1 // Copyright 2011 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "drivers/list_tests.hpp" 30 31 extern "C" { 32 #include <sys/stat.h> 33 34 #include <unistd.h> 35 } 36 37 #include <map> 38 #include <set> 39 #include <string> 40 41 #include <atf-c++.hpp> 42 43 #include "cli/cmd_list.hpp" 44 #include "cli/common.ipp" 45 #include "engine/atf.hpp" 46 #include "engine/config.hpp" 47 #include "engine/exceptions.hpp" 48 #include "engine/filters.hpp" 49 #include "engine/scheduler.hpp" 50 #include "model/metadata.hpp" 51 #include "model/test_case.hpp" 52 #include "model/test_program.hpp" 53 #include "utils/config/tree.ipp" 54 #include "utils/env.hpp" 55 #include "utils/format/macros.hpp" 56 #include "utils/optional.ipp" 57 #include "utils/test_utils.ipp" 58 59 namespace config = utils::config; 60 namespace fs = utils::fs; 61 namespace scheduler = engine::scheduler; 62 63 using utils::none; 64 using utils::optional; 65 66 67 namespace { 68 69 70 /// Gets the path to the helpers for this test program. 71 /// 72 /// \param test_case A pointer to the currently running test case. 73 /// 74 /// \return The path to the helpers binary. 75 static fs::path 76 helpers(const atf::tests::tc* test_case) 77 { 78 return fs::path(test_case->get_config_var("srcdir")) / 79 "list_tests_helpers"; 80 } 81 82 83 /// Hooks to capture the incremental listing of test cases. 84 class capture_hooks : public drivers::list_tests::base_hooks { 85 public: 86 /// Set of the listed test cases in a program:test_case form. 87 std::set< std::string > test_cases; 88 89 /// Set of the listed test cases in a program:test_case form. 90 std::map< std::string, model::metadata > metadatas; 91 92 /// Called when a test case is identified in a test suite. 93 /// 94 /// \param test_program The test program containing the test case. 95 /// \param test_case_name The name of the located test case. 96 virtual void 97 got_test_case(const model::test_program& test_program, 98 const std::string& test_case_name) 99 { 100 const std::string ident = F("%s:%s") % 101 test_program.relative_path() % test_case_name; 102 test_cases.insert(ident); 103 104 metadatas.insert(std::map< std::string, model::metadata >::value_type( 105 ident, test_program.find(test_case_name).get_metadata())); 106 } 107 }; 108 109 110 /// Creates a mock test suite. 111 /// 112 /// \param tc Pointer to the caller test case; needed to obtain the srcdir 113 /// variable of the caller. 114 /// \param source_root Basename of the directory that will contain the 115 /// Kyuafiles. 116 /// \param build_root Basename of the directory that will contain the test 117 /// programs. May or may not be the same as source_root. 118 static void 119 create_helpers(const atf::tests::tc* tc, const fs::path& source_root, 120 const fs::path& build_root) 121 { 122 ATF_REQUIRE(::mkdir(source_root.c_str(), 0755) != -1); 123 ATF_REQUIRE(::mkdir((source_root / "dir").c_str(), 0755) != -1); 124 if (source_root != build_root) { 125 ATF_REQUIRE(::mkdir(build_root.c_str(), 0755) != -1); 126 ATF_REQUIRE(::mkdir((build_root / "dir").c_str(), 0755) != -1); 127 } 128 ATF_REQUIRE(::symlink(helpers(tc).c_str(), 129 (build_root / "dir/program").c_str()) != -1); 130 131 atf::utils::create_file( 132 (source_root / "Kyuafile").str(), 133 "syntax(2)\n" 134 "include('dir/Kyuafile')\n"); 135 136 atf::utils::create_file( 137 (source_root / "dir/Kyuafile").str(), 138 "syntax(2)\n" 139 "atf_test_program{name='program', test_suite='suite-name'}\n"); 140 } 141 142 143 /// Runs the mock test suite. 144 /// 145 /// \param source_root Path to the directory that contains the Kyuafiles. 146 /// \param build_root If not none, path to the directory that contains the test 147 /// programs. 148 /// \param hooks The hooks to use during the listing. 149 /// \param filter_program If not null, the filter on the test program name. 150 /// \param filter_test_case If not null, the filter on the test case name. 151 /// \param the_variable If not null, the value to pass to the test program as 152 /// its "the-variable" configuration property. 153 /// 154 /// \return The result data of the driver. 155 static drivers::list_tests::result 156 run_helpers(const fs::path& source_root, 157 const optional< fs::path > build_root, 158 drivers::list_tests::base_hooks& hooks, 159 const char* filter_program = NULL, 160 const char* filter_test_case = NULL, 161 const char* the_variable = NULL) 162 { 163 std::set< engine::test_filter > filters; 164 if (filter_program != NULL && filter_test_case != NULL) 165 filters.insert(engine::test_filter(fs::path(filter_program), 166 filter_test_case)); 167 168 config::tree user_config = engine::empty_config(); 169 if (the_variable != NULL) { 170 user_config.set_string("test_suites.suite-name.the-variable", 171 the_variable); 172 } 173 174 return drivers::list_tests::drive(source_root / "Kyuafile", build_root, 175 filters, user_config, hooks); 176 } 177 178 179 } // anonymous namespace 180 181 182 ATF_TEST_CASE_WITHOUT_HEAD(one_test_case); 183 ATF_TEST_CASE_BODY(one_test_case) 184 { 185 utils::setenv("TESTS", "some_properties"); 186 capture_hooks hooks; 187 create_helpers(this, fs::path("root"), fs::path("root")); 188 run_helpers(fs::path("root"), none, hooks); 189 190 std::set< std::string > exp_test_cases; 191 exp_test_cases.insert("dir/program:some_properties"); 192 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 193 } 194 195 196 ATF_TEST_CASE_WITHOUT_HEAD(many_test_cases); 197 ATF_TEST_CASE_BODY(many_test_cases) 198 { 199 utils::setenv("TESTS", "no_properties some_properties"); 200 capture_hooks hooks; 201 create_helpers(this, fs::path("root"), fs::path("root")); 202 run_helpers(fs::path("root"), none, hooks); 203 204 std::set< std::string > exp_test_cases; 205 exp_test_cases.insert("dir/program:no_properties"); 206 exp_test_cases.insert("dir/program:some_properties"); 207 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 208 } 209 210 211 ATF_TEST_CASE_WITHOUT_HEAD(filter_match); 212 ATF_TEST_CASE_BODY(filter_match) 213 { 214 utils::setenv("TESTS", "no_properties some_properties"); 215 capture_hooks hooks; 216 create_helpers(this, fs::path("root"), fs::path("root")); 217 run_helpers(fs::path("root"), none, hooks, "dir/program", 218 "some_properties"); 219 220 std::set< std::string > exp_test_cases; 221 exp_test_cases.insert("dir/program:some_properties"); 222 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 223 } 224 225 226 ATF_TEST_CASE_WITHOUT_HEAD(build_root); 227 ATF_TEST_CASE_BODY(build_root) 228 { 229 utils::setenv("TESTS", "no_properties some_properties"); 230 capture_hooks hooks; 231 create_helpers(this, fs::path("source"), fs::path("build")); 232 run_helpers(fs::path("source"), utils::make_optional(fs::path("build")), 233 hooks); 234 235 std::set< std::string > exp_test_cases; 236 exp_test_cases.insert("dir/program:no_properties"); 237 exp_test_cases.insert("dir/program:some_properties"); 238 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 239 } 240 241 242 ATF_TEST_CASE_WITHOUT_HEAD(config_in_head); 243 ATF_TEST_CASE_BODY(config_in_head) 244 { 245 utils::setenv("TESTS", "config_in_head"); 246 capture_hooks hooks; 247 create_helpers(this, fs::path("source"), fs::path("build")); 248 run_helpers(fs::path("source"), utils::make_optional(fs::path("build")), 249 hooks, NULL, NULL, "magic value"); 250 251 std::set< std::string > exp_test_cases; 252 exp_test_cases.insert("dir/program:config_in_head"); 253 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 254 255 const model::metadata& metadata = hooks.metadatas.find( 256 "dir/program:config_in_head")->second; 257 ATF_REQUIRE_EQ("the-variable is magic value", metadata.description()); 258 } 259 260 261 ATF_TEST_CASE_WITHOUT_HEAD(crash); 262 ATF_TEST_CASE_BODY(crash) 263 { 264 utils::setenv("TESTS", "crash_list some_properties"); 265 capture_hooks hooks; 266 create_helpers(this, fs::path("root"), fs::path("root")); 267 run_helpers(fs::path("root"), none, hooks, "dir/program"); 268 269 std::set< std::string > exp_test_cases; 270 exp_test_cases.insert("dir/program:__test_cases_list__"); 271 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 272 } 273 274 275 ATF_INIT_TEST_CASES(tcs) 276 { 277 scheduler::register_interface( 278 "atf", std::shared_ptr< scheduler::interface >( 279 new engine::atf_interface())); 280 281 ATF_ADD_TEST_CASE(tcs, one_test_case); 282 ATF_ADD_TEST_CASE(tcs, many_test_cases); 283 ATF_ADD_TEST_CASE(tcs, filter_match); 284 ATF_ADD_TEST_CASE(tcs, build_root); 285 ATF_ADD_TEST_CASE(tcs, config_in_head); 286 ATF_ADD_TEST_CASE(tcs, crash); 287 } 288