1 // Copyright 2014 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/atf.hpp" 30 31 extern "C" { 32 #include <sys/stat.h> 33 34 #include <signal.h> 35 } 36 37 #include <atf-c++.hpp> 38 39 #include "engine/config.hpp" 40 #include "engine/scheduler.hpp" 41 #include "model/metadata.hpp" 42 #include "model/test_case.hpp" 43 #include "model/test_program_fwd.hpp" 44 #include "model/test_result.hpp" 45 #include "utils/config/tree.ipp" 46 #include "utils/datetime.hpp" 47 #include "utils/env.hpp" 48 #include "utils/format/containers.ipp" 49 #include "utils/format/macros.hpp" 50 #include "utils/fs/operations.hpp" 51 #include "utils/fs/path.hpp" 52 #include "utils/optional.ipp" 53 #include "utils/stacktrace.hpp" 54 #include "utils/test_utils.ipp" 55 56 namespace config = utils::config; 57 namespace datetime = utils::datetime; 58 namespace fs = utils::fs; 59 namespace scheduler = engine::scheduler; 60 61 using utils::none; 62 63 64 namespace { 65 66 67 /// Lists the test cases associated with an ATF test program. 68 /// 69 /// \param program_name Basename of the test program to run. 70 /// \param root Path to the base of the test suite. 71 /// \param names_filter Whitespace-separated list of test cases that the helper 72 /// test program is allowed to expose. 73 /// \param user_config User-provided configuration. 74 /// 75 /// \return The list of loaded test cases. 76 static model::test_cases_map 77 list_one(const char* program_name, 78 const fs::path& root, 79 const char* names_filter = NULL, 80 config::tree user_config = engine::empty_config()) 81 { 82 scheduler::scheduler_handle handle = scheduler::setup(); 83 84 const scheduler::lazy_test_program program( 85 "atf", fs::path(program_name), root, "the-suite", 86 model::metadata_builder().build(), user_config, handle); 87 88 if (names_filter != NULL) 89 utils::setenv("TEST_CASES", names_filter); 90 const model::test_cases_map test_cases = handle.list_tests( 91 &program, user_config); 92 93 handle.cleanup(); 94 95 return test_cases; 96 } 97 98 99 /// Runs a bogus test program and checks the error result. 100 /// 101 /// \param exp_error Expected error string to find. 102 /// \param program_name Basename of the test program to run. 103 /// \param root Path to the base of the test suite. 104 /// \param names_filter Whitespace-separated list of test cases that the helper 105 /// test program is allowed to expose. 106 static void 107 check_list_one_fail(const char* exp_error, 108 const char* program_name, 109 const fs::path& root, 110 const char* names_filter = NULL) 111 { 112 const model::test_cases_map test_cases = list_one( 113 program_name, root, names_filter); 114 115 ATF_REQUIRE_EQ(1, test_cases.size()); 116 const model::test_case& test_case = test_cases.begin()->second; 117 ATF_REQUIRE_EQ("__test_cases_list__", test_case.name()); 118 ATF_REQUIRE(test_case.fake_result()); 119 ATF_REQUIRE_MATCH(exp_error, 120 test_case.fake_result().get().reason()); 121 } 122 123 124 /// Runs one ATF test program and checks its result. 125 /// 126 /// \param tc Pointer to the calling test case, to obtain srcdir. 127 /// \param test_case_name Name of the "test case" to select from the helper 128 /// program. 129 /// \param exp_result The expected result. 130 /// \param user_config User-provided configuration. 131 /// \param check_empty_output If true, verify that the output of the test is 132 /// silent. This is just a hack to implement one of the test cases; we'd 133 /// easily have a nicer abstraction here... 134 static void 135 run_one(const atf::tests::tc* tc, const char* test_case_name, 136 const model::test_result& exp_result, 137 config::tree user_config = engine::empty_config(), 138 const bool check_empty_output = false) 139 { 140 scheduler::scheduler_handle handle = scheduler::setup(); 141 142 const model::test_program_ptr program(new scheduler::lazy_test_program( 143 "atf", fs::path("atf_helpers"), fs::path(tc->get_config_var("srcdir")), 144 "the-suite", model::metadata_builder().build(), 145 user_config, handle)); 146 147 (void)handle.spawn_test(program, test_case_name, user_config); 148 149 scheduler::result_handle_ptr result_handle = handle.wait_any(); 150 const scheduler::test_result_handle* test_result_handle = 151 dynamic_cast< const scheduler::test_result_handle* >( 152 result_handle.get()); 153 atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: "); 154 atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: "); 155 ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); 156 if (check_empty_output) { 157 ATF_REQUIRE(atf::utils::compare_file(result_handle->stdout_file().str(), 158 "")); 159 ATF_REQUIRE(atf::utils::compare_file(result_handle->stderr_file().str(), 160 "")); 161 } 162 result_handle->cleanup(); 163 result_handle.reset(); 164 165 handle.cleanup(); 166 } 167 168 169 } // anonymous namespace 170 171 172 ATF_TEST_CASE_WITHOUT_HEAD(list__ok); 173 ATF_TEST_CASE_BODY(list__ok) 174 { 175 const model::test_cases_map test_cases = list_one( 176 "atf_helpers", fs::path(get_config_var("srcdir")), "pass crash"); 177 178 const model::test_cases_map exp_test_cases = model::test_cases_map_builder() 179 .add("crash") 180 .add("pass", model::metadata_builder() 181 .set_description("Always-passing test case") 182 .build()) 183 .build(); 184 ATF_REQUIRE_EQ(exp_test_cases, test_cases); 185 } 186 187 188 ATF_TEST_CASE_WITHOUT_HEAD(list__configuration_variables); 189 ATF_TEST_CASE_BODY(list__configuration_variables) 190 { 191 config::tree user_config = engine::empty_config(); 192 user_config.set_string("test_suites.the-suite.var1", "value1"); 193 user_config.set_string("test_suites.the-suite.var2", "value2"); 194 195 const model::test_cases_map test_cases = list_one( 196 "atf_helpers", fs::path(get_config_var("srcdir")), "check_list_config", 197 user_config); 198 199 const model::test_cases_map exp_test_cases = model::test_cases_map_builder() 200 .add("check_list_config", model::metadata_builder() 201 .set_description("Found: var1=value1 var2=value2") 202 .build()) 203 .build(); 204 ATF_REQUIRE_EQ(exp_test_cases, test_cases); 205 } 206 207 208 ATF_TEST_CASE_WITHOUT_HEAD(list__current_directory); 209 ATF_TEST_CASE_BODY(list__current_directory) 210 { 211 const fs::path helpers = fs::path(get_config_var("srcdir")) / "atf_helpers"; 212 ATF_REQUIRE(::symlink(helpers.c_str(), "atf_helpers") != -1); 213 const model::test_cases_map test_cases = list_one( 214 "atf_helpers", fs::path("."), "pass"); 215 216 const model::test_cases_map exp_test_cases = model::test_cases_map_builder() 217 .add("pass", model::metadata_builder() 218 .set_description("Always-passing test case") 219 .build()) 220 .build(); 221 ATF_REQUIRE_EQ(exp_test_cases, test_cases); 222 } 223 224 225 ATF_TEST_CASE_WITHOUT_HEAD(list__relative_path); 226 ATF_TEST_CASE_BODY(list__relative_path) 227 { 228 const fs::path helpers = fs::path(get_config_var("srcdir")) / "atf_helpers"; 229 ATF_REQUIRE(::mkdir("dir1", 0755) != -1); 230 ATF_REQUIRE(::mkdir("dir1/dir2", 0755) != -1); 231 ATF_REQUIRE(::symlink(helpers.c_str(), "dir1/dir2/atf_helpers") != -1); 232 const model::test_cases_map test_cases = list_one( 233 "dir2/atf_helpers", fs::path("dir1"), "pass"); 234 235 const model::test_cases_map exp_test_cases = model::test_cases_map_builder() 236 .add("pass", model::metadata_builder() 237 .set_description("Always-passing test case") 238 .build()) 239 .build(); 240 ATF_REQUIRE_EQ(exp_test_cases, test_cases); 241 } 242 243 244 ATF_TEST_CASE_WITHOUT_HEAD(list__missing_test_program); 245 ATF_TEST_CASE_BODY(list__missing_test_program) 246 { 247 check_list_one_fail("Cannot find test program", "non-existent", 248 fs::current_path()); 249 } 250 251 252 ATF_TEST_CASE_WITHOUT_HEAD(list__not_a_test_program); 253 ATF_TEST_CASE_BODY(list__not_a_test_program) 254 { 255 atf::utils::create_file("not-valid", "garbage\n"); 256 ATF_REQUIRE(::chmod("not-valid", 0755) != -1); 257 check_list_one_fail("Invalid test program format", "not-valid", 258 fs::current_path()); 259 } 260 261 262 ATF_TEST_CASE_WITHOUT_HEAD(list__no_permissions); 263 ATF_TEST_CASE_BODY(list__no_permissions) 264 { 265 atf::utils::create_file("not-executable", "garbage\n"); 266 check_list_one_fail("Permission denied to run test program", 267 "not-executable", fs::current_path()); 268 } 269 270 271 ATF_TEST_CASE_WITHOUT_HEAD(list__abort); 272 ATF_TEST_CASE_BODY(list__abort) 273 { 274 check_list_one_fail("Test program received signal", "atf_helpers", 275 fs::path(get_config_var("srcdir")), 276 "crash_head"); 277 } 278 279 280 ATF_TEST_CASE_WITHOUT_HEAD(list__empty); 281 ATF_TEST_CASE_BODY(list__empty) 282 { 283 check_list_one_fail("No test cases", "atf_helpers", 284 fs::path(get_config_var("srcdir")), 285 ""); 286 } 287 288 289 ATF_TEST_CASE_WITHOUT_HEAD(list__stderr_not_quiet); 290 ATF_TEST_CASE_BODY(list__stderr_not_quiet) 291 { 292 check_list_one_fail("Test case list wrote to stderr", "atf_helpers", 293 fs::path(get_config_var("srcdir")), 294 "output_in_list"); 295 } 296 297 298 ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__passes); 299 ATF_TEST_CASE_BODY(test__body_only__passes) 300 { 301 const model::test_result exp_result(model::test_result_passed); 302 run_one(this, "pass", exp_result); 303 } 304 305 306 ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__crashes); 307 ATF_TEST_CASE_BODY(test__body_only__crashes) 308 { 309 utils::prepare_coredump_test(this); 310 311 const model::test_result exp_result( 312 model::test_result_broken, 313 F("Premature exit; test case received signal %s (core dumped)") % 314 SIGABRT); 315 run_one(this, "crash", exp_result); 316 } 317 318 319 ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__times_out); 320 ATF_TEST_CASE_BODY(test__body_only__times_out) 321 { 322 config::tree user_config = engine::empty_config(); 323 user_config.set_string("test_suites.the-suite.control_dir", 324 fs::current_path().str()); 325 user_config.set_string("test_suites.the-suite.timeout", "1"); 326 327 const model::test_result exp_result( 328 model::test_result_broken, "Test case body timed out"); 329 run_one(this, "timeout_body", exp_result, user_config); 330 331 ATF_REQUIRE(!atf::utils::file_exists("cookie")); 332 } 333 334 335 ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__configuration_variables); 336 ATF_TEST_CASE_BODY(test__body_only__configuration_variables) 337 { 338 config::tree user_config = engine::empty_config(); 339 user_config.set_string("test_suites.the-suite.first", "some value"); 340 user_config.set_string("test_suites.the-suite.second", "some other value"); 341 342 const model::test_result exp_result(model::test_result_passed); 343 run_one(this, "check_configuration_variables", exp_result, user_config); 344 } 345 346 347 ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__no_atf_run_warning); 348 ATF_TEST_CASE_BODY(test__body_only__no_atf_run_warning) 349 { 350 const model::test_result exp_result(model::test_result_passed); 351 run_one(this, "pass", exp_result, engine::empty_config(), true); 352 } 353 354 355 ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__body_times_out); 356 ATF_TEST_CASE_BODY(test__body_and_cleanup__body_times_out) 357 { 358 config::tree user_config = engine::empty_config(); 359 user_config.set_string("test_suites.the-suite.control_dir", 360 fs::current_path().str()); 361 user_config.set_string("test_suites.the-suite.timeout", "1"); 362 363 const model::test_result exp_result( 364 model::test_result_broken, "Test case body timed out"); 365 run_one(this, "timeout_body", exp_result, user_config); 366 367 ATF_REQUIRE(!atf::utils::file_exists("cookie")); 368 ATF_REQUIRE(atf::utils::file_exists("cookie.cleanup")); 369 } 370 371 372 ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__cleanup_crashes); 373 ATF_TEST_CASE_BODY(test__body_and_cleanup__cleanup_crashes) 374 { 375 const model::test_result exp_result( 376 model::test_result_broken, 377 "Test case cleanup did not terminate successfully"); 378 run_one(this, "crash_cleanup", exp_result); 379 } 380 381 382 ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__cleanup_times_out); 383 ATF_TEST_CASE_BODY(test__body_and_cleanup__cleanup_times_out) 384 { 385 config::tree user_config = engine::empty_config(); 386 user_config.set_string("test_suites.the-suite.control_dir", 387 fs::current_path().str()); 388 389 scheduler::cleanup_timeout = datetime::delta(1, 0); 390 const model::test_result exp_result( 391 model::test_result_broken, "Test case cleanup timed out"); 392 run_one(this, "timeout_cleanup", exp_result, user_config); 393 394 ATF_REQUIRE(!atf::utils::file_exists("cookie")); 395 } 396 397 398 ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__expect_timeout); 399 ATF_TEST_CASE_BODY(test__body_and_cleanup__expect_timeout) 400 { 401 config::tree user_config = engine::empty_config(); 402 user_config.set_string("test_suites.the-suite.control_dir", 403 fs::current_path().str()); 404 user_config.set_string("test_suites.the-suite.timeout", "1"); 405 406 const model::test_result exp_result( 407 model::test_result_expected_failure, "Times out on purpose"); 408 run_one(this, "expect_timeout", exp_result, user_config); 409 410 ATF_REQUIRE(!atf::utils::file_exists("cookie")); 411 ATF_REQUIRE(atf::utils::file_exists("cookie.cleanup")); 412 } 413 414 415 ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__shared_workdir); 416 ATF_TEST_CASE_BODY(test__body_and_cleanup__shared_workdir) 417 { 418 const model::test_result exp_result(model::test_result_passed); 419 run_one(this, "shared_workdir", exp_result); 420 } 421 422 423 ATF_INIT_TEST_CASES(tcs) 424 { 425 scheduler::register_interface( 426 "atf", std::shared_ptr< scheduler::interface >( 427 new engine::atf_interface())); 428 429 ATF_ADD_TEST_CASE(tcs, list__ok); 430 ATF_ADD_TEST_CASE(tcs, list__configuration_variables); 431 ATF_ADD_TEST_CASE(tcs, list__current_directory); 432 ATF_ADD_TEST_CASE(tcs, list__relative_path); 433 ATF_ADD_TEST_CASE(tcs, list__missing_test_program); 434 ATF_ADD_TEST_CASE(tcs, list__not_a_test_program); 435 ATF_ADD_TEST_CASE(tcs, list__no_permissions); 436 ATF_ADD_TEST_CASE(tcs, list__abort); 437 ATF_ADD_TEST_CASE(tcs, list__empty); 438 ATF_ADD_TEST_CASE(tcs, list__stderr_not_quiet); 439 440 ATF_ADD_TEST_CASE(tcs, test__body_only__passes); 441 ATF_ADD_TEST_CASE(tcs, test__body_only__crashes); 442 ATF_ADD_TEST_CASE(tcs, test__body_only__times_out); 443 ATF_ADD_TEST_CASE(tcs, test__body_only__configuration_variables); 444 ATF_ADD_TEST_CASE(tcs, test__body_only__no_atf_run_warning); 445 ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__body_times_out); 446 ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__cleanup_crashes); 447 ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__cleanup_times_out); 448 ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__expect_timeout); 449 ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__shared_workdir); 450 } 451