1 // Copyright 2024 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/googletest.hpp" 30 31 extern "C" { 32 #include <sys/stat.h> 33 34 #include <signal.h> 35 } 36 37 #include <atf-c++.hpp> 38 39 #include "engine/config.hpp" 40 #include "engine/scheduler.hpp" 41 #include "model/metadata.hpp" 42 #include "model/test_case.hpp" 43 #include "model/test_program_fwd.hpp" 44 #include "model/test_result.hpp" 45 #include "utils/config/tree.ipp" 46 #include "utils/datetime.hpp" 47 #include "utils/env.hpp" 48 #include "utils/format/containers.ipp" 49 #include "utils/format/macros.hpp" 50 #include "utils/fs/operations.hpp" 51 #include "utils/fs/path.hpp" 52 #include "utils/optional.ipp" 53 #include "utils/stacktrace.hpp" 54 #include "utils/test_utils.ipp" 55 56 namespace config = utils::config; 57 namespace datetime = utils::datetime; 58 namespace fs = utils::fs; 59 namespace scheduler = engine::scheduler; 60 61 using utils::none; 62 63 64 namespace { 65 66 67 /// Prefix for all testcases. 68 const std::string test_suite = "Suite."; 69 70 71 /// Runs one googletest test program and checks its result. 72 /// 73 /// \param program_name Basename of the test program to run. 74 /// \param root Path to the base of the test suite. 75 /// \param user_config User-provided configuration. 76 /// 77 /// \return The list of loaded test cases. 78 static model::test_cases_map 79 list(const char* program_name, 80 const fs::path& root, 81 config::tree user_config = engine::empty_config()) 82 { 83 scheduler::scheduler_handle handle = scheduler::setup(); 84 85 const scheduler::lazy_test_program program( 86 "googletest", fs::path(program_name), root, "the-suite", 87 model::metadata_builder().build(), user_config, handle); 88 89 const model::test_cases_map test_cases = handle.list_tests( 90 &program, user_config); 91 92 handle.cleanup(); 93 94 return test_cases; 95 } 96 97 98 /// Runs a bogus test program and checks the error result. 99 /// 100 /// \param exp_error Expected error string to find. 101 /// \param program_name Basename of the test program to run. 102 /// \param root Path to the base of the test suite. 103 static void 104 check_list_fail(const char* exp_error, 105 const char* program_name, 106 const fs::path& root) 107 { 108 const model::test_cases_map test_cases = list( 109 program_name, root); 110 111 ATF_REQUIRE_EQ(1, test_cases.size()); 112 const model::test_case& test_case = test_cases.begin()->second; 113 ATF_REQUIRE_EQ("__test_cases_list__", test_case.name()); 114 ATF_REQUIRE(test_case.fake_result()); 115 ATF_REQUIRE_MATCH(exp_error, 116 test_case.fake_result().get().reason()); 117 } 118 119 120 /// Runs one googletest test program and checks its result. 121 /// 122 /// \param tc Pointer to the calling test case, to obtain srcdir. 123 /// \param test_case_name Name of the "test case" to select from the helper 124 /// program. 125 /// \param exp_result The expected result. 126 /// \param metadata The test case metadata. 127 /// \param user_config User-provided configuration. 128 static void 129 run_one(const atf::tests::tc* tc, const char* test_case_name, 130 const model::test_result& exp_result, 131 const model::metadata& metadata = model::metadata_builder().build(), 132 config::tree user_config = engine::empty_config()) 133 { 134 scheduler::scheduler_handle handle = scheduler::setup(); 135 136 const std::string test_name = test_suite + std::string(test_case_name); 137 138 const model::test_program_ptr program(new scheduler::lazy_test_program( 139 "googletest", fs::path("googletest_helpers"), 140 fs::path(tc->get_config_var("srcdir")), 141 "the-suite", metadata, user_config, handle)); 142 143 (void)handle.spawn_test(program, test_name.c_str(), user_config); 144 145 scheduler::result_handle_ptr result_handle = handle.wait_any(); 146 const scheduler::test_result_handle* test_result_handle = 147 dynamic_cast< const scheduler::test_result_handle* >( 148 result_handle.get()); 149 atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: "); 150 atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: "); 151 ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); 152 result_handle->cleanup(); 153 result_handle.reset(); 154 155 handle.cleanup(); 156 } 157 158 159 } // anonymous namespace 160 161 162 ATF_TEST_CASE_WITHOUT_HEAD(list__ok); 163 ATF_TEST_CASE_BODY(list__ok) 164 { 165 const model::test_cases_map test_cases = list( 166 "googletest_helpers", fs::path(get_config_var("srcdir"))); 167 168 const model::test_cases_map exp_test_cases = model::test_cases_map_builder() 169 .add(test_suite + "check_configuration_variables") 170 .add(test_suite + "crash") 171 .add(test_suite + "fail") 172 .add(test_suite + "pass") 173 .add(test_suite + "pass_but_exit_failure") 174 .add(test_suite + "timeout", model::metadata_builder() 175 .build()) 176 .build(); 177 ATF_REQUIRE_EQ(exp_test_cases, test_cases); 178 } 179 180 181 ATF_TEST_CASE_WITHOUT_HEAD(list__current_directory); 182 ATF_TEST_CASE_BODY(list__current_directory) 183 { 184 const fs::path helpers = 185 fs::path(get_config_var("srcdir")) / "googletest_helpers"; 186 ATF_REQUIRE(::symlink(helpers.c_str(), "googletest_helpers") != -1); 187 const model::test_cases_map test_cases = list( 188 "googletest_helpers", fs::path(".")); 189 190 const model::test_cases_map exp_test_cases = model::test_cases_map_builder() 191 .add(test_suite + "check_configuration_variables") 192 .add(test_suite + "crash") 193 .add(test_suite + "fail") 194 .add(test_suite + "pass") 195 .add(test_suite + "pass_but_exit_failure") 196 .add(test_suite + "timeout", model::metadata_builder() 197 .build()) 198 .build(); 199 ATF_REQUIRE_EQ(exp_test_cases, test_cases); 200 } 201 202 203 ATF_TEST_CASE_WITHOUT_HEAD(list__relative_path); 204 ATF_TEST_CASE_BODY(list__relative_path) 205 { 206 const fs::path helpers = 207 fs::path(get_config_var("srcdir")) / "googletest_helpers"; 208 ATF_REQUIRE(::mkdir("dir1", 0755) != -1); 209 ATF_REQUIRE(::mkdir("dir1/dir2", 0755) != -1); 210 ATF_REQUIRE_EQ(::symlink(helpers.c_str(), 211 "dir1/dir2/googletest_helpers"), 0); 212 const model::test_cases_map test_cases = list( 213 "dir2/googletest_helpers", fs::path("dir1")); 214 215 const model::test_cases_map exp_test_cases = model::test_cases_map_builder() 216 .add(test_suite + "check_configuration_variables") 217 .add(test_suite + "crash") 218 .add(test_suite + "fail") 219 .add(test_suite + "pass") 220 .add(test_suite + "pass_but_exit_failure") 221 .add(test_suite + "timeout", model::metadata_builder() 222 .build()) 223 .build(); 224 ATF_REQUIRE_EQ(exp_test_cases, test_cases); 225 } 226 227 228 ATF_TEST_CASE_WITHOUT_HEAD(list__missing_test_program); 229 ATF_TEST_CASE_BODY(list__missing_test_program) 230 { 231 check_list_fail("Cannot find test program", "non-existent", 232 fs::current_path()); 233 } 234 235 236 ATF_TEST_CASE_WITHOUT_HEAD(list__not_a_test_program); 237 ATF_TEST_CASE_BODY(list__not_a_test_program) 238 { 239 atf::utils::create_file("not-valid", "garbage\n"); 240 ATF_REQUIRE_EQ(::chmod("not-valid", 0755), 0); 241 check_list_fail("Invalid test program format", "not-valid", 242 fs::current_path()); 243 } 244 245 246 ATF_TEST_CASE_WITHOUT_HEAD(list__no_permissions); 247 ATF_TEST_CASE_BODY(list__no_permissions) 248 { 249 atf::utils::create_file("not-executable", "garbage\n"); 250 check_list_fail("Permission denied to run test program", 251 "not-executable", fs::current_path()); 252 } 253 254 255 ATF_TEST_CASE_WITHOUT_HEAD(list__abort); 256 ATF_TEST_CASE_BODY(list__abort) 257 { 258 atf::utils::create_file("kills_self", "#!/bin/sh\nkill -2 $$\n"); 259 ATF_REQUIRE_EQ(::chmod("kills_self", 0755), 0); 260 check_list_fail("Test program received signal", "kills_self", 261 fs::current_path()); 262 } 263 264 265 ATF_TEST_CASE_WITHOUT_HEAD(list__empty); 266 ATF_TEST_CASE_BODY(list__empty) 267 { 268 atf::utils::create_file("empty_list", "#!/bin/sh\n"); 269 ATF_REQUIRE_EQ(::chmod("empty_list", 0755), 0); 270 check_list_fail("No test cases", "empty_list", fs::current_path()); 271 } 272 273 274 ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__passes); 275 ATF_TEST_CASE_BODY(test__body_only__passes) 276 { 277 const model::test_result exp_result(model::test_result_passed); 278 run_one(this, "pass", exp_result); 279 } 280 281 282 ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__fails); 283 ATF_TEST_CASE_BODY(test__body_only__fails) 284 { 285 const std::string failure_message = ( 286 "gtest_macros_demo.cc:4: Failure\n" 287 "Failed\n" 288 "with a reason\n" 289 ); 290 const model::test_result exp_result(model::test_result_failed, 291 failure_message); 292 run_one(this, "fail", exp_result); 293 } 294 295 296 ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__crashes); 297 ATF_TEST_CASE_BODY(test__body_only__crashes) 298 { 299 utils::prepare_coredump_test(this); 300 301 const model::test_result exp_result( 302 model::test_result_broken, 303 F("Error: Premature exit. Test case received signal %s (core dumped)") % 304 SIGABRT); 305 run_one(this, "crash", exp_result); 306 } 307 308 309 ATF_TEST_CASE(test__body_only__times_out); 310 ATF_TEST_CASE_HEAD(test__body_only__times_out) 311 { 312 set_md_var("timeout", "60"); 313 } 314 ATF_TEST_CASE_BODY(test__body_only__times_out) 315 { 316 config::tree user_config = engine::empty_config(); 317 utils::setenv("CONTROL_DIR", fs::current_path().str()); 318 const model::metadata metadata = model::metadata_builder() 319 .set_timeout(datetime::delta(1, 0)).build(); 320 const model::test_result exp_result( 321 model::test_result_broken, "Test case body timed out"); 322 run_one(this, "timeout", exp_result, metadata, user_config); 323 324 ATF_REQUIRE(!atf::utils::file_exists("cookie")); 325 } 326 327 328 ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__configuration_variables); 329 ATF_TEST_CASE_BODY(test__body_only__configuration_variables) 330 { 331 config::tree user_config = engine::empty_config(); 332 user_config.set_string("test_suites.the-suite.first", "some value"); 333 user_config.set_string("test_suites.the-suite.second", "some other value"); 334 335 const model::test_result exp_result(model::test_result_passed); 336 run_one(this, "check_configuration_variables", exp_result, 337 model::metadata_builder().build(), user_config); 338 } 339 340 341 ATF_INIT_TEST_CASES(tcs) 342 { 343 scheduler::register_interface( 344 "googletest", std::shared_ptr< scheduler::interface >( 345 new engine::googletest_interface())); 346 347 ATF_ADD_TEST_CASE(tcs, list__ok); 348 ATF_ADD_TEST_CASE(tcs, list__current_directory); 349 ATF_ADD_TEST_CASE(tcs, list__relative_path); 350 ATF_ADD_TEST_CASE(tcs, list__missing_test_program); 351 ATF_ADD_TEST_CASE(tcs, list__not_a_test_program); 352 ATF_ADD_TEST_CASE(tcs, list__no_permissions); 353 ATF_ADD_TEST_CASE(tcs, list__abort); 354 ATF_ADD_TEST_CASE(tcs, list__empty); 355 356 ATF_ADD_TEST_CASE(tcs, test__body_only__passes); 357 ATF_ADD_TEST_CASE(tcs, test__body_only__fails); 358 ATF_ADD_TEST_CASE(tcs, test__body_only__crashes); 359 ATF_ADD_TEST_CASE(tcs, test__body_only__times_out); 360 ATF_ADD_TEST_CASE(tcs, test__body_only__configuration_variables); 361 } 362