1 // Copyright 2014 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/plain.hpp" 30 31 extern "C" { 32 #include <signal.h> 33 } 34 35 #include <atf-c++.hpp> 36 37 #include "engine/config.hpp" 38 #include "engine/scheduler.hpp" 39 #include "model/metadata.hpp" 40 #include "model/test_case.hpp" 41 #include "model/test_program.hpp" 42 #include "model/test_result.hpp" 43 #include "utils/config/tree.ipp" 44 #include "utils/datetime.hpp" 45 #include "utils/env.hpp" 46 #include "utils/format/containers.ipp" 47 #include "utils/format/macros.hpp" 48 #include "utils/fs/operations.hpp" 49 #include "utils/fs/path.hpp" 50 #include "utils/optional.ipp" 51 52 namespace config = utils::config; 53 namespace datetime = utils::datetime; 54 namespace fs = utils::fs; 55 namespace scheduler = engine::scheduler; 56 57 using utils::none; 58 59 60 namespace { 61 62 63 /// Copies the plain helper to the work directory, selecting a specific helper. 64 /// 65 /// \param tc Pointer to the calling test case, to obtain srcdir. 66 /// \param name Name of the new binary to create. Must match the name of a 67 /// valid helper, as the binary name is used to select it. 68 static void 69 copy_plain_helper(const atf::tests::tc* tc, const char* name) 70 { 71 const fs::path srcdir(tc->get_config_var("srcdir")); 72 atf::utils::copy_file((srcdir / "plain_helpers").str(), name); 73 } 74 75 76 /// Runs one plain test program and checks its result. 77 /// 78 /// \param tc Pointer to the calling test case, to obtain srcdir. 79 /// \param test_case_name Name of the "test case" to select from the helper 80 /// program. 81 /// \param exp_result The expected result. 82 /// \param metadata The test case metadata. 83 /// \param user_config User-provided configuration variables. 84 static void 85 run_one(const atf::tests::tc* tc, const char* test_case_name, 86 const model::test_result& exp_result, 87 const model::metadata& metadata = model::metadata_builder().build(), 88 const config::tree& user_config = engine::empty_config()) 89 { 90 copy_plain_helper(tc, test_case_name); 91 const model::test_program_ptr program = model::test_program_builder( 92 "plain", fs::path(test_case_name), fs::current_path(), "the-suite") 93 .add_test_case("main", metadata).build_ptr(); 94 95 scheduler::scheduler_handle handle = scheduler::setup(); 96 (void)handle.spawn_test(program, "main", user_config); 97 98 scheduler::result_handle_ptr result_handle = handle.wait_any(); 99 const scheduler::test_result_handle* test_result_handle = 100 dynamic_cast< const scheduler::test_result_handle* >( 101 result_handle.get()); 102 atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: "); 103 atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: "); 104 ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); 105 result_handle->cleanup(); 106 result_handle.reset(); 107 108 handle.cleanup(); 109 } 110 111 112 } // anonymous namespace 113 114 115 ATF_TEST_CASE_WITHOUT_HEAD(list); 116 ATF_TEST_CASE_BODY(list) 117 { 118 const model::test_program program = model::test_program_builder( 119 "plain", fs::path("non-existent"), fs::path("."), "unused-suite") 120 .build(); 121 122 scheduler::scheduler_handle handle = scheduler::setup(); 123 const model::test_cases_map test_cases = handle.list_tests( 124 &program, engine::empty_config()); 125 handle.cleanup(); 126 127 const model::test_cases_map exp_test_cases = model::test_cases_map_builder() 128 .add("main").build(); 129 ATF_REQUIRE_EQ(exp_test_cases, test_cases); 130 } 131 132 133 ATF_TEST_CASE_WITHOUT_HEAD(test__exit_success_is_pass); 134 ATF_TEST_CASE_BODY(test__exit_success_is_pass) 135 { 136 const model::test_result exp_result(model::test_result_passed); 137 run_one(this, "pass", exp_result); 138 } 139 140 141 ATF_TEST_CASE_WITHOUT_HEAD(test__exit_non_zero_is_fail); 142 ATF_TEST_CASE_BODY(test__exit_non_zero_is_fail) 143 { 144 const model::test_result exp_result( 145 model::test_result_failed, 146 "Returned non-success exit status 8"); 147 run_one(this, "fail", exp_result); 148 } 149 150 151 ATF_TEST_CASE_WITHOUT_HEAD(test__signal_is_broken); 152 ATF_TEST_CASE_BODY(test__signal_is_broken) 153 { 154 const model::test_result exp_result(model::test_result_broken, 155 F("Received signal %s") % SIGABRT); 156 run_one(this, "crash", exp_result); 157 } 158 159 160 ATF_TEST_CASE(test__timeout_is_broken); 161 ATF_TEST_CASE_HEAD(test__timeout_is_broken) 162 { 163 set_md_var("timeout", "60"); 164 } 165 ATF_TEST_CASE_BODY(test__timeout_is_broken) 166 { 167 utils::setenv("CONTROL_DIR", fs::current_path().str()); 168 169 const model::metadata metadata = model::metadata_builder() 170 .set_timeout(datetime::delta(1, 0)).build(); 171 const model::test_result exp_result(model::test_result_broken, 172 "Test case timed out"); 173 run_one(this, "timeout", exp_result, metadata); 174 175 ATF_REQUIRE(!atf::utils::file_exists("cookie")); 176 } 177 178 179 ATF_TEST_CASE_WITHOUT_HEAD(test__configuration_variables); 180 ATF_TEST_CASE_BODY(test__configuration_variables) 181 { 182 config::tree user_config = engine::empty_config(); 183 user_config.set_string("test_suites.a-suite.first", "unused"); 184 user_config.set_string("test_suites.the-suite.first", "some value"); 185 user_config.set_string("test_suites.the-suite.second", "some other value"); 186 user_config.set_string("test_suites.other-suite.first", "unused"); 187 188 const model::test_result exp_result(model::test_result_passed); 189 run_one(this, "check_configuration_variables", exp_result, 190 model::metadata_builder().build(), user_config); 191 } 192 193 194 ATF_INIT_TEST_CASES(tcs) 195 { 196 scheduler::register_interface( 197 "plain", std::shared_ptr< scheduler::interface >( 198 new engine::plain_interface())); 199 200 ATF_ADD_TEST_CASE(tcs, list); 201 202 ATF_ADD_TEST_CASE(tcs, test__exit_success_is_pass); 203 ATF_ADD_TEST_CASE(tcs, test__exit_non_zero_is_fail); 204 ATF_ADD_TEST_CASE(tcs, test__signal_is_broken); 205 ATF_ADD_TEST_CASE(tcs, test__timeout_is_broken); 206 ATF_ADD_TEST_CASE(tcs, test__configuration_variables); 207 } 208