1 // Copyright 2024 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/googletest_result.hpp" 30 31 extern "C" { 32 #include <signal.h> 33 } 34 35 #include <cstdlib> 36 #include <fstream> 37 #include <sstream> 38 #include <stdexcept> 39 40 #include <atf-c++.hpp> 41 42 #include "engine/exceptions.hpp" 43 #include "model/test_result.hpp" 44 #include "utils/format/macros.hpp" 45 #include "utils/fs/path.hpp" 46 #include "utils/process/status.hpp" 47 48 namespace fs = utils::fs; 49 namespace process = utils::process; 50 51 using utils::none; 52 using utils::optional; 53 54 55 namespace { 56 57 58 /// Performs a test for results::parse() that should succeed. 59 /// 60 /// \param exp_type The expected type of the result. 61 /// \param exp_reason The expected reason describing the result, if any. 62 /// \param text The literal input to parse; can include multiple lines. 63 static void 64 parse_ok_test(const engine::googletest_result::types& exp_type, 65 const char* exp_reason, const char* text) 66 { 67 std::istringstream input(text); 68 const engine::googletest_result actual = 69 engine::googletest_result::parse(input); 70 ATF_REQUIRE_EQ(exp_type, actual.type()); 71 if (exp_reason != NULL) { 72 ATF_REQUIRE(actual.reason()); 73 ATF_REQUIRE_EQ(exp_reason, actual.reason().get()); 74 } else { 75 ATF_REQUIRE(!actual.reason()); 76 } 77 } 78 79 80 /// Wrapper around parse_ok_test to define a test case. 81 /// 82 /// \param name The name of the test case; will be prefixed with 83 /// "googletest_result__parse__". 84 /// \param exp_type The expected type of the result. 85 /// \param exp_reason The expected reason describing the result, if any. 86 /// \param input The literal input to parse. 87 #define PARSE(name, exp_type, exp_reason, input) \ 88 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__parse__ ## name); \ 89 ATF_TEST_CASE_BODY(googletest_result__parse__ ## name) \ 90 { \ 91 parse_ok_test(exp_type, exp_reason, input); \ 92 } 93 94 95 } // anonymous namespace 96 97 98 PARSE(broken, 99 engine::googletest_result::broken, "invalid output", 100 "invalid input"); 101 102 const char disabled_context[] = ( 103 "YOU HAVE 1 DISABLED TEST" 104 ); 105 106 const char disabled_message[] = ( 107 "[==========] Running 0 tests from 0 test cases.\n" 108 "[==========] 0 tests from 0 test cases ran. (0 ms total)\n" 109 "[ PASSED ] 0 tests.\n" 110 "\n" 111 " YOU HAVE 1 DISABLED TEST\n" 112 "\n" 113 "\n" 114 ); 115 116 PARSE(disabled, 117 engine::googletest_result::disabled, disabled_context, 118 disabled_message); 119 120 const char failed_context[] = ( 121 "pass_fail_demo.cc:8: Failure\n" 122 "Expected equality of these values:\n" 123 " false\n" 124 " true\n" 125 ); 126 127 const char failed_message[] = ( 128 "Note: Google Test filter = PassFailTest.Fails\n" 129 "[==========] Running 1 test from 1 test case.\n" 130 "[----------] Global test environment set-up.\n" 131 "[----------] 1 test from PassFailTest\n" 132 "[ RUN ] PassFailTest.Fails\n" 133 "pass_fail_demo.cc:8: Failure\n" 134 "Expected equality of these values:\n" 135 " false\n" 136 " true\n" 137 "[ FAILED ] PassFailTest.Fails (0 ms)\n" 138 "[----------] 1 test from PassFailTest (0 ms total)\n" 139 "\n" 140 "[----------] Global test environment tear-down\n" 141 "[==========] 1 test from 1 test case ran. (0 ms total)\n" 142 "[ PASSED ] 0 tests.\n" 143 "[ FAILED ] 1 test, listed below:\n" 144 "[ FAILED ] PassFailTest.Fails\n" 145 "\n" 146 " 1 FAILED TEST\n" 147 ); 148 149 PARSE(failed, 150 engine::googletest_result::failed, failed_context, 151 failed_message); 152 153 const char skipped_message[] = ( 154 "Note: Google Test filter = SkipTest.DoesSkip\n" 155 "[==========] Running 1 test from 1 test suite.\n" 156 "[----------] Global test environment set-up.\n" 157 "[----------] 1 test from SkipTest\n" 158 "[ RUN ] SkipTest.DoesSkip\n" 159 "[ SKIPPED ] SkipTest.DoesSkip (0 ms)\n" 160 "[----------] 1 test from SkipTest (0 ms total)\n" 161 "\n" 162 "[----------] Global test environment tear-down\n" 163 "[==========] 1 test from 1 test suite ran. (0 ms total)\n" 164 "[ PASSED ] 0 tests.\n" 165 "[ SKIPPED ] 1 test, listed below:\n" 166 "[ SKIPPED ] SkipTest.DoesSkip\n" 167 ); 168 169 PARSE(skipped, 170 engine::googletest_result::skipped, 171 engine::bogus_googletest_skipped_nul_message.c_str(), 172 skipped_message 173 ); 174 175 const char skipped_with_reason_context[] = ( 176 "This is a reason\n" 177 ); 178 179 const char skipped_with_reason_message[] = ( 180 "Note: Google Test filter = SkipTest.SkipWithReason\n" 181 "[==========] Running 1 test from 1 test suite.\n" 182 "[----------] Global test environment set-up.\n" 183 "[----------] 1 test from SkipTest\n" 184 "[ RUN ] SkipTest.SkipWithReason\n" 185 "This is a reason\n" 186 "[ SKIPPED ] SkipTest.SkipWithReason (0 ms)\n" 187 "[----------] 1 test from SkipTest (0 ms total)\n" 188 "\n" 189 "[----------] Global test environment tear-down\n" 190 "[==========] 1 test from 1 test suite ran. (0 ms total)\n" 191 "[ PASSED ] 0 tests.\n" 192 "[ SKIPPED ] 1 test, listed below:\n" 193 "[ SKIPPED ] SkipTest.SkipWithReason\n" 194 ); 195 196 PARSE(skipped_with_reason, 197 engine::googletest_result::skipped, skipped_with_reason_context, 198 skipped_with_reason_message); 199 200 const char successful_message[] = ( 201 "Note: Google Test filter = PassFailTest.Passes\n" 202 "[==========] Running 1 test from 1 test case.\n" 203 "[----------] Global test environment set-up.\n" 204 "[----------] 1 test from PassFailTest\n" 205 "[ RUN ] PassFailTest.Passes\n" 206 "[ OK ] PassFailTest.Passes (0 ms)\n" 207 "[----------] 1 test from PassFailTest (0 ms total)\n" 208 "\n" 209 "[----------] Global test environment tear-down\n" 210 "[==========] 1 test from 1 test case ran. (0 ms total)\n" 211 "[ PASSED ] 1 test.\n" 212 ); 213 214 PARSE(successful, 215 engine::googletest_result::successful, NULL, 216 successful_message); 217 218 const char successful_message2[] = ( 219 "Note: Google Test filter = ValuesTest.ValuesWorks\n" 220 "[==========] Running 1 test from 1 test case.\n" 221 "[----------] Global test environment set-up.\n" 222 "[----------] 1 test from ValuesTest\n" 223 "[ RUN ] ValuesTest.ValuesWorks\n" 224 "[ OK ] ValuesTest.ValuesWorks (0 ms)\n" 225 "[----------] 1 test from ValuesTest (0 ms total)\n" 226 " \n" 227 "[----------] Global test environment tear-down\n" 228 "[==========] 1 test from 1 test case ran. (0 ms total)\n" 229 "[ PASSED ] 1 test.\n" 230 ); 231 232 PARSE(successful2, 233 engine::googletest_result::successful, NULL, 234 successful_message2); 235 236 const char successful_parameterized_message[] = ( 237 "Note: Google Test filter = RangeZeroToFive/ParamDerivedTest/0\n" 238 "[==========] Running 5 tests from 1 test case.\n" 239 "[----------] Global test environment set-up.\n" 240 "[----------] 5 tests from RangeZeroToFive/ParamDerivedTest\n" 241 "[ RUN ] RangeZeroToFive/ParamDerivedTest.SeesSequence/0\n" 242 "[ OK ] RangeZeroToFive/ParamDerivedTest.SeesSequence/0 (0 ms)\n" 243 "[----------] 1 test from RangeZeroToFive/ParamDerivedTest/0 (0 ms total)\n" 244 "\n" 245 "[----------] Global test environment tear-down\n" 246 "[==========] 1 tests from 1 test case ran. (0 ms total)\n" 247 "[ PASSED ] 1 tests.\n" 248 ); 249 250 PARSE(successful_parameterized, 251 engine::googletest_result::successful, NULL, 252 successful_parameterized_message); 253 254 const char successful_message_with_reason[] = ( 255 "Note: Google Test filter = PassFailTest.PassesWithReason\n" 256 "[==========] Running 1 test from 1 test suite.\n" 257 "[----------] Global test environment set-up.\n" 258 "[----------] 1 test from PassFailTest\n" 259 "[ RUN ] PassFailTest.PassesWithReason\n" 260 "This is a reason\n" 261 "[ OK ] PassFailTest.PassesWithReason (0 ms)\n" 262 "[----------] 1 test from PassFailTest (0 ms total)\n" 263 "\n" 264 "[----------] Global test environment tear-down\n" 265 "[==========] 1 test from 1 test suite ran. (0 ms total)\n" 266 "[ PASSED ] 1 tests.\n" 267 ); 268 269 PARSE(successful_with_reason, 270 engine::googletest_result::successful, NULL, 271 successful_message_with_reason); 272 273 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__load__ok); 274 ATF_TEST_CASE_BODY(googletest_result__load__ok) 275 { 276 std::ofstream output("result.txt"); 277 ATF_REQUIRE(output); 278 output << skipped_with_reason_message; 279 output.close(); 280 281 const engine::googletest_result result = engine::googletest_result::load( 282 utils::fs::path("result.txt")); 283 ATF_REQUIRE_EQ(engine::googletest_result::skipped, result.type()); 284 ATF_REQUIRE(result.reason()); 285 ATF_REQUIRE_EQ(skipped_with_reason_context, result.reason().get()); 286 } 287 288 289 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__load__missing_file); 290 ATF_TEST_CASE_BODY(googletest_result__load__missing_file) 291 { 292 ATF_REQUIRE_THROW_RE( 293 std::runtime_error, "Cannot open", 294 engine::googletest_result::load(utils::fs::path("result.txt"))); 295 } 296 297 298 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__broken); 299 ATF_TEST_CASE_BODY(googletest_result__apply__broken) 300 { 301 const process::status status = process::status::fake_exited(EXIT_FAILURE); 302 const engine::googletest_result broken(engine::googletest_result::broken, 303 "The reason"); 304 ATF_REQUIRE_EQ(broken, broken.apply(utils::make_optional(status))); 305 } 306 307 308 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__disabled); 309 ATF_TEST_CASE_BODY(googletest_result__apply__disabled) 310 { 311 const process::status status = process::status::fake_exited(EXIT_SUCCESS); 312 const engine::googletest_result disabled( 313 engine::googletest_result::disabled, "The reason"); 314 ATF_REQUIRE_EQ(disabled, disabled.apply(utils::make_optional(status))); 315 } 316 317 318 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__failed); 319 ATF_TEST_CASE_BODY(googletest_result__apply__failed) 320 { 321 const process::status status = process::status::fake_exited(EXIT_FAILURE); 322 const engine::googletest_result failed(engine::googletest_result::failed, 323 "The reason"); 324 ATF_REQUIRE_EQ(failed, failed.apply(utils::make_optional(status))); 325 } 326 327 328 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__skipped); 329 ATF_TEST_CASE_BODY(googletest_result__apply__skipped) 330 { 331 const process::status status = process::status::fake_exited(EXIT_SUCCESS); 332 const engine::googletest_result skipped(engine::googletest_result::skipped, 333 "The reason"); 334 ATF_REQUIRE_EQ(skipped, skipped.apply(utils::make_optional(status))); 335 } 336 337 338 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__successful); 339 ATF_TEST_CASE_BODY(googletest_result__apply__successful) 340 { 341 const process::status status = process::status::fake_exited(EXIT_SUCCESS); 342 const engine::googletest_result 343 successful(engine::googletest_result::successful); 344 ATF_REQUIRE_EQ(successful, successful.apply(utils::make_optional(status))); 345 } 346 347 348 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__broken); 349 ATF_TEST_CASE_BODY(googletest_result__externalize__broken) 350 { 351 const engine::googletest_result raw(engine::googletest_result::broken, 352 "The reason"); 353 const model::test_result expected(model::test_result_broken, 354 "The reason"); 355 ATF_REQUIRE(expected == raw.externalize()); 356 } 357 358 359 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__disabled); 360 ATF_TEST_CASE_BODY(googletest_result__externalize__disabled) 361 { 362 const engine::googletest_result raw(engine::googletest_result::disabled, 363 "The reason"); 364 const model::test_result expected(model::test_result_skipped, 365 "The reason"); 366 ATF_REQUIRE(expected == raw.externalize()); 367 } 368 369 370 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__failed); 371 ATF_TEST_CASE_BODY(googletest_result__externalize__failed) 372 { 373 const engine::googletest_result raw(engine::googletest_result::failed, 374 "The reason"); 375 const model::test_result expected(model::test_result_failed, 376 "The reason"); 377 ATF_REQUIRE(expected == raw.externalize()); 378 } 379 380 381 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__skipped); 382 ATF_TEST_CASE_BODY(googletest_result__externalize__skipped) 383 { 384 const engine::googletest_result raw(engine::googletest_result::skipped, 385 "The reason"); 386 const model::test_result expected(model::test_result_skipped, 387 "The reason"); 388 ATF_REQUIRE(expected == raw.externalize()); 389 } 390 391 392 ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__successful); 393 ATF_TEST_CASE_BODY(googletest_result__externalize__successful) 394 { 395 const engine::googletest_result raw(engine::googletest_result::successful); 396 const model::test_result expected(model::test_result_passed); 397 ATF_REQUIRE_EQ(expected, raw.externalize()); 398 } 399 400 401 ATF_INIT_TEST_CASES(tcs) 402 { 403 ATF_ADD_TEST_CASE(tcs, googletest_result__parse__broken); 404 ATF_ADD_TEST_CASE(tcs, googletest_result__parse__disabled); 405 ATF_ADD_TEST_CASE(tcs, googletest_result__parse__failed); 406 ATF_ADD_TEST_CASE(tcs, googletest_result__parse__skipped); 407 ATF_ADD_TEST_CASE(tcs, googletest_result__parse__skipped_with_reason); 408 ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful); 409 ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful2); 410 ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful_parameterized); 411 ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful_with_reason); 412 413 ATF_ADD_TEST_CASE(tcs, googletest_result__load__ok); 414 ATF_ADD_TEST_CASE(tcs, googletest_result__load__missing_file); 415 416 ATF_ADD_TEST_CASE(tcs, googletest_result__apply__broken); 417 ATF_ADD_TEST_CASE(tcs, googletest_result__apply__disabled); 418 ATF_ADD_TEST_CASE(tcs, googletest_result__apply__failed); 419 ATF_ADD_TEST_CASE(tcs, googletest_result__apply__skipped); 420 ATF_ADD_TEST_CASE(tcs, googletest_result__apply__successful); 421 422 ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__broken); 423 ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__disabled); 424 ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__failed); 425 ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__skipped); 426 ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__successful); 427 } 428