1 // Copyright 2010 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/atf_result.hpp" 30 31 extern "C" { 32 #include <signal.h> 33 } 34 35 #include <cstdlib> 36 #include <fstream> 37 #include <sstream> 38 #include <stdexcept> 39 40 #include <atf-c++.hpp> 41 42 #include "engine/exceptions.hpp" 43 #include "model/test_result.hpp" 44 #include "utils/format/macros.hpp" 45 #include "utils/fs/path.hpp" 46 #include "utils/process/status.hpp" 47 48 namespace fs = utils::fs; 49 namespace process = utils::process; 50 51 using utils::none; 52 using utils::optional; 53 54 55 namespace { 56 57 58 /// Performs a test for results::parse() that should succeed. 59 /// 60 /// \param exp_type The expected type of the result. 61 /// \param exp_argument The expected argument in the result, if any. 62 /// \param exp_reason The expected reason describing the result, if any. 63 /// \param text The literal input to parse; can include multiple lines. 64 static void 65 parse_ok_test(const engine::atf_result::types& exp_type, 66 const optional< int >& exp_argument, 67 const char* exp_reason, const char* text) 68 { 69 std::istringstream input(text); 70 const engine::atf_result actual = engine::atf_result::parse(input); 71 ATF_REQUIRE(exp_type == actual.type()); 72 ATF_REQUIRE_EQ(exp_argument, actual.argument()); 73 if (exp_reason != NULL) { 74 ATF_REQUIRE(actual.reason()); 75 ATF_REQUIRE_EQ(exp_reason, actual.reason().get()); 76 } else { 77 ATF_REQUIRE(!actual.reason()); 78 } 79 } 80 81 82 /// Wrapper around parse_ok_test to define a test case. 83 /// 84 /// \param name The name of the test case; will be prefixed with 85 /// "atf_result__parse__". 86 /// \param exp_type The expected type of the result. 87 /// \param exp_argument The expected argument in the result, if any. 88 /// \param exp_reason The expected reason describing the result, if any. 89 /// \param input The literal input to parse. 90 #define PARSE_OK(name, exp_type, exp_argument, exp_reason, input) \ 91 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__parse__ ## name); \ 92 ATF_TEST_CASE_BODY(atf_result__parse__ ## name) \ 93 { \ 94 parse_ok_test(exp_type, exp_argument, exp_reason, input); \ 95 } 96 97 98 /// Performs a test for results::parse() that should fail. 99 /// 100 /// \param reason_regexp The reason to match against the broken reason. 101 /// \param text The literal input to parse; can include multiple lines. 102 static void 103 parse_broken_test(const char* reason_regexp, const char* text) 104 { 105 std::istringstream input(text); 106 ATF_REQUIRE_THROW_RE(engine::format_error, reason_regexp, 107 engine::atf_result::parse(input)); 108 } 109 110 111 /// Wrapper around parse_broken_test to define a test case. 112 /// 113 /// \param name The name of the test case; will be prefixed with 114 /// "atf_result__parse__". 115 /// \param reason_regexp The reason to match against the broken reason. 116 /// \param input The literal input to parse. 117 #define PARSE_BROKEN(name, reason_regexp, input) \ 118 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__parse__ ## name); \ 119 ATF_TEST_CASE_BODY(atf_result__parse__ ## name) \ 120 { \ 121 parse_broken_test(reason_regexp, input); \ 122 } 123 124 125 } // anonymous namespace 126 127 128 PARSE_BROKEN(empty, 129 "Empty.*no new line", 130 ""); 131 PARSE_BROKEN(no_newline__unknown, 132 "Empty.*no new line", 133 "foo"); 134 PARSE_BROKEN(no_newline__known, 135 "Empty.*no new line", 136 "passed"); 137 PARSE_BROKEN(multiline__no_newline, 138 "multiple lines.*foo<<NEWLINE>>bar", 139 "failed: foo\nbar"); 140 PARSE_BROKEN(multiline__with_newline, 141 "multiple lines.*foo<<NEWLINE>>bar", 142 "failed: foo\nbar\n"); 143 PARSE_BROKEN(unknown_status__no_reason, 144 "Unknown.*result.*'cba'", 145 "cba\n"); 146 PARSE_BROKEN(unknown_status__with_reason, 147 "Unknown.*result.*'hgf'", 148 "hgf: foo\n"); 149 PARSE_BROKEN(missing_reason__no_delim, 150 "failed.*followed by.*reason", 151 "failed\n"); 152 PARSE_BROKEN(missing_reason__bad_delim, 153 "failed.*followed by.*reason", 154 "failed:\n"); 155 PARSE_BROKEN(missing_reason__empty, 156 "failed.*followed by.*reason", 157 "failed: \n"); 158 159 160 PARSE_OK(broken__ok, 161 engine::atf_result::broken, none, "a b c", 162 "broken: a b c\n"); 163 PARSE_OK(broken__blanks, 164 engine::atf_result::broken, none, " ", 165 "broken: \n"); 166 167 168 PARSE_OK(expected_death__ok, 169 engine::atf_result::expected_death, none, "a b c", 170 "expected_death: a b c\n"); 171 PARSE_OK(expected_death__blanks, 172 engine::atf_result::expected_death, none, " ", 173 "expected_death: \n"); 174 175 176 PARSE_OK(expected_exit__ok__any, 177 engine::atf_result::expected_exit, none, "any exit code", 178 "expected_exit: any exit code\n"); 179 PARSE_OK(expected_exit__ok__specific, 180 engine::atf_result::expected_exit, optional< int >(712), 181 "some known exit code", 182 "expected_exit(712): some known exit code\n"); 183 PARSE_BROKEN(expected_exit__bad_int, 184 "Invalid integer.*45a3", 185 "expected_exit(45a3): this is broken\n"); 186 187 188 PARSE_OK(expected_failure__ok, 189 engine::atf_result::expected_failure, none, "a b c", 190 "expected_failure: a b c\n"); 191 PARSE_OK(expected_failure__blanks, 192 engine::atf_result::expected_failure, none, " ", 193 "expected_failure: \n"); 194 195 196 PARSE_OK(expected_signal__ok__any, 197 engine::atf_result::expected_signal, none, "any signal code", 198 "expected_signal: any signal code\n"); 199 PARSE_OK(expected_signal__ok__specific, 200 engine::atf_result::expected_signal, optional< int >(712), 201 "some known signal code", 202 "expected_signal(712): some known signal code\n"); 203 PARSE_BROKEN(expected_signal__bad_int, 204 "Invalid integer.*45a3", 205 "expected_signal(45a3): this is broken\n"); 206 207 208 PARSE_OK(expected_timeout__ok, 209 engine::atf_result::expected_timeout, none, "a b c", 210 "expected_timeout: a b c\n"); 211 PARSE_OK(expected_timeout__blanks, 212 engine::atf_result::expected_timeout, none, " ", 213 "expected_timeout: \n"); 214 215 216 PARSE_OK(failed__ok, 217 engine::atf_result::failed, none, "a b c", 218 "failed: a b c\n"); 219 PARSE_OK(failed__blanks, 220 engine::atf_result::failed, none, " ", 221 "failed: \n"); 222 223 224 PARSE_OK(passed__ok, 225 engine::atf_result::passed, none, NULL, 226 "passed\n"); 227 PARSE_BROKEN(passed__reason, 228 "cannot have a reason", 229 "passed a b c\n"); 230 231 232 PARSE_OK(skipped__ok, 233 engine::atf_result::skipped, none, "a b c", 234 "skipped: a b c\n"); 235 PARSE_OK(skipped__blanks, 236 engine::atf_result::skipped, none, " ", 237 "skipped: \n"); 238 239 240 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__ok); 241 ATF_TEST_CASE_BODY(atf_result__load__ok) 242 { 243 std::ofstream output("result.txt"); 244 ATF_REQUIRE(output); 245 output << "skipped: a b c\n"; 246 output.close(); 247 248 const engine::atf_result result = engine::atf_result::load( 249 utils::fs::path("result.txt")); 250 ATF_REQUIRE(engine::atf_result::skipped == result.type()); 251 ATF_REQUIRE(!result.argument()); 252 ATF_REQUIRE(result.reason()); 253 ATF_REQUIRE_EQ("a b c", result.reason().get()); 254 } 255 256 257 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__missing_file); 258 ATF_TEST_CASE_BODY(atf_result__load__missing_file) 259 { 260 ATF_REQUIRE_THROW_RE( 261 std::runtime_error, "Cannot open", 262 engine::atf_result::load(utils::fs::path("result.txt"))); 263 } 264 265 266 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__format_error); 267 ATF_TEST_CASE_BODY(atf_result__load__format_error) 268 { 269 std::ofstream output("abc.txt"); 270 ATF_REQUIRE(output); 271 output << "passed: foo\n"; 272 output.close(); 273 274 ATF_REQUIRE_THROW_RE(engine::format_error, "cannot have a reason", 275 engine::atf_result::load(utils::fs::path("abc.txt"))); 276 } 277 278 279 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__broken__ok); 280 ATF_TEST_CASE_BODY(atf_result__apply__broken__ok) 281 { 282 const engine::atf_result in_result(engine::atf_result::broken, 283 "Passthrough"); 284 const process::status status = process::status::fake_exited(EXIT_SUCCESS); 285 ATF_REQUIRE_EQ(in_result, in_result.apply(utils::make_optional(status))); 286 } 287 288 289 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__timed_out); 290 ATF_TEST_CASE_BODY(atf_result__apply__timed_out) 291 { 292 const engine::atf_result timed_out(engine::atf_result::broken, 293 "Some arbitrary error"); 294 ATF_REQUIRE_EQ(engine::atf_result(engine::atf_result::broken, 295 "Test case body timed out"), 296 timed_out.apply(none)); 297 } 298 299 300 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_death__ok); 301 ATF_TEST_CASE_BODY(atf_result__apply__expected_death__ok) 302 { 303 const engine::atf_result in_result(engine::atf_result::expected_death, 304 "Passthrough"); 305 const process::status status = process::status::fake_signaled(SIGINT, true); 306 ATF_REQUIRE_EQ(in_result, in_result.apply(utils::make_optional(status))); 307 } 308 309 310 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__ok); 311 ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__ok) 312 { 313 const process::status success = process::status::fake_exited(EXIT_SUCCESS); 314 const process::status failure = process::status::fake_exited(EXIT_FAILURE); 315 316 const engine::atf_result any_code(engine::atf_result::expected_exit, none, 317 "The reason"); 318 ATF_REQUIRE_EQ(any_code, any_code.apply(utils::make_optional(success))); 319 ATF_REQUIRE_EQ(any_code, any_code.apply(utils::make_optional(failure))); 320 321 const engine::atf_result a_code(engine::atf_result::expected_exit, 322 utils::make_optional(EXIT_FAILURE), "The reason"); 323 ATF_REQUIRE_EQ(a_code, a_code.apply(utils::make_optional(failure))); 324 } 325 326 327 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__failed); 328 ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__failed) 329 { 330 const process::status success = process::status::fake_exited(EXIT_SUCCESS); 331 332 const engine::atf_result a_code(engine::atf_result::expected_exit, 333 utils::make_optional(EXIT_FAILURE), "The reason"); 334 ATF_REQUIRE_EQ( 335 engine::atf_result(engine::atf_result::failed, 336 "Test case expected to exit with code 1 but got " 337 "code 0"), 338 a_code.apply(utils::make_optional(success))); 339 } 340 341 342 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__broken); 343 ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__broken) 344 { 345 const process::status sig3 = process::status::fake_signaled(3, false); 346 347 const engine::atf_result any_code(engine::atf_result::expected_exit, none, 348 "The reason"); 349 ATF_REQUIRE_EQ( 350 engine::atf_result(engine::atf_result::broken, 351 "Expected clean exit but received signal 3"), 352 any_code.apply(utils::make_optional(sig3))); 353 } 354 355 356 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_failure__ok); 357 ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__ok) 358 { 359 const process::status status = process::status::fake_exited(EXIT_SUCCESS); 360 const engine::atf_result xfailure(engine::atf_result::expected_failure, 361 "The reason"); 362 ATF_REQUIRE_EQ(xfailure, xfailure.apply(utils::make_optional(status))); 363 } 364 365 366 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_failure__broken); 367 ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__broken) 368 { 369 const process::status failure = process::status::fake_exited(EXIT_FAILURE); 370 const process::status sig3 = process::status::fake_signaled(3, true); 371 const process::status sig4 = process::status::fake_signaled(4, false); 372 373 const engine::atf_result xfailure(engine::atf_result::expected_failure, 374 "The reason"); 375 ATF_REQUIRE_EQ( 376 engine::atf_result(engine::atf_result::broken, 377 "Expected failure should have reported success but " 378 "exited with code 1"), 379 xfailure.apply(utils::make_optional(failure))); 380 ATF_REQUIRE_EQ( 381 engine::atf_result(engine::atf_result::broken, 382 "Expected failure should have reported success but " 383 "received signal 3 (core dumped)"), 384 xfailure.apply(utils::make_optional(sig3))); 385 ATF_REQUIRE_EQ( 386 engine::atf_result(engine::atf_result::broken, 387 "Expected failure should have reported success but " 388 "received signal 4"), 389 xfailure.apply(utils::make_optional(sig4))); 390 } 391 392 393 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__ok); 394 ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__ok) 395 { 396 const process::status sig1 = process::status::fake_signaled(1, false); 397 const process::status sig3 = process::status::fake_signaled(3, true); 398 399 const engine::atf_result any_sig(engine::atf_result::expected_signal, none, 400 "The reason"); 401 ATF_REQUIRE_EQ(any_sig, any_sig.apply(utils::make_optional(sig1))); 402 ATF_REQUIRE_EQ(any_sig, any_sig.apply(utils::make_optional(sig3))); 403 404 const engine::atf_result a_sig(engine::atf_result::expected_signal, 405 utils::make_optional(3), "The reason"); 406 ATF_REQUIRE_EQ(a_sig, a_sig.apply(utils::make_optional(sig3))); 407 } 408 409 410 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__failed); 411 ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__failed) 412 { 413 const process::status sig5 = process::status::fake_signaled(5, false); 414 415 const engine::atf_result a_sig(engine::atf_result::expected_signal, 416 utils::make_optional(4), "The reason"); 417 ATF_REQUIRE_EQ( 418 engine::atf_result(engine::atf_result::failed, 419 "Test case expected to receive signal 4 but got 5"), 420 a_sig.apply(utils::make_optional(sig5))); 421 } 422 423 424 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__broken); 425 ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__broken) 426 { 427 const process::status success = process::status::fake_exited(EXIT_SUCCESS); 428 429 const engine::atf_result any_sig(engine::atf_result::expected_signal, none, 430 "The reason"); 431 ATF_REQUIRE_EQ( 432 engine::atf_result(engine::atf_result::broken, 433 "Expected signal but exited with code 0"), 434 any_sig.apply(utils::make_optional(success))); 435 } 436 437 438 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_timeout__ok); 439 ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__ok) 440 { 441 const engine::atf_result timeout(engine::atf_result::expected_timeout, 442 "The reason"); 443 ATF_REQUIRE_EQ(timeout, timeout.apply(none)); 444 } 445 446 447 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_timeout__broken); 448 ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__broken) 449 { 450 const process::status status = process::status::fake_exited(EXIT_SUCCESS); 451 const engine::atf_result timeout(engine::atf_result::expected_timeout, 452 "The reason"); 453 ATF_REQUIRE_EQ( 454 engine::atf_result(engine::atf_result::broken, 455 "Expected timeout but exited with code 0"), 456 timeout.apply(utils::make_optional(status))); 457 } 458 459 460 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__failed__ok); 461 ATF_TEST_CASE_BODY(atf_result__apply__failed__ok) 462 { 463 const process::status status = process::status::fake_exited(EXIT_FAILURE); 464 const engine::atf_result failed(engine::atf_result::failed, "The reason"); 465 ATF_REQUIRE_EQ(failed, failed.apply(utils::make_optional(status))); 466 } 467 468 469 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__failed__broken); 470 ATF_TEST_CASE_BODY(atf_result__apply__failed__broken) 471 { 472 const process::status success = process::status::fake_exited(EXIT_SUCCESS); 473 const process::status sig3 = process::status::fake_signaled(3, true); 474 const process::status sig4 = process::status::fake_signaled(4, false); 475 476 const engine::atf_result failed(engine::atf_result::failed, "The reason"); 477 ATF_REQUIRE_EQ( 478 engine::atf_result(engine::atf_result::broken, 479 "Failed test case should have reported failure but " 480 "exited with code 0"), 481 failed.apply(utils::make_optional(success))); 482 ATF_REQUIRE_EQ( 483 engine::atf_result(engine::atf_result::broken, 484 "Failed test case should have reported failure but " 485 "received signal 3 (core dumped)"), 486 failed.apply(utils::make_optional(sig3))); 487 ATF_REQUIRE_EQ( 488 engine::atf_result(engine::atf_result::broken, 489 "Failed test case should have reported failure but " 490 "received signal 4"), 491 failed.apply(utils::make_optional(sig4))); 492 } 493 494 495 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__passed__ok); 496 ATF_TEST_CASE_BODY(atf_result__apply__passed__ok) 497 { 498 const process::status status = process::status::fake_exited(EXIT_SUCCESS); 499 const engine::atf_result passed(engine::atf_result::passed); 500 ATF_REQUIRE_EQ(passed, passed.apply(utils::make_optional(status))); 501 } 502 503 504 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__passed__broken); 505 ATF_TEST_CASE_BODY(atf_result__apply__passed__broken) 506 { 507 const process::status failure = process::status::fake_exited(EXIT_FAILURE); 508 const process::status sig3 = process::status::fake_signaled(3, true); 509 const process::status sig4 = process::status::fake_signaled(4, false); 510 511 const engine::atf_result passed(engine::atf_result::passed); 512 ATF_REQUIRE_EQ( 513 engine::atf_result(engine::atf_result::broken, 514 "Passed test case should have reported success but " 515 "exited with code 1"), 516 passed.apply(utils::make_optional(failure))); 517 ATF_REQUIRE_EQ( 518 engine::atf_result(engine::atf_result::broken, 519 "Passed test case should have reported success but " 520 "received signal 3 (core dumped)"), 521 passed.apply(utils::make_optional(sig3))); 522 ATF_REQUIRE_EQ( 523 engine::atf_result(engine::atf_result::broken, 524 "Passed test case should have reported success but " 525 "received signal 4"), 526 passed.apply(utils::make_optional(sig4))); 527 } 528 529 530 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__skipped__ok); 531 ATF_TEST_CASE_BODY(atf_result__apply__skipped__ok) 532 { 533 const process::status status = process::status::fake_exited(EXIT_SUCCESS); 534 const engine::atf_result skipped(engine::atf_result::skipped, "The reason"); 535 ATF_REQUIRE_EQ(skipped, skipped.apply(utils::make_optional(status))); 536 } 537 538 539 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__skipped__broken); 540 ATF_TEST_CASE_BODY(atf_result__apply__skipped__broken) 541 { 542 const process::status failure = process::status::fake_exited(EXIT_FAILURE); 543 const process::status sig3 = process::status::fake_signaled(3, true); 544 const process::status sig4 = process::status::fake_signaled(4, false); 545 546 const engine::atf_result skipped(engine::atf_result::skipped, "The reason"); 547 ATF_REQUIRE_EQ( 548 engine::atf_result(engine::atf_result::broken, 549 "Skipped test case should have reported success but " 550 "exited with code 1"), 551 skipped.apply(utils::make_optional(failure))); 552 ATF_REQUIRE_EQ( 553 engine::atf_result(engine::atf_result::broken, 554 "Skipped test case should have reported success but " 555 "received signal 3 (core dumped)"), 556 skipped.apply(utils::make_optional(sig3))); 557 ATF_REQUIRE_EQ( 558 engine::atf_result(engine::atf_result::broken, 559 "Skipped test case should have reported success but " 560 "received signal 4"), 561 skipped.apply(utils::make_optional(sig4))); 562 } 563 564 565 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__broken); 566 ATF_TEST_CASE_BODY(atf_result__externalize__broken) 567 { 568 const engine::atf_result raw(engine::atf_result::broken, "The reason"); 569 const model::test_result expected(model::test_result_broken, 570 "The reason"); 571 ATF_REQUIRE_EQ(expected, raw.externalize()); 572 } 573 574 575 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_death); 576 ATF_TEST_CASE_BODY(atf_result__externalize__expected_death) 577 { 578 const engine::atf_result raw(engine::atf_result::expected_death, 579 "The reason"); 580 const model::test_result expected(model::test_result_expected_failure, 581 "The reason"); 582 ATF_REQUIRE_EQ(expected, raw.externalize()); 583 } 584 585 586 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_exit); 587 ATF_TEST_CASE_BODY(atf_result__externalize__expected_exit) 588 { 589 const engine::atf_result raw(engine::atf_result::expected_exit, 590 "The reason"); 591 const model::test_result expected(model::test_result_expected_failure, 592 "The reason"); 593 ATF_REQUIRE_EQ(expected, raw.externalize()); 594 } 595 596 597 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_failure); 598 ATF_TEST_CASE_BODY(atf_result__externalize__expected_failure) 599 { 600 const engine::atf_result raw(engine::atf_result::expected_failure, 601 "The reason"); 602 const model::test_result expected(model::test_result_expected_failure, 603 "The reason"); 604 ATF_REQUIRE_EQ(expected, raw.externalize()); 605 } 606 607 608 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_signal); 609 ATF_TEST_CASE_BODY(atf_result__externalize__expected_signal) 610 { 611 const engine::atf_result raw(engine::atf_result::expected_signal, 612 "The reason"); 613 const model::test_result expected(model::test_result_expected_failure, 614 "The reason"); 615 ATF_REQUIRE_EQ(expected, raw.externalize()); 616 } 617 618 619 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_timeout); 620 ATF_TEST_CASE_BODY(atf_result__externalize__expected_timeout) 621 { 622 const engine::atf_result raw(engine::atf_result::expected_timeout, 623 "The reason"); 624 const model::test_result expected(model::test_result_expected_failure, 625 "The reason"); 626 ATF_REQUIRE_EQ(expected, raw.externalize()); 627 } 628 629 630 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__failed); 631 ATF_TEST_CASE_BODY(atf_result__externalize__failed) 632 { 633 const engine::atf_result raw(engine::atf_result::failed, "The reason"); 634 const model::test_result expected(model::test_result_failed, 635 "The reason"); 636 ATF_REQUIRE(expected == raw.externalize()); 637 } 638 639 640 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__passed); 641 ATF_TEST_CASE_BODY(atf_result__externalize__passed) 642 { 643 const engine::atf_result raw(engine::atf_result::passed); 644 const model::test_result expected(model::test_result_passed); 645 ATF_REQUIRE_EQ(expected, raw.externalize()); 646 } 647 648 649 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__skipped); 650 ATF_TEST_CASE_BODY(atf_result__externalize__skipped) 651 { 652 const engine::atf_result raw(engine::atf_result::skipped, "The reason"); 653 const model::test_result expected(model::test_result_skipped, 654 "The reason"); 655 ATF_REQUIRE_EQ(expected, raw.externalize()); 656 } 657 658 659 ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__missing_file); 660 ATF_TEST_CASE_BODY(calculate_atf_result__missing_file) 661 { 662 using process::status; 663 664 const status body_status = status::fake_exited(EXIT_SUCCESS); 665 const model::test_result expected( 666 model::test_result_broken, 667 "Premature exit; test case exited with code 0"); 668 ATF_REQUIRE_EQ(expected, engine::calculate_atf_result( 669 utils::make_optional(body_status), fs::path("foo"))); 670 } 671 672 673 ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__bad_file); 674 ATF_TEST_CASE_BODY(calculate_atf_result__bad_file) 675 { 676 using process::status; 677 678 const status body_status = status::fake_exited(EXIT_SUCCESS); 679 atf::utils::create_file("foo", "invalid\n"); 680 const model::test_result expected(model::test_result_broken, 681 "Unknown test result 'invalid'"); 682 ATF_REQUIRE_EQ(expected, engine::calculate_atf_result( 683 utils::make_optional(body_status), fs::path("foo"))); 684 } 685 686 687 ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__body_ok); 688 ATF_TEST_CASE_BODY(calculate_atf_result__body_ok) 689 { 690 using process::status; 691 692 atf::utils::create_file("result.txt", "skipped: Something\n"); 693 const status body_status = status::fake_exited(EXIT_SUCCESS); 694 ATF_REQUIRE_EQ( 695 model::test_result(model::test_result_skipped, "Something"), 696 engine::calculate_atf_result(utils::make_optional(body_status), 697 fs::path("result.txt"))); 698 } 699 700 701 ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__body_bad); 702 ATF_TEST_CASE_BODY(calculate_atf_result__body_bad) 703 { 704 using process::status; 705 706 atf::utils::create_file("result.txt", "skipped: Something\n"); 707 const status body_status = status::fake_exited(EXIT_FAILURE); 708 ATF_REQUIRE_EQ( 709 model::test_result(model::test_result_broken, "Skipped test case " 710 "should have reported success but exited with " 711 "code 1"), 712 engine::calculate_atf_result(utils::make_optional(body_status), 713 fs::path("result.txt"))); 714 } 715 716 717 ATF_INIT_TEST_CASES(tcs) 718 { 719 ATF_ADD_TEST_CASE(tcs, atf_result__parse__empty); 720 ATF_ADD_TEST_CASE(tcs, atf_result__parse__no_newline__unknown); 721 ATF_ADD_TEST_CASE(tcs, atf_result__parse__no_newline__known); 722 ATF_ADD_TEST_CASE(tcs, atf_result__parse__multiline__no_newline); 723 ATF_ADD_TEST_CASE(tcs, atf_result__parse__multiline__with_newline); 724 ATF_ADD_TEST_CASE(tcs, atf_result__parse__unknown_status__no_reason); 725 ATF_ADD_TEST_CASE(tcs, atf_result__parse__unknown_status__with_reason); 726 ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__no_delim); 727 ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__bad_delim); 728 ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__empty); 729 ATF_ADD_TEST_CASE(tcs, atf_result__parse__broken__ok); 730 ATF_ADD_TEST_CASE(tcs, atf_result__parse__broken__blanks); 731 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_death__ok); 732 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_death__blanks); 733 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__ok__any); 734 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__ok__specific); 735 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__bad_int); 736 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_failure__ok); 737 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_failure__blanks); 738 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__ok__any); 739 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__ok__specific); 740 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__bad_int); 741 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_timeout__ok); 742 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_timeout__blanks); 743 ATF_ADD_TEST_CASE(tcs, atf_result__parse__failed__ok); 744 ATF_ADD_TEST_CASE(tcs, atf_result__parse__failed__blanks); 745 ATF_ADD_TEST_CASE(tcs, atf_result__parse__passed__ok); 746 ATF_ADD_TEST_CASE(tcs, atf_result__parse__passed__reason); 747 ATF_ADD_TEST_CASE(tcs, atf_result__parse__skipped__ok); 748 ATF_ADD_TEST_CASE(tcs, atf_result__parse__skipped__blanks); 749 750 ATF_ADD_TEST_CASE(tcs, atf_result__load__ok); 751 ATF_ADD_TEST_CASE(tcs, atf_result__load__missing_file); 752 ATF_ADD_TEST_CASE(tcs, atf_result__load__format_error); 753 754 ATF_ADD_TEST_CASE(tcs, atf_result__apply__broken__ok); 755 ATF_ADD_TEST_CASE(tcs, atf_result__apply__timed_out); 756 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_death__ok); 757 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__ok); 758 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__failed); 759 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__broken); 760 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_failure__ok); 761 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_failure__broken); 762 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__ok); 763 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__failed); 764 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__broken); 765 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_timeout__ok); 766 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_timeout__broken); 767 ATF_ADD_TEST_CASE(tcs, atf_result__apply__failed__ok); 768 ATF_ADD_TEST_CASE(tcs, atf_result__apply__failed__broken); 769 ATF_ADD_TEST_CASE(tcs, atf_result__apply__passed__ok); 770 ATF_ADD_TEST_CASE(tcs, atf_result__apply__passed__broken); 771 ATF_ADD_TEST_CASE(tcs, atf_result__apply__skipped__ok); 772 ATF_ADD_TEST_CASE(tcs, atf_result__apply__skipped__broken); 773 774 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__broken); 775 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_death); 776 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_exit); 777 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_failure); 778 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_signal); 779 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_timeout); 780 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__failed); 781 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__passed); 782 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__skipped); 783 784 ATF_ADD_TEST_CASE(tcs, calculate_atf_result__missing_file); 785 ATF_ADD_TEST_CASE(tcs, calculate_atf_result__bad_file); 786 ATF_ADD_TEST_CASE(tcs, calculate_atf_result__body_ok); 787 ATF_ADD_TEST_CASE(tcs, calculate_atf_result__body_bad); 788 } 789