1 // 2 // Automated Testing Framework (atf) 3 // 4 // Copyright (c) 2007 The NetBSD Foundation, Inc. 5 // All rights reserved. 6 // 7 // Redistribution and use in source and binary forms, with or without 8 // modification, are permitted provided that the following conditions 9 // are met: 10 // 1. Redistributions of source code must retain the above copyright 11 // notice, this list of conditions and the following disclaimer. 12 // 2. Redistributions in binary form must reproduce the above copyright 13 // notice, this list of conditions and the following disclaimer in the 14 // documentation and/or other materials provided with the distribution. 15 // 16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 // 29 30 extern "C" { 31 #include <sys/types.h> 32 #include <sys/stat.h> 33 #include <sys/time.h> 34 #include <sys/wait.h> 35 #include <signal.h> 36 #include <unistd.h> 37 } 38 39 #include <algorithm> 40 #include <cctype> 41 #include <cerrno> 42 #include <cstdlib> 43 #include <cstring> 44 #include <fstream> 45 #include <iostream> 46 #include <map> 47 #include <memory> 48 #include <sstream> 49 #include <stdexcept> 50 #include <vector> 51 52 extern "C" { 53 #include "atf-c/error.h" 54 #include "atf-c/tc.h" 55 #include "atf-c/utils.h" 56 } 57 58 #include "tests.hpp" 59 60 #include "detail/application.hpp" 61 #include "detail/env.hpp" 62 #include "detail/exceptions.hpp" 63 #include "detail/fs.hpp" 64 #include "detail/parser.hpp" 65 #include "detail/sanity.hpp" 66 #include "detail/text.hpp" 67 68 namespace impl = atf::tests; 69 namespace detail = atf::tests::detail; 70 #define IMPL_NAME "atf::tests" 71 72 // ------------------------------------------------------------------------ 73 // The "atf_tp_writer" class. 74 // ------------------------------------------------------------------------ 75 76 detail::atf_tp_writer::atf_tp_writer(std::ostream& os) : 77 m_os(os), 78 m_is_first(true) 79 { 80 atf::parser::headers_map hm; 81 atf::parser::attrs_map ct_attrs; 82 ct_attrs["version"] = "1"; 83 hm["Content-Type"] = atf::parser::header_entry("Content-Type", 84 "application/X-atf-tp", ct_attrs); 85 atf::parser::write_headers(hm, m_os); 86 } 87 88 void 89 detail::atf_tp_writer::start_tc(const std::string& ident) 90 { 91 if (!m_is_first) 92 m_os << "\n"; 93 m_os << "ident: " << ident << "\n"; 94 m_os.flush(); 95 } 96 97 void 98 detail::atf_tp_writer::end_tc(void) 99 { 100 if (m_is_first) 101 m_is_first = false; 102 } 103 104 void 105 detail::atf_tp_writer::tc_meta_data(const std::string& name, 106 const std::string& value) 107 { 108 PRE(name != "ident"); 109 m_os << name << ": " << value << "\n"; 110 m_os.flush(); 111 } 112 113 // ------------------------------------------------------------------------ 114 // Free helper functions. 115 // ------------------------------------------------------------------------ 116 117 bool 118 detail::match(const std::string& regexp, const std::string& str) 119 { 120 return atf::text::match(str, regexp); 121 } 122 123 // ------------------------------------------------------------------------ 124 // The "tc" class. 125 // ------------------------------------------------------------------------ 126 127 static std::map< atf_tc_t*, impl::tc* > wraps; 128 static std::map< const atf_tc_t*, const impl::tc* > cwraps; 129 130 struct impl::tc_impl : atf::utils::noncopyable { 131 std::string m_ident; 132 atf_tc_t m_tc; 133 bool m_has_cleanup; 134 135 tc_impl(const std::string& ident, const bool has_cleanup) : 136 m_ident(ident), 137 m_has_cleanup(has_cleanup) 138 { 139 } 140 141 static void 142 wrap_head(atf_tc_t *tc) 143 { 144 std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc); 145 INV(iter != wraps.end()); 146 (*iter).second->head(); 147 } 148 149 static void 150 wrap_body(const atf_tc_t *tc) 151 { 152 std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter = 153 cwraps.find(tc); 154 INV(iter != cwraps.end()); 155 try { 156 (*iter).second->body(); 157 } catch (const std::exception& e) { 158 (*iter).second->fail("Caught unhandled exception: " + std::string( 159 e.what())); 160 } catch (...) { 161 (*iter).second->fail("Caught unknown exception"); 162 } 163 } 164 165 static void 166 wrap_cleanup(const atf_tc_t *tc) 167 { 168 std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter = 169 cwraps.find(tc); 170 INV(iter != cwraps.end()); 171 (*iter).second->cleanup(); 172 } 173 }; 174 175 impl::tc::tc(const std::string& ident, const bool has_cleanup) : 176 pimpl(new tc_impl(ident, has_cleanup)) 177 { 178 } 179 180 impl::tc::~tc(void) 181 { 182 cwraps.erase(&pimpl->m_tc); 183 wraps.erase(&pimpl->m_tc); 184 185 atf_tc_fini(&pimpl->m_tc); 186 } 187 188 void 189 impl::tc::init(const vars_map& config) 190 { 191 atf_error_t err; 192 193 utils::auto_array< const char * > array( 194 new const char*[(config.size() * 2) + 1]); 195 const char **ptr = array.get(); 196 for (vars_map::const_iterator iter = config.begin(); 197 iter != config.end(); iter++) { 198 *ptr = (*iter).first.c_str(); 199 *(ptr + 1) = (*iter).second.c_str(); 200 ptr += 2; 201 } 202 *ptr = NULL; 203 204 wraps[&pimpl->m_tc] = this; 205 cwraps[&pimpl->m_tc] = this; 206 207 err = atf_tc_init(&pimpl->m_tc, pimpl->m_ident.c_str(), pimpl->wrap_head, 208 pimpl->wrap_body, pimpl->m_has_cleanup ? pimpl->wrap_cleanup : NULL, 209 array.get()); 210 if (atf_is_error(err)) 211 throw_atf_error(err); 212 } 213 214 bool 215 impl::tc::has_config_var(const std::string& var) 216 const 217 { 218 return atf_tc_has_config_var(&pimpl->m_tc, var.c_str()); 219 } 220 221 bool 222 impl::tc::has_md_var(const std::string& var) 223 const 224 { 225 return atf_tc_has_md_var(&pimpl->m_tc, var.c_str()); 226 } 227 228 const std::string 229 impl::tc::get_config_var(const std::string& var) 230 const 231 { 232 return atf_tc_get_config_var(&pimpl->m_tc, var.c_str()); 233 } 234 235 const std::string 236 impl::tc::get_config_var(const std::string& var, const std::string& defval) 237 const 238 { 239 return atf_tc_get_config_var_wd(&pimpl->m_tc, var.c_str(), defval.c_str()); 240 } 241 242 const std::string 243 impl::tc::get_md_var(const std::string& var) 244 const 245 { 246 return atf_tc_get_md_var(&pimpl->m_tc, var.c_str()); 247 } 248 249 const impl::vars_map 250 impl::tc::get_md_vars(void) 251 const 252 { 253 vars_map vars; 254 255 char **array = atf_tc_get_md_vars(&pimpl->m_tc); 256 try { 257 char **ptr; 258 for (ptr = array; *ptr != NULL; ptr += 2) 259 vars[*ptr] = *(ptr + 1); 260 } catch (...) { 261 atf_utils_free_charpp(array); 262 throw; 263 } 264 265 return vars; 266 } 267 268 void 269 impl::tc::set_md_var(const std::string& var, const std::string& val) 270 { 271 atf_error_t err = atf_tc_set_md_var(&pimpl->m_tc, var.c_str(), val.c_str()); 272 if (atf_is_error(err)) 273 throw_atf_error(err); 274 } 275 276 void 277 impl::tc::run(const std::string& resfile) 278 const 279 { 280 atf_error_t err = atf_tc_run(&pimpl->m_tc, resfile.c_str()); 281 if (atf_is_error(err)) 282 throw_atf_error(err); 283 } 284 285 void 286 impl::tc::run_cleanup(void) 287 const 288 { 289 atf_error_t err = atf_tc_cleanup(&pimpl->m_tc); 290 if (atf_is_error(err)) 291 throw_atf_error(err); 292 } 293 294 void 295 impl::tc::head(void) 296 { 297 } 298 299 void 300 impl::tc::cleanup(void) 301 const 302 { 303 } 304 305 void 306 impl::tc::require_prog(const std::string& prog) 307 const 308 { 309 atf_tc_require_prog(prog.c_str()); 310 } 311 312 void 313 impl::tc::pass(void) 314 { 315 atf_tc_pass(); 316 } 317 318 void 319 impl::tc::fail(const std::string& reason) 320 { 321 atf_tc_fail("%s", reason.c_str()); 322 } 323 324 void 325 impl::tc::fail_nonfatal(const std::string& reason) 326 { 327 atf_tc_fail_nonfatal("%s", reason.c_str()); 328 } 329 330 void 331 impl::tc::skip(const std::string& reason) 332 { 333 atf_tc_skip("%s", reason.c_str()); 334 } 335 336 void 337 impl::tc::check_errno(const char* file, const int line, const int exp_errno, 338 const char* expr_str, const bool result) 339 { 340 atf_tc_check_errno(file, line, exp_errno, expr_str, result); 341 } 342 343 void 344 impl::tc::require_errno(const char* file, const int line, const int exp_errno, 345 const char* expr_str, const bool result) 346 { 347 atf_tc_require_errno(file, line, exp_errno, expr_str, result); 348 } 349 350 void 351 impl::tc::expect_pass(void) 352 { 353 atf_tc_expect_pass(); 354 } 355 356 void 357 impl::tc::expect_fail(const std::string& reason) 358 { 359 atf_tc_expect_fail("%s", reason.c_str()); 360 } 361 362 void 363 impl::tc::expect_exit(const int exitcode, const std::string& reason) 364 { 365 atf_tc_expect_exit(exitcode, "%s", reason.c_str()); 366 } 367 368 void 369 impl::tc::expect_signal(const int signo, const std::string& reason) 370 { 371 atf_tc_expect_signal(signo, "%s", reason.c_str()); 372 } 373 374 void 375 impl::tc::expect_death(const std::string& reason) 376 { 377 atf_tc_expect_death("%s", reason.c_str()); 378 } 379 380 void 381 impl::tc::expect_timeout(const std::string& reason) 382 { 383 atf_tc_expect_timeout("%s", reason.c_str()); 384 } 385 386 // ------------------------------------------------------------------------ 387 // The "tp" class. 388 // ------------------------------------------------------------------------ 389 390 class tp : public atf::application::app { 391 public: 392 typedef std::vector< impl::tc * > tc_vector; 393 394 private: 395 static const char* m_description; 396 397 bool m_lflag; 398 atf::fs::path m_resfile; 399 std::string m_srcdir_arg; 400 atf::fs::path m_srcdir; 401 402 atf::tests::vars_map m_vars; 403 404 std::string specific_args(void) const; 405 options_set specific_options(void) const; 406 void process_option(int, const char*); 407 408 void (*m_add_tcs)(tc_vector&); 409 tc_vector m_tcs; 410 411 void parse_vflag(const std::string&); 412 void handle_srcdir(void); 413 414 tc_vector init_tcs(void); 415 416 enum tc_part { 417 BODY, 418 CLEANUP, 419 }; 420 421 void list_tcs(void); 422 impl::tc* find_tc(tc_vector, const std::string&); 423 static std::pair< std::string, tc_part > process_tcarg(const std::string&); 424 int run_tc(const std::string&); 425 426 public: 427 tp(void (*)(tc_vector&)); 428 ~tp(void); 429 430 int main(void); 431 }; 432 433 const char* tp::m_description = 434 "This is an independent atf test program."; 435 436 tp::tp(void (*add_tcs)(tc_vector&)) : 437 app(m_description, "atf-test-program(1)", "atf(7)", false), 438 m_lflag(false), 439 m_resfile("/dev/stdout"), 440 m_srcdir("."), 441 m_add_tcs(add_tcs) 442 { 443 } 444 445 tp::~tp(void) 446 { 447 for (tc_vector::iterator iter = m_tcs.begin(); 448 iter != m_tcs.end(); iter++) { 449 impl::tc* tc = *iter; 450 451 delete tc; 452 } 453 } 454 455 std::string 456 tp::specific_args(void) 457 const 458 { 459 return "test_case"; 460 } 461 462 tp::options_set 463 tp::specific_options(void) 464 const 465 { 466 using atf::application::option; 467 options_set opts; 468 opts.insert(option('l', "", "List test cases and their purpose")); 469 opts.insert(option('r', "resfile", "The file to which the test program " 470 "will write the results of the " 471 "executed test case")); 472 opts.insert(option('s', "srcdir", "Directory where the test's data " 473 "files are located")); 474 opts.insert(option('v', "var=value", "Sets the configuration variable " 475 "`var' to `value'")); 476 return opts; 477 } 478 479 void 480 tp::process_option(int ch, const char* arg) 481 { 482 switch (ch) { 483 case 'l': 484 m_lflag = true; 485 break; 486 487 case 'r': 488 m_resfile = atf::fs::path(arg); 489 break; 490 491 case 's': 492 m_srcdir_arg = arg; 493 break; 494 495 case 'v': 496 parse_vflag(arg); 497 break; 498 499 default: 500 UNREACHABLE; 501 } 502 } 503 504 void 505 tp::parse_vflag(const std::string& str) 506 { 507 if (str.empty()) 508 throw std::runtime_error("-v requires a non-empty argument"); 509 510 std::vector< std::string > ws = atf::text::split(str, "="); 511 if (ws.size() == 1 && str[str.length() - 1] == '=') { 512 m_vars[ws[0]] = ""; 513 } else { 514 if (ws.size() != 2) 515 throw std::runtime_error("-v requires an argument of the form " 516 "var=value"); 517 518 m_vars[ws[0]] = ws[1]; 519 } 520 } 521 522 void 523 tp::handle_srcdir(void) 524 { 525 if (m_srcdir_arg.empty()) { 526 m_srcdir = atf::fs::path(m_argv0).branch_path(); 527 if (m_srcdir.leaf_name() == ".libs") 528 m_srcdir = m_srcdir.branch_path(); 529 } else 530 m_srcdir = atf::fs::path(m_srcdir_arg); 531 532 if (!atf::fs::exists(m_srcdir / m_prog_name)) 533 throw std::runtime_error("Cannot find the test program in the " 534 "source directory `" + m_srcdir.str() + "'"); 535 536 if (!m_srcdir.is_absolute()) 537 m_srcdir = m_srcdir.to_absolute(); 538 539 m_vars["srcdir"] = m_srcdir.str(); 540 } 541 542 tp::tc_vector 543 tp::init_tcs(void) 544 { 545 m_add_tcs(m_tcs); 546 for (tc_vector::iterator iter = m_tcs.begin(); 547 iter != m_tcs.end(); iter++) { 548 impl::tc* tc = *iter; 549 550 tc->init(m_vars); 551 } 552 return m_tcs; 553 } 554 555 // 556 // An auxiliary unary predicate that compares the given test case's 557 // identifier to the identifier stored in it. 558 // 559 class tc_equal_to_ident { 560 const std::string& m_ident; 561 562 public: 563 tc_equal_to_ident(const std::string& i) : 564 m_ident(i) 565 { 566 } 567 568 bool operator()(const impl::tc* tc) 569 { 570 return tc->get_md_var("ident") == m_ident; 571 } 572 }; 573 574 void 575 tp::list_tcs(void) 576 { 577 tc_vector tcs = init_tcs(); 578 detail::atf_tp_writer writer(std::cout); 579 580 for (tc_vector::const_iterator iter = tcs.begin(); 581 iter != tcs.end(); iter++) { 582 const impl::vars_map vars = (*iter)->get_md_vars(); 583 584 { 585 impl::vars_map::const_iterator iter2 = vars.find("ident"); 586 INV(iter2 != vars.end()); 587 writer.start_tc((*iter2).second); 588 } 589 590 for (impl::vars_map::const_iterator iter2 = vars.begin(); 591 iter2 != vars.end(); iter2++) { 592 const std::string& key = (*iter2).first; 593 if (key != "ident") 594 writer.tc_meta_data(key, (*iter2).second); 595 } 596 597 writer.end_tc(); 598 } 599 } 600 601 impl::tc* 602 tp::find_tc(tc_vector tcs, const std::string& name) 603 { 604 std::vector< std::string > ids; 605 for (tc_vector::iterator iter = tcs.begin(); 606 iter != tcs.end(); iter++) { 607 impl::tc* tc = *iter; 608 609 if (tc->get_md_var("ident") == name) 610 return tc; 611 } 612 throw atf::application::usage_error("Unknown test case `%s'", 613 name.c_str()); 614 } 615 616 std::pair< std::string, tp::tc_part > 617 tp::process_tcarg(const std::string& tcarg) 618 { 619 const std::string::size_type pos = tcarg.find(':'); 620 if (pos == std::string::npos) { 621 return std::make_pair(tcarg, BODY); 622 } else { 623 const std::string tcname = tcarg.substr(0, pos); 624 625 const std::string partname = tcarg.substr(pos + 1); 626 if (partname == "body") 627 return std::make_pair(tcname, BODY); 628 else if (partname == "cleanup") 629 return std::make_pair(tcname, CLEANUP); 630 else { 631 using atf::application::usage_error; 632 throw usage_error("Invalid test case part `%s'", partname.c_str()); 633 } 634 } 635 } 636 637 int 638 tp::run_tc(const std::string& tcarg) 639 { 640 const std::pair< std::string, tc_part > fields = process_tcarg(tcarg); 641 642 impl::tc* tc = find_tc(init_tcs(), fields.first); 643 644 if (!atf::env::has("__RUNNING_INSIDE_ATF_RUN") || atf::env::get( 645 "__RUNNING_INSIDE_ATF_RUN") != "internal-yes-value") 646 { 647 std::cerr << m_prog_name << ": WARNING: Running test cases without " 648 "atf-run(1) is unsupported\n"; 649 std::cerr << m_prog_name << ": WARNING: No isolation nor timeout " 650 "control is being applied; you may get unexpected failures; see " 651 "atf-test-case(4)\n"; 652 } 653 654 try { 655 switch (fields.second) { 656 case BODY: 657 tc->run(m_resfile.str()); 658 break; 659 case CLEANUP: 660 tc->run_cleanup(); 661 break; 662 default: 663 UNREACHABLE; 664 } 665 return EXIT_SUCCESS; 666 } catch (const std::runtime_error& e) { 667 std::cerr << "ERROR: " << e.what() << "\n"; 668 return EXIT_FAILURE; 669 } 670 } 671 672 int 673 tp::main(void) 674 { 675 using atf::application::usage_error; 676 677 int errcode; 678 679 handle_srcdir(); 680 681 if (m_lflag) { 682 if (m_argc > 0) 683 throw usage_error("Cannot provide test case names with -l"); 684 685 list_tcs(); 686 errcode = EXIT_SUCCESS; 687 } else { 688 if (m_argc == 0) 689 throw usage_error("Must provide a test case name"); 690 else if (m_argc > 1) 691 throw usage_error("Cannot provide more than one test case name"); 692 INV(m_argc == 1); 693 694 errcode = run_tc(m_argv[0]); 695 } 696 697 return errcode; 698 } 699 700 namespace atf { 701 namespace tests { 702 int run_tp(int, char* const*, void (*)(tp::tc_vector&)); 703 } 704 } 705 706 int 707 impl::run_tp(int argc, char* const* argv, void (*add_tcs)(tp::tc_vector&)) 708 { 709 return tp(add_tcs).run(argc, argv); 710 } 711