1 // Copyright 2011 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "cli/common.hpp" 30 31 #include <fstream> 32 33 #include <atf-c++.hpp> 34 35 #include "engine/exceptions.hpp" 36 #include "engine/filters.hpp" 37 #include "model/metadata.hpp" 38 #include "model/test_program.hpp" 39 #include "model/test_result.hpp" 40 #include "store/layout.hpp" 41 #include "utils/cmdline/exceptions.hpp" 42 #include "utils/cmdline/globals.hpp" 43 #include "utils/cmdline/options.hpp" 44 #include "utils/cmdline/parser.ipp" 45 #include "utils/cmdline/ui_mock.hpp" 46 #include "utils/datetime.hpp" 47 #include "utils/env.hpp" 48 #include "utils/format/macros.hpp" 49 #include "utils/fs/exceptions.hpp" 50 #include "utils/fs/operations.hpp" 51 #include "utils/fs/path.hpp" 52 #include "utils/optional.ipp" 53 #include "utils/sanity.hpp" 54 55 namespace cmdline = utils::cmdline; 56 namespace config = utils::config; 57 namespace datetime = utils::datetime; 58 namespace fs = utils::fs; 59 namespace layout = store::layout; 60 61 using utils::optional; 62 63 64 namespace { 65 66 67 /// Syntactic sugar to instantiate engine::test_filter objects. 68 /// 69 /// \param test_program Test program. 70 /// \param test_case Test case. 71 /// 72 /// \return A \code test_filter \endcode object, based on \p test_program and 73 /// \p test_case. 74 inline engine::test_filter 75 mkfilter(const char* test_program, const char* test_case) 76 { 77 return engine::test_filter(fs::path(test_program), test_case); 78 } 79 80 81 } // anonymous namespace 82 83 84 ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__default); 85 ATF_TEST_CASE_BODY(build_root_path__default) 86 { 87 std::map< std::string, std::vector< std::string > > options; 88 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 89 90 ATF_REQUIRE(!cli::build_root_path(mock_cmdline)); 91 } 92 93 94 ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__explicit); 95 ATF_TEST_CASE_BODY(build_root_path__explicit) 96 { 97 std::map< std::string, std::vector< std::string > > options; 98 options["build-root"].push_back("/my//path"); 99 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 100 101 ATF_REQUIRE(cli::build_root_path(mock_cmdline)); 102 ATF_REQUIRE_EQ("/my/path", cli::build_root_path(mock_cmdline).get().str()); 103 } 104 105 106 ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__default); 107 ATF_TEST_CASE_BODY(kyuafile_path__default) 108 { 109 std::map< std::string, std::vector< std::string > > options; 110 options["kyuafile"].push_back(cli::kyuafile_option.default_value()); 111 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 112 113 ATF_REQUIRE_EQ(cli::kyuafile_option.default_value(), 114 cli::kyuafile_path(mock_cmdline).str()); 115 } 116 117 118 ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__explicit); 119 ATF_TEST_CASE_BODY(kyuafile_path__explicit) 120 { 121 std::map< std::string, std::vector< std::string > > options; 122 options["kyuafile"].push_back("/my//path"); 123 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 124 125 ATF_REQUIRE_EQ("/my/path", cli::kyuafile_path(mock_cmdline).str()); 126 } 127 128 129 ATF_TEST_CASE_WITHOUT_HEAD(result_types__default); 130 ATF_TEST_CASE_BODY(result_types__default) 131 { 132 std::map< std::string, std::vector< std::string > > options; 133 options["results-filter"].push_back( 134 cli::results_filter_option.default_value()); 135 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 136 137 cli::result_types exp_types; 138 exp_types.push_back(model::test_result_skipped); 139 exp_types.push_back(model::test_result_expected_failure); 140 exp_types.push_back(model::test_result_broken); 141 exp_types.push_back(model::test_result_failed); 142 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 143 } 144 145 146 ATF_TEST_CASE_WITHOUT_HEAD(result_types__empty); 147 ATF_TEST_CASE_BODY(result_types__empty) 148 { 149 std::map< std::string, std::vector< std::string > > options; 150 options["results-filter"].push_back(""); 151 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 152 153 cli::result_types exp_types; 154 exp_types.push_back(model::test_result_passed); 155 exp_types.push_back(model::test_result_skipped); 156 exp_types.push_back(model::test_result_expected_failure); 157 exp_types.push_back(model::test_result_broken); 158 exp_types.push_back(model::test_result_failed); 159 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 160 } 161 162 163 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__all); 164 ATF_TEST_CASE_BODY(result_types__explicit__all) 165 { 166 std::map< std::string, std::vector< std::string > > options; 167 options["results-filter"].push_back("passed,skipped,xfail,broken,failed"); 168 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 169 170 cli::result_types exp_types; 171 exp_types.push_back(model::test_result_passed); 172 exp_types.push_back(model::test_result_skipped); 173 exp_types.push_back(model::test_result_expected_failure); 174 exp_types.push_back(model::test_result_broken); 175 exp_types.push_back(model::test_result_failed); 176 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 177 } 178 179 180 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__some); 181 ATF_TEST_CASE_BODY(result_types__explicit__some) 182 { 183 std::map< std::string, std::vector< std::string > > options; 184 options["results-filter"].push_back("skipped,broken"); 185 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 186 187 cli::result_types exp_types; 188 exp_types.push_back(model::test_result_skipped); 189 exp_types.push_back(model::test_result_broken); 190 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 191 } 192 193 194 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__invalid); 195 ATF_TEST_CASE_BODY(result_types__explicit__invalid) 196 { 197 std::map< std::string, std::vector< std::string > > options; 198 options["results-filter"].push_back("skipped,foo,broken"); 199 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 200 201 ATF_REQUIRE_THROW_RE(std::runtime_error, "Unknown result type 'foo'", 202 cli::get_result_types(mock_cmdline)); 203 } 204 205 206 ATF_TEST_CASE_WITHOUT_HEAD(results_file_create__default__new); 207 ATF_TEST_CASE_BODY(results_file_create__default__new) 208 { 209 std::map< std::string, std::vector< std::string > > options; 210 options["results-file"].push_back( 211 cli::results_file_create_option.default_value()); 212 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 213 214 const fs::path home("homedir"); 215 utils::setenv("HOME", home.str()); 216 217 ATF_REQUIRE_EQ(cli::results_file_create_option.default_value(), 218 cli::results_file_create(mock_cmdline)); 219 ATF_REQUIRE(!fs::exists(home / ".kyua")); 220 } 221 222 223 ATF_TEST_CASE_WITHOUT_HEAD(results_file_create__default__historical); 224 ATF_TEST_CASE_BODY(results_file_create__default__historical) 225 { 226 std::map< std::string, std::vector< std::string > > options; 227 options["results-file"].push_back( 228 cli::results_file_create_option.default_value()); 229 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 230 231 const fs::path home("homedir"); 232 utils::setenv("HOME", home.str()); 233 fs::mkdir_p(fs::path("homedir/.kyua"), 0755); 234 atf::utils::create_file("homedir/.kyua/store.db", "fake store"); 235 236 ATF_REQUIRE_EQ(fs::path("homedir/.kyua/store.db").to_absolute(), 237 fs::path(cli::results_file_create(mock_cmdline))); 238 } 239 240 241 ATF_TEST_CASE_WITHOUT_HEAD(results_file_create__explicit); 242 ATF_TEST_CASE_BODY(results_file_create__explicit) 243 { 244 std::map< std::string, std::vector< std::string > > options; 245 options["results-file"].push_back("/my//path/f.db"); 246 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 247 248 ATF_REQUIRE_EQ("/my//path/f.db", 249 cli::results_file_create(mock_cmdline)); 250 } 251 252 253 ATF_TEST_CASE_WITHOUT_HEAD(results_file_open__default__latest); 254 ATF_TEST_CASE_BODY(results_file_open__default__latest) 255 { 256 std::map< std::string, std::vector< std::string > > options; 257 options["results-file"].push_back( 258 cli::results_file_open_option.default_value()); 259 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 260 261 const fs::path home("homedir"); 262 utils::setenv("HOME", home.str()); 263 264 ATF_REQUIRE_EQ(cli::results_file_open_option.default_value(), 265 cli::results_file_open(mock_cmdline)); 266 ATF_REQUIRE(!fs::exists(home / ".kyua")); 267 } 268 269 270 ATF_TEST_CASE_WITHOUT_HEAD(results_file_open__default__historical); 271 ATF_TEST_CASE_BODY(results_file_open__default__historical) 272 { 273 std::map< std::string, std::vector< std::string > > options; 274 options["results-file"].push_back( 275 cli::results_file_open_option.default_value()); 276 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 277 278 const fs::path home("homedir"); 279 utils::setenv("HOME", home.str()); 280 fs::mkdir_p(fs::path("homedir/.kyua"), 0755); 281 atf::utils::create_file("homedir/.kyua/store.db", "fake store"); 282 283 ATF_REQUIRE_EQ(fs::path("homedir/.kyua/store.db").to_absolute(), 284 fs::path(cli::results_file_open(mock_cmdline))); 285 } 286 287 288 ATF_TEST_CASE_WITHOUT_HEAD(results_file_open__explicit); 289 ATF_TEST_CASE_BODY(results_file_open__explicit) 290 { 291 std::map< std::string, std::vector< std::string > > options; 292 options["results-file"].push_back("/my//path/f.db"); 293 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 294 295 ATF_REQUIRE_EQ("/my//path/f.db", cli::results_file_open(mock_cmdline)); 296 } 297 298 299 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__none); 300 ATF_TEST_CASE_BODY(parse_filters__none) 301 { 302 const cmdline::args_vector args; 303 const std::set< engine::test_filter > filters = cli::parse_filters(args); 304 ATF_REQUIRE(filters.empty()); 305 } 306 307 308 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__ok); 309 ATF_TEST_CASE_BODY(parse_filters__ok) 310 { 311 cmdline::args_vector args; 312 args.push_back("foo"); 313 args.push_back("bar/baz"); 314 args.push_back("other:abc"); 315 args.push_back("other:bcd"); 316 const std::set< engine::test_filter > filters = cli::parse_filters(args); 317 318 std::set< engine::test_filter > exp_filters; 319 exp_filters.insert(mkfilter("foo", "")); 320 exp_filters.insert(mkfilter("bar/baz", "")); 321 exp_filters.insert(mkfilter("other", "abc")); 322 exp_filters.insert(mkfilter("other", "bcd")); 323 324 ATF_REQUIRE(exp_filters == filters); 325 } 326 327 328 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__duplicate); 329 ATF_TEST_CASE_BODY(parse_filters__duplicate) 330 { 331 cmdline::args_vector args; 332 args.push_back("foo/bar//baz"); 333 args.push_back("hello/world:yes"); 334 args.push_back("foo//bar/baz"); 335 ATF_REQUIRE_THROW_RE(cmdline::error, "Duplicate.*'foo/bar/baz'", 336 cli::parse_filters(args)); 337 } 338 339 340 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__nondisjoint); 341 ATF_TEST_CASE_BODY(parse_filters__nondisjoint) 342 { 343 cmdline::args_vector args; 344 args.push_back("foo/bar"); 345 args.push_back("hello/world:yes"); 346 args.push_back("foo/bar:baz"); 347 ATF_REQUIRE_THROW_RE(cmdline::error, "'foo/bar'.*'foo/bar:baz'.*disjoint", 348 cli::parse_filters(args)); 349 } 350 351 352 ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__none); 353 ATF_TEST_CASE_BODY(report_unused_filters__none) 354 { 355 std::set< engine::test_filter > unused; 356 357 cmdline::ui_mock ui; 358 ATF_REQUIRE(!cli::report_unused_filters(unused, &ui)); 359 ATF_REQUIRE(ui.out_log().empty()); 360 ATF_REQUIRE(ui.err_log().empty()); 361 } 362 363 364 ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__some); 365 ATF_TEST_CASE_BODY(report_unused_filters__some) 366 { 367 std::set< engine::test_filter > unused; 368 unused.insert(mkfilter("a/b", "")); 369 unused.insert(mkfilter("hey/d", "yes")); 370 371 cmdline::ui_mock ui; 372 cmdline::init("progname"); 373 ATF_REQUIRE(cli::report_unused_filters(unused, &ui)); 374 ATF_REQUIRE(ui.out_log().empty()); 375 ATF_REQUIRE_EQ(2, ui.err_log().size()); 376 ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'a/b'", 377 ui.err_log())); 378 ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'hey/d:yes'", 379 ui.err_log())); 380 } 381 382 383 ATF_TEST_CASE_WITHOUT_HEAD(format_delta); 384 ATF_TEST_CASE_BODY(format_delta) 385 { 386 ATF_REQUIRE_EQ("0.000s", cli::format_delta(datetime::delta())); 387 ATF_REQUIRE_EQ("0.012s", cli::format_delta(datetime::delta(0, 12300))); 388 ATF_REQUIRE_EQ("0.999s", cli::format_delta(datetime::delta(0, 999000))); 389 ATF_REQUIRE_EQ("51.321s", cli::format_delta(datetime::delta(51, 321000))); 390 } 391 392 393 ATF_TEST_CASE_WITHOUT_HEAD(format_result__no_reason); 394 ATF_TEST_CASE_BODY(format_result__no_reason) 395 { 396 ATF_REQUIRE_EQ("passed", cli::format_result( 397 model::test_result(model::test_result_passed))); 398 ATF_REQUIRE_EQ("failed", cli::format_result( 399 model::test_result(model::test_result_failed))); 400 } 401 402 403 ATF_TEST_CASE_WITHOUT_HEAD(format_result__with_reason); 404 ATF_TEST_CASE_BODY(format_result__with_reason) 405 { 406 ATF_REQUIRE_EQ("broken: Something", cli::format_result( 407 model::test_result(model::test_result_broken, "Something"))); 408 ATF_REQUIRE_EQ("expected_failure: A B C", cli::format_result( 409 model::test_result(model::test_result_expected_failure, "A B C"))); 410 ATF_REQUIRE_EQ("failed: More text", cli::format_result( 411 model::test_result(model::test_result_failed, "More text"))); 412 ATF_REQUIRE_EQ("skipped: Bye", cli::format_result( 413 model::test_result(model::test_result_skipped, "Bye"))); 414 } 415 416 417 ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_case); 418 ATF_TEST_CASE_BODY(format_test_case_id__test_case) 419 { 420 const model::test_program test_program = model::test_program_builder( 421 "mock", fs::path("foo/bar/baz"), fs::path("unused-root"), 422 "unused-suite-name") 423 .add_test_case("abc") 424 .build(); 425 ATF_REQUIRE_EQ("foo/bar/baz:abc", 426 cli::format_test_case_id(test_program, "abc")); 427 } 428 429 430 ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_filter); 431 ATF_TEST_CASE_BODY(format_test_case_id__test_filter) 432 { 433 const engine::test_filter filter(fs::path("foo/bar"), "baz"); 434 ATF_REQUIRE_EQ("foo/bar:baz", cli::format_test_case_id(filter)); 435 } 436 437 438 ATF_TEST_CASE_WITHOUT_HEAD(write_version_header); 439 ATF_TEST_CASE_BODY(write_version_header) 440 { 441 cmdline::ui_mock ui; 442 cli::write_version_header(&ui); 443 ATF_REQUIRE_EQ(1, ui.out_log().size()); 444 ATF_REQUIRE_MATCH("^kyua .*[0-9]+\\.[0-9]+$", ui.out_log()[0]); 445 ATF_REQUIRE(ui.err_log().empty()); 446 } 447 448 449 ATF_INIT_TEST_CASES(tcs) 450 { 451 ATF_ADD_TEST_CASE(tcs, build_root_path__default); 452 ATF_ADD_TEST_CASE(tcs, build_root_path__explicit); 453 454 ATF_ADD_TEST_CASE(tcs, kyuafile_path__default); 455 ATF_ADD_TEST_CASE(tcs, kyuafile_path__explicit); 456 457 ATF_ADD_TEST_CASE(tcs, result_types__default); 458 ATF_ADD_TEST_CASE(tcs, result_types__empty); 459 ATF_ADD_TEST_CASE(tcs, result_types__explicit__all); 460 ATF_ADD_TEST_CASE(tcs, result_types__explicit__some); 461 ATF_ADD_TEST_CASE(tcs, result_types__explicit__invalid); 462 463 ATF_ADD_TEST_CASE(tcs, results_file_create__default__new); 464 ATF_ADD_TEST_CASE(tcs, results_file_create__default__historical); 465 ATF_ADD_TEST_CASE(tcs, results_file_create__explicit); 466 467 ATF_ADD_TEST_CASE(tcs, results_file_open__default__latest); 468 ATF_ADD_TEST_CASE(tcs, results_file_open__default__historical); 469 ATF_ADD_TEST_CASE(tcs, results_file_open__explicit); 470 471 ATF_ADD_TEST_CASE(tcs, parse_filters__none); 472 ATF_ADD_TEST_CASE(tcs, parse_filters__ok); 473 ATF_ADD_TEST_CASE(tcs, parse_filters__duplicate); 474 ATF_ADD_TEST_CASE(tcs, parse_filters__nondisjoint); 475 476 ATF_ADD_TEST_CASE(tcs, report_unused_filters__none); 477 ATF_ADD_TEST_CASE(tcs, report_unused_filters__some); 478 479 ATF_ADD_TEST_CASE(tcs, format_delta); 480 481 ATF_ADD_TEST_CASE(tcs, format_result__no_reason); 482 ATF_ADD_TEST_CASE(tcs, format_result__with_reason); 483 484 ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_case); 485 ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_filter); 486 487 ATF_ADD_TEST_CASE(tcs, write_version_header); 488 } 489