1 // Copyright 2014 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/scanner.hpp" 30 31 #include <cstdarg> 32 #include <cstddef> 33 #include <typeinfo> 34 35 #include <atf-c++.hpp> 36 37 #include "engine/filters.hpp" 38 #include "model/metadata.hpp" 39 #include "model/test_case.hpp" 40 #include "model/test_program.hpp" 41 #include "utils/format/containers.ipp" 42 #include "utils/fs/path.hpp" 43 #include "utils/optional.ipp" 44 45 namespace fs = utils::fs; 46 47 using utils::optional; 48 49 50 namespace { 51 52 53 /// Test program that implements a mock test_cases() lazy call. 54 class mock_test_program : public model::test_program { 55 /// Number of times test_cases has been called. 56 mutable std::size_t _num_calls; 57 58 /// Collection of test cases; lazily initialized. 59 mutable model::test_cases_map _test_cases; 60 61 public: 62 /// Constructs a new test program. 63 /// 64 /// \param binary_ The name of the test program binary relative to root_. 65 mock_test_program(const fs::path& binary_) : 66 test_program("unused-interface", binary_, fs::path("unused-root"), 67 "unused-suite", model::metadata_builder().build(), 68 model::test_cases_map()), 69 _num_calls(0) 70 { 71 } 72 73 /// Gets or loads the list of test cases from the test program. 74 /// 75 /// \return The list of test cases provided by the test program. 76 const model::test_cases_map& 77 test_cases(void) const 78 { 79 if (_num_calls == 0) { 80 const model::metadata metadata = model::metadata_builder().build(); 81 const model::test_case tc1("one", metadata); 82 const model::test_case tc2("two", metadata); 83 _test_cases.insert(model::test_cases_map::value_type("one", tc1)); 84 _test_cases.insert(model::test_cases_map::value_type("two", tc2)); 85 } 86 _num_calls++; 87 return _test_cases; 88 } 89 90 /// Returns the number of times test_cases() has been called. 91 /// 92 /// \return A counter. 93 std::size_t 94 num_calls(void) const 95 { 96 return _num_calls; 97 } 98 }; 99 100 101 /// Syntactic sugar to instantiate a test program with various test cases. 102 /// 103 /// The scanner only cares about the relative path of the test program object 104 /// and the names of the test cases. This function helps in instantiating a 105 /// test program that has the minimum set of details only. 106 /// 107 /// \param relative_path Relative path to the test program. 108 /// \param ... List of test case names to add to the test program. Must be 109 /// NULL-terminated. 110 /// 111 /// \return A constructed test program. 112 static model::test_program_ptr 113 new_test_program(const char* relative_path, ...) 114 { 115 model::test_program_builder builder( 116 "unused-interface", fs::path(relative_path), fs::path("unused-root"), 117 "unused-suite"); 118 119 va_list ap; 120 va_start(ap, relative_path); 121 const char* test_case_name; 122 while ((test_case_name = va_arg(ap, const char*)) != NULL) { 123 builder.add_test_case(test_case_name); 124 } 125 va_end(ap); 126 127 return builder.build_ptr(); 128 } 129 130 131 /// Yields all test cases in the scanner for simplicity of testing. 132 /// 133 /// In most of the tests below, we just care about the scanner returning the 134 /// full set of matching test cases, not the specific behavior of every single 135 /// yield() call. This function just returns the whole set, which helps in 136 /// writing functional tests. 137 /// 138 /// \param scanner The scanner on which to iterate. 139 /// 140 /// \return The full collection of results yielded by the scanner. 141 static std::set< engine::scan_result > 142 yield_all(engine::scanner& scanner) 143 { 144 std::set< engine::scan_result > results; 145 while (!scanner.done()) { 146 const optional< engine::scan_result > result = scanner.yield(); 147 ATF_REQUIRE(result); 148 results.insert(result.get()); 149 } 150 ATF_REQUIRE(!scanner.yield()); 151 ATF_REQUIRE(scanner.done()); 152 return results; 153 } 154 155 156 } // anonymous namespace 157 158 159 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__no_tests); 160 ATF_TEST_CASE_BODY(scanner__no_filters__no_tests) 161 { 162 const model::test_programs_vector test_programs; 163 const std::set< engine::test_filter > filters; 164 165 engine::scanner scanner(test_programs, filters); 166 ATF_REQUIRE(scanner.done()); 167 ATF_REQUIRE(!scanner.yield()); 168 ATF_REQUIRE(scanner.unused_filters().empty()); 169 } 170 171 172 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__one_test_in_one_program); 173 ATF_TEST_CASE_BODY(scanner__no_filters__one_test_in_one_program) 174 { 175 const model::test_program_ptr test_program = new_test_program( 176 "dir/program", "lone_test", NULL); 177 178 model::test_programs_vector test_programs; 179 test_programs.push_back(test_program); 180 181 const std::set< engine::test_filter > filters; 182 183 std::set< engine::scan_result > exp_results; 184 exp_results.insert(engine::scan_result(test_program, "lone_test")); 185 186 engine::scanner scanner(test_programs, filters); 187 const std::set< engine::scan_result > results = yield_all(scanner); 188 ATF_REQUIRE_EQ(exp_results, results); 189 ATF_REQUIRE(scanner.unused_filters().empty()); 190 } 191 192 193 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__one_test_per_many_programs); 194 ATF_TEST_CASE_BODY(scanner__no_filters__one_test_per_many_programs) 195 { 196 const model::test_program_ptr test_program1 = new_test_program( 197 "dir/program1", "foo_test", NULL); 198 const model::test_program_ptr test_program2 = new_test_program( 199 "program2", "bar_test", NULL); 200 const model::test_program_ptr test_program3 = new_test_program( 201 "a/b/c/d/e/program3", "baz_test", NULL); 202 203 model::test_programs_vector test_programs; 204 test_programs.push_back(test_program1); 205 test_programs.push_back(test_program2); 206 test_programs.push_back(test_program3); 207 208 const std::set< engine::test_filter > filters; 209 210 std::set< engine::scan_result > exp_results; 211 exp_results.insert(engine::scan_result(test_program1, "foo_test")); 212 exp_results.insert(engine::scan_result(test_program2, "bar_test")); 213 exp_results.insert(engine::scan_result(test_program3, "baz_test")); 214 215 engine::scanner scanner(test_programs, filters); 216 const std::set< engine::scan_result > results = yield_all(scanner); 217 ATF_REQUIRE_EQ(exp_results, results); 218 ATF_REQUIRE(scanner.unused_filters().empty()); 219 } 220 221 222 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__many_tests_in_one_program); 223 ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_in_one_program) 224 { 225 const model::test_program_ptr test_program = new_test_program( 226 "dir/program", "first_test", "second_test", "third_test", NULL); 227 228 model::test_programs_vector test_programs; 229 test_programs.push_back(test_program); 230 231 const std::set< engine::test_filter > filters; 232 233 std::set< engine::scan_result > exp_results; 234 exp_results.insert(engine::scan_result(test_program, "first_test")); 235 exp_results.insert(engine::scan_result(test_program, "second_test")); 236 exp_results.insert(engine::scan_result(test_program, "third_test")); 237 238 engine::scanner scanner(test_programs, filters); 239 const std::set< engine::scan_result > results = yield_all(scanner); 240 ATF_REQUIRE_EQ(exp_results, results); 241 ATF_REQUIRE(scanner.unused_filters().empty()); 242 } 243 244 245 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__many_tests_per_many_programs); 246 ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_per_many_programs) 247 { 248 const model::test_program_ptr test_program1 = new_test_program( 249 "dir/program1", "foo_test", "bar_test", "baz_test", NULL); 250 const model::test_program_ptr test_program2 = new_test_program( 251 "program2", "lone_test", NULL); 252 const model::test_program_ptr test_program3 = new_test_program( 253 "a/b/c/d/e/program3", "another_test", "last_test", NULL); 254 255 model::test_programs_vector test_programs; 256 test_programs.push_back(test_program1); 257 test_programs.push_back(test_program2); 258 test_programs.push_back(test_program3); 259 260 const std::set< engine::test_filter > filters; 261 262 std::set< engine::scan_result > exp_results; 263 exp_results.insert(engine::scan_result(test_program1, "foo_test")); 264 exp_results.insert(engine::scan_result(test_program1, "bar_test")); 265 exp_results.insert(engine::scan_result(test_program1, "baz_test")); 266 exp_results.insert(engine::scan_result(test_program2, "lone_test")); 267 exp_results.insert(engine::scan_result(test_program3, "another_test")); 268 exp_results.insert(engine::scan_result(test_program3, "last_test")); 269 270 engine::scanner scanner(test_programs, filters); 271 const std::set< engine::scan_result > results = yield_all(scanner); 272 ATF_REQUIRE_EQ(exp_results, results); 273 ATF_REQUIRE(scanner.unused_filters().empty()); 274 } 275 276 277 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__verify_lazy_loads); 278 ATF_TEST_CASE_BODY(scanner__no_filters__verify_lazy_loads) 279 { 280 const model::test_program_ptr test_program1(new mock_test_program( 281 fs::path("first"))); 282 const mock_test_program* mock_program1 = 283 dynamic_cast< const mock_test_program* >(test_program1.get()); 284 const model::test_program_ptr test_program2(new mock_test_program( 285 fs::path("second"))); 286 const mock_test_program* mock_program2 = 287 dynamic_cast< const mock_test_program* >(test_program2.get()); 288 289 model::test_programs_vector test_programs; 290 test_programs.push_back(test_program1); 291 test_programs.push_back(test_program2); 292 293 const std::set< engine::test_filter > filters; 294 295 std::set< engine::scan_result > exp_results; 296 exp_results.insert(engine::scan_result(test_program1, "one")); 297 exp_results.insert(engine::scan_result(test_program1, "two")); 298 exp_results.insert(engine::scan_result(test_program2, "one")); 299 exp_results.insert(engine::scan_result(test_program2, "two")); 300 301 engine::scanner scanner(test_programs, filters); 302 std::set< engine::scan_result > results; 303 ATF_REQUIRE_EQ(0, mock_program1->num_calls()); 304 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 305 306 // This abuses the internal implementation of the scanner by making 307 // assumptions on the order of the results. 308 results.insert(scanner.yield().get()); 309 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 310 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 311 results.insert(scanner.yield().get()); 312 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 313 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 314 results.insert(scanner.yield().get()); 315 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 316 ATF_REQUIRE_EQ(1, mock_program2->num_calls()); 317 results.insert(scanner.yield().get()); 318 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 319 ATF_REQUIRE_EQ(1, mock_program2->num_calls()); 320 ATF_REQUIRE(scanner.done()); 321 322 ATF_REQUIRE_EQ(exp_results, results); 323 ATF_REQUIRE(scanner.unused_filters().empty()); 324 325 // Make sure we are still talking to the original objects. 326 for (std::set< engine::scan_result >::const_iterator iter = results.begin(); 327 iter != results.end(); ++iter) { 328 const mock_test_program* mock_program = 329 dynamic_cast< const mock_test_program* >((*iter).first.get()); 330 ATF_REQUIRE_EQ(1, mock_program->num_calls()); 331 } 332 } 333 334 335 ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__no_tests); 336 ATF_TEST_CASE_BODY(scanner__with_filters__no_tests) 337 { 338 const model::test_programs_vector test_programs; 339 340 std::set< engine::test_filter > filters; 341 filters.insert(engine::test_filter(fs::path("foo"), "bar")); 342 343 engine::scanner scanner(test_programs, filters); 344 ATF_REQUIRE(scanner.done()); 345 ATF_REQUIRE(!scanner.yield()); 346 ATF_REQUIRE_EQ(filters, scanner.unused_filters()); 347 } 348 349 350 ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__no_matches); 351 ATF_TEST_CASE_BODY(scanner__with_filters__no_matches) 352 { 353 const model::test_program_ptr test_program1 = new_test_program( 354 "dir/program1", "foo_test", "bar_test", "baz_test", NULL); 355 const model::test_program_ptr test_program2 = new_test_program( 356 "dir/program2", "bar_test", NULL); 357 const model::test_program_ptr test_program3 = new_test_program( 358 "program3", "another_test", "last_test", NULL); 359 360 model::test_programs_vector test_programs; 361 test_programs.push_back(test_program1); 362 test_programs.push_back(test_program2); 363 test_programs.push_back(test_program3); 364 365 std::set< engine::test_filter > filters; 366 filters.insert(engine::test_filter(fs::path("dir/program2"), "baz_test")); 367 filters.insert(engine::test_filter(fs::path("program4"), "another_test")); 368 filters.insert(engine::test_filter(fs::path("dir/program3"), "")); 369 370 const std::set< engine::scan_result > exp_results; 371 372 engine::scanner scanner(test_programs, filters); 373 const std::set< engine::scan_result > results = yield_all(scanner); 374 ATF_REQUIRE_EQ(exp_results, results); 375 ATF_REQUIRE_EQ(filters, scanner.unused_filters()); 376 } 377 378 379 ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__some_matches); 380 ATF_TEST_CASE_BODY(scanner__with_filters__some_matches) 381 { 382 const model::test_program_ptr test_program1 = new_test_program( 383 "dir/program1", "foo_test", "bar_test", "baz_test", NULL); 384 const model::test_program_ptr test_program2 = new_test_program( 385 "dir/program2", "bar_test", NULL); 386 const model::test_program_ptr test_program3 = new_test_program( 387 "program3", "another_test", "last_test", NULL); 388 const model::test_program_ptr test_program4 = new_test_program( 389 "program4", "more_test", NULL); 390 391 model::test_programs_vector test_programs; 392 test_programs.push_back(test_program1); 393 test_programs.push_back(test_program2); 394 test_programs.push_back(test_program3); 395 test_programs.push_back(test_program4); 396 397 std::set< engine::test_filter > filters; 398 filters.insert(engine::test_filter(fs::path("dir/program1"), "baz_test")); 399 filters.insert(engine::test_filter(fs::path("dir/program2"), "foo_test")); 400 filters.insert(engine::test_filter(fs::path("program3"), "")); 401 402 std::set< engine::test_filter > exp_filters; 403 exp_filters.insert(engine::test_filter(fs::path("dir/program2"), 404 "foo_test")); 405 406 std::set< engine::scan_result > exp_results; 407 exp_results.insert(engine::scan_result(test_program1, "baz_test")); 408 exp_results.insert(engine::scan_result(test_program3, "another_test")); 409 exp_results.insert(engine::scan_result(test_program3, "last_test")); 410 411 engine::scanner scanner(test_programs, filters); 412 const std::set< engine::scan_result > results = yield_all(scanner); 413 ATF_REQUIRE_EQ(exp_results, results); 414 415 ATF_REQUIRE_EQ(exp_filters, scanner.unused_filters()); 416 } 417 418 419 ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__verify_lazy_loads); 420 ATF_TEST_CASE_BODY(scanner__with_filters__verify_lazy_loads) 421 { 422 const model::test_program_ptr test_program1(new mock_test_program( 423 fs::path("first"))); 424 const mock_test_program* mock_program1 = 425 dynamic_cast< const mock_test_program* >(test_program1.get()); 426 const model::test_program_ptr test_program2(new mock_test_program( 427 fs::path("second"))); 428 const mock_test_program* mock_program2 = 429 dynamic_cast< const mock_test_program* >(test_program2.get()); 430 431 model::test_programs_vector test_programs; 432 test_programs.push_back(test_program1); 433 test_programs.push_back(test_program2); 434 435 std::set< engine::test_filter > filters; 436 filters.insert(engine::test_filter(fs::path("first"), "")); 437 438 std::set< engine::scan_result > exp_results; 439 exp_results.insert(engine::scan_result(test_program1, "one")); 440 exp_results.insert(engine::scan_result(test_program1, "two")); 441 442 engine::scanner scanner(test_programs, filters); 443 std::set< engine::scan_result > results; 444 ATF_REQUIRE_EQ(0, mock_program1->num_calls()); 445 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 446 447 results.insert(scanner.yield().get()); 448 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 449 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 450 results.insert(scanner.yield().get()); 451 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 452 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 453 ATF_REQUIRE(scanner.done()); 454 455 ATF_REQUIRE_EQ(exp_results, results); 456 ATF_REQUIRE(scanner.unused_filters().empty()); 457 458 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 459 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 460 } 461 462 463 ATF_INIT_TEST_CASES(tcs) 464 { 465 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__no_tests); 466 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__one_test_in_one_program); 467 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__one_test_per_many_programs); 468 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__many_tests_in_one_program); 469 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__many_tests_per_many_programs); 470 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__verify_lazy_loads); 471 472 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__no_tests); 473 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__no_matches); 474 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__some_matches); 475 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__verify_lazy_loads); 476 } 477