1 // Copyright 2014 The Kyua Authors.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "engine/scanner.hpp"
30
31 #include <cstdarg>
32 #include <cstddef>
33 #include <typeinfo>
34
35 #include <atf-c++.hpp>
36
37 #include "engine/filters.hpp"
38 #include "model/metadata.hpp"
39 #include "model/test_case.hpp"
40 #include "model/test_program.hpp"
41 #include "utils/format/containers.ipp"
42 #include "utils/fs/path.hpp"
43 #include "utils/optional.ipp"
44
45 namespace fs = utils::fs;
46
47 using utils::optional;
48
49
50 namespace {
51
52
53 /// Test program that implements a mock test_cases() lazy call.
54 class mock_test_program : public model::test_program {
55 /// Number of times test_cases has been called.
56 mutable std::size_t _num_calls;
57
58 /// Collection of test cases; lazily initialized.
59 mutable model::test_cases_map _test_cases;
60
61 public:
62 /// Constructs a new test program.
63 ///
64 /// \param binary_ The name of the test program binary relative to root_.
mock_test_program(const fs::path & binary_)65 mock_test_program(const fs::path& binary_) :
66 test_program("unused-interface", binary_, fs::path("unused-root"),
67 "unused-suite", model::metadata_builder().build(),
68 model::test_cases_map()),
69 _num_calls(0)
70 {
71 }
72
73 /// Gets or loads the list of test cases from the test program.
74 ///
75 /// \return The list of test cases provided by the test program.
76 const model::test_cases_map&
test_cases(void) const77 test_cases(void) const
78 {
79 if (_num_calls == 0) {
80 const model::metadata metadata = model::metadata_builder().build();
81 const model::test_case tc1("one", metadata);
82 const model::test_case tc2("two", metadata);
83 _test_cases.insert(model::test_cases_map::value_type("one", tc1));
84 _test_cases.insert(model::test_cases_map::value_type("two", tc2));
85 }
86 _num_calls++;
87 return _test_cases;
88 }
89
90 /// Returns the number of times test_cases() has been called.
91 ///
92 /// \return A counter.
93 std::size_t
num_calls(void) const94 num_calls(void) const
95 {
96 return _num_calls;
97 }
98 };
99
100
101 /// Syntactic sugar to instantiate a test program with various test cases.
102 ///
103 /// The scanner only cares about the relative path of the test program object
104 /// and the names of the test cases. This function helps in instantiating a
105 /// test program that has the minimum set of details only.
106 ///
107 /// \param relative_path Relative path to the test program.
108 /// \param ... List of test case names to add to the test program. Must be
109 /// NULL-terminated.
110 ///
111 /// \return A constructed test program.
112 static model::test_program_ptr
new_test_program(const char * relative_path,...)113 new_test_program(const char* relative_path, ...)
114 {
115 model::test_program_builder builder(
116 "unused-interface", fs::path(relative_path), fs::path("unused-root"),
117 "unused-suite");
118
119 va_list ap;
120 va_start(ap, relative_path);
121 const char* test_case_name;
122 while ((test_case_name = va_arg(ap, const char*)) != NULL) {
123 builder.add_test_case(test_case_name);
124 }
125 va_end(ap);
126
127 return builder.build_ptr();
128 }
129
130
131 /// Yields all test cases in the scanner for simplicity of testing.
132 ///
133 /// In most of the tests below, we just care about the scanner returning the
134 /// full set of matching test cases, not the specific behavior of every single
135 /// yield() call. This function just returns the whole set, which helps in
136 /// writing functional tests.
137 ///
138 /// \param scanner The scanner on which to iterate.
139 ///
140 /// \return The full collection of results yielded by the scanner.
141 static std::set< engine::scan_result >
yield_all(engine::scanner & scanner)142 yield_all(engine::scanner& scanner)
143 {
144 std::set< engine::scan_result > results;
145 while (!scanner.done()) {
146 const optional< engine::scan_result > result = scanner.yield();
147 ATF_REQUIRE(result);
148 results.insert(result.get());
149 }
150 ATF_REQUIRE(!scanner.yield());
151 ATF_REQUIRE(scanner.done());
152 return results;
153 }
154
155
156 } // anonymous namespace
157
158
159 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__no_tests);
ATF_TEST_CASE_BODY(scanner__no_filters__no_tests)160 ATF_TEST_CASE_BODY(scanner__no_filters__no_tests)
161 {
162 const model::test_programs_vector test_programs;
163 const std::set< engine::test_filter > filters;
164
165 engine::scanner scanner(test_programs, filters);
166 ATF_REQUIRE(scanner.done());
167 ATF_REQUIRE(!scanner.yield());
168 ATF_REQUIRE(scanner.unused_filters().empty());
169 }
170
171
172 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__one_test_in_one_program);
ATF_TEST_CASE_BODY(scanner__no_filters__one_test_in_one_program)173 ATF_TEST_CASE_BODY(scanner__no_filters__one_test_in_one_program)
174 {
175 const model::test_program_ptr test_program = new_test_program(
176 "dir/program", "lone_test", NULL);
177
178 model::test_programs_vector test_programs;
179 test_programs.push_back(test_program);
180
181 const std::set< engine::test_filter > filters;
182
183 std::set< engine::scan_result > exp_results;
184 exp_results.insert(engine::scan_result(test_program, "lone_test"));
185
186 engine::scanner scanner(test_programs, filters);
187 const std::set< engine::scan_result > results = yield_all(scanner);
188 ATF_REQUIRE_EQ(exp_results, results);
189 ATF_REQUIRE(scanner.unused_filters().empty());
190 }
191
192
193 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__one_test_per_many_programs);
ATF_TEST_CASE_BODY(scanner__no_filters__one_test_per_many_programs)194 ATF_TEST_CASE_BODY(scanner__no_filters__one_test_per_many_programs)
195 {
196 const model::test_program_ptr test_program1 = new_test_program(
197 "dir/program1", "foo_test", NULL);
198 const model::test_program_ptr test_program2 = new_test_program(
199 "program2", "bar_test", NULL);
200 const model::test_program_ptr test_program3 = new_test_program(
201 "a/b/c/d/e/program3", "baz_test", NULL);
202
203 model::test_programs_vector test_programs;
204 test_programs.push_back(test_program1);
205 test_programs.push_back(test_program2);
206 test_programs.push_back(test_program3);
207
208 const std::set< engine::test_filter > filters;
209
210 std::set< engine::scan_result > exp_results;
211 exp_results.insert(engine::scan_result(test_program1, "foo_test"));
212 exp_results.insert(engine::scan_result(test_program2, "bar_test"));
213 exp_results.insert(engine::scan_result(test_program3, "baz_test"));
214
215 engine::scanner scanner(test_programs, filters);
216 const std::set< engine::scan_result > results = yield_all(scanner);
217 ATF_REQUIRE_EQ(exp_results, results);
218 ATF_REQUIRE(scanner.unused_filters().empty());
219 }
220
221
222 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__many_tests_in_one_program);
ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_in_one_program)223 ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_in_one_program)
224 {
225 const model::test_program_ptr test_program = new_test_program(
226 "dir/program", "first_test", "second_test", "third_test", NULL);
227
228 model::test_programs_vector test_programs;
229 test_programs.push_back(test_program);
230
231 const std::set< engine::test_filter > filters;
232
233 std::set< engine::scan_result > exp_results;
234 exp_results.insert(engine::scan_result(test_program, "first_test"));
235 exp_results.insert(engine::scan_result(test_program, "second_test"));
236 exp_results.insert(engine::scan_result(test_program, "third_test"));
237
238 engine::scanner scanner(test_programs, filters);
239 const std::set< engine::scan_result > results = yield_all(scanner);
240 ATF_REQUIRE_EQ(exp_results, results);
241 ATF_REQUIRE(scanner.unused_filters().empty());
242 }
243
244
245 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__many_tests_per_many_programs);
ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_per_many_programs)246 ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_per_many_programs)
247 {
248 const model::test_program_ptr test_program1 = new_test_program(
249 "dir/program1", "foo_test", "bar_test", "baz_test", NULL);
250 const model::test_program_ptr test_program2 = new_test_program(
251 "program2", "lone_test", NULL);
252 const model::test_program_ptr test_program3 = new_test_program(
253 "a/b/c/d/e/program3", "another_test", "last_test", NULL);
254
255 model::test_programs_vector test_programs;
256 test_programs.push_back(test_program1);
257 test_programs.push_back(test_program2);
258 test_programs.push_back(test_program3);
259
260 const std::set< engine::test_filter > filters;
261
262 std::set< engine::scan_result > exp_results;
263 exp_results.insert(engine::scan_result(test_program1, "foo_test"));
264 exp_results.insert(engine::scan_result(test_program1, "bar_test"));
265 exp_results.insert(engine::scan_result(test_program1, "baz_test"));
266 exp_results.insert(engine::scan_result(test_program2, "lone_test"));
267 exp_results.insert(engine::scan_result(test_program3, "another_test"));
268 exp_results.insert(engine::scan_result(test_program3, "last_test"));
269
270 engine::scanner scanner(test_programs, filters);
271 const std::set< engine::scan_result > results = yield_all(scanner);
272 ATF_REQUIRE_EQ(exp_results, results);
273 ATF_REQUIRE(scanner.unused_filters().empty());
274 }
275
276
277 ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__verify_lazy_loads);
ATF_TEST_CASE_BODY(scanner__no_filters__verify_lazy_loads)278 ATF_TEST_CASE_BODY(scanner__no_filters__verify_lazy_loads)
279 {
280 const model::test_program_ptr test_program1(new mock_test_program(
281 fs::path("first")));
282 const mock_test_program* mock_program1 =
283 dynamic_cast< const mock_test_program* >(test_program1.get());
284 const model::test_program_ptr test_program2(new mock_test_program(
285 fs::path("second")));
286 const mock_test_program* mock_program2 =
287 dynamic_cast< const mock_test_program* >(test_program2.get());
288
289 model::test_programs_vector test_programs;
290 test_programs.push_back(test_program1);
291 test_programs.push_back(test_program2);
292
293 const std::set< engine::test_filter > filters;
294
295 std::set< engine::scan_result > exp_results;
296 exp_results.insert(engine::scan_result(test_program1, "one"));
297 exp_results.insert(engine::scan_result(test_program1, "two"));
298 exp_results.insert(engine::scan_result(test_program2, "one"));
299 exp_results.insert(engine::scan_result(test_program2, "two"));
300
301 engine::scanner scanner(test_programs, filters);
302 std::set< engine::scan_result > results;
303 ATF_REQUIRE_EQ(0, mock_program1->num_calls());
304 ATF_REQUIRE_EQ(0, mock_program2->num_calls());
305
306 // This abuses the internal implementation of the scanner by making
307 // assumptions on the order of the results.
308 results.insert(scanner.yield().get());
309 ATF_REQUIRE_EQ(1, mock_program1->num_calls());
310 ATF_REQUIRE_EQ(0, mock_program2->num_calls());
311 results.insert(scanner.yield().get());
312 ATF_REQUIRE_EQ(1, mock_program1->num_calls());
313 ATF_REQUIRE_EQ(0, mock_program2->num_calls());
314 results.insert(scanner.yield().get());
315 ATF_REQUIRE_EQ(1, mock_program1->num_calls());
316 ATF_REQUIRE_EQ(1, mock_program2->num_calls());
317 results.insert(scanner.yield().get());
318 ATF_REQUIRE_EQ(1, mock_program1->num_calls());
319 ATF_REQUIRE_EQ(1, mock_program2->num_calls());
320 ATF_REQUIRE(scanner.done());
321
322 ATF_REQUIRE_EQ(exp_results, results);
323 ATF_REQUIRE(scanner.unused_filters().empty());
324
325 // Make sure we are still talking to the original objects.
326 for (std::set< engine::scan_result >::const_iterator iter = results.begin();
327 iter != results.end(); ++iter) {
328 const mock_test_program* mock_program =
329 dynamic_cast< const mock_test_program* >((*iter).first.get());
330 ATF_REQUIRE_EQ(1, mock_program->num_calls());
331 }
332 }
333
334
335 ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__no_tests);
ATF_TEST_CASE_BODY(scanner__with_filters__no_tests)336 ATF_TEST_CASE_BODY(scanner__with_filters__no_tests)
337 {
338 const model::test_programs_vector test_programs;
339
340 std::set< engine::test_filter > filters;
341 filters.insert(engine::test_filter(fs::path("foo"), "bar"));
342
343 engine::scanner scanner(test_programs, filters);
344 ATF_REQUIRE(scanner.done());
345 ATF_REQUIRE(!scanner.yield());
346 ATF_REQUIRE_EQ(filters, scanner.unused_filters());
347 }
348
349
350 ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__no_matches);
ATF_TEST_CASE_BODY(scanner__with_filters__no_matches)351 ATF_TEST_CASE_BODY(scanner__with_filters__no_matches)
352 {
353 const model::test_program_ptr test_program1 = new_test_program(
354 "dir/program1", "foo_test", "bar_test", "baz_test", NULL);
355 const model::test_program_ptr test_program2 = new_test_program(
356 "dir/program2", "bar_test", NULL);
357 const model::test_program_ptr test_program3 = new_test_program(
358 "program3", "another_test", "last_test", NULL);
359
360 model::test_programs_vector test_programs;
361 test_programs.push_back(test_program1);
362 test_programs.push_back(test_program2);
363 test_programs.push_back(test_program3);
364
365 std::set< engine::test_filter > filters;
366 filters.insert(engine::test_filter(fs::path("dir/program2"), "baz_test"));
367 filters.insert(engine::test_filter(fs::path("program4"), "another_test"));
368 filters.insert(engine::test_filter(fs::path("dir/program3"), ""));
369
370 const std::set< engine::scan_result > exp_results;
371
372 engine::scanner scanner(test_programs, filters);
373 const std::set< engine::scan_result > results = yield_all(scanner);
374 ATF_REQUIRE_EQ(exp_results, results);
375 ATF_REQUIRE_EQ(filters, scanner.unused_filters());
376 }
377
378
379 ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__some_matches);
ATF_TEST_CASE_BODY(scanner__with_filters__some_matches)380 ATF_TEST_CASE_BODY(scanner__with_filters__some_matches)
381 {
382 const model::test_program_ptr test_program1 = new_test_program(
383 "dir/program1", "foo_test", "bar_test", "baz_test", NULL);
384 const model::test_program_ptr test_program2 = new_test_program(
385 "dir/program2", "bar_test", NULL);
386 const model::test_program_ptr test_program3 = new_test_program(
387 "program3", "another_test", "last_test", NULL);
388 const model::test_program_ptr test_program4 = new_test_program(
389 "program4", "more_test", NULL);
390
391 model::test_programs_vector test_programs;
392 test_programs.push_back(test_program1);
393 test_programs.push_back(test_program2);
394 test_programs.push_back(test_program3);
395 test_programs.push_back(test_program4);
396
397 std::set< engine::test_filter > filters;
398 filters.insert(engine::test_filter(fs::path("dir/program1"), "baz_test"));
399 filters.insert(engine::test_filter(fs::path("dir/program2"), "foo_test"));
400 filters.insert(engine::test_filter(fs::path("program3"), ""));
401
402 std::set< engine::test_filter > exp_filters;
403 exp_filters.insert(engine::test_filter(fs::path("dir/program2"),
404 "foo_test"));
405
406 std::set< engine::scan_result > exp_results;
407 exp_results.insert(engine::scan_result(test_program1, "baz_test"));
408 exp_results.insert(engine::scan_result(test_program3, "another_test"));
409 exp_results.insert(engine::scan_result(test_program3, "last_test"));
410
411 engine::scanner scanner(test_programs, filters);
412 const std::set< engine::scan_result > results = yield_all(scanner);
413 ATF_REQUIRE_EQ(exp_results, results);
414
415 ATF_REQUIRE_EQ(exp_filters, scanner.unused_filters());
416 }
417
418
419 ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__verify_lazy_loads);
ATF_TEST_CASE_BODY(scanner__with_filters__verify_lazy_loads)420 ATF_TEST_CASE_BODY(scanner__with_filters__verify_lazy_loads)
421 {
422 const model::test_program_ptr test_program1(new mock_test_program(
423 fs::path("first")));
424 const mock_test_program* mock_program1 =
425 dynamic_cast< const mock_test_program* >(test_program1.get());
426 const model::test_program_ptr test_program2(new mock_test_program(
427 fs::path("second")));
428 const mock_test_program* mock_program2 =
429 dynamic_cast< const mock_test_program* >(test_program2.get());
430
431 model::test_programs_vector test_programs;
432 test_programs.push_back(test_program1);
433 test_programs.push_back(test_program2);
434
435 std::set< engine::test_filter > filters;
436 filters.insert(engine::test_filter(fs::path("first"), ""));
437
438 std::set< engine::scan_result > exp_results;
439 exp_results.insert(engine::scan_result(test_program1, "one"));
440 exp_results.insert(engine::scan_result(test_program1, "two"));
441
442 engine::scanner scanner(test_programs, filters);
443 std::set< engine::scan_result > results;
444 ATF_REQUIRE_EQ(0, mock_program1->num_calls());
445 ATF_REQUIRE_EQ(0, mock_program2->num_calls());
446
447 results.insert(scanner.yield().get());
448 ATF_REQUIRE_EQ(1, mock_program1->num_calls());
449 ATF_REQUIRE_EQ(0, mock_program2->num_calls());
450 results.insert(scanner.yield().get());
451 ATF_REQUIRE_EQ(1, mock_program1->num_calls());
452 ATF_REQUIRE_EQ(0, mock_program2->num_calls());
453 ATF_REQUIRE(scanner.done());
454
455 ATF_REQUIRE_EQ(exp_results, results);
456 ATF_REQUIRE(scanner.unused_filters().empty());
457
458 ATF_REQUIRE_EQ(1, mock_program1->num_calls());
459 ATF_REQUIRE_EQ(0, mock_program2->num_calls());
460 }
461
462
ATF_INIT_TEST_CASES(tcs)463 ATF_INIT_TEST_CASES(tcs)
464 {
465 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__no_tests);
466 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__one_test_in_one_program);
467 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__one_test_per_many_programs);
468 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__many_tests_in_one_program);
469 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__many_tests_per_many_programs);
470 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__verify_lazy_loads);
471
472 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__no_tests);
473 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__no_matches);
474 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__some_matches);
475 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__verify_lazy_loads);
476 }
477