1 // Copyright 2013 The Kyua Authors.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include <map>
30
31 #include <atf-c++.hpp>
32
33 #include "model/context.hpp"
34 #include "model/metadata.hpp"
35 #include "model/test_program.hpp"
36 #include "model/test_result.hpp"
37 #include "store/migrate.hpp"
38 #include "store/read_backend.hpp"
39 #include "store/read_transaction.hpp"
40 #include "store/write_backend.hpp"
41 #include "utils/datetime.hpp"
42 #include "utils/env.hpp"
43 #include "utils/format/macros.hpp"
44 #include "utils/fs/path.hpp"
45 #include "utils/logging/operations.hpp"
46 #include "utils/sqlite/database.hpp"
47 #include "utils/stream.hpp"
48 #include "utils/units.hpp"
49
50 namespace datetime = utils::datetime;
51 namespace fs = utils::fs;
52 namespace logging = utils::logging;
53 namespace sqlite = utils::sqlite;
54 namespace units = utils::units;
55
56
57 namespace {
58
59
60 /// Gets a data file from the tests directory.
61 ///
62 /// We cannot use the srcdir property because the required files are not there
63 /// when building with an object directory. In those cases, the data files
64 /// remainin the source directory while the resulting test program is in the
65 /// object directory, thus having the wrong value for its srcdir property.
66 ///
67 /// \param name Basename of the test data file to query.
68 ///
69 /// \return The actual path to the requested data file.
70 static fs::path
testdata_file(const std::string & name)71 testdata_file(const std::string& name)
72 {
73 const fs::path testdatadir(utils::getenv_with_default(
74 "KYUA_STORETESTDATADIR", KYUA_STORETESTDATADIR));
75 return testdatadir / name;
76 }
77
78
79 /// Validates the contents of the action with identifier 1.
80 ///
81 /// \param dbpath Path to the database in which to check the action contents.
82 static void
check_action_1(const fs::path & dbpath)83 check_action_1(const fs::path& dbpath)
84 {
85 store::read_backend backend = store::read_backend::open_ro(dbpath);
86 store::read_transaction transaction = backend.start_read();
87
88 const fs::path root("/some/root");
89 std::map< std::string, std::string > environment;
90 const model::context context(root, environment);
91
92 ATF_REQUIRE_EQ(context, transaction.get_context());
93
94 store::results_iterator iter = transaction.get_results();
95 ATF_REQUIRE(!iter);
96 }
97
98
99 /// Validates the contents of the action with identifier 2.
100 ///
101 /// \param dbpath Path to the database in which to check the action contents.
102 static void
check_action_2(const fs::path & dbpath)103 check_action_2(const fs::path& dbpath)
104 {
105 store::read_backend backend = store::read_backend::open_ro(dbpath);
106 store::read_transaction transaction = backend.start_read();
107
108 const fs::path root("/test/suite/root");
109 std::map< std::string, std::string > environment;
110 environment["HOME"] = "/home/test";
111 environment["PATH"] = "/bin:/usr/bin";
112 const model::context context(root, environment);
113
114 ATF_REQUIRE_EQ(context, transaction.get_context());
115
116 const model::test_program test_program_1 = model::test_program_builder(
117 "plain", fs::path("foo_test"), fs::path("/test/suite/root"),
118 "suite-name")
119 .add_test_case("main")
120 .build();
121 const model::test_result result_1(model::test_result_passed);
122
123 const model::test_program test_program_2 = model::test_program_builder(
124 "plain", fs::path("subdir/another_test"), fs::path("/test/suite/root"),
125 "subsuite-name")
126 .add_test_case("main",
127 model::metadata_builder()
128 .set_timeout(datetime::delta(10, 0))
129 .build())
130 .set_metadata(model::metadata_builder()
131 .set_timeout(datetime::delta(10, 0))
132 .build())
133 .build();
134 const model::test_result result_2(model::test_result_failed,
135 "Exited with code 1");
136
137 const model::test_program test_program_3 = model::test_program_builder(
138 "plain", fs::path("subdir/bar_test"), fs::path("/test/suite/root"),
139 "subsuite-name")
140 .add_test_case("main")
141 .build();
142 const model::test_result result_3(model::test_result_broken,
143 "Received signal 1");
144
145 const model::test_program test_program_4 = model::test_program_builder(
146 "plain", fs::path("top_test"), fs::path("/test/suite/root"),
147 "suite-name")
148 .add_test_case("main")
149 .build();
150 const model::test_result result_4(model::test_result_expected_failure,
151 "Known bug");
152
153 const model::test_program test_program_5 = model::test_program_builder(
154 "plain", fs::path("last_test"), fs::path("/test/suite/root"),
155 "suite-name")
156 .add_test_case("main")
157 .build();
158 const model::test_result result_5(model::test_result_skipped,
159 "Does not apply");
160
161 store::results_iterator iter = transaction.get_results();
162 ATF_REQUIRE(iter);
163 ATF_REQUIRE_EQ(test_program_1, *iter.test_program());
164 ATF_REQUIRE_EQ("main", iter.test_case_name());
165 ATF_REQUIRE_EQ(result_1, iter.result());
166 ATF_REQUIRE(iter.stdout_contents().empty());
167 ATF_REQUIRE(iter.stderr_contents().empty());
168 ATF_REQUIRE_EQ(1357643611000000LL, iter.start_time().to_microseconds());
169 ATF_REQUIRE_EQ(1357643621000500LL, iter.end_time().to_microseconds());
170
171 ++iter;
172 ATF_REQUIRE(iter);
173 ATF_REQUIRE_EQ(test_program_5, *iter.test_program());
174 ATF_REQUIRE_EQ("main", iter.test_case_name());
175 ATF_REQUIRE_EQ(result_5, iter.result());
176 ATF_REQUIRE(iter.stdout_contents().empty());
177 ATF_REQUIRE(iter.stderr_contents().empty());
178 ATF_REQUIRE_EQ(1357643632000000LL, iter.start_time().to_microseconds());
179 ATF_REQUIRE_EQ(1357643638000000LL, iter.end_time().to_microseconds());
180
181 ++iter;
182 ATF_REQUIRE(iter);
183 ATF_REQUIRE_EQ(test_program_2, *iter.test_program());
184 ATF_REQUIRE_EQ("main", iter.test_case_name());
185 ATF_REQUIRE_EQ(result_2, iter.result());
186 ATF_REQUIRE_EQ("Test stdout", iter.stdout_contents());
187 ATF_REQUIRE_EQ("Test stderr", iter.stderr_contents());
188 ATF_REQUIRE_EQ(1357643622001200LL, iter.start_time().to_microseconds());
189 ATF_REQUIRE_EQ(1357643622900021LL, iter.end_time().to_microseconds());
190
191 ++iter;
192 ATF_REQUIRE(iter);
193 ATF_REQUIRE_EQ(test_program_3, *iter.test_program());
194 ATF_REQUIRE_EQ("main", iter.test_case_name());
195 ATF_REQUIRE_EQ(result_3, iter.result());
196 ATF_REQUIRE(iter.stdout_contents().empty());
197 ATF_REQUIRE(iter.stderr_contents().empty());
198 ATF_REQUIRE_EQ(1357643623500000LL, iter.start_time().to_microseconds());
199 ATF_REQUIRE_EQ(1357643630981932LL, iter.end_time().to_microseconds());
200
201 ++iter;
202 ATF_REQUIRE(iter);
203 ATF_REQUIRE_EQ(test_program_4, *iter.test_program());
204 ATF_REQUIRE_EQ("main", iter.test_case_name());
205 ATF_REQUIRE_EQ(result_4, iter.result());
206 ATF_REQUIRE(iter.stdout_contents().empty());
207 ATF_REQUIRE(iter.stderr_contents().empty());
208 ATF_REQUIRE_EQ(1357643631000000LL, iter.start_time().to_microseconds());
209 ATF_REQUIRE_EQ(1357643631020000LL, iter.end_time().to_microseconds());
210
211 ++iter;
212 ATF_REQUIRE(!iter);
213 }
214
215
216 /// Validates the contents of the action with identifier 3.
217 ///
218 /// \param dbpath Path to the database in which to check the action contents.
219 static void
check_action_3(const fs::path & dbpath)220 check_action_3(const fs::path& dbpath)
221 {
222 store::read_backend backend = store::read_backend::open_ro(dbpath);
223 store::read_transaction transaction = backend.start_read();
224
225 const fs::path root("/usr/tests");
226 std::map< std::string, std::string > environment;
227 environment["PATH"] = "/bin:/usr/bin";
228 const model::context context(root, environment);
229
230 ATF_REQUIRE_EQ(context, transaction.get_context());
231
232 const model::test_program test_program_6 = model::test_program_builder(
233 "atf", fs::path("complex_test"), fs::path("/usr/tests"),
234 "suite-name")
235 .add_test_case("this_passes")
236 .add_test_case("this_fails",
237 model::metadata_builder()
238 .set_description("Test description")
239 .set_has_cleanup(true)
240 .set_required_memory(units::bytes(128))
241 .set_required_user("root")
242 .build())
243 .add_test_case("this_skips",
244 model::metadata_builder()
245 .add_allowed_architecture("powerpc")
246 .add_allowed_architecture("x86_64")
247 .add_allowed_platform("amd64")
248 .add_allowed_platform("macppc")
249 .add_required_config("X-foo")
250 .add_required_config("unprivileged_user")
251 .add_required_file(fs::path("/the/data/file"))
252 .add_required_program(fs::path("/bin/ls"))
253 .add_required_program(fs::path("cp"))
254 .set_description("Test explanation")
255 .set_has_cleanup(true)
256 .set_required_memory(units::bytes(512))
257 .set_required_user("unprivileged")
258 .set_timeout(datetime::delta(600, 0))
259 .build())
260 .build();
261 const model::test_result result_6(model::test_result_passed);
262 const model::test_result result_7(model::test_result_failed,
263 "Some reason");
264 const model::test_result result_8(model::test_result_skipped,
265 "Another reason");
266
267 const model::test_program test_program_7 = model::test_program_builder(
268 "atf", fs::path("simple_test"), fs::path("/usr/tests"),
269 "subsuite-name")
270 .add_test_case("main",
271 model::metadata_builder()
272 .set_description("More text")
273 .set_has_cleanup(true)
274 .set_required_memory(units::bytes(128))
275 .set_required_user("unprivileged")
276 .build())
277 .build();
278 const model::test_result result_9(model::test_result_failed,
279 "Exited with code 1");
280
281 store::results_iterator iter = transaction.get_results();
282 ATF_REQUIRE(iter);
283 ATF_REQUIRE_EQ(test_program_6, *iter.test_program());
284 ATF_REQUIRE_EQ("this_fails", iter.test_case_name());
285 ATF_REQUIRE_EQ(result_7, iter.result());
286 ATF_REQUIRE(iter.stdout_contents().empty());
287 ATF_REQUIRE(iter.stderr_contents().empty());
288 ATF_REQUIRE_EQ(1357648719000000LL, iter.start_time().to_microseconds());
289 ATF_REQUIRE_EQ(1357648720897182LL, iter.end_time().to_microseconds());
290
291 ++iter;
292 ATF_REQUIRE(iter);
293 ATF_REQUIRE_EQ(test_program_6, *iter.test_program());
294 ATF_REQUIRE_EQ("this_passes", iter.test_case_name());
295 ATF_REQUIRE_EQ(result_6, iter.result());
296 ATF_REQUIRE(iter.stdout_contents().empty());
297 ATF_REQUIRE(iter.stderr_contents().empty());
298 ATF_REQUIRE_EQ(1357648712000000LL, iter.start_time().to_microseconds());
299 ATF_REQUIRE_EQ(1357648718000000LL, iter.end_time().to_microseconds());
300
301 ++iter;
302 ATF_REQUIRE(iter);
303 ATF_REQUIRE_EQ(test_program_6, *iter.test_program());
304 ATF_REQUIRE_EQ("this_skips", iter.test_case_name());
305 ATF_REQUIRE_EQ(result_8, iter.result());
306 ATF_REQUIRE_EQ("Another stdout", iter.stdout_contents());
307 ATF_REQUIRE(iter.stderr_contents().empty());
308 ATF_REQUIRE_EQ(1357648729182013LL, iter.start_time().to_microseconds());
309 ATF_REQUIRE_EQ(1357648730000000LL, iter.end_time().to_microseconds());
310
311 ++iter;
312 ATF_REQUIRE(iter);
313 ATF_REQUIRE_EQ(test_program_7, *iter.test_program());
314 ATF_REQUIRE_EQ("main", iter.test_case_name());
315 ATF_REQUIRE_EQ(result_9, iter.result());
316 ATF_REQUIRE(iter.stdout_contents().empty());
317 ATF_REQUIRE_EQ("Another stderr", iter.stderr_contents());
318 ATF_REQUIRE_EQ(1357648740120000LL, iter.start_time().to_microseconds());
319 ATF_REQUIRE_EQ(1357648750081700LL, iter.end_time().to_microseconds());
320
321 ++iter;
322 ATF_REQUIRE(!iter);
323 }
324
325
326 /// Validates the contents of the action with identifier 4.
327 ///
328 /// \param dbpath Path to the database in which to check the action contents.
329 static void
check_action_4(const fs::path & dbpath)330 check_action_4(const fs::path& dbpath)
331 {
332 store::read_backend backend = store::read_backend::open_ro(dbpath);
333 store::read_transaction transaction = backend.start_read();
334
335 const fs::path root("/usr/tests");
336 std::map< std::string, std::string > environment;
337 environment["LANG"] = "C";
338 environment["PATH"] = "/bin:/usr/bin";
339 environment["TERM"] = "xterm";
340 const model::context context(root, environment);
341
342 ATF_REQUIRE_EQ(context, transaction.get_context());
343
344 const model::test_program test_program_8 = model::test_program_builder(
345 "plain", fs::path("subdir/another_test"), fs::path("/usr/tests"),
346 "subsuite-name")
347 .add_test_case("main",
348 model::metadata_builder()
349 .set_timeout(datetime::delta(10, 0))
350 .build())
351 .set_metadata(model::metadata_builder()
352 .set_timeout(datetime::delta(10, 0))
353 .build())
354 .build();
355 const model::test_result result_10(model::test_result_failed,
356 "Exit failure");
357
358 const model::test_program test_program_9 = model::test_program_builder(
359 "atf", fs::path("complex_test"), fs::path("/usr/tests"),
360 "suite-name")
361 .add_test_case("this_passes")
362 .add_test_case("this_fails",
363 model::metadata_builder()
364 .set_description("Test description")
365 .set_required_user("root")
366 .build())
367 .build();
368 const model::test_result result_11(model::test_result_passed);
369 const model::test_result result_12(model::test_result_failed,
370 "Some reason");
371
372 store::results_iterator iter = transaction.get_results();
373 ATF_REQUIRE(iter);
374 ATF_REQUIRE_EQ(test_program_9, *iter.test_program());
375 ATF_REQUIRE_EQ("this_fails", iter.test_case_name());
376 ATF_REQUIRE_EQ(result_12, iter.result());
377 ATF_REQUIRE(iter.stdout_contents().empty());
378 ATF_REQUIRE(iter.stderr_contents().empty());
379 ATF_REQUIRE_EQ(1357644397100000LL, iter.start_time().to_microseconds());
380 ATF_REQUIRE_EQ(1357644399005000LL, iter.end_time().to_microseconds());
381
382 ++iter;
383 ATF_REQUIRE(iter);
384 ATF_REQUIRE_EQ(test_program_9, *iter.test_program());
385 ATF_REQUIRE_EQ("this_passes", iter.test_case_name());
386 ATF_REQUIRE_EQ(result_11, iter.result());
387 ATF_REQUIRE(iter.stdout_contents().empty());
388 ATF_REQUIRE(iter.stderr_contents().empty());
389 ATF_REQUIRE_EQ(1357644396500000LL, iter.start_time().to_microseconds());
390 ATF_REQUIRE_EQ(1357644397000000LL, iter.end_time().to_microseconds());
391
392 ++iter;
393 ATF_REQUIRE(iter);
394 ATF_REQUIRE_EQ(test_program_8, *iter.test_program());
395 ATF_REQUIRE_EQ("main", iter.test_case_name());
396 ATF_REQUIRE_EQ(result_10, iter.result());
397 ATF_REQUIRE_EQ("Test stdout", iter.stdout_contents());
398 ATF_REQUIRE_EQ("Test stderr", iter.stderr_contents());
399 ATF_REQUIRE_EQ(1357644395000000LL, iter.start_time().to_microseconds());
400 ATF_REQUIRE_EQ(1357644396000000LL, iter.end_time().to_microseconds());
401
402 ++iter;
403 ATF_REQUIRE(!iter);
404 }
405
406
407 } // anonymous namespace
408
409
410 #define CURRENT_SCHEMA_TEST(dataset) \
411 ATF_TEST_CASE(current_schema_ ##dataset); \
412 ATF_TEST_CASE_HEAD(current_schema_ ##dataset) \
413 { \
414 logging::set_inmemory(); \
415 const std::string required_files = \
416 store::detail::schema_file().str() + " " + \
417 testdata_file("testdata_v3_" #dataset ".sql").str(); \
418 set_md_var("require.files", required_files); \
419 } \
420 ATF_TEST_CASE_BODY(current_schema_ ##dataset) \
421 { \
422 const fs::path testpath("test.db"); \
423 \
424 sqlite::database db = sqlite::database::open( \
425 testpath, sqlite::open_readwrite | sqlite::open_create); \
426 db.exec(utils::read_file(store::detail::schema_file())); \
427 db.exec(utils::read_file(testdata_file(\
428 "testdata_v3_" #dataset ".sql"))); \
429 db.close(); \
430 \
431 check_action_ ## dataset (testpath); \
432 }
433 CURRENT_SCHEMA_TEST(1);
434 CURRENT_SCHEMA_TEST(2);
435 CURRENT_SCHEMA_TEST(3);
436 CURRENT_SCHEMA_TEST(4);
437
438
439 #define MIGRATE_SCHEMA_TEST(from_version) \
440 ATF_TEST_CASE(migrate_schema__from_v ##from_version); \
441 ATF_TEST_CASE_HEAD(migrate_schema__from_v ##from_version) \
442 { \
443 logging::set_inmemory(); \
444 \
445 const char* schema = "schema_v" #from_version ".sql"; \
446 const char* testdata = "testdata_v" #from_version ".sql"; \
447 \
448 std::string required_files = \
449 testdata_file(schema).str() + " " + testdata_file(testdata).str(); \
450 for (int i = from_version; i < store::detail::current_schema_version; \
451 ++i) \
452 required_files += " " + store::detail::migration_file( \
453 i, i + 1).str(); \
454 \
455 set_md_var("require.files", required_files); \
456 } \
457 ATF_TEST_CASE_BODY(migrate_schema__from_v ##from_version) \
458 { \
459 const char* schema = "schema_v" #from_version ".sql"; \
460 const char* testdata = "testdata_v" #from_version ".sql"; \
461 \
462 const fs::path testpath("test.db"); \
463 \
464 sqlite::database db = sqlite::database::open( \
465 testpath, sqlite::open_readwrite | sqlite::open_create); \
466 db.exec(utils::read_file(testdata_file(schema))); \
467 db.exec(utils::read_file(testdata_file(testdata))); \
468 db.close(); \
469 \
470 store::migrate_schema(fs::path("test.db")); \
471 \
472 check_action_2(fs::path(".kyua/store/" \
473 "results.test_suite_root.20130108-111331-000000.db")); \
474 check_action_3(fs::path(".kyua/store/" \
475 "results.usr_tests.20130108-123832-000000.db")); \
476 check_action_4(fs::path(".kyua/store/" \
477 "results.usr_tests.20130108-112635-000000.db")); \
478 }
479 MIGRATE_SCHEMA_TEST(1);
480 MIGRATE_SCHEMA_TEST(2);
481
482
ATF_INIT_TEST_CASES(tcs)483 ATF_INIT_TEST_CASES(tcs)
484 {
485 ATF_ADD_TEST_CASE(tcs, current_schema_1);
486 ATF_ADD_TEST_CASE(tcs, current_schema_2);
487 ATF_ADD_TEST_CASE(tcs, current_schema_3);
488 ATF_ADD_TEST_CASE(tcs, current_schema_4);
489
490 ATF_ADD_TEST_CASE(tcs, migrate_schema__from_v1);
491 ATF_ADD_TEST_CASE(tcs, migrate_schema__from_v2);
492 }
493