1 // Copyright 2013 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include <map> 30 31 #include <atf-c++.hpp> 32 33 #include "model/context.hpp" 34 #include "model/metadata.hpp" 35 #include "model/test_program.hpp" 36 #include "model/test_result.hpp" 37 #include "store/migrate.hpp" 38 #include "store/read_backend.hpp" 39 #include "store/read_transaction.hpp" 40 #include "store/write_backend.hpp" 41 #include "utils/datetime.hpp" 42 #include "utils/env.hpp" 43 #include "utils/format/macros.hpp" 44 #include "utils/fs/path.hpp" 45 #include "utils/logging/operations.hpp" 46 #include "utils/sqlite/database.hpp" 47 #include "utils/stream.hpp" 48 #include "utils/units.hpp" 49 50 namespace datetime = utils::datetime; 51 namespace fs = utils::fs; 52 namespace logging = utils::logging; 53 namespace sqlite = utils::sqlite; 54 namespace units = utils::units; 55 56 57 namespace { 58 59 60 /// Gets a data file from the tests directory. 61 /// 62 /// We cannot use the srcdir property because the required files are not there 63 /// when building with an object directory. In those cases, the data files 64 /// remainin the source directory while the resulting test program is in the 65 /// object directory, thus having the wrong value for its srcdir property. 66 /// 67 /// \param name Basename of the test data file to query. 68 /// 69 /// \return The actual path to the requested data file. 70 static fs::path 71 testdata_file(const std::string& name) 72 { 73 const fs::path testdatadir(utils::getenv_with_default( 74 "KYUA_STORETESTDATADIR", KYUA_STORETESTDATADIR)); 75 return testdatadir / name; 76 } 77 78 79 /// Validates the contents of the action with identifier 1. 80 /// 81 /// \param dbpath Path to the database in which to check the action contents. 82 static void 83 check_action_1(const fs::path& dbpath) 84 { 85 store::read_backend backend = store::read_backend::open_ro(dbpath); 86 store::read_transaction transaction = backend.start_read(); 87 88 const fs::path root("/some/root"); 89 std::map< std::string, std::string > environment; 90 const model::context context(root, environment); 91 92 ATF_REQUIRE_EQ(context, transaction.get_context()); 93 94 store::results_iterator iter = transaction.get_results(); 95 ATF_REQUIRE(!iter); 96 } 97 98 99 /// Validates the contents of the action with identifier 2. 100 /// 101 /// \param dbpath Path to the database in which to check the action contents. 102 static void 103 check_action_2(const fs::path& dbpath) 104 { 105 store::read_backend backend = store::read_backend::open_ro(dbpath); 106 store::read_transaction transaction = backend.start_read(); 107 108 const fs::path root("/test/suite/root"); 109 std::map< std::string, std::string > environment; 110 environment["HOME"] = "/home/test"; 111 environment["PATH"] = "/bin:/usr/bin"; 112 const model::context context(root, environment); 113 114 ATF_REQUIRE_EQ(context, transaction.get_context()); 115 116 const model::test_program test_program_1 = model::test_program_builder( 117 "plain", fs::path("foo_test"), fs::path("/test/suite/root"), 118 "suite-name") 119 .add_test_case("main") 120 .build(); 121 const model::test_result result_1(model::test_result_passed); 122 123 const model::test_program test_program_2 = model::test_program_builder( 124 "plain", fs::path("subdir/another_test"), fs::path("/test/suite/root"), 125 "subsuite-name") 126 .add_test_case("main", 127 model::metadata_builder() 128 .set_timeout(datetime::delta(10, 0)) 129 .build()) 130 .set_metadata(model::metadata_builder() 131 .set_timeout(datetime::delta(10, 0)) 132 .build()) 133 .build(); 134 const model::test_result result_2(model::test_result_failed, 135 "Exited with code 1"); 136 137 const model::test_program test_program_3 = model::test_program_builder( 138 "plain", fs::path("subdir/bar_test"), fs::path("/test/suite/root"), 139 "subsuite-name") 140 .add_test_case("main") 141 .build(); 142 const model::test_result result_3(model::test_result_broken, 143 "Received signal 1"); 144 145 const model::test_program test_program_4 = model::test_program_builder( 146 "plain", fs::path("top_test"), fs::path("/test/suite/root"), 147 "suite-name") 148 .add_test_case("main") 149 .build(); 150 const model::test_result result_4(model::test_result_expected_failure, 151 "Known bug"); 152 153 const model::test_program test_program_5 = model::test_program_builder( 154 "plain", fs::path("last_test"), fs::path("/test/suite/root"), 155 "suite-name") 156 .add_test_case("main") 157 .build(); 158 const model::test_result result_5(model::test_result_skipped, 159 "Does not apply"); 160 161 store::results_iterator iter = transaction.get_results(); 162 ATF_REQUIRE(iter); 163 ATF_REQUIRE_EQ(test_program_1, *iter.test_program()); 164 ATF_REQUIRE_EQ("main", iter.test_case_name()); 165 ATF_REQUIRE_EQ(result_1, iter.result()); 166 ATF_REQUIRE(iter.stdout_contents().empty()); 167 ATF_REQUIRE(iter.stderr_contents().empty()); 168 ATF_REQUIRE_EQ(1357643611000000LL, iter.start_time().to_microseconds()); 169 ATF_REQUIRE_EQ(1357643621000500LL, iter.end_time().to_microseconds()); 170 171 ++iter; 172 ATF_REQUIRE(iter); 173 ATF_REQUIRE_EQ(test_program_5, *iter.test_program()); 174 ATF_REQUIRE_EQ("main", iter.test_case_name()); 175 ATF_REQUIRE_EQ(result_5, iter.result()); 176 ATF_REQUIRE(iter.stdout_contents().empty()); 177 ATF_REQUIRE(iter.stderr_contents().empty()); 178 ATF_REQUIRE_EQ(1357643632000000LL, iter.start_time().to_microseconds()); 179 ATF_REQUIRE_EQ(1357643638000000LL, iter.end_time().to_microseconds()); 180 181 ++iter; 182 ATF_REQUIRE(iter); 183 ATF_REQUIRE_EQ(test_program_2, *iter.test_program()); 184 ATF_REQUIRE_EQ("main", iter.test_case_name()); 185 ATF_REQUIRE_EQ(result_2, iter.result()); 186 ATF_REQUIRE_EQ("Test stdout", iter.stdout_contents()); 187 ATF_REQUIRE_EQ("Test stderr", iter.stderr_contents()); 188 ATF_REQUIRE_EQ(1357643622001200LL, iter.start_time().to_microseconds()); 189 ATF_REQUIRE_EQ(1357643622900021LL, iter.end_time().to_microseconds()); 190 191 ++iter; 192 ATF_REQUIRE(iter); 193 ATF_REQUIRE_EQ(test_program_3, *iter.test_program()); 194 ATF_REQUIRE_EQ("main", iter.test_case_name()); 195 ATF_REQUIRE_EQ(result_3, iter.result()); 196 ATF_REQUIRE(iter.stdout_contents().empty()); 197 ATF_REQUIRE(iter.stderr_contents().empty()); 198 ATF_REQUIRE_EQ(1357643623500000LL, iter.start_time().to_microseconds()); 199 ATF_REQUIRE_EQ(1357643630981932LL, iter.end_time().to_microseconds()); 200 201 ++iter; 202 ATF_REQUIRE(iter); 203 ATF_REQUIRE_EQ(test_program_4, *iter.test_program()); 204 ATF_REQUIRE_EQ("main", iter.test_case_name()); 205 ATF_REQUIRE_EQ(result_4, iter.result()); 206 ATF_REQUIRE(iter.stdout_contents().empty()); 207 ATF_REQUIRE(iter.stderr_contents().empty()); 208 ATF_REQUIRE_EQ(1357643631000000LL, iter.start_time().to_microseconds()); 209 ATF_REQUIRE_EQ(1357643631020000LL, iter.end_time().to_microseconds()); 210 211 ++iter; 212 ATF_REQUIRE(!iter); 213 } 214 215 216 /// Validates the contents of the action with identifier 3. 217 /// 218 /// \param dbpath Path to the database in which to check the action contents. 219 static void 220 check_action_3(const fs::path& dbpath) 221 { 222 store::read_backend backend = store::read_backend::open_ro(dbpath); 223 store::read_transaction transaction = backend.start_read(); 224 225 const fs::path root("/usr/tests"); 226 std::map< std::string, std::string > environment; 227 environment["PATH"] = "/bin:/usr/bin"; 228 const model::context context(root, environment); 229 230 ATF_REQUIRE_EQ(context, transaction.get_context()); 231 232 const model::test_program test_program_6 = model::test_program_builder( 233 "atf", fs::path("complex_test"), fs::path("/usr/tests"), 234 "suite-name") 235 .add_test_case("this_passes") 236 .add_test_case("this_fails", 237 model::metadata_builder() 238 .set_description("Test description") 239 .set_has_cleanup(true) 240 .set_required_memory(units::bytes(128)) 241 .set_required_user("root") 242 .build()) 243 .add_test_case("this_skips", 244 model::metadata_builder() 245 .add_allowed_architecture("powerpc") 246 .add_allowed_architecture("x86_64") 247 .add_allowed_platform("amd64") 248 .add_allowed_platform("macppc") 249 .add_required_config("X-foo") 250 .add_required_config("unprivileged_user") 251 .add_required_file(fs::path("/the/data/file")) 252 .add_required_program(fs::path("/bin/ls")) 253 .add_required_program(fs::path("cp")) 254 .set_description("Test explanation") 255 .set_has_cleanup(true) 256 .set_required_memory(units::bytes(512)) 257 .set_required_user("unprivileged") 258 .set_timeout(datetime::delta(600, 0)) 259 .build()) 260 .build(); 261 const model::test_result result_6(model::test_result_passed); 262 const model::test_result result_7(model::test_result_failed, 263 "Some reason"); 264 const model::test_result result_8(model::test_result_skipped, 265 "Another reason"); 266 267 const model::test_program test_program_7 = model::test_program_builder( 268 "atf", fs::path("simple_test"), fs::path("/usr/tests"), 269 "subsuite-name") 270 .add_test_case("main", 271 model::metadata_builder() 272 .set_description("More text") 273 .set_has_cleanup(true) 274 .set_required_memory(units::bytes(128)) 275 .set_required_user("unprivileged") 276 .build()) 277 .build(); 278 const model::test_result result_9(model::test_result_failed, 279 "Exited with code 1"); 280 281 store::results_iterator iter = transaction.get_results(); 282 ATF_REQUIRE(iter); 283 ATF_REQUIRE_EQ(test_program_6, *iter.test_program()); 284 ATF_REQUIRE_EQ("this_fails", iter.test_case_name()); 285 ATF_REQUIRE_EQ(result_7, iter.result()); 286 ATF_REQUIRE(iter.stdout_contents().empty()); 287 ATF_REQUIRE(iter.stderr_contents().empty()); 288 ATF_REQUIRE_EQ(1357648719000000LL, iter.start_time().to_microseconds()); 289 ATF_REQUIRE_EQ(1357648720897182LL, iter.end_time().to_microseconds()); 290 291 ++iter; 292 ATF_REQUIRE(iter); 293 ATF_REQUIRE_EQ(test_program_6, *iter.test_program()); 294 ATF_REQUIRE_EQ("this_passes", iter.test_case_name()); 295 ATF_REQUIRE_EQ(result_6, iter.result()); 296 ATF_REQUIRE(iter.stdout_contents().empty()); 297 ATF_REQUIRE(iter.stderr_contents().empty()); 298 ATF_REQUIRE_EQ(1357648712000000LL, iter.start_time().to_microseconds()); 299 ATF_REQUIRE_EQ(1357648718000000LL, iter.end_time().to_microseconds()); 300 301 ++iter; 302 ATF_REQUIRE(iter); 303 ATF_REQUIRE_EQ(test_program_6, *iter.test_program()); 304 ATF_REQUIRE_EQ("this_skips", iter.test_case_name()); 305 ATF_REQUIRE_EQ(result_8, iter.result()); 306 ATF_REQUIRE_EQ("Another stdout", iter.stdout_contents()); 307 ATF_REQUIRE(iter.stderr_contents().empty()); 308 ATF_REQUIRE_EQ(1357648729182013LL, iter.start_time().to_microseconds()); 309 ATF_REQUIRE_EQ(1357648730000000LL, iter.end_time().to_microseconds()); 310 311 ++iter; 312 ATF_REQUIRE(iter); 313 ATF_REQUIRE_EQ(test_program_7, *iter.test_program()); 314 ATF_REQUIRE_EQ("main", iter.test_case_name()); 315 ATF_REQUIRE_EQ(result_9, iter.result()); 316 ATF_REQUIRE(iter.stdout_contents().empty()); 317 ATF_REQUIRE_EQ("Another stderr", iter.stderr_contents()); 318 ATF_REQUIRE_EQ(1357648740120000LL, iter.start_time().to_microseconds()); 319 ATF_REQUIRE_EQ(1357648750081700LL, iter.end_time().to_microseconds()); 320 321 ++iter; 322 ATF_REQUIRE(!iter); 323 } 324 325 326 /// Validates the contents of the action with identifier 4. 327 /// 328 /// \param dbpath Path to the database in which to check the action contents. 329 static void 330 check_action_4(const fs::path& dbpath) 331 { 332 store::read_backend backend = store::read_backend::open_ro(dbpath); 333 store::read_transaction transaction = backend.start_read(); 334 335 const fs::path root("/usr/tests"); 336 std::map< std::string, std::string > environment; 337 environment["LANG"] = "C"; 338 environment["PATH"] = "/bin:/usr/bin"; 339 environment["TERM"] = "xterm"; 340 const model::context context(root, environment); 341 342 ATF_REQUIRE_EQ(context, transaction.get_context()); 343 344 const model::test_program test_program_8 = model::test_program_builder( 345 "plain", fs::path("subdir/another_test"), fs::path("/usr/tests"), 346 "subsuite-name") 347 .add_test_case("main", 348 model::metadata_builder() 349 .set_timeout(datetime::delta(10, 0)) 350 .build()) 351 .set_metadata(model::metadata_builder() 352 .set_timeout(datetime::delta(10, 0)) 353 .build()) 354 .build(); 355 const model::test_result result_10(model::test_result_failed, 356 "Exit failure"); 357 358 const model::test_program test_program_9 = model::test_program_builder( 359 "atf", fs::path("complex_test"), fs::path("/usr/tests"), 360 "suite-name") 361 .add_test_case("this_passes") 362 .add_test_case("this_fails", 363 model::metadata_builder() 364 .set_description("Test description") 365 .set_required_user("root") 366 .build()) 367 .build(); 368 const model::test_result result_11(model::test_result_passed); 369 const model::test_result result_12(model::test_result_failed, 370 "Some reason"); 371 372 store::results_iterator iter = transaction.get_results(); 373 ATF_REQUIRE(iter); 374 ATF_REQUIRE_EQ(test_program_9, *iter.test_program()); 375 ATF_REQUIRE_EQ("this_fails", iter.test_case_name()); 376 ATF_REQUIRE_EQ(result_12, iter.result()); 377 ATF_REQUIRE(iter.stdout_contents().empty()); 378 ATF_REQUIRE(iter.stderr_contents().empty()); 379 ATF_REQUIRE_EQ(1357644397100000LL, iter.start_time().to_microseconds()); 380 ATF_REQUIRE_EQ(1357644399005000LL, iter.end_time().to_microseconds()); 381 382 ++iter; 383 ATF_REQUIRE(iter); 384 ATF_REQUIRE_EQ(test_program_9, *iter.test_program()); 385 ATF_REQUIRE_EQ("this_passes", iter.test_case_name()); 386 ATF_REQUIRE_EQ(result_11, iter.result()); 387 ATF_REQUIRE(iter.stdout_contents().empty()); 388 ATF_REQUIRE(iter.stderr_contents().empty()); 389 ATF_REQUIRE_EQ(1357644396500000LL, iter.start_time().to_microseconds()); 390 ATF_REQUIRE_EQ(1357644397000000LL, iter.end_time().to_microseconds()); 391 392 ++iter; 393 ATF_REQUIRE(iter); 394 ATF_REQUIRE_EQ(test_program_8, *iter.test_program()); 395 ATF_REQUIRE_EQ("main", iter.test_case_name()); 396 ATF_REQUIRE_EQ(result_10, iter.result()); 397 ATF_REQUIRE_EQ("Test stdout", iter.stdout_contents()); 398 ATF_REQUIRE_EQ("Test stderr", iter.stderr_contents()); 399 ATF_REQUIRE_EQ(1357644395000000LL, iter.start_time().to_microseconds()); 400 ATF_REQUIRE_EQ(1357644396000000LL, iter.end_time().to_microseconds()); 401 402 ++iter; 403 ATF_REQUIRE(!iter); 404 } 405 406 407 } // anonymous namespace 408 409 410 #define CURRENT_SCHEMA_TEST(dataset) \ 411 ATF_TEST_CASE(current_schema_ ##dataset); \ 412 ATF_TEST_CASE_HEAD(current_schema_ ##dataset) \ 413 { \ 414 logging::set_inmemory(); \ 415 const std::string required_files = \ 416 store::detail::schema_file().str() + " " + \ 417 testdata_file("testdata_v3_" #dataset ".sql").str(); \ 418 set_md_var("require.files", required_files); \ 419 } \ 420 ATF_TEST_CASE_BODY(current_schema_ ##dataset) \ 421 { \ 422 const fs::path testpath("test.db"); \ 423 \ 424 sqlite::database db = sqlite::database::open( \ 425 testpath, sqlite::open_readwrite | sqlite::open_create); \ 426 db.exec(utils::read_file(store::detail::schema_file())); \ 427 db.exec(utils::read_file(testdata_file(\ 428 "testdata_v3_" #dataset ".sql"))); \ 429 db.close(); \ 430 \ 431 check_action_ ## dataset (testpath); \ 432 } 433 CURRENT_SCHEMA_TEST(1); 434 CURRENT_SCHEMA_TEST(2); 435 CURRENT_SCHEMA_TEST(3); 436 CURRENT_SCHEMA_TEST(4); 437 438 439 #define MIGRATE_SCHEMA_TEST(from_version) \ 440 ATF_TEST_CASE(migrate_schema__from_v ##from_version); \ 441 ATF_TEST_CASE_HEAD(migrate_schema__from_v ##from_version) \ 442 { \ 443 logging::set_inmemory(); \ 444 \ 445 const char* schema = "schema_v" #from_version ".sql"; \ 446 const char* testdata = "testdata_v" #from_version ".sql"; \ 447 \ 448 std::string required_files = \ 449 testdata_file(schema).str() + " " + testdata_file(testdata).str(); \ 450 for (int i = from_version; i < store::detail::current_schema_version; \ 451 ++i) \ 452 required_files += " " + store::detail::migration_file( \ 453 i, i + 1).str(); \ 454 \ 455 set_md_var("require.files", required_files); \ 456 } \ 457 ATF_TEST_CASE_BODY(migrate_schema__from_v ##from_version) \ 458 { \ 459 const char* schema = "schema_v" #from_version ".sql"; \ 460 const char* testdata = "testdata_v" #from_version ".sql"; \ 461 \ 462 const fs::path testpath("test.db"); \ 463 \ 464 sqlite::database db = sqlite::database::open( \ 465 testpath, sqlite::open_readwrite | sqlite::open_create); \ 466 db.exec(utils::read_file(testdata_file(schema))); \ 467 db.exec(utils::read_file(testdata_file(testdata))); \ 468 db.close(); \ 469 \ 470 store::migrate_schema(fs::path("test.db")); \ 471 \ 472 check_action_2(fs::path(".kyua/store/" \ 473 "results.test_suite_root.20130108-111331-000000.db")); \ 474 check_action_3(fs::path(".kyua/store/" \ 475 "results.usr_tests.20130108-123832-000000.db")); \ 476 check_action_4(fs::path(".kyua/store/" \ 477 "results.usr_tests.20130108-112635-000000.db")); \ 478 } 479 MIGRATE_SCHEMA_TEST(1); 480 MIGRATE_SCHEMA_TEST(2); 481 482 483 ATF_INIT_TEST_CASES(tcs) 484 { 485 ATF_ADD_TEST_CASE(tcs, current_schema_1); 486 ATF_ADD_TEST_CASE(tcs, current_schema_2); 487 ATF_ADD_TEST_CASE(tcs, current_schema_3); 488 ATF_ADD_TEST_CASE(tcs, current_schema_4); 489 490 ATF_ADD_TEST_CASE(tcs, migrate_schema__from_v1); 491 ATF_ADD_TEST_CASE(tcs, migrate_schema__from_v2); 492 } 493