1 // Copyright 2014 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "drivers/report_junit.hpp" 30 31 #include <sstream> 32 #include <vector> 33 34 #include <atf-c++.hpp> 35 36 #include "drivers/scan_results.hpp" 37 #include "engine/filters.hpp" 38 #include "model/context.hpp" 39 #include "model/metadata.hpp" 40 #include "model/test_case.hpp" 41 #include "model/test_program.hpp" 42 #include "model/test_result.hpp" 43 #include "store/write_backend.hpp" 44 #include "store/write_transaction.hpp" 45 #include "utils/datetime.hpp" 46 #include "utils/format/macros.hpp" 47 #include "utils/fs/path.hpp" 48 #include "utils/optional.ipp" 49 #include "utils/units.hpp" 50 51 namespace datetime = utils::datetime; 52 namespace fs = utils::fs; 53 namespace units = utils::units; 54 55 using utils::none; 56 57 58 namespace { 59 60 61 /// Formatted metadata for a test case with defaults. 62 static const char* const default_metadata = 63 "allowed_architectures is empty\n" 64 "allowed_platforms is empty\n" 65 "description is empty\n" 66 "execenv is empty\n" 67 "execenv_jail_params is empty\n" 68 "has_cleanup = false\n" 69 "is_exclusive = false\n" 70 "required_configs is empty\n" 71 "required_disk_space = 0\n" 72 "required_files is empty\n" 73 "required_kmods is empty\n" 74 "required_memory = 0\n" 75 "required_programs is empty\n" 76 "required_user is empty\n" 77 "timeout = 300\n"; 78 79 80 /// Formatted metadata for a test case constructed with the "with_metadata" flag 81 /// set to true in add_tests. 82 static const char* const overriden_metadata = 83 "allowed_architectures is empty\n" 84 "allowed_platforms is empty\n" 85 "description = Textual description\n" 86 "execenv is empty\n" 87 "execenv_jail_params is empty\n" 88 "has_cleanup = false\n" 89 "is_exclusive = false\n" 90 "required_configs is empty\n" 91 "required_disk_space = 0\n" 92 "required_files is empty\n" 93 "required_kmods is empty\n" 94 "required_memory = 0\n" 95 "required_programs is empty\n" 96 "required_user is empty\n" 97 "timeout = 5678\n"; 98 99 100 /// Populates the context of the given database. 101 /// 102 /// \param tx Transaction to use for the writes to the database. 103 /// \param env_vars Number of environment variables to add to the context. 104 static void 105 add_context(store::write_transaction& tx, const std::size_t env_vars) 106 { 107 std::map< std::string, std::string > env; 108 for (std::size_t i = 0; i < env_vars; i++) 109 env[F("VAR%s") % i] = F("Value %s") % i; 110 const model::context context(fs::path("/root"), env); 111 (void)tx.put_context(context); 112 } 113 114 115 /// Adds a new test program with various test cases to the given database. 116 /// 117 /// \param tx Transaction to use for the writes to the database. 118 /// \param prog Test program name. 119 /// \param results Collection of results for the added test cases. The size of 120 /// this vector indicates the number of tests in the test program. 121 /// \param with_metadata Whether to add metadata overrides to the test cases. 122 /// \param with_output Whether to add stdout/stderr messages to the test cases. 123 static void 124 add_tests(store::write_transaction& tx, 125 const char* prog, 126 const std::vector< model::test_result >& results, 127 const bool with_metadata, const bool with_output) 128 { 129 model::test_program_builder test_program_builder( 130 "plain", fs::path(prog), fs::path("/root"), "suite"); 131 132 for (std::size_t j = 0; j < results.size(); j++) { 133 model::metadata_builder builder; 134 if (with_metadata) { 135 builder.set_description("Textual description"); 136 builder.set_timeout(datetime::delta(5678, 0)); 137 } 138 test_program_builder.add_test_case(F("t%s") % j, builder.build()); 139 } 140 141 const model::test_program test_program = test_program_builder.build(); 142 const int64_t tp_id = tx.put_test_program(test_program); 143 144 for (std::size_t j = 0; j < results.size(); j++) { 145 const int64_t tc_id = tx.put_test_case(test_program, F("t%s") % j, 146 tp_id); 147 const datetime::timestamp start = 148 datetime::timestamp::from_microseconds(0); 149 const datetime::timestamp end = 150 datetime::timestamp::from_microseconds(j * 1000000 + 500000); 151 tx.put_result(results[j], tc_id, start, end); 152 153 if (with_output) { 154 atf::utils::create_file("fake-out", F("stdout file %s") % j); 155 tx.put_test_case_file("__STDOUT__", fs::path("fake-out"), tc_id); 156 atf::utils::create_file("fake-err", F("stderr file %s") % j); 157 tx.put_test_case_file("__STDERR__", fs::path("fake-err"), tc_id); 158 } 159 } 160 } 161 162 163 } // anonymous namespace 164 165 166 ATF_TEST_CASE_WITHOUT_HEAD(junit_classname); 167 ATF_TEST_CASE_BODY(junit_classname) 168 { 169 const model::test_program test_program = model::test_program_builder( 170 "plain", fs::path("dir1/dir2/program"), fs::path("/root"), "suite") 171 .build(); 172 173 ATF_REQUIRE_EQ("dir1.dir2.program", drivers::junit_classname(test_program)); 174 } 175 176 177 ATF_TEST_CASE_WITHOUT_HEAD(junit_duration); 178 ATF_TEST_CASE_BODY(junit_duration) 179 { 180 ATF_REQUIRE_EQ("0.457", 181 drivers::junit_duration(datetime::delta(0, 456700))); 182 ATF_REQUIRE_EQ("3.120", 183 drivers::junit_duration(datetime::delta(3, 120000))); 184 ATF_REQUIRE_EQ("5.000", drivers::junit_duration(datetime::delta(5, 0))); 185 } 186 187 188 ATF_TEST_CASE_WITHOUT_HEAD(junit_metadata__defaults); 189 ATF_TEST_CASE_BODY(junit_metadata__defaults) 190 { 191 const model::metadata metadata = model::metadata_builder().build(); 192 193 const std::string expected = std::string() 194 + drivers::junit_metadata_header 195 + default_metadata; 196 197 ATF_REQUIRE_EQ(expected, drivers::junit_metadata(metadata)); 198 } 199 200 201 ATF_TEST_CASE_WITHOUT_HEAD(junit_metadata__overrides); 202 ATF_TEST_CASE_BODY(junit_metadata__overrides) 203 { 204 const model::metadata metadata = model::metadata_builder() 205 .add_allowed_architecture("arch1") 206 .add_allowed_platform("platform1") 207 .set_description("This is a test") 208 .set_execenv("jail") 209 .set_execenv_jail_params("vnet") 210 .set_has_cleanup(true) 211 .set_is_exclusive(true) 212 .add_required_config("config1") 213 .set_required_disk_space(units::bytes(456)) 214 .add_required_file(fs::path("file1")) 215 .set_required_memory(units::bytes(123)) 216 .add_required_program(fs::path("prog1")) 217 .set_required_user("root") 218 .set_timeout(datetime::delta(10, 0)) 219 .build(); 220 221 const std::string expected = std::string() 222 + drivers::junit_metadata_header 223 + "allowed_architectures = arch1\n" 224 + "allowed_platforms = platform1\n" 225 + "description = This is a test\n" 226 + "execenv = jail\n" 227 + "execenv_jail_params = vnet\n" 228 + "has_cleanup = true\n" 229 + "is_exclusive = true\n" 230 + "required_configs = config1\n" 231 + "required_disk_space = 456\n" 232 + "required_files = file1\n" 233 + "required_kmods is empty\n" 234 + "required_memory = 123\n" 235 + "required_programs = prog1\n" 236 + "required_user = root\n" 237 + "timeout = 10\n"; 238 239 ATF_REQUIRE_EQ(expected, drivers::junit_metadata(metadata)); 240 } 241 242 243 ATF_TEST_CASE_WITHOUT_HEAD(junit_timing); 244 ATF_TEST_CASE_BODY(junit_timing) 245 { 246 const std::string expected = std::string() 247 + drivers::junit_timing_header + 248 "Start time: 2015-06-12T01:02:35.123456Z\n" 249 "End time: 2016-07-13T18:47:10.000001Z\n" 250 "Duration: 34364674.877s\n"; 251 252 const datetime::timestamp start_time = 253 datetime::timestamp::from_values(2015, 6, 12, 1, 2, 35, 123456); 254 const datetime::timestamp end_time = 255 datetime::timestamp::from_values(2016, 7, 13, 18, 47, 10, 1); 256 257 ATF_REQUIRE_EQ(expected, drivers::junit_timing(start_time, end_time)); 258 } 259 260 261 ATF_TEST_CASE_WITHOUT_HEAD(report_junit_hooks__minimal); 262 ATF_TEST_CASE_BODY(report_junit_hooks__minimal) 263 { 264 store::write_backend backend = store::write_backend::open_rw( 265 fs::path("test.db")); 266 store::write_transaction tx = backend.start_write(); 267 add_context(tx, 0); 268 tx.commit(); 269 backend.close(); 270 271 std::ostringstream output; 272 273 drivers::report_junit_hooks hooks(output); 274 drivers::scan_results::drive(fs::path("test.db"), 275 std::set< engine::test_filter >(), 276 hooks); 277 278 const char* expected = 279 "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n" 280 "<testsuite>\n" 281 "<properties>\n" 282 "<property name=\"cwd\" value=\"/root\"/>\n" 283 "</properties>\n" 284 "</testsuite>\n"; 285 ATF_REQUIRE_EQ(expected, output.str()); 286 } 287 288 289 ATF_TEST_CASE_WITHOUT_HEAD(report_junit_hooks__some_tests); 290 ATF_TEST_CASE_BODY(report_junit_hooks__some_tests) 291 { 292 std::vector< model::test_result > results1; 293 results1.push_back(model::test_result( 294 model::test_result_broken, "Broken")); 295 results1.push_back(model::test_result( 296 model::test_result_expected_failure, "XFail")); 297 results1.push_back(model::test_result( 298 model::test_result_failed, "Failed")); 299 std::vector< model::test_result > results2; 300 results2.push_back(model::test_result( 301 model::test_result_passed)); 302 results2.push_back(model::test_result( 303 model::test_result_skipped, "Skipped")); 304 305 store::write_backend backend = store::write_backend::open_rw( 306 fs::path("test.db")); 307 store::write_transaction tx = backend.start_write(); 308 add_context(tx, 2); 309 add_tests(tx, "dir/prog-1", results1, false, false); 310 add_tests(tx, "dir/sub/prog-2", results2, true, true); 311 tx.commit(); 312 backend.close(); 313 314 std::ostringstream output; 315 316 drivers::report_junit_hooks hooks(output); 317 drivers::scan_results::drive(fs::path("test.db"), 318 std::set< engine::test_filter >(), 319 hooks); 320 321 const std::string expected = std::string() + 322 "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n" 323 "<testsuite>\n" 324 "<properties>\n" 325 "<property name=\"cwd\" value=\"/root\"/>\n" 326 "<property name=\"env.VAR0\" value=\"Value 0\"/>\n" 327 "<property name=\"env.VAR1\" value=\"Value 1\"/>\n" 328 "</properties>\n" 329 330 "<testcase classname=\"dir.prog-1\" name=\"t0\" time=\"0.500\">\n" 331 "<error message=\"Broken\"/>\n" 332 "<system-err>" 333 + drivers::junit_metadata_header + 334 default_metadata 335 + drivers::junit_timing_header + 336 "Start time: 1970-01-01T00:00:00.000000Z\n" 337 "End time: 1970-01-01T00:00:00.500000Z\n" 338 "Duration: 0.500s\n" 339 + drivers::junit_stderr_header + 340 "<EMPTY>\n" 341 "</system-err>\n" 342 "</testcase>\n" 343 344 "<testcase classname=\"dir.prog-1\" name=\"t1\" time=\"1.500\">\n" 345 "<system-err>" 346 "Expected failure result details\n" 347 "-------------------------------\n" 348 "\n" 349 "XFail\n" 350 "\n" 351 + drivers::junit_metadata_header + 352 default_metadata 353 + drivers::junit_timing_header + 354 "Start time: 1970-01-01T00:00:00.000000Z\n" 355 "End time: 1970-01-01T00:00:01.500000Z\n" 356 "Duration: 1.500s\n" 357 + drivers::junit_stderr_header + 358 "<EMPTY>\n" 359 "</system-err>\n" 360 "</testcase>\n" 361 362 "<testcase classname=\"dir.prog-1\" name=\"t2\" time=\"2.500\">\n" 363 "<failure message=\"Failed\"/>\n" 364 "<system-err>" 365 + drivers::junit_metadata_header + 366 default_metadata 367 + drivers::junit_timing_header + 368 "Start time: 1970-01-01T00:00:00.000000Z\n" 369 "End time: 1970-01-01T00:00:02.500000Z\n" 370 "Duration: 2.500s\n" 371 + drivers::junit_stderr_header + 372 "<EMPTY>\n" 373 "</system-err>\n" 374 "</testcase>\n" 375 376 "<testcase classname=\"dir.sub.prog-2\" name=\"t0\" time=\"0.500\">\n" 377 "<system-out>stdout file 0</system-out>\n" 378 "<system-err>" 379 + drivers::junit_metadata_header + 380 overriden_metadata 381 + drivers::junit_timing_header + 382 "Start time: 1970-01-01T00:00:00.000000Z\n" 383 "End time: 1970-01-01T00:00:00.500000Z\n" 384 "Duration: 0.500s\n" 385 + drivers::junit_stderr_header + 386 "stderr file 0</system-err>\n" 387 "</testcase>\n" 388 389 "<testcase classname=\"dir.sub.prog-2\" name=\"t1\" time=\"1.500\">\n" 390 "<skipped/>\n" 391 "<system-out>stdout file 1</system-out>\n" 392 "<system-err>" 393 "Skipped result details\n" 394 "----------------------\n" 395 "\n" 396 "Skipped\n" 397 "\n" 398 + drivers::junit_metadata_header + 399 overriden_metadata 400 + drivers::junit_timing_header + 401 "Start time: 1970-01-01T00:00:00.000000Z\n" 402 "End time: 1970-01-01T00:00:01.500000Z\n" 403 "Duration: 1.500s\n" 404 + drivers::junit_stderr_header + 405 "stderr file 1</system-err>\n" 406 "</testcase>\n" 407 408 "</testsuite>\n"; 409 ATF_REQUIRE_EQ(expected, output.str()); 410 } 411 412 413 ATF_INIT_TEST_CASES(tcs) 414 { 415 ATF_ADD_TEST_CASE(tcs, junit_classname); 416 417 ATF_ADD_TEST_CASE(tcs, junit_duration); 418 419 ATF_ADD_TEST_CASE(tcs, junit_metadata__defaults); 420 ATF_ADD_TEST_CASE(tcs, junit_metadata__overrides); 421 422 ATF_ADD_TEST_CASE(tcs, junit_timing); 423 424 ATF_ADD_TEST_CASE(tcs, report_junit_hooks__minimal); 425 ATF_ADD_TEST_CASE(tcs, report_junit_hooks__some_tests); 426 } 427