1 // Copyright 2014 The Kyua Authors.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "drivers/report_junit.hpp"
30
31 #include <sstream>
32 #include <vector>
33
34 #include <atf-c++.hpp>
35
36 #include "drivers/scan_results.hpp"
37 #include "engine/filters.hpp"
38 #include "model/context.hpp"
39 #include "model/metadata.hpp"
40 #include "model/test_case.hpp"
41 #include "model/test_program.hpp"
42 #include "model/test_result.hpp"
43 #include "store/write_backend.hpp"
44 #include "store/write_transaction.hpp"
45 #include "utils/datetime.hpp"
46 #include "utils/format/macros.hpp"
47 #include "utils/fs/path.hpp"
48 #include "utils/optional.ipp"
49 #include "utils/units.hpp"
50
51 namespace datetime = utils::datetime;
52 namespace fs = utils::fs;
53 namespace units = utils::units;
54
55 using utils::none;
56
57
58 namespace {
59
60
61 /// Formatted metadata for a test case with defaults.
62 static const char* const default_metadata =
63 "allowed_architectures is empty\n"
64 "allowed_platforms is empty\n"
65 "description is empty\n"
66 "execenv is empty\n"
67 "execenv_jail_params is empty\n"
68 "has_cleanup = false\n"
69 "is_exclusive = false\n"
70 "required_configs is empty\n"
71 "required_disk_space = 0\n"
72 "required_files is empty\n"
73 "required_kmods is empty\n"
74 "required_memory = 0\n"
75 "required_programs is empty\n"
76 "required_user is empty\n"
77 "timeout = 300\n";
78
79
80 /// Formatted metadata for a test case constructed with the "with_metadata" flag
81 /// set to true in add_tests.
82 static const char* const overriden_metadata =
83 "allowed_architectures is empty\n"
84 "allowed_platforms is empty\n"
85 "description = Textual description\n"
86 "execenv is empty\n"
87 "execenv_jail_params is empty\n"
88 "has_cleanup = false\n"
89 "is_exclusive = false\n"
90 "required_configs is empty\n"
91 "required_disk_space = 0\n"
92 "required_files is empty\n"
93 "required_kmods is empty\n"
94 "required_memory = 0\n"
95 "required_programs is empty\n"
96 "required_user is empty\n"
97 "timeout = 5678\n";
98
99
100 /// Populates the context of the given database.
101 ///
102 /// \param tx Transaction to use for the writes to the database.
103 /// \param env_vars Number of environment variables to add to the context.
104 static void
add_context(store::write_transaction & tx,const std::size_t env_vars)105 add_context(store::write_transaction& tx, const std::size_t env_vars)
106 {
107 std::map< std::string, std::string > env;
108 for (std::size_t i = 0; i < env_vars; i++)
109 env[F("VAR%s") % i] = F("Value %s") % i;
110 const model::context context(fs::path("/root"), env);
111 (void)tx.put_context(context);
112 }
113
114
115 /// Adds a new test program with various test cases to the given database.
116 ///
117 /// \param tx Transaction to use for the writes to the database.
118 /// \param prog Test program name.
119 /// \param results Collection of results for the added test cases. The size of
120 /// this vector indicates the number of tests in the test program.
121 /// \param with_metadata Whether to add metadata overrides to the test cases.
122 /// \param with_output Whether to add stdout/stderr messages to the test cases.
123 static void
add_tests(store::write_transaction & tx,const char * prog,const std::vector<model::test_result> & results,const bool with_metadata,const bool with_output)124 add_tests(store::write_transaction& tx,
125 const char* prog,
126 const std::vector< model::test_result >& results,
127 const bool with_metadata, const bool with_output)
128 {
129 model::test_program_builder test_program_builder(
130 "plain", fs::path(prog), fs::path("/root"), "suite");
131
132 for (std::size_t j = 0; j < results.size(); j++) {
133 model::metadata_builder builder;
134 if (with_metadata) {
135 builder.set_description("Textual description");
136 builder.set_timeout(datetime::delta(5678, 0));
137 }
138 test_program_builder.add_test_case(F("t%s") % j, builder.build());
139 }
140
141 const model::test_program test_program = test_program_builder.build();
142 const int64_t tp_id = tx.put_test_program(test_program);
143
144 for (std::size_t j = 0; j < results.size(); j++) {
145 const int64_t tc_id = tx.put_test_case(test_program, F("t%s") % j,
146 tp_id);
147 const datetime::timestamp start =
148 datetime::timestamp::from_microseconds(0);
149 const datetime::timestamp end =
150 datetime::timestamp::from_microseconds(j * 1000000 + 500000);
151 tx.put_result(results[j], tc_id, start, end);
152
153 if (with_output) {
154 atf::utils::create_file("fake-out", F("stdout file %s") % j);
155 tx.put_test_case_file("__STDOUT__", fs::path("fake-out"), tc_id);
156 atf::utils::create_file("fake-err", F("stderr file %s") % j);
157 tx.put_test_case_file("__STDERR__", fs::path("fake-err"), tc_id);
158 }
159 }
160 }
161
162
163 } // anonymous namespace
164
165
166 ATF_TEST_CASE_WITHOUT_HEAD(junit_classname);
ATF_TEST_CASE_BODY(junit_classname)167 ATF_TEST_CASE_BODY(junit_classname)
168 {
169 const model::test_program test_program = model::test_program_builder(
170 "plain", fs::path("dir1/dir2/program"), fs::path("/root"), "suite")
171 .build();
172
173 ATF_REQUIRE_EQ("dir1.dir2.program", drivers::junit_classname(test_program));
174 }
175
176
177 ATF_TEST_CASE_WITHOUT_HEAD(junit_duration);
ATF_TEST_CASE_BODY(junit_duration)178 ATF_TEST_CASE_BODY(junit_duration)
179 {
180 ATF_REQUIRE_EQ("0.457",
181 drivers::junit_duration(datetime::delta(0, 456700)));
182 ATF_REQUIRE_EQ("3.120",
183 drivers::junit_duration(datetime::delta(3, 120000)));
184 ATF_REQUIRE_EQ("5.000", drivers::junit_duration(datetime::delta(5, 0)));
185 }
186
187
188 ATF_TEST_CASE_WITHOUT_HEAD(junit_metadata__defaults);
ATF_TEST_CASE_BODY(junit_metadata__defaults)189 ATF_TEST_CASE_BODY(junit_metadata__defaults)
190 {
191 const model::metadata metadata = model::metadata_builder().build();
192
193 const std::string expected = std::string()
194 + drivers::junit_metadata_header
195 + default_metadata;
196
197 ATF_REQUIRE_EQ(expected, drivers::junit_metadata(metadata));
198 }
199
200
201 ATF_TEST_CASE_WITHOUT_HEAD(junit_metadata__overrides);
ATF_TEST_CASE_BODY(junit_metadata__overrides)202 ATF_TEST_CASE_BODY(junit_metadata__overrides)
203 {
204 const model::metadata metadata = model::metadata_builder()
205 .add_allowed_architecture("arch1")
206 .add_allowed_platform("platform1")
207 .set_description("This is a test")
208 .set_execenv("jail")
209 .set_execenv_jail_params("vnet")
210 .set_has_cleanup(true)
211 .set_is_exclusive(true)
212 .add_required_config("config1")
213 .set_required_disk_space(units::bytes(456))
214 .add_required_file(fs::path("file1"))
215 .set_required_memory(units::bytes(123))
216 .add_required_program(fs::path("prog1"))
217 .set_required_user("root")
218 .set_timeout(datetime::delta(10, 0))
219 .build();
220
221 const std::string expected = std::string()
222 + drivers::junit_metadata_header
223 + "allowed_architectures = arch1\n"
224 + "allowed_platforms = platform1\n"
225 + "description = This is a test\n"
226 + "execenv = jail\n"
227 + "execenv_jail_params = vnet\n"
228 + "has_cleanup = true\n"
229 + "is_exclusive = true\n"
230 + "required_configs = config1\n"
231 + "required_disk_space = 456\n"
232 + "required_files = file1\n"
233 + "required_kmods is empty\n"
234 + "required_memory = 123\n"
235 + "required_programs = prog1\n"
236 + "required_user = root\n"
237 + "timeout = 10\n";
238
239 ATF_REQUIRE_EQ(expected, drivers::junit_metadata(metadata));
240 }
241
242
243 ATF_TEST_CASE_WITHOUT_HEAD(junit_timing);
ATF_TEST_CASE_BODY(junit_timing)244 ATF_TEST_CASE_BODY(junit_timing)
245 {
246 const std::string expected = std::string()
247 + drivers::junit_timing_header +
248 "Start time: 2015-06-12T01:02:35.123456Z\n"
249 "End time: 2016-07-13T18:47:10.000001Z\n"
250 "Duration: 34364674.877s\n";
251
252 const datetime::timestamp start_time =
253 datetime::timestamp::from_values(2015, 6, 12, 1, 2, 35, 123456);
254 const datetime::timestamp end_time =
255 datetime::timestamp::from_values(2016, 7, 13, 18, 47, 10, 1);
256
257 ATF_REQUIRE_EQ(expected, drivers::junit_timing(start_time, end_time));
258 }
259
260
261 ATF_TEST_CASE_WITHOUT_HEAD(report_junit_hooks__minimal);
ATF_TEST_CASE_BODY(report_junit_hooks__minimal)262 ATF_TEST_CASE_BODY(report_junit_hooks__minimal)
263 {
264 store::write_backend backend = store::write_backend::open_rw(
265 fs::path("test.db"));
266 store::write_transaction tx = backend.start_write();
267 add_context(tx, 0);
268 tx.commit();
269 backend.close();
270
271 std::ostringstream output;
272
273 drivers::report_junit_hooks hooks(output);
274 drivers::scan_results::drive(fs::path("test.db"),
275 std::set< engine::test_filter >(),
276 hooks);
277
278 const char* expected =
279 "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"
280 "<testsuite>\n"
281 "<properties>\n"
282 "<property name=\"cwd\" value=\"/root\"/>\n"
283 "</properties>\n"
284 "</testsuite>\n";
285 ATF_REQUIRE_EQ(expected, output.str());
286 }
287
288
289 ATF_TEST_CASE_WITHOUT_HEAD(report_junit_hooks__some_tests);
ATF_TEST_CASE_BODY(report_junit_hooks__some_tests)290 ATF_TEST_CASE_BODY(report_junit_hooks__some_tests)
291 {
292 std::vector< model::test_result > results1;
293 results1.push_back(model::test_result(
294 model::test_result_broken, "Broken"));
295 results1.push_back(model::test_result(
296 model::test_result_expected_failure, "XFail"));
297 results1.push_back(model::test_result(
298 model::test_result_failed, "Failed"));
299 std::vector< model::test_result > results2;
300 results2.push_back(model::test_result(
301 model::test_result_passed));
302 results2.push_back(model::test_result(
303 model::test_result_skipped, "Skipped"));
304
305 store::write_backend backend = store::write_backend::open_rw(
306 fs::path("test.db"));
307 store::write_transaction tx = backend.start_write();
308 add_context(tx, 2);
309 add_tests(tx, "dir/prog-1", results1, false, false);
310 add_tests(tx, "dir/sub/prog-2", results2, true, true);
311 tx.commit();
312 backend.close();
313
314 std::ostringstream output;
315
316 drivers::report_junit_hooks hooks(output);
317 drivers::scan_results::drive(fs::path("test.db"),
318 std::set< engine::test_filter >(),
319 hooks);
320
321 const std::string expected = std::string() +
322 "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"
323 "<testsuite>\n"
324 "<properties>\n"
325 "<property name=\"cwd\" value=\"/root\"/>\n"
326 "<property name=\"env.VAR0\" value=\"Value 0\"/>\n"
327 "<property name=\"env.VAR1\" value=\"Value 1\"/>\n"
328 "</properties>\n"
329
330 "<testcase classname=\"dir.prog-1\" name=\"t0\" time=\"0.500\">\n"
331 "<error message=\"Broken\"/>\n"
332 "<system-err>"
333 + drivers::junit_metadata_header +
334 default_metadata
335 + drivers::junit_timing_header +
336 "Start time: 1970-01-01T00:00:00.000000Z\n"
337 "End time: 1970-01-01T00:00:00.500000Z\n"
338 "Duration: 0.500s\n"
339 + drivers::junit_stderr_header +
340 "<EMPTY>\n"
341 "</system-err>\n"
342 "</testcase>\n"
343
344 "<testcase classname=\"dir.prog-1\" name=\"t1\" time=\"1.500\">\n"
345 "<system-err>"
346 "Expected failure result details\n"
347 "-------------------------------\n"
348 "\n"
349 "XFail\n"
350 "\n"
351 + drivers::junit_metadata_header +
352 default_metadata
353 + drivers::junit_timing_header +
354 "Start time: 1970-01-01T00:00:00.000000Z\n"
355 "End time: 1970-01-01T00:00:01.500000Z\n"
356 "Duration: 1.500s\n"
357 + drivers::junit_stderr_header +
358 "<EMPTY>\n"
359 "</system-err>\n"
360 "</testcase>\n"
361
362 "<testcase classname=\"dir.prog-1\" name=\"t2\" time=\"2.500\">\n"
363 "<failure message=\"Failed\"/>\n"
364 "<system-err>"
365 + drivers::junit_metadata_header +
366 default_metadata
367 + drivers::junit_timing_header +
368 "Start time: 1970-01-01T00:00:00.000000Z\n"
369 "End time: 1970-01-01T00:00:02.500000Z\n"
370 "Duration: 2.500s\n"
371 + drivers::junit_stderr_header +
372 "<EMPTY>\n"
373 "</system-err>\n"
374 "</testcase>\n"
375
376 "<testcase classname=\"dir.sub.prog-2\" name=\"t0\" time=\"0.500\">\n"
377 "<system-out>stdout file 0</system-out>\n"
378 "<system-err>"
379 + drivers::junit_metadata_header +
380 overriden_metadata
381 + drivers::junit_timing_header +
382 "Start time: 1970-01-01T00:00:00.000000Z\n"
383 "End time: 1970-01-01T00:00:00.500000Z\n"
384 "Duration: 0.500s\n"
385 + drivers::junit_stderr_header +
386 "stderr file 0</system-err>\n"
387 "</testcase>\n"
388
389 "<testcase classname=\"dir.sub.prog-2\" name=\"t1\" time=\"1.500\">\n"
390 "<skipped/>\n"
391 "<system-out>stdout file 1</system-out>\n"
392 "<system-err>"
393 "Skipped result details\n"
394 "----------------------\n"
395 "\n"
396 "Skipped\n"
397 "\n"
398 + drivers::junit_metadata_header +
399 overriden_metadata
400 + drivers::junit_timing_header +
401 "Start time: 1970-01-01T00:00:00.000000Z\n"
402 "End time: 1970-01-01T00:00:01.500000Z\n"
403 "Duration: 1.500s\n"
404 + drivers::junit_stderr_header +
405 "stderr file 1</system-err>\n"
406 "</testcase>\n"
407
408 "</testsuite>\n";
409 ATF_REQUIRE_EQ(expected, output.str());
410 }
411
412
ATF_INIT_TEST_CASES(tcs)413 ATF_INIT_TEST_CASES(tcs)
414 {
415 ATF_ADD_TEST_CASE(tcs, junit_classname);
416
417 ATF_ADD_TEST_CASE(tcs, junit_duration);
418
419 ATF_ADD_TEST_CASE(tcs, junit_metadata__defaults);
420 ATF_ADD_TEST_CASE(tcs, junit_metadata__overrides);
421
422 ATF_ADD_TEST_CASE(tcs, junit_timing);
423
424 ATF_ADD_TEST_CASE(tcs, report_junit_hooks__minimal);
425 ATF_ADD_TEST_CASE(tcs, report_junit_hooks__some_tests);
426 }
427