1 // Copyright 2014 The Kyua Authors.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "drivers/report_junit.hpp"
30
31 #include <sstream>
32 #include <vector>
33
34 #include <atf-c++.hpp>
35
36 #include "drivers/scan_results.hpp"
37 #include "engine/filters.hpp"
38 #include "model/context.hpp"
39 #include "model/metadata.hpp"
40 #include "model/test_case.hpp"
41 #include "model/test_program.hpp"
42 #include "model/test_result.hpp"
43 #include "store/write_backend.hpp"
44 #include "store/write_transaction.hpp"
45 #include "utils/datetime.hpp"
46 #include "utils/format/macros.hpp"
47 #include "utils/fs/path.hpp"
48 #include "utils/optional.ipp"
49 #include "utils/units.hpp"
50
51 namespace datetime = utils::datetime;
52 namespace fs = utils::fs;
53 namespace units = utils::units;
54
55 using utils::none;
56
57
58 namespace {
59
60
61 /// Formatted metadata for a test case with defaults.
62 static const char* const default_metadata =
63 "allowed_architectures is empty\n"
64 "allowed_platforms is empty\n"
65 "description is empty\n"
66 "execenv is empty\n"
67 "execenv_jail_params is empty\n"
68 "has_cleanup = false\n"
69 "is_exclusive = false\n"
70 "required_configs is empty\n"
71 "required_disk_space = 0\n"
72 "required_files is empty\n"
73 "required_memory = 0\n"
74 "required_programs is empty\n"
75 "required_user is empty\n"
76 "timeout = 300\n";
77
78
79 /// Formatted metadata for a test case constructed with the "with_metadata" flag
80 /// set to true in add_tests.
81 static const char* const overriden_metadata =
82 "allowed_architectures is empty\n"
83 "allowed_platforms is empty\n"
84 "description = Textual description\n"
85 "execenv is empty\n"
86 "execenv_jail_params is empty\n"
87 "has_cleanup = false\n"
88 "is_exclusive = false\n"
89 "required_configs is empty\n"
90 "required_disk_space = 0\n"
91 "required_files is empty\n"
92 "required_memory = 0\n"
93 "required_programs is empty\n"
94 "required_user is empty\n"
95 "timeout = 5678\n";
96
97
98 /// Populates the context of the given database.
99 ///
100 /// \param tx Transaction to use for the writes to the database.
101 /// \param env_vars Number of environment variables to add to the context.
102 static void
add_context(store::write_transaction & tx,const std::size_t env_vars)103 add_context(store::write_transaction& tx, const std::size_t env_vars)
104 {
105 std::map< std::string, std::string > env;
106 for (std::size_t i = 0; i < env_vars; i++)
107 env[F("VAR%s") % i] = F("Value %s") % i;
108 const model::context context(fs::path("/root"), env);
109 (void)tx.put_context(context);
110 }
111
112
113 /// Adds a new test program with various test cases to the given database.
114 ///
115 /// \param tx Transaction to use for the writes to the database.
116 /// \param prog Test program name.
117 /// \param results Collection of results for the added test cases. The size of
118 /// this vector indicates the number of tests in the test program.
119 /// \param with_metadata Whether to add metadata overrides to the test cases.
120 /// \param with_output Whether to add stdout/stderr messages to the test cases.
121 static void
add_tests(store::write_transaction & tx,const char * prog,const std::vector<model::test_result> & results,const bool with_metadata,const bool with_output)122 add_tests(store::write_transaction& tx,
123 const char* prog,
124 const std::vector< model::test_result >& results,
125 const bool with_metadata, const bool with_output)
126 {
127 model::test_program_builder test_program_builder(
128 "plain", fs::path(prog), fs::path("/root"), "suite");
129
130 for (std::size_t j = 0; j < results.size(); j++) {
131 model::metadata_builder builder;
132 if (with_metadata) {
133 builder.set_description("Textual description");
134 builder.set_timeout(datetime::delta(5678, 0));
135 }
136 test_program_builder.add_test_case(F("t%s") % j, builder.build());
137 }
138
139 const model::test_program test_program = test_program_builder.build();
140 const int64_t tp_id = tx.put_test_program(test_program);
141
142 for (std::size_t j = 0; j < results.size(); j++) {
143 const int64_t tc_id = tx.put_test_case(test_program, F("t%s") % j,
144 tp_id);
145 const datetime::timestamp start =
146 datetime::timestamp::from_microseconds(0);
147 const datetime::timestamp end =
148 datetime::timestamp::from_microseconds(j * 1000000 + 500000);
149 tx.put_result(results[j], tc_id, start, end);
150
151 if (with_output) {
152 atf::utils::create_file("fake-out", F("stdout file %s") % j);
153 tx.put_test_case_file("__STDOUT__", fs::path("fake-out"), tc_id);
154 atf::utils::create_file("fake-err", F("stderr file %s") % j);
155 tx.put_test_case_file("__STDERR__", fs::path("fake-err"), tc_id);
156 }
157 }
158 }
159
160
161 } // anonymous namespace
162
163
164 ATF_TEST_CASE_WITHOUT_HEAD(junit_classname);
ATF_TEST_CASE_BODY(junit_classname)165 ATF_TEST_CASE_BODY(junit_classname)
166 {
167 const model::test_program test_program = model::test_program_builder(
168 "plain", fs::path("dir1/dir2/program"), fs::path("/root"), "suite")
169 .build();
170
171 ATF_REQUIRE_EQ("dir1.dir2.program", drivers::junit_classname(test_program));
172 }
173
174
175 ATF_TEST_CASE_WITHOUT_HEAD(junit_duration);
ATF_TEST_CASE_BODY(junit_duration)176 ATF_TEST_CASE_BODY(junit_duration)
177 {
178 ATF_REQUIRE_EQ("0.457",
179 drivers::junit_duration(datetime::delta(0, 456700)));
180 ATF_REQUIRE_EQ("3.120",
181 drivers::junit_duration(datetime::delta(3, 120000)));
182 ATF_REQUIRE_EQ("5.000", drivers::junit_duration(datetime::delta(5, 0)));
183 }
184
185
186 ATF_TEST_CASE_WITHOUT_HEAD(junit_metadata__defaults);
ATF_TEST_CASE_BODY(junit_metadata__defaults)187 ATF_TEST_CASE_BODY(junit_metadata__defaults)
188 {
189 const model::metadata metadata = model::metadata_builder().build();
190
191 const std::string expected = std::string()
192 + drivers::junit_metadata_header
193 + default_metadata;
194
195 ATF_REQUIRE_EQ(expected, drivers::junit_metadata(metadata));
196 }
197
198
199 ATF_TEST_CASE_WITHOUT_HEAD(junit_metadata__overrides);
ATF_TEST_CASE_BODY(junit_metadata__overrides)200 ATF_TEST_CASE_BODY(junit_metadata__overrides)
201 {
202 const model::metadata metadata = model::metadata_builder()
203 .add_allowed_architecture("arch1")
204 .add_allowed_platform("platform1")
205 .set_description("This is a test")
206 .set_execenv("jail")
207 .set_execenv_jail_params("vnet")
208 .set_has_cleanup(true)
209 .set_is_exclusive(true)
210 .add_required_config("config1")
211 .set_required_disk_space(units::bytes(456))
212 .add_required_file(fs::path("file1"))
213 .set_required_memory(units::bytes(123))
214 .add_required_program(fs::path("prog1"))
215 .set_required_user("root")
216 .set_timeout(datetime::delta(10, 0))
217 .build();
218
219 const std::string expected = std::string()
220 + drivers::junit_metadata_header
221 + "allowed_architectures = arch1\n"
222 + "allowed_platforms = platform1\n"
223 + "description = This is a test\n"
224 + "execenv = jail\n"
225 + "execenv_jail_params = vnet\n"
226 + "has_cleanup = true\n"
227 + "is_exclusive = true\n"
228 + "required_configs = config1\n"
229 + "required_disk_space = 456\n"
230 + "required_files = file1\n"
231 + "required_memory = 123\n"
232 + "required_programs = prog1\n"
233 + "required_user = root\n"
234 + "timeout = 10\n";
235
236 ATF_REQUIRE_EQ(expected, drivers::junit_metadata(metadata));
237 }
238
239
240 ATF_TEST_CASE_WITHOUT_HEAD(junit_timing);
ATF_TEST_CASE_BODY(junit_timing)241 ATF_TEST_CASE_BODY(junit_timing)
242 {
243 const std::string expected = std::string()
244 + drivers::junit_timing_header +
245 "Start time: 2015-06-12T01:02:35.123456Z\n"
246 "End time: 2016-07-13T18:47:10.000001Z\n"
247 "Duration: 34364674.877s\n";
248
249 const datetime::timestamp start_time =
250 datetime::timestamp::from_values(2015, 6, 12, 1, 2, 35, 123456);
251 const datetime::timestamp end_time =
252 datetime::timestamp::from_values(2016, 7, 13, 18, 47, 10, 1);
253
254 ATF_REQUIRE_EQ(expected, drivers::junit_timing(start_time, end_time));
255 }
256
257
258 ATF_TEST_CASE_WITHOUT_HEAD(report_junit_hooks__minimal);
ATF_TEST_CASE_BODY(report_junit_hooks__minimal)259 ATF_TEST_CASE_BODY(report_junit_hooks__minimal)
260 {
261 store::write_backend backend = store::write_backend::open_rw(
262 fs::path("test.db"));
263 store::write_transaction tx = backend.start_write();
264 add_context(tx, 0);
265 tx.commit();
266 backend.close();
267
268 std::ostringstream output;
269
270 drivers::report_junit_hooks hooks(output);
271 drivers::scan_results::drive(fs::path("test.db"),
272 std::set< engine::test_filter >(),
273 hooks);
274
275 const char* expected =
276 "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"
277 "<testsuite>\n"
278 "<properties>\n"
279 "<property name=\"cwd\" value=\"/root\"/>\n"
280 "</properties>\n"
281 "</testsuite>\n";
282 ATF_REQUIRE_EQ(expected, output.str());
283 }
284
285
286 ATF_TEST_CASE_WITHOUT_HEAD(report_junit_hooks__some_tests);
ATF_TEST_CASE_BODY(report_junit_hooks__some_tests)287 ATF_TEST_CASE_BODY(report_junit_hooks__some_tests)
288 {
289 std::vector< model::test_result > results1;
290 results1.push_back(model::test_result(
291 model::test_result_broken, "Broken"));
292 results1.push_back(model::test_result(
293 model::test_result_expected_failure, "XFail"));
294 results1.push_back(model::test_result(
295 model::test_result_failed, "Failed"));
296 std::vector< model::test_result > results2;
297 results2.push_back(model::test_result(
298 model::test_result_passed));
299 results2.push_back(model::test_result(
300 model::test_result_skipped, "Skipped"));
301
302 store::write_backend backend = store::write_backend::open_rw(
303 fs::path("test.db"));
304 store::write_transaction tx = backend.start_write();
305 add_context(tx, 2);
306 add_tests(tx, "dir/prog-1", results1, false, false);
307 add_tests(tx, "dir/sub/prog-2", results2, true, true);
308 tx.commit();
309 backend.close();
310
311 std::ostringstream output;
312
313 drivers::report_junit_hooks hooks(output);
314 drivers::scan_results::drive(fs::path("test.db"),
315 std::set< engine::test_filter >(),
316 hooks);
317
318 const std::string expected = std::string() +
319 "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"
320 "<testsuite>\n"
321 "<properties>\n"
322 "<property name=\"cwd\" value=\"/root\"/>\n"
323 "<property name=\"env.VAR0\" value=\"Value 0\"/>\n"
324 "<property name=\"env.VAR1\" value=\"Value 1\"/>\n"
325 "</properties>\n"
326
327 "<testcase classname=\"dir.prog-1\" name=\"t0\" time=\"0.500\">\n"
328 "<error message=\"Broken\"/>\n"
329 "<system-err>"
330 + drivers::junit_metadata_header +
331 default_metadata
332 + drivers::junit_timing_header +
333 "Start time: 1970-01-01T00:00:00.000000Z\n"
334 "End time: 1970-01-01T00:00:00.500000Z\n"
335 "Duration: 0.500s\n"
336 + drivers::junit_stderr_header +
337 "<EMPTY>\n"
338 "</system-err>\n"
339 "</testcase>\n"
340
341 "<testcase classname=\"dir.prog-1\" name=\"t1\" time=\"1.500\">\n"
342 "<system-err>"
343 "Expected failure result details\n"
344 "-------------------------------\n"
345 "\n"
346 "XFail\n"
347 "\n"
348 + drivers::junit_metadata_header +
349 default_metadata
350 + drivers::junit_timing_header +
351 "Start time: 1970-01-01T00:00:00.000000Z\n"
352 "End time: 1970-01-01T00:00:01.500000Z\n"
353 "Duration: 1.500s\n"
354 + drivers::junit_stderr_header +
355 "<EMPTY>\n"
356 "</system-err>\n"
357 "</testcase>\n"
358
359 "<testcase classname=\"dir.prog-1\" name=\"t2\" time=\"2.500\">\n"
360 "<failure message=\"Failed\"/>\n"
361 "<system-err>"
362 + drivers::junit_metadata_header +
363 default_metadata
364 + drivers::junit_timing_header +
365 "Start time: 1970-01-01T00:00:00.000000Z\n"
366 "End time: 1970-01-01T00:00:02.500000Z\n"
367 "Duration: 2.500s\n"
368 + drivers::junit_stderr_header +
369 "<EMPTY>\n"
370 "</system-err>\n"
371 "</testcase>\n"
372
373 "<testcase classname=\"dir.sub.prog-2\" name=\"t0\" time=\"0.500\">\n"
374 "<system-out>stdout file 0</system-out>\n"
375 "<system-err>"
376 + drivers::junit_metadata_header +
377 overriden_metadata
378 + drivers::junit_timing_header +
379 "Start time: 1970-01-01T00:00:00.000000Z\n"
380 "End time: 1970-01-01T00:00:00.500000Z\n"
381 "Duration: 0.500s\n"
382 + drivers::junit_stderr_header +
383 "stderr file 0</system-err>\n"
384 "</testcase>\n"
385
386 "<testcase classname=\"dir.sub.prog-2\" name=\"t1\" time=\"1.500\">\n"
387 "<skipped/>\n"
388 "<system-out>stdout file 1</system-out>\n"
389 "<system-err>"
390 "Skipped result details\n"
391 "----------------------\n"
392 "\n"
393 "Skipped\n"
394 "\n"
395 + drivers::junit_metadata_header +
396 overriden_metadata
397 + drivers::junit_timing_header +
398 "Start time: 1970-01-01T00:00:00.000000Z\n"
399 "End time: 1970-01-01T00:00:01.500000Z\n"
400 "Duration: 1.500s\n"
401 + drivers::junit_stderr_header +
402 "stderr file 1</system-err>\n"
403 "</testcase>\n"
404
405 "</testsuite>\n";
406 ATF_REQUIRE_EQ(expected, output.str());
407 }
408
409
ATF_INIT_TEST_CASES(tcs)410 ATF_INIT_TEST_CASES(tcs)
411 {
412 ATF_ADD_TEST_CASE(tcs, junit_classname);
413
414 ATF_ADD_TEST_CASE(tcs, junit_duration);
415
416 ATF_ADD_TEST_CASE(tcs, junit_metadata__defaults);
417 ATF_ADD_TEST_CASE(tcs, junit_metadata__overrides);
418
419 ATF_ADD_TEST_CASE(tcs, junit_timing);
420
421 ATF_ADD_TEST_CASE(tcs, report_junit_hooks__minimal);
422 ATF_ADD_TEST_CASE(tcs, report_junit_hooks__some_tests);
423 }
424