1 // Copyright 2010 The Kyua Authors.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "engine/atf_result.hpp"
30
31 extern "C" {
32 #include <signal.h>
33 }
34
35 #include <cstdlib>
36 #include <fstream>
37 #include <sstream>
38 #include <stdexcept>
39
40 #include <atf-c++.hpp>
41
42 #include "engine/exceptions.hpp"
43 #include "model/test_result.hpp"
44 #include "utils/format/macros.hpp"
45 #include "utils/fs/path.hpp"
46 #include "utils/process/status.hpp"
47
48 namespace fs = utils::fs;
49 namespace process = utils::process;
50
51 using utils::none;
52 using utils::optional;
53
54
55 namespace {
56
57
58 /// Performs a test for results::parse() that should succeed.
59 ///
60 /// \param exp_type The expected type of the result.
61 /// \param exp_argument The expected argument in the result, if any.
62 /// \param exp_reason The expected reason describing the result, if any.
63 /// \param text The literal input to parse; can include multiple lines.
64 static void
parse_ok_test(const engine::atf_result::types & exp_type,const optional<int> & exp_argument,const char * exp_reason,const char * text)65 parse_ok_test(const engine::atf_result::types& exp_type,
66 const optional< int >& exp_argument,
67 const char* exp_reason, const char* text)
68 {
69 std::istringstream input(text);
70 const engine::atf_result actual = engine::atf_result::parse(input);
71 ATF_REQUIRE(exp_type == actual.type());
72 ATF_REQUIRE_EQ(exp_argument, actual.argument());
73 if (exp_reason != NULL) {
74 ATF_REQUIRE(actual.reason());
75 ATF_REQUIRE_EQ(exp_reason, actual.reason().get());
76 } else {
77 ATF_REQUIRE(!actual.reason());
78 }
79 }
80
81
82 /// Wrapper around parse_ok_test to define a test case.
83 ///
84 /// \param name The name of the test case; will be prefixed with
85 /// "atf_result__parse__".
86 /// \param exp_type The expected type of the result.
87 /// \param exp_argument The expected argument in the result, if any.
88 /// \param exp_reason The expected reason describing the result, if any.
89 /// \param input The literal input to parse.
90 #define PARSE_OK(name, exp_type, exp_argument, exp_reason, input) \
91 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__parse__ ## name); \
92 ATF_TEST_CASE_BODY(atf_result__parse__ ## name) \
93 { \
94 parse_ok_test(exp_type, exp_argument, exp_reason, input); \
95 }
96
97
98 /// Performs a test for results::parse() that should fail.
99 ///
100 /// \param reason_regexp The reason to match against the broken reason.
101 /// \param text The literal input to parse; can include multiple lines.
102 static void
parse_broken_test(const char * reason_regexp,const char * text)103 parse_broken_test(const char* reason_regexp, const char* text)
104 {
105 std::istringstream input(text);
106 ATF_REQUIRE_THROW_RE(engine::format_error, reason_regexp,
107 engine::atf_result::parse(input));
108 }
109
110
111 /// Wrapper around parse_broken_test to define a test case.
112 ///
113 /// \param name The name of the test case; will be prefixed with
114 /// "atf_result__parse__".
115 /// \param reason_regexp The reason to match against the broken reason.
116 /// \param input The literal input to parse.
117 #define PARSE_BROKEN(name, reason_regexp, input) \
118 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__parse__ ## name); \
119 ATF_TEST_CASE_BODY(atf_result__parse__ ## name) \
120 { \
121 parse_broken_test(reason_regexp, input); \
122 }
123
124
125 } // anonymous namespace
126
127
128 PARSE_BROKEN(empty,
129 "Empty.*no new line",
130 "");
131 PARSE_BROKEN(no_newline__unknown,
132 "Empty.*no new line",
133 "foo");
134 PARSE_BROKEN(no_newline__known,
135 "Empty.*no new line",
136 "passed");
137 PARSE_BROKEN(multiline__no_newline,
138 "multiple lines.*foo<<NEWLINE>>bar",
139 "failed: foo\nbar");
140 PARSE_BROKEN(multiline__with_newline,
141 "multiple lines.*foo<<NEWLINE>>bar",
142 "failed: foo\nbar\n");
143 PARSE_BROKEN(unknown_status__no_reason,
144 "Unknown.*result.*'cba'",
145 "cba\n");
146 PARSE_BROKEN(unknown_status__with_reason,
147 "Unknown.*result.*'hgf'",
148 "hgf: foo\n");
149 PARSE_BROKEN(missing_reason__no_delim,
150 "failed.*followed by.*reason",
151 "failed\n");
152 PARSE_BROKEN(missing_reason__bad_delim,
153 "failed.*followed by.*reason",
154 "failed:\n");
155 PARSE_BROKEN(missing_reason__empty,
156 "failed.*followed by.*reason",
157 "failed: \n");
158
159
160 PARSE_OK(broken__ok,
161 engine::atf_result::broken, none, "a b c",
162 "broken: a b c\n");
163 PARSE_OK(broken__blanks,
164 engine::atf_result::broken, none, " ",
165 "broken: \n");
166
167
168 PARSE_OK(expected_death__ok,
169 engine::atf_result::expected_death, none, "a b c",
170 "expected_death: a b c\n");
171 PARSE_OK(expected_death__blanks,
172 engine::atf_result::expected_death, none, " ",
173 "expected_death: \n");
174
175
176 PARSE_OK(expected_exit__ok__any,
177 engine::atf_result::expected_exit, none, "any exit code",
178 "expected_exit: any exit code\n");
179 PARSE_OK(expected_exit__ok__specific,
180 engine::atf_result::expected_exit, optional< int >(712),
181 "some known exit code",
182 "expected_exit(712): some known exit code\n");
183 PARSE_BROKEN(expected_exit__bad_int,
184 "Invalid integer.*45a3",
185 "expected_exit(45a3): this is broken\n");
186
187
188 PARSE_OK(expected_failure__ok,
189 engine::atf_result::expected_failure, none, "a b c",
190 "expected_failure: a b c\n");
191 PARSE_OK(expected_failure__blanks,
192 engine::atf_result::expected_failure, none, " ",
193 "expected_failure: \n");
194
195
196 PARSE_OK(expected_signal__ok__any,
197 engine::atf_result::expected_signal, none, "any signal code",
198 "expected_signal: any signal code\n");
199 PARSE_OK(expected_signal__ok__specific,
200 engine::atf_result::expected_signal, optional< int >(712),
201 "some known signal code",
202 "expected_signal(712): some known signal code\n");
203 PARSE_BROKEN(expected_signal__bad_int,
204 "Invalid integer.*45a3",
205 "expected_signal(45a3): this is broken\n");
206
207
208 PARSE_OK(expected_timeout__ok,
209 engine::atf_result::expected_timeout, none, "a b c",
210 "expected_timeout: a b c\n");
211 PARSE_OK(expected_timeout__blanks,
212 engine::atf_result::expected_timeout, none, " ",
213 "expected_timeout: \n");
214
215
216 PARSE_OK(failed__ok,
217 engine::atf_result::failed, none, "a b c",
218 "failed: a b c\n");
219 PARSE_OK(failed__blanks,
220 engine::atf_result::failed, none, " ",
221 "failed: \n");
222
223
224 PARSE_OK(passed__ok,
225 engine::atf_result::passed, none, NULL,
226 "passed\n");
227 PARSE_BROKEN(passed__reason,
228 "cannot have a reason",
229 "passed a b c\n");
230
231
232 PARSE_OK(skipped__ok,
233 engine::atf_result::skipped, none, "a b c",
234 "skipped: a b c\n");
235 PARSE_OK(skipped__blanks,
236 engine::atf_result::skipped, none, " ",
237 "skipped: \n");
238
239
240 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__ok);
ATF_TEST_CASE_BODY(atf_result__load__ok)241 ATF_TEST_CASE_BODY(atf_result__load__ok)
242 {
243 std::ofstream output("result.txt");
244 ATF_REQUIRE(output);
245 output << "skipped: a b c\n";
246 output.close();
247
248 const engine::atf_result result = engine::atf_result::load(
249 utils::fs::path("result.txt"));
250 ATF_REQUIRE(engine::atf_result::skipped == result.type());
251 ATF_REQUIRE(!result.argument());
252 ATF_REQUIRE(result.reason());
253 ATF_REQUIRE_EQ("a b c", result.reason().get());
254 }
255
256
257 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__missing_file);
ATF_TEST_CASE_BODY(atf_result__load__missing_file)258 ATF_TEST_CASE_BODY(atf_result__load__missing_file)
259 {
260 ATF_REQUIRE_THROW_RE(
261 std::runtime_error, "Cannot open",
262 engine::atf_result::load(utils::fs::path("result.txt")));
263 }
264
265
266 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__format_error);
ATF_TEST_CASE_BODY(atf_result__load__format_error)267 ATF_TEST_CASE_BODY(atf_result__load__format_error)
268 {
269 std::ofstream output("abc.txt");
270 ATF_REQUIRE(output);
271 output << "passed: foo\n";
272 output.close();
273
274 ATF_REQUIRE_THROW_RE(engine::format_error, "cannot have a reason",
275 engine::atf_result::load(utils::fs::path("abc.txt")));
276 }
277
278
279 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__broken__ok);
ATF_TEST_CASE_BODY(atf_result__apply__broken__ok)280 ATF_TEST_CASE_BODY(atf_result__apply__broken__ok)
281 {
282 const engine::atf_result in_result(engine::atf_result::broken,
283 "Passthrough");
284 const process::status status = process::status::fake_exited(EXIT_SUCCESS);
285 ATF_REQUIRE_EQ(in_result, in_result.apply(utils::make_optional(status)));
286 }
287
288
289 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__timed_out);
ATF_TEST_CASE_BODY(atf_result__apply__timed_out)290 ATF_TEST_CASE_BODY(atf_result__apply__timed_out)
291 {
292 const engine::atf_result timed_out(engine::atf_result::broken,
293 "Some arbitrary error");
294 ATF_REQUIRE_EQ(engine::atf_result(engine::atf_result::broken,
295 "Test case body timed out"),
296 timed_out.apply(none));
297 }
298
299
300 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_death__ok);
ATF_TEST_CASE_BODY(atf_result__apply__expected_death__ok)301 ATF_TEST_CASE_BODY(atf_result__apply__expected_death__ok)
302 {
303 const engine::atf_result in_result(engine::atf_result::expected_death,
304 "Passthrough");
305 const process::status status = process::status::fake_signaled(SIGINT, true);
306 ATF_REQUIRE_EQ(in_result, in_result.apply(utils::make_optional(status)));
307 }
308
309
310 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__ok);
ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__ok)311 ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__ok)
312 {
313 const process::status success = process::status::fake_exited(EXIT_SUCCESS);
314 const process::status failure = process::status::fake_exited(EXIT_FAILURE);
315
316 const engine::atf_result any_code(engine::atf_result::expected_exit, none,
317 "The reason");
318 ATF_REQUIRE_EQ(any_code, any_code.apply(utils::make_optional(success)));
319 ATF_REQUIRE_EQ(any_code, any_code.apply(utils::make_optional(failure)));
320
321 const engine::atf_result a_code(engine::atf_result::expected_exit,
322 utils::make_optional(EXIT_FAILURE), "The reason");
323 ATF_REQUIRE_EQ(a_code, a_code.apply(utils::make_optional(failure)));
324 }
325
326
327 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__failed);
ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__failed)328 ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__failed)
329 {
330 const process::status success = process::status::fake_exited(EXIT_SUCCESS);
331
332 const engine::atf_result a_code(engine::atf_result::expected_exit,
333 utils::make_optional(EXIT_FAILURE), "The reason");
334 ATF_REQUIRE_EQ(
335 engine::atf_result(engine::atf_result::failed,
336 "Test case expected to exit with code 1 but got "
337 "code 0"),
338 a_code.apply(utils::make_optional(success)));
339 }
340
341
342 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__broken);
ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__broken)343 ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__broken)
344 {
345 const process::status sig3 = process::status::fake_signaled(3, false);
346
347 const engine::atf_result any_code(engine::atf_result::expected_exit, none,
348 "The reason");
349 ATF_REQUIRE_EQ(
350 engine::atf_result(engine::atf_result::broken,
351 "Expected clean exit but received signal 3"),
352 any_code.apply(utils::make_optional(sig3)));
353 }
354
355
356 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_failure__ok);
ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__ok)357 ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__ok)
358 {
359 const process::status status = process::status::fake_exited(EXIT_SUCCESS);
360 const engine::atf_result xfailure(engine::atf_result::expected_failure,
361 "The reason");
362 ATF_REQUIRE_EQ(xfailure, xfailure.apply(utils::make_optional(status)));
363 }
364
365
366 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_failure__broken);
ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__broken)367 ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__broken)
368 {
369 const process::status failure = process::status::fake_exited(EXIT_FAILURE);
370 const process::status sig3 = process::status::fake_signaled(3, true);
371 const process::status sig4 = process::status::fake_signaled(4, false);
372
373 const engine::atf_result xfailure(engine::atf_result::expected_failure,
374 "The reason");
375 ATF_REQUIRE_EQ(
376 engine::atf_result(engine::atf_result::broken,
377 "Expected failure should have reported success but "
378 "exited with code 1"),
379 xfailure.apply(utils::make_optional(failure)));
380 ATF_REQUIRE_EQ(
381 engine::atf_result(engine::atf_result::broken,
382 "Expected failure should have reported success but "
383 "received signal 3 (core dumped)"),
384 xfailure.apply(utils::make_optional(sig3)));
385 ATF_REQUIRE_EQ(
386 engine::atf_result(engine::atf_result::broken,
387 "Expected failure should have reported success but "
388 "received signal 4"),
389 xfailure.apply(utils::make_optional(sig4)));
390 }
391
392
393 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__ok);
ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__ok)394 ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__ok)
395 {
396 const process::status sig1 = process::status::fake_signaled(1, false);
397 const process::status sig3 = process::status::fake_signaled(3, true);
398
399 const engine::atf_result any_sig(engine::atf_result::expected_signal, none,
400 "The reason");
401 ATF_REQUIRE_EQ(any_sig, any_sig.apply(utils::make_optional(sig1)));
402 ATF_REQUIRE_EQ(any_sig, any_sig.apply(utils::make_optional(sig3)));
403
404 const engine::atf_result a_sig(engine::atf_result::expected_signal,
405 utils::make_optional(3), "The reason");
406 ATF_REQUIRE_EQ(a_sig, a_sig.apply(utils::make_optional(sig3)));
407 }
408
409
410 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__failed);
ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__failed)411 ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__failed)
412 {
413 const process::status sig5 = process::status::fake_signaled(5, false);
414
415 const engine::atf_result a_sig(engine::atf_result::expected_signal,
416 utils::make_optional(4), "The reason");
417 ATF_REQUIRE_EQ(
418 engine::atf_result(engine::atf_result::failed,
419 "Test case expected to receive signal 4 but got 5"),
420 a_sig.apply(utils::make_optional(sig5)));
421 }
422
423
424 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__broken);
ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__broken)425 ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__broken)
426 {
427 const process::status success = process::status::fake_exited(EXIT_SUCCESS);
428
429 const engine::atf_result any_sig(engine::atf_result::expected_signal, none,
430 "The reason");
431 ATF_REQUIRE_EQ(
432 engine::atf_result(engine::atf_result::broken,
433 "Expected signal but exited with code 0"),
434 any_sig.apply(utils::make_optional(success)));
435 }
436
437
438 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_timeout__ok);
ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__ok)439 ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__ok)
440 {
441 const engine::atf_result timeout(engine::atf_result::expected_timeout,
442 "The reason");
443 ATF_REQUIRE_EQ(timeout, timeout.apply(none));
444 }
445
446
447 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_timeout__broken);
ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__broken)448 ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__broken)
449 {
450 const process::status status = process::status::fake_exited(EXIT_SUCCESS);
451 const engine::atf_result timeout(engine::atf_result::expected_timeout,
452 "The reason");
453 ATF_REQUIRE_EQ(
454 engine::atf_result(engine::atf_result::broken,
455 "Expected timeout but exited with code 0"),
456 timeout.apply(utils::make_optional(status)));
457 }
458
459
460 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__failed__ok);
ATF_TEST_CASE_BODY(atf_result__apply__failed__ok)461 ATF_TEST_CASE_BODY(atf_result__apply__failed__ok)
462 {
463 const process::status status = process::status::fake_exited(EXIT_FAILURE);
464 const engine::atf_result failed(engine::atf_result::failed, "The reason");
465 ATF_REQUIRE_EQ(failed, failed.apply(utils::make_optional(status)));
466 }
467
468
469 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__failed__broken);
ATF_TEST_CASE_BODY(atf_result__apply__failed__broken)470 ATF_TEST_CASE_BODY(atf_result__apply__failed__broken)
471 {
472 const process::status success = process::status::fake_exited(EXIT_SUCCESS);
473 const process::status sig3 = process::status::fake_signaled(3, true);
474 const process::status sig4 = process::status::fake_signaled(4, false);
475
476 const engine::atf_result failed(engine::atf_result::failed, "The reason");
477 ATF_REQUIRE_EQ(
478 engine::atf_result(engine::atf_result::broken,
479 "Failed test case should have reported failure but "
480 "exited with code 0"),
481 failed.apply(utils::make_optional(success)));
482 ATF_REQUIRE_EQ(
483 engine::atf_result(engine::atf_result::broken,
484 "Failed test case should have reported failure but "
485 "received signal 3 (core dumped)"),
486 failed.apply(utils::make_optional(sig3)));
487 ATF_REQUIRE_EQ(
488 engine::atf_result(engine::atf_result::broken,
489 "Failed test case should have reported failure but "
490 "received signal 4"),
491 failed.apply(utils::make_optional(sig4)));
492 }
493
494
495 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__passed__ok);
ATF_TEST_CASE_BODY(atf_result__apply__passed__ok)496 ATF_TEST_CASE_BODY(atf_result__apply__passed__ok)
497 {
498 const process::status status = process::status::fake_exited(EXIT_SUCCESS);
499 const engine::atf_result passed(engine::atf_result::passed);
500 ATF_REQUIRE_EQ(passed, passed.apply(utils::make_optional(status)));
501 }
502
503
504 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__passed__broken);
ATF_TEST_CASE_BODY(atf_result__apply__passed__broken)505 ATF_TEST_CASE_BODY(atf_result__apply__passed__broken)
506 {
507 const process::status failure = process::status::fake_exited(EXIT_FAILURE);
508 const process::status sig3 = process::status::fake_signaled(3, true);
509 const process::status sig4 = process::status::fake_signaled(4, false);
510
511 const engine::atf_result passed(engine::atf_result::passed);
512 ATF_REQUIRE_EQ(
513 engine::atf_result(engine::atf_result::broken,
514 "Passed test case should have reported success but "
515 "exited with code 1"),
516 passed.apply(utils::make_optional(failure)));
517 ATF_REQUIRE_EQ(
518 engine::atf_result(engine::atf_result::broken,
519 "Passed test case should have reported success but "
520 "received signal 3 (core dumped)"),
521 passed.apply(utils::make_optional(sig3)));
522 ATF_REQUIRE_EQ(
523 engine::atf_result(engine::atf_result::broken,
524 "Passed test case should have reported success but "
525 "received signal 4"),
526 passed.apply(utils::make_optional(sig4)));
527 }
528
529
530 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__skipped__ok);
ATF_TEST_CASE_BODY(atf_result__apply__skipped__ok)531 ATF_TEST_CASE_BODY(atf_result__apply__skipped__ok)
532 {
533 const process::status status = process::status::fake_exited(EXIT_SUCCESS);
534 const engine::atf_result skipped(engine::atf_result::skipped, "The reason");
535 ATF_REQUIRE_EQ(skipped, skipped.apply(utils::make_optional(status)));
536 }
537
538
539 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__skipped__broken);
ATF_TEST_CASE_BODY(atf_result__apply__skipped__broken)540 ATF_TEST_CASE_BODY(atf_result__apply__skipped__broken)
541 {
542 const process::status failure = process::status::fake_exited(EXIT_FAILURE);
543 const process::status sig3 = process::status::fake_signaled(3, true);
544 const process::status sig4 = process::status::fake_signaled(4, false);
545
546 const engine::atf_result skipped(engine::atf_result::skipped, "The reason");
547 ATF_REQUIRE_EQ(
548 engine::atf_result(engine::atf_result::broken,
549 "Skipped test case should have reported success but "
550 "exited with code 1"),
551 skipped.apply(utils::make_optional(failure)));
552 ATF_REQUIRE_EQ(
553 engine::atf_result(engine::atf_result::broken,
554 "Skipped test case should have reported success but "
555 "received signal 3 (core dumped)"),
556 skipped.apply(utils::make_optional(sig3)));
557 ATF_REQUIRE_EQ(
558 engine::atf_result(engine::atf_result::broken,
559 "Skipped test case should have reported success but "
560 "received signal 4"),
561 skipped.apply(utils::make_optional(sig4)));
562 }
563
564
565 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__broken);
ATF_TEST_CASE_BODY(atf_result__externalize__broken)566 ATF_TEST_CASE_BODY(atf_result__externalize__broken)
567 {
568 const engine::atf_result raw(engine::atf_result::broken, "The reason");
569 const model::test_result expected(model::test_result_broken,
570 "The reason");
571 ATF_REQUIRE_EQ(expected, raw.externalize());
572 }
573
574
575 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_death);
ATF_TEST_CASE_BODY(atf_result__externalize__expected_death)576 ATF_TEST_CASE_BODY(atf_result__externalize__expected_death)
577 {
578 const engine::atf_result raw(engine::atf_result::expected_death,
579 "The reason");
580 const model::test_result expected(model::test_result_expected_failure,
581 "The reason");
582 ATF_REQUIRE_EQ(expected, raw.externalize());
583 }
584
585
586 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_exit);
ATF_TEST_CASE_BODY(atf_result__externalize__expected_exit)587 ATF_TEST_CASE_BODY(atf_result__externalize__expected_exit)
588 {
589 const engine::atf_result raw(engine::atf_result::expected_exit,
590 "The reason");
591 const model::test_result expected(model::test_result_expected_failure,
592 "The reason");
593 ATF_REQUIRE_EQ(expected, raw.externalize());
594 }
595
596
597 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_failure);
ATF_TEST_CASE_BODY(atf_result__externalize__expected_failure)598 ATF_TEST_CASE_BODY(atf_result__externalize__expected_failure)
599 {
600 const engine::atf_result raw(engine::atf_result::expected_failure,
601 "The reason");
602 const model::test_result expected(model::test_result_expected_failure,
603 "The reason");
604 ATF_REQUIRE_EQ(expected, raw.externalize());
605 }
606
607
608 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_signal);
ATF_TEST_CASE_BODY(atf_result__externalize__expected_signal)609 ATF_TEST_CASE_BODY(atf_result__externalize__expected_signal)
610 {
611 const engine::atf_result raw(engine::atf_result::expected_signal,
612 "The reason");
613 const model::test_result expected(model::test_result_expected_failure,
614 "The reason");
615 ATF_REQUIRE_EQ(expected, raw.externalize());
616 }
617
618
619 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_timeout);
ATF_TEST_CASE_BODY(atf_result__externalize__expected_timeout)620 ATF_TEST_CASE_BODY(atf_result__externalize__expected_timeout)
621 {
622 const engine::atf_result raw(engine::atf_result::expected_timeout,
623 "The reason");
624 const model::test_result expected(model::test_result_expected_failure,
625 "The reason");
626 ATF_REQUIRE_EQ(expected, raw.externalize());
627 }
628
629
630 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__failed);
ATF_TEST_CASE_BODY(atf_result__externalize__failed)631 ATF_TEST_CASE_BODY(atf_result__externalize__failed)
632 {
633 const engine::atf_result raw(engine::atf_result::failed, "The reason");
634 const model::test_result expected(model::test_result_failed,
635 "The reason");
636 ATF_REQUIRE(expected == raw.externalize());
637 }
638
639
640 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__passed);
ATF_TEST_CASE_BODY(atf_result__externalize__passed)641 ATF_TEST_CASE_BODY(atf_result__externalize__passed)
642 {
643 const engine::atf_result raw(engine::atf_result::passed);
644 const model::test_result expected(model::test_result_passed);
645 ATF_REQUIRE_EQ(expected, raw.externalize());
646 }
647
648
649 ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__skipped);
ATF_TEST_CASE_BODY(atf_result__externalize__skipped)650 ATF_TEST_CASE_BODY(atf_result__externalize__skipped)
651 {
652 const engine::atf_result raw(engine::atf_result::skipped, "The reason");
653 const model::test_result expected(model::test_result_skipped,
654 "The reason");
655 ATF_REQUIRE_EQ(expected, raw.externalize());
656 }
657
658
659 ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__missing_file);
ATF_TEST_CASE_BODY(calculate_atf_result__missing_file)660 ATF_TEST_CASE_BODY(calculate_atf_result__missing_file)
661 {
662 using process::status;
663
664 const status body_status = status::fake_exited(EXIT_SUCCESS);
665 const model::test_result expected(
666 model::test_result_broken,
667 "Premature exit; test case exited with code 0");
668 ATF_REQUIRE_EQ(expected, engine::calculate_atf_result(
669 utils::make_optional(body_status), fs::path("foo")));
670 }
671
672
673 ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__bad_file);
ATF_TEST_CASE_BODY(calculate_atf_result__bad_file)674 ATF_TEST_CASE_BODY(calculate_atf_result__bad_file)
675 {
676 using process::status;
677
678 const status body_status = status::fake_exited(EXIT_SUCCESS);
679 atf::utils::create_file("foo", "invalid\n");
680 const model::test_result expected(model::test_result_broken,
681 "Unknown test result 'invalid'");
682 ATF_REQUIRE_EQ(expected, engine::calculate_atf_result(
683 utils::make_optional(body_status), fs::path("foo")));
684 }
685
686
687 ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__body_ok);
ATF_TEST_CASE_BODY(calculate_atf_result__body_ok)688 ATF_TEST_CASE_BODY(calculate_atf_result__body_ok)
689 {
690 using process::status;
691
692 atf::utils::create_file("result.txt", "skipped: Something\n");
693 const status body_status = status::fake_exited(EXIT_SUCCESS);
694 ATF_REQUIRE_EQ(
695 model::test_result(model::test_result_skipped, "Something"),
696 engine::calculate_atf_result(utils::make_optional(body_status),
697 fs::path("result.txt")));
698 }
699
700
701 ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__body_bad);
ATF_TEST_CASE_BODY(calculate_atf_result__body_bad)702 ATF_TEST_CASE_BODY(calculate_atf_result__body_bad)
703 {
704 using process::status;
705
706 atf::utils::create_file("result.txt", "skipped: Something\n");
707 const status body_status = status::fake_exited(EXIT_FAILURE);
708 ATF_REQUIRE_EQ(
709 model::test_result(model::test_result_broken, "Skipped test case "
710 "should have reported success but exited with "
711 "code 1"),
712 engine::calculate_atf_result(utils::make_optional(body_status),
713 fs::path("result.txt")));
714 }
715
716
ATF_INIT_TEST_CASES(tcs)717 ATF_INIT_TEST_CASES(tcs)
718 {
719 ATF_ADD_TEST_CASE(tcs, atf_result__parse__empty);
720 ATF_ADD_TEST_CASE(tcs, atf_result__parse__no_newline__unknown);
721 ATF_ADD_TEST_CASE(tcs, atf_result__parse__no_newline__known);
722 ATF_ADD_TEST_CASE(tcs, atf_result__parse__multiline__no_newline);
723 ATF_ADD_TEST_CASE(tcs, atf_result__parse__multiline__with_newline);
724 ATF_ADD_TEST_CASE(tcs, atf_result__parse__unknown_status__no_reason);
725 ATF_ADD_TEST_CASE(tcs, atf_result__parse__unknown_status__with_reason);
726 ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__no_delim);
727 ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__bad_delim);
728 ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__empty);
729 ATF_ADD_TEST_CASE(tcs, atf_result__parse__broken__ok);
730 ATF_ADD_TEST_CASE(tcs, atf_result__parse__broken__blanks);
731 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_death__ok);
732 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_death__blanks);
733 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__ok__any);
734 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__ok__specific);
735 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__bad_int);
736 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_failure__ok);
737 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_failure__blanks);
738 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__ok__any);
739 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__ok__specific);
740 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__bad_int);
741 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_timeout__ok);
742 ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_timeout__blanks);
743 ATF_ADD_TEST_CASE(tcs, atf_result__parse__failed__ok);
744 ATF_ADD_TEST_CASE(tcs, atf_result__parse__failed__blanks);
745 ATF_ADD_TEST_CASE(tcs, atf_result__parse__passed__ok);
746 ATF_ADD_TEST_CASE(tcs, atf_result__parse__passed__reason);
747 ATF_ADD_TEST_CASE(tcs, atf_result__parse__skipped__ok);
748 ATF_ADD_TEST_CASE(tcs, atf_result__parse__skipped__blanks);
749
750 ATF_ADD_TEST_CASE(tcs, atf_result__load__ok);
751 ATF_ADD_TEST_CASE(tcs, atf_result__load__missing_file);
752 ATF_ADD_TEST_CASE(tcs, atf_result__load__format_error);
753
754 ATF_ADD_TEST_CASE(tcs, atf_result__apply__broken__ok);
755 ATF_ADD_TEST_CASE(tcs, atf_result__apply__timed_out);
756 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_death__ok);
757 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__ok);
758 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__failed);
759 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__broken);
760 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_failure__ok);
761 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_failure__broken);
762 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__ok);
763 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__failed);
764 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__broken);
765 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_timeout__ok);
766 ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_timeout__broken);
767 ATF_ADD_TEST_CASE(tcs, atf_result__apply__failed__ok);
768 ATF_ADD_TEST_CASE(tcs, atf_result__apply__failed__broken);
769 ATF_ADD_TEST_CASE(tcs, atf_result__apply__passed__ok);
770 ATF_ADD_TEST_CASE(tcs, atf_result__apply__passed__broken);
771 ATF_ADD_TEST_CASE(tcs, atf_result__apply__skipped__ok);
772 ATF_ADD_TEST_CASE(tcs, atf_result__apply__skipped__broken);
773
774 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__broken);
775 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_death);
776 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_exit);
777 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_failure);
778 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_signal);
779 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_timeout);
780 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__failed);
781 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__passed);
782 ATF_ADD_TEST_CASE(tcs, atf_result__externalize__skipped);
783
784 ATF_ADD_TEST_CASE(tcs, calculate_atf_result__missing_file);
785 ATF_ADD_TEST_CASE(tcs, calculate_atf_result__bad_file);
786 ATF_ADD_TEST_CASE(tcs, calculate_atf_result__body_ok);
787 ATF_ADD_TEST_CASE(tcs, calculate_atf_result__body_bad);
788 }
789