1 // Copyright 2015 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/tap_parser.hpp" 30 31 #include <fstream> 32 33 #include <atf-c++.hpp> 34 35 #include "engine/exceptions.hpp" 36 #include "utils/format/containers.ipp" 37 #include "utils/format/macros.hpp" 38 #include "utils/fs/path.hpp" 39 40 namespace fs = utils::fs; 41 42 43 namespace { 44 45 46 /// Helper to execute parse_tap_output() on inline text contents. 47 /// 48 /// \param contents The TAP output to parse. 49 /// 50 /// \return The tap_summary object resultingafter the parse. 51 /// 52 /// \throw engine::load_error If parse_tap_output() fails. 53 static engine::tap_summary 54 do_parse(const std::string& contents) 55 { 56 std::ofstream output("tap.txt"); 57 ATF_REQUIRE(output); 58 output << contents; 59 output.close(); 60 return engine::parse_tap_output(fs::path("tap.txt")); 61 } 62 63 64 } // anonymous namespace 65 66 67 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__bailed_out); 68 ATF_TEST_CASE_BODY(tap_summary__bailed_out) 69 { 70 const engine::tap_summary summary = engine::tap_summary::new_bailed_out(); 71 ATF_REQUIRE(summary.bailed_out()); 72 } 73 74 75 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__some_results); 76 ATF_TEST_CASE_BODY(tap_summary__some_results) 77 { 78 const engine::tap_summary summary = engine::tap_summary::new_results( 79 engine::tap_plan(1, 5), 3, 2); 80 ATF_REQUIRE(!summary.bailed_out()); 81 ATF_REQUIRE_EQ(engine::tap_plan(1, 5), summary.plan()); 82 ATF_REQUIRE_EQ(3, summary.ok_count()); 83 ATF_REQUIRE_EQ(2, summary.not_ok_count()); 84 } 85 86 87 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__all_skipped); 88 ATF_TEST_CASE_BODY(tap_summary__all_skipped) 89 { 90 const engine::tap_summary summary = engine::tap_summary::new_all_skipped( 91 "Skipped"); 92 ATF_REQUIRE(!summary.bailed_out()); 93 ATF_REQUIRE_EQ(engine::tap_plan(1, 0), summary.plan()); 94 ATF_REQUIRE_EQ("Skipped", summary.all_skipped_reason()); 95 } 96 97 98 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__equality_operators); 99 ATF_TEST_CASE_BODY(tap_summary__equality_operators) 100 { 101 const engine::tap_summary bailed_out = 102 engine::tap_summary::new_bailed_out(); 103 const engine::tap_summary all_skipped_1 = 104 engine::tap_summary::new_all_skipped("Reason 1"); 105 const engine::tap_summary results_1 = 106 engine::tap_summary::new_results(engine::tap_plan(1, 5), 3, 2); 107 108 // Self-equality checks. 109 ATF_REQUIRE( bailed_out == bailed_out); 110 ATF_REQUIRE(!(bailed_out != bailed_out)); 111 ATF_REQUIRE( all_skipped_1 == all_skipped_1); 112 ATF_REQUIRE(!(all_skipped_1 != all_skipped_1)); 113 ATF_REQUIRE( results_1 == results_1); 114 ATF_REQUIRE(!(results_1 != results_1)); 115 116 // Cross-equality checks. 117 ATF_REQUIRE(!(bailed_out == all_skipped_1)); 118 ATF_REQUIRE( bailed_out != all_skipped_1); 119 ATF_REQUIRE(!(bailed_out == results_1)); 120 ATF_REQUIRE( bailed_out != results_1); 121 ATF_REQUIRE(!(all_skipped_1 == results_1)); 122 ATF_REQUIRE( all_skipped_1 != results_1); 123 124 // Checks for the all_skipped "type". 125 const engine::tap_summary all_skipped_2 = 126 engine::tap_summary::new_all_skipped("Reason 2"); 127 ATF_REQUIRE(!(all_skipped_1 == all_skipped_2)); 128 ATF_REQUIRE( all_skipped_1 != all_skipped_2); 129 130 131 // Checks for the results "type", different plan. 132 const engine::tap_summary results_2 = 133 engine::tap_summary::new_results(engine::tap_plan(2, 6), 134 results_1.ok_count(), 135 results_1.not_ok_count()); 136 ATF_REQUIRE(!(results_1 == results_2)); 137 ATF_REQUIRE( results_1 != results_2); 138 139 140 // Checks for the results "type", different counts. 141 const engine::tap_summary results_3 = 142 engine::tap_summary::new_results(results_1.plan(), 143 results_1.not_ok_count(), 144 results_1.ok_count()); 145 ATF_REQUIRE(!(results_1 == results_3)); 146 ATF_REQUIRE( results_1 != results_3); 147 } 148 149 150 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__output); 151 ATF_TEST_CASE_BODY(tap_summary__output) 152 { 153 { 154 const engine::tap_summary summary = 155 engine::tap_summary::new_bailed_out(); 156 ATF_REQUIRE_EQ( 157 "tap_summary{bailed_out=true}", 158 (F("%s") % summary).str()); 159 } 160 161 { 162 const engine::tap_summary summary = 163 engine::tap_summary::new_results(engine::tap_plan(5, 10), 2, 4); 164 ATF_REQUIRE_EQ( 165 "tap_summary{bailed_out=false, plan=5..10, ok_count=2, " 166 "not_ok_count=4}", 167 (F("%s") % summary).str()); 168 } 169 170 { 171 const engine::tap_summary summary = 172 engine::tap_summary::new_all_skipped("Who knows"); 173 ATF_REQUIRE_EQ( 174 "tap_summary{bailed_out=false, plan=1..0, " 175 "all_skipped_reason=Who knows}", 176 (F("%s") % summary).str()); 177 } 178 } 179 180 181 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__only_one_result); 182 ATF_TEST_CASE_BODY(parse_tap_output__only_one_result) 183 { 184 const engine::tap_summary summary = do_parse( 185 "1..1\n" 186 "ok - 1\n"); 187 188 const engine::tap_summary exp_summary = 189 engine::tap_summary::new_results(engine::tap_plan(1, 1), 1, 0); 190 ATF_REQUIRE_EQ(exp_summary, summary); 191 } 192 193 194 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__all_pass); 195 ATF_TEST_CASE_BODY(parse_tap_output__all_pass) 196 { 197 const engine::tap_summary summary = do_parse( 198 "1..8\n" 199 "ok - 1\n" 200 " Some diagnostic message\n" 201 "ok - 2 This test also passed\n" 202 "garbage line\n" 203 "ok - 3 This test passed\n" 204 "not ok 4 # SKIP Some reason\n" 205 "not ok 5 # TODO Another reason\n" 206 "ok - 6 Doesn't make a difference SKIP\n" 207 "ok - 7 Doesn't make a difference either TODO\n" 208 "ok # Also works without a number\n"); 209 210 const engine::tap_summary exp_summary = 211 engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0); 212 ATF_REQUIRE_EQ(exp_summary, summary); 213 } 214 215 216 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__some_fail); 217 ATF_TEST_CASE_BODY(parse_tap_output__some_fail) 218 { 219 const engine::tap_summary summary = do_parse( 220 "garbage line\n" 221 "not ok - 1 This test failed\n" 222 "ok - 2 This test passed\n" 223 "not ok - 3 This test failed\n" 224 "1..6\n" 225 "not ok - 4 This test failed\n" 226 "ok - 5 This test passed\n" 227 "not ok # Fails as well without a number\n"); 228 229 const engine::tap_summary exp_summary = 230 engine::tap_summary::new_results(engine::tap_plan(1, 6), 2, 4); 231 ATF_REQUIRE_EQ(exp_summary, summary); 232 } 233 234 235 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_and_todo_variants); 236 ATF_TEST_CASE_BODY(parse_tap_output__skip_and_todo_variants) 237 { 238 const engine::tap_summary summary = do_parse( 239 "1..8\n" 240 "not ok - 1 # SKIP Some reason\n" 241 "not ok - 2 # skip Some reason\n" 242 "not ok - 3 # Skipped Some reason\n" 243 "not ok - 4 # skipped Some reason\n" 244 "not ok - 5 # Skipped: Some reason\n" 245 "not ok - 6 # skipped: Some reason\n" 246 "not ok - 7 # TODO Some reason\n" 247 "not ok - 8 # todo Some reason\n"); 248 249 const engine::tap_summary exp_summary = 250 engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0); 251 ATF_REQUIRE_EQ(exp_summary, summary); 252 } 253 254 255 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_with_reason); 256 ATF_TEST_CASE_BODY(parse_tap_output__skip_all_with_reason) 257 { 258 const engine::tap_summary summary = do_parse( 259 "1..0 SKIP Some reason for skipping\n" 260 "ok - 1\n" 261 " Some diagnostic message\n" 262 "ok - 6 Doesn't make a difference SKIP\n" 263 "ok - 7 Doesn't make a difference either TODO\n"); 264 265 const engine::tap_summary exp_summary = 266 engine::tap_summary::new_all_skipped("Some reason for skipping"); 267 ATF_REQUIRE_EQ(exp_summary, summary); 268 } 269 270 271 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_without_reason); 272 ATF_TEST_CASE_BODY(parse_tap_output__skip_all_without_reason) 273 { 274 const engine::tap_summary summary = do_parse( 275 "1..0 unrecognized # garbage skip\n"); 276 277 const engine::tap_summary exp_summary = 278 engine::tap_summary::new_all_skipped("No reason specified"); 279 ATF_REQUIRE_EQ(exp_summary, summary); 280 } 281 282 283 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_invalid); 284 ATF_TEST_CASE_BODY(parse_tap_output__skip_all_invalid) 285 { 286 ATF_REQUIRE_THROW_RE(engine::load_error, 287 "Skipped plan must be 1\\.\\.0", 288 do_parse("1..3 # skip\n")); 289 } 290 291 292 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__plan_at_end); 293 ATF_TEST_CASE_BODY(parse_tap_output__plan_at_end) 294 { 295 const engine::tap_summary summary = do_parse( 296 "ok - 1\n" 297 " Some diagnostic message\n" 298 "ok - 2 This test also passed\n" 299 "garbage line\n" 300 "ok - 3 This test passed\n" 301 "not ok 4 # SKIP Some reason\n" 302 "not ok 5 # TODO Another reason\n" 303 "ok - 6 Doesn't make a difference SKIP\n" 304 "ok - 7 Doesn't make a difference either TODO\n" 305 "1..7\n"); 306 307 const engine::tap_summary exp_summary = 308 engine::tap_summary::new_results(engine::tap_plan(1, 7), 7, 0); 309 ATF_REQUIRE_EQ(exp_summary, summary); 310 } 311 312 313 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__stray_oks); 314 ATF_TEST_CASE_BODY(parse_tap_output__stray_oks) 315 { 316 const engine::tap_summary summary = do_parse( 317 "1..3\n" 318 "ok - 1\n" 319 "ok\n" 320 "ok - 2 This test also passed\n" 321 "not ok\n" 322 "ok - 3 This test passed\n"); 323 324 const engine::tap_summary exp_summary = 325 engine::tap_summary::new_results(engine::tap_plan(1, 3), 3, 0); 326 ATF_REQUIRE_EQ(exp_summary, summary); 327 } 328 329 330 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__no_plan); 331 ATF_TEST_CASE_BODY(parse_tap_output__no_plan) 332 { 333 ATF_REQUIRE_THROW_RE( 334 engine::load_error, 335 "Output did not contain any TAP plan", 336 do_parse( 337 "not ok - 1 This test failed\n" 338 "ok - 2 This test passed\n")); 339 } 340 341 342 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__double_plan); 343 ATF_TEST_CASE_BODY(parse_tap_output__double_plan) 344 { 345 ATF_REQUIRE_THROW_RE( 346 engine::load_error, 347 "Found duplicate plan", 348 do_parse( 349 "garbage line\n" 350 "1..5\n" 351 "not ok - 1 This test failed\n" 352 "ok - 2 This test passed\n" 353 "1..8\n" 354 "ok\n")); 355 } 356 357 358 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_plan); 359 ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_plan) 360 { 361 ATF_REQUIRE_THROW_RE( 362 engine::load_error, 363 "Reported plan differs from actual executed tests", 364 do_parse( 365 "1..3\n" 366 "not ok - 1 This test failed\n" 367 "ok - 2 This test passed\n")); 368 } 369 370 371 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_trailing_plan); 372 ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_trailing_plan) 373 { 374 ATF_REQUIRE_THROW_RE( 375 engine::load_error, 376 "Reported plan differs from actual executed tests", 377 do_parse( 378 "not ok - 1 This test failed\n" 379 "ok - 2 This test passed\n" 380 "1..3\n")); 381 } 382 383 384 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__insane_plan); 385 ATF_TEST_CASE_BODY(parse_tap_output__insane_plan) 386 { 387 ATF_REQUIRE_THROW_RE( 388 engine::load_error, "Invalid value", 389 do_parse("120830981209831..234891793874080981092803981092312\n")); 390 } 391 392 393 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__reversed_plan); 394 ATF_TEST_CASE_BODY(parse_tap_output__reversed_plan) 395 { 396 ATF_REQUIRE_THROW_RE(engine::load_error, 397 "Found reversed plan 8\\.\\.5", 398 do_parse("8..5\n")); 399 } 400 401 402 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out); 403 ATF_TEST_CASE_BODY(parse_tap_output__bail_out) 404 { 405 const engine::tap_summary summary = do_parse( 406 "1..3\n" 407 "not ok - 1 This test failed\n" 408 "Bail out! There is some unknown problem\n" 409 "ok - 2 This test passed\n"); 410 411 const engine::tap_summary exp_summary = 412 engine::tap_summary::new_bailed_out(); 413 ATF_REQUIRE_EQ(exp_summary, summary); 414 } 415 416 417 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out_wins_over_no_plan); 418 ATF_TEST_CASE_BODY(parse_tap_output__bail_out_wins_over_no_plan) 419 { 420 const engine::tap_summary summary = do_parse( 421 "not ok - 1 This test failed\n" 422 "Bail out! There is some unknown problem\n" 423 "ok - 2 This test passed\n"); 424 425 const engine::tap_summary exp_summary = 426 engine::tap_summary::new_bailed_out(); 427 ATF_REQUIRE_EQ(exp_summary, summary); 428 } 429 430 431 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__open_failure); 432 ATF_TEST_CASE_BODY(parse_tap_output__open_failure) 433 { 434 ATF_REQUIRE_THROW_RE(engine::load_error, "hello.txt.*Failed to open", 435 engine::parse_tap_output(fs::path("hello.txt"))); 436 } 437 438 439 ATF_INIT_TEST_CASES(tcs) 440 { 441 ATF_ADD_TEST_CASE(tcs, tap_summary__bailed_out); 442 ATF_ADD_TEST_CASE(tcs, tap_summary__some_results); 443 ATF_ADD_TEST_CASE(tcs, tap_summary__all_skipped); 444 ATF_ADD_TEST_CASE(tcs, tap_summary__equality_operators); 445 ATF_ADD_TEST_CASE(tcs, tap_summary__output); 446 447 ATF_ADD_TEST_CASE(tcs, parse_tap_output__only_one_result); 448 ATF_ADD_TEST_CASE(tcs, parse_tap_output__all_pass); 449 ATF_ADD_TEST_CASE(tcs, parse_tap_output__some_fail); 450 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_and_todo_variants); 451 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_without_reason); 452 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_with_reason); 453 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_invalid); 454 ATF_ADD_TEST_CASE(tcs, parse_tap_output__plan_at_end); 455 ATF_ADD_TEST_CASE(tcs, parse_tap_output__stray_oks); 456 ATF_ADD_TEST_CASE(tcs, parse_tap_output__no_plan); 457 ATF_ADD_TEST_CASE(tcs, parse_tap_output__double_plan); 458 ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_plan); 459 ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_trailing_plan); 460 ATF_ADD_TEST_CASE(tcs, parse_tap_output__insane_plan); 461 ATF_ADD_TEST_CASE(tcs, parse_tap_output__reversed_plan); 462 ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out); 463 ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out_wins_over_no_plan); 464 ATF_ADD_TEST_CASE(tcs, parse_tap_output__open_failure); 465 } 466