1 // Copyright 2015 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "engine/tap_parser.hpp" 30 31 #include <fstream> 32 33 #include "engine/exceptions.hpp" 34 #include "utils/format/macros.hpp" 35 #include "utils/noncopyable.hpp" 36 #include "utils/optional.ipp" 37 #include "utils/sanity.hpp" 38 #include "utils/text/exceptions.hpp" 39 #include "utils/text/operations.ipp" 40 #include "utils/text/regex.hpp" 41 42 namespace fs = utils::fs; 43 namespace text = utils::text; 44 45 using utils::optional; 46 47 48 /// TAP plan representing all tests being skipped. 49 const engine::tap_plan engine::all_skipped_plan(1, 0); 50 51 52 namespace { 53 54 55 /// Implementation of the TAP parser. 56 /// 57 /// This is a class only to simplify keeping global constant values around (like 58 /// prebuilt regular expressions). 59 class tap_parser : utils::noncopyable { 60 /// Regular expression to match plan lines. 61 text::regex _plan_regex; 62 63 /// Regular expression to match a TODO and extract the reason. 64 text::regex _todo_regex; 65 66 /// Regular expression to match a SKIP and extract the reason. 67 text::regex _skip_regex; 68 69 /// Regular expression to match a single test result. 70 text::regex _result_regex; 71 72 /// Checks if a line contains a TAP plan and extracts its data. 73 /// 74 /// \param line The line to try to parse. 75 /// \param [in,out] out_plan Used to store the found plan, if any. The same 76 /// output variable should be given to all calls to this function so 77 /// that duplicate plan entries can be discovered. 78 /// \param [out] out_all_skipped_reason Used to store the reason for all 79 /// tests being skipped, if any. If this is set to a non-empty value, 80 /// then the out_plan is set to 1..0. 81 /// 82 /// \return True if the line matched a plan; false otherwise. 83 /// 84 /// \throw engine::format_error If the input is invalid. 85 /// \throw text::error If the input is invalid. 86 bool 87 try_parse_plan(const std::string& line, 88 optional< engine::tap_plan >& out_plan, 89 std::string& out_all_skipped_reason) 90 { 91 const text::regex_matches plan_matches = _plan_regex.match(line); 92 if (!plan_matches) 93 return false; 94 const engine::tap_plan plan( 95 text::to_type< std::size_t >(plan_matches.get(1)), 96 text::to_type< std::size_t >(plan_matches.get(2))); 97 98 if (out_plan) 99 throw engine::format_error( 100 F("Found duplicate plan %s..%s (saw %s..%s earlier)") % 101 plan.first % plan.second % 102 out_plan.get().first % out_plan.get().second); 103 104 std::string all_skipped_reason; 105 const text::regex_matches skip_matches = _skip_regex.match(line); 106 if (skip_matches) { 107 if (plan != engine::all_skipped_plan) { 108 throw engine::format_error(F("Skipped plan must be %s..%s") % 109 engine::all_skipped_plan.first % 110 engine::all_skipped_plan.second); 111 } 112 all_skipped_reason = skip_matches.get(2); 113 if (all_skipped_reason.empty()) 114 all_skipped_reason = "No reason specified"; 115 } else { 116 if (plan.first > plan.second) 117 throw engine::format_error(F("Found reversed plan %s..%s") % 118 plan.first % plan.second); 119 } 120 121 INV(!out_plan); 122 out_plan = plan; 123 out_all_skipped_reason = all_skipped_reason; 124 125 POST(out_plan); 126 POST(out_all_skipped_reason.empty() || 127 out_plan.get() == engine::all_skipped_plan); 128 129 return true; 130 } 131 132 /// Checks if a line contains a TAP test result and extracts its data. 133 /// 134 /// \param line The line to try to parse. 135 /// \param [in,out] out_ok_count Accumulator for 'ok' results. 136 /// \param [in,out] out_not_ok_count Accumulator for 'not ok' results. 137 /// \param [out] out_bailed_out Set to true if the test bailed out. 138 /// 139 /// \return True if the line matched a result; false otherwise. 140 /// 141 /// \throw engine::format_error If the input is invalid. 142 /// \throw text::error If the input is invalid. 143 bool 144 try_parse_result(const std::string& line, std::size_t& out_ok_count, 145 std::size_t& out_not_ok_count, bool& out_bailed_out) 146 { 147 PRE(!out_bailed_out); 148 149 const text::regex_matches result_matches = _result_regex.match(line); 150 if (result_matches) { 151 if (result_matches.get(1) == "ok") { 152 ++out_ok_count; 153 } else { 154 INV(result_matches.get(1) == "not ok"); 155 if (_todo_regex.match(line) || _skip_regex.match(line)) { 156 ++out_ok_count; 157 } else { 158 ++out_not_ok_count; 159 } 160 } 161 return true; 162 } else { 163 if (line.find("Bail out!") == 0) { 164 out_bailed_out = true; 165 return true; 166 } else { 167 return false; 168 } 169 } 170 } 171 172 public: 173 /// Sets up the TAP parser state. 174 tap_parser(void) : 175 _plan_regex(text::regex::compile("^([0-9]+)\\.\\.([0-9]+)", 2)), 176 _todo_regex(text::regex::compile("TODO[ \t]*(.*)$", 2, true)), 177 _skip_regex(text::regex::compile("(SKIP|Skipped:?)[ \t]*(.*)$", 2, 178 true)), 179 _result_regex(text::regex::compile("^(not ok|ok)[ \t-]+[0-9]*", 1)) 180 { 181 } 182 183 /// Parses an input file containing TAP output. 184 /// 185 /// \param input The stream to read from. 186 /// 187 /// \return The results of the parsing in the form of a tap_summary object. 188 /// 189 /// \throw engine::format_error If there are any syntax errors in the input. 190 /// \throw text::error If there are any syntax errors in the input. 191 engine::tap_summary 192 parse(std::ifstream& input) 193 { 194 optional< engine::tap_plan > plan; 195 std::string all_skipped_reason; 196 bool bailed_out = false; 197 std::size_t ok_count = 0, not_ok_count = 0; 198 199 std::string line; 200 while (!bailed_out && std::getline(input, line)) { 201 if (try_parse_result(line, ok_count, not_ok_count, bailed_out)) 202 continue; 203 (void)try_parse_plan(line, plan, all_skipped_reason); 204 } 205 206 if (bailed_out) { 207 return engine::tap_summary::new_bailed_out(); 208 } else { 209 if (!plan) 210 throw engine::format_error( 211 "Output did not contain any TAP plan and the program did " 212 "not bail out"); 213 214 if (plan.get() == engine::all_skipped_plan) { 215 return engine::tap_summary::new_all_skipped(all_skipped_reason); 216 } else { 217 const std::size_t exp_count = plan.get().second - 218 plan.get().first + 1; 219 const std::size_t actual_count = ok_count + not_ok_count; 220 if (exp_count != actual_count) { 221 throw engine::format_error( 222 "Reported plan differs from actual executed tests"); 223 } 224 return engine::tap_summary::new_results(plan.get(), ok_count, 225 not_ok_count); 226 } 227 } 228 } 229 }; 230 231 232 } // anonymous namespace 233 234 235 /// Constructs a TAP summary with the results of parsing a TAP output. 236 /// 237 /// \param bailed_out_ Whether the test program bailed out early or not. 238 /// \param plan_ The TAP plan. 239 /// \param all_skipped_reason_ The reason for skipping all tests, if any. 240 /// \param ok_count_ Number of 'ok' test results. 241 /// \param not_ok_count_ Number of 'not ok' test results. 242 engine::tap_summary::tap_summary(const bool bailed_out_, 243 const tap_plan& plan_, 244 const std::string& all_skipped_reason_, 245 const std::size_t ok_count_, 246 const std::size_t not_ok_count_) : 247 _bailed_out(bailed_out_), _plan(plan_), 248 _all_skipped_reason(all_skipped_reason_), 249 _ok_count(ok_count_), _not_ok_count(not_ok_count_) 250 { 251 } 252 253 254 /// Constructs a TAP summary for a bailed out test program. 255 /// 256 /// \return The new tap_summary object. 257 engine::tap_summary 258 engine::tap_summary::new_bailed_out(void) 259 { 260 return tap_summary(true, tap_plan(0, 0), "", 0, 0); 261 } 262 263 264 /// Constructs a TAP summary for a test program that skipped all tests. 265 /// 266 /// \param reason Textual reason describing why the tests were skipped. 267 /// 268 /// \return The new tap_summary object. 269 engine::tap_summary 270 engine::tap_summary::new_all_skipped(const std::string& reason) 271 { 272 return tap_summary(false, all_skipped_plan, reason, 0, 0); 273 } 274 275 276 /// Constructs a TAP summary for a test program that reported results. 277 /// 278 /// \param plan_ The TAP plan. 279 /// \param ok_count_ Total number of 'ok' results. 280 /// \param not_ok_count_ Total number of 'not ok' results. 281 /// 282 /// \return The new tap_summary object. 283 engine::tap_summary 284 engine::tap_summary::new_results(const tap_plan& plan_, 285 const std::size_t ok_count_, 286 const std::size_t not_ok_count_) 287 { 288 PRE((plan_.second - plan_.first + 1) == (ok_count_ + not_ok_count_)); 289 return tap_summary(false, plan_, "", ok_count_, not_ok_count_); 290 } 291 292 293 /// Checks whether the test program bailed out early or not. 294 /// 295 /// \return True if the test program aborted execution before completing. 296 bool 297 engine::tap_summary::bailed_out(void) const 298 { 299 return _bailed_out; 300 } 301 302 303 /// Gets the TAP plan of the test program. 304 /// 305 /// \pre bailed_out() must be false. 306 /// 307 /// \return The TAP plan. If 1..0, then all_skipped_reason() will have some 308 /// contents. 309 const engine::tap_plan& 310 engine::tap_summary::plan(void) const 311 { 312 PRE(!_bailed_out); 313 return _plan; 314 } 315 316 317 /// Gets the reason for skipping all the tests, if any. 318 /// 319 /// \pre bailed_out() must be false. 320 /// \pre plan() returns 1..0. 321 /// 322 /// \return The reason for skipping all the tests. 323 const std::string& 324 engine::tap_summary::all_skipped_reason(void) const 325 { 326 PRE(!_bailed_out); 327 PRE(_plan == all_skipped_plan); 328 return _all_skipped_reason; 329 } 330 331 332 /// Gets the number of 'ok' test results. 333 /// 334 /// \pre bailed_out() must be false. 335 /// 336 /// \return The number of test results that reported 'ok'. 337 std::size_t 338 engine::tap_summary::ok_count(void) const 339 { 340 PRE(!bailed_out()); 341 PRE(_all_skipped_reason.empty()); 342 return _ok_count; 343 } 344 345 346 /// Gets the number of 'not ok' test results. 347 /// 348 /// \pre bailed_out() must be false. 349 /// 350 /// \return The number of test results that reported 'not ok'. 351 std::size_t 352 engine::tap_summary::not_ok_count(void) const 353 { 354 PRE(!_bailed_out); 355 PRE(_all_skipped_reason.empty()); 356 return _not_ok_count; 357 } 358 359 360 /// Checks two tap_summary objects for equality. 361 /// 362 /// \param other The object to compare this one to. 363 /// 364 /// \return True if the two objects are equal; false otherwise. 365 bool 366 engine::tap_summary::operator==(const tap_summary& other) const 367 { 368 return (_bailed_out == other._bailed_out && 369 _plan == other._plan && 370 _all_skipped_reason == other._all_skipped_reason && 371 _ok_count == other._ok_count && 372 _not_ok_count == other._not_ok_count); 373 } 374 375 376 /// Checks two tap_summary objects for inequality. 377 /// 378 /// \param other The object to compare this one to. 379 /// 380 /// \return True if the two objects are different; false otherwise. 381 bool 382 engine::tap_summary::operator!=(const tap_summary& other) const 383 { 384 return !(*this == other); 385 } 386 387 388 /// Formats a tap_summary into a stream. 389 /// 390 /// \param output The stream into which to inject the object. 391 /// \param summary The summary to format. 392 /// 393 /// \return The output stream. 394 std::ostream& 395 engine::operator<<(std::ostream& output, const tap_summary& summary) 396 { 397 output << "tap_summary{"; 398 if (summary.bailed_out()) { 399 output << "bailed_out=true"; 400 } else { 401 const tap_plan& plan = summary.plan(); 402 output << "bailed_out=false" 403 << ", plan=" << plan.first << ".." << plan.second; 404 if (plan == all_skipped_plan) { 405 output << ", all_skipped_reason=" << summary.all_skipped_reason(); 406 } else { 407 output << ", ok_count=" << summary.ok_count() 408 << ", not_ok_count=" << summary.not_ok_count(); 409 } 410 } 411 output << "}"; 412 return output; 413 } 414 415 416 /// Parses an input file containing the TAP output of a test program. 417 /// 418 /// \param filename Path to the file to parse. 419 /// 420 /// \return The parsed data in the form of a tap_summary. 421 /// 422 /// \throw load_error If there are any problems parsing the file. Such problems 423 /// should be considered as test program breakage. 424 engine::tap_summary 425 engine::parse_tap_output(const utils::fs::path& filename) 426 { 427 std::ifstream input(filename.str().c_str()); 428 if (!input) 429 throw engine::load_error(filename, "Failed to open TAP output file"); 430 431 try { 432 return tap_summary(tap_parser().parse(input)); 433 } catch (const engine::format_error& e) { 434 throw engine::load_error(filename, e.what()); 435 } catch (const text::error& e) { 436 throw engine::load_error(filename, e.what()); 437 } 438 } 439