1 // Copyright 2012 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "cli/cmd_report_html.hpp" 30 31 #include <algorithm> 32 #include <cerrno> 33 #include <cstdlib> 34 #include <set> 35 #include <stdexcept> 36 37 #include "cli/common.ipp" 38 #include "drivers/scan_results.hpp" 39 #include "engine/filters.hpp" 40 #include "model/context.hpp" 41 #include "model/metadata.hpp" 42 #include "model/test_case.hpp" 43 #include "model/test_program.hpp" 44 #include "model/test_result.hpp" 45 #include "store/layout.hpp" 46 #include "store/read_transaction.hpp" 47 #include "utils/cmdline/options.hpp" 48 #include "utils/cmdline/parser.ipp" 49 #include "utils/cmdline/ui.hpp" 50 #include "utils/datetime.hpp" 51 #include "utils/env.hpp" 52 #include "utils/format/macros.hpp" 53 #include "utils/fs/exceptions.hpp" 54 #include "utils/fs/operations.hpp" 55 #include "utils/fs/path.hpp" 56 #include "utils/optional.ipp" 57 #include "utils/text/templates.hpp" 58 59 namespace cmdline = utils::cmdline; 60 namespace config = utils::config; 61 namespace datetime = utils::datetime; 62 namespace fs = utils::fs; 63 namespace layout = store::layout; 64 namespace text = utils::text; 65 66 using utils::optional; 67 68 69 namespace { 70 71 72 /// Creates the report's top directory and fails if it exists. 73 /// 74 /// \param directory The directory to create. 75 /// \param force Whether to wipe an existing directory or not. 76 /// 77 /// \throw std::runtime_error If the directory already exists; this is a user 78 /// error that the user must correct. 79 /// \throw fs::error If the directory creation fails for any other reason. 80 static void 81 create_top_directory(const fs::path& directory, const bool force) 82 { 83 if (force) { 84 if (fs::exists(directory)) 85 fs::rm_r(directory); 86 } 87 88 try { 89 fs::mkdir(directory, 0755); 90 } catch (const fs::system_error& e) { 91 if (e.original_errno() == EEXIST) 92 throw std::runtime_error(F("Output directory '%s' already exists; " 93 "maybe use --force?") % 94 directory); 95 else 96 throw e; 97 } 98 } 99 100 101 /// Generates a flat unique filename for a given test case. 102 /// 103 /// \param test_program The test program for which to genereate the name. 104 /// \param test_case_name The test case name. 105 /// 106 /// \return A filename unique within a directory with a trailing HTML extension. 107 static std::string 108 test_case_filename(const model::test_program& test_program, 109 const std::string& test_case_name) 110 { 111 static const char* special_characters = "/:"; 112 113 std::string name = cli::format_test_case_id(test_program, test_case_name); 114 std::string::size_type pos = name.find_first_of(special_characters); 115 while (pos != std::string::npos) { 116 name.replace(pos, 1, "_"); 117 pos = name.find_first_of(special_characters, pos + 1); 118 } 119 return name + ".html"; 120 } 121 122 123 /// Adds a string to string map to the templates. 124 /// 125 /// \param [in,out] templates The templates to add the map to. 126 /// \param props The map to add to the templates. 127 /// \param key_vector Name of the template vector that holds the keys. 128 /// \param value_vector Name of the template vector that holds the values. 129 static void 130 add_map(text::templates_def& templates, const config::properties_map& props, 131 const std::string& key_vector, const std::string& value_vector) 132 { 133 templates.add_vector(key_vector); 134 templates.add_vector(value_vector); 135 136 for (config::properties_map::const_iterator iter = props.begin(); 137 iter != props.end(); ++iter) { 138 templates.add_to_vector(key_vector, (*iter).first); 139 templates.add_to_vector(value_vector, (*iter).second); 140 } 141 } 142 143 144 /// Generates an HTML report. 145 class html_hooks : public drivers::scan_results::base_hooks { 146 /// User interface object where to report progress. 147 cmdline::ui* _ui; 148 149 /// The top directory in which to create the HTML files. 150 fs::path _directory; 151 152 /// Collection of result types to include in the report. 153 const cli::result_types& _results_filters; 154 155 /// The start time of the first test. 156 optional< utils::datetime::timestamp > _start_time; 157 158 /// The end time of the last test. 159 optional< utils::datetime::timestamp > _end_time; 160 161 /// The total run time of the tests. Note that we cannot subtract _end_time 162 /// from _start_time to compute this due to parallel execution. 163 utils::datetime::delta _runtime; 164 165 /// Templates accumulator to generate the index.html file. 166 text::templates_def _summary_templates; 167 168 /// Mapping of result types to the amount of tests with such result. 169 std::map< model::test_result_type, std::size_t > _types_count; 170 171 /// Generates a common set of templates for all of our files. 172 /// 173 /// \return A new templates object with common parameters. 174 static text::templates_def 175 common_templates(void) 176 { 177 text::templates_def templates; 178 templates.add_variable("css", "report.css"); 179 return templates; 180 } 181 182 /// Adds a test case result to the summary. 183 /// 184 /// \param test_program The test program with the test case to be added. 185 /// \param test_case_name Name of the test case. 186 /// \param result The result of the test case. 187 /// \param has_detail If true, the result of the test case has not been 188 /// filtered and therefore there exists a separate file for the test 189 /// with all of its information. 190 void 191 add_to_summary(const model::test_program& test_program, 192 const std::string& test_case_name, 193 const model::test_result& result, 194 const bool has_detail) 195 { 196 ++_types_count[result.type()]; 197 198 if (!has_detail) 199 return; 200 201 std::string test_cases_vector; 202 std::string test_cases_file_vector; 203 switch (result.type()) { 204 case model::test_result_broken: 205 test_cases_vector = "broken_test_cases"; 206 test_cases_file_vector = "broken_test_cases_file"; 207 break; 208 209 case model::test_result_expected_failure: 210 test_cases_vector = "xfail_test_cases"; 211 test_cases_file_vector = "xfail_test_cases_file"; 212 break; 213 214 case model::test_result_failed: 215 test_cases_vector = "failed_test_cases"; 216 test_cases_file_vector = "failed_test_cases_file"; 217 break; 218 219 case model::test_result_passed: 220 test_cases_vector = "passed_test_cases"; 221 test_cases_file_vector = "passed_test_cases_file"; 222 break; 223 224 case model::test_result_skipped: 225 test_cases_vector = "skipped_test_cases"; 226 test_cases_file_vector = "skipped_test_cases_file"; 227 break; 228 } 229 INV(!test_cases_vector.empty()); 230 INV(!test_cases_file_vector.empty()); 231 232 _summary_templates.add_to_vector( 233 test_cases_vector, 234 cli::format_test_case_id(test_program, test_case_name)); 235 _summary_templates.add_to_vector( 236 test_cases_file_vector, 237 test_case_filename(test_program, test_case_name)); 238 } 239 240 /// Instantiate a template to generate an HTML file in the output directory. 241 /// 242 /// \param templates The templates to use. 243 /// \param template_name The name of the template. This is automatically 244 /// searched for in the installed directory, so do not provide a path. 245 /// \param output_name The name of the output file. This is a basename to 246 /// be created within the output directory. 247 /// 248 /// \throw text::error If there is any problem applying the templates. 249 void 250 generate(const text::templates_def& templates, 251 const std::string& template_name, 252 const std::string& output_name) const 253 { 254 const fs::path miscdir(utils::getenv_with_default( 255 "KYUA_MISCDIR", KYUA_MISCDIR)); 256 const fs::path template_file = miscdir / template_name; 257 const fs::path output_path(_directory / output_name); 258 259 _ui->out(F("Generating %s") % output_path); 260 text::instantiate(templates, template_file, output_path); 261 } 262 263 /// Gets the number of tests with a given result type. 264 /// 265 /// \param type The type to be queried. 266 /// 267 /// \return The number of tests of the given type, or 0 if none have yet 268 /// been registered by add_to_summary(). 269 std::size_t 270 get_count(const model::test_result_type type) const 271 { 272 const std::map< model::test_result_type, std::size_t >::const_iterator 273 iter = _types_count.find(type); 274 if (iter == _types_count.end()) 275 return 0; 276 else 277 return (*iter).second; 278 } 279 280 public: 281 /// Constructor for the hooks. 282 /// 283 /// \param ui_ User interface object where to report progress. 284 /// \param directory_ The directory in which to create the HTML files. 285 /// \param results_filters_ The result types to include in the report. 286 /// Cannot be empty. 287 html_hooks(cmdline::ui* ui_, const fs::path& directory_, 288 const cli::result_types& results_filters_) : 289 _ui(ui_), 290 _directory(directory_), 291 _results_filters(results_filters_), 292 _summary_templates(common_templates()) 293 { 294 PRE(!results_filters_.empty()); 295 296 // Keep in sync with add_to_summary(). 297 _summary_templates.add_vector("broken_test_cases"); 298 _summary_templates.add_vector("broken_test_cases_file"); 299 _summary_templates.add_vector("xfail_test_cases"); 300 _summary_templates.add_vector("xfail_test_cases_file"); 301 _summary_templates.add_vector("failed_test_cases"); 302 _summary_templates.add_vector("failed_test_cases_file"); 303 _summary_templates.add_vector("passed_test_cases"); 304 _summary_templates.add_vector("passed_test_cases_file"); 305 _summary_templates.add_vector("skipped_test_cases"); 306 _summary_templates.add_vector("skipped_test_cases_file"); 307 } 308 309 /// Callback executed when the context is loaded. 310 /// 311 /// \param context The context loaded from the database. 312 void 313 got_context(const model::context& context) 314 { 315 text::templates_def templates = common_templates(); 316 templates.add_variable("cwd", context.cwd().str()); 317 add_map(templates, context.env(), "env_var", "env_var_value"); 318 generate(templates, "context.html", "context.html"); 319 } 320 321 /// Callback executed when a test results is found. 322 /// 323 /// \param iter Container for the test result's data. 324 void 325 got_result(store::results_iterator& iter) 326 { 327 const model::test_program_ptr test_program = iter.test_program(); 328 const std::string& test_case_name = iter.test_case_name(); 329 const model::test_result result = iter.result(); 330 331 if (std::find(_results_filters.begin(), _results_filters.end(), 332 result.type()) == _results_filters.end()) { 333 add_to_summary(*test_program, test_case_name, result, false); 334 return; 335 } 336 337 add_to_summary(*test_program, test_case_name, result, true); 338 339 if (!_start_time || _start_time.get() > iter.start_time()) 340 _start_time = iter.start_time(); 341 if (!_end_time || _end_time.get() < iter.end_time()) 342 _end_time = iter.end_time(); 343 344 const datetime::delta duration = iter.end_time() - iter.start_time(); 345 346 _runtime += duration; 347 348 text::templates_def templates = common_templates(); 349 templates.add_variable("test_case", 350 cli::format_test_case_id(*test_program, 351 test_case_name)); 352 templates.add_variable("test_program", 353 test_program->absolute_path().str()); 354 templates.add_variable("result", cli::format_result(result)); 355 templates.add_variable("start_time", 356 iter.start_time().to_iso8601_in_utc()); 357 templates.add_variable("end_time", 358 iter.end_time().to_iso8601_in_utc()); 359 templates.add_variable("duration", cli::format_delta(duration)); 360 361 const model::test_case& test_case = test_program->find(test_case_name); 362 add_map(templates, test_case.get_metadata().to_properties(), 363 "metadata_var", "metadata_value"); 364 365 { 366 const std::string stdout_text = iter.stdout_contents(); 367 if (!stdout_text.empty()) 368 templates.add_variable("stdout", stdout_text); 369 } 370 { 371 const std::string stderr_text = iter.stderr_contents(); 372 if (!stderr_text.empty()) 373 templates.add_variable("stderr", stderr_text); 374 } 375 376 generate(templates, "test_result.html", 377 test_case_filename(*test_program, test_case_name)); 378 } 379 380 /// Writes the index.html file in the output directory. 381 /// 382 /// This should only be called once all the processing has been done; 383 /// i.e. when the scan_results driver returns. 384 void 385 write_summary(void) 386 { 387 const std::size_t n_passed = get_count(model::test_result_passed); 388 const std::size_t n_failed = get_count(model::test_result_failed); 389 const std::size_t n_skipped = get_count(model::test_result_skipped); 390 const std::size_t n_xfail = get_count( 391 model::test_result_expected_failure); 392 const std::size_t n_broken = get_count(model::test_result_broken); 393 394 const std::size_t n_bad = n_broken + n_failed; 395 396 if (_start_time) { 397 INV(_end_time); 398 _summary_templates.add_variable( 399 "start_time", _start_time.get().to_iso8601_in_utc()); 400 _summary_templates.add_variable( 401 "end_time", _end_time.get().to_iso8601_in_utc()); 402 } else { 403 _summary_templates.add_variable("start_time", "No tests run"); 404 _summary_templates.add_variable("end_time", "No tests run"); 405 } 406 _summary_templates.add_variable("duration", 407 cli::format_delta(_runtime)); 408 _summary_templates.add_variable("passed_tests_count", 409 F("%s") % n_passed); 410 _summary_templates.add_variable("failed_tests_count", 411 F("%s") % n_failed); 412 _summary_templates.add_variable("skipped_tests_count", 413 F("%s") % n_skipped); 414 _summary_templates.add_variable("xfail_tests_count", 415 F("%s") % n_xfail); 416 _summary_templates.add_variable("broken_tests_count", 417 F("%s") % n_broken); 418 _summary_templates.add_variable("bad_tests_count", F("%s") % n_bad); 419 420 generate(text::templates_def(), "report.css", "report.css"); 421 generate(_summary_templates, "index.html", "index.html"); 422 } 423 }; 424 425 426 } // anonymous namespace 427 428 429 /// Default constructor for cmd_report_html. 430 cli::cmd_report_html::cmd_report_html(void) : cli_command( 431 "report-html", "", 0, 0, 432 "Generates an HTML report with the result of a test suite run") 433 { 434 add_option(results_file_open_option); 435 add_option(cmdline::bool_option( 436 "force", "Wipe the output directory before generating the new report; " 437 "use care")); 438 add_option(cmdline::path_option( 439 "output", "The directory in which to store the HTML files", 440 "path", "html")); 441 add_option(cmdline::list_option( 442 "results-filter", "Comma-separated list of result types to include in " 443 "the report", "types", "skipped,xfail,broken,failed")); 444 } 445 446 447 /// Entry point for the "report-html" subcommand. 448 /// 449 /// \param ui Object to interact with the I/O of the program. 450 /// \param cmdline Representation of the command line to the subcommand. 451 /// 452 /// \return 0 if everything is OK, 1 if the statement is invalid or if there is 453 /// any other problem. 454 int 455 cli::cmd_report_html::run(cmdline::ui* ui, 456 const cmdline::parsed_cmdline& cmdline, 457 const config::tree& /* user_config */) 458 { 459 const result_types types = get_result_types(cmdline); 460 461 const fs::path results_file = layout::find_results( 462 results_file_open(cmdline)); 463 464 const fs::path directory = 465 cmdline.get_option< cmdline::path_option >("output"); 466 create_top_directory(directory, cmdline.has_option("force")); 467 html_hooks hooks(ui, directory, types); 468 drivers::scan_results::drive(results_file, 469 std::set< engine::test_filter >(), 470 hooks); 471 hooks.write_summary(); 472 473 return EXIT_SUCCESS; 474 } 475