1 // Copyright 2010 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "cli/cmd_test.hpp" 30 31 #include <cstdlib> 32 33 #include "cli/common.ipp" 34 #include "drivers/run_tests.hpp" 35 #include "model/test_program.hpp" 36 #include "model/test_result.hpp" 37 #include "store/layout.hpp" 38 #include "utils/cmdline/options.hpp" 39 #include "utils/cmdline/parser.ipp" 40 #include "utils/cmdline/ui.hpp" 41 #include "utils/config/tree.ipp" 42 #include "utils/datetime.hpp" 43 #include "utils/format/macros.hpp" 44 #include "utils/fs/path.hpp" 45 46 namespace cmdline = utils::cmdline; 47 namespace config = utils::config; 48 namespace datetime = utils::datetime; 49 namespace fs = utils::fs; 50 namespace layout = store::layout; 51 52 using cli::cmd_test; 53 54 55 namespace { 56 57 58 /// Hooks to print a progress report of the execution of the tests. 59 class print_hooks : public drivers::run_tests::base_hooks { 60 /// Object to interact with the I/O of the program. 61 cmdline::ui* _ui; 62 63 /// Whether the tests are executed in parallel or not. 64 bool _parallel; 65 66 public: 67 /// The amount of test results per type. 68 std::map<enum model::test_result_type, unsigned long> type_count; 69 70 /// Constructor for the hooks. 71 /// 72 /// \param ui_ Object to interact with the I/O of the program. 73 /// \param parallel_ True if we are executing more than one test at once. 74 print_hooks(cmdline::ui* ui_, const bool parallel_) : 75 _ui(ui_), 76 _parallel(parallel_) 77 { 78 for (const auto& pair : model::test_result_types) 79 type_count[pair.first] = 0; 80 } 81 82 /// Called when the processing of a test case begins. 83 /// 84 /// \param test_program The test program containing the test case. 85 /// \param test_case_name The name of the test case being executed. 86 virtual void 87 got_test_case(const model::test_program& test_program, 88 const std::string& test_case_name) 89 { 90 if (!_parallel) { 91 _ui->out(F("%s -> ") % 92 cli::format_test_case_id(test_program, test_case_name), 93 false); 94 } 95 } 96 97 /// Called when a result of a test case becomes available. 98 /// 99 /// \param test_program The test program containing the test case. 100 /// \param test_case_name The name of the test case being executed. 101 /// \param result The result of the execution of the test case. 102 /// \param duration The time it took to run the test. 103 virtual void 104 got_result(const model::test_program& test_program, 105 const std::string& test_case_name, 106 const model::test_result& result, 107 const datetime::delta& duration) 108 { 109 if (_parallel) { 110 _ui->out(F("%s -> ") % 111 cli::format_test_case_id(test_program, test_case_name), 112 false); 113 } 114 _ui->out(F("%s [%s]") % cli::format_result(result) % 115 cli::format_delta(duration)); 116 117 type_count[result.type()]++; 118 } 119 }; 120 121 122 } // anonymous namespace 123 124 125 /// Default constructor for cmd_test. 126 cmd_test::cmd_test(void) : cli_command( 127 "test", "[test-program ...]", 0, -1, "Run tests") 128 { 129 add_option(build_root_option); 130 add_option(kyuafile_option); 131 add_option(results_file_create_option); 132 } 133 134 135 /// Entry point for the "test" subcommand. 136 /// 137 /// \param ui Object to interact with the I/O of the program. 138 /// \param cmdline Representation of the command line to the subcommand. 139 /// \param user_config The runtime configuration of the program. 140 /// 141 /// \return 0 if all tests passed, 1 otherwise. 142 int 143 cmd_test::run(cmdline::ui* ui, const cmdline::parsed_cmdline& cmdline, 144 const config::tree& user_config) 145 { 146 const layout::results_id_file_pair results = layout::new_db( 147 results_file_create(cmdline), kyuafile_path(cmdline).branch_path()); 148 149 const bool parallel = (user_config.lookup< config::positive_int_node >( 150 "parallelism") > 1); 151 152 print_hooks hooks(ui, parallel); 153 const drivers::run_tests::result result = drivers::run_tests::drive( 154 kyuafile_path(cmdline), build_root_path(cmdline), results.second, 155 parse_filters(cmdline.arguments()), user_config, hooks); 156 157 unsigned long total = 0; 158 unsigned long good = 0; 159 unsigned long bad = 0; 160 for (const auto& pair : model::test_result_types) { 161 const auto& type = pair.second; 162 const auto count = hooks.type_count[type.id]; 163 total += count; 164 if (type.is_run && type.is_good) 165 good += count; 166 if (!type.is_good) 167 bad += count; 168 } 169 170 int exit_code; 171 if (total > 0) { 172 ui->out(""); 173 if (!results.first.empty()) { 174 ui->out(F("Results file id is %s") % results.first); 175 } 176 ui->out(F("Results saved to %s") % results.second); 177 ui->out(""); 178 179 ui->out(F("%s/%s passed (") % good % total, false); 180 const auto& types = model::test_result_types; 181 for (auto it = types.begin(); it != types.end(); it++) { 182 const auto& type = it->second; 183 if (!type.is_run || !type.is_good) { 184 if (it != types.begin()) 185 ui->out(", ", false); 186 ui->out(F("%s %s") % hooks.type_count[type.id] % type.name, 187 false); 188 } 189 } 190 ui->out(")"); 191 192 exit_code = (bad == 0 ? EXIT_SUCCESS : EXIT_FAILURE); 193 } else { 194 // TODO(jmmv): Delete created empty file; it's useless! 195 if (!results.first.empty()) { 196 ui->out(F("Results file id is %s") % results.first); 197 } 198 ui->out(F("Results saved to %s") % results.second); 199 exit_code = EXIT_SUCCESS; 200 } 201 202 return report_unused_filters(result.unused_filters, ui) ? 203 EXIT_FAILURE : exit_code; 204 } 205