1 // Copyright 2010 The Kyua Authors.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "cli/cmd_test.hpp"
30
31 #include <cstdlib>
32
33 #include "cli/common.ipp"
34 #include "drivers/run_tests.hpp"
35 #include "model/test_program.hpp"
36 #include "model/test_result.hpp"
37 #include "store/layout.hpp"
38 #include "utils/cmdline/options.hpp"
39 #include "utils/cmdline/parser.ipp"
40 #include "utils/cmdline/ui.hpp"
41 #include "utils/config/tree.ipp"
42 #include "utils/datetime.hpp"
43 #include "utils/format/macros.hpp"
44 #include "utils/fs/path.hpp"
45
46 namespace cmdline = utils::cmdline;
47 namespace config = utils::config;
48 namespace datetime = utils::datetime;
49 namespace fs = utils::fs;
50 namespace layout = store::layout;
51
52 using cli::cmd_test;
53
54
55 namespace {
56
57
58 /// Hooks to print a progress report of the execution of the tests.
59 class print_hooks : public drivers::run_tests::base_hooks {
60 /// Object to interact with the I/O of the program.
61 cmdline::ui* _ui;
62
63 /// Whether the tests are executed in parallel or not.
64 bool _parallel;
65
66 public:
67 /// The amount of test results per type.
68 std::map<enum model::test_result_type, unsigned long> type_count;
69
70 /// Constructor for the hooks.
71 ///
72 /// \param ui_ Object to interact with the I/O of the program.
73 /// \param parallel_ True if we are executing more than one test at once.
print_hooks(cmdline::ui * ui_,const bool parallel_)74 print_hooks(cmdline::ui* ui_, const bool parallel_) :
75 _ui(ui_),
76 _parallel(parallel_)
77 {
78 for (const auto& pair : model::test_result_types)
79 type_count[pair.first] = 0;
80 }
81
82 /// Called when the processing of a test case begins.
83 ///
84 /// \param test_program The test program containing the test case.
85 /// \param test_case_name The name of the test case being executed.
86 virtual void
got_test_case(const model::test_program & test_program,const std::string & test_case_name)87 got_test_case(const model::test_program& test_program,
88 const std::string& test_case_name)
89 {
90 if (!_parallel) {
91 _ui->out(F("%s -> ") %
92 cli::format_test_case_id(test_program, test_case_name),
93 false);
94 }
95 }
96
97 /// Called when a result of a test case becomes available.
98 ///
99 /// \param test_program The test program containing the test case.
100 /// \param test_case_name The name of the test case being executed.
101 /// \param result The result of the execution of the test case.
102 /// \param duration The time it took to run the test.
103 virtual void
got_result(const model::test_program & test_program,const std::string & test_case_name,const model::test_result & result,const datetime::delta & duration)104 got_result(const model::test_program& test_program,
105 const std::string& test_case_name,
106 const model::test_result& result,
107 const datetime::delta& duration)
108 {
109 if (_parallel) {
110 _ui->out(F("%s -> ") %
111 cli::format_test_case_id(test_program, test_case_name),
112 false);
113 }
114 _ui->out(F("%s [%s]") % cli::format_result(result) %
115 cli::format_delta(duration));
116
117 type_count[result.type()]++;
118 }
119 };
120
121
122 } // anonymous namespace
123
124
125 /// Default constructor for cmd_test.
cmd_test(void)126 cmd_test::cmd_test(void) : cli_command(
127 "test", "[test-program ...]", 0, -1, "Run tests")
128 {
129 add_option(build_root_option);
130 add_option(kyuafile_option);
131 add_option(results_file_create_option);
132 }
133
134
135 /// Entry point for the "test" subcommand.
136 ///
137 /// \param ui Object to interact with the I/O of the program.
138 /// \param cmdline Representation of the command line to the subcommand.
139 /// \param user_config The runtime configuration of the program.
140 ///
141 /// \return 0 if all tests passed, 1 otherwise.
142 int
run(cmdline::ui * ui,const cmdline::parsed_cmdline & cmdline,const config::tree & user_config)143 cmd_test::run(cmdline::ui* ui, const cmdline::parsed_cmdline& cmdline,
144 const config::tree& user_config)
145 {
146 const layout::results_id_file_pair results = layout::new_db(
147 results_file_create(cmdline), kyuafile_path(cmdline).branch_path());
148
149 const bool parallel = (user_config.lookup< config::positive_int_node >(
150 "parallelism") > 1);
151
152 print_hooks hooks(ui, parallel);
153 const drivers::run_tests::result result = drivers::run_tests::drive(
154 kyuafile_path(cmdline), build_root_path(cmdline), results.second,
155 parse_filters(cmdline.arguments()), user_config, hooks);
156
157 unsigned long total = 0;
158 unsigned long good = 0;
159 unsigned long bad = 0;
160 for (const auto& pair : model::test_result_types) {
161 const auto& type = pair.second;
162 const auto count = hooks.type_count[type.id];
163 total += count;
164 if (type.is_run && type.is_good)
165 good += count;
166 if (!type.is_good)
167 bad += count;
168 }
169
170 int exit_code;
171 if (total > 0) {
172 ui->out("");
173 if (!results.first.empty()) {
174 ui->out(F("Results file id is %s") % results.first);
175 }
176 ui->out(F("Results saved to %s") % results.second);
177 ui->out("");
178
179 ui->out(F("%s/%s passed (") % good % total, false);
180 const auto& types = model::test_result_types;
181 for (auto it = types.begin(); it != types.end(); it++) {
182 const auto& type = it->second;
183 if (!type.is_run || !type.is_good) {
184 if (it != types.begin())
185 ui->out(", ", false);
186 ui->out(F("%s %s") % hooks.type_count[type.id] % type.name,
187 false);
188 }
189 }
190 ui->out(")");
191
192 exit_code = (bad == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
193 } else {
194 // TODO(jmmv): Delete created empty file; it's useless!
195 if (!results.first.empty()) {
196 ui->out(F("Results file id is %s") % results.first);
197 }
198 ui->out(F("Results saved to %s") % results.second);
199 exit_code = EXIT_SUCCESS;
200 }
201
202 return report_unused_filters(result.unused_filters, ui) ?
203 EXIT_FAILURE : exit_code;
204 }
205