1 // Copyright 2011 The Kyua Authors.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "drivers/scan_results.hpp"
30
31 #include <set>
32
33 #include <atf-c++.hpp>
34
35 #include "engine/filters.hpp"
36 #include "model/context.hpp"
37 #include "model/metadata.hpp"
38 #include "model/test_case.hpp"
39 #include "model/test_program.hpp"
40 #include "model/test_result.hpp"
41 #include "store/exceptions.hpp"
42 #include "store/read_transaction.hpp"
43 #include "store/write_backend.hpp"
44 #include "store/write_transaction.hpp"
45 #include "utils/datetime.hpp"
46 #include "utils/format/containers.ipp"
47 #include "utils/format/macros.hpp"
48 #include "utils/optional.ipp"
49 #include "utils/sanity.hpp"
50
51 namespace datetime = utils::datetime;
52 namespace fs = utils::fs;
53
54 using utils::none;
55 using utils::optional;
56
57
58 namespace {
59
60
61 /// Records the callback values for futher investigation.
62 class capture_hooks : public drivers::scan_results::base_hooks {
63 public:
64 /// Whether begin() was called or not.
65 bool _begin_called;
66
67 /// The captured driver result, if any.
68 optional< drivers::scan_results::result > _end_result;
69
70 /// The captured context, if any.
71 optional< model::context > _context;
72
73 /// The captured results, flattened as "program:test_case:result".
74 std::set< std::string > _results;
75
76 /// Constructor.
capture_hooks(void)77 capture_hooks(void) :
78 _begin_called(false)
79 {
80 }
81
82 /// Callback executed before any operation is performed.
83 void
begin(void)84 begin(void)
85 {
86 _begin_called = true;
87 }
88
89 /// Callback executed after all operations are performed.
90 ///
91 /// \param r A structure with all results computed by this driver. Note
92 /// that this is also returned by the drive operation.
93 void
end(const drivers::scan_results::result & r)94 end(const drivers::scan_results::result& r)
95 {
96 PRE(!_end_result);
97 _end_result = r;
98 }
99
100 /// Callback executed when the context is loaded.
101 ///
102 /// \param context The context loaded from the database.
got_context(const model::context & context)103 void got_context(const model::context& context)
104 {
105 PRE(!_context);
106 _context = context;
107 }
108
109 /// Callback executed when a test results is found.
110 ///
111 /// \param iter Container for the test result's data.
got_result(store::results_iterator & iter)112 void got_result(store::results_iterator& iter)
113 {
114 const char* type;
115 switch (iter.result().type()) {
116 case model::test_result_passed: type = "passed"; break;
117 case model::test_result_skipped: type = "skipped"; break;
118 default:
119 UNREACHABLE_MSG("Formatting unimplemented");
120 }
121 const datetime::delta duration = iter.end_time() - iter.start_time();
122 _results.insert(F("%s:%s:%s:%s:%s:%s") %
123 iter.test_program()->absolute_path() %
124 iter.test_case_name() % type % iter.result().reason() %
125 duration.seconds % duration.useconds);
126 }
127 };
128
129
130 /// Populates a results file.
131 ///
132 /// It is not OK to call this function multiple times on the same file.
133 ///
134 /// \param db_name The database to update.
135 /// \param count The number of "elements" to insert into the results file.
136 /// Determines the number of test programs and the number of test cases
137 /// each has. Can be used to determine from the caller which particular
138 /// results file has been loaded.
139 static void
populate_results_file(const char * db_name,const int count)140 populate_results_file(const char* db_name, const int count)
141 {
142 store::write_backend backend = store::write_backend::open_rw(
143 fs::path(db_name));
144
145 store::write_transaction tx = backend.start_write();
146
147 std::map< std::string, std::string > env;
148 for (int i = 0; i < count; i++)
149 env[F("VAR%s") % i] = F("Value %s") % i;
150 const model::context context(fs::path("/root"), env);
151 tx.put_context(context);
152
153 for (int i = 0; i < count; i++) {
154 model::test_program_builder test_program_builder(
155 "fake", fs::path(F("dir/prog_%s") % i), fs::path("/root"),
156 F("suite_%s") % i);
157 for (int j = 0; j < count; j++) {
158 test_program_builder.add_test_case(F("case_%s") % j);
159 }
160 const model::test_program test_program = test_program_builder.build();
161 const int64_t tp_id = tx.put_test_program(test_program);
162
163 for (int j = 0; j < count; j++) {
164 const model::test_result result(model::test_result_skipped,
165 F("Count %s") % j);
166 const int64_t tc_id = tx.put_test_case(test_program,
167 F("case_%s") % j, tp_id);
168 const datetime::timestamp start =
169 datetime::timestamp::from_microseconds(1000010);
170 const datetime::timestamp end =
171 datetime::timestamp::from_microseconds(5000020 + i + j);
172 tx.put_result(result, tc_id, start, end);
173 }
174 }
175
176 tx.commit();
177 }
178
179
180 } // anonymous namespace
181
182
183 ATF_TEST_CASE_WITHOUT_HEAD(ok__all);
ATF_TEST_CASE_BODY(ok__all)184 ATF_TEST_CASE_BODY(ok__all)
185 {
186 populate_results_file("test.db", 2);
187
188 capture_hooks hooks;
189 const drivers::scan_results::result result = drivers::scan_results::drive(
190 fs::path("test.db"), std::set< engine::test_filter >(), hooks);
191 ATF_REQUIRE(result.unused_filters.empty());
192 ATF_REQUIRE(hooks._begin_called);
193 ATF_REQUIRE(hooks._end_result);
194
195 std::map< std::string, std::string > env;
196 env["VAR0"] = "Value 0";
197 env["VAR1"] = "Value 1";
198 const model::context context(fs::path("/root"), env);
199 ATF_REQUIRE(context == hooks._context.get());
200
201 std::set< std::string > results;
202 results.insert("/root/dir/prog_0:case_0:skipped:Count 0:4:10");
203 results.insert("/root/dir/prog_0:case_1:skipped:Count 1:4:11");
204 results.insert("/root/dir/prog_1:case_0:skipped:Count 0:4:11");
205 results.insert("/root/dir/prog_1:case_1:skipped:Count 1:4:12");
206 ATF_REQUIRE_EQ(results, hooks._results);
207 }
208
209
210 ATF_TEST_CASE_WITHOUT_HEAD(ok__filters);
ATF_TEST_CASE_BODY(ok__filters)211 ATF_TEST_CASE_BODY(ok__filters)
212 {
213 populate_results_file("test.db", 3);
214
215 std::set< engine::test_filter > filters;
216 filters.insert(engine::test_filter(fs::path("dir/prog_1"), ""));
217 filters.insert(engine::test_filter(fs::path("dir/prog_1"), "no"));
218 filters.insert(engine::test_filter(fs::path("dir/prog_2"), "case_1"));
219 filters.insert(engine::test_filter(fs::path("dir/prog_3"), ""));
220
221 capture_hooks hooks;
222 const drivers::scan_results::result result = drivers::scan_results::drive(
223 fs::path("test.db"), filters, hooks);
224 ATF_REQUIRE(hooks._begin_called);
225 ATF_REQUIRE(hooks._end_result);
226
227 std::set< engine::test_filter > unused_filters;
228 unused_filters.insert(engine::test_filter(fs::path("dir/prog_1"), "no"));
229 unused_filters.insert(engine::test_filter(fs::path("dir/prog_3"), ""));
230 ATF_REQUIRE_EQ(unused_filters, result.unused_filters);
231
232 std::set< std::string > results;
233 results.insert("/root/dir/prog_1:case_0:skipped:Count 0:4:11");
234 results.insert("/root/dir/prog_1:case_1:skipped:Count 1:4:12");
235 results.insert("/root/dir/prog_1:case_2:skipped:Count 2:4:13");
236 results.insert("/root/dir/prog_2:case_1:skipped:Count 1:4:13");
237 ATF_REQUIRE_EQ(results, hooks._results);
238 }
239
240
241 ATF_TEST_CASE_WITHOUT_HEAD(missing_db);
ATF_TEST_CASE_BODY(missing_db)242 ATF_TEST_CASE_BODY(missing_db)
243 {
244 capture_hooks hooks;
245 ATF_REQUIRE_THROW(
246 store::error,
247 drivers::scan_results::drive(fs::path("test.db"),
248 std::set< engine::test_filter >(),
249 hooks));
250 }
251
252
ATF_INIT_TEST_CASES(tcs)253 ATF_INIT_TEST_CASES(tcs)
254 {
255 ATF_ADD_TEST_CASE(tcs, ok__all);
256 ATF_ADD_TEST_CASE(tcs, ok__filters);
257 ATF_ADD_TEST_CASE(tcs, missing_db);
258 }
259