1 // Copyright 2011 The Kyua Authors. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // * Neither the name of Google Inc. nor the names of its contributors 14 // may be used to endorse or promote products derived from this software 15 // without specific prior written permission. 16 // 17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 #include "drivers/scan_results.hpp" 30 31 #include <set> 32 33 #include <atf-c++.hpp> 34 35 #include "engine/filters.hpp" 36 #include "model/context.hpp" 37 #include "model/metadata.hpp" 38 #include "model/test_case.hpp" 39 #include "model/test_program.hpp" 40 #include "model/test_result.hpp" 41 #include "store/exceptions.hpp" 42 #include "store/read_transaction.hpp" 43 #include "store/write_backend.hpp" 44 #include "store/write_transaction.hpp" 45 #include "utils/datetime.hpp" 46 #include "utils/format/containers.ipp" 47 #include "utils/format/macros.hpp" 48 #include "utils/optional.ipp" 49 #include "utils/sanity.hpp" 50 51 namespace datetime = utils::datetime; 52 namespace fs = utils::fs; 53 54 using utils::none; 55 using utils::optional; 56 57 58 namespace { 59 60 61 /// Records the callback values for futher investigation. 62 class capture_hooks : public drivers::scan_results::base_hooks { 63 public: 64 /// Whether begin() was called or not. 65 bool _begin_called; 66 67 /// The captured driver result, if any. 68 optional< drivers::scan_results::result > _end_result; 69 70 /// The captured context, if any. 71 optional< model::context > _context; 72 73 /// The captured results, flattened as "program:test_case:result". 74 std::set< std::string > _results; 75 76 /// Constructor. 77 capture_hooks(void) : 78 _begin_called(false) 79 { 80 } 81 82 /// Callback executed before any operation is performed. 83 void 84 begin(void) 85 { 86 _begin_called = true; 87 } 88 89 /// Callback executed after all operations are performed. 90 /// 91 /// \param r A structure with all results computed by this driver. Note 92 /// that this is also returned by the drive operation. 93 void 94 end(const drivers::scan_results::result& r) 95 { 96 PRE(!_end_result); 97 _end_result = r; 98 } 99 100 /// Callback executed when the context is loaded. 101 /// 102 /// \param context The context loaded from the database. 103 void got_context(const model::context& context) 104 { 105 PRE(!_context); 106 _context = context; 107 } 108 109 /// Callback executed when a test results is found. 110 /// 111 /// \param iter Container for the test result's data. 112 void got_result(store::results_iterator& iter) 113 { 114 const char* type; 115 switch (iter.result().type()) { 116 case model::test_result_passed: type = "passed"; break; 117 case model::test_result_skipped: type = "skipped"; break; 118 default: 119 UNREACHABLE_MSG("Formatting unimplemented"); 120 } 121 const datetime::delta duration = iter.end_time() - iter.start_time(); 122 _results.insert(F("%s:%s:%s:%s:%s:%s") % 123 iter.test_program()->absolute_path() % 124 iter.test_case_name() % type % iter.result().reason() % 125 duration.seconds % duration.useconds); 126 } 127 }; 128 129 130 /// Populates a results file. 131 /// 132 /// It is not OK to call this function multiple times on the same file. 133 /// 134 /// \param db_name The database to update. 135 /// \param count The number of "elements" to insert into the results file. 136 /// Determines the number of test programs and the number of test cases 137 /// each has. Can be used to determine from the caller which particular 138 /// results file has been loaded. 139 static void 140 populate_results_file(const char* db_name, const int count) 141 { 142 store::write_backend backend = store::write_backend::open_rw( 143 fs::path(db_name)); 144 145 store::write_transaction tx = backend.start_write(); 146 147 std::map< std::string, std::string > env; 148 for (int i = 0; i < count; i++) 149 env[F("VAR%s") % i] = F("Value %s") % i; 150 const model::context context(fs::path("/root"), env); 151 tx.put_context(context); 152 153 for (int i = 0; i < count; i++) { 154 model::test_program_builder test_program_builder( 155 "fake", fs::path(F("dir/prog_%s") % i), fs::path("/root"), 156 F("suite_%s") % i); 157 for (int j = 0; j < count; j++) { 158 test_program_builder.add_test_case(F("case_%s") % j); 159 } 160 const model::test_program test_program = test_program_builder.build(); 161 const int64_t tp_id = tx.put_test_program(test_program); 162 163 for (int j = 0; j < count; j++) { 164 const model::test_result result(model::test_result_skipped, 165 F("Count %s") % j); 166 const int64_t tc_id = tx.put_test_case(test_program, 167 F("case_%s") % j, tp_id); 168 const datetime::timestamp start = 169 datetime::timestamp::from_microseconds(1000010); 170 const datetime::timestamp end = 171 datetime::timestamp::from_microseconds(5000020 + i + j); 172 tx.put_result(result, tc_id, start, end); 173 } 174 } 175 176 tx.commit(); 177 } 178 179 180 } // anonymous namespace 181 182 183 ATF_TEST_CASE_WITHOUT_HEAD(ok__all); 184 ATF_TEST_CASE_BODY(ok__all) 185 { 186 populate_results_file("test.db", 2); 187 188 capture_hooks hooks; 189 const drivers::scan_results::result result = drivers::scan_results::drive( 190 fs::path("test.db"), std::set< engine::test_filter >(), hooks); 191 ATF_REQUIRE(result.unused_filters.empty()); 192 ATF_REQUIRE(hooks._begin_called); 193 ATF_REQUIRE(hooks._end_result); 194 195 std::map< std::string, std::string > env; 196 env["VAR0"] = "Value 0"; 197 env["VAR1"] = "Value 1"; 198 const model::context context(fs::path("/root"), env); 199 ATF_REQUIRE(context == hooks._context.get()); 200 201 std::set< std::string > results; 202 results.insert("/root/dir/prog_0:case_0:skipped:Count 0:4:10"); 203 results.insert("/root/dir/prog_0:case_1:skipped:Count 1:4:11"); 204 results.insert("/root/dir/prog_1:case_0:skipped:Count 0:4:11"); 205 results.insert("/root/dir/prog_1:case_1:skipped:Count 1:4:12"); 206 ATF_REQUIRE_EQ(results, hooks._results); 207 } 208 209 210 ATF_TEST_CASE_WITHOUT_HEAD(ok__filters); 211 ATF_TEST_CASE_BODY(ok__filters) 212 { 213 populate_results_file("test.db", 3); 214 215 std::set< engine::test_filter > filters; 216 filters.insert(engine::test_filter(fs::path("dir/prog_1"), "")); 217 filters.insert(engine::test_filter(fs::path("dir/prog_1"), "no")); 218 filters.insert(engine::test_filter(fs::path("dir/prog_2"), "case_1")); 219 filters.insert(engine::test_filter(fs::path("dir/prog_3"), "")); 220 221 capture_hooks hooks; 222 const drivers::scan_results::result result = drivers::scan_results::drive( 223 fs::path("test.db"), filters, hooks); 224 ATF_REQUIRE(hooks._begin_called); 225 ATF_REQUIRE(hooks._end_result); 226 227 std::set< engine::test_filter > unused_filters; 228 unused_filters.insert(engine::test_filter(fs::path("dir/prog_1"), "no")); 229 unused_filters.insert(engine::test_filter(fs::path("dir/prog_3"), "")); 230 ATF_REQUIRE_EQ(unused_filters, result.unused_filters); 231 232 std::set< std::string > results; 233 results.insert("/root/dir/prog_1:case_0:skipped:Count 0:4:11"); 234 results.insert("/root/dir/prog_1:case_1:skipped:Count 1:4:12"); 235 results.insert("/root/dir/prog_1:case_2:skipped:Count 2:4:13"); 236 results.insert("/root/dir/prog_2:case_1:skipped:Count 1:4:13"); 237 ATF_REQUIRE_EQ(results, hooks._results); 238 } 239 240 241 ATF_TEST_CASE_WITHOUT_HEAD(missing_db); 242 ATF_TEST_CASE_BODY(missing_db) 243 { 244 capture_hooks hooks; 245 ATF_REQUIRE_THROW( 246 store::error, 247 drivers::scan_results::drive(fs::path("test.db"), 248 std::set< engine::test_filter >(), 249 hooks)); 250 } 251 252 253 ATF_INIT_TEST_CASES(tcs) 254 { 255 ATF_ADD_TEST_CASE(tcs, ok__all); 256 ATF_ADD_TEST_CASE(tcs, ok__filters); 257 ATF_ADD_TEST_CASE(tcs, missing_db); 258 } 259