xref: /freebsd/contrib/atf/atf-c++/tests.cpp (revision e960cb4c033c8f65af1e8b09310c1bd5414f21ac) 
1 // Copyright (c) 2007 The NetBSD Foundation, Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions
6 // are met:
7 // 1. Redistributions of source code must retain the above copyright
8 //    notice, this list of conditions and the following disclaimer.
9 // 2. Redistributions in binary form must reproduce the above copyright
10 //    notice, this list of conditions and the following disclaimer in the
11 //    documentation and/or other materials provided with the distribution.
12 //
13 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
14 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
15 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
16 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
18 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
20 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
24 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 
26 #include "atf-c++/tests.hpp"
27 
28 #if defined(HAVE_CONFIG_H)
29 #include "config.h"
30 #endif
31 
32 extern "C" {
33 #include <sys/types.h>
34 #include <sys/stat.h>
35 #include <sys/time.h>
36 #include <sys/wait.h>
37 #include <signal.h>
38 #include <unistd.h>
39 }
40 
41 #include <algorithm>
42 #include <cctype>
43 #include <cerrno>
44 #include <cstdlib>
45 #include <cstring>
46 #include <fstream>
47 #include <iostream>
48 #include <map>
49 #include <memory>
50 #include <sstream>
51 #include <stdexcept>
52 #include <vector>
53 
54 extern "C" {
55 #include "atf-c/error.h"
56 #include "atf-c/tc.h"
57 #include "atf-c/utils.h"
58 }
59 
60 #include "atf-c++/detail/application.hpp"
61 #include "atf-c++/detail/env.hpp"
62 #include "atf-c++/detail/exceptions.hpp"
63 #include "atf-c++/detail/fs.hpp"
64 #include "atf-c++/detail/sanity.hpp"
65 #include "atf-c++/detail/text.hpp"
66 
67 #if defined(HAVE_GNU_GETOPT)
68 #   define GETOPT_POSIX "+"
69 #else
70 #   define GETOPT_POSIX ""
71 #endif
72 
73 namespace impl = atf::tests;
74 namespace detail = atf::tests::detail;
75 #define IMPL_NAME "atf::tests"
76 
77 using atf::application::usage_error;
78 
79 // ------------------------------------------------------------------------
80 // The "atf_tp_writer" class.
81 // ------------------------------------------------------------------------
82 
83 detail::atf_tp_writer::atf_tp_writer(std::ostream& os) :
84     m_os(os),
85     m_is_first(true)
86 {
87     m_os << "Content-Type: application/X-atf-tp; version=\"1\"\n\n";
88 }
89 
90 void
91 detail::atf_tp_writer::start_tc(const std::string& ident)
92 {
93     if (!m_is_first)
94         m_os << "\n";
95     m_os << "ident: " << ident << "\n";
96     m_os.flush();
97 }
98 
99 void
100 detail::atf_tp_writer::end_tc(void)
101 {
102     if (m_is_first)
103         m_is_first = false;
104 }
105 
106 void
107 detail::atf_tp_writer::tc_meta_data(const std::string& name,
108                                     const std::string& value)
109 {
110     PRE(name != "ident");
111     m_os << name << ": " << value << "\n";
112     m_os.flush();
113 }
114 
115 // ------------------------------------------------------------------------
116 // Free helper functions.
117 // ------------------------------------------------------------------------
118 
119 std::string Program_Name;
120 
121 static void
122 set_program_name(const char* argv0)
123 {
124     const std::string program_name = atf::fs::path(argv0).leaf_name();
125     Program_Name = program_name;
126 }
127 
128 bool
129 detail::match(const std::string& regexp, const std::string& str)
130 {
131     return atf::text::match(str, regexp);
132 }
133 
134 // ------------------------------------------------------------------------
135 // The "tc" class.
136 // ------------------------------------------------------------------------
137 
138 static std::map< atf_tc_t*, impl::tc* > wraps;
139 static std::map< const atf_tc_t*, const impl::tc* > cwraps;
140 
141 struct impl::tc_impl {
142 private:
143     // Non-copyable.
144     tc_impl(const tc_impl&);
145     tc_impl& operator=(const tc_impl&);
146 
147 public:
148     std::string m_ident;
149     atf_tc_t m_tc;
150     bool m_has_cleanup;
151 
152     tc_impl(const std::string& ident, const bool has_cleanup) :
153         m_ident(ident),
154         m_has_cleanup(has_cleanup)
155     {
156     }
157 
158     static void
159     wrap_head(atf_tc_t *tc)
160     {
161         std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc);
162         INV(iter != wraps.end());
163         (*iter).second->head();
164     }
165 
166     static void
167     wrap_body(const atf_tc_t *tc)
168     {
169         std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
170             cwraps.find(tc);
171         INV(iter != cwraps.end());
172         (*iter).second->body();
173     }
174 
175     static void
176     wrap_cleanup(const atf_tc_t *tc)
177     {
178         std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
179             cwraps.find(tc);
180         INV(iter != cwraps.end());
181         (*iter).second->cleanup();
182     }
183 };
184 
185 impl::tc::tc(const std::string& ident, const bool has_cleanup) :
186     pimpl(new tc_impl(ident, has_cleanup))
187 {
188 }
189 
190 impl::tc::~tc(void)
191 {
192     cwraps.erase(&pimpl->m_tc);
193     wraps.erase(&pimpl->m_tc);
194 
195     atf_tc_fini(&pimpl->m_tc);
196 }
197 
198 void
199 impl::tc::init(const vars_map& config)
200 {
201     atf_error_t err;
202 
203     std::vector< const char * > array;
204     array.reserve((config.size() * 2) + 1);
205 
206     for (vars_map::const_iterator iter = config.begin();
207          iter != config.end(); iter++) {
208          array.push_back((*iter).first.c_str());
209          array.push_back((*iter).second.c_str());
210     }
211     array.push_back(nullptr);
212 
213     wraps[&pimpl->m_tc] = this;
214     cwraps[&pimpl->m_tc] = this;
215 
216     err = atf_tc_init(&pimpl->m_tc, pimpl->m_ident.c_str(), pimpl->wrap_head,
217         pimpl->wrap_body, pimpl->m_has_cleanup ? pimpl->wrap_cleanup : NULL,
218         array.data());
219     if (atf_is_error(err))
220         throw_atf_error(err);
221 }
222 
223 bool
224 impl::tc::has_config_var(const std::string& var)
225     const
226 {
227     return atf_tc_has_config_var(&pimpl->m_tc, var.c_str());
228 }
229 
230 bool
231 impl::tc::has_md_var(const std::string& var)
232     const
233 {
234     return atf_tc_has_md_var(&pimpl->m_tc, var.c_str());
235 }
236 
237 const std::string
238 impl::tc::get_config_var(const std::string& var)
239     const
240 {
241     return atf_tc_get_config_var(&pimpl->m_tc, var.c_str());
242 }
243 
244 const std::string
245 impl::tc::get_config_var(const std::string& var, const std::string& defval)
246     const
247 {
248     return atf_tc_get_config_var_wd(&pimpl->m_tc, var.c_str(), defval.c_str());
249 }
250 
251 const std::string
252 impl::tc::get_md_var(const std::string& var)
253     const
254 {
255     return atf_tc_get_md_var(&pimpl->m_tc, var.c_str());
256 }
257 
258 const impl::vars_map
259 impl::tc::get_md_vars(void)
260     const
261 {
262     vars_map vars;
263 
264     char **array = atf_tc_get_md_vars(&pimpl->m_tc);
265     try {
266         char **ptr;
267         for (ptr = array; *ptr != NULL; ptr += 2)
268             vars[*ptr] = *(ptr + 1);
269         atf_utils_free_charpp(array);
270     } catch (...) {
271         atf_utils_free_charpp(array);
272         throw;
273     }
274 
275     return vars;
276 }
277 
278 void
279 impl::tc::set_md_var(const std::string& var, const std::string& val)
280 {
281     atf_error_t err = atf_tc_set_md_var(&pimpl->m_tc, var.c_str(), val.c_str());
282     if (atf_is_error(err))
283         throw_atf_error(err);
284 }
285 
286 void
287 impl::tc::run(const std::string& resfile)
288     const
289 {
290     atf_error_t err = atf_tc_run(&pimpl->m_tc, resfile.c_str());
291     if (atf_is_error(err))
292         throw_atf_error(err);
293 }
294 
295 void
296 impl::tc::run_cleanup(void)
297     const
298 {
299     atf_error_t err = atf_tc_cleanup(&pimpl->m_tc);
300     if (atf_is_error(err))
301         throw_atf_error(err);
302 }
303 
304 void
305 impl::tc::head(void)
306 {
307 }
308 
309 void
310 impl::tc::cleanup(void)
311     const
312 {
313 }
314 
315 void
316 impl::tc::require_kmod(const std::string& kmod)
317     const
318 {
319     atf_tc_require_kmod(kmod.c_str());
320 }
321 
322 void
323 impl::tc::require_prog(const std::string& prog)
324     const
325 {
326     atf_tc_require_prog(prog.c_str());
327 }
328 
329 void
330 impl::tc::pass(void)
331 {
332     atf_tc_pass();
333 }
334 
335 void
336 impl::tc::fail(const std::string& reason)
337 {
338     atf_tc_fail("%s", reason.c_str());
339 }
340 
341 void
342 impl::tc::fail_nonfatal(const std::string& reason)
343 {
344     atf_tc_fail_nonfatal("%s", reason.c_str());
345 }
346 
347 void
348 impl::tc::skip(const std::string& reason)
349 {
350     atf_tc_skip("%s", reason.c_str());
351 }
352 
353 void
354 impl::tc::check_errno(const char* file, const int line, const int exp_errno,
355                       const char* expr_str, const bool result)
356 {
357     atf_tc_check_errno(file, line, exp_errno, expr_str, result);
358 }
359 
360 void
361 impl::tc::require_errno(const char* file, const int line, const int exp_errno,
362                         const char* expr_str, const bool result)
363 {
364     atf_tc_require_errno(file, line, exp_errno, expr_str, result);
365 }
366 
367 void
368 impl::tc::expect_pass(void)
369 {
370     atf_tc_expect_pass();
371 }
372 
373 void
374 impl::tc::expect_fail(const std::string& reason)
375 {
376     atf_tc_expect_fail("%s", reason.c_str());
377 }
378 
379 void
380 impl::tc::expect_exit(const int exitcode, const std::string& reason)
381 {
382     atf_tc_expect_exit(exitcode, "%s", reason.c_str());
383 }
384 
385 void
386 impl::tc::expect_signal(const int signo, const std::string& reason)
387 {
388     atf_tc_expect_signal(signo, "%s", reason.c_str());
389 }
390 
391 void
392 impl::tc::expect_death(const std::string& reason)
393 {
394     atf_tc_expect_death("%s", reason.c_str());
395 }
396 
397 void
398 impl::tc::expect_timeout(const std::string& reason)
399 {
400     atf_tc_expect_timeout("%s", reason.c_str());
401 }
402 
403 // ------------------------------------------------------------------------
404 // Test program main code.
405 // ------------------------------------------------------------------------
406 
407 namespace {
408 
409 typedef std::vector< impl::tc * > tc_vector;
410 
411 enum tc_part { BODY, CLEANUP };
412 
413 static void
414 parse_vflag(const std::string& str, atf::tests::vars_map& vars)
415 {
416     if (str.empty())
417         throw std::runtime_error("-v requires a non-empty argument");
418 
419     std::vector< std::string > ws = atf::text::split(str, "=");
420     if (ws.size() == 1 && str[str.length() - 1] == '=') {
421         vars[ws[0]] = "";
422     } else {
423         if (ws.size() != 2)
424             throw std::runtime_error("-v requires an argument of the form "
425                                      "var=value");
426 
427         vars[ws[0]] = ws[1];
428     }
429 }
430 
431 static atf::fs::path
432 handle_srcdir(const char* argv0, const std::string& srcdir_arg)
433 {
434     atf::fs::path srcdir(".");
435 
436     if (srcdir_arg.empty()) {
437         srcdir = atf::fs::path(argv0).branch_path();
438         if (srcdir.leaf_name() == ".libs")
439             srcdir = srcdir.branch_path();
440     } else
441         srcdir = atf::fs::path(srcdir_arg);
442 
443     if (!atf::fs::exists(srcdir / Program_Name))
444         throw usage_error("Cannot find the test program in the source "
445                           "directory `%s'", srcdir.c_str());
446 
447     if (!srcdir.is_absolute())
448         srcdir = srcdir.to_absolute();
449 
450     return srcdir;
451 }
452 
453 static void
454 init_tcs(void (*add_tcs)(tc_vector&), tc_vector& tcs,
455          const atf::tests::vars_map& vars)
456 {
457     add_tcs(tcs);
458     for (auto& tc : tcs) {
459         tc->init(vars);
460     }
461 }
462 
463 static int
464 list_tcs(const tc_vector& tcs)
465 {
466     detail::atf_tp_writer writer(std::cout);
467 
468     for (tc_vector::const_iterator iter = tcs.begin();
469          iter != tcs.end(); iter++) {
470         const impl::vars_map vars = (*iter)->get_md_vars();
471 
472         {
473             impl::vars_map::const_iterator iter2 = vars.find("ident");
474             INV(iter2 != vars.end());
475             writer.start_tc((*iter2).second);
476         }
477 
478         for (impl::vars_map::const_iterator iter2 = vars.begin();
479              iter2 != vars.end(); iter2++) {
480             const std::string& key = (*iter2).first;
481             if (key != "ident")
482                 writer.tc_meta_data(key, (*iter2).second);
483         }
484 
485         writer.end_tc();
486     }
487 
488     return EXIT_SUCCESS;
489 }
490 
491 static impl::tc*
492 find_tc(tc_vector tcs, const std::string& name)
493 {
494     std::vector< std::string > ids;
495     for (tc_vector::iterator iter = tcs.begin();
496          iter != tcs.end(); iter++) {
497         impl::tc* tc = *iter;
498 
499         if (tc->get_md_var("ident") == name)
500             return tc;
501     }
502     throw usage_error("Unknown test case `%s'", name.c_str());
503 }
504 
505 static std::pair< std::string, tc_part >
506 process_tcarg(const std::string& tcarg)
507 {
508     const std::string::size_type pos = tcarg.find(':');
509     if (pos == std::string::npos) {
510         return std::make_pair(tcarg, BODY);
511     } else {
512         const std::string tcname = tcarg.substr(0, pos);
513 
514         const std::string partname = tcarg.substr(pos + 1);
515         if (partname == "body")
516             return std::make_pair(tcname, BODY);
517         else if (partname == "cleanup")
518             return std::make_pair(tcname, CLEANUP);
519         else {
520             throw usage_error("Invalid test case part `%s'", partname.c_str());
521         }
522     }
523 }
524 
525 static int
526 run_tc(tc_vector& tcs, const std::string& tcarg, const atf::fs::path& resfile)
527 {
528     const std::pair< std::string, tc_part > fields = process_tcarg(tcarg);
529 
530     impl::tc* tc = find_tc(tcs, fields.first);
531 
532     if (!atf::env::has("__RUNNING_INSIDE_ATF_RUN") || atf::env::get(
533         "__RUNNING_INSIDE_ATF_RUN") != "internal-yes-value")
534     {
535         std::cerr << Program_Name << ": WARNING: Running test cases outside "
536             "of kyua(1) is unsupported\n";
537         std::cerr << Program_Name << ": WARNING: No isolation nor timeout "
538             "control is being applied; you may get unexpected failures; see "
539             "atf-test-case(4)\n";
540     }
541 
542     switch (fields.second) {
543     case BODY:
544         tc->run(resfile.str());
545         break;
546     case CLEANUP:
547         tc->run_cleanup();
548         break;
549     default:
550         UNREACHABLE;
551     }
552     return EXIT_SUCCESS;
553 }
554 
555 static int
556 safe_main(int argc, char** argv, void (*add_tcs)(tc_vector&))
557 {
558     const char* argv0 = argv[0];
559 
560     bool lflag = false;
561     atf::fs::path resfile("/dev/stdout");
562     std::string srcdir_arg;
563     atf::tests::vars_map vars;
564 
565     int ch;
566     int old_opterr;
567 
568     old_opterr = opterr;
569     ::opterr = 0;
570     while ((ch = ::getopt(argc, argv, GETOPT_POSIX ":lr:s:v:")) != -1) {
571         switch (ch) {
572         case 'l':
573             lflag = true;
574             break;
575 
576         case 'r':
577             resfile = atf::fs::path(::optarg);
578             break;
579 
580         case 's':
581             srcdir_arg = ::optarg;
582             break;
583 
584         case 'v':
585             parse_vflag(::optarg, vars);
586             break;
587 
588         case ':':
589             throw usage_error("Option -%c requires an argument.", ::optopt);
590             break;
591 
592         case '?':
593         default:
594             throw usage_error("Unknown option -%c.", ::optopt);
595         }
596     }
597     argc -= optind;
598     argv += optind;
599 
600     // Clear getopt state just in case the test wants to use it.
601     ::opterr = old_opterr;
602     ::optind = 1;
603 #if defined(HAVE_OPTRESET)
604     ::optreset = 1;
605 #endif
606 
607     vars["srcdir"] = handle_srcdir(argv0, srcdir_arg).str();
608 
609     int errcode;
610 
611     if (lflag) {
612         if (argc > 0)
613             throw usage_error("Cannot provide test case names with -l");
614     } else {
615         if (argc == 0)
616             throw usage_error("Must provide a test case name");
617         else if (argc > 1)
618             throw usage_error("Cannot provide more than one test case name");
619         INV(argc == 1);
620     }
621     tc_vector tcs;
622     try {
623         init_tcs(add_tcs, tcs, vars);
624         errcode = lflag ? list_tcs(tcs) : run_tc(tcs, argv[0], resfile);
625     } catch (...) {
626         for (auto& tc: tcs) {
627             delete tc;
628         }
629         throw;
630     }
631 
632     return errcode;
633 }
634 
635 }  // anonymous namespace
636 
637 namespace atf {
638     namespace tests {
639         int run_tp(int, char**, void (*)(tc_vector&));
640     }
641 }
642 
643 int
644 impl::run_tp(int argc, char** argv, void (*add_tcs)(tc_vector&))
645 {
646     try {
647         set_program_name(argv[0]);
648         return ::safe_main(argc, argv, add_tcs);
649     } catch (const usage_error& e) {
650         std::cerr
651             << Program_Name << ": ERROR: " << e.what() << '\n'
652             << Program_Name << ": See atf-test-program(1) for usage details.\n";
653         return EXIT_FAILURE;
654     }
655 }
656