xref: /freebsd/contrib/atf/atf-c++/tests.cpp (revision b5daf675efc746611c7cfcd1fa474b8905064c4b) 
1 // Copyright (c) 2007 The NetBSD Foundation, Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions
6 // are met:
7 // 1. Redistributions of source code must retain the above copyright
8 //    notice, this list of conditions and the following disclaimer.
9 // 2. Redistributions in binary form must reproduce the above copyright
10 //    notice, this list of conditions and the following disclaimer in the
11 //    documentation and/or other materials provided with the distribution.
12 //
13 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
14 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
15 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
16 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
18 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
20 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
24 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 
26 #include "atf-c++/tests.hpp"
27 
28 #if defined(HAVE_CONFIG_H)
29 #include "config.h"
30 #endif
31 
32 extern "C" {
33 #include <sys/types.h>
34 #include <sys/stat.h>
35 #include <sys/time.h>
36 #include <sys/wait.h>
37 #include <signal.h>
38 #include <unistd.h>
39 }
40 
41 #include <algorithm>
42 #include <cctype>
43 #include <cerrno>
44 #include <cstdlib>
45 #include <cstring>
46 #include <fstream>
47 #include <iostream>
48 #include <map>
49 #include <memory>
50 #include <sstream>
51 #include <stdexcept>
52 #include <vector>
53 
54 extern "C" {
55 #include "atf-c/error.h"
56 #include "atf-c/tc.h"
57 #include "atf-c/utils.h"
58 }
59 
60 #include "atf-c++/detail/application.hpp"
61 #include "atf-c++/detail/auto_array.hpp"
62 #include "atf-c++/detail/env.hpp"
63 #include "atf-c++/detail/exceptions.hpp"
64 #include "atf-c++/detail/fs.hpp"
65 #include "atf-c++/detail/sanity.hpp"
66 #include "atf-c++/detail/text.hpp"
67 
68 #if defined(HAVE_GNU_GETOPT)
69 #   define GETOPT_POSIX "+"
70 #else
71 #   define GETOPT_POSIX ""
72 #endif
73 
74 namespace impl = atf::tests;
75 namespace detail = atf::tests::detail;
76 #define IMPL_NAME "atf::tests"
77 
78 using atf::application::usage_error;
79 
80 // ------------------------------------------------------------------------
81 // The "atf_tp_writer" class.
82 // ------------------------------------------------------------------------
83 
84 detail::atf_tp_writer::atf_tp_writer(std::ostream& os) :
85     m_os(os),
86     m_is_first(true)
87 {
88     m_os << "Content-Type: application/X-atf-tp; version=\"1\"\n\n";
89 }
90 
91 void
92 detail::atf_tp_writer::start_tc(const std::string& ident)
93 {
94     if (!m_is_first)
95         m_os << "\n";
96     m_os << "ident: " << ident << "\n";
97     m_os.flush();
98 }
99 
100 void
101 detail::atf_tp_writer::end_tc(void)
102 {
103     if (m_is_first)
104         m_is_first = false;
105 }
106 
107 void
108 detail::atf_tp_writer::tc_meta_data(const std::string& name,
109                                     const std::string& value)
110 {
111     PRE(name != "ident");
112     m_os << name << ": " << value << "\n";
113     m_os.flush();
114 }
115 
116 // ------------------------------------------------------------------------
117 // Free helper functions.
118 // ------------------------------------------------------------------------
119 
120 std::string Program_Name;
121 
122 static void
123 set_program_name(const char* argv0)
124 {
125     const std::string program_name = atf::fs::path(argv0).leaf_name();
126     // Libtool workaround: if running from within the source tree (binaries
127     // that are not installed yet), skip the "lt-" prefix added to files in
128     // the ".libs" directory to show the real (not temporary) name.
129     if (program_name.substr(0, 3) == "lt-")
130         Program_Name = program_name.substr(3);
131     else
132         Program_Name = program_name;
133 }
134 
135 bool
136 detail::match(const std::string& regexp, const std::string& str)
137 {
138     return atf::text::match(str, regexp);
139 }
140 
141 // ------------------------------------------------------------------------
142 // The "tc" class.
143 // ------------------------------------------------------------------------
144 
145 static std::map< atf_tc_t*, impl::tc* > wraps;
146 static std::map< const atf_tc_t*, const impl::tc* > cwraps;
147 
148 struct impl::tc_impl {
149 private:
150     // Non-copyable.
151     tc_impl(const tc_impl&);
152     tc_impl& operator=(const tc_impl&);
153 
154 public:
155     std::string m_ident;
156     atf_tc_t m_tc;
157     bool m_has_cleanup;
158 
159     tc_impl(const std::string& ident, const bool has_cleanup) :
160         m_ident(ident),
161         m_has_cleanup(has_cleanup)
162     {
163     }
164 
165     static void
166     wrap_head(atf_tc_t *tc)
167     {
168         std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc);
169         INV(iter != wraps.end());
170         (*iter).second->head();
171     }
172 
173     static void
174     wrap_body(const atf_tc_t *tc)
175     {
176         std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
177             cwraps.find(tc);
178         INV(iter != cwraps.end());
179         (*iter).second->body();
180     }
181 
182     static void
183     wrap_cleanup(const atf_tc_t *tc)
184     {
185         std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
186             cwraps.find(tc);
187         INV(iter != cwraps.end());
188         (*iter).second->cleanup();
189     }
190 };
191 
192 impl::tc::tc(const std::string& ident, const bool has_cleanup) :
193     pimpl(new tc_impl(ident, has_cleanup))
194 {
195 }
196 
197 impl::tc::~tc(void)
198 {
199     cwraps.erase(&pimpl->m_tc);
200     wraps.erase(&pimpl->m_tc);
201 
202     atf_tc_fini(&pimpl->m_tc);
203 }
204 
205 void
206 impl::tc::init(const vars_map& config)
207 {
208     atf_error_t err;
209 
210     auto_array< const char * > array(new const char*[(config.size() * 2) + 1]);
211     const char **ptr = array.get();
212     for (vars_map::const_iterator iter = config.begin();
213          iter != config.end(); iter++) {
214          *ptr = (*iter).first.c_str();
215          *(ptr + 1) = (*iter).second.c_str();
216          ptr += 2;
217     }
218     *ptr = NULL;
219 
220     wraps[&pimpl->m_tc] = this;
221     cwraps[&pimpl->m_tc] = this;
222 
223     err = atf_tc_init(&pimpl->m_tc, pimpl->m_ident.c_str(), pimpl->wrap_head,
224         pimpl->wrap_body, pimpl->m_has_cleanup ? pimpl->wrap_cleanup : NULL,
225         array.get());
226     if (atf_is_error(err))
227         throw_atf_error(err);
228 }
229 
230 bool
231 impl::tc::has_config_var(const std::string& var)
232     const
233 {
234     return atf_tc_has_config_var(&pimpl->m_tc, var.c_str());
235 }
236 
237 bool
238 impl::tc::has_md_var(const std::string& var)
239     const
240 {
241     return atf_tc_has_md_var(&pimpl->m_tc, var.c_str());
242 }
243 
244 const std::string
245 impl::tc::get_config_var(const std::string& var)
246     const
247 {
248     return atf_tc_get_config_var(&pimpl->m_tc, var.c_str());
249 }
250 
251 const std::string
252 impl::tc::get_config_var(const std::string& var, const std::string& defval)
253     const
254 {
255     return atf_tc_get_config_var_wd(&pimpl->m_tc, var.c_str(), defval.c_str());
256 }
257 
258 const std::string
259 impl::tc::get_md_var(const std::string& var)
260     const
261 {
262     return atf_tc_get_md_var(&pimpl->m_tc, var.c_str());
263 }
264 
265 const impl::vars_map
266 impl::tc::get_md_vars(void)
267     const
268 {
269     vars_map vars;
270 
271     char **array = atf_tc_get_md_vars(&pimpl->m_tc);
272     try {
273         char **ptr;
274         for (ptr = array; *ptr != NULL; ptr += 2)
275             vars[*ptr] = *(ptr + 1);
276     } catch (...) {
277         atf_utils_free_charpp(array);
278         throw;
279     }
280 
281     return vars;
282 }
283 
284 void
285 impl::tc::set_md_var(const std::string& var, const std::string& val)
286 {
287     atf_error_t err = atf_tc_set_md_var(&pimpl->m_tc, var.c_str(), val.c_str());
288     if (atf_is_error(err))
289         throw_atf_error(err);
290 }
291 
292 void
293 impl::tc::run(const std::string& resfile)
294     const
295 {
296     atf_error_t err = atf_tc_run(&pimpl->m_tc, resfile.c_str());
297     if (atf_is_error(err))
298         throw_atf_error(err);
299 }
300 
301 void
302 impl::tc::run_cleanup(void)
303     const
304 {
305     atf_error_t err = atf_tc_cleanup(&pimpl->m_tc);
306     if (atf_is_error(err))
307         throw_atf_error(err);
308 }
309 
310 void
311 impl::tc::head(void)
312 {
313 }
314 
315 void
316 impl::tc::cleanup(void)
317     const
318 {
319 }
320 
321 void
322 impl::tc::require_kmod(const std::string& kmod)
323     const
324 {
325     atf_tc_require_kmod(kmod.c_str());
326 }
327 
328 void
329 impl::tc::require_prog(const std::string& prog)
330     const
331 {
332     atf_tc_require_prog(prog.c_str());
333 }
334 
335 void
336 impl::tc::pass(void)
337 {
338     atf_tc_pass();
339 }
340 
341 void
342 impl::tc::fail(const std::string& reason)
343 {
344     atf_tc_fail("%s", reason.c_str());
345 }
346 
347 void
348 impl::tc::fail_nonfatal(const std::string& reason)
349 {
350     atf_tc_fail_nonfatal("%s", reason.c_str());
351 }
352 
353 void
354 impl::tc::skip(const std::string& reason)
355 {
356     atf_tc_skip("%s", reason.c_str());
357 }
358 
359 void
360 impl::tc::check_errno(const char* file, const int line, const int exp_errno,
361                       const char* expr_str, const bool result)
362 {
363     atf_tc_check_errno(file, line, exp_errno, expr_str, result);
364 }
365 
366 void
367 impl::tc::require_errno(const char* file, const int line, const int exp_errno,
368                         const char* expr_str, const bool result)
369 {
370     atf_tc_require_errno(file, line, exp_errno, expr_str, result);
371 }
372 
373 void
374 impl::tc::expect_pass(void)
375 {
376     atf_tc_expect_pass();
377 }
378 
379 void
380 impl::tc::expect_fail(const std::string& reason)
381 {
382     atf_tc_expect_fail("%s", reason.c_str());
383 }
384 
385 void
386 impl::tc::expect_exit(const int exitcode, const std::string& reason)
387 {
388     atf_tc_expect_exit(exitcode, "%s", reason.c_str());
389 }
390 
391 void
392 impl::tc::expect_signal(const int signo, const std::string& reason)
393 {
394     atf_tc_expect_signal(signo, "%s", reason.c_str());
395 }
396 
397 void
398 impl::tc::expect_death(const std::string& reason)
399 {
400     atf_tc_expect_death("%s", reason.c_str());
401 }
402 
403 void
404 impl::tc::expect_timeout(const std::string& reason)
405 {
406     atf_tc_expect_timeout("%s", reason.c_str());
407 }
408 
409 // ------------------------------------------------------------------------
410 // Test program main code.
411 // ------------------------------------------------------------------------
412 
413 namespace {
414 
415 typedef std::vector< impl::tc * > tc_vector;
416 
417 enum tc_part { BODY, CLEANUP };
418 
419 static void
420 parse_vflag(const std::string& str, atf::tests::vars_map& vars)
421 {
422     if (str.empty())
423         throw std::runtime_error("-v requires a non-empty argument");
424 
425     std::vector< std::string > ws = atf::text::split(str, "=");
426     if (ws.size() == 1 && str[str.length() - 1] == '=') {
427         vars[ws[0]] = "";
428     } else {
429         if (ws.size() != 2)
430             throw std::runtime_error("-v requires an argument of the form "
431                                      "var=value");
432 
433         vars[ws[0]] = ws[1];
434     }
435 }
436 
437 static atf::fs::path
438 handle_srcdir(const char* argv0, const std::string& srcdir_arg)
439 {
440     atf::fs::path srcdir(".");
441 
442     if (srcdir_arg.empty()) {
443         srcdir = atf::fs::path(argv0).branch_path();
444         if (srcdir.leaf_name() == ".libs")
445             srcdir = srcdir.branch_path();
446     } else
447         srcdir = atf::fs::path(srcdir_arg);
448 
449     if (!atf::fs::exists(srcdir / Program_Name))
450         throw usage_error("Cannot find the test program in the source "
451                           "directory `%s'", srcdir.c_str());
452 
453     if (!srcdir.is_absolute())
454         srcdir = srcdir.to_absolute();
455 
456     return srcdir;
457 }
458 
459 static void
460 init_tcs(void (*add_tcs)(tc_vector&), tc_vector& tcs,
461          const atf::tests::vars_map& vars)
462 {
463     add_tcs(tcs);
464     for (tc_vector::iterator iter = tcs.begin(); iter != tcs.end(); iter++) {
465         impl::tc* tc = *iter;
466 
467         tc->init(vars);
468     }
469 }
470 
471 static int
472 list_tcs(const tc_vector& tcs)
473 {
474     detail::atf_tp_writer writer(std::cout);
475 
476     for (tc_vector::const_iterator iter = tcs.begin();
477          iter != tcs.end(); iter++) {
478         const impl::vars_map vars = (*iter)->get_md_vars();
479 
480         {
481             impl::vars_map::const_iterator iter2 = vars.find("ident");
482             INV(iter2 != vars.end());
483             writer.start_tc((*iter2).second);
484         }
485 
486         for (impl::vars_map::const_iterator iter2 = vars.begin();
487              iter2 != vars.end(); iter2++) {
488             const std::string& key = (*iter2).first;
489             if (key != "ident")
490                 writer.tc_meta_data(key, (*iter2).second);
491         }
492 
493         writer.end_tc();
494     }
495 
496     return EXIT_SUCCESS;
497 }
498 
499 static impl::tc*
500 find_tc(tc_vector tcs, const std::string& name)
501 {
502     std::vector< std::string > ids;
503     for (tc_vector::iterator iter = tcs.begin();
504          iter != tcs.end(); iter++) {
505         impl::tc* tc = *iter;
506 
507         if (tc->get_md_var("ident") == name)
508             return tc;
509     }
510     throw usage_error("Unknown test case `%s'", name.c_str());
511 }
512 
513 static std::pair< std::string, tc_part >
514 process_tcarg(const std::string& tcarg)
515 {
516     const std::string::size_type pos = tcarg.find(':');
517     if (pos == std::string::npos) {
518         return std::make_pair(tcarg, BODY);
519     } else {
520         const std::string tcname = tcarg.substr(0, pos);
521 
522         const std::string partname = tcarg.substr(pos + 1);
523         if (partname == "body")
524             return std::make_pair(tcname, BODY);
525         else if (partname == "cleanup")
526             return std::make_pair(tcname, CLEANUP);
527         else {
528             throw usage_error("Invalid test case part `%s'", partname.c_str());
529         }
530     }
531 }
532 
533 static int
534 run_tc(tc_vector& tcs, const std::string& tcarg, const atf::fs::path& resfile)
535 {
536     const std::pair< std::string, tc_part > fields = process_tcarg(tcarg);
537 
538     impl::tc* tc = find_tc(tcs, fields.first);
539 
540     if (!atf::env::has("__RUNNING_INSIDE_ATF_RUN") || atf::env::get(
541         "__RUNNING_INSIDE_ATF_RUN") != "internal-yes-value")
542     {
543         std::cerr << Program_Name << ": WARNING: Running test cases outside "
544             "of kyua(1) is unsupported\n";
545         std::cerr << Program_Name << ": WARNING: No isolation nor timeout "
546             "control is being applied; you may get unexpected failures; see "
547             "atf-test-case(4)\n";
548     }
549 
550     switch (fields.second) {
551     case BODY:
552         tc->run(resfile.str());
553         break;
554     case CLEANUP:
555         tc->run_cleanup();
556         break;
557     default:
558         UNREACHABLE;
559     }
560     return EXIT_SUCCESS;
561 }
562 
563 static int
564 safe_main(int argc, char** argv, void (*add_tcs)(tc_vector&))
565 {
566     const char* argv0 = argv[0];
567 
568     bool lflag = false;
569     atf::fs::path resfile("/dev/stdout");
570     std::string srcdir_arg;
571     atf::tests::vars_map vars;
572 
573     int ch;
574     int old_opterr;
575 
576     old_opterr = opterr;
577     ::opterr = 0;
578     while ((ch = ::getopt(argc, argv, GETOPT_POSIX ":lr:s:v:")) != -1) {
579         switch (ch) {
580         case 'l':
581             lflag = true;
582             break;
583 
584         case 'r':
585             resfile = atf::fs::path(::optarg);
586             break;
587 
588         case 's':
589             srcdir_arg = ::optarg;
590             break;
591 
592         case 'v':
593             parse_vflag(::optarg, vars);
594             break;
595 
596         case ':':
597             throw usage_error("Option -%c requires an argument.", ::optopt);
598             break;
599 
600         case '?':
601         default:
602             throw usage_error("Unknown option -%c.", ::optopt);
603         }
604     }
605     argc -= optind;
606     argv += optind;
607 
608     // Clear getopt state just in case the test wants to use it.
609     ::opterr = old_opterr;
610     ::optind = 1;
611 #if defined(HAVE_OPTRESET)
612     ::optreset = 1;
613 #endif
614 
615     vars["srcdir"] = handle_srcdir(argv0, srcdir_arg).str();
616 
617     int errcode;
618 
619     tc_vector tcs;
620     if (lflag) {
621         if (argc > 0)
622             throw usage_error("Cannot provide test case names with -l");
623 
624         init_tcs(add_tcs, tcs, vars);
625         errcode = list_tcs(tcs);
626     } else {
627         if (argc == 0)
628             throw usage_error("Must provide a test case name");
629         else if (argc > 1)
630             throw usage_error("Cannot provide more than one test case name");
631         INV(argc == 1);
632 
633         init_tcs(add_tcs, tcs, vars);
634         errcode = run_tc(tcs, argv[0], resfile);
635     }
636     for (tc_vector::iterator iter = tcs.begin(); iter != tcs.end(); iter++) {
637         impl::tc* tc = *iter;
638 
639         delete tc;
640     }
641 
642     return errcode;
643 }
644 
645 }  // anonymous namespace
646 
647 namespace atf {
648     namespace tests {
649         int run_tp(int, char**, void (*)(tc_vector&));
650     }
651 }
652 
653 int
654 impl::run_tp(int argc, char** argv, void (*add_tcs)(tc_vector&))
655 {
656     try {
657         set_program_name(argv[0]);
658         return ::safe_main(argc, argv, add_tcs);
659     } catch (const usage_error& e) {
660         std::cerr
661             << Program_Name << ": ERROR: " << e.what() << '\n'
662             << Program_Name << ": See atf-test-program(1) for usage details.\n";
663         return EXIT_FAILURE;
664     }
665 }
666