xref: /freebsd/contrib/atf/atf-c++/tests.cpp (revision 4928135658a9d0eaee37003df6137ab363fcb0b4) 
1 // Copyright (c) 2007 The NetBSD Foundation, Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions
6 // are met:
7 // 1. Redistributions of source code must retain the above copyright
8 //    notice, this list of conditions and the following disclaimer.
9 // 2. Redistributions in binary form must reproduce the above copyright
10 //    notice, this list of conditions and the following disclaimer in the
11 //    documentation and/or other materials provided with the distribution.
12 //
13 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
14 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
15 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
16 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
18 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
20 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
24 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 
26 #include "atf-c++/tests.hpp"
27 
28 #if defined(HAVE_CONFIG_H)
29 #include "config.h"
30 #endif
31 
32 extern "C" {
33 #include <sys/types.h>
34 #include <sys/stat.h>
35 #include <sys/time.h>
36 #include <sys/wait.h>
37 #include <signal.h>
38 #include <unistd.h>
39 }
40 
41 #include <algorithm>
42 #include <cctype>
43 #include <cerrno>
44 #include <cstdlib>
45 #include <cstring>
46 #include <fstream>
47 #include <iostream>
48 #include <map>
49 #include <memory>
50 #include <sstream>
51 #include <stdexcept>
52 #include <vector>
53 
54 extern "C" {
55 #include "atf-c/error.h"
56 #include "atf-c/tc.h"
57 #include "atf-c/utils.h"
58 }
59 
60 #include "atf-c++/detail/application.hpp"
61 #include "atf-c++/detail/auto_array.hpp"
62 #include "atf-c++/detail/env.hpp"
63 #include "atf-c++/detail/exceptions.hpp"
64 #include "atf-c++/detail/fs.hpp"
65 #include "atf-c++/detail/sanity.hpp"
66 #include "atf-c++/detail/text.hpp"
67 
68 #if defined(HAVE_GNU_GETOPT)
69 #   define GETOPT_POSIX "+"
70 #else
71 #   define GETOPT_POSIX ""
72 #endif
73 
74 namespace impl = atf::tests;
75 namespace detail = atf::tests::detail;
76 #define IMPL_NAME "atf::tests"
77 
78 using atf::application::usage_error;
79 
80 // ------------------------------------------------------------------------
81 // The "atf_tp_writer" class.
82 // ------------------------------------------------------------------------
83 
84 detail::atf_tp_writer::atf_tp_writer(std::ostream& os) :
85     m_os(os),
86     m_is_first(true)
87 {
88     m_os << "Content-Type: application/X-atf-tp; version=\"1\"\n\n";
89 }
90 
91 void
92 detail::atf_tp_writer::start_tc(const std::string& ident)
93 {
94     if (!m_is_first)
95         m_os << "\n";
96     m_os << "ident: " << ident << "\n";
97     m_os.flush();
98 }
99 
100 void
101 detail::atf_tp_writer::end_tc(void)
102 {
103     if (m_is_first)
104         m_is_first = false;
105 }
106 
107 void
108 detail::atf_tp_writer::tc_meta_data(const std::string& name,
109                                     const std::string& value)
110 {
111     PRE(name != "ident");
112     m_os << name << ": " << value << "\n";
113     m_os.flush();
114 }
115 
116 // ------------------------------------------------------------------------
117 // Free helper functions.
118 // ------------------------------------------------------------------------
119 
120 std::string Program_Name;
121 
122 static void
123 set_program_name(const char* argv0)
124 {
125     const std::string program_name = atf::fs::path(argv0).leaf_name();
126     // Libtool workaround: if running from within the source tree (binaries
127     // that are not installed yet), skip the "lt-" prefix added to files in
128     // the ".libs" directory to show the real (not temporary) name.
129     if (program_name.substr(0, 3) == "lt-")
130         Program_Name = program_name.substr(3);
131     else
132         Program_Name = program_name;
133 }
134 
135 bool
136 detail::match(const std::string& regexp, const std::string& str)
137 {
138     return atf::text::match(str, regexp);
139 }
140 
141 // ------------------------------------------------------------------------
142 // The "tc" class.
143 // ------------------------------------------------------------------------
144 
145 static std::map< atf_tc_t*, impl::tc* > wraps;
146 static std::map< const atf_tc_t*, const impl::tc* > cwraps;
147 
148 struct impl::tc_impl {
149 private:
150     // Non-copyable.
151     tc_impl(const tc_impl&);
152     tc_impl& operator=(const tc_impl&);
153 
154 public:
155     std::string m_ident;
156     atf_tc_t m_tc;
157     bool m_has_cleanup;
158 
159     tc_impl(const std::string& ident, const bool has_cleanup) :
160         m_ident(ident),
161         m_has_cleanup(has_cleanup)
162     {
163     }
164 
165     static void
166     wrap_head(atf_tc_t *tc)
167     {
168         std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc);
169         INV(iter != wraps.end());
170         (*iter).second->head();
171     }
172 
173     static void
174     wrap_body(const atf_tc_t *tc)
175     {
176         std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
177             cwraps.find(tc);
178         INV(iter != cwraps.end());
179         (*iter).second->body();
180     }
181 
182     static void
183     wrap_cleanup(const atf_tc_t *tc)
184     {
185         std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
186             cwraps.find(tc);
187         INV(iter != cwraps.end());
188         (*iter).second->cleanup();
189     }
190 };
191 
192 impl::tc::tc(const std::string& ident, const bool has_cleanup) :
193     pimpl(new tc_impl(ident, has_cleanup))
194 {
195 }
196 
197 impl::tc::~tc(void)
198 {
199     cwraps.erase(&pimpl->m_tc);
200     wraps.erase(&pimpl->m_tc);
201 
202     atf_tc_fini(&pimpl->m_tc);
203 }
204 
205 void
206 impl::tc::init(const vars_map& config)
207 {
208     atf_error_t err;
209 
210     auto_array< const char * > array(new const char*[(config.size() * 2) + 1]);
211     const char **ptr = array.get();
212     for (vars_map::const_iterator iter = config.begin();
213          iter != config.end(); iter++) {
214          *ptr = (*iter).first.c_str();
215          *(ptr + 1) = (*iter).second.c_str();
216          ptr += 2;
217     }
218     *ptr = NULL;
219 
220     wraps[&pimpl->m_tc] = this;
221     cwraps[&pimpl->m_tc] = this;
222 
223     err = atf_tc_init(&pimpl->m_tc, pimpl->m_ident.c_str(), pimpl->wrap_head,
224         pimpl->wrap_body, pimpl->m_has_cleanup ? pimpl->wrap_cleanup : NULL,
225         array.get());
226     if (atf_is_error(err))
227         throw_atf_error(err);
228 }
229 
230 bool
231 impl::tc::has_config_var(const std::string& var)
232     const
233 {
234     return atf_tc_has_config_var(&pimpl->m_tc, var.c_str());
235 }
236 
237 bool
238 impl::tc::has_md_var(const std::string& var)
239     const
240 {
241     return atf_tc_has_md_var(&pimpl->m_tc, var.c_str());
242 }
243 
244 const std::string
245 impl::tc::get_config_var(const std::string& var)
246     const
247 {
248     return atf_tc_get_config_var(&pimpl->m_tc, var.c_str());
249 }
250 
251 const std::string
252 impl::tc::get_config_var(const std::string& var, const std::string& defval)
253     const
254 {
255     return atf_tc_get_config_var_wd(&pimpl->m_tc, var.c_str(), defval.c_str());
256 }
257 
258 const std::string
259 impl::tc::get_md_var(const std::string& var)
260     const
261 {
262     return atf_tc_get_md_var(&pimpl->m_tc, var.c_str());
263 }
264 
265 const impl::vars_map
266 impl::tc::get_md_vars(void)
267     const
268 {
269     vars_map vars;
270 
271     char **array = atf_tc_get_md_vars(&pimpl->m_tc);
272     try {
273         char **ptr;
274         for (ptr = array; *ptr != NULL; ptr += 2)
275             vars[*ptr] = *(ptr + 1);
276     } catch (...) {
277         atf_utils_free_charpp(array);
278         throw;
279     }
280 
281     return vars;
282 }
283 
284 void
285 impl::tc::set_md_var(const std::string& var, const std::string& val)
286 {
287     atf_error_t err = atf_tc_set_md_var(&pimpl->m_tc, var.c_str(), val.c_str());
288     if (atf_is_error(err))
289         throw_atf_error(err);
290 }
291 
292 void
293 impl::tc::run(const std::string& resfile)
294     const
295 {
296     atf_error_t err = atf_tc_run(&pimpl->m_tc, resfile.c_str());
297     if (atf_is_error(err))
298         throw_atf_error(err);
299 }
300 
301 void
302 impl::tc::run_cleanup(void)
303     const
304 {
305     atf_error_t err = atf_tc_cleanup(&pimpl->m_tc);
306     if (atf_is_error(err))
307         throw_atf_error(err);
308 }
309 
310 void
311 impl::tc::head(void)
312 {
313 }
314 
315 void
316 impl::tc::cleanup(void)
317     const
318 {
319 }
320 
321 void
322 impl::tc::require_prog(const std::string& prog)
323     const
324 {
325     atf_tc_require_prog(prog.c_str());
326 }
327 
328 void
329 impl::tc::pass(void)
330 {
331     atf_tc_pass();
332 }
333 
334 void
335 impl::tc::fail(const std::string& reason)
336 {
337     atf_tc_fail("%s", reason.c_str());
338 }
339 
340 void
341 impl::tc::fail_nonfatal(const std::string& reason)
342 {
343     atf_tc_fail_nonfatal("%s", reason.c_str());
344 }
345 
346 void
347 impl::tc::skip(const std::string& reason)
348 {
349     atf_tc_skip("%s", reason.c_str());
350 }
351 
352 void
353 impl::tc::check_errno(const char* file, const int line, const int exp_errno,
354                       const char* expr_str, const bool result)
355 {
356     atf_tc_check_errno(file, line, exp_errno, expr_str, result);
357 }
358 
359 void
360 impl::tc::require_errno(const char* file, const int line, const int exp_errno,
361                         const char* expr_str, const bool result)
362 {
363     atf_tc_require_errno(file, line, exp_errno, expr_str, result);
364 }
365 
366 void
367 impl::tc::expect_pass(void)
368 {
369     atf_tc_expect_pass();
370 }
371 
372 void
373 impl::tc::expect_fail(const std::string& reason)
374 {
375     atf_tc_expect_fail("%s", reason.c_str());
376 }
377 
378 void
379 impl::tc::expect_exit(const int exitcode, const std::string& reason)
380 {
381     atf_tc_expect_exit(exitcode, "%s", reason.c_str());
382 }
383 
384 void
385 impl::tc::expect_signal(const int signo, const std::string& reason)
386 {
387     atf_tc_expect_signal(signo, "%s", reason.c_str());
388 }
389 
390 void
391 impl::tc::expect_death(const std::string& reason)
392 {
393     atf_tc_expect_death("%s", reason.c_str());
394 }
395 
396 void
397 impl::tc::expect_timeout(const std::string& reason)
398 {
399     atf_tc_expect_timeout("%s", reason.c_str());
400 }
401 
402 // ------------------------------------------------------------------------
403 // Test program main code.
404 // ------------------------------------------------------------------------
405 
406 namespace {
407 
408 typedef std::vector< impl::tc * > tc_vector;
409 
410 enum tc_part { BODY, CLEANUP };
411 
412 static void
413 parse_vflag(const std::string& str, atf::tests::vars_map& vars)
414 {
415     if (str.empty())
416         throw std::runtime_error("-v requires a non-empty argument");
417 
418     std::vector< std::string > ws = atf::text::split(str, "=");
419     if (ws.size() == 1 && str[str.length() - 1] == '=') {
420         vars[ws[0]] = "";
421     } else {
422         if (ws.size() != 2)
423             throw std::runtime_error("-v requires an argument of the form "
424                                      "var=value");
425 
426         vars[ws[0]] = ws[1];
427     }
428 }
429 
430 static atf::fs::path
431 handle_srcdir(const char* argv0, const std::string& srcdir_arg)
432 {
433     atf::fs::path srcdir(".");
434 
435     if (srcdir_arg.empty()) {
436         srcdir = atf::fs::path(argv0).branch_path();
437         if (srcdir.leaf_name() == ".libs")
438             srcdir = srcdir.branch_path();
439     } else
440         srcdir = atf::fs::path(srcdir_arg);
441 
442     if (!atf::fs::exists(srcdir / Program_Name))
443         throw usage_error("Cannot find the test program in the source "
444                           "directory `%s'", srcdir.c_str());
445 
446     if (!srcdir.is_absolute())
447         srcdir = srcdir.to_absolute();
448 
449     return srcdir;
450 }
451 
452 static void
453 init_tcs(void (*add_tcs)(tc_vector&), tc_vector& tcs,
454          const atf::tests::vars_map& vars)
455 {
456     add_tcs(tcs);
457     for (tc_vector::iterator iter = tcs.begin(); iter != tcs.end(); iter++) {
458         impl::tc* tc = *iter;
459 
460         tc->init(vars);
461     }
462 }
463 
464 static int
465 list_tcs(const tc_vector& tcs)
466 {
467     detail::atf_tp_writer writer(std::cout);
468 
469     for (tc_vector::const_iterator iter = tcs.begin();
470          iter != tcs.end(); iter++) {
471         const impl::vars_map vars = (*iter)->get_md_vars();
472 
473         {
474             impl::vars_map::const_iterator iter2 = vars.find("ident");
475             INV(iter2 != vars.end());
476             writer.start_tc((*iter2).second);
477         }
478 
479         for (impl::vars_map::const_iterator iter2 = vars.begin();
480              iter2 != vars.end(); iter2++) {
481             const std::string& key = (*iter2).first;
482             if (key != "ident")
483                 writer.tc_meta_data(key, (*iter2).second);
484         }
485 
486         writer.end_tc();
487     }
488 
489     return EXIT_SUCCESS;
490 }
491 
492 static impl::tc*
493 find_tc(tc_vector tcs, const std::string& name)
494 {
495     std::vector< std::string > ids;
496     for (tc_vector::iterator iter = tcs.begin();
497          iter != tcs.end(); iter++) {
498         impl::tc* tc = *iter;
499 
500         if (tc->get_md_var("ident") == name)
501             return tc;
502     }
503     throw usage_error("Unknown test case `%s'", name.c_str());
504 }
505 
506 static std::pair< std::string, tc_part >
507 process_tcarg(const std::string& tcarg)
508 {
509     const std::string::size_type pos = tcarg.find(':');
510     if (pos == std::string::npos) {
511         return std::make_pair(tcarg, BODY);
512     } else {
513         const std::string tcname = tcarg.substr(0, pos);
514 
515         const std::string partname = tcarg.substr(pos + 1);
516         if (partname == "body")
517             return std::make_pair(tcname, BODY);
518         else if (partname == "cleanup")
519             return std::make_pair(tcname, CLEANUP);
520         else {
521             throw usage_error("Invalid test case part `%s'", partname.c_str());
522         }
523     }
524 }
525 
526 static int
527 run_tc(tc_vector& tcs, const std::string& tcarg, const atf::fs::path& resfile)
528 {
529     const std::pair< std::string, tc_part > fields = process_tcarg(tcarg);
530 
531     impl::tc* tc = find_tc(tcs, fields.first);
532 
533     if (!atf::env::has("__RUNNING_INSIDE_ATF_RUN") || atf::env::get(
534         "__RUNNING_INSIDE_ATF_RUN") != "internal-yes-value")
535     {
536         std::cerr << Program_Name << ": WARNING: Running test cases outside "
537             "of kyua(1) is unsupported\n";
538         std::cerr << Program_Name << ": WARNING: No isolation nor timeout "
539             "control is being applied; you may get unexpected failures; see "
540             "atf-test-case(4)\n";
541     }
542 
543     switch (fields.second) {
544     case BODY:
545         tc->run(resfile.str());
546         break;
547     case CLEANUP:
548         tc->run_cleanup();
549         break;
550     default:
551         UNREACHABLE;
552     }
553     return EXIT_SUCCESS;
554 }
555 
556 static int
557 safe_main(int argc, char** argv, void (*add_tcs)(tc_vector&))
558 {
559     const char* argv0 = argv[0];
560 
561     bool lflag = false;
562     atf::fs::path resfile("/dev/stdout");
563     std::string srcdir_arg;
564     atf::tests::vars_map vars;
565 
566     int ch;
567     int old_opterr;
568 
569     old_opterr = opterr;
570     ::opterr = 0;
571     while ((ch = ::getopt(argc, argv, GETOPT_POSIX ":lr:s:v:")) != -1) {
572         switch (ch) {
573         case 'l':
574             lflag = true;
575             break;
576 
577         case 'r':
578             resfile = atf::fs::path(::optarg);
579             break;
580 
581         case 's':
582             srcdir_arg = ::optarg;
583             break;
584 
585         case 'v':
586             parse_vflag(::optarg, vars);
587             break;
588 
589         case ':':
590             throw usage_error("Option -%c requires an argument.", ::optopt);
591             break;
592 
593         case '?':
594         default:
595             throw usage_error("Unknown option -%c.", ::optopt);
596         }
597     }
598     argc -= optind;
599     argv += optind;
600 
601     // Clear getopt state just in case the test wants to use it.
602     ::opterr = old_opterr;
603     ::optind = 1;
604 #if defined(HAVE_OPTRESET)
605     ::optreset = 1;
606 #endif
607 
608     vars["srcdir"] = handle_srcdir(argv0, srcdir_arg).str();
609 
610     int errcode;
611 
612     tc_vector tcs;
613     if (lflag) {
614         if (argc > 0)
615             throw usage_error("Cannot provide test case names with -l");
616 
617         init_tcs(add_tcs, tcs, vars);
618         errcode = list_tcs(tcs);
619     } else {
620         if (argc == 0)
621             throw usage_error("Must provide a test case name");
622         else if (argc > 1)
623             throw usage_error("Cannot provide more than one test case name");
624         INV(argc == 1);
625 
626         init_tcs(add_tcs, tcs, vars);
627         errcode = run_tc(tcs, argv[0], resfile);
628     }
629     for (tc_vector::iterator iter = tcs.begin(); iter != tcs.end(); iter++) {
630         impl::tc* tc = *iter;
631 
632         delete tc;
633     }
634 
635     return errcode;
636 }
637 
638 }  // anonymous namespace
639 
640 namespace atf {
641     namespace tests {
642         int run_tp(int, char**, void (*)(tc_vector&));
643     }
644 }
645 
646 int
647 impl::run_tp(int argc, char** argv, void (*add_tcs)(tc_vector&))
648 {
649     try {
650         set_program_name(argv[0]);
651         return ::safe_main(argc, argv, add_tcs);
652     } catch (const usage_error& e) {
653         std::cerr
654             << Program_Name << ": ERROR: " << e.what() << '\n'
655             << Program_Name << ": See atf-test-program(1) for usage details.\n";
656         return EXIT_FAILURE;
657     }
658 }
659