3 * Copyright (C)1997-2012 Net Integration Technologies and contributors.
4 * Licensed under the GNU Library General Public License, version 2.
5 * See the included file named LICENSE for license information.
6 * You can get wvtest from: http://github.com/apenwarr/wvtest
24 #ifdef HAVE_VALGRIND_MEMCHECK_H
25 # include <valgrind/memcheck.h>
26 # include <valgrind/valgrind.h>
28 # define VALGRIND_COUNT_ERRORS 0
29 # define VALGRIND_DO_LEAK_CHECK
30 # define VALGRIND_COUNT_LEAKS(a,b,c,d) (a=b=c=d=0)
33 #define MAX_TEST_TIME 40 // max seconds for a single test to run
34 #define MAX_TOTAL_TIME 120*60 // max seconds for the entire suite to run
36 #define TEST_START_FORMAT "! %s:%-5d %-40s "
40 return (int)VALGRIND_COUNT_ERRORS;
45 int leaked = 0, dubious = 0, reachable = 0, suppressed = 0;
46 VALGRIND_DO_LEAK_CHECK;
47 VALGRIND_COUNT_LEAKS(leaked, dubious, reachable, suppressed);
48 printf("memleaks: sure:%d dubious:%d reachable:%d suppress:%d\n",
49 leaked, dubious, reachable, suppressed);
52 // dubious+reachable are normally non-zero because of globals...
53 // return leaked+dubious+reachable;
57 // Return 1 if no children are running or zombies, 0 if there are any running
58 // or zombie children.
59 // Will wait for any already-terminated children first.
60 // Passes if no rogue children were running, fails otherwise.
61 // If your test gets a failure in here, either you're not killing all your
62 // children, or you're not calling waitpid(2) on all of them.
63 static bool no_running_children()
68 // Acknowledge and complain about any zombie children
72 wait_result = waitpid(-1, &status, WNOHANG);
77 snprintf(buf, sizeof(buf) - 1, "%d", wait_result);
78 buf[sizeof(buf)-1] = '\0';
79 WVFAILEQ("Unclaimed dead child process", buf);
81 } while (wait_result > 0);
83 // There should not be any running children, so waitpid should return -1
84 WVPASSEQ(errno, ECHILD);
85 WVPASSEQ(wait_result, -1);
86 return (wait_result == -1 && errno == ECHILD);
92 WvTest *WvTest::first, *WvTest::last;
93 int WvTest::fails, WvTest::runs, WvTest::xpasses, WvTest::xfails, WvTest::skips;
94 time_t WvTest::start_time;
95 bool WvTest::run_twice = false;
97 void WvTest::alarm_handler(int)
99 printf("\n! WvTest Current test took longer than %d seconds! FAILED\n",
106 static const char *pathstrip(const char *filename)
109 cptr = strrchr(filename, '/');
110 if (cptr) filename = cptr + 1;
111 cptr = strrchr(filename, '\\');
112 if (cptr) filename = cptr + 1;
117 WvTest::WvTest(const char *_descr, const char *_idstr, MainFunc *_main,
120 idstr(pathstrip(_idstr)),
133 static bool prefix_match(const char *s, const char * const *prefixes)
135 for (const char * const *prefix = prefixes; prefix && *prefix; prefix++)
137 if (!strncasecmp(s, *prefix, strlen(*prefix)))
144 int WvTest::run_all(const char * const *prefixes)
146 int old_valgrind_errs = 0, new_valgrind_errs;
147 int old_valgrind_leaks = 0, new_valgrind_leaks;
150 /* I should be doing something to do with SetTimer here,
151 * not sure exactly what just yet */
153 char *disable(getenv("WVTEST_DISABLE_TIMEOUT"));
154 if (disable != NULL && disable[0] != '\0' && disable[0] != '0')
155 signal(SIGALRM, SIG_IGN);
157 signal(SIGALRM, alarm_handler);
158 alarm(MAX_TEST_TIME);
160 start_time = time(NULL);
162 // make sure we can always start out in the same directory, so tests have
163 // access to their files. If a test uses chdir(), we want to be able to
166 if (!getcwd(wd, sizeof(wd)))
169 const char *slowstr1 = getenv("WVTEST_MIN_SLOWNESS");
170 const char *slowstr2 = getenv("WVTEST_MAX_SLOWNESS");
171 int min_slowness = 0, max_slowness = 65535;
172 if (slowstr1) min_slowness = atoi(slowstr1);
173 if (slowstr2) max_slowness = atoi(slowstr2);
178 char *parallel_str = getenv("WVTEST_PARALLEL");
180 run_twice = atoi(parallel_str) > 0;
183 // there are lots of fflush() calls in here because stupid win32 doesn't
184 // flush very often by itself.
185 fails = runs = xpasses = xfails = skips = 0;
186 for (WvTest *cur = first; cur; cur = cur->next)
188 if (cur->slowness <= max_slowness
189 && cur->slowness >= min_slowness
191 || prefix_match(cur->idstr, prefixes)
192 || prefix_match(cur->descr, prefixes)))
195 // set SIGPIPE back to default, helps catch tests which don't set
196 // this signal to SIG_IGN (which is almost always what you want)
198 signal(SIGPIPE, SIG_DFL);
203 // I see everything twice!
204 printf("Running test in parallel.\n");
209 printf("\nTesting \"%s\" in %s:\n", cur->descr, cur->idstr);
214 perror("Unable to change back to original directory");
217 new_valgrind_errs = memerrs();
218 WVPASS(new_valgrind_errs == old_valgrind_errs);
219 old_valgrind_errs = new_valgrind_errs;
221 new_valgrind_leaks = memleaks();
222 WVPASS(new_valgrind_leaks == old_valgrind_leaks);
223 old_valgrind_leaks = new_valgrind_leaks;
234 // I see everything once!
235 printf("Child exiting.\n");
240 printf("Waiting for child to exit.\n");
242 while ((result = waitpid(child, NULL, 0)) == -1 &&
244 printf("Waitpid interrupted, retrying.\n");
249 WVPASS(no_running_children());
255 if (prefixes && *prefixes && **prefixes)
256 printf("WvTest: WARNING: only ran tests starting with "
257 "specifed prefix(es).\n");
259 printf("WvTest: ran all tests.\n");
260 printf("WvTest: %d test%s, %d failure%s.\n",
261 runs, runs==1 ? "" : "s",
262 fails, fails==1 ? "": "s");
263 printf("WvTest: %d test%s skipped, %d known breakage%s, %d fixed breakage%s.\n",
264 skips, skips==1 ? "" : "s",
265 xfails, xfails==1 ? "" : "s",
266 xpasses, xpasses==1 ? "" : "s");
273 // If we aren't running in parallel, we want to output the name of the test
274 // before we run it, so we know what happened if it crashes. If we are
275 // running in parallel, outputting this information in multiple printf()s
276 // can confuse parsers, so we want to output everything in one printf().
278 // This function gets called by both start() and check(). If we're not
279 // running in parallel, just print the data. If we're running in parallel,
280 // and we're starting a test, save a copy of the file/line/description until
281 // the test is done and we can output it all at once.
283 // Yes, this is probably the worst API of all time.
284 void WvTest::print_result(bool start, const char *_file, int _line,
285 const char *_condstr, const char *result)
288 static char *condstr;
297 file = strdup(pathstrip(_file));
298 condstr = strdup(_condstr);
301 for (char *cptr = condstr; *cptr; cptr++)
303 if (!isprint((unsigned char)*cptr))
311 printf(TEST_START_FORMAT "%s\n", file, line, condstr, result);
316 printf(TEST_START_FORMAT, file, line, condstr);
318 printf("%s\n", result);
328 file = condstr = NULL;
333 void WvTest::start(const char *file, int line, const char *condstr)
335 // Either print the file, line, and condstr, or save them for later.
336 print_result(true, file, line, condstr, NULL);
340 void WvTest::check_prologue()
343 alarm(MAX_TEST_TIME); // restart per-test timeout
345 if (!start_time) start_time = time(NULL);
347 if (time(NULL) - start_time > MAX_TOTAL_TIME)
349 printf("\n! WvTest Total run time exceeded %d seconds! FAILED\n",
360 void WvTest::check(bool cond)
363 print_result(false, NULL, 0, NULL, cond ? "ok" : "FAILED");
369 if (getenv("WVTEST_DIE_FAST"))
375 void WvTest::check_xfail(bool cond)
378 print_result(false, NULL, 0, NULL, cond ? "xpass ok" : "xfail ok");
387 void WvTest::skip(const char *file, int line, const char *condstr)
389 start(file, line, condstr);
390 print_result(false, NULL, 0, NULL, "skip ok");
395 bool WvTest::start_check_eq(const char *file, int line,
396 const char *a, const char *b, bool expect_pass)
401 size_t len = strlen(a) + strlen(b) + 8 + 1;
402 char *str = new char[len];
403 sprintf(str, "[%s] %s [%s]", a, expect_pass ? "==" : "!=", b);
405 start(file, line, str);
408 bool cond = !strcmp(a, b);
417 bool WvTest::start_check_eq(const char *file, int line,
418 const std::string &a, const std::string &b,
421 return start_check_eq(file, line, a.c_str(), b.c_str(), expect_pass);
425 bool WvTest::start_check_eq(const char *file, int line,
426 int a, int b, bool expect_pass)
428 size_t len = 128 + 128 + 8 + 1;
429 char *str = new char[len];
430 sprintf(str, "%d %s %d", a, expect_pass ? "==" : "!=", b);
432 start(file, line, str);
435 bool cond = (a == b);
444 bool WvTest::start_check_eq(const char *file, int line,
445 double a, double b, double c, bool expect_pass)
447 size_t len = 128 + 128 + 128 + 8 + 1;
448 char *str = new char[len];
449 sprintf(str, "%f %s %f eps %f", a, expect_pass ? "==" : "!=", b, c);
451 start(file, line, str);
454 bool cond = ( WVABS(a - b) <= WVABS(c) );
463 bool WvTest::start_check_lt(const char *file, int line,
464 const char *a, const char *b)
469 size_t len = strlen(a) + strlen(b) + 8 + 1;
470 char *str = new char[len];
471 sprintf(str, "[%s] < [%s]", a, b);
473 start(file, line, str);
476 bool cond = strcmp(a, b) < 0;
482 bool WvTest::start_check_lt(const char *file, int line, int a, int b)
484 size_t len = 128 + 128 + 8 + 1;
485 char *str = new char[len];
486 sprintf(str, "%d < %d", a, b);
488 start(file, line, str);