/* * Some utility routines for writing tests. * * Here are a variety of utility routines for writing tests compatible with * the TAP protocol. All routines of the form ok() or is*() take a test * number and some number of appropriate arguments, check to be sure the * results match the expected output using the arguments, and print out * something appropriate for that test number. Other utility routines help in * constructing more complex tests, skipping tests, reporting errors, setting * up the TAP output format, or finding things in the test environment. * * This file is part of C TAP Harness. The current version plus supporting * documentation is at . * * Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 * Russ Allbery * Copyright 2001, 2002, 2004, 2005, 2006, 2007, 2008, 2011, 2012, 2013, 2014 * The Board of Trustees of the Leland Stanford Junior University * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include /* * The test count. Always contains the number that will be used for the next * test status. This is exported to callers of the library. */ unsigned long testnum = 1; /* * Status information stored so that we can give a test summary at the end of * the test case. We store the planned final test and the count of failures. * We can get the highest test count from testnum. */ static unsigned long _planned = 0; static unsigned long _failed = 0; /* * Store the PID of the process that called plan() and only summarize * results when that process exits, so as to not misreport results in forked * processes. */ static pid_t _process = 0; /* * If true, we're doing lazy planning and will print out the plan based on the * last test number at the end of testing. */ static int _lazy = 0; /* * If true, the test was aborted by calling bail(). Currently, this is only * used to ensure that we pass a false value to any cleanup functions even if * all tests to that point have passed. */ static int _aborted = 0; /* * Registered cleanup functions. These are stored as a linked list and run in * registered order by finish when the test program exits. Each function is * passed a boolean value indicating whether all tests were successful. */ struct cleanup_func { test_cleanup_func func; struct cleanup_func *next; }; static struct cleanup_func *cleanup_funcs = NULL; /* * Registered diag files. Any output found in these files will be printed out * as if it were passed to diag() before any other output we do. This allows * background processes to log to a file and have that output interleaved with * the test output. */ struct diag_file { char *name; FILE *file; char *buffer; size_t bufsize; struct diag_file *next; }; static struct diag_file *diag_files = NULL; /* * Print a specified prefix and then the test description. Handles turning * the argument list into a va_args structure suitable for passing to * print_desc, which has to be done in a macro. Assumes that format is the * argument immediately before the variadic arguments. */ #define PRINT_DESC(prefix, format) \ do { \ if (format != NULL) { \ va_list args; \ if (prefix != NULL) \ printf("%s", prefix); \ va_start(args, format); \ vprintf(format, args); \ va_end(args); \ } \ } while (0) /* * Form a new string by concatenating multiple strings. The arguments must be * terminated by (const char *) 0. * * This function only exists because we can't assume asprintf. We can't * simulate asprintf with snprintf because we're only assuming SUSv3, which * does not require that snprintf with a NULL buffer return the required * length. When those constraints are relaxed, this should be ripped out and * replaced with asprintf or a more trivial replacement with snprintf. */ static char * concat(const char *first, ...) { va_list args; char *result; const char *string; size_t offset; size_t length = 0; /* * Find the total memory required. Ensure we don't overflow length. See * the comment for breallocarray for why we're using UINT_MAX here. */ va_start(args, first); for (string = first; string != NULL; string = va_arg(args, const char *)) { if (length >= UINT_MAX - strlen(string)) bail("strings too long in concat"); length += strlen(string); } va_end(args); length++; /* Create the string. */ result = bmalloc(length); va_start(args, first); offset = 0; for (string = first; string != NULL; string = va_arg(args, const char *)) { memcpy(result + offset, string, strlen(string)); offset += strlen(string); } va_end(args); result[offset] = '\0'; return result; } /* * Check all registered diag_files for any output. We only print out the * output if we see a complete line; otherwise, we wait for the next newline. */ static void check_diag_files(void) { struct diag_file *file; fpos_t where; size_t length; int size, incomplete; /* * Walk through each file and read each line of output available. The * general scheme here used is as follows: try to read a line of output at * a time. If we get NULL, check for EOF; on EOF, advance to the next * file. * * If we get some data, see if it ends in a newline. If it doesn't end in * a newline, we have one of two cases: our buffer isn't large enough, in * which case we resize it and try again, or we have incomplete data in * the file, in which case we rewind the file and will try again next * time. */ for (file = diag_files; file != NULL; file = file->next) { clearerr(file->file); /* Store the current position in case we have to rewind. */ if (fgetpos(file->file, &where) < 0) sysbail("cannot get position in %s", file->name); /* Continue until we get EOF or an incomplete line of data. */ incomplete = 0; while (!feof(file->file) && !incomplete) { size = file->bufsize > INT_MAX ? INT_MAX : (int) file->bufsize; if (fgets(file->buffer, size, file->file) == NULL) { if (ferror(file->file)) sysbail("cannot read from %s", file->name); continue; } /* * See if the line ends in a newline. If not, see which error * case we have. Use UINT_MAX as a substitute for SIZE_MAX (see * the comment for breallocarray). */ length = strlen(file->buffer); if (file->buffer[length - 1] != '\n') { if (length < file->bufsize - 1) incomplete = 1; else { if (file->bufsize >= UINT_MAX - BUFSIZ) sysbail("line too long in %s", file->name); file->bufsize += BUFSIZ; file->buffer = brealloc(file->buffer, file->bufsize); } /* * On either incomplete lines or too small of a buffer, rewind * and read the file again (on the next pass, if incomplete). * It's simpler than trying to double-buffer the file. */ if (fsetpos(file->file, &where) < 0) sysbail("cannot set position in %s", file->name); continue; } /* We saw a complete line. Print it out. */ printf("# %s", file->buffer); } } } /* * Our exit handler. Called on completion of the test to report a summary of * results provided we're still in the original process. This also handles * printing out the plan if we used plan_lazy(), although that's suppressed if * we never ran a test (due to an early bail, for example), and running any * registered cleanup functions. */ static void finish(void) { int success, primary; struct cleanup_func *current; unsigned long highest = testnum - 1; struct diag_file *file, *tmp; /* Check for pending diag_file output. */ check_diag_files(); /* Free the diag_files. */ file = diag_files; while (file != NULL) { tmp = file; file = file->next; fclose(tmp->file); free(tmp->name); free(tmp->buffer); free(tmp); } diag_files = NULL; /* * Determine whether all tests were successful, which is needed before * calling cleanup functions since we pass that fact to the functions. */ if (_planned == 0 && _lazy) _planned = highest; success = (!_aborted && _planned == highest && _failed == 0); /* * If there are any registered cleanup functions, we run those first. We * always run them, even if we didn't run a test. Don't do anything * except free the diag_files and call cleanup functions if we aren't the * primary process (the process in which plan or plan_lazy was called), * and tell the cleanup functions that fact. */ primary = (_process == 0 || getpid() == _process); while (cleanup_funcs != NULL) { cleanup_funcs->func(success, primary); current = cleanup_funcs; cleanup_funcs = cleanup_funcs->next; free(current); } if (!primary) return; /* Don't do anything further if we never planned a test. */ if (_planned == 0) return; /* If we're aborting due to bail, don't print summaries. */ if (_aborted) return; /* Print out the lazy plan if needed. */ fflush(stderr); if (_lazy && _planned > 0) printf("1..%lu\n", _planned); /* Print out a summary of the results. */ if (_planned > highest) diag("Looks like you planned %lu test%s but only ran %lu", _planned, (_planned > 1 ? "s" : ""), highest); else if (_planned < highest) diag("Looks like you planned %lu test%s but ran %lu extra", _planned, (_planned > 1 ? "s" : ""), highest - _planned); else if (_failed > 0) diag("Looks like you failed %lu test%s of %lu", _failed, (_failed > 1 ? "s" : ""), _planned); else if (_planned != 1) diag("All %lu tests successful or skipped", _planned); else diag("%lu test successful or skipped", _planned); } /* * Initialize things. Turns on line buffering on stdout and then prints out * the number of tests in the test suite. We intentionally don't check for * pending diag_file output here, since it should really come after the plan. */ void plan(unsigned long count) { if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0) sysdiag("cannot set stdout to line buffered"); fflush(stderr); printf("1..%lu\n", count); testnum = 1; _planned = count; _process = getpid(); if (atexit(finish) != 0) { sysdiag("cannot register exit handler"); diag("cleanups will not be run"); } } /* * Initialize things for lazy planning, where we'll automatically print out a * plan at the end of the program. Turns on line buffering on stdout as well. */ void plan_lazy(void) { if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0) sysdiag("cannot set stdout to line buffered"); testnum = 1; _process = getpid(); _lazy = 1; if (atexit(finish) != 0) sysbail("cannot register exit handler to display plan"); } /* * Skip the entire test suite and exits. Should be called instead of plan(), * not after it, since it prints out a special plan line. Ignore diag_file * output here, since it's not clear if it's allowed before the plan. */ void skip_all(const char *format, ...) { fflush(stderr); printf("1..0 # skip"); PRINT_DESC(" ", format); putchar('\n'); exit(0); } /* * Takes a boolean success value and assumes the test passes if that value * is true and fails if that value is false. */ int ok(int success, const char *format, ...) { fflush(stderr); check_diag_files(); printf("%sok %lu", success ? "" : "not ", testnum++); if (!success) _failed++; PRINT_DESC(" - ", format); putchar('\n'); return success; } /* * Same as ok(), but takes the format arguments as a va_list. */ int okv(int success, const char *format, va_list args) { fflush(stderr); check_diag_files(); printf("%sok %lu", success ? "" : "not ", testnum++); if (!success) _failed++; if (format != NULL) { printf(" - "); vprintf(format, args); } putchar('\n'); return success; } /* * Skip a test. */ void skip(const char *reason, ...) { fflush(stderr); check_diag_files(); printf("ok %lu # skip", testnum++); PRINT_DESC(" ", reason); putchar('\n'); } /* * Report the same status on the next count tests. */ int ok_block(unsigned long count, int success, const char *format, ...) { unsigned long i; fflush(stderr); check_diag_files(); for (i = 0; i < count; i++) { printf("%sok %lu", success ? "" : "not ", testnum++); if (!success) _failed++; PRINT_DESC(" - ", format); putchar('\n'); } return success; } /* * Skip the next count tests. */ void skip_block(unsigned long count, const char *reason, ...) { unsigned long i; fflush(stderr); check_diag_files(); for (i = 0; i < count; i++) { printf("ok %lu # skip", testnum++); PRINT_DESC(" ", reason); putchar('\n'); } } /* * Takes an expected boolean value and a seen boolean value and assumes the * test passes if the truth value of both match. */ int is_bool(int wanted, int seen, const char *format, ...) { int success; fflush(stderr); check_diag_files(); success = (!!wanted == !!seen); if (success) printf("ok %lu", testnum++); else { diag("wanted: %s", !!wanted ? "true" : "false"); diag(" seen: %s", !!seen ? "true" : "false"); printf("not ok %lu", testnum++); _failed++; } PRINT_DESC(" - ", format); putchar('\n'); return success; } /* * Takes an expected integer and a seen integer and assumes the test passes * if those two numbers match. */ int is_int(long wanted, long seen, const char *format, ...) { int success; fflush(stderr); check_diag_files(); success = (wanted == seen); if (success) printf("ok %lu", testnum++); else { diag("wanted: %ld", wanted); diag(" seen: %ld", seen); printf("not ok %lu", testnum++); _failed++; } PRINT_DESC(" - ", format); putchar('\n'); return success; } /* * Takes a string and what the string should be, and assumes the test passes * if those strings match (using strcmp). */ int is_string(const char *wanted, const char *seen, const char *format, ...) { int success; if (wanted == NULL) wanted = "(null)"; if (seen == NULL) seen = "(null)"; fflush(stderr); check_diag_files(); success = (strcmp(wanted, seen) == 0); if (success) printf("ok %lu", testnum++); else { diag("wanted: %s", wanted); diag(" seen: %s", seen); printf("not ok %lu", testnum++); _failed++; } PRINT_DESC(" - ", format); putchar('\n'); return success; } /* * Takes an expected unsigned long and a seen unsigned long and assumes the * test passes if the two numbers match. Otherwise, reports them in hex. */ int is_hex(unsigned long wanted, unsigned long seen, const char *format, ...) { int success; fflush(stderr); check_diag_files(); success = (wanted == seen); if (success) printf("ok %lu", testnum++); else { diag("wanted: %lx", (unsigned long) wanted); diag(" seen: %lx", (unsigned long) seen); printf("not ok %lu", testnum++); _failed++; } PRINT_DESC(" - ", format); putchar('\n'); return success; } /* * Bail out with an error. */ void bail(const char *format, ...) { va_list args; _aborted = 1; fflush(stderr); check_diag_files(); fflush(stdout); printf("Bail out! "); va_start(args, format); vprintf(format, args); va_end(args); printf("\n"); exit(255); } /* * Bail out with an error, appending strerror(errno). */ void sysbail(const char *format, ...) { va_list args; int oerrno = errno; _aborted = 1; fflush(stderr); check_diag_files(); fflush(stdout); printf("Bail out! "); va_start(args, format); vprintf(format, args); va_end(args); printf(": %s\n", strerror(oerrno)); exit(255); } /* * Report a diagnostic to stderr. Always returns 1 to allow embedding in * compound statements. */ int diag(const char *format, ...) { va_list args; fflush(stderr); check_diag_files(); fflush(stdout); printf("# "); va_start(args, format); vprintf(format, args); va_end(args); printf("\n"); return 1; } /* * Report a diagnostic to stderr, appending strerror(errno). Always returns 1 * to allow embedding in compound statements. */ int sysdiag(const char *format, ...) { va_list args; int oerrno = errno; fflush(stderr); check_diag_files(); fflush(stdout); printf("# "); va_start(args, format); vprintf(format, args); va_end(args); printf(": %s\n", strerror(oerrno)); return 1; } /* * Register a new file for diag_file processing. */ void diag_file_add(const char *name) { struct diag_file *file, *prev; file = bcalloc(1, sizeof(struct diag_file)); file->name = bstrdup(name); file->file = fopen(file->name, "r"); if (file->file == NULL) sysbail("cannot open %s", name); file->buffer = bmalloc(BUFSIZ); file->bufsize = BUFSIZ; if (diag_files == NULL) diag_files = file; else { for (prev = diag_files; prev->next != NULL; prev = prev->next) ; prev->next = file; } } /* * Remove a file from diag_file processing. If the file is not found, do * nothing, since there are some situations where it can be removed twice * (such as if it's removed from a cleanup function, since cleanup functions * are called after freeing all the diag_files). */ void diag_file_remove(const char *name) { struct diag_file *file; struct diag_file **prev = &diag_files; for (file = diag_files; file != NULL; file = file->next) { if (strcmp(file->name, name) == 0) { *prev = file->next; fclose(file->file); free(file->name); free(file->buffer); free(file); return; } prev = &file->next; } } /* * Allocate cleared memory, reporting a fatal error with bail on failure. */ void * bcalloc(size_t n, size_t size) { void *p; p = calloc(n, size); if (p == NULL) sysbail("failed to calloc %lu", (unsigned long)(n * size)); return p; } /* * Allocate memory, reporting a fatal error with bail on failure. */ void * bmalloc(size_t size) { void *p; p = malloc(size); if (p == NULL) sysbail("failed to malloc %lu", (unsigned long) size); return p; } /* * Reallocate memory, reporting a fatal error with bail on failure. */ void * brealloc(void *p, size_t size) { p = realloc(p, size); if (p == NULL) sysbail("failed to realloc %lu bytes", (unsigned long) size); return p; } /* * The same as brealloc, but determine the size by multiplying an element * count by a size, similar to calloc. The multiplication is checked for * integer overflow. * * We should technically use SIZE_MAX here for the overflow check, but * SIZE_MAX is C99 and we're only assuming C89 + SUSv3, which does not * guarantee that it exists. They do guarantee that UINT_MAX exists, and we * can assume that UINT_MAX <= SIZE_MAX. * * (In theory, C89 and C99 permit size_t to be smaller than unsigned int, but * I disbelieve in the existence of such systems and they will have to cope * without overflow checks.) */ void * breallocarray(void *p, size_t n, size_t size) { if (n > 0 && UINT_MAX / n <= size) bail("reallocarray too large"); p = realloc(p, n * size); if (p == NULL) sysbail("failed to realloc %lu bytes", (unsigned long) (n * size)); return p; } /* * Copy a string, reporting a fatal error with bail on failure. */ char * bstrdup(const char *s) { char *p; size_t len; len = strlen(s) + 1; p = malloc(len); if (p == NULL) sysbail("failed to strdup %lu bytes", (unsigned long) len); memcpy(p, s, len); return p; } /* * Copy up to n characters of a string, reporting a fatal error with bail on * failure. Don't use the system strndup function, since it may not exist and * the TAP library doesn't assume any portability support. */ char * bstrndup(const char *s, size_t n) { const char *p; char *copy; size_t length; /* Don't assume that the source string is nul-terminated. */ for (p = s; (size_t) (p - s) < n && *p != '\0'; p++) ; length = (size_t) (p - s); copy = malloc(length + 1); if (p == NULL) sysbail("failed to strndup %lu bytes", (unsigned long) length); memcpy(copy, s, length); copy[length] = '\0'; return copy; } /* * Locate a test file. Given the partial path to a file, look under * C_TAP_BUILD and then C_TAP_SOURCE for the file and return the full path to * the file. Returns NULL if the file doesn't exist. A non-NULL return * should be freed with free(). */ char * test_file_path(const char *file) { char *base; char *path = NULL; const char *envs[] = { "C_TAP_BUILD", "C_TAP_SOURCE", NULL }; int i; for (i = 0; envs[i] != NULL; i++) { base = getenv(envs[i]); if (base == NULL) continue; path = concat(base, "/", file, (const char *) 0); if (access(path, R_OK) == 0) break; free(path); path = NULL; } return path; } /* * Create a temporary directory, tmp, under C_TAP_BUILD if set and the current * directory if it does not. Returns the path to the temporary directory in * newly allocated memory, and calls bail on any failure. The return value * should be freed with test_tmpdir_free. * * This function uses sprintf because it attempts to be independent of all * other portability layers. The use immediately after a memory allocation * should be safe without using snprintf or strlcpy/strlcat. */ char * test_tmpdir(void) { const char *build; char *path = NULL; build = getenv("C_TAP_BUILD"); if (build == NULL) build = "."; path = concat(build, "/tmp", (const char *) 0); if (access(path, X_OK) < 0) if (mkdir(path, 0777) < 0) sysbail("error creating temporary directory %s", path); return path; } /* * Free a path returned from test_tmpdir() and attempt to remove the * directory. If we can't delete the directory, don't worry; something else * that hasn't yet cleaned up may still be using it. */ void test_tmpdir_free(char *path) { if (path != NULL) rmdir(path); free(path); } /* * Register a cleanup function that is called when testing ends. All such * registered functions will be run by finish. */ void test_cleanup_register(test_cleanup_func func) { struct cleanup_func *cleanup, **last; cleanup = bmalloc(sizeof(struct cleanup_func)); cleanup->func = func; cleanup->next = NULL; last = &cleanup_funcs; while (*last != NULL) last = &(*last)->next; *last = cleanup; }