From 78b63d329c2e3bb185c0b78119ccb5fc5c0781b9 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Sun, 7 Dec 2008 18:53:02 -0200 Subject: [PATCH] Add dynamic Python implementation A new implementation is added. This implementation is dynamic. Test suites must be compiled as dynamically linked shared objects (.so) and a Python program (using ctypes module) inspects the shared objects, looking for test cases, running them and collecting statistics. The advantage of this implementation is that test suites are completely isolated, name clashes between test suites can't be possible. The testing program is completely decoupled from the test suites, and is less "hackish", in the sense that no code-generation is needed. You compile your test suites as shared object, and run the tester on them, that's it. Is much easier to extend too, since is implemented in Python. The downside is that the test suites are less "debuggeable", you can't easily plug a gdb to see what's going on there (AFAIK). tmp --- py/mutest | 222 ++++++++++++++++++++++++++++++++++++++++ py/mutest.h | 68 ++++++++++++ sample/.gitignore | 1 + sample/Makefile | 17 ++- sample/README | 4 + sample/factorial_test.c | 7 +- sample/sum_test.c | 7 +- 7 files changed, 323 insertions(+), 3 deletions(-) create mode 100755 py/mutest create mode 100644 py/mutest.h diff --git a/py/mutest b/py/mutest new file mode 100755 index 0000000..0c10bd7 --- /dev/null +++ b/py/mutest @@ -0,0 +1,222 @@ +#!/usr/bin/env python + +import re +from sys import stdout, stderr +from optparse import OptionParser +from glob import glob +from os.path import basename, splitext, abspath +from subprocess import Popen, PIPE +from ctypes import cdll, c_int + +API_VERSION = 1 + +V_QUIET = 0 +V_ERROR = 1 +V_SUMMARY = 2 +V_SUITE = 3 +V_CASE = 4 +V_CHECK = 5 + +verbose_level = V_ERROR + + +SUMMARY_TEXT = ''' +Tests done: + %(passed_suites)s test suite(s) passed, %(failed_suites)s failed, \ +%(skipped_suites)s skipped. + %(passed_cases)s test case(s) passed, %(failed_cases)s failed. + %(passed_checks)s check(s) passed, %(failed_checks)s failed. +''' + +def log(level, msg, *args): + global verbose_level + out = stdout + if level == V_ERROR: + stderr + if verbose_level >= level: + out.write((msg % args) + '\n') + + +#class SOError (Exception): +# pass + + +class TestCase(object): + + def __init__(self, so, name): + self.so = so + self.name = name + self.testcase = self.get_fun(name) + self.reset_counters = self.get_fun('mutest_reset_counters') + self.set_verbose_level = self.get_fun( + 'mutest_set_verbose_level', argtype=[c_int]) + + @property + def passed_count(self): + return self.get_val('mutest_passed_count') + + @property + def failed_count(self): + return self.get_val('mutest_failed_count') + + def get_fun(self, name, argtype=None, restype=None): + f = getattr(self.so, name) + f.argtypes = argtype + f.restype = restype + return f + + def get_val(self, name): + return c_int.in_dll(self.so, name).value + + def run(self): + global verbose_level + self.set_verbose_level(verbose_level) + self.reset_counters() + self.testcase() + return (self.passed_count, self.failed_count) + + +class TestSuiteResult (object): + failed = False + passed_cases = 0 + failed_cases = 0 + passed_checks = 0 + failed_checks = 0 + + def __repr__(self): + return 'TestSuiteResult(failed=%s, passed_cases=%s, '\ + 'failed_cases=%s, passed_checks=%s, failed_checks=%s)'\ + % (self.failed, + self.passed_cases, self.failed_cases, + self.passed_checks, self.failed_checks) + + +class TestSuite (object): + + def __init__(self, so, name, case_names): + self.name = name + self.so = so + try: + self.api_version = c_int.in_dll(self.so, + 'mutest_api_version').value + except ValueError: + self.api_version = 0 + return + self.cases = [TestCase(self.so, name) for name in case_names] + + def run(self): + r = TestSuiteResult() + for case in self.cases: + log(V_CASE, "\t* Executing test case '%s'...", + case.name) + (case_passed_checks, case_failed_checks) = case.run() + log(V_CASE, '\t Results: %s check(s) passed, %s ' + 'failed.', case_passed_checks, + case_failed_checks) + if case_failed_checks: + r.failed = True + r.failed_cases += 1 + else: + r.passed_cases += 1 + r.passed_checks += case_passed_checks + r.failed_checks += case_failed_checks + return r + + +case_names_re = re.compile(r'[0-9a-f]{8} T (mu_test_\w+)', re.I) + +def get_case_names(so_name): + proc = Popen(['nm', '-p', so_name], stdout=PIPE) + output = proc.communicate()[0] + return case_names_re.findall(output) + + +def parse_arguments(args): + verbose_help = ('Show a short result summary, add more for extra ' + 'verbosity: -vv for test suites progress, -vvv for ' + 'test cases progress and -vvvv for printing each ' + 'and every check done') + quiet_help = ('Be quiet (overrides -v)') + search_help = ('Search for all test suites in the current directory ' + '(*.so) and add them') + parser = OptionParser() + parser.add_option('-v', dest='verbose_level', action='count', + default=1, help=verbose_help) + parser.add_option('-q', '--verbose-level', dest='quiet', + action='store_true', default=False, help=quiet_help) + parser.add_option('-a', '--search-all', dest='search_all', + action='store_true', default=False, help=search_help) + return parser.parse_args() + + +def main(args): + global verbose_level + + (opts, args) = parse_arguments(args) + + if opts.quiet: + verbose_level = 0 + else: + verbose_level = opts.verbose_level + + if opts.search_all: + args.extend(glob('*.so')) + + if not args: + log(V_SUMMARY, 'No test suites to run') + return 0 + + results = dict(passed_suites=0, failed_suites=0, skipped_suites=0, + passed_cases=0, failed_cases=0, + passed_checks=0, failed_checks=0) + + for so_name in args: + suite_name = splitext(basename(so_name))[0] + log(V_SUITE, '\nRunning test suite "%s"...', suite_name) + + try: + so = cdll.LoadLibrary(abspath(so_name)) + except OSError, e: + log(V_ERROR, 'Error loading "%s" (%s), skipping ' + 'test suite "%s"', so_name, e, + suite_name) + results['skipped_suites'] += 1 + continue + + case_names = get_case_names(so_name) + + suite = TestSuite(so, suite_name, case_names) + + if suite.api_version != API_VERSION: + log(V_ERROR, 'Wrong API version (%s expected, %s ' + 'found), skipping test suite "%s"', + API_VERSION, suite.api_version, + suite.name) + results['skipped_suites'] += 1 + continue + + r = suite.run() + + log(V_SUITE, 'Results: %s test case(s) passed, %s failed, ' + '%s check(s) passed, %s failed.', + r.passed_cases, r.failed_cases, + r.passed_checks, r.failed_checks) + + if r.failed: + results['failed_suites'] += 1 + else: + results['passed_suites'] += 1 + results['failed_cases'] += r.failed_cases + results['passed_cases'] += r.passed_cases + results['failed_checks'] += r.failed_checks + results['passed_checks'] += r.passed_checks + + log(V_SUMMARY, SUMMARY_TEXT % results) + + return 0 + + +if __name__ == '__main__': + import sys + sys.exit(main(sys.argv[1:])) + diff --git a/py/mutest.h b/py/mutest.h new file mode 100644 index 0000000..ace3b9c --- /dev/null +++ b/py/mutest.h @@ -0,0 +1,68 @@ + +#include /* printf(), fprintf() */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* this increments when the "API" changes */ +int mutest_api_version = 1; + +int mutest_passed_count; +int mutest_failed_count; +void mutest_reset_counters() { + mutest_passed_count = 0; + mutest_failed_count = 0; +} + +/* + * Verbose level: + * 0: quiet + * 1: errors + * 2: summary + * 3: suites + * 4: cases + * 5: checks + */ +int mutest_verbose_level = 1; +void mutest_set_verbose_level(int val) { + mutest_verbose_level = val; +} + +#define mu_check(exp) \ + do { \ + if (exp) { \ + ++mutest_passed_count; \ + if (mutest_verbose_level >= 5) \ + printf("%s:%d: mu_check(%s) passed\n", \ + __FILE__, __LINE__, #exp); \ + } else { \ + ++mutest_failed_count; \ + if (mutest_verbose_level) \ + fprintf(stderr, "%s:%d: mu_check(%s) " \ + "failed, resuming test case\n", \ + __FILE__, __LINE__, #exp); \ + } \ + } while (0) + +#define mu_ensure(exp) \ + do { \ + if (exp) { \ + ++mutest_passed_count; \ + if (mutest_verbose_level >=5) \ + printf("%s:%d: mu_ensure(%s) passed\n", \ + __FILE__, __LINE__, #exp); \ + } else { \ + ++mutest_failed_count; \ + if (mutest_verbose_level) \ + fprintf(stderr, "%s:%d: mu_ensure(%s) " \ + "failed, aborting test case\n", \ + __FILE__, __LINE__, #exp); \ + return; \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + diff --git a/sample/.gitignore b/sample/.gitignore index 9d1f6d3..a261a18 100644 --- a/sample/.gitignore +++ b/sample/.gitignore @@ -1,2 +1,3 @@ *.o +*.so tester diff --git a/sample/Makefile b/sample/Makefile index df8b16c..d352654 100644 --- a/sample/Makefile +++ b/sample/Makefile @@ -7,20 +7,35 @@ TARGET=tester OBJS = factorial.o sum.o TESTS = factorial_test.o sum_test.o TESTER = tester.o +SO = factorial.so sum.so ALL = $(TESTER) $(OBJS) $(TESTS) all: $(TARGET) +py: $(SO) + $(TARGET): $(ALL) $(TESTER): $(OBJS) $(TESTS) ../mkmutest $(TESTS) | gcc -xc -o $(TESTER) -c - +factorial.so: factorial_test.c + +sum.so: sum_test.c + test: $(TARGET) ./$(TARGET) $(V) +test-py: $(SO) + ../py/mutest $(V) -a + clean: - $(RM) $(TARGET) $(ALL) + $(RM) $(TARGET) $(SO) $(ALL) + +.c.so: + $(CC) $(CFLAGS) $(LDFLAGS) -DMUTEST_PY -fPIC -shared -o $@ $^ + +.SUFFIXES: .so .PHONY: all test clean diff --git a/sample/README b/sample/README index 2d5c710..a687fb0 100644 --- a/sample/README +++ b/sample/README @@ -5,4 +5,8 @@ make test If you want extra verbosity, try: make test V=-vvv +To try out the python implementation, try: +make test-py + +(same for the verbosity level) diff --git a/sample/factorial_test.c b/sample/factorial_test.c index 10209e0..0e0191a 100644 --- a/sample/factorial_test.c +++ b/sample/factorial_test.c @@ -1,7 +1,12 @@ -#include "../mutest.h" #include "factorial.h" +#ifdef MUTEST_PY +#include "../py/mutest.h" +#else +#include "../mutest.h" +#endif + void mu_test_factorial_zero() { unsigned x = factorial(0); mu_check(x == 1); diff --git a/sample/sum_test.c b/sample/sum_test.c index 7fea630..398ca08 100644 --- a/sample/sum_test.c +++ b/sample/sum_test.c @@ -2,9 +2,14 @@ /* see factorial_test.c for more complete examples, this file is mostly to show * how to have multiple test suites, and a test suite that succeed. */ -#include "../mutest.h" #include "sum.h" +#ifdef MUTEST_PY +#include "../py/mutest.h" +#else +#include "../mutest.h" +#endif + void mu_test_sum() { mu_check(sum(4, 5) == 9); mu_check(sum(-4, -5) == -9); -- 2.43.0