Implement test suite initialization and termination

This patch adds test suite initialization and termination support to both
C/bash and Python implementations.

Any exported function in a test suite that starts with "mu_init" is used
as an initialization function (returning 0 if the initialization
succeeded) and any function starting with "mu_term" is used as
a termination function. If initialization fails (any initialization
funtion fails), the test suite is skipped.

diff --git a/mkmutest b/mkmutest
index 1bfec88a20850dae61d1bc18528e93838100b84e..c6b941b6e1ac1b5f5879d52f4edb6e62e739642a 100755 (executable)
--- a/mkmutest
+++ b/mkmutest
@@ -17,6 +17,12 @@
 # cases must have unique names, even across test suites.
 
 # the first argument should be mutest.h
 # cases must have unique names, even across test suites.
 
 # the first argument should be mutest.h

+if [ -z "$1" ]
+then
+       echo "Too few arguments" >&2
+       echo "Usage: $0 mutest_h_location [object files...]" >&2
+       exit 1
+fi

 mutest_h="$1"
 shift
 echo "#include \"$mutest_h\""
 mutest_h="$1"
 shift
 echo "#include \"$mutest_h\""


@@ -24,18 +30,31 @@ echo "void mu_run_suites() {"
 echo
 for file in "$@"
 do
 echo
 for file in "$@"
 do

-       suite="`basename "$file" .o | sed 's/\"/\\\\\"/g'`"
-       echo -e '\tmutest_suite_name = "'"$suite"'";'
-       echo -e '\tmu_print(MU_SUITE, "\\nRunning suite \"'"$suite"'\"\\n");'
-       for symbol in `nm -p "$file" \
-                       | egrep '^[[:xdigit:]]{8} T mu_test\w*$' \
-                       | cut -c12-`
+       pr_file=`echo "$file" | sed 's/\"/\\\\\"/g'`
+       suite=`basename "$file" .o | sed 's/\"/\\\\\"/g'`
+       symbols=`nm -p "$file" | egrep '^[[:xdigit:]]{8} T mu_\w+$' | cut -c12-`
+       tests=`echo "$symbols" | egrep '^mu_test'`
+       inits=`echo "$symbols" | egrep '^mu_init'`
+       terms=`echo "$symbols" | egrep '^mu_term'`
+       echo -e '\tdo {'
+       echo -e '\t\tmutest_suite_name = "'"$suite"'";'
+       echo -e '\t\tmu_print(MU_SUITE, "\\nRunning suite '"'$suite'"'\\n");'
+       for init in $inits

        do
        do

-               echo -e "\tmu_run_case($symbol);"
+               echo -e "\\t\\tmu_run_init($init);"

        done
        done

-       echo -e "\tif (mutest_suite_failed) ++mutest_failed_suites;"
-       echo -e "\telse                     ++mutest_passed_suites;"
-       echo -e "\tmutest_suite_failed = 0;"
+       for testcase in $tests
+       do
+               echo -e "\t\tmu_run_case($testcase);"
+       done
+       for term in $terms
+       do
+               echo -e "\t\tmu_run_term($term);"
+       done
+       echo -e "\t\tif (mutest_suite_failed) ++mutest_failed_suites;"
+       echo -e "\t\telse                     ++mutest_passed_suites;"
+       echo -e "\t\tmutest_suite_failed = 0;"
+       echo -e '\t} while (0);'

        echo
 done
 echo "}"
        echo
 done
 echo "}"

diff --git a/mutest.c b/mutest.c
index 2971480b15119633b500879f2185d565b6add487..291fdd458c1224e3753c256217c9a723f764dc01 100644 (file)
--- a/mutest.c
+++ b/mutest.c
@@ -13,6 +13,7 @@
 const char* mutest_suite_name;
 int mutest_failed_suites;
 int mutest_passed_suites;
 const char* mutest_suite_name;
 int mutest_failed_suites;
 int mutest_passed_suites;

+int mutest_skipped_suites;

 int mutest_suite_failed;
 
 
 int mutest_suite_failed;
 
 


@@ -60,14 +61,15 @@ int main(int argc, char* argv[]) {
 
        mu_print(MU_SUMMARY, "\n"
                        "Tests done:\n"
 
        mu_print(MU_SUMMARY, "\n"
                        "Tests done:\n"

-                       "\t%d test suite(s) passed, %d failed.\n"
+                       "\t%d test suite(s) passed, %d failed, %d skipped.\n"

                        "\t%d test case(s) passed, %d failed.\n"
                        "\t%d check(s) passed, %d failed.\n"
                        "\n",
                        mutest_passed_suites, mutest_failed_suites,
                        "\t%d test case(s) passed, %d failed.\n"
                        "\t%d check(s) passed, %d failed.\n"
                        "\n",
                        mutest_passed_suites, mutest_failed_suites,

+                       mutest_skipped_suites,

                        mutest_passed_cases, mutest_failed_cases,
                        mutest_passed_checks, mutest_failed_checks);
 
                        mutest_passed_cases, mutest_failed_cases,
                        mutest_passed_checks, mutest_failed_checks);
 

-       return mutest_failed_checks ? 1 : 0;
+       return (mutest_failed_suites + mutest_skipped_suites) ? 1 : 0;

 }
 
 }
 

diff --git a/mutest.h b/mutest.h
index 1f5561d745a22deef43776c04b37b9f7590200aa..938918f6449cae707d472b09154cf7f1212d3315 100644 (file)
--- a/mutest.h
+++ b/mutest.h
@@ -70,13 +70,32 @@ enum {
  * if you want to implement your own customized version */
 void mu_run_suites();
 
  * if you want to implement your own customized version */
 void mu_run_suites();
 

+/* macro for running a single initialization function */
+#ifndef mu_run_init
+#define mu_run_init(name) \
+       { \
+               int name(); \
+               int r; \
+               mu_print(MU_CASE, "\t+ Executing initialization function " \
+                               "'" #name "'...\n"); \
+               if ((r = name())) { \
+                       mu_print(MU_ERROR, "%s:" #name ": initialization " \
+                                       "function failed (returned %d), " \
+                                       "skipping test suite...\n", \
+                                       mutest_suite_name, r); \
+                       ++mutest_skipped_suites; \
+                       break; \
+               } \
+       } do { } while (0)
+#endif /* mu_run_init */
+

 /* macro for running a single test case */
 #ifndef mu_run_case
 #define mu_run_case(name) \
        do { \
 /* macro for running a single test case */
 #ifndef mu_run_case
 #define mu_run_case(name) \
        do { \

-               mu_print(MU_CASE, "\t- executing test case \"" #name "\"\n"); \
-               void name(); \
+               mu_print(MU_CASE, "\t* Executing test case '" #name "'...\n");\

                mutest_case_name = #name; \
                mutest_case_name = #name; \

+               void name(); \

                name(); \
                if (mutest_case_failed) { \
                        ++mutest_failed_cases; \
                name(); \
                if (mutest_case_failed) { \
                        ++mutest_failed_cases; \


@@ -86,6 +105,17 @@ void mu_run_suites();
        } while (0)
 #endif /* mu_run_case */
 
        } while (0)
 #endif /* mu_run_case */
 

+/* macro for running a single termination function */
+#ifndef mu_run_term
+#define mu_run_term(name) \
+       do { \
+               mu_print(MU_CASE, "\t- Executing termination function '" \
+                               #name "'...\n"); \
+               void name(); \
+               name(); \
+       } while (0)
+#endif /* mu_run_term */
+

 /*
  * mutest exported variables for internal use, do not use directly unless you
  *  know what you're doing.
 /*
  * mutest exported variables for internal use, do not use directly unless you
  *  know what you're doing.


@@ -93,6 +123,7 @@ void mu_run_suites();
 extern const char* mutest_suite_name;
 extern int mutest_failed_suites;
 extern int mutest_passed_suites;
 extern const char* mutest_suite_name;
 extern int mutest_failed_suites;
 extern int mutest_passed_suites;

+extern int mutest_skipped_suites;

 extern int mutest_suite_failed;
 /* test cases */
 extern const char* mutest_case_name;
 extern int mutest_suite_failed;
 /* test cases */
 extern const char* mutest_case_name;

diff --git a/py/mutest b/py/mutest
index 4945552faa3298a1cde8a5c05d042cb6487175c5..8535523298b3379f2f0d3969e526766ae8405a18 100755 (executable)
--- a/py/mutest
+++ b/py/mutest
@@ -17,6 +17,10 @@ V_SUITE   = 3
 V_CASE    = 4
 V_CHECK   = 5
 
 V_CASE    = 4
 V_CHECK   = 5
 

+R_OK      = 1
+R_FAILED  = 2
+R_SKIPPED = 3
+

 verbose_level = V_ERROR
 
 
 verbose_level = V_ERROR
 
 


@@ -78,8 +82,22 @@ class TestCase(object):
                return (self.passed_count, self.failed_count)
 
 
                return (self.passed_count, self.failed_count)
 
 

+class TestSuiteInfo (object):
+
+       inits_re = re.compile(r'[0-9a-f]{8} T (mu_init\w*)', re.I)
+       terms_re = re.compile(r'[0-9a-f]{8} T (mu_term\w*)', re.I)
+       cases_re = re.compile(r'[0-9a-f]{8} T (mu_test\w*)', re.I)
+
+       def __init__(self, so_name):
+               proc = Popen(['nm', '-p', so_name], stdout=PIPE)
+               output = proc.communicate()[0]
+               self.inits = self.inits_re.findall(output)
+               self.terms = self.terms_re.findall(output)
+               self.cases = self.cases_re.findall(output)
+
+

 class TestSuiteResult (object):
 class TestSuiteResult (object):

-       failed = False
+       result = R_OK

        passed_cases = 0
        failed_cases = 0
        passed_checks = 0
        passed_cases = 0
        failed_cases = 0
        passed_checks = 0


@@ -88,14 +106,14 @@ class TestSuiteResult (object):
        def __repr__(self):
                return 'TestSuiteResult(failed=%s, passed_cases=%s, '\
                        'failed_cases=%s, passed_checks=%s, failed_checks=%s)'\
        def __repr__(self):
                return 'TestSuiteResult(failed=%s, passed_cases=%s, '\
                        'failed_cases=%s, passed_checks=%s, failed_checks=%s)'\

-                               % (self.failed,
+                               % (self.result,

                                        self.passed_cases, self.failed_cases,
                                        self.passed_checks, self.failed_checks)
 
 
 class TestSuite (object):
 
                                        self.passed_cases, self.failed_cases,
                                        self.passed_checks, self.failed_checks)
 
 
 class TestSuite (object):
 

-       def __init__(self, so, name, case_names):
+       def __init__(self, so, name, info):

                self.name = name
                self.so = so
                try:
                self.name = name
                self.so = so
                try:


@@ -103,10 +121,28 @@ class TestSuite (object):
                except ValueError:
                        self.api_version = 0
                        return
                except ValueError:
                        self.api_version = 0
                        return

-               self.cases = [TestCase(self.so, name) for name in case_names]
+               self.inits = dict([(name, get_fun(so, name, restype=c_int))
+                               for name in info.inits])
+               self.terms = dict([(name, get_fun(so, name))
+                               for name in info.terms])
+               self.cases = [TestCase(self.so, name)
+                               for name in info.cases]

 
        def run(self):
                r = TestSuiteResult()
 
        def run(self):
                r = TestSuiteResult()

+
+               for name, func in self.inits.items():
+                       log(V_CASE, "\t+ Executing initialization function "
+                                       "'%s'...", name);
+                       res = func()
+                       if res:
+                               log(V_ERROR, "%s:%s: initialization function "
+                                               "failed (returned %d), "
+                                               "skipping test suite...\n",
+                                               self.name, name, res);
+                               r.result = R_SKIPPED
+                               return r
+

                for case in self.cases:
                        log(V_CASE, "\t* Executing test case '%s'...",
                                        case.name)
                for case in self.cases:
                        log(V_CASE, "\t* Executing test case '%s'...",
                                        case.name)


@@ -115,21 +151,19 @@ class TestSuite (object):
                                        'failed.', case_passed_checks,
                                        case_failed_checks)
                        if case_failed_checks:
                                        'failed.', case_passed_checks,
                                        case_failed_checks)
                        if case_failed_checks:

-                               r.failed = True
+                               r.result = R_FAILED

                                r.failed_cases += 1
                        else:
                                r.passed_cases += 1
                        r.passed_checks += case_passed_checks
                        r.failed_checks += case_failed_checks
                                r.failed_cases += 1
                        else:
                                r.passed_cases += 1
                        r.passed_checks += case_passed_checks
                        r.failed_checks += case_failed_checks

-               return r
-

 
 

-case_names_re = re.compile(r'[0-9a-f]{8} T (mu_test\w*)', re.I)
+               for name, func in self.terms.items():
+                       log(V_CASE, "\t- Executing termination function "
+                                       "'%s'...", name)
+                       func()

 
 

-def get_case_names(so_name):
-       proc = Popen(['nm', '-p', so_name], stdout=PIPE)
-       output = proc.communicate()[0]
-       return case_names_re.findall(output)
+               return r

 
 
 def parse_arguments(args):
 
 
 def parse_arguments(args):


@@ -184,9 +218,9 @@ def main(args):
                        results['skipped_suites'] += 1
                        continue
 
                        results['skipped_suites'] += 1
                        continue
 

-               case_names = get_case_names(so_name)
+               info = TestSuiteInfo(so_name)

 
 

-               suite = TestSuite(so, suite_name, case_names)
+               suite = TestSuite(so, suite_name, info)

 
                if suite.api_version != API_VERSION:
                        log(V_ERROR, 'Wrong API version (%s expected, %s '
 
                if suite.api_version != API_VERSION:
                        log(V_ERROR, 'Wrong API version (%s expected, %s '


@@ -203,8 +237,10 @@ def main(args):
                                r.passed_cases, r.failed_cases,
                                r.passed_checks, r.failed_checks)
 
                                r.passed_cases, r.failed_cases,
                                r.passed_checks, r.failed_checks)
 

-               if r.failed:
+               if r.result == R_FAILED:

                        results['failed_suites'] += 1
                        results['failed_suites'] += 1

+               elif r.result == R_SKIPPED:
+                       results['skipped_suites'] += 1

                else:
                        results['passed_suites'] += 1
                results['failed_cases'] += r.failed_cases
                else:
                        results['passed_suites'] += 1
                results['failed_cases'] += r.failed_cases

diff --git a/sample/.gitignore b/sample/.gitignore
index a261a18f1d0569798d8d9f1cf82c41b19ac51981..5c220f291c194e556de1b0f3b2795546b4424856 100644 (file)
--- a/sample/.gitignore
+++ b/sample/.gitignore
@@ -1,3 +1,4 @@
 *.o
 *.so
 tester
 *.o
 *.so
 tester

+test_suite_runner.c

diff --git a/sample/Makefile b/sample/Makefile
index e67a320033c789672ba526188706bbfb5509cfed..e7c764d54ed374ff79907f2de7f25e486130cf4a 100644 (file)
--- a/sample/Makefile
+++ b/sample/Makefile
@@ -4,23 +4,28 @@ V=-v
 
 CFLAGS = -Wall -std=c89
 
 
 CFLAGS = -Wall -std=c89
 

-TARGET=tester
+TARGET = tester
+SUITES_RUNNER_SRC = test_suite_runner.c
+MKMUTEST = ../mkmutest
+MUTEST_H = ../mutest.h
+MUTEST_C = ../mutest.c

 
 OBJS = factorial.o sum.o
 
 OBJS = factorial.o sum.o

-TESTS = factorial_test.o sum_test.o
+TESTS = factorial_test.o sum_test.o init_fail_test.o

 MUTEST = mutest.o
 MUTEST = mutest.o

-TESTER = tester.o
-SO = factorial.so sum.so
-ALL = $(TESTER) $(OBJS) $(TESTS) $(MUTEST)
+SUITES_RUNNER = $(SUITES_RUNNER_SRC:.c=.o)
+SO = factorial.so sum.so init_fail_test.so
+ALL = $(SUITES_RUNNER) $(OBJS) $(TESTS) $(MUTEST)

 
 all: $(TARGET)
 
 py: $(SO)
 
 $(TARGET): $(ALL)
 
 all: $(TARGET)
 
 py: $(SO)
 
 $(TARGET): $(ALL)

+       $(CC) $(LDFLAGS) -o $@ $^

 
 

-$(TESTER): $(OBJS) $(TESTS) $(MUTEST)
-       ../mkmutest ../mutest.h $(TESTS) | gcc -xc -o $(TESTER) -c -
+$(SUITES_RUNNER_SRC): $(MKMUTEST) $(MUTEST_H) $(TESTS)
+       $(MKMUTEST) $(MUTEST_H) $(TESTS) > $@

 
 factorial.so: factorial_test.c
 
 
 factorial.so: factorial_test.c
 


@@ -36,7 +41,7 @@ test-py: $(SO)
        ../py/mutest $(V) -a
 
 clean:
        ../py/mutest $(V) -a
 
 clean:

-       $(RM) $(TARGET) $(SO) $(ALL)
+       $(RM) $(TARGET) $(SO) $(ALL) $(SUITES_RUNNER_SRC)

 
 .c.so:
        $(CC) $(CFLAGS) $(LDFLAGS) -DMUTEST_PY -fPIC -shared -o $@ $^
 
 .c.so:
        $(CC) $(CFLAGS) $(LDFLAGS) -DMUTEST_PY -fPIC -shared -o $@ $^

diff --git a/sample/init_fail_test.c b/sample/init_fail_test.c
new file mode 100644 (file)
index 0000000..c92bd7d
--- /dev/null
+++ b/sample/init_fail_test.c
@@ -0,0 +1,27 @@
+
+/* dummy test to illustrate how a test suite initialization could fail */
+
+#ifdef MUTEST_PY
+#include "../py/mutest.h"
+#else
+#include "../mutest.h"
+#endif
+
+static int ret = 0;
+
+int mu_init_success() {
+       return ret++;
+}
+
+int mu_init_fail() {
+       return ret;
+}
+
+/* this test will never be executed because the initialization failed */
+void mu_test_dummy() {
+}
+
+/* this test will never be executed either */
+void mu_term_success() {
+}
+

diff --git a/sample/sum_test.c b/sample/sum_test.c
index 398ca08230eda8f94b6075d75852a28ae25e0c2e..7ef32e89a7c5bcdc050ae680de7b8c9d916221ac 100644 (file)
--- a/sample/sum_test.c
+++ b/sample/sum_test.c
@@ -1,8 +1,10 @@
 
 /* see factorial_test.c for more complete examples, this file is mostly to show
 
 /* see factorial_test.c for more complete examples, this file is mostly to show

- * how to have multiple test suites, and a test suite that succeed. */
+ * how to have multiple test suites, test suite initialization and
+ * termination, and a test suite that succeed. */

 
 #include "sum.h"
 
 #include "sum.h"

+#include <stdlib.h> /* malloc(), free() */

 
 #ifdef MUTEST_PY
 #include "../py/mutest.h"
 
 #ifdef MUTEST_PY
 #include "../py/mutest.h"


@@ -10,6 +12,15 @@
 #include "../mutest.h"
 #endif
 
 #include "../mutest.h"
 #endif
 

+/* unused, just for ilustrate the test suite initialization/termination */
+static char* global;
+
+int mu_init_sum() {
+       global = (char*) malloc(1024);
+
+       return 0; /* initialization OK */
+}
+

 void mu_test_sum() {
        mu_check(sum(4, 5) == 9);
        mu_check(sum(-4, -5) == -9);
 void mu_test_sum() {
        mu_check(sum(4, 5) == 9);
        mu_check(sum(-4, -5) == -9);


@@ -17,3 +28,7 @@ void mu_test_sum() {
        mu_check(sum(1, -1) == 0);
 }
 
        mu_check(sum(1, -1) == 0);
 }
 

+void mu_term_sum() {
+       free(global);
+}
+