mutest

   1 #!/usr/bin/env python
   2 #
   3 # This file is part of mutest, a simple micro unit testing framework for C.
   4 #
   5 # mutest was written by Leandro Lucarella <llucax@gmail.com> and is released
   6 # under the BOLA license, please see the LICENSE file or visit:
   7 # http://blitiri.com.ar/p/bola/
   8 #
   9 # This is a Python implementation of the mutest main idea. This script load
  10 # the dynamic shared object (.so file) passed as argument, search them for test
  11 # cases and run them, collecting statistics.
  12 #
  13 # Please, read the README file for more details.
  14 #
  15
  16 import re
  17 from sys import stdout, stderr
  18 from optparse import OptionParser
  19 from glob import glob
  20 from os.path import basename, splitext, abspath
  21 from subprocess import Popen, PIPE
  22 from ctypes import cdll, c_int
  23
  24 API_VERSION = 1
  25
  26 V_QUIET   = 0
  27 V_ERROR   = 1
  28 V_SUMMARY = 2
  29 V_SUITE   = 3
  30 V_CASE    = 4
  31 V_CHECK   = 5
  32
  33 R_OK      = 1
  34 R_FAILED  = 2
  35 R_SKIPPED = 3
  36
  37 verbose_level = V_ERROR
  38
  39
  40 SUMMARY_TEXT = '''
  41 Tests done:
  42         %(passed_suites)s test suite(s) passed, %(failed_suites)s failed, \
  43 %(skipped_suites)s skipped.
  44         %(passed_cases)s test case(s) passed, %(failed_cases)s failed.
  45         %(passed_checks)s check(s) passed, %(failed_checks)s failed.
  46 '''
  47
  48 def log(level, msg, *args):
  49         global verbose_level
  50         out = stdout
  51         if level == V_ERROR:
  52                 stderr
  53         if verbose_level >= level:
  54                 out.write((msg % args) + '\n')
  55
  56
  57 def get_fun(so, name, argtype=None, restype=None):
  58         f = getattr(so, name)
  59         f.argtypes = argtype
  60         f.restype = restype
  61         return f
  62
  63
  64 def get_val(so, name):
  65         return c_int.in_dll(so, name).value
  66
  67
  68 #class SOError (Exception):
  69 #       pass
  70
  71
  72 class TestCase(object):
  73
  74         def __init__(self, so, name):
  75                 self.so = so
  76                 self.name = name
  77                 self.testcase = get_fun(so, name)
  78                 self.reset_counters = get_fun(so, 'mutest_reset_counters')
  79                 self.set_verbose_level = get_fun(so,
  80                                 'mutest_set_verbose_level', argtype=[c_int])
  81
  82         @property
  83         def passed_checks(self):
  84                 return get_val(self.so, 'mutest_passed_checks')
  85
  86         @property
  87         def failed_checks(self):
  88                 return get_val(self.so, 'mutest_failed_checks')
  89
  90         def run(self):
  91                 global verbose_level
  92                 self.set_verbose_level(verbose_level)
  93                 self.reset_counters()
  94                 self.testcase()
  95                 return (self.passed_checks, self.failed_checks)
  96
  97
  98 class TestSuiteInfo (object):
  99
 100         inits_re = re.compile(r'[0-9a-f]{8} T (mu_init\w*)', re.I)
 101         terms_re = re.compile(r'[0-9a-f]{8} T (mu_term\w*)', re.I)
 102         cases_re = re.compile(r'[0-9a-f]{8} T (mu_test\w*)', re.I)
 103
 104         def __init__(self, so_name):
 105                 proc = Popen(['nm', '-p', so_name], stdout=PIPE)
 106                 output = proc.communicate()[0]
 107                 self.inits = self.inits_re.findall(output)
 108                 self.terms = self.terms_re.findall(output)
 109                 self.cases = self.cases_re.findall(output)
 110
 111
 112 class TestSuiteResult (object):
 113         result = R_OK
 114         passed_cases = 0
 115         failed_cases = 0
 116         passed_checks = 0
 117         failed_checks = 0
 118
 119         def __repr__(self):
 120                 return 'TestSuiteResult(failed=%s, passed_cases=%s, '\
 121                         'failed_cases=%s, passed_checks=%s, failed_checks=%s)'\
 122                                 % (self.result,
 123                                         self.passed_cases, self.failed_cases,
 124                                         self.passed_checks, self.failed_checks)
 125
 126
 127 class TestSuite (object):
 128
 129         def __init__(self, so, name, info):
 130                 self.name = name
 131                 self.so = so
 132                 try:
 133                         self.api_version = get_val(so, 'mutest_api_version')
 134                 except ValueError:
 135                         self.api_version = 0
 136                         return
 137                 self.inits = dict([(name, get_fun(so, name, restype=c_int))
 138                                 for name in info.inits])
 139                 self.terms = dict([(name, get_fun(so, name))
 140                                 for name in info.terms])
 141                 self.cases = [TestCase(self.so, name)
 142                                 for name in info.cases]
 143
 144         def run(self):
 145                 r = TestSuiteResult()
 146
 147                 for name, func in self.inits.items():
 148                         log(V_CASE, "\t+ Executing initialization function "
 149                                         "'%s'...", name);
 150                         res = func()
 151                         if res:
 152                                 log(V_ERROR, "%s:%s: initialization function "
 153                                                 "failed (returned %d), "
 154                                                 "skipping test suite...\n",
 155                                                 self.name, name, res);
 156                                 r.result = R_SKIPPED
 157                                 return r
 158
 159                 for case in self.cases:
 160                         log(V_CASE, "\t* Executing test case '%s'...",
 161                                         case.name)
 162                         (case_passed_checks, case_failed_checks) = case.run()
 163                         log(V_CASE, '\t  Results: %s check(s) passed, %s '
 164                                         'failed.', case_passed_checks,
 165                                         case_failed_checks)
 166                         if case_failed_checks:
 167                                 r.result = R_FAILED
 168                                 r.failed_cases += 1
 169                         else:
 170                                 r.passed_cases += 1
 171                         r.passed_checks += case_passed_checks
 172                         r.failed_checks += case_failed_checks
 173
 174                 for name, func in self.terms.items():
 175                         log(V_CASE, "\t- Executing termination function "
 176                                         "'%s'...", name)
 177                         func()
 178
 179                 return r
 180
 181
 182 def parse_arguments(args):
 183         verbose_help = ('Show a short result summary, add more for extra '
 184                         'verbosity: -vv for test suites progress, -vvv for '
 185                         'test cases progress and -vvvv for printing each '
 186                         'and every check done')
 187         quiet_help = ('Be quiet (overrides -v)')
 188         search_help = ('Search for all test suites in the current directory '
 189                         '(*.so) and add them')
 190         parser = OptionParser()
 191         parser.add_option('-v', dest='verbose_level', action='count',
 192                         default=1, help=verbose_help)
 193         parser.add_option('-q', '--verbose-level', dest='quiet',
 194                         action='store_true', default=False, help=quiet_help)
 195         parser.add_option('-a', '--search-all', dest='search_all',
 196                         action='store_true', default=False, help=search_help)
 197         return parser.parse_args()
 198
 199
 200 def main(args):
 201         global verbose_level
 202
 203         (opts, args) = parse_arguments(args)
 204
 205         if opts.quiet:
 206                 verbose_level = 0
 207         else:
 208                 verbose_level = opts.verbose_level
 209
 210         if opts.search_all:
 211                 args.extend(glob('*.so'))
 212
 213         if not args:
 214                 log(V_SUMMARY, 'No test suites to run')
 215                 return 0
 216
 217         results = dict(passed_suites=0, failed_suites=0, skipped_suites=0,
 218                         passed_cases=0, failed_cases=0,
 219                         passed_checks=0, failed_checks=0)
 220
 221         for so_name in args:
 222                 suite_name = splitext(basename(so_name))[0]
 223                 log(V_SUITE, '\nRunning test suite "%s"...', suite_name)
 224
 225                 try:
 226                         so = cdll.LoadLibrary(abspath(so_name))
 227                 except OSError, e:
 228                         log(V_ERROR, 'Error loading "%s" (%s), skipping '
 229                                         'test suite "%s"', so_name, e,
 230                                         suite_name)
 231                         results['skipped_suites'] += 1
 232                         continue
 233
 234                 info = TestSuiteInfo(so_name)
 235
 236                 suite = TestSuite(so, suite_name, info)
 237
 238                 if suite.api_version != API_VERSION:
 239                         log(V_ERROR, 'Wrong API version (%s expected, %s '
 240                                         'found), skipping test suite "%s"',
 241                                         API_VERSION, suite.api_version,
 242                                         suite.name)
 243                         results['skipped_suites'] += 1
 244                         continue
 245
 246                 r = suite.run()
 247
 248                 log(V_SUITE, 'Results: %s test case(s) passed, %s failed, '
 249                                 '%s check(s) passed, %s failed.',
 250                                 r.passed_cases, r.failed_cases,
 251                                 r.passed_checks, r.failed_checks)
 252
 253                 if r.result == R_FAILED:
 254                         results['failed_suites'] += 1
 255                 elif r.result == R_SKIPPED:
 256                         results['skipped_suites'] += 1
 257                 else:
 258                         results['passed_suites'] += 1
 259                 results['failed_cases'] += r.failed_cases
 260                 results['passed_cases'] += r.passed_cases
 261                 results['failed_checks'] += r.failed_checks
 262                 results['passed_checks'] += r.passed_checks
 263
 264         log(V_SUMMARY, SUMMARY_TEXT % results)
 265
 266         return 0
 267
 268
 269 if __name__ == '__main__':
 270         import sys
 271         sys.exit(main(sys.argv[1:]))
 272