py/mutest

   1 #!/usr/bin/env python
   2
   3 import re
   4 from sys import stdout, stderr
   5 from optparse import OptionParser
   6 from glob import glob
   7 from os.path import basename, splitext, abspath
   8 from subprocess import Popen, PIPE
   9 from ctypes import cdll, c_int
  10
  11 API_VERSION = 1
  12
  13 V_QUIET   = 0
  14 V_ERROR   = 1
  15 V_SUMMARY = 2
  16 V_SUITE   = 3
  17 V_CASE    = 4
  18 V_CHECK   = 5
  19
  20 R_OK      = 1
  21 R_FAILED  = 2
  22 R_SKIPPED = 3
  23
  24 verbose_level = V_ERROR
  25
  26
  27 SUMMARY_TEXT = '''
  28 Tests done:
  29         %(passed_suites)s test suite(s) passed, %(failed_suites)s failed, \
  30 %(skipped_suites)s skipped.
  31         %(passed_cases)s test case(s) passed, %(failed_cases)s failed.
  32         %(passed_checks)s check(s) passed, %(failed_checks)s failed.
  33 '''
  34
  35 def log(level, msg, *args):
  36         global verbose_level
  37         out = stdout
  38         if level == V_ERROR:
  39                 stderr
  40         if verbose_level >= level:
  41                 out.write((msg % args) + '\n')
  42
  43
  44 def get_fun(so, name, argtype=None, restype=None):
  45         f = getattr(so, name)
  46         f.argtypes = argtype
  47         f.restype = restype
  48         return f
  49
  50
  51 def get_val(so, name):
  52         return c_int.in_dll(so, name).value
  53
  54
  55 #class SOError (Exception):
  56 #       pass
  57
  58
  59 class TestCase(object):
  60
  61         def __init__(self, so, name):
  62                 self.so = so
  63                 self.name = name
  64                 self.testcase = get_fun(so, name)
  65                 self.reset_counters = get_fun(so, 'mutest_reset_counters')
  66                 self.set_verbose_level = get_fun(so,
  67                                 'mutest_set_verbose_level', argtype=[c_int])
  68
  69         @property
  70         def passed_count(self):
  71                 return get_val(self.so, 'mutest_passed_count')
  72
  73         @property
  74         def failed_count(self):
  75                 return get_val(self.so, 'mutest_failed_count')
  76
  77         def run(self):
  78                 global verbose_level
  79                 self.set_verbose_level(verbose_level)
  80                 self.reset_counters()
  81                 self.testcase()
  82                 return (self.passed_count, self.failed_count)
  83
  84
  85 class TestSuiteInfo (object):
  86
  87         inits_re = re.compile(r'[0-9a-f]{8} T (mu_init\w*)', re.I)
  88         terms_re = re.compile(r'[0-9a-f]{8} T (mu_term\w*)', re.I)
  89         cases_re = re.compile(r'[0-9a-f]{8} T (mu_test\w*)', re.I)
  90
  91         def __init__(self, so_name):
  92                 proc = Popen(['nm', '-p', so_name], stdout=PIPE)
  93                 output = proc.communicate()[0]
  94                 self.inits = self.inits_re.findall(output)
  95                 self.terms = self.terms_re.findall(output)
  96                 self.cases = self.cases_re.findall(output)
  97
  98
  99 class TestSuiteResult (object):
 100         result = R_OK
 101         passed_cases = 0
 102         failed_cases = 0
 103         passed_checks = 0
 104         failed_checks = 0
 105
 106         def __repr__(self):
 107                 return 'TestSuiteResult(failed=%s, passed_cases=%s, '\
 108                         'failed_cases=%s, passed_checks=%s, failed_checks=%s)'\
 109                                 % (self.result,
 110                                         self.passed_cases, self.failed_cases,
 111                                         self.passed_checks, self.failed_checks)
 112
 113
 114 class TestSuite (object):
 115
 116         def __init__(self, so, name, info):
 117                 self.name = name
 118                 self.so = so
 119                 try:
 120                         self.api_version = get_val(so, 'mutest_api_version')
 121                 except ValueError:
 122                         self.api_version = 0
 123                         return
 124                 self.inits = dict([(name, get_fun(so, name, restype=c_int))
 125                                 for name in info.inits])
 126                 self.terms = dict([(name, get_fun(so, name))
 127                                 for name in info.terms])
 128                 self.cases = [TestCase(self.so, name)
 129                                 for name in info.cases]
 130
 131         def run(self):
 132                 r = TestSuiteResult()
 133
 134                 for name, func in self.inits.items():
 135                         log(V_CASE, "\t+ Executing initialization function "
 136                                         "'%s'...", name);
 137                         res = func()
 138                         if res:
 139                                 log(V_ERROR, "%s:%s: initialization function "
 140                                                 "failed (returned %d), "
 141                                                 "skipping test suite...\n",
 142                                                 self.name, name, res);
 143                                 r.result = R_SKIPPED
 144                                 return r
 145
 146                 for case in self.cases:
 147                         log(V_CASE, "\t* Executing test case '%s'...",
 148                                         case.name)
 149                         (case_passed_checks, case_failed_checks) = case.run()
 150                         log(V_CASE, '\t  Results: %s check(s) passed, %s '
 151                                         'failed.', case_passed_checks,
 152                                         case_failed_checks)
 153                         if case_failed_checks:
 154                                 r.result = R_FAILED
 155                                 r.failed_cases += 1
 156                         else:
 157                                 r.passed_cases += 1
 158                         r.passed_checks += case_passed_checks
 159                         r.failed_checks += case_failed_checks
 160
 161                 for name, func in self.terms.items():
 162                         log(V_CASE, "\t- Executing termination function "
 163                                         "'%s'...", name)
 164                         func()
 165
 166                 return r
 167
 168
 169 def parse_arguments(args):
 170         verbose_help = ('Show a short result summary, add more for extra '
 171                         'verbosity: -vv for test suites progress, -vvv for '
 172                         'test cases progress and -vvvv for printing each '
 173                         'and every check done')
 174         quiet_help = ('Be quiet (overrides -v)')
 175         search_help = ('Search for all test suites in the current directory '
 176                         '(*.so) and add them')
 177         parser = OptionParser()
 178         parser.add_option('-v', dest='verbose_level', action='count',
 179                         default=1, help=verbose_help)
 180         parser.add_option('-q', '--verbose-level', dest='quiet',
 181                         action='store_true', default=False, help=quiet_help)
 182         parser.add_option('-a', '--search-all', dest='search_all',
 183                         action='store_true', default=False, help=search_help)
 184         return parser.parse_args()
 185
 186
 187 def main(args):
 188         global verbose_level
 189
 190         (opts, args) = parse_arguments(args)
 191
 192         if opts.quiet:
 193                 verbose_level = 0
 194         else:
 195                 verbose_level = opts.verbose_level
 196
 197         if opts.search_all:
 198                 args.extend(glob('*.so'))
 199
 200         if not args:
 201                 log(V_SUMMARY, 'No test suites to run')
 202                 return 0
 203
 204         results = dict(passed_suites=0, failed_suites=0, skipped_suites=0,
 205                         passed_cases=0, failed_cases=0,
 206                         passed_checks=0, failed_checks=0)
 207
 208         for so_name in args:
 209                 suite_name = splitext(basename(so_name))[0]
 210                 log(V_SUITE, '\nRunning test suite "%s"...', suite_name)
 211
 212                 try:
 213                         so = cdll.LoadLibrary(abspath(so_name))
 214                 except OSError, e:
 215                         log(V_ERROR, 'Error loading "%s" (%s), skipping '
 216                                         'test suite "%s"', so_name, e,
 217                                         suite_name)
 218                         results['skipped_suites'] += 1
 219                         continue
 220
 221                 info = TestSuiteInfo(so_name)
 222
 223                 suite = TestSuite(so, suite_name, info)
 224
 225                 if suite.api_version != API_VERSION:
 226                         log(V_ERROR, 'Wrong API version (%s expected, %s '
 227                                         'found), skipping test suite "%s"',
 228                                         API_VERSION, suite.api_version,
 229                                         suite.name)
 230                         results['skipped_suites'] += 1
 231                         continue
 232
 233                 r = suite.run()
 234
 235                 log(V_SUITE, 'Results: %s test case(s) passed, %s failed, '
 236                                 '%s check(s) passed, %s failed.',
 237                                 r.passed_cases, r.failed_cases,
 238                                 r.passed_checks, r.failed_checks)
 239
 240                 if r.result == R_FAILED:
 241                         results['failed_suites'] += 1
 242                 elif r.result == R_SKIPPED:
 243                         results['skipped_suites'] += 1
 244                 else:
 245                         results['passed_suites'] += 1
 246                 results['failed_cases'] += r.failed_cases
 247                 results['passed_cases'] += r.passed_cases
 248                 results['failed_checks'] += r.failed_checks
 249                 results['passed_checks'] += r.passed_checks
 250
 251         log(V_SUMMARY, SUMMARY_TEXT % results)
 252
 253         return 0
 254
 255
 256 if __name__ == '__main__':
 257         import sys
 258         sys.exit(main(sys.argv[1:]))
 259