py/mutest

   1 #!/usr/bin/env python
   2
   3 import re
   4 from sys import stdout, stderr
   5 from optparse import OptionParser
   6 from glob import glob
   7 from os.path import basename, splitext, abspath
   8 from subprocess import Popen, PIPE
   9 from ctypes import cdll, c_int
  10
  11 API_VERSION = 1
  12
  13 V_QUIET   = 0
  14 V_ERROR   = 1
  15 V_SUMMARY = 2
  16 V_SUITE   = 3
  17 V_CASE    = 4
  18 V_CHECK   = 5
  19
  20 verbose_level = V_ERROR
  21
  22
  23 SUMMARY_TEXT = '''
  24 Tests done:
  25         %(passed_suites)s test suite(s) passed, %(failed_suites)s failed, \
  26 %(skipped_suites)s skipped.
  27         %(passed_cases)s test case(s) passed, %(failed_cases)s failed.
  28         %(passed_checks)s check(s) passed, %(failed_checks)s failed.
  29 '''
  30
  31 def log(level, msg, *args):
  32         global verbose_level
  33         out = stdout
  34         if level == V_ERROR:
  35                 stderr
  36         if verbose_level >= level:
  37                 out.write((msg % args) + '\n')
  38
  39
  40 #class SOError (Exception):
  41 #       pass
  42
  43
  44 class TestCase(object):
  45
  46         def __init__(self, so, name):
  47                 self.so = so
  48                 self.name = name
  49                 self.testcase = self.get_fun(name)
  50                 self.reset_counters = self.get_fun('mutest_reset_counters')
  51                 self.set_verbose_level = self.get_fun(
  52                                 'mutest_set_verbose_level', argtype=[c_int])
  53
  54         @property
  55         def passed_count(self):
  56                 return self.get_val('mutest_passed_count')
  57
  58         @property
  59         def failed_count(self):
  60                 return self.get_val('mutest_failed_count')
  61
  62         def get_fun(self, name, argtype=None, restype=None):
  63                 f = getattr(self.so, name)
  64                 f.argtypes = argtype
  65                 f.restype = restype
  66                 return f
  67
  68         def get_val(self, name):
  69                 return c_int.in_dll(self.so, name).value
  70
  71         def run(self):
  72                 global verbose_level
  73                 self.set_verbose_level(verbose_level)
  74                 self.reset_counters()
  75                 self.testcase()
  76                 return (self.passed_count, self.failed_count)
  77
  78
  79 class TestSuiteResult (object):
  80         failed = False
  81         passed_cases = 0
  82         failed_cases = 0
  83         passed_checks = 0
  84         failed_checks = 0
  85
  86         def __repr__(self):
  87                 return 'TestSuiteResult(failed=%s, passed_cases=%s, '\
  88                         'failed_cases=%s, passed_checks=%s, failed_checks=%s)'\
  89                                 % (self.failed,
  90                                         self.passed_cases, self.failed_cases,
  91                                         self.passed_checks, self.failed_checks)
  92
  93
  94 class TestSuite (object):
  95
  96         def __init__(self, so, name, case_names):
  97                 self.name = name
  98                 self.so = so
  99                 try:
 100                         self.api_version = c_int.in_dll(self.so,
 101                                         'mutest_api_version').value
 102                 except ValueError:
 103                         self.api_version = 0
 104                         return
 105                 self.cases = [TestCase(self.so, name) for name in case_names]
 106
 107         def run(self):
 108                 r = TestSuiteResult()
 109                 for case in self.cases:
 110                         log(V_CASE, "\t* Executing test case '%s'...",
 111                                         case.name)
 112                         (case_passed_checks, case_failed_checks) = case.run()
 113                         log(V_CASE, '\t  Results: %s check(s) passed, %s '
 114                                         'failed.', case_passed_checks,
 115                                         case_failed_checks)
 116                         if case_failed_checks:
 117                                 r.failed = True
 118                                 r.failed_cases += 1
 119                         else:
 120                                 r.passed_cases += 1
 121                         r.passed_checks += case_passed_checks
 122                         r.failed_checks += case_failed_checks
 123                 return r
 124
 125
 126 case_names_re = re.compile(r'[0-9a-f]{8} T (mu_test_\w+)', re.I)
 127
 128 def get_case_names(so_name):
 129         proc = Popen(['nm', '-p', so_name], stdout=PIPE)
 130         output = proc.communicate()[0]
 131         return case_names_re.findall(output)
 132
 133
 134 def parse_arguments(args):
 135         verbose_help = ('Show a short result summary, add more for extra '
 136                         'verbosity: -vv for test suites progress, -vvv for '
 137                         'test cases progress and -vvvv for printing each '
 138                         'and every check done')
 139         quiet_help = ('Be quiet (overrides -v)')
 140         search_help = ('Search for all test suites in the current directory '
 141                         '(*.so) and add them')
 142         parser = OptionParser()
 143         parser.add_option('-v', dest='verbose_level', action='count',
 144                         default=1, help=verbose_help)
 145         parser.add_option('-q', '--verbose-level', dest='quiet',
 146                         action='store_true', default=False, help=quiet_help)
 147         parser.add_option('-a', '--search-all', dest='search_all',
 148                         action='store_true', default=False, help=search_help)
 149         return parser.parse_args()
 150
 151
 152 def main(args):
 153         global verbose_level
 154
 155         (opts, args) = parse_arguments(args)
 156
 157         if opts.quiet:
 158                 verbose_level = 0
 159         else:
 160                 verbose_level = opts.verbose_level
 161
 162         if opts.search_all:
 163                 args.extend(glob('*.so'))
 164
 165         if not args:
 166                 log(V_SUMMARY, 'No test suites to run')
 167                 return 0
 168
 169         results = dict(passed_suites=0, failed_suites=0, skipped_suites=0,
 170                         passed_cases=0, failed_cases=0,
 171                         passed_checks=0, failed_checks=0)
 172
 173         for so_name in args:
 174                 suite_name = splitext(basename(so_name))[0]
 175                 log(V_SUITE, '\nRunning test suite "%s"...', suite_name)
 176
 177                 try:
 178                         so = cdll.LoadLibrary(abspath(so_name))
 179                 except OSError, e:
 180                         log(V_ERROR, 'Error loading "%s" (%s), skipping '
 181                                         'test suite "%s"', so_name, e,
 182                                         suite_name)
 183                         results['skipped_suites'] += 1
 184                         continue
 185
 186                 case_names = get_case_names(so_name)
 187
 188                 suite = TestSuite(so, suite_name, case_names)
 189
 190                 if suite.api_version != API_VERSION:
 191                         log(V_ERROR, 'Wrong API version (%s expected, %s '
 192                                         'found), skipping test suite "%s"',
 193                                         API_VERSION, suite.api_version,
 194                                         suite.name)
 195                         results['skipped_suites'] += 1
 196                         continue
 197
 198                 r = suite.run()
 199
 200                 log(V_SUITE, 'Results: %s test case(s) passed, %s failed, '
 201                                 '%s check(s) passed, %s failed.',
 202                                 r.passed_cases, r.failed_cases,
 203                                 r.passed_checks, r.failed_checks)
 204
 205                 if r.failed:
 206                         results['failed_suites'] += 1
 207                 else:
 208                         results['passed_suites'] += 1
 209                 results['failed_cases'] += r.failed_cases
 210                 results['passed_cases'] += r.passed_cases
 211                 results['failed_checks'] += r.failed_checks
 212                 results['passed_checks'] += r.passed_checks
 213
 214         log(V_SUMMARY, SUMMARY_TEXT % results)
 215
 216         return 0
 217
 218
 219 if __name__ == '__main__':
 220         import sys
 221         sys.exit(main(sys.argv[1:]))
 222