py/mutest

   1 #!/usr/bin/env python
   2
   3 import re
   4 from sys import stdout, stderr
   5 from optparse import OptionParser
   6 from glob import glob
   7 from os.path import basename, splitext, abspath
   8 from subprocess import Popen, PIPE
   9 from ctypes import cdll, c_int
  10
  11 API_VERSION = 1
  12
  13 V_QUIET   = 0
  14 V_ERROR   = 1
  15 V_SUMMARY = 2
  16 V_SUITE   = 3
  17 V_CASE    = 4
  18 V_CHECK   = 5
  19
  20 verbose_level = V_ERROR
  21
  22
  23 SUMMARY_TEXT = '''
  24 Tests done:
  25         %(passed_suites)s test suite(s) passed, %(failed_suites)s failed, \
  26 %(skipped_suites)s skipped.
  27         %(passed_cases)s test case(s) passed, %(failed_cases)s failed.
  28         %(passed_checks)s check(s) passed, %(failed_checks)s failed.
  29 '''
  30
  31 def log(level, msg, *args):
  32         global verbose_level
  33         out = stdout
  34         if level == V_ERROR:
  35                 stderr
  36         if verbose_level >= level:
  37                 out.write((msg % args) + '\n')
  38
  39
  40 def get_fun(so, name, argtype=None, restype=None):
  41         f = getattr(so, name)
  42         f.argtypes = argtype
  43         f.restype = restype
  44         return f
  45
  46
  47 def get_val(so, name):
  48         return c_int.in_dll(so, name).value
  49
  50
  51 #class SOError (Exception):
  52 #       pass
  53
  54
  55 class TestCase(object):
  56
  57         def __init__(self, so, name):
  58                 self.so = so
  59                 self.name = name
  60                 self.testcase = get_fun(so, name)
  61                 self.reset_counters = get_fun(so, 'mutest_reset_counters')
  62                 self.set_verbose_level = get_fun(so,
  63                                 'mutest_set_verbose_level', argtype=[c_int])
  64
  65         @property
  66         def passed_count(self):
  67                 return get_val(self.so, 'mutest_passed_count')
  68
  69         @property
  70         def failed_count(self):
  71                 return get_val(self.so, 'mutest_failed_count')
  72
  73         def run(self):
  74                 global verbose_level
  75                 self.set_verbose_level(verbose_level)
  76                 self.reset_counters()
  77                 self.testcase()
  78                 return (self.passed_count, self.failed_count)
  79
  80
  81 class TestSuiteResult (object):
  82         failed = False
  83         passed_cases = 0
  84         failed_cases = 0
  85         passed_checks = 0
  86         failed_checks = 0
  87
  88         def __repr__(self):
  89                 return 'TestSuiteResult(failed=%s, passed_cases=%s, '\
  90                         'failed_cases=%s, passed_checks=%s, failed_checks=%s)'\
  91                                 % (self.failed,
  92                                         self.passed_cases, self.failed_cases,
  93                                         self.passed_checks, self.failed_checks)
  94
  95
  96 class TestSuite (object):
  97
  98         def __init__(self, so, name, case_names):
  99                 self.name = name
 100                 self.so = so
 101                 try:
 102                         self.api_version = get_val(so, 'mutest_api_version')
 103                 except ValueError:
 104                         self.api_version = 0
 105                         return
 106                 self.cases = [TestCase(self.so, name) for name in case_names]
 107
 108         def run(self):
 109                 r = TestSuiteResult()
 110                 for case in self.cases:
 111                         log(V_CASE, "\t* Executing test case '%s'...",
 112                                         case.name)
 113                         (case_passed_checks, case_failed_checks) = case.run()
 114                         log(V_CASE, '\t  Results: %s check(s) passed, %s '
 115                                         'failed.', case_passed_checks,
 116                                         case_failed_checks)
 117                         if case_failed_checks:
 118                                 r.failed = True
 119                                 r.failed_cases += 1
 120                         else:
 121                                 r.passed_cases += 1
 122                         r.passed_checks += case_passed_checks
 123                         r.failed_checks += case_failed_checks
 124                 return r
 125
 126
 127 case_names_re = re.compile(r'[0-9a-f]{8} T (mu_test\w*)', re.I)
 128
 129 def get_case_names(so_name):
 130         proc = Popen(['nm', '-p', so_name], stdout=PIPE)
 131         output = proc.communicate()[0]
 132         return case_names_re.findall(output)
 133
 134
 135 def parse_arguments(args):
 136         verbose_help = ('Show a short result summary, add more for extra '
 137                         'verbosity: -vv for test suites progress, -vvv for '
 138                         'test cases progress and -vvvv for printing each '
 139                         'and every check done')
 140         quiet_help = ('Be quiet (overrides -v)')
 141         search_help = ('Search for all test suites in the current directory '
 142                         '(*.so) and add them')
 143         parser = OptionParser()
 144         parser.add_option('-v', dest='verbose_level', action='count',
 145                         default=1, help=verbose_help)
 146         parser.add_option('-q', '--verbose-level', dest='quiet',
 147                         action='store_true', default=False, help=quiet_help)
 148         parser.add_option('-a', '--search-all', dest='search_all',
 149                         action='store_true', default=False, help=search_help)
 150         return parser.parse_args()
 151
 152
 153 def main(args):
 154         global verbose_level
 155
 156         (opts, args) = parse_arguments(args)
 157
 158         if opts.quiet:
 159                 verbose_level = 0
 160         else:
 161                 verbose_level = opts.verbose_level
 162
 163         if opts.search_all:
 164                 args.extend(glob('*.so'))
 165
 166         if not args:
 167                 log(V_SUMMARY, 'No test suites to run')
 168                 return 0
 169
 170         results = dict(passed_suites=0, failed_suites=0, skipped_suites=0,
 171                         passed_cases=0, failed_cases=0,
 172                         passed_checks=0, failed_checks=0)
 173
 174         for so_name in args:
 175                 suite_name = splitext(basename(so_name))[0]
 176                 log(V_SUITE, '\nRunning test suite "%s"...', suite_name)
 177
 178                 try:
 179                         so = cdll.LoadLibrary(abspath(so_name))
 180                 except OSError, e:
 181                         log(V_ERROR, 'Error loading "%s" (%s), skipping '
 182                                         'test suite "%s"', so_name, e,
 183                                         suite_name)
 184                         results['skipped_suites'] += 1
 185                         continue
 186
 187                 case_names = get_case_names(so_name)
 188
 189                 suite = TestSuite(so, suite_name, case_names)
 190
 191                 if suite.api_version != API_VERSION:
 192                         log(V_ERROR, 'Wrong API version (%s expected, %s '
 193                                         'found), skipping test suite "%s"',
 194                                         API_VERSION, suite.api_version,
 195                                         suite.name)
 196                         results['skipped_suites'] += 1
 197                         continue
 198
 199                 r = suite.run()
 200
 201                 log(V_SUITE, 'Results: %s test case(s) passed, %s failed, '
 202                                 '%s check(s) passed, %s failed.',
 203                                 r.passed_cases, r.failed_cases,
 204                                 r.passed_checks, r.failed_checks)
 205
 206                 if r.failed:
 207                         results['failed_suites'] += 1
 208                 else:
 209                         results['passed_suites'] += 1
 210                 results['failed_cases'] += r.failed_cases
 211                 results['passed_cases'] += r.passed_cases
 212                 results['failed_checks'] += r.failed_checks
 213                 results['passed_checks'] += r.passed_checks
 214
 215         log(V_SUMMARY, SUMMARY_TEXT % results)
 216
 217         return 0
 218
 219
 220 if __name__ == '__main__':
 221         import sys
 222         sys.exit(main(sys.argv[1:]))
 223