tests/tester.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import os.path
   5 import sys
   6 import re
   7 import shutil
   8 import subprocess
   9 import time
  10 from xml.sax.saxutils import escape
  11 import fnmatch
  12
  13 invokeDir = os.getcwd();
  14 testsRoot = os.path.dirname(os.path.abspath(__file__))
  15 if not os.path.exists(os.path.join(testsRoot, "tester.py")): raise "Can't find tests root directory!"
  16 os.environ['OMK_TESTSROOT'] = testsRoot
  17
  18 sys.path.append(os.path.join(testsRoot, ".."))
  19 import rulesdef
  20
  21 class Results(dict):
  22     def __init__(self):
  23         self.time = time.gmtime()
  24         self.datetime = time.strftime("%Y-%m-%d %H:%M:%S +0000", self.time)
  25         self.filename = "results-"+time.strftime("%Y%m%d-%H%M%S", self.time)+".html"
  26         self.stats = None
  27
  28     def genStats(self):
  29         self.stats = Stats(self)
  30
  31     def toHtml(self):
  32         s="""
  33 <?xml version="1.0" encoding="iso-8859-1" ?>
  34 <!DOCTYPE html
  35   PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  36   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
  37 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  38 <head>
  39   <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1" />
  40   <title>OMK test report %(datetime)s</title>
  41 </head>
  42 <body>
  43 <h2>Summary</h2>
  44         """ % self.__dict__
  45         s+=self.stats.toHtml()
  46         s+="""
  47 <h2>Chart</h2>
  48 <table cellpadding='2' border='1'>
  49 <tbody>
  50         """ % self.__dict__
  51         tests = sorted(self.keys())
  52         for t in tests:
  53             s+=self[t].toHtml()
  54         s+="""
  55 </tbody></table>
  56 <h2>Outputs of tests</h2>
  57         """
  58         for t in tests:
  59             s+=self[t].toHtmlOutputs()
  60         s+="""
  61 </body></html>
  62         """
  63         return s
  64
  65     def save(self):
  66         f = file(self.filename, "w+")
  67         f.write(self.toHtml())
  68         try:
  69             os.remove("results.html")
  70         except:
  71             pass
  72         os.symlink(self.filename, "results.html")
  73         print "Results written to "+self.filename
  74
  75 class TestCaseResult(dict):
  76     def __init__(self, tcname):
  77         self.tcname = tcname
  78
  79     def toHtml(self):
  80         rules = sorted(self.keys())
  81         s="""
  82   <tr><td colspan='6'><strong>%s</strong></td></tr>
  83         """ % self.tcname
  84         for r in rules:
  85             s+=self[r].toHtml()
  86         return s
  87
  88     def toHtmlOutputs(self):
  89         rules = sorted(self.keys())
  90         s="<h3>Testcase: %s</h3>" % self.tcname
  91         for r in rules:
  92             s+=self[r].toHtmlOutputs()
  93         return s
  94
  95 class ResultEntry:
  96     def __init__(self, tcname, rules):
  97         self.tcname = tcname
  98         self.rules = rules
  99
 100     def toHtml(self):
 101         if self.exitcode == 0: color=''
 102         else:
 103             if self.exitcode == 1:   color=' bgcolor="red"'
 104             elif self.exitcode == 2: color=' bgcolor="gray"'
 105             else: color=' bgcolor="gray"'
 106         if self.stdout: stdoutlink="<a href='#stdout-%(tcname)s-%(rules)s'>stdout</a>" % self.__dict__
 107         else: stdoutlink=''
 108         if self.stderr: stderrlink="<a href='#stderr-%(tcname)s-%(rules)s'>stderr</a>" % self.__dict__
 109         else: stderrlink=''
 110         s="""
 111   <tr%(color)s>
 112     <td>%(rules)s</td>
 113     <td>%(exitcode)d</td>
 114     <td>%(message)s</td>
 115     <td>%(time).1f s</td>
 116     <td>%(stdoutlink)s</td>
 117     <td>%(stderrlink)s</td>
 118   </tr>
 119         """ % {
 120             'color' : color,
 121             'tcname' : self.rules,
 122             'rules' : self.rules,
 123             'exitcode' : self.exitcode,
 124             'message' : escape(self.message),
 125             'time' : self.time,
 126             'stdoutlink' : stdoutlink,
 127             'stderrlink' : stderrlink,
 128             }
 129         return s
 130     def toHtmlOutputs(self):
 131         vals = {
 132             'tcname':self.tcname,
 133             'rules':self.rules,
 134             'stdout':escape(self.stdout),
 135             'stderr':escape(self.stderr)
 136             }
 137         s=""
 138         if self.stdout: s+="""
 139 <a name='stdout-%(tcname)s-%(rules)s'/>
 140 <h5>Test %(tcname)s, rules %(rules)s, stdout</h5>
 141 <pre>%(stdout)s</pre>""" % vals
 142         if self.stderr: s+="""
 143 <a name='stderr-%(tcname)s-%(rules)s'/>
 144 <h5>Test %(tcname)s, rules %(rules)s, stderr</h5>
 145 <pre>%(stderr)s</pre>""" % vals
 146         return s
 147
 148 class RulesStat:
 149     def __init__(self, rules):
 150         self.rules = rules
 151         self.tests = 0
 152         self.success = 0
 153         self.errors = 0
 154         self.canttest = 0
 155         self.unknown = 0
 156     def update(self, testCaseResult):
 157         try:
 158             resultEntry = testCaseResult[self.rules]
 159             self.tests+=1
 160             if resultEntry.exitcode == 0: self.success+=1
 161             elif resultEntry.exitcode == 1: self.errors+=1
 162             elif resultEntry.exitcode == 2: self.canttest+=1
 163             else: self.unknown+=1
 164         except KeyError:
 165             pass
 166     def toHtml(self):
 167         if self.errors == 0 and self.canttest == 0: self.color=''
 168         elif self.errors != 0:   self.color=' bgcolor="red"'
 169         elif self.canttest != 0: self.color=' bgcolor="gray"'
 170         else: self.color = ' bgcolor="gray"'
 171         s="""
 172   <tr%(color)s>
 173     <td>%(rules)s</td>
 174     <td>%(tests)d</td>
 175     <td>%(success)d</td>
 176     <td>%(errors)d</td>
 177     <td>%(canttest)d</td>
 178     <td>%(unknown)d</td>
 179   </tr>
 180         """ % self.__dict__
 181         return s
 182
 183 class Stats(dict):
 184     def __init__(self, results):
 185         rules = rulesdef.rules.keys()
 186         for rule in rules:
 187             rulesStat = RulesStat(rule)
 188             self[rule]=rulesStat
 189             for resultEntry in results.values():
 190                 rulesStat.update(resultEntry)
 191
 192     def toHtml(self):
 193         s="""
 194 <table cellpadding='2' border='1'>
 195 <col />
 196 <col span='5' align='right' />
 197 <thead><tr>
 198   <td>Rules</td>
 199   <td>Total</td>
 200   <td>Success</td>
 201   <td>Errors</td>
 202   <td>Can't test</td>
 203   <td>Unknown</td>
 204 </tr></thead>
 205 <tbody>
 206         """
 207         rules = sorted(self.keys())
 208         for r in rules:
 209             s+=self[r].toHtml()
 210         s+="""
 211 </tbody></table>"""
 212         return s;
 213
 214 class TestCase:
 215     def __init__(self, directory, executable):
 216         self.directory = directory      # Absolute directory
 217         self.executable = executable
 218         self.name = self._getName()
 219         self._whichRules()
 220
 221     def _getName(self):
 222         name = self.directory
 223         if name.startswith(testsRoot+"/"):
 224             name = name[len(testsRoot)+1:]
 225         testSuffix = re.match("^runtest[-_. :]*(.*)", self.executable).group(1)
 226         if testSuffix:
 227             name+=" "+testSuffix
 228         return name
 229
 230     def _whichRules(self):
 231         """Reads the rules file and creates the self.rules list of all
 232         rules to test"""
 233         self.rules = []
 234         try:
 235             f = open(os.path.join(self.directory, self.executable+'.rules'))
 236         except IOError:
 237             self.rules = rulesdef.rules.keys()
 238             return
 239         line = f.readline()
 240         colonMatch = re.search('([^:]*) *: *(.*)', line)
 241         if colonMatch:
 242             if colonMatch.group(1) == "all":
 243                 # all:
 244                 self.rules = rulesdef.rules.keys()
 245             elif colonMatch.group(1) == "snip":
 246                 # snip: ...
 247                 snip = colonMatch.group(2)
 248                 for r in rulesdef.rules:
 249                     if snip in rulesdef.rules[r]:
 250                         self.rules.append(r)
 251             elif colonMatch.group(1) == "python":
 252                 # python: ...
 253                 expr = colonMatch.group(2)
 254                 for r in rulesdef.rules:
 255                     if eval(expr, {'rules': r, 'snippets': rulesdef.rules[r]}):
 256                         self.rules.append(r)
 257         else:
 258             # rule name
 259             line = line.strip()
 260             if line in rulesdef.rules: self.rules = [ line ]
 261         #print self.rules
 262
 263
 264     def run(self):
 265         self.results = TestCaseResult(self.name)
 266         print "Testing %s:\n" % self.name,
 267         os.chdir(os.path.join(testsRoot, self.directory))
 268 #         if os.path.exists("Makefile.test"):
 269 #             self._exec = self._execMake
 270         if os.path.exists(self.executable):
 271             self._exec = self._execRuntest
 272         else: return
 273         for rules in self.rules:
 274             resultEntry = ResultEntry(self.name, rules)
 275             self.results[rules] = resultEntry
 276             os.environ['OMK_RULES'] = rules
 277             filesBefore = self._getFileSet()
 278             self._copyRules(rules)
 279             try:
 280                 self._doRun(resultEntry)
 281             finally:
 282                 filesAfter = self._getFileSet()
 283                 self._clean(filesBefore, filesAfter)
 284         print
 285
 286     def _getFileSet(self):
 287         files = set()
 288         for f in os.listdir("."):
 289             files.add(f)
 290         return files
 291
 292     def _clean(self, filesBefore, filesAfter):
 293         remove = filesAfter - filesBefore
 294         for f in remove:
 295             os.system("rm -rf "+f)
 296
 297 #     def _execMake(self):
 298 #         return os.system("make -k -f Makefile.test > /dev/null 2>&1")
 299
 300     def _execRuntest(self, log):
 301         startTime = time.clock()
 302         pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 303         endTime = time.clock()
 304         (output, errors) = pipe.communicate()
 305         ret = pipe.returncode
 306         log.exitcode = ret
 307         log.time = endTime - startTime
 308         log.stdout = output
 309         log.stderr = errors
 310         if ret != 0 and os.path.exists("_error"):
 311             log.message = file("_error").read()
 312         else: log.message = ''
 313
 314         return ret
 315
 316     def _copyRules(self, rules):
 317         "Copies the rules to the current directory"
 318         src = os.path.join(testsRoot, "..", "rules", rules, "Makefile.rules")
 319         shutil.copy(src, ".")
 320
 321     def _doRun(self, log):
 322         "Runs the teset in current directory."
 323         print "    ",os.environ['OMK_RULES'],
 324         sys.stdout.flush()
 325         ret = self._exec(log)
 326         if ret == 0: retstr = "OK"
 327         elif ret == 1: retstr = "FAILED"
 328         elif ret == 2: retstr = "--"
 329         else: retstr = "???"
 330         print "%*s%s" % (20-len(os.environ['OMK_RULES']), "", retstr)
 331
 332
 333 results = Results()
 334
 335 for dirpath, dirnames, filenames in os.walk(invokeDir):
 336     executables = fnmatch.filter(filenames, "runtest*")
 337     if not executables: continue
 338     for exe in executables:
 339         if exe[-1] == "~": continue
 340         if re.search(".rules$", exe): continue
 341         t = TestCase(dirpath, exe)
 342         t.run()
 343         results[t.name] = t.results
 344
 345 os.chdir(invokeDir)
 346 results.genStats()
 347 results.save()
 348
 349 # Local Variables:
 350 # compile-command: "python tester.py"
 351 # End: