tests/tester.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import os.path
   5 import sys
   6 import re
   7 import shutil
   8 import subprocess
   9 import time
  10 from xml.sax.saxutils import escape
  11 import fnmatch
  12
  13 invokeDir = os.getcwd();
  14 testsRoot = os.path.dirname(os.path.abspath(__file__))
  15 if not os.path.exists(os.path.join(testsRoot, "tester.py")): raise "Can't find tests root directory!"
  16 os.environ['OMK_TESTSROOT'] = testsRoot
  17
  18 sys.path.append(os.path.join(testsRoot, ".."))
  19 import rulesdef
  20
  21 class Results(dict):
  22     def __init__(self):
  23         self.time = time.gmtime()
  24         self.datetime = time.strftime("%Y-%m-%d %H:%M:%S +0000", self.time)
  25         self.filename = "results-"+time.strftime("%Y%m%d-%H%M%S", self.time)+".html"
  26         self.stats = None
  27
  28     def genStats(self):
  29         self.stats = Stats(self)
  30
  31     def toHtml(self):
  32         s="""
  33 <?xml version="1.0" encoding="iso-8859-1" ?>
  34 <!DOCTYPE html
  35   PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  36   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
  37 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  38 <head>
  39   <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1" />
  40   <title>OMK test report %(datetime)s</title>
  41 </head>
  42 <body>
  43 <h2>Summary</h2>
  44         """ % self.__dict__
  45         s+=self.stats.toHtml()
  46         s+="""
  47 <h2>Chart by rules</h2>
  48         """
  49         s+=self.toHtmlByRules();
  50         s+="""
  51 <h2>Chart by testcase</h2>
  52 <table cellpadding='2' border='1'>
  53 <tbody>
  54         """
  55         tests = sorted(self.keys())
  56         for t in tests:
  57             s+=self[t].toHtml()
  58         s+="""
  59 </tbody></table>
  60 <h2>Outputs of tests</h2>
  61         """
  62         for t in tests:
  63             s+=self[t].toHtmlOutputs()
  64         s+="""
  65 </body></html>
  66         """
  67         return s
  68
  69     def toHtmlByRules(self):
  70         rules = sorted(rulesdef.rules.keys())
  71         tests = sorted(self.keys())
  72         s=''
  73         for r in rules:
  74             s+="""
  75 <a name='results-%(rules)s' />
  76 <h3>Rules: %(rules)s</h3>
  77 <table cellpadding='2' border='1'>
  78 <tbody>
  79 """ % { 'rules':r }
  80             for t in tests:
  81                 if self[t].has_key(r):
  82                     s+=self[t][r].toHtml("<a href='%s'>/</a><a href='%s'>%s</a>" %
  83                                          (self[t].tc.dirRelative, self[t].tc.scriptNameRelative, self[t].tc.name))
  84             s+="""
  85 </tbody></table>
  86 """
  87         return s
  88
  89     def save(self):
  90         f = file(self.filename, "w+")
  91         f.write(self.toHtml())
  92         try:
  93             os.remove("results.html")
  94         except:
  95             pass
  96         os.symlink(self.filename, "results.html")
  97         print "Results written to "+self.filename
  98
  99 class TestCaseResult(dict):
 100     def __init__(self, tc):
 101         self.tc = tc
 102
 103     def toHtml(self):
 104         rules = sorted(self.keys())
 105         s="""
 106   <tr><td colspan='5'><strong><a href='%s'>%s</a></strong></td></tr>
 107         """ % (self.tc.scriptNameRelative, self.tc.name)
 108         for r in rules:
 109             s+=self[r].toHtml(r)
 110         return s
 111
 112     def toHtmlOutputs(self):
 113         rules = sorted(self.keys())
 114         s="<h3>Testcase: %s</h3>" % self.tc.name
 115         for r in rules:
 116             s+=self[r].toHtmlOutputs()
 117         return s
 118
 119 class ResultEntry:
 120     def __init__(self, tcname, rules):
 121         self.tcname = tcname
 122         self.rules = rules
 123         self.canttest = 0
 124
 125     def toHtml(self, title):
 126         if self.exitcode == 0: color=''
 127         elif self.canttest: color=' bgcolor="gold"'
 128         else: color=' bgcolor="red"'
 129         if self.stdout: stdoutlink="<a href='#output-%(tcname)s-%(rules)s'>output</a>" % self.__dict__
 130         else: stdoutlink=''
 131         s="""
 132   <tr%(color)s>
 133     <td>%(title)s</td>
 134     <td>%(exitcode)d</td>
 135     <td>%(message)s</td>
 136     <td>%(time).1f s</td>
 137     <td>%(stdoutlink)s</td>
 138   </tr>
 139         """ % {
 140             'color' : color,
 141             'tcname' : self.rules,
 142             'title' : title,
 143             'exitcode' : self.exitcode,
 144             'message' : escape(self.message),
 145             'time' : self.time,
 146             'stdoutlink' : stdoutlink,
 147             }
 148         return s
 149     def toHtmlOutputs(self):
 150         vals = {
 151             'tcname':self.tcname,
 152             'rules':self.rules,
 153             'stdout':escape(self.stdout),
 154             }
 155         s=""
 156         if self.stdout: s+="""
 157 <a name='output-%(tcname)s-%(rules)s'/>
 158 <h5>Output of test: %(tcname)s, rules: %(rules)s</h5>
 159 <pre>%(stdout)s</pre>""" % vals
 160         return s
 161
 162 class RulesStat:
 163     def __init__(self, rules):
 164         self.rules = rules
 165         self.tests = 0
 166         self.success = 0
 167         self.errors = 0
 168         self.canttest = 0
 169     def update(self, testCaseResult):
 170         try:
 171             resultEntry = testCaseResult[self.rules]
 172             self.tests+=1
 173             if resultEntry.exitcode == 0: self.success+=1
 174             elif resultEntry.canttest: self.canttest+=1
 175             else: self.errors+=1
 176         except KeyError:
 177             pass
 178     def toHtml(self):
 179         if self.errors == 0 and self.canttest == 0: self.color=''
 180         elif self.errors != 0:   self.color=' bgcolor="red"'
 181         else: self.color = ' bgcolor="gold"'
 182         s="""
 183   <tr%(color)s>
 184     <td><a href='#results-%(rules)s'>%(rules)s</a></td>
 185     <td>%(tests)d</td>
 186     <td>%(success)d</td>
 187     <td>%(errors)d</td>
 188     <td>%(canttest)d</td>
 189   </tr>
 190         """ % self.__dict__
 191         return s
 192
 193 class Stats(dict):
 194     def __init__(self, results):
 195         rules = rulesdef.rules.keys()
 196         for rule in rules:
 197             rulesStat = RulesStat(rule)
 198             self[rule]=rulesStat
 199             for resultEntry in results.values():
 200                 rulesStat.update(resultEntry)
 201
 202     def toHtml(self):
 203         s="""
 204 <table cellpadding='2' border='1'>
 205 <col />
 206 <col span='5' align='right' />
 207 <thead><tr>
 208   <td>Rules</td>
 209   <td>Total</td>
 210   <td>Success</td>
 211   <td>Errors</td>
 212   <td>Can't test</td>
 213 </tr></thead>
 214 <tbody>
 215         """
 216         rules = sorted(self.keys())
 217         for r in rules:
 218             s+=self[r].toHtml()
 219         s+="""
 220 </tbody></table>"""
 221         return s;
 222
 223 class TestCase:
 224     def __init__(self, directory, executable):
 225         self.directory = directory      # Absolute directory
 226         self.executable = executable
 227         self.name = self._getName()
 228         self.scriptNameRelative = self._getScriptRelative();
 229         self.dirRelative = os.path.dirname(self.scriptNameRelative)
 230         self._whichRules()
 231
 232     def _getName(self):
 233         name = self.directory
 234         if name.startswith(testsRoot+"/"):
 235             name = name[len(testsRoot)+1:]
 236         testSuffix = re.match("^runtest[-_. :]*(.*)", self.executable).group(1)
 237         if testSuffix:
 238             name+=" "+testSuffix
 239         return name
 240
 241     def _getScriptRelative(self):
 242         script = os.path.join(self.directory, self.executable);
 243         if script.startswith(invokeDir+"/"):
 244             script = script[len(invokeDir)+1:]
 245         return script
 246
 247     def _whichRules(self):
 248         """Reads the rules file and creates the self.rules list of all
 249         rules to test"""
 250         self.rules = []
 251         try:
 252             f = open(os.path.join(self.directory, self.executable+'.rules'))
 253         except IOError:
 254             self.rules = rulesdef.rules.keys()
 255             return
 256         line = f.readline()
 257         colonMatch = re.search('([^:]*) *: *(.*)', line)
 258         if colonMatch:
 259             if colonMatch.group(1) == "all":
 260                 # all:
 261                 self.rules = rulesdef.rules.keys()
 262             elif colonMatch.group(1) == "snip":
 263                 # snip: ...
 264                 snip = colonMatch.group(2)
 265                 for r in rulesdef.rules:
 266                     if snip in rulesdef.rules[r]:
 267                         self.rules.append(r)
 268             elif colonMatch.group(1) == "python":
 269                 # python: ...
 270                 expr = colonMatch.group(2)
 271                 for r in rulesdef.rules:
 272                     if eval(expr, {'rules': r, 'snippets': rulesdef.rules[r]}):
 273                         self.rules.append(r)
 274         else:
 275             # rule name
 276             line = line.strip()
 277             if line in rulesdef.rules: self.rules = [ line ]
 278         #print self.rules
 279
 280
 281     def run(self):
 282         self.results = TestCaseResult(self)
 283         print "Testing %s:\n" % self.name,
 284         os.chdir(os.path.join(testsRoot, self.directory))
 285 #         if os.path.exists("Makefile.test"):
 286 #             self._exec = self._execMake
 287         if os.path.exists(self.executable):
 288             self._exec = self._execRuntest
 289         else: return
 290         for rules in self.rules:
 291             resultEntry = ResultEntry(self.name, rules)
 292             self.results[rules] = resultEntry
 293             os.environ['OMK_RULES'] = rules
 294             filesBefore = self._getFileSet()
 295             self._copyRules(rules)
 296             try:
 297                 self._doRun(resultEntry)
 298             finally:
 299                 filesAfter = self._getFileSet()
 300                 self._clean(filesBefore, filesAfter)
 301         print
 302
 303     def _getFileSet(self):
 304         files = set()
 305         for f in os.listdir("."):
 306             files.add(f)
 307         return files
 308
 309     def _clean(self, filesBefore, filesAfter):
 310         remove = filesAfter - filesBefore
 311         for f in remove:
 312             os.system("rm -rf "+f)
 313
 314 #     def _execMake(self):
 315 #         return os.system("make -k -f Makefile.test > /dev/null 2>&1")
 316
 317     def _execRuntest(self, log):
 318         startTime = time.clock()
 319         pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
 320         (output, error) = pipe.communicate()
 321         endTime = time.clock()
 322         ret = pipe.returncode
 323         log.time = endTime - startTime
 324         log.exitcode = ret
 325         log.stdout = output
 326         log.message = ''
 327         if os.path.exists("_canttest"):
 328             log.message = file("_canttest").read()
 329             log.canttest = 1
 330             ret = 2                     # If Makefile sets canttest, override the exit code which might be zero
 331             log.exitcode = ret
 332         elif ret != 0:
 333             if os.path.exists("_error"):
 334                 log.message = file("_error").read()
 335         return ret
 336
 337     def _copyRules(self, rules):
 338         "Copies the rules to the current directory"
 339         src = os.path.join(testsRoot, "..", "rules", rules, "Makefile.rules")
 340         shutil.copy(src, ".")
 341
 342     def _doRun(self, log):
 343         "Runs the teset in current directory."
 344         print "    ",os.environ['OMK_RULES'],
 345         sys.stdout.flush()
 346         ret = self._exec(log)
 347         if log.canttest: retstr = "--"
 348         elif ret == 0: retstr = "OK"
 349         else: retstr = "FAILED"
 350         print "%*s%s" % (20-len(os.environ['OMK_RULES']), "", retstr)
 351
 352
 353 results = Results()
 354
 355 for dirpath, dirnames, filenames in os.walk(invokeDir):
 356     executables = fnmatch.filter(filenames, "runtest*")
 357     if not executables: continue
 358     for exe in executables:
 359         if exe[-1] == "~": continue
 360         if re.search(".rules$", exe): continue
 361         t = TestCase(dirpath, exe)
 362         t.run()
 363         results[t.name] = t.results
 364
 365 os.chdir(invokeDir)
 366 results.genStats()
 367 results.save()
 368
 369 # Local Variables:
 370 # compile-command: "python tester.py"
 371 # End: