tests/tester.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import os.path
   5 import sys
   6 import re
   7 import shutil
   8 import subprocess
   9 import time
  10 from xml.sax.saxutils import escape
  11 import fnmatch
  12
  13 invokeDir = os.getcwd();
  14 testsRoot = os.path.dirname(os.path.abspath(__file__))
  15 if not os.path.exists(os.path.join(testsRoot, "tester.py")): raise "Can't find tests root directory!"
  16 os.environ['OMK_TESTSROOT'] = testsRoot
  17
  18 class Rules(dict):
  19     "List of all rules and their snippets"
  20     def __init__(self):
  21         for r in os.listdir("../rules"):
  22             snippets = {}
  23             fn = os.path.join("..", "rules", r, "Makefile.rules")
  24             f = open(fn)
  25             for line in f:
  26                 m = re.search("#OMK:(.*)\.omk", line);
  27                 if m:
  28                     snippets[m.group(1)] = 1;
  29             self[r] = snippets.keys()
  30         print self
  31
  32 rules = Rules()
  33
  34 class Results(dict):
  35     def __init__(self):
  36         self.time = time.gmtime()
  37         self.datetime = time.strftime("%Y-%m-%d %H:%M:%S +0000", self.time)
  38         self.filename = "results-"+time.strftime("%Y%m%d-%H%M%S", self.time)+".html"
  39         self.stats = None
  40
  41     def genStats(self):
  42         self.stats = Stats(self)
  43
  44     def toHtml(self):
  45         s="""
  46 <?xml version="1.0" encoding="iso-8859-1" ?>
  47 <!DOCTYPE html
  48   PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  49   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
  50 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  51 <head>
  52   <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1" />
  53   <title>OMK test report %(datetime)s</title>
  54 </head>
  55 <body>
  56 <h2>Summary</h2>
  57         """ % self.__dict__
  58         s+=self.stats.toHtml()
  59         s+="""
  60 <h2>Chart by rules</h2>
  61         """
  62         s+=self.toHtmlByRules();
  63         s+="""
  64 <h2>Chart by testcase</h2>
  65 <table cellpadding='2' border='1'>
  66 <tbody>
  67         """
  68         tests = sorted(self.keys())
  69         for t in tests:
  70             s+=self[t].toHtml()
  71         s+="""
  72 </tbody></table>
  73 <h2>Outputs of tests</h2>
  74         """
  75         for t in tests:
  76             s+=self[t].toHtmlOutputs()
  77         s+="""
  78 </body></html>
  79         """
  80         return s
  81
  82     def toHtmlByRules(self):
  83         rk = sorted(rules.keys())
  84         tests = sorted(self.keys())
  85         s=''
  86         for r in rk:
  87             s+="""
  88 <a name='results-%(rules)s' />
  89 <h3>Rules: %(rules)s</h3>
  90 <table cellpadding='2' border='1'>
  91 <tbody>
  92 """ % { 'rules':r }
  93             for t in tests:
  94                 if self[t].has_key(r):
  95                     s+=self[t][r].toHtml("<a href='%s'>/</a><a href='%s'>%s</a>" %
  96                                          (self[t].tc.dirRelative, self[t].tc.scriptNameRelative, self[t].tc.name))
  97             s+="""
  98 </tbody></table>
  99 """
 100         return s
 101
 102     def save(self):
 103         f = file(self.filename, "w+")
 104         f.write(self.toHtml())
 105         try:
 106             os.remove("results.html")
 107         except:
 108             pass
 109         os.symlink(self.filename, "results.html")
 110         print "Results written to "+self.filename
 111
 112 class TestCaseResult(dict):
 113     def __init__(self, tc):
 114         self.tc = tc
 115
 116     def toHtml(self):
 117         rules = sorted(self.keys())
 118         s="""
 119   <tr><td colspan='5'><strong><a href='%s'>%s</a></strong></td></tr>
 120         """ % (self.tc.scriptNameRelative, self.tc.name)
 121         for r in rules:
 122             s+=self[r].toHtml(r)
 123         return s
 124
 125     def toHtmlOutputs(self):
 126         rules = sorted(self.keys())
 127         s="<h3>Testcase: %s</h3>" % self.tc.name
 128         for r in rules:
 129             s+=self[r].toHtmlOutputs()
 130         return s
 131
 132 class ResultEntry:
 133     def __init__(self, tcname, rules):
 134         self.tcname = tcname
 135         self.rules = rules
 136         self.canttest = 0
 137
 138     def toHtml(self, title):
 139         if self.exitcode == 0: color=''
 140         elif self.canttest: color=' bgcolor="gold"'
 141         else: color=' bgcolor="red"'
 142         if self.stdout: stdoutlink="<a href='#output-%(tcname)s-%(rules)s'>output</a>" % self.__dict__
 143         else: stdoutlink=''
 144         s="""
 145   <tr%(color)s>
 146     <td>%(title)s</td>
 147     <td>%(exitcode)d</td>
 148     <td>%(message)s</td>
 149     <td>%(time).1f s</td>
 150     <td>%(stdoutlink)s</td>
 151   </tr>
 152         """ % {
 153             'color' : color,
 154             'tcname' : self.rules,
 155             'title' : title,
 156             'exitcode' : self.exitcode,
 157             'message' : escape(self.message),
 158             'time' : self.time,
 159             'stdoutlink' : stdoutlink,
 160             }
 161         return s
 162     def toHtmlOutputs(self):
 163         vals = {
 164             'tcname':self.tcname,
 165             'rules':self.rules,
 166             'stdout':escape(self.stdout),
 167             }
 168         s=""
 169         if self.stdout: s+="""
 170 <a name='output-%(tcname)s-%(rules)s'/>
 171 <h5>Output of test: %(tcname)s, rules: %(rules)s</h5>
 172 <pre>%(stdout)s</pre>""" % vals
 173         return s
 174
 175 class RulesStat:
 176     def __init__(self, rules):
 177         self.rules = rules
 178         self.tests = 0
 179         self.success = 0
 180         self.errors = 0
 181         self.canttest = 0
 182     def update(self, testCaseResult):
 183         try:
 184             resultEntry = testCaseResult[self.rules]
 185             self.tests+=1
 186             if resultEntry.exitcode == 0: self.success+=1
 187             elif resultEntry.canttest: self.canttest+=1
 188             else: self.errors+=1
 189         except KeyError:
 190             pass
 191     def toHtml(self):
 192         if self.errors == 0 and self.canttest == 0: self.color=''
 193         elif self.errors != 0:   self.color=' bgcolor="red"'
 194         else: self.color = ' bgcolor="gold"'
 195         s="""
 196   <tr%(color)s>
 197     <td><a href='#results-%(rules)s'>%(rules)s</a></td>
 198     <td>%(tests)d</td>
 199     <td>%(success)d</td>
 200     <td>%(errors)d</td>
 201     <td>%(canttest)d</td>
 202   </tr>
 203         """ % self.__dict__
 204         return s
 205
 206 class Stats(dict):
 207     def __init__(self, results):
 208         rk = rules.keys()
 209         for rule in rk:
 210             rulesStat = RulesStat(rule)
 211             self[rule]=rulesStat
 212             for resultEntry in results.values():
 213                 rulesStat.update(resultEntry)
 214
 215     def toHtml(self):
 216         s="""
 217 <table cellpadding='2' border='1'>
 218 <col />
 219 <col span='5' align='right' />
 220 <thead><tr>
 221   <td>Rules</td>
 222   <td>Total</td>
 223   <td>Success</td>
 224   <td>Errors</td>
 225   <td>Can't test</td>
 226 </tr></thead>
 227 <tbody>
 228         """
 229         rules = sorted(self.keys())
 230         for r in rules:
 231             s+=self[r].toHtml()
 232         s+="""
 233 </tbody></table>"""
 234         return s;
 235
 236 class TestCase:
 237     def __init__(self, directory, executable):
 238         self.directory = directory      # Absolute directory
 239         self.executable = executable
 240         self.name = self._getName()
 241         self.scriptNameRelative = self._getScriptRelative();
 242         self.dirRelative = os.path.dirname(self.scriptNameRelative)
 243         self._whichRules()
 244
 245     def _getName(self):
 246         name = self.directory
 247         if name.startswith(testsRoot+"/"):
 248             name = name[len(testsRoot)+1:]
 249         testSuffix = re.match("^runtest[-_. :]*(.*)", self.executable).group(1)
 250         if testSuffix:
 251             name+=" "+testSuffix
 252         return name
 253
 254     def _getScriptRelative(self):
 255         script = os.path.join(self.directory, self.executable);
 256         if script.startswith(invokeDir+"/"):
 257             script = script[len(invokeDir)+1:]
 258         return script
 259
 260     def _whichRules(self):
 261         """Reads the rules file and creates the self.rules list of all
 262         rules to test"""
 263         self.rules = []
 264         try:
 265             f = open(os.path.join(self.directory, self.executable+'.rules'))
 266         except IOError:
 267             self.rules = rules.keys()
 268             return
 269         line = f.readline()
 270         colonMatch = re.search('([^:]*) *: *(.*)', line)
 271         if colonMatch:
 272             if colonMatch.group(1) == "all":
 273                 # all:
 274                 self.rules = rules.keys()
 275             elif colonMatch.group(1) == "snip":
 276                 # snip: ...
 277                 snip = colonMatch.group(2)
 278                 for r in rules:
 279                     if snip in rules[r]:
 280                         self.rules.append(r)
 281             elif colonMatch.group(1) == "python":
 282                 # python: ...
 283                 expr = colonMatch.group(2)
 284                 for r in rules:
 285                     if eval(expr, {'rules': r, 'snippets': rules[r]}):
 286                         self.rules.append(r)
 287         else:
 288             # rule name
 289             line = line.strip()
 290             if line in rules: self.rules = [ line ]
 291         #print self.rules
 292
 293
 294     def run(self):
 295         self.results = TestCaseResult(self)
 296         print "Testing \"%s\" in %s:\n" % (self.name, self.executable),
 297         os.chdir(os.path.join(testsRoot, self.directory))
 298         # Chose how to run the test - currently there is only one option
 299 #         if os.path.exists("Makefile.test"):
 300 #             self._exec = self._execMake
 301         if os.path.exists(self.executable):
 302             self._exec = self._execRuntest
 303         else: return
 304         self.failed = False
 305         for rules in self.rules:
 306             resultEntry = ResultEntry(self.name, rules)
 307             self.results[rules] = resultEntry
 308             os.environ['OMK_RULES'] = rules
 309             filesBefore = self._getFileSet()
 310             self._copyRules(rules)
 311             try:
 312                 self._doRun(resultEntry)
 313             finally:
 314                 filesAfter = self._getFileSet()
 315                 self._clean(filesBefore, filesAfter)
 316         print
 317
 318     def _getFileSet(self):
 319         files = set()
 320         for f in os.listdir("."):
 321             files.add(f)
 322         return files
 323
 324     def _clean(self, filesBefore, filesAfter):
 325         remove = filesAfter - filesBefore
 326         for f in remove:
 327             os.system("rm -rf "+f)
 328
 329 #     def _execMake(self):
 330 #         return os.system("make -k -f Makefile.test > /dev/null 2>&1")
 331
 332     def _execRuntest(self, log):
 333         startTime = time.clock()
 334         pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
 335         (output, error) = pipe.communicate()
 336         endTime = time.clock()
 337         ret = pipe.returncode
 338         log.time = endTime - startTime
 339         log.exitcode = ret
 340         log.stdout = output
 341         log.message = ''
 342         if os.path.exists("_canttest"):
 343             log.message = file("_canttest").read()
 344             log.canttest = 1
 345             ret = 2                     # If Makefile sets canttest, override the exit code which might be zero
 346             log.exitcode = ret
 347         elif ret != 0:
 348             if os.path.exists("_error"):
 349                 log.message = file("_error").read()
 350         return ret
 351
 352     def _copyRules(self, rules):
 353         "Copies the rules to the current directory"
 354         src = os.path.join(testsRoot, "..", "rules", rules, "Makefile.rules")
 355         shutil.copy(src, ".")
 356
 357     def _doRun(self, log):
 358         "Runs the teset in current directory."
 359 #         print "    ",os.environ['OMK_RULES'],
 360 #         sys.stdout.flush()
 361         ret = self._exec(log)
 362 #         if log.canttest: retstr = "--"
 363 #         elif ret == 0: retstr = "ok"
 364 #         else:
 365 #             retstr = "FAILED"
 366 #             self.failed = True
 367 #         print "%*s%s" % (20-len(os.environ['OMK_RULES']), "", retstr)
 368         if log.canttest: retstr = "[not tested] ok"
 369         elif ret == 0: retstr = "ok"
 370         else:
 371             retstr = "FAILED"
 372             self.failed = True
 373         print "! %s (%s) %s" % (self.name, os.environ['OMK_RULES'], retstr)
 374         sys.stdout.flush()
 375
 376
 377 results = Results()
 378
 379 failed = False;
 380 for dirpath, dirnames, filenames in os.walk(invokeDir):
 381     executables = fnmatch.filter(filenames, "runtest*")
 382     if not executables: continue
 383     for exe in executables:
 384         if exe[-1] == "~": continue
 385         if re.search(".rules$", exe): continue
 386         t = TestCase(dirpath, exe)
 387         t.run()
 388         if t.failed: failed = True
 389         results[t.name] = t.results
 390
 391 os.chdir(invokeDir)
 392 results.genStats()
 393 results.save()
 394
 395 sys.exit(failed)
 396
 397 # Local Variables:
 398 # compile-command: "python tester.py"
 399 # End: