tests/tester.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import os.path
   5 import sys
   6 import re
   7 import shutil
   8 import subprocess
   9 import time
  10 from xml.sax.saxutils import escape
  11 import fnmatch
  12
  13 invokeDir = os.getcwd();
  14 testsRoot = os.path.dirname(os.path.abspath(__file__))
  15 if not os.path.exists(os.path.join(testsRoot, "tester.py")): raise "Can't find tests root directory!"
  16 os.environ['OMK_TESTSROOT'] = testsRoot
  17
  18 sys.path.append(os.path.join(testsRoot, ".."))
  19 import rulesdef
  20
  21 class Results(dict):
  22     def __init__(self):
  23         self.time = time.gmtime()
  24         self.datetime = time.strftime("%Y-%m-%d %H:%M:%S +0000", self.time)
  25         self.filename = "results-"+time.strftime("%Y%m%d-%H%M%S", self.time)+".html"
  26
  27     def toHtml(self):
  28         s="""
  29 <?xml version="1.0" encoding="iso-8859-1" ?>
  30 <!DOCTYPE html
  31   PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  32   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
  33 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  34 <head>
  35   <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1" />
  36   <title>OMK test report %(datetime)s</title>
  37 </head>
  38 <body>
  39 <h2>Summary</h2>TODO
  40         """ % self.__dict__
  41         s+="""
  42 <h2>Chart</h2>
  43 <table cellpadding='2' border='1'>
  44 <tbody>
  45         """ % self.__dict__
  46         tests = sorted(self.keys())
  47         for t in tests:
  48             s+=self[t].toHtml()
  49         s+="""
  50 </tbody></table>
  51 <h2>Outputs of tests</h2>
  52         """
  53         for t in tests:
  54             s+=self[t].toHtmlOutputs()
  55         s+="""
  56 </body></html>
  57         """
  58         return s
  59
  60     def save(self):
  61         f = file(self.filename, "w+")
  62         f.write(self.toHtml())
  63         try:
  64             os.remove("results.html")
  65         except:
  66             pass
  67         os.symlink(self.filename, "results.html")
  68         print "Results written to "+self.filename
  69
  70 class TestCaseResult(dict):
  71     def __init__(self, tcname):
  72         self.tcname = tcname
  73
  74     def toHtml(self):
  75         rules = sorted(self.keys())
  76         s="""
  77   <tr><td colspan='6'><strong>%s</strong></td></tr>
  78         """ % self.tcname
  79         for r in rules:
  80             s+=self[r].toHtml()
  81         return s
  82
  83     def toHtmlOutputs(self):
  84         rules = sorted(self.keys())
  85         s="<h3>Testcase: %s</h3>" % self.tcname
  86         for r in rules:
  87             s+=self[r].toHtmlOutputs()
  88         return s
  89
  90 class ResultEntry:
  91     def __init__(self, tcname, rules):
  92         self.tcname = tcname
  93         self.rules = rules
  94
  95     def toHtml(self):
  96         if self.exitcode == 0: color=''
  97         else:
  98             if self.exitcode == 1:   color=' bgcolor="red"'
  99             elif self.exitcode == 2: color=' bgcolor="yellow"'
 100             else: color=' bgcolor="gray"'
 101         if self.stdout: stdoutlink="<a href='#stdout-%(tcname)s-%(rules)s'>stdout</a>" % self.__dict__
 102         else: stdoutlink=''
 103         if self.stderr: stderrlink="<a href='#stderr-%(tcname)s-%(rules)s'>stderr</a>" % self.__dict__
 104         else: stderrlink=''
 105         s="""
 106   <tr%(color)s>
 107     <td>%(rules)s</td>
 108     <td>%(exitcode)d</td>
 109     <td>%(message)s</td>
 110     <td>%(time).1f s</td>
 111     <td>%(stdoutlink)s</td>
 112     <td>%(stderrlink)s</td>
 113   </tr>
 114         """ % {
 115             'color' : color,
 116             'tcname' : self.rules,
 117             'rules' : self.rules,
 118             'exitcode' : self.exitcode,
 119             'message' : escape(self.message),
 120             'time' : self.time,
 121             'stdoutlink' : stdoutlink,
 122             'stderrlink' : stderrlink,
 123             }
 124         return s
 125     def toHtmlOutputs(self):
 126         vals = {
 127             'tcname':self.tcname,
 128             'rules':self.rules,
 129             'stdout':escape(self.stdout),
 130             'stderr':escape(self.stderr)
 131             }
 132         s=""
 133         if self.stdout: s+="""
 134 <a name='stdout-%(tcname)s-%(rules)s'/>
 135 <h5>Test %(tcname)s, rules %(rules)s, stdout</h5>
 136 <pre>%(stdout)s</pre>""" % vals
 137         if self.stderr: s+="""
 138 <a name='stderr-%(tcname)s-%(rules)s'/>
 139 <h5>Test %(tcname)s, rules %(rules)s, stderr</h5>
 140 <pre>%(stderr)s</pre>""" % vals
 141         return s
 142
 143 class TestCase:
 144     def __init__(self, directory, executable):
 145         self.directory = directory      # Absolute directory
 146         self.executable = executable
 147         self.name = self._getName()
 148         self._whichRules()
 149
 150     def _getName(self):
 151         name = self.directory
 152         if name.startswith(testsRoot+"/"):
 153             name = name[len(testsRoot)+1:]
 154         testSuffix = re.match("^runtest[-_. :]*(.*)", self.executable).group(1)
 155         if testSuffix:
 156             name+=" "+testSuffix
 157         return name
 158
 159     def _whichRules(self):
 160         """Reads the rules file and creates the self.rules list of all
 161         rules to test"""
 162         self.rules = []
 163         try:
 164             f = open(os.path.join(self.directory, 'rules'))
 165         except IOError:
 166             self.rules = rulesdef.rules.keys()
 167             return
 168         line = f.readline()
 169         colonMatch = re.search('([^:]*) *: *(.*)', line)
 170         if colonMatch:
 171             if colonMatch.group(1) == "all":
 172                 # all:
 173                 self.rules = rulesdef.rules.keys()
 174             elif colonMatch.group(1) == "snip":
 175                 # snip: ...
 176                 snip = colonMatch.group(2)
 177                 for r in rulesdef.rules:
 178                     if snip in rulesdef.rules[r]:
 179                         self.rules.append(r)
 180             elif colonMatch.group(1) == "python":
 181                 # python: ...
 182                 expr = colonMatch.group(2)
 183                 for r in rulesdef.rules:
 184                     if eval(expr, {'rules': r, 'snippets': rulesdef.rules[r]}):
 185                         self.rules.append(r)
 186         else:
 187             # rule name
 188             if line in rulesdef.rules: self.rules = [ line ]
 189         #print self.rules
 190
 191
 192     def run(self):
 193         self.results = TestCaseResult(self.name)
 194         print "Testing %s:" % self.name,
 195         os.chdir(os.path.join(testsRoot, self.directory))
 196 #         if os.path.exists("Makefile.test"):
 197 #             self._exec = self._execMake
 198         if os.path.exists("runtest"):
 199             self._exec = self._execRuntest
 200         else: return
 201         for rules in self.rules:
 202             resultEntry = ResultEntry(self.name, rules)
 203             self.results[rules] = resultEntry
 204             print rules,
 205             os.environ['OMK_RULES'] = rules
 206             filesBefore = self._getFileSet()
 207             self._copyRules(rules)
 208             try:
 209                 self._doRun(resultEntry)
 210             finally:
 211                 filesAfter = self._getFileSet()
 212                 self._clean(filesBefore, filesAfter)
 213         print
 214
 215     def _getFileSet(self):
 216         files = set()
 217         for f in os.listdir("."):
 218             files.add(f)
 219         return files
 220
 221     def _clean(self, filesBefore, filesAfter):
 222         remove = filesAfter - filesBefore
 223         for f in remove:
 224             os.system("rm -rf "+f)
 225
 226 #     def _execMake(self):
 227 #         return os.system("make -k -f Makefile.test > /dev/null 2>&1")
 228
 229     def _execRuntest(self, log):
 230         startTime = time.clock()
 231         pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 232         endTime = time.clock()
 233         (output, errors) = pipe.communicate()
 234         ret = pipe.returncode
 235         log.exitcode = ret
 236         log.time = endTime - startTime
 237         log.stdout = output
 238         log.stderr = errors
 239         if ret != 0 and os.path.exists("_error"):
 240             log.message = file("_error").read()
 241         else: log.message = ''
 242
 243         return ret
 244
 245     def _copyRules(self, rules):
 246         "Copies the rules to the current directory"
 247         src = os.path.join(testsRoot, "..", "rules", rules, "Makefile.rules")
 248         shutil.copy(src, ".")
 249
 250     def _doRun(self, log):
 251         "Runs the teset in current directory."
 252         ret = self._exec(log)
 253         print ret,
 254
 255
 256 results = Results()
 257
 258 for dirpath, dirnames, filenames in os.walk(invokeDir):
 259     executables = fnmatch.filter(filenames, "runtest*")
 260     if not executables: continue
 261     for exe in executables:
 262         if exe[-1] == "~": continue
 263         t = TestCase(dirpath, exe)
 264         t.run()
 265         results[t.name] = t.results
 266
 267 os.chdir(invokeDir)
 268 results.save()
 269
 270 # Local Variables:
 271 # compile-command: "python runtests.py"
 272 # End: