tests/tester.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import os.path
   5 import sys
   6 import re
   7 import shutil
   8 import subprocess
   9 import time
  10 from xml.sax.saxutils import escape
  11 import fnmatch
  12
  13 sys.path.append("..")
  14 import rulesdef
  15
  16 class Results(dict):
  17     def __init__(self):
  18         self.time = time.gmtime()
  19         self.datetime = time.strftime("%Y-%m-%d %H:%M:%S +0000", self.time)
  20         self.filename = "results-"+time.strftime("%Y%m%d-%H%M%S", self.time)+".html"
  21
  22     def toHtml(self):
  23         s="""
  24 <?xml version="1.0" encoding="iso-8859-1" ?>
  25 <!DOCTYPE html
  26   PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  27   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
  28 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  29 <head>
  30   <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1" />
  31   <title>OMK test report %(datetime)s</title>
  32 </head>
  33 <body>
  34 <h2>Summary</h2>TODO
  35         """ % self.__dict__
  36         s+="""
  37 <h2>Chart</h2>
  38 <table cellpadding='2' border='1'>
  39 <tbody>
  40         """ % self.__dict__
  41         tests = sorted(self.keys())
  42         for t in tests:
  43             s+=self[t].toHtml()
  44         s+="""
  45 </tbody></table>
  46 <h2>Outputs of tests</h2>
  47         """
  48         for t in tests:
  49             s+=self[t].toHtmlOutputs()
  50         s+="""
  51 </body></html>
  52         """
  53         return s
  54
  55     def save(self):
  56         f = file(self.filename, "w+")
  57         f.write(self.toHtml())
  58         try:
  59             os.remove("results.html")
  60         except:
  61             pass
  62         os.symlink(self.filename, "results.html")
  63         print "Results written to "+self.filename
  64
  65 class TestCaseResult(dict):
  66     def __init__(self, tcname):
  67         self.tcname = tcname
  68
  69     def toHtml(self):
  70         rules = sorted(self.keys())
  71         s="""
  72   <tr><td colspan='6'><strong>%s</strong></td></tr>
  73         """ % self.tcname
  74         for r in rules:
  75             s+=self[r].toHtml()
  76         return s
  77
  78     def toHtmlOutputs(self):
  79         rules = sorted(self.keys())
  80         s="<h3>Testcase: %s</h3>" % self.tcname
  81         for r in rules:
  82             s+=self[r].toHtmlOutputs()
  83         return s
  84
  85 class ResultEntry:
  86     def __init__(self, tcname, rules):
  87         self.tcname = tcname
  88         self.rules = rules
  89
  90     def toHtml(self):
  91         if self.exitcode == 0: color=''
  92         else:
  93             if self.exitcode == 1:   color=' bgcolor="red"'
  94             elif self.exitcode == 2: color=' bgcolor="yellow"'
  95             else: color=' bgcolor="gray"'
  96         if self.stdout: stdoutlink="<a href='#stdout-%(tcname)s-%(rules)s'>stdout</a>" % self.__dict__
  97         else: stdoutlink=''
  98         if self.stderr: stderrlink="<a href='#stderr-%(tcname)s-%(rules)s'>stderr</a>" % self.__dict__
  99         else: stderrlink=''
 100         s="""
 101   <tr%(color)s>
 102     <td>%(rules)s</td>
 103     <td>%(exitcode)d</td>
 104     <td>%(message)s</td>
 105     <td>%(time).1f s</td>
 106     <td>%(stdoutlink)s</td>
 107     <td>%(stderrlink)s</td>
 108   </tr>
 109         """ % {
 110             'color' : color,
 111             'tcname' : self.rules,
 112             'rules' : self.rules,
 113             'exitcode' : self.exitcode,
 114             'message' : escape(self.message),
 115             'time' : self.time,
 116             'stdoutlink' : stdoutlink,
 117             'stderrlink' : stderrlink,
 118             }
 119         return s
 120     def toHtmlOutputs(self):
 121         vals = {
 122             'tcname':self.tcname,
 123             'rules':self.rules,
 124             'stdout':escape(self.stdout),
 125             'stderr':escape(self.stderr)
 126             }
 127         s=""
 128         if self.stdout: s+="""
 129 <a name='stdout-%(tcname)s-%(rules)s'/>
 130 <h5>Test %(tcname)s, rules %(rules)s, stdout</h5>
 131 <pre>%(stdout)s</pre>""" % vals
 132         if self.stderr: s+="""
 133 <a name='stderr-%(tcname)s-%(rules)s'/>
 134 <h5>Test %(tcname)s, rules %(rules)s, stderr</h5>
 135 <pre>%(stderr)s</pre>""" % vals
 136         return s
 137
 138 class TestCase:
 139     def __init__(self, directory, executable):
 140         self.directory = directory      # Absolute directory
 141         self.executable = executable
 142         self.name = self._getName()
 143         self._whichRules()
 144
 145     def _getName(self):
 146         name = self.directory
 147         if name.startswith(testsRoot+"/"):
 148             name = name[len(testsRoot)+1:]
 149         testSuffix = re.match("^runtest[-_. :]*(.*)", self.executable).group(1)
 150         if testSuffix:
 151             name+=" "+testSuffix
 152         return name
 153
 154     def _whichRules(self):
 155         """Reads the rules file and creates the self.rules list of all
 156         rules to test"""
 157         self.rules = []
 158         try:
 159             f = open(os.path.join(self.directory, 'rules'))
 160         except IOError:
 161             self.rules = rulesdef.rules.keys()
 162             return
 163         line = f.readline()
 164         colonMatch = re.search('([^:]*) *: *(.*)', line)
 165         if colonMatch:
 166             if colonMatch.group(1) == "all":
 167                 # all:
 168                 self.rules = rulesdef.rules.keys()
 169             elif colonMatch.group(1) == "snip":
 170                 # snip: ...
 171                 snip = colonMatch.group(2)
 172                 for r in rulesdef.rules:
 173                     if snip in rulesdef.rules[r]:
 174                         self.rules.append(r)
 175             elif colonMatch.group(1) == "python":
 176                 # python: ...
 177                 expr = colonMatch.group(2)
 178                 for r in rulesdef.rules:
 179                     if eval(expr, {'rules': r, 'snippets': rulesdef.rules[r]}):
 180                         self.rules.append(r)
 181         else:
 182             # rule name
 183             if line in rulesdef.rules: self.rules = [ line ]
 184         #print self.rules
 185
 186
 187     def run(self):
 188         self.results = TestCaseResult(self.name)
 189         print "Testing %s:" % self.name,
 190         os.chdir(os.path.join(testsRoot, self.directory))
 191 #         if os.path.exists("Makefile.test"):
 192 #             self._exec = self._execMake
 193         if os.path.exists("runtest"):
 194             self._exec = self._execRuntest
 195         else: return
 196         for rules in self.rules:
 197             resultEntry = ResultEntry(self.name, rules)
 198             self.results[rules] = resultEntry
 199             print rules,
 200             os.environ['OMK_RULES'] = rules
 201             filesBefore = self._getFileSet()
 202             self._copyRules(rules)
 203             try:
 204                 self._doRun(resultEntry)
 205             finally:
 206                 filesAfter = self._getFileSet()
 207                 self._clean(filesBefore, filesAfter)
 208         print
 209
 210     def _getFileSet(self):
 211         files = set()
 212         for f in os.listdir("."):
 213             files.add(f)
 214         return files
 215
 216     def _clean(self, filesBefore, filesAfter):
 217         remove = filesAfter - filesBefore
 218         for f in remove:
 219             os.system("rm -rf "+f)
 220
 221 #     def _execMake(self):
 222 #         return os.system("make -k -f Makefile.test > /dev/null 2>&1")
 223
 224     def _execRuntest(self, log):
 225         startTime = time.clock()
 226         pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 227         endTime = time.clock()
 228         (output, errors) = pipe.communicate()
 229         ret = pipe.returncode
 230         log.exitcode = ret
 231         log.time = endTime - startTime
 232         log.stdout = output
 233         log.stderr = errors
 234         if ret != 0 and os.path.exists("_error"):
 235             log.message = file("_error").read()
 236         else: log.message = ''
 237
 238         return ret
 239
 240     def _copyRules(self, rules):
 241         "Copies the rules to the current directory"
 242         src = os.path.join(testsRoot, "..", "rules", rules, "Makefile.rules")
 243         shutil.copy(src, ".")
 244
 245     def _doRun(self, log):
 246         "Runs the teset in current directory."
 247         ret = self._exec(log)
 248         print ret,
 249
 250
 251 testsRoot = os.path.dirname(os.path.abspath(__file__))
 252 if not os.path.exists(os.path.join(testsRoot, "tester.py")): raise "Can't find tests root directory!"
 253 os.environ['OMK_TESTSROOT'] = testsRoot
 254
 255 results = Results()
 256
 257 for dirpath, dirnames, filenames in os.walk(testsRoot):
 258     executables = fnmatch.filter(filenames, "runtest*")
 259     if not executables: continue
 260     for exe in executables:
 261         if exe[-1] == "~": continue
 262         t = TestCase(dirpath, exe)
 263         t.run()
 264         results[t.name] = t.results
 265
 266 os.chdir(testsRoot)
 267 results.save()
 268
 269 # Local Variables:
 270 # compile-command: "python runtests.py"
 271 # End: