tests/runtests.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import os.path
   5 import sys
   6 import re
   7 import shutil
   8 import subprocess
   9 import time
  10
  11 sys.path.append("..")
  12 import rulesdef
  13
  14 class Results(dict):
  15     def __init__(self):
  16         self.time = time.gmtime()
  17         self.datetime = time.strftime("%Y-%m-%d %H:%M:%S +0000", self.time)
  18         self.filename = "results-"+time.strftime("%Y%m%d-%H%M%S", self.time)+".html"
  19
  20     def toHtml(self):
  21         s="""
  22 <?xml version="1.0" encoding="iso-8859-1" ?>
  23 <!DOCTYPE html
  24   PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  25   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
  26 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  27 <head>
  28   <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1" />
  29   <title>OMK test report %(datetime)s</title>
  30 </head>
  31 <body>
  32 <h2>Summary</h2>TODO
  33         """
  34         s+="""
  35 <h2>Chart</h2>
  36 <table cellpadding='2' border='1'>
  37 <tbody>
  38         """ % self.__dict__
  39         tests = sorted(self.keys())
  40         for t in tests:
  41             s+=self[t].toHtml()
  42         s+="""
  43 </tbody></table>
  44 <h2>Outputs of tests</h2>
  45         """
  46         for t in tests:
  47             s+=self[t].toHtmlOutputs()
  48         s+="""
  49 </body></html>
  50         """
  51         return s
  52
  53     def save(self):
  54         f = file(self.filename, "w+")
  55         f.write(self.toHtml())
  56         try:
  57             os.remove("results.html")
  58         except:
  59             pass
  60         os.symlink(self.filename, "results.html")
  61         print "Results written to "+self.filename
  62
  63 class TestCaseResult(dict):
  64     def __init__(self, tcname):
  65         self.tcname = tcname
  66
  67     def toHtml(self):
  68         rules = sorted(self.keys())
  69         s="""
  70   <tr><td colspan='6'><strong>%s</strong></td></tr>
  71         """ % self.tcname
  72         for r in rules:
  73             s+=self[r].toHtml()
  74         return s
  75
  76     def toHtmlOutputs(self):
  77         rules = sorted(self.keys())
  78         s="<h3>Testcase: %s</h3>" % self.tcname
  79         for r in rules:
  80             s+=self[r].toHtmlOutputs()
  81         return s
  82
  83 class ResultEntry:
  84     def __init__(self, tcname, rules):
  85         self.tcname = tcname
  86         self.rules = rules
  87
  88     def toHtml(self):
  89         if self.exitcode == 0: color=''
  90         else:
  91             if self.exitcode == 1:   color=' bgcolor="red"'
  92             elif self.exitcode == 2: color=' bgcolor="yellow"'
  93             else: color=' bgcolor="silver"'
  94         if self.stdout: stdoutlink="<a href='#stdout-%(tcname)s-%(rules)s'>stdout</a>" % self.__dict__
  95         else: stdoutlink=''
  96         if self.stderr: stderrlink="<a href='#stderr-%(tcname)s-%(rules)s'>stderr</a>" % self.__dict__
  97         else: stderrlink=''
  98         s="""
  99   <tr%(color)s>
 100     <td>%(rules)s</td>
 101     <td>%(exitcode)d</td>
 102     <td>%(message)s</td>
 103     <td>%(time).1f s</td>
 104     <td>%(stdoutlink)s</td>
 105     <td>%(stderrlink)s</td>
 106   </tr>
 107         """ % {
 108             'color' : color,
 109             'tcname' : self.rules,
 110             'rules' : self.rules,
 111             'exitcode' : self.exitcode,
 112             'message' : self.message,
 113             'time' : self.time,
 114             'stdoutlink' : stdoutlink,
 115             'stderrlink' : stderrlink,
 116             }
 117         return s
 118     def toHtmlOutputs(self):
 119         vals = {
 120             'tcname':self.tcname,
 121             'rules':self.rules,
 122             'stdout':self.stdout,
 123             'stderr':self.stderr
 124             }
 125         s=""
 126         if self.stdout: s+="""
 127 <a name='stdout-%(tcname)s-%(rules)s'/>
 128 <h5>Test %(tcname)s, rules %(rules)s, stdout</h5>
 129 <pre>%(stdout)s</pre>""" % vals
 130         if self.stderr: s+="""
 131 <a name='stderr-%(tcname)s-%(rules)s'/>
 132 <h5>Test %(tcname)s, rules %(rules)s, stderr</h5>
 133 <pre>%(stderr)s</pre>""" % vals
 134         return s
 135
 136 class TestCase:
 137     def __init__(self, directory):
 138         self.directory = directory      # Absolute directory
 139         self.name = self._getName()
 140         self._whichRules()
 141
 142     def _getName(self):
 143         name = self.directory
 144         if name.startswith(testsRoot+"/"):
 145             name = name[len(testsRoot)+1:]
 146         return name
 147
 148     def _whichRules(self):
 149         """Reads the rules file and creates the self.rules list of all
 150         rules to test"""
 151         self.rules = []
 152         try:
 153             f = open(os.path.join(self.directory, 'rules'))
 154         except IOError:
 155             self.rules = rulesdef.rules.keys()
 156             return
 157         line = f.readline()
 158         colonMatch = re.search('([^:]*) *: *(.*)', line)
 159         if colonMatch:
 160             if colonMatch.group(1) == "all":
 161                 # all:
 162                 self.rules = rulesdef.rules.keys()
 163             elif colonMatch.group(1) == "snip":
 164                 # snip: ...
 165                 snip = colonMatch.group(2)
 166                 for r in rulesdef.rules:
 167                     if snip in rulesdef.rules[r]:
 168                         self.rules.append(r)
 169             elif colonMatch.group(1) == "python":
 170                 # python: ...
 171                 expr = colonMatch.group(2)
 172                 for r in rulesdef.rules:
 173                     if eval(expr, {'rules': r, 'snippets': rulesdef.rules[r]}):
 174                         self.rules.append(r)
 175         else:
 176             # rule name
 177             if line in rulesdef.rules: self.rules = [ line ]
 178         #print self.rules
 179
 180
 181     def run(self):
 182         self.results = TestCaseResult(self.name)
 183         print "Testing %s:" % self.name,
 184         os.chdir(os.path.join(testsRoot, self.directory))
 185 #         if os.path.exists("Makefile.test"):
 186 #             self._exec = self._execMake
 187         if os.path.exists("runtest"):
 188             self._exec = self._execRuntest
 189         else: return
 190         for rules in self.rules:
 191             resultEntry = ResultEntry(self.name, rules)
 192             self.results[rules] = resultEntry
 193             print rules,
 194             os.environ['OMK_RULES'] = rules
 195             filesBefore = self._getFileSet()
 196             self._copyRules(rules)
 197             self._doRun(resultEntry)
 198             filesAfter = self._getFileSet()
 199             self._clean(filesBefore, filesAfter)
 200         print
 201
 202     def _getFileSet(self):
 203         files = set()
 204         for f in os.listdir("."):
 205             files.add(f)
 206         return files
 207
 208     def _clean(self, filesBefore, filesAfter):
 209         remove = filesAfter - filesBefore
 210         for f in remove:
 211             os.system("rm -rf "+f)
 212
 213 #     def _execMake(self):
 214 #         return os.system("make -k -f Makefile.test > /dev/null 2>&1")
 215
 216     def _execRuntest(self, log):
 217         startTime = time.clock()
 218         pipe = subprocess.Popen("./runtest", stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 219         endTime = time.clock()
 220         (output, errors) = pipe.communicate()
 221         ret = pipe.returncode
 222         log.exitcode = ret
 223         log.time = endTime - startTime
 224         log.stdout = output
 225         log.stderr = errors
 226         if ret != 0 and os.path.exists("_error"):
 227             log.message = file("_error").read()
 228         else: log.message = ''
 229
 230         return ret
 231
 232     def _copyRules(self, rules):
 233         "Copies the rules to the current directory"
 234         src = os.path.join(testsRoot, "..", "rules", rules, "Makefile.rules")
 235         shutil.copy(src, ".")
 236
 237     def _doRun(self, log):
 238         "Runs the teset in current directory."
 239         ret = self._exec(log)
 240         print ret,
 241
 242
 243 testsRoot = os.path.dirname(os.path.abspath(__file__))
 244 if not os.path.exists(os.path.join(testsRoot, "runtests.py")): raise "Can't find tests root directory!"
 245
 246 results = Results()
 247
 248 for dirpath, dirnames, filenames in os.walk(testsRoot):
 249     if not ("Makefile.test" in filenames or \
 250             "runtest" in filenames):
 251         continue
 252     t = TestCase(dirpath)
 253     t.run()
 254     results[t.name] = t.results
 255
 256 os.chdir(testsRoot)
 257 results.save()
 258
 259 # Local Variables:
 260 # compile-command: "python runtests.py"
 261 # End: