X-Git-Url: http://rtime.felk.cvut.cz/gitweb/omk.git/blobdiff_plain/9a4417fc2f4ee4db242a101941d63a1fec9809cd..45996dec1b0b7712238f4c5b2f5a048b6072c08c:/tests/tester.py

diff --git a/tests/tester.py b/tests/tester.py
index 21b8b12..bd84969 100755
--- a/tests/tester.py
+++ b/tests/tester.py
@@ -44,10 +44,14 @@ class Results(dict):
         """ % self.__dict__
         s+=self.stats.toHtml()
         s+="""
-<h2>Chart</h2>
+<h2>Chart by rules</h2>
+        """
+        s+=self.toHtmlByRules();
+        s+="""
+<h2>Chart by testcase</h2>
 <table cellpadding='2' border='1'>
 <tbody>
-        """ % self.__dict__
+        """
         tests = sorted(self.keys())
         for t in tests:
             s+=self[t].toHtml()
@@ -61,7 +65,27 @@ class Results(dict):
 </body></html>
         """
         return s
-    
+
+    def toHtmlByRules(self):
+        rules = sorted(rulesdef.rules.keys())
+        tests = sorted(self.keys())
+        s=''
+        for r in rules:
+            s+="""
+<a name='results-%(rules)s' />
+<h3>Rules: %(rules)s</h3>
+<table cellpadding='2' border='1'>
+<tbody>
+""" % { 'rules':r }
+            for t in tests:
+                if self[t].has_key(r):
+                    s+=self[t][r].toHtml("<a href='%s'>/</a><a href='%s'>%s</a>" %
+                                         (self[t].tc.dirRelative, self[t].tc.scriptNameRelative, self[t].tc.name))
+            s+="""
+</tbody></table>
+"""
+        return s
+
     def save(self):
         f = file(self.filename, "w+")
         f.write(self.toHtml())
@@ -73,21 +97,21 @@ class Results(dict):
         print "Results written to "+self.filename
 
 class TestCaseResult(dict):
-    def __init__(self, tcname):
-        self.tcname = tcname
+    def __init__(self, tc):
+        self.tc = tc
         
     def toHtml(self):
         rules = sorted(self.keys())
         s="""
-  <tr><td colspan='6'><strong>%s</strong></td></tr>
-        """ % self.tcname
+  <tr><td colspan='5'><strong><a href='%s'>%s</a></strong></td></tr>
+        """ % (self.tc.scriptNameRelative, self.tc.name)
         for r in rules:
-            s+=self[r].toHtml()
+            s+=self[r].toHtml(r)
         return s
 
     def toHtmlOutputs(self):
         rules = sorted(self.keys())
-        s="<h3>Testcase: %s</h3>" % self.tcname
+        s="<h3>Testcase: %s</h3>" % self.tc.name
         for r in rules:
             s+=self[r].toHtmlOutputs()
         return s
@@ -96,35 +120,30 @@ class ResultEntry:
     def __init__(self, tcname, rules):
         self.tcname = tcname
         self.rules = rules
+        self.canttest = 0
         
-    def toHtml(self):
+    def toHtml(self, title):
         if self.exitcode == 0: color=''
-        else:
-            if self.exitcode == 1:   color=' bgcolor="red"'
-            elif self.exitcode == 2: color=' bgcolor="gray"'
-            else: color=' bgcolor="gray"'
-        if self.stdout: stdoutlink="<a href='#stdout-%(tcname)s-%(rules)s'>stdout</a>" % self.__dict__
+        elif self.canttest: color=' bgcolor="gold"'
+        else: color=' bgcolor="red"'
+        if self.stdout: stdoutlink="<a href='#output-%(tcname)s-%(rules)s'>output</a>" % self.__dict__
         else: stdoutlink=''
-        if self.stderr: stderrlink="<a href='#stderr-%(tcname)s-%(rules)s'>stderr</a>" % self.__dict__
-        else: stderrlink=''
         s="""
   <tr%(color)s>
-    <td>%(rules)s</td>
+    <td>%(title)s</td>
     <td>%(exitcode)d</td>
     <td>%(message)s</td>
     <td>%(time).1f s</td>
     <td>%(stdoutlink)s</td>
-    <td>%(stderrlink)s</td>
   </tr>
         """ % {
             'color' : color,
             'tcname' : self.rules,
-            'rules' : self.rules,
+            'title' : title,
             'exitcode' : self.exitcode,
             'message' : escape(self.message),
             'time' : self.time,
             'stdoutlink' : stdoutlink,
-            'stderrlink' : stderrlink,
             }
         return s
     def toHtmlOutputs(self):
@@ -132,17 +151,12 @@ class ResultEntry:
             'tcname':self.tcname,
             'rules':self.rules,
             'stdout':escape(self.stdout),
-            'stderr':escape(self.stderr)
             }
         s=""
         if self.stdout: s+="""
-<a name='stdout-%(tcname)s-%(rules)s'/>
-<h5>Test %(tcname)s, rules %(rules)s, stdout</h5>
+<a name='output-%(tcname)s-%(rules)s'/>
+<h5>Output of test: %(tcname)s, rules: %(rules)s</h5>
 <pre>%(stdout)s</pre>""" % vals
-        if self.stderr: s+="""
-<a name='stderr-%(tcname)s-%(rules)s'/>
-<h5>Test %(tcname)s, rules %(rules)s, stderr</h5>
-<pre>%(stderr)s</pre>""" % vals
         return s
 
 class RulesStat:
@@ -152,30 +166,26 @@ class RulesStat:
         self.success = 0
         self.errors = 0
         self.canttest = 0
-        self.unknown = 0
     def update(self, testCaseResult):
         try:
             resultEntry = testCaseResult[self.rules]
             self.tests+=1
             if resultEntry.exitcode == 0: self.success+=1
-            elif resultEntry.exitcode == 1: self.errors+=1
-            elif resultEntry.exitcode == 2: self.canttest+=1
-            else: self.unknown+=1
+            elif resultEntry.canttest: self.canttest+=1
+            else: self.errors+=1
         except KeyError:
             pass
     def toHtml(self):
         if self.errors == 0 and self.canttest == 0: self.color=''
         elif self.errors != 0:   self.color=' bgcolor="red"'
-        elif self.canttest != 0: self.color=' bgcolor="gray"'
-        else: self.color = ' bgcolor="gray"'
+        else: self.color = ' bgcolor="gold"'
         s="""
   <tr%(color)s>
-    <td>%(rules)s</td>
+    <td><a href='#results-%(rules)s'>%(rules)s</a></td>
     <td>%(tests)d</td>
     <td>%(success)d</td>
     <td>%(errors)d</td>
     <td>%(canttest)d</td>
-    <td>%(unknown)d</td>
   </tr>
         """ % self.__dict__
         return s
@@ -200,7 +210,6 @@ class Stats(dict):
   <td>Success</td>
   <td>Errors</td>
   <td>Can't test</td>
-  <td>Unknown</td>
 </tr></thead>
 <tbody>
         """
@@ -216,6 +225,8 @@ class TestCase:
         self.directory = directory      # Absolute directory
         self.executable = executable
         self.name = self._getName()
+        self.scriptNameRelative = self._getScriptRelative();
+        self.dirRelative = os.path.dirname(self.scriptNameRelative)
         self._whichRules()
 
     def _getName(self):
@@ -227,6 +238,12 @@ class TestCase:
             name+=" "+testSuffix
         return name
 
+    def _getScriptRelative(self):
+        script = os.path.join(self.directory, self.executable);
+        if script.startswith(invokeDir+"/"):
+            script = script[len(invokeDir)+1:]
+        return script
+
     def _whichRules(self):
         """Reads the rules file and creates the self.rules list of all
         rules to test"""
@@ -262,7 +279,7 @@ class TestCase:
         
 
     def run(self):
-        self.results = TestCaseResult(self.name)
+        self.results = TestCaseResult(self)
         print "Testing %s:\n" % self.name,
         os.chdir(os.path.join(testsRoot, self.directory))
 #         if os.path.exists("Makefile.test"):
@@ -299,18 +316,22 @@ class TestCase:
 
     def _execRuntest(self, log):
         startTime = time.clock()
-        pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+        pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+        (output, error) = pipe.communicate()
         endTime = time.clock()
-        (output, errors) = pipe.communicate()
         ret = pipe.returncode
-        log.exitcode = ret
         log.time = endTime - startTime
+        log.exitcode = ret
         log.stdout = output
-        log.stderr = errors
-        if ret != 0 and os.path.exists("_error"):
-            log.message = file("_error").read()
-        else: log.message = ''
-        
+        log.message = ''
+        if os.path.exists("_canttest"):
+            log.message = file("_canttest").read()
+            log.canttest = 1
+            ret = 2                     # If Makefile sets canttest, override the exit code which might be zero
+            log.exitcode = ret
+        elif ret != 0:
+            if os.path.exists("_error"):
+                log.message = file("_error").read()
         return ret
 
     def _copyRules(self, rules):
@@ -323,10 +344,9 @@ class TestCase:
         print "    ",os.environ['OMK_RULES'],
         sys.stdout.flush()
         ret = self._exec(log)
-        if ret == 0: retstr = "OK"
-        elif ret == 1: retstr = "FAILED"
-        elif ret == 2: retstr = "--"
-        else: retstr = "???"
+        if log.canttest: retstr = "--"
+        elif ret == 0: retstr = "OK"
+        else: retstr = "FAILED"
         print "%*s%s" % (20-len(os.environ['OMK_RULES']), "", retstr)