X-Git-Url: http://rtime.felk.cvut.cz/gitweb/omk.git/blobdiff_plain/9080379202e60755224be96fdd3aa59432a2e7f2..45996dec1b0b7712238f4c5b2f5a048b6072c08c:/tests/tester.py diff --git a/tests/tester.py b/tests/tester.py index 5ab454e..bd84969 100755 --- a/tests/tester.py +++ b/tests/tester.py @@ -10,7 +10,12 @@ import time from xml.sax.saxutils import escape import fnmatch -sys.path.append("..") +invokeDir = os.getcwd(); +testsRoot = os.path.dirname(os.path.abspath(__file__)) +if not os.path.exists(os.path.join(testsRoot, "tester.py")): raise "Can't find tests root directory!" +os.environ['OMK_TESTSROOT'] = testsRoot + +sys.path.append(os.path.join(testsRoot, "..")) import rulesdef class Results(dict): @@ -18,6 +23,10 @@ class Results(dict): self.time = time.gmtime() self.datetime = time.strftime("%Y-%m-%d %H:%M:%S +0000", self.time) self.filename = "results-"+time.strftime("%Y%m%d-%H%M%S", self.time)+".html" + self.stats = None + + def genStats(self): + self.stats = Stats(self) def toHtml(self): s=""" @@ -31,13 +40,18 @@ class Results(dict): OMK test report %(datetime)s -

Summary

TODO +

Summary

""" % self.__dict__ + s+=self.stats.toHtml() + s+=""" +

Chart by rules

+ """ + s+=self.toHtmlByRules(); s+=""" -

Chart

+

Chart by testcase

- """ % self.__dict__ + """ tests = sorted(self.keys()) for t in tests: s+=self[t].toHtml() @@ -51,7 +65,27 @@ class Results(dict): """ return s - + + def toHtmlByRules(self): + rules = sorted(rulesdef.rules.keys()) + tests = sorted(self.keys()) + s='' + for r in rules: + s+=""" + +

Rules: %(rules)s

+
+ +""" % { 'rules':r } + for t in tests: + if self[t].has_key(r): + s+=self[t][r].toHtml("/%s" % + (self[t].tc.dirRelative, self[t].tc.scriptNameRelative, self[t].tc.name)) + s+=""" +
+""" + return s + def save(self): f = file(self.filename, "w+") f.write(self.toHtml()) @@ -63,21 +97,21 @@ class Results(dict): print "Results written to "+self.filename class TestCaseResult(dict): - def __init__(self, tcname): - self.tcname = tcname + def __init__(self, tc): + self.tc = tc def toHtml(self): rules = sorted(self.keys()) s=""" - %s - """ % self.tcname + %s + """ % (self.tc.scriptNameRelative, self.tc.name) for r in rules: - s+=self[r].toHtml() + s+=self[r].toHtml(r) return s def toHtmlOutputs(self): rules = sorted(self.keys()) - s="

Testcase: %s

" % self.tcname + s="

Testcase: %s

" % self.tc.name for r in rules: s+=self[r].toHtmlOutputs() return s @@ -86,35 +120,30 @@ class ResultEntry: def __init__(self, tcname, rules): self.tcname = tcname self.rules = rules + self.canttest = 0 - def toHtml(self): + def toHtml(self, title): if self.exitcode == 0: color='' - else: - if self.exitcode == 1: color=' bgcolor="red"' - elif self.exitcode == 2: color=' bgcolor="yellow"' - else: color=' bgcolor="gray"' - if self.stdout: stdoutlink="stdout" % self.__dict__ + elif self.canttest: color=' bgcolor="gold"' + else: color=' bgcolor="red"' + if self.stdout: stdoutlink="output" % self.__dict__ else: stdoutlink='' - if self.stderr: stderrlink="stderr" % self.__dict__ - else: stderrlink='' s=""" - %(rules)s + %(title)s %(exitcode)d %(message)s %(time).1f s %(stdoutlink)s - %(stderrlink)s """ % { 'color' : color, 'tcname' : self.rules, - 'rules' : self.rules, + 'title' : title, 'exitcode' : self.exitcode, 'message' : escape(self.message), 'time' : self.time, 'stdoutlink' : stdoutlink, - 'stderrlink' : stderrlink, } return s def toHtmlOutputs(self): @@ -122,24 +151,82 @@ class ResultEntry: 'tcname':self.tcname, 'rules':self.rules, 'stdout':escape(self.stdout), - 'stderr':escape(self.stderr) } s="" if self.stdout: s+=""" - -
Test %(tcname)s, rules %(rules)s, stdout
+ +
Output of test: %(tcname)s, rules: %(rules)s
%(stdout)s
""" % vals - if self.stderr: s+=""" - -
Test %(tcname)s, rules %(rules)s, stderr
-
%(stderr)s
""" % vals return s +class RulesStat: + def __init__(self, rules): + self.rules = rules + self.tests = 0 + self.success = 0 + self.errors = 0 + self.canttest = 0 + def update(self, testCaseResult): + try: + resultEntry = testCaseResult[self.rules] + self.tests+=1 + if resultEntry.exitcode == 0: self.success+=1 + elif resultEntry.canttest: self.canttest+=1 + else: self.errors+=1 + except KeyError: + pass + def toHtml(self): + if self.errors == 0 and self.canttest == 0: self.color='' + elif self.errors != 0: self.color=' bgcolor="red"' + else: self.color = ' bgcolor="gold"' + s=""" + + %(rules)s + %(tests)d + %(success)d + %(errors)d + %(canttest)d + + """ % self.__dict__ + return s + +class Stats(dict): + def __init__(self, results): + rules = rulesdef.rules.keys() + for rule in rules: + rulesStat = RulesStat(rule) + self[rule]=rulesStat + for resultEntry in results.values(): + rulesStat.update(resultEntry) + + def toHtml(self): + s=""" + +++ + + + + + + + + """ + rules = sorted(self.keys()) + for r in rules: + s+=self[r].toHtml() + s+=""" +
RulesTotalSuccessErrorsCan't test
""" + return s; + class TestCase: def __init__(self, directory, executable): self.directory = directory # Absolute directory self.executable = executable self.name = self._getName() + self.scriptNameRelative = self._getScriptRelative(); + self.dirRelative = os.path.dirname(self.scriptNameRelative) self._whichRules() def _getName(self): @@ -151,12 +238,18 @@ class TestCase: name+=" "+testSuffix return name + def _getScriptRelative(self): + script = os.path.join(self.directory, self.executable); + if script.startswith(invokeDir+"/"): + script = script[len(invokeDir)+1:] + return script + def _whichRules(self): """Reads the rules file and creates the self.rules list of all rules to test""" self.rules = [] try: - f = open(os.path.join(self.directory, 'rules')) + f = open(os.path.join(self.directory, self.executable+'.rules')) except IOError: self.rules = rulesdef.rules.keys() return @@ -180,23 +273,23 @@ class TestCase: self.rules.append(r) else: # rule name + line = line.strip() if line in rulesdef.rules: self.rules = [ line ] #print self.rules def run(self): - self.results = TestCaseResult(self.name) - print "Testing %s:" % self.name, + self.results = TestCaseResult(self) + print "Testing %s:\n" % self.name, os.chdir(os.path.join(testsRoot, self.directory)) # if os.path.exists("Makefile.test"): # self._exec = self._execMake - if os.path.exists("runtest"): + if os.path.exists(self.executable): self._exec = self._execRuntest else: return for rules in self.rules: resultEntry = ResultEntry(self.name, rules) self.results[rules] = resultEntry - print rules, os.environ['OMK_RULES'] = rules filesBefore = self._getFileSet() self._copyRules(rules) @@ -223,18 +316,22 @@ class TestCase: def _execRuntest(self, log): startTime = time.clock() - pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) + pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) + (output, error) = pipe.communicate() endTime = time.clock() - (output, errors) = pipe.communicate() ret = pipe.returncode - log.exitcode = ret log.time = endTime - startTime + log.exitcode = ret log.stdout = output - log.stderr = errors - if ret != 0 and os.path.exists("_error"): - log.message = file("_error").read() - else: log.message = '' - + log.message = '' + if os.path.exists("_canttest"): + log.message = file("_canttest").read() + log.canttest = 1 + ret = 2 # If Makefile sets canttest, override the exit code which might be zero + log.exitcode = ret + elif ret != 0: + if os.path.exists("_error"): + log.message = file("_error").read() return ret def _copyRules(self, rules): @@ -244,28 +341,31 @@ class TestCase: def _doRun(self, log): "Runs the teset in current directory." + print " ",os.environ['OMK_RULES'], + sys.stdout.flush() ret = self._exec(log) - print ret, - + if log.canttest: retstr = "--" + elif ret == 0: retstr = "OK" + else: retstr = "FAILED" + print "%*s%s" % (20-len(os.environ['OMK_RULES']), "", retstr) -testsRoot = os.path.dirname(os.path.abspath(__file__)) -if not os.path.exists(os.path.join(testsRoot, "tester.py")): raise "Can't find tests root directory!" -os.environ['OMK_TESTSROOT'] = testsRoot results = Results() -for dirpath, dirnames, filenames in os.walk(testsRoot): +for dirpath, dirnames, filenames in os.walk(invokeDir): executables = fnmatch.filter(filenames, "runtest*") if not executables: continue for exe in executables: if exe[-1] == "~": continue + if re.search(".rules$", exe): continue t = TestCase(dirpath, exe) t.run() results[t.name] = t.results -os.chdir(testsRoot) +os.chdir(invokeDir) +results.genStats() results.save() # Local Variables: -# compile-command: "python runtests.py" +# compile-command: "python tester.py" # End: