if not os.path.exists(os.path.join(testsRoot, "tester.py")): raise "Can't find tests root directory!"
os.environ['OMK_TESTSROOT'] = testsRoot
-sys.path.append(os.path.join(testsRoot, ".."))
-import rulesdef
+class Rules(dict):
+ "List of all rules and their snippets"
+ def __init__(self):
+ for r in os.listdir("../rules"):
+ snippets = {}
+ fn = os.path.join("..", "rules", r, "Makefile.rules")
+ f = open(fn)
+ for line in f:
+ m = re.search("#OMK:(.*)\.omk", line);
+ if m:
+ snippets[m.group(1)] = 1;
+ self[r] = snippets.keys()
+ print self
+
+rules = Rules()
class Results(dict):
def __init__(self):
""" % self.__dict__
s+=self.stats.toHtml()
s+="""
-<h2>Chart</h2>
+<h2>Chart by rules</h2>
+ """
+ s+=self.toHtmlByRules();
+ s+="""
+<h2>Chart by testcase</h2>
<table cellpadding='2' border='1'>
<tbody>
- """ % self.__dict__
+ """
tests = sorted(self.keys())
for t in tests:
s+=self[t].toHtml()
</body></html>
"""
return s
-
+
+ def toHtmlByRules(self):
+ rk = sorted(rules.keys())
+ tests = sorted(self.keys())
+ s=''
+ for r in rk:
+ s+="""
+<a name='results-%(rules)s' />
+<h3>Rules: %(rules)s</h3>
+<table cellpadding='2' border='1'>
+<tbody>
+""" % { 'rules':r }
+ for t in tests:
+ if self[t].has_key(r):
+ s+=self[t][r].toHtml("<a href='%s'>/</a><a href='%s'>%s</a>" %
+ (self[t].tc.dirRelative, self[t].tc.scriptNameRelative, self[t].tc.name))
+ s+="""
+</tbody></table>
+"""
+ return s
+
def save(self):
f = file(self.filename, "w+")
f.write(self.toHtml())
print "Results written to "+self.filename
class TestCaseResult(dict):
- def __init__(self, tcname):
- self.tcname = tcname
+ def __init__(self, tc):
+ self.tc = tc
def toHtml(self):
rules = sorted(self.keys())
s="""
- <tr><td colspan='6'><strong>%s</strong></td></tr>
- """ % self.tcname
+ <tr><td colspan='5'><strong><a href='%s'>%s</a></strong></td></tr>
+ """ % (self.tc.scriptNameRelative, self.tc.name)
for r in rules:
- s+=self[r].toHtml()
+ s+=self[r].toHtml(r)
return s
def toHtmlOutputs(self):
rules = sorted(self.keys())
- s="<h3>Testcase: %s</h3>" % self.tcname
+ s="<h3>Testcase: %s</h3>" % self.tc.name
for r in rules:
s+=self[r].toHtmlOutputs()
return s
def __init__(self, tcname, rules):
self.tcname = tcname
self.rules = rules
+ self.canttest = 0
- def toHtml(self):
+ def toHtml(self, title):
if self.exitcode == 0: color=''
- else:
- if self.exitcode == 1: color=' bgcolor="red"'
- elif self.exitcode == 2: color=' bgcolor="yellow"'
- else: color=' bgcolor="gray"'
- if self.stdout: stdoutlink="<a href='#stdout-%(tcname)s-%(rules)s'>stdout</a>" % self.__dict__
+ elif self.canttest: color=' bgcolor="gold"'
+ else: color=' bgcolor="red"'
+ if self.stdout: stdoutlink="<a href='#output-%(tcname)s-%(rules)s'>output</a>" % self.__dict__
else: stdoutlink=''
- if self.stderr: stderrlink="<a href='#stderr-%(tcname)s-%(rules)s'>stderr</a>" % self.__dict__
- else: stderrlink=''
s="""
<tr%(color)s>
- <td>%(rules)s</td>
+ <td>%(title)s</td>
<td>%(exitcode)d</td>
<td>%(message)s</td>
<td>%(time).1f s</td>
<td>%(stdoutlink)s</td>
- <td>%(stderrlink)s</td>
</tr>
""" % {
'color' : color,
'tcname' : self.rules,
- 'rules' : self.rules,
+ 'title' : title,
'exitcode' : self.exitcode,
'message' : escape(self.message),
'time' : self.time,
'stdoutlink' : stdoutlink,
- 'stderrlink' : stderrlink,
}
return s
def toHtmlOutputs(self):
'tcname':self.tcname,
'rules':self.rules,
'stdout':escape(self.stdout),
- 'stderr':escape(self.stderr)
}
s=""
if self.stdout: s+="""
-<a name='stdout-%(tcname)s-%(rules)s'/>
-<h5>Test %(tcname)s, rules %(rules)s, stdout</h5>
+<a name='output-%(tcname)s-%(rules)s'/>
+<h5>Output of test: %(tcname)s, rules: %(rules)s</h5>
<pre>%(stdout)s</pre>""" % vals
- if self.stderr: s+="""
-<a name='stderr-%(tcname)s-%(rules)s'/>
-<h5>Test %(tcname)s, rules %(rules)s, stderr</h5>
-<pre>%(stderr)s</pre>""" % vals
return s
class RulesStat:
self.success = 0
self.errors = 0
self.canttest = 0
- self.unknown = 0
def update(self, testCaseResult):
try:
resultEntry = testCaseResult[self.rules]
self.tests+=1
if resultEntry.exitcode == 0: self.success+=1
- elif resultEntry.exitcode == 1: self.errors+=1
- elif resultEntry.exitcode == 2: self.canttest+=1
- else: self.unknown+=1
+ elif resultEntry.canttest: self.canttest+=1
+ else: self.errors+=1
except KeyError:
pass
def toHtml(self):
if self.errors == 0 and self.canttest == 0: self.color=''
elif self.errors != 0: self.color=' bgcolor="red"'
- elif self.canttest != 0: self.color=' bgcolor="yellow"'
- else: self.color = ' bgcolor="gray"'
+ else: self.color = ' bgcolor="gold"'
s="""
<tr%(color)s>
- <td>%(rules)s</td>
+ <td><a href='#results-%(rules)s'>%(rules)s</a></td>
<td>%(tests)d</td>
<td>%(success)d</td>
<td>%(errors)d</td>
<td>%(canttest)d</td>
- <td>%(unknown)d</td>
</tr>
""" % self.__dict__
return s
class Stats(dict):
def __init__(self, results):
- rules = rulesdef.rules.keys()
- for rule in rules:
+ rk = rules.keys()
+ for rule in rk:
rulesStat = RulesStat(rule)
self[rule]=rulesStat
for resultEntry in results.values():
<td>Success</td>
<td>Errors</td>
<td>Can't test</td>
- <td>Unknown</td>
</tr></thead>
<tbody>
"""
self.directory = directory # Absolute directory
self.executable = executable
self.name = self._getName()
+ self.scriptNameRelative = self._getScriptRelative();
+ self.dirRelative = os.path.dirname(self.scriptNameRelative)
self._whichRules()
def _getName(self):
name+=" "+testSuffix
return name
+ def _getScriptRelative(self):
+ script = os.path.join(self.directory, self.executable);
+ if script.startswith(invokeDir+"/"):
+ script = script[len(invokeDir)+1:]
+ return script
+
def _whichRules(self):
"""Reads the rules file and creates the self.rules list of all
rules to test"""
try:
f = open(os.path.join(self.directory, self.executable+'.rules'))
except IOError:
- self.rules = rulesdef.rules.keys()
+ self.rules = rules.keys()
return
line = f.readline()
colonMatch = re.search('([^:]*) *: *(.*)', line)
if colonMatch:
if colonMatch.group(1) == "all":
# all:
- self.rules = rulesdef.rules.keys()
+ self.rules = rules.keys()
elif colonMatch.group(1) == "snip":
# snip: ...
snip = colonMatch.group(2)
- for r in rulesdef.rules:
- if snip in rulesdef.rules[r]:
+ for r in rules:
+ if snip in rules[r]:
self.rules.append(r)
elif colonMatch.group(1) == "python":
# python: ...
expr = colonMatch.group(2)
- for r in rulesdef.rules:
- if eval(expr, {'rules': r, 'snippets': rulesdef.rules[r]}):
+ for r in rules:
+ if eval(expr, {'rules': r, 'snippets': rules[r]}):
self.rules.append(r)
else:
# rule name
line = line.strip()
- if line in rulesdef.rules: self.rules = [ line ]
+ if line in rules: self.rules = [ line ]
#print self.rules
def run(self):
- self.results = TestCaseResult(self.name)
- print "Testing %s:\n\t" % self.name,
+ self.results = TestCaseResult(self)
+ print "Testing \"%s\" in %s:\n" % (self.name, self.executable),
os.chdir(os.path.join(testsRoot, self.directory))
+ # Chose how to run the test - currently there is only one option
# if os.path.exists("Makefile.test"):
# self._exec = self._execMake
if os.path.exists(self.executable):
self._exec = self._execRuntest
else: return
+ self.failed = False
for rules in self.rules:
resultEntry = ResultEntry(self.name, rules)
self.results[rules] = resultEntry
- print rules,
os.environ['OMK_RULES'] = rules
filesBefore = self._getFileSet()
self._copyRules(rules)
def _execRuntest(self, log):
startTime = time.clock()
- pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+ pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+ (output, error) = pipe.communicate()
endTime = time.clock()
- (output, errors) = pipe.communicate()
ret = pipe.returncode
- log.exitcode = ret
log.time = endTime - startTime
+ log.exitcode = ret
log.stdout = output
- log.stderr = errors
- if ret != 0 and os.path.exists("_error"):
- log.message = file("_error").read()
- else: log.message = ''
-
+ log.message = ''
+ if os.path.exists("_canttest"):
+ log.message = file("_canttest").read()
+ log.canttest = 1
+ ret = 2 # If Makefile sets canttest, override the exit code which might be zero
+ log.exitcode = ret
+ elif ret != 0:
+ if os.path.exists("_error"):
+ log.message = file("_error").read()
return ret
def _copyRules(self, rules):
def _doRun(self, log):
"Runs the teset in current directory."
+# print " ",os.environ['OMK_RULES'],
+# sys.stdout.flush()
ret = self._exec(log)
- print ret,
+# if log.canttest: retstr = "--"
+# elif ret == 0: retstr = "ok"
+# else:
+# retstr = "FAILED"
+# self.failed = True
+# print "%*s%s" % (20-len(os.environ['OMK_RULES']), "", retstr)
+ if log.canttest: retstr = "[not tested] ok"
+ elif ret == 0: retstr = "ok"
+ else:
+ retstr = "FAILED"
+ self.failed = True
+ print "! %s (%s) %s" % (self.name, os.environ['OMK_RULES'], retstr)
+ sys.stdout.flush()
results = Results()
+failed = False;
for dirpath, dirnames, filenames in os.walk(invokeDir):
executables = fnmatch.filter(filenames, "runtest*")
if not executables: continue
if re.search(".rules$", exe): continue
t = TestCase(dirpath, exe)
t.run()
+ if t.failed: failed = True
results[t.name] = t.results
os.chdir(invokeDir)
results.genStats()
results.save()
+sys.exit(failed)
+
# Local Variables:
# compile-command: "python tester.py"
# End:
projects / omk.git / blobdiff