@$(MAKE) -C $(MAKERULES_DIR) OMK_SERIALIZE_INCLUDED=n SOURCES_DIR=$(MAKERULES_DIR) RELATIVE_DIR="" $(MAKECMDGOALS) W=0
endif
+ifdef OMK_TESTSROOT
+# Usage: $(call canttest,<error message>)
+define canttest
+ ( echo "$(1)" > $(MAKERULES_DIR)/_canttest; echo "$(1)"; exit 1 )
+endef
+else
+define canttest
+ echo "$(1)"
+endef
+endif
+
#=========================
# Include the config file
SOURCESDIR_MAKEFILE=$(SOURCES_DIR)/Makefile
endif
-# Syntax: $(call omk_pass_template,<pass name>,<build dir>,[<local make flags>],[<local condition>])
+ifdef OMK_TESTSROOT
+check-target = $$(@:%=%-check)
+endif
+
+# Syntax: $(call omk_pass_template,<pass name(s)>,<build dir>,[<local make flags>],[<local condition>])
define omk_pass_template
-.PHNOY: $(1) $(1)-local
+.PHNOY: $(1) $(addsuffix -local,$(1)) $(addsuffix -check,$(1))
$(1):
+@$(foreach dir,$(SUBDIRS),$(call mkdir_def,$(2)/$(dir)); \
$(MAKE) SOURCES_DIR=$(SOURCES_DIR)/$(dir) $(NO_PRINT_DIRECTORY) \
@echo "make[omk]: $$@ in $(RELATIVE_DIR)"; \
$(call mkdir_def,$(2)); \
$(MAKE) $(NO_PRINT_DIRECTORY) SOURCES_DIR=$(SOURCES_DIR) RELATIVE_DIR=$(RELATIVE_DIR) -C $(2) \
- -f $(SOURCESDIR_MAKEFILE) $(3) $$(@:%=%-local)
+ -f $(SOURCESDIR_MAKEFILE) $(3) $(check-target) $$(@:%=%-local)
endif
endef
override OMK_SERIALIZE_INCLUDED = y
endif
+# Checks for OMK tester
+ifdef OMK_TESTSROOT
+default-config-pass-check include-pass-check:
+library-pass-check binary-pass-check:
+ @[ -x "$(shell which $(CC))" ] || $(call canttest,Cannot file compiler: $(CC))
+endif
+
#=====================================================================
# User-space rules and templates to compile programs, libraries etc.
# tar_EMBEDFILES .. list of tars with embedded files
# Include RTEMS target configuration defining RTEMS_MAKEFILE_PATH
-include $(MAKERULES_DIR)/config.target
+-include $(MAKERULES_DIR)/config.target
+
+ifdef OMK_TESTSROOT
+ifeq ($(RTEMS_MAKEFILE_PATH),)
+$(error $(shell $(call canttest,RTEMS_MAKEFILE_PATH not defined)))
+endif
+endif
include $(RTEMS_MAKEFILE_PATH)/Makefile.inc
include $(RTEMS_CUSTOM)
LIB_DIR := $(USER_LIB_DIR)
OBJS_DIR := $(USER_OBJS_DIR)
+# Checks for OMK tester
+ifdef OMK_TESTSROOT
+default-config-pass-check include-pass-check:
+library-pass-check binary-pass-check:
+ @[ -x "$(shell which $(CC))" ] || $(call canttest,Cannot file compiler: $(CC))
+ @[ -z "$(wildcard $(USER_LIB_DIR)/$(LD_SCRIPT).ld*)" ] && $(call canttest,LD_SCRIPT $(LD_SCRIPT).ld* not found)
+endif
+
#=====================================================================
# User-space rules and templates to compile programs, libraries etc.
endif # HOST_RULE_TEMPLATES
+# Checks for OMK tester
+ifdef OMK_TESTSROOT
+default-config-pass-check include-pass-check:
+library-pass-check binary-pass-check:
+ @[ -x "$(shell which $(CC))" ] || $(call canttest,Cannot file compiler: $(CC))
+endif
#=====================================================================
# Automatic loading of compiled program by issuing "make load"
# Include VxWorks target configuration containig definition of these
# variables: WIND_BASE CPU TOOL
-include $(MAKERULES_DIR)/config.target
+-include $(MAKERULES_DIR)/config.target
ifndef WRENV
-$(error WRENV is not defined in config.target)
+$(error $(shell $(call canttest,WRENV is not defined in config.target)))
endif
ifndef WRPACKAGE
-$(error WRPACKAGE is not defined in config.target)
+$(error $(shell $(call canttest,WRPACKAGE is not defined in config.target)))
endif
ifndef CPU
-$(error CPU is not defined in config.target)
+$(error $(shell $(call canttest,CPU is not defined in config.target)))
endif
ifndef TOOL
-$(error TOOL is not defined in config.target)
+$(error $(shell $(call canttest,TOOL is not defined in config.target)))
endif
BUILD_DIR_NAME = _build/$(CPU)$(TOOL)
override OMK_SERIALIZE_INCLUDED = y
endif
+# Checks for OMK tester
+ifdef OMK_TESTSROOT
+default-config-pass-check include-pass-check:
+library-pass-check binary-pass-check:
+ @[ -x "$(shell which $(CC))" ] || $(call canttest,Cannot file compiler: $(CC))
+ @[ -x "$(WRENV)" ] || $(call canttest,Cannot execute wrenv script)
+endif
+
# Export some variables to VxWorks makefiles
CPPFLAGS += -I $(USER_INCLUDE_DIR)
- runtest*.rules: Specifies the set of rules the testcase applies
to. If there is more runtest files, every testcase can have its
- own .rules file. The syntax of this files is describes in section
+ own .rules file. The syntax of this files is described in section
`Rules description'.
* Testcase Execution:
Each execution of testcase should produce the following outputs:
- Exit status of make or runtest:
- 0 - the testcase was successfully passed
- 1 - the testcase failed
- 2 - the testcase cannot be executed (e.g. because the needed
- compiler is not available)
+ zero - the testcase was successfully passed
+ other than zero - the testcase failed or cannot be executed
+ (e.g. because the needed compiler is not
+ available)
- Error message:
- In the case of error, the file '_error' should contain more
- detailed description of the error.
+ In the case of nonzero exit status, if file '_canttest' exists in
+ the same directory as Makefile.rules, it is considered that the
+ test cannot be executed (i.e. the compiler is missing) and the
+ content of that file is provided as error message in results
+ log. Otherwise, the test is considered as failed and file '_error'
+ can contain more detailed description of the error.
- stdout and stderr: The output of testcase execution is captured
and is included in the results log. It should contain useful
- information to ease debugging problems.
+ information to ease the debugging process.
Rules Description
-----------------
# -*-sh-*-
+# Exit on first error
+set -e
+
function findup() {
local arg="$1"
if test -z "$arg"; then return 1; fi
}
function canttest() {
- echo $1 > _error
- exit 2
+ [ -f _canttest ] || echo $1 > _canttest
+ exit 1
}
if [ -z "$OMK_TESTSROOT" ]; then
source ../../functions.sh
-make default-config || canttest
+make default-config || error "Can't run make default-config"
make include-pass || error "Can't run make include-pass"
GC=./_compiled/include/global.h
-test -f "$GC" || canttest "Can't find the produced global config"
+test -f "$GC" || error "Can't find the produced global config"
grep -Fv "/* config file:" $GC | diff -u correct/global.h - || error "Global config differs"
-make || canttest "Can't run make to produce local config"
-make CFLAGS=-Wall || canttest "Make failed with custom CFLAGS"
+make || error "Can't run make to produce local config"
+make CFLAGS=-Wall || error "Make failed with custom CFLAGS"
LC=$(find _build -wholename '*/lincan/lincan_config.h')
-test -f "$LC" || canttest "Can't find the produced local config"
+test -f "$LC" || error "Can't find the produced local config"
grep -Fv "/* config file:" $LC | diff -u correct/lincan_config.h - || error "Local config differs"
TLC=$(find _build -name 'toplevel_config.h')
-test -f "$TLC" || canttest "Can't find the produced local config"
+test -f "$TLC" || error "Can't find the produced local config"
grep -Fv "/* config file:" $TLC | diff -u correct/toplevel_config.h - || error "Toplevel local config differs"
touch -t 200001010000 $LC
source ../../functions.sh
-make default-config || canttest
+make default-config || error default-config
make distclean
LC_MESSAGES=C
export LC_MESSAGES
-make 2> stderr || canttest
-grep "control reaches end of non-void function" stderr || error "Default CFLAGS are not applied"
+set -o pipefail
+make 2>&1 | tee stderr || error make
+grep "control reaches end of non-void function" stderr || error "Default CFLAGS are not applied or no GCC is used"
#!/bin/sh
-set -e
source ../functions.sh
OUTPUT=$(make 2>&1)
touch config.omk-default
echo "CFLAGS=-DNUMBER=123" > config.omk
-make||canttest "Can't compile"
+make||error "Can't compile"
[ $OMK_RULES != linux ] && canttest "Should work only with Linux rules"
_compiled/bin/test|grep 123 || error "Custom CFLAGS didn't influence the output"
source ../functions.sh
touch config.omk-default
-make CFLAGS="-DNONSENSE=abc -DNUMBER=123"||canttest "Can't compile"
+make CFLAGS="-DNONSENSE=abc -DNUMBER=123"||error "Can't compile"
[ $OMK_RULES != linux ] && canttest "Should work only with Linux rules"
_compiled/bin/test|grep 123 || error "Custom CFLAGS didn't influence the output"
touch config.omk-default
echo "DEFS=-DNUMBER=123" > config.omk
-make||canttest "Can't compile"
+make||error "Can't compile"
[ $OMK_RULES != linux ] && canttest "Should work only with Linux rules"
_compiled/bin/test|grep 123 || error "Variable DEFS didn't influence the output"
touch config.omk-default
rm -rf _compiled
-make MAKEFILE_OMK=Makefile.omk-test_PROGRAMS || canttest
+make MAKEFILE_OMK=Makefile.omk-test_PROGRAMS || error make
FOUND=`find _compiled -wholename '*/bin-tests/test*'`
echo "Found: $FOUND"
[ -n "$FOUND" ] || error "No test program created"
""" % self.__dict__
s+=self.stats.toHtml()
s+="""
-<h2>Chart</h2>
+<h2>Chart by rules</h2>
+ """
+ s+=self.toHtmlByRules();
+ s+="""
+<h2>Chart by testcase</h2>
<table cellpadding='2' border='1'>
<tbody>
- """ % self.__dict__
+ """
tests = sorted(self.keys())
for t in tests:
s+=self[t].toHtml()
</body></html>
"""
return s
-
+
+ def toHtmlByRules(self):
+ rules = sorted(rulesdef.rules.keys())
+ tests = sorted(self.keys())
+ s=''
+ for r in rules:
+ s+="""
+<a name='results-%(rules)s' />
+<h3>Rules: %(rules)s</h3>
+<table cellpadding='2' border='1'>
+<tbody>
+""" % { 'rules':r }
+ for t in tests:
+ if self[t].has_key(r):
+ s+=self[t][r].toHtml("<a href='%s'>/</a><a href='%s'>%s</a>" %
+ (self[t].tc.dirRelative, self[t].tc.scriptNameRelative, self[t].tc.name))
+ s+="""
+</tbody></table>
+"""
+ return s
+
def save(self):
f = file(self.filename, "w+")
f.write(self.toHtml())
print "Results written to "+self.filename
class TestCaseResult(dict):
- def __init__(self, tcname):
- self.tcname = tcname
+ def __init__(self, tc):
+ self.tc = tc
def toHtml(self):
rules = sorted(self.keys())
s="""
- <tr><td colspan='6'><strong>%s</strong></td></tr>
- """ % self.tcname
+ <tr><td colspan='5'><strong><a href='%s'>%s</a></strong></td></tr>
+ """ % (self.tc.scriptNameRelative, self.tc.name)
for r in rules:
- s+=self[r].toHtml()
+ s+=self[r].toHtml(r)
return s
def toHtmlOutputs(self):
rules = sorted(self.keys())
- s="<h3>Testcase: %s</h3>" % self.tcname
+ s="<h3>Testcase: %s</h3>" % self.tc.name
for r in rules:
s+=self[r].toHtmlOutputs()
return s
def __init__(self, tcname, rules):
self.tcname = tcname
self.rules = rules
+ self.canttest = 0
- def toHtml(self):
+ def toHtml(self, title):
if self.exitcode == 0: color=''
- else:
- if self.exitcode == 1: color=' bgcolor="red"'
- elif self.exitcode == 2: color=' bgcolor="gray"'
- else: color=' bgcolor="gray"'
- if self.stdout: stdoutlink="<a href='#stdout-%(tcname)s-%(rules)s'>stdout</a>" % self.__dict__
+ elif self.canttest: color=' bgcolor="gold"'
+ else: color=' bgcolor="red"'
+ if self.stdout: stdoutlink="<a href='#output-%(tcname)s-%(rules)s'>output</a>" % self.__dict__
else: stdoutlink=''
- if self.stderr: stderrlink="<a href='#stderr-%(tcname)s-%(rules)s'>stderr</a>" % self.__dict__
- else: stderrlink=''
s="""
<tr%(color)s>
- <td>%(rules)s</td>
+ <td>%(title)s</td>
<td>%(exitcode)d</td>
<td>%(message)s</td>
<td>%(time).1f s</td>
<td>%(stdoutlink)s</td>
- <td>%(stderrlink)s</td>
</tr>
""" % {
'color' : color,
'tcname' : self.rules,
- 'rules' : self.rules,
+ 'title' : title,
'exitcode' : self.exitcode,
'message' : escape(self.message),
'time' : self.time,
'stdoutlink' : stdoutlink,
- 'stderrlink' : stderrlink,
}
return s
def toHtmlOutputs(self):
'tcname':self.tcname,
'rules':self.rules,
'stdout':escape(self.stdout),
- 'stderr':escape(self.stderr)
}
s=""
if self.stdout: s+="""
-<a name='stdout-%(tcname)s-%(rules)s'/>
-<h5>Test %(tcname)s, rules %(rules)s, stdout</h5>
+<a name='output-%(tcname)s-%(rules)s'/>
+<h5>Output of test: %(tcname)s, rules: %(rules)s</h5>
<pre>%(stdout)s</pre>""" % vals
- if self.stderr: s+="""
-<a name='stderr-%(tcname)s-%(rules)s'/>
-<h5>Test %(tcname)s, rules %(rules)s, stderr</h5>
-<pre>%(stderr)s</pre>""" % vals
return s
class RulesStat:
self.success = 0
self.errors = 0
self.canttest = 0
- self.unknown = 0
def update(self, testCaseResult):
try:
resultEntry = testCaseResult[self.rules]
self.tests+=1
if resultEntry.exitcode == 0: self.success+=1
- elif resultEntry.exitcode == 1: self.errors+=1
- elif resultEntry.exitcode == 2: self.canttest+=1
- else: self.unknown+=1
+ elif resultEntry.canttest: self.canttest+=1
+ else: self.errors+=1
except KeyError:
pass
def toHtml(self):
if self.errors == 0 and self.canttest == 0: self.color=''
elif self.errors != 0: self.color=' bgcolor="red"'
- elif self.canttest != 0: self.color=' bgcolor="gray"'
- else: self.color = ' bgcolor="gray"'
+ else: self.color = ' bgcolor="gold"'
s="""
<tr%(color)s>
- <td>%(rules)s</td>
+ <td><a href='#results-%(rules)s'>%(rules)s</a></td>
<td>%(tests)d</td>
<td>%(success)d</td>
<td>%(errors)d</td>
<td>%(canttest)d</td>
- <td>%(unknown)d</td>
</tr>
""" % self.__dict__
return s
<td>Success</td>
<td>Errors</td>
<td>Can't test</td>
- <td>Unknown</td>
</tr></thead>
<tbody>
"""
self.directory = directory # Absolute directory
self.executable = executable
self.name = self._getName()
+ self.scriptNameRelative = self._getScriptRelative();
+ self.dirRelative = os.path.dirname(self.scriptNameRelative)
self._whichRules()
def _getName(self):
name+=" "+testSuffix
return name
+ def _getScriptRelative(self):
+ script = os.path.join(self.directory, self.executable);
+ if script.startswith(invokeDir+"/"):
+ script = script[len(invokeDir)+1:]
+ return script
+
def _whichRules(self):
"""Reads the rules file and creates the self.rules list of all
rules to test"""
def run(self):
- self.results = TestCaseResult(self.name)
+ self.results = TestCaseResult(self)
print "Testing %s:\n" % self.name,
os.chdir(os.path.join(testsRoot, self.directory))
# if os.path.exists("Makefile.test"):
def _execRuntest(self, log):
startTime = time.clock()
- pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+ pipe = subprocess.Popen("./"+self.executable, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+ (output, error) = pipe.communicate()
endTime = time.clock()
- (output, errors) = pipe.communicate()
ret = pipe.returncode
- log.exitcode = ret
log.time = endTime - startTime
+ log.exitcode = ret
log.stdout = output
- log.stderr = errors
- if ret != 0 and os.path.exists("_error"):
- log.message = file("_error").read()
- else: log.message = ''
-
+ log.message = ''
+ if ret != 0:
+ if os.path.exists("_canttest"):
+ log.message = file("_canttest").read()
+ log.canttest = 1
+ elif os.path.exists("_error"):
+ log.message = file("_error").read()
return ret
def _copyRules(self, rules):
sys.stdout.flush()
ret = self._exec(log)
if ret == 0: retstr = "OK"
- elif ret == 1: retstr = "FAILED"
- elif ret == 2: retstr = "--"
- else: retstr = "???"
+ elif log.canttest: retstr = "--"
+ else: retstr = "FAILED"
print "%*s%s" % (20-len(os.environ['OMK_RULES']), "", retstr)
projects / omk.git / commitdiff