--- /dev/null
+#!/usr/bin/env python
+#
+# WvTest:
+# Copyright (C) 2012, 2014 Michal Sojka <sojka@os.inf.tu-dresden.de>
+# Licensed under the GNU Library General Public License, version 2.
+# See the included file named LICENSE for license information.
+#
+# This script converts a sequence of wvtest protocol outputs with
+# results of performance measurements to interactive graphs (HTML +
+# JavaScript). An example can be seen at
+# http://os.inf.tu-dresden.de/~sojka/nul/performance.html.
+
+from __future__ import print_function
+import sys
+import re
+import os
+import os.path
+import string
+import time
+import numpy as np
+import json
+
+
+re_prefix = "\([0-9]+\) (?:# )?"
+re_date = re.compile('^Date: (.*)')
+re_testing = re.compile('^('+re_prefix+')?\s*Testing "(.*)" in (.*):\s*$')
+re_commit = re.compile('.*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*, commit: (.*)')
+re_commithash = re.compile('([0-9a-f]{7})(-dirty)? \(')
+re_assertion = re.compile('^('+re_prefix+')?!\s*(.*?)\s+(\S+)\s*$')
+re_perf = re.compile('^('+re_prefix+')?!\s*(.*?)\s+PERF:\s*(.*?)\s+(\S+)\s*$')
+re_perfaxis = re.compile('axis="([^"]+)"')
+
+def dateConv(date):
+ d = time.gmtime(time.mktime(date))
+ return int(time.mktime(d))*1000
+
+
+class Axis:
+ def __init__(self, name=None, units=None):
+ self.name = name
+ self.units = units
+ self.num = None
+ def getLabel(self):
+ if self.units and self.name:
+ return "%s [%s]" % (self.name, self.units)
+ elif self.units:
+ return self.units
+ else:
+ return self.name
+ def __repr__(self): return "Axis(name=%s units=%s, id=%s)" % (self.name, self.units, hex(id(self)))
+
+class Column:
+ def __init__(self, name, units, axis):
+ self.name = name
+ self.units = units
+ self.axis = axis
+ def __repr__(self): return "Column(name=%s units=%s axis=%s)" % (self.name, self.units, repr(self.axis))
+
+class Row(dict):
+ def __init__(self, graph, date):
+ self.graph = graph
+ self.date = date
+ def __getitem__(self, column):
+ try:
+ return dict.__getitem__(self, column)
+ except KeyError:
+ return None
+ def getDate(self):
+ return dateConv(self.date)
+
+class Graph:
+ def __init__(self, id, title):
+ self.columns = {}
+ self.columns_ordered = []
+ self.id = id
+ self.title = title
+ self.rows = []
+ self.date2row = {}
+ self.axes = {}
+ self.axes_ordered = []
+ self.dataname = title.translate(string.maketrans(" /", "--"), "\"':,+()").lower()+".json"
+
+ def __getitem__(self, date):
+ try:
+ rownum = self.date2row[date]
+ except KeyError:
+ rownum = len(self.rows)
+ self.date2row[date] = rownum
+ try:
+ return self.rows[rownum]
+ except IndexError:
+ self.rows[rownum:rownum] = [Row(self, date)]
+ return self.rows[rownum]
+
+ def addValue(self, date, col, val, units):
+ row = self[date]
+ row[col] = val
+ if col not in self.columns:
+ axis=self.getAxis(units) or self.addAxis(units, Axis(units=units))
+ column = Column(col, units, axis)
+ self.columns[col] = column
+ self.columns_ordered.append(column)
+ else:
+ column = self.columns[col]
+ self.columns_ordered.remove(column)
+ self.columns_ordered.append(column)
+ self.columns[col].units=units
+ self.columns[col].axis.units=units
+
+ def addAxis(self, key, axis):
+ self.axes[key] = axis
+ return axis
+
+ def setAxis(self, col, key):
+ self.columns[col].axis = self.getAxis(key) or self.addAxis(key, Axis(name=key))
+
+ def getAxis(self, key):
+ if key not in self.axes: return None
+ return self.axes[key]
+
+ def findRanges(self):
+ for axis in list(self.axes.values()):
+ cols = [col for col in list(self.columns.values()) if col.axis == axis]
+ low = None
+ high = None
+ all_in_range = True
+ for col in cols:
+ values = np.array([row[col.name] for row in self.rows if row[col.name] != None], np.float64)
+ if low == None and high == None:
+ lastmonth = values[-30:]
+ median = np.median(lastmonth)
+ low = median * 0.95
+ high = median * 1.05
+
+ if (values > high).any() or (values < low).any():
+ all_in_range = False
+ if all_in_range:
+ axis.yrange_max = high
+ axis.yrange_min = low
+ else:
+ axis.yrange_max = None
+ axis.yrange_min = None
+
+ def fixupAxisNumbers(self):
+ # Sort axes according to the columns and number them
+ num = 0
+ for column in self.columns_ordered:
+ axis = column.axis
+ if axis not in self.axes_ordered:
+ self.axes_ordered.insert(0, axis)
+ axis.num = num
+ num += 1
+ num = 0
+ for axis in self.axes_ordered:
+ axis.num = num
+ num += 1
+
+ def options_json(self):
+ options = {
+ 'rangeSelector': {
+ 'selected': 2 # 6m
+ },
+ 'title': {
+ 'text': self.title
+ },
+ 'legend': {
+ 'enabled': True,
+ 'floating': False,
+ 'verticalAlign': "top",
+ 'x': 100,
+ 'y': 60,
+ },
+ 'tooltip': {
+ 'formatter': "FUN(tooltip_formatter)END",
+ 'style': {
+ 'whiteSpace': 'normal',
+ 'width': '400px',
+ },
+ },
+ 'plotOptions': {
+ 'series': {
+ 'events': {
+ 'click': "FUN(series_onclick)END",
+ }
+ }
+ },
+ 'yAxis': [{
+ 'lineWidth': 1,
+ 'labels': { 'align': 'right',
+ 'x': -3 },
+ 'title': { 'text': axis.getLabel() },
+ 'min': axis.yrange_min,
+ 'max': axis.yrange_max,
+ } for axis in self.axes_ordered],
+ 'series': [{ 'name': '%s [%s]' % (col.name, col.units),
+ 'yAxis': col.axis.num }
+ for col in self.columns_ordered]
+ }
+ return json.dumps(options, indent=True).replace('"FUN(', '').replace(')END"', '')
+
+ def getData(self):
+ data = [[[row.getDate(), row[col.name]] for row in self.rows] for col in self.columns_ordered]
+ return json.dumps(data).replace('], [', '],\n[')
+
+
+class Graphs(dict):
+ pass
+
+graphs = Graphs()
+date2commit = {}
+commit2msg = {}
+
+date = time.localtime(time.time())
+
+for line in sys.stdin:
+ line = line.rstrip()
+
+ match = re_date.match(line)
+ if (match):
+ date = time.strptime(match.group(1), "%a, %d %b %Y %H:%M:%S +0200")
+ continue
+
+ match = re_testing.match(line)
+ if match:
+ what = match.group(2)
+ where = match.group(3)
+
+ match = re_commit.match(what)
+ if match:
+ date = time.strptime(match.group(1), "%Y-%m-%d %H:%M:%S")
+ commit = match.group(2)
+ match = re_commithash.search(commit);
+ if match:
+ commithash = match.group(1)
+ else:
+ commithash = None
+ date2commit[dateConv(date)] = commithash
+ commit2msg[commithash] = commit
+
+ (basename, ext) = os.path.splitext(os.path.basename(where))
+
+ if what != "all": title = what
+ else: title = basename
+ try:
+ graph = graphs[basename]
+ except KeyError:
+ graph = Graph(basename, title)
+ graphs[basename] = graph
+ continue
+
+ match = re_perf.match(line)
+ if match:
+ perfstr = match.group(3)
+ perf = perfstr.split()
+ col = perf[0]
+ try:
+ val = float(perf[1])
+ except ValueError:
+ val = None
+ try:
+ units = perf[2]
+ if '=' in units: units = None
+ except:
+ units = None
+ if match.group(4) != "ok":
+ val=None
+
+ graph.addValue(date, col, val, units)
+
+ match = re_perfaxis.search(perfstr)
+ if match:
+ graph.setAxis(col, match.group(1));
+
+graphs = [g for g in list(graphs.values()) if len(g.columns)]
+graphs = sorted(graphs, key=lambda g: g.title.lower())
+
+for g in graphs:
+ g.findRanges()
+ g.fixupAxisNumbers()
+
+print("""
+<!DOCTYPE HTML>
+<html>
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+ <title>NUL Performance Plots</title>
+
+ <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js"></script>
+ <script type="text/javascript">
+ function tooltip_formatter() {
+ var s = '<b>'+ Highcharts.dateFormat('%a, %d %b %Y %H:%M:%S', this.x) +'</b><br/>';
+ s += commit2msg[date2commit[this.x]];
+ $.each(this.points, function(i, point) {
+ s += '<br/><span style="color:'+ point.series.color+';">'+ point.series.name +'</span>: '+point.y;
+ });
+ return s;
+ }
+ function series_onclick(event) {
+ var lastpoint = null;
+ for (var i in this.data) {
+ if (event.point == this.data[i]) {
+ if (i > 0) lastpoint = this.data[i-1];
+ break;
+ }
+ }
+ if (lastpoint)
+ window.location = "http://os.inf.tu-dresden.de/~jsteckli/cgi-bin/cgit.cgi/nul/log/?qt=range&q="+date2commit[lastpoint.x]+'..'+date2commit[event.point.x].hash;
+ else
+ window.location = "http://os.inf.tu-dresden.de/~jsteckli/cgi-bin/cgit.cgi/nul/log/?id="+date2commit[event.point.x];
+ }""")
+
+def make_int_keys(json):
+ r = re.compile('"(\d+)"(.*)')
+ s = ''
+ for match in r.finditer(json):
+ s += match.expand('\\1\\2')
+ return s
+
+print("var date2commit = {%s};" % ",\n".join(["%d: '%s'" % (k, date2commit[k]) for k in sorted(date2commit.keys())]))
+print("var commit2msg = %s;" % json.dumps(commit2msg, indent=True))
+# for d in sorted(date2commit.keys()):
+# v = commits[d];
+# print('\t%d: { msg: "%s", hash: "%s" },' % (1000*time.mktime(d), v[0].replace('"', '\\"'), str(v[1]).replace('"', '\\"')))
+print("""
+ </script>
+ <script type="text/javascript" src="js/highstock.js"></script>
+ </head>
+
+ <body>
+ <h1>NUL Performance Plots</h1>
+ <p>The graphs below show performance numbers from various
+ benchmarks that run nightly on NUL repository.</p>
+ <p>Table of content:</p>
+ <ul>
+""")
+for graph in graphs:
+ print(" <li><a href='#%s'>%s</a></li>" % (graph.title, graph.title))
+print(" </ul>")
+for graph in graphs:
+ print("""
+ <h2><a name='%(title)s'>%(title)s</a></h2>
+ <div id="%(id)s" style="height: 400px"></div>
+ <script>
+$.getJSON('%(dataname)s', function(data) {
+ var options = %(options)s
+ for (var i=0; i < data.length; i++)
+ options['series'][i]['data'] = data[i];
+ $("#%(id)s").highcharts("StockChart", options);
+});
+</script>
+""" % {'id': graph.id,
+ 'title': graph.title,
+ 'dataname': graph.dataname,
+ 'options': graph.options_json()})
+ print(graph.getData(), file=open(graph.dataname, 'w'))
+print("""
+ </body>
+</html>
+""")
+
+# Local Variables:
+# compile-command: "cat nul-nightly/nul_*.log|./wvperfpreprocess.py|./wvperf2html.py > graphs.html"
+# End:
--- /dev/null
+#!/usr/bin/env python2
+
+import sys
+import re
+import os
+import os.path
+import string
+import time
+
+re_date = re.compile('^Date: (.*)')
+re_testing = re.compile('^(\([0-9]+\) (# )?)?\s*Testing "(.*)" in (.*):\s*$')
+re_commit = re.compile('.*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*, commit: (.*)')
+re_commithash = re.compile('([0-9a-f]{7})(-dirty)? \(')
+re_check = re.compile('^(\([0-9]+\) (# )?)?!\s*(.*?)\s+(\S+)\s*$')
+re_perf = re.compile('^(\([0-9]+\) (# )?)?!\s*(.*?)\s+PERF:\s*(.*?)\s+(\S+)\s*$')
+
+# State variables
+date = time.localtime(time.time())
+linetype = None
+what = None
+where = ""
+commit = None
+commithash = None
+basename = None
+ext = None
+perf = None
+key = None
+val = None
+units = None
+tag = None
+
+def matches(re):
+ global match, line
+ match = re.match(line)
+ return match
+
+for line in sys.stdin:
+ line = line.rstrip()
+ match = None
+
+ # Parse known lines
+ if matches(re_date):
+ linetype='date'
+ date = time.strptime(match.group(1), "%a, %d %b %Y %H:%M:%S +0200")
+ elif matches(re_testing):
+ linetype='testing'
+ what = match.group(3)
+ where = match.group(4)
+
+ match = re_commit.match(what)
+ if match:
+ linetype='commitid'
+ date = time.strptime(match.group(1), "%Y-%m-%d %H:%M:%S")
+ commit = match.group(2)
+ match = re_commithash.search(commit);
+ if match:
+ commithash = match.group(1)
+ else:
+ commithash = None
+
+ (basename, ext) = os.path.splitext(os.path.basename(where))
+ elif matches(re_perf):
+ linetype='perf'
+ perf = match.group(4)
+ perf = perf.split()
+ key = perf[0]
+ try:
+ val = float(perf[1])
+ except ValueError:
+ val = None
+ try:
+ units = perf[2]
+ except:
+ units = None
+ else:
+ linetype='other'
+ continue
+
+ # Rewriting rules
+ if '/vancouver-kernelbuild' in where:
+ if linetype == 'testing':
+ m = re.compile('vancouver-kernelbuild-(.*).wv').search(where)
+ if m: tag = m.group(1)
+ else: tag = 'ept-vpid'
+
+ line='Testing "Kernel compile in ramdisk" in kernelbuild-ramdisk:'
+ if linetype == 'perf':
+ line = line.replace('kbuild', "vm-"+tag);
+ line = line.replace('ok', 'axis="kbuild" ok');
+ if '/kernelbuild-bare-metal.wv' in where:
+ if linetype == 'testing':
+ line='Testing "Kernel compile in ramdisk" in kernelbuild-ramdisk:'
+ if linetype == 'perf':
+ line = line.replace('kbuild', 'bare-metal');
+ line = line.replace('ok', 'axis="kbuild" ok');
+
+ if '/diskbench-vm.wv' in where and linetype == 'perf' and commithash == '7459b8c':
+ # Skip results of test with forgotten debugging output
+ continue
+
+ if 'standalone/basicperf.c' in where and linetype == 'perf' and "PERF: warmup_" in line:
+ # Skip warmup results
+ continue
+
+ if 'vancouver-linux-basic' in where:
+ continue # Ignore the old test
+
+ if 'diskbench-ramdisk.wv' in where or 'diskbench-ramdisk-old.wv' in where:
+ # Merge graphs for old and new disk protocol
+ if linetype == 'testing' and 'diskbench-ramdisk-old.wv' in where:
+ line = line.replace('diskbench-ramdisk-old.wv', 'diskbench-ramdisk.wv');
+ if linetype == 'perf' and key == 'request_rate':
+ continue # Do not plot request rate
+ if linetype == 'perf' and key == 'throughput' and units:
+ line = line.replace('ok', 'axis="throughput" ok');
+ if 'diskbench-ramdisk-old.wv' in where:
+ line = line.replace('throughput', 'old-protocol', 1)
+
+ if 'parentperf.' in where or 'parentperfsmp.' in where:
+ if linetype == 'testing':
+ if 'parentperf.wv' in where: smp = False
+ elif 'parentperfsmp.wv' in where: smp = True
+ if what == 'Service without sessions': tag = 'nosess'
+ elif what == 'Service with sessions': tag = 'sess'
+ elif what == 'Service with sessions (implemented as a subclass of SService)': tag = 'sserv'
+ elif what == 'Service with sessions represented by portals (implemented as a subclass of NoXlateSService)': tag = 'noxsserv'
+ else: tag = None
+ elif linetype == 'perf':
+ if key == 'min' or key == 'max': continue
+ if key == 'open_session':
+ if 'pre-vnet-removal-239-g910c152' in commit or 'pre-vnet-removal-240-g9b2fa79' in commit: continue # Broken measurements
+ if not smp: print 'Testing "Parent protocol open_session performance" in parentperf_open:'
+ else: continue
+ else:
+ if not smp: print 'Testing "Parent protocol call performance" in parentperf_call:'
+ else: print 'Testing "Parent protocol call performance (4 CPUs in parallel)" in parentperf_call_smp:'
+ line = line.replace(key, tag+'_'+key);
+ print line
+ continue
+
+ if 'loc.wv' in where:
+ if linetype == 'testing' and 'PASSIVE' in what: line = line.replace('loc.wv', 'loc-passive.wv');
+ if linetype == 'perf' and key != 'files':
+ line = line.replace('ok', 'axis="lines" ok');
+
+ if 'pingpong.wv' in where:
+ if linetype == 'perf':
+ if 'min' in key or 'max' in key: continue
+
+ if 'vancouver-boottime.wv' in where and linetype == 'perf' and key == 'tsc':
+ line = "! PERF: boottime %f s ok" % (val/2.66e9)
+ #print >>sys.stderr, line
+
+ # Output (possibly modified) line
+ print line
+
+# Local Variables:
+# compile-command: "cat nul-nightly/nul_*.log|./wvperfpreprocess.py|./wvperf2html.py > graphs.html"
+# End:
--- /dev/null
+#!/usr/bin/env python
+#
+# WvTest:
+# Copyright (C) 2012 Michal Sojka <sojka@os.inf.tu-dresden.de>
+# Licensed under the GNU Library General Public License, version 2.
+# See the included file named LICENSE for license information.
+#
+# This script converts wvtest protocol output to HTML pages.
+
+import sys
+import re
+import os
+import os.path
+import string
+import time
+import numpy as np
+import cgi
+
+re_prefix = "\([0-9]+\) (?:# )?"
+re_date = re.compile('^Date: (.*)')
+re_testing = re.compile('^('+re_prefix+')?\s*Testing "(.*)" in (.*):\s*$')
+re_commit = re.compile('.*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*, commit: (.*)')
+re_commithash = re.compile('([0-9a-f]{7})(-dirty)? \(')
+re_assertion = re.compile('^('+re_prefix+')?!\s*(.*?)\s+(\S+)\s*$')
+re_perf = re.compile('^('+re_prefix+')?!\s*(.*?)\s+PERF:\s*(.*?)\s+(\S+)\s*$')
+re_perfaxis = re.compile('axis="([^"]+)"')
+
+date = time.localtime(time.time())
+
+class Test:
+ def __init__(self, what, where):
+ self.what = what
+ self.where = where
+ self.output = []
+ self.status = 'ok'
+ self.check_count = 0
+ self.failures = 0
+ self.num = None
+
+ def add_line(self, line):
+ self.output.append(line)
+ match = re_assertion.match(line)
+ if match:
+ self.check_count += 1
+ result = match.group(3)
+ if result != "ok":
+ self.status = result
+ self.failures += 1
+ def title(self):
+ if self.what == "all":
+ title = self.where
+ else:
+ title = '%s (%s)' % (self.what, self.where)
+ return title
+
+ def printSummaryHtml(self, file):
+ if self.status == "ok": status_class="ok"
+ else: status_class = "failed"
+ file.write("<tr class='testheader status-%s'><td class='testnum'>%d.</td><td class='testname'><a href='test%d.html'>%s</a></td>"
+ % (status_class, self.num, self.num, cgi.escape(self.title())))
+ file.write("<td>%s</td></tr>\n" % (cgi.escape(self.status)))
+
+ def printDetailHtml(self, file):
+ file.write("""\
+<!DOCTYPE HTML>
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+<title>NUL Test Report</title>
+<link rel="stylesheet" href="wvtest.css" type="text/css" />
+</head>
+
+<body>
+<h1>NUL Test Report</h1>
+%s
+<h2>%d. %s</h2>
+<table class='output'>
+""" % (date_and_commit, self.num, cgi.escape(self.title())))
+ for line in self.output:
+ match = re_assertion.match(line)
+ if match:
+ result = match.group(3)
+ if result == "ok":
+ status_class = "ok"
+ else:
+ status_class = "failed"
+ linestatus = " status-%s" % status_class
+ resultstatus = " class='status-%s'" % status_class
+ else:
+ linestatus = ''
+ resultstatus = ''
+ result = ''
+
+ file.write("<tr><td class='outputline%s'>%s</td><td%s>%s</td></tr>\n" % \
+ (linestatus, cgi.escape(line), resultstatus, cgi.escape(result)))
+ file.write("</table></body></html>")
+
+tests = []
+test = None
+
+for line in sys.stdin.readlines():
+ line = line.rstrip()
+
+ match = re_date.match(line)
+ if (match):
+ date = time.strptime(match.group(1), "%a, %d %b %Y %H:%M:%S +0200")
+ continue
+
+ match = re_testing.match(line)
+ if match:
+ what = match.group(2)
+ where = match.group(3)
+
+ test = Test(what, where)
+ tests.append(test)
+
+ match = re_commit.match(what)
+ if match:
+ date = time.strptime(match.group(1), "%Y-%m-%d %H:%M:%S")
+ commit = match.group(2)
+ match = re_commithash.search(commit);
+ if match:
+ commithash = match.group(1)
+ else:
+ commithash = None
+ continue
+
+ if test: test.add_line(line)
+
+tests_nonempty = [t for t in tests if t.check_count > 0]
+num = 1
+for t in tests_nonempty:
+ t.num = num
+ num += 1
+
+try:
+ date_and_commit = (time.strftime("%a, %d %b %Y %H:%M:%S +0000", date) + " " + commit)
+except:
+ date_and_commit = time.strftime("%a, %d %b %Y %H:%M:%S %Z")
+ pass
+
+targetDir = sys.argv[1]
+if not os.path.isdir(targetDir):
+ os.mkdir(targetDir)
+
+wvtest_css = open(os.path.join(targetDir, "wvtest.css"), 'w')
+wvtest_css.write("""\
+table {
+ border: solid 1px black;
+ max-width: 100%%;
+}
+.status-ok { background: lightgreen; }
+.status-failed { background: red; }
+.testnum { text-align: right; }
+.outputrow { display: none; }
+.output { width: 100%%; }
+.outputline { white-space: pre-wrap; font-family: monospace; }
+.testheader { font-weight: bold; }
+""")
+wvtest_css.close()
+
+index_html = open(os.path.join(targetDir, "index.html"), 'w')
+
+index_html.write("""\
+<!DOCTYPE HTML>
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+<title>NUL Test Report</title>
+<link rel="stylesheet" href="wvtest.css" type="text/css" />
+</head>
+
+<body>
+<h1>NUL Test Report</h1>
+%s
+<table>
+""" % date_and_commit)
+for test in tests_nonempty:
+ test.printSummaryHtml(index_html)
+index_html.write("""\
+</table>
+</body>
+</html>
+""")
+
+for test in tests_nonempty:
+ f = open(os.path.join(targetDir, "test%d.html" % test.num), 'w')
+ test.printDetailHtml(f)
+ f.close()
+
+
+# Local Variables:
+# compile-command: "cat $(ls nul-nightly/nul_*.log|tail -n 1)|./wvtest2html.py html"
+# End:
projects / nul-nightly.git / commitdiff