Update evaluation script

diff --git a/scripts/eval2.py b/scripts/eval2.py
index 3e66e6a0f995d3ff6b437769043766f06486e788..1e627d9539aef78617a14ec03ed12304ac04ac8a 100755 (executable)
--- a/scripts/eval2.py
+++ b/scripts/eval2.py
@@ -10,7 +10,7 @@ import collections
 import psycopg2
 import psycopg2.extras
 import subprocess
-
+import json
 import utils
 
 
@@ -42,6 +42,9 @@ def view_failed():
                         proc.communicate(input=row.output.encode('utf-8'))
 
 class Config(dict):
+        """Linux configuration fetched from DB, key is option name, value is
+        'y' or 'n'.
+        """
         def __init__(self, id):
                 cur.execute("SELECT config FROM configurations WHERE id=%s;", (id,))
                 conf = cur.fetchone()
@@ -50,8 +53,11 @@ class Config(dict):
                         self[key] = val
 
 class VariableOptions(dict):
+        """Dictionary of config options that change value during experiments.
+        Key is the config name, value is the index to the matrix A
+        used for evaluation.
+        """
         def __init__(self, measurements):
-                self.order = []
                 all_options = {}
 
                 for m in measurements:
@@ -62,8 +68,15 @@ class VariableOptions(dict):
                                         all_options[key] = val
                                 else:
                                         if all_options[key] != val and key not in self:
-                                                self[key] = len(self)
-                                                self.order.append(key)
+                                                self[key] = None
+                self.update_order()
+
+        def update_order(self):
+                self.order = []
+                for key in sorted(self.keys()):
+                        self[key] = len(self.order)
+                        self.order.append(key)
+
         def __iter__(self):
                 return self.order.__iter__()
 
@@ -74,32 +87,72 @@ class VariableOptions(dict):
                 for k in self:
                         print("%-40s %s" % (k, self[k]))
 
-def evaluate():
-        cur.execute("SELECT conf, value FROM measure WHERE result='nominal' AND toolgit IN (1, 11);")
-        measurements = cur.fetchall()
-
-        options = VariableOptions(measurements)
-        options.print()
+configs = {}
 
+def construct_ab(options, measurements):
         A = np.zeros((len(measurements), len(options) + 1))
         B = np.zeros((len(measurements), 1))
 
         for i in range(len(measurements)):
-                config = Config(measurements[i].conf)
+                try:
+                        config = configs[measurements[i].conf]
+                except KeyError:
+                        config = Config(measurements[i].conf)
+                        configs[measurements[i].conf] = config
                 A[i, len(options)] = 1
                 B[i] = measurements[i].value
                 for j in range(len(options)):
                         A[i,j] = 1 if config[options.name_by_index(j)] == 'y' else 0
+        return (A, B)
+
+
+def evaluate():
+        cur.execute("SELECT conf, value FROM measure WHERE result='nominal' AND toolgit IN (1, 11);")
+        measurements = cur.fetchall()
+
+        if os.path.exists('options.json'):
+            options = VariableOptions([])
+            options.update(json.load(open('options.json', 'r')))
+            options.update_order()
+        else:
+            options = VariableOptions(measurements)
+            json.dump(options, open('options.json', 'w'), indent='  ', sort_keys=True)
+        #options.print()
+
+        (A,B) = construct_ab(options, measurements)
+        rank = nplag.matrix_rank(A)
+
+        o = options.copy()
+        for k in sorted(o.keys()):
+                del options[k]
+                options.update_order()
+                (A,B) = construct_ab(options, measurements)
+                if nplag.matrix_rank(A) != rank:
+                        options[k]=None
+                        options.update_order()
+                else:
+                        print("Removed", k)
+
+        (A,B) = construct_ab(options, measurements)
+
         #np.set_printoptions(threshold = float('nan'), linewidth = 300)
-        np.set_printoptions(edgeitems = 20, linewidth = 300)
-        print(A, nplag.matrix_rank(A))
-        print(B)
+        np.set_printoptions(edgeitems = 9, linewidth = 80)
+#         print(A)
+#         print(B)
         result = nplag.lstsq(A, B)
         print(result)
 
+        x=result[0]
+
+        results = []
         for k in options:
-                print("%-40s %g" % (k, result[0][options[k]]))
-        print("%-40s %g" % ("common", result[0][len(options)]))
+                idx = options[k]
+                results.append((k, x[idx], A[:,idx].sum(), A[:,idx].sum()/A.shape[0]*100))
+        results.sort(key=lambda r: r[1])
+        for r in results:
+                print("%-40s %7.4g %8d %3d%%" % r)
+        print("%-40s %g" % ("common", x[len(options)]))
+        print(A.shape)
 
 if __name__ == '__main__':
        evaluate()