From bf1ed3f323a038b620f560f2bf1aaa4424e0f9bd Mon Sep 17 00:00:00 2001 From: =?utf8?q?Karel=20Ko=C4=8D=C3=AD?= Date: Fri, 20 Nov 2015 10:05:51 +0100 Subject: [PATCH] Implement database using Django framework Implement current database using Django framework. This changes database access in project. W: Only database reimplemented, no other changes. This means breaking database access in other scripts --- Makefile | 16 ++-- scripts/database.py | 161 ----------------------------------- scripts/database/__init__.py | 4 + scripts/database/manage.py | 16 ++++ scripts/database/models.py | 42 +++++++++ scripts/database/settings.py | 18 ++++ 6 files changed, 89 insertions(+), 168 deletions(-) delete mode 100644 scripts/database.py create mode 100644 scripts/database/__init__.py create mode 100755 scripts/database/manage.py create mode 100644 scripts/database/models.py create mode 100644 scripts/database/settings.py diff --git a/Makefile b/Makefile index d511ff1..76babfb 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ help: HELP+="psql - Launch PostgreSQL interactive terminal.\n" .PHONY: psql psql: - PGPASSWORD="$(CONF_DB_PASSWORD)" psql -d "$(CONF_DB_DATABASE)" -h "$(CONF_DB_HOST)" -p "$(CONF_DB_PORT)" + psql -d "$(CONF_DB_DATABASE)" .PHONY: mbuildroot mbuildroot: @@ -50,11 +50,13 @@ init: initialize initialize: all scripts/initialize.py -HELP+="initdb - Initialize database.\n" -.PHONY: initdb initialize_database +HELP+="dbmigrate - Migrate database.\n" +.PHONY: initdb initialize_database dbmigrate initdb: initialize_database -initialize_database: - PGPASSWORD="$(CONF_DB_PASSWORD)" psql -d "$(CONF_DB_DATABASE)" -h "$(CONF_DB_HOST)" -p "$(CONF_DB_PORT)" -f scripts/databaseinit.sql +initialize_database: dbmigrate +dbmigrate: + ./scripts/database/manage.py makemigrations + ./scripts/database/manage.py migrate HELP+="test - Executes boot and benchmark test. You should use this before\n" HELP+=" target run. This target is for testing if initial kernel\n" @@ -105,11 +107,11 @@ clean_measure: $(RM) -r result $(RM) $(CONF_DOT_CONFIG) -HELP+="clean_database - Drop all tables in database.\n" +HELP+="clean_database - Drop all entries in database.\n" .PHONY: cleandb clean_database cleandb: clean_database clean_database: - PGPASSWORD="$(CONF_DB_PASSWORD)" psql -d "$(CONF_DB_DATABASE)" -h "$(CONF_DB_HOST)" -p "$(CONF_DB_PORT)" -f scripts/databaseclean.sql + ./scripts/database/manage.py flush HELP+="distclean - Cleans all configurations and generated files.\n" HELP+=" Including linux and buildroot.\n" diff --git a/scripts/database.py b/scripts/database.py deleted file mode 100644 index 58fdf9e..0000000 --- a/scripts/database.py +++ /dev/null @@ -1,161 +0,0 @@ -import os -import datetime -import postgresql -import collections - -import utils -import exceptions -from conf import conf -from conf import sf - -def __git_describe__(): - return utils.callsubprocess('git_describe', - conf.git_describe_cmd, False, True)[0] - -def __git_commit__(): - return utils.callsubprocess('git_rev_parse', - conf.git_commit_cmd, False, True)[0] - -def __timestamp__(): - return datetime.datetime.now().strftime('%y-%m-%d-%H-%M-%S') - -Config = collections.namedtuple('Config', 'id hash config') # Named tuple for configuration -Measure = collections.namedtuple('Measure', 'id conf_id output value') # Named tuple for measurement - -class database: - "Class used for accessing PostgreSQL project database." - def __init__(self): - self.db = postgresql.open(database = conf.db_database, - unix='/var/run/postgresql/.s.PGSQL.5432', - password='') - # check if tables are present - tables = ('toolsgit', 'configurations', 'measure') - for tab in tables: - val = self.db.prepare("""SELECT COUNT(*) FROM pg_class - WHERE relname = $1""")(tab)[0][0] - if val < 1: - raise exceptions.DatabaseUninitialized() - - def check_toolsgit(self): - "Return id of toolsgit row. If missing, it is inserted" - ds = __git_describe__() - cm = __git_commit__() - ps = self.db.prepare("""SELECT id FROM toolsgit - WHERE git_describe = $1 AND git_commit = $2 - """) - id = ps(ds, cm) - if id: - return id[0][0] - ps = self.db.prepare("""INSERT INTO toolsgit - (git_describe, git_commit) - VALUES - ($1, $2); - """) - ps(ds, cm) - return self.check_toolsgit() - - def check_linuxgit(self): - "Return id of linuxgit row. If missing, it is inserted." - wd = os.getcwd() - os.chdir(sf(conf.linux_sources)) - ds = __git_describe__() - cm = __git_commit__() - os.chdir(wd) - ps = self.db.prepare("""SELECT id FROM linuxgit - WHERE git_describe = $1 AND git_commit = $2 - """) - id = ps(ds, cm) - if id: - return id[0][0] - ps = self.db.prepare("""INSERT INTO linuxgit - (git_describe, git_commit) - VALUES - ($1, $2); - """) - ps(ds, cm) - return self.check_linuxgit() - - def add_configuration(self, hash, txtconfig, generator): - "Add configuration to database." - ps = self.db.prepare("""INSERT INTO configurations - (hash, config, gtime, toolgit, linuxgit, generator) - VALUES - ($1, $2, $3, $4, $5, $6); - """) - gt = self.check_toolsgit() - lgt = self.check_linuxgit() - tm = datetime.datetime.now() - ps(hash, '\n'.join(txtconfig), tm, gt, lgt, generator) - - def get_configration(self, hash): - "Return configration id for inserted hash." - ps = self.db.prepare("""SELECT id, config FROM configurations - WHERE hash = $1""") - rtn = [] - for dt in ps(hash): - rtn.append(Config(dt[0], hash, dt[1].split('\n'))) - return rtn - - def add_measure(self, output, result, conf_id, value = None): - "Add measurement." - ps = self.db.prepare("""INSERT INTO measure - (conf, output, value, mtime, toolgit, - linuxgit, measurement, result) - VALUES - ($1, $2, $3, $4, $5, $6, $7, $8); - """) - gt = self.check_toolsgit() - lgt = self.check_linuxgit() - tm = datetime.datetime.now() - ps(conf_id, output, value, tm, gt, lgt, conf.measure_identifier, result) - - def update_measure(self, measure_id, value): - "Update measured value" - ps = self.db.prepare("""UPDATE measure SET - (value) = ($2) - WHERE - id = $1; - """) - ps(measure_id, value) - - def get_measures(self, conf_id): - "Get measures for configuration with conf_id id" - ps = self.db.prepare("""SELECT id, output, value FROM measure - WHERE conf = $1; - """) - rtn = [] - for dt in ps(conf_id): - rtn.append(Measure(dt[0], conf_id, dt[1], dt[2])) - return rtn - - def get_unmeasured(self): - "Returns list of all unmeasured configurations." - # FIXME: Take into account case when we want the same - # configuration for either different experiment or kernel - # version or target. - ps = self.db.prepare("""SELECT id, hash, config FROM configurations - WHERE id NOT IN - (SELECT conf FROM measure) - """) - rtn = [] - for dt in ps(): - rtn.append(Config(dt[0], dt[1], dt[2].split('\n'))) - return rtn - - def add_configsort(self, configopt): - "Add configuration option to sorted list" - ps = self.db.prepare("""INSERT INTO configopt - (configopt) VALUES ($1) - """) - ps(configopt) - - def get_configsort(self): - "Returns sorted list of all configuration options" - ps = self.db.prepare("""SELECT id, configopt FROM configopt - ORDER BY id ASC - """) - rtn = [] - itms = ps() - for id, config in itms: - rtn.append(config) - return rtn diff --git a/scripts/database/__init__.py b/scripts/database/__init__.py new file mode 100644 index 0000000..c67d001 --- /dev/null +++ b/scripts/database/__init__.py @@ -0,0 +1,4 @@ +import os +import sys +from .settings import * +from .models import * diff --git a/scripts/database/manage.py b/scripts/database/manage.py new file mode 100755 index 0000000..fc9ef57 --- /dev/null +++ b/scripts/database/manage.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +import os +import sys + +if __name__ == "__main__": + + # Django searches for module 'database' this adds suitable search path + database_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + if not database_path in sys.path: + sys.path.insert(0, database_path) + + import database.settings + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/scripts/database/models.py b/scripts/database/models.py new file mode 100644 index 0000000..d9b5fdd --- /dev/null +++ b/scripts/database/models.py @@ -0,0 +1,42 @@ +import os +import sys +from django.db import models + +class ToolsGit(models.Model): + git_describe = models.TextField() + git_commit = models.TextField() + class Meta: + db_table = "toolsgit" + +class LinuxGit(models.Model): + git_describe = models.TextField() + git_commit = models.TextField() + class Meta: + db_table = "linuxgit" + +class Configurations(models.Model): + hash = models.CharField(max_length=32) + generator = models.TextField() + config = models.TextField() + gtime = models.DateTimeField() + linuxgit = models.ForeignKey('linuxgit', db_column='linuxgit') + toolgit = models.ForeignKey('toolsgit', db_column='toolgit') + class Meta: + db_table = "configurations" + +class Measure(models.Model): + conf = models.ForeignKey('configurations') + measurement = models.TextField() + output = models.TextField() + result = models.TextField() + value = models.FloatField() + mtime = models.DateTimeField() + linuxgit = models.ForeignKey('linuxgit', db_column='linuxgit') + toolgit = models.ForeignKey('toolsgit', db_column='toolgit') + class Meta: + db_table = "measure" + +class Configopt(models.Model): + configopt = models.TextField() + class Meta: + db_table = "configopt" diff --git a/scripts/database/settings.py b/scripts/database/settings.py new file mode 100644 index 0000000..a2a31f2 --- /dev/null +++ b/scripts/database/settings.py @@ -0,0 +1,18 @@ +import os + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +from django.conf import settings +if not settings.configured: + settings.configure( + INSTALLED_APPS = ['database'], + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'HOST': '', + 'NAME': 'linux-conf-perf', + } + }, + MIDDLEWARE_CLASSES = ( + ) + ) -- 2.39.2