Also initial implementation of multithread execution.
A lot of functionality changed.
Phases removed.
Output parsing is now part of measure (boot) process.
Utils cleared.
Add dot_measure file for inverted dot_config.
Configuration generating is now prepared for multiple generating types. Fist implemented is generating configurations with single selected configuration.
Test is modified to be compatible with new changes.
nbscript = 'scripts/nbscript'
# boot_command
# Command executed for booting. Output of this command is saved to output folder.
-boot_command = ['scripts/novaboot/novaboot', nbscript] + novaboot_args
+boot_command = ['echo', 'bootit']
+
+# parse_command
+# Command to parse double value from boot output
+parse_command = ['echo', '0']
# picosat_args
# Additional arguments passed to PicoSAT.
# Port of PotgreSQL database server
db_port = 5432
+# multithread
+# Define if measurement and kernel build should be executed in parallel.
+multithread = False
+# multithread_buffer
+# Defines maximal number of buffered configurations before generating is suspended.
+multithread_buffer = 32
+
# git_describe_cmd
# Command used for getting tools version and status from git
git_describe_cmd = ['git', 'describe', '--always', '--tags', '--dirty']
kernel_config_output = True
kernel_make_output = True
boot_output = True
+parse_output = False
## Configs for debugging
-step_by_step = False # Executes only single step and exits.
single_loop = False # Executes only one loop and exits.
only_config = False # Executes only to configuration phase. Building and booting phases are skipped.
ignore_misconfig = False # Ignore if configuration wasn't applied correctly.
build_folder = 'jobfiles/'
jobfolder_linux_image = build_folder + 'linuxImage'
-phase_file = build_folder + 'phase'
symbol_map_file = build_folder + 'symbol_map' # Also defined in parse_kconfig
rules_file = build_folder + 'rules' # Also defined in parse_kconfig
variable_count_file = build_folder + 'variable_count' # Also defined in parse_kconfig
-required_file = build_folder + 'required'
+fixed_file = build_folder + 'fixed'
+measure_file = build_folder + 'measure'
+dot_measure_file = build_folder + 'dot_measure'
dot_config_back_file = build_folder + 'dot_config_back'
-iteration_file = build_folder + 'iteration'
+single_generated_file = build_folder + 'single_generated'
configurations_folder = 'configurations/'
hashconfigsort = configurations_folder + 'hashconfigsort'
## Programs paths
parse_kconfig = 'scripts/parse_kconfig/parse'
-write_config = 'scripts/write_config/write'
+write_config = 'scripts/write_config/write_config'
picosat = 'scripts/picosat-959/picosat'
+allconfig = 'scripts/allconfig/allconfig'
absroot = os.path.dirname(os.path.realpath(__file__))
from conf import conf
from conf import sf
from exceptions import MissingFile
+import database
-def boot():
+def boot(config, to_database = True):
try:
os.mkdir(sf(conf.output_folder))
except FileExistsError:
pass
- wd = os.getcwd()
-
- sprc = subprocess.Popen(conf.boot_command,
- stdout = subprocess.PIPE)
- with open(os.path.join(sf(conf.output_folder), utils.get_last_configuration()), "a") as f:
+ sprc = subprocess.Popen(conf.boot_command, stdout = subprocess.PIPE)
+ with open(os.path.join(sf(conf.output_folder), config.cfile), "a") as f:
for linen in sprc.stdout:
line = linen.decode('utf-8')
if conf.boot_output:
print(line, end="")
f.write(line)
- os.chdir(wd)
+ # Let user script parse double value
+ out = utils.callsubprocess('parse_command', conf.parse_command,
+ conf.parse_output, True)
+ value = float(out[0])
+
+ if to_database:
+ dtb = database.database()
+ dtb.add_measure(config.cfile, config.id, value)
+
+ return config.cfile
picosat_cmd = [sf(conf.picosat), file]
picosat_cmd += conf.picosat_args
stdout = utils.callsubprocess('picosat', picosat_cmd, conf.picosat_output,
- True, allowed_exit_codes = [10])
+ True, allow_all_exit_codes = True)
rtn = []
solut = []
pass
return rtn
-def __write_temp_config_file__(con):
+def __write_temp_config_file__(con, conf_num):
# Ensure smap existence
utils.build_symbol_map()
- # Load variable count
- with open(sf(conf.variable_count_file)) as f:
- f.readline()
- var_num = int(f.readline())
# Write temporally file
wfile = tempfile.NamedTemporaryFile(delete=False)
for s in con:
s *= -1
else:
nt = False
- if s > var_num:
+ if s > conf_num:
break;
if 'NONAMEGEN' in utils.smap[s]: # ignore generated names
continue
hsh = hashlib.md5(bytes(cstr, 'UTF-8'))
return hsh.hexdigest()
-def __register_conf__(con):
+def __register_conf__(con, conf_num):
dtb = database.database()
# Solution to configuration
- wfile = __write_temp_config_file__(con)
+ wfile = __write_temp_config_file__(con, conf_num)
hsh = __calchash__(wfile)
filen = os.path.join(sf(conf.configurations_folder), hsh)
hshf = hsh
shutil.move(wfile, filen)
dtb.add_configuration(hsh, hshf)
+def __generate_single__(var_num, conf_num):
+ if os.path.isfile(sf(conf.single_generated_file)):
+ return False
+ measure_list = []
+ with open(sf(conf.measure_file), 'r') as f:
+ for ln in f:
+ measure_list.append(int(ln))
+ for measure in measure_list:
+ tfile = __buildtempcnf__(var_num, (sf(conf.rules_file),
+ sf(conf.fixed_file)), (str(measure)))
+ try:
+ confs = __exec_sat__(tfile, ['-i', '0'])
+ for con in confs:
+ __register_conf__(con, conf_num)
+ except exceptions.NoSolution:
+ pass
+ finally:
+ os.remove(tfile)
+ with open(sf(conf.single_generated_file), 'w') as f:
+ f.write("This file informs scripts, that all single selected configurations are already generated.\n")
+ f.write("Remove this file if you want run generating process again.")
+ return True
+
+def __generate_random__(var_num, conf_num):
+ # TODO
+ pass
def generate():
"""Collect boolean equations from files rules and required
# Check if rules_file exist. If it was generated.
if not os.path.isfile(sf(conf.rules_file)):
raise exceptions.MissingFile(conf.rules_file,"Run parse_kconfig.")
- if not os.path.isfile(sf(conf.required_file)):
- raise exceptions.MissingFile(conf.required_file,"Run allconfig.")
+ if not os.path.isfile(sf(conf.fixed_file)):
+ raise exceptions.MissingFile(conf.required_file,"Run allconfig and initialization process.")
- # Load variable clount
+ # Load variable count
with open(sf(conf.variable_count_file)) as f:
var_num = f.readline()
- tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.required_file)), ())
- try:
- confs = __exec_sat__(tfile, [])
- os.remove(tfile)
- for con in confs:
- __register_conf__(con)
- except exceptions.NoSolution:
- os.remove(tfile)
- raise exceptions.NoSolution()
+ conf_num = f.readline()
+
+ if __generate_single__(var_num, conf_num):
+ return
+
+ #tfile = __buildtempcnf__(var_num, (sf(conf.rules_file), sf(conf.fixed_file)), ())
+ #try:
+ #confs = __exec_sat__(tfile, [])
+ #for con in confs:
+ #__register_conf__(con, conf_num)
+ #finally:
+ #os.remove(tfile)
def compare(file1, file2):
"""Compared two configuration"""
def __str__(self):
return "SAT solver found no solution. Statement is not satisfiable."
-class PhaseMismatch(Exception):
- def __init__(self):
- pass
- def __str__(self):
- return "Phase in " + conf.phase_file + " is unknown."
-
class ConfigurationError(Exception):
def __init__(self, message):
self.message = message;
def __str__(self):
return "Configuration error: " + message
-class NoApplicableSolution(Exception):
+class NoApplicableConfiguration(Exception):
def __init__(self):
pass
def __str__(self):
- return "No applicable solution find. All generated solutions were already applied."
+ return "No applicable configuration find. All generated configurations were already applied."
class ProcessFailed(Exception):
def __init__(self, process, returncode):
from conf import conf
from conf import sf
import exceptions
-import loop
def all():
base()
parse_kconfig()
- gen_requred()
+ gen_fixed()
# check if database is initialized
database.database()
except FileExistsError:
pass
- if os.path.isfile(sf(conf.phase_file)):
- print("Warning: file " + conf.phase_file + " already exists. Not overwritten.")
- else:
- loop.phase_set(1)
-
- if os.path.isfile(sf(conf.iteration_file)):
- print("Warning: file " + conf.iteration_file + " already exists. Not overwritten.")
- else:
- loop.iteration_reset()
-
-
def parse_kconfig():
"Execute parse_kconfig in linux_sources directory."
if os.path.isfile(sf(conf.symbol_map_file)) and \
print('Warning: parse_kconfig not executed. Files already exists.')
return
print('Executing parse_kconfig...')
- env = dict(os.environ)
wd = os.getcwd()
os.chdir(sf(conf.linux_sources))
parse_kconfig_cmd = [sf(conf.parse_kconfig)]
os.chdir(wd)
-def gen_requred():
- "Generates required depenpency from dot_config file."
+def __gen_allconfig_fixed__():
+ wd = os.getcwd()
+ os.chdir(sf(conf.linux_sources))
+ allconfig_cmd = [sf(conf.allconfig)]
+ allconfig_cmd += ['Kconfig', sf(conf.dot_config), sf(conf.dot_measure_file)]
+ allconfig_cmd += ['--inv']
+ utils.callsubprocess("allconfig_fixed", allconfig_cmd, False,
+ env = utils.get_kernel_env())
+ os.chdir(wd)
+
+def gen_fixed():
+ "Generates fixed depenpency from dot_config file."
print('Generating required configuration...')
if not os.path.isfile(sf(conf.dot_config)):
srmap = {value:key for key, value in utils.smap.items()} # swap dictionary
shutil.copy(sf(conf.dot_config), sf(conf.dot_config_back_file))
+ __gen_allconfig_fixed__()
with open(sf(conf.dot_config), 'r') as f:
- with open(sf(conf.required_file), 'w') as freq:
+ with open(sf(conf.fixed_file), 'w') as ffix:
for line in f:
if (line[0] == '#') or (not '=' in line):
continue
if (line[indx + 1] == 'y'):
if line[7:indx] == "MODULES": # exception if modules set
raise exceptions.ConfigurationError("Fixed kernel configuration must have MODULES disabled.")
- freq.write(str(srmap[line[7:indx]]) + "\n")
+ ffix.write(str(srmap[line[7:indx]]) + "\n")
elif (line[indx + 1] == 'n' or line[indx + 1] == 'm'):
- freq.write("-" + str(srmap[line[7:indx]]) + "\n")
+ ffix.write("-" + str(srmap[line[7:indx]]) + "\n")
+ with open(sf(conf.dot_measure_file), 'r') as f:
+ with open(sf(conf.measure_file), 'w') as fmes:
+ for line in f:
+ if (line[0] == '#') or (not '=' in line):
+ continue
+ indx = line.index('=')
+ if line[7:indx] == "MODULES":
+ raise exceptions.ConfigurationError("Can't measure configuraion option MODULES. Not supported.")
+ fmes.write(str(srmap[line[7:indx]]) + "\n")
#################################################################################
import subprocess
import signal
from threading import Thread
+from threading import Lock
from conf import conf
from conf import sf
import kernel
import boot
import exceptions
+import database
-def step():
- phs = phase_get()
- if phs == 0 or phs == 1:
- phase_message(1)
- initialize.all()
- phase_set(2)
- elif phs == 2:
- phase_message(2)
- phase_set(3)
- elif phs == 3:
- phase_message(3)
- try:
- configurations.apply()
- except exceptions.NoApplicableSolution:
- try:
- os.mkdir(sf(conf.result_folder))
- except FileExistsError:
- pass
- print('\nAll done.')
- exit(0)
- phase_set(4)
- elif phs == 4:
- phase_message(4)
- phase_set(5)
- elif phs == 5:
- phase_message(5)
- try:
- kernel.config()
- except exceptions.ConfigurationError:
- if not conf.ignore_misconfig:
- print("Configuration mismatch. Exiting.")
- sys.exit(-2)
- phase_set(6)
- elif phs == 6:
- phase_message(6)
- if conf.only_config:
- phase_set(3)
- else:
- phase_set(7)
- elif phs == 7:
- phase_message(7)
- kernel.make()
- phase_set(8)
- elif phs == 8:
- phase_message(8)
- phase_set(9)
- elif phs == 9:
- phase_message(9)
- boot.boot()
- phase_set(10)
- elif phs == 10:
- phase_message(10)
- phase_set(3)
+__confs_unmeasured__ = []
-# Phase #
-phases = ("Not Initialized", #0
- "Initializing", #1
- "Initialized", #2
- "Solution applying", #3
- "Solution applied", #4
- "Kernel configuration", #5
- "Kernel configured", #6
- "Kernel build", #7
- "Kernel built", #8
- "System boot", #9
- "Benchmark successful" #10
- )
+def prepare():
+ """Prepare for measuring
+ Outcome is Linux image for generated configuration."""
+ global __confs_unmeasured__
+ if len(__confs_unmeasured__) == 0:
+ dtb = database.database()
+ confs = dtb.get_unmeasured()
+ if len(confs) == 0:
+ configurations.generate()
+ confs = dtb.get_unmeasured()
+ if len(confs) == 0:
+ raise exceptions.NoApplicableConfiguration()
+ __confs_unmeasured__ = list(confs)
+ con = __confs_unmeasured__.pop()
+ kernel.config(con.cfile)
+ img = kernel.make(con.hash)
+ print("Prepared image: " + img)
+ return img, con
-def phase_get():
+def measure(kernelimg, con):
try:
- with open(sf(conf.phase_file)) as f:
- txtPhase = f.readline().rstrip()
- if not txtPhase in phases:
- raise PhaseMismatch()
- return phases.index(txtPhase)
+ os.remove(sf(conf.jobfolder_linux_image))
except FileNotFoundError:
- return 0
-
-def phase_set(phs):
- # TODO
- try:
- global thr
- if thr.term:
- return
- except NameError:
pass
- with open(sf(conf.phase_file), 'w') as f:
- f.write(phases[phs])
+ os.symlink(os.path.join(sf(conf.build_folder), kernelimg),
+ sf(conf.jobfolder_linux_image))
+ boot.boot(con)
+ print("Configuration '" + con.hash + "' measured.")
-def phase_message(phs):
- "Prints message signaling running phase_"
- print("-- " + phases[phs])
+# Threads #
+__terminate__ = False
+class mainThread(Thread):
+ def run(self):
+ if conf.single_loop:
+ img, config = prepare()
+ measure(img, config)
+ else:
+ while not __terminate__:
+ img, config = prepare()
+ measure(img, config)
-# Iteration #
-def iteration_reset():
- with open(sf(conf.iteration_file), 'w') as f:
- f.write('0')
+# Multithread section #
+__conflist__ = []
+__listlock__ = Lock()
-def iteration_inc():
- with open(sf(conf.iteration_file), 'r') as f:
- it = int(f.readline())
- it += 1
- with open(sf(conf.iteration_file), 'w') as f:
- f.write(str(it))
+class prepareThread(Thread):
+ def __init__(self, name='prepare'):
+ Thread.__init__(self, name=name)
+ def run(self):
+ __listlock__.aquire()
+ while not __terminate__ and len(__conflist__) <= conf.multithread_buffer:
+ __listlock__.release()
+ config = prepare()
+ __listlock__.aquire()
+ __conflist__.append(config)
+ if not __measurethread__.isActive():
+ __measurethread__.start()
+ __listlock__.release()
-# Thread #
-class mainThread(Thread):
- def __init__(self, name):
+class measureThread(Thread):
+ def __init__(self, name='measure'):
Thread.__init__(self, name=name)
- self.term = False
def run(self):
- if conf.step_by_step:
- step()
- elif conf.single_loop:
- while not phase_get() == 2:
- step()
- step()
- while not phase_get() == 2:
- step()
- else:
- while not self.term:
- step()
+ __listlock__.aquire()
+ while not __terminate__ and len(__conflist__) > 0:
+ config = __conflist__[0]
+ del __conflist__[0]
+ __listlock__.release()
+ if not __preparethread__.isActive():
+ __preparethread__.start()
+ measure(config)
+ __listlock__.aquire()
+ __listlock__.release()
-def loop_term():
- global thr
- thr.term = True
+__preparethread__ = prepareThread()
+__measurethread__ = measureThread()
+# Start and sigterm handler #
def sigterm_handler(_signo, _stack_frame):
- loop_term()
+ __terminate__ = True
def loop():
+ initialize.all()
global thr
- thr = mainThread("thred")
+ thr = mainThread()
thr.start()
try:
thr.join()
except KeyboardInterrupt:
- loop_term()
+ __terminate__ = True
#################################################################################
import initialize
import kernel
import boot
+import database
def test():
initialize.base()
initialize.parse_kconfig()
- initialize.gen_requred() # Call this to check initial solution
+ print("-- Make --")
conf.kernel_make_output = True
- kernel.make()
+ img = kernel.make('test')
+ try:
+ os.remove(sf(conf.jobfolder_linux_image))
+ except FileNotFoundError:
+ pass
+ os.symlink(os.path.join(sf(conf.build_folder), img),
+ sf(conf.jobfolder_linux_image))
conf.boot_output = True
- boot.boot()
+ conf.parse_output = True
+ print("-- Boot --")
+ config = database.Config('0', 'test', img)
+ boot.boot(config, False)
#################################################################################
def callsubprocess(process_name, process, show_output = True,
- return_output = False, env=os.environ, allowed_exit_codes = [0]):
+ return_output = False, env=os.environ, allowed_exit_codes = [0],
+ allow_all_exit_codes = False):
sprc = subprocess.Popen(process, stdout = subprocess.PIPE, env = env)
try:
rtn.append(line.rstrip())
rtncode = sprc.wait()
- if rtncode not in allowed_exit_codes:
+ if rtncode not in allowed_exit_codes and not allow_all_exit_codes:
raise exceptions.ProcessFailed(process, rtncode)
return rtn
env = dict(os.environ)
env.update(conf.kernel_env)
return env
-
-
-def hash_config(cf):
- """Hashes configuration using MD5 hash.
- """
- try:
- cf.remove(0)
- except ValueError:
- pass
- str = ""
- for c in cf:
- if c < 0:
- str += '-'
- else:
- str += '+'
- hsh = hashlib.md5(bytes(str, sys.getdefaultencoding()))
- return hsh.hexdigest()
-
-def config_strtoint(str, full):
- """Reads list of configured symbols from string
- """
- rtn = []
- if full:
- for s in str.rstrip().split(sep=' '):
- rtn.append(int(s))
- else:
- count = 0
- with open(sf(conf.variable_count_file)) as f:
- f.readline()
- count = int(f.readline())
- for s in str.rstrip().split(sep=' '):
- val = int(s)
- if abs(val) <= count:
- rtn.append(val)
- else:
- break;
- try:
- rtn.remove(0)
- except ValueError:
- pass
- return rtn
-
-def get_config_from_hash(hash):
- with open(sf(conf.config_map_file), "r") as f:
- for line in f:
- w = line.rstrip().split(sep=':')
- if w[0] == hash:
- return config_strtoint(w[1], True)
- return None
-
-def get_last_configuration():
- hsh = ""
- try:
- with open(sf(conf.config_solved_file), "r") as f:
- for line in f:
- sline = line.rstrip()
- if sline != '':
- hsh = sline
- except FileNotFoundError:
- try:
- with open(sf(conf.config_map_file), "r") as f:
- w = f.readline().split(sep=':')
- hsh = w[0]
- except FileNotFoundError:
- pass
-
- if hsh != '':
- return hsh
- else:
- return 'NoConfig'
projects / linux-conf-perf.git / commitdiff