From 341f20fbc131a3104da33d079f04228554cff2ad Mon Sep 17 00:00:00 2001 From: Markus Date: Fri, 17 May 2019 17:15:42 +0200 Subject: [PATCH 01/31] Add new feature to fluid2d: EMS EMS stands for Experiment Management System and is a new way to handle big sets of experiments. It is an optional feature that does not influence the way fluid2d works unless activated by the user. An example of its usage is shown in the new experiment `Waves with EMS`. --- core/ems.py | 395 +++++++++++++++++++ experiments/Waves with EMS/wave_breaking.exp | 41 ++ experiments/Waves with EMS/wave_breaking.py | 150 +++++++ 3 files changed, 586 insertions(+) create mode 100644 core/ems.py create mode 100644 experiments/Waves with EMS/wave_breaking.exp create mode 100644 experiments/Waves with EMS/wave_breaking.py diff --git a/core/ems.py b/core/ems.py new file mode 100644 index 0000000..c73b577 --- /dev/null +++ b/core/ems.py @@ -0,0 +1,395 @@ +# Markus Reinert, May 2019 +# +# The fluid2d Experiment Management System (EMS). +# +# This module provides a way to handle big sets of fluid2d-experiments. +# It creates a database in which information about all the experiments +# is stored. This allows to easily show, compare, search for, filter +# and save comments to already executed experiments. Furthermore, it +# provides, after set up for a specific experiment, a simpler interface +# to modify the parameteres of interest. To avoid overwriting files, +# it automatically assigns a unique identifier to every new experiment. +# +# Its usage is explained in the breaking waves experiment. + +import os +import datetime +import sqlite3 as dbsys + + +# Columns with the following names are automatically added to every table of the database. +# Therefore they must not be used as parameter names. +RESERVED_COLUMN_NAMES = [ + 'id', + 'datetime', + 'duration', + 'size_total', + 'size_mp4', + 'size_his', + 'size_diag', + 'size_flux', + 'comment', +] + + +class InputMismatch(Exception): + """Incompatibility of given experiment file and database.""" + pass + + +class ParamError(Exception): + """Non-conformance of user-set attributes of param.""" + pass + + +class ExpFileError(Exception): + """Malformed experiment file.""" + pass + + +class EMS: + """Experiment Management System""" + + def __init__(self, param: "param.Param", experiment_file: str=""): + """Constructor of the Experiment Management System. + + Set up a connection to the database. If an experiment file is + given, a new entry in the database is created for this file. + Otherwise an existing entry corresponding to `param.expname` is + used. The parameters of this entry are loaded. + They can be accessed via the method `get_parameters`.""" + + # Get the path to the database and create a connection + datadir = param.datadir + if datadir.startswith("~"): + datadir = os.path.expanduser(param.datadir) + dbpath = os.path.join(datadir, "experiments.db") + print("-"*50) + print(" Opening database {}.".format(dbpath)) + self.connection = dbsys.connect(dbpath) + cursor = self.connection.cursor() + + if experiment_file: + # If an experiment file is given, parse it and add a new entry to the database. + self.exp_class, self.description, param_list = parse_experiment_file(experiment_file) + if not self.exp_class: + raise ExpFileError("no name given in file {}.".format(experiment_file)) + # Add static parameter fields + param_list = ( + [ + ["INTEGER", "id", -1], + ["TEXT", "datetime", datetime.datetime.now().isoformat()], + ] + + param_list + + [ + ["REAL", "duration", -1], + ["REAL", "size_total", -1], + ["REAL", "size_mp4", -1], + ["REAL", "size_his", -1], + ["REAL", "size_diag", -1], + ["REAL", "size_flux", -1], + ["TEXT", "comment", self.description], + ] + # Everything added here should be in RESERVED_COLUMN_NAMES + ) + # Check whether table exists already + if table_exists(cursor, self.exp_class): + # Check if table has the same columns + cursor.execute('PRAGMA table_info("{}")'.format(self.exp_class)) + for column in cursor.fetchall(): + col_index = column[0] + col_name = column[1] + col_type = column[2] + if (col_type != param_list[col_index][0] or + col_name != param_list[col_index][1]): + raise InputMismatch( + "column {} of the database ({} {}) does not match " + "the corresponding parameter ({} {}) in the file {}." + .format(col_index + 1, col_type, col_name, + *param_list[col_index][:2], experiment_file) + ) + # Get the highest index + cursor.execute('SELECT id from "{}" ORDER BY id DESC'.format(self.exp_class)) + self.id_ = cursor.fetchone()[0] + 1 + else: + # Create a new table + print(' Creating new table "{}".'.format(self.exp_class)) + param_string = ", ".join(['"{}" {}'.format(n, t) for t, n, v in param_list]) + sql_command = 'CREATE TABLE "{}" ({})'.format(self.exp_class, param_string) + cursor.execute(sql_command) + # TODO: make string formating safe against SQL injection, + # possibly by using the "?"-format + # First entry has index 1 (one) + self.id_ = 1 + # Set id in the parameter list + param_list[0][2] = self.id_ + # Add a new entry to the table + print(' Adding new entry #{} to table "{}".'.format(self.id_, self.exp_class)) + value_string = ", ".join(['"{}"'.format(v) for t, n, v in param_list]) + sql_command = 'INSERT INTO "{}" VALUES ({})'.format(self.exp_class, value_string) + cursor.execute(sql_command) + # TODO: make string formating safe against SQL injection, + # possibly by using the "?"-format + # Save the database + self.connection.commit() + # Set the name of the experiment + param.expname = "{}_{:03}".format(self.exp_class, self.id_) + else: + # Get name and id of the experiment + expname_parts = param.expname.split('_') + if len(expname_parts) == 1: + raise ParamError( + 'param.expname is not a valid database entry: "{}"'.format(param.expname) + ) + self.exp_class = '_'.join(expname_parts[:-1]) + try: + self.id_ = int(expname_parts[-1]) + except ValueError: + raise ParamError( + 'param.expname is not a valid database entry: "{}"'.format(param.expname) + ) + + # Remember directory of the experiment + self.output_dir = os.path.join(datadir, param.expname) + + # Get columns of the table and their value for the current experiment + self.params = dict() + cursor.execute('PRAGMA table_info("{}")'.format(self.exp_class)) + columns = cursor.fetchall() + cursor.execute('SELECT * FROM "{}" WHERE id = ?'.format(self.exp_class), (self.id_,)) + values = cursor.fetchone() + for column in columns: + col_index = column[0] + col_name = column[1] + value = values[col_index] + if col_name in RESERVED_COLUMN_NAMES: + continue + else: + if value == "True": + value = True + elif value == "False": + value = False + self.params[col_name] = value + + def __del__(self): + """Destructor of the Experiment Management System. + + Save the database and close the connection to it.""" + # Save and close database + print(" Closing database.") + print("-"*50) + self.connection.commit() + self.connection.close() + + def get_parameters(self): + """Get user-set experiment parameters and values as dictionary.""" + return self.params + + def finalize(self, fluid2d): + """Save integration time and size of output files in database. + + This method must be called when the simulation is finished, + that means, after the line with `f2d.loop()`. + It also sets the field `datetime` to the current time.""" + + # Get duration of the run and size of the output files + integration_time = fluid2d.t + + # Get list of files in the output directory + output_files = [os.path.join(self.output_dir, f) for f in os.listdir(self.output_dir)] + # To get size in MB, divide by 1000*1000 = 1e6. + total_size = sum(os.path.getsize(path) for path in output_files) / 1e6 + his_size = os.path.getsize(fluid2d.output.hisfile) / 1e6 + diag_size = os.path.getsize(fluid2d.output.diagfile) / 1e6 + if fluid2d.plot_interactive and hasattr(fluid2d.plotting, 'mp4file'): + mp4_size = os.path.getsize(fluid2d.plotting.mp4file) / 1e6 + else: + mp4_size = 0.0 + if fluid2d.diag_fluxes: + flux_size = os.path.getsize(fluid2d.output.flxfile) / 1e6 + else: + flux_size = 0.0 + + # Update values in the database + self.connection.execute( + 'UPDATE "{}" SET duration = ? WHERE id = ?'.format(self.exp_class), + (round(integration_time, 2), self.id_,) + ) + self.connection.execute( + 'UPDATE "{}" SET size_total = ? WHERE id = ?'.format(self.exp_class), + (round(total_size, 3), self.id_,) + ) + self.connection.execute( + 'UPDATE "{}" SET size_mp4 = ? WHERE id = ?'.format(self.exp_class), + (round(mp4_size, 3), self.id_,) + ) + self.connection.execute( + 'UPDATE "{}" SET size_his = ? WHERE id = ?'.format(self.exp_class), + (round(his_size, 3), self.id_,) + ) + self.connection.execute( + 'UPDATE "{}" SET size_diag = ? WHERE id = ?'.format(self.exp_class), + (round(diag_size, 3), self.id_,) + ) + self.connection.execute( + 'UPDATE "{}" SET size_flux = ? WHERE id = ?'.format(self.exp_class), + (round(flux_size, 3), self.id_,) + ) + self.connection.execute( + 'UPDATE "{}" SET datetime = ? WHERE id = ?'.format(self.exp_class), + (datetime.datetime.now().isoformat(), self.id_) + ) + + # Save the database + self.connection.commit() + + +def parse_experiment_file(path: str): + """Parse the given experiment file. + + An experiment file + - must provide a name, + - can provide a description, + - can provide parameters with values. + + The name is written in a line starting with "Name:". It must be a + valid string to be used as a filename; in particular, it must not + contain the slash "/". + + The description begins in or after a line starting with + "Description:" and goes until the beginning of the parameters + or the end of the file. + + The parameters follow after a line starting with "Parameters:". + Every parameter is written in its own line. This line contains the + datatype, the name and the value of the parameter seperated by one + or several whitespaces. + The datatype must be one of "int", "float", "bool" or "str". + The name must not contain whitespace characters and must not be in + the list of reserved column names. + The value must be a valid value for the given datatype. If the + value is omitted, it defaults to zero for numbers, True for booleans + and the empty string for strings. The values "True" and "False" can + only be used for parameters of boolean type, not as strings. + It is planned to include the possibility of providing several values + in order to run multiple experiments from one experiment file. + + Lines in the experiment file starting with the #-symbol are ignored. + It is not possible to write in-line comments.""" + # TODO: allow parameter names and string-values with whitespace like "long name". + # TODO: allow to give several values to run multiple experiments. + with open(path) as f: + experiment_lines = f.readlines() + name = "" + description = "" + param_list = [] + reading_params = False + reading_description = False + for line in experiment_lines: + if line.startswith("#"): + # Skip comments + continue + elif not line.strip(): + # Skip empty lines except in the description + if reading_description: + description += line + else: + continue + elif line.lower().startswith("name:"): + if not name: + name = line[5:] + else: + raise ExpFileError( + "name defined more than once in file {}.".format(path) + ) + elif line.lower().startswith("description:"): + reading_description = True + reading_params = False + description += line[12:] + elif line.lower().startswith("parameters:"): + reading_description = False + reading_params = True + elif reading_description: + description += line + elif reading_params: + words = line.split() + if len(words) < 2: + raise ExpFileError( + 'type or name missing for parameter "{}" in file {}.'.format(words[0], path) + ) + param_type = words[0].lower() + param_name = words[1] + param_values = words[2:] + # Check values + if len(param_values) == 0: + param_value = None + elif len(param_values) == 1: + param_value = param_values[0] + else: + raise NotImplementedError( + 'multiple values given for parameter "{}" in file {}.' + .format(param_name, path) + ) + # Check type + if param_type == "int": + sql_type = "INTEGER" + if param_value is None: + param_value = 0 + else: + param_value = int(param_value) + elif param_type == "float": + sql_type = "REAL" + if param_value is None: + param_value = 0.0 + else: + param_value = float(param_value) + elif param_type == "bool": + if param_value is None: + param_value = "True" + elif param_value.lower() == "true": + param_value = "True" + elif param_value.lower() == "false": + param_value = "False" + else: + raise ExpFileError( + 'boolean parameter "{}" is neither "True" nor "False" ' + 'but "{}" in file {}.'.format(param_name, param_value, path) + ) + sql_type = "TEXT" + elif param_type == "str": + sql_type = "TEXT" + if param_value is None: + param_value = "" + elif param_value == "True" or param_value == "False": + raise ExpFileError( + 'the words "True" and "False" cannot be used as value for ' + 'parameter "{}" of type string. Instead, use boolean type ' + 'parameter in file {}.'.format(param_name, path) + ) + else: + raise ExpFileError( + 'unknown parameter type "{}" in file {}.'.format(param_type, path) + ) + # Check name + if param_name in RESERVED_COLUMN_NAMES: + raise ExpFileError( + 'reserved name used for parameter "{}" in file {}.' + .format(param_name, path) + ) + param_list.append([sql_type, param_name, param_value]) + else: + raise ExpFileError( + "unexpected line in file {}:\n{}".format(path, repr(line)) + ) + return name.strip(), description.strip(), param_list + + +def table_exists(cursor: dbsys.Cursor, name: str): + """Check if table with given name exists in the connected database.""" + # Get all tables + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + # This gives a list like that: [('table1',), ('table2',)] + for table in cursor.fetchall(): + if table[0] == name: + return True + return False diff --git a/experiments/Waves with EMS/wave_breaking.exp b/experiments/Waves with EMS/wave_breaking.exp new file mode 100644 index 0000000..a9e3111 --- /dev/null +++ b/experiments/Waves with EMS/wave_breaking.exp @@ -0,0 +1,41 @@ +# Markus Reinert, May 2019 +# +# Fluid2d-experiment with EMS: wave breaking at the coast (experiment file) +# +# Refer to the documentation of `ems.parse_experiment_file` for a +# detailed description of the format of experiment files. + + +# Name of the experiment class. +# If the name is changed, a new table is created in the database. +Name: Breaking_Waves + + +# Descriptions are optional. +# They can go over multiple lines and can also contain empty lines. +Description: Simulate interface waves, their propagation and breaking at a sloping coast. +Various parameters can be modified in this file. + + +# Here are the parameters that we want to modify in this class of experiments. +# Parameters which we intend to keep constant are in the Python file. +Parameters: + +### Physics +# activate or deactivate diffusion +bool diffusion False +# diffusion coefficient (if diffusion is True) +float Kdiff 1e-2 + +### Coast +float slope 0.5 +float height 1.05 +# beginning of the flat coast +float x_start 4.0 + +### Initial Condition +# perturbation can be of type "sin" or "gauss" +str perturbation gauss +# height and width of the perturbation +float intensity 0.5 +float sigma 0.3 diff --git a/experiments/Waves with EMS/wave_breaking.py b/experiments/Waves with EMS/wave_breaking.py new file mode 100644 index 0000000..007185f --- /dev/null +++ b/experiments/Waves with EMS/wave_breaking.py @@ -0,0 +1,150 @@ +# Markus Reinert, April/May 2019 +# +# Fluid2d-experiment with EMS: wave breaking at the coast (Python file) +# +# This file aims to be both, an interesting experiment and a helpful +# introduction to the Experiment Management System (EMS) of fluid2d. +# +# To use the EMS, we import the module `ems` into the Python script of +# our experiment and create an instance of the class `EMS`, giving the +# usual Param-object and an experiment file. This creates an entry in +# the experiment-database. The database is stored as an SQLite file +# with the name `experiments.db` in the data directory of fluid2d, i.e., +# the path defined in `param.datadir`. Furthermore, it sets the value +# of `param.expname` to refer to this entry. Therefore, when the EMS is +# used, the value of `param.expname` must not be set manually. +# Exceptions to this rule are explained below. +# At the beginning, some information is missing in the new entry of the +# database. To complete it, we call the method `finalize` of the EMS at +# the end of the simulation. Through this, the actual integration time +# and the size of the output files (in MB) is stored in the database. +# The field datetime is also updated and set to the current time. +# The parameters defined in the experiment file are accessible in the +# Python script through the dictionary of experiment parameters (EP). +# +# If the experiment parameters are also needed somewhere else in the +# code where the EMS-object is not accessible, for example in a forcing +# module, then a new EMS-object can be created. This new EMS-object +# must be initialized with the same Param-object and without an +# experiment file. In this way, both EMS-objects refer to the same +# entry of the database and share the same experiment parameters. +# +# It is possible to re-run experiments with overwriting the files, for +# example because the integration was stopped too early. +# To do this, we specify as `param.expname` the name of the experiment +# we want to run again, consisting of the name of the experiment class +# and a 3-digit number. Then we call the constructor of EMS without an +# experiment file. Example: +# param.expname = "Breaking_Waves_001" +# ems = EMS(param) +# This replaces the files in the folder `Breaking_Waves_001` and updates +# the entry 1 of the table `Breaking_Waves` at the end of the run. +# +# In general, whenever an EMS-object is created with an experiment file +# given, it creates a new entry in the database with the information +# from the experiment file and it sets `param.expname` to refer to this +# new entry. In contrast, if an EMS-object is created without an +# experiment file, it loads the data from the database entry which +# corresponds to the value of `param.expname`. + + +from fluid2d import Fluid2d +from param import Param +from grid import Grid +from ems import EMS + +import numpy as np + + +# Load default values and set type of model +param = Param("default.xml") +param.modelname = "boussinesq" + +# Activate the Experiment Management System (EMS) with an experiment file +ems = EMS(param, "wave_breaking.exp") +# Fetch experiment parameters (EP) +EP = ems.get_parameters() + +# Set domain type, size and resolution +param.geometry = "closed" +param.ny = 2 * 64 +param.nx = 3 * param.ny +param.Ly = 2 +param.Lx = 3 * param.Ly +# Set number of CPU cores used +param.npx = 1 +param.npy = 1 + +# Set time settings +param.tend = 100.0 +param.cfl = 1.2 +param.adaptable_dt = False +param.dt = 0.01 +param.dtmax = 0.1 + +# Choose discretization +param.order = 5 + +# Set output settings +param.var_to_save = ["vorticity", "buoyancy", "psi"] +param.list_diag = "all" +param.freq_his = 0.2 +param.freq_diag = 0.1 + +# Set plot settings +param.plot_var = "buoyancy" +param.plot_interactive = True +param.generate_mp4 = True +param.freq_plot = 10 +param.colorscheme = "imposed" +param.cax = [0, 1] +param.cmap = "Blues_r" # reversed blue colour axis + +# Configure physics +param.gravity = 1.0 +param.forcing = False +param.noslip = False +param.diffusion = EP["diffusion"] +param.Kdiff = EP["Kdiff"] * param.Lx / param.nx + +# Initialize geometry +grid = Grid(param) +xr, yr = grid.xr, grid.yr + +LAND = 0 +AQUA = 1 + +# Add linear sloping shore +m = EP["slope"] +t = EP["height"] - EP["x_start"] * m +grid.msk[(yr <= m*xr + t) & (yr < EP["height"])] = LAND +grid.finalize_msk() + +# Create model +f2d = Fluid2d(param, grid) +model = f2d.model + +# Set initial perturbation of the surface (or interface) +buoy = model.var.get("buoyancy") +buoy[:] = 1 +if EP["perturbation"].lower() == "sin": + # Use a sinusoidal perturbation + buoy[ + (yr < EP["intensity"] * np.sin(2 * np.pi * xr/EP["sigma"]) + 1.0) + & (xr < EP["x_start"]) + ] = 0 +elif EP["perturbation"].lower() == "gauss": + # Use a Gaussian perturbation + buoy[ + (yr < EP["intensity"] * np.exp(-(xr/EP["sigma"])**2) + 1.0) + & (xr < EP["x_start"]) + ] = 0 +else: + raise ValueError("unknown type of perturbation: {}.".format(EP["perturbation"])) +buoy *= grid.msk + +# Start simulation +f2d.loop() + +# Make it a complete database entry +ems.finalize(f2d) From 22563419933816ced6ab00e889f366d6d9081fc5 Mon Sep 17 00:00:00 2001 From: Markus Date: Fri, 17 May 2019 18:13:46 +0200 Subject: [PATCH 02/31] Make EMS module safe against SQL injection. --- core/ems.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/core/ems.py b/core/ems.py index c73b577..44f9758 100644 --- a/core/ems.py +++ b/core/ems.py @@ -114,22 +114,22 @@ def __init__(self, param: "param.Param", experiment_file: str=""): else: # Create a new table print(' Creating new table "{}".'.format(self.exp_class)) - param_string = ", ".join(['"{}" {}'.format(n, t) for t, n, v in param_list]) - sql_command = 'CREATE TABLE "{}" ({})'.format(self.exp_class, param_string) + column_string = ", ".join(['"{}" {}'.format(n, t) for t, n, v in param_list]) + sql_command = 'CREATE TABLE "{}" ({})'.format(self.exp_class, column_string) cursor.execute(sql_command) - # TODO: make string formating safe against SQL injection, - # possibly by using the "?"-format # First entry has index 1 (one) self.id_ = 1 # Set id in the parameter list param_list[0][2] = self.id_ # Add a new entry to the table print(' Adding new entry #{} to table "{}".'.format(self.id_, self.exp_class)) - value_string = ", ".join(['"{}"'.format(v) for t, n, v in param_list]) - sql_command = 'INSERT INTO "{}" VALUES ({})'.format(self.exp_class, value_string) - cursor.execute(sql_command) - # TODO: make string formating safe against SQL injection, - # possibly by using the "?"-format + value_list = [v for t, n, v in param_list] + sql_command = ( + 'INSERT INTO "{}" VALUES ('.format(self.exp_class) + + ', '.join(['?'] * len(value_list)) + + ')' + ) + cursor.execute(sql_command, value_list) # Save the database self.connection.commit() # Set the name of the experiment @@ -254,7 +254,7 @@ def parse_experiment_file(path: str): The name is written in a line starting with "Name:". It must be a valid string to be used as a filename; in particular, it must not - contain the slash "/". + contain the slash (/) or the quotation mark ("). The description begins in or after a line starting with "Description:" and goes until the beginning of the parameters @@ -265,8 +265,8 @@ def parse_experiment_file(path: str): datatype, the name and the value of the parameter seperated by one or several whitespaces. The datatype must be one of "int", "float", "bool" or "str". - The name must not contain whitespace characters and must not be in - the list of reserved column names. + The name must not contain whitespace characters or quotation marks + and must not be in the list of reserved column names. The value must be a valid value for the given datatype. If the value is omitted, it defaults to zero for numbers, True for booleans and the empty string for strings. The values "True" and "False" can @@ -298,6 +298,8 @@ def parse_experiment_file(path: str): elif line.lower().startswith("name:"): if not name: name = line[5:] + if '"' in name: + raise ExpFileError('name must not contain the symbol ".') else: raise ExpFileError( "name defined more than once in file {}.".format(path) @@ -376,6 +378,11 @@ def parse_experiment_file(path: str): 'reserved name used for parameter "{}" in file {}.' .format(param_name, path) ) + if '"' in param_name: + raise ExpFileError( + 'name of parameter "{}" must not contain the symbol ".' + .format(param_name) + ) param_list.append([sql_type, param_name, param_value]) else: raise ExpFileError( From 969c5390f5d7f9fe518223f6424035b2a57eb747 Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 20 May 2019 19:21:46 +0200 Subject: [PATCH 03/31] Add a simple Command-Line-Interface for the EMS The development of this tool is not yet finished, but already works and provides some useful functionality: - looking at the database of experiments - opening his/diag/mp4 files for specific experiments --- experiments/Experiment-Manager.py | 340 ++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 experiments/Experiment-Manager.py diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py new file mode 100644 index 0000000..dbefdb2 --- /dev/null +++ b/experiments/Experiment-Manager.py @@ -0,0 +1,340 @@ +# Markus Reinert, May 2019 +# +# Command-line interface for the fluid2d Experiment Management System (EMS). +# +# This programme can be started directly with Python if fluid2d is activated. +# Otherwise, the path to the folder with the experiment-files (that is the +# value of param.datadir) must be specified as a command-line argument. + +import os +import sys +import cmd +import sqlite3 as dbsys + + +MP4_PLAYER = "mplayer" + + +class EMDBConnection: + """Experiment Management Database Connection""" + + def __init__(self, dbpath: str): + # Create a connection to the given database + print("-"*50) + print("Opening database {}.".format(dbpath)) + self.connection = dbsys.connect(dbpath) + cursor = self.connection.cursor() + + # Get all tables of the database + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + # This gives a list like that: [('table1',), ('table2',)] + self.tables = [EMDBTable(cursor, t[0]) for t in cursor.fetchall()] + + def show_table_overview(self, title=True): + if title: + print("Experiments in database:") + for table in self.tables: + print( + " - {}: {} experiments, {} columns" + .format(table.name, table.get_length(), len(table.columns)) + ) + + def show_full_tables(self): + print("-"*50) + for table in self.tables: + print(table) + print("-"*50) + + def show_table(self, name): + for table in self.tables: + if table.name == name: + print("-"*50) + print(table) + print("-"*50) + return True + return False + + +class EMDBTable: + """Experiment Management Database Table""" + + def __init__(self, cursor: dbsys.Cursor, name: str): + self.name = str(name) + self.c = cursor + # Get columns + self.c.execute('PRAGMA table_info("{}")'.format(self.name)) + self.columns = [(column[1], column[2]) for column in self.c.fetchall()] + # column[0]: index from 0, column[1]: name, column[2]: type + + def __str__(self): + COMMENT_MAX_LENGTH = 15 + LIMITER = " " + CUT_DATE = False + # If CUT_DATE is True, CUT_MONTH is ignored + CUT_MONTH = True + # If CUT_MONTH is True, CUT_YEAR is ignored + CUT_YEAR = False + CUT_SECONDS = True + # Get entries + self.c.execute('SELECT * from "{}"'.format(self.name)) + rows = [list(row) for row in self.c.fetchall()] + lengths = [len(c[0]) if c[0] != "comment" else COMMENT_MAX_LENGTH + for c in self.columns] + table_size = 0 + for row in rows: + for i, val in enumerate(row): + # Check name of column + if self.columns[i][0] == "comment": + # Cut comments that are too long and end them with an ellipsis + if len(val) > COMMENT_MAX_LENGTH: + row[i] = val[:COMMENT_MAX_LENGTH-1] + "…" + # No need to determine length of value + continue + elif self.columns[i][0] == "datetime": + if CUT_DATE: + val = val.split("T")[1] + elif CUT_MONTH: + val = val[8:] + elif CUT_YEAR: + val = val[5:] + if CUT_SECONDS: + val = val[:-10] + val = val.replace("T", ",") + row[i] = val + elif self.columns[i][0] == "size_total": + table_size += val + + # Check type of column + if self.columns[i][1] == "TEXT" or self.columns[i][1] == "INTEGER": + lengths[i] = max(len(str(val)), lengths[i]) + elif self.columns[i][1] == "REAL": + lengths[i] = max(len("{:.3f}".format(val)), lengths[i]) + else: + raise Exception( + "unknown type {} of column {} in table {}." + .format(*self.columns[i], self.name) + ) + text = "Experiment: " + self.name + " ({:.3f} MB)\n".format(table_size) + text_cols = [] + for i, (n, t) in enumerate(self.columns): + text_cols.append(("{:^" + str(lengths[i]) + "}").format(n)) + text += LIMITER.join(text_cols) + "\n" + format_strings = [] + for i, l in enumerate(lengths): + if self.columns[i][1] == "REAL": + format_strings.append("{:^" + str(l) + ".3f}") + elif self.columns[i][0] == "comment": + format_strings.append("{:" + str(l) + "}") + else: + format_strings.append("{:^" + str(l) + "}") + for row in rows: + text_cols = [f_str.format(val) for f_str, val in zip(format_strings, row)] + text += LIMITER.join(text_cols) + "\n" + return text.strip() + + def get_length(self): + self.c.execute('SELECT Count(*) FROM "{}"'.format(self.name)) + return self.c.fetchone()[0] + + def entry_exists(self, id_): + self.c.execute('SELECT id FROM "{}" WHERE id = ?'.format(self.name), (id_,)) + if self.c.fetchone() is not None: + return True + return False + + +# https://docs.python.org/3/library/cmd.html +class EMShell(cmd.Cmd): + """Experiment Management (System) Shell""" + + intro = ( + "-" * 50 + + "\nFluid2d Experiment Management System (EMS)\n" + + "-" * 50 + + "\nType help or ? to list available commands." + + "\nType exit or Ctrl+D or Ctrl+C to exit." + + "\nPress Tab-key for auto-completion of commands or table names.\n" + ) + prompt = "(EMS) " + + def __init__(self, experiments_dir: str): + super().__init__() + self.exp_dir = experiments_dir + self.con = EMDBConnection(os.path.join(self.exp_dir, "experiments.db")) + self.selected_table = "" + + ### Functionality to SHOW the content of the database + def do_show(self, table_name): + """Show the content of a table. + + If no table name is specified or selected, the content of every table is shown.""" + if table_name: + # Try to open the specified table. + if not self.con.show_table(table_name): + # If it fails, print a message. + print('Unknown experiment: "{}"'.format(name)) + else: + if self.selected_table: + self.con.show_table(self.selected_table) + else: + # No table name given and no table selected + self.con.show_full_tables() + + def complete_show(self, text, line, begidx, endidx): + return self.table_name_completion(text) + + def help_show(self): + print( + "Show all information in the database about a class of experiments.\n" + "If no experiment is specified and no experiment is selected, " + "information about all the experiments is shown." + ) + + ### Functionality to OPEN experiment files + def do_open_mp4(self, params): + """Open the mp4-file for an experiment specified by its name and ID.""" + expname = self.parse_params_to_experiment(params) + if not expname: + return + dir_ = os.path.join(self.exp_dir, expname) + for f in os.listdir(dir_): + if f.endswith(".mp4") and f.startswith(expname): + break + else: + print("No mp4-file found in folder:", dir_) + path = os.path.join(self.exp_dir, expname, f) + os.system('{} "{}" &'.format(MP4_PLAYER, path)) + + def complete_open_mp4(self, text, line, begidx, endidx): + return self.table_name_completion(text) + + def do_open_his(self, params): + """Open the his-file for an experiment specified by its name and ID.""" + expname = self.parse_params_to_experiment(params) + if not expname: + return + path = os.path.join(self.exp_dir, expname, expname + "_his.nc") + if not os.path.isfile(path): + print("File does not exist:", path) + return + os.system('ncview "{}" &'.format(path)) + + def complete_open_his(self, text, line, begidx, endidx): + return self.table_name_completion(text) + + def do_open_diag(self, params): + """Open the diag-file for an experiment specified by its name and ID.""" + expname = self.parse_params_to_experiment(params) + if not expname: + return + path = os.path.join(self.exp_dir, expname, expname + "_diag.nc") + if not os.path.isfile(path): + print("File does not exist:", path) + return + os.system('ncview "{}" &'.format(path)) + + def complete_open_diag(self, text, line, begidx, endidx): + return self.table_name_completion(text) + + ### Functionality to SELECT a specific table + def do_select(self, params): + if params == "": + self.prompt = "(EMS) " + self.selected_table = params + elif self.check_table_exists(params): + self.prompt = "({}) ".format(params) + self.selected_table = params + else: + print('Unknown experiment: "{}"'.format(params)) + + def complete_select(self, text, line, begidx, endidx): + return self.table_name_completion(text) + + def help_select(self): + print( + 'Select an experiment by specifing its name.\n' + 'When an experiment is selected, every operation is automatically ' + 'executed for this experiment, except if specified differently.\n' + 'To unselect again, use the "select"-command with no argument or press Ctrl+D.' + ) + + ### Functionality to QUIT the program + def do_exit(self, params): + return True + + def do_EOF(self, params): + print("") + if self.selected_table: + self.prompt = "(EMS) " + self.selected_table = "" + else: + return True + + ### Behaviour for empty input + def emptyline(self): + self.con.show_table_overview() + + ### Helper functions + def table_name_completion(self, text): + completions = [] + for table in self.con.tables: + if table.name.startswith(text): + completions.append(table.name) + return completions + + def check_table_exists(self, name): + return name in [table.name for table in self.con.tables] + + def parse_params_to_experiment(self, params): + params = params.split(" ") + # Check for correct parameter input + if len(params) < 2: + if self.selected_table: + if len(params) == 1: + name = self.selected_table + else: + print("Exactly 1 ID must be specified.") + return + else: + print("Name and ID of experiment must be specified.") + return + else: + # Extract name from input + name = " ".join(params[:-1]).strip() + # Check name + for table in self.con.tables: + if table.name == name: + break + else: + print('Unknown experiment: "{}"'.format(name)) + return + # Extract and check ID + try: + id_ = int(params[-1]) + except ValueError: + print('Last argument "{}" is not a valid ID.'.format(params[-1])) + return + if not table.entry_exists(id_): + print("No entry exists in table {} with ID {}.".format(name, id_)) + return + # Return name of experiment folder + return "{}_{:03}".format(name, id_) + + +if len(sys.argv) == 1: + try: + from param import Param + except ModuleNotFoundError: + raise Exception( + "When fluid2d is not available, this programme has to be started " + "with the experiments-folder as argument." + ) + param = Param(None) # it is not necessary to specify a defaultfile for Param + datadir = param.datadir + if datadir.startswith("~"): + datadir = os.path.expanduser(param.datadir) + del param +else: + datadir = sys.argv[1] +ems_cli = EMShell(datadir) +ems_cli.cmdloop() From 15bb33b1ea0e7d29400861c4212a2c69575701fb Mon Sep 17 00:00:00 2001 From: Markus Date: Tue, 21 May 2019 11:14:41 +0200 Subject: [PATCH 04/31] EMShell: Improve file opening and output messages Shell of the Experiment-Manager has a new command "verbose" to configure whether one wants to see the messages printed by external programmes or not. --- experiments/Experiment-Manager.py | 98 ++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 14 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index dbefdb2..a77fbb4 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -10,9 +10,13 @@ import sys import cmd import sqlite3 as dbsys +import subprocess +# Command to open mp4-files MP4_PLAYER = "mplayer" +# Command to open NetCDF (his or diag) files +NETCDF_VIEWER = "ncview" class EMDBConnection: @@ -30,14 +34,22 @@ def __init__(self, dbpath: str): # This gives a list like that: [('table1',), ('table2',)] self.tables = [EMDBTable(cursor, t[0]) for t in cursor.fetchall()] - def show_table_overview(self, title=True): - if title: - print("Experiments in database:") - for table in self.tables: - print( - " - {}: {} experiments, {} columns" - .format(table.name, table.get_length(), len(table.columns)) - ) + def __del__(self): + print("Closing database.") + print("-"*50) + self.connection.close() + + def get_table_overview(self): + if self.tables: + text = "Experiments in database:" + for table in self.tables: + text += ( + "\n - {}: {} experiments, {} columns" + .format(table.name, table.get_length(), len(table.columns)) + ) + else: + text = "No experiments in database." + return text def show_full_tables(self): print("-"*50) @@ -149,8 +161,6 @@ class EMShell(cmd.Cmd): intro = ( "-" * 50 - + "\nFluid2d Experiment Management System (EMS)\n" - + "-" * 50 + "\nType help or ? to list available commands." + "\nType exit or Ctrl+D or Ctrl+C to exit." + "\nPress Tab-key for auto-completion of commands or table names.\n" @@ -159,11 +169,45 @@ class EMShell(cmd.Cmd): def __init__(self, experiments_dir: str): super().__init__() + print("-"*50) + print("Fluid2d Experiment Management System (EMS)") self.exp_dir = experiments_dir self.con = EMDBConnection(os.path.join(self.exp_dir, "experiments.db")) + self.intro += "\n" + self.con.get_table_overview() + "\n" self.selected_table = "" + self.silent_mode = True + + ### Functionality to MODIFY how the programme acts + def do_verbose(self, params): + if params == "" or params.lower() == "on": + self.silent_mode = False + elif params.lower() == "off": + self.silent_mode = True + else: + print('Unknown parameter. Please use "verbose on" or "verbose off".') + + def complete_verbose(self, text, line, begidx, endidx): + if text == "" or text == "o": + return ["on", "off"] + if text == "of": + return ["off",] + + def help_verbose(self): + print( + "Toggle between verbose- and silent-mode.\n" + 'Use "verbose on" or "verbose" to see the output of external programmes ' + 'started from this shell.\n' + 'Use "verbose off" to hide all output of external programmes (default).\n' + 'No error message is displayed in silent-mode when the opening of a file fails.\n' + 'In any case, external programmes are started in the background, so the shell can ' + 'still be used, even if the input prompt is polluted.' + ) ### Functionality to SHOW the content of the database + def do_list(self, params): + """List all experiment classes (tables) in the database.""" + print(self.con.get_table_overview()) + def do_show(self, table_name): """Show the content of a table. @@ -203,11 +247,18 @@ def do_open_mp4(self, params): else: print("No mp4-file found in folder:", dir_) path = os.path.join(self.exp_dir, expname, f) - os.system('{} "{}" &'.format(MP4_PLAYER, path)) + self.open_file(MP4_PLAYER, path) def complete_open_mp4(self, text, line, begidx, endidx): return self.table_name_completion(text) + def help_open_mp4(self): + print("Open the mp4-file for an experiment specified by its name and ID.\n" + "It is not possible to interact with the video player via input in the shell, " + "for example with mplayer.\n" + "User input via the graphical interface is not affected by this." + ) + def do_open_his(self, params): """Open the his-file for an experiment specified by its name and ID.""" expname = self.parse_params_to_experiment(params) @@ -217,7 +268,7 @@ def do_open_his(self, params): if not os.path.isfile(path): print("File does not exist:", path) return - os.system('ncview "{}" &'.format(path)) + self.open_file(NETCDF_VIEWER, path) def complete_open_his(self, text, line, begidx, endidx): return self.table_name_completion(text) @@ -231,7 +282,7 @@ def do_open_diag(self, params): if not os.path.isfile(path): print("File does not exist:", path) return - os.system('ncview "{}" &'.format(path)) + self.open_file(NETCDF_VIEWER, path) def complete_open_diag(self, text, line, begidx, endidx): return self.table_name_completion(text) @@ -272,7 +323,7 @@ def do_EOF(self, params): ### Behaviour for empty input def emptyline(self): - self.con.show_table_overview() + pass ### Helper functions def table_name_completion(self, text): @@ -320,6 +371,25 @@ def parse_params_to_experiment(self, params): # Return name of experiment folder return "{}_{:03}".format(name, id_) + def open_file(self, command, path): + if self.silent_mode: + print("Opening file {} with {} in silent-mode.".format(path, command)) + subprocess.Popen( + [command, path], + # Disable standard input via the shell, for example with mplayer. + stdin=subprocess.DEVNULL, + # Throw away output and error messages. + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + else: + print("Opening file {} with {} in verbose-mode.".format(path, command)) + subprocess.Popen( + [command, path], + # Disable standard input via the shell, for example with mplayer. + stdin=subprocess.DEVNULL, + ) + if len(sys.argv) == 1: try: From ebd6bc61159a12e69f4cc2147c5df4da5ca60e11 Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 22 May 2019 16:59:06 +0200 Subject: [PATCH 05/31] EMShell: Filter and sort experiments Also the way to enable and disable columns in the table view was greatly improved. --- experiments/Experiment-Manager.py | 379 +++++++++++++++++++++++++----- 1 file changed, 316 insertions(+), 63 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index a77fbb4..4a7bad7 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -18,6 +18,24 @@ # Command to open NetCDF (his or diag) files NETCDF_VIEWER = "ncview" +# Settings for the output of a table +COMMENT_MAX_LENGTH = 21 +FLOAT_FORMAT = "{:.4f}" +LIMITER = " " + +# Hide specific information in the table +# The values of the following variables can be modified +# with "enable" and "disable" during runtime. +HIDE_COMMENT = False +HIDE_MP4_SIZE = False +HIDE_HIS_SIZE = False +HIDE_DIAG_SIZE = True +HIDE_FLUX_SIZE = True +HIDE_TOTAL_SIZE = False +HIDE_DATETIME = False +# If HIDE_DATETIME is True, the following 2 have no effect +HIDE_YEAR = False +HIDE_SECONDS = True class EMDBConnection: """Experiment Management Database Connection""" @@ -66,6 +84,20 @@ def show_table(self, name): return True return False + def show_filtered_table(self, table_name, statement): + for table in self.tables: + if table.name == table_name or table_name == "": + print("-"*50) + table.print_selection(statement) + print("-"*50) + + def show_sorted_table(self, table_name, statement): + for table in self.tables: + if table.name == table_name or table_name == "": + print("-"*50) + table.print_sorted(statement) + print("-"*50) + class EMDBTable: """Experiment Management Database Table""" @@ -79,70 +111,9 @@ def __init__(self, cursor: dbsys.Cursor, name: str): # column[0]: index from 0, column[1]: name, column[2]: type def __str__(self): - COMMENT_MAX_LENGTH = 15 - LIMITER = " " - CUT_DATE = False - # If CUT_DATE is True, CUT_MONTH is ignored - CUT_MONTH = True - # If CUT_MONTH is True, CUT_YEAR is ignored - CUT_YEAR = False - CUT_SECONDS = True # Get entries self.c.execute('SELECT * from "{}"'.format(self.name)) - rows = [list(row) for row in self.c.fetchall()] - lengths = [len(c[0]) if c[0] != "comment" else COMMENT_MAX_LENGTH - for c in self.columns] - table_size = 0 - for row in rows: - for i, val in enumerate(row): - # Check name of column - if self.columns[i][0] == "comment": - # Cut comments that are too long and end them with an ellipsis - if len(val) > COMMENT_MAX_LENGTH: - row[i] = val[:COMMENT_MAX_LENGTH-1] + "…" - # No need to determine length of value - continue - elif self.columns[i][0] == "datetime": - if CUT_DATE: - val = val.split("T")[1] - elif CUT_MONTH: - val = val[8:] - elif CUT_YEAR: - val = val[5:] - if CUT_SECONDS: - val = val[:-10] - val = val.replace("T", ",") - row[i] = val - elif self.columns[i][0] == "size_total": - table_size += val - - # Check type of column - if self.columns[i][1] == "TEXT" or self.columns[i][1] == "INTEGER": - lengths[i] = max(len(str(val)), lengths[i]) - elif self.columns[i][1] == "REAL": - lengths[i] = max(len("{:.3f}".format(val)), lengths[i]) - else: - raise Exception( - "unknown type {} of column {} in table {}." - .format(*self.columns[i], self.name) - ) - text = "Experiment: " + self.name + " ({:.3f} MB)\n".format(table_size) - text_cols = [] - for i, (n, t) in enumerate(self.columns): - text_cols.append(("{:^" + str(lengths[i]) + "}").format(n)) - text += LIMITER.join(text_cols) + "\n" - format_strings = [] - for i, l in enumerate(lengths): - if self.columns[i][1] == "REAL": - format_strings.append("{:^" + str(l) + ".3f}") - elif self.columns[i][0] == "comment": - format_strings.append("{:" + str(l) + "}") - else: - format_strings.append("{:^" + str(l) + "}") - for row in rows: - text_cols = [f_str.format(val) for f_str, val in zip(format_strings, row)] - text += LIMITER.join(text_cols) + "\n" - return text.strip() + return string_format_table(self.c.fetchall(), self.columns, self.name) def get_length(self): self.c.execute('SELECT Count(*) FROM "{}"'.format(self.name)) @@ -154,6 +125,24 @@ def entry_exists(self, id_): return True return False + def print_selection(self, statement): + try: + self.c.execute('SELECT * FROM "{}" WHERE {}' + .format(self.name, statement)) + except dbsys.OperationalError as e: + print('SQL error for experiment "{}":'.format(self.name), e) + else: + print(string_format_table(self.c.fetchall(), self.columns, self.name)) + + def print_sorted(self, statement): + try: + self.c.execute('SELECT * FROM "{}" ORDER BY {}' + .format(self.name, statement)) + except dbsys.OperationalError as e: + print('SQL error for experiment "{}":'.format(self.name), e) + else: + print(string_format_table(self.c.fetchall(), self.columns, self.name)) + # https://docs.python.org/3/library/cmd.html class EMShell(cmd.Cmd): @@ -163,7 +152,7 @@ class EMShell(cmd.Cmd): "-" * 50 + "\nType help or ? to list available commands." + "\nType exit or Ctrl+D or Ctrl+C to exit." - + "\nPress Tab-key for auto-completion of commands or table names.\n" + + "\nPress Tab-key for auto-completion of commands, table names or column names." ) prompt = "(EMS) " @@ -203,6 +192,125 @@ def help_verbose(self): 'still be used, even if the input prompt is polluted.' ) + def do_enable(self, params): + global HIDE_COMMENT + global HIDE_MP4_SIZE, HIDE_HIS_SIZE, HIDE_DIAG_SIZE, HIDE_FLUX_SIZE, HIDE_TOTAL_SIZE + global HIDE_DATETIME, HIDE_YEAR, HIDE_SECONDS + + params = params.lower() + + if params == "all": + HIDE_COMMENT = False + HIDE_MP4_SIZE = False + HIDE_HIS_SIZE = False + HIDE_DIAG_SIZE = False + HIDE_FLUX_SIZE = False + HIDE_TOTAL_SIZE = False + HIDE_DATETIME = False + HIDE_YEAR = False + HIDE_SECONDS = False + elif params == "size": + HIDE_MP4_SIZE = False + HIDE_HIS_SIZE = False + HIDE_DIAG_SIZE = False + HIDE_FLUX_SIZE = False + HIDE_TOTAL_SIZE = False + elif params == "size_mp4": + HIDE_MP4_SIZE = False + elif params == "size_his": + HIDE_HIS_SIZE = False + elif params == "size_diag": + HIDE_DIAG_SIZE = False + elif params == "size_flux": + HIDE_FLUX_SIZE = False + elif params == "size_total": + HIDE_TOTAL_SIZE = False + elif params == "datetime": + HIDE_DATETIME = False + elif params == "year": + HIDE_YEAR = False + elif params == "seconds": + HIDE_SECONDS = False + elif params == "comment": + HIDE_COMMENT = False + else: + print("Unknown argument.") + + def complete_enable(self, text, line, begidx, endidx): + parameters = [ + 'comment', + 'size', + 'size_mp4', + 'size_his', + 'size_diag', + 'size_flux', + 'size_total', + 'datetime', + 'year', + 'seconds', + ] + completions = [] + for p in parameters: + if p.startswith(text): + completions.append(p) + return completions + + def help_enable(self): + print( + 'Show hidden information in the experiment table.\n' + 'Use "enable all" to show all available information.\n' + 'See the help of "disable" for further explanations.' + ) + + def do_disable(self, params): + global HIDE_COMMENT + global HIDE_MP4_SIZE, HIDE_HIS_SIZE, HIDE_DIAG_SIZE, HIDE_FLUX_SIZE, HIDE_TOTAL_SIZE + global HIDE_DATETIME, HIDE_YEAR, HIDE_SECONDS + + params = params.lower() + + if params == "size": + HIDE_MP4_SIZE = True + HIDE_HIS_SIZE = True + HIDE_DIAG_SIZE = True + HIDE_FLUX_SIZE = True + HIDE_TOTAL_SIZE = True + elif params == "size_mp4": + HIDE_MP4_SIZE = True + elif params == "size_his": + HIDE_HIS_SIZE = True + elif params == "size_diag": + HIDE_DIAG_SIZE = True + elif params == "size_flux": + HIDE_FLUX_SIZE = True + elif params == "size_total": + HIDE_TOTAL_SIZE = True + elif params == "datetime": + HIDE_DATETIME = True + elif params == "year": + HIDE_YEAR = True + elif params == "seconds": + HIDE_SECONDS = True + if params == "comment": + HIDE_COMMENT = True + else: + print("Unknown argument.") + + def complete_disable(self, text, line, begidx, endidx): + return self.complete_enable(text, line, begidx, endidx) + + def help_disable(self): + print( + 'Hide unnecessary information in the experiment table.\n' + 'The following parameters can be hidden:\n' + ' - datetime, size_total, size_mp4, size_his, size_diag, size_flux, comment\n' + 'Furthermore, the shorthand "size" can be used to disable at once ' + 'size_total, size_mp4, size_his, size_diag and size_flux.\n' + 'When datetime is not disabled, the commands "disable year" and "disable seconds" ' + 'can be used to make the datetime column slimmer.\n' + 'To show the parameters again, use "enable".' + ) + ### Functionality to SHOW the content of the database def do_list(self, params): """List all experiment classes (tables) in the database.""" @@ -234,6 +342,48 @@ def help_show(self): "information about all the experiments is shown." ) + def do_filter(self, params): + self.con.show_filtered_table(self.selected_table, params) + + def complete_filter(self, text, line, begidx, endidx): + return self.column_name_completion(self.selected_table, text) + + def help_filter(self): + print( + 'Filter experiments by the given value of a given parameter.\n' + 'Any valid SQLite WHERE-statement can be used as a filter, for example:\n' + ' - filter intensity <= 0.2\n' + ' - filter slope = 0.5\n' + ' - filter diffusion = "True"\n' + ' - filter perturbation = "gauss" AND duration > 20\n' + 'It is necessary to put the value for string or boolean arguments in quotation ' + 'marks like this: "True", "False", "Some text".\n' + 'Quotation marks are also necessary if the name of the parameter contains ' + 'whitespace.\n' + 'To sort the experiments, use "sort".\n' + 'To filter and sort the experiments, use SQLite syntax.\n' + 'Examples:\n' + ' - filter intensity > 0.1 ORDER BY intensity\n' + ' - filter perturbation != "gauss" ORDER BY size_total DESC' + ) + + def do_sort(self, params): + self.con.show_sorted_table(self.selected_table, params) + + def complete_sort(self, text, line, begidx, endidx): + return self.column_name_completion(self.selected_table, text) + + def help_sort(self): + print( + 'Sort the experiments by the value of a given parameter.\n' + 'To invert the order, add "desc" (descending) to the command.\n' + 'Example usages:\n' + ' - sort intensity: show experiments with lowest intensity on top of the table.\n' + ' - sort size_total desc: show experiments with biggest file size on top.\n' + 'Quotation marks are necessary if the name of the parameter contains whitespace.\n' + 'To filter and sort the experiments, see the help of "filter".' + ) + ### Functionality to OPEN experiment files def do_open_mp4(self, params): """Open the mp4-file for an experiment specified by its name and ID.""" @@ -333,6 +483,15 @@ def table_name_completion(self, text): completions.append(table.name) return completions + def column_name_completion(self, table_name, text): + completions = [] + for table in self.con.tables: + if table.name == table_name or table_name == "": + for column_name, column_type in table.columns: + if column_name.startswith(text): + completions.append(column_name) + return completions + def check_table_exists(self, name): return name in [table.name for table in self.con.tables] @@ -391,6 +550,100 @@ def open_file(self, command, path): ) +def string_format_table(rows, columns, table_name): + # Convert rows to list of lists (since fetchall() returns a list of tuples) + rows = [list(row) for row in rows] + columns = columns.copy() + # Remove ignored columns + remove_indices = [] + for i in range(len(columns)): + if ( + (HIDE_COMMENT and columns[i][0] == "comment") or + (HIDE_DATETIME and columns[i][0] == "datetime") or + (HIDE_MP4_SIZE and columns[i][0] == "size_mp4") or + (HIDE_HIS_SIZE and columns[i][0] == "size_his") or + (HIDE_DIAG_SIZE and columns[i][0] == "size_diag") or + (HIDE_FLUX_SIZE and columns[i][0] == "size_flux") or + (HIDE_TOTAL_SIZE and columns[i][0] == "size_total") + ): + remove_indices.append(i) + for i in sorted(remove_indices, reverse=True): + columns.pop(i) + for row in rows: + row.pop(i) + # To calculate the total size of the files associated with the table + table_size = 0 + # Get necessary length for each column and process table + lengths = [len(n) for n, t in columns] + for row in rows: + for i, val in enumerate(row): + # Check name of column + if columns[i][0] == "comment": + # Cut comments which are too long and end them with an ellipsis + if len(val) > COMMENT_MAX_LENGTH: + val = val[:COMMENT_MAX_LENGTH-1] + "…" + row[i] = val + elif columns[i][0] == "datetime": + # Cut unnecessary parts of the date and time + if HIDE_YEAR: + val = val[5:] + if HIDE_SECONDS: + val = val[:-10] + row[i] = val.replace('T', ',') + elif columns[i][0] == "size_total": + if val > 0: + table_size += val + + # Check type of column and adopt + if columns[i][1] == "TEXT" or columns[i][1] == "INTEGER": + lengths[i] = max(lengths[i], len(str(val))) + elif columns[i][0] in ["size_diag", "size_flux", "size_his", + "size_mp4", "size_total"]: + lengths[i] = max(lengths[i], len("{:.3f}".format(val))) + elif columns[i][1] == "REAL": + lengths[i] = max(lengths[i], len(FLOAT_FORMAT.format(val))) + else: + # This is an unexpected situation, which probably means that + # sqlite3 does not work as it was, when this script was written. + raise Exception( + "unknown type {} of column {} in table {}." + .format(*columns[i], table_name) + ) + # Create top line of the text to be returned + text = "Experiment: " + table_name + if not HIDE_TOTAL_SIZE: + text += " ({:.3f} MB)".format(table_size) + if len(remove_indices) == 1: + text += " (1 parameter hidden)" + elif len(remove_indices) > 1: + text += " ({} parameters hidden)".format(len(remove_indices)) + text += "\n" + # Add column name + text += LIMITER.join([ + ("{:^" + str(l) + "}").format(n) + for (n, t), l in zip(columns, lengths) + ]) + "\n" + format_strings = [] + for (n, t), l in zip(columns, lengths): + # Numbers right justified, + # comments left justified, + # text centered + if n in ["size_diag", "size_flux", "size_his", "size_mp4", "size_total"]: + format_strings.append("{:>" + str(l) + ".3f}") + elif t == "REAL": + format_strings.append(FLOAT_FORMAT.replace(":", ":>"+str(l))) + elif t == "INTEGER": + format_strings.append("{:>" + str(l) + "}") + elif n == "comment": + format_strings.append("{:" + str(l) + "}") + else: + format_strings.append("{:^" + str(l) + "}") + for row in rows: + text_cols = [f_str.format(val) for f_str, val in zip(format_strings, row)] + text += LIMITER.join(text_cols) + "\n" + return text.strip() + + if len(sys.argv) == 1: try: from param import Param From 90c54127f231c7960f0b68b3f2bfbac2343fdc47 Mon Sep 17 00:00:00 2001 From: Markus Date: Thu, 23 May 2019 11:42:33 +0200 Subject: [PATCH 06/31] EMShell: Improve the hiding of columns New internal behaviour is implemented that allows to hide all the columns of the experiment table in the EMS Interface. With this, the help-text of the functions for this was updated. --- experiments/Experiment-Manager.py | 219 ++++++++++++------------------ 1 file changed, 88 insertions(+), 131 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 4a7bad7..fb34849 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -23,19 +23,14 @@ FLOAT_FORMAT = "{:.4f}" LIMITER = " " -# Hide specific information in the table -# The values of the following variables can be modified -# with "enable" and "disable" during runtime. -HIDE_COMMENT = False -HIDE_MP4_SIZE = False -HIDE_HIS_SIZE = False -HIDE_DIAG_SIZE = True -HIDE_FLUX_SIZE = True -HIDE_TOTAL_SIZE = False -HIDE_DATETIME = False -# If HIDE_DATETIME is True, the following 2 have no effect -HIDE_YEAR = False -HIDE_SECONDS = True +# Hide the following information in the table +# They can be activated with "enable" during runtime. +# More information can be hidden with "disable". +hidden_information = { + 'size_diag', + 'size_flux', + 'datetime_seconds', +} class EMDBConnection: """Experiment Management Database Connection""" @@ -98,6 +93,13 @@ def show_sorted_table(self, table_name, statement): table.print_sorted(statement) print("-"*50) + def is_valid_column(self, name): + for table in self.tables: + for n, t in table.columns: + if name == n: + return True + return False + class EMDBTable: """Experiment Management Database Table""" @@ -113,7 +115,7 @@ def __init__(self, cursor: dbsys.Cursor, name: str): def __str__(self): # Get entries self.c.execute('SELECT * from "{}"'.format(self.name)) - return string_format_table(self.c.fetchall(), self.columns, self.name) + return string_format_table(self.name, self.columns, self.c.fetchall()) def get_length(self): self.c.execute('SELECT Count(*) FROM "{}"'.format(self.name)) @@ -132,7 +134,7 @@ def print_selection(self, statement): except dbsys.OperationalError as e: print('SQL error for experiment "{}":'.format(self.name), e) else: - print(string_format_table(self.c.fetchall(), self.columns, self.name)) + print(string_format_table(self.name, self.columns, self.c.fetchall())) def print_sorted(self, statement): try: @@ -141,7 +143,7 @@ def print_sorted(self, statement): except dbsys.OperationalError as e: print('SQL error for experiment "{}":'.format(self.name), e) else: - print(string_format_table(self.c.fetchall(), self.columns, self.name)) + print(string_format_table(self.name, self.columns, self.c.fetchall())) # https://docs.python.org/3/library/cmd.html @@ -153,6 +155,7 @@ class EMShell(cmd.Cmd): + "\nType help or ? to list available commands." + "\nType exit or Ctrl+D or Ctrl+C to exit." + "\nPress Tab-key for auto-completion of commands, table names or column names." + + "\n" ) prompt = "(EMS) " @@ -193,62 +196,25 @@ def help_verbose(self): ) def do_enable(self, params): - global HIDE_COMMENT - global HIDE_MP4_SIZE, HIDE_HIS_SIZE, HIDE_DIAG_SIZE, HIDE_FLUX_SIZE, HIDE_TOTAL_SIZE - global HIDE_DATETIME, HIDE_YEAR, HIDE_SECONDS - - params = params.lower() - + global hidden_information if params == "all": - HIDE_COMMENT = False - HIDE_MP4_SIZE = False - HIDE_HIS_SIZE = False - HIDE_DIAG_SIZE = False - HIDE_FLUX_SIZE = False - HIDE_TOTAL_SIZE = False - HIDE_DATETIME = False - HIDE_YEAR = False - HIDE_SECONDS = False - elif params == "size": - HIDE_MP4_SIZE = False - HIDE_HIS_SIZE = False - HIDE_DIAG_SIZE = False - HIDE_FLUX_SIZE = False - HIDE_TOTAL_SIZE = False - elif params == "size_mp4": - HIDE_MP4_SIZE = False - elif params == "size_his": - HIDE_HIS_SIZE = False - elif params == "size_diag": - HIDE_DIAG_SIZE = False - elif params == "size_flux": - HIDE_FLUX_SIZE = False - elif params == "size_total": - HIDE_TOTAL_SIZE = False - elif params == "datetime": - HIDE_DATETIME = False - elif params == "year": - HIDE_YEAR = False - elif params == "seconds": - HIDE_SECONDS = False - elif params == "comment": - HIDE_COMMENT = False + hidden_information.clear() else: - print("Unknown argument.") + for param in params.split(): + if param == "size": + hidden_information.difference_update( + {'size_diag', 'size_flux', 'size_his', 'size_mp4', 'size_total'} + ) + else: + hidden_information.discard(param) def complete_enable(self, text, line, begidx, endidx): - parameters = [ - 'comment', - 'size', - 'size_mp4', - 'size_his', - 'size_diag', - 'size_flux', - 'size_total', - 'datetime', - 'year', - 'seconds', - ] + parameters = hidden_information.copy() + if len(parameters) > 0: + parameters.add('all') + if not parameters.isdisjoint({'size_diag', 'size_flux', 'size_his', + 'size_mp4', 'size_total'}): + parameters.add('size') completions = [] for p in parameters: if p.startswith(text): @@ -257,58 +223,57 @@ def complete_enable(self, text, line, begidx, endidx): def help_enable(self): print( - 'Show hidden information in the experiment table.\n' - 'Use "enable all" to show all available information.\n' + 'Make hidden information in the experiment table visible.\n' 'See the help of "disable" for further explanations.' ) def do_disable(self, params): - global HIDE_COMMENT - global HIDE_MP4_SIZE, HIDE_HIS_SIZE, HIDE_DIAG_SIZE, HIDE_FLUX_SIZE, HIDE_TOTAL_SIZE - global HIDE_DATETIME, HIDE_YEAR, HIDE_SECONDS - - params = params.lower() - - if params == "size": - HIDE_MP4_SIZE = True - HIDE_HIS_SIZE = True - HIDE_DIAG_SIZE = True - HIDE_FLUX_SIZE = True - HIDE_TOTAL_SIZE = True - elif params == "size_mp4": - HIDE_MP4_SIZE = True - elif params == "size_his": - HIDE_HIS_SIZE = True - elif params == "size_diag": - HIDE_DIAG_SIZE = True - elif params == "size_flux": - HIDE_FLUX_SIZE = True - elif params == "size_total": - HIDE_TOTAL_SIZE = True - elif params == "datetime": - HIDE_DATETIME = True - elif params == "year": - HIDE_YEAR = True - elif params == "seconds": - HIDE_SECONDS = True - if params == "comment": - HIDE_COMMENT = True - else: - print("Unknown argument.") + global hidden_information + for param in params.split(): + if param == "size": + # Short notation for the union of two sets + hidden_information |= {'size_diag', 'size_flux', 'size_his', + 'size_mp4', 'size_total'} + elif (param == "datetime_year" + or param == "datetime_seconds" + or self.con.is_valid_column(param) + ): + hidden_information.add(param) + else: + print("Unknown argument:", param) def complete_disable(self, text, line, begidx, endidx): - return self.complete_enable(text, line, begidx, endidx) + parameters = [ + 'datetime_year', + 'datetime_seconds', + ] + # If not all size columns are hidden, add shorthand for all sizes + if not hidden_information.issuperset({'size_diag', 'size_flux', 'size_his', + 'size_mp4', 'size_total'}): + parameters += ['size'] + # Add columns of selected table to the list of auto-completions + for table in self.con.tables: + if self.selected_table == "" or self.selected_table == table.name: + parameters += [n for n,t in table.columns] + completions = [] + for p in parameters: + if p.startswith(text) and p not in hidden_information: + completions.append(p) + return completions def help_disable(self): print( - 'Hide unnecessary information in the experiment table.\n' - 'The following parameters can be hidden:\n' - ' - datetime, size_total, size_mp4, size_his, size_diag, size_flux, comment\n' - 'Furthermore, the shorthand "size" can be used to disable at once ' - 'size_total, size_mp4, size_his, size_diag and size_flux.\n' - 'When datetime is not disabled, the commands "disable year" and "disable seconds" ' - 'can be used to make the datetime column slimmer.\n' - 'To show the parameters again, use "enable".' + 'Hide unneeded information in the experiment table.\n' + 'Every parameter/column can be hidden, for example:\n' + ' - disable comment\n' + 'Furthermore, the year and number of seconds can be hidden from the datetime ' + 'column by using "disable datetime_year" and "disable datetime_seconds".\n' + 'Multiple parameters can be specified at once, for example:\n' + ' - disable size_his size_diag size_flux\n' + 'To hide all file sizes, use "disable size".\n' + 'To show hidden parameters again, use "enable".\n' + 'In addition to the behaviour described here, it also supports the command ' + '"enable all".' ) ### Functionality to SHOW the content of the database @@ -550,24 +515,16 @@ def open_file(self, command, path): ) -def string_format_table(rows, columns, table_name): +def string_format_table(table_name, columns, rows): # Convert rows to list of lists (since fetchall() returns a list of tuples) rows = [list(row) for row in rows] columns = columns.copy() # Remove ignored columns - remove_indices = [] - for i in range(len(columns)): - if ( - (HIDE_COMMENT and columns[i][0] == "comment") or - (HIDE_DATETIME and columns[i][0] == "datetime") or - (HIDE_MP4_SIZE and columns[i][0] == "size_mp4") or - (HIDE_HIS_SIZE and columns[i][0] == "size_his") or - (HIDE_DIAG_SIZE and columns[i][0] == "size_diag") or - (HIDE_FLUX_SIZE and columns[i][0] == "size_flux") or - (HIDE_TOTAL_SIZE and columns[i][0] == "size_total") - ): - remove_indices.append(i) - for i in sorted(remove_indices, reverse=True): + indices_to_remove = [] + for i, (n, t) in enumerate(columns): + if n in hidden_information: + indices_to_remove.append(i) + for i in sorted(indices_to_remove, reverse=True): columns.pop(i) for row in rows: row.pop(i) @@ -585,9 +542,9 @@ def string_format_table(rows, columns, table_name): row[i] = val elif columns[i][0] == "datetime": # Cut unnecessary parts of the date and time - if HIDE_YEAR: + if "datetime_year" in hidden_information: val = val[5:] - if HIDE_SECONDS: + if "datetime_seconds" in hidden_information: val = val[:-10] row[i] = val.replace('T', ',') elif columns[i][0] == "size_total": @@ -611,12 +568,12 @@ def string_format_table(rows, columns, table_name): ) # Create top line of the text to be returned text = "Experiment: " + table_name - if not HIDE_TOTAL_SIZE: + if "size_total" not in hidden_information: text += " ({:.3f} MB)".format(table_size) - if len(remove_indices) == 1: + if len(indices_to_remove) == 1: text += " (1 parameter hidden)" - elif len(remove_indices) > 1: - text += " ({} parameters hidden)".format(len(remove_indices)) + elif len(indices_to_remove) > 1: + text += " ({} parameters hidden)".format(len(indices_to_remove)) text += "\n" # Add column name text += LIMITER.join([ From c44f7fa30f5971f2a5188471aabd1556522955d1 Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 27 May 2019 16:39:46 +0200 Subject: [PATCH 07/31] EMShell: add command to remove entries Further changes: Add the switch -v to some commands to display all information in the database, even hidden parameters. Improve help messages for these commands. --- experiments/Experiment-Manager.py | 256 ++++++++++++++++++++++-------- 1 file changed, 190 insertions(+), 66 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index fb34849..3bfe464 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -9,6 +9,7 @@ import os import sys import cmd +import shutil import sqlite3 as dbsys import subprocess @@ -32,6 +33,10 @@ 'datetime_seconds', } +# Switch for temporarily showing all information, including full comments: +DISPLAY_ALL = False + + class EMDBConnection: """Experiment Management Database Connection""" @@ -52,6 +57,9 @@ def __del__(self): print("-"*50) self.connection.close() + def save_database(self): + self.connection.commit() + def get_table_overview(self): if self.tables: text = "Experiments in database:" @@ -64,20 +72,18 @@ def get_table_overview(self): text = "No experiments in database." return text - def show_full_tables(self): - print("-"*50) + def show_all_tables(self): for table in self.tables: - print(table) print("-"*50) + print(table) def show_table(self, name): + print("-"*50) for table in self.tables: if table.name == name: - print("-"*50) print(table) - print("-"*50) - return True - return False + return + print('Unknown experiment: "{}"'.format(name)) def show_filtered_table(self, table_name, statement): for table in self.tables: @@ -100,6 +106,17 @@ def is_valid_column(self, name): return True return False + def entry_exists(self, table_name, id_): + for table in self.tables: + if table.name == table_name: + return table.entry_exists(id_) + + def delete_entry(self, table_name, id_): + for table in self.tables: + if table.name == table_name: + table.delete_entry(id_) + return + class EMDBTable: """Experiment Management Database Table""" @@ -127,6 +144,9 @@ def entry_exists(self, id_): return True return False + def delete_entry(self, id_): + self.c.execute('DELETE FROM "{}" WHERE id = ?'.format(self.name), (id_,)) + def print_selection(self, statement): try: self.c.execute('SELECT * FROM "{}" WHERE {}' @@ -281,73 +301,99 @@ def do_list(self, params): """List all experiment classes (tables) in the database.""" print(self.con.get_table_overview()) - def do_show(self, table_name): - """Show the content of a table. - - If no table name is specified or selected, the content of every table is shown.""" - if table_name: - # Try to open the specified table. - if not self.con.show_table(table_name): - # If it fails, print a message. - print('Unknown experiment: "{}"'.format(name)) + def do_show(self, params): + """Show the content of a table.""" + global DISPLAY_ALL + params = set(params.split()) + if "-v" in params: + DISPLAY_ALL = True + params.remove('-v') else: + DISPLAY_ALL = False + if len(params) == 0: + # No table name given if self.selected_table: self.con.show_table(self.selected_table) else: - # No table name given and no table selected - self.con.show_full_tables() + self.con.show_all_tables() + else: + for table_name in sorted(params): + # Try to open the specified table. + self.con.show_table(table_name) + print("-"*50) def complete_show(self, text, line, begidx, endidx): return self.table_name_completion(text) def help_show(self): - print( - "Show all information in the database about a class of experiments.\n" - "If no experiment is specified and no experiment is selected, " - "information about all the experiments is shown." - ) + print("""> show [name(s) of experiment class(es)] [-v] + Show all entries in the database for one or more classes of experiments. + Specify as parameter the name of the experiment class to show. Multiple + names may be specified. If no name is specified, then the experiments of + the currently selected class are shown (cf. "select"). If no name name is + specified and no experiment selected, then all entries are shown. + Use the commands "enable" and "disable" to specify which information + (i.e., which column) is displayed. To display all information instead, use + "show" with the parameter "-v". + + See also: filter, sort.""") def do_filter(self, params): + global DISPLAY_ALL + if params.endswith(" -v"): + params = params[:-3] + DISPLAY_ALL = True + else: + DISPLAY_ALL = False self.con.show_filtered_table(self.selected_table, params) def complete_filter(self, text, line, begidx, endidx): return self.column_name_completion(self.selected_table, text) def help_filter(self): - print( - 'Filter experiments by the given value of a given parameter.\n' - 'Any valid SQLite WHERE-statement can be used as a filter, for example:\n' - ' - filter intensity <= 0.2\n' - ' - filter slope = 0.5\n' - ' - filter diffusion = "True"\n' - ' - filter perturbation = "gauss" AND duration > 20\n' - 'It is necessary to put the value for string or boolean arguments in quotation ' - 'marks like this: "True", "False", "Some text".\n' - 'Quotation marks are also necessary if the name of the parameter contains ' - 'whitespace.\n' - 'To sort the experiments, use "sort".\n' - 'To filter and sort the experiments, use SQLite syntax.\n' - 'Examples:\n' - ' - filter intensity > 0.1 ORDER BY intensity\n' - ' - filter perturbation != "gauss" ORDER BY size_total DESC' - ) + print("""> filter [condition] [-v] + Filter experiments of the currently selected class according to the given + condition. Any valid SQLite WHERE-statement can be used as a filter. + Examples: + - filter intensity <= 0.2 + - filter slope = 0.5 + - filter diffusion = "True" + - filter datetime >= "2019-03-2 + - filter perturbation != "gauss" AND duration > 20 + It is necessary to put the value for string, datetime or boolean argument + in quotation marks as shown. + To sort the filtered experiments, use SQLite syntax. + Examples: + - filter intensity > 0.1 ORDER BY intensity + - filter perturbation != "gauss" ORDER BY size_total DESC + The see all information about the filtered experiments, add "-v" at the end + of the command. + + See also: sort, show.""") def do_sort(self, params): + global DISPLAY_ALL + if params.endswith(" -v"): + params = params[:-3] + DISPLAY_ALL = True + else: + DISPLAY_ALL = False self.con.show_sorted_table(self.selected_table, params) def complete_sort(self, text, line, begidx, endidx): return self.column_name_completion(self.selected_table, text) def help_sort(self): - print( - 'Sort the experiments by the value of a given parameter.\n' - 'To invert the order, add "desc" (descending) to the command.\n' - 'Example usages:\n' - ' - sort intensity: show experiments with lowest intensity on top of the table.\n' - ' - sort size_total desc: show experiments with biggest file size on top.\n' - 'Quotation marks are necessary if the name of the parameter contains whitespace.\n' - 'To filter and sort the experiments, see the help of "filter".' - ) + print("""> sort [parameter] [desc] [-v] + Sort the experiments by the value of a given parameter. + To invert the order, add "desc" (descending) to the command. + Examples: + - sort intensity: show experiments with lowest intensity on top of the table. + - sort size_total desc: show experiments with biggest file size on top. + The see all information about the sorted experiments, add "-v" at the end + of the command. + + See also: filter, show.""") ### Functionality to OPEN experiment files def do_open_mp4(self, params): @@ -356,11 +402,17 @@ def do_open_mp4(self, params): if not expname: return dir_ = os.path.join(self.exp_dir, expname) - for f in os.listdir(dir_): + try: + files = os.listdir(dir_) + except FileNotFoundError: + print("Folder does not exist:", dir_) + return + for f in files: if f.endswith(".mp4") and f.startswith(expname): break else: - print("No mp4-file found in folder:", dir_) + print("No mp4-file found in folder:", dir_) + return path = os.path.join(self.exp_dir, expname, f) self.open_file(MP4_PLAYER, path) @@ -402,6 +454,77 @@ def do_open_diag(self, params): def complete_open_diag(self, text, line, begidx, endidx): return self.table_name_completion(text) + ### Functionality to CLEAN up + def do_remove(self, params): + global DISPLAY_ALL + DISPLAY_ALL = True + + # Check conditions and parameters + if not self.selected_table: + print('No table selected. Select a table to remove entries.') + return + if not params: + print('No ids given. Specify ids of the entries to remove.') + return + + # Parse parameters + ids = set() + for p in params.split(): + try: + id_ = int(p) + except ValueError: + print('Parameter is not a valid id: ' + p + '. No data removed.') + return + if not self.con.entry_exists(self.selected_table, id_): + print('No entry with id', id_, 'exists in the selected table. No data removed.') + return + ids.add(id_) + + # Print full information of selected entries + if len(ids) == 1: + statement = "id = {}".format(*ids) + else: + statement = "id IN {}".format(tuple(ids)) + print('WARNING: the following entries will be DELETED:') + self.con.show_filtered_table(self.selected_table, statement) + + # Print full information of related folders + folders = [] + print('WARNING: the following folders and files will be DELETED:') + for id_ in ids: + expname = "{}_{:03}".format(self.selected_table, id_) + folder = os.path.join(self.exp_dir, expname) + try: + files = os.listdir(folder) + except FileNotFoundError: + print(' x Folder does not exist:', folder) + else: + folders.append(folder) + print(" -", folder) + for f in sorted(files): + print(" -", f) + + # Final check + print('Do you really want to permanently delete these files, folders and entries?') + print('This cannot be undone.') + answer = input('Continue [yes/no] ? ') + if answer == 'yes': + # Remove entries + for id_ in ids: + self.con.delete_entry(self.selected_table, id_) + print('Deleted entry', id_, 'from experiment "{}".'.format(self.selected_table)) + self.con.save_database() + # Remove files + for folder in folders: + try: + shutil.rmtree(folder) + except OSError as e: + print('Error deleting folder {}:'.format(folder), e) + else: + print('Deleted folder {}.'.format(folder)) + else: + print('Answer was not "yes". No data removed.') + ### Functionality to SELECT a specific table def do_select(self, params): if params == "": @@ -521,13 +644,14 @@ def string_format_table(table_name, columns, rows): columns = columns.copy() # Remove ignored columns indices_to_remove = [] - for i, (n, t) in enumerate(columns): - if n in hidden_information: - indices_to_remove.append(i) - for i in sorted(indices_to_remove, reverse=True): - columns.pop(i) - for row in rows: - row.pop(i) + if not DISPLAY_ALL: + for i, (n, t) in enumerate(columns): + if n in hidden_information: + indices_to_remove.append(i) + for i in sorted(indices_to_remove, reverse=True): + columns.pop(i) + for row in rows: + row.pop(i) # To calculate the total size of the files associated with the table table_size = 0 # Get necessary length for each column and process table @@ -537,14 +661,14 @@ def string_format_table(table_name, columns, rows): # Check name of column if columns[i][0] == "comment": # Cut comments which are too long and end them with an ellipsis - if len(val) > COMMENT_MAX_LENGTH: + if len(val) > COMMENT_MAX_LENGTH and not DISPLAY_ALL: val = val[:COMMENT_MAX_LENGTH-1] + "…" row[i] = val elif columns[i][0] == "datetime": # Cut unnecessary parts of the date and time - if "datetime_year" in hidden_information: + if "datetime_year" in hidden_information and not DISPLAY_ALL: val = val[5:] - if "datetime_seconds" in hidden_information: + if "datetime_seconds" in hidden_information and not DISPLAY_ALL: val = val[:-10] row[i] = val.replace('T', ',') elif columns[i][0] == "size_total": @@ -568,23 +692,23 @@ def string_format_table(table_name, columns, rows): ) # Create top line of the text to be returned text = "Experiment: " + table_name - if "size_total" not in hidden_information: + if "size_total" not in hidden_information or DISPLAY_ALL: text += " ({:.3f} MB)".format(table_size) if len(indices_to_remove) == 1: text += " (1 parameter hidden)" elif len(indices_to_remove) > 1: text += " ({} parameters hidden)".format(len(indices_to_remove)) text += "\n" - # Add column name + # Add column name centred, except the last one text += LIMITER.join([ ("{:^" + str(l) + "}").format(n) - for (n, t), l in zip(columns, lengths) - ]) + "\n" + for (n, t), l in zip(columns[:-1], lengths[:-1]) + ] + [columns[-1][0]]) + "\n" format_strings = [] for (n, t), l in zip(columns, lengths): # Numbers right justified, # comments left justified, - # text centered + # text centred if n in ["size_diag", "size_flux", "size_his", "size_mp4", "size_total"]: format_strings.append("{:>" + str(l) + ".3f}") elif t == "REAL": From 3c8f42168f13b57a72393a3021428001b4136024 Mon Sep 17 00:00:00 2001 From: Markus Date: Thu, 30 May 2019 14:07:37 +0200 Subject: [PATCH 08/31] EMShell: auto-set optimal width for comments The last column in the table view of the EMShell, which displays the comment of an experiment, can now automatically adopt its width to the width available in the terminal window. Also the full comment man be displayed every time instead. The option to start the EMShell without fluid2d activated is removed. It did not fit well into the general behaviour of the fluid2d-system. New function is added to the EMShell to display free disk space. --- core/ems.py | 1 + experiments/Experiment-Manager.py | 167 +++++++++++++++++++++--------- 2 files changed, 117 insertions(+), 51 deletions(-) diff --git a/core/ems.py b/core/ems.py index 44f9758..7b0acae 100644 --- a/core/ems.py +++ b/core/ems.py @@ -148,6 +148,7 @@ def __init__(self, param: "param.Param", experiment_file: str=""): raise ParamError( 'param.expname is not a valid database entry: "{}"'.format(param.expname) ) + print(' Reading from entry #{} of table "{}".'.format(self.id_, self.exp_class)) # Remember directory of the experiment self.output_dir = os.path.join(datadir, param.expname) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 3bfe464..a168e1a 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -7,12 +7,18 @@ # value of param.datadir) must be specified as a command-line argument. import os -import sys import cmd import shutil import sqlite3 as dbsys import subprocess +try: + from param import Param +except ModuleNotFoundError: + raise Exception( + "Please activate fluid2d to use this programme!" + ) + # Command to open mp4-files MP4_PLAYER = "mplayer" @@ -20,13 +26,19 @@ NETCDF_VIEWER = "ncview" # Settings for the output of a table -COMMENT_MAX_LENGTH = 21 +# The COMMENT_MAX_LENGTH can be "AUTO", "FULL" or a positive number +COMMENT_MAX_LENGTH = "AUTO" FLOAT_FORMAT = "{:.4f}" +# Display sizes with 2 decimals, because when 3 decimals are used, +# the dot (or comma) can be mistaken for the separator after 1000. +SIZE_FORMAT = "{:.2f}" LIMITER = " " +# Symbol of the linebreak and its replacement in comments +LINEBREAK_REPLACE = ("\n", "|") # Hide the following information in the table -# They can be activated with "enable" during runtime. -# More information can be hidden with "disable". +# They can be activated with the command "enable" during runtime. +# More information can be hidden with the command "disable". hidden_information = { 'size_diag', 'size_flux', @@ -34,7 +46,7 @@ } # Switch for temporarily showing all information, including full comments: -DISPLAY_ALL = False +display_all = False class EMDBConnection: @@ -217,7 +229,9 @@ def help_verbose(self): def do_enable(self, params): global hidden_information - if params == "all": + if not params: + print("Please specify information to enable.") + elif params == "all": hidden_information.clear() else: for param in params.split(): @@ -249,6 +263,9 @@ def help_enable(self): def do_disable(self, params): global hidden_information + if not params: + print("Please specify information to disable.") + return for param in params.split(): if param == "size": # Short notation for the union of two sets @@ -303,13 +320,13 @@ def do_list(self, params): def do_show(self, params): """Show the content of a table.""" - global DISPLAY_ALL + global display_all params = set(params.split()) if "-v" in params: - DISPLAY_ALL = True + display_all = True params.remove('-v') else: - DISPLAY_ALL = False + display_all = False if len(params) == 0: # No table name given if self.selected_table: @@ -339,12 +356,12 @@ def help_show(self): See also: filter, sort.""") def do_filter(self, params): - global DISPLAY_ALL + global display_all if params.endswith(" -v"): params = params[:-3] - DISPLAY_ALL = True + display_all = True else: - DISPLAY_ALL = False + display_all = False self.con.show_filtered_table(self.selected_table, params) def complete_filter(self, text, line, begidx, endidx): @@ -372,12 +389,12 @@ def help_filter(self): See also: sort, show.""") def do_sort(self, params): - global DISPLAY_ALL + global display_all if params.endswith(" -v"): params = params[:-3] - DISPLAY_ALL = True + display_all = True else: - DISPLAY_ALL = False + display_all = False self.con.show_sorted_table(self.selected_table, params) def complete_sort(self, text, line, begidx, endidx): @@ -456,8 +473,8 @@ def complete_open_diag(self, text, line, begidx, endidx): ### Functionality to CLEAN up def do_remove(self, params): - global DISPLAY_ALL - DISPLAY_ALL = True + global display_all + display_all = True # Check conditions and parameters if not self.selected_table: @@ -525,6 +542,32 @@ def do_remove(self, params): else: print('Answer was not "yes". No data removed.') + def do_print_disk_info(self, params): + # Explanation: https://stackoverflow.com/a/12327880/3661532 + statvfs = os.statvfs(self.exp_dir) + available_space = statvfs.f_frsize * statvfs.f_bavail + print( + "Available disk space in the experiment directory:", + SIZE_FORMAT.format(available_space / 1024**3), + "GiB" + ) + print("Experiment directory:", self.exp_dir) + + def help_print_disk_info(self): + print("""> print_disk_info + Display the available disk space in the experiment directory and its path. + The available disk space is printed in Gibibytes (GiB), this means: + 1 GiB = 1024^3 bytes. + This is used instead of the Gigabyte (GB), which is defined as: + 1 GB = 1000^3 bytes, + because the GiB underestimates the free disk space, which is considered + safer than overestimating it. In contrast, the size used by experiments is + displayed in Megabytes (MB), where + 1 MB = 1000^2 bytes, + since the used space should rather be overestimated. + More information about the unit: https://en.wikipedia.org/wiki/Gibibyte .""" + ) + ### Functionality to SELECT a specific table def do_select(self, params): if params == "": @@ -639,12 +682,12 @@ def open_file(self, command, path): def string_format_table(table_name, columns, rows): - # Convert rows to list of lists (since fetchall() returns a list of tuples) + # Convert rows to list of lists (because fetchall() returns a list of tuples) rows = [list(row) for row in rows] - columns = columns.copy() # Remove ignored columns + columns = columns.copy() indices_to_remove = [] - if not DISPLAY_ALL: + if not display_all: for i, (n, t) in enumerate(columns): if n in hidden_information: indices_to_remove.append(i) @@ -652,35 +695,49 @@ def string_format_table(table_name, columns, rows): columns.pop(i) for row in rows: row.pop(i) - # To calculate the total size of the files associated with the table - table_size = 0 - # Get necessary length for each column and process table + # Get width necessary for each column, get total size of the files + # associated with the table and process the entries of the table. lengths = [len(n) for n, t in columns] + table_size = 0 for row in rows: for i, val in enumerate(row): # Check name of column if columns[i][0] == "comment": - # Cut comments which are too long and end them with an ellipsis - if len(val) > COMMENT_MAX_LENGTH and not DISPLAY_ALL: - val = val[:COMMENT_MAX_LENGTH-1] + "…" + if display_all or COMMENT_MAX_LENGTH == "FULL": + # No formatting needed + pass + else: + # Replace linebreaks + val = val.replace(*LINEBREAK_REPLACE) + if type(COMMENT_MAX_LENGTH) == int and COMMENT_MAX_LENGTH > 0: + # Cut comments which are too long and end them with an ellipsis + if len(val) > COMMENT_MAX_LENGTH: + val = val[:COMMENT_MAX_LENGTH-1] + "…" + elif COMMENT_MAX_LENGTH == "AUTO": + # Cut comments later + pass + else: + raise ValueError( + 'COMMENT_MAX_LENGTH has to be "AUTO" or "FULL" ' + 'or a positive integer, not "{}".'.format(COMMENT_MAX_LENGTH) + ) row[i] = val elif columns[i][0] == "datetime": # Cut unnecessary parts of the date and time - if "datetime_year" in hidden_information and not DISPLAY_ALL: + if "datetime_year" in hidden_information and not display_all: val = val[5:] - if "datetime_seconds" in hidden_information and not DISPLAY_ALL: + if "datetime_seconds" in hidden_information and not display_all: val = val[:-10] row[i] = val.replace('T', ',') elif columns[i][0] == "size_total": if val > 0: table_size += val - # Check type of column and adopt if columns[i][1] == "TEXT" or columns[i][1] == "INTEGER": lengths[i] = max(lengths[i], len(str(val))) elif columns[i][0] in ["size_diag", "size_flux", "size_his", "size_mp4", "size_total"]: - lengths[i] = max(lengths[i], len("{:.3f}".format(val))) + lengths[i] = max(lengths[i], len(SIZE_FORMAT.format(val))) elif columns[i][1] == "REAL": lengths[i] = max(lengths[i], len(FLOAT_FORMAT.format(val))) else: @@ -690,10 +747,25 @@ def string_format_table(table_name, columns, rows): "unknown type {} of column {} in table {}." .format(*columns[i], table_name) ) + if ( + COMMENT_MAX_LENGTH == "AUTO" + and "comment" not in hidden_information + and not display_all + ): + line_length = shutil.get_terminal_size().columns + total_length = sum(lengths[:-1]) + len(LIMITER) * len(lengths[:-1]) + comment_length = line_length - total_length % line_length + lengths[-1] = comment_length + for row in rows: + comment = row[-1] + # Cut comments which are too long and end them with an ellipsis + if len(comment) > comment_length: + comment = comment[:comment_length-1] + "…" + row[-1] = comment # Create top line of the text to be returned text = "Experiment: " + table_name - if "size_total" not in hidden_information or DISPLAY_ALL: - text += " ({:.3f} MB)".format(table_size) + if "size_total" not in hidden_information or display_all: + text += " (" + SIZE_FORMAT.format(table_size) + " MB)" if len(indices_to_remove) == 1: text += " (1 parameter hidden)" elif len(indices_to_remove) > 1: @@ -707,16 +779,16 @@ def string_format_table(table_name, columns, rows): format_strings = [] for (n, t), l in zip(columns, lengths): # Numbers right justified, - # comments left justified, - # text centred + # text centred, + # comments unformatted if n in ["size_diag", "size_flux", "size_his", "size_mp4", "size_total"]: - format_strings.append("{:>" + str(l) + ".3f}") + format_strings.append(SIZE_FORMAT.replace(":", ":>"+str(l))) elif t == "REAL": format_strings.append(FLOAT_FORMAT.replace(":", ":>"+str(l))) elif t == "INTEGER": format_strings.append("{:>" + str(l) + "}") elif n == "comment": - format_strings.append("{:" + str(l) + "}") + format_strings.append("{}") else: format_strings.append("{:^" + str(l) + "}") for row in rows: @@ -725,20 +797,13 @@ def string_format_table(table_name, columns, rows): return text.strip() -if len(sys.argv) == 1: - try: - from param import Param - except ModuleNotFoundError: - raise Exception( - "When fluid2d is not available, this programme has to be started " - "with the experiments-folder as argument." - ) - param = Param(None) # it is not necessary to specify a defaultfile for Param - datadir = param.datadir - if datadir.startswith("~"): - datadir = os.path.expanduser(param.datadir) - del param -else: - datadir = sys.argv[1] +# Get the directory of the experiments +param = Param(None) # it is not necessary to specify a defaultfile for Param +datadir = param.datadir +del param +if datadir.startswith("~"): + datadir = os.path.expanduser(datadir) + +# Start the shell ems_cli = EMShell(datadir) ems_cli.cmdloop() From 6b0288c07fac39a64af41e199d060bbf04a8c22a Mon Sep 17 00:00:00 2001 From: Markus Date: Thu, 30 May 2019 17:22:38 +0200 Subject: [PATCH 09/31] EMShell: print tables in colour optionally --- experiments/Experiment-Manager.py | 76 +++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index a168e1a..ec18b65 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -10,6 +10,7 @@ import cmd import shutil import sqlite3 as dbsys +import itertools import subprocess try: @@ -36,6 +37,39 @@ # Symbol of the linebreak and its replacement in comments LINEBREAK_REPLACE = ("\n", "|") +### Settings to print tables in colour +# Set COLOURS to False or None to disable colours. +# Otherwise, COLOURS must be list, of which each element describes the colours used in one row. +# The colours of every row are described by a list of colour codes, +# cf. https://en.wikipedia.org/wiki/ANSI_escape_code#Colors. +# An arbitrary number of colours can be specified. +# Apart from background colours, also text colours and font styles can be specified. +# Example to colour rows alternately with white and default colour: +# COLOURS = ( +# ("\033[47m",), +# ("\033[49m",), +# ) +# Example to colour columns alternately with white and default colour: +# COLOURS = ( +# ("\033[47m", "\033[49m",), +# ) +# Example for a check-pattern: +# COLOURS = ( +# ("\033[47m", "\033[49m",), +# ("\033[49m", "\033[47m",), +# ) +# Example for alternating background colours in rows and alternating font colours in columns: +# COLOURS = ( +# ("\033[31;47m", "\033[39;47m",), +# ("\033[31;49m", "\033[39;49m",), +# ) +COLOURS = ( + ("\033[107m",), + ("\033[49m",), +) +# Colour-code to reset to default colour and default font +COLOURS_END = "\033[39;49m" + # Hide the following information in the table # They can be activated with the command "enable" during runtime. # More information can be hidden with the command "disable". @@ -788,13 +822,23 @@ def string_format_table(table_name, columns, rows): elif t == "INTEGER": format_strings.append("{:>" + str(l) + "}") elif n == "comment": - format_strings.append("{}") + format_strings.append("{:" + str(l) + "}") else: format_strings.append("{:^" + str(l) + "}") + if COLOURS: + row_colours = itertools.cycle(COLOURS) for row in rows: - text_cols = [f_str.format(val) for f_str, val in zip(format_strings, row)] - text += LIMITER.join(text_cols) + "\n" - return text.strip() + if COLOURS: + col_colours = itertools.cycle(next(row_colours)) + text_cols = [next(col_colours) + f_str.format(val) if COLOURS + else f_str.format(val) for f_str, val in zip(format_strings, row)] + text += LIMITER.join(text_cols) + text += "\n" + if text.endswith("\n"): + text = text[:-1] + if COLOURS: + text = text + COLOURS_END + return text # Get the directory of the experiments @@ -804,6 +848,30 @@ def string_format_table(table_name, columns, rows): if datadir.startswith("~"): datadir = os.path.expanduser(datadir) +# Use fancy colours during 6 days of Carnival +try: + import datetime + from dateutil.easter import easter + date_today = datetime.date.today() + carnival_start = easter(date_today.year)-datetime.timedelta(days=46+6) # Weiberfastnacht + carnival_end = easter(date_today.year)-datetime.timedelta(days=46) # Aschermittwoch + if carnival_start <= date_today < carnival_end or COLOURS == "HAPPY": + print("It's carnival! Let's hope your terminal supports colours!") + # This looks like a rainbow on many terminals, e.g. xterm. + COLOURS = ( + ("\033[41m",), + ("\033[101m",), + ("\033[43m",), + ("\033[103m",), + ("\033[102m",), + ("\033[106m",), + ("\033[104m",), + ("\033[105m",), + ) +except: + # At least we tried! + pass + # Start the shell ems_cli = EMShell(datadir) ems_cli.cmdloop() From a468795e02a2671e7c86f06e7e757ba441d01e18 Mon Sep 17 00:00:00 2001 From: Markus Date: Sun, 2 Jun 2019 19:38:14 +0200 Subject: [PATCH 10/31] EMShell: print datetime in an easy-to-read format A new setting is added to the EMShell which, when activated, prints the value of the datetime column in a format which is easily readable by humans, for example by saying "15 min ago" instead of the exact date and time. The ems was slightly modified to prevent unexpected behaviour in the rare case that datetime.now() returns a time with 0 microseconds. --- core/ems.py | 5 +- experiments/Experiment-Manager.py | 101 ++++++++++++++++++++++++++---- 2 files changed, 92 insertions(+), 14 deletions(-) diff --git a/core/ems.py b/core/ems.py index 7b0acae..feedef6 100644 --- a/core/ems.py +++ b/core/ems.py @@ -78,7 +78,8 @@ def __init__(self, param: "param.Param", experiment_file: str=""): param_list = ( [ ["INTEGER", "id", -1], - ["TEXT", "datetime", datetime.datetime.now().isoformat()], + ["TEXT", "datetime", + datetime.datetime.now().isoformat(timespec="microseconds")], ] + param_list + [ @@ -238,7 +239,7 @@ def finalize(self, fluid2d): ) self.connection.execute( 'UPDATE "{}" SET datetime = ? WHERE id = ?'.format(self.exp_class), - (datetime.datetime.now().isoformat(), self.id_) + (datetime.datetime.now().isoformat(timespec="microseconds"), self.id_) ) # Save the database diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index ec18b65..3bb6d5c 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -10,6 +10,7 @@ import cmd import shutil import sqlite3 as dbsys +import datetime import itertools import subprocess @@ -26,16 +27,25 @@ # Command to open NetCDF (his or diag) files NETCDF_VIEWER = "ncview" -# Settings for the output of a table -# The COMMENT_MAX_LENGTH can be "AUTO", "FULL" or a positive number +### Settings for the output of a table +# Maximal length of comments (possible values: "AUTO", "FULL" or a positive integer) +# This setting is ignored if display_all is True. COMMENT_MAX_LENGTH = "AUTO" +# Format-string for real numbers FLOAT_FORMAT = "{:.4f}" -# Display sizes with 2 decimals, because when 3 decimals are used, -# the dot (or comma) can be mistaken for the separator after 1000. +# Format-string for file sizes (MB) or disk space (GiB) +# Two decimals are used instead of three, to avoid confusion between the dot as +# a decimal separator and a delimiter after 1000. SIZE_FORMAT = "{:.2f}" +# Symbol to separate two columns of the table LIMITER = " " # Symbol of the linebreak and its replacement in comments +# This is ignored if display_all is True or COMMENT_MAX_LENGTH is "FULL". LINEBREAK_REPLACE = ("\n", "|") +# Show date and time in ISO-format or easy-to-read-format +# The easy-to-read-format does not include seconds, independent whether seconds +# are hidden or not. This setting is ignored if display_all is True. +ISO_DATETIME = False ### Settings to print tables in colour # Set COLOURS to False or None to disable colours. @@ -70,6 +80,35 @@ # Colour-code to reset to default colour and default font COLOURS_END = "\033[39;49m" + +### Language settings (currently only for dates in easy-to-read-format) +# Definition of languages +class English: + JUST_NOW = "just now" + AGO_MINUTES = "{} min ago" + AGO_HOURS = "{}:{:02} h ago" + YESTERDAY = "yesterday" + FUTURE = "in the future" + +class French: + JUST_NOW = "maintenant" + AGO_MINUTES = "il y a {} min" + AGO_HOURS = "il y a {}h{:02}" + YESTERDAY = "hier" + FUTURE = "à l'avenir" + +class German: + JUST_NOW = "gerade eben" + AGO_MINUTES = "vor {} Min." + AGO_HOURS = "vor {}:{:02} Std." + YESTERDAY = "gestern" + FUTURE = "in der Zukunft" + +# Selection of a language +LANG = English + + +### Global variables modifiable during runtime # Hide the following information in the table # They can be activated with the command "enable" during runtime. # More information can be hidden with the command "disable". @@ -79,7 +118,8 @@ 'datetime_seconds', } -# Switch for temporarily showing all information, including full comments: +# Temporarily show all information, including full comments +# This is set to True by the argument "-v" to commands which print tables. display_all = False @@ -716,6 +756,9 @@ def open_file(self, command, path): def string_format_table(table_name, columns, rows): + # Get current date and time to make datetime easy to read + if not ISO_DATETIME: + dt_now = datetime.datetime.now() # Convert rows to list of lists (because fetchall() returns a list of tuples) rows = [list(row) for row in rows] # Remove ignored columns @@ -757,12 +800,20 @@ def string_format_table(table_name, columns, rows): ) row[i] = val elif columns[i][0] == "datetime": - # Cut unnecessary parts of the date and time - if "datetime_year" in hidden_information and not display_all: - val = val[5:] - if "datetime_seconds" in hidden_information and not display_all: - val = val[:-10] - row[i] = val.replace('T', ',') + if display_all: + # No formatting needed + pass + elif ISO_DATETIME: + # Cut unnecessary parts of the date and time + if "datetime_year" in hidden_information: + val = val[5:] + if "datetime_seconds" in hidden_information: + val = val[:-10] + row[i] = val.replace('T', ',') + else: + # Create datetime-object from ISO-format + dt_obj = datetime.datetime.strptime(val, "%Y-%m-%dT%H:%M:%S.%f") + row[i] = make_nice_time_string(dt_obj, dt_now) elif columns[i][0] == "size_total": if val > 0: table_size += val @@ -841,6 +892,33 @@ def string_format_table(table_name, columns, rows): return text +def make_nice_time_string(datetime_object, datetime_reference): + """Create an easy to read representation of the datetime object.""" + if datetime_object > datetime_reference: + return LANG.FUTURE + elif datetime_object.date() == datetime_reference.date(): + # same day + dt_diff = datetime_reference - datetime_object + dt_diff_minutes = dt_diff.seconds / 60 + if dt_diff_minutes < 1: + return LANG.JUST_NOW + elif dt_diff_minutes < 59.5: + return LANG.AGO_MINUTES.format(round(dt_diff_minutes)) + else: + return LANG.AGO_HOURS.format(int(dt_diff_minutes / 60), int(dt_diff_minutes % 60)) + elif datetime_object.date() == (datetime_reference - datetime.timedelta(days=1)).date(): + # yesterday + return datetime_object.strftime(LANG.YESTERDAY + ", %H:%M") + elif datetime_object.date() > (datetime_reference - datetime.timedelta(days=7)).date(): + # this week, i.e., within the last six days + return datetime_object.strftime("%a, %H:%M") + else: + format_string = "%b-%d, %H:%M" + if "datetime_year" not in hidden_information: + format_string = "%Y-" + format_string + return datetime_object.strftime(format_string) + + # Get the directory of the experiments param = Param(None) # it is not necessary to specify a defaultfile for Param datadir = param.datadir @@ -850,7 +928,6 @@ def string_format_table(table_name, columns, rows): # Use fancy colours during 6 days of Carnival try: - import datetime from dateutil.easter import easter date_today = datetime.date.today() carnival_start = easter(date_today.year)-datetime.timedelta(days=46+6) # Weiberfastnacht From 8cd39b123411c032e0d16ddcdfa0c58ed089dbbb Mon Sep 17 00:00:00 2001 From: Markus Date: Sun, 2 Jun 2019 21:06:45 +0200 Subject: [PATCH 11/31] EMShell: fix bug in calculation of column width --- experiments/Experiment-Manager.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 3bb6d5c..b4926ee 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -255,7 +255,7 @@ def print_sorted(self, statement): # https://docs.python.org/3/library/cmd.html class EMShell(cmd.Cmd): """Experiment Management (System) Shell""" - + intro = ( "-" * 50 + "\nType help or ? to list available commands." @@ -809,11 +809,12 @@ def string_format_table(table_name, columns, rows): val = val[5:] if "datetime_seconds" in hidden_information: val = val[:-10] - row[i] = val.replace('T', ',') + val = val.replace('T', ',') else: # Create datetime-object from ISO-format dt_obj = datetime.datetime.strptime(val, "%Y-%m-%dT%H:%M:%S.%f") - row[i] = make_nice_time_string(dt_obj, dt_now) + val = make_nice_time_string(dt_obj, dt_now) + row[i] = val elif columns[i][0] == "size_total": if val > 0: table_size += val @@ -934,7 +935,7 @@ def make_nice_time_string(datetime_object, datetime_reference): carnival_end = easter(date_today.year)-datetime.timedelta(days=46) # Aschermittwoch if carnival_start <= date_today < carnival_end or COLOURS == "HAPPY": print("It's carnival! Let's hope your terminal supports colours!") - # This looks like a rainbow on many terminals, e.g. xterm. + # This looks like a rainbow on many terminals, e.g. xterm. COLOURS = ( ("\033[41m",), ("\033[101m",), From 66b49647fc72ccc00fcedf411853b7ee1f34b674 Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 5 Jun 2019 19:33:59 +0200 Subject: [PATCH 12/31] EMShell: add extensions and plots --- experiments/EMShell_Extensions.py | 31 ++ experiments/Experiment-Manager.py | 716 ++++++++++++++++++++++++++++-- 2 files changed, 715 insertions(+), 32 deletions(-) create mode 100644 experiments/EMShell_Extensions.py diff --git a/experiments/EMShell_Extensions.py b/experiments/EMShell_Extensions.py new file mode 100644 index 0000000..f4947f1 --- /dev/null +++ b/experiments/EMShell_Extensions.py @@ -0,0 +1,31 @@ +# Markus Reinert, June 2019 +# +# Extensions for the EMShell, the command-line interface of the fluid2d Experiment Management System (EMS) + +import numpy as np +import netCDF4 as nc +from scipy.fftpack import fft, fftshift, fftfreq + + +def get_strongest_wavenumber(his_filename): + # Open data file + dataset_his = nc.Dataset(his_filename) + # Load data + ny = dataset_his.ny + dy = dataset_his.Ly / ny + # Save the data as masked numpy arrays + x = dataset_his["x"][:] + psi = dataset_his["psi"][:] + psi.mask = 1 - dataset_his["msk"][:] + # Set length of zero-padded signal + fft_ny = ny + # Caculate zero-padded Fourier-transform in y + fft_psi = fftshift(fft(psi, n=fft_ny, axis=1), axes=1) + # Its sampling frequency is dky = 1/dy/fft_ny + # Calculate the corresponding axis in Fourier-space + ky = fftshift(fftfreq(fft_ny, dy)) + # Remove the zero-frequency because it is always very large + fft_psi[:, ky==0] = 0 + # Calculate the frequency of maximal intensity (apart from the zero frequency) + ky_max = np.abs(ky)[np.argmax(np.max(np.abs(fft_psi), axis=2), axis=1)] + return np.max(ky_max) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index b4926ee..2e00eee 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -1,6 +1,6 @@ # Markus Reinert, May 2019 # -# Command-line interface for the fluid2d Experiment Management System (EMS). +# Command-line interface for the fluid2d Experiment Management System (EMS) # # This programme can be started directly with Python if fluid2d is activated. # Otherwise, the path to the folder with the experiment-files (that is the @@ -8,11 +8,16 @@ import os import cmd +import time +import numpy as np import shutil import sqlite3 as dbsys import datetime import itertools import subprocess +import matplotlib.pyplot as plt + +import EMShell_Extensions as EMExt try: from param import Param @@ -27,6 +32,17 @@ # Command to open NetCDF (his or diag) files NETCDF_VIEWER = "ncview" +### Extensions +# Make new shell extensions available by adding them to this dictionary. +# The key is the name under which the function is called in the shell, +# which must not contain any whitespace. +# The value is the function, which takes as only argument the path to the +# his-file of an experiment and can return the calculated value. +extra_tools = { + "wavenumber": EMExt.get_strongest_wavenumber, + "wavelength": lambda hisname: 1/EMExt.get_strongest_wavenumber(hisname), +} + ### Settings for the output of a table # Maximal length of comments (possible values: "AUTO", "FULL" or a positive integer) # This setting is ignored if display_all is True. @@ -158,6 +174,18 @@ def get_table_overview(self): text = "No experiments in database." return text + def get_data(self, table_name, column_name, condition=""): + for table in self.tables: + if table_name == table.name: + return table.get_data(column_name, condition) + print(f'No table with name {table_name} found.') + return [] + + def get_column_names(self, table_name): + for table in self.tables: + if table.name == table_name: + return [n for n,t in table.columns] + def show_all_tables(self): for table in self.tables: print("-"*50) @@ -185,11 +213,12 @@ def show_sorted_table(self, table_name, statement): table.print_sorted(statement) print("-"*50) - def is_valid_column(self, name): + def is_valid_column(self, column_name, table_name=""): for table in self.tables: - for n, t in table.columns: - if name == n: - return True + if not table_name or table_name == table.name: + for n, t in table.columns: + if column_name == n: + return True return False def entry_exists(self, table_name, id_): @@ -224,6 +253,16 @@ def get_length(self): self.c.execute('SELECT Count(*) FROM "{}"'.format(self.name)) return self.c.fetchone()[0] + def get_data(self, column_name, condition=""): + try: + self.c.execute(f'SELECT {column_name} FROM "{self.name}" WHERE {condition}' if condition else + f'SELECT {column_name} FROM "{self.name}"') + except dbsys.OperationalError as e: + print(f'SQL error for experiment "{self.name}":', e) + return [] + else: + return [e[0] for e in self.c.fetchall()] + def entry_exists(self, id_): self.c.execute('SELECT id FROM "{}" WHERE id = ?'.format(self.name), (id_,)) if self.c.fetchone() is not None: @@ -260,7 +299,7 @@ class EMShell(cmd.Cmd): "-" * 50 + "\nType help or ? to list available commands." + "\nType exit or Ctrl+D or Ctrl+C to exit." - + "\nPress Tab-key for auto-completion of commands, table names or column names." + + "\nPress Tab-key for auto-completion of commands, table names, etc." + "\n" ) prompt = "(EMS) " @@ -289,6 +328,7 @@ def complete_verbose(self, text, line, begidx, endidx): return ["on", "off"] if text == "of": return ["off",] + return [] def help_verbose(self): print( @@ -323,11 +363,7 @@ def complete_enable(self, text, line, begidx, endidx): if not parameters.isdisjoint({'size_diag', 'size_flux', 'size_his', 'size_mp4', 'size_total'}): parameters.add('size') - completions = [] - for p in parameters: - if p.startswith(text): - completions.append(p) - return completions + return [p for p in parameters if p.startswith(text)] def help_enable(self): print( @@ -366,11 +402,7 @@ def complete_disable(self, text, line, begidx, endidx): for table in self.con.tables: if self.selected_table == "" or self.selected_table == table.name: parameters += [n for n,t in table.columns] - completions = [] - for p in parameters: - if p.startswith(text) and p not in hidden_information: - completions.append(p) - return completions + return [p for p in parameters if p.startswith(text) and p not in hidden_information] def help_disable(self): print( @@ -436,13 +468,16 @@ def do_filter(self, params): display_all = True else: display_all = False - self.con.show_filtered_table(self.selected_table, params) + if not params: + print('No condition to filter given. Type "help filter" for further information.') + else: + self.con.show_filtered_table(self.selected_table, params) def complete_filter(self, text, line, begidx, endidx): return self.column_name_completion(self.selected_table, text) def help_filter(self): - print("""> filter [condition] [-v] + print("""> filter [-v] Filter experiments of the currently selected class according to the given condition. Any valid SQLite WHERE-statement can be used as a filter. Examples: @@ -469,13 +504,16 @@ def do_sort(self, params): display_all = True else: display_all = False - self.con.show_sorted_table(self.selected_table, params) + if not params: + print('No parameter to sort given. Type "help sort" for further information.') + else: + self.con.show_sorted_table(self.selected_table, params) def complete_sort(self, text, line, begidx, endidx): return self.column_name_completion(self.selected_table, text) def help_sort(self): - print("""> sort [parameter] [desc] [-v] + print("""> sort [desc] [-v] Sort the experiments by the value of a given parameter. To invert the order, add "desc" (descending) to the command. Examples: @@ -545,6 +583,592 @@ def do_open_diag(self, params): def complete_open_diag(self, text, line, begidx, endidx): return self.table_name_completion(text) + ### Functionality for Data Analysis + def do_calculate(self, params): + if not self.selected_table: + print('No experiment selected. Select an experiment to do calculations on it.') + return + try: + toolname, id_ = params.split() + except ValueError: + print('Invalid syntax. The correct syntax is: calculate . ' + 'Make sure the name of the tool contains no space.') + return + if not extra_tools: + print('There are no tools for calculations loaded. ' + 'Add some tools in the code and restart the programme.') + return + if toolname not in extra_tools: + print(f'Unknown tool: "{toolname}". The available tools are:') + for t in extra_tools: + print(" -", t) + return + try: + id_ = int(id_) + except ValueError: + print(f'Second parameter is not a valid id: "{id_}".') + return + if not self.con.entry_exists(self.selected_table, id_): + print(f'No entry with the id {id_} exists for the selected experiment.') + return + expname = "{}_{:03}".format(self.selected_table, id_) + path = os.path.join(self.exp_dir, expname, expname + '_his.nc') + try: + val = extra_tools[toolname](path) + except Exception as e: + print(f'Tool "{toolname}" did not succeed on experiment {id_}. ' + 'The error message is:') + print(e) + return + if val: + print(f'-> {toolname}({id_}) = {val}') + else: + print(f'-> {toolname}({id_}) succeeded without return value.') + + def complete_calculate(self, text, line, begidx, endidx): + return [p for p in extra_tools if p.startswith(text)] + + def help_calculate(self): + print("""> calculate + Call an EMShell-Extension on an experiment and print the result. + This works always on the entries of the currently selected experiment. + To make a tool available, load it into the Python script of the + Experiment-Manager and add it to the dictionary "extra_tools". + Its name must not contain any whitespace.""") + + def do_plot(self, params): + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to plot its data.') + return + # Parse the arguments + param_list = params.split(" ") + # Two parameters are necessary + if len(param_list) < 2: + print('No two parameters given. Type "help plot" for further information.') + return + param1 = param_list[0] + param_list.pop(0) + param2 = param_list[0] + param_list.pop(0) + # The last parameters can be used to modify the plot + save_as = None + draw_grid = False + format_string = "" + for param in reversed(param_list): + if param == "-png": + save_as = "PNG" + param_list.pop() + elif param == "-pdf": + save_as = "PDF" + param_list.pop() + elif param == "-grid": + draw_grid = True + param_list.pop() + elif param.startswith("-f="): + format_string = param[3:] + param_list.pop() + else: + # End of extra parameters, beginning of the SQL statement + break + # Every other parameter is an SQL condition to filter the data + condition = " ".join(param_list).strip() + # Get the data belonging to the parameters + data1 = self.get_data(self.selected_table, param1, condition) + if not data1: + print(f'Get data for first parameter "{param1}" ', end="") + if condition: + print(f'under condition "{condition}" ', end="") + print('failed.') + return + data2 = self.get_data(self.selected_table, param2, condition) + if not data2: + print(f'Get data for second parameter "{param2}" ', end="") + if condition: + print(f'under condition "{condition}" ', end="") + print('failed.') + return + + print("-> x:", data1) + print("-> y:", data2) + plt.xlabel(param1) + plt.ylabel(param2) + if draw_grid: + plt.grid() + try: + plt.plot( + data1, data2, format_string, + label=f'{self.selected_table} ({condition if condition else "all data"})', + ) + except Exception as e: + print("Plot did not succeed. Error message:") + print(e) + else: + plt.legend() + if save_as == "PDF": + figid = 0 + filename = f"figure_{figid}.pdf" + while os.path.exists(filename): + figid += 1 + filename = f"figure_{figid}.pdf" + print(f'Saving plot as "{filename}".') + plt.savefig(filename) + elif save_as == "PNG": + figid = 0 + filename = f"figure_{figid}.png" + while os.path.exists(filename): + figid += 1 + filename = f"figure_{figid}.png" + print(f'Saving plot as "{filename}".') + plt.savefig(filename) + plt.show() + + def complete_plot(self, text, line, begidx, endidx): + if not self.selected_table: + return [] + parameters = ["f=", "grid", "png", "pdf"] + parameters += self.con.get_column_names(self.selected_table) + parameters.extend(extra_tools.keys()) + return [p for p in parameters if p.startswith(text)] + + def help_plot(self): + print("""> plot [condition] [-f=] [-grid] [-{png|pdf}] + Make a diagram showing the relation between the two parameters for the + selected experiment. + + The parameters can be read from the database or calculated using shell- + extensions. Type "help calculate" for further information on extensions. + + A condition can be used to filter or sort the data. Type "help filter" for + further information on filtering and sorting. + + To specify the type of plot, use "-f=", where is a format + string for matplotlib like "-f=o" to plot with dots instead of a line, or + "-f=rx" to plot with red crosses. For further information on format strings + in matplotlib, see the Notes section on the following website: + https://matplotlib.org/3.1.0/api/_as_gen/matplotlib.pyplot.plot.html + If multiple format strings are specified, only the first one is considered. + + To draw a grid on the plot, add "-grid" to the command. + + Add either "-png" or "-pdf" to the command to save the figure in a file of + the corresponding filetype. If both are specified, only the first argument + is taken into account. + + The order of the qualifiers starting with a dash ("-") does not play a role, + but they must be at the end of the command. + + While the window with the plot is open, the shell can be used as usual. + If another plot command is executed with the figure still open, the new plot + is superimposed onto the previous one. Matplotlib automatically selects a + new colour if no colour is specified explicitely. If the -grid argument is + specified a second time in another plot command, the grid is switched off. + + Example: + - plot size_his duration id <= 10 -f=g.- -grid + This command creates a plot in green with dots and lines on a grid + comparing the integration time of the first ten experiments in the + selected class of experiments with the size of their history-file.""") + + def do_scatter(self, params): + # Check the run condition + if not self.selected_table: + print('No experiment selected. ' + 'Select an experiment to make a scatter plot of its data.') + return + # Parse the arguments + param_list = params.split(" ") + # Two parameters are necessary + if len(param_list) < 3: + print('No three parameters given. Type "help scatter" for further information.') + return + parameters = [] + for i in range(3): + parameters.append(param_list.pop(0)) + # The last parameters can be used to modify the plot + save_as = None + draw_grid = False + cmap = None + xmin = None + xmax = None + ymin = None + ymax = None + zmin = None + zmax = None + for param in reversed(param_list): + if param == "-png": + save_as = "PNG" + param_list.pop() + elif param == "-pdf": + save_as = "PDF" + param_list.pop() + elif param == "-grid": + draw_grid = True + param_list.pop() + elif param.startswith("-cmap="): + cmap = param[6:] + param_list.pop() + elif param.startswith("-xmin="): + try: + xmin = float(param[6:]) + except ValueError: + print(f'Value for xmin cannot be converted to float: "{xmin}" ' + '-- ignoring it.') + param_list.pop() + elif param.startswith("-xmax="): + try: + xmax = float(param[6:]) + except ValueError: + print(f'Value for xmax cannot be converted to float: "{xmax}" ' + '-- ignoring it.') + param_list.pop() + elif param.startswith("-ymin="): + try: + ymin = float(param[6:]) + except ValueError: + print(f'Value for ymin cannot be converted to float: "{ymin}" ' + '-- ignoring it.') + param_list.pop() + elif param.startswith("-ymax="): + try: + ymax = float(param[6:]) + except ValueError: + print(f'Value for ymax cannot be converted to float: "{ymax}" ' + '-- ignoring it.') + param_list.pop() + elif param.startswith("-zmin="): + try: + zmin = float(param[6:]) + except ValueError: + print(f'Value for zmin cannot be converted to float: "{zmin}" ' + '-- ignoring it.') + param_list.pop() + elif param.startswith("-zmax="): + try: + zmax = float(param[6:]) + except ValueError: + print(f'Value for zmax cannot be converted to float: "{zmax}" ' + '-- ignoring it.') + param_list.pop() + else: + # End of extra parameters, beginning of the SQL statement + break + # Every other parameter is an SQL condition to filter the data + condition = " ".join(param_list).strip() + # Get the data belonging to the parameters + datas = [] + for parameter in parameters: + data = self.get_data(self.selected_table, parameter, condition) + if not data: + print(f'Get data for parameter "{parameter}" ', end="") + if condition: + print(f'under condition "{condition}" ', end="") + print('failed.') + return + datas.append(data) + print("-> x:", datas[0]) + print("-> y:", datas[1]) + print("-> z:", datas[2]) + plt.xlabel(parameters[0]) + plt.ylabel(parameters[1]) + plt.title(parameters[2]) + if xmin is not None: + plt.xlim(left=xmin) + if xmax is not None: + plt.xlim(right=xmax) + if ymin is not None: + plt.ylim(bottom=ymin) + if ymax is not None: + plt.ylim(top=ymax) + if draw_grid: + plt.grid() + try: + plt.scatter( + datas[0], datas[1], c=datas[2], cmap=cmap, vmin=zmin, vmax=zmax, + ) + except Exception as e: + print("Plot did not succeed. Error message:") + print(e) + else: + plt.colorbar() + if save_as == "PDF": + figid = 0 + filename = f"figure_{figid}.pdf" + while os.path.exists(filename): + figid += 1 + filename = f"figure_{figid}.pdf" + print(f'Saving plot as "{filename}".') + plt.savefig(filename) + elif save_as == "PNG": + figid = 0 + filename = f"figure_{figid}.png" + while os.path.exists(filename): + figid += 1 + filename = f"figure_{figid}.png" + print(f'Saving plot as "{filename}".') + plt.savefig(filename) + plt.show() + + def complete_scatter(self, text, line, begidx, endidx): + if not self.selected_table: + return [] + parameters = ["cmap=", "grid", "png", "pdf"] + parameters += self.con.get_column_names(self.selected_table) + parameters.extend(extra_tools.keys()) + return [p for p in parameters if p.startswith(text)] + + def help_scatter(self): + print("""> scatter [condition] [-cmap=] [-grid] [-{x|y|y}{min|max}=] [-{png|pdf}] + Make a scatter plot showing the values of parameter_3 in colour in relation + to parameter_1 and parameter_2 for the selected experiment. + + The parameters can be read from the database or calculated using shell- + extensions. Type "help calculate" for further information on extensions. + + A condition can be used to filter or sort the data. Type "help filter" for + further information on filtering and sorting. + + To specify the colour map of plot, use "-cmap=", where is the + name of a colour_map for matplotlib like "-cmap=bwr" to plot in blue-white- + red or "-cmap=jet" for a colourful rainbow. For further examples, consider + https://matplotlib.org/users/colormaps.html . + If multiple colour maps are specified, only the first one is considered. + + To draw a grid on the plot, add "-grid" to the command. + + To specify the range in either x-, y- or z-direction, use the attributes + -xmin=, -xmax=, -ymin=, -ymax=, -zmin=, -zmax= with a number as argument. + The z-axis refers to the colourbar. + + Add either "-png" or "-pdf" to the command to save the figure in a file of + the corresponding filetype. If both are specified, only the first argument + is taken into account. + + The order of the qualifiers starting with a dash ("-") does not play a role, + but they must be at the end of the command. + + While the window with the plot is open, the shell can be used as usual. + If another plot command is executed with the figure still open, the new plot + is superimposed onto the previous one. Matplotlib automatically selects a + new colour if no colour is specified explicitely. + + Example: + - scatter intensity sigma duration diffusion="False" OR Kdiff=0 -zmin=0 + This command creates a scatter plot with the default colourmap of + matplotlib on a colour-axis starting from 0 which shows the integration + time in relation to the intensity and the value of sigma for experiments + of the current class without diffusion.""") + + def do_pcolor(self, params): + # Check the run condition + if not self.selected_table: + print('No experiment selected. ' + 'Select an experiment to make a pseudocolour plot of its data.') + return + # Parse the arguments + param_list = params.split(" ") + # Two parameters are necessary + if len(param_list) < 3: + print('No three parameters given. Type "help pcolor" for further information.') + return + parameters = [] + for i in range(3): + parameters.append(param_list.pop(0)) + # The last parameters can be used to modify the plot + save_as = None + draw_grid = False + shading = 'flat' + cmap = None + xmin = None + xmax = None + ymin = None + ymax = None + zmin = None + zmax = None + for param in reversed(param_list): + if param == "-png": + save_as = "PNG" + param_list.pop() + elif param == "-pdf": + save_as = "PDF" + param_list.pop() + elif param == "-grid": + draw_grid = True + param_list.pop() + elif param == "-shading": + shading = 'gouraud' + param_list.pop() + elif param.startswith("-cmap="): + cmap = param[6:] + param_list.pop() + elif param.startswith("-xmin="): + try: + xmin = float(param[6:]) + except ValueError: + print(f'Value for xmin cannot be converted to float: "{xmin}" ' + '-- ignoring it.') + param_list.pop() + elif param.startswith("-xmax="): + try: + xmax = float(param[6:]) + except ValueError: + print(f'Value for xmax cannot be converted to float: "{xmax}" ' + '-- ignoring it.') + param_list.pop() + elif param.startswith("-ymin="): + try: + ymin = float(param[6:]) + except ValueError: + print(f'Value for ymin cannot be converted to float: "{ymin}" ' + '-- ignoring it.') + param_list.pop() + elif param.startswith("-ymax="): + try: + ymax = float(param[6:]) + except ValueError: + print(f'Value for ymax cannot be converted to float: "{ymax}" ' + '-- ignoring it.') + param_list.pop() + elif param.startswith("-zmin="): + try: + zmin = float(param[6:]) + except ValueError: + print(f'Value for zmin cannot be converted to float: "{zmin}" ' + '-- ignoring it.') + param_list.pop() + elif param.startswith("-zmax="): + try: + zmax = float(param[6:]) + except ValueError: + print(f'Value for zmax cannot be converted to float: "{zmax}" ' + '-- ignoring it.') + param_list.pop() + else: + # End of extra parameters, beginning of the SQL statement + break + # Every other parameter is an SQL condition to filter the data + condition = " ".join(param_list).strip() + # Get the data belonging to the parameters + datas = [] + for parameter in parameters: + data = self.get_data(self.selected_table, parameter, condition) + if not data: + print(f'Get data for parameter "{parameter}" ', end="") + if condition: + print(f'under condition "{condition}" ', end="") + print('failed.') + return + datas.append(data) + # Arrange the data in a grid + xvalues = sorted(set(datas[0])) + yvalues = sorted(set(datas[1])) + print(f'There are {len(xvalues)} unique x-values and {len(yvalues)} unique y-values.') + data_grid = np.full((len(yvalues), len(xvalues)), np.nan) + for i, zval in enumerate(datas[2]): + data_grid[yvalues.index(datas[1][i]), xvalues.index(datas[0][i])] = zval + print(f"-> x ({parameters[0]}):", xvalues) + print(f"-> y ({parameters[1]}):", yvalues) + print(f"-> z ({parameters[2]}):") + print(data_grid) + if shading == 'flat': + xdiff2 = np.diff(xvalues)/2 + xaxis = [xvalues[0] - xdiff2[0], *(xvalues[:-1] + xdiff2), xvalues[-1] + xdiff2[-1]] + ydiff2 = np.diff(yvalues)/2 + yaxis = [yvalues[0] - ydiff2[0], *(yvalues[:-1] + ydiff2), yvalues[-1] + ydiff2[-1]] + else: + xaxis = xvalues + yaxis = yvalues + plt.xlabel(parameters[0]) + plt.ylabel(parameters[1]) + plt.title(f'{parameters[2]} ({condition if condition else "all data"})' ) + if xmin is not None: + plt.xlim(left=xmin) + if xmax is not None: + plt.xlim(right=xmax) + if ymin is not None: + plt.ylim(bottom=ymin) + if ymax is not None: + plt.ylim(top=ymax) + if draw_grid: + plt.grid() + try: + plt.pcolormesh( + xaxis, yaxis, data_grid, cmap=cmap, vmin=zmin, vmax=zmax, + shading=shading, + ) + except Exception as e: + print("Plot did not succeed. Error message:") + print(e) + else: + plt.colorbar() + if save_as == "PDF": + figid = 0 + filename = f"figure_{figid}.pdf" + while os.path.exists(filename): + figid += 1 + filename = f"figure_{figid}.pdf" + print(f'Saving plot as "{filename}".') + plt.savefig(filename) + elif save_as == "PNG": + figid = 0 + filename = f"figure_{figid}.png" + while os.path.exists(filename): + figid += 1 + filename = f"figure_{figid}.png" + print(f'Saving plot as "{filename}".') + plt.savefig(filename) + plt.show() + + def complete_pcolor(self, text, line, begidx, endidx): + if not self.selected_table: + return [] + parameters = ["cmap=", "grid", "png", "pdf", "shading"] + parameters += self.con.get_column_names(self.selected_table) + parameters.extend(extra_tools.keys()) + return [p for p in parameters if p.startswith(text)] + + def help_scatter(self): + print("""> scatter [condition] [-cmap=] [-grid] [-{x|y|y}{min|max}=] [-{png|pdf}] + Make a scatter plot showing the values of parameter_3 in colour in relation + to parameter_1 and parameter_2 for the selected experiment. + + The parameters can be read from the database or calculated using shell- + extensions. Type "help calculate" for further information on extensions. + + A condition can be used to filter or sort the data. Type "help filter" for + further information on filtering and sorting. + + To specify the colour map of plot, use "-cmap=", where is the + name of a colour_map for matplotlib like "-cmap=bwr" to plot in blue-white- + red or "-cmap=jet" for a colourful rainbow. For further examples, consider + https://matplotlib.org/users/colormaps.html . + If multiple colour maps are specified, only the first one is considered. + + To draw a grid on the plot, add "-grid" to the command. + + To specify the range in either x-, y- or z-direction, use the attributes + -xmin=, -xmax=, -ymin=, -ymax=, -zmin=, -zmax= with a number as argument. + The z-axis refers to the colourbar. + + Add either "-png" or "-pdf" to the command to save the figure in a file of + the corresponding filetype. If both are specified, only the first argument + is taken into account. + + The order of the qualifiers starting with a dash ("-") does not play a role, + but they must be at the end of the command. + + While the window with the plot is open, the shell can be used as usual. + If another plot command is executed with the figure still open, the new plot + is superimposed onto the previous one. Matplotlib automatically selects a + new colour if no colour is specified explicitely. + + Example: + - scatter intensity sigma duration diffusion="False" OR Kdiff=0 -zmin=0 + This command creates a scatter plot with the default colourmap of + matplotlib on a colour-axis starting from 0 which shows the integration + time in relation to the intensity and the value of sigma for experiments + of the current class without diffusion.""") + ### Functionality to CLEAN up def do_remove(self, params): global display_all @@ -552,7 +1176,7 @@ def do_remove(self, params): # Check conditions and parameters if not self.selected_table: - print('No table selected. Select a table to remove entries.') + print('No experiment selected. Select an experiment to remove entries from it.') return if not params: print('No ids given. Specify ids of the entries to remove.') @@ -669,11 +1293,12 @@ def do_exit(self, params): return True def do_EOF(self, params): - print("") if self.selected_table: + print("select") self.prompt = "(EMS) " self.selected_table = "" else: + print("exit") return True ### Behaviour for empty input @@ -682,11 +1307,7 @@ def emptyline(self): ### Helper functions def table_name_completion(self, text): - completions = [] - for table in self.con.tables: - if table.name.startswith(text): - completions.append(table.name) - return completions + return [table.name for table in self.con.tables if table.name.startswith(text)] def column_name_completion(self, table_name, text): completions = [] @@ -754,6 +1375,33 @@ def open_file(self, command, path): stdin=subprocess.DEVNULL, ) + def get_data(self, table_name, parameter, condition): + if self.con.is_valid_column(parameter, table_name): + return self.con.get_data(table_name, parameter, condition) + elif parameter in extra_tools: + data = [] + ids = self.con.get_data(table_name, "id", condition) + print(f'Calculating data with tool "{parameter}" for {len(ids)} experiments.') + tstart = time.time() + for id_ in ids: + expname = "{}_{:03}".format(table_name, id_) + path = os.path.join(self.exp_dir, expname, expname + '_his.nc') + try: + val = extra_tools[parameter](path) + except Exception as e: + print(f'Tool "{parameter}" did not succeed on experiment {id_}. ' + 'The error message is:') + print(e) + print('Using value 0 (zero) as fallback.') + val = 0 + data.append(val) + print('Calculation finished in {:.3f} seconds.'.format(time.time() - tstart)) + return data + else: + print(f'Parameter "{parameter}" is neither a parameter of the ' + 'selected experiment nor a loaded extension.') + return [] + def string_format_table(table_name, columns, rows): # Get current date and time to make datetime easy to read @@ -903,8 +1551,8 @@ def make_nice_time_string(datetime_object, datetime_reference): dt_diff_minutes = dt_diff.seconds / 60 if dt_diff_minutes < 1: return LANG.JUST_NOW - elif dt_diff_minutes < 59.5: - return LANG.AGO_MINUTES.format(round(dt_diff_minutes)) + elif dt_diff_minutes < 60: + return LANG.AGO_MINUTES.format(int(dt_diff_minutes)) else: return LANG.AGO_HOURS.format(int(dt_diff_minutes / 60), int(dt_diff_minutes % 60)) elif datetime_object.date() == (datetime_reference - datetime.timedelta(days=1)).date(): @@ -930,6 +1578,10 @@ def make_nice_time_string(datetime_object, datetime_reference): # Use fancy colours during 6 days of Carnival try: from dateutil.easter import easter +except ModuleNotFoundError: + # No fun at carnival possible + pass +else: date_today = datetime.date.today() carnival_start = easter(date_today.year)-datetime.timedelta(days=46+6) # Weiberfastnacht carnival_end = easter(date_today.year)-datetime.timedelta(days=46) # Aschermittwoch @@ -946,9 +1598,9 @@ def make_nice_time_string(datetime_object, datetime_reference): ("\033[104m",), ("\033[105m",), ) -except: - # At least we tried! - pass + +# Activate interactive plotting +plt.ion() # Start the shell ems_cli = EMShell(datadir) From c2659f1046324232bd8ded48bf5026171055351d Mon Sep 17 00:00:00 2001 From: Markus Date: Thu, 6 Jun 2019 10:27:12 +0200 Subject: [PATCH 13/31] EMShell: restructure and simplify the code Fix some problems in the last commit, so that the code is more flexible and a lot shorter now. --- experiments/EMShell_Extensions.py | 14 +- experiments/Experiment-Manager.py | 724 +++++++++++------------------- 2 files changed, 281 insertions(+), 457 deletions(-) diff --git a/experiments/EMShell_Extensions.py b/experiments/EMShell_Extensions.py index f4947f1..f581a0f 100644 --- a/experiments/EMShell_Extensions.py +++ b/experiments/EMShell_Extensions.py @@ -8,16 +8,20 @@ def get_strongest_wavenumber(his_filename): - # Open data file + """Calculate the most intense wavenumber in y-direction. + + This function opens the given history-file, performs a Fourier + transform in y on the masked streamfunction psi and returns the + highest wavenumber which has at some point in time the highest + intensity apart from the wavenumber zero.""" + # Open history file and load the data dataset_his = nc.Dataset(his_filename) - # Load data ny = dataset_his.ny dy = dataset_his.Ly / ny - # Save the data as masked numpy arrays - x = dataset_his["x"][:] + # Save the data as a masked numpy array psi = dataset_his["psi"][:] psi.mask = 1 - dataset_his["msk"][:] - # Set length of zero-padded signal + # Set length of zero-padded signal (use ny for no zero-padding) fft_ny = ny # Caculate zero-padded Fourier-transform in y fft_psi = fftshift(fft(psi, n=fft_ny, axis=1), axes=1) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 2e00eee..10d7603 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -1,12 +1,11 @@ -# Markus Reinert, May 2019 +# Markus Reinert, May/June 2019 # -# Command-line interface for the fluid2d Experiment Management System (EMS) +# EMShell: Command-line interface for the fluid2d Experiment Management System (EMS) # -# This programme can be started directly with Python if fluid2d is activated. -# Otherwise, the path to the folder with the experiment-files (that is the -# value of param.datadir) must be specified as a command-line argument. +# This programme requires fluid2d to be activated. import os +import re import cmd import time import numpy as np @@ -16,9 +15,8 @@ import itertools import subprocess import matplotlib.pyplot as plt - +# Local imports import EMShell_Extensions as EMExt - try: from param import Param except ModuleNotFoundError: @@ -214,8 +212,12 @@ def show_sorted_table(self, table_name, statement): print("-"*50) def is_valid_column(self, column_name, table_name=""): + """Check whether a column with the given name exists. + + If no table_name is specified, check whether any table has such + a column, otherwise look only in the table with the given name.""" for table in self.tables: - if not table_name or table_name == table.name: + if table.name == table_name or table_name == "": for n, t in table.columns: if column_name == n: return True @@ -620,7 +622,7 @@ def do_calculate(self, params): 'The error message is:') print(e) return - if val: + if val is not None: print(f'-> {toolname}({id_}) = {val}') else: print(f'-> {toolname}({id_}) succeeded without return value.') @@ -642,128 +644,54 @@ def do_plot(self, params): print('No experiment selected. Select an experiment to plot its data.') return # Parse the arguments - param_list = params.split(" ") - # Two parameters are necessary - if len(param_list) < 2: - print('No two parameters given. Type "help plot" for further information.') + parameters = self.parse_plot_parameters(params, 2) + if not parameters: return - param1 = param_list[0] - param_list.pop(0) - param2 = param_list[0] - param_list.pop(0) - # The last parameters can be used to modify the plot - save_as = None - draw_grid = False - format_string = "" - for param in reversed(param_list): - if param == "-png": - save_as = "PNG" - param_list.pop() - elif param == "-pdf": - save_as = "PDF" - param_list.pop() - elif param == "-grid": - draw_grid = True - param_list.pop() - elif param.startswith("-f="): - format_string = param[3:] - param_list.pop() - else: - # End of extra parameters, beginning of the SQL statement - break - # Every other parameter is an SQL condition to filter the data - condition = " ".join(param_list).strip() + variables = parameters["variables"] # Get the data belonging to the parameters - data1 = self.get_data(self.selected_table, param1, condition) - if not data1: - print(f'Get data for first parameter "{param1}" ', end="") - if condition: - print(f'under condition "{condition}" ', end="") - print('failed.') + datas = self.get_multiple_data(self.selected_table, variables, parameters["condition"]) + if not datas: return - data2 = self.get_data(self.selected_table, param2, condition) - if not data2: - print(f'Get data for second parameter "{param2}" ', end="") - if condition: - print(f'under condition "{condition}" ', end="") - print('failed.') - return - - print("-> x:", data1) - print("-> y:", data2) - plt.xlabel(param1) - plt.ylabel(param2) - if draw_grid: - plt.grid() + # Print and plot the data + for i, c in enumerate(["x", "y"]): + print(f'-> {c} ({variables[i]}):', datas[i]) + plot_label = variables[1] + if parameters["condition"]: + plot_label += ' (' + parameters["condition"] + ')' + plt.title(self.selected_table) + plt.xlabel(variables[0]) try: plt.plot( - data1, data2, format_string, - label=f'{self.selected_table} ({condition if condition else "all data"})', + datas[0], datas[1], parameters["format_string"], label=plot_label, ) except Exception as e: print("Plot did not succeed. Error message:") print(e) else: plt.legend() - if save_as == "PDF": - figid = 0 - filename = f"figure_{figid}.pdf" - while os.path.exists(filename): - figid += 1 - filename = f"figure_{figid}.pdf" - print(f'Saving plot as "{filename}".') - plt.savefig(filename) - elif save_as == "PNG": - figid = 0 - filename = f"figure_{figid}.png" - while os.path.exists(filename): - figid += 1 - filename = f"figure_{figid}.png" - print(f'Saving plot as "{filename}".') - plt.savefig(filename) + if parameters["xmin"] is not None or parameters["xmax"] is not None: + plt.xlim(parameters["xmin"], parameters["xmax"]) + if parameters["ymin"] is not None or parameters["ymax"] is not None: + plt.ylim(parameters["ymin"], parameters["ymax"]) + if parameters["grid"]: + plt.grid(True) + if parameters["save_as"]: + plt.savefig(get_unique_save_filename("plot_{}." + parameters["save_as"])) plt.show() def complete_plot(self, text, line, begidx, endidx): - if not self.selected_table: - return [] - parameters = ["f=", "grid", "png", "pdf"] - parameters += self.con.get_column_names(self.selected_table) - parameters.extend(extra_tools.keys()) - return [p for p in parameters if p.startswith(text)] + return self.plot_attribute_completion(self, text, [ + 'f=', 'grid', + 'png', 'pdf', + 'xmin=', 'xmax=', 'ymin=', 'ymax=', + ]) def help_plot(self): - print("""> plot [condition] [-f=] [-grid] [-{png|pdf}] - Make a diagram showing the relation between the two parameters for the - selected experiment. - - The parameters can be read from the database or calculated using shell- - extensions. Type "help calculate" for further information on extensions. - - A condition can be used to filter or sort the data. Type "help filter" for - further information on filtering and sorting. - - To specify the type of plot, use "-f=", where is a format - string for matplotlib like "-f=o" to plot with dots instead of a line, or - "-f=rx" to plot with red crosses. For further information on format strings - in matplotlib, see the Notes section on the following website: - https://matplotlib.org/3.1.0/api/_as_gen/matplotlib.pyplot.plot.html - If multiple format strings are specified, only the first one is considered. - - To draw a grid on the plot, add "-grid" to the command. - - Add either "-png" or "-pdf" to the command to save the figure in a file of - the corresponding filetype. If both are specified, only the first argument - is taken into account. - - The order of the qualifiers starting with a dash ("-") does not play a role, - but they must be at the end of the command. - - While the window with the plot is open, the shell can be used as usual. - If another plot command is executed with the figure still open, the new plot - is superimposed onto the previous one. Matplotlib automatically selects a - new colour if no colour is specified explicitely. If the -grid argument is - specified a second time in another plot command, the grid is switched off. - + print("""> plot [condition] [-grid] [-f=] [-{x|y}{min|max}=] [-{png|pdf}] + Make a diagram showing the relation between the two variables for the + selected experiment.""") + self.print_general_plot_help("2d") + print(""" Example: - plot size_his duration id <= 10 -f=g.- -grid This command creates a plot in green with dots and lines on a grid @@ -773,185 +701,60 @@ def help_plot(self): def do_scatter(self, params): # Check the run condition if not self.selected_table: - print('No experiment selected. ' - 'Select an experiment to make a scatter plot of its data.') + print('No experiment selected. Select an experiment to plot its data.') return # Parse the arguments - param_list = params.split(" ") - # Two parameters are necessary - if len(param_list) < 3: - print('No three parameters given. Type "help scatter" for further information.') + parameters = self.parse_plot_parameters(params, 3) + if not parameters: return - parameters = [] - for i in range(3): - parameters.append(param_list.pop(0)) - # The last parameters can be used to modify the plot - save_as = None - draw_grid = False - cmap = None - xmin = None - xmax = None - ymin = None - ymax = None - zmin = None - zmax = None - for param in reversed(param_list): - if param == "-png": - save_as = "PNG" - param_list.pop() - elif param == "-pdf": - save_as = "PDF" - param_list.pop() - elif param == "-grid": - draw_grid = True - param_list.pop() - elif param.startswith("-cmap="): - cmap = param[6:] - param_list.pop() - elif param.startswith("-xmin="): - try: - xmin = float(param[6:]) - except ValueError: - print(f'Value for xmin cannot be converted to float: "{xmin}" ' - '-- ignoring it.') - param_list.pop() - elif param.startswith("-xmax="): - try: - xmax = float(param[6:]) - except ValueError: - print(f'Value for xmax cannot be converted to float: "{xmax}" ' - '-- ignoring it.') - param_list.pop() - elif param.startswith("-ymin="): - try: - ymin = float(param[6:]) - except ValueError: - print(f'Value for ymin cannot be converted to float: "{ymin}" ' - '-- ignoring it.') - param_list.pop() - elif param.startswith("-ymax="): - try: - ymax = float(param[6:]) - except ValueError: - print(f'Value for ymax cannot be converted to float: "{ymax}" ' - '-- ignoring it.') - param_list.pop() - elif param.startswith("-zmin="): - try: - zmin = float(param[6:]) - except ValueError: - print(f'Value for zmin cannot be converted to float: "{zmin}" ' - '-- ignoring it.') - param_list.pop() - elif param.startswith("-zmax="): - try: - zmax = float(param[6:]) - except ValueError: - print(f'Value for zmax cannot be converted to float: "{zmax}" ' - '-- ignoring it.') - param_list.pop() - else: - # End of extra parameters, beginning of the SQL statement - break - # Every other parameter is an SQL condition to filter the data - condition = " ".join(param_list).strip() + variables = parameters["variables"] # Get the data belonging to the parameters - datas = [] - for parameter in parameters: - data = self.get_data(self.selected_table, parameter, condition) - if not data: - print(f'Get data for parameter "{parameter}" ', end="") - if condition: - print(f'under condition "{condition}" ', end="") - print('failed.') - return - datas.append(data) - print("-> x:", datas[0]) - print("-> y:", datas[1]) - print("-> z:", datas[2]) - plt.xlabel(parameters[0]) - plt.ylabel(parameters[1]) - plt.title(parameters[2]) - if xmin is not None: - plt.xlim(left=xmin) - if xmax is not None: - plt.xlim(right=xmax) - if ymin is not None: - plt.ylim(bottom=ymin) - if ymax is not None: - plt.ylim(top=ymax) - if draw_grid: - plt.grid() + datas = self.get_multiple_data(self.selected_table, variables, parameters["condition"]) + if not datas: + return + # Print and plot the data + for i, c in enumerate(["x", "y", "z"]): + print('-> {} ({}):'.format(c, variables[i]), datas[i]) + plot_title = self.selected_table + ": " + variables[2] + if parameters["condition"]: + plot_title += ' (' + parameters["condition"] + ')' + plt.title(plot_title) + plt.xlabel(variables[0]) + plt.ylabel(variables[1]) try: plt.scatter( - datas[0], datas[1], c=datas[2], cmap=cmap, vmin=zmin, vmax=zmax, + datas[0], datas[1], c=datas[2], + cmap=parameters["cmap"], + vmin=parameters["zmin"], vmax=parameters["zmax"], ) except Exception as e: - print("Plot did not succeed. Error message:") + print("Scatter did not succeed. Error message:") print(e) else: plt.colorbar() - if save_as == "PDF": - figid = 0 - filename = f"figure_{figid}.pdf" - while os.path.exists(filename): - figid += 1 - filename = f"figure_{figid}.pdf" - print(f'Saving plot as "{filename}".') - plt.savefig(filename) - elif save_as == "PNG": - figid = 0 - filename = f"figure_{figid}.png" - while os.path.exists(filename): - figid += 1 - filename = f"figure_{figid}.png" - print(f'Saving plot as "{filename}".') - plt.savefig(filename) + if parameters["xmin"] is not None or parameters["xmax"] is not None: + plt.xlim(parameters["xmin"], parameters["xmax"]) + if parameters["ymin"] is not None or parameters["ymax"] is not None: + plt.ylim(parameters["ymin"], parameters["ymax"]) + if parameters["grid"]: + plt.grid(True) + if parameters["save_as"]: + plt.savefig(get_unique_save_filename("scatter_{}." + parameters["save_as"])) plt.show() def complete_scatter(self, text, line, begidx, endidx): - if not self.selected_table: - return [] - parameters = ["cmap=", "grid", "png", "pdf"] - parameters += self.con.get_column_names(self.selected_table) - parameters.extend(extra_tools.keys()) - return [p for p in parameters if p.startswith(text)] + return self.plot_attribute_completion(self, text, [ + 'cmap=', 'grid', + 'png', 'pdf', + 'xmin=', 'xmax=', 'ymin=', 'ymax=', 'zmin=', 'zmax=', + ]) def help_scatter(self): - print("""> scatter [condition] [-cmap=] [-grid] [-{x|y|y}{min|max}=] [-{png|pdf}] - Make a scatter plot showing the values of parameter_3 in colour in relation - to parameter_1 and parameter_2 for the selected experiment. - - The parameters can be read from the database or calculated using shell- - extensions. Type "help calculate" for further information on extensions. - - A condition can be used to filter or sort the data. Type "help filter" for - further information on filtering and sorting. - - To specify the colour map of plot, use "-cmap=", where is the - name of a colour_map for matplotlib like "-cmap=bwr" to plot in blue-white- - red or "-cmap=jet" for a colourful rainbow. For further examples, consider - https://matplotlib.org/users/colormaps.html . - If multiple colour maps are specified, only the first one is considered. - - To draw a grid on the plot, add "-grid" to the command. - - To specify the range in either x-, y- or z-direction, use the attributes - -xmin=, -xmax=, -ymin=, -ymax=, -zmin=, -zmax= with a number as argument. - The z-axis refers to the colourbar. - - Add either "-png" or "-pdf" to the command to save the figure in a file of - the corresponding filetype. If both are specified, only the first argument - is taken into account. - - The order of the qualifiers starting with a dash ("-") does not play a role, - but they must be at the end of the command. - - While the window with the plot is open, the shell can be used as usual. - If another plot command is executed with the figure still open, the new plot - is superimposed onto the previous one. Matplotlib automatically selects a - new colour if no colour is specified explicitely. - + print("""> scatter [condition] [-grid] [-cmap=] [-{x|y|z}{min|max}=] [-{png|pdf}] + Make a scatter plot showing the values of the third variable in colour in + relation to the other two variables for the selected experiment.""") + self.print_general_plot_help("3d") + print(""" Example: - scatter intensity sigma duration diffusion="False" OR Kdiff=0 -zmin=0 This command creates a scatter plot with the default colourmap of @@ -962,103 +765,17 @@ def help_scatter(self): def do_pcolor(self, params): # Check the run condition if not self.selected_table: - print('No experiment selected. ' - 'Select an experiment to make a pseudocolour plot of its data.') + print('No experiment selected. Select an experiment to plot its data.') return # Parse the arguments - param_list = params.split(" ") - # Two parameters are necessary - if len(param_list) < 3: - print('No three parameters given. Type "help pcolor" for further information.') + parameters = self.parse_plot_parameters(params, 3) + if not parameters: return - parameters = [] - for i in range(3): - parameters.append(param_list.pop(0)) - # The last parameters can be used to modify the plot - save_as = None - draw_grid = False - shading = 'flat' - cmap = None - xmin = None - xmax = None - ymin = None - ymax = None - zmin = None - zmax = None - for param in reversed(param_list): - if param == "-png": - save_as = "PNG" - param_list.pop() - elif param == "-pdf": - save_as = "PDF" - param_list.pop() - elif param == "-grid": - draw_grid = True - param_list.pop() - elif param == "-shading": - shading = 'gouraud' - param_list.pop() - elif param.startswith("-cmap="): - cmap = param[6:] - param_list.pop() - elif param.startswith("-xmin="): - try: - xmin = float(param[6:]) - except ValueError: - print(f'Value for xmin cannot be converted to float: "{xmin}" ' - '-- ignoring it.') - param_list.pop() - elif param.startswith("-xmax="): - try: - xmax = float(param[6:]) - except ValueError: - print(f'Value for xmax cannot be converted to float: "{xmax}" ' - '-- ignoring it.') - param_list.pop() - elif param.startswith("-ymin="): - try: - ymin = float(param[6:]) - except ValueError: - print(f'Value for ymin cannot be converted to float: "{ymin}" ' - '-- ignoring it.') - param_list.pop() - elif param.startswith("-ymax="): - try: - ymax = float(param[6:]) - except ValueError: - print(f'Value for ymax cannot be converted to float: "{ymax}" ' - '-- ignoring it.') - param_list.pop() - elif param.startswith("-zmin="): - try: - zmin = float(param[6:]) - except ValueError: - print(f'Value for zmin cannot be converted to float: "{zmin}" ' - '-- ignoring it.') - param_list.pop() - elif param.startswith("-zmax="): - try: - zmax = float(param[6:]) - except ValueError: - print(f'Value for zmax cannot be converted to float: "{zmax}" ' - '-- ignoring it.') - param_list.pop() - else: - # End of extra parameters, beginning of the SQL statement - break - # Every other parameter is an SQL condition to filter the data - condition = " ".join(param_list).strip() + variables = parameters["variables"] # Get the data belonging to the parameters - datas = [] - for parameter in parameters: - data = self.get_data(self.selected_table, parameter, condition) - if not data: - print(f'Get data for parameter "{parameter}" ', end="") - if condition: - print(f'under condition "{condition}" ', end="") - print('failed.') - return - datas.append(data) + datas = self.get_multiple_data(self.selected_table, variables, parameters["condition"]) + if not datas: + return # Arrange the data in a grid xvalues = sorted(set(datas[0])) yvalues = sorted(set(datas[1])) @@ -1066,11 +783,9 @@ def do_pcolor(self, params): data_grid = np.full((len(yvalues), len(xvalues)), np.nan) for i, zval in enumerate(datas[2]): data_grid[yvalues.index(datas[1][i]), xvalues.index(datas[0][i])] = zval - print(f"-> x ({parameters[0]}):", xvalues) - print(f"-> y ({parameters[1]}):", yvalues) - print(f"-> z ({parameters[2]}):") - print(data_grid) - if shading == 'flat': + # If no shading is applied, extend the axes to have each rectangle + # centred at the corresponding (x,y)-value + if parameters["shading"] == 'flat': xdiff2 = np.diff(xvalues)/2 xaxis = [xvalues[0] - xdiff2[0], *(xvalues[:-1] + xdiff2), xvalues[-1] + xdiff2[-1]] ydiff2 = np.diff(yvalues)/2 @@ -1078,93 +793,54 @@ def do_pcolor(self, params): else: xaxis = xvalues yaxis = yvalues - plt.xlabel(parameters[0]) - plt.ylabel(parameters[1]) - plt.title(f'{parameters[2]} ({condition if condition else "all data"})' ) - if xmin is not None: - plt.xlim(left=xmin) - if xmax is not None: - plt.xlim(right=xmax) - if ymin is not None: - plt.ylim(bottom=ymin) - if ymax is not None: - plt.ylim(top=ymax) - if draw_grid: - plt.grid() + # Print and plot the data + print(f"-> x ({variables[0]}):", xvalues) + print(f"-> y ({variables[1]}):", yvalues) + print(f"-> z ({variables[2]}):") + print(data_grid) + plot_title = self.selected_table + ": " + variables[2] + if parameters["condition"]: + plot_title += ' (' + parameters["condition"] + ')' + plt.title(plot_title) + plt.xlabel(variables[0]) + plt.ylabel(variables[1]) try: plt.pcolormesh( - xaxis, yaxis, data_grid, cmap=cmap, vmin=zmin, vmax=zmax, - shading=shading, + xaxis, yaxis, data_grid, + cmap=parameters["cmap"], shading=parameters["shading"], + vmin=parameters["zmin"], vmax=parameters["zmax"], ) except Exception as e: - print("Plot did not succeed. Error message:") + print("Pcolormesh did not succeed. Error message:") print(e) else: plt.colorbar() - if save_as == "PDF": - figid = 0 - filename = f"figure_{figid}.pdf" - while os.path.exists(filename): - figid += 1 - filename = f"figure_{figid}.pdf" - print(f'Saving plot as "{filename}".') - plt.savefig(filename) - elif save_as == "PNG": - figid = 0 - filename = f"figure_{figid}.png" - while os.path.exists(filename): - figid += 1 - filename = f"figure_{figid}.png" - print(f'Saving plot as "{filename}".') - plt.savefig(filename) + if parameters["xmin"] is not None or parameters["xmax"] is not None: + plt.xlim(parameters["xmin"], parameters["xmax"]) + if parameters["ymin"] is not None or parameters["ymax"] is not None: + plt.ylim(parameters["ymin"], parameters["ymax"]) + if parameters["grid"]: + plt.grid(True) + if parameters["save_as"]: + plt.savefig(get_unique_save_filename("pcolor_{}." + parameters["save_as"])) plt.show() def complete_pcolor(self, text, line, begidx, endidx): - if not self.selected_table: - return [] - parameters = ["cmap=", "grid", "png", "pdf", "shading"] - parameters += self.con.get_column_names(self.selected_table) - parameters.extend(extra_tools.keys()) - return [p for p in parameters if p.startswith(text)] - - def help_scatter(self): - print("""> scatter [condition] [-cmap=] [-grid] [-{x|y|y}{min|max}=] [-{png|pdf}] - Make a scatter plot showing the values of parameter_3 in colour in relation - to parameter_1 and parameter_2 for the selected experiment. - - The parameters can be read from the database or calculated using shell- - extensions. Type "help calculate" for further information on extensions. - - A condition can be used to filter or sort the data. Type "help filter" for - further information on filtering and sorting. - - To specify the colour map of plot, use "-cmap=", where is the - name of a colour_map for matplotlib like "-cmap=bwr" to plot in blue-white- - red or "-cmap=jet" for a colourful rainbow. For further examples, consider - https://matplotlib.org/users/colormaps.html . - If multiple colour maps are specified, only the first one is considered. - - To draw a grid on the plot, add "-grid" to the command. - - To specify the range in either x-, y- or z-direction, use the attributes - -xmin=, -xmax=, -ymin=, -ymax=, -zmin=, -zmax= with a number as argument. - The z-axis refers to the colourbar. - - Add either "-png" or "-pdf" to the command to save the figure in a file of - the corresponding filetype. If both are specified, only the first argument - is taken into account. - - The order of the qualifiers starting with a dash ("-") does not play a role, - but they must be at the end of the command. - - While the window with the plot is open, the shell can be used as usual. - If another plot command is executed with the figure still open, the new plot - is superimposed onto the previous one. Matplotlib automatically selects a - new colour if no colour is specified explicitely. - + return self.plot_attribute_completion(self, text, [ + 'cmap=', 'shading', 'grid', + 'png', 'pdf', + 'xmin=', 'xmax=', 'ymin=', 'ymax=', 'zmin=', 'zmax=', + ]) + + def help_pcolor(self): + print("""> pcolor [condition] [-grid] [-shading] [-cmap=] [-{x|y|z}{min|max}=] [-{png|pdf}] + Make a pseudo-colour plot showing the values of the third variable in colour + in relation to the two other variables for the selected experiment.""") + self.print_general_plot_help("3d", shading=True) + print(""" Example: - - scatter intensity sigma duration diffusion="False" OR Kdiff=0 -zmin=0 - This command creates a scatter plot with the default colourmap of + - pcolor intensity sigma duration diffusion="False" OR Kdiff=0 -zmin=0 + This command creates a pseudo-colour plot with the default colourmap of matplotlib on a colour-axis starting from 0 which shows the integration time in relation to the intensity and the value of sigma for experiments of the current class without diffusion.""") @@ -1309,6 +985,13 @@ def emptyline(self): def table_name_completion(self, text): return [table.name for table in self.con.tables if table.name.startswith(text)] + def plot_attribute_completion(self, text, parameters): + if not self.selected_table: + return [] + parameters += self.con.get_column_names(self.selected_table) + parameters.extend(extra_tools.keys()) + return [p for p in parameters if p.startswith(text)] + def column_name_completion(self, table_name, text): completions = [] for table in self.con.tables: @@ -1356,6 +1039,63 @@ def parse_params_to_experiment(self, params): # Return name of experiment folder return "{}_{:03}".format(name, id_) + @staticmethod + def parse_plot_parameters(params, n_necessary): + parameters = { + "variables": [], + "condition": "", + "save_as": None, + "grid": False, + "xmin": None, + "xmax": None, + "ymin": None, + "ymax": None, + "zmin": None, + "zmax": None, + "cmap": None, + "shading": "flat", + "format_string": "", + } + param_list = params.split(" ") + # Necessary parameters are at the beginning + if len(param_list) < n_necessary: + print(f'Not enough parameters given, {n_necessary} are necessary.') + return None + for i in range(n_necessary): + parameters["variables"].append(param_list.pop(0)) + # The last parameters can be used to modify the plot + for param in reversed(param_list): + if not param: + # Ignore empty strings + pass + elif param == "-png": + parameters["save_as"] = "png" + elif param == "-pdf": + parameters["save_as"] = "pdf" + elif param == "-grid": + parameters["grid"] = True + elif param == "-shading": + parameters["shading"] = "gouraud" + elif param.startswith("-f="): + parameters["format_string"] = param[3:] + elif param.startswith("-cmap="): + parameters["cmap"] = param[6:] + elif xyzminmax.match(param): + param_name = param[1:5] + param_value = param[6:] + try: + parameters[param_name] = float(param_value) + except ValueError: + print(f'Value for {param_name} cannot be converted to ' + f'float: "{param_value}".') + else: + # End of extra parameters, beginning of the SQL statement + break + param_list.pop() + # Every other parameter is treated as an SQL condition to filter the data + parameters["condition"] = " ".join(param_list).strip() + return parameters + def open_file(self, command, path): if self.silent_mode: print("Opening file {} with {} in silent-mode.".format(path, command)) @@ -1375,6 +1115,19 @@ def open_file(self, command, path): stdin=subprocess.DEVNULL, ) + def get_multiple_data(self, table_name, variables, condition): + datas = [] + for var in variables: + data = self.get_data(table_name, var, condition) + if not data: + print(f'Get data for parameter "{var}" ', end="") + if condition: + print(f'under condition "{condition}" ', end="") + print('failed.') + return None + datas.append(data) + return datas + def get_data(self, table_name, parameter, condition): if self.con.is_valid_column(parameter, table_name): return self.con.get_data(table_name, parameter, condition) @@ -1402,6 +1155,69 @@ def get_data(self, table_name, parameter, condition): 'selected experiment nor a loaded extension.') return [] + @staticmethod + def print_general_plot_help(plot_type, shading=False): + print(""" + The variables can be read from the database or calculated using shell- + extensions. Type "help calculate" for further information on extensions. + + A condition can be used to filter or sort the data. Type "help filter" for + further information on filtering and sorting. + + To draw a grid on the plot, add "-grid" to the command.""" + ) + if shading: + print(""" + To make a smooth plot instead of rectangles of solid colour, add "-shading" + to the command. This enables the Gouraud-shading of matplotlib.""" + ) + if plot_type == "2d": + print(""" + To specify the type of plot, use "-f=", where is a format + string for matplotlib like "-f=o" to plot with dots instead of a line, or + "-f=rx" to plot with red crosses. For further information on format strings + in matplotlib, see the Notes section on the following website: + https://matplotlib.org/3.1.0/api/_as_gen/matplotlib.pyplot.plot.html + + To specify the range in x- or y-direction, use the following attributes: + -xmin=, -xmax=, -ymin=, -ymax= + with a number as the argument .""" + ) + elif plot_type == "3d": + print(""" + To specify the colour map of the plot, use "-cmap=", where is + the name of a colour map for matplotlib like "-cmap=bwr" to plot in blue- + white-red or "-cmap=jet" for a colourful rainbow. For further examples, + consider the website https://matplotlib.org/users/colormaps.html . + + To specify the range in x-, y- or z-direction, use the following attributes: + -xmin=, -xmax=, -ymin=, -ymax=, -zmin=, -zmax= + with a number as the argument . The z-axis refers to the colourbar.""" + ) + print(""" + Add either "-png" or "-pdf" to the command to save the figure in a file of + the corresponding filetype. + + The order of the qualifiers starting with a dash ("-") does not play a role, + but they must be at the end of the command, i.e., after the variables and + after the filter condition (if any). If a qualifier is given more than + once, only the first occurence is taken into account. + + While the window with the plot is open, the shell can be used as usual. + If another plot command is executed with the figure still open, the new plot + is superimposed onto the previous one.""" + ) + + +def get_unique_save_filename(template): + id_ = 0 + filename = template.format(id_) + while os.path.exists(filename): + id_ += 1 + filename = template.format(id_) + print(f'Saving as "{filename}".') + return filename + def string_format_table(table_name, columns, rows): # Get current date and time to make datetime easy to read @@ -1602,6 +1418,10 @@ def make_nice_time_string(datetime_object, datetime_reference): # Activate interactive plotting plt.ion() +# Compile regular expression to match parameters +# This matches -xmin=, -xmax=, -ymin=, -ymax=, -zmin=, -zmax= +xyzminmax = re.compile("-[xyz]m(in|ax)=") + # Start the shell ems_cli = EMShell(datadir) ems_cli.cmdloop() From 81f11665fa000c06e91c29ace1a4c1f87109b2a1 Mon Sep 17 00:00:00 2001 From: Markus Date: Thu, 6 Jun 2019 15:28:02 +0200 Subject: [PATCH 14/31] EMShell: fix an obvious bug in the last commit --- experiments/Experiment-Manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 10d7603..ec407ca 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -680,7 +680,7 @@ def do_plot(self, params): plt.show() def complete_plot(self, text, line, begidx, endidx): - return self.plot_attribute_completion(self, text, [ + return self.plot_attribute_completion(text, [ 'f=', 'grid', 'png', 'pdf', 'xmin=', 'xmax=', 'ymin=', 'ymax=', @@ -743,7 +743,7 @@ def do_scatter(self, params): plt.show() def complete_scatter(self, text, line, begidx, endidx): - return self.plot_attribute_completion(self, text, [ + return self.plot_attribute_completion(text, [ 'cmap=', 'grid', 'png', 'pdf', 'xmin=', 'xmax=', 'ymin=', 'ymax=', 'zmin=', 'zmax=', @@ -826,7 +826,7 @@ def do_pcolor(self, params): plt.show() def complete_pcolor(self, text, line, begidx, endidx): - return self.plot_attribute_completion(self, text, [ + return self.plot_attribute_completion(text, [ 'cmap=', 'shading', 'grid', 'png', 'pdf', 'xmin=', 'xmax=', 'ymin=', 'ymax=', 'zmin=', 'zmax=', @@ -1250,7 +1250,7 @@ def string_format_table(table_name, columns, rows): else: # Replace linebreaks val = val.replace(*LINEBREAK_REPLACE) - if type(COMMENT_MAX_LENGTH) == int and COMMENT_MAX_LENGTH > 0: + if type(COMMENT_MAX_LENGTH) is int and COMMENT_MAX_LENGTH > 0: # Cut comments which are too long and end them with an ellipsis if len(val) > COMMENT_MAX_LENGTH: val = val[:COMMENT_MAX_LENGTH-1] + "…" From 1609077ca5ecc2d78220bffd9f9d3443d8156cf7 Mon Sep 17 00:00:00 2001 From: Markus Date: Thu, 6 Jun 2019 17:06:15 +0200 Subject: [PATCH 15/31] EMShell: save as svg and save after plot is finished --- experiments/Experiment-Manager.py | 45 ++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index ec407ca..90d4992 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -682,12 +682,12 @@ def do_plot(self, params): def complete_plot(self, text, line, begidx, endidx): return self.plot_attribute_completion(text, [ 'f=', 'grid', - 'png', 'pdf', + 'png', 'pdf', 'svg', 'xmin=', 'xmax=', 'ymin=', 'ymax=', ]) def help_plot(self): - print("""> plot [condition] [-grid] [-f=] [-{x|y}{min|max}=] [-{png|pdf}] + print("""> plot [condition] [-grid] [-f=] [-{x|y}{min|max}=] [-{png|pdf|svg}] Make a diagram showing the relation between the two variables for the selected experiment.""") self.print_general_plot_help("2d") @@ -745,12 +745,12 @@ def do_scatter(self, params): def complete_scatter(self, text, line, begidx, endidx): return self.plot_attribute_completion(text, [ 'cmap=', 'grid', - 'png', 'pdf', + 'png', 'pdf', 'svg', 'xmin=', 'xmax=', 'ymin=', 'ymax=', 'zmin=', 'zmax=', ]) def help_scatter(self): - print("""> scatter [condition] [-grid] [-cmap=] [-{x|y|z}{min|max}=] [-{png|pdf}] + print("""> scatter [condition] [-grid] [-cmap=] [-{x|y|z}{min|max}=] [-{png|pdf|svg}] Make a scatter plot showing the values of the third variable in colour in relation to the other two variables for the selected experiment.""") self.print_general_plot_help("3d") @@ -828,12 +828,12 @@ def do_pcolor(self, params): def complete_pcolor(self, text, line, begidx, endidx): return self.plot_attribute_completion(text, [ 'cmap=', 'shading', 'grid', - 'png', 'pdf', + 'png', 'pdf', 'svg', 'xmin=', 'xmax=', 'ymin=', 'ymax=', 'zmin=', 'zmax=', ]) def help_pcolor(self): - print("""> pcolor [condition] [-grid] [-shading] [-cmap=] [-{x|y|z}{min|max}=] [-{png|pdf}] + print("""> pcolor [condition] [-grid] [-shading] [-cmap=] [-{x|y|z}{min|max}=] [-{png|pdf|svg}] Make a pseudo-colour plot showing the values of the third variable in colour in relation to the two other variables for the selected experiment.""") self.print_general_plot_help("3d", shading=True) @@ -845,6 +845,28 @@ def help_pcolor(self): time in relation to the intensity and the value of sigma for experiments of the current class without diffusion.""") + def do_save_figure(self, params): + if not params: + params = "png" + if params in ["png", "pdf", "svg"]: + plt.savefig(get_unique_save_filename("figure_{}." + params)) + else: + plt.savefig(params) + + def complete_save_figure(self, text, line, begidx, endidx): + if "." not in text: + return [e for e in ("png", "pdf", "svg") if e.startswith(text)] + stub = text[:text.rfind(".")] + return [n for n in (stub + ".png", stub + ".pdf", stub + ".svg") if n.startswith(text)] + + def help_save_figure(self): + print("""> save_figure + Save the currently opened figure in a file. One can either specify the full + filename or one of the filetypes png, pdf or svg alone. If only the type is + specified, then the name is automatically set to "figure_#.type", where + "#" is an integer such that the filename is unique and "type" is the given + filetype.""") + ### Functionality to CLEAN up def do_remove(self, params): global display_all @@ -1068,10 +1090,8 @@ def parse_plot_parameters(params, n_necessary): if not param: # Ignore empty strings pass - elif param == "-png": - parameters["save_as"] = "png" - elif param == "-pdf": - parameters["save_as"] = "pdf" + elif param in ["-png", "-pdf", "-svg"]: + parameters["save_as"] = param[1:] elif param == "-grid": parameters["grid"] = True elif param == "-shading": @@ -1195,8 +1215,9 @@ def print_general_plot_help(plot_type, shading=False): with a number as the argument . The z-axis refers to the colourbar.""" ) print(""" - Add either "-png" or "-pdf" to the command to save the figure in a file of - the corresponding filetype. + Add either "-png" or "-pdf" or "-svg" to the command to save the figure in + a file of the corresponding filetype. Alternatively, use the command + "save_figure" after creating the plot. The order of the qualifiers starting with a dash ("-") does not play a role, but they must be at the end of the command, i.e., after the variables and From eac0290e053aeca00e8da4d078d74da4f78e4f05 Mon Sep 17 00:00:00 2001 From: Markus Date: Fri, 7 Jun 2019 15:46:33 +0200 Subject: [PATCH 16/31] EMShell: preserve command history after restart This commit also contains changes in same names, docstrings and comments. --- ...ell_Extensions.py => EMShellExtensions.py} | 15 ++++-- experiments/Experiment-Manager.py | 53 +++++++++++++++---- 2 files changed, 55 insertions(+), 13 deletions(-) rename experiments/{EMShell_Extensions.py => EMShellExtensions.py} (74%) diff --git a/experiments/EMShell_Extensions.py b/experiments/EMShellExtensions.py similarity index 74% rename from experiments/EMShell_Extensions.py rename to experiments/EMShellExtensions.py index f581a0f..1410af2 100644 --- a/experiments/EMShell_Extensions.py +++ b/experiments/EMShellExtensions.py @@ -1,13 +1,20 @@ -# Markus Reinert, June 2019 -# -# Extensions for the EMShell, the command-line interface of the fluid2d Experiment Management System (EMS) +"""Extensions for the EMShell of fluid2d + +The EMShell is the command-line interface of the fluid2d Experiment +Management System (EMS). Its functionality for data analysis can be +extended by adding new functions to the dictionary "extra_tools" in the +EMShell. These extensions are defined here, but they can also be +imported from other files. + +Author: Markus REINERT, June 2019 +""" import numpy as np import netCDF4 as nc from scipy.fftpack import fft, fftshift, fftfreq -def get_strongest_wavenumber(his_filename): +def get_strongest_wavenumber_y(his_filename: str) -> float: """Calculate the most intense wavenumber in y-direction. This function opens the given history-file, performs a Fourier diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 90d4992..78e8aa7 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -1,8 +1,29 @@ -# Markus Reinert, May/June 2019 -# -# EMShell: Command-line interface for the fluid2d Experiment Management System (EMS) -# -# This programme requires fluid2d to be activated. +"""EMShell: Command-line interface for the EMS of fluid2d + +EMS is the Experiment Management System of fluid2d, a powerful and +convenient way to handle big sets of experiments, cf. core/ems.py. +The EMShell is a command line interface to access, inspect, modify +and analyse the database and its entries. + +To run this programme, activate fluid2d, then run this script with +Python (version 3.6 or newer). + +This code uses f-strings, which were introduced in Python 3.6: + https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep498 +Unfortunately, they create a SyntaxError in older Python versions. + +It is possible to access the experiment database from multiple +processes at the same time, so one can add new entries to the database +with the EMS of fluid2d while looking at the database and performing +data analysis in the EMShell; the EMShell shows automatically the +updated version of the database. However, this works less well if the +database is accessed by processes on different computers. In this +case it can be necessary to restart the EMShell to see changes in the +database. It is always necessary to restart the EMShell when a new +class of experiments was added to the database. + +Author: Markus Reinert, May/June 2019 +""" import os import re @@ -12,11 +33,12 @@ import shutil import sqlite3 as dbsys import datetime +import readline import itertools import subprocess import matplotlib.pyplot as plt # Local imports -import EMShell_Extensions as EMExt +import EMShellExtensions as EMExt try: from param import Param except ModuleNotFoundError: @@ -37,8 +59,8 @@ # The value is the function, which takes as only argument the path to the # his-file of an experiment and can return the calculated value. extra_tools = { - "wavenumber": EMExt.get_strongest_wavenumber, - "wavelength": lambda hisname: 1/EMExt.get_strongest_wavenumber(hisname), + "wavenumber": EMExt.get_strongest_wavenumber_y, + "wavelength": lambda hisname: 1/EMExt.get_strongest_wavenumber_y(hisname), } ### Settings for the output of a table @@ -315,6 +337,18 @@ def __init__(self, experiments_dir: str): self.intro += "\n" + self.con.get_table_overview() + "\n" self.selected_table = "" self.silent_mode = True + # Settings for saving the command history + self.command_history_file = os.path.join(self.exp_dir, ".emshell_history") + self.command_history_length = 1000 + + + def preloop(self): + if os.path.exists(self.command_history_file): + readline.read_history_file(self.command_history_file) + + def postloop(self): + readline.set_history_length(self.command_history_length) + readline.write_history_file(self.command_history_file) ### Functionality to MODIFY how the programme acts def do_verbose(self, params): @@ -486,7 +520,7 @@ def help_filter(self): - filter intensity <= 0.2 - filter slope = 0.5 - filter diffusion = "True" - - filter datetime >= "2019-03-2 + - filter datetime >= "2019-03-21" - filter perturbation != "gauss" AND duration > 20 It is necessary to put the value for string, datetime or boolean argument in quotation marks as shown. @@ -1009,6 +1043,7 @@ def table_name_completion(self, text): def plot_attribute_completion(self, text, parameters): if not self.selected_table: + print("\nError: select an experiment first!") return [] parameters += self.con.get_column_names(self.selected_table) parameters.extend(extra_tools.keys()) From dbd581599555177a9d6bc1fc1052e41760d97f00 Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 12 Jun 2019 14:24:13 +0200 Subject: [PATCH 17/31] EMShell: Add new functionality and unify behaviour With this commit it is possible to remove tables from the database and to change to comment of entries in the database. The behaviour and the help texts of several commands are made uniform. Instead of an ID, it is now possible to say "last" or "-1". A bug in the ems is fixed which caused a crash when a table was empty. --- core/ems.py | 6 +- experiments/Experiment-Manager.py | 330 ++++++++++++++++++++++-------- 2 files changed, 252 insertions(+), 84 deletions(-) diff --git a/core/ems.py b/core/ems.py index feedef6..00968d9 100644 --- a/core/ems.py +++ b/core/ems.py @@ -111,7 +111,11 @@ def __init__(self, param: "param.Param", experiment_file: str=""): ) # Get the highest index cursor.execute('SELECT id from "{}" ORDER BY id DESC'.format(self.exp_class)) - self.id_ = cursor.fetchone()[0] + 1 + highest_entry = cursor.fetchone() + if highest_entry: + self.id_ = highest_entry[0] + 1 + else: + self.id_ = 1 else: # Create a new table print(' Creating new table "{}".'.format(self.exp_class)) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 78e8aa7..5860b7f 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -186,10 +186,11 @@ def get_table_overview(self): if self.tables: text = "Experiments in database:" for table in self.tables: - text += ( - "\n - {}: {} experiments, {} columns" - .format(table.name, table.get_length(), len(table.columns)) - ) + n_entries = len(table) + n_columns = len(table.columns) + text += f"\n - {table.name}: " + text += f"{n_entries} experiment{'s' if n_entries != 1 else ''}, " + text += f"{n_columns} columns" else: text = "No experiments in database." return text @@ -206,6 +207,23 @@ def get_column_names(self, table_name): if table.name == table_name: return [n for n,t in table.columns] + def get_table_length(self, table_name): + for table in self.tables: + if table.name == table_name: + return len(table) + + def get_highest_index(self, table_name): + for table in self.tables: + if table.name == table_name: + return table.get_highest_index() + + def set_comment(self, table_name, id_, new_comment): + for table in self.tables: + if table.name == table_name: + return table.set_comment(id_, new_comment) + print(f'Unknown experiment: "{table_name}"') + return False + def show_all_tables(self): for table in self.tables: print("-"*50) @@ -245,6 +263,12 @@ def is_valid_column(self, column_name, table_name=""): return True return False + def table_exists(self, table_name): + for table in self.tables: + if table.name == table_name: + return True + return False + def entry_exists(self, table_name, id_): for table in self.tables: if table.name == table_name: @@ -256,6 +280,9 @@ def delete_entry(self, table_name, id_): table.delete_entry(id_) return + def delete_table(self, table_name): + self.connection.execute(f'DROP TABLE "{table_name}"') + class EMDBTable: """Experiment Management Database Table""" @@ -273,7 +300,7 @@ def __str__(self): self.c.execute('SELECT * from "{}"'.format(self.name)) return string_format_table(self.name, self.columns, self.c.fetchall()) - def get_length(self): + def __len__(self): self.c.execute('SELECT Count(*) FROM "{}"'.format(self.name)) return self.c.fetchone()[0] @@ -287,6 +314,11 @@ def get_data(self, column_name, condition=""): else: return [e[0] for e in self.c.fetchall()] + def get_highest_index(self): + self.c.execute('SELECT id from "{}" ORDER BY id DESC'.format(self.name)) + result = self.c.fetchone() + return result[0] if result else None + def entry_exists(self, id_): self.c.execute('SELECT id FROM "{}" WHERE id = ?'.format(self.name), (id_,)) if self.c.fetchone() is not None: @@ -314,6 +346,18 @@ def print_sorted(self, statement): else: print(string_format_table(self.name, self.columns, self.c.fetchall())) + def set_comment(self, id_, new_comment): + try: + self.c.execute( + 'UPDATE "{}" SET comment = ? WHERE id = ?'.format(self.name), + (new_comment, id_,) + ) + except dbsys.OperationalError as e: + print('SQL error for experiment "{}":'.format(self.name), e) + return False + else: + return True + # https://docs.python.org/3/library/cmd.html class EMShell(cmd.Cmd): @@ -340,13 +384,11 @@ def __init__(self, experiments_dir: str): # Settings for saving the command history self.command_history_file = os.path.join(self.exp_dir, ".emshell_history") self.command_history_length = 1000 - - - def preloop(self): + # Load previously saved command history if os.path.exists(self.command_history_file): readline.read_history_file(self.command_history_file) - def postloop(self): + def __del__(self): readline.set_history_length(self.command_history_length) readline.write_history_file(self.command_history_file) @@ -563,9 +605,10 @@ def help_sort(self): ### Functionality to OPEN experiment files def do_open_mp4(self, params): """Open the mp4-file for an experiment specified by its name and ID.""" - expname = self.parse_params_to_experiment(params) - if not expname: + expname_id = self.parse_params_to_experiment(params) + if not expname_id: return + expname = "{}_{:03}".format(*expname_id) dir_ = os.path.join(self.exp_dir, expname) try: files = os.listdir(dir_) @@ -585,17 +628,19 @@ def complete_open_mp4(self, text, line, begidx, endidx): return self.table_name_completion(text) def help_open_mp4(self): - print("Open the mp4-file for an experiment specified by its name and ID.\n" - "It is not possible to interact with the video player via input in the shell, " - "for example with mplayer.\n" - "User input via the graphical interface is not affected by this." - ) + print(f"""> open_mp4 [experiment] + Open the mp4-file of an experiment with {MP4_PLAYER}.""") + self.print_param_parser_help() + print(""" + The programme to open mp4-files with can be configured in the Python script + of the Experiment-Manager with the constant "MP4_PLAYER".""") def do_open_his(self, params): """Open the his-file for an experiment specified by its name and ID.""" - expname = self.parse_params_to_experiment(params) - if not expname: + expname_id = self.parse_params_to_experiment(params) + if not expname_id: return + expname = "{}_{:03}".format(*expname_id) path = os.path.join(self.exp_dir, expname, expname + "_his.nc") if not os.path.isfile(path): print("File does not exist:", path) @@ -605,11 +650,20 @@ def do_open_his(self, params): def complete_open_his(self, text, line, begidx, endidx): return self.table_name_completion(text) + def help_open_his(self): + print(f"""> open_his [experiment] + Open the NetCDF history-file of an experiment with {NETCDF_VIEWER}.""") + self.print_param_parser_help() + print(""" + The programme to open NetCDF-files with can be configured in the Python + script of the Experiment-Manager with the constant "NETCDF_VIEWER".""") + def do_open_diag(self, params): """Open the diag-file for an experiment specified by its name and ID.""" - expname = self.parse_params_to_experiment(params) - if not expname: + expname_id = self.parse_params_to_experiment(params) + if not expname_id: return + expname = "{}_{:03}".format(*expname_id) path = os.path.join(self.exp_dir, expname, expname + "_diag.nc") if not os.path.isfile(path): print("File does not exist:", path) @@ -619,17 +673,22 @@ def do_open_diag(self, params): def complete_open_diag(self, text, line, begidx, endidx): return self.table_name_completion(text) + def help_open_diag(self): + print(f"""> open_diag [experiment] + Open the NetCDF diagnostics-file of an experiment with {NETCDF_VIEWER}.""") + self.print_param_parser_help() + print(""" + The programme to open NetCDF-files with can be configured in the Python + script of the Experiment-Manager with the constant "NETCDF_VIEWER".""") + ### Functionality for Data Analysis def do_calculate(self, params): - if not self.selected_table: - print('No experiment selected. Select an experiment to do calculations on it.') - return - try: - toolname, id_ = params.split() - except ValueError: - print('Invalid syntax. The correct syntax is: calculate . ' - 'Make sure the name of the tool contains no space.') + if " " not in params: + print("At least two arguments are needed.") return + # Parse and check toolname + i = params.find(" ") + toolname = params[:i] if not extra_tools: print('There are no tools for calculations loaded. ' 'Add some tools in the code and restart the programme.') @@ -639,15 +698,12 @@ def do_calculate(self, params): for t in extra_tools: print(" -", t) return - try: - id_ = int(id_) - except ValueError: - print(f'Second parameter is not a valid id: "{id_}".') - return - if not self.con.entry_exists(self.selected_table, id_): - print(f'No entry with the id {id_} exists for the selected experiment.') + # Parse ID and optionally table name + experiment_name_id = self.parse_params_to_experiment(params[i+1:]) + if experiment_name_id is None: return - expname = "{}_{:03}".format(self.selected_table, id_) + table_name, id_ = experiment_name_id + expname = "{}_{:03}".format(table_name, id_) path = os.path.join(self.exp_dir, expname, expname + '_his.nc') try: val = extra_tools[toolname](path) @@ -657,17 +713,18 @@ def do_calculate(self, params): print(e) return if val is not None: - print(f'-> {toolname}({id_}) = {val}') + print(f'-> {toolname}({table_name}:{id_}) = {val}') else: - print(f'-> {toolname}({id_}) succeeded without return value.') + print(f'-> {toolname}({table_name}:{id_}) succeeded without return value.') def complete_calculate(self, text, line, begidx, endidx): return [p for p in extra_tools if p.startswith(text)] def help_calculate(self): - print("""> calculate - Call an EMShell-Extension on an experiment and print the result. - This works always on the entries of the currently selected experiment. + print("""> calculate [experiment] + Call an EMShell-Extension on an experiment and print the result.""") + self.print_param_parser_help() + print(""" To make a tool available, load it into the Python script of the Experiment-Manager and add it to the dictionary "extra_tools". Its name must not contain any whitespace.""") @@ -901,33 +958,72 @@ def help_save_figure(self): "#" is an integer such that the filename is unique and "type" is the given filetype.""") - ### Functionality to CLEAN up - def do_remove(self, params): + ### Functionality to MODIFY the entries + def do_new_comment(self, params): + # Check and parse parameters + table_name, id_ = self.parse_params_to_experiment(params) + # Print the current entry fully global display_all display_all = True + self.con.show_filtered_table(table_name, f"id = {id_}") + # Ask for user input + print("Write a new comment for this entry (Ctrl+D to finish, Ctrl+C to cancel):") + comment_lines = [] + while True: + try: + line = input() + except EOFError: + break + except KeyboardInterrupt: + print("Cancelling.") + return + comment_lines.append(line) + comment = "\n".join(comment_lines).strip() + # Update the comment + if self.con.set_comment(table_name, id_, comment): + self.con.save_database() + print("New comment was saved.") + else: + print("An error occured.") + + def help_new_comment(self): + print("""> new_comment [experiment] + Ask the user to enter a new comment for an experiment.""") + self.print_param_parser_help() + ### Functionality to CLEAN up + def do_remove(self, params): # Check conditions and parameters if not self.selected_table: print('No experiment selected. Select an experiment to remove entries from it.') return if not params: - print('No ids given. Specify ids of the entries to remove.') + print('No IDs given. Specify at least one ID to remove its entry.') return # Parse parameters ids = set() for p in params.split(): - try: - id_ = int(p) - except ValueError: - print('Parameter is not a valid id: ' + p + '. No data removed.') - return - if not self.con.entry_exists(self.selected_table, id_): - print('No entry with id', id_, 'exists in the selected table. No data removed.') - return + if p == "last" or p == "-1": + id_ = self.con.get_highest_index(self.selected_table) + if id_ is None: + print(f'No entries exist for the experiment class "{self.selected_table}".') + return + else: + try: + id_ = int(p) + except ValueError: + print('Parameter', p, 'is not a valid ID. No data removed.') + return + if not self.con.entry_exists(self.selected_table, id_): + print('No entry with ID', id_, 'exists in the selected table. ' + 'No data removed.') + return ids.add(id_) # Print full information of selected entries + global display_all + display_all = True if len(ids) == 1: statement = "id = {}".format(*ids) else: @@ -972,6 +1068,55 @@ def do_remove(self, params): else: print('Answer was not "yes". No data removed.') + def help_remove(self): + print("""> remove + Delete the entry with the given ID from the currently selected class of + experiments and all files associated with it. Multiple IDs can be specified + to remove several entries and their folders at once. Instead of an ID, the + value "last" can be used to choose the entry with the highest ID. The value + "-1" is an alias for "last". Before the data is deleted, the user is asked + to confirm, which must be answered with "yes". + The remove an empty class of experiments, use "remove_selected_class".""") + + def do_remove_selected_class(self, params): + global display_all + display_all = True + + # Check conditions and parameters + if not self.selected_table: + print('No experiment selected. Select an experiment to remove it.') + return + if params: + print('This command takes no attributes. Cancelling.') + return + + if self.con.get_table_length(self.selected_table): + print('Selected experiment class contains experiments. Remove these ' + 'experiments first before removing the class. Cancelling.') + return + + print('Do you really want to permanently delete the experiment class ' + f'"{self.selected_table}"?') + print('This cannot be undone.') + print('The EMShell will exit after the deletion.') + answer = input('Continue [yes/no] ? ') + if answer == 'yes': + self.con.delete_table(self.selected_table) + self.con.save_database() + print(f'Deleted experiment class "{self.selected_table}".') + print(f'EMShell must be restarted to update the list of tables.') + return True + else: + print('Answer was not "yes". No data removed.') + + def help_remove_selected_class(self): + print("""> remove_selected_class + Delete the currently selected class of experiments from the database. + This works only if no entries are associated with this experiment class. + Before the class is removed, the user is asked to confirm, which must be + answered with "yes". To remove entries from the selected class, use the + command "remove".""") + def do_print_disk_info(self, params): # Explanation: https://stackoverflow.com/a/12327880/3661532 statvfs = os.statvfs(self.exp_dir) @@ -1061,40 +1206,50 @@ def column_name_completion(self, table_name, text): def check_table_exists(self, name): return name in [table.name for table in self.con.tables] - def parse_params_to_experiment(self, params): + def parse_params_to_experiment(self, params: str) -> [str, int]: + """Parse and check input of the form "[experiment] ". + + The argument "experiment" is optional, the ID is necessary. + Instead of an ID, "last" can be used to refer to the entry with + the highest ID. The value "-1" is an alias for "last". + If no experiment is given, the selected experiment is taken. + Return None and print a message if input is not valid, otherwise + return the experiment name and the ID.""" + if not params: + print("No ID given.") + return None params = params.split(" ") - # Check for correct parameter input - if len(params) < 2: - if self.selected_table: - if len(params) == 1: - name = self.selected_table - else: - print("Exactly 1 ID must be specified.") - return - else: - print("Name and ID of experiment must be specified.") - return + # Parse the ID + specifier = params.pop() + if specifier == "last": + id_ = -1 else: - # Extract name from input - name = " ".join(params[:-1]).strip() - # Check name - for table in self.con.tables: - if table.name == name: - break + try: + id_ = int(specifier) + except ValueError: + print(f'Last argument is not a valid ID: "{specifier}".') + return None + # Parse and check the table name + table_name = " ".join(params).strip() + if table_name: + if not self.con.table_exists(table_name): + print(f'No experiment class with the name "{table_name}" exists.') + return None else: - print('Unknown experiment: "{}"'.format(name)) - return - # Extract and check ID - try: - id_ = int(params[-1]) - except ValueError: - print('Last argument "{}" is not a valid ID.'.format(params[-1])) - return - if not table.entry_exists(id_): - print("No entry exists in table {} with ID {}.".format(name, id_)) - return - # Return name of experiment folder - return "{}_{:03}".format(name, id_) + if not self.selected_table: + print("No experiment selected and no experiment name given.") + return None + table_name = self.selected_table + # Check the ID + if id_ == -1: + id_ = self.con.get_highest_index(table_name) + if id_ is None: + print(f'No entries exist for the experiment class "{table_name}".') + return None + elif not self.con.entry_exists(table_name, id_): + print(f'No entry with ID {id_} exists for the experiment class "{table_name}".') + return None + return table_name, id_ @staticmethod def parse_plot_parameters(params, n_necessary): @@ -1264,6 +1419,15 @@ def print_general_plot_help(plot_type, shading=False): is superimposed onto the previous one.""" ) + @staticmethod + def print_param_parser_help(): + print(""" + If only an ID and no experiment name is given, take the currently selected + class of experiments. Instead of an ID, the value "last" can be used to + choose the entry with the highest ID. The value "-1" is an alias for + "last".""" + ) + def get_unique_save_filename(template): id_ = 0 From f0c7c3c5ce0e099ad4a9769538b9403dc0ecd41c Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 12 Jun 2019 15:49:41 +0200 Subject: [PATCH 18/31] EMShell: Fix bug which locks database after using 'last'. --- experiments/Experiment-Manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 5860b7f..6f31957 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -317,6 +317,7 @@ def get_data(self, column_name, condition=""): def get_highest_index(self): self.c.execute('SELECT id from "{}" ORDER BY id DESC'.format(self.name)) result = self.c.fetchone() + self.c.fetchall() # otherwise the database stays locked return result[0] if result else None def entry_exists(self, id_): From 5f5895cbd80f1929f0db760813edf301894d54af Mon Sep 17 00:00:00 2001 From: Markus Date: Thu, 13 Jun 2019 14:58:13 +0200 Subject: [PATCH 19/31] Fix output when fluid2d runs on multiple cores When fluid2d runs on multiple cores, the name of the his file displayed at the end was wrong. The filename that was printed after the simulation finished was the name of one of the his-files which was joined into one big his-file and then removed. With this commit, fluid2d prints the correct name of the his-file after the run even when using multiple cores. The same is true for the flux file. Creating an mp4-file while fluid2d runs on multiple cores was possible so far, however, it resulted in the creation of just one file which shows only some part of the domain, not fully. Therefore it was only limited usefully. Furthermore it led to a crash, since several fluid2d-processes called "finalize" on the same file. Also creating an mp4-file, which is not really helpful wastes resouces. Therefore, this commit disables the possbility to generate mp4s when fluid2d runs on multiple cores. A warning is printed instead. The member variables of Output hisfile_joined and flxfile_joined were unused and are now removed. --- core/output.py | 17 +++++++++-------- core/plotting.py | 8 ++++++++ core/restart.py | 6 ------ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/core/output.py b/core/output.py index d75f285..2d12443 100644 --- a/core/output.py +++ b/core/output.py @@ -26,7 +26,6 @@ def __init__(self, param, grid, diag, flxlist=None): self.template = self.expdir+'/%s_his_%03i.nc' if self.nbproc > 1: self.hisfile = self.template % (self.expname, self.myrank) - self.hisfile_joined = '%s/%s_his.nc' % (self.expdir, self.expname) else: self.hisfile = '%s/%s_his.nc' % (self.expdir, self.expname) @@ -43,7 +42,6 @@ def __init__(self, param, grid, diag, flxlist=None): template = self.expdir+'/%s_flx_%03i.nc' if self.nbproc > 1: self.flxfile = template % (self.expname, self.myrank) - self.flxfile_joined = '%s/%s_flx.nc' % (self.expdir, self.expname) else: self.flxfile = '%s/%s_flx.nc' % (self.expdir, self.expname) @@ -154,13 +152,13 @@ def dump_diag(self): def join(self): if self.nbproc > 1: filename = self.hisfile.split('his')[0]+'his' - join(filename) + self.hisfile = join(filename) if self.diag_fluxes: filename = self.flxfile.split('flx')[0]+'flx' - join(filename) - - - + self.flxfile = join(filename) + + + class NcfileIO(object): """Allow to create() and write() a Netcdf file of 'history' type, i.e. a set of model 2D snapshots. Variables were originally the @@ -250,7 +248,8 @@ def join(filename): ''' Join history files without having to mpirun Useful when the run has been broken and one wants to join - things from an interactive session ''' + things from an interactive session. + Return the name of the new, joined history file. ''' template = filename+'_%03i.nc' hisfile_joined = filename+'.nc' @@ -390,3 +389,5 @@ def join(filename): os.remove(ncfile) print('-'*50) + + return hisfile_joined diff --git a/core/plotting.py b/core/plotting.py index 296fee0..38ec41c 100644 --- a/core/plotting.py +++ b/core/plotting.py @@ -22,6 +22,14 @@ def __init__(self, param, grid, var, diag): 'generate_mp4', 'myrank', 'modelname'] param.copy(self, self.list_param) + if param.npx * param.npy > 1 and self.generate_mp4: + print( + 'Warning: It is not possible to generate an mp4-file when ' + 'fluid2d runs on multiple cores.\n' + 'The parameter generate_mp4 is automatically changed to False.' + ) + self.generate_mp4 = False + nh = self.nh nx = param.nx ny = param.ny diff --git a/core/restart.py b/core/restart.py index c012a02..e3541f3 100644 --- a/core/restart.py +++ b/core/restart.py @@ -78,16 +78,10 @@ def __init__(self, param, grid, f2d, launch=True): f2d.output.template = self.expdir +'/%s_%02i_his' % ( self.expname, self.nextrestart)+'_%03i.nc' f2d.output.hisfile = f2d.output.template % (self.myrank) - f2d.output.hisfile_joined = self.expdir + '/%s_%02i_his.nc' % ( - self.expname, self.nextrestart) if self.diag_fluxes: f2d.output.template = self.expdir +'/%s_%02i_flx' % ( self.expname, self.nextrestart)+'_%03i.nc' f2d.output.flxfile = f2d.output.template % (self.myrank) - f2d.output.flxfile_joined = self.expdir + '/%s_%02i_flx.nc' % ( - self.expname, self.nextrestart) - - # split the integration in 'ninterrestart' intervals and # save a restart at the end of each From 5438741b9a05b83ccc5cb4bac302c3d3698f180a Mon Sep 17 00:00:00 2001 From: Markus Date: Thu, 13 Jun 2019 16:18:37 +0200 Subject: [PATCH 20/31] EMS: Make "finalize" save against missing files Also improve the handling of verbose and silent mode in EMShell. --- core/ems.py | 105 ++++++++++++++++++------------ experiments/Experiment-Manager.py | 74 ++++++++++----------- 2 files changed, 98 insertions(+), 81 deletions(-) diff --git a/core/ems.py b/core/ems.py index 00968d9..0acf938 100644 --- a/core/ems.py +++ b/core/ems.py @@ -198,54 +198,77 @@ def finalize(self, fluid2d): that means, after the line with `f2d.loop()`. It also sets the field `datetime` to the current time.""" - # Get duration of the run and size of the output files - integration_time = fluid2d.t - - # Get list of files in the output directory - output_files = [os.path.join(self.output_dir, f) for f in os.listdir(self.output_dir)] - # To get size in MB, divide by 1000*1000 = 1e6. - total_size = sum(os.path.getsize(path) for path in output_files) / 1e6 - his_size = os.path.getsize(fluid2d.output.hisfile) / 1e6 - diag_size = os.path.getsize(fluid2d.output.diagfile) / 1e6 - if fluid2d.plot_interactive and hasattr(fluid2d.plotting, 'mp4file'): - mp4_size = os.path.getsize(fluid2d.plotting.mp4file) / 1e6 - else: - mp4_size = 0.0 - if fluid2d.diag_fluxes: - flux_size = os.path.getsize(fluid2d.output.flxfile) / 1e6 - else: - flux_size = 0.0 - - # Update values in the database + # Write duration of the run into the database self.connection.execute( 'UPDATE "{}" SET duration = ? WHERE id = ?'.format(self.exp_class), - (round(integration_time, 2), self.id_,) - ) - self.connection.execute( - 'UPDATE "{}" SET size_total = ? WHERE id = ?'.format(self.exp_class), - (round(total_size, 3), self.id_,) - ) - self.connection.execute( - 'UPDATE "{}" SET size_mp4 = ? WHERE id = ?'.format(self.exp_class), - (round(mp4_size, 3), self.id_,) - ) - self.connection.execute( - 'UPDATE "{}" SET size_his = ? WHERE id = ?'.format(self.exp_class), - (round(his_size, 3), self.id_,) - ) - self.connection.execute( - 'UPDATE "{}" SET size_diag = ? WHERE id = ?'.format(self.exp_class), - (round(diag_size, 3), self.id_,) - ) - self.connection.execute( - 'UPDATE "{}" SET size_flux = ? WHERE id = ?'.format(self.exp_class), - (round(flux_size, 3), self.id_,) + (round(fluid2d.t, 2), self.id_,) ) + # Update date and time in the database self.connection.execute( 'UPDATE "{}" SET datetime = ? WHERE id = ?'.format(self.exp_class), (datetime.datetime.now().isoformat(timespec="microseconds"), self.id_) ) - + # Write size of output into the database + # Divide size by 1000*1000 = 1e6 to get value in MB + try: + # Get list of files in the output directory + output_files = [os.path.join(self.output_dir, f) for f in os.listdir(self.output_dir)] + total_size = sum(os.path.getsize(path) for path in output_files) + except FileNotFoundError as e: + print(" Error getting total size of output:", e) + else: + self.connection.execute( + 'UPDATE "{}" SET size_total = ? WHERE id = ?'.format(self.exp_class), + (round(total_size / 1e6, 3), self.id_,) + ) + # History file + try: + his_size = os.path.getsize(fluid2d.output.hisfile) + except FileNotFoundError as e: + print(" Error getting size of his-file:", e) + else: + self.connection.execute( + 'UPDATE "{}" SET size_his = ? WHERE id = ?'.format(self.exp_class), + (round(his_size / 1e6, 3), self.id_,) + ) + # Diagnostics file + try: + diag_size = os.path.getsize(fluid2d.output.diagfile) + except FileNotFoundError: + print(" Error getting size of diag-file:", e) + else: + self.connection.execute( + 'UPDATE "{}" SET size_diag = ? WHERE id = ?'.format(self.exp_class), + (round(diag_size / 1e6, 3), self.id_,) + ) + # MP4 file + mp4_size = -1.0 + if fluid2d.plot_interactive and hasattr(fluid2d.plotting, 'mp4file'): + try: + mp4_size = os.path.getsize(fluid2d.plotting.mp4file) + except FileNotFoundError: + print(" Error getting size of mp4-file:", e) + else: + mp4_size = 0.0 + if mp4_size >= 0: + self.connection.execute( + 'UPDATE "{}" SET size_mp4 = ? WHERE id = ?'.format(self.exp_class), + (round(mp4_size / 1e6, 3), self.id_,) + ) + # Flux file + flux_size = -1.0 + if fluid2d.diag_fluxes: + try: + flux_size = os.path.getsize(fluid2d.output.flxfile) + except FileNotFoundError: + print(" Error getting size of flux-file:", e) + else: + flux_size = 0.0 + if flux_size >= 0: + self.connection.execute( + 'UPDATE "{}" SET size_flux = ? WHERE id = ?'.format(self.exp_class), + (round(flux_size / 1e6, 3), self.id_,) + ) # Save the database self.connection.commit() diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 6f31957..4757725 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -381,7 +381,6 @@ def __init__(self, experiments_dir: str): self.con = EMDBConnection(os.path.join(self.exp_dir, "experiments.db")) self.intro += "\n" + self.con.get_table_overview() + "\n" self.selected_table = "" - self.silent_mode = True # Settings for saving the command history self.command_history_file = os.path.join(self.exp_dir, ".emshell_history") self.command_history_length = 1000 @@ -394,32 +393,6 @@ def __del__(self): readline.write_history_file(self.command_history_file) ### Functionality to MODIFY how the programme acts - def do_verbose(self, params): - if params == "" or params.lower() == "on": - self.silent_mode = False - elif params.lower() == "off": - self.silent_mode = True - else: - print('Unknown parameter. Please use "verbose on" or "verbose off".') - - def complete_verbose(self, text, line, begidx, endidx): - if text == "" or text == "o": - return ["on", "off"] - if text == "of": - return ["off",] - return [] - - def help_verbose(self): - print( - "Toggle between verbose- and silent-mode.\n" - 'Use "verbose on" or "verbose" to see the output of external programmes ' - 'started from this shell.\n' - 'Use "verbose off" to hide all output of external programmes (default).\n' - 'No error message is displayed in silent-mode when the opening of a file fails.\n' - 'In any case, external programmes are started in the background, so the shell can ' - 'still be used, even if the input prompt is polluted.' - ) - def do_enable(self, params): global hidden_information if not params: @@ -606,6 +579,11 @@ def help_sort(self): ### Functionality to OPEN experiment files def do_open_mp4(self, params): """Open the mp4-file for an experiment specified by its name and ID.""" + if params.endswith(" -v"): + verbose = True + params = params[:-3].rstrip() + else: + verbose = False expname_id = self.parse_params_to_experiment(params) if not expname_id: return @@ -623,21 +601,28 @@ def do_open_mp4(self, params): print("No mp4-file found in folder:", dir_) return path = os.path.join(self.exp_dir, expname, f) - self.open_file(MP4_PLAYER, path) + self.open_file(MP4_PLAYER, path, verbose) def complete_open_mp4(self, text, line, begidx, endidx): return self.table_name_completion(text) def help_open_mp4(self): - print(f"""> open_mp4 [experiment] + print(f"""> open_mp4 [experiment] [-v] Open the mp4-file of an experiment with {MP4_PLAYER}.""") self.print_param_parser_help() print(""" + Add "-v" to the command to see the output of the external programme. + The programme to open mp4-files with can be configured in the Python script of the Experiment-Manager with the constant "MP4_PLAYER".""") def do_open_his(self, params): """Open the his-file for an experiment specified by its name and ID.""" + if params.endswith(" -v"): + verbose = True + params = params[:-3].rstrip() + else: + verbose = False expname_id = self.parse_params_to_experiment(params) if not expname_id: return @@ -646,21 +631,28 @@ def do_open_his(self, params): if not os.path.isfile(path): print("File does not exist:", path) return - self.open_file(NETCDF_VIEWER, path) + self.open_file(NETCDF_VIEWER, path, verbose) def complete_open_his(self, text, line, begidx, endidx): return self.table_name_completion(text) def help_open_his(self): - print(f"""> open_his [experiment] + print(f"""> open_his [experiment] [-v] Open the NetCDF history-file of an experiment with {NETCDF_VIEWER}.""") self.print_param_parser_help() print(""" + Add "-v" to the command to see the output of the external programme. + The programme to open NetCDF-files with can be configured in the Python script of the Experiment-Manager with the constant "NETCDF_VIEWER".""") def do_open_diag(self, params): """Open the diag-file for an experiment specified by its name and ID.""" + if params.endswith(" -v"): + verbose = True + params = params[:-3].rstrip() + else: + verbose = False expname_id = self.parse_params_to_experiment(params) if not expname_id: return @@ -669,16 +661,18 @@ def do_open_diag(self, params): if not os.path.isfile(path): print("File does not exist:", path) return - self.open_file(NETCDF_VIEWER, path) + self.open_file(NETCDF_VIEWER, path, verbose) def complete_open_diag(self, text, line, begidx, endidx): return self.table_name_completion(text) def help_open_diag(self): - print(f"""> open_diag [experiment] + print(f"""> open_diag [experiment] [-v] Open the NetCDF diagnostics-file of an experiment with {NETCDF_VIEWER}.""") self.print_param_parser_help() print(""" + Add "-v" to the command to see the output of the external programme. + The programme to open NetCDF-files with can be configured in the Python script of the Experiment-Manager with the constant "NETCDF_VIEWER".""") @@ -1307,23 +1301,23 @@ def parse_plot_parameters(params, n_necessary): parameters["condition"] = " ".join(param_list).strip() return parameters - def open_file(self, command, path): - if self.silent_mode: - print("Opening file {} with {} in silent-mode.".format(path, command)) + def open_file(self, command, path, verbose=False): + if verbose: + print("Opening file {} with {} in verbose-mode.".format(path, command)) subprocess.Popen( [command, path], # Disable standard input via the shell, for example with mplayer. stdin=subprocess.DEVNULL, - # Throw away output and error messages. - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, ) else: - print("Opening file {} with {} in verbose-mode.".format(path, command)) + print("Opening file {} with {} silently.".format(path, command)) subprocess.Popen( [command, path], # Disable standard input via the shell, for example with mplayer. stdin=subprocess.DEVNULL, + # Throw away output and error messages. + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) def get_multiple_data(self, table_name, variables, condition): From f0a72f49d335aa1ba8facd771b939bf2a003cfc9 Mon Sep 17 00:00:00 2001 From: Markus Date: Fri, 14 Jun 2019 13:39:13 +0200 Subject: [PATCH 21/31] EMS: add support for Python 3.5 Usage of the parameter timespec in datetime.isoformat() made it impossible to use EMS with Python 3.5. The parameter timespec is removed now, which required a bit of fine-tuning in th EMShell. The EMShell itself still requires Python 3.6 however. Also improve one error message in the EMS. --- core/ems.py | 8 ++++++-- experiments/Experiment-Manager.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/ems.py b/core/ems.py index 0acf938..c627921 100644 --- a/core/ems.py +++ b/core/ems.py @@ -79,7 +79,7 @@ def __init__(self, param: "param.Param", experiment_file: str=""): [ ["INTEGER", "id", -1], ["TEXT", "datetime", - datetime.datetime.now().isoformat(timespec="microseconds")], + datetime.datetime.now().isoformat()], ] + param_list + [ @@ -164,6 +164,10 @@ def __init__(self, param: "param.Param", experiment_file: str=""): columns = cursor.fetchall() cursor.execute('SELECT * FROM "{}" WHERE id = ?'.format(self.exp_class), (self.id_,)) values = cursor.fetchone() + if values is None: + raise ParamError( + 'No entry with id {} exists in table "{}".'.format(self.id_, self.exp_class) + ) for column in columns: col_index = column[0] col_name = column[1] @@ -206,7 +210,7 @@ def finalize(self, fluid2d): # Update date and time in the database self.connection.execute( 'UPDATE "{}" SET datetime = ? WHERE id = ?'.format(self.exp_class), - (datetime.datetime.now().isoformat(timespec="microseconds"), self.id_) + (datetime.datetime.now().isoformat(), self.id_) ) # Write size of output into the database # Divide size by 1000*1000 = 1e6 to get value in MB diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 4757725..5012789 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -1487,7 +1487,7 @@ def string_format_table(table_name, columns, rows): if "datetime_year" in hidden_information: val = val[5:] if "datetime_seconds" in hidden_information: - val = val[:-10] + val = val.split(".")[0][:-3] val = val.replace('T', ',') else: # Create datetime-object from ISO-format From 94defebf43d5ee5ff414da811c66bfa3c630248b Mon Sep 17 00:00:00 2001 From: Markus Date: Fri, 14 Jun 2019 15:31:23 +0200 Subject: [PATCH 22/31] EMShell: (Re)add the option to run without fluid2d It proved useful to start the EMShell by specifying the path to the experiment folder instead of always taking the default expdir path. This possibility, which existed before, is re-added. --- experiments/Experiment-Manager.py | 35 +++++++++++++++++++------------ 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 5012789..baafdc2 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -5,8 +5,10 @@ The EMShell is a command line interface to access, inspect, modify and analyse the database and its entries. -To run this programme, activate fluid2d, then run this script with -Python (version 3.6 or newer). +To start this programme, activate fluid2d, then run this script with +Python (version 3.6 or newer). Alternatively, without the need to +activate fluid2d, specify the path to the experiment folder as a +command-line argument when launching this script with Python. This code uses f-strings, which were introduced in Python 3.6: https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep498 @@ -27,6 +29,7 @@ class of experiments was added to the database. import os import re +import sys import cmd import time import numpy as np @@ -39,12 +42,6 @@ class of experiments was added to the database. import matplotlib.pyplot as plt # Local imports import EMShellExtensions as EMExt -try: - from param import Param -except ModuleNotFoundError: - raise Exception( - "Please activate fluid2d to use this programme!" - ) # Command to open mp4-files @@ -1600,11 +1597,23 @@ def make_nice_time_string(datetime_object, datetime_reference): # Get the directory of the experiments -param = Param(None) # it is not necessary to specify a defaultfile for Param -datadir = param.datadir -del param -if datadir.startswith("~"): - datadir = os.path.expanduser(datadir) +if len(sys.argv) == 1: + try: + from param import Param + except ModuleNotFoundError: + raise Exception( + "Please activate fluid2d or specify the experiment-folder as " + "argument when starting this programme." + ) + param = Param(None) # it is not necessary to specify a defaultfile for Param + datadir = param.datadir + del param + if datadir.startswith("~"): + datadir = os.path.expanduser(datadir) +elif len(sys.argv) == 2: + datadir = sys.argv[1] +else: + raise Exception("More than one argument given.") # Use fancy colours during 6 days of Carnival try: From 40d6ca99843d1e5707e51e9a5f2c3dba55d2d53f Mon Sep 17 00:00:00 2001 From: Markus Date: Fri, 14 Jun 2019 16:33:21 +0200 Subject: [PATCH 23/31] EMShell: Add command to make >1 plots at same time --- experiments/Experiment-Manager.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index baafdc2..de22811 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -943,12 +943,16 @@ def complete_save_figure(self, text, line, begidx, endidx): return [n for n in (stub + ".png", stub + ".pdf", stub + ".svg") if n.startswith(text)] def help_save_figure(self): - print("""> save_figure + print("""> save_figure [filename or -type] Save the currently opened figure in a file. One can either specify the full filename or one of the filetypes png, pdf or svg alone. If only the type is specified, then the name is automatically set to "figure_#.type", where "#" is an integer such that the filename is unique and "type" is the given - filetype.""") + filetype. If no type and no name is given, the figure is saved as png.""") + + def do_new_figure(self, params): + """Open a window to draw the next plot in a new figure.""" + plt.figure() ### Functionality to MODIFY the entries def do_new_comment(self, params): @@ -1408,7 +1412,9 @@ def print_general_plot_help(plot_type, shading=False): While the window with the plot is open, the shell can be used as usual. If another plot command is executed with the figure still open, the new plot - is superimposed onto the previous one.""" + is superimposed onto the previous one. To draw the next plot command in a + new figure instead, while keeping the previous window open, use the command + "new_figure".""" ) @staticmethod From ff606a228ad47be468c471bdd6db1e6e477b8351 Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 17 Jun 2019 14:39:49 +0200 Subject: [PATCH 24/31] EMShell: Add command to compare entries in a table --- experiments/Experiment-Manager.py | 226 +++++++++++++++++++++++------- 1 file changed, 172 insertions(+), 54 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index de22811..472612c 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -81,18 +81,18 @@ class of experiments was added to the database. ISO_DATETIME = False ### Settings to print tables in colour -# Set COLOURS to False or None to disable colours. +# Set COLOURS to False or None to disable colours (this does not disable highlighting). # Otherwise, COLOURS must be list, of which each element describes the colours used in one row. # The colours of every row are described by a list of colour codes, # cf. https://en.wikipedia.org/wiki/ANSI_escape_code#Colors. # An arbitrary number of colours can be specified. # Apart from background colours, also text colours and font styles can be specified. -# Example to colour rows alternately with white and default colour: +# Example to fill rows alternately with white and default colour: # COLOURS = ( # ("\033[47m",), # ("\033[49m",), # ) -# Example to colour columns alternately with white and default colour: +# Example to fill columns alternately with white and default colour: # COLOURS = ( # ("\033[47m", "\033[49m",), # ) @@ -110,6 +110,8 @@ class of experiments was added to the database. ("\033[107m",), ("\033[49m",), ) +# Colour-code to highlight columns (used by "compare -h") +COLOURS_HIGHLIGHT = "\033[103m" # Colour-code to reset to default colour and default font COLOURS_END = "\033[39;49m" @@ -160,6 +162,9 @@ class EMDBConnection: """Experiment Management Database Connection""" def __init__(self, dbpath: str): + self.connection = None + if not os.path.isfile(dbpath): + raise FileNotFoundError(f"Database file {dbpath} does not exist.") # Create a connection to the given database print("-"*50) print("Opening database {}.".format(dbpath)) @@ -172,9 +177,10 @@ def __init__(self, dbpath: str): self.tables = [EMDBTable(cursor, t[0]) for t in cursor.fetchall()] def __del__(self): - print("Closing database.") - print("-"*50) - self.connection.close() + if self.connection: + print("Closing database.") + print("-"*50) + self.connection.close() def save_database(self): self.connection.commit() @@ -248,6 +254,14 @@ def show_sorted_table(self, table_name, statement): table.print_sorted(statement) print("-"*50) + def show_comparison(self, table_name, ids, highlight=False): + for table in self.tables: + if table.name == table_name: + print("-"*50) + table.print_comparison(ids, highlight) + print("-"*50) + return + def is_valid_column(self, column_name, table_name=""): """Check whether a column with the given name exists. @@ -303,8 +317,11 @@ def __len__(self): def get_data(self, column_name, condition=""): try: - self.c.execute(f'SELECT {column_name} FROM "{self.name}" WHERE {condition}' if condition else - f'SELECT {column_name} FROM "{self.name}"') + self.c.execute( + f'SELECT {column_name} FROM "{self.name}" WHERE {condition}' + if condition else + f'SELECT {column_name} FROM "{self.name}"' + ) except dbsys.OperationalError as e: print(f'SQL error for experiment "{self.name}":', e) return [] @@ -328,8 +345,7 @@ def delete_entry(self, id_): def print_selection(self, statement): try: - self.c.execute('SELECT * FROM "{}" WHERE {}' - .format(self.name, statement)) + self.c.execute('SELECT * FROM "{}" WHERE {}'.format(self.name, statement)) except dbsys.OperationalError as e: print('SQL error for experiment "{}":'.format(self.name), e) else: @@ -337,13 +353,47 @@ def print_selection(self, statement): def print_sorted(self, statement): try: - self.c.execute('SELECT * FROM "{}" ORDER BY {}' - .format(self.name, statement)) + self.c.execute('SELECT * FROM "{}" ORDER BY {}'.format(self.name, statement)) except dbsys.OperationalError as e: print('SQL error for experiment "{}":'.format(self.name), e) else: print(string_format_table(self.name, self.columns, self.c.fetchall())) + def print_comparison(self, ids, highlight=False): + try: + self.c.execute( + 'SELECT * FROM "{}" WHERE id IN {}'.format(self.name, tuple(ids)) + if ids else + 'SELECT * FROM "{}"'.format(self.name) + ) + except dbsys.OperationalError as e: + print('SQL error for experiment "{}":'.format(self.name), e) + return + full_entries = self.c.fetchall() + different_columns = [] + for i, row in enumerate(full_entries): + if i == 0: + continue + for c, value in enumerate(row): + if c not in different_columns and value != full_entries[i-1][c]: + different_columns.append(c) + if not highlight: + # Print only the columns of the table with differences + print(string_format_table( + self.name, + [col for c, col in enumerate(self.columns) if c in different_columns], + [ + [val for c, val in enumerate(row) if c in different_columns] + for row in full_entries + ] + )) + else: + # Print the full table and highlight the columns with differences + print(string_format_table( + self.name, self.columns, full_entries, + [col[0] for c, col in enumerate(self.columns) if c in different_columns] + )) + def set_comment(self, id_, new_comment): try: self.c.execute( @@ -369,8 +419,10 @@ class EMShell(cmd.Cmd): + "\n" ) prompt = "(EMS) " + ruler = "-" def __init__(self, experiments_dir: str): + self.initialized = False super().__init__() print("-"*50) print("Fluid2d Experiment Management System (EMS)") @@ -380,14 +432,16 @@ def __init__(self, experiments_dir: str): self.selected_table = "" # Settings for saving the command history self.command_history_file = os.path.join(self.exp_dir, ".emshell_history") - self.command_history_length = 1000 + readline.set_history_length(1000) # Load previously saved command history if os.path.exists(self.command_history_file): readline.read_history_file(self.command_history_file) + self.initialized = True def __del__(self): - readline.set_history_length(self.command_history_length) - readline.write_history_file(self.command_history_file) + if self.initialized: + print("Saving command history.") + readline.write_history_file(self.command_history_file) ### Functionality to MODIFY how the programme acts def do_enable(self, params): @@ -573,6 +627,49 @@ def help_sort(self): See also: filter, show.""") + def do_compare(self, params): + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to make a comparison.') + return + # Parse the arguments + global display_all + display_all = False + highlight = False + other_params = [] + for p in params.split(): + if p == "-v": + display_all = True + elif p == "-h": + highlight = True + else: + other_params.append(p) + if other_params: + ids = self.parse_multiple_ids(other_params) + if ids is None: + return + if len(ids) < 2: + print("Please specify at least 2 different IDs.") + return + elif self.con.get_table_length(self.selected_table) < 2: + print("Selected experiment contains less than 2 entries.") + return + else: + # No IDs means no filtering, i.e., all entries are compared + ids = [] + self.con.show_comparison(self.selected_table, ids, highlight) + + def help_compare(self): + print("""> compare [IDs] [-h] [-v] + Show the difference between two or more entries of the selected experiment. + This prints a table which includes only the parameters in which the + specified entries differ. Alternatively, add "-h" to show all parameters + and highlight the differences in colour. Disabled columns are not shown by + default. Add the argument "-v" to show disabled columns, the full date-time + and the full comment. Instead of an ID, "last" or "-1" can be used to + compare with the entry of highest ID. If no ID is specified, all entries of + the selected experiment class are compared.""") + ### Functionality to OPEN experiment files def do_open_mp4(self, params): """Open the mp4-file for an experiment specified by its name and ID.""" @@ -996,26 +1093,11 @@ def do_remove(self, params): if not params: print('No IDs given. Specify at least one ID to remove its entry.') return - # Parse parameters - ids = set() - for p in params.split(): - if p == "last" or p == "-1": - id_ = self.con.get_highest_index(self.selected_table) - if id_ is None: - print(f'No entries exist for the experiment class "{self.selected_table}".') - return - else: - try: - id_ = int(p) - except ValueError: - print('Parameter', p, 'is not a valid ID. No data removed.') - return - if not self.con.entry_exists(self.selected_table, id_): - print('No entry with ID', id_, 'exists in the selected table. ' - 'No data removed.') - return - ids.add(id_) + ids = self.parse_multiple_ids(params.split()) + if ids is None: + print("No data removed.") + return # Print full information of selected entries global display_all @@ -1065,13 +1147,13 @@ def do_remove(self, params): print('Answer was not "yes". No data removed.') def help_remove(self): - print("""> remove + print("""> remove Delete the entry with the given ID from the currently selected class of experiments and all files associated with it. Multiple IDs can be specified to remove several entries and their folders at once. Instead of an ID, the - value "last" can be used to choose the entry with the highest ID. The value - "-1" is an alias for "last". Before the data is deleted, the user is asked - to confirm, which must be answered with "yes". + argument "last" or "-1" can be used to choose the entry with the highest ID. + Before the data is deleted, the user is asked to confirm, which must be + answered with "yes". The remove an empty class of experiments, use "remove_selected_class".""") def do_remove_selected_class(self, params): @@ -1240,7 +1322,7 @@ def parse_params_to_experiment(self, params: str) -> [str, int]: if id_ == -1: id_ = self.con.get_highest_index(table_name) if id_ is None: - print(f'No entries exist for the experiment class "{table_name}".') + print(f'No entry exists for the experiment class "{table_name}".') return None elif not self.con.entry_exists(table_name, id_): print(f'No entry with ID {id_} exists for the experiment class "{table_name}".') @@ -1302,6 +1384,36 @@ def parse_plot_parameters(params, n_necessary): parameters["condition"] = " ".join(param_list).strip() return parameters + def parse_multiple_ids(self, param_list): + """Transform the given list of strings into a list of IDs. + + This method also checks that the IDs are valid entries of the + selected database and returns None if an invalid ID is given. + Otherwise, a sorted list of unique values is returned. + It correctly parses "last" or "-1".""" + ids = [] + for id_ in param_list: + if id_ == "last" or id_ == "-1": + id_ = self.con.get_highest_index(self.selected_table) + if id_ is None: + print(f'No entry exists for the experiment class "{self.selected_table}".') + return + elif id_ in ids: + continue + else: + try: + id_ = int(id_) + except ValueError: + print('Parameter', id_, 'is not a valid ID.') + return + if id_ in ids: + continue + if not self.con.entry_exists(self.selected_table, id_): + print('No entry with ID', id_, 'exists for the selected experiment.') + return + ids.append(id_) + return sorted(ids) + def open_file(self, command, path, verbose=False): if verbose: print("Opening file {} with {} in verbose-mode.".format(path, command)) @@ -1437,11 +1549,12 @@ def get_unique_save_filename(template): return filename -def string_format_table(table_name, columns, rows): +def string_format_table(table_name, columns, rows, highlight_columns=[]): # Get current date and time to make datetime easy to read if not ISO_DATETIME: dt_now = datetime.datetime.now() - # Convert rows to list of lists (because fetchall() returns a list of tuples) + # Convert rows to list of lists (because fetchall() returns a list of tuples + # and to work on a copy of it) rows = [list(row) for row in rows] # Remove ignored columns columns = columns.copy() @@ -1517,8 +1630,8 @@ def string_format_table(table_name, columns, rows): ) if ( COMMENT_MAX_LENGTH == "AUTO" - and "comment" not in hidden_information and not display_all + and "comment" == columns[-1][0] ): line_length = shutil.get_terminal_size().columns total_length = sum(lengths[:-1]) + len(LIMITER) * len(lengths[:-1]) @@ -1539,26 +1652,31 @@ def string_format_table(table_name, columns, rows): elif len(indices_to_remove) > 1: text += " ({} parameters hidden)".format(len(indices_to_remove)) text += "\n" - # Add column name centred, except the last one - text += LIMITER.join([ - ("{:^" + str(l) + "}").format(n) - for (n, t), l in zip(columns[:-1], lengths[:-1]) - ] + [columns[-1][0]]) + "\n" + column_names_formatted = [] format_strings = [] for (n, t), l in zip(columns, lengths): + # Column names centred, except comment, since it is the last column. + cname_form = f"{n:^{l}}" # Numbers right justified, - # text centred, - # comments unformatted + # comments left justified, + # other texts centred. if n in ["size_diag", "size_flux", "size_his", "size_mp4", "size_total"]: - format_strings.append(SIZE_FORMAT.replace(":", ":>"+str(l))) + f_string = SIZE_FORMAT.replace(":", ":>"+str(l)) elif t == "REAL": - format_strings.append(FLOAT_FORMAT.replace(":", ":>"+str(l))) + f_string = FLOAT_FORMAT.replace(":", ":>"+str(l)) elif t == "INTEGER": - format_strings.append("{:>" + str(l) + "}") + f_string = "{:>" + str(l) + "}" elif n == "comment": - format_strings.append("{:" + str(l) + "}") + f_string = "{:" + str(l) + "}" + cname_form = n else: - format_strings.append("{:^" + str(l) + "}") + f_string = "{:^" + str(l) + "}" + if n in highlight_columns: + f_string = COLOURS_HIGHLIGHT + f_string + COLOURS_END + cname_form = COLOURS_HIGHLIGHT + cname_form + COLOURS_END + format_strings.append(f_string) + column_names_formatted.append(cname_form) + text += LIMITER.join(column_names_formatted) + "\n" if COLOURS: row_colours = itertools.cycle(COLOURS) for row in rows: From 9c2c3f4240b924b8f8f8ce764ea1fc36b453d938 Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 17 Jun 2019 15:00:56 +0200 Subject: [PATCH 25/31] EMShell: Change def-value from empty list to None It is proposed by Codacy to not use an empty list as the default value of a function: http://pylint-messages.wikidot.com/messages:w0102 So instead, the value None is used. --- experiments/Experiment-Manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index 472612c..f868b73 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -1549,7 +1549,7 @@ def get_unique_save_filename(template): return filename -def string_format_table(table_name, columns, rows, highlight_columns=[]): +def string_format_table(table_name, columns, rows, highlight_columns=None): # Get current date and time to make datetime easy to read if not ISO_DATETIME: dt_now = datetime.datetime.now() @@ -1671,7 +1671,7 @@ def string_format_table(table_name, columns, rows, highlight_columns=[]): cname_form = n else: f_string = "{:^" + str(l) + "}" - if n in highlight_columns: + if highlight_columns is not None and n in highlight_columns: f_string = COLOURS_HIGHLIGHT + f_string + COLOURS_END cname_form = COLOURS_HIGHLIGHT + cname_form + COLOURS_END format_strings.append(f_string) From 46aa07412c44508cfb54c3c38284feae37f2815c Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 19 Jun 2019 01:00:01 +0200 Subject: [PATCH 26/31] EMS: Implement support for multi values and cores The EMS is almost fully reworked in this commit. It now contains the long desired feature of specifying multiple values in the experiment file and automatically running one experiment for every possible combination of these values. Once again, this is demonstrated in the Waves experiment. Furthermore, the EMS supports now natively multi-threading, which required some manual hacks before. However, this required to add the EMS to the Param class, which, at the end, made the whole implementation a lot cleaner and easier for the user. In consequence, this commit modified, for the first time, files of the standard distribution of fluid2d. The influence was kept as tiny as possbile. As before, the EMS does not make a difference for the user unless he activates it, which brings now a whole new experience! --- core/ems.py | 656 ++++++++++--------- core/fluid2d.py | 1 + core/output.py | 6 +- core/param.py | 26 +- core/plotting.py | 14 +- experiments/Experiment-Manager.py | 37 +- experiments/Waves with EMS/wave_breaking.exp | 38 +- experiments/Waves with EMS/wave_breaking.py | 282 ++++---- 8 files changed, 559 insertions(+), 501 deletions(-) diff --git a/core/ems.py b/core/ems.py index c627921..23ef03c 100644 --- a/core/ems.py +++ b/core/ems.py @@ -1,20 +1,29 @@ -# Markus Reinert, May 2019 -# -# The fluid2d Experiment Management System (EMS). -# -# This module provides a way to handle big sets of fluid2d-experiments. -# It creates a database in which information about all the experiments -# is stored. This allows to easily show, compare, search for, filter -# and save comments to already executed experiments. Furthermore, it -# provides, after set up for a specific experiment, a simpler interface -# to modify the parameteres of interest. To avoid overwriting files, -# it automatically assigns a unique identifier to every new experiment. -# -# Its usage is explained in the breaking waves experiment. +"""EMS: The fluid2d Experiment Management System + +The Experiment Management System provides a convenient way to handle big +sets of fluid2d-experiments. It creates a database in which information +about all the performed experiments is stored. This allows to easily +show, compare, search, filter, organise, analyse them, and a lot more. +Furthermore, the EMS offers a simpler way to modify the parameters of +interest in an experiment, including the automation of running an +experiment several times with different parameter values. To prevent +overwriting files, the EMS automatically assigns a unique identifier for +every new run of an experiment. However, if fluid2d runs on multiple +cores, an ID has to be specified initially. + +Its usage is explained in detail in the experiment on breaking waves. + +With the EMShell, a command line interface is provided to access the +experiment-database created by the EMS. + +Author: Markus Reinert, May/June 2019 +""" import os import datetime import sqlite3 as dbsys +from collections import namedtuple +from itertools import product # Columns with the following names are automatically added to every table of the database. @@ -33,249 +42,272 @@ class InputMismatch(Exception): - """Incompatibility of given experiment file and database.""" + """Incompatibility between experiment file and corresponding table in the database.""" pass -class ParamError(Exception): - """Non-conformance of user-set attributes of param.""" +class ExpFileError(Exception): + """Malformed experiment file.""" pass -class ExpFileError(Exception): - """Malformed experiment file.""" +class DatabaseError(Exception): + """Malformed table in the experiment database.""" + pass + + +class NotInitializedError(Exception): + """Call to `EMS.initialize` required before this action.""" pass class EMS: """Experiment Management System""" - def __init__(self, param: "param.Param", experiment_file: str=""): - """Constructor of the Experiment Management System. - - Set up a connection to the database. If an experiment file is - given, a new entry in the database is created for this file. - Otherwise an existing entry corresponding to `param.expname` is - used. The parameters of this entry are loaded. - They can be accessed via the method `get_parameters`.""" + def __init__(self, experiment_file: str): + """Load the parameters from the experiment file.""" + self.connection = None + self.cursor = None + self.exp_class, self.exp_id, self.description, self.param_name_list, param_values_list = parse_experiment_file(experiment_file) + self.param_combinations = product(*param_values_list) + self.setup_next_parameters(increase_id=False) - # Get the path to the database and create a connection - datadir = param.datadir - if datadir.startswith("~"): - datadir = os.path.expanduser(param.datadir) - dbpath = os.path.join(datadir, "experiments.db") - print("-"*50) - print(" Opening database {}.".format(dbpath)) - self.connection = dbsys.connect(dbpath) - cursor = self.connection.cursor() - - if experiment_file: - # If an experiment file is given, parse it and add a new entry to the database. - self.exp_class, self.description, param_list = parse_experiment_file(experiment_file) - if not self.exp_class: - raise ExpFileError("no name given in file {}.".format(experiment_file)) - # Add static parameter fields - param_list = ( - [ - ["INTEGER", "id", -1], - ["TEXT", "datetime", - datetime.datetime.now().isoformat()], - ] - + param_list - + [ - ["REAL", "duration", -1], - ["REAL", "size_total", -1], - ["REAL", "size_mp4", -1], - ["REAL", "size_his", -1], - ["REAL", "size_diag", -1], - ["REAL", "size_flux", -1], - ["TEXT", "comment", self.description], - ] - # Everything added here should be in RESERVED_COLUMN_NAMES + def __del__(self): + """Close the connection to the experiment database if any.""" + if self.connection: + print(" Closing database.") + print("-"*50) + self.connection.close() + + def get_expname(self): + if self.exp_id is None: + raise NotInitializedError("ID not set.") + return "{}_{:03}".format(self.exp_class, self.exp_id) + + def setup_next_parameters(self, increase_id=True): + try: + self.parameters = { + name: val for name, val in + zip(self.param_name_list, next(self.param_combinations)) + } + except StopIteration: + self.parameters = None + if increase_id: + self.exp_id += 1 + + def initialize(self, data_directory: str): + if not self.connection: + self.connect(data_directory) + # All of the names in the next two lists should be in RESERVED_COLUMN_NAMES + DBColumn = namedtuple("DBColumn", ["sql_type", "name", "value"]) + static_columns_start = [ + DBColumn("INTEGER", "id", -1), + DBColumn("TEXT", "datetime", datetime.datetime.now().isoformat()), + ] + static_columns_end = [ + DBColumn("REAL", "duration", -1), + DBColumn("REAL", "size_total", -1), + DBColumn("REAL", "size_mp4", -1), + DBColumn("REAL", "size_his", -1), + DBColumn("REAL", "size_diag", -1), + DBColumn("REAL", "size_flux", -1), + DBColumn("TEXT", "comment", self.description), + ] + new_columns = ( + static_columns_start + + [DBColumn(sql_type(val), name, val) for name, val in self.parameters.items()] + + static_columns_end + ) + # If the table exists, check its columns, otherwise create it + if self.table_exists(self.exp_class): + self.cursor.execute('PRAGMA table_info("{}")'.format(self.exp_class)) + columns = self.cursor.fetchall() + # Check static columns + for st_col in static_columns_start: + column = columns.pop(0) + col_index = column[0] + col_name = column[1] + col_type = column[2] + if col_name != st_col.name or col_type != st_col.sql_type: + raise DatabaseError( + "expected column {} in the database to be {} {} but is {} {}." + .format(col_index, st_col.sql_type, st_col.name, col_type, col_name) + ) + for st_col in reversed(static_columns_end): + column = columns.pop() + col_index = column[0] + col_name = column[1] + col_type = column[2] + if col_name != st_col.name or col_type != st_col.sql_type: + raise DatabaseError( + "expected column {} in the database to be {} {} but is {} {}." + .format(col_index, st_col.sql_type, st_col.name, col_type, col_name) + ) + # Check user-defined columns + # TODO: be more flexible here: + # - allow to skip columns which are no longer needed + # - allow to add new columns if needed + # - allow to convert types + for name, val in self.parameters.items(): + column = columns.pop(0) + col_index = column[0] + col_name = column[1] + col_type = column[2] + type_ = sql_type(val) + if col_name != name or col_type != type_: + print(repr(val), sql_type(val)) + raise InputMismatch( + "parameter {} of type {} does not fit into column {} " + "with name {} and type {}." + .format(name, type_, col_index, col_name, col_type) + ) + else: + print(' Creating new table "{}".'.format(self.exp_class)) + column_string = ", ".join( + ['"{}" {}'.format(col.name, col.sql_type) for col in new_columns] ) - # Check whether table exists already - if table_exists(cursor, self.exp_class): - # Check if table has the same columns - cursor.execute('PRAGMA table_info("{}")'.format(self.exp_class)) - for column in cursor.fetchall(): - col_index = column[0] - col_name = column[1] - col_type = column[2] - if (col_type != param_list[col_index][0] or - col_name != param_list[col_index][1]): - raise InputMismatch( - "column {} of the database ({} {}) does not match " - "the corresponding parameter ({} {}) in the file {}." - .format(col_index + 1, col_type, col_name, - *param_list[col_index][:2], experiment_file) - ) - # Get the highest index - cursor.execute('SELECT id from "{}" ORDER BY id DESC'.format(self.exp_class)) - highest_entry = cursor.fetchone() - if highest_entry: - self.id_ = highest_entry[0] + 1 - else: - self.id_ = 1 - else: - # Create a new table - print(' Creating new table "{}".'.format(self.exp_class)) - column_string = ", ".join(['"{}" {}'.format(n, t) for t, n, v in param_list]) - sql_command = 'CREATE TABLE "{}" ({})'.format(self.exp_class, column_string) - cursor.execute(sql_command) - # First entry has index 1 (one) - self.id_ = 1 - # Set id in the parameter list - param_list[0][2] = self.id_ - # Add a new entry to the table - print(' Adding new entry #{} to table "{}".'.format(self.id_, self.exp_class)) - value_list = [v for t, n, v in param_list] - sql_command = ( - 'INSERT INTO "{}" VALUES ('.format(self.exp_class) - + ', '.join(['?'] * len(value_list)) - + ')' + self.cursor.execute( + 'CREATE TABLE "{}" ({})'.format(self.exp_class, column_string) ) - cursor.execute(sql_command, value_list) - # Save the database - self.connection.commit() - # Set the name of the experiment - param.expname = "{}_{:03}".format(self.exp_class, self.id_) + # Set the experiment ID if it was not defined in the experiment file + if self.exp_id is None: + new_entry = True + # Get the highest index of the table or start with ID 1 if table is empty + self.cursor.execute( + 'SELECT id from "{}" ORDER BY id DESC'.format(self.exp_class) + ) + highest_entry = self.cursor.fetchone() + self.exp_id = highest_entry[0] + 1 if highest_entry else 1 else: - # Get name and id of the experiment - expname_parts = param.expname.split('_') - if len(expname_parts) == 1: - raise ParamError( - 'param.expname is not a valid database entry: "{}"'.format(param.expname) - ) - self.exp_class = '_'.join(expname_parts[:-1]) - try: - self.id_ = int(expname_parts[-1]) - except ValueError: - raise ParamError( - 'param.expname is not a valid database entry: "{}"'.format(param.expname) - ) - print(' Reading from entry #{} of table "{}".'.format(self.id_, self.exp_class)) - - # Remember directory of the experiment - self.output_dir = os.path.join(datadir, param.expname) - - # Get columns of the table and their value for the current experiment - self.params = dict() - cursor.execute('PRAGMA table_info("{}")'.format(self.exp_class)) - columns = cursor.fetchall() - cursor.execute('SELECT * FROM "{}" WHERE id = ?'.format(self.exp_class), (self.id_,)) - values = cursor.fetchone() - if values is None: - raise ParamError( - 'No entry with id {} exists in table "{}".'.format(self.id_, self.exp_class) + # If no entry with this ID exists, create a new one + self.cursor.execute( + 'SELECT id from "{}" WHERE id = ?'.format(self.exp_class), + [self.exp_id], ) - for column in columns: - col_index = column[0] - col_name = column[1] - value = values[col_index] - if col_name in RESERVED_COLUMN_NAMES: - continue - else: - if value == "True": - value = True - elif value == "False": - value = False - self.params[col_name] = value - - def __del__(self): - """Destructor of the Experiment Management System. - - Save the database and close the connection to it.""" - # Save and close database - print(" Closing database.") - print("-"*50) + new_entry = self.cursor.fetchone() is None + new_columns[0] = DBColumn("INTEGER", "id", self.exp_id) + if new_entry: + print(' Adding new entry #{} to table "{}".'.format(self.exp_id, self.exp_class)) + # Use the question mark as a placeholder to profit from string formatting of sqlite + value_string = ', '.join(['?'] * len(new_columns)) + self.cursor.execute( + 'INSERT INTO "{}" VALUES ({})'.format(self.exp_class, value_string), + [sql_value(col.value) for col in new_columns], + ) + else: + print(' Overwriting entry #{} of table "{}".'.format(self.exp_id, self.exp_class)) + # Use the question mark as a placeholder to profit from string formatting of sqlite + column_name_string = ", ".join('"{}" = ?'.format(col.name) for col in new_columns) + self.cursor.execute( + 'UPDATE "{}" SET {} WHERE id = ?'.format(self.exp_class, column_name_string), + [sql_value(col.value) for col in new_columns] + [self.exp_id], + ) + # Save the database self.connection.commit() - self.connection.close() - - def get_parameters(self): - """Get user-set experiment parameters and values as dictionary.""" - return self.params def finalize(self, fluid2d): - """Save integration time and size of output files in database. + """Save information about the completed run in the database. This method must be called when the simulation is finished, - that means, after the line with `f2d.loop()`. - It also sets the field `datetime` to the current time.""" - - # Write duration of the run into the database - self.connection.execute( - 'UPDATE "{}" SET duration = ? WHERE id = ?'.format(self.exp_class), - (round(fluid2d.t, 2), self.id_,) - ) - # Update date and time in the database - self.connection.execute( - 'UPDATE "{}" SET datetime = ? WHERE id = ?'.format(self.exp_class), - (datetime.datetime.now().isoformat(), self.id_) - ) - # Write size of output into the database - # Divide size by 1000*1000 = 1e6 to get value in MB + that means, after the line `f2d.loop()`. + It writes the integration time and the sizes of the created + output files into the database. If a blow-up was detected, this + is stored in the comment-field. Furthermore, the datetime-field + of the database entry is set to the current time.""" + if not self.connection: + return + # Divide every size by 1000*1000 = 1e6 to get the value in MB + # Total size + output_dir = os.path.dirname(fluid2d.output.hisfile) try: - # Get list of files in the output directory - output_files = [os.path.join(self.output_dir, f) for f in os.listdir(self.output_dir)] - total_size = sum(os.path.getsize(path) for path in output_files) + output_files = [os.path.join(output_dir, f) for f in os.listdir(output_dir)] + total_size = sum(os.path.getsize(path) for path in output_files) / 1e6 except FileNotFoundError as e: print(" Error getting total size of output:", e) - else: - self.connection.execute( - 'UPDATE "{}" SET size_total = ? WHERE id = ?'.format(self.exp_class), - (round(total_size / 1e6, 3), self.id_,) - ) + total_size = -1 # History file try: - his_size = os.path.getsize(fluid2d.output.hisfile) + his_size = os.path.getsize(fluid2d.output.hisfile) / 1e6 except FileNotFoundError as e: print(" Error getting size of his-file:", e) - else: - self.connection.execute( - 'UPDATE "{}" SET size_his = ? WHERE id = ?'.format(self.exp_class), - (round(his_size / 1e6, 3), self.id_,) - ) + his_size = -1 # Diagnostics file try: - diag_size = os.path.getsize(fluid2d.output.diagfile) + diag_size = os.path.getsize(fluid2d.output.diagfile) / 1e6 except FileNotFoundError: print(" Error getting size of diag-file:", e) - else: - self.connection.execute( - 'UPDATE "{}" SET size_diag = ? WHERE id = ?'.format(self.exp_class), - (round(diag_size / 1e6, 3), self.id_,) - ) + diag_size = -1 # MP4 file - mp4_size = -1.0 if fluid2d.plot_interactive and hasattr(fluid2d.plotting, 'mp4file'): try: - mp4_size = os.path.getsize(fluid2d.plotting.mp4file) + mp4_size = os.path.getsize(fluid2d.plotting.mp4file) / 1e6 except FileNotFoundError: print(" Error getting size of mp4-file:", e) + mp4_size = -1 else: - mp4_size = 0.0 - if mp4_size >= 0: - self.connection.execute( - 'UPDATE "{}" SET size_mp4 = ? WHERE id = ?'.format(self.exp_class), - (round(mp4_size / 1e6, 3), self.id_,) - ) + mp4_size = 0 # Flux file - flux_size = -1.0 if fluid2d.diag_fluxes: try: - flux_size = os.path.getsize(fluid2d.output.flxfile) + flux_size = os.path.getsize(fluid2d.output.flxfile) / 1e6 except FileNotFoundError: print(" Error getting size of flux-file:", e) + flux_size = -1 else: - flux_size = 0.0 - if flux_size >= 0: - self.connection.execute( - 'UPDATE "{}" SET size_flux = ? WHERE id = ?'.format(self.exp_class), - (round(flux_size / 1e6, 3), self.id_,) - ) - # Save the database + flux_size = 0 + # Check for blow-up + if hasattr(fluid2d, "blow_up") and fluid2d.blow_up: + comment = "Blow-up! " + self.description + else: + comment = self.description + # Update and save the database + self.cursor.execute( + """UPDATE "{}" SET + duration = ?, + datetime = ?, + size_total = ?, + size_mp4 = ?, + size_his = ?, + size_diag = ?, + size_flux = ?, + comment = ? + WHERE id = ?""".format(self.exp_class), + ( + round(fluid2d.t, 2), + datetime.datetime.now().isoformat(), + round(total_size, 3), + round(mp4_size, 3), + round(his_size, 3), + round(diag_size, 3), + round(flux_size, 3), + comment, + self.exp_id, + ), + ) self.connection.commit() + def connect(self, data_dir: str): + print("-"*50) + if data_dir.startswith("~"): + data_dir = os.path.expanduser(data_dir) + if not os.path.isdir(data_dir): + print(" Creating directory {}.".format(data_dir)) + os.makedirs(data_dir) + dbpath = os.path.join(data_dir, "experiments.db") + print(" Opening database {}.".format(dbpath)) + self.connection = dbsys.connect(dbpath) + self.cursor = self.connection.cursor() + + def table_exists(self, name: str): + """Check if table with given name exists in the connected database.""" + # Get all tables + self.cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + # This gives a list like that: [('table1',), ('table2',)] + for table in self.cursor.fetchall(): + if table[0] == name: + return True + return False + def parse_experiment_file(path: str): """Parse the given experiment file. @@ -287,149 +319,129 @@ def parse_experiment_file(path: str): The name is written in a line starting with "Name:". It must be a valid string to be used as a filename; in particular, it must not - contain the slash (/) or the quotation mark ("). + contain a slash (/) and it must not contain a quotation mark ("). + It is advised to use underscores instead of spaces in the name. The description begins in or after a line starting with - "Description:" and goes until the beginning of the parameters - or the end of the file. + "Description:" and goes until the beginning of the parameters. + If no parameters are defined below the description, it goes until + the end of the file. The parameters follow after a line starting with "Parameters:". - Every parameter is written in its own line. This line contains the - datatype, the name and the value of the parameter seperated by one - or several whitespaces. - The datatype must be one of "int", "float", "bool" or "str". - The name must not contain whitespace characters or quotation marks - and must not be in the list of reserved column names. - The value must be a valid value for the given datatype. If the - value is omitted, it defaults to zero for numbers, True for booleans - and the empty string for strings. The values "True" and "False" can - only be used for parameters of boolean type, not as strings. - It is planned to include the possibility of providing several values - in order to run multiple experiments from one experiment file. - - Lines in the experiment file starting with the #-symbol are ignored. - It is not possible to write in-line comments.""" - # TODO: allow parameter names and string-values with whitespace like "long name". - # TODO: allow to give several values to run multiple experiments. + Every parameter is written in its own line. This line contains a + name and one value or several values, each separated by one or + several whitespaces. The name must not contain any whitespace + characters (otherwise it is not recognised as such) and must not + contain a quotation mark. The name must not be in the list of + reserved column names. If the value is True, False, an integer or a + floation point number, it is interpreted as the corresponding Python + type, otherwise it is considered a string. Quotation marks in the + value are taken literally and are not interpreted. + + Everything after a #-symbol is considered a comment and ignored. + + TODO: + - allow to use quotation marks for multi-word strings as values + """ with open(path) as f: experiment_lines = f.readlines() name = "" - description = "" - param_list = [] + id_ = None + description_lines = [] + param_name_list = [] + param_values_list = [] reading_params = False reading_description = False for line in experiment_lines: - if line.startswith("#"): - # Skip comments - continue - elif not line.strip(): + # Remove comments and whitespace at the beginning and the end + line = line.split("#")[0].strip() + if not line: # Skip empty lines except in the description if reading_description: - description += line + description_lines.append(line) else: continue elif line.lower().startswith("name:"): if not name: - name = line[5:] + name = line[5:].strip() if '"' in name: - raise ExpFileError('name must not contain the symbol ".') + raise ExpFileError("Name must not contain a quotation mark.") else: raise ExpFileError( - "name defined more than once in file {}.".format(path) + "Name defined more than once." ) + elif line.lower().startswith("id:"): + if id_ is None: + value = line[3:].strip() + if value != "": + try: + id_ = int(value) + except ValueError: + raise ExpFileError("ID is not an integer.") + else: + raise ExpFileError("ID defined more than once.") elif line.lower().startswith("description:"): reading_description = True reading_params = False - description += line[12:] + description_lines.append(line[12:].lstrip()) elif line.lower().startswith("parameters:"): reading_description = False reading_params = True elif reading_description: - description += line + description_lines.append(line) elif reading_params: - words = line.split() - if len(words) < 2: - raise ExpFileError( - 'type or name missing for parameter "{}" in file {}.'.format(words[0], path) - ) - param_type = words[0].lower() - param_name = words[1] - param_values = words[2:] - # Check values - if len(param_values) == 0: - param_value = None - elif len(param_values) == 1: - param_value = param_values[0] - else: - raise NotImplementedError( - 'multiple values given for parameter "{}" in file {}.' - .format(param_name, path) - ) - # Check type - if param_type == "int": - sql_type = "INTEGER" - if param_value is None: - param_value = 0 - else: - param_value = int(param_value) - elif param_type == "float": - sql_type = "REAL" - if param_value is None: - param_value = 0.0 - else: - param_value = float(param_value) - elif param_type == "bool": - if param_value is None: - param_value = "True" - elif param_value.lower() == "true": - param_value = "True" - elif param_value.lower() == "false": - param_value = "False" - else: - raise ExpFileError( - 'boolean parameter "{}" is neither "True" nor "False" ' - 'but "{}" in file {}.'.format(param_name, param_value, path) - ) - sql_type = "TEXT" - elif param_type == "str": - sql_type = "TEXT" - if param_value is None: - param_value = "" - elif param_value == "True" or param_value == "False": - raise ExpFileError( - 'the words "True" and "False" cannot be used as value for ' - 'parameter "{}" of type string. Instead, use boolean type ' - 'parameter in file {}.'.format(param_name, path) - ) - else: - raise ExpFileError( - 'unknown parameter type "{}" in file {}.'.format(param_type, path) - ) - # Check name + # Parse the parameter name + param_name = line.split()[0] if param_name in RESERVED_COLUMN_NAMES: raise ExpFileError( - 'reserved name used for parameter "{}" in file {}.' - .format(param_name, path) + "reserved name {} must not be used as a parameter.".format(param_name) ) if '"' in param_name: - raise ExpFileError( - 'name of parameter "{}" must not contain the symbol ".' - .format(param_name) - ) - param_list.append([sql_type, param_name, param_value]) + raise ExpFileError("parameter name must not contain a quotation mark.") + # Parse the value(s) of the parameter + param_val_text = line[len(param_name):].lstrip() + if not param_val_text: + raise ExpFileError("no value given for parameter {}.".format(param_name)) + # TODO: allow multi-word strings containing quotation marks + param_values = [cast_string(val) for val in param_val_text.split()] + param_name_list.append(param_name) + param_values_list.append(param_values) else: - raise ExpFileError( - "unexpected line in file {}:\n{}".format(path, repr(line)) - ) - return name.strip(), description.strip(), param_list - - -def table_exists(cursor: dbsys.Cursor, name: str): - """Check if table with given name exists in the connected database.""" - # Get all tables - cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") - # This gives a list like that: [('table1',), ('table2',)] - for table in cursor.fetchall(): - if table[0] == name: - return True - return False + raise ExpFileError("unexpected line:\n{!r}".format(line)) + if not name: + raise ExpFileError("Name must be defined in every experiment file.") + return name, id_, "\n".join(description_lines).strip(), param_name_list, param_values_list + + +def sql_type(value): + if isinstance(value, bool): + return "TEXT" + elif isinstance(value, int): + return "INTEGER" + elif isinstance(value, float): + return "REAL" + return "TEXT" + + +def sql_value(value): + if isinstance(value, bool): + return str(value) + return value + + +def cast_string(value: str): + """Cast into the most specialised Python type possible. + + This method can recognize "True", "False", integers and floats, + everything else is treated as a string and returned unchanged.""" + if value == "True": + return True + if value == "False": + return False + try: + return int(value) + except ValueError: + try: + return float(value) + except ValueError: + return value diff --git a/core/fluid2d.py b/core/fluid2d.py index 8a269fa..7e79b40 100644 --- a/core/fluid2d.py +++ b/core/fluid2d.py @@ -285,6 +285,7 @@ def signal_handler(signal, frame): # check for blow-up if self.model.diags['maxspeed'] > 1e3: self.stop = True + self.blow_up = True if self.myrank == 0: print() print('max|u| > 1000, blow-up detected, stopping') diff --git a/core/output.py b/core/output.py index 2d12443..43c7fa3 100644 --- a/core/output.py +++ b/core/output.py @@ -184,7 +184,11 @@ def create(self): # store all the parameters as NetCDF global attributes dparam = self.param.__dict__ for k in dparam.keys(): - if type(dparam[k]) == type([0, 1]): + if k == "ems": + # The EMS stores itself and does not need to be stored in the netCDF file + pass + + elif type(dparam[k]) == type([0, 1]): # it is not straightforward to store a list in a netcdf file pass diff --git a/core/param.py b/core/param.py index 9dfed68..cdf5289 100644 --- a/core/param.py +++ b/core/param.py @@ -3,6 +3,9 @@ import sys import getopt +# Local import +import ems + class Param(object): """class to set up the default parameters value of the model @@ -17,7 +20,7 @@ class Param(object): """ - def __init__(self, defaultfile): + def __init__(self, defaultfile=None, ems_file=""): """defaultfile is a sequel, it's no longer used the default file is systematically the defaults.json located in the fluid2d/core @@ -42,6 +45,11 @@ def __init__(self, defaultfile): else: self.print_param = False + if ems_file: + self.ems = ems.EMS(ems_file) + else: + self.ems = None + def set_parameters(self, namelist): avail = {} doc = {} @@ -76,6 +84,10 @@ def manall(self): self.man(p) def checkall(self): + if self.ems: + if self.myrank == 0: + self.ems.initialize(self.datadir) + self.expname = self.ems.get_expname() for p, avail in self.avail.items(): if getattr(self, p) in avail: # the parameter 'p' is well set @@ -109,6 +121,18 @@ def copy(self, obj, list_param): missing.append(k) return missing + def get_experiment_parameters(self): + return self.ems.parameters + + def loop_experiment_parameters(self): + while self.ems.parameters: + yield self.ems.parameters + self.ems.setup_next_parameters() + + def finalize(self, fluid2d): + if self.ems: + self.ems.finalize(fluid2d) + if __name__ == "__main__": param = Param('default.xml') diff --git a/core/plotting.py b/core/plotting.py index 38ec41c..8845efb 100644 --- a/core/plotting.py +++ b/core/plotting.py @@ -23,11 +23,12 @@ def __init__(self, param, grid, var, diag): param.copy(self, self.list_param) if param.npx * param.npy > 1 and self.generate_mp4: - print( - 'Warning: It is not possible to generate an mp4-file when ' - 'fluid2d runs on multiple cores.\n' - 'The parameter generate_mp4 is automatically changed to False.' - ) + if self.myrank == 0: + print( + 'Warning: It is not possible to generate an mp4-file when ' + 'fluid2d runs on multiple cores.\n' + 'The parameter generate_mp4 is automatically changed to False.' + ) self.generate_mp4 = False nh = self.nh @@ -217,9 +218,10 @@ def update_fig(self, t, dt, kt): self.process.stdin.write(string) def finalize(self): - """ do nothing but close the mp4 thread if any""" + """Close the mp4 thread if any and the current figure.""" if self.generate_mp4: self.process.communicate() + plt.close() def set_cax(self, z): """ set self.cax, the color range""" diff --git a/experiments/Experiment-Manager.py b/experiments/Experiment-Manager.py index f868b73..baac850 100644 --- a/experiments/Experiment-Manager.py +++ b/experiments/Experiment-Manager.py @@ -215,10 +215,10 @@ def get_table_length(self, table_name): if table.name == table_name: return len(table) - def get_highest_index(self, table_name): + def get_latest_entry(self, table_name): for table in self.tables: if table.name == table_name: - return table.get_highest_index() + return table.get_latest_entry() def set_comment(self, table_name, id_, new_comment): for table in self.tables: @@ -328,8 +328,8 @@ def get_data(self, column_name, condition=""): else: return [e[0] for e in self.c.fetchall()] - def get_highest_index(self): - self.c.execute('SELECT id from "{}" ORDER BY id DESC'.format(self.name)) + def get_latest_entry(self): + self.c.execute('SELECT id from "{}" ORDER BY datetime DESC'.format(self.name)) result = self.c.fetchone() self.c.fetchall() # otherwise the database stays locked return result[0] if result else None @@ -666,9 +666,9 @@ def help_compare(self): specified entries differ. Alternatively, add "-h" to show all parameters and highlight the differences in colour. Disabled columns are not shown by default. Add the argument "-v" to show disabled columns, the full date-time - and the full comment. Instead of an ID, "last" or "-1" can be used to - compare with the entry of highest ID. If no ID is specified, all entries of - the selected experiment class are compared.""") + and the full comment. Instead of an ID, "last" can be used to compare with + the latest entry. If no ID is specified, all entries of the selected + experiment class are compared.""") ### Functionality to OPEN experiment files def do_open_mp4(self, params): @@ -1143,6 +1143,9 @@ def do_remove(self, params): print('Error deleting folder {}:'.format(folder), e) else: print('Deleted folder {}.'.format(folder)) + elif answer == "no": + # Do nothing. + pass else: print('Answer was not "yes". No data removed.') @@ -1151,7 +1154,7 @@ def help_remove(self): Delete the entry with the given ID from the currently selected class of experiments and all files associated with it. Multiple IDs can be specified to remove several entries and their folders at once. Instead of an ID, the - argument "last" or "-1" can be used to choose the entry with the highest ID. + argument "last" can be used to choose the latest entry. Before the data is deleted, the user is asked to confirm, which must be answered with "yes". The remove an empty class of experiments, use "remove_selected_class".""") @@ -1288,8 +1291,7 @@ def parse_params_to_experiment(self, params: str) -> [str, int]: """Parse and check input of the form "[experiment] ". The argument "experiment" is optional, the ID is necessary. - Instead of an ID, "last" can be used to refer to the entry with - the highest ID. The value "-1" is an alias for "last". + Instead of an ID, "last" can be used to refer to the latest entry. If no experiment is given, the selected experiment is taken. Return None and print a message if input is not valid, otherwise return the experiment name and the ID.""" @@ -1300,7 +1302,7 @@ def parse_params_to_experiment(self, params: str) -> [str, int]: # Parse the ID specifier = params.pop() if specifier == "last": - id_ = -1 + id_ = "last" else: try: id_ = int(specifier) @@ -1319,8 +1321,8 @@ def parse_params_to_experiment(self, params: str) -> [str, int]: return None table_name = self.selected_table # Check the ID - if id_ == -1: - id_ = self.con.get_highest_index(table_name) + if id_ == "last": + id_ = self.con.get_latest_entry(table_name) if id_ is None: print(f'No entry exists for the experiment class "{table_name}".') return None @@ -1390,11 +1392,11 @@ def parse_multiple_ids(self, param_list): This method also checks that the IDs are valid entries of the selected database and returns None if an invalid ID is given. Otherwise, a sorted list of unique values is returned. - It correctly parses "last" or "-1".""" + It parses "last" as the latest entry.""" ids = [] for id_ in param_list: - if id_ == "last" or id_ == "-1": - id_ = self.con.get_highest_index(self.selected_table) + if id_ == "last": + id_ = self.con.get_latest_entry(self.selected_table) if id_ is None: print(f'No entry exists for the experiment class "{self.selected_table}".') return @@ -1534,8 +1536,7 @@ def print_param_parser_help(): print(""" If only an ID and no experiment name is given, take the currently selected class of experiments. Instead of an ID, the value "last" can be used to - choose the entry with the highest ID. The value "-1" is an alias for - "last".""" + choose the latest entry.""" ) diff --git a/experiments/Waves with EMS/wave_breaking.exp b/experiments/Waves with EMS/wave_breaking.exp index a9e3111..cf1a43d 100644 --- a/experiments/Waves with EMS/wave_breaking.exp +++ b/experiments/Waves with EMS/wave_breaking.exp @@ -1,9 +1,9 @@ -# Markus Reinert, May 2019 -# # Fluid2d-experiment with EMS: wave breaking at the coast (experiment file) # # Refer to the documentation of `ems.parse_experiment_file` for a # detailed description of the format of experiment files. +# +# Author: Markus Reinert, May/June 2019 # Name of the experiment class. @@ -11,31 +11,45 @@ Name: Breaking_Waves +# ID of the experiment within its class. +# Usually, an ID should NOT be specified, so that the EMS chooses automatically a unique ID. +# It is only necessary to specify an ID explicitly if fluid2d runs on multiple cores, for +# example with mpirun. Furthermore, it is possible to overwrite existing entries in the +# database by specifying their ID. +# The ID is automatically increased by 1 for every new combination if multiple values are +# given for one or several parameters. +# Uncomment the following line to set an ID manually: +#ID: 24 + + # Descriptions are optional. # They can go over multiple lines and can also contain empty lines. -Description: Simulate interface waves, their propagation and breaking at a sloping coast. -Various parameters can be modified in this file. +# Empty lines at the beginning and at the end of the description are ignored. +Description: Simulate waves on the interface of two fluids. +Modify the parameters in this file and observe the creation, propagation and breaking of waves at a sloping coast. # Here are the parameters that we want to modify in this class of experiments. # Parameters which we intend to keep constant are in the Python file. +# For every parameter, at least one value has to be specified. Multiple values can be +# specified to automatically run one experiment for every combination of the given values. Parameters: ### Physics # activate or deactivate diffusion -bool diffusion False +diffusion False # diffusion coefficient (if diffusion is True) -float Kdiff 1e-2 +Kdiff 1e-2 ### Coast -float slope 0.5 -float height 1.05 +slope 0.5 +height 1.05 # beginning of the flat coast -float x_start 4.0 +x_start 4.0 ### Initial Condition # perturbation can be of type "sin" or "gauss" -str perturbation gauss +perturbation gauss # height and width of the perturbation -float intensity 0.5 -float sigma 0.3 +intensity 0.5 +sigma 0.2 0.4 diff --git a/experiments/Waves with EMS/wave_breaking.py b/experiments/Waves with EMS/wave_breaking.py index 007185f..362c543 100644 --- a/experiments/Waves with EMS/wave_breaking.py +++ b/experiments/Waves with EMS/wave_breaking.py @@ -1,150 +1,150 @@ -# Markus Reinert, April/May 2019 -# -# Fluid2d-experiment with EMS: wave breaking at the coast (Python file) -# -# This file aims to be both, an interesting experiment and a helpful -# introduction to the Experiment Management System (EMS) of fluid2d. -# -# To use the EMS, we import the module `ems` into the Python script of -# our experiment and create an instance of the class `EMS`, giving the -# usual Param-object and an experiment file. This creates an entry in -# the experiment-database. The database is stored as an SQLite file -# with the name `experiments.db` in the data directory of fluid2d, i.e., -# the path defined in `param.datadir`. Furthermore, it sets the value -# of `param.expname` to refer to this entry. Therefore, when the EMS is -# used, the value of `param.expname` must not be set manually. -# Exceptions to this rule are explained below. -# At the beginning, some information is missing in the new entry of the -# database. To complete it, we call the method `finalize` of the EMS at -# the end of the simulation. Through this, the actual integration time -# and the size of the output files (in MB) is stored in the database. -# The field datetime is also updated and set to the current time. -# The parameters defined in the experiment file are accessible in the -# Python script through the dictionary of experiment parameters (EP). -# -# If the experiment parameters are also needed somewhere else in the -# code where the EMS-object is not accessible, for example in a forcing -# module, then a new EMS-object can be created. This new EMS-object -# must be initialized with the same Param-object and without an -# experiment file. In this way, both EMS-objects refer to the same -# entry of the database and share the same experiment parameters. -# -# It is possible to re-run experiments with overwriting the files, for -# example because the integration was stopped too early. -# To do this, we specify as `param.expname` the name of the experiment -# we want to run again, consisting of the name of the experiment class -# and a 3-digit number. Then we call the constructor of EMS without an -# experiment file. Example: -# param.expname = "Breaking_Waves_001" -# ems = EMS(param) -# This replaces the files in the folder `Breaking_Waves_001` and updates -# the entry 1 of the table `Breaking_Waves` at the end of the run. -# -# In general, whenever an EMS-object is created with an experiment file -# given, it creates a new entry in the database with the information -# from the experiment file and it sets `param.expname` to refer to this -# new entry. In contrast, if an EMS-object is created without an -# experiment file, it loads the data from the database entry which -# corresponds to the value of `param.expname`. +"""Fluid2d-experiment with EMS: wave breaking at the coast (Python file) + +This file aims to be both, an interesting experiment and a helpful +introduction to the Experiment Management System (EMS) of fluid2d. + +Compared to a simple fluid2d experiment without EMS, three changes are +necessary to profit from the full functionality of the EMS: + + 1. Create Param by specifying the path to the experiment file. + 2. Fetch the experiment parameters from Param. + 3. Call the finalize-method of Param at the end. + +Furthermore, the line with param.expname should be removed, since the +EMS takes care of setting the filename of the experiment. + +When the Experiment Management System is activated like this, an entry +in the experiment-database is created. This entry contains the unique +ID of the experiment, the date and time of the run, the last point in +time before the integration stopped, the sizes of the output files in +MB, a comment or description and -- most importantly -- the parameters +that are chosen by the user to keep track off. These parameters are +defined in the experiment file. After the EMS is set-up, only the +experiment file needs to be changed by the user. Thanks to the EMS, +it is not necessary to modify the Python file more than once at the +beginning. + +Author: Markus Reinert, April/May/June 2019 +""" from fluid2d import Fluid2d from param import Param from grid import Grid -from ems import EMS import numpy as np -# Load default values and set type of model -param = Param("default.xml") -param.modelname = "boussinesq" - -# Activate the Experiment Management System (EMS) with an experiment file -ems = EMS(param, "wave_breaking.exp") -# Fetch experiment parameters (EP) -EP = ems.get_parameters() - -# Set domain type, size and resolution -param.geometry = "closed" -param.ny = 2 * 64 -param.nx = 3 * param.ny -param.Ly = 2 -param.Lx = 3 * param.Ly -# Set number of CPU cores used -param.npx = 1 -param.npy = 1 - -# Set time settings -param.tend = 100.0 -param.cfl = 1.2 -param.adaptable_dt = False -param.dt = 0.01 -param.dtmax = 0.1 - -# Choose discretization -param.order = 5 - -# Set output settings -param.var_to_save = ["vorticity", "buoyancy", "psi"] -param.list_diag = "all" -param.freq_his = 0.2 -param.freq_diag = 0.1 - -# Set plot settings -param.plot_var = "buoyancy" -param.plot_interactive = True -param.generate_mp4 = True -param.freq_plot = 10 -param.colorscheme = "imposed" -param.cax = [0, 1] -param.cmap = "Blues_r" # reversed blue colour axis - -# Configure physics -param.gravity = 1.0 -param.forcing = False -param.noslip = False -param.diffusion = EP["diffusion"] -param.Kdiff = EP["Kdiff"] * param.Lx / param.nx - -# Initialize geometry -grid = Grid(param) -xr, yr = grid.xr, grid.yr - -LAND = 0 -AQUA = 1 - -# Add linear sloping shore -m = EP["slope"] -t = EP["height"] - EP["x_start"] * m -grid.msk[(yr <= m*xr + t) & (yr < EP["height"])] = LAND -grid.finalize_msk() - -# Create model -f2d = Fluid2d(param, grid) -model = f2d.model - -# Set initial perturbation of the surface (or interface) -buoy = model.var.get("buoyancy") -buoy[:] = 1 -if EP["perturbation"].lower() == "sin": - # Use a sinusoidal perturbation - buoy[ - (yr < EP["intensity"] * np.sin(2 * np.pi * xr/EP["sigma"]) + 1.0) - & (xr < EP["x_start"]) - ] = 0 -elif EP["perturbation"].lower() == "gauss": - # Use a Gaussian perturbation - buoy[ - (yr < EP["intensity"] * np.exp(-(xr/EP["sigma"])**2) + 1.0) - & (xr < EP["x_start"]) - ] = 0 -else: - raise ValueError("unknown type of perturbation: {}.".format(EP["perturbation"])) -buoy *= grid.msk - -# Start simulation -f2d.loop() - -# Make it a complete database entry -ems.finalize(f2d) +### STEP 1 +# Load default parameter values, load specific parameters from the given +# experiment file and set-up the EMS. It is advised to specify the experiment +# file explicitly with the keyword `ems_file` like this: +param = Param(ems_file="wave_breaking.exp") + +# Do not set `param.expname` because it will be ignored. + +### STEP 2 +# There are two ways to get the dictionary with the experiment parameters. +# (a) The quick and easy way (using "get"): +# To run exactly one experiment with only the values specified at first, use +# EP = param.get_experiment_parameters() +# In this case, it is not necessary to indent the whole file as below. +# (b) The recommended way (using "loop"): +# To run one experiment for every possible combination of all the values +# specified in the experiment file, use +# for EP in param.loop_experiment_parameters(): +# and put the rest of the Python file within this loop by indenting every line. +# +# In both cases, the dictionary EP is now available. Its keys are the names +# of the parameters in the experiment file. Every key refers to the +# corresponding value. If multiple values are given in the experiment file, +# in every iteration of the loop, another combination of these values is in EP. +# Within the following lines of the Python code, the values of the EP are used +# to implement the desired behaviour of the experiment. +for EP in param.loop_experiment_parameters(): + # Set model type, domain type, size and resolution + param.modelname = "boussinesq" + param.geometry = "closed" + param.ny = 2 * 64 + param.nx = 3 * param.ny + param.Ly = 2 + param.Lx = 3 * param.Ly + # Set number of CPU cores used + param.npx = 1 + param.npy = 1 + + # Set time settings + param.tend = 20.0 + param.adaptable_dt = False + param.dt = 0.02 + + # Choose discretization + param.order = 5 + + # Set output settings + param.var_to_save = ["vorticity", "buoyancy", "psi"] + param.list_diag = "all" + param.freq_his = 0.2 + param.freq_diag = 0.1 + + # Set plot settings + param.plot_var = "buoyancy" + param.plot_interactive = True + param.generate_mp4 = True + param.freq_plot = 10 + param.colorscheme = "imposed" + param.cax = [0, 1] + param.cmap = "Blues_r" # reversed blue colour axis + + # Configure physics + param.gravity = 1.0 + param.forcing = False + param.noslip = False + param.diffusion = EP["diffusion"] + param.Kdiff = EP["Kdiff"] * param.Lx / param.nx + + # Initialize geometry + grid = Grid(param) + xr, yr = grid.xr, grid.yr + + LAND = 0 + AQUA = 1 + + # Add linear sloping shore + m = EP["slope"] + t = EP["height"] - EP["x_start"] * m + grid.msk[(yr <= m*xr + t) & (yr < EP["height"])] = LAND + grid.finalize_msk() + + # Create model + f2d = Fluid2d(param, grid) + model = f2d.model + + # Set initial perturbation of the surface (or interface) + buoy = model.var.get("buoyancy") + buoy[:] = 1 + if EP["perturbation"].lower() == "sin": + # Use a sinusoidal perturbation + buoy[ + (yr < EP["intensity"] * np.sin(2 * np.pi * xr/EP["sigma"]) + 1.0) + & (xr < EP["x_start"]) + ] = 0 + elif EP["perturbation"].lower() == "gauss": + # Use a Gaussian perturbation + buoy[ + (yr < EP["intensity"] * np.exp(-(xr/EP["sigma"])**2) + 1.0) + & (xr < EP["x_start"]) + ] = 0 + else: + raise ValueError("unknown type of perturbation: {}.".format(EP["perturbation"])) + buoy *= grid.msk + + # Start simulation + f2d.loop() + + ### STEP 3 + # Finish off the EMS database entry. + # Without this call to `finalize`, the size of the output files cannot be + # saved in the database. It is important to have this line within the + # for-loop if the multi-run strategy (b) is used. + param.finalize(f2d) From ad97ad2ef87c55effaca26ff7f596c3b1eca60f5 Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 19 Jun 2019 11:44:44 +0200 Subject: [PATCH 27/31] EMS: Extend and improve comments and doc-strings The Experiment-Manager EMShell together with its extensions is moved into its own directory to have a clearer arrangement and structure. --- core/ems.py | 57 ++++++++++++-------- core/param.py | 29 +++++++++- {experiments => ems}/EMShellExtensions.py | 0 {experiments => ems}/Experiment-Manager.py | 0 experiments/Waves with EMS/wave_breaking.exp | 6 +-- experiments/Waves with EMS/wave_breaking.py | 50 +++++++++-------- 6 files changed, 93 insertions(+), 49 deletions(-) rename {experiments => ems}/EMShellExtensions.py (100%) rename {experiments => ems}/Experiment-Manager.py (100%) diff --git a/core/ems.py b/core/ems.py index 23ef03c..e338cec 100644 --- a/core/ems.py +++ b/core/ems.py @@ -9,19 +9,21 @@ experiment several times with different parameter values. To prevent overwriting files, the EMS automatically assigns a unique identifier for every new run of an experiment. However, if fluid2d runs on multiple -cores, an ID has to be specified initially. +cores, an initial ID has to be specified manually. -Its usage is explained in detail in the experiment on breaking waves. +The usage of the EMS is explained in detail in the experiment on waves, +located in the folder `experiments/Waves with EMS` of fluid2d. With the EMShell, a command line interface is provided to access the -experiment-database created by the EMS. +experiment-database created by the EMS. The EMShell is run via the file +`Experiment-Manager.py` in the folder `ems` of fluid2d. Author: Markus Reinert, May/June 2019 """ import os import datetime -import sqlite3 as dbsys +import sqlite3 from collections import namedtuple from itertools import product @@ -213,7 +215,8 @@ def finalize(self, fluid2d): It writes the integration time and the sizes of the created output files into the database. If a blow-up was detected, this is stored in the comment-field. Furthermore, the datetime-field - of the database entry is set to the current time.""" + of the database entry is set to the current time. + """ if not self.connection: return # Divide every size by 1000*1000 = 1e6 to get the value in MB @@ -295,7 +298,7 @@ def connect(self, data_dir: str): os.makedirs(data_dir) dbpath = os.path.join(data_dir, "experiments.db") print(" Opening database {}.".format(dbpath)) - self.connection = dbsys.connect(dbpath) + self.connection = sqlite3.connect(dbpath) self.cursor = self.connection.cursor() def table_exists(self, name: str): @@ -314,8 +317,9 @@ def parse_experiment_file(path: str): An experiment file - must provide a name, - - can provide a description, - - can provide parameters with values. + - can provide an ID, + - can provide a multi-line description, + - can provide parameters with one or several values each. The name is written in a line starting with "Name:". It must be a valid string to be used as a filename; in particular, it must not @@ -328,20 +332,23 @@ def parse_experiment_file(path: str): the end of the file. The parameters follow after a line starting with "Parameters:". - Every parameter is written in its own line. This line contains a - name and one value or several values, each separated by one or - several whitespaces. The name must not contain any whitespace - characters (otherwise it is not recognised as such) and must not - contain a quotation mark. The name must not be in the list of - reserved column names. If the value is True, False, an integer or a - floation point number, it is interpreted as the corresponding Python - type, otherwise it is considered a string. Quotation marks in the - value are taken literally and are not interpreted. + Every parameter is written in its own line. This line begins with + the name of the parameter followed by one value or several values, + separated by one or several spaces or tabs. The name must not + contain any whitespace characters (otherwise it is not recognised as + one name) and must not contain a quotation mark. The name must not + be in the list of reserved column names. The datatype of the value + must be unambiguous, i.e., the values True and False are interpreted + as Boolean variables, numbers like 3 are treated as integers, + numbers like 3.14 or 1e3 (=1000) are treated as floats and all other + values are taken as strings literally. If a string contains a + whitespace, this is considered as multiple values for one parameter + and not as a string of multiple words. Everything after a #-symbol is considered a comment and ignored. TODO: - - allow to use quotation marks for multi-word strings as values + - use of quotation marks for multi-word strings in values """ with open(path) as f: experiment_lines = f.readlines() @@ -353,7 +360,7 @@ def parse_experiment_file(path: str): reading_params = False reading_description = False for line in experiment_lines: - # Remove comments and whitespace at the beginning and the end + # Remove comments as well as leading and trailing whitespace line = line.split("#")[0].strip() if not line: # Skip empty lines except in the description @@ -367,9 +374,7 @@ def parse_experiment_file(path: str): if '"' in name: raise ExpFileError("Name must not contain a quotation mark.") else: - raise ExpFileError( - "Name defined more than once." - ) + raise ExpFileError("Name defined more than once.") elif line.lower().startswith("id:"): if id_ is None: value = line[3:].strip() @@ -424,6 +429,11 @@ def sql_type(value): def sql_value(value): + """Cast into a type suitable for writing the value in a databse. + + This returns for Boolean variables a string representing the given + value and returns the value unchanged otherwise. + """ if isinstance(value, bool): return str(value) return value @@ -433,7 +443,8 @@ def cast_string(value: str): """Cast into the most specialised Python type possible. This method can recognize "True", "False", integers and floats, - everything else is treated as a string and returned unchanged.""" + everything else is treated as a string and returned unchanged. + """ if value == "True": return True if value == "False": diff --git a/core/param.py b/core/param.py index cdf5289..fa83474 100644 --- a/core/param.py +++ b/core/param.py @@ -21,9 +21,15 @@ class Param(object): """ def __init__(self, defaultfile=None, ems_file=""): - """defaultfile is a sequel, it's no longer used the default file is - systematically the defaults.json located in the fluid2d/core + """Load default parameters and optionally experiment parameters. + The parameter `defaultfile` is no longer used and exists only + for backwards compatibility. The default file is always + the file `defaults.json` located in the core-folder of fluid2d. + + The parameter `ems_file` takes optionally the name of an + experiment file. If given, the Experiment Management System + (EMS) is activated and the experiment file is parsed. """ import grid @@ -85,6 +91,7 @@ def manall(self): def checkall(self): if self.ems: + # Only create a new database entry once, not by every core if self.myrank == 0: self.ems.initialize(self.datadir) self.expname = self.ems.get_expname() @@ -122,14 +129,32 @@ def copy(self, obj, list_param): return missing def get_experiment_parameters(self): + """Return the experiment parameters dictionary loaded by the EMS. + + The EMS must be activated in the constructor of `Param` to use + this method. It returns the dictionary of experiment parameters + and exits. It is advised to use, when possible, the method + `loop_experiment_parameters` instead. + """ return self.ems.parameters def loop_experiment_parameters(self): + """Iterate over the experiment parameters loaded by the EMS. + + In every iteration, this method returns a new dictionary of + experiment parameters containing a combination of the values + specified in the experiment file. This experiment file for the + EMS must be specified in the constructor of `Param`. If only + one value is given for every parameter in the experiment file, + the method `get_experiment_parameters` can be used instead. + The ID of the experiment is increased in every iteration. + """ while self.ems.parameters: yield self.ems.parameters self.ems.setup_next_parameters() def finalize(self, fluid2d): + """Invoke the finalize method of the EMS if activated.""" if self.ems: self.ems.finalize(fluid2d) diff --git a/experiments/EMShellExtensions.py b/ems/EMShellExtensions.py similarity index 100% rename from experiments/EMShellExtensions.py rename to ems/EMShellExtensions.py diff --git a/experiments/Experiment-Manager.py b/ems/Experiment-Manager.py similarity index 100% rename from experiments/Experiment-Manager.py rename to ems/Experiment-Manager.py diff --git a/experiments/Waves with EMS/wave_breaking.exp b/experiments/Waves with EMS/wave_breaking.exp index cf1a43d..b3da53e 100644 --- a/experiments/Waves with EMS/wave_breaking.exp +++ b/experiments/Waves with EMS/wave_breaking.exp @@ -1,7 +1,7 @@ # Fluid2d-experiment with EMS: wave breaking at the coast (experiment file) # # Refer to the documentation of `ems.parse_experiment_file` for a -# detailed description of the format of experiment files. +# detailed description of the experiment file format. # # Author: Markus Reinert, May/June 2019 @@ -19,7 +19,7 @@ Name: Breaking_Waves # The ID is automatically increased by 1 for every new combination if multiple values are # given for one or several parameters. # Uncomment the following line to set an ID manually: -#ID: 24 +#ID: 21 # Descriptions are optional. @@ -52,4 +52,4 @@ x_start 4.0 perturbation gauss # height and width of the perturbation intensity 0.5 -sigma 0.2 0.4 +sigma 0.4 0.8 diff --git a/experiments/Waves with EMS/wave_breaking.py b/experiments/Waves with EMS/wave_breaking.py index 362c543..108568d 100644 --- a/experiments/Waves with EMS/wave_breaking.py +++ b/experiments/Waves with EMS/wave_breaking.py @@ -1,33 +1,37 @@ """Fluid2d-experiment with EMS: wave breaking at the coast (Python file) -This file aims to be both, an interesting experiment and a helpful -introduction to the Experiment Management System (EMS) of fluid2d. +This file, together with its corresponding experiment file, aims to be +both, an interesting experiment and a helpful introduction to the +Experiment Management System (EMS) of fluid2d. Compared to a simple fluid2d experiment without EMS, three changes are necessary to profit from the full functionality of the EMS: 1. Create Param by specifying the path to the experiment file. - 2. Fetch the experiment parameters from Param. + 2. Fetch and use the experiment parameters from Param. 3. Call the finalize-method of Param at the end. -Furthermore, the line with param.expname should be removed, since the -EMS takes care of setting the filename of the experiment. +Furthermore, the variable `param.expname` should not be set in the +Python file, since the EMS takes care of setting it. In this way, the +loss of old output files by accidentally replacing them is avoided. +Nevertheless, it is possible to explicitly ask the EMS to replace old +experiments. See the experiment file for more details. When the Experiment Management System is activated like this, an entry -in the experiment-database is created. This entry contains the unique -ID of the experiment, the date and time of the run, the last point in -time before the integration stopped, the sizes of the output files in -MB, a comment or description and -- most importantly -- the parameters -that are chosen by the user to keep track off. These parameters are -defined in the experiment file. After the EMS is set-up, only the -experiment file needs to be changed by the user. Thanks to the EMS, -it is not necessary to modify the Python file more than once at the -beginning. +in the experiment-database is created. The database is stored in the +same directory as the output of the experiments, which is defined in +`param.datadir`. Every entry contains the unique ID of the experiment, +the date and time of the run, the last point in time of the integration, +the sizes of the output files in MB, a comment or description and +-- most importantly -- the parameters that are chosen by the user to +keep track off. These parameters are defined in the experiment file. +After the EMS is set-up, only the experiment file needs to be changed by +the user. Thanks to the EMS, it is not necessary to modify the Python +file more than once at the beginning to set it up. Author: Markus Reinert, April/May/June 2019 """ - from fluid2d import Fluid2d from param import Param from grid import Grid @@ -37,8 +41,7 @@ ### STEP 1 # Load default parameter values, load specific parameters from the given -# experiment file and set-up the EMS. It is advised to specify the experiment -# file explicitly with the keyword `ems_file` like this: +# experiment file and set-up the EMS. param = Param(ems_file="wave_breaking.exp") # Do not set `param.expname` because it will be ignored. @@ -54,13 +57,16 @@ # specified in the experiment file, use # for EP in param.loop_experiment_parameters(): # and put the rest of the Python file within this loop by indenting every line. +# The behaviour of both ways is the same if only one value is given for every +# parameter in the experiment file. Therefore it is recommended to use the +# multi-run implementation, since it is more versatile. # # In both cases, the dictionary EP is now available. Its keys are the names # of the parameters in the experiment file. Every key refers to the # corresponding value. If multiple values are given in the experiment file, # in every iteration of the loop, another combination of these values is in EP. -# Within the following lines of the Python code, the values of the EP are used -# to implement the desired behaviour of the experiment. +# Within the following lines of Python code, the values of the EP are used to +# implement the desired behaviour of the experiment. for EP in param.loop_experiment_parameters(): # Set model type, domain type, size and resolution param.modelname = "boussinesq" @@ -70,10 +76,12 @@ param.Ly = 2 param.Lx = 3 * param.Ly # Set number of CPU cores used + # Remember to set a fixed (initial) ID in the experiment file, if multiple + # cores are used. param.npx = 1 param.npy = 1 - # Set time settings + # Use a fixed time stepping for a constant framerate in the mp4-file param.tend = 20.0 param.adaptable_dt = False param.dt = 0.02 @@ -120,7 +128,7 @@ f2d = Fluid2d(param, grid) model = f2d.model - # Set initial perturbation of the surface (or interface) + # Set initial perturbation of the interface buoy = model.var.get("buoyancy") buoy[:] = 1 if EP["perturbation"].lower() == "sin": From 9b07f713e3dda87ea3683fc28a177f25b6d5b1d3 Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 19 Jun 2019 15:29:11 +0200 Subject: [PATCH 28/31] EMS: Made compatible with Python < 3.7 --- core/ems.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/ems.py b/core/ems.py index e338cec..4c9d419 100644 --- a/core/ems.py +++ b/core/ems.py @@ -117,7 +117,8 @@ def initialize(self, data_directory: str): ] new_columns = ( static_columns_start - + [DBColumn(sql_type(val), name, val) for name, val in self.parameters.items()] + + [DBColumn(sql_type(self.parameters[name]), name, self.parameters[name]) + for name in self.param_name_list] + static_columns_end ) # If the table exists, check its columns, otherwise create it @@ -150,14 +151,13 @@ def initialize(self, data_directory: str): # - allow to skip columns which are no longer needed # - allow to add new columns if needed # - allow to convert types - for name, val in self.parameters.items(): + for name in self.param_name_list: column = columns.pop(0) col_index = column[0] col_name = column[1] col_type = column[2] - type_ = sql_type(val) + type_ = sql_type(self.parameters[name]) if col_name != name or col_type != type_: - print(repr(val), sql_type(val)) raise InputMismatch( "parameter {} of type {} does not fit into column {} " "with name {} and type {}." From cb234695a3d01412d70e0bb6ccdbf612e9cf7f38 Mon Sep 17 00:00:00 2001 From: Markus Reinert Date: Wed, 8 Jan 2020 09:54:37 +0100 Subject: [PATCH 29/31] EMS: Fix bug in method that changes comments --- ems/Experiment-Manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ems/Experiment-Manager.py b/ems/Experiment-Manager.py index baac850..33a6ddc 100644 --- a/ems/Experiment-Manager.py +++ b/ems/Experiment-Manager.py @@ -1054,7 +1054,10 @@ def do_new_figure(self, params): ### Functionality to MODIFY the entries def do_new_comment(self, params): # Check and parse parameters - table_name, id_ = self.parse_params_to_experiment(params) + experiment_name_id = self.parse_params_to_experiment(params) + if experiment_name_id is None: + return + table_name, id_ = experiment_name_id # Print the current entry fully global display_all display_all = True From 6edac45d06549f68d0cb8853e6e8a25fafecd1ea Mon Sep 17 00:00:00 2001 From: Markus Reinert Date: Wed, 8 Jan 2020 11:19:44 +0100 Subject: [PATCH 30/31] EMS: Explain better the filter-method --- ems/Experiment-Manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ems/Experiment-Manager.py b/ems/Experiment-Manager.py index 33a6ddc..7a19134 100644 --- a/ems/Experiment-Manager.py +++ b/ems/Experiment-Manager.py @@ -590,8 +590,14 @@ def help_filter(self): - filter datetime >= "2019-03-21" - filter perturbation != "gauss" AND duration > 20 It is necessary to put the value for string, datetime or boolean argument - in quotation marks as shown. - To sort the filtered experiments, use SQLite syntax. + in quotation marks as shown. The command + - filter Kdiff + shows all entries where Kdiff is non-zero, whereas the command + - filter NOT Kdiff + shows all entries where Kdiff is zero, following the usual Python convention + that zero is interpreted as False. Note that this does not work for boolean + variables in the database, since they are saved as strings. + To sort the filtered experiments, use SQLite syntax, as in the following Examples: - filter intensity > 0.1 ORDER BY intensity - filter perturbation != "gauss" ORDER BY size_total DESC From 5ee80bb1dfe6236def6d03852e7b9491f30c18f2 Mon Sep 17 00:00:00 2001 From: Markus Reinert Date: Thu, 11 Jun 2020 11:45:26 +0200 Subject: [PATCH 31/31] EMS: Improve code and add new functionality to modify columns This commit brings a lot of code improvements, most of them in the DB-connection class, which was simplified a lot. Furthermore, new functionality was implemented to add or remove columns from a table. The requirement to have numpy installed was removed from the EMShell (apart from the EMShellExtensions, which are considered optional) and matplotlib was made an optional requirement. A lot (but not all of the functionality of EMS) can be used without matplotlib, so it is now possible to run EMShell on a computer without matplotlib. When matplotlib is used, a different backend is now chosen by default, which allows the user to modify the plot manually. This is a big advantage when the user wants to create nice figures. Also a new colormap was introduced that can be chosen by the user. Finally, comments were added and improved. --- ems/Experiment-Manager.py | 628 +++++++++++++++++++++++++++----------- 1 file changed, 448 insertions(+), 180 deletions(-) diff --git a/ems/Experiment-Manager.py b/ems/Experiment-Manager.py index 7a19134..c7028a9 100644 --- a/ems/Experiment-Manager.py +++ b/ems/Experiment-Manager.py @@ -1,53 +1,63 @@ -"""EMShell: Command-line interface for the EMS of fluid2d +"""EMShell: Command-line interface for the EMS of Fluid2D -EMS is the Experiment Management System of fluid2d, a powerful and +EMS is the Experiment Management System of Fluid2D, a powerful and convenient way to handle big sets of experiments, cf. core/ems.py. The EMShell is a command line interface to access, inspect, modify and analyse the database and its entries. -To start this programme, activate fluid2d, then run this script with +To start this programme, activate Fluid2D, then run this script with Python (version 3.6 or newer). Alternatively, without the need to -activate fluid2d, specify the path to the experiment folder as a +activate Fluid2D, specify the path to the experiment folder as a command-line argument when launching this script with Python. This code uses f-strings, which were introduced in Python 3.6: - https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep498 +https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep498 Unfortunately, they create a SyntaxError in older Python versions. It is possible to access the experiment database from multiple processes at the same time, so one can add new entries to the database -with the EMS of fluid2d while looking at the database and performing +with the EMS of Fluid2D while looking at the database and performing data analysis in the EMShell; the EMShell shows automatically the updated version of the database. However, this works less well if the database is accessed by processes on different computers. In this case it can be necessary to restart the EMShell to see changes in the database. It is always necessary to restart the EMShell when a new -class of experiments was added to the database. +class of experiments (a new table) was added to the database. -Author: Markus Reinert, May/June 2019 +Author: Markus Reinert, May/June 2019, June 2020 """ +# Standard library imports import os import re import sys import cmd import time -import numpy as np import shutil -import sqlite3 as dbsys +import sqlite3 import datetime import readline import itertools import subprocess -import matplotlib.pyplot as plt -# Local imports -import EMShellExtensions as EMExt +# Optional matplotlib import +try: + import matplotlib + # Before importing pyplot, choose the Qt5Agg backend of matplotlib, + # which allows to modify graphs and labels in a figure manually. + # If this causes problems, comment or change the following line. + matplotlib.use("Qt5Agg") + import matplotlib.pyplot as plt + from matplotlib.colors import LinearSegmentedColormap +except Exception as e: + print("Warning: matplotlib cannot be imported:", e) + print("Some functionality is deactivated.") + print("Install matplotlib to use all the features of the EMShell.") + matplotlib = None + +# Local import +import EMShellExtensions as EMExt -# Command to open mp4-files -MP4_PLAYER = "mplayer" -# Command to open NetCDF (his or diag) files -NETCDF_VIEWER = "ncview" ### Extensions # Make new shell extensions available by adding them to this dictionary. @@ -60,15 +70,21 @@ class of experiments was added to the database. "wavelength": lambda hisname: 1/EMExt.get_strongest_wavenumber_y(hisname), } +### External programmes called by the EMShell +# Command to open mp4-files +MP4_PLAYER = "mplayer" +# Command to open NetCDF (his or diag) files +NETCDF_VIEWER = "ncview" + ### Settings for the output of a table -# Maximal length of comments (possible values: "AUTO", "FULL" or a positive integer) +# Maximal length of comments (possible values: "AUTO", "FULL", or a positive integer) # This setting is ignored if display_all is True. COMMENT_MAX_LENGTH = "AUTO" # Format-string for real numbers FLOAT_FORMAT = "{:.4f}" # Format-string for file sizes (MB) or disk space (GiB) # Two decimals are used instead of three, to avoid confusion between the dot as -# a decimal separator and a delimiter after 1000. +# a decimal separator and the dot as a delimiter after 1000. SIZE_FORMAT = "{:.2f}" # Symbol to separate two columns of the table LIMITER = " " @@ -116,7 +132,7 @@ class of experiments was added to the database. COLOURS_END = "\033[39;49m" -### Language settings (currently only for dates in easy-to-read-format) +### Language settings (applies currently only for dates in easy-to-read-format) # Definition of languages class English: JUST_NOW = "just now" @@ -143,6 +159,21 @@ class German: LANG = English +# Defintion of a new colormap "iridescent" by Paul Tol +# (https://personal.sron.nl/~pault/#fig:scheme_iridescent) +if matplotlib: + iridescent = LinearSegmentedColormap.from_list( + "iridescent", + ['#FEFBE9', '#FCF7D5', '#F5F3C1', '#EAF0B5', '#DDECBF', + '#D0E7CA', '#C2E3D2', '#B5DDD8', '#A8D8DC', '#9BD2E1', + '#8DCBE4', '#81C4E7', '#7BBCE7', '#7EB2E4', '#88A5DD', + '#9398D2', '#9B8AC4', '#9D7DB2', '#9A709E', '#906388', + '#805770', '#684957', '#46353A'] + ) + iridescent.set_bad('#999999') + iridescent_r = iridescent.reversed() + + ### Global variables modifiable during runtime # Hide the following information in the table # They can be activated with the command "enable" during runtime. @@ -162,13 +193,15 @@ class EMDBConnection: """Experiment Management Database Connection""" def __init__(self, dbpath: str): + """Establish a connection to the database in dbpath and initialise all tables.""" self.connection = None if not os.path.isfile(dbpath): raise FileNotFoundError(f"Database file {dbpath} does not exist.") + # Create a connection to the given database print("-"*50) print("Opening database {}.".format(dbpath)) - self.connection = dbsys.connect(dbpath) + self.connection = sqlite3.connect(dbpath) cursor = self.connection.cursor() # Get all tables of the database @@ -177,6 +210,7 @@ def __init__(self, dbpath: str): self.tables = [EMDBTable(cursor, t[0]) for t in cursor.fetchall()] def __del__(self): + """Close the connection to the database (if any).""" if self.connection: print("Closing database.") print("-"*50) @@ -185,7 +219,11 @@ def __del__(self): def save_database(self): self.connection.commit() + def delete_table(self, table_name): + self.connection.execute(f'DROP TABLE "{table_name}"') + def get_table_overview(self): + """Return a text with metadata about the tables in the database.""" if self.tables: text = "Experiments in database:" for table in self.tables: @@ -198,107 +236,44 @@ def get_table_overview(self): text = "No experiments in database." return text - def get_data(self, table_name, column_name, condition=""): - for table in self.tables: - if table_name == table.name: - return table.get_data(column_name, condition) - print(f'No table with name {table_name} found.') - return [] - - def get_column_names(self, table_name): + def is_valid_table(self, table_name): + """Check if a table with the given name exists.""" for table in self.tables: if table.name == table_name: - return [n for n,t in table.columns] - - def get_table_length(self, table_name): - for table in self.tables: - if table.name == table_name: - return len(table) - - def get_latest_entry(self, table_name): - for table in self.tables: - if table.name == table_name: - return table.get_latest_entry() - - def set_comment(self, table_name, id_, new_comment): - for table in self.tables: - if table.name == table_name: - return table.set_comment(id_, new_comment) - print(f'Unknown experiment: "{table_name}"') + return True return False - def show_all_tables(self): - for table in self.tables: - print("-"*50) - print(table) - - def show_table(self, name): - print("-"*50) - for table in self.tables: - if table.name == name: - print(table) - return - print('Unknown experiment: "{}"'.format(name)) - - def show_filtered_table(self, table_name, statement): + def is_valid_column(self, column_name): + """Check if a column with the given name exists in any table.""" for table in self.tables: - if table.name == table_name or table_name == "": - print("-"*50) - table.print_selection(statement) - print("-"*50) - - def show_sorted_table(self, table_name, statement): - for table in self.tables: - if table.name == table_name or table_name == "": - print("-"*50) - table.print_sorted(statement) - print("-"*50) - - def show_comparison(self, table_name, ids, highlight=False): - for table in self.tables: - if table.name == table_name: - print("-"*50) - table.print_comparison(ids, highlight) - print("-"*50) - return - - def is_valid_column(self, column_name, table_name=""): - """Check whether a column with the given name exists. - - If no table_name is specified, check whether any table has such - a column, otherwise look only in the table with the given name.""" - for table in self.tables: - if table.name == table_name or table_name == "": - for n, t in table.columns: - if column_name == n: - return True + for n, t in table.columns: + if column_name == n: + return True return False - def table_exists(self, table_name): - for table in self.tables: - if table.name == table_name: - return True - return False + def get_all_column_names(self): + """Return a list of the names of all columns in all tables.""" + return [n for table in self.tables for n, t in table.columns] - def entry_exists(self, table_name, id_): - for table in self.tables: - if table.name == table_name: - return table.entry_exists(id_) + def do(self, function, table_name, *args, **kwargs): + """Call a function on a table specified by its name and return the result. - def delete_entry(self, table_name, id_): + This method calls function(table, *args, **kwargs), where table + is the EMDBTable object with the name table_name, and returns + the result of this function call. If no table with the given + name exists, then a warning is printed and None is returned. + """ for table in self.tables: if table.name == table_name: - table.delete_entry(id_) - return - - def delete_table(self, table_name): - self.connection.execute(f'DROP TABLE "{table_name}"') + return function(table, *args, **kwargs) + print('No table exists with name "{}".'.format(table_name)) + return None class EMDBTable: """Experiment Management Database Table""" - def __init__(self, cursor: dbsys.Cursor, name: str): + def __init__(self, cursor: sqlite3.Cursor, name: str): self.name = str(name) self.c = cursor # Get columns @@ -307,7 +282,7 @@ def __init__(self, cursor: dbsys.Cursor, name: str): # column[0]: index from 0, column[1]: name, column[2]: type def __str__(self): - # Get entries + """Return a text representation of all entries formated as a table.""" self.c.execute('SELECT * from "{}"'.format(self.name)) return string_format_table(self.name, self.columns, self.c.fetchall()) @@ -322,17 +297,24 @@ def get_data(self, column_name, condition=""): if condition else f'SELECT {column_name} FROM "{self.name}"' ) - except dbsys.OperationalError as e: + except sqlite3.OperationalError as e: print(f'SQL error for experiment "{self.name}":', e) return [] else: return [e[0] for e in self.c.fetchall()] + def get_column_names(self): + return [n for n, t in self.columns] + def get_latest_entry(self): self.c.execute('SELECT id from "{}" ORDER BY datetime DESC'.format(self.name)) result = self.c.fetchone() self.c.fetchall() # otherwise the database stays locked - return result[0] if result else None + if result: + return result[0] + else: + print('Table "{}" is empty.'.format(self.name)) + return None def entry_exists(self, id_): self.c.execute('SELECT id FROM "{}" WHERE id = ?'.format(self.name), (id_,)) @@ -346,7 +328,7 @@ def delete_entry(self, id_): def print_selection(self, statement): try: self.c.execute('SELECT * FROM "{}" WHERE {}'.format(self.name, statement)) - except dbsys.OperationalError as e: + except sqlite3.OperationalError as e: print('SQL error for experiment "{}":'.format(self.name), e) else: print(string_format_table(self.name, self.columns, self.c.fetchall())) @@ -354,7 +336,7 @@ def print_selection(self, statement): def print_sorted(self, statement): try: self.c.execute('SELECT * FROM "{}" ORDER BY {}'.format(self.name, statement)) - except dbsys.OperationalError as e: + except sqlite3.OperationalError as e: print('SQL error for experiment "{}":'.format(self.name), e) else: print(string_format_table(self.name, self.columns, self.c.fetchall())) @@ -366,7 +348,7 @@ def print_comparison(self, ids, highlight=False): if ids else 'SELECT * FROM "{}"'.format(self.name) ) - except dbsys.OperationalError as e: + except sqlite3.OperationalError as e: print('SQL error for experiment "{}":'.format(self.name), e) return full_entries = self.c.fetchall() @@ -385,13 +367,13 @@ def print_comparison(self, ids, highlight=False): [ [val for c, val in enumerate(row) if c in different_columns] for row in full_entries - ] + ], )) else: # Print the full table and highlight the columns with differences print(string_format_table( self.name, self.columns, full_entries, - [col[0] for c, col in enumerate(self.columns) if c in different_columns] + [col[0] for c, col in enumerate(self.columns) if c in different_columns], )) def set_comment(self, id_, new_comment): @@ -400,14 +382,146 @@ def set_comment(self, id_, new_comment): 'UPDATE "{}" SET comment = ? WHERE id = ?'.format(self.name), (new_comment, id_,) ) - except dbsys.OperationalError as e: + except sqlite3.OperationalError as e: print('SQL error for experiment "{}":'.format(self.name), e) return False else: return True + def add_column(self, column_name, data): + """Add a new column to the table with the given name and data. + + The new column is always of datatype REAL and is added just + before the column "duration", i.e., at the end of the + user-defined columns. + + Adding a new column is a bit tricky in SQLite, because this is + not supported natively. Therefore, it is necessary to create a + new table with the additional column and move all the data there + from the original table. For this, we rename, at first, the + existing table, then we create a new one with the original name, + we migrate all the data from the old to the new table and we + remove the old table. Then, finally, we fill the new table with + the given data. If anything went wrong during this process, + there should be two tables, one with the original name of the + table, and one with the prefix "tmp_", so it should be possible + for the user to recover the data by manually moving it from the + temporary to the new table. + """ + old_column_name_string = ", ".join( + ['"{}"'.format(col[0]) for col in self.columns] + ) + for i, column in enumerate(self.columns): + if column[0] == "duration": + i_new_column = i + break + else: + print( + 'Cannot add a new column before "duration" because ' + 'no column "duration" exists in the table "{}".'.format(self.name) + ) + return False + self.columns = ( + self.columns[:i_new_column] + [(column_name, "REAL")] + self.columns[i_new_column:] + ) + new_column_name_type_string = ", ".join([ + '"{}" {}'.format(column_name, column_type) + for column_name, column_type in self.columns + ]) + tmp_name = "tmp_" + self.name + try: + # Rename the current table + self.c.execute( + 'ALTER TABLE "{}" RENAME TO "{}"'.format(self.name, tmp_name) + ) + # Create a new table with its name and the new column + self.c.execute( + 'CREATE TABLE "{}" ({})'.format(self.name, new_column_name_type_string) + ) + # Copy the data from the old to the new table + self.c.execute( + 'INSERT INTO "{}" ({}) SELECT {} FROM "{}"'.format( + self.name, old_column_name_string, old_column_name_string, tmp_name + ) + ) + # Remove the old table + self.c.execute('DROP TABLE "{}"'.format(tmp_name)) + except sqlite3.OperationalError as e: + print( + 'SQL error for experiment "{}" when adding column "{}": {}' + .format(self.name, column_name, e) + ) + return False + # Get the IDs of the entries in the table + try: + self.c.execute(f'SELECT id FROM "{self.name}"') + except sqlite3.OperationalError as e: + print(f'SQL error for experiment "{self.name}" when fetching IDs:', e) + return False + # Fill every entry + return_status = True + for entry, value in zip(self.c.fetchall(), data): + id_ = entry[0] + try: + # TODO: do this with ?-notation instead of string format + self.c.execute( + f'UPDATE "{self.name}" SET {column_name} = {value} WHERE id = {id_}' + ) + except sqlite3.OperationalError as e: + print( + 'SQL error for experiment "{}" when adding value {} to entry {}:' + .format(self.name, repr(value), id_), e + ) + return_status = False + return return_status + + def remove_column(self, column_name): + """Remove the column with the given name from the table. + + A similar caveat as for add_column applies. + """ + for column in self.columns: + if column[0] == column_name: + self.columns.remove(column) + break + else: + print('Cannot find column "{}" in table "{}".'.format(column_name, self.name)) + return False + new_column_name_string = ", ".join([ + '"{}"'.format(column_name) + for column_name, column_type in self.columns + ]) + new_column_name_type_string = ", ".join([ + '"{}" {}'.format(column_name, column_type) + for column_name, column_type in self.columns + ]) + tmp_name = "tmp_" + self.name + try: + # Rename the current table + self.c.execute( + 'ALTER TABLE "{}" RENAME TO "{}"'.format(self.name, tmp_name) + ) + # Create a new table with its name and without the column + self.c.execute( + 'CREATE TABLE "{}" ({})'.format(self.name, new_column_name_type_string) + ) + # Copy the data from the old to the new table + self.c.execute( + 'INSERT INTO "{}" SELECT {} FROM "{}"'.format( + self.name, new_column_name_string, tmp_name + ) + ) + # Remove the old table + self.c.execute('DROP TABLE "{}"'.format(tmp_name)) + except sqlite3.OperationalError as e: + print( + 'SQL error for experiment "{}" when removing column "{}": {}' + .format(self.name, column_name, e) + ) + return False + return True + -# https://docs.python.org/3/library/cmd.html class EMShell(cmd.Cmd): """Experiment Management (System) Shell""" @@ -422,14 +536,19 @@ class EMShell(cmd.Cmd): ruler = "-" def __init__(self, experiments_dir: str): + """Establish a connection to the database in the given directory. + + This initialises also the command history. + """ self.initialized = False super().__init__() print("-"*50) - print("Fluid2d Experiment Management System (EMS)") + print("Fluid2D Experiment Management System (EMS)") self.exp_dir = experiments_dir self.con = EMDBConnection(os.path.join(self.exp_dir, "experiments.db")) self.intro += "\n" + self.con.get_table_overview() + "\n" self.selected_table = "" + # Settings for saving the command history self.command_history_file = os.path.join(self.exp_dir, ".emshell_history") readline.set_history_length(1000) @@ -439,6 +558,7 @@ def __init__(self, experiments_dir: str): self.initialized = True def __del__(self): + """Save the command history (not the database).""" if self.initialized: print("Saving command history.") readline.write_history_file(self.command_history_file) @@ -501,10 +621,12 @@ def complete_disable(self, text, line, begidx, endidx): if not hidden_information.issuperset({'size_diag', 'size_flux', 'size_his', 'size_mp4', 'size_total'}): parameters += ['size'] - # Add columns of selected table to the list of auto-completions - for table in self.con.tables: - if self.selected_table == "" or self.selected_table == table.name: - parameters += [n for n,t in table.columns] + # Add columns of the selected table (or of all tables if none is + # selected) to the list of auto-completions + if self.selected_table: + parameters += self.con.do(EMDBTable.get_column_names, self.selected_table) + else: + parameters += self.con.get_all_column_names() return [p for p in parameters if p.startswith(text) and p not in hidden_information] def help_disable(self): @@ -539,13 +661,16 @@ def do_show(self, params): if len(params) == 0: # No table name given if self.selected_table: - self.con.show_table(self.selected_table) + print("-"*50) + self.con.do(print, self.selected_table) else: - self.con.show_all_tables() + for table in self.con.tables: + print("-"*50) + print(table) else: for table_name in sorted(params): - # Try to open the specified table. - self.con.show_table(table_name) + print("-"*50) + self.con.do(print, table_name) print("-"*50) def complete_show(self, text, line, begidx, endidx): @@ -565,6 +690,10 @@ def help_show(self): See also: filter, sort.""") def do_filter(self, params): + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to filter it.') + return global display_all if params.endswith(" -v"): params = params[:-3] @@ -573,8 +702,10 @@ def do_filter(self, params): display_all = False if not params: print('No condition to filter given. Type "help filter" for further information.') - else: - self.con.show_filtered_table(self.selected_table, params) + return + print("-"*50) + self.con.do(EMDBTable.print_selection, self.selected_table, params) + print("-"*50) def complete_filter(self, text, line, begidx, endidx): return self.column_name_completion(self.selected_table, text) @@ -607,6 +738,10 @@ def help_filter(self): See also: sort, show.""") def do_sort(self, params): + # Check the run condition + if not self.selected_table: + print("No experiment selected. Select an experiment to sort it.") + return global display_all if params.endswith(" -v"): params = params[:-3] @@ -616,7 +751,9 @@ def do_sort(self, params): if not params: print('No parameter to sort given. Type "help sort" for further information.') else: - self.con.show_sorted_table(self.selected_table, params) + print("-"*50) + self.con.do(EMDBTable.print_sorted, self.selected_table, params) + print("-"*50) def complete_sort(self, text, line, begidx, endidx): return self.column_name_completion(self.selected_table, text) @@ -657,13 +794,15 @@ def do_compare(self, params): if len(ids) < 2: print("Please specify at least 2 different IDs.") return - elif self.con.get_table_length(self.selected_table) < 2: + elif self.con.do(len, self.selected_table) < 2: print("Selected experiment contains less than 2 entries.") return else: # No IDs means no filtering, i.e., all entries are compared ids = [] - self.con.show_comparison(self.selected_table, ids, highlight) + print("-"*50) + self.con.do(EMDBTable.print_comparison, self.selected_table, ids, highlight) + print("-"*50) def help_compare(self): print("""> compare [IDs] [-h] [-v] @@ -825,6 +964,11 @@ def help_calculate(self): Its name must not contain any whitespace.""") def do_plot(self, params): + # Check that the required module is loaded + if not matplotlib: + print("This feature requires matplotlib, which has not been imported successfully.") + print("The error message can be found at the beginning of the programme output.") + return # Check the run condition if not self.selected_table: print('No experiment selected. Select an experiment to plot its data.') @@ -885,6 +1029,11 @@ def help_plot(self): selected class of experiments with the size of their history-file.""") def do_scatter(self, params): + # Check that the required module is loaded + if not matplotlib: + print("This feature requires matplotlib, which has not been imported successfully.") + print("The error message can be found at the beginning of the programme output.") + return # Check the run condition if not self.selected_table: print('No experiment selected. Select an experiment to plot its data.') @@ -949,6 +1098,11 @@ def help_scatter(self): of the current class without diffusion.""") def do_pcolor(self, params): + # Check that the required module is loaded + if not matplotlib: + print("This feature requires matplotlib, which has not been imported successfully.") + print("The error message can be found at the beginning of the programme output.") + return # Check the run condition if not self.selected_table: print('No experiment selected. Select an experiment to plot its data.') @@ -962,20 +1116,27 @@ def do_pcolor(self, params): datas = self.get_multiple_data(self.selected_table, variables, parameters["condition"]) if not datas: return + x_data = datas[0] + y_data = datas[1] + z_data = datas[2] # Arrange the data in a grid - xvalues = sorted(set(datas[0])) - yvalues = sorted(set(datas[1])) - print(f'There are {len(xvalues)} unique x-values and {len(yvalues)} unique y-values.') - data_grid = np.full((len(yvalues), len(xvalues)), np.nan) - for i, zval in enumerate(datas[2]): - data_grid[yvalues.index(datas[1][i]), xvalues.index(datas[0][i])] = zval - # If no shading is applied, extend the axes to have each rectangle - # centred at the corresponding (x,y)-value + xvalues = sorted(set(x_data)) + yvalues = sorted(set(y_data)) + nx = len(xvalues) + ny = len(yvalues) + print(f"There are {nx} unique x-values and {ny} unique y-values.") + data_grid = [[float("nan")] * nx for i in range(ny)] + for i, zval in enumerate(z_data): + data_grid[yvalues.index(y_data[i])][xvalues.index(x_data[i])] = zval if parameters["shading"] == 'flat': - xdiff2 = np.diff(xvalues)/2 - xaxis = [xvalues[0] - xdiff2[0], *(xvalues[:-1] + xdiff2), xvalues[-1] + xdiff2[-1]] - ydiff2 = np.diff(yvalues)/2 - yaxis = [yvalues[0] - ydiff2[0], *(yvalues[:-1] + ydiff2), yvalues[-1] + ydiff2[-1]] + # If no shading is applied, extend the axes to have each rectangle + # centred at the corresponding (x,y)-value + xaxis = [xvalues[0] - (xvalues[1] - xvalues[0]) / 2] + xaxis += [xvalues[i] + (xvalues[i+1] - xvalues[i]) / 2 for i in range(len(xvalues) - 1)] + xaxis += [xvalues[-1] + (xvalues[-1] - xvalues[-2]) / 2] + yaxis = [yvalues[0] - (yvalues[1] - yvalues[0]) / 2] + yaxis += [yvalues[i] + (yvalues[i+1] - yvalues[i]) / 2 for i in range(len(yvalues) - 1)] + yaxis += [yvalues[-1] + (yvalues[-1] - yvalues[-2]) / 2] else: xaxis = xvalues yaxis = yvalues @@ -990,10 +1151,15 @@ def do_pcolor(self, params): plt.title(plot_title) plt.xlabel(variables[0]) plt.ylabel(variables[1]) + cmap = parameters["cmap"] + if cmap == "iridescent": + cmap = iridescent + if cmap == "iridescent_r": + cmap = iridescent_r try: plt.pcolormesh( xaxis, yaxis, data_grid, - cmap=parameters["cmap"], shading=parameters["shading"], + cmap=cmap, shading=parameters["shading"], vmin=parameters["zmin"], vmax=parameters["zmax"], ) except Exception as e: @@ -1032,6 +1198,11 @@ def help_pcolor(self): of the current class without diffusion.""") def do_save_figure(self, params): + # Check that the required module is loaded + if not matplotlib: + print("This feature requires matplotlib, which has not been imported successfully.") + print("The error message can be found at the beginning of the programme output.") + return if not params: params = "png" if params in ["png", "pdf", "svg"]: @@ -1055,6 +1226,11 @@ def help_save_figure(self): def do_new_figure(self, params): """Open a window to draw the next plot in a new figure.""" + # Check that the required module is loaded + if not matplotlib: + print("This feature requires matplotlib, which has not been imported successfully.") + print("The error message can be found at the beginning of the programme output.") + return plt.figure() ### Functionality to MODIFY the entries @@ -1067,7 +1243,9 @@ def do_new_comment(self, params): # Print the current entry fully global display_all display_all = True - self.con.show_filtered_table(table_name, f"id = {id_}") + print("-"*50) + self.con.do(EMDBTable.print_selection, table_name, f"id = {id_}") + print("-"*50) # Ask for user input print("Write a new comment for this entry (Ctrl+D to finish, Ctrl+C to cancel):") comment_lines = [] @@ -1082,17 +1260,108 @@ def do_new_comment(self, params): comment_lines.append(line) comment = "\n".join(comment_lines).strip() # Update the comment - if self.con.set_comment(table_name, id_, comment): + if self.con.do(EMDBTable.set_comment, table_name, id_, comment): self.con.save_database() print("New comment was saved.") - else: - print("An error occured.") def help_new_comment(self): print("""> new_comment [experiment] Ask the user to enter a new comment for an experiment.""") self.print_param_parser_help() + def do_add_column(self, params): + # TODO: this adds currently only REAL data (float) before the "duration" column + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to add a column.') + return + params = params.split() + if len(params) != 2: + print( + "Exactly two arguments are needed: name of the new column " + "and name of the tool to fill it." + ) + return + column_name, tool_name = params + if column_name in self.con.do(EMDBTable.get_column_names, self.selected_table): + print( + "A column with the name {} exists already in {}." + .format(column_name, self.selected_table) + ) + return + data = self.get_data(self.selected_table, tool_name, condition="") + if not data: + print("Getting data failed.") + return + if self.con.do(EMDBTable.add_column, self.selected_table, column_name, data): + self.con.save_database() + print("Done.") + else: + print("An error occured.") + + def complete_add_column(self, text, line, begidx, endidx): + return self.plot_attribute_completion(text, []) + + def help_add_column(self): + print("""> add_column + Add a new column with the given name to the selected table. The new + column will be added before the "duration" column, that means, at + the end of the user-defined columns. It will be of type REAL + (float) and will be filled with data computed by the specified tool. + For more information about tools, see "calculate". Instead of a + tool, also the name of an already existing column can be given to + copy the data from there. + + Warning: If you want to run more experiments of the selected class + after adding a new column to the table, you must also add a + corresponding parameter in the experiment file with the same + datatype and at the same position.""" + ) + + def do_remove_column(self, params): + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to remove a column.') + return + column_name = params + if column_name not in self.con.do(EMDBTable.get_column_names, self.selected_table): + print( + "No column with the name {} exists in {}." + .format(column_name, self.selected_table) + ) + return + print( + 'Do you really want to permanently delete the column {!r} from {!r}?' + .format(column_name, self.selected_table) + ) + print('This cannot be undone.') + answer = input('Continue [yes/no] ? ') + if answer == 'yes': + if self.con.do(EMDBTable.remove_column, self.selected_table, column_name): + self.con.save_database() + print("Done.") + else: + print("An error occured.") + elif answer == "no": + # Do nothing. + pass + else: + print('Answer was not "yes". No data removed.') + + def complete_remove_column(self, text, line, begidx, endidx): + return self.column_name_completion(self.selected_table, text) + + def help_remove_column(self): + print("""> remove_column + Remove the specified column from the selected table. + + The user is asked for confirmation before the command is executed. + + Warning: If you want to run more experiments of the selected class + after removing a column from the table, you must also remove the + corresponding parameter in the experiment file.""" + ) + ### Functionality to CLEAN up def do_remove(self, params): # Check conditions and parameters @@ -1112,11 +1381,13 @@ def do_remove(self, params): global display_all display_all = True if len(ids) == 1: - statement = "id = {}".format(*ids) + statement = "id = {}".format(ids[0]) else: statement = "id IN {}".format(tuple(ids)) print('WARNING: the following entries will be DELETED:') - self.con.show_filtered_table(self.selected_table, statement) + print("-"*50) + self.con.do(EMDBTable.print_selection, self.selected_table, statement) + print("-"*50) # Print full information of related folders folders = [] @@ -1135,13 +1406,13 @@ def do_remove(self, params): print(" -", f) # Final check - print('Do you really want to permanently delete these files, folders and entries?') + print('Do you really want to permanently delete these files, folders, and entries?') print('This cannot be undone.') answer = input('Continue [yes/no] ? ') if answer == 'yes': # Remove entries for id_ in ids: - self.con.delete_entry(self.selected_table, id_) + self.con.do(EMDBTable.delete_entry, self.selected_table, id_) print('Deleted entry', id_, 'from experiment "{}".'.format(self.selected_table)) self.con.save_database() # Remove files @@ -1180,7 +1451,7 @@ def do_remove_selected_class(self, params): print('This command takes no attributes. Cancelling.') return - if self.con.get_table_length(self.selected_table): + if self.con.do(len, self.selected_table) > 0: print('Selected experiment class contains experiments. Remove these ' 'experiments first before removing the class. Cancelling.') return @@ -1238,7 +1509,7 @@ def do_select(self, params): if params == "": self.prompt = "(EMS) " self.selected_table = params - elif self.check_table_exists(params): + elif self.con.is_valid_table(params): self.prompt = "({}) ".format(params) self.selected_table = params else: @@ -1260,6 +1531,10 @@ def do_exit(self, params): return True def do_EOF(self, params): + """This is called when Ctrl+D is pressed. + If an experiment is selected, it will be deselected. + If no experiment is selected, the programme will exit. + """ if self.selected_table: print("select") self.prompt = "(EMS) " @@ -1280,21 +1555,13 @@ def plot_attribute_completion(self, text, parameters): if not self.selected_table: print("\nError: select an experiment first!") return [] - parameters += self.con.get_column_names(self.selected_table) + parameters += self.con.do(EMDBTable.get_column_names, self.selected_table) parameters.extend(extra_tools.keys()) return [p for p in parameters if p.startswith(text)] def column_name_completion(self, table_name, text): - completions = [] - for table in self.con.tables: - if table.name == table_name or table_name == "": - for column_name, column_type in table.columns: - if column_name.startswith(text): - completions.append(column_name) - return completions - - def check_table_exists(self, name): - return name in [table.name for table in self.con.tables] + columns = self.con.do(EMDBTable.get_column_names, table_name) + return [c for c in columns if c.startswith(text)] def parse_params_to_experiment(self, params: str) -> [str, int]: """Parse and check input of the form "[experiment] ". @@ -1321,7 +1588,7 @@ def parse_params_to_experiment(self, params: str) -> [str, int]: # Parse and check the table name table_name = " ".join(params).strip() if table_name: - if not self.con.table_exists(table_name): + if not self.con.is_valid_table(table_name): print(f'No experiment class with the name "{table_name}" exists.') return None else: @@ -1331,11 +1598,10 @@ def parse_params_to_experiment(self, params: str) -> [str, int]: table_name = self.selected_table # Check the ID if id_ == "last": - id_ = self.con.get_latest_entry(table_name) + id_ = self.con.do(EMDBTable.get_latest_entry, table_name) if id_ is None: - print(f'No entry exists for the experiment class "{table_name}".') return None - elif not self.con.entry_exists(table_name, id_): + elif not self.con.do(EMDBTable.entry_exists, table_name, id_): print(f'No entry with ID {id_} exists for the experiment class "{table_name}".') return None return table_name, id_ @@ -1405,11 +1671,10 @@ def parse_multiple_ids(self, param_list): ids = [] for id_ in param_list: if id_ == "last": - id_ = self.con.get_latest_entry(self.selected_table) + id_ = self.con.do(EMDBTable.get_latest_entry, self.selected_table) if id_ is None: - print(f'No entry exists for the experiment class "{self.selected_table}".') return - elif id_ in ids: + if id_ in ids: continue else: try: @@ -1419,7 +1684,7 @@ def parse_multiple_ids(self, param_list): return if id_ in ids: continue - if not self.con.entry_exists(self.selected_table, id_): + if not self.con.do(EMDBTable.entry_exists, self.selected_table, id_): print('No entry with ID', id_, 'exists for the selected experiment.') return ids.append(id_) @@ -1458,11 +1723,11 @@ def get_multiple_data(self, table_name, variables, condition): return datas def get_data(self, table_name, parameter, condition): - if self.con.is_valid_column(parameter, table_name): - return self.con.get_data(table_name, parameter, condition) + if parameter in self.con.do(EMDBTable.get_column_names, table_name): + return self.con.do(EMDBTable.get_data, table_name, parameter, condition) elif parameter in extra_tools: data = [] - ids = self.con.get_data(table_name, "id", condition) + ids = self.con.do(EMDBTable.get_data, table_name, "id", condition) print(f'Calculating data with tool "{parameter}" for {len(ids)} experiments.') tstart = time.time() for id_ in ids: @@ -1514,10 +1779,12 @@ def print_general_plot_help(plot_type, shading=False): ) elif plot_type == "3d": print(""" - To specify the colour map of the plot, use "-cmap=", where is - the name of a colour map for matplotlib like "-cmap=bwr" to plot in blue- - white-red or "-cmap=jet" for a colourful rainbow. For further examples, - consider the website https://matplotlib.org/users/colormaps.html . + To specify the colour map of the plot, use "-cmap=", where + is the name of a colour map for matplotlib, like "-cmap=bwr" + to plot in blue-white-red or "-cmap=iridescent" to use Paul Tol's + beautiful colour scheme that also works in colour-blind vision. + For further inspiration, consider the following website: + https://matplotlib.org/users/colormaps.html To specify the range in x-, y- or z-direction, use the following attributes: -xmin=, -xmax=, -ymin=, -ymax=, -zmin=, -zmax= @@ -1774,7 +2041,8 @@ def make_nice_time_string(datetime_object, datetime_reference): ) # Activate interactive plotting -plt.ion() +if matplotlib: + plt.ion() # Compile regular expression to match parameters # This matches -xmin=, -xmax=, -ymin=, -ymax=, -zmin=, -zmax=