diff --git a/initial_conf.py b/initial_conf.py
index 9adcb382..efbcc879 100755
--- a/initial_conf.py
+++ b/initial_conf.py
@@ -16,6 +16,9 @@
from getpass import getpass
import pandas as pd
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
def initial_conf(save_user=True, replace_user=False, global_config_flag=True):
@@ -29,13 +32,13 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True):
try_find_conf_file()
except FileNotFoundError:
pass
- print("DataJoint configuration file not found. Running configuration script...")
+ logger.info("DataJoint configuration file not found. Running configuration script...")
if global_config_flag:
- print(
+ logger.info(
"Global configuration flag is set to True. The configuration will be saved in the global configuration file."
)
else:
- print(
+ logger.info(
"Global configuration flag is set to False. The configuration will be saved in the local configuration file."
)
@@ -44,14 +47,11 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True):
host = "datajoint00.pni.princeton.edu"
user_already = False
- if (
- "database.user" in dj.config
- and dj.config.instance._conf["database.user"] is not None
- ):
+ if "database.user" in dj.config and dj.config.instance._conf["database.user"] is not None:
user_already = True
if replace_user or not user_already:
- print("Enter your username (Princeton NETID):")
+ logger.info("Enter your username (Princeton NETID):")
user = input()
password = getpass()
dj.conn(host=host, user=user, password=password)
@@ -65,7 +65,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True):
dj.config["database.host"] = host
if "custom" not in dj.config:
- dj.config["custom"] = dict()
+ dj.config["custom"] = {}
dj.config["custom"]["database.prefix"] = "u19_"
import u19_pipeline.lab as lab
@@ -76,9 +76,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True):
# Transform variables to list and path if applicable
for custom_var in custom_vars_names:
- this_var = custom_vars.loc[
- custom_vars["custom_variable"] == custom_var, "value"
- ].tolist()
+ this_var = custom_vars.loc[custom_vars["custom_variable"] == custom_var, "value"].tolist()
# If custom variables are directories, get local path for this system
if "dir" in custom_var:
@@ -92,22 +90,20 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True):
# Get store info
if "stores" not in dj.config:
- dj.config["stores"] = dict()
+ dj.config["stores"] = {}
dj_stores = lab.DjStores.fetch(as_dict=True)
- dj_stores_dict = dict()
+ dj_stores_dict = {}
for i in dj_stores:
store_name = i.pop("store_name")
dj_stores_dict[store_name] = i
- dj_stores_dict[store_name]["location"] = (
- lab.Path().get_local_path2(i["location"]).as_posix()
- )
+ dj_stores_dict[store_name]["location"] = lab.Path().get_local_path2(i["location"]).as_posix()
dj.config["stores"] = dj_stores_dict
if global_config_flag:
- print(
+ logger.info(
"Global configuration flag is set to True. The configuration will be saved in the global configuration file."
)
dj.config.save_global()
@@ -123,9 +119,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True):
help="prevent to save user into conf file",
action="store_true",
)
- parser.add_argument(
- "--replace_user", "-r", help="replace user in conf file", action="store_true"
- )
+ parser.add_argument("--replace_user", "-r", help="replace user in conf file", action="store_true")
parser.add_argument(
"--not_global_config",
"-ng",
@@ -133,7 +127,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True):
action="store_true",
)
- default_args = dict()
+ default_args = {}
default_args["save_user"] = True
default_args["replace_user"] = False
default_args["global_config"] = False
diff --git a/notebooks/Marking_dead_mice_batch.ipynb b/notebooks/Marking_dead_mice_batch.ipynb
index 6e50e8cc..fd6fd9e4 100644
--- a/notebooks/Marking_dead_mice_batch.ipynb
+++ b/notebooks/Marking_dead_mice_batch.ipynb
@@ -6,8 +6,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from u19_pipeline import subject, action, acquisition\n",
- "import pandas as pd"
+ "from u19_pipeline import action, subject"
]
},
{
@@ -30,15 +29,18 @@
],
"source": [
"query1 = 'subject_fullname like \"joun%\"'\n",
- "query2 = 'subject_fullname != \"jounhong_Ai228_686\"' # This mouse is still alive\n",
+ "query2 = 'subject_fullname != \"jounhong_Ai228_686\"' # This mouse is still alive\n",
"query3 = 'subject_status = \"InExperiments\"'\n",
"query4 = 'effective_date < \"2024-08-01\"'\n",
- "data = (action.SubjectStatus() * subject.Subject() & (' and '.join([query1, query2, query3, query4]))).fetch(format='frame')\n",
+ "data = (\n",
+ " action.SubjectStatus() * subject.Subject()\n",
+ " & (\" and \".join([query1, query2, query3, query4]))\n",
+ ").fetch(format=\"frame\")\n",
"\n",
- "df = data.reset_index().sort_values(by='effective_date', ascending=False)\n",
+ "df = data.reset_index().sort_values(by=\"effective_date\", ascending=False)\n",
"\n",
"\n",
- "df['subject_fullname']"
+ "df[\"subject_fullname\"]"
]
},
{
@@ -59,8 +61,8 @@
"query1 = 'subject_fullname like \"joun%\"'\n",
"\n",
"# These will need to be properly handled in the future if the lens are not 0\n",
- "print(len(subject.SubjectActionAutomatic() & query1 & ' valid_until_date is NULL'))\n",
- "print(len(subject.SubjectActionManual() & query1 & ' valid_until_date is NULL'))"
+ "print(len(subject.SubjectActionAutomatic() & query1 & \" valid_until_date is NULL\"))\n",
+ "print(len(subject.SubjectActionManual() & query1 & \" valid_until_date is NULL\"))"
]
},
{
@@ -78,38 +80,28 @@
}
],
"source": [
- "for sub in df['subject_fullname']:\n",
- " key = {\n",
- " 'subject_fullname': sub,\n",
- " 'cage': '(grave)'}\n",
+ "for sub in df[\"subject_fullname\"]:\n",
+ " key = {\"subject_fullname\": sub, \"cage\": \"(grave)\"}\n",
" subject.CagingStatus.update1(key)\n",
"\n",
- " key = {\n",
- " 'subject_fullname': sub,\n",
- " 'location': 'valhalla'}\n",
+ " key = {\"subject_fullname\": sub, \"location\": \"valhalla\"}\n",
"\n",
" subject.Subject.update1(key)\n",
"\n",
- " key = {\n",
- " 'subject_fullname': sub,\n",
- " 'death_date': '2025-02-05'\n",
- " }\n",
+ " key = {\"subject_fullname\": sub, \"death_date\": \"2025-02-05\"}\n",
"\n",
" subject.Death.insert1(key, skip_duplicates=True)\n",
"\n",
" key = {\n",
- " 'subject_fullname': sub,\n",
- " 'effective_date': '2025-02-05',\n",
- " 'subject_status': 'Dead',\n",
- " 'water_per_day': 1,\n",
- " 'schedule': 'Nothing/Nothing/Nothing/Nothing/Nothing/Nothing/Nothing'\n",
+ " \"subject_fullname\": sub,\n",
+ " \"effective_date\": \"2025-02-05\",\n",
+ " \"subject_status\": \"Dead\",\n",
+ " \"water_per_day\": 1,\n",
+ " \"schedule\": \"Nothing/Nothing/Nothing/Nothing/Nothing/Nothing/Nothing\",\n",
" }\n",
"\n",
" action.SubjectStatus.insert1(key)\n",
"\n",
- "\n",
- "\n",
- "\n",
" print(sub)"
]
},
diff --git a/notebooks/TestAlertSystem.ipynb b/notebooks/TestAlertSystem.ipynb
index b3a59a11..8183f757 100644
--- a/notebooks/TestAlertSystem.ipynb
+++ b/notebooks/TestAlertSystem.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -47,10 +48,8 @@
],
"source": [
"import datajoint as dj\n",
- "import pandas as pd\n",
- "import datetime\n",
- "import u19_pipeline.alert_system.main_alert_system as mas\n",
- "\n"
+ "\n",
+ "import u19_pipeline.alert_system.main_alert_system as mas"
]
},
{
@@ -123,6 +122,7 @@
],
"source": [
"import u19_pipeline.alert_system.custom_alerts.subject_trial as subject_trial\n",
+ "\n",
"df = subject_trial.main()\n",
"df"
]
@@ -152,7 +152,7 @@
}
],
"source": [
- "mas.main_alert_system()\n"
+ "mas.main_alert_system()"
]
},
{
@@ -161,8 +161,8 @@
"metadata": {},
"outputs": [],
"source": [
- "import u19_pipeline.subject as subject\n",
- "import u19_pipeline.action as action"
+ "import u19_pipeline.action as action\n",
+ "import u19_pipeline.subject as subject"
]
},
{
diff --git a/notebooks/TestAutomaticPipeline.ipynb b/notebooks/TestAutomaticPipeline.ipynb
index a14e48b3..61bb0d57 100644
--- a/notebooks/TestAutomaticPipeline.ipynb
+++ b/notebooks/TestAutomaticPipeline.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -42,16 +43,13 @@
"import datajoint as dj\n",
"import pandas as pd\n",
"\n",
- "import u19_pipeline.ephys_pipeline as ephys_pipeline\n",
- "from u19_pipeline.ephys_pipeline import ephys_element\n",
- "\n",
- "import u19_pipeline.imaging_pipeline as imaging_pipeline\n",
- "from u19_pipeline.imaging_pipeline import imaging_element, scan_element\n",
- "\n",
"import u19_pipeline.automatic_job.recording_handler as rec_handler\n",
"import u19_pipeline.automatic_job.recording_process_handler as rec_process_handler\n",
- "\n",
- "from u19_pipeline import recording, recording_process\n"
+ "import u19_pipeline.ephys_pipeline as ephys_pipeline\n",
+ "import u19_pipeline.imaging_pipeline as imaging_pipeline\n",
+ "from u19_pipeline import recording, recording_process\n",
+ "from u19_pipeline.ephys_pipeline import ephys_element\n",
+ "from u19_pipeline.imaging_pipeline import imaging_element, scan_element"
]
},
{
@@ -114,7 +112,7 @@
"metadata": {},
"outputs": [],
"source": [
- "function_status_process = getattr(rec_process_handler.RecProcessHandler, 'transfer_request')"
+ "function_status_process = rec_process_handler.RecProcessHandler.transfer_request"
]
},
{
@@ -201,10 +199,10 @@
"metadata": {},
"outputs": [],
"source": [
- "lab = dj.create_virtual_module('lab', 'u19_lab')\n",
+ "lab = dj.create_virtual_module(\"lab\", \"u19_lab\")\n",
"user = dict()\n",
- "user['user_id'] = 'nb1689'\n",
- "user['slack_webhook'] = '(webhook url from slack API)222'\n",
+ "user[\"user_id\"] = \"nb1689\"\n",
+ "user[\"slack_webhook\"] = \"(webhook url from slack API)222\"\n",
"lab.User.update1(user)"
]
},
@@ -426,84 +424,128 @@
}
],
"source": [
- "from datetime import datetime\n",
"import copy\n",
- "import pathlib\n",
- "\n",
- "from u19_pipeline.automatic_job import recording_handler\n",
+ "from datetime import datetime\n",
"\n",
- "import u19_pipeline.utils.dj_shortcuts as dj_short\n",
- "import u19_pipeline.utils.slack_utils as slack_utils\n",
"import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n",
- "import u19_pipeline.automatic_job.slurm_creator as slurmlib\n",
- "import u19_pipeline.automatic_job.parameter_file_creator as paramfilelib\n",
"import u19_pipeline.automatic_job.params_config as config\n",
- "import u19_pipeline.automatic_job.ephys_element_populate as ep\n",
- "import u19_pipeline.automatic_job.imaging_element_populate as ip\n",
- "\n",
"\n",
"update_value_dict = copy.deepcopy(config.default_update_value_dict)\n",
"\n",
"# Get jobs with error status or finished\n",
- "df_inactive_jobs = rec_process_handler.RecProcessHandler.get_active_process_jobs(active=False)\n",
+ "df_inactive_jobs = rec_process_handler.RecProcessHandler.get_active_process_jobs(\n",
+ " active=False\n",
+ ")\n",
"\n",
- "jobs_ids = df_inactive_jobs['job_id'].to_frame().to_dict('records')\n",
+ "jobs_ids = df_inactive_jobs[\"job_id\"].to_frame().to_dict(\"records\")\n",
"\n",
- "#Get latest time they changed status\n",
+ "# Get latest time they changed status\n",
"now = datetime.now()\n",
- "timestamp_jobs = pd.DataFrame((recording_process.Processing & jobs_ids).aggr(recording_process.LogStatus, max_date=\"max(status_timestamp)\").fetch(as_dict=True))\n",
- "df_inactive_jobs = df_inactive_jobs.merge(timestamp_jobs, on='job_id')\n",
+ "timestamp_jobs = pd.DataFrame(\n",
+ " (recording_process.Processing & jobs_ids)\n",
+ " .aggr(recording_process.LogStatus, max_date=\"max(status_timestamp)\")\n",
+ " .fetch(as_dict=True)\n",
+ ")\n",
+ "df_inactive_jobs = df_inactive_jobs.merge(timestamp_jobs, on=\"job_id\")\n",
"\n",
"# Get jobs that were processed in cluster\n",
- "df_inactive_jobs['local_or_cluster'] = df_inactive_jobs['program_selection_params'].map(lambda x: x['local_or_cluster'])\n",
- "df_inactive_jobs['process_cluster'] = df_inactive_jobs['program_selection_params'].map(lambda x: x['process_cluster'])\n",
- "\n",
- "#Update all local jobs\n",
- "df_not_cluster = df_inactive_jobs.loc[df_inactive_jobs['local_or_cluster'] == 'local'].reset_index(drop=True)\n",
+ "df_inactive_jobs[\"local_or_cluster\"] = df_inactive_jobs[\"program_selection_params\"].map(\n",
+ " lambda x: x[\"local_or_cluster\"]\n",
+ ")\n",
+ "df_inactive_jobs[\"process_cluster\"] = df_inactive_jobs[\"program_selection_params\"].map(\n",
+ " lambda x: x[\"process_cluster\"]\n",
+ ")\n",
+ "\n",
+ "# Update all local jobs\n",
+ "df_not_cluster = df_inactive_jobs.loc[\n",
+ " df_inactive_jobs[\"local_or_cluster\"] == \"local\"\n",
+ "].reset_index(drop=True)\n",
"if df_not_cluster.shape[0] > 0:\n",
" # Update local jobs post-processed jobs\n",
- " df_not_cluster_post_processed = df_not_cluster.loc[df_not_cluster['status_processing_id'] == config.JOB_STATUS_PROCESSED, :]\n",
+ " df_not_cluster_post_processed = df_not_cluster.loc[\n",
+ " df_not_cluster[\"status_processing_id\"] == config.JOB_STATUS_PROCESSED, :\n",
+ " ]\n",
" df_not_cluster_post_processed = df_not_cluster_post_processed.reset_index(drop=True)\n",
" for i in range(df_not_cluster_post_processed.shape[0]):\n",
- " rec_process_handler.RecProcessHandler.update_status_pipeline(df_not_cluster_post_processed.loc[i, 'query_key'], config.JOB_STATUS_POST_PROCESSED)\n",
- " rec_process_handler.RecProcessHandler.update_job_id_log(df_not_cluster_post_processed.loc[i, 'job_id'], config.JOB_STATUS_PROCESSED, config.JOB_STATUS_POST_PROCESSED, update_value_dict['error_info'])\n",
+ " rec_process_handler.RecProcessHandler.update_status_pipeline(\n",
+ " df_not_cluster_post_processed.loc[i, \"query_key\"],\n",
+ " config.JOB_STATUS_POST_PROCESSED,\n",
+ " )\n",
+ " rec_process_handler.RecProcessHandler.update_job_id_log(\n",
+ " df_not_cluster_post_processed.loc[i, \"job_id\"],\n",
+ " config.JOB_STATUS_PROCESSED,\n",
+ " config.JOB_STATUS_POST_PROCESSED,\n",
+ " update_value_dict[\"error_info\"],\n",
+ " )\n",
"\n",
" # Update also local jobs post-error jobs\n",
- " df_not_cluster_post_error = df_not_cluster.loc[df_not_cluster['status_processing_id'] == config.JOB_STATUS_ERROR_ID, :]\n",
+ " df_not_cluster_post_error = df_not_cluster.loc[\n",
+ " df_not_cluster[\"status_processing_id\"] == config.JOB_STATUS_ERROR_ID, :\n",
+ " ]\n",
" df_not_cluster_post_error = df_not_cluster_post_error.reset_index(drop=True)\n",
" for i in range(df_not_cluster_post_error.shape[0]):\n",
- " rec_process_handler.RecProcessHandler.update_status_pipeline(df_not_cluster_post_error.loc[i, 'query_key'], config.JOB_STATUS_ERROR_DELETED)\n",
- " rec_process_handler.RecProcessHandler.update_job_id_log(df_not_cluster_post_error.loc[i, 'job_id'], config.JOB_STATUS_ERROR_ID, config.JOB_STATUS_ERROR_DELETED, update_value_dict['error_info'])\n",
- "\n",
- "#If no inactive job, after locals the end\n",
- "df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs['local_or_cluster'] == 'cluster'].reset_index(drop=True)\n",
+ " rec_process_handler.RecProcessHandler.update_status_pipeline(\n",
+ " df_not_cluster_post_error.loc[i, \"query_key\"],\n",
+ " config.JOB_STATUS_ERROR_DELETED,\n",
+ " )\n",
+ " rec_process_handler.RecProcessHandler.update_job_id_log(\n",
+ " df_not_cluster_post_error.loc[i, \"job_id\"],\n",
+ " config.JOB_STATUS_ERROR_ID,\n",
+ " config.JOB_STATUS_ERROR_DELETED,\n",
+ " update_value_dict[\"error_info\"],\n",
+ " )\n",
+ "\n",
+ "# If no inactive job, after locals the end\n",
+ "df_inactive_jobs = df_inactive_jobs.loc[\n",
+ " df_inactive_jobs[\"local_or_cluster\"] == \"cluster\"\n",
+ "].reset_index(drop=True)\n",
"\n",
"df_inactive_jobsz = df_inactive_jobs.copy()\n",
"\n",
"print(df_inactive_jobs)\n",
"\n",
"# Get jobs to delete (finished or with error >= 7 days)\n",
- "df_inactive_jobs['days_from_last_update'] = (now - df_inactive_jobs['max_date']).dt.days\n",
- "df_inactive_jobs['ready_delete'] = 1\n",
- "df_inactive_jobs.loc[(df_inactive_jobs['status_processing_id'] == config.JOB_STATUS_ERROR_ID) & (df_inactive_jobs['days_from_last_update'] < 7), 'ready_delete'] = 0\n",
- "df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs['ready_delete'] == 1,:]\n",
+ "df_inactive_jobs[\"days_from_last_update\"] = (now - df_inactive_jobs[\"max_date\"]).dt.days\n",
+ "df_inactive_jobs[\"ready_delete\"] = 1\n",
+ "df_inactive_jobs.loc[\n",
+ " (df_inactive_jobs[\"status_processing_id\"] == config.JOB_STATUS_ERROR_ID)\n",
+ " & (df_inactive_jobs[\"days_from_last_update\"] < 7),\n",
+ " \"ready_delete\",\n",
+ "] = 0\n",
+ "df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs[\"ready_delete\"] == 1, :]\n",
"\n",
"# Check if raw directories are present in cluster\n",
- "df_inactive_jobs['raw_dir'] = df_inactive_jobs.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_pre_path'],\\\n",
- " x['process_cluster'], x['recording_modality'], type_dir='raw'), axis=1)\n",
- "df_inactive_jobs['raw_dir'] = df_inactive_jobs['raw_dir'].astype(str)\n",
- "df_inactive_jobs['raw_dir_deleted'] = 0\n",
- "df_inactive_jobs.loc[df_inactive_jobs['raw_dir'] == \"0\", 'raw_dir_deleted'] = 1\n",
+ "df_inactive_jobs[\"raw_dir\"] = df_inactive_jobs.apply(\n",
+ " lambda x: ft.check_directory_exists_cluster(\n",
+ " x[\"recording_process_pre_path\"],\n",
+ " x[\"process_cluster\"],\n",
+ " x[\"recording_modality\"],\n",
+ " type_dir=\"raw\",\n",
+ " ),\n",
+ " axis=1,\n",
+ ")\n",
+ "df_inactive_jobs[\"raw_dir\"] = df_inactive_jobs[\"raw_dir\"].astype(str)\n",
+ "df_inactive_jobs[\"raw_dir_deleted\"] = 0\n",
+ "df_inactive_jobs.loc[df_inactive_jobs[\"raw_dir\"] == \"0\", \"raw_dir_deleted\"] = 1\n",
"\n",
"print(df_inactive_jobs)\n",
"\n",
"# Check if processed directories are present in cluster\n",
- "df_inactive_jobs['processed_dir'] = df_inactive_jobs.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_post_path'],\\\n",
- " x['process_cluster'], x['recording_modality'], type_dir='processed'), axis=1)\n",
- "df_inactive_jobs['processed_dir'] = df_inactive_jobs['processed_dir'].astype(str)\n",
- "df_inactive_jobs['processed_dir_deleted'] = 0\n",
- "df_inactive_jobs.loc[df_inactive_jobs['processed_dir'] == \"0\", 'processed_dir_deleted'] = 1\n",
- "df_inactive_jobs = df_inactive_jobs.reset_index(drop=True)\n"
+ "df_inactive_jobs[\"processed_dir\"] = df_inactive_jobs.apply(\n",
+ " lambda x: ft.check_directory_exists_cluster(\n",
+ " x[\"recording_process_post_path\"],\n",
+ " x[\"process_cluster\"],\n",
+ " x[\"recording_modality\"],\n",
+ " type_dir=\"processed\",\n",
+ " ),\n",
+ " axis=1,\n",
+ ")\n",
+ "df_inactive_jobs[\"processed_dir\"] = df_inactive_jobs[\"processed_dir\"].astype(str)\n",
+ "df_inactive_jobs[\"processed_dir_deleted\"] = 0\n",
+ "df_inactive_jobs.loc[\n",
+ " df_inactive_jobs[\"processed_dir\"] == \"0\", \"processed_dir_deleted\"\n",
+ "] = 1\n",
+ "df_inactive_jobs = df_inactive_jobs.reset_index(drop=True)"
]
},
{
@@ -612,11 +654,25 @@
}
],
"source": [
- "df_inactive_jobsz['lala'] = df_inactive_jobsz.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_pre_path'],\\\n",
- " x['process_cluster'], x['recording_modality'], type_dir='raw'), axis=1)\n",
- "\n",
- "sasa = df_inactive_jobsz.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_post_path'],\\\n",
- " x['process_cluster'], x['recording_modality'], type_dir='processed'), axis=1)\n",
+ "df_inactive_jobsz[\"lala\"] = df_inactive_jobsz.apply(\n",
+ " lambda x: ft.check_directory_exists_cluster(\n",
+ " x[\"recording_process_pre_path\"],\n",
+ " x[\"process_cluster\"],\n",
+ " x[\"recording_modality\"],\n",
+ " type_dir=\"raw\",\n",
+ " ),\n",
+ " axis=1,\n",
+ ")\n",
+ "\n",
+ "sasa = df_inactive_jobsz.apply(\n",
+ " lambda x: ft.check_directory_exists_cluster(\n",
+ " x[\"recording_process_post_path\"],\n",
+ " x[\"process_cluster\"],\n",
+ " x[\"recording_modality\"],\n",
+ " type_dir=\"processed\",\n",
+ " ),\n",
+ " axis=1,\n",
+ ")\n",
"sasa"
]
},
@@ -648,7 +704,12 @@
}
],
"source": [
- "su = ft.check_directory_exists_cluster(df_inactive_jobsz.loc[0,'recording_process_post_path'], 'tiger', 'electrophysiology', type_dir='processed')\n",
+ "su = ft.check_directory_exists_cluster(\n",
+ " df_inactive_jobsz.loc[0, \"recording_process_post_path\"],\n",
+ " \"tiger\",\n",
+ " \"electrophysiology\",\n",
+ " type_dir=\"processed\",\n",
+ ")\n",
"su"
]
},
@@ -705,37 +766,66 @@
"from u19_pipeline import utility\n",
"\n",
"rec_process_keys = dict()\n",
- "rec_process_keys['sdsd'] = 207\n",
+ "rec_process_keys[\"sdsd\"] = 207\n",
"\n",
"\n",
- "params_df = pd.DataFrame((imaging_pipeline.imaging_element.ProcessingParamSet.proj('params', 'processing_method') * \\\n",
- "recording_process.Processing.ImagingParams.proj('paramset_idx') & rec_process_keys).fetch(as_dict=True))\n",
- "params_df = params_df.drop('paramset_idx', axis=1)\n",
+ "params_df = pd.DataFrame(\n",
+ " (\n",
+ " imaging_pipeline.imaging_element.ProcessingParamSet.proj(\n",
+ " \"params\", \"processing_method\"\n",
+ " )\n",
+ " * recording_process.Processing.ImagingParams.proj(\"paramset_idx\")\n",
+ " & rec_process_keys\n",
+ " ).fetch(as_dict=True)\n",
+ ")\n",
+ "params_df = params_df.drop(\"paramset_idx\", axis=1)\n",
"\n",
- "#Insert processing_method in params itself (for BrainCogsImagingSorters)\n",
- "params_df['params'] = params_df.apply(lambda x: {**x['params'], **{'processing_method':x['processing_method']}},axis=1)\n",
+ "# Insert processing_method in params itself (for BrainCogsImagingSorters)\n",
+ "params_df[\"params\"] = params_df.apply(\n",
+ " lambda x: {**x[\"params\"], **{\"processing_method\": x[\"processing_method\"]}}, axis=1\n",
+ ")\n",
"\n",
"# Get preprocess param sets\n",
- "preparams_df = pd.DataFrame((imaging_pipeline.imaging_element.PreProcessParamSteps * \\\n",
- "utility.smart_dj_join(imaging_pipeline.imaging_element.PreProcessParamSteps.Step, imaging_pipeline.imaging_element.PreProcessParamSet.proj('preprocess_method', 'params')) *\n",
- "recording_process.Processing.ImagingParams.proj('preprocess_param_steps_id') & rec_process_keys).fetch(as_dict=True))\n",
+ "preparams_df = pd.DataFrame(\n",
+ " (\n",
+ " imaging_pipeline.imaging_element.PreProcessParamSteps\n",
+ " * utility.smart_dj_join(\n",
+ " imaging_pipeline.imaging_element.PreProcessParamSteps.Step,\n",
+ " imaging_pipeline.imaging_element.PreProcessParamSet.proj(\n",
+ " \"preprocess_method\", \"params\"\n",
+ " ),\n",
+ " )\n",
+ " * recording_process.Processing.ImagingParams.proj(\"preprocess_param_steps_id\")\n",
+ " & rec_process_keys\n",
+ " ).fetch(as_dict=True)\n",
+ ")\n",
"\n",
"if preparams_df.shape[0] > 0:\n",
- "\n",
- " # Join precluster params for the same recording_process\n",
- " preparams_df['preparams'] = preparams_df.apply(lambda x : {x['preprocess_method']: x['params']}, axis=1)\n",
- " preparams_df = preparams_df.sort_values(by=['job_id', 'step_number'])\n",
- " preparams_df = preparams_df[['job_id', 'preparams']].groupby(\"job_id\").agg(lambda x: list(x))\n",
- " preparams_df = preparams_df.reset_index()\n",
+ " # Join precluster params for the same recording_process\n",
+ " preparams_df[\"preparams\"] = preparams_df.apply(\n",
+ " lambda x: {x[\"preprocess_method\"]: x[\"params\"]}, axis=1\n",
+ " )\n",
+ " preparams_df = preparams_df.sort_values(by=[\"job_id\", \"step_number\"])\n",
+ " preparams_df = (\n",
+ " preparams_df[[\"job_id\", \"preparams\"]].groupby(\"job_id\").agg(lambda x: list(x))\n",
+ " )\n",
+ " preparams_df = preparams_df.reset_index()\n",
"\n",
"else:\n",
- " preparams_df = pd.DataFrame((imaging_pipeline.imaging_element.PreProcessParamSteps * \\\n",
- " recording_process.Processing.ImagingParams.proj('preprocess_param_steps_id') & rec_process_keys).fetch(as_dict=True))\n",
- " preparams_df['preparams'] = None\n",
+ " preparams_df = pd.DataFrame(\n",
+ " (\n",
+ " imaging_pipeline.imaging_element.PreProcessParamSteps\n",
+ " * recording_process.Processing.ImagingParams.proj(\n",
+ " \"preprocess_param_steps_id\"\n",
+ " )\n",
+ " & rec_process_keys\n",
+ " ).fetch(as_dict=True)\n",
+ " )\n",
+ " preparams_df[\"preparams\"] = None\n",
"\n",
"params_df = params_df.merge(preparams_df)\n",
"\n",
- "print('params_df 3')\n",
+ "print(\"params_df 3\")\n",
"print(params_df)"
]
},
@@ -868,36 +958,37 @@
}
],
"source": [
- "cat_gt_params = {\n",
- " \"apfilter\": [\"biquad\",2,300,0],\n",
- " \"gfix\": [0.40,0.10,0.02],\n",
- " \"extras\": [\"prb_fld\", \"t_miss_ok\", \"ap\", \"gblcar\", \"out_prb_fld\"],\n",
- " \"lazy\": True\n",
- " }\n",
- " \n",
+ "cat_gt_params = {\n",
+ " \"apfilter\": [\"biquad\", 2, 300, 0],\n",
+ " \"gfix\": [0.40, 0.10, 0.02],\n",
+ " \"extras\": [\"prb_fld\", \"t_miss_ok\", \"ap\", \"gblcar\", \"out_prb_fld\"],\n",
+ " \"lazy\": True,\n",
+ "}\n",
+ "\n",
"ephys_element.PreClusterParamSet.insert_new_params(\n",
- " precluster_method='catgt',\n",
+ " precluster_method=\"catgt\",\n",
" paramset_idx=2,\n",
" params=cat_gt_params,\n",
- " paramset_desc='Spike sorting using Kilosort w lazy')\n",
+ " paramset_desc=\"Spike sorting using Kilosort w lazy\",\n",
+ ")\n",
+ "\n",
"\n",
+ "cat_gt_params = {\n",
+ " \"apfilter\": [\"biquad\", 2, 301, 0],\n",
+ " \"gfix\": [0.40, 0.10, 0.02],\n",
+ " \"extras\": [\"prb_fld\", \"t_miss_ok\", \"ap\", \"gblcar\", \"out_prb_fld\"],\n",
+ " \"lazy\": True,\n",
+ "}\n",
"\n",
- "cat_gt_params = {\n",
- " \"apfilter\": [\"biquad\",2,301,0],\n",
- " \"gfix\": [0.40,0.10,0.02],\n",
- " \"extras\": [\"prb_fld\", \"t_miss_ok\", \"ap\", \"gblcar\", \"out_prb_fld\"],\n",
- " \"lazy\": True\n",
- " }\n",
- " \n",
"ephys_element.PreClusterParamSet.insert_new_params(\n",
- " precluster_method='catgt2',\n",
+ " precluster_method=\"catgt2\",\n",
" paramset_idx=3,\n",
" params=cat_gt_params,\n",
- " paramset_desc='Spike sorting using Kilosort w lazy')\n",
+ " paramset_desc=\"Spike sorting using Kilosort w lazy\",\n",
+ ")\n",
"\n",
"\n",
- "ephys_element.PreClusterParamSet()\n",
- " \n"
+ "ephys_element.PreClusterParamSet()"
]
},
{
@@ -1008,45 +1099,29 @@
}
],
"source": [
- "paramlist = {'precluster_param_steps_id':0}\n",
+ "paramlist = {\"precluster_param_steps_id\": 0}\n",
"ephys_element.PreClusterParamSteps.insert1(paramlist, skip_duplicates=True)\n",
"\n",
- "paramorder = {\n",
- "'precluster_param_steps_id': 0,\n",
- "'step_number': 1,\n",
- "'paramset_idx': 0\n",
- "}\n",
+ "paramorder = {\"precluster_param_steps_id\": 0, \"step_number\": 1, \"paramset_idx\": 0}\n",
"ephys_element.PreClusterParamSteps.Step.insert1(paramorder, skip_duplicates=True)\n",
"\n",
"\n",
- "paramlist = {'precluster_param_steps_id':1}\n",
+ "paramlist = {\"precluster_param_steps_id\": 1}\n",
"ephys_element.PreClusterParamSteps.insert1(paramlist, skip_duplicates=True)\n",
"\n",
- "paramorder = {\n",
- "'precluster_param_steps_id': 1,\n",
- "'step_number': 1,\n",
- "'paramset_idx': 1\n",
- "}\n",
+ "paramorder = {\"precluster_param_steps_id\": 1, \"step_number\": 1, \"paramset_idx\": 1}\n",
"ephys_element.PreClusterParamSteps.Step.insert1(paramorder, skip_duplicates=True)\n",
"\n",
"\n",
- "paramlist = {'precluster_param_steps_id':2}\n",
- "ephys_element.PreClusterParamSteps.insert1(paramlist,skip_duplicates=True)\n",
+ "paramlist = {\"precluster_param_steps_id\": 2}\n",
+ "ephys_element.PreClusterParamSteps.insert1(paramlist, skip_duplicates=True)\n",
"\n",
- "paramorder = {\n",
- "'precluster_param_steps_id': 2,\n",
- "'step_number': 1,\n",
- "'paramset_idx': 0\n",
- "}\n",
+ "paramorder = {\"precluster_param_steps_id\": 2, \"step_number\": 1, \"paramset_idx\": 0}\n",
"ephys_element.PreClusterParamSteps.Step.insert1(paramorder, skip_duplicates=True)\n",
- "paramorder = {\n",
- "'precluster_param_steps_id': 2,\n",
- "'step_number': 2,\n",
- "'paramset_idx': 1\n",
- "}\n",
+ "paramorder = {\"precluster_param_steps_id\": 2, \"step_number\": 2, \"paramset_idx\": 1}\n",
"ephys_element.PreClusterParamSteps.Step.insert1(paramorder, skip_duplicates=True)\n",
"\n",
- "ephys_element.PreClusterParamSteps.Step()\n"
+ "ephys_element.PreClusterParamSteps.Step()"
]
},
{
@@ -1167,75 +1242,77 @@
],
"source": [
"params = {\n",
- " \"fs\": 30000,\n",
- " \"fshigh\": 150,\n",
- " \"minfr_goodchannels\": 0.1,\n",
- " \"Th\": [10, 4],\n",
- " \"lam\": 10,\n",
- " \"AUCsplit\": 0.9,\n",
- " \"minFR\": 0.02,\n",
- " \"momentum\": [20, 400],\n",
- " \"sigmaMask\": 30,\n",
- " \"ThPre\": 8,\n",
- " \"CAR\": 1, \n",
- " \"spkTh\": -6,\n",
- " \"reorder\": 1,\n",
- " \"nskip\": 25,\n",
- " \"GPU\": 1,\n",
- " \"Nfilt\": 1024,\n",
- " \"nfilt_factor\": 4,\n",
- " \"ntbuff\": 64,\n",
- " \"NT\": 32832,\n",
- " \"whiteningRange\": 32,\n",
- " \"nSkipCov\": 25,\n",
- " \"scaleproc\": 200,\n",
- " \"nPCs\": 3,\n",
- " \"useRAM\": 0,\n",
- " \"trange\": [0, 1000000000],\n",
- "\"NchanTOT\": 384\n",
+ " \"fs\": 30000,\n",
+ " \"fshigh\": 150,\n",
+ " \"minfr_goodchannels\": 0.1,\n",
+ " \"Th\": [10, 4],\n",
+ " \"lam\": 10,\n",
+ " \"AUCsplit\": 0.9,\n",
+ " \"minFR\": 0.02,\n",
+ " \"momentum\": [20, 400],\n",
+ " \"sigmaMask\": 30,\n",
+ " \"ThPre\": 8,\n",
+ " \"CAR\": 1,\n",
+ " \"spkTh\": -6,\n",
+ " \"reorder\": 1,\n",
+ " \"nskip\": 25,\n",
+ " \"GPU\": 1,\n",
+ " \"Nfilt\": 1024,\n",
+ " \"nfilt_factor\": 4,\n",
+ " \"ntbuff\": 64,\n",
+ " \"NT\": 32832,\n",
+ " \"whiteningRange\": 32,\n",
+ " \"nSkipCov\": 25,\n",
+ " \"scaleproc\": 200,\n",
+ " \"nPCs\": 3,\n",
+ " \"useRAM\": 0,\n",
+ " \"trange\": [0, 1000000000],\n",
+ " \"NchanTOT\": 384,\n",
"}\n",
"\n",
"ephys_element.ClusteringParamSet.insert_new_params(\n",
- " processing_method='kilosort2',\n",
+ " processing_method=\"kilosort2\",\n",
" paramset_idx=0,\n",
" params=params,\n",
- " paramset_desc='Spike sorting using Kilosort2')\n",
+ " paramset_desc=\"Spike sorting using Kilosort2\",\n",
+ ")\n",
"\n",
"\n",
"params = {\n",
- " \"fs\": 30000,\n",
- " \"fshigh\": 150,\n",
- " \"minfr_goodchannels\": 0.1,\n",
- " \"Th\": [10, 4],\n",
- " \"lam\": 10,\n",
- " \"AUCsplit\": 0.9,\n",
- " \"minFR\": 0.02,\n",
- " \"momentum\": [20, 400],\n",
- " \"sigmaMask\": 30,\n",
- " \"ThPre\": 8,\n",
- " \"CAR\": 1, \n",
- " \"spkTh\": -6,\n",
- " \"reorder\": 1,\n",
- " \"nskip\": 25,\n",
- " \"GPU\": 2,\n",
- " \"Nfilt\": 1024,\n",
- " \"nfilt_factor\": 4,\n",
- " \"ntbuff\": 64,\n",
- " \"NT\": 32832,\n",
- " \"whiteningRange\": 32,\n",
- " \"nSkipCov\": 25,\n",
- " \"scaleproc\": 200,\n",
- " \"nPCs\": 3,\n",
- " \"useRAM\": 0,\n",
- " \"trange\": [0, 1000000000],\n",
- "\"NchanTOT\": 384\n",
+ " \"fs\": 30000,\n",
+ " \"fshigh\": 150,\n",
+ " \"minfr_goodchannels\": 0.1,\n",
+ " \"Th\": [10, 4],\n",
+ " \"lam\": 10,\n",
+ " \"AUCsplit\": 0.9,\n",
+ " \"minFR\": 0.02,\n",
+ " \"momentum\": [20, 400],\n",
+ " \"sigmaMask\": 30,\n",
+ " \"ThPre\": 8,\n",
+ " \"CAR\": 1,\n",
+ " \"spkTh\": -6,\n",
+ " \"reorder\": 1,\n",
+ " \"nskip\": 25,\n",
+ " \"GPU\": 2,\n",
+ " \"Nfilt\": 1024,\n",
+ " \"nfilt_factor\": 4,\n",
+ " \"ntbuff\": 64,\n",
+ " \"NT\": 32832,\n",
+ " \"whiteningRange\": 32,\n",
+ " \"nSkipCov\": 25,\n",
+ " \"scaleproc\": 200,\n",
+ " \"nPCs\": 3,\n",
+ " \"useRAM\": 0,\n",
+ " \"trange\": [0, 1000000000],\n",
+ " \"NchanTOT\": 384,\n",
"}\n",
"\n",
"ephys_element.ClusteringParamSet.insert_new_params(\n",
- " processing_method='kilosort2',\n",
+ " processing_method=\"kilosort2\",\n",
" paramset_idx=1,\n",
" params=params,\n",
- " paramset_desc='Spike sorting using Kilosort2')\n"
+ " paramset_desc=\"Spike sorting using Kilosort2\",\n",
+ ")"
]
},
{
@@ -1363,64 +1440,68 @@
}
],
"source": [
- "\n",
"ops = dict()\n",
"# sample rate\n",
- "ops['fs'] = 30000; \n",
- "ops['fshigh'] = 300; \n",
+ "ops[\"fs\"] = 30000\n",
+ "ops[\"fshigh\"] = 300\n",
"\n",
"# minimum firing rate on a \"good\" channel (0 to skip)\n",
- "ops['minfr_goodchannels'] = 0; \n",
+ "ops[\"minfr_goodchannels\"] = 0\n",
"\n",
"# threshold on projections (like in Kilosort1, can be different for last pass like [10 4])\n",
- "ops['Th'] = [10, 4]; \n",
+ "ops[\"Th\"] = [10, 4]\n",
"\n",
- "# how important is the amplitude penalty (like in Kilosort1, 0 means not used, 10 is average, 50 is a lot) \n",
- "ops['lam'] = 10; \n",
+ "# how important is the amplitude penalty (like in Kilosort1, 0 means not used, 10 is average, 50 is a lot)\n",
+ "ops[\"lam\"] = 10\n",
"\n",
"# splitting a cluster at the end requires at least this much isolation for each sub-cluster (max = 1)\n",
- "ops['AUCsplit'] = 0.9; \n",
+ "ops[\"AUCsplit\"] = 0.9\n",
"\n",
"# minimum spike rate (Hz), if a cluster falls below this for too long it gets removed\n",
- "ops['minFR'] = 1/50; \n",
+ "ops[\"minFR\"] = 1 / 50\n",
"\n",
- "# number of samples to average over (annealed from first to second value) \n",
- "ops['momentum'] = [20, 400]; \n",
+ "# number of samples to average over (annealed from first to second value)\n",
+ "ops[\"momentum\"] = [20, 400]\n",
"\n",
"# spatial constant in um for computing residual variance of spike\n",
- "ops['sigmaMask'] = 30; \n",
+ "ops[\"sigmaMask\"] = 30\n",
"\n",
"# threshold crossings for pre-clustering (in PCA projection space)\n",
- "ops['ThPre'] = 8; \n",
- "ops['reorder'] = 1; # whether to reorder batches for drift correction. \n",
- "ops['nskip'] = 25; # how many batches to skip for determining spike PCs\n",
+ "ops[\"ThPre\"] = 8\n",
+ "ops[\"reorder\"] = 1 # whether to reorder batches for drift correction.\n",
+ "ops[\"nskip\"] = 25 # how many batches to skip for determining spike PCs\n",
"\n",
"# danger, changing these settings can lead to fatal errors\n",
"# options for determining PCs\n",
- "ops['spkTh'] = -6; # spike threshold in standard deviations (-6)\n",
+ "ops[\"spkTh\"] = -6 # spike threshold in standard deviations (-6)\n",
"\n",
- "ops['GPU'] = 1; # has to be 1, no CPU version yet, sorry\n",
+ "ops[\"GPU\"] = 1 # has to be 1, no CPU version yet, sorry\n",
"# ops.Nfilt = 1024; # max number of clusters\n",
- "ops['nfilt_factor'] = 4; # max number of clusters per good channel (even temporary ones)\n",
- "ops['ntbuff'] = 64; # samples of symmetrical buffer for whitening and spike detection\n",
- "ops['NT'] = 64*1024+ ops['ntbuff']; # must be multiple of 32 + ntbuff. This is the batch size (try decreasing if out of memory). \n",
- "ops['whiteningRange'] = 32; # number of channels to use for whitening each channel\n",
- "ops['nSkipCov'] = 25; # compute whitening matrix from every N-th batch\n",
- "ops['scaleproc'] = 200; # int16 scaling of whitened data\n",
- "ops['nPCs'] = 3; # how many PCs to project the spikes into\n",
- "ops['useRAM'] = 0; # not yet available\n",
- "\n",
- "\n",
- "ops['trange'] = [0, 100000000000000]\n",
- "ops['NchanTOT'] = 384\n",
- "ops['sig'] = 20; #spatial smoothness constant for registration\n",
- "ops['nblocks'] = 5; # blocks for registration. 0 turns it off, 1 does rigid registration. Replaces \"datashift\" option. \n",
+ "ops[\"nfilt_factor\"] = 4 # max number of clusters per good channel (even temporary ones)\n",
+ "ops[\"ntbuff\"] = 64 # samples of symmetrical buffer for whitening and spike detection\n",
+ "ops[\"NT\"] = (\n",
+ " 64 * 1024 + ops[\"ntbuff\"]\n",
+ ") # must be multiple of 32 + ntbuff. This is the batch size (try decreasing if out of memory).\n",
+ "ops[\"whiteningRange\"] = 32 # number of channels to use for whitening each channel\n",
+ "ops[\"nSkipCov\"] = 25 # compute whitening matrix from every N-th batch\n",
+ "ops[\"scaleproc\"] = 200 # int16 scaling of whitened data\n",
+ "ops[\"nPCs\"] = 3 # how many PCs to project the spikes into\n",
+ "ops[\"useRAM\"] = 0 # not yet available\n",
+ "\n",
+ "\n",
+ "ops[\"trange\"] = [0, 100000000000000]\n",
+ "ops[\"NchanTOT\"] = 384\n",
+ "ops[\"sig\"] = 20 # spatial smoothness constant for registration\n",
+ "ops[\"nblocks\"] = (\n",
+ " 5 # blocks for registration. 0 turns it off, 1 does rigid registration. Replaces \"datashift\" option.\n",
+ ")\n",
"\n",
"ephys_element.ClusteringParamSet.insert_new_params(\n",
- " processing_method='kilosort',\n",
+ " processing_method=\"kilosort\",\n",
" paramset_idx=2,\n",
" params=ops,\n",
- " paramset_desc='Spike sorting using Kilosort2')\n",
+ " paramset_desc=\"Spike sorting using Kilosort2\",\n",
+ ")\n",
"\n",
"\n",
"ephys_element.ClusteringParamSet()"
@@ -1440,75 +1521,76 @@
"outputs": [],
"source": [
"ops = {\n",
- " \"look_one_level_down\": 0.0,\n",
- " \"fast_disk\": [],\n",
- " \"delete_bin\": False,\n",
- " \"mesoscan\": False,\n",
- " \"h5py\": [],\n",
- " \"h5py_key\": \"data\",\n",
- " \"save_path0\": [],\n",
- " \"subfolders\": [],\n",
- " \"nplanes\": 1,\n",
- " \"nchannels\": 1,\n",
- " \"functional_chan\": 1,\n",
- " \"tau\": 1.0,\n",
- " \"fs\": 10.0,\n",
- " \"force_sktiff\": False,\n",
- " \"preclassify\": 0.0,\n",
- " \"save_mat\": False,\n",
- " \"combined\": True,\n",
- " \"aspect\": 1.0,\n",
- " \"do_bidiphase\": False,\n",
- " \"bidiphase\": 0.0,\n",
- " \"do_registration\": True,\n",
- " \"keep_movie_raw\": False,\n",
- " \"nimg_init\": 300,\n",
- " \"batch_size\": 500,\n",
- " \"maxregshift\": 0.1,\n",
- " \"align_by_chan\": 1,\n",
- " \"reg_tif\": False,\n",
- " \"reg_tif_chan2\": False,\n",
- " \"subpixel\": 10,\n",
- " \"smooth_sigma\": 1.15,\n",
- " \"th_badframes\": 1.0,\n",
- " \"pad_fft\": False,\n",
- " \"nonrigid\": False,\n",
- " \"block_size\": [128, 128],\n",
- " \"snr_thresh\": 1.2,\n",
- " \"maxregshiftNR\": 5.0,\n",
- " \"1Preg\": False,\n",
- " \"spatial_hp\": 50.0,\n",
- " \"pre_smooth\": 2.0,\n",
- " \"spatial_taper\": 50.0,\n",
- " \"roidetect\": True,\n",
- " \"sparse_mode\": False,\n",
- " \"diameter\": 12,\n",
- " \"spatial_scale\": 0,\n",
- " \"connected\": True,\n",
- " \"nbinned\": 5000,\n",
- " \"max_iterations\": 20,\n",
- " \"threshold_scaling\": 1.0,\n",
- " \"max_overlap\": 0.75,\n",
- " \"high_pass\": 100.0,\n",
- " \"inner_neuropil_radius\": 2,\n",
- " \"min_neuropil_pixels\": 350,\n",
- " \"allow_overlap\": False,\n",
- " \"chan2_thres\": 0.65,\n",
- " \"baseline\": \"maximin\",\n",
- " \"win_baseline\": 60.0,\n",
- " \"sig_baseline\": 10.0,\n",
- " \"prctile_baseline\": 8.0,\n",
- " \"neucoeff\": 0.7,\n",
- " \"xrange\": [0, 0],\n",
- " \"yrange\": [0, 0]\n",
+ " \"look_one_level_down\": 0.0,\n",
+ " \"fast_disk\": [],\n",
+ " \"delete_bin\": False,\n",
+ " \"mesoscan\": False,\n",
+ " \"h5py\": [],\n",
+ " \"h5py_key\": \"data\",\n",
+ " \"save_path0\": [],\n",
+ " \"subfolders\": [],\n",
+ " \"nplanes\": 1,\n",
+ " \"nchannels\": 1,\n",
+ " \"functional_chan\": 1,\n",
+ " \"tau\": 1.0,\n",
+ " \"fs\": 10.0,\n",
+ " \"force_sktiff\": False,\n",
+ " \"preclassify\": 0.0,\n",
+ " \"save_mat\": False,\n",
+ " \"combined\": True,\n",
+ " \"aspect\": 1.0,\n",
+ " \"do_bidiphase\": False,\n",
+ " \"bidiphase\": 0.0,\n",
+ " \"do_registration\": True,\n",
+ " \"keep_movie_raw\": False,\n",
+ " \"nimg_init\": 300,\n",
+ " \"batch_size\": 500,\n",
+ " \"maxregshift\": 0.1,\n",
+ " \"align_by_chan\": 1,\n",
+ " \"reg_tif\": False,\n",
+ " \"reg_tif_chan2\": False,\n",
+ " \"subpixel\": 10,\n",
+ " \"smooth_sigma\": 1.15,\n",
+ " \"th_badframes\": 1.0,\n",
+ " \"pad_fft\": False,\n",
+ " \"nonrigid\": False,\n",
+ " \"block_size\": [128, 128],\n",
+ " \"snr_thresh\": 1.2,\n",
+ " \"maxregshiftNR\": 5.0,\n",
+ " \"1Preg\": False,\n",
+ " \"spatial_hp\": 50.0,\n",
+ " \"pre_smooth\": 2.0,\n",
+ " \"spatial_taper\": 50.0,\n",
+ " \"roidetect\": True,\n",
+ " \"sparse_mode\": False,\n",
+ " \"diameter\": 12,\n",
+ " \"spatial_scale\": 0,\n",
+ " \"connected\": True,\n",
+ " \"nbinned\": 5000,\n",
+ " \"max_iterations\": 20,\n",
+ " \"threshold_scaling\": 1.0,\n",
+ " \"max_overlap\": 0.75,\n",
+ " \"high_pass\": 100.0,\n",
+ " \"inner_neuropil_radius\": 2,\n",
+ " \"min_neuropil_pixels\": 350,\n",
+ " \"allow_overlap\": False,\n",
+ " \"chan2_thres\": 0.65,\n",
+ " \"baseline\": \"maximin\",\n",
+ " \"win_baseline\": 60.0,\n",
+ " \"sig_baseline\": 10.0,\n",
+ " \"prctile_baseline\": 8.0,\n",
+ " \"neucoeff\": 0.7,\n",
+ " \"xrange\": [0, 0],\n",
+ " \"yrange\": [0, 0],\n",
"}\n",
"\n",
"\n",
"imaging_element.ProcessingParamSet.insert_new_params(\n",
- " processing_method='suite2p',\n",
+ " processing_method=\"suite2p\",\n",
" paramset_idx=0,\n",
" params=ops,\n",
- " paramset_desc='Default params Suite2p')"
+ " paramset_desc=\"Default params Suite2p\",\n",
+ ")"
]
},
{
@@ -1525,11 +1607,13 @@
"outputs": [],
"source": [
"PreProcessParamStepsRecord = dict()\n",
- "PreProcessParamStepsRecord['preprocess_param_steps_id'] = 0 \n",
- "PreProcessParamStepsRecord['preprocess_param_steps_name'] = 'No preprocessing'\n",
- "PreProcessParamStepsRecord['preprocess_param_steps_desc'] = 'No preprocessing steps'\n",
+ "PreProcessParamStepsRecord[\"preprocess_param_steps_id\"] = 0\n",
+ "PreProcessParamStepsRecord[\"preprocess_param_steps_name\"] = \"No preprocessing\"\n",
+ "PreProcessParamStepsRecord[\"preprocess_param_steps_desc\"] = \"No preprocessing steps\"\n",
"\n",
- "imaging_element.PreProcessParamSteps.insert1(PreProcessParamStepsRecord, skip_duplicates=True)"
+ "imaging_element.PreProcessParamSteps.insert1(\n",
+ " PreProcessParamStepsRecord, skip_duplicates=True\n",
+ ")"
]
},
{
@@ -1679,70 +1763,69 @@
}
],
"source": [
- "\n",
"# For recording 39, all probes equal, list = 0\n",
"default_params_key = dict()\n",
- "default_params_key['recording_id'] = 39\n",
- "default_params_key['fragment_number'] = 0\n",
- "default_params_key['preprocess_param_steps_id'] = 0\n",
- "default_params_key['paramset_idx'] = 0\n",
+ "default_params_key[\"recording_id\"] = 39\n",
+ "default_params_key[\"fragment_number\"] = 0\n",
+ "default_params_key[\"preprocess_param_steps_id\"] = 0\n",
+ "default_params_key[\"paramset_idx\"] = 0\n",
"\n",
"\n",
"recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n",
"\n",
"# For recording 41, all probes equal, list = 2 (list 2 has two different preparams catgt & catgt2)\n",
"default_params_key = dict()\n",
- "default_params_key['recording_id'] = 41\n",
- "default_params_key['fragment_number'] = 0\n",
- "default_params_key['preprocess_param_steps_id'] = 2\n",
- "default_params_key['paramset_idx'] = 0\n",
+ "default_params_key[\"recording_id\"] = 41\n",
+ "default_params_key[\"fragment_number\"] = 0\n",
+ "default_params_key[\"preprocess_param_steps_id\"] = 2\n",
+ "default_params_key[\"paramset_idx\"] = 0\n",
"\n",
"\n",
"recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n",
"\n",
"# For recording 40, all probes different params\n",
"default_params_key = dict()\n",
- "default_params_key['recording_id'] = 40\n",
- "default_params_key['fragment_number'] = 0\n",
- "default_params_key['preprocess_param_steps_id'] = 0\n",
- "default_params_key['paramset_idx'] = 0\n",
- "default_params_key['default_same_preparams_all'] = 0\n",
- "default_params_key['default_same_params_all'] = 0\n",
+ "default_params_key[\"recording_id\"] = 40\n",
+ "default_params_key[\"fragment_number\"] = 0\n",
+ "default_params_key[\"preprocess_param_steps_id\"] = 0\n",
+ "default_params_key[\"paramset_idx\"] = 0\n",
+ "default_params_key[\"default_same_preparams_all\"] = 0\n",
+ "default_params_key[\"default_same_params_all\"] = 0\n",
"\n",
"recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n",
"\n",
"default_params_key = dict()\n",
- "default_params_key['recording_id'] = 40\n",
- "default_params_key['fragment_number'] = 1\n",
- "default_params_key['preprocess_param_steps_id'] = 0\n",
- "default_params_key['paramset_idx'] = 1\n",
- "default_params_key['default_same_preparams_all'] = 0\n",
- "default_params_key['default_same_params_all'] = 0\n",
+ "default_params_key[\"recording_id\"] = 40\n",
+ "default_params_key[\"fragment_number\"] = 1\n",
+ "default_params_key[\"preprocess_param_steps_id\"] = 0\n",
+ "default_params_key[\"paramset_idx\"] = 1\n",
+ "default_params_key[\"default_same_preparams_all\"] = 0\n",
+ "default_params_key[\"default_same_params_all\"] = 0\n",
"\n",
"recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n",
"\n",
"default_params_key = dict()\n",
- "default_params_key['recording_id'] = 40\n",
- "default_params_key['fragment_number'] = 2\n",
- "default_params_key['preprocess_param_steps_id'] = 1\n",
- "default_params_key['paramset_idx'] = 0\n",
- "default_params_key['default_same_preparams_all'] = 0\n",
- "default_params_key['default_same_params_all'] = 0\n",
+ "default_params_key[\"recording_id\"] = 40\n",
+ "default_params_key[\"fragment_number\"] = 2\n",
+ "default_params_key[\"preprocess_param_steps_id\"] = 1\n",
+ "default_params_key[\"paramset_idx\"] = 0\n",
+ "default_params_key[\"default_same_preparams_all\"] = 0\n",
+ "default_params_key[\"default_same_params_all\"] = 0\n",
"\n",
"recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n",
"\n",
"default_params_key = dict()\n",
- "default_params_key['recording_id'] = 40\n",
- "default_params_key['fragment_number'] = 3\n",
- "default_params_key['preprocess_param_steps_id'] = 1\n",
- "default_params_key['paramset_idx'] = 1\n",
- "default_params_key['default_same_preparams_all'] = 0\n",
- "default_params_key['default_same_params_all'] = 0\n",
+ "default_params_key[\"recording_id\"] = 40\n",
+ "default_params_key[\"fragment_number\"] = 3\n",
+ "default_params_key[\"preprocess_param_steps_id\"] = 1\n",
+ "default_params_key[\"paramset_idx\"] = 1\n",
+ "default_params_key[\"default_same_preparams_all\"] = 0\n",
+ "default_params_key[\"default_same_params_all\"] = 0\n",
"\n",
"\n",
"recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n",
"\n",
- "recording.DefaultParams()\n"
+ "recording.DefaultParams()"
]
},
{
@@ -1758,14 +1841,16 @@
"metadata": {},
"outputs": [],
"source": [
- "ephys_pipeline = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')\n",
- "ephys_element = dj.create_virtual_module('ephys_element', 'u19_pipeline_ephys_element')\n",
- "probe_element = dj.create_virtual_module('probe_element', 'u19_pipeline_probe_element')\n",
+ "ephys_pipeline = dj.create_virtual_module(\"ephys_pipeline\", \"u19_ephys_pipeline\")\n",
+ "ephys_element = dj.create_virtual_module(\"ephys_element\", \"u19_pipeline_ephys_element\")\n",
+ "probe_element = dj.create_virtual_module(\"probe_element\", \"u19_pipeline_probe_element\")\n",
"\n",
"\n",
- "imaging_pipeline = dj.create_virtual_module('imaging_pipeline', 'u19_imaging_pipeline')\n",
- "scan_element = dj.create_virtual_module('scan_element', 'u19_pipeline_scan_element')\n",
- "imaging_element = dj.create_virtual_module('imaging_element', 'u19_pipeline_imaging_element')\n"
+ "imaging_pipeline = dj.create_virtual_module(\"imaging_pipeline\", \"u19_imaging_pipeline\")\n",
+ "scan_element = dj.create_virtual_module(\"scan_element\", \"u19_pipeline_scan_element\")\n",
+ "imaging_element = dj.create_virtual_module(\n",
+ " \"imaging_element\", \"u19_pipeline_imaging_element\"\n",
+ ")"
]
},
{
@@ -1786,15 +1871,17 @@
}
],
"source": [
- "from u19_pipeline.ephys_pipeline import get_spikeglx_meta_filepath, get_session_directory\n",
+ "from u19_pipeline.ephys_pipeline import (\n",
+ " get_session_directory,\n",
+ " get_spikeglx_meta_filepath,\n",
+ ")\n",
"\n",
"so = recording.Recording.fetch(\"KEY\", as_dict=True)\n",
"\n",
"\n",
- "\n",
"print(so[-1])\n",
"print(get_session_directory(so[-1]))\n",
- "get_spikeglx_meta_filepath(so[-1])\n"
+ "get_spikeglx_meta_filepath(so[-1])"
]
},
{
@@ -1815,7 +1902,14 @@
}
],
"source": [
- "dj.ERD(recording) + dj.ERD(recording_process) + dj.ERD(ephys_pipeline) + dj.ERD(ephys_element.EphysRecording) + dj.ERD(ephys_element.PreClusterTask) + dj.ERD(ephys_element.PreClusterParamList)\n"
+ "(\n",
+ " dj.ERD(recording)\n",
+ " + dj.ERD(recording_process)\n",
+ " + dj.ERD(ephys_pipeline)\n",
+ " + dj.ERD(ephys_element.EphysRecording)\n",
+ " + dj.ERD(ephys_element.PreClusterTask)\n",
+ " + dj.ERD(ephys_element.PreClusterParamList)\n",
+ ")"
]
},
{
@@ -1836,7 +1930,14 @@
}
],
"source": [
- "dj.ERD(recording) + dj.ERD(recording_process) + dj.ERD(imaging_pipeline) + dj.ERD(scan_element.Scan) + dj.ERD(imaging_element.ProcessingTask) + dj.ERD(imaging_element.ProcessingParamSet)\n"
+ "(\n",
+ " dj.ERD(recording)\n",
+ " + dj.ERD(recording_process)\n",
+ " + dj.ERD(imaging_pipeline)\n",
+ " + dj.ERD(scan_element.Scan)\n",
+ " + dj.ERD(imaging_element.ProcessingTask)\n",
+ " + dj.ERD(imaging_element.ProcessingParamSet)\n",
+ ")"
]
},
{
@@ -2064,8 +2165,8 @@
}
],
"source": [
- "#ephys_pipeline_db = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')\n",
- "#ephys_pipeline_db.EphysPipelineSession.delete()"
+ "# ephys_pipeline_db = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')\n",
+ "# ephys_pipeline_db.EphysPipelineSession.delete()"
]
}
],
diff --git a/notebooks/ad_hoc_psychometrics_plot.ipynb b/notebooks/ad_hoc_psychometrics_plot.ipynb
index 8964144b..0b171c5a 100644
--- a/notebooks/ad_hoc_psychometrics_plot.ipynb
+++ b/notebooks/ad_hoc_psychometrics_plot.ipynb
@@ -14,14 +14,13 @@
"outputs": [],
"source": [
"import datajoint as dj\n",
- "import pandas as pd\n",
+ "import matplotlib.patches as mpatches\n",
"import numpy as np\n",
+ "import pandas as pd\n",
"import pylab as plt\n",
- "import matplotlib.patches as mpatches\n",
- "\n",
"from matplotlib import cm\n",
- "from u19_pipeline import utility\n",
- "from inspect import getmembers, isfunction\n"
+ "\n",
+ "from u19_pipeline import utility"
]
},
{
@@ -74,11 +73,11 @@
"metadata": {},
"outputs": [],
"source": [
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n",
- "optogenetics = dj.create_virtual_module('optogenetics', 'u19_optogenetics')\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "subject = dj.create_virtual_module('subject', 'u19_subject')\n",
- "puffs = dj.create_virtual_module('puffs', 'u19_puffs')"
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
+ "optogenetics = dj.create_virtual_module(\"optogenetics\", \"u19_optogenetics\")\n",
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n",
+ "puffs = dj.create_virtual_module(\"puffs\", \"u19_puffs\")"
]
},
{
@@ -94,36 +93,40 @@
"metadata": {},
"outputs": [],
"source": [
- "#List of conditions to filter sessions, trials, etc for analysis\n",
- "key_selection = ['subject_fullname like \"efonseca_Vgat%\"', 'level = 14', 'is_bad_session = 0']\n",
- "\n",
- "#List of tables to \"prefilter data\" (generally session or block tables)\n",
+ "# List of conditions to filter sessions, trials, etc for analysis\n",
+ "key_selection = [\n",
+ " 'subject_fullname like \"efonseca_Vgat%\"',\n",
+ " \"level = 14\",\n",
+ " \"is_bad_session = 0\",\n",
+ "]\n",
+ "\n",
+ "# List of tables to \"prefilter data\" (generally session or block tables)\n",
"table_prefilters = [behavior.TowersBlock, acquisition.Session]\n",
"\n",
- "#List of conditions to compare (data will be filtered for all conditions and saved plotted)\n",
- "condition_key = ['stim_on = 0', 'stim_on = 1']\n",
- "#Labels for legend (if empty will be the smae as condition_key)\n",
- "condition_labels = ['Laser Off', 'Laser On']\n",
- "#List of Tables with all condition info, (normally trial tables)\n",
+ "# List of conditions to compare (data will be filtered for all conditions and saved plotted)\n",
+ "condition_key = [\"stim_on = 0\", \"stim_on = 1\"]\n",
+ "# Labels for legend (if empty will be the smae as condition_key)\n",
+ "condition_labels = [\"Laser Off\", \"Laser On\"]\n",
+ "# List of Tables with all condition info, (normally trial tables)\n",
"data_tables = [behavior.TowersBlock().Trial(), optogenetics.OptogeneticSession().Trial]\n",
"\n",
- "#Extra field to split plots by (e.g. one plot per subject)\n",
- "split_plots_by = 'subject_fullname'\n",
+ "# Extra field to split plots by (e.g. one plot per subject)\n",
+ "split_plots_by = \"subject_fullname\"\n",
"\n",
- "#Bins for trials .... (for now, manual selection)\n",
- "#Recommendation. np.arange(min_evidence, max_evidence+1, step=(max_evidence - min_evidence)/10)\n",
+ "# Bins for trials .... (for now, manual selection)\n",
+ "# Recommendation. np.arange(min_evidence, max_evidence+1, step=(max_evidence - min_evidence)/10)\n",
"deltaBins = np.arange(-15, 16, step=3)\n",
"\n",
- "#How to color plots \n",
- "#('Pick colors from a list, from a perceptually uniform colormap, a qualitative colormap or custom colormap')\n",
+ "# How to color plots\n",
+ "# ('Pick colors from a list, from a perceptually uniform colormap, a qualitative colormap or custom colormap')\n",
"# [list, uniform, qualitative, custom]\n",
- "color_type = 'list'\n",
+ "color_type = \"list\"\n",
"\n",
- "#List of colors (if applicable, select from list)\n",
- "color_list = ['k', 'b']\n",
+ "# List of colors (if applicable, select from list)\n",
+ "color_list = [\"k\", \"b\"]\n",
"\n",
- "#custom colormap name, (If applicable, select from custom colormap)\n",
- "'''\n",
+ "# custom colormap name, (If applicable, select from custom colormap)\n",
+ "\"\"\"\n",
"cmaps['Perceptually Uniform Sequential'] = [\n",
" 'viridis', 'plasma', 'inferno', 'magma', 'cividis']\n",
"cmaps['Sequential'] = [\n",
@@ -146,14 +149,14 @@
" 'gnuplot', 'gnuplot2', 'CMRmap', 'cubehelix', 'brg',\n",
" 'gist_rainbow', 'rainbow', 'jet', 'turbo', 'nipy_spectral',\n",
" 'gist_ncar']\n",
- "'''\n",
- "custom_colormap = 'autumn'\n",
+ "\"\"\"\n",
+ "custom_colormap = \"autumn\"\n",
"\n",
- "fig_size = (13,14)\n",
+ "fig_size = (13, 14)\n",
"\n",
- "big_font_size = (20,16)\n",
- "medium_font_size = (16,13)\n",
- "small_font_size = (12,10)\n"
+ "big_font_size = (20, 16)\n",
+ "medium_font_size = (16, 13)\n",
+ "small_font_size = (12, 10)"
]
},
{
@@ -169,55 +172,68 @@
"metadata": {},
"outputs": [],
"source": [
- "#Here are parameter examples for some use cases\n",
- "example = 'None'\n",
- "\n",
- "#Compare stimulation trials vs non stimulation trials\n",
- "if example == 'Optogenetic_on_vs_off':\n",
- " key_selection = ['subject_fullname like \"efonseca_Vgat%\"', 'level = 14', 'is_bad_session = 0']\n",
+ "# Here are parameter examples for some use cases\n",
+ "example = \"None\"\n",
+ "\n",
+ "# Compare stimulation trials vs non stimulation trials\n",
+ "if example == \"Optogenetic_on_vs_off\":\n",
+ " key_selection = [\n",
+ " 'subject_fullname like \"efonseca_Vgat%\"',\n",
+ " \"level = 14\",\n",
+ " \"is_bad_session = 0\",\n",
+ " ]\n",
" table_prefilters = [behavior.TowersBlock, acquisition.Session]\n",
- " condition_key = ['stim_on = 0', 'stim_on = 1']\n",
- " condition_labels = ['Laser Off', 'Laser On']\n",
- " data_tables = [behavior.TowersBlock().Trial(), optogenetics.OptogeneticSession().Trial]\n",
- " split_plots_by = 'subject_fullname'\n",
- " \n",
+ " condition_key = [\"stim_on = 0\", \"stim_on = 1\"]\n",
+ " condition_labels = [\"Laser Off\", \"Laser On\"]\n",
+ " data_tables = [\n",
+ " behavior.TowersBlock().Trial(),\n",
+ " optogenetics.OptogeneticSession().Trial,\n",
+ " ]\n",
+ " split_plots_by = \"subject_fullname\"\n",
+ "\n",
" deltaBins = np.arange(-15, 16, step=3)\n",
- " color_type = 'list'\n",
- " color_list = ['k', 'b']\n",
- "elif example == 'Compare_subjects':\n",
- " key_selection = ['level = 11', 'block_performance > 0.65', 'subject_fullname like \"%ms%\"']\n",
+ " color_type = \"list\"\n",
+ " color_list = [\"k\", \"b\"]\n",
+ "elif example == \"Compare_subjects\":\n",
+ " key_selection = [\n",
+ " \"level = 11\",\n",
+ " \"block_performance > 0.65\",\n",
+ " 'subject_fullname like \"%ms%\"',\n",
+ " ]\n",
" table_prefilters = [behavior.TowersBlock]\n",
- " #Here, the \"conditions to compare\" are the subjects themselves\n",
- " condition_key = (subject.Subject & 'user_id like \"%ms%\"').fetch('subject_fullname', as_dict=True)\n",
+ " # Here, the \"conditions to compare\" are the subjects themselves\n",
+ " condition_key = (subject.Subject & 'user_id like \"%ms%\"').fetch(\n",
+ " \"subject_fullname\", as_dict=True\n",
+ " )\n",
" condition_labels = []\n",
" data_tables = [behavior.TowersBlock().Trial()]\n",
- " split_plots_by = ''\n",
- " \n",
+ " split_plots_by = \"\"\n",
+ "\n",
" deltaBins = np.arange(-12, 13, step=3)\n",
- " color_type = 'qualitative'\n",
- " color_list = ['k', 'b']\n",
- "elif example == 'MainLevel_vs_Training_level':\n",
- " key_selection = ['level = 11', 'subject_fullname like \"emdiamanti_%\"']\n",
+ " color_type = \"qualitative\"\n",
+ " color_list = [\"k\", \"b\"]\n",
+ "elif example == \"MainLevel_vs_Training_level\":\n",
+ " key_selection = [\"level = 11\", 'subject_fullname like \"emdiamanti_%\"']\n",
" table_prefilters = [acquisition.Session]\n",
- " condition_key = ['level = main_level', 'level<>main_level']\n",
- " condition_labels = ['Main Level', 'Training Level']\n",
+ " condition_key = [\"level = main_level\", \"level<>main_level\"]\n",
+ " condition_labels = [\"Main Level\", \"Training Level\"]\n",
" data_tables = [behavior.TowersBlock, behavior.TowersBlock().Trial()]\n",
- " split_plots_by = 'subject_fullname'\n",
- " \n",
+ " split_plots_by = \"subject_fullname\"\n",
+ "\n",
" deltaBins = np.arange(-15, 16, step=3)\n",
- " color_type = 'list'\n",
- " color_list = ['r', 'k']\n",
- "elif example == 'PuffsTrials':\n",
- " key_selection = ['level=9', 'rig = 1']\n",
+ " color_type = \"list\"\n",
+ " color_list = [\"r\", \"k\"]\n",
+ "elif example == \"PuffsTrials\":\n",
+ " key_selection = [\"level=9\", \"rig = 1\"]\n",
" table_prefilters = [puffs.PuffsSession(), puffs.PuffsSession().TrialOld()]\n",
- " condition_key = ['trial_duration < 11.5', 'trial_duration > 11.5']\n",
+ " condition_key = [\"trial_duration < 11.5\", \"trial_duration > 11.5\"]\n",
" condition_labels = []\n",
" data_tables = [puffs.PuffsSession().TrialOld()]\n",
- " split_plots_by = ''\n",
- " \n",
+ " split_plots_by = \"\"\n",
+ "\n",
" deltaBins = np.arange(-12, 13, step=3)\n",
- " color_type = 'list'\n",
- " color_list = ['r', 'k']\n"
+ " color_type = \"list\"\n",
+ " color_list = [\"r\", \"k\"]"
]
},
{
@@ -233,62 +249,69 @@
"metadata": {},
"outputs": [],
"source": [
- "#Transform dictionary of condition keys to a list of strings\n",
+ "# Transform dictionary of condition keys to a list of strings\n",
"if type(condition_key[0]) is dict:\n",
- " if isinstance(list(condition_key[0].values())[0],str):\n",
+ " if isinstance(list(condition_key[0].values())[0], str):\n",
" ap_str = '\"'\n",
" else:\n",
- " ap_str = ''\n",
- " aux_condition_var = [list(x.keys())[0]+ ' = ' + ap_str + list(x.values())[0] + ap_str for x in condition_key]\n",
+ " ap_str = \"\"\n",
+ " aux_condition_var = [\n",
+ " list(x.keys())[0] + \" = \" + ap_str + list(x.values())[0] + ap_str\n",
+ " for x in condition_key\n",
+ " ]\n",
" condition_key = aux_condition_var\n",
- " \n",
+ "\n",
"if len(condition_labels) == 0:\n",
" condition_labels = condition_key\n",
"\n",
"\n",
- "if color_type == 'list':\n",
- " #If we have enough colors on the list:\n",
+ "if color_type == \"list\":\n",
+ " # If we have enough colors on the list:\n",
" if len(condition_labels) <= len(color_list):\n",
- " colors = color_list[0:len(condition_labels)]\n",
- " #If not, we will check if there are less than 10 things to plot \n",
+ " colors = color_list[0 : len(condition_labels)]\n",
+ " # If not, we will check if there are less than 10 things to plot\n",
" elif len(condition_labels) <= 10:\n",
- " color_type = 'qualitative'\n",
- " print('Not enough colors on list, swithcing to qualitative colormap')\n",
- " #If more than 10 things to plot, let's go with uniform colormap\n",
+ " color_type = \"qualitative\"\n",
+ " print(\"Not enough colors on list, swithcing to qualitative colormap\")\n",
+ " # If more than 10 things to plot, let's go with uniform colormap\n",
" else:\n",
- " color_type = 'uniform'\n",
- " print('Not enough colors on list, swithcing to uniform colormap')\n",
- " \n",
- "if color_type == 'qualitative':\n",
+ " color_type = \"uniform\"\n",
+ " print(\"Not enough colors on list, swithcing to uniform colormap\")\n",
+ "\n",
+ "if color_type == \"qualitative\":\n",
" if len(condition_labels) <= 10:\n",
- " colors = cm.get_cmap('tab10', 10)\n",
- " colors = colors(np.linspace(0,len(condition_labels)/10,len(condition_labels)))\n",
+ " colors = cm.get_cmap(\"tab10\", 10)\n",
+ " colors = colors(\n",
+ " np.linspace(0, len(condition_labels) / 10, len(condition_labels))\n",
+ " )\n",
" else:\n",
- " color_type = 'uniform'\n",
- " print('Not enough colors on qualitative colormap, swithcing to uniform colormap')\n",
- " \n",
- "if color_type == 'uniform':\n",
- " colors = cm.get_cmap('viridis', len(condition_labels))\n",
+ " color_type = \"uniform\"\n",
+ " print(\n",
+ " \"Not enough colors on qualitative colormap, swithcing to uniform colormap\"\n",
+ " )\n",
+ "\n",
+ "if color_type == \"uniform\":\n",
+ " colors = cm.get_cmap(\"viridis\", len(condition_labels))\n",
"\n",
- "if color_type == 'custom':\n",
+ "if color_type == \"custom\":\n",
" colors = cm.get_cmap(custom_colormap, len(condition_labels))\n",
"\n",
- "if color_type != 'list' and color_type != 'qualitative':\n",
- " colors = colors(np.linspace(0,1,len(condition_labels)))\n",
- " \n",
+ "if color_type != \"list\" and color_type != \"qualitative\":\n",
+ " colors = colors(np.linspace(0, 1, len(condition_labels)))\n",
+ "\n",
"\n",
- "#Assign a color to each condition\n",
- "zip_iterator = zip(condition_key, colors)\n",
+ "# Assign a color to each condition\n",
+ "zip_iterator = zip(condition_key, colors)\n",
"color_dict_keys = dict(zip_iterator)\n",
"\n",
- "#Assign a color to each label\n",
+ "# Assign a color to each label\n",
"zip_iterator = zip(condition_labels, colors)\n",
"color_dict = dict(zip_iterator)\n",
"\n",
- "#Create a color patch for plot legend\n",
+ "# Create a color patch for plot legend\n",
"patches_legend = []\n",
"for key in color_dict:\n",
- " patches_legend.append(mpatches.Patch(color=color_dict[key], label=key))\n"
+ " patches_legend.append(mpatches.Patch(color=color_dict[key], label=key))"
]
},
{
@@ -308,8 +331,8 @@
"table_filter = table_prefilters[0]\n",
"\n",
"if len(table_prefilters) > 1:\n",
- " for i in range(1,len(table_prefilters)):\n",
- " table_filter = utility.smart_dj_join(table_filter,table_prefilters[i])\n",
+ " for i in range(1, len(table_prefilters)):\n",
+ " table_filter = utility.smart_dj_join(table_filter, table_prefilters[i])\n",
"\n",
"# Filter by all conditions selected\n",
"for filter_key in key_selection:\n",
@@ -318,9 +341,8 @@
"# Join all data tables\n",
"data_table = data_tables[0]\n",
"if len(data_tables) > 1:\n",
- " for i in range(1,len(data_tables)):\n",
- " data_table = utility.smart_dj_join(data_table,data_tables[i]) \n",
- "\n"
+ " for i in range(1, len(data_tables)):\n",
+ " data_table = utility.smart_dj_join(data_table, data_tables[i])"
]
},
{
@@ -336,7 +358,7 @@
"metadata": {},
"outputs": [],
"source": [
- "plots_records = ['']\n",
+ "plots_records = [\"\"]\n",
"if len(split_plots_by) > 0:\n",
" plots_records = table_filter.fetch(split_plots_by)\n",
" plots_records = list(set(plots_records))\n",
@@ -367,87 +389,108 @@
}
],
"source": [
- "df_summary = pd.DataFrame(columns = [split_plots_by, 'Type', '%Correct', 'Total trials', 'Sigmoid fit'])\n",
- "\n",
- "rows, cols = utility.get_cols_rows_plot(num_figs, fig_size) \n",
- "fig, axs = plt.subplots(rows, cols, figsize=fig_size, sharex=True, sharey=True, squeeze=False)\n",
+ "df_summary = pd.DataFrame(\n",
+ " columns=[split_plots_by, \"Type\", \"%Correct\", \"Total trials\", \"Sigmoid fit\"]\n",
+ ")\n",
+ "\n",
+ "rows, cols = utility.get_cols_rows_plot(num_figs, fig_size)\n",
+ "fig, axs = plt.subplots(\n",
+ " rows, cols, figsize=fig_size, sharex=True, sharey=True, squeeze=False\n",
+ ")\n",
"renderer = fig.canvas.get_renderer()\n",
"\n",
- "#Decide font size for plots\n",
- "if rows*cols >= 9:\n",
+ "# Decide font size for plots\n",
+ "if rows * cols >= 9:\n",
" ac_fs = small_font_size\n",
- "elif rows*cols >= 5:\n",
+ "elif rows * cols >= 5:\n",
" ac_fs = medium_font_size\n",
"else:\n",
" ac_fs = big_font_size\n",
"\n",
"for split_idx, split in enumerate(plots_records):\n",
+ " print(split_idx + 1, \"/\", len(plots_records))\n",
+ "\n",
+ " ac_col = split_idx % cols\n",
+ " ac_row = int(split_idx / cols)\n",
"\n",
- " print(split_idx+1, '/', len(plots_records))\n",
- " \n",
- " ac_col = split_idx%cols\n",
- " ac_row = int(split_idx/cols)\n",
- " \n",
- " #Create dictionary for split key plot (if applicable)\n",
+ " # Create dictionary for split key plot (if applicable)\n",
" split_key = dict()\n",
" split_key[split_plots_by] = split\n",
"\n",
" for key_idx, key in enumerate(condition_key):\n",
- "\n",
- " #Fetch session info (if split, add split_key to query)\n",
+ " # Fetch session info (if split, add split_key to query)\n",
" if len(split_plots_by) > 0:\n",
" thisCondition = data_table & table_filter.proj() & key & split_key\n",
" else:\n",
" thisCondition = data_table & table_filter.proj() & key\n",
- " \n",
+ "\n",
" # Fetch all fields of table except external (for speed)\n",
" if key_idx == 0:\n",
- " fields_query = pd.DataFrame.from_dict(thisCondition.heading.attributes, orient='index')\n",
- " fields_query = fields_query.loc[fields_query['is_external'] == False, :]\n",
+ " fields_query = pd.DataFrame.from_dict(\n",
+ " thisCondition.heading.attributes, orient=\"index\"\n",
+ " )\n",
+ " fields_query = fields_query.loc[fields_query[\"is_external\"] == False, :]\n",
" fields_query = fields_query.index.to_list()\n",
"\n",
- " # Fetch session \n",
+ " # Fetch session\n",
" session_info = pd.DataFrame(thisCondition.fetch(*fields_query, as_dict=True))\n",
"\n",
" # Choice and trial type as integer\n",
" if session_info.shape[0] > 0:\n",
- "\n",
" session_info = utility.translate_choice_trials_cues(session_info)\n",
"\n",
" # Call Fit to sigmoid function\n",
- " fit_dict = utility.psychFit(deltaBins, session_info['cue_presence_right'].values, \\\n",
- " session_info['cue_presence_left'].values, session_info['choice_int'].values)\n",
+ " fit_dict = utility.psychFit(\n",
+ " deltaBins,\n",
+ " session_info[\"cue_presence_right\"].values,\n",
+ " session_info[\"cue_presence_left\"].values,\n",
+ " session_info[\"choice_int\"].values,\n",
+ " )\n",
"\n",
" dict_summary = dict()\n",
- " dict_summary[split_plots_by] = split\n",
- " dict_summary['Type'] = key\n",
- " dict_summary['%Correct'] = np.sum(session_info['choice'].values == session_info['trial_type'].values)/session_info.shape[0]\n",
- " dict_summary['Total trials'] = session_info.shape[0]\n",
- " dict_summary['Sigmoid fit'] = fit_dict\n",
- " \n",
- " df_summary = df_summary.append(dict_summary, ignore_index = True)\n",
- "\n",
- " #Select color from color dictionary\n",
+ " dict_summary[split_plots_by] = split\n",
+ " dict_summary[\"Type\"] = key\n",
+ " dict_summary[\"%Correct\"] = (\n",
+ " np.sum(\n",
+ " session_info[\"choice\"].values == session_info[\"trial_type\"].values\n",
+ " )\n",
+ " / session_info.shape[0]\n",
+ " )\n",
+ " dict_summary[\"Total trials\"] = session_info.shape[0]\n",
+ " dict_summary[\"Sigmoid fit\"] = fit_dict\n",
+ "\n",
+ " df_summary = df_summary.append(dict_summary, ignore_index=True)\n",
+ "\n",
+ " # Select color from color dictionary\n",
" color_plot = color_dict_keys[key]\n",
- " #Plot results\n",
- " axs[ac_row, ac_col].plot(fit_dict['delta_bins'], fit_dict['pright_data'], 'o', color = color_plot)\n",
- " if fit_dict['delta_fit'].shape[0] > 0:\n",
- " axs[ac_row, ac_col].plot(fit_dict['delta_fit'], fit_dict['pright_fit'], '--', color = color_plot)\n",
- " \n",
+ " # Plot results\n",
+ " axs[ac_row, ac_col].plot(\n",
+ " fit_dict[\"delta_bins\"], fit_dict[\"pright_data\"], \"o\", color=color_plot\n",
+ " )\n",
+ " if fit_dict[\"delta_fit\"].shape[0] > 0:\n",
+ " axs[ac_row, ac_col].plot(\n",
+ " fit_dict[\"delta_fit\"],\n",
+ " fit_dict[\"pright_fit\"],\n",
+ " \"--\",\n",
+ " color=color_plot,\n",
+ " )\n",
+ "\n",
" # Axes, legends and labels\n",
" if ac_col == 0:\n",
" axs[ac_row, ac_col].set_ylabel(\"% Turned left\", fontsize=ac_fs[1])\n",
- " \n",
+ "\n",
" axs[ac_row, ac_col].set_ylim([-1, 101])\n",
- " mean_correct_string = f\"{df_summary.loc[df_summary[split_plots_by] == split, '%Correct'].mean()*100:2.1f}\"\n",
+ " mean_correct_string = f\"{df_summary.loc[df_summary[split_plots_by] == split, '%Correct'].mean() * 100:2.1f}\"\n",
" if len(split_plots_by) > 7:\n",
- " title_label = str(split) + '-%Correct: ' + mean_correct_string\n",
+ " title_label = str(split) + \"-%Correct: \" + mean_correct_string\n",
" else:\n",
- " title_label = split_plots_by + \" : \" + str(split) + '%Correct: ' + mean_correct_string\n",
+ " title_label = (\n",
+ " split_plots_by + \" : \" + str(split) + \"%Correct: \" + mean_correct_string\n",
+ " )\n",
" axs[ac_row, ac_col].set_title(title_label, fontsize=ac_fs[0])\n",
- " \n",
+ "\n",
" fig.canvas.draw()\n",
- " plt.draw() \n",
+ " plt.draw()\n",
" axs[ac_row, ac_col].draw(renderer)\n",
"\n",
" xlabels = axs[ac_row, ac_col].get_xticklabels()\n",
@@ -459,10 +502,10 @@
" if len(ylabels) > 0:\n",
" axs[ac_row, ac_col].set_yticklabels(ylabels, fontsize=ac_fs[1])\n",
"\n",
- "for i in range(0,cols):\n",
- " axs[rows-1, i].set_xlabel(\"total evidence (L-R)\", fontsize=ac_fs[1])\n",
- " \n",
- "axs[rows-1, cols-1].legend(handles=patches_legend, fontsize=ac_fs[0])\n"
+ "for i in range(0, cols):\n",
+ " axs[rows - 1, i].set_xlabel(\"total evidence (L-R)\", fontsize=ac_fs[1])\n",
+ "\n",
+ "axs[rows - 1, cols - 1].legend(handles=patches_legend, fontsize=ac_fs[0])"
]
},
{
@@ -590,17 +633,19 @@
}
],
"source": [
- "mean_correct = df_summary['%Correct'].mean()\n",
- "total_trials = df_summary['Total trials'].sum()\n",
+ "mean_correct = df_summary[\"%Correct\"].mean()\n",
+ "total_trials = df_summary[\"Total trials\"].sum()\n",
"\n",
- "total_df = pd.DataFrame([['', 'Totals', mean_correct, total_trials, None]], columns=df_summary.columns)\n",
+ "total_df = pd.DataFrame(\n",
+ " [[\"\", \"Totals\", mean_correct, total_trials, None]], columns=df_summary.columns\n",
+ ")\n",
"\n",
"df_final = df_summary.append(total_df, ignore_index=True)\n",
"\n",
"if len(split_plots_by) > 0:\n",
- " df_final = df_final.set_index([split_plots_by, 'Type'])\n",
+ " df_final = df_final.set_index([split_plots_by, \"Type\"])\n",
"else:\n",
- " df_final = df_final.set_index(['Type'])\n",
+ " df_final = df_final.set_index([\"Type\"])\n",
"df_final"
]
},
diff --git a/notebooks/behavior_metrics_performance.ipynb b/notebooks/behavior_metrics_performance.ipynb
index 5fdba49d..2d9e11aa 100644
--- a/notebooks/behavior_metrics_performance.ipynb
+++ b/notebooks/behavior_metrics_performance.ipynb
@@ -23,17 +23,14 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"\n",
+ "\n",
"import datajoint as dj\n",
- "import pandas as pd\n",
"import numpy as np\n",
- "import pylab as plt\n",
- "import matplotlib.patches as mpatches\n",
- "\n",
- "from matplotlib import cm\n",
- "from u19_pipeline import utility\n",
- "from inspect import getmembers, isfunction\n"
+ "import pandas as pd\n",
+ "import pylab as plt"
]
},
{
@@ -542,14 +539,21 @@
}
],
"source": [
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('acquisition', 'u19_behavior')\n",
- "subject = dj.create_virtual_module('acquisition', 'u19_subject')\n",
- "recording = dj.create_virtual_module('recording', 'u19_recording')\n",
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"acquisition\", \"u19_behavior\")\n",
+ "subject = dj.create_virtual_module(\"acquisition\", \"u19_subject\")\n",
+ "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n",
"\n",
- "key = subject.Subject() & \"user_id = 'ms81'\";\n",
- "su = pd.DataFrame((recording.Recording.BehaviorSession * behavior.TowersBlock & key & \"level = 11\" & \"block_performance>0.6\").fetch(as_dict=True, order_by='block_performance DESC'))\n",
- "su\n"
+ "key = subject.Subject() & \"user_id = 'ms81'\"\n",
+ "su = pd.DataFrame(\n",
+ " (\n",
+ " recording.Recording.BehaviorSession * behavior.TowersBlock\n",
+ " & key\n",
+ " & \"level = 11\"\n",
+ " & \"block_performance>0.6\"\n",
+ " ).fetch(as_dict=True, order_by=\"block_performance DESC\")\n",
+ ")\n",
+ "su"
]
},
{
@@ -566,41 +570,42 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
"## Filter rigs\n",
"filter_rigs = True\n",
- "rigs = ['165I-Rig1-T',\n",
- " '165I-Rig2-T',\n",
- " '165I-Rig3-T',\n",
- " '165I-Rig4-T',\n",
- " '165A-Rig5-T',\n",
- " '165A-Rig6-T',\n",
- " '165A-Rig7-T',\n",
- " '165A-Rig8-T', \n",
- " '170b-Imaging0642',\n",
- " '185f-rig1',\n",
- " '188-Rig1',\n",
- " '188-Rig2',\n",
- " 'BezosMeso',\n",
- " 'C42-Bay2-Rig1-I']\n",
- "rigs_label = ', '.join(f'\"{w}\"' for w in rigs)\n",
- "rigs_query = 'session_location in (' + rigs_label + ')'\n",
+ "rigs = [\n",
+ " \"165I-Rig1-T\",\n",
+ " \"165I-Rig2-T\",\n",
+ " \"165I-Rig3-T\",\n",
+ " \"165I-Rig4-T\",\n",
+ " \"165A-Rig5-T\",\n",
+ " \"165A-Rig6-T\",\n",
+ " \"165A-Rig7-T\",\n",
+ " \"165A-Rig8-T\",\n",
+ " \"170b-Imaging0642\",\n",
+ " \"185f-rig1\",\n",
+ " \"188-Rig1\",\n",
+ " \"188-Rig2\",\n",
+ " \"BezosMeso\",\n",
+ " \"C42-Bay2-Rig1-I\",\n",
+ "]\n",
+ "rigs_label = \", \".join(f'\"{w}\"' for w in rigs)\n",
+ "rigs_query = \"session_location in (\" + rigs_label + \")\"\n",
"\n",
- "#Filter Dates\n",
+ "# Filter Dates\n",
"filter_dates = True\n",
- "dates = ['2016-01-01', '2022-03-01']\n",
- "date_label = ' and '.join(f'\"{w}\"' for w in dates)\n",
- "date_query = 'session_date between ' + date_label \n",
+ "dates = [\"2016-01-01\", \"2022-03-01\"]\n",
+ "date_label = \" and \".join(f'\"{w}\"' for w in dates)\n",
+ "date_query = \"session_date between \" + date_label\n",
"\n",
- "#Filter users\n",
+ "# Filter users\n",
"filter_user = True\n",
- "users = ['sbolkan', 'jounhong']\n",
+ "users = [\"sbolkan\", \"jounhong\"]\n",
"users = ['subject_fullname like \"' + x + '%\"' for x in users]\n",
- "user_query = ' or '.join(f'{w}' for w in users) \n",
+ "user_query = \" or \".join(f\"{w}\" for w in users)\n",
"\n",
"\n",
- "#Min sessions to be a \"real\" subject\n",
- "min_sessions = 15\n"
+ "# Min sessions to be a \"real\" subject\n",
+ "min_sessions = 15"
]
},
{
@@ -1010,18 +1015,18 @@
"query = []\n",
"if filter_rigs:\n",
" query.append(rigs_query)\n",
- " \n",
+ "\n",
"if filter_dates:\n",
" query.append(date_query)\n",
- " \n",
+ "\n",
"if filter_user:\n",
" query.append(user_query)\n",
- " \n",
+ "\n",
"# Filter by all conditions selected\n",
"session_table_filtered = acquisition.Session\n",
"for filter_key in query:\n",
" session_table_filtered = session_table_filtered & filter_key\n",
- " \n",
+ "\n",
"session_df = pd.DataFrame(session_table_filtered.fetch(as_dict=True))\n",
"session_df"
]
@@ -1163,11 +1168,12 @@
}
],
"source": [
- "\n",
- "protocol_df = session_df.groupby('session_protocol').agg({'subject_fullname': [('num_sessions', 'count'), ('num_subjects', 'nunique')]})\n",
+ "protocol_df = session_df.groupby(\"session_protocol\").agg(\n",
+ " {\"subject_fullname\": [(\"num_sessions\", \"count\"), (\"num_subjects\", \"nunique\")]}\n",
+ ")\n",
"protocol_df.columns = protocol_df.columns.droplevel()\n",
- "#protocol_df = protocol_df.reset_index()\n",
- "protocol_df = protocol_df.sort_values(by='num_sessions', ascending=False)\n",
+ "# protocol_df = protocol_df.reset_index()\n",
+ "protocol_df = protocol_df.sort_values(by=\"num_sessions\", ascending=False)\n",
"protocol_df"
]
},
@@ -1288,43 +1294,54 @@
}
],
"source": [
- "#Analyze only one protocol\n",
- "session_p_df = session_df.loc[session_df['session_protocol'] == protocol_df.index[0], :]\n",
+ "# Analyze only one protocol\n",
+ "session_p_df = session_df.loc[session_df[\"session_protocol\"] == protocol_df.index[0], :]\n",
"\n",
- "#Get max level\n",
- "max_level = session_p_df['level'].max()\n",
+ "# Get max level\n",
+ "max_level = session_p_df[\"level\"].max()\n",
"max_level = 10\n",
"\n",
- "#Sort by subject and date\n",
- "session_p_df = session_p_df.sort_values(by=['subject_fullname', 'session_date'])\n",
+ "# Sort by subject and date\n",
+ "session_p_df = session_p_df.sort_values(by=[\"subject_fullname\", \"session_date\"])\n",
"session_p_df = session_p_df.reset_index(drop=True)\n",
"\n",
"# Count sessions for each subject (sequential count and total count)\n",
- "sequential_sessions = session_p_df.groupby(['subject_fullname']).cumcount() + 1\n",
- "num_sessions_df = session_p_df.groupby('subject_fullname').agg({'session_date': [('total_sessions', 'count')]})\n",
+ "sequential_sessions = session_p_df.groupby([\"subject_fullname\"]).cumcount() + 1\n",
+ "num_sessions_df = session_p_df.groupby(\"subject_fullname\").agg(\n",
+ " {\"session_date\": [(\"total_sessions\", \"count\")]}\n",
+ ")\n",
"num_sessions_df.columns = num_sessions_df.columns.droplevel()\n",
"num_sessions_df = num_sessions_df.reset_index()\n",
"\n",
"# Merge session count to session df\n",
- "session_p_df['seq_sessions'] = sequential_sessions\n",
+ "session_p_df[\"seq_sessions\"] = sequential_sessions\n",
"session_p_df = session_p_df.merge(num_sessions_df)\n",
"\n",
"# Filter only max level sessions\n",
- "session_p_df_max = session_p_df.loc[session_p_df['level'] == max_level, :]\n",
+ "session_p_df_max = session_p_df.loc[session_p_df[\"level\"] == max_level, :]\n",
"\n",
"# Get only the first day that they got to max level and filter \"fake\" subjects\n",
- "first_session_max_level = session_p_df_max.drop_duplicates(subset = [\"subject_fullname\"], keep='first')\n",
- "first_session_max_level = first_session_max_level.loc[first_session_max_level['total_sessions'] >= min_sessions, :]\n",
+ "first_session_max_level = session_p_df_max.drop_duplicates(\n",
+ " subset=[\"subject_fullname\"], keep=\"first\"\n",
+ ")\n",
+ "first_session_max_level = first_session_max_level.loc[\n",
+ " first_session_max_level[\"total_sessions\"] >= min_sessions, :\n",
+ "]\n",
"first_session_max_level = first_session_max_level.reset_index(drop=True)\n",
"\n",
"\n",
- "#min_sessions to max level per rig\n",
- "sessions_max_level_rig = first_session_max_level.groupby('session_location').agg(\n",
- " {'seq_sessions': [('mean_sessions', 'mean'), ('std_sessions', 'std'), ('num_subjects', 'count')]})\n",
+ "# min_sessions to max level per rig\n",
+ "sessions_max_level_rig = first_session_max_level.groupby(\"session_location\").agg(\n",
+ " {\n",
+ " \"seq_sessions\": [\n",
+ " (\"mean_sessions\", \"mean\"),\n",
+ " (\"std_sessions\", \"std\"),\n",
+ " (\"num_subjects\", \"count\"),\n",
+ " ]\n",
+ " }\n",
+ ")\n",
"sessions_max_level_rig.columns = sessions_max_level_rig.columns.droplevel()\n",
- "sessions_max_level_rig\n",
- "\n",
- "\n"
+ "sessions_max_level_rig"
]
},
{
@@ -1354,13 +1371,29 @@
}
],
"source": [
- "fig, ax = plt.subplots(1, 1, figsize=(15,12))\n",
- "x_pos = np.linspace(1, sessions_max_level_rig.shape[0], num=sessions_max_level_rig.shape[0])\n",
- "ax.bar(x_pos, sessions_max_level_rig['mean_sessions'], yerr=sessions_max_level_rig['std_sessions'], align='center', alpha=0.5, ecolor='black', capsize=10)\n",
- "ax.set_ylabel('Num sessions to max level', fontsize=16)\n",
+ "fig, ax = plt.subplots(1, 1, figsize=(15, 12))\n",
+ "x_pos = np.linspace(\n",
+ " 1, sessions_max_level_rig.shape[0], num=sessions_max_level_rig.shape[0]\n",
+ ")\n",
+ "ax.bar(\n",
+ " x_pos,\n",
+ " sessions_max_level_rig[\"mean_sessions\"],\n",
+ " yerr=sessions_max_level_rig[\"std_sessions\"],\n",
+ " align=\"center\",\n",
+ " alpha=0.5,\n",
+ " ecolor=\"black\",\n",
+ " capsize=10,\n",
+ ")\n",
+ "ax.set_ylabel(\"Num sessions to max level\", fontsize=16)\n",
"ax.set_xticks(x_pos)\n",
"ax.set_xticklabels(sessions_max_level_rig.index)\n",
- "ax.set_title('Sessions to max level per rig protocol:' + protocol_df.index[0][0:20] + ' max_level= ' + str(max_level), fontsize=18)\n",
+ "ax.set_title(\n",
+ " \"Sessions to max level per rig protocol:\"\n",
+ " + protocol_df.index[0][0:20]\n",
+ " + \" max_level= \"\n",
+ " + str(max_level),\n",
+ " fontsize=18,\n",
+ ")\n",
"ax.yaxis.grid(True)\n",
"\n",
"# Save the figure and show\n",
diff --git a/notebooks/check_pos_image_quality.ipynb b/notebooks/check_pos_image_quality.ipynb
index 6e7b5591..87bc8b76 100644
--- a/notebooks/check_pos_image_quality.ipynb
+++ b/notebooks/check_pos_image_quality.ipynb
@@ -23,6 +23,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -32,15 +33,11 @@
"metadata": {},
"outputs": [],
"source": [
- "import datajoint as dj\n",
- "import os\n",
- "import pathlib\n",
- "import pandas as pd\n",
"import cv2\n",
- "import glob\n",
- "import numpy as np\n",
+ "import datajoint as dj\n",
"import matplotlib.pyplot as plt\n",
- "from matplotlib import patches\n"
+ "import numpy as np\n",
+ "import pandas as pd"
]
},
{
@@ -59,7 +56,7 @@
],
"source": [
"dj.conn()\n",
- "action_db = dj.create_virtual_module('action', 'u19_action')"
+ "action_db = dj.create_virtual_module(\"action\", \"u19_action\")"
]
},
{
@@ -76,8 +73,10 @@
"outputs": [],
"source": [
"query = dict()\n",
- "query['location'] = '165A-miniVR-T-2'\n",
- "AllImageRecords = pd.DataFrame((action_db.DailySubjectPositionData & query).fetch(as_dict=True))"
+ "query[\"location\"] = \"165A-miniVR-T-2\"\n",
+ "AllImageRecords = pd.DataFrame(\n",
+ " (action_db.DailySubjectPositionData & query).fetch(as_dict=True)\n",
+ ")"
]
},
{
@@ -315,7 +314,7 @@
"metadata": {},
"outputs": [],
"source": [
- "#AllImageRecords = AllImageRecords_copy.copy(deep=True)"
+ "# AllImageRecords = AllImageRecords_copy.copy(deep=True)"
]
},
{
@@ -331,11 +330,10 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
- "def analyze_image_quality(_image, type='lateral'):\n",
- " #_image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)\n",
- " if type != 'lateral':\n",
- " _image = _image[:,200:600]\n",
+ "def analyze_image_quality(_image, type=\"lateral\"):\n",
+ " # _image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)\n",
+ " if type != \"lateral\":\n",
+ " _image = _image[:, 200:600]\n",
"\n",
" # Sharpness\n",
" _laplacian = cv2.Laplacian(_image, cv2.CV_64F)\n",
@@ -353,11 +351,7 @@
" _sobel = np.sqrt(_sobel_x**2 + _sobel_y**2)\n",
" _resolution = np.mean(_sobel)\n",
"\n",
- " return {\n",
- " \"sharpness\": _sharpness,\n",
- " \"clarity\": _clarity,\n",
- " \"resolution\": _resolution\n",
- " }"
+ " return {\"sharpness\": _sharpness, \"clarity\": _clarity, \"resolution\": _resolution}"
]
},
{
@@ -366,8 +360,12 @@
"metadata": {},
"outputs": [],
"source": [
- "AllImageRecords['top_image_metrics'] = AllImageRecords['top_image'].apply(analyze_image_quality, type='top')\n",
- "AllImageRecords['lateral_image_metrics'] = AllImageRecords['lateral_image'].apply(analyze_image_quality, type='lateral')"
+ "AllImageRecords[\"top_image_metrics\"] = AllImageRecords[\"top_image\"].apply(\n",
+ " analyze_image_quality, type=\"top\"\n",
+ ")\n",
+ "AllImageRecords[\"lateral_image_metrics\"] = AllImageRecords[\"lateral_image\"].apply(\n",
+ " analyze_image_quality, type=\"lateral\"\n",
+ ")"
]
},
{
@@ -693,15 +691,17 @@
}
],
"source": [
- "df_aux = pd.json_normalize(AllImageRecords['top_image_metrics'])\n",
- "df_aux = df_aux.rename(columns=lambda column: 'top_image_' + column)\n",
+ "df_aux = pd.json_normalize(AllImageRecords[\"top_image_metrics\"])\n",
+ "df_aux = df_aux.rename(columns=lambda column: \"top_image_\" + column)\n",
"AllImageRecords = pd.concat([AllImageRecords, df_aux], axis=1)\n",
"\n",
- "df_aux = pd.json_normalize(AllImageRecords['lateral_image_metrics'])\n",
- "df_aux = df_aux.rename(columns=lambda column: 'lateral_image_' + column)\n",
+ "df_aux = pd.json_normalize(AllImageRecords[\"lateral_image_metrics\"])\n",
+ "df_aux = df_aux.rename(columns=lambda column: \"lateral_image_\" + column)\n",
"AllImageRecords = pd.concat([AllImageRecords, df_aux], axis=1)\n",
"\n",
- "AllImageRecords = AllImageRecords.drop(columns=['top_image_metrics', 'lateral_image_metrics'])\n",
+ "AllImageRecords = AllImageRecords.drop(\n",
+ " columns=[\"top_image_metrics\", \"lateral_image_metrics\"]\n",
+ ")\n",
"AllImageRecords"
]
},
@@ -743,22 +743,22 @@
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
"\n",
- "#fig, (ax1, ax2) = plt.subplots(1,2)\n",
+ "# fig, (ax1, ax2) = plt.subplots(1,2)\n",
"# plt.rcParams['figure.dpi'] = 250\n",
"fig.add_subplot(1, 2, 1)\n",
"\n",
- "plt.plot(AllImageRecords['top_image_resolution'], linewidth=2)\n",
+ "plt.plot(AllImageRecords[\"top_image_resolution\"], linewidth=2)\n",
"# Filter by using a moving average window of 10 samples\n",
- "plt.xlabel('# Image ~ Date')\n",
- "plt.ylabel('Image Sharpness')\n",
- "plt.title('Top Camera')\n",
+ "plt.xlabel(\"# Image ~ Date\")\n",
+ "plt.ylabel(\"Image Sharpness\")\n",
+ "plt.title(\"Top Camera\")\n",
"\n",
"fig.add_subplot(1, 2, 2)\n",
- "plt.plot(AllImageRecords['lateral_image_resolution'], linewidth=2)\n",
+ "plt.plot(AllImageRecords[\"lateral_image_resolution\"], linewidth=2)\n",
"# Filter by using a moving average window of 10 samples\n",
- "plt.xlabel('# Image ~ Date')\n",
+ "plt.xlabel(\"# Image ~ Date\")\n",
"\n",
- "plt.title('Lateral Camera')"
+ "plt.title(\"Lateral Camera\")"
]
},
{
@@ -793,19 +793,17 @@
"fig = plt.figure(figsize=(20, 10))\n",
"\n",
"\n",
- "\n",
- "#fig, (ax1, ax2) = plt.subplots(1,2)\n",
+ "# fig, (ax1, ax2) = plt.subplots(1,2)\n",
"# plt.rcParams['figure.dpi'] = 250\n",
"fig.add_subplot(1, 2, 1)\n",
"\n",
- "plt.imshow(AllImageRecords.loc[5,'lateral_image'], cmap='gray')\n",
- "plt.axis('off')\n",
+ "plt.imshow(AllImageRecords.loc[5, \"lateral_image\"], cmap=\"gray\")\n",
+ "plt.axis(\"off\")\n",
"\n",
"\n",
"fig.add_subplot(1, 2, 2)\n",
- "plt.imshow(AllImageRecords.loc[60,'lateral_image'], cmap='gray')\n",
- "plt.axis('off')\n",
- "\n"
+ "plt.imshow(AllImageRecords.loc[60, \"lateral_image\"], cmap=\"gray\")\n",
+ "plt.axis(\"off\")"
]
},
{
@@ -840,19 +838,17 @@
"fig = plt.figure(figsize=(20, 10))\n",
"\n",
"\n",
- "\n",
- "#fig, (ax1, ax2) = plt.subplots(1,2)\n",
+ "# fig, (ax1, ax2) = plt.subplots(1,2)\n",
"# plt.rcParams['figure.dpi'] = 250\n",
"fig.add_subplot(1, 2, 1)\n",
"\n",
- "plt.imshow(AllImageRecords.loc[5,'top_image'], cmap='gray')\n",
- "#plt.axis('off')\n",
+ "plt.imshow(AllImageRecords.loc[5, \"top_image\"], cmap=\"gray\")\n",
+ "# plt.axis('off')\n",
"\n",
"\n",
"fig.add_subplot(1, 2, 2)\n",
- "plt.imshow(AllImageRecords.loc[59,'top_image'], cmap='gray')\n",
- "#plt.axis('off')\n",
- "\n"
+ "plt.imshow(AllImageRecords.loc[59, \"top_image\"], cmap=\"gray\")\n",
+ "# plt.axis('off')"
]
},
{
@@ -887,19 +883,17 @@
"fig = plt.figure(figsize=(20, 10))\n",
"\n",
"\n",
- "\n",
- "#fig, (ax1, ax2) = plt.subplots(1,2)\n",
+ "# fig, (ax1, ax2) = plt.subplots(1,2)\n",
"# plt.rcParams['figure.dpi'] = 250\n",
"fig.add_subplot(1, 2, 1)\n",
"\n",
- "plt.imshow(AllImageRecords.loc[5,'top_image'][:,200:600], cmap='gray')\n",
- "plt.axis('off')\n",
+ "plt.imshow(AllImageRecords.loc[5, \"top_image\"][:, 200:600], cmap=\"gray\")\n",
+ "plt.axis(\"off\")\n",
"\n",
"\n",
"fig.add_subplot(1, 2, 2)\n",
- "plt.imshow(AllImageRecords.loc[59,'top_image'][:,200:600], cmap='gray')\n",
- "plt.axis('off')\n",
- "\n"
+ "plt.imshow(AllImageRecords.loc[59, \"top_image\"][:, 200:600], cmap=\"gray\")\n",
+ "plt.axis(\"off\")"
]
},
{
@@ -919,7 +913,7 @@
}
],
"source": [
- "AllImageRecords.loc[5,'top_image'].shape"
+ "AllImageRecords.loc[5, \"top_image\"].shape"
]
},
{
@@ -952,7 +946,7 @@
],
"source": [
"fig = plt.figure(figsize=(14, 7))\n",
- "plt.imshow(AllImageRecords.loc[60,'lateral_image'])"
+ "plt.imshow(AllImageRecords.loc[60, \"lateral_image\"])"
]
},
{
@@ -1458,19 +1452,19 @@
}
],
"source": [
- "import cv2\n",
- "\n",
- "video_name = 'top_image_video4.mp4' # Name of the output video file\n",
+ "video_name = \"top_image_video4.mp4\" # Name of the output video file\n",
"\n",
- "height, width = AllImageRecords.loc[0,'top_image'].shape\n",
+ "height, width = AllImageRecords.loc[0, \"top_image\"].shape\n",
"\n",
- "fourcc = cv2.VideoWriter_fourcc(*'mp4v') # Codec for mp4 video\n",
- "video = cv2.VideoWriter(video_name, fourcc, 5, (width, height), isColor=False) # 30 is the frame rate\n",
+ "fourcc = cv2.VideoWriter_fourcc(*\"mp4v\") # Codec for mp4 video\n",
+ "video = cv2.VideoWriter(\n",
+ " video_name, fourcc, 5, (width, height), isColor=False\n",
+ ") # 30 is the frame rate\n",
"\n",
"for i in range(AllImageRecords.shape[0]):\n",
- " print(AllImageRecords.loc[i,'top_image'])\n",
+ " print(AllImageRecords.loc[i, \"top_image\"])\n",
"\n",
- " video.write(AllImageRecords.loc[i,'top_image'])\n",
+ " video.write(AllImageRecords.loc[i, \"top_image\"])\n",
"\n",
"video.release()"
]
diff --git a/notebooks/create_tech_responsibilites_table.ipynb b/notebooks/create_tech_responsibilites_table.ipynb
index 634ffe27..7ba28208 100644
--- a/notebooks/create_tech_responsibilites_table.ipynb
+++ b/notebooks/create_tech_responsibilites_table.ipynb
@@ -13,13 +13,12 @@
"metadata": {},
"outputs": [],
"source": [
+ "import json\n",
+ "\n",
"import datajoint as dj\n",
- "import numpy as np\n",
- "import pylab as plt\n",
"import pandas as pd\n",
- "import json\n",
- "import datetime\n",
- "pd.set_option('display.max_rows', 50)"
+ "\n",
+ "pd.set_option(\"display.max_rows\", 50)"
]
},
{
@@ -52,9 +51,9 @@
],
"source": [
"conn = dj.conn()\n",
- "action = dj.create_virtual_module('action', 'u19_test_action')\n",
- "subject = dj.create_virtual_module('subject', 'u19_test_subject')\n",
- "lab = dj.create_virtual_module('lab', 'u19_test_lab')"
+ "action = dj.create_virtual_module(\"action\", \"u19_test_action\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"u19_test_subject\")\n",
+ "lab = dj.create_virtual_module(\"lab\", \"u19_test_lab\")"
]
},
{
@@ -76,18 +75,38 @@
"metadata": {},
"outputs": [],
"source": [
- "all_subject_status = pd.DataFrame((action.SubjectStatus * subject.Subject.proj('user_id') * lab.User.proj('tech_responsibility')).fetch(as_dict=True))\n",
- "all_subject_status = all_subject_status.sort_values(by='effective_date', ascending=False)\n",
- "all_subject_status = all_subject_status.drop_duplicates(subset='subject_fullname', keep='first')\n",
+ "all_subject_status = pd.DataFrame(\n",
+ " (\n",
+ " action.SubjectStatus\n",
+ " * subject.Subject.proj(\"user_id\")\n",
+ " * lab.User.proj(\"tech_responsibility\")\n",
+ " ).fetch(as_dict=True)\n",
+ ")\n",
+ "all_subject_status = all_subject_status.sort_values(\n",
+ " by=\"effective_date\", ascending=False\n",
+ ")\n",
+ "all_subject_status = all_subject_status.drop_duplicates(\n",
+ " subset=\"subject_fullname\", keep=\"first\"\n",
+ ")\n",
"\n",
- "all_subject_status = all_subject_status.sort_values(by='effective_date', ascending=False)\n",
+ "all_subject_status = all_subject_status.sort_values(\n",
+ " by=\"effective_date\", ascending=False\n",
+ ")\n",
"\n",
"all_subject_status = all_subject_status.reset_index(drop=True)\n",
"\n",
- "all_subject_status['schedule_list'] = all_subject_status['schedule'].apply(lambda x: x.split('/'))\n",
- "all_subject_status['schedule_set'] = all_subject_status['schedule_list'].apply(lambda x: set(x))\n",
- "all_subject_status['num_act'] = all_subject_status['schedule_set'].apply(lambda x: len(x))\n",
- "all_subject_status['index_act'] = all_subject_status['schedule_list'].apply(lambda x: index_act(x))"
+ "all_subject_status[\"schedule_list\"] = all_subject_status[\"schedule\"].apply(\n",
+ " lambda x: x.split(\"/\")\n",
+ ")\n",
+ "all_subject_status[\"schedule_set\"] = all_subject_status[\"schedule_list\"].apply(\n",
+ " lambda x: set(x)\n",
+ ")\n",
+ "all_subject_status[\"num_act\"] = all_subject_status[\"schedule_set\"].apply(\n",
+ " lambda x: len(x)\n",
+ ")\n",
+ "all_subject_status[\"index_act\"] = all_subject_status[\"schedule_list\"].apply(\n",
+ " lambda x: index_act(x)\n",
+ ")"
]
},
{
@@ -104,28 +123,27 @@
"outputs": [],
"source": [
"subject_status_dict_owner = {\n",
- " 'Dead': ['Nothing'],\n",
- " 'AdLibWater': ['Nothing'],\n",
- " 'Missing': ['Nothing'],\n",
- " 'WaterRestrictionOnly': ['Watering', 'Weighing'],\n",
- " 'InExperiments': ['Watering', 'Weighing', 'Training']\n",
+ " \"Dead\": [\"Nothing\"],\n",
+ " \"AdLibWater\": [\"Nothing\"],\n",
+ " \"Missing\": [\"Nothing\"],\n",
+ " \"WaterRestrictionOnly\": [\"Watering\", \"Weighing\"],\n",
+ " \"InExperiments\": [\"Watering\", \"Weighing\", \"Training\"],\n",
"}\n",
"subject_status_dict_technician = {\n",
- " 'Dead': ['Nothing'],\n",
- " 'AdLibWater': ['Nothing'],\n",
- " 'Missing': ['Nothing'],\n",
- " 'WaterRestrictionOnly': ['Watering', 'Weighing', 'Transport'],\n",
- " 'InExperiments': ['Watering', 'Weighing', 'Training', 'Transport']\n",
+ " \"Dead\": [\"Nothing\"],\n",
+ " \"AdLibWater\": [\"Nothing\"],\n",
+ " \"Missing\": [\"Nothing\"],\n",
+ " \"WaterRestrictionOnly\": [\"Watering\", \"Weighing\", \"Transport\"],\n",
+ " \"InExperiments\": [\"Watering\", \"Weighing\", \"Training\", \"Transport\"],\n",
"}\n",
"\n",
"act_to_status_dict = {\n",
- " 'Water': 'WaterRestrictionOnly',\n",
- " 'Weigh': 'WaterRestrictionOnly',\n",
- " 'Train': 'InExperiments',\n",
- " 'OnlyTrain': 'InExperiments',\n",
- " 'Nothing': 'Dead',\n",
- " 'Transport': 'Dead'\n",
- "\n",
+ " \"Water\": \"WaterRestrictionOnly\",\n",
+ " \"Weigh\": \"WaterRestrictionOnly\",\n",
+ " \"Train\": \"InExperiments\",\n",
+ " \"OnlyTrain\": \"InExperiments\",\n",
+ " \"Nothing\": \"Dead\",\n",
+ " \"Transport\": \"Dead\",\n",
"}"
]
},
@@ -143,28 +161,27 @@
"outputs": [],
"source": [
"subject_status_dict_owner_shared = {\n",
- " 'Dead': ['Nothing'],\n",
- " 'AdLibWater': ['Nothing'],\n",
- " 'Missing': ['Nothing'],\n",
- " 'WaterRestrictionOnly': ['Watering', 'Weighing'],\n",
- " 'InExperiments': ['Watering', 'Weighing', 'Training']\n",
+ " \"Dead\": [\"Nothing\"],\n",
+ " \"AdLibWater\": [\"Nothing\"],\n",
+ " \"Missing\": [\"Nothing\"],\n",
+ " \"WaterRestrictionOnly\": [\"Watering\", \"Weighing\"],\n",
+ " \"InExperiments\": [\"Watering\", \"Weighing\", \"Training\"],\n",
"}\n",
"subject_status_dict_technician_shared = {\n",
- " 'Dead': ['Nothing'],\n",
- " 'AdLibWater': ['Nothing'],\n",
- " 'Missing': ['Nothing'],\n",
- " 'WaterRestrictionOnly': ['Watering', 'Weighing', 'Transport'],\n",
- " 'InExperiments': ['Watering', 'Weighing', 'Training', 'Transport']\n",
+ " \"Dead\": [\"Nothing\"],\n",
+ " \"AdLibWater\": [\"Nothing\"],\n",
+ " \"Missing\": [\"Nothing\"],\n",
+ " \"WaterRestrictionOnly\": [\"Watering\", \"Weighing\", \"Transport\"],\n",
+ " \"InExperiments\": [\"Watering\", \"Weighing\", \"Training\", \"Transport\"],\n",
"}\n",
"\n",
"act_to_status_dict_shared = {\n",
- " 'Water': 'WaterRestrictionOnly',\n",
- " 'Weigh': 'WaterRestrictionOnly',\n",
- " 'Train': 'InExperiments',\n",
- " 'OnlyTrain': 'InExperiments',\n",
- " 'Nothing': 'Dead',\n",
- " 'Transport': ['Transport']\n",
- "\n",
+ " \"Water\": \"WaterRestrictionOnly\",\n",
+ " \"Weigh\": \"WaterRestrictionOnly\",\n",
+ " \"Train\": \"InExperiments\",\n",
+ " \"OnlyTrain\": \"InExperiments\",\n",
+ " \"Nothing\": \"Dead\",\n",
+ " \"Transport\": [\"Transport\"],\n",
"}"
]
},
@@ -175,27 +192,27 @@
"outputs": [],
"source": [
"base_weekly_dict = {\n",
- " \"assignment_style\": \"Specify for the whole week\",\n",
- " \"weekly\": ['Nothing']\n",
+ " \"assignment_style\": \"Specify for the whole week\",\n",
+ " \"weekly\": [\"Nothing\"],\n",
"}\n",
"\n",
"base_weekdays_weekends_dict = {\n",
- " \"assignment_style\": \"Specify by weekday/weekend\",\n",
- " \"weekdays\": ['Nothing'],\n",
- " \"weekends\": ['Nothing']\n",
+ " \"assignment_style\": \"Specify by weekday/weekend\",\n",
+ " \"weekdays\": [\"Nothing\"],\n",
+ " \"weekends\": [\"Nothing\"],\n",
"}\n",
"\n",
"\n",
"base_days_dict = {\n",
- "\"assignment_style\":\"Specify each day\",\n",
- "\"Monday\": ['Nothing'],\n",
- "\"Tuesday\": ['Nothing'],\n",
- "\"Wednesday\": ['Nothing'],\n",
- "\"Thursday\": ['Nothing'],\n",
- "\"Friday\": ['Nothing'],\n",
- "\"Saturday\": ['Nothing'],\n",
- "\"Sunday\": ['Nothing'],\n",
- "}\n"
+ " \"assignment_style\": \"Specify each day\",\n",
+ " \"Monday\": [\"Nothing\"],\n",
+ " \"Tuesday\": [\"Nothing\"],\n",
+ " \"Wednesday\": [\"Nothing\"],\n",
+ " \"Thursday\": [\"Nothing\"],\n",
+ " \"Friday\": [\"Nothing\"],\n",
+ " \"Saturday\": [\"Nothing\"],\n",
+ " \"Sunday\": [\"Nothing\"],\n",
+ "}"
]
},
{
@@ -206,89 +223,115 @@
"source": [
"def translate_schedule(df_row):\n",
"\n",
- " user_id = df_row['user_id']\n",
- " schedule_list = df_row['schedule_list']\n",
- " tech_responsibility = df_row['tech_responsibility']\n",
- " subject_status = df_row['subject_status']\n",
- " num_act = df_row['num_act']\n",
- " index_act = df_row['index_act']\n",
- "\n",
+ " user_id = df_row[\"user_id\"]\n",
+ " schedule_list = df_row[\"schedule_list\"]\n",
+ " tech_responsibility = df_row[\"tech_responsibility\"]\n",
+ " subject_status = df_row[\"subject_status\"]\n",
+ " num_act = df_row[\"num_act\"]\n",
+ " index_act = df_row[\"index_act\"]\n",
"\n",
" tech_act_dict = base_weekly_dict.copy()\n",
- " tech_act_dict['weekly'] = ['Nothing']\n",
+ " tech_act_dict[\"weekly\"] = [\"Nothing\"]\n",
" owner_act_dict = base_weekly_dict.copy()\n",
- " owner_act_dict['weekly'] = ['Nothing']\n",
+ " owner_act_dict[\"weekly\"] = [\"Nothing\"]\n",
"\n",
- " if tech_responsibility == 'yes':\n",
- " if subject_status in ('Dead', 'AdLibWater', 'Missing'):\n",
- " tech_act_dict = base_weekly_dict.copy()\n",
- " tech_act_dict['weekly'] = subject_status_dict_technician[subject_status]\n",
- " elif num_act == 1:\n",
+ " if tech_responsibility == \"yes\":\n",
+ " if subject_status in (\"Dead\", \"AdLibWater\", \"Missing\") or num_act == 1:\n",
" tech_act_dict = base_weekly_dict.copy()\n",
- " tech_act_dict['weekly'] = subject_status_dict_technician[subject_status]\n",
- " elif num_act == 2 and list(index_act.values())[0] == [1,2,3,4,5] or list(index_act.values())[0] == [0,6]:\n",
- "\n",
+ " tech_act_dict[\"weekly\"] = subject_status_dict_technician[subject_status]\n",
+ " elif (\n",
+ " num_act == 2\n",
+ " and list(index_act.values())[0] == [1, 2, 3, 4, 5]\n",
+ " or list(index_act.values())[0] == [0, 6]\n",
+ " ):\n",
" tech_act_dict = base_weekdays_weekends_dict.copy()\n",
" weekdays_act = schedule_list[1]\n",
" weekdends_act = schedule_list[6]\n",
- " if weekdays_act == 'Transport' or weekdends_act == 'Transport':\n",
+ " if weekdays_act == \"Transport\" or weekdends_act == \"Transport\":\n",
" owner_act_dict = base_weekdays_weekends_dict\n",
- " if weekdays_act == 'Transport':\n",
- " tech_act_dict['weekdays'] = ['Transport']\n",
- " owner_act_dict['weekdays'] = subject_status_dict_owner[subject_status]\n",
+ " if weekdays_act == \"Transport\":\n",
+ " tech_act_dict[\"weekdays\"] = [\"Transport\"]\n",
+ " owner_act_dict[\"weekdays\"] = subject_status_dict_owner[subject_status]\n",
" else:\n",
" weekdays_status = act_to_status_dict[weekdays_act]\n",
- " tech_act_dict['weekdays'] = subject_status_dict_technician[weekdays_status]\n",
- " if weekdends_act == 'Transport':\n",
- " tech_act_dict['weekends'] = ['Transport']\n",
- " owner_act_dict['weekends'] = subject_status_dict_owner[subject_status]\n",
+ " tech_act_dict[\"weekdays\"] = subject_status_dict_technician[\n",
+ " weekdays_status\n",
+ " ]\n",
+ " if weekdends_act == \"Transport\":\n",
+ " tech_act_dict[\"weekends\"] = [\"Transport\"]\n",
+ " owner_act_dict[\"weekends\"] = subject_status_dict_owner[subject_status]\n",
" else:\n",
" weekdends_status = act_to_status_dict[weekdends_act]\n",
- " tech_act_dict['weekends'] = subject_status_dict_technician[weekdends_status]\n",
+ " tech_act_dict[\"weekends\"] = subject_status_dict_technician[\n",
+ " weekdends_status\n",
+ " ]\n",
" else:\n",
" tech_act_dict = base_days_dict.copy()\n",
"\n",
- " \n",
+ " tech_act_dict[\"Monday\"] = subject_status_dict_technician[\n",
+ " act_to_status_dict[schedule_list[0]]\n",
+ " ]\n",
+ " tech_act_dict[\"Tuesday\"] = subject_status_dict_technician[\n",
+ " act_to_status_dict[schedule_list[1]]\n",
+ " ]\n",
+ " tech_act_dict[\"Wednesday\"] = subject_status_dict_technician[\n",
+ " act_to_status_dict[schedule_list[2]]\n",
+ " ]\n",
+ " tech_act_dict[\"Thursday\"] = subject_status_dict_technician[\n",
+ " act_to_status_dict[schedule_list[3]]\n",
+ " ]\n",
+ " tech_act_dict[\"Friday\"] = subject_status_dict_technician[\n",
+ " act_to_status_dict[schedule_list[4]]\n",
+ " ]\n",
+ " tech_act_dict[\"Saturday\"] = subject_status_dict_technician[\n",
+ " act_to_status_dict[schedule_list[5]]\n",
+ " ]\n",
+ " tech_act_dict[\"Sunday\"] = subject_status_dict_technician[\n",
+ " act_to_status_dict[schedule_list[6]]\n",
+ " ]\n",
"\n",
- " tech_act_dict['Monday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[0]]]\n",
- " tech_act_dict['Tuesday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[1]]]\n",
- " tech_act_dict['Wednesday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[2]]]\n",
- " tech_act_dict['Thursday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[3]]]\n",
- " tech_act_dict['Friday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[4]]]\n",
- " tech_act_dict['Saturday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[5]]]\n",
- " tech_act_dict['Sunday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[6]]]\n",
- " \n",
" # Tech_responsibility = no\n",
" else:\n",
- " if subject_status not in ('Dead', 'AdLibWater', 'Missing'):\n",
- " tech_act_dict['weekly'] = ['Transport']\n",
+ " if subject_status not in (\"Dead\", \"AdLibWater\", \"Missing\"):\n",
+ " tech_act_dict[\"weekly\"] = [\"Transport\"]\n",
" if num_act == 1:\n",
- " owner_act_dict['weekly'] = subject_status_dict_owner[subject_status]\n",
- " elif num_act == 2 and list(index_act.values())[0] == [1,2,3,4,5] or list(index_act.values())[0] == [0,6]:\n",
+ " owner_act_dict[\"weekly\"] = subject_status_dict_owner[subject_status]\n",
+ " elif (\n",
+ " num_act == 2\n",
+ " and list(index_act.values())[0] == [1, 2, 3, 4, 5]\n",
+ " or list(index_act.values())[0] == [0, 6]\n",
+ " ):\n",
" owner_act_dict = base_weekdays_weekends_dict.copy()\n",
" weekdays_act = schedule_list[1]\n",
" weekdends_act = schedule_list[6]\n",
- " owner_act_dict['weekdays'] = subject_status_dict_owner[subject_status]\n",
- " owner_act_dict['weekends'] = subject_status_dict_owner[subject_status]\n",
+ " owner_act_dict[\"weekdays\"] = subject_status_dict_owner[subject_status]\n",
+ " owner_act_dict[\"weekends\"] = subject_status_dict_owner[subject_status]\n",
" else:\n",
" owner_act_dict = base_days_dict.copy()\n",
- " schedule_list = [act_to_status_dict[x] if x!='Transport' else x for x in schedule_list]\n",
- " schedule_list = [subject_status if x=='Transport' else x for x in schedule_list]\n",
- "\n",
- " owner_act_dict['Monday'] = subject_status_dict_owner[schedule_list[0]]\n",
- " owner_act_dict['Tuesday'] = subject_status_dict_owner[schedule_list[1]]\n",
- " owner_act_dict['Wednesday'] = subject_status_dict_owner[schedule_list[2]]\n",
- " owner_act_dict['Thursday'] = subject_status_dict_owner[schedule_list[3]]\n",
- " owner_act_dict['Friday'] = subject_status_dict_owner[schedule_list[4]]\n",
- " owner_act_dict['Saturday'] = subject_status_dict_owner[schedule_list[5]]\n",
- " owner_act_dict['Sunday'] = subject_status_dict_owner[schedule_list[6]]\n",
- " \n",
- " \n",
- " return pd.Series({'technician_duties': json.dumps(tech_act_dict), 'owner_duties': json.dumps(owner_act_dict)}) \n",
+ " schedule_list = [\n",
+ " act_to_status_dict[x] if x != \"Transport\" else x\n",
+ " for x in schedule_list\n",
+ " ]\n",
+ " schedule_list = [\n",
+ " subject_status if x == \"Transport\" else x for x in schedule_list\n",
+ " ]\n",
"\n",
+ " owner_act_dict[\"Monday\"] = subject_status_dict_owner[schedule_list[0]]\n",
+ " owner_act_dict[\"Tuesday\"] = subject_status_dict_owner[schedule_list[1]]\n",
+ " owner_act_dict[\"Wednesday\"] = subject_status_dict_owner[\n",
+ " schedule_list[2]\n",
+ " ]\n",
+ " owner_act_dict[\"Thursday\"] = subject_status_dict_owner[schedule_list[3]]\n",
+ " owner_act_dict[\"Friday\"] = subject_status_dict_owner[schedule_list[4]]\n",
+ " owner_act_dict[\"Saturday\"] = subject_status_dict_owner[schedule_list[5]]\n",
+ " owner_act_dict[\"Sunday\"] = subject_status_dict_owner[schedule_list[6]]\n",
"\n",
- " \n",
- "\n"
+ " return pd.Series(\n",
+ " {\n",
+ " \"technician_duties\": json.dumps(tech_act_dict),\n",
+ " \"owner_duties\": json.dumps(owner_act_dict),\n",
+ " }\n",
+ " )"
]
},
{
@@ -297,7 +340,9 @@
"metadata": {},
"outputs": [],
"source": [
- "all_subject_status[['technician_duties', 'owner_duties']] = all_subject_status.apply(translate_schedule, axis=1)"
+ "all_subject_status[[\"technician_duties\", \"owner_duties\"]] = all_subject_status.apply(\n",
+ " translate_schedule, axis=1\n",
+ ")"
]
},
{
@@ -649,7 +694,7 @@
}
],
"source": [
- "all_subject_status['technician_duties'].unique()"
+ "all_subject_status[\"technician_duties\"].unique()"
]
},
{
@@ -679,7 +724,7 @@
}
],
"source": [
- "all_subject_status['owner_duties'].unique()"
+ "all_subject_status[\"owner_duties\"].unique()"
]
},
{
@@ -842,7 +887,12 @@
}
],
"source": [
- "all_subject_status.loc[all_subject_status['owner_duties'].str.contains('{\"assignment_style\": \"Specify each day\",'),:]"
+ "all_subject_status.loc[\n",
+ " all_subject_status[\"owner_duties\"].str.contains(\n",
+ " '{\"assignment_style\": \"Specify each day\",'\n",
+ " ),\n",
+ " :,\n",
+ "]"
]
},
{
@@ -1021,7 +1071,16 @@
}
],
"source": [
- "all_subject_status_final = all_subject_status.loc[:, ['subject_fullname', 'subject_status', 'water_per_day', 'technician_duties', 'owner_duties']]\n",
+ "all_subject_status_final = all_subject_status.loc[\n",
+ " :,\n",
+ " [\n",
+ " \"subject_fullname\",\n",
+ " \"subject_status\",\n",
+ " \"water_per_day\",\n",
+ " \"technician_duties\",\n",
+ " \"owner_duties\",\n",
+ " ],\n",
+ "]\n",
"all_subject_status_final"
]
},
@@ -6042,7 +6101,7 @@
}
],
"source": [
- "all_subject_status_final.to_dict('records')"
+ "all_subject_status_final.to_dict(\"records\")"
]
},
{
@@ -6051,7 +6110,7 @@
"metadata": {},
"outputs": [],
"source": [
- "action.Responsibilities.insert(all_subject_status_final.to_dict('records'))"
+ "action.Responsibilities.insert(all_subject_status_final.to_dict(\"records\"))"
]
},
{
@@ -6076,7 +6135,7 @@
}
],
"source": [
- "all_subject_status_final.loc[288,:]"
+ "all_subject_status_final.loc[288, :]"
]
},
{
diff --git a/notebooks/create_weighingGUI_ss.ipynb b/notebooks/create_weighingGUI_ss.ipynb
index a2408ac5..b7fb7ea7 100644
--- a/notebooks/create_weighingGUI_ss.ipynb
+++ b/notebooks/create_weighingGUI_ss.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -41,41 +42,42 @@
}
],
"source": [
- "import datajoint as dj\n",
- "import pandas as pd\n",
- "import time\n",
- "from zoneinfo import ZoneInfo\n",
+ "import base64\n",
"import datetime\n",
"import pathlib\n",
- "import numpy as np\n",
- "import time\n",
- "import base64\n",
"import shutil\n",
+ "import time\n",
+ "\n",
+ "import datajoint as dj\n",
+ "import numpy as np\n",
"import openpyxl\n",
- "from openpyxl.utils.dataframe import dataframe_to_rows\n",
+ "import pandas as pd\n",
"from openpyxl.drawing.image import Image\n",
+ "from openpyxl.utils.dataframe import dataframe_to_rows\n",
"\n",
"time.sleep(1)\n",
"\n",
- "import u19_pipeline.alert_system.water_weigh_alert.water_weigh_alert as wwa\n",
- "import u19_pipeline.scheduler as scheduler\n",
"import u19_pipeline.acquisition as acquisition\n",
+ "import u19_pipeline.alert_system.water_weigh_alert.water_weigh_alert as wwa\n",
"import u19_pipeline.behavior as behavior\n",
- "import u19_pipeline.subject as subject\n",
"import u19_pipeline.lab as lab\n",
+ "import u19_pipeline.scheduler as scheduler\n",
+ "import u19_pipeline.subject as subject\n",
"\n",
+ "DJ_CUSTOM_VARIABLES_FILENAME = \"DJCustomVariables.csv\"\n",
+ "SLACK_WEBHOOK_FILENAME = \"SlackChannels.csv\"\n",
+ "USER_SLACK_FILENAME = \"UserSlack.csv\"\n",
+ "RIG_STATUS_FILENAME = \"RigStatusTable.csv\"\n",
+ "DAY_SCHEDULE_FILENAME = \"ScheduleDay.csv\"\n",
+ "PAST_SESSION_PERFORMANCE_FILENAME = \"PastSessions.csv\"\n",
+ "SUBJECT_MOTOR_POSITION_FILENAME = \"SubjectMotorPosition.csv\"\n",
"\n",
- "DJ_CUSTOM_VARIABLES_FILENAME = 'DJCustomVariables.csv'\n",
- "SLACK_WEBHOOK_FILENAME = 'SlackChannels.csv'\n",
- "USER_SLACK_FILENAME = 'UserSlack.csv'\n",
- "RIG_STATUS_FILENAME = 'RigStatusTable.csv'\n",
- "DAY_SCHEDULE_FILENAME = 'ScheduleDay.csv'\n",
- "PAST_SESSION_PERFORMANCE_FILENAME = 'PastSessions.csv'\n",
- "SUBJECT_MOTOR_POSITION_FILENAME = 'SubjectMotorPosition.csv'\n",
- "\n",
- "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = 'Weighing_GUI_Replacement_SpreadSheet_Template.xlsx'\n",
- "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = 'Weighing_GUI_Replacement_SpreadSheet.xlsx'\n",
- "\n",
+ "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = (\n",
+ " \"Weighing_GUI_Replacement_SpreadSheet_Template.xlsx\"\n",
+ ")\n",
+ "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = (\n",
+ " \"Weighing_GUI_Replacement_SpreadSheet.xlsx\"\n",
+ ")\n",
"\n",
"\n",
"MAX_SESSIONS_HISTORY = 75"
@@ -90,11 +92,11 @@
"def cast_choice(choice_array):\n",
"\n",
" new_array = choice_array[0].copy()\n",
- " new_array[new_array>2] = 127\n",
+ " new_array[new_array > 2] = 127\n",
"\n",
" new_array = np.array(new_array, dtype=np.uint8)\n",
"\n",
- " return new_array\n"
+ " return new_array"
]
},
{
@@ -105,8 +107,7 @@
"source": [
"def encode_webhook(webhook):\n",
"\n",
- " return (base64.b64encode(webhook.encode('utf-8'))).decode('utf-8')\n",
- " "
+ " return (base64.b64encode(webhook.encode(\"utf-8\"))).decode(\"utf-8\")"
]
},
{
@@ -117,23 +118,41 @@
"source": [
"conf = dj.config\n",
"\n",
- "nodb_virmen_backup_dir = pathlib.Path(pathlib.Path(conf['custom']['root_data_dir'][0]).parent.parent,'Shared','NoDBVirmenBackup')\n",
- "\n",
- "DJ_CUSTOM_VARIABLES_FILENAME = pathlib.Path(nodb_virmen_backup_dir,DJ_CUSTOM_VARIABLES_FILENAME).as_posix()\n",
- "SLACK_WEBHOOK_FILENAME = pathlib.Path(nodb_virmen_backup_dir,SLACK_WEBHOOK_FILENAME).as_posix()\n",
- "USER_SLACK_FILENAME = pathlib.Path(nodb_virmen_backup_dir,USER_SLACK_FILENAME).as_posix()\n",
- "RIG_STATUS_FILENAME = pathlib.Path(nodb_virmen_backup_dir,RIG_STATUS_FILENAME).as_posix()\n",
- "DAY_SCHEDULE_FILENAME = pathlib.Path(nodb_virmen_backup_dir,DAY_SCHEDULE_FILENAME).as_posix()\n",
- "PAST_SESSION_PERFORMANCE_FILENAME = pathlib.Path(nodb_virmen_backup_dir,PAST_SESSION_PERFORMANCE_FILENAME).as_posix()\n",
- "SUBJECT_MOTOR_POSITION_FILENAME = pathlib.Path(nodb_virmen_backup_dir,SUBJECT_MOTOR_POSITION_FILENAME).as_posix()\n",
+ "nodb_virmen_backup_dir = pathlib.Path(\n",
+ " pathlib.Path(conf[\"custom\"][\"root_data_dir\"][0]).parent.parent,\n",
+ " \"Shared\",\n",
+ " \"NoDBVirmenBackup\",\n",
+ ")\n",
"\n",
- "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = pathlib.Path(nodb_virmen_backup_dir,\\\n",
- " WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME).as_posix()\n",
+ "DJ_CUSTOM_VARIABLES_FILENAME = pathlib.Path(\n",
+ " nodb_virmen_backup_dir, DJ_CUSTOM_VARIABLES_FILENAME\n",
+ ").as_posix()\n",
+ "SLACK_WEBHOOK_FILENAME = pathlib.Path(\n",
+ " nodb_virmen_backup_dir, SLACK_WEBHOOK_FILENAME\n",
+ ").as_posix()\n",
+ "USER_SLACK_FILENAME = pathlib.Path(\n",
+ " nodb_virmen_backup_dir, USER_SLACK_FILENAME\n",
+ ").as_posix()\n",
+ "RIG_STATUS_FILENAME = pathlib.Path(\n",
+ " nodb_virmen_backup_dir, RIG_STATUS_FILENAME\n",
+ ").as_posix()\n",
+ "DAY_SCHEDULE_FILENAME = pathlib.Path(\n",
+ " nodb_virmen_backup_dir, DAY_SCHEDULE_FILENAME\n",
+ ").as_posix()\n",
+ "PAST_SESSION_PERFORMANCE_FILENAME = pathlib.Path(\n",
+ " nodb_virmen_backup_dir, PAST_SESSION_PERFORMANCE_FILENAME\n",
+ ").as_posix()\n",
+ "SUBJECT_MOTOR_POSITION_FILENAME = pathlib.Path(\n",
+ " nodb_virmen_backup_dir, SUBJECT_MOTOR_POSITION_FILENAME\n",
+ ").as_posix()\n",
"\n",
- "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = pathlib.Path(nodb_virmen_backup_dir,\\\n",
- " WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE).as_posix()\n",
+ "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = pathlib.Path(\n",
+ " nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME\n",
+ ").as_posix()\n",
"\n",
- "\n"
+ "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = pathlib.Path(\n",
+ " nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE\n",
+ ").as_posix()"
]
},
{
@@ -142,7 +161,9 @@
"metadata": {},
"outputs": [],
"source": [
- "pd.DataFrame(lab.DjCustomVariables.fetch(as_dict=True)).to_csv(DJ_CUSTOM_VARIABLES_FILENAME)\n"
+ "pd.DataFrame(lab.DjCustomVariables.fetch(as_dict=True)).to_csv(\n",
+ " DJ_CUSTOM_VARIABLES_FILENAME\n",
+ ")"
]
},
{
@@ -152,9 +173,9 @@
"outputs": [],
"source": [
"slack_webhooks = pd.DataFrame(lab.SlackWebhooks.fetch(as_dict=True))\n",
- "slack_webhooks['webhook_url'] = slack_webhooks['webhook_url'].astype(str)\n",
+ "slack_webhooks[\"webhook_url\"] = slack_webhooks[\"webhook_url\"].astype(str)\n",
"\n",
- "slack_webhooks['webhook_url'] = slack_webhooks['webhook_url'].apply(encode_webhook)\n",
+ "slack_webhooks[\"webhook_url\"] = slack_webhooks[\"webhook_url\"].apply(encode_webhook)\n",
"slack_webhooks.to_csv(SLACK_WEBHOOK_FILENAME, index=False)"
]
},
@@ -164,11 +185,15 @@
"metadata": {},
"outputs": [],
"source": [
- "user_data = pd.DataFrame(lab.User.fetch('user_id', 'slack', 'tech_responsibility', 'slack_webhook',as_dict=True))\n",
- "user_data['slack_webhook'] = user_data['slack_webhook'].astype(str)\n",
- "user_data['slack_webhook'] = user_data['slack_webhook'].fillna('')\n",
+ "user_data = pd.DataFrame(\n",
+ " lab.User.fetch(\n",
+ " \"user_id\", \"slack\", \"tech_responsibility\", \"slack_webhook\", as_dict=True\n",
+ " )\n",
+ ")\n",
+ "user_data[\"slack_webhook\"] = user_data[\"slack_webhook\"].astype(str)\n",
+ "user_data[\"slack_webhook\"] = user_data[\"slack_webhook\"].fillna(\"\")\n",
"\n",
- "user_data['slack_webhook'] = user_data['slack_webhook'].apply(encode_webhook)\n",
+ "user_data[\"slack_webhook\"] = user_data[\"slack_webhook\"].apply(encode_webhook)\n",
"user_data.to_csv(USER_SLACK_FILENAME, index=False)"
]
},
@@ -178,7 +203,11 @@
"metadata": {},
"outputs": [],
"source": [
- "df_rig_status = pd.DataFrame(scheduler.RigStatus.fetch('location', 'input_output_name','current_status',as_dict=True))\n",
+ "df_rig_status = pd.DataFrame(\n",
+ " scheduler.RigStatus.fetch(\n",
+ " \"location\", \"input_output_name\", \"current_status\", as_dict=True\n",
+ " )\n",
+ ")\n",
"df_rig_status.to_csv(RIG_STATUS_FILENAME, index=False)"
]
},
@@ -1371,13 +1400,21 @@
],
"source": [
"schedule_query = dict()\n",
- "schedule_query['date'] = datetime.date.today() \n",
+ "schedule_query[\"date\"] = datetime.date.today()\n",
"\n",
- "subject_query = 'subject_fullname is not null'\n",
+ "subject_query = \"subject_fullname is not null\"\n",
"\n",
- "day_schedule = pd.DataFrame((scheduler.Schedule * scheduler.TrainingProfile * subject.Subject.proj(subject_user_id='user_id') & schedule_query & subject_query).fetch(as_dict=True))\n",
- "day_schedule = day_schedule.drop(columns='user_id')\n",
- "day_schedule = day_schedule.rename(columns={'subject_user_id':'user_id'})\n",
+ "day_schedule = pd.DataFrame(\n",
+ " (\n",
+ " scheduler.Schedule\n",
+ " * scheduler.TrainingProfile\n",
+ " * subject.Subject.proj(subject_user_id=\"user_id\")\n",
+ " & schedule_query\n",
+ " & subject_query\n",
+ " ).fetch(as_dict=True)\n",
+ ")\n",
+ "day_schedule = day_schedule.drop(columns=\"user_id\")\n",
+ "day_schedule = day_schedule.rename(columns={\"subject_user_id\": \"user_id\"})\n",
"\n",
"\n",
"day_schedule.to_csv(DAY_SCHEDULE_FILENAME, index=False)\n",
@@ -1390,12 +1427,12 @@
"metadata": {},
"outputs": [],
"source": [
- "#shcedule = pd.read_csv(DAY_SCHEDULE_FILENAME)\n",
- "#shcedule = shcedule.loc[:,['date', 'location','timeslot','subject_fullname','training_profile_id','recording_profile_id','input_output_profile_id','experimenters_instructions']]\n",
- "#shcedule.loc[shcedule['experimenters_instructions'].isna(),'experimenters_instructions'] = ''\n",
- "#shcedule\n",
- "#list_of_dicts = shcedule.to_dict(orient='records')\n",
- "#scheduler.Schedule.insert(list_of_dicts)"
+ "# shcedule = pd.read_csv(DAY_SCHEDULE_FILENAME)\n",
+ "# shcedule = shcedule.loc[:,['date', 'location','timeslot','subject_fullname','training_profile_id','recording_profile_id','input_output_profile_id','experimenters_instructions']]\n",
+ "# shcedule.loc[shcedule['experimenters_instructions'].isna(),'experimenters_instructions'] = ''\n",
+ "# shcedule\n",
+ "# list_of_dicts = shcedule.to_dict(orient='records')\n",
+ "# scheduler.Schedule.insert(list_of_dicts)"
]
},
{
@@ -1415,8 +1452,8 @@
}
],
"source": [
- "all_subjects_schedule = \"', '\".join(day_schedule['subject_fullname'])\n",
- "all_subjects_schedule = \"subject_fullname in ('\" +all_subjects_schedule+ \"')\"\n",
+ "all_subjects_schedule = \"', '\".join(day_schedule[\"subject_fullname\"])\n",
+ "all_subjects_schedule = \"subject_fullname in ('\" + all_subjects_schedule + \"')\"\n",
"all_subjects_schedule"
]
},
@@ -1780,7 +1817,9 @@
}
],
"source": [
- "sp = pd.DataFrame((subject.HeadMotorPosition & all_subjects_schedule).fetch(as_dict=True))\n",
+ "sp = pd.DataFrame(\n",
+ " (subject.HeadMotorPosition & all_subjects_schedule).fetch(as_dict=True)\n",
+ ")\n",
"\n",
"sp.to_csv(SUBJECT_MOTOR_POSITION_FILENAME, index=False)\n",
"sp"
@@ -1949,20 +1988,34 @@
}
],
"source": [
- "ss = pd.DataFrame((behavior.TowersSession & all_subjects_schedule).fetch('KEY', order_by='subject_fullname, session_date desc, session_number desc', as_dict=True))\n",
- "ss['session_date'] = ss['session_date'].astype(str)\n",
- "ss['num_sessions'] = ss.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')\n",
- "ss['num_sessions'] = ss['num_sessions'].astype(int)\n",
- "ss = ss.loc[ss['num_sessions'] <= MAX_SESSIONS_HISTORY, :]\n",
+ "ss = pd.DataFrame(\n",
+ " (behavior.TowersSession & all_subjects_schedule).fetch(\n",
+ " \"KEY\",\n",
+ " order_by=\"subject_fullname, session_date desc, session_number desc\",\n",
+ " as_dict=True,\n",
+ " )\n",
+ ")\n",
+ "ss[\"session_date\"] = ss[\"session_date\"].astype(str)\n",
+ "ss[\"num_sessions\"] = ss.groupby([\"subject_fullname\"])[\"subject_fullname\"].rank(\n",
+ " method=\"first\"\n",
+ ")\n",
+ "ss[\"num_sessions\"] = ss[\"num_sessions\"].astype(int)\n",
+ "ss = ss.loc[ss[\"num_sessions\"] <= MAX_SESSIONS_HISTORY, :]\n",
"\n",
- "#ss2 = ss.copy()\n",
+ "# ss2 = ss.copy()\n",
"\n",
- "max_value_indices = ss.groupby('subject_fullname')['num_sessions'].idxmax()\n",
+ "max_value_indices = ss.groupby(\"subject_fullname\")[\"num_sessions\"].idxmax()\n",
"ss = ss.loc[max_value_indices]\n",
"ss = ss.reset_index(drop=True)\n",
- "ss['query'] = \"(subject_fullname='\" + ss['subject_fullname'] + \"' and session_date >= '\" + ss['session_date'] + \"')\"\n",
+ "ss[\"query\"] = (\n",
+ " \"(subject_fullname='\"\n",
+ " + ss[\"subject_fullname\"]\n",
+ " + \"' and session_date >= '\"\n",
+ " + ss[\"session_date\"]\n",
+ " + \"')\"\n",
+ ")\n",
"\n",
- "block_query = ' OR '.join(ss['query'])\n",
+ "block_query = \" OR \".join(ss[\"query\"])\n",
"ss"
]
},
@@ -1972,33 +2025,48 @@
"metadata": {},
"outputs": [],
"source": [
- "sstable=(acquisition.SessionStarted.proj('local_path_behavior_file', 'session_location'))\n",
- "stable = (acquisition.Session).proj(stimulusBank='stimulus_bank')\n",
- "tstable = (behavior.TowersSession.proj(trialType='rewarded_side',choice='chosen_side', stimulusSet='stimulus_set')) \n",
- "tbtable = (behavior.TowersBlock.proj('first_trial','n_trials', 'sublevel', mazeID='level', mainMazeID='main_level', easyBlockFlag='easy_block',\\\n",
- " duration='block_duration', rewardMil='reward_mil', medianTrialDur='trial_duration_median', start='block_start_time')) \n",
- "table_fetch = sstable * stable* tstable * tbtable\n",
- "\n",
- "allblocks = pd.DataFrame((table_fetch & block_query).fetch(order_by='subject_fullname, session_date desc, session_number desc, block desc',as_dict=True))\n",
- "allblocks['from_DB'] = 1\n",
- "allblocks['session_date'] = allblocks['session_date'].astype(str)\n",
- "\n",
- "allblocks['sublevel'] = allblocks['sublevel'].astype('Int64')\n",
- "allblocks['choice'] = allblocks['choice'].apply(cast_choice)\n",
- "allblocks['trialType'] = allblocks['trialType'].apply(cast_choice)\n",
+ "sstable = acquisition.SessionStarted.proj(\n",
+ " \"local_path_behavior_file\", \"session_location\"\n",
+ ")\n",
+ "stable = (acquisition.Session).proj(stimulusBank=\"stimulus_bank\")\n",
+ "tstable = behavior.TowersSession.proj(\n",
+ " trialType=\"rewarded_side\", choice=\"chosen_side\", stimulusSet=\"stimulus_set\"\n",
+ ")\n",
+ "tbtable = behavior.TowersBlock.proj(\n",
+ " \"first_trial\",\n",
+ " \"n_trials\",\n",
+ " \"sublevel\",\n",
+ " mazeID=\"level\",\n",
+ " mainMazeID=\"main_level\",\n",
+ " easyBlockFlag=\"easy_block\",\n",
+ " duration=\"block_duration\",\n",
+ " rewardMil=\"reward_mil\",\n",
+ " medianTrialDur=\"trial_duration_median\",\n",
+ " start=\"block_start_time\",\n",
+ ")\n",
+ "table_fetch = sstable * stable * tstable * tbtable\n",
"\n",
- "#allblocks['session_date'] = allblocks['session_date'].astype(str)\n",
- "#allblocks['num_blocks'] = allblocks.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')\n",
+ "allblocks = pd.DataFrame(\n",
+ " (table_fetch & block_query).fetch(\n",
+ " order_by=\"subject_fullname, session_date desc, session_number desc, block desc\",\n",
+ " as_dict=True,\n",
+ " )\n",
+ ")\n",
+ "allblocks[\"from_DB\"] = 1\n",
+ "allblocks[\"session_date\"] = allblocks[\"session_date\"].astype(str)\n",
"\n",
- "#all_sessions_blocks = allblocks.drop_duplicates(subset=['subject_fullname','session_date','session_number']).copy()\n",
- "#all_sessions_blocks['num_session_blocks'] = all_sessions_blocks.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')\n",
- "#all_sessions_blocks2 = all_sessions_blocks.copy()\n",
- "#max_value_indices = all_sessions_blocks.groupby('subject_fullname')['num_session_blocks'].idxmax()\n",
- "#all_sessions_blocks = all_sessions_blocks.loc[max_value_indices]\n",
+ "allblocks[\"sublevel\"] = allblocks[\"sublevel\"].astype(\"Int64\")\n",
+ "allblocks[\"choice\"] = allblocks[\"choice\"].apply(cast_choice)\n",
+ "allblocks[\"trialType\"] = allblocks[\"trialType\"].apply(cast_choice)\n",
"\n",
+ "# allblocks['session_date'] = allblocks['session_date'].astype(str)\n",
+ "# allblocks['num_blocks'] = allblocks.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')\n",
"\n",
- "\n",
- " "
+ "# all_sessions_blocks = allblocks.drop_duplicates(subset=['subject_fullname','session_date','session_number']).copy()\n",
+ "# all_sessions_blocks['num_session_blocks'] = all_sessions_blocks.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')\n",
+ "# all_sessions_blocks2 = all_sessions_blocks.copy()\n",
+ "# max_value_indices = all_sessions_blocks.groupby('subject_fullname')['num_session_blocks'].idxmax()\n",
+ "# all_sessions_blocks = all_sessions_blocks.loc[max_value_indices]"
]
},
{
@@ -2421,7 +2489,6 @@
}
],
"source": [
- "\n",
"allblocks.to_csv(PAST_SESSION_PERFORMANCE_FILENAME, index=False)\n",
"allblocks"
]
@@ -2433,33 +2500,42 @@
"outputs": [],
"source": [
"subject_data = wwa.get_subject_data()\n",
- "subject_data = subject_data.loc[:, ['user_id', 'subject_fullname', 'cage', 'headplate_image_path', 'last_weight', 'water_per_day']]\n",
- "#subject_data = subject_data.loc[subject_data['user_id'] != 'testuser',:]\n",
+ "subject_data = subject_data.loc[\n",
+ " :,\n",
+ " [\n",
+ " \"user_id\",\n",
+ " \"subject_fullname\",\n",
+ " \"cage\",\n",
+ " \"headplate_image_path\",\n",
+ " \"last_weight\",\n",
+ " \"water_per_day\",\n",
+ " ],\n",
+ "]\n",
+ "# subject_data = subject_data.loc[subject_data['user_id'] != 'testuser',:]\n",
"subject_data = subject_data.reset_index(drop=True)\n",
"\n",
- "dictionary_rename_cols =\\\n",
- "{'user_id': 'owner',\n",
- " 'subject_fullname': 'subject name',\n",
- " 'cage': 'cage',\n",
- " 'headplate_image_path': 'headplate',\n",
- " 'last_weight': 'last weight',\n",
- " 'water_per_day': 'water per day',\n",
- " }\n",
+ "dictionary_rename_cols = {\n",
+ " \"user_id\": \"owner\",\n",
+ " \"subject_fullname\": \"subject name\",\n",
+ " \"cage\": \"cage\",\n",
+ " \"headplate_image_path\": \"headplate\",\n",
+ " \"last_weight\": \"last weight\",\n",
+ " \"water_per_day\": \"water per day\",\n",
+ "}\n",
"\n",
"subject_data = subject_data.rename(columns=dictionary_rename_cols)\n",
"subject_data = subject_data[list(dictionary_rename_cols.values())]\n",
- "subject_data['headplate'] = subject_data['headplate'].astype(str)\n",
+ "subject_data[\"headplate\"] = subject_data[\"headplate\"].astype(str)\n",
"\n",
- "headplate_image_paths = subject_data['headplate'].copy()\n",
+ "headplate_image_paths = subject_data[\"headplate\"].copy()\n",
"\n",
- "subject_data['last weight'] = subject_data['last weight'].round(1)\n",
- "subject_data['water per day'] = subject_data['water per day'].round(1)\n",
+ "subject_data[\"last weight\"] = subject_data[\"last weight\"].round(1)\n",
+ "subject_data[\"water per day\"] = subject_data[\"water per day\"].round(1)\n",
"\n",
"\n",
- "subject_data['headplate'] = ''\n",
- "subject_data['today weight'] = ''\n",
- "subject_data['today water'] = ''\n",
- "\n"
+ "subject_data[\"headplate\"] = \"\"\n",
+ "subject_data[\"today weight\"] = \"\"\n",
+ "subject_data[\"today water\"] = \"\""
]
},
{
@@ -2469,13 +2545,15 @@
"outputs": [],
"source": [
"# Copy the source file to the destination, replacing if it exists\n",
- "shutil.copy2(WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME)\n",
+ "shutil.copy2(\n",
+ " WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE,\n",
+ " WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME,\n",
+ ")\n",
"\n",
"book = openpyxl.load_workbook(WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME)\n",
- "sheet = book['Sheet1'] # Replace 'Sheet1' with your sheet name\n",
+ "sheet = book[\"Sheet1\"] # Replace 'Sheet1' with your sheet name\n",
"\n",
- "#sheet.delete_rows(3, sheet.max_row - 1) #\n",
- "\n"
+ "# sheet.delete_rows(3, sheet.max_row - 1) #"
]
},
{
@@ -2484,8 +2562,8 @@
"metadata": {},
"outputs": [],
"source": [
- "#for r in dataframe_to_rows(subject_data, index=False, header=False):\n",
- " #sheet.append(r)\n"
+ "# for r in dataframe_to_rows(subject_data, index=False, header=False):\n",
+ "# sheet.append(r)\n"
]
},
{
@@ -2494,7 +2572,9 @@
"metadata": {},
"outputs": [],
"source": [
- "for r_idx, row in enumerate(dataframe_to_rows(subject_data, index=False, header=False), start=3):\n",
+ "for r_idx, row in enumerate(\n",
+ " dataframe_to_rows(subject_data, index=False, header=False), start=3\n",
+ "):\n",
" for c_idx, value in enumerate(row, start=1):\n",
" sheet.cell(row=r_idx, column=c_idx, value=value)"
]
@@ -2505,23 +2585,22 @@
"metadata": {},
"outputs": [],
"source": [
- "column_headplate = subject_data.columns.get_loc('headplate') + 1\n",
+ "column_headplate = subject_data.columns.get_loc(\"headplate\") + 1\n",
"column_headplate = openpyxl.utils.get_column_letter(column_headplate)\n",
"\n",
"\n",
"for i in range(headplate_image_paths.shape[0]):\n",
- "\n",
" headplate_path = headplate_image_paths[i]\n",
- " cell_image = column_headplate+str(i+3)\n",
+ " cell_image = column_headplate + str(i + 3)\n",
"\n",
" if pathlib.Path.is_file(pathlib.Path(headplate_path)):\n",
" img = Image(headplate_path)\n",
" img.width = 60\n",
" img.height = 60\n",
" sheet.add_image(img, cell_image)\n",
- " sheet.row_dimensions[i+3].height = 50 \n",
+ " sheet.row_dimensions[i + 3].height = 50\n",
" else:\n",
- " sheet.row_dimensions[i+3].height = 15"
+ " sheet.row_dimensions[i + 3].height = 15"
]
},
{
diff --git a/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb b/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb
index 8508dbdc..ad4625af 100644
--- a/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb
+++ b/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb
@@ -31,6 +31,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -51,11 +52,13 @@
}
],
"source": [
- "import datajoint as dj\n",
- "import os\n",
"import getpass\n",
+ "import os\n",
+ "\n",
"import datajoint as dj\n",
+ "\n",
"import u19_pipeline.utility as utility\n",
+ "\n",
"dj.__version__"
]
},
@@ -82,9 +85,9 @@
}
],
"source": [
- "dj.config['database.host'] = 'datajoint00.pni.princeton.edu'\n",
- "dj.config['database.user'] = 'alvaros' # enter yo\n",
- "dj.config['database.password'] = getpass.getpass() # enter the password securily"
+ "dj.config[\"database.host\"] = \"datajoint00.pni.princeton.edu\"\n",
+ "dj.config[\"database.user\"] = \"alvaros\" # enter yo\n",
+ "dj.config[\"database.password\"] = getpass.getpass() # enter the password securily"
]
},
{
@@ -135,9 +138,9 @@
"metadata": {},
"outputs": [],
"source": [
- "if os.name == 'nt':\n",
- " if 'C:\\\\Program Files\\\\Graphviz\\\\bin' not in os.environ[\"PATH\"]:\n",
- " os.environ[\"PATH\"] += ';C:\\\\Program Files\\\\Graphviz\\\\bin'"
+ "if os.name == \"nt\":\n",
+ " if \"C:\\\\Program Files\\\\Graphviz\\\\bin\" not in os.environ[\"PATH\"]:\n",
+ " os.environ[\"PATH\"] += \";C:\\\\Program Files\\\\Graphviz\\\\bin\""
]
},
{
@@ -156,8 +159,8 @@
"metadata": {},
"outputs": [],
"source": [
- "if 'custom' not in dj.config:\n",
- " dj.config['custom'] = dict()"
+ "if \"custom\" not in dj.config:\n",
+ " dj.config[\"custom\"] = dict()"
]
},
{
@@ -166,7 +169,7 @@
"metadata": {},
"outputs": [],
"source": [
- "dj.config['custom']['database.prefix'] = 'u19_'"
+ "dj.config[\"custom\"][\"database.prefix\"] = \"u19_\""
]
},
{
@@ -182,13 +185,9 @@
"metadata": {},
"outputs": [],
"source": [
- "location_extstorage = utility.get_network_path('u19_dj') + '/external_dj_blobs'\n",
- "dj.config['stores'] = {\n",
- " 'extstorage':\n",
- " {\n",
- " 'location': location_extstorage,\n",
- " 'protocol': 'file'\n",
- " }\n",
+ "location_extstorage = utility.get_network_path(\"u19_dj\") + \"/external_dj_blobs\"\n",
+ "dj.config[\"stores\"] = {\n",
+ " \"extstorage\": {\"location\": location_extstorage, \"protocol\": \"file\"}\n",
"}"
]
},
diff --git a/notebooks/developer_notebooks/00-Set-up-configuration.ipynb b/notebooks/developer_notebooks/00-Set-up-configuration.ipynb
index f62935c2..3493106f 100644
--- a/notebooks/developer_notebooks/00-Set-up-configuration.ipynb
+++ b/notebooks/developer_notebooks/00-Set-up-configuration.ipynb
@@ -24,10 +24,10 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
- "import os\n",
- "import getpass\n",
"import datajoint as dj\n",
+ "\n",
"import u19_pipeline.utility as utility"
]
},
@@ -96,9 +96,11 @@
"outputs": [],
"source": [
"# If there is only one root path.\n",
- "dj.config['custom']['ephys_root_data_dir'] = utility.get_network_path('Bezos') + '/Manuel/neuropixel/manuel'\n",
+ "dj.config[\"custom\"][\"ephys_root_data_dir\"] = (\n",
+ " utility.get_network_path(\"Bezos\") + \"/Manuel/neuropixel/manuel\"\n",
+ ")\n",
"# If there are multiple possible root paths:\n",
- "#dj.config['custom']['ephys_root_data_dir'] = ['/mnt/bucket/labs/tank']"
+ "# dj.config['custom']['ephys_root_data_dir'] = ['/mnt/bucket/labs/tank']"
]
},
{
@@ -120,7 +122,7 @@
}
],
"source": [
- "dj.config['custom']"
+ "dj.config[\"custom\"]"
]
},
{
diff --git a/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb b/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb
index ad7a7297..ddffd45d 100644
--- a/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb
+++ b/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb
@@ -39,6 +39,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -47,9 +48,7 @@
"execution_count": 2,
"metadata": {},
"outputs": [],
- "source": [
- "import os"
- ]
+ "source": []
},
{
"cell_type": "markdown",
@@ -83,8 +82,9 @@
],
"source": [
"import datajoint as dj\n",
- "from u19_pipeline import subject, acquisition, ephys_pipeline\n",
- "from u19_pipeline.ephys_pipeline import probe_element, ephys_element"
+ "\n",
+ "from u19_pipeline import acquisition, ephys_pipeline, subject\n",
+ "from u19_pipeline.ephys_pipeline import ephys_element, probe_element"
]
},
{
@@ -1090,8 +1090,8 @@
],
"source": [
"# plot diagram of tables in multiple schemas\n",
- "#dj.Diagram(subject) + dj.Diagram(acquisition) + dj.Diagram(ephys_pipeline) + dj.Diagram(ephys_element)\n",
- "#dj.Diagram(acquisition)\n",
+ "# dj.Diagram(subject) + dj.Diagram(acquisition) + dj.Diagram(ephys_pipeline) + dj.Diagram(ephys_element)\n",
+ "# dj.Diagram(acquisition)\n",
"dj.Diagram(ephys_pipeline) + dj.Diagram(ephys_element)"
]
},
@@ -1627,7 +1627,12 @@
],
"source": [
"# plot diagram of selected tables and schemas\n",
- "dj.Diagram(subject.Subject) + dj.Diagram(acquisition.Session) + dj.Diagram(ephys_pipeline) + dj.Diagram(ephys_element)"
+ "(\n",
+ " dj.Diagram(subject.Subject)\n",
+ " + dj.Diagram(acquisition.Session)\n",
+ " + dj.Diagram(ephys_pipeline)\n",
+ " + dj.Diagram(ephys_element)\n",
+ ")"
]
},
{
@@ -1996,7 +2001,11 @@
],
"source": [
"# subject, session, and ephys session\n",
- "dj.Diagram(subject.Subject) + dj.Diagram(acquisition.Session) + dj.Diagram(ephys_pipeline.EphysPipelineSession)"
+ "(\n",
+ " dj.Diagram(subject.Subject)\n",
+ " + dj.Diagram(acquisition.Session)\n",
+ " + dj.Diagram(ephys_pipeline.EphysPipelineSession)\n",
+ ")"
]
},
{
diff --git a/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb b/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb
index 2d0e427d..c9bceae2 100644
--- a/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb
+++ b/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb
@@ -24,7 +24,8 @@
"outputs": [],
"source": [
"import os\n",
- "os.chdir('../..')"
+ "\n",
+ "os.chdir(\"../..\")"
]
},
{
@@ -44,11 +45,11 @@
"source": [
"import datajoint as dj\n",
"\n",
- "# modules within original u19_pipeline\n",
- "from u19_pipeline import lab, subject, acquisition, imaging\n",
- "\n",
"# modules defined in DataJoint Calcium Imaging element, installed to u19_pipeline\n",
- "from u19_pipeline.imaging_element import scan_element, imaging_element"
+ "from u19_pipeline.imaging_element import imaging_element, scan_element\n",
+ "\n",
+ "# modules within original u19_pipeline\n",
+ "from u19_pipeline import acquisition, subject"
]
},
{
@@ -2136,7 +2137,12 @@
],
"source": [
"# plot diagram of tables in multiple schemas\n",
- "dj.Diagram(subject) + dj.Diagram(acquisition) + dj.Diagram(scan_element) + dj.Diagram(imaging_element)"
+ "(\n",
+ " dj.Diagram(subject)\n",
+ " + dj.Diagram(acquisition)\n",
+ " + dj.Diagram(scan_element)\n",
+ " + dj.Diagram(imaging_element)\n",
+ ")"
]
},
{
@@ -2682,7 +2688,11 @@
],
"source": [
"# plot diagram of selected tables and schemas\n",
- "dj.Diagram(subject.Subject) + dj.Diagram(acquisition.Session) + dj.Diagram(imaging_element)"
+ "(\n",
+ " dj.Diagram(subject.Subject)\n",
+ " + dj.Diagram(acquisition.Session)\n",
+ " + dj.Diagram(imaging_element)\n",
+ ")"
]
},
{
@@ -2883,7 +2893,11 @@
],
"source": [
"# subject, session and scan\n",
- "dj.Diagram(subject.Subject) + dj.Diagram(acquisition.Session) + dj.Diagram(scan_element.Scan)"
+ "(\n",
+ " dj.Diagram(subject.Subject)\n",
+ " + dj.Diagram(acquisition.Session)\n",
+ " + dj.Diagram(scan_element.Scan)\n",
+ ")"
]
},
{
diff --git a/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb b/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb
index ba0ddf88..943864e0 100644
--- a/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb
+++ b/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb
@@ -2,13 +2,14 @@
"cells": [
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"# Run ephys element workflow"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"This notebook walks you through the steps to run the ephys workflow. \n",
"The workflow requires neuropixels meta file and kilosort output data. \n",
@@ -39,142 +40,129 @@
"\n",
"\n",
"Let's start by importing the relevant modules."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": 1,
- "source": [
- "from scripts.conf_file_finding import try_find_conf_file\r\n",
- "try_find_conf_file()"
- ],
+ "metadata": {},
"outputs": [
{
- "output_type": "stream",
"name": "stdout",
+ "output_type": "stream",
"text": [
"Local configuration file found !!, no need to run the configuration (unless configuration has changed)\n"
]
}
],
- "metadata": {}
+ "source": [
+ "from scripts.conf_file_finding import try_find_conf_file\r\n",
+ "\r\n",
+ "try_find_conf_file()\r"
+ ]
},
{
"cell_type": "code",
"execution_count": 2,
- "source": [
- "import datetime\r\n",
- "import pathlib\r\n",
- "import numpy as np\r\n",
- "import pylab as pl\r\n",
- "from u19_pipeline import acquisition, ephys\r\n",
- "from u19_pipeline.ephys import ephys_element, probe_element, get_session_directory\r\n",
- "import datajoint as dj"
- ],
+ "metadata": {},
"outputs": [
{
- "output_type": "stream",
"name": "stdout",
+ "output_type": "stream",
"text": [
"Connecting alvaros@datajoint00.pni.princeton.edu:3306\n"
]
}
],
- "metadata": {}
+ "source": [
+ "import datetime\r\n",
+ "\r\n",
+ "import datajoint as dj\r\n",
+ "import numpy as np\r\n",
+ "import pylab as pl\r\n",
+ "from u19_pipeline.ephys import ephys_element, get_session_directory, probe_element\r\n",
+ "\r\n",
+ "from u19_pipeline import acquisition, ephys\r"
+ ]
},
{
"cell_type": "code",
"execution_count": 9,
- "source": [
- "dj.conn()"
- ],
+ "metadata": {},
"outputs": [
{
- "output_type": "execute_result",
"data": {
"text/plain": [
"DataJoint connection (connected) alvaros@datajoint00.pni.princeton.edu:3306"
]
},
+ "execution_count": 9,
"metadata": {},
- "execution_count": 9
+ "output_type": "execute_result"
}
],
- "metadata": {}
+ "source": [
+ "dj.conn()\r"
+ ]
},
{
"cell_type": "code",
"execution_count": 3,
- "source": [
- "key = {\r\n",
- " 'subject_fullname': 'ms81_M005',\r\n",
- " 'session_date': datetime.date(2021, 5, 5)}"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "key = {\"subject_fullname\": \"ms81_M005\", \"session_date\": datetime.date(2021, 5, 5)}\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"The module ephys_element contains all the tables designed in the DataJoint ephys element, we could draw the diagram to see the schema structures"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": 11,
- "source": [
- "dj.Diagram(acquisition.Session) + ephys.EphysSession + dj.Diagram(ephys_element)"
- ],
+ "metadata": {},
"outputs": [
{
- "output_type": "execute_result",
"data": {
+ "image/svg+xml": "",
"text/plain": [
""
- ],
- "image/svg+xml": ""
+ ]
},
+ "execution_count": 11,
"metadata": {},
- "execution_count": 11
+ "output_type": "execute_result"
}
],
- "metadata": {}
+ "source": [
+ "dj.Diagram(acquisition.Session) + ephys.EphysSession + dj.Diagram(ephys_element)\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"## Ingest Probe and ProbeInsertion by ephys_element_ingest"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"The original U19 pipeline contains a table `ephys.EphysSession` where datapath to the neuropixel meta file and the kilosort output folder were stored."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": 4,
- "source": [
- "ephys.EphysSession() # Current sessions"
- ],
+ "metadata": {},
"outputs": [
{
- "output_type": "execute_result",
"data": {
- "text/plain": [
- "*subject_fulln *session_date *session_numbe ephys_director\n",
- "+------------+ +------------+ +------------+ +------------+\n",
- "hnieh_E105 2021-01-15 0 /Volumes/Tank/\n",
- "ms81_M004 2021-05-07 0 /Users/ms81/pr\n",
- "ms81_M005 2021-05-05 0 /M005/20210505\n",
- "ms81_M005 2021-05-06 0 /M005/20210506\n",
- " (Total: 4)"
- ],
"text/html": [
"\n",
" \n",
@@ -259,190 +247,218 @@
" \n",
" Total: 4
\n",
" "
+ ],
+ "text/plain": [
+ "*subject_fulln *session_date *session_numbe ephys_director\n",
+ "+------------+ +------------+ +------------+ +------------+\n",
+ "hnieh_E105 2021-01-15 0 /Volumes/Tank/\n",
+ "ms81_M004 2021-05-07 0 /Users/ms81/pr\n",
+ "ms81_M005 2021-05-05 0 /M005/20210505\n",
+ "ms81_M005 2021-05-06 0 /M005/20210506\n",
+ " (Total: 4)"
]
},
+ "execution_count": 4,
"metadata": {},
- "execution_count": 4
+ "output_type": "execute_result"
}
],
- "metadata": {}
+ "source": [
+ "ephys.EphysSession() # Current sessions\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"# Do you have a new one to insert? This is how it is done:\r\n",
"\r\n",
"key = {\r\n",
- " 'subject_fullname': 'ms81_M005',\r\n",
- " 'session_date': datetime.date(2021, 5, 5),\r\n",
- " 'session_number': 0,\r\n",
- " 'ephys_directory': '/Users/ms81/project_Neuropixel/analyses/data/210507_M004/'\r\n",
+ " \"subject_fullname\": \"ms81_M005\",\r\n",
+ " \"session_date\": datetime.date(2021, 5, 5),\r\n",
+ " \"session_number\": 0,\r\n",
+ " \"ephys_directory\": \"/Users/ms81/project_Neuropixel/analyses/data/210507_M004/\",\r\n",
"}\r\n",
"\r\n",
- "ephys.EphysSession.insert1(key, skip_duplicates=True) # insert1 only works for one entry\r\n",
- "ephys.EphysSession()"
- ],
- "outputs": [],
- "metadata": {}
+ "ephys.EphysSession.insert1(\r\n",
+ " key, skip_duplicates=True\r\n",
+ ") # insert1 only works for one entry\r\n",
+ "ephys.EphysSession()\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "dj.Diagram(ephys.EphysSession) + dj.Diagram(probe_element.Probe) + dj.Diagram(ephys_element.ProbeInsertion)"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "(\r\n",
+ " dj.Diagram(ephys.EphysSession)\r\n",
+ " + dj.Diagram(probe_element.Probe)\r\n",
+ " + dj.Diagram(ephys_element.ProbeInsertion)\r\n",
+ ")\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"A module `ephys_element_ingest` was provided to process a ephys session based on the neuropixel meta file: ingest entries into tables `Probe` and `ProbeInsertion`"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "from u19_pipeline.ingest import ephys_element_ingest"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "from u19_pipeline.ingest import ephys_element_ingest\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "for sess_key in ephys.EphysSession.fetch('KEY'):\r\n",
- " ephys_element_ingest.process_session(sess_key)"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "for sess_key in ephys.EphysSession.fetch(\"KEY\"):\r\n",
+ " ephys_element_ingest.process_session(sess_key)\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "probe_element.Probe()"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "probe_element.Probe()\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.ProbeInsertion()"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.ProbeInsertion()\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"## Populate EphysRecording"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"By populating `ephys_element.EphysRecording`, three tables will be ingested:\n",
"+ `probe_element.EelectrodeConfig` table contains the configuration information of the electrodes used, i.e. which 384 electrodes out of the total 960 on the probe were used in this ephys session\n",
"+ `ephys_element.EphysRecording` table specifies which ElectrodeConfig is used in a particular ephys session\n",
"+ `ephys_element.EphysRecording.EphysFile` table saves the file path of SpikeGLX meta file."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "dj.Diagram(ephys_element.ProbeInsertion) + probe_element.ElectrodeConfig\\\r\n",
- "+ ephys_element.EphysRecording + ephys_element.EphysRecording.EphysFile"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "(\r\n",
+ " dj.Diagram(ephys_element.ProbeInsertion)\r\n",
+ " + probe_element.ElectrodeConfig\r\n",
+ " + ephys_element.EphysRecording\r\n",
+ " + ephys_element.EphysRecording.EphysFile\r\n",
+ ")\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.EphysRecording()"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.EphysRecording()\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.EphysRecording.populate(display_progress=True)"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.EphysRecording.populate(display_progress=True)\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"Here is an overview of the Electrode used in a EphysRecording for a particular probe insertion"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "probe_insertion_key = ephys_element.ProbeInsertion.fetch('KEY', limit=1)[0]\r\n",
- "ephys_element.EphysRecording * probe_element.ElectrodeConfig.Electrode & probe_insertion_key"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "probe_insertion_key = ephys_element.ProbeInsertion.fetch(\"KEY\", limit=1)[0]\r\n",
+ "ephys_element.EphysRecording * probe_element.ElectrodeConfig.Electrode & probe_insertion_key\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.EphysRecording.EphysFile()"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.EphysRecording.EphysFile()\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"Note that the filepath is a relative path to the `ephys_root_data_dir` in `dj.config['custom']`"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"## Populate clustering results"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"The next major table in the ephys pipeline is the `ClusteringTask`, which is a manual table that is inserted when a Kilosort2 clustering task is finished and the clustering results are ready for processing. The `ClusteringTask` table depends on the table `ClusteringParamSet`, which are the parameters of the clustering task and needed to be inserted first. A method of the class `ClusteringParamSet` called `insert_new_params` helps on the insertion of params_set"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "dj.Diagram(ephys_element.EphysRecording) + ephys_element.ClusteringParamSet + ephys_element.ClusteringTask"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "(\r\n",
+ " dj.Diagram(ephys_element.EphysRecording)\r\n",
+ " + ephys_element.ClusteringParamSet\r\n",
+ " + ephys_element.ClusteringTask\r\n",
+ ")\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"# insert clustering task manually\r\n",
"params_ks = {\r\n",
@@ -467,277 +483,288 @@
" \"nSkipCov\": 25,\r\n",
" \"scaleproc\": 200,\r\n",
" \"nPCs\": 3,\r\n",
- " \"useRAM\": 0\r\n",
+ " \"useRAM\": 0,\r\n",
"}\r\n",
"ephys_element.ClusteringParamSet.insert_new_params(\r\n",
- " 'kilosort2', 0, 'Spike sorting using Kilosort2', params_ks)\r\n",
- "ephys_element.ClusteringParamSet()"
- ],
- "outputs": [],
- "metadata": {}
+ " \"kilosort2\", 0, \"Spike sorting using Kilosort2\", params_ks\r\n",
+ ")\r\n",
+ "ephys_element.ClusteringParamSet()\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"We are then able to insert an entry into the `ClusteringTask` table. One important field of the table is `clustering_output_dir`, which specifies the Kilosort2 output directory for the later processing. For the current pipeline, the directory could be reconstructed from directories stored in existing tables"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "dj.Diagram(ephys_element.EphysRecording) + ephys_element.ClusteringTask + ephys_element.Clustering"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "(\r\n",
+ " dj.Diagram(ephys_element.EphysRecording)\r\n",
+ " + ephys_element.ClusteringTask\r\n",
+ " + ephys_element.Clustering\r\n",
+ ")\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.ClusteringTask.describe();"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.ClusteringTask.describe();\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"# Specify path and animal information\r\n",
"\r\n",
"ephys_key = dict(\r\n",
- " subject_fullname='ms81_M004', session_date='2021-05-07', session_number=0, insertion_number=0)\r\n",
- "ephys_dir = '210507_M004/towersTask_g0_imec0/'"
- ],
- "outputs": [],
- "metadata": {}
+ " subject_fullname=\"ms81_M004\",\r\n",
+ " session_date=\"2021-05-07\",\r\n",
+ " session_number=0,\r\n",
+ " insertion_number=0,\r\n",
+ ")\r\n",
+ "ephys_dir = \"210507_M004/towersTask_g0_imec0/\"\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"ephys_element.ClusteringTask.insert1(\r\n",
- " dict(**ephys_key, paramset_idx=0, clustering_output_dir=ephys_dir), skip_duplicates=True)\r\n",
- "ephys_element.ClusteringTask()"
- ],
- "outputs": [],
- "metadata": {}
+ " dict(**ephys_key, paramset_idx=0, clustering_output_dir=ephys_dir),\r\n",
+ " skip_duplicates=True,\r\n",
+ ")\r\n",
+ "ephys_element.ClusteringTask()\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"We are then able to populate the clustering results. The `Clustering` table now validates the Kilosort2 outcomes. In the future release of elements-ephys, this table will be used to trigger Kilosort2."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.Clustering.populate(display_progress=True)"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.Clustering.populate(display_progress=True)\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"# Curation and CuratedClustering"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"The next step in the pipeline is the curation of spike sorting results. If a manual curation was implemented, an entry needs to be manually inserted into the table `Curation`, which specifies the directory to the curated results in `curation_output_dir`. If we would like to process the Kilosort2 outcome directly, an entry is also needed in `Curation`. A method `create1_from_clustering_task` was provided to help this insertion. It copies the `clustering_output_dir` in `ClusteringTask` to the field `curation_output_dir` in the table `Curation` with a new `curation_id`."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "dj.Diagram(ephys_element.Clustering) + ephys_element.Curation + ephys_element.CuratedClustering \\\r\n",
- "+ ephys_element.CuratedClustering.Unit"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "(\r\n",
+ " dj.Diagram(ephys_element.Clustering)\r\n",
+ " + ephys_element.Curation\r\n",
+ " + ephys_element.CuratedClustering\r\n",
+ " + ephys_element.CuratedClustering.Unit\r\n",
+ ")\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "key = (ephys_element.ClusteringTask & ephys_key).fetch1('KEY')\r\n",
- "ephys_element.Curation().create1_from_clustering_task(key)"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "key = (ephys_element.ClusteringTask & ephys_key).fetch1(\"KEY\")\r\n",
+ "ephys_element.Curation().create1_from_clustering_task(key)\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.Curation()"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.Curation()\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"Then we could populate table `CuratedClustering`, ingesting either the output of Kilosort2 or the curated results."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.CuratedClustering.populate(display_progress=True)"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.CuratedClustering.populate(display_progress=True)\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"The part table `CuratedClustering.Unit` contains the spike sorted units"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.CuratedClustering.Unit()"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.CuratedClustering.Unit()\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"## Populate LFP and waveform"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.LFP.populate(display_progress=True)"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.LFP.populate(display_progress=True)\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys_element.LFP()"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys_element.LFP()\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"The current workflow also contain tables to save spike waveforms:\n",
"\n",
"`WaveformSet`: a table to drive the processing of all spikes waveforms resulting from a CuratedClustering. \n",
"`WaveformSet.Waveform`: mean waveform across spikes for a given unit and electrode. \n",
"`WaveformSet.PeakWaveform`: mean waveform across spikes for a given unit at the electrode with peak spike amplitude."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"# May take a while to populate depending on data size.\r\n",
- "ephys_element.WaveformSet.populate(display_progress=True)"
- ],
- "outputs": [],
- "metadata": {}
+ "ephys_element.WaveformSet.populate(display_progress=True)\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"# Synchronize ePhys and Behavior"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"Populate the tables, using the nidaq file produced by spikeGLX. This is done automatically and produces a record of VR iteration numbers measured in the time of the ePhys setup."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys.BehaviorSync()"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys.BehaviorSync()\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "dj.config"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "dj.config\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "get_session_directory(key)"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "get_session_directory(key)\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys.BehaviorSync.populate(key)"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys.BehaviorSync.populate(key)\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "ephys.BehaviorSync()"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "ephys.BehaviorSync()\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"Once the synchronization files have been produces, the all Clusters/spikes can be binned into iterations."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": 7,
- "source": [
- "ephys.CuratedClustersIteration.populate(key)"
- ],
+ "metadata": {},
"outputs": [
{
- "output_type": "stream",
"name": "stdout",
+ "output_type": "stream",
"text": [
"\\\\bucket.pni.princeton.edu\\Bezos-center\\Manuel\\neuropixel\\manuel\n",
"\n",
@@ -756,32 +783,24 @@
]
}
],
- "metadata": {}
+ "source": [
+ "ephys.CuratedClustersIteration.populate(key)\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"This allows to plot neuronal responses responses. Here is an example"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": 10,
- "source": [
- "ephys.CuratedClustersIteration.Unit & key & 'unit = 7'"
- ],
+ "metadata": {},
"outputs": [
{
- "output_type": "execute_result",
"data": {
- "text/plain": [
- "*subject_fulln *session_date *session_numbe *insertion_num *paramset_idx *curation_id *unit spike_coun firing_rate_be firing_rate_af\n",
- "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------+ +--------+ +------------+ +------------+\n",
- "ms81_M005 2021-05-05 0 0 0 1 7 =BLOB= 0.321505 3.72292 \n",
- "ms81_M005 2021-05-05 0 1 0 1 7 =BLOB= 6.21089 2.39695 \n",
- " (Total: 2)"
- ],
"text/html": [
"\n",
" \n",
@@ -890,63 +909,70 @@
" \n",
" Total: 2
\n",
" "
+ ],
+ "text/plain": [
+ "*subject_fulln *session_date *session_numbe *insertion_num *paramset_idx *curation_id *unit spike_coun firing_rate_be firing_rate_af\n",
+ "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------+ +--------+ +------------+ +------------+\n",
+ "ms81_M005 2021-05-05 0 0 0 1 7 =BLOB= 0.321505 3.72292 \n",
+ "ms81_M005 2021-05-05 0 1 0 1 7 =BLOB= 6.21089 2.39695 \n",
+ " (Total: 2)"
]
},
+ "execution_count": 10,
"metadata": {},
- "execution_count": 10
+ "output_type": "execute_result"
}
],
- "metadata": {}
+ "source": [
+ "ephys.CuratedClustersIteration.Unit & key & \"unit = 7\"\r"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
- "source": [
- "key = {\"subject_fullname\": \"ms81_M004\", \"insertion_number\": 0, \"session_date\": '2021-05-07', 'curation_id': 1}\n",
- "\n",
- "ephys.BehaviorSync() & key"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "key = {\r\n",
+ " \"subject_fullname\": \"ms81_M004\",\r\n",
+ " \"insertion_number\": 0,\r\n",
+ " \"session_date\": \"2021-05-07\",\r\n",
+ " \"curation_id\": 1,\r\n",
+ "}\r\n",
+ "\r\n",
+ "ephys.BehaviorSync() & key\r"
+ ]
},
{
"cell_type": "code",
"execution_count": 11,
- "source": [
- "spikes = (ephys.CuratedClustersIteration.Unit & key & 'insertion_number = 0' & 'unit = 7').fetch1('spike_counts_iteration')\n",
- "\n",
- "behavior = dj.create_virtual_module('subject', 'u19_behavior')\n",
- "pos, time = (behavior.TowersBlock().Trial() & key).fetch('position', 'trial_time')\n",
- "\n",
- "position_at_each_iteration = []\n",
- "for trial in range(len(pos)):\n",
- " pp = pos[trial][:,1]\n",
- " snip = np.zeros(len(time[trial]))*np.NaN\n",
- " snip[0:len(pp)] = pp\n",
- " position_at_each_iteration = np.append(position_at_each_iteration, snip)"
- ],
+ "metadata": {},
"outputs": [],
- "metadata": {}
+ "source": [
+ "spikes = (\r\n",
+ " ephys.CuratedClustersIteration.Unit & key & \"insertion_number = 0\" & \"unit = 7\"\r\n",
+ ").fetch1(\"spike_counts_iteration\")\r\n",
+ "\r\n",
+ "behavior = dj.create_virtual_module(\"subject\", \"u19_behavior\")\r\n",
+ "pos, time = (behavior.TowersBlock().Trial() & key).fetch(\"position\", \"trial_time\")\r\n",
+ "\r\n",
+ "position_at_each_iteration = []\r\n",
+ "for trial in range(len(pos)):\r\n",
+ " pp = pos[trial][:, 1]\r\n",
+ " snip = np.zeros(len(time[trial])) * np.NaN\r\n",
+ " snip[0 : len(pp)] = pp\r\n",
+ " position_at_each_iteration = np.append(position_at_each_iteration, snip)\r"
+ ]
},
{
"cell_type": "code",
"execution_count": 15,
- "source": [
- "pl.figure(figsize=(15,5))\n",
- "pl.subplot(1,2,1)\n",
- "pl.plot(spikes)\n",
- "pl.plot(position_at_each_iteration/50)\n",
- "pl.ylim([0,10])\n",
- "pl.xlim([0,30000])\n",
- "\n",
- "pl.subplot(1,2,2)\n",
- "pl.plot(position_at_each_iteration[1:], spikes, '.')"
- ],
+ "metadata": {},
"outputs": [
{
- "output_type": "error",
"ename": "ValueError",
"evalue": "x and y must have same first dimension, but have shapes (470786,) and (470789,)",
+ "output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
@@ -959,22 +985,33 @@
]
},
{
- "output_type": "display_data",
"data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAA28AAAEzCAYAAACmHDtZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABIY0lEQVR4nO3deZwkdX3/8fene46dWRb2YDncBQFZRKIcugIKMRBALhM0UYN4B+VnPKImJmK8o4nGExGUEG9jxAsVZUEU5JZjl3MXWHZZlt1l72N27unr+/ujqnt6evqo7q6uPub1fDzm0d3V1d/6VlX3dH368z3MOScAAAAAQGuLNbsCAAAAAIDKCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2kDF4M3MvmNm281sZd6y+Wb2ezNb49/Oa2w1AQBoDcW+FwueNzO73MzWmtkjZvbiqOsIAOhMQTJv35N0TsGySyXd7JxbIulm/zEAADPB9zT9ezHfuZKW+H+XSPpmBHUCAMwAFYM359ztknYXLL5A0vf9+9+X9OpwqwUAQGsq8b2Y7wJJP3CeeyTNNbODo6kdAKCT1drn7UDn3BZJ8m8PCK9KAAC0tUWSNuY93uQvAwCgLl2N3oCZXSKv2Yhmz579kqOPPrrRmwQANNmKFSt2OucWNrseTWJFlrlpK/H9CAAzUj3fkbUGb9vM7GDn3Ba/Kcj2Uis6566WdLUkLV261C1fvrzGTQIA2oWZPdPsOjTRJkmH5D1eLGlz4Up8PwLAzFTPd2StzSavk/RW//5bJf261goAANBhrpP0Fn/UyZMl7c12NQAAoB4VM29m9mNJp0na38w2SfqkpM9L+qmZXSxpg6TXNbKSAAC0ihLfi92S5Jy7StIySedJWitpVNLbm1NTAECnqRi8OefeUOKpM0KuCwAALa/M92L2eSfpPRFVBwAwg9TabBIAAAAAECGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANlBX8GZmHzSzVWa20sx+bGazwqoYAACtyMzOMbPVZrbWzC4t8vx+ZvYbM3vY/458ezPqCQDoPDUHb2a2SNI/SlrqnHuhpLikC8OqGAAArcbM4pKulHSupGMkvcHMjilY7T2SHnPOHSfpNElfNrOeSCsKAOhI9Tab7JLUZ2Zdkvolba6/SgAAtKwTJa11zq1zziUkXSPpgoJ1nKQ5ZmaS9pG0W1Iq2moCADpRzcGbc+5ZSV+StEHSFkl7nXM3Fa5nZpeY2XIzW75jx47aawoAQPMtkrQx7/Emf1m+KyS9QN4Pmo9Ker9zLlNYEN+PAIBq1dNscp68XxsPl/QcSbPN7E2F6znnrnbOLXXOLV24cGHtNQUAoPmsyDJX8PhsSQ/J+248XtIVZrbvtBfx/QgAqFI9zSbPlPS0c26Hcy4p6VpJLw+nWgAAtKRNkg7Je7xY07sMvF3Stc6zVtLTko6OqH4AgA5WT/C2QdLJZtbvt+s/Q9Lj4VQLAICWdL+kJWZ2uD8IyYWSritYZ4O870SZ2YGSni9pXaS1BAB0pK5aX+icu9fMfi7pAXkdsR+UdHVYFQMAoNU451Jm9l5Jv5M3yvJ3nHOrzOxd/vNXSfqMpO+Z2aPymll+2Dm3s2mVBgB0jJqDN0lyzn1S0idDqgsAAC3PObdM0rKCZVfl3d8s6ZVR1wsA0PnqnSoAAAAAABABgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AZaO3hbf5f0i3dKv3q39Ov3St9+pbT+zubVJ52Sfv9JaddT4Za75WHpZ2+X9qwPr8zdT0ubH5Qy6fDKBAAAANA0Xc2uQFn3/be0+gZp9kJp8FlvWWKkOXUZ2iZ9+Sjv/iEnSQueF17Z626VVl0rnffF8Mq855ve8fvYDikWD69cAAAAAE1RV+bNzOaa2c/N7Akze9zMXhZWxSRJu9dJR5wmfXDV5LLFLw11E2UNbJQuO1a69pLJwE2Sjjo73O3sekrqXyDN3j+8Mu/7b++2qye8MgEAMrNzzGy1ma01s0tLrHOamT1kZqvM7Lao6wgA6Ez1Zt6+JulG59xrzaxHUn8IdZo0slM6+DjJTDrlA9LeTVL//FA3Mc3DP5E23C391dekO78qDTzj/WVdujH8TNb2x6QDjgmvvHTSuz3kpPDKBADIzOKSrpR0lqRNku43s+ucc4/lrTNX0jckneOc22BmBzSlsgCAjlNz8GZm+0p6haS3SZJzLiEpEU61fGN7pD4/WDvr06EWXdIvL/Fuz/y0tPzbk8vfc5+08Pnhby+TkbY9Jp3wpvDK3OMHmy95e3hlAgAk6URJa51z6yTJzK6RdIGkx/LWuUjStc65DZLknNseeS0BAB2pnmaTR0jaIem7ZvagmX3LzGaHVC8pMSqlxqW+eaEVWVEqL/Zcf4d3O/8I6UNrGhO4SV5fvuSIdMDR4ZW5a613u+DI8MoEAEjSIkkb8x5v8pflO0rSPDO71cxWmNlbIqsdAKCj1RO8dUl6saRvOudOkDQiaVrbfzO7xMyWm9nyHTt2BC99bI932+hmkvn25n0f/8TPhJ39OWmfBrZ42fmkd7tgSXhl5oK3EAdVAQBIkhVZ5goed0l6iaTzJZ0t6eNmdlThi2r+fgQAzFj1BG+bJG1yzt3rP/65vGBuCufc1c65pc65pQsXLgxeejZ4izLzNrJz+rIj/qKx28xOO7B/yMFb37xoA18AmBk2STok7/FiSZuLrHOjc27EObdT0u2SjissqObvRwDAjFVznzfn3FYz22hmz3fOrZZ0hqa2+a/P2G7vti/CAGTUD94uudUbQCSdkLr7GrvNoc1SrFva58Dwyty9TppP1g0AGuB+SUvM7HBJz0q6UF4ft3y/lnSFmXVJ6pF0kqSvRlpLAEBHqne0yfdJ+pE/0uQ6SeGNkNGMzNvoLu+2f3+pq9f7i2Kb/fO9ETXDsme9dMiJ4ZUHAJAkOedSZvZeSb+TFJf0HefcKjN7l//8Vc65x83sRkmPSMpI+pZzbmXzag0A6BR1BW/OuYckLQ2nKgVG/cxblE3/ss0m+xdEt83R3eFuL5ORBjdL+y0Or0wAQI5zbpmkZQXLrip4/EVJX4yyXgCAzlfXJN0N1azMW1ef1BPudHUVtxlm8DayXcokCd4AAACADtPCwdtuqWtW4/uc5RvdLc3eP7rtZbcZZnZx7ybvdl+CNwAAAKCTtHDwtifarJvkDVgS9QiNo7vCHZQlG7ztVzjtEAAAAIB21rrB2/headbcaLc5ussbrCQqmYyXYQyz2eT4Xu826sAXAAAAQEO1ePC2b7TbHNkZ7WAl4wOSy4S7zeSod9sdYb89AAAAAA3XwsHboDRrv2i3GfbIj0G2J4W7zcSId9szO7wyAQAAADRd6wZvE4NSb4SZt9SElBiSZkcYvI01YDqE5KhkcSneE16ZAAAAAJqudYO3qJtN5ibojjLzlt1miMFbYtTLuoU56TcAAACApmvN4M256JtN5gKpCAcsaUTAmByhvxsAAADQgVozeEuOeRNNR9lscmSnd9uMzFuYUwUkRqKdZBwAAABAJFozeJsY9G6b0Wwyykm6x/dKFpN654RXZrbZJAAAAICO0prB23g2eJsb3TYbMfJjJRNDXuAWZv+05IjUTfAGAAAAdJoWDd78iaajbDY5ukuSRTu59fig1Btyv77EKM0mAQAAgA7UmsHbhB+8RdlscmJI6tlHisUj3OZg+PuYHGXAEgAAAKADtWbwlms2GeFok4nh6PuKZZtNhikxQp83AAAAoAO1aPDWhGaTzRilcXxv+PuYYKoAAAAAoBO1ZvA2MeTdhp2VKqcZGauJocY0myTzBgAAAHSc1g7eevaJbpvJkWi3J3l93sLMvGUyBG8AAABAh2rN4C0x7A8eEmH1os68OeePNhlidjE15t3SbBIAAADoOK0ZvGVHfoxS1H3FUhNSJhlus8nEqHdL5g0AAADoOK0ZvCWGpd6og7fRaAPGCX9EzTCbTSZHvFsybwAAAEDHac3gbWK4CZm3iKcKGG9A8Jbwgzcm6QYAAAA6TmsGb4nhaEealKKfKiCbeWtIs8mIA18AAAAADdeawVvUmbd0SkpP0GwSAAAAQMtqzeAt6j5v2aCnGc0mG5J5I3gDAAAAOk3rBm9RZsESTchYNWIi8qQfvHUz2iQAAADQaVozeJuIOPOWG+ijzZtNMmAJAAAA0LFaL3hLp7zJpnsiHLAk0YRmkxPD3m2YmbdmZBABAAAARKL1grdENqhpRuYt4tEmu2ZJ8e7wymxG3z0AAAAAkWjd4K0Zfd4ibTY5FP50CIlRKdYlxXvCLRcAAABA07Ve8DbRjMxbNmCMMGPViLnskqPeYCVm4ZYLAAAAoOlaL3jLBVIR9nnLjtIYaZ+3ofAzfVFPNA4AAAAgMq0XvOWG0G/GVAERD1gS5kiTkp95I3gDAAAAOlHrBW/je73bsAObcprRbHJisAF93si8AQAAAJ2q9YK3sT3ebf/86LaZGJUsLnX1RrfNiaHws4uJESboBgAAADpU6wZvffOi22ZixMu6RTnQR6MGLGGaAAAAAKAjtV7wNj7gzX/W3RfdNhPD0Qc9jZoqgGaTAAAAQEdqveBtbE+0WTcp+oxVOimlxsMfUTNJs0kAAACgUxG8SX5fsQgzVrkRNcm8AQAAAAimBYO3geYEb2HPuVZOo6ZDiDoIBQAAABCZFgzempF5i7jPW25qghCDt0xGSo0xYAkAAADQoVozeJs1N9ptRt3cMDspeJiZt+Sod0vmDQAAAOhIrRm89c2NdptRN5tsROYtG7yReQMAAAA6UmsFb8lxLwjp9GaTE9ngLcRtZrN5BG8AAABAR2qt4G18wLvt9KkCcoEWzSYBAAAABFN38GZmcTN70Mx+W3dtxvZ4t1EGb6mElE5EOz9aI5pNknkDgEiY2TlmttrM1prZpWXWe6mZpc3stVHWDwDQucLIvL1f0uMhlNOc4C3ZhKAn0cBmk2TeAKBhzCwu6UpJ50o6RtIbzOyYEuv9l6TfRVtDAEAnqyt4M7PFks6X9K1QatOM4K0ZGavEiGQxqbsvvDJzA5YQvAFAA50oaa1zbp1zLiHpGkkXFFnvfZJ+IWl7lJUDAHS2ejNvl0n6V0mZ+quiJgVvTRilMTu6pVmIZWb3I8JRMwFg5lkkaWPe403+shwzWyTpNZKuirBeAIAZoObgzcxeJWm7c25FhfUuMbPlZrZ8x44d5QttSvDWgCaMlUwMhb+9JM0mASACxX51cwWPL5P0YedcumxB1Xw/AgCg+jJvp0j6azNbL6/ZyF+a2f8WruScu9o5t9Q5t3ThwoXlSxzbI1lc6p1TR7Wq1Kxmk2FnyBI0mwSACGySdEje48WSNhess1TSNf7342slfcPMXl1YUFXfjwAAqI7gzTn3EefcYufcYZIulHSLc+5NddVmbI+XdQuzOWEluSH2ow7eQt5ebsASRpsEgAa6X9ISMzvczHrkff9dl7+Cc+5w59xh/vfjzyW92zn3q8hrCgDoOF3NrsAU2eAtSs1oNpkYDj/zlhyRYl1SV0+45QIAcpxzKTN7r7xRJOOSvuOcW2Vm7/Kfp58bAKBhQgnenHO3Srq17oKaErw1aaqAOQeHXOYoWTcAiIBzbpmkZQXLigZtzrm3RVEnAMDMEMY8b+EZ2yP1zY12m00bbbIBA5YwQTcAAADQsVoseBuQZs2NdptNGW1yuAF93kYZrAQAAADoYK0VvI3vbULmze8rFo+wr1hiROoJeUTN5CjTBAAAAAAdrHWCt0zGC94iz7z5zQ2jGuHSOX/AkgaMNkmzSQAAAKBjtU7wNjEoyUWfeUuORDvQR3JUkmtM8EbmDQAAAOhYrRO8jQ94t7P2i3a7UWessqNb9oY9VQB93gAAAIBO1jrB29iAd9usZpORbS87QErIwRtTBQAAAAAdrXWCt2zmrRlTBUQ90qTEVAEAAAAAqtJCwdte77YZUwU0o9lkIzJvNJsEAAAAOlbrBG/ZZpPNmCogyoE+GhG8ZTJSaoxmkwAAAEAHa53gLTdgydxot5sYlnpDnnOt7PaGvNsws33JUb9MMm8AAABAp2qd4G1sQLJ49P22JqIO3how2mQ2eGOqAAAAAKBjtU7wNj7gNZmMarJsyWtumBgKv/9ZOY1oNknwBgAAAHS81gnexgaibzKZbNCca+VMNKLZ5JhfJsEbAAAA0KlaJ3gb3xv9YCXZYfujbjYZ65a6ekMsk8wbAAAA0OlaKHgbaM5gJZLUE3HwFvocbwRvAAAAQKdrneBtbKAJmbdB7zbKZpOJ4fD72BG8AQAAAB2vdYK38QFp1n7RbjPbbDLSAUuGww8WmSoAAAAA6HitEbw515wBSxJN6vMWdrPJXJ+3vnDLBQAAANAyWiN4SwxLLj0zBiyZGG5gn7eI58gDAAAAEJnWCN7G93q3UWfesn3eop7nLewBUpJk3gAAAIBO1xrB29iAd1sm85ZIZfTLBzfJOVf0+Zsf36adwxPVbTfXbDLqAUvCzrz587wxYAl8yx7doqHxZLOrUdSabUNa8cyeZlcDAACg7bRG8DY+4N2WybxdccsaffAnD+uGlVunPTeWSOvi7y/XW759X3XbnRiWLBZt0NOI4C0xInXNkmKtcTrRXGu2DendP3pA//rzR5pdlaLO+urt+ttv3t3sagAAALSd1rjaz2beyow2uX3Iy6rtHZueTUj72bhndo1Ut93EsNeE0ay619UjMdKA0SbHyLohZzSRliQ9OzDW5JoAAAAgTK0RvGUzb33zot3uxFC0TSYzaa9/WiPmeQs7mwcAAACgpbRG8Dbm938JMNpksS5vpfrBVTQxFP00AVJjRptksBIAAACgo7VI8Dbg9T0rMwpjkJaNVm3zx8Rw9CNNSuFvMzFKs0lMU+tvGgAAAGhNrRG8jQ94/d2iHnBjYjjikSYbFLwlCd4wKcounAAAAIhOawRvYwN1zfFWc4JhYijizNuQd9uI0Sbp8wYAAAB0tBYJ3vYE6u8mSa72UG26xLDUu2945VXcnp95Czvb14jpBwAAAAC0lNYI3sYHAmTeSrcFq7mVWNSjTTZqwJJGTD+AthfqDx0AAABoutYI3sb2SP3zo92mc9EPWDKRbTYZ8jYnIt4PtDSr/ecMAAAAtLDWCd4CzvFWdKqAWraZGpcyqfYfsKQZQSgAAACAyDU/eMtkvAFLKgRvgaYKqGa7E8PebTP6vIXZbDI1Lrk0fd4AAACADtf84G18QJKT+hdEu91Eg5owlt3mcPjbbNT0A2h7zPMGAADQWZofvI3u9m77Iu7zlsu8RRy8dc2S4l3hlikxYAkAAADQ4VogeNvl3QbMvIWWTGjU4CHlNGI+tmwQSrNJFGCybgAAgM7S/OBtzM+89Vfo8xb2dhNN6PM20YD52Gg2CQAAAMwIzQ/ess0m6+jzVlPfnmzmLepmkz1zQi6zCRlEtAX6vAEAAHSWFgje/GaTQfu8lbsirSY914jBQypuswHNJhs18TfaFs0lAQAAOlPzg7ex3VKsS+otn5EK/YK0aZm3BvV5Y8ASAAAAoKM1P3gb3eU1mawnOqup2WSTMm9hB1n0eQMAAABmhBYI3nZHP02A5GXBumdLsXi02ww7yGpG80+0Bfq8AQAAdJbWCN6qGKyk6PVoLUm7iaHomxo2ZLTJYcniUldvuOUCAAAAaCktELztqjhNgCRZ2JMFTAxFn61KjDQg8+aXySgVAAAAQEdrfvA2vE3a56D6yqileVhiuOIgKaFKJ6X0RPjB28Qwg5UAAAAAM0DNwZuZHWJmfzSzx81slZm9v+pCMmlpfK/UH06ft6pyTxMRB2+5vmkNaDbJNAEogi5vQGOY2TlmttrM1prZpUWef6OZPeL/3W1mxzWjngCAzlNP5i0l6Z+dcy+QdLKk95jZMVWVML5XkqtqwJLQBmFIRNxsMjsqZOijTRK8YSpa0AKNY2ZxSVdKOlfSMZLeUOS772lJf+GcO1bSZyRdHW0tAQCdqubgzTm3xTn3gH9/SNLjkhZVVcjobu+2f74e2zwo55xWbd4rVyRCG0umS9fFzzEMjqeCb3tiqGLm7bHNg/rVg88qmc4okcroyW1Dwcsv1KjJtIv0o3PO6aGNA0qkMlq9tY46B7Bm25AmUqXPDcK3euuQUulMs6tRNeecHts8OG35lr1j2j2SiKQO2wbHtXN4ouw6a7YNKZFqv+MbllLnCTknSlrrnFvnnEtIukbSBfkrOOfuds7t8R/eI2lxxHUEAHSoUPq8mdlhkk6QdG+R5y4xs+VmtnzHjh1TnxzzgreHd8V03uV36J0/WKHzL79Tv3jg2Wnb+PmKTWFUdVKFvmKrNu/VeZffoQ/85CG94ep79JnfPqZXfvV2bR4Yq317UmOmCigo8661u/TqK+/Syz9/i86+7Hat3d6YAG7H0ITO+urt+sSvVjWkfEz39M4RnX3Z7frC71aXXKdVpwj43t3rdd7ld0xb/rLP3aKXfPb3kdThpP+8WUs/+4eSz28fHNdZX71dn7xuZST1aUXfvcs7T/eu29XsqrSqRZI25j3epPI/XF4s6YZiT5T9fgQAoIi6gzcz20fSLyR9wDk37eda59zVzrmlzrmlCxcunPqkn3l7ZrRHkvSHx7dJUtUZrppGoqww59qWgfHc/eXP7NH96726Dowmq99WdntSYwYsKcjmPb3T21Y2w7B9qHymoVaD496xyB4bNN4O/1w+uGFPhTVbz6oy2ZxWCTiz7+n7np657+mVm/dKkjbuqfGHqs5X7Aun+Cw2ZqfLC94+XOz5st+PAAAUUVfwZmbd8gK3Hznnrq26AD/zNtE9t55qVC+dlFLjEQ9Y0qBmk8nRymU2+MK4Ra674aPPG9BQmyQdkvd4saTNhSuZ2bGSviXpAuccaUwAQCjqGW3SJH1b0uPOua/UVMiYlz0Y796v1mrUZsLP7JUJ3kIPSBqVeSvW5y3cLZREjNA8rZKpqgbvl/ZSrO8xJEn3S1piZoebWY+kCyVdl7+CmR0q6VpJb3bOPdmEOgIAOlRXHa89RdKbJT1qZg/5y/7NObcscAmjuyWLK9FVZwas2qvCRgVSQbYZ5miTmYwfvDHa5ExBVg2NVlMz9BnEOZcys/dK+p2kuKTvOOdWmdm7/OevkvQJSQskfcP7nVMp59zSZtUZANA5ag7enHN3qt4f08d2S33zqroiLfprcLU/EE80IJCqpBHNJlNjklzFMhv9+zm/0EcvyBHnvKAWjobQFfk/Ui4rWHZV3v13SHpH1PUCAHS+UEabrNmoH7xVIZTLilzmrUyzybAvfLMBY3eIwVuj+tEFZKSBIhfkiLdq5oS3S3vh8w0AQOtpbvA2tlvqDz5Bd2gm/FHvahiwpObrmcSwF7jFQjzkJZp/Flax0QkYfqcHOg+ZWwAAWk+Tg7c9Ul91wVso1xN1NJusefuN6JuWGPVue/qnLGbAks7XjhfWrZoRxFScJwAAWleTm03uqTrzVuySteo+Gs0asCT04K25zSaz2jCOAAAAANpO85tN9s2LPouQy7xF2WxyJPwBUgIGoY0agIAuMc1Trj9Sq56XVq0XiuM3GQAAWk/zgrfkuDfBdLWZtzACvew8b2WCnsKt1L3ZxHBj5niTmp55Q/TasdkkAAAA6tO84G1st3db5WiTxVR9HZsYkuK9UldP3dsObKJ5zSYbP2AJgURUyF4hKrzVAABoPc0L3kazwVszRpscrroJY90XzYmRBmTemtB3Lw8DG6AaBJ4AAAD1aWLmbY93W3WzyRC2HaAJY+F26m822YDgLemPNtndX369BqMFX/Q45Gg03mMAALSeFmg2We1ok9MvKaq+yJgYknr3rfZV9UkMNa7ZZJOCNzIpzdDOB72d6z5z8LkGAKB1Nb/ZZFMm6R6q2Gyy8AKmrgsa5xo02uRI0Ym/o86EkXmLHse8MTiuHAMAAFpZC2TeqhuwpC2bTaYTUibVgMxbsEFQuBbrHO2cFWnnugMAALSC5mbeuvqk7r66i6p62PQaBizJqukCdKJBA4skRpgmYIap5q3eahmUVqtPMQSYHAMAAFpZEzNvAzU1mQzl+m9iqKYJuqUaL0AbNSpkIwZBQdvj4hsAAKAzNbfZZA3TBITXbLJS8BZimqBRk2knRqSeyoOVMKFz52jnwKyd6z4j8W8DAICW09xmk/31T9BdtUzGC96ibDbZ0MwbzSYBAACAmaAlMm9V9eOpd6qAib3ebZUDpdQlG7w1YrTJJg5YQialeYKc02KflWbi7QIAAFCf5mbeagig6m4BmJ0cfNbcOguqQkObTdLnbSYJEgAZYRLCwNsIAICW05zgzTkviGrGHG/Z4C3KzFtutMnmTBXQaPSpawKOORqNtxgAAC2nOcHbxKDk0rlmk/U2v6vqOjZg8DZtnrd6rmRyfd5qG+GydLkjUnflAUsadRFmtJuMXDsf8zau+ozCaQIAoHV1RbkxJ2lwPKk5o7tlklzfPA2OJZXKTI0uJpJpOeeUzjiNpzLqiU/GmM45OeeUTDv1dMVyy8oZmUhJkrrjMfWMDUiSEj1zFEtnZGZKpDJ+/Zy6YjHFTEqXKHNoPKWJVFoxMyXTGfV1x6fUJV8m45R2Tm5sSD2SRq1X8VRaPfGYRhNpzeqOK5HKKOOcuuKWW94VN/V2xUvuz56RhObNiknpCSW7+jU2ntSsrriS6YziMa9e+YYnUhpNpNTfU//pTqQyGk2k1NcTV9I/bomC7Y0mUprVFc8t747HFI95x7krZkplih+vbPndcasYpCRSmSllpPxzmd1OqfKLKXw/5S8fTaTV3xOXmSmd8d6TZt4+TaTS6u2KK5XOvn+kVNp7vrcrpuzbOh4zOeeUSGfUHYsp7ZwyzuXOcfZ4xWLePifTGaUzTrO647l9zTineGzymCTSXl0SqYx6u7wyu+NT65/9XKUz3memq+BzlEw7dcdNybRXt554TD1dMaULPo/Z7ZY6Ttnjn8o49XbFiq7j7dPU14wn07l9zO1XgHM3lkjnPp+z/XNT+HxPVyxX7+x5Kyb/fZPPuan7lN1GqfoVLi+1nnNOA6NJ9XTFNKs7Pu28FEqmM4qb95kp9blIZ5ySfl2z5zBraDzpb8d7nPHvOCfvM+yXn3ZOu0cS6u+JayyZ9o5jMp17L3bFTGPJtObM6i66fxOptH88vZ+5CAABAGiMSIO3lc/u1bGfuknH2lO6rle6+GfrdEvmpmnrff9Pz+jIA+fo479aOe25a+7fqO54TJ+74Qk98PGzNH92jy754YqS2/z6zWv05d8/mXu87GXP6BhJL7/sIe3U04HrvncsKUl63VV/mrK8r9u72PnS647Ta1+yeMpz7/vxg7r+0S36p66H9Z646ZjP3K5ylzUfOHOJLvvDGknSzf/8F3rewun92f7mG3fpgQ0DOuPwWfq2pM/fvFHfvmn6MSyshyT9+j2n6LhD5lbY0/KO+tgN05btHE7k7v9x9Xa9/bv36+iD5uiJrUOSpBccvK+W/eOpOupjN+jYxfvpkU179YO/P1GvOGrhlHLSGaejPnaD3n7KYfrkX/1ZyTrcvXanLvrWvfrZu16mlx7mZW+P/OgNevGhc/XR81+gv/3mn/S/F5+kU5fsH2ifrrl/oz5y7aO688Ona/G8yUzmv//2MX33rvX6xzOW6J/OOkqv/+8/acUzXub2mktO1oVX35O7LfSBM5fof25fp/7eLt3/0TP1v/c8o4//epWO2H+21u30+kD+6B0n6dk9Y/rXXzyiYxfvp+vee6okaclHvWP81H+epye2Dur8y+/Mlfur95wiSXp8y6Ce92/LJEmnHLlAd63dpfWfP1+S9L27vff1uh3edk7/0q3atGdU6z53fq6cL9/0pK7441q97y+P1NdvWZtbvv7z5+uEf79JXfGYJpLpXP0l6dt3Pq3PXv+4VnzsTC3Yp3fK/h7p1/nIA/bR2u3D+u37TtULF+2Xez67T/mO/viNWvsf5+YeP75lUOd+7Q5d9aaX6JwXHjRtfUl6asewzvjybbnH573oIH3jjS/JPR5PpvWCT9yY2xdJuuDKO7Xy2cGi5b3oUzdpv75u3fNvZ0iazLiv2zmS26cPnnmU3n/mEt2xZofe/O379It/eLle8tzJzP3a7cM68yu36WsXHq8Ljl+khzcO6IIr79J33/ZSnX70AVO2984frNAfHt8mSVo4p1fD4yk9/plzitZN8o7bq449WL99ZIv+5ezn6z2nHzltnZP+8w9TPoPZ/b758W26+PvLS5b9xw+dptO/dGuu/EKfvG6VPnndqinL3vKy5+ptLz9Mf/nl2/TVvztOrzlhsa5/ZIve838PlNwOAAAIT1OaTc4zrxnhgCs92MZvHtpcdPmmPWO69oFnJUnbBsclSQ9tHChZzmU3r5n6+s1euXtVXV+x/IujfNlfqW9cOf3i5/pHvWWzNaERzVKl36O/fcdkMPn4luIXmw9sGJAkrXza249RzSpbZr5s4NFIt63eIUm5wE3y9iWb+Hhkkzfa5x1rdkx7bSrjpWZ+dM+Gstu466mdkqR7nto1ZfkDGwZ0z7rdkqQ71+4MXOfr/QvXbLCT9d271kuSvnOnd17yj9/d/rbvWTe1Dlk/vX+jRhJp7RiakCRd97B3vrKBW7aO2fdI9rjkS6YzZd/bWXetnVqHXz049bOzYfeoChNP19zvHePv3b1+WnmD4yntHklMqb+k3Oduy97xknVZu937bAeptyQl05MVe2ST95qb/eCmmCe2DE15vOzRrVMejyXS015TKnCTvM/v1sHJ/SmWoPvp8o2SpDvWeO+p+9fvnvJ89rN60yqv3tn3yW1PTn+P/yFv33YMTeT+f5STDax+vmJT0edL/W/65YPPli139dbBKeUH8YM/PaPV/mf7xpXesf9DmfMFAADC1ZTgba68L/8BNX6utcJV+9JDGnG9SlaZdIzV0Q6oX+OBgqxq9mu2eRecIy548BaFUvsQ5qAm5UZTrKVfVaXXFHs+uyjobpWqc8Vt19AALdgxaFS5tQuyr7Wcq6rqUOb1pZ4qfE32cbMH8mlU/8jCYpu9nwAAzCTNCd7Myz7sKZN5CzJASC3XDH3poZqCxkoXluXqso+NBQqy8i+2Ku1bv7zgrZrMWxRKHafC3Qnjeq9YEdnt1zLATKlXlDvzpV4zbXmJQhpxeR1oOoE6gtwg5y7o0Z9ynrLll6tDhfIaOk1Chf3P7kujahB2kFR7cdk+kJNL/iL2sH7S8+86WMUz0QAAIBxNajY5pIwz7S0TRAW7QKz+6qM/Pai9ZYLGkipckZWrSX+u2WSFTeRto9Ke9ctrzjaq3gprtobpo3dWXqeUQEFEDRem1Vwc585VideEEpy6IlmdAK9r9IiUgT53AQ9AsdVaLZFT+L4o3H8rEsx464VcjxYrL/t6cxl9rfsKnRR7QufFp/f/BAAA4WlSs8lhDapfmTKbL/vruxW/WCq+7tTHXvBW/dxolZpNlrvwn23Bmk3GpmTeyu9cv99scrRdmk0WnNF6LtBzcVORd4kFyN5Mf002W1dcrIY2s9Mv8Mtvu1QZtYRhwSbyrqXc4J+74Jm3/PKzy0q/umJc2oC4NReklNj/wh8Tqvn/VFU9qiyv0qGotX6F+3vQxNO51hTHxJ6prVAAABBIU4K3BTZYtsmkVD54qeb6rLAZlddsspbgrUKzyTLPzVawZpPVxAiz/cxbkIxeVhQJjVK7EGTevMAXk2UujicTYlVk0SYrVf75KpS6wC9cp2yTzCKZt0ACvKauZpMB1g16+PPPkwXaQPmKNyLpOBmUldhmdr1ss0mb+ji0elRZXqVjUWv98n5ikiQdPXKfJGlV5rk6wdYWfQ0AAAhHU4K3hbZX21Vhkuwyz1XT96bwWq8vNVh2lMuAxVQleLPJKvq85TJvrdVsstkTMVf13ih8TYl3XbHs2GTfuoDbqGHAElfkdUGOb7DMWw0Dlvi3QQLjoMFz8cxbmTpUHOAlfIV92SrtW7WD2QSuR8iZt1oVZhZPGLpVj2Weq5+nX6EjYlt1mAUfvRIAAFSnOcGbBrTDza359fUECH3pobJ97UqplHkrZ7aNNyDz5o822WoDltRxnMLIVNQTlJRS7LxUGySWPizhX2IHOQe1ZfQaG5mHEZg2sr9f4KIrNMNtVz1KSoVNedMpPWdine7IvFBPu4MlSRfG/9iM6gEAMCNEOkl31gE2oNsyx0W+3V4l1O0SNfV5qzhgSZkrtdkBpwqoRquONlnKtGaTZQaqqBTEBcnQNFtodaujeWMgrXwQm6CWbFkt2d5WUE19Z2tMD/T+P/VaSmPXL9Yve2apZ+t+0te2qcsltdYt0l2ZF0qS3hm/Xp9PXdSgWgMAMLNFnnnr07jm2Ji2V8i8NWK0ybnyJhCudoJuKUDn/zLPzNa4hqsMsirtW79NKOViSlQRfzezRWOgqR8CllVppMdqyprymhAvvmsZTGd6Ga0ZDQTq8xa0rKJBfPP2O0hfzOnVK55pa3azyYrlVbHuC229ei2lNZlFGtj/xRpyfep1o9KgNxH48szzc3Nnxs1piRWfUBwAANQn8uDthJjXoX2wQgBVfq6nKkabzLs/17zgbaCW0SYrtGksdcE5SwnFzAUaFTJWRZ+3yWxekzuZFQg8YEmRHcwuq7Tv5fqb1dbnrfz7qXifN387AWd6KznYRcU+b9UL0sS3nnFQAv2wUsNwk0HGK6nULLIxfd6yZRd/300bbTKs7TY4iK2m/KNiGyVJb0p8RI+d/CW9JfkR/cdBl0uf3KMPLbkh12Ty7yY+Lkn6aNePwq8wAACIPng7Kfa4JGll5rDyK5YbbbKKUe/yr/X2kzecdW2TdNdmnyr6plUTvAUdBCVyJacKKP+41LKimwgQoFWTla3UDLPYLlXOmhWWUWLAkgplFAYrQfr0BRqwJFtuFW/syaqEl0XNP09BfpSp3Oct4IZLlj+9gMLRJqef29yaBXUIebTJKoO5sPr/7asRXdr1Yw242dqmeVP//5opEevLrXuve4EezByp0+IPh7JtAAAwVeTB20XxmyVJj7gjyq5XPvPmr1PlxUw281bLJN0VpwooUZXsqJBBBiypZpLu2TbeciNNSt7Fb4+SWj/rIq2fdZEO0B5J089VGJMzF5/nLXhWdvI12e2XGm2yTB1KbKdUdqaaslVhKoFSGjVmR3WZt4CjTVadeQtUbMgKRpssUcPJzFv178HgtQiuYlPvgAUu6/2IZtuEP5ebTdu/wmL+L/2X1VQTAABUIfLgbaENasz1qK7GRVVcweX/kr6fP5FsTX3earxqnF3FwCLVjGjZr/GWzLyZSa+J35l7fGvvP6lXiWAXnsVWSiWqugqu7SxVaorXwBEMm9TstbZ53prfRLfRVSibsQ248UYNYPK8zDPS2EDR5/bViE6wNUWecYorXdd2H80cLkn6bPKN3oIKh+Fn6dP02/RJdW0TAAAUF/lok0kX13fT51RcL0jTqZqbTdbQ563Wi8bZGpMUtNnk5P1KmYvZFv4IlmEweReSWf02odWz3qbM1+Zr/azdkys+5P89/3xv0IMtD2nf7n6tnzXqPf+NP5OWnCnd9TWpu1965WelB34gHXW2Dho8WufHHtRRu9ZLj62WLKazYg8oI9MhO3fqtNhGHbl3q7R2q3fiLOb9DW3zHnfNktIJKZ2Uuvt0zNhmxWNbddAzGyV3gBTvkeLd+vPYI3pX/DfakD5CWjWkM2KPKKkupRTX6x78Dx3bHdPYtvP0pvh2xZRRTE4xOZkymp2JazyeVExOuuMJ/fXgBh0fH/bWMadDbZue+/RB2pro04u6BhVXRrrxLsml9c3uFXo0c4Rm//DLeknXofpo14S2uXk60PZo4cMP6x3xTUopnvubrXHvAv2upyQzvTGzRkPxjPf5uGej3hp/zLt/32b/JJlenXpSu+Mp9SqusXjGr7VJD+7Ra+OPyDmTkz8w/J/WS939OmNsvZ4bG9e8ddul4f2keLeUSUsrf67relZqwO2jWzInaI5GderjaSnzZ1J3nxTv1hvjTyipuBKuW2PqUa+SurznSumL0ue6TteIZun0Wz6l/+3u1sDOE6UbfyZtWyktOWvy/FlMi7cN601xr/9VRjGvzit2+s+b4mmn18RWefV+eEga3Kw3xjcqrrS6lVaX0l42/OYVUnef3h9/XDHLSH9YLrm0Dhga1R96rteP0mfoINutZ93+Gs4cKD2R0NE71um82HYdvW2NdMtPpTkHSRbXIZsH9Ob4er1gcLb0p/t19DM79fXuu9S37SjpzudJmaR3nDIpXdr1hP42fru2u7na5fbVoPql394subTkMlImk3c/ra93b1JMGR1lz2pJ8lnpvz4o7btIsrhXbjqph3tHtZ95n5uNmYXSVZ+T0il9ZO+gvjLLG1DknInPa41bpLTiuY9g39gW/b7nX/THzPH6z9RFKhWVpRXTU5mD9a30+bnPuFT+/+96d1CZZwEAQK0iDd66lFa3pbXZLai4brlfwGv9ZXuuDSujuIbVV3nlwm1WeL5UfWfbhKSgzSYLe8+U1qcJ7XT7VSwzambeVBBjrkefSL1NX+y+WpIUG9td/AWrr598bXJ0cvn2Vd6fJCVHpev/ybu/5SG9XtLreyQ96f9J+p8e/3UPSWf1SHrK/wvgnySpR9J9/p/vh36Zp6RXST/7jb7dk/eiIenAuKSnH9XZ3UUKdZKyy2+WXqe8x1nbpIxMqXjMu6h+oEeKxXVufK/Ojd8vbZEO0wq9M/9Tev8yfazY9iTp997NB/O3daP06ez9ZZOrfih/nfzyfi19qbD833k375a843T79E0f6+fwXxF/1LuzWdLmX+ae/49SdZb0hi5/XrBB6dS4pF2rpF3+k0/fNmXdoyR9trCs30ze7ZX01ex5+mWZbd/h3Xww+9zdXVKsS/NT45ofkz4Z++HkuhlJ10jnSTqvR9Ia/8/3Akmf6Za0W9LvpBMknRCXtOMe6Q9527SYLo6bui2thbZ3cvlja71gLBb3g9C4FPOC1WNsTBnFNMcPznTs33nrSFLMq/Mv731WTqZ5NqRupXTIvgukeJc2TozJJoa0vw3qxt5LJUkPZo7UOneQFttOnXTTE1JMWhJ7Vpd0Xa8zJr6op9yiaYdqfxvUTk3+r5lsmlz6v9TazPRyAABA/SIP3iRpR4CgI1hgVnmlKaNNalhjXfuqcig2Xc193qoYsGTKFgKMNrlBB1Yss4oiQ3OYbdMz7kD9LH2afpY+TXM0qjs+cpaO/9xduXXetnShPnWik/Y9WFp3m7TklRoYHtFfXX6rDooN6mevP0g6+Dhp7nO9i9rRXdLABmn7Y1r2dEZXPjChv3np4br4lOdKzum8r90mk9O7/vwwfeuOp3T+iw7UJace7u21y0z+ZdJS/4Jcdk3JMX3h1/fptvWj+ui5R+nlRx/iZeTSSf3tlbfpiNgWjfUfoivecYZeddkt6lFKsyyhd7yoS//1SL9edfKx+t6fnlFG5ufeTE6mOX092jOWUkYxrf7sefr776/QrWt2eZkimeJK652vOFIb9oxp2aNbJUnrP+1lNo649DdabDt03T+cqPs2jOgTv12tfptQt1L6yrv+Rn/3zdu9H0L8TNKEujWuHq361FkySad/6Y/aNTwhk9PDnzhLJ/z772SSHvjYmf7xcHrNN+7Us3vGNKc3rpGJpExSTBnd9eHTdeoXbvH3QjI53fah06TuPv3DD+/Vqk179PXXv0jHLdrHO06pcam7T6d+bYVSLiaTtFtz9MnT99dFf7nUz3Cm9NLP3KBupdVrSfUqqTka1cG2S//+1nN0yne3K6W4vnThS/Xha+7VGS88VF9/7VFS0itbLpOr951rtun9P37Qy2j6mc57P3K69wF0GY0lUzr7q7cppoxu/efTJJfWS798v1KKK624kop75+Qz50ixLh3+0RvlFNP6z3vHfvXWIb3xst+oR0nN9vurLpglXfP24/ST5Rv1rft368KXHamLTz3cC6LiPbpj7W69/6eP6uVHLtQVb1yqXzy4WZ++bqX++vhF+uzfnJALshSLacmlkz9WZMcTXf+p84t+jlzG6Yx/m4y4D5jTq/v+5sxp633qruunPD7/Iq+8H/30Yf1i+ya9OX6Tzot5v0ocYHt0bux+9duEMhbXtalT9Nq4F43/ffxGfTR18bTy99derXaLi9axlHsyL6hqfQAAEEykwVu278XeGkZ7zFdN6JUfsMy1YY11zaltm1WOLpi1j3nNJgM1caxix/ptQiOZVmw2aTrMtuop95zcsiH1y3X1T1kvEe+TDn2R9+DFb5YkpW1CG92B2uIOko47b2rB+z7H+zv0ZD09tFar3Gr9ed8R0oFHS5Iec89IknbP/TM97EzHzX6udOgLA9X5qf5xrXLbNDj/RdIBB+eWr3BbtSL9fC2K90kH/plWuvXeE0469eDna/XDq3V6zwLt0sC0MmPq1piS3oOuXmViXcrkdTFNy8uyFOvzllFMG9yByiw4SqO7dmirdubeyJn4LA2rf9prJMn1zJHFTCM2W4PZj3b/fO3Rvt79fRbm1t0dm6/tGtWYujSk1GQh856rTe6AqQUveJ4kaVf8IG1wPRrf7wjpgKnZ801uw5THw33P8QKvbi/LvUPz/ErmV1j6xKKTNOKnpywW06hmKWNxadZ+3l+B1KyUdqlg+X6TgUVmIqUNzv9RY/8j/W2vnVZOtl6uoNuvk5vMMvl13Wnd0qEna+uTC7TGPam9sxZL8ycHXEr2x7Vb+2o4vq/UN1fp7iENarbG4/tIPcXPlSeaPoQ/TL9SP0y/smCp05dfd7w+9LOH9bv0Uv1Pz1c0rp6ir19gg9qVycu8FZZU5J/fVlVuXQEAAKoX6YAlXcpIkvYEGO0xrLmkpmXe4rU1Nay12WS/vGaTw0GaTQYoL8ub560FR5t0aR1q27TeTc0KTpsqoNhok1Vuq9wxqmWwiNLzvNVSVv15zqJz4ZWdDt5TVX1r2bcg6wTc/eLvg6hyxEW2HeB9Wap2jR5dMtziLVfe7zNL9VTmYB1o05s2dymleTasXW7f6fVr3mkCAGDGijR4y2beBmoYqj9fNaPe5a8710b8ZpON3Wa+xow26Vp2tMn9ElvVa6lpAxYECWaCXgyWO0w1jaJYabTJIk9PTtgcrNIlt1Chvo3YnwCbDfFF1RQfoN4NHm6ylvfWtHo3aLTJalVzqLa6+TrI9kxbPk9DkqRdmvy/2QKDjgIAMGNFHLx5mbcgk2QHmuetyu17zSZrzLzVOtqkjSnp4koEaKEadJLuXiXVZZmmzfN2lG3U6t636uzYfdOeWzjijeTwRObQKcsDZWtCyC1Umour6GsqzC9WNqgIHHAWL6PCNG/Tl5XZXjZADvJerSUIqmqet4BlTpmkO0DQ04y4YXL+tqmPS6nlPVhNPYKq5lht1XwdVCTztr8NStKUwZGq/eECAACEJ/Lgbcz1aKJE34p85TI11Yw2WdhscrzmzFv550sPWDLhZ8iCZBXyyiuzXp/fFLNZUwW8M369ei2p/+65TF/o+m/1KpF77rl775ekKX3epGLHp/Z2k7lgqtj6dU3SXf75oq8pUWbh8lJnv1wQ5Vx1QVau2WSAdXPr1HKcArwo8PHPWy9IcNTwed6KvS0LguJp+1/wXrBKb47aaxduaXk7u8XN14Hao5j/A1vWfD942+0m+wo3ah47AABQWeSjTe5RbcFTMdX0K+pWSvvYuMa6a8u8VRxtssTy2TU2byy3a/tY8BEsG2GR7czdf33XbXp9123Sp94mzT1Uxw94A1cMFQyqUXjBW0+ftyDXxrVcV5YKSmqKF0K4sHX+iI9Bi81liBo/k3WAVYIdgKLZxRbL6FTqrzkZdBa8x+vdbmF5DexTt9XNV7eltUB7JweXkXSwn43blressD6tdbYAAOhsdWXezOwcM1ttZmvN7NJK68cto70B+7uVbzqVbbYT3Hx5vyCPdU2/CAmiYn+0EpWZbWMaDTBYSeE2yl3AZif+HnbVz1cXhkW2U79Ov1z/L/GBos+/O/GP0xcW7E7xDEd19SgWvFfTtG/yNeWzdcWCocoD2BSWUWrbVRSi8j9YVBX01NGXrlEDlgRrNtmEPm8FQXGlXSsVzNWr2tIqxvB5BW5x8yVNBmtZh9h2pZ1ps9t/stzcy2du2Fbpu888l/vPP2JmL25GPQEAnafmzJuZxSVdKeksSZsk3W9m1znnHiv1mrjSgUaalCpcGFTTbMdfd39/UtyR7vmBtl+imJJKTtKdazZZWSwvlC6becsGb1VONh7GZW9MGT3Hdum3mZP1u8yJOmz8/yQpN0/WV3//pJbdvGba66aPnFfdSIr5ygVokxes1Qcypfu8lVbqAn368iIBoJW/wHaqrpngZOat8rq1vBca0Vxu6jmvHBw1Y7CMipm3gqa6jWo1WW0wWCnQzT/2u/w+bfNtMFfxg7RLZ8Qe1BYtUDL/q2KGN5sM+N13rqQl/t9Jkr7p3wIAUJd6Mm8nSlrrnFvnnEtIukbSBeVe0KVMoMFKKqllJPSFfvA2WmPwVmvk02/jGgmYeQuaVZhjzcu8Hag96ra0NrmFRZ+P4uK67IiANZyoSq8o1+ct8DZqybypxv1p1IAljR5tskFBZ1iCnsN2HI1xt7w+bQv80SUl6Z5Z79MLY+un9GmVGp/9bANBvvsukPQD57lH0lwzO7iwIAAAqmW1Nu0xs9dKOsc59w7/8ZslneSce2+p1xz/nB73r39/vv4t9Y6atlnM7J64RhLpKY/zZZ/729jt+nLPVfrzia9qY8EcZGHWpXDby3o+omfd/npn8p/rLi9b5qtif9IVPV/XmRNf0Fq3uMgrqyuzGkcnH9Mvej+ttyQ+rNszx00rN/9c5OvrjmssOfW5wrpknHLrlKtnsfNdbLtB97XU+6dweal9qyToawv3ZVZ3TOPJTLmXTNHfE5epdL1L7VuluhY7xqU+Z8VeV257pfax1LmrtB0nabRgf8u9pvDYBD0uQd8n1R6nfPn7Um7dIOerkn00qpWz3qH/SF6k/0m/SpK0ftZFkqQrU3+tL6YuLPq6cu/tZ/7rVSucc0sDV6JNBPnuM7PfSvq8c+5O//HNkj7snFteqtylS5e65ctLPg0A6CBmVvN3ZD3B2+sknV3wBXaic+59BetdIukS/+ELJa2saYPtb39JOyuu1blm8v7P5H2XZvb+z+R9f75zecNUdogg331mdr2kzxUEb//qnFtRUBbfj/WZyZ+vWnC8qsPxqg7Hqzo1f0fWM9rkJkmH5D1eLGlz4UrOuaslXS1JZra8E3+JDWIm77s0s/d/Ju+7NLP3f6bve7Pr0CBBvvv4fowAx6w6HK/qcLyqw/GqTj3fkfX0ebtf0hIzO9zMeiRdKOm6OsoDAKDVBfnuu07SW/xRJ0+WtNc5tyXqigIAOk/NmTfnXMrM3ivpd5Likr7jnFsVWs0AAGgxpb77zOxd/vNXSVom6TxJayWNSnp7s+oLAOgsdU3S7ZxbJu9LKqir69lem5vJ+y7N7P2fyfsuzez9Z987ULHvPj9oy953kt5TZbEde7waiGNWHY5XdThe1eF4Vafm41XzgCUAAAAAgOjU0+cNAAAAABCRSII3MzvHzFab2VozuzSKbUbBzNab2aNm9lB21Bgzm29mvzezNf7tvLz1P+Ifg9Vmdnbe8pf45aw1s8utlhmUI2Bm3zGz7Wa2Mm9ZaPtrZr1m9hN/+b1mdlikO1hGiX3/lJk965//h8zsvLznOmnfDzGzP5rZ42a2ysze7y+fKee+1P53/Pk3s1lmdp+ZPezv+6f95TPi3DeCVfg+NM/l/vOPmNmLm1HPVhHgeL3RP06PmNndZnZcsXJmikrHK2+9l5pZ2rx5C2esIMfLzE7z/8evMrPboq5jKwnwedzPzH6T950xo/v7WpFrx4Lna/t/75xr6J+8Dt1PSTpCUo+khyUd0+jtRvEnab2k/QuWfUHSpf79SyX9l3//GH/feyUd7h+TuP/cfZJeJskk3SDp3GbvW4n9fYWkF0ta2Yj9lfRuSVf59y+U9JNm73OFff+UpA8VWbfT9v1gSS/278+R9KS/jzPl3Jfa/44//3499/Hvd0u6V9LJM+XcN+B4Vvw+lDfQyQ3+cTpZ0r3NrneLH6+XS5rn3z+X41X5estf7xZ5/TZf2+x6t/LxkjRX0mOSDvUfH9Dserf48fq3vO+DhZJ2S+ppdt2beMymXTsWPF/T//soMm8nSlrrnFvnnEtIukbSBRFst1kukPR9//73Jb06b/k1zrkJ59zT8kYhO9HMDpa0r3PuT847kz/Ie01Lcc7dLu+DmC/M/c0v6+eSzsj+Ot9sJfa9lE7b9y3OuQf8+0OSHpe0SDPn3Jfa/1I6Zv+dZ9h/2O3/Oc2Qc98AQb4PL5D0A//Y3yNprn/8ZqKKx8s5d7dzbo//8B55c+rNVEGvt94n6ReStkdZuRYU5HhdJOla59wGSXLOzeRjFuR4OUlz/P/h+8i7bkpFW83WEeDasab/91EEb4skbcx7vEnlL3zaiZN0k5mtMLNL/GUHOn8+H//2AH95qeOwyL9fuLxdhLm/udc451KS9kpa0LCah+O9fqr7O3lNxzp23/0mbSfIy8DMuHNfsP/SDDj/ZhY3s4fkXej93jk3I899SIJ8H3byd2a1qj0WF8v7FXumqni8zGyRpNdIukoI8v46StI8M7vVv9Z7S2S1az1BjtcVkl4gabOkRyW93zmXiaZ6bamm//dRBG/FfkHtlCEuT3HOvVheU433mNkryqxb6jh06vGpZX/b7Vh8U9LzJB0vaYukL/vLO3LfzWwfeb/WfsA5N1hu1SLLOnH/Z8T5d86lnXPHy8tonGhmLyyzekftewME2deZdDwqCXwszOx0ecHbhxtao9YW5HhdJunDzrl046vT8oIcry5JL5F0vqSzJX3czI5qdMVaVJDjdbakhyQ9R9534xVmtm9jq9XWavp/H0XwtknSIXmPF8uLyNuec26zf7td0i/lpZS3ZVOe/m02xV7qOGzS1GYe7XZ8wtzf3GvMrEvSfgreVDFyzrlt/oVtRtL/yDv/Ugfuu5l1ywtcfuScu9ZfPGPOfbH9n0nnX5KccwOSbpV0jmbQuQ9ZkO/Djv3OrEGgY2Fmx0r6lqQLnHO7IqpbKwpyvJZKusbM1kt6raRvmNmrI6ld6wn6ebzROTfinNsp6XZJM3VQnCDH6+3ympk659xaSU9LOjqi+rWjmv7fRxG83S9piZkdbmY98jqkXxfBdhvKzGab2ZzsfUmvlLRS3r691V/trZJ+7d+/TtKF5o2sdrikJZLu85scDZnZyX4b4bfkvaYdhLm/+WW9VtItfv+YllTQLvk18s6/1GH77tf125Ied859Je+pGXHuS+3/TDj/ZrbQzOb69/sknSnpCc2Qc98AQb4Pr5P0Fn8UspMl7c02UZ2BKh4vMztU0rWS3uyce7IJdWwlFY+Xc+5w59xhzrnD5PUxfbdz7leR17Q1BPk8/lrSn5tZl5n1SzpJXr/nmSjI8dog6QxJMrMDJT1f0rpIa9leavt/76IZbeU8eSO0PSXpo1FsM4J9OkLeSDsPS1qV3S95fTVulrTGv52f95qP+sdgtfJGlJT3S9hK/7kr5E+e3mp/kn4sr3lYUt6vBReHub+SZkn6mbxBDu6TdESz97nCvv9QXpvuR/wP4MEduu+nykvjPyKvOcRD/md6ppz7Uvvf8edf0rGSHvT3caWkT/jLZ8S5b9AxnfZ9KOldkt7l3zdJV/rPPyppabPr3OLH61uS9uR9Npc3u86tfLwK1v2eZvBok0GPl6R/kTfi5Ep5zeabXu9WPV7ymkve5P/vWinpTc2uc5OPV7Frx7r/32e/PAEAAAAALSySSboBAAAAAPUheAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA38f1tmrwdPSF2lAAAAAElFTkSuQmCC",
"text/plain": [
""
- ],
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAA28AAAEzCAYAAACmHDtZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABIY0lEQVR4nO3deZwkdX3/8fene46dWRb2YDncBQFZRKIcugIKMRBALhM0UYN4B+VnPKImJmK8o4nGExGUEG9jxAsVZUEU5JZjl3MXWHZZlt1l72N27unr+/ujqnt6evqo7q6uPub1fDzm0d3V1d/6VlX3dH368z3MOScAAAAAQGuLNbsCAAAAAIDKCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2kDF4M3MvmNm281sZd6y+Wb2ezNb49/Oa2w1AQBoDcW+FwueNzO73MzWmtkjZvbiqOsIAOhMQTJv35N0TsGySyXd7JxbIulm/zEAADPB9zT9ezHfuZKW+H+XSPpmBHUCAMwAFYM359ztknYXLL5A0vf9+9+X9OpwqwUAQGsq8b2Y7wJJP3CeeyTNNbODo6kdAKCT1drn7UDn3BZJ8m8PCK9KAAC0tUWSNuY93uQvAwCgLl2N3oCZXSKv2Yhmz579kqOPPrrRmwQANNmKFSt2OucWNrseTWJFlrlpK/H9CAAzUj3fkbUGb9vM7GDn3Ba/Kcj2Uis6566WdLUkLV261C1fvrzGTQIA2oWZPdPsOjTRJkmH5D1eLGlz4Up8PwLAzFTPd2StzSavk/RW//5bJf261goAANBhrpP0Fn/UyZMl7c12NQAAoB4VM29m9mNJp0na38w2SfqkpM9L+qmZXSxpg6TXNbKSAAC0ihLfi92S5Jy7StIySedJWitpVNLbm1NTAECnqRi8OefeUOKpM0KuCwAALa/M92L2eSfpPRFVBwAwg9TabBIAAAAAECGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANlBX8GZmHzSzVWa20sx+bGazwqoYAACtyMzOMbPVZrbWzC4t8vx+ZvYbM3vY/458ezPqCQDoPDUHb2a2SNI/SlrqnHuhpLikC8OqGAAArcbM4pKulHSupGMkvcHMjilY7T2SHnPOHSfpNElfNrOeSCsKAOhI9Tab7JLUZ2Zdkvolba6/SgAAtKwTJa11zq1zziUkXSPpgoJ1nKQ5ZmaS9pG0W1Iq2moCADpRzcGbc+5ZSV+StEHSFkl7nXM3Fa5nZpeY2XIzW75jx47aawoAQPMtkrQx7/Emf1m+KyS9QN4Pmo9Ker9zLlNYEN+PAIBq1dNscp68XxsPl/QcSbPN7E2F6znnrnbOLXXOLV24cGHtNQUAoPmsyDJX8PhsSQ/J+248XtIVZrbvtBfx/QgAqFI9zSbPlPS0c26Hcy4p6VpJLw+nWgAAtKRNkg7Je7xY07sMvF3Stc6zVtLTko6OqH4AgA5WT/C2QdLJZtbvt+s/Q9Lj4VQLAICWdL+kJWZ2uD8IyYWSritYZ4O870SZ2YGSni9pXaS1BAB0pK5aX+icu9fMfi7pAXkdsR+UdHVYFQMAoNU451Jm9l5Jv5M3yvJ3nHOrzOxd/vNXSfqMpO+Z2aPymll+2Dm3s2mVBgB0jJqDN0lyzn1S0idDqgsAAC3PObdM0rKCZVfl3d8s6ZVR1wsA0PnqnSoAAAAAABABgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AZaO3hbf5f0i3dKv3q39Ov3St9+pbT+zubVJ52Sfv9JaddT4Za75WHpZ2+X9qwPr8zdT0ubH5Qy6fDKBAAAANA0Xc2uQFn3/be0+gZp9kJp8FlvWWKkOXUZ2iZ9+Sjv/iEnSQueF17Z626VVl0rnffF8Mq855ve8fvYDikWD69cAAAAAE1RV+bNzOaa2c/N7Akze9zMXhZWxSRJu9dJR5wmfXDV5LLFLw11E2UNbJQuO1a69pLJwE2Sjjo73O3sekrqXyDN3j+8Mu/7b++2qye8MgEAMrNzzGy1ma01s0tLrHOamT1kZqvM7Lao6wgA6Ez1Zt6+JulG59xrzaxHUn8IdZo0slM6+DjJTDrlA9LeTVL//FA3Mc3DP5E23C391dekO78qDTzj/WVdujH8TNb2x6QDjgmvvHTSuz3kpPDKBADIzOKSrpR0lqRNku43s+ucc4/lrTNX0jckneOc22BmBzSlsgCAjlNz8GZm+0p6haS3SZJzLiEpEU61fGN7pD4/WDvr06EWXdIvL/Fuz/y0tPzbk8vfc5+08Pnhby+TkbY9Jp3wpvDK3OMHmy95e3hlAgAk6URJa51z6yTJzK6RdIGkx/LWuUjStc65DZLknNseeS0BAB2pnmaTR0jaIem7ZvagmX3LzGaHVC8pMSqlxqW+eaEVWVEqL/Zcf4d3O/8I6UNrGhO4SV5fvuSIdMDR4ZW5a613u+DI8MoEAEjSIkkb8x5v8pflO0rSPDO71cxWmNlbIqsdAKCj1RO8dUl6saRvOudOkDQiaVrbfzO7xMyWm9nyHTt2BC99bI932+hmkvn25n0f/8TPhJ39OWmfBrZ42fmkd7tgSXhl5oK3EAdVAQBIkhVZ5goed0l6iaTzJZ0t6eNmdlThi2r+fgQAzFj1BG+bJG1yzt3rP/65vGBuCufc1c65pc65pQsXLgxeejZ4izLzNrJz+rIj/qKx28xOO7B/yMFb37xoA18AmBk2STok7/FiSZuLrHOjc27EObdT0u2SjissqObvRwDAjFVznzfn3FYz22hmz3fOrZZ0hqa2+a/P2G7vti/CAGTUD94uudUbQCSdkLr7GrvNoc1SrFva58Dwyty9TppP1g0AGuB+SUvM7HBJz0q6UF4ft3y/lnSFmXVJ6pF0kqSvRlpLAEBHqne0yfdJ+pE/0uQ6SeGNkNGMzNvoLu+2f3+pq9f7i2Kb/fO9ETXDsme9dMiJ4ZUHAJAkOedSZvZeSb+TFJf0HefcKjN7l//8Vc65x83sRkmPSMpI+pZzbmXzag0A6BR1BW/OuYckLQ2nKgVG/cxblE3/ss0m+xdEt83R3eFuL5ORBjdL+y0Or0wAQI5zbpmkZQXLrip4/EVJX4yyXgCAzlfXJN0N1azMW1ef1BPudHUVtxlm8DayXcokCd4AAACADtPCwdtuqWtW4/uc5RvdLc3eP7rtZbcZZnZx7ybvdl+CNwAAAKCTtHDwtifarJvkDVgS9QiNo7vCHZQlG7ztVzjtEAAAAIB21rrB2/headbcaLc5ussbrCQqmYyXYQyz2eT4Xu826sAXAAAAQEO1ePC2b7TbHNkZ7WAl4wOSy4S7zeSod9sdYb89AAAAAA3XwsHboDRrv2i3GfbIj0G2J4W7zcSId9szO7wyAQAAADRd6wZvE4NSb4SZt9SElBiSZkcYvI01YDqE5KhkcSneE16ZAAAAAJqudYO3qJtN5ibojjLzlt1miMFbYtTLuoU56TcAAACApmvN4M256JtN5gKpCAcsaUTAmByhvxsAAADQgVozeEuOeRNNR9lscmSnd9uMzFuYUwUkRqKdZBwAAABAJFozeJsY9G6b0Wwyykm6x/dKFpN654RXZrbZJAAAAICO0prB23g2eJsb3TYbMfJjJRNDXuAWZv+05IjUTfAGAAAAdJoWDd78iaajbDY5ukuSRTu59fig1Btyv77EKM0mAQAAgA7UmsHbhB+8RdlscmJI6tlHisUj3OZg+PuYHGXAEgAAAKADtWbwlms2GeFok4nh6PuKZZtNhikxQp83AAAAoAO1aPDWhGaTzRilcXxv+PuYYKoAAAAAoBO1ZvA2MeTdhp2VKqcZGauJocY0myTzBgAAAHSc1g7eevaJbpvJkWi3J3l93sLMvGUyBG8AAABAh2rN4C0x7A8eEmH1os68OeePNhlidjE15t3SbBIAAADoOK0ZvGVHfoxS1H3FUhNSJhlus8nEqHdL5g0AAADoOK0ZvCWGpd6og7fRaAPGCX9EzTCbTSZHvFsybwAAAEDHac3gbWK4CZm3iKcKGG9A8Jbwgzcm6QYAAAA6TmsGb4nhaEealKKfKiCbeWtIs8mIA18AAAAADdeawVvUmbd0SkpP0GwSAAAAQMtqzeAt6j5v2aCnGc0mG5J5I3gDAAAAOk3rBm9RZsESTchYNWIi8qQfvHUz2iQAAADQaVozeJuIOPOWG+ijzZtNMmAJAAAA0LFaL3hLp7zJpnsiHLAk0YRmkxPD3m2YmbdmZBABAAAARKL1grdENqhpRuYt4tEmu2ZJ8e7wymxG3z0AAAAAkWjd4K0Zfd4ibTY5FP50CIlRKdYlxXvCLRcAAABA07Ve8DbRjMxbNmCMMGPViLnskqPeYCVm4ZYLAAAAoOlaL3jLBVIR9nnLjtIYaZ+3ofAzfVFPNA4AAAAgMq0XvOWG0G/GVAERD1gS5kiTkp95I3gDAAAAOlHrBW/je73bsAObcprRbHJisAF93si8AQAAAJ2q9YK3sT3ebf/86LaZGJUsLnX1RrfNiaHws4uJESboBgAAADpU6wZvffOi22ZixMu6RTnQR6MGLGGaAAAAAKAjtV7wNj7gzX/W3RfdNhPD0Qc9jZoqgGaTAAAAQEdqveBtbE+0WTcp+oxVOimlxsMfUTNJs0kAAACgUxG8SX5fsQgzVrkRNcm8AQAAAAimBYO3geYEb2HPuVZOo6ZDiDoIBQAAABCZFgzempF5i7jPW25qghCDt0xGSo0xYAkAAADQoVozeJs1N9ptRt3cMDspeJiZt+Sod0vmDQAAAOhIrRm89c2NdptRN5tsROYtG7yReQMAAAA6UmsFb8lxLwjp9GaTE9ngLcRtZrN5BG8AAABAR2qt4G18wLvt9KkCcoEWzSYBAAAABFN38GZmcTN70Mx+W3dtxvZ4t1EGb6mElE5EOz9aI5pNknkDgEiY2TlmttrM1prZpWXWe6mZpc3stVHWDwDQucLIvL1f0uMhlNOc4C3ZhKAn0cBmk2TeAKBhzCwu6UpJ50o6RtIbzOyYEuv9l6TfRVtDAEAnqyt4M7PFks6X9K1QatOM4K0ZGavEiGQxqbsvvDJzA5YQvAFAA50oaa1zbp1zLiHpGkkXFFnvfZJ+IWl7lJUDAHS2ejNvl0n6V0mZ+quiJgVvTRilMTu6pVmIZWb3I8JRMwFg5lkkaWPe403+shwzWyTpNZKuirBeAIAZoObgzcxeJWm7c25FhfUuMbPlZrZ8x44d5QttSvDWgCaMlUwMhb+9JM0mASACxX51cwWPL5P0YedcumxB1Xw/AgCg+jJvp0j6azNbL6/ZyF+a2f8WruScu9o5t9Q5t3ThwoXlSxzbI1lc6p1TR7Wq1Kxmk2FnyBI0mwSACGySdEje48WSNhess1TSNf7342slfcPMXl1YUFXfjwAAqI7gzTn3EefcYufcYZIulHSLc+5NddVmbI+XdQuzOWEluSH2ow7eQt5ebsASRpsEgAa6X9ISMzvczHrkff9dl7+Cc+5w59xh/vfjzyW92zn3q8hrCgDoOF3NrsAU2eAtSs1oNpkYDj/zlhyRYl1SV0+45QIAcpxzKTN7r7xRJOOSvuOcW2Vm7/Kfp58bAKBhQgnenHO3Srq17oKaErw1aaqAOQeHXOYoWTcAiIBzbpmkZQXLigZtzrm3RVEnAMDMEMY8b+EZ2yP1zY12m00bbbIBA5YwQTcAAADQsVoseBuQZs2NdptNGW1yuAF93kYZrAQAAADoYK0VvI3vbULmze8rFo+wr1hiROoJeUTN5CjTBAAAAAAdrHWCt0zGC94iz7z5zQ2jGuHSOX/AkgaMNkmzSQAAAKBjtU7wNjEoyUWfeUuORDvQR3JUkmtM8EbmDQAAAOhYrRO8jQ94t7P2i3a7UWessqNb9oY9VQB93gAAAIBO1jrB29iAd9usZpORbS87QErIwRtTBQAAAAAdrXWCt2zmrRlTBUQ90qTEVAEAAAAAqtJCwdte77YZUwU0o9lkIzJvNJsEAAAAOlbrBG/ZZpPNmCogyoE+GhG8ZTJSaoxmkwAAAEAHa53gLTdgydxot5sYlnpDnnOt7PaGvNsws33JUb9MMm8AAABAp2qd4G1sQLJ49P22JqIO3how2mQ2eGOqAAAAAKBjtU7wNj7gNZmMarJsyWtumBgKv/9ZOY1oNknwBgAAAHS81gnexgaibzKZbNCca+VMNKLZ5JhfJsEbAAAA0KlaJ3gb3xv9YCXZYfujbjYZ65a6ekMsk8wbAAAA0OlaKHgbaM5gJZLUE3HwFvocbwRvAAAAQKdrneBtbKAJmbdB7zbKZpOJ4fD72BG8AQAAAB2vdYK38QFp1n7RbjPbbDLSAUuGww8WmSoAAAAA6HitEbw515wBSxJN6vMWdrPJXJ+3vnDLBQAAANAyWiN4SwxLLj0zBiyZGG5gn7eI58gDAAAAEJnWCN7G93q3UWfesn3eop7nLewBUpJk3gAAAIBO1xrB29iAd1sm85ZIZfTLBzfJOVf0+Zsf36adwxPVbTfXbDLqAUvCzrz587wxYAl8yx7doqHxZLOrUdSabUNa8cyeZlcDAACg7bRG8DY+4N2WybxdccsaffAnD+uGlVunPTeWSOvi7y/XW759X3XbnRiWLBZt0NOI4C0xInXNkmKtcTrRXGu2DendP3pA//rzR5pdlaLO+urt+ttv3t3sagAAALSd1rjaz2beyow2uX3Iy6rtHZueTUj72bhndo1Ut93EsNeE0ay619UjMdKA0SbHyLohZzSRliQ9OzDW5JoAAAAgTK0RvGUzb33zot3uxFC0TSYzaa9/WiPmeQs7mwcAAACgpbRG8Dbm938JMNpksS5vpfrBVTQxFP00AVJjRptksBIAAACgo7VI8Dbg9T0rMwpjkJaNVm3zx8Rw9CNNSuFvMzFKs0lMU+tvGgAAAGhNrRG8jQ94/d2iHnBjYjjikSYbFLwlCd4wKcounAAAAIhOawRvYwN1zfFWc4JhYijizNuQd9uI0Sbp8wYAAAB0tBYJ3vYE6u8mSa72UG26xLDUu2945VXcnp95Czvb14jpBwAAAAC0lNYI3sYHAmTeSrcFq7mVWNSjTTZqwJJGTD+AthfqDx0AAABoutYI3sb2SP3zo92mc9EPWDKRbTYZ8jYnIt4PtDSr/ecMAAAAtLDWCd4CzvFWdKqAWraZGpcyqfYfsKQZQSgAAACAyDU/eMtkvAFLKgRvgaYKqGa7E8PebTP6vIXZbDI1Lrk0fd4AAACADtf84G18QJKT+hdEu91Eg5owlt3mcPjbbNT0A2h7zPMGAADQWZofvI3u9m77Iu7zlsu8RRy8dc2S4l3hlikxYAkAAADQ4VogeNvl3QbMvIWWTGjU4CHlNGI+tmwQSrNJFGCybgAAgM7S/OBtzM+89Vfo8xb2dhNN6PM20YD52Gg2CQAAAMwIzQ/ess0m6+jzVlPfnmzmLepmkz1zQi6zCRlEtAX6vAEAAHSWFgje/GaTQfu8lbsirSY914jBQypuswHNJhs18TfaFs0lAQAAOlPzg7ex3VKsS+otn5EK/YK0aZm3BvV5Y8ASAAAAoKM1P3gb3eU1mawnOqup2WSTMm9hB1n0eQMAAABmhBYI3nZHP02A5GXBumdLsXi02ww7yGpG80+0Bfq8AQAAdJbWCN6qGKyk6PVoLUm7iaHomxo2ZLTJYcniUldvuOUCAAAAaCktELztqjhNgCRZ2JMFTAxFn61KjDQg8+aXySgVAAAAQEdrfvA2vE3a56D6yqileVhiuOIgKaFKJ6X0RPjB28Qwg5UAAAAAM0DNwZuZHWJmfzSzx81slZm9v+pCMmlpfK/UH06ft6pyTxMRB2+5vmkNaDbJNAEogi5vQGOY2TlmttrM1prZpUWef6OZPeL/3W1mxzWjngCAzlNP5i0l6Z+dcy+QdLKk95jZMVWVML5XkqtqwJLQBmFIRNxsMjsqZOijTRK8YSpa0AKNY2ZxSVdKOlfSMZLeUOS772lJf+GcO1bSZyRdHW0tAQCdqubgzTm3xTn3gH9/SNLjkhZVVcjobu+2f74e2zwo55xWbd4rVyRCG0umS9fFzzEMjqeCb3tiqGLm7bHNg/rVg88qmc4okcroyW1Dwcsv1KjJtIv0o3PO6aGNA0qkMlq9tY46B7Bm25AmUqXPDcK3euuQUulMs6tRNeecHts8OG35lr1j2j2SiKQO2wbHtXN4ouw6a7YNKZFqv+MbllLnCTknSlrrnFvnnEtIukbSBfkrOOfuds7t8R/eI2lxxHUEAHSoUPq8mdlhkk6QdG+R5y4xs+VmtnzHjh1TnxzzgreHd8V03uV36J0/WKHzL79Tv3jg2Wnb+PmKTWFUdVKFvmKrNu/VeZffoQ/85CG94ep79JnfPqZXfvV2bR4Yq317UmOmCigo8661u/TqK+/Syz9/i86+7Hat3d6YAG7H0ITO+urt+sSvVjWkfEz39M4RnX3Z7frC71aXXKdVpwj43t3rdd7ld0xb/rLP3aKXfPb3kdThpP+8WUs/+4eSz28fHNdZX71dn7xuZST1aUXfvcs7T/eu29XsqrSqRZI25j3epPI/XF4s6YZiT5T9fgQAoIi6gzcz20fSLyR9wDk37eda59zVzrmlzrmlCxcunPqkn3l7ZrRHkvSHx7dJUtUZrppGoqww59qWgfHc/eXP7NH96726Dowmq99WdntSYwYsKcjmPb3T21Y2w7B9qHymoVaD496xyB4bNN4O/1w+uGFPhTVbz6oy2ZxWCTiz7+n7np657+mVm/dKkjbuqfGHqs5X7Aun+Cw2ZqfLC94+XOz5st+PAAAUUVfwZmbd8gK3Hznnrq26AD/zNtE9t55qVC+dlFLjEQ9Y0qBmk8nRymU2+MK4Ra674aPPG9BQmyQdkvd4saTNhSuZ2bGSviXpAuccaUwAQCjqGW3SJH1b0uPOua/UVMiYlz0Y796v1mrUZsLP7JUJ3kIPSBqVeSvW5y3cLZREjNA8rZKpqgbvl/ZSrO8xJEn3S1piZoebWY+kCyVdl7+CmR0q6VpJb3bOPdmEOgIAOlRXHa89RdKbJT1qZg/5y/7NObcscAmjuyWLK9FVZwas2qvCRgVSQbYZ5miTmYwfvDHa5ExBVg2NVlMz9BnEOZcys/dK+p2kuKTvOOdWmdm7/OevkvQJSQskfcP7nVMp59zSZtUZANA5ag7enHN3qt4f08d2S33zqroiLfprcLU/EE80IJCqpBHNJlNjklzFMhv9+zm/0EcvyBHnvKAWjobQFfk/Ui4rWHZV3v13SHpH1PUCAHS+UEabrNmoH7xVIZTLilzmrUyzybAvfLMBY3eIwVuj+tEFZKSBIhfkiLdq5oS3S3vh8w0AQOtpbvA2tlvqDz5Bd2gm/FHvahiwpObrmcSwF7jFQjzkJZp/Flax0QkYfqcHOg+ZWwAAWk+Tg7c9Ul91wVso1xN1NJusefuN6JuWGPVue/qnLGbAks7XjhfWrZoRxFScJwAAWleTm03uqTrzVuySteo+Gs0asCT04K25zSaz2jCOAAAAANpO85tN9s2LPouQy7xF2WxyJPwBUgIGoY0agIAuMc1Trj9Sq56XVq0XiuM3GQAAWk/zgrfkuDfBdLWZtzACvew8b2WCnsKt1L3ZxHBj5niTmp55Q/TasdkkAAAA6tO84G1st3db5WiTxVR9HZsYkuK9UldP3dsObKJ5zSYbP2AJgURUyF4hKrzVAABoPc0L3kazwVszRpscrroJY90XzYmRBmTemtB3Lw8DG6AaBJ4AAAD1aWLmbY93W3WzyRC2HaAJY+F26m822YDgLemPNtndX369BqMFX/Q45Gg03mMAALSeFmg2We1ok9MvKaq+yJgYknr3rfZV9UkMNa7ZZJOCNzIpzdDOB72d6z5z8LkGAKB1Nb/ZZFMm6R6q2Gyy8AKmrgsa5xo02uRI0Ym/o86EkXmLHse8MTiuHAMAAFpZC2TeqhuwpC2bTaYTUibVgMxbsEFQuBbrHO2cFWnnugMAALSC5mbeuvqk7r66i6p62PQaBizJqukCdKJBA4skRpgmYIap5q3eahmUVqtPMQSYHAMAAFpZEzNvAzU1mQzl+m9iqKYJuqUaL0AbNSpkIwZBQdvj4hsAAKAzNbfZZA3TBITXbLJS8BZimqBRk2knRqSeyoOVMKFz52jnwKyd6z4j8W8DAICW09xmk/31T9BdtUzGC96ibDbZ0MwbzSYBAACAmaAlMm9V9eOpd6qAib3ebZUDpdQlG7w1YrTJJg5YQialeYKc02KflWbi7QIAAFCf5mbeagig6m4BmJ0cfNbcOguqQkObTdLnbSYJEgAZYRLCwNsIAICW05zgzTkviGrGHG/Z4C3KzFtutMnmTBXQaPSpawKOORqNtxgAAC2nOcHbxKDk0rlmk/U2v6vqOjZg8DZtnrd6rmRyfd5qG+GydLkjUnflAUsadRFmtJuMXDsf8zau+ozCaQIAoHV1RbkxJ2lwPKk5o7tlklzfPA2OJZXKTI0uJpJpOeeUzjiNpzLqiU/GmM45OeeUTDv1dMVyy8oZmUhJkrrjMfWMDUiSEj1zFEtnZGZKpDJ+/Zy6YjHFTEqXKHNoPKWJVFoxMyXTGfV1x6fUJV8m45R2Tm5sSD2SRq1X8VRaPfGYRhNpzeqOK5HKKOOcuuKWW94VN/V2xUvuz56RhObNiknpCSW7+jU2ntSsrriS6YziMa9e+YYnUhpNpNTfU//pTqQyGk2k1NcTV9I/bomC7Y0mUprVFc8t747HFI95x7krZkplih+vbPndcasYpCRSmSllpPxzmd1OqfKLKXw/5S8fTaTV3xOXmSmd8d6TZt4+TaTS6u2KK5XOvn+kVNp7vrcrpuzbOh4zOeeUSGfUHYsp7ZwyzuXOcfZ4xWLePifTGaUzTrO647l9zTineGzymCTSXl0SqYx6u7wyu+NT65/9XKUz3memq+BzlEw7dcdNybRXt554TD1dMaULPo/Z7ZY6Ttnjn8o49XbFiq7j7dPU14wn07l9zO1XgHM3lkjnPp+z/XNT+HxPVyxX7+x5Kyb/fZPPuan7lN1GqfoVLi+1nnNOA6NJ9XTFNKs7Pu28FEqmM4qb95kp9blIZ5ySfl2z5zBraDzpb8d7nPHvOCfvM+yXn3ZOu0cS6u+JayyZ9o5jMp17L3bFTGPJtObM6i66fxOptH88vZ+5CAABAGiMSIO3lc/u1bGfuknH2lO6rle6+GfrdEvmpmnrff9Pz+jIA+fo479aOe25a+7fqO54TJ+74Qk98PGzNH92jy754YqS2/z6zWv05d8/mXu87GXP6BhJL7/sIe3U04HrvncsKUl63VV/mrK8r9u72PnS647Ta1+yeMpz7/vxg7r+0S36p66H9Z646ZjP3K5ylzUfOHOJLvvDGknSzf/8F3rewun92f7mG3fpgQ0DOuPwWfq2pM/fvFHfvmn6MSyshyT9+j2n6LhD5lbY0/KO+tgN05btHE7k7v9x9Xa9/bv36+iD5uiJrUOSpBccvK+W/eOpOupjN+jYxfvpkU179YO/P1GvOGrhlHLSGaejPnaD3n7KYfrkX/1ZyTrcvXanLvrWvfrZu16mlx7mZW+P/OgNevGhc/XR81+gv/3mn/S/F5+kU5fsH2ifrrl/oz5y7aO688Ona/G8yUzmv//2MX33rvX6xzOW6J/OOkqv/+8/acUzXub2mktO1oVX35O7LfSBM5fof25fp/7eLt3/0TP1v/c8o4//epWO2H+21u30+kD+6B0n6dk9Y/rXXzyiYxfvp+vee6okaclHvWP81H+epye2Dur8y+/Mlfur95wiSXp8y6Ce92/LJEmnHLlAd63dpfWfP1+S9L27vff1uh3edk7/0q3atGdU6z53fq6cL9/0pK7441q97y+P1NdvWZtbvv7z5+uEf79JXfGYJpLpXP0l6dt3Pq3PXv+4VnzsTC3Yp3fK/h7p1/nIA/bR2u3D+u37TtULF+2Xez67T/mO/viNWvsf5+YeP75lUOd+7Q5d9aaX6JwXHjRtfUl6asewzvjybbnH573oIH3jjS/JPR5PpvWCT9yY2xdJuuDKO7Xy2cGi5b3oUzdpv75u3fNvZ0iazLiv2zmS26cPnnmU3n/mEt2xZofe/O379It/eLle8tzJzP3a7cM68yu36WsXHq8Ljl+khzcO6IIr79J33/ZSnX70AVO2984frNAfHt8mSVo4p1fD4yk9/plzitZN8o7bq449WL99ZIv+5ezn6z2nHzltnZP+8w9TPoPZ/b758W26+PvLS5b9xw+dptO/dGuu/EKfvG6VPnndqinL3vKy5+ptLz9Mf/nl2/TVvztOrzlhsa5/ZIve838PlNwOAAAIT1OaTc4zrxnhgCs92MZvHtpcdPmmPWO69oFnJUnbBsclSQ9tHChZzmU3r5n6+s1euXtVXV+x/IujfNlfqW9cOf3i5/pHvWWzNaERzVKl36O/fcdkMPn4luIXmw9sGJAkrXza249RzSpbZr5s4NFIt63eIUm5wE3y9iWb+Hhkkzfa5x1rdkx7bSrjpWZ+dM+Gstu466mdkqR7nto1ZfkDGwZ0z7rdkqQ71+4MXOfr/QvXbLCT9d271kuSvnOnd17yj9/d/rbvWTe1Dlk/vX+jRhJp7RiakCRd97B3vrKBW7aO2fdI9rjkS6YzZd/bWXetnVqHXz049bOzYfeoChNP19zvHePv3b1+WnmD4yntHklMqb+k3Oduy97xknVZu937bAeptyQl05MVe2ST95qb/eCmmCe2DE15vOzRrVMejyXS015TKnCTvM/v1sHJ/SmWoPvp8o2SpDvWeO+p+9fvnvJ89rN60yqv3tn3yW1PTn+P/yFv33YMTeT+f5STDax+vmJT0edL/W/65YPPli139dbBKeUH8YM/PaPV/mf7xpXesf9DmfMFAADC1ZTgba68L/8BNX6utcJV+9JDGnG9SlaZdIzV0Q6oX+OBgqxq9mu2eRecIy548BaFUvsQ5qAm5UZTrKVfVaXXFHs+uyjobpWqc8Vt19AALdgxaFS5tQuyr7Wcq6rqUOb1pZ4qfE32cbMH8mlU/8jCYpu9nwAAzCTNCd7Myz7sKZN5CzJASC3XDH3poZqCxkoXluXqso+NBQqy8i+2Ku1bv7zgrZrMWxRKHafC3Qnjeq9YEdnt1zLATKlXlDvzpV4zbXmJQhpxeR1oOoE6gtwg5y7o0Z9ynrLll6tDhfIaOk1Chf3P7kujahB2kFR7cdk+kJNL/iL2sH7S8+86WMUz0QAAIBxNajY5pIwz7S0TRAW7QKz+6qM/Pai9ZYLGkipckZWrSX+u2WSFTeRto9Ke9ctrzjaq3gprtobpo3dWXqeUQEFEDRem1Vwc585VideEEpy6IlmdAK9r9IiUgT53AQ9AsdVaLZFT+L4o3H8rEsx464VcjxYrL/t6cxl9rfsKnRR7QufFp/f/BAAA4WlSs8lhDapfmTKbL/vruxW/WCq+7tTHXvBW/dxolZpNlrvwn23Bmk3GpmTeyu9cv99scrRdmk0WnNF6LtBzcVORd4kFyN5Mf002W1dcrIY2s9Mv8Mtvu1QZtYRhwSbyrqXc4J+74Jm3/PKzy0q/umJc2oC4NReklNj/wh8Tqvn/VFU9qiyv0qGotX6F+3vQxNO51hTHxJ6prVAAABBIU4K3BTZYtsmkVD54qeb6rLAZlddsspbgrUKzyTLPzVawZpPVxAiz/cxbkIxeVhQJjVK7EGTevMAXk2UujicTYlVk0SYrVf75KpS6wC9cp2yTzCKZt0ACvKauZpMB1g16+PPPkwXaQPmKNyLpOBmUldhmdr1ss0mb+ji0elRZXqVjUWv98n5ikiQdPXKfJGlV5rk6wdYWfQ0AAAhHU4K3hbZX21Vhkuwyz1XT96bwWq8vNVh2lMuAxVQleLPJKvq85TJvrdVsstkTMVf13ih8TYl3XbHs2GTfuoDbqGHAElfkdUGOb7DMWw0Dlvi3QQLjoMFz8cxbmTpUHOAlfIV92SrtW7WD2QSuR8iZt1oVZhZPGLpVj2Weq5+nX6EjYlt1mAUfvRIAAFSnOcGbBrTDza359fUECH3pobJ97UqplHkrZ7aNNyDz5o822WoDltRxnMLIVNQTlJRS7LxUGySWPizhX2IHOQe1ZfQaG5mHEZg2sr9f4KIrNMNtVz1KSoVNedMpPWdine7IvFBPu4MlSRfG/9iM6gEAMCNEOkl31gE2oNsyx0W+3V4l1O0SNfV5qzhgSZkrtdkBpwqoRquONlnKtGaTZQaqqBTEBcnQNFtodaujeWMgrXwQm6CWbFkt2d5WUE19Z2tMD/T+P/VaSmPXL9Yve2apZ+t+0te2qcsltdYt0l2ZF0qS3hm/Xp9PXdSgWgMAMLNFnnnr07jm2Ji2V8i8NWK0ybnyJhCudoJuKUDn/zLPzNa4hqsMsirtW79NKOViSlQRfzezRWOgqR8CllVppMdqyprymhAvvmsZTGd6Ga0ZDQTq8xa0rKJBfPP2O0hfzOnVK55pa3azyYrlVbHuC229ei2lNZlFGtj/xRpyfep1o9KgNxH48szzc3Nnxs1piRWfUBwAANQn8uDthJjXoX2wQgBVfq6nKkabzLs/17zgbaCW0SYrtGksdcE5SwnFzAUaFTJWRZ+3yWxekzuZFQg8YEmRHcwuq7Tv5fqb1dbnrfz7qXifN387AWd6KznYRcU+b9UL0sS3nnFQAv2wUsNwk0HGK6nULLIxfd6yZRd/300bbTKs7TY4iK2m/KNiGyVJb0p8RI+d/CW9JfkR/cdBl0uf3KMPLbkh12Ty7yY+Lkn6aNePwq8wAACIPng7Kfa4JGll5rDyK5YbbbKKUe/yr/X2kzecdW2TdNdmnyr6plUTvAUdBCVyJacKKP+41LKimwgQoFWTla3UDLPYLlXOmhWWUWLAkgplFAYrQfr0BRqwJFtuFW/syaqEl0XNP09BfpSp3Oct4IZLlj+9gMLRJqef29yaBXUIebTJKoO5sPr/7asRXdr1Yw242dqmeVP//5opEevLrXuve4EezByp0+IPh7JtAAAwVeTB20XxmyVJj7gjyq5XPvPmr1PlxUw281bLJN0VpwooUZXsqJBBBiypZpLu2TbeciNNSt7Fb4+SWj/rIq2fdZEO0B5J089VGJMzF5/nLXhWdvI12e2XGm2yTB1KbKdUdqaaslVhKoFSGjVmR3WZt4CjTVadeQtUbMgKRpssUcPJzFv178HgtQiuYlPvgAUu6/2IZtuEP5ebTdu/wmL+L/2X1VQTAABUIfLgbaENasz1qK7GRVVcweX/kr6fP5FsTX3earxqnF3FwCLVjGjZr/GWzLyZSa+J35l7fGvvP6lXiWAXnsVWSiWqugqu7SxVaorXwBEMm9TstbZ53prfRLfRVSibsQ248UYNYPK8zDPS2EDR5/bViE6wNUWecYorXdd2H80cLkn6bPKN3oIKh+Fn6dP02/RJdW0TAAAUF/lok0kX13fT51RcL0jTqZqbTdbQ563Wi8bZGpMUtNnk5P1KmYvZFv4IlmEweReSWf02odWz3qbM1+Zr/azdkys+5P89/3xv0IMtD2nf7n6tnzXqPf+NP5OWnCnd9TWpu1965WelB34gHXW2Dho8WufHHtRRu9ZLj62WLKazYg8oI9MhO3fqtNhGHbl3q7R2q3fiLOb9DW3zHnfNktIJKZ2Uuvt0zNhmxWNbddAzGyV3gBTvkeLd+vPYI3pX/DfakD5CWjWkM2KPKKkupRTX6x78Dx3bHdPYtvP0pvh2xZRRTE4xOZkymp2JazyeVExOuuMJ/fXgBh0fH/bWMadDbZue+/RB2pro04u6BhVXRrrxLsml9c3uFXo0c4Rm//DLeknXofpo14S2uXk60PZo4cMP6x3xTUopnvubrXHvAv2upyQzvTGzRkPxjPf5uGej3hp/zLt/32b/JJlenXpSu+Mp9SqusXjGr7VJD+7Ra+OPyDmTkz8w/J/WS939OmNsvZ4bG9e8ddul4f2keLeUSUsrf67relZqwO2jWzInaI5GderjaSnzZ1J3nxTv1hvjTyipuBKuW2PqUa+SurznSumL0ue6TteIZun0Wz6l/+3u1sDOE6UbfyZtWyktOWvy/FlMi7cN601xr/9VRjGvzit2+s+b4mmn18RWefV+eEga3Kw3xjcqrrS6lVaX0l42/OYVUnef3h9/XDHLSH9YLrm0Dhga1R96rteP0mfoINutZ93+Gs4cKD2R0NE71um82HYdvW2NdMtPpTkHSRbXIZsH9Ob4er1gcLb0p/t19DM79fXuu9S37SjpzudJmaR3nDIpXdr1hP42fru2u7na5fbVoPql394subTkMlImk3c/ra93b1JMGR1lz2pJ8lnpvz4o7btIsrhXbjqph3tHtZ95n5uNmYXSVZ+T0il9ZO+gvjLLG1DknInPa41bpLTiuY9g39gW/b7nX/THzPH6z9RFKhWVpRXTU5mD9a30+bnPuFT+/+96d1CZZwEAQK0iDd66lFa3pbXZLai4brlfwGv9ZXuuDSujuIbVV3nlwm1WeL5UfWfbhKSgzSYLe8+U1qcJ7XT7VSwzambeVBBjrkefSL1NX+y+WpIUG9td/AWrr598bXJ0cvn2Vd6fJCVHpev/ybu/5SG9XtLreyQ96f9J+p8e/3UPSWf1SHrK/wvgnySpR9J9/p/vh36Zp6RXST/7jb7dk/eiIenAuKSnH9XZ3UUKdZKyy2+WXqe8x1nbpIxMqXjMu6h+oEeKxXVufK/Ojd8vbZEO0wq9M/9Tev8yfazY9iTp997NB/O3daP06ez9ZZOrfih/nfzyfi19qbD833k375a843T79E0f6+fwXxF/1LuzWdLmX+ae/49SdZb0hi5/XrBB6dS4pF2rpF3+k0/fNmXdoyR9trCs30ze7ZX01ex5+mWZbd/h3Xww+9zdXVKsS/NT45ofkz4Z++HkuhlJ10jnSTqvR9Ia/8/3Akmf6Za0W9LvpBMknRCXtOMe6Q9527SYLo6bui2thbZ3cvlja71gLBb3g9C4FPOC1WNsTBnFNMcPznTs33nrSFLMq/Mv731WTqZ5NqRupXTIvgukeJc2TozJJoa0vw3qxt5LJUkPZo7UOneQFttOnXTTE1JMWhJ7Vpd0Xa8zJr6op9yiaYdqfxvUTk3+r5lsmlz6v9TazPRyAABA/SIP3iRpR4CgI1hgVnmlKaNNalhjXfuqcig2Xc193qoYsGTKFgKMNrlBB1Yss4oiQ3OYbdMz7kD9LH2afpY+TXM0qjs+cpaO/9xduXXetnShPnWik/Y9WFp3m7TklRoYHtFfXX6rDooN6mevP0g6+Dhp7nO9i9rRXdLABmn7Y1r2dEZXPjChv3np4br4lOdKzum8r90mk9O7/vwwfeuOp3T+iw7UJace7u21y0z+ZdJS/4Jcdk3JMX3h1/fptvWj+ui5R+nlRx/iZeTSSf3tlbfpiNgWjfUfoivecYZeddkt6lFKsyyhd7yoS//1SL9edfKx+t6fnlFG5ufeTE6mOX092jOWUkYxrf7sefr776/QrWt2eZkimeJK652vOFIb9oxp2aNbJUnrP+1lNo649DdabDt03T+cqPs2jOgTv12tfptQt1L6yrv+Rn/3zdu9H0L8TNKEujWuHq361FkySad/6Y/aNTwhk9PDnzhLJ/z772SSHvjYmf7xcHrNN+7Us3vGNKc3rpGJpExSTBnd9eHTdeoXbvH3QjI53fah06TuPv3DD+/Vqk179PXXv0jHLdrHO06pcam7T6d+bYVSLiaTtFtz9MnT99dFf7nUz3Cm9NLP3KBupdVrSfUqqTka1cG2S//+1nN0yne3K6W4vnThS/Xha+7VGS88VF9/7VFS0itbLpOr951rtun9P37Qy2j6mc57P3K69wF0GY0lUzr7q7cppoxu/efTJJfWS798v1KKK624kop75+Qz50ixLh3+0RvlFNP6z3vHfvXWIb3xst+oR0nN9vurLpglXfP24/ST5Rv1rft368KXHamLTz3cC6LiPbpj7W69/6eP6uVHLtQVb1yqXzy4WZ++bqX++vhF+uzfnJALshSLacmlkz9WZMcTXf+p84t+jlzG6Yx/m4y4D5jTq/v+5sxp633qruunPD7/Iq+8H/30Yf1i+ya9OX6Tzot5v0ocYHt0bux+9duEMhbXtalT9Nq4F43/ffxGfTR18bTy99derXaLi9axlHsyL6hqfQAAEEykwVu278XeGkZ7zFdN6JUfsMy1YY11zaltm1WOLpi1j3nNJgM1caxix/ptQiOZVmw2aTrMtuop95zcsiH1y3X1T1kvEe+TDn2R9+DFb5YkpW1CG92B2uIOko47b2rB+z7H+zv0ZD09tFar3Gr9ed8R0oFHS5Iec89IknbP/TM97EzHzX6udOgLA9X5qf5xrXLbNDj/RdIBB+eWr3BbtSL9fC2K90kH/plWuvXeE0469eDna/XDq3V6zwLt0sC0MmPq1piS3oOuXmViXcrkdTFNy8uyFOvzllFMG9yByiw4SqO7dmirdubeyJn4LA2rf9prJMn1zJHFTCM2W4PZj3b/fO3Rvt79fRbm1t0dm6/tGtWYujSk1GQh856rTe6AqQUveJ4kaVf8IG1wPRrf7wjpgKnZ801uw5THw33P8QKvbi/LvUPz/ErmV1j6xKKTNOKnpywW06hmKWNxadZ+3l+B1KyUdqlg+X6TgUVmIqUNzv9RY/8j/W2vnVZOtl6uoNuvk5vMMvl13Wnd0qEna+uTC7TGPam9sxZL8ycHXEr2x7Vb+2o4vq/UN1fp7iENarbG4/tIPcXPlSeaPoQ/TL9SP0y/smCp05dfd7w+9LOH9bv0Uv1Pz1c0rp6ir19gg9qVycu8FZZU5J/fVlVuXQEAAKoX6YAlXcpIkvYEGO0xrLmkpmXe4rU1Nay12WS/vGaTw0GaTQYoL8ub560FR5t0aR1q27TeTc0KTpsqoNhok1Vuq9wxqmWwiNLzvNVSVv15zqJz4ZWdDt5TVX1r2bcg6wTc/eLvg6hyxEW2HeB9Wap2jR5dMtziLVfe7zNL9VTmYB1o05s2dymleTasXW7f6fVr3mkCAGDGijR4y2beBmoYqj9fNaPe5a8710b8ZpON3Wa+xow26Vp2tMn9ElvVa6lpAxYECWaCXgyWO0w1jaJYabTJIk9PTtgcrNIlt1Chvo3YnwCbDfFF1RQfoN4NHm6ylvfWtHo3aLTJalVzqLa6+TrI9kxbPk9DkqRdmvy/2QKDjgIAMGNFHLx5mbcgk2QHmuetyu17zSZrzLzVOtqkjSnp4koEaKEadJLuXiXVZZmmzfN2lG3U6t636uzYfdOeWzjijeTwRObQKcsDZWtCyC1Umour6GsqzC9WNqgIHHAWL6PCNG/Tl5XZXjZADvJerSUIqmqet4BlTpmkO0DQ04y4YXL+tqmPS6nlPVhNPYKq5lht1XwdVCTztr8NStKUwZGq/eECAACEJ/Lgbcz1aKJE34p85TI11Yw2WdhscrzmzFv550sPWDLhZ8iCZBXyyiuzXp/fFLNZUwW8M369ei2p/+65TF/o+m/1KpF77rl775ekKX3epGLHp/Z2k7lgqtj6dU3SXf75oq8pUWbh8lJnv1wQ5Vx1QVau2WSAdXPr1HKcArwo8PHPWy9IcNTwed6KvS0LguJp+1/wXrBKb47aaxduaXk7u8XN14Hao5j/A1vWfD942+0m+wo3ah47AABQWeSjTe5RbcFTMdX0K+pWSvvYuMa6a8u8VRxtssTy2TU2byy3a/tY8BEsG2GR7czdf33XbXp9123Sp94mzT1Uxw94A1cMFQyqUXjBW0+ftyDXxrVcV5YKSmqKF0K4sHX+iI9Bi81liBo/k3WAVYIdgKLZxRbL6FTqrzkZdBa8x+vdbmF5DexTt9XNV7eltUB7JweXkXSwn43blressD6tdbYAAOhsdWXezOwcM1ttZmvN7NJK68cto70B+7uVbzqVbbYT3Hx5vyCPdU2/CAmiYn+0EpWZbWMaDTBYSeE2yl3AZif+HnbVz1cXhkW2U79Ov1z/L/GBos+/O/GP0xcW7E7xDEd19SgWvFfTtG/yNeWzdcWCocoD2BSWUWrbVRSi8j9YVBX01NGXrlEDlgRrNtmEPm8FQXGlXSsVzNWr2tIqxvB5BW5x8yVNBmtZh9h2pZ1ps9t/stzcy2du2Fbpu888l/vPP2JmL25GPQEAnafmzJuZxSVdKeksSZsk3W9m1znnHiv1mrjSgUaalCpcGFTTbMdfd39/UtyR7vmBtl+imJJKTtKdazZZWSwvlC6becsGb1VONh7GZW9MGT3Hdum3mZP1u8yJOmz8/yQpN0/WV3//pJbdvGba66aPnFfdSIr5ygVokxes1Qcypfu8lVbqAn368iIBoJW/wHaqrpngZOat8rq1vBca0Vxu6jmvHBw1Y7CMipm3gqa6jWo1WW0wWCnQzT/2u/w+bfNtMFfxg7RLZ8Qe1BYtUDL/q2KGN5sM+N13rqQl/t9Jkr7p3wIAUJd6Mm8nSlrrnFvnnEtIukbSBeVe0KVMoMFKKqllJPSFfvA2WmPwVmvk02/jGgmYeQuaVZhjzcu8Hag96ra0NrmFRZ+P4uK67IiANZyoSq8o1+ct8DZqybypxv1p1IAljR5tskFBZ1iCnsN2HI1xt7w+bQv80SUl6Z5Z79MLY+un9GmVGp/9bANBvvsukPQD57lH0lwzO7iwIAAAqmW1Nu0xs9dKOsc59w7/8ZslneSce2+p1xz/nB73r39/vv4t9Y6atlnM7J64RhLpKY/zZZ/729jt+nLPVfrzia9qY8EcZGHWpXDby3o+omfd/npn8p/rLi9b5qtif9IVPV/XmRNf0Fq3uMgrqyuzGkcnH9Mvej+ttyQ+rNszx00rN/9c5OvrjmssOfW5wrpknHLrlKtnsfNdbLtB97XU+6dweal9qyToawv3ZVZ3TOPJTLmXTNHfE5epdL1L7VuluhY7xqU+Z8VeV257pfax1LmrtB0nabRgf8u9pvDYBD0uQd8n1R6nfPn7Um7dIOerkn00qpWz3qH/SF6k/0m/SpK0ftZFkqQrU3+tL6YuLPq6cu/tZ/7rVSucc0sDV6JNBPnuM7PfSvq8c+5O//HNkj7snFteqtylS5e65ctLPg0A6CBmVvN3ZD3B2+sknV3wBXaic+59BetdIukS/+ELJa2saYPtb39JOyuu1blm8v7P5H2XZvb+z+R9f75zecNUdogg331mdr2kzxUEb//qnFtRUBbfj/WZyZ+vWnC8qsPxqg7Hqzo1f0fWM9rkJkmH5D1eLGlz4UrOuaslXS1JZra8E3+JDWIm77s0s/d/Ju+7NLP3f6bve7Pr0CBBvvv4fowAx6w6HK/qcLyqw/GqTj3fkfX0ebtf0hIzO9zMeiRdKOm6OsoDAKDVBfnuu07SW/xRJ0+WtNc5tyXqigIAOk/NmTfnXMrM3ivpd5Likr7jnFsVWs0AAGgxpb77zOxd/vNXSVom6TxJayWNSnp7s+oLAOgsdU3S7ZxbJu9LKqir69lem5vJ+y7N7P2fyfsuzez9Z987ULHvPj9oy953kt5TZbEde7waiGNWHY5XdThe1eF4Vafm41XzgCUAAAAAgOjU0+cNAAAAABCRSII3MzvHzFab2VozuzSKbUbBzNab2aNm9lB21Bgzm29mvzezNf7tvLz1P+Ifg9Vmdnbe8pf45aw1s8utlhmUI2Bm3zGz7Wa2Mm9ZaPtrZr1m9hN/+b1mdlikO1hGiX3/lJk965//h8zsvLznOmnfDzGzP5rZ42a2ysze7y+fKee+1P53/Pk3s1lmdp+ZPezv+6f95TPi3DeCVfg+NM/l/vOPmNmLm1HPVhHgeL3RP06PmNndZnZcsXJmikrHK2+9l5pZ2rx5C2esIMfLzE7z/8evMrPboq5jKwnwedzPzH6T950xo/v7WpFrx4Lna/t/75xr6J+8Dt1PSTpCUo+khyUd0+jtRvEnab2k/QuWfUHSpf79SyX9l3//GH/feyUd7h+TuP/cfZJeJskk3SDp3GbvW4n9fYWkF0ta2Yj9lfRuSVf59y+U9JNm73OFff+UpA8VWbfT9v1gSS/278+R9KS/jzPl3Jfa/44//3499/Hvd0u6V9LJM+XcN+B4Vvw+lDfQyQ3+cTpZ0r3NrneLH6+XS5rn3z+X41X5estf7xZ5/TZf2+x6t/LxkjRX0mOSDvUfH9Dserf48fq3vO+DhZJ2S+ppdt2beMymXTsWPF/T//soMm8nSlrrnFvnnEtIukbSBRFst1kukPR9//73Jb06b/k1zrkJ59zT8kYhO9HMDpa0r3PuT847kz/Ie01Lcc7dLu+DmC/M/c0v6+eSzsj+Ot9sJfa9lE7b9y3OuQf8+0OSHpe0SDPn3Jfa/1I6Zv+dZ9h/2O3/Oc2Qc98AQb4PL5D0A//Y3yNprn/8ZqKKx8s5d7dzbo//8B55c+rNVEGvt94n6ReStkdZuRYU5HhdJOla59wGSXLOzeRjFuR4OUlz/P/h+8i7bkpFW83WEeDasab/91EEb4skbcx7vEnlL3zaiZN0k5mtMLNL/GUHOn8+H//2AH95qeOwyL9fuLxdhLm/udc451KS9kpa0LCah+O9fqr7O3lNxzp23/0mbSfIy8DMuHNfsP/SDDj/ZhY3s4fkXej93jk3I899SIJ8H3byd2a1qj0WF8v7FXumqni8zGyRpNdIukoI8v46StI8M7vVv9Z7S2S1az1BjtcVkl4gabOkRyW93zmXiaZ6bamm//dRBG/FfkHtlCEuT3HOvVheU433mNkryqxb6jh06vGpZX/b7Vh8U9LzJB0vaYukL/vLO3LfzWwfeb/WfsA5N1hu1SLLOnH/Z8T5d86lnXPHy8tonGhmLyyzekftewME2deZdDwqCXwszOx0ecHbhxtao9YW5HhdJunDzrl046vT8oIcry5JL5F0vqSzJX3czI5qdMVaVJDjdbakhyQ9R9534xVmtm9jq9XWavp/H0XwtknSIXmPF8uLyNuec26zf7td0i/lpZS3ZVOe/m02xV7qOGzS1GYe7XZ8wtzf3GvMrEvSfgreVDFyzrlt/oVtRtL/yDv/Ugfuu5l1ywtcfuScu9ZfPGPOfbH9n0nnX5KccwOSbpV0jmbQuQ9ZkO/Djv3OrEGgY2Fmx0r6lqQLnHO7IqpbKwpyvJZKusbM1kt6raRvmNmrI6ld6wn6ebzROTfinNsp6XZJM3VQnCDH6+3ympk659xaSU9LOjqi+rWjmv7fRxG83S9piZkdbmY98jqkXxfBdhvKzGab2ZzsfUmvlLRS3r691V/trZJ+7d+/TtKF5o2sdrikJZLu85scDZnZyX4b4bfkvaYdhLm/+WW9VtItfv+YllTQLvk18s6/1GH77tf125Ied859Je+pGXHuS+3/TDj/ZrbQzOb69/sknSnpCc2Qc98AQb4Pr5P0Fn8UspMl7c02UZ2BKh4vMztU0rWS3uyce7IJdWwlFY+Xc+5w59xhzrnD5PUxfbdz7leR17Q1BPk8/lrSn5tZl5n1SzpJXr/nmSjI8dog6QxJMrMDJT1f0rpIa9leavt/76IZbeU8eSO0PSXpo1FsM4J9OkLeSDsPS1qV3S95fTVulrTGv52f95qP+sdgtfJGlJT3S9hK/7kr5E+e3mp/kn4sr3lYUt6vBReHub+SZkn6mbxBDu6TdESz97nCvv9QXpvuR/wP4MEduu+nykvjPyKvOcRD/md6ppz7Uvvf8edf0rGSHvT3caWkT/jLZ8S5b9AxnfZ9KOldkt7l3zdJV/rPPyppabPr3OLH61uS9uR9Npc3u86tfLwK1v2eZvBok0GPl6R/kTfi5Ep5zeabXu9WPV7ymkve5P/vWinpTc2uc5OPV7Frx7r/32e/PAEAAAAALSySSboBAAAAAPUheAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA38f1tmrwdPSF2lAAAAAElFTkSuQmCC"
+ ]
},
"metadata": {
"needs_background": "light"
- }
+ },
+ "output_type": "display_data"
}
],
- "metadata": {}
+ "source": [
+ "pl.figure(figsize=(15, 5))\r\n",
+ "pl.subplot(1, 2, 1)\r\n",
+ "pl.plot(spikes)\r\n",
+ "pl.plot(position_at_each_iteration / 50)\r\n",
+ "pl.ylim([0, 10])\r\n",
+ "pl.xlim([0, 30000])\r\n",
+ "\r\n",
+ "pl.subplot(1, 2, 2)\r\n",
+ "pl.plot(position_at_each_iteration[1:], spikes, \".\")\r"
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"TODO: \n",
" - Sync table under ephys module.\n",
@@ -987,17 +1024,19 @@
" ephys_element.CuratedClustering.make = ironclust_make\n",
" ephys_element.CuratedClustering.populate()\n",
" ```"
- ],
- "metadata": {}
+ ]
}
],
"metadata": {
+ "interpreter": {
+ "hash": "ad2050938946b9745fdc6507c7561603613194cae6ce583d2036ae212150e70f"
+ },
"jupytext": {
"encoding": "# -*- coding: utf-8 -*-"
},
"kernelspec": {
- "name": "python3",
- "display_name": "Python 3.7.10 64-bit ('imaging_elements': conda)"
+ "display_name": "Python 3.7.10 64-bit ('imaging_elements': conda)",
+ "name": "python3"
},
"language_info": {
"codemirror_mode": {
@@ -1010,9 +1049,6 @@
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.10"
- },
- "interpreter": {
- "hash": "ad2050938946b9745fdc6507c7561603613194cae6ce583d2036ae212150e70f"
}
},
"nbformat": 4,
diff --git a/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb b/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb
index 853a3a86..8e6d73e0 100644
--- a/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb
+++ b/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb
@@ -72,6 +72,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -97,13 +98,12 @@
}
],
"source": [
- "import os \n",
"import datajoint as dj\n",
"import numpy as np\n",
- "from u19_pipeline import imaging, acquisition, subject\n",
- "from u19_pipeline.imaging_element import imaging_element, scan_element, get_suite2p_dir\n",
+ "from u19_pipeline.imaging_element import get_suite2p_dir, imaging_element, scan_element\n",
"from u19_pipeline.ingest.imaging_element_ingest import process_scan\n",
- "import pathlib"
+ "\n",
+ "from u19_pipeline import acquisition, imaging, subject"
]
},
{
@@ -155,10 +155,10 @@
}
],
"source": [
- "subject = 'jeremyjc_j016'\n",
- "date = '2021-11-21'\n",
+ "subject = \"jeremyjc_j016\"\n",
+ "date = \"2021-11-21\"\n",
"\n",
- "key = (imaging.Scan & dict(session_date =date, subject_fullname=subject)).fetch1('KEY')\n",
+ "key = (imaging.Scan & dict(session_date=date, subject_fullname=subject)).fetch1(\"KEY\")\n",
"key"
]
},
@@ -177,7 +177,7 @@
"metadata": {},
"outputs": [],
"source": [
- "#(scan_element.ScanInfo & key).delete()"
+ "# (scan_element.ScanInfo & key).delete()"
]
},
{
@@ -227,71 +227,76 @@
"source": [
"# ingest parameters for Suite2p\n",
"pars = {\n",
- " 'look_one_level_down': 0.0,\n",
- " 'fast_disk': [],\n",
- " 'delete_bin': False,\n",
- " 'mesoscan': False,\n",
- " 'h5py': [],\n",
- " 'h5py_key': 'data',\n",
- " 'save_path0': [],\n",
- " 'subfolders': [],\n",
- " 'nplanes': 1,\n",
- " 'nchannels': 1,\n",
- " 'functional_chan': 1,\n",
- " 'tau': 1.0,\n",
- " 'fs': 10.0,\n",
- " 'force_sktiff': False,\n",
- " 'preclassify': 0.0,\n",
- " 'save_mat': False,\n",
- " 'combined': True,\n",
- " 'aspect': 1.0,\n",
- " 'do_bidiphase': False,\n",
- " 'bidiphase': 0.0,\n",
- " 'do_registration': True,\n",
- " 'keep_movie_raw': False,\n",
- " 'nimg_init': 300,\n",
- " 'batch_size': 500,\n",
- " 'maxregshift': 0.1,\n",
- " 'align_by_chan': 1,\n",
- " 'reg_tif': False,\n",
- " 'reg_tif_chan2': False,\n",
- " 'subpixel': 10,\n",
- " 'smooth_sigma': 1.15,\n",
- " 'th_badframes': 1.0,\n",
- " 'pad_fft': False,\n",
- " 'nonrigid': True,\n",
- " 'block_size': [128, 128],\n",
- " 'snr_thresh': 1.2,\n",
- " 'maxregshiftNR': 5.0,\n",
- " '1Preg': False,\n",
- " 'spatial_hp': 50.0,\n",
- " 'pre_smooth': 2.0,\n",
- " 'spatial_taper': 50.0,\n",
- " 'roidetect': True,\n",
- " 'sparse_mode': False,\n",
- " 'diameter': 12,\n",
- " 'spatial_scale': 0,\n",
- " 'connected': True,\n",
- " 'nbinned': 5000,\n",
- " 'max_iterations': 20,\n",
- " 'threshold_scaling': 1.0,\n",
- " 'max_overlap': 0.75,\n",
- " 'high_pass': 100.0,\n",
- " 'inner_neuropil_radius': 2,\n",
- " 'min_neuropil_pixels': 350,\n",
- " 'allow_overlap': False,\n",
- " 'chan2_thres': 0.65,\n",
- " 'baseline': 'maximin',\n",
- " 'win_baseline': 60.0,\n",
- " 'sig_baseline': 10.0,\n",
- " 'prctile_baseline': 8.0,\n",
- " 'neucoeff': 0.7,\n",
- " 'xrange': np.array([0, 0]),\n",
- " 'yrange': np.array([0, 0])}\n",
+ " \"look_one_level_down\": 0.0,\n",
+ " \"fast_disk\": [],\n",
+ " \"delete_bin\": False,\n",
+ " \"mesoscan\": False,\n",
+ " \"h5py\": [],\n",
+ " \"h5py_key\": \"data\",\n",
+ " \"save_path0\": [],\n",
+ " \"subfolders\": [],\n",
+ " \"nplanes\": 1,\n",
+ " \"nchannels\": 1,\n",
+ " \"functional_chan\": 1,\n",
+ " \"tau\": 1.0,\n",
+ " \"fs\": 10.0,\n",
+ " \"force_sktiff\": False,\n",
+ " \"preclassify\": 0.0,\n",
+ " \"save_mat\": False,\n",
+ " \"combined\": True,\n",
+ " \"aspect\": 1.0,\n",
+ " \"do_bidiphase\": False,\n",
+ " \"bidiphase\": 0.0,\n",
+ " \"do_registration\": True,\n",
+ " \"keep_movie_raw\": False,\n",
+ " \"nimg_init\": 300,\n",
+ " \"batch_size\": 500,\n",
+ " \"maxregshift\": 0.1,\n",
+ " \"align_by_chan\": 1,\n",
+ " \"reg_tif\": False,\n",
+ " \"reg_tif_chan2\": False,\n",
+ " \"subpixel\": 10,\n",
+ " \"smooth_sigma\": 1.15,\n",
+ " \"th_badframes\": 1.0,\n",
+ " \"pad_fft\": False,\n",
+ " \"nonrigid\": True,\n",
+ " \"block_size\": [128, 128],\n",
+ " \"snr_thresh\": 1.2,\n",
+ " \"maxregshiftNR\": 5.0,\n",
+ " \"1Preg\": False,\n",
+ " \"spatial_hp\": 50.0,\n",
+ " \"pre_smooth\": 2.0,\n",
+ " \"spatial_taper\": 50.0,\n",
+ " \"roidetect\": True,\n",
+ " \"sparse_mode\": False,\n",
+ " \"diameter\": 12,\n",
+ " \"spatial_scale\": 0,\n",
+ " \"connected\": True,\n",
+ " \"nbinned\": 5000,\n",
+ " \"max_iterations\": 20,\n",
+ " \"threshold_scaling\": 1.0,\n",
+ " \"max_overlap\": 0.75,\n",
+ " \"high_pass\": 100.0,\n",
+ " \"inner_neuropil_radius\": 2,\n",
+ " \"min_neuropil_pixels\": 350,\n",
+ " \"allow_overlap\": False,\n",
+ " \"chan2_thres\": 0.65,\n",
+ " \"baseline\": \"maximin\",\n",
+ " \"win_baseline\": 60.0,\n",
+ " \"sig_baseline\": 10.0,\n",
+ " \"prctile_baseline\": 8.0,\n",
+ " \"neucoeff\": 0.7,\n",
+ " \"xrange\": np.array([0, 0]),\n",
+ " \"yrange\": np.array([0, 0]),\n",
+ "}\n",
"\n",
"\n",
"imaging_element.ProcessingParamSet.insert_new_params(\n",
- " 'suite2p', 0, 'Calcium imaging analysis with Suite2p using default Suite2p parameters', pars)"
+ " \"suite2p\",\n",
+ " 0,\n",
+ " \"Calcium imaging analysis with Suite2p using default Suite2p parameters\",\n",
+ " pars,\n",
+ ")"
]
},
{
@@ -417,7 +422,7 @@
"metadata": {},
"outputs": [],
"source": [
- "#(imaging_element.ProcessingTask() & key).delete()"
+ "# (imaging_element.ProcessingTask() & key).delete()"
]
},
{
@@ -542,10 +547,13 @@
}
],
"source": [
- "scan_keys = (scan_element.Scan & key).fetch('KEY')\n",
+ "scan_keys = (scan_element.Scan & key).fetch(\"KEY\")\n",
"for scan_key in scan_keys:\n",
" output_dir = get_suite2p_dir(scan_key)\n",
- " imaging_element.ProcessingTask.insert1(dict(**scan_key, paramset_idx=0, processing_output_dir=output_dir), skip_duplicates=True)\n",
+ " imaging_element.ProcessingTask.insert1(\n",
+ " dict(**scan_key, paramset_idx=0, processing_output_dir=output_dir),\n",
+ " skip_duplicates=True,\n",
+ " )\n",
"\n",
"imaging_element.ProcessingTask() & key"
]
@@ -716,7 +724,7 @@
"metadata": {},
"outputs": [],
"source": [
- "processing_keys = imaging_element.Processing.fetch('KEY')\n",
+ "processing_keys = imaging_element.Processing.fetch(\"KEY\")\n",
"for processing_key in processing_keys:\n",
" imaging_element.Curation().create1_from_processing_task(processing_key)"
]
@@ -1651,7 +1659,7 @@
"metadata": {},
"outputs": [],
"source": [
- "scan_key = (scan_element.Scan & key).fetch('KEY')[0]"
+ "scan_key = (scan_element.Scan & key).fetch(\"KEY\")[0]"
]
},
{
@@ -1660,8 +1668,10 @@
"metadata": {},
"outputs": [],
"source": [
- "px_height, px_width = (scan_element.ScanInfo().Field & scan_key).fetch1('px_height', 'px_width')\n",
- "masks = (imaging_element.Segmentation.Mask & scan_key & 'curation_id=1').fetch()"
+ "px_height, px_width = (scan_element.ScanInfo().Field & scan_key).fetch1(\n",
+ " \"px_height\", \"px_width\"\n",
+ ")\n",
+ "masks = (imaging_element.Segmentation.Mask & scan_key & \"curation_id=1\").fetch()"
]
},
{
@@ -1680,7 +1690,7 @@
"outputs": [],
"source": [
"for mask in masks:\n",
- " img[mask['mask_xpix'], mask['mask_ypix']] = mask['mask_weights']"
+ " img[mask[\"mask_xpix\"], mask[\"mask_ypix\"]] = mask[\"mask_weights\"]"
]
},
{
@@ -1690,6 +1700,7 @@
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib notebook"
]
},
diff --git a/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb b/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb
index 71a731c7..24afd8fe 100644
--- a/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb
+++ b/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb
@@ -29,6 +29,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -50,7 +51,6 @@
],
"source": [
"import datajoint as dj\n",
- "import u19_pipeline.utility as utility\n",
"\n",
"dj.__version__"
]
@@ -61,7 +61,7 @@
"metadata": {},
"outputs": [],
"source": [
- "db_name = 'u19_'"
+ "db_name = \"u19_\""
]
},
{
@@ -71,9 +71,9 @@
"outputs": [],
"source": [
"connect_mod = lambda x: dj.VirtualModule(x, db_name + x)\n",
- "lab = connect_mod('lab')\n",
- "subject = connect_mod('subject')\n",
- "scheduler = connect_mod('scheduler')\n",
+ "lab = connect_mod(\"lab\")\n",
+ "subject = connect_mod(\"subject\")\n",
+ "scheduler = connect_mod(\"scheduler\")\n",
"# lab = dj.VirtualModule('lab', db_name + 'lab')\n",
"# lab = dj.VirtualModule('lab', db_name + 'lab')"
]
@@ -85,7 +85,6 @@
"outputs": [],
"source": [
"# lab.Location()\n",
- "import u19_pipeline.scheduler\n",
"# dj.Diagram(lab)"
]
},
@@ -95,9 +94,10 @@
"metadata": {},
"outputs": [],
"source": [
- "from u19_pipeline import scheduler\n",
"# scheduler = connect_mod('scheduler')\n",
- "import json"
+ "import json\n",
+ "\n",
+ "from u19_pipeline import scheduler"
]
},
{
@@ -145,7 +145,7 @@
"source": [
"# scheduler.BehaviorProfile.drop()\n",
"# scheduler.RecordingProfile.drop()\n",
- "scheduler.Schedule.drop()\n"
+ "scheduler.Schedule.drop()"
]
},
{
@@ -165,8 +165,7 @@
}
],
"source": [
- "import json\n",
- "behavior_profile_variables = {'nested_key': 'nested_value'}\n",
+ "behavior_profile_variables = {\"nested_key\": \"nested_value\"}\n",
"json.dumps(behavior_profile_variables)\n",
"# json.dumps(behavior_profile_variables, separators=(',', \":\"))"
]
@@ -177,22 +176,22 @@
"metadata": {},
"outputs": [],
"source": [
- "import numpy as np\n",
"import json\n",
- "json_dump = json.dumps(behavior_profile_variables, separators=(',', \":\"))\n",
+ "\n",
+ "import numpy as np\n",
+ "\n",
+ "json_dump = json.dumps(behavior_profile_variables, separators=(\",\", \":\"))\n",
"length = len(json_dump)\n",
"# Encode the string using Latin-1\n",
"# byte_array = bytearray(json_dump, 'latin1')\n",
"\n",
"\n",
- "\n",
- "\n",
"behavior_dict = {\n",
- " 'user_id': 'testuser',\n",
- " 'date_created': '2024-10-01',\n",
- " 'behavior_profile_name': 'TestUserBehavior1',\n",
- " 'behavior_profile_description': 'This is a sample profile description.',\n",
- " 'behavior_profile_variables': np.array(['hi']), # Example binary data\n",
+ " \"user_id\": \"testuser\",\n",
+ " \"date_created\": \"2024-10-01\",\n",
+ " \"behavior_profile_name\": \"TestUserBehavior1\",\n",
+ " \"behavior_profile_description\": \"This is a sample profile description.\",\n",
+ " \"behavior_profile_variables\": np.array([\"hi\"]), # Example binary data\n",
" # 'behavior_profile_variables': np.array(['hi'],dtype=f'U{length}') # Example binary data\n",
" # 'behavior_profile_variables': np.array([json_dump],dtype=f'U{length}') # Example binary data\n",
" # 'behavior_profile_variables': # Example binary data\n",
@@ -216,7 +215,7 @@
}
],
"source": [
- "repr(np.array(['hi']))\n",
+ "repr(np.array([\"hi\"]))\n",
"scheduler.BehaviorProfile.insert1()"
]
},
@@ -248,7 +247,7 @@
}
],
"source": [
- "(scheduler.BehaviorProfile).fetch(as_dict=True)\n"
+ "(scheduler.BehaviorProfile).fetch(as_dict=True)"
]
},
{
@@ -276,7 +275,7 @@
}
],
"source": [
- "(scheduler.BehaviorProfile & 'behavior_profile_id > 1').delete()\n"
+ "(scheduler.BehaviorProfile & \"behavior_profile_id > 1\").delete()"
]
},
{
@@ -303,7 +302,6 @@
}
],
"source": [
- "\n",
"scheduler.BehaviorProfile.heading\n",
"\n",
"# scheduler.RecordingProfile.heading"
@@ -315,12 +313,13 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
"recording_dict = {\n",
- " 'date_created': '2024-10-28',\n",
- " 'recording_profile_name': '',\n",
- " 'recording_profile_description': 'This is a sample profile description.',\n",
- " 'recording_profile_variables': {'nested_key': 'nested_value'} # Example binary data\n",
+ " \"date_created\": \"2024-10-28\",\n",
+ " \"recording_profile_name\": \"\",\n",
+ " \"recording_profile_description\": \"This is a sample profile description.\",\n",
+ " \"recording_profile_variables\": {\n",
+ " \"nested_key\": \"nested_value\"\n",
+ " }, # Example binary data\n",
"}"
]
},
@@ -330,7 +329,6 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
"scheduler.RecordingProfile.insert1(recording_dict)"
]
}
diff --git a/notebooks/developer_notebooks/brightness_video_analysis.ipynb b/notebooks/developer_notebooks/brightness_video_analysis.ipynb
index a518197c..683b7215 100644
--- a/notebooks/developer_notebooks/brightness_video_analysis.ipynb
+++ b/notebooks/developer_notebooks/brightness_video_analysis.ipynb
@@ -22,18 +22,14 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"\n",
- "import datajoint as dj\n",
- "import pandas as pd\n",
- "import numpy as np\n",
- "import pylab as plt\n",
- "import matplotlib.patches as mpatches\n",
- "from os.path import exists\n",
- "import cv2\n",
"\n",
- "from matplotlib import cm\n",
- "from u19_pipeline import utility as uti\n"
+ "import cv2\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import pylab as plt"
]
},
{
@@ -54,9 +50,9 @@
}
],
"source": [
- "import numpy as np\n",
"import json\n",
- "npy_data = np.load('/Users/alvaros/Downloads/ops_mika.npy', allow_pickle=True)\n",
+ "\n",
+ "npy_data = np.load(\"/Users/alvaros/Downloads/ops_mika.npy\", allow_pickle=True)\n",
"dict_data = npy_data[()]\n",
"with open(\"/Users/alvaros/Downloads/ops_mika.json\", \"w\") as write_file:\n",
" json.dump(dict_data, write_file)\n",
@@ -93,11 +89,11 @@
"from u19_pipeline.imaging_pipeline import imaging_element as ie\n",
"\n",
"query = dict()\n",
- "query['paramset_idx'] = 1\n",
+ "query[\"paramset_idx\"] = 1\n",
"\n",
"imaging_paramset = (ie.ProcessingParamSet & query).fetch(as_dict=True)\n",
"\n",
- "imset = imaging_paramset[0]['params']\n"
+ "imset = imaging_paramset[0][\"params\"]"
]
},
{
@@ -161,10 +157,11 @@
" shared_keys = d1_keys.intersection(d2_keys)\n",
" added = d1_keys - d2_keys\n",
" removed = d2_keys - d1_keys\n",
- " modified = {o : (d1[o], d2[o]) for o in shared_keys if d1[o] != d2[o]}\n",
+ " modified = {o: (d1[o], d2[o]) for o in shared_keys if d1[o] != d2[o]}\n",
" same = set(o for o in shared_keys if d1[o] == d2[o])\n",
" return added, removed, modified, same\n",
"\n",
+ "\n",
"added, removed, modified, same = dict_compare(imset, a)"
]
},
@@ -188,12 +185,15 @@
"metadata": {},
"outputs": [],
"source": [
- "npy_manual = np.load('/Users/alvaros/Downloads/ops_dir/ops_manual.npy', allow_pickle=True)\n",
- "npy_pipeline = np.load('/Users/alvaros/Downloads/ops_dir/ops_pipeline.npy', allow_pickle=True)\n",
+ "npy_manual = np.load(\n",
+ " \"/Users/alvaros/Downloads/ops_dir/ops_manual.npy\", allow_pickle=True\n",
+ ")\n",
+ "npy_pipeline = np.load(\n",
+ " \"/Users/alvaros/Downloads/ops_dir/ops_pipeline.npy\", allow_pickle=True\n",
+ ")\n",
"\n",
"npy_manual_dict = npy_manual.tolist()\n",
- "npy_pipeline_dict = npy_pipeline.tolist()\n",
- "\n"
+ "npy_pipeline_dict = npy_pipeline.tolist()"
]
},
{
@@ -1080,17 +1080,30 @@
}
],
"source": [
- "cols_trials = ['subject_fullname', 'session_date', 'session_number', 'block', 'trial_idx', 'cue_presence_left', 'cue_presence_right', 'cue_onset_left', 'cue_onset_right', 'video_path']\n",
- "order_by=('block', 'trial_idx')\n",
+ "cols_trials = [\n",
+ " \"subject_fullname\",\n",
+ " \"session_date\",\n",
+ " \"session_number\",\n",
+ " \"block\",\n",
+ " \"trial_idx\",\n",
+ " \"cue_presence_left\",\n",
+ " \"cue_presence_right\",\n",
+ " \"cue_onset_left\",\n",
+ " \"cue_onset_right\",\n",
+ " \"video_path\",\n",
+ "]\n",
+ "order_by = (\"block\", \"trial_idx\")\n",
"\n",
"for i in range(1):\n",
- "\n",
- " key_dict = blocks_video.loc[i+10, :].to_dict()\n",
- " video_df = pd.DataFrame(((behavior.TowersBlockTrialVideo * behavior.TowersBlock.Trial) & key_dict).fetch(*cols_trials, as_dict=True, order_by=order_by))\n",
- "\n",
+ " key_dict = blocks_video.loc[i + 10, :].to_dict()\n",
+ " video_df = pd.DataFrame(\n",
+ " (\n",
+ " (behavior.TowersBlockTrialVideo * behavior.TowersBlock.Trial) & key_dict\n",
+ " ).fetch(*cols_trials, as_dict=True, order_by=order_by)\n",
+ " )\n",
"\n",
" for j in range(1):\n",
- " path_video = video_df.loc[j, 'video_path']\n",
+ " path_video = video_df.loc[j, \"video_path\"]\n",
" path_video = path_video.replace(drive_videos, root_path)\n",
" print(path_video)\n",
"\n",
@@ -1099,21 +1112,18 @@
" cap = cv2.VideoCapture(path_video)\n",
"\n",
" # Check if camera opened successfully\n",
- " if (cap.isOpened()== False): \n",
+ " if cap.isOpened() == False:\n",
" print(\"Error opening video stream or file\")\n",
"\n",
" total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
- " mean_value = np.zeros((total_frames))\n",
+ " mean_value = np.zeros(total_frames)\n",
" for k in range(total_frames):\n",
- " \n",
" # Capture frame-by-frame\n",
" ret, frame = cap.read()\n",
" if ret == True:\n",
" mean_value[k] = np.mean(frame.flatten())\n",
"\n",
- "\n",
- " cap.release()\n",
- "\n"
+ " cap.release()"
]
},
{
@@ -1274,7 +1284,7 @@
}
],
"source": [
- "video_df.head()\n"
+ "video_df.head()"
]
},
{
@@ -1297,16 +1307,15 @@
}
],
"source": [
- " \n",
- "fig, axs = plt.subplots(1, 1, figsize=(15,12))\n",
+ "fig, axs = plt.subplots(1, 1, figsize=(15, 12))\n",
"\n",
- "plt.plot(mean_value, linewidth=3, color ='k')\n",
+ "plt.plot(mean_value, linewidth=3, color=\"k\")\n",
"\n",
"plt.xticks(fontsize=16)\n",
"plt.yticks(fontsize=16)\n",
"plt.xlabel(\"Iteration # \", fontsize=18)\n",
- "plt.ylabel(\"Mean value RGB \", fontsize=18);\n",
- "plt.title('Color change across time towers task', fontsize=20);\n"
+ "plt.ylabel(\"Mean value RGB \", fontsize=18)\n",
+ "plt.title(\"Color change across time towers task\", fontsize=20);"
]
},
{
diff --git a/notebooks/developer_notebooks/delete_logs_periodically.ipynb b/notebooks/developer_notebooks/delete_logs_periodically.ipynb
index dd0c7c91..0ffd2076 100644
--- a/notebooks/developer_notebooks/delete_logs_periodically.ipynb
+++ b/notebooks/developer_notebooks/delete_logs_periodically.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -31,11 +32,11 @@
"metadata": {},
"outputs": [],
"source": [
- "import pandas as pd\n",
+ "import datetime\n",
"import pathlib\n",
"import re\n",
- "import datetime\n",
- "\n"
+ "\n",
+ "import pandas as pd"
]
},
{
@@ -65,10 +66,10 @@
}
],
"source": [
- "#Behavior data\n",
- "directory_path = pathlib.Path('Z:\\Shared\\log')\n",
+ "# Behavior data\n",
+ "directory_path = pathlib.Path(\"Z:\\Shared\\log\")\n",
"date_pattern = r\"_\\d{8}_\"\n",
- "reference_date = datetime.date.today() - datetime.timedelta(days=1)\n"
+ "reference_date = datetime.date.today() - datetime.timedelta(days=1)"
]
},
{
@@ -77,8 +78,11 @@
"metadata": {},
"outputs": [],
"source": [
- "files = [p for p in directory_path.rglob(\"*\") if p.is_file() and re.search(date_pattern, p.name)]\n",
- "\n"
+ "files = [\n",
+ " p\n",
+ " for p in directory_path.rglob(\"*\")\n",
+ " if p.is_file() and re.search(date_pattern, p.name)\n",
+ "]"
]
},
{
@@ -87,18 +91,23 @@
"metadata": {},
"outputs": [],
"source": [
- "files = [p for p in directory_path.rglob(\"*\") if p.is_file() and re.search(date_pattern, p.name)]\n",
- "\n",
+ "files = [\n",
+ " p\n",
+ " for p in directory_path.rglob(\"*\")\n",
+ " if p.is_file() and re.search(date_pattern, p.name)\n",
+ "]\n",
"\n",
- "files_df = pd.DataFrame(files, columns=['filepaths'])\n",
- "files_df['filename'] = files_df['filepaths'].apply(extract_filename)\n",
- "files_df['exctract_date'] = files_df['filename'].str.extract(r'(\\d{8,' + str(8) + r'})')\n",
- "files_df['exctract_date'] = pd.to_datetime(files_df['exctract_date'], format='%Y%m%d')\n",
- "files_df['before_ref_date'] = files_df['exctract_date'] <= pd.Timestamp(reference_date)\n",
"\n",
+ "files_df = pd.DataFrame(files, columns=[\"filepaths\"])\n",
+ "files_df[\"filename\"] = files_df[\"filepaths\"].apply(extract_filename)\n",
+ "files_df[\"exctract_date\"] = files_df[\"filename\"].str.extract(r\"(\\d{8,\" + str(8) + r\"})\")\n",
+ "files_df[\"exctract_date\"] = pd.to_datetime(files_df[\"exctract_date\"], format=\"%Y%m%d\")\n",
+ "files_df[\"before_ref_date\"] = files_df[\"exctract_date\"] <= pd.Timestamp(reference_date)\n",
"\n",
"\n",
- "files_for_deletion = files_df.loc[files_df['before_ref_date']==True, 'filepaths'].to_list()"
+ "files_for_deletion = files_df.loc[\n",
+ " files_df[\"before_ref_date\"] == True, \"filepaths\"\n",
+ "].to_list()"
]
},
{
diff --git a/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb b/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb
index 008fb014..f35bbe46 100644
--- a/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb
+++ b/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -42,9 +43,7 @@
}
],
"source": [
- "\n",
- "import datajoint as dj\n",
- "\n"
+ "import datajoint as dj"
]
},
{
@@ -61,7 +60,7 @@
}
],
"source": [
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')"
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")"
]
},
{
@@ -78,17 +77,18 @@
}
],
"source": [
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
"\n",
"old_session_stats_query = \"current_datetime < NOW() - INTERVAL 15 DAY\"\n",
"\n",
"order_cols = \"'subject_fullname', 'session_date', 'session_number', 'trial_idx'\"\n",
"connection = acquisition.LiveSessionStats.connection\n",
"with connection.transaction:\n",
- " old_session_stats = (acquisition.LiveSessionStats & old_session_stats_query).fetch(as_dict=True, order_by=[order_cols])\n",
+ " old_session_stats = (acquisition.LiveSessionStats & old_session_stats_query).fetch(\n",
+ " as_dict=True, order_by=[order_cols]\n",
+ " )\n",
" (acquisition.HistoricSessionStats).insert(old_session_stats, skip_duplicates=True)\n",
- " (acquisition.LiveSessionStats & old_session_stats_query).delete(safemode=False)\n",
- "\n"
+ " (acquisition.LiveSessionStats & old_session_stats_query).delete(safemode=False)"
]
},
{
diff --git a/notebooks/developer_notebooks/query_for_tech_home_based_off_alvaros_big_query.ipynb b/notebooks/developer_notebooks/query_for_tech_home_based_off_alvaros_big_query.ipynb
index b1aea32f..6f72c408 100644
--- a/notebooks/developer_notebooks/query_for_tech_home_based_off_alvaros_big_query.ipynb
+++ b/notebooks/developer_notebooks/query_for_tech_home_based_off_alvaros_big_query.ipynb
@@ -420,12 +420,17 @@
],
"source": [
"# Get max effective_date per subject\n",
- "max_dates = dj.U('subject_fullname').aggr(\n",
- " action.SubjectStatus,\n",
- " last_status='MAX(effective_date)'\n",
+ "max_dates = dj.U(\"subject_fullname\").aggr(\n",
+ " action.SubjectStatus, last_status=\"MAX(effective_date)\"\n",
")\n",
"\n",
- "sub_cag_status_exclusion = (action.SubjectStatus & max_dates.proj(effective_date='last_status')).proj('subject_status','water_per_day','schedule',unused_date ='effective_date') * subject.Subject() * subject.CagingStatus()\n",
+ "sub_cag_status_exclusion = (\n",
+ " (action.SubjectStatus & max_dates.proj(effective_date=\"last_status\")).proj(\n",
+ " \"subject_status\", \"water_per_day\", \"schedule\", unused_date=\"effective_date\"\n",
+ " )\n",
+ " * subject.Subject()\n",
+ " * subject.CagingStatus()\n",
+ ")\n",
"display(sub_cag_status_exclusion)"
]
},
@@ -1682,34 +1687,37 @@
],
"source": [
"# Obtain if training occured today\n",
- "today_training_sessions = dj.U('subject_fullname').aggr(\n",
+ "today_training_sessions = dj.U(\"subject_fullname\").aggr(\n",
" acquisition.Session() & f'DATE(session_date) = \"{today}\"',\n",
- " last_session='MAX(session_date)'\n",
+ " last_session=\"MAX(session_date)\",\n",
")\n",
"\n",
- "todays_water_admin = (action.WaterAdministration & f'DATE(administration_date) = \"{today}\"').proj(\n",
+ "todays_water_admin = (\n",
+ " action.WaterAdministration & f'DATE(administration_date) = \"{today}\"'\n",
+ ").proj(\n",
" \"earned\",\n",
" \"supplement\",\n",
" \"received\",\n",
" \"watertype_name\",\n",
- " effective_date = 'administration_date')\n",
+ " effective_date=\"administration_date\",\n",
+ ")\n",
"\n",
"\n",
"# First, get the max weighing times for today\n",
- "today_max_weighing_times = dj.U('subject_fullname').aggr(\n",
+ "today_max_weighing_times = dj.U(\"subject_fullname\").aggr(\n",
" action.Weighing() & f'DATE(weighing_time) = \"{today}\"', # Filter for today\n",
" # last_weigh='MAX(weighing_time)',\n",
- " weigh='round(avg(weight),2)',\n",
- " effective_date = 'date(MAX(weighing_time))'\n",
+ " weigh=\"round(avg(weight),2)\",\n",
+ " effective_date=\"date(MAX(weighing_time))\",\n",
")\n",
"\n",
"\n",
"# Water and training. Let's add weighing\n",
- "weight_water_training = todays_water_admin.join(today_training_sessions, left=True).join(\n",
- " today_max_weighing_times, left=True\n",
- ")\n",
+ "weight_water_training = todays_water_admin.join(\n",
+ " today_training_sessions, left=True\n",
+ ").join(today_max_weighing_times, left=True)\n",
"\n",
- "weight_water_training_fetch = weight_water_training.fetch(format='frame')\n",
+ "weight_water_training_fetch = weight_water_training.fetch(format=\"frame\")\n",
"weight_water_training_fetch"
]
},
@@ -3738,7 +3746,7 @@
" sub_cag_status_exclusion.join(weight_water_training, left=True)\n",
" & 'subject_status not in (\"Dead\", \"Missing\", \"AdLibWater\")'\n",
")\n",
- "all_together_fetch = all_together.fetch(format='frame')\n",
+ "all_together_fetch = all_together.fetch(format=\"frame\")\n",
"all_together_fetch"
]
},
diff --git a/notebooks/developer_notebooks/review_ephys_sync.ipynb b/notebooks/developer_notebooks/review_ephys_sync.ipynb
index cefe7767..8afe7e0c 100644
--- a/notebooks/developer_notebooks/review_ephys_sync.ipynb
+++ b/notebooks/developer_notebooks/review_ephys_sync.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -42,13 +43,10 @@
"source": [
"import datetime\n",
"import pathlib\n",
+ "\n",
"import datajoint as dj\n",
- "import numpy as np\n",
- "import pylab as pl\n",
- "import matplotlib\n",
"import matplotlib.pyplot as plt\n",
- "from scipy.signal import find_peaks\n",
- "\n"
+ "import numpy as np"
]
},
{
@@ -66,10 +64,9 @@
}
],
"source": [
- "from u19_pipeline import acquisition\n",
- "from u19_pipeline import ephys_pipeline\n",
- "recording = dj.create_virtual_module(\"u19_recording\",\"u19_recording\")\n",
- "\n"
+ "from u19_pipeline import acquisition, ephys_pipeline\n",
+ "\n",
+ "recording = dj.create_virtual_module(\"u19_recording\", \"u19_recording\")"
]
},
{
@@ -111,11 +108,10 @@
}
],
"source": [
- "key = {'subject_fullname': 'jyanar_ya008',\n",
- " 'session_date': datetime.date(2024, 5, 28)}\n",
+ "key = {\"subject_fullname\": \"jyanar_ya008\", \"session_date\": datetime.date(2024, 5, 28)}\n",
"\n",
"\n",
- "session_key = (acquisition.Session & key).fetch1('KEY')\n",
+ "session_key = (acquisition.Session & key).fetch1(\"KEY\")\n",
"session_key"
]
},
@@ -136,7 +132,9 @@
}
],
"source": [
- "id_recording = (recording.Recording.BehaviorSession & session_key).fetch('recording_id', as_dict=True)[0]\n",
+ "id_recording = (recording.Recording.BehaviorSession & session_key).fetch(\n",
+ " \"recording_id\", as_dict=True\n",
+ ")[0]\n",
"id_recording"
]
},
@@ -199,7 +197,7 @@
}
],
"source": [
- "plt.plot(behavior_sync[0]['trial_index_nidq'])"
+ "plt.plot(behavior_sync[0][\"trial_index_nidq\"])"
]
},
{
@@ -260,18 +258,22 @@
"source": [
"session_dir = pathlib.Path(get_session_directory(key))\n",
"print(session_dir)\n",
- "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
- "nidq_bin_full_path = list(session_dir.glob('*nidq.bin*'))[0]\n",
+ "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
+ "nidq_bin_full_path = list(session_dir.glob(\"*nidq.bin*\"))[0]\n",
"\n",
- "#Nidaq file\n",
- "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n",
+ "# Nidaq file\n",
+ "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n",
"nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n",
- "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta)\n",
+ "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " nidq_bin_full_path, nidq_meta\n",
+ ")\n",
"\n",
- "#Behavior data\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n",
+ "# Behavior data\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
"thissession = behavior.TowersBlock().Trial() & key\n",
- "behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n"
+ "behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n",
+ " \"trial_time\", \"vi_start\", \"iterations\"\n",
+ ")"
]
},
{
@@ -303,7 +305,7 @@
}
],
"source": [
- "plt.plot(digital_array[1,97225711-10000:97225711+1000])"
+ "plt.plot(digital_array[1, 97225711 - 10000 : 97225711 + 1000])"
]
},
{
@@ -335,8 +337,8 @@
}
],
"source": [
- "plt.plot(digital_array[1,97225711-1000:97225711+1000])\n",
- "plt.plot(digital_array[2,97225711-1000:97225711+1000])"
+ "plt.plot(digital_array[1, 97225711 - 1000 : 97225711 + 1000])\n",
+ "plt.plot(digital_array[2, 97225711 - 1000 : 97225711 + 1000])"
]
},
{
@@ -1861,21 +1863,34 @@
}
],
"source": [
- "mode = None #Default for sessions before 12/01/2021\n",
- "#mode = 'pulses' #Default for sessions after 12/01/2021\n",
- "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[1,:], digital_array[2,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n",
+ "mode = None # Default for sessions before 12/01/2021\n",
+ "# mode = 'pulses' #Default for sessions after 12/01/2021\n",
+ "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n",
+ " digital_array[1, :],\n",
+ " digital_array[2, :],\n",
+ " nidq_sampling_rate,\n",
+ " behavior_time.shape[0],\n",
+ " behavior_time,\n",
+ " mode=mode,\n",
+ ")\n",
"# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n",
"\n",
"\n",
"# Check # of trials and iterations match\n",
- "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n",
+ "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n",
+ " ephys_utils.assert_iteration_samples_count(\n",
+ " iteration_dict[\"iter_start_idx\"], behavior_time\n",
+ " )\n",
+ ")\n",
"\n",
"print(trial_count_diff)\n",
"print(trials_diff_iteration_big)\n",
"print(trials_diff_iteration_small)\n",
"\n",
"\n",
- "status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small)\n"
+ "status = ephys_utils.evaluate_sync_process(\n",
+ " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small\n",
+ ")"
]
},
{
@@ -1904,7 +1919,7 @@
"metadata": {},
"outputs": [],
"source": [
- "it_times = iteration_dict['iter_times_idx'][trials_diff_iteration_small[0]]\n",
+ "it_times = iteration_dict[\"iter_times_idx\"][trials_diff_iteration_small[0]]\n",
"beh_times = behavior_time[trials_diff_iteration_small[0]].flatten()"
]
},
@@ -1926,11 +1941,12 @@
],
"source": [
"if it_times.shape[0] >= beh_times.shape[0]:\n",
- " print('More pulses than behavior iterations, check other method')\n",
+ " print(\"More pulses than behavior iterations, check other method\")\n",
"else:\n",
- " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n",
+ " diff_vector = np.diff(it_times - beh_times[: it_times.shape[0]])\n",
"\n",
"import scipy\n",
+ "\n",
"peaks, _ = scipy.signal.find_peaks(diff_vector, height=0.05, distance=20)\n",
"peaks"
]
@@ -1965,7 +1981,7 @@
"metadata": {},
"outputs": [],
"source": [
- "difo_iters = np.diff(iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]])\n",
+ "difo_iters = np.diff(iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]])\n",
"peaks2, _ = scipy.signal.find_peaks(difo_iters, height=1500, distance=20)"
]
},
@@ -1986,16 +2002,16 @@
}
],
"source": [
- "original_iter = iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]]\n",
- "new_iter_start = iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]]\n",
+ "original_iter = iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]]\n",
+ "new_iter_start = iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]]\n",
"\n",
"for i in range(peaks.shape[0]):\n",
- " value_insert = (original_iter[peaks[i]] + original_iter[peaks[i]+1] ) /2\n",
+ " value_insert = (original_iter[peaks[i]] + original_iter[peaks[i] + 1]) / 2\n",
" new_iter_start = np.insert(new_iter_start, peaks[i], value_insert)\n",
"\n",
"\n",
- "if new_iter_start.shape[0] != beh_times[:it_times.shape[0]]:\n",
- " print('with peak strategy, could not find correct missing iterations')\n"
+ "if new_iter_start.shape[0] != beh_times[: it_times.shape[0]]:\n",
+ " print(\"with peak strategy, could not find correct missing iterations\")"
]
},
{
@@ -2048,7 +2064,7 @@
}
],
"source": [
- "times = new_iter_start/nidq_sampling_rate\n",
+ "times = new_iter_start / nidq_sampling_rate\n",
"times = times - times[0]\n",
"new_diff_times = times - beh_times\n",
"plt.plot(new_diff_times)"
@@ -2091,9 +2107,9 @@
],
"source": [
"print(iteration_dict.keys())\n",
- "so = np.where(iteration_dict['trialnumber_vector_samples'] == 12)\n",
+ "so = np.where(iteration_dict[\"trialnumber_vector_samples\"] == 12)\n",
"\n",
- "print(so[0])\n"
+ "print(so[0])"
]
},
{
@@ -2139,20 +2155,37 @@
"end_iter = 2704\n",
"samp_after = 60000\n",
"\n",
- "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n",
- "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 1,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot][end_iter],\n",
+ " ]\n",
+ " + samp_after\n",
+ ")\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 2,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot][end_iter],\n",
+ " ]\n",
+ " + samp_after\n",
+ ")\n",
"\n",
"\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
"\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
"\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -2190,20 +2223,37 @@
"start_iter = 2704\n",
"samp_after = 600\n",
"\n",
- "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+samp_after])\n",
- "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+samp_after])\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 2,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n",
+ " + samp_after,\n",
+ " ]\n",
+ ")\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 1,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n",
+ " + samp_after,\n",
+ " ]\n",
+ ")\n",
"\n",
"\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
"\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
"\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -2222,8 +2272,8 @@
],
"source": [
"trial_plot = 280\n",
- "print(iteration_dict['iter_start_idx'][trial_plot][2704])\n",
- "print(iteration_dict['iter_start_idx'][trial_plot+1][0])"
+ "print(iteration_dict[\"iter_start_idx\"][trial_plot][2704])\n",
+ "print(iteration_dict[\"iter_start_idx\"][trial_plot + 1][0])"
]
},
{
@@ -2255,7 +2305,7 @@
}
],
"source": [
- "plt.plot(np.diff(iteration_dict['iter_start_idx'][trial_plot]/nidq_sampling_rate))"
+ "plt.plot(np.diff(iteration_dict[\"iter_start_idx\"][trial_plot] / nidq_sampling_rate))"
]
},
{
@@ -2335,14 +2385,14 @@
"source": [
"x = np.array([])\n",
"mean_x = np.array([])\n",
- "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n",
- " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n",
+ "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n",
+ " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n",
" mean_time_trial = np.mean(s)\n",
- " x = np.append(x,s, axis=0)\n",
+ " x = np.append(x, s, axis=0)\n",
" mean_x = np.append(mean_x, mean_time_trial)\n",
"\n",
- "#plt.plot(x)\n",
- "plt.plot(mean_x)\n"
+ "# plt.plot(x)\n",
+ "plt.plot(mean_x)"
]
},
{
diff --git a/notebooks/developer_notebooks/startup_time_analysis.ipynb b/notebooks/developer_notebooks/startup_time_analysis.ipynb
index efe3cb47..9313af2e 100644
--- a/notebooks/developer_notebooks/startup_time_analysis.ipynb
+++ b/notebooks/developer_notebooks/startup_time_analysis.ipynb
@@ -34,7 +34,7 @@
}
],
"source": [
- "action = dj.create_virtual_module('action', 'u19_action')\n"
+ "action = dj.create_virtual_module(\"action\", \"u19_action\")"
]
},
{
@@ -44,7 +44,7 @@
"metadata": {},
"outputs": [],
"source": [
- "startup_times = action.RigStartupTime().fetch(format='frame')\n"
+ "startup_times = action.RigStartupTime().fetch(format=\"frame\")"
]
},
{
@@ -252,37 +252,44 @@
}
],
"source": [
- "import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
- "import matplotlib.ticker as ticker\n",
"import numpy as np\n",
+ "import pandas as pd\n",
"\n",
"# --- Deduplicate: within each location, if two startups are within 45s, keep the older one ---\n",
"df = startup_times.reset_index().copy()\n",
- "df['startup_datetime'] = pd.to_datetime(df['startup_datetime'])\n",
- "df = df.sort_values(['location', 'startup_datetime'])\n",
+ "df[\"startup_datetime\"] = pd.to_datetime(df[\"startup_datetime\"])\n",
+ "df = df.sort_values([\"location\", \"startup_datetime\"])\n",
+ "\n",
"\n",
"def drop_close_duplicates(group, threshold_sec=45):\n",
- " group = group.sort_values('startup_datetime').reset_index(drop=True)\n",
+ " group = group.sort_values(\"startup_datetime\").reset_index(drop=True)\n",
" keep = [True]\n",
" for i in range(1, len(group)):\n",
- " delta = (group.loc[i, 'startup_datetime'] - group.loc[i-1, 'startup_datetime']).total_seconds()\n",
+ " delta = (\n",
+ " group.loc[i, \"startup_datetime\"] - group.loc[i - 1, \"startup_datetime\"]\n",
+ " ).total_seconds()\n",
" keep.append(delta > threshold_sec)\n",
" return group[keep]\n",
"\n",
+ "\n",
"df_clean = (\n",
- " df.groupby('location', group_keys=True)\n",
+ " df.groupby(\"location\", group_keys=True)\n",
" .apply(drop_close_duplicates)\n",
" .reset_index(drop=False)\n",
")\n",
- "df_clean['startup_time'] = pd.to_numeric(df_clean['startup_time'], errors='coerce')\n",
- "df_clean['num_subj_scheduled'] = pd.to_numeric(df_clean['num_subj_scheduled'], errors='coerce')\n",
+ "df_clean[\"startup_time\"] = pd.to_numeric(df_clean[\"startup_time\"], errors=\"coerce\")\n",
+ "df_clean[\"num_subj_scheduled\"] = pd.to_numeric(\n",
+ " df_clean[\"num_subj_scheduled\"], errors=\"coerce\"\n",
+ ")\n",
"\n",
"# Remove rows with startup_time > 5000s\n",
"n_before = len(df_clean)\n",
- "df_clean = df_clean[df_clean['startup_time'] <= 5000].copy()\n",
- "print(f\"Raw rows: {len(df)}, after dedup: {n_before}, after removing >5000s: {len(df_clean)} ({n_before - len(df_clean)} dropped)\")\n",
- "df_clean.head()\n"
+ "df_clean = df_clean[df_clean[\"startup_time\"] <= 5000].copy()\n",
+ "print(\n",
+ " f\"Raw rows: {len(df)}, after dedup: {n_before}, after removing >5000s: {len(df_clean)} ({n_before - len(df_clean)} dropped)\"\n",
+ ")\n",
+ "df_clean.head()"
]
},
{
@@ -335,22 +342,30 @@
"source": [
"# --- Startup time by rig (location) ---\n",
"rig_stats = (\n",
- " df_clean.groupby('location')['startup_time']\n",
- " .agg(median='median', mean='mean', std='std', count='count', p90=lambda x: x.quantile(0.9))\n",
- " .sort_values('median', ascending=False)\n",
+ " df_clean.groupby(\"location\")[\"startup_time\"]\n",
+ " .agg(\n",
+ " median=\"median\",\n",
+ " mean=\"mean\",\n",
+ " std=\"std\",\n",
+ " count=\"count\",\n",
+ " p90=lambda x: x.quantile(0.9),\n",
+ " )\n",
+ " .sort_values(\"median\", ascending=False)\n",
")\n",
"print(rig_stats.to_string())\n",
"\n",
"fig, ax = plt.subplots(figsize=(max(8, len(rig_stats) * 0.6), 5))\n",
"order = rig_stats.index.tolist()\n",
- "data_by_rig = [df_clean[df_clean['location'] == loc]['startup_time'].dropna() for loc in order]\n",
+ "data_by_rig = [\n",
+ " df_clean[df_clean[\"location\"] == loc][\"startup_time\"].dropna() for loc in order\n",
+ "]\n",
"ax.boxplot(data_by_rig, tick_labels=order, vert=True)\n",
- "ax.set_xlabel('Rig (location)')\n",
- "ax.set_ylabel('Startup time (s)')\n",
- "ax.set_title('Startup time distribution by rig')\n",
- "plt.xticks(rotation=45, ha='right')\n",
+ "ax.set_xlabel(\"Rig (location)\")\n",
+ "ax.set_ylabel(\"Startup time (s)\")\n",
+ "ax.set_title(\"Startup time distribution by rig\")\n",
+ "plt.xticks(rotation=45, ha=\"right\")\n",
"plt.tight_layout()\n",
- "plt.show()\n"
+ "plt.show()"
]
},
{
@@ -385,9 +400,15 @@
"source": [
"# --- Startup time by startup_type ---\n",
"type_stats = (\n",
- " df_clean.groupby('startup_type')['startup_time']\n",
- " .agg(median='median', mean='mean', std='std', count='count', p90=lambda x: x.quantile(0.9))\n",
- " .sort_values('median', ascending=False)\n",
+ " df_clean.groupby(\"startup_type\")[\"startup_time\"]\n",
+ " .agg(\n",
+ " median=\"median\",\n",
+ " mean=\"mean\",\n",
+ " std=\"std\",\n",
+ " count=\"count\",\n",
+ " p90=lambda x: x.quantile(0.9),\n",
+ " )\n",
+ " .sort_values(\"median\", ascending=False)\n",
")\n",
"print(type_stats.to_string())\n",
"\n",
@@ -396,25 +417,25 @@
"# Boxplot by startup_type\n",
"order_t = type_stats.index.tolist()\n",
"axes[0].boxplot(\n",
- " [df_clean[df_clean['startup_type'] == t]['startup_time'].dropna() for t in order_t],\n",
+ " [df_clean[df_clean[\"startup_type\"] == t][\"startup_time\"].dropna() for t in order_t],\n",
" tick_labels=order_t,\n",
")\n",
- "axes[0].set_xlabel('startup_type')\n",
- "axes[0].set_ylabel('Startup time (s)')\n",
- "axes[0].set_title('Startup time by startup_type')\n",
- "axes[0].tick_params(axis='x', rotation=30)\n",
+ "axes[0].set_xlabel(\"startup_type\")\n",
+ "axes[0].set_ylabel(\"Startup time (s)\")\n",
+ "axes[0].set_title(\"Startup time by startup_type\")\n",
+ "axes[0].tick_params(axis=\"x\", rotation=30)\n",
"\n",
"# Mean + 95% CI bar chart\n",
- "means = type_stats['mean']\n",
- "sems = df_clean.groupby('startup_type')['startup_time'].sem()\n",
+ "means = type_stats[\"mean\"]\n",
+ "sems = df_clean.groupby(\"startup_type\")[\"startup_time\"].sem()\n",
"axes[1].bar(order_t, means[order_t], yerr=1.96 * sems[order_t], capsize=5)\n",
- "axes[1].set_xlabel('startup_type')\n",
- "axes[1].set_ylabel('Mean startup time (s)')\n",
- "axes[1].set_title('Mean ± 95% CI by startup_type')\n",
- "axes[1].tick_params(axis='x', rotation=30)\n",
+ "axes[1].set_xlabel(\"startup_type\")\n",
+ "axes[1].set_ylabel(\"Mean startup time (s)\")\n",
+ "axes[1].set_title(\"Mean ± 95% CI by startup_type\")\n",
+ "axes[1].tick_params(axis=\"x\", rotation=30)\n",
"\n",
"plt.tight_layout()\n",
- "plt.show()\n"
+ "plt.show()"
]
},
{
@@ -451,42 +472,46 @@
],
"source": [
"# --- Effect of num_subj_scheduled on startup_time ---\n",
- "corr = df_clean[['num_subj_scheduled', 'startup_time']].corr().iloc[0, 1]\n",
+ "corr = df_clean[[\"num_subj_scheduled\", \"startup_time\"]].corr().iloc[0, 1]\n",
"print(f\"Pearson r(num_subj_scheduled, startup_time) = {corr:.3f}\")\n",
"\n",
"subj_stats = (\n",
- " df_clean.groupby('num_subj_scheduled')['startup_time']\n",
- " .agg(median='median', mean='mean', count='count')\n",
+ " df_clean.groupby(\"num_subj_scheduled\")[\"startup_time\"]\n",
+ " .agg(median=\"median\", mean=\"mean\", count=\"count\")\n",
" .reset_index()\n",
- " .sort_values('num_subj_scheduled')\n",
+ " .sort_values(\"num_subj_scheduled\")\n",
")\n",
"print(subj_stats.to_string(index=False))\n",
"\n",
"fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n",
"\n",
"# Scatter with regression line\n",
- "x = df_clean['num_subj_scheduled'].dropna()\n",
- "y = df_clean.loc[x.index, 'startup_time']\n",
+ "x = df_clean[\"num_subj_scheduled\"].dropna()\n",
+ "y = df_clean.loc[x.index, \"startup_time\"]\n",
"m, b = np.polyfit(x, y, 1)\n",
"axes[0].scatter(x, y, alpha=0.3, s=15)\n",
"xs = np.linspace(x.min(), x.max(), 100)\n",
- "axes[0].plot(xs, m * xs + b, 'r-', label=f'slope={m:.1f}s/subj, r={corr:.2f}')\n",
- "axes[0].set_xlabel('num_subj_scheduled')\n",
- "axes[0].set_ylabel('Startup time (s)')\n",
- "axes[0].set_title('Startup time vs. num_subj_scheduled')\n",
+ "axes[0].plot(xs, m * xs + b, \"r-\", label=f\"slope={m:.1f}s/subj, r={corr:.2f}\")\n",
+ "axes[0].set_xlabel(\"num_subj_scheduled\")\n",
+ "axes[0].set_ylabel(\"Startup time (s)\")\n",
+ "axes[0].set_title(\"Startup time vs. num_subj_scheduled\")\n",
"axes[0].legend()\n",
"\n",
"# Mean startup time per num_subj_scheduled (size ∝ count)\n",
"sc = axes[1].scatter(\n",
- " subj_stats['num_subj_scheduled'], subj_stats['mean'],\n",
- " s=subj_stats['count'] * 5, alpha=0.7,\n",
+ " subj_stats[\"num_subj_scheduled\"],\n",
+ " subj_stats[\"mean\"],\n",
+ " s=subj_stats[\"count\"] * 5,\n",
+ " alpha=0.7,\n",
+ ")\n",
+ "axes[1].set_xlabel(\"num_subj_scheduled\")\n",
+ "axes[1].set_ylabel(\"Mean startup time (s)\")\n",
+ "axes[1].set_title(\n",
+ " \"Mean startup time per num_subj_scheduled\\n(bubble size ∝ sample count)\"\n",
")\n",
- "axes[1].set_xlabel('num_subj_scheduled')\n",
- "axes[1].set_ylabel('Mean startup time (s)')\n",
- "axes[1].set_title('Mean startup time per num_subj_scheduled\\n(bubble size ∝ sample count)')\n",
"\n",
"plt.tight_layout()\n",
- "plt.show()\n"
+ "plt.show()"
]
},
{
@@ -530,12 +555,14 @@
],
"source": [
"# --- Filter to \"good data\" (after May 15th) ---\n",
- "cutoff_date = pd.to_datetime('2026-05-15')\n",
- "df_good = df_clean[df_clean['startup_datetime'] >= cutoff_date].copy()\n",
- "print(f\"Filtered to {cutoff_date.date()} onwards: {len(df_good)} rows (from {len(df_clean)})\")\n",
+ "cutoff_date = pd.to_datetime(\"2026-05-15\")\n",
+ "df_good = df_clean[df_clean[\"startup_datetime\"] >= cutoff_date].copy()\n",
+ "print(\n",
+ " f\"Filtered to {cutoff_date.date()} onwards: {len(df_good)} rows (from {len(df_clean)})\"\n",
+ ")\n",
"\n",
"print(\"\\nAll unique locations in good data:\")\n",
- "print(df_good['location'].value_counts())\n"
+ "print(df_good[\"location\"].value_counts())"
]
},
{
@@ -590,27 +617,35 @@
],
"source": [
"# === ALL RIGS ANALYSIS (after May 15th) ===\n",
- "print(\"\\n\" + \"=\"*70)\n",
+ "print(\"\\n\" + \"=\" * 70)\n",
"print(\"STARTUP TIME BY RIG (after 2026-05-15)\")\n",
- "print(\"=\"*70)\n",
+ "print(\"=\" * 70)\n",
"\n",
"rig_stats_good = (\n",
- " df_good.groupby('location')['startup_time']\n",
- " .agg(median='median', mean='mean', std='std', count='count', p95=lambda x: x.quantile(0.95))\n",
- " .sort_values('median', ascending=False)\n",
+ " df_good.groupby(\"location\")[\"startup_time\"]\n",
+ " .agg(\n",
+ " median=\"median\",\n",
+ " mean=\"mean\",\n",
+ " std=\"std\",\n",
+ " count=\"count\",\n",
+ " p95=lambda x: x.quantile(0.95),\n",
+ " )\n",
+ " .sort_values(\"median\", ascending=False)\n",
")\n",
"print(rig_stats_good.to_string())\n",
"\n",
"fig, ax = plt.subplots(figsize=(max(10, len(rig_stats_good) * 0.6), 5))\n",
"order = rig_stats_good.index.tolist()\n",
- "data_by_rig = [df_good[df_good['location'] == loc]['startup_time'].dropna() for loc in order]\n",
+ "data_by_rig = [\n",
+ " df_good[df_good[\"location\"] == loc][\"startup_time\"].dropna() for loc in order\n",
+ "]\n",
"ax.boxplot(data_by_rig, tick_labels=order, vert=True)\n",
- "ax.set_xlabel('Rig (location)')\n",
- "ax.set_ylabel('Startup time (s)')\n",
- "ax.set_title('Startup time by rig (after 2026-05-15)')\n",
- "plt.xticks(rotation=45, ha='right')\n",
+ "ax.set_xlabel(\"Rig (location)\")\n",
+ "ax.set_ylabel(\"Startup time (s)\")\n",
+ "ax.set_title(\"Startup time by rig (after 2026-05-15)\")\n",
+ "plt.xticks(rotation=45, ha=\"right\")\n",
"plt.tight_layout()\n",
- "plt.show()\n"
+ "plt.show()"
]
},
{
@@ -648,14 +683,20 @@
],
"source": [
"# === STARTUP TYPE ANALYSIS (after May 15th, all rigs) ===\n",
- "print(\"\\n\" + \"=\"*70)\n",
+ "print(\"\\n\" + \"=\" * 70)\n",
"print(\"STARTUP TYPE ANALYSIS (after 2026-05-15, all rigs)\")\n",
- "print(\"=\"*70)\n",
+ "print(\"=\" * 70)\n",
"\n",
"type_stats_good = (\n",
- " df_good.groupby('startup_type')['startup_time']\n",
- " .agg(median='median', mean='mean', std='std', count='count', p95=lambda x: x.quantile(0.95))\n",
- " .sort_values('median', ascending=False)\n",
+ " df_good.groupby(\"startup_type\")[\"startup_time\"]\n",
+ " .agg(\n",
+ " median=\"median\",\n",
+ " mean=\"mean\",\n",
+ " std=\"std\",\n",
+ " count=\"count\",\n",
+ " p95=lambda x: x.quantile(0.95),\n",
+ " )\n",
+ " .sort_values(\"median\", ascending=False)\n",
")\n",
"print(type_stats_good.to_string())\n",
"\n",
@@ -663,24 +704,24 @@
"\n",
"order_t = type_stats_good.index.tolist()\n",
"axes[0].boxplot(\n",
- " [df_good[df_good['startup_type'] == t]['startup_time'].dropna() for t in order_t],\n",
+ " [df_good[df_good[\"startup_type\"] == t][\"startup_time\"].dropna() for t in order_t],\n",
" tick_labels=order_t,\n",
")\n",
- "axes[0].set_xlabel('startup_type')\n",
- "axes[0].set_ylabel('Startup time (s)')\n",
- "axes[0].set_title('Startup time by startup_type (after 2026-05-15)')\n",
- "axes[0].tick_params(axis='x', rotation=30)\n",
+ "axes[0].set_xlabel(\"startup_type\")\n",
+ "axes[0].set_ylabel(\"Startup time (s)\")\n",
+ "axes[0].set_title(\"Startup time by startup_type (after 2026-05-15)\")\n",
+ "axes[0].tick_params(axis=\"x\", rotation=30)\n",
"\n",
- "means_t = type_stats_good['mean']\n",
- "sems_t = df_good.groupby('startup_type')['startup_time'].sem()\n",
+ "means_t = type_stats_good[\"mean\"]\n",
+ "sems_t = df_good.groupby(\"startup_type\")[\"startup_time\"].sem()\n",
"axes[1].bar(order_t, means_t[order_t], yerr=1.96 * sems_t[order_t], capsize=5)\n",
- "axes[1].set_xlabel('startup_type')\n",
- "axes[1].set_ylabel('Mean startup time (s)')\n",
- "axes[1].set_title('Mean ± 95% CI by startup_type (after 2026-05-15)')\n",
- "axes[1].tick_params(axis='x', rotation=30)\n",
+ "axes[1].set_xlabel(\"startup_type\")\n",
+ "axes[1].set_ylabel(\"Mean startup time (s)\")\n",
+ "axes[1].set_title(\"Mean ± 95% CI by startup_type (after 2026-05-15)\")\n",
+ "axes[1].tick_params(axis=\"x\", rotation=30)\n",
"\n",
"plt.tight_layout()\n",
- "plt.show()\n"
+ "plt.show()"
]
},
{
@@ -722,18 +763,18 @@
],
"source": [
"# === NUM_SUBJ_SCHEDULED ANALYSIS (after May 15th, all rigs) ===\n",
- "print(\"\\n\" + \"=\"*70)\n",
+ "print(\"\\n\" + \"=\" * 70)\n",
"print(\"NUM_SUBJ_SCHEDULED ANALYSIS (after 2026-05-15, all rigs)\")\n",
- "print(\"=\"*70)\n",
+ "print(\"=\" * 70)\n",
"\n",
- "corr_good = df_good[['num_subj_scheduled', 'startup_time']].corr().iloc[0, 1]\n",
+ "corr_good = df_good[[\"num_subj_scheduled\", \"startup_time\"]].corr().iloc[0, 1]\n",
"print(f\"Pearson r = {corr_good:.3f}\")\n",
"\n",
"subj_stats_good = (\n",
- " df_good.groupby('num_subj_scheduled')['startup_time']\n",
- " .agg(median='median', mean='mean', count='count')\n",
+ " df_good.groupby(\"num_subj_scheduled\")[\"startup_time\"]\n",
+ " .agg(median=\"median\", mean=\"mean\", count=\"count\")\n",
" .reset_index()\n",
- " .sort_values('num_subj_scheduled')\n",
+ " .sort_values(\"num_subj_scheduled\")\n",
")\n",
"print(\"\\nStartup time by num_subj_scheduled:\")\n",
"print(subj_stats_good.to_string(index=False))\n",
@@ -741,30 +782,40 @@
"fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n",
"\n",
"# Scatter with regression\n",
- "x_good = df_good['num_subj_scheduled'].dropna()\n",
- "y_good = df_good.loc[x_good.index, 'startup_time']\n",
+ "x_good = df_good[\"num_subj_scheduled\"].dropna()\n",
+ "y_good = df_good.loc[x_good.index, \"startup_time\"]\n",
"m_good, b_good = np.polyfit(x_good, y_good, 1)\n",
"\n",
"axes[0].scatter(x_good, y_good, alpha=0.3, s=20)\n",
"xs_good = np.linspace(x_good.min(), x_good.max(), 100)\n",
- "axes[0].plot(xs_good, m_good * xs_good + b_good, 'r-', linewidth=2, label=f'slope={m_good:.2f}s/subj, r={corr_good:.2f}')\n",
- "axes[0].set_xlabel('num_subj_scheduled')\n",
- "axes[0].set_ylabel('Startup time (s)')\n",
- "axes[0].set_title('Startup time vs. num_subj_scheduled (after 2026-05-15)')\n",
+ "axes[0].plot(\n",
+ " xs_good,\n",
+ " m_good * xs_good + b_good,\n",
+ " \"r-\",\n",
+ " linewidth=2,\n",
+ " label=f\"slope={m_good:.2f}s/subj, r={corr_good:.2f}\",\n",
+ ")\n",
+ "axes[0].set_xlabel(\"num_subj_scheduled\")\n",
+ "axes[0].set_ylabel(\"Startup time (s)\")\n",
+ "axes[0].set_title(\"Startup time vs. num_subj_scheduled (after 2026-05-15)\")\n",
"axes[0].legend()\n",
"axes[0].grid(True, alpha=0.3)\n",
"\n",
"# Bubble chart\n",
"axes[1].scatter(\n",
- " subj_stats_good['num_subj_scheduled'], subj_stats_good['mean'],\n",
- " s=subj_stats_good['count'] * 5, alpha=0.7,\n",
+ " subj_stats_good[\"num_subj_scheduled\"],\n",
+ " subj_stats_good[\"mean\"],\n",
+ " s=subj_stats_good[\"count\"] * 5,\n",
+ " alpha=0.7,\n",
+ ")\n",
+ "axes[1].set_xlabel(\"num_subj_scheduled\")\n",
+ "axes[1].set_ylabel(\"Mean startup time (s)\")\n",
+ "axes[1].set_title(\n",
+ " \"Mean startup time per num_subj_scheduled\\n(bubble size ∝ sample count, after 2026-05-15)\"\n",
")\n",
- "axes[1].set_xlabel('num_subj_scheduled')\n",
- "axes[1].set_ylabel('Mean startup time (s)')\n",
- "axes[1].set_title('Mean startup time per num_subj_scheduled\\n(bubble size ∝ sample count, after 2026-05-15)')\n",
"\n",
"plt.tight_layout()\n",
- "plt.show()\n"
+ "plt.show()"
]
},
{
@@ -791,7 +842,7 @@
"# === Which rigs take longest: filtered by startup_type ===\n",
"# Show all startup_types present in good data so we can confirm the labels\n",
"print(\"Startup types in good data:\")\n",
- "print(df_good['startup_type'].value_counts())\n"
+ "print(df_good[\"startup_type\"].value_counts())"
]
},
{
@@ -876,40 +927,47 @@
"source": [
"# === Rig Tester vs Training Flow GUI: which rigs take the longest? ===\n",
"# Adjust these strings to match the exact startup_type values printed above\n",
- "df_rig_tester = df_good[df_good['startup_type'].str.contains('Rig Tester', case=False, na=False)]\n",
- "df_training_gui = df_good[df_good['startup_type'].str.contains('Training Flow GUI', case=False, na=False)]\n",
+ "df_rig_tester = df_good[\n",
+ " df_good[\"startup_type\"].str.contains(\"Rig Tester\", case=False, na=False)\n",
+ "]\n",
+ "df_training_gui = df_good[\n",
+ " df_good[\"startup_type\"].str.contains(\"Training Flow GUI\", case=False, na=False)\n",
+ "]\n",
"\n",
"print(f\"Rig Tester startup_type rows: {len(df_rig_tester)}\")\n",
"print(f\"Training Flow GUI startup_type rows: {len(df_training_gui)}\")\n",
"\n",
"# --- Which rigs take longest under each startup_type ---\n",
- "for label, subset in [('Rig Tester', df_rig_tester), ('Training Flow GUI', df_training_gui)]:\n",
+ "for label, subset in [\n",
+ " (\"Rig Tester\", df_rig_tester),\n",
+ " (\"Training Flow GUI\", df_training_gui),\n",
+ "]:\n",
" if len(subset) == 0:\n",
" print(f\"\\n[{label}] No rows found — check startup_type label above\")\n",
" continue\n",
"\n",
" stats = (\n",
- " subset.groupby('location')['startup_time']\n",
- " .agg(median='median', mean='mean', std='std', count='count')\n",
- " .sort_values('median', ascending=False)\n",
+ " subset.groupby(\"location\")[\"startup_time\"]\n",
+ " .agg(median=\"median\", mean=\"mean\", std=\"std\", count=\"count\")\n",
+ " .sort_values(\"median\", ascending=False)\n",
" )\n",
- " print(f\"\\n{'='*60}\")\n",
+ " print(f\"\\n{'=' * 60}\")\n",
" print(f\" {label}: startup time by rig (sorted slowest → fastest)\")\n",
- " print(f\"{'='*60}\")\n",
+ " print(f\"{'=' * 60}\")\n",
" print(stats.to_string())\n",
"\n",
" fig, ax = plt.subplots(figsize=(max(8, len(stats) * 0.6), 5))\n",
" locs = stats.index.tolist()\n",
" ax.boxplot(\n",
- " [subset[subset['location'] == loc]['startup_time'].dropna() for loc in locs],\n",
+ " [subset[subset[\"location\"] == loc][\"startup_time\"].dropna() for loc in locs],\n",
" tick_labels=locs,\n",
" )\n",
- " ax.set_xlabel('Rig (location)')\n",
- " ax.set_ylabel('Startup time (s)')\n",
- " ax.set_title(f'Startup time by rig — {label} (after 2026-05-15)')\n",
- " plt.xticks(rotation=45, ha='right')\n",
+ " ax.set_xlabel(\"Rig (location)\")\n",
+ " ax.set_ylabel(\"Startup time (s)\")\n",
+ " ax.set_title(f\"Startup time by rig — {label} (after 2026-05-15)\")\n",
+ " plt.xticks(rotation=45, ha=\"right\")\n",
" plt.tight_layout()\n",
- " plt.show()\n"
+ " plt.show()"
]
},
{
@@ -973,54 +1031,67 @@
],
"source": [
"# === Does num_subj_scheduled matter within each startup_type? ===\n",
- "for label, subset in [('Rig Tester', df_rig_tester), ('Training Flow GUI', df_training_gui)]:\n",
+ "for label, subset in [\n",
+ " (\"Rig Tester\", df_rig_tester),\n",
+ " (\"Training Flow GUI\", df_training_gui),\n",
+ "]:\n",
" if len(subset) == 0:\n",
" continue\n",
"\n",
- " x = subset['num_subj_scheduled'].dropna()\n",
- " y = subset.loc[x.index, 'startup_time']\n",
+ " x = subset[\"num_subj_scheduled\"].dropna()\n",
+ " y = subset.loc[x.index, \"startup_time\"]\n",
"\n",
" if x.nunique() < 2:\n",
- " print(f\"\\n[{label}] Not enough variation in num_subj_scheduled to fit regression\")\n",
+ " print(\n",
+ " f\"\\n[{label}] Not enough variation in num_subj_scheduled to fit regression\"\n",
+ " )\n",
" continue\n",
"\n",
- " corr = subset[['num_subj_scheduled', 'startup_time']].corr().iloc[0, 1]\n",
+ " corr = subset[[\"num_subj_scheduled\", \"startup_time\"]].corr().iloc[0, 1]\n",
" m, b = np.polyfit(x, y, 1)\n",
"\n",
" subj_stats = (\n",
- " subset.groupby('num_subj_scheduled')['startup_time']\n",
- " .agg(median='median', mean='mean', count='count')\n",
+ " subset.groupby(\"num_subj_scheduled\")[\"startup_time\"]\n",
+ " .agg(median=\"median\", mean=\"mean\", count=\"count\")\n",
" .reset_index()\n",
- " .sort_values('num_subj_scheduled')\n",
+ " .sort_values(\"num_subj_scheduled\")\n",
" )\n",
"\n",
- " print(f\"\\n{'='*60}\")\n",
- " print(f\" {label}: effect of num_subj_scheduled (r={corr:.2f}, slope={m:.2f}s/subj)\")\n",
- " print(f\"{'='*60}\")\n",
+ " print(f\"\\n{'=' * 60}\")\n",
+ " print(\n",
+ " f\" {label}: effect of num_subj_scheduled (r={corr:.2f}, slope={m:.2f}s/subj)\"\n",
+ " )\n",
+ " print(f\"{'=' * 60}\")\n",
" print(subj_stats.to_string(index=False))\n",
"\n",
" fig, axes = plt.subplots(1, 2, figsize=(14, 4))\n",
"\n",
" axes[0].scatter(x, y, alpha=0.4, s=20)\n",
" xs = np.linspace(x.min(), x.max(), 100)\n",
- " axes[0].plot(xs, m * xs + b, 'r-', linewidth=2, label=f'slope={m:.2f}s/subj, r={corr:.2f}')\n",
- " axes[0].set_xlabel('num_subj_scheduled')\n",
- " axes[0].set_ylabel('Startup time (s)')\n",
- " axes[0].set_title(f'{label}: startup time vs. num_subj_scheduled')\n",
+ " axes[0].plot(\n",
+ " xs, m * xs + b, \"r-\", linewidth=2, label=f\"slope={m:.2f}s/subj, r={corr:.2f}\"\n",
+ " )\n",
+ " axes[0].set_xlabel(\"num_subj_scheduled\")\n",
+ " axes[0].set_ylabel(\"Startup time (s)\")\n",
+ " axes[0].set_title(f\"{label}: startup time vs. num_subj_scheduled\")\n",
" axes[0].legend()\n",
" axes[0].grid(True, alpha=0.3)\n",
"\n",
" axes[1].scatter(\n",
- " subj_stats['num_subj_scheduled'], subj_stats['mean'],\n",
- " s=subj_stats['count'] * 10, alpha=0.7,\n",
+ " subj_stats[\"num_subj_scheduled\"],\n",
+ " subj_stats[\"mean\"],\n",
+ " s=subj_stats[\"count\"] * 10,\n",
+ " alpha=0.7,\n",
+ " )\n",
+ " axes[1].set_xlabel(\"num_subj_scheduled\")\n",
+ " axes[1].set_ylabel(\"Mean startup time (s)\")\n",
+ " axes[1].set_title(\n",
+ " f\"{label}: mean startup time per num_subj_scheduled\\n(bubble size ∝ count)\"\n",
" )\n",
- " axes[1].set_xlabel('num_subj_scheduled')\n",
- " axes[1].set_ylabel('Mean startup time (s)')\n",
- " axes[1].set_title(f'{label}: mean startup time per num_subj_scheduled\\n(bubble size ∝ count)')\n",
" axes[1].grid(True, alpha=0.3)\n",
"\n",
" plt.tight_layout()\n",
- " plt.show()\n"
+ " plt.show()"
]
},
{
@@ -1096,43 +1167,57 @@
],
"source": [
"# === Startup time as a function of calendar date, per rig, split by startup_type ===\n",
- "df_good['date_ordinal'] = df_good['startup_datetime'].map(pd.Timestamp.toordinal)\n",
+ "df_good[\"date_ordinal\"] = df_good[\"startup_datetime\"].map(pd.Timestamp.toordinal)\n",
"\n",
- "for label, subset in [('Rig Tester', df_rig_tester), ('Training Flow GUI', df_training_gui)]:\n",
+ "for label, subset in [\n",
+ " (\"Rig Tester\", df_rig_tester),\n",
+ " (\"Training Flow GUI\", df_training_gui),\n",
+ "]:\n",
" if len(subset) == 0:\n",
" continue\n",
"\n",
" subset = subset.copy()\n",
- " subset['date_ordinal'] = subset['startup_datetime'].map(pd.Timestamp.toordinal)\n",
+ " subset[\"date_ordinal\"] = subset[\"startup_datetime\"].map(pd.Timestamp.toordinal)\n",
"\n",
- " rigs = subset['location'].unique()\n",
+ " rigs = subset[\"location\"].unique()\n",
" n_rigs = len(rigs)\n",
" cols = 3\n",
" rows = int(np.ceil(n_rigs / cols))\n",
"\n",
" # --- Per-rig subplots with trend lines ---\n",
" fig, axes = plt.subplots(rows, cols, figsize=(cols * 5, rows * 3.5), squeeze=False)\n",
- " fig.suptitle(f'Startup time over time — {label} (after 2026-05-15)', fontsize=13, y=1.01)\n",
+ " fig.suptitle(\n",
+ " f\"Startup time over time — {label} (after 2026-05-15)\", fontsize=13, y=1.01\n",
+ " )\n",
"\n",
" trend_rows = []\n",
" for i, rig in enumerate(sorted(rigs)):\n",
" ax = axes[i // cols][i % cols]\n",
- " grp = subset[subset['location'] == rig].dropna(subset=['startup_time', 'date_ordinal'])\n",
+ " grp = subset[subset[\"location\"] == rig].dropna(\n",
+ " subset=[\"startup_time\", \"date_ordinal\"]\n",
+ " )\n",
"\n",
- " ax.scatter(grp['startup_datetime'], grp['startup_time'], s=15, alpha=0.6)\n",
+ " ax.scatter(grp[\"startup_datetime\"], grp[\"startup_time\"], s=15, alpha=0.6)\n",
"\n",
" if len(grp) >= 2:\n",
- " m, b = np.polyfit(grp['date_ordinal'], grp['startup_time'], 1)\n",
- " ax.plot(grp['startup_datetime'], m * grp['date_ordinal'] + b, 'r-', linewidth=1.5)\n",
- " corr = grp[['date_ordinal', 'startup_time']].corr().iloc[0, 1]\n",
- " trend_rows.append({'location': rig, 'slope_s_per_day': m, 'r': corr, 'n': len(grp)})\n",
- " ax.set_title(f'{rig}\\nslope={m:.2f}s/day, r={corr:.2f}', fontsize=8)\n",
+ " m, b = np.polyfit(grp[\"date_ordinal\"], grp[\"startup_time\"], 1)\n",
+ " ax.plot(\n",
+ " grp[\"startup_datetime\"],\n",
+ " m * grp[\"date_ordinal\"] + b,\n",
+ " \"r-\",\n",
+ " linewidth=1.5,\n",
+ " )\n",
+ " corr = grp[[\"date_ordinal\", \"startup_time\"]].corr().iloc[0, 1]\n",
+ " trend_rows.append(\n",
+ " {\"location\": rig, \"slope_s_per_day\": m, \"r\": corr, \"n\": len(grp)}\n",
+ " )\n",
+ " ax.set_title(f\"{rig}\\nslope={m:.2f}s/day, r={corr:.2f}\", fontsize=8)\n",
" else:\n",
- " ax.set_title(f'{rig} (n={len(grp)})', fontsize=8)\n",
+ " ax.set_title(f\"{rig} (n={len(grp)})\", fontsize=8)\n",
"\n",
- " ax.set_ylabel('Startup time (s)', fontsize=7)\n",
- " ax.tick_params(axis='x', labelrotation=30, labelsize=6)\n",
- " ax.tick_params(axis='y', labelsize=7)\n",
+ " ax.set_ylabel(\"Startup time (s)\", fontsize=7)\n",
+ " ax.tick_params(axis=\"x\", labelrotation=30, labelsize=6)\n",
+ " ax.tick_params(axis=\"y\", labelsize=7)\n",
" ax.grid(True, alpha=0.3)\n",
"\n",
" # Hide unused subplots\n",
@@ -1146,11 +1231,11 @@
" if trend_rows:\n",
" trend_df = (\n",
" pd.DataFrame(trend_rows)\n",
- " .sort_values('slope_s_per_day', ascending=False)\n",
+ " .sort_values(\"slope_s_per_day\", ascending=False)\n",
" .reset_index(drop=True)\n",
" )\n",
" print(f\"\\n{label} — trend summary (positive slope = getting slower over time):\")\n",
- " print(trend_df.to_string(index=False))\n"
+ " print(trend_df.to_string(index=False))"
]
},
{
diff --git a/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb b/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb
index 42f017b2..6f828883 100644
--- a/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb
+++ b/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -33,9 +34,10 @@
}
],
"source": [
- "import pandas as pd\n",
- "import datajoint as dj\n",
"import pathlib\n",
+ "\n",
+ "import datajoint as dj\n",
+ "\n",
"import u19_pipeline.automatic_job.params_config as config"
]
},
@@ -54,8 +56,9 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
- "recording_process = dj.create_virtual_module('recording_process', 'u19_recording_process')\n"
+ "recording_process = dj.create_virtual_module(\n",
+ " \"recording_process\", \"u19_recording_process\"\n",
+ ")"
]
},
{
@@ -85,7 +88,7 @@
"outputs": [],
"source": [
"key = dict()\n",
- "key['job_id'] = 684\n",
+ "key[\"job_id\"] = 684\n",
"recording_process_info = (recording_process.Processing & key).fetch(as_dict=True)"
]
},
@@ -133,7 +136,7 @@
"metadata": {},
"outputs": [],
"source": [
- "ephys_root = dj_config['custom']['ephys_root_data_dir'][0]"
+ "ephys_root = dj_config[\"custom\"][\"ephys_root_data_dir\"][0]"
]
},
{
@@ -153,7 +156,9 @@
}
],
"source": [
- "probe_directory = pathlib.Path(ephys_root, recording_process_info[0]['recording_process_pre_path'])\n",
+ "probe_directory = pathlib.Path(\n",
+ " ephys_root, recording_process_info[0][\"recording_process_pre_path\"]\n",
+ ")\n",
"probe_directory"
]
},
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb
index 371c8262..0d152516 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -40,15 +41,15 @@
],
"source": [
"import datetime\n",
- "import numpy as np\n",
- "import pandas as pd\n",
- "import matplotlib.pyplot as plt\n",
"\n",
"import datajoint as dj\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
"\n",
"import u19_pipeline.ephys_pipeline as ephys_pipeline\n",
- "import u19_pipeline.utils.ephys_utils as ephys_utils\n",
- "import u19_pipeline.utils.ephys_fix_sync_code as ephys_fix_sync_code\n"
+ "import u19_pipeline.utils.ephys_fix_sync_code as ephys_fix_sync_code\n",
+ "import u19_pipeline.utils.ephys_utils as ephys_utils"
]
},
{
@@ -69,12 +70,16 @@
],
"source": [
"recording_query = \"recording_id > 501 and recording_id < 503\"\n",
- "recording = dj.create_virtual_module('recording', 'u19_recording')\n",
- "recording_keys = (recording.Recording & recording_query).fetch('recording_id', as_dict=True, order_by='recording_id')\n",
+ "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n",
+ "recording_keys = (recording.Recording & recording_query).fetch(\n",
+ " \"recording_id\", as_dict=True, order_by=\"recording_id\"\n",
+ ")\n",
"\n",
"\n",
- "session_fields = ['subject_fullname', 'session_date', 'session_number']\n",
- "session_keys = (recording.Recording.BehaviorSession & recording_query).fetch(*session_fields, as_dict=True, order_by='recording_id')\n",
+ "session_fields = [\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ "session_keys = (recording.Recording.BehaviorSession & recording_query).fetch(\n",
+ " *session_fields, as_dict=True, order_by=\"recording_id\"\n",
+ ")\n",
"\n",
"\n",
"recording_keys"
@@ -86,100 +91,128 @@
"metadata": {},
"outputs": [],
"source": [
- "def main_ephys_fix_sync_code(iter_start_idx, iter_times_idx, behavior_time, nidq_sampling_rate, key):\n",
+ "def main_ephys_fix_sync_code(\n",
+ " iter_start_idx, iter_times_idx, behavior_time, nidq_sampling_rate, key\n",
+ "):\n",
" iteration_dict = dict()\n",
- " iteration_dict['iter_start_idx'] = list()\n",
- " iteration_dict['iter_times_idx'] = list()\n",
- " iteration_dict['trial_sync_stats'] = list()\n",
+ " iteration_dict[\"iter_start_idx\"] = list()\n",
+ " iteration_dict[\"iter_times_idx\"] = list()\n",
+ " iteration_dict[\"trial_sync_stats\"] = list()\n",
"\n",
" for i in range(len(iter_start_idx)):\n",
+ " trial_stats_dict = {}\n",
+ " trial_stats_dict[\"num_trial\"] = i + 1\n",
"\n",
- " trial_stats_dict= {}\n",
- " trial_stats_dict['num_trial'] = i+1\n",
- "\n",
- " #print('fixing trial ',i)\n",
+ " # print('fixing trial ',i)\n",
" behavior_time_vector = behavior_time[i].flatten()\n",
"\n",
- " #time_iteration = np.median(np.diff(behavior_time[0:5])\n",
+ " # time_iteration = np.median(np.diff(behavior_time[0:5])\n",
"\n",
- " iter_times_idx[i] = iter_times_idx[i]+behavior_time_vector[1]\n",
- " #iter_times_idx[i] = iter_times_idx[i]+behavior_time_vector[1]-(behavior_time_vector[2]-iter_times_idx[i][2])\n",
- " #iter_times_idx[i] = iter_times_idx[i]+behavior_time_vector[1]\n",
- " #synced_time_vector = np.insert(synced_time_vector, 1, behavior_time_vector[1])\n",
- " #synced_time_vector[2:] = synced_time_vector[2:]+(behavior_time_vector[2]-synced_time_vector[2])\n",
- " #synced_time_vector[0] = 0\n",
+ " iter_times_idx[i] = iter_times_idx[i] + behavior_time_vector[1]\n",
+ " # iter_times_idx[i] = iter_times_idx[i]+behavior_time_vector[1]-(behavior_time_vector[2]-iter_times_idx[i][2])\n",
+ " # iter_times_idx[i] = iter_times_idx[i]+behavior_time_vector[1]\n",
+ " # synced_time_vector = np.insert(synced_time_vector, 1, behavior_time_vector[1])\n",
+ " # synced_time_vector[2:] = synced_time_vector[2:]+(behavior_time_vector[2]-synced_time_vector[2])\n",
+ " # synced_time_vector[0] = 0\n",
"\n",
- " synced_time_vector, shift_vec, median_vec = ephys_fix_sync_code.get_shift_vector(iter_times_idx[i],behavior_time_vector)\n",
+ " synced_time_vector, shift_vec, median_vec = (\n",
+ " ephys_fix_sync_code.get_shift_vector(\n",
+ " iter_times_idx[i], behavior_time_vector\n",
+ " )\n",
+ " )\n",
"\n",
- " #if median_vec[1] < 0:\n",
+ " # if median_vec[1] < 0:\n",
" # print('Trial # median more', i)\n",
" # raise('Median less than 0')\n",
- " \n",
+ "\n",
" # break\n",
"\n",
- " trial_stats_dict['max_shift'] = np.max(shift_vec)\n",
- " trial_stats_dict['min_shift'] = np.min(shift_vec)\n",
- " trial_stats_dict['median_diff'] = median_vec[1]\n",
- " trial_stats_dict['min_diff'] = median_vec[0]\n",
- " trial_stats_dict['max_diff'] = median_vec[2]\n",
+ " trial_stats_dict[\"max_shift\"] = np.max(shift_vec)\n",
+ " trial_stats_dict[\"min_shift\"] = np.min(shift_vec)\n",
+ " trial_stats_dict[\"median_diff\"] = median_vec[1]\n",
+ " trial_stats_dict[\"min_diff\"] = median_vec[0]\n",
+ " trial_stats_dict[\"max_diff\"] = median_vec[2]\n",
"\n",
- " synced_time_vector,trial_stats_dict['borrow_step2'] =\\\n",
- " ephys_fix_sync_code.fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, shift_vec)\n",
+ " synced_time_vector, trial_stats_dict[\"borrow_step2\"] = (\n",
+ " ephys_fix_sync_code.fix_shifted_sync_vector(\n",
+ " synced_time_vector, behavior_time_vector, shift_vec\n",
+ " )\n",
+ " )\n",
"\n",
- " #synced_time_vector, trial_stats_dict['borrow_step3'] =\\\n",
+ " # synced_time_vector, trial_stats_dict['borrow_step3'] =\\\n",
" # fix_sync_vector_greater(synced_time_vector, behavior_time_vector)\n",
- " synced_time_vector,trial_stats_dict['borrow_step4'] =\\\n",
- " ephys_fix_sync_code.complete_last_part_sync_vec(synced_time_vector, behavior_time_vector)\n",
- "\n",
- " synced_iteration_vector =\\\n",
- " ephys_fix_sync_code.fix_iter_vector(iter_start_idx[i],synced_time_vector, iter_times_idx[i], nidq_sampling_rate)\n",
- " \n",
- " trial_stats_dict['num_iterations'] = synced_iteration_vector.shape[0] \n",
- " \n",
- " trial_stats_dict = trial_stats_dict | key\n",
- " iteration_dict['trial_sync_stats'].append(trial_stats_dict)\n",
+ " synced_time_vector, trial_stats_dict[\"borrow_step4\"] = (\n",
+ " ephys_fix_sync_code.complete_last_part_sync_vec(\n",
+ " synced_time_vector, behavior_time_vector\n",
+ " )\n",
+ " )\n",
"\n",
+ " synced_iteration_vector = ephys_fix_sync_code.fix_iter_vector(\n",
+ " iter_start_idx[i], synced_time_vector, iter_times_idx[i], nidq_sampling_rate\n",
+ " )\n",
+ "\n",
+ " trial_stats_dict[\"num_iterations\"] = synced_iteration_vector.shape[0]\n",
+ "\n",
+ " trial_stats_dict = trial_stats_dict | key\n",
+ " iteration_dict[\"trial_sync_stats\"].append(trial_stats_dict)\n",
"\n",
- " iteration_dict['iter_start_idx'].append(synced_iteration_vector.copy())\n",
- " iteration_dict['iter_times_idx'].append(synced_time_vector.copy())\n",
+ " iteration_dict[\"iter_start_idx\"].append(synced_iteration_vector.copy())\n",
+ " iteration_dict[\"iter_times_idx\"].append(synced_time_vector.copy())\n",
"\n",
- " print('end fix sync code 1')\n",
+ " print(\"end fix sync code 1\")\n",
"\n",
- " iteration_dict['iter_start_idx'] = np.asarray(iteration_dict['iter_start_idx'].copy(), dtype=object)\n",
- " iteration_dict['iter_times_idx'] = np.asarray(iteration_dict['iter_times_idx'].copy(), dtype=object)\n",
+ " iteration_dict[\"iter_start_idx\"] = np.asarray(\n",
+ " iteration_dict[\"iter_start_idx\"].copy(), dtype=object\n",
+ " )\n",
+ " iteration_dict[\"iter_times_idx\"] = np.asarray(\n",
+ " iteration_dict[\"iter_times_idx\"].copy(), dtype=object\n",
+ " )\n",
"\n",
- " print('end fix sync code')\n",
+ " print(\"end fix sync code\")\n",
"\n",
" # Check # of trials and iterations match\n",
- " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n",
+ " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n",
+ " ephys_utils.assert_iteration_samples_count(\n",
+ " iteration_dict[\"iter_start_idx\"], behavior_time\n",
+ " )\n",
+ " )\n",
"\n",
- " print('after assert_iteration_samples_count fix sync code')\n",
+ " print(\"after assert_iteration_samples_count fix sync code\")\n",
"\n",
" if trial_count_diff != 0:\n",
- " print('trial_count_diff', trial_count_diff)\n",
+ " print(\"trial_count_diff\", trial_count_diff)\n",
" if len(trials_diff_iteration_big) > 0:\n",
- " print('trials_diff_iteration_big', trials_diff_iteration_big)\n",
+ " print(\"trials_diff_iteration_big\", trials_diff_iteration_big)\n",
" if len(trials_diff_iteration_small) > 0:\n",
- " print('trials_diff_iteration_small', trials_diff_iteration_small)\n",
+ " print(\"trials_diff_iteration_small\", trials_diff_iteration_small)\n",
"\n",
+ " status = ephys_utils.evaluate_sync_process(\n",
+ " trial_count_diff,\n",
+ " trials_diff_iteration_big,\n",
+ " trials_diff_iteration_small,\n",
+ " behavior_time.shape[0],\n",
+ " )\n",
"\n",
- " status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n",
+ " print(\"after evaluate_sync_process fix sync code\")\n",
"\n",
- " print('after evaluate_sync_process fix sync code')\n",
- "\n",
- " for i in range(len(iteration_dict['iter_start_idx'])):\n",
- " synced_time_vector = iteration_dict['iter_times_idx'][i]\n",
+ " for i in range(len(iteration_dict[\"iter_start_idx\"])):\n",
+ " synced_time_vector = iteration_dict[\"iter_times_idx\"][i]\n",
" behavior_time_vector = behavior_time[i].flatten()\n",
"\n",
- " status = ephys_fix_sync_code.sync_evaluation_process2(synced_time_vector, behavior_time_vector)\n",
+ " status = ephys_fix_sync_code.sync_evaluation_process2(\n",
+ " synced_time_vector, behavior_time_vector\n",
+ " )\n",
" if status == -1:\n",
" break\n",
"\n",
- " print('after sync_evaluation_process2', status)\n",
+ " print(\"after sync_evaluation_process2\", status)\n",
"\n",
" if status == 1:\n",
- " iteration_dict['trial_start_idx'] = ephys_utils.get_index_trial_vector_from_iteration(iteration_dict['iter_start_idx'])\n",
- "\n",
+ " iteration_dict[\"trial_start_idx\"] = (\n",
+ " ephys_utils.get_index_trial_vector_from_iteration(\n",
+ " iteration_dict[\"iter_start_idx\"]\n",
+ " )\n",
+ " )\n",
"\n",
" return status, iteration_dict"
]
@@ -196,69 +229,108 @@
"\n",
" # Get behavior key\n",
" behavior_key = (recording.Recording.BehaviorSession & key).fetch1()\n",
- " behavior_key.pop('recording_id')\n",
+ " behavior_key.pop(\"recording_id\")\n",
"\n",
- " if 'testuser' in behavior_key['subject_fullname']:\n",
+ " if \"testuser\" in behavior_key[\"subject_fullname\"]:\n",
" return\n",
"\n",
" # If a specific block is requested, add that to our behavior_key. It should be an int referring to virmen block number.\n",
" # This is useful for sessions in which the nidaq stream was interrupted due to restarting virmen\n",
- " if 'block' in kwargs:\n",
- " print('block: ', kwargs['block'])\n",
- " behavior_key['block'] = kwargs['block']\n",
+ " if \"block\" in kwargs:\n",
+ " print(\"block: \", kwargs[\"block\"])\n",
+ " behavior_key[\"block\"] = kwargs[\"block\"]\n",
"\n",
" print(behavior_key)\n",
"\n",
" # And get the datajoint record\n",
- " behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n",
+ " behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
" thissession = behavior.TowersBlock().Trial() & behavior_key\n",
- " behavior_time, iterstart = thissession.fetch('trial_time', 'vi_start')\n",
+ " behavior_time, iterstart = thissession.fetch(\"trial_time\", \"vi_start\")\n",
"\n",
- " print('len iterstart', len(iterstart))\n",
+ " print(\"len iterstart\", len(iterstart))\n",
"\n",
" if len(iterstart) == 0:\n",
- " raise ValueError('No behavior found')\n",
+ " raise ValueError(\"No behavior found\")\n",
"\n",
- " print('after reading behavior data')\n",
+ " print(\"after reading behavior data\")\n",
"\n",
" # 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n",
- " nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate(ephys_session_fullpath)\n",
+ " nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate(\n",
+ " ephys_session_fullpath\n",
+ " )\n",
"\n",
- " trial_pulse_signal, iteration_pulse_signal = ephys_utils.load_trial_iteration_signals(ephys_session_fullpath, nidq_meta)\n",
+ " trial_pulse_signal, iteration_pulse_signal = (\n",
+ " ephys_utils.load_trial_iteration_signals(ephys_session_fullpath, nidq_meta)\n",
+ " )\n",
"\n",
- " print('after reading spikeglx data')\n",
+ " print(\"after reading spikeglx data\")\n",
"\n",
" # Synchronize between pulses and get iteration # vector for each sample\n",
- " recent_recording = behavior_key['session_date'] > datetime.date(2021,6,1) # Everything past June 1 2021\n",
+ " recent_recording = behavior_key[\"session_date\"] > datetime.date(\n",
+ " 2021, 6, 1\n",
+ " ) # Everything past June 1 2021\n",
" if recent_recording:\n",
" # New synchronization method: digital_array[1,2] contain pulses for trial and frame number.\n",
- " mode=None\n",
- " iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal, nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode)\n",
+ " mode = None\n",
+ " iteration_dict = (\n",
+ " ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n",
+ " trial_pulse_signal,\n",
+ " iteration_pulse_signal,\n",
+ " nidq_sampling_rate,\n",
+ " behavior_time.shape[0],\n",
+ " behavior_time,\n",
+ " mode,\n",
+ " )\n",
+ " )\n",
" else:\n",
" # Old synchronization: digital_array[0:7] contain a digital word that counts the virmen frames.\n",
- " raise ValueError('Old sessions < 2022 not suported anymore')\n",
- " #iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_word(digital_array, behavior_time, iterstart)\n",
+ " raise ValueError(\"Old sessions < 2022 not suported anymore\")\n",
+ " # iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_word(digital_array, behavior_time, iterstart)\n",
"\n",
" # Check # of trials (from database record of behavior in `behavior_time`) and iterations (extracted from NIDAQ in `iter_start_idx`) match\n",
- " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n",
- "\n",
- " print('metrics to evaluate...')\n",
- " print(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n",
- "\n",
- " status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n",
+ " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n",
+ " ephys_utils.assert_iteration_samples_count(\n",
+ " iteration_dict[\"iter_start_idx\"], behavior_time\n",
+ " )\n",
+ " )\n",
+ "\n",
+ " print(\"metrics to evaluate...\")\n",
+ " print(\n",
+ " trial_count_diff,\n",
+ " trials_diff_iteration_big,\n",
+ " trials_diff_iteration_small,\n",
+ " behavior_time.shape[0],\n",
+ " )\n",
+ "\n",
+ " status = ephys_utils.evaluate_sync_process(\n",
+ " trial_count_diff,\n",
+ " trials_diff_iteration_big,\n",
+ " trials_diff_iteration_small,\n",
+ " behavior_time.shape[0],\n",
+ " )\n",
"\n",
" if status == 1:\n",
- " iteration_dict['trial_start_idx'] = ephys_utils.get_index_trial_vector_from_iteration(iteration_dict['iter_start_idx'])\n",
+ " iteration_dict[\"trial_start_idx\"] = (\n",
+ " ephys_utils.get_index_trial_vector_from_iteration(\n",
+ " iteration_dict[\"iter_start_idx\"]\n",
+ " )\n",
+ " )\n",
"\n",
- " #Failed sync by a lot, error\n",
+ " # Failed sync by a lot, error\n",
" status_regular = 1\n",
" status_fix = 0\n",
" if status < 1:\n",
" status_regular = 0\n",
- " print('Regular ephys sync failed')\n",
- " \n",
+ " print(\"Regular ephys sync failed\")\n",
+ "\n",
" try:\n",
- " status_fix, iteration_dict2 = main_ephys_fix_sync_code(iteration_dict['iter_start_idx'], iteration_dict['iter_times_idx'], behavior_time, nidq_sampling_rate, key)\n",
+ " status_fix, iteration_dict2 = main_ephys_fix_sync_code(\n",
+ " iteration_dict[\"iter_start_idx\"],\n",
+ " iteration_dict[\"iter_times_idx\"],\n",
+ " behavior_time,\n",
+ " nidq_sampling_rate,\n",
+ " key,\n",
+ " )\n",
" except:\n",
" status_fix = 0\n",
" iteration_dict2 = {}\n",
@@ -367,10 +439,8 @@
}
],
"source": [
- "\n",
"num_session = 0\n",
"for idx_session in range(len(recording_keys)):\n",
- "\n",
" recording_key = recording_keys[idx_session]\n",
" session_key = session_keys[idx_session]\n",
"\n",
@@ -378,22 +448,25 @@
" if status_fix == 0:\n",
" continue\n",
"\n",
- " print('session_key', session_key)\n",
+ " print(\"session_key\", session_key)\n",
"\n",
- " for i in range(len(iteration_dict2['trial_sync_stats'])):\n",
- " iteration_dict2['trial_sync_stats'][i] = iteration_dict2['trial_sync_stats'][i] | session_key\n",
+ " for i in range(len(iteration_dict2[\"trial_sync_stats\"])):\n",
+ " iteration_dict2[\"trial_sync_stats\"][i] = (\n",
+ " iteration_dict2[\"trial_sync_stats\"][i] | session_key\n",
+ " )\n",
+ "\n",
+ " print(\"status_fix\", status_fix)\n",
+ " print(\"num_session\", num_session)\n",
"\n",
- " print('status_fix', status_fix)\n",
- " print('num_session', num_session)\n",
- " \n",
" if num_session == 0:\n",
- " trial_sync_stats_df = pd.DataFrame(iteration_dict2['trial_sync_stats'])\n",
+ " trial_sync_stats_df = pd.DataFrame(iteration_dict2[\"trial_sync_stats\"])\n",
" else:\n",
- " trial_sync_stats_df = pd.concat([trial_sync_stats_df, pd.DataFrame(iteration_dict2['trial_sync_stats'])], axis=0)\n",
- "\n",
- " num_session += 1\n",
+ " trial_sync_stats_df = pd.concat(\n",
+ " [trial_sync_stats_df, pd.DataFrame(iteration_dict2[\"trial_sync_stats\"])],\n",
+ " axis=0,\n",
+ " )\n",
"\n",
- " \n"
+ " num_session += 1"
]
},
{
@@ -405,9 +478,9 @@
"def count_borrowed_iter(borrow_list):\n",
" num_borrow = 0\n",
" for i in range(len(borrow_list)):\n",
- " num_borrow = num_borrow+ borrow_list[i][1] - borrow_list[i][0] + 1\n",
+ " num_borrow = num_borrow + borrow_list[i][1] - borrow_list[i][0] + 1\n",
"\n",
- " return num_borrow\n"
+ " return num_borrow"
]
},
{
@@ -780,22 +853,38 @@
}
],
"source": [
+ "trial_sync_stats_df[\"min_shift_abs\"] = trial_sync_stats_df[\"min_shift\"].abs()\n",
+ "trial_sync_stats_df[\"shift\"] = trial_sync_stats_df[[\"max_shift\", \"min_shift_abs\"]].max(\n",
+ " axis=1\n",
+ ")\n",
"\n",
- "trial_sync_stats_df['min_shift_abs'] = trial_sync_stats_df['min_shift'].abs()\n",
- "trial_sync_stats_df['shift'] = trial_sync_stats_df[['max_shift', 'min_shift_abs']].max(axis=1)\n",
- "\n",
- "trial_sync_stats_df.loc[trial_sync_stats_df['shift']==trial_sync_stats_df['min_shift_abs'], 'shift'] = trial_sync_stats_df.loc[trial_sync_stats_df['shift']==trial_sync_stats_df['min_shift_abs'], 'shift']*-1\n",
+ "trial_sync_stats_df.loc[\n",
+ " trial_sync_stats_df[\"shift\"] == trial_sync_stats_df[\"min_shift_abs\"], \"shift\"\n",
+ "] = (\n",
+ " trial_sync_stats_df.loc[\n",
+ " trial_sync_stats_df[\"shift\"] == trial_sync_stats_df[\"min_shift_abs\"], \"shift\"\n",
+ " ]\n",
+ " * -1\n",
+ ")\n",
"\n",
- "trial_sync_stats_df['num_borrow_step2'] = trial_sync_stats_df['borrow_step2'].apply(count_borrowed_iter)\n",
- "#trial_sync_stats_df['num_borrow_step3'] = trial_sync_stats_df['borrow_step3'].apply(count_borrowed_iter)\n",
- "trial_sync_stats_df['num_borrow_step4'] = trial_sync_stats_df['borrow_step4'].apply(count_borrowed_iter)\n",
+ "trial_sync_stats_df[\"num_borrow_step2\"] = trial_sync_stats_df[\"borrow_step2\"].apply(\n",
+ " count_borrowed_iter\n",
+ ")\n",
+ "# trial_sync_stats_df['num_borrow_step3'] = trial_sync_stats_df['borrow_step3'].apply(count_borrowed_iter)\n",
+ "trial_sync_stats_df[\"num_borrow_step4\"] = trial_sync_stats_df[\"borrow_step4\"].apply(\n",
+ " count_borrowed_iter\n",
+ ")\n",
"\n",
"\n",
- "#trial_sync_stats_df['total_borrow'] = trial_sync_stats_df['num_borrow_step2'] + trial_sync_stats_df['num_borrow_step3'] + trial_sync_stats_df['num_borrow_step4'] \n",
+ "# trial_sync_stats_df['total_borrow'] = trial_sync_stats_df['num_borrow_step2'] + trial_sync_stats_df['num_borrow_step3'] + trial_sync_stats_df['num_borrow_step4']\n",
"\n",
- "trial_sync_stats_df['total_borrow'] = trial_sync_stats_df['num_borrow_step4'] + trial_sync_stats_df['num_borrow_step2']\n",
+ "trial_sync_stats_df[\"total_borrow\"] = (\n",
+ " trial_sync_stats_df[\"num_borrow_step4\"] + trial_sync_stats_df[\"num_borrow_step2\"]\n",
+ ")\n",
"\n",
- "trial_sync_stats_df['percentage_borrow'] = trial_sync_stats_df['total_borrow']*100 / trial_sync_stats_df['num_iterations']\n",
+ "trial_sync_stats_df[\"percentage_borrow\"] = (\n",
+ " trial_sync_stats_df[\"total_borrow\"] * 100 / trial_sync_stats_df[\"num_iterations\"]\n",
+ ")\n",
"\n",
"\n",
"trial_sync_stats_df"
@@ -1158,7 +1247,7 @@
}
],
"source": [
- "trial_sync_stats_df = trial_sync_stats_df.sort_values(by='median_diff', ascending=False)\n",
+ "trial_sync_stats_df = trial_sync_stats_df.sort_values(by=\"median_diff\", ascending=False)\n",
"trial_sync_stats_df"
]
},
@@ -1226,7 +1315,7 @@
}
],
"source": [
- "trial_sync_stats_df.loc[trial_sync_stats_df['median_diff'] < -0.0001, :]"
+ "trial_sync_stats_df.loc[trial_sync_stats_df[\"median_diff\"] < -0.0001, :]"
]
},
{
@@ -1246,7 +1335,7 @@
}
],
"source": [
- "trial_sync_stats_df['min_shift_abs'].max()"
+ "trial_sync_stats_df[\"min_shift_abs\"].max()"
]
},
{
@@ -1278,18 +1367,18 @@
"source": [
"plt.figure(figsize=(8, 6))\n",
"\n",
- "shift_stats = trial_sync_stats_df['median_diff'].copy()\n",
+ "shift_stats = trial_sync_stats_df[\"median_diff\"].copy()\n",
"num_0_shifts = (shift_stats == 0).sum()\n",
"\n",
"num_shifts = shift_stats[shift_stats != 0]\n",
"\n",
"\n",
- "plt.hist(num_shifts,bins=61)\n",
+ "plt.hist(num_shifts, bins=61)\n",
"\n",
"\n",
- "plt.xlabel('Time(s)')\n",
- "plt.ylabel('# Trials')\n",
- "plt.title('Median diff NIDAQ - ViRMEn')"
+ "plt.xlabel(\"Time(s)\")\n",
+ "plt.ylabel(\"# Trials\")\n",
+ "plt.title(\"Median diff NIDAQ - ViRMEn\")"
]
},
{
@@ -1298,11 +1387,19 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
- "trial_sync_stats_df['min_shift_abs'] = trial_sync_stats_df['min_shift'].abs()\n",
- "trial_sync_stats_df['shift'] = trial_sync_stats_df[['max_shift', 'min_shift_abs']].max(axis=1)\n",
- "\n",
- "trial_sync_stats_df.loc[trial_sync_stats_df['shift']==trial_sync_stats_df['min_shift_abs'], 'shift'] = trial_sync_stats_df.loc[trial_sync_stats_df['shift']==trial_sync_stats_df['min_shift_abs'], 'shift']*-1\n"
+ "trial_sync_stats_df[\"min_shift_abs\"] = trial_sync_stats_df[\"min_shift\"].abs()\n",
+ "trial_sync_stats_df[\"shift\"] = trial_sync_stats_df[[\"max_shift\", \"min_shift_abs\"]].max(\n",
+ " axis=1\n",
+ ")\n",
+ "\n",
+ "trial_sync_stats_df.loc[\n",
+ " trial_sync_stats_df[\"shift\"] == trial_sync_stats_df[\"min_shift_abs\"], \"shift\"\n",
+ "] = (\n",
+ " trial_sync_stats_df.loc[\n",
+ " trial_sync_stats_df[\"shift\"] == trial_sync_stats_df[\"min_shift_abs\"], \"shift\"\n",
+ " ]\n",
+ " * -1\n",
+ ")"
]
},
{
@@ -1334,7 +1431,7 @@
"source": [
"plt.figure(figsize=(8, 6))\n",
"\n",
- "shift_stats = trial_sync_stats_df['shift'].copy()\n",
+ "shift_stats = trial_sync_stats_df[\"shift\"].copy()\n",
"num_0_shifts = (shift_stats == 0).sum()\n",
"\n",
"num_shifts = shift_stats[shift_stats != 0]\n",
@@ -1342,10 +1439,14 @@
"\n",
"plt.hist(num_shifts)\n",
"\n",
- "plt.yscale('log')\n",
- "plt.xlabel('# Shift iterations')\n",
- "plt.ylabel('# Trials')\n",
- "plt.title('\"Max\" shift in trial to preserve median: '+ str(num_0_shifts)+ ' trials without shift')"
+ "plt.yscale(\"log\")\n",
+ "plt.xlabel(\"# Shift iterations\")\n",
+ "plt.ylabel(\"# Trials\")\n",
+ "plt.title(\n",
+ " '\"Max\" shift in trial to preserve median: '\n",
+ " + str(num_0_shifts)\n",
+ " + \" trials without shift\"\n",
+ ")"
]
},
{
@@ -1378,13 +1479,12 @@
"plt.figure(figsize=(8, 6))\n",
"\n",
"\n",
+ "plt.hist(trial_sync_stats_df[\"percentage_borrow\"])\n",
"\n",
- "plt.hist(trial_sync_stats_df['percentage_borrow'])\n",
- "\n",
- "plt.yscale('log')\n",
- "plt.xlabel('% Trial')\n",
- "plt.ylabel('# Trials')\n",
- "plt.title('Trial NIDAQ iterations borrowed from ViRMEn')"
+ "plt.yscale(\"log\")\n",
+ "plt.xlabel(\"% Trial\")\n",
+ "plt.ylabel(\"# Trials\")\n",
+ "plt.title(\"Trial NIDAQ iterations borrowed from ViRMEn\")"
]
},
{
@@ -1404,7 +1504,7 @@
}
],
"source": [
- "np.max(trial_sync_stats_df['percentage_borrow'])"
+ "np.max(trial_sync_stats_df[\"percentage_borrow\"])"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb
index b50391ba..91436d0f 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -41,23 +42,20 @@
"source": [
"import datetime\n",
"import pathlib\n",
- "import numpy as np\n",
- "import pylab as pl\n",
- "import matplotlib\n",
- "import matplotlib.pyplot as plt\n",
- "from scipy.signal import find_peaks\n",
- "from scipy import signal as sp\n",
- "\n",
"\n",
- "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory, get_ephys_root_data_dir\n",
- "# import u19_pipeline.ephys_sync as ephys\n",
- "import u19_pipeline.acquisition as acquisition\n",
"import datajoint as dj\n",
- "\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
"from element_interface.utils import find_full_path\n",
"\n",
+ "# import u19_pipeline.ephys_sync as ephys\n",
+ "import u19_pipeline.acquisition as acquisition\n",
"import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n",
- "import u19_pipeline.utils.ephys_utils as ephys_utils\n"
+ "import u19_pipeline.utils.ephys_utils as ephys_utils\n",
+ "from u19_pipeline.ephys_pipeline import (\n",
+ " get_ephys_root_data_dir,\n",
+ " get_session_directory,\n",
+ ")"
]
},
{
@@ -77,25 +75,26 @@
}
],
"source": [
- "key = {'subject_fullname': 'jyanar_ya054',\n",
- " 'session_date': datetime.date(2025, 10, 10)}\n",
- "#key = {\n",
+ "key = {\"subject_fullname\": \"jyanar_ya054\", \"session_date\": datetime.date(2025, 10, 10)}\n",
+ "# key = {\n",
"# 'subject_fullname': 'jk8386_jk83',\n",
"# 'session_date': datetime.date(2025, 7, 31),\n",
"# 'session_number': 0 }\n",
"\n",
- "#key2 = 'subject_fullname = \"jjulian_jj046\" and session_date = \"2022-04-19\" and block >=2'\n",
- "#key = {\n",
+ "# key2 = 'subject_fullname = \"jjulian_jj046\" and session_date = \"2022-04-19\" and block >=2'\n",
+ "# key = {\n",
"# 'subject_fullname': 'jjulian_jj042',\n",
"# 'session_date': datetime.date(2021, 10, 31)}\n",
"\n",
"\n",
- "key = (acquisition.Session & key).fetch1('KEY')\n",
+ "key = (acquisition.Session & key).fetch1(\"KEY\")\n",
"key\n",
"\n",
- "recording = dj.create_virtual_module('recording', 'u19_recording')\n",
+ "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n",
"\n",
- "recording_key = ((acquisition.Session * recording.Recording.BehaviorSession) & key).fetch('recording_id', as_dict=True)\n",
+ "recording_key = (\n",
+ " (acquisition.Session * recording.Recording.BehaviorSession) & key\n",
+ ").fetch(\"recording_id\", as_dict=True)\n",
"\n",
"recording_key"
]
@@ -118,7 +117,7 @@
],
"source": [
"session_dir = pathlib.Path(get_session_directory(recording_key[0]))\n",
- "session_dir\n"
+ "session_dir"
]
},
{
@@ -136,38 +135,43 @@
}
],
"source": [
- "session_dir = find_full_path(get_ephys_root_data_dir(),\n",
- " get_session_directory(recording_key))\n",
+ "session_dir = find_full_path(\n",
+ " get_ephys_root_data_dir(), get_session_directory(recording_key)\n",
+ ")\n",
"print(session_dir)\n",
- "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
- "#Check if session is Nidq or OneBox\n",
- "nidq_session = list(session_dir.glob('*nidq.bin*'))\n",
- "obx_session = list(session_dir.glob('*obx.bin*'))\n",
+ "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
+ "# Check if session is Nidq or OneBox\n",
+ "nidq_session = list(session_dir.glob(\"*nidq.bin*\"))\n",
+ "obx_session = list(session_dir.glob(\"*obx.bin*\"))\n",
"\n",
"if len(nidq_session) == 0 and len(obx_session) == 0:\n",
- " print('No session found')\n",
+ " print(\"No session found\")\n",
"elif len(nidq_session) > 0:\n",
" ephys_session_fullpath = nidq_session[0]\n",
"else:\n",
" ephys_session_fullpath = obx_session[0]\n",
"\n",
- "#Nidaq file\n",
- "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n",
+ "# Nidaq file\n",
+ "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n",
"nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n",
"\n",
"\n",
- "# 1: load meta data, and the content of the NIDAQ file. Its content is digital. \n",
+ "# 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n",
"new_trial_channel = 1\n",
"new_iteration_channel = 2\n",
"# If PXIe card (nidq) card use for recording deduce digital channels\n",
- "if nidq_meta['typeThis'] == 'nidq':\n",
- " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)\n",
+ "if nidq_meta[\"typeThis\"] == \"nidq\":\n",
+ " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " ephys_session_fullpath, nidq_meta\n",
+ " )\n",
"# If onebox card (obx) card use for recording digital channels are 0-2\n",
"else:\n",
- " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])\n",
+ " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]\n",
+ " )\n",
" # If no sync pulse found trial and iteration signals are 0 & 1 respectively\n",
- " channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]\n",
- " channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]\n",
+ " channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]\n",
+ " channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]\n",
"\n",
" if channel0_pulses > channel1_pulses:\n",
" new_trial_channel = 1\n",
@@ -204,18 +208,20 @@
"metadata": {},
"outputs": [],
"source": [
- "#Behavior data\n",
+ "# Behavior data\n",
"behavior_key = key.copy()\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
"\n",
"block = 0\n",
"# If a specific block is requested, add that to our behavior_key. It should be an int referring to virmen block number.\n",
"# This is useful for sessions in which the nidaq stream was interrupted due to restarting virmen\n",
"if block > 0:\n",
- " behavior_key['block'] = block\n",
+ " behavior_key[\"block\"] = block\n",
"\n",
"thissession = behavior.TowersBlock().Trial() & behavior_key\n",
- "behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n"
+ "behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n",
+ " \"trial_time\", \"vi_start\", \"iterations\"\n",
+ ")"
]
},
{
@@ -292,21 +298,37 @@
}
],
"source": [
- "mode = None #Default for sessions before 12/01/2021\n",
- "#mode = 'pulses' #Default for sessions after 12/01/2021\n",
- "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[new_trial_channel,:], digital_array[new_iteration_channel,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n",
+ "mode = None # Default for sessions before 12/01/2021\n",
+ "# mode = 'pulses' #Default for sessions after 12/01/2021\n",
+ "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n",
+ " digital_array[new_trial_channel, :],\n",
+ " digital_array[new_iteration_channel, :],\n",
+ " nidq_sampling_rate,\n",
+ " behavior_time.shape[0],\n",
+ " behavior_time,\n",
+ " mode=mode,\n",
+ ")\n",
"# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n",
"\n",
"\n",
"# Check # of trials and iterations match\n",
- "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n",
+ "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n",
+ " ephys_utils.assert_iteration_samples_count(\n",
+ " iteration_dict[\"iter_start_idx\"], behavior_time\n",
+ " )\n",
+ ")\n",
"\n",
"print(trial_count_diff)\n",
"print(trials_diff_iteration_big)\n",
"print(trials_diff_iteration_small)\n",
"\n",
"\n",
- "status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n"
+ "status = ephys_utils.evaluate_sync_process(\n",
+ " trial_count_diff,\n",
+ " trials_diff_iteration_big,\n",
+ " trials_diff_iteration_small,\n",
+ " behavior_time.shape[0],\n",
+ ")"
]
},
{
@@ -315,8 +337,11 @@
"metadata": {},
"outputs": [],
"source": [
- "for i in range(len(iteration_dict['iter_start_idx'])):\n",
- " if iteration_dict['iter_start_idx'][i].shape[0] > behavior_time[i].flatten().shape[0]:\n",
+ "for i in range(len(iteration_dict[\"iter_start_idx\"])):\n",
+ " if (\n",
+ " iteration_dict[\"iter_start_idx\"][i].shape[0]\n",
+ " > behavior_time[i].flatten().shape[0]\n",
+ " ):\n",
" print(i)"
]
},
@@ -336,8 +361,8 @@
],
"source": [
"trial_fix = 287\n",
- "synced_iteration_vector = iteration_dict['iter_start_idx'][trial_fix]\n",
- "synced_time_vector = iteration_dict['iter_times_idx'][trial_fix]\n",
+ "synced_iteration_vector = iteration_dict[\"iter_start_idx\"][trial_fix]\n",
+ "synced_time_vector = iteration_dict[\"iter_times_idx\"][trial_fix]\n",
"behavior_time_vector = behavior_time[trial_fix].flatten()\n",
"\n",
"print(synced_iteration_vector.shape[0])\n",
@@ -361,8 +386,7 @@
}
],
"source": [
- "\n",
- "'''\n",
+ "\"\"\"\n",
"#synced_time_vector = synced_time_vector+behavior_time_vector[1]-(behavior_time_vector[2]-synced_time_vector[2])\n",
"synced_time_vector = synced_time_vector+behavior_time_vector[1]\n",
"synced_time_vector = np.insert(synced_time_vector, 1, behavior_time_vector[1])\n",
@@ -375,7 +399,7 @@
"\n",
"print(synced_time_vector[0:5])\n",
"print(behavior_time_vector[0:5])\n",
- "'''"
+ "\"\"\""
]
},
{
@@ -408,51 +432,58 @@
"initial_sample = 0\n",
"base_size = 40\n",
"\n",
- "baseline_diff = synced_time_vector[initial_sample:base_size] - behavior_time_vector[initial_sample:base_size] \n",
+ "baseline_diff = (\n",
+ " synced_time_vector[initial_sample:base_size]\n",
+ " - behavior_time_vector[initial_sample:base_size]\n",
+ ")\n",
"\n",
"count_iter_first = 0\n",
- "while(1):\n",
- "\n",
- " count_iter_first = count_iter_first+1\n",
- "\n",
+ "while 1:\n",
+ " count_iter_first = count_iter_first + 1\n",
"\n",
- " base_greater = np.where(baseline_diff >0)\n",
+ " base_greater = np.where(baseline_diff > 0)\n",
" base_greater = base_greater[0]\n",
"\n",
- " if base_greater.shape[0] > 0 and synced_time_vector.shape[0] < behavior_time_vector.shape[0]:\n",
+ " if (\n",
+ " base_greater.shape[0] > 0\n",
+ " and synced_time_vector.shape[0] < behavior_time_vector.shape[0]\n",
+ " ):\n",
" print(count_iter_first)\n",
" idx_first = base_greater[0]\n",
- " time_bef = behavior_time_vector[idx_first] - behavior_time_vector[idx_first-1]\n",
+ " time_bef = behavior_time_vector[idx_first] - behavior_time_vector[idx_first - 1]\n",
"\n",
+ " print(\"idx_first\", idx_first)\n",
+ " print(\"time_bef\", time_bef)\n",
+ " print(\"synced_time_vector\", synced_time_vector[idx_first - 2 : idx_first + 2])\n",
+ " print(\n",
+ " \"behavior_time_vector\", behavior_time_vector[idx_first - 2 : idx_first + 2]\n",
+ " )\n",
"\n",
- " print('idx_first', idx_first)\n",
- " print('time_bef', time_bef)\n",
- " print('synced_time_vector', synced_time_vector[idx_first-2:idx_first+2])\n",
- " print('behavior_time_vector', behavior_time_vector[idx_first-2:idx_first+2])\n",
+ " synced_time_vector = np.insert(\n",
+ " synced_time_vector, idx_first, synced_time_vector[idx_first - 1] + time_bef\n",
+ " )\n",
"\n",
- " synced_time_vector = np.insert(synced_time_vector, idx_first, synced_time_vector[idx_first-1]+time_bef)\n",
+ " print(\"synced_time_vector\", synced_time_vector[idx_first - 2 : idx_first + 2])\n",
"\n",
- " print('synced_time_vector', synced_time_vector[idx_first-2:idx_first+2])\n",
- "\n",
- " baseline_diff = synced_time_vector[initial_sample:base_size] - behavior_time_vector[initial_sample:base_size]\n",
+ " baseline_diff = (\n",
+ " synced_time_vector[initial_sample:base_size]\n",
+ " - behavior_time_vector[initial_sample:base_size]\n",
+ " )\n",
" else:\n",
- " break \n",
- "\n",
- "\n",
- "\n",
+ " break\n",
"\n",
"\n",
"median_diff = np.median(baseline_diff)\n",
- "max_diff = np.median(baseline_diff)+0.007\n",
- "min_diff = np.median(baseline_diff)-0.007\n",
- "time_vec = np.arange(base_size-initial_sample)\n",
- "#plt.plot([0, base_size-initial_sample], [median_diff, median_diff],'k')\n",
+ "max_diff = np.median(baseline_diff) + 0.007\n",
+ "min_diff = np.median(baseline_diff) - 0.007\n",
+ "time_vec = np.arange(base_size - initial_sample)\n",
+ "# plt.plot([0, base_size-initial_sample], [median_diff, median_diff],'k')\n",
"plt.plot(baseline_diff)\n",
- "plt.plot([0, base_size-initial_sample], [max_diff, max_diff],'r')\n",
- "plt.plot([0, base_size-initial_sample], [min_diff, min_diff],'g')\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time (s)')\n",
- "plt.title('Difference first iterations NIDAQ Pulse time - Virmen Behavior time')"
+ "plt.plot([0, base_size - initial_sample], [max_diff, max_diff], \"r\")\n",
+ "plt.plot([0, base_size - initial_sample], [min_diff, min_diff], \"g\")\n",
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time (s)\")\n",
+ "plt.title(\"Difference first iterations NIDAQ Pulse time - Virmen Behavior time\")"
]
},
{
@@ -472,7 +503,7 @@
"source": [
"print(synced_time_vector[0:5])\n",
"\n",
- "print(behavior_time_vector[0:5])\n"
+ "print(behavior_time_vector[0:5])"
]
},
{
@@ -570,20 +601,15 @@
}
],
"source": [
- "\n",
- "vec_shift = np.zeros((synced_time_vector.shape[0]-base_size), dtype=int)\n",
- "for i in range(synced_time_vector.shape[0]-base_size):\n",
- "\n",
- "\n",
- " idx_start = initial_sample+i+1\n",
- " idx_end = base_size+i+1\n",
+ "vec_shift = np.zeros((synced_time_vector.shape[0] - base_size), dtype=int)\n",
+ "for i in range(synced_time_vector.shape[0] - base_size):\n",
+ " idx_start = initial_sample + i + 1\n",
+ " idx_end = base_size + i + 1\n",
"\n",
" this_bt = behavior_time_vector[idx_start:idx_end]\n",
" this_iv = synced_time_vector[idx_start:idx_end]\n",
"\n",
- " median_ori = np.median(this_iv-this_bt)\n",
- "\n",
- " \n",
+ " median_ori = np.median(this_iv - this_bt)\n",
"\n",
" if median_ori >= min_diff and median_ori < max_diff:\n",
" vec_shift[i] = 0\n",
@@ -594,26 +620,24 @@
" else:\n",
" sign = -1\n",
" print(i, -1)\n",
- " \n",
+ "\n",
" new_sign = sign\n",
" if sign != 0:\n",
- "\n",
" for j in range(1, 100):\n",
- "\n",
" if new_sign == 1:\n",
- " if idx_end+j > synced_time_vector.shape[0]:\n",
- " this_iv = synced_time_vector[idx_start+j:]\n",
+ " if idx_end + j > synced_time_vector.shape[0]:\n",
+ " this_iv = synced_time_vector[idx_start + j :]\n",
" this_bt = this_bt[:-1]\n",
" else:\n",
- " this_iv = synced_time_vector[idx_start+j:idx_end+j]\n",
+ " this_iv = synced_time_vector[idx_start + j : idx_end + j]\n",
" else:\n",
- " this_iv = synced_time_vector[idx_start-j:idx_end-j]\n",
- " \n",
- " median_now = np.median(this_iv-this_bt)\n",
- " #print('median_now', median_now, 'sign', sign)\n",
- " \n",
+ " this_iv = synced_time_vector[idx_start - j : idx_end - j]\n",
+ "\n",
+ " median_now = np.median(this_iv - this_bt)\n",
+ " # print('median_now', median_now, 'sign', sign)\n",
+ "\n",
" if median_now >= min_diff and median_now < max_diff:\n",
- " vec_shift[i] = j*new_sign\n",
+ " vec_shift[i] = j * new_sign\n",
" break\n",
" elif median_now < min_diff:\n",
" new_sign = 1\n",
@@ -621,16 +645,14 @@
" new_sign = -1\n",
"\n",
" if new_sign != sign:\n",
- " if np.abs(median_ori-median_diff) < np.abs(median_now-median_diff):\n",
+ " if np.abs(median_ori - median_diff) < np.abs(median_now - median_diff):\n",
" vec_shift[i] = 0\n",
" else:\n",
- " vec_shift[i] = j*sign \n",
+ " vec_shift[i] = j * sign\n",
" break\n",
- " \n",
"\n",
" if j == 99:\n",
- " vec_shift[i] = j*sign\n",
- "\n",
+ " vec_shift[i] = j * sign\n",
"\n",
"\n",
"print(vec_shift)"
@@ -663,15 +685,15 @@
}
],
"source": [
- "#plt.plot(np.diff(new_synced_time_vector))\n",
+ "# plt.plot(np.diff(new_synced_time_vector))\n",
"plt.plot(np.diff(synced_time_vector))\n",
"plt.plot(np.diff(behavior_time_vector))\n",
- "plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n",
- "#plt.plot(np.diff(new_synced_time_vector3))\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time (s)')\n",
- "plt.title('Difference iterations NIDAQ Pulse time - Virmen Behavior time')\n",
- "#plt.title('Iteration Duration NIDAQ')\n",
+ "plt.plot(synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]])\n",
+ "# plt.plot(np.diff(new_synced_time_vector3))\n",
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time (s)\")\n",
+ "plt.title(\"Difference iterations NIDAQ Pulse time - Virmen Behavior time\")\n",
+ "# plt.title('Iteration Duration NIDAQ')\n",
"plt.ylim([-0.04, 0.04])\n",
"plt.xlim([1000, 1100])"
]
@@ -683,17 +705,16 @@
"outputs": [],
"source": [
"new_synced_time_vector = synced_time_vector.copy()\n",
- "mid_point = int(np.floor(initial_sample+base_size/2))\n",
+ "mid_point = int(np.floor(initial_sample + base_size / 2))\n",
"for i in range(vec_shift.shape[0]):\n",
- " new_synced_time_vector[mid_point+i] = synced_time_vector[mid_point+i+vec_shift[i]]\n",
- " \n",
- "idx_end = mid_point+vec_shift.shape[0]-1\n",
- "for i in range(idx_end, new_synced_time_vector.shape[0]):\n",
- "\n",
- " if i+vec_shift[-1] < new_synced_time_vector.shape[0]:\n",
- " new_synced_time_vector[i] = synced_time_vector[i+vec_shift[-1]]\n",
+ " new_synced_time_vector[mid_point + i] = synced_time_vector[\n",
+ " mid_point + i + vec_shift[i]\n",
+ " ]\n",
"\n",
- "\n"
+ "idx_end = mid_point + vec_shift.shape[0] - 1\n",
+ "for i in range(idx_end, new_synced_time_vector.shape[0]):\n",
+ " if i + vec_shift[-1] < new_synced_time_vector.shape[0]:\n",
+ " new_synced_time_vector[i] = synced_time_vector[i + vec_shift[-1]]"
]
},
{
@@ -711,18 +732,16 @@
}
],
"source": [
- "\n",
"diff_vec_shift = np.diff(vec_shift)\n",
"where_insert_iteration = np.where(diff_vec_shift != 0)\n",
"where_insert_iteration = where_insert_iteration[0]\n",
- "print('where_insert_iteration', where_insert_iteration+mid_point)\n",
+ "print(\"where_insert_iteration\", where_insert_iteration + mid_point)\n",
"\n",
"find = np.full(4, 1)\n",
"correlation_result = np.correlate(diff_vec_shift == 0, find)\n",
"consecutive_zeros = np.flatnonzero(correlation_result == 4)\n",
"\n",
"\n",
- "\n",
"index_shift = 0\n",
"index_borrow_virmen = list()\n",
"for i in range(len(where_insert_iteration)):\n",
@@ -730,71 +749,78 @@
" stable_parts = stable_parts[0]\n",
"\n",
" if stable_parts.shape[0] > 0:\n",
- " index_borrow_virmen.append([where_insert_iteration[index_shift], consecutive_zeros[stable_parts[0]]])\n",
- " next_borrow_start = np.where(where_insert_iteration > consecutive_zeros[stable_parts[0]])\n",
+ " index_borrow_virmen.append(\n",
+ " [where_insert_iteration[index_shift], consecutive_zeros[stable_parts[0]]]\n",
+ " )\n",
+ " next_borrow_start = np.where(\n",
+ " where_insert_iteration > consecutive_zeros[stable_parts[0]]\n",
+ " )\n",
" next_borrow_start = next_borrow_start[0]\n",
" if next_borrow_start.shape[0] > 0:\n",
- " index_shift = next_borrow_start[0]\n",
+ " index_shift = next_borrow_start[0]\n",
" else:\n",
" break\n",
" else:\n",
- " print('Extreme case diff vec shift')\n",
+ " print(\"Extreme case diff vec shift\")\n",
"\n",
- " index_borrow_virmen.append([where_insert_iteration[index_shift], diff_vec_shift.shape[0]])\n",
- " print('index_borrow_virmen', index_borrow_virmen)\n",
+ " index_borrow_virmen.append(\n",
+ " [where_insert_iteration[index_shift], diff_vec_shift.shape[0]]\n",
+ " )\n",
+ " print(\"index_borrow_virmen\", index_borrow_virmen)\n",
" break\n",
"\n",
"\n",
- "\n",
- "print('index_borrow_virmen', index_borrow_virmen)\n",
+ "print(\"index_borrow_virmen\", index_borrow_virmen)\n",
"\n",
"\n",
"new_synced_time_vector2 = new_synced_time_vector.copy()\n",
"\n",
- "#for i in range(where_insert_iteration.shape[0]):\n",
+ "# for i in range(where_insert_iteration.shape[0]):\n",
"index_diff_e = -1\n",
"for i in range(len(index_borrow_virmen)):\n",
- "\n",
- " if index_diff_e > index_borrow_virmen[i][0]+mid_point:\n",
+ " if index_diff_e > index_borrow_virmen[i][0] + mid_point:\n",
" continue\n",
"\n",
" for j in range(100):\n",
- "\n",
- " index_diff_s = index_borrow_virmen[i][0]+mid_point\n",
+ " index_diff_s = index_borrow_virmen[i][0] + mid_point\n",
" index_diff_vec = index_borrow_virmen[i][0]\n",
- " index_diff_e = index_borrow_virmen[i][1]+mid_point\n",
+ " index_diff_e = index_borrow_virmen[i][1] + mid_point\n",
"\n",
- " if j>=1:\n",
- " index_diff_s = index_diff_s-1\n",
- " if j >1:\n",
- " index_diff_e = index_diff_e+j-1\n",
+ " if j >= 1:\n",
+ " index_diff_s = index_diff_s - 1\n",
+ " if j > 1:\n",
+ " index_diff_e = index_diff_e + j - 1\n",
"\n",
" print(j)\n",
"\n",
- " print('index_diff_vec', index_diff_s, index_diff_e)\n",
- " #print('diff_vec_shift[index_diff] ', diff_vec_shift[index_diff_vec-1:index_diff_vec+1] )\n",
+ " print(\"index_diff_vec\", index_diff_s, index_diff_e)\n",
+ " # print('diff_vec_shift[index_diff] ', diff_vec_shift[index_diff_vec-1:index_diff_vec+1] )\n",
+ "\n",
+ " # print('behavior_time_vector', behavior_time_vector[index_diff_s-1:index_diff_e+2])\n",
+ " # print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])\n",
"\n",
- " #print('behavior_time_vector', behavior_time_vector[index_diff_s-1:index_diff_e+2])\n",
- " #print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])\n",
+ " time_iteration = (\n",
+ " behavior_time_vector[index_diff_s : index_diff_e + 1]\n",
+ " - behavior_time_vector[index_diff_s]\n",
+ " )\n",
+ " # print('time_iteration', time_iteration)\n",
"\n",
- " time_iteration = behavior_time_vector[index_diff_s:index_diff_e+1] - behavior_time_vector[index_diff_s]\n",
- " #print('time_iteration', time_iteration)\n",
+ " new_synced_time_vector2[index_diff_s : index_diff_e + 1] = (\n",
+ " np.repeat(\n",
+ " new_synced_time_vector2[index_diff_s], index_diff_e - index_diff_s + 1\n",
+ " )\n",
+ " + time_iteration\n",
+ " )\n",
"\n",
- " new_synced_time_vector2[index_diff_s:index_diff_e+1] = np.repeat(new_synced_time_vector2[index_diff_s], index_diff_e-index_diff_s+1) +\\\n",
- " time_iteration\n",
- " \n",
- " check_diff = np.diff(new_synced_time_vector2[index_diff_s:index_diff_e+2])\n",
+ " check_diff = np.diff(new_synced_time_vector2[index_diff_s : index_diff_e + 2])\n",
" idx_back_time = np.where(check_diff <= 0.0005)\n",
" idx_back_time = idx_back_time[0]\n",
" if idx_back_time.shape[0] == 0:\n",
" break\n",
- " \n",
"\n",
- " \n",
- " #print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])\n",
- " #print('diff new_synced_time_vector2', np.diff(new_synced_time_vector2[index_diff_s-1:index_diff_e+2]))\n",
- " #print('diff new_synced_time_vector2', np.diff(behavior_time_vector[index_diff_s-1:index_diff_e+2]))\n",
- "\n"
+ " # print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])\n",
+ " # print('diff new_synced_time_vector2', np.diff(new_synced_time_vector2[index_diff_s-1:index_diff_e+2]))\n",
+ " # print('diff new_synced_time_vector2', np.diff(behavior_time_vector[index_diff_s-1:index_diff_e+2]))"
]
},
{
@@ -832,10 +858,12 @@
],
"source": [
"new_synced_time_vector3 = new_synced_time_vector2.copy()\n",
- "diff_vecs = (new_synced_time_vector2 - behavior_time_vector[:new_synced_time_vector2.shape[0]])\n",
+ "diff_vecs = (\n",
+ " new_synced_time_vector2 - behavior_time_vector[: new_synced_time_vector2.shape[0]]\n",
+ ")\n",
"\n",
"idx_plus = np.where(diff_vecs > 1000)\n",
- "#idx_plus = np.where(diff_vecs > max_diff)\n",
+ "# idx_plus = np.where(diff_vecs > max_diff)\n",
"idx_plus = idx_plus[0]\n",
"\n",
"break_points = np.where(np.diff(idx_plus) != 1)[0] + 1\n",
@@ -844,17 +872,18 @@
"print(idx_plus)\n",
"print(grouped_sections)\n",
"for i in range(len(grouped_sections)):\n",
- "\n",
" if grouped_sections[i].shape[0] > 0:\n",
- "\n",
- " idx_start = grouped_sections[i][0]-1\n",
+ " idx_start = grouped_sections[i][0] - 1\n",
" idx_end = grouped_sections[i][-1]\n",
"\n",
- " time_vector = behavior_time_vector[idx_start:idx_end+1]-behavior_time_vector[idx_start]\n",
- " new_synced_time_vector3[idx_start:idx_end+1] =\\\n",
- " np.repeat(new_synced_time_vector3[idx_start], idx_end-idx_start+1) + time_vector\n",
- "\n",
- " "
+ " time_vector = (\n",
+ " behavior_time_vector[idx_start : idx_end + 1]\n",
+ " - behavior_time_vector[idx_start]\n",
+ " )\n",
+ " new_synced_time_vector3[idx_start : idx_end + 1] = (\n",
+ " np.repeat(new_synced_time_vector3[idx_start], idx_end - idx_start + 1)\n",
+ " + time_vector\n",
+ " )"
]
},
{
@@ -881,19 +910,18 @@
"size_vec = new_synced_time_vector4.shape[0]\n",
"diff_size = behavior_time_vector.shape[0] - new_synced_time_vector3.shape[0]\n",
"\n",
- "print('diff_size', diff_size)\n",
+ "print(\"diff_size\", diff_size)\n",
"\n",
"if diff_size > 0:\n",
- " diff_size = diff_size+1\n",
+ " diff_size = diff_size + 1\n",
"\n",
- " last_part_bt = behavior_time_vector[-diff_size:]- behavior_time_vector[-diff_size]\n",
+ " last_part_bt = behavior_time_vector[-diff_size:] - behavior_time_vector[-diff_size]\n",
" insert_part = np.repeat(new_synced_time_vector4[-1], diff_size) + last_part_bt\n",
- " print('last_part_bt',last_part_bt)\n",
+ " print(\"last_part_bt\", last_part_bt)\n",
"\n",
- " print('insert_part',insert_part)\n",
+ " print(\"insert_part\", insert_part)\n",
"\n",
- " new_synced_time_vector4 = np.append(new_synced_time_vector4, insert_part[1:])\n",
- "\n"
+ " new_synced_time_vector4 = np.append(new_synced_time_vector4, insert_part[1:])"
]
},
{
@@ -942,41 +970,44 @@
}
],
"source": [
- "\n",
"plt.figure(figsize=(8, 6))\n",
- "#plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n",
- "\n",
- "\n",
- "\n",
- "#plt.plot(new_synced_time_vector2 - behavior_time_vector[:new_synced_time_vector2.shape[0]])\n",
- "#plt.plot(np.diff(new_synced_time_vector))\n",
- "#plt.plot(np.diff(new_synced_time_vector2))\n",
- "#plt.plot(np.diff(behavior_time_vector))\n",
- "#plt.plot(np.diff(new_synced_time_vector3))\n",
- "plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n",
- "#plt.plot(new_synced_time_vector - behavior_time_vector[:new_synced_time_vector.shape[0]])\n",
- "#plt.plot(new_synced_time_vector2 - behavior_time_vector[:new_synced_time_vector3.shape[0]])\n",
- "#plt.plot(new_synced_time_vector4 - behavior_time_vector[:new_synced_time_vector4.shape[0]], color ='#ff7f0e')\n",
- "#plt.plot(new_synced_time_vector3 - behavior_time_vector[:new_synced_time_vector3.shape[0]], color='#1f77b4')\n",
- "\n",
- "plt.plot((new_synced_time_vector4 - behavior_time_vector[:new_synced_time_vector4.shape[0]]))\n",
- "\n",
- "time_shift = np.arange(int(initial_sample+base_size/2), vec_shift.shape[0]+int(initial_sample+base_size/2))\n",
- "#plt.plot([time_shift[0], time_shift[-1]],[min_diff, min_diff], 'g')\n",
- "#plt.plot([time_shift[0], time_shift[-1]],[max_diff, max_diff], 'r')\n",
- "#plt.plot([time_shift[0], 500],[0, 0], 'k')\n",
- "\n",
- "#plt.plot([491, 491],[min_diff, max_diff], 'k')\n",
- "\n",
- "#plt.plot(time_shift,vec_shift*0.01)\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time (s)')\n",
- "plt.title('Difference iterations NIDAQ Pulse time - Virmen Behavior time')\n",
- "#plt.title('Iteration Duration NIDAQ')\n",
+ "# plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n",
+ "\n",
+ "\n",
+ "# plt.plot(new_synced_time_vector2 - behavior_time_vector[:new_synced_time_vector2.shape[0]])\n",
+ "# plt.plot(np.diff(new_synced_time_vector))\n",
+ "# plt.plot(np.diff(new_synced_time_vector2))\n",
+ "# plt.plot(np.diff(behavior_time_vector))\n",
+ "# plt.plot(np.diff(new_synced_time_vector3))\n",
+ "plt.plot(synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]])\n",
+ "# plt.plot(new_synced_time_vector - behavior_time_vector[:new_synced_time_vector.shape[0]])\n",
+ "# plt.plot(new_synced_time_vector2 - behavior_time_vector[:new_synced_time_vector3.shape[0]])\n",
+ "# plt.plot(new_synced_time_vector4 - behavior_time_vector[:new_synced_time_vector4.shape[0]], color ='#ff7f0e')\n",
+ "# plt.plot(new_synced_time_vector3 - behavior_time_vector[:new_synced_time_vector3.shape[0]], color='#1f77b4')\n",
+ "\n",
+ "plt.plot(\n",
+ " new_synced_time_vector4 - behavior_time_vector[: new_synced_time_vector4.shape[0]]\n",
+ ")\n",
+ "\n",
+ "time_shift = np.arange(\n",
+ " int(initial_sample + base_size / 2),\n",
+ " vec_shift.shape[0] + int(initial_sample + base_size / 2),\n",
+ ")\n",
+ "# plt.plot([time_shift[0], time_shift[-1]],[min_diff, min_diff], 'g')\n",
+ "# plt.plot([time_shift[0], time_shift[-1]],[max_diff, max_diff], 'r')\n",
+ "# plt.plot([time_shift[0], 500],[0, 0], 'k')\n",
+ "\n",
+ "# plt.plot([491, 491],[min_diff, max_diff], 'k')\n",
+ "\n",
+ "# plt.plot(time_shift,vec_shift*0.01)\n",
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time (s)\")\n",
+ "plt.title(\"Difference iterations NIDAQ Pulse time - Virmen Behavior time\")\n",
+ "# plt.title('Iteration Duration NIDAQ')\n",
"plt.ylim([-0.04, 0.2])\n",
- "#plt.ylim([-0.01, 0.03])\n",
- "#plt.ylim([-0.005, 0.05])\n",
- "#plt.xlim([480, 500])"
+ "# plt.ylim([-0.01, 0.03])\n",
+ "# plt.ylim([-0.005, 0.05])\n",
+ "# plt.xlim([480, 500])"
]
},
{
@@ -1007,21 +1038,21 @@
],
"source": [
"plt.figure(figsize=(8, 6))\n",
- "plt.plot(0,0)\n",
- "plt.plot(0,0)\n",
+ "plt.plot(0, 0)\n",
+ "plt.plot(0, 0)\n",
"\n",
"\n",
"plt.plot(np.diff(new_synced_time_vector4))\n",
- "#plt.plot(np.diff(new_synced_time_vector3))\n",
- "#plt.plot(np.diff(behavior_time_vector))\n",
+ "# plt.plot(np.diff(new_synced_time_vector3))\n",
+ "# plt.plot(np.diff(behavior_time_vector))\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time (s)')\n",
- "plt.title('ViRMEn behavior time iteration duration')\n",
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time (s)\")\n",
+ "plt.title(\"ViRMEn behavior time iteration duration\")\n",
"plt.ylim([-0.05, 0.05])\n",
- "#plt.ylim([-0.01, 0.03])\n",
- "#plt.ylim([-0.005, 0.05])\n",
- "#plt.xlim([305, 320])"
+ "# plt.ylim([-0.01, 0.03])\n",
+ "# plt.ylim([-0.005, 0.05])\n",
+ "# plt.xlim([305, 320])"
]
},
{
@@ -1098,12 +1129,11 @@
"source": [
"difo = np.diff(new_synced_time_vector)\n",
"\n",
- "idx_r = np.where(difo <= 0 )\n",
+ "idx_r = np.where(difo <= 0)\n",
"idx_r = idx_r[0]\n",
"print(idx_r)\n",
- "idx_vs = np.where(vec_shift != 0 )\n",
- "idx_vs = idx_vs[0]\n",
- "\n"
+ "idx_vs = np.where(vec_shift != 0)\n",
+ "idx_vs = idx_vs[0]"
]
},
{
@@ -1152,9 +1182,9 @@
}
],
"source": [
- "plt.plot(np.diff(synced_time_vector[idx_r[0]-5:]))\n",
- "plt.plot(np.diff(behavior_time_vector[idx_r[0]-5:]))\n",
- "plt.ylim(-0.02,0.07)"
+ "plt.plot(np.diff(synced_time_vector[idx_r[0] - 5 :]))\n",
+ "plt.plot(np.diff(behavior_time_vector[idx_r[0] - 5 :]))\n",
+ "plt.ylim(-0.02, 0.07)"
]
},
{
@@ -1173,7 +1203,7 @@
],
"source": [
"it_times = new_synced_time_vector\n",
- "#it_times = synced_time_vector\n",
+ "# it_times = synced_time_vector\n",
"beh_times = behavior_time_vector\n",
"\n",
"print(it_times.shape[0])\n",
@@ -1198,11 +1228,12 @@
],
"source": [
"if it_times.shape[0] >= beh_times.shape[0]:\n",
- " diff_vector = np.diff(it_times[:beh_times.shape[0]] - beh_times)\n",
+ " diff_vector = np.diff(it_times[: beh_times.shape[0]] - beh_times)\n",
"else:\n",
- " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n",
+ " diff_vector = np.diff(it_times - beh_times[: it_times.shape[0]])\n",
"\n",
"import scipy\n",
+ "\n",
"peaks, _ = scipy.signal.find_peaks(diff_vector, height=0.05, distance=20)\n",
"peaks"
]
@@ -1224,13 +1255,13 @@
}
],
"source": [
- "#plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n",
- "plt.plot((new_synced_time_vector - beh_times[:it_times.shape[0]]))\n",
- "#plt.plot((it_times - beh_times))\n",
- "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
+ "# plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n",
+ "plt.plot(new_synced_time_vector - beh_times[: it_times.shape[0]])\n",
+ "# plt.plot((it_times - beh_times))\n",
+ "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
"plt.plot(peaks, diff_vector[peaks], \"x\")\n",
- "plt.ylabel('Time diff in current iteration TTL - Virmen (s)')\n",
- "plt.xlabel('Iteration')\n",
+ "plt.ylabel(\"Time diff in current iteration TTL - Virmen (s)\")\n",
+ "plt.xlabel(\"Iteration\")\n",
"plt.ylim([-0.05, 0.02])\n",
"plt.show()"
]
@@ -1263,11 +1294,11 @@
],
"source": [
"plt.plot(np.diff(new_synced_time_vector))\n",
- "plt.plot(np.diff(beh_times)+0.01)\n",
- "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
- "#plt.plot(peaks, diff_vector[peaks], \"x\")\n",
+ "plt.plot(np.diff(beh_times) + 0.01)\n",
+ "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
+ "# plt.plot(peaks, diff_vector[peaks], \"x\")\n",
"plt.ylim([-0.01, 0.05])\n",
- "plt.xlabel('Iteration')\n"
+ "plt.xlabel(\"Iteration\")"
]
},
{
@@ -1287,8 +1318,10 @@
}
],
"source": [
- "difo = np.pad(np.diff(new_synced_time_vector), (1, 0), 'constant', constant_values=(0,0))\n",
- "difo2 = np.pad(np.diff(beh_times), (1, 0), 'constant', constant_values=(0,0))\n",
+ "difo = np.pad(\n",
+ " np.diff(new_synced_time_vector), (1, 0), \"constant\", constant_values=(0, 0)\n",
+ ")\n",
+ "difo2 = np.pad(np.diff(beh_times), (1, 0), \"constant\", constant_values=(0, 0))\n",
"\n",
"peaks_ttl_pulses, _ = scipy.signal.find_peaks(difo, height=0.02, distance=20)\n",
"peaks_virmen_iter, _ = scipy.signal.find_peaks(difo2, height=0.02, distance=20)\n",
@@ -1355,8 +1388,8 @@
}
],
"source": [
- "print(new_synced_time_vector[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n",
- "print(beh_times[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n"
+ "print(new_synced_time_vector[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])\n",
+ "print(beh_times[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])"
]
},
{
@@ -1375,7 +1408,7 @@
],
"source": [
"print(new_synced_time_vector[-4:])\n",
- "print(beh_times[-4:])\n"
+ "print(beh_times[-4:])"
]
},
{
@@ -1405,10 +1438,9 @@
}
],
"source": [
- "\n",
"plt.plot(np.diff(new_synced_time_vector))\n",
- "#plt.plot(np.diff(synced_time_vector))\n",
- "plt.plot(np.diff(behavior_time_vector)+0.01)\n",
+ "# plt.plot(np.diff(synced_time_vector))\n",
+ "plt.plot(np.diff(behavior_time_vector) + 0.01)\n",
"plt.ylim(0, 0.11)"
]
},
@@ -1459,8 +1491,8 @@
],
"source": [
"trial_plot = 1\n",
- "plt.plot((iteration_dict['iter_times_idx'][trial_plot]))\n",
- "plt.plot((behavior_time[trial_plot].flatten()))\n"
+ "plt.plot(iteration_dict[\"iter_times_idx\"][trial_plot])\n",
+ "plt.plot(behavior_time[trial_plot].flatten())"
]
},
{
@@ -1482,15 +1514,24 @@
"samp_before = 300\n",
"samp_after = 300\n",
"if trial_plot == 0:\n",
- " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot][0]-np.int64(samp_before)\n",
+ " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot][0] - np.int64(\n",
+ " samp_before\n",
+ " )\n",
"else:\n",
- " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot-1][-1]-np.int64(samp_before)\n",
- "first_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][0]+np.int64(samp_after)\n",
- "second_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][1]\n",
+ " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1] - np.int64(\n",
+ " samp_before\n",
+ " )\n",
+ "first_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][0] + np.int64(\n",
+ " samp_after\n",
+ ")\n",
+ "second_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][1]\n",
"\n",
- "samp_diff = iteration_dict['iter_start_idx'][trial_plot][0] - iteration_dict['iter_start_idx'][trial_plot-1][-1]\n",
- "print('last_iter_trial0', last_iter_trial0)\n",
- "print('first_iter_trial1', first_iter_trial1)"
+ "samp_diff = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1]\n",
+ ")\n",
+ "print(\"last_iter_trial0\", last_iter_trial0)\n",
+ "print(\"first_iter_trial1\", first_iter_trial1)"
]
},
{
@@ -1520,8 +1561,8 @@
"outputs": [],
"source": [
"trial_plot2 = 2\n",
- "lo2 = iteration_dict['iter_times_idx'][trial_plot2][2:]\n",
- "lo3 = np.append([0], lo2)\n"
+ "lo2 = iteration_dict[\"iter_times_idx\"][trial_plot2][2:]\n",
+ "lo3 = np.append([0], lo2)"
]
},
{
@@ -1551,11 +1592,13 @@
}
],
"source": [
- "\n",
"shape_beh = behavior_time[trial_plot2].shape[0]\n",
- "shape_iter = iteration_dict['iter_times_idx'][trial_plot2].shape[0]\n",
- "#plt.plot(iteration_dict['iter_times_idx'][trial_plot2][:shape_beh] - behavior_time[trial_plot2].flatten())\n",
- "plt.plot(iteration_dict['iter_times_idx'][trial_plot2] - behavior_time[trial_plot2].flatten()[:shape_iter])\n",
+ "shape_iter = iteration_dict[\"iter_times_idx\"][trial_plot2].shape[0]\n",
+ "# plt.plot(iteration_dict['iter_times_idx'][trial_plot2][:shape_beh] - behavior_time[trial_plot2].flatten())\n",
+ "plt.plot(\n",
+ " iteration_dict[\"iter_times_idx\"][trial_plot2]\n",
+ " - behavior_time[trial_plot2].flatten()[:shape_iter]\n",
+ ")\n",
"plt.ylim(-0.1, 0.11)"
]
},
@@ -1609,7 +1652,7 @@
],
"source": [
"print(lo3[:5])\n",
- "print(iteration_dict['iter_times_idx'][trial_plot2][:5])\n",
+ "print(iteration_dict[\"iter_times_idx\"][trial_plot2][:5])\n",
"print(behavior_time[trial_plot2].flatten()[:5])"
]
},
@@ -1653,55 +1696,64 @@
}
],
"source": [
- "\n",
- "\n",
"start_iter = 1\n",
"end_iter = 1\n",
"\n",
"\n",
- "sample_start =(iteration_dict['iter_start_idx'][trial_plot][start_iter]-np.int64(samp_before))\n",
- "sample_end = (iteration_dict['iter_start_idx'][trial_plot][end_iter]+np.int64(samp_after))\n",
- "\n",
- "iter_sample = digital_array[new_trial_channel,last_iter_trial0:first_iter_trial1]\n",
+ "sample_start = iteration_dict[\"iter_start_idx\"][trial_plot][start_iter] - np.int64(\n",
+ " samp_before\n",
+ ")\n",
+ "sample_end = iteration_dict[\"iter_start_idx\"][trial_plot][end_iter] + np.int64(\n",
+ " samp_after\n",
+ ")\n",
"\n",
- "time_vector = np.linspace(start=0, stop=iter_sample.shape[0]-1,num=iter_sample.shape[0])\n",
- "time_vector = (time_vector*1000/nidq_sampling_rate)\n",
- "time_vector -= ((iter_sample.shape[0]-samp_after)*1000)/nidq_sampling_rate\n",
+ "iter_sample = digital_array[new_trial_channel, last_iter_trial0:first_iter_trial1]\n",
"\n",
- "time_last_trial = time_vector[0] +(samp_before*1000)/nidq_sampling_rate\n",
+ "time_vector = np.linspace(\n",
+ " start=0, stop=iter_sample.shape[0] - 1, num=iter_sample.shape[0]\n",
+ ")\n",
+ "time_vector = time_vector * 1000 / nidq_sampling_rate\n",
+ "time_vector -= ((iter_sample.shape[0] - samp_after) * 1000) / nidq_sampling_rate\n",
"\n",
+ "time_last_trial = time_vector[0] + (samp_before * 1000) / nidq_sampling_rate\n",
"\n",
"\n",
"print(iter_sample.shape[0])\n",
- "samples_after_second_pulse = (second_iter_trial1-(first_iter_trial1-samp_after))\n",
- "print('samples_after_second_pulse', samples_after_second_pulse)\n",
- "print(iter_sample.shape[0]-samp_after)\n",
+ "samples_after_second_pulse = second_iter_trial1 - (first_iter_trial1 - samp_after)\n",
+ "print(\"samples_after_second_pulse\", samples_after_second_pulse)\n",
+ "print(iter_sample.shape[0] - samp_after)\n",
"\n",
- "idx_time_zero = np.where((time_vector >= 0))\n",
+ "idx_time_zero = np.where(time_vector >= 0)\n",
"idx_time_zero = idx_time_zero[0]\n",
"idx_time_zero = idx_time_zero[0]\n",
"\n",
"\n",
- "print('new_iteration_channel', new_iteration_channel)\n",
- "print(digital_array[new_iteration_channel,:].shape)\n",
+ "print(\"new_iteration_channel\", new_iteration_channel)\n",
+ "print(digital_array[new_iteration_channel, :].shape)\n",
"\n",
- "plt.plot(time_vector,iter_sample)\n",
- "plt.plot(time_vector,digital_array[new_iteration_channel,last_iter_trial0:first_iter_trial1]+0.02)\n",
- "plt.plot(0,1,\"x\")\n",
- "if (idx_time_zero+samples_after_second_pulse) < time_vector.shape[0]:\n",
- " plt.plot(time_vector[idx_time_zero+samples_after_second_pulse],1,\"x\")\n",
- "plt.plot(time_last_trial,1,\"x\")\n",
+ "plt.plot(time_vector, iter_sample)\n",
+ "plt.plot(\n",
+ " time_vector,\n",
+ " digital_array[new_iteration_channel, last_iter_trial0:first_iter_trial1] + 0.02,\n",
+ ")\n",
+ "plt.plot(0, 1, \"x\")\n",
+ "if (idx_time_zero + samples_after_second_pulse) < time_vector.shape[0]:\n",
+ " plt.plot(time_vector[idx_time_zero + samples_after_second_pulse], 1, \"x\")\n",
+ "plt.plot(time_last_trial, 1, \"x\")\n",
"\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
"\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
"\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -1721,7 +1773,7 @@
}
],
"source": [
- "iteration_dict['iter_start_idx'][trial_plot].shape"
+ "iteration_dict[\"iter_start_idx\"][trial_plot].shape"
]
},
{
@@ -1757,20 +1809,37 @@
"start_iter = 790\n",
"samp_after = 600\n",
"\n",
- "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n",
- "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 2,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n",
+ " + np.int64(samp_after),\n",
+ " ]\n",
+ ")\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 1,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n",
+ " + np.int64(samp_after),\n",
+ " ]\n",
+ ")\n",
"\n",
"\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
"\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
"\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -1789,8 +1858,8 @@
],
"source": [
"trial_plot = 280\n",
- "print(iteration_dict['iter_start_idx'][trial_plot][2704])\n",
- "print(iteration_dict['iter_start_idx'][trial_plot+1][0])"
+ "print(iteration_dict[\"iter_start_idx\"][trial_plot][2704])\n",
+ "print(iteration_dict[\"iter_start_idx\"][trial_plot + 1][0])"
]
},
{
@@ -1822,7 +1891,7 @@
}
],
"source": [
- "plt.plot(np.diff(iteration_dict['iter_start_idx'][trial_plot]/nidq_sampling_rate))"
+ "plt.plot(np.diff(iteration_dict[\"iter_start_idx\"][trial_plot] / nidq_sampling_rate))"
]
},
{
@@ -1902,14 +1971,14 @@
"source": [
"x = np.array([])\n",
"mean_x = np.array([])\n",
- "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n",
- " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n",
+ "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n",
+ " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n",
" mean_time_trial = np.mean(s)\n",
- " x = np.append(x,s, axis=0)\n",
+ " x = np.append(x, s, axis=0)\n",
" mean_x = np.append(mean_x, mean_time_trial)\n",
"\n",
- "#plt.plot(x)\n",
- "plt.plot(mean_x)\n"
+ "# plt.plot(x)\n",
+ "plt.plot(mean_x)"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb
index f560c837..f7e67ad1 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -39,25 +40,17 @@
}
],
"source": [
- "import datetime\n",
"import pathlib\n",
- "import numpy as np\n",
- "import pylab as pl\n",
- "import matplotlib\n",
- "import matplotlib.pyplot as plt\n",
- "from scipy.signal import find_peaks\n",
- "from scipy import signal as sp\n",
"\n",
- "\n",
- "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory, get_ephys_root_data_dir\n",
- "import u19_pipeline.utils.ephys_fix_sync_code as ephys_fix_sync_code\n",
- "import u19_pipeline.acquisition as acquisition\n",
"import datajoint as dj\n",
- "\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
"from element_interface.utils import find_full_path\n",
"\n",
"import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n",
- "import u19_pipeline.utils.ephys_utils as ephys_utils\n"
+ "import u19_pipeline.utils.ephys_fix_sync_code as ephys_fix_sync_code\n",
+ "import u19_pipeline.utils.ephys_utils as ephys_utils\n",
+ "from u19_pipeline.ephys_pipeline import get_ephys_root_data_dir, get_session_directory"
]
},
{
@@ -80,12 +73,14 @@
],
"source": [
"recording_query = \"recording_id = 502\"\n",
- "recording = dj.create_virtual_module('recording', 'u19_recording')\n",
- "recording_key = (recording.Recording & recording_query).fetch('recording_id', as_dict=True, order_by='recording_id')\n",
+ "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n",
+ "recording_key = (recording.Recording & recording_query).fetch(\n",
+ " \"recording_id\", as_dict=True, order_by=\"recording_id\"\n",
+ ")\n",
"\n",
"\n",
"key = (recording.Recording.BehaviorSession & recording_query).fetch1()\n",
- "del key['recording_id']\n",
+ "del key[\"recording_id\"]\n",
"\n",
"key"
]
@@ -108,7 +103,7 @@
],
"source": [
"session_dir = pathlib.Path(get_session_directory(recording_key[0]))\n",
- "session_dir\n"
+ "session_dir"
]
},
{
@@ -126,38 +121,43 @@
}
],
"source": [
- "session_dir = find_full_path(get_ephys_root_data_dir(),\n",
- " get_session_directory(recording_key))\n",
+ "session_dir = find_full_path(\n",
+ " get_ephys_root_data_dir(), get_session_directory(recording_key)\n",
+ ")\n",
"print(session_dir)\n",
- "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
- "#Check if session is Nidq or OneBox\n",
- "nidq_session = list(session_dir.glob('*nidq.bin*'))\n",
- "obx_session = list(session_dir.glob('*obx.bin*'))\n",
+ "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
+ "# Check if session is Nidq or OneBox\n",
+ "nidq_session = list(session_dir.glob(\"*nidq.bin*\"))\n",
+ "obx_session = list(session_dir.glob(\"*obx.bin*\"))\n",
"\n",
"if len(nidq_session) == 0 and len(obx_session) == 0:\n",
- " print('No session found')\n",
+ " print(\"No session found\")\n",
"elif len(nidq_session) > 0:\n",
" ephys_session_fullpath = nidq_session[0]\n",
"else:\n",
" ephys_session_fullpath = obx_session[0]\n",
"\n",
- "#Nidaq file\n",
- "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n",
+ "# Nidaq file\n",
+ "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n",
"nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n",
"\n",
"\n",
- "# 1: load meta data, and the content of the NIDAQ file. Its content is digital. \n",
+ "# 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n",
"new_trial_channel = 1\n",
"new_iteration_channel = 2\n",
"# If PXIe card (nidq) card use for recording deduce digital channels\n",
- "if nidq_meta['typeThis'] == 'nidq':\n",
- " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)\n",
+ "if nidq_meta[\"typeThis\"] == \"nidq\":\n",
+ " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " ephys_session_fullpath, nidq_meta\n",
+ " )\n",
"# If onebox card (obx) card use for recording digital channels are 0-2\n",
"else:\n",
- " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])\n",
+ " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]\n",
+ " )\n",
" # If no sync pulse found trial and iteration signals are 0 & 1 respectively\n",
- " channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]\n",
- " channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]\n",
+ " channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]\n",
+ " channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]\n",
"\n",
" if channel0_pulses > channel1_pulses:\n",
" new_trial_channel = 1\n",
@@ -194,18 +194,20 @@
"metadata": {},
"outputs": [],
"source": [
- "#Behavior data\n",
+ "# Behavior data\n",
"behavior_key = key.copy()\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
"\n",
"block = 0\n",
"# If a specific block is requested, add that to our behavior_key. It should be an int referring to virmen block number.\n",
"# This is useful for sessions in which the nidaq stream was interrupted due to restarting virmen\n",
"if block > 0:\n",
- " behavior_key['block'] = block\n",
+ " behavior_key[\"block\"] = block\n",
"\n",
"thissession = behavior.TowersBlock().Trial() & behavior_key\n",
- "behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n"
+ "behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n",
+ " \"trial_time\", \"vi_start\", \"iterations\"\n",
+ ")"
]
},
{
@@ -214,8 +216,8 @@
"metadata": {},
"outputs": [],
"source": [
- "#plt.plot(np.diff(digital_array[new_trial_channel,:5000000]))\n",
- "#plt.plot(digital_array[new_trial_channel,:5000000])\n"
+ "# plt.plot(np.diff(digital_array[new_trial_channel,:5000000]))\n",
+ "# plt.plot(digital_array[new_trial_channel,:5000000])\n"
]
},
{
@@ -308,21 +310,37 @@
}
],
"source": [
- "mode = None #Default for sessions before 12/01/2021\n",
- "#mode = 'pulses' #Default for sessions after 12/01/2021\n",
- "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[new_trial_channel,:], digital_array[new_iteration_channel,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n",
+ "mode = None # Default for sessions before 12/01/2021\n",
+ "# mode = 'pulses' #Default for sessions after 12/01/2021\n",
+ "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n",
+ " digital_array[new_trial_channel, :],\n",
+ " digital_array[new_iteration_channel, :],\n",
+ " nidq_sampling_rate,\n",
+ " behavior_time.shape[0],\n",
+ " behavior_time,\n",
+ " mode=mode,\n",
+ ")\n",
"# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n",
"\n",
"\n",
"# Check # of trials and iterations match\n",
- "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n",
+ "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n",
+ " ephys_utils.assert_iteration_samples_count(\n",
+ " iteration_dict[\"iter_start_idx\"], behavior_time\n",
+ " )\n",
+ ")\n",
"\n",
"print(trial_count_diff)\n",
"print(trials_diff_iteration_big)\n",
"print(trials_diff_iteration_small)\n",
"\n",
"\n",
- "status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n"
+ "status = ephys_utils.evaluate_sync_process(\n",
+ " trial_count_diff,\n",
+ " trials_diff_iteration_big,\n",
+ " trials_diff_iteration_small,\n",
+ " behavior_time.shape[0],\n",
+ ")"
]
},
{
@@ -345,8 +363,8 @@
],
"source": [
"trial_fix = 3\n",
- "synced_iteration_vector = iteration_dict['iter_start_idx'][trial_fix]\n",
- "synced_time_vector = (iteration_dict['iter_times_idx'][trial_fix]).copy()\n",
+ "synced_iteration_vector = iteration_dict[\"iter_start_idx\"][trial_fix]\n",
+ "synced_time_vector = (iteration_dict[\"iter_times_idx\"][trial_fix]).copy()\n",
"\n",
"behavior_time_vector = behavior_time[trial_fix].flatten()\n",
"\n",
@@ -355,10 +373,10 @@
"\n",
"print(old_synced_time_vector[0:5])\n",
"print(behavior_time_vector[0:5])\n",
- "print('--------------------')\n",
+ "print(\"--------------------\")\n",
"print(np.diff(old_synced_time_vector[0:6]))\n",
"print(np.diff(behavior_time_vector[0:6]))\n",
- "print(iteration_dict['iter_times_idx'][trial_fix][0:5])"
+ "print(iteration_dict[\"iter_times_idx\"][trial_fix][0:5])"
]
},
{
@@ -398,8 +416,7 @@
}
],
"source": [
- "\n",
- "'''\n",
+ "\"\"\"\n",
"#synced_time_vector = synced_time_vector+behavior_time_vector[1]-(behavior_time_vector[2]-synced_time_vector[2])\n",
"synced_time_vector = synced_time_vector+behavior_time_vector[1]\n",
"synced_time_vector = np.insert(synced_time_vector, 1, behavior_time_vector[1])\n",
@@ -412,7 +429,7 @@
"\n",
"print(synced_time_vector[0:5])\n",
"print(behavior_time_vector[0:5])\n",
- "'''"
+ "\"\"\""
]
},
{
@@ -449,29 +466,28 @@
}
],
"source": [
- "\n",
- "synced_time_vector2, shift_vec, median_vec = ephys_fix_sync_code.get_shift_vector(synced_time_vector,behavior_time_vector)\n",
+ "synced_time_vector2, shift_vec, median_vec = ephys_fix_sync_code.get_shift_vector(\n",
+ " synced_time_vector, behavior_time_vector\n",
+ ")\n",
"\n",
"print(median_vec)\n",
"\n",
"\n",
+ "# median_diff = np.median(baseline_diff)\n",
+ "# max_diff = np.median(baseline_diff)+0.007\n",
+ "# min_diff = np.median(baseline_diff)-0.007\n",
+ "# time_vec = np.arange(base_size-initial_sample)\n",
+ "plt.plot([0, synced_time_vector.shape[0]], [median_vec[1], median_vec[1]], \"k\")\n",
+ "plt.plot([0, synced_time_vector.shape[0]], [median_vec[0], median_vec[0]], \"r\")\n",
+ "plt.plot([0, synced_time_vector.shape[0]], [median_vec[2], median_vec[2]], \"g\")\n",
"\n",
- "#median_diff = np.median(baseline_diff)\n",
- "#max_diff = np.median(baseline_diff)+0.007\n",
- "#min_diff = np.median(baseline_diff)-0.007\n",
- "#time_vec = np.arange(base_size-initial_sample)\n",
- "plt.plot([0, synced_time_vector.shape[0]], [median_vec[1], median_vec[1]],'k')\n",
- "plt.plot([0, synced_time_vector.shape[0]], [median_vec[0], median_vec[0]],'r')\n",
- "plt.plot([0, synced_time_vector.shape[0]], [median_vec[2], median_vec[2]],'g')\n",
- "\n",
- "plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n",
- "plt.plot(synced_time_vector2 - behavior_time_vector[:synced_time_vector2.shape[0]])\n",
+ "plt.plot(synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]])\n",
+ "plt.plot(synced_time_vector2 - behavior_time_vector[: synced_time_vector2.shape[0]])\n",
"\n",
"\n",
- "\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time difference (s)')\n",
- "plt.title('Before & After applying (get_shift_vector))')\n",
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time difference (s)\")\n",
+ "plt.title(\"Before & After applying (get_shift_vector))\")\n",
"\n",
"plt.ylim([-0.05, 0.15])"
]
@@ -503,13 +519,13 @@
}
],
"source": [
- "plt.plot([0, 40], [median_vec[1], median_vec[1]],'k')\n",
- "plt.plot([0, 40], [median_vec[0], median_vec[0]],'r')\n",
- "plt.plot([0, 40], [median_vec[2], median_vec[2]],'g')\n",
+ "plt.plot([0, 40], [median_vec[1], median_vec[1]], \"k\")\n",
+ "plt.plot([0, 40], [median_vec[0], median_vec[0]], \"r\")\n",
+ "plt.plot([0, 40], [median_vec[2], median_vec[2]], \"g\")\n",
"\n",
- "plt.plot([0, 40], [0, 0],'y')\n",
+ "plt.plot([0, 40], [0, 0], \"y\")\n",
"\n",
- "plt.plot(synced_time_vector[0:40] - behavior_time_vector[0:40])\n"
+ "plt.plot(synced_time_vector[0:40] - behavior_time_vector[0:40])"
]
},
{
@@ -531,15 +547,24 @@
"samp_before = 1000\n",
"samp_after = 1600\n",
"if trial_plot == 0:\n",
- " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot][0]-np.int64(samp_before)\n",
+ " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot][0] - np.int64(\n",
+ " samp_before\n",
+ " )\n",
"else:\n",
- " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot-1][-1]-np.int64(samp_before)\n",
- "first_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][0]+np.int64(samp_after)\n",
- "second_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][1]\n",
- "\n",
- "samp_diff = iteration_dict['iter_start_idx'][trial_plot][0] - iteration_dict['iter_start_idx'][trial_plot-1][-1]\n",
- "print('last_iter_trial0', last_iter_trial0)\n",
- "print('first_iter_trial1', first_iter_trial1)"
+ " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1] - np.int64(\n",
+ " samp_before\n",
+ " )\n",
+ "first_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][0] + np.int64(\n",
+ " samp_after\n",
+ ")\n",
+ "second_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][1]\n",
+ "\n",
+ "samp_diff = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1]\n",
+ ")\n",
+ "print(\"last_iter_trial0\", last_iter_trial0)\n",
+ "print(\"first_iter_trial1\", first_iter_trial1)"
]
},
{
@@ -575,57 +600,70 @@
}
],
"source": [
- "\n",
- "\n",
"start_iter = 1\n",
"end_iter = 1\n",
"\n",
"\n",
- "sample_start =(iteration_dict['iter_start_idx'][trial_plot][start_iter]-np.int64(samp_before))\n",
- "sample_end = (iteration_dict['iter_start_idx'][trial_plot][end_iter]+np.int64(samp_after))\n",
- "\n",
- "iter_sample = digital_array[new_trial_channel,last_iter_trial0:first_iter_trial1]\n",
+ "sample_start = iteration_dict[\"iter_start_idx\"][trial_plot][start_iter] - np.int64(\n",
+ " samp_before\n",
+ ")\n",
+ "sample_end = iteration_dict[\"iter_start_idx\"][trial_plot][end_iter] + np.int64(\n",
+ " samp_after\n",
+ ")\n",
"\n",
- "time_vector = np.linspace(start=0, stop=iter_sample.shape[0]-1,num=iter_sample.shape[0])\n",
- "time_vector = (time_vector*1000/nidq_sampling_rate)\n",
- "time_vector -= ((iter_sample.shape[0]-samp_after)*1000)/nidq_sampling_rate\n",
+ "iter_sample = digital_array[new_trial_channel, last_iter_trial0:first_iter_trial1]\n",
"\n",
- "time_last_trial = time_vector[0] +(samp_before*1000)/nidq_sampling_rate\n",
+ "time_vector = np.linspace(\n",
+ " start=0, stop=iter_sample.shape[0] - 1, num=iter_sample.shape[0]\n",
+ ")\n",
+ "time_vector = time_vector * 1000 / nidq_sampling_rate\n",
+ "time_vector -= ((iter_sample.shape[0] - samp_after) * 1000) / nidq_sampling_rate\n",
"\n",
+ "time_last_trial = time_vector[0] + (samp_before * 1000) / nidq_sampling_rate\n",
"\n",
"\n",
"print(iter_sample.shape[0])\n",
- "samples_after_second_pulse = (second_iter_trial1-(first_iter_trial1-samp_after))\n",
- "print('samples_after_second_pulse', samples_after_second_pulse)\n",
- "print(iter_sample.shape[0]-samp_after)\n",
+ "samples_after_second_pulse = second_iter_trial1 - (first_iter_trial1 - samp_after)\n",
+ "print(\"samples_after_second_pulse\", samples_after_second_pulse)\n",
+ "print(iter_sample.shape[0] - samp_after)\n",
"\n",
- "idx_time_zero = np.where((time_vector >= 0))\n",
+ "idx_time_zero = np.where(time_vector >= 0)\n",
"idx_time_zero = idx_time_zero[0]\n",
"idx_time_zero = idx_time_zero[0]\n",
"\n",
"\n",
- "print('new_iteration_channel', new_iteration_channel)\n",
- "print(digital_array[new_iteration_channel,:].shape)\n",
- "\n",
- "plt.plot(time_vector,iter_sample)\n",
- "plt.plot(time_vector,digital_array[new_iteration_channel,last_iter_trial0:first_iter_trial1]+0.02)\n",
- "plt.plot(0,1,\"rx\",'')\n",
- "if (idx_time_zero+samples_after_second_pulse) < time_vector.shape[0]:\n",
- " plt.plot(time_vector[idx_time_zero+samples_after_second_pulse],1,\"gx\")\n",
- "plt.plot(time_last_trial,1,\"bx\",)\n",
- "\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
- "\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
- "\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)\n",
- "\n",
- "#plt.xlim([-20, 20])"
+ "print(\"new_iteration_channel\", new_iteration_channel)\n",
+ "print(digital_array[new_iteration_channel, :].shape)\n",
+ "\n",
+ "plt.plot(time_vector, iter_sample)\n",
+ "plt.plot(\n",
+ " time_vector,\n",
+ " digital_array[new_iteration_channel, last_iter_trial0:first_iter_trial1] + 0.02,\n",
+ ")\n",
+ "plt.plot(0, 1, \"rx\", \"\")\n",
+ "if (idx_time_zero + samples_after_second_pulse) < time_vector.shape[0]:\n",
+ " plt.plot(time_vector[idx_time_zero + samples_after_second_pulse], 1, \"gx\")\n",
+ "plt.plot(\n",
+ " time_last_trial,\n",
+ " 1,\n",
+ " \"bx\",\n",
+ ")\n",
+ "\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
+ "\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
+ "\n",
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)\n",
+ "\n",
+ "# plt.xlim([-20, 20])"
]
},
{
@@ -655,18 +693,19 @@
}
],
"source": [
- "synced_time_vector3,_ = ephys_fix_sync_code.fix_shifted_sync_vector(synced_time_vector2, behavior_time_vector, shift_vec)\n",
+ "synced_time_vector3, _ = ephys_fix_sync_code.fix_shifted_sync_vector(\n",
+ " synced_time_vector2, behavior_time_vector, shift_vec\n",
+ ")\n",
"\n",
- "plt.plot(synced_time_vector2 - behavior_time_vector[:synced_time_vector2.shape[0]])\n",
- "plt.plot(synced_time_vector3 - behavior_time_vector[:synced_time_vector3.shape[0]])\n",
+ "plt.plot(synced_time_vector2 - behavior_time_vector[: synced_time_vector2.shape[0]])\n",
+ "plt.plot(synced_time_vector3 - behavior_time_vector[: synced_time_vector3.shape[0]])\n",
"\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time difference (s)')\n",
- "plt.title('Before & After applying (fix_shifted_sync_vector)')\n",
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time difference (s)\")\n",
+ "plt.title(\"Before & After applying (fix_shifted_sync_vector)\")\n",
"\n",
- "plt.ylim([-0.4,0.4])\n",
- "\n"
+ "plt.ylim([-0.4, 0.4])"
]
},
{
@@ -696,16 +735,17 @@
}
],
"source": [
- "synced_time_vector4,_ =\\\n",
- " ephys_fix_sync_code.complete_last_part_sync_vec(synced_time_vector3, behavior_time_vector)\n",
+ "synced_time_vector4, _ = ephys_fix_sync_code.complete_last_part_sync_vec(\n",
+ " synced_time_vector3, behavior_time_vector\n",
+ ")\n",
"\n",
- "plt.plot(synced_time_vector3 - behavior_time_vector[:synced_time_vector3.shape[0]])\n",
- "plt.plot(synced_time_vector4 - behavior_time_vector[:synced_time_vector4.shape[0]])\n",
+ "plt.plot(synced_time_vector3 - behavior_time_vector[: synced_time_vector3.shape[0]])\n",
+ "plt.plot(synced_time_vector4 - behavior_time_vector[: synced_time_vector4.shape[0]])\n",
"\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time difference (s)')\n",
- "plt.title('Before & After applying (complete_last_part_sync_vec)')\n",
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time difference (s)\")\n",
+ "plt.title(\"Before & After applying (complete_last_part_sync_vec)\")\n",
"\n",
"plt.ylim([-0.4, 0.4])"
]
@@ -737,16 +777,17 @@
}
],
"source": [
- "synced_time_vector4,_ =\\\n",
- " ephys_fix_sync_code.complete_last_part_sync_vec(synced_time_vector3, behavior_time_vector)\n",
+ "synced_time_vector4, _ = ephys_fix_sync_code.complete_last_part_sync_vec(\n",
+ " synced_time_vector3, behavior_time_vector\n",
+ ")\n",
"\n",
- "plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n",
- "plt.plot(synced_time_vector4 - behavior_time_vector[:synced_time_vector4.shape[0]])\n",
+ "plt.plot(synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]])\n",
+ "plt.plot(synced_time_vector4 - behavior_time_vector[: synced_time_vector4.shape[0]])\n",
"\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time difference (s)')\n",
- "plt.title('Before & After applying (complete_last_part_sync_vec)')\n",
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time difference (s)\")\n",
+ "plt.title(\"Before & After applying (complete_last_part_sync_vec)\")\n",
"\n",
"plt.ylim([-0.4, 2.2])"
]
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb
index 35443af1..5b103705 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb
@@ -33,29 +33,24 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"\n",
"import datetime\n",
"import pathlib\n",
- "import matplotlib\n",
+ "\n",
+ "import datajoint as dj\n",
"import matplotlib.pyplot as plt\n",
- "from scipy.signal import find_peaks\n",
- "from scipy import signal as sp\n",
- "import pathlib\n",
+ "import numpy as np\n",
"import pynapple as nap\n",
"import tqdm\n",
- "\n",
- "import datajoint as dj\n",
"from element_interface.utils import find_full_path\n",
- "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n",
- "import u19_pipeline.utils.ephys_utils as ephys_utils\n",
"\n",
- "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory, get_ephys_root_data_dir\n",
- "\n",
- "import numpy as np\n",
- "import u19_pipeline.ephys_pipeline as ep\n",
"import u19_pipeline.acquisition as acquisition\n",
- "from u19_pipeline import recording"
+ "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n",
+ "import u19_pipeline.utils.ephys_utils as ephys_utils\n",
+ "from u19_pipeline import recording\n",
+ "from u19_pipeline.ephys_pipeline import get_ephys_root_data_dir, get_session_directory"
]
},
{
@@ -65,10 +60,12 @@
"metadata": {},
"outputs": [],
"source": [
- "key = {'subject_fullname': 'jyanar_ya054', 'session_date': datetime.date(2025, 10, 14)}\n",
- "key = (acquisition.Session & key).fetch1('KEY')\n",
- "recording = dj.create_virtual_module('recording', 'u19_recording')\n",
- "recording_key = ((acquisition.Session * recording.Recording.BehaviorSession) & key).fetch('recording_id', as_dict=True)\n",
+ "key = {\"subject_fullname\": \"jyanar_ya054\", \"session_date\": datetime.date(2025, 10, 14)}\n",
+ "key = (acquisition.Session & key).fetch1(\"KEY\")\n",
+ "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n",
+ "recording_key = (\n",
+ " (acquisition.Session * recording.Recording.BehaviorSession) & key\n",
+ ").fetch(\"recording_id\", as_dict=True)\n",
"session_dir = pathlib.Path(get_session_directory(recording_key[0]))"
]
},
@@ -96,37 +93,42 @@
}
],
"source": [
- "session_dir = find_full_path(get_ephys_root_data_dir(),\n",
- " get_session_directory(recording_key))\n",
+ "session_dir = find_full_path(\n",
+ " get_ephys_root_data_dir(), get_session_directory(recording_key)\n",
+ ")\n",
"print(session_dir)\n",
- "#Check if session is Nidq or OneBox\n",
- "nidq_session = list(session_dir.glob('*nidq.bin*'))\n",
- "obx_session = list(session_dir.glob('*obx.bin*'))\n",
+ "# Check if session is Nidq or OneBox\n",
+ "nidq_session = list(session_dir.glob(\"*nidq.bin*\"))\n",
+ "obx_session = list(session_dir.glob(\"*obx.bin*\"))\n",
"\n",
"if len(nidq_session) == 0 and len(obx_session) == 0:\n",
- " print('No session found')\n",
+ " print(\"No session found\")\n",
"elif len(nidq_session) > 0:\n",
" ephys_session_fullpath = nidq_session[0]\n",
"else:\n",
" ephys_session_fullpath = obx_session[0]\n",
"\n",
- "#Nidaq file\n",
- "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n",
+ "# Nidaq file\n",
+ "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n",
"nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n",
"\n",
"\n",
- "# 1: load meta data, and the content of the NIDAQ file. Its content is digital. \n",
+ "# 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n",
"new_trial_channel = 1\n",
"new_iteration_channel = 2\n",
"# If PXIe card (nidq) card use for recording deduce digital channels\n",
- "if nidq_meta['typeThis'] == 'nidq':\n",
- " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)\n",
+ "if nidq_meta[\"typeThis\"] == \"nidq\":\n",
+ " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " ephys_session_fullpath, nidq_meta\n",
+ " )\n",
"# If onebox card (obx) card use for recording digital channels are 0-2\n",
"else:\n",
- " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])\n",
+ " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]\n",
+ " )\n",
" # If no sync pulse found trial and iteration signals are 0 & 1 respectively\n",
- " channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]\n",
- " channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]\n",
+ " channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]\n",
+ " channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]\n",
"\n",
" if channel0_pulses > channel1_pulses:\n",
" new_trial_channel = 1\n",
@@ -135,10 +137,12 @@
" new_trial_channel = 0\n",
" new_iteration_channel = 1\n",
"\n",
- "obx_iters_ttl = digital_array[new_iteration_channel,:]\n",
- "obx_trials_ttl = digital_array[new_trial_channel,:]\n",
+ "obx_iters_ttl = digital_array[new_iteration_channel, :]\n",
+ "obx_trials_ttl = digital_array[new_trial_channel, :]\n",
"obx_trial_start_idx = ephys_utils.get_idx_trial_start(obx_trials_ttl)\n",
- "clock_time_s = np.arange(0, len(obx_iters_ttl)/nidq_sampling_rate, step=1/nidq_sampling_rate)\n",
+ "clock_time_s = np.arange(\n",
+ " 0, len(obx_iters_ttl) / nidq_sampling_rate, step=1 / nidq_sampling_rate\n",
+ ")\n",
"obx_trial_start_s = clock_time_s[obx_trial_start_idx]"
]
},
@@ -157,9 +161,11 @@
"metadata": {},
"outputs": [],
"source": [
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n",
- "session = behavior.TowersBlock().Trial & key\n",
- "virmen_time, virmen_iterstart, virmen_num_iterations = session.fetch('trial_time', 'vi_start', 'iterations')"
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
+ "session = behavior.TowersBlock().Trial & key\n",
+ "virmen_time, virmen_iterstart, virmen_num_iterations = session.fetch(\n",
+ " \"trial_time\", \"vi_start\", \"iterations\"\n",
+ ")"
]
},
{
@@ -186,8 +192,8 @@
],
"source": [
"iter_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n",
- " digital_array[new_trial_channel,:],\n",
- " digital_array[new_iteration_channel,:],\n",
+ " digital_array[new_trial_channel, :],\n",
+ " digital_array[new_iteration_channel, :],\n",
" nidq_sampling_rate,\n",
" virmen_time.shape[0],\n",
" virmen_time,\n",
@@ -210,9 +216,11 @@
"metadata": {},
"outputs": [],
"source": [
- "#spikes = nap.load_file('Z:/20251014_g0/ya054_20251014_g0/ya054_20251014_g0_imec0/job_id_882/catgt_output/kilosort4/ya054_20251014.nwb')['units']\n",
+ "# spikes = nap.load_file('Z:/20251014_g0/ya054_20251014_g0/ya054_20251014_g0_imec0/job_id_882/catgt_output/kilosort4/ya054_20251014.nwb')['units']\n",
"\n",
- "spikes = nap.load_file('Z:/Data/Processed/electrophysiology/jyanar/jyanar_ya054/20251014_g0/ya054_20251014_g0/ya054_20251014_g0_imec0/job_id_882/kilosort4_output/ya054_20251014.nwb')['units']\n",
+ "spikes = nap.load_file(\n",
+ " \"Z:/Data/Processed/electrophysiology/jyanar/jyanar_ya054/20251014_g0/ya054_20251014_g0/ya054_20251014_g0_imec0/job_id_882/kilosort4_output/ya054_20251014.nwb\"\n",
+ ")[\"units\"]\n",
"spks_binned = spikes.count(bin_size=0.05)"
]
},
@@ -279,11 +287,11 @@
"obx_trial_stop_idx = []\n",
"for itrial in range(NTRIALS):\n",
" trl_start_idx = obx_trial_start_idx[itrial]\n",
- " trl_start_s = obx_trial_start_s[itrial]\n",
+ " trl_start_s = obx_trial_start_s[itrial]\n",
" virmen_s = virmen_time[itrial]\n",
" trl_stop_idx = np.where(virmen_s[-1] + trl_start_s <= clock_time_s)[0][0] - 1\n",
" obx_trial_stop_idx.append(trl_stop_idx)\n",
- " trial_index[trl_start_idx : trl_stop_idx] = itrial + 1"
+ " trial_index[trl_start_idx:trl_stop_idx] = itrial + 1"
]
},
{
@@ -294,7 +302,7 @@
"outputs": [],
"source": [
"obx_trial_stop_idx = np.asarray(obx_trial_stop_idx)\n",
- "obx_trial_stop_s = clock_time_s[obx_trial_stop_idx]"
+ "obx_trial_stop_s = clock_time_s[obx_trial_stop_idx]"
]
},
{
@@ -323,28 +331,28 @@
"for itrial in tqdm.tqdm(range(NTRIALS)):\n",
" # Compute the trial start and stop indices\n",
" trl_start_idx = obx_trial_start_idx[itrial]\n",
- " trl_stop_idx = obx_trial_stop_idx[itrial]\n",
- " trl_start_s = obx_trial_start_s[itrial]\n",
- " trl_stop_s = clock_time_s[trl_stop_idx]\n",
+ " trl_stop_idx = obx_trial_stop_idx[itrial]\n",
+ " trl_start_s = obx_trial_start_s[itrial]\n",
+ " trl_stop_s = clock_time_s[trl_stop_idx]\n",
" # Now iterate over virmen timestamps, filling in each frame\n",
" NFRAMES = virmen_time[itrial].shape[0]\n",
" iter_start_idx_thistrial = []\n",
" for iframe in range(1, NFRAMES):\n",
- " itr_start_s = virmen_time[itrial][iframe - 1] # Relative time to trial start\n",
- " itr_stop_s = virmen_time[itrial][iframe] # Relative time to trial start\n",
+ " itr_start_s = virmen_time[itrial][iframe - 1] # Relative time to trial start\n",
+ " itr_stop_s = virmen_time[itrial][iframe] # Relative time to trial start\n",
" # Compute the number of frames to get there\n",
" itr_start_idx = int(np.floor(itr_start_s * nidq_sampling_rate))\n",
- " itr_stop_idx = int(np.floor(itr_stop_s * nidq_sampling_rate))\n",
+ " itr_stop_idx = int(np.floor(itr_stop_s * nidq_sampling_rate))\n",
" # Express in absolute time\n",
" itr_start_idx += trl_start_idx\n",
" itr_stop_idx += trl_start_idx\n",
" iter_start_idx_thistrial.append(itr_start_idx)\n",
" # SLOW\n",
- " #itr_start_s = trl_start_s + virmen_time[itrial][iframe - 1] # Express start and stop in absolute time\n",
- " #itr_stop_s = trl_start_s + virmen_time[itrial][iframe]\n",
- " #itr_start_idx = np.where(itr_start_s <= clock_time_s)[0][0] - 1\n",
- " #itr_stop_idx = np.where(itr_stop_s <= clock_time_s)[0][0] - 1\n",
- " frame_index[itr_start_idx : itr_stop_idx] = iframe\n",
+ " # itr_start_s = trl_start_s + virmen_time[itrial][iframe - 1] # Express start and stop in absolute time\n",
+ " # itr_stop_s = trl_start_s + virmen_time[itrial][iframe]\n",
+ " # itr_start_idx = np.where(itr_start_s <= clock_time_s)[0][0] - 1\n",
+ " # itr_stop_idx = np.where(itr_stop_s <= clock_time_s)[0][0] - 1\n",
+ " frame_index[itr_start_idx:itr_stop_idx] = iframe\n",
"\n",
" last_start_rel = virmen_time[itrial][-1]\n",
" last_start_idx = trl_start_idx + int(np.floor(last_start_rel * nidq_sampling_rate))\n",
@@ -383,7 +391,7 @@
],
"source": [
"plt.plot(clock_time_s, frame_index)\n",
- "plt.plot(clock_time_s, iter_dict['framenumber_vector_samples'])"
+ "plt.plot(clock_time_s, iter_dict[\"framenumber_vector_samples\"])"
]
},
{
@@ -415,7 +423,7 @@
],
"source": [
"plt.plot(clock_time_s, trial_index)\n",
- "plt.plot(clock_time_s, iter_dict['trialnumber_vector_samples'])"
+ "plt.plot(clock_time_s, iter_dict[\"trialnumber_vector_samples\"])"
]
},
{
@@ -437,7 +445,12 @@
],
"source": [
"# Are the manually-computed trial_index and frame_index arrays different from what we already had?\n",
- "plt.hist(np.diff(np.stack((trial_index, iter_dict['trialnumber_vector_samples']), axis=1), axis=1),bins=100);"
+ "plt.hist(\n",
+ " np.diff(\n",
+ " np.stack((trial_index, iter_dict[\"trialnumber_vector_samples\"]), axis=1), axis=1\n",
+ " ),\n",
+ " bins=100,\n",
+ ");"
]
},
{
@@ -459,7 +472,12 @@
],
"source": [
"# What about the iteration index?\n",
- "plt.hist(np.diff(np.stack((frame_index, iter_dict['framenumber_vector_samples']), axis=1), axis=1), density=True);"
+ "plt.hist(\n",
+ " np.diff(\n",
+ " np.stack((frame_index, iter_dict[\"framenumber_vector_samples\"]), axis=1), axis=1\n",
+ " ),\n",
+ " density=True,\n",
+ ");"
]
},
{
@@ -478,7 +496,7 @@
}
],
"source": [
- "lolo = frame_index - iter_dict['framenumber_vector_samples']\n",
+ "lolo = frame_index - iter_dict[\"framenumber_vector_samples\"]\n",
"\n",
"print(np.nanmax(lolo))\n",
"print(np.nanmin(lolo))"
@@ -510,7 +528,11 @@
}
],
"source": [
- "plt.plot(np.diff(np.stack((frame_index, iter_dict['framenumber_vector_samples']), axis=1), axis=1));"
+ "plt.plot(\n",
+ " np.diff(\n",
+ " np.stack((frame_index, iter_dict[\"framenumber_vector_samples\"]), axis=1), axis=1\n",
+ " )\n",
+ ");"
]
},
{
@@ -546,15 +568,19 @@
"\n",
"spk_resp = []\n",
"for itrial in range(NTRIALS):\n",
- " epoch = nap.IntervalSet(obx_trial_start_s[itrial] - pre_s, obx_trial_start_s[itrial] + post_s)\n",
- " spk_resp.append( spikes.restrict(epoch).count(bin_size=bin_size) )\n",
+ " epoch = nap.IntervalSet(\n",
+ " obx_trial_start_s[itrial] - pre_s, obx_trial_start_s[itrial] + post_s\n",
+ " )\n",
+ " spk_resp.append(spikes.restrict(epoch).count(bin_size=bin_size))\n",
"\n",
- "t = spk_resp[0].t - obx_trial_start_s[0] # time axis\n",
+ "t = spk_resp[0].t - obx_trial_start_s[0] # time axis\n",
"spk_resp = np.stack(spk_resp, axis=2)\n",
"mean_resp = spk_resp.mean(axis=2).mean(axis=1).T\n",
"\n",
"plt.plot(t, mean_resp)\n",
- "plt.gca().set(xlabel='time [s]', ylabel='spks/s', title='mean population response to trial onset');"
+ "plt.gca().set(\n",
+ " xlabel=\"time [s]\", ylabel=\"spks/s\", title=\"mean population response to trial onset\"\n",
+ ");"
]
},
{
@@ -572,13 +598,14 @@
"metadata": {},
"outputs": [],
"source": [
- "np.savez('ya054_2025-10-14_manual_idx_vecs_v2.npz', \n",
- " trial_index_nidq=trial_index, \n",
- " iteration_index_nidq=frame_index, \n",
- " obx_trial_start_idx=obx_trial_start_idx,\n",
- " obx_trial_stop_idx=obx_trial_stop_idx,\n",
- " iter_start_idx=np.array(iter_start_idx, dtype='object'),\n",
- " sampling_rate=nidq_sampling_rate,\n",
+ "np.savez(\n",
+ " \"ya054_2025-10-14_manual_idx_vecs_v2.npz\",\n",
+ " trial_index_nidq=trial_index,\n",
+ " iteration_index_nidq=frame_index,\n",
+ " obx_trial_start_idx=obx_trial_start_idx,\n",
+ " obx_trial_stop_idx=obx_trial_stop_idx,\n",
+ " iter_start_idx=np.array(iter_start_idx, dtype=\"object\"),\n",
+ " sampling_rate=nidq_sampling_rate,\n",
")"
]
},
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb
index c30e8a2b..f3aa9439 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -42,11 +43,8 @@
}
],
"source": [
- "import pandas as pd\n",
- "import numpy as np\n",
"import datajoint as dj\n",
- "\n",
- "\n"
+ "import pandas as pd"
]
},
{
@@ -63,10 +61,12 @@
}
],
"source": [
- "#Behavior data\n",
- "recording_process = dj.create_virtual_module('recording_process', 'u19_recording_process')\n",
- "recording = dj.create_virtual_module('recording', 'u19_recording')\n",
- "#EphysPipeline = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')"
+ "# Behavior data\n",
+ "recording_process = dj.create_virtual_module(\n",
+ " \"recording_process\", \"u19_recording_process\"\n",
+ ")\n",
+ "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n",
+ "# EphysPipeline = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')"
]
},
{
@@ -75,12 +75,17 @@
"metadata": {},
"outputs": [],
"source": [
- "query_subjects =\\\n",
- " \"subject_fullname LIKE '%jknpx4%' OR subject_fullname LIKE '%jknpx5%' OR subject_fullname LIKE '%jk85%' OR subject_fullname LIKE '%jk86%'\"\n",
+ "query_subjects = \"subject_fullname LIKE '%jknpx4%' OR subject_fullname LIKE '%jknpx5%' OR subject_fullname LIKE '%jk85%' OR subject_fullname LIKE '%jk86%'\"\n",
"\n",
"\n",
- "\n",
- "selected_process = pd.DataFrame((recording_process.Processing * recording.Recording * recording.Recording.BehaviorSession & query_subjects).fetch(as_dict=True, order_by='job_id'))"
+ "selected_process = pd.DataFrame(\n",
+ " (\n",
+ " recording_process.Processing\n",
+ " * recording.Recording\n",
+ " * recording.Recording.BehaviorSession\n",
+ " & query_subjects\n",
+ " ).fetch(as_dict=True, order_by=\"job_id\")\n",
+ ")"
]
},
{
@@ -741,10 +746,10 @@
}
],
"source": [
- "selected_process_copy['status_processing_id'] = -1\n",
- "selected_process_copy['slurm_id'] = None\n",
- "selected_process_copy = selected_process_copy.drop(columns={'job_id'}, axis=1)\n",
- "selected_process_copy = selected_process_copy.iloc[:,0:9]\n",
+ "selected_process_copy[\"status_processing_id\"] = -1\n",
+ "selected_process_copy[\"slurm_id\"] = None\n",
+ "selected_process_copy = selected_process_copy.drop(columns={\"job_id\"}, axis=1)\n",
+ "selected_process_copy = selected_process_copy.iloc[:, 0:9]\n",
"selected_process_copy"
]
},
@@ -754,7 +759,7 @@
"metadata": {},
"outputs": [],
"source": [
- "#recording_process.Processing.insert(selected_process_copy)"
+ "# recording_process.Processing.insert(selected_process_copy)"
]
},
{
@@ -763,11 +768,18 @@
"metadata": {},
"outputs": [],
"source": [
- "query_job_id = 'job_id > 900'\n",
- "\n",
+ "query_job_id = \"job_id > 900\"\n",
"\n",
"\n",
- "new_process = pd.DataFrame((recording_process.Processing * recording.Recording * recording.Recording.BehaviorSession & query_subjects & query_job_id).fetch(as_dict=True, order_by='job_id'))\n"
+ "new_process = pd.DataFrame(\n",
+ " (\n",
+ " recording_process.Processing\n",
+ " * recording.Recording\n",
+ " * recording.Recording.BehaviorSession\n",
+ " & query_subjects\n",
+ " & query_job_id\n",
+ " ).fetch(as_dict=True, order_by=\"job_id\")\n",
+ ")"
]
},
{
@@ -1308,12 +1320,18 @@
}
],
"source": [
- "new_process_params = new_process[['job_id', 'fragment_number','status_processing_id']].copy()\n",
+ "new_process_params = new_process[\n",
+ " [\"job_id\", \"fragment_number\", \"status_processing_id\"]\n",
+ "].copy()\n",
"\n",
- "new_process_params['fragment_number'] = 1\n",
- "new_process_params['status_processing_id'] = 4\n",
- "new_process_params = new_process_params.rename(columns={'fragment_number': 'precluster_param_steps_id',\\\n",
- " 'status_processing_id':'paramset_idx'})\n",
+ "new_process_params[\"fragment_number\"] = 1\n",
+ "new_process_params[\"status_processing_id\"] = 4\n",
+ "new_process_params = new_process_params.rename(\n",
+ " columns={\n",
+ " \"fragment_number\": \"precluster_param_steps_id\",\n",
+ " \"status_processing_id\": \"paramset_idx\",\n",
+ " }\n",
+ ")\n",
"\n",
"\n",
"new_process_params"
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb
index 3d08d9e7..0a4c57ee 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -44,19 +45,11 @@
}
],
"source": [
- "import datetime\n",
"import pathlib\n",
- "import numpy as np\n",
- "import pylab as pl\n",
- "import matplotlib\n",
- "import matplotlib.pyplot as plt\n",
- "from scipy.signal import find_peaks\n",
"\n",
- "\n",
- "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory\n",
"# import u19_pipeline.ephys_sync as ephys\n",
- "import u19_pipeline.acquisition as acquisition\n",
- "import datajoint as dj\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
"\n",
"import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n",
"import u19_pipeline.utils.ephys_utils as ephys_utils"
@@ -84,22 +77,22 @@
}
],
"source": [
- "session_dir = pathlib.Path('//cup.pni.princeton.edu/braininit/Data/Raw/electrophysiology/jyanar/jyanar_ya054/20251012_g0/ya054_20251012_g0/')\n",
- "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
- "\n",
- "\n",
+ "session_dir = pathlib.Path(\n",
+ " \"//cup.pni.princeton.edu/braininit/Data/Raw/electrophysiology/jyanar/jyanar_ya054/20251012_g0/ya054_20251012_g0/\"\n",
+ ")\n",
+ "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
"\n",
"\n",
- "nidq_bin_full_path = list(session_dir.glob('*obx.bin*'))[0]\n",
+ "nidq_bin_full_path = list(session_dir.glob(\"*obx.bin*\"))[0]\n",
"\n",
- "#Nidaq file\n",
- "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n",
+ "# Nidaq file\n",
+ "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n",
"\n",
"print(nidq_meta)\n",
- "#nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n",
- "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta, d_line_list=[0,3])\n",
- "\n",
- "\n"
+ "# nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n",
+ "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " nidq_bin_full_path, nidq_meta, d_line_list=[0, 3]\n",
+ ")"
]
},
{
@@ -172,7 +165,7 @@
}
],
"source": [
- "plt.plot(digital_array[0,:])"
+ "plt.plot(digital_array[0, :])"
]
},
{
@@ -192,7 +185,7 @@
}
],
"source": [
- "trial_start_idx = (np.where(np.diff(digital_array[0,:]) == 1))\n",
+ "trial_start_idx = np.where(np.diff(digital_array[0, :]) == 1)\n",
"len(trial_start_idx[0])"
]
},
@@ -223,8 +216,8 @@
}
],
"source": [
- "plt.plot(digital_array[0,trial_start_idx[0][0]-1000:trial_start_idx[0][0]+2000])\n",
- "plt.plot(digital_array[1,trial_start_idx[0][0]-1000:trial_start_idx[0][0]+2000])"
+ "plt.plot(digital_array[0, trial_start_idx[0][0] - 1000 : trial_start_idx[0][0] + 2000])\n",
+ "plt.plot(digital_array[1, trial_start_idx[0][0] - 1000 : trial_start_idx[0][0] + 2000])"
]
},
{
@@ -1749,21 +1742,34 @@
}
],
"source": [
- "mode = None #Default for sessions before 12/01/2021\n",
- "#mode = 'pulses' #Default for sessions after 12/01/2021\n",
- "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[1,:], digital_array[2,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n",
+ "mode = None # Default for sessions before 12/01/2021\n",
+ "# mode = 'pulses' #Default for sessions after 12/01/2021\n",
+ "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n",
+ " digital_array[1, :],\n",
+ " digital_array[2, :],\n",
+ " nidq_sampling_rate,\n",
+ " behavior_time.shape[0],\n",
+ " behavior_time,\n",
+ " mode=mode,\n",
+ ")\n",
"# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n",
"\n",
"\n",
"# Check # of trials and iterations match\n",
- "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n",
+ "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n",
+ " ephys_utils.assert_iteration_samples_count(\n",
+ " iteration_dict[\"iter_start_idx\"], behavior_time\n",
+ " )\n",
+ ")\n",
"\n",
"print(trial_count_diff)\n",
"print(trials_diff_iteration_big)\n",
"print(trials_diff_iteration_small)\n",
"\n",
"\n",
- "status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small)\n"
+ "status = ephys_utils.evaluate_sync_process(\n",
+ " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small\n",
+ ")"
]
},
{
@@ -1792,7 +1798,7 @@
"metadata": {},
"outputs": [],
"source": [
- "it_times = iteration_dict['iter_times_idx'][trials_diff_iteration_small[0]]\n",
+ "it_times = iteration_dict[\"iter_times_idx\"][trials_diff_iteration_small[0]]\n",
"beh_times = behavior_time[trials_diff_iteration_small[0]].flatten()"
]
},
@@ -1814,11 +1820,12 @@
],
"source": [
"if it_times.shape[0] >= beh_times.shape[0]:\n",
- " print('More pulses than behavior iterations, check other method')\n",
+ " print(\"More pulses than behavior iterations, check other method\")\n",
"else:\n",
- " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n",
+ " diff_vector = np.diff(it_times - beh_times[: it_times.shape[0]])\n",
"\n",
"import scipy\n",
+ "\n",
"peaks, _ = scipy.signal.find_peaks(diff_vector, height=0.05, distance=20)\n",
"peaks"
]
@@ -1853,7 +1860,7 @@
"metadata": {},
"outputs": [],
"source": [
- "difo_iters = np.diff(iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]])\n",
+ "difo_iters = np.diff(iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]])\n",
"peaks2, _ = scipy.signal.find_peaks(difo_iters, height=1500, distance=20)"
]
},
@@ -1874,16 +1881,16 @@
}
],
"source": [
- "original_iter = iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]]\n",
- "new_iter_start = iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]]\n",
+ "original_iter = iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]]\n",
+ "new_iter_start = iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]]\n",
"\n",
"for i in range(peaks.shape[0]):\n",
- " value_insert = (original_iter[peaks[i]] + original_iter[peaks[i]+1] ) /2\n",
+ " value_insert = (original_iter[peaks[i]] + original_iter[peaks[i] + 1]) / 2\n",
" new_iter_start = np.insert(new_iter_start, peaks[i], value_insert)\n",
"\n",
"\n",
- "if new_iter_start.shape[0] != beh_times[:it_times.shape[0]]:\n",
- " print('with peak strategy, could not find correct missing iterations')\n"
+ "if new_iter_start.shape[0] != beh_times[: it_times.shape[0]]:\n",
+ " print(\"with peak strategy, could not find correct missing iterations\")"
]
},
{
@@ -1936,7 +1943,7 @@
}
],
"source": [
- "times = new_iter_start/nidq_sampling_rate\n",
+ "times = new_iter_start / nidq_sampling_rate\n",
"times = times - times[0]\n",
"new_diff_times = times - beh_times\n",
"plt.plot(new_diff_times)"
@@ -1979,9 +1986,9 @@
],
"source": [
"print(iteration_dict.keys())\n",
- "so = np.where(iteration_dict['trialnumber_vector_samples'] == 12)\n",
+ "so = np.where(iteration_dict[\"trialnumber_vector_samples\"] == 12)\n",
"\n",
- "print(so[0])\n"
+ "print(so[0])"
]
},
{
@@ -2027,20 +2034,37 @@
"end_iter = 2704\n",
"samp_after = 60000\n",
"\n",
- "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n",
- "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 1,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot][end_iter],\n",
+ " ]\n",
+ " + samp_after\n",
+ ")\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 2,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot][end_iter],\n",
+ " ]\n",
+ " + samp_after\n",
+ ")\n",
"\n",
"\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
"\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
"\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -2078,20 +2102,37 @@
"start_iter = 2704\n",
"samp_after = 600\n",
"\n",
- "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+samp_after])\n",
- "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+samp_after])\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 2,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n",
+ " + samp_after,\n",
+ " ]\n",
+ ")\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 1,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n",
+ " + samp_after,\n",
+ " ]\n",
+ ")\n",
"\n",
"\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
"\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
"\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -2110,8 +2151,8 @@
],
"source": [
"trial_plot = 280\n",
- "print(iteration_dict['iter_start_idx'][trial_plot][2704])\n",
- "print(iteration_dict['iter_start_idx'][trial_plot+1][0])"
+ "print(iteration_dict[\"iter_start_idx\"][trial_plot][2704])\n",
+ "print(iteration_dict[\"iter_start_idx\"][trial_plot + 1][0])"
]
},
{
@@ -2143,7 +2184,7 @@
}
],
"source": [
- "plt.plot(np.diff(iteration_dict['iter_start_idx'][trial_plot]/nidq_sampling_rate))"
+ "plt.plot(np.diff(iteration_dict[\"iter_start_idx\"][trial_plot] / nidq_sampling_rate))"
]
},
{
@@ -2223,14 +2264,14 @@
"source": [
"x = np.array([])\n",
"mean_x = np.array([])\n",
- "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n",
- " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n",
+ "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n",
+ " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n",
" mean_time_trial = np.mean(s)\n",
- " x = np.append(x,s, axis=0)\n",
+ " x = np.append(x, s, axis=0)\n",
" mean_x = np.append(mean_x, mean_time_trial)\n",
"\n",
- "#plt.plot(x)\n",
- "plt.plot(mean_x)\n"
+ "# plt.plot(x)\n",
+ "plt.plot(mean_x)"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb
index 5c7213fb..19e9bee3 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -43,23 +44,20 @@
"source": [
"import datetime\n",
"import pathlib\n",
- "import numpy as np\n",
- "import pylab as pl\n",
- "import matplotlib\n",
- "import matplotlib.pyplot as plt\n",
- "from scipy.signal import find_peaks\n",
- "from scipy import signal as sp\n",
"\n",
- "\n",
- "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory, get_ephys_root_data_dir\n",
- "# import u19_pipeline.ephys_sync as ephys\n",
- "import u19_pipeline.acquisition as acquisition\n",
"import datajoint as dj\n",
- "\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
"from element_interface.utils import find_full_path\n",
"\n",
+ "# import u19_pipeline.ephys_sync as ephys\n",
+ "import u19_pipeline.acquisition as acquisition\n",
"import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n",
- "import u19_pipeline.utils.ephys_utils as ephys_utils\n"
+ "import u19_pipeline.utils.ephys_utils as ephys_utils\n",
+ "from u19_pipeline.ephys_pipeline import (\n",
+ " get_ephys_root_data_dir,\n",
+ " get_session_directory,\n",
+ ")"
]
},
{
@@ -79,25 +77,26 @@
}
],
"source": [
- "key = {'subject_fullname': 'jyanar_ya054',\n",
- " 'session_date': datetime.date(2025, 10, 11)}\n",
- "#key = {\n",
+ "key = {\"subject_fullname\": \"jyanar_ya054\", \"session_date\": datetime.date(2025, 10, 11)}\n",
+ "# key = {\n",
"# 'subject_fullname': 'jk8386_jk83',\n",
"# 'session_date': datetime.date(2025, 7, 31),\n",
"# 'session_number': 0 }\n",
"\n",
- "#key2 = 'subject_fullname = \"jjulian_jj046\" and session_date = \"2022-04-19\" and block >=2'\n",
- "#key = {\n",
+ "# key2 = 'subject_fullname = \"jjulian_jj046\" and session_date = \"2022-04-19\" and block >=2'\n",
+ "# key = {\n",
"# 'subject_fullname': 'jjulian_jj042',\n",
"# 'session_date': datetime.date(2021, 10, 31)}\n",
"\n",
"\n",
- "key = (acquisition.Session & key).fetch1('KEY')\n",
+ "key = (acquisition.Session & key).fetch1(\"KEY\")\n",
"key\n",
"\n",
- "recording = dj.create_virtual_module('recording', 'u19_recording')\n",
+ "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n",
"\n",
- "recording_key = ((acquisition.Session * recording.Recording.BehaviorSession) & key).fetch('recording_id', as_dict=True)\n",
+ "recording_key = (\n",
+ " (acquisition.Session * recording.Recording.BehaviorSession) & key\n",
+ ").fetch(\"recording_id\", as_dict=True)\n",
"recording_key"
]
},
@@ -119,7 +118,7 @@
],
"source": [
"session_dir = pathlib.Path(get_session_directory(recording_key[0]))\n",
- "session_dir\n"
+ "session_dir"
]
},
{
@@ -137,38 +136,43 @@
}
],
"source": [
- "session_dir = find_full_path(get_ephys_root_data_dir(),\n",
- " get_session_directory(recording_key))\n",
+ "session_dir = find_full_path(\n",
+ " get_ephys_root_data_dir(), get_session_directory(recording_key)\n",
+ ")\n",
"print(session_dir)\n",
- "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
- "#Check if session is Nidq or OneBox\n",
- "nidq_session = list(session_dir.glob('*nidq.bin*'))\n",
- "obx_session = list(session_dir.glob('*obx.bin*'))\n",
+ "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
+ "# Check if session is Nidq or OneBox\n",
+ "nidq_session = list(session_dir.glob(\"*nidq.bin*\"))\n",
+ "obx_session = list(session_dir.glob(\"*obx.bin*\"))\n",
"\n",
"if len(nidq_session) == 0 and len(obx_session) == 0:\n",
- " print('No session found')\n",
+ " print(\"No session found\")\n",
"elif len(nidq_session) > 0:\n",
" ephys_session_fullpath = nidq_session[0]\n",
"else:\n",
" ephys_session_fullpath = obx_session[0]\n",
"\n",
- "#Nidaq file\n",
- "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n",
+ "# Nidaq file\n",
+ "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n",
"nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n",
"\n",
"\n",
- "# 1: load meta data, and the content of the NIDAQ file. Its content is digital. \n",
+ "# 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n",
"new_trial_channel = 1\n",
"new_iteration_channel = 2\n",
"# If PXIe card (nidq) card use for recording deduce digital channels\n",
- "if nidq_meta['typeThis'] == 'nidq':\n",
- " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)\n",
+ "if nidq_meta[\"typeThis\"] == \"nidq\":\n",
+ " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " ephys_session_fullpath, nidq_meta\n",
+ " )\n",
"# If onebox card (obx) card use for recording digital channels are 0-2\n",
"else:\n",
- " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])\n",
+ " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]\n",
+ " )\n",
" # If no sync pulse found trial and iteration signals are 0 & 1 respectively\n",
- " channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]\n",
- " channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]\n",
+ " channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]\n",
+ " channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]\n",
"\n",
" if channel0_pulses > channel1_pulses:\n",
" new_trial_channel = 1\n",
@@ -205,10 +209,12 @@
"metadata": {},
"outputs": [],
"source": [
- "#Behavior data\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n",
+ "# Behavior data\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
"thissession = behavior.TowersBlock().Trial() & key\n",
- "behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n"
+ "behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n",
+ " \"trial_time\", \"vi_start\", \"iterations\"\n",
+ ")"
]
},
{
@@ -281,21 +287,37 @@
}
],
"source": [
- "mode = None #Default for sessions before 12/01/2021\n",
- "#mode = 'pulses' #Default for sessions after 12/01/2021\n",
- "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[new_trial_channel,:], digital_array[new_iteration_channel,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n",
+ "mode = None # Default for sessions before 12/01/2021\n",
+ "# mode = 'pulses' #Default for sessions after 12/01/2021\n",
+ "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n",
+ " digital_array[new_trial_channel, :],\n",
+ " digital_array[new_iteration_channel, :],\n",
+ " nidq_sampling_rate,\n",
+ " behavior_time.shape[0],\n",
+ " behavior_time,\n",
+ " mode=mode,\n",
+ ")\n",
"# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n",
"\n",
"\n",
"# Check # of trials and iterations match\n",
- "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n",
+ "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n",
+ " ephys_utils.assert_iteration_samples_count(\n",
+ " iteration_dict[\"iter_start_idx\"], behavior_time\n",
+ " )\n",
+ ")\n",
"\n",
"print(trial_count_diff)\n",
"print(trials_diff_iteration_big)\n",
"print(trials_diff_iteration_small)\n",
"\n",
"\n",
- "status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n"
+ "status = ephys_utils.evaluate_sync_process(\n",
+ " trial_count_diff,\n",
+ " trials_diff_iteration_big,\n",
+ " trials_diff_iteration_small,\n",
+ " behavior_time.shape[0],\n",
+ ")"
]
},
{
@@ -312,8 +334,11 @@
}
],
"source": [
- "for i in range(len(iteration_dict['iter_start_idx'])):\n",
- " if iteration_dict['iter_start_idx'][i].shape[0] > behavior_time[i].flatten().shape[0]:\n",
+ "for i in range(len(iteration_dict[\"iter_start_idx\"])):\n",
+ " if (\n",
+ " iteration_dict[\"iter_start_idx\"][i].shape[0]\n",
+ " > behavior_time[i].flatten().shape[0]\n",
+ " ):\n",
" print(i)"
]
},
@@ -324,9 +349,9 @@
"outputs": [],
"source": [
"trial_fix = 1\n",
- "synced_iteration_vector = iteration_dict['iter_start_idx'][trial_fix]\n",
- "synced_time_vector = iteration_dict['iter_times_idx'][trial_fix]\n",
- "behavior_time_vector = behavior_time[trial_fix].flatten()\n"
+ "synced_iteration_vector = iteration_dict[\"iter_start_idx\"][trial_fix]\n",
+ "synced_time_vector = iteration_dict[\"iter_times_idx\"][trial_fix]\n",
+ "behavior_time_vector = behavior_time[trial_fix].flatten()"
]
},
{
@@ -347,43 +372,58 @@
}
],
"source": [
- "\n",
"# Check where is more likely we miss an iteration pulse and insert it to iteration_vector\n",
- "print('synced_iteration_vector', synced_iteration_vector.shape[0])\n",
- "print('synced_time_vector', synced_time_vector.shape[0])\n",
- "print('behavior_time_vector', behavior_time_vector.shape[0])\n",
+ "print(\"synced_iteration_vector\", synced_iteration_vector.shape[0])\n",
+ "print(\"synced_time_vector\", synced_time_vector.shape[0])\n",
+ "print(\"behavior_time_vector\", behavior_time_vector.shape[0])\n",
"\n",
"new_synced_iteration_vector = synced_iteration_vector.copy()\n",
"new_synced_time_vector = synced_time_vector.copy()\n",
"\n",
"diff_time_test = new_synced_time_vector[2:50] - behavior_time_vector[1:49]\n",
"pulse_greater_than_virmen = np.where(diff_time_test > 0)\n",
- "pulse_greater_than_virmen = pulse_greater_than_virmen[0]+0\n",
+ "pulse_greater_than_virmen = pulse_greater_than_virmen[0] + 0\n",
"if pulse_greater_than_virmen.shape[0] == 0:\n",
- " #print('we have a special case of extra pulse beginning')\n",
+ " # print('we have a special case of extra pulse beginning')\n",
" new_synced_iteration_vector = np.delete(new_synced_iteration_vector, 1)\n",
- " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n",
+ " new_synced_time_vector = (\n",
+ " new_synced_iteration_vector - new_synced_iteration_vector[0]\n",
+ " ) / nidq_sampling_rate\n",
"\n",
"for i in range(100):\n",
" sir = 0\n",
- " diff_time = new_synced_time_vector[sir:] - behavior_time_vector[sir:new_synced_time_vector.shape[0]]\n",
+ " diff_time = (\n",
+ " new_synced_time_vector[sir:]\n",
+ " - behavior_time_vector[sir : new_synced_time_vector.shape[0]]\n",
+ " )\n",
" pulse_greater_than_virmen = np.where(diff_time > 0)\n",
- " pulse_greater_than_virmen = pulse_greater_than_virmen[0]+sir\n",
+ " pulse_greater_than_virmen = pulse_greater_than_virmen[0] + sir\n",
"\n",
" if pulse_greater_than_virmen.shape[0] > 0:\n",
" fix_trial = 1\n",
- " \n",
+ "\n",
" else:\n",
" fix_trial = 0\n",
" break\n",
"\n",
" print(i)\n",
- " print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
- " print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
+ " print(\n",
+ " new_synced_time_vector[\n",
+ " pulse_greater_than_virmen[0] - 2 : pulse_greater_than_virmen[0] + 4\n",
+ " ]\n",
+ " )\n",
+ " print(\n",
+ " behavior_time_vector[\n",
+ " pulse_greater_than_virmen[0] - 2 : pulse_greater_than_virmen[0] + 4\n",
+ " ]\n",
+ " )\n",
" print(pulse_greater_than_virmen[0])\n",
"\n",
" if pulse_greater_than_virmen[0] > 3:\n",
- " median_diff = np.median(new_synced_time_vector[0:pulse_greater_than_virmen[0]-3]-behavior_time_vector[0:pulse_greater_than_virmen[0]-3])\n",
+ " median_diff = np.median(\n",
+ " new_synced_time_vector[0 : pulse_greater_than_virmen[0] - 3]\n",
+ " - behavior_time_vector[0 : pulse_greater_than_virmen[0] - 3]\n",
+ " )\n",
" else:\n",
" median_diff = -0.015\n",
"\n",
@@ -391,63 +431,118 @@
" comp_value_l = -0.015\n",
" comp_value_g = -0.003\n",
" else:\n",
- " comp_value_l = median_diff*1.3\n",
- " comp_value_g = median_diff*0.7\n",
- "\n",
- " print('median_diff',median_diff, 'comp_value_l', comp_value_l,'comp_value_g', comp_value_g)\n",
- "\n",
- " diff_time_iter_missing_pulse = behavior_time_vector[pulse_greater_than_virmen[0]] - behavior_time_vector[pulse_greater_than_virmen[0]-1]\n",
- " print('missing time in iter pulses', behavior_time_vector[pulse_greater_than_virmen[0]])\n",
- " print('time_next_missing_pulse', diff_time_iter_missing_pulse)\n",
- " diff_iter_add = np.floor(diff_time_iter_missing_pulse*nidq_sampling_rate)\n",
- " value_insert_iteration = new_synced_iteration_vector[pulse_greater_than_virmen[0]-1] + diff_iter_add\n",
- "\n",
- " print('value_insert_iteration', value_insert_iteration)\n",
- "\n",
- " new_synced_iteration_vector = np.insert(new_synced_iteration_vector, pulse_greater_than_virmen[0], value_insert_iteration)\n",
+ " comp_value_l = median_diff * 1.3\n",
+ " comp_value_g = median_diff * 0.7\n",
+ "\n",
+ " print(\n",
+ " \"median_diff\",\n",
+ " median_diff,\n",
+ " \"comp_value_l\",\n",
+ " comp_value_l,\n",
+ " \"comp_value_g\",\n",
+ " comp_value_g,\n",
+ " )\n",
+ "\n",
+ " diff_time_iter_missing_pulse = (\n",
+ " behavior_time_vector[pulse_greater_than_virmen[0]]\n",
+ " - behavior_time_vector[pulse_greater_than_virmen[0] - 1]\n",
+ " )\n",
+ " print(\n",
+ " \"missing time in iter pulses\",\n",
+ " behavior_time_vector[pulse_greater_than_virmen[0]],\n",
+ " )\n",
+ " print(\"time_next_missing_pulse\", diff_time_iter_missing_pulse)\n",
+ " diff_iter_add = np.floor(diff_time_iter_missing_pulse * nidq_sampling_rate)\n",
+ " value_insert_iteration = (\n",
+ " new_synced_iteration_vector[pulse_greater_than_virmen[0] - 1] + diff_iter_add\n",
+ " )\n",
+ "\n",
+ " print(\"value_insert_iteration\", value_insert_iteration)\n",
+ "\n",
+ " new_synced_iteration_vector = np.insert(\n",
+ " new_synced_iteration_vector,\n",
+ " pulse_greater_than_virmen[0],\n",
+ " value_insert_iteration,\n",
+ " )\n",
"\n",
" if new_synced_iteration_vector.shape[0] > behavior_time_vector.shape[0]:\n",
- " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, pulse_greater_than_virmen[0]+1)\n",
- " #new_synced_iteration_vector = new_synced_iteration_vector[:-1]\n",
+ " new_synced_iteration_vector = np.delete(\n",
+ " new_synced_iteration_vector, pulse_greater_than_virmen[0] + 1\n",
+ " )\n",
+ " # new_synced_iteration_vector = new_synced_iteration_vector[:-1]\n",
"\n",
- " if pulse_greater_than_virmen[0] < (new_synced_time_vector.shape[0]-1):\n",
+ " if pulse_greater_than_virmen[0] < (new_synced_time_vector.shape[0] - 1):\n",
" idx = pulse_greater_than_virmen[0]\n",
- " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n",
- " diff_vectorin = (new_synced_time_vector[idx+1:idx+10]- behavior_time_vector[idx+1:idx+10])\n",
+ " new_synced_time_vector = (\n",
+ " new_synced_iteration_vector - new_synced_iteration_vector[0]\n",
+ " ) / nidq_sampling_rate\n",
+ " diff_vectorin = (\n",
+ " new_synced_time_vector[idx + 1 : idx + 10]\n",
+ " - behavior_time_vector[idx + 1 : idx + 10]\n",
+ " )\n",
" print(diff_vectorin)\n",
" diff_inserted_iter = np.median(diff_vectorin)\n",
- " print('diff_in new inserted iter', diff_inserted_iter)\n",
+ " print(\"diff_in new inserted iter\", diff_inserted_iter)\n",
" if diff_inserted_iter < comp_value_l:\n",
- " print('extra diff !!!!')\n",
- " diff_behavior = behavior_time_vector[idx+1] - behavior_time_vector[idx]\n",
- " idx_next_iter = np.where(new_synced_time_vector > behavior_time_vector[idx+1])\n",
- " print('diff_behavior', diff_behavior)\n",
+ " print(\"extra diff !!!!\")\n",
+ " diff_behavior = behavior_time_vector[idx + 1] - behavior_time_vector[idx]\n",
+ " idx_next_iter = np.where(\n",
+ " new_synced_time_vector > behavior_time_vector[idx + 1]\n",
+ " )\n",
+ " print(\"diff_behavior\", diff_behavior)\n",
" idx_next_iter = idx_next_iter[0][0]\n",
- " print('idx_next_iter', idx_next_iter)\n",
- " print('idx+1', idx+1)\n",
- " if diff_behavior > 0.1 and idx_next_iter > idx+3:\n",
- " print('bajale a indx_next_iter', idx_next_iter)\n",
- " idx_next_iter = idx+3\n",
- " if idx_next_iter > idx+2:\n",
- " print('more than one to remove')\n",
- " int_array_step = np.arange(idx+1, idx_next_iter-1, dtype=np.int64)\n",
- " print('int_array_step', int_array_step)\n",
- " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, int_array_step)\n",
+ " print(\"idx_next_iter\", idx_next_iter)\n",
+ " print(\"idx+1\", idx + 1)\n",
+ " if diff_behavior > 0.1 and idx_next_iter > idx + 3:\n",
+ " print(\"bajale a indx_next_iter\", idx_next_iter)\n",
+ " idx_next_iter = idx + 3\n",
+ " if idx_next_iter > idx + 2:\n",
+ " print(\"more than one to remove\")\n",
+ " int_array_step = np.arange(idx + 1, idx_next_iter - 1, dtype=np.int64)\n",
+ " print(\"int_array_step\", int_array_step)\n",
+ " new_synced_iteration_vector = np.delete(\n",
+ " new_synced_iteration_vector, int_array_step\n",
+ " )\n",
" else:\n",
- " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, [idx+1])\n",
- " print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
+ " new_synced_iteration_vector = np.delete(\n",
+ " new_synced_iteration_vector, [idx + 1]\n",
+ " )\n",
+ " print(\n",
+ " new_synced_time_vector[\n",
+ " pulse_greater_than_virmen[0] - 2 : pulse_greater_than_virmen[0] + 4\n",
+ " ]\n",
+ " )\n",
" elif diff_inserted_iter > comp_value_g and diff_inserted_iter < 0:\n",
- " print('less diff now, insert one iteration')\n",
- " value_insert = int((new_synced_iteration_vector[pulse_greater_than_virmen[0]]+new_synced_iteration_vector[pulse_greater_than_virmen[0]+1])/2)\n",
- " new_synced_iteration_vector = np.insert(new_synced_iteration_vector, pulse_greater_than_virmen[0]+1, value_insert)\n",
- "\n",
- "\n",
- " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n",
- "\n",
- " print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
- " print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
- "\n",
- " '''\n",
+ " print(\"less diff now, insert one iteration\")\n",
+ " value_insert = int(\n",
+ " (\n",
+ " new_synced_iteration_vector[pulse_greater_than_virmen[0]]\n",
+ " + new_synced_iteration_vector[pulse_greater_than_virmen[0] + 1]\n",
+ " )\n",
+ " / 2\n",
+ " )\n",
+ " new_synced_iteration_vector = np.insert(\n",
+ " new_synced_iteration_vector,\n",
+ " pulse_greater_than_virmen[0] + 1,\n",
+ " value_insert,\n",
+ " )\n",
+ "\n",
+ " new_synced_time_vector = (\n",
+ " new_synced_iteration_vector - new_synced_iteration_vector[0]\n",
+ " ) / nidq_sampling_rate\n",
+ "\n",
+ " print(\n",
+ " new_synced_time_vector[\n",
+ " pulse_greater_than_virmen[0] - 2 : pulse_greater_than_virmen[0] + 4\n",
+ " ]\n",
+ " )\n",
+ " print(\n",
+ " behavior_time_vector[\n",
+ " pulse_greater_than_virmen[0] - 2 : pulse_greater_than_virmen[0] + 4\n",
+ " ]\n",
+ " )\n",
+ "\n",
+ " \"\"\"\n",
" diff_vector = np.diff(new_synced_time_vector - behavior_time_vector[:new_synced_time_vector.shape[0]])\n",
" #In case last peak is at the end of trial (append 0 to detect it)\n",
" diff_vector = np.append(diff_vector, np.array([0]))\n",
@@ -477,9 +572,7 @@
"\n",
" #print(new_synced_iteration_vector[peaks[0]-2:peaks[0]+5])\n",
" print(new_synced_time_vector[peaks[0]-2:peaks[0]+5])\n",
- " '''\n",
- " \n",
- "\n"
+ " \"\"\""
]
},
{
@@ -501,7 +594,7 @@
],
"source": [
"it_times = new_synced_time_vector\n",
- "#it_times = synced_time_vector\n",
+ "# it_times = synced_time_vector\n",
"beh_times = behavior_time_vector\n",
"\n",
"print(it_times.shape[0])\n",
@@ -527,11 +620,12 @@
],
"source": [
"if it_times.shape[0] >= beh_times.shape[0]:\n",
- " diff_vector = np.diff(it_times[:beh_times.shape[0]] - beh_times)\n",
+ " diff_vector = np.diff(it_times[: beh_times.shape[0]] - beh_times)\n",
"else:\n",
- " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n",
+ " diff_vector = np.diff(it_times - beh_times[: it_times.shape[0]])\n",
"\n",
"import scipy\n",
+ "\n",
"peaks, _ = scipy.signal.find_peaks(diff_vector, height=0.05, distance=20)\n",
"peaks"
]
@@ -554,13 +648,13 @@
}
],
"source": [
- "#plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n",
- "plt.plot((new_synced_time_vector - beh_times[:it_times.shape[0]]))\n",
- "#plt.plot((it_times - beh_times))\n",
- "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
+ "# plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n",
+ "plt.plot(new_synced_time_vector - beh_times[: it_times.shape[0]])\n",
+ "# plt.plot((it_times - beh_times))\n",
+ "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
"plt.plot(peaks, diff_vector[peaks], \"x\")\n",
- "plt.ylabel('Time diff in current iteration TTL - Virmen (s)')\n",
- "plt.xlabel('Iteration')\n",
+ "plt.ylabel(\"Time diff in current iteration TTL - Virmen (s)\")\n",
+ "plt.xlabel(\"Iteration\")\n",
"plt.ylim([-0.05, 0.02])\n",
"plt.show()"
]
@@ -584,11 +678,11 @@
],
"source": [
"plt.plot(np.diff(new_synced_time_vector))\n",
- "plt.plot(np.diff(beh_times)+0.01)\n",
- "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
- "#plt.plot(peaks, diff_vector[peaks], \"x\")\n",
+ "plt.plot(np.diff(beh_times) + 0.01)\n",
+ "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
+ "# plt.plot(peaks, diff_vector[peaks], \"x\")\n",
"plt.ylim([-0.01, 0.05])\n",
- "plt.xlabel('Iteration')\n"
+ "plt.xlabel(\"Iteration\")"
]
},
{
@@ -609,8 +703,10 @@
}
],
"source": [
- "difo = np.pad(np.diff(new_synced_time_vector), (1, 0), 'constant', constant_values=(0,0))\n",
- "difo2 = np.pad(np.diff(beh_times), (1, 0), 'constant', constant_values=(0,0))\n",
+ "difo = np.pad(\n",
+ " np.diff(new_synced_time_vector), (1, 0), \"constant\", constant_values=(0, 0)\n",
+ ")\n",
+ "difo2 = np.pad(np.diff(beh_times), (1, 0), \"constant\", constant_values=(0, 0))\n",
"\n",
"peaks_ttl_pulses, _ = scipy.signal.find_peaks(difo, height=0.02, distance=20)\n",
"peaks_virmen_iter, _ = scipy.signal.find_peaks(difo2, height=0.02, distance=20)\n",
@@ -680,8 +776,8 @@
}
],
"source": [
- "print(new_synced_time_vector[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n",
- "print(beh_times[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n"
+ "print(new_synced_time_vector[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])\n",
+ "print(beh_times[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])"
]
},
{
@@ -703,7 +799,7 @@
],
"source": [
"print(new_synced_time_vector[-4:])\n",
- "print(beh_times[-4:])\n"
+ "print(beh_times[-4:])"
]
},
{
@@ -724,10 +820,9 @@
}
],
"source": [
- "\n",
"plt.plot(np.diff(new_synced_time_vector))\n",
- "#plt.plot(np.diff(synced_time_vector))\n",
- "plt.plot(np.diff(behavior_time_vector)+0.01)\n",
+ "# plt.plot(np.diff(synced_time_vector))\n",
+ "plt.plot(np.diff(behavior_time_vector) + 0.01)\n",
"plt.ylim(0, 0.11)"
]
},
@@ -778,8 +873,8 @@
],
"source": [
"trial_plot = 116\n",
- "plt.plot((iteration_dict['iter_times_idx'][trial_plot]))\n",
- "plt.plot((behavior_time[trial_plot].flatten()))\n"
+ "plt.plot(iteration_dict[\"iter_times_idx\"][trial_plot])\n",
+ "plt.plot(behavior_time[trial_plot].flatten())"
]
},
{
@@ -801,15 +896,24 @@
"samp_before = 1000\n",
"samp_after = 1600\n",
"if trial_plot == 0:\n",
- " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot][0]-np.int64(samp_before)\n",
+ " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot][0] - np.int64(\n",
+ " samp_before\n",
+ " )\n",
"else:\n",
- " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot-1][-1]-np.int64(samp_before)\n",
- "first_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][0]+np.int64(samp_after)\n",
- "second_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][1]\n",
- "\n",
- "samp_diff = iteration_dict['iter_start_idx'][trial_plot][0] - iteration_dict['iter_start_idx'][trial_plot-1][-1]\n",
- "print('last_iter_trial0', last_iter_trial0)\n",
- "print('first_iter_trial1', first_iter_trial1)"
+ " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1] - np.int64(\n",
+ " samp_before\n",
+ " )\n",
+ "first_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][0] + np.int64(\n",
+ " samp_after\n",
+ ")\n",
+ "second_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][1]\n",
+ "\n",
+ "samp_diff = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1]\n",
+ ")\n",
+ "print(\"last_iter_trial0\", last_iter_trial0)\n",
+ "print(\"first_iter_trial1\", first_iter_trial1)"
]
},
{
@@ -852,55 +956,64 @@
}
],
"source": [
- "\n",
- "\n",
"start_iter = 1\n",
"end_iter = 1\n",
"\n",
"\n",
- "sample_start =(iteration_dict['iter_start_idx'][trial_plot][start_iter]-np.int64(samp_before))\n",
- "sample_end = (iteration_dict['iter_start_idx'][trial_plot][end_iter]+np.int64(samp_after))\n",
+ "sample_start = iteration_dict[\"iter_start_idx\"][trial_plot][start_iter] - np.int64(\n",
+ " samp_before\n",
+ ")\n",
+ "sample_end = iteration_dict[\"iter_start_idx\"][trial_plot][end_iter] + np.int64(\n",
+ " samp_after\n",
+ ")\n",
"\n",
- "iter_sample = digital_array[new_trial_channel,last_iter_trial0:first_iter_trial1]\n",
+ "iter_sample = digital_array[new_trial_channel, last_iter_trial0:first_iter_trial1]\n",
"\n",
- "time_vector = np.linspace(start=0, stop=iter_sample.shape[0]-1,num=iter_sample.shape[0])\n",
- "time_vector = (time_vector*1000/nidq_sampling_rate)\n",
- "time_vector -= ((iter_sample.shape[0]-samp_after)*1000)/nidq_sampling_rate\n",
- "\n",
- "time_last_trial = time_vector[0] +(samp_before*1000)/nidq_sampling_rate\n",
+ "time_vector = np.linspace(\n",
+ " start=0, stop=iter_sample.shape[0] - 1, num=iter_sample.shape[0]\n",
+ ")\n",
+ "time_vector = time_vector * 1000 / nidq_sampling_rate\n",
+ "time_vector -= ((iter_sample.shape[0] - samp_after) * 1000) / nidq_sampling_rate\n",
"\n",
+ "time_last_trial = time_vector[0] + (samp_before * 1000) / nidq_sampling_rate\n",
"\n",
"\n",
"print(iter_sample.shape[0])\n",
- "samples_after_second_pulse = (second_iter_trial1-(first_iter_trial1-samp_after))\n",
- "print('samples_after_second_pulse', samples_after_second_pulse)\n",
- "print(iter_sample.shape[0]-samp_after)\n",
+ "samples_after_second_pulse = second_iter_trial1 - (first_iter_trial1 - samp_after)\n",
+ "print(\"samples_after_second_pulse\", samples_after_second_pulse)\n",
+ "print(iter_sample.shape[0] - samp_after)\n",
"\n",
- "idx_time_zero = np.where((time_vector >= 0))\n",
+ "idx_time_zero = np.where(time_vector >= 0)\n",
"idx_time_zero = idx_time_zero[0]\n",
"idx_time_zero = idx_time_zero[0]\n",
"\n",
"\n",
- "print('new_iteration_channel', new_iteration_channel)\n",
- "print(digital_array[new_iteration_channel,:].shape)\n",
- "\n",
- "plt.plot(time_vector,iter_sample)\n",
- "plt.plot(time_vector,digital_array[new_iteration_channel,last_iter_trial0:first_iter_trial1]+0.02)\n",
- "plt.plot(0,1,\"x\")\n",
- "if (idx_time_zero+samples_after_second_pulse) < time_vector.shape[0]:\n",
- " plt.plot(time_vector[idx_time_zero+samples_after_second_pulse],1,\"x\")\n",
- "plt.plot(time_last_trial,1,\"x\")\n",
- "\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
- "\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
- "\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "print(\"new_iteration_channel\", new_iteration_channel)\n",
+ "print(digital_array[new_iteration_channel, :].shape)\n",
+ "\n",
+ "plt.plot(time_vector, iter_sample)\n",
+ "plt.plot(\n",
+ " time_vector,\n",
+ " digital_array[new_iteration_channel, last_iter_trial0:first_iter_trial1] + 0.02,\n",
+ ")\n",
+ "plt.plot(0, 1, \"x\")\n",
+ "if (idx_time_zero + samples_after_second_pulse) < time_vector.shape[0]:\n",
+ " plt.plot(time_vector[idx_time_zero + samples_after_second_pulse], 1, \"x\")\n",
+ "plt.plot(time_last_trial, 1, \"x\")\n",
+ "\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
+ "\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
+ "\n",
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -926,20 +1039,37 @@
"start_iter = 2704\n",
"samp_after = 600\n",
"\n",
- "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n",
- "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n",
- "\n",
- "\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
- "\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
- "\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 2,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n",
+ " + np.int64(samp_after),\n",
+ " ]\n",
+ ")\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 1,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n",
+ " + np.int64(samp_after),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
+ "\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
+ "\n",
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -958,8 +1088,8 @@
],
"source": [
"trial_plot = 280\n",
- "print(iteration_dict['iter_start_idx'][trial_plot][2704])\n",
- "print(iteration_dict['iter_start_idx'][trial_plot+1][0])"
+ "print(iteration_dict[\"iter_start_idx\"][trial_plot][2704])\n",
+ "print(iteration_dict[\"iter_start_idx\"][trial_plot + 1][0])"
]
},
{
@@ -991,7 +1121,7 @@
}
],
"source": [
- "plt.plot(np.diff(iteration_dict['iter_start_idx'][trial_plot]/nidq_sampling_rate))"
+ "plt.plot(np.diff(iteration_dict[\"iter_start_idx\"][trial_plot] / nidq_sampling_rate))"
]
},
{
@@ -1071,14 +1201,14 @@
"source": [
"x = np.array([])\n",
"mean_x = np.array([])\n",
- "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n",
- " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n",
+ "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n",
+ " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n",
" mean_time_trial = np.mean(s)\n",
- " x = np.append(x,s, axis=0)\n",
+ " x = np.append(x, s, axis=0)\n",
" mean_x = np.append(mean_x, mean_time_trial)\n",
"\n",
- "#plt.plot(x)\n",
- "plt.plot(mean_x)\n"
+ "# plt.plot(x)\n",
+ "plt.plot(mean_x)"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb
index 6a2f27a3..b1582c65 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -41,25 +42,15 @@
}
],
"source": [
- "import datetime\n",
- "import pathlib\n",
- "import numpy as np\n",
- "import pylab as pl\n",
- "import matplotlib\n",
- "import matplotlib.pyplot as plt\n",
- "from scipy.signal import find_peaks\n",
- "from scipy import signal as sp\n",
- "\n",
- "\n",
- "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory, get_ephys_root_data_dir\n",
"# import u19_pipeline.ephys_sync as ephys\n",
- "import u19_pipeline.acquisition as acquisition\n",
"import datajoint as dj\n",
- "\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
"from element_interface.utils import find_full_path\n",
"\n",
"import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n",
- "import u19_pipeline.utils.ephys_utils as ephys_utils\n"
+ "import u19_pipeline.utils.ephys_utils as ephys_utils\n",
+ "from u19_pipeline.ephys_pipeline import get_ephys_root_data_dir, get_session_directory"
]
},
{
@@ -68,8 +59,8 @@
"metadata": {},
"outputs": [],
"source": [
- "#Behavior data\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "# Behavior data\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -78,35 +69,48 @@
"metadata": {},
"outputs": [],
"source": [
- "def fix_trial_times(synced_iteration_vector, synced_time_vector, behavior_time_vector, nidq_sampling_rate):\n",
- "# Check where is more likely we miss an iteration pulse and insert it to iteration_vector\n",
+ "def fix_trial_times(\n",
+ " synced_iteration_vector,\n",
+ " synced_time_vector,\n",
+ " behavior_time_vector,\n",
+ " nidq_sampling_rate,\n",
+ "):\n",
+ " # Check where is more likely we miss an iteration pulse and insert it to iteration_vector\n",
"\n",
" new_synced_iteration_vector = synced_iteration_vector.copy()\n",
" new_synced_time_vector = synced_time_vector.copy()\n",
"\n",
" diff_time_test = new_synced_time_vector[2:50] - behavior_time_vector[1:49]\n",
" pulse_greater_than_virmen = np.where(diff_time_test > 0)\n",
- " pulse_greater_than_virmen = pulse_greater_than_virmen[0]+0\n",
+ " pulse_greater_than_virmen = pulse_greater_than_virmen[0] + 0\n",
" if pulse_greater_than_virmen.shape[0] == 0:\n",
- " #print('we have a special case of extra pulse beginning')\n",
+ " # print('we have a special case of extra pulse beginning')\n",
" new_synced_iteration_vector = np.delete(new_synced_iteration_vector, 1)\n",
- " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n",
+ " new_synced_time_vector = (\n",
+ " new_synced_iteration_vector - new_synced_iteration_vector[0]\n",
+ " ) / nidq_sampling_rate\n",
"\n",
" for i in range(100):\n",
- " diff_time = new_synced_time_vector[0:] - behavior_time_vector[0:new_synced_time_vector.shape[0]]\n",
+ " diff_time = (\n",
+ " new_synced_time_vector[0:]\n",
+ " - behavior_time_vector[0 : new_synced_time_vector.shape[0]]\n",
+ " )\n",
" pulse_greater_than_virmen = np.where(diff_time > 0)\n",
- " pulse_greater_than_virmen = pulse_greater_than_virmen[0]+0\n",
+ " pulse_greater_than_virmen = pulse_greater_than_virmen[0] + 0\n",
"\n",
" if pulse_greater_than_virmen.shape[0] == 0:\n",
" break\n",
"\n",
- " #print(i)\n",
- " #print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+3])\n",
- " #print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+3])\n",
- " #print(pulse_greater_than_virmen[0])\n",
+ " # print(i)\n",
+ " # print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+3])\n",
+ " # print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+3])\n",
+ " # print(pulse_greater_than_virmen[0])\n",
"\n",
" if pulse_greater_than_virmen[0] > 3:\n",
- " median_diff = np.median(new_synced_time_vector[0:pulse_greater_than_virmen[0]-3]-behavior_time_vector[0:pulse_greater_than_virmen[0]-3])\n",
+ " median_diff = np.median(\n",
+ " new_synced_time_vector[0 : pulse_greater_than_virmen[0] - 3]\n",
+ " - behavior_time_vector[0 : pulse_greater_than_virmen[0] - 3]\n",
+ " )\n",
" else:\n",
" median_diff = -0.015\n",
"\n",
@@ -114,69 +118,111 @@
" comp_value_l = -0.015\n",
" comp_value_g = -0.005\n",
" else:\n",
- " comp_value_l = median_diff*1.3\n",
- " comp_value_g = median_diff*0.7\n",
- "\n",
- " diff_time_iter_missing_pulse = behavior_time_vector[pulse_greater_than_virmen[0]] - behavior_time_vector[pulse_greater_than_virmen[0]-1]\n",
- " #print('missing time in iter pulses', behavior_time_vector[pulse_greater_than_virmen[0]])\n",
- " #print('time_next_missing_pulse', diff_time_iter_missing_pulse)\n",
- " diff_iter_add = np.floor(diff_time_iter_missing_pulse*nidq_sampling_rate)\n",
- " value_insert_iteration = new_synced_iteration_vector[pulse_greater_than_virmen[0]-1] + diff_iter_add\n",
- "\n",
- " #print('value_insert_iteration', value_insert_iteration)\n",
- "\n",
- " new_synced_iteration_vector = np.insert(new_synced_iteration_vector, pulse_greater_than_virmen[0], value_insert_iteration)\n",
+ " comp_value_l = median_diff * 1.3\n",
+ " comp_value_g = median_diff * 0.7\n",
+ "\n",
+ " diff_time_iter_missing_pulse = (\n",
+ " behavior_time_vector[pulse_greater_than_virmen[0]]\n",
+ " - behavior_time_vector[pulse_greater_than_virmen[0] - 1]\n",
+ " )\n",
+ " # print('missing time in iter pulses', behavior_time_vector[pulse_greater_than_virmen[0]])\n",
+ " # print('time_next_missing_pulse', diff_time_iter_missing_pulse)\n",
+ " diff_iter_add = np.floor(diff_time_iter_missing_pulse * nidq_sampling_rate)\n",
+ " value_insert_iteration = (\n",
+ " new_synced_iteration_vector[pulse_greater_than_virmen[0] - 1]\n",
+ " + diff_iter_add\n",
+ " )\n",
+ "\n",
+ " # print('value_insert_iteration', value_insert_iteration)\n",
+ "\n",
+ " new_synced_iteration_vector = np.insert(\n",
+ " new_synced_iteration_vector,\n",
+ " pulse_greater_than_virmen[0],\n",
+ " value_insert_iteration,\n",
+ " )\n",
"\n",
" if new_synced_iteration_vector.shape[0] > behavior_time_vector.shape[0]:\n",
- " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, pulse_greater_than_virmen[0]+1)\n",
- " #new_synced_iteration_vector = new_synced_iteration_vector[:-1]\n",
+ " new_synced_iteration_vector = np.delete(\n",
+ " new_synced_iteration_vector, pulse_greater_than_virmen[0] + 1\n",
+ " )\n",
+ " # new_synced_iteration_vector = new_synced_iteration_vector[:-1]\n",
"\n",
- " if pulse_greater_than_virmen[0] < (new_synced_time_vector.shape[0]-1):\n",
+ " if pulse_greater_than_virmen[0] < (new_synced_time_vector.shape[0] - 1):\n",
" idx = pulse_greater_than_virmen[0]\n",
- " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n",
- " if idx+10 > (new_synced_time_vector.shape[0]-1):\n",
- " idx_final = new_synced_time_vector.shape[0]-1\n",
- " diff_vectorin = (new_synced_time_vector[idx+1:idx_final]- behavior_time_vector[idx+1:idx_final])\n",
+ " new_synced_time_vector = (\n",
+ " new_synced_iteration_vector - new_synced_iteration_vector[0]\n",
+ " ) / nidq_sampling_rate\n",
+ " if idx + 10 > (new_synced_time_vector.shape[0] - 1):\n",
+ " idx_final = new_synced_time_vector.shape[0] - 1\n",
+ " diff_vectorin = (\n",
+ " new_synced_time_vector[idx + 1 : idx_final]\n",
+ " - behavior_time_vector[idx + 1 : idx_final]\n",
+ " )\n",
" else:\n",
- " diff_vectorin = (new_synced_time_vector[idx+1:idx+10]- behavior_time_vector[idx+1:idx+10])\n",
- " #print(diff_vectorin)\n",
+ " diff_vectorin = (\n",
+ " new_synced_time_vector[idx + 1 : idx + 10]\n",
+ " - behavior_time_vector[idx + 1 : idx + 10]\n",
+ " )\n",
+ " # print(diff_vectorin)\n",
" diff_inserted_iter = np.median(diff_vectorin)\n",
- " #print('diff_in new inserted iter', diff_inserted_iter)\n",
+ " # print('diff_in new inserted iter', diff_inserted_iter)\n",
" if diff_inserted_iter < comp_value_l:\n",
- " #print('extra diff !!!!')\n",
- " diff_behavior = behavior_time_vector[idx+1] - behavior_time_vector[idx]\n",
- " idx_next_iter = np.where(new_synced_time_vector > behavior_time_vector[idx+1])\n",
- " #print('diff_behavior', diff_behavior)\n",
+ " # print('extra diff !!!!')\n",
+ " diff_behavior = (\n",
+ " behavior_time_vector[idx + 1] - behavior_time_vector[idx]\n",
+ " )\n",
+ " idx_next_iter = np.where(\n",
+ " new_synced_time_vector > behavior_time_vector[idx + 1]\n",
+ " )\n",
+ " # print('diff_behavior', diff_behavior)\n",
" idx_next_iter = idx_next_iter[0][0]\n",
- " #print('idx_next_iter', idx_next_iter)\n",
- " #print('idx+1', idx+1)\n",
- " if diff_behavior > 0.1 and idx_next_iter > idx+3:\n",
- " #print('bajale a indx_next_iter', idx_next_iter)\n",
- " idx_next_iter = idx+3\n",
- " if idx_next_iter > idx+2:\n",
- " #print('more than one to remove')\n",
- " int_array_step = np.arange(idx+1, idx_next_iter-1, dtype=np.int64)\n",
- " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, int_array_step)\n",
+ " # print('idx_next_iter', idx_next_iter)\n",
+ " # print('idx+1', idx+1)\n",
+ " if diff_behavior > 0.1 and idx_next_iter > idx + 3:\n",
+ " # print('bajale a indx_next_iter', idx_next_iter)\n",
+ " idx_next_iter = idx + 3\n",
+ " if idx_next_iter > idx + 2:\n",
+ " # print('more than one to remove')\n",
+ " int_array_step = np.arange(\n",
+ " idx + 1, idx_next_iter - 1, dtype=np.int64\n",
+ " )\n",
+ " new_synced_iteration_vector = np.delete(\n",
+ " new_synced_iteration_vector, int_array_step\n",
+ " )\n",
" else:\n",
- " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, [idx+1])\n",
- " #print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
+ " new_synced_iteration_vector = np.delete(\n",
+ " new_synced_iteration_vector, [idx + 1]\n",
+ " )\n",
+ " # print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
" elif diff_inserted_iter > comp_value_g and diff_inserted_iter < 0:\n",
" if new_synced_iteration_vector.shape[0] < behavior_time_vector.shape[0]:\n",
- " #print('less diff now, insert one iteration')\n",
- " #print('diff_inserted_iter', diff_inserted_iter)\n",
- " #print('median_diff', median_diff, 'comp_value_g', comp_value_g)\n",
- " #print('diff_vectorin', diff_vectorin)\n",
- " value_insert = int((new_synced_iteration_vector[pulse_greater_than_virmen[0]]+new_synced_iteration_vector[pulse_greater_than_virmen[0]+1])/2) \n",
- " new_synced_iteration_vector = np.insert(new_synced_iteration_vector, pulse_greater_than_virmen[0]+1, value_insert)\n",
- "\n",
- " \n",
- " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n",
- "\n",
- " #print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
- " #print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
- "\n",
- " return new_synced_iteration_vector, new_synced_time_vector\n",
- "\n"
+ " # print('less diff now, insert one iteration')\n",
+ " # print('diff_inserted_iter', diff_inserted_iter)\n",
+ " # print('median_diff', median_diff, 'comp_value_g', comp_value_g)\n",
+ " # print('diff_vectorin', diff_vectorin)\n",
+ " value_insert = int(\n",
+ " (\n",
+ " new_synced_iteration_vector[pulse_greater_than_virmen[0]]\n",
+ " + new_synced_iteration_vector[\n",
+ " pulse_greater_than_virmen[0] + 1\n",
+ " ]\n",
+ " )\n",
+ " / 2\n",
+ " )\n",
+ " new_synced_iteration_vector = np.insert(\n",
+ " new_synced_iteration_vector,\n",
+ " pulse_greater_than_virmen[0] + 1,\n",
+ " value_insert,\n",
+ " )\n",
+ "\n",
+ " new_synced_time_vector = (\n",
+ " new_synced_iteration_vector - new_synced_iteration_vector[0]\n",
+ " ) / nidq_sampling_rate\n",
+ "\n",
+ " # print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
+ " # print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n",
+ "\n",
+ " return new_synced_iteration_vector, new_synced_time_vector"
]
},
{
@@ -194,15 +240,20 @@
" max_diff = max(diff_vector)\n",
" median_general = np.median(diff_vector)\n",
"\n",
- " num_div= 10\n",
+ " num_div = 10\n",
" median_diff_percent = np.empty([num_div])\n",
" median_diff_abs = np.empty([num_div])\n",
" for j in range(num_div):\n",
- " start_iter = int(j*num_iter/num_div)\n",
- " end_iter = int((j+1)*num_iter/num_div)\n",
- " median_diff_percent[j] = (np.median(diff_vector[start_iter:end_iter])-median_general)*100/median_general\n",
- " median_diff_abs[j] = (np.median(diff_vector[start_iter:end_iter])-median_general)\n",
- "\n",
+ " start_iter = int(j * num_iter / num_div)\n",
+ " end_iter = int((j + 1) * num_iter / num_div)\n",
+ " median_diff_percent[j] = (\n",
+ " (np.median(diff_vector[start_iter:end_iter]) - median_general)\n",
+ " * 100\n",
+ " / median_general\n",
+ " )\n",
+ " median_diff_abs[j] = (\n",
+ " np.median(diff_vector[start_iter:end_iter]) - median_general\n",
+ " )\n",
"\n",
" if max_diff <= 0 and np.max(np.abs(median_diff_abs)) < 0.005:\n",
" pass\n",
@@ -211,9 +262,7 @@
" print(median_general)\n",
" plt.plot(median_diff_abs)\n",
"\n",
- " return status\n",
- " \n",
- "\n"
+ " return status"
]
},
{
@@ -241,12 +290,16 @@
],
"source": [
"recording_query = \"recording_id >= 500\"\n",
- "recording = dj.create_virtual_module('recording', 'u19_recording')\n",
- "recording_keys = (recording.Recording & recording_query).fetch('recording_id', as_dict=True, order_by='recording_id')\n",
+ "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n",
+ "recording_keys = (recording.Recording & recording_query).fetch(\n",
+ " \"recording_id\", as_dict=True, order_by=\"recording_id\"\n",
+ ")\n",
"\n",
"\n",
- "session_fields = ['subject_fullname', 'session_date', 'session_number']\n",
- "session_keys = (recording.Recording.BehaviorSession & recording_query).fetch(*session_fields, as_dict=True, order_by='recording_id')\n",
+ "session_fields = [\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ "session_keys = (recording.Recording.BehaviorSession & recording_query).fetch(\n",
+ " *session_fields, as_dict=True, order_by=\"recording_id\"\n",
+ ")\n",
"\n",
"\n",
"recording_keys"
@@ -311,38 +364,42 @@
" recording_key = recording_keys[idx_session]\n",
" session_key = session_keys[idx_session]\n",
"\n",
- " session_dir = find_full_path(get_ephys_root_data_dir(),\n",
- " get_session_directory(recording_key))\n",
+ " session_dir = find_full_path(\n",
+ " get_ephys_root_data_dir(), get_session_directory(recording_key)\n",
+ " )\n",
" print(session_dir)\n",
- " #session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
- " #Check if session is Nidq or OneBox\n",
- " nidq_session = list(session_dir.glob('*nidq.bin*'))\n",
- " obx_session = list(session_dir.glob('*obx.bin*'))\n",
+ " # session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
+ " # Check if session is Nidq or OneBox\n",
+ " nidq_session = list(session_dir.glob(\"*nidq.bin*\"))\n",
+ " obx_session = list(session_dir.glob(\"*obx.bin*\"))\n",
"\n",
" if len(nidq_session) == 0 and len(obx_session) == 0:\n",
- " print('No session found')\n",
+ " print(\"No session found\")\n",
" elif len(nidq_session) > 0:\n",
" ephys_session_fullpath = nidq_session[0]\n",
" else:\n",
" ephys_session_fullpath = obx_session[0]\n",
"\n",
- " #Nidaq file\n",
- " nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n",
+ " # Nidaq file\n",
+ " nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n",
" nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n",
"\n",
- "\n",
- " # 1: load meta data, and the content of the NIDAQ file. Its content is digital. \n",
+ " # 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n",
" new_trial_channel = 1\n",
" new_iteration_channel = 2\n",
" # If PXIe card (nidq) card use for recording deduce digital channels\n",
- " if nidq_meta['typeThis'] == 'nidq':\n",
- " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)\n",
+ " if nidq_meta[\"typeThis\"] == \"nidq\":\n",
+ " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " ephys_session_fullpath, nidq_meta\n",
+ " )\n",
" # If onebox card (obx) card use for recording digital channels are 0-2\n",
" else:\n",
- " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])\n",
+ " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]\n",
+ " )\n",
" # If no sync pulse found trial and iteration signals are 0 & 1 respectively\n",
- " channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]\n",
- " channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]\n",
+ " channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]\n",
+ " channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]\n",
"\n",
" if channel0_pulses > channel1_pulses:\n",
" new_trial_channel = 1\n",
@@ -351,47 +408,65 @@
" new_trial_channel = 0\n",
" new_iteration_channel = 1\n",
"\n",
- " \n",
" thissession = behavior.TowersBlock().Trial() & session_key\n",
- " behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n",
- "\n",
- " mode = None #Default for sessions before 12/01/2021\n",
- " #mode = 'pulses' #Default for sessions after 12/01/2021\n",
- " iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[new_trial_channel,:], digital_array[new_iteration_channel,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n",
+ " behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n",
+ " \"trial_time\", \"vi_start\", \"iterations\"\n",
+ " )\n",
+ "\n",
+ " mode = None # Default for sessions before 12/01/2021\n",
+ " # mode = 'pulses' #Default for sessions after 12/01/2021\n",
+ " iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n",
+ " digital_array[new_trial_channel, :],\n",
+ " digital_array[new_iteration_channel, :],\n",
+ " nidq_sampling_rate,\n",
+ " behavior_time.shape[0],\n",
+ " behavior_time,\n",
+ " mode=mode,\n",
+ " )\n",
" # get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n",
"\n",
- "\n",
- " for i in range(len(iteration_dict['iter_start_idx'])):\n",
- " #print('fixing trial ',i)\n",
- " iteration_dict['iter_start_idx'][i],iteration_dict['iter_times_idx'][i] =\\\n",
- " fix_trial_times(iteration_dict['iter_start_idx'][i], iteration_dict['iter_times_idx'][i], behavior_time[i].flatten(), nidq_sampling_rate)\n",
- "\n",
+ " for i in range(len(iteration_dict[\"iter_start_idx\"])):\n",
+ " # print('fixing trial ',i)\n",
+ " iteration_dict[\"iter_start_idx\"][i], iteration_dict[\"iter_times_idx\"][i] = (\n",
+ " fix_trial_times(\n",
+ " iteration_dict[\"iter_start_idx\"][i],\n",
+ " iteration_dict[\"iter_times_idx\"][i],\n",
+ " behavior_time[i].flatten(),\n",
+ " nidq_sampling_rate,\n",
+ " )\n",
+ " )\n",
"\n",
" # Check # of trials and iterations match\n",
- " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n",
- "\n",
+ " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n",
+ " ephys_utils.assert_iteration_samples_count(\n",
+ " iteration_dict[\"iter_start_idx\"], behavior_time\n",
+ " )\n",
+ " )\n",
"\n",
" print(trial_count_diff)\n",
" print(trials_diff_iteration_big)\n",
" print(trials_diff_iteration_small)\n",
"\n",
+ " status = ephys_utils.evaluate_sync_process(\n",
+ " trial_count_diff,\n",
+ " trials_diff_iteration_big,\n",
+ " trials_diff_iteration_small,\n",
+ " behavior_time.shape[0],\n",
+ " )\n",
"\n",
- " status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n",
- "\n",
- " for i in range(len(iteration_dict['iter_start_idx'])):\n",
- " synced_time_vector = iteration_dict['iter_times_idx'][i]\n",
+ " for i in range(len(iteration_dict[\"iter_start_idx\"])):\n",
+ " synced_time_vector = iteration_dict[\"iter_times_idx\"][i]\n",
" behavior_time_vector = behavior_time[i].flatten()\n",
" status = sync_evaluation_process2(synced_time_vector, behavior_time_vector)\n",
" if status == 0:\n",
" print(session_key)\n",
" print(recording_key)\n",
" print(i)\n",
- " if i!=0:\n",
+ " if i != 0:\n",
" break\n",
- " \n",
+ "\n",
" if status == 0:\n",
- " break\n",
- " \n"
+ " break"
]
},
{
@@ -434,11 +509,12 @@
],
"source": [
"if it_times.shape[0] >= beh_times.shape[0]:\n",
- " diff_vector = np.diff(it_times[:beh_times.shape[0]] - beh_times)\n",
+ " diff_vector = np.diff(it_times[: beh_times.shape[0]] - beh_times)\n",
"else:\n",
- " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n",
+ " diff_vector = np.diff(it_times - beh_times[: it_times.shape[0]])\n",
"\n",
"import scipy\n",
+ "\n",
"peaks, _ = scipy.signal.find_peaks(diff_vector, height=0.05, distance=20)\n",
"peaks"
]
@@ -460,12 +536,12 @@
}
],
"source": [
- "#plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n",
- "plt.plot((it_times - beh_times[:it_times.shape[0]]))\n",
- "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
+ "# plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n",
+ "plt.plot(it_times - beh_times[: it_times.shape[0]])\n",
+ "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
"plt.plot(peaks, diff_vector[peaks], \"x\")\n",
- "plt.ylabel('Time diff in current iteration TTL - Virmen (s)')\n",
- "plt.xlabel('Iteration')\n",
+ "plt.ylabel(\"Time diff in current iteration TTL - Virmen (s)\")\n",
+ "plt.xlabel(\"Iteration\")\n",
"plt.ylim([-0.05, 0.05])\n",
"plt.show()"
]
@@ -498,11 +574,11 @@
],
"source": [
"plt.plot(np.diff(synced_time_vector))\n",
- "plt.plot(np.diff(beh_times)+0.01)\n",
- "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
- "#plt.plot(peaks, diff_vector[peaks], \"x\")\n",
+ "plt.plot(np.diff(beh_times) + 0.01)\n",
+ "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n",
+ "# plt.plot(peaks, diff_vector[peaks], \"x\")\n",
"plt.ylim([-0.01, 0.05])\n",
- "plt.xlabel('Iteration')\n"
+ "plt.xlabel(\"Iteration\")"
]
},
{
@@ -520,8 +596,8 @@
}
],
"source": [
- "difo = np.pad(np.diff(synced_time_vector), (1, 0), 'constant', constant_values=(0,0))\n",
- "difo2 = np.pad(np.diff(beh_times), (1, 0), 'constant', constant_values=(0,0))\n",
+ "difo = np.pad(np.diff(synced_time_vector), (1, 0), \"constant\", constant_values=(0, 0))\n",
+ "difo2 = np.pad(np.diff(beh_times), (1, 0), \"constant\", constant_values=(0, 0))\n",
"\n",
"peaks_ttl_pulses, _ = scipy.signal.find_peaks(difo, height=0.02, distance=20)\n",
"peaks_virmen_iter, _ = scipy.signal.find_peaks(difo2, height=0.02, distance=20)\n",
@@ -566,8 +642,8 @@
}
],
"source": [
- "print(new_synced_time_vector[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n",
- "print(beh_times[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n"
+ "print(new_synced_time_vector[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])\n",
+ "print(beh_times[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])"
]
},
{
@@ -586,7 +662,7 @@
],
"source": [
"print(new_synced_time_vector[-4:])\n",
- "print(beh_times[-4:])\n"
+ "print(beh_times[-4:])"
]
},
{
@@ -616,10 +692,9 @@
}
],
"source": [
- "\n",
"plt.plot(np.diff(new_synced_time_vector))\n",
- "#plt.plot(np.diff(synced_time_vector))\n",
- "plt.plot(np.diff(behavior_time_vector)+0.01)\n",
+ "# plt.plot(np.diff(synced_time_vector))\n",
+ "plt.plot(np.diff(behavior_time_vector) + 0.01)\n",
"plt.ylim(0, 0.11)"
]
},
@@ -670,8 +745,8 @@
],
"source": [
"trial_plot = 99\n",
- "plt.plot((iteration_dict['iter_times_idx'][trial_plot]))\n",
- "plt.plot((behavior_time[trial_plot].flatten()))\n"
+ "plt.plot(iteration_dict[\"iter_times_idx\"][trial_plot])\n",
+ "plt.plot(behavior_time[trial_plot].flatten())"
]
},
{
@@ -694,11 +769,18 @@
"trial_plot = 117\n",
"samp_before = 800\n",
"samp_after = 800\n",
- "last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot-1][-1]-np.int64(samp_before)\n",
- "first_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][0]+np.int64(samp_after)\n",
- "second_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][0]\n",
- "\n",
- "samp_diff = iteration_dict['iter_start_idx'][trial_plot][0] - iteration_dict['iter_start_idx'][trial_plot-1][-1]\n",
+ "last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1] - np.int64(\n",
+ " samp_before\n",
+ ")\n",
+ "first_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][0] + np.int64(\n",
+ " samp_after\n",
+ ")\n",
+ "second_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ "\n",
+ "samp_diff = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1]\n",
+ ")\n",
"samp_diff"
]
},
@@ -740,53 +822,60 @@
}
],
"source": [
- "\n",
- "\n",
"start_iter = 1\n",
"end_iter = 1\n",
"\n",
"\n",
- "sample_start =(iteration_dict['iter_start_idx'][trial_plot][start_iter]-np.int64(samp_before))\n",
- "sample_end = (iteration_dict['iter_start_idx'][trial_plot][end_iter]+np.int64(samp_after))\n",
- "\n",
- "iter_sample = digital_array[new_trial_channel,last_iter_trial0:first_iter_trial1]\n",
+ "sample_start = iteration_dict[\"iter_start_idx\"][trial_plot][start_iter] - np.int64(\n",
+ " samp_before\n",
+ ")\n",
+ "sample_end = iteration_dict[\"iter_start_idx\"][trial_plot][end_iter] + np.int64(\n",
+ " samp_after\n",
+ ")\n",
"\n",
- "time_vector = np.linspace(start=0, stop=iter_sample.shape[0]-1,num=iter_sample.shape[0])\n",
- "time_vector = (time_vector*1000/nidq_sampling_rate)\n",
- "time_vector -= ((iter_sample.shape[0]-samp_after)*1000)/nidq_sampling_rate\n",
+ "iter_sample = digital_array[new_trial_channel, last_iter_trial0:first_iter_trial1]\n",
"\n",
- "time_last_trial = time_vector[0] +(samp_before*1000)/nidq_sampling_rate\n",
+ "time_vector = np.linspace(\n",
+ " start=0, stop=iter_sample.shape[0] - 1, num=iter_sample.shape[0]\n",
+ ")\n",
+ "time_vector = time_vector * 1000 / nidq_sampling_rate\n",
+ "time_vector -= ((iter_sample.shape[0] - samp_after) * 1000) / nidq_sampling_rate\n",
"\n",
+ "time_last_trial = time_vector[0] + (samp_before * 1000) / nidq_sampling_rate\n",
"\n",
"\n",
"print(iter_sample.shape[0])\n",
- "samples_after_second_pulse = (second_iter_trial1-(first_iter_trial1-samp_after))\n",
- "print('samples_after_second_pulse', samples_after_second_pulse)\n",
- "print(iter_sample.shape[0]-samp_after)\n",
+ "samples_after_second_pulse = second_iter_trial1 - (first_iter_trial1 - samp_after)\n",
+ "print(\"samples_after_second_pulse\", samples_after_second_pulse)\n",
+ "print(iter_sample.shape[0] - samp_after)\n",
"\n",
- "idx_time_zero = np.where((time_vector >= 0))\n",
+ "idx_time_zero = np.where(time_vector >= 0)\n",
"idx_time_zero = idx_time_zero[0]\n",
"idx_time_zero = idx_time_zero[0]\n",
"\n",
"\n",
- "\n",
- "\n",
- "plt.plot(time_vector,iter_sample)\n",
- "plt.plot(time_vector,digital_array[new_iteration_channel,last_iter_trial0:first_iter_trial1]+0.02)\n",
- "plt.plot(0,1,\"x\")\n",
- "plt.plot(time_vector[idx_time_zero+samples_after_second_pulse],1,\"x\")\n",
- "plt.plot(time_last_trial,1,\"x\")\n",
- "\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
- "\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
- "\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "plt.plot(time_vector, iter_sample)\n",
+ "plt.plot(\n",
+ " time_vector,\n",
+ " digital_array[new_iteration_channel, last_iter_trial0:first_iter_trial1] + 0.02,\n",
+ ")\n",
+ "plt.plot(0, 1, \"x\")\n",
+ "plt.plot(time_vector[idx_time_zero + samples_after_second_pulse], 1, \"x\")\n",
+ "plt.plot(time_last_trial, 1, \"x\")\n",
+ "\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
+ "\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
+ "\n",
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -812,20 +901,37 @@
"start_iter = 2704\n",
"samp_after = 600\n",
"\n",
- "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n",
- "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n",
- "\n",
- "\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
- "\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
- "\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 2,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n",
+ " + np.int64(samp_after),\n",
+ " ]\n",
+ ")\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 1,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n",
+ " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n",
+ " + np.int64(samp_after),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
+ "\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
+ "\n",
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -844,8 +950,8 @@
],
"source": [
"trial_plot = 280\n",
- "print(iteration_dict['iter_start_idx'][trial_plot][2704])\n",
- "print(iteration_dict['iter_start_idx'][trial_plot+1][0])"
+ "print(iteration_dict[\"iter_start_idx\"][trial_plot][2704])\n",
+ "print(iteration_dict[\"iter_start_idx\"][trial_plot + 1][0])"
]
},
{
@@ -877,7 +983,7 @@
}
],
"source": [
- "plt.plot(np.diff(iteration_dict['iter_start_idx'][trial_plot]/nidq_sampling_rate))"
+ "plt.plot(np.diff(iteration_dict[\"iter_start_idx\"][trial_plot] / nidq_sampling_rate))"
]
},
{
@@ -957,14 +1063,14 @@
"source": [
"x = np.array([])\n",
"mean_x = np.array([])\n",
- "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n",
- " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n",
+ "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n",
+ " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n",
" mean_time_trial = np.mean(s)\n",
- " x = np.append(x,s, axis=0)\n",
+ " x = np.append(x, s, axis=0)\n",
" mean_x = np.append(mean_x, mean_time_trial)\n",
"\n",
- "#plt.plot(x)\n",
- "plt.plot(mean_x)\n"
+ "# plt.plot(x)\n",
+ "plt.plot(mean_x)"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb
index 4255e2ab..452c07e9 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -39,11 +40,10 @@
}
],
"source": [
- "from u19_pipeline import ephys\n",
- "#from u19_pipeline import ephys_pipeline as ephys\n",
- "from u19_pipeline import recording\n",
+ "import pandas as pd\n",
"\n",
- "import pandas as pd"
+ "# from u19_pipeline import ephys_pipeline as ephys\n",
+ "from u19_pipeline import ephys"
]
},
{
@@ -91,21 +91,34 @@
"source": [
"key = \"subject_fullname like 'sbolkan%'\"\n",
"\n",
- "#keys_session = pd.DataFrame((ephys.EphysPipelineSession * recording.Recording.BehaviorSession & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n",
- "keys_session = pd.DataFrame((ephys.EphysSession & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n",
+ "# keys_session = pd.DataFrame((ephys.EphysPipelineSession * recording.Recording.BehaviorSession & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n",
+ "keys_session = pd.DataFrame(\n",
+ " (ephys.EphysSession & key).fetch(\"KEY\", as_dict=True, order_by=\"session_date DESC\")\n",
+ ")\n",
"\n",
- "#already_synced = pd.DataFrame((ephys.EphysPipelineSession * recording.Recording.BehaviorSession * ephys.BehaviorSync & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n",
- "already_synced = pd.DataFrame((ephys.EphysSession * ephys.BehaviorSync & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n",
+ "# already_synced = pd.DataFrame((ephys.EphysPipelineSession * recording.Recording.BehaviorSession * ephys.BehaviorSync & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n",
+ "already_synced = pd.DataFrame(\n",
+ " (ephys.EphysSession * ephys.BehaviorSync & key).fetch(\n",
+ " \"KEY\", as_dict=True, order_by=\"session_date DESC\"\n",
+ " )\n",
+ ")\n",
"\n",
"\n",
- "keys_session_missing = keys_session.merge(already_synced, how='left', on=['subject_fullname', 'session_date', 'session_number'], indicator=True)\n",
- "#keys_session_missing = keys_session.merge(already_synced, how='left', on=['recording_id'], indicator=True)\n",
+ "keys_session_missing = keys_session.merge(\n",
+ " already_synced,\n",
+ " how=\"left\",\n",
+ " on=[\"subject_fullname\", \"session_date\", \"session_number\"],\n",
+ " indicator=True,\n",
+ ")\n",
+ "# keys_session_missing = keys_session.merge(already_synced, how='left', on=['recording_id'], indicator=True)\n",
"\n",
"\n",
- "keys_session_missing = keys_session_missing.loc[keys_session_missing['_merge'] == 'left_only', :]\n",
+ "keys_session_missing = keys_session_missing.loc[\n",
+ " keys_session_missing[\"_merge\"] == \"left_only\", :\n",
+ "]\n",
"keys_session_missing = keys_session_missing.reset_index(drop=True)\n",
- "keys_session_missing = keys_session_missing.drop(['_merge'], axis=1)\n",
- "keys_session_missing = keys_session_missing.to_dict(orient='records')\n",
+ "keys_session_missing = keys_session_missing.drop([\"_merge\"], axis=1)\n",
+ "keys_session_missing = keys_session_missing.to_dict(orient=\"records\")\n",
"keys_session_missing"
]
},
@@ -1301,8 +1314,8 @@
],
"source": [
"key = dict()\n",
- "key['session_date'] = '2022-08-18'\n",
- "key['subject_fullname'] = 'sbolkan_a2a_487' \n",
+ "key[\"session_date\"] = \"2022-08-18\"\n",
+ "key[\"subject_fullname\"] = \"sbolkan_a2a_487\"\n",
"\n",
"ephys.BehaviorSync.populate(key)"
]
@@ -1475,18 +1488,22 @@
"source": [
"session_dir = pathlib.Path(get_session_directory(key))\n",
"print(session_dir)\n",
- "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
- "nidq_bin_full_path = list(session_dir.glob('*nidq.bin*'))[0]\n",
+ "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n",
+ "nidq_bin_full_path = list(session_dir.glob(\"*nidq.bin*\"))[0]\n",
"\n",
- "#Nidaq file\n",
- "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n",
+ "# Nidaq file\n",
+ "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n",
"nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n",
- "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta)\n",
+ "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n",
+ " nidq_bin_full_path, nidq_meta\n",
+ ")\n",
"\n",
- "#Behavior data\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n",
+ "# Behavior data\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
"thissession = behavior.TowersBlock().Trial() & key\n",
- "behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n"
+ "behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n",
+ " \"trial_time\", \"vi_start\", \"iterations\"\n",
+ ")"
]
},
{
@@ -1518,7 +1535,7 @@
}
],
"source": [
- "plt.plot(digital_array[1,1292075-10000:1292075+1000])"
+ "plt.plot(digital_array[1, 1292075 - 10000 : 1292075 + 1000])"
]
},
{
@@ -1550,8 +1567,8 @@
}
],
"source": [
- "plt.plot(digital_array[1,1292075-1000:1292075+1000])\n",
- "plt.plot(digital_array[2,1292075-1000:1292075+1000])"
+ "plt.plot(digital_array[1, 1292075 - 1000 : 1292075 + 1000])\n",
+ "plt.plot(digital_array[2, 1292075 - 1000 : 1292075 + 1000])"
]
},
{
@@ -16175,15 +16192,24 @@
}
],
"source": [
- "mode = 'counter_bit0' #Default for sessions before 12/01/2021\n",
- "#mode = 'pulses' #Default for sessions after 12/01/2021\n",
- "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[1,:], digital_array[2,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n",
+ "mode = \"counter_bit0\" # Default for sessions before 12/01/2021\n",
+ "# mode = 'pulses' #Default for sessions after 12/01/2021\n",
+ "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n",
+ " digital_array[1, :],\n",
+ " digital_array[2, :],\n",
+ " nidq_sampling_rate,\n",
+ " behavior_time.shape[0],\n",
+ " behavior_time,\n",
+ " mode=mode,\n",
+ ")\n",
"# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n",
"\n",
"print(iteration_dict)\n",
"\n",
- "status = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n",
- "status\n"
+ "status = ephys_utils.assert_iteration_samples_count(\n",
+ " iteration_dict[\"iter_start_idx\"], behavior_time\n",
+ ")\n",
+ "status"
]
},
{
@@ -16202,9 +16228,9 @@
],
"source": [
"print(iteration_dict.keys())\n",
- "so = np.where(iteration_dict['trialnumber_vector_samples'] == 12)\n",
+ "so = np.where(iteration_dict[\"trialnumber_vector_samples\"] == 12)\n",
"\n",
- "print(so[0])\n"
+ "print(so[0])"
]
},
{
@@ -16249,20 +16275,39 @@
"end_iter = 5\n",
"samp_after = 30\n",
"\n",
- "plt.plot(digital_array[0,iteration_dict['iter_start_idx'][trial_plot][0]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n",
- "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][0]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 0,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][0] - samp_before : iteration_dict[\n",
+ " \"iter_start_idx\"\n",
+ " ][trial_plot][end_iter],\n",
+ " ]\n",
+ " + samp_after\n",
+ ")\n",
+ "plt.plot(\n",
+ " digital_array[\n",
+ " 1,\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][0] - samp_before : iteration_dict[\n",
+ " \"iter_start_idx\"\n",
+ " ][trial_plot][end_iter],\n",
+ " ]\n",
+ " + samp_after\n",
+ ")\n",
"\n",
"\n",
- "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n",
- "time = iter_nidaq/nidq_sampling_rate\n",
- "iter_virmen = time*120\n",
+ "iter_nidaq = (\n",
+ " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n",
+ " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n",
+ ")\n",
+ "time = iter_nidaq / nidq_sampling_rate\n",
+ "iter_virmen = time * 120\n",
"\n",
- "print('time from niDAQ', time)\n",
- "print('time behavior', behavior_time[trial_plot][-1])\n",
+ "print(\"time from niDAQ\", time)\n",
+ "print(\"time behavior\", behavior_time[trial_plot][-1])\n",
"\n",
- "print('samples nidaq', iter_nidaq)\n",
- "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n",
- "print('iter_virmen',behavior_time[trial_plot].shape)"
+ "print(\"samples nidaq\", iter_nidaq)\n",
+ "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n",
+ "print(\"iter_virmen\", behavior_time[trial_plot].shape)"
]
},
{
@@ -16303,14 +16348,14 @@
"source": [
"x = np.array([])\n",
"mean_x = np.array([])\n",
- "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n",
- " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n",
+ "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n",
+ " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n",
" mean_time_trial = np.mean(s)\n",
- " x = np.append(x,s, axis=0)\n",
+ " x = np.append(x, s, axis=0)\n",
" mean_x = np.append(mean_x, mean_time_trial)\n",
"\n",
- "#plt.plot(x)\n",
- "plt.plot(mean_x)\n"
+ "# plt.plot(x)\n",
+ "plt.plot(mean_x)"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb
index e88eab83..c03cf8bc 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -34,7 +35,8 @@
],
"source": [
"import numpy as np\n",
- "import u19_pipeline.ephys_pipeline as ep\n"
+ "\n",
+ "import u19_pipeline.ephys_pipeline as ep"
]
},
{
@@ -43,11 +45,11 @@
"metadata": {},
"outputs": [],
"source": [
- "recs2 = (ep.BehaviorSync).fetch('recording_id', as_dict=True)\n",
+ "recs2 = (ep.BehaviorSync).fetch(\"recording_id\", as_dict=True)\n",
"bad_Sessions = list()\n",
"for i in recs2:\n",
" s = (ep.BehaviorSync & i).fetch(as_dict=True)\n",
- " if np.all(np.isnan(s[0]['trial_index_nidq'])):\n",
+ " if np.all(np.isnan(s[0][\"trial_index_nidq\"])):\n",
" bad_Sessions.append(i)"
]
},
@@ -57,7 +59,7 @@
"metadata": {},
"outputs": [],
"source": [
- "recs2 = (ep.BehaviorSync).fetch('recording_id', as_dict=True)"
+ "recs2 = (ep.BehaviorSync).fetch(\"recording_id\", as_dict=True)"
]
},
{
@@ -76,9 +78,9 @@
"outputs": [],
"source": [
"missing_recs = []\n",
- "for i in range(450,457):\n",
+ "for i in range(450, 457):\n",
" dicto = {}\n",
- " dicto['recording_id'] = i\n",
+ " dicto[\"recording_id\"] = i\n",
" missing_recs.append(dicto)"
]
},
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb
index 9fe654e7..0c745c04 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -35,7 +36,8 @@
],
"source": [
"import numpy as np\n",
- "import u19_pipeline.ephys_pipeline as ep\n"
+ "\n",
+ "import u19_pipeline.ephys_pipeline as ep"
]
},
{
@@ -44,11 +46,11 @@
"metadata": {},
"outputs": [],
"source": [
- "recs2 = (ep.BehaviorSync & \"recording_id = 482\").fetch('recording_id', as_dict=True)\n",
+ "recs2 = (ep.BehaviorSync & \"recording_id = 482\").fetch(\"recording_id\", as_dict=True)\n",
"bad_Sessions = list()\n",
"for i in recs2:\n",
" s = (ep.BehaviorSync & i).fetch(as_dict=True)\n",
- " if np.all(np.isnan(s[0]['trial_index_nidq'])):\n",
+ " if np.all(np.isnan(s[0][\"trial_index_nidq\"])):\n",
" bad_Sessions.append(i)"
]
},
@@ -58,8 +60,7 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
- "bad_Sessions = {'recording_id = 205'}"
+ "bad_Sessions = {\"recording_id = 205\"}"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb
index 67316156..994afb8a 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -33,17 +34,12 @@
],
"source": [
"import datajoint as dj\n",
- "import pandas as pd\n",
- "import numpy as np\n",
"import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
"\n",
"import u19_pipeline.ephys_pipeline as ep\n",
"import u19_pipeline.utils.ephys_utils as eu\n",
- "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n",
- "\n",
- "\n",
- "from u19_pipeline import recording\n",
- "\n"
+ "from u19_pipeline import recording"
]
},
{
@@ -52,8 +48,8 @@
"metadata": {},
"outputs": [],
"source": [
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n",
- "ephys_pipeline = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')"
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
+ "ephys_pipeline = dj.create_virtual_module(\"ephys_pipeline\", \"u19_ephys_pipeline\")"
]
},
{
@@ -62,7 +58,9 @@
"metadata": {},
"outputs": [],
"source": [
- "recording_synced = (ep.BehaviorSync & \"sync_data is null\").fetch('KEY', order_by='recording_id DESC')"
+ "recording_synced = (ep.BehaviorSync & \"sync_data is null\").fetch(\n",
+ " \"KEY\", order_by=\"recording_id DESC\"\n",
+ ")"
]
},
{
@@ -127,7 +125,7 @@
"metadata": {},
"outputs": [],
"source": [
- "recording_synced = [recording_synced[0]]\n"
+ "recording_synced = [recording_synced[0]]"
]
},
{
@@ -174,105 +172,109 @@
],
"source": [
"for rec_key in recording_synced:\n",
- "\n",
- " print('rec_key', rec_key)\n",
+ " print(\"rec_key\", rec_key)\n",
" behavior_key = (recording.Recording.BehaviorSession & rec_key).fetch1()\n",
"\n",
" full_session_path = ep.get_full_session_directory(rec_key)\n",
- " \n",
- " print('full_session_path', full_session_path)\n",
+ "\n",
+ " print(\"full_session_path\", full_session_path)\n",
"\n",
" if isinstance(full_session_path, str) and len(full_session_path) == 0:\n",
" continue\n",
"\n",
" nidq_meta, nidq_sampling_rate = eu.read_nidq_meta_samp_rate(full_session_path)\n",
"\n",
- " trial_pulses_signal, _ = eu.load_trial_iteration_signals(full_session_path, nidq_meta)\n",
+ " trial_pulses_signal, _ = eu.load_trial_iteration_signals(\n",
+ " full_session_path, nidq_meta\n",
+ " )\n",
"\n",
" try:\n",
" all_sync_data = (ep.BehaviorSync & rec_key).fetch1()\n",
" except:\n",
- " print('No sync data was sound for this session')\n",
+ " print(\"No sync data was sound for this session\")\n",
" continue\n",
"\n",
- " if all_sync_data['sync_data'] != None:\n",
- " print('sync_data is already inserted')\n",
+ " if all_sync_data[\"sync_data\"] != None:\n",
+ " print(\"sync_data is already inserted\")\n",
" pass\n",
- " #continue \n",
+ " # continue\n",
+ "\n",
+ " behavior_key.pop(\"recording_id\")\n",
"\n",
- " behavior_key.pop('recording_id')\n",
- " \n",
" thissession = behavior.TowersBlock().Trial() & behavior_key\n",
- " behavior_time, iterstart = thissession.fetch('trial_time', 'vi_start')\n",
+ " behavior_time, iterstart = thissession.fetch(\"trial_time\", \"vi_start\")\n",
"\n",
- " if np.where(np.isnan(all_sync_data['trial_index_nidq']))[0].shape[0] == all_sync_data['trial_index_nidq'].shape[0]:\n",
- " print('All nan session')\n",
+ " if (\n",
+ " np.where(np.isnan(all_sync_data[\"trial_index_nidq\"]))[0].shape[0]\n",
+ " == all_sync_data[\"trial_index_nidq\"].shape[0]\n",
+ " ):\n",
+ " print(\"All nan session\")\n",
" continue\n",
"\n",
- " if int(np.nanmax(all_sync_data['trial_index_nidq'])) != behavior_time.shape[0]:\n",
- " print('doing only one block')\n",
- " behavior_key['block'] = 1\n",
+ " if int(np.nanmax(all_sync_data[\"trial_index_nidq\"])) != behavior_time.shape[0]:\n",
+ " print(\"doing only one block\")\n",
+ " behavior_key[\"block\"] = 1\n",
" thissession = behavior.TowersBlock().Trial() & behavior_key\n",
- " behavior_time, iterstart = thissession.fetch('trial_time', 'vi_start')\n",
+ " behavior_time, iterstart = thissession.fetch(\"trial_time\", \"vi_start\")\n",
"\n",
- " if int(np.nanmax(all_sync_data['trial_index_nidq'])) != behavior_time.shape[0]:\n",
+ " if int(np.nanmax(all_sync_data[\"trial_index_nidq\"])) != behavior_time.shape[0]:\n",
" continue\n",
"\n",
- " status, trial_idx_vector, iteration_idx_vector =\\\n",
- " eu.get_index_type_vectors(all_sync_data['trial_index_nidq'], all_sync_data['iteration_index_nidq'], nidq_sampling_rate)\n",
- "\n",
+ " status, trial_idx_vector, iteration_idx_vector = eu.get_index_type_vectors(\n",
+ " all_sync_data[\"trial_index_nidq\"],\n",
+ " all_sync_data[\"iteration_index_nidq\"],\n",
+ " nidq_sampling_rate,\n",
+ " )\n",
"\n",
" if not status:\n",
" print(behavior_key)\n",
" print(rec_key)\n",
- " print('Could not reproduce nidq full vector')\n",
+ " print(\"Could not reproduce nidq full vector\")\n",
" break\n",
"\n",
- " trial_idx_vector_virmen, iteration_idx_vector_virmen =\\\n",
- " eu.get_iteration_intertrial_from_virmen_time(trial_pulses_signal, nidq_sampling_rate, behavior_time.shape[0], behavior_time)\n",
- "\n",
- " trial_index_nidq_virmen, iteration_index_nidq_virmen =\\\n",
- " eu.get_full_vector_samples(iteration_idx_vector_virmen,nidq_sampling_rate,all_sync_data['iteration_index_nidq'].shape[0])\n",
+ " trial_idx_vector_virmen, iteration_idx_vector_virmen = (\n",
+ " eu.get_iteration_intertrial_from_virmen_time(\n",
+ " trial_pulses_signal,\n",
+ " nidq_sampling_rate,\n",
+ " behavior_time.shape[0],\n",
+ " behavior_time,\n",
+ " )\n",
+ " )\n",
"\n",
+ " trial_index_nidq_virmen, iteration_index_nidq_virmen = eu.get_full_vector_samples(\n",
+ " iteration_idx_vector_virmen,\n",
+ " nidq_sampling_rate,\n",
+ " all_sync_data[\"iteration_index_nidq\"].shape[0],\n",
+ " )\n",
"\n",
- " diff_vector = all_sync_data['iteration_index_nidq'] - iteration_index_nidq_virmen\n",
+ " diff_vector = all_sync_data[\"iteration_index_nidq\"] - iteration_index_nidq_virmen\n",
"\n",
" max_diff_virmen = np.nanmax(diff_vector)\n",
- " print('max_diff_virmen', max_diff_virmen)\n",
+ " print(\"max_diff_virmen\", max_diff_virmen)\n",
"\n",
" min_diff_virmen = np.nanmin(diff_vector)\n",
- " print('min_diff_virmen', min_diff_virmen)\n",
+ " print(\"min_diff_virmen\", min_diff_virmen)\n",
"\n",
" dictionary_sync_data = dict()\n",
"\n",
- "\n",
" if max_diff_virmen > 20 or min_diff_virmen < -20:\n",
- " dictionary_sync_data['virmen_sync_status'] = True\n",
+ " dictionary_sync_data[\"virmen_sync_status\"] = True\n",
" else:\n",
- " dictionary_sync_data['virmen_sync_status'] = False\n",
- "\n",
+ " dictionary_sync_data[\"virmen_sync_status\"] = False\n",
"\n",
- " dictionary_sync_data['regular_sync_status'] = False\n",
+ " dictionary_sync_data[\"regular_sync_status\"] = False\n",
"\n",
- " dictionary_sync_data['trial_idx_vector'] = trial_idx_vector\n",
- " dictionary_sync_data['iteration_idx_vector'] = iteration_idx_vector\n",
+ " dictionary_sync_data[\"trial_idx_vector\"] = trial_idx_vector\n",
+ " dictionary_sync_data[\"iteration_idx_vector\"] = iteration_idx_vector\n",
"\n",
- " dictionary_sync_data['trial_idx_vector_from_virmen'] = trial_idx_vector_virmen\n",
- " dictionary_sync_data['iteration_idx_vector_from_virmen'] = iteration_idx_vector_virmen\n",
+ " dictionary_sync_data[\"trial_idx_vector_from_virmen\"] = trial_idx_vector_virmen\n",
+ " dictionary_sync_data[\"iteration_idx_vector_from_virmen\"] = (\n",
+ " iteration_idx_vector_virmen\n",
+ " )\n",
"\n",
" update_key = rec_key.copy()\n",
- " update_key['sync_data'] = dictionary_sync_data\n",
- " (ephys_pipeline.BehaviorSync & rec_key).update1(update_key)\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- " \n",
- "\n",
- " \n",
- "\n",
- " "
+ " update_key[\"sync_data\"] = dictionary_sync_data\n",
+ " (ephys_pipeline.BehaviorSync & rec_key).update1(update_key)"
]
},
{
@@ -281,9 +283,11 @@
"metadata": {},
"outputs": [],
"source": [
- "trial_index_nidq, iteration_index_nidq =\\\n",
- " eu.get_full_vector_samples(iteration_idx_vector,nidq_sampling_rate,all_sync_data['iteration_index_nidq'].shape[0])\n",
- "\n"
+ "trial_index_nidq, iteration_index_nidq = eu.get_full_vector_samples(\n",
+ " iteration_idx_vector,\n",
+ " nidq_sampling_rate,\n",
+ " all_sync_data[\"iteration_index_nidq\"].shape[0],\n",
+ ")"
]
},
{
@@ -292,7 +296,9 @@
"metadata": {},
"outputs": [],
"source": [
- "idx_diff = np.where((all_sync_data['iteration_index_nidq'] - iteration_index_nidq) > 0 )[0]"
+ "idx_diff = np.where((all_sync_data[\"iteration_index_nidq\"] - iteration_index_nidq) > 0)[\n",
+ " 0\n",
+ "]"
]
},
{
@@ -343,7 +349,7 @@
}
],
"source": [
- "all_sync_data['trial_index_nidq'][idx_diff]"
+ "all_sync_data[\"trial_index_nidq\"][idx_diff]"
]
},
{
@@ -373,7 +379,7 @@
}
],
"source": [
- "plt.plot((all_sync_data['iteration_index_nidq'] - iteration_index_nidq))"
+ "plt.plot(all_sync_data[\"iteration_index_nidq\"] - iteration_index_nidq)"
]
},
{
@@ -403,7 +409,7 @@
}
],
"source": [
- "plt.plot(all_sync_data['iteration_index_nidq'][idx_diff])\n",
+ "plt.plot(all_sync_data[\"iteration_index_nidq\"][idx_diff])\n",
"plt.plot(iteration_index_nidq[idx_diff])"
]
},
@@ -422,8 +428,8 @@
}
],
"source": [
- "print((all_sync_data['iteration_index_nidq'][idx_diff[0:5]]))\n",
- "print((iteration_index_nidq[idx_diff[0:5]]))"
+ "print(all_sync_data[\"iteration_index_nidq\"][idx_diff[0:5]])\n",
+ "print(iteration_index_nidq[idx_diff[0:5]])"
]
},
{
@@ -441,8 +447,8 @@
}
],
"source": [
- "print((all_sync_data['iteration_index_nidq'][idx_diff[0:5]-3]))\n",
- "print((iteration_index_nidq[idx_diff[0:5]-3]))"
+ "print(all_sync_data[\"iteration_index_nidq\"][idx_diff[0:5] - 3])\n",
+ "print(iteration_index_nidq[idx_diff[0:5] - 3])"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb
index 8f530801..9f56ad6e 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -35,17 +36,8 @@
],
"source": [
"import datajoint as dj\n",
- "import pandas as pd\n",
- "import numpy as np\n",
- "import matplotlib.pyplot as plt\n",
- "\n",
- "import u19_pipeline.ephys_pipeline as ep\n",
- "import u19_pipeline.utils.ephys_utils as eu\n",
- "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n",
"\n",
- "\n",
- "from u19_pipeline import recording\n",
- "\n"
+ "import u19_pipeline.ephys_pipeline as ep"
]
},
{
@@ -54,8 +46,8 @@
"metadata": {},
"outputs": [],
"source": [
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n",
- "ephys_pipeline = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')"
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
+ "ephys_pipeline = dj.create_virtual_module(\"ephys_pipeline\", \"u19_ephys_pipeline\")"
]
},
{
@@ -64,7 +56,9 @@
"metadata": {},
"outputs": [],
"source": [
- "recording_synced = (ep.BehaviorSync & \"sync_data is not null\").fetch('KEY', 'sync_data', order_by='recording_id DESC', as_dict=True)"
+ "recording_synced = (ep.BehaviorSync & \"sync_data is not null\").fetch(\n",
+ " \"KEY\", \"sync_data\", order_by=\"recording_id DESC\", as_dict=True\n",
+ ")"
]
},
{
@@ -83,28 +77,26 @@
],
"source": [
"for rec_key in recording_synced:\n",
- "\n",
- " print('rec_key', rec_key['recording_id'])\n",
+ " print(\"rec_key\", rec_key[\"recording_id\"])\n",
"\n",
" update_key = dict()\n",
- " update_key['recording_id'] = rec_key['recording_id']\n",
- " if rec_key['sync_data']['regular_sync_status']:\n",
- " update_key['regular_sync_status'] = 1\n",
- " update_key['fixed_sync_status'] = 0\n",
+ " update_key[\"recording_id\"] = rec_key[\"recording_id\"]\n",
+ " if rec_key[\"sync_data\"][\"regular_sync_status\"]:\n",
+ " update_key[\"regular_sync_status\"] = 1\n",
+ " update_key[\"fixed_sync_status\"] = 0\n",
" else:\n",
- " update_key['regular_sync_status'] = 0\n",
- " update_key['fixed_sync_status'] = 1\n",
+ " update_key[\"regular_sync_status\"] = 0\n",
+ " update_key[\"fixed_sync_status\"] = 1\n",
"\n",
- " if rec_key['sync_data']['virmen_sync_status']:\n",
- " update_key['virmen_sync_status'] = 1\n",
+ " if rec_key[\"sync_data\"][\"virmen_sync_status\"]:\n",
+ " update_key[\"virmen_sync_status\"] = 1\n",
" else:\n",
- " update_key['regular_sync_status'] = 0\n",
+ " update_key[\"regular_sync_status\"] = 0\n",
"\n",
- " print('update_key', update_key)\n",
+ " print(\"update_key\", update_key)\n",
" break\n",
"\n",
- " #(ephys_pipeline.BehaviorSync).update1(update_key)\n",
- " \n"
+ " # (ephys_pipeline.BehaviorSync).update1(update_key)"
]
},
{
@@ -124,7 +116,7 @@
}
],
"source": [
- "rec_key['sync_data']['trial_idx_vector'].shape"
+ "rec_key[\"sync_data\"][\"trial_idx_vector\"].shape"
]
},
{
@@ -34543,7 +34535,7 @@
}
],
"source": [
- "rec_key['sync_data']['iteration_idx_vector']"
+ "rec_key[\"sync_data\"][\"iteration_idx_vector\"]"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb
index 915cff08..633f990f 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb
@@ -24,7 +24,8 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
- "try_find_conf_file()\n"
+ "\n",
+ "try_find_conf_file()"
]
},
{
@@ -44,11 +45,11 @@
}
],
"source": [
- "import u19_pipeline.ephys_pipeline as ep\n",
- "import u19_pipeline.recording as recording\n",
- "import matplotlib.pyplot as plt\n",
"import datajoint as dj\n",
- " \n"
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import u19_pipeline.ephys_pipeline as ep\n",
+ "import u19_pipeline.recording as recording"
]
},
{
@@ -80,9 +81,9 @@
],
"source": [
"beh_key = dict()\n",
- "beh_key['subject_fullname'] = 'emdia_nicky'\n",
- "beh_key['session_date'] = '2026-03-23'\n",
- "beh_key['session_number'] = 0\n",
+ "beh_key[\"subject_fullname\"] = \"emdia_nicky\"\n",
+ "beh_key[\"session_date\"] = \"2026-03-23\"\n",
+ "beh_key[\"session_number\"] = 0\n",
"beh_key"
]
},
@@ -112,13 +113,13 @@
"recording_beh_key = (recording.Recording.BehaviorSession & beh_key).fetch(as_dict=True)\n",
"\n",
"if len(recording_beh_key) == 0:\n",
- " print('No recording data found for this session')\n",
+ " print(\"No recording data found for this session\")\n",
"elif len(recording_beh_key) > 1:\n",
- " print('Multiple recordings for this behavior key, select only one')\n",
+ " print(\"Multiple recordings for this behavior key, select only one\")\n",
"elif len(recording_beh_key) == 1:\n",
" recording_key = dict()\n",
- " recording_key['recording_id'] = recording_beh_key[0]['recording_id']\n",
- " print(recording_key)\n"
+ " recording_key[\"recording_id\"] = recording_beh_key[0][\"recording_id\"]\n",
+ " print(recording_key)"
]
},
{
@@ -149,13 +150,14 @@
}
],
"source": [
- "\n",
- "sync_record = (ep.BehaviorSync & recording_key).fetch1('sync_data')\n",
+ "sync_record = (ep.BehaviorSync & recording_key).fetch1(\"sync_data\")\n",
"full_sync_record = (ep.BehaviorSync & recording_key).fetch1()\n",
"\n",
- "behavior = dj.create_virtual_module('behavior','u19_behavior')\n",
- "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(as_dict=True)[0]\n",
- "behavior_session.pop('recording_id')\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
+ "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(\n",
+ " as_dict=True\n",
+ ")[0]\n",
+ "behavior_session.pop(\"recording_id\")\n",
"behavior_time = (behavior.SpatialTimeBlobs & behavior_session).fetch(as_dict=True)\n",
"\n",
"behavior_session"
@@ -187,8 +189,8 @@
}
],
"source": [
- "print(full_sync_record['iteration_index_nidq'])\n",
- "print(full_sync_record['iteration_index_nidq'])"
+ "print(full_sync_record[\"iteration_index_nidq\"])\n",
+ "print(full_sync_record[\"iteration_index_nidq\"])"
]
},
{
@@ -246,8 +248,8 @@
}
],
"source": [
- "print(sync_record['trial_idx_vector'].shape)\n",
- "sync_record['trial_idx_vector'][0:100]"
+ "print(sync_record[\"trial_idx_vector\"].shape)\n",
+ "sync_record[\"trial_idx_vector\"][0:100]"
]
},
{
@@ -304,11 +306,20 @@
"outputs": [],
"source": [
"import numpy as np\n",
- "time_as_behavior_fullsession = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession']]), axis=0) \n",
- "time_as_behavior_trial_ind = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind']]), axis=0) \n",
"\n",
- "time_as_behavior_fullsession_v = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession_virmen']]), axis=0) \n",
- "time_as_behavior_trial_ind_v = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind_virmen']]), axis=0) \n"
+ "time_as_behavior_fullsession = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_fullsession\"]]), axis=0\n",
+ ")\n",
+ "time_as_behavior_trial_ind = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_trial_ind\"]]), axis=0\n",
+ ")\n",
+ "\n",
+ "time_as_behavior_fullsession_v = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_fullsession_virmen\"]]), axis=0\n",
+ ")\n",
+ "time_as_behavior_trial_ind_v = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_trial_ind_virmen\"]]), axis=0\n",
+ ")"
]
},
{
@@ -326,11 +337,21 @@
"metadata": {},
"outputs": [],
"source": [
- "diff_time_trial_ind = behavior_time[0]['trial_time'].squeeze() - time_as_behavior_trial_ind\n",
- "diff_time_trial_full_session = (behavior_time[0]['cumulative_session_time'].squeeze() - behavior_time[0]['cumulative_session_time'][0]) - time_as_behavior_fullsession\n",
+ "diff_time_trial_ind = (\n",
+ " behavior_time[0][\"trial_time\"].squeeze() - time_as_behavior_trial_ind\n",
+ ")\n",
+ "diff_time_trial_full_session = (\n",
+ " behavior_time[0][\"cumulative_session_time\"].squeeze()\n",
+ " - behavior_time[0][\"cumulative_session_time\"][0]\n",
+ ") - time_as_behavior_fullsession\n",
"\n",
- "diff_time_trial_ind_v = behavior_time[0]['trial_time'].squeeze() - time_as_behavior_trial_ind_v\n",
- "diff_time_trial_full_session_v = (behavior_time[0]['cumulative_session_time'].squeeze() - behavior_time[0]['cumulative_session_time'][0]) - time_as_behavior_fullsession_v\n"
+ "diff_time_trial_ind_v = (\n",
+ " behavior_time[0][\"trial_time\"].squeeze() - time_as_behavior_trial_ind_v\n",
+ ")\n",
+ "diff_time_trial_full_session_v = (\n",
+ " behavior_time[0][\"cumulative_session_time\"].squeeze()\n",
+ " - behavior_time[0][\"cumulative_session_time\"][0]\n",
+ ") - time_as_behavior_fullsession_v"
]
},
{
@@ -373,9 +394,11 @@
"plt.figure(figsize=(8, 6))\n",
"plt.plot(diff_time_trial_full_session)\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time diff (s)')\n",
- "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session')"
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time diff (s)\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n",
+ ")"
]
},
{
@@ -408,13 +431,14 @@
"source": [
"plt.figure(figsize=(8, 6))\n",
"\n",
- "plt.hist(diff_time_trial_full_session,bins=100)\n",
+ "plt.hist(diff_time_trial_full_session, bins=100)\n",
"\n",
- "plt.yscale('log')\n",
- "plt.xlabel('Iteration start time difference (s)')\n",
- "plt.ylabel('Counts')\n",
- "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session')\n",
- "\n"
+ "plt.yscale(\"log\")\n",
+ "plt.xlabel(\"Iteration start time difference (s)\")\n",
+ "plt.ylabel(\"Counts\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session'\n",
+ ")"
]
},
{
@@ -448,9 +472,11 @@
"plt.figure(figsize=(8, 6))\n",
"plt.plot(diff_time_trial_full_session_v)\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time diff (s)')\n",
- "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session')"
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time diff (s)\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n",
+ ")"
]
},
{
@@ -483,13 +509,14 @@
"source": [
"plt.figure(figsize=(8, 6))\n",
"\n",
- "plt.hist(diff_time_trial_full_session_v,bins=100)\n",
+ "plt.hist(diff_time_trial_full_session_v, bins=100)\n",
"\n",
- "plt.yscale('log')\n",
- "plt.xlabel('Iteration start time difference (s)')\n",
- "plt.ylabel('Counts')\n",
- "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session')\n",
- "\n"
+ "plt.yscale(\"log\")\n",
+ "plt.xlabel(\"Iteration start time difference (s)\")\n",
+ "plt.ylabel(\"Counts\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session'\n",
+ ")"
]
}
],
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb
index e0777bb0..eea28d9f 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb
@@ -16,7 +16,8 @@
"outputs": [],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
- "try_find_conf_file()\n"
+ "\n",
+ "try_find_conf_file()"
]
},
{
@@ -34,11 +35,11 @@
}
],
"source": [
- "import u19_pipeline.ephys_pipeline as ep\n",
- "import u19_pipeline.recording as recording\n",
- "import matplotlib.pyplot as plt\n",
"import datajoint as dj\n",
- " \n"
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import u19_pipeline.ephys_pipeline as ep\n",
+ "import u19_pipeline.recording as recording"
]
},
{
@@ -57,7 +58,7 @@
"outputs": [],
"source": [
"recording_key = dict()\n",
- "recording_key['recording_id'] = 611"
+ "recording_key[\"recording_id\"] = 611"
]
},
{
@@ -88,12 +89,13 @@
}
],
"source": [
+ "sync_record = (ep.BehaviorSync & recording_key).fetch1(\"sync_data\")\n",
"\n",
- "sync_record = (ep.BehaviorSync & recording_key).fetch1('sync_data')\n",
- "\n",
- "behavior = dj.create_virtual_module('behavior','u19_behavior')\n",
- "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(as_dict=True)[0]\n",
- "behavior_session.pop('recording_id')\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
+ "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(\n",
+ " as_dict=True\n",
+ ")[0]\n",
+ "behavior_session.pop(\"recording_id\")\n",
"behavior_time = (behavior.SpatialTimeBlobs & behavior_session).fetch(as_dict=True)\n",
"\n",
"behavior_session"
@@ -171,12 +173,20 @@
"outputs": [],
"source": [
"import numpy as np\n",
- "time_as_behavior_fullsession = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession']]), axis=0) \n",
- "time_as_behavior_trial_ind = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind']]), axis=0) \n",
"\n",
- "time_as_behavior_fullsession_v = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession_virmen']]), axis=0) \n",
- "time_as_behavior_trial_ind_v = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind_virmen']]), axis=0) \n",
- "\n"
+ "time_as_behavior_fullsession = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_fullsession\"]]), axis=0\n",
+ ")\n",
+ "time_as_behavior_trial_ind = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_trial_ind\"]]), axis=0\n",
+ ")\n",
+ "\n",
+ "time_as_behavior_fullsession_v = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_fullsession_virmen\"]]), axis=0\n",
+ ")\n",
+ "time_as_behavior_trial_ind_v = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_trial_ind_virmen\"]]), axis=0\n",
+ ")"
]
},
{
@@ -194,11 +204,21 @@
"metadata": {},
"outputs": [],
"source": [
- "diff_time_trial_ind = behavior_time[0]['trial_time'].squeeze() - time_as_behavior_trial_ind\n",
- "diff_time_trial_ind_v = behavior_time[0]['trial_time'].squeeze() - time_as_behavior_trial_ind_v\n",
+ "diff_time_trial_ind = (\n",
+ " behavior_time[0][\"trial_time\"].squeeze() - time_as_behavior_trial_ind\n",
+ ")\n",
+ "diff_time_trial_ind_v = (\n",
+ " behavior_time[0][\"trial_time\"].squeeze() - time_as_behavior_trial_ind_v\n",
+ ")\n",
"\n",
- "diff_time_trial_full_session = (behavior_time[0]['cumulative_session_time'].squeeze() - behavior_time[0]['cumulative_session_time'][0]) - time_as_behavior_fullsession\n",
- "diff_time_trial_full_session_v = (behavior_time[0]['cumulative_session_time'].squeeze() - behavior_time[0]['cumulative_session_time'][0]) - time_as_behavior_fullsession_v\n"
+ "diff_time_trial_full_session = (\n",
+ " behavior_time[0][\"cumulative_session_time\"].squeeze()\n",
+ " - behavior_time[0][\"cumulative_session_time\"][0]\n",
+ ") - time_as_behavior_fullsession\n",
+ "diff_time_trial_full_session_v = (\n",
+ " behavior_time[0][\"cumulative_session_time\"].squeeze()\n",
+ " - behavior_time[0][\"cumulative_session_time\"][0]\n",
+ ") - time_as_behavior_fullsession_v"
]
},
{
@@ -241,9 +261,11 @@
"plt.figure(figsize=(8, 6))\n",
"plt.plot(diff_time_trial_ind)\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time diff (s)')\n",
- "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Each trial ind. time')"
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time diff (s)\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Each trial ind. time'\n",
+ ")"
]
},
{
@@ -285,13 +307,14 @@
"source": [
"plt.figure(figsize=(8, 6))\n",
"\n",
- "plt.hist(diff_time_trial_ind,bins=100)\n",
+ "plt.hist(diff_time_trial_ind, bins=100)\n",
"\n",
- "plt.yscale('log')\n",
- "plt.xlabel('Iteration start time difference (s)')\n",
- "plt.ylabel('Counts')\n",
- "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Each trial ind. time')\n",
- "\n"
+ "plt.yscale(\"log\")\n",
+ "plt.xlabel(\"Iteration start time difference (s)\")\n",
+ "plt.ylabel(\"Counts\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Each trial ind. time'\n",
+ ")"
]
},
{
@@ -363,9 +386,11 @@
"plt.figure(figsize=(8, 6))\n",
"plt.plot(diff_time_trial_full_session)\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time diff (s)')\n",
- "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session')"
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time diff (s)\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n",
+ ")"
]
},
{
@@ -398,13 +423,14 @@
"source": [
"plt.figure(figsize=(8, 6))\n",
"\n",
- "plt.hist(diff_time_trial_full_session,bins=100)\n",
+ "plt.hist(diff_time_trial_full_session, bins=100)\n",
"\n",
- "plt.yscale('log')\n",
- "plt.xlabel('Iteration start time difference (s)')\n",
- "plt.ylabel('Counts')\n",
- "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session')\n",
- "\n"
+ "plt.yscale(\"log\")\n",
+ "plt.xlabel(\"Iteration start time difference (s)\")\n",
+ "plt.ylabel(\"Counts\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session'\n",
+ ")"
]
},
{
@@ -444,15 +470,14 @@
}
],
"source": [
- "\n",
- "\n",
"plt.figure(figsize=(8, 6))\n",
"plt.plot(diff_time_trial_full_session_v)\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time diff (s)')\n",
- "plt.title('iteration \"start time\" sanity check comparison -- behavior file - nidaq pulse signal. Single vector time all session')\n",
- "\n"
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time diff (s)\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" sanity check comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n",
+ ")"
]
},
{
@@ -511,15 +536,14 @@
}
],
"source": [
- "\n",
- "\n",
"plt.figure(figsize=(8, 6))\n",
"plt.plot(diff_times_fullsessions)\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time diff (s)')\n",
- "plt.title('iteration \"start time\" comparison -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1')\n",
- "\n"
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time diff (s)\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" comparison -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1'\n",
+ ")"
]
},
{
@@ -552,12 +576,14 @@
"source": [
"plt.figure(figsize=(8, 6))\n",
"\n",
- "plt.hist(diff_times_fullsessions,bins=100)\n",
+ "plt.hist(diff_times_fullsessions, bins=100)\n",
"\n",
- "plt.yscale('log')\n",
- "plt.xlabel('Iteration start time difference (s)')\n",
- "plt.ylabel('Counts')\n",
- "plt.title('iteration \"start time\" histogram -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1')\n"
+ "plt.yscale(\"log\")\n",
+ "plt.xlabel(\"Iteration start time difference (s)\")\n",
+ "plt.ylabel(\"Counts\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" histogram -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1'\n",
+ ")"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb
index c2f94799..52af6449 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb
@@ -23,8 +23,8 @@
}
],
"source": [
- "\n",
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -43,14 +43,13 @@
}
],
"source": [
- "import u19_pipeline.ephys_pipeline as ep\n",
- "import u19_pipeline.utils.ephys_utils as ephys_utils\n",
- "import u19_pipeline.recording as recording\n",
- "import matplotlib.pyplot as plt\n",
"import datajoint as dj\n",
+ "import matplotlib.pyplot as plt\n",
"import numpy as np\n",
- " \n",
- "\n"
+ "\n",
+ "import u19_pipeline.ephys_pipeline as ep\n",
+ "import u19_pipeline.recording as recording\n",
+ "import u19_pipeline.utils.ephys_utils as ephys_utils"
]
},
{
@@ -69,7 +68,7 @@
"outputs": [],
"source": [
"recording_key = dict()\n",
- "recording_key['recording_id'] = 446"
+ "recording_key[\"recording_id\"] = 446"
]
},
{
@@ -100,15 +99,18 @@
}
],
"source": [
+ "sync_record = (ep.BehaviorSync & recording_key).fetch1(\"sync_data\")\n",
"\n",
- "sync_record = (ep.BehaviorSync & recording_key).fetch1('sync_data')\n",
- "\n",
- "behavior = dj.create_virtual_module('behavior','u19_behavior')\n",
- "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(as_dict=True)[0]\n",
- "behavior_session.pop('recording_id')\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n",
+ "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(\n",
+ " as_dict=True\n",
+ ")[0]\n",
+ "behavior_session.pop(\"recording_id\")\n",
"behavior_time_old = (behavior.SpatialTimeBlobs & behavior_session).fetch(as_dict=True)\n",
"\n",
- "behavior_time, behavior_fullsession = ephys_utils.get_real_behavior_time(behavior_session)\n",
+ "behavior_time, behavior_fullsession = ephys_utils.get_real_behavior_time(\n",
+ " behavior_session\n",
+ ")\n",
"behavior_time_single_vec = np.concatenate(behavior_time)\n",
"\n",
"behavior_session"
@@ -121,7 +123,9 @@
"metadata": {},
"outputs": [],
"source": [
- "comp_times = behavior_fullsession-behavior_time_old[0]['cumulative_session_time'].squeeze()"
+ "comp_times = (\n",
+ " behavior_fullsession - behavior_time_old[0][\"cumulative_session_time\"].squeeze()\n",
+ ")"
]
},
{
@@ -162,7 +166,7 @@
"metadata": {},
"outputs": [],
"source": [
- "comp_times2 = behavior_time_single_vec -behavior_time_old[0]['trial_time'].squeeze()"
+ "comp_times2 = behavior_time_single_vec - behavior_time_old[0][\"trial_time\"].squeeze()"
]
},
{
@@ -213,7 +217,7 @@
}
],
"source": [
- "plt.plot(comp_times2)\n"
+ "plt.plot(comp_times2)"
]
},
{
@@ -240,7 +244,9 @@
}
],
"source": [
- "all_vectors = ep.get_full_vectors_from_key(recording_key, apply_behavior_shift=True, behavior_time=behavior_time)"
+ "all_vectors = ep.get_full_vectors_from_key(\n",
+ " recording_key, apply_behavior_shift=True, behavior_time=behavior_time\n",
+ ")"
]
},
{
@@ -282,7 +288,7 @@
}
],
"source": [
- "all_vectors['time_as_behavior_fullsession'].shape"
+ "all_vectors[\"time_as_behavior_fullsession\"].shape"
]
},
{
@@ -300,13 +306,19 @@
"metadata": {},
"outputs": [],
"source": [
+ "time_as_behavior_fullsession = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_fullsession\"]]), axis=0\n",
+ ")\n",
+ "time_as_behavior_trial_ind = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_trial_ind\"]]), axis=0\n",
+ ")\n",
"\n",
- "time_as_behavior_fullsession = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession']]), axis=0) \n",
- "time_as_behavior_trial_ind = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind']]), axis=0) \n",
- "\n",
- "time_as_behavior_fullsession_v = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession_virmen']]), axis=0) \n",
- "time_as_behavior_trial_ind_v = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind_virmen']]), axis=0) \n",
- "\n"
+ "time_as_behavior_fullsession_v = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_fullsession_virmen\"]]), axis=0\n",
+ ")\n",
+ "time_as_behavior_trial_ind_v = np.concatenate(\n",
+ " ([x for x in all_vectors[\"time_as_behavior_trial_ind_virmen\"]]), axis=0\n",
+ ")"
]
},
{
@@ -327,8 +339,8 @@
"diff_time_trial_ind = time_as_behavior_trial_ind - behavior_time_single_vec\n",
"diff_time_trial_ind_v = time_as_behavior_trial_ind_v - behavior_time_single_vec\n",
"\n",
- "diff_time_trial_full_session = time_as_behavior_fullsession - behavior_fullsession \n",
- "diff_time_trial_full_session_v = time_as_behavior_fullsession_v - behavior_fullsession\n"
+ "diff_time_trial_full_session = time_as_behavior_fullsession - behavior_fullsession\n",
+ "diff_time_trial_full_session_v = time_as_behavior_fullsession_v - behavior_fullsession"
]
},
{
@@ -371,11 +383,13 @@
"plt.figure(figsize=(8, 6))\n",
"plt.plot(diff_time_trial_ind)\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time diff (s)')\n",
- "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Each trial ind. time')\n",
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time diff (s)\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Each trial ind. time'\n",
+ ")\n",
"\n",
- "#plt.ylim([-0.75,0.1])"
+ "# plt.ylim([-0.75,0.1])"
]
},
{
@@ -417,7 +431,9 @@
}
],
"source": [
- "consecutive_trials = ephys_utils.get_consecutive_trials_vector_from_iteration_matrix(behavior_time_old[0]['iteration_matrix'])\n",
+ "consecutive_trials = ephys_utils.get_consecutive_trials_vector_from_iteration_matrix(\n",
+ " behavior_time_old[0][\"iteration_matrix\"]\n",
+ ")\n",
"consecutive_trials.max()"
]
},
@@ -440,12 +456,12 @@
}
],
"source": [
- "print('Where minimum difference')\n",
+ "print(\"Where minimum difference\")\n",
"idx_min_diff = diff_time_trial_ind.argmin()\n",
- "print('block', behavior_time_old[0]['iteration_matrix'][idx_min_diff,0])\n",
- "print('trial', behavior_time_old[0]['iteration_matrix'][idx_min_diff,1])\n",
- "print('consecutive_trial', consecutive_trials[idx_min_diff])\n",
- "print('iteration', behavior_time_old[0]['iteration_matrix'][idx_min_diff,2])\n"
+ "print(\"block\", behavior_time_old[0][\"iteration_matrix\"][idx_min_diff, 0])\n",
+ "print(\"trial\", behavior_time_old[0][\"iteration_matrix\"][idx_min_diff, 1])\n",
+ "print(\"consecutive_trial\", consecutive_trials[idx_min_diff])\n",
+ "print(\"iteration\", behavior_time_old[0][\"iteration_matrix\"][idx_min_diff, 2])"
]
},
{
@@ -488,12 +504,14 @@
}
],
"source": [
- "print(time_as_behavior_trial_ind[idx_min_diff-5:idx_min_diff+5])\n",
+ "print(time_as_behavior_trial_ind[idx_min_diff - 5 : idx_min_diff + 5])\n",
"\n",
- "print(behavior_time_single_vec[idx_min_diff-5:idx_min_diff+5])\n",
+ "print(behavior_time_single_vec[idx_min_diff - 5 : idx_min_diff + 5])\n",
"\n",
"\n",
- "print(behavior_time_old[0]['iteration_matrix'][idx_min_diff-5:idx_min_diff+300,2])"
+ "print(\n",
+ " behavior_time_old[0][\"iteration_matrix\"][idx_min_diff - 5 : idx_min_diff + 300, 2]\n",
+ ")"
]
},
{
@@ -554,14 +572,15 @@
"\n",
"abs_diff_time_trial_ind = np.abs(diff_time_trial_ind)\n",
"\n",
- "plt.hist(diff_time_trial_ind,bins=100)\n",
+ "plt.hist(diff_time_trial_ind, bins=100)\n",
"\n",
- "plt.yscale('log')\n",
- "#plt.xscale('log')\n",
- "plt.xlabel('Iteration start time difference (s)')\n",
- "plt.ylabel('Counts')\n",
- "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Each trial ind. time')\n",
- "\n"
+ "plt.yscale(\"log\")\n",
+ "# plt.xscale('log')\n",
+ "plt.xlabel(\"Iteration start time difference (s)\")\n",
+ "plt.ylabel(\"Counts\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Each trial ind. time'\n",
+ ")"
]
},
{
@@ -633,9 +652,11 @@
"plt.figure(figsize=(8, 6))\n",
"plt.plot(diff_time_trial_full_session)\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time diff (s)')\n",
- "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session')"
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time diff (s)\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n",
+ ")"
]
},
{
@@ -668,13 +689,14 @@
"source": [
"plt.figure(figsize=(8, 6))\n",
"\n",
- "plt.hist(diff_time_trial_full_session,bins=100)\n",
+ "plt.hist(diff_time_trial_full_session, bins=100)\n",
"\n",
- "plt.yscale('log')\n",
- "plt.xlabel('Iteration start time difference (s)')\n",
- "plt.ylabel('Counts')\n",
- "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session')\n",
- "\n"
+ "plt.yscale(\"log\")\n",
+ "plt.xlabel(\"Iteration start time difference (s)\")\n",
+ "plt.ylabel(\"Counts\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session'\n",
+ ")"
]
},
{
@@ -714,15 +736,14 @@
}
],
"source": [
- "\n",
- "\n",
"plt.figure(figsize=(8, 6))\n",
"plt.plot(diff_time_trial_full_session_v)\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time diff (s)')\n",
- "plt.title('iteration \"start time\" sanity check comparison -- behavior file - nidaq pulse signal. Single vector time all session')\n",
- "\n"
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time diff (s)\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" sanity check comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n",
+ ")"
]
},
{
@@ -781,15 +802,14 @@
}
],
"source": [
- "\n",
- "\n",
"plt.figure(figsize=(8, 6))\n",
"plt.plot(diff_times_fullsessions)\n",
"\n",
- "plt.xlabel('Iteration #')\n",
- "plt.ylabel('Time diff (s)')\n",
- "plt.title('iteration \"start time\" comparison -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1')\n",
- "\n"
+ "plt.xlabel(\"Iteration #\")\n",
+ "plt.ylabel(\"Time diff (s)\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" comparison -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1'\n",
+ ")"
]
},
{
@@ -822,12 +842,14 @@
"source": [
"plt.figure(figsize=(8, 6))\n",
"\n",
- "plt.hist(diff_times_fullsessions,bins=100)\n",
+ "plt.hist(diff_times_fullsessions, bins=100)\n",
"\n",
- "plt.yscale('log')\n",
- "plt.xlabel('Iteration start time difference (s)')\n",
- "plt.ylabel('Counts')\n",
- "plt.title('iteration \"start time\" histogram -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1')\n"
+ "plt.yscale(\"log\")\n",
+ "plt.xlabel(\"Iteration start time difference (s)\")\n",
+ "plt.ylabel(\"Counts\")\n",
+ "plt.title(\n",
+ " 'iteration \"start time\" histogram -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1'\n",
+ ")"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb
index 31d15b51..5c289361 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -39,12 +40,12 @@
}
],
"source": [
- "import numpy as np\n",
- "import u19_pipeline.ephys_pipeline as ep\n",
- "import u19_pipeline.recording as recording\n",
- "import datajoint as dj\n",
+ "import datetime\n",
+ "\n",
"import pandas as pd\n",
- "import datetime\n"
+ "\n",
+ "import u19_pipeline.ephys_pipeline as ep\n",
+ "import u19_pipeline.recording as recording"
]
},
{
@@ -53,7 +54,7 @@
"metadata": {},
"outputs": [],
"source": [
- "#(ep.BehaviorSync & \"recording_id=502\").delete()"
+ "# (ep.BehaviorSync & \"recording_id=502\").delete()"
]
},
{
@@ -70,12 +71,25 @@
"outputs": [],
"source": [
"past_date = datetime.date.today() - datetime.timedelta(days=9000)\n",
- "query_session_date = 'session_date >=\"' + past_date.strftime('%Y-%m-%d') +'\"'\n",
+ "query_session_date = 'session_date >=\"' + past_date.strftime(\"%Y-%m-%d\") + '\"'\n",
"\n",
- "all_recs = ((recording.Recording & \"recording_modality='electrophysiology'\") * (recording.Recording.BehaviorSession & query_session_date)).join(ep.BehaviorSync, left=True)\n",
- "all_recs = pd.DataFrame(all_recs.fetch('recording_id','subject_fullname','session_date','session_number', as_dict=True))\n",
+ "all_recs = (\n",
+ " (recording.Recording & \"recording_modality='electrophysiology'\")\n",
+ " * (recording.Recording.BehaviorSession & query_session_date)\n",
+ ").join(ep.BehaviorSync, left=True)\n",
+ "all_recs = pd.DataFrame(\n",
+ " all_recs.fetch(\n",
+ " \"recording_id\",\n",
+ " \"subject_fullname\",\n",
+ " \"session_date\",\n",
+ " \"session_number\",\n",
+ " as_dict=True,\n",
+ " )\n",
+ ")\n",
"\n",
- "not_sync_recs = pd.DataFrame((recording.Recording * ep.BehaviorSync).fetch('KEY', as_dict=True))"
+ "not_sync_recs = pd.DataFrame(\n",
+ " (recording.Recording * ep.BehaviorSync).fetch(\"KEY\", as_dict=True)\n",
+ ")"
]
},
{
@@ -86,7 +100,7 @@
"source": [
"def get_rec_key_dict(recording_id):\n",
"\n",
- " return {'recording_id': recording_id}"
+ " return {\"recording_id\": recording_id}"
]
},
{
@@ -371,10 +385,10 @@
}
],
"source": [
- "not_sync_recs2 = pd.merge(all_recs,not_sync_recs, how='left', indicator=True)\n",
- "not_sync_recs2 = not_sync_recs2.loc[not_sync_recs2['_merge']=='left_only',:]\n",
- "not_sync_recs2 = not_sync_recs2.sort_values(by='session_date', ascending=False)\n",
- "not_sync_recs2['rec_key'] = not_sync_recs2['recording_id'].apply(get_rec_key_dict)\n",
+ "not_sync_recs2 = pd.merge(all_recs, not_sync_recs, how=\"left\", indicator=True)\n",
+ "not_sync_recs2 = not_sync_recs2.loc[not_sync_recs2[\"_merge\"] == \"left_only\", :]\n",
+ "not_sync_recs2 = not_sync_recs2.sort_values(by=\"session_date\", ascending=False)\n",
+ "not_sync_recs2[\"rec_key\"] = not_sync_recs2[\"recording_id\"].apply(get_rec_key_dict)\n",
"not_sync_recs2 = not_sync_recs2.reset_index(drop=True)\n",
"not_sync_recs2"
]
@@ -481,8 +495,8 @@
}
],
"source": [
- "#for i in range(2):\n",
- "ep.BehaviorSync.populate(not_sync_recs2.loc[8,'rec_key'])"
+ "# for i in range(2):\n",
+ "ep.BehaviorSync.populate(not_sync_recs2.loc[8, \"rec_key\"])"
]
},
{
@@ -53823,7 +53837,7 @@
],
"source": [
"lo = (ep.BehaviorSync & \"recording_id=502\").fetch(as_dict=True)\n",
- "lo[0]['sync_data']"
+ "lo[0][\"sync_data\"]"
]
},
{
@@ -53843,7 +53857,7 @@
}
],
"source": [
- "lo[0]['sync_data'].keys()"
+ "lo[0][\"sync_data\"].keys()"
]
},
{
diff --git a/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb
index 76d6e992..11382e59 100644
--- a/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb
+++ b/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb
@@ -26,6 +26,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -46,8 +47,8 @@
],
"source": [
"import numpy as np\n",
+ "\n",
"import u19_pipeline.ephys_pipeline as ep\n",
- "import u19_pipeline.acquisition as acquisition\n",
"from u19_pipeline import recording"
]
},
@@ -69,10 +70,10 @@
}
],
"source": [
- "key = {'subject_fullname': \"jk8386_jknpx6\", 'session_date': \"2025-11-24\"}\n",
+ "key = {\"subject_fullname\": \"jk8386_jknpx6\", \"session_date\": \"2025-11-24\"}\n",
"\n",
"rid = (recording.Recording.BehaviorSession & key).fetch(as_dict=True)[0]\n",
- "key = {'recording_id': rid['recording_id']}\n",
+ "key = {\"recording_id\": rid[\"recording_id\"]}\n",
"key"
]
},
@@ -82,7 +83,7 @@
"metadata": {},
"outputs": [],
"source": [
- "key = {'recording_id': 309}"
+ "key = {\"recording_id\": 309}"
]
},
{
@@ -103,6 +104,7 @@
],
"source": [
"import importlib\n",
+ "\n",
"importlib.reload(ep)"
]
},
@@ -172,8 +174,10 @@
}
],
"source": [
- "block = 1 # block numbering starts at 1. Set to whatever block you want to use as the virmen_start time.\n",
- "ep.BehaviorSync.populate(key, make_kwargs={'block': block}) # when block is passed, it only starts the session at that specific block."
+ "block = 1 # block numbering starts at 1. Set to whatever block you want to use as the virmen_start time.\n",
+ "ep.BehaviorSync.populate(\n",
+ " key, make_kwargs={\"block\": block}\n",
+ ") # when block is passed, it only starts the session at that specific block."
]
},
{
@@ -202,7 +206,7 @@
}
],
"source": [
- "np.where(np.isnan(lo[0]['iteration_index_nidq']))[0].shape"
+ "np.where(np.isnan(lo[0][\"iteration_index_nidq\"]))[0].shape"
]
},
{
diff --git a/notebooks/ephys_element/read_ephys_parameters.ipynb b/notebooks/ephys_element/read_ephys_parameters.ipynb
index 7e9fbd64..36524143 100644
--- a/notebooks/ephys_element/read_ephys_parameters.ipynb
+++ b/notebooks/ephys_element/read_ephys_parameters.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -180,7 +181,7 @@
}
],
"source": [
- "all_params.loc[0, 'params']"
+ "all_params.loc[0, \"params\"]"
]
}
],
diff --git a/notebooks/ephys_element/test_paramspy_for_os.ipynb b/notebooks/ephys_element/test_paramspy_for_os.ipynb
index 3425ec85..2c91b0d3 100644
--- a/notebooks/ephys_element/test_paramspy_for_os.ipynb
+++ b/notebooks/ephys_element/test_paramspy_for_os.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -24,9 +25,10 @@
"metadata": {},
"outputs": [],
"source": [
- "import pandas as pd\n",
+ "import pathlib\n",
+ "\n",
"import datajoint as dj\n",
- "import pathlib"
+ "import pandas as pd"
]
},
{
@@ -35,10 +37,9 @@
"metadata": {},
"outputs": [],
"source": [
- "import u19_pipeline.ephys_pipeline as ep\n",
+ "import u19_pipeline.lab as lab\n",
"import u19_pipeline.recording as recording\n",
- "import u19_pipeline.recording_process as rp\n",
- "import u19_pipeline.lab as lab"
+ "import u19_pipeline.recording_process as rp"
]
},
{
@@ -47,11 +48,13 @@
"metadata": {},
"outputs": [],
"source": [
- "df_path = pd.DataFrame((lab.Path & \"global_path='/braininit'\").fetch('system', 'local_path',as_dict=True))\n",
+ "df_path = pd.DataFrame(\n",
+ " (lab.Path & \"global_path='/braininit'\").fetch(\"system\", \"local_path\", as_dict=True)\n",
+ ")\n",
"\n",
- "linux_path = df_path.loc[df_path['system']=='linux','local_path'].values[0]\n",
- "windows_path = df_path.loc[df_path['system']=='windows','local_path'].values[0]\n",
- "mac_path = df_path.loc[df_path['system']=='mac','local_path'].values[0]"
+ "linux_path = df_path.loc[df_path[\"system\"] == \"linux\", \"local_path\"].values[0]\n",
+ "windows_path = df_path.loc[df_path[\"system\"] == \"windows\", \"local_path\"].values[0]\n",
+ "mac_path = df_path.loc[df_path[\"system\"] == \"mac\", \"local_path\"].values[0]"
]
},
{
@@ -217,10 +220,14 @@
],
"source": [
"rec_query = dict()\n",
- "rec_query['recording_modality'] = 'electrophysiology'\n",
+ "rec_query[\"recording_modality\"] = \"electrophysiology\"\n",
"rec_query_2 = \"recording_id > 230\"\n",
"\n",
- "rec_data = pd.DataFrame((recording.Recording * rp.Processing & rec_query & rec_query_2).fetch('recording_process_post_path',as_dict=True))\n",
+ "rec_data = pd.DataFrame(\n",
+ " (recording.Recording * rp.Processing & rec_query & rec_query_2).fetch(\n",
+ " \"recording_process_post_path\", as_dict=True\n",
+ " )\n",
+ ")\n",
"rec_data"
]
},
@@ -239,33 +246,29 @@
}
],
"source": [
- "\n",
"for i in range(rec_data.shape[0]):\n",
- " proc_dir = rec_data.loc[i,'recording_process_post_path']\n",
- " output_dir = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][1],proc_dir)\n",
+ " proc_dir = rec_data.loc[i, \"recording_process_post_path\"]\n",
+ " output_dir = pathlib.Path(dj.config[\"custom\"][\"ephys_root_data_dir\"][1], proc_dir)\n",
"\n",
- " kilo_output = output_dir.glob('*kilosort*')\n",
+ " kilo_output = output_dir.glob(\"*kilosort*\")\n",
" for kilo_dir in kilo_output:\n",
- " params_file = pathlib.Path(kilo_dir,'params.py')\n",
+ " params_file = pathlib.Path(kilo_dir, \"params.py\")\n",
" if params_file.is_file():\n",
- " \n",
- " with open(params_file.as_posix(), \"r\") as file:\n",
+ " with open(params_file.as_posix()) as file:\n",
" params_text = file.read()\n",
- " \n",
+ "\n",
" try:\n",
- " windows_params = params_text.replace(linux_path,windows_path)\n",
- " windows_param_file = pathlib.Path(kilo_dir,'win_params.py')\n",
+ " windows_params = params_text.replace(linux_path, windows_path)\n",
+ " windows_param_file = pathlib.Path(kilo_dir, \"win_params.py\")\n",
" with open(windows_param_file, \"w\") as file:\n",
" file.write(windows_params)\n",
"\n",
- " mac_params = params_text.replace(linux_path,mac_path)\n",
- " mac_param_file = pathlib.Path(kilo_dir,'mac_params.py')\n",
+ " mac_params = params_text.replace(linux_path, mac_path)\n",
+ " mac_param_file = pathlib.Path(kilo_dir, \"mac_params.py\")\n",
" with open(mac_param_file, \"w\") as file:\n",
" file.write(mac_params)\n",
" except Exception as e:\n",
- " print(e)\n",
- "\n",
- "\n"
+ " print(e)"
]
},
{
diff --git a/notebooks/fetch_session_data.ipynb b/notebooks/fetch_session_data.ipynb
index ab5baf24..ba28308b 100644
--- a/notebooks/fetch_session_data.ipynb
+++ b/notebooks/fetch_session_data.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -33,11 +34,11 @@
],
"source": [
"import datajoint as dj\n",
- "import pandas as pd\n",
"import numpy as np\n",
+ "import pandas as pd\n",
"\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n"
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -1117,9 +1118,9 @@
],
"source": [
"key = dict()\n",
- "key['subject_fullname'] = 'ms81_M005'\n",
- "#key = \"subject_fullname like 'emdia%'\" \n",
- "#key['session_date'] = '2021-05-05'\n",
+ "key[\"subject_fullname\"] = \"ms81_M005\"\n",
+ "# key = \"subject_fullname like 'emdia%'\"\n",
+ "# key['session_date'] = '2021-05-05'\n",
"\n",
"print(key)\n",
"\n",
@@ -1517,10 +1518,15 @@
],
"source": [
"blocktable = behavior.TowersBlock() * behavior.TowersBlock.Trial()\n",
- "#Remove position, velocity fields because they take a lot to load\n",
- "all_fields = pd.DataFrame.from_dict(blocktable.heading.attributes, orient='index')\n",
- "sel_fields = all_fields.loc[~all_fields['name'].isin(['position', 'velocity', 'sensor_dots', 'collision', 'trial_time']), :]\n",
- "sel_fields = sel_fields['name'].tolist()\n",
+ "# Remove position, velocity fields because they take a lot to load\n",
+ "all_fields = pd.DataFrame.from_dict(blocktable.heading.attributes, orient=\"index\")\n",
+ "sel_fields = all_fields.loc[\n",
+ " ~all_fields[\"name\"].isin(\n",
+ " [\"position\", \"velocity\", \"sensor_dots\", \"collision\", \"trial_time\"]\n",
+ " ),\n",
+ " :,\n",
+ "]\n",
+ "sel_fields = sel_fields[\"name\"].tolist()\n",
"\n",
"\n",
"all_trials_df = pd.DataFrame((blocktable & key).fetch(*sel_fields, as_dict=True))\n",
@@ -1533,14 +1539,22 @@
"metadata": {},
"outputs": [],
"source": [
- "all_trials_df['towers_left'] = all_trials_df.apply(lambda x: np.sum(x['cue_presence_left'][0]),axis=1)\n",
- "all_trials_df['towers_right'] = all_trials_df.apply(lambda x: np.sum(x['cue_presence_right'][0]),axis=1)\n",
- "all_trials_df['stimulus'] = all_trials_df['towers_left'] - all_trials_df['towers_right']\n",
- "all_trials_df['choice'] = all_trials_df['choice'].replace({'L': 0, 'R': 1, 'nil': -1})\n",
- "all_trials_df['trial_type'] = all_trials_df['trial_type'].replace({'L': 0, 'R': 1, 'nil': -1})\n",
+ "all_trials_df[\"towers_left\"] = all_trials_df.apply(\n",
+ " lambda x: np.sum(x[\"cue_presence_left\"][0]), axis=1\n",
+ ")\n",
+ "all_trials_df[\"towers_right\"] = all_trials_df.apply(\n",
+ " lambda x: np.sum(x[\"cue_presence_right\"][0]), axis=1\n",
+ ")\n",
+ "all_trials_df[\"stimulus\"] = all_trials_df[\"towers_left\"] - all_trials_df[\"towers_right\"]\n",
+ "all_trials_df[\"choice\"] = all_trials_df[\"choice\"].replace({\"L\": 0, \"R\": 1, \"nil\": -1})\n",
+ "all_trials_df[\"trial_type\"] = all_trials_df[\"trial_type\"].replace(\n",
+ " {\"L\": 0, \"R\": 1, \"nil\": -1}\n",
+ ")\n",
"\n",
- "all_trials_df = all_trials_df.loc[(all_trials_df['main_level'] == 11) & (all_trials_df['level'] == 11), :]\n",
- "all_trials_df = all_trials_df.reset_index(drop=True)\n"
+ "all_trials_df = all_trials_df.loc[\n",
+ " (all_trials_df[\"main_level\"] == 11) & (all_trials_df[\"level\"] == 11), :\n",
+ "]\n",
+ "all_trials_df = all_trials_df.reset_index(drop=True)"
]
},
{
@@ -2054,7 +2068,7 @@
}
],
"source": [
- "all_trials_df[['cue_presence_left', 'cue_presence_right']]"
+ "all_trials_df[[\"cue_presence_left\", \"cue_presence_right\"]]"
]
},
{
@@ -2085,7 +2099,7 @@
}
],
"source": [
- "all_trials_df['stimulus']"
+ "all_trials_df[\"stimulus\"]"
]
},
{
@@ -3105,7 +3119,7 @@
}
],
"source": [
- "all_trials_df['stimulus'].tolist()"
+ "all_trials_df[\"stimulus\"].tolist()"
]
},
{
diff --git a/notebooks/imaging_element/00-Set-up-configuration.ipynb b/notebooks/imaging_element/00-Set-up-configuration.ipynb
index e22e83a7..0bea86fe 100644
--- a/notebooks/imaging_element/00-Set-up-configuration.ipynb
+++ b/notebooks/imaging_element/00-Set-up-configuration.ipynb
@@ -24,10 +24,10 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
- "import os\n",
- "import getpass\n",
"import datajoint as dj\n",
+ "\n",
"import u19_pipeline.utility as utility"
]
},
@@ -94,9 +94,11 @@
"outputs": [],
"source": [
"# If there is only one root path:\n",
- "dj.config['custom']['imaging_root_data_dir'] = utility.get_network_path('braininit') + '/RigData/mesoscope/imaging/'\n",
+ "dj.config[\"custom\"][\"imaging_root_data_dir\"] = (\n",
+ " utility.get_network_path(\"braininit\") + \"/RigData/mesoscope/imaging/\"\n",
+ ")\n",
"# If there are multiple possible root paths (write them as a list):\n",
- "#dj.config['custom']['imaging_root_data_dir'] = ['/mnt/bucket/braininit/RigData/mesoscope/imaging/imaging_pipe1/']"
+ "# dj.config['custom']['imaging_root_data_dir'] = ['/mnt/bucket/braininit/RigData/mesoscope/imaging/imaging_pipe1/']"
]
},
{
@@ -117,7 +119,7 @@
}
],
"source": [
- "dj.config['custom']"
+ "dj.config[\"custom\"]"
]
},
{
diff --git a/notebooks/imaging_element/03-imaging-data-analysis.ipynb b/notebooks/imaging_element/03-imaging-data-analysis.ipynb
index 3b5872f4..4f4fef33 100644
--- a/notebooks/imaging_element/03-imaging-data-analysis.ipynb
+++ b/notebooks/imaging_element/03-imaging-data-analysis.ipynb
@@ -29,6 +29,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -54,18 +55,16 @@
}
],
"source": [
- "import os \n",
"import datajoint as dj\n",
+ "import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import pandas as pd\n",
- "from u19_pipeline import acquisition, subject\n",
- "from u19_pipeline.imaging_element import imaging_element, scan_element, get_suite2p_dir\n",
- "from u19_pipeline.ingest.imaging_element_ingest import process_scan\n",
- "import pathlib\n",
- "import matplotlib.pyplot as plt\n",
+ "from u19_pipeline.imaging_element import imaging_element\n",
"\n",
- "imaging = dj.create_virtual_module('imaging', 'u19_imaging')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "from u19_pipeline import subject\n",
+ "\n",
+ "imaging = dj.create_virtual_module(\"imaging\", \"u19_imaging\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -81,10 +80,10 @@
"metadata": {},
"outputs": [],
"source": [
- "subject = 'jeremyjc_j016'\n",
- "date = '2021-11-21'\n",
+ "subject = \"jeremyjc_j016\"\n",
+ "date = \"2021-11-21\"\n",
"\n",
- "key = (imaging.Scan & dict(session_date =date, subject_fullname=subject)).fetch1('KEY')"
+ "key = (imaging.Scan & dict(session_date=date, subject_fullname=subject)).fetch1(\"KEY\")"
]
},
{
@@ -100,10 +99,23 @@
"metadata": {},
"outputs": [],
"source": [
- "cols_sync = ['sync_im_frame', 'sync_im_frame_global', 'sync_behav_block_by_im_frame', 'sync_behav_trial_by_im_frame', 'sync_behav_iter_by_im_frame', 'sync_im_frame_span_by_behav_block'] \n",
- "sync_data = pd.DataFrame((imaging.SyncImagingBehavior & key).fetch(*cols_sync, as_dict=True))\n",
- "mask_data = pd.DataFrame((imaging_element.MaskClassification.MaskType & key).fetch(as_dict=True))\n",
- "trace_data = pd.DataFrame((imaging_element.Fluorescence.Trace & key).fetch(as_dict=True))\n",
+ "cols_sync = [\n",
+ " \"sync_im_frame\",\n",
+ " \"sync_im_frame_global\",\n",
+ " \"sync_behav_block_by_im_frame\",\n",
+ " \"sync_behav_trial_by_im_frame\",\n",
+ " \"sync_behav_iter_by_im_frame\",\n",
+ " \"sync_im_frame_span_by_behav_block\",\n",
+ "]\n",
+ "sync_data = pd.DataFrame(\n",
+ " (imaging.SyncImagingBehavior & key).fetch(*cols_sync, as_dict=True)\n",
+ ")\n",
+ "mask_data = pd.DataFrame(\n",
+ " (imaging_element.MaskClassification.MaskType & key).fetch(as_dict=True)\n",
+ ")\n",
+ "trace_data = pd.DataFrame(\n",
+ " (imaging_element.Fluorescence.Trace & key).fetch(as_dict=True)\n",
+ ")\n",
"behavior_data = pd.DataFrame((behavior.TowersBlock.Trial & key).fetch(as_dict=True))"
]
},
@@ -144,12 +156,16 @@
}
],
"source": [
- "#Which mask shall we use for plots\n",
+ "# Which mask shall we use for plots\n",
"idx_mask = 1\n",
"\n",
- "mask_data = mask_data.sort_values(by=['confidence'], ascending=[False])\n",
+ "mask_data = mask_data.sort_values(by=[\"confidence\"], ascending=[False])\n",
"mask_data = mask_data.reset_index(drop=True)\n",
- "plt.plot(trace_data.loc[trace_data['mask'] == mask_data.loc[idx_mask, 'mask'], 'fluorescence'].values[0])"
+ "plt.plot(\n",
+ " trace_data.loc[\n",
+ " trace_data[\"mask\"] == mask_data.loc[idx_mask, \"mask\"], \"fluorescence\"\n",
+ " ].values[0]\n",
+ ")"
]
},
{
@@ -165,25 +181,33 @@
"metadata": {},
"outputs": [],
"source": [
- "blocks_imaging_data = np.unique(sync_data.loc[0, 'sync_behav_block_by_im_frame'])\n",
+ "blocks_imaging_data = np.unique(sync_data.loc[0, \"sync_behav_block_by_im_frame\"])\n",
"blocks_imaging_data = np.delete(blocks_imaging_data, 0)\n",
"\n",
- "trials_block = behavior_data[['block', 'trial_idx']].copy()\n",
- "trials_block = trials_block.loc[trials_block['block'].isin(blocks_imaging_data), :]\n",
- "trials_block_group = trials_block.groupby(by=['block']).count()\n",
+ "trials_block = behavior_data[[\"block\", \"trial_idx\"]].copy()\n",
+ "trials_block = trials_block.loc[trials_block[\"block\"].isin(blocks_imaging_data), :]\n",
+ "trials_block_group = trials_block.groupby(by=[\"block\"]).count()\n",
"trials_block_group = trials_block_group.reset_index()\n",
- "trials_block_group.sort_values(by=['trial_idx'], ascending=[False])\n",
+ "trials_block_group.sort_values(by=[\"trial_idx\"], ascending=[False])\n",
"trials_block_group.reset_index(drop=True)\n",
- "sel_block = trials_block_group.loc[0, 'block']\n",
+ "sel_block = trials_block_group.loc[0, \"block\"]\n",
"\n",
- "trials_sel_block = behavior_data.loc[behavior_data['block'] == sel_block, :]\n",
+ "trials_sel_block = behavior_data.loc[behavior_data[\"block\"] == sel_block, :]\n",
"trials_sel_block = trials_sel_block.reset_index(drop=True)\n",
"\n",
- "#Samples imaging of the selected block\n",
- "idx_samples_block = np.where(np.squeeze(sync_data.loc[0, 'sync_behav_block_by_im_frame']) == sel_block)\n",
- "imaging_trial_data_idx = np.squeeze(sync_data.loc[0, 'sync_behav_trial_by_im_frame'])[idx_samples_block]\n",
- "imaging_iter_data_idx = np.squeeze(sync_data.loc[0, 'sync_behav_iter_by_im_frame'])[idx_samples_block]\n",
- "fluorescence_data = trace_data.loc[trace_data['mask'] == mask_data.loc[idx_mask, 'mask'], 'fluorescence'].values[0][idx_samples_block]\n"
+ "# Samples imaging of the selected block\n",
+ "idx_samples_block = np.where(\n",
+ " np.squeeze(sync_data.loc[0, \"sync_behav_block_by_im_frame\"]) == sel_block\n",
+ ")\n",
+ "imaging_trial_data_idx = np.squeeze(sync_data.loc[0, \"sync_behav_trial_by_im_frame\"])[\n",
+ " idx_samples_block\n",
+ "]\n",
+ "imaging_iter_data_idx = np.squeeze(sync_data.loc[0, \"sync_behav_iter_by_im_frame\"])[\n",
+ " idx_samples_block\n",
+ "]\n",
+ "fluorescence_data = trace_data.loc[\n",
+ " trace_data[\"mask\"] == mask_data.loc[idx_mask, \"mask\"], \"fluorescence\"\n",
+ "].values[0][idx_samples_block]"
]
},
{
@@ -199,34 +223,35 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
"trace_vec = list()\n",
"position_vec = list()\n",
"for i in range(trials_sel_block.shape[0]):\n",
- "\n",
- " idx_trial = np.where(imaging_trial_data_idx == i+1)\n",
+ " idx_trial = np.where(imaging_trial_data_idx == i + 1)\n",
" iters_trial = imaging_iter_data_idx[idx_trial]\n",
" fluorescence_trial = fluorescence_data[idx_trial]\n",
- " position_vec_trial = trials_sel_block.loc[trials_sel_block['trial_idx'] == i+1, 'position'].values[0]\n",
- " sensor_dots = trials_sel_block.loc[trials_sel_block['trial_idx'] == i+1, 'sensor_dots'].values[0]\n",
- " iter_in_trial = trials_sel_block.loc[trials_sel_block['trial_idx'] == i+1, 'iterations'].values[0]\n",
+ " position_vec_trial = trials_sel_block.loc[\n",
+ " trials_sel_block[\"trial_idx\"] == i + 1, \"position\"\n",
+ " ].values[0]\n",
+ " sensor_dots = trials_sel_block.loc[\n",
+ " trials_sel_block[\"trial_idx\"] == i + 1, \"sensor_dots\"\n",
+ " ].values[0]\n",
+ " iter_in_trial = trials_sel_block.loc[\n",
+ " trials_sel_block[\"trial_idx\"] == i + 1, \"iterations\"\n",
+ " ].values[0]\n",
"\n",
" unique_iter = np.unique(iters_trial)\n",
"\n",
" for idx, iter in enumerate(unique_iter):\n",
- "\n",
- "\n",
- "\n",
" if iter <= iter_in_trial:\n",
" curr_iter = np.where(iters_trial == iter)\n",
" trace_iter = fluorescence_trial[curr_iter[0]]\n",
- " \n",
- " trace_vec.append(np.mean(trace_iter)) \n",
- " position_vec.append(position_vec_trial[int(iter)-1,1])\n",
+ "\n",
+ " trace_vec.append(np.mean(trace_iter))\n",
+ " position_vec.append(position_vec_trial[int(iter) - 1, 1])\n",
"\n",
"\n",
- "trace_vec = np.asarray(trace_vec)\n",
- "position_vec = np.asarray(position_vec)\n"
+ "trace_vec = np.asarray(trace_vec)\n",
+ "position_vec = np.asarray(position_vec)"
]
},
{
diff --git a/notebooks/imaging_element/getSyncInfo_python.ipynb b/notebooks/imaging_element/getSyncInfo_python.ipynb
index 44514496..da60ae01 100644
--- a/notebooks/imaging_element/getSyncInfo_python.ipynb
+++ b/notebooks/imaging_element/getSyncInfo_python.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -24,17 +25,18 @@
"metadata": {},
"outputs": [],
"source": [
- "import pandas as pd\n",
- "import numpy as np\n",
+ "import pathlib\n",
+ "\n",
"import datajoint as dj\n",
- "import u19_pipeline.imaging_pipeline as ip\n",
"import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
"\n",
- "import u19_pipeline.recording as recording\n",
- "import u19_pipeline.lab as lab\n",
"import u19_pipeline.acquisition as acquisition\n",
- "import u19_pipeline.utils.matlab_utils as mu\n",
- "import pathlib"
+ "import u19_pipeline.imaging_pipeline as ip\n",
+ "import u19_pipeline.lab as lab\n",
+ "import u19_pipeline.recording as recording\n",
+ "import u19_pipeline.utils.matlab_utils as mu"
]
},
{
@@ -43,7 +45,7 @@
"metadata": {},
"outputs": [],
"source": [
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -52,15 +54,16 @@
"metadata": {},
"outputs": [],
"source": [
- "import numpy as np\n",
- "import tifffile\n",
"import re\n",
"import struct\n",
"\n",
+ "import tifffile\n",
+ "\n",
+ "\n",
"def get_sync_info(input_file, data_type, frame_skip):\n",
" if data_type == \"uint16\":\n",
" num_bytes = 2\n",
- " struct_fmt = ' 0 else [])\n",
" else:\n",
" if len(d) != n_data:\n",
- " raise ValueError(f\"Inconsistent number of I2CData entries (expected {n_data}, got {len(d)}).\")\n",
+ " raise ValueError(\n",
+ " f\"Inconsistent number of I2CData entries (expected {n_data}, got {len(d)}).\"\n",
+ " )\n",
" final_data.append(d)\n",
"\n",
" epoch = np.array([epoch], dtype=np.float64)\n",
@@ -183,7 +200,7 @@
"metadata": {},
"outputs": [],
"source": [
- "imaging_root_dir = dj.config['custom']['imaging_root_data_dir'][0]"
+ "imaging_root_dir = dj.config[\"custom\"][\"imaging_root_data_dir\"][0]"
]
},
{
@@ -385,7 +402,11 @@
}
],
"source": [
- "imaging_sessions = pd.DataFrame((recording.Recording.BehaviorSession * ip.ImagingPipelineSession).fetch(order_by='session_date desc',as_dict=True))\n",
+ "imaging_sessions = pd.DataFrame(\n",
+ " (recording.Recording.BehaviorSession * ip.ImagingPipelineSession).fetch(\n",
+ " order_by=\"session_date desc\", as_dict=True\n",
+ " )\n",
+ ")\n",
"imaging_sessions"
]
},
@@ -408,10 +429,10 @@
}
],
"source": [
- "key = imaging_sessions.loc[0,:].to_dict()\n",
+ "key = imaging_sessions.loc[0, :].to_dict()\n",
"key\n",
"img_key = dict()\n",
- "img_key['recording_id'] = key.pop('recording_id')\n",
+ "img_key[\"recording_id\"] = key.pop(\"recording_id\")\n",
"session_key = key\n",
"session_key"
]
@@ -433,7 +454,7 @@
}
],
"source": [
- "imaging_dir = (recording.Recording & img_key).fetch1('recording_directory')\n",
+ "imaging_dir = (recording.Recording & img_key).fetch1(\"recording_directory\")\n",
"imaging_dir = pathlib.Path(imaging_root_dir, imaging_dir)\n",
"imaging_dir"
]
@@ -541,7 +562,9 @@
}
],
"source": [
- "imaging_files = [file.as_posix() for file in imaging_dir.rglob(\"*.tif\") if file.is_file()]\n",
+ "imaging_files = [\n",
+ " file.as_posix() for file in imaging_dir.rglob(\"*.tif\") if file.is_file()\n",
+ "]\n",
"imaging_files.sort()\n",
"imaging_files"
]
@@ -570,8 +593,8 @@
],
"source": [
"behavior_data = (behavior.SpatialTimeBlobs & session_key).fetch(as_dict=True)\n",
- "session_time = behavior_data[0]['cumulative_session_time'].flatten()\n",
- "beh_iteration_matrix = behavior_data[0]['iteration_matrix']\n",
+ "session_time = behavior_data[0][\"cumulative_session_time\"].flatten()\n",
+ "beh_iteration_matrix = behavior_data[0][\"iteration_matrix\"]\n",
"beh_iteration_matrix"
]
},
@@ -589,7 +612,9 @@
}
],
"source": [
- "path = (acquisition.SessionStarted & session_key).fetch1('new_remote_path_behavior_file')\n",
+ "path = (acquisition.SessionStarted & session_key).fetch1(\n",
+ " \"new_remote_path_behavior_file\"\n",
+ ")\n",
"mat_file = lab.Path().get_local_path2(path)\n",
"df_behavior_file = mu.convert_behavior_file(mat_file)"
]
@@ -636,7 +661,7 @@
}
],
"source": [
- "data['log']"
+ "data[\"log\"]"
]
},
{
@@ -645,7 +670,7 @@
"metadata": {},
"outputs": [],
"source": [
- "input_file = '/mnt/cup/braininit/Data/Raw/imaging/efonseca/efonseca_ef932_act131/20260518_g1/05182026/ef932_act131_ppc_05182026b_00003_00001.tif'"
+ "input_file = \"/mnt/cup/braininit/Data/Raw/imaging/efonseca/efonseca_ef932_act131/20260518_g1/05182026/ef932_act131_ppc_05182026b_00003_00001.tif\""
]
},
{
diff --git a/notebooks/imaging_element/read_imaging_parameters.ipynb b/notebooks/imaging_element/read_imaging_parameters.ipynb
index 34ec326b..d9f2f28b 100644
--- a/notebooks/imaging_element/read_imaging_parameters.ipynb
+++ b/notebooks/imaging_element/read_imaging_parameters.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -34,8 +35,9 @@
}
],
"source": [
- "import pandas as pd\n",
"import numpy as np\n",
+ "import pandas as pd\n",
+ "\n",
"import u19_pipeline.imaging_pipeline as ip"
]
},
@@ -57,7 +59,7 @@
}
],
"source": [
- "so = np.load('suite2p_volum_params_05162026.npy', allow_pickle=True)\n",
+ "so = np.load(\"suite2p_volum_params_05162026.npy\", allow_pickle=True)\n",
"so"
]
},
@@ -77,7 +79,7 @@
"outputs": [],
"source": [
"keyo = dict()\n",
- "keyo['recording_id'] = 678"
+ "keyo[\"recording_id\"] = 678"
]
},
{
@@ -324,7 +326,7 @@
}
],
"source": [
- "suite2p_params = all_params.loc[6, 'params']\n",
+ "suite2p_params = all_params.loc[6, \"params\"]\n",
"suite2p_params"
]
},
@@ -334,7 +336,7 @@
"metadata": {},
"outputs": [],
"source": [
- "suite2p_params['fs'] = 50\n"
+ "suite2p_params[\"fs\"] = 50"
]
},
{
@@ -343,7 +345,7 @@
"metadata": {},
"outputs": [],
"source": [
- "suite2p_params['neuropil_extract'] = False\n"
+ "suite2p_params[\"neuropil_extract\"] = False"
]
},
{
@@ -352,7 +354,9 @@
"metadata": {},
"outputs": [],
"source": [
- "ip.imaging_element.ProcessingParamSet.insert_new_params(all_params.loc[2, 'processing_method'],6,\"182-suite2p-params-2026-5\",lo)"
+ "ip.imaging_element.ProcessingParamSet.insert_new_params(\n",
+ " all_params.loc[2, \"processing_method\"], 6, \"182-suite2p-params-2026-5\", lo\n",
+ ")"
]
},
{
@@ -361,7 +365,12 @@
"metadata": {},
"outputs": [],
"source": [
- "ip.imaging_element.ProcessingParamSet.insert_new_params(all_params.loc[2, 'processing_method'],4,\"182-suite2p-params-2026-3\",suite2p_params)"
+ "ip.imaging_element.ProcessingParamSet.insert_new_params(\n",
+ " all_params.loc[2, \"processing_method\"],\n",
+ " 4,\n",
+ " \"182-suite2p-params-2026-3\",\n",
+ " suite2p_params,\n",
+ ")"
]
},
{
@@ -371,6 +380,7 @@
"outputs": [],
"source": [
"import json\n",
+ "\n",
"with open(\"parameters_imaging.json\", \"w\") as f:\n",
" json.dump(suite2p_params, f)"
]
diff --git a/notebooks/imaging_element/read_params.py b/notebooks/imaging_element/read_params.py
index 595cf0d7..88dee707 100644
--- a/notebooks/imaging_element/read_params.py
+++ b/notebooks/imaging_element/read_params.py
@@ -1,24 +1,27 @@
-
import os
+
from scipy.io import savemat
this_dir = os.path.dirname(__file__)
os.chdir(this_dir)
import datajoint as dj
+
dj.conn()
-ephys_element =dj.create_virtual_module('u19_pipeline_ephys_element','u19_pipeline_ephys_element')
-imaging_element =dj.create_virtual_module('u19_pipeline_imaging_element','u19_pipeline_imaging_element')
+ephys_element = dj.create_virtual_module("u19_pipeline_ephys_element", "u19_pipeline_ephys_element")
+imaging_element = dj.create_virtual_module("u19_pipeline_imaging_element", "u19_pipeline_imaging_element")
-modalities = ['electrophysiology', 'imaging']
+modalities = ["electrophysiology", "imaging"]
-params_tables = [ephys_element.ClusteringParamSet, imaging_element.ProcessingParamSet]
-preparams_steps_tables = [(ephys_element.PreClusterParamSteps * ephys_element.PreClusterParamSteps.Step * ephys_element.PreClusterParamSet)]
-preparams_tables = [ephys_element.PreClusterParamSet]
+params_tables = [ephys_element.ClusteringParamSet, imaging_element.ProcessingParamSet]
+preparams_steps_tables = [
+ (ephys_element.PreClusterParamSteps * ephys_element.PreClusterParamSteps.Step * ephys_element.PreClusterParamSet)
+]
+preparams_tables = [ephys_element.PreClusterParamSet]
-#method_tables = [ephys_element.ClusteringMethod, imaging_element.ProcessingMethod]
-#pre_method_tables = [ephys_element.PreClusterMethod]
+# method_tables = [ephys_element.ClusteringMethod, imaging_element.ProcessingMethod]
+# pre_method_tables = [ephys_element.PreClusterMethod]
###################################Fetch all params from all modalities
params_dict_list = []
@@ -26,42 +29,42 @@
params_dict_list.append(table.fetch(as_dict=True))
-#Append all params in the same dictionary
+# Append all params in the same dictionary
params_dict_dict = {}
num_params = 0
for idx, param_modality_list in enumerate(params_dict_list):
- for dict in param_modality_list:
- dict['recording_modality'] = modalities[idx]
- dict['param_set_hash'] = str(dict['param_set_hash'])
- if 'clustering_method' in dict:
- dict['processing_method'] = dict.pop('clustering_method')
-
- params_dict_dict['param_'+str(num_params)] = dict
- num_params +=1
+ for entry in param_modality_list:
+ entry["recording_modality"] = modalities[idx]
+ entry["param_set_hash"] = str(entry["param_set_hash"])
+ if "clustering_method" in entry:
+ entry["processing_method"] = entry.pop("clustering_method")
+
+ params_dict_dict["param_" + str(num_params)] = entry
+ num_params += 1
#################################################Fetch all preparamsStepList from all modalities
preparams_steps = []
for table in preparams_steps_tables:
preparams_steps.append(table.fetch(as_dict=True))
-#Append all preparams in the same dictionary
+# Append all preparams in the same dictionary
preparams_steps_dict_dict = {}
num_preparams_steps = 0
for idx, preparam_modality_list in enumerate(preparams_steps):
- for dict in preparam_modality_list:
- dict['param_set_hash'] = str(dict['param_set_hash'])
- dict['recording_modality'] = modalities[idx]
- if 'precluster_param_steps_id' in dict:
- dict['preprocess_param_steps_id'] = dict.pop('precluster_param_steps_id')
- if 'precluster_method' in dict:
- dict['preprocess_method'] = dict.pop('precluster_method')
- if 'precluster_param_steps_name' in dict:
- dict['preprocess_param_steps_name'] = dict.pop('precluster_param_steps_name')
- if 'precluster_param_steps_desc' in dict:
- dict['preprocess_param_steps_desc'] = dict.pop('precluster_param_steps_desc')
-
- preparams_steps_dict_dict['param_'+str(num_preparams_steps)] = dict
- num_preparams_steps +=1
+ for entry in preparam_modality_list:
+ entry["param_set_hash"] = str(entry["param_set_hash"])
+ entry["recording_modality"] = modalities[idx]
+ if "precluster_param_steps_id" in entry:
+ entry["preprocess_param_steps_id"] = entry.pop("precluster_param_steps_id")
+ if "precluster_method" in entry:
+ entry["preprocess_method"] = entry.pop("precluster_method")
+ if "precluster_param_steps_name" in entry:
+ entry["preprocess_param_steps_name"] = entry.pop("precluster_param_steps_name")
+ if "precluster_param_steps_desc" in entry:
+ entry["preprocess_param_steps_desc"] = entry.pop("precluster_param_steps_desc")
+
+ preparams_steps_dict_dict["param_" + str(num_preparams_steps)] = entry
+ num_preparams_steps += 1
#################################################Fetch all preparams from all modalities
preparams_dict_list = []
@@ -69,20 +72,20 @@
preparams_dict_list.append(table.fetch(as_dict=True))
-#Append all preparams in the same dictionary
+# Append all preparams in the same dictionary
preparams_dict_dict = {}
num_preparams = 0
for idx, preparam_modality_list in enumerate(preparams_dict_list):
- for dict in preparam_modality_list:
- dict['recording_modality'] = modalities[idx]
- dict['param_set_hash'] = str(dict['param_set_hash'])
- if 'precluster_method' in dict:
- dict['preprocess_method'] = dict.pop('precluster_method')
-
- preparams_dict_dict['param_'+str(num_preparams)] = dict
- num_preparams +=1
+ for entry in preparam_modality_list:
+ entry["recording_modality"] = modalities[idx]
+ entry["param_set_hash"] = str(entry["param_set_hash"])
+ if "precluster_method" in entry:
+ entry["preprocess_method"] = entry.pop("precluster_method")
+
+ preparams_dict_dict["param_" + str(num_preparams)] = entry
+ num_preparams += 1
-'''
+"""
#################################################Fetch all methods from all modalities
all_methods_data = []
for table in method_tables:
@@ -114,14 +117,14 @@
premethods_dict['premethod_'+str(num_preparams_steps)] = dict
num_premethods +=1
-'''
+"""
dj.conn().close()
-savemat('params.mat', params_dict_dict)
-savemat('preparams.mat', preparams_dict_dict)
-savemat('preparams_list.mat', preparams_steps_dict_dict)
+savemat("params.mat", params_dict_dict)
+savemat("preparams.mat", preparams_dict_dict)
+savemat("preparams_list.mat", preparams_steps_dict_dict)
-#savemat('methods.mat', methods_dict)
-#savemat('premethods.mat', premethods_dict)
+# savemat('methods.mat', methods_dict)
+# savemat('premethods.mat', premethods_dict)
diff --git a/notebooks/imaging_element/test_Matlab_Python_conv.ipynb b/notebooks/imaging_element/test_Matlab_Python_conv.ipynb
index c801ccc8..2879c21e 100644
--- a/notebooks/imaging_element/test_Matlab_Python_conv.ipynb
+++ b/notebooks/imaging_element/test_Matlab_Python_conv.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -34,13 +35,12 @@
}
],
"source": [
- "import pandas as pd\n",
- "import numpy as np\n",
"import datajoint as dj\n",
+ "import pandas as pd\n",
+ "\n",
"import u19_pipeline.imaging_pipeline as ip\n",
- "import matplotlib.pyplot as plt\n",
- "import u19_pipeline.utils.tiff_utils as tu\n",
- "import u19_pipeline.utils.tiff_matlab_imaging_utils as tmiu\n"
+ "import u19_pipeline.utils.tiff_matlab_imaging_utils as tmiu\n",
+ "import u19_pipeline.utils.tiff_utils as tu"
]
},
{
@@ -61,6 +61,7 @@
],
"source": [
"import importlib\n",
+ "\n",
"importlib.reload(ip)\n",
"importlib.reload(tu)\n",
"importlib.reload(tmiu)"
@@ -72,7 +73,7 @@
"metadata": {},
"outputs": [],
"source": [
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -115,7 +116,7 @@
}
],
"source": [
- "acquired_tiff_keys = ip.AcquiredTiff.fetch('KEY')\n",
+ "acquired_tiff_keys = ip.AcquiredTiff.fetch(\"KEY\")\n",
"acquired_tiff_keys"
]
},
@@ -137,8 +138,8 @@
],
"source": [
"imaging_key = dict()\n",
- "imaging_key['recording_id'] = 684\n",
- "#imaging_key = acquired_tiff_keys[16].copy()\n",
+ "imaging_key[\"recording_id\"] = 684\n",
+ "# imaging_key = acquired_tiff_keys[16].copy()\n",
"imaging_key\n",
"original_imaging_key = imaging_key.copy()\n",
"original_imaging_key"
@@ -237,7 +238,7 @@
"outputs": [],
"source": [
"acquired_tiff_entry = (ip.AcquiredTiff & original_imaging_key).fetch(as_dict=True)\n",
- "tiff_split_entry = (ip.TiffSplit & original_imaging_key).fetch(as_dict=True)\n",
+ "tiff_split_entry = (ip.TiffSplit & original_imaging_key).fetch(as_dict=True)\n",
"tiff_split_file_entry = (ip.TiffSplit.File & original_imaging_key).fetch(as_dict=True)"
]
},
diff --git a/notebooks/imaging_storage_calculator.ipynb b/notebooks/imaging_storage_calculator.ipynb
index 7da8a08e..8caf76ad 100644
--- a/notebooks/imaging_storage_calculator.ipynb
+++ b/notebooks/imaging_storage_calculator.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -36,7 +37,8 @@
],
"source": [
"import numpy as np\n",
- "(np.round(100.76 * 2)/2)"
+ "\n",
+ "(np.round(100.76 * 2) / 2)"
]
},
{
@@ -46,10 +48,10 @@
"outputs": [],
"source": [
"import datajoint as dj\n",
+ "import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
- "import subprocess\n",
- "import u19_pipeline.utils.path_utils as pu\n",
- "import matplotlib.pyplot as plt"
+ "\n",
+ "import u19_pipeline.utils.path_utils as pu"
]
},
{
@@ -82,9 +84,9 @@
}
],
"source": [
- "lab = dj.create_virtual_module('lab', 'u19_lab')\n",
+ "lab = dj.create_virtual_module(\"lab\", \"u19_lab\")\n",
"\n",
- "lab.SlackWebhooks.fetch(as_dict=True)\n"
+ "lab.SlackWebhooks.fetch(as_dict=True)"
]
},
{
@@ -102,11 +104,11 @@
],
"source": [
"a = dict()\n",
- "a['s'] = 2\n",
- "if 't' not in a:\n",
- " print('not here t')\n",
- "if 's' not in a:\n",
- " print('not s either')"
+ "a[\"s\"] = 2\n",
+ "if \"t\" not in a:\n",
+ " print(\"not here t\")\n",
+ "if \"s\" not in a:\n",
+ " print(\"not s either\")"
]
},
{
@@ -329,9 +331,13 @@
}
],
"source": [
- "imaging = dj.create_virtual_module('imaging', 'u19_imaging')\n",
- "subject = dj.create_virtual_module('subject', 'u19_subject')\n",
- "df_imaging = pd.DataFrame((imaging.Scan * subject.Subject.proj('subject_fullname', 'user_id')).fetch(as_dict=True))\n",
+ "imaging = dj.create_virtual_module(\"imaging\", \"u19_imaging\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n",
+ "df_imaging = pd.DataFrame(\n",
+ " (imaging.Scan * subject.Subject.proj(\"subject_fullname\", \"user_id\")).fetch(\n",
+ " as_dict=True\n",
+ " )\n",
+ ")\n",
"df_imaging"
]
},
@@ -341,9 +347,11 @@
"metadata": {},
"outputs": [],
"source": [
- "#for i in range(df_imaging.shape[0])\n",
- "df_imaging['size (TB)'] = df_imaging.apply(lambda x: pu.get_size_directory(x['scan_directory']),axis=1)\n",
- "df_imaging['size (TB)'] = df_imaging['size (TB)']/(1024*1024*1024)"
+ "# for i in range(df_imaging.shape[0])\n",
+ "df_imaging[\"size (TB)\"] = df_imaging.apply(\n",
+ " lambda x: pu.get_size_directory(x[\"scan_directory\"]), axis=1\n",
+ ")\n",
+ "df_imaging[\"size (TB)\"] = df_imaging[\"size (TB)\"] / (1024 * 1024 * 1024)"
]
},
{
@@ -352,9 +360,24 @@
"metadata": {},
"outputs": [],
"source": [
- "df_imaging['is_raw_imaging'] = df_imaging.apply(lambda x: pu.check_file_pattern_dir(x['scan_directory'], pu.file_patterns_acq['raw_imaging']),axis=1)\n",
- "df_imaging['is_commpressed_imaging'] = df_imaging.apply(lambda x: pu.check_file_pattern_dir(x['scan_directory'], pu.file_patterns_acq['commpressed_imaging']),axis=1)\n",
- "df_imaging['is_segmented'] = df_imaging.apply(lambda x: pu.check_file_pattern_dir(x['scan_directory'], pu.file_patterns_acq['segmented_imaging_files']),axis=1)"
+ "df_imaging[\"is_raw_imaging\"] = df_imaging.apply(\n",
+ " lambda x: pu.check_file_pattern_dir(\n",
+ " x[\"scan_directory\"], pu.file_patterns_acq[\"raw_imaging\"]\n",
+ " ),\n",
+ " axis=1,\n",
+ ")\n",
+ "df_imaging[\"is_commpressed_imaging\"] = df_imaging.apply(\n",
+ " lambda x: pu.check_file_pattern_dir(\n",
+ " x[\"scan_directory\"], pu.file_patterns_acq[\"commpressed_imaging\"]\n",
+ " ),\n",
+ " axis=1,\n",
+ ")\n",
+ "df_imaging[\"is_segmented\"] = df_imaging.apply(\n",
+ " lambda x: pu.check_file_pattern_dir(\n",
+ " x[\"scan_directory\"], pu.file_patterns_acq[\"segmented_imaging_files\"]\n",
+ " ),\n",
+ " axis=1,\n",
+ ")"
]
},
{
@@ -378,7 +401,12 @@
"metadata": {},
"outputs": [],
"source": [
- "df_imaging['session_type'] = df_imaging.apply(lambda x: session_type(x['is_commpressed_imaging'], x['is_raw_imaging'], x['is_segmented']),axis=1)"
+ "df_imaging[\"session_type\"] = df_imaging.apply(\n",
+ " lambda x: session_type(\n",
+ " x[\"is_commpressed_imaging\"], x[\"is_raw_imaging\"], x[\"is_segmented\"]\n",
+ " ),\n",
+ " axis=1,\n",
+ ")"
]
},
{
@@ -387,8 +415,8 @@
"metadata": {},
"outputs": [],
"source": [
- "df_imaging['session_date'] = df_imaging['session_date'].astype('str')\n",
- "df_imaging['year'] = df_imaging['session_date'].str.slice(0,4).astype('int')"
+ "df_imaging[\"session_date\"] = df_imaging[\"session_date\"].astype(\"str\")\n",
+ "df_imaging[\"year\"] = df_imaging[\"session_date\"].str.slice(0, 4).astype(\"int\")"
]
},
{
@@ -397,7 +425,7 @@
"metadata": {},
"outputs": [],
"source": [
- "df_imaging = df_imaging.loc[df_imaging['size (TB)'] > 0.001, :]"
+ "df_imaging = df_imaging.loc[df_imaging[\"size (TB)\"] > 0.001, :]"
]
},
{
@@ -2500,10 +2528,12 @@
}
],
"source": [
- "n, bins, patches = plt.hist(df_imaging['size (TB)'].values , 100, facecolor='g', alpha=0.75)\n",
- "plt.xlabel('Size per session (TB)')\n",
- "plt.ylabel('Count ')\n",
- "plt.title('Histogram of Imaging Sessions')\n",
+ "n, bins, patches = plt.hist(\n",
+ " df_imaging[\"size (TB)\"].values, 100, facecolor=\"g\", alpha=0.75\n",
+ ")\n",
+ "plt.xlabel(\"Size per session (TB)\")\n",
+ "plt.ylabel(\"Count \")\n",
+ "plt.title(\"Histogram of Imaging Sessions\")\n",
"plt.grid(True)\n",
"plt.show()"
]
@@ -2626,7 +2656,6 @@
}
],
"source": [
- "\n",
"df_imaging[[\"user_id\", \"size (TB)\", \"year\"]].groupby(by=[\"user_id\", \"year\"]).sum()"
]
},
@@ -2763,7 +2792,9 @@
}
],
"source": [
- "df_imaging[[\"subject_fullname\", \"user_id\", \"session_type\"]].groupby(by=[\"user_id\", \"session_type\"]).count()"
+ "df_imaging[[\"subject_fullname\", \"user_id\", \"session_type\"]].groupby(\n",
+ " by=[\"user_id\", \"session_type\"]\n",
+ ").count()"
]
},
{
@@ -2986,7 +3017,9 @@
}
],
"source": [
- "df_imaging[[\"user_id\", \"session_type\", \"size (TB)\"]].groupby(by=[\"user_id\", \"session_type\"]).mean()\n"
+ "df_imaging[[\"user_id\", \"session_type\", \"size (TB)\"]].groupby(\n",
+ " by=[\"user_id\", \"session_type\"]\n",
+ ").mean()"
]
},
{
diff --git a/notebooks/live_stats_monitor.ipynb b/notebooks/live_stats_monitor.ipynb
index 3022d9b4..bbdfec85 100644
--- a/notebooks/live_stats_monitor.ipynb
+++ b/notebooks/live_stats_monitor.ipynb
@@ -22,9 +22,8 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
- "try_find_conf_file()\n",
"\n",
- "\n"
+ "try_find_conf_file()"
]
},
{
@@ -33,12 +32,13 @@
"metadata": {},
"outputs": [],
"source": [
+ "import time\n",
+ "from datetime import datetime, timedelta\n",
+ "from zoneinfo import ZoneInfo\n",
+ "\n",
"import datajoint as dj\n",
"import numpy as np\n",
"import pandas as pd\n",
- "import time\n",
- "from zoneinfo import ZoneInfo\n",
- "from datetime import datetime, timedelta\n",
"\n",
"import u19_pipeline.utils.slack_utils as su"
]
@@ -59,7 +59,7 @@
"dj.conn()\n",
"\n",
"MINUTES_ALERT = 20\n",
- "SECONDS_ALERT = MINUTES_ALERT*60\n",
+ "SECONDS_ALERT = MINUTES_ALERT * 60\n",
"MIN_SESSIONS_COMPLETED = 3"
]
},
@@ -76,9 +76,9 @@
"metadata": {},
"outputs": [],
"source": [
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "lab = dj.create_virtual_module('lab', 'u19_lab')\n",
- "subject = dj.create_virtual_module('subject', 'u19_subject')"
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "lab = dj.create_virtual_module(\"lab\", \"u19_lab\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")"
]
},
{
@@ -87,9 +87,7 @@
"metadata": {},
"outputs": [],
"source": [
- "slack_configuration_dictionary = {\n",
- " 'slack_notification_channel': ['alvaro_luna']\n",
- "}"
+ "slack_configuration_dictionary = {\"slack_notification_channel\": [\"alvaro_luna\"]}"
]
},
{
@@ -98,49 +96,58 @@
"metadata": {},
"outputs": [],
"source": [
- "def slack_alert_message_format_live_stats(alert_dictionary1, alert_dictionary2, time_no_response):\n",
+ "def slack_alert_message_format_live_stats(\n",
+ " alert_dictionary1, alert_dictionary2, time_no_response\n",
+ "):\n",
"\n",
" now = datetime.now()\n",
- " datestr = now.strftime('%d-%b-%Y %H:%M:%S')\n",
+ " datestr = now.strftime(\"%d-%b-%Y %H:%M:%S\")\n",
"\n",
" msep = dict()\n",
- " msep['type'] = \"divider\"\n",
+ " msep[\"type\"] = \"divider\"\n",
"\n",
- " #Title#\n",
+ " # Title#\n",
" m1 = dict()\n",
- " m1['type'] = 'section'\n",
+ " m1[\"type\"] = \"section\"\n",
" m1_1 = dict()\n",
" m1_1[\"type\"] = \"mrkdwn\"\n",
- " m1_1[\"text\"] = ':rotating_light: * Live Monitor Alert* on ' + datestr + '\\n' +\\\n",
- " 'More than ' + str(int(time_no_response/60)) + ' min without valid new trial' + '\\n'\n",
- " m1['text'] = m1_1\n",
- "\n",
- " #Info#\n",
+ " m1_1[\"text\"] = (\n",
+ " \":rotating_light: * Live Monitor Alert* on \"\n",
+ " + datestr\n",
+ " + \"\\n\"\n",
+ " + \"More than \"\n",
+ " + str(int(time_no_response / 60))\n",
+ " + \" min without valid new trial\"\n",
+ " + \"\\n\"\n",
+ " )\n",
+ " m1[\"text\"] = m1_1\n",
+ "\n",
+ " # Info#\n",
" m2 = dict()\n",
- " m2['type'] = 'section'\n",
+ " m2[\"type\"] = \"section\"\n",
" m2_1 = dict()\n",
" m2_1[\"type\"] = \"mrkdwn\"\n",
"\n",
- " m2_1[\"text\"] = '*Session Reported:*' + '\\n'\n",
+ " m2_1[\"text\"] = \"*Session Reported:*\" + \"\\n\"\n",
" for key in alert_dictionary1.keys():\n",
- " m2_1[\"text\"] += '*' + key + '* : ' + str(alert_dictionary1[key]) + '\\n'\n",
- " m2_1[\"text\"] += '\\n'\n",
- " m2['text'] = m2_1\n",
+ " m2_1[\"text\"] += \"*\" + key + \"* : \" + str(alert_dictionary1[key]) + \"\\n\"\n",
+ " m2_1[\"text\"] += \"\\n\"\n",
+ " m2[\"text\"] = m2_1\n",
"\n",
" m4 = dict()\n",
- " m4['type'] = 'section'\n",
+ " m4[\"type\"] = \"section\"\n",
" m4_1 = dict()\n",
" m4_1[\"type\"] = \"mrkdwn\"\n",
"\n",
- " m4_1[\"text\"] = '*Last Stats Reported*:' + '\\n'\n",
+ " m4_1[\"text\"] = \"*Last Stats Reported*:\" + \"\\n\"\n",
" for key in alert_dictionary2.keys():\n",
- " m4_1[\"text\"] += '*' + key + '* : ' + str(alert_dictionary2[key]) + '\\n'\n",
- " m4_1[\"text\"] += '\\n'\n",
- " m4['text'] = m4_1\n",
+ " m4_1[\"text\"] += \"*\" + key + \"* : \" + str(alert_dictionary2[key]) + \"\\n\"\n",
+ " m4_1[\"text\"] += \"\\n\"\n",
+ " m4[\"text\"] = m4_1\n",
"\n",
" message = dict()\n",
- " message['blocks'] = [m1,msep,m2,msep,m4,msep]\n",
- " message['text'] = 'Live Monitor Alert'\n",
+ " message[\"blocks\"] = [m1, msep, m2, msep, m4, msep]\n",
+ " message[\"text\"] = \"Live Monitor Alert\"\n",
"\n",
" return message"
]
@@ -225,165 +232,253 @@
"source": [
"# Query today's started sessions that are not finished\n",
"query = {}\n",
- "query['session_date'] = datetime.today().strftime('%Y-%m-%d')\n",
- "query['is_finished'] = 0\n",
+ "query[\"session_date\"] = datetime.today().strftime(\"%Y-%m-%d\")\n",
+ "query[\"is_finished\"] = 0\n",
"\n",
- "#Only look for sessions started in the last 1:30\n",
- "last_time_start = datetime.now(tz=ZoneInfo('America/New_York')) - timedelta(hours=2,minutes=30)\n",
- "last_time_start = last_time_start.replace(tzinfo=None).strftime('%Y-%m-%d %H:%M:%S')\n",
+ "# Only look for sessions started in the last 1:30\n",
+ "last_time_start = datetime.now(tz=ZoneInfo(\"America/New_York\")) - timedelta(\n",
+ " hours=2, minutes=30\n",
+ ")\n",
+ "last_time_start = last_time_start.replace(tzinfo=None).strftime(\"%Y-%m-%d %H:%M:%S\")\n",
"\n",
- "query_started_recently = \"session_start_time > '\" + last_time_start + \"'\" \n",
- "sessions = pd.DataFrame((acquisition.SessionStarted & query & query_started_recently).fetch('KEY','session_location','session_start_time',as_dict=True))\n",
- "#sessions = sessions.loc[~sessions['subject_fullname'].str.startswith('testuser'),:]\n",
+ "query_started_recently = \"session_start_time > '\" + last_time_start + \"'\"\n",
+ "sessions = pd.DataFrame(\n",
+ " (acquisition.SessionStarted & query & query_started_recently).fetch(\n",
+ " \"KEY\", \"session_location\", \"session_start_time\", as_dict=True\n",
+ " )\n",
+ ")\n",
+ "# sessions = sessions.loc[~sessions['subject_fullname'].str.startswith('testuser'),:]\n",
"\n",
- "print('sessions STARTED RECENTLY\\n', sessions)\n",
+ "print(\"sessions STARTED RECENTLY\\n\", sessions)\n",
"\n",
"if sessions.shape[0] > 0:\n",
- "\n",
- " #If more than one \"not finished\" session in same rig, grab the last one started\n",
- " sessions2 = sessions.groupby('session_location').agg({'session_start_time': [('session_start_time', 'max')]})\n",
+ " # If more than one \"not finished\" session in same rig, grab the last one started\n",
+ " sessions2 = sessions.groupby(\"session_location\").agg(\n",
+ " {\"session_start_time\": [(\"session_start_time\", \"max\")]}\n",
+ " )\n",
" sessions2.columns = sessions2.columns.droplevel()\n",
" sessions2 = sessions2.reset_index()\n",
- " sessions = pd.merge(sessions, sessions2, on=['session_location', 'session_start_time'])\n",
- " sessions = sessions.drop(columns=['session_location'])\n",
+ " sessions = pd.merge(\n",
+ " sessions, sessions2, on=[\"session_location\", \"session_start_time\"]\n",
+ " )\n",
+ " sessions = sessions.drop(columns=[\"session_location\"])\n",
" sessions = sessions.reset_index(drop=True)\n",
"\n",
- " print('Last session started on rig\\n', sessions)\n",
+ " print(\"Last session started on rig\\n\", sessions)\n",
"\n",
- " #Only analyze sessions that have not been reported\n",
- " query_reported = {} \n",
- " query_reported['session_date'] = datetime.today().strftime('%Y-%m-%d')\n",
- " sessions_reported = pd.DataFrame((acquisition.ReportedLiveSessionStats & query_reported).fetch('KEY', as_dict = True))\n",
+ " # Only analyze sessions that have not been reported\n",
+ " query_reported = {}\n",
+ " query_reported[\"session_date\"] = datetime.today().strftime(\"%Y-%m-%d\")\n",
+ " sessions_reported = pd.DataFrame(\n",
+ " (acquisition.ReportedLiveSessionStats & query_reported).fetch(\n",
+ " \"KEY\", as_dict=True\n",
+ " )\n",
+ " )\n",
"\n",
"if sessions_reported.shape[0] > 0:\n",
- "\n",
- " sessions = pd.merge(sessions,sessions_reported, how='left', indicator=True)\n",
- " sessions = sessions.loc[sessions['_merge'] == 'left_only']\n",
- " sessions = sessions.drop(columns='_merge')\n",
+ " sessions = pd.merge(sessions, sessions_reported, how=\"left\", indicator=True)\n",
+ " sessions = sessions.loc[sessions[\"_merge\"] == \"left_only\"]\n",
+ " sessions = sessions.drop(columns=\"_merge\")\n",
" sessions = sessions.reset_index(drop=True)\n",
"\n",
- "print('Sessions not reported\\n', sessions)\n",
+ "print(\"Sessions not reported\\n\", sessions)\n",
"\n",
- "#Only analyze sessions subjects > NUM_SESSIONS_COMPLETED have not been reported\n",
+ "# Only analyze sessions subjects > NUM_SESSIONS_COMPLETED have not been reported\n",
"if sessions.shape[0] > 0:\n",
+ " query_subjects = (\n",
+ " \"subject_fullname in ('\" + \"', '\".join(sessions[\"subject_fullname\"]) + \"')\"\n",
+ " )\n",
"\n",
- " query_subjects = \"subject_fullname in ('\"+ \"', '\".join(sessions['subject_fullname']) + \"')\"\n",
- "\n",
- " count_sessions_table = (subject.Subject).aggr((acquisition.Session & query_subjects), num_sessions=\"count(subject_fullname)\")\n",
+ " count_sessions_table = (subject.Subject).aggr(\n",
+ " (acquisition.Session & query_subjects), num_sessions=\"count(subject_fullname)\"\n",
+ " )\n",
" count_sessions_df = pd.DataFrame(count_sessions_table.fetch(as_dict=True))\n",
"\n",
- " sessions = pd.merge(sessions,count_sessions_df, how='left')\n",
- " sessions['num_sessions'] = sessions['num_sessions'].fillna(0)\n",
+ " sessions = pd.merge(sessions, count_sessions_df, how=\"left\")\n",
+ " sessions[\"num_sessions\"] = sessions[\"num_sessions\"].fillna(0)\n",
"\n",
- " sessions = sessions.loc[sessions['num_sessions']>= MIN_SESSIONS_COMPLETED, :]\n",
+ " sessions = sessions.loc[sessions[\"num_sessions\"] >= MIN_SESSIONS_COMPLETED, :]\n",
"\n",
- "print('Subjects with completed min_num_sessions\\n', sessions)\n",
+ "print(\"Subjects with completed min_num_sessions\\n\", sessions)\n",
"\n",
"if sessions.shape[0] > 0:\n",
- "\n",
" # Query last live stat from the started sessions\n",
- " query_live_stats = sessions.to_dict('records')\n",
- " \n",
- " #Last non violation trial in sessions\n",
+ " query_live_stats = sessions.to_dict(\"records\")\n",
+ "\n",
+ " # Last non violation trial in sessions\n",
" query_no_violation_trial = dict()\n",
- " query_no_violation_trial['violation_trial'] = 0\n",
- " lss_nvio = acquisition.SessionStarted.aggr((acquisition.LiveSessionStats & query_no_violation_trial).proj('current_datetime'), current_datetime=\"max(current_datetime)\")\n",
+ " query_no_violation_trial[\"violation_trial\"] = 0\n",
+ " lss_nvio = acquisition.SessionStarted.aggr(\n",
+ " (acquisition.LiveSessionStats & query_no_violation_trial).proj(\n",
+ " \"current_datetime\"\n",
+ " ),\n",
+ " current_datetime=\"max(current_datetime)\",\n",
+ " )\n",
" live_stats_nvio = pd.DataFrame((lss_nvio & query_live_stats).fetch(as_dict=True))\n",
- " live_stats_nvio = live_stats_nvio.rename({'current_datetime': 'last_non_violation_trial'}, axis=1)\n",
- " \n",
- " #Last violation trial in sessions\n",
- " query_violation_trial = dict()\n",
- " query_violation_trial['violation_trial'] = 1\n",
- " lss_vio = acquisition.SessionStarted.aggr((acquisition.LiveSessionStats & query_violation_trial).proj('current_datetime'), current_datetime=\"max(current_datetime)\")\n",
- " live_stats_vio = pd.DataFrame((lss_vio & query_live_stats & query_violation_trial).fetch(as_dict=True))\n",
- " live_stats_vio = live_stats_vio.rename({'current_datetime': 'last_violation_trial'}, axis=1)\n",
+ " live_stats_nvio = live_stats_nvio.rename(\n",
+ " {\"current_datetime\": \"last_non_violation_trial\"}, axis=1\n",
+ " )\n",
"\n",
+ " # Last violation trial in sessions\n",
+ " query_violation_trial = dict()\n",
+ " query_violation_trial[\"violation_trial\"] = 1\n",
+ " lss_vio = acquisition.SessionStarted.aggr(\n",
+ " (acquisition.LiveSessionStats & query_violation_trial).proj(\"current_datetime\"),\n",
+ " current_datetime=\"max(current_datetime)\",\n",
+ " )\n",
+ " live_stats_vio = pd.DataFrame(\n",
+ " (lss_vio & query_live_stats & query_violation_trial).fetch(as_dict=True)\n",
+ " )\n",
+ " live_stats_vio = live_stats_vio.rename(\n",
+ " {\"current_datetime\": \"last_violation_trial\"}, axis=1\n",
+ " )\n",
"\n",
" # Merge last violation trial and last non violation Trials from sessions\n",
" if live_stats_nvio.shape[0] > 0 and live_stats_vio.shape[0] > 0:\n",
- " live_stats = pd.merge(live_stats_nvio,live_stats_vio, how='outer')\n",
+ " live_stats = pd.merge(live_stats_nvio, live_stats_vio, how=\"outer\")\n",
" elif live_stats_nvio.shape[0] > 0:\n",
" live_stats = live_stats_nvio.copy()\n",
- " live_stats['last_violation_trial'] = pd.NaT\n",
+ " live_stats[\"last_violation_trial\"] = pd.NaT\n",
" elif live_stats_vio.shape[0] > 0:\n",
" live_stats = live_stats_vio.copy()\n",
- " live_stats['last_non_violation_trial'] = pd.NaT\n",
+ " live_stats[\"last_non_violation_trial\"] = pd.NaT\n",
" else:\n",
" live_stats = pd.DataFrame()\n",
"\n",
- "\n",
" # If there are any sessions with live stats\n",
" if live_stats.shape[0] > 0:\n",
- "\n",
- " live_stats = pd.merge(sessions,live_stats, how='inner')\n",
- " fake_date = pd.Timestamp('1900-01-01')\n",
+ " live_stats = pd.merge(sessions, live_stats, how=\"inner\")\n",
+ " fake_date = pd.Timestamp(\"1900-01-01\")\n",
"\n",
" # Filter sessions whose last trial info is greater than 300s\n",
- " right_now_est = datetime.now(tz=ZoneInfo('America/New_York'))\n",
+ " right_now_est = datetime.now(tz=ZoneInfo(\"America/New_York\"))\n",
" right_now_est = right_now_est.replace(tzinfo=None)\n",
- " live_stats['seconds_elapsed_last_stat_nvio'] = (right_now_est- live_stats['last_non_violation_trial']).dt.total_seconds()\n",
- " live_stats['alert_nvio'] = live_stats['seconds_elapsed_last_stat_nvio'] > SECONDS_ALERT\n",
- "\n",
- " live_stats['seconds_elapsed_session_started'] = (right_now_est- live_stats['session_start_time']).dt.total_seconds()\n",
- " live_stats['alert_vio'] = live_stats['seconds_elapsed_session_started'] > SECONDS_ALERT & pd.isna(live_stats['last_non_violation_trial'].isna()) & ~pd.isna(live_stats['last_violation_trial'].isna())\n",
- "\n",
- "\n",
- " live_stats = live_stats.loc[(live_stats['alert_nvio']==True) | (live_stats['alert_vio']==True),:]\n",
+ " live_stats[\"seconds_elapsed_last_stat_nvio\"] = (\n",
+ " right_now_est - live_stats[\"last_non_violation_trial\"]\n",
+ " ).dt.total_seconds()\n",
+ " live_stats[\"alert_nvio\"] = (\n",
+ " live_stats[\"seconds_elapsed_last_stat_nvio\"] > SECONDS_ALERT\n",
+ " )\n",
+ "\n",
+ " live_stats[\"seconds_elapsed_session_started\"] = (\n",
+ " right_now_est - live_stats[\"session_start_time\"]\n",
+ " ).dt.total_seconds()\n",
+ " live_stats[\"alert_vio\"] = live_stats[\n",
+ " \"seconds_elapsed_session_started\"\n",
+ " ] > SECONDS_ALERT & pd.isna(\n",
+ " live_stats[\"last_non_violation_trial\"].isna()\n",
+ " ) & ~pd.isna(live_stats[\"last_violation_trial\"].isna())\n",
+ "\n",
+ " live_stats = live_stats.loc[\n",
+ " (live_stats[\"alert_nvio\"] == True) | (live_stats[\"alert_vio\"] == True), :\n",
+ " ]\n",
"\n",
" print(live_stats)\n",
"\n",
- " #If there are any sessions to alert (more then 300s)\n",
+ " # If there are any sessions to alert (more then 300s)\n",
" if live_stats.shape[0] > 0:\n",
- "\n",
- " live_stats['current_datetime'] = live_stats[['last_violation_trial', 'last_non_violation_trial']].fillna(fake_date).max(axis=1)\n",
- " live_stats['seconds_elapsed_last_valid_stat'] = live_stats[['seconds_elapsed_last_stat_nvio', 'seconds_elapsed_session_started']].fillna(-np.inf).max(axis=1)\n",
- "\n",
- " #get_session_info to alert (plus slack researcher)\n",
- " query_live_stats_sessions = live_stats[['subject_fullname', 'session_date', 'session_number']].to_dict('records')\n",
- "\n",
- " session_data_df = pd.DataFrame(((lab.User.proj('slack') * subject.Subject.proj('user_id') *\\\n",
- " acquisition.SessionStarted.proj('session_location')) & query_live_stats_sessions).fetch(as_dict=True))\n",
- " \n",
- " session_data_df = session_data_df.rename({'slack': 'researcher'}, axis=1)\n",
- " session_data_df['researcher'] = '<@'+ session_data_df['researcher'] + '>'\n",
- " session_data_df = session_data_df[['researcher', 'subject_fullname', 'session_date', 'session_number']]\n",
- "\n",
- " #Query full live stat table\n",
- " #session_stats = live_stats.copy()\n",
- " #session_stats = session_stats.rename({'current_datetime': 'last_live_stat'}, axis=1)\n",
- " query_live_stats = live_stats[['subject_fullname', 'session_date', 'session_number', 'current_datetime']].to_dict('records')\n",
- " live_stats_mini = live_stats[['subject_fullname', 'session_date', 'session_number', 'seconds_elapsed_last_valid_stat']].copy()\n",
- " ls_full_df = pd.DataFrame((acquisition.LiveSessionStats & query_live_stats).fetch(as_dict=True))\n",
- " ls_full_df = pd.merge(ls_full_df, live_stats_mini, on=['subject_fullname', 'session_date', 'session_number'])\n",
- " ls_full_df = ls_full_df.drop(columns=['subject_fullname', 'session_date', 'session_number'])\n",
- " ls_full_df = ls_full_df.rename({'current_datetime': 'last_trial_time'}, axis=1)\n",
- "\n",
- "\n",
- " mid = ls_full_df['last_trial_time']\n",
- " ls_full_df = ls_full_df.drop(columns=['last_trial_time'])\n",
- " ls_full_df.insert(0, 'last_trial_time', mid)\n",
- "\n",
- " ls_full_dict = ls_full_df.to_dict('records')\n",
+ " live_stats[\"current_datetime\"] = (\n",
+ " live_stats[[\"last_violation_trial\", \"last_non_violation_trial\"]]\n",
+ " .fillna(fake_date)\n",
+ " .max(axis=1)\n",
+ " )\n",
+ " live_stats[\"seconds_elapsed_last_valid_stat\"] = (\n",
+ " live_stats[\n",
+ " [\n",
+ " \"seconds_elapsed_last_stat_nvio\",\n",
+ " \"seconds_elapsed_session_started\",\n",
+ " ]\n",
+ " ]\n",
+ " .fillna(-np.inf)\n",
+ " .max(axis=1)\n",
+ " )\n",
+ "\n",
+ " # get_session_info to alert (plus slack researcher)\n",
+ " query_live_stats_sessions = live_stats[\n",
+ " [\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ " ].to_dict(\"records\")\n",
+ "\n",
+ " session_data_df = pd.DataFrame(\n",
+ " (\n",
+ " (\n",
+ " lab.User.proj(\"slack\")\n",
+ " * subject.Subject.proj(\"user_id\")\n",
+ " * acquisition.SessionStarted.proj(\"session_location\")\n",
+ " )\n",
+ " & query_live_stats_sessions\n",
+ " ).fetch(as_dict=True)\n",
+ " )\n",
+ "\n",
+ " session_data_df = session_data_df.rename({\"slack\": \"researcher\"}, axis=1)\n",
+ " session_data_df[\"researcher\"] = \"<@\" + session_data_df[\"researcher\"] + \">\"\n",
+ " session_data_df = session_data_df[\n",
+ " [\"researcher\", \"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ " ]\n",
+ "\n",
+ " # Query full live stat table\n",
+ " # session_stats = live_stats.copy()\n",
+ " # session_stats = session_stats.rename({'current_datetime': 'last_live_stat'}, axis=1)\n",
+ " query_live_stats = live_stats[\n",
+ " [\n",
+ " \"subject_fullname\",\n",
+ " \"session_date\",\n",
+ " \"session_number\",\n",
+ " \"current_datetime\",\n",
+ " ]\n",
+ " ].to_dict(\"records\")\n",
+ " live_stats_mini = live_stats[\n",
+ " [\n",
+ " \"subject_fullname\",\n",
+ " \"session_date\",\n",
+ " \"session_number\",\n",
+ " \"seconds_elapsed_last_valid_stat\",\n",
+ " ]\n",
+ " ].copy()\n",
+ " ls_full_df = pd.DataFrame(\n",
+ " (acquisition.LiveSessionStats & query_live_stats).fetch(as_dict=True)\n",
+ " )\n",
+ " ls_full_df = pd.merge(\n",
+ " ls_full_df,\n",
+ " live_stats_mini,\n",
+ " on=[\"subject_fullname\", \"session_date\", \"session_number\"],\n",
+ " )\n",
+ " ls_full_df = ls_full_df.drop(\n",
+ " columns=[\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ " )\n",
+ " ls_full_df = ls_full_df.rename(\n",
+ " {\"current_datetime\": \"last_trial_time\"}, axis=1\n",
+ " )\n",
+ "\n",
+ " mid = ls_full_df[\"last_trial_time\"]\n",
+ " ls_full_df = ls_full_df.drop(columns=[\"last_trial_time\"])\n",
+ " ls_full_df.insert(0, \"last_trial_time\", mid)\n",
+ "\n",
+ " ls_full_dict = ls_full_df.to_dict(\"records\")\n",
"\n",
" # Send one alert per session found\n",
" idx_alert = 0\n",
" for this_alert_record in ls_full_dict:\n",
- "\n",
- " #Format message for session and live stat dictionary\n",
- " this_session_stats = session_data_df.iloc[idx_alert,:]\n",
- " slack_json_message = slack_alert_message_format_live_stats(this_session_stats.to_dict(), this_alert_record, int(this_alert_record['seconds_elapsed_last_valid_stat']))\n",
- "\n",
- " #Send alert\n",
+ " # Format message for session and live stat dictionary\n",
+ " this_session_stats = session_data_df.iloc[idx_alert, :]\n",
+ " slack_json_message = slack_alert_message_format_live_stats(\n",
+ " this_session_stats.to_dict(),\n",
+ " this_alert_record,\n",
+ " int(this_alert_record[\"seconds_elapsed_last_valid_stat\"]),\n",
+ " )\n",
+ "\n",
+ " # Send alert\n",
" webhooks_list = su.get_webhook_list(slack_configuration_dictionary, lab)\n",
" for this_webhook in webhooks_list:\n",
" su.send_slack_notification(this_webhook, slack_json_message)\n",
" time.sleep(1)\n",
"\n",
- " reported_session = this_session_stats[['subject_fullname', 'session_date', 'session_number']].copy()\n",
- " reported_session['report_datetime'] = right_now_est\n",
+ " reported_session = this_session_stats[\n",
+ " [\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ " ].copy()\n",
+ " reported_session[\"report_datetime\"] = right_now_est\n",
"\n",
" acquisition.ReportedLiveSessionStats.insert1(reported_session.to_dict())\n",
- " idx_alert += 1\n"
+ " idx_alert += 1"
]
},
{
diff --git a/notebooks/numtrials_and_bias_check.ipynb b/notebooks/numtrials_and_bias_check.ipynb
index e2711f28..c78f0536 100644
--- a/notebooks/numtrials_and_bias_check.ipynb
+++ b/notebooks/numtrials_and_bias_check.ipynb
@@ -22,18 +22,15 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"\n",
- "import datajoint as dj\n",
"from datetime import date, timedelta\n",
+ "\n",
+ "import datajoint as dj\n",
"import pandas as pd\n",
- "import numpy as np\n",
- "import pylab as plt\n",
- "import matplotlib.patches as mpatches\n",
"\n",
- "from matplotlib import cm\n",
- "from u19_pipeline import utility\n",
- "from inspect import getmembers, isfunction\n"
+ "from u19_pipeline import utility"
]
},
{
@@ -51,8 +48,8 @@
"source": [
"utility.basic_dj_configuration(dj)\n",
"dj.conn()\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -68,50 +65,71 @@
"metadata": {},
"outputs": [],
"source": [
- "#Filter Dates\n",
+ "# Filter Dates\n",
"filter_dates = True\n",
"num_days_check = 60\n",
"current_date = date.today()\n",
"first_date_to_check = current_date - timedelta(days=num_days_check)\n",
- "dates = [first_date_to_check.strftime(\"%Y-%m-%d\"), current_date.strftime(\"%Y-%m-%d\")]\n",
- "date_label = ' and '.join(f'\"{w}\"' for w in dates)\n",
- "date_query = 'session_date between ' + date_label \n",
+ "dates = [first_date_to_check.strftime(\"%Y-%m-%d\"), current_date.strftime(\"%Y-%m-%d\")]\n",
+ "date_label = \" and \".join(f'\"{w}\"' for w in dates)\n",
+ "date_query = \"session_date between \" + date_label\n",
"\n",
"\n",
"## Filter rigs\n",
"filter_rigs = False\n",
- "rigs = ['165I-Rig1-T',\n",
- " '165I-Rig2-T',\n",
- " '165I-Rig3-T',\n",
- " '165I-Rig4-T',\n",
- " '165A-Rig5-T',\n",
- " '165A-Rig6-T',\n",
- " '165A-Rig7-T',\n",
- " '165A-Rig8-T', \n",
- " '170b-Imaging0642',\n",
- " '185f-rig1',\n",
- " '188-Rig1',\n",
- " '188-Rig2',\n",
- " 'BezosMeso',\n",
- " 'C42-Bay2-Rig1-I']\n",
- "rigs_label = ', '.join(f'\"{w}\"' for w in rigs)\n",
- "rigs_query = 'session_location in (' + rigs_label + ')'\n",
+ "rigs = [\n",
+ " \"165I-Rig1-T\",\n",
+ " \"165I-Rig2-T\",\n",
+ " \"165I-Rig3-T\",\n",
+ " \"165I-Rig4-T\",\n",
+ " \"165A-Rig5-T\",\n",
+ " \"165A-Rig6-T\",\n",
+ " \"165A-Rig7-T\",\n",
+ " \"165A-Rig8-T\",\n",
+ " \"170b-Imaging0642\",\n",
+ " \"185f-rig1\",\n",
+ " \"188-Rig1\",\n",
+ " \"188-Rig2\",\n",
+ " \"BezosMeso\",\n",
+ " \"C42-Bay2-Rig1-I\",\n",
+ "]\n",
+ "rigs_label = \", \".join(f'\"{w}\"' for w in rigs)\n",
+ "rigs_query = \"session_location in (\" + rigs_label + \")\"\n",
"\n",
"\n",
- "#Filter users\n",
+ "# Filter users\n",
"filter_user = True\n",
- "users = ['efonseca_ef255_actgp002']\n",
+ "users = [\"efonseca_ef255_actgp002\"]\n",
"users = ['subject_fullname like \"' + x + '%\"' for x in users]\n",
- "user_query = ' or '.join(f'{w}' for w in users) \n",
+ "user_query = \" or \".join(f\"{w}\" for w in users)\n",
"\n",
- "#Filter extras\n",
- "not_testuser = 'subject_fullname not like \"testuser%\"'\n",
+ "# Filter extras\n",
+ "not_testuser = 'subject_fullname not like \"testuser%\"'\n",
"not_testsubjects = 'subject_fullname not like \"%_test\"'\n",
- "not_bad_sessions = 'is_bad_session = 0'\n",
- "extra_queries = [not_testuser, not_testsubjects, not_bad_sessions]\n",
+ "not_bad_sessions = \"is_bad_session = 0\"\n",
+ "extra_queries = [not_testuser, not_testsubjects, not_bad_sessions]\n",
"\n",
- "columns_session_table = ['subject_fullname', 'session_date', 'session_number', 'session_location', 'session_performance', 'task', 'level', 'stimulus_bank', 'session_protocol', 'num_trials']\n",
- "columns_trial_table = ['subject_fullname', 'session_date', 'session_number', 'block', 'trial_idx', 'trial_type', 'choice']\n"
+ "columns_session_table = [\n",
+ " \"subject_fullname\",\n",
+ " \"session_date\",\n",
+ " \"session_number\",\n",
+ " \"session_location\",\n",
+ " \"session_performance\",\n",
+ " \"task\",\n",
+ " \"level\",\n",
+ " \"stimulus_bank\",\n",
+ " \"session_protocol\",\n",
+ " \"num_trials\",\n",
+ "]\n",
+ "columns_trial_table = [\n",
+ " \"subject_fullname\",\n",
+ " \"session_date\",\n",
+ " \"session_number\",\n",
+ " \"block\",\n",
+ " \"trial_idx\",\n",
+ " \"trial_type\",\n",
+ " \"choice\",\n",
+ "]"
]
},
{
@@ -549,23 +567,25 @@
"query = []\n",
"if filter_rigs:\n",
" query.append(rigs_query)\n",
- " \n",
+ "\n",
"if filter_dates:\n",
" query.append(date_query)\n",
- " \n",
+ "\n",
"if filter_user:\n",
" query.append(user_query)\n",
"\n",
- "#All extra stuff to filter\n",
+ "# All extra stuff to filter\n",
"for i in extra_queries:\n",
" query.append(i)\n",
- " \n",
+ "\n",
"# Filter by all conditions selected\n",
"session_table_filtered = acquisition.Session\n",
"for filter_key in query:\n",
" session_table_filtered = session_table_filtered & filter_key\n",
- " \n",
- "session_df = pd.DataFrame(session_table_filtered.fetch(*columns_session_table, as_dict=True))\n",
+ "\n",
+ "session_df = pd.DataFrame(\n",
+ " session_table_filtered.fetch(*columns_session_table, as_dict=True)\n",
+ ")\n",
"session_df"
]
},
@@ -633,11 +653,12 @@
}
],
"source": [
- "\n",
- "protocol_df = session_df.groupby('session_protocol').agg({'subject_fullname': [('num_sessions', 'count'), ('num_subjects', 'nunique')]})\n",
+ "protocol_df = session_df.groupby(\"session_protocol\").agg(\n",
+ " {\"subject_fullname\": [(\"num_sessions\", \"count\"), (\"num_subjects\", \"nunique\")]}\n",
+ ")\n",
"protocol_df.columns = protocol_df.columns.droplevel()\n",
- "#protocol_df = protocol_df.reset_index()\n",
- "protocol_df = protocol_df.sort_values(by='num_sessions', ascending=False)\n",
+ "# protocol_df = protocol_df.reset_index()\n",
+ "protocol_df = protocol_df.sort_values(by=\"num_sessions\", ascending=False)\n",
"protocol_df"
]
},
@@ -734,23 +755,32 @@
"source": [
"session_p_df = session_df.copy()\n",
"\n",
- "#Sort by subject, date and session number\n",
- "session_p_df = session_p_df.sort_values(by=['subject_fullname', 'session_date', 'session_number'], ascending=[True, False, False])\n",
+ "# Sort by subject, date and session number\n",
+ "session_p_df = session_p_df.sort_values(\n",
+ " by=[\"subject_fullname\", \"session_date\", \"session_number\"],\n",
+ " ascending=[True, False, False],\n",
+ ")\n",
"session_p_df = session_p_df.reset_index(drop=True)\n",
"\n",
"# Group metrics for each subject (total num sessions, mean num_trials, std num_trials)\n",
- "num_sessions_df = session_p_df.groupby('subject_fullname').agg({'session_date': [('total_sessions', 'count')], \\\n",
- " 'num_trials': [('avg_trials', 'mean'), ('std_trials', 'std')]})\n",
+ "num_sessions_df = session_p_df.groupby(\"subject_fullname\").agg(\n",
+ " {\n",
+ " \"session_date\": [(\"total_sessions\", \"count\")],\n",
+ " \"num_trials\": [(\"avg_trials\", \"mean\"), (\"std_trials\", \"std\")],\n",
+ " }\n",
+ ")\n",
"num_sessions_df.columns = num_sessions_df.columns.droplevel()\n",
"num_sessions_df = num_sessions_df.reset_index()\n",
"\n",
"# Merge session count to session df\n",
"session_p_df = session_p_df.merge(num_sessions_df)\n",
"\n",
- "session_p_df['z_score_trials'] = (session_p_df['num_trials'] - session_p_df['avg_trials']) / session_p_df['std_trials']\n",
+ "session_p_df[\"z_score_trials\"] = (\n",
+ " session_p_df[\"num_trials\"] - session_p_df[\"avg_trials\"]\n",
+ ") / session_p_df[\"std_trials\"]\n",
"\n",
"# Keep only today's session\n",
- "session_p_df = session_p_df.loc[session_p_df['session_date'] == current_date,:]\n",
+ "session_p_df = session_p_df.loc[session_p_df[\"session_date\"] == current_date, :]\n",
"session_p_df = session_p_df.reset_index(drop=True)\n",
"session_p_df"
]
@@ -1182,33 +1212,59 @@
}
],
"source": [
- "sort_columns = ['subject_fullname', 'session_date', 'session_number', 'block', 'trial_idx']\n",
- "session_columns = ['subject_fullname', 'session_date', 'session_number']\n",
- "trials_df = trials_df.sort_values(by=sort_columns, ascending=[True, False, False, True, True])\n",
+ "sort_columns = [\n",
+ " \"subject_fullname\",\n",
+ " \"session_date\",\n",
+ " \"session_number\",\n",
+ " \"block\",\n",
+ " \"trial_idx\",\n",
+ "]\n",
+ "session_columns = [\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ "trials_df = trials_df.sort_values(\n",
+ " by=sort_columns, ascending=[True, False, False, True, True]\n",
+ ")\n",
"trials_df = trials_df.reset_index(drop=True)\n",
"\n",
"\n",
"# Correct trials, correct to left and correct to right\n",
- "trials_df['left_trial'] = (trials_df['trial_type'] == 'L').astype(int)\n",
- "trials_df['right_trial'] = (trials_df['trial_type'] == 'R').astype(int)\n",
+ "trials_df[\"left_trial\"] = (trials_df[\"trial_type\"] == \"L\").astype(int)\n",
+ "trials_df[\"right_trial\"] = (trials_df[\"trial_type\"] == \"R\").astype(int)\n",
"\n",
- "trials_df['cum_left_trials'] = trials_df.groupby(['subject_fullname', 'session_date', 'session_number'])['left_trial'].cumsum()\n",
- "trials_df['cum_right_trials'] = trials_df.groupby(['subject_fullname', 'session_date', 'session_number'])['right_trial'].cumsum()\n",
+ "trials_df[\"cum_left_trials\"] = trials_df.groupby(\n",
+ " [\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ ")[\"left_trial\"].cumsum()\n",
+ "trials_df[\"cum_right_trials\"] = trials_df.groupby(\n",
+ " [\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ ")[\"right_trial\"].cumsum()\n",
"\n",
- "trials_df['correct_trial'] = (trials_df['trial_type'] == trials_df['choice']).astype(int)\n",
- "trials_df['correct_left'] = ((trials_df['correct_trial'] == 1) & (trials_df['trial_type'] == 'L')).astype(int)\n",
- "trials_df['correct_right'] = ((trials_df['correct_trial'] == 1) & (trials_df['trial_type'] == 'R')).astype(int)\n",
+ "trials_df[\"correct_trial\"] = (trials_df[\"trial_type\"] == trials_df[\"choice\"]).astype(\n",
+ " int\n",
+ ")\n",
+ "trials_df[\"correct_left\"] = (\n",
+ " (trials_df[\"correct_trial\"] == 1) & (trials_df[\"trial_type\"] == \"L\")\n",
+ ").astype(int)\n",
+ "trials_df[\"correct_right\"] = (\n",
+ " (trials_df[\"correct_trial\"] == 1) & (trials_df[\"trial_type\"] == \"R\")\n",
+ ").astype(int)\n",
"\n",
- "trials_df['cum_correct_trials'] = trials_df.groupby(['subject_fullname', 'session_date', 'session_number'])['correct_trial'].cumsum()\n",
- "trials_df['cum_correct_left_trials'] = trials_df.groupby(['subject_fullname', 'session_date', 'session_number'])['correct_left'].cumsum()\n",
- "trials_df['cum_correct_right_trials'] = trials_df.groupby(['subject_fullname', 'session_date', 'session_number'])['correct_right'].cumsum()\n",
+ "trials_df[\"cum_correct_trials\"] = trials_df.groupby(\n",
+ " [\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ ")[\"correct_trial\"].cumsum()\n",
+ "trials_df[\"cum_correct_left_trials\"] = trials_df.groupby(\n",
+ " [\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ ")[\"correct_left\"].cumsum()\n",
+ "trials_df[\"cum_correct_right_trials\"] = trials_df.groupby(\n",
+ " [\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ ")[\"correct_right\"].cumsum()\n",
"\n",
- "trials_df = trials_df.loc[~trials_df.duplicated(subset=session_columns, keep='last'), :]\n",
+ "trials_df = trials_df.loc[~trials_df.duplicated(subset=session_columns, keep=\"last\"), :]\n",
"trials_df = trials_df.reset_index(drop=True)\n",
"\n",
- "trials_df['bias'] = (trials_df['cum_correct_right_trials'] / trials_df['cum_right_trials']) - (trials_df['cum_correct_left_trials'] / trials_df['cum_left_trials'])\n",
+ "trials_df[\"bias\"] = (\n",
+ " trials_df[\"cum_correct_right_trials\"] / trials_df[\"cum_right_trials\"]\n",
+ ") - (trials_df[\"cum_correct_left_trials\"] / trials_df[\"cum_left_trials\"])\n",
"\n",
- "bias_df = trials_df[session_columns + ['bias']]\n",
+ "bias_df = trials_df[session_columns + [\"bias\"]]\n",
"\n",
"bias_df"
]
diff --git a/notebooks/performance_metrics_2022.ipynb b/notebooks/performance_metrics_2022.ipynb
index 4853514d..69245199 100644
--- a/notebooks/performance_metrics_2022.ipynb
+++ b/notebooks/performance_metrics_2022.ipynb
@@ -22,19 +22,14 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"\n",
+ "\n",
"import datajoint as dj\n",
- "from datetime import date, timedelta\n",
"import pandas as pd\n",
- "import numpy as np\n",
- "import pylab as plt\n",
- "import matplotlib.patches as mpatches\n",
"\n",
- "from matplotlib import cm\n",
- "from u19_pipeline import utility\n",
- "from inspect import getmembers, isfunction\n",
- "import u19_pipeline.alert_system.behavior_metrics as bm\n"
+ "import u19_pipeline.alert_system.behavior_metrics as bm"
]
},
{
@@ -59,8 +54,8 @@
],
"source": [
"dj.conn()\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -78,28 +73,28 @@
"source": [
"query_2021 = 'session_date between \"2021-01-01\" and \"2021-12-31\"'\n",
"query_2022 = 'session_date between \"2022-01-01\" and \"2022-12-31\"'\n",
- "not_testuser = 'subject_fullname not like \"testuser%\"'\n",
+ "not_testuser = 'subject_fullname not like \"testuser%\"'\n",
"not_testsubjects = 'subject_fullname not like \"%_test\"'\n",
- "not_bad_sessions = 'is_bad_session = 0'\n",
+ "not_bad_sessions = \"is_bad_session = 0\"\n",
"\n",
- "query_2021 = [query_2021, not_testuser, not_testsubjects, not_bad_sessions]\n",
- "query_2022 = [query_2022, not_testuser, not_testsubjects, not_bad_sessions]\n",
+ "query_2021 = [query_2021, not_testuser, not_testsubjects, not_bad_sessions]\n",
+ "query_2022 = [query_2022, not_testuser, not_testsubjects, not_bad_sessions]\n",
"\n",
- "session_table_filtered = acquisition.Session \n",
+ "session_table_filtered = acquisition.Session\n",
"for filter_key in query_2021:\n",
" session_table_filtered = session_table_filtered & filter_key\n",
- "subject_2021 = set((session_table_filtered).fetch('subject_fullname').tolist())\n",
+ "subject_2021 = set((session_table_filtered).fetch(\"subject_fullname\").tolist())\n",
"\n",
- "session_table_filtered = acquisition.Session \n",
+ "session_table_filtered = acquisition.Session\n",
"for filter_key in query_2022:\n",
" session_table_filtered = session_table_filtered & filter_key\n",
- "subject_2022 = set((session_table_filtered).fetch('subject_fullname').tolist())\n",
+ "subject_2022 = set((session_table_filtered).fetch(\"subject_fullname\").tolist())\n",
"subject_2022\n",
"\n",
"subjects_only_2022 = list(subject_2022 - subject_2021)\n",
"len(subjects_only_2022)\n",
"\n",
- "subjects_only_2022query = [{'subject_fullname': i} for i in subjects_only_2022]"
+ "subjects_only_2022query = [{\"subject_fullname\": i} for i in subjects_only_2022]"
]
},
{
@@ -253,10 +248,16 @@
}
],
"source": [
- "#session_protocol_q = 'session_protocol like \"%\"'\n",
+ "# session_protocol_q = 'session_protocol like \"%\"'\n",
"session_protocol_q = 'session_protocol like \"%PoissonBlocksCondensed3m%\"'\n",
- "session_df = pd.DataFrame((acquisition.Session & subjects_only_2022query & session_protocol_q).fetch('KEY','session_protocol', order_by='session_start_time', as_dict=True))\n",
- "session_keys = ((acquisition.Session & subjects_only_2022query & session_protocol_q).fetch('KEY', as_dict=True))\n",
+ "session_df = pd.DataFrame(\n",
+ " (acquisition.Session & subjects_only_2022query & session_protocol_q).fetch(\n",
+ " \"KEY\", \"session_protocol\", order_by=\"session_start_time\", as_dict=True\n",
+ " )\n",
+ ")\n",
+ "session_keys = (\n",
+ " acquisition.Session & subjects_only_2022query & session_protocol_q\n",
+ ").fetch(\"KEY\", as_dict=True)\n",
"session_df"
]
},
@@ -384,12 +385,14 @@
}
],
"source": [
- "protocol_df = session_df.groupby('session_protocol').agg({'subject_fullname': [('num_sessions', 'count'), ('num_subjects', 'nunique')]})\n",
+ "protocol_df = session_df.groupby(\"session_protocol\").agg(\n",
+ " {\"subject_fullname\": [(\"num_sessions\", \"count\"), (\"num_subjects\", \"nunique\")]}\n",
+ ")\n",
"protocol_df.columns = protocol_df.columns.droplevel()\n",
- "#protocol_df = protocol_df.reset_index()\n",
- "protocol_df = protocol_df.sort_values(by='num_sessions', ascending=False)\n",
- "protocol_df.loc['Total']= protocol_df.sum(numeric_only=True, axis=0)\n",
- "#protocol_df = pd.concat([protocol_df, ])\n",
+ "# protocol_df = protocol_df.reset_index()\n",
+ "protocol_df = protocol_df.sort_values(by=\"num_sessions\", ascending=False)\n",
+ "protocol_df.loc[\"Total\"] = protocol_df.sum(numeric_only=True, axis=0)\n",
+ "# protocol_df = pd.concat([protocol_df, ])\n",
"protocol_df.head(45)"
]
},
@@ -613,8 +616,28 @@
"source": [
"block_query = \"main_level = level and level in (11,12,15,16)\"\n",
"\n",
- "trials_df = pd.DataFrame((acquisition.Session.proj('session_location') * behavior.TowersBlock * behavior.TowersBlock.Trial() & block_query & session_keys).fetch('KEY', 'level', 'trial_type', 'choice', 'session_location', order_by='session_date', as_dict=True))\n",
- "all_sessions_df = pd.DataFrame((acquisition.Session & session_keys).fetch('KEY','session_location', order_by='session_date', as_dict=True))\n",
+ "trials_df = pd.DataFrame(\n",
+ " (\n",
+ " acquisition.Session.proj(\"session_location\")\n",
+ " * behavior.TowersBlock\n",
+ " * behavior.TowersBlock.Trial()\n",
+ " & block_query\n",
+ " & session_keys\n",
+ " ).fetch(\n",
+ " \"KEY\",\n",
+ " \"level\",\n",
+ " \"trial_type\",\n",
+ " \"choice\",\n",
+ " \"session_location\",\n",
+ " order_by=\"session_date\",\n",
+ " as_dict=True,\n",
+ " )\n",
+ ")\n",
+ "all_sessions_df = pd.DataFrame(\n",
+ " (acquisition.Session & session_keys).fetch(\n",
+ " \"KEY\", \"session_location\", order_by=\"session_date\", as_dict=True\n",
+ " )\n",
+ ")\n",
"trials_df"
]
},
@@ -631,7 +654,9 @@
"source": [
"all_sessions_df\n",
"\n",
- "summary_alls_df = all_sessions_df.groupby('session_location').agg({'subject_fullname': [('num_subjects', 'nunique'), ('num_sessions', 'count')]})\n",
+ "summary_alls_df = all_sessions_df.groupby(\"session_location\").agg(\n",
+ " {\"subject_fullname\": [(\"num_subjects\", \"nunique\"), (\"num_sessions\", \"count\")]}\n",
+ ")\n",
"summary_alls_df.columns = summary_alls_df.columns.droplevel()\n",
"summary_alls_df = summary_alls_df.reset_index()"
]
@@ -1037,9 +1062,13 @@
}
],
"source": [
- "metrics_df = bm.BehaviorMetrics.get_bias_from_trial_df(trials_df, return_all_metrics=True)\n",
- "metrics_df['performance'] = metrics_df['cum_correct_trials']*100 / metrics_df['cum_trials']\n",
- "metrics_df['bias'] = metrics_df['bias'].abs()\n",
+ "metrics_df = bm.BehaviorMetrics.get_bias_from_trial_df(\n",
+ " trials_df, return_all_metrics=True\n",
+ ")\n",
+ "metrics_df[\"performance\"] = (\n",
+ " metrics_df[\"cum_correct_trials\"] * 100 / metrics_df[\"cum_trials\"]\n",
+ ")\n",
+ "metrics_df[\"bias\"] = metrics_df[\"bias\"].abs()\n",
"metrics_df = metrics_df.reset_index(drop=True)\n",
"metrics_df"
]
@@ -1057,7 +1086,7 @@
"metadata": {},
"outputs": [],
"source": [
- "len(list(set(metrics_df['subject_fullname'].tolist())))"
+ "len(list(set(metrics_df[\"subject_fullname\"].tolist())))"
]
},
{
@@ -1068,9 +1097,9 @@
"source": [
"performance_min = 70\n",
"perf_str = str(performance_min)\n",
- "group_variable = 'session_location'\n",
+ "group_variable = \"session_location\"\n",
"\n",
- "metrics_df\n"
+ "metrics_df"
]
},
{
@@ -1426,66 +1455,107 @@
}
],
"source": [
+ "metrics_df_f = metrics_df.loc[\n",
+ " (metrics_df[\"performance\"] >= performance_min) & (metrics_df[\"cum_trials\"] > 100), :\n",
+ "]\n",
"\n",
- "\n",
- "metrics_df_f = metrics_df.loc[(metrics_df['performance'] >= performance_min) & (metrics_df['cum_trials'] > 100), :]\n",
- "\n",
- "#Group all sessions by rig\n",
- "summary_alls_df = all_sessions_df.groupby('session_location').agg({'subject_fullname': [('num_subjects', 'nunique'), ('num_sessions', 'count')]})\n",
+ "# Group all sessions by rig\n",
+ "summary_alls_df = all_sessions_df.groupby(\"session_location\").agg(\n",
+ " {\"subject_fullname\": [(\"num_subjects\", \"nunique\"), (\"num_sessions\", \"count\")]}\n",
+ ")\n",
"summary_alls_df.columns = summary_alls_df.columns.droplevel()\n",
"summary_alls_df = summary_alls_df.reset_index()\n",
"\n",
- "#Group l11 sessions by rig\n",
- "summary_l11_df = metrics_df.groupby('session_location').agg({'subject_fullname': [('num_subjects_l11', 'nunique'), ('num_sessions_l11', 'count')], 'performance': [('performance_l11', 'mean')],\\\n",
- " 'bias': [('bias_l11', 'mean')]})\n",
+ "# Group l11 sessions by rig\n",
+ "summary_l11_df = metrics_df.groupby(\"session_location\").agg(\n",
+ " {\n",
+ " \"subject_fullname\": [\n",
+ " (\"num_subjects_l11\", \"nunique\"),\n",
+ " (\"num_sessions_l11\", \"count\"),\n",
+ " ],\n",
+ " \"performance\": [(\"performance_l11\", \"mean\")],\n",
+ " \"bias\": [(\"bias_l11\", \"mean\")],\n",
+ " }\n",
+ ")\n",
"summary_l11_df.columns = summary_l11_df.columns.droplevel()\n",
"summary_l11_df = summary_l11_df.reset_index()\n",
"\n",
- "#Group l11 70 % sessions by rig\n",
- "summary_l1170_df = metrics_df_f.groupby('session_location').agg({'subject_fullname': [('num_subjects_l11>'+perf_str, 'nunique'), ('num_sessions_l11>'+perf_str, 'count')], 'performance': [('performance_l11>'+perf_str, 'mean')],\\\n",
- " 'bias': [('bias_l11>'+perf_str, 'mean')]})\n",
+ "# Group l11 70 % sessions by rig\n",
+ "summary_l1170_df = metrics_df_f.groupby(\"session_location\").agg(\n",
+ " {\n",
+ " \"subject_fullname\": [\n",
+ " (\"num_subjects_l11>\" + perf_str, \"nunique\"),\n",
+ " (\"num_sessions_l11>\" + perf_str, \"count\"),\n",
+ " ],\n",
+ " \"performance\": [(\"performance_l11>\" + perf_str, \"mean\")],\n",
+ " \"bias\": [(\"bias_l11>\" + perf_str, \"mean\")],\n",
+ " }\n",
+ ")\n",
"summary_l1170_df.columns = summary_l1170_df.columns.droplevel()\n",
"summary_l1170_df = summary_l1170_df.reset_index()\n",
"\n",
"# Merge group dataframes\n",
- "summary_l11_df = summary_alls_df.merge(summary_l11_df, on='session_location', how='left')\n",
- "summary_df = summary_l11_df.merge(summary_l1170_df, on='session_location', how='left')\n",
+ "summary_l11_df = summary_alls_df.merge(\n",
+ " summary_l11_df, on=\"session_location\", how=\"left\"\n",
+ ")\n",
+ "summary_df = summary_l11_df.merge(summary_l1170_df, on=\"session_location\", how=\"left\")\n",
"\n",
"\n",
"# Fillna, new metrics\n",
- "summary_df['num_subjects_l11'] = summary_df['num_subjects_l11'].fillna(0)\n",
- "summary_df['num_sessions_l11'] = summary_df['num_sessions_l11'].fillna(0)\n",
- "summary_df['num_sessions_l11>'+perf_str] = summary_df['num_sessions_l11>'+perf_str].fillna(0)\n",
- "summary_df['num_subjects_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str].fillna(0)\n",
+ "summary_df[\"num_subjects_l11\"] = summary_df[\"num_subjects_l11\"].fillna(0)\n",
+ "summary_df[\"num_sessions_l11\"] = summary_df[\"num_sessions_l11\"].fillna(0)\n",
+ "summary_df[\"num_sessions_l11>\" + perf_str] = summary_df[\n",
+ " \"num_sessions_l11>\" + perf_str\n",
+ "].fillna(0)\n",
+ "summary_df[\"num_subjects_l11>\" + perf_str] = summary_df[\n",
+ " \"num_subjects_l11>\" + perf_str\n",
+ "].fillna(0)\n",
"\n",
"\n",
"# Calculate Total rows\n",
- "summary_df.loc['Total']= summary_df.sum(numeric_only=True, axis=0)\n",
- "mean_columns = ['performance_l11', 'bias_l11', 'performance_l11>'+perf_str, 'bias_l11>'+perf_str]\n",
- "summary_df.loc['Total', mean_columns]= summary_df.loc[summary_df.iloc[range(summary_df.shape[0]-1)].index, mean_columns].mean(numeric_only=True, axis=0)\n",
+ "summary_df.loc[\"Total\"] = summary_df.sum(numeric_only=True, axis=0)\n",
+ "mean_columns = [\n",
+ " \"performance_l11\",\n",
+ " \"bias_l11\",\n",
+ " \"performance_l11>\" + perf_str,\n",
+ " \"bias_l11>\" + perf_str,\n",
+ "]\n",
+ "summary_df.loc[\"Total\", mean_columns] = summary_df.loc[\n",
+ " summary_df.iloc[range(summary_df.shape[0] - 1)].index, mean_columns\n",
+ "].mean(numeric_only=True, axis=0)\n",
"\n",
"# Ratio sessions / session in level & performance\n",
- "summary_df['ratio_l11'] = summary_df['num_subjects_l11']*100 /summary_df['num_subjects']\n",
- "summary_df['ratio_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str]*100 /summary_df['num_subjects']\n",
+ "summary_df[\"ratio_l11\"] = (\n",
+ " summary_df[\"num_subjects_l11\"] * 100 / summary_df[\"num_subjects\"]\n",
+ ")\n",
+ "summary_df[\"ratio_l11>\" + perf_str] = (\n",
+ " summary_df[\"num_subjects_l11>\" + perf_str] * 100 / summary_df[\"num_subjects\"]\n",
+ ")\n",
"\n",
"# Make int columns pretty\n",
- "summary_df['num_subjects'] = summary_df['num_subjects'].astype('Int64')\n",
- "summary_df['num_sessions'] = summary_df['num_sessions'].astype('Int64')\n",
- "summary_df['num_sessions_l11'] = summary_df['num_sessions_l11'].astype('Int64')\n",
- "summary_df['num_subjects_l11'] = summary_df['num_subjects_l11'].astype('Int64')\n",
- "summary_df['num_sessions_l11>'+perf_str] = summary_df['num_sessions_l11>'+perf_str].astype('Int64')\n",
- "summary_df['num_subjects_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str].astype('Int64')\n",
+ "summary_df[\"num_subjects\"] = summary_df[\"num_subjects\"].astype(\"Int64\")\n",
+ "summary_df[\"num_sessions\"] = summary_df[\"num_sessions\"].astype(\"Int64\")\n",
+ "summary_df[\"num_sessions_l11\"] = summary_df[\"num_sessions_l11\"].astype(\"Int64\")\n",
+ "summary_df[\"num_subjects_l11\"] = summary_df[\"num_subjects_l11\"].astype(\"Int64\")\n",
+ "summary_df[\"num_sessions_l11>\" + perf_str] = summary_df[\n",
+ " \"num_sessions_l11>\" + perf_str\n",
+ "].astype(\"Int64\")\n",
+ "summary_df[\"num_subjects_l11>\" + perf_str] = summary_df[\n",
+ " \"num_subjects_l11>\" + perf_str\n",
+ "].astype(\"Int64\")\n",
"\n",
"# Sort properly\n",
- "summary_df = summary_df.sort_values(by=['ratio_l11>'+perf_str, 'num_sessions_l11>'+perf_str], ascending=False)\n",
- "summary_df = summary_df.set_index('session_location')\n",
+ "summary_df = summary_df.sort_values(\n",
+ " by=[\"ratio_l11>\" + perf_str, \"num_sessions_l11>\" + perf_str], ascending=False\n",
+ ")\n",
+ "summary_df = summary_df.set_index(\"session_location\")\n",
"\n",
"# Total row at the end\n",
- "summary_df.index = summary_df.index.fillna('Total')\n",
+ "summary_df.index = summary_df.index.fillna(\"Total\")\n",
"idx = summary_df.index\n",
- "idx = idx.drop('Total')\n",
+ "idx = idx.drop(\"Total\")\n",
"idx = idx.to_list()\n",
- "summary_df = summary_df.reindex(idx + ['Total'])\n",
+ "summary_df = summary_df.reindex(idx + [\"Total\"])\n",
"summary_df"
]
},
@@ -1506,7 +1576,7 @@
}
],
"source": [
- "96/168"
+ "96 / 168"
]
},
{
@@ -1930,7 +2000,7 @@
}
],
"source": [
- "metrics_df_f = metrics_df_f.sort_values(by='performance', ascending=False)\n",
+ "metrics_df_f = metrics_df_f.sort_values(by=\"performance\", ascending=False)\n",
"metrics_df_f"
]
},
@@ -2045,8 +2115,8 @@
],
"source": [
"query_s = dict()\n",
- "query_s['subject_fullname'] = 'ms81_M024'\n",
- "query_s['session_date'] = '2022-10-01'\n",
+ "query_s[\"subject_fullname\"] = \"ms81_M024\"\n",
+ "query_s[\"session_date\"] = \"2022-10-01\"\n",
"a = pd.DataFrame((acquisition.Session & query_s).fetch(as_dict=True))\n",
"a"
]
diff --git a/notebooks/psychometrics_plot.ipynb b/notebooks/psychometrics_plot.ipynb
index b2231f31..0dc025c1 100644
--- a/notebooks/psychometrics_plot.ipynb
+++ b/notebooks/psychometrics_plot.ipynb
@@ -13,14 +13,15 @@
"metadata": {},
"outputs": [],
"source": [
+ "import datetime\n",
+ "\n",
"import datajoint as dj\n",
- "import numpy as np\n",
- "import pylab as plt\n",
"import matplotlib.patches as mpatches\n",
+ "import numpy as np\n",
"import pandas as pd\n",
- "from scipy.optimize import curve_fit\n",
+ "import pylab as plt\n",
"from astropy.stats import binom_conf_interval\n",
- "import datetime"
+ "from scipy.optimize import curve_fit"
]
},
{
@@ -36,15 +37,11 @@
"metadata": {},
"outputs": [],
"source": [
- "dj.config['database.host'] = 'datajoint00.pni.princeton.edu'\n",
- "dj.config['enable_python_native_blobs'] = True\n",
+ "dj.config[\"database.host\"] = \"datajoint00.pni.princeton.edu\"\n",
+ "dj.config[\"enable_python_native_blobs\"] = True\n",
"\n",
- "dj.config['stores'] = {\n",
- " 'extstorage':\n",
- " {\n",
- " 'location': '/Volumes/u19_dj/external_dj_blobs',\n",
- " 'protocol': 'file'\n",
- " }\n",
+ "dj.config[\"stores\"] = {\n",
+ " \"extstorage\": {\"location\": \"/Volumes/u19_dj/external_dj_blobs\", \"protocol\": \"file\"}\n",
"}"
]
},
@@ -72,7 +69,7 @@
],
"source": [
"dj.conn()\n",
- "behavior = dj.create_virtual_module('subject', 'u19_behavior')"
+ "behavior = dj.create_virtual_module(\"subject\", \"u19_behavior\")"
]
},
{
@@ -89,7 +86,7 @@
"outputs": [],
"source": [
"def psychometrics_function(x, O, A, lambd, x0):\n",
- " return O + A/(1+np.exp(-(x-x0)/lambd))"
+ " return O + A / (1 + np.exp(-(x - x0) / lambd))"
]
},
{
@@ -107,95 +104,107 @@
"source": [
"def psychFit(deltaBins, numR, numL, choices):\n",
"\n",
- " numRight = np.zeros(len(deltaBins))\n",
- " numTrials = np.zeros(len(deltaBins))\n",
- " trialDelta = np.zeros(len(deltaBins))\n",
- " phat = np.zeros(len(deltaBins))\n",
- " pci = np.zeros((2, len(deltaBins)))\n",
- " \n",
- " #Evidence variable\n",
- " nCues_RminusL = numR - numL\n",
- " #Correct deltaBin & trialBin to produce same result as Matlab psychFit\n",
- " deltaBins_search = deltaBins.astype(float) - 1.5\n",
- " trialBin = np.searchsorted(deltaBins_search, nCues_RminusL, side='right')\n",
- " trialBin -= 1;\n",
- " \n",
- " #Put into evidence bins all Trials with corresponding choices\n",
+ " numRight = np.zeros(len(deltaBins))\n",
+ " numTrials = np.zeros(len(deltaBins))\n",
+ " trialDelta = np.zeros(len(deltaBins))\n",
+ " phat = np.zeros(len(deltaBins))\n",
+ " pci = np.zeros((2, len(deltaBins)))\n",
+ "\n",
+ " # Evidence variable\n",
+ " nCues_RminusL = numR - numL\n",
+ " # Correct deltaBin & trialBin to produce same result as Matlab psychFit\n",
+ " deltaBins_search = deltaBins.astype(float) - 1.5\n",
+ " trialBin = np.searchsorted(deltaBins_search, nCues_RminusL, side=\"right\")\n",
+ " trialBin -= 1\n",
+ "\n",
+ " # Put into evidence bins all Trials with corresponding choices\n",
" for iTrial in range(len(choices)):\n",
- " numTrials[trialBin[iTrial]] = numTrials[trialBin[iTrial]] + 1\n",
+ " numTrials[trialBin[iTrial]] = numTrials[trialBin[iTrial]] + 1\n",
" if choices[iTrial] == 2:\n",
- " numRight[trialBin[iTrial]] = numRight[trialBin[iTrial]] + 1\n",
- " \n",
- " trialDelta[trialBin[iTrial]] = trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial]\n",
- " \n",
- " with np.errstate(divide='ignore', invalid='ignore'):\n",
- " trialDelta = np.true_divide(trialDelta, numTrials);\n",
- " \n",
+ " numRight[trialBin[iTrial]] = numRight[trialBin[iTrial]] + 1\n",
+ "\n",
+ " trialDelta[trialBin[iTrial]] = (\n",
+ " trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial]\n",
+ " )\n",
+ "\n",
+ " with np.errstate(divide=\"ignore\", invalid=\"ignore\"):\n",
+ " trialDelta = np.true_divide(trialDelta, numTrials)\n",
+ "\n",
" # Select only bins with trials\n",
- " idx_zero = numTrials == 0 \n",
+ " idx_zero = numTrials == 0\n",
" numTrials_nz = numTrials[~idx_zero]\n",
- " numRight_nz = numRight[~idx_zero]\n",
- " \n",
- " #(Binomial proportion confidence interval given k successes, n trials)\n",
- " phat_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=0, interval='jeffreys')\n",
- " pci_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=1-0.1587, interval='jeffreys')\n",
- " \n",
- " #Correct confidence intervals and expected outcomes for bins with no trials (ci = [0 1], hat = 0.5)\n",
+ " numRight_nz = numRight[~idx_zero]\n",
+ "\n",
+ " # (Binomial proportion confidence interval given k successes, n trials)\n",
+ " phat_nz = binom_conf_interval(\n",
+ " numRight_nz, numTrials_nz, confidence_level=0, interval=\"jeffreys\"\n",
+ " )\n",
+ " pci_nz = binom_conf_interval(\n",
+ " numRight_nz, numTrials_nz, confidence_level=1 - 0.1587, interval=\"jeffreys\"\n",
+ " )\n",
+ "\n",
+ " # Correct confidence intervals and expected outcomes for bins with no trials (ci = [0 1], hat = 0.5)\n",
" phat_nz = phat_nz[0]\n",
" phat[~idx_zero] = phat_nz\n",
- " phat[idx_zero] = 0.5\n",
+ " phat[idx_zero] = 0.5\n",
" pci[0][~idx_zero] = pci_nz[0]\n",
- " pci[0][idx_zero] = 0\n",
+ " pci[0][idx_zero] = 0\n",
" pci[1][~idx_zero] = pci_nz[1]\n",
- " pci[1][idx_zero] = 1\n",
- " \n",
+ " pci[1][idx_zero] = 1\n",
+ "\n",
" # (Logistic function fit) only valid if we have at least 5 bins with trials\n",
" if np.count_nonzero(~idx_zero) < 5:\n",
" is_there_psychometric = False\n",
" else:\n",
" is_there_psychometric = True\n",
- " #Get weight matrix to \"reproduce\" Matlab fit \n",
- " #https://stackoverflow.com/questions/58983113/scipy-curve-fit-vs-matlab-fit-weighted-nonlinear-least-squares\n",
- " # matlab -> 'Weights' , ((pci(sel,2) - pci(sel,1))/2).^-2 \n",
+ " # Get weight matrix to \"reproduce\" Matlab fit\n",
+ " # https://stackoverflow.com/questions/58983113/scipy-curve-fit-vs-matlab-fit-weighted-nonlinear-least-squares\n",
+ " # matlab -> 'Weights' , ((pci(sel,2) - pci(sel,1))/2).^-2\n",
" # python -> sigma = diagonal_matrix(1/weights)\n",
- " \n",
- " weight_array = np.power((pci[1][~idx_zero] - pci[0][~idx_zero])/2,2)\n",
- " sigma_fit = np.diag(weight_array)\n",
- " \n",
- " psychometric, pcov = curve_fit(psychometrics_function, deltaBins[~idx_zero], phat[~idx_zero], \\\n",
- " p0 = (0, 1, 3, 0),sigma = sigma_fit, maxfev=40000)\n",
"\n",
- " #Append a row of nans to confidence intervals . whyy ??\n",
- " aux_vec = np.empty((1,pci.shape[1]))\n",
+ " weight_array = np.power((pci[1][~idx_zero] - pci[0][~idx_zero]) / 2, 2)\n",
+ " sigma_fit = np.diag(weight_array)\n",
+ "\n",
+ " psychometric, pcov = curve_fit(\n",
+ " psychometrics_function,\n",
+ " deltaBins[~idx_zero],\n",
+ " phat[~idx_zero],\n",
+ " p0=(0, 1, 3, 0),\n",
+ " sigma=sigma_fit,\n",
+ " maxfev=40000,\n",
+ " )\n",
+ "\n",
+ " # Append a row of nans to confidence intervals . whyy ??\n",
+ " aux_vec = np.empty((1, pci.shape[1]))\n",
" aux_vec[:] = np.nan\n",
" pci = np.vstack((pci, aux_vec))\n",
- " \n",
+ "\n",
" # x vector for plotting\n",
- " delta = np.linspace(deltaBins[0]-2, deltaBins[-1]+2, num=50)\n",
+ " delta = np.linspace(deltaBins[0] - 2, deltaBins[-1] + 2, num=50)\n",
"\n",
" # Repeat trialDelta 3 times for errorX why ??\n",
- " errorX = np.tile(trialDelta[~idx_zero], 3);\n",
- " \n",
+ " errorX = np.tile(trialDelta[~idx_zero], 3)\n",
+ "\n",
" # Confidence intervals are errorY, as a vector\n",
- " errorY = np.stack(pci[:,~idx_zero])\n",
+ " errorY = np.stack(pci[:, ~idx_zero])\n",
" errorY = errorY.flatten()\n",
- " \n",
+ "\n",
" # Fill dictionary of results\n",
" fit_results = dict()\n",
- " fit_results['delta_bins'] = deltaBins[~idx_zero]\n",
- " fit_results['delta_data'] = trialDelta[~idx_zero];\n",
- " fit_results['pright_data'] = 100*phat[~idx_zero];\n",
- " fit_results['delta_error'] = errorX;\n",
- " fit_results['pright_error'] = 100*errorY;\n",
- " \n",
+ " fit_results[\"delta_bins\"] = deltaBins[~idx_zero]\n",
+ " fit_results[\"delta_data\"] = trialDelta[~idx_zero]\n",
+ " fit_results[\"pright_data\"] = 100 * phat[~idx_zero]\n",
+ " fit_results[\"delta_error\"] = errorX\n",
+ " fit_results[\"pright_error\"] = 100 * errorY\n",
+ "\n",
" if is_there_psychometric:\n",
- " fit_results['delta_fit'] = delta\n",
- " fit_results['pright_fit'] = psychometrics_function(delta, *psychometric)*100\n",
+ " fit_results[\"delta_fit\"] = delta\n",
+ " fit_results[\"pright_fit\"] = psychometrics_function(delta, *psychometric) * 100\n",
" else:\n",
- " fit_results['delta_fit'] = np.empty([0])\n",
- " fit_results['pright_fit'] = np.empty([0])\n",
- " \n",
- " return fit_results\n"
+ " fit_results[\"delta_fit\"] = np.empty([0])\n",
+ " fit_results[\"pright_fit\"] = np.empty([0])\n",
+ "\n",
+ " return fit_results"
]
},
{
@@ -211,51 +220,58 @@
"metadata": {},
"outputs": [],
"source": [
- "keys = [{\n",
- " 'subject_fullname': 'ms81_M005',\n",
- " 'session_date': datetime.date(2021, 5, 8),\n",
- " 'session_number': 0\n",
- "}, {\n",
- " 'subject_fullname': 'ms81_M004',\n",
- " 'session_date': datetime.date(2021, 5, 8),\n",
- " 'session_number': 0\n",
- "}, {\n",
- " 'subject_fullname': 'ms81_M004',\n",
- " 'session_date': datetime.date(2021, 5, 7),\n",
- " 'session_number': 0\n",
- "}, {\n",
- " 'subject_fullname': 'ms81_M003',\n",
- " 'session_date': datetime.date(2021, 5, 7),\n",
- " 'session_number': 0\n",
- "}, {\n",
- " 'subject_fullname': 'ms81_M004',\n",
- " 'session_date': datetime.date(2021, 5, 6),\n",
- " 'session_number': 0\n",
- "}, {\n",
- " 'subject_fullname': 'ms81_M005',\n",
- " 'session_date': datetime.date(2021, 5, 5),\n",
- " 'session_number': 0\n",
- "}, {\n",
- " 'subject_fullname': 'ms81_M002',\n",
- " 'session_date': datetime.date(2021, 5, 3),\n",
- " 'session_number': 0\n",
- "}]\n",
+ "keys = [\n",
+ " {\n",
+ " \"subject_fullname\": \"ms81_M005\",\n",
+ " \"session_date\": datetime.date(2021, 5, 8),\n",
+ " \"session_number\": 0,\n",
+ " },\n",
+ " {\n",
+ " \"subject_fullname\": \"ms81_M004\",\n",
+ " \"session_date\": datetime.date(2021, 5, 8),\n",
+ " \"session_number\": 0,\n",
+ " },\n",
+ " {\n",
+ " \"subject_fullname\": \"ms81_M004\",\n",
+ " \"session_date\": datetime.date(2021, 5, 7),\n",
+ " \"session_number\": 0,\n",
+ " },\n",
+ " {\n",
+ " \"subject_fullname\": \"ms81_M003\",\n",
+ " \"session_date\": datetime.date(2021, 5, 7),\n",
+ " \"session_number\": 0,\n",
+ " },\n",
+ " {\n",
+ " \"subject_fullname\": \"ms81_M004\",\n",
+ " \"session_date\": datetime.date(2021, 5, 6),\n",
+ " \"session_number\": 0,\n",
+ " },\n",
+ " {\n",
+ " \"subject_fullname\": \"ms81_M005\",\n",
+ " \"session_date\": datetime.date(2021, 5, 5),\n",
+ " \"session_number\": 0,\n",
+ " },\n",
+ " {\n",
+ " \"subject_fullname\": \"ms81_M002\",\n",
+ " \"session_date\": datetime.date(2021, 5, 3),\n",
+ " \"session_number\": 0,\n",
+ " },\n",
+ "]\n",
"\n",
- "#List of colors\n",
- "colors = ['r', 'y', 'k', 'b']\n",
+ "# List of colors\n",
+ "colors = [\"r\", \"y\", \"k\", \"b\"]\n",
"\n",
- "#Assign a color to each subject\n",
+ "# Assign a color to each subject\n",
"keys_df = pd.DataFrame(keys)\n",
"subjects = keys_df.subject_fullname.unique()\n",
"subjects.sort()\n",
"zip_iterator = zip(subjects, colors)\n",
"color_dict = dict(zip_iterator)\n",
"\n",
- "#Create a color patch for plot legend\n",
+ "# Create a color patch for plot legend\n",
"patches_legend = []\n",
"for key in color_dict:\n",
- " patches_legend.append(mpatches.Patch(color=color_dict[key], label=key))\n",
- "\n"
+ " patches_legend.append(mpatches.Patch(color=color_dict[key], label=key))"
]
},
{
@@ -327,51 +343,67 @@
}
],
"source": [
- "\n",
"plt.figure(figsize=(15, 12))\n",
"fraction_correct = np.zeros(len(keys))\n",
"for key_idx, key in enumerate(keys):\n",
- "\n",
" key_spec = behavior.TowersBlock() & key & \"level = 11\" & \"block_performance > 0.65\"\n",
" thissession = behavior.TowersBlock().Trial() & key_spec\n",
- " \n",
+ "\n",
" # Fetch session\n",
- " session_info = pd.DataFrame(thissession.fetch('choice', 'trial_type', 'cue_presence_left', 'cue_presence_right', as_dict=True))\n",
- " \n",
+ " session_info = pd.DataFrame(\n",
+ " thissession.fetch(\n",
+ " \"choice\",\n",
+ " \"trial_type\",\n",
+ " \"cue_presence_left\",\n",
+ " \"cue_presence_right\",\n",
+ " as_dict=True,\n",
+ " )\n",
+ " )\n",
+ "\n",
" # Choice and trial type as integer\n",
- " session_info['trial_type_int'] = 0\n",
- " session_info.loc[session_info['trial_type'] == 'L','trial_type_int'] = 1 \n",
- " session_info.loc[session_info['trial_type'] == 'R','trial_type_int'] = 2 \n",
- " session_info['choice_int'] = 0\n",
- " session_info.loc[session_info['choice'] == 'L','choice_int'] = 1 \n",
- " session_info.loc[session_info['choice'] == 'R','choice_int'] = 2\n",
- " \n",
+ " session_info[\"trial_type_int\"] = 0\n",
+ " session_info.loc[session_info[\"trial_type\"] == \"L\", \"trial_type_int\"] = 1\n",
+ " session_info.loc[session_info[\"trial_type\"] == \"R\", \"trial_type_int\"] = 2\n",
+ " session_info[\"choice_int\"] = 0\n",
+ " session_info.loc[session_info[\"choice\"] == \"L\", \"choice_int\"] = 1\n",
+ " session_info.loc[session_info[\"choice\"] == \"R\", \"choice_int\"] = 2\n",
+ "\n",
" # number of towers per trial\n",
- " session_info['cue_presence_left'] = session_info['cue_presence_left'].apply(lambda x: np.count_nonzero(x))\n",
- " session_info['cue_presence_right'] = session_info['cue_presence_right'].apply(lambda x: np.count_nonzero(x))\n",
- " \n",
- " \n",
- " fraction_correct[key_idx] = np.sum(session_info['choice'].values == session_info['trial_type'].values)/session_info.shape[0]\n",
- " \n",
+ " session_info[\"cue_presence_left\"] = session_info[\"cue_presence_left\"].apply(\n",
+ " lambda x: np.count_nonzero(x)\n",
+ " )\n",
+ " session_info[\"cue_presence_right\"] = session_info[\"cue_presence_right\"].apply(\n",
+ " lambda x: np.count_nonzero(x)\n",
+ " )\n",
+ "\n",
+ " fraction_correct[key_idx] = (\n",
+ " np.sum(session_info[\"choice\"].values == session_info[\"trial_type\"].values)\n",
+ " / session_info.shape[0]\n",
+ " )\n",
+ "\n",
" # Call Fit to sigmoid function\n",
- " fit_dict = psychFit(deltaBins, session_info['cue_presence_right'].values, \\\n",
- " session_info['cue_presence_left'].values, session_info['choice_int'].values)\n",
- " \n",
- " #Select color from color dictionary\n",
- " color_plot = color_dict[key['subject_fullname']]\n",
- " #Plot results\n",
- " plt.plot(fit_dict['delta_bins'], fit_dict['pright_data'], 'o', color = color_plot)\n",
- " if fit_dict['delta_fit'].shape[0] > 0:\n",
- " plt.plot(fit_dict['delta_fit'], fit_dict['pright_fit'], '--', color = color_plot)\n",
- " \n",
+ " fit_dict = psychFit(\n",
+ " deltaBins,\n",
+ " session_info[\"cue_presence_right\"].values,\n",
+ " session_info[\"cue_presence_left\"].values,\n",
+ " session_info[\"choice_int\"].values,\n",
+ " )\n",
+ "\n",
+ " # Select color from color dictionary\n",
+ " color_plot = color_dict[key[\"subject_fullname\"]]\n",
+ " # Plot results\n",
+ " plt.plot(fit_dict[\"delta_bins\"], fit_dict[\"pright_data\"], \"o\", color=color_plot)\n",
+ " if fit_dict[\"delta_fit\"].shape[0] > 0:\n",
+ " plt.plot(fit_dict[\"delta_fit\"], fit_dict[\"pright_fit\"], \"--\", color=color_plot)\n",
+ "\n",
"plt.xticks(fontsize=16)\n",
"plt.yticks(fontsize=16)\n",
"plt.xlabel(\"total evidence (L-R)\", fontsize=18)\n",
"plt.ylabel(\"% Turned left\", fontsize=18)\n",
"plt.ylim([-1, 101])\n",
- "#pl.xlim([-11,11])\n",
+ "# pl.xlim([-11,11])\n",
"mean_correct_string = f\"{np.mean(fraction_correct):2.2f}\"\n",
- "plt.title('Fraction correct responses '+mean_correct_string, fontsize=20)\n",
+ "plt.title(\"Fraction correct responses \" + mean_correct_string, fontsize=20)\n",
"plt.legend(handles=patches_legend, fontsize=18)\n",
"print()"
]
diff --git a/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb b/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb
index e7fa38e1..060513b6 100644
--- a/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb
+++ b/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb
@@ -24,6 +24,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -33,18 +34,17 @@
"metadata": {},
"outputs": [],
"source": [
- "import datajoint as dj\n",
- "import os\n",
+ "import glob\n",
"import pathlib\n",
- "import pandas as pd\n",
+ "\n",
"import cv2\n",
- "import glob\n",
- "import numpy as np\n",
+ "import datajoint as dj\n",
"import matplotlib.pyplot as plt\n",
- "from matplotlib import patches\n",
- "from skimage.measure import EllipseModel\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "from scipy import stats\n",
"from skimage.draw import ellipse_perimeter\n",
- "from scipy import stats"
+ "from skimage.measure import EllipseModel"
]
},
{
@@ -62,8 +62,8 @@
],
"source": [
"dj.conn()\n",
- "pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')\n",
- "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')"
+ "pupillometry_db = dj.create_virtual_module(\"pupillometry\", \"u19_pupillometry\")\n",
+ "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")"
]
},
{
@@ -93,8 +93,8 @@
"frameNumber = 1000\n",
"\n",
"key = dict()\n",
- "key['subject_fullname'] = 'efonseca_ef111_act111'\n",
- "key['session_date'] = '2023-06-24'\n",
+ "key[\"subject_fullname\"] = \"efonseca_ef111_act111\"\n",
+ "key[\"session_date\"] = \"2023-06-24\"\n",
"\n",
"key"
]
@@ -122,26 +122,34 @@
],
"source": [
"# Get video location\n",
- "key_pupil = (acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key).fetch(as_dict=True)[0]\n",
- "db_pupil_data = (pupillometry_db.PupillometrySessionModelData & key).fetch(as_dict=True)[0]\n",
- "pupillometry_dir = dj.config.get('custom', {}).get('pupillometry_root_data_dir',None)\n",
+ "key_pupil = (\n",
+ " acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key\n",
+ ").fetch(as_dict=True)[0]\n",
+ "db_pupil_data = (pupillometry_db.PupillometrySessionModelData & key).fetch(\n",
+ " as_dict=True\n",
+ ")[0]\n",
+ "pupillometry_dir = dj.config.get(\"custom\", {}).get(\"pupillometry_root_data_dir\", None)\n",
"if pupillometry_dir is None:\n",
- " raise Exception('pupillometry_root_data_dir not found in config, run initial_conf.py again')\n",
+ " raise Exception(\n",
+ " \"pupillometry_root_data_dir not found in config, run initial_conf.py again\"\n",
+ " )\n",
"pupillometry_raw_dir = pupillometry_dir[0]\n",
- "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil['remote_path_video_file'])\n",
- "print('videoPath', videoPath)\n",
+ "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil[\"remote_path_video_file\"])\n",
+ "print(\"videoPath\", videoPath)\n",
"\n",
"# Get output location\n",
"pupillometry_processed_dir = pupillometry_dir[1]\n",
- "output_dir = pathlib.Path(pupillometry_processed_dir,pathlib.Path(key_pupil['remote_path_video_file']).parent)\n",
+ "output_dir = pathlib.Path(\n",
+ " pupillometry_processed_dir, pathlib.Path(key_pupil[\"remote_path_video_file\"]).parent\n",
+ ")\n",
"\n",
- "#Find h5 files (output from deeplabcut)\n",
- "h5_files = glob.glob(str(output_dir) + '/*.h5')\n",
+ "# Find h5 files (output from deeplabcut)\n",
+ "h5_files = glob.glob(str(output_dir) + \"/*.h5\")\n",
"if len(h5_files) != 1:\n",
- " raise Exception('Didn''t find any h5 files after deeplabcut analyze_video')\n",
+ " raise Exception(\"Didnt find any h5 files after deeplabcut analyze_video\")\n",
"else:\n",
" h5_files = h5_files[0]\n",
- "print('h5_files', h5_files)"
+ "print(\"h5_files\", h5_files)"
]
},
{
@@ -208,7 +216,7 @@
}
],
"source": [
- "labels.loc[:, (bodyparts[0],'likelihood')]"
+ "labels.loc[:, (bodyparts[0], \"likelihood\")]"
]
},
{
@@ -218,8 +226,12 @@
"outputs": [],
"source": [
"from scipy.signal import find_peaks\n",
- "x = labels.loc[:, (bodyparts[2],'y')],\n",
- "peaks, _ = find_peaks(x, threshold=30, )\n",
+ "\n",
+ "x = (labels.loc[:, (bodyparts[2], \"y\")],)\n",
+ "peaks, _ = find_peaks(\n",
+ " x,\n",
+ " threshold=30,\n",
+ ")\n",
"plt.plot(x)\n",
"plt.plot(peaks, x[peaks], \"x\")\n",
"plt.plot(np.zeros_like(x), \"--\", color=\"gray\")\n",
@@ -245,7 +257,7 @@
}
],
"source": [
- "plt.plot(labels.loc[:, (bodyparts[2],'y')], linewidth=0.5, color ='k')\n",
+ "plt.plot(labels.loc[:, (bodyparts[2], \"y\")], linewidth=0.5, color=\"k\")\n",
"\n",
"\n",
"plt.xlabel(\"frame#\", fontsize=12)\n",
@@ -332,14 +344,12 @@
}
],
"source": [
- "\n",
- "\n",
"# Take a subset from the data\n",
"subset = labels.loc[frameNumber]\n",
"# Get the pupil markers (0 to 8) from the multiindex array obtained from deeplabcut\n",
- "x = subset.xs('x', level='coords').to_numpy()[0:8]\n",
- "y = subset.xs('y', level='coords').to_numpy()[0:8]\n",
- "xy = np.column_stack((x,y))\n",
+ "x = subset.xs(\"x\", level=\"coords\").to_numpy()[0:8]\n",
+ "y = subset.xs(\"y\", level=\"coords\").to_numpy()[0:8]\n",
+ "xy = np.column_stack((x, y))\n",
"\n",
"# Estimate an ellipse based on the points generated around the pupil, this is enough to get an estimate of the pupil area in pixels\n",
"ellipse = EllipseModel()\n",
@@ -347,11 +357,17 @@
"\n",
"# Validate by ploting the points and the fitting ellipse on the image from the video\n",
"fig, ax = plt.subplots()\n",
- "plt.rcParams['figure.dpi'] = 250\n",
- "img = cv2.imread(str(frameNumber) + '.jpg')\n",
- "ax.scatter(xy[:,0], xy[:,1], s=0.1, color='orange')\n",
+ "plt.rcParams[\"figure.dpi\"] = 250\n",
+ "img = cv2.imread(str(frameNumber) + \".jpg\")\n",
+ "ax.scatter(xy[:, 0], xy[:, 1], s=0.1, color=\"orange\")\n",
"# Draw the ellipse on the original image\n",
- "cy, cx = ellipse_perimeter(int(ellipse.params[1]), int(ellipse.params[0]), int(ellipse.params[3]), int(ellipse.params[2]), ellipse.params[4])\n",
+ "cy, cx = ellipse_perimeter(\n",
+ " int(ellipse.params[1]),\n",
+ " int(ellipse.params[0]),\n",
+ " int(ellipse.params[3]),\n",
+ " int(ellipse.params[2]),\n",
+ " ellipse.params[4],\n",
+ ")\n",
"img[cy, cx] = 255\n",
"ax.imshow(img)\n",
"\n",
@@ -365,7 +381,7 @@
"metadata": {},
"outputs": [],
"source": [
- "df = pd.DataFrame(db_pupil_data['pupil_diameter'], columns=['PupilDiameter'])"
+ "df = pd.DataFrame(db_pupil_data[\"pupil_diameter\"], columns=[\"PupilDiameter\"])"
]
},
{
@@ -387,15 +403,15 @@
}
],
"source": [
- "\n",
"# Get a boolean array where true correspond to the frame with an outlier diameter\n",
- "zscore = np.abs(stats.zscore(df, nan_policy='omit'))\n",
+ "zscore = np.abs(stats.zscore(df, nan_policy=\"omit\"))\n",
"outlierFlags = np.abs(zscore) > 2\n",
"outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: \"OutlierFlag\"})\n",
"\n",
"# Get the list of outlier frames\n",
"from itertools import compress\n",
- "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags['OutlierFlag']))\n",
+ "\n",
+ "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags[\"OutlierFlag\"]))\n",
"\n",
"\n",
"# Retrieve an outlier frames sample from the video for visual inspection\n",
@@ -403,20 +419,20 @@
"fig = plt.figure(figsize=(7, 7))\n",
"columns = 5\n",
"rows = 5\n",
- "for i in range(1, columns*rows +1):\n",
+ "for i in range(1, columns * rows + 1):\n",
" # Get total number of frames\n",
" totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n",
" # Set frame position\n",
- " cap.set(cv2.CAP_PROP_POS_FRAMES,outlierFrames[i])\n",
+ " cap.set(cv2.CAP_PROP_POS_FRAMES, outlierFrames[i])\n",
" ret, frame = cap.read()\n",
"\n",
" fig.add_subplot(rows, columns, i)\n",
" plt.title(\"Frame \" + str(outlierFrames[i]), fontsize=5)\n",
" plt.imshow(frame)\n",
- " plt.axis('off')\n",
+ " plt.axis(\"off\")\n",
"plt.show()\n",
"cap.release()\n",
- "cv2.destroyAllWindows()\n"
+ "cv2.destroyAllWindows()"
]
},
{
@@ -470,29 +486,33 @@
"source": [
"# Concatenate outlier flags array to remove outliers from pupil diameter array\n",
"temp = pd.concat([df, outlierFlags], axis=1)\n",
- "temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None\n",
- "pupilDiameter = temp['PupilDiameter']\n",
+ "temp.loc[temp[\"OutlierFlag\"] == True, \"PupilDiameter\"] = None\n",
+ "pupilDiameter = temp[\"PupilDiameter\"]\n",
"\n",
"# This might be better to leave open for the experimenter.\n",
"# Replace the outlier diameter data using a cubic spline interpolator\n",
"# pupilDiameter = temp['PupilDiameter'].interpolate(method='spline', order=3, s=0)\n",
"\n",
- "fig, (ax1, ax2) = plt.subplots(1,2)\n",
+ "fig, (ax1, ax2) = plt.subplots(1, 2)\n",
"# plt.rcParams['figure.dpi'] = 250\n",
"\n",
"ax1.plot(pupilDiameter[0:3600], linewidth=0.25)\n",
"# Filter by using a moving average window of 10 samples\n",
- "ax1.plot(pupilDiameter[0:3600].rolling(10).mean(), linewidth=0.5, color='red')\n",
- "ax1.set_xlabel('Frame')\n",
- "ax1.set_ylabel('Pupil diameter [px]')\n",
- "ax1.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n",
+ "ax1.plot(pupilDiameter[0:3600].rolling(10).mean(), linewidth=0.5, color=\"red\")\n",
+ "ax1.set_xlabel(\"Frame\")\n",
+ "ax1.set_ylabel(\"Pupil diameter [px]\")\n",
+ "ax1.legend(\n",
+ " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n",
+ ")\n",
"\n",
"ax2.plot(pupilDiameter[0:20000], linewidth=0.25)\n",
"# Filter by using a moving average window of 10 samples\n",
- "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color='red')\n",
- "ax2.set_xlabel('Frame')\n",
- "ax2.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n",
- "#fig.savefig('test.eps', format='eps')"
+ "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color=\"red\")\n",
+ "ax2.set_xlabel(\"Frame\")\n",
+ "ax2.legend(\n",
+ " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n",
+ ")\n",
+ "# fig.savefig('test.eps', format='eps')"
]
},
{
diff --git a/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb b/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb
index 092cc7ef..15fb28ad 100644
--- a/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb
+++ b/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb
@@ -22,7 +22,8 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
- "try_find_conf_file()\n"
+ "\n",
+ "try_find_conf_file()"
]
},
{
@@ -31,16 +32,13 @@
"metadata": {},
"outputs": [],
"source": [
+ "import pathlib\n",
+ "import pickle\n",
+ "\n",
"import datajoint as dj\n",
"import pandas as pd\n",
- "import numpy as np\n",
- "\n",
- "import u19_pipeline.utils.path_utils as pu\n",
"\n",
- "from matplotlib import cm\n",
- "from u19_pipeline import utility\n",
- "import pathlib\n",
- "import pickle"
+ "import u19_pipeline.utils.path_utils as pu"
]
},
{
@@ -56,10 +54,9 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
"dj.conn()\n",
- "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')"
+ "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "pupillometry_db = dj.create_virtual_module(\"pupillometry\", \"u19_pupillometry\")"
]
},
{
@@ -79,13 +76,15 @@
}
],
"source": [
- "root_videopath = dj.config['custom']['dlc_root_data_dir']\n",
+ "root_videopath = dj.config[\"custom\"][\"dlc_root_data_dir\"]\n",
"\n",
- "modelPath = pathlib.Path(root_videopath, 'MODELS', 'Pupillometry2-Ryan-2022-04-07').as_posix()\n",
- "configPath = pathlib.Path(modelPath,'config.yaml').as_posix()\n",
+ "modelPath = pathlib.Path(\n",
+ " root_videopath, \"MODELS\", \"Pupillometry2-Ryan-2022-04-07\"\n",
+ ").as_posix()\n",
+ "configPath = pathlib.Path(modelPath, \"config.yaml\").as_posix()\n",
"\n",
- "proccesed_rootpath = dj.config['custom']['dlc_processed_data_dir']\n",
- "proccesed_rootpath = pathlib.Path(proccesed_rootpath, 'pupillometry').as_posix()\n",
+ "proccesed_rootpath = dj.config[\"custom\"][\"dlc_processed_data_dir\"]\n",
+ "proccesed_rootpath = pathlib.Path(proccesed_rootpath, \"pupillometry\").as_posix()\n",
"configPath"
]
},
@@ -323,12 +322,20 @@
],
"source": [
"key = 'subject_fullname like \"jounhong%\"'\n",
- "video_data_df = pd.DataFrame((pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\"KEY\", 'remote_path_video_file', as_dict=True))\n",
+ "video_data_df = pd.DataFrame(\n",
+ " (pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\n",
+ " \"KEY\", \"remote_path_video_file\", as_dict=True\n",
+ " )\n",
+ ")\n",
"\n",
- "video_data_df['remote_path_video_file'] = 'pupillometry/' + video_data_df['remote_path_video_file']\n",
- "video_data_df.loc[0,'remote_path_video_file']\n",
+ "video_data_df[\"remote_path_video_file\"] = (\n",
+ " \"pupillometry/\" + video_data_df[\"remote_path_video_file\"]\n",
+ ")\n",
+ "video_data_df.loc[0, \"remote_path_video_file\"]\n",
"\n",
- "video_data_df['processed_path'] = video_data_df['remote_path_video_file'].apply(lambda x : pathlib.Path(x).parent.as_posix())\n",
+ "video_data_df[\"processed_path\"] = video_data_df[\"remote_path_video_file\"].apply(\n",
+ " lambda x: pathlib.Path(x).parent.as_posix()\n",
+ ")\n",
"\n",
"video_data_df"
]
@@ -383,10 +390,13 @@
}
],
"source": [
- "\n",
"for i in range(video_data_df.shape[0]):\n",
- " this_video_path = pathlib.Path(root_videopath, video_data_df.loc[i, 'remote_path_video_file']).as_posix()\n",
- " this_results_path = pathlib.Path(proccesed_rootpath, video_data_df.loc[i, 'processed_path']).as_posix()\n",
+ " this_video_path = pathlib.Path(\n",
+ " root_videopath, video_data_df.loc[i, \"remote_path_video_file\"]\n",
+ " ).as_posix()\n",
+ " this_results_path = pathlib.Path(\n",
+ " proccesed_rootpath, video_data_df.loc[i, \"processed_path\"]\n",
+ " ).as_posix()\n",
"\n",
" pickle_file = pu.get_filepattern_paths(this_results_path, \"/*pupil_diameter.pickle\")\n",
"\n",
@@ -394,27 +404,26 @@
"\n",
" if len(pickle_file) > 0:\n",
" pickle_file = pickle_file[0]\n",
- " \n",
+ "\n",
" objects = []\n",
- " with (open(pickle_file, \"rb\")) as openfile:\n",
+ " with open(pickle_file, \"rb\") as openfile:\n",
" while True:\n",
" try:\n",
" object = pickle.load(openfile)\n",
" except EOFError:\n",
" break\n",
"\n",
- "\n",
" pupillometry_data_array = object\n",
"\n",
" print(type(pupillometry_data_array))\n",
" print(pupillometry_data_array)\n",
"\n",
" key = dict()\n",
- " key['subject_fullname'] = video_data_df.loc[i, 'subject_fullname']\n",
- " key['session_date'] = video_data_df.loc[i, 'session_date']\n",
+ " key[\"subject_fullname\"] = video_data_df.loc[i, \"subject_fullname\"]\n",
+ " key[\"session_date\"] = video_data_df.loc[i, \"session_date\"]\n",
" new_key = (acquisition_db.Session & key).fetch1(\"KEY\")\n",
- " new_key['pupil_diameter'] = pupillometry_data_array\n",
- " pupillometry_db.PupillometryData.insert1(new_key, allow_direct_insert=True)\n"
+ " new_key[\"pupil_diameter\"] = pupillometry_data_array\n",
+ " pupillometry_db.PupillometryData.insert1(new_key, allow_direct_insert=True)"
]
},
{
@@ -422,13 +431,7 @@
"execution_count": 15,
"metadata": {},
"outputs": [],
- "source": [
- "\n",
- "\n",
- "\n",
- "\n",
- "\n"
- ]
+ "source": []
}
],
"metadata": {
diff --git a/notebooks/pupillometry_notebooks/pupillometry.ipynb b/notebooks/pupillometry_notebooks/pupillometry.ipynb
index 6fa2b624..3924ac9d 100644
--- a/notebooks/pupillometry_notebooks/pupillometry.ipynb
+++ b/notebooks/pupillometry_notebooks/pupillometry.ipynb
@@ -22,16 +22,15 @@
"outputs": [],
"source": [
"import os\n",
- "import deeplabcut\n",
- "\n",
"\n",
+ "import deeplabcut\n",
"\n",
- "modelPath = 'Pupillometry2-Ryan-2022-04-07'\n",
- "configPath = os.path.join(modelPath,'config.yaml')\n",
- "videoPath = '/mnt/cup/braininit/Data/Raw/video_pupillometry/jounhong/jounhong_TH_276/20220401_g0/jounhong_TH_276_20220401_g0.mj2'\n",
- "output_path = '/mnt/cup/braininit/Data/Processed/video_pupillometry/jounhong/jounhong_TH_276/20220401_g0/jounhong_TH_276_20220401_g0DLC_resnet101_Pupillometry2Apr7shuffle1_1030000.h5'\n",
- "rawAnalysisResultPath = 'rawAnalysisResults/'\n",
- "#deeplabcut.analyze_videos(configPath, videoPath, destfolder=rawAnalysisResultPath)"
+ "modelPath = \"Pupillometry2-Ryan-2022-04-07\"\n",
+ "configPath = os.path.join(modelPath, \"config.yaml\")\n",
+ "videoPath = \"/mnt/cup/braininit/Data/Raw/video_pupillometry/jounhong/jounhong_TH_276/20220401_g0/jounhong_TH_276_20220401_g0.mj2\"\n",
+ "output_path = \"/mnt/cup/braininit/Data/Processed/video_pupillometry/jounhong/jounhong_TH_276/20220401_g0/jounhong_TH_276_20220401_g0DLC_resnet101_Pupillometry2Apr7shuffle1_1030000.h5\"\n",
+ "rawAnalysisResultPath = \"rawAnalysisResults/\"\n",
+ "# deeplabcut.analyze_videos(configPath, videoPath, destfolder=rawAnalysisResultPath)"
]
},
{
@@ -462,8 +461,8 @@
}
],
"source": [
- "import pandas as pd\n",
"import numpy as np\n",
+ "import pandas as pd\n",
"\n",
"labels = pd.read_hdf(output_path)\n",
"labels"
@@ -476,6 +475,7 @@
"outputs": [],
"source": [
"import cv2\n",
+ "\n",
"cap = cv2.VideoCapture(videoPath)\n",
"\n",
"frameNumber = 1000\n",
@@ -483,10 +483,10 @@
"# Get total number of frames\n",
"totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n",
"# Set frame position\n",
- "cap.set(cv2.CAP_PROP_POS_FRAMES,frameNumber)\n",
+ "cap.set(cv2.CAP_PROP_POS_FRAMES, frameNumber)\n",
"\n",
"ret, frame = cap.read()\n",
- "name = str(frameNumber) + '.jpg'\n",
+ "name = str(frameNumber) + \".jpg\"\n",
"cv2.imwrite(name, frame)\n",
"\n",
"cap.release()\n",
@@ -513,16 +513,15 @@
],
"source": [
"import matplotlib.pyplot as plt\n",
- "from matplotlib import patches\n",
- "from skimage.measure import EllipseModel\n",
"from skimage.draw import ellipse_perimeter\n",
+ "from skimage.measure import EllipseModel\n",
"\n",
"# Take a subset from the data\n",
"subset = labels.loc[frameNumber]\n",
"# Get the pupil markers (0 to 8) from the multiindex array obtained from deeplabcut\n",
- "x = subset.xs('x', level='coords').to_numpy()[0:8]\n",
- "y = subset.xs('y', level='coords').to_numpy()[0:8]\n",
- "xy = np.column_stack((x,y))\n",
+ "x = subset.xs(\"x\", level=\"coords\").to_numpy()[0:8]\n",
+ "y = subset.xs(\"y\", level=\"coords\").to_numpy()[0:8]\n",
+ "xy = np.column_stack((x, y))\n",
"\n",
"# Estimate an ellipse based on the points generated around the pupil, this is enough to get an estimate of the pupil area in pixels\n",
"ellipse = EllipseModel()\n",
@@ -530,11 +529,17 @@
"\n",
"# Validate by ploting the points and the fitting ellipse on the image from the video\n",
"fig, ax = plt.subplots()\n",
- "plt.rcParams['figure.dpi'] = 250\n",
- "img = cv2.imread(str(frameNumber) + '.jpg')\n",
- "ax.scatter(xy[:,0], xy[:,1], s=0.1, color='orange')\n",
+ "plt.rcParams[\"figure.dpi\"] = 250\n",
+ "img = cv2.imread(str(frameNumber) + \".jpg\")\n",
+ "ax.scatter(xy[:, 0], xy[:, 1], s=0.1, color=\"orange\")\n",
"# Draw the ellipse on the original image\n",
- "cy, cx = ellipse_perimeter(int(ellipse.params[1]), int(ellipse.params[0]), int(ellipse.params[3]), int(ellipse.params[2]), ellipse.params[4])\n",
+ "cy, cx = ellipse_perimeter(\n",
+ " int(ellipse.params[1]),\n",
+ " int(ellipse.params[0]),\n",
+ " int(ellipse.params[3]),\n",
+ " int(ellipse.params[2]),\n",
+ " ellipse.params[4],\n",
+ ")\n",
"img[cy, cx] = 255\n",
"ax.imshow(img)\n",
"\n",
@@ -562,20 +567,20 @@
"metadata": {},
"outputs": [],
"source": [
- "df = pd.DataFrame(np.zeros(1), columns=['PupilDiameter'])\n",
+ "df = pd.DataFrame(np.zeros(1), columns=[\"PupilDiameter\"])\n",
"for i in range(labels.index.size):\n",
" subset = labels.loc[i]\n",
- " x = subset.xs('x', level='coords').to_numpy()[0:8]\n",
- " y = subset.xs('y', level='coords').to_numpy()[0:8]\n",
- " xy = np.column_stack((x,y))\n",
+ " x = subset.xs(\"x\", level=\"coords\").to_numpy()[0:8]\n",
+ " y = subset.xs(\"y\", level=\"coords\").to_numpy()[0:8]\n",
+ " xy = np.column_stack((x, y))\n",
" # Fit the points to an ellipse and get the parameters (estimate X center coordinate, estimate Y center coordinate, a, b, theta)\n",
" ellipse = EllipseModel()\n",
" ellipse.estimate(xy)\n",
" # Calculate the area of the ellipse from the parameters a and b\n",
" ellipseArea = np.pi * ellipse.params[2] * ellipse.params[3]\n",
" # Get the diameter of a circle from the area of the ellipse\n",
- " pupilDiameter = 2 * np.sqrt(ellipseArea/np.pi)\n",
- " df.loc[i] = pupilDiameter\n"
+ " pupilDiameter = 2 * np.sqrt(ellipseArea / np.pi)\n",
+ " df.loc[i] = pupilDiameter"
]
},
{
@@ -627,6 +632,7 @@
],
"source": [
"from scipy import stats\n",
+ "\n",
"# Get a boolean array where true correspond to the frame with an outlier diameter\n",
"zscore = np.abs(stats.zscore(df))\n",
"outlierFlags = np.abs(stats.zscore(df)) > 2\n",
@@ -634,27 +640,28 @@
"\n",
"# Get the list of outlier frames\n",
"from itertools import compress\n",
- "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags['OutlierFlag']))\n",
+ "\n",
+ "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags[\"OutlierFlag\"]))\n",
"\n",
"# Retrieve an outlier frames sample from the video for visual inspection\n",
- "cap = cv2.VideoCapture('jounhong_TH_276_20220406_g0.mj2')\n",
+ "cap = cv2.VideoCapture(\"jounhong_TH_276_20220406_g0.mj2\")\n",
"fig = plt.figure(figsize=(7, 7))\n",
"columns = 5\n",
"rows = 5\n",
- "for i in range(1, columns*rows +1):\n",
+ "for i in range(1, columns * rows + 1):\n",
" # Get total number of frames\n",
" totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n",
" # Set frame position\n",
- " cap.set(cv2.CAP_PROP_POS_FRAMES,outlierFrames[i])\n",
+ " cap.set(cv2.CAP_PROP_POS_FRAMES, outlierFrames[i])\n",
" ret, frame = cap.read()\n",
"\n",
" fig.add_subplot(rows, columns, i)\n",
" plt.title(\"Frame \" + str(outlierFrames[i]), fontsize=5)\n",
" plt.imshow(frame)\n",
- " plt.axis('off')\n",
+ " plt.axis(\"off\")\n",
"plt.show()\n",
"cap.release()\n",
- "cv2.destroyAllWindows()\n"
+ "cv2.destroyAllWindows()"
]
},
{
@@ -688,29 +695,33 @@
"source": [
"# Concatenate outlier flags array to remove outliers from pupil diameter array\n",
"temp = pd.concat([df, outlierFlags], axis=1)\n",
- "temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None\n",
- "pupilDiameter = temp['PupilDiameter']\n",
+ "temp.loc[temp[\"OutlierFlag\"] == True, \"PupilDiameter\"] = None\n",
+ "pupilDiameter = temp[\"PupilDiameter\"]\n",
"\n",
"# This might be better to leave open for the experimenter.\n",
"# Replace the outlier diameter data using a cubic spline interpolator\n",
"# pupilDiameter = temp['PupilDiameter'].interpolate(method='spline', order=3, s=0)\n",
"\n",
- "fig, (ax1, ax2) = plt.subplots(1,2)\n",
+ "fig, (ax1, ax2) = plt.subplots(1, 2)\n",
"# plt.rcParams['figure.dpi'] = 250\n",
"\n",
"ax1.plot(pupilDiameter[0:3600], linewidth=0.25)\n",
"# Filter by using a moving average window of 10 samples\n",
- "ax1.plot(pupilDiameter[0:3600].rolling(10).mean(), linewidth=0.5, color='red')\n",
- "ax1.set_xlabel('Frame')\n",
- "ax1.set_ylabel('Pupil diameter [px]')\n",
- "ax1.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n",
+ "ax1.plot(pupilDiameter[0:3600].rolling(10).mean(), linewidth=0.5, color=\"red\")\n",
+ "ax1.set_xlabel(\"Frame\")\n",
+ "ax1.set_ylabel(\"Pupil diameter [px]\")\n",
+ "ax1.legend(\n",
+ " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n",
+ ")\n",
"\n",
"ax2.plot(pupilDiameter[0:20000], linewidth=0.25)\n",
"# Filter by using a moving average window of 10 samples\n",
- "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color='red')\n",
- "ax2.set_xlabel('Frame')\n",
- "ax2.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n",
- "#fig.savefig('test.eps', format='eps')"
+ "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color=\"red\")\n",
+ "ax2.set_xlabel(\"Frame\")\n",
+ "ax2.legend(\n",
+ " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n",
+ ")\n",
+ "# fig.savefig('test.eps', format='eps')"
]
},
{
@@ -728,6 +739,7 @@
"source": [
"# Pupil diameter pipeline functions\n",
"\n",
+ "\n",
"def analyzeVideo(videoPath, modelPath, destinationFolder):\n",
" \"\"\"\n",
" Stores the analized video data from videoPath as h5 file in the destination folder using the DLC model in modelPath\n",
@@ -738,10 +750,11 @@
" \"\"\"\n",
" # Analyze the video using the selected modelPath and videoPath\n",
" import os\n",
- " import deeplabcut\n",
- " configPath = os.path.join(modelPath,'config.yaml')\n",
+ "\n",
+ " configPath = os.path.join(modelPath, \"config.yaml\")\n",
" deeplabcut.analyze_videos(configPath, videoPath, destfolder=destinationFolder)\n",
"\n",
+ "\n",
"def getPupilDiameter(analyzedVideoDataPath):\n",
" \"\"\"\n",
" Returns a pupil diameter numpy array from an analized video data stored in analyzedVideoDataPath\n",
@@ -752,30 +765,30 @@
" \"\"\"\n",
" # TODO make the function\n",
" from skimage.measure import EllipseModel\n",
- " from skimage.draw import ellipse_perimeter\n",
- " \n",
+ "\n",
" # Read the analyzed video data h5 file\n",
" labels = pd.read_hdf(analyzedVideoDataPath)\n",
"\n",
" # Create a data frame of the same size ad the analyzed video data filled with zeros\n",
- " df = pd.DataFrame(np.zeros(1), columns=['PupilDiameter'])\n",
+ " df = pd.DataFrame(np.zeros(1), columns=[\"PupilDiameter\"])\n",
" # For each frame, get the x and y coordinates of the points around the pupil, fit an ellipse and calculate the diameter of a circle with the same area as the ellipse\n",
" for i in range(labels.index.size):\n",
" subset = labels.loc[i]\n",
- " x = subset.xs('x', level='coords').to_numpy()[0:8]\n",
- " y = subset.xs('y', level='coords').to_numpy()[0:8]\n",
- " xy = np.column_stack((x,y))\n",
+ " x = subset.xs(\"x\", level=\"coords\").to_numpy()[0:8]\n",
+ " y = subset.xs(\"y\", level=\"coords\").to_numpy()[0:8]\n",
+ " xy = np.column_stack((x, y))\n",
" # Fit the points to an ellipse and get the parameters (estimate X center coordinate, estimate Y center coordinate, a, b, theta)\n",
" ellipse = EllipseModel()\n",
" ellipse.estimate(xy)\n",
" # Calculate the area of the ellipse from the parameters a and b\n",
" ellipseArea = np.pi * ellipse.params[2] * ellipse.params[3]\n",
" # Get the diameter of a circle from the area of the ellipse\n",
- " pupilDiameter = 2 * np.sqrt(ellipseArea/np.pi)\n",
+ " pupilDiameter = 2 * np.sqrt(ellipseArea / np.pi)\n",
" df.loc[i] = pupilDiameter\n",
"\n",
" # Get outliers (frames where either the mice have the eyes closed (blink or groom) or deeplabcut fails to track the pupil correctly)\n",
" from scipy import stats\n",
+ "\n",
" # Calculate the zscore of the data frame\n",
" zscore = np.abs(stats.zscore(df))\n",
" # Set a treshold for a valid zscore value (determined empirically)\n",
@@ -784,11 +797,10 @@
" outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: \"OutlierFlag\"})\n",
" # Concatenate outlier flags array to remove outliers from pupil diameter array\n",
" temp = pd.concat([df, outlierFlags], axis=1)\n",
- " temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None\n",
- " pupilDiameter = temp['PupilDiameter']\n",
- " \n",
- " return pupilDiameter.to_numpy()\n",
- " "
+ " temp.loc[temp[\"OutlierFlag\"] == True, \"PupilDiameter\"] = None\n",
+ " pupilDiameter = temp[\"PupilDiameter\"]\n",
+ "\n",
+ " return pupilDiameter.to_numpy()"
]
},
{
@@ -806,7 +818,8 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
- "try_find_conf_file()\n"
+ "\n",
+ "try_find_conf_file()"
]
},
{
@@ -815,9 +828,10 @@
"metadata": {},
"outputs": [],
"source": [
- "import datajoint as dj\n",
"import os\n",
"import pathlib\n",
+ "\n",
+ "import datajoint as dj\n",
"import pandas as pd"
]
},
@@ -836,8 +850,8 @@
],
"source": [
"dj.conn()\n",
- "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')"
+ "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "pupillometry_db = dj.create_virtual_module(\"pupillometry\", \"u19_pupillometry\")"
]
},
{
@@ -951,7 +965,7 @@
}
],
"source": [
- "dj.ERD(pupillometry_db) -1"
+ "dj.ERD(pupillometry_db) - 1"
]
},
{
@@ -971,13 +985,15 @@
}
],
"source": [
- "root_videopath = dj.config['custom']['dlc_root_data_dir2']\n",
+ "root_videopath = dj.config[\"custom\"][\"dlc_root_data_dir2\"]\n",
"\n",
- "modelPath = pathlib.Path(root_videopath, 'MODELS', 'Pupillometry2-Ryan-2022-04-07').as_posix()\n",
- "configPath = pathlib.Path(modelPath,'config.yaml').as_posix()\n",
+ "modelPath = pathlib.Path(\n",
+ " root_videopath, \"MODELS\", \"Pupillometry2-Ryan-2022-04-07\"\n",
+ ").as_posix()\n",
+ "configPath = pathlib.Path(modelPath, \"config.yaml\").as_posix()\n",
"\n",
- "proccesed_rootpath = dj.config['custom']['dlc_processed_data_dir2']\n",
- "proccesed_rootpath = pathlib.Path(proccesed_rootpath, 'pupillometry').as_posix()\n",
+ "proccesed_rootpath = dj.config[\"custom\"][\"dlc_processed_data_dir2\"]\n",
+ "proccesed_rootpath = pathlib.Path(proccesed_rootpath, \"pupillometry\").as_posix()\n",
"configPath"
]
},
@@ -1215,12 +1231,20 @@
],
"source": [
"key = 'subject_fullname like \"jounhong%\"'\n",
- "video_data_df = pd.DataFrame((pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\"KEY\", 'remote_path_video_file', as_dict=True))\n",
+ "video_data_df = pd.DataFrame(\n",
+ " (pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\n",
+ " \"KEY\", \"remote_path_video_file\", as_dict=True\n",
+ " )\n",
+ ")\n",
"\n",
- "video_data_df['remote_path_video_file'] = 'pupillometry/' + video_data_df['remote_path_video_file']\n",
- "video_data_df.loc[0,'remote_path_video_file']\n",
+ "video_data_df[\"remote_path_video_file\"] = (\n",
+ " \"pupillometry/\" + video_data_df[\"remote_path_video_file\"]\n",
+ ")\n",
+ "video_data_df.loc[0, \"remote_path_video_file\"]\n",
"\n",
- "video_data_df['processed_path'] = video_data_df['remote_path_video_file'].apply(lambda x : pathlib.Path(x).parent.as_posix())\n",
+ "video_data_df[\"processed_path\"] = video_data_df[\"remote_path_video_file\"].apply(\n",
+ " lambda x: pathlib.Path(x).parent.as_posix()\n",
+ ")\n",
"\n",
"video_data_df"
]
@@ -1242,7 +1266,7 @@
}
],
"source": [
- "'''\n",
+ "\"\"\"\n",
"for i in range(video_data_df.shape[0]-1,0,-1):\n",
" this_video_path = pathlib.Path(root_videopath, video_data_df.loc[i, 'remote_path_video_file']).as_posix()\n",
" this_results_path = pathlib.Path(proccesed_rootpath, video_data_df.loc[i, 'processed_path']).as_posix()\n",
@@ -1252,7 +1276,7 @@
" print(this_video_path)\n",
" print(this_results_path)\n",
" deeplabcut.analyze_videos(configPath, this_video_path, destfolder=this_results_path)\n",
- "'''"
+ "\"\"\""
]
},
{
@@ -1272,7 +1296,7 @@
}
],
"source": [
- "'''\n",
+ "\"\"\"\n",
"import u19_pipeline.utils.dlc_process as dlc\n",
"\n",
"\n",
@@ -1283,7 +1307,7 @@
" dlc.getPupilDiameter(destinationFolder=this_results_path)\n",
" except BaseException as e:\n",
" print(str(e))\n",
- "'''"
+ "\"\"\""
]
},
{
@@ -1303,7 +1327,7 @@
}
],
"source": [
- "'''\n",
+ "\"\"\"\n",
"import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n",
"import u19_pipeline.automatic_job.slurm_creator as sc\n",
"import subprocess\n",
@@ -1372,7 +1396,7 @@
"\n",
"\n",
"\n",
- "'''"
+ "\"\"\""
]
},
{
@@ -1383,13 +1407,14 @@
"source": [
"import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n",
"\n",
+ "\n",
"def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars):\n",
- " '''\n",
+ " \"\"\"\n",
" Create scp command from cluster directories and local slurm file\n",
- " '''\n",
+ " \"\"\"\n",
"\n",
- " user_host = cluster_vars['user']+'@'+cluster_vars['hostname']\n",
- " slurm_destination = user_host+':'+slurm_destination\n",
+ " user_host = cluster_vars[\"user\"] + \"@\" + cluster_vars[\"hostname\"]\n",
+ " slurm_destination = user_host + \":\" + slurm_destination\n",
" status = ft.scp_file_transfer(slurm_file_local_path, slurm_destination)\n",
"\n",
" return status"
@@ -1421,79 +1446,97 @@
}
],
"source": [
- "import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n",
- "import u19_pipeline.automatic_job.slurm_creator as sc\n",
+ "import re\n",
"import subprocess\n",
+ "\n",
"import u19_pipeline.automatic_job.params_config as config\n",
- "import re\n",
+ "import u19_pipeline.automatic_job.slurm_creator as sc\n",
"from u19_pipeline.utils.file_utils import write_file\n",
"\n",
- "repo_dir = '/scratch/gpfs/BRAINCOGS/U19-pipeline_python'\n",
- "script_path = pathlib.Path(repo_dir, 'u19_pipeline/utils/dlc_process.py').as_posix()\n",
+ "repo_dir = \"/scratch/gpfs/BRAINCOGS/U19-pipeline_python\"\n",
+ "script_path = pathlib.Path(repo_dir, \"u19_pipeline/utils/dlc_process.py\").as_posix()\n",
"\n",
- "local_repo_dir = '/Users/alvaros/Documents/MATLAB/BrainCogsProjects/Datajoint_proj/U19-pipeline_python/'\n",
+ "local_repo_dir = \"/Users/alvaros/Documents/MATLAB/BrainCogsProjects/Datajoint_proj/U19-pipeline_python/\"\n",
"\n",
- "slurms_filepath = pathlib.Path(local_repo_dir, 'u19_pipeline/automatic_job/SlurmFiles').as_posix()\n",
- "default_slurm_filename = 'slurm_real.slurm'\n",
+ "slurms_filepath = pathlib.Path(\n",
+ " local_repo_dir, \"u19_pipeline/automatic_job/SlurmFiles\"\n",
+ ").as_posix()\n",
+ "default_slurm_filename = \"slurm_real.slurm\"\n",
"\n",
"for i in range(9, 10):\n",
- " this_video_path = pathlib.Path(root_videopath, video_data_df.loc[i, 'remote_path_video_file']).as_posix()\n",
- " this_results_path = pathlib.Path(proccesed_rootpath, video_data_df.loc[i, 'processed_path']).as_posix()\n",
+ " this_video_path = pathlib.Path(\n",
+ " root_videopath, video_data_df.loc[i, \"remote_path_video_file\"]\n",
+ " ).as_posix()\n",
+ " this_results_path = pathlib.Path(\n",
+ " proccesed_rootpath, video_data_df.loc[i, \"processed_path\"]\n",
+ " ).as_posix()\n",
"\n",
" recording_process_id = str(i)\n",
"\n",
- " #Get all associated directories given the selected processing cluster\n",
- " cluster_vars = ft.get_cluster_vars('tiger')\n",
+ " # Get all associated directories given the selected processing cluster\n",
+ " cluster_vars = ft.get_cluster_vars(\"tiger\")\n",
"\n",
" # Start with default values\n",
- " slurm_dict = cluster_vars['slurm_default'].copy()\n",
- " label_rec_process = 'rec'+recording_process_id\n",
- " print('label_rec_process', label_rec_process)\n",
- " slurm_dict['job-name'] = label_rec_process\n",
- " slurm_dict['output'] = str(pathlib.Path(cluster_vars['log_files_dir'],label_rec_process+ '.log'))\n",
+ " slurm_dict = cluster_vars[\"slurm_default\"].copy()\n",
+ " label_rec_process = \"rec\" + recording_process_id\n",
+ " print(\"label_rec_process\", label_rec_process)\n",
+ " slurm_dict[\"job-name\"] = label_rec_process\n",
+ " slurm_dict[\"output\"] = str(\n",
+ " pathlib.Path(cluster_vars[\"log_files_dir\"], label_rec_process + \".log\")\n",
+ " )\n",
" slurm_text = sc.generate_slurm_dlc2(slurm_dict)\n",
"\n",
" slurm_file_name = default_slurm_filename\n",
- " slurm_file_local_path = pathlib.Path(slurms_filepath,slurm_file_name).as_posix()\n",
+ " slurm_file_local_path = pathlib.Path(slurms_filepath, slurm_file_name).as_posix()\n",
"\n",
" write_file(slurm_file_local_path, slurm_text)\n",
"\n",
- " slurm_destination = pathlib.Path(cluster_vars['slurm_files_dir'], slurm_file_name).as_posix()\n",
+ " slurm_destination = pathlib.Path(\n",
+ " cluster_vars[\"slurm_files_dir\"], slurm_file_name\n",
+ " ).as_posix()\n",
"\n",
" print(slurm_file_local_path)\n",
" print(slurm_destination)\n",
" print(cluster_vars)\n",
"\n",
" status = transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars)\n",
- " \n",
+ "\n",
" processed_data_directory = pathlib.Path(this_results_path).as_posix()\n",
" repository_dir = repo_dir\n",
"\n",
- " command = ['ssh', cluster_vars['user']+\"@\"+cluster_vars['hostname'], 'sbatch', \n",
- " \"--export=recording_process_id=\"+recording_process_id+\n",
- " \",raw_data_directory=\"+this_video_path+\n",
- " \",processed_data_directory=\"+processed_data_directory+\n",
- " \",repository_dir='\"+repository_dir+\n",
- " \"',process_script_path=\"+script_path+\n",
- " \",model_path=\"+modelPath\n",
- " , slurm_destination\n",
+ " command = [\n",
+ " \"ssh\",\n",
+ " cluster_vars[\"user\"] + \"@\" + cluster_vars[\"hostname\"],\n",
+ " \"sbatch\",\n",
+ " \"--export=recording_process_id=\"\n",
+ " + recording_process_id\n",
+ " + \",raw_data_directory=\"\n",
+ " + this_video_path\n",
+ " + \",processed_data_directory=\"\n",
+ " + processed_data_directory\n",
+ " + \",repository_dir='\"\n",
+ " + repository_dir\n",
+ " + \"',process_script_path=\"\n",
+ " + script_path\n",
+ " + \",model_path=\"\n",
+ " + modelPath,\n",
+ " slurm_destination,\n",
" ]\n",
"\n",
" print(command)\n",
" p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
- " #p = os.popen(command_new).read()\n",
+ " # p = os.popen(command_new).read()\n",
" p.wait()\n",
" stdout, stderr = p.communicate()\n",
- " print('stdout', stdout.decode('UTF-8'))\n",
- " print('stderr', stderr.decode('UTF-8'))\n",
- " print('p.returncode', p.returncode)\n",
- "\n",
- " if p.returncode == config.system_process['SUCCESS']:\n",
- " batch_job_sentence = stdout.decode('UTF-8')\n",
- " print('batch_job_sentence', batch_job_sentence)\n",
- " id_slurm_job = batch_job_sentence.replace(\"Submitted batch job \",\"\")\n",
- " id_slurm_job = re.sub(r\"[\\n\\t\\s]*\", \"\", id_slurm_job)\n",
- "\n"
+ " print(\"stdout\", stdout.decode(\"UTF-8\"))\n",
+ " print(\"stderr\", stderr.decode(\"UTF-8\"))\n",
+ " print(\"p.returncode\", p.returncode)\n",
+ "\n",
+ " if p.returncode == config.system_process[\"SUCCESS\"]:\n",
+ " batch_job_sentence = stdout.decode(\"UTF-8\")\n",
+ " print(\"batch_job_sentence\", batch_job_sentence)\n",
+ " id_slurm_job = batch_job_sentence.replace(\"Submitted batch job \", \"\")\n",
+ " id_slurm_job = re.sub(r\"[\\n\\t\\s]*\", \"\", id_slurm_job)"
]
},
{
diff --git a/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb b/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb
index 233b32e7..1e8ef8c0 100644
--- a/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb
+++ b/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb
@@ -24,6 +24,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -59,11 +60,11 @@
}
],
"source": [
- "import datajoint as dj\n",
- "import os\n",
"import pathlib\n",
- "import pandas as pd\n",
- "import deeplabcut"
+ "\n",
+ "import datajoint as dj\n",
+ "import deeplabcut\n",
+ "import pandas as pd"
]
},
{
@@ -81,8 +82,8 @@
],
"source": [
"dj.conn()\n",
- "pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')\n",
- "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')"
+ "pupillometry_db = dj.create_virtual_module(\"pupillometry\", \"u19_pupillometry\")\n",
+ "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")"
]
},
{
@@ -105,11 +106,15 @@
],
"source": [
"key = dict()\n",
- "key['subject_fullname'] = 'efonseca_ef111_act111'\n",
- "key['session_date'] = '2023-04-25'\n",
+ "key[\"subject_fullname\"] = \"efonseca_ef111_act111\"\n",
+ "key[\"session_date\"] = \"2023-04-25\"\n",
"\n",
- "key_pupil = (acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key).fetch(as_dict=True)[0]\n",
- "model_key = (pupillometry_db.PupillometryModels & \"model_id = \"+str(key_pupil['model_id'])).fetch(as_dict=True)[0]\n",
+ "key_pupil = (\n",
+ " acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key\n",
+ ").fetch(as_dict=True)[0]\n",
+ "model_key = (\n",
+ " pupillometry_db.PupillometryModels & \"model_id = \" + str(key_pupil[\"model_id\"])\n",
+ ").fetch(as_dict=True)[0]\n",
"\n",
"model_key"
]
@@ -157,23 +162,26 @@
}
],
"source": [
- "models_dir = dj.config.get('custom', {}).get('root_data_dir',None)\n",
+ "models_dir = dj.config.get(\"custom\", {}).get(\"root_data_dir\", None)\n",
"if models_dir is None:\n",
- " raise Exception('root_data_dir in not found in config, , run initial_conf.py again')\n",
+ " raise Exception(\"root_data_dir in not found in config, , run initial_conf.py again\")\n",
"models_dir = models_dir[0]\n",
- "configPath = pathlib.Path(models_dir, model_key['model_path'],'config.yaml')\n",
+ "configPath = pathlib.Path(models_dir, model_key[\"model_path\"], \"config.yaml\")\n",
"\n",
- "pupillometry_dir = dj.config.get('custom', {}).get('pupillometry_root_data_dir',None)\n",
+ "pupillometry_dir = dj.config.get(\"custom\", {}).get(\"pupillometry_root_data_dir\", None)\n",
"\n",
"if pupillometry_dir is None:\n",
- " raise Exception('pupillometry_root_data_dir not found in config, run initial_conf.py again')\n",
+ " raise Exception(\n",
+ " \"pupillometry_root_data_dir not found in config, run initial_conf.py again\"\n",
+ " )\n",
"pupillometry_raw_dir = pupillometry_dir[0]\n",
"pupillometry_processed_dir = pupillometry_dir[1]\n",
"\n",
- "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil['remote_path_video_file'])\n",
- "output_dir = pathlib.Path(pupillometry_processed_dir,pathlib.Path(key_pupil['remote_path_video_file']).parent)\n",
- "output_dir\n",
- "\n"
+ "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil[\"remote_path_video_file\"])\n",
+ "output_dir = pathlib.Path(\n",
+ " pupillometry_processed_dir, pathlib.Path(key_pupil[\"remote_path_video_file\"]).parent\n",
+ ")\n",
+ "output_dir"
]
},
{
@@ -265,7 +273,7 @@
}
],
"source": [
- "dj.config.get('custom', {})"
+ "dj.config.get(\"custom\", {})"
]
},
{
@@ -320,7 +328,9 @@
}
],
"source": [
- "deeplabcut.analyze_videos(configPath.as_posix(), videoPath.as_posix(), destfolder=output_dir.as_posix())"
+ "deeplabcut.analyze_videos(\n",
+ " configPath.as_posix(), videoPath.as_posix(), destfolder=output_dir.as_posix()\n",
+ ")"
]
},
{
@@ -330,8 +340,8 @@
"outputs": [],
"source": [
"dj.conn()\n",
- "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "#pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')"
+ "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "# pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')"
]
},
{
@@ -454,7 +464,7 @@
}
],
"source": [
- "dj.ERD(pupillometry_db) -1"
+ "dj.ERD(pupillometry_db) - 1"
]
},
{
@@ -474,13 +484,15 @@
}
],
"source": [
- "root_videopath = dj.config['custom']['dlc_root_data_dir2']\n",
+ "root_videopath = dj.config[\"custom\"][\"dlc_root_data_dir2\"]\n",
"\n",
- "modelPath = pathlib.Path(root_videopath, 'MODELS', 'Pupillometry2-Ryan-2022-04-07').as_posix()\n",
- "configPath = pathlib.Path(modelPath,'config.yaml').as_posix()\n",
+ "modelPath = pathlib.Path(\n",
+ " root_videopath, \"MODELS\", \"Pupillometry2-Ryan-2022-04-07\"\n",
+ ").as_posix()\n",
+ "configPath = pathlib.Path(modelPath, \"config.yaml\").as_posix()\n",
"\n",
- "proccesed_rootpath = dj.config['custom']['dlc_processed_data_dir2']\n",
- "proccesed_rootpath = pathlib.Path(proccesed_rootpath, 'pupillometry').as_posix()\n",
+ "proccesed_rootpath = dj.config[\"custom\"][\"dlc_processed_data_dir2\"]\n",
+ "proccesed_rootpath = pathlib.Path(proccesed_rootpath, \"pupillometry\").as_posix()\n",
"configPath"
]
},
@@ -718,12 +730,20 @@
],
"source": [
"key = 'subject_fullname like \"jounhong%\"'\n",
- "video_data_df = pd.DataFrame((pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\"KEY\", 'remote_path_video_file', as_dict=True))\n",
+ "video_data_df = pd.DataFrame(\n",
+ " (pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\n",
+ " \"KEY\", \"remote_path_video_file\", as_dict=True\n",
+ " )\n",
+ ")\n",
"\n",
- "video_data_df['remote_path_video_file'] = 'pupillometry/' + video_data_df['remote_path_video_file']\n",
- "video_data_df.loc[0,'remote_path_video_file']\n",
+ "video_data_df[\"remote_path_video_file\"] = (\n",
+ " \"pupillometry/\" + video_data_df[\"remote_path_video_file\"]\n",
+ ")\n",
+ "video_data_df.loc[0, \"remote_path_video_file\"]\n",
"\n",
- "video_data_df['processed_path'] = video_data_df['remote_path_video_file'].apply(lambda x : pathlib.Path(x).parent.as_posix())\n",
+ "video_data_df[\"processed_path\"] = video_data_df[\"remote_path_video_file\"].apply(\n",
+ " lambda x: pathlib.Path(x).parent.as_posix()\n",
+ ")\n",
"\n",
"video_data_df"
]
@@ -745,7 +765,7 @@
}
],
"source": [
- "'''\n",
+ "\"\"\"\n",
"for i in range(video_data_df.shape[0]-1,0,-1):\n",
" this_video_path = pathlib.Path(root_videopath, video_data_df.loc[i, 'remote_path_video_file']).as_posix()\n",
" this_results_path = pathlib.Path(proccesed_rootpath, video_data_df.loc[i, 'processed_path']).as_posix()\n",
@@ -755,7 +775,7 @@
" print(this_video_path)\n",
" print(this_results_path)\n",
" deeplabcut.analyze_videos(configPath, this_video_path, destfolder=this_results_path)\n",
- "'''"
+ "\"\"\""
]
},
{
@@ -775,7 +795,7 @@
}
],
"source": [
- "'''\n",
+ "\"\"\"\n",
"import u19_pipeline.utils.dlc_process as dlc\n",
"\n",
"\n",
@@ -786,7 +806,7 @@
" dlc.getPupilDiameter(destinationFolder=this_results_path)\n",
" except BaseException as e:\n",
" print(str(e))\n",
- "'''"
+ "\"\"\""
]
},
{
@@ -806,7 +826,7 @@
}
],
"source": [
- "'''\n",
+ "\"\"\"\n",
"import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n",
"import u19_pipeline.automatic_job.slurm_creator as sc\n",
"import subprocess\n",
@@ -875,7 +895,7 @@
"\n",
"\n",
"\n",
- "'''"
+ "\"\"\""
]
},
{
@@ -886,13 +906,14 @@
"source": [
"import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n",
"\n",
+ "\n",
"def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars):\n",
- " '''\n",
+ " \"\"\"\n",
" Create scp command from cluster directories and local slurm file\n",
- " '''\n",
+ " \"\"\"\n",
"\n",
- " user_host = cluster_vars['user']+'@'+cluster_vars['hostname']\n",
- " slurm_destination = user_host+':'+slurm_destination\n",
+ " user_host = cluster_vars[\"user\"] + \"@\" + cluster_vars[\"hostname\"]\n",
+ " slurm_destination = user_host + \":\" + slurm_destination\n",
" status = ft.scp_file_transfer(slurm_file_local_path, slurm_destination)\n",
"\n",
" return status"
@@ -924,79 +945,97 @@
}
],
"source": [
- "import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n",
- "import u19_pipeline.automatic_job.slurm_creator as sc\n",
+ "import re\n",
"import subprocess\n",
+ "\n",
"import u19_pipeline.automatic_job.params_config as config\n",
- "import re\n",
+ "import u19_pipeline.automatic_job.slurm_creator as sc\n",
"from u19_pipeline.utils.file_utils import write_file\n",
"\n",
- "repo_dir = '/scratch/gpfs/BRAINCOGS/U19-pipeline_python'\n",
- "script_path = pathlib.Path(repo_dir, 'u19_pipeline/utils/dlc_process.py').as_posix()\n",
+ "repo_dir = \"/scratch/gpfs/BRAINCOGS/U19-pipeline_python\"\n",
+ "script_path = pathlib.Path(repo_dir, \"u19_pipeline/utils/dlc_process.py\").as_posix()\n",
"\n",
- "local_repo_dir = '/Users/alvaros/Documents/MATLAB/BrainCogsProjects/Datajoint_proj/U19-pipeline_python/'\n",
+ "local_repo_dir = \"/Users/alvaros/Documents/MATLAB/BrainCogsProjects/Datajoint_proj/U19-pipeline_python/\"\n",
"\n",
- "slurms_filepath = pathlib.Path(local_repo_dir, 'u19_pipeline/automatic_job/SlurmFiles').as_posix()\n",
- "default_slurm_filename = 'slurm_real.slurm'\n",
+ "slurms_filepath = pathlib.Path(\n",
+ " local_repo_dir, \"u19_pipeline/automatic_job/SlurmFiles\"\n",
+ ").as_posix()\n",
+ "default_slurm_filename = \"slurm_real.slurm\"\n",
"\n",
"for i in range(9, 10):\n",
- " this_video_path = pathlib.Path(root_videopath, video_data_df.loc[i, 'remote_path_video_file']).as_posix()\n",
- " this_results_path = pathlib.Path(proccesed_rootpath, video_data_df.loc[i, 'processed_path']).as_posix()\n",
+ " this_video_path = pathlib.Path(\n",
+ " root_videopath, video_data_df.loc[i, \"remote_path_video_file\"]\n",
+ " ).as_posix()\n",
+ " this_results_path = pathlib.Path(\n",
+ " proccesed_rootpath, video_data_df.loc[i, \"processed_path\"]\n",
+ " ).as_posix()\n",
"\n",
" recording_process_id = str(i)\n",
"\n",
- " #Get all associated directories given the selected processing cluster\n",
- " cluster_vars = ft.get_cluster_vars('tiger')\n",
+ " # Get all associated directories given the selected processing cluster\n",
+ " cluster_vars = ft.get_cluster_vars(\"tiger\")\n",
"\n",
" # Start with default values\n",
- " slurm_dict = cluster_vars['slurm_default'].copy()\n",
- " label_rec_process = 'rec'+recording_process_id\n",
- " print('label_rec_process', label_rec_process)\n",
- " slurm_dict['job-name'] = label_rec_process\n",
- " slurm_dict['output'] = str(pathlib.Path(cluster_vars['log_files_dir'],label_rec_process+ '.log'))\n",
+ " slurm_dict = cluster_vars[\"slurm_default\"].copy()\n",
+ " label_rec_process = \"rec\" + recording_process_id\n",
+ " print(\"label_rec_process\", label_rec_process)\n",
+ " slurm_dict[\"job-name\"] = label_rec_process\n",
+ " slurm_dict[\"output\"] = str(\n",
+ " pathlib.Path(cluster_vars[\"log_files_dir\"], label_rec_process + \".log\")\n",
+ " )\n",
" slurm_text = sc.generate_slurm_dlc2(slurm_dict)\n",
"\n",
" slurm_file_name = default_slurm_filename\n",
- " slurm_file_local_path = pathlib.Path(slurms_filepath,slurm_file_name).as_posix()\n",
+ " slurm_file_local_path = pathlib.Path(slurms_filepath, slurm_file_name).as_posix()\n",
"\n",
" write_file(slurm_file_local_path, slurm_text)\n",
"\n",
- " slurm_destination = pathlib.Path(cluster_vars['slurm_files_dir'], slurm_file_name).as_posix()\n",
+ " slurm_destination = pathlib.Path(\n",
+ " cluster_vars[\"slurm_files_dir\"], slurm_file_name\n",
+ " ).as_posix()\n",
"\n",
" print(slurm_file_local_path)\n",
" print(slurm_destination)\n",
" print(cluster_vars)\n",
"\n",
" status = transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars)\n",
- " \n",
+ "\n",
" processed_data_directory = pathlib.Path(this_results_path).as_posix()\n",
" repository_dir = repo_dir\n",
"\n",
- " command = ['ssh', cluster_vars['user']+\"@\"+cluster_vars['hostname'], 'sbatch', \n",
- " \"--export=recording_process_id=\"+recording_process_id+\n",
- " \",raw_data_directory=\"+this_video_path+\n",
- " \",processed_data_directory=\"+processed_data_directory+\n",
- " \",repository_dir='\"+repository_dir+\n",
- " \"',process_script_path=\"+script_path+\n",
- " \",model_path=\"+modelPath\n",
- " , slurm_destination\n",
+ " command = [\n",
+ " \"ssh\",\n",
+ " cluster_vars[\"user\"] + \"@\" + cluster_vars[\"hostname\"],\n",
+ " \"sbatch\",\n",
+ " \"--export=recording_process_id=\"\n",
+ " + recording_process_id\n",
+ " + \",raw_data_directory=\"\n",
+ " + this_video_path\n",
+ " + \",processed_data_directory=\"\n",
+ " + processed_data_directory\n",
+ " + \",repository_dir='\"\n",
+ " + repository_dir\n",
+ " + \"',process_script_path=\"\n",
+ " + script_path\n",
+ " + \",model_path=\"\n",
+ " + modelPath,\n",
+ " slurm_destination,\n",
" ]\n",
"\n",
" print(command)\n",
" p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
- " #p = os.popen(command_new).read()\n",
+ " # p = os.popen(command_new).read()\n",
" p.wait()\n",
" stdout, stderr = p.communicate()\n",
- " print('stdout', stdout.decode('UTF-8'))\n",
- " print('stderr', stderr.decode('UTF-8'))\n",
- " print('p.returncode', p.returncode)\n",
+ " print(\"stdout\", stdout.decode(\"UTF-8\"))\n",
+ " print(\"stderr\", stderr.decode(\"UTF-8\"))\n",
+ " print(\"p.returncode\", p.returncode)\n",
"\n",
- " if p.returncode == config.system_process['SUCCESS']:\n",
- " batch_job_sentence = stdout.decode('UTF-8')\n",
- " print('batch_job_sentence', batch_job_sentence)\n",
- " id_slurm_job = batch_job_sentence.replace(\"Submitted batch job \",\"\")\n",
- " id_slurm_job = re.sub(r\"[\\n\\t\\s]*\", \"\", id_slurm_job)\n",
- "\n"
+ " if p.returncode == config.system_process[\"SUCCESS\"]:\n",
+ " batch_job_sentence = stdout.decode(\"UTF-8\")\n",
+ " print(\"batch_job_sentence\", batch_job_sentence)\n",
+ " id_slurm_job = batch_job_sentence.replace(\"Submitted batch job \", \"\")\n",
+ " id_slurm_job = re.sub(r\"[\\n\\t\\s]*\", \"\", id_slurm_job)"
]
},
{
diff --git a/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb b/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb
index 80f1f479..e27eeb69 100644
--- a/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb
+++ b/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb
@@ -24,6 +24,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -33,18 +34,17 @@
"metadata": {},
"outputs": [],
"source": [
- "import datajoint as dj\n",
- "import os\n",
+ "import glob\n",
"import pathlib\n",
- "import pandas as pd\n",
+ "\n",
"import cv2\n",
- "import glob\n",
- "import numpy as np\n",
+ "import datajoint as dj\n",
"import matplotlib.pyplot as plt\n",
- "from matplotlib import patches\n",
- "from skimage.measure import EllipseModel\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "from scipy import stats\n",
"from skimage.draw import ellipse_perimeter\n",
- "from scipy import stats"
+ "from skimage.measure import EllipseModel"
]
},
{
@@ -62,8 +62,8 @@
],
"source": [
"dj.conn()\n",
- "pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')\n",
- "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')"
+ "pupillometry_db = dj.create_virtual_module(\"pupillometry\", \"u19_pupillometry\")\n",
+ "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")"
]
},
{
@@ -93,8 +93,8 @@
"frameNumber = 1000\n",
"\n",
"key = dict()\n",
- "key['subject_fullname'] = 'efonseca_ef111_act111'\n",
- "key['session_date'] = '2023-06-24'\n",
+ "key[\"subject_fullname\"] = \"efonseca_ef111_act111\"\n",
+ "key[\"session_date\"] = \"2023-06-24\"\n",
"\n",
"key"
]
@@ -122,26 +122,34 @@
],
"source": [
"# Get video location\n",
- "key_pupil = (acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key).fetch(as_dict=True)[0]\n",
- "db_pupil_data = (pupillometry_db.PupillometrySessionModelData & key).fetch(as_dict=True)[0]\n",
- "pupillometry_dir = dj.config.get('custom', {}).get('pupillometry_root_data_dir',None)\n",
+ "key_pupil = (\n",
+ " acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key\n",
+ ").fetch(as_dict=True)[0]\n",
+ "db_pupil_data = (pupillometry_db.PupillometrySessionModelData & key).fetch(\n",
+ " as_dict=True\n",
+ ")[0]\n",
+ "pupillometry_dir = dj.config.get(\"custom\", {}).get(\"pupillometry_root_data_dir\", None)\n",
"if pupillometry_dir is None:\n",
- " raise Exception('pupillometry_root_data_dir not found in config, run initial_conf.py again')\n",
+ " raise Exception(\n",
+ " \"pupillometry_root_data_dir not found in config, run initial_conf.py again\"\n",
+ " )\n",
"pupillometry_raw_dir = pupillometry_dir[0]\n",
- "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil['remote_path_video_file'])\n",
- "print('videoPath', videoPath)\n",
+ "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil[\"remote_path_video_file\"])\n",
+ "print(\"videoPath\", videoPath)\n",
"\n",
"# Get output location\n",
"pupillometry_processed_dir = pupillometry_dir[1]\n",
- "output_dir = pathlib.Path(pupillometry_processed_dir,pathlib.Path(key_pupil['remote_path_video_file']).parent)\n",
+ "output_dir = pathlib.Path(\n",
+ " pupillometry_processed_dir, pathlib.Path(key_pupil[\"remote_path_video_file\"]).parent\n",
+ ")\n",
"\n",
- "#Find h5 files (output from deeplabcut)\n",
- "h5_files = glob.glob(str(output_dir) + '/*.h5')\n",
+ "# Find h5 files (output from deeplabcut)\n",
+ "h5_files = glob.glob(str(output_dir) + \"/*.h5\")\n",
"if len(h5_files) != 1:\n",
- " raise Exception('Didn''t find any h5 files after deeplabcut analyze_video')\n",
+ " raise Exception(\"Didnt find any h5 files after deeplabcut analyze_video\")\n",
"else:\n",
" h5_files = h5_files[0]\n",
- "print('h5_files', h5_files)"
+ "print(\"h5_files\", h5_files)"
]
},
{
@@ -172,13 +180,11 @@
"\n",
"cap = cv2.VideoCapture(str(videoPath))\n",
"\n",
- "# count the number of frames \n",
- "frames = cap.get(cv2.CAP_PROP_FRAME_COUNT) \n",
+ "# count the number of frames\n",
+ "frames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n",
"\n",
- "print('total # of frames', int(frames))\n",
- "print('Length output deeplabcut', labels.shape[0])\n",
- "\n",
- "\n"
+ "print(\"total # of frames\", int(frames))\n",
+ "print(\"Length output deeplabcut\", labels.shape[0])"
]
},
{
@@ -192,10 +198,9 @@
"\n",
"totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n",
"# Set frame position\n",
- "cap.set(cv2.CAP_PROP_POS_FRAMES,frameNumber)\n",
+ "cap.set(cv2.CAP_PROP_POS_FRAMES, frameNumber)\n",
"\n",
- "ret, frame = cap.read()\n",
- "\n"
+ "ret, frame = cap.read()"
]
},
{
@@ -217,14 +222,12 @@
}
],
"source": [
- "\n",
- "\n",
"# Take a subset from the data\n",
"subset = labels.loc[frameNumber]\n",
"# Get the pupil markers (0 to 8) from the multiindex array obtained from deeplabcut\n",
- "x = subset.xs('x', level='coords').to_numpy()[0:8]\n",
- "y = subset.xs('y', level='coords').to_numpy()[0:8]\n",
- "xy = np.column_stack((x,y))\n",
+ "x = subset.xs(\"x\", level=\"coords\").to_numpy()[0:8]\n",
+ "y = subset.xs(\"y\", level=\"coords\").to_numpy()[0:8]\n",
+ "xy = np.column_stack((x, y))\n",
"\n",
"# Estimate an ellipse based on the points generated around the pupil, this is enough to get an estimate of the pupil area in pixels\n",
"ellipse = EllipseModel()\n",
@@ -232,11 +235,17 @@
"\n",
"# Validate by ploting the points and the fitting ellipse on the image from the video\n",
"fig, ax = plt.subplots()\n",
- "plt.rcParams['figure.dpi'] = 250\n",
+ "plt.rcParams[\"figure.dpi\"] = 250\n",
"img = frame\n",
- "ax.scatter(xy[:,0], xy[:,1], s=0.1, color='orange')\n",
+ "ax.scatter(xy[:, 0], xy[:, 1], s=0.1, color=\"orange\")\n",
"# Draw the ellipse on the original image\n",
- "cy, cx = ellipse_perimeter(int(ellipse.params[1]), int(ellipse.params[0]), int(ellipse.params[3]), int(ellipse.params[2]), ellipse.params[4])\n",
+ "cy, cx = ellipse_perimeter(\n",
+ " int(ellipse.params[1]),\n",
+ " int(ellipse.params[0]),\n",
+ " int(ellipse.params[3]),\n",
+ " int(ellipse.params[2]),\n",
+ " ellipse.params[4],\n",
+ ")\n",
"img[cy, cx] = 255\n",
"ax.imshow(img)\n",
"\n",
@@ -250,7 +259,7 @@
"metadata": {},
"outputs": [],
"source": [
- "df = pd.DataFrame(db_pupil_data['pupil_diameter'], columns=['PupilDiameter'])"
+ "df = pd.DataFrame(db_pupil_data[\"pupil_diameter\"], columns=[\"PupilDiameter\"])"
]
},
{
@@ -272,15 +281,15 @@
}
],
"source": [
- "\n",
"# Get a boolean array where true correspond to the frame with an outlier diameter\n",
- "zscore = np.abs(stats.zscore(df, nan_policy='omit'))\n",
+ "zscore = np.abs(stats.zscore(df, nan_policy=\"omit\"))\n",
"outlierFlags = np.abs(zscore) > 2\n",
"outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: \"OutlierFlag\"})\n",
"\n",
"# Get the list of outlier frames\n",
"from itertools import compress\n",
- "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags['OutlierFlag']))\n",
+ "\n",
+ "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags[\"OutlierFlag\"]))\n",
"\n",
"\n",
"# Retrieve an outlier frames sample from the video for visual inspection\n",
@@ -288,20 +297,20 @@
"fig = plt.figure(figsize=(7, 7))\n",
"columns = 5\n",
"rows = 5\n",
- "for i in range(1, columns*rows +1):\n",
+ "for i in range(1, columns * rows + 1):\n",
" # Get total number of frames\n",
" totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n",
" # Set frame position\n",
- " cap.set(cv2.CAP_PROP_POS_FRAMES,outlierFrames[i])\n",
+ " cap.set(cv2.CAP_PROP_POS_FRAMES, outlierFrames[i])\n",
" ret, frame = cap.read()\n",
"\n",
" fig.add_subplot(rows, columns, i)\n",
" plt.title(\"Frame \" + str(outlierFrames[i]), fontsize=5)\n",
" plt.imshow(frame)\n",
- " plt.axis('off')\n",
+ " plt.axis(\"off\")\n",
"plt.show()\n",
"cap.release()\n",
- "cv2.destroyAllWindows()\n"
+ "cv2.destroyAllWindows()"
]
},
{
@@ -355,29 +364,33 @@
"source": [
"# Concatenate outlier flags array to remove outliers from pupil diameter array\n",
"temp = pd.concat([df, outlierFlags], axis=1)\n",
- "temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None\n",
- "pupilDiameter = temp['PupilDiameter']\n",
+ "temp.loc[temp[\"OutlierFlag\"] == True, \"PupilDiameter\"] = None\n",
+ "pupilDiameter = temp[\"PupilDiameter\"]\n",
"\n",
"# This might be better to leave open for the experimenter.\n",
"# Replace the outlier diameter data using a cubic spline interpolator\n",
"# pupilDiameter = temp['PupilDiameter'].interpolate(method='spline', order=3, s=0)\n",
"\n",
- "fig, (ax1, ax2) = plt.subplots(1,2)\n",
+ "fig, (ax1, ax2) = plt.subplots(1, 2)\n",
"# plt.rcParams['figure.dpi'] = 250\n",
"\n",
"ax1.plot(pupilDiameter[500:1500], linewidth=0.25)\n",
"# Filter by using a moving average window of 10 samples\n",
- "ax1.plot(pupilDiameter[500:1500].rolling(10).mean(), linewidth=0.5, color='red')\n",
- "ax1.set_xlabel('Frame')\n",
- "ax1.set_ylabel('Pupil diameter [px]')\n",
- "ax1.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n",
+ "ax1.plot(pupilDiameter[500:1500].rolling(10).mean(), linewidth=0.5, color=\"red\")\n",
+ "ax1.set_xlabel(\"Frame\")\n",
+ "ax1.set_ylabel(\"Pupil diameter [px]\")\n",
+ "ax1.legend(\n",
+ " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n",
+ ")\n",
"\n",
"ax2.plot(pupilDiameter[0:20000], linewidth=0.25)\n",
"# Filter by using a moving average window of 10 samples\n",
- "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color='red')\n",
- "ax2.set_xlabel('Frame')\n",
- "ax2.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n",
- "#fig.savefig('test.eps', format='eps')"
+ "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color=\"red\")\n",
+ "ax2.set_xlabel(\"Frame\")\n",
+ "ax2.legend(\n",
+ " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n",
+ ")\n",
+ "# fig.savefig('test.eps', format='eps')"
]
},
{
diff --git a/notebooks/read_virmen_behavior_file.ipynb b/notebooks/read_virmen_behavior_file.ipynb
index 0896b02d..f4075108 100644
--- a/notebooks/read_virmen_behavior_file.ipynb
+++ b/notebooks/read_virmen_behavior_file.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -32,24 +33,15 @@
}
],
"source": [
- "import datajoint as dj\n",
- "import pandas as pd\n",
- "import numpy as np\n",
- "import subprocess\n",
"import os\n",
- "import pathlib\n",
- "import numpy as np\n",
- "import u19_pipeline.utils.path_utils as pu\n",
- "from sklearn.linear_model import LinearRegression\n",
- "import matplotlib.pyplot as plt\n",
- "import datetime\n",
"from shutil import which\n",
"\n",
+ "import datajoint as dj\n",
+ "\n",
"import u19_pipeline.utils.matlab_utils as mu\n",
- "from scipy.io import loadmat, matlab\n",
"\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "from u19_pipeline import lab\n"
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "from u19_pipeline import lab"
]
},
{
@@ -65,9 +57,9 @@
"metadata": {},
"outputs": [],
"source": [
- "local_matlab_path = '/Applications/MATLAB_R2020a.app/bin/'\n",
- "if which('matlab') is None:\n",
- " os.environ['PATH'] += ':'+local_matlab_path"
+ "local_matlab_path = \"/Applications/MATLAB_R2020a.app/bin/\"\n",
+ "if which(\"matlab\") is None:\n",
+ " os.environ[\"PATH\"] += \":\" + local_matlab_path"
]
},
{
@@ -84,10 +76,12 @@
}
],
"source": [
- "if which('matlab') is None:\n",
- " print('Matlab command not found, add correct path installation or proceed (Choice & TrialType data will not be read from the behavior file')\n",
+ "if which(\"matlab\") is None:\n",
+ " print(\n",
+ " \"Matlab command not found, add correct path installation or proceed (Choice & TrialType data will not be read from the behavior file\"\n",
+ " )\n",
"else:\n",
- " print('Matlab installation found !!')"
+ " print(\"Matlab installation found !!\")"
]
},
{
@@ -109,10 +103,12 @@
}
],
"source": [
- "subject = 'efonseca_ef884_actp003'\n",
- "date = '2021-12-03'\n",
+ "subject = \"efonseca_ef884_actp003\"\n",
+ "date = \"2021-12-03\"\n",
"\n",
- "path = (acquisition.SessionStarted & dict(session_date =date, subject_fullname=subject)).fetch1('new_remote_path_behavior_file')\n",
+ "path = (\n",
+ " acquisition.SessionStarted & dict(session_date=date, subject_fullname=subject)\n",
+ ").fetch1(\"new_remote_path_behavior_file\")\n",
"\n",
"\n",
"mat_file = lab.Path().get_local_path2(path)\n",
@@ -146,7 +142,7 @@
}
],
"source": [
- "matin['log']['block']['trial'][12]['trialType']"
+ "matin[\"log\"][\"block\"][\"trial\"][12][\"trialType\"]"
]
},
{
@@ -662,7 +658,7 @@
}
],
"source": [
- "df_behavior_file.loc[6, 'trial_type']"
+ "df_behavior_file.loc[6, \"trial_type\"]"
]
},
{
diff --git a/notebooks/storage_calculator_across_time.ipynb b/notebooks/storage_calculator_across_time.ipynb
index 650567bc..e3ecbc51 100644
--- a/notebooks/storage_calculator_across_time.ipynb
+++ b/notebooks/storage_calculator_across_time.ipynb
@@ -15,6 +15,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -24,16 +25,14 @@
"metadata": {},
"outputs": [],
"source": [
- "import datajoint as dj\n",
- "import pandas as pd\n",
- "import subprocess\n",
+ "import matplotlib.pyplot as plt\n",
"import numpy as np\n",
- "import u19_pipeline.utils.path_utils as pu\n",
+ "import pandas as pd\n",
"from sklearn.linear_model import LinearRegression\n",
- "import matplotlib.pyplot as plt\n",
- "import datetime\n",
"\n",
- "fig_size = (15,11)"
+ "import u19_pipeline.utils.path_utils as pu\n",
+ "\n",
+ "fig_size = (15, 11)"
]
},
{
@@ -65,28 +64,38 @@
}
],
"source": [
- "folder_list_to_read_from = ['/mnt/cup/PNI-centers/Bezos/Ryan/', '/mnt/cup/PNI-centers/Bezos/Scott/twoP', '/mnt/cup/labs/witten/Scott/twoPhoton', '/mnt/cup/braininit/RigData/mesoscope/imaging', '/mnt/cup/PNI-centers/Bezos/RigData/scope/bay3/sakoay', '/mnt/cup/PNI-centers/Bezos/RigData/scope/bay3/edward', '/mnt/cup/braininit/RigData/training/npx/electrophysiology']\n",
+ "folder_list_to_read_from = [\n",
+ " \"/mnt/cup/PNI-centers/Bezos/Ryan/\",\n",
+ " \"/mnt/cup/PNI-centers/Bezos/Scott/twoP\",\n",
+ " \"/mnt/cup/labs/witten/Scott/twoPhoton\",\n",
+ " \"/mnt/cup/braininit/RigData/mesoscope/imaging\",\n",
+ " \"/mnt/cup/PNI-centers/Bezos/RigData/scope/bay3/sakoay\",\n",
+ " \"/mnt/cup/PNI-centers/Bezos/RigData/scope/bay3/edward\",\n",
+ " \"/mnt/cup/braininit/RigData/training/npx/electrophysiology\",\n",
+ "]\n",
"\n",
"for idx, folder in enumerate(folder_list_to_read_from):\n",
" print(idx, folder)\n",
" if idx == 0:\n",
" df_storage = pd.DataFrame(pu.get_size_directory_time(folder))\n",
- " df_storage['parent_folder'] = folder\n",
+ " df_storage[\"parent_folder\"] = folder\n",
" else:\n",
" df_storage_current = pd.DataFrame(pu.get_size_directory_time(folder))\n",
- " df_storage_current['parent_folder'] = folder\n",
+ " df_storage_current[\"parent_folder\"] = folder\n",
" df_storage = df_storage.append(df_storage_current)\n",
"\n",
"df_storage.reset_index(drop=True)\n",
- "df_storage['size'] = df_storage['size'].astype(int)\n",
- "df_storage['size'] = df_storage['size'].astype(int)\n",
- "df_storage['size (TB)'] = df_storage['size']/(1024*1024*1024)\n",
- "df_storage['date'] = pd.to_datetime(df_storage['date'])\n",
- "df_storage['date'] = df_storage['date'].dt.normalize()\n",
- "df_storage = df_storage.drop(['size', 'directory'], axis=1)\n",
- "df_storage = df_storage.sort_values(by=['date'])\n",
- "df_storage = df_storage.loc[df_storage['date'] > pd.to_datetime(\"'2018-01-01'\".replace(\"'\",\"\")), :]\n",
- "copy_df_storage = df_storage.copy()\n"
+ "df_storage[\"size\"] = df_storage[\"size\"].astype(int)\n",
+ "df_storage[\"size\"] = df_storage[\"size\"].astype(int)\n",
+ "df_storage[\"size (TB)\"] = df_storage[\"size\"] / (1024 * 1024 * 1024)\n",
+ "df_storage[\"date\"] = pd.to_datetime(df_storage[\"date\"])\n",
+ "df_storage[\"date\"] = df_storage[\"date\"].dt.normalize()\n",
+ "df_storage = df_storage.drop([\"size\", \"directory\"], axis=1)\n",
+ "df_storage = df_storage.sort_values(by=[\"date\"])\n",
+ "df_storage = df_storage.loc[\n",
+ " df_storage[\"date\"] > pd.to_datetime(\"'2018-01-01'\".replace(\"'\", \"\")), :\n",
+ "]\n",
+ "copy_df_storage = df_storage.copy()"
]
},
{
@@ -102,11 +111,11 @@
"metadata": {},
"outputs": [],
"source": [
- "df_storage_day = df_storage.groupby(by=['date']).sum()\n",
+ "df_storage_day = df_storage.groupby(by=[\"date\"]).sum()\n",
"df_storage_day = df_storage_day.reset_index()\n",
- "df_storage_day = df_storage.sort_values(by=['date'])\n",
+ "df_storage_day = df_storage.sort_values(by=[\"date\"])\n",
"df_storage_day = df_storage_day.reset_index(drop=True)\n",
- "df_storage_day['size (TB) sum'] = df_storage_day['size (TB)'].cumsum() \n"
+ "df_storage_day[\"size (TB) sum\"] = df_storage_day[\"size (TB)\"].cumsum()"
]
},
{
@@ -217,15 +226,18 @@
}
],
"source": [
- "\n",
- "df_storage_day_log = df_storage_day.loc[df_storage_day['size (TB) sum'] > 5, :]\n",
+ "df_storage_day_log = df_storage_day.loc[df_storage_day[\"size (TB) sum\"] > 5, :]\n",
"df_storage_day_log.reset_index(drop=True)\n",
- "df_storage_day_log['numeric_time'] = (df_storage_day_log['date'] - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta('1s')\n",
+ "df_storage_day_log[\"numeric_time\"] = (\n",
+ " df_storage_day_log[\"date\"] - pd.Timestamp(\"1970-01-01\")\n",
+ ") // pd.Timedelta(\"1s\")\n",
"\n",
- "df_storage_day_log = df_storage_day_log.drop_duplicates(subset=['numeric_time'])\n",
+ "df_storage_day_log = df_storage_day_log.drop_duplicates(subset=[\"numeric_time\"])\n",
"\n",
- "X = df_storage_day_log['numeric_time'].values\n",
- "y = np.log(df_storage_day_log['size (TB) sum'].values) # Apply natural log function to the target\n",
+ "X = df_storage_day_log[\"numeric_time\"].values\n",
+ "y = np.log(\n",
+ " df_storage_day_log[\"size (TB) sum\"].values\n",
+ ") # Apply natural log function to the target\n",
"\n",
"X = X.reshape(-1, 1)\n",
"y = y.reshape(-1, 1)\n",
@@ -233,27 +245,31 @@
"model = LinearRegression()\n",
"model.fit(X, y)\n",
"X2 = X\n",
- "ini_date = df_storage_day_log.loc[df_storage_day_log.index[-1],'date']\n",
+ "ini_date = df_storage_day_log.loc[df_storage_day_log.index[-1], \"date\"]\n",
"for i in range(6):\n",
" ini_date += pd.DateOffset(months=4)\n",
- " num_time = (ini_date - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta('1s')\n",
+ " num_time = (ini_date - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta(\"1s\")\n",
" X2 = np.append(X2, num_time)\n",
"\n",
"X2 = X2.reshape(-1, 1)\n",
- "y_pred = np.exp(model.predict(X2)) # Apply exponential function (inverse of natural log) to the predictions\n",
+ "y_pred = np.exp(\n",
+ " model.predict(X2)\n",
+ ") # Apply exponential function (inverse of natural log) to the predictions\n",
"\n",
"\n",
- "X2_date = pd.to_datetime(np.squeeze(X2),unit='s')\n",
- "df_pred = pd.DataFrame(data=y_pred, index=X2_date, columns=['TB pred'])\n",
- "df_pred.index.name = 'Date'\n",
+ "X2_date = pd.to_datetime(np.squeeze(X2), unit=\"s\")\n",
+ "df_pred = pd.DataFrame(data=y_pred, index=X2_date, columns=[\"TB pred\"])\n",
+ "df_pred.index.name = \"Date\"\n",
"df_pred = df_pred.reset_index()\n",
- "print('R^2', model.score(X, y))\n",
- "print(df_storage_day_log.loc[df_storage_day_log.index[-1], 'size (TB) sum'])\n",
- "current_TB = \"{:0.2f}\".format(df_storage_day_log.loc[df_storage_day_log.index[-1], 'size (TB) sum']) + \" TB\"\n",
- "df_pred.tail(8)\n",
- "\n",
- "\n",
- "\n"
+ "print(\"R^2\", model.score(X, y))\n",
+ "print(df_storage_day_log.loc[df_storage_day_log.index[-1], \"size (TB) sum\"])\n",
+ "current_TB = (\n",
+ " \"{:0.2f}\".format(\n",
+ " df_storage_day_log.loc[df_storage_day_log.index[-1], \"size (TB) sum\"]\n",
+ " )\n",
+ " + \" TB\"\n",
+ ")\n",
+ "df_pred.tail(8)"
]
},
{
@@ -285,15 +301,20 @@
"source": [
"fig, axs = plt.subplots(1, 1, figsize=fig_size)\n",
"\n",
- "plt.plot(df_storage_day_log['date'] , df_storage_day_log['size (TB) sum'], linewidth=3, color ='k')\n",
- "plt.plot(X2_date, y_pred) \n",
- "plt.yscale('log')\n",
+ "plt.plot(\n",
+ " df_storage_day_log[\"date\"],\n",
+ " df_storage_day_log[\"size (TB) sum\"],\n",
+ " linewidth=3,\n",
+ " color=\"k\",\n",
+ ")\n",
+ "plt.plot(X2_date, y_pred)\n",
+ "plt.yscale(\"log\")\n",
"\n",
"plt.xticks(fontsize=16)\n",
"plt.yticks(fontsize=16)\n",
"plt.xlabel(\"Date\", fontsize=18)\n",
- "plt.ylabel(\"Storage space (TB)\", fontsize=18);\n",
- "plt.title('Storage on imaging & electrophysiology U19: ' + current_TB, fontsize=20);\n"
+ "plt.ylabel(\"Storage space (TB)\", fontsize=18)\n",
+ "plt.title(\"Storage on imaging & electrophysiology U19: \" + current_TB, fontsize=20);"
]
},
{
@@ -404,14 +425,20 @@
}
],
"source": [
- "df_storage_day_lin = df_storage_day.loc[df_storage_day['date'] > pd.to_datetime('20210301', format='%Y%m%d'), :]\n",
+ "df_storage_day_lin = df_storage_day.loc[\n",
+ " df_storage_day[\"date\"] > pd.to_datetime(\"20210301\", format=\"%Y%m%d\"), :\n",
+ "]\n",
"df_storage_day_lin.reset_index(drop=True)\n",
- "df_storage_day_lin['numeric_time'] = (df_storage_day_lin['date'] - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta('1s')\n",
+ "df_storage_day_lin[\"numeric_time\"] = (\n",
+ " df_storage_day_lin[\"date\"] - pd.Timestamp(\"1970-01-01\")\n",
+ ") // pd.Timedelta(\"1s\")\n",
"\n",
- "df_storage_day_lin = df_storage_day_lin.drop_duplicates(subset=['numeric_time'])\n",
+ "df_storage_day_lin = df_storage_day_lin.drop_duplicates(subset=[\"numeric_time\"])\n",
"\n",
- "X = df_storage_day_lin['numeric_time'].values\n",
- "y = (df_storage_day_lin['size (TB) sum'].values) # Apply natural log function to the target\n",
+ "X = df_storage_day_lin[\"numeric_time\"].values\n",
+ "y = df_storage_day_lin[\n",
+ " \"size (TB) sum\"\n",
+ "].values # Apply natural log function to the target\n",
"\n",
"X = X.reshape(-1, 1)\n",
"y = y.reshape(-1, 1)\n",
@@ -419,22 +446,24 @@
"model = LinearRegression()\n",
"model.fit(X, y)\n",
"X2 = X\n",
- "ini_date = df_storage_day_log.loc[df_storage_day_log.index[-1],'date']\n",
+ "ini_date = df_storage_day_log.loc[df_storage_day_log.index[-1], \"date\"]\n",
"for i in range(6):\n",
" ini_date += pd.DateOffset(months=4)\n",
- " num_time = (ini_date - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta('1s')\n",
+ " num_time = (ini_date - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta(\"1s\")\n",
" X2 = np.append(X2, num_time)\n",
"\n",
"X2 = X2.reshape(-1, 1)\n",
- "y_pred = (model.predict(X2)) # Apply exponential function (inverse of natural log) to the predictions\n",
+ "y_pred = model.predict(\n",
+ " X2\n",
+ ") # Apply exponential function (inverse of natural log) to the predictions\n",
"\n",
"\n",
- "X2_date = pd.to_datetime(np.squeeze(X2),unit='s')\n",
- "df_pred = pd.DataFrame(data=y_pred, index=X2_date, columns=['TB pred'])\n",
- "df_pred.index.name = 'Date'\n",
+ "X2_date = pd.to_datetime(np.squeeze(X2), unit=\"s\")\n",
+ "df_pred = pd.DataFrame(data=y_pred, index=X2_date, columns=[\"TB pred\"])\n",
+ "df_pred.index.name = \"Date\"\n",
"df_pred = df_pred.reset_index()\n",
- "print('R^2', model.score(X, y))\n",
- "print(df_storage_day_log.loc[df_storage_day_log.index[-1], 'size (TB) sum'])\n",
+ "print(\"R^2\", model.score(X, y))\n",
+ "print(df_storage_day_log.loc[df_storage_day_log.index[-1], \"size (TB) sum\"])\n",
"df_pred.tail(8)"
]
},
@@ -460,15 +489,17 @@
"source": [
"fig, axs = plt.subplots(1, 1, figsize=fig_size)\n",
"\n",
- "plt.plot(df_storage_day['date'] , df_storage_day['size (TB) sum'], linewidth=3, color ='k')\n",
- "plt.plot(X2_date, y_pred) \n",
+ "plt.plot(\n",
+ " df_storage_day[\"date\"], df_storage_day[\"size (TB) sum\"], linewidth=3, color=\"k\"\n",
+ ")\n",
+ "plt.plot(X2_date, y_pred)\n",
"\n",
"\n",
"plt.xticks(fontsize=16)\n",
"plt.yticks(fontsize=16)\n",
"plt.xlabel(\"Date\", fontsize=18)\n",
- "plt.ylabel(\"Storage space (TB)\", fontsize=18);\n",
- "plt.title('Storage on imaging & electrophysiology U19: ' + current_TB, fontsize=20);"
+ "plt.ylabel(\"Storage space (TB)\", fontsize=18)\n",
+ "plt.title(\"Storage on imaging & electrophysiology U19: \" + current_TB, fontsize=20);"
]
},
{
@@ -485,11 +516,10 @@
"outputs": [],
"source": [
"df_storage_year = df_storage.copy()\n",
- "df_storage_year = df_storage_year.sort_values(by=['date'])\n",
- "df_storage_year['year'] = df_storage_year['date'].dt.year\n",
- "df_storage_year = df_storage_year.groupby(df_storage_year['year']).sum()\n",
- "df_storage_year = df_storage_year.reset_index()\n",
- "\n"
+ "df_storage_year = df_storage_year.sort_values(by=[\"date\"])\n",
+ "df_storage_year[\"year\"] = df_storage_year[\"date\"].dt.year\n",
+ "df_storage_year = df_storage_year.groupby(df_storage_year[\"year\"]).sum()\n",
+ "df_storage_year = df_storage_year.reset_index()"
]
},
{
@@ -576,18 +606,18 @@
"source": [
"fig, axs = plt.subplots(1, 1, figsize=fig_size)\n",
"\n",
- "#plt.scatter(df_storage_day['date'] , df_storage_day['size (TB) sum'], s=100, color = 'k')\n",
- "plt.bar(df_storage_year['year'], df_storage_year['size (TB)'])\n",
+ "# plt.scatter(df_storage_day['date'] , df_storage_day['size (TB) sum'], s=100, color = 'k')\n",
+ "plt.bar(df_storage_year[\"year\"], df_storage_year[\"size (TB)\"])\n",
"\n",
"plt.xticks(fontsize=16)\n",
"plt.yticks(fontsize=16)\n",
"plt.xlabel(\"Year\", fontsize=18)\n",
"plt.ylabel(\"Storage space (TB)\", fontsize=18)\n",
- "plt.title('Storage on imaging & electrophysiology U19 by year', fontsize=20)\n",
+ "plt.title(\"Storage on imaging & electrophysiology U19 by year\", fontsize=20)\n",
"\n",
- "all_years = df_storage_year['year'].values\n",
- "axs.set_xticks(all_years);\n",
- "df_storage_year\n"
+ "all_years = df_storage_year[\"year\"].values\n",
+ "axs.set_xticks(all_years)\n",
+ "df_storage_year"
]
},
{
@@ -684,12 +714,18 @@
}
],
"source": [
- "df_size_parent_folder = df_storage.groupby(by=['parent_folder']).sum()\n",
+ "df_size_parent_folder = df_storage.groupby(by=[\"parent_folder\"]).sum()\n",
"df_size_parent_folder = df_size_parent_folder.reset_index()\n",
- "df_size_parent_folder['parent_folder'] = df_size_parent_folder['parent_folder'].str.replace('/mnt/cup/', '')\n",
- "df_size_parent_folder['parent_folder'] = df_size_parent_folder['parent_folder'].str.replace('braininit/RigData/', '')\n",
- "df_size_parent_folder['parent_folder'] = df_size_parent_folder['parent_folder'].str.replace('PNI-centers/Bezos/RigData/', '')\n",
- "df_size_parent_folder\n"
+ "df_size_parent_folder[\"parent_folder\"] = df_size_parent_folder[\n",
+ " \"parent_folder\"\n",
+ "].str.replace(\"/mnt/cup/\", \"\")\n",
+ "df_size_parent_folder[\"parent_folder\"] = df_size_parent_folder[\n",
+ " \"parent_folder\"\n",
+ "].str.replace(\"braininit/RigData/\", \"\")\n",
+ "df_size_parent_folder[\"parent_folder\"] = df_size_parent_folder[\n",
+ " \"parent_folder\"\n",
+ "].str.replace(\"PNI-centers/Bezos/RigData/\", \"\")\n",
+ "df_size_parent_folder"
]
},
{
@@ -714,17 +750,17 @@
"source": [
"fig, axs = plt.subplots(1, 1, figsize=fig_size)\n",
"index = np.arange(df_size_parent_folder.shape[0])\n",
- "plt.bar(index, df_size_parent_folder['size (TB)'])\n",
+ "plt.bar(index, df_size_parent_folder[\"size (TB)\"])\n",
"\n",
"plt.xticks(fontsize=16)\n",
"plt.yticks(fontsize=16)\n",
"plt.xlabel(\"Directory\", fontsize=18)\n",
"plt.ylabel(\"Storage space (TB)\", fontsize=18)\n",
- "plt.title('Storage on imaging & electrophysiology U19 by main dir', fontsize=20)\n",
+ "plt.title(\"Storage on imaging & electrophysiology U19 by main dir\", fontsize=20)\n",
"\n",
- "all_folders = df_size_parent_folder['parent_folder'].values\n",
- "axs.set_xticks(index);\n",
- "axs.set_xticklabels(all_folders, rotation = 45, ha=\"right\");"
+ "all_folders = df_size_parent_folder[\"parent_folder\"].values\n",
+ "axs.set_xticks(index)\n",
+ "axs.set_xticklabels(all_folders, rotation=45, ha=\"right\");"
]
},
{
diff --git a/notebooks/training_days_per_protocol.ipynb b/notebooks/training_days_per_protocol.ipynb
index 7bc7663d..7fae76ba 100644
--- a/notebooks/training_days_per_protocol.ipynb
+++ b/notebooks/training_days_per_protocol.ipynb
@@ -22,20 +22,18 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"\n",
+ "import copy\n",
+ "from datetime import date, timedelta\n",
+ "\n",
"import datajoint as dj\n",
- "from datetime import date, datetime, timedelta\n",
- "import pandas as pd\n",
"import numpy as np\n",
+ "import pandas as pd\n",
"import pylab as plt\n",
- "import matplotlib.patches as mpatches\n",
"\n",
- "from matplotlib import cm\n",
- "from u19_pipeline import utility\n",
- "from inspect import getmembers, isfunction\n",
- "import u19_pipeline.alert_system.behavior_metrics as bm\n",
- "import copy\n"
+ "import u19_pipeline.alert_system.behavior_metrics as bm"
]
},
{
@@ -52,8 +50,8 @@
"outputs": [],
"source": [
"dj.conn()\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -170,12 +168,15 @@
}
],
"source": [
- "\n",
- "all_protocols_df = pd.DataFrame(acquisition.Session.fetch('session_protocol', as_dict=True))\n",
- "all_protocols_df = all_protocols_df.groupby('session_protocol').agg({'session_protocol': [('num_sessions', 'count')]})\n",
+ "all_protocols_df = pd.DataFrame(\n",
+ " acquisition.Session.fetch(\"session_protocol\", as_dict=True)\n",
+ ")\n",
+ "all_protocols_df = all_protocols_df.groupby(\"session_protocol\").agg(\n",
+ " {\"session_protocol\": [(\"num_sessions\", \"count\")]}\n",
+ ")\n",
"all_protocols_df.columns = all_protocols_df.columns.droplevel()\n",
- "all_protocols_df = all_protocols_df.sort_values(by='num_sessions', ascending=False)\n",
- "all_protocols_df = all_protocols_df.loc[all_protocols_df['num_sessions'] > 100, :]\n",
+ "all_protocols_df = all_protocols_df.sort_values(by=\"num_sessions\", ascending=False)\n",
+ "all_protocols_df = all_protocols_df.loc[all_protocols_df[\"num_sessions\"] > 100, :]\n",
"all_protocols_df"
]
},
@@ -196,7 +197,7 @@
}
],
"source": [
- "session_protocol_q = 'session_protocol = \"' +all_protocols_df.index[1] + '\"'\n",
+ "session_protocol_q = 'session_protocol = \"' + all_protocols_df.index[1] + '\"'\n",
"session_protocol_q"
]
},
@@ -224,38 +225,38 @@
"source": [
"today = date.today() + timedelta(days=-1)\n",
"start_week = today - timedelta(days=today.weekday())\n",
- "#end_lastweek = start_week - timedelta(days=1)\n",
- "start_week_str = start_week.strftime('%Y-%m-%d')\n",
- "#end_lastweek_str = end_lastweek.strftime('%Y-%m-%d')\n",
+ "# end_lastweek = start_week - timedelta(days=1)\n",
+ "start_week_str = start_week.strftime(\"%Y-%m-%d\")\n",
+ "# end_lastweek_str = end_lastweek.strftime('%Y-%m-%d')\n",
"\n",
- "query_old_subjects = 'session_date <= \"'+ start_week_str + '\"'\n",
- "query_new_subjects = 'session_date > \"'+ start_week_str + '\"'\n",
- "not_testuser = 'subject_fullname not like \"testuser%\"'\n",
- "not_testsubjects = 'subject_fullname not like \"%_test\"'\n",
- "not_bad_sessions = 'is_bad_session = 0'\n",
- "base_query = [session_protocol_q, not_testuser, not_testsubjects, not_bad_sessions]\n",
+ "query_old_subjects = 'session_date <= \"' + start_week_str + '\"'\n",
+ "query_new_subjects = 'session_date > \"' + start_week_str + '\"'\n",
+ "not_testuser = 'subject_fullname not like \"testuser%\"'\n",
+ "not_testsubjects = 'subject_fullname not like \"%_test\"'\n",
+ "not_bad_sessions = \"is_bad_session = 0\"\n",
+ "base_query = [session_protocol_q, not_testuser, not_testsubjects, not_bad_sessions]\n",
"\n",
- "query_old_subjects = [query_old_subjects] + base_query\n",
- "query_new_subjects = [query_new_subjects] + base_query\n",
+ "query_old_subjects = [query_old_subjects] + base_query\n",
+ "query_new_subjects = [query_new_subjects] + base_query\n",
"\n",
"base_session_table_filtered = acquisition.Session & base_query\n",
- "session_table_filtered = copy.deepcopy(base_session_table_filtered)\n",
+ "session_table_filtered = copy.deepcopy(base_session_table_filtered)\n",
"for filter_key in query_old_subjects:\n",
" session_table_filtered = session_table_filtered & filter_key\n",
- "subject_old_sessions = set((session_table_filtered).fetch('subject_fullname').tolist())\n",
+ "subject_old_sessions = set((session_table_filtered).fetch(\"subject_fullname\").tolist())\n",
"\n",
- "session_table_filtered = copy.deepcopy(base_session_table_filtered)\n",
+ "session_table_filtered = copy.deepcopy(base_session_table_filtered)\n",
"for filter_key in query_new_subjects:\n",
" session_table_filtered = session_table_filtered & filter_key\n",
- "subject_new_sessions = set((session_table_filtered).fetch('subject_fullname').tolist())\n",
+ "subject_new_sessions = set((session_table_filtered).fetch(\"subject_fullname\").tolist())\n",
"\n",
"subjects_only_old = list(subject_old_sessions - subject_new_sessions)\n",
"\n",
- "subjects_only_oldquery = [{'subject_fullname': i} for i in subjects_only_old]\n",
- "subjects_new_sessions_query = [{'subject_fullname': i} for i in subject_new_sessions]\n",
+ "subjects_only_oldquery = [{\"subject_fullname\": i} for i in subjects_only_old]\n",
+ "subjects_new_sessions_query = [{\"subject_fullname\": i} for i in subject_new_sessions]\n",
"\n",
- "print('subjects this protocol did not train this week: ', len(subjects_only_oldquery))\n",
- "print('subjects this protocol did train this week: ', len(subjects_new_sessions_query))"
+ "print(\"subjects this protocol did not train this week: \", len(subjects_only_oldquery))\n",
+ "print(\"subjects this protocol did train this week: \", len(subjects_new_sessions_query))"
]
},
{
@@ -272,10 +273,21 @@
}
],
"source": [
- "session_old_df = pd.DataFrame((base_session_table_filtered & subjects_only_oldquery).fetch('KEY','session_protocol', 'level',order_by='session_start_time', as_dict=True))\n",
- "session_new_df = pd.DataFrame((base_session_table_filtered & subjects_new_sessions_query).fetch('KEY','session_protocol', 'level', order_by='session_start_time', as_dict=True))\n",
+ "session_old_df = pd.DataFrame(\n",
+ " (base_session_table_filtered & subjects_only_oldquery).fetch(\n",
+ " \"KEY\", \"session_protocol\", \"level\", order_by=\"session_start_time\", as_dict=True\n",
+ " )\n",
+ ")\n",
+ "session_new_df = pd.DataFrame(\n",
+ " (base_session_table_filtered & subjects_new_sessions_query).fetch(\n",
+ " \"KEY\", \"session_protocol\", \"level\", order_by=\"session_start_time\", as_dict=True\n",
+ " )\n",
+ ")\n",
"\n",
- "print('Sessions this protocol from subjects did not train this week: ', session_old_df.shape[0])"
+ "print(\n",
+ " \"Sessions this protocol from subjects did not train this week: \",\n",
+ " session_old_df.shape[0],\n",
+ ")"
]
},
{
@@ -480,8 +492,8 @@
}
],
"source": [
- "max_level_calculated = np.percentile(session_old_df['level'].values, 90)\n",
- "plt.hist(session_old_df['level'].values, bins=session_old_df['level'].max()-1)\n",
+ "max_level_calculated = np.percentile(session_old_df[\"level\"].values, 90)\n",
+ "plt.hist(session_old_df[\"level\"].values, bins=session_old_df[\"level\"].max() - 1)\n",
"max_level_calculated"
]
},
@@ -500,13 +512,29 @@
}
],
"source": [
- "level_query = 'level >= '+ str(max_level_calculated)\n",
- "session_table_filtered = base_session_table_filtered & subjects_only_oldquery & level_query\n",
- "subjects_only_old_level = set((session_table_filtered).fetch('subject_fullname').tolist())\n",
- "subjects_only_oldquery_level = [{'subject_fullname': i} for i in subjects_only_old_level]\n",
- "session_old_df = pd.DataFrame((base_session_table_filtered & subjects_only_oldquery_level).fetch('KEY', 'level', 'session_location',order_by='session_start_time', as_dict=True))\n",
- "print('subjects this protocol did not train this week to get to max level: ', len(subjects_only_oldquery_level))\n",
- "print('Sessions this protocol from subjects did not train this week that got to max level: ', session_old_df.shape[0])"
+ "level_query = \"level >= \" + str(max_level_calculated)\n",
+ "session_table_filtered = (\n",
+ " base_session_table_filtered & subjects_only_oldquery & level_query\n",
+ ")\n",
+ "subjects_only_old_level = set(\n",
+ " (session_table_filtered).fetch(\"subject_fullname\").tolist()\n",
+ ")\n",
+ "subjects_only_oldquery_level = [\n",
+ " {\"subject_fullname\": i} for i in subjects_only_old_level\n",
+ "]\n",
+ "session_old_df = pd.DataFrame(\n",
+ " (base_session_table_filtered & subjects_only_oldquery_level).fetch(\n",
+ " \"KEY\", \"level\", \"session_location\", order_by=\"session_start_time\", as_dict=True\n",
+ " )\n",
+ ")\n",
+ "print(\n",
+ " \"subjects this protocol did not train this week to get to max level: \",\n",
+ " len(subjects_only_oldquery_level),\n",
+ ")\n",
+ "print(\n",
+ " \"Sessions this protocol from subjects did not train this week that got to max level: \",\n",
+ " session_old_df.shape[0],\n",
+ ")"
]
},
{
@@ -697,18 +725,22 @@
],
"source": [
"# Count sessions for each subject (sequential count and total count)\n",
- "session_old_df = session_old_df.sort_values(by=['subject_fullname', 'session_date', 'session_number'])\n",
+ "session_old_df = session_old_df.sort_values(\n",
+ " by=[\"subject_fullname\", \"session_date\", \"session_number\"]\n",
+ ")\n",
"\n",
- "sequential_sessions = session_old_df.groupby(['subject_fullname']).cumcount() + 1\n",
- "num_sessions_df = session_old_df.groupby('subject_fullname').agg({'session_date': [('total_sessions', 'count')]})\n",
+ "sequential_sessions = session_old_df.groupby([\"subject_fullname\"]).cumcount() + 1\n",
+ "num_sessions_df = session_old_df.groupby(\"subject_fullname\").agg(\n",
+ " {\"session_date\": [(\"total_sessions\", \"count\")]}\n",
+ ")\n",
"num_sessions_df.columns = num_sessions_df.columns.droplevel()\n",
"num_sessions_df = num_sessions_df.reset_index()\n",
"\n",
- "#num_sessions_level_df = session_old_df.groupby(['subject_fullname').agg({'session_date': [('total_sessions', 'count')]})\n",
- "#num_sessions_df.columns = num_sessions_df.columns.droplevel()\n",
- "#num_sessions_df = num_sessions_df.reset_index()\n",
+ "# num_sessions_level_df = session_old_df.groupby(['subject_fullname').agg({'session_date': [('total_sessions', 'count')]})\n",
+ "# num_sessions_df.columns = num_sessions_df.columns.droplevel()\n",
+ "# num_sessions_df = num_sessions_df.reset_index()\n",
"\n",
- "session_old_df['seq_sessions'] = sequential_sessions\n",
+ "session_old_df[\"seq_sessions\"] = sequential_sessions\n",
"session_old_df = session_old_df.merge(num_sessions_df)\n",
"\n",
"session_old_df"
@@ -1390,7 +1422,7 @@
}
],
"source": [
- "session_old_df.loc[session_old_df['subject_fullname'] == 'jounhong_CaMKII_tetO_7', :]"
+ "session_old_df.loc[session_old_df[\"subject_fullname\"] == \"jounhong_CaMKII_tetO_7\", :]"
]
},
{
@@ -1517,12 +1549,14 @@
}
],
"source": [
- "protocol_df = session_df.groupby('session_protocol').agg({'subject_fullname': [('num_sessions', 'count'), ('num_subjects', 'nunique')]})\n",
+ "protocol_df = session_df.groupby(\"session_protocol\").agg(\n",
+ " {\"subject_fullname\": [(\"num_sessions\", \"count\"), (\"num_subjects\", \"nunique\")]}\n",
+ ")\n",
"protocol_df.columns = protocol_df.columns.droplevel()\n",
- "#protocol_df = protocol_df.reset_index()\n",
- "protocol_df = protocol_df.sort_values(by='num_sessions', ascending=False)\n",
- "protocol_df.loc['Total']= protocol_df.sum(numeric_only=True, axis=0)\n",
- "#protocol_df = pd.concat([protocol_df, ])\n",
+ "# protocol_df = protocol_df.reset_index()\n",
+ "protocol_df = protocol_df.sort_values(by=\"num_sessions\", ascending=False)\n",
+ "protocol_df.loc[\"Total\"] = protocol_df.sum(numeric_only=True, axis=0)\n",
+ "# protocol_df = pd.concat([protocol_df, ])\n",
"protocol_df.head(45)"
]
},
@@ -1746,8 +1780,28 @@
"source": [
"block_query = \"main_level = level and level>=11\"\n",
"\n",
- "trials_df = pd.DataFrame((acquisition.Session.proj('session_location') * behavior.TowersBlock * behavior.TowersBlock.Trial() & block_query & session_keys).fetch('KEY', 'level', 'trial_type', 'choice', 'session_location', order_by='session_date', as_dict=True))\n",
- "all_sessions_df = pd.DataFrame((acquisition.Session & session_keys).fetch('KEY','session_location', order_by='session_date', as_dict=True))\n",
+ "trials_df = pd.DataFrame(\n",
+ " (\n",
+ " acquisition.Session.proj(\"session_location\")\n",
+ " * behavior.TowersBlock\n",
+ " * behavior.TowersBlock.Trial()\n",
+ " & block_query\n",
+ " & session_keys\n",
+ " ).fetch(\n",
+ " \"KEY\",\n",
+ " \"level\",\n",
+ " \"trial_type\",\n",
+ " \"choice\",\n",
+ " \"session_location\",\n",
+ " order_by=\"session_date\",\n",
+ " as_dict=True,\n",
+ " )\n",
+ ")\n",
+ "all_sessions_df = pd.DataFrame(\n",
+ " (acquisition.Session & session_keys).fetch(\n",
+ " \"KEY\", \"session_location\", order_by=\"session_date\", as_dict=True\n",
+ " )\n",
+ ")\n",
"trials_df"
]
},
@@ -2152,9 +2206,13 @@
}
],
"source": [
- "metrics_df = bm.BehaviorMetrics.get_bias_from_trial_df(trials_df, return_all_metrics=True)\n",
- "metrics_df['performance'] = metrics_df['cum_correct_trials']*100 / metrics_df['cum_trials']\n",
- "metrics_df['bias'] = metrics_df['bias'].abs()\n",
+ "metrics_df = bm.BehaviorMetrics.get_bias_from_trial_df(\n",
+ " trials_df, return_all_metrics=True\n",
+ ")\n",
+ "metrics_df[\"performance\"] = (\n",
+ " metrics_df[\"cum_correct_trials\"] * 100 / metrics_df[\"cum_trials\"]\n",
+ ")\n",
+ "metrics_df[\"bias\"] = metrics_df[\"bias\"].abs()\n",
"metrics_df = metrics_df.reset_index(drop=True)\n",
"metrics_df"
]
@@ -2172,7 +2230,7 @@
"metadata": {},
"outputs": [],
"source": [
- "len(list(set(metrics_df['subject_fullname'].tolist())))"
+ "len(list(set(metrics_df[\"subject_fullname\"].tolist())))"
]
},
{
@@ -2530,64 +2588,107 @@
"source": [
"performance_min = 70\n",
"perf_str = str(performance_min)\n",
- "metrics_df_f = metrics_df.loc[(metrics_df['performance'] >= performance_min) & (metrics_df['cum_trials'] > 100), :]\n",
+ "metrics_df_f = metrics_df.loc[\n",
+ " (metrics_df[\"performance\"] >= performance_min) & (metrics_df[\"cum_trials\"] > 100), :\n",
+ "]\n",
"\n",
- "#Group all sessions by rig\n",
- "summary_alls_df = all_sessions_df.groupby('session_location').agg({'subject_fullname': [('num_subjects', 'nunique'), ('num_sessions', 'count')]})\n",
+ "# Group all sessions by rig\n",
+ "summary_alls_df = all_sessions_df.groupby(\"session_location\").agg(\n",
+ " {\"subject_fullname\": [(\"num_subjects\", \"nunique\"), (\"num_sessions\", \"count\")]}\n",
+ ")\n",
"summary_alls_df.columns = summary_alls_df.columns.droplevel()\n",
"summary_alls_df = summary_alls_df.reset_index()\n",
"\n",
- "#Group l11 sessions by rig\n",
- "summary_l11_df = metrics_df.groupby('session_location').agg({'subject_fullname': [('num_subjects_l11', 'nunique'), ('num_sessions_l11', 'count')], 'performance': [('performance_l11', 'mean')],\\\n",
- " 'bias': [('bias_l11', 'mean')]})\n",
+ "# Group l11 sessions by rig\n",
+ "summary_l11_df = metrics_df.groupby(\"session_location\").agg(\n",
+ " {\n",
+ " \"subject_fullname\": [\n",
+ " (\"num_subjects_l11\", \"nunique\"),\n",
+ " (\"num_sessions_l11\", \"count\"),\n",
+ " ],\n",
+ " \"performance\": [(\"performance_l11\", \"mean\")],\n",
+ " \"bias\": [(\"bias_l11\", \"mean\")],\n",
+ " }\n",
+ ")\n",
"summary_l11_df.columns = summary_l11_df.columns.droplevel()\n",
"summary_l11_df = summary_l11_df.reset_index()\n",
"\n",
- "#Group l11 70 % sessions by rig\n",
- "summary_l1170_df = metrics_df_f.groupby('session_location').agg({'subject_fullname': [('num_subjects_l11>'+perf_str, 'nunique'), ('num_sessions_l11>'+perf_str, 'count')], 'performance': [('performance_l11>'+perf_str, 'mean')],\\\n",
- " 'bias': [('bias_l11>'+perf_str, 'mean')]})\n",
+ "# Group l11 70 % sessions by rig\n",
+ "summary_l1170_df = metrics_df_f.groupby(\"session_location\").agg(\n",
+ " {\n",
+ " \"subject_fullname\": [\n",
+ " (\"num_subjects_l11>\" + perf_str, \"nunique\"),\n",
+ " (\"num_sessions_l11>\" + perf_str, \"count\"),\n",
+ " ],\n",
+ " \"performance\": [(\"performance_l11>\" + perf_str, \"mean\")],\n",
+ " \"bias\": [(\"bias_l11>\" + perf_str, \"mean\")],\n",
+ " }\n",
+ ")\n",
"summary_l1170_df.columns = summary_l1170_df.columns.droplevel()\n",
"summary_l1170_df = summary_l1170_df.reset_index()\n",
"\n",
"# Merge group dataframes\n",
- "summary_l11_df = summary_alls_df.merge(summary_l11_df, on='session_location', how='left')\n",
- "summary_df = summary_l11_df.merge(summary_l1170_df, on='session_location', how='left')\n",
+ "summary_l11_df = summary_alls_df.merge(\n",
+ " summary_l11_df, on=\"session_location\", how=\"left\"\n",
+ ")\n",
+ "summary_df = summary_l11_df.merge(summary_l1170_df, on=\"session_location\", how=\"left\")\n",
"\n",
"\n",
"# Fillna, new metrics\n",
- "summary_df['num_subjects_l11'] = summary_df['num_subjects_l11'].fillna(0)\n",
- "summary_df['num_sessions_l11'] = summary_df['num_sessions_l11'].fillna(0)\n",
- "summary_df['num_sessions_l11>'+perf_str] = summary_df['num_sessions_l11>'+perf_str].fillna(0)\n",
- "summary_df['num_subjects_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str].fillna(0)\n",
+ "summary_df[\"num_subjects_l11\"] = summary_df[\"num_subjects_l11\"].fillna(0)\n",
+ "summary_df[\"num_sessions_l11\"] = summary_df[\"num_sessions_l11\"].fillna(0)\n",
+ "summary_df[\"num_sessions_l11>\" + perf_str] = summary_df[\n",
+ " \"num_sessions_l11>\" + perf_str\n",
+ "].fillna(0)\n",
+ "summary_df[\"num_subjects_l11>\" + perf_str] = summary_df[\n",
+ " \"num_subjects_l11>\" + perf_str\n",
+ "].fillna(0)\n",
"\n",
"\n",
"# Calculate Total rows\n",
- "summary_df.loc['Total']= summary_df.sum(numeric_only=True, axis=0)\n",
- "mean_columns = ['performance_l11', 'bias_l11', 'performance_l11>'+perf_str, 'bias_l11>'+perf_str]\n",
- "summary_df.loc['Total', mean_columns]= summary_df.loc[summary_df.iloc[range(summary_df.shape[0]-1)].index, mean_columns].mean(numeric_only=True, axis=0)\n",
+ "summary_df.loc[\"Total\"] = summary_df.sum(numeric_only=True, axis=0)\n",
+ "mean_columns = [\n",
+ " \"performance_l11\",\n",
+ " \"bias_l11\",\n",
+ " \"performance_l11>\" + perf_str,\n",
+ " \"bias_l11>\" + perf_str,\n",
+ "]\n",
+ "summary_df.loc[\"Total\", mean_columns] = summary_df.loc[\n",
+ " summary_df.iloc[range(summary_df.shape[0] - 1)].index, mean_columns\n",
+ "].mean(numeric_only=True, axis=0)\n",
"\n",
"# Ratio sessions / session in level & performance\n",
- "summary_df['ratio_l11'] = summary_df['num_subjects_l11']*100 /summary_df['num_subjects']\n",
- "summary_df['ratio_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str]*100 /summary_df['num_subjects']\n",
+ "summary_df[\"ratio_l11\"] = (\n",
+ " summary_df[\"num_subjects_l11\"] * 100 / summary_df[\"num_subjects\"]\n",
+ ")\n",
+ "summary_df[\"ratio_l11>\" + perf_str] = (\n",
+ " summary_df[\"num_subjects_l11>\" + perf_str] * 100 / summary_df[\"num_subjects\"]\n",
+ ")\n",
"\n",
"# Make int columns pretty\n",
- "summary_df['num_subjects'] = summary_df['num_subjects'].astype('Int64')\n",
- "summary_df['num_sessions'] = summary_df['num_sessions'].astype('Int64')\n",
- "summary_df['num_sessions_l11'] = summary_df['num_sessions_l11'].astype('Int64')\n",
- "summary_df['num_subjects_l11'] = summary_df['num_subjects_l11'].astype('Int64')\n",
- "summary_df['num_sessions_l11>'+perf_str] = summary_df['num_sessions_l11>'+perf_str].astype('Int64')\n",
- "summary_df['num_subjects_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str].astype('Int64')\n",
+ "summary_df[\"num_subjects\"] = summary_df[\"num_subjects\"].astype(\"Int64\")\n",
+ "summary_df[\"num_sessions\"] = summary_df[\"num_sessions\"].astype(\"Int64\")\n",
+ "summary_df[\"num_sessions_l11\"] = summary_df[\"num_sessions_l11\"].astype(\"Int64\")\n",
+ "summary_df[\"num_subjects_l11\"] = summary_df[\"num_subjects_l11\"].astype(\"Int64\")\n",
+ "summary_df[\"num_sessions_l11>\" + perf_str] = summary_df[\n",
+ " \"num_sessions_l11>\" + perf_str\n",
+ "].astype(\"Int64\")\n",
+ "summary_df[\"num_subjects_l11>\" + perf_str] = summary_df[\n",
+ " \"num_subjects_l11>\" + perf_str\n",
+ "].astype(\"Int64\")\n",
"\n",
"# Sort properly\n",
- "summary_df = summary_df.sort_values(by=['ratio_l11>'+perf_str, 'num_sessions_l11>'+perf_str], ascending=False)\n",
- "summary_df = summary_df.set_index('session_location')\n",
+ "summary_df = summary_df.sort_values(\n",
+ " by=[\"ratio_l11>\" + perf_str, \"num_sessions_l11>\" + perf_str], ascending=False\n",
+ ")\n",
+ "summary_df = summary_df.set_index(\"session_location\")\n",
"\n",
"# Total row at the end\n",
- "summary_df.index = summary_df.index.fillna('Total')\n",
+ "summary_df.index = summary_df.index.fillna(\"Total\")\n",
"idx = summary_df.index\n",
- "idx = idx.drop('Total')\n",
+ "idx = idx.drop(\"Total\")\n",
"idx = idx.to_list()\n",
- "summary_df = summary_df.reindex(idx + ['Total'])\n",
+ "summary_df = summary_df.reindex(idx + [\"Total\"])\n",
"summary_df"
]
},
@@ -2608,7 +2709,7 @@
}
],
"source": [
- "96/168"
+ "96 / 168"
]
},
{
@@ -3032,7 +3133,7 @@
}
],
"source": [
- "metrics_df_f = metrics_df_f.sort_values(by='performance', ascending=False)\n",
+ "metrics_df_f = metrics_df_f.sort_values(by=\"performance\", ascending=False)\n",
"metrics_df_f"
]
},
@@ -3147,8 +3248,8 @@
],
"source": [
"query_s = dict()\n",
- "query_s['subject_fullname'] = 'jounhong_CaMKII_tetO_7'\n",
- "query_s['session_date'] = '2021-07-22'\n",
+ "query_s[\"subject_fullname\"] = \"jounhong_CaMKII_tetO_7\"\n",
+ "query_s[\"session_date\"] = \"2021-07-22\"\n",
"a = pd.DataFrame((acquisition.Session & query_s).fetch(as_dict=True))\n",
"a"
]
diff --git a/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb b/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb
index 0130f596..31982bec 100644
--- a/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb
+++ b/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb
@@ -26,6 +26,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"import datajoint as dj"
]
@@ -502,12 +503,14 @@
"metadata": {},
"outputs": [],
"source": [
- "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n",
- "task = dj.create_virtual_module('task', 'u19_task') \n",
- "subject = dj.create_virtual_module('subject', 'u19_subject')\n",
- "action = dj.create_virtual_module('action', 'u19_action')\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "lab = dj.create_virtual_module(\n",
+ " \"lab\", \"u19_lab\"\n",
+ ") # the first argument here is the __name__ of the virtual module\n",
+ "task = dj.create_virtual_module(\"task\", \"u19_task\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n",
+ "action = dj.create_virtual_module(\"action\", \"u19_action\")\n",
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -1216,7 +1219,12 @@
}
],
"source": [
- "dj.Diagram(subject.Subject) + dj.Diagram(subject.Death) + dj.Diagram(subject.HealthStatus) + dj.Diagram(subject.Weaning)"
+ "(\n",
+ " dj.Diagram(subject.Subject)\n",
+ " + dj.Diagram(subject.Death)\n",
+ " + dj.Diagram(subject.HealthStatus)\n",
+ " + dj.Diagram(subject.Weaning)\n",
+ ")"
]
},
{
@@ -2118,7 +2126,7 @@
],
"source": [
"# restrict by dictionary\n",
- "subject.Subject & {'subject_nickname': 'B205'}"
+ "subject.Subject & {\"subject_nickname\": \"B205\"}"
]
},
{
@@ -2412,7 +2420,7 @@
}
],
"source": [
- "subject.Subject & {'sex': 'Male'}"
+ "subject.Subject & {\"sex\": \"Male\"}"
]
},
{
@@ -2713,7 +2721,7 @@
}
],
"source": [
- "subject.Subject & [{'user_id': 'hnieh'}, {'user_id': 'emanuele'}, 'line=\"DAT-IRES-CRE\"']"
+ "subject.Subject & [{\"user_id\": \"hnieh\"}, {\"user_id\": \"emanuele\"}, 'line=\"DAT-IRES-CRE\"']"
]
},
{
@@ -6324,7 +6332,7 @@
}
],
"source": [
- "(subject.Subject * acquisition.Session).proj('dob', 'sex', 'location')"
+ "(subject.Subject * acquisition.Session).proj(\"dob\", \"sex\", \"location\")"
]
},
{
@@ -6460,7 +6468,7 @@
}
],
"source": [
- "subject.Subject.proj(gender='sex', birth_date='dob') # 'sex->gender'"
+ "subject.Subject.proj(gender=\"sex\", birth_date=\"dob\") # 'sex->gender'"
]
},
{
@@ -6483,7 +6491,9 @@
"metadata": {},
"outputs": [],
"source": [
- "weighing_with_date = action.Weighing.proj(weighing_date='date(weighing_time)') # more options, check MySQL syntax"
+ "weighing_with_date = action.Weighing.proj(\n",
+ " weighing_date=\"date(weighing_time)\"\n",
+ ") # more options, check MySQL syntax"
]
},
{
@@ -6765,7 +6775,7 @@
"source": [
"# First get the date of birth and the session date into the same query\n",
"q = subject.Subject * acquisition.Session\n",
- "q = q.proj('dob')\n",
+ "q = q.proj(\"dob\")\n",
"q"
]
},
@@ -6926,7 +6936,7 @@
],
"source": [
"# Then compute the age\n",
- "q_with_age = q.proj('dob', age='datediff(session_date, dob)') & 'dob is not NULL'\n",
+ "q_with_age = q.proj(\"dob\", age=\"datediff(session_date, dob)\") & \"dob is not NULL\"\n",
"q_with_age"
]
},
@@ -7070,7 +7080,9 @@
}
],
"source": [
- "subject.Subject.aggr(acquisition.Session, n='count(*)', lastest_session_date='max(session_date)')"
+ "subject.Subject.aggr(\n",
+ " acquisition.Session, n=\"count(*)\", lastest_session_date=\"max(session_date)\"\n",
+ ")"
]
},
{
@@ -25971,7 +25983,7 @@
}
],
"source": [
- "subjs['subject_fullname']"
+ "subjs[\"subject_fullname\"]"
]
},
{
@@ -26202,7 +26214,7 @@
}
],
"source": [
- "subjs['dob']"
+ "subjs[\"dob\"]"
]
},
{
@@ -52432,7 +52444,7 @@
],
"source": [
"# fetch as pandas dataframe\n",
- "subjs_df = subject.Subject.fetch(format='frame').reset_index()\n",
+ "subjs_df = subject.Subject.fetch(format=\"frame\").reset_index()\n",
"subjs_df"
]
},
@@ -53011,7 +53023,7 @@
],
"source": [
"# fetch the primary key\n",
- "pk = subject.Subject.fetch('KEY')\n",
+ "pk = subject.Subject.fetch(\"KEY\")\n",
"pk"
]
},
@@ -53022,7 +53034,7 @@
"outputs": [],
"source": [
"# fetch specific attributes\n",
- "dob, sex = subject.Subject.fetch('dob', 'sex')"
+ "dob, sex = subject.Subject.fetch(\"dob\", \"sex\")"
]
},
{
@@ -53831,7 +53843,7 @@
],
"source": [
"# fetch specific attributes as a list of dictionary\n",
- "info = subject.Subject.fetch('dob', 'sex', as_dict=True)\n",
+ "info = subject.Subject.fetch(\"dob\", \"sex\", as_dict=True)\n",
"info"
]
},
@@ -53848,7 +53860,9 @@
"metadata": {},
"outputs": [],
"source": [
- "B205 = (subject.Subject & {'subject_nickname': 'B205'}).fetch1() # \"fetch1()\" because we know there's only one"
+ "B205 = (\n",
+ " subject.Subject & {\"subject_nickname\": \"B205\"}\n",
+ ").fetch1() # \"fetch1()\" because we know there's only one"
]
},
{
@@ -53938,7 +53952,7 @@
"metadata": {},
"outputs": [],
"source": [
- "B205_key = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('KEY')"
+ "B205_key = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\"KEY\")"
]
},
{
@@ -53967,7 +53981,9 @@
"metadata": {},
"outputs": [],
"source": [
- "B205_init_weight = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('initial_weight')"
+ "B205_init_weight = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\n",
+ " \"initial_weight\"\n",
+ ")"
]
},
{
@@ -54007,7 +54023,7 @@
}
],
"source": [
- "(subject.Subject & {'subject_nickname': 'B205'}).fetch('initial_weight')"
+ "(subject.Subject & {\"subject_nickname\": \"B205\"}).fetch(\"initial_weight\")"
]
},
{
diff --git a/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb b/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb
index 8a1f7a73..06a0de3d 100644
--- a/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb
+++ b/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb
@@ -26,18 +26,21 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
+ "import datetime\n",
+ "\n",
"import datajoint as dj\n",
"import numpy as np\n",
- "import matplotlib.pyplot as plt\n",
- "import pandas as pd\n",
- "import datetime\n",
- "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n",
- "task = dj.create_virtual_module('task', 'u19_task') \n",
- "subject = dj.create_virtual_module('subject', 'u19_subject')\n",
- "action = dj.create_virtual_module('action', 'u19_action')\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "\n",
+ "lab = dj.create_virtual_module(\n",
+ " \"lab\", \"u19_lab\"\n",
+ ") # the first argument here is the __name__ of the virtual module\n",
+ "task = dj.create_virtual_module(\"task\", \"u19_task\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n",
+ "action = dj.create_virtual_module(\"action\", \"u19_action\")\n",
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -691,7 +694,7 @@
}
],
"source": [
- "behavior.TowersBlock() # bracket necessary"
+ "behavior.TowersBlock() # bracket necessary"
]
},
{
@@ -842,8 +845,10 @@
}
],
"source": [
- "behavior.TowersSession.aggr(behavior.TowersBlock.proj('block_performance'), \n",
- " avg_performance='avg(block_performance)')"
+ "behavior.TowersSession.aggr(\n",
+ " behavior.TowersBlock.proj(\"block_performance\"),\n",
+ " avg_performance=\"avg(block_performance)\",\n",
+ ")"
]
},
{
@@ -874,7 +879,7 @@
"metadata": {},
"outputs": [],
"source": [
- "schema = dj.schema('alvaros_tutorial2021')"
+ "schema = dj.schema(\"alvaros_tutorial2021\")"
]
},
{
@@ -1288,8 +1293,7 @@
}
],
"source": [
- "behavior.TowersSession.aggr(behavior.TowersBlock.proj(),\n",
- " n_sessions='count(*)')"
+ "behavior.TowersSession.aggr(behavior.TowersBlock.proj(), n_sessions=\"count(*)\")"
]
},
{
@@ -1306,9 +1310,9 @@
"outputs": [],
"source": [
"key = {\n",
- " 'subject_fullname': 'emanuele_B205',\n",
- " 'session_date': datetime.date(2018, 7, 13),\n",
- " 'session_number': 0\n",
+ " \"subject_fullname\": \"emanuele_B205\",\n",
+ " \"session_date\": datetime.date(2018, 7, 13),\n",
+ " \"session_number\": 0,\n",
"}"
]
},
@@ -1561,10 +1565,10 @@
"metadata": {},
"outputs": [],
"source": [
- "performances = (behavior.TowersBlock & key).fetch('block_performance')\n",
+ "performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n",
"\n",
"# create another field in the dictionary key\n",
- "key['avg_performance'] = np.mean(performances)"
+ "key[\"avg_performance\"] = np.mean(performances)"
]
},
{
@@ -1603,7 +1607,9 @@
"metadata": {},
"outputs": [],
"source": [
- "SessionPerformanceManual.insert1(key, skip_duplicates=True) # insert1 only works for one entry"
+ "SessionPerformanceManual.insert1(\n",
+ " key, skip_duplicates=True\n",
+ ") # insert1 only works for one entry"
]
},
{
@@ -1820,7 +1826,9 @@
}
],
"source": [
- "subject.Subject.aggr(behavior.TowersSession.proj(), n_sessions='count(*)') & 'n_sessions=100'"
+ "subject.Subject.aggr(\n",
+ " behavior.TowersSession.proj(), n_sessions=\"count(*)\"\n",
+ ") & \"n_sessions=100\""
]
},
{
@@ -1846,16 +1854,16 @@
"source": [
"# loop through sessions of subject B205 and insert one by one, and compute time\n",
"import time\n",
+ "\n",
"start_time = time.time()\n",
"\n",
- "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E57\"').fetch('KEY'):\n",
- " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n",
+ "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E57\"').fetch(\"KEY\"):\n",
+ " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n",
" # create another field in the dictionary key\n",
" avg_performance = np.mean(performances)\n",
" if np.isnan(avg_performance):\n",
" continue\n",
- " entry = dict(**i_session, \n",
- " avg_performance=avg_performance)\n",
+ " entry = dict(**i_session, avg_performance=avg_performance)\n",
" SessionPerformanceManual.insert1(entry)\n",
"\n",
"print(\"--- %s seconds ---\" % (time.time() - start_time))"
@@ -1879,16 +1887,15 @@
"start_time = time.time()\n",
"\n",
"perf_entries = []\n",
- "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E77\"').fetch('KEY'):\n",
- " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n",
+ "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E77\"').fetch(\"KEY\"):\n",
+ " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n",
" # create another field in the dictionary key\n",
" avg_performance = np.mean(performances)\n",
" if np.isnan(avg_performance):\n",
" continue\n",
- " entry = dict(**i_session, \n",
- " avg_performance=avg_performance)\n",
+ " entry = dict(**i_session, avg_performance=avg_performance)\n",
" perf_entries.append(entry)\n",
- " \n",
+ "\n",
"SessionPerformanceManual.insert(perf_entries)\n",
"print(\"--- %s seconds ---\" % (time.time() - start_time))"
]
@@ -1914,14 +1921,19 @@
" ---\n",
" avg_performance: float # a final product in this table\n",
" \"\"\"\n",
- " \n",
- " key_source = acquisition.Session() & 'subject_fullname=\"hnieh_E57\"' # bracket necessary\n",
- " def make(self, key): # key is one primary key of the entries in table acquisition.Session\n",
+ "\n",
+ " key_source = (\n",
+ " acquisition.Session() & 'subject_fullname=\"hnieh_E57\"'\n",
+ " ) # bracket necessary\n",
+ "\n",
+ " def make(\n",
+ " self, key\n",
+ " ): # key is one primary key of the entries in table acquisition.Session\n",
" # fetch the performance for each block\n",
- " performances = (behavior.TowersBlock & key).fetch('block_performance')\n",
- " \n",
+ " performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n",
+ "\n",
" # create another field in the dictionary key\n",
- " key['avg_performance'] = np.mean(performances)\n",
+ " key[\"avg_performance\"] = np.mean(performances)\n",
" self.insert1(key)"
]
},
@@ -1946,7 +1958,7 @@
}
],
"source": [
- "SessionPerformanceComputed.populate(display_progress=True) \n",
+ "SessionPerformanceComputed.populate(display_progress=True)\n",
"# first argument could be some restrictor to control the populate"
]
},
@@ -2132,21 +2144,23 @@
" ---\n",
" avg_performance: float # a final product in this table\n",
" \"\"\"\n",
- " key_source = subject.Subject() \n",
- " def make(self, key): # key is one primary key of the entries in table subject.Subject()!\n",
+ " key_source = subject.Subject()\n",
+ "\n",
+ " def make(\n",
+ " self, key\n",
+ " ): # key is one primary key of the entries in table subject.Subject()!\n",
"\n",
" perf_entries = []\n",
- " for i_session in (behavior.TowersSession & key).fetch('KEY'):\n",
+ " for i_session in (behavior.TowersSession & key).fetch(\"KEY\"):\n",
" # fetch performance of each block\n",
- " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n",
+ " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n",
" # create another field in the dictionary key\n",
" avg_performance = np.mean(performances)\n",
" if np.isnan(avg_performance):\n",
" continue\n",
- " entry = dict(**i_session, \n",
- " avg_performance=avg_performance)\n",
+ " entry = dict(**i_session, avg_performance=avg_performance)\n",
" perf_entries.append(entry)\n",
- " \n",
+ "\n",
" self.insert(perf_entries)"
]
},
@@ -2174,7 +2188,12 @@
}
],
"source": [
- "SessionPerformanceComputedFromSubject.populate('subject_fullname=\"hnieh_E77\"', display_progress=True, suppress_errors=True, reserve_jobs=True)"
+ "SessionPerformanceComputedFromSubject.populate(\n",
+ " 'subject_fullname=\"hnieh_E77\"',\n",
+ " display_progress=True,\n",
+ " suppress_errors=True,\n",
+ " reserve_jobs=True,\n",
+ ")"
]
},
{
@@ -2209,7 +2228,9 @@
}
],
"source": [
- "(SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"').delete() # any restrictor would work here"
+ "(\n",
+ " SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"'\n",
+ ").delete() # any restrictor would work here"
]
},
{
diff --git a/notebooks/tutorials/3-Build a simple data pipeline.ipynb b/notebooks/tutorials/3-Build a simple data pipeline.ipynb
index ef2b5c2a..a3eabe31 100644
--- a/notebooks/tutorials/3-Build a simple data pipeline.ipynb
+++ b/notebooks/tutorials/3-Build a simple data pipeline.ipynb
@@ -42,6 +42,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"import datajoint as dj"
]
@@ -234,7 +235,7 @@
],
"source": [
"# create your own schema object\n",
- "schema = dj.schema('alvaros_tutorial2021')"
+ "schema = dj.schema(\"alvaros_tutorial2021\")"
]
},
{
@@ -399,7 +400,7 @@
"outputs": [],
"source": [
"# inserting the first mouse\n",
- "Mouse.insert1((0, '2017-03-01', 'M'))"
+ "Mouse.insert1((0, \"2017-03-01\", \"M\"))"
]
},
{
@@ -512,7 +513,7 @@
"metadata": {},
"outputs": [],
"source": [
- "Mouse.insert1((20, datetime.date(2020, 5, 3), 'F'))"
+ "Mouse.insert1((20, datetime.date(2020, 5, 3), \"F\"))"
]
},
{
@@ -626,11 +627,7 @@
"metadata": {},
"outputs": [],
"source": [
- "data = {\n",
- " 'mouse_id': 100,\n",
- " 'dob': '2017-05-12',\n",
- " 'sex': 'F'\n",
- "}"
+ "data = {\"mouse_id\": 100, \"dob\": \"2017-05-12\", \"sex\": \"F\"}"
]
},
{
@@ -757,11 +754,7 @@
"metadata": {},
"outputs": [],
"source": [
- "data = [\n",
- " (1, '2016-11-19', 'M'),\n",
- " (2, '2016-11-20', 'unknown'),\n",
- " (5, '2016-12-25', 'F')\n",
- "]"
+ "data = [(1, \"2016-11-19\", \"M\"), (2, \"2016-11-20\", \"unknown\"), (5, \"2016-12-25\", \"F\")]"
]
},
{
@@ -898,8 +891,8 @@
"outputs": [],
"source": [
"data = [\n",
- " {'mouse_id': 10, 'dob': '2017-01-01', 'sex': 'F'},\n",
- " {'mouse_id': 11, 'dob': '2017-01-03', 'sex': 'F'},\n",
+ " {\"mouse_id\": 10, \"dob\": \"2017-01-01\", \"sex\": \"F\"},\n",
+ " {\"mouse_id\": 11, \"dob\": \"2017-01-03\", \"sex\": \"F\"},\n",
"]\n",
"\n",
"# insert them all\n",
@@ -1057,9 +1050,10 @@
],
"source": [
"Mouse.insert1(\n",
- " {'mouse_id': 0,\n",
- " 'dob': '2018-01-01',\n",
- " 'sex': 'M',\n",
+ " {\n",
+ " \"mouse_id\": 0,\n",
+ " \"dob\": \"2018-01-01\",\n",
+ " \"sex\": \"M\",\n",
" }\n",
")"
]
@@ -1106,7 +1100,7 @@
}
],
"source": [
- "(Mouse & 'mouse_id=11').delete()"
+ "(Mouse & \"mouse_id=11\").delete()"
]
},
{
@@ -1240,7 +1234,7 @@
}
],
"source": [
- "(Mouse & 'mouse_id=0')._update('sex', 'M')"
+ "(Mouse & \"mouse_id=0\")._update(\"sex\", \"M\")"
]
},
{
@@ -1532,10 +1526,10 @@
"outputs": [],
"source": [
"data = {\n",
- " 'mouse_id': 0,\n",
- " 'session_date': '2017-05-15',\n",
- " 'experiment_setup': 0,\n",
- " 'experimenter': 'Edgar Y. Walker'\n",
+ " \"mouse_id\": 0,\n",
+ " \"session_date\": \"2017-05-15\",\n",
+ " \"experiment_setup\": 0,\n",
+ " \"experimenter\": \"Edgar Y. Walker\",\n",
"}\n",
"\n",
"Session.insert1(data)"
@@ -1748,10 +1742,10 @@
],
"source": [
"data = {\n",
- " 'mouse_id': 0,\n",
- " 'session_date': '2018-01-15',\n",
- " 'experiment_setup': 100,\n",
- " 'experimenter': 'Jacob Reimer'\n",
+ " \"mouse_id\": 0,\n",
+ " \"session_date\": \"2018-01-15\",\n",
+ " \"experiment_setup\": 100,\n",
+ " \"experimenter\": \"Jacob Reimer\",\n",
"}\n",
"\n",
"# insert them all\n",
@@ -1774,10 +1768,10 @@
"outputs": [],
"source": [
"data = {\n",
- " 'mouse_id': 1,\n",
- " 'session_date': '2018-01-15',\n",
- " 'experiment_setup': 101,\n",
- " 'experimenter': 'Jacob Reimer'\n",
+ " \"mouse_id\": 1,\n",
+ " \"session_date\": \"2018-01-15\",\n",
+ " \"experiment_setup\": 101,\n",
+ " \"experimenter\": \"Jacob Reimer\",\n",
"}\n",
"\n",
"# insert them all\n",
@@ -1905,10 +1899,10 @@
"outputs": [],
"source": [
"bad_data = {\n",
- " 'mouse_id': 9999, # this mouse doesn't exist!\n",
- " 'session_date': '2017-05-15',\n",
- " 'experiment_setup': 0,\n",
- " 'experimenter': 'Edgar Y. Walker'\n",
+ " \"mouse_id\": 9999, # this mouse doesn't exist!\n",
+ " \"session_date\": \"2017-05-15\",\n",
+ " \"experiment_setup\": 0,\n",
+ " \"experimenter\": \"Edgar Y. Walker\",\n",
"}"
]
},
@@ -1986,7 +1980,7 @@
}
],
"source": [
- "(Mouse & 'mouse_id = 100').delete()"
+ "(Mouse & \"mouse_id = 100\").delete()"
]
},
{
diff --git a/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb b/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb
index 6af32a1d..165441f5 100644
--- a/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb
+++ b/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb
@@ -18,9 +18,10 @@
"metadata": {},
"outputs": [],
"source": [
- "import pylab as pl\n",
"import datajoint as dj\n",
"import numpy as np\n",
+ "import pylab as pl\n",
+ "\n",
"dj.__version__"
]
},
@@ -37,7 +38,7 @@
"metadata": {},
"outputs": [],
"source": [
- "dj.config['database.port'] = 6306"
+ "dj.config[\"database.port\"] = 6306"
]
},
{
@@ -46,7 +47,7 @@
"metadata": {},
"outputs": [],
"source": [
- "#Overview across database schemas\n",
+ "# Overview across database schemas\n",
"dj.list_schemas()"
]
},
@@ -56,10 +57,10 @@
"metadata": {},
"outputs": [],
"source": [
- "#Single entity relationship diagram. This is how the data is organized\n",
- "acqu = dj.schema('pni_acquisition')\n",
- "subj = dj.schema('pni_subject')\n",
- "lab = dj.schema('pni_lab')\n",
+ "# Single entity relationship diagram. This is how the data is organized\n",
+ "acqu = dj.schema(\"pni_acquisition\")\n",
+ "subj = dj.schema(\"pni_subject\")\n",
+ "lab = dj.schema(\"pni_lab\")\n",
"\n",
"dj.ERD(subj)"
]
@@ -77,11 +78,11 @@
"metadata": {},
"outputs": [],
"source": [
- "acquisition = dj.create_virtual_module('acquisition', 'pni_acquisition')\n",
- "subject = dj.create_virtual_module('subject', 'pni_subject')\n",
- "action = dj.create_virtual_module('action', 'pni_action')\n",
- "lab = dj.create_virtual_module('lab', 'pni_lab')\n",
- "task = dj.create_virtual_module('task', 'pni_task')"
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"pni_acquisition\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"pni_subject\")\n",
+ "action = dj.create_virtual_module(\"action\", \"pni_action\")\n",
+ "lab = dj.create_virtual_module(\"lab\", \"pni_lab\")\n",
+ "task = dj.create_virtual_module(\"task\", \"pni_task\")"
]
},
{
@@ -116,10 +117,7 @@
"outputs": [],
"source": [
"new_user = dict(\n",
- " user_id='testuser3',\n",
- " contact_via='Slack',\n",
- " presence='Away',\n",
- " day_cutoff_time=' '\n",
+ " user_id=\"testuser3\", contact_via=\"Slack\", presence=\"Away\", day_cutoff_time=\" \"\n",
")"
]
},
@@ -147,7 +145,7 @@
"metadata": {},
"outputs": [],
"source": [
- "dj.Table._update(lab.User & 'user_id=\"testuser3\"', 'email', 'bla@princeton.edu')"
+ "dj.Table._update(lab.User & 'user_id=\"testuser3\"', \"email\", \"bla@princeton.edu\")"
]
},
{
@@ -202,7 +200,7 @@
"outputs": [],
"source": [
"# Very fat mice\n",
- "subject.Subject() & 'initial_weight > 35'"
+ "subject.Subject() & \"initial_weight > 35\""
]
},
{
@@ -220,7 +218,9 @@
"metadata": {},
"outputs": [],
"source": [
- "q = (subject.Subject & 'subject_id=\"B205\"').proj('sex', 'dob', 'initial_weight', animal_location='location')"
+ "q = (subject.Subject & 'subject_id=\"B205\"').proj(\n",
+ " \"sex\", \"dob\", \"initial_weight\", animal_location=\"location\"\n",
+ ")"
]
},
{
@@ -239,9 +239,9 @@
"outputs": [],
"source": [
"# On which days of week does people like to work harder?\n",
- "Session = acquisition.Session();\n",
+ "Session = acquisition.Session()\n",
"for day in range(7):\n",
- " print(len(Session & 'WEEKDAY(session_date) = ' + str(day)))"
+ " print(len(Session & \"WEEKDAY(session_date) = \" + str(day)))"
]
},
{
@@ -278,7 +278,7 @@
"outputs": [],
"source": [
"# some magic to get weight on the date of each session\n",
- "weight_with_date = action.Weighing.proj('weight', session_date=\"DATE(weighing_time)\")"
+ "weight_with_date = action.Weighing.proj(\"weight\", session_date=\"DATE(weighing_time)\")"
]
},
{
@@ -306,7 +306,9 @@
"metadata": {},
"outputs": [],
"source": [
- "weight, performance = (query & 'subject_id = \"E47\"').fetch('weight','session_performance')"
+ "weight, performance = (query & 'subject_id = \"E47\"').fetch(\n",
+ " \"weight\", \"session_performance\"\n",
+ ")"
]
},
{
@@ -325,8 +327,8 @@
"outputs": [],
"source": [
"pl.scatter(weight, performance)\n",
- "pl.xlabel('bodyweight [g]')\n",
- "pl.ylabel('performance')"
+ "pl.xlabel(\"bodyweight [g]\")\n",
+ "pl.ylabel(\"performance\")"
]
},
{
@@ -342,23 +344,25 @@
"metadata": {},
"outputs": [],
"source": [
- "subjects = subject.Subject.proj('sex') & query & 'user_id = \"edward\"'\n",
+ "subjects = subject.Subject.proj(\"sex\") & query & 'user_id = \"edward\"'\n",
"corrs = []\n",
"for s in subjects:\n",
" # fetch all related info: weight, performance and sex\n",
- " sex, weight, performance = (subject.Subject.proj('sex') * query & s).fetch('sex', 'weight','session_performance')\n",
- " \n",
+ " sex, weight, performance = (subject.Subject.proj(\"sex\") * query & s).fetch(\n",
+ " \"sex\", \"weight\", \"session_performance\"\n",
+ " )\n",
+ "\n",
" weight = weight[performance > 0]\n",
" performance = performance[performance > 0]\n",
- " if sex[0] == 'Male':\n",
+ " if sex[0] == \"Male\":\n",
" pl.scatter(weight, performance, c=[[0, 0, 1]])\n",
- " else: \n",
+ " else:\n",
" pl.scatter(weight, performance, c=[[1, 0, 0]])\n",
" corrs.append(np.corrcoef(weight, performance)[0, 1])\n",
- " \n",
+ "\n",
"pl.xlim([19, 27])\n",
- "pl.xlabel('bodyweight [g]')\n",
- "pl.ylabel('performance')"
+ "pl.xlabel(\"bodyweight [g]\")\n",
+ "pl.ylabel(\"performance\")"
]
},
{
@@ -367,10 +371,10 @@
"metadata": {},
"outputs": [],
"source": [
- "subjects = (subject.Subject() & 'sex = \"Male\"' & 'user_id = \"edward\"' ).fetch('KEY')\n",
- "boymean = np.mean((Session & subjects).fetch('session_performance'))\n",
- "subjects = (subject.Subject() & 'sex = \"Female\"' & 'user_id = \"edward\"' ).fetch('KEY')\n",
- "girlmean = np.mean((Session & subjects).fetch('session_performance'))\n",
+ "subjects = (subject.Subject() & 'sex = \"Male\"' & 'user_id = \"edward\"').fetch(\"KEY\")\n",
+ "boymean = np.mean((Session & subjects).fetch(\"session_performance\"))\n",
+ "subjects = (subject.Subject() & 'sex = \"Female\"' & 'user_id = \"edward\"').fetch(\"KEY\")\n",
+ "girlmean = np.mean((Session & subjects).fetch(\"session_performance\"))\n",
"print(boymean)\n",
"print(girlmean)"
]
@@ -388,9 +392,9 @@
"metadata": {},
"outputs": [],
"source": [
- "pl.hist(corrs,np.arange(-1,1,0.1))\n",
+ "pl.hist(corrs, np.arange(-1, 1, 0.1))\n",
"print(np.mean(corrs))\n",
- "print(np.std(corrs)/np.sqrt(len(corrs)))"
+ "print(np.std(corrs) / np.sqrt(len(corrs)))"
]
},
{
@@ -417,7 +421,7 @@
"outputs": [],
"source": [
"trials_subset = acquisition.TowersBlock.Trial & sessions\n",
- "acquisition.Session.aggr(trials_subset, n='count(*)')"
+ "acquisition.Session.aggr(trials_subset, n=\"count(*)\")"
]
},
{
@@ -433,7 +437,7 @@
"metadata": {},
"outputs": [],
"source": [
- "pos, time = trials_subset.fetch('position', 'trial_time')"
+ "pos, time = trials_subset.fetch(\"position\", \"trial_time\")"
]
},
{
@@ -444,9 +448,9 @@
"source": [
"for trial_idx in range(len(pos)):\n",
" N_samples = len(pos[trial_idx])\n",
- " pl.plot(time[trial_idx][0:N_samples], pos[trial_idx][:,1])\n",
- "pl.xlabel('Time [s]');\n",
- "pl.ylabel('position [cm]');"
+ " pl.plot(time[trial_idx][0:N_samples], pos[trial_idx][:, 1])\n",
+ "pl.xlabel(\"Time [s]\")\n",
+ "pl.ylabel(\"position [cm]\");"
]
},
{
diff --git a/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb b/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb
index 07311511..5dccf245 100644
--- a/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb
+++ b/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb
@@ -152,7 +152,7 @@
"metadata": {},
"outputs": [],
"source": [
- "dj.config['database.host'] = 'datajoint00.pni.princeton.edu'"
+ "dj.config[\"database.host\"] = \"datajoint00.pni.princeton.edu\""
]
},
{
@@ -296,7 +296,7 @@
" \"display.show_tuple_count\": true,\n",
" \"database.use_tls\": null,\n",
" \"enable_python_native_blobs\": false,\n",
- " \"database.prefix\": null\n",
+ " \"database.prefix\": null,\n",
"}"
]
},
@@ -327,7 +327,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from getpass import getpass # use this to type password in without showing it"
+ "from getpass import getpass # use this to type password in without showing it"
]
},
{
@@ -344,8 +344,8 @@
}
],
"source": [
- "dj.config['database.user'] = 'shans'\n",
- "dj.config['database.password'] = getpass('Type password:')"
+ "dj.config[\"database.user\"] = \"shans\"\n",
+ "dj.config[\"database.password\"] = getpass(\"Type password:\")"
]
},
{
diff --git a/notebooks/tutorials/Old_tutorials/202001/1-Build a simple data pipeline.ipynb b/notebooks/tutorials/Old_tutorials/202001/1-Build a simple data pipeline.ipynb
index ce7bc7b3..a16fb405 100644
--- a/notebooks/tutorials/Old_tutorials/202001/1-Build a simple data pipeline.ipynb
+++ b/notebooks/tutorials/Old_tutorials/202001/1-Build a simple data pipeline.ipynb
@@ -216,7 +216,7 @@
"outputs": [],
"source": [
"# create your own schema object\n",
- "schema = dj.schema('shans_tutorial')"
+ "schema = dj.schema(\"shans_tutorial\")"
]
},
{
@@ -399,7 +399,7 @@
"outputs": [],
"source": [
"# inserting the first mouse\n",
- "Mouse.insert1((0, '2017-03-01', 'M'))"
+ "Mouse.insert1((0, \"2017-03-01\", \"M\"))"
]
},
{
@@ -511,11 +511,7 @@
"metadata": {},
"outputs": [],
"source": [
- "data = {\n",
- " 'mouse_id': 100,\n",
- " 'dob': '2017-05-12',\n",
- " 'sex': 'F'\n",
- "}"
+ "data = {\"mouse_id\": 100, \"dob\": \"2017-05-12\", \"sex\": \"F\"}"
]
},
{
@@ -640,11 +636,7 @@
"metadata": {},
"outputs": [],
"source": [
- "data = [\n",
- " (1, '2016-11-19', 'M'),\n",
- " (2, '2016-11-20', 'unknown'),\n",
- " (5, '2016-12-25', 'F')\n",
- "]"
+ "data = [(1, \"2016-11-19\", \"M\"), (2, \"2016-11-20\", \"unknown\"), (5, \"2016-12-25\", \"F\")]"
]
},
{
@@ -779,8 +771,8 @@
"outputs": [],
"source": [
"data = [\n",
- " {'mouse_id': 10, 'dob': '2017-01-01', 'sex': 'F'},\n",
- " {'mouse_id': 11, 'dob': '2017-01-03', 'sex': 'F'},\n",
+ " {\"mouse_id\": 10, \"dob\": \"2017-01-01\", \"sex\": \"F\"},\n",
+ " {\"mouse_id\": 11, \"dob\": \"2017-01-03\", \"sex\": \"F\"},\n",
"]\n",
"\n",
"# insert them all\n",
@@ -922,10 +914,12 @@
"outputs": [],
"source": [
"Mouse.insert1(\n",
- "{'mouse_id': 0,\n",
- " 'dob': '2018-01-01',\n",
- " 'sex': 'M',\n",
- "})"
+ " {\n",
+ " \"mouse_id\": 0,\n",
+ " \"dob\": \"2018-01-01\",\n",
+ " \"sex\": \"M\",\n",
+ " }\n",
+ ")"
]
},
{
@@ -961,7 +955,7 @@
}
],
"source": [
- "(Mouse & 'mouse_id=11').delete()"
+ "(Mouse & \"mouse_id=11\").delete()"
]
},
{
@@ -1081,7 +1075,7 @@
"metadata": {},
"outputs": [],
"source": [
- "dj.Table._update(Mouse & 'mouse_id=0', 'sex', 'F')"
+ "dj.Table._update(Mouse & \"mouse_id=0\", \"sex\", \"F\")"
]
},
{
@@ -1153,7 +1147,7 @@
"metadata": {},
"outputs": [],
"source": [
- "schema2 = dj.schema('shans_tutorial2')"
+ "schema2 = dj.schema(\"shans_tutorial2\")"
]
},
{
@@ -1256,10 +1250,10 @@
"outputs": [],
"source": [
"data = {\n",
- " 'mouse_id': 0,\n",
- " 'session_date': '2017-05-15',\n",
- " 'experiment_setup': 0,\n",
- " 'experimenter': 'Edgar Y. Walker'\n",
+ " \"mouse_id\": 0,\n",
+ " \"session_date\": \"2017-05-15\",\n",
+ " \"experiment_setup\": 0,\n",
+ " \"experimenter\": \"Edgar Y. Walker\",\n",
"}\n",
"\n",
"Session.insert1(data)"
@@ -1474,10 +1468,10 @@
],
"source": [
"data = {\n",
- " 'mouse_id': 0,\n",
- " 'session_date': '2018-01-15',\n",
- " 'experiment_setup': 100,\n",
- " 'experimenter': 'Jacob Reimer'\n",
+ " \"mouse_id\": 0,\n",
+ " \"session_date\": \"2018-01-15\",\n",
+ " \"experiment_setup\": 100,\n",
+ " \"experimenter\": \"Jacob Reimer\",\n",
"}\n",
"\n",
"# insert them all\n",
@@ -1500,10 +1494,10 @@
"outputs": [],
"source": [
"data = {\n",
- " 'mouse_id': 1,\n",
- " 'session_date': '2018-01-15',\n",
- " 'experiment_setup': 101,\n",
- " 'experimenter': 'Jacob Reimer'\n",
+ " \"mouse_id\": 1,\n",
+ " \"session_date\": \"2018-01-15\",\n",
+ " \"experiment_setup\": 101,\n",
+ " \"experimenter\": \"Jacob Reimer\",\n",
"}\n",
"\n",
"# insert them all\n",
@@ -1632,10 +1626,10 @@
"outputs": [],
"source": [
"bad_data = {\n",
- " 'mouse_id': 9999, # this mouse doesn't exist!\n",
- " 'session_date': '2017-05-15',\n",
- " 'experiment_setup': 0,\n",
- " 'experimenter': 'Edgar Y. Walker'\n",
+ " \"mouse_id\": 9999, # this mouse doesn't exist!\n",
+ " \"session_date\": \"2017-05-15\",\n",
+ " \"experiment_setup\": 0,\n",
+ " \"experimenter\": \"Edgar Y. Walker\",\n",
"}"
]
},
@@ -1704,7 +1698,7 @@
}
],
"source": [
- "(Mouse & 'mouse_id = 100').delete()"
+ "(Mouse & \"mouse_id = 100\").delete()"
]
},
{
diff --git a/notebooks/tutorials/Old_tutorials/202001/2-Explore U19 data pipeline with DataJoint.ipynb b/notebooks/tutorials/Old_tutorials/202001/2-Explore U19 data pipeline with DataJoint.ipynb
index 1ef28255..db743c4e 100644
--- a/notebooks/tutorials/Old_tutorials/202001/2-Explore U19 data pipeline with DataJoint.ipynb
+++ b/notebooks/tutorials/Old_tutorials/202001/2-Explore U19 data pipeline with DataJoint.ipynb
@@ -123,12 +123,14 @@
"metadata": {},
"outputs": [],
"source": [
- "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n",
- "task = dj.create_virtual_module('task', 'u19_task') \n",
- "subject = dj.create_virtual_module('subject', 'u19_subject')\n",
- "action = dj.create_virtual_module('action', 'u19_action')\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "lab = dj.create_virtual_module(\n",
+ " \"lab\", \"u19_lab\"\n",
+ ") # the first argument here is the __name__ of the virtual module\n",
+ "task = dj.create_virtual_module(\"task\", \"u19_task\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n",
+ "action = dj.create_virtual_module(\"action\", \"u19_action\")\n",
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -1069,7 +1071,12 @@
}
],
"source": [
- "dj.Diagram(subject.Subject) + dj.Diagram(subject.Death) + dj.Diagram(subject.HealthStatus) + dj.Diagram(subject.Weaning)"
+ "(\n",
+ " dj.Diagram(subject.Subject)\n",
+ " + dj.Diagram(subject.Death)\n",
+ " + dj.Diagram(subject.HealthStatus)\n",
+ " + dj.Diagram(subject.Weaning)\n",
+ ")"
]
},
{
@@ -1861,7 +1868,7 @@
],
"source": [
"# restrict by dictionary\n",
- "subject.Subject & {'subject_nickname': 'B205'}"
+ "subject.Subject & {\"subject_nickname\": \"B205\"}"
]
},
{
@@ -2126,7 +2133,7 @@
}
],
"source": [
- "subject.Subject & {'sex': 'Male'}"
+ "subject.Subject & {\"sex\": \"Male\"}"
]
},
{
@@ -2398,7 +2405,7 @@
}
],
"source": [
- "subject.Subject & [{'user_id': 'hnieh'}, {'user_id': 'emanuele'}]"
+ "subject.Subject & [{\"user_id\": \"hnieh\"}, {\"user_id\": \"emanuele\"}]"
]
},
{
@@ -5478,7 +5485,7 @@
}
],
"source": [
- "subject.Subject.proj('dob', 'sex')"
+ "subject.Subject.proj(\"dob\", \"sex\")"
]
},
{
@@ -5615,7 +5622,7 @@
}
],
"source": [
- "subject.Subject.proj(gender='sex', birth_date='dob')"
+ "subject.Subject.proj(gender=\"sex\", birth_date=\"dob\")"
]
},
{
@@ -5638,7 +5645,9 @@
"metadata": {},
"outputs": [],
"source": [
- "weighing_with_date = action.Weighing.proj(weighing_date='date(weighing_time)') # more options, check MySQL syntax"
+ "weighing_with_date = action.Weighing.proj(\n",
+ " weighing_date=\"date(weighing_time)\"\n",
+ ") # more options, check MySQL syntax"
]
},
{
@@ -5922,7 +5931,7 @@
"source": [
"# First get the date of birth and the session date into the same query\n",
"q = subject.Subject * acquisition.Session\n",
- "q = q.proj('dob')\n",
+ "q = q.proj(\"dob\")\n",
"q"
]
},
@@ -6084,7 +6093,7 @@
],
"source": [
"# Then compute the age\n",
- "q_with_age = q.proj('dob', age='datediff(session_date, dob)') & 'dob is not NULL'\n",
+ "q_with_age = q.proj(\"dob\", age=\"datediff(session_date, dob)\") & \"dob is not NULL\"\n",
"q_with_age"
]
},
@@ -6229,7 +6238,9 @@
}
],
"source": [
- "subject.Subject.aggr(acquisition.Session, n='count(*)', lastest_session='min(session_date)')"
+ "subject.Subject.aggr(\n",
+ " acquisition.Session, n=\"count(*)\", lastest_session=\"min(session_date)\"\n",
+ ")"
]
},
{
@@ -14825,7 +14836,7 @@
}
],
"source": [
- "subjs['subject_fullname']"
+ "subjs[\"subject_fullname\"]"
]
},
{
@@ -14902,7 +14913,7 @@
}
],
"source": [
- "subjs['dob']"
+ "subjs[\"dob\"]"
]
},
{
@@ -26702,7 +26713,7 @@
],
"source": [
"# fetch as pandas dataframe\n",
- "subjs_df = subject.Subject.fetch(format='frame')\n",
+ "subjs_df = subject.Subject.fetch(format=\"frame\")\n",
"subjs_df"
]
},
@@ -26914,7 +26925,7 @@
],
"source": [
"# fetch the primary key\n",
- "pk = subject.Subject.fetch('KEY')\n",
+ "pk = subject.Subject.fetch(\"KEY\")\n",
"pk"
]
},
@@ -26925,7 +26936,7 @@
"outputs": [],
"source": [
"# fetch specific attributes\n",
- "dob, sex = subject.Subject.fetch('dob', 'sex')"
+ "dob, sex = subject.Subject.fetch(\"dob\", \"sex\")"
]
},
{
@@ -27213,7 +27224,7 @@
],
"source": [
"# fetch specific attributes as a list of dictionary\n",
- "info = subject.Subject.fetch('dob', 'sex', as_dict=True)\n",
+ "info = subject.Subject.fetch(\"dob\", \"sex\", as_dict=True)\n",
"info"
]
},
@@ -27230,7 +27241,9 @@
"metadata": {},
"outputs": [],
"source": [
- "B205 = (subject.Subject & {'subject_nickname': 'B205'}).fetch1() # \"fetch1()\" because we know there's only one"
+ "B205 = (\n",
+ " subject.Subject & {\"subject_nickname\": \"B205\"}\n",
+ ").fetch1() # \"fetch1()\" because we know there's only one"
]
},
{
@@ -27318,7 +27331,7 @@
"metadata": {},
"outputs": [],
"source": [
- "B205_key = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('KEY')"
+ "B205_key = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\"KEY\")"
]
},
{
@@ -27347,7 +27360,9 @@
"metadata": {},
"outputs": [],
"source": [
- "B205_init_weight = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('initial_weight')"
+ "B205_init_weight = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\n",
+ " \"initial_weight\"\n",
+ ")"
]
},
{
diff --git a/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb b/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb
index 152983fa..4dfc9e6d 100644
--- a/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb
+++ b/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb
@@ -24,17 +24,19 @@
}
],
"source": [
+ "import datetime\n",
+ "\n",
"import datajoint as dj\n",
"import numpy as np\n",
- "import matplotlib.pyplot as plt\n",
- "import pandas as pd\n",
- "import datetime\n",
- "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n",
- "task = dj.create_virtual_module('task', 'u19_task') \n",
- "subject = dj.create_virtual_module('subject', 'u19_subject')\n",
- "action = dj.create_virtual_module('action', 'u19_action')\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "\n",
+ "lab = dj.create_virtual_module(\n",
+ " \"lab\", \"u19_lab\"\n",
+ ") # the first argument here is the __name__ of the virtual module\n",
+ "task = dj.create_virtual_module(\"task\", \"u19_task\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n",
+ "action = dj.create_virtual_module(\"action\", \"u19_action\")\n",
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -502,7 +504,7 @@
}
],
"source": [
- "behavior.TowersBlock() # bracket necessary"
+ "behavior.TowersBlock() # bracket necessary"
]
},
{
@@ -654,8 +656,10 @@
}
],
"source": [
- "behavior.TowersSession.aggr(behavior.TowersBlock.proj('block_performance'), \n",
- " avg_performance='avg(block_performance)')"
+ "behavior.TowersSession.aggr(\n",
+ " behavior.TowersBlock.proj(\"block_performance\"),\n",
+ " avg_performance=\"avg(block_performance)\",\n",
+ ")"
]
},
{
@@ -686,7 +690,7 @@
"metadata": {},
"outputs": [],
"source": [
- "schema = dj.schema('shans_tutorial')"
+ "schema = dj.schema(\"shans_tutorial\")"
]
},
{
@@ -1050,8 +1054,7 @@
}
],
"source": [
- "behavior.TowersSession.aggr(behavior.TowersBlock.proj(),\n",
- " n_sessions='count(*)')"
+ "behavior.TowersSession.aggr(behavior.TowersBlock.proj(), n_sessions=\"count(*)\")"
]
},
{
@@ -1068,9 +1071,9 @@
"outputs": [],
"source": [
"key = {\n",
- " 'subject_fullname': 'emanuele_B205',\n",
- " 'session_date': datetime.date(2018, 7, 13),\n",
- " 'session_number': 0\n",
+ " \"subject_fullname\": \"emanuele_B205\",\n",
+ " \"session_date\": datetime.date(2018, 7, 13),\n",
+ " \"session_number\": 0,\n",
"}"
]
},
@@ -1080,10 +1083,10 @@
"metadata": {},
"outputs": [],
"source": [
- "performances = (behavior.TowersBlock & key).fetch('block_performance')\n",
+ "performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n",
"\n",
"# create another field in the dictionary key\n",
- "key['avg_performance'] = np.mean(performances)"
+ "key[\"avg_performance\"] = np.mean(performances)"
]
},
{
@@ -1122,7 +1125,9 @@
"metadata": {},
"outputs": [],
"source": [
- "SessionPerformanceManual.insert1(key, skip_duplicates=True) # insert1 only works for one entry"
+ "SessionPerformanceManual.insert1(\n",
+ " key, skip_duplicates=True\n",
+ ") # insert1 only works for one entry"
]
},
{
@@ -1341,7 +1346,9 @@
}
],
"source": [
- "subject.Subject.aggr(behavior.TowersSession.proj(), n_sessions='count(*)') & 'n_sessions=80'"
+ "subject.Subject.aggr(\n",
+ " behavior.TowersSession.proj(), n_sessions=\"count(*)\"\n",
+ ") & \"n_sessions=80\""
]
},
{
@@ -1367,16 +1374,16 @@
"source": [
"# loop through sessions of subject B205 and insert one by one, and compute time\n",
"import time\n",
+ "\n",
"start_time = time.time()\n",
"\n",
- "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E78\"').fetch('KEY'):\n",
- " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n",
+ "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E78\"').fetch(\"KEY\"):\n",
+ " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n",
" # create another field in the dictionary key\n",
" avg_performance = np.mean(performances)\n",
" if np.isnan(avg_performance):\n",
" continue\n",
- " entry = dict(**i_session, \n",
- " avg_performance=avg_performance)\n",
+ " entry = dict(**i_session, avg_performance=avg_performance)\n",
" SessionPerformanceManual.insert1(entry)\n",
"\n",
"print(\"--- %s seconds ---\" % (time.time() - start_time))"
@@ -1400,16 +1407,17 @@
"start_time = time.time()\n",
"\n",
"perf_entries = []\n",
- "for i_session in (behavior.TowersSession & 'subject_fullname=\"lpinto_vg28\"').fetch('KEY'):\n",
- " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n",
+ "for i_session in (behavior.TowersSession & 'subject_fullname=\"lpinto_vg28\"').fetch(\n",
+ " \"KEY\"\n",
+ "):\n",
+ " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n",
" # create another field in the dictionary key\n",
" avg_performance = np.mean(performances)\n",
" if np.isnan(avg_performance):\n",
" continue\n",
- " entry = dict(**i_session, \n",
- " avg_performance=avg_performance)\n",
+ " entry = dict(**i_session, avg_performance=avg_performance)\n",
" perf_entries.append(entry)\n",
- " \n",
+ "\n",
"SessionPerformanceManual.insert(perf_entries)\n",
"print(\"--- %s seconds ---\" % (time.time() - start_time))"
]
@@ -1435,13 +1443,18 @@
" ---\n",
" avg_performance: float # a final product in this table\n",
" \"\"\"\n",
- " key_source = acquisition.Session() & 'subject_fullname=\"lpinto_vg28\"' # bracket necessary\n",
- " def make(self, key): # key is one primary key of the entries in table acquisition.Session\n",
+ " key_source = (\n",
+ " acquisition.Session() & 'subject_fullname=\"lpinto_vg28\"'\n",
+ " ) # bracket necessary\n",
+ "\n",
+ " def make(\n",
+ " self, key\n",
+ " ): # key is one primary key of the entries in table acquisition.Session\n",
" # fetch the performance for each block\n",
- " performances = (behavior.TowersBlock & key).fetch('block_performance')\n",
- " \n",
+ " performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n",
+ "\n",
" # create another field in the dictionary key\n",
- " key['avg_performance'] = np.mean(performances)\n",
+ " key[\"avg_performance\"] = np.mean(performances)\n",
" self.insert1(key)"
]
},
@@ -1466,7 +1479,7 @@
}
],
"source": [
- "SessionPerformanceComputed.populate(display_progress=True) \n",
+ "SessionPerformanceComputed.populate(display_progress=True)\n",
"# first argument could be some restrictor to control the populate"
]
},
@@ -1654,20 +1667,22 @@
" avg_performance: float # a final product in this table\n",
" \"\"\"\n",
" key_source = subject.Subject()\n",
- " def make(self, key): # key is one primary key of the entries in table subject.Subject()!\n",
+ "\n",
+ " def make(\n",
+ " self, key\n",
+ " ): # key is one primary key of the entries in table subject.Subject()!\n",
"\n",
" perf_entries = []\n",
- " for i_session in (behavior.TowersSession & key).fetch('KEY'):\n",
+ " for i_session in (behavior.TowersSession & key).fetch(\"KEY\"):\n",
" # fetch performance of each block\n",
- " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n",
+ " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n",
" # create another field in the dictionary key\n",
" avg_performance = np.mean(performances)\n",
" if np.isnan(avg_performance):\n",
" continue\n",
- " entry = dict(**i_session, \n",
- " avg_performance=avg_performance)\n",
+ " entry = dict(**i_session, avg_performance=avg_performance)\n",
" perf_entries.append(entry)\n",
- " \n",
+ "\n",
" self.insert(perf_entries)"
]
},
@@ -9987,7 +10002,9 @@
}
],
"source": [
- "SessionPerformanceComputedFromSubject.populate(display_progress=True, suppress_errors=True, reserved_jobs=True)"
+ "SessionPerformanceComputedFromSubject.populate(\n",
+ " display_progress=True, suppress_errors=True, reserved_jobs=True\n",
+ ")"
]
},
{
@@ -10003,7 +10020,9 @@
"metadata": {},
"outputs": [],
"source": [
- "(SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"').delete() # any restrictor would work here"
+ "(\n",
+ " SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"'\n",
+ ").delete() # any restrictor would work here"
]
},
{
diff --git a/notebooks/tutorials/Old_tutorials/202001/subject.py b/notebooks/tutorials/Old_tutorials/202001/subject.py
index 13623dde..7e463db7 100644
--- a/notebooks/tutorials/Old_tutorials/202001/subject.py
+++ b/notebooks/tutorials/Old_tutorials/202001/subject.py
@@ -1,12 +1,11 @@
"""This module was auto-generated by datajoint from an existing schema"""
-
import datajoint as dj
-schema = dj.schema('U19_subject')
+schema = dj.schema("U19_subject")
-vmod0 = dj.create_virtual_module('vmod0', 'U19_lab')
+vmod0 = dj.create_virtual_module("vmod0", "U19_lab")
@schema
@@ -134,7 +133,6 @@ class HealthStatus(dj.Manual):
comments=null : varchar(255)
"""
-
class Action(dj.Part):
definition = """
-> HealthStatus
@@ -252,4 +250,4 @@ class AlleleSequence(dj.Lookup):
definition = """
-> Allele
-> Sequence
- """
\ No newline at end of file
+ """
diff --git a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb
index ac5141d8..ae1aae44 100644
--- a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb
+++ b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb
@@ -23,6 +23,7 @@
"outputs": [],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -122,7 +123,7 @@
" \"display.show_tuple_count\": true,\n",
" \"database.use_tls\": null,\n",
" \"enable_python_native_blobs\": false,\n",
- " \"database.prefix\": null\n",
+ " \"database.prefix\": null,\n",
"}"
]
},
@@ -153,7 +154,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from getpass import getpass # use this to type password in without showing it"
+ "from getpass import getpass # use this to type password in without showing it"
]
},
{
@@ -162,8 +163,8 @@
"metadata": {},
"outputs": [],
"source": [
- "dj.config['database.user'] = 'shans'\n",
- "dj.config['database.password'] = getpass('Type password:')"
+ "dj.config[\"database.user\"] = \"shans\"\n",
+ "dj.config[\"database.password\"] = getpass(\"Type password:\")"
]
},
{
diff --git a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb
index 6bd9a9e5..507fdccb 100644
--- a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb
+++ b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb
@@ -162,7 +162,7 @@
" \"display.show_tuple_count\": true,\n",
" \"database.use_tls\": null,\n",
" \"enable_python_native_blobs\": false,\n",
- " \"database.prefix\": null\n",
+ " \"database.prefix\": null,\n",
"}"
]
},
@@ -193,7 +193,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from getpass import getpass # use this to type password in without showing it"
+ "from getpass import getpass # use this to type password in without showing it"
]
},
{
@@ -202,8 +202,8 @@
"metadata": {},
"outputs": [],
"source": [
- "dj.config['database.user'] = 'shans'\n",
- "dj.config['database.password'] = getpass('Type password:')"
+ "dj.config[\"database.user\"] = \"shans\"\n",
+ "dj.config[\"database.password\"] = getpass(\"Type password:\")"
]
},
{
diff --git a/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb b/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb
index 39e4a077..fdb18ad0 100644
--- a/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb
+++ b/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb
@@ -26,6 +26,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"import datajoint as dj"
]
@@ -501,12 +502,14 @@
"metadata": {},
"outputs": [],
"source": [
- "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n",
- "task = dj.create_virtual_module('task', 'u19_task') \n",
- "subject = dj.create_virtual_module('subject', 'u19_subject')\n",
- "action = dj.create_virtual_module('action', 'u19_action')\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "lab = dj.create_virtual_module(\n",
+ " \"lab\", \"u19_lab\"\n",
+ ") # the first argument here is the __name__ of the virtual module\n",
+ "task = dj.create_virtual_module(\"task\", \"u19_task\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n",
+ "action = dj.create_virtual_module(\"action\", \"u19_action\")\n",
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -1215,7 +1218,12 @@
}
],
"source": [
- "dj.Diagram(subject.Subject) + dj.Diagram(subject.Death) + dj.Diagram(subject.HealthStatus) + dj.Diagram(subject.Weaning)"
+ "(\n",
+ " dj.Diagram(subject.Subject)\n",
+ " + dj.Diagram(subject.Death)\n",
+ " + dj.Diagram(subject.HealthStatus)\n",
+ " + dj.Diagram(subject.Weaning)\n",
+ ")"
]
},
{
@@ -2117,7 +2125,7 @@
],
"source": [
"# restrict by dictionary\n",
- "subject.Subject & {'subject_nickname': 'B205'}"
+ "subject.Subject & {\"subject_nickname\": \"B205\"}"
]
},
{
@@ -2411,7 +2419,7 @@
}
],
"source": [
- "subject.Subject & {'sex': 'Male'}"
+ "subject.Subject & {\"sex\": \"Male\"}"
]
},
{
@@ -2712,7 +2720,7 @@
}
],
"source": [
- "subject.Subject & [{'user_id': 'hnieh'}, {'user_id': 'emanuele'}, 'line=\"DAT-IRES-CRE\"']"
+ "subject.Subject & [{\"user_id\": \"hnieh\"}, {\"user_id\": \"emanuele\"}, 'line=\"DAT-IRES-CRE\"']"
]
},
{
@@ -6323,7 +6331,7 @@
}
],
"source": [
- "(subject.Subject * acquisition.Session).proj('dob', 'sex', 'location')"
+ "(subject.Subject * acquisition.Session).proj(\"dob\", \"sex\", \"location\")"
]
},
{
@@ -6459,7 +6467,7 @@
}
],
"source": [
- "subject.Subject.proj(gender='sex', birth_date='dob') # 'sex->gender'"
+ "subject.Subject.proj(gender=\"sex\", birth_date=\"dob\") # 'sex->gender'"
]
},
{
@@ -6482,7 +6490,9 @@
"metadata": {},
"outputs": [],
"source": [
- "weighing_with_date = action.Weighing.proj(weighing_date='date(weighing_time)') # more options, check MySQL syntax"
+ "weighing_with_date = action.Weighing.proj(\n",
+ " weighing_date=\"date(weighing_time)\"\n",
+ ") # more options, check MySQL syntax"
]
},
{
@@ -6764,7 +6774,7 @@
"source": [
"# First get the date of birth and the session date into the same query\n",
"q = subject.Subject * acquisition.Session\n",
- "q = q.proj('dob')\n",
+ "q = q.proj(\"dob\")\n",
"q"
]
},
@@ -6925,7 +6935,7 @@
],
"source": [
"# Then compute the age\n",
- "q_with_age = q.proj('dob', age='datediff(session_date, dob)') & 'dob is not NULL'\n",
+ "q_with_age = q.proj(\"dob\", age=\"datediff(session_date, dob)\") & \"dob is not NULL\"\n",
"q_with_age"
]
},
@@ -7069,7 +7079,9 @@
}
],
"source": [
- "subject.Subject.aggr(acquisition.Session, n='count(*)', lastest_session_date='max(session_date)')"
+ "subject.Subject.aggr(\n",
+ " acquisition.Session, n=\"count(*)\", lastest_session_date=\"max(session_date)\"\n",
+ ")"
]
},
{
@@ -25970,7 +25982,7 @@
}
],
"source": [
- "subjs['subject_fullname']"
+ "subjs[\"subject_fullname\"]"
]
},
{
@@ -26201,7 +26213,7 @@
}
],
"source": [
- "subjs['dob']"
+ "subjs[\"dob\"]"
]
},
{
@@ -52431,7 +52443,7 @@
],
"source": [
"# fetch as pandas dataframe\n",
- "subjs_df = subject.Subject.fetch(format='frame').reset_index()\n",
+ "subjs_df = subject.Subject.fetch(format=\"frame\").reset_index()\n",
"subjs_df"
]
},
@@ -53010,7 +53022,7 @@
],
"source": [
"# fetch the primary key\n",
- "pk = subject.Subject.fetch('KEY')\n",
+ "pk = subject.Subject.fetch(\"KEY\")\n",
"pk"
]
},
@@ -53021,7 +53033,7 @@
"outputs": [],
"source": [
"# fetch specific attributes\n",
- "dob, sex = subject.Subject.fetch('dob', 'sex')"
+ "dob, sex = subject.Subject.fetch(\"dob\", \"sex\")"
]
},
{
@@ -53830,7 +53842,7 @@
],
"source": [
"# fetch specific attributes as a list of dictionary\n",
- "info = subject.Subject.fetch('dob', 'sex', as_dict=True)\n",
+ "info = subject.Subject.fetch(\"dob\", \"sex\", as_dict=True)\n",
"info"
]
},
@@ -53847,7 +53859,9 @@
"metadata": {},
"outputs": [],
"source": [
- "B205 = (subject.Subject & {'subject_nickname': 'B205'}).fetch1() # \"fetch1()\" because we know there's only one"
+ "B205 = (\n",
+ " subject.Subject & {\"subject_nickname\": \"B205\"}\n",
+ ").fetch1() # \"fetch1()\" because we know there's only one"
]
},
{
@@ -53937,7 +53951,7 @@
"metadata": {},
"outputs": [],
"source": [
- "B205_key = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('KEY')"
+ "B205_key = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\"KEY\")"
]
},
{
@@ -53966,7 +53980,9 @@
"metadata": {},
"outputs": [],
"source": [
- "B205_init_weight = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('initial_weight')"
+ "B205_init_weight = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\n",
+ " \"initial_weight\"\n",
+ ")"
]
},
{
@@ -54006,7 +54022,7 @@
}
],
"source": [
- "(subject.Subject & {'subject_nickname': 'B205'}).fetch('initial_weight')"
+ "(subject.Subject & {\"subject_nickname\": \"B205\"}).fetch(\"initial_weight\")"
]
},
{
diff --git a/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb b/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb
index b9098b41..3b157f20 100644
--- a/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb
+++ b/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb
@@ -26,18 +26,21 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
+ "import datetime\n",
+ "\n",
"import datajoint as dj\n",
"import numpy as np\n",
- "import matplotlib.pyplot as plt\n",
- "import pandas as pd\n",
- "import datetime\n",
- "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n",
- "task = dj.create_virtual_module('task', 'u19_task') \n",
- "subject = dj.create_virtual_module('subject', 'u19_subject')\n",
- "action = dj.create_virtual_module('action', 'u19_action')\n",
- "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n",
- "behavior = dj.create_virtual_module('behavior', 'u19_behavior')"
+ "\n",
+ "lab = dj.create_virtual_module(\n",
+ " \"lab\", \"u19_lab\"\n",
+ ") # the first argument here is the __name__ of the virtual module\n",
+ "task = dj.create_virtual_module(\"task\", \"u19_task\")\n",
+ "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n",
+ "action = dj.create_virtual_module(\"action\", \"u19_action\")\n",
+ "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n",
+ "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")"
]
},
{
@@ -691,7 +694,7 @@
}
],
"source": [
- "behavior.TowersBlock() # bracket necessary"
+ "behavior.TowersBlock() # bracket necessary"
]
},
{
@@ -842,8 +845,10 @@
}
],
"source": [
- "behavior.TowersSession.aggr(behavior.TowersBlock.proj('block_performance'), \n",
- " avg_performance='avg(block_performance)')"
+ "behavior.TowersSession.aggr(\n",
+ " behavior.TowersBlock.proj(\"block_performance\"),\n",
+ " avg_performance=\"avg(block_performance)\",\n",
+ ")"
]
},
{
@@ -874,7 +879,7 @@
"metadata": {},
"outputs": [],
"source": [
- "schema = dj.schema('alvaros_tutorial2021')"
+ "schema = dj.schema(\"alvaros_tutorial2021\")"
]
},
{
@@ -1288,8 +1293,7 @@
}
],
"source": [
- "behavior.TowersSession.aggr(behavior.TowersBlock.proj(),\n",
- " n_sessions='count(*)')"
+ "behavior.TowersSession.aggr(behavior.TowersBlock.proj(), n_sessions=\"count(*)\")"
]
},
{
@@ -1306,9 +1310,9 @@
"outputs": [],
"source": [
"key = {\n",
- " 'subject_fullname': 'emanuele_B205',\n",
- " 'session_date': datetime.date(2018, 7, 13),\n",
- " 'session_number': 0\n",
+ " \"subject_fullname\": \"emanuele_B205\",\n",
+ " \"session_date\": datetime.date(2018, 7, 13),\n",
+ " \"session_number\": 0,\n",
"}"
]
},
@@ -1561,10 +1565,10 @@
"metadata": {},
"outputs": [],
"source": [
- "performances = (behavior.TowersBlock & key).fetch('block_performance')\n",
+ "performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n",
"\n",
"# create another field in the dictionary key\n",
- "key['avg_performance'] = np.mean(performances)"
+ "key[\"avg_performance\"] = np.mean(performances)"
]
},
{
@@ -1603,7 +1607,9 @@
"metadata": {},
"outputs": [],
"source": [
- "SessionPerformanceManual.insert1(key, skip_duplicates=True) # insert1 only works for one entry"
+ "SessionPerformanceManual.insert1(\n",
+ " key, skip_duplicates=True\n",
+ ") # insert1 only works for one entry"
]
},
{
@@ -1820,7 +1826,9 @@
}
],
"source": [
- "subject.Subject.aggr(behavior.TowersSession.proj(), n_sessions='count(*)') & 'n_sessions=100'"
+ "subject.Subject.aggr(\n",
+ " behavior.TowersSession.proj(), n_sessions=\"count(*)\"\n",
+ ") & \"n_sessions=100\""
]
},
{
@@ -1846,16 +1854,16 @@
"source": [
"# loop through sessions of subject B205 and insert one by one, and compute time\n",
"import time\n",
+ "\n",
"start_time = time.time()\n",
"\n",
- "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E57\"').fetch('KEY'):\n",
- " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n",
+ "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E57\"').fetch(\"KEY\"):\n",
+ " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n",
" # create another field in the dictionary key\n",
" avg_performance = np.mean(performances)\n",
" if np.isnan(avg_performance):\n",
" continue\n",
- " entry = dict(**i_session, \n",
- " avg_performance=avg_performance)\n",
+ " entry = dict(**i_session, avg_performance=avg_performance)\n",
" SessionPerformanceManual.insert1(entry)\n",
"\n",
"print(\"--- %s seconds ---\" % (time.time() - start_time))"
@@ -1879,16 +1887,15 @@
"start_time = time.time()\n",
"\n",
"perf_entries = []\n",
- "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E77\"').fetch('KEY'):\n",
- " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n",
+ "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E77\"').fetch(\"KEY\"):\n",
+ " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n",
" # create another field in the dictionary key\n",
" avg_performance = np.mean(performances)\n",
" if np.isnan(avg_performance):\n",
" continue\n",
- " entry = dict(**i_session, \n",
- " avg_performance=avg_performance)\n",
+ " entry = dict(**i_session, avg_performance=avg_performance)\n",
" perf_entries.append(entry)\n",
- " \n",
+ "\n",
"SessionPerformanceManual.insert(perf_entries)\n",
"print(\"--- %s seconds ---\" % (time.time() - start_time))"
]
@@ -1914,14 +1921,19 @@
" ---\n",
" avg_performance: float # a final product in this table\n",
" \"\"\"\n",
- " \n",
- " key_source = acquisition.Session() & 'subject_fullname=\"hnieh_E57\"' # bracket necessary\n",
- " def make(self, key): # key is one primary key of the entries in table acquisition.Session\n",
+ "\n",
+ " key_source = (\n",
+ " acquisition.Session() & 'subject_fullname=\"hnieh_E57\"'\n",
+ " ) # bracket necessary\n",
+ "\n",
+ " def make(\n",
+ " self, key\n",
+ " ): # key is one primary key of the entries in table acquisition.Session\n",
" # fetch the performance for each block\n",
- " performances = (behavior.TowersBlock & key).fetch('block_performance')\n",
- " \n",
+ " performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n",
+ "\n",
" # create another field in the dictionary key\n",
- " key['avg_performance'] = np.mean(performances)\n",
+ " key[\"avg_performance\"] = np.mean(performances)\n",
" self.insert1(key)"
]
},
@@ -1946,7 +1958,7 @@
}
],
"source": [
- "SessionPerformanceComputed.populate(display_progress=True) \n",
+ "SessionPerformanceComputed.populate(display_progress=True)\n",
"# first argument could be some restrictor to control the populate"
]
},
@@ -2132,21 +2144,23 @@
" ---\n",
" avg_performance: float # a final product in this table\n",
" \"\"\"\n",
- " key_source = subject.Subject() \n",
- " def make(self, key): # key is one primary key of the entries in table subject.Subject()!\n",
+ " key_source = subject.Subject()\n",
+ "\n",
+ " def make(\n",
+ " self, key\n",
+ " ): # key is one primary key of the entries in table subject.Subject()!\n",
"\n",
" perf_entries = []\n",
- " for i_session in (behavior.TowersSession & key).fetch('KEY'):\n",
+ " for i_session in (behavior.TowersSession & key).fetch(\"KEY\"):\n",
" # fetch performance of each block\n",
- " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n",
+ " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n",
" # create another field in the dictionary key\n",
" avg_performance = np.mean(performances)\n",
" if np.isnan(avg_performance):\n",
" continue\n",
- " entry = dict(**i_session, \n",
- " avg_performance=avg_performance)\n",
+ " entry = dict(**i_session, avg_performance=avg_performance)\n",
" perf_entries.append(entry)\n",
- " \n",
+ "\n",
" self.insert(perf_entries)"
]
},
@@ -2174,7 +2188,12 @@
}
],
"source": [
- "SessionPerformanceComputedFromSubject.populate('subject_fullname=\"hnieh_E77\"', display_progress=True, suppress_errors=True, reserve_jobs=True)"
+ "SessionPerformanceComputedFromSubject.populate(\n",
+ " 'subject_fullname=\"hnieh_E77\"',\n",
+ " display_progress=True,\n",
+ " suppress_errors=True,\n",
+ " reserve_jobs=True,\n",
+ ")"
]
},
{
@@ -2209,7 +2228,9 @@
}
],
"source": [
- "(SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"').delete() # any restrictor would work here"
+ "(\n",
+ " SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"'\n",
+ ").delete() # any restrictor would work here"
]
},
{
diff --git a/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb b/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb
index ef2b5c2a..a3eabe31 100644
--- a/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb
+++ b/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb
@@ -42,6 +42,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"import datajoint as dj"
]
@@ -234,7 +235,7 @@
],
"source": [
"# create your own schema object\n",
- "schema = dj.schema('alvaros_tutorial2021')"
+ "schema = dj.schema(\"alvaros_tutorial2021\")"
]
},
{
@@ -399,7 +400,7 @@
"outputs": [],
"source": [
"# inserting the first mouse\n",
- "Mouse.insert1((0, '2017-03-01', 'M'))"
+ "Mouse.insert1((0, \"2017-03-01\", \"M\"))"
]
},
{
@@ -512,7 +513,7 @@
"metadata": {},
"outputs": [],
"source": [
- "Mouse.insert1((20, datetime.date(2020, 5, 3), 'F'))"
+ "Mouse.insert1((20, datetime.date(2020, 5, 3), \"F\"))"
]
},
{
@@ -626,11 +627,7 @@
"metadata": {},
"outputs": [],
"source": [
- "data = {\n",
- " 'mouse_id': 100,\n",
- " 'dob': '2017-05-12',\n",
- " 'sex': 'F'\n",
- "}"
+ "data = {\"mouse_id\": 100, \"dob\": \"2017-05-12\", \"sex\": \"F\"}"
]
},
{
@@ -757,11 +754,7 @@
"metadata": {},
"outputs": [],
"source": [
- "data = [\n",
- " (1, '2016-11-19', 'M'),\n",
- " (2, '2016-11-20', 'unknown'),\n",
- " (5, '2016-12-25', 'F')\n",
- "]"
+ "data = [(1, \"2016-11-19\", \"M\"), (2, \"2016-11-20\", \"unknown\"), (5, \"2016-12-25\", \"F\")]"
]
},
{
@@ -898,8 +891,8 @@
"outputs": [],
"source": [
"data = [\n",
- " {'mouse_id': 10, 'dob': '2017-01-01', 'sex': 'F'},\n",
- " {'mouse_id': 11, 'dob': '2017-01-03', 'sex': 'F'},\n",
+ " {\"mouse_id\": 10, \"dob\": \"2017-01-01\", \"sex\": \"F\"},\n",
+ " {\"mouse_id\": 11, \"dob\": \"2017-01-03\", \"sex\": \"F\"},\n",
"]\n",
"\n",
"# insert them all\n",
@@ -1057,9 +1050,10 @@
],
"source": [
"Mouse.insert1(\n",
- " {'mouse_id': 0,\n",
- " 'dob': '2018-01-01',\n",
- " 'sex': 'M',\n",
+ " {\n",
+ " \"mouse_id\": 0,\n",
+ " \"dob\": \"2018-01-01\",\n",
+ " \"sex\": \"M\",\n",
" }\n",
")"
]
@@ -1106,7 +1100,7 @@
}
],
"source": [
- "(Mouse & 'mouse_id=11').delete()"
+ "(Mouse & \"mouse_id=11\").delete()"
]
},
{
@@ -1240,7 +1234,7 @@
}
],
"source": [
- "(Mouse & 'mouse_id=0')._update('sex', 'M')"
+ "(Mouse & \"mouse_id=0\")._update(\"sex\", \"M\")"
]
},
{
@@ -1532,10 +1526,10 @@
"outputs": [],
"source": [
"data = {\n",
- " 'mouse_id': 0,\n",
- " 'session_date': '2017-05-15',\n",
- " 'experiment_setup': 0,\n",
- " 'experimenter': 'Edgar Y. Walker'\n",
+ " \"mouse_id\": 0,\n",
+ " \"session_date\": \"2017-05-15\",\n",
+ " \"experiment_setup\": 0,\n",
+ " \"experimenter\": \"Edgar Y. Walker\",\n",
"}\n",
"\n",
"Session.insert1(data)"
@@ -1748,10 +1742,10 @@
],
"source": [
"data = {\n",
- " 'mouse_id': 0,\n",
- " 'session_date': '2018-01-15',\n",
- " 'experiment_setup': 100,\n",
- " 'experimenter': 'Jacob Reimer'\n",
+ " \"mouse_id\": 0,\n",
+ " \"session_date\": \"2018-01-15\",\n",
+ " \"experiment_setup\": 100,\n",
+ " \"experimenter\": \"Jacob Reimer\",\n",
"}\n",
"\n",
"# insert them all\n",
@@ -1774,10 +1768,10 @@
"outputs": [],
"source": [
"data = {\n",
- " 'mouse_id': 1,\n",
- " 'session_date': '2018-01-15',\n",
- " 'experiment_setup': 101,\n",
- " 'experimenter': 'Jacob Reimer'\n",
+ " \"mouse_id\": 1,\n",
+ " \"session_date\": \"2018-01-15\",\n",
+ " \"experiment_setup\": 101,\n",
+ " \"experimenter\": \"Jacob Reimer\",\n",
"}\n",
"\n",
"# insert them all\n",
@@ -1905,10 +1899,10 @@
"outputs": [],
"source": [
"bad_data = {\n",
- " 'mouse_id': 9999, # this mouse doesn't exist!\n",
- " 'session_date': '2017-05-15',\n",
- " 'experiment_setup': 0,\n",
- " 'experimenter': 'Edgar Y. Walker'\n",
+ " \"mouse_id\": 9999, # this mouse doesn't exist!\n",
+ " \"session_date\": \"2017-05-15\",\n",
+ " \"experiment_setup\": 0,\n",
+ " \"experimenter\": \"Edgar Y. Walker\",\n",
"}"
]
},
@@ -1986,7 +1980,7 @@
}
],
"source": [
- "(Mouse & 'mouse_id = 100').delete()"
+ "(Mouse & \"mouse_id = 100\").delete()"
]
},
{
diff --git a/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb b/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb
index b20c803f..61866e32 100644
--- a/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb
+++ b/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb
@@ -39,12 +39,10 @@
"metadata": {},
"outputs": [],
"source": [
- "import pandas as pd\n",
- "import numpy as np\n",
"import datajoint as dj\n",
"import matplotlib\n",
- "import matplotlib.pyplot as plt\n",
- "matplotlib.style.use('ggplot')\n",
+ "\n",
+ "matplotlib.style.use(\"ggplot\")\n",
"%matplotlib inline"
]
},
@@ -65,7 +63,7 @@
"metadata": {},
"outputs": [],
"source": [
- "netid = 'ahoag' # change this to your netid "
+ "netid = \"ahoag\" # change this to your netid"
]
},
{
@@ -93,8 +91,8 @@
}
],
"source": [
- "dj.config['database.host'] = 'datajoint00.pni.princeton.edu'\n",
- "dj.config['database.user'] = netid\n",
+ "dj.config[\"database.host\"] = \"datajoint00.pni.princeton.edu\"\n",
+ "dj.config[\"database.user\"] = netid\n",
"dj.conn()"
]
},
@@ -111,7 +109,9 @@
"metadata": {},
"outputs": [],
"source": [
- "db_lightsheet = dj.create_virtual_module('u19lightserv_lightsheet','u19lightserv_lightsheet')"
+ "db_lightsheet = dj.create_virtual_module(\n",
+ " \"u19lightserv_lightsheet\", \"u19lightserv_lightsheet\"\n",
+ ")"
]
},
{
@@ -314,7 +314,7 @@
],
"source": [
"# Let's take a look at the schema design to get an overview of the tables and their relationships\n",
- "dj.ERD(db_lightsheet.User) + 1 "
+ "dj.ERD(db_lightsheet.User) + 1"
]
},
{
@@ -598,8 +598,10 @@
}
],
"source": [
- "username = 'jverpeut' # change this if you want to look up requests from a different netid\n",
- "db_lightsheet.Request() & {'username':username} "
+ "username = (\n",
+ " \"jverpeut\" # change this if you want to look up requests from a different netid\n",
+ ")\n",
+ "db_lightsheet.Request() & {\"username\": username}"
]
},
{
@@ -732,8 +734,8 @@
}
],
"source": [
- "request_name='natneuroreviews_tompisano_CTB' # change this if you want to look at a different request\n",
- "db_lightsheet.Request.Sample() & {'username':username,'request_name':request_name}\n",
+ "request_name = \"natneuroreviews_tompisano_CTB\" # change this if you want to look at a different request\n",
+ "db_lightsheet.Request.Sample() & {\"username\": username, \"request_name\": request_name}\n",
"# notice the syntax used. Sample is a part table of Request, hence the Request.Sample() syntax"
]
},
@@ -891,7 +893,10 @@
}
],
"source": [
- "db_lightsheet.Request.ClearingBatch() & {'username':username,'request_name':request_name}"
+ "db_lightsheet.Request.ClearingBatch() & {\n",
+ " \"username\": username,\n",
+ " \"request_name\": request_name,\n",
+ "}"
]
},
{
@@ -1980,7 +1985,10 @@
}
],
"source": [
- "db_lightsheet.Request.IdiscoPlusClearing & {'username':username,'request_name':request_name}"
+ "db_lightsheet.Request.IdiscoPlusClearing & {\n",
+ " \"username\": username,\n",
+ " \"request_name\": request_name,\n",
+ "}"
]
},
{
@@ -3274,7 +3282,10 @@
}
],
"source": [
- "db_lightsheet.Request.ImagingRequest() & {'username':username,'request_name':request_name}"
+ "db_lightsheet.Request.ImagingRequest() & {\n",
+ " \"username\": username,\n",
+ " \"request_name\": request_name,\n",
+ "}"
]
},
{
@@ -3409,7 +3420,10 @@
}
],
"source": [
- "db_lightsheet.Request.ImagingResolutionRequest() & {'username':username,'request_name':request_name}"
+ "db_lightsheet.Request.ImagingResolutionRequest() & {\n",
+ " \"username\": username,\n",
+ " \"request_name\": request_name,\n",
+ "}"
]
},
{
@@ -3438,9 +3452,11 @@
}
],
"source": [
- "request_contents = db_lightsheet.Request.ImagingResolutionRequest() & \\\n",
- " {'username':'ejdennis','request_name':'201905_atlas00x_where_x=1:n'}\n",
- "request_contents.fetch1('notes_from_imaging')"
+ "request_contents = db_lightsheet.Request.ImagingResolutionRequest() & {\n",
+ " \"username\": \"ejdennis\",\n",
+ " \"request_name\": \"201905_atlas00x_where_x=1:n\",\n",
+ "}\n",
+ "request_contents.fetch1(\"notes_from_imaging\")"
]
},
{
@@ -3727,7 +3743,10 @@
}
],
"source": [
- "db_lightsheet.Request.ImagingChannel() & {'username':username,'request_name':request_name}"
+ "db_lightsheet.Request.ImagingChannel() & {\n",
+ " \"username\": username,\n",
+ " \"request_name\": request_name,\n",
+ "}"
]
},
{
@@ -4122,7 +4141,10 @@
}
],
"source": [
- "db_lightsheet.Request.ProcessingRequest() & {'username':username,'request_name':request_name}"
+ "db_lightsheet.Request.ProcessingRequest() & {\n",
+ " \"username\": username,\n",
+ " \"request_name\": request_name,\n",
+ "}"
]
},
{
@@ -4282,7 +4304,10 @@
}
],
"source": [
- "db_lightsheet.Request.ProcessingResolutionRequest() & {'username':username,'request_name':request_name}"
+ "db_lightsheet.Request.ProcessingResolutionRequest() & {\n",
+ " \"username\": username,\n",
+ " \"request_name\": request_name,\n",
+ "}"
]
},
{
@@ -4445,7 +4470,10 @@
}
],
"source": [
- "db_lightsheet.Request.ProcessingChannel() & {'username':username,'request_name':request_name}"
+ "db_lightsheet.Request.ProcessingChannel() & {\n",
+ " \"username\": username,\n",
+ " \"request_name\": request_name,\n",
+ "}"
]
},
{
@@ -4506,14 +4534,18 @@
}
],
"source": [
- "username1 = 'ejdennis' # change as needed\n",
- "request_name1 = '201905_atlas00x_where_x=1:n' # change as needed\n",
+ "username1 = \"ejdennis\" # change as needed\n",
+ "request_name1 = \"201905_atlas00x_where_x=1:n\" # change as needed\n",
"# Let's check the archival column of this request\n",
- "request_contents = db_lightsheet.Request() & \\\n",
- " {'username':username1,'request_name':request_name1}\n",
- "print(\"is_archival column = \",request_contents.fetch1('is_archival'))\n",
- "contents = db_lightsheet.Request.ImagingResolutionRequest() & \\\n",
- " {'username':username1,'request_name':request_name1}\n",
+ "request_contents = db_lightsheet.Request() & {\n",
+ " \"username\": username1,\n",
+ " \"request_name\": request_name1,\n",
+ "}\n",
+ "print(\"is_archival column = \", request_contents.fetch1(\"is_archival\"))\n",
+ "contents = db_lightsheet.Request.ImagingResolutionRequest() & {\n",
+ " \"username\": username1,\n",
+ " \"request_name\": request_name1,\n",
+ "}\n",
"contents.fetch(as_dict=True)"
]
},
@@ -4550,24 +4582,42 @@
}
],
"source": [
- "username2 = 'ejdennis' # change as needed\n",
- "request_name2 = 'three_female_atlas_brains' # change as needed\n",
- "sample_contents = db_lightsheet.Request.Sample() & \\\n",
- " {'username':username2,'request_name':request_name2}\n",
- "print(f\"Have {len(sample_contents)} samples in this request:\\n {sample_contents.fetch('sample_name')}\")\n",
- "clearing_batch_contents = db_lightsheet.Request.ClearingBatch() & \\\n",
- " {'username':username2,'request_name':request_name2}\n",
- "print(f\"Have {len(clearing_batch_contents)} clearing batches in this request, each cleared by: {clearing_batch_contents.fetch('clearer')}\")\n",
- "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & \\\n",
- " {'username':username2,'request_name':request_name2}\n",
- "imagers = imaging_request_contents.fetch('imager')\n",
- "print(f\"Samples each imaged by: {imagers} \",)\n",
- "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & \\\n",
- " {'username':username2,'request_name':request_name2}\n",
- "processing_request_contents = db_lightsheet.Request.ProcessingRequest() & \\\n",
- " {'username':username2,'request_name':request_name2}\n",
- "processors = processing_request_contents.fetch('processor')\n",
- "print(f\"Samples each processed by: {processors} \",)"
+ "username2 = \"ejdennis\" # change as needed\n",
+ "request_name2 = \"three_female_atlas_brains\" # change as needed\n",
+ "sample_contents = db_lightsheet.Request.Sample() & {\n",
+ " \"username\": username2,\n",
+ " \"request_name\": request_name2,\n",
+ "}\n",
+ "print(\n",
+ " f\"Have {len(sample_contents)} samples in this request:\\n {sample_contents.fetch('sample_name')}\"\n",
+ ")\n",
+ "clearing_batch_contents = db_lightsheet.Request.ClearingBatch() & {\n",
+ " \"username\": username2,\n",
+ " \"request_name\": request_name2,\n",
+ "}\n",
+ "print(\n",
+ " f\"Have {len(clearing_batch_contents)} clearing batches in this request, each cleared by: {clearing_batch_contents.fetch('clearer')}\"\n",
+ ")\n",
+ "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & {\n",
+ " \"username\": username2,\n",
+ " \"request_name\": request_name2,\n",
+ "}\n",
+ "imagers = imaging_request_contents.fetch(\"imager\")\n",
+ "print(\n",
+ " f\"Samples each imaged by: {imagers} \",\n",
+ ")\n",
+ "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & {\n",
+ " \"username\": username2,\n",
+ " \"request_name\": request_name2,\n",
+ "}\n",
+ "processing_request_contents = db_lightsheet.Request.ProcessingRequest() & {\n",
+ " \"username\": username2,\n",
+ " \"request_name\": request_name2,\n",
+ "}\n",
+ "processors = processing_request_contents.fetch(\"processor\")\n",
+ "print(\n",
+ " f\"Samples each processed by: {processors} \",\n",
+ ")"
]
},
{
@@ -4597,22 +4647,32 @@
}
],
"source": [
- "username3 = 'ejdennis' # change as needed\n",
- "request_name3 = 'three_female_atlas_brains' # change as needed\n",
- "sample_contents = db_lightsheet.Request.Sample() & \\\n",
- " {'username':username3,'request_name':request_name3}\n",
- "print(f\"Have {len(sample_contents)} samples in this request:\\n {sample_contents.fetch('sample_name')}\\n\")\n",
- "clearing_batch_contents = db_lightsheet.Request.ClearingBatch() & \\\n",
- " {'username':username3,'request_name':request_name3}\n",
- "clearing_progress = clearing_batch_contents.fetch1('clearing_progress')\n",
- "print(\"Clearing progress is: \",clearing_progress)\n",
- "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & \\\n",
- " {'username':username3,'request_name':request_name3}\n",
- "imaging_progress = imaging_request_contents.fetch('imaging_progress')\n",
+ "username3 = \"ejdennis\" # change as needed\n",
+ "request_name3 = \"three_female_atlas_brains\" # change as needed\n",
+ "sample_contents = db_lightsheet.Request.Sample() & {\n",
+ " \"username\": username3,\n",
+ " \"request_name\": request_name3,\n",
+ "}\n",
+ "print(\n",
+ " f\"Have {len(sample_contents)} samples in this request:\\n {sample_contents.fetch('sample_name')}\\n\"\n",
+ ")\n",
+ "clearing_batch_contents = db_lightsheet.Request.ClearingBatch() & {\n",
+ " \"username\": username3,\n",
+ " \"request_name\": request_name3,\n",
+ "}\n",
+ "clearing_progress = clearing_batch_contents.fetch1(\"clearing_progress\")\n",
+ "print(\"Clearing progress is: \", clearing_progress)\n",
+ "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & {\n",
+ " \"username\": username3,\n",
+ " \"request_name\": request_name3,\n",
+ "}\n",
+ "imaging_progress = imaging_request_contents.fetch(\"imaging_progress\")\n",
"print(f\"Imaging progress is: {imaging_progress}\")\n",
- "processing_request_contents = db_lightsheet.Request.ProcessingRequest() & \\\n",
- " {'username':username3,'request_name':request_name3}\n",
- "processing_progress = processing_request_contents.fetch('processing_progress')\n",
+ "processing_request_contents = db_lightsheet.Request.ProcessingRequest() & {\n",
+ " \"username\": username3,\n",
+ " \"request_name\": request_name3,\n",
+ "}\n",
+ "processing_progress = processing_request_contents.fetch(\"processing_progress\")\n",
"print(f\"Processing progress is: {processing_progress}\")"
]
},
@@ -4647,16 +4707,23 @@
}
],
"source": [
- "username4 = 'ejdennis' # change as needed\n",
- "request_name4 = 'three_female_atlas_brains' # change as needed\n",
- "sample_contents = db_lightsheet.Request.Sample() & \\\n",
- " {'username':username4,'request_name':request_name4}\n",
+ "username4 = \"ejdennis\" # change as needed\n",
+ "request_name4 = \"three_female_atlas_brains\" # change as needed\n",
+ "sample_contents = db_lightsheet.Request.Sample() & {\n",
+ " \"username\": username4,\n",
+ " \"request_name\": request_name4,\n",
+ "}\n",
"print(f\"Have {len(sample_contents)} samples in this request\\n\")\n",
"for sample_dict in sample_contents:\n",
- " sample_name = sample_dict['sample_name']\n",
- " imaging_channel_contents = db_lightsheet.Request.ImagingChannel() & \\\n",
- " {'username':username4,'request_name':request_name4,'sample_name':sample_name}\n",
- " image_resolutions,channel_names = imaging_channel_contents.fetch('image_resolution','channel_name')\n",
+ " sample_name = sample_dict[\"sample_name\"]\n",
+ " imaging_channel_contents = db_lightsheet.Request.ImagingChannel() & {\n",
+ " \"username\": username4,\n",
+ " \"request_name\": request_name4,\n",
+ " \"sample_name\": sample_name,\n",
+ " }\n",
+ " image_resolutions, channel_names = imaging_channel_contents.fetch(\n",
+ " \"image_resolution\", \"channel_name\"\n",
+ " )\n",
" print(f\"Sample: {sample_name}:\")\n",
" for ii in range(len(image_resolutions)):\n",
" image_resolution = image_resolutions[ii]\n",
diff --git a/notebooks/water_weight_alert.ipynb b/notebooks/water_weight_alert.ipynb
index ff19cc31..015eb8c9 100644
--- a/notebooks/water_weight_alert.ipynb
+++ b/notebooks/water_weight_alert.ipynb
@@ -22,6 +22,7 @@
],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()"
]
},
@@ -40,13 +41,14 @@
}
],
"source": [
- "import datajoint as dj\n",
- "import pandas as pd\n",
- "import time\n",
- "from zoneinfo import ZoneInfo\n",
"import datetime\n",
"import pathlib\n",
- "import u19_pipeline.utils.slack_utils as su\n"
+ "import time\n",
+ "\n",
+ "import datajoint as dj\n",
+ "import pandas as pd\n",
+ "\n",
+ "import u19_pipeline.utils.slack_utils as su"
]
},
{
@@ -57,95 +59,132 @@
"source": [
"def get_subject_data():\n",
"\n",
- " #Query from file\n",
+ " # Query from file\n",
" current_directory = pathlib.Path.cwd()\n",
- " query_file = pathlib.Path(current_directory, 'useful_queries', 'get_subject_data.sql').as_posix()\n",
- "\n",
- " with open(query_file, \"r\", encoding=\"utf-8\") as file:\n",
+ " query_file = pathlib.Path(\n",
+ " current_directory, \"useful_queries\", \"get_subject_data.sql\"\n",
+ " ).as_posix()\n",
"\n",
+ " with open(query_file, encoding=\"utf-8\") as file:\n",
" subject_query = file.read()\n",
"\n",
" conn = dj.conn()\n",
" subject_data = pd.DataFrame(conn.query(subject_query, as_dict=True).fetchall())\n",
"\n",
- "\n",
- "\n",
- " #Get subject fullname only once\n",
- " subject_data = subject_data.loc[:,~subject_data.columns.duplicated()]\n",
- " #subject_data = subject_data.rename({'new_subject_fullname': 'subject_fullname'}, axis=1)\n",
+ " # Get subject fullname only once\n",
+ " subject_data = subject_data.loc[:, ~subject_data.columns.duplicated()]\n",
+ " # subject_data = subject_data.rename({'new_subject_fullname': 'subject_fullname'}, axis=1)\n",
"\n",
" # Get column list\n",
" cols = subject_data.columns.tolist()\n",
"\n",
" # Remove and insert the column\n",
- " cols.insert(0, cols.pop(cols.index('subject_fullname')))\n",
+ " cols.insert(0, cols.pop(cols.index(\"subject_fullname\")))\n",
" subject_data = subject_data.reindex(columns=cols)\n",
"\n",
- "\n",
- " #Get today's responsibility based on schedule \"Nothing/Nothing etc\" field\n",
- " today_idx = (datetime.datetime.today().weekday() + 1) % 7\n",
- " subject_data['schedule_today'] = subject_data['schedule'].str.split('/')\n",
- " subject_data['schedule_today'] = subject_data['schedule_today'].apply(lambda x: x[today_idx])\n",
- "\n",
- "\n",
- "\n",
- " subject_data = subject_data.loc[(((subject_data['schedule_today'] != \"Nothing\") &\\\n",
- " (subject_data['subject_status'] == \"InExperiments\")) |\\\n",
- " (subject_data['subject_status'] == \"WaterRestrictionOnly\")), :]\n",
+ " # Get today's responsibility based on schedule \"Nothing/Nothing etc\" field\n",
+ " today_idx = (datetime.datetime.today().weekday() + 1) % 7\n",
+ " subject_data[\"schedule_today\"] = subject_data[\"schedule\"].str.split(\"/\")\n",
+ " subject_data[\"schedule_today\"] = subject_data[\"schedule_today\"].apply(\n",
+ " lambda x: x[today_idx]\n",
+ " )\n",
+ "\n",
+ " subject_data = subject_data.loc[\n",
+ " (\n",
+ " (\n",
+ " (subject_data[\"schedule_today\"] != \"Nothing\")\n",
+ " & (subject_data[\"subject_status\"] == \"InExperiments\")\n",
+ " )\n",
+ " | (subject_data[\"subject_status\"] == \"WaterRestrictionOnly\")\n",
+ " ),\n",
+ " :,\n",
+ " ]\n",
"\n",
" subject_data = subject_data.reset_index(drop=True)\n",
"\n",
- "\n",
- "\n",
" with pd.option_context(\"future.no_silent_downcasting\", True):\n",
- "\n",
- " subject_data['earned'] = subject_data['earned'].fillna(0).infer_objects(copy=False)\n",
- " subject_data['received'] = subject_data['received'].fillna(0).infer_objects(copy=False)\n",
- " subject_data['supplement'] = subject_data['supplement'].fillna(0).infer_objects(copy=False)\n",
- " subject_data['prescribed_extra_supplement_amount'] = subject_data['prescribed_extra_supplement_amount'].fillna(0).infer_objects(copy=False)\n",
- "\n",
- "\n",
- " #Calculated fields \n",
- " subject_data['already_water'] = subject_data['supplement']>0\n",
- " subject_data['upper_cage'] = subject_data['cage'].str.upper()\n",
- " subject_data['total_water_received'] = subject_data['received']\n",
- " subject_data.loc[subject_data['total_water_received'].isnull(), 'total_water_received'] = 0\n",
- "\n",
- " subject_data['need_supplement'] = 0\n",
- " subject_data['need_supplement'] = (subject_data['total_water_received'] < (subject_data['water_per_day'] -0.05)) & ~subject_data['already_water']\n",
- "\n",
- " subject_data.loc[subject_data['already_received'].isnull(), 'already_received'] = 0\n",
- " subject_data['need_extra_water_now'] = 0\n",
- " subject_data.loc[((subject_data['prescribed_extra_supplement_amount'] > 0) & (subject_data['already_received'] == 0)),'need_extra_water_now'] = 1\n",
- "\n",
- " subject_data['water_status'] = \"Already Watered\"\n",
- " subject_data.loc[subject_data['need_supplement'] == 1, 'water_status'] = \"Need Supplement\"\n",
- " subject_data.loc[subject_data['need_extra_water_now'] == 1, 'water_status'] = \"Need Extra Supplement\"\n",
- "\n",
- " subject_data['need_water'] = 0\n",
- " subject_data.loc[(subject_data['need_supplement'] | subject_data['need_extra_water_now']), 'need_water'] = 1\n",
- "\n",
- " subject_data['current_need_water'] = subject_data['suggested_water']\n",
- " subject_data.loc[subject_data['need_extra_water_now']==1, 'current_need_water'] =\\\n",
- " subject_data.loc[subject_data['need_extra_water_now']==1, 'prescribed_extra_supplement_amount']\n",
- "\n",
- "\n",
- " subject_data['training_status'] = 0\n",
- " subject_data.loc[~subject_data['scheduled_rig'].isnull(), 'training_status'] = 1\n",
- " subject_data.loc[~subject_data['session_location'].isnull(), 'training_status'] = 2\n",
- "\n",
- " subject_data['training_status_label'] = \"Not scheduled\"\n",
- " subject_data.loc[subject_data['training_status'] == 1, 'training_status_label'] = \"Scheduled\"\n",
- " subject_data.loc[subject_data['training_status'] == 2, 'training_status_label'] = \"Training Started\"\n",
- "\n",
- " #Calculated fields \n",
- " subject_data['already_weighted'] = ~subject_data['weight'].isnull()\n",
- " subject_data['need_weight'] = ~subject_data['already_weighted'] | subject_data['need_reweight']\n",
- "\n",
- " subject_data['weight_status'] = \"Already Weighted\"\n",
- " subject_data.loc[subject_data['already_weighted'] == 0, 'weight_status'] = \"Need Weight\"\n",
- " subject_data.loc[subject_data['need_reweight'] == 1, 'weight_status'] = \"REWEIGHT\"\n",
- "\n",
+ " subject_data[\"earned\"] = (\n",
+ " subject_data[\"earned\"].fillna(0).infer_objects(copy=False)\n",
+ " )\n",
+ " subject_data[\"received\"] = (\n",
+ " subject_data[\"received\"].fillna(0).infer_objects(copy=False)\n",
+ " )\n",
+ " subject_data[\"supplement\"] = (\n",
+ " subject_data[\"supplement\"].fillna(0).infer_objects(copy=False)\n",
+ " )\n",
+ " subject_data[\"prescribed_extra_supplement_amount\"] = (\n",
+ " subject_data[\"prescribed_extra_supplement_amount\"]\n",
+ " .fillna(0)\n",
+ " .infer_objects(copy=False)\n",
+ " )\n",
+ "\n",
+ " # Calculated fields\n",
+ " subject_data[\"already_water\"] = subject_data[\"supplement\"] > 0\n",
+ " subject_data[\"upper_cage\"] = subject_data[\"cage\"].str.upper()\n",
+ " subject_data[\"total_water_received\"] = subject_data[\"received\"]\n",
+ " subject_data.loc[\n",
+ " subject_data[\"total_water_received\"].isnull(), \"total_water_received\"\n",
+ " ] = 0\n",
+ "\n",
+ " subject_data[\"need_supplement\"] = 0\n",
+ " subject_data[\"need_supplement\"] = (\n",
+ " subject_data[\"total_water_received\"] < (subject_data[\"water_per_day\"] - 0.05)\n",
+ " ) & ~subject_data[\"already_water\"]\n",
+ "\n",
+ " subject_data.loc[subject_data[\"already_received\"].isnull(), \"already_received\"] = 0\n",
+ " subject_data[\"need_extra_water_now\"] = 0\n",
+ " subject_data.loc[\n",
+ " (\n",
+ " (subject_data[\"prescribed_extra_supplement_amount\"] > 0)\n",
+ " & (subject_data[\"already_received\"] == 0)\n",
+ " ),\n",
+ " \"need_extra_water_now\",\n",
+ " ] = 1\n",
+ "\n",
+ " subject_data[\"water_status\"] = \"Already Watered\"\n",
+ " subject_data.loc[subject_data[\"need_supplement\"] == 1, \"water_status\"] = (\n",
+ " \"Need Supplement\"\n",
+ " )\n",
+ " subject_data.loc[subject_data[\"need_extra_water_now\"] == 1, \"water_status\"] = (\n",
+ " \"Need Extra Supplement\"\n",
+ " )\n",
+ "\n",
+ " subject_data[\"need_water\"] = 0\n",
+ " subject_data.loc[\n",
+ " (subject_data[\"need_supplement\"] | subject_data[\"need_extra_water_now\"]),\n",
+ " \"need_water\",\n",
+ " ] = 1\n",
+ "\n",
+ " subject_data[\"current_need_water\"] = subject_data[\"suggested_water\"]\n",
+ " subject_data.loc[\n",
+ " subject_data[\"need_extra_water_now\"] == 1, \"current_need_water\"\n",
+ " ] = subject_data.loc[\n",
+ " subject_data[\"need_extra_water_now\"] == 1, \"prescribed_extra_supplement_amount\"\n",
+ " ]\n",
+ "\n",
+ " subject_data[\"training_status\"] = 0\n",
+ " subject_data.loc[~subject_data[\"scheduled_rig\"].isnull(), \"training_status\"] = 1\n",
+ " subject_data.loc[~subject_data[\"session_location\"].isnull(), \"training_status\"] = 2\n",
+ "\n",
+ " subject_data[\"training_status_label\"] = \"Not scheduled\"\n",
+ " subject_data.loc[subject_data[\"training_status\"] == 1, \"training_status_label\"] = (\n",
+ " \"Scheduled\"\n",
+ " )\n",
+ " subject_data.loc[subject_data[\"training_status\"] == 2, \"training_status_label\"] = (\n",
+ " \"Training Started\"\n",
+ " )\n",
+ "\n",
+ " # Calculated fields\n",
+ " subject_data[\"already_weighted\"] = ~subject_data[\"weight\"].isnull()\n",
+ " subject_data[\"need_weight\"] = (\n",
+ " ~subject_data[\"already_weighted\"] | subject_data[\"need_reweight\"]\n",
+ " )\n",
+ "\n",
+ " subject_data[\"weight_status\"] = \"Already Weighted\"\n",
+ " subject_data.loc[subject_data[\"already_weighted\"] == 0, \"weight_status\"] = (\n",
+ " \"Need Weight\"\n",
+ " )\n",
+ " subject_data.loc[subject_data[\"need_reweight\"] == 1, \"weight_status\"] = \"REWEIGHT\"\n",
"\n",
" return subject_data"
]
@@ -573,7 +612,7 @@
}
],
"source": [
- "pd.set_option('display.max_rows', 20)\n",
+ "pd.set_option(\"display.max_rows\", 20)\n",
"subject_data"
]
},
@@ -652,15 +691,25 @@
}
],
"source": [
- "subject_data = subject_data.loc[~subject_data['subject_fullname'].str.contains('test'),:]\n",
- "subjects_not_watered = subject_data.loc[subject_data['current_need_water'] > 0, ['subject_fullname', 'current_need_water']]\n",
+ "subject_data = subject_data.loc[\n",
+ " ~subject_data[\"subject_fullname\"].str.contains(\"test\"), :\n",
+ "]\n",
+ "subjects_not_watered = subject_data.loc[\n",
+ " subject_data[\"current_need_water\"] > 0, [\"subject_fullname\", \"current_need_water\"]\n",
+ "]\n",
"subjects_not_watered = subjects_not_watered.reset_index(drop=True)\n",
- "subjects_not_watered['current_need_water'] = subjects_not_watered['current_need_water'].apply(lambda x: f\"{x:.1f}\")\n",
+ "subjects_not_watered[\"current_need_water\"] = subjects_not_watered[\n",
+ " \"current_need_water\"\n",
+ "].apply(lambda x: f\"{x:.1f}\")\n",
"subjects_not_watered = subjects_not_watered.head()\n",
- "subjects_not_weighted = subject_data.loc[subject_data['need_weight'], ['subject_fullname', 'need_weight']]\n",
+ "subjects_not_weighted = subject_data.loc[\n",
+ " subject_data[\"need_weight\"], [\"subject_fullname\", \"need_weight\"]\n",
+ "]\n",
"subjects_not_weighted = subjects_not_weighted.reset_index(drop=True)\n",
"subjects_not_weighted = subjects_not_weighted.head()\n",
- "subjects_not_trained = subject_data.loc[subject_data['training_status']==1, ['subject_fullname', 'scheduled_rig']]\n",
+ "subjects_not_trained = subject_data.loc[\n",
+ " subject_data[\"training_status\"] == 1, [\"subject_fullname\", \"scheduled_rig\"]\n",
+ "]\n",
"subjects_not_trained = subjects_not_trained.reset_index(drop=True)\n",
"subjects_not_trained = subjects_not_trained.head()\n",
"subjects_not_trained"
@@ -680,13 +729,16 @@
}
],
"source": [
- "lab = dj.create_virtual_module('lab', 'u19_lab')\n",
- "slack_configuration_dictionary = {\n",
- " 'slack_notification_channel': ['alvaro_luna']\n",
- "}\n",
+ "lab = dj.create_virtual_module(\"lab\", \"u19_lab\")\n",
+ "slack_configuration_dictionary = {\"slack_notification_channel\": [\"alvaro_luna\"]}\n",
"webhooks_list = []\n",
- "query_slack_webhooks = [{'webhook_name' : x} for x in slack_configuration_dictionary['slack_notification_channel']]\n",
- "webhooks_list += (lab.SlackWebhooks & query_slack_webhooks).fetch('webhook_url').tolist()"
+ "query_slack_webhooks = [\n",
+ " {\"webhook_name\": x}\n",
+ " for x in slack_configuration_dictionary[\"slack_notification_channel\"]\n",
+ "]\n",
+ "webhooks_list += (\n",
+ " (lab.SlackWebhooks & query_slack_webhooks).fetch(\"webhook_url\").tolist()\n",
+ ")"
]
},
{
@@ -695,58 +747,72 @@
"metadata": {},
"outputs": [],
"source": [
- "def slack_alert_message_format_weight_water(subjects_not_watered, subjects_not_weighted, subjects_not_trained):\n",
+ "def slack_alert_message_format_weight_water(\n",
+ " subjects_not_watered, subjects_not_weighted, subjects_not_trained\n",
+ "):\n",
"\n",
" now = datetime.datetime.now()\n",
- " datestr = now.strftime('%d-%b-%Y %H:%M:%S')\n",
+ " datestr = now.strftime(\"%d-%b-%Y %H:%M:%S\")\n",
"\n",
" msep = dict()\n",
- " msep['type'] = \"divider\"\n",
+ " msep[\"type\"] = \"divider\"\n",
"\n",
- " #Title#\n",
+ " # Title#\n",
" m1 = dict()\n",
- " m1['type'] = 'section'\n",
+ " m1[\"type\"] = \"section\"\n",
" m1_1 = dict()\n",
" m1_1[\"type\"] = \"mrkdwn\"\n",
- " m1_1[\"text\"] = ':rotating_light: * Subjects Status Alert *'\n",
- " m1['text'] = m1_1\n",
+ " m1_1[\"text\"] = \":rotating_light: * Subjects Status Alert *\"\n",
+ " m1[\"text\"] = m1_1\n",
"\n",
- " #Info#\n",
+ " # Info#\n",
" m2 = dict()\n",
- " m2['type'] = 'section'\n",
+ " m2[\"type\"] = \"section\"\n",
" m2_1 = dict()\n",
" m2_1[\"type\"] = \"mrkdwn\"\n",
"\n",
- " m2_1[\"text\"] = '*Subjects missing water:*' + '\\n'\n",
+ " m2_1[\"text\"] = \"*Subjects missing water:*\" + \"\\n\"\n",
" for i in range(subjects_not_watered.shape[0]):\n",
- " m2_1[\"text\"] += '*' + subjects_not_watered.loc[i, 'subject_fullname'] + '* : ' + str(subjects_not_watered.loc[i, 'current_need_water']) + ' ml\\n'\n",
- " #m2_1[\"text\"] += '\\n'\n",
- " m2['text'] = m2_1\n",
+ " m2_1[\"text\"] += (\n",
+ " \"*\"\n",
+ " + subjects_not_watered.loc[i, \"subject_fullname\"]\n",
+ " + \"* : \"\n",
+ " + str(subjects_not_watered.loc[i, \"current_need_water\"])\n",
+ " + \" ml\\n\"\n",
+ " )\n",
+ " # m2_1[\"text\"] += '\\n'\n",
+ " m2[\"text\"] = m2_1\n",
"\n",
" m4 = dict()\n",
- " m4['type'] = 'section'\n",
+ " m4[\"type\"] = \"section\"\n",
" m4_1 = dict()\n",
" m4_1[\"type\"] = \"mrkdwn\"\n",
"\n",
- " m4_1[\"text\"] = '*Subjects missing weighing:*' + '\\n'\n",
+ " m4_1[\"text\"] = \"*Subjects missing weighing:*\" + \"\\n\"\n",
" for i in range(subjects_not_weighted.shape[0]):\n",
- " m4_1[\"text\"] += '*' + subjects_not_weighted.loc[i, 'subject_fullname'] + '*\\n' \n",
- " m4['text'] = m4_1\n",
+ " m4_1[\"text\"] += \"*\" + subjects_not_weighted.loc[i, \"subject_fullname\"] + \"*\\n\"\n",
+ " m4[\"text\"] = m4_1\n",
"\n",
" m5 = dict()\n",
- " m5['type'] = 'section'\n",
+ " m5[\"type\"] = \"section\"\n",
" m5_1 = dict()\n",
" m5_1[\"type\"] = \"mrkdwn\"\n",
"\n",
- " m5_1[\"text\"] = '*Subjects missing training:*' + '\\n'\n",
+ " m5_1[\"text\"] = \"*Subjects missing training:*\" + \"\\n\"\n",
" for i in range(subjects_not_trained.shape[0]):\n",
- " m5_1[\"text\"] += '*' + subjects_not_trained.loc[i, 'subject_fullname'] + '* : ' + subjects_not_trained.loc[i, 'scheduled_rig'] + '\\n'\n",
+ " m5_1[\"text\"] += (\n",
+ " \"*\"\n",
+ " + subjects_not_trained.loc[i, \"subject_fullname\"]\n",
+ " + \"* : \"\n",
+ " + subjects_not_trained.loc[i, \"scheduled_rig\"]\n",
+ " + \"\\n\"\n",
+ " )\n",
" #\n",
- " m5['text'] = m5_1\n",
+ " m5[\"text\"] = m5_1\n",
"\n",
" message = dict()\n",
- " message['blocks'] = [m1,msep,m2,msep,m4,msep,m5,msep]\n",
- " message['text'] = 'Subject Status Alert'\n",
+ " message[\"blocks\"] = [m1, msep, m2, msep, m4, msep, m5, msep]\n",
+ " message[\"text\"] = \"Subject Status Alert\"\n",
"\n",
" return message"
]
@@ -757,13 +823,15 @@
"metadata": {},
"outputs": [],
"source": [
- "slack_json_message = slack_alert_message_format_weight_water(subjects_not_watered, subjects_not_weighted, subjects_not_trained)\n",
+ "slack_json_message = slack_alert_message_format_weight_water(\n",
+ " subjects_not_watered, subjects_not_weighted, subjects_not_trained\n",
+ ")\n",
"\n",
"\n",
- "#Send alert\n",
+ "# Send alert\n",
"for this_webhook in webhooks_list:\n",
" su.send_slack_notification(this_webhook, slack_json_message)\n",
- " time.sleep(1)\n"
+ " time.sleep(1)"
]
},
{
diff --git a/notebooks/xyz_picks_notebook.ipynb b/notebooks/xyz_picks_notebook.ipynb
index 06d2bdc2..5d362a0e 100644
--- a/notebooks/xyz_picks_notebook.ipynb
+++ b/notebooks/xyz_picks_notebook.ipynb
@@ -14,17 +14,13 @@
"outputs": [],
"source": [
"from scripts.conf_file_finding import try_find_conf_file\n",
+ "\n",
"try_find_conf_file()\n",
"\n",
- "import pandas as pd\n",
- "import datajoint as dj\n",
"import numpy as np\n",
"import pylab as pl\n",
- "from scipy.io import loadmat\n",
- "from scipy.spatial.transform import Rotation as R\n",
"\n",
- "from u19_pipeline.utils import ephys_utils\n",
- "\n"
+ "from u19_pipeline.utils import ephys_utils"
]
},
{
@@ -33,19 +29,21 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
"recording_id = 81\n",
- "chanmaps_jobs = ['298','299','300','301']\n",
+ "chanmaps_jobs = [\"298\", \"299\", \"300\", \"301\"]\n",
"\n",
"list_probes = list()\n",
"for idx in range(4):\n",
" job_id = chanmaps_jobs[idx]\n",
- " procesed_data_directory = f'/Volumes/braininit/Data/Processed/electrophysiology/ms81/ms81_M019/20220611/TowersTask_g0/TowersTask_g0_imec{idx}/job_id_{job_id}'\n",
- " chanmap_file = f'/Volumes/braininit//Shared/TestData/ChanMapFiles/chanmap_{job_id}.mat'\n",
- " list_probes.append(ephys_utils.xyz_pick_file_creator.main_xyz_pick_file_function(recording_id, idx, chanmap_file, procesed_data_directory))\n",
- "\n",
- "\n",
- "\n"
+ " procesed_data_directory = f\"/Volumes/braininit/Data/Processed/electrophysiology/ms81/ms81_M019/20220611/TowersTask_g0/TowersTask_g0_imec{idx}/job_id_{job_id}\"\n",
+ " chanmap_file = (\n",
+ " f\"/Volumes/braininit//Shared/TestData/ChanMapFiles/chanmap_{job_id}.mat\"\n",
+ " )\n",
+ " list_probes.append(\n",
+ " ephys_utils.xyz_pick_file_creator.main_xyz_pick_file_function(\n",
+ " recording_id, idx, chanmap_file, procesed_data_directory\n",
+ " )\n",
+ " )"
]
},
{
@@ -69,26 +67,26 @@
"source": [
"import matplotlib.pyplot as pl\n",
"\n",
- "fig = pl.figure(figsize = (12,12))\n",
- "ax = fig.add_subplot(111,projection='3d')\n",
+ "fig = pl.figure(figsize=(12, 12))\n",
+ "ax = fig.add_subplot(111, projection=\"3d\")\n",
"for i in range(4):\n",
" this_probe_data = list_probes[i]\n",
- " for this_shank_data in this_probe_data: \n",
- " pt=np.array([np.array(xi) for xi in this_shank_data])\n",
- " ax.scatter(pt[:,0], pt[:,1], pt[:,2])\n",
+ " for this_shank_data in this_probe_data:\n",
+ " pt = np.array([np.array(xi) for xi in this_shank_data])\n",
+ " ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2])\n",
"\n",
"\n",
"# Just for plotting:\n",
- "#xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n",
- "#ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n",
- "ax.scatter(xs=0, ys=0, zs=0, c='k')\n",
- "ax.set_xlim(-5000,5000)\n",
- "ax.set_ylim(-5000,5000)\n",
- "ax.set_zlim(-5000,0)\n",
+ "# xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n",
+ "# ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n",
+ "ax.scatter(xs=0, ys=0, zs=0, c=\"k\")\n",
+ "ax.set_xlim(-5000, 5000)\n",
+ "ax.set_ylim(-5000, 5000)\n",
+ "ax.set_zlim(-5000, 0)\n",
"ax.set_xlabel(\"x | ML\")\n",
"ax.set_ylabel(\"y | AP\")\n",
"ax.set_zlabel(\"z | DV\")\n",
- "ax.view_init(elev=90., azim=90)\n",
+ "ax.view_init(elev=90.0, azim=90)\n",
"\n",
"pl.show()"
]
@@ -114,22 +112,22 @@
"source": [
"import matplotlib.pyplot as pl\n",
"\n",
- "fig = pl.figure(figsize = (12,12))\n",
- "ax = fig.add_subplot(111,projection='3d')\n",
+ "fig = pl.figure(figsize=(12, 12))\n",
+ "ax = fig.add_subplot(111, projection=\"3d\")\n",
"for i in range(4):\n",
" this_probe_data = list_probes[i]\n",
- " for this_shank_data in this_probe_data: \n",
- " pt=np.array([np.array(xi) for xi in this_shank_data])\n",
- " ax.scatter(pt[:,0], pt[:,1], pt[:,2])\n",
+ " for this_shank_data in this_probe_data:\n",
+ " pt = np.array([np.array(xi) for xi in this_shank_data])\n",
+ " ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2])\n",
"\n",
"\n",
"# Just for plotting:\n",
- "#xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n",
- "#ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n",
- "ax.scatter(xs=0, ys=0, zs=0, c='k')\n",
- "ax.set_xlim(-2500,2500)\n",
- "ax.set_ylim(-2500,2500)\n",
- "ax.set_zlim(-5000,0)\n",
+ "# xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n",
+ "# ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n",
+ "ax.scatter(xs=0, ys=0, zs=0, c=\"k\")\n",
+ "ax.set_xlim(-2500, 2500)\n",
+ "ax.set_ylim(-2500, 2500)\n",
+ "ax.set_zlim(-5000, 0)\n",
"ax.set_xlabel(\"x | ML\")\n",
"ax.set_ylabel(\"y | AP\")\n",
"ax.set_zlabel(\"z | DV\")\n",
@@ -157,22 +155,22 @@
"source": [
"import matplotlib.pyplot as pl\n",
"\n",
- "fig = pl.figure(figsize = (12,12))\n",
- "ax = fig.add_subplot(111,projection='3d')\n",
+ "fig = pl.figure(figsize=(12, 12))\n",
+ "ax = fig.add_subplot(111, projection=\"3d\")\n",
"for i in range(4):\n",
" this_probe_data = list_probes[i]\n",
- " for this_shank_data in this_probe_data: \n",
- " pt=np.array([np.array(xi) for xi in this_shank_data])\n",
- " ax.scatter(pt[:,0], pt[:,1], pt[:,2])\n",
+ " for this_shank_data in this_probe_data:\n",
+ " pt = np.array([np.array(xi) for xi in this_shank_data])\n",
+ " ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2])\n",
"\n",
"\n",
"# Just for plotting:\n",
- "#xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n",
- "#ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n",
- "ax.scatter(xs=0, ys=0, zs=0, c='k')\n",
- "ax.set_xlim(-2500,2500)\n",
- "ax.set_ylim(-2500,2500)\n",
- "ax.set_zlim(-5000,0)\n",
+ "# xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n",
+ "# ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n",
+ "ax.scatter(xs=0, ys=0, zs=0, c=\"k\")\n",
+ "ax.set_xlim(-2500, 2500)\n",
+ "ax.set_ylim(-2500, 2500)\n",
+ "ax.set_zlim(-5000, 0)\n",
"ax.set_xlabel(\"x | ML\")\n",
"ax.set_ylabel(\"y | AP\")\n",
"ax.set_zlabel(\"z | DV\")\n",
diff --git a/pyproject.toml b/pyproject.toml
index 138d6796..958dec98 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -21,7 +21,8 @@ dependencies = [
"scipy",
"tables>=3.8",
"astropy>=6.0",
- "openpyxl"
+ "openpyxl",
+ "rich>=15.0.0",
]
requires-python = ">=3.12"
@@ -64,8 +65,32 @@ include = [
]
[tool.ruff]
-# Allow lines to be as long as 120.
-line-length = 120
+# if this is a library, enter the _minimum_ version you
+# want to support, otherwise do py313
+target-version = "py312"
+line-length = 120 # use whatever number makes you happy
+extend-exclude = ["*.ipynb", "**/*.ipynb"]
+
+[tool.ruff.lint]
+# you can see the looong list of rules here:
+# https://docs.astral.sh/ruff/rules/
+# here's a couple to start with
+select = [
+ "A", # warn about shadowing built-ins
+ "E", # style stuff, whitespaces
+ "F", # important pyflakes lints
+ "I", # import sorting
+ "N", # naming
+ "T100", # breakpoints (probably don't want these in prod!)
+ "B", # flake8-bugbear: likely bugs and design problems
+ "SIM", # flake8-simplify: code that can be simplified
+ "UP", # pyupgrade: syntax that can be modernized (e.g., old Python idioms)
+ "C4", # flake8-comprehensions: better list/dict/set comprehension patterns
+]
+# if you're feeling confident you can do:
+# select = ["ALL"]
+# and then manually ignore annoying ones:
+ignore = ["N815", "A002", "N802", "N803", "N806", "E402", "E501", "SIM108"]
[dependency-groups]
dev = [
diff --git a/ruff.toml b/ruff.toml
deleted file mode 100644
index 6510f6fe..00000000
--- a/ruff.toml
+++ /dev/null
@@ -1,20 +0,0 @@
-[lint]
-select = [
- # pycodestyle
- "E",
- # Pyflakes
- "F",
- # pyupgrade
- "UP",
- # flake8-bugbear
- "B",
- # flake8-simplify
- "SIM",
- # isort
- "I",
-]
-
-ignore = [
- "E501"
-]
-
diff --git a/scripts/__init__.py b/scripts/__init__.py
index e1424ed0..260c070a 100644
--- a/scripts/__init__.py
+++ b/scripts/__init__.py
@@ -1 +1 @@
-__version__ = '0.3.1'
+__version__ = "0.3.1"
diff --git a/scripts/conf_file_finding.py b/scripts/conf_file_finding.py
index 77adfbb0..61fe31e5 100644
--- a/scripts/conf_file_finding.py
+++ b/scripts/conf_file_finding.py
@@ -1,6 +1,9 @@
-
import os
import pathlib
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
def chdir_to_root():
@@ -8,16 +11,15 @@ def chdir_to_root():
conf_file_found = 0
starting_directory = os.getcwd()
while 1:
-
current_dir = os.getcwd()
- u19_dir = pathlib.Path(current_dir,'u19_pipeline')
+ u19_dir = pathlib.Path(current_dir, "u19_pipeline")
if os.path.isdir(u19_dir):
root_dir_found = 1
- if os.path.isfile(pathlib.Path(current_dir,'dj_local_conf.json')):
+ if os.path.isfile(pathlib.Path(current_dir, "dj_local_conf.json")):
conf_file_found = 1
if root_dir_found:
break
- os.chdir('..')
+ os.chdir("..")
new_current_dir = os.getcwd()
if str(current_dir) == str(new_current_dir):
os.chdir(starting_directory)
@@ -25,14 +27,14 @@ def chdir_to_root():
return root_dir_found, conf_file_found
+
def get_root_directory():
root_dir_found = 0
current_dir = pathlib.Path(os.getcwd())
while 1:
-
- u19_dir = pathlib.Path(current_dir,'U19-pipeline_python')
- u19_dir2 = pathlib.Path(current_dir,'U19-pipeline-python')
+ u19_dir = pathlib.Path(current_dir, "U19-pipeline_python")
+ u19_dir2 = pathlib.Path(current_dir, "U19-pipeline-python")
if os.path.isdir(u19_dir) or os.path.isdir(u19_dir2):
root_dir_found = 1
break
@@ -48,10 +50,12 @@ def try_find_conf_file():
root_dir_found, conf_file_found = chdir_to_root()
if root_dir_found and conf_file_found:
- print('Local configuration file found !!, no need to run the configuration (unless configuration has changed)')
+ logger.info("Local configuration file found !!, no need to run the configuration (unless configuration has changed)")
elif root_dir_found:
- raise FileNotFoundError('Local configuration file not found. Ignore this if you have a global config. Run configuration script (initial_conf.py) otherwise')
- elif os.path.isfile(pathlib.Path(pathlib.Path.home(),".datajoint_config.json")):
- print("Global configuration file found !!, no need to run the configuration (unless configuration has changed)")
+ raise FileNotFoundError(
+ "Local configuration file not found. Ignore this if you have a global config. Run configuration script (initial_conf.py) otherwise"
+ )
+ elif os.path.isfile(pathlib.Path(pathlib.Path.home(), ".datajoint_config.json")):
+ logger.info("Global configuration file found !!, no need to run the configuration (unless configuration has changed)")
else:
- raise FileNotFoundError('Root dir not found, change this notebook to the project folder')
+ raise FileNotFoundError("Root dir not found, change this notebook to the project folder")
diff --git a/scripts/create_schemas.py b/scripts/create_schemas.py
index 1e05e21d..e69de29b 100644
--- a/scripts/create_schemas.py
+++ b/scripts/create_schemas.py
@@ -1 +0,0 @@
-from u19_pipeline import lab, reference, subject, task, action, acquisition, behavior, puffs
diff --git a/scripts/drop_schemas.py b/scripts/drop_schemas.py
index 2cccf92e..96e4042d 100644
--- a/scripts/drop_schemas.py
+++ b/scripts/drop_schemas.py
@@ -1,6 +1,18 @@
-from u19_pipeline import lab, subject, acquisition, behavior, task, reference, recording, recording_process, ephys_pipeline, imaging_pipeline, ephys_sync
-from u19_pipeline.imaging_pipeline import scan_element, imaging_element
-from u19_pipeline.ephys_pipeline import probe_element, ephys_element
+from u19_pipeline import (
+ acquisition,
+ behavior,
+ ephys_pipeline,
+ ephys_sync,
+ imaging_pipeline,
+ lab,
+ recording,
+ recording_process,
+ reference,
+ subject,
+ task,
+)
+from u19_pipeline.ephys_pipeline import ephys_element, probe_element
+from u19_pipeline.imaging_pipeline import imaging_element, scan_element
behavior.schema.drop()
recording_process.schema.drop()
@@ -16,4 +28,4 @@
subject.schema.drop()
lab.schema.drop()
reference.schema.drop()
-task.schema.drop()
\ No newline at end of file
+task.schema.drop()
diff --git a/scripts/ingest_hdf5_to_db.py b/scripts/ingest_hdf5_to_db.py
index 4ec21b9e..acdffb15 100644
--- a/scripts/ingest_hdf5_to_db.py
+++ b/scripts/ingest_hdf5_to_db.py
@@ -4,475 +4,510 @@
# Folder structure for hdf5 files on bucket will be:
# /jukebox/braininit/puffs/{netid}/{project_name}/{cohort}/{rig}/{hdf5_filename}
+import glob
+import json
+import os
+import sys
+from datetime import timedelta
+
import datajoint as dj
-import os, sys, glob, json
-import pandas as pd
import numpy as np
-import time
-from datetime import datetime, timedelta
-
-dj.config['database.host'] = 'datajoint00.pni.princeton.edu'
-# load dj creds from file
-credfile = '/jukebox/wang/ahoag/.djenv'
-with open(credfile,'r') as infile:
- cred_dict = json.load(infile)
-
-dj.config['database.user'] = cred_dict.get('DJ_DB_USER')
-dj.config['database.password'] = cred_dict.get('DJ_DB_PASS')
-
-# Link to existing databases
-u19_lab = dj.create_virtual_module('u19_lab','u19_lab')
-u19_subject = dj.create_virtual_module('u19_subject','u19_subject')
-u19_acq = dj.create_virtual_module('u19_acquisition','u19_acquisition')
-
-# Link to new Puffs dbs
-u19_puffs = dj.create_virtual_module('u19_puffs','u19_puffs')
-
-# Currently a hardcoded data folder to one of Marlies' cohorts for this example,
-# but in the future will loop over folders starting from
-# The root folder: /jukebox/braininit/puffs/
-data_folder = '/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation/cohort_10/rig0/'
-h5_files = glob.glob(data_folder + '/data*.h5')
-mouse_info_file = os.path.join('/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation',
- 'mouse_info_0821_datajoint_format.csv')
-
-# first read in the mouse_info file into a pandas dataframe
-df_info = pd.read_csv(mouse_info_file,keep_default_na=False)
-
-# # Use mouse_info file and do the u19_subject and u19_lab inserts
-# Tables we need to insert into (**In this order**) are:
-# - u19_lab.User # in case user is not already in the database
-# - u19_lab.Protocol # in case protocol is not already in the database
-# - u19_lab.Location # in case the location of each subject is not already in the database
-#
-# - u19_subject.Species # in case they are NOT training on mice
-# - u19_subject.Strain # in case they are NOT training on a strain of mice already in the db
-# - u19_subject.Line # in case they are using a new genetic line
-# - u19_subject.Subject # in case they are using new subjects
-
-
-# u19_lab.User() inserts
-unique_netids = df_info['netid'].unique()
-full_names = df_info['experimenter_fullname']
-user_insert_list = []
-for netid in unique_netids:
- netid_mask = df_info['netid'] == netid
- full_name = full_names[netid_mask].iloc[0]
- user_insert_dict = {
- 'user_id': netid,
- 'user_nickname': netid,
- 'full_name': full_name,
- 'email': f'{netid}@princeton.edu',
- 'contact_via': 'Email',
- 'presence': 'Available',
- 'tech_responsibility': 'no',
- 'day_cutoff_time': np.array([[18., 0.]])
- }
- user_insert_list.append(user_insert_dict)
-u19_lab.User.insert(user_insert_list,skip_duplicates=True)
-
-# u19_lab.Protocol() inserts
-unique_protocols = df_info['protocol'].unique()
-protocol_insert_list = []
-for protocol in unique_protocols:
- protocol_insert_dict = {
- 'protocol': protocol,
- 'protocol_description': '',
- }
- protocol_insert_list.append(protocol_insert_dict)
-u19_lab.Protocol.insert(protocol_insert_list,skip_duplicates=True)
-
-
-# u19_lab.Location() inserts
-unique_locations = df_info['location'].unique()
-location_descriptions = df_info['location_description']
-location_insert_list = []
-for location in unique_locations:
- location_mask = df_info['location'] == location
- location_description = location_descriptions[netid_mask].iloc[0]
- location_insert_dict = {
- 'location': location,
- 'location_description': location_description,
- }
- location_insert_list.append(location_insert_dict)
-u19_lab.Location.insert(location_insert_list,skip_duplicates=True)
-
-# u19_subject.Species() inserts
-unique_species = df_info['binomial'].unique()
-species_insert_list = []
-for species in unique_species:
- species_insert_dict = {
- 'binomial': species,
- 'species_nickname': '',
- }
- species_insert_list.append(species_insert_dict)
-u19_subject.Species.insert(species_insert_list,skip_duplicates=True)
-
-# u19_subject.Strain() inserts
-unique_strains = df_info['strain'].unique()
-strain_insert_list = []
-for strain in unique_strains:
- strain_insert_dict = {
- 'strain_name': strain,
- 'strain_description': '',
- }
- strain_insert_list.append(strain_insert_dict)
-u19_subject.Strain.insert(strain_insert_list,skip_duplicates=True)
-
-# u19_subject.Line() inserts
-unique_lines = df_info['genetic_line'].unique()
-binomials = df_info['binomial']
-strains = df_info['strain']
-line_insert_list = []
-for line in unique_lines:
- line_mask = df_info['genetic_line'] == line
- binomial = binomials[line_mask].iloc[0]
- strain = strains[line_mask].iloc[0]
- line_insert_dict = {
- 'line': line,
- 'binomial': binomial,
- 'strain_name':strain,
- 'line_description':'',
- 'target_phenotype':'',
- 'is_active':1,
- }
- line_insert_list.append(line_insert_dict)
-u19_subject.Line.insert(line_insert_list,skip_duplicates=True)
-
-
-# u19_subject.Subject() inserts
-unique_subjects = df_info['subj'].unique()
-project_names = df_info['project_name']
-subject_descriptions = df_info['subj_description']
-netids = df_info['netid']
-sexes = df_info['sex']
-DOBs = df_info['DOB']
-locations = df_info['location']
-lines = df_info['genetic_line']
-protocols = df_info['protocol']
-subject_insert_list = []
-for subject in unique_subjects:
- subject_mask = df_info['subj'] == subject
- netid = netids[subject_mask].iloc[0]
- project_name = project_names[subject_mask].iloc[0]
- subject_fullname = '_'.join([netid,project_name,str(subject)])
- sex = sexes[subject_mask].iloc[0]
- DOB = DOBs[subject_mask].iloc[0]
- location = locations[subject_mask].iloc[0]
- line = lines[subject_mask].iloc[0]
- protocol = protocols[subject_mask].iloc[0]
- subject_description = subject_descriptions[subject_mask].iloc[0]
- subject_insert_dict = {
- 'subject_fullname': subject_fullname,
- 'subject_nickname': str(subject),
- 'subject_description': subject_description,
- 'user_id':netid,
- 'sex': 'Male' if sex == 'M' else 'Female',
- 'dob': DOB,
- 'location':location,
- 'line':line,
- 'protocol':protocol
- }
- subject_insert_list.append(subject_insert_dict)
-u19_subject.Subject.insert(subject_insert_list,skip_duplicates=True)
-
-
-# ## Now use h5 files to make inserts into u19_acquisition tables and puffs specific tables
-#
-# Tables we need to insert into (**In this order**) are:
-# - u19_puffs.PuffsCohort # in case cohort is not already in the database
-# - u19_puffs.PuffsFileAcquisition # as we process the various files
-#
-# - u19_acquisition.SessionStarted # as we process the session data in each h5 file
-# - u19_acquisition.Session # as we process the session data in each h5 file
-#
-# - u19_puffs.PuffsSession
-# - u19_puffs.PuffsSession.Trial
-# - u19_puffs.PuffsSession.TrialPhase
-# - u19_puffs.PuffsSession.Puff
-#
-
-# first find list of h5 files that are already processed so we do not
-# repeat the ingestion on those
-already_processed_filenames = (u19_puffs.PuffsFileAcquisition() & 'ingested=1').fetch(
- 'h5_filename')
-
-# Loop over h5 files in this cohort/rig folder
-for h5_file in h5_files:
- if h5_file in already_processed_filenames:
- print(f"file: {h5_file} already processed!")
- continue
- print(h5_file)
-
- """ Grab the relevant dataframes from the hdf5 file """
- with pd.HDFStore(h5_file) as data:
- df_trials = data.trials
- df_puffs = data.trials_timing
- df_phases = data.phases
-
- """ Sometimes there are extra session metadata stored in other dataframes
- with the key names:
- 'sessions/{session_number}/notes'
- 'sessions/{session_number}/params'
- 'sessions/{session_number}/sync'
- If they exist we want to ingest their metadata, but they do not always exist
- """
- stored_session_keys = [x for x in data if 'sessions' in x]
-
- """ Figure out the information from the h5 filepath """
- rig = h5_file.split('/')[-2][-1] # just want the 0 or 1 (not "rig0" or "rig1")
- cohort = h5_file.split('/')[-3].split('_')[-1] # 'want C10' or just '10'
- project_name = h5_file.split('/')[-4].strip()
- username = h5_file.split('/')[-5].strip()
- puffs_cohort_insert_dict = {
- 'user_id':username,
- 'project_name':project_name,
- 'cohort':cohort
+import pandas as pd
+from u19_pipeline.utils.logging_config import get_logger, setup_logging
+
+logger = get_logger(__name__)
+
+
+def main():
+ setup_logging()
+
+ dj.config["database.host"] = "datajoint00.pni.princeton.edu"
+ # load dj creds from file
+ credfile = "/jukebox/wang/ahoag/.djenv"
+ with open(credfile) as infile:
+ cred_dict = json.load(infile)
+
+ dj.config["database.user"] = cred_dict.get("DJ_DB_USER")
+ dj.config["database.password"] = cred_dict.get("DJ_DB_PASS")
+
+ # Link to existing databases
+ u19_lab = dj.create_virtual_module("u19_lab", "u19_lab")
+ u19_subject = dj.create_virtual_module("u19_subject", "u19_subject")
+ u19_acq = dj.create_virtual_module("u19_acquisition", "u19_acquisition")
+
+ # Link to new Puffs dbs
+ u19_puffs = dj.create_virtual_module("u19_puffs", "u19_puffs")
+
+ # Currently a hardcoded data folder to one of Marlies' cohorts for this example,
+ # but in the future will loop over folders starting from
+ # The root folder: /jukebox/braininit/puffs/
+ data_folder = "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation/cohort_10/rig0/"
+ h5_files = glob.glob(data_folder + "/data*.h5")
+ mouse_info_file = os.path.join(
+ "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation",
+ "mouse_info_0821_datajoint_format.csv",
+ )
+
+ # first read in the mouse_info file into a pandas dataframe
+ df_info = pd.read_csv(mouse_info_file, keep_default_na=False)
+
+ # # Use mouse_info file and do the u19_subject and u19_lab inserts
+ # Tables we need to insert into (**In this order**) are:
+ # - u19_lab.User # in case user is not already in the database
+ # - u19_lab.Protocol # in case protocol is not already in the database
+ # - u19_lab.Location # in case the location of each subject is not already in the database
+ #
+ # - u19_subject.Species # in case they are NOT training on mice
+ # - u19_subject.Strain # in case they are NOT training on a strain of mice already in the db
+ # - u19_subject.Line # in case they are using a new genetic line
+ # - u19_subject.Subject # in case they are using new subjects
+
+
+ # u19_lab.User() inserts
+ unique_netids = df_info["netid"].unique()
+ full_names = df_info["experimenter_fullname"]
+ user_insert_list = []
+ for netid in unique_netids:
+ netid_mask = df_info["netid"] == netid
+ full_name = full_names[netid_mask].iloc[0]
+ user_insert_dict = {
+ "user_id": netid,
+ "user_nickname": netid,
+ "full_name": full_name,
+ "email": f"{netid}@princeton.edu",
+ "contact_via": "Email",
+ "presence": "Available",
+ "tech_responsibility": "no",
+ "day_cutoff_time": np.array([[18.0, 0.0]]),
}
- u19_puffs.PuffsCohort().insert1(puffs_cohort_insert_dict,skip_duplicates=True)
-
- df_trials['rig'] = rig
- df_trials['cohort'] = cohort
- df_trials['project_name'] = project_name
-
- """ Make choice column """
- df_trials['choice'] = (df_trials['side'] == df_trials['outcome'].astype(int))
- """ set choice to -1 if outcome was > 1 """
- bad_outcome_mask = df_trials['outcome']>1
- df_trials.loc[bad_outcome_mask,'choice'] = -1
- """ Map choice to string values: L,R and nil """
- df_trials['choice'] = df_trials['choice'].apply(
- lambda x: 'L' if x==0 else ('R' if x==1 else 'nil'))
-
- """ Make an answered correct column from outcome column """
- df_trials['answered_correct'] = (df_trials['outcome'] == 1).astype('int')
-
- """ Turn side column into string type """
- df_trials['side'] = df_trials['side'].apply(lambda x: 'L' if x==0 else ('R' if x==1 else 'nil'))
-
- """ Make a new dataframe for unique sessions, i.e. where subj and session are unique
- in trials """
- df_sessions = df_trials.groupby(['subj','session']).max().reset_index().sort_values(
- ['subj','session'])
- sessions = df_sessions['session']
- last_levels = df_sessions['level']
- session_ends_rel = df_sessions['end']
- """ make a session_dates column """
- session_dates = sessions.apply(lambda x: str(x)[:10])
- df_sessions['session_date'] = session_dates
- subjs = df_sessions['subj']
- """ look up where these subjects are """
- rigs = df_sessions['rig'].astype('int')
- """ make a session number column - i.e.
- the 0-indexed counter for the session on a day for a single subject """
- session_numbers = df_sessions.groupby(['session_date','subj']).cumcount()
- df_sessions['session_number'] = session_numbers
-
- """ Figure out fraction correct for each session """
- fraction_trials_correct_df = df_trials.groupby(
- ['session','subj']).agg({'answered_correct': 'sum','side':'count'})
- fraction_trials_correct_df['fraction_correct'] = \
- fraction_trials_correct_df['answered_correct']/fraction_trials_correct_df['side']
- fraction_trials_correct_df = fraction_trials_correct_df.sort_values(['subj','session']).reset_index()
- df_sessions['fraction_correct'] = fraction_trials_correct_df['fraction_correct']
- fractions_correct = df_sessions['fraction_correct']
-
- """ Initialize lists to fill and then insert into db """
- session_started_insert_list = []
- session_insert_list = []
- puffs_session_insert_list = []
-
- """ Loop over sessions and assemble the inserts """
- n_sessions = len(df_sessions)
- for ii in range(len(df_sessions)):
- subj = str(int(subjs.iloc[ii]))
- subject_fullname = '_'.join([username,project_name,subj])
- last_level = last_levels.iloc[ii]
- date = session_dates.iloc[ii]
- print(f"session {ii+1}/{len(df_sessions)}: {subject_fullname}, {date}")
- session_datetime = sessions.iloc[ii]
- session_end_rel = session_ends_rel[ii]
- session_end_datetime = session_datetime + timedelta(seconds=session_end_rel)
- session_number = session_numbers.iloc[ii]
- percentage_correct = fractions_correct.iloc[ii]*100
- rig = rigs.iloc[ii]
-
- if rig == 0:
- location = "pni-ltl016-05"
- elif rig == 1:
- location = "wang-behavior"
- else:
- sys.exit(f'Rig: {rig} is not one of the rigs. Check data type (needs to be integer)')
- session_compressed_str = session_datetime.strftime('%Y%m%d%H%M%S')
-
- """ Check to see if the session metadata dataframes exist """
- this_session_params_key = f'sessions/{session_compressed_str}/params'
- if this_session_params_key in stored_session_keys:
- session_param_dict = json.loads(data[this_session_params_key].iloc[0])
- else:
- session_param_dict = None
- this_session_notes_key = f'sessions/{session_compressed_str}/notes'
- if this_session_notes_key in stored_session_keys:
- session_notes_dict = json.loads(data[this_session_notes_key].iloc[0])
- if 'notes' in session_notes_dict:
- session_notes = session_notes_dict['notes']
+ user_insert_list.append(user_insert_dict)
+ u19_lab.User.insert(user_insert_list, skip_duplicates=True)
+
+ # u19_lab.Protocol() inserts
+ unique_protocols = df_info["protocol"].unique()
+ protocol_insert_list = []
+ for protocol in unique_protocols:
+ protocol_insert_dict = {
+ "protocol": protocol,
+ "protocol_description": "",
+ }
+ protocol_insert_list.append(protocol_insert_dict)
+ u19_lab.Protocol.insert(protocol_insert_list, skip_duplicates=True)
+
+
+ # u19_lab.Location() inserts
+ unique_locations = df_info["location"].unique()
+ location_descriptions = df_info["location_description"]
+ location_insert_list = []
+ for location in unique_locations:
+ location_mask = df_info["location"] == location
+ location_description = location_descriptions[netid_mask].iloc[0]
+ location_insert_dict = {
+ "location": location,
+ "location_description": location_description,
+ }
+ location_insert_list.append(location_insert_dict)
+ u19_lab.Location.insert(location_insert_list, skip_duplicates=True)
+
+ # u19_subject.Species() inserts
+ unique_species = df_info["binomial"].unique()
+ species_insert_list = []
+ for species in unique_species:
+ species_insert_dict = {
+ "binomial": species,
+ "species_nickname": "",
+ }
+ species_insert_list.append(species_insert_dict)
+ u19_subject.Species.insert(species_insert_list, skip_duplicates=True)
+
+ # u19_subject.Strain() inserts
+ unique_strains = df_info["strain"].unique()
+ strain_insert_list = []
+ for strain in unique_strains:
+ strain_insert_dict = {
+ "strain_name": strain,
+ "strain_description": "",
+ }
+ strain_insert_list.append(strain_insert_dict)
+ u19_subject.Strain.insert(strain_insert_list, skip_duplicates=True)
+
+ # u19_subject.Line() inserts
+ unique_lines = df_info["genetic_line"].unique()
+ binomials = df_info["binomial"]
+ strains = df_info["strain"]
+ line_insert_list = []
+ for line in unique_lines:
+ line_mask = df_info["genetic_line"] == line
+ binomial = binomials[line_mask].iloc[0]
+ strain = strains[line_mask].iloc[0]
+ line_insert_dict = {
+ "line": line,
+ "binomial": binomial,
+ "strain_name": strain,
+ "line_description": "",
+ "target_phenotype": "",
+ "is_active": 1,
+ }
+ line_insert_list.append(line_insert_dict)
+ u19_subject.Line.insert(line_insert_list, skip_duplicates=True)
+
+
+ # u19_subject.Subject() inserts
+ unique_subjects = df_info["subj"].unique()
+ project_names = df_info["project_name"]
+ subject_descriptions = df_info["subj_description"]
+ netids = df_info["netid"]
+ sexes = df_info["sex"]
+ DOBs = df_info["DOB"]
+ locations = df_info["location"]
+ lines = df_info["genetic_line"]
+ protocols = df_info["protocol"]
+ subject_insert_list = []
+ for subject in unique_subjects:
+ subject_mask = df_info["subj"] == subject
+ netid = netids[subject_mask].iloc[0]
+ project_name = project_names[subject_mask].iloc[0]
+ subject_fullname = "_".join([netid, project_name, str(subject)])
+ sex = sexes[subject_mask].iloc[0]
+ DOB = DOBs[subject_mask].iloc[0]
+ location = locations[subject_mask].iloc[0]
+ line = lines[subject_mask].iloc[0]
+ protocol = protocols[subject_mask].iloc[0]
+ subject_description = subject_descriptions[subject_mask].iloc[0]
+ subject_insert_dict = {
+ "subject_fullname": subject_fullname,
+ "subject_nickname": str(subject),
+ "subject_description": subject_description,
+ "user_id": netid,
+ "sex": "Male" if sex == "M" else "Female",
+ "dob": DOB,
+ "location": location,
+ "line": line,
+ "protocol": protocol,
+ }
+ subject_insert_list.append(subject_insert_dict)
+ u19_subject.Subject.insert(subject_insert_list, skip_duplicates=True)
+
+
+ # ## Now use h5 files to make inserts into u19_acquisition tables and puffs specific tables
+ #
+ # Tables we need to insert into (**In this order**) are:
+ # - u19_puffs.PuffsCohort # in case cohort is not already in the database
+ # - u19_puffs.PuffsFileAcquisition # as we process the various files
+ #
+ # - u19_acquisition.SessionStarted # as we process the session data in each h5 file
+ # - u19_acquisition.Session # as we process the session data in each h5 file
+ #
+ # - u19_puffs.PuffsSession
+ # - u19_puffs.PuffsSession.Trial
+ # - u19_puffs.PuffsSession.TrialPhase
+ # - u19_puffs.PuffsSession.Puff
+ #
+
+ # first find list of h5 files that are already processed so we do not
+ # repeat the ingestion on those
+ already_processed_filenames = (u19_puffs.PuffsFileAcquisition() & "ingested=1").fetch("h5_filename")
+
+ # Loop over h5 files in this cohort/rig folder
+ for h5_file in h5_files:
+ if h5_file in already_processed_filenames:
+ logger.info("file: %s already processed!", h5_file)
+ continue
+ logger.info("processing file %s", h5_file)
+
+ """ Grab the relevant dataframes from the hdf5 file """
+ with pd.HDFStore(h5_file) as data:
+ df_trials = data.trials
+ df_puffs = data.trials_timing
+ df_phases = data.phases
+
+ """ Sometimes there are extra session metadata stored in other dataframes
+ with the key names:
+ 'sessions/{session_number}/notes'
+ 'sessions/{session_number}/params'
+ 'sessions/{session_number}/sync'
+ If they exist we want to ingest their metadata, but they do not always exist
+ """
+ stored_session_keys = [x for x in data if "sessions" in x]
+
+ """ Figure out the information from the h5 filepath """
+ rig = h5_file.split("/")[-2][-1] # just want the 0 or 1 (not "rig0" or "rig1")
+ cohort = h5_file.split("/")[-3].split("_")[-1] # 'want C10' or just '10'
+ project_name = h5_file.split("/")[-4].strip()
+ username = h5_file.split("/")[-5].strip()
+ puffs_cohort_insert_dict = {
+ "user_id": username,
+ "project_name": project_name,
+ "cohort": cohort,
+ }
+ u19_puffs.PuffsCohort().insert1(puffs_cohort_insert_dict, skip_duplicates=True)
+
+ df_trials["rig"] = rig
+ df_trials["cohort"] = cohort
+ df_trials["project_name"] = project_name
+
+ """ Make choice column """
+ df_trials["choice"] = df_trials["side"] == df_trials["outcome"].astype(int)
+ """ set choice to -1 if outcome was > 1 """
+ bad_outcome_mask = df_trials["outcome"] > 1
+ df_trials.loc[bad_outcome_mask, "choice"] = -1
+ """ Map choice to string values: L,R and nil """
+ df_trials["choice"] = df_trials["choice"].apply(lambda x: "L" if x == 0 else ("R" if x == 1 else "nil"))
+
+ """ Make an answered correct column from outcome column """
+ df_trials["answered_correct"] = (df_trials["outcome"] == 1).astype("int")
+
+ """ Turn side column into string type """
+ df_trials["side"] = df_trials["side"].apply(lambda x: "L" if x == 0 else ("R" if x == 1 else "nil"))
+
+ """ Make a new dataframe for unique sessions, i.e. where subj and session are unique
+ in trials """
+ df_sessions = df_trials.groupby(["subj", "session"]).max().reset_index().sort_values(["subj", "session"])
+ sessions = df_sessions["session"]
+ last_levels = df_sessions["level"]
+ session_ends_rel = df_sessions["end"]
+ """ make a session_dates column """
+ session_dates = sessions.apply(lambda x: str(x)[:10])
+ df_sessions["session_date"] = session_dates
+ subjs = df_sessions["subj"]
+ """ look up where these subjects are """
+ rigs = df_sessions["rig"].astype("int")
+ """ make a session number column - i.e.
+ the 0-indexed counter for the session on a day for a single subject """
+ session_numbers = df_sessions.groupby(["session_date", "subj"]).cumcount()
+ df_sessions["session_number"] = session_numbers
+
+ """ Figure out fraction correct for each session """
+ fraction_trials_correct_df = df_trials.groupby(["session", "subj"]).agg(
+ {"answered_correct": "sum", "side": "count"}
+ )
+ fraction_trials_correct_df["fraction_correct"] = (
+ fraction_trials_correct_df["answered_correct"] / fraction_trials_correct_df["side"]
+ )
+ fraction_trials_correct_df = fraction_trials_correct_df.sort_values(["subj", "session"]).reset_index()
+ df_sessions["fraction_correct"] = fraction_trials_correct_df["fraction_correct"]
+ fractions_correct = df_sessions["fraction_correct"]
+
+ """ Initialize lists to fill and then insert into db """
+ session_started_insert_list = []
+ session_insert_list = []
+ puffs_session_insert_list = []
+
+ """ Loop over sessions and assemble the inserts """
+ n_sessions = len(df_sessions)
+ for ii in range(len(df_sessions)):
+ subj = str(int(subjs.iloc[ii]))
+ subject_fullname = "_".join([username, project_name, subj])
+ last_level = last_levels.iloc[ii]
+ date = session_dates.iloc[ii]
+ logger.info("session %s/%s: %s, %s", ii + 1, len(df_sessions), subject_fullname, date)
+ session_datetime = sessions.iloc[ii]
+ session_end_rel = session_ends_rel[ii]
+ session_end_datetime = session_datetime + timedelta(seconds=session_end_rel)
+ session_number = session_numbers.iloc[ii]
+ percentage_correct = fractions_correct.iloc[ii] * 100
+ rig = rigs.iloc[ii]
+
+ if rig == 0:
+ location = "pni-ltl016-05"
+ elif rig == 1:
+ location = "wang-behavior"
else:
- session_notes = None
- if 'stdout' in session_notes_dict:
- session_stdout = session_notes_dict['stdout']
+ sys.exit(f"Rig: {rig} is not one of the rigs. Check data type (needs to be integer)")
+ session_compressed_str = session_datetime.strftime("%Y%m%d%H%M%S")
+
+ """ Check to see if the session metadata dataframes exist """
+ this_session_params_key = f"sessions/{session_compressed_str}/params"
+ if this_session_params_key in stored_session_keys:
+ session_param_dict = json.loads(data[this_session_params_key].iloc[0])
else:
- session_stdout = None
- if 'stderr' in session_notes_dict:
- session_stderr = session_notes_dict['stderr']
+ session_param_dict = None
+ this_session_notes_key = f"sessions/{session_compressed_str}/notes"
+ if this_session_notes_key in stored_session_keys:
+ session_notes_dict = json.loads(data[this_session_notes_key].iloc[0])
+ session_notes = session_notes_dict.get("notes", None)
+ session_stdout = session_notes_dict.get("stdout", None)
+ session_stderr = session_notes_dict.get("stderr", None)
else:
+ session_notes = None
+ session_stdout = None
session_stderr = None
- else:
- session_notes = None
- session_stdout = None
- session_stderr = None
- this_session_sync_key = f'sessions/{session_compressed_str}/sync'
- if this_session_sync_key in stored_session_keys:
- session_sync_dict = data[this_session_notes_key].todict()
- else:
- session_sync_dict = None
-
- """ Assemble the u19_acquistion inserts for this session """
- session_started_insert_dict = {
- 'subject_fullname':subject_fullname,
- 'session_date':date,
- 'session_number':session_number,
- 'session_start_time':session_datetime.strftime('%Y-%m-%d %H:%M:%S'),
- 'session_location':location,
- 'task':'AirPuffs',
- 'local_path_behavior_file':'',
- 'remote_path_behavior_file':h5_file,
- 'is_finished':1
- }
- session_insert_dict = {
- 'subject_fullname':subject_fullname,
- 'session_date':date,
- 'session_number':session_number,
- 'session_start_time':session_datetime.strftime('%Y-%m-%d %H:%M:%S'),
- 'session_end_time':session_end_datetime.strftime('%Y-%m-%d %H:%M:%S'),
- 'session_location':location,
- 'task':'AirPuffs',
- 'level':last_level,
- 'set_id':1,
- 'stimulus_bank':"",
- 'stimulus_commit':"",
- 'session_performance':percentage_correct,
- 'session_narrative': '',
+ this_session_sync_key = f"sessions/{session_compressed_str}/sync"
+ if this_session_sync_key in stored_session_keys:
+ session_sync_dict = data[this_session_notes_key].todict()
+ else:
+ session_sync_dict = None
- }
- """ Assemble the puffs inserts for this session """
- # PuffsSession() table
- puffs_session_insert_dict = {
- 'subject_fullname':subject_fullname,
- 'session_date':date,
- 'session_number':session_number,
- 'session_params':session_param_dict,
- 'rig':rig,
- 'notes':session_notes,
- 'stdout':session_stdout,
- 'stderr':session_stderr,
- 'sync':session_sync_dict,
-
- }
- # Trial() table
- df_trials_this_session = df_trials[df_trials['session'] == session_datetime][[
- 'idx','level','side','choice','answered_correct',
- 'draw_p','start','end','dur','nL_intended',
- 'nL','nR_intended','nR','reward','reward_scale','rule']]
- num_trials_this_session = len(df_trials_this_session)
- session_insert_dict['num_trials'] = num_trials_this_session
- df_trials_this_session['subject_fullname'] = subject_fullname
- df_trials_this_session['session_date'] = date
- df_trials_this_session['session_number'] = session_number
- df_trials_this_session['task'] = 'AirPuffs'
- df_trials_this_session['set_id'] = 1
- df_trials_this_session['trial_duration'] = df_trials_this_session['end'] - df_trials_this_session['start']
- df_trials_this_session.rename(columns={
- 'idx': 'trial_idx',
- 'draw_p': 'trial_prior_p_left',
- 'start':'trial_rel_start',
- 'end':'trial_rel_finish',
- 'dur':'cue_period',
- 'nL':'num_puffs_received_l',
- 'nL_intended':'num_puffs_intended_l',
- 'nR':'num_puffs_received_r',
- 'nR_intended':'num_puffs_intended_r',
- 'side':'trial_type',
- 'reward':'reward_rel_start'
- },inplace=True)
-
- trials_insert_list = df_trials_this_session.to_dict('records')
-
- # Puff() table
- df_puffs_this_session = df_puffs[df_puffs['session'] == session_datetime].copy()
- df_puffs_this_session['subject_fullname'] = subject_fullname
- df_puffs_this_session['session_date'] = date
- df_puffs_this_session['session_number'] = session_number
- df_puffs_this_session['puff_idx'] = df_puffs_this_session.groupby('trial').cumcount()
- df_puffs_this_session.rename(columns={
- 'time':'puff_rel_time',
- 'trial':'trial_idx',
- },inplace=True)
- df_puffs_this_session['session_date'] = date
- df_puffs_this_session['session_number'] = session_number
- df_puffs_this_session = df_puffs_this_session[[
- 'subject_fullname',
- 'session_date',
- 'session_number',
- 'trial_idx',
- 'puff_idx',
- 'side',
- 'puff_rel_time'
- ]]
- puffs_insert_list = df_puffs_this_session.to_dict('records')
-
- # TrialPhase() table
- df_phases_this_session = df_phases[df_phases['session'] == session_datetime].copy()
- df_phases_this_session['subject_fullname'] = subject_fullname
- df_phases_this_session['session_date'] = date
- df_phases_this_session['session_number'] = session_number
- df_phases_this_session.rename(columns={
- 'trial':'trial_idx',
- 'start_time':'phase_rel_start',
- 'end_time':'phase_rel_finish',
- },inplace=True)
- df_phases_this_session = df_phases_this_session[[
- 'subject_fullname',
- 'session_date',
- 'session_number',
- 'trial_idx',
- 'phase',
- 'phase_rel_start',
- 'phase_rel_finish'
- ]]
- """ Some phases seem to reference trials that do not exist
- restrict phase rows to trials that exist """
- trial_idxs = df_trials_this_session['trial_idx']
- phase_trial_mask = df_phases_this_session['trial_idx'].isin(trial_idxs.values)
- df_phases_this_session_goodtrials = df_phases_this_session[phase_trial_mask]
- phases_insert_list = df_phases_this_session_goodtrials.to_dict('records')
- """ Start a transaction and do the inserts for this session """
- connection = u19_acq.SessionStarted.connection
- with connection.transaction:
- u19_acq.SessionStarted().insert1(session_started_insert_dict,skip_duplicates=True)
- u19_acq.Session().insert1(session_insert_dict,skip_duplicates=True)
- u19_puffs.PuffsSession().insert1(puffs_session_insert_dict,skip_duplicates=True)
- u19_puffs.PuffsSession.Trial().insert(trials_insert_list,skip_duplicates=True)
- # u19_puffs.PuffsSession.Puff().insert(puffs_insert_list,skip_duplicates=True)
- # u19_puffs.PuffsSession.TrialPhase().insert(phases_insert_list,skip_duplicates=True)
- """ If the inserts for all sessions in this hdf5 file were successful,
- then we need to mark this hdf5 file as processed by inserting it
- into the PuffsFileAcquisition() table in u19_puffs_acquisition table """
- file_acq_insert_dict = {
- 'user_id':username,
- 'project_name':project_name,
- 'cohort':cohort,
- 'rig':rig,
- 'h5_filename':h5_file,
- 'ingested':1
- }
- u19_puffs.PuffsFileAcquisition.insert1(file_acq_insert_dict,replace=True) # want to overwrite with ingested=1 if ingested=0 for some reason
\ No newline at end of file
+ """ Assemble the u19_acquistion inserts for this session """
+ session_started_insert_dict = {
+ "subject_fullname": subject_fullname,
+ "session_date": date,
+ "session_number": session_number,
+ "session_start_time": session_datetime.strftime("%Y-%m-%d %H:%M:%S"),
+ "session_location": location,
+ "task": "AirPuffs",
+ "local_path_behavior_file": "",
+ "remote_path_behavior_file": h5_file,
+ "is_finished": 1,
+ }
+ session_insert_dict = {
+ "subject_fullname": subject_fullname,
+ "session_date": date,
+ "session_number": session_number,
+ "session_start_time": session_datetime.strftime("%Y-%m-%d %H:%M:%S"),
+ "session_end_time": session_end_datetime.strftime("%Y-%m-%d %H:%M:%S"),
+ "session_location": location,
+ "task": "AirPuffs",
+ "level": last_level,
+ "set_id": 1,
+ "stimulus_bank": "",
+ "stimulus_commit": "",
+ "session_performance": percentage_correct,
+ "session_narrative": "",
+ }
+ """ Assemble the puffs inserts for this session """
+ # PuffsSession() table
+ puffs_session_insert_dict = {
+ "subject_fullname": subject_fullname,
+ "session_date": date,
+ "session_number": session_number,
+ "session_params": session_param_dict,
+ "rig": rig,
+ "notes": session_notes,
+ "stdout": session_stdout,
+ "stderr": session_stderr,
+ "sync": session_sync_dict,
+ }
+ # Trial() table
+ df_trials_this_session = df_trials[df_trials["session"] == session_datetime][
+ [
+ "idx",
+ "level",
+ "side",
+ "choice",
+ "answered_correct",
+ "draw_p",
+ "start",
+ "end",
+ "dur",
+ "nL_intended",
+ "nL",
+ "nR_intended",
+ "nR",
+ "reward",
+ "reward_scale",
+ "rule",
+ ]
+ ]
+ num_trials_this_session = len(df_trials_this_session)
+ session_insert_dict["num_trials"] = num_trials_this_session
+ df_trials_this_session["subject_fullname"] = subject_fullname
+ df_trials_this_session["session_date"] = date
+ df_trials_this_session["session_number"] = session_number
+ df_trials_this_session["task"] = "AirPuffs"
+ df_trials_this_session["set_id"] = 1
+ df_trials_this_session["trial_duration"] = df_trials_this_session["end"] - df_trials_this_session["start"]
+ df_trials_this_session.rename(
+ columns={
+ "idx": "trial_idx",
+ "draw_p": "trial_prior_p_left",
+ "start": "trial_rel_start",
+ "end": "trial_rel_finish",
+ "dur": "cue_period",
+ "nL": "num_puffs_received_l",
+ "nL_intended": "num_puffs_intended_l",
+ "nR": "num_puffs_received_r",
+ "nR_intended": "num_puffs_intended_r",
+ "side": "trial_type",
+ "reward": "reward_rel_start",
+ },
+ inplace=True,
+ )
+
+ trials_insert_list = df_trials_this_session.to_dict("records")
+
+ # Puff() table
+ df_puffs_this_session = df_puffs[df_puffs["session"] == session_datetime].copy()
+ df_puffs_this_session["subject_fullname"] = subject_fullname
+ df_puffs_this_session["session_date"] = date
+ df_puffs_this_session["session_number"] = session_number
+ df_puffs_this_session["puff_idx"] = df_puffs_this_session.groupby("trial").cumcount()
+ df_puffs_this_session.rename(
+ columns={
+ "time": "puff_rel_time",
+ "trial": "trial_idx",
+ },
+ inplace=True,
+ )
+ df_puffs_this_session["session_date"] = date
+ df_puffs_this_session["session_number"] = session_number
+ df_puffs_this_session = df_puffs_this_session[
+ [
+ "subject_fullname",
+ "session_date",
+ "session_number",
+ "trial_idx",
+ "puff_idx",
+ "side",
+ "puff_rel_time",
+ ]
+ ]
+ puffs_insert_list = df_puffs_this_session.to_dict("records")
+
+ # TrialPhase() table
+ df_phases_this_session = df_phases[df_phases["session"] == session_datetime].copy()
+ df_phases_this_session["subject_fullname"] = subject_fullname
+ df_phases_this_session["session_date"] = date
+ df_phases_this_session["session_number"] = session_number
+ df_phases_this_session.rename(
+ columns={
+ "trial": "trial_idx",
+ "start_time": "phase_rel_start",
+ "end_time": "phase_rel_finish",
+ },
+ inplace=True,
+ )
+ df_phases_this_session = df_phases_this_session[
+ [
+ "subject_fullname",
+ "session_date",
+ "session_number",
+ "trial_idx",
+ "phase",
+ "phase_rel_start",
+ "phase_rel_finish",
+ ]
+ ]
+ """ Some phases seem to reference trials that do not exist
+ restrict phase rows to trials that exist """
+ trial_idxs = df_trials_this_session["trial_idx"]
+ phase_trial_mask = df_phases_this_session["trial_idx"].isin(trial_idxs.values)
+ df_phases_this_session_goodtrials = df_phases_this_session[phase_trial_mask]
+ phases_insert_list = df_phases_this_session_goodtrials.to_dict("records")
+ """ Start a transaction and do the inserts for this session """
+ connection = u19_acq.SessionStarted.connection
+ with connection.transaction:
+ u19_acq.SessionStarted().insert1(session_started_insert_dict, skip_duplicates=True)
+ u19_acq.Session().insert1(session_insert_dict, skip_duplicates=True)
+ u19_puffs.PuffsSession().insert1(puffs_session_insert_dict, skip_duplicates=True)
+ u19_puffs.PuffsSession.Trial().insert(trials_insert_list, skip_duplicates=True)
+ # u19_puffs.PuffsSession.Puff().insert(puffs_insert_list,skip_duplicates=True)
+ # u19_puffs.PuffsSession.TrialPhase().insert(phases_insert_list,skip_duplicates=True)
+ """ If the inserts for all sessions in this hdf5 file were successful,
+ then we need to mark this hdf5 file as processed by inserting it
+ into the PuffsFileAcquisition() table in u19_puffs_acquisition table """
+ file_acq_insert_dict = {
+ "user_id": username,
+ "project_name": project_name,
+ "cohort": cohort,
+ "rig": rig,
+ "h5_filename": h5_file,
+ "ingested": 1,
+ }
+ u19_puffs.PuffsFileAcquisition.insert1(
+ file_acq_insert_dict, replace=True
+ ) # want to overwrite with ingested=1 if ingested=0 for some reason
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/populate_imaging.py b/scripts/populate_imaging.py
index 14dead71..5845def6 100644
--- a/scripts/populate_imaging.py
+++ b/scripts/populate_imaging.py
@@ -1,5 +1,4 @@
-from u19_pipeline_python import lab, reference, subject, task, action, acquisition, imaging
+from u19_pipeline_python import acquisition
-
-kargs = dict(supress_errors=True, display_progress=True)
+kargs = {"supress_errors": True, "display_progress": True}
acquisition.Scan.populate(**kargs)
diff --git a/scripts/rig_maintenance_demo.py b/scripts/rig_maintenance_demo.py
index 68ad8204..ebc07313 100755
--- a/scripts/rig_maintenance_demo.py
+++ b/scripts/rig_maintenance_demo.py
@@ -15,105 +15,108 @@
from datetime import date
# Add the u19_pipeline to the path
-sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
try:
from u19_pipeline import rig_maintenance
+ from u19_pipeline.utils.logging_config import get_logger, setup_logging
except ImportError as e:
- print(f"Error importing modules: {e}")
- print("Make sure u19_pipeline is properly installed and configured.")
+ import sys as _sys
+ _sys.stderr.write(f"Error importing modules: {e}\n")
+ _sys.stderr.write("Make sure u19_pipeline is properly installed and configured.\n")
sys.exit(1)
+logger = get_logger(__name__)
+
def demo_insert_maintenance_record():
"""Demo: Insert a maintenance record."""
- print("=== Demo: Inserting Maintenance Record ===")
-
+ logger.info("=== Demo: Inserting Maintenance Record ===")
+
# Example maintenance record
maintenance_data = {
- 'location': 'Bezos2', # Must exist in lab.Location
- 'maintenance_type': 'Replacing lines', # Must exist in MaintenanceType
- 'maintenance_date': date.today(),
- 'user': 'demo_user', # Must exist in lab.User
- 'maintenance_notes': 'Replaced all water lines. System tested and working normally.'
+ "location": "Bezos2", # Must exist in lab.Location
+ "maintenance_type": "Replacing lines", # Must exist in MaintenanceType
+ "maintenance_date": date.today(),
+ "user": "demo_user", # Must exist in lab.User
+ "maintenance_notes": "Replaced all water lines. System tested and working normally.",
}
-
+
try:
# Insert the record (commented out to prevent actual DB modifications in demo)
# rig_maintenance.RigMaintenance.insert1(maintenance_data)
- print("Would insert maintenance record:")
+ logger.info("Would insert maintenance record:")
for key, value in maintenance_data.items():
- print(f" {key}: {value}")
- print("✅ Maintenance record ready for insertion")
+ logger.info(" %s: %s", key, value)
+ logger.info("Maintenance record ready for insertion")
except Exception as e:
- print(f"❌ Error inserting maintenance record: {e}")
+ logger.error("Error inserting maintenance record: %s", e)
def demo_query_maintenance_history():
"""Demo: Query maintenance history for a rig."""
- print("\n=== Demo: Querying Maintenance History ===")
-
- rig_name = 'Bezos2'
-
+ logger.info("=== Demo: Querying Maintenance History ===")
+
+ rig_name = "Bezos2"
+
try:
# Query maintenance history (commented out to prevent DB access in demo)
# records = (rig_maintenance.RigMaintenance & {'location': rig_name}).fetch(
# as_dict=True, order_by='maintenance_date DESC'
# )
-
- print(f"Would query maintenance history for rig: {rig_name}")
- print("Query would return records ordered by most recent first")
- print("✅ Query structure is correct")
+
+ logger.info("Would query maintenance history for rig: %s", rig_name)
+ logger.info("Query would return records ordered by most recent first")
+ logger.info("Query structure is correct")
except Exception as e:
- print(f"❌ Error querying maintenance history: {e}")
+ logger.error("Error querying maintenance history: %s", e)
def demo_check_overdue_maintenance():
"""Demo: Check for overdue maintenance."""
- print("\n=== Demo: Checking Overdue Maintenance ===")
-
- print("This demo shows the logic used in check_rig_maintenance.py")
-
+ logger.info("=== Demo: Checking Overdue Maintenance ===")
+
+ logger.info("This demo shows the logic used in check_rig_maintenance.py")
+
# Show the maintenance types and their intervals
- print("\nMaintenance intervals defined:")
+ logger.info("Maintenance intervals defined:")
for mtype in rig_maintenance.MaintenanceType.contents:
maintenance_type, description, interval_days = mtype
- print(f" • {maintenance_type}: every {interval_days} days")
-
+ logger.info(" %s: every %s days", maintenance_type, interval_days)
+
# Show the logic for checking overdue maintenance
current_date = date.today()
- print(f"\nCurrent date: {current_date}")
- print("For each rig and maintenance type combination:")
- print("1. Find most recent maintenance record")
- print("2. Calculate days since last maintenance")
- print("3. Compare against required interval")
- print("4. Flag as overdue if interval exceeded")
- print("✅ Overdue checking logic is sound")
+ logger.info("Current date: %s", current_date)
+ logger.info("For each rig and maintenance type combination:")
+ logger.info("1. Find most recent maintenance record")
+ logger.info("2. Calculate days since last maintenance")
+ logger.info("3. Compare against required interval")
+ logger.info("4. Flag as overdue if interval exceeded")
+ logger.info("Overdue checking logic is sound")
def main():
"""Main demo function."""
- print("Rig Maintenance Schema Demo")
- print("=" * 50)
-
- print("This demo shows how to use the new rig maintenance schema.")
- print("The schema includes:")
- print("- MaintenanceType: Lookup table with maintenance types and intervals")
- print("- RigMaintenance: Records of maintenance performed on rigs")
- print()
-
+ logger.info("Rig Maintenance Schema Demo")
+ logger.info("=" * 50)
+ logger.info("This demo shows how to use the new rig maintenance schema.")
+ logger.info("The schema includes:")
+ logger.info("- MaintenanceType: Lookup table with maintenance types and intervals")
+ logger.info("- RigMaintenance: Records of maintenance performed on rigs")
+
# Run demo functions
demo_insert_maintenance_record()
demo_query_maintenance_history()
demo_check_overdue_maintenance()
-
- print("\n" + "=" * 50)
- print("Demo complete! To use the schema:")
- print("1. Configure DataJoint connection")
- print("2. Create the schema tables in your database")
- print("3. Use the check_rig_maintenance.py script to monitor maintenance")
- print("4. Insert maintenance records as work is completed")
+
+ logger.info("=" * 50)
+ logger.info("Demo complete! To use the schema:")
+ logger.info("1. Configure DataJoint connection")
+ logger.info("2. Create the schema tables in your database")
+ logger.info("3. Use the check_rig_maintenance.py script to monitor maintenance")
+ logger.info("4. Insert maintenance records as work is completed")
if __name__ == "__main__":
- main()
\ No newline at end of file
+ setup_logging()
+ main()
diff --git a/u19_pipeline/__init__.py b/u19_pipeline/__init__.py
index 79f1ef59..e69de29b 100644
--- a/u19_pipeline/__init__.py
+++ b/u19_pipeline/__init__.py
@@ -1,3 +0,0 @@
-from os.path import join, dirname
-import os
-import datajoint as dj
\ No newline at end of file
diff --git a/u19_pipeline/acquisition.py b/u19_pipeline/acquisition.py
index af69bc37..3bdc6d83 100644
--- a/u19_pipeline/acquisition.py
+++ b/u19_pipeline/acquisition.py
@@ -1,7 +1,6 @@
import datajoint as dj
-from u19_pipeline import lab, task, subject
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'acquisition')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "acquisition")
@schema
diff --git a/u19_pipeline/alert_system/alert_code_skeleton.py b/u19_pipeline/alert_system/alert_code_skeleton.py
index f5d262b5..4d6fa257 100644
--- a/u19_pipeline/alert_system/alert_code_skeleton.py
+++ b/u19_pipeline/alert_system/alert_code_skeleton.py
@@ -1,27 +1,15 @@
-
-import pandas as pd
-import datajoint as dj
-import datetime
-import numpy as np
-
-import u19_pipeline.alert_system.behavior_metrics as bm
-import u19_pipeline.alert_system.alert_system_utility as asu
-
-
# Slack Configuration dictionary
slack_configuration_dictionary = {
- 'slack_notification_channel': ['custom_alerts'],
- 'slack_users_channel': ['alvaros']
+ "slack_notification_channel": ["custom_alerts"],
+ "slack_users_channel": ["alvaros"],
}
+
def main():
- '''
+ """
Main function for subject "new_customalert" alert
This function should return a pandas DataFrame where each row will be a slack alert message on configured channels.
You can use datajoint to get data for the alert (e.g. custom_alerts/rig_bias.py) or simply call os scripts (e.g. custom_alerts/braininit_storage.py)
All columns of the dataframe will be included in the alert. (Don't add too many !!)
You can check examples of some alers in the u19_pipeline/alert_system/custom_alerts directory
- '''
-
-
-
+ """
diff --git a/u19_pipeline/alert_system/alert_system_utility.py b/u19_pipeline/alert_system/alert_system_utility.py
index bd65016c..a0bb93d8 100644
--- a/u19_pipeline/alert_system/alert_system_utility.py
+++ b/u19_pipeline/alert_system/alert_system_utility.py
@@ -1,45 +1,44 @@
+import datetime
-
-import pandas as pd
import datajoint as dj
-import datetime
+import pandas as pd
import u19_pipeline.utils.dj_shortcuts as djs
-def get_acquisition_data_alert_system(type='subject_fullname', data_days=60, min_sessions=20):
- '''
+
+def get_acquisition_data_alert_system(type="subject_fullname", data_days=60, min_sessions=20):
+ """
Get and filter data for alert system
Inputs:
- type = subject_fullname/location => Data grouping by subject or by rig
+ type = subject_fullname/location => Data grouping by subject or by rig
data_days = Amount of days to look back for sessions
min_sessions = Min amount of sessions for a subject/rig to draw conclusions about it
Outputs:
session_df = filtered DataFrame from acquisition.Session table
- key_list = list of dictionaries with primary key fields for each session (for querying behavior)
- '''
+ key_list = list of dictionaries with primary key fields for each session (for querying behavior)
+ """
- acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')
+ acquisition = dj.create_virtual_module("acquisition", "u19_acquisition")
# Filter sessiond for # data days only
today = datetime.date.today()
past_date = today - datetime.timedelta(days=data_days)
- query_sessions = 'session_date >="' + past_date.strftime('%Y-%m-%d') +'"'
- query_sessions += 'and session_date < "' + today.strftime('%Y-%m-%d') +'"'
+ query_sessions = 'session_date >="' + past_date.strftime("%Y-%m-%d") + '"'
+ query_sessions += 'and session_date < "' + today.strftime("%Y-%m-%d") + '"'
query_sessions += ' and subject_fullname not like "testuser%"'
session_df = pd.DataFrame((acquisition.Session & query_sessions).fetch(as_dict=True))
- #Filter subjects/rigs with >= min_sessions
- num_sessions_df = session_df.groupby(type).agg({'session_date': [('total_sessions', 'count')]})
+ # Filter subjects/rigs with >= min_sessions
+ num_sessions_df = session_df.groupby(type).agg({"session_date": [("total_sessions", "count")]})
num_sessions_df.columns = num_sessions_df.columns.droplevel()
num_sessions_df = num_sessions_df.reset_index()
- num_sessions_df = num_sessions_df.loc[num_sessions_df['total_sessions'] >= min_sessions, :]
+ num_sessions_df = num_sessions_df.loc[num_sessions_df["total_sessions"] >= min_sessions, :]
session_df = session_df.merge(num_sessions_df, on=type)
# Get key list of sessions for further (behavior) querying
key_fields = djs.get_primary_key_fields(acquisition.Session)
key_list = session_df.loc[:, key_fields]
- key_list = key_list.to_dict('records')
-
+ key_list = key_list.to_dict("records")
- return session_df, key_list
\ No newline at end of file
+ return session_df, key_list
diff --git a/u19_pipeline/alert_system/behavior_metrics.py b/u19_pipeline/alert_system/behavior_metrics.py
index 4d365702..75ac5e96 100644
--- a/u19_pipeline/alert_system/behavior_metrics.py
+++ b/u19_pipeline/alert_system/behavior_metrics.py
@@ -1,74 +1,75 @@
-
-import pandas as pd
-import datajoint as dj
-
-
-class BehaviorMetrics():
-
+class BehaviorMetrics:
@staticmethod
def get_bias_from_trial_df(trials_df, return_all_metrics=False):
- '''
+ """
From a trial by trial df of multiple sessions, calculate a per session bias
- '''
+ """
original_columns = trials_df.columns.tolist()
- sort_columns = ['subject_fullname', 'session_date', 'session_number', 'block', 'trial_idx']
- session_columns = ['subject_fullname', 'session_date', 'session_number']
+ sort_columns = [
+ "subject_fullname",
+ "session_date",
+ "session_number",
+ "block",
+ "trial_idx",
+ ]
+ session_columns = ["subject_fullname", "session_date", "session_number"]
trials_df = trials_df.sort_values(by=sort_columns, ascending=[True, False, False, True, True])
trials_df = trials_df.reset_index(drop=True)
# Left & right trials as integers
- trials_df['left_trial'] = (trials_df['trial_type'] == 'L').astype(int)
- trials_df['right_trial'] = (trials_df['trial_type'] == 'R').astype(int)
- trials_df['trial'] = 1
+ trials_df["left_trial"] = (trials_df["trial_type"] == "L").astype(int)
+ trials_df["right_trial"] = (trials_df["trial_type"] == "R").astype(int)
+ trials_df["trial"] = 1
# Cumulative sum of Left & right trials as integers per session
- trials_df['cum_left_trials'] = trials_df.groupby(session_columns)['left_trial'].cumsum()
- trials_df['cum_right_trials'] = trials_df.groupby(session_columns)['right_trial'].cumsum()
- trials_df['cum_trials'] = trials_df.groupby(session_columns)['trial'].cumsum()
+ trials_df["cum_left_trials"] = trials_df.groupby(session_columns)["left_trial"].cumsum()
+ trials_df["cum_right_trials"] = trials_df.groupby(session_columns)["right_trial"].cumsum()
+ trials_df["cum_trials"] = trials_df.groupby(session_columns)["trial"].cumsum()
# Correct trials per side as integers
- trials_df['correct_trial'] = (trials_df['trial_type'] == trials_df['choice']).astype(int)
- trials_df['correct_left'] = ((trials_df['correct_trial'] == 1) & (trials_df['trial_type'] == 'L')).astype(int)
- trials_df['correct_right'] = ((trials_df['correct_trial'] == 1) & (trials_df['trial_type'] == 'R')).astype(int)
+ trials_df["correct_trial"] = (trials_df["trial_type"] == trials_df["choice"]).astype(int)
+ trials_df["correct_left"] = ((trials_df["correct_trial"] == 1) & (trials_df["trial_type"] == "L")).astype(int)
+ trials_df["correct_right"] = ((trials_df["correct_trial"] == 1) & (trials_df["trial_type"] == "R")).astype(int)
# Cumulative sum of per side correct trials
- trials_df['cum_correct_left_trials'] = trials_df.groupby(session_columns)['correct_left'].cumsum()
- trials_df['cum_correct_right_trials'] = trials_df.groupby(session_columns)['correct_right'].cumsum()
- trials_df['cum_correct_trials'] = trials_df.groupby(session_columns)['correct_trial'].cumsum()
+ trials_df["cum_correct_left_trials"] = trials_df.groupby(session_columns)["correct_left"].cumsum()
+ trials_df["cum_correct_right_trials"] = trials_df.groupby(session_columns)["correct_right"].cumsum()
+ trials_df["cum_correct_trials"] = trials_df.groupby(session_columns)["correct_trial"].cumsum()
# Get only last trial count
- trials_df = trials_df.loc[~trials_df.duplicated(subset=session_columns, keep='last'), :]
+ trials_df = trials_df.loc[~trials_df.duplicated(subset=session_columns, keep="last"), :]
trials_df = trials_df.reset_index(drop=True)
# Calculate bias
- trials_df['bias'] = (trials_df['cum_correct_right_trials'] / trials_df['cum_right_trials']) - (trials_df['cum_correct_left_trials'] / trials_df['cum_left_trials'])
-
- if return_all_metrics:
- bias_df = trials_df
- else:
- bias_df = trials_df[original_columns + ['bias']]
+ trials_df["bias"] = (trials_df["cum_correct_right_trials"] / trials_df["cum_right_trials"]) - (
+ trials_df["cum_correct_left_trials"] / trials_df["cum_left_trials"]
+ )
+
+ bias_df = trials_df if return_all_metrics else trials_df[original_columns + ["bias"]]
return bias_df
-
@staticmethod
def get_zscore_metric_session_df(session_df, metric, groupby_column):
- '''
+ """
Get zscore from a "generic" metric of a session dataframe
- '''
+ """
- #Sort by subject, date and session number
- session_df = session_df.sort_values(by=['subject_fullname', 'session_date', 'session_number'], ascending=[True, False, False])
+ # Sort by subject, date and session number
+ session_df = session_df.sort_values(
+ by=["subject_fullname", "session_date", "session_number"],
+ ascending=[True, False, False],
+ )
session_df = session_df.reset_index(drop=True)
- # Lables for avg an std metric columns
- avg_metric_l = 'avg_'+ metric
- std_metric_l = 'std_'+metric
- z_score_metric_l = 'z_score_'+metric
+ # Lables for avg an std metric columns
+ avg_metric_l = "avg_" + metric
+ std_metric_l = "std_" + metric
+ z_score_metric_l = "z_score_" + metric
# Get mean and std for the selected metric
- avg_df = session_df.groupby(groupby_column).agg({metric: [(avg_metric_l, 'mean'), (std_metric_l, 'std')]})
+ avg_df = session_df.groupby(groupby_column).agg({metric: [(avg_metric_l, "mean"), (std_metric_l, "std")]})
avg_df.columns = avg_df.columns.droplevel()
avg_df = avg_df.reset_index()
@@ -78,4 +79,4 @@ def get_zscore_metric_session_df(session_df, metric, groupby_column):
# Calculate proper zscore
session_df[z_score_metric_l] = (session_df[metric] - session_df[avg_metric_l]) / session_df[std_metric_l]
- return session_df
\ No newline at end of file
+ return session_df
diff --git a/u19_pipeline/alert_system/cronjob_alert.py b/u19_pipeline/alert_system/cronjob_alert.py
index 592bf6d1..84b76a0f 100644
--- a/u19_pipeline/alert_system/cronjob_alert.py
+++ b/u19_pipeline/alert_system/cronjob_alert.py
@@ -1,18 +1,16 @@
-
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(1)
-import datajoint as dj
-import u19_pipeline.alert_system.main_alert_system as mas
-import u19_pipeline.alert_system.log_deletion.old_log_deletion as old
import u19_pipeline.alert_system.live_session_stats_deletion.live_session_stats_deletion as lssd
-import u19_pipeline.alert_system.noDB_backup_creation.noDB_backup_creation_script as noDBbcs
-
+import u19_pipeline.alert_system.log_deletion.old_log_deletion as old
+import u19_pipeline.alert_system.noDB_backup_creation.no_db_backup_creation_script as no_db_bcs
-#mas.main_alert_system()
+# mas.main_alert_system()
old.main_old_log_deletion()
lssd.main_live_session_stats_deletion()
-noDBbcs.main_noDB_backup()
+no_db_bcs.main_noDB_backup()
diff --git a/u19_pipeline/alert_system/custom_alerts/__init__.py b/u19_pipeline/alert_system/custom_alerts/__init__.py
index 2f5c78e4..f77af49c 100644
--- a/u19_pipeline/alert_system/custom_alerts/__init__.py
+++ b/u19_pipeline/alert_system/custom_alerts/__init__.py
@@ -1,2 +1,3 @@
import pkgutil
-__path__ = pkgutil.extend_path(__path__, __name__)
\ No newline at end of file
+
+__path__ = pkgutil.extend_path(__path__, __name__)
diff --git a/u19_pipeline/alert_system/custom_alerts/braininit_storage.py b/u19_pipeline/alert_system/custom_alerts/braininit_storage.py
index 82c7ea42..641b4bc5 100644
--- a/u19_pipeline/alert_system/custom_alerts/braininit_storage.py
+++ b/u19_pipeline/alert_system/custom_alerts/braininit_storage.py
@@ -1,40 +1,36 @@
+import subprocess
-import pandas as pd
import datajoint as dj
-
-import u19_pipeline.lab as lab
-
-import u19_pipeline.alert_system.behavior_metrics as bm
-import u19_pipeline.alert_system.alert_system_utility as asu
-import os
-import subprocess
+import pandas as pd
# Slack Configuration dictionary
slack_configuration_dictionary = {
- 'slack_notification_channel': ['custom_alerts'],
- 'slack_users_channel': []
+ "slack_notification_channel": ["custom_alerts"],
+ "slack_users_channel": [],
}
+
def main():
conf = dj.config
- braininit_path = conf['custom']['root_data_dir'][0]
+ braininit_path = conf["custom"]["root_data_dir"][0]
- #braininit_path = lab.Path().get_local_path2('braininit/Data').as_posix()
- command = "df "+ braininit_path + " | tail -1 | awk '{print $4}'"
+ # braininit_path = lab.Path().get_local_path2('braininit/Data').as_posix()
+ command = "df " + braininit_path + " | tail -1 | awk '{print $4}'"
# a = os.popen(command).read()
storage_left = subprocess.check_output(command, shell=True)
storage_left = storage_left.decode().strip()
storage_left_kb = int(storage_left)
- storage_left_tb = storage_left_kb/1024/1024/1024
+ storage_left_tb = storage_left_kb / 1024 / 1024 / 1024
if storage_left_tb < 4:
- data = {'alert_message': ['Very little space left in braininit'], 'space(tb)': [str(storage_left_tb)]}
+ data = {
+ "alert_message": ["Very little space left in braininit"],
+ "space(tb)": [str(storage_left_tb)],
+ }
alert_df = pd.DataFrame.from_dict(data)
else:
alert_df = pd.DataFrame([])
return alert_df
-
-
diff --git a/u19_pipeline/alert_system/custom_alerts/rig_bias.py b/u19_pipeline/alert_system/custom_alerts/rig_bias.py
index 2f883542..ef77fbf0 100644
--- a/u19_pipeline/alert_system/custom_alerts/rig_bias.py
+++ b/u19_pipeline/alert_system/custom_alerts/rig_bias.py
@@ -1,71 +1,80 @@
+import datetime
-import pandas as pd
import datajoint as dj
-import datetime
import numpy as np
+import pandas as pd
-import u19_pipeline.alert_system.behavior_metrics as bm
import u19_pipeline.alert_system.alert_system_utility as asu
-
+import u19_pipeline.alert_system.behavior_metrics as bm
# Slack Configuration dictionary
-slack_configuration_dictionary = {
- 'slack_notification_channel': ['custom_alerts']
-}
+slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]}
zscore_alert = 2
+
+
def main():
- '''
+ """
Main function for subject "num trials & bias" alert
- '''
+ """
# Get sessions
- _, rig_session_key_list = asu.get_acquisition_data_alert_system(type='session_location')
+ _, rig_session_key_list = asu.get_acquisition_data_alert_system(type="session_location")
# Get trials
- behavior = dj.create_virtual_module('behavior', dj.config['custom']['database.prefix']+'behavior')
- acquisition = dj.create_virtual_module('acquisition', dj.config['custom']['database.prefix']+'acquisition')
-
- rig_trial_df = pd.DataFrame((behavior.TowersBlock.Trial * acquisition.SessionStarted.proj('session_location') \
- & rig_session_key_list).fetch('KEY', 'trial_type', 'choice', 'session_location', as_dict=True))
+ behavior = dj.create_virtual_module("behavior", dj.config["custom"]["database.prefix"] + "behavior")
+ acquisition = dj.create_virtual_module("acquisition", dj.config["custom"]["database.prefix"] + "acquisition")
+
+ rig_trial_df = pd.DataFrame(
+ (behavior.TowersBlock.Trial * acquisition.SessionStarted.proj("session_location") & rig_session_key_list).fetch(
+ "KEY", "trial_type", "choice", "session_location", as_dict=True
+ )
+ )
# Get zscores for bias
bias_df = bm.BehaviorMetrics.get_bias_from_trial_df(rig_trial_df)
- bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df(bias_df, 'bias', 'subject_fullname')
+ bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df(bias_df, "bias", "subject_fullname")
# Filter df for today
today = datetime.date.today() - datetime.timedelta(days=1)
- bias_df = bias_df.loc[bias_df['session_date'] == today, :]
- bias_df['abs_z_score_bias'] = np.abs(bias_df['z_score_bias'])
+ bias_df = bias_df.loc[bias_df["session_date"] == today, :]
+ bias_df["abs_z_score_bias"] = np.abs(bias_df["z_score_bias"])
bias_df = bias_df.reset_index(drop=True)
# Filter if today we got > 3 zscore of trials for a session
- bias_df = bias_df.loc[bias_df['abs_z_score_bias'] >= zscore_alert, :]
+ bias_df = bias_df.loc[bias_df["abs_z_score_bias"] >= zscore_alert, :]
# Get sign of bias (only group subjects with bias to same side)
- bias_df['sign_bias'] = np.sign(bias_df['z_score_bias'])
- bias_df['sign_bias'] = bias_df['sign_bias'].astype(int)
-
- #Count how many subjects were biassed by rig
- bias_location = bias_df.groupby(['session_location', 'sign_bias']).agg({'session_location': [('num_subjects', 'size')],\
- 'subject_fullname': [('subject_fullnames', lambda x: ','.join(x))]})
+ bias_df["sign_bias"] = np.sign(bias_df["z_score_bias"])
+ bias_df["sign_bias"] = bias_df["sign_bias"].astype(int)
+
+ # Count how many subjects were biassed by rig
+ bias_location = bias_df.groupby(["session_location", "sign_bias"]).agg(
+ {
+ "session_location": [("num_subjects", "size")],
+ "subject_fullname": [("subject_fullnames", lambda x: ",".join(x))],
+ }
+ )
bias_location.columns = bias_location.columns.droplevel()
bias_location = bias_location.reset_index()
- #Filter if there were subjects biased to different sides
- bias_location2 = bias_location.groupby(['session_location']).agg({'session_location': [('num_bias_sides', 'size')]})
+ # Filter if there were subjects biased to different sides
+ bias_location2 = bias_location.groupby(["session_location"]).agg({"session_location": [("num_bias_sides", "size")]})
bias_location2.columns = bias_location2.columns.droplevel()
bias_location2 = bias_location2.reset_index()
- bias_location2 = bias_location2.loc[bias_location2['num_bias_sides'] == 1, :]
- bias_location = bias_location.merge(bias_location2, on='session_location')
+ bias_location2 = bias_location2.loc[bias_location2["num_bias_sides"] == 1, :]
+ bias_location = bias_location.merge(bias_location2, on="session_location")
# Only alert if more than 1 subject was biased today
- bias_location = bias_location.loc[bias_location['num_subjects'] > 1, :]
+ bias_location = bias_location.loc[bias_location["num_subjects"] > 1, :]
# Filter columns and set message
- columns_alert = ['session_location', 'sign_bias', 'num_subjects', 'subject_fullnames']
+ columns_alert = [
+ "session_location",
+ "sign_bias",
+ "num_subjects",
+ "subject_fullnames",
+ ]
bias_location = bias_location[columns_alert]
- bias_location['alert_message'] = 'Multiple subjects were biased in this rig'
+ bias_location["alert_message"] = "Multiple subjects were biased in this rig"
return bias_location
-
-
diff --git a/u19_pipeline/alert_system/custom_alerts/rig_trial.py b/u19_pipeline/alert_system/custom_alerts/rig_trial.py
index 9f37778d..3ad79122 100644
--- a/u19_pipeline/alert_system/custom_alerts/rig_trial.py
+++ b/u19_pipeline/alert_system/custom_alerts/rig_trial.py
@@ -1,62 +1,74 @@
-
-import pandas as pd
-import datajoint as dj
import datetime
+
import numpy as np
-import u19_pipeline.alert_system.behavior_metrics as bm
import u19_pipeline.alert_system.alert_system_utility as asu
-
+import u19_pipeline.alert_system.behavior_metrics as bm
# Slack Configuration dictionary
-slack_configuration_dictionary = {
- 'slack_notification_channel': ['custom_alerts']
-}
+slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]}
zscore_alert = 2
+
+
def main():
- '''
+ """
Main function for subject "num trials & bias" alert
- '''
+ """
# Get sessions
- subject_rig_df, _ = asu.get_acquisition_data_alert_system(type='session_location')
+ subject_rig_df, _ = asu.get_acquisition_data_alert_system(type="session_location")
# Get zscores for each session (group by rig)
- subject_rig_df = bm.BehaviorMetrics.get_zscore_metric_session_df(subject_rig_df, 'num_trials', 'session_location')
-
- # Get only todays data
+ subject_rig_df = bm.BehaviorMetrics.get_zscore_metric_session_df(subject_rig_df, "num_trials", "session_location")
+
+ # Get only todays data
today = datetime.date.today() - datetime.timedelta(days=1)
- subject_rig_df = subject_rig_df.loc[subject_rig_df['session_date'] == today, :]
+ subject_rig_df = subject_rig_df.loc[subject_rig_df["session_date"] == today, :]
# Filter if today we got > 3 zscore of trials for a session
- subject_rig_df['abs_z_score_num_trials'] = np.abs(subject_rig_df['z_score_num_trials'])
- num_trials_alert_df = subject_rig_df.loc[subject_rig_df['abs_z_score_num_trials'] >= zscore_alert, :].copy(deep=True)
+ subject_rig_df["abs_z_score_num_trials"] = np.abs(subject_rig_df["z_score_num_trials"])
+ num_trials_alert_df = subject_rig_df.loc[subject_rig_df["abs_z_score_num_trials"] >= zscore_alert, :].copy(
+ deep=True
+ )
# Get sign of zscpre (only group subjects with same deviation)
- num_trials_alert_df['sign_zscore'] = np.sign(num_trials_alert_df['z_score_num_trials'])
- num_trials_alert_df['sign_zscore'] = num_trials_alert_df['sign_zscore'].astype(int)
+ num_trials_alert_df["sign_zscore"] = np.sign(num_trials_alert_df["z_score_num_trials"])
+ num_trials_alert_df["sign_zscore"] = num_trials_alert_df["sign_zscore"].astype(int)
- #Count how many subjects were biassed by rig
- num_trials_alert_location_df = num_trials_alert_df.groupby(['session_location', 'sign_zscore']).agg({'session_location': [('num_subjects', 'size')],\
- 'subject_fullname': [('subject_fullnames', lambda x: ','.join(x))]})
+ # Count how many subjects were biassed by rig
+ num_trials_alert_location_df = num_trials_alert_df.groupby(["session_location", "sign_zscore"]).agg(
+ {
+ "session_location": [("num_subjects", "size")],
+ "subject_fullname": [("subject_fullnames", lambda x: ",".join(x))],
+ }
+ )
num_trials_alert_location_df.columns = num_trials_alert_location_df.columns.droplevel()
num_trials_alert_location_df = num_trials_alert_location_df.reset_index()
- #Filter if there were subjects with a lot of trials and very little trials
- num_trials_alert_location_df2 = num_trials_alert_location_df.groupby(['session_location']).agg({'session_location': [('num_bias_sides', 'size')]})
+ # Filter if there were subjects with a lot of trials and very little trials
+ num_trials_alert_location_df2 = num_trials_alert_location_df.groupby(["session_location"]).agg(
+ {"session_location": [("num_bias_sides", "size")]}
+ )
num_trials_alert_location_df2.columns = num_trials_alert_location_df2.columns.droplevel()
num_trials_alert_location_df2 = num_trials_alert_location_df2.reset_index()
- num_trials_alert_location_df2 = num_trials_alert_location_df2.loc[num_trials_alert_location_df2['num_bias_sides'] == 1, :]
- num_trials_alert_location_df = num_trials_alert_location_df.merge(num_trials_alert_location_df2, on='session_location')
+ num_trials_alert_location_df2 = num_trials_alert_location_df2.loc[
+ num_trials_alert_location_df2["num_bias_sides"] == 1, :
+ ]
+ num_trials_alert_location_df = num_trials_alert_location_df.merge(
+ num_trials_alert_location_df2, on="session_location"
+ )
# Only alert if more than 1 subject was biased today
- num_trials_alert_location_df = num_trials_alert_location_df.loc[num_trials_alert_location_df['num_subjects'] > 1, :]
+ num_trials_alert_location_df = num_trials_alert_location_df.loc[num_trials_alert_location_df["num_subjects"] > 1, :]
# Filter columns and set message
- columns_alert = ['session_location', 'sign_zscore', 'num_subjects', 'subject_fullnames']
+ columns_alert = [
+ "session_location",
+ "sign_zscore",
+ "num_subjects",
+ "subject_fullnames",
+ ]
num_trials_alert_location_df = num_trials_alert_location_df[columns_alert]
- num_trials_alert_location_df['alert_message'] = 'Multiple subjects had abnormal number of trials in this rig'
+ num_trials_alert_location_df["alert_message"] = "Multiple subjects had abnormal number of trials in this rig"
return num_trials_alert_location_df
-
-
diff --git a/u19_pipeline/alert_system/custom_alerts/subject_bias.py b/u19_pipeline/alert_system/custom_alerts/subject_bias.py
index 43c7a0b3..0bd7d7d7 100644
--- a/u19_pipeline/alert_system/custom_alerts/subject_bias.py
+++ b/u19_pipeline/alert_system/custom_alerts/subject_bias.py
@@ -1,49 +1,58 @@
+import datetime
-import pandas as pd
import datajoint as dj
-import datetime
import numpy as np
+import pandas as pd
-import u19_pipeline.alert_system.behavior_metrics as bm
import u19_pipeline.alert_system.alert_system_utility as asu
-
+import u19_pipeline.alert_system.behavior_metrics as bm
# Slack Configuration dictionary
-slack_configuration_dictionary = {
- 'slack_notification_channel': ['custom_alerts']
-}
+slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]}
zscore_alert = 2
+
+
def main():
- '''
+ """
Main function for subject "num trials & bias" alert
- '''
+ """
# Get sessions
- _, subject_session_key_list = asu.get_acquisition_data_alert_system(type='subject_fullname')
+ _, subject_session_key_list = asu.get_acquisition_data_alert_system(type="subject_fullname")
# Get trials
- behavior = dj.create_virtual_module('behavior', dj.config['custom']['database.prefix']+'behavior')
- acquisition = dj.create_virtual_module('acquisition', dj.config['custom']['database.prefix']+'acquisition')
+ behavior = dj.create_virtual_module("behavior", dj.config["custom"]["database.prefix"] + "behavior")
+ acquisition = dj.create_virtual_module("acquisition", dj.config["custom"]["database.prefix"] + "acquisition")
- subject_trial_df = pd.DataFrame((behavior.TowersBlock.Trial * acquisition.SessionStarted & subject_session_key_list).fetch('KEY', 'trial_type', 'choice', 'session_location', as_dict=True))
+ subject_trial_df = pd.DataFrame(
+ (behavior.TowersBlock.Trial * acquisition.SessionStarted & subject_session_key_list).fetch(
+ "KEY", "trial_type", "choice", "session_location", as_dict=True
+ )
+ )
# Get zscores for bias
bias_df = bm.BehaviorMetrics.get_bias_from_trial_df(subject_trial_df)
- bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df(bias_df, 'bias', 'subject_fullname')
+ bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df(bias_df, "bias", "subject_fullname")
# Filter df for todays alert
today = datetime.date.today() - datetime.timedelta(days=1)
- bias_df = bias_df.loc[bias_df['session_date'] == today, :]
- bias_df['abs_z_score_bias'] = np.abs(bias_df['z_score_bias'])
+ bias_df = bias_df.loc[bias_df["session_date"] == today, :]
+ bias_df["abs_z_score_bias"] = np.abs(bias_df["z_score_bias"])
# Filter if today we got > 3 zscore of trials for a session
- alert_bias_df = bias_df.loc[bias_df['abs_z_score_bias'] >= zscore_alert, :]
-
- columns_alert = ['subject_fullname', 'session_date', 'session_number', 'session_location', 'avg_bias', 'bias', 'z_score_bias']
+ alert_bias_df = bias_df.loc[bias_df["abs_z_score_bias"] >= zscore_alert, :]
+
+ columns_alert = [
+ "subject_fullname",
+ "session_date",
+ "session_number",
+ "session_location",
+ "avg_bias",
+ "bias",
+ "z_score_bias",
+ ]
alert_bias_df = alert_bias_df[columns_alert]
- alert_bias_df['alert_message'] = 'Session had abnormal bias'
+ alert_bias_df["alert_message"] = "Session had abnormal bias"
return alert_bias_df
-
-
diff --git a/u19_pipeline/alert_system/custom_alerts/subject_trial.py b/u19_pipeline/alert_system/custom_alerts/subject_trial.py
index d69c4366..00858029 100644
--- a/u19_pipeline/alert_system/custom_alerts/subject_trial.py
+++ b/u19_pipeline/alert_system/custom_alerts/subject_trial.py
@@ -1,42 +1,47 @@
-
-import pandas as pd
-import datajoint as dj
import datetime
+
import numpy as np
-import u19_pipeline.alert_system.behavior_metrics as bm
import u19_pipeline.alert_system.alert_system_utility as asu
-
+import u19_pipeline.alert_system.behavior_metrics as bm
# Slack Configuration dictionary
-slack_configuration_dictionary = {
- 'slack_notification_channel': ['custom_alerts']
-}
+slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]}
zscore_alert = 2
+
+
def main():
- '''
+ """
Main function for subject "num trials & bias" alert
- '''
+ """
# Get sessions
- subject_session_df, _ = asu.get_acquisition_data_alert_system(type='subject_fullname')
+ subject_session_df, _ = asu.get_acquisition_data_alert_system(type="subject_fullname")
# Get zscores for num_trials
- subject_session_df = bm.BehaviorMetrics.get_zscore_metric_session_df(subject_session_df, 'num_trials', 'subject_fullname')
+ subject_session_df = bm.BehaviorMetrics.get_zscore_metric_session_df(
+ subject_session_df, "num_trials", "subject_fullname"
+ )
# Filter df for todays alert
today = datetime.date.today() - datetime.timedelta(days=1)
- subject_session_df = subject_session_df.loc[subject_session_df['session_date'] == today, :]
- subject_session_df['abs_z_score_num_trials'] = np.abs(subject_session_df['z_score_num_trials'])
+ subject_session_df = subject_session_df.loc[subject_session_df["session_date"] == today, :]
+ subject_session_df["abs_z_score_num_trials"] = np.abs(subject_session_df["z_score_num_trials"])
# Filter if today we got > 3 zscore of trials for a session
- alert_subjtect_trial_df = subject_session_df.loc[subject_session_df['abs_z_score_num_trials'] >= zscore_alert, :]
-
- columns_alert = ['subject_fullname', 'session_date', 'session_number', 'session_location', 'avg_num_trials', 'num_trials', 'z_score_num_trials']
+ alert_subjtect_trial_df = subject_session_df.loc[subject_session_df["abs_z_score_num_trials"] >= zscore_alert, :]
+
+ columns_alert = [
+ "subject_fullname",
+ "session_date",
+ "session_number",
+ "session_location",
+ "avg_num_trials",
+ "num_trials",
+ "z_score_num_trials",
+ ]
alert_subjtect_trial_df = alert_subjtect_trial_df[columns_alert]
- alert_subjtect_trial_df['alert_message'] = 'Session had abnormal number of trials'
+ alert_subjtect_trial_df["alert_message"] = "Session had abnormal number of trials"
return alert_subjtect_trial_df
-
-
diff --git a/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py b/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py
index 5250566d..9f7c6c4f 100644
--- a/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py
+++ b/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py
@@ -1,6 +1,7 @@
-
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(0.1)
@@ -8,4 +9,3 @@
import u19_pipeline.alert_system.live_monitor_alert.live_monitor_alert as lma
lma.main_live_monitor_alert()
-
diff --git a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py
index 0756ff6c..47dd628a 100644
--- a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py
+++ b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py
@@ -1,65 +1,73 @@
-import datajoint as dj
-import numpy as np
-import pandas as pd
import time
-from zoneinfo import ZoneInfo
from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
+
+import datajoint as dj
+import pandas as pd
import u19_pipeline.utils.slack_utils as su
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
MINUTES_ALERT = 20
-SECONDS_ALERT = MINUTES_ALERT*60
+SECONDS_ALERT = MINUTES_ALERT * 60
MIN_SESSIONS_COMPLETED = 3
-slack_configuration_dictionary = {
- 'slack_notification_channel': ['rig_training_error_notification']
-}
+slack_configuration_dictionary = {"slack_notification_channel": ["rig_training_error_notification"]}
def slack_alert_message_format_live_stats(alert_dictionary1, alert_dictionary2, time_no_response):
- 'Format dictionaries for live monitor slack alert json'
+ "Format dictionaries for live monitor slack alert json"
now = datetime.now()
- datestr = now.strftime('%d-%b-%Y %H:%M:%S')
+ datestr = now.strftime("%d-%b-%Y %H:%M:%S")
- msep = dict()
- msep['type'] = "divider"
+ msep = {}
+ msep["type"] = "divider"
- #Title#
- m1 = dict()
- m1['type'] = 'section'
- m1_1 = dict()
+ # Title#
+ m1 = {}
+ m1["type"] = "section"
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
- m1_1["text"] = ':rotating_light: *Live Monitor Alert* on ' + datestr + '\n' +\
- 'More than ' + str(time_no_response) + ' min without new trial' + '\n'
- m1['text'] = m1_1
-
- #Info#
- m2 = dict()
- m2['type'] = 'section'
- m2_1 = dict()
+ m1_1["text"] = (
+ ":rotating_light: *Live Monitor Alert* on "
+ + datestr
+ + "\n"
+ + "More than "
+ + str(time_no_response)
+ + " min without new trial"
+ + "\n"
+ )
+ m1["text"] = m1_1
+
+ # Info#
+ m2 = {}
+ m2["type"] = "section"
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
- m2_1["text"] = '*Session Reported:*' + '\n'
- for key in alert_dictionary1.keys():
- m2_1["text"] += '*' + key + '* : ' + str(alert_dictionary1[key]) + '\n'
- m2_1["text"] += '\n'
- m2['text'] = m2_1
+ m2_1["text"] = "*Session Reported:*" + "\n"
+ for key in alert_dictionary1:
+ m2_1["text"] += "*" + key + "* : " + str(alert_dictionary1[key]) + "\n"
+ m2_1["text"] += "\n"
+ m2["text"] = m2_1
- m4 = dict()
- m4['type'] = 'section'
- m4_1 = dict()
+ m4 = {}
+ m4["type"] = "section"
+ m4_1 = {}
m4_1["type"] = "mrkdwn"
- m4_1["text"] = '*Last Stats Reported*:' + '\n'
- for key in alert_dictionary2.keys():
- m4_1["text"] += '*' + key + '* : ' + str(alert_dictionary2[key]) + '\n'
- m4_1["text"] += '\n'
- m4['text'] = m4_1
+ m4_1["text"] = "*Last Stats Reported*:" + "\n"
+ for key in alert_dictionary2:
+ m4_1["text"] += "*" + key + "* : " + str(alert_dictionary2[key]) + "\n"
+ m4_1["text"] += "\n"
+ m4["text"] = m4_1
- message = dict()
- message['blocks'] = [m1,msep,m2,msep,m4,msep]
- message['text'] = 'Live Monitor Alert'
+ message = {}
+ message["blocks"] = [m1, msep, m2, msep, m4, msep]
+ message["text"] = "Live Monitor Alert"
return message
@@ -68,178 +76,227 @@ def main_live_monitor_alert():
# Connect to DB and create virtual modules
dj.conn()
- acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')
- lab = dj.create_virtual_module('lab', 'u19_lab')
- subject = dj.create_virtual_module('subject', 'u19_subject')
+ acquisition = dj.create_virtual_module("acquisition", "u19_acquisition")
+ lab = dj.create_virtual_module("lab", "u19_lab")
+ subject = dj.create_virtual_module("subject", "u19_subject")
# Query today's started sessions that are not finished
query = {}
- query['session_date'] = datetime.today().strftime('%Y-%m-%d')
- query['is_finished'] = 0
+ query["session_date"] = datetime.today().strftime("%Y-%m-%d")
+ query["is_finished"] = 0
- #Only look for sessions started in the last 1:30
- last_time_start = datetime.now(tz=ZoneInfo('America/New_York')) - timedelta(hours=2,minutes=30)
- last_time_start = last_time_start.replace(tzinfo=None).strftime('%Y-%m-%d %H:%M:%S')
+ # Only look for sessions started in the last 1:30
+ last_time_start = datetime.now(tz=ZoneInfo("America/New_York")) - timedelta(hours=2, minutes=30)
+ last_time_start = last_time_start.replace(tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
- query_started_recently = "session_start_time > '" + last_time_start + "'"
- sessions = pd.DataFrame((acquisition.SessionStarted & query & query_started_recently).fetch('KEY','session_location','session_start_time',as_dict=True))
- #sessions = sessions.loc[~sessions['subject_fullname'].str.startswith('testuser'),:]
+ query_started_recently = "session_start_time > '" + last_time_start + "'"
+ sessions = pd.DataFrame(
+ (acquisition.SessionStarted & query & query_started_recently).fetch(
+ "KEY", "session_location", "session_start_time", as_dict=True
+ )
+ )
+ # sessions = sessions.loc[~sessions['subject_fullname'].str.startswith('testuser'),:]
- #print('sessions STARTED RECENTLY\n', sessions)
+ # print('sessions STARTED RECENTLY\n', sessions)
if sessions.shape[0] > 0:
-
- #If more than one "not finished" session in same rig, grab the last one started
- sessions2 = sessions.groupby('session_location').agg({'session_start_time': [('session_start_time', 'max')]})
+ # If more than one "not finished" session in same rig, grab the last one started
+ sessions2 = sessions.groupby("session_location").agg({"session_start_time": [("session_start_time", "max")]})
sessions2.columns = sessions2.columns.droplevel()
sessions2 = sessions2.reset_index()
- sessions = pd.merge(sessions, sessions2, on=['session_location', 'session_start_time'])
- sessions = sessions.drop(columns=['session_location'])
+ sessions = pd.merge(sessions, sessions2, on=["session_location", "session_start_time"])
+ sessions = sessions.drop(columns=["session_location"])
sessions = sessions.reset_index(drop=True)
- print('Last session started on rig\n', sessions)
+ logger.info("Last session started on rig\n%s", sessions)
- #Only analyze sessions that have not been reported
- query_reported = {}
- query_reported['session_date'] = datetime.today().strftime('%Y-%m-%d')
- sessions_reported = pd.DataFrame((acquisition.ReportedLiveSessionStats & query_reported).fetch('KEY', as_dict = True))
+ # Only analyze sessions that have not been reported
+ query_reported = {}
+ query_reported["session_date"] = datetime.today().strftime("%Y-%m-%d")
+ sessions_reported = pd.DataFrame(
+ (acquisition.ReportedLiveSessionStats & query_reported).fetch("KEY", as_dict=True)
+ )
if sessions_reported.shape[0] > 0:
-
- sessions = pd.merge(sessions,sessions_reported, how='left', indicator=True)
- sessions = sessions.loc[sessions['_merge'] == 'left_only']
- sessions = sessions.drop(columns='_merge')
+ sessions = pd.merge(sessions, sessions_reported, how="left", indicator=True)
+ sessions = sessions.loc[sessions["_merge"] == "left_only"]
+ sessions = sessions.drop(columns="_merge")
sessions = sessions.reset_index(drop=True)
- #print('Sessions not reported\n', sessions)
+ # print('Sessions not reported\n', sessions)
- #Only analyze sessions subjects > NUM_SESSIONS_COMPLETED have not been reported
+ # Only analyze sessions subjects > NUM_SESSIONS_COMPLETED have not been reported
if sessions.shape[0] > 0:
+ query_subjects = "subject_fullname in ('" + "', '".join(sessions["subject_fullname"]) + "')"
- query_subjects = "subject_fullname in ('"+ "', '".join(sessions['subject_fullname']) + "')"
-
- count_sessions_table = (subject.Subject).aggr((acquisition.Session & query_subjects), num_sessions="count(subject_fullname)")
+ count_sessions_table = (subject.Subject).aggr(
+ (acquisition.Session & query_subjects),
+ num_sessions="count(subject_fullname)",
+ )
count_sessions_df = pd.DataFrame(count_sessions_table.fetch(as_dict=True))
- sessions = pd.merge(sessions,count_sessions_df, how='left')
- sessions['num_sessions'] = sessions['num_sessions'].fillna(0)
-
- #print('Subjects with min_num_sessions\n', sessions)
+ sessions = pd.merge(sessions, count_sessions_df, how="left")
+ sessions["num_sessions"] = sessions["num_sessions"].fillna(0)
- sessions = sessions.loc[sessions['num_sessions']>= MIN_SESSIONS_COMPLETED, :]
+ # print('Subjects with min_num_sessions\n', sessions)
+ sessions = sessions.loc[sessions["num_sessions"] >= MIN_SESSIONS_COMPLETED, :]
if sessions.shape[0] > 0:
-
# Query last live stat from the started sessions
- query_live_stats = sessions.to_dict('records')
-
- #Last non violation trial in sessions
- query_no_violation_trial = dict()
- query_no_violation_trial['violation_trial'] = 0
- lss_nvio = acquisition.SessionStarted.aggr((acquisition.LiveSessionStats & query_no_violation_trial).proj('current_datetime'), current_datetime="max(current_datetime)")
+ query_live_stats = sessions.to_dict("records")
+
+ # Last non violation trial in sessions
+ query_no_violation_trial = {}
+ query_no_violation_trial["violation_trial"] = 0
+ lss_nvio = acquisition.SessionStarted.aggr(
+ (acquisition.LiveSessionStats & query_no_violation_trial).proj("current_datetime"),
+ current_datetime="max(current_datetime)",
+ )
live_stats_nvio = pd.DataFrame((lss_nvio & query_live_stats).fetch(as_dict=True))
- live_stats_nvio = live_stats_nvio.rename({'current_datetime': 'last_non_violation_trial'}, axis=1)
-
- #Last violation trial in sessions
- query_violation_trial = dict()
- query_violation_trial['violation_trial'] = 1
- lss_vio = acquisition.SessionStarted.aggr((acquisition.LiveSessionStats & query_violation_trial).proj('current_datetime'), current_datetime="max(current_datetime)")
+ live_stats_nvio = live_stats_nvio.rename({"current_datetime": "last_non_violation_trial"}, axis=1)
+
+ # Last violation trial in sessions
+ query_violation_trial = {}
+ query_violation_trial["violation_trial"] = 1
+ lss_vio = acquisition.SessionStarted.aggr(
+ (acquisition.LiveSessionStats & query_violation_trial).proj("current_datetime"),
+ current_datetime="max(current_datetime)",
+ )
live_stats_vio = pd.DataFrame((lss_vio & query_live_stats & query_violation_trial).fetch(as_dict=True))
- live_stats_vio = live_stats_vio.rename({'current_datetime': 'last_violation_trial'}, axis=1)
-
+ live_stats_vio = live_stats_vio.rename({"current_datetime": "last_violation_trial"}, axis=1)
# Merge last violation trial and last non violation Trials from sessions
if live_stats_nvio.shape[0] > 0 and live_stats_vio.shape[0] > 0:
- live_stats = pd.merge(live_stats_nvio,live_stats_vio, how='outer')
+ live_stats = pd.merge(live_stats_nvio, live_stats_vio, how="outer")
elif live_stats_nvio.shape[0] > 0:
live_stats = live_stats_nvio.copy()
- live_stats['last_violation_trial'] = pd.NaT
+ live_stats["last_violation_trial"] = pd.NaT
elif live_stats_vio.shape[0] > 0:
live_stats = live_stats_vio.copy()
- live_stats['last_non_violation_trial'] = pd.NaT
+ live_stats["last_non_violation_trial"] = pd.NaT
else:
live_stats = pd.DataFrame()
-
# If there are any sessions with live stats
if live_stats.shape[0] > 0:
-
- live_stats = pd.merge(sessions,live_stats, how='inner')
- fake_date = pd.Timestamp('1900-01-01')
+ live_stats = pd.merge(sessions, live_stats, how="inner")
+ fake_date = pd.Timestamp("1900-01-01")
# Filter sessions whose last trial info is greater than 300s
- right_now_est = datetime.now(tz=ZoneInfo('America/New_York'))
+ right_now_est = datetime.now(tz=ZoneInfo("America/New_York"))
right_now_est = right_now_est.replace(tzinfo=None)
- live_stats['seconds_elapsed_last_stat_nvio'] = ((right_now_est- live_stats['last_non_violation_trial']).dt.total_seconds()).astype(int)
- live_stats['alert_nvio'] = live_stats['seconds_elapsed_last_stat_nvio'] > SECONDS_ALERT
-
- live_stats['seconds_elapsed_session_started'] = ((right_now_est- live_stats['session_start_time']).dt.total_seconds()).astype(int)
- live_stats['alert_vio'] = live_stats['seconds_elapsed_session_started'] > SECONDS_ALERT
- live_stats['alert_vio'] = live_stats['alert_vio'] & (pd.isna(live_stats['last_non_violation_trial']))
- live_stats['alert_vio'] = live_stats['alert_vio'] & (~pd.isna(live_stats['last_violation_trial']))
-
- print(live_stats.T)
-
- live_stats = live_stats.loc[(live_stats['alert_nvio']==True) | (live_stats['alert_vio']==True),:]
-
- #If there are any sessions to alert (more then SECONDS_ALERT)
- if live_stats.shape[0] > 0:
-
- live_stats['current_datetime'] = fake_date
- live_stats.loc[live_stats['alert_vio']==True, 'current_datetime'] = live_stats.loc[live_stats['alert_vio']==True, 'last_violation_trial']
- live_stats.loc[live_stats['alert_nvio']==True, 'current_datetime'] = live_stats.loc[live_stats['alert_nvio']==True, 'last_non_violation_trial']
+ live_stats["seconds_elapsed_last_stat_nvio"] = (
+ (right_now_est - live_stats["last_non_violation_trial"]).dt.total_seconds()
+ ).astype(int)
+ live_stats["alert_nvio"] = live_stats["seconds_elapsed_last_stat_nvio"] > SECONDS_ALERT
- live_stats['seconds_elapsed_last_valid_stat'] = 0
- live_stats.loc[live_stats['alert_vio']==True, 'seconds_elapsed_last_valid_stat'] = live_stats.loc[live_stats['alert_vio']==True, 'seconds_elapsed_session_started']
- live_stats.loc[live_stats['alert_nvio']==True, 'seconds_elapsed_last_valid_stat'] = live_stats.loc[live_stats['alert_nvio']==True, 'seconds_elapsed_last_stat_nvio']
+ live_stats["seconds_elapsed_session_started"] = (
+ (right_now_est - live_stats["session_start_time"]).dt.total_seconds()
+ ).astype(int)
+ live_stats["alert_vio"] = live_stats["seconds_elapsed_session_started"] > SECONDS_ALERT
+ live_stats["alert_vio"] = live_stats["alert_vio"] & (pd.isna(live_stats["last_non_violation_trial"]))
+ live_stats["alert_vio"] = live_stats["alert_vio"] & (~pd.isna(live_stats["last_violation_trial"]))
- #get_session_info to alert (plus slack researcher)
- query_live_stats_sessions = live_stats[['subject_fullname', 'session_date', 'session_number']].to_dict('records')
+ logger.debug("live_stats %s", live_stats.T)
- session_data_df = pd.DataFrame(((lab.User.proj('slack') * subject.Subject.proj('user_id') *\
- acquisition.SessionStarted.proj('session_location')) & query_live_stats_sessions).fetch(as_dict=True))
-
- session_data_df = session_data_df.rename({'slack': 'researcher'}, axis=1)
- session_data_df['researcher'] = '<@'+ session_data_df['researcher'] + '>'
- session_data_df = session_data_df[['researcher', 'subject_fullname', 'session_date', 'session_number']]
+ live_stats = live_stats.loc[
+ (live_stats["alert_nvio"]) | (live_stats["alert_vio"]),
+ :,
+ ]
- #Query full live stat table
- #session_stats = live_stats.copy()
- #session_stats = session_stats.rename({'current_datetime': 'last_live_stat'}, axis=1)
- query_live_stats = live_stats[['subject_fullname', 'session_date', 'session_number', 'current_datetime']].to_dict('records')
- live_stats_mini = live_stats[['subject_fullname', 'session_date', 'session_number', 'seconds_elapsed_last_valid_stat']].copy()
+ # If there are any sessions to alert (more then SECONDS_ALERT)
+ if live_stats.shape[0] > 0:
+ live_stats["current_datetime"] = fake_date
+ live_stats.loc[live_stats["alert_vio"], "current_datetime"] = live_stats.loc[
+ live_stats["alert_vio"], "last_violation_trial"
+ ]
+ live_stats.loc[live_stats["alert_nvio"], "current_datetime"] = live_stats.loc[
+ live_stats["alert_nvio"], "last_non_violation_trial"
+ ]
+
+ live_stats["seconds_elapsed_last_valid_stat"] = 0
+ live_stats.loc[live_stats["alert_vio"], "seconds_elapsed_last_valid_stat"] = live_stats.loc[
+ live_stats["alert_vio"], "seconds_elapsed_session_started"
+ ]
+ live_stats.loc[live_stats["alert_nvio"], "seconds_elapsed_last_valid_stat"] = live_stats.loc[
+ live_stats["alert_nvio"], "seconds_elapsed_last_stat_nvio"
+ ]
+
+ # get_session_info to alert (plus slack researcher)
+ query_live_stats_sessions = live_stats[["subject_fullname", "session_date", "session_number"]].to_dict(
+ "records"
+ )
+
+ session_data_df = pd.DataFrame(
+ (
+ (
+ lab.User.proj("slack")
+ * subject.Subject.proj("user_id")
+ * acquisition.SessionStarted.proj("session_location")
+ )
+ & query_live_stats_sessions
+ ).fetch(as_dict=True)
+ )
+
+ session_data_df = session_data_df.rename({"slack": "researcher"}, axis=1)
+ session_data_df["researcher"] = "<@" + session_data_df["researcher"] + ">"
+ session_data_df = session_data_df[["researcher", "subject_fullname", "session_date", "session_number"]]
+
+ # Query full live stat table
+ # session_stats = live_stats.copy()
+ # session_stats = session_stats.rename({'current_datetime': 'last_live_stat'}, axis=1)
+ query_live_stats = live_stats[
+ [
+ "subject_fullname",
+ "session_date",
+ "session_number",
+ "current_datetime",
+ ]
+ ].to_dict("records")
+ live_stats_mini = live_stats[
+ [
+ "subject_fullname",
+ "session_date",
+ "session_number",
+ "seconds_elapsed_last_valid_stat",
+ ]
+ ].copy()
ls_full_df = pd.DataFrame((acquisition.LiveSessionStats & query_live_stats).fetch(as_dict=True))
- ls_full_df = pd.merge(ls_full_df, live_stats_mini, on=['subject_fullname', 'session_date', 'session_number'])
- ls_full_df = ls_full_df.drop(columns=['subject_fullname', 'session_date', 'session_number'])
- ls_full_df = ls_full_df.rename({'current_datetime': 'last_trial_time'}, axis=1)
+ ls_full_df = pd.merge(
+ ls_full_df,
+ live_stats_mini,
+ on=["subject_fullname", "session_date", "session_number"],
+ )
+ ls_full_df = ls_full_df.drop(columns=["subject_fullname", "session_date", "session_number"])
+ ls_full_df = ls_full_df.rename({"current_datetime": "last_trial_time"}, axis=1)
+ mid = ls_full_df["last_trial_time"]
+ ls_full_df = ls_full_df.drop(columns=["last_trial_time"])
+ ls_full_df.insert(0, "last_trial_time", mid)
- mid = ls_full_df['last_trial_time']
- ls_full_df = ls_full_df.drop(columns=['last_trial_time'])
- ls_full_df.insert(0, 'last_trial_time', mid)
-
- ls_full_dict = ls_full_df.to_dict('records')
+ ls_full_dict = ls_full_df.to_dict("records")
# Send one alert per session found
- idx_alert = 0
- for this_alert_record in ls_full_dict:
-
- #Format message for session and live stat dictionary
- this_session_stats = session_data_df.iloc[idx_alert,:]
- slack_json_message = slack_alert_message_format_live_stats(this_session_stats.to_dict(), this_alert_record, int(this_alert_record['seconds_elapsed_last_valid_stat']/60))
-
- #Send alert
+ for idx_alert, this_alert_record in enumerate(ls_full_dict):
+ # Format message for session and live stat dictionary
+ this_session_stats = session_data_df.iloc[idx_alert, :]
+ slack_json_message = slack_alert_message_format_live_stats(
+ this_session_stats.to_dict(),
+ this_alert_record,
+ int(this_alert_record["seconds_elapsed_last_valid_stat"] / 60),
+ )
+
+ # Send alert
webhooks_list = su.get_webhook_list(slack_configuration_dictionary, lab)
for this_webhook in webhooks_list:
su.send_slack_notification(this_webhook, slack_json_message)
time.sleep(1)
- reported_session = this_session_stats[['subject_fullname', 'session_date', 'session_number']].copy()
- reported_session['report_datetime'] = right_now_est
+ reported_session = this_session_stats[["subject_fullname", "session_date", "session_number"]].copy()
+ reported_session["report_datetime"] = right_now_est
acquisition.ReportedLiveSessionStats.insert1(reported_session.to_dict())
- idx_alert += 1
diff --git a/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py b/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py
index 7e199601..f08de720 100644
--- a/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py
+++ b/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py
@@ -1,6 +1,7 @@
-
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(0.1)
@@ -8,4 +9,3 @@
import u19_pipeline.alert_system.live_session_stats_deletion.live_session_stats_deletion as lssd
lssd.main_live_session_stats_deletion()
-
diff --git a/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py b/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py
index f22e071a..7a900943 100644
--- a/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py
+++ b/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py
@@ -1,16 +1,17 @@
-
import datajoint as dj
+
def main_live_session_stats_deletion():
- acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')
+ acquisition = dj.create_virtual_module("acquisition", "u19_acquisition")
old_session_stats_query = "current_datetime < NOW() - INTERVAL 5 DAY"
order_cols = "'subject_fullname', 'session_date', 'session_number', 'trial_idx'"
connection = acquisition.LiveSessionStats.connection
with connection.transaction:
- old_session_stats = (acquisition.LiveSessionStats & old_session_stats_query).fetch(as_dict=True, order_by=[order_cols])
+ old_session_stats = (acquisition.LiveSessionStats & old_session_stats_query).fetch(
+ as_dict=True, order_by=[order_cols]
+ )
(acquisition.HistoricSessionStats).insert(old_session_stats, skip_duplicates=True)
(acquisition.LiveSessionStats & old_session_stats_query).delete(safemode=False)
-
diff --git a/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py b/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py
index f7f04033..03fb6034 100644
--- a/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py
+++ b/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py
@@ -1,6 +1,7 @@
-
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(0.1)
@@ -8,4 +9,3 @@
import u19_pipeline.alert_system.locked_tables_alert.locked_tables_alert as lta
lta.main_locked_tables_alert()
-
diff --git a/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py b/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py
index 4f2aa442..6772acdd 100644
--- a/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py
+++ b/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py
@@ -4,18 +4,16 @@
import datajoint as dj
import pandas as pd
-import u19_pipeline.utils.slack_utils as su
import u19_pipeline.lab as lab
+import u19_pipeline.utils.slack_utils as su
# Slack Configuration dictionary
-slack_configuration_dictionary = {
- 'slack_notification_channel': ['dev_notifications']
-}
+slack_configuration_dictionary = {"slack_notification_channel": ["dev_notifications"]}
def main_locked_tables_alert():
- locked_tables_query = 'show open tables where in_use > 0'
+ locked_tables_query = "show open tables where in_use > 0"
conn = dj.conn()
locked_tables_df = pd.DataFrame(conn.query(locked_tables_query, as_dict=True).fetchall())
@@ -23,7 +21,7 @@ def main_locked_tables_alert():
return
else:
locked_tables_df = locked_tables_df.head()
- locked_tables_df = locked_tables_df.drop('Name_locked',axis=1)
+ locked_tables_df = locked_tables_df.drop("Name_locked", axis=1)
locked_tables_df = su.format_df_for_slack_message(locked_tables_df)
slack_json_message = slack_alert_message_format_locked_tables(locked_tables_df)
@@ -36,30 +34,30 @@ def main_locked_tables_alert():
def slack_alert_message_format_locked_tables(locked_tables_df):
now = datetime.datetime.now()
- datestr = now.strftime("%d-%b-%Y %H:%M:%S")
+ now.strftime("%d-%b-%Y %H:%M:%S")
- msep = dict()
+ msep = {}
msep["type"] = "divider"
# Title#
- m1 = dict()
+ m1 = {}
m1["type"] = "section"
- m1_1 = dict()
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
m1_1["text"] = ":rotating_light: *Locked Tables Alert *"
m1["text"] = m1_1
# Info for subjects missing water
- m2 = dict()
+ m2 = {}
m2["type"] = "section"
- m2_1 = dict()
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
m2_1["text"] = "*Locked tables:*" + "\n"
m2_1["text"] += locked_tables_df
m2["text"] = m2_1
- message = dict()
+ message = {}
message["blocks"] = [m1, msep, m2, msep]
message["text"] = "Locked Tables Alert"
diff --git a/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py b/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py
index 9f49f5fd..f1954636 100644
--- a/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py
+++ b/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py
@@ -1,6 +1,7 @@
-
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(0.1)
@@ -8,4 +9,3 @@
import u19_pipeline.alert_system.log_deletion.old_log_deletion as old
old.main_old_log_deletion()
-
diff --git a/u19_pipeline/alert_system/log_deletion/old_log_deletion.py b/u19_pipeline/alert_system/log_deletion/old_log_deletion.py
index 0969658c..395208e1 100644
--- a/u19_pipeline/alert_system/log_deletion/old_log_deletion.py
+++ b/u19_pipeline/alert_system/log_deletion/old_log_deletion.py
@@ -1,8 +1,8 @@
-
-import pandas as pd
+import datetime
import pathlib
import re
-import datetime
+
+import pandas as pd
def extract_filename(x):
@@ -11,21 +11,20 @@ def extract_filename(x):
def main_old_log_deletion():
- DIRECTORY_PATH = pathlib.Path('/home/u19prod@pu.win.princeton.edu/log')
+ DIRECTORY_PATH = pathlib.Path("/home/u19prod@pu.win.princeton.edu/log")
date_pattern = r"_\d{8}_"
reference_date = datetime.date.today() - datetime.timedelta(days=10)
files = [p for p in DIRECTORY_PATH.rglob("*") if p.is_file() and re.search(date_pattern, p.name)]
+ files_df = pd.DataFrame(files, columns=["filepaths"])
+ files_df["filename"] = files_df["filepaths"].apply(extract_filename)
+ files_df["exctract_date"] = files_df["filename"].str.extract(r"(\d{8," + str(8) + r"})")
+ files_df["exctract_date"] = pd.to_datetime(files_df["exctract_date"], format="%Y%m%d")
+ files_df["before_ref_date"] = files_df["exctract_date"] <= pd.Timestamp(reference_date)
- files_df = pd.DataFrame(files, columns=['filepaths'])
- files_df['filename'] = files_df['filepaths'].apply(extract_filename)
- files_df['exctract_date'] = files_df['filename'].str.extract(r'(\d{8,' + str(8) + r'})')
- files_df['exctract_date'] = pd.to_datetime(files_df['exctract_date'], format='%Y%m%d')
- files_df['before_ref_date'] = files_df['exctract_date'] <= pd.Timestamp(reference_date)
-
- files_for_deletion = files_df.loc[files_df['before_ref_date']==True, 'filepaths'].to_list()
+ files_for_deletion = files_df.loc[files_df["before_ref_date"], "filepaths"].to_list()
for file_path in files_for_deletion:
if file_path.is_file(): # Check if it's actually a file before attempting to delete
- file_path.unlink()
\ No newline at end of file
+ file_path.unlink()
diff --git a/u19_pipeline/alert_system/main_alert_system.py b/u19_pipeline/alert_system/main_alert_system.py
index 5ca5ae44..f3f12883 100644
--- a/u19_pipeline/alert_system/main_alert_system.py
+++ b/u19_pipeline/alert_system/main_alert_system.py
@@ -1,21 +1,19 @@
-
-import pandas as pd
-import datajoint as dj
-import pkgutil
-import importlib
import datetime
+import importlib
+import pkgutil
+import time
import traceback
import u19_pipeline.alert_system.custom_alerts as ca
import u19_pipeline.lab as lab
import u19_pipeline.utils.slack_utils as su
+from u19_pipeline.utils.logging_config import get_logger
-import time
+logger = get_logger(__name__)
# Slack Configuration dictionary
-slack_configuration_dictionary = {
- 'slack_notification_channel': ['custom_alerts']
-}
+slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]}
+
def main_alert_system():
'Call main function of all alerts defined in "custom_alerts'
@@ -23,29 +21,29 @@ def main_alert_system():
all_alert_submodules = pkgutil.iter_modules(ca.__path__)
for this_alert_submodule in all_alert_submodules:
+ logger.info("executing %s alert code", this_alert_submodule.name)
- print('executing '+ this_alert_submodule.name+ " alert code")
-
- my_alert_module = importlib.import_module('u19_pipeline.alert_system.custom_alerts.'+this_alert_submodule.name)
+ my_alert_module = importlib.import_module(
+ "u19_pipeline.alert_system.custom_alerts." + this_alert_submodule.name
+ )
try:
alert_df = my_alert_module.main()
- if hasattr(my_alert_module, 'slack_configuration_dictionary'):
+ if hasattr(my_alert_module, "slack_configuration_dictionary"):
slack_dict = my_alert_module.slack_configuration_dictionary
else:
slack_dict = slack_configuration_dictionary
if alert_df.shape[0] > 0:
-
- alert_dict = alert_df.to_dict('records')
+ alert_dict = alert_df.to_dict("records")
webhooks_list = []
- if 'slack_notification_channel' in slack_dict:
- query_slack_webhooks = [{'webhook_name' : x} for x in slack_dict['slack_notification_channel']]
- webhooks_list += (lab.SlackWebhooks & query_slack_webhooks).fetch('webhook_url').tolist()
+ if "slack_notification_channel" in slack_dict:
+ query_slack_webhooks = [{"webhook_name": x} for x in slack_dict["slack_notification_channel"]]
+ webhooks_list += (lab.SlackWebhooks & query_slack_webhooks).fetch("webhook_url").tolist()
- if 'slack_users_channel' in slack_dict:
- query_slack_user_channels = [{'user_id' : x} for x in slack_dict['slack_users_channel']]
- webhooks_list += (lab.User & query_slack_user_channels).fetch('slack_webhook').tolist()
+ if "slack_users_channel" in slack_dict:
+ query_slack_user_channels = [{"user_id": x} for x in slack_dict["slack_users_channel"]]
+ webhooks_list += (lab.User & query_slack_user_channels).fetch("slack_webhook").tolist()
for this_alert_record in alert_dict:
slack_json_message = slack_alert_message_format(this_alert_record, this_alert_submodule.name)
@@ -55,45 +53,46 @@ def main_alert_system():
time.sleep(1)
except Exception as e:
- dict_error = dict()
- dict_error['message'] = 'error while executing ' + this_alert_submodule.name + " alert code"
- dict_error['error_exception'] = (''.join(traceback.format_exception(type(e), value=e, tb=e.__traceback__)))
+ dict_error = {}
+ dict_error["message"] = "error while executing " + this_alert_submodule.name + " alert code"
+ dict_error["error_exception"] = "".join(traceback.format_exception(type(e), value=e, tb=e.__traceback__))
slack_json_message = slack_alert_message_format(dict_error, this_alert_submodule.name)
- webhook_custom = (lab.SlackWebhooks & "webhook_name='custom_alerts'").fetch('webhook_url').tolist()
+ webhook_custom = (lab.SlackWebhooks & "webhook_name='custom_alerts'").fetch("webhook_url").tolist()
if webhook_custom:
su.send_slack_notification(webhook_custom[0], slack_json_message)
del my_alert_module
+
def slack_alert_message_format(alert_dictionaty, alert_module_name):
now = datetime.datetime.now()
- datestr = now.strftime('%d-%b-%Y %H:%M:%S')
+ datestr = now.strftime("%d-%b-%Y %H:%M:%S")
- msep = dict()
- msep['type'] = "divider"
+ msep = {}
+ msep["type"] = "divider"
- #Title#
- m1 = dict()
- m1['type'] = 'section'
- m1_1 = dict()
+ # Title#
+ m1 = {}
+ m1["type"] = "section"
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
- m1_1["text"] = ':rotating_light: *' +alert_module_name + '* on ' + datestr + '\n\n'
- m1['text'] = m1_1
+ m1_1["text"] = ":rotating_light: *" + alert_module_name + "* on " + datestr + "\n\n"
+ m1["text"] = m1_1
- #Info#
- m2 = dict()
- m2['type'] = 'section'
- m2_1 = dict()
+ # Info#
+ m2 = {}
+ m2["type"] = "section"
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
- m2_1["text"] = ''
- for key in alert_dictionaty.keys():
- m2_1["text"] += '*' + key + '* : ' + str(alert_dictionaty[key]) + '\n'
- m2['text'] = m2_1
+ m2_1["text"] = ""
+ for key in alert_dictionaty:
+ m2_1["text"] += "*" + key + "* : " + str(alert_dictionaty[key]) + "\n"
+ m2["text"] = m2_1
- message = dict()
- message['blocks'] = [m1,msep,m2,msep,msep]
- message['text'] = alert_module_name+ ' alert'
+ message = {}
+ message["blocks"] = [m1, msep, m2, msep, msep]
+ message["text"] = alert_module_name + " alert"
return message
diff --git a/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py b/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py
deleted file mode 100644
index f143beb1..00000000
--- a/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py
+++ /dev/null
@@ -1,239 +0,0 @@
-
-from scripts.conf_file_finding import try_find_conf_file
-try_find_conf_file()
-
-import datajoint as dj
-import pandas as pd
-import time
-from zoneinfo import ZoneInfo
-import datetime
-import pathlib
-import numpy as np
-import time
-import base64
-import shutil
-import openpyxl
-from openpyxl.utils.dataframe import dataframe_to_rows
-from openpyxl.drawing.image import Image
-
-time.sleep(1)
-
-import u19_pipeline.alert_system.water_weigh_alert.water_weigh_alert as wwa
-import u19_pipeline.scheduler as scheduler
-import u19_pipeline.acquisition as acquisition
-import u19_pipeline.behavior as behavior
-import u19_pipeline.subject as subject
-import u19_pipeline.lab as lab
-
-
-DJ_CUSTOM_VARIABLES_FILENAME = 'DJCustomVariables.csv'
-SLACK_WEBHOOK_FILENAME = 'SlackChannels.csv'
-USER_SLACK_FILENAME = 'UserSlack.csv'
-RIG_STATUS_FILENAME = 'RigStatusTable.csv'
-DAY_SCHEDULE_FILENAME = 'ScheduleDay.csv'
-PAST_SESSION_PERFORMANCE_FILENAME = 'PastSessions.csv'
-SUBJECT_MOTOR_POSITION_FILENAME = 'SubjectMotorPosition.csv'
-
-WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = 'Weighing_GUI_Replacement_SpreadSheet_Template.xlsx'
-WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = 'Weighing_GUI_Replacement_SpreadSheet.xlsx'
-
-conf = dj.config
-nodb_virmen_backup_dir = pathlib.Path(pathlib.Path(conf['custom']['root_data_dir'][0]).parent.parent,'Shared','NoDBVirmenBackup')
-
-MAX_SESSIONS_HISTORY = 75
-
-def cast_choice(choice_array):
-
- new_array = choice_array[0].copy()
- new_array[new_array>2] = 127
-
- new_array = np.array(new_array, dtype=np.uint8)
-
- return new_array
-
-
-def encode_webhook(webhook):
-
- return (base64.b64encode(webhook.encode('utf-8'))).decode('utf-8')
-
-
-
-def write_dj_custom_vars_file():
-
- file_write = pathlib.Path(nodb_virmen_backup_dir, DJ_CUSTOM_VARIABLES_FILENAME)
- pd.DataFrame(lab.DjCustomVariables.fetch(as_dict=True)).to_csv(file_write)
-
-def write_slack_webhooks_file():
-
- slack_webhooks = pd.DataFrame(lab.SlackWebhooks.fetch(as_dict=True))
- slack_webhooks['webhook_url'] = slack_webhooks['webhook_url'].astype(str)
-
- slack_webhooks['webhook_url'] = slack_webhooks['webhook_url'].apply(encode_webhook)
-
- file_write = pathlib.Path(nodb_virmen_backup_dir, SLACK_WEBHOOK_FILENAME)
- slack_webhooks.to_csv(file_write, index=False)
-
-def write_user_data_file():
-
-
- user_data = pd.DataFrame(lab.User.fetch('user_id', 'slack', 'tech_responsibility', 'slack_webhook',as_dict=True))
- user_data['slack_webhook'] = user_data['slack_webhook'].astype(str)
- user_data['slack_webhook'] = user_data['slack_webhook'].fillna('')
-
- user_data['slack_webhook'] = user_data['slack_webhook'].apply(encode_webhook)
-
- file_write = pathlib.Path(nodb_virmen_backup_dir, USER_SLACK_FILENAME)
- user_data.to_csv(file_write, index=False)
-
-def write_rig_status_file():
-
- file_write = pathlib.Path(nodb_virmen_backup_dir, RIG_STATUS_FILENAME)
-
- df_rig_status = pd.DataFrame(scheduler.RigStatus.fetch('location', 'input_output_name','current_status',as_dict=True))
- df_rig_status.to_csv(file_write, index=False)
-
-
-def write_schedule_file():
-
- schedule_query = dict()
- schedule_query['date'] = datetime.date.today()
-
- subject_query = 'subject_fullname is not null'
-
- day_schedule = pd.DataFrame((scheduler.Schedule * scheduler.TrainingProfile * subject.Subject.proj(subject_user_id='user_id') & schedule_query & subject_query).fetch(as_dict=True))
- day_schedule = day_schedule.drop(columns='user_id')
- day_schedule = day_schedule.rename(columns={'subject_user_id':'user_id'})
-
- file_write = pathlib.Path(nodb_virmen_backup_dir, DAY_SCHEDULE_FILENAME)
- day_schedule.to_csv(file_write, index=False)
-
- return day_schedule
-
-
-def write_past_sessions_file(day_schedule):
-
- all_subjects_schedule = "', '".join(day_schedule['subject_fullname'])
- all_subjects_schedule = "subject_fullname in ('" +all_subjects_schedule+ "')"
-
- ss = pd.DataFrame((behavior.TowersSession & all_subjects_schedule).fetch('KEY', order_by='subject_fullname, session_date desc, session_number desc', as_dict=True))
- ss['session_date'] = ss['session_date'].astype(str)
- ss['num_sessions'] = ss.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')
- ss['num_sessions'] = ss['num_sessions'].astype(int)
- ss = ss.loc[ss['num_sessions'] <= MAX_SESSIONS_HISTORY, :]
-
- max_value_indices = ss.groupby('subject_fullname')['num_sessions'].idxmax()
- ss = ss.loc[max_value_indices]
- ss = ss.reset_index(drop=True)
- ss['query'] = "(subject_fullname='" + ss['subject_fullname'] + "' and session_date >= '" + ss['session_date'] + "')"
-
- block_query = ' OR '.join(ss['query'])
-
- sstable=(acquisition.SessionStarted.proj('local_path_behavior_file', 'session_location'))
- stable = (acquisition.Session).proj(stimulusBank='stimulus_bank')
- tstable = (behavior.TowersSession.proj(trialType='rewarded_side',choice='chosen_side', stimulusSet='stimulus_set'))
- tbtable = (behavior.TowersBlock.proj('first_trial','n_trials', 'sublevel', mazeID='level', mainMazeID='main_level', easyBlockFlag='easy_block',\
- duration='block_duration', rewardMil='reward_mil', medianTrialDur='trial_duration_median', start='block_start_time'))
- table_fetch = sstable * stable* tstable * tbtable
-
- allblocks = pd.DataFrame((table_fetch & block_query).fetch(order_by='subject_fullname, session_date desc, session_number desc, block desc',as_dict=True))
- allblocks['from_DB'] = 1
- allblocks['session_date'] = allblocks['session_date'].astype(str)
-
- allblocks['sublevel'] = allblocks['sublevel'].astype('Int64')
- allblocks['choice'] = allblocks['choice'].apply(cast_choice)
- allblocks['trialType'] = allblocks['trialType'].apply(cast_choice)
-
- file_write = pathlib.Path(nodb_virmen_backup_dir, PAST_SESSION_PERFORMANCE_FILENAME)
- allblocks.to_csv(file_write, index=False)
-
-def write_subject_motor_position(day_schedule):
-
- all_subjects_schedule = "', '".join(day_schedule['subject_fullname'])
- all_subjects_schedule = "subject_fullname in ('" +all_subjects_schedule+ "')"
-
- sp = pd.DataFrame((subject.HeadMotorPosition & all_subjects_schedule).fetch(as_dict=True))
-
- file_write = pathlib.Path(nodb_virmen_backup_dir, SUBJECT_MOTOR_POSITION_FILENAME)
- sp.to_csv(file_write, index=False)
-
-
-def write_weighinig_gui_ss_file():
-
- subject_data = wwa.get_subject_data()
- subject_data = subject_data.loc[:, ['user_id', 'subject_fullname', 'cage', 'headplate_image_path', 'last_weight', 'water_per_day']]
- #subject_data = subject_data.loc[subject_data['user_id'] != 'testuser',:]
- subject_data = subject_data.reset_index(drop=True)
-
- dictionary_rename_cols =\
- {'user_id': 'owner',
- 'subject_fullname': 'subject name',
- 'cage': 'cage',
- 'headplate_image_path': 'headplate',
- 'last_weight': 'last weight',
- 'water_per_day': 'water per day',
- }
-
- subject_data = subject_data.rename(columns=dictionary_rename_cols)
- subject_data = subject_data[list(dictionary_rename_cols.values())]
- subject_data['headplate'] = subject_data['headplate'].astype(str)
-
- headplate_image_paths = subject_data['headplate'].copy()
-
- subject_data['last weight'] = subject_data['last weight'].round(1)
- subject_data['water per day'] = subject_data['water per day'].round(1)
-
-
- subject_data['headplate'] = ''
- subject_data['today weight'] = ''
- subject_data['today water'] = ''
-
- # Copy the source file to the destination, replacing if it exists
- template_ss_file = pathlib.Path(nodb_virmen_backup_dir,WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE)
- ss_file = pathlib.Path(nodb_virmen_backup_dir,WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME)
- shutil.copy2(template_ss_file, ss_file)
-
- book = openpyxl.load_workbook(ss_file)
- sheet = book['Sheet1']
-
-
- for r_idx, row in enumerate(dataframe_to_rows(subject_data, index=False, header=False), start=3):
- for c_idx, value in enumerate(row, start=1):
- sheet.cell(row=r_idx, column=c_idx, value=value)
-
-
- column_headplate = subject_data.columns.get_loc('headplate') + 1
- column_headplate = openpyxl.utils.get_column_letter(column_headplate)
-
-
- for i in range(headplate_image_paths.shape[0]):
-
- headplate_path = headplate_image_paths[i]
- cell_image = column_headplate+str(i+3)
-
- if pathlib.Path.is_file(pathlib.Path(headplate_path)):
- img = Image(headplate_path)
- img.width = 50
- img.height = 50
- sheet.add_image(img, cell_image)
- sheet.row_dimensions[i+3].height = 60
- else:
- sheet.row_dimensions[i+3].height = 15
-
- book.save(ss_file)
-
-
-def main_noDB_backup():
-
- write_dj_custom_vars_file()
- write_slack_webhooks_file()
- write_user_data_file()
- write_rig_status_file()
- day_schedule = write_schedule_file()
- write_past_sessions_file(day_schedule)
- write_subject_motor_position(day_schedule)
- write_weighinig_gui_ss_file()
-
-
-if __name__ == "__main__":
- main_noDB_backup()
-
diff --git a/u19_pipeline/alert_system/noDB_backup_creation/no_db_backup_creation_script.py b/u19_pipeline/alert_system/noDB_backup_creation/no_db_backup_creation_script.py
new file mode 100644
index 00000000..35be59ff
--- /dev/null
+++ b/u19_pipeline/alert_system/noDB_backup_creation/no_db_backup_creation_script.py
@@ -0,0 +1,277 @@
+from scripts.conf_file_finding import try_find_conf_file
+
+try_find_conf_file()
+
+import base64
+import datetime
+import pathlib
+import shutil
+import time
+
+import datajoint as dj
+import numpy as np
+import openpyxl
+import pandas as pd
+from openpyxl.drawing.image import Image
+from openpyxl.utils.dataframe import dataframe_to_rows
+
+time.sleep(1)
+
+import u19_pipeline.acquisition as acquisition
+import u19_pipeline.alert_system.water_weigh_alert.water_weigh_alert as wwa
+import u19_pipeline.behavior as behavior
+import u19_pipeline.lab as lab
+import u19_pipeline.scheduler as scheduler
+import u19_pipeline.subject as subject
+
+DJ_CUSTOM_VARIABLES_FILENAME = "DJCustomVariables.csv"
+SLACK_WEBHOOK_FILENAME = "SlackChannels.csv"
+USER_SLACK_FILENAME = "UserSlack.csv"
+RIG_STATUS_FILENAME = "RigStatusTable.csv"
+DAY_SCHEDULE_FILENAME = "ScheduleDay.csv"
+PAST_SESSION_PERFORMANCE_FILENAME = "PastSessions.csv"
+SUBJECT_MOTOR_POSITION_FILENAME = "SubjectMotorPosition.csv"
+
+WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = "Weighing_GUI_Replacement_SpreadSheet_Template.xlsx"
+WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = "Weighing_GUI_Replacement_SpreadSheet.xlsx"
+
+conf = dj.config
+nodb_virmen_backup_dir = pathlib.Path(
+ pathlib.Path(conf["custom"]["root_data_dir"][0]).parent.parent,
+ "Shared",
+ "NoDBVirmenBackup",
+)
+
+MAX_SESSIONS_HISTORY = 75
+
+
+def cast_choice(choice_array):
+
+ new_array = choice_array[0].copy()
+ new_array[new_array > 2] = 127
+
+ new_array = np.array(new_array, dtype=np.uint8)
+
+ return new_array
+
+
+def encode_webhook(webhook):
+
+ return (base64.b64encode(webhook.encode("utf-8"))).decode("utf-8")
+
+
+def write_dj_custom_vars_file():
+
+ file_write = pathlib.Path(nodb_virmen_backup_dir, DJ_CUSTOM_VARIABLES_FILENAME)
+ pd.DataFrame(lab.DjCustomVariables.fetch(as_dict=True)).to_csv(file_write)
+
+
+def write_slack_webhooks_file():
+
+ slack_webhooks = pd.DataFrame(lab.SlackWebhooks.fetch(as_dict=True))
+ slack_webhooks["webhook_url"] = slack_webhooks["webhook_url"].astype(str)
+
+ slack_webhooks["webhook_url"] = slack_webhooks["webhook_url"].apply(encode_webhook)
+
+ file_write = pathlib.Path(nodb_virmen_backup_dir, SLACK_WEBHOOK_FILENAME)
+ slack_webhooks.to_csv(file_write, index=False)
+
+
+def write_user_data_file():
+
+ user_data = pd.DataFrame(lab.User.fetch("user_id", "slack", "tech_responsibility", "slack_webhook", as_dict=True))
+ user_data["slack_webhook"] = user_data["slack_webhook"].astype(str)
+ user_data["slack_webhook"] = user_data["slack_webhook"].fillna("")
+
+ user_data["slack_webhook"] = user_data["slack_webhook"].apply(encode_webhook)
+
+ file_write = pathlib.Path(nodb_virmen_backup_dir, USER_SLACK_FILENAME)
+ user_data.to_csv(file_write, index=False)
+
+
+def write_rig_status_file():
+
+ file_write = pathlib.Path(nodb_virmen_backup_dir, RIG_STATUS_FILENAME)
+
+ df_rig_status = pd.DataFrame(
+ scheduler.RigStatus.fetch("location", "input_output_name", "current_status", as_dict=True)
+ )
+ df_rig_status.to_csv(file_write, index=False)
+
+
+def write_schedule_file():
+
+ schedule_query = {}
+ schedule_query["date"] = datetime.date.today()
+
+ subject_query = "subject_fullname is not null"
+
+ day_schedule = pd.DataFrame(
+ (
+ scheduler.Schedule * scheduler.TrainingProfile * subject.Subject.proj(subject_user_id="user_id")
+ & schedule_query
+ & subject_query
+ ).fetch(as_dict=True)
+ )
+ day_schedule = day_schedule.drop(columns="user_id")
+ day_schedule = day_schedule.rename(columns={"subject_user_id": "user_id"})
+
+ file_write = pathlib.Path(nodb_virmen_backup_dir, DAY_SCHEDULE_FILENAME)
+ day_schedule.to_csv(file_write, index=False)
+
+ return day_schedule
+
+
+def write_past_sessions_file(day_schedule):
+
+ all_subjects_schedule = "', '".join(day_schedule["subject_fullname"])
+ all_subjects_schedule = "subject_fullname in ('" + all_subjects_schedule + "')"
+
+ ss = pd.DataFrame(
+ (behavior.TowersSession & all_subjects_schedule).fetch(
+ "KEY",
+ order_by="subject_fullname, session_date desc, session_number desc",
+ as_dict=True,
+ )
+ )
+ ss["session_date"] = ss["session_date"].astype(str)
+ ss["num_sessions"] = ss.groupby(["subject_fullname"])["subject_fullname"].rank(method="first")
+ ss["num_sessions"] = ss["num_sessions"].astype(int)
+ ss = ss.loc[ss["num_sessions"] <= MAX_SESSIONS_HISTORY, :]
+
+ max_value_indices = ss.groupby("subject_fullname")["num_sessions"].idxmax()
+ ss = ss.loc[max_value_indices]
+ ss = ss.reset_index(drop=True)
+ ss["query"] = "(subject_fullname='" + ss["subject_fullname"] + "' and session_date >= '" + ss["session_date"] + "')"
+
+ block_query = " OR ".join(ss["query"])
+
+ sstable = acquisition.SessionStarted.proj("local_path_behavior_file", "session_location")
+ stable = (acquisition.Session).proj(stimulusBank="stimulus_bank")
+ tstable = behavior.TowersSession.proj(trialType="rewarded_side", choice="chosen_side", stimulusSet="stimulus_set")
+ tbtable = behavior.TowersBlock.proj(
+ "first_trial",
+ "n_trials",
+ "sublevel",
+ mazeID="level",
+ mainMazeID="main_level",
+ easyBlockFlag="easy_block",
+ duration="block_duration",
+ rewardMil="reward_mil",
+ medianTrialDur="trial_duration_median",
+ start="block_start_time",
+ )
+ table_fetch = sstable * stable * tstable * tbtable
+
+ allblocks = pd.DataFrame(
+ (table_fetch & block_query).fetch(
+ order_by="subject_fullname, session_date desc, session_number desc, block desc",
+ as_dict=True,
+ )
+ )
+ allblocks["from_DB"] = 1
+ allblocks["session_date"] = allblocks["session_date"].astype(str)
+
+ allblocks["sublevel"] = allblocks["sublevel"].astype("Int64")
+ allblocks["choice"] = allblocks["choice"].apply(cast_choice)
+ allblocks["trialType"] = allblocks["trialType"].apply(cast_choice)
+
+ file_write = pathlib.Path(nodb_virmen_backup_dir, PAST_SESSION_PERFORMANCE_FILENAME)
+ allblocks.to_csv(file_write, index=False)
+
+
+def write_subject_motor_position(day_schedule):
+
+ all_subjects_schedule = "', '".join(day_schedule["subject_fullname"])
+ all_subjects_schedule = "subject_fullname in ('" + all_subjects_schedule + "')"
+
+ sp = pd.DataFrame((subject.HeadMotorPosition & all_subjects_schedule).fetch(as_dict=True))
+
+ file_write = pathlib.Path(nodb_virmen_backup_dir, SUBJECT_MOTOR_POSITION_FILENAME)
+ sp.to_csv(file_write, index=False)
+
+
+def write_weighinig_gui_ss_file():
+
+ subject_data = wwa.get_subject_data()
+ subject_data = subject_data.loc[
+ :,
+ [
+ "user_id",
+ "subject_fullname",
+ "cage",
+ "headplate_image_path",
+ "last_weight",
+ "water_per_day",
+ ],
+ ]
+ # subject_data = subject_data.loc[subject_data['user_id'] != 'testuser',:]
+ subject_data = subject_data.reset_index(drop=True)
+
+ dictionary_rename_cols = {
+ "user_id": "owner",
+ "subject_fullname": "subject name",
+ "cage": "cage",
+ "headplate_image_path": "headplate",
+ "last_weight": "last weight",
+ "water_per_day": "water per day",
+ }
+
+ subject_data = subject_data.rename(columns=dictionary_rename_cols)
+ subject_data = subject_data[list(dictionary_rename_cols.values())]
+ subject_data["headplate"] = subject_data["headplate"].astype(str)
+
+ headplate_image_paths = subject_data["headplate"].copy()
+
+ subject_data["last weight"] = subject_data["last weight"].round(1)
+ subject_data["water per day"] = subject_data["water per day"].round(1)
+
+ subject_data["headplate"] = ""
+ subject_data["today weight"] = ""
+ subject_data["today water"] = ""
+
+ # Copy the source file to the destination, replacing if it exists
+ template_ss_file = pathlib.Path(nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE)
+ ss_file = pathlib.Path(nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME)
+ shutil.copy2(template_ss_file, ss_file)
+
+ book = openpyxl.load_workbook(ss_file)
+ sheet = book["Sheet1"]
+
+ for r_idx, row in enumerate(dataframe_to_rows(subject_data, index=False, header=False), start=3):
+ for c_idx, value in enumerate(row, start=1):
+ sheet.cell(row=r_idx, column=c_idx, value=value)
+
+ column_headplate = subject_data.columns.get_loc("headplate") + 1
+ column_headplate = openpyxl.utils.get_column_letter(column_headplate)
+
+ for i in range(headplate_image_paths.shape[0]):
+ headplate_path = headplate_image_paths[i]
+ cell_image = column_headplate + str(i + 3)
+
+ if pathlib.Path.is_file(pathlib.Path(headplate_path)):
+ img = Image(headplate_path)
+ img.width = 50
+ img.height = 50
+ sheet.add_image(img, cell_image)
+ sheet.row_dimensions[i + 3].height = 60
+ else:
+ sheet.row_dimensions[i + 3].height = 15
+
+ book.save(ss_file)
+
+
+def main_noDB_backup():
+
+ write_dj_custom_vars_file()
+ write_slack_webhooks_file()
+ write_user_data_file()
+ write_rig_status_file()
+ day_schedule = write_schedule_file()
+ write_past_sessions_file(day_schedule)
+ write_subject_motor_position(day_schedule)
+ write_weighinig_gui_ss_file()
+
+
+if __name__ == "__main__":
+ main_noDB_backup()
diff --git a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py
index 4fb3050d..0aad616c 100755
--- a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py
+++ b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py
@@ -12,10 +12,12 @@
- Uses the rigs_issues_and_troubleshooting webhook for notifications
"""
+import contextlib
import json
import logging
import os
import sys
+from u19_pipeline.utils.logging_config import get_logger
from datetime import date, datetime, timedelta
from pathlib import Path
@@ -26,11 +28,13 @@
from u19_pipeline import lab, rig_maintenance, scheduler
from u19_pipeline.utils import slack_utils as su
except Exception as e: # pragma: no cover - only happens when package not available
- print(f"Error importing modules: {e}")
- print("Make sure u19_pipeline is properly installed and configured.")
+ # Cannot use logger here since get_logger may not be importable
+ import sys as _sys
+ _sys.stderr.write(f"Error importing modules: {e}\n")
+ _sys.stderr.write("Make sure u19_pipeline is properly installed and configured.\n")
sys.exit(1)
-logger = logging.getLogger("rig_maintenance")
+logger = get_logger(__name__)
def setup_logging():
@@ -43,19 +47,13 @@ def setup_logging():
# Create logs directory in the user's home directory (allow override via env var)
# Default: ~/.u19_pipeline/logs
env_log_dir = os.getenv("U19_PIPELINE_LOG_DIR")
- if env_log_dir:
- log_dir = Path(env_log_dir)
- else:
- log_dir = Path.home() / "u19_pipeline_logs"
+ log_dir = Path(env_log_dir) if env_log_dir else Path.home() / "u19_pipeline_logs"
log_dir.mkdir(parents=True, exist_ok=True)
# Create log file with timestamp
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
log_file = log_dir / f"rig_maintenance_check_{timestamp}.log"
- # Open log file handle
- log_file_handle = open(log_file, "a", encoding="utf-8")
-
# Configure standard logging: StreamHandler (console) + FileHandler (file)
logger = logging.getLogger("rig_maintenance")
logger.setLevel(logging.INFO)
@@ -76,7 +74,7 @@ def setup_logging():
# Prevent messages from also being handled by the root logger
logger.propagate = False
- return logger, log_file, log_file_handle
+ return logger, log_file
def get_slack_webhook():
@@ -200,7 +198,7 @@ def check_overdue_maintenance():
unique_locations = sorted(set(locations))
# Get all maintenance types and their intervals
- maintenance_fetch = getattr(rig_maintenance.MaintenanceType, "fetch")
+ maintenance_fetch = rig_maintenance.MaintenanceType.fetch
maintenance_types = maintenance_fetch(as_dict=True)
# Separate maintenance types by system type
@@ -219,7 +217,7 @@ def check_overdue_maintenance():
rig_loc_query = lab.Location & merged_rig_queries
- rig_loc_fetch = getattr(rig_loc_query, "fetch")
+ rig_loc_fetch = rig_loc_query.fetch
rig_locations = rig_loc_fetch(as_dict=True)
logger.info(f"🔍 Checking maintenance status as of {current_date}")
@@ -252,7 +250,7 @@ def check_overdue_maintenance():
"location": location_name,
"maintenance_type": maintenance_type,
}
- recent_fetch = getattr(recent_q, "fetch")
+ recent_fetch = recent_q.fetch
recent_maintenance = recent_fetch("maintenance_date", order_by="maintenance_date DESC", limit=1)
if len(recent_maintenance) == 0:
@@ -277,9 +275,7 @@ def check_overdue_maintenance():
if interval_days <= 0:
# If the interval is 0 or negative, it is an as-needed maintenance type,
# so we only log if there is no record
- logger.info(
- f" {maintenance_type:.<30} AS-NEEDED (record exists, no interval) ✅"
- )
+ logger.info(f" {maintenance_type:.<30} AS-NEEDED (record exists, no interval) ✅")
elif days_since_last > interval_days:
# Maintenance is overdue
days_overdue = days_since_last - interval_days
@@ -330,7 +326,7 @@ def check_overdue_maintenance():
hosting_loc_query = lab.Location & merged_hosting_queries
- hosting_loc_fetch = getattr(hosting_loc_query, "fetch")
+ hosting_loc_fetch = hosting_loc_query.fetch
hosting_locations = hosting_loc_fetch(as_dict=True)
# Check hosting VMs
@@ -350,7 +346,7 @@ def check_overdue_maintenance():
"location": location_name,
"maintenance_type": maintenance_type,
}
- recent_fetch = getattr(recent_q, "fetch")
+ recent_fetch = recent_q.fetch
recent_maintenance = recent_fetch("maintenance_date", order_by="maintenance_date DESC", limit=1)
if len(recent_maintenance) == 0:
@@ -531,10 +527,9 @@ def dataframe_to_slack_table_blocks(df: pd.DataFrame, chunk_size: int = 15, **kw
def main():
"""Main function to run the maintenance check."""
- log_file_handle = None
try:
# Set up logging
- _, log_file, log_file_handle = setup_logging()
+ _, log_file = setup_logging()
logger.info("🔧 Rig Maintenance Status Checker")
logger.info("=" * 60)
@@ -548,7 +543,7 @@ def main():
records = pd.DataFrame(overdue_items)
records.columns = [" ".join(w.capitalize() for w in col.split("_")) for col in records.columns]
- print(records)
+ logger.info("records\n%s", records)
records = records[["Location", "Maintenance Type", "Status"]]
@@ -608,12 +603,10 @@ def main():
logger.exception(f"❌ Error during maintenance check: {e}")
sys.exit(1)
finally:
- # Close the log file handle
- if log_file_handle:
- try:
- log_file_handle.close()
- except Exception:
- pass
+ # Close any file handlers to flush logs
+ for handler in logging.getLogger("rig_maintenance").handlers[:]:
+ with contextlib.suppress(Exception):
+ handler.close()
if __name__ == "__main__":
diff --git a/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py b/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py
index 3238e705..8eca53ca 100644
--- a/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py
+++ b/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py
@@ -1,6 +1,7 @@
-
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(0.1)
@@ -8,4 +9,3 @@
import u19_pipeline.alert_system.schedule_check_alert.schedule_check_alert as sca
sca.main_schedule_check_alert()
-
diff --git a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py
index 706602cc..ce610f68 100644
--- a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py
+++ b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py
@@ -4,23 +4,28 @@
import datajoint as dj
import pandas as pd
-import u19_pipeline.utils.slack_utils as su
import u19_pipeline.lab as lab
+import u19_pipeline.utils.slack_utils as su
+from u19_pipeline.utils.logging_config import get_logger
+logger = get_logger(__name__)
# Slack Configuration dictionary
slack_configuration_dictionary = {
- 'slack_notification_channel': ['dev_notifications'],
- #"slack_users_channel": ["alvaros"]
+ "slack_notification_channel": ["dev_notifications"],
+ # "slack_users_channel": ["alvaros"]
}
+
def get_schedule_query():
scheduler = dj.create_virtual_module("scheduler", "u19_scheduler")
- tomorrow = (datetime.date.today() + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
- today = (datetime.date.today()).strftime('%Y-%m-%d')
+ tomorrow = (datetime.date.today() + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
+ today = (datetime.date.today()).strftime("%Y-%m-%d")
schedule_query = 'date >= "' + today + '" and date <= "' + tomorrow + '"'
- schedule_df = pd.DataFrame((scheduler.Schedule & schedule_query).fetch('date', 'location', 'subject_fullname', as_dict=True))
+ schedule_df = pd.DataFrame(
+ (scheduler.Schedule & schedule_query).fetch("date", "location", "subject_fullname", as_dict=True)
+ )
return schedule_df
@@ -29,46 +34,51 @@ def main_schedule_check_alert():
alert = 0
schedule_df = get_schedule_query()
- tomorrow_schedule = schedule_df.loc[schedule_df['date'] == (datetime.date.today() + datetime.timedelta(days=1)),:]
+ tomorrow_schedule = schedule_df.loc[schedule_df["date"] == (datetime.date.today() + datetime.timedelta(days=1)), :]
if tomorrow_schedule.shape[0] == 0:
alert = 1
slack_json_message = slack_alert_empty_schedule()
else:
- schedule_df = schedule_df.groupby(['date', 'location']).agg({'subject_fullname': [('#subj', 'count')]})
+ schedule_df = schedule_df.groupby(["date", "location"]).agg({"subject_fullname": [("#subj", "count")]})
schedule_df.columns = schedule_df.columns.droplevel()
schedule_df = schedule_df.reset_index()
- todays_summary_df = schedule_df.loc[schedule_df['date'] == datetime.date.today(),
- ['location', '#subj']].copy()
- tomorrow_summary_df = schedule_df.loc[schedule_df['date'] == (datetime.date.today() + datetime.timedelta(days=1)),
- ['location', '#subj']].copy()
-
- schedule_comp = pd.merge(todays_summary_df, tomorrow_summary_df, on=['location'], suffixes=['_today', '_tomorrow'])
- schedule_comp
+ todays_summary_df = schedule_df.loc[schedule_df["date"] == datetime.date.today(), ["location", "#subj"]].copy()
+ tomorrow_summary_df = schedule_df.loc[
+ schedule_df["date"] == (datetime.date.today() + datetime.timedelta(days=1)),
+ ["location", "#subj"],
+ ].copy()
+
+ schedule_comp = pd.merge(
+ todays_summary_df,
+ tomorrow_summary_df,
+ on=["location"],
+ suffixes=["_today", "_tomorrow"],
+ )
+ logger.debug("schedule_comp %s", schedule_comp)
- schedule_comp['diff_subjects'] = schedule_comp['#subj_tomorrow'] - schedule_comp['#subj_today']
- schedule_comp['rig_less_subjects'] = schedule_comp['diff_subjects'] < -2
+ schedule_comp["diff_subjects"] = schedule_comp["#subj_tomorrow"] - schedule_comp["#subj_today"]
+ schedule_comp["rig_less_subjects"] = schedule_comp["diff_subjects"] < -2
- subjects_today = schedule_comp['#subj_today'].sum()
- subjects_tomorrow = schedule_comp['#subj_tomorrow'].sum()
- total_rigs_less_subjects = schedule_comp['rig_less_subjects'].sum()
+ subjects_today = schedule_comp["#subj_today"].sum()
+ subjects_tomorrow = schedule_comp["#subj_tomorrow"].sum()
+ total_rigs_less_subjects = schedule_comp["rig_less_subjects"].sum()
- print(schedule_comp)
+ logger.debug("schedule_comp %s", schedule_comp)
- if subjects_tomorrow/subjects_today < 0.7:
+ if subjects_tomorrow / subjects_today < 0.7:
alert = 1
if total_rigs_less_subjects >= 3:
alert = 1
if alert == 1:
- schedule_comp['location'] = '*'+schedule_comp['location']+'*~'
- schedule_comp = schedule_comp.drop(['diff_subjects', 'rig_less_subjects'], axis=1)
+ schedule_comp["location"] = "*" + schedule_comp["location"] + "*~"
+ schedule_comp = schedule_comp.drop(["diff_subjects", "rig_less_subjects"], axis=1)
schedule_comp_df_string = su.format_df_for_slack_message(schedule_comp)
slack_json_message = slack_alert_message_format_schedule(schedule_comp_df_string)
-
if alert == 0:
return
else:
@@ -81,61 +91,62 @@ def main_schedule_check_alert():
def slack_alert_message_format_schedule(schedule_df_string):
now = datetime.datetime.now()
- datestr = now.strftime("%d-%b-%Y %H:%M:%S")
+ now.strftime("%d-%b-%Y %H:%M:%S")
- msep = dict()
+ msep = {}
msep["type"] = "divider"
# Title#
- m1 = dict()
+ m1 = {}
m1["type"] = "section"
- m1_1 = dict()
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
m1_1["text"] = ":rotating_light: *Schedule Alert *"
m1["text"] = m1_1
# Info for subjects missing water
- m2 = dict()
+ m2 = {}
m2["type"] = "section"
- m2_1 = dict()
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
m2_1["text"] = "Significantly less subjects scheduled tomorrow\nSchedule per rig:" + "\n\n"
m2_1["text"] += schedule_df_string
m2["text"] = m2_1
- message = dict()
+ message = {}
message["blocks"] = [m1, msep, m2, msep]
message["text"] = "Suspicious Schedule Alert"
return message
+
def slack_alert_empty_schedule():
now = datetime.datetime.now()
- datestr = now.strftime("%d-%b-%Y %H:%M:%S")
+ now.strftime("%d-%b-%Y %H:%M:%S")
- msep = dict()
+ msep = {}
msep["type"] = "divider"
# Title#
- m1 = dict()
+ m1 = {}
m1["type"] = "section"
- m1_1 = dict()
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
m1_1["text"] = ":rotating_light: *Schedule Alert *"
m1["text"] = m1_1
# Info for subjects missing water
- m2 = dict()
+ m2 = {}
m2["type"] = "section"
- m2_1 = dict()
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
m2_1["text"] = "*No Schedule found for tomorrow:*" + "\n"
m2["text"] = m2_1
- message = dict()
+ message = {}
message["blocks"] = [m1, msep, m2, msep]
message["text"] = "Schedule Empty Alert"
- return message
\ No newline at end of file
+ return message
diff --git a/u19_pipeline/alert_system/tech_alert/tech_alert.py b/u19_pipeline/alert_system/tech_alert/tech_alert.py
index cf7eb52b..97d82a8f 100644
--- a/u19_pipeline/alert_system/tech_alert/tech_alert.py
+++ b/u19_pipeline/alert_system/tech_alert/tech_alert.py
@@ -8,6 +8,9 @@
from icalevents.icalevents import events
from u19_pipeline.utils import slack_utils as su
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
def tech_schedule():
@@ -36,25 +39,25 @@ def slack_alert_message_format_tech_alert(schedule_data):
# Divider #
- msep = dict()
+ msep = {}
msep["type"] = "divider"
# Title#
- m1 = dict()
+ m1 = {}
m1["type"] = "section"
- m1_1 = dict()
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
m1_1["text"] = "🗓 *Today's Tech Schedule:*"
m1["text"] = m1_1
- message = dict()
+ message = {}
message["blocks"] = [m1, msep]
message["text"] = "🗓 *Today's Tech Schedule:*"
# Info#
- m2 = dict()
+ m2 = {}
m2["type"] = "section"
- m2_1 = dict()
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
# If there are shifts today, list them; otherwise provide a short fallback
@@ -116,9 +119,9 @@ def slack_alert_message_format_tech_alert(schedule_data):
if alerts:
alert_text = "\n".join(alerts)
- m3 = dict()
+ m3 = {}
m3["type"] = "section"
- m3_1 = dict()
+ m3_1 = {}
m3_1["type"] = "mrkdwn"
m3_1["text"] = f"*:rotating_light: Upcoming Shifts: :rotating_light:*\n{alert_text}"
m3["text"] = m3_1
@@ -260,7 +263,7 @@ def fetch_and_parse_icalevents(weburl: str):
try:
vr_events = events(url=weburl, start=start_date, end=end_date)
except Exception as e:
- print(f"Error fetching events from WhenIWork: {e}")
+ logger.error("Error fetching events from WhenIWork: %s", e)
return []
# Define mapping of substrings to event types and their corresponding colors
@@ -268,7 +271,12 @@ def fetch_and_parse_icalevents(weburl: str):
(r"as\s*vr\s*water\s*at", "VR Water", "blue", "VR Watering only"),
(r"as\s*vr\s*train\s*at", "VR Train", "orange", "VR Onboarding"),
(r"as\s*vr\s*at", "Regular VR", "green", "All VR Duties"),
- (r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at", "VR Brody Mice", "purple", " VR with Brody (Mice)"),
+ (
+ r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at",
+ "VR Brody Mice",
+ "purple",
+ " VR with Brody (Mice)",
+ ),
]
filtered_events: list[dict[str, str | int]] = []
@@ -282,7 +290,7 @@ def fetch_and_parse_icalevents(weburl: str):
event for event in vr_events if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower())
]
if lab_clean_up_events:
- unique_days = sorted(set(event.start.date() for event in lab_clean_up_events))
+ unique_days = sorted({event.start.date() for event in lab_clean_up_events})
for day in unique_days:
filtered_events.append(
{
@@ -320,7 +328,7 @@ def fetch_and_parse_icalevents(weburl: str):
if __name__ == "__main__":
- print("Command-line arguments:", sys.argv)
+ logger.info("Command-line arguments: %s", sys.argv)
if len(sys.argv) < 2:
main_loop()
else:
diff --git a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py
index 23749d9b..fd0003ba 100644
--- a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py
+++ b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py
@@ -14,7 +14,12 @@
import u19_pipeline.lab as lab
import u19_pipeline.utils.slack_utils as su
from u19_pipeline import subject
-from u19_pipeline.utils.subject_metadata import fetch_slack_handles_for_lab_managers_by_subject
+from u19_pipeline.utils.subject_metadata import (
+ fetch_slack_handles_for_lab_managers_by_subject,
+)
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
slack_configuration_dictionary = {
"slack_notification_channel": ["subject_health"],
@@ -25,7 +30,7 @@
def get_subject_data():
- with open(QUERY_FILE, "r", encoding="utf-8") as file:
+ with open(QUERY_FILE, encoding="utf-8") as file:
subject_query = file.read()
conn = dj.conn()
@@ -90,7 +95,10 @@ def get_subject_data():
subject_data.loc[subject_data["need_extra_water_now"] == 1, "water_status"] = "Need Extra Supplement"
subject_data["need_water"] = 0
- subject_data.loc[(subject_data["need_supplement"] | subject_data["need_extra_water_now"]), "need_water"] = 1
+ subject_data.loc[
+ (subject_data["need_supplement"] | subject_data["need_extra_water_now"]),
+ "need_water",
+ ] = 1
subject_data["current_need_water"] = subject_data["suggested_water"]
subject_data.loc[subject_data["need_extra_water_now"] == 1, "current_need_water"] = subject_data.loc[
@@ -136,7 +144,12 @@ def fetch_and_parse_icalevents(weburl: str):
(r"as\s*vr\s*water\s*at", "VR Water", "blue", "VR Watering only"),
(r"as\s*vr\s*train\s*at", "VR Train", "orange", "VR Onboarding"),
(r"as\s*vr\s*at", "Regular VR", "green", "All VR Duties"),
- (r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at", "VR Brody Mice", "purple", " VR with Brody (Mice)"),
+ (
+ r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at",
+ "VR Brody Mice",
+ "purple",
+ " VR with Brody (Mice)",
+ ),
]
filtered_events: list[dict[str, str | int]] = []
@@ -210,7 +223,7 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame:
(lab.LabManager().proj("lab", "lab_manager") & 'lab = "technician"')
* lab.User().proj(lab_manager="user_id", manager_slack="slack")
).fetch("manager_slack", as_dict=True)
- print(technician_manager_slack)
+ logger.debug("technician_manager_slack %s", technician_manager_slack)
technician_manager_slack = [item["manager_slack"] for item in technician_manager_slack]
people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch(format="frame")
@@ -259,7 +272,7 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame:
def resolve_responsible_slack(row):
schedule_today = row.get("schedule_today")
- lab_name = row.get("lab")
+ row.get("lab")
token = str(schedule_today).strip() if schedule_today is not None else None
use_coowners = (token is None) or (token.lower() == "transport") or (token.lower() == "nothing")
@@ -322,13 +335,15 @@ def slack_alert_message_format_weight_water(
individual_alert: bool = False,
):
- print(missing_transport)
+ logger.debug("missing_transport %s", missing_transport)
temp_missing_transport = missing_transport.copy().reset_index()
- notifiable_subjects = list(set(
- temp_missing_transport["subject_fullname"].tolist()
- + subjects_not_watered["subject_fullname"].tolist()
- + subjects_not_weighted["subject_fullname"].tolist()
- ))
+ notifiable_subjects = list(
+ set(
+ temp_missing_transport["subject_fullname"].tolist()
+ + subjects_not_watered["subject_fullname"].tolist()
+ + subjects_not_weighted["subject_fullname"].tolist()
+ )
+ )
slack_handles: list[str] = fetch_slack_handles_for_lab_managers_by_subject(notifiable_subjects)
lab_manager_text = "\n\n"
@@ -343,22 +358,22 @@ def slack_alert_message_format_weight_water(
" " + slack_handles_formatted + ", please be advised that your labs' subjects are listed below."
)
- msep = dict()
+ msep = {}
msep["type"] = "divider"
# Title#
- m1 = dict()
+ m1 = {}
m1["type"] = "section"
- m1_1 = dict()
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
m1_1["text"] = ":rotating_light: *Subjects Status Alert *" + lab_manager_text
m1["text"] = m1_1
# Info for subjects missing water
- m2 = dict()
+ m2 = {}
m2["type"] = "section"
- m2_1 = dict()
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
if subjects_not_watered.empty:
@@ -388,9 +403,9 @@ def slack_alert_message_format_weight_water(
m2["text"] = m2_1
# Info for subjects missing weighing
- m4 = dict()
+ m4 = {}
m4["type"] = "section"
- m4_1 = dict()
+ m4_1 = {}
m4_1["type"] = "mrkdwn"
if subjects_not_weighted.empty:
@@ -421,9 +436,9 @@ def slack_alert_message_format_weight_water(
m4["text"] = m4_1
# Info for subjects missing training
- m5 = dict()
+ m5 = {}
m5["type"] = "section"
- m5_1 = dict()
+ m5_1 = {}
m5_1["type"] = "mrkdwn"
if subjects_not_trained.empty:
@@ -441,9 +456,9 @@ def slack_alert_message_format_weight_water(
m5["text"] = m5_1
# Info for missing transport
- m6 = dict()
+ m6 = {}
m6["type"] = "section"
- m6_1 = dict()
+ m6_1 = {}
m6_1["type"] = "mrkdwn"
if missing_transport.empty:
@@ -472,7 +487,7 @@ def slack_alert_message_format_weight_water(
m6_1["text"] += line + "\n"
m6["text"] = m6_1
- message = dict()
+ message = {}
message["blocks"] = [m1, msep, m2, msep, m4, msep, m5, msep, m6, msep]
# msg_groups = [m1, m2, m4, m5, m6]
msg_groups = [m2]
@@ -539,7 +554,7 @@ def split_text_to_fit(text: str, max_payload_size: int, text_type: str = "mrkdwn
if _estimate_section_size(line, text_type) > max_payload_size:
# Single line too big - this shouldn't happen often, but handle it
# by truncating (better than crashing)
- print("Warning: Single line too large, may be truncated")
+ logger.warning("Warning: Single line too large, may be truncated")
else:
# Even first line doesn't fit - add it anyway (will be handled later)
current_lines = [line]
@@ -647,7 +662,7 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int):
su.send_slack_notification(webhook, {"blocks": [small_block]})
time.sleep(0.5)
except HTTPError:
- print("Failed to send block even after splitting")
+ logger.error("Failed to send block even after splitting")
else:
raise
else:
@@ -658,7 +673,7 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int):
su.send_slack_notification(webhook, {"blocks": [small_block]})
time.sleep(0.5)
except HTTPError:
- print("Failed to send small block")
+ logger.error("Failed to send small block")
def main_water_weigh_alert():
@@ -672,21 +687,22 @@ def main_water_weigh_alert():
subject_data = subject_data.loc[~subject_data["subject_fullname"].str.contains("test"), :]
subjects_not_watered = subject_data.loc[
- subject_data["current_need_water"] > 0, ["subject_fullname", "current_need_water", "responsible_slack_tags"]
+ subject_data["current_need_water"] > 0,
+ ["subject_fullname", "current_need_water", "responsible_slack_tags"],
]
subjects_not_watered = subjects_not_watered.reset_index(drop=True)
subjects_not_watered["current_need_water"] = subjects_not_watered["current_need_water"].apply(lambda x: f"{x:.1f}")
# subjects_not_watered = subjects_not_watered.head()
subjects_not_weighted = subject_data.loc[
- subject_data["need_weight"], ["subject_fullname", "need_weight", "responsible_slack_tags"]
+ subject_data["need_weight"],
+ ["subject_fullname", "need_weight", "responsible_slack_tags"],
]
subjects_not_weighted = subjects_not_weighted.reset_index(drop=True)
# subjects_not_weighted = subjects_not_weighted.head()
subjects_not_trained = subject_data.loc[
- (subject_data["training_status"] == 1)
- & (subject_data["schedule_today"].str.lower() != "water"),
+ (subject_data["training_status"] == 1) & (subject_data["schedule_today"].str.lower() != "water"),
["subject_fullname", "scheduled_rig"],
]
subjects_not_trained = subjects_not_trained.reset_index(drop=True)
@@ -705,10 +721,12 @@ def main_water_weigh_alert():
# Send alert
slack_json_messages = slack_alert_message_format_weight_water(
- subjects_not_watered, subjects_not_weighted, subjects_not_trained, missing_transport=subject_not_returned
+ subjects_not_watered,
+ subjects_not_weighted,
+ subjects_not_trained,
+ missing_transport=subject_not_returned,
)
-
# Send each message's blocks safely (splitting large blocks as needed)
for this_webhook in webhooks_list:
for message in slack_json_messages:
diff --git a/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py b/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py
index 22d50e21..1321c082 100644
--- a/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py
+++ b/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py
@@ -1,6 +1,7 @@
-
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(0.1)
@@ -8,4 +9,3 @@
import u19_pipeline.alert_system.water_weigh_alert.water_weigh_alert as wwa
wwa.main_water_weigh_alert()
-
diff --git a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py
index 23749d9b..fd0003ba 100644
--- a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py
+++ b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py
@@ -14,7 +14,12 @@
import u19_pipeline.lab as lab
import u19_pipeline.utils.slack_utils as su
from u19_pipeline import subject
-from u19_pipeline.utils.subject_metadata import fetch_slack_handles_for_lab_managers_by_subject
+from u19_pipeline.utils.subject_metadata import (
+ fetch_slack_handles_for_lab_managers_by_subject,
+)
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
slack_configuration_dictionary = {
"slack_notification_channel": ["subject_health"],
@@ -25,7 +30,7 @@
def get_subject_data():
- with open(QUERY_FILE, "r", encoding="utf-8") as file:
+ with open(QUERY_FILE, encoding="utf-8") as file:
subject_query = file.read()
conn = dj.conn()
@@ -90,7 +95,10 @@ def get_subject_data():
subject_data.loc[subject_data["need_extra_water_now"] == 1, "water_status"] = "Need Extra Supplement"
subject_data["need_water"] = 0
- subject_data.loc[(subject_data["need_supplement"] | subject_data["need_extra_water_now"]), "need_water"] = 1
+ subject_data.loc[
+ (subject_data["need_supplement"] | subject_data["need_extra_water_now"]),
+ "need_water",
+ ] = 1
subject_data["current_need_water"] = subject_data["suggested_water"]
subject_data.loc[subject_data["need_extra_water_now"] == 1, "current_need_water"] = subject_data.loc[
@@ -136,7 +144,12 @@ def fetch_and_parse_icalevents(weburl: str):
(r"as\s*vr\s*water\s*at", "VR Water", "blue", "VR Watering only"),
(r"as\s*vr\s*train\s*at", "VR Train", "orange", "VR Onboarding"),
(r"as\s*vr\s*at", "Regular VR", "green", "All VR Duties"),
- (r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at", "VR Brody Mice", "purple", " VR with Brody (Mice)"),
+ (
+ r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at",
+ "VR Brody Mice",
+ "purple",
+ " VR with Brody (Mice)",
+ ),
]
filtered_events: list[dict[str, str | int]] = []
@@ -210,7 +223,7 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame:
(lab.LabManager().proj("lab", "lab_manager") & 'lab = "technician"')
* lab.User().proj(lab_manager="user_id", manager_slack="slack")
).fetch("manager_slack", as_dict=True)
- print(technician_manager_slack)
+ logger.debug("technician_manager_slack %s", technician_manager_slack)
technician_manager_slack = [item["manager_slack"] for item in technician_manager_slack]
people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch(format="frame")
@@ -259,7 +272,7 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame:
def resolve_responsible_slack(row):
schedule_today = row.get("schedule_today")
- lab_name = row.get("lab")
+ row.get("lab")
token = str(schedule_today).strip() if schedule_today is not None else None
use_coowners = (token is None) or (token.lower() == "transport") or (token.lower() == "nothing")
@@ -322,13 +335,15 @@ def slack_alert_message_format_weight_water(
individual_alert: bool = False,
):
- print(missing_transport)
+ logger.debug("missing_transport %s", missing_transport)
temp_missing_transport = missing_transport.copy().reset_index()
- notifiable_subjects = list(set(
- temp_missing_transport["subject_fullname"].tolist()
- + subjects_not_watered["subject_fullname"].tolist()
- + subjects_not_weighted["subject_fullname"].tolist()
- ))
+ notifiable_subjects = list(
+ set(
+ temp_missing_transport["subject_fullname"].tolist()
+ + subjects_not_watered["subject_fullname"].tolist()
+ + subjects_not_weighted["subject_fullname"].tolist()
+ )
+ )
slack_handles: list[str] = fetch_slack_handles_for_lab_managers_by_subject(notifiable_subjects)
lab_manager_text = "\n\n"
@@ -343,22 +358,22 @@ def slack_alert_message_format_weight_water(
" " + slack_handles_formatted + ", please be advised that your labs' subjects are listed below."
)
- msep = dict()
+ msep = {}
msep["type"] = "divider"
# Title#
- m1 = dict()
+ m1 = {}
m1["type"] = "section"
- m1_1 = dict()
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
m1_1["text"] = ":rotating_light: *Subjects Status Alert *" + lab_manager_text
m1["text"] = m1_1
# Info for subjects missing water
- m2 = dict()
+ m2 = {}
m2["type"] = "section"
- m2_1 = dict()
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
if subjects_not_watered.empty:
@@ -388,9 +403,9 @@ def slack_alert_message_format_weight_water(
m2["text"] = m2_1
# Info for subjects missing weighing
- m4 = dict()
+ m4 = {}
m4["type"] = "section"
- m4_1 = dict()
+ m4_1 = {}
m4_1["type"] = "mrkdwn"
if subjects_not_weighted.empty:
@@ -421,9 +436,9 @@ def slack_alert_message_format_weight_water(
m4["text"] = m4_1
# Info for subjects missing training
- m5 = dict()
+ m5 = {}
m5["type"] = "section"
- m5_1 = dict()
+ m5_1 = {}
m5_1["type"] = "mrkdwn"
if subjects_not_trained.empty:
@@ -441,9 +456,9 @@ def slack_alert_message_format_weight_water(
m5["text"] = m5_1
# Info for missing transport
- m6 = dict()
+ m6 = {}
m6["type"] = "section"
- m6_1 = dict()
+ m6_1 = {}
m6_1["type"] = "mrkdwn"
if missing_transport.empty:
@@ -472,7 +487,7 @@ def slack_alert_message_format_weight_water(
m6_1["text"] += line + "\n"
m6["text"] = m6_1
- message = dict()
+ message = {}
message["blocks"] = [m1, msep, m2, msep, m4, msep, m5, msep, m6, msep]
# msg_groups = [m1, m2, m4, m5, m6]
msg_groups = [m2]
@@ -539,7 +554,7 @@ def split_text_to_fit(text: str, max_payload_size: int, text_type: str = "mrkdwn
if _estimate_section_size(line, text_type) > max_payload_size:
# Single line too big - this shouldn't happen often, but handle it
# by truncating (better than crashing)
- print("Warning: Single line too large, may be truncated")
+ logger.warning("Warning: Single line too large, may be truncated")
else:
# Even first line doesn't fit - add it anyway (will be handled later)
current_lines = [line]
@@ -647,7 +662,7 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int):
su.send_slack_notification(webhook, {"blocks": [small_block]})
time.sleep(0.5)
except HTTPError:
- print("Failed to send block even after splitting")
+ logger.error("Failed to send block even after splitting")
else:
raise
else:
@@ -658,7 +673,7 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int):
su.send_slack_notification(webhook, {"blocks": [small_block]})
time.sleep(0.5)
except HTTPError:
- print("Failed to send small block")
+ logger.error("Failed to send small block")
def main_water_weigh_alert():
@@ -672,21 +687,22 @@ def main_water_weigh_alert():
subject_data = subject_data.loc[~subject_data["subject_fullname"].str.contains("test"), :]
subjects_not_watered = subject_data.loc[
- subject_data["current_need_water"] > 0, ["subject_fullname", "current_need_water", "responsible_slack_tags"]
+ subject_data["current_need_water"] > 0,
+ ["subject_fullname", "current_need_water", "responsible_slack_tags"],
]
subjects_not_watered = subjects_not_watered.reset_index(drop=True)
subjects_not_watered["current_need_water"] = subjects_not_watered["current_need_water"].apply(lambda x: f"{x:.1f}")
# subjects_not_watered = subjects_not_watered.head()
subjects_not_weighted = subject_data.loc[
- subject_data["need_weight"], ["subject_fullname", "need_weight", "responsible_slack_tags"]
+ subject_data["need_weight"],
+ ["subject_fullname", "need_weight", "responsible_slack_tags"],
]
subjects_not_weighted = subjects_not_weighted.reset_index(drop=True)
# subjects_not_weighted = subjects_not_weighted.head()
subjects_not_trained = subject_data.loc[
- (subject_data["training_status"] == 1)
- & (subject_data["schedule_today"].str.lower() != "water"),
+ (subject_data["training_status"] == 1) & (subject_data["schedule_today"].str.lower() != "water"),
["subject_fullname", "scheduled_rig"],
]
subjects_not_trained = subjects_not_trained.reset_index(drop=True)
@@ -705,10 +721,12 @@ def main_water_weigh_alert():
# Send alert
slack_json_messages = slack_alert_message_format_weight_water(
- subjects_not_watered, subjects_not_weighted, subjects_not_trained, missing_transport=subject_not_returned
+ subjects_not_watered,
+ subjects_not_weighted,
+ subjects_not_trained,
+ missing_transport=subject_not_returned,
)
-
# Send each message's blocks safely (splitting large blocks as needed)
for this_webhook in webhooks_list:
for message in slack_json_messages:
diff --git a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py
index 0aa3e749..f58888ee 100644
--- a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py
+++ b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py
@@ -1,117 +1,119 @@
-import datajoint as dj
-import pathlib
-import subprocess
import json
-import re
import os
+import pathlib
+import subprocess
import time
-from datetime import datetime
-from element_interface.utils import dict_to_uuid
+import datajoint as dj
import u19_pipeline.automatic_job.params_config as config
-#Functions to transfer files (globus, scp, smbclient)
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
-#FOR PNI endpoint
-#pni_ep_id = '6ce834d6-ff8a-11e6-bad1-22000b9a448b'
-pni_ep_id = '005329dc-f31c-11ec-b3c1-15403b7b75ed' # pni BRAINCOGS ep points to /braininit/Data/
-pni_data_dir = ''
+# Functions to transfer files (globus, scp, smbclient)
-#For tiger endpoint
-public_key_location = '~/.ssh/id_ed25519.pub'
-default_user = 'u19prod' # This will change to our automatic client for globus transfers
-tiger_gpu_host = 'della.princeton.edu'
-#tiger_ep_dir = 'a9df83d2-42f0-11e6-80cf-22000b1701d1' # tigress ep
-tiger_ep_dir = '8e1bbdfe-d234-4a7c-93a5-86086a249918' # Endpoint of Della's /scratch/gpfs/. Our directory is ./BRAINCOGS/
+# FOR PNI endpoint
+# pni_ep_id = '6ce834d6-ff8a-11e6-bad1-22000b9a448b'
+pni_ep_id = "005329dc-f31c-11ec-b3c1-15403b7b75ed" # pni BRAINCOGS ep points to /braininit/Data/
+pni_data_dir = ""
-tiger_home_dir_globus = '/BRAINCOGS/Data/'
+# For tiger endpoint
+public_key_location = "~/.ssh/id_ed25519.pub"
+default_user = "u19prod" # This will change to our automatic client for globus transfers
+tiger_gpu_host = "della.princeton.edu"
+# tiger_ep_dir = 'a9df83d2-42f0-11e6-80cf-22000b1701d1' # tigress ep
+tiger_ep_dir = (
+ "8e1bbdfe-d234-4a7c-93a5-86086a249918" # Endpoint of Della's /scratch/gpfs/. Our directory is ./BRAINCOGS/
+)
-#Slurm default values for queue job
+tiger_home_dir_globus = "/BRAINCOGS/Data/"
+
+# Slurm default values for queue job
slurm_dict_tiger_default = {
- 'job-name': 'kilosort2',
- 'nodes': 1,
- 'ntasks': 1,
- 'time': '10:00:00',
- 'mem': '100G',
- 'gres': 'gpu:1',
- 'mail-user': 'alvaros@princeton.edu',
- 'mail-type': ['END'],
- 'output': 'OutputLog/job_id_${job_id}".log',
- 'error': 'ErrorLog/job_id_${job_id}".log'
+ "job-name": "kilosort2",
+ "nodes": 1,
+ "ntasks": 1,
+ "time": "10:00:00",
+ "mem": "100G",
+ "gres": "gpu:1",
+ "mail-user": "alvaros@princeton.edu",
+ "mail-type": ["END"],
+ "output": 'OutputLog/job_id_${job_id}".log',
+ "error": 'ErrorLog/job_id_${job_id}".log',
}
slurm_dict_spockmk2_ephys = {
- 'job-name': 'kilosort2',
- 'nodes': 1,
- 'ntasks': 1,
- 'time': '30:00:00',
- 'mem': '50G',
- 'gres': 'gpu:2',
- 'mail-user': 'alvaros@princeton.edu',
- 'mail-type': ['END'],
- 'output': 'OutputLog/job_id_${job_id}".log',
- 'error': 'ErrorLog/job_id_${job_id}".log'
+ "job-name": "kilosort2",
+ "nodes": 1,
+ "ntasks": 1,
+ "time": "30:00:00",
+ "mem": "50G",
+ "gres": "gpu:2",
+ "mail-user": "alvaros@princeton.edu",
+ "mail-type": ["END"],
+ "output": 'OutputLog/job_id_${job_id}".log',
+ "error": 'ErrorLog/job_id_${job_id}".log',
}
slurm_dict_spock_default = {
- 'job-name': 'kilosort2',
- 'nodes': 1,
- 'ntasks': 1,
- 'time': '30:00:00',
- 'mem': '50G',
- 'gres': 'gpu:2',
- 'mail-user': 'alvaros@princeton.edu',
- 'mail-type': ['END'],
- 'output': 'OutputLog/job_id_${job_id}".log',
- 'error': 'ErrorLog/job_id_${job_id}".log'
+ "job-name": "kilosort2",
+ "nodes": 1,
+ "ntasks": 1,
+ "time": "30:00:00",
+ "mem": "50G",
+ "gres": "gpu:2",
+ "mail-user": "alvaros@princeton.edu",
+ "mail-type": ["END"],
+ "output": 'OutputLog/job_id_${job_id}".log',
+ "error": 'ErrorLog/job_id_${job_id}".log',
}
-#PNI directories
-pni_root_data_dir = dj.config['custom']['root_data_dir']
+# PNI directories
+pni_root_data_dir = dj.config["custom"]["root_data_dir"]
-tiger_home_dir = '/scratch/gpfs/BRAINCOGS'
-spock_home_dir = '/mnt/cup/braininit/Shared/repos/AutomaticPipelineProcessing'
+tiger_home_dir = "/scratch/gpfs/BRAINCOGS"
+spock_home_dir = "/mnt/cup/braininit/Shared/repos/AutomaticPipelineProcessing"
-#Cluster directories
+# Cluster directories
cluster_vars = {
"tiger": {
- "home_dir": tiger_home_dir,
- "root_data_dir_globus": tiger_home_dir_globus + "/Raw",
- "processed_data_dir_globus": tiger_home_dir_globus + "/Processed",
- "root_data_dir": tiger_home_dir + "/Data/Raw",
- "processed_data_dir": tiger_home_dir + "/Data/Processed",
- "slurm_files_dir": tiger_home_dir + "/SlurmFiles",
- "params_files_dir": tiger_home_dir + "/ParameterFiles",
- "chanmap_files_dir": tiger_home_dir + "/ChanMapFiles",
+ "home_dir": tiger_home_dir,
+ "root_data_dir_globus": tiger_home_dir_globus + "/Raw",
+ "processed_data_dir_globus": tiger_home_dir_globus + "/Processed",
+ "root_data_dir": tiger_home_dir + "/Data/Raw",
+ "processed_data_dir": tiger_home_dir + "/Data/Processed",
+ "slurm_files_dir": tiger_home_dir + "/SlurmFiles",
+ "params_files_dir": tiger_home_dir + "/ParameterFiles",
+ "chanmap_files_dir": tiger_home_dir + "/ChanMapFiles",
"electrophysiology_process_dir": tiger_home_dir + "/electrophysiology_processing",
- "imaging_process_dir": tiger_home_dir + "/imaging_processing",
- "log_files_dir": tiger_home_dir + "/OutputLog",
- "error_files_dir": tiger_home_dir + "/ErrorLog",
- "user": default_user,
- "slurm_default": slurm_dict_tiger_default,
- "hostname": "della-gpu.princeton.edu",
- "script_path": "",
- "conda_env": '/home/alvaros/.conda/envs/BrainCogsEphysSorters_env'
-
+ "imaging_process_dir": tiger_home_dir + "/imaging_processing",
+ "log_files_dir": tiger_home_dir + "/OutputLog",
+ "error_files_dir": tiger_home_dir + "/ErrorLog",
+ "user": default_user,
+ "slurm_default": slurm_dict_tiger_default,
+ "hostname": "della-gpu.princeton.edu",
+ "script_path": "",
+ "conda_env": "/home/alvaros/.conda/envs/BrainCogsEphysSorters_env",
},
"spock": {
- "home_dir": spock_home_dir,
- "root_data_dir_globus": pni_data_dir + "/Raw",
- "processed_data_dir_globus": pni_data_dir + "/Processed",
- "root_data_dir": spock_home_dir + "/Raw",
- "processed_data_dir": spock_home_dir + "/Processed",
- "slurm_files_dir": spock_home_dir + "/SlurmFiles",
- "params_files_dir": spock_home_dir + "/ParameterFiles",
- "chanmap_files_dir": spock_home_dir + "/ChanMapFiles",
+ "home_dir": spock_home_dir,
+ "root_data_dir_globus": pni_data_dir + "/Raw",
+ "processed_data_dir_globus": pni_data_dir + "/Processed",
+ "root_data_dir": spock_home_dir + "/Raw",
+ "processed_data_dir": spock_home_dir + "/Processed",
+ "slurm_files_dir": spock_home_dir + "/SlurmFiles",
+ "params_files_dir": spock_home_dir + "/ParameterFiles",
+ "chanmap_files_dir": spock_home_dir + "/ChanMapFiles",
"electrophysiology_process_dir": spock_home_dir + "/electrophysiology_processing",
- "imaging_process_dir": spock_home_dir + "/imaging_processing",
- "log_files_dir": spock_home_dir + "/u19_pipeline/automatic_job/OutputLog",
- "error_files_dir": spock_home_dir + "/u19_pipeline/automatic_job/ErrorLog",
- "user": default_user,
- "slurm_default": slurm_dict_spock_default,
- "hostname": "spockmk2-loginvm.pni.princeton.edu",
- "script_path": "",
- "conda_env": 'u19_pipeline_python_env'
- }
+ "imaging_process_dir": spock_home_dir + "/imaging_processing",
+ "log_files_dir": spock_home_dir + "/u19_pipeline/automatic_job/OutputLog",
+ "error_files_dir": spock_home_dir + "/u19_pipeline/automatic_job/ErrorLog",
+ "user": default_user,
+ "slurm_default": slurm_dict_spock_default,
+ "hostname": "spockmk2-loginvm.pni.princeton.edu",
+ "script_path": "",
+ "conda_env": "u19_pipeline_python_env",
+ },
}
@@ -120,13 +122,13 @@ def get_cluster_vars(cluster):
if cluster in cluster_vars:
return cluster_vars[cluster]
else:
- raise('Non existing cluster')
+ raise ValueError("Non existing cluster")
-def scp_file_transfer(source, dest):
- print("scp", source, dest)
+def scp_file_transfer(source, dest):
- print(["scp", "-i", public_key_location, source, dest])
+ logger.debug("scp %s %s", source, dest)
+ logger.debug("scp command %s", ["scp", "-i", public_key_location, source, dest])
p = subprocess.Popen(["scp", "-i", public_key_location, source, dest])
transfer_status = p.wait()
@@ -135,126 +137,143 @@ def scp_file_transfer(source, dest):
def scp_file_transfer_big_files(source, dest, source_filepath, dest_filepath):
- print("scp", source, dest)
+ logger.debug("scp big files %s %s", source, dest)
# IMPLEMENT HERE
def cp_file_transfer(source, dest):
- print("cp", source, dest)
+ logger.debug("cp %s %s", source, dest)
p = subprocess.Popen(["cp", source, dest])
transfer_status = p.wait()
return transfer_status
-def request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, dest_filepath):
- source_fullpath = source_ep+ ":" + source_filepath
- dest_fullpath = dest_ep + ":" + dest_filepath
+def request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, dest_filepath):
- globus_command = ["globus", "transfer", source_fullpath, dest_fullpath, '--label', job_id_str, '--recursive', '--format', 'json']
- print('**********************************')
- print(globus_command)
- print('**********************************')
+ source_fullpath = source_ep + ":" + source_filepath
+ dest_fullpath = dest_ep + ":" + dest_filepath
+
+ globus_command = [
+ "globus",
+ "transfer",
+ source_fullpath,
+ dest_fullpath,
+ "--label",
+ job_id_str,
+ "--recursive",
+ "--format",
+ "json",
+ ]
+ logger.debug("**********************************")
+ logger.debug("globus_command %s", globus_command)
+ logger.debug("**********************************")
p = subprocess.run(globus_command, capture_output=True)
- print(p)
- transfer_request = dict()
- print('p.stderr',p.stderr)
- print('p.stdout', p.stdout)
+ logger.debug("globus run result %s", p)
+ transfer_request = {}
+ logger.debug("p.stderr %s", p.stderr)
+ logger.debug("p.stdout %s", p.stdout)
if len(p.stderr) == 0:
- try:
- dict_output = json.loads(p.stdout.decode('UTF-8'))
- transfer_request['status'] = config.system_process['SUCCESS']
- transfer_request['task_id'] = dict_output['task_id']
- except Exception as e:
- print('stdout is not a valid json, probably an error')
- transfer_request['status'] = config.system_process['ERROR']
- transfer_request['error_info'] = p.stdout.decode('UTF-8')
- #dict_output = translate_globus_output(p.stdout)
+ try:
+ dict_output = json.loads(p.stdout.decode("UTF-8"))
+ transfer_request["status"] = config.system_process["SUCCESS"]
+ transfer_request["task_id"] = dict_output["task_id"]
+ except Exception:
+ logger.warning("stdout is not a valid json, probably an error")
+ transfer_request["status"] = config.system_process["ERROR"]
+ transfer_request["error_info"] = p.stdout.decode("UTF-8")
+ # dict_output = translate_globus_output(p.stdout)
else:
- transfer_request['status'] = config.system_process['ERROR']
- transfer_request['error_info'] = p.stderr.decode('UTF-8')
+ transfer_request["status"] = config.system_process["ERROR"]
+ transfer_request["error_info"] = p.stderr.decode("UTF-8")
return transfer_request
def request_globus_transfer_status(job_id):
- globus_command = ["globus", "task", "show", job_id, '--format', 'json']
+ globus_command = ["globus", "task", "show", job_id, "--format", "json"]
s = subprocess.run(globus_command, capture_output=True)
- task_output = json.loads(s.stdout.decode('UTF-8'))
+ task_output = json.loads(s.stdout.decode("UTF-8"))
- transfer_request = dict()
- if task_output['status'] == 'SUCCEEDED':
- transfer_request['status'] = config.system_process['COMPLETED']
- elif task_output['status'] in ['PENDING','RETRYING', 'ACTIVE']:
- transfer_request['status'] = config.system_process['SUCCESS']
+ transfer_request = {}
+ if task_output["status"] == "SUCCEEDED":
+ transfer_request["status"] = config.system_process["COMPLETED"]
+ elif task_output["status"] in ["PENDING", "RETRYING", "ACTIVE"]:
+ transfer_request["status"] = config.system_process["SUCCESS"]
else:
- transfer_request['status'] = config.system_process['ERROR']
+ transfer_request["status"] = config.system_process["ERROR"]
return transfer_request
def globus_transfer_to_tiger(job_id, raw_rel_path, modality):
- job_id_str = "job_id_"+str(job_id)+"_raw_transfer"
+ job_id_str = "job_id_" + str(job_id) + "_raw_transfer"
source_ep = pni_ep_id
- dest_ep = tiger_ep_dir
+ dest_ep = tiger_ep_dir
- source_filepath = pathlib.Path(cluster_vars['spock']['root_data_dir_globus'], modality, raw_rel_path).as_posix()
- dest_filepath = pathlib.Path(cluster_vars['tiger']['root_data_dir_globus'], modality, raw_rel_path).as_posix()
+ source_filepath = pathlib.Path(cluster_vars["spock"]["root_data_dir_globus"], modality, raw_rel_path).as_posix()
+ dest_filepath = pathlib.Path(cluster_vars["tiger"]["root_data_dir_globus"], modality, raw_rel_path).as_posix()
transfer_request = request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, dest_filepath)
return transfer_request
+
def globus_transfer_to_pni(job_id, processed_rel_path, modality):
- job_id_str = "job_id_"+str(job_id)+"_processed_transfer"
+ job_id_str = "job_id_" + str(job_id) + "_processed_transfer"
source_ep = tiger_ep_dir
- dest_ep = pni_ep_id
+ dest_ep = pni_ep_id
- dest_filepath = pathlib.Path(cluster_vars['spock']['processed_data_dir_globus'], modality, processed_rel_path).as_posix()
- source_filepath = pathlib.Path(cluster_vars['tiger']['processed_data_dir_globus'], modality, processed_rel_path).as_posix()
+ dest_filepath = pathlib.Path(
+ cluster_vars["spock"]["processed_data_dir_globus"], modality, processed_rel_path
+ ).as_posix()
+ source_filepath = pathlib.Path(
+ cluster_vars["tiger"]["processed_data_dir_globus"], modality, processed_rel_path
+ ).as_posix()
transfer_request = request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, dest_filepath)
return transfer_request
+
def translate_globus_output(stdout_process):
- u = stdout_process.decode('UTF-8')
+ u = stdout_process.decode("UTF-8")
- n = u.split(sep='\n')
- n2 = [x.split(sep=':', maxsplit=1) for x in n]
+ n = u.split(sep="\n")
+ n2 = [x.split(sep=":", maxsplit=1) for x in n]
- flat_list = [item for l in n2 for item in l]
+ flat_list = [item for sublist in n2 for item in sublist]
flat_list2 = [x.strip() for x in flat_list]
- d1 = dict(zip(flat_list2[::2], flat_list2[1::2]))
+ d1 = dict(zip(flat_list2[::2], flat_list2[1::2], strict=False))
return d1
-def transfer_log_file(recording_process_id, program_selection_params, user_host, log_type='ERROR'):
- '''
+def transfer_log_file(recording_process_id, program_selection_params, user_host, log_type="ERROR"):
+ """
Transfer and send parameter files for processing
- '''
- this_cluster_vars = get_cluster_vars(program_selection_params['process_cluster'])
- if log_type == 'ERROR':
- cluster_log_file_dir = this_cluster_vars['error_files_dir']
- local_log_file_dir = dj.config['custom']['error_logs_dir']
+ """
+ this_cluster_vars = get_cluster_vars(program_selection_params["process_cluster"])
+ if log_type == "ERROR":
+ cluster_log_file_dir = this_cluster_vars["error_files_dir"]
+ local_log_file_dir = dj.config["custom"]["error_logs_dir"]
else:
- cluster_log_file_dir = this_cluster_vars['log_files_dir']
- local_log_file_dir = dj.config['custom']['output_logs_dir']
+ cluster_log_file_dir = this_cluster_vars["log_files_dir"]
+ local_log_file_dir = dj.config["custom"]["output_logs_dir"]
- user_host = this_cluster_vars['user']+'@'+this_cluster_vars['hostname']
+ user_host = this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"]
- default_log_filename = 'job_id_%s.log'
+ default_log_filename = "job_id_%s.log"
log_filename = default_log_filename % (recording_process_id)
- log_file_local_path = pathlib.Path(local_log_file_dir,log_filename).as_posix()
- log_file_cluster_path = pathlib.Path(cluster_log_file_dir,log_filename).as_posix()
- chanmap_file_full_path = user_host+':'+log_file_cluster_path
+ log_file_local_path = pathlib.Path(local_log_file_dir, log_filename).as_posix()
+ log_file_cluster_path = pathlib.Path(cluster_log_file_dir, log_filename).as_posix()
+ chanmap_file_full_path = user_host + ":" + log_file_cluster_path
status = scp_file_transfer(chanmap_file_full_path, log_file_local_path)
@@ -263,48 +282,47 @@ def transfer_log_file(recording_process_id, program_selection_params, user_host,
def get_error_log_str(recording_process_id):
- error_log_data = ''
- default_log_filename = 'job_id_%s.log'
+ error_log_data = ""
+ default_log_filename = "job_id_%s.log"
- local_log_file_dir = dj.config['custom']['error_logs_dir']
+ local_log_file_dir = dj.config["custom"]["error_logs_dir"]
log_filename = default_log_filename % (recording_process_id)
- log_file_local_path = pathlib.Path(local_log_file_dir,log_filename).as_posix()
+ log_file_local_path = pathlib.Path(local_log_file_dir, log_filename).as_posix()
if os.path.exists(log_file_local_path):
- with open(log_file_local_path, 'r') as error_log_file:
- error_log_data = ' '.join(error_log_file.readlines())
+ with open(log_file_local_path) as error_log_file:
+ error_log_data = " ".join(error_log_file.readlines())
- error_log_data = error_log_data.replace("activate the default environment with 'conda activate' or create a new environment to customize with 'conda create'.\n",'')
+ error_log_data = error_log_data.replace(
+ "activate the default environment with 'conda activate' or create a new environment to customize with 'conda create'.\n",
+ "",
+ )
- print('error_log_data xxxxxxxxxx')
- print('type(error_log_data)', type(error_log_data))
- print('len(error_log_data)', len(error_log_data))
- #if len(error_log_data) > 400:
- # print(error_log_data[-400:])
- #else:
- # print(error_log_data)
- print('error_log_data xxxxxxxxxx')
+ logger.debug("error_log_data xxxxxxxxxx")
+ logger.debug("type(error_log_data) %s", type(error_log_data))
+ logger.debug("len(error_log_data) %s", len(error_log_data))
+ logger.debug("error_log_data xxxxxxxxxx")
return error_log_data
-def check_directory_exists_cluster(directory, cluster, modality, type_dir='raw'):
- '''
+def check_directory_exists_cluster(directory, cluster, modality, type_dir="raw"):
+ """
Check if directory exists in cluster, runs check_directory_script (check_directory.sh) in cluster machine
Output
dir_exists 1 if directory exists / 0 otherwise
- '''
+ """
this_cluster_vars = get_cluster_vars(cluster)
- if type_dir=='raw':
- final_directory = pathlib.Path(cluster_vars[cluster]['root_data_dir'], modality, directory).as_posix()
+ if type_dir == "raw":
+ final_directory = pathlib.Path(cluster_vars[cluster]["root_data_dir"], modality, directory).as_posix()
else:
- final_directory = pathlib.Path(cluster_vars[cluster]['processed_data_dir'], modality, directory).as_posix()
+ final_directory = pathlib.Path(cluster_vars[cluster]["processed_data_dir"], modality, directory).as_posix()
- base_command = "ssh " + this_cluster_vars['user']+'@'+this_cluster_vars['hostname'] + " "
+ base_command = "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " "
- command = "'if [ -d " + final_directory + " ]; "
+ command = "'if [ -d " + final_directory + " ]; "
post_command = """then
echo "1"
else echo "0"
@@ -320,14 +338,13 @@ def check_directory_exists_cluster(directory, cluster, modality, type_dir='raw')
def delete_directory_cluster(directory, cluster):
- '''
+ """
Delete directory in cluster
- '''
+ """
this_cluster_vars = get_cluster_vars(cluster)
- command = "ssh " + this_cluster_vars['user']+'@'+this_cluster_vars['hostname']\
- + " 'rm -R " + directory + " '"
+ command = "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " 'rm -R " + directory + " '"
p = subprocess.run(command, shell=True)
output = p.returncode
@@ -339,36 +356,33 @@ def delete_directory_cluster(directory, cluster):
def delete_directory_tiger_globus(modality, raw_rel_path):
- '''
+ """
Delete directory in cluster with globus command (for Raw directory globus)
- '''
+ """
source_ep = tiger_ep_dir
- source_filepath = pathlib.Path(cluster_vars['tiger']['root_data_dir_globus'], modality, raw_rel_path).as_posix()
+ source_filepath = pathlib.Path(cluster_vars["tiger"]["root_data_dir_globus"], modality, raw_rel_path).as_posix()
- source_fullpath = source_ep+ ":" + source_filepath
+ source_fullpath = source_ep + ":" + source_filepath
- globus_command = ["globus", "delete", source_fullpath, '--recursive']
+ globus_command = ["globus", "delete", source_fullpath, "--recursive"]
p = subprocess.run(globus_command, capture_output=True)
output = p.returncode
return output
-def delete_empty_data_directory_cluster(cluster, type='raw'):
+def delete_empty_data_directory_cluster(cluster, type="raw"):
"""
Check if directory (or its childs) contains files and if not delete them
"""
max_deletion = 10
this_cluster_vars = get_cluster_vars(cluster)
- base_command = "ssh " + this_cluster_vars['user']+'@'+this_cluster_vars['hostname'] + ' '
+ base_command = "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " "
# Check base directory to delete
- if type == 'raw':
- filepath = this_cluster_vars['root_data_dir']
- else:
- filepath = this_cluster_vars['processed_data_dir']
+ filepath = this_cluster_vars["root_data_dir"] if type == "raw" else this_cluster_vars["processed_data_dir"]
# Repeat always if we find a directory to delete
total_deletion = 0
@@ -376,52 +390,49 @@ def delete_empty_data_directory_cluster(cluster, type='raw'):
deleted_dirs = 0
# List all directories on base (raw/processed) directory
- command_list_dir = base_command + 'find ' + filepath + ' -type d -print'
+ command_list_dir = base_command + "find " + filepath + " -type d -print"
p = subprocess.run(command_list_dir, shell=True, capture_output=True)
list_dir = p.stdout.decode()
- list_dir = list_dir.split('\n')
- if '.' in list_dir:
- list_dir.remove('.')
- if '' in list_dir:
- list_dir.remove('')
-
- for dir in list_dir:
+ list_dir = list_dir.split("\n")
+ if "." in list_dir:
+ list_dir.remove(".")
+ if "" in list_dir:
+ list_dir.remove("")
+ for directory in list_dir:
# Check if directory has no files in it (empty)
- command = base_command + 'find ' + dir + ' -type f | wc -l'
+ command = base_command + "find " + directory + " -type f | wc -l"
num_files = subprocess.check_output(command, shell=True)
num_files = int(num_files.decode().strip())
- #If directory empty, delete it
+ # If directory empty, delete it
if num_files == 0:
- if type == 'raw':
-
+ if type == "raw":
# For raw directories, delete with globus
# Translate local directory to globus dir
- dir_globus = dir.replace(this_cluster_vars['root_data_dir'], '')
+ dir_globus = dir.replace(this_cluster_vars["root_data_dir"], "")
dir_globus = dir_globus[1:]
index_sep = dir_globus.index("/")
modality = dir_globus[0:index_sep]
- dir_globus = dir_globus[index_sep+1:]
+ dir_globus = dir_globus[index_sep + 1 :]
- #Delete with globus
+ # Delete with globus
status = delete_directory_tiger_globus(modality, dir_globus)
time.sleep(2)
- if status == config.system_process['SUCCESS']:
+ if status == config.system_process["SUCCESS"]:
deleted_dirs = 1
break
- total_deletion +=1
+ total_deletion += 1
else:
- #Delete processed files with "normal" ssh
+ # Delete processed files with "normal" ssh
status = delete_directory_cluster(dir, cluster)
- if status == config.system_process['SUCCESS']:
+ if status == config.system_process["SUCCESS"]:
deleted_dirs = 1
break
- total_deletion +=1
+ total_deletion += 1
# If in this round no directories were deleted we are done
if deleted_dirs == 0 or total_deletion >= max_deletion:
break
-
diff --git a/u19_pipeline/automatic_job/create_missing_lfps_script.py b/u19_pipeline/automatic_job/create_missing_lfps_script.py
index 240930fb..887b19c1 100644
--- a/u19_pipeline/automatic_job/create_missing_lfps_script.py
+++ b/u19_pipeline/automatic_job/create_missing_lfps_script.py
@@ -1,31 +1,43 @@
-
import pathlib
import sys
-import datajoint as dj
-from u19_pipeline import recording, ephys_pipeline, recording, recording_process
+import datajoint as dj
import u19_pipeline.automatic_job.params_config as config
+from u19_pipeline import ephys_pipeline, recording, recording_process
+from u19_pipeline.utils.logging_config import get_logger
+logger = get_logger(__name__)
-def main(recording_id):
- rec_query = dict()
- rec_query['recording_id'] = recording_id
- recording_processes = (recording_process.Processing() & rec_query).fetch('job_id', 'recording_id', 'fragment_number', 'recording_process_pre_path', as_dict=True)
- #Create lfp trace if needed (neuropixel 2.0 probes)
- recording_directory = (recording.Recording & rec_query).fetch1('recording_directory')
- recording_directory = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][0], recording_directory).parent.as_posix()
+def main(recording_id):
+ rec_query = {}
+ rec_query["recording_id"] = recording_id
+
+ recording_processes = (recording_process.Processing() & rec_query).fetch(
+ "job_id",
+ "recording_id",
+ "fragment_number",
+ "recording_process_pre_path",
+ as_dict=True,
+ )
+ # Create lfp trace if needed (neuropixel 2.0 probes)
+ recording_directory = (recording.Recording & rec_query).fetch1("recording_directory")
+ recording_directory = pathlib.Path(
+ dj.config["custom"]["ephys_root_data_dir"][0], recording_directory
+ ).parent.as_posix()
for i in recording_processes:
- probe_dir = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][0], i['recording_process_pre_path']).as_posix()
+ probe_dir = pathlib.Path(
+ dj.config["custom"]["ephys_root_data_dir"][0],
+ i["recording_process_pre_path"],
+ ).as_posix()
ephys_pipeline.create_lfp_trace(config.catgt_script, recording_directory, probe_dir)
if __name__ == "__main__":
-
-
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
args = sys.argv[1:]
- print(args)
- main(args[0])
\ No newline at end of file
+ logger.debug("args %s", args)
+ main(args[0])
diff --git a/u19_pipeline/automatic_job/cronjob_automatic_job.py b/u19_pipeline/automatic_job/cronjob_automatic_job.py
index 84a052a6..35ff5767 100644
--- a/u19_pipeline/automatic_job/cronjob_automatic_job.py
+++ b/u19_pipeline/automatic_job/cronjob_automatic_job.py
@@ -1,6 +1,7 @@
-
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(1)
@@ -16,7 +17,5 @@
rec_process_handler.RecProcessHandler.pipeline_handler_main()
time.sleep(5)
-#Check if we need to delete empty directories
-#rec_process_handler.RecProcessHandler.check_job_process_deletion()
-
-
+# Check if we need to delete empty directories
+# rec_process_handler.RecProcessHandler.check_job_process_deletion()
diff --git a/u19_pipeline/automatic_job/ephys_element_ingest.py b/u19_pipeline/automatic_job/ephys_element_ingest.py
index 7b6c674d..cd39a825 100644
--- a/u19_pipeline/automatic_job/ephys_element_ingest.py
+++ b/u19_pipeline/automatic_job/ephys_element_ingest.py
@@ -1,13 +1,20 @@
-import re
import pathlib
-from u19_pipeline import ephys_pipeline
+import re
-from u19_pipeline.ephys_pipeline import (probe_element, ephys_element,
- get_ephys_root_data_dir, get_session_directory)
from element_array_ephys.readers import spikeglx
-
from element_interface.utils import find_full_path
+from u19_pipeline import ephys_pipeline
+from u19_pipeline.ephys_pipeline import (
+ ephys_element,
+ get_ephys_root_data_dir,
+ get_session_directory,
+ probe_element,
+)
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
"""
The ingestion routine for imaging element includes:
@@ -22,7 +29,7 @@
(for an example, see: https://github.com/ttngu207/workflow-ephys/blob/main/notebooks/run_workflow.ipynb)
"""
-acq_software = 'SpikeGLX'
+acq_software = "SpikeGLX"
def process_session(sess_key):
@@ -33,36 +40,44 @@ def process_session(sess_key):
:param scan_key: a `KEY` of `acquisition.Session`
"""
- sess_dir = pathlib.Path(find_full_path(get_ephys_root_data_dir(),
- get_session_directory(sess_key)))
- ephys_meta_filepaths = [fp for fp in sess_dir.rglob('*.ap.meta')]
+ sess_dir = pathlib.Path(find_full_path(get_ephys_root_data_dir(), get_session_directory(sess_key)))
+ ephys_meta_filepaths = list(sess_dir.rglob("*.ap.meta"))
- print('ephys_meta_filepaths', ephys_meta_filepaths)
+ logger.debug("ephys_meta_filepaths %s", ephys_meta_filepaths)
if not len(ephys_meta_filepaths):
- print(f'No SpikeGLX data found for session:{sess_key} - at {sess_dir}')
+ logger.warning("No SpikeGLX data found for session:%s - at %s", sess_key, sess_dir)
return
probe_list, probe_insertion_list = [], []
for meta_filepath in ephys_meta_filepaths:
- print('meta_filepath', meta_filepath)
+ logger.debug("meta_filepath %s", meta_filepath)
spikeglx_meta = spikeglx.SpikeGLXMeta(meta_filepath)
- probe_key = {'probe_type': spikeglx_meta.probe_model, 'probe': spikeglx_meta.probe_SN}
- if probe_key['probe'] not in [p['probe'] for p in probe_list] and probe_key not in probe_element.Probe():
+ probe_key = {
+ "probe_type": spikeglx_meta.probe_model,
+ "probe": spikeglx_meta.probe_SN,
+ }
+ if probe_key["probe"] not in [p["probe"] for p in probe_list] and probe_key not in probe_element.Probe():
probe_list.append(probe_key)
probe_dir = meta_filepath.parent
- probe_number = re.search('(imec)?\d{1}$', probe_dir.name).group()
- probe_number = int(probe_number.replace('imec', ''))
-
- probe_insertion_list.append({**sess_key, 'probe': spikeglx_meta.probe_SN, 'insertion_number': int(probe_number)})
-
- print(f"{probe_list =}")
+ probe_number = re.search("(imec)?\d{1}$", probe_dir.name).group()
+ probe_number = int(probe_number.replace("imec", ""))
+
+ probe_insertion_list.append(
+ {
+ **sess_key,
+ "probe": spikeglx_meta.probe_SN,
+ "insertion_number": int(probe_number),
+ }
+ )
+
+ logger.debug("probe_list %s", probe_list)
probe_element.Probe.insert(probe_list, skip_duplicates=True)
ephys_element.ProbeInsertion.insert(probe_insertion_list, skip_duplicates=True)
-if __name__ == '__main__':
- for sess_key in ephys_pipeline.EphysPipelineSession.fetch('KEY'):
- process_session(sess_key)
\ No newline at end of file
+if __name__ == "__main__":
+ for sess_key in ephys_pipeline.EphysPipelineSession.fetch("KEY"):
+ process_session(sess_key)
diff --git a/u19_pipeline/automatic_job/ephys_element_populate.py b/u19_pipeline/automatic_job/ephys_element_populate.py
index 40ae3744..65f19ece 100644
--- a/u19_pipeline/automatic_job/ephys_element_populate.py
+++ b/u19_pipeline/automatic_job/ephys_element_populate.py
@@ -1,66 +1,66 @@
-from u19_pipeline.ephys_pipeline import probe_element, ephys_element
+import u19_pipeline.automatic_job.params_config as config
from u19_pipeline import recording, recording_process
+from u19_pipeline.ephys_pipeline import ephys_element, probe_element
-import u19_pipeline.automatic_job.params_config as config
def populate_element_data(job_id, display_progress=True, reserve_jobs=False, suppress_errors=False):
- populate_settings = {'display_progress': display_progress,
- 'reserve_jobs': reserve_jobs,
- 'suppress_errors': suppress_errors}
+ populate_settings = {
+ "display_progress": display_progress,
+ "reserve_jobs": reserve_jobs,
+ "suppress_errors": suppress_errors,
+ }
- process_key = (recording_process.Processing * recording.Recording &
- dict(recording_modality='electrophysiology',
- job_id=job_id)).fetch1('KEY')
+ process_key = (
+ recording_process.Processing * recording.Recording & {"recording_modality": "electrophysiology", "job_id": job_id}
+ ).fetch1("KEY")
- fragment_number, recording_process_pre_path, recording_process_post_path = \
- (recording_process.Processing & process_key).fetch1(
- 'fragment_number',
- 'recording_process_pre_path',
- 'recording_process_post_path')
+ fragment_number, recording_process_pre_path, recording_process_post_path = (
+ recording_process.Processing & process_key
+ ).fetch1("fragment_number", "recording_process_pre_path", "recording_process_post_path")
- precluster_param_steps_id, paramset_idx = \
- (recording_process.Processing.EphysParams & process_key
- ).fetch1('precluster_param_steps_id',
- 'paramset_idx')
+ precluster_param_steps_id, paramset_idx = (recording_process.Processing.EphysParams & process_key).fetch1(
+ "precluster_param_steps_id", "paramset_idx"
+ )
- precluster_paramsets = (ephys_element.PreClusterParamSteps.Step() &
- dict(
- precluster_param_steps_id=precluster_param_steps_id)
- ).fetch('paramset_idx')
+ precluster_paramsets = (
+ ephys_element.PreClusterParamSteps.Step() & {"precluster_param_steps_id": precluster_param_steps_id}
+ ).fetch("paramset_idx")
- clustering_method = (ephys_element.ClusteringParamSet &
- dict(paramset_idx=paramset_idx)).fetch1(
- 'clustering_method')
+ clustering_method = (ephys_element.ClusteringParamSet & {"paramset_idx": paramset_idx}).fetch1("clustering_method")
- if len(precluster_paramsets)==0:
- task_mode = 'none'
- else:
- task_mode = 'load'
+ task_mode = "none" if len(precluster_paramsets) == 0 else "load"
- precluster_key = dict(recording_id=process_key['recording_id'],
- insertion_number=fragment_number,
- precluster_param_steps_id=precluster_param_steps_id)
+ precluster_key = {
+ "recording_id": process_key["recording_id"],
+ "insertion_number": fragment_number,
+ "precluster_param_steps_id": precluster_param_steps_id,
+ }
ephys_element.PreClusterTask.insert1(
- dict(**precluster_key,
- precluster_output_dir=recording_process_pre_path,
- task_mode=task_mode),
- skip_duplicates=True)
+ dict(
+ **precluster_key,
+ precluster_output_dir=recording_process_pre_path,
+ task_mode=task_mode,
+ ),
+ skip_duplicates=True,
+ )
ephys_element.PreCluster.populate(precluster_key, **populate_settings)
- if '1.0' in (ephys_element.ProbeInsertion * probe_element.Probe &
- precluster_key).fetch1('probe_type'):
+ if "1.0" in (ephys_element.ProbeInsertion * probe_element.Probe & precluster_key).fetch1("probe_type"):
ephys_element.LFP.populate(precluster_key, **populate_settings)
- cluster_key = dict(**precluster_key,
- paramset_idx=paramset_idx)
+ cluster_key = dict(**precluster_key, paramset_idx=paramset_idx)
ephys_element.ClusteringTask.insert1(
- dict(**cluster_key,
- clustering_output_dir=f'{recording_process_post_path}/{clustering_method}_output',
- task_mode='load'), skip_duplicates=True)
+ dict(
+ **cluster_key,
+ clustering_output_dir=f"{recording_process_post_path}/{clustering_method}_output",
+ task_mode="load",
+ ),
+ skip_duplicates=True,
+ )
ephys_element.Clustering.populate(cluster_key, **populate_settings)
@@ -72,8 +72,8 @@ def populate_element_data(job_id, display_progress=True, reserve_jobs=False, sup
# ephys_element.WaveformSet.populate(cluster_key, **populate_settings)
- return config.status_update_idx['NEXT_STATUS']
-
+ return config.status_update_idx["NEXT_STATUS"]
+
-if __name__ == '__main__':
+if __name__ == "__main__":
populate_element_data()
diff --git a/u19_pipeline/automatic_job/imaging_element.py b/u19_pipeline/automatic_job/imaging_element.py
index 57095fcc..9a7a1ea4 100644
--- a/u19_pipeline/automatic_job/imaging_element.py
+++ b/u19_pipeline/automatic_job/imaging_element.py
@@ -1,20 +1,22 @@
-from concurrent.futures import process
-import os
-import datajoint as dj
-import numpy as np
-from u19_pipeline import lab, imaging_rec, acquisition, subject, recording
-from u19_pipeline.imaging_element import imaging_element, scan_element, \
- get_processed_dir, Equipment,\
- get_imaging_root_data_dir, \
- get_scan_image_files
-from u19_pipeline.ingest.imaging_element_ingest import process_scan
-from element_interface.scanimage_utils import get_scanimage_acq_time, parse_scanimage_header
-from element_interface.utils import find_full_path
-import scanreader
+import os
import pathlib
+
+import scanreader
import tifffile
-import datetime
-import h5py
+from element_interface.scanimage_utils import (
+ parse_scanimage_header,
+)
+
+from u19_pipeline import imaging_rec, lab, recording
+from u19_pipeline.imaging_element import (
+ get_imaging_root_data_dir,
+ get_scan_image_files,
+ imaging_element,
+ scan_element,
+)
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
# subject_fullname = 'koay_K65'
# session_date = '2018-02-02'
@@ -23,11 +25,11 @@
# session_date = '2021-02-27'
# session_number = 0
-acq_software = 'ScanImage'
-#recording_id = os.environ['recording_id']
-recording_process_id = os.environ['recording_process_id']
-#process_method = os.environ['process_method']
-#paramset_idx = os.environ['paramset_idx']
+acq_software = "ScanImage"
+# recording_id = os.environ['recording_id']
+recording_process_id = os.environ["RECORDING_PROCESS_ID"]
+# process_method = os.environ['process_method']
+# paramset_idx = os.environ['paramset_idx']
# parameters = {
# 'look_one_level_down': 0.0,
@@ -93,102 +95,108 @@
# 'yrange': np.array([0, 0])}
-
-# parameters_caiman = {'fr': 30,
+# parameters_caiman = {'fr': 30,
# 'decay_time': 0.4,
-# 'strides': (48, 48),
-# 'overlaps': (24, 24),
-# 'max_shifts': (6,6),
-# 'max_deviation_rigid': 3,
-# 'pw_rigid': True,
-# 'p': 1,
-# 'gnb': 2,
-# 'merge_thr': 0.85,
-# 'rf': 15,
-# 'stride_cnmf': 6,
-# 'K': 4,
-# 'gSig': [4, 4],
-# 'method_init': 'greedy_roi',
-# 'ssub': 1,
-# 'tsub': 1,
-# 'min_SNR': 2.0,
-# 'rval_thr': 0.85,
-# 'cnn_thr': 0.99,
+# 'strides': (48, 48),
+# 'overlaps': (24, 24),
+# 'max_shifts': (6,6),
+# 'max_deviation_rigid': 3,
+# 'pw_rigid': True,
+# 'p': 1,
+# 'gnb': 2,
+# 'merge_thr': 0.85,
+# 'rf': 15,
+# 'stride_cnmf': 6,
+# 'K': 4,
+# 'gSig': [4, 4],
+# 'method_init': 'greedy_roi',
+# 'ssub': 1,
+# 'tsub': 1,
+# 'min_SNR': 2.0,
+# 'rval_thr': 0.85,
+# 'cnn_thr': 0.99,
# 'cnn_lowest': 0.1}
-# imaging_element.ProcessingParamSet.insert_new_params('suite2p', 0, 'Calcium imaging analysis with Suite2p using default Suite2p parameters', parameters)
-# imaging_element.ProcessingParamSet.insert_new_params('caiman', 1, 'Calcium imaging analysis with CaImAn using default CaImAn parameters', parameters_caiman)
+# imaging_element.ProcessingParamSet.insert_new_params('suite2p', 0, 'Calcium imaging analysis with Suite2p using default Suite2p parameters', parameters)
+# imaging_element.ProcessingParamSet.insert_new_params('caiman', 1, 'Calcium imaging analysis with CaImAn using default CaImAn parameters', parameters_caiman)
-#scan_key = (imaging_rec.Scan & dict(recording_id=recording_id)).fetch1('KEY')
+# scan_key = (imaging_rec.Scan & dict(recording_id=recording_id)).fetch1('KEY')
-#Recording process key
-rec_process_key = dict(recording_process_id=recording_process_id)
-rec_process_str_key = 'recording_process_id_'+str(recording_process_id)
+# Recording process key
+rec_process_key = {"recording_process_id": recording_process_id}
+rec_process_str_key = "recording_process_id_" + str(recording_process_id)
-#Get fov key
+# Get fov key
rec_process = (imaging_rec.ImagingProcessing & rec_process_key).fetch1()
fov_key = rec_process.copy()
-fov_key.pop('recording_process_id')
+fov_key.pop("recording_process_id")
-#Recording process extra info
+# Recording process extra info
recording_process_info = (lab.Location * recording.RecordingProcess * recording.Recording & rec_process_key).fetch(
- 'acquisition_type', 'preprocess_paramset_idx', 'process_paramset_idx', as_dict=True)
+ "acquisition_type", "preprocess_paramset_idx", "process_paramset_idx", as_dict=True
+)
-#Paramset idx and key
-paramset_idx = recording_process_info[0]['process_paramset_idx']
-paramset_idx_key = dict()
-paramset_idx_key['paramset_idx'] = paramset_idx
-scanner = recording_process_info[0]['acquisition_type']
+# Paramset idx and key
+paramset_idx = recording_process_info[0]["process_paramset_idx"]
+paramset_idx_key = {}
+paramset_idx_key["paramset_idx"] = paramset_idx
+scanner = recording_process_info[0]["acquisition_type"]
-#Scan id always 0 because we will control that on recording_process
+# Scan id always 0 because we will control that on recording_process
scan_id = 0
-#Get preprocess params
-preprocess_params_key = dict()
-preprocess_params_key['preprocess_paramset_idx'] = recording_process_info[0]['preprocess_paramset_idx']
+# Get preprocess params
+preprocess_params_key = {}
+preprocess_params_key["preprocess_paramset_idx"] = recording_process_info[0]["preprocess_paramset_idx"]
preprocess_params = recording.PreprocessParamSet().get_preprocess_params(preprocess_params_key)
-processing_method = preprocess_params['processing_method']
-task_mode = preprocess_params['task_mode']
+processing_method = preprocess_params["processing_method"]
+task_mode = preprocess_params["task_mode"]
-print('got processing_method', processing_method)
-print('got task_mode', task_mode)
+logger.info("got processing_method %s", processing_method)
+logger.info("got task_mode %s", task_mode)
-print('got paramset_idx', paramset_idx)
+logger.info("got paramset_idx %s", paramset_idx)
-#Get directories for kov
+# Get directories for kov
scan_filepaths = get_scan_image_files(fov_key)
scan_filepaths = scan_filepaths[:1]
-print(scan_filepaths)
+logger.debug("scan_filepaths %s", scan_filepaths)
if rec_process_key not in scan_element.Scan():
- try:
- #TODO: Can use tiffile function to loads
- print('LOADED Scan using Scanreader')
+ try:
+ # TODO: Can use tiffile function to loads
+ logger.info("LOADED Scan using Scanreader")
loaded_scan = scanreader.read_scan(scan_filepaths)
header = parse_scanimage_header(loaded_scan)
- #scanner = header['SI_imagingSystem'].strip('\'') #TODO: If using tiffile, hardcode it to `mesoscope`
- except Exception as e:
- print('LOADED Scan using Tifffile')
- scan_filepaths = scan_filepaths # TODO load all TIFF files from session possibly using TIFFSequence
+ # scanner = header['SI_imagingSystem'].strip('\'') #TODO: If using tiffile, hardcode it to `mesoscope`
+ except Exception:
+ logger.info("LOADED Scan using Tifffile")
+ scan_filepaths = scan_filepaths # TODO load all TIFF files from session possibly using TIFFSequence
loaded_scan = tifffile.imread(scan_filepaths)
- #scanner = 'mesoscope'
- except: #TODO: Use except instead of else)
- print(f'ScanImage loading error') #TODO: Modify the error message
+ # scanner = 'mesoscope'
+ except BaseException: # TODO: Use except instead of else)
+ logger.error("ScanImage loading error") # TODO: Modify the error message
- #Equipment.insert1({'scanner': scanner}, skip_duplicates=True)
+ # Equipment.insert1({'scanner': scanner}, skip_duplicates=True)
scan_element.Scan.insert1(
- {**rec_process_key, 'scan_id': scan_id, 'scanner': scanner, 'acq_software': acq_software})
+ {
+ **rec_process_key,
+ "scan_id": scan_id,
+ "scanner": scanner,
+ "acq_software": acq_software,
+ }
+ )
scan_element.ScanInfo.populate(rec_process_key, display_progress=True)
# output_dir = [x.rsplit('/', maxsplit=1)[0] for x in scan_filepaths]
# output_dir = [pathlib.Path(x) for x in output_dir]
# scan_folder = [x / process_method for x in output_dir]
-fov_directory = (imaging_rec.FieldOfView & fov_key).fetch1('fov_directory')
-#output_dir = pathlib.Path('/usr/people/gs6614/temp_output') / fov_directory / processing_method #TODO fix to possibly work with existing suite2p directories
+fov_directory = (imaging_rec.FieldOfView & fov_key).fetch1("fov_directory")
+# output_dir = pathlib.Path('/usr/people/gs6614/temp_output') / fov_directory / processing_method #TODO fix to possibly work with existing suite2p directories
recording_process_id_folder_found = True
generic_process_folder_found = True
@@ -205,40 +213,41 @@
# No results to load from
-if task_mode == 'load' and not generic_process_folder_found:
- FileNotFoundError(processing_method + ' FOLDER NOT FOUND cannot load results!!!')
+if task_mode == "load" and not generic_process_folder_found:
+ FileNotFoundError(processing_method + " FOLDER NOT FOUND cannot load results!!!")
# Results from this specific recording_process_id already triggered
-if task_mode == 'trigger' and recording_process_id_folder_found:
- print('Overwritting process output from original processing folder', output_dir)
+if task_mode == "trigger" and recording_process_id_folder_found:
+ logger.warning("Overwritting process output from original processing folder %s", output_dir)
-#If trigger and not recording_process_id_folder_found make it
-if task_mode == 'trigger':
+# If trigger and not recording_process_id_folder_found make it
+if task_mode == "trigger":
relative_output_dir = (pathlib.Path(fov_directory) / rec_process_str_key).as_posix()
output_dir = pathlib.Path(get_imaging_root_data_dir(), relative_output_dir)
- output_dir.mkdir(parents=True,exist_ok=True)
+ output_dir.mkdir(parents=True, exist_ok=True)
-print('RELATIVE OUTPUT DIR')
-print(relative_output_dir)
-print(output_dir)
-#output_dir.mkdir(parents=True,exist_ok=True)
+logger.info("RELATIVE OUTPUT DIR")
+logger.info("relative_output_dir %s", relative_output_dir)
+logger.info("output_dir %s", output_dir)
+# output_dir.mkdir(parents=True,exist_ok=True)
-#Check if found output dir is correct
-if task_mode == 'load':
- if processing_method == 'suite2p':
- print('SUITE2P METHOD SELECTED')
+# Check if found output dir is correct
+if task_mode == "load":
+ if processing_method == "suite2p":
+ logger.info("SUITE2P METHOD SELECTED")
# output_dir = get_suite2p_dir(scan_key)
- p = pathlib.Path(output_dir).glob('**/*')
+ p = pathlib.Path(output_dir).glob("**/*")
plane_filepaths = [x for x in p if x.is_dir()]
for plane_filepath in plane_filepaths:
- ops_fp = plane_filepath / 'ops.npy'
- iscell_fp = plane_filepath / 'iscell.npy'
+ ops_fp = plane_filepath / "ops.npy"
+ iscell_fp = plane_filepath / "iscell.npy"
if not ops_fp.exists() or not iscell_fp.exists():
raise FileNotFoundError(
- 'No "ops.npy" or "iscell.npy" found. Invalid suite2p plane folder: {}'.format(plane_filepath))
+ f'No "ops.npy" or "iscell.npy" found. Invalid suite2p plane folder: {plane_filepath}'
+ )
- elif processing_method == 'caiman':
+ elif processing_method == "caiman":
raise ValueError("caiman not supported yet")
# _required_hdf5_fields = ['/motion_correction/reference_image',
# '/motion_correction/correlation_image',
@@ -264,31 +273,38 @@
if paramset_idx_key not in imaging_element.ProcessingParamSet():
-
- #Get all information from process params from recording.ProcessParamSet schema
- process_params_key = dict()
- process_params_key['process_paramset_idx'] = paramset_idx
+ # Get all information from process params from recording.ProcessParamSet schema
+ process_params_key = {}
+ process_params_key["process_paramset_idx"] = paramset_idx
process_params_info = (recording.ProcessParamSet() & process_params_key).fetch(as_dict=True)
process_params = recording.ProcessParamSet().get_process_params(process_params_key)
- #print('process_params_info', process_params_info)
- print('process_params', process_params)
- print('description', process_params_info[0]['process_paramset_desc'])
+ # print('process_params_info', process_params_info)
+ logger.info("process_params %s", process_params)
+ logger.info("description %s", process_params_info[0]["process_paramset_desc"])
- #Insert in imaging element equivalent ProcessParamSet
+ # Insert in imaging element equivalent ProcessParamSet
imaging_element.ProcessingParamSet.insert_new_params(
- processing_method=processing_method, paramset_idx=paramset_idx+1, paramset_desc=process_params_info[0]['process_paramset_desc'], params=process_params)
-
-imaging_element.ProcessingTask.insert1(dict(**rec_process_key,
- scan_id=scan_id,
- paramset_idx=paramset_idx+1,
- processing_output_dir=relative_output_dir,
- task_mode=task_mode),
- skip_duplicates=True)
+ processing_method=processing_method,
+ paramset_idx=paramset_idx + 1,
+ paramset_desc=process_params_info[0]["process_paramset_desc"],
+ params=process_params,
+ )
+
+imaging_element.ProcessingTask.insert1(
+ dict(
+ **rec_process_key,
+ scan_id=scan_id,
+ paramset_idx=paramset_idx + 1,
+ processing_output_dir=relative_output_dir,
+ task_mode=task_mode,
+ ),
+ skip_duplicates=True,
+)
imaging_element.Processing.populate(rec_process_key, display_progress=True)
-processing_keys = imaging_element.Processing.fetch('KEY')
+processing_keys = imaging_element.Processing.fetch("KEY")
for processing_key in processing_keys:
imaging_element.Curation().create1_from_processing_task(processing_key)
diff --git a/u19_pipeline/automatic_job/imaging_element_populate.py b/u19_pipeline/automatic_job/imaging_element_populate.py
index 31f88154..989a6602 100644
--- a/u19_pipeline/automatic_job/imaging_element_populate.py
+++ b/u19_pipeline/automatic_job/imaging_element_populate.py
@@ -1,71 +1,76 @@
-from u19_pipeline import recording, recording_process
-from u19_pipeline.imaging_pipeline import imaging_element
import pathlib
import warnings
import u19_pipeline.automatic_job.params_config as config
+from u19_pipeline import recording, recording_process
+from u19_pipeline.imaging_pipeline import imaging_element
+
def populate_element_data(job_id, display_progress=True, reserve_jobs=False, suppress_errors=False):
- populate_settings = {'display_progress': display_progress,
- 'reserve_jobs': reserve_jobs,
- 'suppress_errors': suppress_errors}
+ populate_settings = {
+ "display_progress": display_progress,
+ "reserve_jobs": reserve_jobs,
+ "suppress_errors": suppress_errors,
+ }
- process_key = (recording_process.Processing * recording.Recording &
- dict(job_id=job_id)).fetch1('KEY')
+ process_key = (recording_process.Processing * recording.Recording & {"job_id": job_id}).fetch1("KEY")
- if (recording.Recording & process_key).fetch1('recording_modality') != 'imaging':
- warnings.warn(f'Recording modality is not `imaging` for job_id: {job_id}')
+ if (recording.Recording & process_key).fetch1("recording_modality") != "imaging":
+ warnings.warn(f"Recording modality is not `imaging` for job_id: {job_id}", stacklevel=2)
return
- fragment_number, recording_process_pre_path, recording_process_post_path = \
- (recording_process.Processing & process_key).fetch1(
- 'fragment_number',
- 'recording_process_pre_path',
- 'recording_process_post_path')
+ fragment_number, recording_process_pre_path, recording_process_post_path = (
+ recording_process.Processing & process_key
+ ).fetch1("fragment_number", "recording_process_pre_path", "recording_process_post_path")
- preprocess_param_steps_id, paramset_idx = \
- (recording_process.Processing.ImagingParams & process_key
- ).fetch1('preprocess_param_steps_id',
- 'paramset_idx')
+ preprocess_param_steps_id, paramset_idx = (recording_process.Processing.ImagingParams & process_key).fetch1(
+ "preprocess_param_steps_id", "paramset_idx"
+ )
- preprocess_paramsets = (imaging_element.PreprocessParamSteps.Step() &
- dict(
- preprocess_param_steps_id=preprocess_param_steps_id)
- ).fetch('paramset_idx')
+ preprocess_paramsets = (
+ imaging_element.PreprocessParamSteps.Step() & {"preprocess_param_steps_id": preprocess_param_steps_id}
+ ).fetch("paramset_idx")
- processing_method = (imaging_element.ProcessingParamSet &
- dict(paramset_idx=paramset_idx)).fetch1(
- 'processing_method')
+ processing_method = (imaging_element.ProcessingParamSet & {"paramset_idx": paramset_idx}).fetch1(
+ "processing_method"
+ )
- if len(preprocess_paramsets)==0:
- preprocess_task_mode = 'none'
- else:
- preprocess_task_mode = 'load'
+ preprocess_task_mode = "none" if len(preprocess_paramsets) == 0 else "load"
- preprocess_key = dict(recording_id=process_key['recording_id'],
- tiff_split=fragment_number,
- scan_id=0,
- preprocess_param_steps_id=preprocess_param_steps_id)
+ preprocess_key = {
+ "recording_id": process_key["recording_id"],
+ "tiff_split": fragment_number,
+ "scan_id": 0,
+ "preprocess_param_steps_id": preprocess_param_steps_id,
+ }
imaging_element.PreprocessTask.insert1(
- dict(**preprocess_key,
- preprocess_output_dir=recording_process_pre_path,
- task_mode=preprocess_task_mode),
- skip_duplicates=True)
+ dict(
+ **preprocess_key,
+ preprocess_output_dir=recording_process_pre_path,
+ task_mode=preprocess_task_mode,
+ ),
+ skip_duplicates=True,
+ )
if not imaging_element.Preprocess & preprocess_key:
imaging_element.Preprocess.populate(preprocess_key, **populate_settings)
- process_key = dict(**preprocess_key,
- paramset_idx=paramset_idx)
+ process_key = dict(**preprocess_key, paramset_idx=paramset_idx)
- pathlib.Path(f'/mnt/cup/braininit/Data/Processed/imaging/{recording_process_post_path}/{processing_method}_output').mkdir(parents=True, exist_ok=True)
+ pathlib.Path(
+ f"/mnt/cup/braininit/Data/Processed/imaging/{recording_process_post_path}/{processing_method}_output"
+ ).mkdir(parents=True, exist_ok=True)
imaging_element.ProcessingTask.insert1(
- dict(**process_key,
- processing_output_dir=f'{recording_process_post_path}/{processing_method}_output',
- task_mode='trigger'), skip_duplicates=True)
+ dict(
+ **process_key,
+ processing_output_dir=f"{recording_process_post_path}/{processing_method}_output",
+ task_mode="trigger",
+ ),
+ skip_duplicates=True,
+ )
if not imaging_element.Processing & process_key:
imaging_element.Processing.populate(process_key, **populate_settings)
@@ -85,8 +90,8 @@ def populate_element_data(job_id, display_progress=True, reserve_jobs=False, sup
if not imaging_element.Activity & process_key:
imaging_element.Activity.populate(process_key, **populate_settings)
- return config.status_update_idx['NEXT_STATUS']
+ return config.status_update_idx["NEXT_STATUS"]
-if __name__ == '__main__':
- populate_element_data()
\ No newline at end of file
+if __name__ == "__main__":
+ populate_element_data()
diff --git a/u19_pipeline/automatic_job/parameter_file_creator.py b/u19_pipeline/automatic_job/parameter_file_creator.py
index e05e2ecc..4e7e5df8 100644
--- a/u19_pipeline/automatic_job/parameter_file_creator.py
+++ b/u19_pipeline/automatic_job/parameter_file_creator.py
@@ -1,78 +1,89 @@
+# import os
+# import pathlib
+import json
+import pathlib
+from scipy.io import savemat
-#import os
-#import pathlib
-import subprocess
-import pathlib
-import json
-import re
import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft
-import u19_pipeline.automatic_job.params_config as config
+import u19_pipeline.automatic_job.params_config as config
from u19_pipeline.utils.file_utils import write_file
-from scipy.io import savemat
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
# Functions to create parameter files and send them
-#parameter_files_filepath = 'u19_pipeline/automatic_job/ParameterFiles'
-#default_preprocess_filename = 'preprocess_paramset_%s.json'
-#default_process_filename = 'process_paramset_%s.json'
+# parameter_files_filepath = 'u19_pipeline/automatic_job/ParameterFiles'
+# default_preprocess_filename = 'preprocess_paramset_%s.json'
+# default_process_filename = 'process_paramset_%s.json'
-#chanmap_files_filepath = 'u19_pipeline/automatic_job/ChanMapFiles'
-#default_chanmap_filename = 'chanmap_%s.mat'
+# chanmap_files_filepath = 'u19_pipeline/automatic_job/ChanMapFiles'
+# default_chanmap_filename = 'chanmap_%s.mat'
default_process_script_path = "scripts/automate_imaging_element.py"
def generate_parameter_file(recording_process_id, params, type_params, program_selection_params):
- '''
+ """
Generate and send parameter files for processing
- '''
- if program_selection_params['local_or_cluster'] == 'cluster':
- cluster_vars = ft.get_cluster_vars(program_selection_params['process_cluster'])
- params_file_cluster_path = cluster_vars['params_files_dir']
- user_host = cluster_vars['user']+'@'+cluster_vars['hostname']
-
- #Write preprocessing parameter file
- if type_params == 'preparams':
+ """
+ if program_selection_params["local_or_cluster"] == "cluster":
+ cluster_vars = ft.get_cluster_vars(program_selection_params["process_cluster"])
+ params_file_cluster_path = cluster_vars["params_files_dir"]
+ user_host = cluster_vars["user"] + "@" + cluster_vars["hostname"]
+
+ # Write preprocessing parameter file
+ if type_params == "preparams":
write_parameter_file(params, recording_process_id, config.default_preprocess_filename)
- if program_selection_params['local_or_cluster'] == 'cluster':
- status = transfer_parameter_file(recording_process_id, config.default_preprocess_filename, params_file_cluster_path, user_host)
+ if program_selection_params["local_or_cluster"] == "cluster":
+ status = transfer_parameter_file(
+ recording_process_id,
+ config.default_preprocess_filename,
+ params_file_cluster_path,
+ user_host,
+ )
else:
- status = config.system_process['SUCCESS']
+ status = config.system_process["SUCCESS"]
else:
write_parameter_file(params, recording_process_id, config.default_process_filename)
- if program_selection_params['local_or_cluster'] == 'cluster':
- status = transfer_parameter_file(recording_process_id, config.default_process_filename, params_file_cluster_path, user_host)
+ if program_selection_params["local_or_cluster"] == "cluster":
+ status = transfer_parameter_file(
+ recording_process_id,
+ config.default_process_filename,
+ params_file_cluster_path,
+ user_host,
+ )
else:
- status = config.system_process['SUCCESS']
+ status = config.system_process["SUCCESS"]
return status
def write_parameter_file(params, recording_process_id, default_param_filename):
- '''
+ """
Write local parameter file to send
- '''
+ """
str_params = json.dumps(params)
param_filename = default_param_filename % (recording_process_id)
- params_file_local_path = str(pathlib.Path(config.parameter_files_filepath,param_filename))
+ params_file_local_path = str(pathlib.Path(config.parameter_files_filepath, param_filename))
write_file(params_file_local_path, str_params)
def transfer_parameter_file(recording_process_id, default_param_filename, cluster_param_dir, user_host):
- '''
+ """
Transfer parameter file to processing cluster
- '''
+ """
param_filename = default_param_filename % (recording_process_id)
- params_file_local_path = str(pathlib.Path(config.parameter_files_filepath,param_filename))
- params_file_cluster_path = str(pathlib.Path(cluster_param_dir,param_filename))
- param_file_full_path = user_host+':'+params_file_cluster_path
+ params_file_local_path = str(pathlib.Path(config.parameter_files_filepath, param_filename))
+ params_file_cluster_path = str(pathlib.Path(cluster_param_dir, param_filename))
+ param_file_full_path = user_host + ":" + params_file_cluster_path
- print('transfer_parameter_file', params_file_local_path, param_file_full_path)
+ logger.debug("transfer_parameter_file %s %s", params_file_local_path, param_file_full_path)
status = ft.scp_file_transfer(params_file_local_path, param_file_full_path)
@@ -80,41 +91,46 @@ def transfer_parameter_file(recording_process_id, default_param_filename, cluste
def generate_chanmap_file(recording_process_id, program_selection_params):
- '''
+ """
Generate and send chanmap files for processing
- '''
-
- cluster_vars = ft.get_cluster_vars(program_selection_params['process_cluster'])
- params_file_cluster_path = cluster_vars['chanmap_files_dir']
- user_host = cluster_vars['user']+'@'+cluster_vars['hostname']
+ """
+
+ cluster_vars = ft.get_cluster_vars(program_selection_params["process_cluster"])
+ params_file_cluster_path = cluster_vars["chanmap_files_dir"]
+ user_host = cluster_vars["user"] + "@" + cluster_vars["hostname"]
- #Write chanmap file
+ # Write chanmap file
# write_chanmap_file(chanmap_df, recording_process_id, config.default_chanmap_filename)
- status = transfer_chanmap_file(recording_process_id, config.default_chanmap_filename, params_file_cluster_path, user_host)
+ status = transfer_chanmap_file(
+ recording_process_id,
+ config.default_chanmap_filename,
+ params_file_cluster_path,
+ user_host,
+ )
return status
+
def write_chanmap_file(chanmap_dict, recording_process_id, default_chanmap_filename):
- '''
+ """
Write local chanmap file to send
- '''
+ """
chanmap_filename = default_chanmap_filename % (recording_process_id)
- chanmap_file_local_path = str(pathlib.Path(config.chanmap_files_filepath,chanmap_filename))
+ chanmap_file_local_path = str(pathlib.Path(config.chanmap_files_filepath, chanmap_filename))
savemat(chanmap_file_local_path, chanmap_dict)
def transfer_chanmap_file(recording_process_id, default_chanmap_filename, cluster_chanmap_dir, user_host):
- '''
+ """
Transfer chanmap file to processing cluster
- '''
+ """
chanmap_filename = default_chanmap_filename % (recording_process_id)
- chanmap_file_local_path = str(pathlib.Path(config.chanmap_files_filepath,chanmap_filename))
- chanmap_file_cluster_path = str(pathlib.Path(cluster_chanmap_dir,chanmap_filename))
- chanmap_file_full_path = user_host+':'+chanmap_file_cluster_path
+ chanmap_file_local_path = str(pathlib.Path(config.chanmap_files_filepath, chanmap_filename))
+ chanmap_file_cluster_path = str(pathlib.Path(cluster_chanmap_dir, chanmap_filename))
+ chanmap_file_full_path = user_host + ":" + chanmap_file_cluster_path
status = ft.scp_file_transfer(chanmap_file_local_path, chanmap_file_full_path)
return status
-
diff --git a/u19_pipeline/automatic_job/params_config.py b/u19_pipeline/automatic_job/params_config.py
index 1140e898..0a5260cd 100644
--- a/u19_pipeline/automatic_job/params_config.py
+++ b/u19_pipeline/automatic_job/params_config.py
@@ -1,289 +1,270 @@
-
-import pandas as pd
-import numpy as np
import os
import pathlib
-from scripts.conf_file_finding import get_root_directory
+import pandas as pd
+
import u19_pipeline.lab as lab
+from scripts.conf_file_finding import get_root_directory
-#Dictionary with main configuration for each modality (ephys or imaging)
+# Dictionary with main configuration for each modality (ephys or imaging)
recording_modality_dict = [
{
- 'recording_modality': 'electrophysiology',
- 'local_or_cluster': 'cluster', # Where processing will happen, locally or cluster
- 'process_repository': 'BrainCogsEphysSorters', # Which repositroy will be used to process
- 'process_cluster': 'spock', # Which cluster will be used to process (check clusters_paths_and_transfers.py file)
- 'process_script': 'main_script.py' # Script in process_repository to
+ "recording_modality": "electrophysiology",
+ "local_or_cluster": "cluster", # Where processing will happen, locally or cluster
+ "process_repository": "BrainCogsEphysSorters", # Which repositroy will be used to process
+ "process_cluster": "spock", # Which cluster will be used to process (check clusters_paths_and_transfers.py file)
+ "process_script": "main_script.py", # Script in process_repository to
},
{
- 'recording_modality': 'imaging',
- 'local_or_cluster': 'local',
- 'process_repository': 'element-calcium-imaging',
- 'process_cluster': 'spock', # Which cluster will be used to process (check clusters_paths_and_transfers.py file)
- 'process_script': 'none'
+ "recording_modality": "imaging",
+ "local_or_cluster": "local",
+ "process_repository": "element-calcium-imaging",
+ "process_cluster": "spock", # Which cluster will be used to process (check clusters_paths_and_transfers.py file)
+ "process_script": "none",
},
]
recording_modality_list = [list(i.values()) for i in recording_modality_dict]
recording_modality_df = pd.DataFrame(recording_modality_dict)
-#Dictionary with status configuration for recordings
+# Dictionary with status configuration for recordings
recording_status_dict = [
{
- 'Value': -1, # Status id
- 'Key': 'ERROR', # Status "nickname"
- 'Label': 'Error in recording handling', # Status description
- 'UpdateField': None, # Which field in the u19_recording.recording table will be updated
- 'ProcessFunction': None, # Which function to execute in workflow for this status
- 'FunctionField': None, # Which field of u19_recording.recording will be used in status function
- 'SlackMessage': None # Slack notification message for this status
+ "Value": -1, # Status id
+ "Key": "ERROR", # Status "nickname"
+ "Label": "Error in recording handling", # Status description
+ "UpdateField": None, # Which field in the u19_recording.recording table will be updated
+ "ProcessFunction": None, # Which function to execute in workflow for this status
+ "FunctionField": None, # Which field of u19_recording.recording will be used in status function
+ "SlackMessage": None, # Slack notification message for this status
},
{
- 'Value': 0,
- 'Key': 'NEW_RECORDING',
- 'Label': 'New recording',
- 'UpdateField': None,
- 'ProcessFunction': None,
- 'FunctionField': None,
- 'SlackMessage': None
+ "Value": 0,
+ "Key": "NEW_RECORDING",
+ "Label": "New recording",
+ "UpdateField": None,
+ "ProcessFunction": None,
+ "FunctionField": None,
+ "SlackMessage": None,
},
{
- 'Value': 1,
- 'Key': 'PNI_DRIVE_TRANSFER_REQUEST',
- 'Label': 'Recording directory transfer to PNI requested',
- 'UpdateField': 'task_copy_id_pni',
- 'ProcessFunction': 'local_transfer_request',
- 'FunctionField': 'recording_process_pre_path',
- 'SlackMessage': None
+ "Value": 1,
+ "Key": "PNI_DRIVE_TRANSFER_REQUEST",
+ "Label": "Recording directory transfer to PNI requested",
+ "UpdateField": "task_copy_id_pni",
+ "ProcessFunction": "local_transfer_request",
+ "FunctionField": "recording_process_pre_path",
+ "SlackMessage": None,
},
{
- 'Value': 2,
- 'Key': 'PNI_DRIVE_TRANSFER_END',
- 'Label': 'Recording directory transferred to PNI',
- 'UpdateField': None,
- 'ProcessFunction': 'local_transfer_check',
- 'FunctionField': 'task_copy_id_pni',
- 'SlackMessage': 'Recording was transferred to braininit (cup) drive'
+ "Value": 2,
+ "Key": "PNI_DRIVE_TRANSFER_END",
+ "Label": "Recording directory transferred to PNI",
+ "UpdateField": None,
+ "ProcessFunction": "local_transfer_check",
+ "FunctionField": "task_copy_id_pni",
+ "SlackMessage": "Recording was transferred to braininit (cup) drive",
},
{
- 'Value': 3,
- 'Key': 'MODALITY_PREINGESTION',
- 'Label': 'modality ingestion & Syncing jobs done',
- 'UpdateField': None,
- 'ProcessFunction': 'modality_preingestion',
- 'FunctionField': None,
- 'SlackMessage': None
+ "Value": 3,
+ "Key": "MODALITY_PREINGESTION",
+ "Label": "modality ingestion & Syncing jobs done",
+ "UpdateField": None,
+ "ProcessFunction": "modality_preingestion",
+ "FunctionField": None,
+ "SlackMessage": None,
},
-
]
-recording_status_list = [[i['Value'], i['Label']] for i in recording_status_dict]
+recording_status_list = [[i["Value"], i["Label"]] for i in recording_status_dict]
recording_status_df = pd.DataFrame(recording_status_dict)
-RECORDING_STATUS_ERROR_ID = recording_status_df.loc[recording_status_df['Key'] == 'ERROR', 'Value'].values[0]
+RECORDING_STATUS_ERROR_ID = recording_status_df.loc[recording_status_df["Key"] == "ERROR", "Value"].values[0]
-#Dictionary with status configuration for recording process (units of each recording)
+# Dictionary with status configuration for recording process (units of each recording)
recording_process_status_dict = [
{
- 'Value': -2,
- 'Key': 'ERROR_DELETED',
- 'Label': 'Error in recording process / Deleted cluster files',
- 'UpdateField': None,
- 'ProcessFunction': None,
- 'FunctionField': None,
- 'SlackMessage': None
+ "Value": -2,
+ "Key": "ERROR_DELETED",
+ "Label": "Error in recording process / Deleted cluster files",
+ "UpdateField": None,
+ "ProcessFunction": None,
+ "FunctionField": None,
+ "SlackMessage": None,
},
{
- 'Value': -1,
- 'Key': 'ERROR',
- 'Label': 'Error in recording process',
- 'UpdateField': None,
- 'ProcessFunction': None,
- 'FunctionField': None,
- 'SlackMessage': None
+ "Value": -1,
+ "Key": "ERROR",
+ "Label": "Error in recording process",
+ "UpdateField": None,
+ "ProcessFunction": None,
+ "FunctionField": None,
+ "SlackMessage": None,
},
{
- 'Value': 0,
- 'Key': 'NEW_RECORDING_PROCESS',
- 'Label': 'New recording process',
- 'UpdateField': None,
- 'ProcessFunction': None,
- 'FunctionField': None,
- 'SlackMessage': None
+ "Value": 0,
+ "Key": "NEW_RECORDING_PROCESS",
+ "Label": "New recording process",
+ "UpdateField": None,
+ "ProcessFunction": None,
+ "FunctionField": None,
+ "SlackMessage": None,
},
{
- 'Value': 1,
- 'Key': 'RAW_FILE_TRANSFER_REQUEST',
- 'Label': 'Raw file transfer requested',
- 'UpdateField': 'task_copy_id_pre',
- 'ProcessFunction': 'transfer_request',
- 'FunctionField': 'recording_process_pre_path',
- 'SlackMessage': None
+ "Value": 1,
+ "Key": "RAW_FILE_TRANSFER_REQUEST",
+ "Label": "Raw file transfer requested",
+ "UpdateField": "task_copy_id_pre",
+ "ProcessFunction": "transfer_request",
+ "FunctionField": "recording_process_pre_path",
+ "SlackMessage": None,
},
{
- 'Value': 2,
- 'Key': 'RAW_FILE_TRANSFER_END',
- 'Label': 'Raw file transferred to cluster',
- 'UpdateField': None,
- 'ProcessFunction': 'transfer_check',
- 'FunctionField': 'task_copy_id_pre',
- 'SlackMessage': None
+ "Value": 2,
+ "Key": "RAW_FILE_TRANSFER_END",
+ "Label": "Raw file transferred to cluster",
+ "UpdateField": None,
+ "ProcessFunction": "transfer_check",
+ "FunctionField": "task_copy_id_pre",
+ "SlackMessage": None,
},
{
- 'Value': 3,
- 'Key': 'JOB_QUEUE',
- 'Label': 'Processing job in queue',
- 'UpdateField': 'slurm_id',
- 'ProcessFunction': 'slurm_job_queue',
- 'FunctionField': None,
- 'SlackMessage': 'Job was queued to be processed in cluster'
+ "Value": 3,
+ "Key": "JOB_QUEUE",
+ "Label": "Processing job in queue",
+ "UpdateField": "slurm_id",
+ "ProcessFunction": "slurm_job_queue",
+ "FunctionField": None,
+ "SlackMessage": "Job was queued to be processed in cluster",
},
{
- 'Value': 4,
- 'Key': 'JOB_FINISHED',
- 'Label': 'Processing job finished',
- 'UpdateField': None,
- 'ProcessFunction': 'slurm_job_check',
- 'FunctionField': 'slurm_id',
- 'SlackMessage': None
+ "Value": 4,
+ "Key": "JOB_FINISHED",
+ "Label": "Processing job finished",
+ "UpdateField": None,
+ "ProcessFunction": "slurm_job_check",
+ "FunctionField": "slurm_id",
+ "SlackMessage": None,
},
{
- 'Value': 5,
- 'Key': 'PROC_FILE_TRANSFER_REQUEST',
- 'Label': 'Processed file transfer requested',
- 'UpdateField': 'task_copy_id_post',
- 'ProcessFunction': 'transfer_request',
- 'FunctionField': 'recording_process_post_path',
- 'SlackMessage': None
+ "Value": 5,
+ "Key": "PROC_FILE_TRANSFER_REQUEST",
+ "Label": "Processed file transfer requested",
+ "UpdateField": "task_copy_id_post",
+ "ProcessFunction": "transfer_request",
+ "FunctionField": "recording_process_post_path",
+ "SlackMessage": None,
},
{
- 'Value': 6,
- 'Key': 'PROC_FILE_TRANSFER_END',
- 'Label': 'Processed file transferred to PNI',
- 'UpdateField': None,
- 'ProcessFunction': 'transfer_check',
- 'FunctionField': 'task_copy_id_post',
- 'SlackMessage': 'Processed data was transferred to cup. Available on the GUI'
+ "Value": 6,
+ "Key": "PROC_FILE_TRANSFER_END",
+ "Label": "Processed file transferred to PNI",
+ "UpdateField": None,
+ "ProcessFunction": "transfer_check",
+ "FunctionField": "task_copy_id_post",
+ "SlackMessage": "Processed data was transferred to cup. Available on the GUI",
},
{
- 'Value': 7,
- 'Key': 'JOB_FINISHED_ELEMENT_WORKFLOW',
- 'Label': 'Data in element, Finished !!',
- 'UpdateField': None,
- 'ProcessFunction': 'populate_element',
- 'FunctionField': None,
- 'SlackMessage': 'Job was successfully processed. Data in element DB'
+ "Value": 7,
+ "Key": "JOB_FINISHED_ELEMENT_WORKFLOW",
+ "Label": "Data in element, Finished !!",
+ "UpdateField": None,
+ "ProcessFunction": "populate_element",
+ "FunctionField": None,
+ "SlackMessage": "Job was successfully processed. Data in element DB",
},
{
- 'Value': 8,
- 'Key': 'JOB_DATA_DELETED_CLUSTER',
- 'Label': 'Data in element, Finished !!',
- 'UpdateField': None,
- 'ProcessFunction': 'populate_element',
- 'FunctionField': None,
- 'SlackMessage': None
+ "Value": 8,
+ "Key": "JOB_DATA_DELETED_CLUSTER",
+ "Label": "Data in element, Finished !!",
+ "UpdateField": None,
+ "ProcessFunction": "populate_element",
+ "FunctionField": None,
+ "SlackMessage": None,
},
]
-recording_process_status_list = [[i['Value'], i['Label']] for i in recording_process_status_dict]
+recording_process_status_list = [[i["Value"], i["Label"]] for i in recording_process_status_dict]
recording_process_status_df = pd.DataFrame(recording_process_status_dict)
# All error and success status values
-JOB_STATUS_ERROR_ID = recording_process_status_df.loc[recording_process_status_df['Key'] == 'ERROR', 'Value'].values[0]
-JOB_STATUS_PROCESSED = recording_process_status_df.loc[recording_process_status_df['Key'] == 'JOB_FINISHED_ELEMENT_WORKFLOW', 'Value'].values[0]
-JOB_STATUS_POST_PROCESSED = recording_process_status_df.loc[recording_process_status_df['Key'] == 'JOB_DATA_DELETED_CLUSTER', 'Value'].values[0]
-JOB_STATUS_ERROR_DELETED = recording_process_status_df.loc[recording_process_status_df['Key'] == 'ERROR_DELETED', 'Value'].values[0]
+JOB_STATUS_ERROR_ID = recording_process_status_df.loc[recording_process_status_df["Key"] == "ERROR", "Value"].values[0]
+JOB_STATUS_PROCESSED = recording_process_status_df.loc[
+ recording_process_status_df["Key"] == "JOB_FINISHED_ELEMENT_WORKFLOW", "Value"
+].values[0]
+JOB_STATUS_POST_PROCESSED = recording_process_status_df.loc[
+ recording_process_status_df["Key"] == "JOB_DATA_DELETED_CLUSTER", "Value"
+].values[0]
+JOB_STATUS_ERROR_DELETED = recording_process_status_df.loc[
+ recording_process_status_df["Key"] == "ERROR_DELETED", "Value"
+].values[0]
# Return code for functions for errors, next state or no status change
-status_update_idx = {
- 'NEXT_STATUS': 1,
- 'NO_CHANGE': 0,
- 'ERROR_STATUS':-1
-}
+status_update_idx = {"NEXT_STATUS": 1, "NO_CHANGE": 0, "ERROR_STATUS": -1}
# Degault Dictionary "skeleton" that is returned in all functions of workflow
-default_update_value_dict ={
- 'value_update': None,
- 'error_info': {
- 'error_message': None,
- 'error_exception': None,
+default_update_value_dict = {
+ "value_update": None,
+ "error_info": {
+ "error_message": None,
+ "error_exception": None,
},
}
-# Return Codes from
-system_process = {
- 'COMPLETED': 1,
- 'SUCCESS': 0,
- 'ERROR': -1
-}
+# Return Codes from
+system_process = {"COMPLETED": 1, "SUCCESS": 0, "ERROR": -1}
# Possible states from slurm queued jobs results and result in pipeline status (next state, no change or error)
slurm_states = {
- 'COMPLETED': {
- 'pipeline_status': status_update_idx['NEXT_STATUS'],
- 'message': ''
- },
- 'PENDING': {
- 'pipeline_status': status_update_idx['NO_CHANGE'],
- 'message': ''
+ "COMPLETED": {"pipeline_status": status_update_idx["NEXT_STATUS"], "message": ""},
+ "PENDING": {"pipeline_status": status_update_idx["NO_CHANGE"], "message": ""},
+ "RUNNING": {"pipeline_status": status_update_idx["NO_CHANGE"], "message": ""},
+ "FAILED": {"pipeline_status": status_update_idx["ERROR_STATUS"], "message": ""},
+ "TIMEOUT": {
+ "pipeline_status": status_update_idx["ERROR_STATUS"],
+ "message": "Timeout for job has expired",
},
- 'RUNNING': {
- 'pipeline_status': status_update_idx['NO_CHANGE'],
- 'message': ''
+ "CANCELLED+": {
+ "pipeline_status": status_update_idx["ERROR_STATUS"],
+ "message": "Job was cancelled",
},
- 'FAILED': {
- 'pipeline_status': status_update_idx['ERROR_STATUS'],
- 'message': ''
- },
- 'TIMEOUT':
- {
- 'pipeline_status': status_update_idx['ERROR_STATUS'],
- 'message': 'Timeout for job has expired'
+ "CANCELLED": {
+ "pipeline_status": status_update_idx["ERROR_STATUS"],
+ "message": "Job was cancelled",
},
- 'CANCELLED+':
- {
- 'pipeline_status': status_update_idx['ERROR_STATUS'],
- 'message': 'Job was cancelled'
- },
- 'CANCELLED':
- {
- 'pipeline_status': status_update_idx['ERROR_STATUS'],
- 'message': 'Job was cancelled'
- }
}
# Look for u19_matlab_dir (Should be present on same directory as U19-Pipeline_Python)
_, u19_pipeline_python_dir = get_root_directory()
datajoint_proj_dir = u19_pipeline_python_dir.parent
-u19_matlab_dir = pathlib.Path(datajoint_proj_dir, 'U19-pipeline-matlab')
-startup_pipeline_matlab_dir = pathlib.Path(u19_matlab_dir, 'scripts').as_posix()
+u19_matlab_dir = pathlib.Path(datajoint_proj_dir, "U19-pipeline-matlab")
+startup_pipeline_matlab_dir = pathlib.Path(u19_matlab_dir, "scripts").as_posix()
this_dir = os.path.dirname(__file__)
-ingest_scaninfo_script = pathlib.Path(this_dir, 'ingest_scaninfo_shell.sh').as_posix()
+ingest_scaninfo_script = pathlib.Path(this_dir, "ingest_scaninfo_shell.sh").as_posix()
# Look for CATGT directory (Should be present on same directory as U19-Pipeline_Python)
-catgt_dir = pathlib.Path(datajoint_proj_dir, 'CatGT-linux')
-catgt_script = pathlib.Path(catgt_dir, 'runit.sh').as_posix()
+catgt_dir = pathlib.Path(datajoint_proj_dir, "CatGT-linux")
+catgt_script = pathlib.Path(catgt_dir, "runit.sh").as_posix()
# For parameter & channmap storing
-parameter_files_filepath = pathlib.Path(this_dir, 'ParameterFiles').as_posix()
-default_preprocess_filename = 'preprocess_paramset_%s.json'
-default_process_filename = 'process_paramset_%s.json'
+parameter_files_filepath = pathlib.Path(this_dir, "ParameterFiles").as_posix()
+default_preprocess_filename = "preprocess_paramset_%s.json"
+default_process_filename = "process_paramset_%s.json"
# default xyzPicksFiles
-xyz_picks_files_dir = pathlib.Path(this_dir, 'xyzPicksFiles')
-xyz_picks_0 = pathlib.Path(xyz_picks_files_dir, 'xyz_picks.json').as_posix()
-xyz_picks_n = pathlib.Path(xyz_picks_files_dir, 'xyz_picks_shank%d.json').as_posix()
+xyz_picks_files_dir = pathlib.Path(this_dir, "xyzPicksFiles")
+xyz_picks_0 = pathlib.Path(xyz_picks_files_dir, "xyz_picks.json").as_posix()
+xyz_picks_n = pathlib.Path(xyz_picks_files_dir, "xyz_picks_shank%d.json").as_posix()
-#chanmap files
-chanmap_files_filepath = pathlib.Path(this_dir, 'ChanMapFiles').as_posix()
-default_chanmap_filename = 'chanmap_%s.mat'
+# chanmap files
+chanmap_files_filepath = pathlib.Path(this_dir, "ChanMapFiles").as_posix()
+default_chanmap_filename = "chanmap_%s.mat"
-#Slack notification channels
+# Slack notification channels
slack_webhooks = lab.SlackWebhooks.fetch()
-slack_webhooks_dict = dict()
+slack_webhooks_dict = {}
for i in range(slack_webhooks.shape[0]):
- slack_webhooks_dict[slack_webhooks[i][0]] = slack_webhooks[i][1]
\ No newline at end of file
+ slack_webhooks_dict[slack_webhooks[i][0]] = slack_webhooks[i][1]
diff --git a/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py b/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py
index 17f62def..d7dbc112 100644
--- a/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py
+++ b/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py
@@ -1,39 +1,54 @@
-
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(1)
-import numpy as np
-import u19_pipeline.ephys_pipeline as ep
-import u19_pipeline.recording as recording
-import datajoint as dj
-import pandas as pd
import datetime
+import pandas as pd
-def get_rec_key_dict(recording_id):
+import u19_pipeline.ephys_pipeline as ep
+import u19_pipeline.recording as recording
+from u19_pipeline.utils.logging_config import get_logger
- return {'recording_id': recording_id}
+logger = get_logger(__name__)
-past_date = datetime.date.today() - datetime.timedelta(days=30)
-query_session_date = 'session_date >="' + past_date.strftime('%Y-%m-%d') +'"'
+def get_rec_key_dict(recording_id):
-all_recs = ((recording.Recording & "recording_modality='electrophysiology'") * (recording.Recording.BehaviorSession & query_session_date)).join(ep.BehaviorSync, left=True)
-all_recs = pd.DataFrame(all_recs.fetch('recording_id','subject_fullname','session_date','session_number', as_dict=True))
+ return {"recording_id": recording_id}
-not_sync_recs = pd.DataFrame((recording.Recording * ep.BehaviorSync).fetch('KEY', as_dict=True))
-not_sync_recs2 = pd.merge(all_recs,not_sync_recs, how='left', indicator=True)
-not_sync_recs2 = not_sync_recs2.loc[not_sync_recs2['_merge']=='left_only',:]
-not_sync_recs2 = not_sync_recs2.sort_values(by='session_date', ascending=False)
-not_sync_recs2['rec_key'] = not_sync_recs2['recording_id'].apply(get_rec_key_dict)
+past_date = datetime.date.today() - datetime.timedelta(days=30)
+query_session_date = 'session_date >="' + past_date.strftime("%Y-%m-%d") + '"'
+
+all_recs = (
+ (recording.Recording & "recording_modality='electrophysiology'")
+ * (recording.Recording.BehaviorSession & query_session_date)
+).join(ep.BehaviorSync, left=True)
+all_recs = pd.DataFrame(
+ all_recs.fetch(
+ "recording_id",
+ "subject_fullname",
+ "session_date",
+ "session_number",
+ as_dict=True,
+ )
+)
+
+not_sync_recs = pd.DataFrame((recording.Recording * ep.BehaviorSync).fetch("KEY", as_dict=True))
+
+not_sync_recs2 = pd.merge(all_recs, not_sync_recs, how="left", indicator=True)
+not_sync_recs2 = not_sync_recs2.loc[not_sync_recs2["_merge"] == "left_only", :]
+not_sync_recs2 = not_sync_recs2.sort_values(by="session_date", ascending=False)
+not_sync_recs2["rec_key"] = not_sync_recs2["recording_id"].apply(get_rec_key_dict)
not_sync_recs2 = not_sync_recs2.reset_index(drop=True)
for i in range(not_sync_recs2.shape[0]):
try:
- ep.BehaviorSync.populate(not_sync_recs2.loc[i,'rec_key'])
+ ep.BehaviorSync.populate(not_sync_recs2.loc[i, "rec_key"])
except Exception as e:
- print(e)
\ No newline at end of file
+ logger.exception("Error populating BehaviorSync: %s", e)
diff --git a/u19_pipeline/automatic_job/pupillometry_check_handler_script.py b/u19_pipeline/automatic_job/pupillometry_check_handler_script.py
index 31e2bd61..0597bc1d 100644
--- a/u19_pipeline/automatic_job/pupillometry_check_handler_script.py
+++ b/u19_pipeline/automatic_job/pupillometry_check_handler_script.py
@@ -1,11 +1,10 @@
-
import time
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(1)
-import datajoint as dj
import u19_pipeline.automatic_job.pupillometry_handler as ph
ph.PupillometryProcessingHandler.check_processed_pupillometry_sessions()
diff --git a/u19_pipeline/automatic_job/pupillometry_handler.py b/u19_pipeline/automatic_job/pupillometry_handler.py
index 07f1bfc2..c1cb7f9a 100644
--- a/u19_pipeline/automatic_job/pupillometry_handler.py
+++ b/u19_pipeline/automatic_job/pupillometry_handler.py
@@ -1,76 +1,79 @@
-import datajoint as dj
-import pathlib
-import pandas as pd
-import numpy as np
+import copy
import glob
-import subprocess
-import re
import os
+import pathlib
+import re
+import subprocess
import sys
-import copy
import traceback
-from skimage.measure import EllipseModel
-from skimage.draw import ellipse_perimeter
-from scipy import stats
+import datajoint as dj
+import numpy as np
+import pandas as pd
+from scipy import stats
+from skimage.measure import EllipseModel
-import u19_pipeline.utils.slack_utils as slack_utils
-import u19_pipeline.automatic_job.slurm_creator as slurmlib
import u19_pipeline.acquisition as acquisition
-import u19_pipeline.pupillometry as pupillometry
import u19_pipeline.automatic_job.params_config as config
-from u19_pipeline.automatic_job import recording_handler
+import u19_pipeline.automatic_job.slurm_creator as slurmlib
+import u19_pipeline.pupillometry as pupillometry
+import u19_pipeline.utils.slack_utils as slack_utils
from u19_pipeline.utils.file_utils import write_file
-import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
def pupillometry_exception_handler(func):
def inner_function(*args, **kwargs):
try:
- argout = func(*args, **kwargs)
- return argout
+ argout = func(*args, **kwargs)
+ return argout
except Exception as e:
- print('Exception HERE ................')
+ logger.exception("Exception HERE ................")
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- update_value_dict['error_info']['error_message'] = str(e)
- update_value_dict['error_info']['error_exception'] = (''.join(traceback.format_exception(type(e), value=e, tb=e.__traceback__)))
-
- print(update_value_dict['error_info']['error_message'])
- print(update_value_dict['error_info']['error_exception'])
+ update_value_dict["error_info"]["error_message"] = str(e)
+ update_value_dict["error_info"]["error_exception"] = "".join(
+ traceback.format_exception(type(e), value=e, tb=e.__traceback__)
+ )
+ logger.error("error_message %s", update_value_dict["error_info"]["error_message"])
+ logger.error("error_exception %s", update_value_dict["error_info"]["error_exception"])
return (config.RECORDING_STATUS_ERROR_ID, update_value_dict)
+
return inner_function
-class PupillometryProcessingHandler():
- spock_home_dir = '/mnt/cup/braininit/Shared/repos/U19-pipeline_python/'
+class PupillometryProcessingHandler:
+ spock_home_dir = "/mnt/cup/braininit/Shared/repos/U19-pipeline_python/"
spock_log_dir = spock_home_dir + "u19_pipeline/automatic_job/OutputLog/"
process_script_path = spock_home_dir + "u19_pipeline/automatic_job/pupillometry_handler.py"
spock_error_dir = spock_home_dir + "u19_pipeline/automatic_job/ErrorLog/"
spock_slurm_filepath = spock_home_dir + "u19_pipeline/"
- spock_system_name = 'spockmk2-loginvm.pni.princeton.edu'
+ spock_system_name = "spockmk2-loginvm.pni.princeton.edu"
- pupillometry_slurm_filepath = os.path.abspath(os.path.realpath(__file__)+ "/../")
+ pupillometry_slurm_filepath = os.path.abspath(os.path.realpath(__file__) + "/../")
- #pupillometry_slurm_filepath = 'u19_pipeline/'
- pupillometry_slurm_filename = 'slurm_pupillometry.slurm'
+ # pupillometry_slurm_filepath = 'u19_pipeline/'
+ pupillometry_slurm_filename = "slurm_pupillometry.slurm"
slurm_dict_pupillometry_spock = {
- 'job-name': 'dj_ingestion',
- 'nodes': 1,
- 'cpus-per-task': 4,
- 'time': '20:00:00',
- 'mem': '50G',
- 'mail-type': ['END', 'FAIL'],
+ "job-name": "dj_ingestion",
+ "nodes": 1,
+ "cpus-per-task": 4,
+ "time": "20:00:00",
+ "mem": "50G",
+ "mail-type": ["END", "FAIL"],
}
@staticmethod
def generate_slurm_pupillometry(slurm_dict):
- slurm_text = '#!/bin/bash\n'
+ slurm_text = "#!/bin/bash\n"
slurm_text += PupillometryProcessingHandler.create_pupillometry_slurm_params_file(slurm_dict)
- slurm_text += '''
+ slurm_text += """
echo "SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}"
echo "VIDEO_DIR: ${video_dir}"
@@ -86,112 +89,131 @@ def generate_slurm_pupillometry(slurm_dict):
cd ${repository_dir}
python ${process_script_path} ${video_dir} ${model_dir} ${output_dir}
#python ${process_script_path} ${recording_process_id}
- '''
+ """
return slurm_text
@staticmethod
def create_pupillometry_slurm_params_file(slurm_dict):
- text_dict = ''
- for slurm_param in slurm_dict.keys():
-
+ text_dict = ""
+ for slurm_param in slurm_dict:
if isinstance(slurm_dict[slurm_param], list):
for list_param in slurm_dict[slurm_param]:
- text_dict += '#SBATCH --' + str(slurm_param) + '=' + str(list_param) + '\n'
+ text_dict += "#SBATCH --" + str(slurm_param) + "=" + str(list_param) + "\n"
else:
- text_dict += '#SBATCH --' + str(slurm_param) + '=' + str(slurm_dict[slurm_param]) + '\n'
+ text_dict += "#SBATCH --" + str(slurm_param) + "=" + str(slurm_dict[slurm_param]) + "\n"
return text_dict
@staticmethod
def generate_slurm_file(video_dir):
- '''
+ """
Generate and send slurm file to be queued in processing cluster
- '''
+ """
- #Get all associated directories given the selected processing cluster
+ # Get all associated directories given the selected processing cluster
# Start with default values
slurm_dict = copy.deepcopy(PupillometryProcessingHandler.slurm_dict_pupillometry_spock)
- label_rec_process = 'job_id_'+str(video_dir.stem)
- slurm_dict['job-name'] = label_rec_process
- print('PupillometryProcessingHandler.spock_log_dir')
- print(PupillometryProcessingHandler.spock_log_dir)
- print('label_rec_process', label_rec_process)
- print('video_dir', video_dir)
- slurm_dict['output'] = PupillometryProcessingHandler.spock_log_dir+label_rec_process+'.log'
- slurm_dict['error'] = PupillometryProcessingHandler.spock_error_dir+label_rec_process+'.log'
+ label_rec_process = "job_id_" + str(video_dir.stem)
+ slurm_dict["job-name"] = label_rec_process
+ logger.debug("PupillometryProcessingHandler.spock_log_dir %s", PupillometryProcessingHandler.spock_log_dir)
+ logger.debug("label_rec_process %s", label_rec_process)
+ logger.debug("video_dir %s", video_dir)
+ slurm_dict["output"] = PupillometryProcessingHandler.spock_log_dir + label_rec_process + ".log"
+ slurm_dict["error"] = PupillometryProcessingHandler.spock_error_dir + label_rec_process + ".log"
- print('slurm_dict', slurm_dict)
+ logger.debug("slurm_dict %s", slurm_dict)
slurm_text = PupillometryProcessingHandler.generate_slurm_pupillometry(slurm_dict)
- slurm_file_local_path = str(pathlib.Path(PupillometryProcessingHandler.pupillometry_slurm_filepath,
- PupillometryProcessingHandler.pupillometry_slurm_filename))
-
- print(slurm_file_local_path)
+ slurm_file_local_path = str(
+ pathlib.Path(
+ PupillometryProcessingHandler.pupillometry_slurm_filepath,
+ PupillometryProcessingHandler.pupillometry_slurm_filename,
+ )
+ )
- print('slurm text', slurm_text)
+ logger.debug("slurm_file_local_path %s", slurm_file_local_path)
+ logger.debug("slurm text %s", slurm_text)
write_file(slurm_file_local_path, slurm_text)
- slurm_destination = pathlib.Path(PupillometryProcessingHandler.spock_slurm_filepath,
- PupillometryProcessingHandler.pupillometry_slurm_filename).as_posix()
+ slurm_destination = pathlib.Path(
+ PupillometryProcessingHandler.spock_slurm_filepath,
+ PupillometryProcessingHandler.pupillometry_slurm_filename,
+ ).as_posix()
- status = PupillometryProcessingHandler.transfer_pupillometry_slurm_file(slurm_file_local_path, slurm_destination)
+ status = PupillometryProcessingHandler.transfer_pupillometry_slurm_file(
+ slurm_file_local_path, slurm_destination
+ )
- print(status)
- print(slurm_destination)
+ logger.debug("status %s", status)
+ logger.debug("slurm_destination %s", slurm_destination)
return status, slurm_destination
@staticmethod
- def queue_pupillometry_slurm_file(video_dir, model_dir, repository_dir, output_dir, process_script_path, slurm_location):
+ def queue_pupillometry_slurm_file(
+ video_dir,
+ model_dir,
+ repository_dir,
+ output_dir,
+ process_script_path,
+ slurm_location,
+ ):
id_slurm_job = -1
- #Get all associated variables given the selected processing cluster
- command = ['ssh', 'u19prod@'+PupillometryProcessingHandler.spock_system_name, 'sbatch',
- "--export=video_dir='"+str(video_dir)+
- "',model_dir='"+str(model_dir)+
- "',repository_dir='"+str(repository_dir)+
- "',process_script_path='"+str(process_script_path)+
- "',output_dir='"+str(output_dir)+"'",
- slurm_location
+ # Get all associated variables given the selected processing cluster
+ command = [
+ "ssh",
+ "u19prod@" + PupillometryProcessingHandler.spock_system_name,
+ "sbatch",
+ "--export=video_dir='"
+ + str(video_dir)
+ + "',model_dir='"
+ + str(model_dir)
+ + "',repository_dir='"
+ + str(repository_dir)
+ + "',process_script_path='"
+ + str(process_script_path)
+ + "',output_dir='"
+ + str(output_dir)
+ + "'",
+ slurm_location,
]
- print(command)
+ logger.debug("command %s", command)
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- #p = os.popen(command_new).read()
+ # p = os.popen(command_new).read()
p.wait()
stdout, stderr = p.communicate()
- print(p.returncode)
- print(stderr)
- print(stdout)
+ logger.debug("returncode %s", p.returncode)
+ logger.debug("stderr %s", stderr)
+ logger.debug("stdout %s", stdout)
if p.returncode == 0:
- error_message = ''
- batch_job_sentence = stdout.decode('UTF-8')
- print('batch_job_sentence', batch_job_sentence)
- id_slurm_job = batch_job_sentence.replace("Submitted batch job ","")
- id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job)
+ error_message = ""
+ batch_job_sentence = stdout.decode("UTF-8")
+ logger.debug("batch_job_sentence %s", batch_job_sentence)
+ id_slurm_job = batch_job_sentence.replace("Submitted batch job ", "")
+ id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job)
else:
- error_message = stderr.decode('UTF-8')
+ error_message = stderr.decode("UTF-8")
return p.returncode, id_slurm_job, error_message
-
@staticmethod
def transfer_pupillometry_slurm_file(slurm_file_local_path, slurm_destination):
- '''
+ """
Create scp command from cluster directories and local slurm file
- '''
-
- print("cp", slurm_file_local_path, slurm_destination)
+ """
- print(["cp", slurm_file_local_path, slurm_destination])
+ logger.debug("cp %s %s", slurm_file_local_path, slurm_destination)
+ logger.debug("cp command %s", ["cp", slurm_file_local_path, slurm_destination])
p = subprocess.Popen(["cp", slurm_file_local_path, slurm_destination])
transfer_status = p.wait()
@@ -210,20 +232,20 @@ def getPupilDiameter(analyzedVideoDataPath):
labels = pd.read_hdf(analyzedVideoDataPath)
# Create a data frame of the same size ad the analyzed video data filled with zeros
- df = pd.DataFrame(np.zeros(1), columns=['PupilDiameter'])
+ df = pd.DataFrame(np.zeros(1), columns=["PupilDiameter"])
# For each frame, get the x and y coordinates of the points around the pupil, fit an ellipse and calculate the diameter of a circle with the same area as the ellipse
for i in range(labels.index.size):
subset = labels.loc[i]
- x = subset.xs('x', level='coords').to_numpy()[0:8]
- y = subset.xs('y', level='coords').to_numpy()[0:8]
- xy = np.column_stack((x,y))
+ x = subset.xs("x", level="coords").to_numpy()[0:8]
+ y = subset.xs("y", level="coords").to_numpy()[0:8]
+ xy = np.column_stack((x, y))
# Fit the points to an ellipse and get the parameters (estimate X center coordinate, estimate Y center coordinate, a, b, theta)
ellipse = EllipseModel()
ellipse.estimate(xy)
# Calculate the area of the ellipse from the parameters a and b
ellipseArea = np.pi * ellipse.params[2] * ellipse.params[3]
# Get the diameter of a circle from the area of the ellipse
- pupilDiameter = 2 * np.sqrt(ellipseArea/np.pi)
+ pupilDiameter = 2 * np.sqrt(ellipseArea / np.pi)
df.loc[i] = pupilDiameter
# Get outliers (frames where either the mice have the eyes closed (blink or groom) or deeplabcut fails to track the pupil correctly)
@@ -236,8 +258,8 @@ def getPupilDiameter(analyzedVideoDataPath):
outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: "OutlierFlag"})
# Concatenate outlier flags array to remove outliers from pupil diameter array
temp = pd.concat([df, outlierFlags], axis=1)
- temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None
- pupilDiameter = temp['PupilDiameter']
+ temp.loc[temp["OutlierFlag"], "PupilDiameter"] = None
+ pupilDiameter = temp["PupilDiameter"]
return pupilDiameter.to_numpy()
@@ -245,140 +267,193 @@ def getPupilDiameter(analyzedVideoDataPath):
def analyze_videos_pupillometry(configPath, videoPath, output_dir):
import deeplabcut
+
# Analyze video and get pupil diameter data
deeplabcut.analyze_videos(configPath, videoPath, destfolder=output_dir)
-
@staticmethod
@pupillometry_exception_handler
def check_pupillometry_sessions_queue():
- status_update = config.status_update_idx['NO_CHANGE']
+ config.status_update_idx["NO_CHANGE"]
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- sessions_missing_process = (acquisition.SessionVideo *
- pupillometry.PupillometrySessionModelData & 'pupillometry_job_id is NULL').fetch(as_dict=True)
+ sessions_missing_process = (
+ acquisition.SessionVideo * pupillometry.PupillometrySessionModelData & "pupillometry_job_id is NULL"
+ ).fetch(as_dict=True)
- print(sessions_missing_process)
+ logger.debug("sessions_missing_process %s", sessions_missing_process)
for pupillometry_2_process in sessions_missing_process:
-
# If error, job id = -1
- key_insert = dict((k, pupillometry_2_process[k]) for k in ('subject_fullname', 'session_date', 'session_number', 'model_id'))
- key_insert['pupillometry_job_id'] = -1
+ key_insert = {
+ k: pupillometry_2_process[k]
+ for k in (
+ "subject_fullname",
+ "session_date",
+ "session_number",
+ "model_id",
+ )
+ }
+ key_insert["pupillometry_job_id"] = -1
# Get model location
- model_key = (pupillometry.PupillometryModels & "model_id = "+str(pupillometry_2_process['model_id'])).fetch(as_dict=True)[0]
- models_dir = dj.config.get('custom', {}).get('root_data_dir',None)
+ model_key = (
+ pupillometry.PupillometryModels & "model_id = " + str(pupillometry_2_process["model_id"])
+ ).fetch(as_dict=True)[0]
+ models_dir = dj.config.get("custom", {}).get("root_data_dir", None)
if models_dir is None:
- raise Exception('root_data_dir in not found in config, , run initial_conf.py again')
+ raise Exception("root_data_dir in not found in config, , run initial_conf.py again")
models_dir = models_dir[0]
- model_path = pathlib.Path(models_dir, model_key['model_path'])
+ model_path = pathlib.Path(models_dir, model_key["model_path"])
- print('model path', model_path)
+ logger.debug("model path %s", model_path)
# Get video location
- pupillometry_dir = dj.config.get('custom', {}).get('pupillometry_root_data_dir',None)
+ pupillometry_dir = dj.config.get("custom", {}).get("pupillometry_root_data_dir", None)
if pupillometry_dir is None:
- raise Exception('pupillometry_root_data_dir not found in config, run initial_conf.py again')
+ raise Exception("pupillometry_root_data_dir not found in config, run initial_conf.py again")
pupillometry_raw_dir = pupillometry_dir[0]
- videoPath = pathlib.Path(pupillometry_raw_dir, pupillometry_2_process['remote_path_video_file'])
+ videoPath = pathlib.Path(pupillometry_raw_dir, pupillometry_2_process["remote_path_video_file"])
- print('videoPath', videoPath)
+ logger.debug("videoPath %s", videoPath)
# Create output location
pupillometry_processed_dir = pupillometry_dir[1]
- output_dir = pathlib.Path(pupillometry_processed_dir,pathlib.Path(pupillometry_2_process['remote_path_video_file']).parent)
+ output_dir = pathlib.Path(
+ pupillometry_processed_dir,
+ pathlib.Path(pupillometry_2_process["remote_path_video_file"]).parent,
+ )
if not output_dir.exists():
output_dir.mkdir(parents=True, exist_ok=True)
# Generate slurm file and transfer it to spock
status, slurm_filepath = PupillometryProcessingHandler.generate_slurm_file(videoPath)
- print('slurm_filepath', slurm_filepath)
+ logger.debug("slurm_filepath %s", slurm_filepath)
# Error handling (generating slurm file)
- if status != config.system_process['SUCCESS']:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'Error while generating/transfering pupillometry slurm file'
+ if status != config.system_process["SUCCESS"]:
+ config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = (
+ "Error while generating/transfering pupillometry slurm file"
+ )
pupillometry.PupillometrySessionModelData.update1(key_insert)
- slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\
- update_value_dict['error_info'] ,pupillometry_2_process)
+ slack_utils.send_slack_error_pupillometry_notification(
+ config.slack_webhooks_dict["automation_pipeline_error_notification"],
+ update_value_dict["error_info"],
+ pupillometry_2_process,
+ )
- #return (status_update, update_value_dict)
+ # return (status_update, update_value_dict)
continue
# Queue slurm file in spock
status, slurm_jobid, error_message = PupillometryProcessingHandler.queue_pupillometry_slurm_file(
- videoPath, model_path, PupillometryProcessingHandler.spock_home_dir, output_dir,
- PupillometryProcessingHandler.process_script_path, slurm_filepath)
+ videoPath,
+ model_path,
+ PupillometryProcessingHandler.spock_home_dir,
+ output_dir,
+ PupillometryProcessingHandler.process_script_path,
+ slurm_filepath,
+ )
# Error handling (queuing slurm file)
- if status != config.system_process['SUCCESS']:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'Error to queue pupillometry slurm file'
+ if status != config.system_process["SUCCESS"]:
+ config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = "Error to queue pupillometry slurm file"
pupillometry.PupillometrySessionModelData.update1(key_insert)
- slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\
- update_value_dict['error_info'] ,pupillometry_2_process)
+ slack_utils.send_slack_error_pupillometry_notification(
+ config.slack_webhooks_dict["automation_pipeline_error_notification"],
+ update_value_dict["error_info"],
+ pupillometry_2_process,
+ )
- #return (status_update, update_value_dict)
+ # return (status_update, update_value_dict)
continue
# If success, store job_id
- key_insert['pupillometry_job_id'] = slurm_jobid
+ key_insert["pupillometry_job_id"] = slurm_jobid
pupillometry.PupillometrySessionModelData.update1(key_insert)
- slack_utils.send_slack_pupillometry_update_notification(config.slack_webhooks_dict['automation_pipeline_update_notification'],\
- 'Pupillometry job submitted', pupillometry_2_process)
+ slack_utils.send_slack_pupillometry_update_notification(
+ config.slack_webhooks_dict["automation_pipeline_update_notification"],
+ "Pupillometry job submitted",
+ pupillometry_2_process,
+ )
@staticmethod
@pupillometry_exception_handler
def check_processed_pupillometry_sessions():
- #status_update = config.status_update_idx['NO_CHANGE']
+ # status_update = config.status_update_idx['NO_CHANGE']
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- sessions_to_check = (acquisition.SessionVideo * pupillometry.PupillometrySessionModelData & 'pupillometry_job_id > 0 and pupil_diameter is NULL').fetch(as_dict=True)
+ sessions_to_check = (
+ acquisition.SessionVideo * pupillometry.PupillometrySessionModelData
+ & "pupillometry_job_id > 0 and pupil_diameter is NULL"
+ ).fetch(as_dict=True)
for session_check in sessions_to_check:
-
- key_update = dict((k, session_check[k]) for k in ('subject_fullname', 'session_date', 'session_number', 'model_id'))
- print('key_update1', key_update)
-
- status_update, message = slurmlib.check_slurm_job('u19prod', PupillometryProcessingHandler.spock_system_name, str(session_check['pupillometry_job_id']), local_user=False)
-
- #If job finished copy over output and/or error log
- if status_update == config.status_update_idx['ERROR_STATUS']:
-
- update_value_dict['error_info']['error_message'] = 'An error occured in processing (check LOG)'
- update_value_dict['error_info']['error_exception'] = message
-
- key_update['pupillometry_job_id'] = -1
+ key_update = {
+ k: session_check[k]
+ for k in (
+ "subject_fullname",
+ "session_date",
+ "session_number",
+ "model_id",
+ )
+ }
+ logger.debug("key_update1 %s", key_update)
+
+ status_update, message = slurmlib.check_slurm_job(
+ "u19prod",
+ PupillometryProcessingHandler.spock_system_name,
+ str(session_check["pupillometry_job_id"]),
+ local_user=False,
+ )
+
+ # If job finished copy over output and/or error log
+ if status_update == config.status_update_idx["ERROR_STATUS"]:
+ update_value_dict["error_info"]["error_message"] = "An error occured in processing (check LOG)"
+ update_value_dict["error_info"]["error_exception"] = message
+
+ key_update["pupillometry_job_id"] = -1
pupillometry.PupillometrySessionModelData.update1(key_update)
- slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\
- update_value_dict['error_info'] ,session_check)
+ slack_utils.send_slack_error_pupillometry_notification(
+ config.slack_webhooks_dict["automation_pipeline_error_notification"],
+ update_value_dict["error_info"],
+ session_check,
+ )
continue
-
# Get video location
- if status_update == config.status_update_idx['NEXT_STATUS']:
- pupillometry_dir = dj.config.get('custom', {}).get('pupillometry_root_data_dir',None)
+ if status_update == config.status_update_idx["NEXT_STATUS"]:
+ pupillometry_dir = dj.config.get("custom", {}).get("pupillometry_root_data_dir", None)
if pupillometry_dir is None:
- raise Exception('pupillometry_root_data_dir not found in config, run initial_conf.py again')
+ raise Exception("pupillometry_root_data_dir not found in config, run initial_conf.py again")
# Create output location
pupillometry_processed_dir = pupillometry_dir[1]
- output_dir = pathlib.Path(pupillometry_processed_dir,pathlib.Path(session_check['remote_path_video_file']).parent)
+ output_dir = pathlib.Path(
+ pupillometry_processed_dir,
+ pathlib.Path(session_check["remote_path_video_file"]).parent,
+ )
- print(output_dir)
+ logger.debug("output_dir %s", output_dir)
- #Find h5 files
- h5_files = glob.glob(str(output_dir) + '/*.h5')
+ # Find h5 files
+ h5_files = glob.glob(str(output_dir) + "/*.h5")
if len(h5_files) != 1:
- update_value_dict['error_info']['error_message'] = 'Didn''t find any h5 files after deeplabcut analyze_video'
- slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\
- update_value_dict['error_info'] ,session_check)
- key_update['pupillometry_job_id'] = -1
+ update_value_dict["error_info"]["error_message"] = (
+ "Didnt find any h5 files after deeplabcut analyze_video"
+ )
+ slack_utils.send_slack_error_pupillometry_notification(
+ config.slack_webhooks_dict["automation_pipeline_error_notification"],
+ update_value_dict["error_info"],
+ session_check,
+ )
+ key_update["pupillometry_job_id"] = -1
pupillometry.PupillometrySessionModelData.update1(key_update)
continue
else:
@@ -386,35 +461,39 @@ def check_processed_pupillometry_sessions():
try:
pupil_data = PupillometryProcessingHandler.getPupilDiameter(h5_files)
- except Exception as e:
- update_value_dict['error_info']['error_message'] = 'Could not get pupil diameter (check h5 or video file)'
- slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\
- update_value_dict['error_info'] ,session_check)
- key_update['pupillometry_job_id'] = -1
+ except Exception:
+ update_value_dict["error_info"]["error_message"] = (
+ "Could not get pupil diameter (check h5 or video file)"
+ )
+ slack_utils.send_slack_error_pupillometry_notification(
+ config.slack_webhooks_dict["automation_pipeline_error_notification"],
+ update_value_dict["error_info"],
+ session_check,
+ )
+ key_update["pupillometry_job_id"] = -1
pupillometry.PupillometrySessionModelData.update1(key_update)
continue
- key_update['pupil_diameter'] = pupil_data
- print('key_update', key_update)
+ key_update["pupil_diameter"] = pupil_data
+ logger.debug("key_update %s", key_update)
pupillometry.PupillometrySessionModelData.update1(key_update)
- slack_utils.send_slack_pupillometry_update_notification(config.slack_webhooks_dict['automation_pipeline_update_notification'],\
- 'Pupillometry job finished', session_check)
-
+ slack_utils.send_slack_pupillometry_update_notification(
+ config.slack_webhooks_dict["automation_pipeline_update_notification"],
+ "Pupillometry job finished",
+ session_check,
+ )
-if __name__ == '__main__':
-
+if __name__ == "__main__":
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(1)
args = sys.argv[1:]
- args[1] = args[1] + '/config.yaml'
- print(args)
- #
-
-
+ args[1] = args[1] + "/config.yaml"
+ logger.debug("args %s", args)
PupillometryProcessingHandler.analyze_videos_pupillometry(args[1], args[0], args[2])
-
diff --git a/u19_pipeline/automatic_job/pupillometry_handler_script.py b/u19_pipeline/automatic_job/pupillometry_handler_script.py
index 19c02b57..42df0e1a 100644
--- a/u19_pipeline/automatic_job/pupillometry_handler_script.py
+++ b/u19_pipeline/automatic_job/pupillometry_handler_script.py
@@ -1,11 +1,10 @@
-
import time
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(1)
-import datajoint as dj
import u19_pipeline.automatic_job.pupillometry_handler as ph
ph.PupillometryProcessingHandler.check_pupillometry_sessions_queue()
diff --git a/u19_pipeline/automatic_job/pupillometry_main.py b/u19_pipeline/automatic_job/pupillometry_main.py
index f2f08b58..b931f0da 100644
--- a/u19_pipeline/automatic_job/pupillometry_main.py
+++ b/u19_pipeline/automatic_job/pupillometry_main.py
@@ -1,18 +1,23 @@
import sys
-if __name__ == '__main__':
-
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+if __name__ == "__main__":
import time
+
from scripts.conf_file_finding import try_find_conf_file
+
try_find_conf_file()
time.sleep(1)
import u19_pipeline.automatic_job.pupillometry_handler as ph
args = sys.argv[1:]
- print(args)
- print(args[0])
- print(args[1])
- print(args[2])
+ logger.debug("args %s", args)
+ logger.debug("args[0] %s", args[0])
+ logger.debug("args[1] %s", args[1])
+ logger.debug("args[2] %s", args[2])
ph.PupillometryProcessingHandler.analyze_videos_pupillometry(args[0], args[1], args[2])
diff --git a/u19_pipeline/automatic_job/recording_handler.py b/u19_pipeline/automatic_job/recording_handler.py
index 46643147..291a0953 100644
--- a/u19_pipeline/automatic_job/recording_handler.py
+++ b/u19_pipeline/automatic_job/recording_handler.py
@@ -1,106 +1,126 @@
-
+import copy
import pathlib
import time
import traceback
-import pandas as pd
-import datajoint as dj
-import copy
-
from datetime import datetime
-from u19_pipeline import recording, ephys_pipeline, imaging_pipeline, recording, recording_process, lab
+import datajoint as dj
+import pandas as pd
+
+import u19_pipeline.automatic_job.params_config as config
import u19_pipeline.utils.dj_shortcuts as dj_short
-import u19_pipeline.utils.slack_utils as slack_utils
import u19_pipeline.utils.scp_transfers as scp_tr
-
+import u19_pipeline.utils.slack_utils as slack_utils
+from u19_pipeline import (
+ ephys_pipeline,
+ imaging_pipeline,
+ lab,
+ recording,
+ recording_process,
+)
from u19_pipeline.automatic_job import ephys_element_ingest
-import u19_pipeline.automatic_job.params_config as config
+from u19_pipeline.utils.logging_config import get_logger
+logger = get_logger(__name__)
def exception_handler(func):
- '''
+ """
Decorator function to get error message when a workflow manager function fails
- '''
+ """
+
def inner_function(*args, **kwargs):
try:
- argout = func(*args, **kwargs)
- return argout
+ argout = func(*args, **kwargs)
+ return argout
except Exception as e:
- print('Exception HERE ................')
+ logger.exception("Exception HERE ................")
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- update_value_dict['error_info']['error_message'] = str(e)
- update_value_dict['error_info']['error_exception'] = (''.join(traceback.format_exception(type(e), value=e, tb=e.__traceback__)))
+ update_value_dict["error_info"]["error_message"] = str(e)
+ update_value_dict["error_info"]["error_exception"] = "".join(
+ traceback.format_exception(type(e), value=e, tb=e.__traceback__)
+ )
return (config.RECORDING_STATUS_ERROR_ID, update_value_dict)
+
return inner_function
-class RecordingHandler():
+class RecordingHandler:
@staticmethod
def pipeline_handler_main():
- '''
+ """
Call all processing functions according to the current status of each recording
Update status of each process job accordingly
- '''
+ """
- #Get info from all the possible status for a processjob
+ # Get info from all the possible status for a processjob
df_all_recordings = RecordingHandler.get_active_recordings()
- #For all active process jobs
+ # For all active process jobs
for i in range(df_all_recordings.shape[0]):
-
- #Filter current process job
+ # Filter current process job
recording_series = df_all_recordings.loc[i, :]
- #Filter current status info
- current_status = recording_series['status_recording_id']
- next_status_series = config.recording_status_df.loc[config.recording_status_df['Value'] == current_status+1, :].squeeze()
+ # Filter current status info
+ current_status = recording_series["status_recording_id"]
+ next_status_series = config.recording_status_df.loc[
+ config.recording_status_df["Value"] == current_status + 1, :
+ ].squeeze()
- print('function to apply:', next_status_series['ProcessFunction'])
+ logger.info("function to apply: %s", next_status_series["ProcessFunction"])
# Get processing function
- function_status_process = getattr(RecordingHandler, next_status_series['ProcessFunction'])
+ function_status_process = getattr(RecordingHandler, next_status_series["ProcessFunction"])
- #Trigger process, if success update recording process record
+ # Trigger process, if success update recording process record
try:
status, update_dict = function_status_process(recording_series)
- #Get dictionary of record process
- key = recording_series['query_key']
+ # Get dictionary of record process
+ key = recording_series["query_key"]
- if status == config.status_update_idx['NEXT_STATUS']:
- #Get values to update
- next_status = next_status_series['Value']
- value_update = update_dict['value_update']
- field_update = next_status_series['UpdateField']
+ if status == config.status_update_idx["NEXT_STATUS"]:
+ # Get values to update
+ next_status = next_status_series["Value"]
+ value_update = update_dict["value_update"]
+ field_update = next_status_series["UpdateField"]
# Update status in u19_recording.recording table (possibly other field as well)
RecordingHandler.update_status_pipeline(key, next_status, field_update, value_update)
# Send slack Message to webhook if slack message activated for status
- if next_status_series['SlackMessage']:
- slack_utils.send_slack_update_notification(config.slack_webhooks_dict['automation_pipeline_update_notification'],\
- next_status_series['SlackMessage'], recording_series)
-
- #An error occurred in process
- if status == config.status_update_idx['ERROR_STATUS']:
+ if next_status_series["SlackMessage"]:
+ slack_utils.send_slack_update_notification(
+ config.slack_webhooks_dict["automation_pipeline_update_notification"],
+ next_status_series["SlackMessage"],
+ recording_series,
+ )
+
+ # An error occurred in process
+ if status == config.status_update_idx["ERROR_STATUS"]:
next_status = config.RECORDING_STATUS_ERROR_ID
- RecordingHandler.update_status_pipeline(key,next_status, None, None)
- slack_utils.send_slack_error_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\
- update_dict['error_info'] ,recording_series)
-
- #if success or error update status timestamps table
- if status != config.status_update_idx['NO_CHANGE']:
- RecordingHandler.update_recording_log(recording_series['recording_id'], current_status, next_status, update_dict['error_info'])
-
+ RecordingHandler.update_status_pipeline(key, next_status, None, None)
+ slack_utils.send_slack_error_notification(
+ config.slack_webhooks_dict["automation_pipeline_error_notification"],
+ update_dict["error_info"],
+ recording_series,
+ )
+
+ # if success or error update status timestamps table
+ if status != config.status_update_idx["NO_CHANGE"]:
+ RecordingHandler.update_recording_log(
+ recording_series["recording_id"],
+ current_status,
+ next_status,
+ update_dict["error_info"],
+ )
except Exception as err:
- raise(err)
+ raise (err)
## Send notification error, update recording to error
time.sleep(2)
-
@staticmethod
@exception_handler
def local_transfer_request(rec_series):
@@ -118,24 +138,30 @@ def local_transfer_request(rec_series):
'error_info': error info to be inserted if error occured }
"""
- status_update = config.status_update_idx['NO_CHANGE']
+ status_update = config.status_update_idx["NO_CHANGE"]
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- data_directory = pathlib.Path(dj.config['custom']['root_data_dir'], rec_series['recording_modality'], rec_series['recording_directory']).as_posix()
+ data_directory = pathlib.Path(
+ dj.config["custom"]["root_data_dir"],
+ rec_series["recording_modality"],
+ rec_series["recording_directory"],
+ ).as_posix()
data_directory = pathlib.Path(data_directory).parent
pathlib.Path(data_directory).mkdir(parents=True, exist_ok=True)
+ # Encode Windows like directory for scp to work
-
- #Encode Windows like directory for scp to work
-
- status, task_id = scp_tr.call_scp_background(ip_address=rec_series['ip_address'], system_user=rec_series['system_user'],
- recording_system_directory=rec_series['local_directory'], data_directory=data_directory.as_posix())
+ status, task_id = scp_tr.call_scp_background(
+ ip_address=rec_series["ip_address"],
+ system_user=rec_series["system_user"],
+ recording_system_directory=rec_series["local_directory"],
+ data_directory=data_directory.as_posix(),
+ )
if status:
- status_update = config.status_update_idx['NEXT_STATUS']
- update_value_dict['value_update'] = task_id
+ status_update = config.status_update_idx["NEXT_STATUS"]
+ update_value_dict["value_update"] = task_id
return (status_update, update_value_dict)
@@ -156,21 +182,21 @@ def local_transfer_check(rec_series):
'error_info': error info to be inserted if error occured }
"""
- status_update = config.status_update_idx['NO_CHANGE']
+ status_update = config.status_update_idx["NO_CHANGE"]
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- id_task = rec_series['task_copy_id_pni']
+ id_task = rec_series["task_copy_id_pni"]
is_finished, exit_code = scp_tr.check_scp_transfer(id_task)
if is_finished:
if exit_code == 0:
- status_update = config.status_update_idx['NEXT_STATUS']
+ status_update = config.status_update_idx["NEXT_STATUS"]
else:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'Return code scp not = 0'
+ status_update = config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = "Return code scp not = 0"
- print('is_finished', is_finished, 'status_update', status_update)
+ logger.debug("is_finished %s status_update %s", is_finished, status_update)
return (status_update, update_value_dict)
@@ -192,42 +218,41 @@ def modality_preingestion(in_rec_series):
rec_series = in_rec_series.copy()
- if rec_series['recording_modality'] == 'electrophysiology':
+ if rec_series["recording_modality"] == "electrophysiology":
status_update, update_value_dict = RecordingHandler.electrophysiology_preingestion(rec_series)
- if rec_series['recording_modality'] == 'imaging':
+ if rec_series["recording_modality"] == "imaging":
status_update, update_value_dict = RecordingHandler.imaging_preingestion(rec_series)
-
return (status_update, update_value_dict)
-
@staticmethod
def get_active_recordings():
- '''
+ """
get all recordings that have to go through some action in the pipeline
Return:
df_recordings (pd.DataFrame): all recordings that are going to be processed in the pipeline
- '''
+ """
- status_query = 'status_recording_id > ' + str(config.recording_status_df['Value'].min())
- status_query += ' and status_recording_id < ' + str(config.recording_status_df['Value'].max())
+ status_query = "status_recording_id > " + str(config.recording_status_df["Value"].min())
+ status_query += " and status_recording_id < " + str(config.recording_status_df["Value"].max())
- print(status_query)
+ logger.debug("status_query %s", status_query)
- recordings_active = recording.Recording * lab.Location.proj('ip_address', 'system_user', 'acquisition_type') & status_query
+ recordings_active = (
+ recording.Recording * lab.Location.proj("ip_address", "system_user", "acquisition_type") & status_query
+ )
- print(recordings_active)
+ logger.debug("recordings_active %s", recordings_active)
df_recordings = pd.DataFrame(recordings_active.fetch(as_dict=True))
if df_recordings.shape[0] > 0:
key_list = dj_short.get_primary_key_fields(recording.Recording)
- df_recordings['query_key'] = df_recordings.loc[:, key_list].to_dict(orient='records')
+ df_recordings["query_key"] = df_recordings.loc[:, key_list].to_dict(orient="records")
return df_recordings
-
@staticmethod
def update_status_pipeline(recording_key_dict, status, update_field=None, update_value=None):
"""
@@ -239,20 +264,20 @@ def update_status_pipeline(recording_key_dict, status, update_field=None, update
update_value (str|int): field value to be inserted on in task_field
"""
- print('recording_key_dict', recording_key_dict)
- print('status', status)
- print('update_field', update_field)
- print('update_value', update_value)
+ logger.debug("recording_key_dict %s", recording_key_dict)
+ logger.debug("status %s", status)
+ logger.debug("update_field %s", update_field)
+ logger.debug("update_value %s", update_value)
if update_field is not None:
update_task_id_dict = recording_key_dict.copy()
update_task_id_dict[update_field] = update_value
- print('update_task_id_dict', update_task_id_dict)
+ logger.debug("update_task_id_dict %s", update_task_id_dict)
recording.Recording.update1(update_task_id_dict)
update_status_dict = recording_key_dict.copy()
- update_status_dict['status_recording_id'] = status
- print('update_status_dict', update_status_dict)
+ update_status_dict["status_recording_id"] = status
+ logger.debug("update_status_dict %s", update_status_dict)
recording.Recording.update1(update_status_dict)
@staticmethod
@@ -266,26 +291,25 @@ def update_recording_log(recording_id, current_status, next_status, error_info_d
now = datetime.now()
date_time = now.strftime("%Y-%m-%d %H:%M:%S")
- print('error_info_dict', error_info_dict)
+ logger.debug("error_info_dict %s", error_info_dict)
- key = dict()
- key['recording_id'] = recording_id
- key['status_recording_id_old'] = current_status
- key['status_recording_id_new'] = next_status
- key['recording_status_timestamp'] = date_time
+ key = {}
+ key["recording_id"] = recording_id
+ key["status_recording_id_old"] = current_status
+ key["status_recording_id_new"] = next_status
+ key["recording_status_timestamp"] = date_time
- if error_info_dict['error_message'] is not None and len(error_info_dict['error_message']) >= 256:
- error_info_dict['error_message'] =error_info_dict['error_message'][:255]
+ if error_info_dict["error_message"] is not None and len(error_info_dict["error_message"]) >= 256:
+ error_info_dict["error_message"] = error_info_dict["error_message"][:255]
- if error_info_dict['error_exception'] is not None and len(error_info_dict['error_exception']) >= 4096:
- error_info_dict['error_exception'] =error_info_dict['error_exception'][:4095]
+ if error_info_dict["error_exception"] is not None and len(error_info_dict["error_exception"]) >= 4096:
+ error_info_dict["error_exception"] = error_info_dict["error_exception"][:4095]
- key['recording_error_message'] = error_info_dict['error_message']
- key['recording_error_exception'] = error_info_dict['error_exception']
+ key["recording_error_message"] = error_info_dict["error_message"]
+ key["recording_error_exception"] = error_info_dict["error_exception"]
recording.LogStatus.insert1(key)
-
@staticmethod
def electrophysiology_preingestion(rec_series):
"""
@@ -301,65 +325,88 @@ def electrophysiology_preingestion(rec_series):
'error_info': error info to be inserted if error occured }
"""
- status_update = config.status_update_idx['NO_CHANGE']
+ status_update = config.status_update_idx["NO_CHANGE"]
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- #Insert first ephysSession and firs tables of ephys elements
- ephys_pipeline.EphysPipelineSession.populate(rec_series['query_key'])
- ephys_element_ingest.process_session(rec_series['query_key'])
- ephys_pipeline.ephys_element.EphysRecording.populate(rec_series['query_key'])
+ # Insert first ephysSession and firs tables of ephys elements
+ ephys_pipeline.EphysPipelineSession.populate(rec_series["query_key"])
+ ephys_element_ingest.process_session(rec_series["query_key"])
+ ephys_pipeline.ephys_element.EphysRecording.populate(rec_series["query_key"])
- ingested_recording = (ephys_pipeline.ephys_element.EphysRecording & rec_series['query_key']).fetch("KEY", as_dict=True)
+ ingested_recording = (ephys_pipeline.ephys_element.EphysRecording & rec_series["query_key"]).fetch(
+ "KEY", as_dict=True
+ )
- print('before BehaviorSync')
- print("rec_series['query_key']", rec_series['query_key'])
+ logger.debug("before BehaviorSync")
+ logger.debug("rec_series['query_key'] %s", rec_series["query_key"])
- ephys_pipeline.BehaviorSync.populate(rec_series['query_key'])
+ ephys_pipeline.BehaviorSync.populate(rec_series["query_key"])
if len(ingested_recording) == 0:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'Ephys Recording was not ingested (probably recording not in location)'
+ status_update = config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = (
+ "Ephys Recording was not ingested (probably recording not in location)"
+ )
return (status_update, update_value_dict)
- #Insert recording processes records
- old_recording_process = (recording_process.Processing() & rec_series['query_key']).fetch("KEY", as_dict=True)
+ # Insert recording processes records
+ old_recording_process = (recording_process.Processing() & rec_series["query_key"]).fetch("KEY", as_dict=True)
if len(old_recording_process) == 0:
-
connection = recording.Recording.connection
with connection.transaction:
-
- probe_files = (ephys_pipeline.ephys_element.EphysRecording.EphysFile & rec_series['query_key']).fetch(as_dict=True)
- probe_files = [dict(item, recording_process_pre_path=pathlib.Path(item['file_path']).parents[0].as_posix()) for item in probe_files]
-
- recording_process.Processing().insert_recording_process(probe_files, 'insertion_number')
-
- #Get parameters for recording processes
- recording_processes = (recording_process.Processing() & rec_series['query_key']).fetch('job_id', 'recording_id', 'fragment_number', 'recording_process_pre_path', as_dict=True)
- default_params_record_df = pd.DataFrame((recording.DefaultParams & rec_series['query_key']).fetch(as_dict=True))
- params_rec_process = recording.DefaultParams.get_default_params_rec_process(recording_processes, default_params_record_df)
+ probe_files = (ephys_pipeline.ephys_element.EphysRecording.EphysFile & rec_series["query_key"]).fetch(
+ as_dict=True
+ )
+ probe_files = [
+ dict(
+ item,
+ recording_process_pre_path=pathlib.Path(item["file_path"]).parents[0].as_posix(),
+ )
+ for item in probe_files
+ ]
+
+ recording_process.Processing().insert_recording_process(probe_files, "insertion_number")
+
+ # Get parameters for recording processes
+ recording_processes = (recording_process.Processing() & rec_series["query_key"]).fetch(
+ "job_id",
+ "recording_id",
+ "fragment_number",
+ "recording_process_pre_path",
+ as_dict=True,
+ )
+ default_params_record_df = pd.DataFrame(
+ (recording.DefaultParams & rec_series["query_key"]).fetch(as_dict=True)
+ )
+ params_rec_process = recording.DefaultParams.get_default_params_rec_process(
+ recording_processes, default_params_record_df
+ )
# Rename preprocess_param_steps_id with the electrophysiology one
for i in params_rec_process:
- i['precluster_param_steps_id'] = i.pop('preprocess_param_steps_id')
+ i["precluster_param_steps_id"] = i.pop("preprocess_param_steps_id")
recording_process.Processing.EphysParams.insert(params_rec_process)
- #Update recording_process_post_path
+ # Update recording_process_post_path
recording_process.Processing().set_recording_process_post_path(recording_processes)
- #Create lfp trace if needed (neuropixel 2.0 probes)
- recording_directory = (recording.Recording & rec_series['query_key']).fetch1('recording_directory')
- recording_directory = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][0], recording_directory).parent.as_posix()
+ # Create lfp trace if needed (neuropixel 2.0 probes)
+ recording_directory = (recording.Recording & rec_series["query_key"]).fetch1("recording_directory")
+ recording_directory = pathlib.Path(
+ dj.config["custom"]["ephys_root_data_dir"][0], recording_directory
+ ).parent.as_posix()
for i in recording_processes:
- probe_dir = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][0], i['recording_process_pre_path']).as_posix()
+ probe_dir = pathlib.Path(
+ dj.config["custom"]["ephys_root_data_dir"][0],
+ i["recording_process_pre_path"],
+ ).as_posix()
ephys_pipeline.create_lfp_trace(config.catgt_script, recording_directory, probe_dir)
-
- status_update = config.status_update_idx['NEXT_STATUS']
+ status_update = config.status_update_idx["NEXT_STATUS"]
return (status_update, update_value_dict)
-
@staticmethod
def imaging_preingestion(rec_series):
"""
@@ -375,75 +422,95 @@ def imaging_preingestion(rec_series):
'error_info': error info to be inserted if error occured }
"""
- status_update = config.status_update_idx['NO_CHANGE']
+ status_update = config.status_update_idx["NO_CHANGE"]
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- print("rec_series['query_key']")
- print(rec_series['query_key'])
+ logger.debug("rec_series['query_key'] %s", rec_series["query_key"])
- #Populate ImagingPipelineSession and call matlab script that handles TiffSplits
- imaging_pipeline.ImagingPipelineSession.populate(rec_series['query_key'])
- imaging_pipeline.AcquiredTiff.populate(rec_series['query_key'])
+ # Populate ImagingPipelineSession and call matlab script that handles TiffSplits
+ imaging_pipeline.ImagingPipelineSession.populate(rec_series["query_key"])
+ imaging_pipeline.AcquiredTiff.populate(rec_series["query_key"])
- #Retrieve all fovs records ingested in matlab Script
- fovs_ingested = (imaging_pipeline.TiffSplit & rec_series['query_key']).fetch("KEY", as_dict=True)
+ # Retrieve all fovs records ingested in matlab Script
+ fovs_ingested = (imaging_pipeline.TiffSplit & rec_series["query_key"]).fetch("KEY", as_dict=True)
if len(fovs_ingested) == 0:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'Imaging TiffSplit process failed'
+ status_update = config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = "Imaging TiffSplit process failed"
return (status_update, update_value_dict)
- #Ingest Scan for each fov from the TiffSplit process
+ # Ingest Scan for each fov from the TiffSplit process
for this_fov in fovs_ingested:
-
# Scan_id always zero because TIFF splitted (FOVs) already on imaging_pipeline schema
- scan_id = 0
# Acquisition type will have Mesoscope or 2Photon
- scanner = rec_series['acquisition_type']
+ scanner = rec_series["acquisition_type"]
# Hardcoded acquisition software
- acq_software = 'ScanImage'
+ acq_software = "ScanImage"
- #Insert Scan and ScanInfo
+ # Insert Scan and ScanInfo
imaging_pipeline.scan_element.Scan.insert1(
- {**this_fov, 'scan_id': 0, 'scanner': scanner, 'acq_software': acq_software}, skip_duplicates=True)
+ {
+ **this_fov,
+ "scan_id": 0,
+ "scanner": scanner,
+ "acq_software": acq_software,
+ },
+ skip_duplicates=True,
+ )
- #Populate ScanInfo for all fovs
- imaging_pipeline.scan_element.ScanInfo.populate(rec_series['query_key'], display_progress=True)
+ # Populate ScanInfo for all fovs
+ imaging_pipeline.scan_element.ScanInfo.populate(rec_series["query_key"], display_progress=True)
- #ingested_recording = (imaging_pipeline.scan_element.Scan & rec_series['query_key']).fetch("KEY", as_dict=True)
+ # ingested_recording = (imaging_pipeline.scan_element.Scan & rec_series['query_key']).fetch("KEY", as_dict=True)
# Get fov directories for each recording process:
- fov_files_df = pd.DataFrame((imaging_pipeline.scan_element.ScanInfo.ScanFile & rec_series['query_key']).fetch(as_dict=True))
-
- fov_files = fov_files_df.groupby('tiff_split').first().reset_index()
+ fov_files_df = pd.DataFrame(
+ (imaging_pipeline.scan_element.ScanInfo.ScanFile & rec_series["query_key"]).fetch(as_dict=True)
+ )
+ fov_files = fov_files_df.groupby("tiff_split").first().reset_index()
- #Insert recording processes records
- old_recording_process = (recording_process.Processing() & rec_series['query_key']).fetch("KEY", as_dict=True)
+ # Insert recording processes records
+ old_recording_process = (recording_process.Processing() & rec_series["query_key"]).fetch("KEY", as_dict=True)
if len(old_recording_process) == 0:
-
connection = recording.Recording.connection
with connection.transaction:
-
# Get fov directories for each recording process:
- fov_files_df = pd.DataFrame((imaging_pipeline.scan_element.ScanInfo.ScanFile & rec_series['query_key']).fetch(as_dict=True))
- fov_files = fov_files_df.groupby('tiff_split').first().reset_index().to_dict('records')
-
- fov_files = [dict(item, recording_process_pre_path=pathlib.Path(item['file_path']).parent.as_posix()) for item in fov_files]
-
- recording_process.Processing().insert_recording_process(fov_files, 'tiff_split')
-
- #Get parameters for recording processes
- recording_processes = (recording_process.Processing() & rec_series['query_key']).fetch('job_id', 'recording_id', 'fragment_number', 'recording_process_pre_path', as_dict=True)
- default_params_record_df = pd.DataFrame((recording.DefaultParams & rec_series['query_key']).fetch(as_dict=True))
- params_rec_process = recording.DefaultParams.get_default_params_rec_process(recording_processes, default_params_record_df)
+ fov_files_df = pd.DataFrame(
+ (imaging_pipeline.scan_element.ScanInfo.ScanFile & rec_series["query_key"]).fetch(as_dict=True)
+ )
+ fov_files = fov_files_df.groupby("tiff_split").first().reset_index().to_dict("records")
+
+ fov_files = [
+ dict(
+ item,
+ recording_process_pre_path=pathlib.Path(item["file_path"]).parent.as_posix(),
+ )
+ for item in fov_files
+ ]
+
+ recording_process.Processing().insert_recording_process(fov_files, "tiff_split")
+
+ # Get parameters for recording processes
+ recording_processes = (recording_process.Processing() & rec_series["query_key"]).fetch(
+ "job_id",
+ "recording_id",
+ "fragment_number",
+ "recording_process_pre_path",
+ as_dict=True,
+ )
+ default_params_record_df = pd.DataFrame(
+ (recording.DefaultParams & rec_series["query_key"]).fetch(as_dict=True)
+ )
+ params_rec_process = recording.DefaultParams.get_default_params_rec_process(
+ recording_processes, default_params_record_df
+ )
recording_process.Processing.ImagingParams.insert(params_rec_process, skip_duplicates=True)
- #Update recording_process_post_path
+ # Update recording_process_post_path
recording_process.Processing().set_recording_process_post_path(recording_processes)
- status_update = config.status_update_idx['NEXT_STATUS']
+ status_update = config.status_update_idx["NEXT_STATUS"]
return (status_update, update_value_dict)
-
diff --git a/u19_pipeline/automatic_job/recording_process_handler.py b/u19_pipeline/automatic_job/recording_process_handler.py
index ef66eb42..f83100ad 100644
--- a/u19_pipeline/automatic_job/recording_process_handler.py
+++ b/u19_pipeline/automatic_job/recording_process_handler.py
@@ -1,127 +1,155 @@
-
-import time
-import traceback
-import pandas as pd
-import datajoint as dj
import copy
import pathlib
-import numpy as np
+import time
+import traceback
+from datetime import datetime
-from u19_pipeline.automatic_job import recording_handler
+import pandas as pd
+from ecephys_spike_sorting.common.SGLXMetaToCoords import MetaToCoords
-import u19_pipeline.utils.dj_shortcuts as dj_short
-import u19_pipeline.utils.slack_utils as slack_utils
import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft
-import u19_pipeline.automatic_job.slurm_creator as slurmlib
-import u19_pipeline.automatic_job.parameter_file_creator as paramfilelib
-import u19_pipeline.automatic_job.params_config as config
import u19_pipeline.automatic_job.ephys_element_populate as ep
import u19_pipeline.automatic_job.imaging_element_populate as ip
+import u19_pipeline.automatic_job.parameter_file_creator as paramfilelib
+import u19_pipeline.automatic_job.params_config as config
+import u19_pipeline.automatic_job.slurm_creator as slurmlib
+import u19_pipeline.utils.dj_shortcuts as dj_short
+import u19_pipeline.utils.slack_utils as slack_utils
+from u19_pipeline import (
+ ephys_pipeline,
+ imaging_pipeline,
+ recording,
+ recording_process,
+ utility,
+)
+from u19_pipeline.automatic_job import recording_handler
+from u19_pipeline.utils.logging_config import get_logger
-from datetime import datetime
-from u19_pipeline import recording, recording_process, ephys_pipeline, imaging_pipeline, utility
-from u19_pipeline.utility import create_str_from_dict, is_this_spock
-from u19_pipeline.utils import ephys_utils
-
-from ecephys_spike_sorting.common.SGLXMetaToCoords import MetaToCoords
+logger = get_logger(__name__)
+from u19_pipeline.utility import is_this_spock
-class RecProcessHandler():
+class RecProcessHandler:
@staticmethod
def pipeline_handler_main():
- '''
+ """
Call all processing functions according to the current status of each process job
Update status of each process job accordingly
- '''
+ """
- #Get info from all the possible status for a processjob
+ # Get info from all the possible status for a processjob
df_all_process_job = RecProcessHandler.get_active_process_jobs()
- #For all active process jobs
+ # For all active process jobs
for i in range(df_all_process_job.shape[0]):
-
- #Filter current process job
+ # Filter current process job
rec_process_series = df_all_process_job.loc[i, :].copy()
- #Filter current status info
- current_status = rec_process_series['status_processing_id']
- current_status_series = config.recording_process_status_df.loc[config.recording_process_status_df['Value'] == current_status, :].squeeze()
- next_status_series = config.recording_process_status_df.loc[config.recording_process_status_df['Value'] == current_status+1, :].squeeze()
+ # Filter current status info
+ current_status = rec_process_series["status_processing_id"]
+ config.recording_process_status_df.loc[
+ config.recording_process_status_df["Value"] == current_status, :
+ ].squeeze()
+ next_status_series = config.recording_process_status_df.loc[
+ config.recording_process_status_df["Value"] == current_status + 1, :
+ ].squeeze()
# Get processing function
- function_status_process = getattr(RecProcessHandler, next_status_series['ProcessFunction'])
+ function_status_process = getattr(RecProcessHandler, next_status_series["ProcessFunction"])
- #Trigger process, if success update recording process record
+ # Trigger process, if success update recording process record
try:
status, update_dict = function_status_process(rec_process_series, next_status_series)
- #print('update_dict', update_dict)
- #Get dictionary of record process
- key = rec_process_series['query_key']
-
- print('rec_process_series')
- print(rec_process_series)
-
- if status == config.status_update_idx['NEXT_STATUS']:
-
+ # print('update_dict', update_dict)
+ # Get dictionary of record process
+ key = rec_process_series["query_key"]
- #Get values to update
- next_status = next_status_series['Value']
- value_update = update_dict['value_update']
- field_update = next_status_series['UpdateField']
+ logger.debug("rec_process_series %s", rec_process_series)
- print('key to update', key)
- print('old status', current_status, 'new status', next_status)
- print('value_update', value_update, 'field_update', field_update)
- print('function executed:', next_status_series['ProcessFunction'])
+ if status == config.status_update_idx["NEXT_STATUS"]:
+ # Get values to update
+ next_status = next_status_series["Value"]
+ value_update = update_dict["value_update"]
+ field_update = next_status_series["UpdateField"]
+ logger.info("key to update %s", key)
+ logger.info("old status %s new status %s", current_status, next_status)
+ logger.info("value_update %s field_update %s", value_update, field_update)
+ logger.info("function executed: %s", next_status_series["ProcessFunction"])
RecProcessHandler.update_status_pipeline(key, next_status, field_update, value_update)
- if next_status_series['SlackMessage']:
- slack_utils.send_slack_update_notification(config.slack_webhooks_dict['automation_pipeline_update_notification'],\
- next_status_series['SlackMessage'], rec_process_series)
+ if next_status_series["SlackMessage"]:
+ slack_utils.send_slack_update_notification(
+ config.slack_webhooks_dict["automation_pipeline_update_notification"],
+ next_status_series["SlackMessage"],
+ rec_process_series,
+ )
- #if success or error update status timestamps table
- if status != config.status_update_idx['NO_CHANGE']:
- if status == config.status_update_idx['ERROR_STATUS']:
+ # if success or error update status timestamps table
+ if status != config.status_update_idx["NO_CHANGE"]:
+ if status == config.status_update_idx["ERROR_STATUS"]:
next_status = config.RECORDING_STATUS_ERROR_ID
- #Crop error messages to fit in DB
- if len(update_dict['error_info']['error_message']) > 255:
- print('Cropping error message')
- update_dict['error_info']['error_message'] = update_dict['error_info']['error_message'][-255:]
-
- if isinstance(update_dict['error_info']['error_exception'], str) and len(update_dict['error_info']['error_exception']) > 4095:
- print('Cropping error error_exception')
- update_dict['error_info']['error_exception'] = update_dict['error_info']['error_exception'][-4095:]
-
- RecProcessHandler.update_job_id_log(rec_process_series['job_id'], current_status, next_status, update_dict['error_info'])
-
- #An error occurred in process
- if status == config.status_update_idx['ERROR_STATUS']:
-
+ # Crop error messages to fit in DB
+ if len(update_dict["error_info"]["error_message"]) > 255:
+ logger.warning("Cropping error message")
+ update_dict["error_info"]["error_message"] = update_dict["error_info"]["error_message"][
+ -255:
+ ]
+
+ if (
+ isinstance(update_dict["error_info"]["error_exception"], str)
+ and len(update_dict["error_info"]["error_exception"]) > 4095
+ ):
+ logger.warning("Cropping error error_exception")
+ update_dict["error_info"]["error_exception"] = update_dict["error_info"]["error_exception"][
+ -4095:
+ ]
+
+ RecProcessHandler.update_job_id_log(
+ rec_process_series["job_id"],
+ current_status,
+ next_status,
+ update_dict["error_info"],
+ )
+
+ # An error occurred in process
+ if status == config.status_update_idx["ERROR_STATUS"]:
next_status = config.RECORDING_STATUS_ERROR_ID
- RecProcessHandler.update_status_pipeline(key,next_status, None, None)
-
- #Crop even more error message to fit in slack notification
- if isinstance(update_dict['error_info']['error_exception'], str) and len(update_dict['error_info']['error_exception']) > 1024:
- print('Cropping error error_exception for slack')
- update_dict['error_info']['error_exception'] = update_dict['error_info']['error_exception'][-1023:]
+ RecProcessHandler.update_status_pipeline(key, next_status, None, None)
+
+ # Crop even more error message to fit in slack notification
+ if (
+ isinstance(update_dict["error_info"]["error_exception"], str)
+ and len(update_dict["error_info"]["error_exception"]) > 1024
+ ):
+ logger.warning("Cropping error error_exception for slack")
+ update_dict["error_info"]["error_exception"] = update_dict["error_info"]["error_exception"][
+ -1023:
+ ]
# Send slack notification with original error exception, retry if exception has invalid format
try:
- slack_utils.send_slack_error_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\
- update_dict['error_info'] ,rec_process_series)
- except:
- update_dict['error_info']['error_exception'] = 'Error exception not accepted in slack, check log file for error'
- slack_utils.send_slack_error_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\
- update_dict['error_info'] ,rec_process_series)
-
+ slack_utils.send_slack_error_notification(
+ config.slack_webhooks_dict["automation_pipeline_error_notification"],
+ update_dict["error_info"],
+ rec_process_series,
+ )
+ except Exception:
+ update_dict["error_info"]["error_exception"] = (
+ "Error exception not accepted in slack, check log file for error"
+ )
+ slack_utils.send_slack_error_notification(
+ config.slack_webhooks_dict["automation_pipeline_error_notification"],
+ update_dict["error_info"],
+ rec_process_series,
+ )
except Exception as err:
- raise(err)
- print(traceback.format_exc())
+ raise (err)
+ logger.exception("Unhandled exception: %s", traceback.format_exc())
## Send notification error, update recording to error
time.sleep(2)
@@ -143,38 +171,36 @@ def transfer_request(rec_series, status_series):
'error_info': error info to be inserted if error occured }
"""
- status_update = config.status_update_idx['NO_CHANGE']
+ status_update = config.status_update_idx["NO_CHANGE"]
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- job_id = rec_series['job_id']
- raw_rel_path = rec_series['recording_process_pre_path']
- proc_rel_path = rec_series['recording_process_post_path']
- modality = rec_series['recording_modality']
+ job_id = rec_series["job_id"]
+ raw_rel_path = rec_series["recording_process_pre_path"]
+ proc_rel_path = rec_series["recording_process_post_path"]
+ modality = rec_series["recording_modality"]
# If tiger, we trigger globus transfer
- if rec_series['program_selection_params']['process_cluster'] == "tiger":
-
- if status_series['Key'] == 'RAW_FILE_TRANSFER_REQUEST':
-
- #status, task_id = scp_tr.call_scp_background(ip_address=rec_series['ip_address'], system_user=rec_series['system_user'],
- #recording_system_directory=rec_series['local_directory'], data_directory=data_directory.as_posix())
+ if rec_series["program_selection_params"]["process_cluster"] == "tiger":
+ if status_series["Key"] == "RAW_FILE_TRANSFER_REQUEST":
+ # status, task_id = scp_tr.call_scp_background(ip_address=rec_series['ip_address'], system_user=rec_series['system_user'],
+ # recording_system_directory=rec_series['local_directory'], data_directory=data_directory.as_posix())
transfer_request = ft.globus_transfer_to_tiger(job_id, raw_rel_path, modality)
- elif status_series['Key'] == 'PROC_FILE_TRANSFER_REQUEST':
- #ALS, which recording directory for processed file
+ elif status_series["Key"] == "PROC_FILE_TRANSFER_REQUEST":
+ # ALS, which recording directory for processed file
transfer_request = ft.globus_transfer_to_pni(job_id, proc_rel_path, modality)
- if transfer_request['status'] == config.system_process['SUCCESS']:
- status_update = config.status_update_idx['NEXT_STATUS']
- update_value_dict['value_update'] = transfer_request['task_id']
+ if transfer_request["status"] == config.system_process["SUCCESS"]:
+ status_update = config.status_update_idx["NEXT_STATUS"]
+ update_value_dict["value_update"] = transfer_request["task_id"]
else:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = transfer_request['error_info']
+ status_update = config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = transfer_request["error_info"]
# If not tiger let's go to next status
else:
- status_update = config.status_update_idx['NEXT_STATUS']
+ status_update = config.status_update_idx["NEXT_STATUS"]
return (status_update, update_value_dict)
@@ -195,27 +221,25 @@ def transfer_check(rec_series, status_series):
'error_info': error info to be inserted if error occured }
"""
- status_update = config.status_update_idx['NO_CHANGE']
+ status_update = config.status_update_idx["NO_CHANGE"]
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- id_task = rec_series[status_series['FunctionField']]
+ id_task = rec_series[status_series["FunctionField"]]
# If tiger, we trigger globus transfer
- if rec_series['program_selection_params']['process_cluster'] == "tiger":
-
+ if rec_series["program_selection_params"]["process_cluster"] == "tiger":
transfer_request = ft.request_globus_transfer_status(str(id_task))
- if transfer_request['status'] == config.system_process['COMPLETED']:
- status_update = config.status_update_idx['NEXT_STATUS']
- elif transfer_request['status'] == config.system_process['ERROR']:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'An error occured during globus transfer'
+ if transfer_request["status"] == config.system_process["COMPLETED"]:
+ status_update = config.status_update_idx["NEXT_STATUS"]
+ elif transfer_request["status"] == config.system_process["ERROR"]:
+ status_update = config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = "An error occured during globus transfer"
else:
- status_update = config.status_update_idx['NEXT_STATUS']
+ status_update = config.status_update_idx["NEXT_STATUS"]
return (status_update, update_value_dict)
-
@staticmethod
@recording_handler.exception_handler
def slurm_job_queue(rec_series, status_series):
@@ -233,81 +257,96 @@ def slurm_job_queue(rec_series, status_series):
'error_info': error info to be inserted if error occured }
"""
- status_update = config.status_update_idx['NO_CHANGE']
+ status_update = config.status_update_idx["NO_CHANGE"]
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- print(rec_series['program_selection_params'])
-
-
- #Create and transfer parameter file
- status = paramfilelib.generate_parameter_file(rec_series['job_id'], rec_series['params'], 'params', rec_series['program_selection_params'])
-
- print(status)
-
- #Create and transfer preparameter file
- if status == config.system_process['SUCCESS']:
- status = paramfilelib.generate_parameter_file(rec_series['job_id'], rec_series['preparams'], 'preparams', rec_series['program_selection_params'])
+ logger.debug("program_selection_params %s", rec_series["program_selection_params"])
+
+ # Create and transfer parameter file
+ status = paramfilelib.generate_parameter_file(
+ rec_series["job_id"],
+ rec_series["params"],
+ "params",
+ rec_series["program_selection_params"],
+ )
+
+ logger.debug("status %s", status)
+
+ # Create and transfer preparameter file
+ if status == config.system_process["SUCCESS"]:
+ status = paramfilelib.generate_parameter_file(
+ rec_series["job_id"],
+ rec_series["preparams"],
+ "preparams",
+ rec_series["program_selection_params"],
+ )
else:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'Error while generating/transfering parameter file'
+ status_update = config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = "Error while generating/transfering parameter file"
return (status_update, update_value_dict)
- #If electrophysiology, transfer chanmap as well
- if status == config.system_process['SUCCESS'] and rec_series['recording_modality'] == 'electrophysiology':
- recording_key = (recording_process.Processing.proj('recording_id', insertion_number='fragment_number') & rec_series['query_key']).fetch1()
+ # If electrophysiology, transfer chanmap as well
+ if status == config.system_process["SUCCESS"] and rec_series["recording_modality"] == "electrophysiology":
+ recording_key = (
+ recording_process.Processing.proj("recording_id", insertion_number="fragment_number")
+ & rec_series["query_key"]
+ ).fetch1()
del recording_key["job_id"]
-
- chanmap_filename = config.default_chanmap_filename % (rec_series['job_id'])
- chanmap_file_local_path = pathlib.Path(config.chanmap_files_filepath,chanmap_filename).as_posix()
- raw_directory_for_chanmap = rec_series['recording_process_pre_path']
+ chanmap_filename = config.default_chanmap_filename % (rec_series["job_id"])
+ chanmap_file_local_path = pathlib.Path(config.chanmap_files_filepath, chanmap_filename).as_posix()
+ rec_series["recording_process_pre_path"]
spikeglx_meta_filepath = ephys_pipeline.get_spikeglx_meta_filepath(recording_key)
# Chanmap mat file generation
- MetaToCoords(spikeglx_meta_filepath, 1, destFullPath =chanmap_file_local_path)
- status = paramfilelib.generate_chanmap_file(rec_series['job_id'], rec_series['program_selection_params'])
- if status != config.system_process['SUCCESS']:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'Error while generating/transfering channelmap file'
+ MetaToCoords(spikeglx_meta_filepath, 1, destFullPath=chanmap_file_local_path)
+ status = paramfilelib.generate_chanmap_file(rec_series["job_id"], rec_series["program_selection_params"])
+ if status != config.system_process["SUCCESS"]:
+ status_update = config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = "Error while generating/transfering channelmap file"
return (status_update, update_value_dict)
# Only queue if processing in tiger
- if rec_series['program_selection_params']['local_or_cluster'] == "cluster":
+ if rec_series["program_selection_params"]["local_or_cluster"] == "cluster":
+ logger.info("lets transfer slum file ..............xxxxxxxx............")
- print('lets transfer slum file ..............xxxxxxxx............')
-
- #Create and transfer slurm file
- if status == config.system_process['SUCCESS']:
- status, slurm_filepath = slurmlib.generate_slurm_file(rec_series['job_id'], rec_series['program_selection_params'])
+ # Create and transfer slurm file
+ if status == config.system_process["SUCCESS"]:
+ status, slurm_filepath = slurmlib.generate_slurm_file(
+ rec_series["job_id"], rec_series["program_selection_params"]
+ )
else:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'Error while generating/transfering slurm file'
+ status_update = config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = "Error while generating/transfering slurm file"
return (status_update, update_value_dict)
- #Queue slurm file
- if status == config.system_process['SUCCESS']:
-
- status, slurm_jobid, error_message = slurmlib.queue_slurm_file(rec_series['job_id'], rec_series['program_selection_params'],
- rec_series['recording_process_pre_path'], rec_series['recording_process_post_path'],
- rec_series['recording_modality'], slurm_filepath
+ # Queue slurm file
+ if status == config.system_process["SUCCESS"]:
+ status, slurm_jobid, error_message = slurmlib.queue_slurm_file(
+ rec_series["job_id"],
+ rec_series["program_selection_params"],
+ rec_series["recording_process_pre_path"],
+ rec_series["recording_process_post_path"],
+ rec_series["recording_modality"],
+ slurm_filepath,
)
else:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'Error while generating/transfering slurm file'
+ status_update = config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = "Error while generating/transfering slurm file"
return (status_update, update_value_dict)
- if status == config.system_process['SUCCESS']:
- status_update = config.status_update_idx['NEXT_STATUS']
- update_value_dict['value_update'] = slurm_jobid
+ if status == config.system_process["SUCCESS"]:
+ status_update = config.status_update_idx["NEXT_STATUS"]
+ update_value_dict["value_update"] = slurm_jobid
else:
- status_update = config.status_update_idx['ERROR_STATUS']
- update_value_dict['error_info']['error_message'] = 'Error while queuing job in SLURM'
+ status_update = config.status_update_idx["ERROR_STATUS"]
+ update_value_dict["error_info"]["error_message"] = "Error while queuing job in SLURM"
return (status_update, update_value_dict)
else:
- #Just check we succeded with parameter files
- if status == config.system_process['SUCCESS']:
- status_update = config.status_update_idx['NEXT_STATUS']
+ # Just check we succeded with parameter files
+ if status == config.system_process["SUCCESS"]:
+ status_update = config.status_update_idx["NEXT_STATUS"]
return (status_update, update_value_dict)
@@ -330,38 +369,50 @@ def slurm_job_check(rec_series, status_series):
update_value_dict = copy.deepcopy(config.default_update_value_dict)
# Only queue if processing in tiger
- if rec_series['program_selection_params']['local_or_cluster'] == "cluster":
- status_update = config.status_update_idx['NO_CHANGE']
+ if rec_series["program_selection_params"]["local_or_cluster"] == "cluster":
+ status_update = config.status_update_idx["NO_CHANGE"]
local_user = False
- program_selection_params = rec_series['program_selection_params']
- if program_selection_params['process_cluster'] == 'spock' and is_this_spock():
+ program_selection_params = rec_series["program_selection_params"]
+ if program_selection_params["process_cluster"] == "spock" and is_this_spock():
local_user = True
- ssh_user = ft.cluster_vars[program_selection_params['process_cluster']]['user']
- ssh_host = ft.cluster_vars[program_selection_params['process_cluster']]['hostname']
- slurm_jobid = str(rec_series['slurm_id'])
+ ssh_user = ft.cluster_vars[program_selection_params["process_cluster"]]["user"]
+ ssh_host = ft.cluster_vars[program_selection_params["process_cluster"]]["hostname"]
+ slurm_jobid = str(rec_series["slurm_id"])
status_update, message = slurmlib.check_slurm_job(ssh_user, ssh_host, slurm_jobid, local_user=local_user)
# Get message from slurm status check
- update_value_dict['error_info']['error_message'] = message
+ update_value_dict["error_info"]["error_message"] = message
# This updates the value
- #If job finished copy over output and/or error log
- if status_update == config.status_update_idx['NEXT_STATUS'] or status_update == config.status_update_idx['ERROR_STATUS']:
-
- ft.transfer_log_file(rec_series['job_id'], program_selection_params, ssh_host, log_type='ERROR')
- ft.transfer_log_file(rec_series['job_id'], program_selection_params, ssh_host, log_type='OUTPUT')
- error_log = ft.get_error_log_str(rec_series['job_id'])
+ # If job finished copy over output and/or error log
+ if (
+ status_update == config.status_update_idx["NEXT_STATUS"]
+ or status_update == config.status_update_idx["ERROR_STATUS"]
+ ):
+ ft.transfer_log_file(
+ rec_series["job_id"],
+ program_selection_params,
+ ssh_host,
+ log_type="ERROR",
+ )
+ ft.transfer_log_file(
+ rec_series["job_id"],
+ program_selection_params,
+ ssh_host,
+ log_type="OUTPUT",
+ )
+ error_log = ft.get_error_log_str(rec_series["job_id"])
# If the program errored, print the log.
# Previous method of capturing the error_log was insufficient since Kilosort wrote to stderr
- if status_update == config.status_update_idx['ERROR_STATUS']:
- update_value_dict['error_info']['error_message'] = 'An error occured in processing (check LOG)'
- update_value_dict['error_info']['error_exception'] = error_log
+ if status_update == config.status_update_idx["ERROR_STATUS"]:
+ update_value_dict["error_info"]["error_message"] = "An error occured in processing (check LOG)"
+ update_value_dict["error_info"]["error_exception"] = error_log
else:
- status_update = config.status_update_idx['NEXT_STATUS']
+ status_update = config.status_update_idx["NEXT_STATUS"]
return (status_update, update_value_dict)
@@ -383,89 +434,98 @@ def populate_element(rec_series, status_series):
"""
update_value_dict = copy.deepcopy(config.default_update_value_dict)
- if rec_series['recording_modality'] == 'electrophysiology':
+ if rec_series["recording_modality"] == "electrophysiology":
# Add extra function to create xyz picks file
- #chanmap_filename = config.default_chanmap_filename % (rec_series['job_id'])
- #chanmap_file_local_path = pathlib.Path(config.chanmap_files_filepath,chanmap_filename).as_posix()
- #ephys_utils.xyz_pick_file_creator.main_xyz_pick_file_function(rec_series['recording_id'], rec_series['fragment_number'], chanmap_file_local_path, rec_series['recording_process_post_path'])
+ # chanmap_filename = config.default_chanmap_filename % (rec_series['job_id'])
+ # chanmap_file_local_path = pathlib.Path(config.chanmap_files_filepath,chanmap_filename).as_posix()
+ # ephys_utils.xyz_pick_file_creator.main_xyz_pick_file_function(rec_series['recording_id'], rec_series['fragment_number'], chanmap_file_local_path, rec_series['recording_process_post_path'])
- status_update = ep.populate_element_data(rec_series['job_id'])
+ status_update = ep.populate_element_data(rec_series["job_id"])
- elif rec_series['recording_modality'] == 'imaging':
- status_update = ip.populate_element_data(rec_series['job_id'])
+ elif rec_series["recording_modality"] == "imaging":
+ status_update = ip.populate_element_data(rec_series["job_id"])
return (status_update, update_value_dict)
@staticmethod
def get_program_selection_params(modality):
- '''
+ """
get default processing variables (cluster, repository to process, etc) for a given modality
Return:
program_selection_params (pd.DataFrame): processing variables default dictionary
- '''
+ """
# Get df from config file
- this_modality_program_selection_params =\
- config.recording_modality_df.loc[config.recording_modality_df['recording_modality'] == modality, :]
+ this_modality_program_selection_params = config.recording_modality_df.loc[
+ config.recording_modality_df["recording_modality"] == modality, :
+ ]
# Pack all features in a dictionary
- this_modality_program_selection_params_dict = this_modality_program_selection_params.to_dict('records')
- this_modality_program_selection_params_dict
+ this_modality_program_selection_params_dict = this_modality_program_selection_params.to_dict("records")
+ logger.debug("program_selection_params_dict %s", this_modality_program_selection_params_dict)
# Get two columns, (recording_modality & "packed" program_selection_params)
- this_modality_program_selection_params = this_modality_program_selection_params.loc[:, 'recording_modality'].to_frame().copy()
- this_modality_program_selection_params['program_selection_params'] = this_modality_program_selection_params_dict
+ this_modality_program_selection_params = (
+ this_modality_program_selection_params.loc[:, "recording_modality"].to_frame().copy()
+ )
+ this_modality_program_selection_params["program_selection_params"] = this_modality_program_selection_params_dict
- this_modality_program_selection_params
+ logger.debug("program_selection_params %s", this_modality_program_selection_params)
return this_modality_program_selection_params
@staticmethod
def get_active_process_jobs(active=True):
- '''
+ """
get all process jobs that have to go through some action in the pipeline
Return:
df_process_jobs (pd.DataFrame): all jobs that are going to be processed in the pipeline
- '''
+ """
if active:
- status_query = 'status_processing_id > ' + str(config.JOB_STATUS_ERROR_ID)
- status_query += ' and status_processing_id < ' + str(config.JOB_STATUS_PROCESSED)
+ status_query = "status_processing_id > " + str(config.JOB_STATUS_ERROR_ID)
+ status_query += " and status_processing_id < " + str(config.JOB_STATUS_PROCESSED)
else:
- status_query = 'status_processing_id = ' + str(config.JOB_STATUS_ERROR_ID)
- status_query += ' or status_processing_id = ' + str(config.JOB_STATUS_PROCESSED)
-
- jobs_active = (recording.Recording.proj('recording_modality', 'location', 'recording_directory') * \
- recording_process.Processing & status_query)
+ status_query = "status_processing_id = " + str(config.JOB_STATUS_ERROR_ID)
+ status_query += " or status_processing_id = " + str(config.JOB_STATUS_PROCESSED)
+
+ jobs_active = (
+ recording.Recording.proj("recording_modality", "location", "recording_directory")
+ * recording_process.Processing
+ & status_query
+ )
df_process_jobs = pd.DataFrame(jobs_active.fetch(as_dict=True))
if df_process_jobs.shape[0] > 0:
key_list = dj_short.get_primary_key_fields(recording_process.Processing)
- df_process_jobs['query_key'] = df_process_jobs.loc[:, key_list].to_dict(orient='records')
+ df_process_jobs["query_key"] = df_process_jobs.loc[:, key_list].to_dict(orient="records")
# Get parameters for all modalities
- all_modalities = df_process_jobs['recording_modality'].unique()
+ all_modalities = df_process_jobs["recording_modality"].unique()
- all_mods_df = list()
+ all_mods_df = []
# Get process df for each modality
for this_modality in all_modalities:
-
- this_mod_df = df_process_jobs.loc[df_process_jobs['recording_modality'] == this_modality,:].copy()
- these_process_keys = this_mod_df['query_key'].to_list()
+ this_mod_df = df_process_jobs.loc[df_process_jobs["recording_modality"] == this_modality, :].copy()
+ these_process_keys = this_mod_df["query_key"].to_list()
this_mod_program_selection_params = RecProcessHandler.get_program_selection_params(this_modality)
- if this_modality == 'electrophysiology':
+ if this_modality == "electrophysiology":
params_df = RecProcessHandler.get_ephys_params_jobs(these_process_keys)
- if this_modality == 'imaging':
+ if this_modality == "imaging":
params_df = RecProcessHandler.get_imaging_params_jobs(these_process_keys)
- this_mod_df = this_mod_df.merge(params_df, on='job_id', how='left')
- this_mod_df = this_mod_df.merge(this_mod_program_selection_params, on='recording_modality', how='left')
+ this_mod_df = this_mod_df.merge(params_df, on="job_id", how="left")
+ this_mod_df = this_mod_df.merge(
+ this_mod_program_selection_params,
+ on="recording_modality",
+ how="left",
+ )
all_mods_df.append(this_mod_df)
- #Concatenate all process_jobs for each modality
+ # Concatenate all process_jobs for each modality
df_process_jobs = all_mods_df[0].copy()
for this_mod_df in all_mods_df[1:]:
@@ -473,38 +533,54 @@ def get_active_process_jobs(active=True):
df_process_jobs = df_process_jobs.reset_index(drop=True)
- print(df_process_jobs)
+ logger.debug("df_process_jobs %s", df_process_jobs)
return df_process_jobs
@staticmethod
def get_ephys_params_jobs(rec_process_keys):
- '''
+ """
get all parameters (precluster & cluster) for each of the recording process
Join precluster param list into a list
Args:
rec_process_keys (dict): key to find recording_process records
Return:
params_df (pd.DataFrame): recording_process & params df
- '''
+ """
# Get cluster param sets
- params_df = pd.DataFrame((ephys_pipeline.ephys_element.ClusteringParamSet.proj('params', 'clustering_method') * \
- recording_process.Processing.EphysParams.proj('paramset_idx') & rec_process_keys).fetch(as_dict=True))
- params_df = params_df.drop('paramset_idx', axis=1)
-
- #Insert clustering method in params itself (for BrainCogsEphysSorters)
- params_df['params'] = params_df.apply(lambda x: {**x['params'], **{'clustering_method':x['clustering_method']}},axis=1)
+ params_df = pd.DataFrame(
+ (
+ ephys_pipeline.ephys_element.ClusteringParamSet.proj("params", "clustering_method")
+ * recording_process.Processing.EphysParams.proj("paramset_idx")
+ & rec_process_keys
+ ).fetch(as_dict=True)
+ )
+ params_df = params_df.drop("paramset_idx", axis=1)
+
+ # Insert clustering method in params itself (for BrainCogsEphysSorters)
+ params_df["params"] = params_df.apply(
+ lambda x: {**x["params"], **{"clustering_method": x["clustering_method"]}},
+ axis=1,
+ )
# Get precluster param sets
- preparams_df = pd.DataFrame((ephys_pipeline.ephys_element.PreClusterParamSteps * \
- utility.smart_dj_join(ephys_pipeline.ephys_element.PreClusterParamSteps.Step, ephys_pipeline.ephys_element.PreClusterParamSet.proj('precluster_method', 'params')) *
- recording_process.Processing.EphysParams.proj('precluster_param_steps_id') & rec_process_keys).fetch(as_dict=True))
+ preparams_df = pd.DataFrame(
+ (
+ ephys_pipeline.ephys_element.PreClusterParamSteps
+ * utility.smart_dj_join(
+ ephys_pipeline.ephys_element.PreClusterParamSteps.Step,
+ ephys_pipeline.ephys_element.PreClusterParamSet.proj("precluster_method", "params"),
+ )
+ * recording_process.Processing.EphysParams.proj("precluster_param_steps_id")
+ & rec_process_keys
+ ).fetch(as_dict=True)
+ )
# Join precluster params for the same recording_process
- preparams_df['preparams'] = preparams_df.apply(lambda x : {x['precluster_method']: x['params']}, axis=1)
- preparams_df = preparams_df.sort_values(by=['job_id', 'step_number'])
- preparams_df = preparams_df[['job_id', 'preparams']].groupby("job_id").agg(lambda x: list(x))
+ preparams_df["preparams"] = preparams_df.apply(lambda x: {x["precluster_method"]: x["params"]}, axis=1)
+ preparams_df = preparams_df.sort_values(by=["job_id", "step_number"])
+ preparams_df = preparams_df[["job_id", "preparams"]].groupby("job_id").agg(lambda x: list(x))
preparams_df = preparams_df.reset_index()
params_df = params_df.merge(preparams_df)
@@ -513,47 +589,66 @@ def get_ephys_params_jobs(rec_process_keys):
@staticmethod
def get_imaging_params_jobs(rec_process_keys):
- '''
+ """
get all parameters (precluster & cluster) for each of the recording process
Join precluster param steps into a list
Args:
rec_process_keys (dict): key to find recording_process records
Return:
params_df (pd.DataFrame): recording_process & params df
- '''
-
- params_df = pd.DataFrame((imaging_pipeline.imaging_element.ProcessingParamSet.proj('params', 'processing_method') * \
- recording_process.Processing.ImagingParams.proj('paramset_idx') & rec_process_keys).fetch(as_dict=True))
- params_df = params_df.drop('paramset_idx', axis=1)
+ """
- #Insert processing_method in params itself (for BrainCogsImagingSorters)
- params_df['params'] = params_df.apply(lambda x: {**x['params'], **{'processing_method':x['processing_method']}},axis=1)
+ params_df = pd.DataFrame(
+ (
+ imaging_pipeline.imaging_element.ProcessingParamSet.proj("params", "processing_method")
+ * recording_process.Processing.ImagingParams.proj("paramset_idx")
+ & rec_process_keys
+ ).fetch(as_dict=True)
+ )
+ params_df = params_df.drop("paramset_idx", axis=1)
+
+ # Insert processing_method in params itself (for BrainCogsImagingSorters)
+ params_df["params"] = params_df.apply(
+ lambda x: {**x["params"], **{"processing_method": x["processing_method"]}},
+ axis=1,
+ )
# Get preprocess param sets
- preparams_df = pd.DataFrame((imaging_pipeline.imaging_element.PreprocessParamSteps * \
- utility.smart_dj_join(imaging_pipeline.imaging_element.PreprocessParamSteps.Step, imaging_pipeline.imaging_element.PreprocessParamSet.proj('preprocess_method', 'params')) *
- recording_process.Processing.ImagingParams.proj('preprocess_param_steps_id') & rec_process_keys).fetch(as_dict=True))
+ preparams_df = pd.DataFrame(
+ (
+ imaging_pipeline.imaging_element.PreprocessParamSteps
+ * utility.smart_dj_join(
+ imaging_pipeline.imaging_element.PreprocessParamSteps.Step,
+ imaging_pipeline.imaging_element.PreprocessParamSet.proj("preprocess_method", "params"),
+ )
+ * recording_process.Processing.ImagingParams.proj("preprocess_param_steps_id")
+ & rec_process_keys
+ ).fetch(as_dict=True)
+ )
- #If there is no preprocess steps for this jobs fill with empty values
+ # If there is no preprocess steps for this jobs fill with empty values
if preparams_df.shape[0] > 0:
-
# Join precluster params for the same recording_process
- preparams_df['preparams'] = preparams_df.apply(lambda x : {x['preprocess_method']: x['params']}, axis=1)
- preparams_df = preparams_df.sort_values(by=['job_id', 'step_number'])
- preparams_df = preparams_df[['job_id', 'preparams']].groupby("job_id").agg(lambda x: list(x))
+ preparams_df["preparams"] = preparams_df.apply(lambda x: {x["preprocess_method"]: x["params"]}, axis=1)
+ preparams_df = preparams_df.sort_values(by=["job_id", "step_number"])
+ preparams_df = preparams_df[["job_id", "preparams"]].groupby("job_id").agg(lambda x: list(x))
preparams_df = preparams_df.reset_index()
else:
- preparams_df = pd.DataFrame((imaging_pipeline.imaging_element.PreprocessParamSteps * \
- recording_process.Processing.ImagingParams.proj('preprocess_param_steps_id') & rec_process_keys).fetch(as_dict=True))
- preparams_df['preparams'] = None
-
- preparams_df = preparams_df[['job_id', 'preparams']]
+ preparams_df = pd.DataFrame(
+ (
+ imaging_pipeline.imaging_element.PreprocessParamSteps
+ * recording_process.Processing.ImagingParams.proj("preprocess_param_steps_id")
+ & rec_process_keys
+ ).fetch(as_dict=True)
+ )
+ preparams_df["preparams"] = None
+
+ preparams_df = preparams_df[["job_id", "preparams"]]
params_df = params_df.merge(preparams_df)
return params_df
-
@staticmethod
def update_status_pipeline(recording_process_key_dict, status, update_field=None, update_value=None):
"""
@@ -571,10 +666,9 @@ def update_status_pipeline(recording_process_key_dict, status, update_field=None
recording_process.Processing.update1(update_task_id_dict)
update_status_dict = recording_process_key_dict.copy()
- update_status_dict['status_processing_id'] = status
+ update_status_dict["status_processing_id"] = status
recording_process.Processing.update1(update_status_dict)
-
@staticmethod
def update_job_id_log(job_id, current_status, next_status, error_info_dict):
"""
@@ -586,26 +680,25 @@ def update_job_id_log(job_id, current_status, next_status, error_info_dict):
now = datetime.now()
date_time = now.strftime("%Y-%m-%d %H:%M:%S")
- print('error_info_dict', error_info_dict)
+ logger.debug("error_info_dict %s", error_info_dict)
- key = dict()
- key['job_id'] = job_id
- key['status_processing_id_old'] = current_status
- key['status_processing_id_new'] = next_status
- key['status_timestamp'] = date_time
+ key = {}
+ key["job_id"] = job_id
+ key["status_processing_id_old"] = current_status
+ key["status_processing_id_new"] = next_status
+ key["status_timestamp"] = date_time
- if error_info_dict['error_message'] is not None and len(error_info_dict['error_message']) >= 256:
- error_info_dict['error_message'] =error_info_dict['error_message'][:255]
+ if error_info_dict["error_message"] is not None and len(error_info_dict["error_message"]) >= 256:
+ error_info_dict["error_message"] = error_info_dict["error_message"][:255]
- if error_info_dict['error_exception'] is not None and len(error_info_dict['error_exception']) >= 4096:
- error_info_dict['error_exception'] =error_info_dict['error_exception'][:4095]
+ if error_info_dict["error_exception"] is not None and len(error_info_dict["error_exception"]) >= 4096:
+ error_info_dict["error_exception"] = error_info_dict["error_exception"][:4095]
- key['error_exception'] = error_info_dict['error_exception']
- key['error_message'] = error_info_dict['error_message']
+ key["error_exception"] = error_info_dict["error_exception"]
+ key["error_message"] = error_info_dict["error_message"]
recording_process.LogStatus.insert1(key)
-
@staticmethod
@recording_handler.exception_handler
def check_job_process_deletion():
@@ -617,107 +710,174 @@ def check_job_process_deletion():
# Get jobs with error status or finished
df_inactive_jobs = RecProcessHandler.get_active_process_jobs(active=False)
- #If no inactive job, the end
+ # If no inactive job, the end
if df_inactive_jobs.shape[0] == 0:
return
- jobs_ids = df_inactive_jobs['job_id'].to_frame().to_dict('records')
+ jobs_ids = df_inactive_jobs["job_id"].to_frame().to_dict("records")
- #Get latest time they changed status
+ # Get latest time they changed status
now = datetime.now()
- timestamp_jobs = pd.DataFrame((recording_process.Processing & jobs_ids).aggr(recording_process.LogStatus, max_date="max(status_timestamp)").fetch(as_dict=True))
- df_inactive_jobs = df_inactive_jobs.merge(timestamp_jobs, on='job_id')
+ timestamp_jobs = pd.DataFrame(
+ (recording_process.Processing & jobs_ids)
+ .aggr(recording_process.LogStatus, max_date="max(status_timestamp)")
+ .fetch(as_dict=True)
+ )
+ df_inactive_jobs = df_inactive_jobs.merge(timestamp_jobs, on="job_id")
# Get jobs that were processed in cluster
- df_inactive_jobs['local_or_cluster'] = df_inactive_jobs['program_selection_params'].map(lambda x: x['local_or_cluster'])
- df_inactive_jobs['process_cluster'] = df_inactive_jobs['program_selection_params'].map(lambda x: x['process_cluster'])
-
- #Update all local jobs
- df_not_cluster = df_inactive_jobs.loc[df_inactive_jobs['local_or_cluster'] == 'local'].reset_index(drop=True)
+ df_inactive_jobs["local_or_cluster"] = df_inactive_jobs["program_selection_params"].map(
+ lambda x: x["local_or_cluster"]
+ )
+ df_inactive_jobs["process_cluster"] = df_inactive_jobs["program_selection_params"].map(
+ lambda x: x["process_cluster"]
+ )
+
+ # Update all local jobs
+ df_not_cluster = df_inactive_jobs.loc[df_inactive_jobs["local_or_cluster"] == "local"].reset_index(drop=True)
if df_not_cluster.shape[0] > 0:
# Update local jobs post-processed jobs
- df_not_cluster_post_processed = df_not_cluster.loc[df_not_cluster['status_processing_id'] == config.JOB_STATUS_PROCESSED, :]
+ df_not_cluster_post_processed = df_not_cluster.loc[
+ df_not_cluster["status_processing_id"] == config.JOB_STATUS_PROCESSED, :
+ ]
df_not_cluster_post_processed = df_not_cluster_post_processed.reset_index(drop=True)
for i in range(df_not_cluster_post_processed.shape[0]):
- RecProcessHandler.update_status_pipeline(df_not_cluster_post_processed.loc[i, 'query_key'], config.JOB_STATUS_POST_PROCESSED)
- RecProcessHandler.update_job_id_log(df_not_cluster_post_processed.loc[i, 'job_id'], config.JOB_STATUS_PROCESSED, config.JOB_STATUS_POST_PROCESSED, update_value_dict['error_info'])
+ RecProcessHandler.update_status_pipeline(
+ df_not_cluster_post_processed.loc[i, "query_key"],
+ config.JOB_STATUS_POST_PROCESSED,
+ )
+ RecProcessHandler.update_job_id_log(
+ df_not_cluster_post_processed.loc[i, "job_id"],
+ config.JOB_STATUS_PROCESSED,
+ config.JOB_STATUS_POST_PROCESSED,
+ update_value_dict["error_info"],
+ )
# Update also local jobs post-error jobs
- df_not_cluster_post_error = df_not_cluster.loc[df_not_cluster['status_processing_id'] == config.JOB_STATUS_ERROR_ID, :]
+ df_not_cluster_post_error = df_not_cluster.loc[
+ df_not_cluster["status_processing_id"] == config.JOB_STATUS_ERROR_ID, :
+ ]
df_not_cluster_post_error = df_not_cluster_post_error.reset_index(drop=True)
for i in range(df_not_cluster_post_error.shape[0]):
- RecProcessHandler.update_status_pipeline(df_not_cluster_post_error.loc[i, 'query_key'], config.JOB_STATUS_ERROR_DELETED)
- RecProcessHandler.update_job_id_log(df_not_cluster_post_error.loc[i, 'job_id'], config.JOB_STATUS_ERROR_ID, config.JOB_STATUS_ERROR_DELETED, update_value_dict['error_info'])
+ RecProcessHandler.update_status_pipeline(
+ df_not_cluster_post_error.loc[i, "query_key"],
+ config.JOB_STATUS_ERROR_DELETED,
+ )
+ RecProcessHandler.update_job_id_log(
+ df_not_cluster_post_error.loc[i, "job_id"],
+ config.JOB_STATUS_ERROR_ID,
+ config.JOB_STATUS_ERROR_DELETED,
+ update_value_dict["error_info"],
+ )
- #If no inactive job, after locals the end
- df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs['local_or_cluster'] == 'cluster'].reset_index(drop=True)
+ # If no inactive job, after locals the end
+ df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs["local_or_cluster"] == "cluster"].reset_index(
+ drop=True
+ )
if df_inactive_jobs.shape[0] == 0:
return
# Get jobs to delete (finished or with error >= 7 days)
- df_inactive_jobs['days_from_last_update'] = (now - df_inactive_jobs['max_date']).dt.days
- df_inactive_jobs['ready_delete'] = 1
- df_inactive_jobs.loc[(df_inactive_jobs['status_processing_id'] == config.JOB_STATUS_ERROR_ID) & (df_inactive_jobs['days_from_last_update'] < 7), 'ready_delete'] = 0
- df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs['ready_delete'] == 1,:]
-
- #If no inactive job, after ready delete data return now
+ df_inactive_jobs["days_from_last_update"] = (now - df_inactive_jobs["max_date"]).dt.days
+ df_inactive_jobs["ready_delete"] = 1
+ df_inactive_jobs.loc[
+ (df_inactive_jobs["status_processing_id"] == config.JOB_STATUS_ERROR_ID)
+ & (df_inactive_jobs["days_from_last_update"] < 7),
+ "ready_delete",
+ ] = 0
+ df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs["ready_delete"] == 1, :]
+
+ # If no inactive job, after ready delete data return now
if df_inactive_jobs.shape[0] == 0:
return
# Check if raw directories are present in cluster
- df_inactive_jobs['raw_dir'] = df_inactive_jobs.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_pre_path'],\
- x['process_cluster'], x['recording_modality'], type_dir='raw'), axis=1)
- df_inactive_jobs['raw_dir'] = df_inactive_jobs['raw_dir'].astype(str)
- df_inactive_jobs['raw_dir_deleted'] = 0
- df_inactive_jobs.loc[df_inactive_jobs['raw_dir'] == "0", 'raw_dir_deleted'] = 1
+ df_inactive_jobs["raw_dir"] = df_inactive_jobs.apply(
+ lambda x: ft.check_directory_exists_cluster(
+ x["recording_process_pre_path"],
+ x["process_cluster"],
+ x["recording_modality"],
+ type_dir="raw",
+ ),
+ axis=1,
+ )
+ df_inactive_jobs["raw_dir"] = df_inactive_jobs["raw_dir"].astype(str)
+ df_inactive_jobs["raw_dir_deleted"] = 0
+ df_inactive_jobs.loc[df_inactive_jobs["raw_dir"] == "0", "raw_dir_deleted"] = 1
# Check if processed directories are present in cluster
- df_inactive_jobs['processed_dir'] = df_inactive_jobs.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_post_path'],\
- x['process_cluster'], x['recording_modality'], type_dir='processed'), axis=1)
- df_inactive_jobs['processed_dir'] = df_inactive_jobs['processed_dir'].astype(str)
- df_inactive_jobs['processed_dir_deleted'] = 0
- df_inactive_jobs.loc[df_inactive_jobs['processed_dir'] == "0", 'processed_dir_deleted'] = 1
+ df_inactive_jobs["processed_dir"] = df_inactive_jobs.apply(
+ lambda x: ft.check_directory_exists_cluster(
+ x["recording_process_post_path"],
+ x["process_cluster"],
+ x["recording_modality"],
+ type_dir="processed",
+ ),
+ axis=1,
+ )
+ df_inactive_jobs["processed_dir"] = df_inactive_jobs["processed_dir"].astype(str)
+ df_inactive_jobs["processed_dir_deleted"] = 0
+ df_inactive_jobs.loc[df_inactive_jobs["processed_dir"] == "0", "processed_dir_deleted"] = 1
df_inactive_jobs = df_inactive_jobs.reset_index(drop=True)
- #Delete raw directories tiger
+ # Delete raw directories tiger
for i in range(df_inactive_jobs.shape[0]):
- modality = df_inactive_jobs.loc[i, 'recording_modality']
- dir_delete = df_inactive_jobs.loc[i, 'recording_process_pre_path']
- to_delete = df_inactive_jobs.loc[i, 'raw_dir']
+ modality = df_inactive_jobs.loc[i, "recording_modality"]
+ dir_delete = df_inactive_jobs.loc[i, "recording_process_pre_path"]
+ to_delete = df_inactive_jobs.loc[i, "raw_dir"]
if str(to_delete) != "0":
status = ft.delete_directory_tiger_globus(modality, dir_delete)
- if status == config.system_process['SUCCESS']:
- df_inactive_jobs.loc[i, 'raw_dir_deleted'] = 1
+ if status == config.system_process["SUCCESS"]:
+ df_inactive_jobs.loc[i, "raw_dir_deleted"] = 1
- #Delete processed directories tiger
+ # Delete processed directories tiger
for i in range(df_inactive_jobs.shape[0]):
- cluster = df_inactive_jobs.loc[i, 'process_cluster']
- dir_delete = df_inactive_jobs.loc[i, 'processed_dir']
+ cluster = df_inactive_jobs.loc[i, "process_cluster"]
+ dir_delete = df_inactive_jobs.loc[i, "processed_dir"]
if str(dir_delete) != "0":
status = ft.delete_directory_cluster(dir_delete, cluster)
- if status == config.system_process['SUCCESS']:
- df_inactive_jobs.loc[i, 'processed_dir_deleted'] = 1
+ if status == config.system_process["SUCCESS"]:
+ df_inactive_jobs.loc[i, "processed_dir_deleted"] = 1
- #Delete empty directories in clusters
- clusters = df_inactive_jobs['process_cluster'].unique().tolist()
+ # Delete empty directories in clusters
+ clusters = df_inactive_jobs["process_cluster"].unique().tolist()
for this_cluster in clusters:
ft.delete_empty_data_directory_cluster(this_cluster, type="raw")
ft.delete_empty_data_directory_cluster(this_cluster, type="processed")
-
# Update status for post-processed jobs
- df_post_processed = df_inactive_jobs.loc[(df_inactive_jobs['raw_dir_deleted'] == 1) & (df_inactive_jobs['processed_dir_deleted'] == 1) & (df_inactive_jobs['status_processing_id'] == config.JOB_STATUS_PROCESSED)]
+ df_post_processed = df_inactive_jobs.loc[
+ (df_inactive_jobs["raw_dir_deleted"] == 1)
+ & (df_inactive_jobs["processed_dir_deleted"] == 1)
+ & (df_inactive_jobs["status_processing_id"] == config.JOB_STATUS_PROCESSED)
+ ]
df_post_processed = df_post_processed.reset_index(drop=True)
for i in range(df_post_processed.shape[0]):
- RecProcessHandler.update_status_pipeline(df_post_processed.loc[i, 'query_key'], config.JOB_STATUS_POST_PROCESSED)
- RecProcessHandler.update_job_id_log(df_post_processed.loc[i, 'job_id'], config.JOB_STATUS_PROCESSED, config.JOB_STATUS_POST_PROCESSED, update_value_dict['error_info'])
+ RecProcessHandler.update_status_pipeline(
+ df_post_processed.loc[i, "query_key"], config.JOB_STATUS_POST_PROCESSED
+ )
+ RecProcessHandler.update_job_id_log(
+ df_post_processed.loc[i, "job_id"],
+ config.JOB_STATUS_PROCESSED,
+ config.JOB_STATUS_POST_PROCESSED,
+ update_value_dict["error_info"],
+ )
# Update status for post-error jobs
- df_post_error = df_inactive_jobs.loc[(df_inactive_jobs['raw_dir_deleted'] == 1) & (df_inactive_jobs['processed_dir_deleted'] == 1) & (df_inactive_jobs['status_processing_id'] == config.JOB_STATUS_ERROR_ID)]
+ df_post_error = df_inactive_jobs.loc[
+ (df_inactive_jobs["raw_dir_deleted"] == 1)
+ & (df_inactive_jobs["processed_dir_deleted"] == 1)
+ & (df_inactive_jobs["status_processing_id"] == config.JOB_STATUS_ERROR_ID)
+ ]
df_post_error = df_post_error.reset_index(drop=True)
for i in range(df_post_error.shape[0]):
- RecProcessHandler.update_status_pipeline(df_post_error.loc[i, 'query_key'], config.JOB_STATUS_ERROR_DELETED)
- RecProcessHandler.update_job_id_log(df_post_error.loc[i, 'job_id'], config.JOB_STATUS_ERROR_ID, config.JOB_STATUS_ERROR_DELETED, update_value_dict['error_info'])
+ RecProcessHandler.update_status_pipeline(df_post_error.loc[i, "query_key"], config.JOB_STATUS_ERROR_DELETED)
+ RecProcessHandler.update_job_id_log(
+ df_post_error.loc[i, "job_id"],
+ config.JOB_STATUS_ERROR_ID,
+ config.JOB_STATUS_ERROR_DELETED,
+ update_value_dict["error_info"],
+ )
return
@@ -736,4 +896,3 @@ def filter_session_status(df_rec_process, status):
df_rec_process_status = df_rec_process.loc[df_sessions['status_processing_id'] == status, :]
df_rec_process_status = df_rec_process_status.reset_index(drop=True)
'''
-
diff --git a/u19_pipeline/automatic_job/slurm_creator.py b/u19_pipeline/automatic_job/slurm_creator.py
index d1d91c1c..290048fa 100644
--- a/u19_pipeline/automatic_job/slurm_creator.py
+++ b/u19_pipeline/automatic_job/slurm_creator.py
@@ -1,115 +1,131 @@
-
-
import copy
-import os
-import subprocess
import pathlib
-import json
import re
+import subprocess
+
import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft
-from u19_pipeline.utility import create_str_from_dict, is_this_spock
import u19_pipeline.automatic_job.params_config as config
+from u19_pipeline.utility import is_this_spock
from u19_pipeline.utils.file_utils import write_file
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
# Functions to create slurm jobs
-slurms_filepath = 'u19_pipeline/automatic_job/SlurmFiles'
-default_slurm_filename = 'slurm_real.slurm'
+slurms_filepath = "u19_pipeline/automatic_job/SlurmFiles"
+default_slurm_filename = "slurm_real.slurm"
default_process_script_path = "scripts/automate_imaging_element.py"
-default_preprocessing_tool = 'kilosort2'
-default_matlab_ver = 'R2020b'
+default_preprocessing_tool = "kilosort2"
+default_matlab_ver = "R2020b"
+
def generate_slurm_file(job_id, program_selection_params):
- '''
+ """
Generate and send slurm file to be queued in processing cluster
- '''
+ """
- #Get all associated directories given the selected processing cluster
- cluster_vars = ft.get_cluster_vars(program_selection_params['process_cluster'])
+ # Get all associated directories given the selected processing cluster
+ cluster_vars = ft.get_cluster_vars(program_selection_params["process_cluster"])
# Start with default values
- slurm_dict = copy.deepcopy(cluster_vars['slurm_default'])
- label_rec_process = 'job_id_'+str(job_id)
- slurm_dict['job-name'] = label_rec_process
+ slurm_dict = copy.deepcopy(cluster_vars["slurm_default"])
+ label_rec_process = "job_id_" + str(job_id)
+ slurm_dict["job-name"] = label_rec_process
- slurm_dict['output'] = str(pathlib.Path(cluster_vars['log_files_dir'],label_rec_process+ '.log'))
- slurm_dict['error'] = str(pathlib.Path(cluster_vars['error_files_dir'],label_rec_process+ '.log'))
+ slurm_dict["output"] = str(pathlib.Path(cluster_vars["log_files_dir"], label_rec_process + ".log"))
+ slurm_dict["error"] = str(pathlib.Path(cluster_vars["error_files_dir"], label_rec_process + ".log"))
- print('slurm_dict', slurm_dict)
+ logger.debug("slurm_dict %s", slurm_dict)
- if program_selection_params['process_cluster'] == 'spock':
+ if program_selection_params["process_cluster"] == "spock":
slurm_text = generate_slurm_spock(slurm_dict)
else:
slurm_text = generate_slurm_tiger(slurm_dict)
slurm_file_name = default_slurm_filename
- slurm_file_local_path = str(pathlib.Path(slurms_filepath,slurm_file_name))
+ slurm_file_local_path = str(pathlib.Path(slurms_filepath, slurm_file_name))
- print(slurm_file_local_path)
- print(cluster_vars['slurm_files_dir'])
- print(slurm_file_name)
+ logger.debug("slurm_file_local_path %s", slurm_file_local_path)
+ logger.debug("slurm_files_dir %s", cluster_vars["slurm_files_dir"])
+ logger.debug("slurm_file_name %s", slurm_file_name)
write_file(slurm_file_local_path, slurm_text)
- if program_selection_params['process_cluster'] == 'spock' and is_this_spock():
- status = config.system_process['SUCCESS']
+ if program_selection_params["process_cluster"] == "spock" and is_this_spock():
+ status = config.system_process["SUCCESS"]
slurm_destination = slurm_file_local_path
else:
- slurm_destination = pathlib.Path(cluster_vars['slurm_files_dir'], slurm_file_name).as_posix()
+ slurm_destination = pathlib.Path(cluster_vars["slurm_files_dir"], slurm_file_name).as_posix()
status = transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars)
- print(status)
- print(slurm_destination)
- print(cluster_vars)
+ logger.debug("status %s", status)
+ logger.debug("slurm_destination %s", slurm_destination)
+ logger.debug("cluster_vars %s", cluster_vars)
return status, slurm_destination
-def queue_slurm_file(job_id, program_selection_params, raw_directory, proc_directory, modality, slurm_location):
+def queue_slurm_file(
+ job_id,
+ program_selection_params,
+ raw_directory,
+ proc_directory,
+ modality,
+ slurm_location,
+):
id_slurm_job = -1
job_id = str(job_id)
- #Get all associated variables given the selected processing cluster
- cluster_vars = ft.get_cluster_vars(program_selection_params['process_cluster'])
-
- print('queue_slurm_file **********************************')
-
-
- processing_repository = program_selection_params['process_repository']
- repository_dir = pathlib.Path(cluster_vars[modality+'_process_dir'],processing_repository).as_posix()
-
- command = ['ssh', cluster_vars['user']+"@"+cluster_vars['hostname'], 'sbatch',
- "--export=recording_process_id="+job_id+
- ",raw_data_directory='"+raw_directory+
- "',processed_data_directory='"+proc_directory+
- "',repository_dir='"+repository_dir+
- "',process_script_path='"+program_selection_params['process_script']+"'"
- , slurm_location
+ # Get all associated variables given the selected processing cluster
+ cluster_vars = ft.get_cluster_vars(program_selection_params["process_cluster"])
+
+ logger.debug("queue_slurm_file **********************************")
+
+ processing_repository = program_selection_params["process_repository"]
+ repository_dir = pathlib.Path(cluster_vars[modality + "_process_dir"], processing_repository).as_posix()
+
+ command = [
+ "ssh",
+ cluster_vars["user"] + "@" + cluster_vars["hostname"],
+ "sbatch",
+ "--export=recording_process_id="
+ + job_id
+ + ",raw_data_directory='"
+ + raw_directory
+ + "',processed_data_directory='"
+ + proc_directory
+ + "',repository_dir='"
+ + repository_dir
+ + "',process_script_path='"
+ + program_selection_params["process_script"]
+ + "'",
+ slurm_location,
]
- if program_selection_params['process_cluster'] == 'spock' and is_this_spock():
+ if program_selection_params["process_cluster"] == "spock" and is_this_spock():
command = command[2:]
- print(command)
+ logger.debug("command %s", command)
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- #p = os.popen(command_new).read()
+ # p = os.popen(command_new).read()
p.wait()
stdout, stderr = p.communicate()
- print(stdout)
- print(stderr)
+ logger.debug("stdout %s", stdout)
+ logger.debug("stderr %s", stderr)
- if p.returncode == config.system_process['SUCCESS']:
- error_message = ''
- batch_job_sentence = stdout.decode('UTF-8')
- print('batch_job_sentence', batch_job_sentence)
- id_slurm_job = batch_job_sentence.replace("Submitted batch job ","")
- id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job)
+ if p.returncode == config.system_process["SUCCESS"]:
+ error_message = ""
+ batch_job_sentence = stdout.decode("UTF-8")
+ logger.debug("batch_job_sentence %s", batch_job_sentence)
+ id_slurm_job = batch_job_sentence.replace("Submitted batch job ", "")
+ id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job)
else:
- error_message = stderr.decode('UTF-8')
+ error_message = stderr.decode("UTF-8")
return p.returncode, id_slurm_job, error_message
@@ -117,41 +133,48 @@ def queue_slurm_file(job_id, program_selection_params, raw_directory, proc_direc
def check_slurm_job(ssh_user, host, jobid, local_user=False):
if local_user:
- command = ['sacct', '--job', jobid, '--format=state']
+ command = ["sacct", "--job", jobid, "--format=state"]
else:
- command = ['ssh', ssh_user+'@'+host, 'sacct', '--job', jobid, '--format=state']
+ command = [
+ "ssh",
+ ssh_user + "@" + host,
+ "sacct",
+ "--job",
+ jobid,
+ "--format=state",
+ ]
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
stdout, stderr = p.communicate()
- stdout = stdout.decode('UTF-8')
+ stdout = stdout.decode("UTF-8")
- print('p.returncode !!!!!!!!!!!!', p.returncode)
- print("config.system_process['SUCCESS']", config.system_process['SUCCESS'])
+ logger.debug("p.returncode %s", p.returncode)
+ logger.debug("config.system_process['SUCCESS'] %s", config.system_process["SUCCESS"])
- if p.returncode == config.system_process['SUCCESS']:
+ if p.returncode == config.system_process["SUCCESS"]:
state_slurm_job = stdout.split("\n")[2].strip()
- state_pipeline = config.slurm_states[state_slurm_job]['pipeline_status']
- error_message = config.slurm_states[state_slurm_job]['message']
+ state_pipeline = config.slurm_states[state_slurm_job]["pipeline_status"]
+ error_message = config.slurm_states[state_slurm_job]["message"]
- print('state_pipeline ....', state_pipeline)
- print('error_message', error_message)
+ logger.debug("state_pipeline %s", state_pipeline)
+ logger.debug("error_message %s", error_message)
else:
- state_pipeline = config.status_update_idx['ERROR_STATUS']
- error_message = 'Failed to retrieve slurm job status'
+ state_pipeline = config.status_update_idx["ERROR_STATUS"]
+ error_message = "Failed to retrieve slurm job status"
return state_pipeline, error_message
def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars):
- '''
+ """
Create scp command from cluster directories and local slurm file
- '''
+ """
- user_host = cluster_vars['user']+'@'+cluster_vars['hostname']
- slurm_destination = user_host+':'+slurm_destination
+ user_host = cluster_vars["user"] + "@" + cluster_vars["hostname"]
+ slurm_destination = user_host + ":" + slurm_destination
status = ft.scp_file_transfer(slurm_file_local_path, slurm_destination)
return status
@@ -159,23 +182,23 @@ def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars):
def create_slurm_params_file(slurm_dict):
- text_dict = ''
- for slurm_param in slurm_dict.keys():
-
+ text_dict = ""
+ for slurm_param in slurm_dict:
if isinstance(slurm_dict[slurm_param], list):
for list_param in slurm_dict[slurm_param]:
- text_dict += '#SBATCH --' + str(slurm_param) + '=' + str(list_param) + '\n'
+ text_dict += "#SBATCH --" + str(slurm_param) + "=" + str(list_param) + "\n"
else:
- text_dict += '#SBATCH --' + str(slurm_param) + '=' + str(slurm_dict[slurm_param]) + '\n'
+ text_dict += "#SBATCH --" + str(slurm_param) + "=" + str(slurm_dict[slurm_param]) + "\n"
return text_dict
+
def generate_slurm_spock(slurm_dict):
- slurm_text = '#!/bin/bash\n'
- #slurm_text += module_defininition_text()
+ slurm_text = "#!/bin/bash\n"
+ # slurm_text += module_defininition_text()
slurm_text += create_slurm_params_file(slurm_dict)
- slurm_text += '''
+ slurm_text += """
echo "SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}"
echo "RECORDING_PROCESS_ID: ${recording_process_id}"
@@ -193,13 +216,14 @@ def generate_slurm_spock(slurm_dict):
cd ${repository_dir}
python -u ${process_script_path}
#python ${process_script_path} ${recording_process_id}
- '''
+ """
return slurm_text
+
def module_defininition_text():
- return '''
+ return """
module ()
{
local _mlredir=1;
@@ -226,14 +250,15 @@ def module_defininition_text():
_module_raw "$@" 2>&1;
fi
}
- '''
+ """
+
def generate_slurm_spockmk2_ephys(slurm_dict):
- slurm_text = '#!/bin/bash\n'
- slurm_text += 'source ~/.bashrc'
+ slurm_text = "#!/bin/bash\n"
+ slurm_text += "source ~/.bashrc"
slurm_text += create_slurm_params_file(slurm_dict)
- slurm_text += '''
+ slurm_text += """
echo "SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}"
echo "RECORDING_PROCESS_ID: ${recording_process_id}"
@@ -250,16 +275,16 @@ def generate_slurm_spockmk2_ephys(slurm_dict):
cd ${repository_dir}
python -u ${process_script_path}
#python ${process_script_path} ${recording_process_id}
- '''
+ """
return slurm_text
def generate_slurm_tiger(slurm_dict):
- slurm_text = '#!/bin/bash\n'
+ slurm_text = "#!/bin/bash\n"
slurm_text += create_slurm_params_file(slurm_dict)
- slurm_text += '''
+ slurm_text += """
echo "SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}"
echo "RECORDING_PROCESS_ID: ${recording_process_id}"
@@ -275,16 +300,16 @@ def generate_slurm_tiger(slurm_dict):
cd ${repository_dir}
python -u ${process_script_path}
- '''
+ """
return slurm_text
def generate_slurm_dlc(slurm_dict):
- slurm_text = '#!/bin/bash\n'
+ slurm_text = "#!/bin/bash\n"
slurm_text += create_slurm_params_file(slurm_dict)
- slurm_text += '''
+ slurm_text += """
echo "SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}"
echo "RAW_DATA_DIRECTORY: ${raw_data_directory}"
@@ -297,16 +322,16 @@ def generate_slurm_dlc(slurm_dict):
conda activate /usr/people/alvaros/.conda/envs/u19_datajoint_py39_env
python -u ${process_script_path} ${raw_data_directory} ${model_path} ${processed_data_directory}
- '''
+ """
return slurm_text
def generate_slurm_dlc2(slurm_dict):
- slurm_text = '#!/bin/bash\n'
+ slurm_text = "#!/bin/bash\n"
slurm_text += create_slurm_params_file(slurm_dict)
- slurm_text += '''
+ slurm_text += """
echo "SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}"
echo "RAW_DATA_DIRECTORY: ${raw_data_directory}"
@@ -319,7 +344,6 @@ def generate_slurm_dlc2(slurm_dict):
conda activate /home/alvaros/.conda/envs/u19_datajoint_py39_env
python -u ${process_script_path} ${raw_data_directory} ${model_path} ${processed_data_directory}
- '''
+ """
return slurm_text
-
diff --git a/u19_pipeline/automatic_job/tigress2pni.py b/u19_pipeline/automatic_job/tigress2pni.py
index 2a9b8300..bbf0acb0 100755
--- a/u19_pipeline/automatic_job/tigress2pni.py
+++ b/u19_pipeline/automatic_job/tigress2pni.py
@@ -13,35 +13,43 @@
previous transfers, so if this script is run twice in quick succession,
the second run won't queue a duplicate transfer."""
+import contextlib
import json
-import sys
import os
-import six
+import sys
-from globus_sdk import (NativeAppAuthClient, TransferClient,
- RefreshTokenAuthorizer, TransferData)
+import six
+from fair_research_login import NativeClient
+from globus_sdk import (
+ NativeAppAuthClient,
+ RefreshTokenAuthorizer,
+ TransferClient,
+ TransferData,
+)
from globus_sdk.exc import GlobusAPIError
from globus_sdk.services.transfer.errors import TransferAPIError
-from fair_research_login import NativeClient
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
# Princeton TIGRESS
-SOURCE_ENDPOINT = 'a9df83d2-42f0-11e6-80cf-22000b1701d1'
+SOURCE_ENDPOINT = "a9df83d2-42f0-11e6-80cf-22000b1701d1"
# SOURCE_ENDPOINT = 'ef3a4e74-e742-11ec-9912-3b4cfda38030'
-DESTINATION_PATH = '/mnt/cup/braininit/Shared/TestGlobusTransfer'
-SOURCE_PATH = '/Data/Processed/imaging/'
-SOURCE_PATH = '/tiger/scratch/gpfs/BRAINCOGS/Data/Processed/imaging/'
+DESTINATION_PATH = "/mnt/cup/braininit/Shared/TestGlobusTransfer"
+SOURCE_PATH = "/Data/Processed/imaging/"
+SOURCE_PATH = "/tiger/scratch/gpfs/BRAINCOGS/Data/Processed/imaging/"
# Princeton PNI
-DESTINATION_ENDPOINT = '6ce834d6-ff8a-11e6-bad1-22000b9a448b'
+DESTINATION_ENDPOINT = "6ce834d6-ff8a-11e6-bad1-22000b9a448b"
# Copy data off of the endpoint share
# SOURCE_PATH = '/tigress/bdsinger/data/'
# Destination Path -- The directory will be created if it doesn't exist
# DESTINATION_PATH = '/mnt/cup/people/bdsinger/data/'
-TRANSFER_LABEL = 'FolderSyncTest'
+TRANSFER_LABEL = "FolderSyncTest"
# You will need to register a *Native App* at https://developers.globus.org/
# Your app should include the following:
@@ -50,31 +58,30 @@
# - "Native App" should be checked
# For more information:
# https://docs.globus.org/api/auth/developer-guide/#register-app
-CLIENT_ID = '71286d4e-778d-45f7-9a30-813305f9e1f3'
-DATA_FILE = 'transfer-data.json'
-REDIRECT_URI = 'https://auth.globus.org/v2/web/auth-code'
-SCOPES = ('openid email profile '
- 'urn:globus:auth:scope:transfer.api.globus.org:all')
+CLIENT_ID = "71286d4e-778d-45f7-9a30-813305f9e1f3"
+DATA_FILE = "transfer-data.json"
+REDIRECT_URI = "https://auth.globus.org/v2/web/auth-code"
+SCOPES = "openid email profile urn:globus:auth:scope:transfer.api.globus.org:all"
-APP_NAME = 'FolderSyncTest'
+APP_NAME = "FolderSyncTest"
# ONLY run new tasks if there was a previous task and it exited with one of the
# following statuses. This is ignored if there was no previous task.
# The previous task is queried from the DATA_FILE
-PREVIOUS_TASK_RUN_CASES = ['SUCCEEDED', 'FAILED']
+PREVIOUS_TASK_RUN_CASES = ["SUCCEEDED", "FAILED"]
# Create the destination folder if it does not already exist
CREATE_DESTINATION_FOLDER = True
-get_input = getattr(__builtins__, 'raw_input', input)
+get_input = getattr(__builtins__, "raw_input", input)
def load_data_from_file(filepath):
"""Load a set of saved tokens."""
if not os.path.exists(filepath):
return []
- with open(filepath, 'r') as f:
+ with open(filepath) as f:
tokens = json.load(f)
return tokens
@@ -84,32 +91,31 @@ def save_data_to_file(filepath, key, data):
"""Save data to a file"""
try:
store = load_data_from_file(filepath)
- except:
+ except Exception:
store = {}
if len(store) > 0:
store[key] = data
- with open(filepath, 'w') as f:
+ with open(filepath, "w") as f:
json.dump(store, f)
def setup_transfer_client(transfer_tokens):
authorizer = RefreshTokenAuthorizer(
- transfer_tokens['refresh_token'],
+ transfer_tokens["refresh_token"],
NativeAppAuthClient(client_id=CLIENT_ID),
- access_token=transfer_tokens['access_token'],
- expires_at=transfer_tokens['expires_at_seconds'])
+ access_token=transfer_tokens["access_token"],
+ expires_at=transfer_tokens["expires_at_seconds"],
+ )
transfer_client = TransferClient(authorizer=authorizer)
try:
transfer_client.endpoint_autoactivate(SOURCE_ENDPOINT)
- r = transfer_client.endpoint_autoactivate(DESTINATION_ENDPOINT)
+ transfer_client.endpoint_autoactivate(DESTINATION_ENDPOINT)
except GlobusAPIError as ex:
if ex.http_status == 401:
- sys.exit('Refresh token has expired. '
- 'Please delete the `tokens` object from '
- '{} and try again.'.format(DATA_FILE))
+ sys.exit(f"Refresh token has expired. Please delete the `tokens` object from {DATA_FILE} and try again.")
else:
raise ex
return transfer_client
@@ -120,10 +126,7 @@ def check_endpoint_path(transfer_client, endpoint, path):
try:
transfer_client.operation_ls(endpoint, path=path)
except TransferAPIError as tapie:
- print('Failed to query endpoint "{}": {}'.format(
- endpoint,
- tapie.message
- ))
+ logger.error('Failed to query endpoint "%s": %s', endpoint, tapie.message)
sys.exit(1)
@@ -134,9 +137,9 @@ def create_destination_directory(transfer_client, dest_ep, dest_path):
except TransferAPIError:
try:
transfer_client.operation_mkdir(dest_ep, dest_path)
- print('Created directory: {}'.format(dest_path))
+ logger.info("Created directory: %s", dest_path)
except TransferAPIError as tapie:
- print('Failed to start transfer: {}'.format(tapie.message))
+ logger.error("Failed to start transfer: %s", tapie.message)
sys.exit(1)
@@ -146,48 +149,41 @@ def main():
try:
# if we already have tokens, load and use the
tokens = client.load_tokens(requested_scopes=SCOPES)
- print(tokens)
- except:
+ logger.debug("tokens %s", tokens)
+ except Exception:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
# need to specify that we want refresh tokens
- print('login in')
+ logger.info("login in")
tokens = client.login(requested_scopes=SCOPES, refresh_tokens=True)
- try:
+ with contextlib.suppress(Exception):
client.save_tokens(tokens)
- except:
- pass
- transfer = setup_transfer_client(tokens['transfer.api.globus.org'])
+ transfer = setup_transfer_client(tokens["transfer.api.globus.org"])
try:
data = load_data_from_file(DATA_FILE)
if len(data) > 0:
- task_data = data['task']
- task = transfer.get_task(task_data['task_id'])
- if task['status'] not in PREVIOUS_TASK_RUN_CASES:
- print('The last transfer status is {}, skipping run...'.format(
- task['status']
- ))
+ task_data = data["task"]
+ task = transfer.get_task(task_data["task_id"])
+ if task["status"] not in PREVIOUS_TASK_RUN_CASES:
+ logger.info("The last transfer status is %s, skipping run...", task["status"])
sys.exit(1)
except KeyError:
# Ignore if there is no previous task
pass
if len(sys.argv) < 2:
- print('Usage: {} tigress_dir pni_dir'.format(
- sys.argv[0]
- ))
+ logger.error("Usage: %s tigress_dir pni_dir", sys.argv[0])
sys.exit(1)
SOURCE_PATH = sys.argv[1]
DESTINATION_PATH = sys.argv[2]
check_endpoint_path(transfer, SOURCE_ENDPOINT, SOURCE_PATH)
if CREATE_DESTINATION_FOLDER:
- create_destination_directory(transfer, DESTINATION_ENDPOINT,
- DESTINATION_PATH)
+ create_destination_directory(transfer, DESTINATION_ENDPOINT, DESTINATION_PATH)
else:
check_endpoint_path(transfer, DESTINATION_ENDPOINT, DESTINATION_PATH)
@@ -196,27 +192,26 @@ def main():
SOURCE_ENDPOINT,
DESTINATION_ENDPOINT,
label=TRANSFER_LABEL,
- sync_level="checksum"
+ sync_level="checksum",
)
tdata.add_item(SOURCE_PATH, DESTINATION_PATH, recursive=True)
task = transfer.submit_transfer(tdata)
- save_data_to_file(DATA_FILE, 'task', task.data)
- print('Transfer has been started from\n {}:{}\nto\n {}:{}'.format(
- SOURCE_ENDPOINT,
- SOURCE_PATH,
- DESTINATION_ENDPOINT,
- DESTINATION_PATH
- ))
- url_string = 'https://globus.org/app/transfer?' + \
- six.moves.urllib.parse.urlencode({
- 'origin_id': SOURCE_ENDPOINT,
- 'origin_path': SOURCE_PATH,
- 'destination_id': DESTINATION_ENDPOINT,
- 'destination_path': DESTINATION_PATH
- })
- print('Visit the link below to see the changes:\n{}'.format(url_string))
-
-
-if __name__ == '__main__':
+ save_data_to_file(DATA_FILE, "task", task.data)
+ logger.info(
+ "Transfer has been started from\n %s:%s\nto\n %s:%s",
+ SOURCE_ENDPOINT, SOURCE_PATH, DESTINATION_ENDPOINT, DESTINATION_PATH,
+ )
+ url_string = "https://globus.org/app/transfer?" + six.moves.urllib.parse.urlencode(
+ {
+ "origin_id": SOURCE_ENDPOINT,
+ "origin_path": SOURCE_PATH,
+ "destination_id": DESTINATION_ENDPOINT,
+ "destination_path": DESTINATION_PATH,
+ }
+ )
+ logger.info("Visit the link below to see the changes:\n%s", url_string)
+
+
+if __name__ == "__main__":
main()
diff --git a/u19_pipeline/behavior.py b/u19_pipeline/behavior.py
index fe7c8eef..9164be9f 100644
--- a/u19_pipeline/behavior.py
+++ b/u19_pipeline/behavior.py
@@ -1,7 +1,7 @@
import datajoint as dj
-from . import acquisition, task, subject
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'behavior')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "behavior")
+
@schema
class DataDirectory(dj.Computed):
@@ -116,4 +116,4 @@ class TowersSessionPsych(dj.Computed):
session_pright_error=null : blob # confidence interval for precentage went right of data
session_delta_fit=null : blob # num of right - num of left, x ticks for fitting results
session_pright_fit=null : blob # fitting results for percent went right
- """
\ No newline at end of file
+ """
diff --git a/u19_pipeline/copy_table.py b/u19_pipeline/copy_table.py
index 445938cd..d6c6779a 100644
--- a/u19_pipeline/copy_table.py
+++ b/u19_pipeline/copy_table.py
@@ -1,34 +1,29 @@
-import datajoint as dj
import traceback
-from u19_pipeline.temp import acquisition, behavior, imaging, meso, meso_analysis
-from u19_pipeline import subject
+
+import datajoint as dj
from tqdm import tqdm
+from u19_pipeline import subject
+from u19_pipeline.temp import acquisition, behavior, imaging, meso, meso_analysis
+from u19_pipeline.utils.logging_config import get_logger
-acquisition_original = dj.create_virtual_module(
- 'acquisition_original', 'u19_acquisition'
-)
+logger = get_logger(__name__)
-behavior_original = dj.create_virtual_module(
- 'behavior_original', 'u19_behavior')
+acquisition_original = dj.create_virtual_module("acquisition_original", "u19_acquisition")
-meso_original = dj.create_virtual_module(
- 'meso_original', 'u19_meso'
-)
+behavior_original = dj.create_virtual_module("behavior_original", "u19_behavior")
-meso_analysis_original = dj.create_virtual_module(
- 'meso_analysis_original', 'u19_meso_analysis'
-)
+meso_original = dj.create_virtual_module("meso_original", "u19_meso")
-imaging_original = dj.create_virtual_module(
- 'imaging_original', 'u19_imaging'
-)
+meso_analysis_original = dj.create_virtual_module("meso_analysis_original", "u19_meso_analysis")
+
+imaging_original = dj.create_virtual_module("imaging_original", "u19_imaging")
def copy_table(target_schema, src_schema, table_name, **kwargs):
- if '.' in table_name:
- attrs = table_name.split('.')
+ if "." in table_name:
+ attrs = table_name.split(".")
target_table = target_schema
src_table = src_schema
@@ -40,9 +35,8 @@ def copy_table(target_schema, src_schema, table_name, **kwargs):
src_table = getattr(src_schema, table_name)
pk = src_table.heading.primary_key
- if 'session_number' in pk:
- q_insert = (src_table & acquisition.SessionStarted.proj()) - \
- target_table.proj()
+ if "session_number" in pk:
+ q_insert = (src_table & acquisition.SessionStarted.proj()) - target_table.proj()
else:
q_insert = src_table - target_table.proj()
@@ -54,174 +48,170 @@ def copy_table(target_schema, src_schema, table_name, **kwargs):
try:
target_table.insert1(t, skip_duplicates=True, **kwargs)
except Exception:
- print("Error when inserting {}".format(t))
+ logger.error(f"Error when inserting {t}")
traceback.print_exc()
def copy_acquisition_tables():
acquisition.SessionTemp.insert(
- (acquisition_original.Session - acquisition.SessionTemp.proj()) &
- acquisition.SessionStarted.proj(),
- skip_duplicates=True)
+ (acquisition_original.Session - acquisition.SessionTemp.proj()) & acquisition.SessionStarted.proj(),
+ skip_duplicates=True,
+ )
acquisition.DataDirectoryTemp.insert(
- (acquisition_original.DataDirectory - acquisition.DataDirectoryTemp.proj()) &
- acquisition.SessionStarted.proj(),
- allow_direct_insert=True
+ (acquisition_original.DataDirectory - acquisition.DataDirectoryTemp.proj()) & acquisition.SessionStarted.proj(),
+ allow_direct_insert=True,
)
def copy_behavior_tables():
BEHAVIOR_TABLES = [
- 'DataDirectory',
- 'TowersSession',
- 'TowersBlock',
- 'TowersBlock.Trial',
- 'TowersBlockTrialVideo',
- 'TowersSubjectCumulativePsych',
- 'TowersSessionPsych',
+ "DataDirectory",
+ "TowersSession",
+ "TowersBlock",
+ "TowersBlock.Trial",
+ "TowersBlockTrialVideo",
+ "TowersSubjectCumulativePsych",
+ "TowersSessionPsych",
]
for table in BEHAVIOR_TABLES:
-
- print(f'Copying table {table}')
- if '.' in table:
- if table == 'TowersBlock.Trial':
- for subj in tqdm((subject.Subject & behavior.TowersBlock).fetch('KEY')):
- behavior.TowersBlock.Trial.insert(
- behavior_original.TowersBlock.Trial & subj,
- skip_duplicates=True)
+ logger.info(f"Copying table {table}")
+ if "." in table:
+ if table == "TowersBlock.Trial":
+ for subj in tqdm((subject.Subject & behavior.TowersBlock).fetch("KEY")):
+ behavior.TowersBlock.Trial.insert(behavior_original.TowersBlock.Trial & subj, skip_duplicates=True)
else:
- if table == 'TowersBlockVideo':
- for subj in tqdm((subject.Subject & behavior.TowersBlock).fetch('KEY')):
+ if table == "TowersBlockVideo":
+ for subj in tqdm((subject.Subject & behavior.TowersBlock).fetch("KEY")):
behavior.TowersBlockTrialVideo.insert(
behavior_original.TowersBlockTrialVideo & subj & behavior.TowersBlock,
- skip_duplicates=True)
+ skip_duplicates=True,
+ )
else:
copy_table(behavior, behavior_original, table)
else:
- copy_table(behavior, behavior_original, table,
- allow_direct_insert=True)
+ copy_table(behavior, behavior_original, table, allow_direct_insert=True)
def copy_imaging_tables():
IMAGING_TABLES = [
- 'Scan',
- 'ScanInfo',
- 'FieldOfView',
- 'FieldOfView.File',
- 'McMethod',
- 'McParameter',
- 'McParameterSet',
- 'McParameterSet.Parameter',
- 'MotionCorrection',
- 'MotionCorrection.WithinFile',
- 'MotionCorrection.AcrossFiles',
- 'SegmentationMethod',
- 'SegParameter',
- 'SegParameterSet',
- 'SegParameterSet.Parameter',
- 'Segmentation',
- 'Segmentation.Roi',
- 'Segmentation.RoiMorphologyAuto',
- 'Segmentation.Chunks',
- 'Segmentation.Background',
- 'Trace'
+ "Scan",
+ "ScanInfo",
+ "FieldOfView",
+ "FieldOfView.File",
+ "McMethod",
+ "McParameter",
+ "McParameterSet",
+ "McParameterSet.Parameter",
+ "MotionCorrection",
+ "MotionCorrection.WithinFile",
+ "MotionCorrection.AcrossFiles",
+ "SegmentationMethod",
+ "SegParameter",
+ "SegParameterSet",
+ "SegParameterSet.Parameter",
+ "Segmentation",
+ "Segmentation.Roi",
+ "Segmentation.RoiMorphologyAuto",
+ "Segmentation.Chunks",
+ "Segmentation.Background",
+ "Trace",
]
for table in IMAGING_TABLES:
+ logger.info(f"Copying table {table}...")
- print(f'Copying table {table}...')
-
- if '.' in table:
+ if "." in table:
copy_table(imaging, imaging_original, table)
else:
temp_table = getattr(imaging, table)
- if isinstance(temp_table, dj.Lookup) or \
- isinstance(temp_table, dj.Manual):
- copy_table(imaging, imaging_original, table,
- skip_duplicates=True)
+ if isinstance(temp_table, (dj.Lookup, dj.Manual)):
+ copy_table(imaging, imaging_original, table, skip_duplicates=True)
else:
- copy_table(imaging, imaging_original, table,
- allow_direct_insert=True,
- skip_duplicates=True)
+ copy_table(
+ imaging,
+ imaging_original,
+ table,
+ allow_direct_insert=True,
+ skip_duplicates=True,
+ )
def copy_meso_tables():
MESO_TABLES = [
- 'Scan',
- 'ScanInfo',
- 'FieldOfView',
- 'FieldOfView.File',
- 'SyncImagingBehavior',
- 'MotionCorrectionMethod',
- 'McParameter',
- 'McParameterSet',
- 'McParameterSet.Parameter',
- 'MotionCorrectionWithinFile',
- 'MotionCorrectionAcrossFiles',
- 'MotionCorrection',
- 'SegmentationMethod',
- 'SegParameter',
- 'SegParameterSet',
- 'SegParameterSet.Parameter',
- 'Segmentation',
- 'Segmentation.Roi',
- 'Segmentation.RoiMorphologyAuto',
- 'Segmentation.Chunks',
- 'Segmentation.Background',
- 'SegmentationRoiMorphologyManual'
+ "Scan",
+ "ScanInfo",
+ "FieldOfView",
+ "FieldOfView.File",
+ "SyncImagingBehavior",
+ "MotionCorrectionMethod",
+ "McParameter",
+ "McParameterSet",
+ "McParameterSet.Parameter",
+ "MotionCorrectionWithinFile",
+ "MotionCorrectionAcrossFiles",
+ "MotionCorrection",
+ "SegmentationMethod",
+ "SegParameter",
+ "SegParameterSet",
+ "SegParameterSet.Parameter",
+ "Segmentation",
+ "Segmentation.Roi",
+ "Segmentation.RoiMorphologyAuto",
+ "Segmentation.Chunks",
+ "Segmentation.Background",
+ "SegmentationRoiMorphologyManual",
]
for table in MESO_TABLES:
+ logger.info(f"Copying table {table}...")
- print(f'Copying table {table}...')
-
- if '.' in table:
+ if "." in table:
copy_table(meso, meso_original, table)
else:
temp_table = getattr(meso, table)
- if isinstance(temp_table, dj.Lookup) or \
- isinstance(temp_table, dj.Manual):
+ if isinstance(temp_table, (dj.Lookup, dj.Manual)):
copy_table(meso, meso_original, table)
else:
- copy_table(meso, meso_original, table,
- allow_direct_insert=True)
+ copy_table(meso, meso_original, table, allow_direct_insert=True)
def copy_meso_analysis_tables():
MESO_ANALYSIS_TABLES = [
- 'TrialSelectionParams',
- 'BinningParameters',
- 'StandardizedTime',
- 'Trialstats',
- 'BinnedBehavior',
- 'TrialSelectionParameters',
- 'BinnedTrace'
+ "TrialSelectionParams",
+ "BinningParameters",
+ "StandardizedTime",
+ "Trialstats",
+ "BinnedBehavior",
+ "TrialSelectionParameters",
+ "BinnedTrace",
]
for table in MESO_ANALYSIS_TABLES:
+ logger.info(f"Copying table {table}...")
- print(f'Copying table {table}...')
-
- if '.' in table:
+ if "." in table:
copy_table(meso_analysis, meso_analysis_original, table)
else:
temp_table = getattr(meso_analysis, table)
- if isinstance(temp_table, dj.Lookup) or \
- isinstance(temp_table, dj.Manual):
+ if isinstance(temp_table, (dj.Lookup, dj.Manual)):
copy_table(meso_analysis, meso_analysis_original, table)
else:
- copy_table(meso_analysis, meso_analysis_original, table,
- allow_direct_insert=True)
+ copy_table(
+ meso_analysis,
+ meso_analysis_original,
+ table,
+ allow_direct_insert=True,
+ )
def main():
@@ -233,5 +223,5 @@ def main():
copy_meso_analysis_tables()
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/u19_pipeline/ephys_pipeline.py b/u19_pipeline/ephys_pipeline.py
index b312c124..7ad5e5e5 100644
--- a/u19_pipeline/ephys_pipeline.py
+++ b/u19_pipeline/ephys_pipeline.py
@@ -16,6 +16,9 @@
import u19_pipeline.utils.ephys_fix_sync_code as efsc
import u19_pipeline.utils.ephys_utils as ephys_utils
from u19_pipeline import recording
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
schema = dj.schema(dj.config["custom"]["database.prefix"] + "ephys_pipeline")
@@ -93,12 +96,12 @@ def get_session_directory(session_key):
def get_full_session_directory(recording_key):
session_dir = find_full_path([get_ephys_root_data_dir()[0]], get_session_directory(recording_key))
- print("ephys dir:", session_dir)
+ logger.info("ephys dir: %s", session_dir)
nidq_session = list(session_dir.glob("*nidq.bin*"))
obx_session = list(session_dir.glob("*obx.bin*"))
if len(nidq_session) == 0 and len(obx_session) == 0:
- print("No ephys session found")
+ logger.warning("No ephys session found")
ephys_session_fullpath = ""
elif len(nidq_session) > 0:
ephys_session_fullpath = nidq_session[0]
@@ -109,7 +112,7 @@ def get_full_session_directory(recording_key):
def append_cat_gt_params_from_probedir(probe_dirname):
- extra_cat_gt_params = dict()
+ extra_cat_gt_params = {}
probe_match = re.search("_imec[0-9]$", probe_dirname)
if probe_match:
@@ -160,13 +163,13 @@ def create_lfp_trace(cat_gt_script, recording_directory, probe_directory):
"-lffilter=" + lfp_filter_params,
]
- print(cat_gt_command)
+ logger.debug("cat_gt_command: %s", cat_gt_command)
p = subprocess.Popen(cat_gt_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
stdout, stderr = p.communicate()
- print('stdout', stdout)
- print('stderr', stderr.decode("UTF-8"))
+ logger.debug("stdout: %s", stdout)
+ logger.debug("stderr: %s", stderr.decode("UTF-8"))
if stderr:
error = json.loads(stderr.decode("UTF-8"))
@@ -188,17 +191,17 @@ def get_spikeglx_meta_filepath(ephys_recording_key):
ephys_element.EphysRecording.EphysFile & ephys_recording_key & 'file_path LIKE "%.ap.meta"'
).fetch1("file_path")
- print("ephys_recording_key", ephys_recording_key)
- print("spikeglx_meta_filepath", spikeglx_meta_filepath)
+ logger.debug("ephys_recording_key: %s", ephys_recording_key)
+ logger.debug("spikeglx_meta_filepath: %s", spikeglx_meta_filepath)
- print("get_ephys_root_data_dir", get_ephys_root_data_dir())
+ logger.debug("get_ephys_root_data_dir: %s", get_ephys_root_data_dir())
spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(), spikeglx_meta_filepath)
- print("spikeglx_meta_filepath", spikeglx_meta_filepath)
+ logger.debug("spikeglx_meta_filepath: %s", spikeglx_meta_filepath)
try:
spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(), spikeglx_meta_filepath)
- print("spikeglx_meta_filepath", spikeglx_meta_filepath)
+ logger.debug("spikeglx_meta_filepath: %s", spikeglx_meta_filepath)
except FileNotFoundError:
# if not found, search in session_dir again
if not spikeglx_meta_filepath.exists():
@@ -207,17 +210,18 @@ def get_spikeglx_meta_filepath(ephys_recording_key):
ephys_element.ProbeInsertion * ephys_element.probe.Probe & ephys_recording_key
).fetch1("probe")
- spikeglx_meta_filepaths = [fp for fp in session_dir.rglob("*.ap.meta")]
+ spikeglx_meta_filepaths = list(session_dir.rglob("*.ap.meta"))
for meta_filepath in spikeglx_meta_filepaths:
spikeglx_meta = spikeglx.SpikeGLXMeta(meta_filepath)
if str(spikeglx_meta.probe_SN) == inserted_probe_serial_number:
spikeglx_meta_filepath = meta_filepath
break
else:
- raise FileNotFoundError("No SpikeGLX data found for probe insertion: {}".format(ephys_recording_key))
+ raise FileNotFoundError(f"No SpikeGLX data found for probe insertion: {ephys_recording_key}")
return spikeglx_meta_filepath
+
def get_full_vectors_from_key(rec_key, single_vec_mode=False):
"""
Get time and "old school" synchronization vectors from a recording key.
@@ -246,7 +250,7 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False):
full_session_path = get_full_session_directory(rec_key)
if isinstance(full_session_path, str) and len(full_session_path) == 0:
- print("No session found for this key")
+ logger.warning("No session found for this key")
return
# Get sampling rate and calculate channels and samples
@@ -257,43 +261,46 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False):
# Read behavior sync record
try:
sync_data = (BehaviorSync & rec_key).fetch1("sync_data")
- except:
- print("No sync data was sound for this session")
+ except Exception:
+ logger.warning("No sync data was found for this session")
return
+ trial_index_nidq_virmen, iteration_index_nidq_virmen = ephys_utils.get_full_vector_samples(
+ sync_data["iteration_idx_vector_from_virmen"],
+ nidq_sampling_rate,
+ num_samples,
+ )
- trial_index_nidq_virmen, iteration_index_nidq_virmen =\
- ephys_utils.get_full_vector_samples(sync_data['iteration_idx_vector_from_virmen'],nidq_sampling_rate,num_samples)
-
- trial_index_nidq, iteration_index_nidq =\
- ephys_utils.get_full_vector_samples(sync_data['iteration_idx_vector'],nidq_sampling_rate,num_samples)
-
+ trial_index_nidq, iteration_index_nidq = ephys_utils.get_full_vector_samples(
+ sync_data["iteration_idx_vector"], nidq_sampling_rate, num_samples
+ )
time_vector = ephys_utils.get_time_vector(trial_index_nidq, nidq_sampling_rate)
- #Calculate time for iteration start samples
- trial_times_ind, trial_times_full =\
- ephys_utils.get_time_vector_as_behavior(sync_data['iteration_idx_vector'], nidq_sampling_rate)
+ # Calculate time for iteration start samples
+ trial_times_ind, trial_times_full = ephys_utils.get_time_vector_as_behavior(
+ sync_data["iteration_idx_vector"], nidq_sampling_rate
+ )
- trial_times_ind_virmen, trial_times_full_virmen =\
- ephys_utils.get_time_vector_as_behavior(sync_data['iteration_idx_vector_from_virmen'], nidq_sampling_rate)
+ trial_times_ind_virmen, trial_times_full_virmen = ephys_utils.get_time_vector_as_behavior(
+ sync_data["iteration_idx_vector_from_virmen"], nidq_sampling_rate
+ )
+ # Store data
+ all_vectors = {}
- #Store data
- all_vectors = dict()
+ all_vectors["time_as_behavior_trial_ind"] = trial_times_ind
+ all_vectors["time_as_behavior_fullsession"] = trial_times_full
- all_vectors['time_as_behavior_trial_ind'] = trial_times_ind
- all_vectors['time_as_behavior_fullsession'] = trial_times_full
-
- all_vectors['time_as_behavior_trial_ind_virmen'] = trial_times_ind_virmen
- all_vectors['time_as_behavior_fullsession_virmen'] = trial_times_full_virmen
+ all_vectors["time_as_behavior_trial_ind_virmen"] = trial_times_ind_virmen
+ all_vectors["time_as_behavior_fullsession_virmen"] = trial_times_full_virmen
if single_vec_mode:
- for this_key in all_vectors.keys():
- print('this_key', this_key)
- print(all_vectors[this_key].shape)
- all_vectors[this_key] = np.concatenate(([x for x in all_vectors[this_key]]), axis=0)
- print(all_vectors[this_key].shape)
+ for this_key in all_vectors:
+ logger.debug("this_key: %s", this_key)
+ logger.debug(all_vectors[this_key].shape)
+ all_vectors[this_key] = np.concatenate((list(all_vectors[this_key])), axis=0)
+ logger.debug(all_vectors[this_key].shape)
all_vectors["trial_index_nidq_virmen"] = trial_index_nidq_virmen
all_vectors["iteration_index_nidq_virmen"] = iteration_index_nidq_virmen
@@ -302,7 +309,6 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False):
all_vectors["iteration_index_nidq"] = iteration_index_nidq
all_vectors["time_vector"] = time_vector
-
return all_vectors
@@ -336,7 +342,7 @@ class ImecSamplingRate(dj.Part):
def make(self, key, **kwargs):
# Pull the Nidaq file/record
- print(key)
+ logger.debug("key: %s", key)
try:
ephys_session_fullpath = get_full_session_directory(key)
@@ -350,27 +356,27 @@ def make(self, key, **kwargs):
# If a specific block is requested, add that to our behavior_key. It should be an int referring to virmen block number.
# This is useful for sessions in which the nidaq stream was interrupted due to restarting virmen
if "block" in kwargs:
- print("block: ", kwargs["block"])
+ logger.debug("block: %s", kwargs["block"])
behavior_key["block"] = kwargs["block"]
- print(behavior_key)
+ logger.debug("behavior_key: %s", behavior_key)
# And get the datajoint record
behavior = dj.create_virtual_module("behavior", "u19_behavior")
thissession = behavior.TowersBlock().Trial() & behavior_key
behavior_time, iterstart = thissession.fetch("trial_time", "vi_start")
- if key['recording_id']==530:
- print(type(behavior_time))
- print(len(behavior_time))
+ if key["recording_id"] == 530:
+ logger.debug("behavior_time type: %s", type(behavior_time))
+ logger.debug("behavior_time len: %d", len(behavior_time))
behavior_time = behavior_time[:-2]
- print("len iterstart", len(iterstart))
+ logger.debug("len iterstart: %d", len(iterstart))
if len(iterstart) == 0:
raise ValueError("No behavior found")
- print("after reading behavior data")
+ logger.debug("after reading behavior data")
# 1: load meta data, and the content of the NIDAQ file. Its content is digital.
nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate(ephys_session_fullpath)
@@ -379,7 +385,7 @@ def make(self, key, **kwargs):
ephys_session_fullpath, nidq_meta
)
- print("after reading spikeglx data")
+ logger.debug("after reading spikeglx data")
# Synchronize between pulses and get iteration # vector for each sample
recent_recording = behavior_key["session_date"] > datetime.date(2021, 6, 1) # Everything past June 1 2021
@@ -404,11 +410,19 @@ def make(self, key, **kwargs):
ephys_utils.assert_iteration_samples_count(iteration_dict["iter_start_idx"], behavior_time)
)
- print("metrics to evaluate...")
- print(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])
+ logger.debug(
+ "metrics to evaluate: trial_count_diff=%s, trials_diff_iteration_big=%s, trials_diff_iteration_small=%s, n_trials=%d",
+ trial_count_diff,
+ trials_diff_iteration_big,
+ trials_diff_iteration_small,
+ behavior_time.shape[0],
+ )
status = ephys_utils.evaluate_sync_process(
- trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0]
+ trial_count_diff,
+ trials_diff_iteration_big,
+ trials_diff_iteration_small,
+ behavior_time.shape[0],
)
if status == 1:
@@ -421,7 +435,7 @@ def make(self, key, **kwargs):
status_fix = 0
if status < 1:
status_regular = 0
- print("Regular ephys sync failed")
+ logger.warning("Regular ephys sync failed")
status_fix, iteration_dict = efsc.main_ephys_fix_sync_code(
iteration_dict["iter_start_idx"],
iteration_dict["iter_times_idx"],
@@ -429,9 +443,9 @@ def make(self, key, **kwargs):
nidq_sampling_rate,
)
- dictionary_sync_data = dict()
+ dictionary_sync_data = {}
- print("after all main ehpys fix sync code", status_regular, status_fix)
+ logger.debug("after all main ephys fix sync code: status_regular=%s, status_fix=%s", status_regular, status_fix)
if status_regular > 0 or status_fix > 0:
dictionary_sync_data["trial_idx_vector"] = iteration_dict["trial_start_idx"]
@@ -440,10 +454,14 @@ def make(self, key, **kwargs):
dictionary_sync_data["trial_idx_vector"] = []
dictionary_sync_data["iteration_idx_vector"] = []
- iteration_dict["trial_start_idx_virmen"], iteration_dict["iter_start_idx_virmen"] = (
- ephys_utils.get_iteration_intertrial_from_virmen_time(
- trial_pulse_signal, nidq_sampling_rate, behavior_time.shape[0], behavior_time
- )
+ (
+ iteration_dict["trial_start_idx_virmen"],
+ iteration_dict["iter_start_idx_virmen"],
+ ) = ephys_utils.get_iteration_intertrial_from_virmen_time(
+ trial_pulse_signal,
+ nidq_sampling_rate,
+ behavior_time.shape[0],
+ behavior_time,
)
dictionary_sync_data["trial_idx_vector_from_virmen"] = iteration_dict["trial_start_idx_virmen"]
@@ -460,19 +478,19 @@ def make(self, key, **kwargs):
virmen_sync_status=1,
)
- print('ephys_session_fullpath', ephys_session_fullpath)
- print('sync code executed sucessfully !!!!!!!!!!!!!!!!!!!!!!!!!!!')
+ logger.info("ephys_session_fullpath: %s", ephys_session_fullpath)
+ logger.info("sync code executed successfully")
- print(kwargs)
+ logger.debug("kwargs: %s", kwargs)
- if 'populate' not in kwargs or kwargs['populate'] == True:
- BehaviorSync.insert1(final_key,allow_direct_insert=True)
+ if "populate" not in kwargs or kwargs["populate"]:
+ BehaviorSync.insert1(final_key, allow_direct_insert=True)
self.insert_imec_sampling_rate(key, ephys_session_fullpath.parent)
else:
return final_key
except Exception as e:
- print(e)
+ logger.exception("Error in BehaviorSync.make: %s", e)
def insert_imec_sampling_rate(self, key, session_dir):
# get the imec sampling rate for a particular probe
diff --git a/u19_pipeline/ephys_sync.py b/u19_pipeline/ephys_sync.py
index 92d8d69a..65bee7d1 100644
--- a/u19_pipeline/ephys_sync.py
+++ b/u19_pipeline/ephys_sync.py
@@ -1,17 +1,19 @@
-import datajoint as dj
import pathlib
+
+import datajoint as dj
import numpy as np
+import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX
+import u19_pipeline.utils.ephys_utils as ephys_utils
from u19_pipeline import behavior
from u19_pipeline.ephys_pipeline import ephys_element, get_session_directory
+from u19_pipeline.utils.logging_config import get_logger
-import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX
-import u19_pipeline.utils.ephys_utils as ephys_utils
-import u19_pipeline.utils.path_utils as pu
-import u19_pipeline.automatic_job.params_config as config
+logger = get_logger(__name__)
# Tables downstream from `ephys_pipeline` module ---------------------------------------
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'ephys_sync')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "ephys_sync")
+
@schema
class BehaviorSync(dj.Imported):
@@ -34,67 +36,77 @@ class ImecSamplingRate(dj.Part):
def make(self, key):
# Pull the Nidaq file/record
session_dir = pathlib.Path(get_session_directory(key))
- nidq_bin_full_path = list(session_dir.glob('*nidq.bin*'))[0]
+ nidq_bin_full_path = list(session_dir.glob("*nidq.bin*"))[0]
# And get the datajoint record
thissession = behavior.TowersBlock().Trial() & key
- behavior_time, iterstart = thissession.fetch('trial_time', 'vi_start')
+ behavior_time, iterstart = thissession.fetch("trial_time", "vi_start")
# 1: load meta data, and the content of the NIDAQ file. Its content is digital.
- nidq_meta = readSGLX.readMeta(nidq_bin_full_path)
+ nidq_meta = readSGLX.readMeta(nidq_bin_full_path)
nidq_sampling_rate = readSGLX.SampRate(nidq_meta)
- if nidq_meta['typeThis'] == 'nidq':
- digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta)
+ if nidq_meta["typeThis"] == "nidq":
+ digital_array = ephys_utils.SpiceGlxUtility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta)
else:
- digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta, d_line_list=[1,2])
+ digital_array = ephys_utils.SpiceGlxUtility.load_spice_glx_digital_file(
+ nidq_bin_full_path, nidq_meta, d_line_list=[1, 2]
+ )
# Synchronize between pulses and get iteration # vector for each sample
- mode='counter_bit0'
- iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[1,:], digital_array[2,:], nidq_sampling_rate, behavior_time.shape[0], mode)
+ mode = "counter_bit0"
+ iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(
+ digital_array[1, :],
+ digital_array[2, :],
+ nidq_sampling_rate,
+ behavior_time.shape[0],
+ mode,
+ )
# Check # of trials and iterations match
- status = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)
+ status = ephys_utils.assert_iteration_samples_count(iteration_dict["iter_start_idx"], behavior_time)
- #They didn't match, try counter method (if available)
+ # They didn't match, try counter method (if available)
if (not status) and (digital_array.shape[0] > 3):
- [framenumber_in_trial, trialnumber] = ephys_utils.behavior_sync_frame_counter_method(digital_array, behavior_time, thissession, nidq_sampling_rate, 3, 5)
- iteration_dict['framenumber_vector_samples'] = framenumber_in_trial
- iteration_dict['trialnumber_vector_samples'] = trialnumber
-
+ [framenumber_in_trial, trialnumber] = ephys_utils.behavior_sync_frame_counter_method(
+ digital_array, behavior_time, thissession, nidq_sampling_rate, 3, 5
+ )
+ iteration_dict["framenumber_vector_samples"] = framenumber_in_trial
+ iteration_dict["trialnumber_vector_samples"] = trialnumber
- final_key = dict(key, nidq_sampling_rate = nidq_sampling_rate,
- iteration_index_nidq = iteration_dict['framenumber_vector_samples'],
- trial_index_nidq = iteration_dict['trialnumber_vector_samples'])
+ final_key = dict(
+ key,
+ nidq_sampling_rate=nidq_sampling_rate,
+ iteration_index_nidq=iteration_dict["framenumber_vector_samples"],
+ trial_index_nidq=iteration_dict["trialnumber_vector_samples"],
+ )
- print(final_key)
+ logger.debug("final_key: %s", final_key)
- self.insert1(final_key,allow_direct_insert=True)
+ self.insert1(final_key, allow_direct_insert=True)
self.insert_imec_sampling_rate(key, session_dir)
-
-
def insert_imec_sampling_rate(self, key, session_dir):
# get the imec sampling rate for a particular probe
here = ephys_element.ProbeInsertion & key
- for probe_insertion in here.fetch('KEY'):
- #imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.bin'.format(probe_insertion['insertion_number'])))
- imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.meta'.format(probe_insertion['insertion_number'])))
+ for probe_insertion in here.fetch("KEY"):
+ # imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.bin'.format(probe_insertion['insertion_number'])))
+ imec_bin_filepath = list(session_dir.glob("*imec{}/*.ap.meta".format(probe_insertion["insertion_number"])))
- if len(imec_bin_filepath) == 1: # find the binary file to get meta data
+ if len(imec_bin_filepath) == 1: # find the binary file to get meta data
imec_bin_filepath = imec_bin_filepath[0]
- else: # if this fails, get the ap.meta file.
- imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.meta'.format(probe_insertion['insertion_number'])))
+ else: # if this fails, get the ap.meta file.
+ imec_bin_filepath = list(
+ session_dir.glob("*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]))
+ )
if len(imec_bin_filepath) == 1:
s = str(imec_bin_filepath[0])
imec_bin_filepath = pathlib.Path(s.replace(".meta", ".bin"))
- else: # If this fails too, no imec file exists at the path.
+ else: # If this fails too, no imec file exists at the path.
raise NameError("No imec meta file found.")
imec_meta = readSGLX.readMeta(imec_bin_filepath)
- self.ImecSamplingRate.insert1(
- dict(probe_insertion,
- ephys_sampling_rate=imec_meta['imSampRate']))
+ self.ImecSamplingRate.insert1(dict(probe_insertion, ephys_sampling_rate=imec_meta["imSampRate"]))
@schema
@@ -118,58 +130,62 @@ def make(self, key):
self.insert1(key)
- nidq_sampling_rate, iteration_index_nidq = \
- (BehaviorSync * BehaviorSync.ImecSamplingRate & key).fetch1(
- 'nidq_sampling_rate', 'iteration_index_nidq')
+ nidq_sampling_rate, iteration_index_nidq = (BehaviorSync * BehaviorSync.ImecSamplingRate & key).fetch1(
+ "nidq_sampling_rate", "iteration_index_nidq"
+ )
key_session = key.copy()
del key_session["insertion_number"]
thissession = behavior.TowersBlock().Trial() & key
- iterstart = thissession.fetch('vi_start')
+ iterstart = thissession.fetch("vi_start")
- first_vr_iteration = iterstart[0]
+ iterstart[0]
# Obtain the precise times when the frames transition.
# This is obtained from iteration_index_nidq
ls = np.diff(iteration_index_nidq)
- ls[ls<0] = 1 # These are the trial transitions (see definition above). To get total number of frames, we define this as a transition like all others.
+ ls[ls < 0] = (
+ 1 # These are the trial transitions (see definition above). To get total number of frames, we define this as a transition like all others.
+ )
ls[np.isnan(ls)] = 0
- iteration_transition_indexes = np.where(ls)[0]
+ np.where(ls)[0]
# First iterations captured not in virmen because vr was not started yet
- #for i in range(first_vr_iteration):
+ # for i in range(first_vr_iteration):
# if iteration_index_nidq[iteration_transition_indexes[i]] <= first_vr_iteration:
# ls[iteration_transition_indexes[i]] = 0
- print('sum_iterationtrans', np.sum(ls))
+ logger.debug("sum_iterationtrans: %s", np.sum(ls))
- iteration_times = np.where(ls)[0]/nidq_sampling_rate
+ iteration_times = np.where(ls)[0] / nidq_sampling_rate
# get end of time from nidq metadata
session_dir = pathlib.Path(get_session_directory(key))
- nidq_bin_full_path = list(session_dir.glob('*nidq.bin*'))[0]
+ nidq_bin_full_path = list(session_dir.glob("*nidq.bin*"))[0]
nidq_meta = readSGLX.readMeta(nidq_bin_full_path)
- t_end = np.float(nidq_meta['fileTimeSecs'])
+ t_end = np.float(nidq_meta["fileTimeSecs"])
unit_spike_counts = []
- for unit_key in (ephys_element.CuratedClustering.Unit & key).fetch('KEY'):
- spike_times = (ephys_element.CuratedClustering.Unit & unit_key).fetch1('spike_times')
+ for unit_key in (ephys_element.CuratedClustering.Unit & key).fetch("KEY"):
+ spike_times = (ephys_element.CuratedClustering.Unit & unit_key).fetch1("spike_times")
# vector with length n_iterations + 1
spike_counts_iteration = np.bincount(np.digitize(spike_times, iteration_times))
- firing_rate_before_first_iteration = spike_counts_iteration[0]/iteration_times[0]
- firing_rate_after_last_iteration = spike_counts_iteration[-1]/(t_end - iteration_times[-1])
+ firing_rate_before_first_iteration = spike_counts_iteration[0] / iteration_times[0]
+ firing_rate_after_last_iteration = spike_counts_iteration[-1] / (t_end - iteration_times[-1])
# remove the first and last entries
spike_counts_iteration = np.delete(spike_counts_iteration, [0, -1])
unit_spike_counts.append(
- dict(unit_key,
- spike_counts_iteration=spike_counts_iteration,
- firing_rate_before_first_iteration=firing_rate_before_first_iteration,
- firing_rate_after_last_iteration=firing_rate_after_last_iteration)
+ dict(
+ unit_key,
+ spike_counts_iteration=spike_counts_iteration,
+ firing_rate_before_first_iteration=firing_rate_before_first_iteration,
+ firing_rate_after_last_iteration=firing_rate_after_last_iteration,
+ )
)
self.Unit.insert(unit_spike_counts)
diff --git a/u19_pipeline/imaging.py b/u19_pipeline/imaging.py
index df5a0f1b..1973a52e 100644
--- a/u19_pipeline/imaging.py
+++ b/u19_pipeline/imaging.py
@@ -1,8 +1,6 @@
import datajoint as dj
-from u19_pipeline import acquisition
-
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "imaging")
@schema
diff --git a/u19_pipeline/imaging_pipeline.py b/u19_pipeline/imaging_pipeline.py
index 87acec7b..ea358756 100644
--- a/u19_pipeline/imaging_pipeline.py
+++ b/u19_pipeline/imaging_pipeline.py
@@ -1,24 +1,22 @@
-
-import datajoint as dj
import pathlib
import subprocess
from pathlib import Path
-from u19_pipeline import lab, subject, recording
+import datajoint as dj
+from element_calcium_imaging import imaging_preprocess as imaging_element
+from element_interface.utils import find_full_path
+
import u19_pipeline.automatic_job.params_config as config
import u19_pipeline.utils.dj_shortcuts as dj_short
import u19_pipeline.utils.tiff_utils as tu
+from u19_pipeline import lab, recording, subject
+from u19_pipeline.utils.logging_config import get_logger
+logger = get_logger(__name__)
-import datajoint as dj
-
-from element_calcium_imaging import scan as scan_element
-from element_calcium_imaging import imaging_preprocess as imaging_element
-from element_interface.utils import find_full_path
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "imaging_pipeline")
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging_pipeline')
-
# Declare upstream imaging tables ------------------------------------------------------
@schema
class ImagingPipelineSession(dj.Computed):
@@ -26,18 +24,17 @@ class ImagingPipelineSession(dj.Computed):
# General information of an imaging session
-> recording.Recording
"""
+
@property
def key_source(self):
- return recording.Recording & {'recording_modality': 'imaging'}
+ return recording.Recording & {"recording_modality": "imaging"}
def make(self, key):
self.insert1(key)
-
@schema
class AcquiredTiff(dj.Imported):
-
definition = """
-> imaging_pipeline.ImagingPipelineSession
---
@@ -73,31 +70,26 @@ class AcquiredTiff(dj.Imported):
stack_definition="N/A" : varchar(64)
"""
- photon_micro_acq = ['2photon', '3photon']
- mesoscope_acq = ['mesoscope']
+ photon_micro_acq = ["2photon", "3photon"]
+ mesoscope_acq = ["mesoscope"]
def make(self, key, test_mode=False):
- scan_info = (
- ImagingPipelineSession
- * recording.Recording
- * lab.Location
- & key
- ).fetch1()
+ scan_info = (ImagingPipelineSession * recording.Recording * lab.Location & key).fetch1()
- imaging_root = dj.config['custom']['imaging_root_data_dir'][0]
- scan_directory = Path(imaging_root) / scan_info['recording_directory']
- acq_type = scan_info['acquisition_type']
+ imaging_root = dj.config["custom"]["imaging_root_data_dir"][0]
+ scan_directory = Path(imaging_root) / scan_info["recording_directory"]
+ acq_type = scan_info["acquisition_type"]
is_mesoscope = acq_type in self.mesoscope_acq
is_2photon = acq_type in self.photon_micro_acq
- print(f'Preparing {scan_directory}')
+ logger.info(f"Preparing {scan_directory}")
if is_mesoscope:
- original_stacks_dir = scan_directory / 'originalStacks'
+ original_stacks_dir = scan_directory / "originalStacks"
- tif_files = list(scan_directory.glob('*tif*'))
+ tif_files = list(scan_directory.glob("*tif*"))
if not tif_files and original_stacks_dir.exists():
tif_dir = original_stacks_dir
@@ -111,47 +103,33 @@ def make(self, key, test_mode=False):
fl, basename, is_compressed = tu.check_tif_files(tif_dir)
- fl = [Path(tif_dir,x).as_posix() for x in fl]
+ fl = [Path(tif_dir, x).as_posix() for x in fl]
if is_mesoscope:
imheader, parsed_info = tu.get_parsed_info_mesoscope(fl)
else:
imheader, parsed_info = tu.get_parsed_info_2photon(fl)
- rec_info, frames_per_file = tu.get_recording_info(
- fl,
- imheader,
- parsed_info
- )
+ rec_info, frames_per_file = tu.get_recording_info(fl, imheader, parsed_info)
- rec_info['nfovs'] = tu.get_nfovs(rec_info, is_mesoscope)
+ rec_info["nfovs"] = tu.get_nfovs(rec_info, is_mesoscope)
- last_good_file, cumulative_frames = tu.get_last_good_frame(
- frames_per_file,
- tif_dir
- )
+ last_good_file, cumulative_frames = tu.get_last_good_frame(frames_per_file, tif_dir)
- rec_info['nframes_good'] = cumulative_frames[last_good_file]
- rec_info['last_good_file'] = last_good_file+1
+ rec_info["nframes_good"] = cumulative_frames[last_good_file]
+ rec_info["last_good_file"] = last_good_file + 1
- rec_info['AcqTime'] = tu.check_acqtime(
- rec_info['AcqTime'],
- scan_directory
- )
+ rec_info["AcqTime"] = tu.check_acqtime(rec_info["AcqTime"], scan_directory)
if is_compressed:
tu.remove_compressed_videos(fl, scan_directory)
- scan_info_key = tu.create_scan_info_key(
- key,
- rec_info,
- scan_info['recording_directory']
- )
+ scan_info_key = tu.create_scan_info_key(key, rec_info, scan_info["recording_directory"])
if not test_mode:
- self.insert1(scan_info_key,allow_direct_insert=True)
+ self.insert1(scan_info_key, allow_direct_insert=True)
if is_mesoscope:
- tiffsplit_mesoscope_keys,tiff_splitfiles_mesoscope_keys = tu.get_fov_mesoscope(
+ tiffsplit_mesoscope_keys, tiff_splitfiles_mesoscope_keys = tu.get_fov_mesoscope(
fl,
key,
skip_parsing,
@@ -160,41 +138,49 @@ def make(self, key, test_mode=False):
basename,
cumulative_frames,
scan_info,
- imaging_root
+ imaging_root,
)
for i in range(len(tiffsplit_mesoscope_keys)):
if not test_mode:
- TiffSplit.insert(tiffsplit_mesoscope_keys[i],allow_direct_insert=True)
- TiffSplit.File.insert(tiff_splitfiles_mesoscope_keys[i],allow_direct_insert=True)
+ TiffSplit.insert(tiffsplit_mesoscope_keys[i], allow_direct_insert=True)
+ TiffSplit.File.insert(tiff_splitfiles_mesoscope_keys[i], allow_direct_insert=True)
if test_mode:
- return scan_info_key, tiffsplit_mesoscope_keys, tiff_splitfiles_mesoscope_keys
+ return (
+ scan_info_key,
+ tiffsplit_mesoscope_keys,
+ tiff_splitfiles_mesoscope_keys,
+ )
elif is_2photon:
tiffsplit_2photon_key = tu.get_fov_photonmicro(key, rec_info, scan_info)
tiffsplitfile_2photon_key = tu.get_fovfile_photonmicro(key, fl, imheader)
if not test_mode:
- TiffSplit.insert([tiffsplit_2photon_key],allow_direct_insert=True)
- TiffSplit.File.insert(tiffsplitfile_2photon_key,allow_direct_insert=True)
+ TiffSplit.insert([tiffsplit_2photon_key], allow_direct_insert=True)
+ TiffSplit.File.insert(tiffsplitfile_2photon_key, allow_direct_insert=True)
if test_mode:
return scan_info_key, tiffsplit_2photon_key, tiffsplitfile_2photon_key
else:
raise ValueError("Invalid acquisition type")
-
+
def old_make(self, key):
str_key = dj_short.get_string_key(key)
- command = [config.ingest_scaninfo_script, config.startup_pipeline_matlab_dir, str_key]
- print(command)
+ command = [
+ config.ingest_scaninfo_script,
+ config.startup_pipeline_matlab_dir,
+ str_key,
+ ]
+ logger.debug("command: %s", command)
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
- print('aftercommand before comm')
+ logger.debug("aftercommand before comm")
stdout, stderr = p.communicate()
- print('aftercommand after comm')
- print(stdout.decode('UTF-8'))
- print(stderr.decode('UTF-8'))
+ logger.debug("aftercommand after comm")
+ logger.debug(stdout.decode("UTF-8"))
+ logger.debug(stderr.decode("UTF-8"))
@schema
@@ -251,14 +237,14 @@ class File(dj.Part):
"""
# 1. Schema names ----------------------------------------------------------------------
-scan_schema_name = dj.config['custom']['database.prefix'] + 'pipeline_scan_element'
-imaging_schema_name = dj.config['custom']['database.prefix'] + 'pipeline_imaging_element'
+scan_schema_name = dj.config["custom"]["database.prefix"] + "pipeline_scan_element"
+imaging_schema_name = dj.config["custom"]["database.prefix"] + "pipeline_imaging_element"
# 2. Upstream tables -------------------------------------------------------------------
-from u19_pipeline.reference import BrainArea as Location
Session = TiffSplit
+
@lab.schema
class Equipment(dj.Manual):
definition = """
@@ -268,69 +254,76 @@ class Equipment(dj.Manual):
description=null : varchar(256)
"""
+
# 3. Utility functions -----------------------------------------------------------------
-from u19_pipeline import recording_process
+
def get_imaging_root_data_dir():
- return dj.config.get('custom', {}).get('imaging_root_data_dir', None)
+ return dj.config.get("custom", {}).get("imaging_root_data_dir", None)
+
def get_scan_image_files(job_id):
- #scan_key = (TiffSplit * recording_process.Processing.proj('recording_id', tiff_split='fragment_number') & job_id).fetch1('KEY')
+ # scan_key = (TiffSplit * recording_process.Processing.proj('recording_id', tiff_split='fragment_number') & job_id).fetch1('KEY')
- scan_key = (TiffSplit & job_id).fetch1('KEY')
+ scan_key = (TiffSplit & job_id).fetch1("KEY")
- filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch('tiff_split_directory', 'tiff_split_filename', as_dict=True)
+ filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch(
+ "tiff_split_directory", "tiff_split_filename", as_dict=True
+ )
- tiff_filepaths = [find_full_path(get_imaging_root_data_dir(),
- pathlib.Path(file['tiff_split_directory']) /
- file['tiff_split_filename']).as_posix()
- for file in filepaths]
+ tiff_filepaths = [
+ find_full_path(
+ get_imaging_root_data_dir(),
+ pathlib.Path(file["tiff_split_directory"]) / file["tiff_split_filename"],
+ ).as_posix()
+ for file in filepaths
+ ]
return tiff_filepaths
-def get_calcium_imaging_files(scan_key, acq_software):
- filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch('tiff_split_directory', 'tiff_split_filename', as_dict=True)
-
- tiff_filepaths = [find_full_path(get_imaging_root_data_dir(),
- pathlib.Path(file['tiff_split_directory']) /
- file['tiff_split_filename']).as_posix()
- for file in filepaths]
-
- return tiff_filepaths
-
def get_calcium_imaging_files(scan_key, acq_software):
- filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch('tiff_split_directory', 'tiff_split_filename', as_dict=True)
+ filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch(
+ "tiff_split_directory", "tiff_split_filename", as_dict=True
+ )
- tiff_filepaths = [find_full_path(get_imaging_root_data_dir(),
- pathlib.Path(file['tiff_split_directory']) /
- file['tiff_split_filename']).as_posix()
- for file in filepaths]
+ tiff_filepaths = [
+ find_full_path(
+ get_imaging_root_data_dir(),
+ pathlib.Path(file["tiff_split_directory"]) / file["tiff_split_filename"],
+ ).as_posix()
+ for file in filepaths
+ ]
return tiff_filepaths
+
def get_processed_dir(processing_task_key, process_method):
- sess_key = (ImagingPipelineSession & processing_task_key).fetch1('KEY')
- bucket_scan_dir = (TiffSplit & sess_key &
- {'tiff_split': processing_task_key['scan_id']}).fetch1('tiff_split_directory')
- user_id = (subject.Subject & processing_task_key).fetch1('user_id')
+ sess_key = (ImagingPipelineSession & processing_task_key).fetch1("KEY")
+ bucket_scan_dir = (TiffSplit & sess_key & {"tiff_split": processing_task_key["scan_id"]}).fetch1(
+ "tiff_split_directory"
+ )
+ (subject.Subject & processing_task_key).fetch1("user_id")
sess_dir = find_full_path(get_imaging_root_data_dir(), bucket_scan_dir)
- relative_suite2p_dir = (pathlib.Path(bucket_scan_dir) / process_method).as_posix()
+ (pathlib.Path(bucket_scan_dir) / process_method).as_posix()
if not sess_dir.exists():
- raise FileNotFoundError(f'Session directory not found ({sess_dir})')
+ raise FileNotFoundError(f"Session directory not found ({sess_dir})")
- if process_method == 'suite2p':
+ if process_method == "suite2p":
# Check if ops.npy is inside suite2p_dir
- suite2p_dirs = set([fp.parent.parent for fp in sess_dir.rglob('*ops.npy')])
+ suite2p_dirs = {fp.parent.parent for fp in sess_dir.rglob("*ops.npy")}
if len(suite2p_dirs) != 1:
- raise FileNotFoundError(f'Error searching for Suite2p output directory in {bucket_scan_dir} - Found {suite2p_dirs}')
- elif process_method == 'caiman':
- raise NotImplementedError('CaImAn is not currented implemented.')
+ raise FileNotFoundError(
+ f"Error searching for Suite2p output directory in {bucket_scan_dir} - Found {suite2p_dirs}"
+ )
+ elif process_method == "caiman":
+ raise NotImplementedError("CaImAn is not currented implemented.")
return sess_dir
+
# 4. Activate imaging schema -----------------------------------------------------------
-imaging_element.activate(imaging_schema_name, scan_schema_name, linking_module=__name__)
\ No newline at end of file
+imaging_element.activate(imaging_schema_name, scan_schema_name, linking_module=__name__)
diff --git a/u19_pipeline/lab.py b/u19_pipeline/lab.py
index 855e9b47..71ef7a86 100644
--- a/u19_pipeline/lab.py
+++ b/u19_pipeline/lab.py
@@ -229,11 +229,35 @@ class Path(dj.Lookup):
contents = [
["/Bezos-center", "windows", "Y:", r"\\cup.pni.princeton.edu\Bezos-center", ""],
- ["/Bezos-center", "mac", "/Volumes/Bezos-center", "//cup.pni.princeton.edu/Bezos-center", ""],
- ["/Bezos-center", "linux", "/mnt/Bezos-center", "//cup.pni.princeton.edu/Bezos-center", ""],
+ [
+ "/Bezos-center",
+ "mac",
+ "/Volumes/Bezos-center",
+ "//cup.pni.princeton.edu/Bezos-center",
+ "",
+ ],
+ [
+ "/Bezos-center",
+ "linux",
+ "/mnt/Bezos-center",
+ "//cup.pni.princeton.edu/Bezos-center",
+ "",
+ ],
["/braininit", "windows", "Z:", r"\\cup.pni.princeton.edu\braininit", ""],
- ["/braininit", "mac", "/Volumes/braininit", "//cup.pni.princeton.edu/Bezos-center", ""],
- ["/braininit", "linux", "/mnt/braininit", "//cup.pni.princeton.edu/Bezos-center", ""],
+ [
+ "/braininit",
+ "mac",
+ "/Volumes/braininit",
+ "//cup.pni.princeton.edu/Bezos-center",
+ "",
+ ],
+ [
+ "/braininit",
+ "linux",
+ "/mnt/braininit",
+ "//cup.pni.princeton.edu/Bezos-center",
+ "",
+ ],
]
def get_local_path(self, path, local_os=None):
@@ -282,11 +306,7 @@ def get_local_path(self, path, local_os=None):
path = os.path.join(mapping[i, local], path[n + 1 :])
break
- if os.path.sep == "\\" and local_os.lower() != "glo":
- path = path.replace("/", "\\")
-
- else:
- path = path.replace("\\", "/")
+ path = path.replace("/", "\\") if os.path.sep == "\\" and local_os.lower() != "glo" else path.replace("\\", "/")
return path
diff --git a/u19_pipeline/lightsheet.py b/u19_pipeline/lightsheet.py
index 154eef7f..1efacefe 100644
--- a/u19_pipeline/lightsheet.py
+++ b/u19_pipeline/lightsheet.py
@@ -1,6 +1,6 @@
import datajoint as dj
-schema = dj.schema('U19_lightsheet')
+schema = dj.schema("U19_lightsheet")
@schema
@@ -11,7 +11,8 @@ class User(dj.Lookup):
---
princeton_email : varchar(50)
"""
-
+
+
@schema
class Experiment(dj.Manual):
definition = """ # Experiments performed using the light sheet microscope
@@ -33,10 +34,13 @@ class Experiment(dj.Manual):
channel555 : enum("","registration","cell_detection","probe_detection","injection_detection")
channel647 : enum("","registration","cell_detection","probe_detection","injection_detection")
channel790 : enum("","registration","cell_detection","probe_detection","injection_detection")
- """
+ """
+
@schema
-class IdiscoPlusClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data
+class IdiscoPlusClearing(
+ dj.Manual
+): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data
definition = """ # Periodic calibration data of the light sheet microscope
-> Experiment # experiment_id, the primary key from the Experiment() table
----
@@ -147,7 +151,9 @@ class IdiscoPlusClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table
@schema
-class IdiscoAbbreviatedClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data
+class IdiscoAbbreviatedClearing(
+ dj.Manual
+): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data
definition = """ # Periodic calibration data of the light sheet microscope
-> Experiment # experiment_id, the primary key from the Experiment() table
----
@@ -186,8 +192,11 @@ class IdiscoAbbreviatedClearing(dj.Manual): # dj.Manual is one of the 4 datajoin
clearing_notes = "" : varchar(500)
"""
+
@schema
-class IdiscoAbbreviatedRatClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data
+class IdiscoAbbreviatedRatClearing(
+ dj.Manual
+): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data
definition = """ # Abbreviated Rat clearing table
-> Experiment # experiment_id, the primary key from the Experiment() table
----
@@ -240,8 +249,11 @@ class IdiscoAbbreviatedRatClearing(dj.Manual): # dj.Manual is one of the 4 dataj
clearing_notes = "" : varchar(500)
"""
+
@schema
-class UdiscoClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data
+class UdiscoClearing(
+ dj.Manual
+): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data
definition = """ # uDISCO clearing table
-> Experiment # experiment_id, the primary key from the Experiment() table
----
@@ -270,4 +282,4 @@ class UdiscoClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table typ
time_clearing_babb_wash1 = NULL : datetime
clearing_babb_wash1_notes = "" : varchar(250)
clearing_notes = "" : varchar(500)
- """
\ No newline at end of file
+ """
diff --git a/u19_pipeline/meso.py b/u19_pipeline/meso.py
index 50e346e1..ece2a913 100644
--- a/u19_pipeline/meso.py
+++ b/u19_pipeline/meso.py
@@ -1,9 +1,7 @@
import datajoint as dj
import numpy as np
-from u19_pipeline import acquisition
-
-schema = dj.schema('u19_meso')
+schema = dj.schema("u19_meso")
@schema
@@ -109,8 +107,8 @@ class MotionCorrectionMethod(dj.Lookup):
"""
contents = [
- ['LinearNormalized', 'Normalized', 'Linear'],
- ['NonLinearNormalized', 'Normalized', 'Nonlinear']
+ ["LinearNormalized", "Normalized", "Linear"],
+ ["NonLinearNormalized", "Normalized", "Nonlinear"],
]
@@ -124,11 +122,11 @@ class McParameter(dj.Lookup):
"""
contents = [
- ['LinearNormalized', 'mc_max_shift', ''],
- ['LinearNormalized', 'mc_max_iter', ''],
- ['LinearNormalized', 'mc_stop_below_shift', ''],
- ['LinearNormalized', 'mc_black_tolerance', ''],
- ['LinearNormalized', 'mc_median_rebin', '']
+ ["LinearNormalized", "mc_max_shift", ""],
+ ["LinearNormalized", "mc_max_iter", ""],
+ ["LinearNormalized", "mc_stop_below_shift", ""],
+ ["LinearNormalized", "mc_black_tolerance", ""],
+ ["LinearNormalized", "mc_median_rebin", ""],
]
@@ -140,7 +138,7 @@ class McParameterSet(dj.Lookup):
mc_parameter_set_id : int # parameter set id
"""
- contents = [['LinearNormalized', 1]]
+ contents = [["LinearNormalized", 1]]
class Parameter(dj.Part):
definition = """
@@ -355,23 +353,15 @@ class Trace(dj.Imported):
"""
-
-
-if __name__ == '__main__':
-
- key = {'mcorr_method': 'LinearNormalized',
- 'mc_parameter_set_id': 1}
+if __name__ == "__main__":
+ key = {"mcorr_method": "LinearNormalized", "mc_parameter_set_id": 1}
parameters = [
- dict(**key, mc_parameter_name='mc_max_shift',
- mc_parameter_value=np.array([15.])),
- dict(**key, mc_parameter_name='mc_max_iter',
- mc_parameter_value=np.array([5.])),
- dict(**key, mc_parameter_name='mc_stop_below_shift',
- mc_parameter_value=0.3),
- dict(**key, mc_parameter_name='mc_black_tolerance',
- mc_parameter_value=-1.),
- dict(**key, mc_parameter_name='mc_median_rebin',
- mc_parameter_value=10.)]
+ dict(**key, mc_parameter_name="mc_max_shift", mc_parameter_value=np.array([15.0])),
+ dict(**key, mc_parameter_name="mc_max_iter", mc_parameter_value=np.array([5.0])),
+ dict(**key, mc_parameter_name="mc_stop_below_shift", mc_parameter_value=0.3),
+ dict(**key, mc_parameter_name="mc_black_tolerance", mc_parameter_value=-1.0),
+ dict(**key, mc_parameter_name="mc_median_rebin", mc_parameter_value=10.0),
+ ]
McParameterSet.Parameter.insert(parameters)
diff --git a/u19_pipeline/meso_analysis.py b/u19_pipeline/meso_analysis.py
index be1dd5f9..7da59d5d 100644
--- a/u19_pipeline/meso_analysis.py
+++ b/u19_pipeline/meso_analysis.py
@@ -1,7 +1,6 @@
import datajoint as dj
-from u19_pipeline import meso
-schema = dj.schema('u19_meso_analysis')
+schema = dj.schema("u19_meso_analysis")
@schema
diff --git a/u19_pipeline/microsocope.py b/u19_pipeline/microsocope.py
index d6b2a5e4..00d52050 100644
--- a/u19_pipeline/microsocope.py
+++ b/u19_pipeline/microsocope.py
@@ -1,6 +1,7 @@
import datajoint as dj
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'microscope')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "microscope")
+
@schema
class Center(dj.Lookup):
@@ -9,9 +10,8 @@ class Center(dj.Lookup):
---
description = "" : varchar(500)
"""
- contents = [
- ["Bezos Center",""],["McDonnell Center",""]
- ]
+ contents = [["Bezos Center", ""], ["McDonnell Center", ""]]
+
@schema
class Microscope(dj.Lookup):
@@ -25,6 +25,7 @@ class Microscope(dj.Lookup):
microscope_description='' : varchar(2047)
"""
+
@schema
class Laser(dj.Manual):
definition = """
@@ -138,9 +139,9 @@ class DaqSystemType(dj.Lookup):
daq_notes='': varchar(255)
"""
contents = [
- ['National Instrument PXI', ''],
- ['National Instrument PCI', ''],
- ['Vidrio hardware', '']
+ ["National Instrument PXI", ""],
+ ["National Instrument PCI", ""],
+ ["Vidrio hardware", ""],
]
@@ -256,6 +257,7 @@ class AcquisitionSoftwareStatus(dj.Manual):
-> AcquisitionSoftware
"""
+
@schema
class PsfStackData(dj.Manual):
definition = """
diff --git a/u19_pipeline/puffs.py b/u19_pipeline/puffs.py
index ee8fe6f7..d52df1c8 100644
--- a/u19_pipeline/puffs.py
+++ b/u19_pipeline/puffs.py
@@ -1,9 +1,8 @@
"""This module defines tables in the schema ahoag_puffs_lab_demo"""
import datajoint as dj
-from . import lab, acquisition, task
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'puffs')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "puffs")
@schema
@@ -14,10 +13,7 @@ class Rig(dj.Lookup):
-> lab.location
"""
# "wang-behavior"
- contents = [
- [0, "pni-ltl016-05"],
- [1, "wang-behavior"]
- ]
+ contents = [[0, "pni-ltl016-05"], [1, "wang-behavior"]]
@schema
diff --git a/u19_pipeline/pupillometry.py b/u19_pipeline/pupillometry.py
index 40efaa0a..b5606e17 100644
--- a/u19_pipeline/pupillometry.py
+++ b/u19_pipeline/pupillometry.py
@@ -2,7 +2,7 @@
from u19_pipeline import acquisition
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'pupillometry')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "pupillometry")
@schema
@@ -26,10 +26,10 @@ class PupillometrySession(dj.Imported):
-> acquisition.Session
---
"""
-
+
@property
def key_source(self):
- return acquisition.SessionVideo & {'video_type': 'pupillometry'}
+ return acquisition.SessionVideo & {"video_type": "pupillometry"}
def make(self, key):
pass
@@ -46,7 +46,7 @@ class PupillometrySyncBehavior(dj.Imported):
"""
def make(self, key):
- pass
+ pass
@schema
@@ -69,4 +69,4 @@ class PupillometrySessionModelData(dj.Imported):
"""
def make(self, key):
- pass
\ No newline at end of file
+ pass
diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py
index 6748f93d..b23a2776 100644
--- a/u19_pipeline/recording.py
+++ b/u19_pipeline/recording.py
@@ -1,15 +1,14 @@
import datajoint as dj
-import copy
-from u19_pipeline import lab, subject, acquisition
import u19_pipeline.automatic_job.params_config as config
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'recording')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "recording")
+
# Declare recording tables -------------------------------------------------------------
@schema
class Modality(dj.Lookup):
- definition = """
+ definition = """
recording_modality: varchar(64) # recording modalities
# (ephys, imaging, video_recording, etc.)
---
@@ -27,22 +26,22 @@ class Modality(dj.Lookup):
process_repository: varchar(64) # Name of the repository that handles the
# processing of these modality
"""
- contents = config.recording_modality_list
+ contents = config.recording_modality_list
@schema
class Status(dj.Lookup):
- definition = """
+ definition = """
status_recording_id: TINYINT(1) # Status in the automatic processing pipeline
---
status_recording_definition: VARCHAR(256) # Status definition
"""
- contents = config.recording_status_list
+ contents = config.recording_status_list
@schema
class Recording(dj.Manual):
- definition = """
+ definition = """
recording_id: INT(11) AUTO_INCREMENT # Unique number assigned to recording
---
-> Modality
@@ -51,17 +50,17 @@ class Recording(dj.Manual):
task_copy_id_pni=null: int(11) # globus transfer task raw file local->cup
recording_directory: varchar(255) # relative directory on cup
local_directory: varchar(255) # local directory where the recording is stored on system
- """
+ """
- class BehaviorSession(dj.Part):
- definition = """
+ class BehaviorSession(dj.Part):
+ definition = """
-> master
---
-> acquisition.Session
"""
-
- class RecordingSession(dj.Part):
- definition = """
+
+ class RecordingSession(dj.Part):
+ definition = """
-> master
---
-> subject.Subject
@@ -71,7 +70,7 @@ class RecordingSession(dj.Part):
@schema
class LogStatus(dj.Manual):
- definition = """
+ definition = """
recording_log_id: INT(11) AUTO_INCREMENT # Unique number assigned to each change
# of status for all recordings
---
@@ -86,7 +85,7 @@ class LogStatus(dj.Manual):
@schema
class DefaultParams(dj.Manual):
- definition = """
+ definition = """
-> Recording
fragment_number: TINYINT(1) # probe/field_of_view # if not always the same
-----
@@ -96,50 +95,58 @@ class DefaultParams(dj.Manual):
paramset_idx: INT(11) # params index for recording (could be imaging/ephys)
"""
- @staticmethod
- def get_default_params_rec_process(recording_processes, default_params_record_df):
- 'Get associated params from DefaultParams record and recording processes (jobs) of recording'
-
- params_rec_process = list()
- for i in recording_processes:
-
- this_params_rec_process = dict()
- this_params_rec_process['job_id'] = i['job_id']
- this_fragment = i['fragment_number']
-
- this_params_rec_process['preprocess_param_steps_id'] = \
- DefaultParams.get_corresponding_param(default_params_record_df, this_fragment, 'default_same_preparams_all', 'preprocess_param_steps_id')
-
- this_params_rec_process['paramset_idx'] = \
- DefaultParams.get_corresponding_param(default_params_record_df, this_fragment, 'default_same_params_all', 'paramset_idx')
-
- params_rec_process.append(this_params_rec_process)
-
- return params_rec_process
-
-
- @staticmethod
- def get_corresponding_param(default_params_record_df, this_fragment, default_label, param_label):
- 'Get corresponding param (preprocess_param_steps_id or paramset_idx) for this fragment'
-
- 'default_label = default_same_preparams_all / default_same_params_all '
- 'param_label = preprocess_param_steps_id / paramset_idx '
-
- #If there is no default params for this recording, get default ones (0 id)
- if default_params_record_df.shape[0] == 0:
- this_fragment_preprocess_param_steps_id = 0
- return this_fragment_preprocess_param_steps_id
-
- else:
- if default_params_record_df.loc[0, default_label] == 1:
+ @staticmethod
+ def get_default_params_rec_process(recording_processes, default_params_record_df):
+ "Get associated params from DefaultParams record and recording processes (jobs) of recording"
+
+ params_rec_process = []
+ for i in recording_processes:
+ this_params_rec_process = {}
+ this_params_rec_process["job_id"] = i["job_id"]
+ this_fragment = i["fragment_number"]
+
+ this_params_rec_process["preprocess_param_steps_id"] = DefaultParams.get_corresponding_param(
+ default_params_record_df,
+ this_fragment,
+ "default_same_preparams_all",
+ "preprocess_param_steps_id",
+ )
+
+ this_params_rec_process["paramset_idx"] = DefaultParams.get_corresponding_param(
+ default_params_record_df,
+ this_fragment,
+ "default_same_params_all",
+ "paramset_idx",
+ )
+
+ params_rec_process.append(this_params_rec_process)
+
+ return params_rec_process
+
+ @staticmethod
+ def get_corresponding_param(default_params_record_df, this_fragment, default_label, param_label):
+ "Get corresponding param (preprocess_param_steps_id or paramset_idx) for this fragment"
+
+ "default_label = default_same_preparams_all / default_same_params_all "
+ "param_label = preprocess_param_steps_id / paramset_idx "
+
+ # If there is no default params for this recording, get default ones (0 id)
+ if default_params_record_df.shape[0] == 0:
+ this_fragment_preprocess_param_steps_id = 0
+ return this_fragment_preprocess_param_steps_id
+
+ else:
+ if default_params_record_df.loc[0, default_label] == 1:
+ this_fragment_preprocess_param_steps_id = default_params_record_df.loc[0, param_label]
+ else:
+ this_fragment_preprocess_param_steps_id = default_params_record_df.loc[
+ default_params_record_df["fragment_number"] == this_fragment,
+ param_label,
+ ]
+ # If there is no list id for this specific fragment, get default one
+ if this_fragment_preprocess_param_steps_id.shape[0] == 0:
this_fragment_preprocess_param_steps_id = default_params_record_df.loc[0, param_label]
- else:
- this_fragment_preprocess_param_steps_id = \
- default_params_record_df.loc[default_params_record_df['fragment_number'] == this_fragment, param_label]
- #If there is no list id for this specific fragment, get default one
- if this_fragment_preprocess_param_steps_id.shape[0] == 0:
- this_fragment_preprocess_param_steps_id = default_params_record_df.loc[0, param_label]
- else:
- this_fragment_preprocess_param_steps_id = this_fragment_preprocess_param_steps_id.values[0]
-
- return this_fragment_preprocess_param_steps_id
+ else:
+ this_fragment_preprocess_param_steps_id = this_fragment_preprocess_param_steps_id.values[0]
+
+ return this_fragment_preprocess_param_steps_id
diff --git a/u19_pipeline/recording_process.py b/u19_pipeline/recording_process.py
index 3efaef30..b30275e7 100644
--- a/u19_pipeline/recording_process.py
+++ b/u19_pipeline/recording_process.py
@@ -1,28 +1,26 @@
-
import pathlib
import datajoint as dj
-from u19_pipeline import recording
-from u19_pipeline.imaging_pipeline import imaging_element
-from u19_pipeline.ephys_pipeline import ephys_element
+
import u19_pipeline.automatic_job.params_config as config
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'recording_process')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "recording_process")
+
# Declare recording processing tables --------------------------------------------------
@schema
class Status(dj.Lookup):
- definition = """
+ definition = """
status_processing_id: TINYINT(1) # Status in the automatic processing pipeline
---
status_processing_definition: VARCHAR(256) # Status definition
"""
- contents = config.recording_process_status_list
+ contents = config.recording_process_status_list
@schema
class Processing(dj.Manual):
- definition = """
+ definition = """
job_id: INT(11) AUTO_INCREMENT
# Unique number assigned to each
# processing job for a recording
@@ -43,66 +41,68 @@ class Processing(dj.Manual):
task_copy_id_post=null: UUID # id for globus transfer task
# sorted file tiger->cup
slurm_id=null: VARCHAR(16) # id for slurm process in tiger
- """
+ """
- class EphysParams(dj.Part):
- definition="""
+ class EphysParams(dj.Part):
+ definition = """
-> master
---
-> ephys_element.PreClusterParamSteps
-> ephys_element.ClusteringParamSet.proj(cluster_paramset_idx='paramset_idx')
"""
- class ImagingParams(dj.Part):
- definition="""
+ class ImagingParams(dj.Part):
+ definition = """
-> master
---
-> imaging_element.PreProcessParamSteps
-> imaging_element.ProcessingParamSet
- """
+ """
- # This table would control ingestion of PreClusteringTask
- def insert_recording_process(self, fragment_keys, fragment_fieldname):
- '''
+ # This table would control ingestion of PreClusteringTask
+ def insert_recording_process(self, fragment_keys, fragment_fieldname):
+ """
# Insert RecordingTask(s) from recording.
# For each processing "unit" of a recording add a new RecordingTask (imaging ->field of view, electrophysiology->probe)
Input:
recording_key (dict) = Dictionary with recording record
rec_unit (dict) = Dictionary of recording "unit" to be processed
unit_fieldname (str) = Unit fieldname to be read (ephys-> probe_, imaging->fov)
- '''
+ """
# Get directory fieldname for specific modality (probe_directory, fov_directory, etc.)
# Append data for the unit to insert
- this_recprocess_key = list()
+ this_recprocess_key = []
for idx, fragment_key in enumerate(fragment_keys):
- this_recprocess_key.append(dict())
- this_recprocess_key[idx]['recording_id'] = fragment_key['recording_id']
- this_recprocess_key[idx]['fragment_number'] = fragment_key[fragment_fieldname]
- this_recprocess_key[idx]['status_processing_id'] = 0
- this_recprocess_key[idx]['recording_process_pre_path'] = fragment_key['recording_process_pre_path']
+ this_recprocess_key.append({})
+ this_recprocess_key[idx]["recording_id"] = fragment_key["recording_id"]
+ this_recprocess_key[idx]["fragment_number"] = fragment_key[fragment_fieldname]
+ this_recprocess_key[idx]["status_processing_id"] = 0
+ this_recprocess_key[idx]["recording_process_pre_path"] = fragment_key["recording_process_pre_path"]
self.insert(this_recprocess_key)
- def set_recording_process_post_path(self, recprocess_keys):
- '''
- # Update recording_process_post_path from recording_process keys
- Input:
- recprocess_keys (dict) = Dictionary with job_ids & recording_process_pre_path
- '''
+ def set_recording_process_post_path(self, recprocess_keys):
+ """
+ # Update recording_process_post_path from recording_process keys
+ Input:
+ recprocess_keys (dict) = Dictionary with job_ids & recording_process_pre_path
+ """
- for rec_process in recprocess_keys:
- this_key_dict = dict()
- this_key_dict['job_id'] = rec_process['job_id']
- this_key_dict['recording_process_post_path'] = \
- pathlib.Path(rec_process['recording_process_pre_path'], 'job_id_' + str(rec_process['job_id'])).as_posix()
- self.update1(this_key_dict)
+ for rec_process in recprocess_keys:
+ this_key_dict = {}
+ this_key_dict["job_id"] = rec_process["job_id"]
+ this_key_dict["recording_process_post_path"] = pathlib.Path(
+ rec_process["recording_process_pre_path"],
+ "job_id_" + str(rec_process["job_id"]),
+ ).as_posix()
+ self.update1(this_key_dict)
@schema
class LogStatus(dj.Manual):
- definition = """
+ definition = """
log_id: INT(11) AUTO_INCREMENT # Unique number assigned to each change
# of status for all processing jobs
---
diff --git a/u19_pipeline/reference.py b/u19_pipeline/reference.py
index 5539a19a..1285dd4d 100644
--- a/u19_pipeline/reference.py
+++ b/u19_pipeline/reference.py
@@ -1,9 +1,8 @@
"""This module defines tables in the schema U19_reference."""
-
import datajoint as dj
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'reference')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "reference")
@schema
@@ -14,10 +13,10 @@ class BrainArea(dj.Lookup):
area_full_name="" : varchar(128)
"""
contents = [
- ['Hippocampus', ''],
- ['Striatum', ''],
- ['PPC', 'Posterior Parietal Cortex'],
- ['EC', 'Entorhinal Cortex']
+ ["Hippocampus", ""],
+ ["Striatum", ""],
+ ["PPC", "Posterior Parietal Cortex"],
+ ["EC", "Entorhinal Cortex"],
]
@@ -26,15 +25,7 @@ class VirusSource(dj.Lookup):
definition = """
virus_source : varchar(32)
"""
- contents = zip([
- 'UNC',
- 'UPenn',
- 'Addgene',
- 'MIT',
- 'Stanford',
- 'Princeton',
- 'custom'
- ])
+ contents = zip(["UNC", "UPenn", "Addgene", "MIT", "Stanford", "Princeton", "custom"])
@schema
@@ -42,12 +33,7 @@ class VirusType(dj.Lookup):
definition = """
virus_type : varchar(64)
"""
- contents = zip([
- 'AAV',
- 'Rabies',
- 'Psedotyped rabies',
- 'Lenti'
- ])
+ contents = zip(["AAV", "Rabies", "Psedotyped rabies", "Lenti"])
@schema
@@ -65,14 +51,74 @@ class Virus(dj.Lookup):
virus_description="" : varchar(512)
"""
contents = [
- ['jRCaMP1a', 1, 'AAV1.Syn.NES.jRCaMP1a.WPRE.SV40', 'AAV', 'Upenn', '', 33.6, None, ''],
- ['GCaMP6f', 1, 'AAV1.Syn.GCaMP6f.WPRE.SV40', 'AAV', 'UPenn', '', 26.5, None, ''],
- ['Syn.RFP', 1, 'AAV5.hSyn.TurboRFP.WPRE.rBG', 'AAV', 'UPenn', '', 44, None, ''],
- ['GFAP.GFP', 1, 'AAV5.GFAP.eGFP.WPRE.hGH', 'AAV', 'UPenn', '', 10.6, None, ''],
- ['CamKII.GFP', 1, 'AAV9.CamKII0.4.eGFP.WPRE.rBG', 'AAV', 'UPenn', '', 34.9, None, ''],
- ['Syn.RFP', 1, 'AAV9.hSyn.TurboRFP.WPRE.rBG', 'AAV', 'UPenn', '', 66.4, None, ''],
- ['ChR2(H134R)-YFP', 1, 'AAV9.hSyn.hChR2(H134R)-eYFP.WPRE.hGH', 'AAV', 'UPenn', '', 33.9, None, ''],
- ['Chronos-GFP', 1, 'AAV9.Syn.Chronos-GFP.WPRE.bGH', 'AAV', 'UPenn', '', 35.1, None, '']
+ [
+ "jRCaMP1a",
+ 1,
+ "AAV1.Syn.NES.jRCaMP1a.WPRE.SV40",
+ "AAV",
+ "Upenn",
+ "",
+ 33.6,
+ None,
+ "",
+ ],
+ [
+ "GCaMP6f",
+ 1,
+ "AAV1.Syn.GCaMP6f.WPRE.SV40",
+ "AAV",
+ "UPenn",
+ "",
+ 26.5,
+ None,
+ "",
+ ],
+ ["Syn.RFP", 1, "AAV5.hSyn.TurboRFP.WPRE.rBG", "AAV", "UPenn", "", 44, None, ""],
+ ["GFAP.GFP", 1, "AAV5.GFAP.eGFP.WPRE.hGH", "AAV", "UPenn", "", 10.6, None, ""],
+ [
+ "CamKII.GFP",
+ 1,
+ "AAV9.CamKII0.4.eGFP.WPRE.rBG",
+ "AAV",
+ "UPenn",
+ "",
+ 34.9,
+ None,
+ "",
+ ],
+ [
+ "Syn.RFP",
+ 1,
+ "AAV9.hSyn.TurboRFP.WPRE.rBG",
+ "AAV",
+ "UPenn",
+ "",
+ 66.4,
+ None,
+ "",
+ ],
+ [
+ "ChR2(H134R)-YFP",
+ 1,
+ "AAV9.hSyn.hChR2(H134R)-eYFP.WPRE.hGH",
+ "AAV",
+ "UPenn",
+ "",
+ 33.9,
+ None,
+ "",
+ ],
+ [
+ "Chronos-GFP",
+ 1,
+ "AAV9.Syn.Chronos-GFP.WPRE.bGH",
+ "AAV",
+ "UPenn",
+ "",
+ 35.1,
+ None,
+ "",
+ ],
]
@@ -86,7 +132,6 @@ class Template(dj.Lookup):
database_field_names : longblob # cell array for database field names
"""
-
class RightNow(dj.Part):
definition = """
# ActionItems template for the information of each column
@@ -96,7 +141,6 @@ class RightNow(dj.Part):
plot_index : int # index of where to plot in GUI
"""
-
class Genotype(dj.Part):
definition = """
# ActionItems template for the information of each column
@@ -106,7 +150,6 @@ class Genotype(dj.Part):
plot_index : int # index of where to plot in GUI
"""
-
class DailyInfo(dj.Part):
definition = """
# Daily Info template for the information of each column
@@ -124,7 +167,6 @@ class DailyInfo(dj.Part):
plot_index : int # index of where to plot in GUI
"""
-
class Animal(dj.Part):
definition = """
# Animal template for the information of each column
@@ -141,7 +183,6 @@ class Animal(dj.Part):
plot_index : int # index of where to plot in GUI
"""
-
class ActionItems(dj.Part):
definition = """
# ActionItems template for the information of each column
diff --git a/u19_pipeline/rig_maintenance.py b/u19_pipeline/rig_maintenance.py
index 504f5d1c..c93f2b0b 100644
--- a/u19_pipeline/rig_maintenance.py
+++ b/u19_pipeline/rig_maintenance.py
@@ -28,10 +28,22 @@ class MaintenanceType(dj.Lookup):
contents = [
["Replacing lines", "Replacing water/reward lines", 30, 7, 1],
["Replacing spheres", "Replacing spheres in the rig", 60, 7, 0],
- ["General Calibration", "General calibration (always after changing lines or solenoids)", 30, 7, 0],
+ [
+ "General Calibration",
+ "General calibration (always after changing lines or solenoids)",
+ 30,
+ 7,
+ 0,
+ ],
["Replacing solenoid valve", "Replacing solenoid valve", 365, 30, 1],
["Deep Clean solenoid valve", "Deep cleaning of solenoid valve", 30, 7, 1],
- ["Replacing reward lines connectors", "Replacing reward lines connectors", 180, 14, 0],
+ [
+ "Replacing reward lines connectors",
+ "Replacing reward lines connectors",
+ 180,
+ 14,
+ 0,
+ ],
]
diff --git a/u19_pipeline/scheduler.py b/u19_pipeline/scheduler.py
index ea794671..aa89aa2a 100644
--- a/u19_pipeline/scheduler.py
+++ b/u19_pipeline/scheduler.py
@@ -125,6 +125,7 @@ class InputOutputProfileList(dj.Manual):
check_type : enum('Mandatory','Optional') # Prevent training if missing this input/output
"""
+
@schema
class RigStatus(dj.Manual):
definition = """
@@ -137,6 +138,7 @@ class RigStatus(dj.Manual):
last_status_update : datetime # at what time status changed
"""
+
@schema
class Shift(dj.Lookup):
definition = """
diff --git a/u19_pipeline/subject.py b/u19_pipeline/subject.py
index bf07ba7a..dbb6a1ee 100644
--- a/u19_pipeline/subject.py
+++ b/u19_pipeline/subject.py
@@ -1,6 +1,5 @@
"""This module defines tables in the schema U19_subject"""
-
import datajoint as dj
prefix = dj.config["custom"]["database.prefix"]
@@ -22,9 +21,7 @@ class Species(dj.Lookup):
---
species_nickname : varchar(32) # nickname
"""
- contents = [
- ['Mus musculus', 'Laboratory mouse']
- ]
+ contents = [["Mus musculus", "Laboratory mouse"]]
@schema
@@ -34,11 +31,7 @@ class Source(dj.Lookup):
---
source_description="" : varchar(255)
"""
- contents = [
- ['Jax Lab', ''],
- ['Princeton', ''],
- ['Allen Institute', '']
- ]
+ contents = [["Jax Lab", ""], ["Princeton", ""], ["Allen Institute", ""]]
@schema
@@ -48,9 +41,7 @@ class Strain(dj.Lookup):
---
strain_description="" : varchar(255) # description
"""
- contents = [
- ['C57BL6/J', '']
- ]
+ contents = [["C57BL6/J", ""]]
@schema
@@ -64,16 +55,40 @@ class Allele(dj.Lookup):
allele_description="" : varchar(1023)
"""
contents = [
- ['Thy1-GP5.3', 'Thy1-GCaMP6f', 'Jax Lab', 'David W. Tank, Princeton University', ''],
- ['Ai93', '(TITL-GCaMP6f)-D', 'Jax Lab', 'Allen Institute', ''],
- ['Ai148', '(TIT2L-GC6f-ICL-tTA2)-D', 'Jax Lab', 'Allen Institute', ''],
- ['Thy1-YFP', '', 'Jax Lab', 'Joshua R Sanes, Harvard University', ''],
- ['VGAT-ChR2-EYFP', 'Slc32a1-COP4*H134R/EYFP', 'Jax Lab', 'Guoping Feng, Massachusetts Institute of Technology', ''],
- ['Emx1-Cre', 'Emx-IRES-Cre', 'Jax Lab', 'Kevin R. Jones, University of Colorado- Boulder', ''],
- ['D1-Cre', 'Drd1-cre', 'Jax Lab', 'Ming Xu, University of Chicago', ''],
- ['D2-Cre', 'Drd2-cre', 'Jax Lab', 'Unknown', ''],
- ['DAT-IRES-Cre', 'Slc6a3tm1.1(cre)Bkmn', 'Jax Lab', 'Cristina M Backman, National Institute on Drug Abuse (NIH)', ''],
- ['Slc17a7-IRES2-Cre', '', 'Jax Lab', 'Allen Institute', '']
+ [
+ "Thy1-GP5.3",
+ "Thy1-GCaMP6f",
+ "Jax Lab",
+ "David W. Tank, Princeton University",
+ "",
+ ],
+ ["Ai93", "(TITL-GCaMP6f)-D", "Jax Lab", "Allen Institute", ""],
+ ["Ai148", "(TIT2L-GC6f-ICL-tTA2)-D", "Jax Lab", "Allen Institute", ""],
+ ["Thy1-YFP", "", "Jax Lab", "Joshua R Sanes, Harvard University", ""],
+ [
+ "VGAT-ChR2-EYFP",
+ "Slc32a1-COP4*H134R/EYFP",
+ "Jax Lab",
+ "Guoping Feng, Massachusetts Institute of Technology",
+ "",
+ ],
+ [
+ "Emx1-Cre",
+ "Emx-IRES-Cre",
+ "Jax Lab",
+ "Kevin R. Jones, University of Colorado- Boulder",
+ "",
+ ],
+ ["D1-Cre", "Drd1-cre", "Jax Lab", "Ming Xu, University of Chicago", ""],
+ ["D2-Cre", "Drd2-cre", "Jax Lab", "Unknown", ""],
+ [
+ "DAT-IRES-Cre",
+ "Slc6a3tm1.1(cre)Bkmn",
+ "Jax Lab",
+ "Cristina M Backman, National Institute on Drug Abuse (NIH)",
+ "",
+ ],
+ ["Slc17a7-IRES2-Cre", "", "Jax Lab", "Allen Institute", ""],
]
@@ -85,11 +100,11 @@ class SequenceType(dj.Lookup):
seq_type_description="" : varchar(255)
"""
contents = [
- ['calcium sensor', ''],
- ['optogenetics', ''],
- ['promoter', ''],
- ['recombinase', ''],
- ['fluorescent protein', '']
+ ["calcium sensor", ""],
+ ["optogenetics", ""],
+ ["promoter", ""],
+ ["recombinase", ""],
+ ["fluorescent protein", ""],
]
@@ -103,15 +118,15 @@ class Sequence(dj.Lookup):
sequence_description="" : varchar(255)
"""
contents = [
- ['GCaMP6f', 'calcium sensor', '', ''],
- ['GCaMP6s', 'calcium sensor', '', ''],
- ['ChR2', 'optogenetics', '', ''],
- ['EYFP', 'fluorescent protein', '', ''],
- ['Thy1', 'promoter', '', ''],
- ['Emx1', 'promoter', '', ''],
- ['Cre', 'recombinase', '', ''],
- ['D1', 'promoter', '', 'dopamine receptor type 1'],
- ['D2', 'promoter', '', 'dopamine receptor type 2']
+ ["GCaMP6f", "calcium sensor", "", ""],
+ ["GCaMP6s", "calcium sensor", "", ""],
+ ["ChR2", "optogenetics", "", ""],
+ ["EYFP", "fluorescent protein", "", ""],
+ ["Thy1", "promoter", "", ""],
+ ["Emx1", "promoter", "", ""],
+ ["Cre", "recombinase", "", ""],
+ ["D1", "promoter", "", "dopamine receptor type 1"],
+ ["D2", "promoter", "", "dopamine receptor type 2"],
]
@@ -122,19 +137,19 @@ class AlleleSequence(dj.Lookup):
-> Sequence
"""
contents = [
- ['Thy1-YFP', 'Thy1'],
- ['Thy1-YFP', 'EYFP'],
- ['Thy1-GP5.3', 'GCaMP6f'],
- ['VGAT-ChR2-EYFP', 'EYFP'],
- ['VGAT-ChR2-EYFP', 'ChR2'],
- ['Emx1-Cre', 'Emx1'],
- ['Emx1-Cre', 'Cre'],
- ['D1-Cre', 'D1'],
- ['D1-Cre', 'Cre'],
- ['D2-Cre', 'D2'],
- ['D2-Cre', 'Cre'],
- ['DAT-IRES-Cre', 'Cre'],
- ['Slc17a7-IRES2-Cre', 'Cre']
+ ["Thy1-YFP", "Thy1"],
+ ["Thy1-YFP", "EYFP"],
+ ["Thy1-GP5.3", "GCaMP6f"],
+ ["VGAT-ChR2-EYFP", "EYFP"],
+ ["VGAT-ChR2-EYFP", "ChR2"],
+ ["Emx1-Cre", "Emx1"],
+ ["Emx1-Cre", "Cre"],
+ ["D1-Cre", "D1"],
+ ["D1-Cre", "Cre"],
+ ["D2-Cre", "D2"],
+ ["D2-Cre", "Cre"],
+ ["DAT-IRES-Cre", "Cre"],
+ ["Slc17a7-IRES2-Cre", "Cre"],
]
@@ -150,17 +165,17 @@ class Line(dj.Lookup):
is_active=1 : tinyint # is active
"""
contents = [
- ['Unknown', 'Mus musculus', 'C57BL6/J', '', '', 1],
- ['C57BL6/J', 'Mus musculus', 'C57BL6/J', 'wild-type mice', '', 1],
- ['VGAT-ChR2-EYFP', 'Mus musculus', 'C57BL6/J', '', '', 1],
- ['Ai93-Emx1', 'Mus musculus', 'C57BL6/J', '', '', 1],
- ['Thy1-GP5.3', 'Mus musculus', 'C57BL6/J', '', '', 1],
- ['Thy1-YFP', 'Mus musculus', 'C57BL6/J', '', '', 1],
- ['DAT-IRES-CRE', 'Mus musculus', 'C57BL6/J', '', '', 1],
- ['DAT-Ai148', 'Mus musculus', 'C57BL6/J', '', '', 1],
- ['Slc17a7-Ai148', 'Mus musculus', 'C57BL6/J', '', '', 1],
- ['D1-CRE', 'Mus musculus', 'C57BL6/J', '', '', 1],
- ['D2-CRE', 'Mus musculus', 'C57BL6/J', '', '', 1]
+ ["Unknown", "Mus musculus", "C57BL6/J", "", "", 1],
+ ["C57BL6/J", "Mus musculus", "C57BL6/J", "wild-type mice", "", 1],
+ ["VGAT-ChR2-EYFP", "Mus musculus", "C57BL6/J", "", "", 1],
+ ["Ai93-Emx1", "Mus musculus", "C57BL6/J", "", "", 1],
+ ["Thy1-GP5.3", "Mus musculus", "C57BL6/J", "", "", 1],
+ ["Thy1-YFP", "Mus musculus", "C57BL6/J", "", "", 1],
+ ["DAT-IRES-CRE", "Mus musculus", "C57BL6/J", "", "", 1],
+ ["DAT-Ai148", "Mus musculus", "C57BL6/J", "", "", 1],
+ ["Slc17a7-Ai148", "Mus musculus", "C57BL6/J", "", "", 1],
+ ["D1-CRE", "Mus musculus", "C57BL6/J", "", "", 1],
+ ["D2-CRE", "Mus musculus", "C57BL6/J", "", "", 1],
]
@@ -171,18 +186,18 @@ class LineAllele(dj.Lookup):
-> Allele
"""
contents = [
- ['VGAT-ChR2-EYFP', 'VGAT-ChR2-EYFP'],
- ['Ai93-Emx1', 'Ai93'],
- ['Ai93-Emx1', 'Emx1-Cre'],
- ['Thy1-GP5.3', 'Thy1-GP5.3'],
- ['Thy1-YFP', 'Thy1-YFP'],
- ['DAT-IRES-CRE', 'DAT-IRES-Cre'],
- ['DAT-Ai148', 'DAT-IRES-Cre'],
- ['DAT-Ai148', 'Ai148'],
- ['Slc17a7-Ai148', 'Ai148'],
- ['Slc17a7-Ai148', 'Slc17a7-IRES2-Cre'],
- ['D1-CRE', 'D1-Cre'],
- ['D2-CRE', 'D2-Cre']
+ ["VGAT-ChR2-EYFP", "VGAT-ChR2-EYFP"],
+ ["Ai93-Emx1", "Ai93"],
+ ["Ai93-Emx1", "Emx1-Cre"],
+ ["Thy1-GP5.3", "Thy1-GP5.3"],
+ ["Thy1-YFP", "Thy1-YFP"],
+ ["DAT-IRES-CRE", "DAT-IRES-Cre"],
+ ["DAT-Ai148", "DAT-IRES-Cre"],
+ ["DAT-Ai148", "Ai148"],
+ ["Slc17a7-Ai148", "Ai148"],
+ ["Slc17a7-Ai148", "Slc17a7-IRES2-Cre"],
+ ["D1-CRE", "D1-Cre"],
+ ["D2-CRE", "D2-Cre"],
]
@@ -210,11 +225,15 @@ class ActItem(dj.Lookup):
definition = """
act_item : varchar(64) # possible act item
"""
- contents = zip(['Post-op painkillers',
- 'Post-op monitoring',
- 'Acclimatize to humans',
- 'Topical antibiotics for headplate scab',
- 'Needs fattening'])
+ contents = zip(
+ [
+ "Post-op painkillers",
+ "Post-op monitoring",
+ "Acclimatize to humans",
+ "Topical antibiotics for headplate scab",
+ "Needs fattening",
+ ]
+ )
@schema
@@ -224,6 +243,7 @@ class SubjectActionManual(dj.Manual):
-> ActItem
"""
+
@schema
class SubjectActionAutomatic(dj.Manual):
definition = """
@@ -233,6 +253,7 @@ class SubjectActionAutomatic(dj.Manual):
notification_message : varchar(255) # Notification message e.g. low bodyweight warning
"""
+
@schema
class SubjectProject(dj.Manual):
definition = """
@@ -295,6 +316,7 @@ class CagingStatus(dj.Manual):
-> Cage
"""
+
@schema
class HealthStatus(dj.Manual):
definition = """
@@ -310,7 +332,6 @@ class HealthStatus(dj.Manual):
comments=null : varchar(255)
"""
-
class Action(dj.Part):
definition = """
-> HealthStatus
@@ -375,6 +396,7 @@ class Zygosity(dj.Manual):
zygosity : enum('Present','Absent','Homozygous','Heterozygous')
"""
+
@schema
class HeadMotorPosition(dj.Manual):
definition = """
diff --git a/u19_pipeline/task.py b/u19_pipeline/task.py
index c7e6b1cd..ae421e1f 100644
--- a/u19_pipeline/task.py
+++ b/u19_pipeline/task.py
@@ -2,8 +2,7 @@
import datajoint as dj
-
-schema = dj.schema(dj.config['custom']['database.prefix'] + 'task')
+schema = dj.schema(dj.config["custom"]["database.prefix"] + "task")
@schema
@@ -13,12 +12,7 @@ class Task(dj.Lookup):
---
task_description="" : varchar(512)
"""
- contents = [
- ['AirPuffs', ''],
- ['Towers', ''],
- ['Clicks', ''],
- ['LinearTrack', '']
- ]
+ contents = [["AirPuffs", ""], ["Towers", ""], ["Clicks", ""], ["LinearTrack", ""]]
@schema
diff --git a/u19_pipeline/temp/acquisition.py b/u19_pipeline/temp/acquisition.py
index 87609495..1169de95 100644
--- a/u19_pipeline/temp/acquisition.py
+++ b/u19_pipeline/temp/acquisition.py
@@ -1,8 +1,6 @@
import datajoint as dj
-from u19_pipeline import lab, task, subject
-
-schema = dj.schema('u19_acquisition')
+schema = dj.schema("u19_acquisition")
@schema
diff --git a/u19_pipeline/temp/behavior.py b/u19_pipeline/temp/behavior.py
index 47354b82..a7c0cd4f 100644
--- a/u19_pipeline/temp/behavior.py
+++ b/u19_pipeline/temp/behavior.py
@@ -1,14 +1,13 @@
"""This module was auto-generated by datajoint from an existing schema"""
-
import datajoint as dj
-schema = dj.schema('u19_behavior_temp')
+schema = dj.schema("u19_behavior_temp")
-vmod0 = dj.create_virtual_module('vmod0', 'u19_acquisition')
-vmod1 = dj.create_virtual_module('vmod1', 'u19_task')
-vmod2 = dj.create_virtual_module('vmod2', 'u19_subject')
+vmod0 = dj.create_virtual_module("vmod0", "u19_acquisition")
+vmod1 = dj.create_virtual_module("vmod1", "u19_task")
+vmod2 = dj.create_virtual_module("vmod2", "u19_subject")
@schema
diff --git a/u19_pipeline/temp/imaging.py b/u19_pipeline/temp/imaging.py
index a644d9e6..bef752e7 100644
--- a/u19_pipeline/temp/imaging.py
+++ b/u19_pipeline/temp/imaging.py
@@ -1,10 +1,9 @@
"""This module was auto-generated by datajoint from an existing schema"""
-
import datajoint as dj
-schema = dj.schema('u19_imaging_temp')
-vmod0 = dj.create_virtual_module('vmod0', 'u19_acquisition')
+schema = dj.schema("u19_imaging_temp")
+vmod0 = dj.create_virtual_module("vmod0", "u19_acquisition")
@schema
diff --git a/u19_pipeline/temp/meso.py b/u19_pipeline/temp/meso.py
index b7d94ef6..3a912f58 100644
--- a/u19_pipeline/temp/meso.py
+++ b/u19_pipeline/temp/meso.py
@@ -1,12 +1,11 @@
"""This module was auto-generated by datajoint from an existing schema"""
-
import datajoint as dj
-schema = dj.schema('u19_meso_temp')
+schema = dj.schema("u19_meso_temp")
-vmod0 = dj.create_virtual_module('vmod0', 'u19_acquisition')
+vmod0 = dj.create_virtual_module("vmod0", "u19_acquisition")
@schema
diff --git a/u19_pipeline/temp/meso_analysis.py b/u19_pipeline/temp/meso_analysis.py
index a1c7c5c8..d17fbde4 100644
--- a/u19_pipeline/temp/meso_analysis.py
+++ b/u19_pipeline/temp/meso_analysis.py
@@ -1,12 +1,11 @@
"""This module was auto-generated by datajoint from an existing schema"""
-
import datajoint as dj
-schema = dj.schema('u19_meso_analysis_temp')
+schema = dj.schema("u19_meso_analysis_temp")
-vmod0 = dj.create_virtual_module('vmod0', 'u19_meso')
+vmod0 = dj.create_virtual_module("vmod0", "u19_meso")
@schema
diff --git a/u19_pipeline/utility.py b/u19_pipeline/utility.py
index 297c9262..bf8f80c3 100644
--- a/u19_pipeline/utility.py
+++ b/u19_pipeline/utility.py
@@ -1,14 +1,12 @@
-
-
-import sys
import os
-import pandas as pd
-from pandas.api.types import is_numeric_dtype
+import subprocess
+import sys
+
+import datajoint as dj
import numpy as np
-from scipy.optimize import curve_fit
+import pandas as pd
from astropy.stats import binom_conf_interval
-import datajoint as dj
-import subprocess
+from scipy.optimize import curve_fit
def is_this_spock():
@@ -23,8 +21,8 @@ def is_this_spock():
stdout, stderr = p.communicate()
if p.returncode == 0:
- hostname = stdout.decode('UTF-8')
- if 'spock' in hostname or 'scotty' in hostname:
+ hostname = stdout.decode("UTF-8")
+ if "spock" in hostname or "scotty" in hostname:
isSpock = True
return isSpock
@@ -35,25 +33,19 @@ def basic_dj_configuration(dj):
Configure host and external storage locations for datajoint
"""
- dj.config['database.host'] = 'datajoint00.pni.princeton.edu'
- dj.config['enable_python_native_blobs'] = True
+ dj.config["database.host"] = "datajoint00.pni.princeton.edu"
+ dj.config["enable_python_native_blobs"] = True
if is_this_spock():
- ext_storage_location = '/mnt/bucket/u19_dj/external_dj_blobs'
+ ext_storage_location = "/mnt/bucket/u19_dj/external_dj_blobs"
elif sys.platform == "darwin":
- ext_storage_location = '/Volumes/u19_dj/external_dj_blobs'
+ ext_storage_location = "/Volumes/u19_dj/external_dj_blobs"
elif sys.platform == "win32":
- ext_storage_location = '//bucket.pni.princeton.edu/u19_dj/external_dj_blobs'
+ ext_storage_location = "//bucket.pni.princeton.edu/u19_dj/external_dj_blobs"
elif sys.platform == "linux" or sys.platform == "linux2":
- ext_storage_location = '/mnt/u19_dj/external_dj_blobs'
+ ext_storage_location = "/mnt/u19_dj/external_dj_blobs"
- dj.config['stores'] = {
- 'extstorage':
- {
- 'location': ext_storage_location,
- 'protocol': 'file'
- }
- }
+ dj.config["stores"] = {"extstorage": {"location": ext_storage_location, "protocol": "file"}}
def get_network_path(path_name):
@@ -65,29 +57,30 @@ def get_network_path(path_name):
network_path: (str): String with network path as mounted by the corresponding os
"""
- key = dict()
- # Check if path name to search starts with needed / at start
- if path_name[0] != '/':
- key['global_path'] = '/' + path_name
+ key = {}
+ # Check if path name to search starts with needed / at start
+ if path_name[0] != "/":
+ key["global_path"] = "/" + path_name
else:
- key['global_path'] = path_name
-
- field_get = ['local_path']
+ key["global_path"] = path_name
+
+ field_get = ["local_path"]
if is_this_spock():
- field_get = ['bucket_path']
- key['system'] = 'linux'
+ field_get = ["bucket_path"]
+ key["system"] = "linux"
elif sys.platform == "darwin":
- key['system'] = 'mac'
- elif os.name == 'nt':
- key['system'] = 'windows'
+ key["system"] = "mac"
+ elif os.name == "nt":
+ key["system"] = "windows"
else:
- key['system'] = 'linux'
+ key["system"] = "linux"
- lab = dj.create_virtual_module('lab', dj.config['custom']['database.prefix']+'lab')
+ lab = dj.create_virtual_module("lab", dj.config["custom"]["database.prefix"] + "lab")
network_path = (lab.Path & key).fetch1(*field_get)
return network_path
+
# Function copied in dj_shorts
def smart_dj_join(t1, t2):
"""
@@ -96,21 +89,21 @@ def smart_dj_join(t1, t2):
"""
# Get all fields from tables
- fields_t1 = pd.DataFrame.from_dict(t1.heading.attributes, orient='index')
- fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient='index')
+ fields_t1 = pd.DataFrame.from_dict(t1.heading.attributes, orient="index")
+ fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient="index")
# Get only secondary fields and check matches
- fields_t1_list = set(fields_t1.loc[fields_t1['in_key'] == False].index.to_list())
- fields_t2_list = set(fields_t2.loc[fields_t2['in_key'] == False].index.to_list())
+ fields_t1_list = set(fields_t1.loc[not fields_t1["in_key"]].index.to_list())
+ fields_t2_list = set(fields_t2.loc[not fields_t2["in_key"]].index.to_list())
intersected_fields = fields_t2_list.intersection(fields_t1_list)
# If there are:
if len(intersected_fields) > 0:
# Create a dictionary to rename matching ones
suffix = t2.table_name
- new_name_attr_dict = dict()
+ new_name_attr_dict = {}
for i in intersected_fields:
- new_name_attr_dict[suffix + '_' + i] = i
+ new_name_attr_dict[suffix + "_" + i] = i
# List non matching ones
non_intersected_fields = list(fields_t2_list - intersected_fields)
@@ -124,11 +117,11 @@ def smart_dj_join(t1, t2):
return t
-def psychometrics_function(x, O, A, lambd, x0):
+def psychometrics_function(x, offset, A, lambd, x0):
"""
Standard sigmoid function
"""
- return O + A/(1+np.exp(-(x-x0)/lambd))
+ return offset + A / (1 + np.exp(-(x - x0) / lambd))
def psychFit(deltaBins, numR, numL, choices):
@@ -146,8 +139,8 @@ def psychFit(deltaBins, numR, numL, choices):
nCues_RminusL = numR - numL
# Correct deltaBin & trialBin to produce same result as Matlab psychFit
deltaBins_search = deltaBins.astype(float) - 1.5
- trialBin = np.searchsorted(deltaBins_search, nCues_RminusL, side='right')
- trialBin -= 1;
+ trialBin = np.searchsorted(deltaBins_search, nCues_RminusL, side="right")
+ trialBin -= 1
# Put into evidence bins all Trials with corresponding choices
for iTrial in range(len(choices)):
@@ -157,8 +150,8 @@ def psychFit(deltaBins, numR, numL, choices):
trialDelta[trialBin[iTrial]] = trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial]
- with np.errstate(divide='ignore', invalid='ignore'):
- trialDelta = np.true_divide(trialDelta, numTrials);
+ with np.errstate(divide="ignore", invalid="ignore"):
+ trialDelta = np.true_divide(trialDelta, numTrials)
# Select only bins with trials
idx_zero = numTrials == 0
@@ -166,8 +159,8 @@ def psychFit(deltaBins, numR, numL, choices):
numRight_nz = numRight[~idx_zero]
# (Binomial proportion confidence interval given k successes, n trials)
- phat_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=0, interval='jeffreys')
- pci_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=1 - 0.1587, interval='jeffreys')
+ phat_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=0, interval="jeffreys")
+ pci_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=1 - 0.1587, interval="jeffreys")
# Correct confidence intervals and expected outcomes for bins with no trials (ci = [0 1], hat = 0.5)
phat_nz = phat_nz[0]
@@ -191,8 +184,14 @@ def psychFit(deltaBins, numR, numL, choices):
weight_array = np.power((pci[1][~idx_zero] - pci[0][~idx_zero]) / 2, 2)
sigma_fit = np.diag(weight_array)
- psychometric, pcov = curve_fit(psychometrics_function, deltaBins[~idx_zero], phat[~idx_zero], \
- p0=(0, 1, 3, 0), sigma=sigma_fit, maxfev=40000)
+ psychometric, pcov = curve_fit(
+ psychometrics_function,
+ deltaBins[~idx_zero],
+ phat[~idx_zero],
+ p0=(0, 1, 3, 0),
+ sigma=sigma_fit,
+ maxfev=40000,
+ )
# Append a row of nans to confidence intervals . whyy ??
aux_vec = np.empty((1, pci.shape[1]))
@@ -203,26 +202,26 @@ def psychFit(deltaBins, numR, numL, choices):
delta = np.linspace(deltaBins[0] - 2, deltaBins[-1] + 2, num=50)
# Repeat trialDelta 3 times for errorX why ??
- errorX = np.tile(trialDelta[~idx_zero], 3);
+ errorX = np.tile(trialDelta[~idx_zero], 3)
# Confidence intervals are errorY, as a vector
errorY = np.stack(pci[:, ~idx_zero])
errorY = errorY.flatten()
# Fill dictionary of results
- fit_results = dict()
- fit_results['delta_bins'] = deltaBins[~idx_zero]
- fit_results['delta_data'] = trialDelta[~idx_zero];
- fit_results['pright_data'] = 100 * phat[~idx_zero];
- fit_results['delta_error'] = errorX;
- fit_results['pright_error'] = 100 * errorY;
+ fit_results = {}
+ fit_results["delta_bins"] = deltaBins[~idx_zero]
+ fit_results["delta_data"] = trialDelta[~idx_zero]
+ fit_results["pright_data"] = 100 * phat[~idx_zero]
+ fit_results["delta_error"] = errorX
+ fit_results["pright_error"] = 100 * errorY
if is_there_psychometric:
- fit_results['delta_fit'] = delta
- fit_results['pright_fit'] = psychometrics_function(delta, *psychometric) * 100
+ fit_results["delta_fit"] = delta
+ fit_results["pright_fit"] = psychometrics_function(delta, *psychometric) * 100
else:
- fit_results['delta_fit'] = np.empty([0])
- fit_results['pright_fit'] = np.empty([0])
+ fit_results["delta_fit"] = np.empty([0])
+ fit_results["pright_fit"] = np.empty([0])
return fit_results
@@ -232,21 +231,21 @@ def translate_choice_trials_cues(session_df):
Transform to numeric values trialtype, choice and left & right cues
"""
- session_df['trial_type_int'] = 0
- session_df.loc[session_df['trial_type'] == 'L','trial_type_int'] = 1
- session_df.loc[session_df['trial_type'] == 'R','trial_type_int'] = 2
- session_df['choice_int'] = 0
- session_df.loc[session_df['choice'] == 'L','choice_int'] = 1
- session_df.loc[session_df['choice'] == 'R','choice_int'] = 2
+ session_df["trial_type_int"] = 0
+ session_df.loc[session_df["trial_type"] == "L", "trial_type_int"] = 1
+ session_df.loc[session_df["trial_type"] == "R", "trial_type_int"] = 2
+ session_df["choice_int"] = 0
+ session_df.loc[session_df["choice"] == "L", "choice_int"] = 1
+ session_df.loc[session_df["choice"] == "R", "choice_int"] = 2
# If we are translating towers task cues
- if 'cue_presence_left' in session_df.columns:
- session_df['cue_presence_left'] = session_df['cue_presence_left'].apply(lambda x: np.count_nonzero(x))
- session_df['cue_presence_right'] = session_df['cue_presence_right'].apply(lambda x: np.count_nonzero(x))
+ if "cue_presence_left" in session_df.columns:
+ session_df["cue_presence_left"] = session_df["cue_presence_left"].apply(lambda x: np.count_nonzero(x))
+ session_df["cue_presence_right"] = session_df["cue_presence_right"].apply(lambda x: np.count_nonzero(x))
# If we are translating puff task cues
- elif 'num_puffs_received_r' in session_df.columns:
- session_df['cue_presence_left'] = session_df['num_puffs_received_l']
- session_df['cue_presence_right'] = session_df['num_puffs_received_r']
+ elif "num_puffs_received_r" in session_df.columns:
+ session_df["cue_presence_left"] = session_df["num_puffs_received_l"]
+ session_df["cue_presence_right"] = session_df["num_puffs_received_r"]
return session_df
@@ -259,11 +258,10 @@ def get_cols_rows_plot(num_plots, fig_size):
# Check width vs height relation
fig_rel = fig_size[1] / fig_size[0]
- #Let's start with square root
+ # Let's start with square root
num_rows = 1
num_cols = 1
while 1:
-
# Add rows and columns to closely match desired width height relationship and surpass number of desired plots
ac_rel = num_cols / num_rows
@@ -280,17 +278,17 @@ def get_cols_rows_plot(num_plots, fig_size):
def create_str_from_dict(key_dict):
- slurm_file_name = ''
- for i in key_dict.keys():
- slurm_file_name += str(i) + '_' + str(key_dict[i])
+ slurm_file_name = ""
+ for i in key_dict:
+ slurm_file_name += str(i) + "_" + str(key_dict[i])
return slurm_file_name
def numpy_array_to_dict(np_array, as_int=True):
- '''
+ """
Transform a numpy array to dictionary:
(numpy array are stored when saving Blobs in MATLAB Datajoint, normally a dictionary will be the fit)
- '''
+ """
# Transform numpy array to DF
out_dict = pd.DataFrame(np_array.flatten())
@@ -298,27 +296,27 @@ def numpy_array_to_dict(np_array, as_int=True):
# Flatten each column and get the "real value of it"
out_dict = out_dict.applymap(lambda x: x.flatten())
- #Get not empty columns to extract fist value of only those columns
+ # Get not empty columns to extract fist value of only those columns
s = out_dict.applymap(lambda x: x.shape[0])
not_empty_columns = s.loc[:, ~(s == 0).any()].columns.to_list()
out_dict = out_dict.apply(lambda x: x[0].flatten() if x.name in not_empty_columns else x)
-
+
if not isinstance(out_dict, pd.DataFrame):
out_dict = out_dict.to_frame()
- #Get columns that are "real" arrays not unique values disguised
+ # Get columns that are "real" arrays not unique values disguised
s = out_dict.applymap(lambda x: x.size).T
real_array_columns = s.loc[:, (s > 1).any()].columns.to_list()
out_dict = out_dict.T
out_dict = out_dict.apply(lambda x: x[0] if x.name not in real_array_columns else x, axis=0)
-
+
columns = out_dict.columns.copy()
out_dict = out_dict.squeeze()
- #Transform numeric columns to int (normally for params)
+ # Transform numeric columns to int (normally for params)
if as_int:
for i in columns:
- if (isinstance(out_dict[i],np.float64) and out_dict[i].is_integer()):
- out_dict[i] = out_dict[i].astype('int')
+ if isinstance(out_dict[i], np.float64) and out_dict[i].is_integer():
+ out_dict[i] = out_dict[i].astype("int")
out_dict = out_dict.to_dict()
diff --git a/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py b/u19_pipeline/utils/DemoReadSGLXData/read_sglx.py
similarity index 66%
rename from u19_pipeline/utils/DemoReadSGLXData/readSGLX.py
rename to u19_pipeline/utils/DemoReadSGLXData/read_sglx.py
index 7fca605f..d736ddc8 100644
--- a/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py
+++ b/u19_pipeline/utils/DemoReadSGLXData/read_sglx.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
"""
Requires python 3
@@ -15,11 +14,16 @@
much easier!
"""
-import numpy as np
-import matplotlib.pyplot as plt
+
from pathlib import Path
-from tkinter import Tk
-from tkinter import filedialog
+from tkinter import Tk, filedialog
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
# Parse ini file returning a dictionary whose keys are the metadata
@@ -40,15 +44,12 @@ def readMeta(binFullPath):
mdatList = f.read().splitlines()
# convert the list entries into key value pairs
for m in mdatList:
- csList = m.split(sep='=')
- if csList[0][0] == '~':
- currKey = csList[0][1:len(csList[0])]
- else:
- currKey = csList[0]
+ csList = m.split(sep="=")
+ currKey = csList[0][1:len(csList[0])] if csList[0][0] == "~" else csList[0]
metaDict.update({currKey: csList[1]})
else:
- print("no meta file")
- return(metaDict)
+ logger.error("no meta file")
+ return metaDict
# Return sample rate as python float.
@@ -56,17 +57,17 @@ def readMeta(binFullPath):
# Use python command sys.float_info to get properties of float on your system.
#
def SampRate(meta):
- if meta['typeThis'] == 'imec':
- srate = float(meta['imSampRate'])
- elif meta['typeThis'] == 'nidq':
- srate = float(meta['niSampRate'])
- elif meta['typeThis'] == 'obx':
- srate = float(meta['obSampRate'])
+ if meta["typeThis"] == "imec":
+ srate = float(meta["imSampRate"])
+ elif meta["typeThis"] == "nidq":
+ srate = float(meta["niSampRate"])
+ elif meta["typeThis"] == "obx":
+ srate = float(meta["obSampRate"])
else:
- print('Error: unknown stream type')
+ logger.error("Error: unknown stream type")
srate = 1
-
- return(srate)
+
+ return srate
# Return a multiplicative factor for converting 16-bit file data
@@ -76,23 +77,20 @@ def SampRate(meta):
# Note that each channel may have its own gain.
#
def Int2Volts(meta):
- if meta['typeThis'] == 'imec':
- if 'imMaxInt' in meta:
- maxInt = int(meta['imMaxInt'])
- else:
- maxInt = 512
- fI2V = float(meta['imAiRangeMax'])/maxInt
- elif meta['typeThis'] == 'nidq':
- maxInt = int(meta['niMaxInt'])
- fI2V = float(meta['niAiRangeMax'])/maxInt
- elif meta['typeThis'] == 'obx':
- maxInt = int(meta['obMaxInt'])
- fI2V = float(meta['obAiRangeMax'])/maxInt
+ if meta["typeThis"] == "imec":
+ maxInt = int(meta["imMaxInt"]) if "imMaxInt" in meta else 512
+ fI2V = float(meta["imAiRangeMax"]) / maxInt
+ elif meta["typeThis"] == "nidq":
+ maxInt = int(meta["niMaxInt"])
+ fI2V = float(meta["niAiRangeMax"]) / maxInt
+ elif meta["typeThis"] == "obx":
+ maxInt = int(meta["obMaxInt"])
+ fI2V = float(meta["obAiRangeMax"]) / maxInt
else:
- print('Error: unknown stream type')
+ logger.error("Error: unknown stream type")
fI2V = 1
-
- return(fI2V)
+
+ return fI2V
# Return array of original channel IDs. As an example, suppose we want the
@@ -104,57 +102,58 @@ def Int2Volts(meta):
# Note that the SpikeGLX channels are 0 based.
#
def OriginalChans(meta):
- if meta['snsSaveChanSubset'] == 'all':
+ if meta["snsSaveChanSubset"] == "all":
# output = int32, 0 to nSavedChans - 1
- chans = np.arange(0, int(meta['nSavedChans']))
+ chans = np.arange(0, int(meta["nSavedChans"]))
else:
# parse the snsSaveChanSubset string
# split at commas
- chStrList = meta['snsSaveChanSubset'].split(sep=',')
+ chStrList = meta["snsSaveChanSubset"].split(sep=",")
chans = np.arange(0, 0) # creates an empty array of int32
for sL in chStrList:
- currList = sL.split(sep=':')
+ currList = sL.split(sep=":")
if len(currList) > 1:
# each set of contiguous channels specified by
# chan1:chan2 inclusive
- newChans = np.arange(int(currList[0]), int(currList[1])+1)
+ newChans = np.arange(int(currList[0]), int(currList[1]) + 1)
else:
- newChans = np.arange(int(currList[0]), int(currList[0])+1)
+ newChans = np.arange(int(currList[0]), int(currList[0]) + 1)
chans = np.append(chans, newChans)
- return(chans)
+ return chans
# Return counts of each nidq channel type that composes the timepoints
# stored in the binary file.
#
def ChannelCountsNI(meta):
- chanCountList = meta['snsMnMaXaDw'].split(sep=',')
+ chanCountList = meta["snsMnMaXaDw"].split(sep=",")
MN = int(chanCountList[0])
MA = int(chanCountList[1])
XA = int(chanCountList[2])
DW = int(chanCountList[3])
- return(MN, MA, XA, DW)
+ return (MN, MA, XA, DW)
# Return counts of each imec channel type that composes the timepoints
# stored in the binary files.
#
def ChannelCountsIM(meta):
- chanCountList = meta['snsApLfSy'].split(sep=',')
+ chanCountList = meta["snsApLfSy"].split(sep=",")
AP = int(chanCountList[0])
LF = int(chanCountList[1])
SY = int(chanCountList[2])
- return(AP, LF, SY)
+ return (AP, LF, SY)
+
# Return counts of each obx channel type that composes the timepoints
# stored in the binary files.
#
def ChannelCountsOBX(meta):
- chanCountList = meta['snsXaDwSy'].split(sep=',')
+ chanCountList = meta["snsXaDwSy"].split(sep=",")
XA = int(chanCountList[0])
DW = int(chanCountList[1])
SY = int(chanCountList[2])
- return(XA, DW, SY)
+ return (XA, DW, SY)
# Return gain for ith channel stored in nidq file.
@@ -163,12 +162,12 @@ def ChannelCountsOBX(meta):
# Note: there is nomatching function for OBX, where the gain is fixed = 1
def ChanGainNI(ichan, savedMN, savedMA, meta):
if ichan < savedMN:
- gain = float(meta['niMNGain'])
+ gain = float(meta["niMNGain"])
elif ichan < (savedMN + savedMA):
- gain = float(meta['niMAGain'])
+ gain = float(meta["niMAGain"])
else:
- gain = 1 # non multiplexed channels have no extra gain
- return(gain)
+ gain = 1 # non multiplexed channels have no extra gain
+ return gain
# Return gain for imec channels.
@@ -176,56 +175,50 @@ def ChanGainNI(ichan, savedMN, savedMA, meta):
#
def ChanGainsIM(meta):
# list of probe types with NP 1.0 imro format
- np1_imro = [0,1020,1030,1200,1100,1120,1121,1122,1123,1300]
+ np1_imro = [0, 1020, 1030, 1200, 1100, 1120, 1121, 1122, 1123, 1300]
# number of channels acquired
- acqCountList = meta['acqApLfSy'].split(sep=',')
- APgain = np.zeros(int(acqCountList[0])) # default type = float64
- LFgain = np.zeros(int(acqCountList[1])) # empty array for 2.0
-
- if 'imDatPrb_type' in meta:
- probeType = int(meta['imDatPrb_type'])
- else:
- probeType = 0
-
+ acqCountList = meta["acqApLfSy"].split(sep=",")
+ APgain = np.zeros(int(acqCountList[0])) # default type = float64
+ LFgain = np.zeros(int(acqCountList[1])) # empty array for 2.0
+
+ probeType = int(meta["imDatPrb_type"]) if "imDatPrb_type" in meta else 0
+
if sum(np.isin(np1_imro, probeType)):
# imro + probe allows setting gain independently for each channel
- imroList = meta['imroTbl'].split(sep=')')
+ imroList = meta["imroTbl"].split(sep=")")
# One entry for each channel plus header entry,
- # plus a final empty entry following the last ')'
+ # plus a final empty entry following the last ')'
for i in range(0, int(acqCountList[0])):
- currList = imroList[i+1].split(sep=' ')
+ currList = imroList[i + 1].split(sep=" ")
APgain[i] = float(currList[3])
LFgain[i] = float(currList[4])
- else:
+ else:
# get gain from imChan0apGain
- if 'imChan0apGain' in meta:
- APgain = APgain + float(meta['imChan0apGain'])
- if int(acqCountList[1]) > 0:
- LFgain = LFgain + float(meta['imChan0lfGain'])
- elif (probeType == 1110):
+ if "imChan0apGain" in meta:
+ APgain = APgain + float(meta["imChan0apGain"])
+ if int(acqCountList[1]) > 0:
+ LFgain = LFgain + float(meta["imChan0lfGain"])
+ elif probeType == 1110:
# active UHD, for metadata lacking imChan0apGain, get gain from
# imro table header
- imroList = meta['imroTbl'].split(sep=')')
- currList = imroList[0].split(sep=',')
+ imroList = meta["imroTbl"].split(sep=")")
+ currList = imroList[0].split(sep=",")
APgain = APgain + float(currList[3])
LFgain = LFgain + float(currList[4])
elif (probeType == 21) or (probeType == 24):
# development NP 2.0; APGain = 80 for all AP
# return 0 for LFgain (no LF channels)
- APgain = APgain + 80
- elif (probeType == 2013):
+ APgain = APgain + 80
+ elif probeType == 2013:
# commercial NP 2.0; APGain = 100 for all AP
APgain = APgain + 100
else:
- print('unknown gain, setting APgain to 1')
+ logger.warning("unknown gain, setting APgain to 1")
APgain = APgain + 1
fI2V = Int2Volts(meta)
- APChan0_to_uV = 1e6*fI2V/APgain[0]
- if LFgain.size > 0:
- LFChan0_to_uV = 1e6*fI2V/LFgain[0]
- else:
- LFChan0_to_uV = 0
- return(APgain, LFgain, APChan0_to_uV, LFChan0_to_uV)
+ APChan0_to_uV = 1e6 * fI2V / APgain[0]
+ LFChan0_to_uV = 1000000.0 * fI2V / LFgain[0] if LFgain.size > 0 else 0
+ return (APgain, LFgain, APChan0_to_uV, LFChan0_to_uV)
# Having accessed a block of raw nidq data using makeMemMapRaw, convert
@@ -247,11 +240,12 @@ def GainCorrectNI(dataArray, chanList, meta):
# in chanList, so output matches that shape
convArray = np.zeros(dataArray.shape, dtype=float)
for i in range(0, len(chanList)):
- j = chanList[i] # index in saved data
- conv = fI2V/ChanGainNI(j, MN, MA, meta)
+ j = chanList[i] # index in saved data
+ conv = fI2V / ChanGainNI(j, MN, MA, meta)
# dataArray contains only the channels in chanList
convArray[i, :] = dataArray[i, :] * conv
- return(convArray)
+ return convArray
+
# Having accessed a block of raw obx data using makeMemMapRaw, convert
# values to volts. The conversion is only applied to the
@@ -269,7 +263,7 @@ def GainCorrectOBX(dataArray, chanList, meta):
for i in range(0, len(chanList)):
# dataArray contains only the channels in chanList
convArray[i, :] = dataArray[i, :] * fI2V
- return(convArray)
+ return convArray
# Having accessed a block of raw imec data using makeMemMapRaw, convert
@@ -294,10 +288,10 @@ def GainCorrectIM(dataArray, chanList, meta):
# make array of floats to return. dataArray contains only the channels
# in chanList, so output matches that shape
- convArray = np.zeros(dataArray.shape, dtype='float')
+ convArray = np.zeros(dataArray.shape, dtype="float")
for i in range(0, len(chanList)):
- j = chanList[i] # index into timepoint
- k = chans[j] # acquisition index
+ j = chanList[i] # index into timepoint
+ k = chans[j] # acquisition index
if k < nAP:
conv = fI2V / APgain[k]
elif k < nNu:
@@ -305,20 +299,27 @@ def GainCorrectIM(dataArray, chanList, meta):
else:
conv = 1
# The dataArray contains only the channels in chanList
- convArray[i, :] = dataArray[i, :]*conv
- return(convArray)
+ convArray[i, :] = dataArray[i, :] * conv
+ return convArray
+
# Return memmap for the raw data
# Fortran ordering is used to match the MATLAB version
# of these tools.
#
def makeMemMapRaw(binFullPath, meta):
- nChan = int(meta['nSavedChans'])
- nFileSamp = int(int(meta['fileSizeBytes'])/(2*nChan))
- print("nChan: %d, nFileSamp: %d" % (nChan, nFileSamp))
- rawData = np.memmap(binFullPath, dtype='int16', mode='r',
- shape=(nChan, nFileSamp), offset=0, order='F')
- return(rawData)
+ nChan = int(meta["nSavedChans"])
+ nFileSamp = int(int(meta["fileSizeBytes"]) / (2 * nChan))
+ logger.info("nChan: %d, nFileSamp: %d", nChan, nFileSamp)
+ rawData = np.memmap(
+ binFullPath,
+ dtype="int16",
+ mode="r",
+ shape=(nChan, nFileSamp),
+ offset=0,
+ order="F",
+ )
+ return rawData
# Return an array [lines X timepoints] of uint8 values for a
@@ -331,49 +332,49 @@ def makeMemMapRaw(binFullPath, meta):
#
def ExtractDigital(rawData, firstSamp, lastSamp, dwReq, dLineList, meta):
# Get channel index of requested digital word dwReq
- if meta['typeThis'] == 'imec':
+ if meta["typeThis"] == "imec":
AP, LF, SY = ChannelCountsIM(meta)
if SY == 0:
- print("No imec sync channel saved.")
- digArray = np.zeros((0), 'uint8')
- return(digArray)
+ logger.warning("No imec sync channel saved.")
+ digArray = np.zeros((0), "uint8")
+ return digArray
else:
digCh = AP + LF + dwReq
- elif meta['typeThis'] == 'nidq':
+ elif meta["typeThis"] == "nidq":
MN, MA, XA, DW = ChannelCountsNI(meta)
- if dwReq > DW-1:
- print("Maximum digital word in file = %d" % (DW-1))
- digArray = np.zeros((0), 'uint8')
- return(digArray)
+ if dwReq > DW - 1:
+ logger.info("Maximum digital word in file = %d", DW - 1)
+ digArray = np.zeros((0), "uint8")
+ return digArray
else:
digCh = MN + MA + XA + dwReq
- elif meta['typeThis'] == 'obx':
+ elif meta["typeThis"] == "obx":
XA, DW, SY = ChannelCountsOBX(meta)
- if dwReq > DW-1:
- print("Maximum digital word in file = %d" % (DW-1))
- digArray = np.zeros((0), 'uint8')
- return(digArray)
+ if dwReq > DW - 1:
+ logger.info("Maximum digital word in file = %d", DW - 1)
+ digArray = np.zeros((0), "uint8")
+ return digArray
else:
digCh = XA + dwReq
else:
- print('unknown data stream')
+ logger.error("unknown data stream")
- selectData = np.ascontiguousarray(rawData[digCh, firstSamp:lastSamp+1], 'int16')
- nSamp = lastSamp-firstSamp + 1
+ selectData = np.ascontiguousarray(rawData[digCh, firstSamp : lastSamp + 1], "int16")
+ nSamp = lastSamp - firstSamp + 1
# unpack bits of selectData; unpack bits works with uint8
# original data is int16
- bitWiseData = np.unpackbits(selectData.view(dtype='uint8'))
+ bitWiseData = np.unpackbits(selectData.view(dtype="uint8"))
# output is 1-D array, nSamp*16. Reshape and transpose
bitWiseData = np.transpose(np.reshape(bitWiseData, (nSamp, 16)))
nLine = len(dLineList)
- digArray = np.zeros((nLine, nSamp), 'uint8')
+ digArray = np.zeros((nLine, nSamp), "uint8")
for i in range(0, nLine):
byteN, bitN = np.divmod(dLineList[i], 8)
- targI = byteN*8 + (7 - bitN)
+ targI = byteN * 8 + (7 - bitN)
digArray[i, :] = bitWiseData[targI, :]
- return(digArray)
+ return digArray
# Sample calling program to get a file from the user,
@@ -386,19 +387,19 @@ def ExtractDigital(rawData, firstSamp, lastSamp, dwReq, dLineList, meta):
def main():
# Get file from user
- root = Tk() # create the Tkinter widget
- root.withdraw() # hide the Tkinter root window
+ root = Tk() # create the Tkinter widget
+ root.withdraw() # hide the Tkinter root window
# Windows specific; forces the window to appear in front
root.attributes("-topmost", True)
binFullPath = Path(filedialog.askopenfilename(title="Select binary file"))
- root.destroy() # destroy the Tkinter widget
+ root.destroy() # destroy the Tkinter widget
# Other parameters about what data to read
tStart = 0
tEnd = 2
- dataType = 'A' # 'A' for analog, 'D' for digital data
+ dataType = "A" # 'A' for analog, 'D' for digital data
# For analog channels: zero-based index of a channel to extract,
# gain correct and plot (plots first channel only)
@@ -417,44 +418,43 @@ def main():
# parameters common to NI and imec data
sRate = SampRate(meta)
- firstSamp = int(sRate*tStart)
- lastSamp = int(sRate*tEnd)
+ firstSamp = int(sRate * tStart)
+ lastSamp = int(sRate * tEnd)
# array of times for plot
- tDat = np.arange(firstSamp, lastSamp+1, dtype='uint64')
- tDat = 1000*tDat/sRate # plot time axis in msec
+ tDat = np.arange(firstSamp, lastSamp + 1, dtype="uint64")
+ tDat = 1000 * tDat / sRate # plot time axis in msec
rawData = makeMemMapRaw(binFullPath, meta)
- if dataType == 'A':
- selectData = rawData[chanList, firstSamp:lastSamp+1]
- if meta['typeThis'] == 'imec':
+ if dataType == "A":
+ selectData = rawData[chanList, firstSamp : lastSamp + 1]
+ if meta["typeThis"] == "imec":
# apply gain correction and convert to uV
- convData = 1e6*GainCorrectIM(selectData, chanList, meta)
- elif meta['typeThis'] == 'nidq':
+ convData = 1e6 * GainCorrectIM(selectData, chanList, meta)
+ elif meta["typeThis"] == "nidq":
MN, MA, XA, DW = ChannelCountsNI(meta)
# print("NI channel counts: %d, %d, %d, %d" % (MN, MA, XA, DW))
# apply gain correction and convert to mV
- convData = 1e3*GainCorrectNI(selectData, chanList, meta)
- elif meta['typeThis'] == 'obx':
- # Gain correct is just conversion to volts
- convData = 1e3*GainCorrectOBX(selectData, chanList, meta)
-
+ convData = 1e3 * GainCorrectNI(selectData, chanList, meta)
+ elif meta["typeThis"] == "obx":
+ # Gain correct is just conversion to volts
+ convData = 1e3 * GainCorrectOBX(selectData, chanList, meta)
+
# Plot the first of the extracted channels
fig, ax = plt.subplots()
ax.plot(tDat, convData[0, :])
plt.show()
else:
- digArray = ExtractDigital(rawData, firstSamp, lastSamp, dw,
- dLineList, meta)
+ digArray = ExtractDigital(rawData, firstSamp, lastSamp, dw, dLineList, meta)
# Plot the first of the extracted channels
fig, ax = plt.subplots()
for i in range(0, len(dLineList)):
- ax.plot(tDat, digArray[i, :])
+ ax.plot(tDat, digArray[i, :])
plt.show()
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/u19_pipeline/utils/dj_shortcuts.py b/u19_pipeline/utils/dj_shortcuts.py
index 0e87490f..9d1b9764 100644
--- a/u19_pipeline/utils/dj_shortcuts.py
+++ b/u19_pipeline/utils/dj_shortcuts.py
@@ -1,17 +1,18 @@
import pandas as pd
+
def get_primary_key_fields(t):
"""
Get list of all fields that compose primary key
Args:
- t (Dj table): Instance of a table in datajoint
+ t (Dj table): Instance of a table in datajoint
Returns:
primary_field_list: (list): List of all fields that make primary key
"""
- fields_t = pd.DataFrame.from_dict(t.heading.attributes, orient='index')
- primary_field_list = fields_t.loc[fields_t['in_key'] == True].index.to_list()
-
+ fields_t = pd.DataFrame.from_dict(t.heading.attributes, orient="index")
+ primary_field_list = fields_t.loc[fields_t["in_key"]].index.to_list()
+
return primary_field_list
@@ -23,21 +24,21 @@ def smart_dj_join(t1, t2):
"""
# Get all fields from tables
- fields_t1 = pd.DataFrame.from_dict(t1.heading.attributes, orient='index')
- fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient='index')
+ fields_t1 = pd.DataFrame.from_dict(t1.heading.attributes, orient="index")
+ fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient="index")
# Get only secondary fields and check matches
- fields_t1_list = set(fields_t1.loc[fields_t1['in_key'] == False].index.to_list())
- fields_t2_list = set(fields_t2.loc[fields_t2['in_key'] == False].index.to_list())
+ fields_t1_list = set(fields_t1.loc[not fields_t1["in_key"]].index.to_list())
+ fields_t2_list = set(fields_t2.loc[not fields_t2["in_key"]].index.to_list())
intersected_fields = fields_t2_list.intersection(fields_t1_list)
# If there are:
if len(intersected_fields) > 0:
# Create a dictionary to rename matching ones
suffix = t2.table_name
- new_name_attr_dict = dict()
+ new_name_attr_dict = {}
for i in intersected_fields:
- new_name_attr_dict[suffix + '_' + i] = i
+ new_name_attr_dict[suffix + "_" + i] = i
# List non matching ones
non_intersected_fields = list(fields_t2_list - intersected_fields)
@@ -56,15 +57,15 @@ def get_string_key(key):
Translate list or dict key to string
"""
- str_key = ''
+ str_key = ""
if isinstance(key, list):
- str_key = [[k + '=' + str(v) for k,v in x.items()] for x in key]
- str_key = ['('+' and '.join(sublist)+')' for sublist in str_key]
- str_key = ' or '.join(str_key)
+ str_key = [[k + "=" + str(v) for k, v in x.items()] for x in key]
+ str_key = ["(" + " and ".join(sublist) + ")" for sublist in str_key]
+ str_key = " or ".join(str_key)
elif isinstance(key, dict):
- str_key = [k + '=' + str(v) for k,v in key.items()]
- str_key = ' and '.join(str_key)
- elif isinstance(key,str):
+ str_key = [k + "=" + str(v) for k, v in key.items()]
+ str_key = " and ".join(str_key)
+ elif isinstance(key, str):
str_key = key
-
- return str_key
\ No newline at end of file
+
+ return str_key
diff --git a/u19_pipeline/utils/dlc_process.py b/u19_pipeline/utils/dlc_process.py
index 7c659fe2..4e5658b2 100644
--- a/u19_pipeline/utils/dlc_process.py
+++ b/u19_pipeline/utils/dlc_process.py
@@ -1,20 +1,22 @@
-
import os
-import deeplabcut
-import pandas as pd
-import numpy as np
-import pickle
import pathlib
+import pickle
import sys
+import deeplabcut
+import numpy as np
+import pandas as pd
from scipy import stats
from skimage.measure import EllipseModel
-from skimage.draw import ellipse_perimeter
import u19_pipeline.utils.path_utils as pu
+from u19_pipeline.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
# Pupil diameter pipeline functions
+
def analyzeVideo(videoPath=None, modelPath=None, destinationFolder=None):
"""
Stores the analized video data from videoPath as h5 file in the destination folder using the DLC model in modelPath
@@ -25,9 +27,10 @@ def analyzeVideo(videoPath=None, modelPath=None, destinationFolder=None):
"""
# Analyze the video using the selected modelPath and videoPath
- configPath = os.path.join(modelPath,'config.yaml')
+ configPath = os.path.join(modelPath, "config.yaml")
deeplabcut.analyze_videos(configPath, videoPath, destfolder=destinationFolder)
+
def getPupilDiameter(destinationFolder=None):
"""
Returns a pupil diameter numpy array from an analized video data stored in analyzedVideoDataPath
@@ -37,35 +40,35 @@ def getPupilDiameter(destinationFolder=None):
An array that contains the pupil diameter (index is the video frame) [numpy Array]
"""
# TODO make the function
-
+
# Read the analyzed video data h5 file
h5_file = pu.get_filepattern_paths(destinationFolder, "/*.h5")
- print(h5_file)
+ logger.debug("h5_file: %s", h5_file)
if len(h5_file) == 0:
- raise Exception('No h5 file in directory: '+ destinationFolder)
+ raise Exception("No h5 file in directory: " + destinationFolder)
if len(h5_file) > 1:
- raise Exception('To many h5 files in directory: '+ destinationFolder)
-
+ raise Exception("To many h5 files in directory: " + destinationFolder)
+
h5_file = h5_file[0]
labels = pd.read_hdf(h5_file)
# Create a data frame of the same size ad the analyzed video data filled with zeros
- df = pd.DataFrame(np.zeros(1), columns=['PupilDiameter'])
+ df = pd.DataFrame(np.zeros(1), columns=["PupilDiameter"])
# For each frame, get the x and y coordinates of the points around the pupil, fit an ellipse and calculate the diameter of a circle with the same area as the ellipse
for i in range(labels.index.size):
subset = labels.loc[i]
- x = subset.xs('x', level='coords').to_numpy()[0:8]
- y = subset.xs('y', level='coords').to_numpy()[0:8]
- xy = np.column_stack((x,y))
+ x = subset.xs("x", level="coords").to_numpy()[0:8]
+ y = subset.xs("y", level="coords").to_numpy()[0:8]
+ xy = np.column_stack((x, y))
# Fit the points to an ellipse and get the parameters (estimate X center coordinate, estimate Y center coordinate, a, b, theta)
ellipse = EllipseModel()
ellipse.estimate(xy)
# Calculate the area of the ellipse from the parameters a and b
ellipseArea = np.pi * ellipse.params[2] * ellipse.params[3]
# Get the diameter of a circle from the area of the ellipse
- pupilDiameter = 2 * np.sqrt(ellipseArea/np.pi)
+ pupilDiameter = 2 * np.sqrt(ellipseArea / np.pi)
df.loc[i] = pupilDiameter
# Get outliers (frames where either the mice have the eyes closed (blink or groom) or deeplabcut fails to track the pupil correctly)
@@ -78,18 +81,17 @@ def getPupilDiameter(destinationFolder=None):
outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: "OutlierFlag"})
# Concatenate outlier flags array to remove outliers from pupil diameter array
temp = pd.concat([df, outlierFlags], axis=1)
- temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None
- pupilDiameter = temp['PupilDiameter'].to_numpy()
+ temp.loc[temp["OutlierFlag"], "PupilDiameter"] = None
+ pupilDiameter = temp["PupilDiameter"].to_numpy()
filename = pathlib.Path(destinationFolder, "pupil_diameter.pickle").as_posix()
- file_to_store = open(filename, "wb")
- pickle.dump(pupilDiameter, file_to_store)
- file_to_store.close()
+ with open(filename, "wb") as file_to_store:
+ pickle.dump(pupilDiameter, file_to_store)
if __name__ == "__main__":
args = sys.argv[1:]
- print(args)
+ logger.debug("args: %s", args)
analyzeVideo(videoPath=args[0], modelPath=args[1], destinationFolder=args[2])
- getPupilDiameter(destinationFolder=args[2])
\ No newline at end of file
+ getPupilDiameter(destinationFolder=args[2])
diff --git a/u19_pipeline/utils/ephys_fix_sync_code.py b/u19_pipeline/utils/ephys_fix_sync_code.py
index e4733778..684b1d2c 100644
--- a/u19_pipeline/utils/ephys_fix_sync_code.py
+++ b/u19_pipeline/utils/ephys_fix_sync_code.py
@@ -1,19 +1,24 @@
-
-import datetime
-import pathlib
import numpy as np
import u19_pipeline.utils.ephys_utils as ephys_utils
+from u19_pipeline.utils.logging_config import get_logger
+logger = get_logger(__name__)
-def get_shift_vector(synced_time_vector, behavior_time_vector, base_size=40,initial_sample=0, samples_shift=100):
+def get_shift_vector(
+ synced_time_vector,
+ behavior_time_vector,
+ base_size=40,
+ initial_sample=0,
+ samples_shift=100,
+):
diff_size = np.abs(synced_time_vector.shape[0] - behavior_time_vector.shape[0])
- baseline_diff = synced_time_vector[initial_sample:base_size] - behavior_time_vector[initial_sample:base_size]
+ baseline_diff = synced_time_vector[initial_sample:base_size] - behavior_time_vector[initial_sample:base_size]
- '''
+ """
while(1):
base_greater = np.where(baseline_diff >0)
@@ -28,25 +33,21 @@ def get_shift_vector(synced_time_vector, behavior_time_vector, base_size=40,init
baseline_diff = synced_time_vector[initial_sample:base_size] - behavior_time_vector[initial_sample:base_size]
else:
break
- '''
-
+ """
median_diff = np.median(baseline_diff)
- max_diff = np.median(baseline_diff)+0.007
- min_diff = np.median(baseline_diff)-0.007
-
-
- vec_shift = np.zeros((synced_time_vector.shape[0]-base_size), dtype=int)
- for i in range(synced_time_vector.shape[0]-base_size):
+ max_diff = np.median(baseline_diff) + 0.007
+ min_diff = np.median(baseline_diff) - 0.007
-
- idx_start = initial_sample+i+1
- idx_end = base_size+i+1
+ vec_shift = np.zeros((synced_time_vector.shape[0] - base_size), dtype=int)
+ for i in range(synced_time_vector.shape[0] - base_size):
+ idx_start = initial_sample + i + 1
+ idx_end = base_size + i + 1
this_bt = behavior_time_vector[idx_start:idx_end]
this_iv = synced_time_vector[idx_start:idx_end]
- median_ori = np.median(this_iv-this_bt)
+ median_ori = np.median(this_iv - this_bt)
if median_ori >= min_diff and median_ori < max_diff:
vec_shift[i] = 0
@@ -55,25 +56,23 @@ def get_shift_vector(synced_time_vector, behavior_time_vector, base_size=40,init
sign = 1
else:
sign = -1
-
+
new_sign = sign
if sign != 0:
-
for j in range(1, samples_shift):
-
if new_sign == 1:
- if idx_end+j > synced_time_vector.shape[0]:
- this_iv = synced_time_vector[idx_start+j:]
+ if idx_end + j > synced_time_vector.shape[0]:
+ this_iv = synced_time_vector[idx_start + j :]
this_bt = this_bt[:-1]
else:
- this_iv = synced_time_vector[idx_start+j:idx_end+j]
+ this_iv = synced_time_vector[idx_start + j : idx_end + j]
else:
- this_iv = synced_time_vector[idx_start-j:idx_end-j]
-
- median_now = np.median(this_iv-this_bt)
-
+ this_iv = synced_time_vector[idx_start - j : idx_end - j]
+
+ median_now = np.median(this_iv - this_bt)
+
if median_now >= min_diff and median_now < max_diff:
- vec_shift[i] = j*new_sign
+ vec_shift[i] = j * new_sign
break
elif median_now < min_diff:
new_sign = 1
@@ -81,61 +80,65 @@ def get_shift_vector(synced_time_vector, behavior_time_vector, base_size=40,init
new_sign = -1
if new_sign != sign:
- if np.abs(median_ori-median_diff) < np.abs(median_now-median_diff):
+ if np.abs(median_ori - median_diff) < np.abs(median_now - median_diff):
vec_shift[i] = 0
else:
- vec_shift[i] = j*sign
+ vec_shift[i] = j * sign
break
-
- if j == samples_shift-1:
- #print('Extreme case !!!')
- vec_shift[i] = j*sign
+
+ if j == samples_shift - 1:
+ # print('Extreme case !!!')
+ vec_shift[i] = j * sign
new_synced_time_vector = synced_time_vector.copy()
- mid_point = int(initial_sample+base_size/2)
+ mid_point = int(initial_sample + base_size / 2)
for i in range(vec_shift.shape[0]):
- new_synced_time_vector[mid_point+i] = synced_time_vector[mid_point+i+vec_shift[i]]
-
- idx_end = mid_point+vec_shift.shape[0]-1
+ new_synced_time_vector[mid_point + i] = synced_time_vector[mid_point + i + vec_shift[i]]
+
+ idx_end = mid_point + vec_shift.shape[0] - 1
for i in range(idx_end, new_synced_time_vector.shape[0]):
- if i+vec_shift[-1] < new_synced_time_vector.shape[0]:
- new_synced_time_vector[i] = synced_time_vector[i+vec_shift[-1]]
+ if i + vec_shift[-1] < new_synced_time_vector.shape[0]:
+ new_synced_time_vector[i] = synced_time_vector[i + vec_shift[-1]]
max_shift = np.max(np.abs(vec_shift))
-
if max_shift > diff_size and diff_size != 1:
pass
- #print('max_shift', max_shift)
- #print('diff_size', diff_size)
- #print('walaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
- #raise ValueError('more max shift than diff size')
+ # print('max_shift', max_shift)
+ # print('diff_size', diff_size)
+ # print('walaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
+ # raise ValueError('more max shift than diff size')
return new_synced_time_vector, vec_shift, [min_diff, median_diff, max_diff]
def fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, vec_shift, initial_sample=0, base_size=40):
- mid_point = int(initial_sample+base_size/2)
+ mid_point = int(initial_sample + base_size / 2)
new_synced_time_vector = synced_time_vector.copy()
diff_vec_shift = np.diff(vec_shift)
where_insert_iteration = np.where(diff_vec_shift != 0)
where_insert_iteration = where_insert_iteration[0]
- #print('where_insert_iteration', where_insert_iteration)
+ # print('where_insert_iteration', where_insert_iteration)
find = np.full(4, 1)
correlation_result = np.correlate(diff_vec_shift == 0, find)
consecutive_zeros = np.flatnonzero(correlation_result == 4)
index_shift = 0
- index_borrow_virmen = list()
- for i in range(len(where_insert_iteration)):
+ index_borrow_virmen = []
+ for _i in range(len(where_insert_iteration)):
stable_parts = np.where(consecutive_zeros > where_insert_iteration[index_shift])
stable_parts = stable_parts[0]
if stable_parts.shape[0] > 0:
- index_borrow_virmen.append([where_insert_iteration[index_shift], consecutive_zeros[stable_parts[0]]])
+ index_borrow_virmen.append(
+ [
+ where_insert_iteration[index_shift],
+ consecutive_zeros[stable_parts[0]],
+ ]
+ )
next_borrow_start = np.where(where_insert_iteration > consecutive_zeros[stable_parts[0]])
next_borrow_start = next_borrow_start[0]
if next_borrow_start.shape[0] > 0:
@@ -143,59 +146,57 @@ def fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, vec_shift,
else:
break
else:
- print('Extreme case diff vec shift')
+ logger.warning("Extreme case diff vec shift")
index_borrow_virmen.append([where_insert_iteration[index_shift], diff_vec_shift.shape[0]])
- print('index_borrow_virmen', index_borrow_virmen)
+ logger.debug("index_borrow_virmen: %s", index_borrow_virmen)
break
+ # print('index_borrow_virmen', index_borrow_virmen)
-
- #print('index_borrow_virmen', index_borrow_virmen)
-
- #for i in range(where_insert_iteration.shape[0]):
- index_diff_e = -1
- borrowed_indexes = list()
+ # for i in range(where_insert_iteration.shape[0]):
+ index_diff_e = -1
+ borrowed_indexes = []
for i in range(len(index_borrow_virmen)):
-
- if index_diff_e > index_borrow_virmen[i][0]+mid_point:
+ if index_diff_e > index_borrow_virmen[i][0] + mid_point:
continue
for j in range(100):
+ index_diff_s = index_borrow_virmen[i][0] + mid_point - 1
+ index_diff_e = index_borrow_virmen[i][1] + mid_point + 1
- index_diff_s = index_borrow_virmen[i][0]+mid_point-1
- index_diff_e = index_borrow_virmen[i][1]+mid_point+1
+ if j >= 1:
+ index_diff_s = index_diff_s - 1
+ if j > 1:
+ index_diff_e = index_diff_e + j - 1
- if j>=1:
- index_diff_s = index_diff_s-1
- if j >1:
- index_diff_e = index_diff_e+j-1
+ # print('index_diff_vec', index_diff_s, index_diff_e)
+ # print('diff_vec_shift[index_diff] ', diff_vec_shift[index_diff_vec-1:index_diff_vec+1] )
- #print('index_diff_vec', index_diff_s, index_diff_e)
- #print('diff_vec_shift[index_diff] ', diff_vec_shift[index_diff_vec-1:index_diff_vec+1] )
+ # print('behavior_time_vector', behavior_time_vector[index_diff_s-1:index_diff_e+2])
+ # print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])
- #print('behavior_time_vector', behavior_time_vector[index_diff_s-1:index_diff_e+2])
- #print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])
+ time_iteration = behavior_time_vector[index_diff_s : index_diff_e + 1] - behavior_time_vector[index_diff_s]
+ # print('time_iteration', time_iteration)
- time_iteration = behavior_time_vector[index_diff_s:index_diff_e+1] - behavior_time_vector[index_diff_s]
- #print('time_iteration', time_iteration)
+ new_synced_time_vector[index_diff_s : index_diff_e + 1] = (
+ np.repeat(
+ new_synced_time_vector[index_diff_s],
+ index_diff_e - index_diff_s + 1,
+ )
+ + time_iteration
+ )
- new_synced_time_vector[index_diff_s:index_diff_e+1] = np.repeat(new_synced_time_vector[index_diff_s], index_diff_e-index_diff_s+1) +\
- time_iteration
-
- check_diff = np.diff(new_synced_time_vector[index_diff_s:index_diff_e+2])
+ check_diff = np.diff(new_synced_time_vector[index_diff_s : index_diff_e + 2])
idx_back_time = np.where(check_diff <= 0.0005)
idx_back_time = idx_back_time[0]
if idx_back_time.shape[0] == 0:
break
-
-
- borrowed_indexes.append([index_diff_s,index_diff_e+1])
- #print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])
- #print('diff new_synced_time_vector2', np.diff(new_synced_time_vector2[index_diff_s-1:index_diff_e+2]))
- #print('diff new_synced_time_vector2', np.diff(behavior_time_vector[index_diff_s-1:index_diff_e+2]))
-
+ borrowed_indexes.append([index_diff_s, index_diff_e + 1])
+ # print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])
+ # print('diff new_synced_time_vector2', np.diff(new_synced_time_vector2[index_diff_s-1:index_diff_e+2]))
+ # print('diff new_synced_time_vector2', np.diff(behavior_time_vector[index_diff_s-1:index_diff_e+2]))
return new_synced_time_vector, borrowed_indexes
@@ -204,7 +205,7 @@ def fix_sync_vector_greater(sync_time_vector, behavior_time_vector):
new_sync_time_vector = sync_time_vector.copy()
- diff_vecs = (new_sync_time_vector - behavior_time_vector[:new_sync_time_vector.shape[0]])
+ diff_vecs = new_sync_time_vector - behavior_time_vector[: new_sync_time_vector.shape[0]]
idx_plus = np.where(diff_vecs > 0)
idx_plus = idx_plus[0]
@@ -214,45 +215,53 @@ def fix_sync_vector_greater(sync_time_vector, behavior_time_vector):
borrowed_indexes = []
for i in range(len(grouped_sections)):
-
if grouped_sections[i].shape[0] > 0:
-
- idx_start = grouped_sections[i][0]-1
+ idx_start = grouped_sections[i][0] - 1
idx_end = grouped_sections[i][-1]
- time_vector = behavior_time_vector[idx_start:idx_end+1]-behavior_time_vector[idx_start]
- new_sync_time_vector[idx_start:idx_end+1] =\
- np.repeat(new_sync_time_vector[idx_start], idx_end-idx_start+1) + time_vector
-
+ time_vector = behavior_time_vector[idx_start : idx_end + 1] - behavior_time_vector[idx_start]
+ new_sync_time_vector[idx_start : idx_end + 1] = (
+ np.repeat(new_sync_time_vector[idx_start], idx_end - idx_start + 1) + time_vector
+ )
+
borrowed_indexes.append([idx_start, idx_end])
-
return new_sync_time_vector, borrowed_indexes
def complete_last_part_sync_vec(sync_time_vector, behavior_time_vector):
new_sync_time_vector = sync_time_vector.copy()
-
+
diff_size = behavior_time_vector.shape[0] - new_sync_time_vector.shape[0]
borrowed_indexes = []
if diff_size > 0:
- diff_size = diff_size+1
+ diff_size = diff_size + 1
- last_part_bt = behavior_time_vector[-diff_size:]- behavior_time_vector[-diff_size]
+ last_part_bt = behavior_time_vector[-diff_size:] - behavior_time_vector[-diff_size]
insert_part = np.repeat(new_sync_time_vector[-1], diff_size) + last_part_bt
- #print('last_part_bt',last_part_bt)
+ # print('last_part_bt',last_part_bt)
- #print('insert_part',insert_part)
+ # print('insert_part',insert_part)
new_sync_time_vector = np.append(new_sync_time_vector, insert_part[1:])
- borrowed_indexes.append([behavior_time_vector.shape[0]-diff_size, behavior_time_vector.shape[0]-1])
-
+ borrowed_indexes.append(
+ [
+ behavior_time_vector.shape[0] - diff_size,
+ behavior_time_vector.shape[0] - 1,
+ ]
+ )
+
return new_sync_time_vector, borrowed_indexes
-
-def fix_iter_vector(synced_iteration_vector, synced_time_vector, original_time_vector, nidq_sampling_rate):
+
+def fix_iter_vector(
+ synced_iteration_vector,
+ synced_time_vector,
+ original_time_vector,
+ nidq_sampling_rate,
+):
new_synced_iteration_vector = synced_iteration_vector.copy()
@@ -263,10 +272,10 @@ def fix_iter_vector(synced_iteration_vector, synced_time_vector, original_time_v
first_iter = synced_iteration_vector[0]
for i in range(diff_iter_times_idx.shape[0]):
if diff_iter_times_idx[i] != 0:
- new_synced_iteration_vector[i] = first_iter+int(synced_time_vector[i]*nidq_sampling_rate)
+ new_synced_iteration_vector[i] = first_iter + int(synced_time_vector[i] * nidq_sampling_rate)
if ori_shape != new_shape:
- last_iterations = (np.round(first_iter+synced_time_vector[ori_shape:]*nidq_sampling_rate))
+ last_iterations = np.round(first_iter + synced_time_vector[ori_shape:] * nidq_sampling_rate)
new_synced_iteration_vector = np.append(new_synced_iteration_vector, last_iterations.astype(np.int64))
return new_synced_iteration_vector
@@ -275,99 +284,100 @@ def fix_iter_vector(synced_iteration_vector, synced_time_vector, original_time_v
def sync_evaluation_process2(synced_time_vector, behavior_time_vector):
status = 1
- diff_vector = synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]]
+ diff_vector = synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]]
num_iter = diff_vector.shape[0]
- #max_diff = max(diff_vector)
+ # max_diff = max(diff_vector)
median_general = np.median(diff_vector)
- num_div= 10
+ num_div = 10
median_diff_percent = np.empty([num_div])
median_diff_abs = np.empty([num_div])
for j in range(num_div):
- start_iter = int(j*num_iter/num_div)
- end_iter = int((j+1)*num_iter/num_div)
- median_diff_percent[j] = (np.median(diff_vector[start_iter:end_iter])-median_general)*100/median_general
- median_diff_abs[j] = (np.median(diff_vector[start_iter:end_iter])-median_general)
-
+ start_iter = int(j * num_iter / num_div)
+ end_iter = int((j + 1) * num_iter / num_div)
+ median_diff_percent[j] = (np.median(diff_vector[start_iter:end_iter]) - median_general) * 100 / median_general
+ median_diff_abs[j] = np.median(diff_vector[start_iter:end_iter]) - median_general
if np.max(np.abs(median_diff_abs)) < 0.005:
pass
else:
status = -1
- print(median_general)
+ logger.debug("median_general: %s", median_general)
return status
def main_ephys_fix_sync_code(iter_start_idx, iter_times_idx, behavior_time, nidq_sampling_rate):
- iteration_dict = dict()
- iteration_dict['iter_start_idx'] = list()
- iteration_dict['iter_times_idx'] = list()
+ iteration_dict = {}
+ iteration_dict["iter_start_idx"] = []
+ iteration_dict["iter_times_idx"] = []
for i in range(len(iter_start_idx)):
-
- #print('fixing trial ',i)
+ # print('fixing trial ',i)
behavior_time_vector = behavior_time[i].flatten()
- synced_time_vector, shift_vec, median_vec = get_shift_vector(iter_times_idx[i],behavior_time_vector)
-
+ synced_time_vector, shift_vec, median_vec = get_shift_vector(iter_times_idx[i], behavior_time_vector)
- synced_time_vector,_ =\
- fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, shift_vec)
+ synced_time_vector, _ = fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, shift_vec)
- #synced_time_vector, trial_stats_dict['borrow_step3'] =\
+ # synced_time_vector, trial_stats_dict['borrow_step3'] =\
# fix_sync_vector_greater(synced_time_vector, behavior_time_vector)
- synced_time_vector,_ =\
- complete_last_part_sync_vec(synced_time_vector, behavior_time_vector)
+ synced_time_vector, _ = complete_last_part_sync_vec(synced_time_vector, behavior_time_vector)
- synced_iteration_vector =\
- fix_iter_vector(iter_start_idx[i],synced_time_vector, iter_times_idx[i], nidq_sampling_rate)
-
+ synced_iteration_vector = fix_iter_vector(
+ iter_start_idx[i], synced_time_vector, iter_times_idx[i], nidq_sampling_rate
+ )
- iteration_dict['iter_start_idx'].append(synced_iteration_vector.copy())
- iteration_dict['iter_times_idx'].append(synced_time_vector.copy())
+ iteration_dict["iter_start_idx"].append(synced_iteration_vector.copy())
+ iteration_dict["iter_times_idx"].append(synced_time_vector.copy())
- print('end fix sync code 1')
+ logger.debug("end fix sync code 1")
- iteration_dict['iter_start_idx'] = np.asarray(iteration_dict['iter_start_idx'].copy(), dtype=object)
- iteration_dict['iter_times_idx'] = np.asarray(iteration_dict['iter_times_idx'].copy(), dtype=object)
+ iteration_dict["iter_start_idx"] = np.asarray(iteration_dict["iter_start_idx"].copy(), dtype=object)
+ iteration_dict["iter_times_idx"] = np.asarray(iteration_dict["iter_times_idx"].copy(), dtype=object)
- print('end fix sync code')
+ logger.debug("end fix sync code")
# Check # of trials and iterations match
- trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)
+ trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (
+ ephys_utils.assert_iteration_samples_count(iteration_dict["iter_start_idx"], behavior_time)
+ )
- print('after assert_iteration_samples_count fix sync code')
+ logger.debug("after assert_iteration_samples_count fix sync code")
if trial_count_diff != 0:
- print('trial_count_diff', trial_count_diff)
+ logger.debug("trial_count_diff: %s", trial_count_diff)
if len(trials_diff_iteration_big) > 0:
- print('trials_diff_iteration_big', trials_diff_iteration_big)
+ logger.debug("trials_diff_iteration_big: %s", trials_diff_iteration_big)
if len(trials_diff_iteration_small) > 0:
- print('trials_diff_iteration_small', trials_diff_iteration_small)
-
+ logger.debug("trials_diff_iteration_small: %s", trials_diff_iteration_small)
+ status = ephys_utils.evaluate_sync_process(
+ trial_count_diff,
+ trials_diff_iteration_big,
+ trials_diff_iteration_small,
+ behavior_time.shape[0],
+ )
- status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])
+ logger.debug("after evaluate_sync_process fix sync code")
- print('after evaluate_sync_process fix sync code')
-
- for i in range(len(iteration_dict['iter_start_idx'])):
- synced_time_vector = iteration_dict['iter_times_idx'][i]
+ for i in range(len(iteration_dict["iter_start_idx"])):
+ synced_time_vector = iteration_dict["iter_times_idx"][i]
behavior_time_vector = behavior_time[i].flatten()
status = sync_evaluation_process2(synced_time_vector, behavior_time_vector)
if status == -1:
break
- print('after sync_evaluation_process2', status)
-
+ logger.debug("after sync_evaluation_process2: status=%s", status)
+
if status == 1:
- iteration_dict['trial_start_idx'] = ephys_utils.get_index_trial_vector_from_iteration(iteration_dict['iter_start_idx'])
+ iteration_dict["trial_start_idx"] = ephys_utils.get_index_trial_vector_from_iteration(
+ iteration_dict["iter_start_idx"]
+ )
- print('after get_index_trial_vector_from_iteration')
+ logger.debug("after get_index_trial_vector_from_iteration")
return status, iteration_dict
-
diff --git a/u19_pipeline/utils/ephys_utils.py b/u19_pipeline/utils/ephys_utils.py
index 9aa18cf8..8e9daa8d 100644
--- a/u19_pipeline/utils/ephys_utils.py
+++ b/u19_pipeline/utils/ephys_utils.py
@@ -1,22 +1,20 @@
-
-import numpy as np
-import datajoint as dj
+import json
import pathlib
import warnings
-import json
+import datajoint as dj
+import numpy as np
+from bitstring import BitArray
from scipy import signal as sp
from scipy.io import loadmat
-from scipy.spatial.transform import Rotation as R
-
-from bitstring import BitArray
-from element_array_ephys import ephys as ephys_element
import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX
+from u19_pipeline.utils.logging_config import get_logger
+logger = get_logger(__name__)
-class spice_glx_utility:
+class SpiceGlxUtility:
@staticmethod
def load_spice_glx_digital_file(file_path, nidq_meta, d_line_list=None):
# Read NIDAQ digital file.
@@ -27,80 +25,88 @@ def load_spice_glx_digital_file(file_path, nidq_meta, d_line_list=None):
# Get all digitial channels if not provided
if d_line_list is None:
- list_start_end_chan = nidq_meta['niXDChans1'].split(sep=':')
- print(list_start_end_chan)
- if len(list_start_end_chan) == 2:
- d_line_list = list(range(int(list_start_end_chan[0]), int(list_start_end_chan[1])+1))
+ list_start_end_chan = nidq_meta["niXDChans1"].split(sep=":")
+ logger.debug("list_start_end_chan: %s", list_start_end_chan)
+ if len(list_start_end_chan) == 2:
+ d_line_list = list(range(int(list_start_end_chan[0]), int(list_start_end_chan[1]) + 1))
else:
raise ValueError('Could not infer channel list from nidq_meta["niXDChans1"] ')
nidq_sampling_rate = readSGLX.SampRate(nidq_meta)
- #Get first and last sample idx from the file
+ # Get first and last sample idx from the file
t_start = 0
- t_end = np.float64(nidq_meta['fileTimeSecs'])
+ t_end = np.float64(nidq_meta["fileTimeSecs"])
dw = 0
first_sample_index = int(nidq_sampling_rate * t_start)
last_sample_index = int(nidq_sampling_rate * t_end) - 1
- #Read binary and digital
+ # Read binary and digital
nidq_raw_data = readSGLX.makeMemMapRaw(file_path, nidq_meta) # Pull raw bin data
- digital_array = readSGLX.ExtractDigital( # extract interation index
- nidq_raw_data, first_sample_index, last_sample_index,
- dw, d_line_list, nidq_meta)
+ digital_array = readSGLX.ExtractDigital( # extract interation index
+ nidq_raw_data,
+ first_sample_index,
+ last_sample_index,
+ dw,
+ d_line_list,
+ nidq_meta,
+ )
return digital_array
-
+
def read_nidq_meta_samp_rate(ephys_session_fullpath):
- #Nidaq file
- nidq_meta = readSGLX.readMeta(ephys_session_fullpath)
+ # Nidaq file
+ nidq_meta = readSGLX.readMeta(ephys_session_fullpath)
nidq_sampling_rate = readSGLX.SampRate(nidq_meta)
return nidq_meta, nidq_sampling_rate
+
def load_trial_iteration_signals(ephys_session_fullpath, nidq_meta):
- # 1: load meta data, and the content of the NIDAQ file. Its content is digital.
+ # 1: load meta data, and the content of the NIDAQ file. Its content is digital.
new_trial_channel = 1
new_iteration_channel = 2
# If PXIe card (nidq) card use for recording deduce digital channels
- if nidq_meta['typeThis'] == 'nidq':
- digital_array = spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)
+ if nidq_meta["typeThis"] == "nidq":
+ digital_array = SpiceGlxUtility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)
# If onebox card (obx) card use for recording digital channels are 0-2
else:
- digital_array = spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])
+ digital_array = SpiceGlxUtility.load_spice_glx_digital_file(
+ ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]
+ )
# If no sync pulse found trial and iteration signals are 0 & 1 respectively
- channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]
- channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]
+ channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]
+ channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]
if channel0_pulses > channel1_pulses:
new_trial_channel = 1
new_iteration_channel = 0
else:
new_trial_channel = 0
- new_iteration_channel = 1
+ new_iteration_channel = 1
- return digital_array[new_trial_channel,:], digital_array[new_iteration_channel,:]
+ return digital_array[new_trial_channel, :], digital_array[new_iteration_channel, :]
def get_idx_trial_start(trial_pulse_signal):
- #Get index of samples when trial has started based on a pulse signal
+ # Get index of samples when trial has started based on a pulse signal
- #Get idx samples trial starts
+ # Get idx samples trial starts
trial_start_idx = np.where(np.diff(trial_pulse_signal) == 1)
trial_start_idx = trial_start_idx[0]
- #Get idx samples trial pulse start end
+ # Get idx samples trial pulse start end
trial_start_pulse_end_idx = np.where(np.diff(trial_pulse_signal) == 255)
trial_start_pulse_end_idx = trial_start_pulse_end_idx[0]
- #Detect fake trial init pulses (a single sample in 1 instead of 5ms signal)
+ # Detect fake trial init pulses (a single sample in 1 instead of 5ms signal)
fake_trial_init = []
for idx, sample in enumerate(trial_start_idx):
- #Get the mean value of next samples after rising edge detected
- mean_pulse = np.mean(trial_pulse_signal[sample+1:sample+10])
+ # Get the mean value of next samples after rising edge detected
+ mean_pulse = np.mean(trial_pulse_signal[sample + 1 : sample + 10])
# Average value should be 1
if mean_pulse < 0.9:
fake_trial_init.append(idx)
@@ -110,10 +116,16 @@ def get_idx_trial_start(trial_pulse_signal):
return trial_start_idx, trial_start_pulse_end_idx
-def get_idx_iter_start_pulsesignal(iteration_pulse_signal_trial, iteration_pulse_start_signal,trial_start_idx, samples_before_pulse_start, behavior_iterations):
- #Get index of iteration starts on a trial based on a pulse start signal
+def get_idx_iter_start_pulsesignal(
+ iteration_pulse_signal_trial,
+ iteration_pulse_start_signal,
+ trial_start_idx,
+ samples_before_pulse_start,
+ behavior_iterations,
+):
+ # Get index of iteration starts on a trial based on a pulse start signal
- #Get idx of iteration start during trial (pulse signal after trial start signal)
+ # Get idx of iteration start during trial (pulse signal after trial start signal)
iter_samples = np.where(np.diff(iteration_pulse_signal_trial) == 1)
iter_samples = np.squeeze(iter_samples)
# First iteration is at trial start, just align first trial start
@@ -122,12 +134,12 @@ def get_idx_iter_start_pulsesignal(iteration_pulse_signal_trial, iteration_pulse
iter_start_samples = np.where(np.diff(iteration_pulse_start_signal) == 1)
iter_start_samples = np.squeeze(iter_start_samples)
- #print('iter_start_samples', iter_start_samples)
+ # print('iter_start_samples', iter_start_samples)
- #print('iter_start_samples.size', iter_start_samples.size)
- #print('type(iter_start_samples)', type(iter_start_samples))
+ # print('iter_start_samples.size', iter_start_samples.size)
+ # print('type(iter_start_samples)', type(iter_start_samples))
- # If we find some kind of pulse at the start of
+ # If we find some kind of pulse at the start of
if iter_start_samples.size > 0:
iter_samples = np.insert(iter_samples, 0, trial_start_idx)
else:
@@ -140,15 +152,14 @@ def get_idx_iter_start_pulsesignal(iteration_pulse_signal_trial, iteration_pulse
def get_idx_iter_start_counterbit(iteration_pulse_signal_trial, trial_start_idx):
- #Get index of iteration starts on a trial based on a iteration bit0 counter
-
+ # Get index of iteration starts on a trial based on a iteration bit0 counter
- #Get idx of odd iteration during trial
+ # Get idx of odd iteration during trial
iter_samples = np.where(np.diff(iteration_pulse_signal_trial) == 1)
iter_samples = iter_samples + trial_start_idx
if iteration_pulse_signal_trial[0] == 1:
- #If last iteration was odd, insert a iteration at start
+ # If last iteration was odd, insert a iteration at start
iter_samples = np.insert(iter_samples, 0, trial_start_idx)
else:
# First iteration is at trial start, just align first trial start
@@ -156,7 +167,7 @@ def get_idx_iter_start_counterbit(iteration_pulse_signal_trial, trial_start_idx)
iter_samples = np.squeeze(iter_samples)
- #Get idx of even iteration during trial
+ # Get idx of even iteration during trial
iter_samples2 = np.where(np.diff(iteration_pulse_signal_trial) == 255)
iter_samples2 = iter_samples2 + trial_start_idx
iter_samples2 = np.squeeze(iter_samples2)
@@ -172,43 +183,50 @@ def get_trial_signal_mode(iteration_pulse_signal_trial, behavior_time_vector_tri
# If iterations in trial are less than the ones in behavior, the mode was the counterbit
iter_samples = np.where(np.diff(iteration_pulse_signal_trial) == 1)
- if iter_samples[0].shape[0] < (behavior_time_vector_trial.shape[0]*3/4):
- mode = 'counter_bit0'
- else:
- mode = 'pulse_signal'
+ mode = "counter_bit0" if iter_samples[0].shape[0] < behavior_time_vector_trial.shape[0] * 3 / 4 else "pulse_signal"
- print('mode deduction: ', mode)
+ logger.debug("mode deduction: %s", mode)
return mode
-def get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal, nidq_sampling_rate, num_behavior_trials, behavior_time_vector, mode=None) -> dict:
+def get_iteration_sample_vector_from_digital_lines_pulses(
+ trial_pulse_signal,
+ iteration_pulse_signal,
+ nidq_sampling_rate,
+ num_behavior_trials,
+ behavior_time_vector,
+ mode=None,
+) -> dict:
- #Output as a dictionary
- iteration_vector_output = dict()
+ # Output as a dictionary
+ iteration_vector_output = {}
- #Vectors that will contain trial # and iter # for each sample on file
- #iteration_vector_output['framenumber_vector_samples'] = np.zeros(trial_pulse_signal.shape[0])*np.nan
- #iteration_vector_output['trialnumber_vector_samples'] = np.zeros(trial_pulse_signal.shape[0])*np.nan
+ # Vectors that will contain trial # and iter # for each sample on file
+ # iteration_vector_output['framenumber_vector_samples'] = np.zeros(trial_pulse_signal.shape[0])*np.nan
+ # iteration_vector_output['trialnumber_vector_samples'] = np.zeros(trial_pulse_signal.shape[0])*np.nan
- #Get idx samples trial starts
+ # Get idx samples trial starts
trial_start_idx, trial_end_pulse_idx = get_idx_trial_start(trial_pulse_signal)
- print('len trial_start_idx', trial_start_idx.shape)
+ logger.debug("len trial_start_idx: %s", trial_start_idx.shape)
if mode is None:
- mode = get_trial_signal_mode(iteration_pulse_signal[trial_start_idx[0]:trial_start_idx[1]], behavior_time_vector[0])
+ mode = get_trial_signal_mode(
+ iteration_pulse_signal[trial_start_idx[0] : trial_start_idx[1]],
+ behavior_time_vector[0],
+ )
# Just to make sure we get corresponding iter pulse (trial and iter pulse at same time !!)
- if mode == 'counter_bit0':
+ if mode == "counter_bit0":
ms_after_trial_start_pulse = 1
ms_before_trial_end = 1
else:
ms_after_trial_start_pulse = -4
ms_before_trial_end = 4
- samples_after_pulse_start = int(nidq_sampling_rate*(ms_after_trial_start_pulse/1000))
- samples_before_pulse_end = int(nidq_sampling_rate*(ms_before_trial_end/1000))
+ samples_after_pulse_start = int(nidq_sampling_rate * (ms_after_trial_start_pulse / 1000))
+ samples_before_pulse_end = int(nidq_sampling_rate * (ms_before_trial_end / 1000))
# num Trials to sync (if behavior stopped before last trial was saved)
num_trials_sync = min([trial_start_idx.shape[0], num_behavior_trials])
@@ -216,44 +234,50 @@ def get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, it
iter_start_idx = []
iter_times_idx = []
for i in range(num_trials_sync):
- #Trial starts for iteration signal when trial pulse ends ()
+ # Trial starts for iteration signal when trial pulse ends ()
idx_start = trial_end_pulse_idx[i]
idx_start_before = trial_start_idx[i] + samples_after_pulse_start
- if i < trial_start_idx.shape[0]-1:
- idx_end = trial_start_idx[i+1] -samples_before_pulse_end
+ if i < trial_start_idx.shape[0] - 1:
+ idx_end = trial_start_idx[i + 1] - samples_before_pulse_end
else:
idx_end = trial_pulse_signal.shape[0] - samples_before_pulse_end
-
-
- #Get idx of iteration start of current trial
- if mode == 'counter_bit0':
- iter_samples = get_idx_iter_start_counterbit(iteration_pulse_signal[idx_start_before:idx_end], trial_start_idx[i])
+ # Get idx of iteration start of current trial
+ if mode == "counter_bit0":
+ iter_samples = get_idx_iter_start_counterbit(
+ iteration_pulse_signal[idx_start_before:idx_end], trial_start_idx[i]
+ )
else:
- iter_samples = get_idx_iter_start_pulsesignal(iteration_pulse_signal[idx_start:idx_end], iteration_pulse_signal[idx_start_before:idx_start], trial_start_idx[i], idx_start, behavior_time_vector[i].shape[0])
-
- #Append as an array of arrays (each trial is an array with idx of iterations)
+ iter_samples = get_idx_iter_start_pulsesignal(
+ iteration_pulse_signal[idx_start:idx_end],
+ iteration_pulse_signal[idx_start_before:idx_start],
+ trial_start_idx[i],
+ idx_start,
+ behavior_time_vector[i].shape[0],
+ )
+
+ # Append as an array of arrays (each trial is an array with idx of iterations)
iter_start_idx.append(iter_samples)
- #Calculate time for each iteration start
- times = iter_samples/nidq_sampling_rate
+ # Calculate time for each iteration start
+ times = iter_samples / nidq_sampling_rate
times = times - times[0]
iter_times_idx.append(times)
- #Fill vector samples
- #for j in range(iter_samples.shape[0]-1):
+ # Fill vector samples
+ # for j in range(iter_samples.shape[0]-1):
# iteration_vector_output['framenumber_vector_samples'][iter_samples[j]:iter_samples[j+1]] = j+1
- #Last iteration # is from start of iteration to end of trial
- #if i < trial_start_idx.shape[0]-1:
+ # Last iteration # is from start of iteration to end of trial
+ # if i < trial_start_idx.shape[0]-1:
# iteration_vector_output['trialnumber_vector_samples'][trial_start_idx[i]:trial_start_idx[i+1]] = i+1
# iteration_vector_output['framenumber_vector_samples'][iter_samples[-1]:trial_start_idx[i+1]] = iter_samples.shape[0]
# For last trial, lets finish it 1s after last iteration detected
- #else:
+ # else:
# iteration_vector_output['trialnumber_vector_samples'][trial_start_idx[i]:iter_samples[-1]+int(nidq_sampling_rate)] = i+1
# iteration_vector_output['framenumber_vector_samples'][iter_samples[-1]:iter_samples[-1]+int(nidq_sampling_rate)] = iter_samples.shape[0]
- iteration_vector_output['iter_start_idx'] = np.asarray(iter_start_idx.copy(), dtype=object)
- iteration_vector_output['iter_times_idx'] = np.asarray(iter_times_idx.copy(), dtype=object)
+ iteration_vector_output["iter_start_idx"] = np.asarray(iter_start_idx.copy(), dtype=object)
+ iteration_vector_output["iter_times_idx"] = np.asarray(iter_times_idx.copy(), dtype=object)
return iteration_vector_output
@@ -264,35 +288,44 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite
# ... and also get start and end time
framenumber = np.zeros(digital_array.shape[1])
for i in range(digital_array.shape[1]):
- a = BitArray(np.flip(digital_array[1:, i])) # ignore 0-bit, as this is the NPX sync puls, and not virmen.
+ a = BitArray(np.flip(digital_array[1:, i])) # ignore 0-bit, as this is the NPX sync puls, and not virmen.
framenumber[i] = a.uint
- iterations_raw = np.array(framenumber, dtype=np.int32) # Transform frames into integer
- recording_start = np.min(np.where(iterations_raw>0)) #first chane of testlist
- recording_end = np.where(np.abs(np.diff(iterations_raw))>0)[0][-1] + 200 # Adding a random 200 measurements, so ~40ms at our usual 5kHz sampling rate.
+ iterations_raw = np.array(framenumber, dtype=np.int32) # Transform frames into integer
+ recording_start = np.min(np.where(iterations_raw > 0)) # first chane of testlist
+ recording_end = (
+ np.where(np.abs(np.diff(iterations_raw)) > 0)[0][-1] + 200
+ ) # Adding a random 200 measurements, so ~40ms at our usual 5kHz sampling rate.
# Second, transform `iterations_raw` into `framenumber_in_trial` and `trialnumber`
- framenumber_in_trial = np.zeros(len(iterations_raw))*np.nan
- trialnumber = np.zeros(len(iterations_raw))*np.nan
+ framenumber_in_trial = np.zeros(len(iterations_raw)) * np.nan
+ trialnumber = np.zeros(len(iterations_raw)) * np.nan
current_trial = 0
overflow = 0
iter_start_idx = []
for idx, frame_number in enumerate(iterations_raw):
- if (idx>recording_start) & (idx recording_start) & (idx < recording_end):
+ if (frame_number == 0) & (iterations_raw[idx - 1] == 127): # At the reset, add 128
overflow = overflow + 1
- if (frame_number==0) & (iterations_raw[idx-1]!=127) & (iterations_raw[idx-1]!=0) & (iterations_raw[idx-2]==127): # Unlucky reset if happened to be sampled at the wrong time
+ if (
+ (frame_number == 0)
+ & (iterations_raw[idx - 1] != 127)
+ & (iterations_raw[idx - 1] != 0)
+ & (iterations_raw[idx - 2] == 127)
+ ): # Unlucky reset if happened to be sampled at the wrong time
overflow = overflow + 1
- framenumber_in_trial[idx-1] = frame_number + overflow*128 - 1 # In case this happened, the previous sample has to be corrected
+ framenumber_in_trial[idx - 1] = (
+ frame_number + overflow * 128 - 1
+ ) # In case this happened, the previous sample has to be corrected
# Keep track of trial number
- endflag = framenumber_in_trial[idx-1] == (len(time[current_trial])) #Trial end has been reached.
- transitionflag = frame_number < 3 # Next trial should start at zero again
- if endflag & transitionflag: # Only at the transitions
+ endflag = framenumber_in_trial[idx - 1] == (len(time[current_trial])) # Trial end has been reached.
+ transitionflag = frame_number < 3 # Next trial should start at zero again
+ if endflag & transitionflag: # Only at the transitions
current_trial = current_trial + 1 # Increases trial count
- overflow = 0 # Reset the 7 bit counter
- iter_start_idx.append(idx) # Make a note when this happened
- framenumber_in_trial[idx] = frame_number + overflow*128 - 1
+ overflow = 0 # Reset the 7 bit counter
+ iter_start_idx.append(idx) # Make a note when this happened
+ framenumber_in_trial[idx] = frame_number + overflow * 128 - 1
trialnumber[idx] = current_trial
- trial_list = np.array(np.unique(trialnumber[np.isfinite(trialnumber)]), dtype = np.int32)
+ trial_list = np.array(np.unique(trialnumber[np.isfinite(trialnumber)]), dtype=np.int32)
# Fourth, find and remove the nidaq glitches
# These are single samples where the iteration number is corrupted
@@ -303,62 +336,68 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite
din = np.diff(framenumber_in_trial)
trial_transitions = np.where(np.diff(trialnumber))
glitches = []
- for candidate in np.where( np.logical_or(din>1, din<0) )[0]: # skipped frames or counting down
+ for candidate in np.where(np.logical_or(din > 1, din < 0))[0]: # skipped frames or counting down
if np.sum(candidate == trial_transitions) == 0:
glitches = np.append(glitches, candidate)
- glitches = np.array(glitches, dtype = np.int32)
+ glitches = np.array(glitches, dtype=np.int32)
# Attempt to remove them
skipped_frames = 0
for g in glitches:
- if framenumber_in_trial[g] < framenumber_in_trial[g+2]:
- if framenumber_in_trial[g+2] - framenumber_in_trial[g] == 2: # skipped frame, should be very rare
- framenumber_in_trial[g+1] = framenumber_in_trial[g]+1
+ if framenumber_in_trial[g] < framenumber_in_trial[g + 2]:
+ if framenumber_in_trial[g + 2] - framenumber_in_trial[g] == 2: # skipped frame, should be very rare
+ framenumber_in_trial[g + 1] = framenumber_in_trial[g] + 1
skipped_frames = skipped_frames + 1
- else: # If random number, nidaq sample in the middle of update.
- framenumber_in_trial[g+1] = framenumber_in_trial[g]
+ else: # If random number, nidaq sample in the middle of update.
+ framenumber_in_trial[g + 1] = framenumber_in_trial[g]
# This point we have framenumber_in_trial and trialnumber. Now just some refactoring to fit into the usual data structure
- iteration_vector_output = dict()
+ iteration_vector_output = {}
- iteration_vector_output['trialnumber_vector_samples'] = trialnumber
- iteration_vector_output['framenumber_vector_samples'] = framenumber_in_trial
+ iteration_vector_output["trialnumber_vector_samples"] = trialnumber
+ iteration_vector_output["framenumber_vector_samples"] = framenumber_in_trial
iter_start_idx = []
for t in trial_list:
- iter_start_idx.append( np.arange(0, framenumber_in_trial[trialnumber==t][-1], 1))
- iteration_vector_output['iter_start_idx'] = np.asarray(iter_start_idx.copy(), dtype=object)
+ iter_start_idx.append(np.arange(0, framenumber_in_trial[trialnumber == t][-1], 1))
+ iteration_vector_output["iter_start_idx"] = np.asarray(iter_start_idx.copy(), dtype=object)
return iteration_vector_output
+
def assert_iteration_samples_count(iteration_sample_idx_output, behavior_time_vector):
- #Assert that vector sync pulses match behavior time vector
+ # Assert that vector sync pulses match behavior time vector
# Count trial count differences
trial_count_diff = np.abs(iteration_sample_idx_output.shape[0] - (behavior_time_vector.shape[0]))
- count = 0
- trials_diff_iteration_small = list()
- trials_diff_iteration_big = list()
- for idx_trial, iter_trials in enumerate(iteration_sample_idx_output):
- if iter_trials.shape[0] != behavior_time_vector[idx_trial].shape[0]:
- print('trial#', count,
- 'iterPulses:', iter_trials.shape[0],
- 'IterBeh:', behavior_time_vector[idx_trial].shape[0],
- 'Difference (', behavior_time_vector[idx_trial].shape[0]-iter_trials.shape[0], ')')
- count += 1
+ trials_diff_iteration_small = []
+ trials_diff_iteration_big = []
+ for count, iter_trials in enumerate(iteration_sample_idx_output):
+ if iter_trials.shape[0] != behavior_time_vector[count].shape[0]:
+ logger.debug(
+ "trial# %d iterPulses: %d IterBeh: %d Difference (%d)",
+ count,
+ iter_trials.shape[0],
+ behavior_time_vector[count].shape[0],
+ behavior_time_vector[count].shape[0] - iter_trials.shape[0],
+ )
# For each trial iteration # should be equal to the behavioral file iterations
- if iter_trials.shape[0] != behavior_time_vector[idx_trial].shape[0]:
- if np.abs(iter_trials.shape[0] - behavior_time_vector[idx_trial].shape[0]) < 6:
- trials_diff_iteration_small.append(idx_trial)
+ if iter_trials.shape[0] != behavior_time_vector[count].shape[0]:
+ if np.abs(iter_trials.shape[0] - behavior_time_vector[count].shape[0]) < 6:
+ trials_diff_iteration_small.append(count)
else:
- trials_diff_iteration_big.append(idx_trial)
-
+ trials_diff_iteration_big.append(count)
return trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small
-def evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, total_trials):
+def evaluate_sync_process(
+ trial_count_diff,
+ trials_diff_iteration_big,
+ trials_diff_iteration_small,
+ total_trials,
+):
"""
Check if all sync process ran smoothly, we need to redo some trials or it's not worth it
@@ -369,85 +408,87 @@ def evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_di
"""
if len(trials_diff_iteration_big) > 1:
- print('Missed by a lot some trials: ', trials_diff_iteration_big)
+ logger.warning("Missed by a lot some trials: %s", trials_diff_iteration_big)
status = -1
return status
- if len(trials_diff_iteration_big) == 1 and trials_diff_iteration_big[0] == total_trials-1:
- print('Missed by a lot last trial: (Assume recording stopped earlier) ', trials_diff_iteration_big)
+ if len(trials_diff_iteration_big) == 1 and trials_diff_iteration_big[0] == total_trials - 1:
+ logger.warning(
+ "Missed by a lot last trial: (Assume recording stopped earlier) %s",
+ trials_diff_iteration_big,
+ )
status = 1
return status
# All trials synced perfectly
- if trial_count_diff ==0 and len(trials_diff_iteration_small) == 0:
- print('Synced perfectly xxxxxxxxxxxxxx')
+ if trial_count_diff == 0 and len(trials_diff_iteration_small) == 0:
+ logger.info("Synced perfectly")
status = 1
return status
# We miss last trial (surely recording was stop before behavior)
if trial_count_diff < 2 and len(trials_diff_iteration_small) == 0:
- print('Missed one trial signal xxxxxxxxxxxxxx')
+ logger.warning("Missed one trial signal")
status = -1
return status
# Iterations differ in more than two trials
if len(trials_diff_iteration_small) > 2:
- print('Missed iteration count on many trials: ', len(trials_diff_iteration_small))
+ logger.warning("Missed iteration count on many trials: %d", len(trials_diff_iteration_small))
status = -1
return status
if trial_count_diff < 2 and len(trials_diff_iteration_small) <= 2:
- print('Missed num trials: ', trial_count_diff)
- print('Missed iteration count in how many trials: ', len(trials_diff_iteration_small))
- print('Trying to fix trials')
+ logger.warning("Missed num trials: %d", trial_count_diff)
+ logger.warning("Missed iteration count in how many trials: %d", len(trials_diff_iteration_small))
+ logger.info("Trying to fix trials")
status = 0
return status
else:
status = -1
- print('Missed by a lot of trials, everything different or missing')
+ logger.warning("Missed by a lot of trials, everything different or missing")
return status
-
def fix_missing_iteration_trials(trials_diff_iteration_small, iteration_dict, behavior_times, nidq_sampling_rate):
# Fix and insert missing synced iteration vectors
- print('trials_diff_iteration_small', trials_diff_iteration_small)
- print(type(trials_diff_iteration_small))
-
+ logger.debug("trials_diff_iteration_small: %s (type: %s)", trials_diff_iteration_small, type(trials_diff_iteration_small))
# For each bad synced trial (Should be only a few)
for i in range(len(trials_diff_iteration_small)):
-
- #Insert missing iterations on synced vector
+ # Insert missing iterations on synced vector
idx_trial = trials_diff_iteration_small[i]
- status, new_iter_start = insert_missing_synced_iteration(iteration_dict['iter_start_idx'][idx_trial],\
- iteration_dict['iter_times_idx'][idx_trial], behavior_times[idx_trial].flatten())
+ status, new_iter_start = insert_missing_synced_iteration(
+ iteration_dict["iter_start_idx"][idx_trial],
+ iteration_dict["iter_times_idx"][idx_trial],
+ behavior_times[idx_trial].flatten(),
+ )
if not status:
raise ValueError("Coud not find missing iteration in trial")
- iteration_dict['iter_start_idx'][idx_trial] = new_iter_start
+ iteration_dict["iter_start_idx"][idx_trial] = new_iter_start
# Get new synced time vector for trial
- new_times = new_iter_start/nidq_sampling_rate
+ new_times = new_iter_start / nidq_sampling_rate
new_times = new_times - new_times[0]
- iteration_dict['iter_times_idx'][idx_trial] = new_times
+ iteration_dict["iter_times_idx"][idx_trial] = new_times
# Fix framenumber in the sample vector
- #for j in range(new_iter_start.shape[0]-1):
+ # for j in range(new_iter_start.shape[0]-1):
# iteration_dict['framenumber_vector_samples'][new_iter_start[j]:new_iter_start[j+1]] = j+1
- #Last iteration fixed as well
- #next_trial = idx_trial+1
- #start_iteration_next_trial = iteration_dict['iter_start_idx'][next_trial][0]
+ # Last iteration fixed as well
+ # next_trial = idx_trial+1
+ # start_iteration_next_trial = iteration_dict['iter_start_idx'][next_trial][0]
- #print('next_trial ........', next_trial)
- #print('last iteration of this trial so far', new_iter_start[-1])
- #print('start next trial iteration', start_iteration_next_trial)
+ # print('next_trial ........', next_trial)
+ # print('last iteration of this trial so far', new_iter_start[-1])
+ # print('start next trial iteration', start_iteration_next_trial)
- #iteration_dict['framenumber_vector_samples'][new_iter_start[-1]:start_iteration_next_trial] = new_iter_start.shape[0]
+ # iteration_dict['framenumber_vector_samples'][new_iter_start[-1]:start_iteration_next_trial] = new_iter_start.shape[0]
return iteration_dict
@@ -456,57 +497,62 @@ def insert_missing_synced_iteration(synced_iteration_vector, synced_time_vector,
# Check where is more likely we miss an iteration pulse and insert it to iteration_vector
status = 1
- print('synced_iteration_vector', synced_iteration_vector.shape[0])
- print('synced_time_vector', synced_time_vector.shape[0])
- print('behavior_time_vector', behavior_time_vector.shape[0])
+ logger.debug("synced_iteration_vector: %d", synced_iteration_vector.shape[0])
+ logger.debug("synced_time_vector: %d", synced_time_vector.shape[0])
+ logger.debug("behavior_time_vector: %d", behavior_time_vector.shape[0])
# Get in which indexes we get a "peak" of non matching times...
if synced_time_vector.shape[0] >= behavior_time_vector.shape[0]:
- print('More pulses than behavior iterations, check other method') # Christian: What is the other method?
+ logger.warning("More pulses than behavior iterations, check other method")
status = -1
return status, np.empty(0)
else:
- diff_vector = np.diff(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])
- #In case last peak is at the end of trial (append 0 to detect it)
+ diff_vector = np.diff(synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]])
+ # In case last peak is at the end of trial (append 0 to detect it)
diff_vector = np.append(diff_vector, np.array([0]))
peaks, _ = sp.find_peaks(diff_vector, height=0.05, distance=20)
- print('peaks here .............', peaks)
- print(peaks.shape)
-
+ logger.debug("peaks: %s (shape: %s)", peaks, peaks.shape)
# Insert extra iterations as a new "iteration" start to match behavior iterations
new_synced_iteration_vector = synced_iteration_vector.copy()
for i in range(peaks.shape[0]):
- value_insert = (synced_iteration_vector[peaks[i]] + synced_iteration_vector[peaks[i]+1] ) /2
+ value_insert = (synced_iteration_vector[peaks[i]] + synced_iteration_vector[peaks[i] + 1]) / 2
new_synced_iteration_vector = np.insert(new_synced_iteration_vector, peaks[i], value_insert)
-
if new_synced_iteration_vector.shape[0] != behavior_time_vector.shape[0]:
- print('with peak strategy, could not find correct missing iterations')
+ logger.warning("with peak strategy, could not find correct missing iterations")
status = -1
return status, np.empty(0)
return status, new_synced_iteration_vector
-
# Deprecated
-def behavior_sync_frame_counter_method(digital_array, behavior_time_vector, session_trial_keys, nidq_sampling_rate, bit_start, number_bits):
-
- max_count = np.power(2, number_bits)-1
-
- # 2: transform the digital lines into a number, save in an array of integers
+def behavior_sync_frame_counter_method(
+ digital_array,
+ behavior_time_vector,
+ session_trial_keys,
+ nidq_sampling_rate,
+ bit_start,
+ number_bits,
+):
+
+ max_count = np.power(2, number_bits) - 1
+
+ # 2: transform the digital lines into a number, save in an array of integers
# ... and also get start and end time
framenumber = np.zeros(digital_array.shape[1])
for i in range(digital_array.shape[1]):
- a = BitArray(np.flip(digital_array[bit_start:, i])) # ignore 0-bit, as this is the NPX sync puls, and not virmen.
+ a = BitArray(
+ np.flip(digital_array[bit_start:, i])
+ ) # ignore 0-bit, as this is the NPX sync puls, and not virmen.
framenumber[i] = a.uint
- iterations_raw = np.array(framenumber, dtype=np.int) # Transform frames into integer
- recording_start = np.min(np.where(iterations_raw>0)) #Get start of recording: first change of testlist
- dt = np.int(0.04*nidq_sampling_rate)
- recording_end = np.where(np.abs(np.diff(iterations_raw))>0)[0][-1] + dt #Get end of recording
+ iterations_raw = np.array(framenumber, dtype=np.int) # Transform frames into integer
+ recording_start = np.min(np.where(iterations_raw > 0)) # Get start of recording: first change of testlist
+ dt = np.int(0.04 * nidq_sampling_rate)
+ recording_end = np.where(np.abs(np.diff(iterations_raw)) > 0)[0][-1] + dt # Get end of recording
# 3: transform `iterations_raw` into `framenumber_in_trial` and `trialnumber`
# iterations_raw is just a number between 0 and max_count+1. Some math has to be done to obtain:
@@ -514,34 +560,42 @@ def behavior_sync_frame_counter_method(digital_array, behavior_time_vector, sess
# trialnumber has the length of the number of samples of the NIDAQ card, and every entry is the current trial.
#
# NOTE: some minor glitches have to be catched, if a NIDAQ sample happenes to be recorded while the VR System updates the iteration number.
- framenumber_in_trial = np.zeros(len(iterations_raw))*np.nan
- trialnumber = np.zeros(len(iterations_raw))*np.nan
+ framenumber_in_trial = np.zeros(len(iterations_raw)) * np.nan
+ trialnumber = np.zeros(len(iterations_raw)) * np.nan
current_trial = 0
- overflow = 0 # This variable keep track whenever the reset from max_count to 0 happens.
+ overflow = 0 # This variable keep track whenever the reset from max_count to 0 happens.
for idx, frame_number in enumerate(iterations_raw):
- if (idx>recording_start) & (idx recording_start) & (idx < recording_end):
+ # print(iterations_raw2[idx], frame_number)
+ if (frame_number == 0) & (iterations_raw[idx - 1] == max_count): # At the reset, add max_count+1
overflow = overflow + 1
- if (frame_number==0) & (iterations_raw[idx-1]!=max_count) & (iterations_raw[idx-1]!=0) & (iterations_raw[idx-2]==max_count): # Unlucky reset if happened to be sampled at the wrong time
+ if (
+ (frame_number == 0)
+ & (iterations_raw[idx - 1] != max_count)
+ & (iterations_raw[idx - 1] != 0)
+ & (iterations_raw[idx - 2] == max_count)
+ ): # Unlucky reset if happened to be sampled at the wrong time
overflow = overflow + 1
- framenumber_in_trial[idx-1] = frame_number + overflow*(max_count+1) - 1 # In case this happened, the previous sample has to be corrected
+ framenumber_in_trial[idx - 1] = (
+ frame_number + overflow * (max_count + 1) - 1
+ ) # In case this happened, the previous sample has to be corrected
# Keep track of trial number
- endflag = framenumber_in_trial[idx-1] == (len(behavior_time_vector[current_trial])) # Trial end has been reached.
-
- transitionflag = frame_number == 2 # Next trial should start at zero again (it starts with two ??)
- if endflag & transitionflag: # Only at the transitions
+ endflag = framenumber_in_trial[idx - 1] == (
+ len(behavior_time_vector[current_trial])
+ ) # Trial end has been reached.
+ transitionflag = frame_number == 2 # Next trial should start at zero again (it starts with two ??)
+ if endflag & transitionflag: # Only at the transitions
current_trial = current_trial + 1 # Increases trial count
- overflow = 0 # Reset the 7 bit counter for the next trial
+ overflow = 0 # Reset the 7 bit counter for the next trial
if overflow == 0:
framenumber_in_trial[idx] = frame_number
else:
- framenumber_in_trial[idx] = frame_number + overflow*(max_count+1) -1
+ framenumber_in_trial[idx] = frame_number + overflow * (max_count + 1) - 1
trialnumber[idx] = current_trial
- trial_list = np.array(np.unique(trialnumber[np.isfinite(trialnumber)]), dtype = np.int64)
+ trial_list = np.array(np.unique(trialnumber[np.isfinite(trialnumber)]), dtype=np.int64)
# 4: find and remove additional NIDAQ glitches of two types:
# a) single samples where the iteration number is corrupted because sampling happened faster than output of the behevior PC.
@@ -550,45 +604,50 @@ def behavior_sync_frame_counter_method(digital_array, behavior_time_vector, sess
din = np.diff(framenumber_in_trial)
trial_transitions = np.where(np.diff(trialnumber))
glitches = []
- for candidate in np.where( np.logical_or(din>1, din<0) )[0]: # skipped frames or counting down
+ for candidate in np.where(np.logical_or(din > 1, din < 0))[0]: # skipped frames or counting down
if np.sum(candidate == trial_transitions) == 0:
glitches = np.append(glitches, candidate)
- glitches = np.array(glitches, dtype = np.int)
+ glitches = np.array(glitches, dtype=np.int)
# Attempt to remove them
skipped_frames = 0
for g in glitches:
- if framenumber_in_trial[g] < framenumber_in_trial[g+2]:
- if framenumber_in_trial[g+2] - framenumber_in_trial[g] == 2: # skipped frame, should be very rare
+ if framenumber_in_trial[g] < framenumber_in_trial[g + 2]:
+ if framenumber_in_trial[g + 2] - framenumber_in_trial[g] == 2: # skipped frame, should be very rare
pass
- framenumber_in_trial[g+1] = framenumber_in_trial[g]+1
+ framenumber_in_trial[g + 1] = framenumber_in_trial[g] + 1
skipped_frames = skipped_frames + 1
- else: # If random number, nidaq sample in the middle of update.
- framenumber_in_trial[g+1] = framenumber_in_trial[g]
+ else: # If random number, nidaq sample in the middle of update.
+ framenumber_in_trial[g + 1] = framenumber_in_trial[g]
# A set of final asserts, making sure that the code worked as intended
- assert len(trial_list) == len(session_trial_keys) # Make sure the trial number is correct.
- assert np.sum(np.diff(framenumber_in_trial)>1) == 0 # No frames should be skipped
- assert np.sum(np.diff(framenumber_in_trial)<0) 1) == 0 # No frames should be skipped
+ assert np.sum(np.diff(framenumber_in_trial) < 0) < len(trial_list) # Negative iterations only at trial transitions
iterations_test = 0
for t in trial_list:
- iterations_test = iterations_test + framenumber_in_trial[trialnumber==t][-1] # Integrate number of iterations
- assert framenumber_in_trial[trialnumber==t][-1] == len(behavior_time_vector[t]) # Make sure number of nidaq-frames in each trial is identical to dj record:
- nidaqtime = np.sum(trialnumber == t)/nidq_sampling_rate
+ iterations_test = iterations_test + framenumber_in_trial[trialnumber == t][-1] # Integrate number of iterations
+ assert framenumber_in_trial[trialnumber == t][-1] == len(
+ behavior_time_vector[t]
+ ) # Make sure number of nidaq-frames in each trial is identical to dj record:
+ nidaqtime = np.sum(trialnumber == t) / nidq_sampling_rate
matlabtime = np.max(behavior_time_vector[t])
- assert ((nidaqtime - matlabtime) / matlabtime) < 0.1 # # Make sure the nidaq-trial-duration and dj records are consistent; 10% arbitrarily chosen
- nidaq_duration = iterations_test + skipped_frames
- #dj_duration = iterstart[-1] + len(behavior_time_vector[-1])
- #assert np.abs(nidaq_duration - dj_duration) < 3 # at most two frames off - sometimes this happens at the beginning/end of the recording
+ assert (
+ (nidaqtime - matlabtime) / matlabtime
+ ) < 0.1 # # Make sure the nidaq-trial-duration and dj records are consistent; 10% arbitrarily chosen
+ iterations_test + skipped_frames
+ # dj_duration = iterstart[-1] + len(behavior_time_vector[-1])
+ # assert np.abs(nidaq_duration - dj_duration) < 3 # at most two frames off - sometimes this happens at the beginning/end of the recording
# If this is done, and the asserts are passed, insert the data into the database
return (framenumber_in_trial, trialnumber)
-
def future_counter_get_signal():
pass
-'''
+
+
+"""
# Cleaner way to get iteration number from counter, still need debugging, if necessary
framenumber = np.zeros(idx_end-idx_start)
@@ -613,23 +672,25 @@ def future_counter_get_signal():
framenumber_in_trial[idx] = frame_number + overflow*(max_count+1)
- print(np.max(framenumber_in_trial))
-'''
+ logger.debug("max framenumber_in_trial: %s", np.max(framenumber_in_trial))
+"""
+
def load_open_ephys_digital_file(file_path):
pass
-def get_iteration_intertrial_from_virmen_time(trial_pulse_signal, nidq_sampling_rate, num_behavior_trials, behavior_time_vector):
+def get_iteration_intertrial_from_virmen_time(
+ trial_pulse_signal, nidq_sampling_rate, num_behavior_trials, behavior_time_vector
+):
- #Get idx samples trial starts
- trial_start_idx,_ = get_idx_trial_start(trial_pulse_signal)
+ # Get idx samples trial starts
+ trial_start_idx, _ = get_idx_trial_start(trial_pulse_signal)
iter_start_idx = []
for i in range(num_behavior_trials):
-
- new_synced_iteration_vector = trial_start_idx[i]+np.int64(behavior_time_vector[i]*nidq_sampling_rate)
+ new_synced_iteration_vector = trial_start_idx[i] + np.int64(behavior_time_vector[i] * nidq_sampling_rate)
iter_start_idx.append(new_synced_iteration_vector.squeeze())
iter_start_idx = np.asarray(iter_start_idx.copy(), dtype=object)
@@ -639,62 +700,67 @@ def get_iteration_intertrial_from_virmen_time(trial_pulse_signal, nidq_sampling_
def get_full_vector_samples(iter_start_idx_vectors, nidq_sampling_rate, total_samples):
- framenumber_vector_samples = np.zeros(total_samples)*np.nan
- trialnumber_vector_samples = np.zeros(total_samples)*np.nan
+ framenumber_vector_samples = np.zeros(total_samples) * np.nan
+ trialnumber_vector_samples = np.zeros(total_samples) * np.nan
for i in range(len(iter_start_idx_vectors)):
-
this_trial_iter_vector = iter_start_idx_vectors[i]
- if i < len(iter_start_idx_vectors)-1:
- next_trial_iter_vector = iter_start_idx_vectors[i+1]
-
- #Fill vector samples
- for j in range(this_trial_iter_vector.shape[0]-1):
- framenumber_vector_samples[this_trial_iter_vector[j]:this_trial_iter_vector[j+1]] = j+1
-
- #Last iteration # is from start of iteration to end of trial
- if i < len(iter_start_idx_vectors)-1:
- trialnumber_vector_samples[this_trial_iter_vector[0]:next_trial_iter_vector[0]] = i+1
- framenumber_vector_samples[this_trial_iter_vector[-1]:next_trial_iter_vector[0]] = this_trial_iter_vector.shape[0]
+ if i < len(iter_start_idx_vectors) - 1:
+ next_trial_iter_vector = iter_start_idx_vectors[i + 1]
+
+ # Fill vector samples
+ for j in range(this_trial_iter_vector.shape[0] - 1):
+ framenumber_vector_samples[this_trial_iter_vector[j] : this_trial_iter_vector[j + 1]] = j + 1
+
+ # Last iteration # is from start of iteration to end of trial
+ if i < len(iter_start_idx_vectors) - 1:
+ trialnumber_vector_samples[this_trial_iter_vector[0] : next_trial_iter_vector[0]] = i + 1
+ framenumber_vector_samples[this_trial_iter_vector[-1] : next_trial_iter_vector[0]] = (
+ this_trial_iter_vector.shape[0]
+ )
# For last trial, lets finish it 1s after last iteration detected
else:
- #print('total_samples', total_samples)
- #print('this_trial_iter_vector[-1]', this_trial_iter_vector[-1])
- #print('this_trial_iter_vector[0]', this_trial_iter_vector[0])
- #print('this_trial_iter_vector[-1]+int(nidq_sampling_rate)', this_trial_iter_vector[-1]+int(nidq_sampling_rate))
- trialnumber_vector_samples[this_trial_iter_vector[0]:this_trial_iter_vector[-1]+int(nidq_sampling_rate)] = i+1
- framenumber_vector_samples[this_trial_iter_vector[-1]:this_trial_iter_vector[-1]+int(nidq_sampling_rate)] = this_trial_iter_vector.shape[0]
+ # print('total_samples', total_samples)
+ # print('this_trial_iter_vector[-1]', this_trial_iter_vector[-1])
+ # print('this_trial_iter_vector[0]', this_trial_iter_vector[0])
+ # print('this_trial_iter_vector[-1]+int(nidq_sampling_rate)', this_trial_iter_vector[-1]+int(nidq_sampling_rate))
+ trialnumber_vector_samples[
+ this_trial_iter_vector[0] : this_trial_iter_vector[-1] + int(nidq_sampling_rate)
+ ] = i + 1
+ framenumber_vector_samples[
+ this_trial_iter_vector[-1] : this_trial_iter_vector[-1] + int(nidq_sampling_rate)
+ ] = this_trial_iter_vector.shape[0]
return trialnumber_vector_samples, framenumber_vector_samples
+
def get_time_vector_as_behavior(trial_idx_vector, nidq_sampling_rate):
first_iter_session = trial_idx_vector[0][0]
- time_0 = first_iter_session/nidq_sampling_rate
+ time_0 = first_iter_session / nidq_sampling_rate
trial_times_ind = []
trial_times_full = []
for i in range(trial_idx_vector.shape[0]):
-
- time_vector = trial_idx_vector[i]/nidq_sampling_rate
+ time_vector = trial_idx_vector[i] / nidq_sampling_rate
time_vector_ind = time_vector - time_vector[0]
time_vector_full = time_vector - time_0
trial_times_ind.append(time_vector_ind)
trial_times_full.append(time_vector_full)
- trial_times_ind= np.asarray(trial_times_ind.copy(), dtype=object)
+ trial_times_ind = np.asarray(trial_times_ind.copy(), dtype=object)
trial_times_full = np.asarray(trial_times_full.copy(), dtype=object)
-
return trial_times_ind, trial_times_full
+
def get_time_vector(trial_index_nidq, nidq_sampling_rate):
time_vector = np.arange(0, trial_index_nidq.shape[0])
time_vector = time_vector.astype(np.float64)
- time_vector = time_vector/nidq_sampling_rate
+ time_vector = time_vector / nidq_sampling_rate
session_start_index = np.where(trial_index_nidq == 1)[0][0]
time_vector -= time_vector[session_start_index]
@@ -702,7 +768,6 @@ def get_time_vector(trial_index_nidq, nidq_sampling_rate):
return time_vector
-
def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling_rate):
status = False
@@ -715,14 +780,13 @@ def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling
trial_index_nidq[:first_non_nan] = 0
iteration_index_nidq[:first_non_nan] = -1
- idx_trial = np.where((np.diff(trial_index_nidq)) == 1 )[0] + 1
+ idx_trial = np.where((np.diff(trial_index_nidq)) == 1)[0] + 1
idx_iteration = np.where((np.diff(iteration_index_nidq)) == 1)[0] + 1
idx_iteration_final = []
for i in range(idx_trial.shape[0]):
-
- if i < idx_trial.shape[0]-1:
- iterations_this_trial = idx_iteration[(idx_iteration > idx_trial[i]) & (idx_iteration idx_trial[i]) & (idx_iteration < idx_trial[i + 1])]
else:
iterations_this_trial = idx_iteration[(idx_iteration > idx_trial[i])]
idx_this_trial_iterations = np.insert(iterations_this_trial, 0, idx_trial[i])
@@ -732,17 +796,24 @@ def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling
trial_index_nidq[:first_non_nan] = np.nan
iteration_index_nidq[:first_non_nan] = np.nan
- trial_index_nidq2, iteration_index_nidq2 = get_full_vector_samples(idx_iteration_final, nidq_sampling_rate, trial_index_nidq.shape[0])
-
+ trial_index_nidq2, iteration_index_nidq2 = get_full_vector_samples(
+ idx_iteration_final, nidq_sampling_rate, trial_index_nidq.shape[0]
+ )
iteration_equal = np.allclose(iteration_index_nidq, iteration_index_nidq2, equal_nan=True)
trial_equal = np.allclose(trial_index_nidq, trial_index_nidq2, equal_nan=True)
- print('iteration_equal', iteration_equal)
- print('trial_equal', trial_equal)
+ logger.debug("iteration_equal: %s", iteration_equal)
+ logger.debug("trial_equal: %s", trial_equal)
- print('np.where(np.isnan(iteration_index_nidq))[0].shape', np.where(np.isnan(iteration_index_nidq))[0].shape)
- print('np.where(np.isnan(iteration_index_nidq2))[0].shape', np.where(np.isnan(iteration_index_nidq2))[0].shape)
+ logger.debug(
+ "np.where(np.isnan(iteration_index_nidq))[0].shape: %s",
+ np.where(np.isnan(iteration_index_nidq))[0].shape,
+ )
+ logger.debug(
+ "np.where(np.isnan(iteration_index_nidq2))[0].shape: %s",
+ np.where(np.isnan(iteration_index_nidq2))[0].shape,
+ )
if trial_equal and iteration_equal:
status = True
@@ -750,22 +821,20 @@ def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling
return status, idx_trial, idx_iteration_final
-
def get_index_trial_vector_from_iteration(iteration_start_idx):
trial_start_idx = np.zeros(len(iteration_start_idx), dtype=np.int64)
for i in range(len(iteration_start_idx)):
-
trial_start_idx[i] = iteration_start_idx[i][0]
return trial_start_idx
-class xyz_pick_file_creator():
- '''
+class XyzPickFileCreator:
+ """
Class that handles probe coordinates locations given initial isertion coordinates & shank coordinates
- '''
+ """
@staticmethod
def main_xyz_pick_file_function(recording_id, fragment_number, chanmap_file, processed_data_directory):
@@ -779,23 +848,27 @@ def main_xyz_pick_file_function(recording_id, fragment_number, chanmap_file, pro
"""
# Check existance of ibl output path
- ibl_output_dir = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][1], processed_data_directory, 'ibl_data')
+ ibl_output_dir = pathlib.Path(
+ dj.config["custom"]["ephys_root_data_dir"][1],
+ processed_data_directory,
+ "ibl_data",
+ )
if not ibl_output_dir.is_dir():
pathlib.Path.mkdir(ibl_output_dir)
# Get recording id
- probe_location = xyz_pick_file_creator.get_probe_insertion_coordinates(recording_id, fragment_number)
- print(probe_location)
+ probe_location = XyzPickFileCreator.get_probe_insertion_coordinates(recording_id, fragment_number)
+ logger.debug("probe_location: %s", probe_location)
- #Load channelmap and check how many probes there are
+ # Load channelmap and check how many probes there are
chanmap = loadmat(chanmap_file)
- max_shank = int(np.max(chanmap['kcoords']))
+ max_shank = int(np.max(chanmap["kcoords"]))
# Calculate probe coordinates and store files
- all_shanks = list()
+ all_shanks = []
for i in range(max_shank):
- probe_track = xyz_pick_file_creator.get_probetrack(chanmap, shank=i+1, **probe_location)
- xyz_pick_file_creator.save_xyz_pick_file(ibl_output_dir, probe_track, shank=i)
+ probe_track = XyzPickFileCreator.get_probetrack(chanmap, shank=i + 1, **probe_location)
+ XyzPickFileCreator.save_xyz_pick_file(ibl_output_dir, probe_track, shank=i)
all_shanks.append(probe_track)
return all_shanks
@@ -809,39 +882,63 @@ def get_probe_insertion_coordinates(recording_id, probe_num):
probe_num (int) = Reference to probe# of current recording
"""
- coordinates_columns = ['real_ap_coordinates', 'real_depth_coordinates', 'real_ml_coordinates', 'phi_angle', 'theta_angle', 'rho_angle']
+ coordinates_columns = [
+ "real_ap_coordinates",
+ "real_depth_coordinates",
+ "real_ml_coordinates",
+ "phi_angle",
+ "theta_angle",
+ "rho_angle",
+ ]
probes_not_found = False
# Create virtual modules of needed DBs
- action_db = dj.create_virtual_module('action', 'u19_action')
- recording_db = dj.create_virtual_module('recording', 'u19_recording')
+ action_db = dj.create_virtual_module("action", "u19_action")
+ recording_db = dj.create_virtual_module("recording", "u19_recording")
# Query subject of recording
- query = {'recording_id': recording_id}
- subject_recording = (recording_db.Recording.BehaviorSession & query).fetch('subject_fullname')
+ query = {"recording_id": recording_id}
+ subject_recording = (recording_db.Recording.BehaviorSession & query).fetch("subject_fullname")
if subject_recording.shape[0] != 0:
# Query probe insertion table
- query_surgery = {'subject_fullname': subject_recording[0], 'device_idx': probe_num}
+ query_surgery = {
+ "subject_fullname": subject_recording[0],
+ "device_idx": probe_num,
+ }
probe_location = (action_db.SurgeryLocation & query_surgery).fetch(*coordinates_columns, as_dict=True)
if len(probe_location) == 0:
probes_not_found = True
else:
probe_location = probe_location[0]
# Convert to float
- probe_location = {k:float(v)for (k,v) in probe_location.items()}
+ probe_location = {k: float(v) for (k, v) in probe_location.items()}
else:
probes_not_found = True
# If insertion decive is not found, create a dummy one but raise a warning
if probes_not_found:
- warnings.warn("Warning probe location was not found on DB for recording_id: " + str(recording_id) + " & probe# " + str(probe_num) )
+ warnings.warn(
+ "Warning probe location was not found on DB for recording_id: "
+ + str(recording_id)
+ + " & probe# "
+ + str(probe_num), stacklevel=2
+ )
probe_location = dict.fromkeys(coordinates_columns, 0)
return probe_location
@staticmethod
- def get_probetrack(chanmap, shank=1, real_ml_coordinates=0, real_ap_coordinates=0, real_depth_coordinates=0, phi_angle=0, theta_angle=0, rho_angle=0):
+ def get_probetrack(
+ chanmap,
+ shank=1,
+ real_ml_coordinates=0,
+ real_ap_coordinates=0,
+ real_depth_coordinates=0,
+ phi_angle=0,
+ theta_angle=0,
+ rho_angle=0,
+ ):
"""
Build numpy array with "brain" coordinates from insertion device and probe features
Input:
@@ -856,30 +953,40 @@ def get_probetrack(chanmap, shank=1, real_ml_coordinates=0, real_ap_coordinates=
"""
# Step 1: Convert degrees to radiants and reformat chanmap
- phi = phi_angle*np.pi/180
- theta = theta_angle*np.pi/180
- roll = rho_angle*np.pi/180
- x = np.array([i[0] for i in chanmap['xcoords']]);
- y = np.array([i[0] for i in chanmap['ycoords']]);
- k = np.array([i[0] for i in chanmap['kcoords']]);
+ phi = phi_angle * np.pi / 180
+ theta = theta_angle * np.pi / 180
+ roll = rho_angle * np.pi / 180
+ x = np.array([i[0] for i in chanmap["xcoords"]])
+ y = np.array([i[0] for i in chanmap["ycoords"]])
+ k = np.array([i[0] for i in chanmap["kcoords"]])
# Step 2: Transform them into 3D, assuming the Probe is perpendicular to x|y plane
- avx = np.mean(x[k==shank]) # Center "X" on middle of probe
- probe_x0 = np.array([np.cos(roll)*avx, -np.sin(roll)*avx]) # coordinates of the shank after "roll" around probe axis
- probe_length = np.max(y[k==shank]) - np.min(y[k==shank]) # Length off the probe per chanmap
- probe_unitVec = np.array([np.sin(theta)*np.cos(phi), np.sin(theta)*np.sin(phi), np.cos(theta)]) # Unit vector point along insertion direction
-
- probe_length = np.arange(-real_depth_coordinates*1000, -real_depth_coordinates*1000 + probe_length, 10) # Resulution of future xyz_pick.json file
+ avx = np.mean(x[k == shank]) # Center "X" on middle of probe
+ probe_x0 = np.array(
+ [np.cos(roll) * avx, -np.sin(roll) * avx]
+ ) # coordinates of the shank after "roll" around probe axis
+ probe_length = np.max(y[k == shank]) - np.min(y[k == shank]) # Length off the probe per chanmap
+ probe_unitVec = np.array(
+ [np.sin(theta) * np.cos(phi), np.sin(theta) * np.sin(phi), np.cos(theta)]
+ ) # Unit vector point along insertion direction
+
+ probe_length = np.arange(
+ -real_depth_coordinates * 1000,
+ -real_depth_coordinates * 1000 + probe_length,
+ 10,
+ ) # Resulution of future xyz_pick.json file
# Step 3: Produce the 3D coordinates along the probe track
- probe_track = np.zeros((len(probe_length),3))
+ probe_track = np.zeros((len(probe_length), 3))
for i in range(len(probe_length)):
- probe_track[i,:] = probe_length[i]*probe_unitVec + np.array([probe_x0[0], probe_x0[1], 0])
+ probe_track[i, :] = probe_length[i] * probe_unitVec + np.array([probe_x0[0], probe_x0[1], 0])
# Step 4: Shift probe my ML|AP insertion coordinates
- probe_track_shifted = np.zeros((probe_track.shape))
+ probe_track_shifted = np.zeros(probe_track.shape)
for i in range(len(probe_track_shifted)):
- probe_track_shifted[i,:] = probe_track[i,:] + np.array([real_ml_coordinates*1000, real_ap_coordinates*1000, 0])
+ probe_track_shifted[i, :] = probe_track[i, :] + np.array(
+ [real_ml_coordinates * 1000, real_ap_coordinates * 1000, 0]
+ )
return probe_track_shifted.tolist()
@@ -892,11 +999,16 @@ def save_xyz_pick_file(save_directory, probe_coord_data, shank=0):
shank (int) = Shank # (0 index based) (different filename depending on it)
"""
- filenames = ['xyz_picks.json', 'xyz_picks_shank1.json', 'xyz_picks_shank2.json', 'xyz_picks_shank3.json']
+ filenames = [
+ "xyz_picks.json",
+ "xyz_picks_shank1.json",
+ "xyz_picks_shank2.json",
+ "xyz_picks_shank3.json",
+ ]
final_filename = pathlib.Path(save_directory, filenames[shank]).as_posix()
- dict_coord = dict()
+ dict_coord = {}
dict_coord["xyz_picks"] = probe_coord_data
- with open(final_filename, 'w') as fp:
- json.dump(dict_coord, fp)
\ No newline at end of file
+ with open(final_filename, "w") as fp:
+ json.dump(dict_coord, fp)
diff --git a/u19_pipeline/utils/file_utils.py b/u19_pipeline/utils/file_utils.py
index 13b0e514..11added6 100644
--- a/u19_pipeline/utils/file_utils.py
+++ b/u19_pipeline/utils/file_utils.py
@@ -1,17 +1,18 @@
import os
+
def write_file(path, text):
os.umask(0)
descriptor = os.open(
- path=path,
- flags=(
- os.O_WRONLY # access mode: write only
- | os.O_CREAT # create if not exists
- | os.O_TRUNC # truncate the file to zero
- ),
- mode=0o664
+ path=path,
+ flags=(
+ os.O_WRONLY # access mode: write only
+ | os.O_CREAT # create if not exists
+ | os.O_TRUNC # truncate the file to zero
+ ),
+ mode=0o664,
)
- with open(descriptor, 'w') as fh:
- fh.write(text)
\ No newline at end of file
+ with open(descriptor, "w") as fh:
+ fh.write(text)
diff --git a/u19_pipeline/utils/insert_miscelaneous_db.py b/u19_pipeline/utils/insert_miscelaneous_db.py
index a1378906..2f550cae 100644
--- a/u19_pipeline/utils/insert_miscelaneous_db.py
+++ b/u19_pipeline/utils/insert_miscelaneous_db.py
@@ -1,34 +1,36 @@
-
import os
import pathlib
-import numpy as np
+
import datajoint as dj
+import numpy as np
def add_researcher_user_table(user_id, full_name, email, phone):
# Values sent by function caller
- new_key = dict()
- new_key['user_id'] = user_id
- new_key['user_nickname'] = user_id
- new_key['full_name'] = full_name
- new_key['email'] = email
- new_key['phone'] = phone
+ new_key = {}
+ new_key["user_id"] = user_id
+ new_key["user_nickname"] = user_id
+ new_key["full_name"] = full_name
+ new_key["email"] = email
+ new_key["phone"] = phone
# Write default values for User
- new_key['mobile_carrier'] = 'none'
- new_key['slack'] = full_name
- new_key['contact_via'] = 'Slack'
- new_key['primary_tech'] = 'N/A'
- new_key['tech_responsibility']='yes'
- new_key['day_cutoff_time']= np.array([18, 0])
+ new_key["mobile_carrier"] = "none"
+ new_key["slack"] = full_name
+ new_key["contact_via"] = "Slack"
+ new_key["primary_tech"] = "N/A"
+ new_key["tech_responsibility"] = "yes"
+ new_key["day_cutoff_time"] = np.array([18, 0])
# Find conf file (to load it even if we are in different path)
- repository_dir = os.path.abspath(os.path.realpath(__file__)+ "/../../..")
- conf_file = pathlib.Path(repository_dir, 'dj_local_conf.json')
+ repository_dir = os.path.abspath(os.path.realpath(__file__) + "/../../..")
+ conf_file = pathlib.Path(repository_dir, "dj_local_conf.json")
if not pathlib.Path.exists(conf_file):
- raise Exception("No configuration file found, configure repository first: https://braincogs.github.io/software/db_access.html#db-access-for-python-repository")
+ raise Exception(
+ "No configuration file found, configure repository first: https://braincogs.github.io/software/db_access.html#db-access-for-python-repository"
+ )
config = dj.settings.Config()
config.load(str(conf_file))
@@ -36,4 +38,5 @@ def add_researcher_user_table(user_id, full_name, email, phone):
# Connect and insert record
dj.conn()
import u19_pipeline.lab as lab
+
lab.User.insert1(new_key)
diff --git a/u19_pipeline/utils/logging_config.py b/u19_pipeline/utils/logging_config.py
new file mode 100644
index 00000000..f0acf2a6
--- /dev/null
+++ b/u19_pipeline/utils/logging_config.py
@@ -0,0 +1,46 @@
+"""Logging helpers for u19_pipeline.
+
+Library usage (u19_pipeline/*):
+ Call get_logger(__name__) at module level. No handlers are attached here;
+ the caller (application or script) is responsible for configuring them.
+
+Script usage (scripts/*):
+ Call setup_logging() once at the top of the __main__ block. This installs
+ a Rich handler on the root logger so all u19_pipeline loggers emit
+ formatted output without disturbing any downstream caller that configures
+ their own handlers.
+"""
+
+import logging
+
+# Silence the library's own loggers by default (PEP 396 / logging HOWTO).
+# Downstream code or scripts call setup_logging() to attach real handlers.
+logging.getLogger("u19_pipeline").addHandler(logging.NullHandler())
+
+
+def get_logger(name: str) -> logging.Logger:
+ """Return a named logger. Never configures handlers — safe for library use."""
+ return logging.getLogger(name)
+
+
+def setup_logging(level: int = logging.INFO) -> None:
+ """Configure the root logger with a Rich handler.
+
+ Call this exactly once at the entry point of a script or application.
+ Safe to call multiple times (idempotent via basicConfig semantics).
+ """
+ from rich.console import Console
+ from rich.logging import RichHandler
+
+ logging.basicConfig(
+ level=level,
+ format="%(message)s",
+ datefmt="[%X]",
+ handlers=[
+ RichHandler(
+ console=Console(stderr=True),
+ rich_tracebacks=True,
+ show_path=True,
+ )
+ ],
+ )
diff --git a/u19_pipeline/utils/matlab_utils.py b/u19_pipeline/utils/matlab_utils.py
index 00aa28c5..9df8db86 100644
--- a/u19_pipeline/utils/matlab_utils.py
+++ b/u19_pipeline/utils/matlab_utils.py
@@ -1,24 +1,22 @@
"""Authors: Ben Dichter, Cody Baker."""
+
import os
import sys
+from collections.abc import Iterable, Sequence
+from datetime import datetime
from pathlib import Path
from shutil import which
-import pandas as pd
import numpy as np
-from datetime import datetime
-from scipy.io import loadmat, matlab
-from collections.abc import Iterable
+from u19_pipeline.utils.logging_config import get_logger
-try:
- from typing import ArrayLike
-except ImportError:
- from numpy import ndarray
- from typing import Union, Sequence
+logger = get_logger(__name__)
+import pandas as pd
+from numpy import ndarray
+from scipy.io import loadmat, matlab
- # adapted from numpy typing
- ArrayLike = Union[bool, int, float, complex, list, ndarray, Sequence]
+ArrayLike = bool | int | float | complex | list | ndarray | Sequence
def check_module(nwbfile, name, description=None):
@@ -48,7 +46,7 @@ def find_discontinuities(tt, factor=10000):
if len(before_jumps):
out = np.array([tt[0], tt[before_jumps[0]]])
- for i, j in zip(before_jumps, before_jumps[1:]):
+ for i, j in zip(before_jumps, before_jumps[1:], strict=False):
out = np.vstack((out, [tt[i + 1], tt[j]]))
out = np.vstack((out, [tt[before_jumps[-1] + 1], tt[-1]]))
return out
@@ -59,7 +57,7 @@ def find_discontinuities(tt, factor=10000):
def mat_obj_to_dict(mat_struct):
"""Recursive function to convert nested matlab struct objects to dictionaries."""
dict_from_struct = {}
- for field_name in mat_struct.__dict__['_fieldnames']:
+ for field_name in mat_struct.__dict__["_fieldnames"]:
dict_from_struct[field_name] = mat_struct.__dict__[field_name]
if isinstance(dict_from_struct[field_name], matlab.mio5_params.mat_struct):
dict_from_struct[field_name] = mat_obj_to_dict(dict_from_struct[field_name])
@@ -85,8 +83,7 @@ def mat_obj_to_array(mat_struct_array):
def has_struct(mat_struct_array):
"""Determines if a matlab cell array contains any mat objects."""
- return any(
- isinstance(mat_struct, matlab.mio5_params.mat_struct) for mat_struct in mat_struct_array)
+ return any(isinstance(mat_struct, matlab.mio5_params.mat_struct) for mat_struct in mat_struct_array)
def convert_mat_file_to_dict(mat_file_name):
@@ -133,7 +130,7 @@ def flatten_nested_dict(nested_dict):
if isinstance(v, dict):
if v:
flatten_sub_dict = flatten_nested_dict(v).items()
- flatten_dict.update({k2: v2 for k2, v2 in flatten_sub_dict})
+ flatten_dict.update(dict(flatten_sub_dict))
else:
flatten_dict[k] = np.array([])
else:
@@ -144,8 +141,8 @@ def flatten_nested_dict(nested_dict):
def convert_function_handle_to_str(mat_file_path):
"""Executes a matlab script which converts function handle values to str
- if matlab is installed on the system."""
- matlab_class = '''
+ if matlab is installed on the system."""
+ matlab_class = """
classdef Choice < uint32
enumeration
@@ -179,8 +176,8 @@ def convert_function_handle_to_str(mat_file_path):
end
end
- '''
- matlab_code = r'''
+ """
+ matlab_code = r"""
%Support when behavioral files has 2 "versions"
if length(log.version) > 1
@@ -230,49 +227,49 @@ def convert_function_handle_to_str(mat_file_path):
fclose(fid);
quit;
- '''
+ """
- with Path('Choice.m').open('w') as f:
+ with Path("Choice.m").open("w") as f:
f.write(matlab_class)
metadata = {}
convert_script_code = f"filePath = '{mat_file_path}';\nload(filePath);{matlab_code}"
convert_script_path = Path("convert_function_to_txt.m")
- with convert_script_path.open('w') as f:
+ with convert_script_path.open("w") as f:
f.write(convert_script_code)
- if 'win' in sys.platform and sys.platform != 'darwin':
- matlab_cmd = '''
+ if "win" in sys.platform and sys.platform != "darwin":
+ matlab_cmd = """
#!/bin/bash
matlab -nosplash -wait -log -r convert_function_to_txt
- '''
+ """
else:
- matlab_cmd = '''
+ matlab_cmd = """
#!/bin/bash
matlab -nosplash -nodisplay -log -r convert_function_to_txt
- '''
+ """
- if which('matlab') is not None:
+ if which("matlab") is not None:
try:
os.system(matlab_cmd)
-
- with open("trial_choice.txt", "r") as f:
+
+ with open("trial_choice.txt") as f:
trial_choice = f.read().splitlines()
- with open("trial_type.txt", "r") as f:
+ with open("trial_type.txt") as f:
trial_type = f.read().splitlines()
- with open("shaping_protocol.txt", "r") as f:
+ with open("shaping_protocol.txt") as f:
shaping_protocol = f.read().splitlines()
- with open("code_version.txt", "r") as f:
+ with open("code_version.txt") as f:
version = f.readline()
- with open("protocol.txt", "r") as f:
+ with open("protocol.txt") as f:
protocol = f.readline()
- metadata['experiment_name'] = version
- metadata['protocol_name'] = protocol
- metadata['trial_choice'] = trial_choice
- metadata['trial_type'] = trial_type
- metadata['shaping_protocol'] = shaping_protocol
+ metadata["experiment_name"] = version
+ metadata["protocol_name"] = protocol
+ metadata["trial_choice"] = trial_choice
+ metadata["trial_type"] = trial_type
+ metadata["shaping_protocol"] = shaping_protocol
os.remove("code_version.txt")
os.remove("protocol.txt")
@@ -281,16 +278,19 @@ def convert_function_handle_to_str(mat_file_path):
os.remove("shaping_protocol.txt")
except Exception as e:
- print(f"There was an error while trying to execute {convert_script_path}:\n{e}")
+ logger.error("There was an error while trying to execute %s: %s", convert_script_path, e)
else:
- print("A working matlab version was not found. "
- "Code version, animal protocol, type of trial, and choice could not be saved to NWB.")
+ logger.warning(
+ "A working matlab version was not found. "
+ "Code version, animal protocol, type of trial, and choice could not be saved to NWB."
+ )
os.remove("Choice.m")
os.remove("convert_function_to_txt.m")
return metadata
+
def convert_towers_block_trial_2_df(current_block_trial, block_num):
"""
Convert block trial data from matlab file to a Pandas DataFrame
@@ -302,26 +302,27 @@ def convert_towers_block_trial_2_df(current_block_trial, block_num):
-------
pandas.DataFrame
"""
-
+
valid_block = 0
# "Normal" blocks are stored as numpy arrays and its length is greater than 0
if isinstance(current_block_trial, np.ndarray) and current_block_trial.shape[0] > 0:
current_block_trial = current_block_trial.tolist()
- valid_block = 1
+ valid_block = 1
# One trial blocks are stored as dictionaries
if isinstance(current_block_trial, dict):
current_block_trial = [current_block_trial]
- valid_block = 1
+ valid_block = 1
if valid_block:
block_trial_df = pd.DataFrame(current_block_trial)
- block_trial_df.insert(loc=0, column='trial_idx', value=np.arange(len(block_trial_df))+1)
- block_trial_df.insert(loc=0, column='block', value=block_num)
+ block_trial_df.insert(loc=0, column="trial_idx", value=np.arange(len(block_trial_df)) + 1)
+ block_trial_df.insert(loc=0, column="block", value=block_num)
else:
block_trial_df = pd.DataFrame()
return valid_block, block_trial_df
+
def convert_towers_block_2_df(current_block, num_block):
"""
Convert block data from matlab file to a Pandas DataFrame
@@ -336,18 +337,18 @@ def convert_towers_block_2_df(current_block, num_block):
valid_block = 0
# "Normal" blocks are stored as numpy arrays and its length is greater than 0
- if isinstance(current_block, np.ndarray) and current_block_trial.shape[0] > 0:
+ if isinstance(current_block, np.ndarray) and current_block.shape[0] > 0:
current_block = current_block.tolist()
- valid_block = 1
+ valid_block = 1
# One trial blocks are stored as dictionaries
if isinstance(current_block, dict):
current_block = [current_block]
- valid_block = 1
+ valid_block = 1
if valid_block:
block_df = pd.DataFrame(current_block)
- block_df.insert(loc=0, column='block', value=num_block)
- block_df = block_df.drop(['trial'], axis=1)
+ block_df.insert(loc=0, column="block", value=num_block)
+ block_df = block_df.drop(["trial"], axis=1)
else:
block_df = pd.DataFrame()
@@ -367,53 +368,48 @@ def convert_behavior_file(mat_file):
matin = convert_mat_file_to_dict(mat_file)
converted_metadata = convert_function_handle_to_str(mat_file_path=mat_file)
- if bool(converted_metadata):
- metadata_read = True
- else:
- metadata_read = False
+ metadata_read = bool(bool(converted_metadata))
session_block_trial_df = pd.DataFrame()
- #Convert all blocks trials to dataframe and append them
+ # Convert all blocks trials to dataframe and append them
- #For a single block sessions
- if isinstance(matin['log']['block'], dict):
+ # For a single block sessions
+ if isinstance(matin["log"]["block"], dict):
length_blocks = 1
dict_block = 1
- #For multiple block sessions
+ # For multiple block sessions
else:
- length_blocks = matin['log']['block'].shape[0]
+ length_blocks = matin["log"]["block"].shape[0]
dict_block = 0
- #Convert all blocks of the session
+ # Convert all blocks of the session
num_blocks_conv = 0
for i in range(length_blocks):
+ block = matin["log"]["block"] if dict_block else matin["log"]["block"][i]
- if dict_block:
- block = matin['log']['block']
- else:
- block = matin['log']['block'][i]
-
- #Convert trial df and block df
- valid_block, block_trial_df = convert_towers_block_trial_2_df(block['trial'],i+1)
- valid_blocks, block_df = convert_towers_block_2_df(block, i+1)
- #Write string of block level Protocol (from matlab obscured data)
+ # Convert trial df and block df
+ valid_block, block_trial_df = convert_towers_block_trial_2_df(block["trial"], i + 1)
+ valid_blocks, block_df = convert_towers_block_2_df(block, i + 1)
+ # Write string of block level Protocol (from matlab obscured data)
if metadata_read:
- block_df['shapingProtocol'] = converted_metadata['shaping_protocol'][i]
+ block_df["shapingProtocol"] = converted_metadata["shaping_protocol"][i]
if valid_block and valid_blocks:
- session_current_block_trial_df = block_trial_df.merge(block_df, on='block', suffixes=['_block', '_trial'])
+ session_current_block_trial_df = block_trial_df.merge(block_df, on="block", suffixes=["_block", "_trial"])
if num_blocks_conv == 0:
session_block_trial_df = session_current_block_trial_df.copy()
else:
- session_block_trial_df = pd.concat([session_block_trial_df, session_current_block_trial_df],ignore_index=True)
- num_blocks_conv +=1
+ session_block_trial_df = pd.concat(
+ [session_block_trial_df, session_current_block_trial_df],
+ ignore_index=True,
+ )
+ num_blocks_conv += 1
-
- #Write choice and trial type of each trial (from matlab obscured data)
+ # Write choice and trial type of each trial (from matlab obscured data)
if metadata_read:
- session_block_trial_df['choice'] = converted_metadata['trial_choice']
- session_block_trial_df['trialType'] = converted_metadata['trial_type']
-
+ session_block_trial_df["choice"] = converted_metadata["trial_choice"]
+ session_block_trial_df["trialType"] = converted_metadata["trial_type"]
+
session_block_trial_df = session_block_trial_df.reset_index(drop=True)
return session_block_trial_df
diff --git a/u19_pipeline/utils/path_utils.py b/u19_pipeline/utils/path_utils.py
index 93fc391e..43f28fde 100644
--- a/u19_pipeline/utils/path_utils.py
+++ b/u19_pipeline/utils/path_utils.py
@@ -1,28 +1,25 @@
-
-import pathlib
-import os
import glob
+import os
import subprocess
-import sys
file_patterns_acq = {
- "raw_imaging": ['/*.tiff', '/*.tif'],
- "commpressed_imaging": ['/*.tiff.gz', '/*.tif.gz'],
- "segmented_imaging_files": ['/*.modeling.mat', '/*suite2p/'],
- "raw_np_files": ['/*ap.bin', '/*ap.meta'],
- "sorted_np_files": ['/*.npy']
+ "raw_imaging": ["/*.tiff", "/*.tif"],
+ "commpressed_imaging": ["/*.tiff.gz", "/*.tif.gz"],
+ "segmented_imaging_files": ["/*.modeling.mat", "/*suite2p/"],
+ "raw_np_files": ["/*ap.bin", "/*ap.meta"],
+ "sorted_np_files": ["/*.npy"],
}
+
def check_file_pattern_dir(filepath, file_patterns):
"""
Check if directory (or its childs) contains some files with specific pattern names
"""
- dirs_with_session_files = []
child_dirs = [x[0] for x in os.walk(filepath)]
patterns_found = 0
- for dir in child_dirs:
+ for directory in child_dirs:
for pat in file_patterns:
- found_file = glob.glob(dir+pat)
+ found_file = glob.glob(directory + pat)
if len(found_file) > 0:
patterns_found = 1
break
@@ -35,25 +32,24 @@ def check_file_pattern_dir(filepath, file_patterns):
else:
return 0
+
def get_filepattern_paths(filepath, file_pattern):
"""
Check directory/files that correspond to file pattern in filepath
"""
- found_patterns = glob.glob(filepath+file_pattern)
+ found_patterns = glob.glob(filepath + file_pattern)
return found_patterns
+
def get_size_directory(path):
"""
get directory size of a folder for linux systems
"""
- command = ["du", path, '-s']
+ command = ["du", path, "-s"]
s = subprocess.run(command, capture_output=True)
- output = s.stdout.decode('UTF-8')
- if len(output) != 0:
- kbytes = int(output.split('\t')[0])
- else:
- kbytes = -1
+ output = s.stdout.decode("UTF-8")
+ kbytes = int(output.split("\t")[0]) if len(output) != 0 else -1
return kbytes
@@ -64,18 +60,16 @@ def get_size_directory_time(path):
command = ["du", "--separate-dirs", "--time", path]
output = subprocess.check_output(command)
- output = output.decode('UTF-8')
+ output = output.decode("UTF-8")
list_values = output.split("\n")
list_return = []
for line in list_values:
line_values = line.split("\t")
- dict_line = dict()
+ dict_line = {}
if len(line_values) == 3:
- dict_line['size'] = line_values[0]
- dict_line['date'] = line_values[1]
- dict_line['directory'] = line_values[2]
+ dict_line["size"] = line_values[0]
+ dict_line["date"] = line_values[1]
+ dict_line["directory"] = line_values[2]
list_return.append(dict_line)
-
- return list_return
-
+ return list_return
diff --git a/u19_pipeline/utils/scp_transfers.py b/u19_pipeline/utils/scp_transfers.py
index 3a8798d0..237143f7 100644
--- a/u19_pipeline/utils/scp_transfers.py
+++ b/u19_pipeline/utils/scp_transfers.py
@@ -1,15 +1,20 @@
-
import os
-import psutil
import subprocess
import sys
-from paramiko import SSHClient, AutoAddPolicy, RSAKey
+
+import psutil
+from paramiko import AutoAddPolicy, RSAKey, SSHClient
from paramiko.auth_handler import AuthenticationException, SSHException
-from scp import SCPClient, SCPException
+from scp import SCPClient
+
+from u19_pipeline.automatic_job.clusters_paths_and_transfers import (
+ public_key_location as public_key_location,
+)
+from u19_pipeline.utils.logging_config import get_logger
-from u19_pipeline.automatic_job.clusters_paths_and_transfers import public_key_location as public_key_location
+logger = get_logger(__name__)
-#Steps on windows machine
+# Steps on windows machine
# https://thesysadminchannel.com/solved-add-windowscapability-failed-error-code-0x800f0954-rsat-fix/
# PowerShell
# Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
@@ -27,6 +32,7 @@
# PowerShell
# restart-service sshd
+
class RemoteClient:
"""Client to interact with a remote host via SSH & SCP."""
@@ -37,21 +43,17 @@ def __init__(self, host, user, ssh_key_filepath, remote_path):
self.remote_path = remote_path
def _get_ssh_key(self):
- """ Fetch locally stored SSH key."""
+ """Fetch locally stored SSH key."""
try:
- self.ssh_key = RSAKey.from_private_key_file(
- self.ssh_key_filepath
- )
- print(
- f"Found SSH key at self {self.ssh_key_filepath}"
- )
+ self.ssh_key = RSAKey.from_private_key_file(self.ssh_key_filepath)
+ logger.info("Found SSH key at %s", self.ssh_key_filepath)
return self.ssh_key
except SSHException as e:
- print(e)
+ logger.error("SSHException: %s", e)
@property
def connection(self):
- """Open connection to remote host. """
+ """Open connection to remote host."""
try:
client = SSHClient()
client.load_system_host_keys()
@@ -64,9 +66,7 @@ def connection(self):
)
return client
except AuthenticationException as e:
- print(
- f"Authentication failed: did you remember to create an SSH key? {e}"
- )
+ logger.error("Authentication failed: did you remember to create an SSH key? %s", e)
raise e
@property
@@ -89,35 +89,49 @@ def download_folder(self, remote_path: str, local_path: str):
def transfer_scp(host=None, username=None, remote_path=None, local_path=None):
rc = RemoteClient(host, username, public_key_location, remote_path)
- print(host)
- print(username)
- print(public_key_location)
- print(remote_path)
- print(local_path)
+ logger.debug("host: %s", host)
+ logger.debug("username: %s", username)
+ logger.debug("public_key_location: %s", public_key_location)
+ logger.debug("remote_path: %s", remote_path)
+ logger.debug("local_path: %s", local_path)
rc._get_ssh_key()
- rc.scp
+ logger.debug("scp: %s", rc.scp)
rc.download_folder(remote_path=remote_path, local_path=local_path)
rc.disconnect()
-def call_scp_background(ip_address=None, system_user=None, recording_system_directory=None, data_directory=None):
- print(ip_address, system_user, data_directory, recording_system_directory)
- #transfer_scp(rec_series['ip_address'], rec_series['system_user'], rec_series['local_directory'], full_remote_path)
+def call_scp_background(
+ ip_address=None,
+ system_user=None,
+ recording_system_directory=None,
+ data_directory=None,
+):
- this_file = os.path.realpath(__file__)
+ logger.debug("ip_address=%s, system_user=%s, data_directory=%s, recording_system_directory=%s", ip_address, system_user, data_directory, recording_system_directory)
+ # transfer_scp(rec_series['ip_address'], rec_series['system_user'], rec_series['local_directory'], full_remote_path)
+ this_file = os.path.realpath(__file__)
- p = subprocess.Popen(["nohup", "python", this_file, ip_address, system_user, recording_system_directory, data_directory, "&"])
+ p = subprocess.Popen(
+ [
+ "nohup",
+ "python",
+ this_file,
+ ip_address,
+ system_user,
+ recording_system_directory,
+ data_directory,
+ "&",
+ ]
+ )
# To test without nohup
- #p = subprocess.run(["python", this_file, ip_address, system_user, recording_system_directory, data_directory], capture_output=True)
- #print('stderr', p.stderr.decode('UTF-8'))
- #print('stdout', p.stdout.decode('UTF-8'))
- #print('p.returncode', p.returncode)
-
- return True,p.pid
-
+ # p = subprocess.run(["python", this_file, ip_address, system_user, recording_system_directory, data_directory], capture_output=True)
+ # print('stderr', p.stderr.decode('UTF-8'))
+ # print('stdout', p.stdout.decode('UTF-8'))
+ # print('p.returncode', p.returncode)
+ return True, p.pid
def check_scp_transfer(pid):
@@ -127,9 +141,9 @@ def check_scp_transfer(pid):
if psutil.pid_exists(pid):
pr = psutil.Process(pid=pid)
- print(pr)
+ logger.debug("process: %s", pr)
gone, _ = psutil.wait_procs([pr], timeout=3)
- print(gone)
+ logger.debug("gone: %s", gone)
if len(gone) > 0:
finished = True
@@ -141,6 +155,7 @@ def check_scp_transfer(pid):
return finished, exit_code
+
def check_directory_copied_correctly():
pass
# diff -r -q /path/to/dir1 /path/to/dir2
@@ -148,9 +163,6 @@ def check_directory_copied_correctly():
if __name__ == "__main__":
args = sys.argv[1:]
- print(args)
+ logger.debug("args: %s", args)
transfer_scp(host=args[0], username=args[1], remote_path=args[2], local_path=args[3])
-
-
-
diff --git a/u19_pipeline/utils/slack_utils.py b/u19_pipeline/utils/slack_utils.py
index d78f1c9f..e514654c 100644
--- a/u19_pipeline/utils/slack_utils.py
+++ b/u19_pipeline/utils/slack_utils.py
@@ -30,9 +30,9 @@ def format_df_for_slack_message(df):
for i in column_list:
df[i] = df[i].astype(str)
max_len = max([df[i].str.len().max(), len(i)])
- df[i] = df[i].str.pad(width=max_len, side='right', fillchar='_')
- column_pad = i.ljust(max_len, '_')
- df = df.rename(columns={i:column_pad})
+ df[i] = df[i].str.pad(width=max_len, side="right", fillchar="_")
+ column_pad = i.ljust(max_len, "_")
+ df = df.rename(columns={i: column_pad})
df = df.to_string(index=False)
df = df.replace(" ", " --- ")
@@ -51,21 +51,21 @@ def send_slack_update_notification(webhook_url, base_message, session_info):
now = datetime.now()
datestr = now.strftime("%d-%b-%Y %H:%M:%S")
- msep = dict()
+ msep = {}
msep["type"] = "divider"
# Title #
- m1 = dict()
+ m1 = {}
m1["type"] = "section"
- m1_1 = dict()
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
m1_1["text"] = ":white_check_mark: *Automation pipeline update* on " + datestr + "\n\n"
m1["text"] = m1_1
# Info #
- m2 = dict()
+ m2 = {}
m2["type"] = "section"
- m2_1 = dict()
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
m2_1["text"] = (
"*"
@@ -88,7 +88,7 @@ def send_slack_update_notification(webhook_url, base_message, session_info):
)
m2["text"] = m2_1
- message = dict()
+ message = {}
message["blocks"] = [m1, msep, m2]
message["text"] = "Automation pipeline update recording:" + str(session_info["recording_id"])
@@ -104,21 +104,21 @@ def send_slack_error_notification(webhook_url, error_info, session_info):
if "job_id" not in session_info:
session_info["job_id"] = "Not a job"
- msep = dict()
+ msep = {}
msep["type"] = "divider"
# Title #
- m1 = dict()
+ m1 = {}
m1["type"] = "section"
- m1_1 = dict()
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
m1_1["text"] = ":rotating_light: *Automation pipeline error* on " + datestr + "\n\n"
m1["text"] = m1_1
# Info #
- m2 = dict()
+ m2 = {}
m2["type"] = "section"
- m2_1 = dict()
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
m2_1["text"] = (
"*Automation pipeline failed for:* \n"
@@ -140,9 +140,9 @@ def send_slack_error_notification(webhook_url, error_info, session_info):
m2["text"] = m2_1
# Error #
- m3 = dict()
+ m3 = {}
m3["type"] = "section"
- m3_1 = dict()
+ m3_1 = {}
m3_1["type"] = "mrkdwn"
m3_1["text"] = (
"*Error info* \n"
@@ -154,7 +154,7 @@ def send_slack_error_notification(webhook_url, error_info, session_info):
)
m3["text"] = m3_1
- message = dict()
+ message = {}
message["blocks"] = [m1, msep, m2, msep, m3]
message["text"] = "Automation pipeline error in recording:" + str(session_info["recording_id"])
@@ -170,21 +170,21 @@ def send_slack_error_pupillometry_notification(webhook_url, error_info, session_
if "job_id" not in session_info:
session_info["job_id"] = "Not a job"
- msep = dict()
+ msep = {}
msep["type"] = "divider"
# Title #
- m1 = dict()
+ m1 = {}
m1["type"] = "section"
- m1_1 = dict()
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
m1_1["text"] = ":rotating_light: *Automation pipeline error* on " + datestr + "\n\n"
m1["text"] = m1_1
# Info #
- m2 = dict()
+ m2 = {}
m2["type"] = "section"
- m2_1 = dict()
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
m2_1["text"] = (
"*Pupillometry pipeline failed for:* \n"
@@ -203,9 +203,9 @@ def send_slack_error_pupillometry_notification(webhook_url, error_info, session_
m2["text"] = m2_1
# Error #
- m3 = dict()
+ m3 = {}
m3["type"] = "section"
- m3_1 = dict()
+ m3_1 = {}
m3_1["type"] = "mrkdwn"
m3_1["text"] = (
"*Error info* \n"
@@ -217,7 +217,7 @@ def send_slack_error_pupillometry_notification(webhook_url, error_info, session_
)
m3["text"] = m3_1
- message = dict()
+ message = {}
message["blocks"] = [m1, msep, m2, msep, m3]
message["text"] = "Pupillometry pipeline error in session"
@@ -230,21 +230,21 @@ def send_slack_pupillometry_update_notification(webhook_url, base_message, sessi
now = datetime.now()
datestr = now.strftime("%d-%b-%Y %H:%M:%S")
- msep = dict()
+ msep = {}
msep["type"] = "divider"
# Title #
- m1 = dict()
+ m1 = {}
m1["type"] = "section"
- m1_1 = dict()
+ m1_1 = {}
m1_1["type"] = "mrkdwn"
m1_1["text"] = ":white_check_mark: *Pupillometry pipeline update* on " + datestr + "\n\n"
m1["text"] = m1_1
# Info #
- m2 = dict()
+ m2 = {}
m2["type"] = "section"
- m2_1 = dict()
+ m2_1 = {}
m2_1["type"] = "mrkdwn"
m2_1["text"] = (
"*"
@@ -264,7 +264,7 @@ def send_slack_pupillometry_update_notification(webhook_url, base_message, sessi
)
m2["text"] = m2_1
- message = dict()
+ message = {}
message["blocks"] = [m1, msep, m2]
message["text"] = "Pupillometry pipeline update"
diff --git a/u19_pipeline/utils/subject_metadata.py b/u19_pipeline/utils/subject_metadata.py
index 671fca8a..2d4449f2 100644
--- a/u19_pipeline/utils/subject_metadata.py
+++ b/u19_pipeline/utils/subject_metadata.py
@@ -1,9 +1,11 @@
-from typing import Iterable
+from collections.abc import Iterable
from u19_pipeline import lab, subject
-def fetch_slack_handles_for_lab_managers_by_subject(subjects: list[str] | str) -> list[str]:
+def fetch_slack_handles_for_lab_managers_by_subject(
+ subjects: list[str] | str,
+) -> list[str]:
"""Returns the Slack handles of lab managers for the labs associated with the given subjects."""
if isinstance(subjects, str):
subjects = [subjects]
@@ -18,7 +20,9 @@ def fetch_slack_handles_for_lab_managers_by_subject(subjects: list[str] | str) -
return slack_tags
-def fetch_slack_handles_for_lab_managers_by_user(user_ids: Iterable[str] | str) -> list[str]:
+def fetch_slack_handles_for_lab_managers_by_user(
+ user_ids: Iterable[str] | str,
+) -> list[str]:
"""Returns the Slack handles of lab managers for the given user IDs."""
if isinstance(user_ids, str):
diff --git a/u19_pipeline/utils/tiff_matlab_imaging_utils.py b/u19_pipeline/utils/tiff_matlab_imaging_utils.py
index 5a032170..92d12dc1 100644
--- a/u19_pipeline/utils/tiff_matlab_imaging_utils.py
+++ b/u19_pipeline/utils/tiff_matlab_imaging_utils.py
@@ -1,20 +1,22 @@
+import ast
+import contextlib
+import re
+import time
from pathlib import Path
+
import numpy as np
import tifffile as tiff
from sklearn.linear_model import HuberRegressor
-import time
-import re
-import ast
+from u19_pipeline.utils.logging_config import get_logger
-xySizeFactor = 1.05; # images are this much larger than nominal size
-zFactor = 1.45; # actual displacement in z vs command
+logger = get_logger(__name__)
+xy_size_factor = 1.05 # images are this much larger than nominal size
+z_factor = 1.45 # actual displacement in z vs command
-def select_files_from_mean_f(
- scan_directory,
- f_decrease_threshold=15
-):
+
+def select_files_from_mean_f(scan_directory, f_decrease_threshold=15):
"""
Estimate bleaching by loading only the first frame
from each TIFF file.
@@ -38,7 +40,7 @@ def select_files_from_mean_f(
start_time = time.time()
- print("Estimating bleaching...", end="", flush=True)
+ logger.info("Estimating bleaching...")
scan_directory = Path(scan_directory)
@@ -47,9 +49,7 @@ def select_files_from_mean_f(
n_files = len(tif_files)
if n_files == 0:
- raise FileNotFoundError(
- f"No TIFF files found in {scan_directory}"
- )
+ raise FileNotFoundError(f"No TIFF files found in {scan_directory}")
mean_f = np.zeros(n_files)
@@ -62,11 +62,9 @@ def select_files_from_mean_f(
# ---------------------------------------------------------
for i, tif_path in enumerate(tif_files):
-
- print(".", end="", flush=True)
+ logger.debug("processing file %d/%d", i + 1, n_files)
with tiff.TiffFile(tif_path) as tif_obj:
-
n_frames = len(tif_obj.pages)
first_frame = tif_obj.pages[0].asarray()
@@ -91,7 +89,6 @@ def select_files_from_mean_f(
x = frame_id.reshape(-1, 1)
if len(mean_f) > 1:
-
try:
# Robust regression equivalent to MATLAB robustfit
model = HuberRegressor()
@@ -101,26 +98,18 @@ def select_files_from_mean_f(
yhat = model.predict(x)
except Exception:
-
# fallback to standard linear fit
coeffs = np.polyfit(frame_id, mean_f, 1)
yhat = np.polyval(coeffs, frame_id)
- threshold_value = (
- yhat[0]
- - (f_decrease_threshold / 100.0) * yhat[0]
- )
+ threshold_value = yhat[0] - (f_decrease_threshold / 100.0) * yhat[0]
valid_idx = np.where(yhat > threshold_value)[0]
- if len(valid_idx) > 0:
- last_good_file = valid_idx[-1]
- else:
- last_good_file = n_files - 1
+ last_good_file = valid_idx[-1] if len(valid_idx) > 0 else n_files - 1
else:
-
last_good_file = 0
# ---------------------------------------------------------
@@ -129,11 +118,11 @@ def select_files_from_mean_f(
cumulative_frames = np.cumsum(frame_id)
- last_good_frame = cumulative_frames[last_good_file]
+ cumulative_frames[last_good_file]
elapsed_minutes = (time.time() - start_time) / 60
- print(f" done after {elapsed_minutes:.1f} min")
+ logger.info("done after %.1f min", elapsed_minutes)
return last_good_file
@@ -166,7 +155,6 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False):
# ---------------------------------------------------------
with tiff.TiffFile(tif_fn) as tif:
-
pages = tif.pages
header = pages
@@ -177,12 +165,9 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False):
image_description = first_page.description
# Some ScanImage versions also use Software tag
- software = first_page.tags.get('Software')
+ software = first_page.tags.get("Software")
- if software is not None:
- scope_str = str(software.value)
- else:
- scope_str = image_description
+ scope_str = str(software.value) if software is not None else image_description
parsed_info = {}
@@ -190,95 +175,60 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False):
# General image info
# -----------------------------------------------------
- parsed_info['Filename'] = str(tif_fn)
+ parsed_info["Filename"] = str(tif_fn)
- parsed_info['Width'] = first_page.imagewidth
+ parsed_info["Width"] = first_page.imagewidth
- parsed_info['Height'] = first_page.imagelength
+ parsed_info["Height"] = first_page.imagelength
# Acquisition time
- acq_match = re.search(
- r'epoch = \[([0-9].+?)\]',
- image_description
- )
+ acq_match = re.search(r"epoch = \[([0-9].+?)\]", image_description)
- parsed_info['AcqTime'] = (
- acq_match.group(1)
- if acq_match
- else ''
- )
+ parsed_info["AcqTime"] = acq_match.group(1) if acq_match else ""
# Number of depths
- depth_match = re.search(
- r'SI\.hFastZ\.numFramesPerVolume = (\d+)',
- scope_str
- )
+ depth_match = re.search(r"SI\.hFastZ\.numFramesPerVolume = (\d+)", scope_str)
if depth_match:
- parsed_info['nDepths'] = int(depth_match.group(1))
+ parsed_info["nDepths"] = int(depth_match.group(1))
else:
- parsed_info['nDepths'] = 0
+ parsed_info["nDepths"] = 0
- parsed_info['Zs'] = -1
+ parsed_info["Zs"] = -1
# Frame rate
- frame_rate_match = re.search(
- r'SI\.hRoiManager\.scanVolumeRate = ([0-9]+\.[0-9]+)',
- scope_str
- )
+ frame_rate_match = re.search(r"SI\.hRoiManager\.scanVolumeRate = ([0-9]+\.[0-9]+)", scope_str)
- parsed_info['frameRate'] = (
- float(frame_rate_match.group(1))
- if frame_rate_match
- else 0
- )
+ parsed_info["frameRate"] = float(frame_rate_match.group(1)) if frame_rate_match else 0
# ROI lag
- lag_match = re.search(
- r'SI\.hScan2D\.flytoTimePerScanfield = ([0-9]+\.[0-9]+)',
- scope_str
- )
+ lag_match = re.search(r"SI\.hScan2D\.flytoTimePerScanfield = ([0-9]+\.[0-9]+)", scope_str)
- parsed_info['interROIlag_sec'] = (
- float(lag_match.group(1))
- if lag_match
- else 0
- )
+ parsed_info["interROIlag_sec"] = float(lag_match.group(1)) if lag_match else 0
# -----------------------------------------------------
# Timing / behavioral sync
# -----------------------------------------------------
if not skip_behav_sync:
-
- parsed_info['Timing'] = {
- 'Frame_ts_sec': np.zeros(len(pages)),
- 'BehavFrames': []
+ parsed_info["Timing"] = {
+ "Frame_ts_sec": np.zeros(len(pages)),
+ "BehavFrames": [],
}
for i, page in enumerate(pages):
-
desc = page.description
# Frame timestamps
- ts_match = re.search(
- r'frameTimestamps_sec = ([0-9]+\.[0-9]+)',
- desc
- )
+ ts_match = re.search(r"frameTimestamps_sec = ([0-9]+\.[0-9]+)", desc)
if ts_match:
- parsed_info['Timing']['Frame_ts_sec'][i] = (
- float(ts_match.group(1))
- )
+ parsed_info["Timing"]["Frame_ts_sec"][i] = float(ts_match.group(1))
# I2C behavioral sync
- i2c_match = re.search(
- r'I2CData = ({.+})',
- desc
- )
+ i2c_match = re.search(r"I2CData = ({.+})", desc)
if i2c_match:
-
raw_data = i2c_match.group(1)
try:
@@ -291,130 +241,72 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False):
else:
behav_data = []
- parsed_info['Timing']['BehavFrames'].append(
- behav_data
- )
+ parsed_info["Timing"]["BehavFrames"].append(behav_data)
# -----------------------------------------------------
# Microscope / ScanImage metadata
# -----------------------------------------------------
- parsed_info['Scope'] = {}
+ parsed_info["Scope"] = {}
# Resolution scaling
- if 'objectiveResolution' in scope_str:
-
+ if "objectiveResolution" in scope_str:
try:
+ res_match = re.search(r"SI\.objectiveResolution = ([0-9]+\.[0-9]+)", scope_str)
- res_match = re.search(
- r'SI\.objectiveResolution = ([0-9]+\.[0-9]+)',
- scope_str
- )
-
- resolution_factor = (
- xySizeFactor * float(res_match.group(1))
- )
+ resolution_factor = xy_size_factor * float(res_match.group(1))
except Exception:
+ res_match = re.search(r"SI\.objectiveResolution = (\d+)", scope_str)
- res_match = re.search(
- r'SI\.objectiveResolution = (\d+)',
- scope_str
- )
-
- resolution_factor = (
- xySizeFactor * float(res_match.group(1))
- )
+ resolution_factor = xy_size_factor * float(res_match.group(1))
else:
-
resolution_factor = 1
# -----------------------------------------------------
# Power
# -----------------------------------------------------
- power_match = re.search(
- r'SI\.hBeams\.powers = (\d+)',
- scope_str
- )
+ power_match = re.search(r"SI\.hBeams\.powers = (\d+)", scope_str)
- parsed_info['Scope']['Power_percent'] = (
- float(power_match.group(1))
- if power_match
- else 0
- )
+ parsed_info["Scope"]["Power_percent"] = float(power_match.group(1)) if power_match else 0
# -----------------------------------------------------
# Channels
# -----------------------------------------------------
- channel_match = re.search(
- r'SI\.hChannels\.channelSave = (\d+)',
- scope_str
- )
+ channel_match = re.search(r"SI\.hChannels\.channelSave = (\d+)", scope_str)
- parsed_info['Scope']['Channels'] = (
- int(channel_match.group(1))
- if channel_match
- else 0
- )
+ parsed_info["Scope"]["Channels"] = int(channel_match.group(1)) if channel_match else 0
# -----------------------------------------------------
# Config / user filenames
# -----------------------------------------------------
- cfg_match = re.search(
- r'SI\.hConfigurationSaver\.cfgFilename = (.+cfg)',
- scope_str
- )
+ cfg_match = re.search(r"SI\.hConfigurationSaver\.cfgFilename = (.+cfg)", scope_str)
- parsed_info['Scope']['cfgFilename'] = (
- cfg_match.group(1).strip()
- if cfg_match
- else ''
- )
+ parsed_info["Scope"]["cfgFilename"] = cfg_match.group(1).strip() if cfg_match else ""
- usr_match = re.search(
- r'SI\.hConfigurationSaver\.usrFilename = (.+usr)',
- scope_str
- )
+ usr_match = re.search(r"SI\.hConfigurationSaver\.usrFilename = (.+usr)", scope_str)
- parsed_info['Scope']['usrFilename'] = (
- usr_match.group(1).strip()
- if usr_match
- else ''
- )
+ parsed_info["Scope"]["usrFilename"] = usr_match.group(1).strip() if usr_match else ""
# -----------------------------------------------------
# Fast-Z lag
# -----------------------------------------------------
- lag_match = re.search(
- r'SI\.hFastZ\.actuatorLag = ([0-9eE\.\-]+)',
- scope_str
- )
+ lag_match = re.search(r"SI\.hFastZ\.actuatorLag = ([0-9eE\.\-]+)", scope_str)
- parsed_info['Scope']['fastZ_lag'] = (
- float(lag_match.group(1))
- if lag_match
- else 0
- )
+ parsed_info["Scope"]["fastZ_lag"] = float(lag_match.group(1)) if lag_match else 0
# -----------------------------------------------------
# Fast-Z flyback
# -----------------------------------------------------
- flyback_match = re.search(
- r'SI\.hFastZ\.flybackTime = ([0-9]+\.[0-9]+)',
- scope_str
- )
+ flyback_match = re.search(r"SI\.hFastZ\.flybackTime = ([0-9]+\.[0-9]+)", scope_str)
- parsed_info['Scope']['fastZ_flybackTime'] = (
- float(flyback_match.group(1))
- if flyback_match
- else 0
- )
+ parsed_info["Scope"]["fastZ_flybackTime"] = float(flyback_match.group(1)) if flyback_match else 0
# -----------------------------------------------------
# Timing-related ScanImage parameters
@@ -426,82 +318,50 @@ def extract_float(pattern, default=0):
return float(match.group(1)) if match else default
- parsed_info['Scope']['linePeriod'] = extract_float(
- r'SI\.hRoiManager\.linePeriod = ([0-9.eE\-]+)'
- )
+ parsed_info["Scope"]["linePeriod"] = extract_float(r"SI\.hRoiManager\.linePeriod = ([0-9.eE\-]+)")
- parsed_info['Scope']['scanFramePeriod'] = extract_float(
- r'SI\.hRoiManager\.scanFramePeriod = ([0-9.eE\-]+)'
- )
+ parsed_info["Scope"]["scanFramePeriod"] = extract_float(r"SI\.hRoiManager\.scanFramePeriod = ([0-9.eE\-]+)")
- parsed_info['Scope']['scanFrameRate'] = extract_float(
- r'SI\.hRoiManager\.scanFrameRate = ([0-9.eE\-]+)'
- )
+ parsed_info["Scope"]["scanFrameRate"] = extract_float(r"SI\.hRoiManager\.scanFrameRate = ([0-9.eE\-]+)")
- parsed_info['Scope']['scanVolumeRate'] = extract_float(
- r'SI\.hRoiManager\.scanVolumeRate = ([0-9.eE\-]+)'
- )
+ parsed_info["Scope"]["scanVolumeRate"] = extract_float(r"SI\.hRoiManager\.scanVolumeRate = ([0-9.eE\-]+)")
- parsed_info['Scope']['flybackTimePerFrame'] = extract_float(
- r'SI\.hScan2D\.flybackTimePerFrame = ([0-9.eE\-]+)'
- )
+ parsed_info["Scope"]["flybackTimePerFrame"] = extract_float(r"SI\.hScan2D\.flybackTimePerFrame = ([0-9.eE\-]+)")
- parsed_info['Scope']['flytoTimePerScanfield'] = extract_float(
- r'SI\.hScan2D\.flytoTimePerScanfield = ([0-9.eE\-]+)'
+ parsed_info["Scope"]["flytoTimePerScanfield"] = extract_float(
+ r"SI\.hScan2D\.flytoTimePerScanfield = ([0-9.eE\-]+)"
)
# -----------------------------------------------------
# FOV corner points
# -----------------------------------------------------
- if 'fovCornerPoints' in scope_str:
-
- fov_match = re.search(
- r'SI\.hScan2D\.fovCornerPoints = (\[.+?\])',
- scope_str,
- re.DOTALL
- )
+ if "fovCornerPoints" in scope_str:
+ fov_match = re.search(r"SI\.hScan2D\.fovCornerPoints = (\[.+?\])", scope_str, re.DOTALL)
if fov_match:
-
try:
-
- new_fov_match = fov_match.group(1).replace(' ',',')
- new_fov_match = new_fov_match.replace(';',',')
+ new_fov_match = fov_match.group(1).replace(" ", ",")
+ new_fov_match = new_fov_match.replace(";", ",")
fov_points = np.array(ast.literal_eval(new_fov_match))
- fov_points = fov_points.reshape(4,2)
+ fov_points = fov_points.reshape(4, 2)
- parsed_info['Scope']['fovCornerPoints'] = (
- resolution_factor * fov_points
- )
+ parsed_info["Scope"]["fovCornerPoints"] = resolution_factor * fov_points
except Exception:
-
- parsed_info['Scope']['fovCornerPoints'] = 0
+ parsed_info["Scope"]["fovCornerPoints"] = 0
else:
-
- parsed_info['Scope']['fovCornerPoints'] = 0
+ parsed_info["Scope"]["fovCornerPoints"] = 0
else:
-
- parsed_info['Scope']['fovCornerPoints'] = 0
+ parsed_info["Scope"]["fovCornerPoints"] = 0
return header, parsed_info
-from pathlib import Path
-import re
-import ast
-import numpy as np
-import tifffile as tiff
-
-
-def parse_tif_header_mesoscope(
- tif_fn,
- skip_behav_sync=False
-):
+def parse_tif_header_mesoscope(tif_fn, skip_behav_sync=False):
"""
Parse ScanImage mesoscope TIFF metadata.
@@ -529,7 +389,6 @@ def parse_tif_header_mesoscope(
# ---------------------------------------------------------
with tiff.TiffFile(tif_fn) as tif:
-
pages = tif.pages
header = pages
@@ -538,21 +397,13 @@ def parse_tif_header_mesoscope(
image_description = first_page.description
- software_tag = first_page.tags.get('Software')
+ software_tag = first_page.tags.get("Software")
- artist_tag = first_page.tags.get('Artist')
+ artist_tag = first_page.tags.get("Artist")
- scope_str = (
- str(software_tag.value)
- if software_tag is not None
- else image_description
- )
+ scope_str = str(software_tag.value) if software_tag is not None else image_description
- roi_info = (
- str(artist_tag.value)
- if artist_tag is not None
- else ""
- )
+ roi_info = str(artist_tag.value) if artist_tag is not None else ""
parsed_info = {}
@@ -560,75 +411,47 @@ def parse_tif_header_mesoscope(
# General image info
# -----------------------------------------------------
- parsed_info['Filename'] = str(tif_fn)
+ parsed_info["Filename"] = str(tif_fn)
- parsed_info['Width'] = first_page.imagewidth
+ parsed_info["Width"] = first_page.imagewidth
- parsed_info['Height'] = first_page.imagelength
+ parsed_info["Height"] = first_page.imagelength
- acq_match = re.search(
- r'(?<=epoch = )\[.+?]',
- image_description
- )
+ acq_match = re.search(r"(?<=epoch = )\[.+?]", image_description)
- parsed_info['AcqTime'] = (
- acq_match.group(0)
- if acq_match
- else ""
- )
+ parsed_info["AcqTime"] = acq_match.group(0) if acq_match else ""
def extract_float(pattern, text, default=0):
match = re.search(pattern, text)
- return (
- float(match.group(0))
- if match
- else default
- )
+ return float(match.group(0)) if match else default
- parsed_info['frameRate'] = extract_float(
- r'(?<=SI\.hRoiManager\.scanVolumeRate = )\d+\.\d+',
- scope_str
- )
+ parsed_info["frameRate"] = extract_float(r"(?<=SI\.hRoiManager\.scanVolumeRate = )\d+\.\d+", scope_str)
- parsed_info['interROIlag_sec'] = extract_float(
- r'(?<=SI\.hScan2D\.flytoTimePerScanfield = )\d+\.\d+',
- scope_str
- )
+ parsed_info["interROIlag_sec"] = extract_float(r"(?<=SI\.hScan2D\.flytoTimePerScanfield = )\d+\.\d+", scope_str)
# -----------------------------------------------------
# Timing / behavior sync
# -----------------------------------------------------
if not skip_behav_sync:
-
- parsed_info['Timing'] = {
- 'Frame_ts_sec': np.zeros(len(pages)),
- 'BehavFrames': []
+ parsed_info["Timing"] = {
+ "Frame_ts_sec": np.zeros(len(pages)),
+ "BehavFrames": [],
}
for i, page in enumerate(pages):
-
desc = page.description
- ts_match = re.search(
- r'(?<=frameTimestamps_sec = )\d+\.\d+',
- desc
- )
+ ts_match = re.search(r"(?<=frameTimestamps_sec = )\d+\.\d+", desc)
if ts_match:
- parsed_info['Timing']['Frame_ts_sec'][i] = (
- float(ts_match.group(0))
- )
+ parsed_info["Timing"]["Frame_ts_sec"][i] = float(ts_match.group(0))
- i2c_match = re.search(
- r'(?<=I2CData = ){.+}',
- desc
- )
+ i2c_match = re.search(r"(?<=I2CData = ){.+}", desc)
if i2c_match:
-
raw_data = i2c_match.group(0)
try:
@@ -638,56 +461,31 @@ def extract_float(pattern, text, default=0):
behav_data = np.nan
else:
-
behav_data = []
- parsed_info['Timing']['BehavFrames'].append(
- behav_data
- )
+ parsed_info["Timing"]["BehavFrames"].append(behav_data)
# -----------------------------------------------------
# ROI parsing
# -----------------------------------------------------
- roi_marks = [
- m.start()
- for m in re.finditer(
- r'"scanimage\.mroi\.Roi"',
- roi_info
- )
- ]
+ roi_marks = [m.start() for m in re.finditer(r'"scanimage\.mroi\.Roi"', roi_info)]
- parsed_info['nROIs'] = len(roi_marks)
+ parsed_info["nROIs"] = len(roi_marks)
- parsed_info['ROI'] = []
+ parsed_info["ROI"] = []
# Resolution scaling
- resolution_match = re.search(
- r'(?<=SI\.objectiveResolution = )\d+\.\d+',
- scope_str
- )
+ resolution_match = re.search(r"(?<=SI\.objectiveResolution = )\d+\.\d+", scope_str)
- if resolution_match:
-
- resolution_factor = (
- xySizeFactor * float(resolution_match.group(0))
- )
-
- else:
-
- resolution_factor = 1
+ resolution_factor = xy_size_factor * float(resolution_match.group(0)) if resolution_match else 1
for i_roi in range(len(roi_marks)):
-
if i_roi != len(roi_marks) - 1:
-
- this_roi = roi_info[
- roi_marks[i_roi]:roi_marks[i_roi + 1]
- ]
+ this_roi = roi_info[roi_marks[i_roi] : roi_marks[i_roi + 1]]
else:
-
- this_roi = roi_info[roi_marks[i_roi]:]
+ this_roi = roi_info[roi_marks[i_roi] :]
roi_dict = {}
@@ -695,31 +493,17 @@ def extract_float(pattern, text, default=0):
# ROI name
# -------------------------------------------------
- name_match = re.search(
- r'(?<="name": ")\w+\d*',
- this_roi
- )
+ name_match = re.search(r'(?<="name": ")\w+\d*', this_roi)
- roi_dict['name'] = (
- name_match.group(0)
- if name_match
- else ''
- )
+ roi_dict["name"] = name_match.group(0) if name_match else ""
# -------------------------------------------------
# Z position
# -------------------------------------------------
- z_match = re.search(
- r'(?<="zs": )(|-)\d+',
- this_roi
- )
+ z_match = re.search(r'(?<="zs": )(|-)\d+', this_roi)
- roi_dict['Zs'] = (
- float(z_match.group(0))
- if z_match
- else 0
- )
+ roi_dict["Zs"] = float(z_match.group(0)) if z_match else 0
# -------------------------------------------------
# Helper for vector parsing
@@ -733,9 +517,7 @@ def parse_array(pattern, text):
return np.array([])
try:
- return np.array(
- ast.literal_eval(match.group(0))
- )
+ return np.array(ast.literal_eval(match.group(0)))
except Exception:
return np.array([])
@@ -744,348 +526,177 @@ def parse_array(pattern, text):
# ROI geometry
# -------------------------------------------------
- roi_dict['centerXY'] = (
- resolution_factor
- * parse_array(
- r'(?<="centerXY": )\[.+?]',
- this_roi
- )
- )
-
- roi_dict['sizeXY'] = (
- resolution_factor
- * parse_array(
- r'(?<="sizeXY": )\[.+?]',
- this_roi
- )
- )
-
- rotation_match = re.search(
- r'(?<="rotationDegrees":) (\d+|\d+\.\d+)',
- this_roi
- )
-
- roi_dict['rotationDegrees'] = (
- float(rotation_match.group(0))
- if rotation_match
- else 0
- )
-
- roi_dict['pixelResolutionXY'] = parse_array(
- r'(?<="pixelResolutionXY": )\[.+?]',
- this_roi
- )
-
- discrete_match = re.search(
- r'(?<="discretePlaneMode":) \d',
- this_roi
- )
-
- roi_dict['discretePlaneMode'] = bool(
- int(discrete_match.group(0))
- ) if discrete_match else False
-
- power_match = re.search(
- r'(?<="powers":) \d+',
- this_roi
- )
+ roi_dict["centerXY"] = resolution_factor * parse_array(r'(?<="centerXY": )\[.+?]', this_roi)
- if power_match:
+ roi_dict["sizeXY"] = resolution_factor * parse_array(r'(?<="sizeXY": )\[.+?]', this_roi)
+
+ rotation_match = re.search(r'(?<="rotationDegrees":) (\d+|\d+\.\d+)', this_roi)
- roi_dict['Power_percent'] = float(
- power_match.group(0)
- )
+ roi_dict["rotationDegrees"] = float(rotation_match.group(0)) if rotation_match else 0
- parsed_info['ROI'].append(roi_dict)
+ roi_dict["pixelResolutionXY"] = parse_array(r'(?<="pixelResolutionXY": )\[.+?]', this_roi)
+
+ discrete_match = re.search(r'(?<="discretePlaneMode":) \d', this_roi)
+
+ roi_dict["discretePlaneMode"] = bool(int(discrete_match.group(0))) if discrete_match else False
+
+ power_match = re.search(r'(?<="powers":) \d+', this_roi)
+
+ if power_match:
+ roi_dict["Power_percent"] = float(power_match.group(0))
+
+ parsed_info["ROI"].append(roi_dict)
# -----------------------------------------------------
# Scope metadata
# -----------------------------------------------------
- parsed_info['Scope'] = {}
-
- if not re.search(
- r'(?<="powers":) \d+',
- roi_info
- ):
+ parsed_info["Scope"] = {}
- power_match = re.search(
- r'(?<=SI\.hBeams\.powers = )\d+',
- scope_str
- )
+ if not re.search(r'(?<="powers":) \d+', roi_info):
+ power_match = re.search(r"(?<=SI\.hBeams\.powers = )\d+", scope_str)
- parsed_info['Scope']['Power_percent'] = (
- float(power_match.group(0))
- if power_match
- else 0
- )
+ parsed_info["Scope"]["Power_percent"] = float(power_match.group(0)) if power_match else 0
else:
-
- parsed_info['Scope']['Power_percent'] = (
- 'discrete powers per ROI'
- )
+ parsed_info["Scope"]["Power_percent"] = "discrete powers per ROI"
# Channels
- channel_match = re.search(
- r'(?<=SI\.hChannels\.channelSave = )\d+',
- scope_str
- )
+ channel_match = re.search(r"(?<=SI\.hChannels\.channelSave = )\d+", scope_str)
- parsed_info['Scope']['Channels'] = (
- int(channel_match.group(0))
- if channel_match
- else 0
- )
+ parsed_info["Scope"]["Channels"] = int(channel_match.group(0)) if channel_match else 0
# Config filenames
- cfg_match = re.search(
- r"(?<=SI\.hConfigurationSaver\.cfgFilename = ').+cfg",
- scope_str
- )
+ cfg_match = re.search(r"(?<=SI\.hConfigurationSaver\.cfgFilename = ').+cfg", scope_str)
- parsed_info['Scope']['cfgFilename'] = (
- cfg_match.group(0)
- if cfg_match
- else ''
- )
+ parsed_info["Scope"]["cfgFilename"] = cfg_match.group(0) if cfg_match else ""
- usr_match = re.search(
- r"(?<=SI\.hConfigurationSaver\.usrFilename = ').+usr",
- scope_str
- )
+ usr_match = re.search(r"(?<=SI\.hConfigurationSaver\.usrFilename = ').+usr", scope_str)
- parsed_info['Scope']['usrFilename'] = (
- usr_match.group(0)
- if usr_match
- else ''
- )
+ parsed_info["Scope"]["usrFilename"] = usr_match.group(0) if usr_match else ""
# -----------------------------------------------------
# Timing metadata
# -----------------------------------------------------
timing_fields = {
- 'fastZ_lag':
- r'(?<=SI\.hFastZ\.actuatorLag = )\d+\.\d+',
-
- 'fastZ_flybackTime':
- r'(?<=SI\.hFastZ\.flybackTime = )\d+\.\d+',
-
- 'linePeriod':
- r'(?<=SI\.hRoiManager\.linePeriod = )\d+\.\d+e-[0-9]+',
-
- 'scanFramePeriod':
- r'(?<=SI\.hRoiManager\.scanFramePeriod = )\d+\.\d+',
-
- 'scanFrameRate':
- r'(?<=SI\.hRoiManager\.scanFrameRate = )\d+\.\d+',
-
- 'scanVolumeRate':
- r'(?<=SI\.hRoiManager\.scanVolumeRate = )\d+\.\d+',
-
- 'flybackTimePerFrame':
- r'(?<=SI\.hScan2D\.flybackTimePerFrame = )\d+\.\d+',
-
- 'flytoTimePerScanfield':
- r'(?<=SI\.hScan2D\.flytoTimePerScanfield = )\d+\.\d+'
+ "fastZ_lag": r"(?<=SI\.hFastZ\.actuatorLag = )\d+\.\d+",
+ "fastZ_flybackTime": r"(?<=SI\.hFastZ\.flybackTime = )\d+\.\d+",
+ "linePeriod": r"(?<=SI\.hRoiManager\.linePeriod = )\d+\.\d+e-[0-9]+",
+ "scanFramePeriod": r"(?<=SI\.hRoiManager\.scanFramePeriod = )\d+\.\d+",
+ "scanFrameRate": r"(?<=SI\.hRoiManager\.scanFrameRate = )\d+\.\d+",
+ "scanVolumeRate": r"(?<=SI\.hRoiManager\.scanVolumeRate = )\d+\.\d+",
+ "flybackTimePerFrame": r"(?<=SI\.hScan2D\.flybackTimePerFrame = )\d+\.\d+",
+ "flytoTimePerScanfield": r"(?<=SI\.hScan2D\.flytoTimePerScanfield = )\d+\.\d+",
}
for key, pattern in timing_fields.items():
-
- parsed_info['Scope'][key] = extract_float(
- pattern,
- scope_str
- )
+ parsed_info["Scope"][key] = extract_float(pattern, scope_str)
# -----------------------------------------------------
# FOV corner points
# -----------------------------------------------------
- fov_match = re.search(
- r'(?<=SI\.hScan2D\.fovCornerPoints = )\[.+?]',
- scope_str
- )
+ fov_match = re.search(r"(?<=SI\.hScan2D\.fovCornerPoints = )\[.+?]", scope_str)
if fov_match:
-
try:
-
- new_fov_match = fov_match.group(0).replace(' ',',')
- new_fov_match = new_fov_match.replace(';',',')
+ new_fov_match = fov_match.group(0).replace(" ", ",")
+ new_fov_match = new_fov_match.replace(";", ",")
fov_points = np.array(ast.literal_eval(new_fov_match))
- fov_points = fov_points.reshape(4,2)
+ fov_points = fov_points.reshape(4, 2)
- parsed_info['Scope']['fovCornerPoints'] = (
- resolution_factor * fov_points
- )
+ parsed_info["Scope"]["fovCornerPoints"] = resolution_factor * fov_points
except Exception:
-
- parsed_info['Scope']['fovCornerPoints'] = 0
+ parsed_info["Scope"]["fovCornerPoints"] = 0
else:
-
- parsed_info['Scope']['fovCornerPoints'] = 0
+ parsed_info["Scope"]["fovCornerPoints"] = 0
# -----------------------------------------------------
# Stack metadata
# -----------------------------------------------------
- stack_match = re.search(
- r'(?<=SI\.hStackManager\.enable = )\w+',
- scope_str
- )
-
- stacks_enabled = (
- stack_match.group(0)
- if stack_match
- else 'false'
- )
+ stack_match = re.search(r"(?<=SI\.hStackManager\.enable = )\w+", scope_str)
- parsed_info['Scope']['stacks_enabled'] = (
- 1 if stacks_enabled == 'true' else 0
- )
+ stacks_enabled = stack_match.group(0) if stack_match else "false"
- if stacks_enabled == 'true':
+ parsed_info["Scope"]["stacks_enabled"] = 1 if stacks_enabled == "true" else 0
- actuator_match = re.search(
- r"(?<=SI\.hStackManager\.stackActuator = ')\w+",
- scope_str
- )
+ if stacks_enabled == "true":
+ actuator_match = re.search(r"(?<=SI\.hStackManager\.stackActuator = ')\w+", scope_str)
- definition_match = re.search(
- r"(?<=SI\.hStackManager\.stackDefinition = ')\w+",
- scope_str
- )
+ definition_match = re.search(r"(?<=SI\.hStackManager\.stackDefinition = ')\w+", scope_str)
- parsed_info['Scope']['stackActuator'] = (
- actuator_match.group(0)
- if actuator_match
- else ''
- )
+ parsed_info["Scope"]["stackActuator"] = actuator_match.group(0) if actuator_match else ""
- parsed_info['Scope']['stackDefinition'] = (
- definition_match.group(0)
- if definition_match
- else ''
- )
+ parsed_info["Scope"]["stackDefinition"] = definition_match.group(0) if definition_match else ""
# -----------------------------------------------------
# Motion correction
# -----------------------------------------------------
- motion_match = re.search(
- r'(?<=SI\.hMotionManager\.enable = )\w+',
- scope_str
- )
+ motion_match = re.search(r"(?<=SI\.hMotionManager\.enable = )\w+", scope_str)
- motion_enabled = (
- motion_match.group(0)
- if motion_match
- else 'false'
- )
+ motion_enabled = motion_match.group(0) if motion_match else "false"
- parsed_info['Scope']['motionCorrection_enabled'] = (
- 1 if motion_enabled == 'true' else 0
- )
+ parsed_info["Scope"]["motionCorrection_enabled"] = 1 if motion_enabled == "true" else 0
- if motion_enabled == 'true':
+ if motion_enabled == "true":
+ correction_z_match = re.search(r"(?<=SI\.hMotionManager\.correctionEnableZ = )\w+", scope_str)
- correction_z_match = re.search(
- r'(?<=SI\.hMotionManager\.correctionEnableZ = )\w+',
- scope_str
- )
-
- if (
- correction_z_match
- and correction_z_match.group(0) == 'true'
- ):
-
- parsed_info['Scope']['motionCorMode'] = (
- 'automated'
- )
+ if correction_z_match and correction_z_match.group(0) == "true":
+ parsed_info["Scope"]["motionCorMode"] = "automated"
else:
-
- parsed_info['Scope']['motionCorMode'] = (
- 'manual'
- )
+ parsed_info["Scope"]["motionCorMode"] = "manual"
# -----------------------------------------------------
# Depths / Z planes
# -----------------------------------------------------
- depth_match = re.search(
- r'SI\.hFastZ\.numFramesPerVolume = \d+',
- scope_str
- )
+ depth_match = re.search(r"SI\.hFastZ\.numFramesPerVolume = \d+", scope_str)
if depth_match:
-
- parsed_info['nDepths'] = int(
- re.search(r'\d+', depth_match.group(0)).group(0)
- )
+ parsed_info["nDepths"] = int(re.search(r"\d+", depth_match.group(0)).group(0))
else:
+ if stacks_enabled == "true":
+ slices_match = re.search(r"(?<=SI\.hStackManager\.actualNumSlices = )\d+", scope_str)
- if stacks_enabled == 'true':
-
- slices_match = re.search(
- r'(?<=SI\.hStackManager\.actualNumSlices = )\d+',
- scope_str
- )
-
- parsed_info['nDepths'] = (
- int(slices_match.group(0))
- if slices_match
- else 1
- )
+ parsed_info["nDepths"] = int(slices_match.group(0)) if slices_match else 1
else:
-
- parsed_info['nDepths'] = 1
+ parsed_info["nDepths"] = 1
# -----------------------------------------------------
# Z positions
# -----------------------------------------------------
- parsed_info['Zs'] = None
+ parsed_info["Zs"] = None
z_patterns = [
- r'(?<=SI\.hFastZ\.userZs = )\[.+?]',
- r'(?<=SI\.hStackManager\.zs = )\[.+?]',
- r'(?<=SI\.hFastZ\.position = )(|-)\d+'
+ r"(?<=SI\.hFastZ\.userZs = )\[.+?]",
+ r"(?<=SI\.hStackManager\.zs = )\[.+?]",
+ r"(?<=SI\.hFastZ\.position = )(|-)\d+",
]
for pattern in z_patterns:
-
z_match = re.search(pattern, scope_str)
if z_match:
-
try:
+ z_val = ast.literal_eval(z_match.group(0))
- z_val = ast.literal_eval(
- z_match.group(0)
- )
-
- parsed_info['Zs'] = (
- zFactor * np.array(z_val)
- )
+ parsed_info["Zs"] = z_factor * np.array(z_val)
except Exception:
+ with contextlib.suppress(Exception):
+ parsed_info["Zs"] = float(z_match.group(0))
- try:
-
- parsed_info['Zs'] = float(
- z_match.group(0)
- )
-
- except Exception:
- pass
break
- return header, parsed_info
\ No newline at end of file
+ return header, parsed_info
diff --git a/u19_pipeline/utils/tiff_utils.py b/u19_pipeline/utils/tiff_utils.py
index f497fcda..41d478e6 100644
--- a/u19_pipeline/utils/tiff_utils.py
+++ b/u19_pipeline/utils/tiff_utils.py
@@ -1,23 +1,24 @@
-import numpy as np
import gzip
-import shutil
-import re
import os
-import tifffile
-
-from pathlib import Path
+import re
+import shutil
from concurrent.futures import ProcessPoolExecutor
from datetime import datetime
+from pathlib import Path
+import numpy as np
+import tifffile
import u19_pipeline.utils.tiff_matlab_imaging_utils as tmiu
+from u19_pipeline.utils.logging_config import get_logger
+logger = get_logger(__name__)
-tif_number_fmt = r'_[0-9]{5}\.tif'
-tif_gz_number_fmt = r'_[0-9]{5}\.tif\.gz'
+tif_number_fmt = r"_[0-9]{5}\.tif"
+tif_gz_number_fmt = r"_[0-9]{5}\.tif\.gz"
-patt_acq_number = r'_[0-9]{5}_'
-patt_file_number = r'_[0-9]{5}\.'
+patt_acq_number = r"_[0-9]{5}_"
+patt_file_number = r"_[0-9]{5}\."
def check_tif_files(tif_dir):
@@ -26,38 +27,31 @@ def check_tif_files(tif_dir):
is_compressed = False
- fl = sorted(tif_dir.glob('*.tif'))
+ fl = sorted(tif_dir.glob("*.tif"))
if not fl:
-
- gz_files = sorted(tif_dir.glob('*.tif.gz'))
+ gz_files = sorted(tif_dir.glob("*.tif.gz"))
if gz_files:
-
is_compressed = True
for gz_file in gz_files:
- with gzip.open(gz_file, 'rb') as f_in:
- with open(gz_file.with_suffix(''), 'wb') as f_out:
- shutil.copyfileobj(f_in, f_out)
+ with gzip.open(gz_file, "rb") as f_in, open(gz_file.with_suffix(""), "wb") as f_out:
+ shutil.copyfileobj(f_in, f_out)
- fl = sorted(tif_dir.glob('*.tif'))
+ fl = sorted(tif_dir.glob("*.tif"))
else:
- raise FileNotFoundError(
- "No tif or tif.gz files found"
- )
+ raise FileNotFoundError("No tif or tif.gz files found")
fl = [f.name for f in fl]
match = re.search(tif_number_fmt, fl[0])
if match is None:
- raise ValueError(
- f'Invalid tif naming format: {fl[0]}'
- )
+ raise ValueError(f"Invalid tif naming format: {fl[0]}")
- basename = fl[0][:match.start()]
+ basename = fl[0][: match.start()]
return fl, basename, is_compressed
@@ -65,8 +59,7 @@ def check_tif_files(tif_dir):
def get_parsed_info_2photon(fl):
with ProcessPoolExecutor(max_workers=16) as exe:
-
- results = list(exe.map(tmiu.parse_tif_header_2photon,fl))
+ results = list(exe.map(tmiu.parse_tif_header_2photon, fl))
imheader = [r[0] for r in results]
parsed_info = [r[1] for r in results]
@@ -77,8 +70,7 @@ def get_parsed_info_2photon(fl):
def get_parsed_info_mesoscope(fl):
with ProcessPoolExecutor(max_workers=16) as exe:
-
- results = list(exe.map(tmiu.parse_tif_header_mesoscope,fl))
+ results = list(exe.map(tmiu.parse_tif_header_mesoscope, fl))
imheader = [r[0] for r in results]
parsed_info = [r[1] for r in results]
@@ -90,40 +82,33 @@ def get_recording_info(fl, imheader, parsed_info):
frames_per_file = np.zeros(len(fl), dtype=int)
- for i, file in enumerate(fl):
-
+ for i, _file in enumerate(fl):
if i == 0:
rec_info = parsed_info[i]
- rec_info['Timing']['BehavFrames'] = np.array(rec_info['Timing']['BehavFrames'], dtype=object)
+ rec_info["Timing"]["BehavFrames"] = np.array(rec_info["Timing"]["BehavFrames"], dtype=object)
else:
-
- if parsed_info[i]['Timing']['Frame_ts_sec'][0] == 0:
-
- parsed_info[i]['Timing']['Frame_ts_sec'] += (
- rec_info['Timing']['Frame_ts_sec'][-1]
- + 1 / rec_info['frameRate']
+ if parsed_info[i]["Timing"]["Frame_ts_sec"][0] == 0:
+ parsed_info[i]["Timing"]["Frame_ts_sec"] += (
+ rec_info["Timing"]["Frame_ts_sec"][-1] + 1 / rec_info["frameRate"]
)
- rec_info['Timing']['Frame_ts_sec'] = np.concatenate([
- rec_info['Timing']['Frame_ts_sec'],
- parsed_info[i]['Timing']['Frame_ts_sec']
- ])
+ rec_info["Timing"]["Frame_ts_sec"] = np.concatenate(
+ [
+ rec_info["Timing"]["Frame_ts_sec"],
+ parsed_info[i]["Timing"]["Frame_ts_sec"],
+ ]
+ )
- aux = np.array(parsed_info[i]['Timing']['BehavFrames'], dtype=object)
+ aux = np.array(parsed_info[i]["Timing"]["BehavFrames"], dtype=object)
aux = np.squeeze(aux)
if aux.size > 0:
- rec_info['Timing']['BehavFrames'] = np.concatenate([
- rec_info['Timing']['BehavFrames'],
- aux
- ])
+ rec_info["Timing"]["BehavFrames"] = np.concatenate([rec_info["Timing"]["BehavFrames"], aux])
frames_per_file[i] = len(imheader[i])
- rec_info['nFrames'] = len(
- rec_info['Timing']['Frame_ts_sec']
- )
+ rec_info["nFrames"] = len(rec_info["Timing"]["Frame_ts_sec"])
return rec_info, frames_per_file
@@ -131,103 +116,96 @@ def get_recording_info(fl, imheader, parsed_info):
def get_nfovs(rec_info, is_mesoscope):
if is_mesoscope:
- return len(rec_info['ROI'])
+ return len(rec_info["ROI"])
return 1
+
def check_acqtime(acq_time, scan_directory):
try:
- dt = datetime.strptime(acq_time,'%Y %m %d %H %M %S.%f'
- )
+ dt = datetime.strptime(acq_time, "%Y %m %d %H %M %S.%f")
- return dt.strftime('%Y-%m-%d %H:%M:%S')
+ return dt.strftime("%Y-%m-%d %H:%M:%S")
except Exception:
-
dirname = Path(scan_directory).name
- date_match = re.search(r'(\d{8})', dirname)
+ date_match = re.search(r"(\d{8})", dirname)
if date_match:
-
d = date_match.group(1)
- return (
- f'{d[:4]}-{d[4:6]}-{d[6:8]} 00:00:00'
- )
+ return f"{d[:4]}-{d[4:6]}-{d[6:8]} 00:00:00"
+
+ return "1000-01-01 00:00:00"
- return '1000-01-01 00:00:00'
-
def get_last_good_frame(frames_per_file, scan_directory):
last_good_file = tmiu.select_files_from_mean_f(scan_directory)
cumulative_frames = np.cumsum(frames_per_file)
-
return last_good_file, cumulative_frames
def create_scan_info_key(key, rec_info, bucket_dir):
- filename = Path(rec_info['Filename']).name
+ filename = Path(rec_info["Filename"]).name
scan_info_key = key.copy()
- scan_info_key['file_name_base'] = str(
- Path('/') / bucket_dir / filename
- )
+ scan_info_key["file_name_base"] = str(Path("/") / bucket_dir / filename)
- scan_info_key['scan_width'] = rec_info['Width']
- scan_info_key['scan_height'] = rec_info['Height']
+ scan_info_key["scan_width"] = rec_info["Width"]
+ scan_info_key["scan_height"] = rec_info["Height"]
try:
- datetime.strptime(rec_info['AcqTime'],'%Y-%m-%d %H:%M:%S')
- scan_info_key['acq_time'] = rec_info['AcqTime']
+ datetime.strptime(rec_info["AcqTime"], "%Y-%m-%d %H:%M:%S")
+ scan_info_key["acq_time"] = rec_info["AcqTime"]
except Exception:
- scan_info_key['acq_time'] = '1000-01-01 00:00:00'
-
- scan_info_key['n_depths'] = rec_info['nDepths']
- scan_info_key['scan_depths'] = rec_info['Zs']
- scan_info_key['frame_rate'] = rec_info['frameRate']
- scan_info_key['inter_fov_lag_sec'] = rec_info['interROIlag_sec']
- scan_info_key['frame_ts_sec'] = rec_info['Timing']['Frame_ts_sec']
-
- scope = rec_info['Scope']
- scan_info_key['power_percent'] = scope.get('Power_percent', 0)
- scan_info_key['channels'] = scope['Channels']
- scan_info_key['cfg_filename'] = scope['cfgFilename']
- scan_info_key['usr_filename'] = scope['usrFilename']
- scan_info_key['fast_z_lag'] = scope['fastZ_lag']
- scan_info_key['fast_z_flyback_time'] = scope['fastZ_flybackTime']
- scan_info_key['line_period'] = scope['linePeriod']
- scan_info_key['scan_frame_period'] = scope['scanFramePeriod']
- scan_info_key['scan_volume_rate'] = scope['scanVolumeRate']
- scan_info_key['flyback_time_per_frame'] = scope['flybackTimePerFrame']
- scan_info_key['flyto_time_per_scan_field'] = scope['flytoTimePerScanfield']
- scan_info_key['fov_corner_points'] = scope['fovCornerPoints']
-
- scan_info_key['nfovs'] = rec_info['nfovs']
- scan_info_key['nframes'] = rec_info['nFrames']
- scan_info_key['nframes_good'] = rec_info['nframes_good']
- scan_info_key['last_good_file'] = rec_info['last_good_file']
-
- if 'stacks_enabled' in scope:
- scan_info_key['stacks_enabled'] = scope['stacks_enabled'];
-
- if 'stackActuator' in scope:
- scan_info_key['stack_actuator'] = scope['stackActuator']
-
- if 'stackDefinition' in scope:
- scan_info_key['stack_definition'] = scope['stackDefinition']
-
- if 'motionCorrection_enabled' in scope:
- scan_info_key['motion_correction_enabled'] = scope['motionCorrection_enabled']
-
- if 'motionCorMode' in scope:
- scan_info_key['motion_correction_mode'] = scope['motionCorMode']
-
+ scan_info_key["acq_time"] = "1000-01-01 00:00:00"
+
+ scan_info_key["n_depths"] = rec_info["nDepths"]
+ scan_info_key["scan_depths"] = rec_info["Zs"]
+ scan_info_key["frame_rate"] = rec_info["frameRate"]
+ scan_info_key["inter_fov_lag_sec"] = rec_info["interROIlag_sec"]
+ scan_info_key["frame_ts_sec"] = rec_info["Timing"]["Frame_ts_sec"]
+
+ scope = rec_info["Scope"]
+ scan_info_key["power_percent"] = scope.get("Power_percent", 0)
+ scan_info_key["channels"] = scope["Channels"]
+ scan_info_key["cfg_filename"] = scope["cfgFilename"]
+ scan_info_key["usr_filename"] = scope["usrFilename"]
+ scan_info_key["fast_z_lag"] = scope["fastZ_lag"]
+ scan_info_key["fast_z_flyback_time"] = scope["fastZ_flybackTime"]
+ scan_info_key["line_period"] = scope["linePeriod"]
+ scan_info_key["scan_frame_period"] = scope["scanFramePeriod"]
+ scan_info_key["scan_volume_rate"] = scope["scanVolumeRate"]
+ scan_info_key["flyback_time_per_frame"] = scope["flybackTimePerFrame"]
+ scan_info_key["flyto_time_per_scan_field"] = scope["flytoTimePerScanfield"]
+ scan_info_key["fov_corner_points"] = scope["fovCornerPoints"]
+
+ scan_info_key["nfovs"] = rec_info["nfovs"]
+ scan_info_key["nframes"] = rec_info["nFrames"]
+ scan_info_key["nframes_good"] = rec_info["nframes_good"]
+ scan_info_key["last_good_file"] = rec_info["last_good_file"]
+
+ if "stacks_enabled" in scope:
+ scan_info_key["stacks_enabled"] = scope["stacks_enabled"]
+
+ if "stackActuator" in scope:
+ scan_info_key["stack_actuator"] = scope["stackActuator"]
+
+ if "stackDefinition" in scope:
+ scan_info_key["stack_definition"] = scope["stackDefinition"]
+
+ if "motionCorrection_enabled" in scope:
+ scan_info_key["motion_correction_enabled"] = scope["motionCorrection_enabled"]
+
+ if "motionCorMode" in scope:
+ scan_info_key["motion_correction_mode"] = scope["motionCorMode"]
+
return scan_info_key
@@ -248,34 +226,29 @@ def remove_compressed_videos(fl, directory):
directory = Path(directory)
for tif_file in fl:
-
file_base = directory / tif_file
- gz_file = Path(str(file_base) + '.gz')
+ gz_file = Path(str(file_base) + ".gz")
if gz_file.exists() and file_base.exists():
-
- print(f"Removing {gz_file}")
+ logger.info("Removing %s", gz_file)
gz_file.unlink()
else:
-
- print(
- f"Could not find compressed pair {file_base}"
- )
-
+ logger.warning("Could not find compressed pair %s", file_base)
# -----------------------------------------------------------------------------
# Helper functions
# -----------------------------------------------------------------------------
+
def replace_frame_timestamp(imdescription, lag):
"""
Replace frameTimestamps_sec value with lag-adjusted value.
"""
- pattern = r'(?<=frameTimestamps_sec = )([0-9]+\.[0-9]+)'
+ pattern = r"(?<=frameTimestamps_sec = )([0-9]+\.[0-9]+)"
match = re.search(pattern, imdescription)
if match:
@@ -293,16 +266,13 @@ def clean_software_string(software):
if software is None:
return ""
- software = software.replace(
- "hRoiManager.mroiEnable = 1",
- "hRoiManager.mroiEnable = 0"
- )
+ software = software.replace("hRoiManager.mroiEnable = 1", "hRoiManager.mroiEnable = 0")
idx = software.find("SI.hRoiManager.imagingFovUm")
if idx != -1:
newline_idx = software.find("\n", idx)
if newline_idx != -1:
- software = software[:idx] + software[newline_idx + 1:]
+ software = software[:idx] + software[newline_idx + 1 :]
return software
@@ -311,6 +281,7 @@ def clean_software_string(software):
# Mesoscope ROI splitting
# -----------------------------------------------------------------------------
+
def get_fov_mesoscope(
fl,
key_data,
@@ -320,7 +291,7 @@ def get_fov_mesoscope(
basename,
cumulative_frames,
scan_dirs_db,
- imaging_root
+ imaging_root,
):
"""
Python translation of MATLAB insert_fov_mesoscope.
@@ -331,32 +302,25 @@ def get_fov_mesoscope(
n_roi = rec_info["nROIs"]
if not skip_parsing:
-
- print("\tparsing ROIs...")
+ logger.debug("parsing ROIs...")
roi_nr = [roi["pixelResolutionXY"][1] for roi in rec_info["ROI"]]
- roi_nc = [roi["pixelResolutionXY"][0] for roi in rec_info["ROI"]]
+ [roi["pixelResolutionXY"][0] for roi in rec_info["ROI"]]
inter_roi_lag = rec_info["interROIlag_sec"]
depths = rec_info["nDepths"]
- which_depths = sorted(
- list(set([roi["Zs"] for roi in rec_info["ROI"]]))
- )
+ which_depths = sorted({roi["Zs"] for roi in rec_info["ROI"]})
# ---------------------------------------------------------------------
# Create directories
# ---------------------------------------------------------------------
for idepth in range(depths):
-
- which_roi = [
- i for i, roi in enumerate(rec_info["ROI"])
- if roi["Zs"] == which_depths[idepth]
- ]
+ which_roi = [i for i, roi in enumerate(rec_info["ROI"]) if roi["Zs"] == which_depths[idepth]]
for iroi in which_roi:
- dirname = f"ROI{iroi+1:02d}_z{idepth+1}"
+ dirname = f"ROI{iroi + 1:02d}_z{idepth + 1}"
os.makedirs(dirname, exist_ok=True)
# ---------------------------------------------------------------------
@@ -364,9 +328,7 @@ def get_fov_mesoscope(
# ---------------------------------------------------------------------
for iF, tif_file in enumerate(fl):
-
with tifffile.TiffFile(tif_file) as tif:
-
pages = tif.pages
current_header = pages[0].tags
@@ -375,32 +337,19 @@ def get_fov_mesoscope(
# Read stack
# -------------------------------------------------------------
- first_page = pages[0].asarray()
+ pages[0].asarray()
- thisstack = np.zeros(
- (
- imheader[iF][0]["Height"],
- 512,
- len(imheader[iF])
- ),
- dtype=np.uint16
- )
+ thisstack = np.zeros((imheader[iF][0]["Height"], 512, len(imheader[iF])), dtype=np.uint16)
pixel2sum = 1
for iframe, page in enumerate(pages):
-
temp_stack = page.asarray()
if temp_stack.shape[1] != thisstack.shape[1]:
-
pixel2sum = imheader[iF][0]["Width"] // 512
- reshaped = temp_stack.reshape(
- temp_stack.shape[0],
- pixel2sum,
- 512
- )
+ reshaped = temp_stack.reshape(temp_stack.shape[0], pixel2sum, 512)
temp_stack = reshaped.sum(axis=1)
@@ -416,36 +365,26 @@ def get_fov_mesoscope(
# -------------------------------------------------------------
for idepth in range(depths):
-
- ilag = 0
rowct = 0
- which_roi = [
- i for i, roi in enumerate(rec_info["ROI"])
- if roi["Zs"] == which_depths[idepth]
- ]
-
- for iroi in which_roi:
+ which_roi = [i for i, roi in enumerate(rec_info["ROI"]) if roi["Zs"] == which_depths[idepth]]
+ for ilag, iroi in enumerate(which_roi):
zidx = list(range(idepth, thisstack.shape[2], depths))
- substack = thisstack[
- rowct:rowct + roi_nr[iroi],
- :nc,
- :
- ][:, :, zidx]
+ substack = thisstack[rowct : rowct + roi_nr[iroi], :nc, :][:, :, zidx]
# -----------------------------------------------------
# Output filename
# -----------------------------------------------------
- match = re.search(r'_[0-9]{5}\.tif', tif_file)
+ match = re.search(r"_[0-9]{5}\.tif", tif_file)
thisfn = (
- f"./ROI{iroi+1:02d}_z{idepth+1}/"
- f"{tif_file[:match.start()]}"
- f"ROI{iroi+1:02d}_z{idepth+1}_"
- f"{tif_file[match.start()+1:]}"
+ f"./ROI{iroi + 1:02d}_z{idepth + 1}/"
+ f"{tif_file[: match.start()]}"
+ f"ROI{iroi + 1:02d}_z{idepth + 1}_"
+ f"{tif_file[match.start() + 1 :]}"
)
# -----------------------------------------------------
@@ -454,62 +393,38 @@ def get_fov_mesoscope(
first_desc = imheader[iF][zidx[0]]["ImageDescription"]
- first_desc = replace_frame_timestamp(
- first_desc,
- inter_roi_lag * ilag
- )
+ first_desc = replace_frame_timestamp(first_desc, inter_roi_lag * ilag)
- software = clean_software_string(
- current_header.get("Software", None).value
- if "Software" in current_header else ""
+ clean_software_string(
+ current_header.get("Software", None).value if "Software" in current_header else ""
)
- artist = (
- current_header.get("Artist", None).value
- if "Artist" in current_header else ""
- )
+ current_header.get("Artist", None).value if "Artist" in current_header else ""
- metadata = {
- "ImageDescription": first_desc,
- "Software": software,
- "Artist": artist
- }
# -----------------------------------------------------
# Write TIFF
# -----------------------------------------------------
with tifffile.TiffWriter(thisfn, bigtiff=True) as writer:
-
for iz in range(substack.shape[2]):
+ desc = imheader[iF][zidx[iz]]["ImageDescription"]
- desc = imheader[iF][zidx[iz]][
- "ImageDescription"
- ]
-
- desc = replace_frame_timestamp(
- desc,
- inter_roi_lag * ilag
- )
+ desc = replace_frame_timestamp(desc, inter_roi_lag * ilag)
writer.write(
substack[:, :, iz],
description=desc,
metadata=None,
- extratags=[]
+ extratags=[],
)
- ilag += 1
-
# -----------------------------------------------------
# Update row counter
# -----------------------------------------------------
if len(which_roi) > 1:
-
- padsize = (
- nr - sum([roi_nr[r] for r in which_roi])
- ) / (len(which_roi) - 1)
+ padsize = (nr - sum([roi_nr[r] for r in which_roi])) / (len(which_roi) - 1)
rowct += int(padsize + roi_nr[iroi])
@@ -519,96 +434,57 @@ def get_fov_mesoscope(
os.makedirs("originalStacks", exist_ok=True)
- shutil.move(
- tif_file,
- os.path.join("originalStacks", tif_file)
- )
-
+ shutil.move(tif_file, os.path.join("originalStacks", tif_file))
ct = 1
- cumulative_frames = np.concatenate(
- ([0], cumulative_frames)
- )
+ cumulative_frames = np.concatenate(([0], cumulative_frames))
fov_keys = []
tiff_files_entries = []
for iroi in range(n_roi):
-
ndepths = len(np.atleast_1d(rec_info["ROI"][iroi]["Zs"]))
for iz in range(ndepths):
-
fov_key = dict(key_data)
fov_key["tiff_split"] = ct
- fov_key["tiff_split_directory"] = (
- f"{scan_dirs_db['recording_directory']}/"
- f"ROI{iroi+1:02d}_z{iz+1}/"
- )
+ fov_key["tiff_split_directory"] = f"{scan_dirs_db['recording_directory']}/ROI{iroi + 1:02d}_z{iz + 1}/"
roi_name = rec_info["ROI"][iroi]["name"]
- if roi_name:
- thisname = f"{roi_name}_z{iz+1}"
- else:
- thisname = f"ROI{iroi+1:02d}_z{iz+1}"
+ thisname = f"{roi_name}_z{iz + 1}" if roi_name else f"ROI{iroi + 1:02d}_z{iz + 1}"
fov_key["tiff_split_name"] = thisname
# Safe assignments
- fov_key["fov_depth"] = (
- rec_info["ROI"][iroi]["Zs"]
- if rec_info["ROI"][iroi]["Zs"] is not None
- else 0
- )
+ fov_key["fov_depth"] = rec_info["ROI"][iroi]["Zs"] if rec_info["ROI"][iroi]["Zs"] is not None else 0
- fov_key["fov_center_xy"] = (
- rec_info["ROI"][iroi].get("centerXY", -1)
- )
+ fov_key["fov_center_xy"] = rec_info["ROI"][iroi].get("centerXY", -1)
- fov_key["fov_size_xy"] = (
- rec_info["ROI"][iroi].get("sizeXY", -1)
- )
+ fov_key["fov_size_xy"] = rec_info["ROI"][iroi].get("sizeXY", -1)
- fov_key["fov_rotation_degrees"] = (
- rec_info["ROI"][iroi].get("rotationDegrees", -1)
- )
+ fov_key["fov_rotation_degrees"] = rec_info["ROI"][iroi].get("rotationDegrees", -1)
- fov_key["fov_pixel_resolution_xy"] = (
- rec_info["ROI"][iroi].get("pixelResolutionXY", -1)
- )
+ fov_key["fov_pixel_resolution_xy"] = rec_info["ROI"][iroi].get("pixelResolutionXY", -1)
- fov_key["fov_discrete_plane_mode"] = (
- rec_info["ROI"][iroi].get("discretePlaneMode", -1)
- )
+ fov_key["fov_discrete_plane_mode"] = rec_info["ROI"][iroi].get("discretePlaneMode", -1)
if not fov_key["fov_discrete_plane_mode"]:
fov_key["fov_discrete_plane_mode"] = 0
- fov_key["power_percent"] = (
- rec_info["ROI"][iroi].get(
- "Power_percent",
- rec_info["Scope"]["Power_percent"]
- )
- )
+ fov_key["power_percent"] = rec_info["ROI"][iroi].get("Power_percent", rec_info["Scope"]["Power_percent"])
fov_keys.append(fov_key)
ct += 1
- tiff_split_directory = (
- Path(imaging_root)
- / fov_key["tiff_split_directory"]
- )
+ tiff_split_directory = Path(imaging_root) / fov_key["tiff_split_directory"]
- tif_files = sorted(
- tiff_split_directory.glob("*.tif")
- )
+ tif_files = sorted(tiff_split_directory.glob("*.tif"))
file_entries = []
for iF, tif_file in enumerate(tif_files):
-
entry = dict(key_data)
entry["tiff_split"] = fov_key["tiff_split"]
@@ -617,13 +493,12 @@ def get_fov_mesoscope(
entry["file_frame_range"] = [
cumulative_frames[iF] + 1,
- cumulative_frames[iF + 1]
+ cumulative_frames[iF + 1],
]
file_entries.append(entry)
-
- tiff_files_entries += file_entries
+ tiff_files_entries += file_entries
return fov_keys, tiff_files_entries
@@ -632,6 +507,7 @@ def get_fov_mesoscope(
# 2-photon FOV insert
# -----------------------------------------------------------------------------
+
def get_fov_photonmicro(key, rec_info, scan_dirs_db):
"""
Insert FOV metadata for 2-photon imaging.
@@ -640,9 +516,7 @@ def get_fov_photonmicro(key, rec_info, scan_dirs_db):
fovkey = dict(key)
fovkey["tiff_split"] = 1
- fovkey["tiff_split_directory"] = (
- scan_dirs_db["recording_directory"]
- )
+ fovkey["tiff_split_directory"] = scan_dirs_db["recording_directory"]
fovkey["fov_depth"] = 0
fovkey["fov_center_xy"] = 0
@@ -651,9 +525,7 @@ def get_fov_photonmicro(key, rec_info, scan_dirs_db):
fovkey["fov_pixel_resolution_xy"] = 0
fovkey["fov_discrete_plane_mode"] = 0
- fovkey["power_percent"] = (
- rec_info["Scope"]["Power_percent"]
- )
+ fovkey["power_percent"] = rec_info["Scope"]["Power_percent"]
return fovkey
@@ -662,13 +534,8 @@ def get_fov_photonmicro(key, rec_info, scan_dirs_db):
# 2-photon file insertion
# -----------------------------------------------------------------------------
-def get_fovfile_photonmicro(
- key,
- fl,
- imheader,
- patt_acq_number=r'_[0-9]{5}_',
- patt_file_number=r'_[0-9]{5}\.'
-):
+
+def get_fovfile_photonmicro(key, fl, imheader, patt_acq_number=r"_[0-9]{5}_", patt_file_number=r"_[0-9]{5}\."):
"""
Insert TIFF split file entries for 2-photon imaging.
"""
@@ -676,38 +543,22 @@ def get_fovfile_photonmicro(
filekeys = []
if len(fl) > 0:
-
prefile_frame_range = 0
for iF, filename in enumerate(fl):
+ acq_string = re.findall(patt_acq_number, filename)
- acq_string = re.findall(
- patt_acq_number,
- filename
- )
-
- number_string = re.findall(
- patt_file_number,
- filename
- )
+ number_string = re.findall(patt_file_number, filename)
if len(acq_string) == 1 and len(number_string) == 1:
-
- file_number = int(
- number_string[0][1:-1]
- )
- acq_number = int(
- acq_string[0][1:-1]
- )
+ file_number = int(number_string[0][1:-1])
+ acq_number = int(acq_string[0][1:-1])
if acq_number != 1:
- file_number = file_number-1+(acq_number-1)*100
+ file_number = file_number - 1 + (acq_number - 1) * 100
nframes = len(imheader[iF])
- frame_range = [
- prefile_frame_range + 1,
- prefile_frame_range + nframes
- ]
+ frame_range = [prefile_frame_range + 1, prefile_frame_range + nframes]
prefile_frame_range = frame_range[1]
@@ -720,4 +571,4 @@ def get_fovfile_photonmicro(
filekeys.append(entry)
- return filekeys
\ No newline at end of file
+ return filekeys
diff --git a/uv.lock b/uv.lock
index e684b35f..4f035520 100644
--- a/uv.lock
+++ b/uv.lock
@@ -2543,6 +2543,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4f/79/d3bbab197e86e0ff4f9c07122895b66a3e0d024247fcff7f12c473cb36d9/llvmlite-0.47.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6842cf6f707ec4be3d985a385ad03f72b2d724439e118fcbe99b2929964f0453", size = 39153839, upload-time = "2026-03-31T18:29:51.004Z" },
]
+[[package]]
+name = "markdown-it-py"
+version = "4.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "mdurl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" },
+]
+
[[package]]
name = "markupsafe"
version = "3.0.3"
@@ -2684,6 +2696,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" },
]
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
+]
+
[[package]]
name = "minio"
version = "7.2.20"
@@ -4418,6 +4439,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/7e/71/44ce230e1b7fadd372515a97e32a83011f906ddded8d03e3c6aafbdedbb7/rfc3987_syntax-1.1.0-py3-none-any.whl", hash = "sha256:6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f", size = 8046, upload-time = "2025-07-18T01:05:03.843Z" },
]
+[[package]]
+name = "rich"
+version = "15.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "markdown-it-py" },
+ { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" },
+]
+
[[package]]
name = "roifile"
version = "2026.2.10"
@@ -5219,6 +5253,7 @@ dependencies = [
{ name = "openpyxl" },
{ name = "pandas" },
{ name = "python-dotenv" },
+ { name = "rich" },
{ name = "scikit-image" },
{ name = "scipy" },
{ name = "tables" },
@@ -5277,6 +5312,7 @@ requires-dist = [
{ name = "plotly", marker = "extra == 'analysis'", specifier = ">=5.14" },
{ name = "psutil", marker = "extra == 'pipeline'" },
{ name = "python-dotenv", specifier = ">=0.20" },
+ { name = "rich", specifier = ">=15.0.0" },
{ name = "scanreader", marker = "extra == 'pipeline'", git = "https://github.com/BrainCOGS/scanreader.git" },
{ name = "scikit-image" },
{ name = "scipy" },