From f722b5a43c138c468930de3835c60a396c729c77 Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Thu, 11 Jun 2026 21:38:50 -0400 Subject: [PATCH 1/6] added ruff rules and fixed it --- notebooks/Marking_dead_mice_batch.ipynb | 3 +- notebooks/TestAlertSystem.ipynb | 10 +- notebooks/TestAutomaticPipeline.ipynb | 81 ++- notebooks/ad_hoc_psychometrics_plot.ipynb | 10 +- notebooks/behavior_metrics_performance.ipynb | 12 +- notebooks/check_pos_image_quality.ipynb | 13 +- .../create_tech_responsibilites_table.ipynb | 12 +- notebooks/create_weighingGUI_ss.ipynb | 23 +- .../00-Datajoint-configuration.ipynb | 7 +- .../00-Set-up-configuration.ipynb | 4 +- .../01-ephys-workflow-structure.ipynb | 10 +- .../01-imaging-workflow-structure.ipynb | 9 +- .../02-process-ephys-workflow.ipynb | 478 +++++++++--------- .../02-process-imaging-workflow.ipynb | 9 +- .../Creating_the_Scheduling_Table.ipynb | 13 +- .../brightness_video_analysis.ipynb | 19 +- .../delete_logs_periodically.ipynb | 6 +- .../delete_old_live_session_stats.ipynb | 1 + .../review_ephys_sync.ipynb | 12 +- .../startup_time_analysis.ipynb | 3 +- .../check_catgt_parameters_lfp.ipynb | 6 +- .../check_sync_notebook-new_method_bulk.ipynb | 11 +- ...sync_notebook-new_method_one_session.ipynb | 33 +- ..._notebook-new_method_one_trial_check.ipynb | 21 +- .../20251119_sync_v4.ipynb | 21 +- .../bulk_reprocess_ephys_recordings.ipynb | 4 +- .../check_daq_signals_jorge.ipynb | 14 +- .../check_sync_notebook-new_method.ipynb | 31 +- .../check_sync_notebook-new_method2.ipynb | 28 +- .../populate_sync_ephys_old_pipeline.ipynb | 8 +- .../reingest_missing_sync_sessions_old.ipynb | 2 + .../reingest_nan_sync_sessions.ipynb | 2 + ...ync_all_sessions_include_virmen_time.ipynb | 17 +- ..._sessions_include_virmen_time_status.ipynb | 9 +- .../get_ephys_sync_data.ipynb | 7 +- .../get_full_vectos_from_sync_data.ipynb | 7 +- ...t_full_vectos_from_sync_data_w_shift.ipynb | 10 +- .../reingest_missing_sync_sessions.ipynb | 11 +- .../reingest_sync_by_virmen_block.ipynb | 4 +- .../ephys_element/read_ephys_parameters.ipynb | 1 + .../ephys_element/test_paramspy_for_os.ipynb | 13 +- notebooks/fetch_session_data.ipynb | 3 +- .../00-Set-up-configuration.ipynb | 4 +- .../03-imaging-data-analysis.ipynb | 12 +- .../imaging_element/getSyncInfo_python.ipynb | 21 +- .../read_imaging_parameters.ipynb | 5 +- notebooks/imaging_element/read_params.py | 2 + .../test_Matlab_Python_conv.ipynb | 11 +- notebooks/imaging_storage_calculator.ipynb | 9 +- notebooks/live_stats_monitor.ipynb | 8 +- notebooks/numtrials_and_bias_check.ipynb | 11 +- notebooks/performance_metrics_2022.ipynb | 9 +- notebooks/psychometrics_plot.ipynb | 23 +- .../grade_pupillometry_processing.ipynb | 18 +- .../insert_pupilometry_data_test.ipynb | 12 +- .../pupillometry_notebooks/pupillometry.ipynb | 24 +- .../pupillometry_final_notebook.ipynb | 16 +- .../review_pupillometry_data.ipynb | 17 +- notebooks/read_virmen_behavior_file.ipynb | 14 +- .../storage_calculator_across_time.ipynb | 19 +- notebooks/training_days_per_protocol.ipynb | 14 +- ...ore U19 data pipeline with DataJoint.ipynb | 1 + ...a with U19 pipeline and save results.ipynb | 8 +- .../3-Build a simple data pipeline.ipynb | 1 + .../Old_tutorials/2019/python_demo.ipynb | 7 +- .../202001/0-Get DataJoint Ready.ipynb | 2 +- ...a with U19 pipeline and save results.ipynb | 7 +- .../202103/0-Get DataJoint Ready-Copy1.ipynb | 3 +- .../202103/0-Get DataJoint Ready.ipynb | 2 +- ...ore U19 data pipeline with DataJoint.ipynb | 1 + ...a with U19 pipeline and save results.ipynb | 8 +- .../3-Build a simple data pipeline.ipynb | 1 + ...ghtsheet_datajoint_tutorial (Python).ipynb | 4 +- notebooks/water_weight_alert.ipynb | 12 +- notebooks/xyz_picks_notebook.ipynb | 5 +- pyproject.toml | 28 +- scripts/conf_file_finding.py | 1 + scripts/create_schemas.py | 1 - scripts/drop_schemas.py | 18 +- scripts/ingest_hdf5_to_db.py | 13 +- scripts/populate_imaging.py | 3 +- u19_pipeline/__init__.py | 5 +- u19_pipeline/acquisition.py | 1 - .../alert_system/alert_code_skeleton.py | 6 - .../alert_system/alert_system_utility.py | 6 +- u19_pipeline/alert_system/behavior_metrics.py | 4 +- u19_pipeline/alert_system/cronjob_alert.py | 7 +- .../alert_system/custom_alerts/__init__.py | 1 + .../custom_alerts/braininit_storage.py | 11 +- .../alert_system/custom_alerts/rig_bias.py | 8 +- .../alert_system/custom_alerts/rig_trial.py | 6 +- .../custom_alerts/subject_bias.py | 8 +- .../custom_alerts/subject_trial.py | 6 +- .../cronjob_live_monitor_alert.py | 2 + .../live_monitor_alert/live_monitor_alert.py | 8 +- .../cronjob_live_session_stats_deletion.py | 2 + .../live_session_stats_deletion.py | 1 + .../cronjob_locked_tables_alert.py | 2 + .../locked_tables_alert.py | 2 +- .../log_deletion/cronjob_old_log_deletion.py | 2 + .../log_deletion/old_log_deletion.py | 5 +- .../alert_system/main_alert_system.py | 9 +- .../noDB_backup_creation_script.py | 23 +- .../rig_maintenance/check_rig_maintenance.py | 10 +- .../cronjob_schedule_check.py | 2 + .../schedule_check_alert.py | 3 +- .../create_weighing_gui_ss.py | 6 +- .../cronjob_water_weigh_alert.py | 2 + .../water_weigh_alert/water_weigh_alert.py | 6 +- .../clusters_paths_and_transfers.py | 14 +- .../create_missing_lfps_script.py | 4 +- .../automatic_job/cronjob_automatic_job.py | 2 + .../automatic_job/ephys_element_ingest.py | 14 +- .../automatic_job/ephys_element_populate.py | 4 +- u19_pipeline/automatic_job/imaging_element.py | 35 +- .../automatic_job/imaging_element_populate.py | 5 +- .../automatic_job/parameter_file_creator.py | 10 +- u19_pipeline/automatic_job/params_config.py | 6 +- .../populate_missing_syncbehavior_ephys.py | 10 +- .../pupillometry_check_handler_script.py | 2 +- .../automatic_job/pupillometry_handler.py | 33 +- .../pupillometry_handler_script.py | 2 +- .../automatic_job/pupillometry_main.py | 1 + .../automatic_job/recording_handler.py | 24 +- .../recording_process_handler.py | 38 +- u19_pipeline/automatic_job/slurm_creator.py | 7 +- u19_pipeline/automatic_job/tigress2pni.py | 41 +- u19_pipeline/behavior.py | 1 - u19_pipeline/copy_table.py | 9 +- u19_pipeline/ephys_pipeline.py | 4 +- u19_pipeline/ephys_sync.py | 10 +- u19_pipeline/imaging.py | 2 - u19_pipeline/imaging_pipeline.py | 15 +- u19_pipeline/meso.py | 2 - u19_pipeline/meso_analysis.py | 1 - u19_pipeline/puffs.py | 1 - u19_pipeline/recording.py | 3 +- u19_pipeline/recording_process.py | 4 +- u19_pipeline/task.py | 1 - u19_pipeline/temp/acquisition.py | 2 - u19_pipeline/utility.py | 26 +- .../utils/DemoReadSGLXData/readSGLX.py | 9 +- u19_pipeline/utils/dj_shortcuts.py | 1 + u19_pipeline/utils/dlc_process.py | 9 +- u19_pipeline/utils/ephys_fix_sync_code.py | 2 - u19_pipeline/utils/ephys_utils.py | 21 +- u19_pipeline/utils/file_utils.py | 1 + u19_pipeline/utils/insert_miscelaneous_db.py | 3 +- u19_pipeline/utils/matlab_utils.py | 21 +- u19_pipeline/utils/path_utils.py | 4 +- u19_pipeline/utils/scp_transfers.py | 11 +- u19_pipeline/utils/subject_metadata.py | 2 +- .../utils/tiff_matlab_imaging_utils.py | 17 +- u19_pipeline/utils/tiff_utils.py | 14 +- 154 files changed, 983 insertions(+), 992 deletions(-) diff --git a/notebooks/Marking_dead_mice_batch.ipynb b/notebooks/Marking_dead_mice_batch.ipynb index 6e50e8cc..27925688 100644 --- a/notebooks/Marking_dead_mice_batch.ipynb +++ b/notebooks/Marking_dead_mice_batch.ipynb @@ -6,8 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "from u19_pipeline import subject, action, acquisition\n", - "import pandas as pd" + "from u19_pipeline import action, subject" ] }, { diff --git a/notebooks/TestAlertSystem.ipynb b/notebooks/TestAlertSystem.ipynb index b3a59a11..46a93538 100644 --- a/notebooks/TestAlertSystem.ipynb +++ b/notebooks/TestAlertSystem.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -46,9 +47,9 @@ } ], "source": [ + "\n", "import datajoint as dj\n", - "import pandas as pd\n", - "import datetime\n", + "\n", "import u19_pipeline.alert_system.main_alert_system as mas\n", "\n" ] @@ -123,6 +124,7 @@ ], "source": [ "import u19_pipeline.alert_system.custom_alerts.subject_trial as subject_trial\n", + "\n", "df = subject_trial.main()\n", "df" ] @@ -161,8 +163,8 @@ "metadata": {}, "outputs": [], "source": [ - "import u19_pipeline.subject as subject\n", - "import u19_pipeline.action as action" + "import u19_pipeline.action as action\n", + "import u19_pipeline.subject as subject" ] }, { diff --git a/notebooks/TestAutomaticPipeline.ipynb b/notebooks/TestAutomaticPipeline.ipynb index a14e48b3..634813e0 100644 --- a/notebooks/TestAutomaticPipeline.ipynb +++ b/notebooks/TestAutomaticPipeline.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -42,16 +43,13 @@ "import datajoint as dj\n", "import pandas as pd\n", "\n", - "import u19_pipeline.ephys_pipeline as ephys_pipeline\n", - "from u19_pipeline.ephys_pipeline import ephys_element\n", - "\n", - "import u19_pipeline.imaging_pipeline as imaging_pipeline\n", - "from u19_pipeline.imaging_pipeline import imaging_element, scan_element\n", - "\n", "import u19_pipeline.automatic_job.recording_handler as rec_handler\n", "import u19_pipeline.automatic_job.recording_process_handler as rec_process_handler\n", - "\n", - "from u19_pipeline import recording, recording_process\n" + "import u19_pipeline.ephys_pipeline as ephys_pipeline\n", + "import u19_pipeline.imaging_pipeline as imaging_pipeline\n", + "from u19_pipeline import recording, recording_process\n", + "from u19_pipeline.ephys_pipeline import ephys_element\n", + "from u19_pipeline.imaging_pipeline import imaging_element, scan_element\n" ] }, { @@ -114,7 +112,7 @@ "metadata": {}, "outputs": [], "source": [ - "function_status_process = getattr(rec_process_handler.RecProcessHandler, 'transfer_request')" + "function_status_process = rec_process_handler.RecProcessHandler.transfer_request" ] }, { @@ -426,21 +424,11 @@ } ], "source": [ - "from datetime import datetime\n", "import copy\n", - "import pathlib\n", - "\n", - "from u19_pipeline.automatic_job import recording_handler\n", + "from datetime import datetime\n", "\n", - "import u19_pipeline.utils.dj_shortcuts as dj_short\n", - "import u19_pipeline.utils.slack_utils as slack_utils\n", "import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n", - "import u19_pipeline.automatic_job.slurm_creator as slurmlib\n", - "import u19_pipeline.automatic_job.parameter_file_creator as paramfilelib\n", "import u19_pipeline.automatic_job.params_config as config\n", - "import u19_pipeline.automatic_job.ephys_element_populate as ep\n", - "import u19_pipeline.automatic_job.imaging_element_populate as ip\n", - "\n", "\n", "update_value_dict = copy.deepcopy(config.default_update_value_dict)\n", "\n", @@ -1366,55 +1354,55 @@ "\n", "ops = dict()\n", "# sample rate\n", - "ops['fs'] = 30000; \n", - "ops['fshigh'] = 300; \n", + "ops['fs'] = 30000 \n", + "ops['fshigh'] = 300 \n", "\n", "# minimum firing rate on a \"good\" channel (0 to skip)\n", - "ops['minfr_goodchannels'] = 0; \n", + "ops['minfr_goodchannels'] = 0 \n", "\n", "# threshold on projections (like in Kilosort1, can be different for last pass like [10 4])\n", - "ops['Th'] = [10, 4]; \n", + "ops['Th'] = [10, 4] \n", "\n", "# how important is the amplitude penalty (like in Kilosort1, 0 means not used, 10 is average, 50 is a lot) \n", - "ops['lam'] = 10; \n", + "ops['lam'] = 10 \n", "\n", "# splitting a cluster at the end requires at least this much isolation for each sub-cluster (max = 1)\n", - "ops['AUCsplit'] = 0.9; \n", + "ops['AUCsplit'] = 0.9 \n", "\n", "# minimum spike rate (Hz), if a cluster falls below this for too long it gets removed\n", - "ops['minFR'] = 1/50; \n", + "ops['minFR'] = 1/50 \n", "\n", "# number of samples to average over (annealed from first to second value) \n", - "ops['momentum'] = [20, 400]; \n", + "ops['momentum'] = [20, 400] \n", "\n", "# spatial constant in um for computing residual variance of spike\n", - "ops['sigmaMask'] = 30; \n", + "ops['sigmaMask'] = 30 \n", "\n", "# threshold crossings for pre-clustering (in PCA projection space)\n", - "ops['ThPre'] = 8; \n", - "ops['reorder'] = 1; # whether to reorder batches for drift correction. \n", - "ops['nskip'] = 25; # how many batches to skip for determining spike PCs\n", + "ops['ThPre'] = 8 \n", + "ops['reorder'] = 1 # whether to reorder batches for drift correction. \n", + "ops['nskip'] = 25 # how many batches to skip for determining spike PCs\n", "\n", "# danger, changing these settings can lead to fatal errors\n", "# options for determining PCs\n", - "ops['spkTh'] = -6; # spike threshold in standard deviations (-6)\n", + "ops['spkTh'] = -6 # spike threshold in standard deviations (-6)\n", "\n", - "ops['GPU'] = 1; # has to be 1, no CPU version yet, sorry\n", + "ops['GPU'] = 1 # has to be 1, no CPU version yet, sorry\n", "# ops.Nfilt = 1024; # max number of clusters\n", - "ops['nfilt_factor'] = 4; # max number of clusters per good channel (even temporary ones)\n", - "ops['ntbuff'] = 64; # samples of symmetrical buffer for whitening and spike detection\n", - "ops['NT'] = 64*1024+ ops['ntbuff']; # must be multiple of 32 + ntbuff. This is the batch size (try decreasing if out of memory). \n", - "ops['whiteningRange'] = 32; # number of channels to use for whitening each channel\n", - "ops['nSkipCov'] = 25; # compute whitening matrix from every N-th batch\n", - "ops['scaleproc'] = 200; # int16 scaling of whitened data\n", - "ops['nPCs'] = 3; # how many PCs to project the spikes into\n", - "ops['useRAM'] = 0; # not yet available\n", + "ops['nfilt_factor'] = 4 # max number of clusters per good channel (even temporary ones)\n", + "ops['ntbuff'] = 64 # samples of symmetrical buffer for whitening and spike detection\n", + "ops['NT'] = 64*1024+ ops['ntbuff'] # must be multiple of 32 + ntbuff. This is the batch size (try decreasing if out of memory). \n", + "ops['whiteningRange'] = 32 # number of channels to use for whitening each channel\n", + "ops['nSkipCov'] = 25 # compute whitening matrix from every N-th batch\n", + "ops['scaleproc'] = 200 # int16 scaling of whitened data\n", + "ops['nPCs'] = 3 # how many PCs to project the spikes into\n", + "ops['useRAM'] = 0 # not yet available\n", "\n", "\n", "ops['trange'] = [0, 100000000000000]\n", "ops['NchanTOT'] = 384\n", - "ops['sig'] = 20; #spatial smoothness constant for registration\n", - "ops['nblocks'] = 5; # blocks for registration. 0 turns it off, 1 does rigid registration. Replaces \"datashift\" option. \n", + "ops['sig'] = 20 #spatial smoothness constant for registration\n", + "ops['nblocks'] = 5 # blocks for registration. 0 turns it off, 1 does rigid registration. Replaces \"datashift\" option. \n", "\n", "ephys_element.ClusteringParamSet.insert_new_params(\n", " processing_method='kilosort',\n", @@ -1786,7 +1774,10 @@ } ], "source": [ - "from u19_pipeline.ephys_pipeline import get_spikeglx_meta_filepath, get_session_directory\n", + "from u19_pipeline.ephys_pipeline import (\n", + " get_session_directory,\n", + " get_spikeglx_meta_filepath,\n", + ")\n", "\n", "so = recording.Recording.fetch(\"KEY\", as_dict=True)\n", "\n", diff --git a/notebooks/ad_hoc_psychometrics_plot.ipynb b/notebooks/ad_hoc_psychometrics_plot.ipynb index 8964144b..5ec81983 100644 --- a/notebooks/ad_hoc_psychometrics_plot.ipynb +++ b/notebooks/ad_hoc_psychometrics_plot.ipynb @@ -13,15 +13,15 @@ "metadata": {}, "outputs": [], "source": [ + "\n", "import datajoint as dj\n", - "import pandas as pd\n", + "import matplotlib.patches as mpatches\n", "import numpy as np\n", + "import pandas as pd\n", "import pylab as plt\n", - "import matplotlib.patches as mpatches\n", - "\n", "from matplotlib import cm\n", - "from u19_pipeline import utility\n", - "from inspect import getmembers, isfunction\n" + "\n", + "from u19_pipeline import utility\n" ] }, { diff --git a/notebooks/behavior_metrics_performance.ipynb b/notebooks/behavior_metrics_performance.ipynb index 5fdba49d..7bd9f21c 100644 --- a/notebooks/behavior_metrics_performance.ipynb +++ b/notebooks/behavior_metrics_performance.ipynb @@ -23,17 +23,15 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "\n", + "\n", "import datajoint as dj\n", - "import pandas as pd\n", "import numpy as np\n", + "import pandas as pd\n", "import pylab as plt\n", - "import matplotlib.patches as mpatches\n", - "\n", - "from matplotlib import cm\n", - "from u19_pipeline import utility\n", - "from inspect import getmembers, isfunction\n" + "\n" ] }, { @@ -547,7 +545,7 @@ "subject = dj.create_virtual_module('acquisition', 'u19_subject')\n", "recording = dj.create_virtual_module('recording', 'u19_recording')\n", "\n", - "key = subject.Subject() & \"user_id = 'ms81'\";\n", + "key = subject.Subject() & \"user_id = 'ms81'\"\n", "su = pd.DataFrame((recording.Recording.BehaviorSession * behavior.TowersBlock & key & \"level = 11\" & \"block_performance>0.6\").fetch(as_dict=True, order_by='block_performance DESC'))\n", "su\n" ] diff --git a/notebooks/check_pos_image_quality.ipynb b/notebooks/check_pos_image_quality.ipynb index 6e7b5591..8706a29f 100644 --- a/notebooks/check_pos_image_quality.ipynb +++ b/notebooks/check_pos_image_quality.ipynb @@ -23,6 +23,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -32,15 +33,12 @@ "metadata": {}, "outputs": [], "source": [ - "import datajoint as dj\n", - "import os\n", - "import pathlib\n", - "import pandas as pd\n", + "\n", "import cv2\n", - "import glob\n", - "import numpy as np\n", + "import datajoint as dj\n", "import matplotlib.pyplot as plt\n", - "from matplotlib import patches\n" + "import numpy as np\n", + "import pandas as pd\n" ] }, { @@ -1458,7 +1456,6 @@ } ], "source": [ - "import cv2\n", "\n", "video_name = 'top_image_video4.mp4' # Name of the output video file\n", "\n", diff --git a/notebooks/create_tech_responsibilites_table.ipynb b/notebooks/create_tech_responsibilites_table.ipynb index 634ffe27..0dd58f3e 100644 --- a/notebooks/create_tech_responsibilites_table.ipynb +++ b/notebooks/create_tech_responsibilites_table.ipynb @@ -13,12 +13,11 @@ "metadata": {}, "outputs": [], "source": [ + "import json\n", + "\n", "import datajoint as dj\n", - "import numpy as np\n", - "import pylab as plt\n", "import pandas as pd\n", - "import json\n", - "import datetime\n", + "\n", "pd.set_option('display.max_rows', 50)" ] }, @@ -220,10 +219,7 @@ " owner_act_dict['weekly'] = ['Nothing']\n", "\n", " if tech_responsibility == 'yes':\n", - " if subject_status in ('Dead', 'AdLibWater', 'Missing'):\n", - " tech_act_dict = base_weekly_dict.copy()\n", - " tech_act_dict['weekly'] = subject_status_dict_technician[subject_status]\n", - " elif num_act == 1:\n", + " if subject_status in ('Dead', 'AdLibWater', 'Missing') or num_act == 1:\n", " tech_act_dict = base_weekly_dict.copy()\n", " tech_act_dict['weekly'] = subject_status_dict_technician[subject_status]\n", " elif num_act == 2 and list(index_act.values())[0] == [1,2,3,4,5] or list(index_act.values())[0] == [0,6]:\n", diff --git a/notebooks/create_weighingGUI_ss.ipynb b/notebooks/create_weighingGUI_ss.ipynb index a2408ac5..8206baa9 100644 --- a/notebooks/create_weighingGUI_ss.ipynb +++ b/notebooks/create_weighingGUI_ss.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -41,29 +42,27 @@ } ], "source": [ - "import datajoint as dj\n", - "import pandas as pd\n", - "import time\n", - "from zoneinfo import ZoneInfo\n", + "import base64\n", "import datetime\n", "import pathlib\n", - "import numpy as np\n", - "import time\n", - "import base64\n", "import shutil\n", + "import time\n", + "\n", + "import datajoint as dj\n", + "import numpy as np\n", "import openpyxl\n", - "from openpyxl.utils.dataframe import dataframe_to_rows\n", + "import pandas as pd\n", "from openpyxl.drawing.image import Image\n", + "from openpyxl.utils.dataframe import dataframe_to_rows\n", "\n", "time.sleep(1)\n", "\n", - "import u19_pipeline.alert_system.water_weigh_alert.water_weigh_alert as wwa\n", - "import u19_pipeline.scheduler as scheduler\n", "import u19_pipeline.acquisition as acquisition\n", + "import u19_pipeline.alert_system.water_weigh_alert.water_weigh_alert as wwa\n", "import u19_pipeline.behavior as behavior\n", - "import u19_pipeline.subject as subject\n", "import u19_pipeline.lab as lab\n", - "\n", + "import u19_pipeline.scheduler as scheduler\n", + "import u19_pipeline.subject as subject\n", "\n", "DJ_CUSTOM_VARIABLES_FILENAME = 'DJCustomVariables.csv'\n", "SLACK_WEBHOOK_FILENAME = 'SlackChannels.csv'\n", diff --git a/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb b/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb index 8508dbdc..4cdbb4a5 100644 --- a/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb +++ b/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb @@ -31,6 +31,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -51,11 +52,13 @@ } ], "source": [ - "import datajoint as dj\n", - "import os\n", "import getpass\n", + "import os\n", + "\n", "import datajoint as dj\n", + "\n", "import u19_pipeline.utility as utility\n", + "\n", "dj.__version__" ] }, diff --git a/notebooks/developer_notebooks/00-Set-up-configuration.ipynb b/notebooks/developer_notebooks/00-Set-up-configuration.ipynb index f62935c2..1be21a9a 100644 --- a/notebooks/developer_notebooks/00-Set-up-configuration.ipynb +++ b/notebooks/developer_notebooks/00-Set-up-configuration.ipynb @@ -24,10 +24,10 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", - "import os\n", - "import getpass\n", "import datajoint as dj\n", + "\n", "import u19_pipeline.utility as utility" ] }, diff --git a/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb b/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb index ad7a7297..42a3f4f5 100644 --- a/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb +++ b/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb @@ -39,6 +39,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -47,9 +48,7 @@ "execution_count": 2, "metadata": {}, "outputs": [], - "source": [ - "import os" - ] + "source": [] }, { "cell_type": "markdown", @@ -83,8 +82,9 @@ ], "source": [ "import datajoint as dj\n", - "from u19_pipeline import subject, acquisition, ephys_pipeline\n", - "from u19_pipeline.ephys_pipeline import probe_element, ephys_element" + "\n", + "from u19_pipeline import acquisition, ephys_pipeline, subject\n", + "from u19_pipeline.ephys_pipeline import ephys_element, probe_element" ] }, { diff --git a/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb b/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb index 2d0e427d..a071d35a 100644 --- a/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb +++ b/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb @@ -24,6 +24,7 @@ "outputs": [], "source": [ "import os\n", + "\n", "os.chdir('../..')" ] }, @@ -44,11 +45,11 @@ "source": [ "import datajoint as dj\n", "\n", - "# modules within original u19_pipeline\n", - "from u19_pipeline import lab, subject, acquisition, imaging\n", - "\n", "# modules defined in DataJoint Calcium Imaging element, installed to u19_pipeline\n", - "from u19_pipeline.imaging_element import scan_element, imaging_element" + "from u19_pipeline.imaging_element import imaging_element, scan_element\n", + "\n", + "# modules within original u19_pipeline\n", + "from u19_pipeline import acquisition, subject" ] }, { diff --git a/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb b/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb index ba0ddf88..de1a4280 100644 --- a/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb +++ b/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb @@ -2,13 +2,14 @@ "cells": [ { "cell_type": "markdown", + "metadata": {}, "source": [ "# Run ephys element workflow" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "This notebook walks you through the steps to run the ephys workflow. \n", "The workflow requires neuropixels meta file and kilosort output data. \n", @@ -39,142 +40,131 @@ "\n", "\n", "Let's start by importing the relevant modules." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 1, - "source": [ - "from scripts.conf_file_finding import try_find_conf_file\r\n", - "try_find_conf_file()" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Local configuration file found !!, no need to run the configuration (unless configuration has changed)\n" ] } ], - "metadata": {} + "source": [ + "from scripts.conf_file_finding import try_find_conf_file\r\n", + "\r\n", + "try_find_conf_file()" + ] }, { "cell_type": "code", "execution_count": 2, - "source": [ - "import datetime\r\n", - "import pathlib\r\n", - "import numpy as np\r\n", - "import pylab as pl\r\n", - "from u19_pipeline import acquisition, ephys\r\n", - "from u19_pipeline.ephys import ephys_element, probe_element, get_session_directory\r\n", - "import datajoint as dj" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Connecting alvaros@datajoint00.pni.princeton.edu:3306\n" ] } ], - "metadata": {} + "source": [ + "import datetime\r\n", + "\r\n", + "import datajoint as dj\r\n", + "import numpy as np\r\n", + "import pylab as pl\r\n", + "from u19_pipeline.ephys import ephys_element, get_session_directory, probe_element\r\n", + "\r\n", + "from u19_pipeline import acquisition, ephys\r" + ] }, { "cell_type": "code", "execution_count": 9, - "source": [ - "dj.conn()" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "DataJoint connection (connected) alvaros@datajoint00.pni.princeton.edu:3306" ] }, + "execution_count": 9, "metadata": {}, - "execution_count": 9 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "dj.conn()" + ] }, { "cell_type": "code", "execution_count": 3, + "metadata": {}, + "outputs": [], "source": [ "key = {\r\n", " 'subject_fullname': 'ms81_M005',\r\n", " 'session_date': datetime.date(2021, 5, 5)}" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "The module ephys_element contains all the tables designed in the DataJoint ephys element, we could draw the diagram to see the schema structures" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 11, - "source": [ - "dj.Diagram(acquisition.Session) + ephys.EphysSession + dj.Diagram(ephys_element)" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { + "image/svg+xml": "\n\n\n\n\nephys_element.ClusteringMethod\n\n\nephys_element.ClusteringMethod\n\n\n\n\n\nephys_element.ClusteringParamSet\n\n\nephys_element.ClusteringParamSet\n\n\n\n\n\nephys_element.ClusteringMethod->ephys_element.ClusteringParamSet\n\n\n\n\nephys_element.LFP\n\n\nephys_element.LFP\n\n\n\n\n\nephys_element.LFP.Electrode\n\n\nephys_element.LFP.Electrode\n\n\n\n\n\nephys_element.LFP->ephys_element.LFP.Electrode\n\n\n\n\nephys_element.ClusterQualityLabel\n\n\nephys_element.ClusterQualityLabel\n\n\n\n\n\nephys_element.CuratedClustering.Unit\n\n\nephys_element.CuratedClustering.Unit\n\n\n\n\n\nephys_element.ClusterQualityLabel->ephys_element.CuratedClustering.Unit\n\n\n\n\nephys_element.InsertionLocation\n\n\nephys_element.InsertionLocation\n\n\n\n\n\nephys_element.ClusteringTask\n\n\nephys_element.ClusteringTask\n\n\n\n\n\nephys_element.ClusteringParamSet->ephys_element.ClusteringTask\n\n\n\n\nephys_element.EphysRecording\n\n\nephys_element.EphysRecording\n\n\n\n\n\nephys_element.EphysRecording->ephys_element.LFP\n\n\n\n\nephys_element.EphysRecording.EphysFile\n\n\nephys_element.EphysRecording.EphysFile\n\n\n\n\n\nephys_element.EphysRecording->ephys_element.EphysRecording.EphysFile\n\n\n\n\nephys_element.EphysRecording->ephys_element.ClusteringTask\n\n\n\n\nephys_element.AcquisitionSoftware\n\n\nephys_element.AcquisitionSoftware\n\n\n\n\n\nephys_element.AcquisitionSoftware->ephys_element.EphysRecording\n\n\n\n\nephys_element.ProbeInsertion\n\n\nephys_element.ProbeInsertion\n\n\n\n\n\nephys_element.ProbeInsertion->ephys_element.InsertionLocation\n\n\n\n\nephys_element.ProbeInsertion->ephys_element.EphysRecording\n\n\n\n\nephys_element.WaveformSet.Waveform\n\n\nephys_element.WaveformSet.Waveform\n\n\n\n\n\nephys_element.CuratedClustering.Unit->ephys_element.WaveformSet.Waveform\n\n\n\n\nephys_element.WaveformSet.PeakWaveform\n\n\nephys_element.WaveformSet.PeakWaveform\n\n\n\n\n\nephys_element.CuratedClustering.Unit->ephys_element.WaveformSet.PeakWaveform\n\n\n\n\nephys_element.Curation\n\n\nephys_element.Curation\n\n\n\n\n\nephys_element.CuratedClustering\n\n\nephys_element.CuratedClustering\n\n\n\n\n\nephys_element.Curation->ephys_element.CuratedClustering\n\n\n\n\nephys_element.WaveformSet\n\n\nephys_element.WaveformSet\n\n\n\n\n\nephys_element.WaveformSet->ephys_element.WaveformSet.Waveform\n\n\n\n\nephys_element.WaveformSet->ephys_element.WaveformSet.PeakWaveform\n\n\n\n\nacquisition.Session\n\n\nacquisition.Session\n\n\n\n\n\nephys.EphysSession\n\n\nephys.EphysSession\n\n\n\n\n\nacquisition.Session->ephys.EphysSession\n\n\n\n\nephys_element.CuratedClustering->ephys_element.CuratedClustering.Unit\n\n\n\n\nephys_element.CuratedClustering->ephys_element.WaveformSet\n\n\n\n\nephys_element.Clustering\n\n\nephys_element.Clustering\n\n\n\n\n\nephys_element.Clustering->ephys_element.Curation\n\n\n\n\nephys_element.ClusteringTask->ephys_element.Clustering\n\n\n\n\nephys.EphysSession->ephys_element.ProbeInsertion\n\n\n\n", "text/plain": [ "" - ], - "image/svg+xml": "\n\n\n\n\nephys_element.ClusteringMethod\n\n\nephys_element.ClusteringMethod\n\n\n\n\n\nephys_element.ClusteringParamSet\n\n\nephys_element.ClusteringParamSet\n\n\n\n\n\nephys_element.ClusteringMethod->ephys_element.ClusteringParamSet\n\n\n\n\nephys_element.LFP\n\n\nephys_element.LFP\n\n\n\n\n\nephys_element.LFP.Electrode\n\n\nephys_element.LFP.Electrode\n\n\n\n\n\nephys_element.LFP->ephys_element.LFP.Electrode\n\n\n\n\nephys_element.ClusterQualityLabel\n\n\nephys_element.ClusterQualityLabel\n\n\n\n\n\nephys_element.CuratedClustering.Unit\n\n\nephys_element.CuratedClustering.Unit\n\n\n\n\n\nephys_element.ClusterQualityLabel->ephys_element.CuratedClustering.Unit\n\n\n\n\nephys_element.InsertionLocation\n\n\nephys_element.InsertionLocation\n\n\n\n\n\nephys_element.ClusteringTask\n\n\nephys_element.ClusteringTask\n\n\n\n\n\nephys_element.ClusteringParamSet->ephys_element.ClusteringTask\n\n\n\n\nephys_element.EphysRecording\n\n\nephys_element.EphysRecording\n\n\n\n\n\nephys_element.EphysRecording->ephys_element.LFP\n\n\n\n\nephys_element.EphysRecording.EphysFile\n\n\nephys_element.EphysRecording.EphysFile\n\n\n\n\n\nephys_element.EphysRecording->ephys_element.EphysRecording.EphysFile\n\n\n\n\nephys_element.EphysRecording->ephys_element.ClusteringTask\n\n\n\n\nephys_element.AcquisitionSoftware\n\n\nephys_element.AcquisitionSoftware\n\n\n\n\n\nephys_element.AcquisitionSoftware->ephys_element.EphysRecording\n\n\n\n\nephys_element.ProbeInsertion\n\n\nephys_element.ProbeInsertion\n\n\n\n\n\nephys_element.ProbeInsertion->ephys_element.InsertionLocation\n\n\n\n\nephys_element.ProbeInsertion->ephys_element.EphysRecording\n\n\n\n\nephys_element.WaveformSet.Waveform\n\n\nephys_element.WaveformSet.Waveform\n\n\n\n\n\nephys_element.CuratedClustering.Unit->ephys_element.WaveformSet.Waveform\n\n\n\n\nephys_element.WaveformSet.PeakWaveform\n\n\nephys_element.WaveformSet.PeakWaveform\n\n\n\n\n\nephys_element.CuratedClustering.Unit->ephys_element.WaveformSet.PeakWaveform\n\n\n\n\nephys_element.Curation\n\n\nephys_element.Curation\n\n\n\n\n\nephys_element.CuratedClustering\n\n\nephys_element.CuratedClustering\n\n\n\n\n\nephys_element.Curation->ephys_element.CuratedClustering\n\n\n\n\nephys_element.WaveformSet\n\n\nephys_element.WaveformSet\n\n\n\n\n\nephys_element.WaveformSet->ephys_element.WaveformSet.Waveform\n\n\n\n\nephys_element.WaveformSet->ephys_element.WaveformSet.PeakWaveform\n\n\n\n\nacquisition.Session\n\n\nacquisition.Session\n\n\n\n\n\nephys.EphysSession\n\n\nephys.EphysSession\n\n\n\n\n\nacquisition.Session->ephys.EphysSession\n\n\n\n\nephys_element.CuratedClustering->ephys_element.CuratedClustering.Unit\n\n\n\n\nephys_element.CuratedClustering->ephys_element.WaveformSet\n\n\n\n\nephys_element.Clustering\n\n\nephys_element.Clustering\n\n\n\n\n\nephys_element.Clustering->ephys_element.Curation\n\n\n\n\nephys_element.ClusteringTask->ephys_element.Clustering\n\n\n\n\nephys.EphysSession->ephys_element.ProbeInsertion\n\n\n\n" + ] }, + "execution_count": 11, "metadata": {}, - "execution_count": 11 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "dj.Diagram(acquisition.Session) + ephys.EphysSession + dj.Diagram(ephys_element)" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## Ingest Probe and ProbeInsertion by ephys_element_ingest" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "The original U19 pipeline contains a table `ephys.EphysSession` where datapath to the neuropixel meta file and the kilosort output folder were stored." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 4, - "source": [ - "ephys.EphysSession() # Current sessions" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { - "text/plain": [ - "*subject_fulln *session_date *session_numbe ephys_director\n", - "+------------+ +------------+ +------------+ +------------+\n", - "hnieh_E105 2021-01-15 0 /Volumes/Tank/\n", - "ms81_M004 2021-05-07 0 /Users/ms81/pr\n", - "ms81_M005 2021-05-05 0 /M005/20210505\n", - "ms81_M005 2021-05-06 0 /M005/20210506\n", - " (Total: 4)" - ], "text/html": [ "\n", " \n", @@ -259,17 +249,31 @@ " \n", "

Total: 4

\n", " " + ], + "text/plain": [ + "*subject_fulln *session_date *session_numbe ephys_director\n", + "+------------+ +------------+ +------------+ +------------+\n", + "hnieh_E105 2021-01-15 0 /Volumes/Tank/\n", + "ms81_M004 2021-05-07 0 /Users/ms81/pr\n", + "ms81_M005 2021-05-05 0 /M005/20210505\n", + "ms81_M005 2021-05-06 0 /M005/20210506\n", + " (Total: 4)" ] }, + "execution_count": 4, "metadata": {}, - "execution_count": 4 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "ephys.EphysSession() # Current sessions" + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Do you have a new one to insert? This is how it is done:\r\n", "\r\n", @@ -282,167 +286,167 @@ "\r\n", "ephys.EphysSession.insert1(key, skip_duplicates=True) # insert1 only works for one entry\r\n", "ephys.EphysSession()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "dj.Diagram(ephys.EphysSession) + dj.Diagram(probe_element.Probe) + dj.Diagram(ephys_element.ProbeInsertion)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "A module `ephys_element_ingest` was provided to process a ephys session based on the neuropixel meta file: ingest entries into tables `Probe` and `ProbeInsertion`" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from u19_pipeline.ingest import ephys_element_ingest" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "for sess_key in ephys.EphysSession.fetch('KEY'):\r\n", " ephys_element_ingest.process_session(sess_key)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "probe_element.Probe()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.ProbeInsertion()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## Populate EphysRecording" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "By populating `ephys_element.EphysRecording`, three tables will be ingested:\n", "+ `probe_element.EelectrodeConfig` table contains the configuration information of the electrodes used, i.e. which 384 electrodes out of the total 960 on the probe were used in this ephys session\n", "+ `ephys_element.EphysRecording` table specifies which ElectrodeConfig is used in a particular ephys session\n", "+ `ephys_element.EphysRecording.EphysFile` table saves the file path of SpikeGLX meta file." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "dj.Diagram(ephys_element.ProbeInsertion) + probe_element.ElectrodeConfig\\\r\n", "+ ephys_element.EphysRecording + ephys_element.EphysRecording.EphysFile" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.EphysRecording()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.EphysRecording.populate(display_progress=True)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Here is an overview of the Electrode used in a EphysRecording for a particular probe insertion" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "probe_insertion_key = ephys_element.ProbeInsertion.fetch('KEY', limit=1)[0]\r\n", "ephys_element.EphysRecording * probe_element.ElectrodeConfig.Electrode & probe_insertion_key" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.EphysRecording.EphysFile()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Note that the filepath is a relative path to the `ephys_root_data_dir` in `dj.config['custom']`" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## Populate clustering results" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "The next major table in the ephys pipeline is the `ClusteringTask`, which is a manual table that is inserted when a Kilosort2 clustering task is finished and the clustering results are ready for processing. The `ClusteringTask` table depends on the table `ClusteringParamSet`, which are the parameters of the clustering task and needed to be inserted first. A method of the class `ClusteringParamSet` called `insert_new_params` helps on the insertion of params_set" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "dj.Diagram(ephys_element.EphysRecording) + ephys_element.ClusteringParamSet + ephys_element.ClusteringTask" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# insert clustering task manually\r\n", "params_ks = {\r\n", @@ -472,272 +476,268 @@ "ephys_element.ClusteringParamSet.insert_new_params(\r\n", " 'kilosort2', 0, 'Spike sorting using Kilosort2', params_ks)\r\n", "ephys_element.ClusteringParamSet()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "We are then able to insert an entry into the `ClusteringTask` table. One important field of the table is `clustering_output_dir`, which specifies the Kilosort2 output directory for the later processing. For the current pipeline, the directory could be reconstructed from directories stored in existing tables" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "dj.Diagram(ephys_element.EphysRecording) + ephys_element.ClusteringTask + ephys_element.Clustering" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.ClusteringTask.describe();" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Specify path and animal information\r\n", "\r\n", "ephys_key = dict(\r\n", " subject_fullname='ms81_M004', session_date='2021-05-07', session_number=0, insertion_number=0)\r\n", "ephys_dir = '210507_M004/towersTask_g0_imec0/'" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.ClusteringTask.insert1(\r\n", " dict(**ephys_key, paramset_idx=0, clustering_output_dir=ephys_dir), skip_duplicates=True)\r\n", "ephys_element.ClusteringTask()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "We are then able to populate the clustering results. The `Clustering` table now validates the Kilosort2 outcomes. In the future release of elements-ephys, this table will be used to trigger Kilosort2." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.Clustering.populate(display_progress=True)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# Curation and CuratedClustering" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "The next step in the pipeline is the curation of spike sorting results. If a manual curation was implemented, an entry needs to be manually inserted into the table `Curation`, which specifies the directory to the curated results in `curation_output_dir`. If we would like to process the Kilosort2 outcome directly, an entry is also needed in `Curation`. A method `create1_from_clustering_task` was provided to help this insertion. It copies the `clustering_output_dir` in `ClusteringTask` to the field `curation_output_dir` in the table `Curation` with a new `curation_id`." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "dj.Diagram(ephys_element.Clustering) + ephys_element.Curation + ephys_element.CuratedClustering \\\r\n", "+ ephys_element.CuratedClustering.Unit" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "key = (ephys_element.ClusteringTask & ephys_key).fetch1('KEY')\r\n", "ephys_element.Curation().create1_from_clustering_task(key)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.Curation()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Then we could populate table `CuratedClustering`, ingesting either the output of Kilosort2 or the curated results." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.CuratedClustering.populate(display_progress=True)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "The part table `CuratedClustering.Unit` contains the spike sorted units" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.CuratedClustering.Unit()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## Populate LFP and waveform" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.LFP.populate(display_progress=True)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys_element.LFP()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "The current workflow also contain tables to save spike waveforms:\n", "\n", "`WaveformSet`: a table to drive the processing of all spikes waveforms resulting from a CuratedClustering. \n", "`WaveformSet.Waveform`: mean waveform across spikes for a given unit and electrode. \n", "`WaveformSet.PeakWaveform`: mean waveform across spikes for a given unit at the electrode with peak spike amplitude." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# May take a while to populate depending on data size.\r\n", "ephys_element.WaveformSet.populate(display_progress=True)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# Synchronize ePhys and Behavior" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Populate the tables, using the nidaq file produced by spikeGLX. This is done automatically and produces a record of VR iteration numbers measured in the time of the ePhys setup." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys.BehaviorSync()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "dj.config" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "get_session_directory(key)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys.BehaviorSync.populate(key)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "ephys.BehaviorSync()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Once the synchronization files have been produces, the all Clusters/spikes can be binned into iterations." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 7, - "source": [ - "ephys.CuratedClustersIteration.populate(key)" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "\\\\bucket.pni.princeton.edu\\Bezos-center\\Manuel\\neuropixel\\manuel\n", "\n", @@ -756,32 +756,24 @@ ] } ], - "metadata": {} + "source": [ + "ephys.CuratedClustersIteration.populate(key)" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "This allows to plot neuronal responses responses. Here is an example" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 10, - "source": [ - "ephys.CuratedClustersIteration.Unit & key & 'unit = 7'" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { - "text/plain": [ - "*subject_fulln *session_date *session_numbe *insertion_num *paramset_idx *curation_id *unit spike_coun firing_rate_be firing_rate_af\n", - "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------+ +--------+ +------------+ +------------+\n", - "ms81_M005 2021-05-05 0 0 0 1 7 =BLOB= 0.321505 3.72292 \n", - "ms81_M005 2021-05-05 0 1 0 1 7 =BLOB= 6.21089 2.39695 \n", - " (Total: 2)" - ], "text/html": [ "\n", " \n", @@ -890,28 +882,40 @@ " \n", "

Total: 2

\n", " " + ], + "text/plain": [ + "*subject_fulln *session_date *session_numbe *insertion_num *paramset_idx *curation_id *unit spike_coun firing_rate_be firing_rate_af\n", + "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------+ +--------+ +------------+ +------------+\n", + "ms81_M005 2021-05-05 0 0 0 1 7 =BLOB= 0.321505 3.72292 \n", + "ms81_M005 2021-05-05 0 1 0 1 7 =BLOB= 6.21089 2.39695 \n", + " (Total: 2)" ] }, + "execution_count": 10, "metadata": {}, - "execution_count": 10 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "ephys.CuratedClustersIteration.Unit & key & 'unit = 7'" + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "key = {\"subject_fullname\": \"ms81_M004\", \"insertion_number\": 0, \"session_date\": '2021-05-07', 'curation_id': 1}\n", "\n", "ephys.BehaviorSync() & key" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 11, + "metadata": {}, + "outputs": [], "source": [ "spikes = (ephys.CuratedClustersIteration.Unit & key & 'insertion_number = 0' & 'unit = 7').fetch1('spike_counts_iteration')\n", "\n", @@ -924,29 +928,17 @@ " snip = np.zeros(len(time[trial]))*np.NaN\n", " snip[0:len(pp)] = pp\n", " position_at_each_iteration = np.append(position_at_each_iteration, snip)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 15, - "source": [ - "pl.figure(figsize=(15,5))\n", - "pl.subplot(1,2,1)\n", - "pl.plot(spikes)\n", - "pl.plot(position_at_each_iteration/50)\n", - "pl.ylim([0,10])\n", - "pl.xlim([0,30000])\n", - "\n", - "pl.subplot(1,2,2)\n", - "pl.plot(position_at_each_iteration[1:], spikes, '.')" - ], + "metadata": {}, "outputs": [ { - "output_type": "error", "ename": "ValueError", "evalue": "x and y must have same first dimension, but have shapes (470786,) and (470789,)", + "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", @@ -959,22 +951,33 @@ ] }, { - "output_type": "display_data", "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA28AAAEzCAYAAACmHDtZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABIY0lEQVR4nO3deZwkdX3/8fene46dWRb2YDncBQFZRKIcugIKMRBALhM0UYN4B+VnPKImJmK8o4nGExGUEG9jxAsVZUEU5JZjl3MXWHZZlt1l72N27unr+/ujqnt6evqo7q6uPub1fDzm0d3V1d/6VlX3dH368z3MOScAAAAAQGuLNbsCAAAAAIDKCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2kDF4M3MvmNm281sZd6y+Wb2ezNb49/Oa2w1AQBoDcW+FwueNzO73MzWmtkjZvbiqOsIAOhMQTJv35N0TsGySyXd7JxbIulm/zEAADPB9zT9ezHfuZKW+H+XSPpmBHUCAMwAFYM359ztknYXLL5A0vf9+9+X9OpwqwUAQGsq8b2Y7wJJP3CeeyTNNbODo6kdAKCT1drn7UDn3BZJ8m8PCK9KAAC0tUWSNuY93uQvAwCgLl2N3oCZXSKv2Yhmz579kqOPPrrRmwQANNmKFSt2OucWNrseTWJFlrlpK/H9CAAzUj3fkbUGb9vM7GDn3Ba/Kcj2Uis6566WdLUkLV261C1fvrzGTQIA2oWZPdPsOjTRJkmH5D1eLGlz4Up8PwLAzFTPd2StzSavk/RW//5bJf261goAANBhrpP0Fn/UyZMl7c12NQAAoB4VM29m9mNJp0na38w2SfqkpM9L+qmZXSxpg6TXNbKSAAC0ihLfi92S5Jy7StIySedJWitpVNLbm1NTAECnqRi8OefeUOKpM0KuCwAALa/M92L2eSfpPRFVBwAwg9TabBIAAAAAECGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANlBX8GZmHzSzVWa20sx+bGazwqoYAACtyMzOMbPVZrbWzC4t8vx+ZvYbM3vY/458ezPqCQDoPDUHb2a2SNI/SlrqnHuhpLikC8OqGAAArcbM4pKulHSupGMkvcHMjilY7T2SHnPOHSfpNElfNrOeSCsKAOhI9Tab7JLUZ2Zdkvolba6/SgAAtKwTJa11zq1zziUkXSPpgoJ1nKQ5ZmaS9pG0W1Iq2moCADpRzcGbc+5ZSV+StEHSFkl7nXM3Fa5nZpeY2XIzW75jx47aawoAQPMtkrQx7/Emf1m+KyS9QN4Pmo9Ker9zLlNYEN+PAIBq1dNscp68XxsPl/QcSbPN7E2F6znnrnbOLXXOLV24cGHtNQUAoPmsyDJX8PhsSQ/J+248XtIVZrbvtBfx/QgAqFI9zSbPlPS0c26Hcy4p6VpJLw+nWgAAtKRNkg7Je7xY07sMvF3Stc6zVtLTko6OqH4AgA5WT/C2QdLJZtbvt+s/Q9Lj4VQLAICWdL+kJWZ2uD8IyYWSritYZ4O870SZ2YGSni9pXaS1BAB0pK5aX+icu9fMfi7pAXkdsR+UdHVYFQMAoNU451Jm9l5Jv5M3yvJ3nHOrzOxd/vNXSfqMpO+Z2aPymll+2Dm3s2mVBgB0jJqDN0lyzn1S0idDqgsAAC3PObdM0rKCZVfl3d8s6ZVR1wsA0PnqnSoAAAAAABABgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AZaO3hbf5f0i3dKv3q39Ov3St9+pbT+zubVJ52Sfv9JaddT4Za75WHpZ2+X9qwPr8zdT0ubH5Qy6fDKBAAAANA0Xc2uQFn3/be0+gZp9kJp8FlvWWKkOXUZ2iZ9+Sjv/iEnSQueF17Z626VVl0rnffF8Mq855ve8fvYDikWD69cAAAAAE1RV+bNzOaa2c/N7Akze9zMXhZWxSRJu9dJR5wmfXDV5LLFLw11E2UNbJQuO1a69pLJwE2Sjjo73O3sekrqXyDN3j+8Mu/7b++2qye8MgEAMrNzzGy1ma01s0tLrHOamT1kZqvM7Lao6wgA6Ez1Zt6+JulG59xrzaxHUn8IdZo0slM6+DjJTDrlA9LeTVL//FA3Mc3DP5E23C391dekO78qDTzj/WVdujH8TNb2x6QDjgmvvHTSuz3kpPDKBADIzOKSrpR0lqRNku43s+ucc4/lrTNX0jckneOc22BmBzSlsgCAjlNz8GZm+0p6haS3SZJzLiEpEU61fGN7pD4/WDvr06EWXdIvL/Fuz/y0tPzbk8vfc5+08Pnhby+TkbY9Jp3wpvDK3OMHmy95e3hlAgAk6URJa51z6yTJzK6RdIGkx/LWuUjStc65DZLknNseeS0BAB2pnmaTR0jaIem7ZvagmX3LzGaHVC8pMSqlxqW+eaEVWVEqL/Zcf4d3O/8I6UNrGhO4SV5fvuSIdMDR4ZW5a613u+DI8MoEAEjSIkkb8x5v8pflO0rSPDO71cxWmNlbIqsdAKCj1RO8dUl6saRvOudOkDQiaVrbfzO7xMyWm9nyHTt2BC99bI932+hmkvn25n0f/8TPhJ39OWmfBrZ42fmkd7tgSXhl5oK3EAdVAQBIkhVZ5goed0l6iaTzJZ0t6eNmdlThi2r+fgQAzFj1BG+bJG1yzt3rP/65vGBuCufc1c65pc65pQsXLgxeejZ4izLzNrJz+rIj/qKx28xOO7B/yMFb37xoA18AmBk2STok7/FiSZuLrHOjc27EObdT0u2SjissqObvRwDAjFVznzfn3FYz22hmz3fOrZZ0hqa2+a/P2G7vti/CAGTUD94uudUbQCSdkLr7GrvNoc1SrFva58Dwyty9TppP1g0AGuB+SUvM7HBJz0q6UF4ft3y/lnSFmXVJ6pF0kqSvRlpLAEBHqne0yfdJ+pE/0uQ6SeGNkNGMzNvoLu+2f3+pq9f7i2Kb/fO9ETXDsme9dMiJ4ZUHAJAkOedSZvZeSb+TFJf0HefcKjN7l//8Vc65x83sRkmPSMpI+pZzbmXzag0A6BR1BW/OuYckLQ2nKgVG/cxblE3/ss0m+xdEt83R3eFuL5ORBjdL+y0Or0wAQI5zbpmkZQXLrip4/EVJX4yyXgCAzlfXJN0N1azMW1ef1BPudHUVtxlm8DayXcokCd4AAACADtPCwdtuqWtW4/uc5RvdLc3eP7rtZbcZZnZx7ybvdl+CNwAAAKCTtHDwtifarJvkDVgS9QiNo7vCHZQlG7ztVzjtEAAAAIB21rrB2/headbcaLc5ussbrCQqmYyXYQyz2eT4Xu826sAXAAAAQEO1ePC2b7TbHNkZ7WAl4wOSy4S7zeSod9sdYb89AAAAAA3XwsHboDRrv2i3GfbIj0G2J4W7zcSId9szO7wyAQAAADRd6wZvE4NSb4SZt9SElBiSZkcYvI01YDqE5KhkcSneE16ZAAAAAJqudYO3qJtN5ibojjLzlt1miMFbYtTLuoU56TcAAACApmvN4M256JtN5gKpCAcsaUTAmByhvxsAAADQgVozeEuOeRNNR9lscmSnd9uMzFuYUwUkRqKdZBwAAABAJFozeJsY9G6b0Wwyykm6x/dKFpN654RXZrbZJAAAAICO0prB23g2eJsb3TYbMfJjJRNDXuAWZv+05IjUTfAGAAAAdJoWDd78iaajbDY5ukuSRTu59fig1Btyv77EKM0mAQAAgA7UmsHbhB+8RdlscmJI6tlHisUj3OZg+PuYHGXAEgAAAKADtWbwlms2GeFok4nh6PuKZZtNhikxQp83AAAAoAO1aPDWhGaTzRilcXxv+PuYYKoAAAAAoBO1ZvA2MeTdhp2VKqcZGauJocY0myTzBgAAAHSc1g7eevaJbpvJkWi3J3l93sLMvGUyBG8AAABAh2rN4C0x7A8eEmH1os68OeePNhlidjE15t3SbBIAAADoOK0ZvGVHfoxS1H3FUhNSJhlus8nEqHdL5g0AAADoOK0ZvCWGpd6og7fRaAPGCX9EzTCbTSZHvFsybwAAAEDHac3gbWK4CZm3iKcKGG9A8Jbwgzcm6QYAAAA6TmsGb4nhaEealKKfKiCbeWtIs8mIA18AAAAADdeawVvUmbd0SkpP0GwSAAAAQMtqzeAt6j5v2aCnGc0mG5J5I3gDAAAAOk3rBm9RZsESTchYNWIi8qQfvHUz2iQAAADQaVozeJuIOPOWG+ijzZtNMmAJAAAA0LFaL3hLp7zJpnsiHLAk0YRmkxPD3m2YmbdmZBABAAAARKL1grdENqhpRuYt4tEmu2ZJ8e7wymxG3z0AAAAAkWjd4K0Zfd4ibTY5FP50CIlRKdYlxXvCLRcAAABA07Ve8DbRjMxbNmCMMGPViLnskqPeYCVm4ZYLAAAAoOlaL3jLBVIR9nnLjtIYaZ+3ofAzfVFPNA4AAAAgMq0XvOWG0G/GVAERD1gS5kiTkp95I3gDAAAAOlHrBW/je73bsAObcprRbHJisAF93si8AQAAAJ2q9YK3sT3ebf/86LaZGJUsLnX1RrfNiaHws4uJESboBgAAADpU6wZvffOi22ZixMu6RTnQR6MGLGGaAAAAAKAjtV7wNj7gzX/W3RfdNhPD0Qc9jZoqgGaTAAAAQEdqveBtbE+0WTcp+oxVOimlxsMfUTNJs0kAAACgUxG8SX5fsQgzVrkRNcm8AQAAAAimBYO3geYEb2HPuVZOo6ZDiDoIBQAAABCZFgzempF5i7jPW25qghCDt0xGSo0xYAkAAADQoVozeJs1N9ptRt3cMDspeJiZt+Sod0vmDQAAAOhIrRm89c2NdptRN5tsROYtG7yReQMAAAA6UmsFb8lxLwjp9GaTE9ngLcRtZrN5BG8AAABAR2qt4G18wLvt9KkCcoEWzSYBAAAABFN38GZmcTN70Mx+W3dtxvZ4t1EGb6mElE5EOz9aI5pNknkDgEiY2TlmttrM1prZpWXWe6mZpc3stVHWDwDQucLIvL1f0uMhlNOc4C3ZhKAn0cBmk2TeAKBhzCwu6UpJ50o6RtIbzOyYEuv9l6TfRVtDAEAnqyt4M7PFks6X9K1QatOM4K0ZGavEiGQxqbsvvDJzA5YQvAFAA50oaa1zbp1zLiHpGkkXFFnvfZJ+IWl7lJUDAHS2ejNvl0n6V0mZ+quiJgVvTRilMTu6pVmIZWb3I8JRMwFg5lkkaWPe403+shwzWyTpNZKuirBeAIAZoObgzcxeJWm7c25FhfUuMbPlZrZ8x44d5QttSvDWgCaMlUwMhb+9JM0mASACxX51cwWPL5P0YedcumxB1Xw/AgCg+jJvp0j6azNbL6/ZyF+a2f8WruScu9o5t9Q5t3ThwoXlSxzbI1lc6p1TR7Wq1Kxmk2FnyBI0mwSACGySdEje48WSNhess1TSNf7342slfcPMXl1YUFXfjwAAqI7gzTn3EefcYufcYZIulHSLc+5NddVmbI+XdQuzOWEluSH2ow7eQt5ebsASRpsEgAa6X9ISMzvczHrkff9dl7+Cc+5w59xh/vfjzyW92zn3q8hrCgDoOF3NrsAU2eAtSs1oNpkYDj/zlhyRYl1SV0+45QIAcpxzKTN7r7xRJOOSvuOcW2Vm7/Kfp58bAKBhQgnenHO3Srq17oKaErw1aaqAOQeHXOYoWTcAiIBzbpmkZQXLigZtzrm3RVEnAMDMEMY8b+EZ2yP1zY12m00bbbIBA5YwQTcAAADQsVoseBuQZs2NdptNGW1yuAF93kYZrAQAAADoYK0VvI3vbULmze8rFo+wr1hiROoJeUTN5CjTBAAAAAAdrHWCt0zGC94iz7z5zQ2jGuHSOX/AkgaMNkmzSQAAAKBjtU7wNjEoyUWfeUuORDvQR3JUkmtM8EbmDQAAAOhYrRO8jQ94t7P2i3a7UWessqNb9oY9VQB93gAAAIBO1jrB29iAd9usZpORbS87QErIwRtTBQAAAAAdrXWCt2zmrRlTBUQ90qTEVAEAAAAAqtJCwdte77YZUwU0o9lkIzJvNJsEAAAAOlbrBG/ZZpPNmCogyoE+GhG8ZTJSaoxmkwAAAEAHa53gLTdgydxot5sYlnpDnnOt7PaGvNsws33JUb9MMm8AAABAp2qd4G1sQLJ49P22JqIO3how2mQ2eGOqAAAAAKBjtU7wNj7gNZmMarJsyWtumBgKv/9ZOY1oNknwBgAAAHS81gnexgaibzKZbNCca+VMNKLZ5JhfJsEbAAAA0KlaJ3gb3xv9YCXZYfujbjYZ65a6ekMsk8wbAAAA0OlaKHgbaM5gJZLUE3HwFvocbwRvAAAAQKdrneBtbKAJmbdB7zbKZpOJ4fD72BG8AQAAAB2vdYK38QFp1n7RbjPbbDLSAUuGww8WmSoAAAAA6HitEbw515wBSxJN6vMWdrPJXJ+3vnDLBQAAANAyWiN4SwxLLj0zBiyZGG5gn7eI58gDAAAAEJnWCN7G93q3UWfesn3eop7nLewBUpJk3gAAAIBO1xrB29iAd1sm85ZIZfTLBzfJOVf0+Zsf36adwxPVbTfXbDLqAUvCzrz587wxYAl8yx7doqHxZLOrUdSabUNa8cyeZlcDAACg7bRG8DY+4N2WybxdccsaffAnD+uGlVunPTeWSOvi7y/XW759X3XbnRiWLBZt0NOI4C0xInXNkmKtcTrRXGu2DendP3pA//rzR5pdlaLO+urt+ttv3t3sagAAALSd1rjaz2beyow2uX3Iy6rtHZueTUj72bhndo1Ut93EsNeE0ay619UjMdKA0SbHyLohZzSRliQ9OzDW5JoAAAAgTK0RvGUzb33zot3uxFC0TSYzaa9/WiPmeQs7mwcAAACgpbRG8Dbm938JMNpksS5vpfrBVTQxFP00AVJjRptksBIAAACgo7VI8Dbg9T0rMwpjkJaNVm3zx8Rw9CNNSuFvMzFKs0lMU+tvGgAAAGhNrRG8jQ94/d2iHnBjYjjikSYbFLwlCd4wKcounAAAAIhOawRvYwN1zfFWc4JhYijizNuQd9uI0Sbp8wYAAAB0tBYJ3vYE6u8mSa72UG26xLDUu2945VXcnp95Czvb14jpBwAAAAC0lNYI3sYHAmTeSrcFq7mVWNSjTTZqwJJGTD+AthfqDx0AAABoutYI3sb2SP3zo92mc9EPWDKRbTYZ8jYnIt4PtDSr/ecMAAAAtLDWCd4CzvFWdKqAWraZGpcyqfYfsKQZQSgAAACAyDU/eMtkvAFLKgRvgaYKqGa7E8PebTP6vIXZbDI1Lrk0fd4AAACADtf84G18QJKT+hdEu91Eg5owlt3mcPjbbNT0A2h7zPMGAADQWZofvI3u9m77Iu7zlsu8RRy8dc2S4l3hlikxYAkAAADQ4VogeNvl3QbMvIWWTGjU4CHlNGI+tmwQSrNJFGCybgAAgM7S/OBtzM+89Vfo8xb2dhNN6PM20YD52Gg2CQAAAMwIzQ/ess0m6+jzVlPfnmzmLepmkz1zQi6zCRlEtAX6vAEAAHSWFgje/GaTQfu8lbsirSY914jBQypuswHNJhs18TfaFs0lAQAAOlPzg7ex3VKsS+otn5EK/YK0aZm3BvV5Y8ASAAAAoKM1P3gb3eU1mawnOqup2WSTMm9hB1n0eQMAAABmhBYI3nZHP02A5GXBumdLsXi02ww7yGpG80+0Bfq8AQAAdJbWCN6qGKyk6PVoLUm7iaHomxo2ZLTJYcniUldvuOUCAAAAaCktELztqjhNgCRZ2JMFTAxFn61KjDQg8+aXySgVAAAAQEdrfvA2vE3a56D6yqileVhiuOIgKaFKJ6X0RPjB28Qwg5UAAAAAM0DNwZuZHWJmfzSzx81slZm9v+pCMmlpfK/UH06ft6pyTxMRB2+5vmkNaDbJNAEogi5vQGOY2TlmttrM1prZpUWef6OZPeL/3W1mxzWjngCAzlNP5i0l6Z+dcy+QdLKk95jZMVWVML5XkqtqwJLQBmFIRNxsMjsqZOijTRK8YSpa0AKNY2ZxSVdKOlfSMZLeUOS772lJf+GcO1bSZyRdHW0tAQCdqubgzTm3xTn3gH9/SNLjkhZVVcjobu+2f74e2zwo55xWbd4rVyRCG0umS9fFzzEMjqeCb3tiqGLm7bHNg/rVg88qmc4okcroyW1Dwcsv1KjJtIv0o3PO6aGNA0qkMlq9tY46B7Bm25AmUqXPDcK3euuQUulMs6tRNeecHts8OG35lr1j2j2SiKQO2wbHtXN4ouw6a7YNKZFqv+MbllLnCTknSlrrnFvnnEtIukbSBfkrOOfuds7t8R/eI2lxxHUEAHSoUPq8mdlhkk6QdG+R5y4xs+VmtnzHjh1TnxzzgreHd8V03uV36J0/WKHzL79Tv3jg2Wnb+PmKTWFUdVKFvmKrNu/VeZffoQ/85CG94ep79JnfPqZXfvV2bR4Yq317UmOmCigo8661u/TqK+/Syz9/i86+7Hat3d6YAG7H0ITO+urt+sSvVjWkfEz39M4RnX3Z7frC71aXXKdVpwj43t3rdd7ld0xb/rLP3aKXfPb3kdThpP+8WUs/+4eSz28fHNdZX71dn7xuZST1aUXfvcs7T/eu29XsqrSqRZI25j3epPI/XF4s6YZiT5T9fgQAoIi6gzcz20fSLyR9wDk37eda59zVzrmlzrmlCxcunPqkn3l7ZrRHkvSHx7dJUtUZrppGoqww59qWgfHc/eXP7NH96726Dowmq99WdntSYwYsKcjmPb3T21Y2w7B9qHymoVaD496xyB4bNN4O/1w+uGFPhTVbz6oy2ZxWCTiz7+n7np657+mVm/dKkjbuqfGHqs5X7Aun+Cw2ZqfLC94+XOz5st+PAAAUUVfwZmbd8gK3Hznnrq26AD/zNtE9t55qVC+dlFLjEQ9Y0qBmk8nRymU2+MK4Ra674aPPG9BQmyQdkvd4saTNhSuZ2bGSviXpAuccaUwAQCjqGW3SJH1b0uPOua/UVMiYlz0Y796v1mrUZsLP7JUJ3kIPSBqVeSvW5y3cLZREjNA8rZKpqgbvl/ZSrO8xJEn3S1piZoebWY+kCyVdl7+CmR0q6VpJb3bOPdmEOgIAOlRXHa89RdKbJT1qZg/5y/7NObcscAmjuyWLK9FVZwas2qvCRgVSQbYZ5miTmYwfvDHa5ExBVg2NVlMz9BnEOZcys/dK+p2kuKTvOOdWmdm7/OevkvQJSQskfcP7nVMp59zSZtUZANA5ag7enHN3qt4f08d2S33zqroiLfprcLU/EE80IJCqpBHNJlNjklzFMhv9+zm/0EcvyBHnvKAWjobQFfk/Ui4rWHZV3v13SHpH1PUCAHS+UEabrNmoH7xVIZTLilzmrUyzybAvfLMBY3eIwVuj+tEFZKSBIhfkiLdq5oS3S3vh8w0AQOtpbvA2tlvqDz5Bd2gm/FHvahiwpObrmcSwF7jFQjzkJZp/Flax0QkYfqcHOg+ZWwAAWk+Tg7c9Ul91wVso1xN1NJusefuN6JuWGPVue/qnLGbAks7XjhfWrZoRxFScJwAAWleTm03uqTrzVuySteo+Gs0asCT04K25zSaz2jCOAAAAANpO85tN9s2LPouQy7xF2WxyJPwBUgIGoY0agIAuMc1Trj9Sq56XVq0XiuM3GQAAWk/zgrfkuDfBdLWZtzACvew8b2WCnsKt1L3ZxHBj5niTmp55Q/TasdkkAAAA6tO84G1st3db5WiTxVR9HZsYkuK9UldP3dsObKJ5zSYbP2AJgURUyF4hKrzVAABoPc0L3kazwVszRpscrroJY90XzYmRBmTemtB3Lw8DG6AaBJ4AAAD1aWLmbY93W3WzyRC2HaAJY+F26m822YDgLemPNtndX369BqMFX/Q45Gg03mMAALSeFmg2We1ok9MvKaq+yJgYknr3rfZV9UkMNa7ZZJOCNzIpzdDOB72d6z5z8LkGAKB1Nb/ZZFMm6R6q2Gyy8AKmrgsa5xo02uRI0Ym/o86EkXmLHse8MTiuHAMAAFpZC2TeqhuwpC2bTaYTUibVgMxbsEFQuBbrHO2cFWnnugMAALSC5mbeuvqk7r66i6p62PQaBizJqukCdKJBA4skRpgmYIap5q3eahmUVqtPMQSYHAMAAFpZEzNvAzU1mQzl+m9iqKYJuqUaL0AbNSpkIwZBQdvj4hsAAKAzNbfZZA3TBITXbLJS8BZimqBRk2knRqSeyoOVMKFz52jnwKyd6z4j8W8DAICW09xmk/31T9BdtUzGC96ibDbZ0MwbzSYBAACAmaAlMm9V9eOpd6qAib3ebZUDpdQlG7w1YrTJJg5YQialeYKc02KflWbi7QIAAFCf5mbeagig6m4BmJ0cfNbcOguqQkObTdLnbSYJEgAZYRLCwNsIAICW05zgzTkviGrGHG/Z4C3KzFtutMnmTBXQaPSpawKOORqNtxgAAC2nOcHbxKDk0rlmk/U2v6vqOjZg8DZtnrd6rmRyfd5qG+GydLkjUnflAUsadRFmtJuMXDsf8zau+ozCaQIAoHV1RbkxJ2lwPKk5o7tlklzfPA2OJZXKTI0uJpJpOeeUzjiNpzLqiU/GmM45OeeUTDv1dMVyy8oZmUhJkrrjMfWMDUiSEj1zFEtnZGZKpDJ+/Zy6YjHFTEqXKHNoPKWJVFoxMyXTGfV1x6fUJV8m45R2Tm5sSD2SRq1X8VRaPfGYRhNpzeqOK5HKKOOcuuKWW94VN/V2xUvuz56RhObNiknpCSW7+jU2ntSsrriS6YziMa9e+YYnUhpNpNTfU//pTqQyGk2k1NcTV9I/bomC7Y0mUprVFc8t747HFI95x7krZkplih+vbPndcasYpCRSmSllpPxzmd1OqfKLKXw/5S8fTaTV3xOXmSmd8d6TZt4+TaTS6u2KK5XOvn+kVNp7vrcrpuzbOh4zOeeUSGfUHYsp7ZwyzuXOcfZ4xWLePifTGaUzTrO647l9zTineGzymCTSXl0SqYx6u7wyu+NT65/9XKUz3memq+BzlEw7dcdNybRXt554TD1dMaULPo/Z7ZY6Ttnjn8o49XbFiq7j7dPU14wn07l9zO1XgHM3lkjnPp+z/XNT+HxPVyxX7+x5Kyb/fZPPuan7lN1GqfoVLi+1nnNOA6NJ9XTFNKs7Pu28FEqmM4qb95kp9blIZ5ySfl2z5zBraDzpb8d7nPHvOCfvM+yXn3ZOu0cS6u+JayyZ9o5jMp17L3bFTGPJtObM6i66fxOptH88vZ+5CAABAGiMSIO3lc/u1bGfuknH2lO6rle6+GfrdEvmpmnrff9Pz+jIA+fo479aOe25a+7fqO54TJ+74Qk98PGzNH92jy754YqS2/z6zWv05d8/mXu87GXP6BhJL7/sIe3U04HrvncsKUl63VV/mrK8r9u72PnS647Ta1+yeMpz7/vxg7r+0S36p66H9Z646ZjP3K5ylzUfOHOJLvvDGknSzf/8F3rewun92f7mG3fpgQ0DOuPwWfq2pM/fvFHfvmn6MSyshyT9+j2n6LhD5lbY0/KO+tgN05btHE7k7v9x9Xa9/bv36+iD5uiJrUOSpBccvK+W/eOpOupjN+jYxfvpkU179YO/P1GvOGrhlHLSGaejPnaD3n7KYfrkX/1ZyTrcvXanLvrWvfrZu16mlx7mZW+P/OgNevGhc/XR81+gv/3mn/S/F5+kU5fsH2ifrrl/oz5y7aO688Ona/G8yUzmv//2MX33rvX6xzOW6J/OOkqv/+8/acUzXub2mktO1oVX35O7LfSBM5fof25fp/7eLt3/0TP1v/c8o4//epWO2H+21u30+kD+6B0n6dk9Y/rXXzyiYxfvp+vee6okaclHvWP81H+epye2Dur8y+/Mlfur95wiSXp8y6Ce92/LJEmnHLlAd63dpfWfP1+S9L27vff1uh3edk7/0q3atGdU6z53fq6cL9/0pK7441q97y+P1NdvWZtbvv7z5+uEf79JXfGYJpLpXP0l6dt3Pq3PXv+4VnzsTC3Yp3fK/h7p1/nIA/bR2u3D+u37TtULF+2Xez67T/mO/viNWvsf5+YeP75lUOd+7Q5d9aaX6JwXHjRtfUl6asewzvjybbnH573oIH3jjS/JPR5PpvWCT9yY2xdJuuDKO7Xy2cGi5b3oUzdpv75u3fNvZ0iazLiv2zmS26cPnnmU3n/mEt2xZofe/O379It/eLle8tzJzP3a7cM68yu36WsXHq8Ljl+khzcO6IIr79J33/ZSnX70AVO2984frNAfHt8mSVo4p1fD4yk9/plzitZN8o7bq449WL99ZIv+5ezn6z2nHzltnZP+8w9TPoPZ/b758W26+PvLS5b9xw+dptO/dGuu/EKfvG6VPnndqinL3vKy5+ptLz9Mf/nl2/TVvztOrzlhsa5/ZIve838PlNwOAAAIT1OaTc4zrxnhgCs92MZvHtpcdPmmPWO69oFnJUnbBsclSQ9tHChZzmU3r5n6+s1euXtVXV+x/IujfNlfqW9cOf3i5/pHvWWzNaERzVKl36O/fcdkMPn4luIXmw9sGJAkrXza249RzSpbZr5s4NFIt63eIUm5wE3y9iWb+Hhkkzfa5x1rdkx7bSrjpWZ+dM+Gstu466mdkqR7nto1ZfkDGwZ0z7rdkqQ71+4MXOfr/QvXbLCT9d271kuSvnOnd17yj9/d/rbvWTe1Dlk/vX+jRhJp7RiakCRd97B3vrKBW7aO2fdI9rjkS6YzZd/bWXetnVqHXz049bOzYfeoChNP19zvHePv3b1+WnmD4yntHklMqb+k3Oduy97xknVZu937bAeptyQl05MVe2ST95qb/eCmmCe2DE15vOzRrVMejyXS015TKnCTvM/v1sHJ/SmWoPvp8o2SpDvWeO+p+9fvnvJ89rN60yqv3tn3yW1PTn+P/yFv33YMTeT+f5STDax+vmJT0edL/W/65YPPli139dbBKeUH8YM/PaPV/mf7xpXesf9DmfMFAADC1ZTgba68L/8BNX6utcJV+9JDGnG9SlaZdIzV0Q6oX+OBgqxq9mu2eRecIy548BaFUvsQ5qAm5UZTrKVfVaXXFHs+uyjobpWqc8Vt19AALdgxaFS5tQuyr7Wcq6rqUOb1pZ4qfE32cbMH8mlU/8jCYpu9nwAAzCTNCd7Myz7sKZN5CzJASC3XDH3poZqCxkoXluXqso+NBQqy8i+2Ku1bv7zgrZrMWxRKHafC3Qnjeq9YEdnt1zLATKlXlDvzpV4zbXmJQhpxeR1oOoE6gtwg5y7o0Z9ynrLll6tDhfIaOk1Chf3P7kujahB2kFR7cdk+kJNL/iL2sH7S8+86WMUz0QAAIBxNajY5pIwz7S0TRAW7QKz+6qM/Pai9ZYLGkipckZWrSX+u2WSFTeRto9Ke9ctrzjaq3gprtobpo3dWXqeUQEFEDRem1Vwc585VideEEpy6IlmdAK9r9IiUgT53AQ9AsdVaLZFT+L4o3H8rEsx464VcjxYrL/t6cxl9rfsKnRR7QufFp/f/BAAA4WlSs8lhDapfmTKbL/vruxW/WCq+7tTHXvBW/dxolZpNlrvwn23Bmk3GpmTeyu9cv99scrRdmk0WnNF6LtBzcVORd4kFyN5Mf002W1dcrIY2s9Mv8Mtvu1QZtYRhwSbyrqXc4J+74Jm3/PKzy0q/umJc2oC4NReklNj/wh8Tqvn/VFU9qiyv0qGotX6F+3vQxNO51hTHxJ6prVAAABBIU4K3BTZYtsmkVD54qeb6rLAZlddsspbgrUKzyTLPzVawZpPVxAiz/cxbkIxeVhQJjVK7EGTevMAXk2UujicTYlVk0SYrVf75KpS6wC9cp2yTzCKZt0ACvKauZpMB1g16+PPPkwXaQPmKNyLpOBmUldhmdr1ss0mb+ji0elRZXqVjUWv98n5ikiQdPXKfJGlV5rk6wdYWfQ0AAAhHU4K3hbZX21Vhkuwyz1XT96bwWq8vNVh2lMuAxVQleLPJKvq85TJvrdVsstkTMVf13ih8TYl3XbHs2GTfuoDbqGHAElfkdUGOb7DMWw0Dlvi3QQLjoMFz8cxbmTpUHOAlfIV92SrtW7WD2QSuR8iZt1oVZhZPGLpVj2Weq5+nX6EjYlt1mAUfvRIAAFSnOcGbBrTDza359fUECH3pobJ97UqplHkrZ7aNNyDz5o822WoDltRxnMLIVNQTlJRS7LxUGySWPizhX2IHOQe1ZfQaG5mHEZg2sr9f4KIrNMNtVz1KSoVNedMpPWdine7IvFBPu4MlSRfG/9iM6gEAMCNEOkl31gE2oNsyx0W+3V4l1O0SNfV5qzhgSZkrtdkBpwqoRquONlnKtGaTZQaqqBTEBcnQNFtodaujeWMgrXwQm6CWbFkt2d5WUE19Z2tMD/T+P/VaSmPXL9Yve2apZ+t+0te2qcsltdYt0l2ZF0qS3hm/Xp9PXdSgWgMAMLNFnnnr07jm2Ji2V8i8NWK0ybnyJhCudoJuKUDn/zLPzNa4hqsMsirtW79NKOViSlQRfzezRWOgqR8CllVppMdqyprymhAvvmsZTGd6Ga0ZDQTq8xa0rKJBfPP2O0hfzOnVK55pa3azyYrlVbHuC229ei2lNZlFGtj/xRpyfep1o9KgNxH48szzc3Nnxs1piRWfUBwAANQn8uDthJjXoX2wQgBVfq6nKkabzLs/17zgbaCW0SYrtGksdcE5SwnFzAUaFTJWRZ+3yWxekzuZFQg8YEmRHcwuq7Tv5fqb1dbnrfz7qXifN387AWd6KznYRcU+b9UL0sS3nnFQAv2wUsNwk0HGK6nULLIxfd6yZRd/300bbTKs7TY4iK2m/KNiGyVJb0p8RI+d/CW9JfkR/cdBl0uf3KMPLbkh12Ty7yY+Lkn6aNePwq8wAACIPng7Kfa4JGll5rDyK5YbbbKKUe/yr/X2kzecdW2TdNdmnyr6plUTvAUdBCVyJacKKP+41LKimwgQoFWTla3UDLPYLlXOmhWWUWLAkgplFAYrQfr0BRqwJFtuFW/syaqEl0XNP09BfpSp3Oct4IZLlj+9gMLRJqef29yaBXUIebTJKoO5sPr/7asRXdr1Yw242dqmeVP//5opEevLrXuve4EezByp0+IPh7JtAAAwVeTB20XxmyVJj7gjyq5XPvPmr1PlxUw281bLJN0VpwooUZXsqJBBBiypZpLu2TbeciNNSt7Fb4+SWj/rIq2fdZEO0B5J089VGJMzF5/nLXhWdvI12e2XGm2yTB1KbKdUdqaaslVhKoFSGjVmR3WZt4CjTVadeQtUbMgKRpssUcPJzFv178HgtQiuYlPvgAUu6/2IZtuEP5ebTdu/wmL+L/2X1VQTAABUIfLgbaENasz1qK7GRVVcweX/kr6fP5FsTX3earxqnF3FwCLVjGjZr/GWzLyZSa+J35l7fGvvP6lXiWAXnsVWSiWqugqu7SxVaorXwBEMm9TstbZ53prfRLfRVSibsQ248UYNYPK8zDPS2EDR5/bViE6wNUWecYorXdd2H80cLkn6bPKN3oIKh+Fn6dP02/RJdW0TAAAUF/lok0kX13fT51RcL0jTqZqbTdbQ563Wi8bZGpMUtNnk5P1KmYvZFv4IlmEweReSWf02odWz3qbM1+Zr/azdkys+5P89/3xv0IMtD2nf7n6tnzXqPf+NP5OWnCnd9TWpu1965WelB34gHXW2Dho8WufHHtRRu9ZLj62WLKazYg8oI9MhO3fqtNhGHbl3q7R2q3fiLOb9DW3zHnfNktIJKZ2Uuvt0zNhmxWNbddAzGyV3gBTvkeLd+vPYI3pX/DfakD5CWjWkM2KPKKkupRTX6x78Dx3bHdPYtvP0pvh2xZRRTE4xOZkymp2JazyeVExOuuMJ/fXgBh0fH/bWMadDbZue+/RB2pro04u6BhVXRrrxLsml9c3uFXo0c4Rm//DLeknXofpo14S2uXk60PZo4cMP6x3xTUopnvubrXHvAv2upyQzvTGzRkPxjPf5uGej3hp/zLt/32b/JJlenXpSu+Mp9SqusXjGr7VJD+7Ra+OPyDmTkz8w/J/WS939OmNsvZ4bG9e8ddul4f2keLeUSUsrf67relZqwO2jWzInaI5GderjaSnzZ1J3nxTv1hvjTyipuBKuW2PqUa+SurznSumL0ue6TteIZun0Wz6l/+3u1sDOE6UbfyZtWyktOWvy/FlMi7cN601xr/9VRjGvzit2+s+b4mmn18RWefV+eEga3Kw3xjcqrrS6lVaX0l42/OYVUnef3h9/XDHLSH9YLrm0Dhga1R96rteP0mfoINutZ93+Gs4cKD2R0NE71um82HYdvW2NdMtPpTkHSRbXIZsH9Ob4er1gcLb0p/t19DM79fXuu9S37SjpzudJmaR3nDIpXdr1hP42fru2u7na5fbVoPql394subTkMlImk3c/ra93b1JMGR1lz2pJ8lnpvz4o7btIsrhXbjqph3tHtZ95n5uNmYXSVZ+T0il9ZO+gvjLLG1DknInPa41bpLTiuY9g39gW/b7nX/THzPH6z9RFKhWVpRXTU5mD9a30+bnPuFT+/+96d1CZZwEAQK0iDd66lFa3pbXZLai4brlfwGv9ZXuuDSujuIbVV3nlwm1WeL5UfWfbhKSgzSYLe8+U1qcJ7XT7VSwzambeVBBjrkefSL1NX+y+WpIUG9td/AWrr598bXJ0cvn2Vd6fJCVHpev/ybu/5SG9XtLreyQ96f9J+p8e/3UPSWf1SHrK/wvgnySpR9J9/p/vh36Zp6RXST/7jb7dk/eiIenAuKSnH9XZ3UUKdZKyy2+WXqe8x1nbpIxMqXjMu6h+oEeKxXVufK/Ojd8vbZEO0wq9M/9Tev8yfazY9iTp997NB/O3daP06ez9ZZOrfih/nfzyfi19qbD833k375a843T79E0f6+fwXxF/1LuzWdLmX+ae/49SdZb0hi5/XrBB6dS4pF2rpF3+k0/fNmXdoyR9trCs30ze7ZX01ex5+mWZbd/h3Xww+9zdXVKsS/NT45ofkz4Z++HkuhlJ10jnSTqvR9Ia/8/3Akmf6Za0W9LvpBMknRCXtOMe6Q9527SYLo6bui2thbZ3cvlja71gLBb3g9C4FPOC1WNsTBnFNMcPznTs33nrSFLMq/Mv731WTqZ5NqRupXTIvgukeJc2TozJJoa0vw3qxt5LJUkPZo7UOneQFttOnXTTE1JMWhJ7Vpd0Xa8zJr6op9yiaYdqfxvUTk3+r5lsmlz6v9TazPRyAABA/SIP3iRpR4CgI1hgVnmlKaNNalhjXfuqcig2Xc193qoYsGTKFgKMNrlBB1Yss4oiQ3OYbdMz7kD9LH2afpY+TXM0qjs+cpaO/9xduXXetnShPnWik/Y9WFp3m7TklRoYHtFfXX6rDooN6mevP0g6+Dhp7nO9i9rRXdLABmn7Y1r2dEZXPjChv3np4br4lOdKzum8r90mk9O7/vwwfeuOp3T+iw7UJace7u21y0z+ZdJS/4Jcdk3JMX3h1/fptvWj+ui5R+nlRx/iZeTSSf3tlbfpiNgWjfUfoivecYZeddkt6lFKsyyhd7yoS//1SL9edfKx+t6fnlFG5ufeTE6mOX092jOWUkYxrf7sefr776/QrWt2eZkimeJK652vOFIb9oxp2aNbJUnrP+1lNo649DdabDt03T+cqPs2jOgTv12tfptQt1L6yrv+Rn/3zdu9H0L8TNKEujWuHq361FkySad/6Y/aNTwhk9PDnzhLJ/z772SSHvjYmf7xcHrNN+7Us3vGNKc3rpGJpExSTBnd9eHTdeoXbvH3QjI53fah06TuPv3DD+/Vqk179PXXv0jHLdrHO06pcam7T6d+bYVSLiaTtFtz9MnT99dFf7nUz3Cm9NLP3KBupdVrSfUqqTka1cG2S//+1nN0yne3K6W4vnThS/Xha+7VGS88VF9/7VFS0itbLpOr951rtun9P37Qy2j6mc57P3K69wF0GY0lUzr7q7cppoxu/efTJJfWS798v1KKK624kop75+Qz50ixLh3+0RvlFNP6z3vHfvXWIb3xst+oR0nN9vurLpglXfP24/ST5Rv1rft368KXHamLTz3cC6LiPbpj7W69/6eP6uVHLtQVb1yqXzy4WZ++bqX++vhF+uzfnJALshSLacmlkz9WZMcTXf+p84t+jlzG6Yx/m4y4D5jTq/v+5sxp633qruunPD7/Iq+8H/30Yf1i+ya9OX6Tzot5v0ocYHt0bux+9duEMhbXtalT9Nq4F43/ffxGfTR18bTy99derXaLi9axlHsyL6hqfQAAEEykwVu278XeGkZ7zFdN6JUfsMy1YY11zaltm1WOLpi1j3nNJgM1caxix/ptQiOZVmw2aTrMtuop95zcsiH1y3X1T1kvEe+TDn2R9+DFb5YkpW1CG92B2uIOko47b2rB+z7H+zv0ZD09tFar3Gr9ed8R0oFHS5Iec89IknbP/TM97EzHzX6udOgLA9X5qf5xrXLbNDj/RdIBB+eWr3BbtSL9fC2K90kH/plWuvXeE0469eDna/XDq3V6zwLt0sC0MmPq1piS3oOuXmViXcrkdTFNy8uyFOvzllFMG9yByiw4SqO7dmirdubeyJn4LA2rf9prJMn1zJHFTCM2W4PZj3b/fO3Rvt79fRbm1t0dm6/tGtWYujSk1GQh856rTe6AqQUveJ4kaVf8IG1wPRrf7wjpgKnZ801uw5THw33P8QKvbi/LvUPz/ErmV1j6xKKTNOKnpywW06hmKWNxadZ+3l+B1KyUdqlg+X6TgUVmIqUNzv9RY/8j/W2vnVZOtl6uoNuvk5vMMvl13Wnd0qEna+uTC7TGPam9sxZL8ycHXEr2x7Vb+2o4vq/UN1fp7iENarbG4/tIPcXPlSeaPoQ/TL9SP0y/smCp05dfd7w+9LOH9bv0Uv1Pz1c0rp6ir19gg9qVycu8FZZU5J/fVlVuXQEAAKoX6YAlXcpIkvYEGO0xrLmkpmXe4rU1Nay12WS/vGaTw0GaTQYoL8ub560FR5t0aR1q27TeTc0KTpsqoNhok1Vuq9wxqmWwiNLzvNVSVv15zqJz4ZWdDt5TVX1r2bcg6wTc/eLvg6hyxEW2HeB9Wap2jR5dMtziLVfe7zNL9VTmYB1o05s2dymleTasXW7f6fVr3mkCAGDGijR4y2beBmoYqj9fNaPe5a8710b8ZpON3Wa+xow26Vp2tMn9ElvVa6lpAxYECWaCXgyWO0w1jaJYabTJIk9PTtgcrNIlt1Chvo3YnwCbDfFF1RQfoN4NHm6ylvfWtHo3aLTJalVzqLa6+TrI9kxbPk9DkqRdmvy/2QKDjgIAMGNFHLx5mbcgk2QHmuetyu17zSZrzLzVOtqkjSnp4koEaKEadJLuXiXVZZmmzfN2lG3U6t636uzYfdOeWzjijeTwRObQKcsDZWtCyC1Umour6GsqzC9WNqgIHHAWL6PCNG/Tl5XZXjZADvJerSUIqmqet4BlTpmkO0DQ04y4YXL+tqmPS6nlPVhNPYKq5lht1XwdVCTztr8NStKUwZGq/eECAACEJ/Lgbcz1aKJE34p85TI11Yw2WdhscrzmzFv550sPWDLhZ8iCZBXyyiuzXp/fFLNZUwW8M369ei2p/+65TF/o+m/1KpF77rl775ekKX3epGLHp/Z2k7lgqtj6dU3SXf75oq8pUWbh8lJnv1wQ5Vx1QVau2WSAdXPr1HKcArwo8PHPWy9IcNTwed6KvS0LguJp+1/wXrBKb47aaxduaXk7u8XN14Hao5j/A1vWfD942+0m+wo3ah47AABQWeSjTe5RbcFTMdX0K+pWSvvYuMa6a8u8VRxtssTy2TU2byy3a/tY8BEsG2GR7czdf33XbXp9123Sp94mzT1Uxw94A1cMFQyqUXjBW0+ftyDXxrVcV5YKSmqKF0K4sHX+iI9Bi81liBo/k3WAVYIdgKLZxRbL6FTqrzkZdBa8x+vdbmF5DexTt9XNV7eltUB7JweXkXSwn43blressD6tdbYAAOhsdWXezOwcM1ttZmvN7NJK68cto70B+7uVbzqVbbYT3Hx5vyCPdU2/CAmiYn+0EpWZbWMaDTBYSeE2yl3AZif+HnbVz1cXhkW2U79Ov1z/L/GBos+/O/GP0xcW7E7xDEd19SgWvFfTtG/yNeWzdcWCocoD2BSWUWrbVRSi8j9YVBX01NGXrlEDlgRrNtmEPm8FQXGlXSsVzNWr2tIqxvB5BW5x8yVNBmtZh9h2pZ1ps9t/stzcy2du2Fbpu888l/vPP2JmL25GPQEAnafmzJuZxSVdKeksSZsk3W9m1znnHiv1mrjSgUaalCpcGFTTbMdfd39/UtyR7vmBtl+imJJKTtKdazZZWSwvlC6becsGb1VONh7GZW9MGT3Hdum3mZP1u8yJOmz8/yQpN0/WV3//pJbdvGba66aPnFfdSIr5ygVokxes1Qcypfu8lVbqAn368iIBoJW/wHaqrpngZOat8rq1vBca0Vxu6jmvHBw1Y7CMipm3gqa6jWo1WW0wWCnQzT/2u/w+bfNtMFfxg7RLZ8Qe1BYtUDL/q2KGN5sM+N13rqQl/t9Jkr7p3wIAUJd6Mm8nSlrrnFvnnEtIukbSBeVe0KVMoMFKKqllJPSFfvA2WmPwVmvk02/jGgmYeQuaVZhjzcu8Hag96ra0NrmFRZ+P4uK67IiANZyoSq8o1+ct8DZqybypxv1p1IAljR5tskFBZ1iCnsN2HI1xt7w+bQv80SUl6Z5Z79MLY+un9GmVGp/9bANBvvsukPQD57lH0lwzO7iwIAAAqmW1Nu0xs9dKOsc59w7/8ZslneSce2+p1xz/nB73r39/vv4t9Y6atlnM7J64RhLpKY/zZZ/729jt+nLPVfrzia9qY8EcZGHWpXDby3o+omfd/npn8p/rLi9b5qtif9IVPV/XmRNf0Fq3uMgrqyuzGkcnH9Mvej+ttyQ+rNszx00rN/9c5OvrjmssOfW5wrpknHLrlKtnsfNdbLtB97XU+6dweal9qyToawv3ZVZ3TOPJTLmXTNHfE5epdL1L7VuluhY7xqU+Z8VeV257pfax1LmrtB0nabRgf8u9pvDYBD0uQd8n1R6nfPn7Um7dIOerkn00qpWz3qH/SF6k/0m/SpK0ftZFkqQrU3+tL6YuLPq6cu/tZ/7rVSucc0sDV6JNBPnuM7PfSvq8c+5O//HNkj7snFteqtylS5e65ctLPg0A6CBmVvN3ZD3B2+sknV3wBXaic+59BetdIukS/+ELJa2saYPtb39JOyuu1blm8v7P5H2XZvb+z+R9f75zecNUdogg331mdr2kzxUEb//qnFtRUBbfj/WZyZ+vWnC8qsPxqg7Hqzo1f0fWM9rkJkmH5D1eLGlz4UrOuaslXS1JZra8E3+JDWIm77s0s/d/Ju+7NLP3f6bve7Pr0CBBvvv4fowAx6w6HK/qcLyqw/GqTj3fkfX0ebtf0hIzO9zMeiRdKOm6OsoDAKDVBfnuu07SW/xRJ0+WtNc5tyXqigIAOk/NmTfnXMrM3ivpd5Likr7jnFsVWs0AAGgxpb77zOxd/vNXSVom6TxJayWNSnp7s+oLAOgsdU3S7ZxbJu9LKqir69lem5vJ+y7N7P2fyfsuzez9Z987ULHvPj9oy953kt5TZbEde7waiGNWHY5XdThe1eF4Vafm41XzgCUAAAAAgOjU0+cNAAAAABCRSII3MzvHzFab2VozuzSKbUbBzNab2aNm9lB21Bgzm29mvzezNf7tvLz1P+Ifg9Vmdnbe8pf45aw1s8utlhmUI2Bm3zGz7Wa2Mm9ZaPtrZr1m9hN/+b1mdlikO1hGiX3/lJk965//h8zsvLznOmnfDzGzP5rZ42a2ysze7y+fKee+1P53/Pk3s1lmdp+ZPezv+6f95TPi3DeCVfg+NM/l/vOPmNmLm1HPVhHgeL3RP06PmNndZnZcsXJmikrHK2+9l5pZ2rx5C2esIMfLzE7z/8evMrPboq5jKwnwedzPzH6T950xo/v7WpFrx4Lna/t/75xr6J+8Dt1PSTpCUo+khyUd0+jtRvEnab2k/QuWfUHSpf79SyX9l3//GH/feyUd7h+TuP/cfZJeJskk3SDp3GbvW4n9fYWkF0ta2Yj9lfRuSVf59y+U9JNm73OFff+UpA8VWbfT9v1gSS/278+R9KS/jzPl3Jfa/44//3499/Hvd0u6V9LJM+XcN+B4Vvw+lDfQyQ3+cTpZ0r3NrneLH6+XS5rn3z+X41X5estf7xZ5/TZf2+x6t/LxkjRX0mOSDvUfH9Dserf48fq3vO+DhZJ2S+ppdt2beMymXTsWPF/T//soMm8nSlrrnFvnnEtIukbSBRFst1kukPR9//73Jb06b/k1zrkJ59zT8kYhO9HMDpa0r3PuT847kz/Ie01Lcc7dLu+DmC/M/c0v6+eSzsj+Ot9sJfa9lE7b9y3OuQf8+0OSHpe0SDPn3Jfa/1I6Zv+dZ9h/2O3/Oc2Qc98AQb4PL5D0A//Y3yNprn/8ZqKKx8s5d7dzbo//8B55c+rNVEGvt94n6ReStkdZuRYU5HhdJOla59wGSXLOzeRjFuR4OUlz/P/h+8i7bkpFW83WEeDasab/91EEb4skbcx7vEnlL3zaiZN0k5mtMLNL/GUHOn8+H//2AH95qeOwyL9fuLxdhLm/udc451KS9kpa0LCah+O9fqr7O3lNxzp23/0mbSfIy8DMuHNfsP/SDDj/ZhY3s4fkXej93jk3I899SIJ8H3byd2a1qj0WF8v7FXumqni8zGyRpNdIukoI8v46StI8M7vVv9Z7S2S1az1BjtcVkl4gabOkRyW93zmXiaZ6bamm//dRBG/FfkHtlCEuT3HOvVheU433mNkryqxb6jh06vGpZX/b7Vh8U9LzJB0vaYukL/vLO3LfzWwfeb/WfsA5N1hu1SLLOnH/Z8T5d86lnXPHy8tonGhmLyyzekftewME2deZdDwqCXwszOx0ecHbhxtao9YW5HhdJunDzrl046vT8oIcry5JL5F0vqSzJX3czI5qdMVaVJDjdbakhyQ9R9534xVmtm9jq9XWavp/H0XwtknSIXmPF8uLyNuec26zf7td0i/lpZS3ZVOe/m02xV7qOGzS1GYe7XZ8wtzf3GvMrEvSfgreVDFyzrlt/oVtRtL/yDv/Ugfuu5l1ywtcfuScu9ZfPGPOfbH9n0nnX5KccwOSbpV0jmbQuQ9ZkO/Djv3OrEGgY2Fmx0r6lqQLnHO7IqpbKwpyvJZKusbM1kt6raRvmNmrI6ld6wn6ebzROTfinNsp6XZJM3VQnCDH6+3ympk659xaSU9LOjqi+rWjmv7fRxG83S9piZkdbmY98jqkXxfBdhvKzGab2ZzsfUmvlLRS3r691V/trZJ+7d+/TtKF5o2sdrikJZLu85scDZnZyX4b4bfkvaYdhLm/+WW9VtItfv+YllTQLvk18s6/1GH77tf125Ied859Je+pGXHuS+3/TDj/ZrbQzOb69/sknSnpCc2Qc98AQb4Pr5P0Fn8UspMl7c02UZ2BKh4vMztU0rWS3uyce7IJdWwlFY+Xc+5w59xhzrnD5PUxfbdz7leR17Q1BPk8/lrSn5tZl5n1SzpJXr/nmSjI8dog6QxJMrMDJT1f0rpIa9leavt/76IZbeU8eSO0PSXpo1FsM4J9OkLeSDsPS1qV3S95fTVulrTGv52f95qP+sdgtfJGlJT3S9hK/7kr5E+e3mp/kn4sr3lYUt6vBReHub+SZkn6mbxBDu6TdESz97nCvv9QXpvuR/wP4MEduu+nykvjPyKvOcRD/md6ppz7Uvvf8edf0rGSHvT3caWkT/jLZ8S5b9AxnfZ9KOldkt7l3zdJV/rPPyppabPr3OLH61uS9uR9Npc3u86tfLwK1v2eZvBok0GPl6R/kTfi5Ep5zeabXu9WPV7ymkve5P/vWinpTc2uc5OPV7Frx7r/32e/PAEAAAAALSySSboBAAAAAPUheAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA38f1tmrwdPSF2lAAAAAElFTkSuQmCC", "text/plain": [ "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA28AAAEzCAYAAACmHDtZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABIY0lEQVR4nO3deZwkdX3/8fene46dWRb2YDncBQFZRKIcugIKMRBALhM0UYN4B+VnPKImJmK8o4nGExGUEG9jxAsVZUEU5JZjl3MXWHZZlt1l72N27unr+/ujqnt6evqo7q6uPub1fDzm0d3V1d/6VlX3dH368z3MOScAAAAAQGuLNbsCAAAAAIDKCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2kDF4M3MvmNm281sZd6y+Wb2ezNb49/Oa2w1AQBoDcW+FwueNzO73MzWmtkjZvbiqOsIAOhMQTJv35N0TsGySyXd7JxbIulm/zEAADPB9zT9ezHfuZKW+H+XSPpmBHUCAMwAFYM359ztknYXLL5A0vf9+9+X9OpwqwUAQGsq8b2Y7wJJP3CeeyTNNbODo6kdAKCT1drn7UDn3BZJ8m8PCK9KAAC0tUWSNuY93uQvAwCgLl2N3oCZXSKv2Yhmz579kqOPPrrRmwQANNmKFSt2OucWNrseTWJFlrlpK/H9CAAzUj3fkbUGb9vM7GDn3Ba/Kcj2Uis6566WdLUkLV261C1fvrzGTQIA2oWZPdPsOjTRJkmH5D1eLGlz4Up8PwLAzFTPd2StzSavk/RW//5bJf261goAANBhrpP0Fn/UyZMl7c12NQAAoB4VM29m9mNJp0na38w2SfqkpM9L+qmZXSxpg6TXNbKSAAC0ihLfi92S5Jy7StIySedJWitpVNLbm1NTAECnqRi8OefeUOKpM0KuCwAALa/M92L2eSfpPRFVBwAwg9TabBIAAAAAECGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANlBX8GZmHzSzVWa20sx+bGazwqoYAACtyMzOMbPVZrbWzC4t8vx+ZvYbM3vY/458ezPqCQDoPDUHb2a2SNI/SlrqnHuhpLikC8OqGAAArcbM4pKulHSupGMkvcHMjilY7T2SHnPOHSfpNElfNrOeSCsKAOhI9Tab7JLUZ2Zdkvolba6/SgAAtKwTJa11zq1zziUkXSPpgoJ1nKQ5ZmaS9pG0W1Iq2moCADpRzcGbc+5ZSV+StEHSFkl7nXM3Fa5nZpeY2XIzW75jx47aawoAQPMtkrQx7/Emf1m+KyS9QN4Pmo9Ker9zLlNYEN+PAIBq1dNscp68XxsPl/QcSbPN7E2F6znnrnbOLXXOLV24cGHtNQUAoPmsyDJX8PhsSQ/J+248XtIVZrbvtBfx/QgAqFI9zSbPlPS0c26Hcy4p6VpJLw+nWgAAtKRNkg7Je7xY07sMvF3Stc6zVtLTko6OqH4AgA5WT/C2QdLJZtbvt+s/Q9Lj4VQLAICWdL+kJWZ2uD8IyYWSritYZ4O870SZ2YGSni9pXaS1BAB0pK5aX+icu9fMfi7pAXkdsR+UdHVYFQMAoNU451Jm9l5Jv5M3yvJ3nHOrzOxd/vNXSfqMpO+Z2aPymll+2Dm3s2mVBgB0jJqDN0lyzn1S0idDqgsAAC3PObdM0rKCZVfl3d8s6ZVR1wsA0PnqnSoAAAAAABABgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AYI3gAAAACgDRC8AQAAAEAbIHgDAAAAgDZA8AYAAAAAbYDgDQAAAADaAMEbAAAAALQBgjcAAAAAaAMEbwAAAADQBgjeAAAAAKANELwBAAAAQBsgeAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA0QvAEAAABAGyB4AwAAAIA2QPAGAAAAAG2A4A0AAAAA2gDBGwAAAAC0AYI3AAAAAGgDBG8AAAAA0AZaO3hbf5f0i3dKv3q39Ov3St9+pbT+zubVJ52Sfv9JaddT4Za75WHpZ2+X9qwPr8zdT0ubH5Qy6fDKBAAAANA0Xc2uQFn3/be0+gZp9kJp8FlvWWKkOXUZ2iZ9+Sjv/iEnSQueF17Z626VVl0rnffF8Mq855ve8fvYDikWD69cAAAAAE1RV+bNzOaa2c/N7Akze9zMXhZWxSRJu9dJR5wmfXDV5LLFLw11E2UNbJQuO1a69pLJwE2Sjjo73O3sekrqXyDN3j+8Mu/7b++2qye8MgEAMrNzzGy1ma01s0tLrHOamT1kZqvM7Lao6wgA6Ez1Zt6+JulG59xrzaxHUn8IdZo0slM6+DjJTDrlA9LeTVL//FA3Mc3DP5E23C391dekO78qDTzj/WVdujH8TNb2x6QDjgmvvHTSuz3kpPDKBADIzOKSrpR0lqRNku43s+ucc4/lrTNX0jckneOc22BmBzSlsgCAjlNz8GZm+0p6haS3SZJzLiEpEU61fGN7pD4/WDvr06EWXdIvL/Fuz/y0tPzbk8vfc5+08Pnhby+TkbY9Jp3wpvDK3OMHmy95e3hlAgAk6URJa51z6yTJzK6RdIGkx/LWuUjStc65DZLknNseeS0BAB2pnmaTR0jaIem7ZvagmX3LzGaHVC8pMSqlxqW+eaEVWVEqL/Zcf4d3O/8I6UNrGhO4SV5fvuSIdMDR4ZW5a613u+DI8MoEAEjSIkkb8x5v8pflO0rSPDO71cxWmNlbIqsdAKCj1RO8dUl6saRvOudOkDQiaVrbfzO7xMyWm9nyHTt2BC99bI932+hmkvn25n0f/8TPhJ39OWmfBrZ42fmkd7tgSXhl5oK3EAdVAQBIkhVZ5goed0l6iaTzJZ0t6eNmdlThi2r+fgQAzFj1BG+bJG1yzt3rP/65vGBuCufc1c65pc65pQsXLgxeejZ4izLzNrJz+rIj/qKx28xOO7B/yMFb37xoA18AmBk2STok7/FiSZuLrHOjc27EObdT0u2SjissqObvRwDAjFVznzfn3FYz22hmz3fOrZZ0hqa2+a/P2G7vti/CAGTUD94uudUbQCSdkLr7GrvNoc1SrFva58Dwyty9TppP1g0AGuB+SUvM7HBJz0q6UF4ft3y/lnSFmXVJ6pF0kqSvRlpLAEBHqne0yfdJ+pE/0uQ6SeGNkNGMzNvoLu+2f3+pq9f7i2Kb/fO9ETXDsme9dMiJ4ZUHAJAkOedSZvZeSb+TFJf0HefcKjN7l//8Vc65x83sRkmPSMpI+pZzbmXzag0A6BR1BW/OuYckLQ2nKgVG/cxblE3/ss0m+xdEt83R3eFuL5ORBjdL+y0Or0wAQI5zbpmkZQXLrip4/EVJX4yyXgCAzlfXJN0N1azMW1ef1BPudHUVtxlm8DayXcokCd4AAACADtPCwdtuqWtW4/uc5RvdLc3eP7rtZbcZZnZx7ybvdl+CNwAAAKCTtHDwtifarJvkDVgS9QiNo7vCHZQlG7ztVzjtEAAAAIB21rrB2/headbcaLc5ussbrCQqmYyXYQyz2eT4Xu826sAXAAAAQEO1ePC2b7TbHNkZ7WAl4wOSy4S7zeSod9sdYb89AAAAAA3XwsHboDRrv2i3GfbIj0G2J4W7zcSId9szO7wyAQAAADRd6wZvE4NSb4SZt9SElBiSZkcYvI01YDqE5KhkcSneE16ZAAAAAJqudYO3qJtN5ibojjLzlt1miMFbYtTLuoU56TcAAACApmvN4M256JtN5gKpCAcsaUTAmByhvxsAAADQgVozeEuOeRNNR9lscmSnd9uMzFuYUwUkRqKdZBwAAABAJFozeJsY9G6b0Wwyykm6x/dKFpN654RXZrbZJAAAAICO0prB23g2eJsb3TYbMfJjJRNDXuAWZv+05IjUTfAGAAAAdJoWDd78iaajbDY5ukuSRTu59fig1Btyv77EKM0mAQAAgA7UmsHbhB+8RdlscmJI6tlHisUj3OZg+PuYHGXAEgAAAKADtWbwlms2GeFok4nh6PuKZZtNhikxQp83AAAAoAO1aPDWhGaTzRilcXxv+PuYYKoAAAAAoBO1ZvA2MeTdhp2VKqcZGauJocY0myTzBgAAAHSc1g7eevaJbpvJkWi3J3l93sLMvGUyBG8AAABAh2rN4C0x7A8eEmH1os68OeePNhlidjE15t3SbBIAAADoOK0ZvGVHfoxS1H3FUhNSJhlus8nEqHdL5g0AAADoOK0ZvCWGpd6og7fRaAPGCX9EzTCbTSZHvFsybwAAAEDHac3gbWK4CZm3iKcKGG9A8Jbwgzcm6QYAAAA6TmsGb4nhaEealKKfKiCbeWtIs8mIA18AAAAADdeawVvUmbd0SkpP0GwSAAAAQMtqzeAt6j5v2aCnGc0mG5J5I3gDAAAAOk3rBm9RZsESTchYNWIi8qQfvHUz2iQAAADQaVozeJuIOPOWG+ijzZtNMmAJAAAA0LFaL3hLp7zJpnsiHLAk0YRmkxPD3m2YmbdmZBABAAAARKL1grdENqhpRuYt4tEmu2ZJ8e7wymxG3z0AAAAAkWjd4K0Zfd4ibTY5FP50CIlRKdYlxXvCLRcAAABA07Ve8DbRjMxbNmCMMGPViLnskqPeYCVm4ZYLAAAAoOlaL3jLBVIR9nnLjtIYaZ+3ofAzfVFPNA4AAAAgMq0XvOWG0G/GVAERD1gS5kiTkp95I3gDAAAAOlHrBW/je73bsAObcprRbHJisAF93si8AQAAAJ2q9YK3sT3ebf/86LaZGJUsLnX1RrfNiaHws4uJESboBgAAADpU6wZvffOi22ZixMu6RTnQR6MGLGGaAAAAAKAjtV7wNj7gzX/W3RfdNhPD0Qc9jZoqgGaTAAAAQEdqveBtbE+0WTcp+oxVOimlxsMfUTNJs0kAAACgUxG8SX5fsQgzVrkRNcm8AQAAAAimBYO3geYEb2HPuVZOo6ZDiDoIBQAAABCZFgzempF5i7jPW25qghCDt0xGSo0xYAkAAADQoVozeJs1N9ptRt3cMDspeJiZt+Sod0vmDQAAAOhIrRm89c2NdptRN5tsROYtG7yReQMAAAA6UmsFb8lxLwjp9GaTE9ngLcRtZrN5BG8AAABAR2qt4G18wLvt9KkCcoEWzSYBAAAABFN38GZmcTN70Mx+W3dtxvZ4t1EGb6mElE5EOz9aI5pNknkDgEiY2TlmttrM1prZpWXWe6mZpc3stVHWDwDQucLIvL1f0uMhlNOc4C3ZhKAn0cBmk2TeAKBhzCwu6UpJ50o6RtIbzOyYEuv9l6TfRVtDAEAnqyt4M7PFks6X9K1QatOM4K0ZGavEiGQxqbsvvDJzA5YQvAFAA50oaa1zbp1zLiHpGkkXFFnvfZJ+IWl7lJUDAHS2ejNvl0n6V0mZ+quiJgVvTRilMTu6pVmIZWb3I8JRMwFg5lkkaWPe403+shwzWyTpNZKuirBeAIAZoObgzcxeJWm7c25FhfUuMbPlZrZ8x44d5QttSvDWgCaMlUwMhb+9JM0mASACxX51cwWPL5P0YedcumxB1Xw/AgCg+jJvp0j6azNbL6/ZyF+a2f8WruScu9o5t9Q5t3ThwoXlSxzbI1lc6p1TR7Wq1Kxmk2FnyBI0mwSACGySdEje48WSNhess1TSNf7342slfcPMXl1YUFXfjwAAqI7gzTn3EefcYufcYZIulHSLc+5NddVmbI+XdQuzOWEluSH2ow7eQt5ebsASRpsEgAa6X9ISMzvczHrkff9dl7+Cc+5w59xh/vfjzyW92zn3q8hrCgDoOF3NrsAU2eAtSs1oNpkYDj/zlhyRYl1SV0+45QIAcpxzKTN7r7xRJOOSvuOcW2Vm7/Kfp58bAKBhQgnenHO3Srq17oKaErw1aaqAOQeHXOYoWTcAiIBzbpmkZQXLigZtzrm3RVEnAMDMEMY8b+EZ2yP1zY12m00bbbIBA5YwQTcAAADQsVoseBuQZs2NdptNGW1yuAF93kYZrAQAAADoYK0VvI3vbULmze8rFo+wr1hiROoJeUTN5CjTBAAAAAAdrHWCt0zGC94iz7z5zQ2jGuHSOX/AkgaMNkmzSQAAAKBjtU7wNjEoyUWfeUuORDvQR3JUkmtM8EbmDQAAAOhYrRO8jQ94t7P2i3a7UWessqNb9oY9VQB93gAAAIBO1jrB29iAd9usZpORbS87QErIwRtTBQAAAAAdrXWCt2zmrRlTBUQ90qTEVAEAAAAAqtJCwdte77YZUwU0o9lkIzJvNJsEAAAAOlbrBG/ZZpPNmCogyoE+GhG8ZTJSaoxmkwAAAEAHa53gLTdgydxot5sYlnpDnnOt7PaGvNsws33JUb9MMm8AAABAp2qd4G1sQLJ49P22JqIO3how2mQ2eGOqAAAAAKBjtU7wNj7gNZmMarJsyWtumBgKv/9ZOY1oNknwBgAAAHS81gnexgaibzKZbNCca+VMNKLZ5JhfJsEbAAAA0KlaJ3gb3xv9YCXZYfujbjYZ65a6ekMsk8wbAAAA0OlaKHgbaM5gJZLUE3HwFvocbwRvAAAAQKdrneBtbKAJmbdB7zbKZpOJ4fD72BG8AQAAAB2vdYK38QFp1n7RbjPbbDLSAUuGww8WmSoAAAAA6HitEbw515wBSxJN6vMWdrPJXJ+3vnDLBQAAANAyWiN4SwxLLj0zBiyZGG5gn7eI58gDAAAAEJnWCN7G93q3UWfesn3eop7nLewBUpJk3gAAAIBO1xrB29iAd1sm85ZIZfTLBzfJOVf0+Zsf36adwxPVbTfXbDLqAUvCzrz587wxYAl8yx7doqHxZLOrUdSabUNa8cyeZlcDAACg7bRG8DY+4N2WybxdccsaffAnD+uGlVunPTeWSOvi7y/XW759X3XbnRiWLBZt0NOI4C0xInXNkmKtcTrRXGu2DendP3pA//rzR5pdlaLO+urt+ttv3t3sagAAALSd1rjaz2beyow2uX3Iy6rtHZueTUj72bhndo1Ut93EsNeE0ay619UjMdKA0SbHyLohZzSRliQ9OzDW5JoAAAAgTK0RvGUzb33zot3uxFC0TSYzaa9/WiPmeQs7mwcAAACgpbRG8Dbm938JMNpksS5vpfrBVTQxFP00AVJjRptksBIAAACgo7VI8Dbg9T0rMwpjkJaNVm3zx8Rw9CNNSuFvMzFKs0lMU+tvGgAAAGhNrRG8jQ94/d2iHnBjYjjikSYbFLwlCd4wKcounAAAAIhOawRvYwN1zfFWc4JhYijizNuQd9uI0Sbp8wYAAAB0tBYJ3vYE6u8mSa72UG26xLDUu2945VXcnp95Czvb14jpBwAAAAC0lNYI3sYHAmTeSrcFq7mVWNSjTTZqwJJGTD+AthfqDx0AAABoutYI3sb2SP3zo92mc9EPWDKRbTYZ8jYnIt4PtDSr/ecMAAAAtLDWCd4CzvFWdKqAWraZGpcyqfYfsKQZQSgAAACAyDU/eMtkvAFLKgRvgaYKqGa7E8PebTP6vIXZbDI1Lrk0fd4AAACADtf84G18QJKT+hdEu91Eg5owlt3mcPjbbNT0A2h7zPMGAADQWZofvI3u9m77Iu7zlsu8RRy8dc2S4l3hlikxYAkAAADQ4VogeNvl3QbMvIWWTGjU4CHlNGI+tmwQSrNJFGCybgAAgM7S/OBtzM+89Vfo8xb2dhNN6PM20YD52Gg2CQAAAMwIzQ/ess0m6+jzVlPfnmzmLepmkz1zQi6zCRlEtAX6vAEAAHSWFgje/GaTQfu8lbsirSY914jBQypuswHNJhs18TfaFs0lAQAAOlPzg7ex3VKsS+otn5EK/YK0aZm3BvV5Y8ASAAAAoKM1P3gb3eU1mawnOqup2WSTMm9hB1n0eQMAAABmhBYI3nZHP02A5GXBumdLsXi02ww7yGpG80+0Bfq8AQAAdJbWCN6qGKyk6PVoLUm7iaHomxo2ZLTJYcniUldvuOUCAAAAaCktELztqjhNgCRZ2JMFTAxFn61KjDQg8+aXySgVAAAAQEdrfvA2vE3a56D6yqileVhiuOIgKaFKJ6X0RPjB28Qwg5UAAAAAM0DNwZuZHWJmfzSzx81slZm9v+pCMmlpfK/UH06ft6pyTxMRB2+5vmkNaDbJNAEogi5vQGOY2TlmttrM1prZpUWef6OZPeL/3W1mxzWjngCAzlNP5i0l6Z+dcy+QdLKk95jZMVWVML5XkqtqwJLQBmFIRNxsMjsqZOijTRK8YSpa0AKNY2ZxSVdKOlfSMZLeUOS772lJf+GcO1bSZyRdHW0tAQCdqubgzTm3xTn3gH9/SNLjkhZVVcjobu+2f74e2zwo55xWbd4rVyRCG0umS9fFzzEMjqeCb3tiqGLm7bHNg/rVg88qmc4okcroyW1Dwcsv1KjJtIv0o3PO6aGNA0qkMlq9tY46B7Bm25AmUqXPDcK3euuQUulMs6tRNeecHts8OG35lr1j2j2SiKQO2wbHtXN4ouw6a7YNKZFqv+MbllLnCTknSlrrnFvnnEtIukbSBfkrOOfuds7t8R/eI2lxxHUEAHSoUPq8mdlhkk6QdG+R5y4xs+VmtnzHjh1TnxzzgreHd8V03uV36J0/WKHzL79Tv3jg2Wnb+PmKTWFUdVKFvmKrNu/VeZffoQ/85CG94ep79JnfPqZXfvV2bR4Yq317UmOmCigo8661u/TqK+/Syz9/i86+7Hat3d6YAG7H0ITO+urt+sSvVjWkfEz39M4RnX3Z7frC71aXXKdVpwj43t3rdd7ld0xb/rLP3aKXfPb3kdThpP+8WUs/+4eSz28fHNdZX71dn7xuZST1aUXfvcs7T/eu29XsqrSqRZI25j3epPI/XF4s6YZiT5T9fgQAoIi6gzcz20fSLyR9wDk37eda59zVzrmlzrmlCxcunPqkn3l7ZrRHkvSHx7dJUtUZrppGoqww59qWgfHc/eXP7NH96726Dowmq99WdntSYwYsKcjmPb3T21Y2w7B9qHymoVaD496xyB4bNN4O/1w+uGFPhTVbz6oy2ZxWCTiz7+n7np657+mVm/dKkjbuqfGHqs5X7Aun+Cw2ZqfLC94+XOz5st+PAAAUUVfwZmbd8gK3Hznnrq26AD/zNtE9t55qVC+dlFLjEQ9Y0qBmk8nRymU2+MK4Ra674aPPG9BQmyQdkvd4saTNhSuZ2bGSviXpAuccaUwAQCjqGW3SJH1b0uPOua/UVMiYlz0Y796v1mrUZsLP7JUJ3kIPSBqVeSvW5y3cLZREjNA8rZKpqgbvl/ZSrO8xJEn3S1piZoebWY+kCyVdl7+CmR0q6VpJb3bOPdmEOgIAOlRXHa89RdKbJT1qZg/5y/7NObcscAmjuyWLK9FVZwas2qvCRgVSQbYZ5miTmYwfvDHa5ExBVg2NVlMz9BnEOZcys/dK+p2kuKTvOOdWmdm7/OevkvQJSQskfcP7nVMp59zSZtUZANA5ag7enHN3qt4f08d2S33zqroiLfprcLU/EE80IJCqpBHNJlNjklzFMhv9+zm/0EcvyBHnvKAWjobQFfk/Ui4rWHZV3v13SHpH1PUCAHS+UEabrNmoH7xVIZTLilzmrUyzybAvfLMBY3eIwVuj+tEFZKSBIhfkiLdq5oS3S3vh8w0AQOtpbvA2tlvqDz5Bd2gm/FHvahiwpObrmcSwF7jFQjzkJZp/Flax0QkYfqcHOg+ZWwAAWk+Tg7c9Ul91wVso1xN1NJusefuN6JuWGPVue/qnLGbAks7XjhfWrZoRxFScJwAAWleTm03uqTrzVuySteo+Gs0asCT04K25zSaz2jCOAAAAANpO85tN9s2LPouQy7xF2WxyJPwBUgIGoY0agIAuMc1Trj9Sq56XVq0XiuM3GQAAWk/zgrfkuDfBdLWZtzACvew8b2WCnsKt1L3ZxHBj5niTmp55Q/TasdkkAAAA6tO84G1st3db5WiTxVR9HZsYkuK9UldP3dsObKJ5zSYbP2AJgURUyF4hKrzVAABoPc0L3kazwVszRpscrroJY90XzYmRBmTemtB3Lw8DG6AaBJ4AAAD1aWLmbY93W3WzyRC2HaAJY+F26m822YDgLemPNtndX369BqMFX/Q45Gg03mMAALSeFmg2We1ok9MvKaq+yJgYknr3rfZV9UkMNa7ZZJOCNzIpzdDOB72d6z5z8LkGAKB1Nb/ZZFMm6R6q2Gyy8AKmrgsa5xo02uRI0Ym/o86EkXmLHse8MTiuHAMAAFpZC2TeqhuwpC2bTaYTUibVgMxbsEFQuBbrHO2cFWnnugMAALSC5mbeuvqk7r66i6p62PQaBizJqukCdKJBA4skRpgmYIap5q3eahmUVqtPMQSYHAMAAFpZEzNvAzU1mQzl+m9iqKYJuqUaL0AbNSpkIwZBQdvj4hsAAKAzNbfZZA3TBITXbLJS8BZimqBRk2knRqSeyoOVMKFz52jnwKyd6z4j8W8DAICW09xmk/31T9BdtUzGC96ibDbZ0MwbzSYBAACAmaAlMm9V9eOpd6qAib3ebZUDpdQlG7w1YrTJJg5YQialeYKc02KflWbi7QIAAFCf5mbeagig6m4BmJ0cfNbcOguqQkObTdLnbSYJEgAZYRLCwNsIAICW05zgzTkviGrGHG/Z4C3KzFtutMnmTBXQaPSpawKOORqNtxgAAC2nOcHbxKDk0rlmk/U2v6vqOjZg8DZtnrd6rmRyfd5qG+GydLkjUnflAUsadRFmtJuMXDsf8zau+ozCaQIAoHV1RbkxJ2lwPKk5o7tlklzfPA2OJZXKTI0uJpJpOeeUzjiNpzLqiU/GmM45OeeUTDv1dMVyy8oZmUhJkrrjMfWMDUiSEj1zFEtnZGZKpDJ+/Zy6YjHFTEqXKHNoPKWJVFoxMyXTGfV1x6fUJV8m45R2Tm5sSD2SRq1X8VRaPfGYRhNpzeqOK5HKKOOcuuKWW94VN/V2xUvuz56RhObNiknpCSW7+jU2ntSsrriS6YziMa9e+YYnUhpNpNTfU//pTqQyGk2k1NcTV9I/bomC7Y0mUprVFc8t747HFI95x7krZkplih+vbPndcasYpCRSmSllpPxzmd1OqfKLKXw/5S8fTaTV3xOXmSmd8d6TZt4+TaTS6u2KK5XOvn+kVNp7vrcrpuzbOh4zOeeUSGfUHYsp7ZwyzuXOcfZ4xWLePifTGaUzTrO647l9zTineGzymCTSXl0SqYx6u7wyu+NT65/9XKUz3memq+BzlEw7dcdNybRXt554TD1dMaULPo/Z7ZY6Ttnjn8o49XbFiq7j7dPU14wn07l9zO1XgHM3lkjnPp+z/XNT+HxPVyxX7+x5Kyb/fZPPuan7lN1GqfoVLi+1nnNOA6NJ9XTFNKs7Pu28FEqmM4qb95kp9blIZ5ySfl2z5zBraDzpb8d7nPHvOCfvM+yXn3ZOu0cS6u+JayyZ9o5jMp17L3bFTGPJtObM6i66fxOptH88vZ+5CAABAGiMSIO3lc/u1bGfuknH2lO6rle6+GfrdEvmpmnrff9Pz+jIA+fo479aOe25a+7fqO54TJ+74Qk98PGzNH92jy754YqS2/z6zWv05d8/mXu87GXP6BhJL7/sIe3U04HrvncsKUl63VV/mrK8r9u72PnS647Ta1+yeMpz7/vxg7r+0S36p66H9Z646ZjP3K5ylzUfOHOJLvvDGknSzf/8F3rewun92f7mG3fpgQ0DOuPwWfq2pM/fvFHfvmn6MSyshyT9+j2n6LhD5lbY0/KO+tgN05btHE7k7v9x9Xa9/bv36+iD5uiJrUOSpBccvK+W/eOpOupjN+jYxfvpkU179YO/P1GvOGrhlHLSGaejPnaD3n7KYfrkX/1ZyTrcvXanLvrWvfrZu16mlx7mZW+P/OgNevGhc/XR81+gv/3mn/S/F5+kU5fsH2ifrrl/oz5y7aO688Ona/G8yUzmv//2MX33rvX6xzOW6J/OOkqv/+8/acUzXub2mktO1oVX35O7LfSBM5fof25fp/7eLt3/0TP1v/c8o4//epWO2H+21u30+kD+6B0n6dk9Y/rXXzyiYxfvp+vee6okaclHvWP81H+epye2Dur8y+/Mlfur95wiSXp8y6Ce92/LJEmnHLlAd63dpfWfP1+S9L27vff1uh3edk7/0q3atGdU6z53fq6cL9/0pK7441q97y+P1NdvWZtbvv7z5+uEf79JXfGYJpLpXP0l6dt3Pq3PXv+4VnzsTC3Yp3fK/h7p1/nIA/bR2u3D+u37TtULF+2Xez67T/mO/viNWvsf5+YeP75lUOd+7Q5d9aaX6JwXHjRtfUl6asewzvjybbnH573oIH3jjS/JPR5PpvWCT9yY2xdJuuDKO7Xy2cGi5b3oUzdpv75u3fNvZ0iazLiv2zmS26cPnnmU3n/mEt2xZofe/O379It/eLle8tzJzP3a7cM68yu36WsXHq8Ljl+khzcO6IIr79J33/ZSnX70AVO2984frNAfHt8mSVo4p1fD4yk9/plzitZN8o7bq449WL99ZIv+5ezn6z2nHzltnZP+8w9TPoPZ/b758W26+PvLS5b9xw+dptO/dGuu/EKfvG6VPnndqinL3vKy5+ptLz9Mf/nl2/TVvztOrzlhsa5/ZIve838PlNwOAAAIT1OaTc4zrxnhgCs92MZvHtpcdPmmPWO69oFnJUnbBsclSQ9tHChZzmU3r5n6+s1euXtVXV+x/IujfNlfqW9cOf3i5/pHvWWzNaERzVKl36O/fcdkMPn4luIXmw9sGJAkrXza249RzSpbZr5s4NFIt63eIUm5wE3y9iWb+Hhkkzfa5x1rdkx7bSrjpWZ+dM+Gstu466mdkqR7nto1ZfkDGwZ0z7rdkqQ71+4MXOfr/QvXbLCT9d271kuSvnOnd17yj9/d/rbvWTe1Dlk/vX+jRhJp7RiakCRd97B3vrKBW7aO2fdI9rjkS6YzZd/bWXetnVqHXz049bOzYfeoChNP19zvHePv3b1+WnmD4yntHklMqb+k3Oduy97xknVZu937bAeptyQl05MVe2ST95qb/eCmmCe2DE15vOzRrVMejyXS015TKnCTvM/v1sHJ/SmWoPvp8o2SpDvWeO+p+9fvnvJ89rN60yqv3tn3yW1PTn+P/yFv33YMTeT+f5STDax+vmJT0edL/W/65YPPli139dbBKeUH8YM/PaPV/mf7xpXesf9DmfMFAADC1ZTgba68L/8BNX6utcJV+9JDGnG9SlaZdIzV0Q6oX+OBgqxq9mu2eRecIy548BaFUvsQ5qAm5UZTrKVfVaXXFHs+uyjobpWqc8Vt19AALdgxaFS5tQuyr7Wcq6rqUOb1pZ4qfE32cbMH8mlU/8jCYpu9nwAAzCTNCd7Myz7sKZN5CzJASC3XDH3poZqCxkoXluXqso+NBQqy8i+2Ku1bv7zgrZrMWxRKHafC3Qnjeq9YEdnt1zLATKlXlDvzpV4zbXmJQhpxeR1oOoE6gtwg5y7o0Z9ynrLll6tDhfIaOk1Chf3P7kujahB2kFR7cdk+kJNL/iL2sH7S8+86WMUz0QAAIBxNajY5pIwz7S0TRAW7QKz+6qM/Pai9ZYLGkipckZWrSX+u2WSFTeRto9Ke9ctrzjaq3gprtobpo3dWXqeUQEFEDRem1Vwc585VideEEpy6IlmdAK9r9IiUgT53AQ9AsdVaLZFT+L4o3H8rEsx464VcjxYrL/t6cxl9rfsKnRR7QufFp/f/BAAA4WlSs8lhDapfmTKbL/vruxW/WCq+7tTHXvBW/dxolZpNlrvwn23Bmk3GpmTeyu9cv99scrRdmk0WnNF6LtBzcVORd4kFyN5Mf002W1dcrIY2s9Mv8Mtvu1QZtYRhwSbyrqXc4J+74Jm3/PKzy0q/umJc2oC4NReklNj/wh8Tqvn/VFU9qiyv0qGotX6F+3vQxNO51hTHxJ6prVAAABBIU4K3BTZYtsmkVD54qeb6rLAZlddsspbgrUKzyTLPzVawZpPVxAiz/cxbkIxeVhQJjVK7EGTevMAXk2UujicTYlVk0SYrVf75KpS6wC9cp2yTzCKZt0ACvKauZpMB1g16+PPPkwXaQPmKNyLpOBmUldhmdr1ss0mb+ji0elRZXqVjUWv98n5ikiQdPXKfJGlV5rk6wdYWfQ0AAAhHU4K3hbZX21Vhkuwyz1XT96bwWq8vNVh2lMuAxVQleLPJKvq85TJvrdVsstkTMVf13ih8TYl3XbHs2GTfuoDbqGHAElfkdUGOb7DMWw0Dlvi3QQLjoMFz8cxbmTpUHOAlfIV92SrtW7WD2QSuR8iZt1oVZhZPGLpVj2Weq5+nX6EjYlt1mAUfvRIAAFSnOcGbBrTDza359fUECH3pobJ97UqplHkrZ7aNNyDz5o822WoDltRxnMLIVNQTlJRS7LxUGySWPizhX2IHOQe1ZfQaG5mHEZg2sr9f4KIrNMNtVz1KSoVNedMpPWdine7IvFBPu4MlSRfG/9iM6gEAMCNEOkl31gE2oNsyx0W+3V4l1O0SNfV5qzhgSZkrtdkBpwqoRquONlnKtGaTZQaqqBTEBcnQNFtodaujeWMgrXwQm6CWbFkt2d5WUE19Z2tMD/T+P/VaSmPXL9Yve2apZ+t+0te2qcsltdYt0l2ZF0qS3hm/Xp9PXdSgWgMAMLNFnnnr07jm2Ji2V8i8NWK0ybnyJhCudoJuKUDn/zLPzNa4hqsMsirtW79NKOViSlQRfzezRWOgqR8CllVppMdqyprymhAvvmsZTGd6Ga0ZDQTq8xa0rKJBfPP2O0hfzOnVK55pa3azyYrlVbHuC229ei2lNZlFGtj/xRpyfep1o9KgNxH48szzc3Nnxs1piRWfUBwAANQn8uDthJjXoX2wQgBVfq6nKkabzLs/17zgbaCW0SYrtGksdcE5SwnFzAUaFTJWRZ+3yWxekzuZFQg8YEmRHcwuq7Tv5fqb1dbnrfz7qXifN387AWd6KznYRcU+b9UL0sS3nnFQAv2wUsNwk0HGK6nULLIxfd6yZRd/300bbTKs7TY4iK2m/KNiGyVJb0p8RI+d/CW9JfkR/cdBl0uf3KMPLbkh12Ty7yY+Lkn6aNePwq8wAACIPng7Kfa4JGll5rDyK5YbbbKKUe/yr/X2kzecdW2TdNdmnyr6plUTvAUdBCVyJacKKP+41LKimwgQoFWTla3UDLPYLlXOmhWWUWLAkgplFAYrQfr0BRqwJFtuFW/syaqEl0XNP09BfpSp3Oct4IZLlj+9gMLRJqef29yaBXUIebTJKoO5sPr/7asRXdr1Yw242dqmeVP//5opEevLrXuve4EezByp0+IPh7JtAAAwVeTB20XxmyVJj7gjyq5XPvPmr1PlxUw281bLJN0VpwooUZXsqJBBBiypZpLu2TbeciNNSt7Fb4+SWj/rIq2fdZEO0B5J089VGJMzF5/nLXhWdvI12e2XGm2yTB1KbKdUdqaaslVhKoFSGjVmR3WZt4CjTVadeQtUbMgKRpssUcPJzFv178HgtQiuYlPvgAUu6/2IZtuEP5ebTdu/wmL+L/2X1VQTAABUIfLgbaENasz1qK7GRVVcweX/kr6fP5FsTX3earxqnF3FwCLVjGjZr/GWzLyZSa+J35l7fGvvP6lXiWAXnsVWSiWqugqu7SxVaorXwBEMm9TstbZ53prfRLfRVSibsQ248UYNYPK8zDPS2EDR5/bViE6wNUWecYorXdd2H80cLkn6bPKN3oIKh+Fn6dP02/RJdW0TAAAUF/lok0kX13fT51RcL0jTqZqbTdbQ563Wi8bZGpMUtNnk5P1KmYvZFv4IlmEweReSWf02odWz3qbM1+Zr/azdkys+5P89/3xv0IMtD2nf7n6tnzXqPf+NP5OWnCnd9TWpu1965WelB34gHXW2Dho8WufHHtRRu9ZLj62WLKazYg8oI9MhO3fqtNhGHbl3q7R2q3fiLOb9DW3zHnfNktIJKZ2Uuvt0zNhmxWNbddAzGyV3gBTvkeLd+vPYI3pX/DfakD5CWjWkM2KPKKkupRTX6x78Dx3bHdPYtvP0pvh2xZRRTE4xOZkymp2JazyeVExOuuMJ/fXgBh0fH/bWMadDbZue+/RB2pro04u6BhVXRrrxLsml9c3uFXo0c4Rm//DLeknXofpo14S2uXk60PZo4cMP6x3xTUopnvubrXHvAv2upyQzvTGzRkPxjPf5uGej3hp/zLt/32b/JJlenXpSu+Mp9SqusXjGr7VJD+7Ra+OPyDmTkz8w/J/WS939OmNsvZ4bG9e8ddul4f2keLeUSUsrf67relZqwO2jWzInaI5GderjaSnzZ1J3nxTv1hvjTyipuBKuW2PqUa+SurznSumL0ue6TteIZun0Wz6l/+3u1sDOE6UbfyZtWyktOWvy/FlMi7cN601xr/9VRjGvzit2+s+b4mmn18RWefV+eEga3Kw3xjcqrrS6lVaX0l42/OYVUnef3h9/XDHLSH9YLrm0Dhga1R96rteP0mfoINutZ93+Gs4cKD2R0NE71um82HYdvW2NdMtPpTkHSRbXIZsH9Ob4er1gcLb0p/t19DM79fXuu9S37SjpzudJmaR3nDIpXdr1hP42fru2u7na5fbVoPql394subTkMlImk3c/ra93b1JMGR1lz2pJ8lnpvz4o7btIsrhXbjqph3tHtZ95n5uNmYXSVZ+T0il9ZO+gvjLLG1DknInPa41bpLTiuY9g39gW/b7nX/THzPH6z9RFKhWVpRXTU5mD9a30+bnPuFT+/+96d1CZZwEAQK0iDd66lFa3pbXZLai4brlfwGv9ZXuuDSujuIbVV3nlwm1WeL5UfWfbhKSgzSYLe8+U1qcJ7XT7VSwzambeVBBjrkefSL1NX+y+WpIUG9td/AWrr598bXJ0cvn2Vd6fJCVHpev/ybu/5SG9XtLreyQ96f9J+p8e/3UPSWf1SHrK/wvgnySpR9J9/p/vh36Zp6RXST/7jb7dk/eiIenAuKSnH9XZ3UUKdZKyy2+WXqe8x1nbpIxMqXjMu6h+oEeKxXVufK/Ojd8vbZEO0wq9M/9Tev8yfazY9iTp997NB/O3daP06ez9ZZOrfih/nfzyfi19qbD833k375a843T79E0f6+fwXxF/1LuzWdLmX+ae/49SdZb0hi5/XrBB6dS4pF2rpF3+k0/fNmXdoyR9trCs30ze7ZX01ex5+mWZbd/h3Xww+9zdXVKsS/NT45ofkz4Z++HkuhlJ10jnSTqvR9Ia/8/3Akmf6Za0W9LvpBMknRCXtOMe6Q9527SYLo6bui2thbZ3cvlja71gLBb3g9C4FPOC1WNsTBnFNMcPznTs33nrSFLMq/Mv731WTqZ5NqRupXTIvgukeJc2TozJJoa0vw3qxt5LJUkPZo7UOneQFttOnXTTE1JMWhJ7Vpd0Xa8zJr6op9yiaYdqfxvUTk3+r5lsmlz6v9TazPRyAABA/SIP3iRpR4CgI1hgVnmlKaNNalhjXfuqcig2Xc193qoYsGTKFgKMNrlBB1Yss4oiQ3OYbdMz7kD9LH2afpY+TXM0qjs+cpaO/9xduXXetnShPnWik/Y9WFp3m7TklRoYHtFfXX6rDooN6mevP0g6+Dhp7nO9i9rRXdLABmn7Y1r2dEZXPjChv3np4br4lOdKzum8r90mk9O7/vwwfeuOp3T+iw7UJace7u21y0z+ZdJS/4Jcdk3JMX3h1/fptvWj+ui5R+nlRx/iZeTSSf3tlbfpiNgWjfUfoivecYZeddkt6lFKsyyhd7yoS//1SL9edfKx+t6fnlFG5ufeTE6mOX092jOWUkYxrf7sefr776/QrWt2eZkimeJK652vOFIb9oxp2aNbJUnrP+1lNo649DdabDt03T+cqPs2jOgTv12tfptQt1L6yrv+Rn/3zdu9H0L8TNKEujWuHq361FkySad/6Y/aNTwhk9PDnzhLJ/z772SSHvjYmf7xcHrNN+7Us3vGNKc3rpGJpExSTBnd9eHTdeoXbvH3QjI53fah06TuPv3DD+/Vqk179PXXv0jHLdrHO06pcam7T6d+bYVSLiaTtFtz9MnT99dFf7nUz3Cm9NLP3KBupdVrSfUqqTka1cG2S//+1nN0yne3K6W4vnThS/Xha+7VGS88VF9/7VFS0itbLpOr951rtun9P37Qy2j6mc57P3K69wF0GY0lUzr7q7cppoxu/efTJJfWS798v1KKK624kop75+Qz50ixLh3+0RvlFNP6z3vHfvXWIb3xst+oR0nN9vurLpglXfP24/ST5Rv1rft368KXHamLTz3cC6LiPbpj7W69/6eP6uVHLtQVb1yqXzy4WZ++bqX++vhF+uzfnJALshSLacmlkz9WZMcTXf+p84t+jlzG6Yx/m4y4D5jTq/v+5sxp633qruunPD7/Iq+8H/30Yf1i+ya9OX6Tzot5v0ocYHt0bux+9duEMhbXtalT9Nq4F43/ffxGfTR18bTy99derXaLi9axlHsyL6hqfQAAEEykwVu278XeGkZ7zFdN6JUfsMy1YY11zaltm1WOLpi1j3nNJgM1caxix/ptQiOZVmw2aTrMtuop95zcsiH1y3X1T1kvEe+TDn2R9+DFb5YkpW1CG92B2uIOko47b2rB+z7H+zv0ZD09tFar3Gr9ed8R0oFHS5Iec89IknbP/TM97EzHzX6udOgLA9X5qf5xrXLbNDj/RdIBB+eWr3BbtSL9fC2K90kH/plWuvXeE0469eDna/XDq3V6zwLt0sC0MmPq1piS3oOuXmViXcrkdTFNy8uyFOvzllFMG9yByiw4SqO7dmirdubeyJn4LA2rf9prJMn1zJHFTCM2W4PZj3b/fO3Rvt79fRbm1t0dm6/tGtWYujSk1GQh856rTe6AqQUveJ4kaVf8IG1wPRrf7wjpgKnZ801uw5THw33P8QKvbi/LvUPz/ErmV1j6xKKTNOKnpywW06hmKWNxadZ+3l+B1KyUdqlg+X6TgUVmIqUNzv9RY/8j/W2vnVZOtl6uoNuvk5vMMvl13Wnd0qEna+uTC7TGPam9sxZL8ycHXEr2x7Vb+2o4vq/UN1fp7iENarbG4/tIPcXPlSeaPoQ/TL9SP0y/smCp05dfd7w+9LOH9bv0Uv1Pz1c0rp6ir19gg9qVycu8FZZU5J/fVlVuXQEAAKoX6YAlXcpIkvYEGO0xrLmkpmXe4rU1Nay12WS/vGaTw0GaTQYoL8ub560FR5t0aR1q27TeTc0KTpsqoNhok1Vuq9wxqmWwiNLzvNVSVv15zqJz4ZWdDt5TVX1r2bcg6wTc/eLvg6hyxEW2HeB9Wap2jR5dMtziLVfe7zNL9VTmYB1o05s2dymleTasXW7f6fVr3mkCAGDGijR4y2beBmoYqj9fNaPe5a8710b8ZpON3Wa+xow26Vp2tMn9ElvVa6lpAxYECWaCXgyWO0w1jaJYabTJIk9PTtgcrNIlt1Chvo3YnwCbDfFF1RQfoN4NHm6ylvfWtHo3aLTJalVzqLa6+TrI9kxbPk9DkqRdmvy/2QKDjgIAMGNFHLx5mbcgk2QHmuetyu17zSZrzLzVOtqkjSnp4koEaKEadJLuXiXVZZmmzfN2lG3U6t636uzYfdOeWzjijeTwRObQKcsDZWtCyC1Umour6GsqzC9WNqgIHHAWL6PCNG/Tl5XZXjZADvJerSUIqmqet4BlTpmkO0DQ04y4YXL+tqmPS6nlPVhNPYKq5lht1XwdVCTztr8NStKUwZGq/eECAACEJ/Lgbcz1aKJE34p85TI11Yw2WdhscrzmzFv550sPWDLhZ8iCZBXyyiuzXp/fFLNZUwW8M369ei2p/+65TF/o+m/1KpF77rl775ekKX3epGLHp/Z2k7lgqtj6dU3SXf75oq8pUWbh8lJnv1wQ5Vx1QVau2WSAdXPr1HKcArwo8PHPWy9IcNTwed6KvS0LguJp+1/wXrBKb47aaxduaXk7u8XN14Hao5j/A1vWfD942+0m+wo3ah47AABQWeSjTe5RbcFTMdX0K+pWSvvYuMa6a8u8VRxtssTy2TU2byy3a/tY8BEsG2GR7czdf33XbXp9123Sp94mzT1Uxw94A1cMFQyqUXjBW0+ftyDXxrVcV5YKSmqKF0K4sHX+iI9Bi81liBo/k3WAVYIdgKLZxRbL6FTqrzkZdBa8x+vdbmF5DexTt9XNV7eltUB7JweXkXSwn43blressD6tdbYAAOhsdWXezOwcM1ttZmvN7NJK68cto70B+7uVbzqVbbYT3Hx5vyCPdU2/CAmiYn+0EpWZbWMaDTBYSeE2yl3AZif+HnbVz1cXhkW2U79Ov1z/L/GBos+/O/GP0xcW7E7xDEd19SgWvFfTtG/yNeWzdcWCocoD2BSWUWrbVRSi8j9YVBX01NGXrlEDlgRrNtmEPm8FQXGlXSsVzNWr2tIqxvB5BW5x8yVNBmtZh9h2pZ1ps9t/stzcy2du2Fbpu888l/vPP2JmL25GPQEAnafmzJuZxSVdKeksSZsk3W9m1znnHiv1mrjSgUaalCpcGFTTbMdfd39/UtyR7vmBtl+imJJKTtKdazZZWSwvlC6becsGb1VONh7GZW9MGT3Hdum3mZP1u8yJOmz8/yQpN0/WV3//pJbdvGba66aPnFfdSIr5ygVokxes1Qcypfu8lVbqAn368iIBoJW/wHaqrpngZOat8rq1vBca0Vxu6jmvHBw1Y7CMipm3gqa6jWo1WW0wWCnQzT/2u/w+bfNtMFfxg7RLZ8Qe1BYtUDL/q2KGN5sM+N13rqQl/t9Jkr7p3wIAUJd6Mm8nSlrrnFvnnEtIukbSBeVe0KVMoMFKKqllJPSFfvA2WmPwVmvk02/jGgmYeQuaVZhjzcu8Hag96ra0NrmFRZ+P4uK67IiANZyoSq8o1+ct8DZqybypxv1p1IAljR5tskFBZ1iCnsN2HI1xt7w+bQv80SUl6Z5Z79MLY+un9GmVGp/9bANBvvsukPQD57lH0lwzO7iwIAAAqmW1Nu0xs9dKOsc59w7/8ZslneSce2+p1xz/nB73r39/vv4t9Y6atlnM7J64RhLpKY/zZZ/729jt+nLPVfrzia9qY8EcZGHWpXDby3o+omfd/npn8p/rLi9b5qtif9IVPV/XmRNf0Fq3uMgrqyuzGkcnH9Mvej+ttyQ+rNszx00rN/9c5OvrjmssOfW5wrpknHLrlKtnsfNdbLtB97XU+6dweal9qyToawv3ZVZ3TOPJTLmXTNHfE5epdL1L7VuluhY7xqU+Z8VeV257pfax1LmrtB0nabRgf8u9pvDYBD0uQd8n1R6nfPn7Um7dIOerkn00qpWz3qH/SF6k/0m/SpK0ftZFkqQrU3+tL6YuLPq6cu/tZ/7rVSucc0sDV6JNBPnuM7PfSvq8c+5O//HNkj7snFteqtylS5e65ctLPg0A6CBmVvN3ZD3B2+sknV3wBXaic+59BetdIukS/+ELJa2saYPtb39JOyuu1blm8v7P5H2XZvb+z+R9f75zecNUdogg331mdr2kzxUEb//qnFtRUBbfj/WZyZ+vWnC8qsPxqg7Hqzo1f0fWM9rkJkmH5D1eLGlz4UrOuaslXS1JZra8E3+JDWIm77s0s/d/Ju+7NLP3f6bve7Pr0CBBvvv4fowAx6w6HK/qcLyqw/GqTj3fkfX0ebtf0hIzO9zMeiRdKOm6OsoDAKDVBfnuu07SW/xRJ0+WtNc5tyXqigIAOk/NmTfnXMrM3ivpd5Likr7jnFsVWs0AAGgxpb77zOxd/vNXSVom6TxJayWNSnp7s+oLAOgsdU3S7ZxbJu9LKqir69lem5vJ+y7N7P2fyfsuzez9Z987ULHvPj9oy953kt5TZbEde7waiGNWHY5XdThe1eF4Vafm41XzgCUAAAAAgOjU0+cNAAAAABCRSII3MzvHzFab2VozuzSKbUbBzNab2aNm9lB21Bgzm29mvzezNf7tvLz1P+Ifg9Vmdnbe8pf45aw1s8utlhmUI2Bm3zGz7Wa2Mm9ZaPtrZr1m9hN/+b1mdlikO1hGiX3/lJk965//h8zsvLznOmnfDzGzP5rZ42a2ysze7y+fKee+1P53/Pk3s1lmdp+ZPezv+6f95TPi3DeCVfg+NM/l/vOPmNmLm1HPVhHgeL3RP06PmNndZnZcsXJmikrHK2+9l5pZ2rx5C2esIMfLzE7z/8evMrPboq5jKwnwedzPzH6T950xo/v7WpFrx4Lna/t/75xr6J+8Dt1PSTpCUo+khyUd0+jtRvEnab2k/QuWfUHSpf79SyX9l3//GH/feyUd7h+TuP/cfZJeJskk3SDp3GbvW4n9fYWkF0ta2Yj9lfRuSVf59y+U9JNm73OFff+UpA8VWbfT9v1gSS/278+R9KS/jzPl3Jfa/44//3499/Hvd0u6V9LJM+XcN+B4Vvw+lDfQyQ3+cTpZ0r3NrneLH6+XS5rn3z+X41X5estf7xZ5/TZf2+x6t/LxkjRX0mOSDvUfH9Dserf48fq3vO+DhZJ2S+ppdt2beMymXTsWPF/T//soMm8nSlrrnFvnnEtIukbSBRFst1kukPR9//73Jb06b/k1zrkJ59zT8kYhO9HMDpa0r3PuT847kz/Ie01Lcc7dLu+DmC/M/c0v6+eSzsj+Ot9sJfa9lE7b9y3OuQf8+0OSHpe0SDPn3Jfa/1I6Zv+dZ9h/2O3/Oc2Qc98AQb4PL5D0A//Y3yNprn/8ZqKKx8s5d7dzbo//8B55c+rNVEGvt94n6ReStkdZuRYU5HhdJOla59wGSXLOzeRjFuR4OUlz/P/h+8i7bkpFW83WEeDasab/91EEb4skbcx7vEnlL3zaiZN0k5mtMLNL/GUHOn8+H//2AH95qeOwyL9fuLxdhLm/udc451KS9kpa0LCah+O9fqr7O3lNxzp23/0mbSfIy8DMuHNfsP/SDDj/ZhY3s4fkXej93jk3I899SIJ8H3byd2a1qj0WF8v7FXumqni8zGyRpNdIukoI8v46StI8M7vVv9Z7S2S1az1BjtcVkl4gabOkRyW93zmXiaZ6bamm//dRBG/FfkHtlCEuT3HOvVheU433mNkryqxb6jh06vGpZX/b7Vh8U9LzJB0vaYukL/vLO3LfzWwfeb/WfsA5N1hu1SLLOnH/Z8T5d86lnXPHy8tonGhmLyyzekftewME2deZdDwqCXwszOx0ecHbhxtao9YW5HhdJunDzrl046vT8oIcry5JL5F0vqSzJX3czI5qdMVaVJDjdbakhyQ9R9534xVmtm9jq9XWavp/H0XwtknSIXmPF8uLyNuec26zf7td0i/lpZS3ZVOe/m02xV7qOGzS1GYe7XZ8wtzf3GvMrEvSfgreVDFyzrlt/oVtRtL/yDv/Ugfuu5l1ywtcfuScu9ZfPGPOfbH9n0nnX5KccwOSbpV0jmbQuQ9ZkO/Djv3OrEGgY2Fmx0r6lqQLnHO7IqpbKwpyvJZKusbM1kt6raRvmNmrI6ld6wn6ebzROTfinNsp6XZJM3VQnCDH6+3ympk659xaSU9LOjqi+rWjmv7fRxG83S9piZkdbmY98jqkXxfBdhvKzGab2ZzsfUmvlLRS3r691V/trZJ+7d+/TtKF5o2sdrikJZLu85scDZnZyX4b4bfkvaYdhLm/+WW9VtItfv+YllTQLvk18s6/1GH77tf125Ied859Je+pGXHuS+3/TDj/ZrbQzOb69/sknSnpCc2Qc98AQb4Pr5P0Fn8UspMl7c02UZ2BKh4vMztU0rWS3uyce7IJdWwlFY+Xc+5w59xhzrnD5PUxfbdz7leR17Q1BPk8/lrSn5tZl5n1SzpJXr/nmSjI8dog6QxJMrMDJT1f0rpIa9leavt/76IZbeU8eSO0PSXpo1FsM4J9OkLeSDsPS1qV3S95fTVulrTGv52f95qP+sdgtfJGlJT3S9hK/7kr5E+e3mp/kn4sr3lYUt6vBReHub+SZkn6mbxBDu6TdESz97nCvv9QXpvuR/wP4MEduu+nykvjPyKvOcRD/md6ppz7Uvvf8edf0rGSHvT3caWkT/jLZ8S5b9AxnfZ9KOldkt7l3zdJV/rPPyppabPr3OLH61uS9uR9Npc3u86tfLwK1v2eZvBok0GPl6R/kTfi5Ep5zeabXu9WPV7ymkve5P/vWinpTc2uc5OPV7Frx7r/32e/PAEAAAAALSySSboBAAAAAPUheAMAAACANkDwBgAAAABtgOANAAAAANoAwRsAAAAAtAGCNwAAAABoAwRvAAAAANAGCN4AAAAAoA38f1tmrwdPSF2lAAAAAElFTkSuQmCC" + ] }, "metadata": { "needs_background": "light" - } + }, + "output_type": "display_data" } ], - "metadata": {} + "source": [ + "pl.figure(figsize=(15,5))\n", + "pl.subplot(1,2,1)\n", + "pl.plot(spikes)\n", + "pl.plot(position_at_each_iteration/50)\n", + "pl.ylim([0,10])\n", + "pl.xlim([0,30000])\n", + "\n", + "pl.subplot(1,2,2)\n", + "pl.plot(position_at_each_iteration[1:], spikes, '.')" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "TODO: \n", " - Sync table under ephys module.\n", @@ -987,17 +990,19 @@ " ephys_element.CuratedClustering.make = ironclust_make\n", " ephys_element.CuratedClustering.populate()\n", " ```" - ], - "metadata": {} + ] } ], "metadata": { + "interpreter": { + "hash": "ad2050938946b9745fdc6507c7561603613194cae6ce583d2036ae212150e70f" + }, "jupytext": { "encoding": "# -*- coding: utf-8 -*-" }, "kernelspec": { - "name": "python3", - "display_name": "Python 3.7.10 64-bit ('imaging_elements': conda)" + "display_name": "Python 3.7.10 64-bit ('imaging_elements': conda)", + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1010,9 +1015,6 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.10" - }, - "interpreter": { - "hash": "ad2050938946b9745fdc6507c7561603613194cae6ce583d2036ae212150e70f" } }, "nbformat": 4, diff --git a/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb b/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb index 853a3a86..75cade0b 100644 --- a/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb +++ b/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb @@ -72,6 +72,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -97,13 +98,12 @@ } ], "source": [ - "import os \n", "import datajoint as dj\n", "import numpy as np\n", - "from u19_pipeline import imaging, acquisition, subject\n", - "from u19_pipeline.imaging_element import imaging_element, scan_element, get_suite2p_dir\n", + "from u19_pipeline.imaging_element import get_suite2p_dir, imaging_element, scan_element\n", "from u19_pipeline.ingest.imaging_element_ingest import process_scan\n", - "import pathlib" + "\n", + "from u19_pipeline import acquisition, imaging, subject" ] }, { @@ -1690,6 +1690,7 @@ "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", + "\n", "%matplotlib notebook" ] }, diff --git a/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb b/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb index 71a731c7..ab588de1 100644 --- a/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb +++ b/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb @@ -29,6 +29,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -50,7 +51,6 @@ ], "source": [ "import datajoint as dj\n", - "import u19_pipeline.utility as utility\n", "\n", "dj.__version__" ] @@ -85,7 +85,6 @@ "outputs": [], "source": [ "# lab.Location()\n", - "import u19_pipeline.scheduler\n", "# dj.Diagram(lab)" ] }, @@ -95,9 +94,10 @@ "metadata": {}, "outputs": [], "source": [ - "from u19_pipeline import scheduler\n", "# scheduler = connect_mod('scheduler')\n", - "import json" + "import json\n", + "\n", + "from u19_pipeline import scheduler" ] }, { @@ -165,7 +165,6 @@ } ], "source": [ - "import json\n", "behavior_profile_variables = {'nested_key': 'nested_value'}\n", "json.dumps(behavior_profile_variables)\n", "# json.dumps(behavior_profile_variables, separators=(',', \":\"))" @@ -177,8 +176,10 @@ "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", "import json\n", + "\n", + "import numpy as np\n", + "\n", "json_dump = json.dumps(behavior_profile_variables, separators=(',', \":\"))\n", "length = len(json_dump)\n", "# Encode the string using Latin-1\n", diff --git a/notebooks/developer_notebooks/brightness_video_analysis.ipynb b/notebooks/developer_notebooks/brightness_video_analysis.ipynb index a518197c..a36b0d20 100644 --- a/notebooks/developer_notebooks/brightness_video_analysis.ipynb +++ b/notebooks/developer_notebooks/brightness_video_analysis.ipynb @@ -22,18 +22,15 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "\n", - "import datajoint as dj\n", - "import pandas as pd\n", + "\n", + "import cv2\n", "import numpy as np\n", + "import pandas as pd\n", "import pylab as plt\n", - "import matplotlib.patches as mpatches\n", - "from os.path import exists\n", - "import cv2\n", - "\n", - "from matplotlib import cm\n", - "from u19_pipeline import utility as uti\n" + "\n" ] }, { @@ -54,8 +51,8 @@ } ], "source": [ - "import numpy as np\n", "import json\n", + "\n", "npy_data = np.load('/Users/alvaros/Downloads/ops_mika.npy', allow_pickle=True)\n", "dict_data = npy_data[()]\n", "with open(\"/Users/alvaros/Downloads/ops_mika.json\", \"w\") as write_file:\n", @@ -1103,7 +1100,7 @@ " print(\"Error opening video stream or file\")\n", "\n", " total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n", - " mean_value = np.zeros((total_frames))\n", + " mean_value = np.zeros(total_frames)\n", " for k in range(total_frames):\n", " \n", " # Capture frame-by-frame\n", @@ -1305,7 +1302,7 @@ "plt.xticks(fontsize=16)\n", "plt.yticks(fontsize=16)\n", "plt.xlabel(\"Iteration # \", fontsize=18)\n", - "plt.ylabel(\"Mean value RGB \", fontsize=18);\n", + "plt.ylabel(\"Mean value RGB \", fontsize=18)\n", "plt.title('Color change across time towers task', fontsize=20);\n" ] }, diff --git a/notebooks/developer_notebooks/delete_logs_periodically.ipynb b/notebooks/developer_notebooks/delete_logs_periodically.ipynb index dd0c7c91..8459e2e9 100644 --- a/notebooks/developer_notebooks/delete_logs_periodically.ipynb +++ b/notebooks/developer_notebooks/delete_logs_periodically.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -31,10 +32,11 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import datetime\n", "import pathlib\n", "import re\n", - "import datetime\n", + "\n", + "import pandas as pd\n", "\n" ] }, diff --git a/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb b/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb index 008fb014..ee87810a 100644 --- a/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb +++ b/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, diff --git a/notebooks/developer_notebooks/review_ephys_sync.ipynb b/notebooks/developer_notebooks/review_ephys_sync.ipynb index cefe7767..a1b7d53b 100644 --- a/notebooks/developer_notebooks/review_ephys_sync.ipynb +++ b/notebooks/developer_notebooks/review_ephys_sync.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -42,12 +43,10 @@ "source": [ "import datetime\n", "import pathlib\n", + "\n", "import datajoint as dj\n", - "import numpy as np\n", - "import pylab as pl\n", - "import matplotlib\n", "import matplotlib.pyplot as plt\n", - "from scipy.signal import find_peaks\n", + "import numpy as np\n", "\n" ] }, @@ -66,8 +65,8 @@ } ], "source": [ - "from u19_pipeline import acquisition\n", - "from u19_pipeline import ephys_pipeline\n", + "from u19_pipeline import acquisition, ephys_pipeline\n", + "\n", "recording = dj.create_virtual_module(\"u19_recording\",\"u19_recording\")\n", "\n" ] @@ -1931,6 +1930,7 @@ " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n", "\n", "import scipy\n", + "\n", "peaks, _ = scipy.signal.find_peaks(diff_vector, height=0.05, distance=20)\n", "peaks" ] diff --git a/notebooks/developer_notebooks/startup_time_analysis.ipynb b/notebooks/developer_notebooks/startup_time_analysis.ipynb index efe3cb47..65bb14c5 100644 --- a/notebooks/developer_notebooks/startup_time_analysis.ipynb +++ b/notebooks/developer_notebooks/startup_time_analysis.ipynb @@ -252,10 +252,9 @@ } ], "source": [ - "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "import matplotlib.ticker as ticker\n", "import numpy as np\n", + "import pandas as pd\n", "\n", "# --- Deduplicate: within each location, if two startups are within 45s, keep the older one ---\n", "df = startup_times.reset_index().copy()\n", diff --git a/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb b/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb index 42f017b2..c35557cc 100644 --- a/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb +++ b/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -33,9 +34,10 @@ } ], "source": [ - "import pandas as pd\n", - "import datajoint as dj\n", "import pathlib\n", + "\n", + "import datajoint as dj\n", + "\n", "import u19_pipeline.automatic_job.params_config as config" ] }, diff --git a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb index 371c8262..c8af2023 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -40,15 +41,15 @@ ], "source": [ "import datetime\n", - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", "\n", "import datajoint as dj\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", "\n", "import u19_pipeline.ephys_pipeline as ephys_pipeline\n", - "import u19_pipeline.utils.ephys_utils as ephys_utils\n", - "import u19_pipeline.utils.ephys_fix_sync_code as ephys_fix_sync_code\n" + "import u19_pipeline.utils.ephys_fix_sync_code as ephys_fix_sync_code\n", + "import u19_pipeline.utils.ephys_utils as ephys_utils\n" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb index b50391ba..0ccda4c7 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -41,23 +42,20 @@ "source": [ "import datetime\n", "import pathlib\n", - "import numpy as np\n", - "import pylab as pl\n", - "import matplotlib\n", - "import matplotlib.pyplot as plt\n", - "from scipy.signal import find_peaks\n", - "from scipy import signal as sp\n", "\n", - "\n", - "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory, get_ephys_root_data_dir\n", - "# import u19_pipeline.ephys_sync as ephys\n", - "import u19_pipeline.acquisition as acquisition\n", "import datajoint as dj\n", - "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "from element_interface.utils import find_full_path\n", "\n", + "# import u19_pipeline.ephys_sync as ephys\n", + "import u19_pipeline.acquisition as acquisition\n", "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", - "import u19_pipeline.utils.ephys_utils as ephys_utils\n" + "import u19_pipeline.utils.ephys_utils as ephys_utils\n", + "from u19_pipeline.ephys_pipeline import (\n", + " get_ephys_root_data_dir,\n", + " get_session_directory,\n", + ")\n" ] }, { @@ -959,7 +957,7 @@ "#plt.plot(new_synced_time_vector4 - behavior_time_vector[:new_synced_time_vector4.shape[0]], color ='#ff7f0e')\n", "#plt.plot(new_synced_time_vector3 - behavior_time_vector[:new_synced_time_vector3.shape[0]], color='#1f77b4')\n", "\n", - "plt.plot((new_synced_time_vector4 - behavior_time_vector[:new_synced_time_vector4.shape[0]]))\n", + "plt.plot(new_synced_time_vector4 - behavior_time_vector[:new_synced_time_vector4.shape[0]])\n", "\n", "time_shift = np.arange(int(initial_sample+base_size/2), vec_shift.shape[0]+int(initial_sample+base_size/2))\n", "#plt.plot([time_shift[0], time_shift[-1]],[min_diff, min_diff], 'g')\n", @@ -1203,6 +1201,7 @@ " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n", "\n", "import scipy\n", + "\n", "peaks, _ = scipy.signal.find_peaks(diff_vector, height=0.05, distance=20)\n", "peaks" ] @@ -1225,7 +1224,7 @@ ], "source": [ "#plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n", - "plt.plot((new_synced_time_vector - beh_times[:it_times.shape[0]]))\n", + "plt.plot(new_synced_time_vector - beh_times[:it_times.shape[0]])\n", "#plt.plot((it_times - beh_times))\n", "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", "plt.plot(peaks, diff_vector[peaks], \"x\")\n", @@ -1459,8 +1458,8 @@ ], "source": [ "trial_plot = 1\n", - "plt.plot((iteration_dict['iter_times_idx'][trial_plot]))\n", - "plt.plot((behavior_time[trial_plot].flatten()))\n" + "plt.plot(iteration_dict['iter_times_idx'][trial_plot])\n", + "plt.plot(behavior_time[trial_plot].flatten())\n" ] }, { @@ -1677,7 +1676,7 @@ "print('samples_after_second_pulse', samples_after_second_pulse)\n", "print(iter_sample.shape[0]-samp_after)\n", "\n", - "idx_time_zero = np.where((time_vector >= 0))\n", + "idx_time_zero = np.where(time_vector >= 0)\n", "idx_time_zero = idx_time_zero[0]\n", "idx_time_zero = idx_time_zero[0]\n", "\n", diff --git a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb index f560c837..49212bef 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -39,25 +40,17 @@ } ], "source": [ - "import datetime\n", "import pathlib\n", - "import numpy as np\n", - "import pylab as pl\n", - "import matplotlib\n", - "import matplotlib.pyplot as plt\n", - "from scipy.signal import find_peaks\n", - "from scipy import signal as sp\n", "\n", - "\n", - "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory, get_ephys_root_data_dir\n", - "import u19_pipeline.utils.ephys_fix_sync_code as ephys_fix_sync_code\n", - "import u19_pipeline.acquisition as acquisition\n", "import datajoint as dj\n", - "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "from element_interface.utils import find_full_path\n", "\n", "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", - "import u19_pipeline.utils.ephys_utils as ephys_utils\n" + "import u19_pipeline.utils.ephys_fix_sync_code as ephys_fix_sync_code\n", + "import u19_pipeline.utils.ephys_utils as ephys_utils\n", + "from u19_pipeline.ephys_pipeline import get_ephys_root_data_dir, get_session_directory\n" ] }, { @@ -599,7 +592,7 @@ "print('samples_after_second_pulse', samples_after_second_pulse)\n", "print(iter_sample.shape[0]-samp_after)\n", "\n", - "idx_time_zero = np.where((time_vector >= 0))\n", + "idx_time_zero = np.where(time_vector >= 0)\n", "idx_time_zero = idx_time_zero[0]\n", "idx_time_zero = idx_time_zero[0]\n", "\n", diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb index 35443af1..1b560339 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb @@ -33,29 +33,24 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "\n", "import datetime\n", "import pathlib\n", - "import matplotlib\n", + "\n", + "import datajoint as dj\n", "import matplotlib.pyplot as plt\n", - "from scipy.signal import find_peaks\n", - "from scipy import signal as sp\n", - "import pathlib\n", + "import numpy as np\n", "import pynapple as nap\n", "import tqdm\n", - "\n", - "import datajoint as dj\n", "from element_interface.utils import find_full_path\n", - "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", - "import u19_pipeline.utils.ephys_utils as ephys_utils\n", - "\n", - "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory, get_ephys_root_data_dir\n", "\n", - "import numpy as np\n", - "import u19_pipeline.ephys_pipeline as ep\n", "import u19_pipeline.acquisition as acquisition\n", - "from u19_pipeline import recording" + "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", + "import u19_pipeline.utils.ephys_utils as ephys_utils\n", + "from u19_pipeline import recording\n", + "from u19_pipeline.ephys_pipeline import get_ephys_root_data_dir, get_session_directory" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb index c30e8a2b..aa8709e3 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -42,9 +43,8 @@ } ], "source": [ - "import pandas as pd\n", - "import numpy as np\n", "import datajoint as dj\n", + "import pandas as pd\n", "\n", "\n" ] diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb index 3d08d9e7..a10d512c 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -44,19 +45,11 @@ } ], "source": [ - "import datetime\n", "import pathlib\n", - "import numpy as np\n", - "import pylab as pl\n", - "import matplotlib\n", - "import matplotlib.pyplot as plt\n", - "from scipy.signal import find_peaks\n", - "\n", "\n", - "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory\n", "# import u19_pipeline.ephys_sync as ephys\n", - "import u19_pipeline.acquisition as acquisition\n", - "import datajoint as dj\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "\n", "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", "import u19_pipeline.utils.ephys_utils as ephys_utils" @@ -1819,6 +1812,7 @@ " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n", "\n", "import scipy\n", + "\n", "peaks, _ = scipy.signal.find_peaks(diff_vector, height=0.05, distance=20)\n", "peaks" ] diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb index 5c7213fb..e9fa2f5f 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -43,23 +44,20 @@ "source": [ "import datetime\n", "import pathlib\n", - "import numpy as np\n", - "import pylab as pl\n", - "import matplotlib\n", - "import matplotlib.pyplot as plt\n", - "from scipy.signal import find_peaks\n", - "from scipy import signal as sp\n", "\n", - "\n", - "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory, get_ephys_root_data_dir\n", - "# import u19_pipeline.ephys_sync as ephys\n", - "import u19_pipeline.acquisition as acquisition\n", "import datajoint as dj\n", - "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "from element_interface.utils import find_full_path\n", "\n", + "# import u19_pipeline.ephys_sync as ephys\n", + "import u19_pipeline.acquisition as acquisition\n", "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", - "import u19_pipeline.utils.ephys_utils as ephys_utils\n" + "import u19_pipeline.utils.ephys_utils as ephys_utils\n", + "from u19_pipeline.ephys_pipeline import (\n", + " get_ephys_root_data_dir,\n", + " get_session_directory,\n", + ")\n" ] }, { @@ -532,6 +530,7 @@ " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n", "\n", "import scipy\n", + "\n", "peaks, _ = scipy.signal.find_peaks(diff_vector, height=0.05, distance=20)\n", "peaks" ] @@ -555,7 +554,7 @@ ], "source": [ "#plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n", - "plt.plot((new_synced_time_vector - beh_times[:it_times.shape[0]]))\n", + "plt.plot(new_synced_time_vector - beh_times[:it_times.shape[0]])\n", "#plt.plot((it_times - beh_times))\n", "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", "plt.plot(peaks, diff_vector[peaks], \"x\")\n", @@ -778,8 +777,8 @@ ], "source": [ "trial_plot = 116\n", - "plt.plot((iteration_dict['iter_times_idx'][trial_plot]))\n", - "plt.plot((behavior_time[trial_plot].flatten()))\n" + "plt.plot(iteration_dict['iter_times_idx'][trial_plot])\n", + "plt.plot(behavior_time[trial_plot].flatten())\n" ] }, { @@ -876,7 +875,7 @@ "print('samples_after_second_pulse', samples_after_second_pulse)\n", "print(iter_sample.shape[0]-samp_after)\n", "\n", - "idx_time_zero = np.where((time_vector >= 0))\n", + "idx_time_zero = np.where(time_vector >= 0)\n", "idx_time_zero = idx_time_zero[0]\n", "idx_time_zero = idx_time_zero[0]\n", "\n", diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb index 6a2f27a3..8f2853a1 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -41,25 +42,15 @@ } ], "source": [ - "import datetime\n", - "import pathlib\n", - "import numpy as np\n", - "import pylab as pl\n", - "import matplotlib\n", - "import matplotlib.pyplot as plt\n", - "from scipy.signal import find_peaks\n", - "from scipy import signal as sp\n", - "\n", - "\n", - "from u19_pipeline.ephys_pipeline import ephys_element, probe_element, get_session_directory, get_ephys_root_data_dir\n", "# import u19_pipeline.ephys_sync as ephys\n", - "import u19_pipeline.acquisition as acquisition\n", "import datajoint as dj\n", - "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "from element_interface.utils import find_full_path\n", "\n", "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", - "import u19_pipeline.utils.ephys_utils as ephys_utils\n" + "import u19_pipeline.utils.ephys_utils as ephys_utils\n", + "from u19_pipeline.ephys_pipeline import get_ephys_root_data_dir, get_session_directory\n" ] }, { @@ -439,6 +430,7 @@ " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n", "\n", "import scipy\n", + "\n", "peaks, _ = scipy.signal.find_peaks(diff_vector, height=0.05, distance=20)\n", "peaks" ] @@ -461,7 +453,7 @@ ], "source": [ "#plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n", - "plt.plot((it_times - beh_times[:it_times.shape[0]]))\n", + "plt.plot(it_times - beh_times[:it_times.shape[0]])\n", "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", "plt.plot(peaks, diff_vector[peaks], \"x\")\n", "plt.ylabel('Time diff in current iteration TTL - Virmen (s)')\n", @@ -670,8 +662,8 @@ ], "source": [ "trial_plot = 99\n", - "plt.plot((iteration_dict['iter_times_idx'][trial_plot]))\n", - "plt.plot((behavior_time[trial_plot].flatten()))\n" + "plt.plot(iteration_dict['iter_times_idx'][trial_plot])\n", + "plt.plot(behavior_time[trial_plot].flatten())\n" ] }, { @@ -764,7 +756,7 @@ "print('samples_after_second_pulse', samples_after_second_pulse)\n", "print(iter_sample.shape[0]-samp_after)\n", "\n", - "idx_time_zero = np.where((time_vector >= 0))\n", + "idx_time_zero = np.where(time_vector >= 0)\n", "idx_time_zero = idx_time_zero[0]\n", "idx_time_zero = idx_time_zero[0]\n", "\n", diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb index 4255e2ab..9a5ebbdc 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -39,11 +40,10 @@ } ], "source": [ - "from u19_pipeline import ephys\n", - "#from u19_pipeline import ephys_pipeline as ephys\n", - "from u19_pipeline import recording\n", + "import pandas as pd\n", "\n", - "import pandas as pd" + "#from u19_pipeline import ephys_pipeline as ephys\n", + "from u19_pipeline import ephys" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb index e88eab83..dbf786a9 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -34,6 +35,7 @@ ], "source": [ "import numpy as np\n", + "\n", "import u19_pipeline.ephys_pipeline as ep\n" ] }, diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb index 9fe654e7..414d5ba1 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -35,6 +36,7 @@ ], "source": [ "import numpy as np\n", + "\n", "import u19_pipeline.ephys_pipeline as ep\n" ] }, diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb index 67316156..aa52df28 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -33,15 +34,11 @@ ], "source": [ "import datajoint as dj\n", - "import pandas as pd\n", - "import numpy as np\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "\n", "import u19_pipeline.ephys_pipeline as ep\n", "import u19_pipeline.utils.ephys_utils as eu\n", - "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", - "\n", - "\n", "from u19_pipeline import recording\n", "\n" ] @@ -373,7 +370,7 @@ } ], "source": [ - "plt.plot((all_sync_data['iteration_index_nidq'] - iteration_index_nidq))" + "plt.plot(all_sync_data['iteration_index_nidq'] - iteration_index_nidq)" ] }, { @@ -422,8 +419,8 @@ } ], "source": [ - "print((all_sync_data['iteration_index_nidq'][idx_diff[0:5]]))\n", - "print((iteration_index_nidq[idx_diff[0:5]]))" + "print(all_sync_data['iteration_index_nidq'][idx_diff[0:5]])\n", + "print(iteration_index_nidq[idx_diff[0:5]])" ] }, { @@ -441,8 +438,8 @@ } ], "source": [ - "print((all_sync_data['iteration_index_nidq'][idx_diff[0:5]-3]))\n", - "print((iteration_index_nidq[idx_diff[0:5]-3]))" + "print(all_sync_data['iteration_index_nidq'][idx_diff[0:5]-3])\n", + "print(iteration_index_nidq[idx_diff[0:5]-3])" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb index 8f530801..0e0a8d4f 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -35,16 +36,8 @@ ], "source": [ "import datajoint as dj\n", - "import pandas as pd\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", "\n", "import u19_pipeline.ephys_pipeline as ep\n", - "import u19_pipeline.utils.ephys_utils as eu\n", - "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", - "\n", - "\n", - "from u19_pipeline import recording\n", "\n" ] }, diff --git a/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb index 915cff08..dd172115 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb @@ -24,6 +24,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n" ] }, @@ -44,10 +45,11 @@ } ], "source": [ + "import datajoint as dj\n", + "import matplotlib.pyplot as plt\n", + "\n", "import u19_pipeline.ephys_pipeline as ep\n", "import u19_pipeline.recording as recording\n", - "import matplotlib.pyplot as plt\n", - "import datajoint as dj\n", " \n" ] }, @@ -304,6 +306,7 @@ "outputs": [], "source": [ "import numpy as np\n", + "\n", "time_as_behavior_fullsession = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession']]), axis=0) \n", "time_as_behavior_trial_ind = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind']]), axis=0) \n", "\n", diff --git a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb index e0777bb0..bccfaa32 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb @@ -16,6 +16,7 @@ "outputs": [], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n" ] }, @@ -34,10 +35,11 @@ } ], "source": [ + "import datajoint as dj\n", + "import matplotlib.pyplot as plt\n", + "\n", "import u19_pipeline.ephys_pipeline as ep\n", "import u19_pipeline.recording as recording\n", - "import matplotlib.pyplot as plt\n", - "import datajoint as dj\n", " \n" ] }, @@ -171,6 +173,7 @@ "outputs": [], "source": [ "import numpy as np\n", + "\n", "time_as_behavior_fullsession = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession']]), axis=0) \n", "time_as_behavior_trial_ind = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind']]), axis=0) \n", "\n", diff --git a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb index c2f94799..245a2dcb 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb @@ -25,6 +25,7 @@ "source": [ "\n", "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -43,12 +44,13 @@ } ], "source": [ - "import u19_pipeline.ephys_pipeline as ep\n", - "import u19_pipeline.utils.ephys_utils as ephys_utils\n", - "import u19_pipeline.recording as recording\n", - "import matplotlib.pyplot as plt\n", "import datajoint as dj\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", + "\n", + "import u19_pipeline.ephys_pipeline as ep\n", + "import u19_pipeline.recording as recording\n", + "import u19_pipeline.utils.ephys_utils as ephys_utils\n", " \n", "\n" ] diff --git a/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb index 31d15b51..b5a255b9 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -39,12 +40,12 @@ } ], "source": [ - "import numpy as np\n", - "import u19_pipeline.ephys_pipeline as ep\n", - "import u19_pipeline.recording as recording\n", - "import datajoint as dj\n", + "import datetime\n", + "\n", "import pandas as pd\n", - "import datetime\n" + "\n", + "import u19_pipeline.ephys_pipeline as ep\n", + "import u19_pipeline.recording as recording\n" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb index 76d6e992..5952360b 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb @@ -26,6 +26,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -46,8 +47,8 @@ ], "source": [ "import numpy as np\n", + "\n", "import u19_pipeline.ephys_pipeline as ep\n", - "import u19_pipeline.acquisition as acquisition\n", "from u19_pipeline import recording" ] }, @@ -103,6 +104,7 @@ ], "source": [ "import importlib\n", + "\n", "importlib.reload(ep)" ] }, diff --git a/notebooks/ephys_element/read_ephys_parameters.ipynb b/notebooks/ephys_element/read_ephys_parameters.ipynb index 7e9fbd64..850d58cc 100644 --- a/notebooks/ephys_element/read_ephys_parameters.ipynb +++ b/notebooks/ephys_element/read_ephys_parameters.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, diff --git a/notebooks/ephys_element/test_paramspy_for_os.ipynb b/notebooks/ephys_element/test_paramspy_for_os.ipynb index 3425ec85..1db3d8f7 100644 --- a/notebooks/ephys_element/test_paramspy_for_os.ipynb +++ b/notebooks/ephys_element/test_paramspy_for_os.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -24,9 +25,10 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import pathlib\n", + "\n", "import datajoint as dj\n", - "import pathlib" + "import pandas as pd" ] }, { @@ -35,10 +37,9 @@ "metadata": {}, "outputs": [], "source": [ - "import u19_pipeline.ephys_pipeline as ep\n", + "import u19_pipeline.lab as lab\n", "import u19_pipeline.recording as recording\n", - "import u19_pipeline.recording_process as rp\n", - "import u19_pipeline.lab as lab" + "import u19_pipeline.recording_process as rp" ] }, { @@ -249,7 +250,7 @@ " params_file = pathlib.Path(kilo_dir,'params.py')\n", " if params_file.is_file():\n", " \n", - " with open(params_file.as_posix(), \"r\") as file:\n", + " with open(params_file.as_posix()) as file:\n", " params_text = file.read()\n", " \n", " try:\n", diff --git a/notebooks/fetch_session_data.ipynb b/notebooks/fetch_session_data.ipynb index ab5baf24..986a0093 100644 --- a/notebooks/fetch_session_data.ipynb +++ b/notebooks/fetch_session_data.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -33,8 +34,8 @@ ], "source": [ "import datajoint as dj\n", - "import pandas as pd\n", "import numpy as np\n", + "import pandas as pd\n", "\n", "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n" diff --git a/notebooks/imaging_element/00-Set-up-configuration.ipynb b/notebooks/imaging_element/00-Set-up-configuration.ipynb index e22e83a7..d63a8c5d 100644 --- a/notebooks/imaging_element/00-Set-up-configuration.ipynb +++ b/notebooks/imaging_element/00-Set-up-configuration.ipynb @@ -24,10 +24,10 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", - "import os\n", - "import getpass\n", "import datajoint as dj\n", + "\n", "import u19_pipeline.utility as utility" ] }, diff --git a/notebooks/imaging_element/03-imaging-data-analysis.ipynb b/notebooks/imaging_element/03-imaging-data-analysis.ipynb index 3b5872f4..de10beda 100644 --- a/notebooks/imaging_element/03-imaging-data-analysis.ipynb +++ b/notebooks/imaging_element/03-imaging-data-analysis.ipynb @@ -29,6 +29,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -54,15 +55,14 @@ } ], "source": [ - "import os \n", + "\n", "import datajoint as dj\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", - "from u19_pipeline import acquisition, subject\n", - "from u19_pipeline.imaging_element import imaging_element, scan_element, get_suite2p_dir\n", - "from u19_pipeline.ingest.imaging_element_ingest import process_scan\n", - "import pathlib\n", - "import matplotlib.pyplot as plt\n", + "from u19_pipeline.imaging_element import imaging_element\n", + "\n", + "from u19_pipeline import subject\n", "\n", "imaging = dj.create_virtual_module('imaging', 'u19_imaging')\n", "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" diff --git a/notebooks/imaging_element/getSyncInfo_python.ipynb b/notebooks/imaging_element/getSyncInfo_python.ipynb index 44514496..182075df 100644 --- a/notebooks/imaging_element/getSyncInfo_python.ipynb +++ b/notebooks/imaging_element/getSyncInfo_python.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -24,17 +25,18 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", - "import numpy as np\n", + "import pathlib\n", + "\n", "import datajoint as dj\n", - "import u19_pipeline.imaging_pipeline as ip\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", "\n", - "import u19_pipeline.recording as recording\n", - "import u19_pipeline.lab as lab\n", "import u19_pipeline.acquisition as acquisition\n", - "import u19_pipeline.utils.matlab_utils as mu\n", - "import pathlib" + "import u19_pipeline.imaging_pipeline as ip\n", + "import u19_pipeline.lab as lab\n", + "import u19_pipeline.recording as recording\n", + "import u19_pipeline.utils.matlab_utils as mu" ] }, { @@ -52,11 +54,12 @@ "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", - "import tifffile\n", "import re\n", "import struct\n", "\n", + "import tifffile\n", + "\n", + "\n", "def get_sync_info(input_file, data_type, frame_skip):\n", " if data_type == \"uint16\":\n", " num_bytes = 2\n", diff --git a/notebooks/imaging_element/read_imaging_parameters.ipynb b/notebooks/imaging_element/read_imaging_parameters.ipynb index 34ec326b..4bb5363c 100644 --- a/notebooks/imaging_element/read_imaging_parameters.ipynb +++ b/notebooks/imaging_element/read_imaging_parameters.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -34,8 +35,9 @@ } ], "source": [ - "import pandas as pd\n", "import numpy as np\n", + "import pandas as pd\n", + "\n", "import u19_pipeline.imaging_pipeline as ip" ] }, @@ -371,6 +373,7 @@ "outputs": [], "source": [ "import json\n", + "\n", "with open(\"parameters_imaging.json\", \"w\") as f:\n", " json.dump(suite2p_params, f)" ] diff --git a/notebooks/imaging_element/read_params.py b/notebooks/imaging_element/read_params.py index 595cf0d7..7536b791 100644 --- a/notebooks/imaging_element/read_params.py +++ b/notebooks/imaging_element/read_params.py @@ -1,11 +1,13 @@ import os + from scipy.io import savemat this_dir = os.path.dirname(__file__) os.chdir(this_dir) import datajoint as dj + dj.conn() ephys_element =dj.create_virtual_module('u19_pipeline_ephys_element','u19_pipeline_ephys_element') diff --git a/notebooks/imaging_element/test_Matlab_Python_conv.ipynb b/notebooks/imaging_element/test_Matlab_Python_conv.ipynb index c801ccc8..10b33689 100644 --- a/notebooks/imaging_element/test_Matlab_Python_conv.ipynb +++ b/notebooks/imaging_element/test_Matlab_Python_conv.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -34,13 +35,12 @@ } ], "source": [ - "import pandas as pd\n", - "import numpy as np\n", "import datajoint as dj\n", + "import pandas as pd\n", + "\n", "import u19_pipeline.imaging_pipeline as ip\n", - "import matplotlib.pyplot as plt\n", - "import u19_pipeline.utils.tiff_utils as tu\n", - "import u19_pipeline.utils.tiff_matlab_imaging_utils as tmiu\n" + "import u19_pipeline.utils.tiff_matlab_imaging_utils as tmiu\n", + "import u19_pipeline.utils.tiff_utils as tu\n" ] }, { @@ -61,6 +61,7 @@ ], "source": [ "import importlib\n", + "\n", "importlib.reload(ip)\n", "importlib.reload(tu)\n", "importlib.reload(tmiu)" diff --git a/notebooks/imaging_storage_calculator.ipynb b/notebooks/imaging_storage_calculator.ipynb index 7da8a08e..72bfe6b4 100644 --- a/notebooks/imaging_storage_calculator.ipynb +++ b/notebooks/imaging_storage_calculator.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -36,6 +37,7 @@ ], "source": [ "import numpy as np\n", + "\n", "(np.round(100.76 * 2)/2)" ] }, @@ -45,11 +47,12 @@ "metadata": {}, "outputs": [], "source": [ + "\n", "import datajoint as dj\n", + "import matplotlib.pyplot as plt\n", "import pandas as pd\n", - "import subprocess\n", - "import u19_pipeline.utils.path_utils as pu\n", - "import matplotlib.pyplot as plt" + "\n", + "import u19_pipeline.utils.path_utils as pu" ] }, { diff --git a/notebooks/live_stats_monitor.ipynb b/notebooks/live_stats_monitor.ipynb index 3022d9b4..bacd033d 100644 --- a/notebooks/live_stats_monitor.ipynb +++ b/notebooks/live_stats_monitor.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "\n", "\n" @@ -33,12 +34,13 @@ "metadata": {}, "outputs": [], "source": [ + "import time\n", + "from datetime import datetime, timedelta\n", + "from zoneinfo import ZoneInfo\n", + "\n", "import datajoint as dj\n", "import numpy as np\n", "import pandas as pd\n", - "import time\n", - "from zoneinfo import ZoneInfo\n", - "from datetime import datetime, timedelta\n", "\n", "import u19_pipeline.utils.slack_utils as su" ] diff --git a/notebooks/numtrials_and_bias_check.ipynb b/notebooks/numtrials_and_bias_check.ipynb index e2711f28..f2d36158 100644 --- a/notebooks/numtrials_and_bias_check.ipynb +++ b/notebooks/numtrials_and_bias_check.ipynb @@ -22,18 +22,15 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "\n", - "import datajoint as dj\n", "from datetime import date, timedelta\n", + "\n", + "import datajoint as dj\n", "import pandas as pd\n", - "import numpy as np\n", - "import pylab as plt\n", - "import matplotlib.patches as mpatches\n", "\n", - "from matplotlib import cm\n", - "from u19_pipeline import utility\n", - "from inspect import getmembers, isfunction\n" + "from u19_pipeline import utility\n" ] }, { diff --git a/notebooks/performance_metrics_2022.ipynb b/notebooks/performance_metrics_2022.ipynb index 4853514d..dc878ea7 100644 --- a/notebooks/performance_metrics_2022.ipynb +++ b/notebooks/performance_metrics_2022.ipynb @@ -22,18 +22,13 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "\n", + "\n", "import datajoint as dj\n", - "from datetime import date, timedelta\n", "import pandas as pd\n", - "import numpy as np\n", - "import pylab as plt\n", - "import matplotlib.patches as mpatches\n", "\n", - "from matplotlib import cm\n", - "from u19_pipeline import utility\n", - "from inspect import getmembers, isfunction\n", "import u19_pipeline.alert_system.behavior_metrics as bm\n" ] }, diff --git a/notebooks/psychometrics_plot.ipynb b/notebooks/psychometrics_plot.ipynb index b2231f31..5c2507b3 100644 --- a/notebooks/psychometrics_plot.ipynb +++ b/notebooks/psychometrics_plot.ipynb @@ -13,14 +13,15 @@ "metadata": {}, "outputs": [], "source": [ + "import datetime\n", + "\n", "import datajoint as dj\n", - "import numpy as np\n", - "import pylab as plt\n", "import matplotlib.patches as mpatches\n", + "import numpy as np\n", "import pandas as pd\n", - "from scipy.optimize import curve_fit\n", + "import pylab as plt\n", "from astropy.stats import binom_conf_interval\n", - "import datetime" + "from scipy.optimize import curve_fit" ] }, { @@ -118,7 +119,7 @@ " #Correct deltaBin & trialBin to produce same result as Matlab psychFit\n", " deltaBins_search = deltaBins.astype(float) - 1.5\n", " trialBin = np.searchsorted(deltaBins_search, nCues_RminusL, side='right')\n", - " trialBin -= 1;\n", + " trialBin -= 1\n", " \n", " #Put into evidence bins all Trials with corresponding choices\n", " for iTrial in range(len(choices)):\n", @@ -129,7 +130,7 @@ " trialDelta[trialBin[iTrial]] = trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial]\n", " \n", " with np.errstate(divide='ignore', invalid='ignore'):\n", - " trialDelta = np.true_divide(trialDelta, numTrials);\n", + " trialDelta = np.true_divide(trialDelta, numTrials)\n", " \n", " # Select only bins with trials\n", " idx_zero = numTrials == 0 \n", @@ -174,7 +175,7 @@ " delta = np.linspace(deltaBins[0]-2, deltaBins[-1]+2, num=50)\n", "\n", " # Repeat trialDelta 3 times for errorX why ??\n", - " errorX = np.tile(trialDelta[~idx_zero], 3);\n", + " errorX = np.tile(trialDelta[~idx_zero], 3)\n", " \n", " # Confidence intervals are errorY, as a vector\n", " errorY = np.stack(pci[:,~idx_zero])\n", @@ -183,10 +184,10 @@ " # Fill dictionary of results\n", " fit_results = dict()\n", " fit_results['delta_bins'] = deltaBins[~idx_zero]\n", - " fit_results['delta_data'] = trialDelta[~idx_zero];\n", - " fit_results['pright_data'] = 100*phat[~idx_zero];\n", - " fit_results['delta_error'] = errorX;\n", - " fit_results['pright_error'] = 100*errorY;\n", + " fit_results['delta_data'] = trialDelta[~idx_zero]\n", + " fit_results['pright_data'] = 100*phat[~idx_zero]\n", + " fit_results['delta_error'] = errorX\n", + " fit_results['pright_error'] = 100*errorY\n", " \n", " if is_there_psychometric:\n", " fit_results['delta_fit'] = delta\n", diff --git a/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb b/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb index e7fa38e1..49cedc90 100644 --- a/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb +++ b/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb @@ -24,6 +24,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -33,18 +34,17 @@ "metadata": {}, "outputs": [], "source": [ - "import datajoint as dj\n", - "import os\n", + "import glob\n", "import pathlib\n", - "import pandas as pd\n", + "\n", "import cv2\n", - "import glob\n", - "import numpy as np\n", + "import datajoint as dj\n", "import matplotlib.pyplot as plt\n", - "from matplotlib import patches\n", - "from skimage.measure import EllipseModel\n", + "import numpy as np\n", + "import pandas as pd\n", + "from scipy import stats\n", "from skimage.draw import ellipse_perimeter\n", - "from scipy import stats" + "from skimage.measure import EllipseModel" ] }, { @@ -218,6 +218,7 @@ "outputs": [], "source": [ "from scipy.signal import find_peaks\n", + "\n", "x = labels.loc[:, (bodyparts[2],'y')],\n", "peaks, _ = find_peaks(x, threshold=30, )\n", "plt.plot(x)\n", @@ -395,6 +396,7 @@ "\n", "# Get the list of outlier frames\n", "from itertools import compress\n", + "\n", "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags['OutlierFlag']))\n", "\n", "\n", diff --git a/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb b/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb index 092cc7ef..19bc95d5 100644 --- a/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb +++ b/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n" ] }, @@ -31,16 +32,13 @@ "metadata": {}, "outputs": [], "source": [ + "import pathlib\n", + "import pickle\n", + "\n", "import datajoint as dj\n", "import pandas as pd\n", - "import numpy as np\n", "\n", - "import u19_pipeline.utils.path_utils as pu\n", - "\n", - "from matplotlib import cm\n", - "from u19_pipeline import utility\n", - "import pathlib\n", - "import pickle" + "import u19_pipeline.utils.path_utils as pu" ] }, { diff --git a/notebooks/pupillometry_notebooks/pupillometry.ipynb b/notebooks/pupillometry_notebooks/pupillometry.ipynb index 6fa2b624..21357268 100644 --- a/notebooks/pupillometry_notebooks/pupillometry.ipynb +++ b/notebooks/pupillometry_notebooks/pupillometry.ipynb @@ -22,9 +22,8 @@ "outputs": [], "source": [ "import os\n", - "import deeplabcut\n", - "\n", "\n", + "import deeplabcut\n", "\n", "modelPath = 'Pupillometry2-Ryan-2022-04-07'\n", "configPath = os.path.join(modelPath,'config.yaml')\n", @@ -462,8 +461,8 @@ } ], "source": [ - "import pandas as pd\n", "import numpy as np\n", + "import pandas as pd\n", "\n", "labels = pd.read_hdf(output_path)\n", "labels" @@ -476,6 +475,7 @@ "outputs": [], "source": [ "import cv2\n", + "\n", "cap = cv2.VideoCapture(videoPath)\n", "\n", "frameNumber = 1000\n", @@ -513,9 +513,8 @@ ], "source": [ "import matplotlib.pyplot as plt\n", - "from matplotlib import patches\n", - "from skimage.measure import EllipseModel\n", "from skimage.draw import ellipse_perimeter\n", + "from skimage.measure import EllipseModel\n", "\n", "# Take a subset from the data\n", "subset = labels.loc[frameNumber]\n", @@ -627,6 +626,7 @@ ], "source": [ "from scipy import stats\n", + "\n", "# Get a boolean array where true correspond to the frame with an outlier diameter\n", "zscore = np.abs(stats.zscore(df))\n", "outlierFlags = np.abs(stats.zscore(df)) > 2\n", @@ -634,6 +634,7 @@ "\n", "# Get the list of outlier frames\n", "from itertools import compress\n", + "\n", "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags['OutlierFlag']))\n", "\n", "# Retrieve an outlier frames sample from the video for visual inspection\n", @@ -738,7 +739,6 @@ " \"\"\"\n", " # Analyze the video using the selected modelPath and videoPath\n", " import os\n", - " import deeplabcut\n", " configPath = os.path.join(modelPath,'config.yaml')\n", " deeplabcut.analyze_videos(configPath, videoPath, destfolder=destinationFolder)\n", "\n", @@ -752,7 +752,6 @@ " \"\"\"\n", " # TODO make the function\n", " from skimage.measure import EllipseModel\n", - " from skimage.draw import ellipse_perimeter\n", " \n", " # Read the analyzed video data h5 file\n", " labels = pd.read_hdf(analyzedVideoDataPath)\n", @@ -806,6 +805,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n" ] }, @@ -815,9 +815,10 @@ "metadata": {}, "outputs": [], "source": [ - "import datajoint as dj\n", "import os\n", "import pathlib\n", + "\n", + "import datajoint as dj\n", "import pandas as pd" ] }, @@ -1383,6 +1384,7 @@ "source": [ "import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n", "\n", + "\n", "def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars):\n", " '''\n", " Create scp command from cluster directories and local slurm file\n", @@ -1421,11 +1423,11 @@ } ], "source": [ - "import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n", - "import u19_pipeline.automatic_job.slurm_creator as sc\n", + "import re\n", "import subprocess\n", + "\n", "import u19_pipeline.automatic_job.params_config as config\n", - "import re\n", + "import u19_pipeline.automatic_job.slurm_creator as sc\n", "from u19_pipeline.utils.file_utils import write_file\n", "\n", "repo_dir = '/scratch/gpfs/BRAINCOGS/U19-pipeline_python'\n", diff --git a/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb b/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb index 233b32e7..01951c0f 100644 --- a/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb +++ b/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb @@ -24,6 +24,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -59,11 +60,11 @@ } ], "source": [ - "import datajoint as dj\n", - "import os\n", "import pathlib\n", - "import pandas as pd\n", - "import deeplabcut" + "\n", + "import datajoint as dj\n", + "import deeplabcut\n", + "import pandas as pd" ] }, { @@ -886,6 +887,7 @@ "source": [ "import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n", "\n", + "\n", "def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars):\n", " '''\n", " Create scp command from cluster directories and local slurm file\n", @@ -924,11 +926,11 @@ } ], "source": [ - "import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n", - "import u19_pipeline.automatic_job.slurm_creator as sc\n", + "import re\n", "import subprocess\n", + "\n", "import u19_pipeline.automatic_job.params_config as config\n", - "import re\n", + "import u19_pipeline.automatic_job.slurm_creator as sc\n", "from u19_pipeline.utils.file_utils import write_file\n", "\n", "repo_dir = '/scratch/gpfs/BRAINCOGS/U19-pipeline_python'\n", diff --git a/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb b/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb index 80f1f479..ccc5a20e 100644 --- a/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb +++ b/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb @@ -24,6 +24,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -33,18 +34,17 @@ "metadata": {}, "outputs": [], "source": [ - "import datajoint as dj\n", - "import os\n", + "import glob\n", "import pathlib\n", - "import pandas as pd\n", + "\n", "import cv2\n", - "import glob\n", - "import numpy as np\n", + "import datajoint as dj\n", "import matplotlib.pyplot as plt\n", - "from matplotlib import patches\n", - "from skimage.measure import EllipseModel\n", + "import numpy as np\n", + "import pandas as pd\n", + "from scipy import stats\n", "from skimage.draw import ellipse_perimeter\n", - "from scipy import stats" + "from skimage.measure import EllipseModel" ] }, { @@ -280,6 +280,7 @@ "\n", "# Get the list of outlier frames\n", "from itertools import compress\n", + "\n", "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags['OutlierFlag']))\n", "\n", "\n", diff --git a/notebooks/read_virmen_behavior_file.ipynb b/notebooks/read_virmen_behavior_file.ipynb index 0896b02d..681e054f 100644 --- a/notebooks/read_virmen_behavior_file.ipynb +++ b/notebooks/read_virmen_behavior_file.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -32,21 +33,12 @@ } ], "source": [ - "import datajoint as dj\n", - "import pandas as pd\n", - "import numpy as np\n", - "import subprocess\n", "import os\n", - "import pathlib\n", - "import numpy as np\n", - "import u19_pipeline.utils.path_utils as pu\n", - "from sklearn.linear_model import LinearRegression\n", - "import matplotlib.pyplot as plt\n", - "import datetime\n", "from shutil import which\n", "\n", + "import datajoint as dj\n", + "\n", "import u19_pipeline.utils.matlab_utils as mu\n", - "from scipy.io import loadmat, matlab\n", "\n", "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", "from u19_pipeline import lab\n" diff --git a/notebooks/storage_calculator_across_time.ipynb b/notebooks/storage_calculator_across_time.ipynb index 650567bc..f62d5e09 100644 --- a/notebooks/storage_calculator_across_time.ipynb +++ b/notebooks/storage_calculator_across_time.ipynb @@ -15,6 +15,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -24,14 +25,12 @@ "metadata": {}, "outputs": [], "source": [ - "import datajoint as dj\n", - "import pandas as pd\n", - "import subprocess\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "import u19_pipeline.utils.path_utils as pu\n", + "import pandas as pd\n", "from sklearn.linear_model import LinearRegression\n", - "import matplotlib.pyplot as plt\n", - "import datetime\n", + "\n", + "import u19_pipeline.utils.path_utils as pu\n", "\n", "fig_size = (15,11)" ] @@ -292,7 +291,7 @@ "plt.xticks(fontsize=16)\n", "plt.yticks(fontsize=16)\n", "plt.xlabel(\"Date\", fontsize=18)\n", - "plt.ylabel(\"Storage space (TB)\", fontsize=18);\n", + "plt.ylabel(\"Storage space (TB)\", fontsize=18)\n", "plt.title('Storage on imaging & electrophysiology U19: ' + current_TB, fontsize=20);\n" ] }, @@ -467,7 +466,7 @@ "plt.xticks(fontsize=16)\n", "plt.yticks(fontsize=16)\n", "plt.xlabel(\"Date\", fontsize=18)\n", - "plt.ylabel(\"Storage space (TB)\", fontsize=18);\n", + "plt.ylabel(\"Storage space (TB)\", fontsize=18)\n", "plt.title('Storage on imaging & electrophysiology U19: ' + current_TB, fontsize=20);" ] }, @@ -586,7 +585,7 @@ "plt.title('Storage on imaging & electrophysiology U19 by year', fontsize=20)\n", "\n", "all_years = df_storage_year['year'].values\n", - "axs.set_xticks(all_years);\n", + "axs.set_xticks(all_years)\n", "df_storage_year\n" ] }, @@ -723,7 +722,7 @@ "plt.title('Storage on imaging & electrophysiology U19 by main dir', fontsize=20)\n", "\n", "all_folders = df_size_parent_folder['parent_folder'].values\n", - "axs.set_xticks(index);\n", + "axs.set_xticks(index)\n", "axs.set_xticklabels(all_folders, rotation = 45, ha=\"right\");" ] }, diff --git a/notebooks/training_days_per_protocol.ipynb b/notebooks/training_days_per_protocol.ipynb index 7bc7663d..ec408b9c 100644 --- a/notebooks/training_days_per_protocol.ipynb +++ b/notebooks/training_days_per_protocol.ipynb @@ -22,20 +22,18 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "\n", + "import copy\n", + "from datetime import date, timedelta\n", + "\n", "import datajoint as dj\n", - "from datetime import date, datetime, timedelta\n", - "import pandas as pd\n", "import numpy as np\n", + "import pandas as pd\n", "import pylab as plt\n", - "import matplotlib.patches as mpatches\n", "\n", - "from matplotlib import cm\n", - "from u19_pipeline import utility\n", - "from inspect import getmembers, isfunction\n", - "import u19_pipeline.alert_system.behavior_metrics as bm\n", - "import copy\n" + "import u19_pipeline.alert_system.behavior_metrics as bm\n" ] }, { diff --git a/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb b/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb index 0130f596..463f0a97 100644 --- a/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb +++ b/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb @@ -26,6 +26,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "import datajoint as dj" ] diff --git a/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb b/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb index 8a1f7a73..d2f72ad1 100644 --- a/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb +++ b/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb @@ -26,12 +26,13 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", + "import datetime\n", + "\n", "import datajoint as dj\n", "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import datetime\n", + "\n", "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n", "task = dj.create_virtual_module('task', 'u19_task') \n", "subject = dj.create_virtual_module('subject', 'u19_subject')\n", @@ -1846,6 +1847,7 @@ "source": [ "# loop through sessions of subject B205 and insert one by one, and compute time\n", "import time\n", + "\n", "start_time = time.time()\n", "\n", "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E57\"').fetch('KEY'):\n", diff --git a/notebooks/tutorials/3-Build a simple data pipeline.ipynb b/notebooks/tutorials/3-Build a simple data pipeline.ipynb index ef2b5c2a..235c71ac 100644 --- a/notebooks/tutorials/3-Build a simple data pipeline.ipynb +++ b/notebooks/tutorials/3-Build a simple data pipeline.ipynb @@ -42,6 +42,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "import datajoint as dj" ] diff --git a/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb b/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb index 6af32a1d..fed95af2 100644 --- a/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb +++ b/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb @@ -18,9 +18,10 @@ "metadata": {}, "outputs": [], "source": [ - "import pylab as pl\n", "import datajoint as dj\n", "import numpy as np\n", + "import pylab as pl\n", + "\n", "dj.__version__" ] }, @@ -239,7 +240,7 @@ "outputs": [], "source": [ "# On which days of week does people like to work harder?\n", - "Session = acquisition.Session();\n", + "Session = acquisition.Session()\n", "for day in range(7):\n", " print(len(Session & 'WEEKDAY(session_date) = ' + str(day)))" ] @@ -445,7 +446,7 @@ "for trial_idx in range(len(pos)):\n", " N_samples = len(pos[trial_idx])\n", " pl.plot(time[trial_idx][0:N_samples], pos[trial_idx][:,1])\n", - "pl.xlabel('Time [s]');\n", + "pl.xlabel('Time [s]')\n", "pl.ylabel('position [cm]');" ] }, diff --git a/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb b/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb index 07311511..2d936f4f 100644 --- a/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb +++ b/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb @@ -327,7 +327,7 @@ "metadata": {}, "outputs": [], "source": [ - "from getpass import getpass # use this to type password in without showing it" + "from getpass import getpass # use this to type password in without showing it" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb b/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb index 152983fa..ebb0167e 100644 --- a/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb +++ b/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb @@ -24,11 +24,11 @@ } ], "source": [ + "import datetime\n", + "\n", "import datajoint as dj\n", "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import datetime\n", + "\n", "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n", "task = dj.create_virtual_module('task', 'u19_task') \n", "subject = dj.create_virtual_module('subject', 'u19_subject')\n", @@ -1367,6 +1367,7 @@ "source": [ "# loop through sessions of subject B205 and insert one by one, and compute time\n", "import time\n", + "\n", "start_time = time.time()\n", "\n", "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E78\"').fetch('KEY'):\n", diff --git a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb index ac5141d8..d67d0814 100644 --- a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb +++ b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb @@ -23,6 +23,7 @@ "outputs": [], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -153,7 +154,7 @@ "metadata": {}, "outputs": [], "source": [ - "from getpass import getpass # use this to type password in without showing it" + "from getpass import getpass # use this to type password in without showing it" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb index 6bd9a9e5..004f4769 100644 --- a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb +++ b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb @@ -193,7 +193,7 @@ "metadata": {}, "outputs": [], "source": [ - "from getpass import getpass # use this to type password in without showing it" + "from getpass import getpass # use this to type password in without showing it" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb b/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb index 39e4a077..5a49aa5f 100644 --- a/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb +++ b/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb @@ -26,6 +26,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "import datajoint as dj" ] diff --git a/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb b/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb index b9098b41..e615f596 100644 --- a/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb +++ b/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb @@ -26,12 +26,13 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", + "import datetime\n", + "\n", "import datajoint as dj\n", "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import datetime\n", + "\n", "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n", "task = dj.create_virtual_module('task', 'u19_task') \n", "subject = dj.create_virtual_module('subject', 'u19_subject')\n", @@ -1846,6 +1847,7 @@ "source": [ "# loop through sessions of subject B205 and insert one by one, and compute time\n", "import time\n", + "\n", "start_time = time.time()\n", "\n", "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E57\"').fetch('KEY'):\n", diff --git a/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb b/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb index ef2b5c2a..235c71ac 100644 --- a/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb +++ b/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb @@ -42,6 +42,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "import datajoint as dj" ] diff --git a/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb b/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb index b20c803f..c633247f 100644 --- a/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb +++ b/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb @@ -39,11 +39,9 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", - "import numpy as np\n", "import datajoint as dj\n", "import matplotlib\n", - "import matplotlib.pyplot as plt\n", + "\n", "matplotlib.style.use('ggplot')\n", "%matplotlib inline" ] diff --git a/notebooks/water_weight_alert.ipynb b/notebooks/water_weight_alert.ipynb index ff19cc31..49478118 100644 --- a/notebooks/water_weight_alert.ipynb +++ b/notebooks/water_weight_alert.ipynb @@ -22,6 +22,7 @@ ], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()" ] }, @@ -40,12 +41,13 @@ } ], "source": [ - "import datajoint as dj\n", - "import pandas as pd\n", - "import time\n", - "from zoneinfo import ZoneInfo\n", "import datetime\n", "import pathlib\n", + "import time\n", + "\n", + "import datajoint as dj\n", + "import pandas as pd\n", + "\n", "import u19_pipeline.utils.slack_utils as su\n" ] }, @@ -61,7 +63,7 @@ " current_directory = pathlib.Path.cwd()\n", " query_file = pathlib.Path(current_directory, 'useful_queries', 'get_subject_data.sql').as_posix()\n", "\n", - " with open(query_file, \"r\", encoding=\"utf-8\") as file:\n", + " with open(query_file, encoding=\"utf-8\") as file:\n", "\n", " subject_query = file.read()\n", "\n", diff --git a/notebooks/xyz_picks_notebook.ipynb b/notebooks/xyz_picks_notebook.ipynb index 06d2bdc2..adab18d9 100644 --- a/notebooks/xyz_picks_notebook.ipynb +++ b/notebooks/xyz_picks_notebook.ipynb @@ -14,14 +14,11 @@ "outputs": [], "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", + "\n", "try_find_conf_file()\n", "\n", - "import pandas as pd\n", - "import datajoint as dj\n", "import numpy as np\n", "import pylab as pl\n", - "from scipy.io import loadmat\n", - "from scipy.spatial.transform import Rotation as R\n", "\n", "from u19_pipeline.utils import ephys_utils\n", "\n" diff --git a/pyproject.toml b/pyproject.toml index 138d6796..ea9caee4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,8 +64,32 @@ include = [ ] [tool.ruff] -# Allow lines to be as long as 120. -line-length = 120 +# if this is a library, enter the _minimum_ version you +# want to support, otherwise do py313 +target-version = "py312" +line-length = 120 # use whatever number makes you happy +extend-exclude = ["*.ipynb", "**/*.ipynb"] + +[tool.ruff.lint] +# you can see the looong list of rules here: +# https://docs.astral.sh/ruff/rules/ +# here's a couple to start with +select = [ + "A", # warn about shadowing built-ins + "E", # style stuff, whitespaces + "F", # important pyflakes lints + "I", # import sorting + "N", # naming + "T100", # breakpoints (probably don't want these in prod!) + "B", # flake8-bugbear: likely bugs and design problems + "SIM", # flake8-simplify: code that can be simplified + "UP", # pyupgrade: syntax that can be modernized (e.g., old Python idioms) + "C4", # flake8-comprehensions: better list/dict/set comprehension patterns +] +# if you're feeling confident you can do: +# select = ["ALL"] +# and then manually ignore annoying ones: +ignore = ["N815","A002", "N806"] [dependency-groups] dev = [ diff --git a/scripts/conf_file_finding.py b/scripts/conf_file_finding.py index 77adfbb0..677198a2 100644 --- a/scripts/conf_file_finding.py +++ b/scripts/conf_file_finding.py @@ -2,6 +2,7 @@ import os import pathlib + def chdir_to_root(): root_dir_found = 0 diff --git a/scripts/create_schemas.py b/scripts/create_schemas.py index 1e05e21d..e69de29b 100644 --- a/scripts/create_schemas.py +++ b/scripts/create_schemas.py @@ -1 +0,0 @@ -from u19_pipeline import lab, reference, subject, task, action, acquisition, behavior, puffs diff --git a/scripts/drop_schemas.py b/scripts/drop_schemas.py index 2cccf92e..b5047991 100644 --- a/scripts/drop_schemas.py +++ b/scripts/drop_schemas.py @@ -1,6 +1,18 @@ -from u19_pipeline import lab, subject, acquisition, behavior, task, reference, recording, recording_process, ephys_pipeline, imaging_pipeline, ephys_sync -from u19_pipeline.imaging_pipeline import scan_element, imaging_element -from u19_pipeline.ephys_pipeline import probe_element, ephys_element +from u19_pipeline import ( + acquisition, + behavior, + ephys_pipeline, + ephys_sync, + imaging_pipeline, + lab, + recording, + recording_process, + reference, + subject, + task, +) +from u19_pipeline.ephys_pipeline import ephys_element, probe_element +from u19_pipeline.imaging_pipeline import imaging_element, scan_element behavior.schema.drop() recording_process.schema.drop() diff --git a/scripts/ingest_hdf5_to_db.py b/scripts/ingest_hdf5_to_db.py index 4ec21b9e..ea6c7afc 100644 --- a/scripts/ingest_hdf5_to_db.py +++ b/scripts/ingest_hdf5_to_db.py @@ -4,17 +4,20 @@ # Folder structure for hdf5 files on bucket will be: # /jukebox/braininit/puffs/{netid}/{project_name}/{cohort}/{rig}/{hdf5_filename} +import glob +import json +import os +import sys +from datetime import timedelta + import datajoint as dj -import os, sys, glob, json -import pandas as pd import numpy as np -import time -from datetime import datetime, timedelta +import pandas as pd dj.config['database.host'] = 'datajoint00.pni.princeton.edu' # load dj creds from file credfile = '/jukebox/wang/ahoag/.djenv' -with open(credfile,'r') as infile: +with open(credfile) as infile: cred_dict = json.load(infile) dj.config['database.user'] = cred_dict.get('DJ_DB_USER') diff --git a/scripts/populate_imaging.py b/scripts/populate_imaging.py index 14dead71..ce6e5ae1 100644 --- a/scripts/populate_imaging.py +++ b/scripts/populate_imaging.py @@ -1,5 +1,4 @@ -from u19_pipeline_python import lab, reference, subject, task, action, acquisition, imaging - +from u19_pipeline_python import acquisition kargs = dict(supress_errors=True, display_progress=True) acquisition.Scan.populate(**kargs) diff --git a/u19_pipeline/__init__.py b/u19_pipeline/__init__.py index 79f1ef59..4c64b43a 100644 --- a/u19_pipeline/__init__.py +++ b/u19_pipeline/__init__.py @@ -1,3 +1,4 @@ -from os.path import join, dirname import os -import datajoint as dj \ No newline at end of file +from os.path import dirname, join + +import datajoint as dj diff --git a/u19_pipeline/acquisition.py b/u19_pipeline/acquisition.py index af69bc37..9349ce1b 100644 --- a/u19_pipeline/acquisition.py +++ b/u19_pipeline/acquisition.py @@ -1,5 +1,4 @@ import datajoint as dj -from u19_pipeline import lab, task, subject schema = dj.schema(dj.config['custom']['database.prefix'] + 'acquisition') diff --git a/u19_pipeline/alert_system/alert_code_skeleton.py b/u19_pipeline/alert_system/alert_code_skeleton.py index f5d262b5..fab6feb9 100644 --- a/u19_pipeline/alert_system/alert_code_skeleton.py +++ b/u19_pipeline/alert_system/alert_code_skeleton.py @@ -1,11 +1,5 @@ -import pandas as pd -import datajoint as dj -import datetime -import numpy as np -import u19_pipeline.alert_system.behavior_metrics as bm -import u19_pipeline.alert_system.alert_system_utility as asu # Slack Configuration dictionary diff --git a/u19_pipeline/alert_system/alert_system_utility.py b/u19_pipeline/alert_system/alert_system_utility.py index bd65016c..5cd5ea65 100644 --- a/u19_pipeline/alert_system/alert_system_utility.py +++ b/u19_pipeline/alert_system/alert_system_utility.py @@ -1,11 +1,13 @@ -import pandas as pd -import datajoint as dj import datetime +import datajoint as dj +import pandas as pd + import u19_pipeline.utils.dj_shortcuts as djs + def get_acquisition_data_alert_system(type='subject_fullname', data_days=60, min_sessions=20): ''' Get and filter data for alert system diff --git a/u19_pipeline/alert_system/behavior_metrics.py b/u19_pipeline/alert_system/behavior_metrics.py index 4d365702..2c259b5c 100644 --- a/u19_pipeline/alert_system/behavior_metrics.py +++ b/u19_pipeline/alert_system/behavior_metrics.py @@ -1,9 +1,7 @@ -import pandas as pd -import datajoint as dj -class BehaviorMetrics(): +class BehaviorMetrics: @staticmethod def get_bias_from_trial_df(trials_df, return_all_metrics=False): diff --git a/u19_pipeline/alert_system/cronjob_alert.py b/u19_pipeline/alert_system/cronjob_alert.py index 592bf6d1..42cf1311 100644 --- a/u19_pipeline/alert_system/cronjob_alert.py +++ b/u19_pipeline/alert_system/cronjob_alert.py @@ -1,17 +1,16 @@ import time + from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(1) -import datajoint as dj -import u19_pipeline.alert_system.main_alert_system as mas -import u19_pipeline.alert_system.log_deletion.old_log_deletion as old import u19_pipeline.alert_system.live_session_stats_deletion.live_session_stats_deletion as lssd +import u19_pipeline.alert_system.log_deletion.old_log_deletion as old import u19_pipeline.alert_system.noDB_backup_creation.noDB_backup_creation_script as noDBbcs - #mas.main_alert_system() old.main_old_log_deletion() lssd.main_live_session_stats_deletion() diff --git a/u19_pipeline/alert_system/custom_alerts/__init__.py b/u19_pipeline/alert_system/custom_alerts/__init__.py index 2f5c78e4..eb1f6e67 100644 --- a/u19_pipeline/alert_system/custom_alerts/__init__.py +++ b/u19_pipeline/alert_system/custom_alerts/__init__.py @@ -1,2 +1,3 @@ import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) \ No newline at end of file diff --git a/u19_pipeline/alert_system/custom_alerts/braininit_storage.py b/u19_pipeline/alert_system/custom_alerts/braininit_storage.py index 82c7ea42..609e433b 100644 --- a/u19_pipeline/alert_system/custom_alerts/braininit_storage.py +++ b/u19_pipeline/alert_system/custom_alerts/braininit_storage.py @@ -1,14 +1,9 @@ -import pandas as pd -import datajoint as dj - -import u19_pipeline.lab as lab - -import u19_pipeline.alert_system.behavior_metrics as bm -import u19_pipeline.alert_system.alert_system_utility as asu -import os import subprocess +import datajoint as dj +import pandas as pd + # Slack Configuration dictionary slack_configuration_dictionary = { 'slack_notification_channel': ['custom_alerts'], diff --git a/u19_pipeline/alert_system/custom_alerts/rig_bias.py b/u19_pipeline/alert_system/custom_alerts/rig_bias.py index 2f883542..bffb7404 100644 --- a/u19_pipeline/alert_system/custom_alerts/rig_bias.py +++ b/u19_pipeline/alert_system/custom_alerts/rig_bias.py @@ -1,12 +1,12 @@ -import pandas as pd -import datajoint as dj import datetime + +import datajoint as dj import numpy as np +import pandas as pd -import u19_pipeline.alert_system.behavior_metrics as bm import u19_pipeline.alert_system.alert_system_utility as asu - +import u19_pipeline.alert_system.behavior_metrics as bm # Slack Configuration dictionary slack_configuration_dictionary = { diff --git a/u19_pipeline/alert_system/custom_alerts/rig_trial.py b/u19_pipeline/alert_system/custom_alerts/rig_trial.py index 9f37778d..a662cef0 100644 --- a/u19_pipeline/alert_system/custom_alerts/rig_trial.py +++ b/u19_pipeline/alert_system/custom_alerts/rig_trial.py @@ -1,12 +1,10 @@ -import pandas as pd -import datajoint as dj import datetime + import numpy as np -import u19_pipeline.alert_system.behavior_metrics as bm import u19_pipeline.alert_system.alert_system_utility as asu - +import u19_pipeline.alert_system.behavior_metrics as bm # Slack Configuration dictionary slack_configuration_dictionary = { diff --git a/u19_pipeline/alert_system/custom_alerts/subject_bias.py b/u19_pipeline/alert_system/custom_alerts/subject_bias.py index 43c7a0b3..3a6037da 100644 --- a/u19_pipeline/alert_system/custom_alerts/subject_bias.py +++ b/u19_pipeline/alert_system/custom_alerts/subject_bias.py @@ -1,12 +1,12 @@ -import pandas as pd -import datajoint as dj import datetime + +import datajoint as dj import numpy as np +import pandas as pd -import u19_pipeline.alert_system.behavior_metrics as bm import u19_pipeline.alert_system.alert_system_utility as asu - +import u19_pipeline.alert_system.behavior_metrics as bm # Slack Configuration dictionary slack_configuration_dictionary = { diff --git a/u19_pipeline/alert_system/custom_alerts/subject_trial.py b/u19_pipeline/alert_system/custom_alerts/subject_trial.py index d69c4366..b11620df 100644 --- a/u19_pipeline/alert_system/custom_alerts/subject_trial.py +++ b/u19_pipeline/alert_system/custom_alerts/subject_trial.py @@ -1,12 +1,10 @@ -import pandas as pd -import datajoint as dj import datetime + import numpy as np -import u19_pipeline.alert_system.behavior_metrics as bm import u19_pipeline.alert_system.alert_system_utility as asu - +import u19_pipeline.alert_system.behavior_metrics as bm # Slack Configuration dictionary slack_configuration_dictionary = { diff --git a/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py b/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py index 5250566d..3fd32f10 100644 --- a/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py +++ b/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py @@ -1,6 +1,8 @@ import time + from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(0.1) diff --git a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py index 0756ff6c..4bdd7a1a 100644 --- a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py +++ b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py @@ -1,9 +1,9 @@ -import datajoint as dj -import numpy as np -import pandas as pd import time -from zoneinfo import ZoneInfo from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +import datajoint as dj +import pandas as pd import u19_pipeline.utils.slack_utils as su diff --git a/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py b/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py index 7e199601..1d31c80d 100644 --- a/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py +++ b/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py @@ -1,6 +1,8 @@ import time + from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(0.1) diff --git a/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py b/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py index f22e071a..4e3793e6 100644 --- a/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py +++ b/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py @@ -1,6 +1,7 @@ import datajoint as dj + def main_live_session_stats_deletion(): acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition') diff --git a/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py b/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py index f7f04033..bd1d4abc 100644 --- a/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py +++ b/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py @@ -1,6 +1,8 @@ import time + from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(0.1) diff --git a/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py b/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py index 4f2aa442..fd593574 100644 --- a/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py +++ b/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py @@ -4,8 +4,8 @@ import datajoint as dj import pandas as pd -import u19_pipeline.utils.slack_utils as su import u19_pipeline.lab as lab +import u19_pipeline.utils.slack_utils as su # Slack Configuration dictionary slack_configuration_dictionary = { diff --git a/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py b/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py index 9f49f5fd..a07fef2e 100644 --- a/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py +++ b/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py @@ -1,6 +1,8 @@ import time + from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(0.1) diff --git a/u19_pipeline/alert_system/log_deletion/old_log_deletion.py b/u19_pipeline/alert_system/log_deletion/old_log_deletion.py index 0969658c..54706b45 100644 --- a/u19_pipeline/alert_system/log_deletion/old_log_deletion.py +++ b/u19_pipeline/alert_system/log_deletion/old_log_deletion.py @@ -1,8 +1,9 @@ -import pandas as pd +import datetime import pathlib import re -import datetime + +import pandas as pd def extract_filename(x): diff --git a/u19_pipeline/alert_system/main_alert_system.py b/u19_pipeline/alert_system/main_alert_system.py index 5ca5ae44..4c481cac 100644 --- a/u19_pipeline/alert_system/main_alert_system.py +++ b/u19_pipeline/alert_system/main_alert_system.py @@ -1,17 +1,14 @@ -import pandas as pd -import datajoint as dj -import pkgutil -import importlib import datetime +import importlib +import pkgutil +import time import traceback import u19_pipeline.alert_system.custom_alerts as ca import u19_pipeline.lab as lab import u19_pipeline.utils.slack_utils as su -import time - # Slack Configuration dictionary slack_configuration_dictionary = { 'slack_notification_channel': ['custom_alerts'] diff --git a/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py b/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py index f143beb1..c10a65cb 100644 --- a/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py +++ b/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py @@ -1,30 +1,29 @@ from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() -import datajoint as dj -import pandas as pd -import time -from zoneinfo import ZoneInfo +import base64 import datetime import pathlib -import numpy as np -import time -import base64 import shutil +import time + +import datajoint as dj +import numpy as np import openpyxl -from openpyxl.utils.dataframe import dataframe_to_rows +import pandas as pd from openpyxl.drawing.image import Image +from openpyxl.utils.dataframe import dataframe_to_rows time.sleep(1) -import u19_pipeline.alert_system.water_weigh_alert.water_weigh_alert as wwa -import u19_pipeline.scheduler as scheduler import u19_pipeline.acquisition as acquisition +import u19_pipeline.alert_system.water_weigh_alert.water_weigh_alert as wwa import u19_pipeline.behavior as behavior -import u19_pipeline.subject as subject import u19_pipeline.lab as lab - +import u19_pipeline.scheduler as scheduler +import u19_pipeline.subject as subject DJ_CUSTOM_VARIABLES_FILENAME = 'DJCustomVariables.csv' SLACK_WEBHOOK_FILENAME = 'SlackChannels.csv' diff --git a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py index 4fb3050d..efb788b3 100755 --- a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py +++ b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py @@ -200,7 +200,7 @@ def check_overdue_maintenance(): unique_locations = sorted(set(locations)) # Get all maintenance types and their intervals - maintenance_fetch = getattr(rig_maintenance.MaintenanceType, "fetch") + maintenance_fetch = rig_maintenance.MaintenanceType.fetch maintenance_types = maintenance_fetch(as_dict=True) # Separate maintenance types by system type @@ -219,7 +219,7 @@ def check_overdue_maintenance(): rig_loc_query = lab.Location & merged_rig_queries - rig_loc_fetch = getattr(rig_loc_query, "fetch") + rig_loc_fetch = rig_loc_query.fetch rig_locations = rig_loc_fetch(as_dict=True) logger.info(f"🔍 Checking maintenance status as of {current_date}") @@ -252,7 +252,7 @@ def check_overdue_maintenance(): "location": location_name, "maintenance_type": maintenance_type, } - recent_fetch = getattr(recent_q, "fetch") + recent_fetch = recent_q.fetch recent_maintenance = recent_fetch("maintenance_date", order_by="maintenance_date DESC", limit=1) if len(recent_maintenance) == 0: @@ -330,7 +330,7 @@ def check_overdue_maintenance(): hosting_loc_query = lab.Location & merged_hosting_queries - hosting_loc_fetch = getattr(hosting_loc_query, "fetch") + hosting_loc_fetch = hosting_loc_query.fetch hosting_locations = hosting_loc_fetch(as_dict=True) # Check hosting VMs @@ -350,7 +350,7 @@ def check_overdue_maintenance(): "location": location_name, "maintenance_type": maintenance_type, } - recent_fetch = getattr(recent_q, "fetch") + recent_fetch = recent_q.fetch recent_maintenance = recent_fetch("maintenance_date", order_by="maintenance_date DESC", limit=1) if len(recent_maintenance) == 0: diff --git a/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py b/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py index 3238e705..1835b0d5 100644 --- a/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py +++ b/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py @@ -1,6 +1,8 @@ import time + from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(0.1) diff --git a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py index 706602cc..5e00eb7b 100644 --- a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py +++ b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py @@ -4,9 +4,8 @@ import datajoint as dj import pandas as pd -import u19_pipeline.utils.slack_utils as su import u19_pipeline.lab as lab - +import u19_pipeline.utils.slack_utils as su # Slack Configuration dictionary slack_configuration_dictionary = { diff --git a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py index 23749d9b..45d4dea1 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py +++ b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py @@ -14,7 +14,9 @@ import u19_pipeline.lab as lab import u19_pipeline.utils.slack_utils as su from u19_pipeline import subject -from u19_pipeline.utils.subject_metadata import fetch_slack_handles_for_lab_managers_by_subject +from u19_pipeline.utils.subject_metadata import ( + fetch_slack_handles_for_lab_managers_by_subject, +) slack_configuration_dictionary = { "slack_notification_channel": ["subject_health"], @@ -25,7 +27,7 @@ def get_subject_data(): - with open(QUERY_FILE, "r", encoding="utf-8") as file: + with open(QUERY_FILE, encoding="utf-8") as file: subject_query = file.read() conn = dj.conn() diff --git a/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py b/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py index 22d50e21..8aee0e7b 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py +++ b/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py @@ -1,6 +1,8 @@ import time + from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(0.1) diff --git a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py index 23749d9b..45d4dea1 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py +++ b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py @@ -14,7 +14,9 @@ import u19_pipeline.lab as lab import u19_pipeline.utils.slack_utils as su from u19_pipeline import subject -from u19_pipeline.utils.subject_metadata import fetch_slack_handles_for_lab_managers_by_subject +from u19_pipeline.utils.subject_metadata import ( + fetch_slack_handles_for_lab_managers_by_subject, +) slack_configuration_dictionary = { "slack_notification_channel": ["subject_health"], @@ -25,7 +27,7 @@ def get_subject_data(): - with open(QUERY_FILE, "r", encoding="utf-8") as file: + with open(QUERY_FILE, encoding="utf-8") as file: subject_query = file.read() conn = dj.conn() diff --git a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py index 0aa3e749..cd747c13 100644 --- a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py +++ b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py @@ -1,15 +1,13 @@ -import datajoint as dj -import pathlib -import subprocess import json -import re import os +import pathlib +import subprocess import time -from datetime import datetime -from element_interface.utils import dict_to_uuid +import datajoint as dj import u19_pipeline.automatic_job.params_config as config + #Functions to transfer files (globus, scp, smbclient) #FOR PNI endpoint @@ -166,7 +164,7 @@ def request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, des dict_output = json.loads(p.stdout.decode('UTF-8')) transfer_request['status'] = config.system_process['SUCCESS'] transfer_request['task_id'] = dict_output['task_id'] - except Exception as e: + except Exception: print('stdout is not a valid json, probably an error') transfer_request['status'] = config.system_process['ERROR'] transfer_request['error_info'] = p.stdout.decode('UTF-8') @@ -271,7 +269,7 @@ def get_error_log_str(recording_process_id): log_file_local_path = pathlib.Path(local_log_file_dir,log_filename).as_posix() if os.path.exists(log_file_local_path): - with open(log_file_local_path, 'r') as error_log_file: + with open(log_file_local_path) as error_log_file: error_log_data = ' '.join(error_log_file.readlines()) error_log_data = error_log_data.replace("activate the default environment with 'conda activate' or create a new environment to customize with 'conda create'.\n",'') diff --git a/u19_pipeline/automatic_job/create_missing_lfps_script.py b/u19_pipeline/automatic_job/create_missing_lfps_script.py index 240930fb..a23be741 100644 --- a/u19_pipeline/automatic_job/create_missing_lfps_script.py +++ b/u19_pipeline/automatic_job/create_missing_lfps_script.py @@ -1,11 +1,11 @@ import pathlib import sys -import datajoint as dj -from u19_pipeline import recording, ephys_pipeline, recording, recording_process +import datajoint as dj import u19_pipeline.automatic_job.params_config as config +from u19_pipeline import ephys_pipeline, recording, recording_process def main(recording_id): diff --git a/u19_pipeline/automatic_job/cronjob_automatic_job.py b/u19_pipeline/automatic_job/cronjob_automatic_job.py index 84a052a6..e2804630 100644 --- a/u19_pipeline/automatic_job/cronjob_automatic_job.py +++ b/u19_pipeline/automatic_job/cronjob_automatic_job.py @@ -1,6 +1,8 @@ import time + from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(1) diff --git a/u19_pipeline/automatic_job/ephys_element_ingest.py b/u19_pipeline/automatic_job/ephys_element_ingest.py index 7b6c674d..a733a9c7 100644 --- a/u19_pipeline/automatic_job/ephys_element_ingest.py +++ b/u19_pipeline/automatic_job/ephys_element_ingest.py @@ -1,13 +1,17 @@ -import re import pathlib -from u19_pipeline import ephys_pipeline +import re -from u19_pipeline.ephys_pipeline import (probe_element, ephys_element, - get_ephys_root_data_dir, get_session_directory) from element_array_ephys.readers import spikeglx - from element_interface.utils import find_full_path +from u19_pipeline import ephys_pipeline +from u19_pipeline.ephys_pipeline import ( + ephys_element, + get_ephys_root_data_dir, + get_session_directory, + probe_element, +) + """ The ingestion routine for imaging element includes: diff --git a/u19_pipeline/automatic_job/ephys_element_populate.py b/u19_pipeline/automatic_job/ephys_element_populate.py index 40ae3744..3df20a53 100644 --- a/u19_pipeline/automatic_job/ephys_element_populate.py +++ b/u19_pipeline/automatic_job/ephys_element_populate.py @@ -1,7 +1,7 @@ -from u19_pipeline.ephys_pipeline import probe_element, ephys_element +import u19_pipeline.automatic_job.params_config as config from u19_pipeline import recording, recording_process +from u19_pipeline.ephys_pipeline import ephys_element, probe_element -import u19_pipeline.automatic_job.params_config as config def populate_element_data(job_id, display_progress=True, reserve_jobs=False, suppress_errors=False): diff --git a/u19_pipeline/automatic_job/imaging_element.py b/u19_pipeline/automatic_job/imaging_element.py index 57095fcc..e802e411 100644 --- a/u19_pipeline/automatic_job/imaging_element.py +++ b/u19_pipeline/automatic_job/imaging_element.py @@ -1,20 +1,19 @@ -from concurrent.futures import process -import os -import datajoint as dj -import numpy as np -from u19_pipeline import lab, imaging_rec, acquisition, subject, recording -from u19_pipeline.imaging_element import imaging_element, scan_element, \ - get_processed_dir, Equipment,\ - get_imaging_root_data_dir, \ - get_scan_image_files -from u19_pipeline.ingest.imaging_element_ingest import process_scan -from element_interface.scanimage_utils import get_scanimage_acq_time, parse_scanimage_header -from element_interface.utils import find_full_path -import scanreader +import os import pathlib + +import scanreader import tifffile -import datetime -import h5py +from element_interface.scanimage_utils import ( + parse_scanimage_header, +) + +from u19_pipeline import imaging_rec, lab, recording +from u19_pipeline.imaging_element import ( + get_imaging_root_data_dir, + get_scan_image_files, + imaging_element, + scan_element, +) # subject_fullname = 'koay_K65' # session_date = '2018-02-02' @@ -170,13 +169,13 @@ loaded_scan = scanreader.read_scan(scan_filepaths) header = parse_scanimage_header(loaded_scan) #scanner = header['SI_imagingSystem'].strip('\'') #TODO: If using tiffile, hardcode it to `mesoscope` - except Exception as e: + except Exception: print('LOADED Scan using Tifffile') scan_filepaths = scan_filepaths # TODO load all TIFF files from session possibly using TIFFSequence loaded_scan = tifffile.imread(scan_filepaths) #scanner = 'mesoscope' except: #TODO: Use except instead of else) - print(f'ScanImage loading error') #TODO: Modify the error message + print('ScanImage loading error') #TODO: Modify the error message #Equipment.insert1({'scanner': scanner}, skip_duplicates=True) scan_element.Scan.insert1( @@ -236,7 +235,7 @@ iscell_fp = plane_filepath / 'iscell.npy' if not ops_fp.exists() or not iscell_fp.exists(): raise FileNotFoundError( - 'No "ops.npy" or "iscell.npy" found. Invalid suite2p plane folder: {}'.format(plane_filepath)) + f'No "ops.npy" or "iscell.npy" found. Invalid suite2p plane folder: {plane_filepath}') elif processing_method == 'caiman': raise ValueError("caiman not supported yet") diff --git a/u19_pipeline/automatic_job/imaging_element_populate.py b/u19_pipeline/automatic_job/imaging_element_populate.py index 31f88154..bb4e6a8c 100644 --- a/u19_pipeline/automatic_job/imaging_element_populate.py +++ b/u19_pipeline/automatic_job/imaging_element_populate.py @@ -1,9 +1,10 @@ -from u19_pipeline import recording, recording_process -from u19_pipeline.imaging_pipeline import imaging_element import pathlib import warnings import u19_pipeline.automatic_job.params_config as config +from u19_pipeline import recording, recording_process +from u19_pipeline.imaging_pipeline import imaging_element + def populate_element_data(job_id, display_progress=True, reserve_jobs=False, suppress_errors=False): diff --git a/u19_pipeline/automatic_job/parameter_file_creator.py b/u19_pipeline/automatic_job/parameter_file_creator.py index e05e2ecc..ba0ac75a 100644 --- a/u19_pipeline/automatic_job/parameter_file_creator.py +++ b/u19_pipeline/automatic_job/parameter_file_creator.py @@ -2,14 +2,14 @@ #import os #import pathlib -import subprocess -import pathlib import json -import re +import pathlib + +from scipy.io import savemat + import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft -import u19_pipeline.automatic_job.params_config as config +import u19_pipeline.automatic_job.params_config as config from u19_pipeline.utils.file_utils import write_file -from scipy.io import savemat # Functions to create parameter files and send them diff --git a/u19_pipeline/automatic_job/params_config.py b/u19_pipeline/automatic_job/params_config.py index 1140e898..b60f8d5c 100644 --- a/u19_pipeline/automatic_job/params_config.py +++ b/u19_pipeline/automatic_job/params_config.py @@ -1,11 +1,11 @@ -import pandas as pd -import numpy as np import os import pathlib -from scripts.conf_file_finding import get_root_directory +import pandas as pd + import u19_pipeline.lab as lab +from scripts.conf_file_finding import get_root_directory #Dictionary with main configuration for each modality (ephys or imaging) recording_modality_dict = [ diff --git a/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py b/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py index 17f62def..49c519e1 100644 --- a/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py +++ b/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py @@ -1,16 +1,18 @@ import time + from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(1) -import numpy as np +import datetime + +import pandas as pd + import u19_pipeline.ephys_pipeline as ep import u19_pipeline.recording as recording -import datajoint as dj -import pandas as pd -import datetime def get_rec_key_dict(recording_id): diff --git a/u19_pipeline/automatic_job/pupillometry_check_handler_script.py b/u19_pipeline/automatic_job/pupillometry_check_handler_script.py index 31e2bd61..039e774b 100644 --- a/u19_pipeline/automatic_job/pupillometry_check_handler_script.py +++ b/u19_pipeline/automatic_job/pupillometry_check_handler_script.py @@ -2,10 +2,10 @@ import time from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(1) -import datajoint as dj import u19_pipeline.automatic_job.pupillometry_handler as ph ph.PupillometryProcessingHandler.check_processed_pupillometry_sessions() diff --git a/u19_pipeline/automatic_job/pupillometry_handler.py b/u19_pipeline/automatic_job/pupillometry_handler.py index 07f1bfc2..cb75a02b 100644 --- a/u19_pipeline/automatic_job/pupillometry_handler.py +++ b/u19_pipeline/automatic_job/pupillometry_handler.py @@ -1,27 +1,25 @@ -import datajoint as dj -import pathlib -import pandas as pd -import numpy as np +import copy import glob -import subprocess -import re import os +import pathlib +import re +import subprocess import sys -import copy import traceback -from skimage.measure import EllipseModel -from skimage.draw import ellipse_perimeter -from scipy import stats +import datajoint as dj +import numpy as np +import pandas as pd +from scipy import stats +from skimage.measure import EllipseModel -import u19_pipeline.utils.slack_utils as slack_utils -import u19_pipeline.automatic_job.slurm_creator as slurmlib import u19_pipeline.acquisition as acquisition -import u19_pipeline.pupillometry as pupillometry import u19_pipeline.automatic_job.params_config as config -from u19_pipeline.automatic_job import recording_handler +import u19_pipeline.automatic_job.slurm_creator as slurmlib +import u19_pipeline.pupillometry as pupillometry +import u19_pipeline.utils.slack_utils as slack_utils from u19_pipeline.utils.file_utils import write_file -import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft + def pupillometry_exception_handler(func): def inner_function(*args, **kwargs): @@ -42,7 +40,7 @@ def inner_function(*args, **kwargs): return (config.RECORDING_STATUS_ERROR_ID, update_value_dict) return inner_function -class PupillometryProcessingHandler(): +class PupillometryProcessingHandler: spock_home_dir = '/mnt/cup/braininit/Shared/repos/U19-pipeline_python/' spock_log_dir = spock_home_dir + "u19_pipeline/automatic_job/OutputLog/" @@ -386,7 +384,7 @@ def check_processed_pupillometry_sessions(): try: pupil_data = PupillometryProcessingHandler.getPupilDiameter(h5_files) - except Exception as e: + except Exception: update_value_dict['error_info']['error_message'] = 'Could not get pupil diameter (check h5 or video file)' slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\ update_value_dict['error_info'] ,session_check) @@ -405,6 +403,7 @@ def check_processed_pupillometry_sessions(): if __name__ == '__main__': import time + from scripts.conf_file_finding import try_find_conf_file try_find_conf_file() time.sleep(1) diff --git a/u19_pipeline/automatic_job/pupillometry_handler_script.py b/u19_pipeline/automatic_job/pupillometry_handler_script.py index 19c02b57..1b595c45 100644 --- a/u19_pipeline/automatic_job/pupillometry_handler_script.py +++ b/u19_pipeline/automatic_job/pupillometry_handler_script.py @@ -2,10 +2,10 @@ import time from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(1) -import datajoint as dj import u19_pipeline.automatic_job.pupillometry_handler as ph ph.PupillometryProcessingHandler.check_pupillometry_sessions_queue() diff --git a/u19_pipeline/automatic_job/pupillometry_main.py b/u19_pipeline/automatic_job/pupillometry_main.py index f2f08b58..ab882904 100644 --- a/u19_pipeline/automatic_job/pupillometry_main.py +++ b/u19_pipeline/automatic_job/pupillometry_main.py @@ -3,6 +3,7 @@ if __name__ == '__main__': import time + from scripts.conf_file_finding import try_find_conf_file try_find_conf_file() time.sleep(1) diff --git a/u19_pipeline/automatic_job/recording_handler.py b/u19_pipeline/automatic_job/recording_handler.py index 46643147..e02b4f7b 100644 --- a/u19_pipeline/automatic_job/recording_handler.py +++ b/u19_pipeline/automatic_job/recording_handler.py @@ -1,21 +1,25 @@ +import copy import pathlib import time import traceback -import pandas as pd -import datajoint as dj -import copy - from datetime import datetime -from u19_pipeline import recording, ephys_pipeline, imaging_pipeline, recording, recording_process, lab +import datajoint as dj +import pandas as pd + +import u19_pipeline.automatic_job.params_config as config import u19_pipeline.utils.dj_shortcuts as dj_short -import u19_pipeline.utils.slack_utils as slack_utils import u19_pipeline.utils.scp_transfers as scp_tr - +import u19_pipeline.utils.slack_utils as slack_utils +from u19_pipeline import ( + ephys_pipeline, + imaging_pipeline, + lab, + recording, + recording_process, +) from u19_pipeline.automatic_job import ephys_element_ingest -import u19_pipeline.automatic_job.params_config as config - def exception_handler(func): @@ -34,7 +38,7 @@ def inner_function(*args, **kwargs): return (config.RECORDING_STATUS_ERROR_ID, update_value_dict) return inner_function -class RecordingHandler(): +class RecordingHandler: @staticmethod def pipeline_handler_main(): diff --git a/u19_pipeline/automatic_job/recording_process_handler.py b/u19_pipeline/automatic_job/recording_process_handler.py index ef66eb42..b0d2737a 100644 --- a/u19_pipeline/automatic_job/recording_process_handler.py +++ b/u19_pipeline/automatic_job/recording_process_handler.py @@ -1,31 +1,33 @@ -import time -import traceback -import pandas as pd -import datajoint as dj import copy import pathlib -import numpy as np +import time +import traceback +from datetime import datetime -from u19_pipeline.automatic_job import recording_handler +import pandas as pd +from ecephys_spike_sorting.common.SGLXMetaToCoords import MetaToCoords -import u19_pipeline.utils.dj_shortcuts as dj_short -import u19_pipeline.utils.slack_utils as slack_utils import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft -import u19_pipeline.automatic_job.slurm_creator as slurmlib -import u19_pipeline.automatic_job.parameter_file_creator as paramfilelib -import u19_pipeline.automatic_job.params_config as config import u19_pipeline.automatic_job.ephys_element_populate as ep import u19_pipeline.automatic_job.imaging_element_populate as ip +import u19_pipeline.automatic_job.parameter_file_creator as paramfilelib +import u19_pipeline.automatic_job.params_config as config +import u19_pipeline.automatic_job.slurm_creator as slurmlib +import u19_pipeline.utils.dj_shortcuts as dj_short +import u19_pipeline.utils.slack_utils as slack_utils +from u19_pipeline import ( + ephys_pipeline, + imaging_pipeline, + recording, + recording_process, + utility, +) +from u19_pipeline.automatic_job import recording_handler +from u19_pipeline.utility import is_this_spock -from datetime import datetime -from u19_pipeline import recording, recording_process, ephys_pipeline, imaging_pipeline, utility -from u19_pipeline.utility import create_str_from_dict, is_this_spock -from u19_pipeline.utils import ephys_utils - -from ecephys_spike_sorting.common.SGLXMetaToCoords import MetaToCoords -class RecProcessHandler(): +class RecProcessHandler: @staticmethod def pipeline_handler_main(): diff --git a/u19_pipeline/automatic_job/slurm_creator.py b/u19_pipeline/automatic_job/slurm_creator.py index d1d91c1c..2dc81797 100644 --- a/u19_pipeline/automatic_job/slurm_creator.py +++ b/u19_pipeline/automatic_job/slurm_creator.py @@ -1,14 +1,13 @@ import copy -import os -import subprocess import pathlib -import json import re +import subprocess + import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft -from u19_pipeline.utility import create_str_from_dict, is_this_spock import u19_pipeline.automatic_job.params_config as config +from u19_pipeline.utility import is_this_spock from u19_pipeline.utils.file_utils import write_file # Functions to create slurm jobs diff --git a/u19_pipeline/automatic_job/tigress2pni.py b/u19_pipeline/automatic_job/tigress2pni.py index 2a9b8300..92130afe 100755 --- a/u19_pipeline/automatic_job/tigress2pni.py +++ b/u19_pipeline/automatic_job/tigress2pni.py @@ -14,17 +14,20 @@ the second run won't queue a duplicate transfer.""" import json -import sys import os -import six +import sys -from globus_sdk import (NativeAppAuthClient, TransferClient, - RefreshTokenAuthorizer, TransferData) +import six +from fair_research_login import NativeClient +from globus_sdk import ( + NativeAppAuthClient, + RefreshTokenAuthorizer, + TransferClient, + TransferData, +) from globus_sdk.exc import GlobusAPIError from globus_sdk.services.transfer.errors import TransferAPIError -from fair_research_login import NativeClient - # Princeton TIGRESS SOURCE_ENDPOINT = 'a9df83d2-42f0-11e6-80cf-22000b1701d1' # SOURCE_ENDPOINT = 'ef3a4e74-e742-11ec-9912-3b4cfda38030' @@ -74,7 +77,7 @@ def load_data_from_file(filepath): """Load a set of saved tokens.""" if not os.path.exists(filepath): return [] - with open(filepath, 'r') as f: + with open(filepath) as f: tokens = json.load(f) return tokens @@ -109,7 +112,7 @@ def setup_transfer_client(transfer_tokens): if ex.http_status == 401: sys.exit('Refresh token has expired. ' 'Please delete the `tokens` object from ' - '{} and try again.'.format(DATA_FILE)) + f'{DATA_FILE} and try again.') else: raise ex return transfer_client @@ -120,10 +123,7 @@ def check_endpoint_path(transfer_client, endpoint, path): try: transfer_client.operation_ls(endpoint, path=path) except TransferAPIError as tapie: - print('Failed to query endpoint "{}": {}'.format( - endpoint, - tapie.message - )) + print(f'Failed to query endpoint "{endpoint}": {tapie.message}') sys.exit(1) @@ -134,9 +134,9 @@ def create_destination_directory(transfer_client, dest_ep, dest_path): except TransferAPIError: try: transfer_client.operation_mkdir(dest_ep, dest_path) - print('Created directory: {}'.format(dest_path)) + print(f'Created directory: {dest_path}') except TransferAPIError as tapie: - print('Failed to start transfer: {}'.format(tapie.message)) + print(f'Failed to start transfer: {tapie.message}') sys.exit(1) @@ -177,9 +177,7 @@ def main(): pass if len(sys.argv) < 2: - print('Usage: {} tigress_dir pni_dir'.format( - sys.argv[0] - )) + print(f'Usage: {sys.argv[0]} tigress_dir pni_dir') sys.exit(1) SOURCE_PATH = sys.argv[1] @@ -202,12 +200,7 @@ def main(): task = transfer.submit_transfer(tdata) save_data_to_file(DATA_FILE, 'task', task.data) - print('Transfer has been started from\n {}:{}\nto\n {}:{}'.format( - SOURCE_ENDPOINT, - SOURCE_PATH, - DESTINATION_ENDPOINT, - DESTINATION_PATH - )) + print(f'Transfer has been started from\n {SOURCE_ENDPOINT}:{SOURCE_PATH}\nto\n {DESTINATION_ENDPOINT}:{DESTINATION_PATH}') url_string = 'https://globus.org/app/transfer?' + \ six.moves.urllib.parse.urlencode({ 'origin_id': SOURCE_ENDPOINT, @@ -215,7 +208,7 @@ def main(): 'destination_id': DESTINATION_ENDPOINT, 'destination_path': DESTINATION_PATH }) - print('Visit the link below to see the changes:\n{}'.format(url_string)) + print(f'Visit the link below to see the changes:\n{url_string}') if __name__ == '__main__': diff --git a/u19_pipeline/behavior.py b/u19_pipeline/behavior.py index fe7c8eef..770d0897 100644 --- a/u19_pipeline/behavior.py +++ b/u19_pipeline/behavior.py @@ -1,5 +1,4 @@ import datajoint as dj -from . import acquisition, task, subject schema = dj.schema(dj.config['custom']['database.prefix'] + 'behavior') diff --git a/u19_pipeline/copy_table.py b/u19_pipeline/copy_table.py index 445938cd..14d3348d 100644 --- a/u19_pipeline/copy_table.py +++ b/u19_pipeline/copy_table.py @@ -1,9 +1,10 @@ -import datajoint as dj import traceback -from u19_pipeline.temp import acquisition, behavior, imaging, meso, meso_analysis -from u19_pipeline import subject + +import datajoint as dj from tqdm import tqdm +from u19_pipeline import subject +from u19_pipeline.temp import acquisition, behavior, imaging, meso, meso_analysis acquisition_original = dj.create_virtual_module( 'acquisition_original', 'u19_acquisition' @@ -54,7 +55,7 @@ def copy_table(target_schema, src_schema, table_name, **kwargs): try: target_table.insert1(t, skip_duplicates=True, **kwargs) except Exception: - print("Error when inserting {}".format(t)) + print(f"Error when inserting {t}") traceback.print_exc() diff --git a/u19_pipeline/ephys_pipeline.py b/u19_pipeline/ephys_pipeline.py index b312c124..dd6a52be 100644 --- a/u19_pipeline/ephys_pipeline.py +++ b/u19_pipeline/ephys_pipeline.py @@ -214,7 +214,7 @@ def get_spikeglx_meta_filepath(ephys_recording_key): spikeglx_meta_filepath = meta_filepath break else: - raise FileNotFoundError("No SpikeGLX data found for probe insertion: {}".format(ephys_recording_key)) + raise FileNotFoundError(f"No SpikeGLX data found for probe insertion: {ephys_recording_key}") return spikeglx_meta_filepath @@ -289,7 +289,7 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): all_vectors['time_as_behavior_fullsession_virmen'] = trial_times_full_virmen if single_vec_mode: - for this_key in all_vectors.keys(): + for this_key in all_vectors: print('this_key', this_key) print(all_vectors[this_key].shape) all_vectors[this_key] = np.concatenate(([x for x in all_vectors[this_key]]), axis=0) diff --git a/u19_pipeline/ephys_sync.py b/u19_pipeline/ephys_sync.py index 92d8d69a..3d18492e 100644 --- a/u19_pipeline/ephys_sync.py +++ b/u19_pipeline/ephys_sync.py @@ -1,14 +1,12 @@ -import datajoint as dj import pathlib -import numpy as np -from u19_pipeline import behavior -from u19_pipeline.ephys_pipeline import ephys_element, get_session_directory +import datajoint as dj +import numpy as np import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX import u19_pipeline.utils.ephys_utils as ephys_utils -import u19_pipeline.utils.path_utils as pu -import u19_pipeline.automatic_job.params_config as config +from u19_pipeline import behavior +from u19_pipeline.ephys_pipeline import ephys_element, get_session_directory # Tables downstream from `ephys_pipeline` module --------------------------------------- schema = dj.schema(dj.config['custom']['database.prefix'] + 'ephys_sync') diff --git a/u19_pipeline/imaging.py b/u19_pipeline/imaging.py index df5a0f1b..2c3d2e6e 100644 --- a/u19_pipeline/imaging.py +++ b/u19_pipeline/imaging.py @@ -1,6 +1,4 @@ import datajoint as dj -from u19_pipeline import acquisition - schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging') diff --git a/u19_pipeline/imaging_pipeline.py b/u19_pipeline/imaging_pipeline.py index 87acec7b..9012bef5 100644 --- a/u19_pipeline/imaging_pipeline.py +++ b/u19_pipeline/imaging_pipeline.py @@ -1,21 +1,16 @@ -import datajoint as dj import pathlib import subprocess from pathlib import Path -from u19_pipeline import lab, subject, recording -import u19_pipeline.automatic_job.params_config as config -import u19_pipeline.utils.dj_shortcuts as dj_short -import u19_pipeline.utils.tiff_utils as tu - - import datajoint as dj - -from element_calcium_imaging import scan as scan_element from element_calcium_imaging import imaging_preprocess as imaging_element from element_interface.utils import find_full_path +import u19_pipeline.automatic_job.params_config as config +import u19_pipeline.utils.dj_shortcuts as dj_short +import u19_pipeline.utils.tiff_utils as tu +from u19_pipeline import lab, recording, subject schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging_pipeline') @@ -255,7 +250,6 @@ class File(dj.Part): imaging_schema_name = dj.config['custom']['database.prefix'] + 'pipeline_imaging_element' # 2. Upstream tables ------------------------------------------------------------------- -from u19_pipeline.reference import BrainArea as Location Session = TiffSplit @@ -269,7 +263,6 @@ class Equipment(dj.Manual): """ # 3. Utility functions ----------------------------------------------------------------- -from u19_pipeline import recording_process def get_imaging_root_data_dir(): return dj.config.get('custom', {}).get('imaging_root_data_dir', None) diff --git a/u19_pipeline/meso.py b/u19_pipeline/meso.py index 50e346e1..2ded474c 100644 --- a/u19_pipeline/meso.py +++ b/u19_pipeline/meso.py @@ -1,7 +1,5 @@ import datajoint as dj import numpy as np -from u19_pipeline import acquisition - schema = dj.schema('u19_meso') diff --git a/u19_pipeline/meso_analysis.py b/u19_pipeline/meso_analysis.py index be1dd5f9..419423ea 100644 --- a/u19_pipeline/meso_analysis.py +++ b/u19_pipeline/meso_analysis.py @@ -1,5 +1,4 @@ import datajoint as dj -from u19_pipeline import meso schema = dj.schema('u19_meso_analysis') diff --git a/u19_pipeline/puffs.py b/u19_pipeline/puffs.py index ee8fe6f7..3394d129 100644 --- a/u19_pipeline/puffs.py +++ b/u19_pipeline/puffs.py @@ -1,7 +1,6 @@ """This module defines tables in the schema ahoag_puffs_lab_demo""" import datajoint as dj -from . import lab, acquisition, task schema = dj.schema(dj.config['custom']['database.prefix'] + 'puffs') diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 6748f93d..ddcbdaf6 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -1,7 +1,6 @@ + import datajoint as dj -import copy -from u19_pipeline import lab, subject, acquisition import u19_pipeline.automatic_job.params_config as config schema = dj.schema(dj.config['custom']['database.prefix'] + 'recording') diff --git a/u19_pipeline/recording_process.py b/u19_pipeline/recording_process.py index 3efaef30..bb60c46d 100644 --- a/u19_pipeline/recording_process.py +++ b/u19_pipeline/recording_process.py @@ -2,9 +2,7 @@ import pathlib import datajoint as dj -from u19_pipeline import recording -from u19_pipeline.imaging_pipeline import imaging_element -from u19_pipeline.ephys_pipeline import ephys_element + import u19_pipeline.automatic_job.params_config as config schema = dj.schema(dj.config['custom']['database.prefix'] + 'recording_process') diff --git a/u19_pipeline/task.py b/u19_pipeline/task.py index c7e6b1cd..9a422388 100644 --- a/u19_pipeline/task.py +++ b/u19_pipeline/task.py @@ -2,7 +2,6 @@ import datajoint as dj - schema = dj.schema(dj.config['custom']['database.prefix'] + 'task') diff --git a/u19_pipeline/temp/acquisition.py b/u19_pipeline/temp/acquisition.py index 87609495..4d14f722 100644 --- a/u19_pipeline/temp/acquisition.py +++ b/u19_pipeline/temp/acquisition.py @@ -1,7 +1,5 @@ import datajoint as dj -from u19_pipeline import lab, task, subject - schema = dj.schema('u19_acquisition') diff --git a/u19_pipeline/utility.py b/u19_pipeline/utility.py index 297c9262..e82a1ae2 100644 --- a/u19_pipeline/utility.py +++ b/u19_pipeline/utility.py @@ -1,14 +1,14 @@ -import sys import os -import pandas as pd -from pandas.api.types import is_numeric_dtype +import subprocess +import sys + +import datajoint as dj import numpy as np -from scipy.optimize import curve_fit +import pandas as pd from astropy.stats import binom_conf_interval -import datajoint as dj -import subprocess +from scipy.optimize import curve_fit def is_this_spock(): @@ -147,7 +147,7 @@ def psychFit(deltaBins, numR, numL, choices): # Correct deltaBin & trialBin to produce same result as Matlab psychFit deltaBins_search = deltaBins.astype(float) - 1.5 trialBin = np.searchsorted(deltaBins_search, nCues_RminusL, side='right') - trialBin -= 1; + trialBin -= 1 # Put into evidence bins all Trials with corresponding choices for iTrial in range(len(choices)): @@ -158,7 +158,7 @@ def psychFit(deltaBins, numR, numL, choices): trialDelta[trialBin[iTrial]] = trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial] with np.errstate(divide='ignore', invalid='ignore'): - trialDelta = np.true_divide(trialDelta, numTrials); + trialDelta = np.true_divide(trialDelta, numTrials) # Select only bins with trials idx_zero = numTrials == 0 @@ -203,7 +203,7 @@ def psychFit(deltaBins, numR, numL, choices): delta = np.linspace(deltaBins[0] - 2, deltaBins[-1] + 2, num=50) # Repeat trialDelta 3 times for errorX why ?? - errorX = np.tile(trialDelta[~idx_zero], 3); + errorX = np.tile(trialDelta[~idx_zero], 3) # Confidence intervals are errorY, as a vector errorY = np.stack(pci[:, ~idx_zero]) @@ -212,10 +212,10 @@ def psychFit(deltaBins, numR, numL, choices): # Fill dictionary of results fit_results = dict() fit_results['delta_bins'] = deltaBins[~idx_zero] - fit_results['delta_data'] = trialDelta[~idx_zero]; - fit_results['pright_data'] = 100 * phat[~idx_zero]; - fit_results['delta_error'] = errorX; - fit_results['pright_error'] = 100 * errorY; + fit_results['delta_data'] = trialDelta[~idx_zero] + fit_results['pright_data'] = 100 * phat[~idx_zero] + fit_results['delta_error'] = errorX + fit_results['pright_error'] = 100 * errorY if is_there_psychometric: fit_results['delta_fit'] = delta diff --git a/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py b/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py index 7fca605f..bcc85579 100644 --- a/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py +++ b/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Requires python 3 @@ -15,11 +14,11 @@ much easier! """ -import numpy as np -import matplotlib.pyplot as plt from pathlib import Path -from tkinter import Tk -from tkinter import filedialog +from tkinter import Tk, filedialog + +import matplotlib.pyplot as plt +import numpy as np # Parse ini file returning a dictionary whose keys are the metadata diff --git a/u19_pipeline/utils/dj_shortcuts.py b/u19_pipeline/utils/dj_shortcuts.py index 0e87490f..ff007fd6 100644 --- a/u19_pipeline/utils/dj_shortcuts.py +++ b/u19_pipeline/utils/dj_shortcuts.py @@ -1,5 +1,6 @@ import pandas as pd + def get_primary_key_fields(t): """ Get list of all fields that compose primary key diff --git a/u19_pipeline/utils/dlc_process.py b/u19_pipeline/utils/dlc_process.py index 7c659fe2..073872ee 100644 --- a/u19_pipeline/utils/dlc_process.py +++ b/u19_pipeline/utils/dlc_process.py @@ -1,15 +1,14 @@ import os -import deeplabcut -import pandas as pd -import numpy as np -import pickle import pathlib +import pickle import sys +import deeplabcut +import numpy as np +import pandas as pd from scipy import stats from skimage.measure import EllipseModel -from skimage.draw import ellipse_perimeter import u19_pipeline.utils.path_utils as pu diff --git a/u19_pipeline/utils/ephys_fix_sync_code.py b/u19_pipeline/utils/ephys_fix_sync_code.py index e4733778..a024f2d5 100644 --- a/u19_pipeline/utils/ephys_fix_sync_code.py +++ b/u19_pipeline/utils/ephys_fix_sync_code.py @@ -1,6 +1,4 @@ -import datetime -import pathlib import numpy as np import u19_pipeline.utils.ephys_utils as ephys_utils diff --git a/u19_pipeline/utils/ephys_utils.py b/u19_pipeline/utils/ephys_utils.py index 9aa18cf8..d9ade41a 100644 --- a/u19_pipeline/utils/ephys_utils.py +++ b/u19_pipeline/utils/ephys_utils.py @@ -1,16 +1,13 @@ -import numpy as np -import datajoint as dj +import json import pathlib import warnings -import json +import datajoint as dj +import numpy as np +from bitstring import BitArray from scipy import signal as sp from scipy.io import loadmat -from scipy.spatial.transform import Rotation as R - -from bitstring import BitArray -from element_array_ephys import ephys as ephys_element import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX @@ -762,7 +759,7 @@ def get_index_trial_vector_from_iteration(iteration_start_idx): return trial_start_idx -class xyz_pick_file_creator(): +class xyz_pick_file_creator: ''' Class that handles probe coordinates locations given initial isertion coordinates & shank coordinates ''' @@ -859,9 +856,9 @@ def get_probetrack(chanmap, shank=1, real_ml_coordinates=0, real_ap_coordinates= phi = phi_angle*np.pi/180 theta = theta_angle*np.pi/180 roll = rho_angle*np.pi/180 - x = np.array([i[0] for i in chanmap['xcoords']]); - y = np.array([i[0] for i in chanmap['ycoords']]); - k = np.array([i[0] for i in chanmap['kcoords']]); + x = np.array([i[0] for i in chanmap['xcoords']]) + y = np.array([i[0] for i in chanmap['ycoords']]) + k = np.array([i[0] for i in chanmap['kcoords']]) # Step 2: Transform them into 3D, assuming the Probe is perpendicular to x|y plane avx = np.mean(x[k==shank]) # Center "X" on middle of probe @@ -877,7 +874,7 @@ def get_probetrack(chanmap, shank=1, real_ml_coordinates=0, real_ap_coordinates= probe_track[i,:] = probe_length[i]*probe_unitVec + np.array([probe_x0[0], probe_x0[1], 0]) # Step 4: Shift probe my ML|AP insertion coordinates - probe_track_shifted = np.zeros((probe_track.shape)) + probe_track_shifted = np.zeros(probe_track.shape) for i in range(len(probe_track_shifted)): probe_track_shifted[i,:] = probe_track[i,:] + np.array([real_ml_coordinates*1000, real_ap_coordinates*1000, 0]) diff --git a/u19_pipeline/utils/file_utils.py b/u19_pipeline/utils/file_utils.py index 13b0e514..901a4529 100644 --- a/u19_pipeline/utils/file_utils.py +++ b/u19_pipeline/utils/file_utils.py @@ -1,5 +1,6 @@ import os + def write_file(path, text): os.umask(0) diff --git a/u19_pipeline/utils/insert_miscelaneous_db.py b/u19_pipeline/utils/insert_miscelaneous_db.py index a1378906..b34eb875 100644 --- a/u19_pipeline/utils/insert_miscelaneous_db.py +++ b/u19_pipeline/utils/insert_miscelaneous_db.py @@ -1,8 +1,9 @@ import os import pathlib -import numpy as np + import datajoint as dj +import numpy as np def add_researcher_user_table(user_id, full_name, email, phone): diff --git a/u19_pipeline/utils/matlab_utils.py b/u19_pipeline/utils/matlab_utils.py index 00aa28c5..dc7d7045 100644 --- a/u19_pipeline/utils/matlab_utils.py +++ b/u19_pipeline/utils/matlab_utils.py @@ -1,21 +1,22 @@ """Authors: Ben Dichter, Cody Baker.""" import os import sys +from collections.abc import Iterable +from datetime import datetime from pathlib import Path from shutil import which -import pandas as pd import numpy as np -from datetime import datetime +import pandas as pd from scipy.io import loadmat, matlab -from collections.abc import Iterable - try: from typing import ArrayLike except ImportError: + from collections.abc import Sequence + from typing import Union + from numpy import ndarray - from typing import Union, Sequence # adapted from numpy typing ArrayLike = Union[bool, int, float, complex, list, ndarray, Sequence] @@ -257,15 +258,15 @@ def convert_function_handle_to_str(mat_file_path): try: os.system(matlab_cmd) - with open("trial_choice.txt", "r") as f: + with open("trial_choice.txt") as f: trial_choice = f.read().splitlines() - with open("trial_type.txt", "r") as f: + with open("trial_type.txt") as f: trial_type = f.read().splitlines() - with open("shaping_protocol.txt", "r") as f: + with open("shaping_protocol.txt") as f: shaping_protocol = f.read().splitlines() - with open("code_version.txt", "r") as f: + with open("code_version.txt") as f: version = f.readline() - with open("protocol.txt", "r") as f: + with open("protocol.txt") as f: protocol = f.readline() metadata['experiment_name'] = version diff --git a/u19_pipeline/utils/path_utils.py b/u19_pipeline/utils/path_utils.py index 93fc391e..9cce7bdf 100644 --- a/u19_pipeline/utils/path_utils.py +++ b/u19_pipeline/utils/path_utils.py @@ -1,9 +1,7 @@ -import pathlib -import os import glob +import os import subprocess -import sys file_patterns_acq = { "raw_imaging": ['/*.tiff', '/*.tif'], diff --git a/u19_pipeline/utils/scp_transfers.py b/u19_pipeline/utils/scp_transfers.py index 3a8798d0..8191f77d 100644 --- a/u19_pipeline/utils/scp_transfers.py +++ b/u19_pipeline/utils/scp_transfers.py @@ -1,13 +1,16 @@ import os -import psutil import subprocess import sys -from paramiko import SSHClient, AutoAddPolicy, RSAKey + +import psutil +from paramiko import AutoAddPolicy, RSAKey, SSHClient from paramiko.auth_handler import AuthenticationException, SSHException -from scp import SCPClient, SCPException +from scp import SCPClient -from u19_pipeline.automatic_job.clusters_paths_and_transfers import public_key_location as public_key_location +from u19_pipeline.automatic_job.clusters_paths_and_transfers import ( + public_key_location as public_key_location, +) #Steps on windows machine # https://thesysadminchannel.com/solved-add-windowscapability-failed-error-code-0x800f0954-rsat-fix/ diff --git a/u19_pipeline/utils/subject_metadata.py b/u19_pipeline/utils/subject_metadata.py index 671fca8a..1fc0e6e3 100644 --- a/u19_pipeline/utils/subject_metadata.py +++ b/u19_pipeline/utils/subject_metadata.py @@ -1,4 +1,4 @@ -from typing import Iterable +from collections.abc import Iterable from u19_pipeline import lab, subject diff --git a/u19_pipeline/utils/tiff_matlab_imaging_utils.py b/u19_pipeline/utils/tiff_matlab_imaging_utils.py index 5a032170..2a7c71d9 100644 --- a/u19_pipeline/utils/tiff_matlab_imaging_utils.py +++ b/u19_pipeline/utils/tiff_matlab_imaging_utils.py @@ -1,14 +1,14 @@ +import ast +import re +import time from pathlib import Path + import numpy as np import tifffile as tiff from sklearn.linear_model import HuberRegressor -import time -import re -import ast - -xySizeFactor = 1.05; # images are this much larger than nominal size -zFactor = 1.45; # actual displacement in z vs command +xySizeFactor = 1.05 # images are this much larger than nominal size +zFactor = 1.45 # actual displacement in z vs command def select_files_from_mean_f( @@ -491,11 +491,6 @@ def extract_float(pattern, default=0): return header, parsed_info -from pathlib import Path -import re -import ast -import numpy as np -import tifffile as tiff def parse_tif_header_mesoscope( diff --git a/u19_pipeline/utils/tiff_utils.py b/u19_pipeline/utils/tiff_utils.py index f497fcda..764f9648 100644 --- a/u19_pipeline/utils/tiff_utils.py +++ b/u19_pipeline/utils/tiff_utils.py @@ -1,18 +1,16 @@ -import numpy as np import gzip -import shutil -import re import os -import tifffile - -from pathlib import Path +import re +import shutil from concurrent.futures import ProcessPoolExecutor from datetime import datetime +from pathlib import Path +import numpy as np +import tifffile import u19_pipeline.utils.tiff_matlab_imaging_utils as tmiu - tif_number_fmt = r'_[0-9]{5}\.tif' tif_gz_number_fmt = r'_[0-9]{5}\.tif\.gz' @@ -214,7 +212,7 @@ def create_scan_info_key(key, rec_info, bucket_dir): scan_info_key['last_good_file'] = rec_info['last_good_file'] if 'stacks_enabled' in scope: - scan_info_key['stacks_enabled'] = scope['stacks_enabled']; + scan_info_key['stacks_enabled'] = scope['stacks_enabled'] if 'stackActuator' in scope: scan_info_key['stack_actuator'] = scope['stackActuator'] From 6c2a88202b36de6cfa71f0afb503518f61f4363e Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Thu, 11 Jun 2026 21:40:24 -0400 Subject: [PATCH 2/6] ruff formatted files --- notebooks/Marking_dead_mice_batch.ipynb | 43 +- notebooks/TestAlertSystem.ipynb | 6 +- notebooks/TestAutomaticPipeline.ipynb | 726 ++++++++------ notebooks/ad_hoc_psychometrics_plot.ipynb | 361 ++++--- notebooks/behavior_metrics_performance.ipynb | 165 ++-- notebooks/check_pos_image_quality.ipynb | 119 ++- .../create_tech_responsibilites_table.ipynb | 305 +++--- notebooks/create_weighingGUI_ss.ipynb | 306 +++--- .../00-Datajoint-configuration.ipynb | 28 +- .../00-Set-up-configuration.ipynb | 8 +- .../01-ephys-workflow-structure.ipynb | 17 +- .../01-imaging-workflow-structure.ipynb | 21 +- .../02-process-ephys-workflow.ipynb | 188 ++-- .../02-process-imaging-workflow.ipynb | 158 +-- .../Creating_the_Scheduling_Table.ipynb | 45 +- .../brightness_video_analysis.ipynb | 66 +- .../delete_logs_periodically.ipynb | 37 +- .../delete_old_live_session_stats.ipynb | 15 +- ...ech_home_based_off_alvaros_big_query.ipynb | 40 +- .../review_ephys_sync.ipynb | 180 ++-- .../startup_time_analysis.ipynb | 398 +++++--- .../check_catgt_parameters_lfp.ipynb | 13 +- .../check_sync_notebook-new_method_bulk.ipynb | 379 +++++--- ...sync_notebook-new_method_one_session.ipynb | 676 +++++++------ ..._notebook-new_method_one_trial_check.ipynb | 296 +++--- .../20251119_sync_v4.ipynb | 150 +-- .../bulk_reprocess_ephys_recordings.ipynb | 66 +- .../check_daq_signals_jorge.ipynb | 167 ++-- .../check_sync_notebook-new_method.ipynb | 505 ++++++---- .../check_sync_notebook-new_method2.ipynb | 508 ++++++---- .../populate_sync_ephys_old_pipeline.ipynb | 135 ++- .../reingest_missing_sync_sessions_old.ipynb | 12 +- .../reingest_nan_sync_sessions.ipynb | 9 +- ...ync_all_sessions_include_virmen_time.ipynb | 145 +-- ..._sessions_include_virmen_time_status.ipynb | 41 +- .../get_ephys_sync_data.ipynb | 114 ++- .../get_full_vectos_from_sync_data.ipynb | 129 ++- ...t_full_vectos_from_sync_data_w_shift.ipynb | 160 +-- .../reingest_missing_sync_sessions.ipynb | 43 +- .../reingest_sync_by_virmen_block.ipynb | 14 +- .../ephys_element/read_ephys_parameters.ipynb | 2 +- .../ephys_element/test_paramspy_for_os.ipynb | 42 +- notebooks/fetch_session_data.ipynb | 51 +- .../00-Set-up-configuration.ipynb | 8 +- .../03-imaging-data-analysis.ipynb | 103 +- .../imaging_element/getSyncInfo_python.ipynb | 86 +- .../read_imaging_parameters.ipynb | 21 +- notebooks/imaging_element/read_params.py | 105 +- .../test_Matlab_Python_conv.ipynb | 12 +- notebooks/imaging_storage_calculator.ipynb | 88 +- notebooks/live_stats_monitor.ipynb | 361 ++++--- notebooks/numtrials_and_bias_check.ipynb | 187 ++-- notebooks/performance_metrics_2022.ipynb | 207 ++-- notebooks/psychometrics_plot.ipynb | 303 +++--- .../grade_pupillometry_processing.ipynb | 112 ++- .../insert_pupilometry_data_test.ipynb | 65 +- .../pupillometry_notebooks/pupillometry.ipynb | 259 ++--- .../pupillometry_final_notebook.ipynb | 185 ++-- .../review_pupillometry_data.ipynb | 114 ++- notebooks/read_virmen_behavior_file.ipynb | 30 +- .../storage_calculator_across_time.ipynb | 185 ++-- notebooks/training_days_per_protocol.ipynb | 291 ++++-- ...ore U19 data pipeline with DataJoint.ipynb | 67 +- ...a with U19 pipeline and save results.ipynb | 107 +- .../3-Build a simple data pipeline.ipynb | 67 +- .../Old_tutorials/2019/python_demo.ipynb | 87 +- .../202001/0-Get DataJoint Ready.ipynb | 8 +- .../1-Build a simple data pipeline.ipynb | 70 +- ...ore U19 data pipeline with DataJoint.ipynb | 65 +- ...a with U19 pipeline and save results.ipynb | 102 +- .../tutorials/Old_tutorials/202001/subject.py | 8 +- .../202103/0-Get DataJoint Ready-Copy1.ipynb | 6 +- .../202103/0-Get DataJoint Ready.ipynb | 6 +- ...ore U19 data pipeline with DataJoint.ipynb | 67 +- ...a with U19 pipeline and save results.ipynb | 107 +- .../3-Build a simple data pipeline.ipynb | 67 +- ...ghtsheet_datajoint_tutorial (Python).ipynb | 207 ++-- notebooks/water_weight_alert.ipynb | 284 +++--- notebooks/xyz_picks_notebook.ipynb | 89 +- scripts/__init__.py | 2 +- scripts/conf_file_finding.py | 32 +- scripts/drop_schemas.py | 2 +- scripts/ingest_hdf5_to_db.py | 604 +++++++----- scripts/rig_maintenance_demo.py | 38 +- u19_pipeline/acquisition.py | 2 +- .../alert_system/alert_code_skeleton.py | 16 +- .../alert_system/alert_system_utility.py | 39 +- u19_pipeline/alert_system/behavior_metrics.py | 104 +- u19_pipeline/alert_system/cronjob_alert.py | 3 +- .../alert_system/custom_alerts/__init__.py | 2 +- .../custom_alerts/braininit_storage.py | 21 +- .../alert_system/custom_alerts/rig_bias.py | 83 +- .../alert_system/custom_alerts/rig_trial.py | 88 +- .../custom_alerts/subject_bias.py | 54 +- .../custom_alerts/subject_trial.py | 47 +- .../cronjob_live_monitor_alert.py | 2 - .../live_monitor_alert/live_monitor_alert.py | 402 +++++--- .../cronjob_live_session_stats_deletion.py | 2 - .../live_session_stats_deletion.py | 12 +- .../cronjob_locked_tables_alert.py | 2 - .../locked_tables_alert.py | 12 +- .../log_deletion/cronjob_old_log_deletion.py | 2 - .../log_deletion/old_log_deletion.py | 40 +- .../alert_system/main_alert_system.py | 94 +- .../noDB_backup_creation_script.py | 253 +++-- .../rig_maintenance/check_rig_maintenance.py | 70 +- .../cronjob_schedule_check.py | 2 - .../schedule_check_alert.py | 75 +- .../alert_system/tech_alert/tech_alert.py | 54 +- .../create_weighing_gui_ss.py | 168 +++- .../cronjob_water_weigh_alert.py | 2 - .../water_weigh_alert/water_weigh_alert.py | 168 +++- .../clusters_paths_and_transfers.py | 436 +++++---- .../create_missing_lfps_script.py | 37 +- .../automatic_job/cronjob_automatic_job.py | 7 +- .../automatic_job/ephys_element_ingest.py | 43 +- .../automatic_job/ephys_element_populate.py | 113 ++- u19_pipeline/automatic_job/imaging_element.py | 216 ++-- .../automatic_job/imaging_element_populate.py | 108 +- .../automatic_job/parameter_file_creator.py | 141 +-- u19_pipeline/automatic_job/params_config.py | 375 ++++--- .../populate_missing_syncbehavior_ephys.py | 42 +- .../pupillometry_check_handler_script.py | 1 - .../automatic_job/pupillometry_handler.py | 459 ++++++--- .../pupillometry_handler_script.py | 1 - .../automatic_job/pupillometry_main.py | 8 +- .../automatic_job/recording_handler.py | 512 ++++++---- .../recording_process_handler.py | 919 ++++++++++++------ u19_pipeline/automatic_job/slurm_creator.py | 218 +++-- u19_pipeline/automatic_job/tigress2pni.py | 102 +- u19_pipeline/behavior.py | 5 +- u19_pipeline/copy_table.py | 230 +++-- u19_pipeline/ephys_pipeline.py | 192 ++-- u19_pipeline/ephys_sync.py | 137 ++- u19_pipeline/imaging.py | 2 +- u19_pipeline/imaging_pipeline.py | 210 ++-- u19_pipeline/lab.py | 44 +- u19_pipeline/lightsheet.py | 28 +- u19_pipeline/meso.py | 46 +- u19_pipeline/meso_analysis.py | 2 +- u19_pipeline/microsocope.py | 16 +- u19_pipeline/puffs.py | 7 +- u19_pipeline/pupillometry.py | 10 +- u19_pipeline/recording.py | 144 +-- u19_pipeline/recording_process.py | 72 +- u19_pipeline/reference.py | 111 ++- u19_pipeline/rig_maintenance.py | 16 +- u19_pipeline/scheduler.py | 2 + u19_pipeline/subject.py | 178 ++-- u19_pipeline/task.py | 9 +- u19_pipeline/temp/acquisition.py | 2 +- u19_pipeline/temp/behavior.py | 9 +- u19_pipeline/temp/imaging.py | 5 +- u19_pipeline/temp/meso.py | 5 +- u19_pipeline/temp/meso_analysis.py | 5 +- u19_pipeline/utility.py | 174 ++-- .../utils/DemoReadSGLXData/readSGLX.py | 257 ++--- u19_pipeline/utils/dj_shortcuts.py | 36 +- u19_pipeline/utils/dlc_process.py | 29 +- u19_pipeline/utils/ephys_fix_sync_code.py | 375 ++++--- u19_pipeline/utils/ephys_utils.py | 804 +++++++++------ u19_pipeline/utils/file_utils.py | 18 +- u19_pipeline/utils/insert_miscelaneous_db.py | 32 +- u19_pipeline/utils/matlab_utils.py | 136 +-- u19_pipeline/utils/path_utils.py | 36 +- u19_pipeline/utils/scp_transfers.py | 64 +- u19_pipeline/utils/slack_utils.py | 46 +- u19_pipeline/utils/subject_metadata.py | 22 +- .../utils/tiff_matlab_imaging_utils.py | 661 ++++--------- u19_pipeline/utils/tiff_utils.py | 398 +++----- 170 files changed, 12815 insertions(+), 9020 deletions(-) diff --git a/notebooks/Marking_dead_mice_batch.ipynb b/notebooks/Marking_dead_mice_batch.ipynb index 27925688..fd6fd9e4 100644 --- a/notebooks/Marking_dead_mice_batch.ipynb +++ b/notebooks/Marking_dead_mice_batch.ipynb @@ -29,15 +29,18 @@ ], "source": [ "query1 = 'subject_fullname like \"joun%\"'\n", - "query2 = 'subject_fullname != \"jounhong_Ai228_686\"' # This mouse is still alive\n", + "query2 = 'subject_fullname != \"jounhong_Ai228_686\"' # This mouse is still alive\n", "query3 = 'subject_status = \"InExperiments\"'\n", "query4 = 'effective_date < \"2024-08-01\"'\n", - "data = (action.SubjectStatus() * subject.Subject() & (' and '.join([query1, query2, query3, query4]))).fetch(format='frame')\n", + "data = (\n", + " action.SubjectStatus() * subject.Subject()\n", + " & (\" and \".join([query1, query2, query3, query4]))\n", + ").fetch(format=\"frame\")\n", "\n", - "df = data.reset_index().sort_values(by='effective_date', ascending=False)\n", + "df = data.reset_index().sort_values(by=\"effective_date\", ascending=False)\n", "\n", "\n", - "df['subject_fullname']" + "df[\"subject_fullname\"]" ] }, { @@ -58,8 +61,8 @@ "query1 = 'subject_fullname like \"joun%\"'\n", "\n", "# These will need to be properly handled in the future if the lens are not 0\n", - "print(len(subject.SubjectActionAutomatic() & query1 & ' valid_until_date is NULL'))\n", - "print(len(subject.SubjectActionManual() & query1 & ' valid_until_date is NULL'))" + "print(len(subject.SubjectActionAutomatic() & query1 & \" valid_until_date is NULL\"))\n", + "print(len(subject.SubjectActionManual() & query1 & \" valid_until_date is NULL\"))" ] }, { @@ -77,38 +80,28 @@ } ], "source": [ - "for sub in df['subject_fullname']:\n", - " key = {\n", - " 'subject_fullname': sub,\n", - " 'cage': '(grave)'}\n", + "for sub in df[\"subject_fullname\"]:\n", + " key = {\"subject_fullname\": sub, \"cage\": \"(grave)\"}\n", " subject.CagingStatus.update1(key)\n", "\n", - " key = {\n", - " 'subject_fullname': sub,\n", - " 'location': 'valhalla'}\n", + " key = {\"subject_fullname\": sub, \"location\": \"valhalla\"}\n", "\n", " subject.Subject.update1(key)\n", "\n", - " key = {\n", - " 'subject_fullname': sub,\n", - " 'death_date': '2025-02-05'\n", - " }\n", + " key = {\"subject_fullname\": sub, \"death_date\": \"2025-02-05\"}\n", "\n", " subject.Death.insert1(key, skip_duplicates=True)\n", "\n", " key = {\n", - " 'subject_fullname': sub,\n", - " 'effective_date': '2025-02-05',\n", - " 'subject_status': 'Dead',\n", - " 'water_per_day': 1,\n", - " 'schedule': 'Nothing/Nothing/Nothing/Nothing/Nothing/Nothing/Nothing'\n", + " \"subject_fullname\": sub,\n", + " \"effective_date\": \"2025-02-05\",\n", + " \"subject_status\": \"Dead\",\n", + " \"water_per_day\": 1,\n", + " \"schedule\": \"Nothing/Nothing/Nothing/Nothing/Nothing/Nothing/Nothing\",\n", " }\n", "\n", " action.SubjectStatus.insert1(key)\n", "\n", - "\n", - "\n", - "\n", " print(sub)" ] }, diff --git a/notebooks/TestAlertSystem.ipynb b/notebooks/TestAlertSystem.ipynb index 46a93538..8183f757 100644 --- a/notebooks/TestAlertSystem.ipynb +++ b/notebooks/TestAlertSystem.ipynb @@ -47,11 +47,9 @@ } ], "source": [ - "\n", "import datajoint as dj\n", "\n", - "import u19_pipeline.alert_system.main_alert_system as mas\n", - "\n" + "import u19_pipeline.alert_system.main_alert_system as mas" ] }, { @@ -154,7 +152,7 @@ } ], "source": [ - "mas.main_alert_system()\n" + "mas.main_alert_system()" ] }, { diff --git a/notebooks/TestAutomaticPipeline.ipynb b/notebooks/TestAutomaticPipeline.ipynb index 634813e0..61bb0d57 100644 --- a/notebooks/TestAutomaticPipeline.ipynb +++ b/notebooks/TestAutomaticPipeline.ipynb @@ -49,7 +49,7 @@ "import u19_pipeline.imaging_pipeline as imaging_pipeline\n", "from u19_pipeline import recording, recording_process\n", "from u19_pipeline.ephys_pipeline import ephys_element\n", - "from u19_pipeline.imaging_pipeline import imaging_element, scan_element\n" + "from u19_pipeline.imaging_pipeline import imaging_element, scan_element" ] }, { @@ -199,10 +199,10 @@ "metadata": {}, "outputs": [], "source": [ - "lab = dj.create_virtual_module('lab', 'u19_lab')\n", + "lab = dj.create_virtual_module(\"lab\", \"u19_lab\")\n", "user = dict()\n", - "user['user_id'] = 'nb1689'\n", - "user['slack_webhook'] = '(webhook url from slack API)222'\n", + "user[\"user_id\"] = \"nb1689\"\n", + "user[\"slack_webhook\"] = \"(webhook url from slack API)222\"\n", "lab.User.update1(user)" ] }, @@ -433,65 +433,119 @@ "update_value_dict = copy.deepcopy(config.default_update_value_dict)\n", "\n", "# Get jobs with error status or finished\n", - "df_inactive_jobs = rec_process_handler.RecProcessHandler.get_active_process_jobs(active=False)\n", + "df_inactive_jobs = rec_process_handler.RecProcessHandler.get_active_process_jobs(\n", + " active=False\n", + ")\n", "\n", - "jobs_ids = df_inactive_jobs['job_id'].to_frame().to_dict('records')\n", + "jobs_ids = df_inactive_jobs[\"job_id\"].to_frame().to_dict(\"records\")\n", "\n", - "#Get latest time they changed status\n", + "# Get latest time they changed status\n", "now = datetime.now()\n", - "timestamp_jobs = pd.DataFrame((recording_process.Processing & jobs_ids).aggr(recording_process.LogStatus, max_date=\"max(status_timestamp)\").fetch(as_dict=True))\n", - "df_inactive_jobs = df_inactive_jobs.merge(timestamp_jobs, on='job_id')\n", + "timestamp_jobs = pd.DataFrame(\n", + " (recording_process.Processing & jobs_ids)\n", + " .aggr(recording_process.LogStatus, max_date=\"max(status_timestamp)\")\n", + " .fetch(as_dict=True)\n", + ")\n", + "df_inactive_jobs = df_inactive_jobs.merge(timestamp_jobs, on=\"job_id\")\n", "\n", "# Get jobs that were processed in cluster\n", - "df_inactive_jobs['local_or_cluster'] = df_inactive_jobs['program_selection_params'].map(lambda x: x['local_or_cluster'])\n", - "df_inactive_jobs['process_cluster'] = df_inactive_jobs['program_selection_params'].map(lambda x: x['process_cluster'])\n", + "df_inactive_jobs[\"local_or_cluster\"] = df_inactive_jobs[\"program_selection_params\"].map(\n", + " lambda x: x[\"local_or_cluster\"]\n", + ")\n", + "df_inactive_jobs[\"process_cluster\"] = df_inactive_jobs[\"program_selection_params\"].map(\n", + " lambda x: x[\"process_cluster\"]\n", + ")\n", "\n", - "#Update all local jobs\n", - "df_not_cluster = df_inactive_jobs.loc[df_inactive_jobs['local_or_cluster'] == 'local'].reset_index(drop=True)\n", + "# Update all local jobs\n", + "df_not_cluster = df_inactive_jobs.loc[\n", + " df_inactive_jobs[\"local_or_cluster\"] == \"local\"\n", + "].reset_index(drop=True)\n", "if df_not_cluster.shape[0] > 0:\n", " # Update local jobs post-processed jobs\n", - " df_not_cluster_post_processed = df_not_cluster.loc[df_not_cluster['status_processing_id'] == config.JOB_STATUS_PROCESSED, :]\n", + " df_not_cluster_post_processed = df_not_cluster.loc[\n", + " df_not_cluster[\"status_processing_id\"] == config.JOB_STATUS_PROCESSED, :\n", + " ]\n", " df_not_cluster_post_processed = df_not_cluster_post_processed.reset_index(drop=True)\n", " for i in range(df_not_cluster_post_processed.shape[0]):\n", - " rec_process_handler.RecProcessHandler.update_status_pipeline(df_not_cluster_post_processed.loc[i, 'query_key'], config.JOB_STATUS_POST_PROCESSED)\n", - " rec_process_handler.RecProcessHandler.update_job_id_log(df_not_cluster_post_processed.loc[i, 'job_id'], config.JOB_STATUS_PROCESSED, config.JOB_STATUS_POST_PROCESSED, update_value_dict['error_info'])\n", + " rec_process_handler.RecProcessHandler.update_status_pipeline(\n", + " df_not_cluster_post_processed.loc[i, \"query_key\"],\n", + " config.JOB_STATUS_POST_PROCESSED,\n", + " )\n", + " rec_process_handler.RecProcessHandler.update_job_id_log(\n", + " df_not_cluster_post_processed.loc[i, \"job_id\"],\n", + " config.JOB_STATUS_PROCESSED,\n", + " config.JOB_STATUS_POST_PROCESSED,\n", + " update_value_dict[\"error_info\"],\n", + " )\n", "\n", " # Update also local jobs post-error jobs\n", - " df_not_cluster_post_error = df_not_cluster.loc[df_not_cluster['status_processing_id'] == config.JOB_STATUS_ERROR_ID, :]\n", + " df_not_cluster_post_error = df_not_cluster.loc[\n", + " df_not_cluster[\"status_processing_id\"] == config.JOB_STATUS_ERROR_ID, :\n", + " ]\n", " df_not_cluster_post_error = df_not_cluster_post_error.reset_index(drop=True)\n", " for i in range(df_not_cluster_post_error.shape[0]):\n", - " rec_process_handler.RecProcessHandler.update_status_pipeline(df_not_cluster_post_error.loc[i, 'query_key'], config.JOB_STATUS_ERROR_DELETED)\n", - " rec_process_handler.RecProcessHandler.update_job_id_log(df_not_cluster_post_error.loc[i, 'job_id'], config.JOB_STATUS_ERROR_ID, config.JOB_STATUS_ERROR_DELETED, update_value_dict['error_info'])\n", - "\n", - "#If no inactive job, after locals the end\n", - "df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs['local_or_cluster'] == 'cluster'].reset_index(drop=True)\n", + " rec_process_handler.RecProcessHandler.update_status_pipeline(\n", + " df_not_cluster_post_error.loc[i, \"query_key\"],\n", + " config.JOB_STATUS_ERROR_DELETED,\n", + " )\n", + " rec_process_handler.RecProcessHandler.update_job_id_log(\n", + " df_not_cluster_post_error.loc[i, \"job_id\"],\n", + " config.JOB_STATUS_ERROR_ID,\n", + " config.JOB_STATUS_ERROR_DELETED,\n", + " update_value_dict[\"error_info\"],\n", + " )\n", + "\n", + "# If no inactive job, after locals the end\n", + "df_inactive_jobs = df_inactive_jobs.loc[\n", + " df_inactive_jobs[\"local_or_cluster\"] == \"cluster\"\n", + "].reset_index(drop=True)\n", "\n", "df_inactive_jobsz = df_inactive_jobs.copy()\n", "\n", "print(df_inactive_jobs)\n", "\n", "# Get jobs to delete (finished or with error >= 7 days)\n", - "df_inactive_jobs['days_from_last_update'] = (now - df_inactive_jobs['max_date']).dt.days\n", - "df_inactive_jobs['ready_delete'] = 1\n", - "df_inactive_jobs.loc[(df_inactive_jobs['status_processing_id'] == config.JOB_STATUS_ERROR_ID) & (df_inactive_jobs['days_from_last_update'] < 7), 'ready_delete'] = 0\n", - "df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs['ready_delete'] == 1,:]\n", + "df_inactive_jobs[\"days_from_last_update\"] = (now - df_inactive_jobs[\"max_date\"]).dt.days\n", + "df_inactive_jobs[\"ready_delete\"] = 1\n", + "df_inactive_jobs.loc[\n", + " (df_inactive_jobs[\"status_processing_id\"] == config.JOB_STATUS_ERROR_ID)\n", + " & (df_inactive_jobs[\"days_from_last_update\"] < 7),\n", + " \"ready_delete\",\n", + "] = 0\n", + "df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs[\"ready_delete\"] == 1, :]\n", "\n", "# Check if raw directories are present in cluster\n", - "df_inactive_jobs['raw_dir'] = df_inactive_jobs.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_pre_path'],\\\n", - " x['process_cluster'], x['recording_modality'], type_dir='raw'), axis=1)\n", - "df_inactive_jobs['raw_dir'] = df_inactive_jobs['raw_dir'].astype(str)\n", - "df_inactive_jobs['raw_dir_deleted'] = 0\n", - "df_inactive_jobs.loc[df_inactive_jobs['raw_dir'] == \"0\", 'raw_dir_deleted'] = 1\n", + "df_inactive_jobs[\"raw_dir\"] = df_inactive_jobs.apply(\n", + " lambda x: ft.check_directory_exists_cluster(\n", + " x[\"recording_process_pre_path\"],\n", + " x[\"process_cluster\"],\n", + " x[\"recording_modality\"],\n", + " type_dir=\"raw\",\n", + " ),\n", + " axis=1,\n", + ")\n", + "df_inactive_jobs[\"raw_dir\"] = df_inactive_jobs[\"raw_dir\"].astype(str)\n", + "df_inactive_jobs[\"raw_dir_deleted\"] = 0\n", + "df_inactive_jobs.loc[df_inactive_jobs[\"raw_dir\"] == \"0\", \"raw_dir_deleted\"] = 1\n", "\n", "print(df_inactive_jobs)\n", "\n", "# Check if processed directories are present in cluster\n", - "df_inactive_jobs['processed_dir'] = df_inactive_jobs.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_post_path'],\\\n", - " x['process_cluster'], x['recording_modality'], type_dir='processed'), axis=1)\n", - "df_inactive_jobs['processed_dir'] = df_inactive_jobs['processed_dir'].astype(str)\n", - "df_inactive_jobs['processed_dir_deleted'] = 0\n", - "df_inactive_jobs.loc[df_inactive_jobs['processed_dir'] == \"0\", 'processed_dir_deleted'] = 1\n", - "df_inactive_jobs = df_inactive_jobs.reset_index(drop=True)\n" + "df_inactive_jobs[\"processed_dir\"] = df_inactive_jobs.apply(\n", + " lambda x: ft.check_directory_exists_cluster(\n", + " x[\"recording_process_post_path\"],\n", + " x[\"process_cluster\"],\n", + " x[\"recording_modality\"],\n", + " type_dir=\"processed\",\n", + " ),\n", + " axis=1,\n", + ")\n", + "df_inactive_jobs[\"processed_dir\"] = df_inactive_jobs[\"processed_dir\"].astype(str)\n", + "df_inactive_jobs[\"processed_dir_deleted\"] = 0\n", + "df_inactive_jobs.loc[\n", + " df_inactive_jobs[\"processed_dir\"] == \"0\", \"processed_dir_deleted\"\n", + "] = 1\n", + "df_inactive_jobs = df_inactive_jobs.reset_index(drop=True)" ] }, { @@ -600,11 +654,25 @@ } ], "source": [ - "df_inactive_jobsz['lala'] = df_inactive_jobsz.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_pre_path'],\\\n", - " x['process_cluster'], x['recording_modality'], type_dir='raw'), axis=1)\n", + "df_inactive_jobsz[\"lala\"] = df_inactive_jobsz.apply(\n", + " lambda x: ft.check_directory_exists_cluster(\n", + " x[\"recording_process_pre_path\"],\n", + " x[\"process_cluster\"],\n", + " x[\"recording_modality\"],\n", + " type_dir=\"raw\",\n", + " ),\n", + " axis=1,\n", + ")\n", "\n", - "sasa = df_inactive_jobsz.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_post_path'],\\\n", - " x['process_cluster'], x['recording_modality'], type_dir='processed'), axis=1)\n", + "sasa = df_inactive_jobsz.apply(\n", + " lambda x: ft.check_directory_exists_cluster(\n", + " x[\"recording_process_post_path\"],\n", + " x[\"process_cluster\"],\n", + " x[\"recording_modality\"],\n", + " type_dir=\"processed\",\n", + " ),\n", + " axis=1,\n", + ")\n", "sasa" ] }, @@ -636,7 +704,12 @@ } ], "source": [ - "su = ft.check_directory_exists_cluster(df_inactive_jobsz.loc[0,'recording_process_post_path'], 'tiger', 'electrophysiology', type_dir='processed')\n", + "su = ft.check_directory_exists_cluster(\n", + " df_inactive_jobsz.loc[0, \"recording_process_post_path\"],\n", + " \"tiger\",\n", + " \"electrophysiology\",\n", + " type_dir=\"processed\",\n", + ")\n", "su" ] }, @@ -693,37 +766,66 @@ "from u19_pipeline import utility\n", "\n", "rec_process_keys = dict()\n", - "rec_process_keys['sdsd'] = 207\n", + "rec_process_keys[\"sdsd\"] = 207\n", "\n", "\n", - "params_df = pd.DataFrame((imaging_pipeline.imaging_element.ProcessingParamSet.proj('params', 'processing_method') * \\\n", - "recording_process.Processing.ImagingParams.proj('paramset_idx') & rec_process_keys).fetch(as_dict=True))\n", - "params_df = params_df.drop('paramset_idx', axis=1)\n", + "params_df = pd.DataFrame(\n", + " (\n", + " imaging_pipeline.imaging_element.ProcessingParamSet.proj(\n", + " \"params\", \"processing_method\"\n", + " )\n", + " * recording_process.Processing.ImagingParams.proj(\"paramset_idx\")\n", + " & rec_process_keys\n", + " ).fetch(as_dict=True)\n", + ")\n", + "params_df = params_df.drop(\"paramset_idx\", axis=1)\n", "\n", - "#Insert processing_method in params itself (for BrainCogsImagingSorters)\n", - "params_df['params'] = params_df.apply(lambda x: {**x['params'], **{'processing_method':x['processing_method']}},axis=1)\n", + "# Insert processing_method in params itself (for BrainCogsImagingSorters)\n", + "params_df[\"params\"] = params_df.apply(\n", + " lambda x: {**x[\"params\"], **{\"processing_method\": x[\"processing_method\"]}}, axis=1\n", + ")\n", "\n", "# Get preprocess param sets\n", - "preparams_df = pd.DataFrame((imaging_pipeline.imaging_element.PreProcessParamSteps * \\\n", - "utility.smart_dj_join(imaging_pipeline.imaging_element.PreProcessParamSteps.Step, imaging_pipeline.imaging_element.PreProcessParamSet.proj('preprocess_method', 'params')) *\n", - "recording_process.Processing.ImagingParams.proj('preprocess_param_steps_id') & rec_process_keys).fetch(as_dict=True))\n", + "preparams_df = pd.DataFrame(\n", + " (\n", + " imaging_pipeline.imaging_element.PreProcessParamSteps\n", + " * utility.smart_dj_join(\n", + " imaging_pipeline.imaging_element.PreProcessParamSteps.Step,\n", + " imaging_pipeline.imaging_element.PreProcessParamSet.proj(\n", + " \"preprocess_method\", \"params\"\n", + " ),\n", + " )\n", + " * recording_process.Processing.ImagingParams.proj(\"preprocess_param_steps_id\")\n", + " & rec_process_keys\n", + " ).fetch(as_dict=True)\n", + ")\n", "\n", "if preparams_df.shape[0] > 0:\n", - "\n", - " # Join precluster params for the same recording_process\n", - " preparams_df['preparams'] = preparams_df.apply(lambda x : {x['preprocess_method']: x['params']}, axis=1)\n", - " preparams_df = preparams_df.sort_values(by=['job_id', 'step_number'])\n", - " preparams_df = preparams_df[['job_id', 'preparams']].groupby(\"job_id\").agg(lambda x: list(x))\n", - " preparams_df = preparams_df.reset_index()\n", + " # Join precluster params for the same recording_process\n", + " preparams_df[\"preparams\"] = preparams_df.apply(\n", + " lambda x: {x[\"preprocess_method\"]: x[\"params\"]}, axis=1\n", + " )\n", + " preparams_df = preparams_df.sort_values(by=[\"job_id\", \"step_number\"])\n", + " preparams_df = (\n", + " preparams_df[[\"job_id\", \"preparams\"]].groupby(\"job_id\").agg(lambda x: list(x))\n", + " )\n", + " preparams_df = preparams_df.reset_index()\n", "\n", "else:\n", - " preparams_df = pd.DataFrame((imaging_pipeline.imaging_element.PreProcessParamSteps * \\\n", - " recording_process.Processing.ImagingParams.proj('preprocess_param_steps_id') & rec_process_keys).fetch(as_dict=True))\n", - " preparams_df['preparams'] = None\n", + " preparams_df = pd.DataFrame(\n", + " (\n", + " imaging_pipeline.imaging_element.PreProcessParamSteps\n", + " * recording_process.Processing.ImagingParams.proj(\n", + " \"preprocess_param_steps_id\"\n", + " )\n", + " & rec_process_keys\n", + " ).fetch(as_dict=True)\n", + " )\n", + " preparams_df[\"preparams\"] = None\n", "\n", "params_df = params_df.merge(preparams_df)\n", "\n", - "print('params_df 3')\n", + "print(\"params_df 3\")\n", "print(params_df)" ] }, @@ -856,36 +958,37 @@ } ], "source": [ - "cat_gt_params = {\n", - " \"apfilter\": [\"biquad\",2,300,0],\n", - " \"gfix\": [0.40,0.10,0.02],\n", - " \"extras\": [\"prb_fld\", \"t_miss_ok\", \"ap\", \"gblcar\", \"out_prb_fld\"],\n", - " \"lazy\": True\n", - " }\n", - " \n", + "cat_gt_params = {\n", + " \"apfilter\": [\"biquad\", 2, 300, 0],\n", + " \"gfix\": [0.40, 0.10, 0.02],\n", + " \"extras\": [\"prb_fld\", \"t_miss_ok\", \"ap\", \"gblcar\", \"out_prb_fld\"],\n", + " \"lazy\": True,\n", + "}\n", + "\n", "ephys_element.PreClusterParamSet.insert_new_params(\n", - " precluster_method='catgt',\n", + " precluster_method=\"catgt\",\n", " paramset_idx=2,\n", " params=cat_gt_params,\n", - " paramset_desc='Spike sorting using Kilosort w lazy')\n", + " paramset_desc=\"Spike sorting using Kilosort w lazy\",\n", + ")\n", "\n", "\n", - "cat_gt_params = {\n", - " \"apfilter\": [\"biquad\",2,301,0],\n", - " \"gfix\": [0.40,0.10,0.02],\n", - " \"extras\": [\"prb_fld\", \"t_miss_ok\", \"ap\", \"gblcar\", \"out_prb_fld\"],\n", - " \"lazy\": True\n", - " }\n", - " \n", + "cat_gt_params = {\n", + " \"apfilter\": [\"biquad\", 2, 301, 0],\n", + " \"gfix\": [0.40, 0.10, 0.02],\n", + " \"extras\": [\"prb_fld\", \"t_miss_ok\", \"ap\", \"gblcar\", \"out_prb_fld\"],\n", + " \"lazy\": True,\n", + "}\n", + "\n", "ephys_element.PreClusterParamSet.insert_new_params(\n", - " precluster_method='catgt2',\n", + " precluster_method=\"catgt2\",\n", " paramset_idx=3,\n", " params=cat_gt_params,\n", - " paramset_desc='Spike sorting using Kilosort w lazy')\n", + " paramset_desc=\"Spike sorting using Kilosort w lazy\",\n", + ")\n", "\n", "\n", - "ephys_element.PreClusterParamSet()\n", - " \n" + "ephys_element.PreClusterParamSet()" ] }, { @@ -996,45 +1099,29 @@ } ], "source": [ - "paramlist = {'precluster_param_steps_id':0}\n", + "paramlist = {\"precluster_param_steps_id\": 0}\n", "ephys_element.PreClusterParamSteps.insert1(paramlist, skip_duplicates=True)\n", "\n", - "paramorder = {\n", - "'precluster_param_steps_id': 0,\n", - "'step_number': 1,\n", - "'paramset_idx': 0\n", - "}\n", + "paramorder = {\"precluster_param_steps_id\": 0, \"step_number\": 1, \"paramset_idx\": 0}\n", "ephys_element.PreClusterParamSteps.Step.insert1(paramorder, skip_duplicates=True)\n", "\n", "\n", - "paramlist = {'precluster_param_steps_id':1}\n", + "paramlist = {\"precluster_param_steps_id\": 1}\n", "ephys_element.PreClusterParamSteps.insert1(paramlist, skip_duplicates=True)\n", "\n", - "paramorder = {\n", - "'precluster_param_steps_id': 1,\n", - "'step_number': 1,\n", - "'paramset_idx': 1\n", - "}\n", + "paramorder = {\"precluster_param_steps_id\": 1, \"step_number\": 1, \"paramset_idx\": 1}\n", "ephys_element.PreClusterParamSteps.Step.insert1(paramorder, skip_duplicates=True)\n", "\n", "\n", - "paramlist = {'precluster_param_steps_id':2}\n", - "ephys_element.PreClusterParamSteps.insert1(paramlist,skip_duplicates=True)\n", + "paramlist = {\"precluster_param_steps_id\": 2}\n", + "ephys_element.PreClusterParamSteps.insert1(paramlist, skip_duplicates=True)\n", "\n", - "paramorder = {\n", - "'precluster_param_steps_id': 2,\n", - "'step_number': 1,\n", - "'paramset_idx': 0\n", - "}\n", + "paramorder = {\"precluster_param_steps_id\": 2, \"step_number\": 1, \"paramset_idx\": 0}\n", "ephys_element.PreClusterParamSteps.Step.insert1(paramorder, skip_duplicates=True)\n", - "paramorder = {\n", - "'precluster_param_steps_id': 2,\n", - "'step_number': 2,\n", - "'paramset_idx': 1\n", - "}\n", + "paramorder = {\"precluster_param_steps_id\": 2, \"step_number\": 2, \"paramset_idx\": 1}\n", "ephys_element.PreClusterParamSteps.Step.insert1(paramorder, skip_duplicates=True)\n", "\n", - "ephys_element.PreClusterParamSteps.Step()\n" + "ephys_element.PreClusterParamSteps.Step()" ] }, { @@ -1155,75 +1242,77 @@ ], "source": [ "params = {\n", - " \"fs\": 30000,\n", - " \"fshigh\": 150,\n", - " \"minfr_goodchannels\": 0.1,\n", - " \"Th\": [10, 4],\n", - " \"lam\": 10,\n", - " \"AUCsplit\": 0.9,\n", - " \"minFR\": 0.02,\n", - " \"momentum\": [20, 400],\n", - " \"sigmaMask\": 30,\n", - " \"ThPre\": 8,\n", - " \"CAR\": 1, \n", - " \"spkTh\": -6,\n", - " \"reorder\": 1,\n", - " \"nskip\": 25,\n", - " \"GPU\": 1,\n", - " \"Nfilt\": 1024,\n", - " \"nfilt_factor\": 4,\n", - " \"ntbuff\": 64,\n", - " \"NT\": 32832,\n", - " \"whiteningRange\": 32,\n", - " \"nSkipCov\": 25,\n", - " \"scaleproc\": 200,\n", - " \"nPCs\": 3,\n", - " \"useRAM\": 0,\n", - " \"trange\": [0, 1000000000],\n", - "\"NchanTOT\": 384\n", + " \"fs\": 30000,\n", + " \"fshigh\": 150,\n", + " \"minfr_goodchannels\": 0.1,\n", + " \"Th\": [10, 4],\n", + " \"lam\": 10,\n", + " \"AUCsplit\": 0.9,\n", + " \"minFR\": 0.02,\n", + " \"momentum\": [20, 400],\n", + " \"sigmaMask\": 30,\n", + " \"ThPre\": 8,\n", + " \"CAR\": 1,\n", + " \"spkTh\": -6,\n", + " \"reorder\": 1,\n", + " \"nskip\": 25,\n", + " \"GPU\": 1,\n", + " \"Nfilt\": 1024,\n", + " \"nfilt_factor\": 4,\n", + " \"ntbuff\": 64,\n", + " \"NT\": 32832,\n", + " \"whiteningRange\": 32,\n", + " \"nSkipCov\": 25,\n", + " \"scaleproc\": 200,\n", + " \"nPCs\": 3,\n", + " \"useRAM\": 0,\n", + " \"trange\": [0, 1000000000],\n", + " \"NchanTOT\": 384,\n", "}\n", "\n", "ephys_element.ClusteringParamSet.insert_new_params(\n", - " processing_method='kilosort2',\n", + " processing_method=\"kilosort2\",\n", " paramset_idx=0,\n", " params=params,\n", - " paramset_desc='Spike sorting using Kilosort2')\n", + " paramset_desc=\"Spike sorting using Kilosort2\",\n", + ")\n", "\n", "\n", "params = {\n", - " \"fs\": 30000,\n", - " \"fshigh\": 150,\n", - " \"minfr_goodchannels\": 0.1,\n", - " \"Th\": [10, 4],\n", - " \"lam\": 10,\n", - " \"AUCsplit\": 0.9,\n", - " \"minFR\": 0.02,\n", - " \"momentum\": [20, 400],\n", - " \"sigmaMask\": 30,\n", - " \"ThPre\": 8,\n", - " \"CAR\": 1, \n", - " \"spkTh\": -6,\n", - " \"reorder\": 1,\n", - " \"nskip\": 25,\n", - " \"GPU\": 2,\n", - " \"Nfilt\": 1024,\n", - " \"nfilt_factor\": 4,\n", - " \"ntbuff\": 64,\n", - " \"NT\": 32832,\n", - " \"whiteningRange\": 32,\n", - " \"nSkipCov\": 25,\n", - " \"scaleproc\": 200,\n", - " \"nPCs\": 3,\n", - " \"useRAM\": 0,\n", - " \"trange\": [0, 1000000000],\n", - "\"NchanTOT\": 384\n", + " \"fs\": 30000,\n", + " \"fshigh\": 150,\n", + " \"minfr_goodchannels\": 0.1,\n", + " \"Th\": [10, 4],\n", + " \"lam\": 10,\n", + " \"AUCsplit\": 0.9,\n", + " \"minFR\": 0.02,\n", + " \"momentum\": [20, 400],\n", + " \"sigmaMask\": 30,\n", + " \"ThPre\": 8,\n", + " \"CAR\": 1,\n", + " \"spkTh\": -6,\n", + " \"reorder\": 1,\n", + " \"nskip\": 25,\n", + " \"GPU\": 2,\n", + " \"Nfilt\": 1024,\n", + " \"nfilt_factor\": 4,\n", + " \"ntbuff\": 64,\n", + " \"NT\": 32832,\n", + " \"whiteningRange\": 32,\n", + " \"nSkipCov\": 25,\n", + " \"scaleproc\": 200,\n", + " \"nPCs\": 3,\n", + " \"useRAM\": 0,\n", + " \"trange\": [0, 1000000000],\n", + " \"NchanTOT\": 384,\n", "}\n", "\n", "ephys_element.ClusteringParamSet.insert_new_params(\n", - " processing_method='kilosort2',\n", + " processing_method=\"kilosort2\",\n", " paramset_idx=1,\n", " params=params,\n", - " paramset_desc='Spike sorting using Kilosort2')\n" + " paramset_desc=\"Spike sorting using Kilosort2\",\n", + ")" ] }, { @@ -1351,64 +1440,68 @@ } ], "source": [ - "\n", "ops = dict()\n", "# sample rate\n", - "ops['fs'] = 30000 \n", - "ops['fshigh'] = 300 \n", + "ops[\"fs\"] = 30000\n", + "ops[\"fshigh\"] = 300\n", "\n", "# minimum firing rate on a \"good\" channel (0 to skip)\n", - "ops['minfr_goodchannels'] = 0 \n", + "ops[\"minfr_goodchannels\"] = 0\n", "\n", "# threshold on projections (like in Kilosort1, can be different for last pass like [10 4])\n", - "ops['Th'] = [10, 4] \n", + "ops[\"Th\"] = [10, 4]\n", "\n", - "# how important is the amplitude penalty (like in Kilosort1, 0 means not used, 10 is average, 50 is a lot) \n", - "ops['lam'] = 10 \n", + "# how important is the amplitude penalty (like in Kilosort1, 0 means not used, 10 is average, 50 is a lot)\n", + "ops[\"lam\"] = 10\n", "\n", "# splitting a cluster at the end requires at least this much isolation for each sub-cluster (max = 1)\n", - "ops['AUCsplit'] = 0.9 \n", + "ops[\"AUCsplit\"] = 0.9\n", "\n", "# minimum spike rate (Hz), if a cluster falls below this for too long it gets removed\n", - "ops['minFR'] = 1/50 \n", + "ops[\"minFR\"] = 1 / 50\n", "\n", - "# number of samples to average over (annealed from first to second value) \n", - "ops['momentum'] = [20, 400] \n", + "# number of samples to average over (annealed from first to second value)\n", + "ops[\"momentum\"] = [20, 400]\n", "\n", "# spatial constant in um for computing residual variance of spike\n", - "ops['sigmaMask'] = 30 \n", + "ops[\"sigmaMask\"] = 30\n", "\n", "# threshold crossings for pre-clustering (in PCA projection space)\n", - "ops['ThPre'] = 8 \n", - "ops['reorder'] = 1 # whether to reorder batches for drift correction. \n", - "ops['nskip'] = 25 # how many batches to skip for determining spike PCs\n", + "ops[\"ThPre\"] = 8\n", + "ops[\"reorder\"] = 1 # whether to reorder batches for drift correction.\n", + "ops[\"nskip\"] = 25 # how many batches to skip for determining spike PCs\n", "\n", "# danger, changing these settings can lead to fatal errors\n", "# options for determining PCs\n", - "ops['spkTh'] = -6 # spike threshold in standard deviations (-6)\n", + "ops[\"spkTh\"] = -6 # spike threshold in standard deviations (-6)\n", "\n", - "ops['GPU'] = 1 # has to be 1, no CPU version yet, sorry\n", + "ops[\"GPU\"] = 1 # has to be 1, no CPU version yet, sorry\n", "# ops.Nfilt = 1024; # max number of clusters\n", - "ops['nfilt_factor'] = 4 # max number of clusters per good channel (even temporary ones)\n", - "ops['ntbuff'] = 64 # samples of symmetrical buffer for whitening and spike detection\n", - "ops['NT'] = 64*1024+ ops['ntbuff'] # must be multiple of 32 + ntbuff. This is the batch size (try decreasing if out of memory). \n", - "ops['whiteningRange'] = 32 # number of channels to use for whitening each channel\n", - "ops['nSkipCov'] = 25 # compute whitening matrix from every N-th batch\n", - "ops['scaleproc'] = 200 # int16 scaling of whitened data\n", - "ops['nPCs'] = 3 # how many PCs to project the spikes into\n", - "ops['useRAM'] = 0 # not yet available\n", - "\n", - "\n", - "ops['trange'] = [0, 100000000000000]\n", - "ops['NchanTOT'] = 384\n", - "ops['sig'] = 20 #spatial smoothness constant for registration\n", - "ops['nblocks'] = 5 # blocks for registration. 0 turns it off, 1 does rigid registration. Replaces \"datashift\" option. \n", + "ops[\"nfilt_factor\"] = 4 # max number of clusters per good channel (even temporary ones)\n", + "ops[\"ntbuff\"] = 64 # samples of symmetrical buffer for whitening and spike detection\n", + "ops[\"NT\"] = (\n", + " 64 * 1024 + ops[\"ntbuff\"]\n", + ") # must be multiple of 32 + ntbuff. This is the batch size (try decreasing if out of memory).\n", + "ops[\"whiteningRange\"] = 32 # number of channels to use for whitening each channel\n", + "ops[\"nSkipCov\"] = 25 # compute whitening matrix from every N-th batch\n", + "ops[\"scaleproc\"] = 200 # int16 scaling of whitened data\n", + "ops[\"nPCs\"] = 3 # how many PCs to project the spikes into\n", + "ops[\"useRAM\"] = 0 # not yet available\n", + "\n", + "\n", + "ops[\"trange\"] = [0, 100000000000000]\n", + "ops[\"NchanTOT\"] = 384\n", + "ops[\"sig\"] = 20 # spatial smoothness constant for registration\n", + "ops[\"nblocks\"] = (\n", + " 5 # blocks for registration. 0 turns it off, 1 does rigid registration. Replaces \"datashift\" option.\n", + ")\n", "\n", "ephys_element.ClusteringParamSet.insert_new_params(\n", - " processing_method='kilosort',\n", + " processing_method=\"kilosort\",\n", " paramset_idx=2,\n", " params=ops,\n", - " paramset_desc='Spike sorting using Kilosort2')\n", + " paramset_desc=\"Spike sorting using Kilosort2\",\n", + ")\n", "\n", "\n", "ephys_element.ClusteringParamSet()" @@ -1428,75 +1521,76 @@ "outputs": [], "source": [ "ops = {\n", - " \"look_one_level_down\": 0.0,\n", - " \"fast_disk\": [],\n", - " \"delete_bin\": False,\n", - " \"mesoscan\": False,\n", - " \"h5py\": [],\n", - " \"h5py_key\": \"data\",\n", - " \"save_path0\": [],\n", - " \"subfolders\": [],\n", - " \"nplanes\": 1,\n", - " \"nchannels\": 1,\n", - " \"functional_chan\": 1,\n", - " \"tau\": 1.0,\n", - " \"fs\": 10.0,\n", - " \"force_sktiff\": False,\n", - " \"preclassify\": 0.0,\n", - " \"save_mat\": False,\n", - " \"combined\": True,\n", - " \"aspect\": 1.0,\n", - " \"do_bidiphase\": False,\n", - " \"bidiphase\": 0.0,\n", - " \"do_registration\": True,\n", - " \"keep_movie_raw\": False,\n", - " \"nimg_init\": 300,\n", - " \"batch_size\": 500,\n", - " \"maxregshift\": 0.1,\n", - " \"align_by_chan\": 1,\n", - " \"reg_tif\": False,\n", - " \"reg_tif_chan2\": False,\n", - " \"subpixel\": 10,\n", - " \"smooth_sigma\": 1.15,\n", - " \"th_badframes\": 1.0,\n", - " \"pad_fft\": False,\n", - " \"nonrigid\": False,\n", - " \"block_size\": [128, 128],\n", - " \"snr_thresh\": 1.2,\n", - " \"maxregshiftNR\": 5.0,\n", - " \"1Preg\": False,\n", - " \"spatial_hp\": 50.0,\n", - " \"pre_smooth\": 2.0,\n", - " \"spatial_taper\": 50.0,\n", - " \"roidetect\": True,\n", - " \"sparse_mode\": False,\n", - " \"diameter\": 12,\n", - " \"spatial_scale\": 0,\n", - " \"connected\": True,\n", - " \"nbinned\": 5000,\n", - " \"max_iterations\": 20,\n", - " \"threshold_scaling\": 1.0,\n", - " \"max_overlap\": 0.75,\n", - " \"high_pass\": 100.0,\n", - " \"inner_neuropil_radius\": 2,\n", - " \"min_neuropil_pixels\": 350,\n", - " \"allow_overlap\": False,\n", - " \"chan2_thres\": 0.65,\n", - " \"baseline\": \"maximin\",\n", - " \"win_baseline\": 60.0,\n", - " \"sig_baseline\": 10.0,\n", - " \"prctile_baseline\": 8.0,\n", - " \"neucoeff\": 0.7,\n", - " \"xrange\": [0, 0],\n", - " \"yrange\": [0, 0]\n", + " \"look_one_level_down\": 0.0,\n", + " \"fast_disk\": [],\n", + " \"delete_bin\": False,\n", + " \"mesoscan\": False,\n", + " \"h5py\": [],\n", + " \"h5py_key\": \"data\",\n", + " \"save_path0\": [],\n", + " \"subfolders\": [],\n", + " \"nplanes\": 1,\n", + " \"nchannels\": 1,\n", + " \"functional_chan\": 1,\n", + " \"tau\": 1.0,\n", + " \"fs\": 10.0,\n", + " \"force_sktiff\": False,\n", + " \"preclassify\": 0.0,\n", + " \"save_mat\": False,\n", + " \"combined\": True,\n", + " \"aspect\": 1.0,\n", + " \"do_bidiphase\": False,\n", + " \"bidiphase\": 0.0,\n", + " \"do_registration\": True,\n", + " \"keep_movie_raw\": False,\n", + " \"nimg_init\": 300,\n", + " \"batch_size\": 500,\n", + " \"maxregshift\": 0.1,\n", + " \"align_by_chan\": 1,\n", + " \"reg_tif\": False,\n", + " \"reg_tif_chan2\": False,\n", + " \"subpixel\": 10,\n", + " \"smooth_sigma\": 1.15,\n", + " \"th_badframes\": 1.0,\n", + " \"pad_fft\": False,\n", + " \"nonrigid\": False,\n", + " \"block_size\": [128, 128],\n", + " \"snr_thresh\": 1.2,\n", + " \"maxregshiftNR\": 5.0,\n", + " \"1Preg\": False,\n", + " \"spatial_hp\": 50.0,\n", + " \"pre_smooth\": 2.0,\n", + " \"spatial_taper\": 50.0,\n", + " \"roidetect\": True,\n", + " \"sparse_mode\": False,\n", + " \"diameter\": 12,\n", + " \"spatial_scale\": 0,\n", + " \"connected\": True,\n", + " \"nbinned\": 5000,\n", + " \"max_iterations\": 20,\n", + " \"threshold_scaling\": 1.0,\n", + " \"max_overlap\": 0.75,\n", + " \"high_pass\": 100.0,\n", + " \"inner_neuropil_radius\": 2,\n", + " \"min_neuropil_pixels\": 350,\n", + " \"allow_overlap\": False,\n", + " \"chan2_thres\": 0.65,\n", + " \"baseline\": \"maximin\",\n", + " \"win_baseline\": 60.0,\n", + " \"sig_baseline\": 10.0,\n", + " \"prctile_baseline\": 8.0,\n", + " \"neucoeff\": 0.7,\n", + " \"xrange\": [0, 0],\n", + " \"yrange\": [0, 0],\n", "}\n", "\n", "\n", "imaging_element.ProcessingParamSet.insert_new_params(\n", - " processing_method='suite2p',\n", + " processing_method=\"suite2p\",\n", " paramset_idx=0,\n", " params=ops,\n", - " paramset_desc='Default params Suite2p')" + " paramset_desc=\"Default params Suite2p\",\n", + ")" ] }, { @@ -1513,11 +1607,13 @@ "outputs": [], "source": [ "PreProcessParamStepsRecord = dict()\n", - "PreProcessParamStepsRecord['preprocess_param_steps_id'] = 0 \n", - "PreProcessParamStepsRecord['preprocess_param_steps_name'] = 'No preprocessing'\n", - "PreProcessParamStepsRecord['preprocess_param_steps_desc'] = 'No preprocessing steps'\n", + "PreProcessParamStepsRecord[\"preprocess_param_steps_id\"] = 0\n", + "PreProcessParamStepsRecord[\"preprocess_param_steps_name\"] = \"No preprocessing\"\n", + "PreProcessParamStepsRecord[\"preprocess_param_steps_desc\"] = \"No preprocessing steps\"\n", "\n", - "imaging_element.PreProcessParamSteps.insert1(PreProcessParamStepsRecord, skip_duplicates=True)" + "imaging_element.PreProcessParamSteps.insert1(\n", + " PreProcessParamStepsRecord, skip_duplicates=True\n", + ")" ] }, { @@ -1667,70 +1763,69 @@ } ], "source": [ - "\n", "# For recording 39, all probes equal, list = 0\n", "default_params_key = dict()\n", - "default_params_key['recording_id'] = 39\n", - "default_params_key['fragment_number'] = 0\n", - "default_params_key['preprocess_param_steps_id'] = 0\n", - "default_params_key['paramset_idx'] = 0\n", + "default_params_key[\"recording_id\"] = 39\n", + "default_params_key[\"fragment_number\"] = 0\n", + "default_params_key[\"preprocess_param_steps_id\"] = 0\n", + "default_params_key[\"paramset_idx\"] = 0\n", "\n", "\n", "recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n", "\n", "# For recording 41, all probes equal, list = 2 (list 2 has two different preparams catgt & catgt2)\n", "default_params_key = dict()\n", - "default_params_key['recording_id'] = 41\n", - "default_params_key['fragment_number'] = 0\n", - "default_params_key['preprocess_param_steps_id'] = 2\n", - "default_params_key['paramset_idx'] = 0\n", + "default_params_key[\"recording_id\"] = 41\n", + "default_params_key[\"fragment_number\"] = 0\n", + "default_params_key[\"preprocess_param_steps_id\"] = 2\n", + "default_params_key[\"paramset_idx\"] = 0\n", "\n", "\n", "recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n", "\n", "# For recording 40, all probes different params\n", "default_params_key = dict()\n", - "default_params_key['recording_id'] = 40\n", - "default_params_key['fragment_number'] = 0\n", - "default_params_key['preprocess_param_steps_id'] = 0\n", - "default_params_key['paramset_idx'] = 0\n", - "default_params_key['default_same_preparams_all'] = 0\n", - "default_params_key['default_same_params_all'] = 0\n", + "default_params_key[\"recording_id\"] = 40\n", + "default_params_key[\"fragment_number\"] = 0\n", + "default_params_key[\"preprocess_param_steps_id\"] = 0\n", + "default_params_key[\"paramset_idx\"] = 0\n", + "default_params_key[\"default_same_preparams_all\"] = 0\n", + "default_params_key[\"default_same_params_all\"] = 0\n", "\n", "recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n", "\n", "default_params_key = dict()\n", - "default_params_key['recording_id'] = 40\n", - "default_params_key['fragment_number'] = 1\n", - "default_params_key['preprocess_param_steps_id'] = 0\n", - "default_params_key['paramset_idx'] = 1\n", - "default_params_key['default_same_preparams_all'] = 0\n", - "default_params_key['default_same_params_all'] = 0\n", + "default_params_key[\"recording_id\"] = 40\n", + "default_params_key[\"fragment_number\"] = 1\n", + "default_params_key[\"preprocess_param_steps_id\"] = 0\n", + "default_params_key[\"paramset_idx\"] = 1\n", + "default_params_key[\"default_same_preparams_all\"] = 0\n", + "default_params_key[\"default_same_params_all\"] = 0\n", "\n", "recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n", "\n", "default_params_key = dict()\n", - "default_params_key['recording_id'] = 40\n", - "default_params_key['fragment_number'] = 2\n", - "default_params_key['preprocess_param_steps_id'] = 1\n", - "default_params_key['paramset_idx'] = 0\n", - "default_params_key['default_same_preparams_all'] = 0\n", - "default_params_key['default_same_params_all'] = 0\n", + "default_params_key[\"recording_id\"] = 40\n", + "default_params_key[\"fragment_number\"] = 2\n", + "default_params_key[\"preprocess_param_steps_id\"] = 1\n", + "default_params_key[\"paramset_idx\"] = 0\n", + "default_params_key[\"default_same_preparams_all\"] = 0\n", + "default_params_key[\"default_same_params_all\"] = 0\n", "\n", "recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n", "\n", "default_params_key = dict()\n", - "default_params_key['recording_id'] = 40\n", - "default_params_key['fragment_number'] = 3\n", - "default_params_key['preprocess_param_steps_id'] = 1\n", - "default_params_key['paramset_idx'] = 1\n", - "default_params_key['default_same_preparams_all'] = 0\n", - "default_params_key['default_same_params_all'] = 0\n", + "default_params_key[\"recording_id\"] = 40\n", + "default_params_key[\"fragment_number\"] = 3\n", + "default_params_key[\"preprocess_param_steps_id\"] = 1\n", + "default_params_key[\"paramset_idx\"] = 1\n", + "default_params_key[\"default_same_preparams_all\"] = 0\n", + "default_params_key[\"default_same_params_all\"] = 0\n", "\n", "\n", "recording.DefaultParams.insert1(default_params_key, skip_duplicates=True)\n", "\n", - "recording.DefaultParams()\n" + "recording.DefaultParams()" ] }, { @@ -1746,14 +1841,16 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_pipeline = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')\n", - "ephys_element = dj.create_virtual_module('ephys_element', 'u19_pipeline_ephys_element')\n", - "probe_element = dj.create_virtual_module('probe_element', 'u19_pipeline_probe_element')\n", + "ephys_pipeline = dj.create_virtual_module(\"ephys_pipeline\", \"u19_ephys_pipeline\")\n", + "ephys_element = dj.create_virtual_module(\"ephys_element\", \"u19_pipeline_ephys_element\")\n", + "probe_element = dj.create_virtual_module(\"probe_element\", \"u19_pipeline_probe_element\")\n", "\n", "\n", - "imaging_pipeline = dj.create_virtual_module('imaging_pipeline', 'u19_imaging_pipeline')\n", - "scan_element = dj.create_virtual_module('scan_element', 'u19_pipeline_scan_element')\n", - "imaging_element = dj.create_virtual_module('imaging_element', 'u19_pipeline_imaging_element')\n" + "imaging_pipeline = dj.create_virtual_module(\"imaging_pipeline\", \"u19_imaging_pipeline\")\n", + "scan_element = dj.create_virtual_module(\"scan_element\", \"u19_pipeline_scan_element\")\n", + "imaging_element = dj.create_virtual_module(\n", + " \"imaging_element\", \"u19_pipeline_imaging_element\"\n", + ")" ] }, { @@ -1782,10 +1879,9 @@ "so = recording.Recording.fetch(\"KEY\", as_dict=True)\n", "\n", "\n", - "\n", "print(so[-1])\n", "print(get_session_directory(so[-1]))\n", - "get_spikeglx_meta_filepath(so[-1])\n" + "get_spikeglx_meta_filepath(so[-1])" ] }, { @@ -1806,7 +1902,14 @@ } ], "source": [ - "dj.ERD(recording) + dj.ERD(recording_process) + dj.ERD(ephys_pipeline) + dj.ERD(ephys_element.EphysRecording) + dj.ERD(ephys_element.PreClusterTask) + dj.ERD(ephys_element.PreClusterParamList)\n" + "(\n", + " dj.ERD(recording)\n", + " + dj.ERD(recording_process)\n", + " + dj.ERD(ephys_pipeline)\n", + " + dj.ERD(ephys_element.EphysRecording)\n", + " + dj.ERD(ephys_element.PreClusterTask)\n", + " + dj.ERD(ephys_element.PreClusterParamList)\n", + ")" ] }, { @@ -1827,7 +1930,14 @@ } ], "source": [ - "dj.ERD(recording) + dj.ERD(recording_process) + dj.ERD(imaging_pipeline) + dj.ERD(scan_element.Scan) + dj.ERD(imaging_element.ProcessingTask) + dj.ERD(imaging_element.ProcessingParamSet)\n" + "(\n", + " dj.ERD(recording)\n", + " + dj.ERD(recording_process)\n", + " + dj.ERD(imaging_pipeline)\n", + " + dj.ERD(scan_element.Scan)\n", + " + dj.ERD(imaging_element.ProcessingTask)\n", + " + dj.ERD(imaging_element.ProcessingParamSet)\n", + ")" ] }, { @@ -2055,8 +2165,8 @@ } ], "source": [ - "#ephys_pipeline_db = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')\n", - "#ephys_pipeline_db.EphysPipelineSession.delete()" + "# ephys_pipeline_db = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')\n", + "# ephys_pipeline_db.EphysPipelineSession.delete()" ] } ], diff --git a/notebooks/ad_hoc_psychometrics_plot.ipynb b/notebooks/ad_hoc_psychometrics_plot.ipynb index 5ec81983..0b171c5a 100644 --- a/notebooks/ad_hoc_psychometrics_plot.ipynb +++ b/notebooks/ad_hoc_psychometrics_plot.ipynb @@ -13,7 +13,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "import datajoint as dj\n", "import matplotlib.patches as mpatches\n", "import numpy as np\n", @@ -21,7 +20,7 @@ "import pylab as plt\n", "from matplotlib import cm\n", "\n", - "from u19_pipeline import utility\n" + "from u19_pipeline import utility" ] }, { @@ -74,11 +73,11 @@ "metadata": {}, "outputs": [], "source": [ - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n", - "optogenetics = dj.create_virtual_module('optogenetics', 'u19_optogenetics')\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "subject = dj.create_virtual_module('subject', 'u19_subject')\n", - "puffs = dj.create_virtual_module('puffs', 'u19_puffs')" + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", + "optogenetics = dj.create_virtual_module(\"optogenetics\", \"u19_optogenetics\")\n", + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n", + "puffs = dj.create_virtual_module(\"puffs\", \"u19_puffs\")" ] }, { @@ -94,36 +93,40 @@ "metadata": {}, "outputs": [], "source": [ - "#List of conditions to filter sessions, trials, etc for analysis\n", - "key_selection = ['subject_fullname like \"efonseca_Vgat%\"', 'level = 14', 'is_bad_session = 0']\n", - "\n", - "#List of tables to \"prefilter data\" (generally session or block tables)\n", + "# List of conditions to filter sessions, trials, etc for analysis\n", + "key_selection = [\n", + " 'subject_fullname like \"efonseca_Vgat%\"',\n", + " \"level = 14\",\n", + " \"is_bad_session = 0\",\n", + "]\n", + "\n", + "# List of tables to \"prefilter data\" (generally session or block tables)\n", "table_prefilters = [behavior.TowersBlock, acquisition.Session]\n", "\n", - "#List of conditions to compare (data will be filtered for all conditions and saved plotted)\n", - "condition_key = ['stim_on = 0', 'stim_on = 1']\n", - "#Labels for legend (if empty will be the smae as condition_key)\n", - "condition_labels = ['Laser Off', 'Laser On']\n", - "#List of Tables with all condition info, (normally trial tables)\n", + "# List of conditions to compare (data will be filtered for all conditions and saved plotted)\n", + "condition_key = [\"stim_on = 0\", \"stim_on = 1\"]\n", + "# Labels for legend (if empty will be the smae as condition_key)\n", + "condition_labels = [\"Laser Off\", \"Laser On\"]\n", + "# List of Tables with all condition info, (normally trial tables)\n", "data_tables = [behavior.TowersBlock().Trial(), optogenetics.OptogeneticSession().Trial]\n", "\n", - "#Extra field to split plots by (e.g. one plot per subject)\n", - "split_plots_by = 'subject_fullname'\n", + "# Extra field to split plots by (e.g. one plot per subject)\n", + "split_plots_by = \"subject_fullname\"\n", "\n", - "#Bins for trials .... (for now, manual selection)\n", - "#Recommendation. np.arange(min_evidence, max_evidence+1, step=(max_evidence - min_evidence)/10)\n", + "# Bins for trials .... (for now, manual selection)\n", + "# Recommendation. np.arange(min_evidence, max_evidence+1, step=(max_evidence - min_evidence)/10)\n", "deltaBins = np.arange(-15, 16, step=3)\n", "\n", - "#How to color plots \n", - "#('Pick colors from a list, from a perceptually uniform colormap, a qualitative colormap or custom colormap')\n", + "# How to color plots\n", + "# ('Pick colors from a list, from a perceptually uniform colormap, a qualitative colormap or custom colormap')\n", "# [list, uniform, qualitative, custom]\n", - "color_type = 'list'\n", + "color_type = \"list\"\n", "\n", - "#List of colors (if applicable, select from list)\n", - "color_list = ['k', 'b']\n", + "# List of colors (if applicable, select from list)\n", + "color_list = [\"k\", \"b\"]\n", "\n", - "#custom colormap name, (If applicable, select from custom colormap)\n", - "'''\n", + "# custom colormap name, (If applicable, select from custom colormap)\n", + "\"\"\"\n", "cmaps['Perceptually Uniform Sequential'] = [\n", " 'viridis', 'plasma', 'inferno', 'magma', 'cividis']\n", "cmaps['Sequential'] = [\n", @@ -146,14 +149,14 @@ " 'gnuplot', 'gnuplot2', 'CMRmap', 'cubehelix', 'brg',\n", " 'gist_rainbow', 'rainbow', 'jet', 'turbo', 'nipy_spectral',\n", " 'gist_ncar']\n", - "'''\n", - "custom_colormap = 'autumn'\n", + "\"\"\"\n", + "custom_colormap = \"autumn\"\n", "\n", - "fig_size = (13,14)\n", + "fig_size = (13, 14)\n", "\n", - "big_font_size = (20,16)\n", - "medium_font_size = (16,13)\n", - "small_font_size = (12,10)\n" + "big_font_size = (20, 16)\n", + "medium_font_size = (16, 13)\n", + "small_font_size = (12, 10)" ] }, { @@ -169,55 +172,68 @@ "metadata": {}, "outputs": [], "source": [ - "#Here are parameter examples for some use cases\n", - "example = 'None'\n", - "\n", - "#Compare stimulation trials vs non stimulation trials\n", - "if example == 'Optogenetic_on_vs_off':\n", - " key_selection = ['subject_fullname like \"efonseca_Vgat%\"', 'level = 14', 'is_bad_session = 0']\n", + "# Here are parameter examples for some use cases\n", + "example = \"None\"\n", + "\n", + "# Compare stimulation trials vs non stimulation trials\n", + "if example == \"Optogenetic_on_vs_off\":\n", + " key_selection = [\n", + " 'subject_fullname like \"efonseca_Vgat%\"',\n", + " \"level = 14\",\n", + " \"is_bad_session = 0\",\n", + " ]\n", " table_prefilters = [behavior.TowersBlock, acquisition.Session]\n", - " condition_key = ['stim_on = 0', 'stim_on = 1']\n", - " condition_labels = ['Laser Off', 'Laser On']\n", - " data_tables = [behavior.TowersBlock().Trial(), optogenetics.OptogeneticSession().Trial]\n", - " split_plots_by = 'subject_fullname'\n", - " \n", + " condition_key = [\"stim_on = 0\", \"stim_on = 1\"]\n", + " condition_labels = [\"Laser Off\", \"Laser On\"]\n", + " data_tables = [\n", + " behavior.TowersBlock().Trial(),\n", + " optogenetics.OptogeneticSession().Trial,\n", + " ]\n", + " split_plots_by = \"subject_fullname\"\n", + "\n", " deltaBins = np.arange(-15, 16, step=3)\n", - " color_type = 'list'\n", - " color_list = ['k', 'b']\n", - "elif example == 'Compare_subjects':\n", - " key_selection = ['level = 11', 'block_performance > 0.65', 'subject_fullname like \"%ms%\"']\n", + " color_type = \"list\"\n", + " color_list = [\"k\", \"b\"]\n", + "elif example == \"Compare_subjects\":\n", + " key_selection = [\n", + " \"level = 11\",\n", + " \"block_performance > 0.65\",\n", + " 'subject_fullname like \"%ms%\"',\n", + " ]\n", " table_prefilters = [behavior.TowersBlock]\n", - " #Here, the \"conditions to compare\" are the subjects themselves\n", - " condition_key = (subject.Subject & 'user_id like \"%ms%\"').fetch('subject_fullname', as_dict=True)\n", + " # Here, the \"conditions to compare\" are the subjects themselves\n", + " condition_key = (subject.Subject & 'user_id like \"%ms%\"').fetch(\n", + " \"subject_fullname\", as_dict=True\n", + " )\n", " condition_labels = []\n", " data_tables = [behavior.TowersBlock().Trial()]\n", - " split_plots_by = ''\n", - " \n", + " split_plots_by = \"\"\n", + "\n", " deltaBins = np.arange(-12, 13, step=3)\n", - " color_type = 'qualitative'\n", - " color_list = ['k', 'b']\n", - "elif example == 'MainLevel_vs_Training_level':\n", - " key_selection = ['level = 11', 'subject_fullname like \"emdiamanti_%\"']\n", + " color_type = \"qualitative\"\n", + " color_list = [\"k\", \"b\"]\n", + "elif example == \"MainLevel_vs_Training_level\":\n", + " key_selection = [\"level = 11\", 'subject_fullname like \"emdiamanti_%\"']\n", " table_prefilters = [acquisition.Session]\n", - " condition_key = ['level = main_level', 'level<>main_level']\n", - " condition_labels = ['Main Level', 'Training Level']\n", + " condition_key = [\"level = main_level\", \"level<>main_level\"]\n", + " condition_labels = [\"Main Level\", \"Training Level\"]\n", " data_tables = [behavior.TowersBlock, behavior.TowersBlock().Trial()]\n", - " split_plots_by = 'subject_fullname'\n", - " \n", + " split_plots_by = \"subject_fullname\"\n", + "\n", " deltaBins = np.arange(-15, 16, step=3)\n", - " color_type = 'list'\n", - " color_list = ['r', 'k']\n", - "elif example == 'PuffsTrials':\n", - " key_selection = ['level=9', 'rig = 1']\n", + " color_type = \"list\"\n", + " color_list = [\"r\", \"k\"]\n", + "elif example == \"PuffsTrials\":\n", + " key_selection = [\"level=9\", \"rig = 1\"]\n", " table_prefilters = [puffs.PuffsSession(), puffs.PuffsSession().TrialOld()]\n", - " condition_key = ['trial_duration < 11.5', 'trial_duration > 11.5']\n", + " condition_key = [\"trial_duration < 11.5\", \"trial_duration > 11.5\"]\n", " condition_labels = []\n", " data_tables = [puffs.PuffsSession().TrialOld()]\n", - " split_plots_by = ''\n", - " \n", + " split_plots_by = \"\"\n", + "\n", " deltaBins = np.arange(-12, 13, step=3)\n", - " color_type = 'list'\n", - " color_list = ['r', 'k']\n" + " color_type = \"list\"\n", + " color_list = [\"r\", \"k\"]" ] }, { @@ -233,62 +249,69 @@ "metadata": {}, "outputs": [], "source": [ - "#Transform dictionary of condition keys to a list of strings\n", + "# Transform dictionary of condition keys to a list of strings\n", "if type(condition_key[0]) is dict:\n", - " if isinstance(list(condition_key[0].values())[0],str):\n", + " if isinstance(list(condition_key[0].values())[0], str):\n", " ap_str = '\"'\n", " else:\n", - " ap_str = ''\n", - " aux_condition_var = [list(x.keys())[0]+ ' = ' + ap_str + list(x.values())[0] + ap_str for x in condition_key]\n", + " ap_str = \"\"\n", + " aux_condition_var = [\n", + " list(x.keys())[0] + \" = \" + ap_str + list(x.values())[0] + ap_str\n", + " for x in condition_key\n", + " ]\n", " condition_key = aux_condition_var\n", - " \n", + "\n", "if len(condition_labels) == 0:\n", " condition_labels = condition_key\n", "\n", "\n", - "if color_type == 'list':\n", - " #If we have enough colors on the list:\n", + "if color_type == \"list\":\n", + " # If we have enough colors on the list:\n", " if len(condition_labels) <= len(color_list):\n", - " colors = color_list[0:len(condition_labels)]\n", - " #If not, we will check if there are less than 10 things to plot \n", + " colors = color_list[0 : len(condition_labels)]\n", + " # If not, we will check if there are less than 10 things to plot\n", " elif len(condition_labels) <= 10:\n", - " color_type = 'qualitative'\n", - " print('Not enough colors on list, swithcing to qualitative colormap')\n", - " #If more than 10 things to plot, let's go with uniform colormap\n", + " color_type = \"qualitative\"\n", + " print(\"Not enough colors on list, swithcing to qualitative colormap\")\n", + " # If more than 10 things to plot, let's go with uniform colormap\n", " else:\n", - " color_type = 'uniform'\n", - " print('Not enough colors on list, swithcing to uniform colormap')\n", - " \n", - "if color_type == 'qualitative':\n", + " color_type = \"uniform\"\n", + " print(\"Not enough colors on list, swithcing to uniform colormap\")\n", + "\n", + "if color_type == \"qualitative\":\n", " if len(condition_labels) <= 10:\n", - " colors = cm.get_cmap('tab10', 10)\n", - " colors = colors(np.linspace(0,len(condition_labels)/10,len(condition_labels)))\n", + " colors = cm.get_cmap(\"tab10\", 10)\n", + " colors = colors(\n", + " np.linspace(0, len(condition_labels) / 10, len(condition_labels))\n", + " )\n", " else:\n", - " color_type = 'uniform'\n", - " print('Not enough colors on qualitative colormap, swithcing to uniform colormap')\n", - " \n", - "if color_type == 'uniform':\n", - " colors = cm.get_cmap('viridis', len(condition_labels))\n", + " color_type = \"uniform\"\n", + " print(\n", + " \"Not enough colors on qualitative colormap, swithcing to uniform colormap\"\n", + " )\n", + "\n", + "if color_type == \"uniform\":\n", + " colors = cm.get_cmap(\"viridis\", len(condition_labels))\n", "\n", - "if color_type == 'custom':\n", + "if color_type == \"custom\":\n", " colors = cm.get_cmap(custom_colormap, len(condition_labels))\n", "\n", - "if color_type != 'list' and color_type != 'qualitative':\n", - " colors = colors(np.linspace(0,1,len(condition_labels)))\n", - " \n", + "if color_type != \"list\" and color_type != \"qualitative\":\n", + " colors = colors(np.linspace(0, 1, len(condition_labels)))\n", "\n", - "#Assign a color to each condition\n", - "zip_iterator = zip(condition_key, colors)\n", + "\n", + "# Assign a color to each condition\n", + "zip_iterator = zip(condition_key, colors)\n", "color_dict_keys = dict(zip_iterator)\n", "\n", - "#Assign a color to each label\n", + "# Assign a color to each label\n", "zip_iterator = zip(condition_labels, colors)\n", "color_dict = dict(zip_iterator)\n", "\n", - "#Create a color patch for plot legend\n", + "# Create a color patch for plot legend\n", "patches_legend = []\n", "for key in color_dict:\n", - " patches_legend.append(mpatches.Patch(color=color_dict[key], label=key))\n" + " patches_legend.append(mpatches.Patch(color=color_dict[key], label=key))" ] }, { @@ -308,8 +331,8 @@ "table_filter = table_prefilters[0]\n", "\n", "if len(table_prefilters) > 1:\n", - " for i in range(1,len(table_prefilters)):\n", - " table_filter = utility.smart_dj_join(table_filter,table_prefilters[i])\n", + " for i in range(1, len(table_prefilters)):\n", + " table_filter = utility.smart_dj_join(table_filter, table_prefilters[i])\n", "\n", "# Filter by all conditions selected\n", "for filter_key in key_selection:\n", @@ -318,9 +341,8 @@ "# Join all data tables\n", "data_table = data_tables[0]\n", "if len(data_tables) > 1:\n", - " for i in range(1,len(data_tables)):\n", - " data_table = utility.smart_dj_join(data_table,data_tables[i]) \n", - "\n" + " for i in range(1, len(data_tables)):\n", + " data_table = utility.smart_dj_join(data_table, data_tables[i])" ] }, { @@ -336,7 +358,7 @@ "metadata": {}, "outputs": [], "source": [ - "plots_records = ['']\n", + "plots_records = [\"\"]\n", "if len(split_plots_by) > 0:\n", " plots_records = table_filter.fetch(split_plots_by)\n", " plots_records = list(set(plots_records))\n", @@ -367,87 +389,108 @@ } ], "source": [ - "df_summary = pd.DataFrame(columns = [split_plots_by, 'Type', '%Correct', 'Total trials', 'Sigmoid fit'])\n", - "\n", - "rows, cols = utility.get_cols_rows_plot(num_figs, fig_size) \n", - "fig, axs = plt.subplots(rows, cols, figsize=fig_size, sharex=True, sharey=True, squeeze=False)\n", + "df_summary = pd.DataFrame(\n", + " columns=[split_plots_by, \"Type\", \"%Correct\", \"Total trials\", \"Sigmoid fit\"]\n", + ")\n", + "\n", + "rows, cols = utility.get_cols_rows_plot(num_figs, fig_size)\n", + "fig, axs = plt.subplots(\n", + " rows, cols, figsize=fig_size, sharex=True, sharey=True, squeeze=False\n", + ")\n", "renderer = fig.canvas.get_renderer()\n", "\n", - "#Decide font size for plots\n", - "if rows*cols >= 9:\n", + "# Decide font size for plots\n", + "if rows * cols >= 9:\n", " ac_fs = small_font_size\n", - "elif rows*cols >= 5:\n", + "elif rows * cols >= 5:\n", " ac_fs = medium_font_size\n", "else:\n", " ac_fs = big_font_size\n", "\n", "for split_idx, split in enumerate(plots_records):\n", + " print(split_idx + 1, \"/\", len(plots_records))\n", + "\n", + " ac_col = split_idx % cols\n", + " ac_row = int(split_idx / cols)\n", "\n", - " print(split_idx+1, '/', len(plots_records))\n", - " \n", - " ac_col = split_idx%cols\n", - " ac_row = int(split_idx/cols)\n", - " \n", - " #Create dictionary for split key plot (if applicable)\n", + " # Create dictionary for split key plot (if applicable)\n", " split_key = dict()\n", " split_key[split_plots_by] = split\n", "\n", " for key_idx, key in enumerate(condition_key):\n", - "\n", - " #Fetch session info (if split, add split_key to query)\n", + " # Fetch session info (if split, add split_key to query)\n", " if len(split_plots_by) > 0:\n", " thisCondition = data_table & table_filter.proj() & key & split_key\n", " else:\n", " thisCondition = data_table & table_filter.proj() & key\n", - " \n", + "\n", " # Fetch all fields of table except external (for speed)\n", " if key_idx == 0:\n", - " fields_query = pd.DataFrame.from_dict(thisCondition.heading.attributes, orient='index')\n", - " fields_query = fields_query.loc[fields_query['is_external'] == False, :]\n", + " fields_query = pd.DataFrame.from_dict(\n", + " thisCondition.heading.attributes, orient=\"index\"\n", + " )\n", + " fields_query = fields_query.loc[fields_query[\"is_external\"] == False, :]\n", " fields_query = fields_query.index.to_list()\n", "\n", - " # Fetch session \n", + " # Fetch session\n", " session_info = pd.DataFrame(thisCondition.fetch(*fields_query, as_dict=True))\n", "\n", " # Choice and trial type as integer\n", " if session_info.shape[0] > 0:\n", - "\n", " session_info = utility.translate_choice_trials_cues(session_info)\n", "\n", " # Call Fit to sigmoid function\n", - " fit_dict = utility.psychFit(deltaBins, session_info['cue_presence_right'].values, \\\n", - " session_info['cue_presence_left'].values, session_info['choice_int'].values)\n", + " fit_dict = utility.psychFit(\n", + " deltaBins,\n", + " session_info[\"cue_presence_right\"].values,\n", + " session_info[\"cue_presence_left\"].values,\n", + " session_info[\"choice_int\"].values,\n", + " )\n", "\n", " dict_summary = dict()\n", - " dict_summary[split_plots_by] = split\n", - " dict_summary['Type'] = key\n", - " dict_summary['%Correct'] = np.sum(session_info['choice'].values == session_info['trial_type'].values)/session_info.shape[0]\n", - " dict_summary['Total trials'] = session_info.shape[0]\n", - " dict_summary['Sigmoid fit'] = fit_dict\n", - " \n", - " df_summary = df_summary.append(dict_summary, ignore_index = True)\n", - "\n", - " #Select color from color dictionary\n", + " dict_summary[split_plots_by] = split\n", + " dict_summary[\"Type\"] = key\n", + " dict_summary[\"%Correct\"] = (\n", + " np.sum(\n", + " session_info[\"choice\"].values == session_info[\"trial_type\"].values\n", + " )\n", + " / session_info.shape[0]\n", + " )\n", + " dict_summary[\"Total trials\"] = session_info.shape[0]\n", + " dict_summary[\"Sigmoid fit\"] = fit_dict\n", + "\n", + " df_summary = df_summary.append(dict_summary, ignore_index=True)\n", + "\n", + " # Select color from color dictionary\n", " color_plot = color_dict_keys[key]\n", - " #Plot results\n", - " axs[ac_row, ac_col].plot(fit_dict['delta_bins'], fit_dict['pright_data'], 'o', color = color_plot)\n", - " if fit_dict['delta_fit'].shape[0] > 0:\n", - " axs[ac_row, ac_col].plot(fit_dict['delta_fit'], fit_dict['pright_fit'], '--', color = color_plot)\n", - " \n", + " # Plot results\n", + " axs[ac_row, ac_col].plot(\n", + " fit_dict[\"delta_bins\"], fit_dict[\"pright_data\"], \"o\", color=color_plot\n", + " )\n", + " if fit_dict[\"delta_fit\"].shape[0] > 0:\n", + " axs[ac_row, ac_col].plot(\n", + " fit_dict[\"delta_fit\"],\n", + " fit_dict[\"pright_fit\"],\n", + " \"--\",\n", + " color=color_plot,\n", + " )\n", + "\n", " # Axes, legends and labels\n", " if ac_col == 0:\n", " axs[ac_row, ac_col].set_ylabel(\"% Turned left\", fontsize=ac_fs[1])\n", - " \n", + "\n", " axs[ac_row, ac_col].set_ylim([-1, 101])\n", - " mean_correct_string = f\"{df_summary.loc[df_summary[split_plots_by] == split, '%Correct'].mean()*100:2.1f}\"\n", + " mean_correct_string = f\"{df_summary.loc[df_summary[split_plots_by] == split, '%Correct'].mean() * 100:2.1f}\"\n", " if len(split_plots_by) > 7:\n", - " title_label = str(split) + '-%Correct: ' + mean_correct_string\n", + " title_label = str(split) + \"-%Correct: \" + mean_correct_string\n", " else:\n", - " title_label = split_plots_by + \" : \" + str(split) + '%Correct: ' + mean_correct_string\n", + " title_label = (\n", + " split_plots_by + \" : \" + str(split) + \"%Correct: \" + mean_correct_string\n", + " )\n", " axs[ac_row, ac_col].set_title(title_label, fontsize=ac_fs[0])\n", - " \n", + "\n", " fig.canvas.draw()\n", - " plt.draw() \n", + " plt.draw()\n", " axs[ac_row, ac_col].draw(renderer)\n", "\n", " xlabels = axs[ac_row, ac_col].get_xticklabels()\n", @@ -459,10 +502,10 @@ " if len(ylabels) > 0:\n", " axs[ac_row, ac_col].set_yticklabels(ylabels, fontsize=ac_fs[1])\n", "\n", - "for i in range(0,cols):\n", - " axs[rows-1, i].set_xlabel(\"total evidence (L-R)\", fontsize=ac_fs[1])\n", - " \n", - "axs[rows-1, cols-1].legend(handles=patches_legend, fontsize=ac_fs[0])\n" + "for i in range(0, cols):\n", + " axs[rows - 1, i].set_xlabel(\"total evidence (L-R)\", fontsize=ac_fs[1])\n", + "\n", + "axs[rows - 1, cols - 1].legend(handles=patches_legend, fontsize=ac_fs[0])" ] }, { @@ -590,17 +633,19 @@ } ], "source": [ - "mean_correct = df_summary['%Correct'].mean()\n", - "total_trials = df_summary['Total trials'].sum()\n", + "mean_correct = df_summary[\"%Correct\"].mean()\n", + "total_trials = df_summary[\"Total trials\"].sum()\n", "\n", - "total_df = pd.DataFrame([['', 'Totals', mean_correct, total_trials, None]], columns=df_summary.columns)\n", + "total_df = pd.DataFrame(\n", + " [[\"\", \"Totals\", mean_correct, total_trials, None]], columns=df_summary.columns\n", + ")\n", "\n", "df_final = df_summary.append(total_df, ignore_index=True)\n", "\n", "if len(split_plots_by) > 0:\n", - " df_final = df_final.set_index([split_plots_by, 'Type'])\n", + " df_final = df_final.set_index([split_plots_by, \"Type\"])\n", "else:\n", - " df_final = df_final.set_index(['Type'])\n", + " df_final = df_final.set_index([\"Type\"])\n", "df_final" ] }, diff --git a/notebooks/behavior_metrics_performance.ipynb b/notebooks/behavior_metrics_performance.ipynb index 7bd9f21c..2d9e11aa 100644 --- a/notebooks/behavior_metrics_performance.ipynb +++ b/notebooks/behavior_metrics_performance.ipynb @@ -30,8 +30,7 @@ "import datajoint as dj\n", "import numpy as np\n", "import pandas as pd\n", - "import pylab as plt\n", - "\n" + "import pylab as plt" ] }, { @@ -540,14 +539,21 @@ } ], "source": [ - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('acquisition', 'u19_behavior')\n", - "subject = dj.create_virtual_module('acquisition', 'u19_subject')\n", - "recording = dj.create_virtual_module('recording', 'u19_recording')\n", + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"acquisition\", \"u19_behavior\")\n", + "subject = dj.create_virtual_module(\"acquisition\", \"u19_subject\")\n", + "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n", "\n", "key = subject.Subject() & \"user_id = 'ms81'\"\n", - "su = pd.DataFrame((recording.Recording.BehaviorSession * behavior.TowersBlock & key & \"level = 11\" & \"block_performance>0.6\").fetch(as_dict=True, order_by='block_performance DESC'))\n", - "su\n" + "su = pd.DataFrame(\n", + " (\n", + " recording.Recording.BehaviorSession * behavior.TowersBlock\n", + " & key\n", + " & \"level = 11\"\n", + " & \"block_performance>0.6\"\n", + " ).fetch(as_dict=True, order_by=\"block_performance DESC\")\n", + ")\n", + "su" ] }, { @@ -564,41 +570,42 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "## Filter rigs\n", "filter_rigs = True\n", - "rigs = ['165I-Rig1-T',\n", - " '165I-Rig2-T',\n", - " '165I-Rig3-T',\n", - " '165I-Rig4-T',\n", - " '165A-Rig5-T',\n", - " '165A-Rig6-T',\n", - " '165A-Rig7-T',\n", - " '165A-Rig8-T', \n", - " '170b-Imaging0642',\n", - " '185f-rig1',\n", - " '188-Rig1',\n", - " '188-Rig2',\n", - " 'BezosMeso',\n", - " 'C42-Bay2-Rig1-I']\n", - "rigs_label = ', '.join(f'\"{w}\"' for w in rigs)\n", - "rigs_query = 'session_location in (' + rigs_label + ')'\n", + "rigs = [\n", + " \"165I-Rig1-T\",\n", + " \"165I-Rig2-T\",\n", + " \"165I-Rig3-T\",\n", + " \"165I-Rig4-T\",\n", + " \"165A-Rig5-T\",\n", + " \"165A-Rig6-T\",\n", + " \"165A-Rig7-T\",\n", + " \"165A-Rig8-T\",\n", + " \"170b-Imaging0642\",\n", + " \"185f-rig1\",\n", + " \"188-Rig1\",\n", + " \"188-Rig2\",\n", + " \"BezosMeso\",\n", + " \"C42-Bay2-Rig1-I\",\n", + "]\n", + "rigs_label = \", \".join(f'\"{w}\"' for w in rigs)\n", + "rigs_query = \"session_location in (\" + rigs_label + \")\"\n", "\n", - "#Filter Dates\n", + "# Filter Dates\n", "filter_dates = True\n", - "dates = ['2016-01-01', '2022-03-01']\n", - "date_label = ' and '.join(f'\"{w}\"' for w in dates)\n", - "date_query = 'session_date between ' + date_label \n", + "dates = [\"2016-01-01\", \"2022-03-01\"]\n", + "date_label = \" and \".join(f'\"{w}\"' for w in dates)\n", + "date_query = \"session_date between \" + date_label\n", "\n", - "#Filter users\n", + "# Filter users\n", "filter_user = True\n", - "users = ['sbolkan', 'jounhong']\n", + "users = [\"sbolkan\", \"jounhong\"]\n", "users = ['subject_fullname like \"' + x + '%\"' for x in users]\n", - "user_query = ' or '.join(f'{w}' for w in users) \n", + "user_query = \" or \".join(f\"{w}\" for w in users)\n", "\n", "\n", - "#Min sessions to be a \"real\" subject\n", - "min_sessions = 15\n" + "# Min sessions to be a \"real\" subject\n", + "min_sessions = 15" ] }, { @@ -1008,18 +1015,18 @@ "query = []\n", "if filter_rigs:\n", " query.append(rigs_query)\n", - " \n", + "\n", "if filter_dates:\n", " query.append(date_query)\n", - " \n", + "\n", "if filter_user:\n", " query.append(user_query)\n", - " \n", + "\n", "# Filter by all conditions selected\n", "session_table_filtered = acquisition.Session\n", "for filter_key in query:\n", " session_table_filtered = session_table_filtered & filter_key\n", - " \n", + "\n", "session_df = pd.DataFrame(session_table_filtered.fetch(as_dict=True))\n", "session_df" ] @@ -1161,11 +1168,12 @@ } ], "source": [ - "\n", - "protocol_df = session_df.groupby('session_protocol').agg({'subject_fullname': [('num_sessions', 'count'), ('num_subjects', 'nunique')]})\n", + "protocol_df = session_df.groupby(\"session_protocol\").agg(\n", + " {\"subject_fullname\": [(\"num_sessions\", \"count\"), (\"num_subjects\", \"nunique\")]}\n", + ")\n", "protocol_df.columns = protocol_df.columns.droplevel()\n", - "#protocol_df = protocol_df.reset_index()\n", - "protocol_df = protocol_df.sort_values(by='num_sessions', ascending=False)\n", + "# protocol_df = protocol_df.reset_index()\n", + "protocol_df = protocol_df.sort_values(by=\"num_sessions\", ascending=False)\n", "protocol_df" ] }, @@ -1286,43 +1294,54 @@ } ], "source": [ - "#Analyze only one protocol\n", - "session_p_df = session_df.loc[session_df['session_protocol'] == protocol_df.index[0], :]\n", + "# Analyze only one protocol\n", + "session_p_df = session_df.loc[session_df[\"session_protocol\"] == protocol_df.index[0], :]\n", "\n", - "#Get max level\n", - "max_level = session_p_df['level'].max()\n", + "# Get max level\n", + "max_level = session_p_df[\"level\"].max()\n", "max_level = 10\n", "\n", - "#Sort by subject and date\n", - "session_p_df = session_p_df.sort_values(by=['subject_fullname', 'session_date'])\n", + "# Sort by subject and date\n", + "session_p_df = session_p_df.sort_values(by=[\"subject_fullname\", \"session_date\"])\n", "session_p_df = session_p_df.reset_index(drop=True)\n", "\n", "# Count sessions for each subject (sequential count and total count)\n", - "sequential_sessions = session_p_df.groupby(['subject_fullname']).cumcount() + 1\n", - "num_sessions_df = session_p_df.groupby('subject_fullname').agg({'session_date': [('total_sessions', 'count')]})\n", + "sequential_sessions = session_p_df.groupby([\"subject_fullname\"]).cumcount() + 1\n", + "num_sessions_df = session_p_df.groupby(\"subject_fullname\").agg(\n", + " {\"session_date\": [(\"total_sessions\", \"count\")]}\n", + ")\n", "num_sessions_df.columns = num_sessions_df.columns.droplevel()\n", "num_sessions_df = num_sessions_df.reset_index()\n", "\n", "# Merge session count to session df\n", - "session_p_df['seq_sessions'] = sequential_sessions\n", + "session_p_df[\"seq_sessions\"] = sequential_sessions\n", "session_p_df = session_p_df.merge(num_sessions_df)\n", "\n", "# Filter only max level sessions\n", - "session_p_df_max = session_p_df.loc[session_p_df['level'] == max_level, :]\n", + "session_p_df_max = session_p_df.loc[session_p_df[\"level\"] == max_level, :]\n", "\n", "# Get only the first day that they got to max level and filter \"fake\" subjects\n", - "first_session_max_level = session_p_df_max.drop_duplicates(subset = [\"subject_fullname\"], keep='first')\n", - "first_session_max_level = first_session_max_level.loc[first_session_max_level['total_sessions'] >= min_sessions, :]\n", + "first_session_max_level = session_p_df_max.drop_duplicates(\n", + " subset=[\"subject_fullname\"], keep=\"first\"\n", + ")\n", + "first_session_max_level = first_session_max_level.loc[\n", + " first_session_max_level[\"total_sessions\"] >= min_sessions, :\n", + "]\n", "first_session_max_level = first_session_max_level.reset_index(drop=True)\n", "\n", "\n", - "#min_sessions to max level per rig\n", - "sessions_max_level_rig = first_session_max_level.groupby('session_location').agg(\n", - " {'seq_sessions': [('mean_sessions', 'mean'), ('std_sessions', 'std'), ('num_subjects', 'count')]})\n", + "# min_sessions to max level per rig\n", + "sessions_max_level_rig = first_session_max_level.groupby(\"session_location\").agg(\n", + " {\n", + " \"seq_sessions\": [\n", + " (\"mean_sessions\", \"mean\"),\n", + " (\"std_sessions\", \"std\"),\n", + " (\"num_subjects\", \"count\"),\n", + " ]\n", + " }\n", + ")\n", "sessions_max_level_rig.columns = sessions_max_level_rig.columns.droplevel()\n", - "sessions_max_level_rig\n", - "\n", - "\n" + "sessions_max_level_rig" ] }, { @@ -1352,13 +1371,29 @@ } ], "source": [ - "fig, ax = plt.subplots(1, 1, figsize=(15,12))\n", - "x_pos = np.linspace(1, sessions_max_level_rig.shape[0], num=sessions_max_level_rig.shape[0])\n", - "ax.bar(x_pos, sessions_max_level_rig['mean_sessions'], yerr=sessions_max_level_rig['std_sessions'], align='center', alpha=0.5, ecolor='black', capsize=10)\n", - "ax.set_ylabel('Num sessions to max level', fontsize=16)\n", + "fig, ax = plt.subplots(1, 1, figsize=(15, 12))\n", + "x_pos = np.linspace(\n", + " 1, sessions_max_level_rig.shape[0], num=sessions_max_level_rig.shape[0]\n", + ")\n", + "ax.bar(\n", + " x_pos,\n", + " sessions_max_level_rig[\"mean_sessions\"],\n", + " yerr=sessions_max_level_rig[\"std_sessions\"],\n", + " align=\"center\",\n", + " alpha=0.5,\n", + " ecolor=\"black\",\n", + " capsize=10,\n", + ")\n", + "ax.set_ylabel(\"Num sessions to max level\", fontsize=16)\n", "ax.set_xticks(x_pos)\n", "ax.set_xticklabels(sessions_max_level_rig.index)\n", - "ax.set_title('Sessions to max level per rig protocol:' + protocol_df.index[0][0:20] + ' max_level= ' + str(max_level), fontsize=18)\n", + "ax.set_title(\n", + " \"Sessions to max level per rig protocol:\"\n", + " + protocol_df.index[0][0:20]\n", + " + \" max_level= \"\n", + " + str(max_level),\n", + " fontsize=18,\n", + ")\n", "ax.yaxis.grid(True)\n", "\n", "# Save the figure and show\n", diff --git a/notebooks/check_pos_image_quality.ipynb b/notebooks/check_pos_image_quality.ipynb index 8706a29f..87bc8b76 100644 --- a/notebooks/check_pos_image_quality.ipynb +++ b/notebooks/check_pos_image_quality.ipynb @@ -33,12 +33,11 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "import cv2\n", "import datajoint as dj\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "import pandas as pd\n" + "import pandas as pd" ] }, { @@ -57,7 +56,7 @@ ], "source": [ "dj.conn()\n", - "action_db = dj.create_virtual_module('action', 'u19_action')" + "action_db = dj.create_virtual_module(\"action\", \"u19_action\")" ] }, { @@ -74,8 +73,10 @@ "outputs": [], "source": [ "query = dict()\n", - "query['location'] = '165A-miniVR-T-2'\n", - "AllImageRecords = pd.DataFrame((action_db.DailySubjectPositionData & query).fetch(as_dict=True))" + "query[\"location\"] = \"165A-miniVR-T-2\"\n", + "AllImageRecords = pd.DataFrame(\n", + " (action_db.DailySubjectPositionData & query).fetch(as_dict=True)\n", + ")" ] }, { @@ -313,7 +314,7 @@ "metadata": {}, "outputs": [], "source": [ - "#AllImageRecords = AllImageRecords_copy.copy(deep=True)" + "# AllImageRecords = AllImageRecords_copy.copy(deep=True)" ] }, { @@ -329,11 +330,10 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "def analyze_image_quality(_image, type='lateral'):\n", - " #_image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)\n", - " if type != 'lateral':\n", - " _image = _image[:,200:600]\n", + "def analyze_image_quality(_image, type=\"lateral\"):\n", + " # _image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)\n", + " if type != \"lateral\":\n", + " _image = _image[:, 200:600]\n", "\n", " # Sharpness\n", " _laplacian = cv2.Laplacian(_image, cv2.CV_64F)\n", @@ -351,11 +351,7 @@ " _sobel = np.sqrt(_sobel_x**2 + _sobel_y**2)\n", " _resolution = np.mean(_sobel)\n", "\n", - " return {\n", - " \"sharpness\": _sharpness,\n", - " \"clarity\": _clarity,\n", - " \"resolution\": _resolution\n", - " }" + " return {\"sharpness\": _sharpness, \"clarity\": _clarity, \"resolution\": _resolution}" ] }, { @@ -364,8 +360,12 @@ "metadata": {}, "outputs": [], "source": [ - "AllImageRecords['top_image_metrics'] = AllImageRecords['top_image'].apply(analyze_image_quality, type='top')\n", - "AllImageRecords['lateral_image_metrics'] = AllImageRecords['lateral_image'].apply(analyze_image_quality, type='lateral')" + "AllImageRecords[\"top_image_metrics\"] = AllImageRecords[\"top_image\"].apply(\n", + " analyze_image_quality, type=\"top\"\n", + ")\n", + "AllImageRecords[\"lateral_image_metrics\"] = AllImageRecords[\"lateral_image\"].apply(\n", + " analyze_image_quality, type=\"lateral\"\n", + ")" ] }, { @@ -691,15 +691,17 @@ } ], "source": [ - "df_aux = pd.json_normalize(AllImageRecords['top_image_metrics'])\n", - "df_aux = df_aux.rename(columns=lambda column: 'top_image_' + column)\n", + "df_aux = pd.json_normalize(AllImageRecords[\"top_image_metrics\"])\n", + "df_aux = df_aux.rename(columns=lambda column: \"top_image_\" + column)\n", "AllImageRecords = pd.concat([AllImageRecords, df_aux], axis=1)\n", "\n", - "df_aux = pd.json_normalize(AllImageRecords['lateral_image_metrics'])\n", - "df_aux = df_aux.rename(columns=lambda column: 'lateral_image_' + column)\n", + "df_aux = pd.json_normalize(AllImageRecords[\"lateral_image_metrics\"])\n", + "df_aux = df_aux.rename(columns=lambda column: \"lateral_image_\" + column)\n", "AllImageRecords = pd.concat([AllImageRecords, df_aux], axis=1)\n", "\n", - "AllImageRecords = AllImageRecords.drop(columns=['top_image_metrics', 'lateral_image_metrics'])\n", + "AllImageRecords = AllImageRecords.drop(\n", + " columns=[\"top_image_metrics\", \"lateral_image_metrics\"]\n", + ")\n", "AllImageRecords" ] }, @@ -741,22 +743,22 @@ "source": [ "fig = plt.figure(figsize=(20, 10))\n", "\n", - "#fig, (ax1, ax2) = plt.subplots(1,2)\n", + "# fig, (ax1, ax2) = plt.subplots(1,2)\n", "# plt.rcParams['figure.dpi'] = 250\n", "fig.add_subplot(1, 2, 1)\n", "\n", - "plt.plot(AllImageRecords['top_image_resolution'], linewidth=2)\n", + "plt.plot(AllImageRecords[\"top_image_resolution\"], linewidth=2)\n", "# Filter by using a moving average window of 10 samples\n", - "plt.xlabel('# Image ~ Date')\n", - "plt.ylabel('Image Sharpness')\n", - "plt.title('Top Camera')\n", + "plt.xlabel(\"# Image ~ Date\")\n", + "plt.ylabel(\"Image Sharpness\")\n", + "plt.title(\"Top Camera\")\n", "\n", "fig.add_subplot(1, 2, 2)\n", - "plt.plot(AllImageRecords['lateral_image_resolution'], linewidth=2)\n", + "plt.plot(AllImageRecords[\"lateral_image_resolution\"], linewidth=2)\n", "# Filter by using a moving average window of 10 samples\n", - "plt.xlabel('# Image ~ Date')\n", + "plt.xlabel(\"# Image ~ Date\")\n", "\n", - "plt.title('Lateral Camera')" + "plt.title(\"Lateral Camera\")" ] }, { @@ -791,19 +793,17 @@ "fig = plt.figure(figsize=(20, 10))\n", "\n", "\n", - "\n", - "#fig, (ax1, ax2) = plt.subplots(1,2)\n", + "# fig, (ax1, ax2) = plt.subplots(1,2)\n", "# plt.rcParams['figure.dpi'] = 250\n", "fig.add_subplot(1, 2, 1)\n", "\n", - "plt.imshow(AllImageRecords.loc[5,'lateral_image'], cmap='gray')\n", - "plt.axis('off')\n", + "plt.imshow(AllImageRecords.loc[5, \"lateral_image\"], cmap=\"gray\")\n", + "plt.axis(\"off\")\n", "\n", "\n", "fig.add_subplot(1, 2, 2)\n", - "plt.imshow(AllImageRecords.loc[60,'lateral_image'], cmap='gray')\n", - "plt.axis('off')\n", - "\n" + "plt.imshow(AllImageRecords.loc[60, \"lateral_image\"], cmap=\"gray\")\n", + "plt.axis(\"off\")" ] }, { @@ -838,19 +838,17 @@ "fig = plt.figure(figsize=(20, 10))\n", "\n", "\n", - "\n", - "#fig, (ax1, ax2) = plt.subplots(1,2)\n", + "# fig, (ax1, ax2) = plt.subplots(1,2)\n", "# plt.rcParams['figure.dpi'] = 250\n", "fig.add_subplot(1, 2, 1)\n", "\n", - "plt.imshow(AllImageRecords.loc[5,'top_image'], cmap='gray')\n", - "#plt.axis('off')\n", + "plt.imshow(AllImageRecords.loc[5, \"top_image\"], cmap=\"gray\")\n", + "# plt.axis('off')\n", "\n", "\n", "fig.add_subplot(1, 2, 2)\n", - "plt.imshow(AllImageRecords.loc[59,'top_image'], cmap='gray')\n", - "#plt.axis('off')\n", - "\n" + "plt.imshow(AllImageRecords.loc[59, \"top_image\"], cmap=\"gray\")\n", + "# plt.axis('off')" ] }, { @@ -885,19 +883,17 @@ "fig = plt.figure(figsize=(20, 10))\n", "\n", "\n", - "\n", - "#fig, (ax1, ax2) = plt.subplots(1,2)\n", + "# fig, (ax1, ax2) = plt.subplots(1,2)\n", "# plt.rcParams['figure.dpi'] = 250\n", "fig.add_subplot(1, 2, 1)\n", "\n", - "plt.imshow(AllImageRecords.loc[5,'top_image'][:,200:600], cmap='gray')\n", - "plt.axis('off')\n", + "plt.imshow(AllImageRecords.loc[5, \"top_image\"][:, 200:600], cmap=\"gray\")\n", + "plt.axis(\"off\")\n", "\n", "\n", "fig.add_subplot(1, 2, 2)\n", - "plt.imshow(AllImageRecords.loc[59,'top_image'][:,200:600], cmap='gray')\n", - "plt.axis('off')\n", - "\n" + "plt.imshow(AllImageRecords.loc[59, \"top_image\"][:, 200:600], cmap=\"gray\")\n", + "plt.axis(\"off\")" ] }, { @@ -917,7 +913,7 @@ } ], "source": [ - "AllImageRecords.loc[5,'top_image'].shape" + "AllImageRecords.loc[5, \"top_image\"].shape" ] }, { @@ -950,7 +946,7 @@ ], "source": [ "fig = plt.figure(figsize=(14, 7))\n", - "plt.imshow(AllImageRecords.loc[60,'lateral_image'])" + "plt.imshow(AllImageRecords.loc[60, \"lateral_image\"])" ] }, { @@ -1456,18 +1452,19 @@ } ], "source": [ + "video_name = \"top_image_video4.mp4\" # Name of the output video file\n", "\n", - "video_name = 'top_image_video4.mp4' # Name of the output video file\n", - "\n", - "height, width = AllImageRecords.loc[0,'top_image'].shape\n", + "height, width = AllImageRecords.loc[0, \"top_image\"].shape\n", "\n", - "fourcc = cv2.VideoWriter_fourcc(*'mp4v') # Codec for mp4 video\n", - "video = cv2.VideoWriter(video_name, fourcc, 5, (width, height), isColor=False) # 30 is the frame rate\n", + "fourcc = cv2.VideoWriter_fourcc(*\"mp4v\") # Codec for mp4 video\n", + "video = cv2.VideoWriter(\n", + " video_name, fourcc, 5, (width, height), isColor=False\n", + ") # 30 is the frame rate\n", "\n", "for i in range(AllImageRecords.shape[0]):\n", - " print(AllImageRecords.loc[i,'top_image'])\n", + " print(AllImageRecords.loc[i, \"top_image\"])\n", "\n", - " video.write(AllImageRecords.loc[i,'top_image'])\n", + " video.write(AllImageRecords.loc[i, \"top_image\"])\n", "\n", "video.release()" ] diff --git a/notebooks/create_tech_responsibilites_table.ipynb b/notebooks/create_tech_responsibilites_table.ipynb index 0dd58f3e..7ba28208 100644 --- a/notebooks/create_tech_responsibilites_table.ipynb +++ b/notebooks/create_tech_responsibilites_table.ipynb @@ -18,7 +18,7 @@ "import datajoint as dj\n", "import pandas as pd\n", "\n", - "pd.set_option('display.max_rows', 50)" + "pd.set_option(\"display.max_rows\", 50)" ] }, { @@ -51,9 +51,9 @@ ], "source": [ "conn = dj.conn()\n", - "action = dj.create_virtual_module('action', 'u19_test_action')\n", - "subject = dj.create_virtual_module('subject', 'u19_test_subject')\n", - "lab = dj.create_virtual_module('lab', 'u19_test_lab')" + "action = dj.create_virtual_module(\"action\", \"u19_test_action\")\n", + "subject = dj.create_virtual_module(\"subject\", \"u19_test_subject\")\n", + "lab = dj.create_virtual_module(\"lab\", \"u19_test_lab\")" ] }, { @@ -75,18 +75,38 @@ "metadata": {}, "outputs": [], "source": [ - "all_subject_status = pd.DataFrame((action.SubjectStatus * subject.Subject.proj('user_id') * lab.User.proj('tech_responsibility')).fetch(as_dict=True))\n", - "all_subject_status = all_subject_status.sort_values(by='effective_date', ascending=False)\n", - "all_subject_status = all_subject_status.drop_duplicates(subset='subject_fullname', keep='first')\n", + "all_subject_status = pd.DataFrame(\n", + " (\n", + " action.SubjectStatus\n", + " * subject.Subject.proj(\"user_id\")\n", + " * lab.User.proj(\"tech_responsibility\")\n", + " ).fetch(as_dict=True)\n", + ")\n", + "all_subject_status = all_subject_status.sort_values(\n", + " by=\"effective_date\", ascending=False\n", + ")\n", + "all_subject_status = all_subject_status.drop_duplicates(\n", + " subset=\"subject_fullname\", keep=\"first\"\n", + ")\n", "\n", - "all_subject_status = all_subject_status.sort_values(by='effective_date', ascending=False)\n", + "all_subject_status = all_subject_status.sort_values(\n", + " by=\"effective_date\", ascending=False\n", + ")\n", "\n", "all_subject_status = all_subject_status.reset_index(drop=True)\n", "\n", - "all_subject_status['schedule_list'] = all_subject_status['schedule'].apply(lambda x: x.split('/'))\n", - "all_subject_status['schedule_set'] = all_subject_status['schedule_list'].apply(lambda x: set(x))\n", - "all_subject_status['num_act'] = all_subject_status['schedule_set'].apply(lambda x: len(x))\n", - "all_subject_status['index_act'] = all_subject_status['schedule_list'].apply(lambda x: index_act(x))" + "all_subject_status[\"schedule_list\"] = all_subject_status[\"schedule\"].apply(\n", + " lambda x: x.split(\"/\")\n", + ")\n", + "all_subject_status[\"schedule_set\"] = all_subject_status[\"schedule_list\"].apply(\n", + " lambda x: set(x)\n", + ")\n", + "all_subject_status[\"num_act\"] = all_subject_status[\"schedule_set\"].apply(\n", + " lambda x: len(x)\n", + ")\n", + "all_subject_status[\"index_act\"] = all_subject_status[\"schedule_list\"].apply(\n", + " lambda x: index_act(x)\n", + ")" ] }, { @@ -103,28 +123,27 @@ "outputs": [], "source": [ "subject_status_dict_owner = {\n", - " 'Dead': ['Nothing'],\n", - " 'AdLibWater': ['Nothing'],\n", - " 'Missing': ['Nothing'],\n", - " 'WaterRestrictionOnly': ['Watering', 'Weighing'],\n", - " 'InExperiments': ['Watering', 'Weighing', 'Training']\n", + " \"Dead\": [\"Nothing\"],\n", + " \"AdLibWater\": [\"Nothing\"],\n", + " \"Missing\": [\"Nothing\"],\n", + " \"WaterRestrictionOnly\": [\"Watering\", \"Weighing\"],\n", + " \"InExperiments\": [\"Watering\", \"Weighing\", \"Training\"],\n", "}\n", "subject_status_dict_technician = {\n", - " 'Dead': ['Nothing'],\n", - " 'AdLibWater': ['Nothing'],\n", - " 'Missing': ['Nothing'],\n", - " 'WaterRestrictionOnly': ['Watering', 'Weighing', 'Transport'],\n", - " 'InExperiments': ['Watering', 'Weighing', 'Training', 'Transport']\n", + " \"Dead\": [\"Nothing\"],\n", + " \"AdLibWater\": [\"Nothing\"],\n", + " \"Missing\": [\"Nothing\"],\n", + " \"WaterRestrictionOnly\": [\"Watering\", \"Weighing\", \"Transport\"],\n", + " \"InExperiments\": [\"Watering\", \"Weighing\", \"Training\", \"Transport\"],\n", "}\n", "\n", "act_to_status_dict = {\n", - " 'Water': 'WaterRestrictionOnly',\n", - " 'Weigh': 'WaterRestrictionOnly',\n", - " 'Train': 'InExperiments',\n", - " 'OnlyTrain': 'InExperiments',\n", - " 'Nothing': 'Dead',\n", - " 'Transport': 'Dead'\n", - "\n", + " \"Water\": \"WaterRestrictionOnly\",\n", + " \"Weigh\": \"WaterRestrictionOnly\",\n", + " \"Train\": \"InExperiments\",\n", + " \"OnlyTrain\": \"InExperiments\",\n", + " \"Nothing\": \"Dead\",\n", + " \"Transport\": \"Dead\",\n", "}" ] }, @@ -142,28 +161,27 @@ "outputs": [], "source": [ "subject_status_dict_owner_shared = {\n", - " 'Dead': ['Nothing'],\n", - " 'AdLibWater': ['Nothing'],\n", - " 'Missing': ['Nothing'],\n", - " 'WaterRestrictionOnly': ['Watering', 'Weighing'],\n", - " 'InExperiments': ['Watering', 'Weighing', 'Training']\n", + " \"Dead\": [\"Nothing\"],\n", + " \"AdLibWater\": [\"Nothing\"],\n", + " \"Missing\": [\"Nothing\"],\n", + " \"WaterRestrictionOnly\": [\"Watering\", \"Weighing\"],\n", + " \"InExperiments\": [\"Watering\", \"Weighing\", \"Training\"],\n", "}\n", "subject_status_dict_technician_shared = {\n", - " 'Dead': ['Nothing'],\n", - " 'AdLibWater': ['Nothing'],\n", - " 'Missing': ['Nothing'],\n", - " 'WaterRestrictionOnly': ['Watering', 'Weighing', 'Transport'],\n", - " 'InExperiments': ['Watering', 'Weighing', 'Training', 'Transport']\n", + " \"Dead\": [\"Nothing\"],\n", + " \"AdLibWater\": [\"Nothing\"],\n", + " \"Missing\": [\"Nothing\"],\n", + " \"WaterRestrictionOnly\": [\"Watering\", \"Weighing\", \"Transport\"],\n", + " \"InExperiments\": [\"Watering\", \"Weighing\", \"Training\", \"Transport\"],\n", "}\n", "\n", "act_to_status_dict_shared = {\n", - " 'Water': 'WaterRestrictionOnly',\n", - " 'Weigh': 'WaterRestrictionOnly',\n", - " 'Train': 'InExperiments',\n", - " 'OnlyTrain': 'InExperiments',\n", - " 'Nothing': 'Dead',\n", - " 'Transport': ['Transport']\n", - "\n", + " \"Water\": \"WaterRestrictionOnly\",\n", + " \"Weigh\": \"WaterRestrictionOnly\",\n", + " \"Train\": \"InExperiments\",\n", + " \"OnlyTrain\": \"InExperiments\",\n", + " \"Nothing\": \"Dead\",\n", + " \"Transport\": [\"Transport\"],\n", "}" ] }, @@ -174,27 +192,27 @@ "outputs": [], "source": [ "base_weekly_dict = {\n", - " \"assignment_style\": \"Specify for the whole week\",\n", - " \"weekly\": ['Nothing']\n", + " \"assignment_style\": \"Specify for the whole week\",\n", + " \"weekly\": [\"Nothing\"],\n", "}\n", "\n", "base_weekdays_weekends_dict = {\n", - " \"assignment_style\": \"Specify by weekday/weekend\",\n", - " \"weekdays\": ['Nothing'],\n", - " \"weekends\": ['Nothing']\n", + " \"assignment_style\": \"Specify by weekday/weekend\",\n", + " \"weekdays\": [\"Nothing\"],\n", + " \"weekends\": [\"Nothing\"],\n", "}\n", "\n", "\n", "base_days_dict = {\n", - "\"assignment_style\":\"Specify each day\",\n", - "\"Monday\": ['Nothing'],\n", - "\"Tuesday\": ['Nothing'],\n", - "\"Wednesday\": ['Nothing'],\n", - "\"Thursday\": ['Nothing'],\n", - "\"Friday\": ['Nothing'],\n", - "\"Saturday\": ['Nothing'],\n", - "\"Sunday\": ['Nothing'],\n", - "}\n" + " \"assignment_style\": \"Specify each day\",\n", + " \"Monday\": [\"Nothing\"],\n", + " \"Tuesday\": [\"Nothing\"],\n", + " \"Wednesday\": [\"Nothing\"],\n", + " \"Thursday\": [\"Nothing\"],\n", + " \"Friday\": [\"Nothing\"],\n", + " \"Saturday\": [\"Nothing\"],\n", + " \"Sunday\": [\"Nothing\"],\n", + "}" ] }, { @@ -205,86 +223,115 @@ "source": [ "def translate_schedule(df_row):\n", "\n", - " user_id = df_row['user_id']\n", - " schedule_list = df_row['schedule_list']\n", - " tech_responsibility = df_row['tech_responsibility']\n", - " subject_status = df_row['subject_status']\n", - " num_act = df_row['num_act']\n", - " index_act = df_row['index_act']\n", - "\n", + " user_id = df_row[\"user_id\"]\n", + " schedule_list = df_row[\"schedule_list\"]\n", + " tech_responsibility = df_row[\"tech_responsibility\"]\n", + " subject_status = df_row[\"subject_status\"]\n", + " num_act = df_row[\"num_act\"]\n", + " index_act = df_row[\"index_act\"]\n", "\n", " tech_act_dict = base_weekly_dict.copy()\n", - " tech_act_dict['weekly'] = ['Nothing']\n", + " tech_act_dict[\"weekly\"] = [\"Nothing\"]\n", " owner_act_dict = base_weekly_dict.copy()\n", - " owner_act_dict['weekly'] = ['Nothing']\n", + " owner_act_dict[\"weekly\"] = [\"Nothing\"]\n", "\n", - " if tech_responsibility == 'yes':\n", - " if subject_status in ('Dead', 'AdLibWater', 'Missing') or num_act == 1:\n", + " if tech_responsibility == \"yes\":\n", + " if subject_status in (\"Dead\", \"AdLibWater\", \"Missing\") or num_act == 1:\n", " tech_act_dict = base_weekly_dict.copy()\n", - " tech_act_dict['weekly'] = subject_status_dict_technician[subject_status]\n", - " elif num_act == 2 and list(index_act.values())[0] == [1,2,3,4,5] or list(index_act.values())[0] == [0,6]:\n", - "\n", + " tech_act_dict[\"weekly\"] = subject_status_dict_technician[subject_status]\n", + " elif (\n", + " num_act == 2\n", + " and list(index_act.values())[0] == [1, 2, 3, 4, 5]\n", + " or list(index_act.values())[0] == [0, 6]\n", + " ):\n", " tech_act_dict = base_weekdays_weekends_dict.copy()\n", " weekdays_act = schedule_list[1]\n", " weekdends_act = schedule_list[6]\n", - " if weekdays_act == 'Transport' or weekdends_act == 'Transport':\n", + " if weekdays_act == \"Transport\" or weekdends_act == \"Transport\":\n", " owner_act_dict = base_weekdays_weekends_dict\n", - " if weekdays_act == 'Transport':\n", - " tech_act_dict['weekdays'] = ['Transport']\n", - " owner_act_dict['weekdays'] = subject_status_dict_owner[subject_status]\n", + " if weekdays_act == \"Transport\":\n", + " tech_act_dict[\"weekdays\"] = [\"Transport\"]\n", + " owner_act_dict[\"weekdays\"] = subject_status_dict_owner[subject_status]\n", " else:\n", " weekdays_status = act_to_status_dict[weekdays_act]\n", - " tech_act_dict['weekdays'] = subject_status_dict_technician[weekdays_status]\n", - " if weekdends_act == 'Transport':\n", - " tech_act_dict['weekends'] = ['Transport']\n", - " owner_act_dict['weekends'] = subject_status_dict_owner[subject_status]\n", + " tech_act_dict[\"weekdays\"] = subject_status_dict_technician[\n", + " weekdays_status\n", + " ]\n", + " if weekdends_act == \"Transport\":\n", + " tech_act_dict[\"weekends\"] = [\"Transport\"]\n", + " owner_act_dict[\"weekends\"] = subject_status_dict_owner[subject_status]\n", " else:\n", " weekdends_status = act_to_status_dict[weekdends_act]\n", - " tech_act_dict['weekends'] = subject_status_dict_technician[weekdends_status]\n", + " tech_act_dict[\"weekends\"] = subject_status_dict_technician[\n", + " weekdends_status\n", + " ]\n", " else:\n", " tech_act_dict = base_days_dict.copy()\n", "\n", - " \n", + " tech_act_dict[\"Monday\"] = subject_status_dict_technician[\n", + " act_to_status_dict[schedule_list[0]]\n", + " ]\n", + " tech_act_dict[\"Tuesday\"] = subject_status_dict_technician[\n", + " act_to_status_dict[schedule_list[1]]\n", + " ]\n", + " tech_act_dict[\"Wednesday\"] = subject_status_dict_technician[\n", + " act_to_status_dict[schedule_list[2]]\n", + " ]\n", + " tech_act_dict[\"Thursday\"] = subject_status_dict_technician[\n", + " act_to_status_dict[schedule_list[3]]\n", + " ]\n", + " tech_act_dict[\"Friday\"] = subject_status_dict_technician[\n", + " act_to_status_dict[schedule_list[4]]\n", + " ]\n", + " tech_act_dict[\"Saturday\"] = subject_status_dict_technician[\n", + " act_to_status_dict[schedule_list[5]]\n", + " ]\n", + " tech_act_dict[\"Sunday\"] = subject_status_dict_technician[\n", + " act_to_status_dict[schedule_list[6]]\n", + " ]\n", "\n", - " tech_act_dict['Monday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[0]]]\n", - " tech_act_dict['Tuesday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[1]]]\n", - " tech_act_dict['Wednesday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[2]]]\n", - " tech_act_dict['Thursday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[3]]]\n", - " tech_act_dict['Friday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[4]]]\n", - " tech_act_dict['Saturday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[5]]]\n", - " tech_act_dict['Sunday'] = subject_status_dict_technician[act_to_status_dict[schedule_list[6]]]\n", - " \n", " # Tech_responsibility = no\n", " else:\n", - " if subject_status not in ('Dead', 'AdLibWater', 'Missing'):\n", - " tech_act_dict['weekly'] = ['Transport']\n", + " if subject_status not in (\"Dead\", \"AdLibWater\", \"Missing\"):\n", + " tech_act_dict[\"weekly\"] = [\"Transport\"]\n", " if num_act == 1:\n", - " owner_act_dict['weekly'] = subject_status_dict_owner[subject_status]\n", - " elif num_act == 2 and list(index_act.values())[0] == [1,2,3,4,5] or list(index_act.values())[0] == [0,6]:\n", + " owner_act_dict[\"weekly\"] = subject_status_dict_owner[subject_status]\n", + " elif (\n", + " num_act == 2\n", + " and list(index_act.values())[0] == [1, 2, 3, 4, 5]\n", + " or list(index_act.values())[0] == [0, 6]\n", + " ):\n", " owner_act_dict = base_weekdays_weekends_dict.copy()\n", " weekdays_act = schedule_list[1]\n", " weekdends_act = schedule_list[6]\n", - " owner_act_dict['weekdays'] = subject_status_dict_owner[subject_status]\n", - " owner_act_dict['weekends'] = subject_status_dict_owner[subject_status]\n", + " owner_act_dict[\"weekdays\"] = subject_status_dict_owner[subject_status]\n", + " owner_act_dict[\"weekends\"] = subject_status_dict_owner[subject_status]\n", " else:\n", " owner_act_dict = base_days_dict.copy()\n", - " schedule_list = [act_to_status_dict[x] if x!='Transport' else x for x in schedule_list]\n", - " schedule_list = [subject_status if x=='Transport' else x for x in schedule_list]\n", - "\n", - " owner_act_dict['Monday'] = subject_status_dict_owner[schedule_list[0]]\n", - " owner_act_dict['Tuesday'] = subject_status_dict_owner[schedule_list[1]]\n", - " owner_act_dict['Wednesday'] = subject_status_dict_owner[schedule_list[2]]\n", - " owner_act_dict['Thursday'] = subject_status_dict_owner[schedule_list[3]]\n", - " owner_act_dict['Friday'] = subject_status_dict_owner[schedule_list[4]]\n", - " owner_act_dict['Saturday'] = subject_status_dict_owner[schedule_list[5]]\n", - " owner_act_dict['Sunday'] = subject_status_dict_owner[schedule_list[6]]\n", - " \n", - " \n", - " return pd.Series({'technician_duties': json.dumps(tech_act_dict), 'owner_duties': json.dumps(owner_act_dict)}) \n", + " schedule_list = [\n", + " act_to_status_dict[x] if x != \"Transport\" else x\n", + " for x in schedule_list\n", + " ]\n", + " schedule_list = [\n", + " subject_status if x == \"Transport\" else x for x in schedule_list\n", + " ]\n", "\n", + " owner_act_dict[\"Monday\"] = subject_status_dict_owner[schedule_list[0]]\n", + " owner_act_dict[\"Tuesday\"] = subject_status_dict_owner[schedule_list[1]]\n", + " owner_act_dict[\"Wednesday\"] = subject_status_dict_owner[\n", + " schedule_list[2]\n", + " ]\n", + " owner_act_dict[\"Thursday\"] = subject_status_dict_owner[schedule_list[3]]\n", + " owner_act_dict[\"Friday\"] = subject_status_dict_owner[schedule_list[4]]\n", + " owner_act_dict[\"Saturday\"] = subject_status_dict_owner[schedule_list[5]]\n", + " owner_act_dict[\"Sunday\"] = subject_status_dict_owner[schedule_list[6]]\n", "\n", - " \n", - "\n" + " return pd.Series(\n", + " {\n", + " \"technician_duties\": json.dumps(tech_act_dict),\n", + " \"owner_duties\": json.dumps(owner_act_dict),\n", + " }\n", + " )" ] }, { @@ -293,7 +340,9 @@ "metadata": {}, "outputs": [], "source": [ - "all_subject_status[['technician_duties', 'owner_duties']] = all_subject_status.apply(translate_schedule, axis=1)" + "all_subject_status[[\"technician_duties\", \"owner_duties\"]] = all_subject_status.apply(\n", + " translate_schedule, axis=1\n", + ")" ] }, { @@ -645,7 +694,7 @@ } ], "source": [ - "all_subject_status['technician_duties'].unique()" + "all_subject_status[\"technician_duties\"].unique()" ] }, { @@ -675,7 +724,7 @@ } ], "source": [ - "all_subject_status['owner_duties'].unique()" + "all_subject_status[\"owner_duties\"].unique()" ] }, { @@ -838,7 +887,12 @@ } ], "source": [ - "all_subject_status.loc[all_subject_status['owner_duties'].str.contains('{\"assignment_style\": \"Specify each day\",'),:]" + "all_subject_status.loc[\n", + " all_subject_status[\"owner_duties\"].str.contains(\n", + " '{\"assignment_style\": \"Specify each day\",'\n", + " ),\n", + " :,\n", + "]" ] }, { @@ -1017,7 +1071,16 @@ } ], "source": [ - "all_subject_status_final = all_subject_status.loc[:, ['subject_fullname', 'subject_status', 'water_per_day', 'technician_duties', 'owner_duties']]\n", + "all_subject_status_final = all_subject_status.loc[\n", + " :,\n", + " [\n", + " \"subject_fullname\",\n", + " \"subject_status\",\n", + " \"water_per_day\",\n", + " \"technician_duties\",\n", + " \"owner_duties\",\n", + " ],\n", + "]\n", "all_subject_status_final" ] }, @@ -6038,7 +6101,7 @@ } ], "source": [ - "all_subject_status_final.to_dict('records')" + "all_subject_status_final.to_dict(\"records\")" ] }, { @@ -6047,7 +6110,7 @@ "metadata": {}, "outputs": [], "source": [ - "action.Responsibilities.insert(all_subject_status_final.to_dict('records'))" + "action.Responsibilities.insert(all_subject_status_final.to_dict(\"records\"))" ] }, { @@ -6072,7 +6135,7 @@ } ], "source": [ - "all_subject_status_final.loc[288,:]" + "all_subject_status_final.loc[288, :]" ] }, { diff --git a/notebooks/create_weighingGUI_ss.ipynb b/notebooks/create_weighingGUI_ss.ipynb index 8206baa9..b7fb7ea7 100644 --- a/notebooks/create_weighingGUI_ss.ipynb +++ b/notebooks/create_weighingGUI_ss.ipynb @@ -64,17 +64,20 @@ "import u19_pipeline.scheduler as scheduler\n", "import u19_pipeline.subject as subject\n", "\n", - "DJ_CUSTOM_VARIABLES_FILENAME = 'DJCustomVariables.csv'\n", - "SLACK_WEBHOOK_FILENAME = 'SlackChannels.csv'\n", - "USER_SLACK_FILENAME = 'UserSlack.csv'\n", - "RIG_STATUS_FILENAME = 'RigStatusTable.csv'\n", - "DAY_SCHEDULE_FILENAME = 'ScheduleDay.csv'\n", - "PAST_SESSION_PERFORMANCE_FILENAME = 'PastSessions.csv'\n", - "SUBJECT_MOTOR_POSITION_FILENAME = 'SubjectMotorPosition.csv'\n", - "\n", - "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = 'Weighing_GUI_Replacement_SpreadSheet_Template.xlsx'\n", - "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = 'Weighing_GUI_Replacement_SpreadSheet.xlsx'\n", + "DJ_CUSTOM_VARIABLES_FILENAME = \"DJCustomVariables.csv\"\n", + "SLACK_WEBHOOK_FILENAME = \"SlackChannels.csv\"\n", + "USER_SLACK_FILENAME = \"UserSlack.csv\"\n", + "RIG_STATUS_FILENAME = \"RigStatusTable.csv\"\n", + "DAY_SCHEDULE_FILENAME = \"ScheduleDay.csv\"\n", + "PAST_SESSION_PERFORMANCE_FILENAME = \"PastSessions.csv\"\n", + "SUBJECT_MOTOR_POSITION_FILENAME = \"SubjectMotorPosition.csv\"\n", "\n", + "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = (\n", + " \"Weighing_GUI_Replacement_SpreadSheet_Template.xlsx\"\n", + ")\n", + "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = (\n", + " \"Weighing_GUI_Replacement_SpreadSheet.xlsx\"\n", + ")\n", "\n", "\n", "MAX_SESSIONS_HISTORY = 75" @@ -89,11 +92,11 @@ "def cast_choice(choice_array):\n", "\n", " new_array = choice_array[0].copy()\n", - " new_array[new_array>2] = 127\n", + " new_array[new_array > 2] = 127\n", "\n", " new_array = np.array(new_array, dtype=np.uint8)\n", "\n", - " return new_array\n" + " return new_array" ] }, { @@ -104,8 +107,7 @@ "source": [ "def encode_webhook(webhook):\n", "\n", - " return (base64.b64encode(webhook.encode('utf-8'))).decode('utf-8')\n", - " " + " return (base64.b64encode(webhook.encode(\"utf-8\"))).decode(\"utf-8\")" ] }, { @@ -116,23 +118,41 @@ "source": [ "conf = dj.config\n", "\n", - "nodb_virmen_backup_dir = pathlib.Path(pathlib.Path(conf['custom']['root_data_dir'][0]).parent.parent,'Shared','NoDBVirmenBackup')\n", - "\n", - "DJ_CUSTOM_VARIABLES_FILENAME = pathlib.Path(nodb_virmen_backup_dir,DJ_CUSTOM_VARIABLES_FILENAME).as_posix()\n", - "SLACK_WEBHOOK_FILENAME = pathlib.Path(nodb_virmen_backup_dir,SLACK_WEBHOOK_FILENAME).as_posix()\n", - "USER_SLACK_FILENAME = pathlib.Path(nodb_virmen_backup_dir,USER_SLACK_FILENAME).as_posix()\n", - "RIG_STATUS_FILENAME = pathlib.Path(nodb_virmen_backup_dir,RIG_STATUS_FILENAME).as_posix()\n", - "DAY_SCHEDULE_FILENAME = pathlib.Path(nodb_virmen_backup_dir,DAY_SCHEDULE_FILENAME).as_posix()\n", - "PAST_SESSION_PERFORMANCE_FILENAME = pathlib.Path(nodb_virmen_backup_dir,PAST_SESSION_PERFORMANCE_FILENAME).as_posix()\n", - "SUBJECT_MOTOR_POSITION_FILENAME = pathlib.Path(nodb_virmen_backup_dir,SUBJECT_MOTOR_POSITION_FILENAME).as_posix()\n", + "nodb_virmen_backup_dir = pathlib.Path(\n", + " pathlib.Path(conf[\"custom\"][\"root_data_dir\"][0]).parent.parent,\n", + " \"Shared\",\n", + " \"NoDBVirmenBackup\",\n", + ")\n", "\n", - "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = pathlib.Path(nodb_virmen_backup_dir,\\\n", - " WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME).as_posix()\n", + "DJ_CUSTOM_VARIABLES_FILENAME = pathlib.Path(\n", + " nodb_virmen_backup_dir, DJ_CUSTOM_VARIABLES_FILENAME\n", + ").as_posix()\n", + "SLACK_WEBHOOK_FILENAME = pathlib.Path(\n", + " nodb_virmen_backup_dir, SLACK_WEBHOOK_FILENAME\n", + ").as_posix()\n", + "USER_SLACK_FILENAME = pathlib.Path(\n", + " nodb_virmen_backup_dir, USER_SLACK_FILENAME\n", + ").as_posix()\n", + "RIG_STATUS_FILENAME = pathlib.Path(\n", + " nodb_virmen_backup_dir, RIG_STATUS_FILENAME\n", + ").as_posix()\n", + "DAY_SCHEDULE_FILENAME = pathlib.Path(\n", + " nodb_virmen_backup_dir, DAY_SCHEDULE_FILENAME\n", + ").as_posix()\n", + "PAST_SESSION_PERFORMANCE_FILENAME = pathlib.Path(\n", + " nodb_virmen_backup_dir, PAST_SESSION_PERFORMANCE_FILENAME\n", + ").as_posix()\n", + "SUBJECT_MOTOR_POSITION_FILENAME = pathlib.Path(\n", + " nodb_virmen_backup_dir, SUBJECT_MOTOR_POSITION_FILENAME\n", + ").as_posix()\n", "\n", - "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = pathlib.Path(nodb_virmen_backup_dir,\\\n", - " WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE).as_posix()\n", + "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = pathlib.Path(\n", + " nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME\n", + ").as_posix()\n", "\n", - "\n" + "WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = pathlib.Path(\n", + " nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE\n", + ").as_posix()" ] }, { @@ -141,7 +161,9 @@ "metadata": {}, "outputs": [], "source": [ - "pd.DataFrame(lab.DjCustomVariables.fetch(as_dict=True)).to_csv(DJ_CUSTOM_VARIABLES_FILENAME)\n" + "pd.DataFrame(lab.DjCustomVariables.fetch(as_dict=True)).to_csv(\n", + " DJ_CUSTOM_VARIABLES_FILENAME\n", + ")" ] }, { @@ -151,9 +173,9 @@ "outputs": [], "source": [ "slack_webhooks = pd.DataFrame(lab.SlackWebhooks.fetch(as_dict=True))\n", - "slack_webhooks['webhook_url'] = slack_webhooks['webhook_url'].astype(str)\n", + "slack_webhooks[\"webhook_url\"] = slack_webhooks[\"webhook_url\"].astype(str)\n", "\n", - "slack_webhooks['webhook_url'] = slack_webhooks['webhook_url'].apply(encode_webhook)\n", + "slack_webhooks[\"webhook_url\"] = slack_webhooks[\"webhook_url\"].apply(encode_webhook)\n", "slack_webhooks.to_csv(SLACK_WEBHOOK_FILENAME, index=False)" ] }, @@ -163,11 +185,15 @@ "metadata": {}, "outputs": [], "source": [ - "user_data = pd.DataFrame(lab.User.fetch('user_id', 'slack', 'tech_responsibility', 'slack_webhook',as_dict=True))\n", - "user_data['slack_webhook'] = user_data['slack_webhook'].astype(str)\n", - "user_data['slack_webhook'] = user_data['slack_webhook'].fillna('')\n", + "user_data = pd.DataFrame(\n", + " lab.User.fetch(\n", + " \"user_id\", \"slack\", \"tech_responsibility\", \"slack_webhook\", as_dict=True\n", + " )\n", + ")\n", + "user_data[\"slack_webhook\"] = user_data[\"slack_webhook\"].astype(str)\n", + "user_data[\"slack_webhook\"] = user_data[\"slack_webhook\"].fillna(\"\")\n", "\n", - "user_data['slack_webhook'] = user_data['slack_webhook'].apply(encode_webhook)\n", + "user_data[\"slack_webhook\"] = user_data[\"slack_webhook\"].apply(encode_webhook)\n", "user_data.to_csv(USER_SLACK_FILENAME, index=False)" ] }, @@ -177,7 +203,11 @@ "metadata": {}, "outputs": [], "source": [ - "df_rig_status = pd.DataFrame(scheduler.RigStatus.fetch('location', 'input_output_name','current_status',as_dict=True))\n", + "df_rig_status = pd.DataFrame(\n", + " scheduler.RigStatus.fetch(\n", + " \"location\", \"input_output_name\", \"current_status\", as_dict=True\n", + " )\n", + ")\n", "df_rig_status.to_csv(RIG_STATUS_FILENAME, index=False)" ] }, @@ -1370,13 +1400,21 @@ ], "source": [ "schedule_query = dict()\n", - "schedule_query['date'] = datetime.date.today() \n", + "schedule_query[\"date\"] = datetime.date.today()\n", "\n", - "subject_query = 'subject_fullname is not null'\n", + "subject_query = \"subject_fullname is not null\"\n", "\n", - "day_schedule = pd.DataFrame((scheduler.Schedule * scheduler.TrainingProfile * subject.Subject.proj(subject_user_id='user_id') & schedule_query & subject_query).fetch(as_dict=True))\n", - "day_schedule = day_schedule.drop(columns='user_id')\n", - "day_schedule = day_schedule.rename(columns={'subject_user_id':'user_id'})\n", + "day_schedule = pd.DataFrame(\n", + " (\n", + " scheduler.Schedule\n", + " * scheduler.TrainingProfile\n", + " * subject.Subject.proj(subject_user_id=\"user_id\")\n", + " & schedule_query\n", + " & subject_query\n", + " ).fetch(as_dict=True)\n", + ")\n", + "day_schedule = day_schedule.drop(columns=\"user_id\")\n", + "day_schedule = day_schedule.rename(columns={\"subject_user_id\": \"user_id\"})\n", "\n", "\n", "day_schedule.to_csv(DAY_SCHEDULE_FILENAME, index=False)\n", @@ -1389,12 +1427,12 @@ "metadata": {}, "outputs": [], "source": [ - "#shcedule = pd.read_csv(DAY_SCHEDULE_FILENAME)\n", - "#shcedule = shcedule.loc[:,['date', 'location','timeslot','subject_fullname','training_profile_id','recording_profile_id','input_output_profile_id','experimenters_instructions']]\n", - "#shcedule.loc[shcedule['experimenters_instructions'].isna(),'experimenters_instructions'] = ''\n", - "#shcedule\n", - "#list_of_dicts = shcedule.to_dict(orient='records')\n", - "#scheduler.Schedule.insert(list_of_dicts)" + "# shcedule = pd.read_csv(DAY_SCHEDULE_FILENAME)\n", + "# shcedule = shcedule.loc[:,['date', 'location','timeslot','subject_fullname','training_profile_id','recording_profile_id','input_output_profile_id','experimenters_instructions']]\n", + "# shcedule.loc[shcedule['experimenters_instructions'].isna(),'experimenters_instructions'] = ''\n", + "# shcedule\n", + "# list_of_dicts = shcedule.to_dict(orient='records')\n", + "# scheduler.Schedule.insert(list_of_dicts)" ] }, { @@ -1414,8 +1452,8 @@ } ], "source": [ - "all_subjects_schedule = \"', '\".join(day_schedule['subject_fullname'])\n", - "all_subjects_schedule = \"subject_fullname in ('\" +all_subjects_schedule+ \"')\"\n", + "all_subjects_schedule = \"', '\".join(day_schedule[\"subject_fullname\"])\n", + "all_subjects_schedule = \"subject_fullname in ('\" + all_subjects_schedule + \"')\"\n", "all_subjects_schedule" ] }, @@ -1779,7 +1817,9 @@ } ], "source": [ - "sp = pd.DataFrame((subject.HeadMotorPosition & all_subjects_schedule).fetch(as_dict=True))\n", + "sp = pd.DataFrame(\n", + " (subject.HeadMotorPosition & all_subjects_schedule).fetch(as_dict=True)\n", + ")\n", "\n", "sp.to_csv(SUBJECT_MOTOR_POSITION_FILENAME, index=False)\n", "sp" @@ -1948,20 +1988,34 @@ } ], "source": [ - "ss = pd.DataFrame((behavior.TowersSession & all_subjects_schedule).fetch('KEY', order_by='subject_fullname, session_date desc, session_number desc', as_dict=True))\n", - "ss['session_date'] = ss['session_date'].astype(str)\n", - "ss['num_sessions'] = ss.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')\n", - "ss['num_sessions'] = ss['num_sessions'].astype(int)\n", - "ss = ss.loc[ss['num_sessions'] <= MAX_SESSIONS_HISTORY, :]\n", + "ss = pd.DataFrame(\n", + " (behavior.TowersSession & all_subjects_schedule).fetch(\n", + " \"KEY\",\n", + " order_by=\"subject_fullname, session_date desc, session_number desc\",\n", + " as_dict=True,\n", + " )\n", + ")\n", + "ss[\"session_date\"] = ss[\"session_date\"].astype(str)\n", + "ss[\"num_sessions\"] = ss.groupby([\"subject_fullname\"])[\"subject_fullname\"].rank(\n", + " method=\"first\"\n", + ")\n", + "ss[\"num_sessions\"] = ss[\"num_sessions\"].astype(int)\n", + "ss = ss.loc[ss[\"num_sessions\"] <= MAX_SESSIONS_HISTORY, :]\n", "\n", - "#ss2 = ss.copy()\n", + "# ss2 = ss.copy()\n", "\n", - "max_value_indices = ss.groupby('subject_fullname')['num_sessions'].idxmax()\n", + "max_value_indices = ss.groupby(\"subject_fullname\")[\"num_sessions\"].idxmax()\n", "ss = ss.loc[max_value_indices]\n", "ss = ss.reset_index(drop=True)\n", - "ss['query'] = \"(subject_fullname='\" + ss['subject_fullname'] + \"' and session_date >= '\" + ss['session_date'] + \"')\"\n", + "ss[\"query\"] = (\n", + " \"(subject_fullname='\"\n", + " + ss[\"subject_fullname\"]\n", + " + \"' and session_date >= '\"\n", + " + ss[\"session_date\"]\n", + " + \"')\"\n", + ")\n", "\n", - "block_query = ' OR '.join(ss['query'])\n", + "block_query = \" OR \".join(ss[\"query\"])\n", "ss" ] }, @@ -1971,33 +2025,48 @@ "metadata": {}, "outputs": [], "source": [ - "sstable=(acquisition.SessionStarted.proj('local_path_behavior_file', 'session_location'))\n", - "stable = (acquisition.Session).proj(stimulusBank='stimulus_bank')\n", - "tstable = (behavior.TowersSession.proj(trialType='rewarded_side',choice='chosen_side', stimulusSet='stimulus_set')) \n", - "tbtable = (behavior.TowersBlock.proj('first_trial','n_trials', 'sublevel', mazeID='level', mainMazeID='main_level', easyBlockFlag='easy_block',\\\n", - " duration='block_duration', rewardMil='reward_mil', medianTrialDur='trial_duration_median', start='block_start_time')) \n", - "table_fetch = sstable * stable* tstable * tbtable\n", - "\n", - "allblocks = pd.DataFrame((table_fetch & block_query).fetch(order_by='subject_fullname, session_date desc, session_number desc, block desc',as_dict=True))\n", - "allblocks['from_DB'] = 1\n", - "allblocks['session_date'] = allblocks['session_date'].astype(str)\n", - "\n", - "allblocks['sublevel'] = allblocks['sublevel'].astype('Int64')\n", - "allblocks['choice'] = allblocks['choice'].apply(cast_choice)\n", - "allblocks['trialType'] = allblocks['trialType'].apply(cast_choice)\n", + "sstable = acquisition.SessionStarted.proj(\n", + " \"local_path_behavior_file\", \"session_location\"\n", + ")\n", + "stable = (acquisition.Session).proj(stimulusBank=\"stimulus_bank\")\n", + "tstable = behavior.TowersSession.proj(\n", + " trialType=\"rewarded_side\", choice=\"chosen_side\", stimulusSet=\"stimulus_set\"\n", + ")\n", + "tbtable = behavior.TowersBlock.proj(\n", + " \"first_trial\",\n", + " \"n_trials\",\n", + " \"sublevel\",\n", + " mazeID=\"level\",\n", + " mainMazeID=\"main_level\",\n", + " easyBlockFlag=\"easy_block\",\n", + " duration=\"block_duration\",\n", + " rewardMil=\"reward_mil\",\n", + " medianTrialDur=\"trial_duration_median\",\n", + " start=\"block_start_time\",\n", + ")\n", + "table_fetch = sstable * stable * tstable * tbtable\n", "\n", - "#allblocks['session_date'] = allblocks['session_date'].astype(str)\n", - "#allblocks['num_blocks'] = allblocks.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')\n", + "allblocks = pd.DataFrame(\n", + " (table_fetch & block_query).fetch(\n", + " order_by=\"subject_fullname, session_date desc, session_number desc, block desc\",\n", + " as_dict=True,\n", + " )\n", + ")\n", + "allblocks[\"from_DB\"] = 1\n", + "allblocks[\"session_date\"] = allblocks[\"session_date\"].astype(str)\n", "\n", - "#all_sessions_blocks = allblocks.drop_duplicates(subset=['subject_fullname','session_date','session_number']).copy()\n", - "#all_sessions_blocks['num_session_blocks'] = all_sessions_blocks.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')\n", - "#all_sessions_blocks2 = all_sessions_blocks.copy()\n", - "#max_value_indices = all_sessions_blocks.groupby('subject_fullname')['num_session_blocks'].idxmax()\n", - "#all_sessions_blocks = all_sessions_blocks.loc[max_value_indices]\n", + "allblocks[\"sublevel\"] = allblocks[\"sublevel\"].astype(\"Int64\")\n", + "allblocks[\"choice\"] = allblocks[\"choice\"].apply(cast_choice)\n", + "allblocks[\"trialType\"] = allblocks[\"trialType\"].apply(cast_choice)\n", "\n", + "# allblocks['session_date'] = allblocks['session_date'].astype(str)\n", + "# allblocks['num_blocks'] = allblocks.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')\n", "\n", - "\n", - " " + "# all_sessions_blocks = allblocks.drop_duplicates(subset=['subject_fullname','session_date','session_number']).copy()\n", + "# all_sessions_blocks['num_session_blocks'] = all_sessions_blocks.groupby(['subject_fullname'])['subject_fullname'].rank(method='first')\n", + "# all_sessions_blocks2 = all_sessions_blocks.copy()\n", + "# max_value_indices = all_sessions_blocks.groupby('subject_fullname')['num_session_blocks'].idxmax()\n", + "# all_sessions_blocks = all_sessions_blocks.loc[max_value_indices]" ] }, { @@ -2420,7 +2489,6 @@ } ], "source": [ - "\n", "allblocks.to_csv(PAST_SESSION_PERFORMANCE_FILENAME, index=False)\n", "allblocks" ] @@ -2432,33 +2500,42 @@ "outputs": [], "source": [ "subject_data = wwa.get_subject_data()\n", - "subject_data = subject_data.loc[:, ['user_id', 'subject_fullname', 'cage', 'headplate_image_path', 'last_weight', 'water_per_day']]\n", - "#subject_data = subject_data.loc[subject_data['user_id'] != 'testuser',:]\n", + "subject_data = subject_data.loc[\n", + " :,\n", + " [\n", + " \"user_id\",\n", + " \"subject_fullname\",\n", + " \"cage\",\n", + " \"headplate_image_path\",\n", + " \"last_weight\",\n", + " \"water_per_day\",\n", + " ],\n", + "]\n", + "# subject_data = subject_data.loc[subject_data['user_id'] != 'testuser',:]\n", "subject_data = subject_data.reset_index(drop=True)\n", "\n", - "dictionary_rename_cols =\\\n", - "{'user_id': 'owner',\n", - " 'subject_fullname': 'subject name',\n", - " 'cage': 'cage',\n", - " 'headplate_image_path': 'headplate',\n", - " 'last_weight': 'last weight',\n", - " 'water_per_day': 'water per day',\n", - " }\n", + "dictionary_rename_cols = {\n", + " \"user_id\": \"owner\",\n", + " \"subject_fullname\": \"subject name\",\n", + " \"cage\": \"cage\",\n", + " \"headplate_image_path\": \"headplate\",\n", + " \"last_weight\": \"last weight\",\n", + " \"water_per_day\": \"water per day\",\n", + "}\n", "\n", "subject_data = subject_data.rename(columns=dictionary_rename_cols)\n", "subject_data = subject_data[list(dictionary_rename_cols.values())]\n", - "subject_data['headplate'] = subject_data['headplate'].astype(str)\n", + "subject_data[\"headplate\"] = subject_data[\"headplate\"].astype(str)\n", "\n", - "headplate_image_paths = subject_data['headplate'].copy()\n", + "headplate_image_paths = subject_data[\"headplate\"].copy()\n", "\n", - "subject_data['last weight'] = subject_data['last weight'].round(1)\n", - "subject_data['water per day'] = subject_data['water per day'].round(1)\n", + "subject_data[\"last weight\"] = subject_data[\"last weight\"].round(1)\n", + "subject_data[\"water per day\"] = subject_data[\"water per day\"].round(1)\n", "\n", "\n", - "subject_data['headplate'] = ''\n", - "subject_data['today weight'] = ''\n", - "subject_data['today water'] = ''\n", - "\n" + "subject_data[\"headplate\"] = \"\"\n", + "subject_data[\"today weight\"] = \"\"\n", + "subject_data[\"today water\"] = \"\"" ] }, { @@ -2468,13 +2545,15 @@ "outputs": [], "source": [ "# Copy the source file to the destination, replacing if it exists\n", - "shutil.copy2(WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME)\n", + "shutil.copy2(\n", + " WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE,\n", + " WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME,\n", + ")\n", "\n", "book = openpyxl.load_workbook(WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME)\n", - "sheet = book['Sheet1'] # Replace 'Sheet1' with your sheet name\n", + "sheet = book[\"Sheet1\"] # Replace 'Sheet1' with your sheet name\n", "\n", - "#sheet.delete_rows(3, sheet.max_row - 1) #\n", - "\n" + "# sheet.delete_rows(3, sheet.max_row - 1) #" ] }, { @@ -2483,8 +2562,8 @@ "metadata": {}, "outputs": [], "source": [ - "#for r in dataframe_to_rows(subject_data, index=False, header=False):\n", - " #sheet.append(r)\n" + "# for r in dataframe_to_rows(subject_data, index=False, header=False):\n", + "# sheet.append(r)\n" ] }, { @@ -2493,7 +2572,9 @@ "metadata": {}, "outputs": [], "source": [ - "for r_idx, row in enumerate(dataframe_to_rows(subject_data, index=False, header=False), start=3):\n", + "for r_idx, row in enumerate(\n", + " dataframe_to_rows(subject_data, index=False, header=False), start=3\n", + "):\n", " for c_idx, value in enumerate(row, start=1):\n", " sheet.cell(row=r_idx, column=c_idx, value=value)" ] @@ -2504,23 +2585,22 @@ "metadata": {}, "outputs": [], "source": [ - "column_headplate = subject_data.columns.get_loc('headplate') + 1\n", + "column_headplate = subject_data.columns.get_loc(\"headplate\") + 1\n", "column_headplate = openpyxl.utils.get_column_letter(column_headplate)\n", "\n", "\n", "for i in range(headplate_image_paths.shape[0]):\n", - "\n", " headplate_path = headplate_image_paths[i]\n", - " cell_image = column_headplate+str(i+3)\n", + " cell_image = column_headplate + str(i + 3)\n", "\n", " if pathlib.Path.is_file(pathlib.Path(headplate_path)):\n", " img = Image(headplate_path)\n", " img.width = 60\n", " img.height = 60\n", " sheet.add_image(img, cell_image)\n", - " sheet.row_dimensions[i+3].height = 50 \n", + " sheet.row_dimensions[i + 3].height = 50\n", " else:\n", - " sheet.row_dimensions[i+3].height = 15" + " sheet.row_dimensions[i + 3].height = 15" ] }, { diff --git a/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb b/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb index 4cdbb4a5..ad4625af 100644 --- a/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb +++ b/notebooks/developer_notebooks/00-Datajoint-configuration.ipynb @@ -85,9 +85,9 @@ } ], "source": [ - "dj.config['database.host'] = 'datajoint00.pni.princeton.edu'\n", - "dj.config['database.user'] = 'alvaros' # enter yo\n", - "dj.config['database.password'] = getpass.getpass() # enter the password securily" + "dj.config[\"database.host\"] = \"datajoint00.pni.princeton.edu\"\n", + "dj.config[\"database.user\"] = \"alvaros\" # enter yo\n", + "dj.config[\"database.password\"] = getpass.getpass() # enter the password securily" ] }, { @@ -138,9 +138,9 @@ "metadata": {}, "outputs": [], "source": [ - "if os.name == 'nt':\n", - " if 'C:\\\\Program Files\\\\Graphviz\\\\bin' not in os.environ[\"PATH\"]:\n", - " os.environ[\"PATH\"] += ';C:\\\\Program Files\\\\Graphviz\\\\bin'" + "if os.name == \"nt\":\n", + " if \"C:\\\\Program Files\\\\Graphviz\\\\bin\" not in os.environ[\"PATH\"]:\n", + " os.environ[\"PATH\"] += \";C:\\\\Program Files\\\\Graphviz\\\\bin\"" ] }, { @@ -159,8 +159,8 @@ "metadata": {}, "outputs": [], "source": [ - "if 'custom' not in dj.config:\n", - " dj.config['custom'] = dict()" + "if \"custom\" not in dj.config:\n", + " dj.config[\"custom\"] = dict()" ] }, { @@ -169,7 +169,7 @@ "metadata": {}, "outputs": [], "source": [ - "dj.config['custom']['database.prefix'] = 'u19_'" + "dj.config[\"custom\"][\"database.prefix\"] = \"u19_\"" ] }, { @@ -185,13 +185,9 @@ "metadata": {}, "outputs": [], "source": [ - "location_extstorage = utility.get_network_path('u19_dj') + '/external_dj_blobs'\n", - "dj.config['stores'] = {\n", - " 'extstorage':\n", - " {\n", - " 'location': location_extstorage,\n", - " 'protocol': 'file'\n", - " }\n", + "location_extstorage = utility.get_network_path(\"u19_dj\") + \"/external_dj_blobs\"\n", + "dj.config[\"stores\"] = {\n", + " \"extstorage\": {\"location\": location_extstorage, \"protocol\": \"file\"}\n", "}" ] }, diff --git a/notebooks/developer_notebooks/00-Set-up-configuration.ipynb b/notebooks/developer_notebooks/00-Set-up-configuration.ipynb index 1be21a9a..3493106f 100644 --- a/notebooks/developer_notebooks/00-Set-up-configuration.ipynb +++ b/notebooks/developer_notebooks/00-Set-up-configuration.ipynb @@ -96,9 +96,11 @@ "outputs": [], "source": [ "# If there is only one root path.\n", - "dj.config['custom']['ephys_root_data_dir'] = utility.get_network_path('Bezos') + '/Manuel/neuropixel/manuel'\n", + "dj.config[\"custom\"][\"ephys_root_data_dir\"] = (\n", + " utility.get_network_path(\"Bezos\") + \"/Manuel/neuropixel/manuel\"\n", + ")\n", "# If there are multiple possible root paths:\n", - "#dj.config['custom']['ephys_root_data_dir'] = ['/mnt/bucket/labs/tank']" + "# dj.config['custom']['ephys_root_data_dir'] = ['/mnt/bucket/labs/tank']" ] }, { @@ -120,7 +122,7 @@ } ], "source": [ - "dj.config['custom']" + "dj.config[\"custom\"]" ] }, { diff --git a/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb b/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb index 42a3f4f5..ddffd45d 100644 --- a/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb +++ b/notebooks/developer_notebooks/01-ephys-workflow-structure.ipynb @@ -1090,8 +1090,8 @@ ], "source": [ "# plot diagram of tables in multiple schemas\n", - "#dj.Diagram(subject) + dj.Diagram(acquisition) + dj.Diagram(ephys_pipeline) + dj.Diagram(ephys_element)\n", - "#dj.Diagram(acquisition)\n", + "# dj.Diagram(subject) + dj.Diagram(acquisition) + dj.Diagram(ephys_pipeline) + dj.Diagram(ephys_element)\n", + "# dj.Diagram(acquisition)\n", "dj.Diagram(ephys_pipeline) + dj.Diagram(ephys_element)" ] }, @@ -1627,7 +1627,12 @@ ], "source": [ "# plot diagram of selected tables and schemas\n", - "dj.Diagram(subject.Subject) + dj.Diagram(acquisition.Session) + dj.Diagram(ephys_pipeline) + dj.Diagram(ephys_element)" + "(\n", + " dj.Diagram(subject.Subject)\n", + " + dj.Diagram(acquisition.Session)\n", + " + dj.Diagram(ephys_pipeline)\n", + " + dj.Diagram(ephys_element)\n", + ")" ] }, { @@ -1996,7 +2001,11 @@ ], "source": [ "# subject, session, and ephys session\n", - "dj.Diagram(subject.Subject) + dj.Diagram(acquisition.Session) + dj.Diagram(ephys_pipeline.EphysPipelineSession)" + "(\n", + " dj.Diagram(subject.Subject)\n", + " + dj.Diagram(acquisition.Session)\n", + " + dj.Diagram(ephys_pipeline.EphysPipelineSession)\n", + ")" ] }, { diff --git a/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb b/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb index a071d35a..c9bceae2 100644 --- a/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb +++ b/notebooks/developer_notebooks/01-imaging-workflow-structure.ipynb @@ -25,7 +25,7 @@ "source": [ "import os\n", "\n", - "os.chdir('../..')" + "os.chdir(\"../..\")" ] }, { @@ -2137,7 +2137,12 @@ ], "source": [ "# plot diagram of tables in multiple schemas\n", - "dj.Diagram(subject) + dj.Diagram(acquisition) + dj.Diagram(scan_element) + dj.Diagram(imaging_element)" + "(\n", + " dj.Diagram(subject)\n", + " + dj.Diagram(acquisition)\n", + " + dj.Diagram(scan_element)\n", + " + dj.Diagram(imaging_element)\n", + ")" ] }, { @@ -2683,7 +2688,11 @@ ], "source": [ "# plot diagram of selected tables and schemas\n", - "dj.Diagram(subject.Subject) + dj.Diagram(acquisition.Session) + dj.Diagram(imaging_element)" + "(\n", + " dj.Diagram(subject.Subject)\n", + " + dj.Diagram(acquisition.Session)\n", + " + dj.Diagram(imaging_element)\n", + ")" ] }, { @@ -2884,7 +2893,11 @@ ], "source": [ "# subject, session and scan\n", - "dj.Diagram(subject.Subject) + dj.Diagram(acquisition.Session) + dj.Diagram(scan_element.Scan)" + "(\n", + " dj.Diagram(subject.Subject)\n", + " + dj.Diagram(acquisition.Session)\n", + " + dj.Diagram(scan_element.Scan)\n", + ")" ] }, { diff --git a/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb b/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb index de1a4280..943864e0 100644 --- a/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb +++ b/notebooks/developer_notebooks/02-process-ephys-workflow.ipynb @@ -58,7 +58,7 @@ "source": [ "from scripts.conf_file_finding import try_find_conf_file\r\n", "\r\n", - "try_find_conf_file()" + "try_find_conf_file()\r" ] }, { @@ -102,7 +102,7 @@ } ], "source": [ - "dj.conn()" + "dj.conn()\r" ] }, { @@ -111,9 +111,7 @@ "metadata": {}, "outputs": [], "source": [ - "key = {\r\n", - " 'subject_fullname': 'ms81_M005',\r\n", - " 'session_date': datetime.date(2021, 5, 5)}" + "key = {\"subject_fullname\": \"ms81_M005\", \"session_date\": datetime.date(2021, 5, 5)}\r" ] }, { @@ -141,7 +139,7 @@ } ], "source": [ - "dj.Diagram(acquisition.Session) + ephys.EphysSession + dj.Diagram(ephys_element)" + "dj.Diagram(acquisition.Session) + ephys.EphysSession + dj.Diagram(ephys_element)\r" ] }, { @@ -266,7 +264,7 @@ } ], "source": [ - "ephys.EphysSession() # Current sessions" + "ephys.EphysSession() # Current sessions\r" ] }, { @@ -278,14 +276,16 @@ "# Do you have a new one to insert? This is how it is done:\r\n", "\r\n", "key = {\r\n", - " 'subject_fullname': 'ms81_M005',\r\n", - " 'session_date': datetime.date(2021, 5, 5),\r\n", - " 'session_number': 0,\r\n", - " 'ephys_directory': '/Users/ms81/project_Neuropixel/analyses/data/210507_M004/'\r\n", + " \"subject_fullname\": \"ms81_M005\",\r\n", + " \"session_date\": datetime.date(2021, 5, 5),\r\n", + " \"session_number\": 0,\r\n", + " \"ephys_directory\": \"/Users/ms81/project_Neuropixel/analyses/data/210507_M004/\",\r\n", "}\r\n", "\r\n", - "ephys.EphysSession.insert1(key, skip_duplicates=True) # insert1 only works for one entry\r\n", - "ephys.EphysSession()" + "ephys.EphysSession.insert1(\r\n", + " key, skip_duplicates=True\r\n", + ") # insert1 only works for one entry\r\n", + "ephys.EphysSession()\r" ] }, { @@ -294,7 +294,11 @@ "metadata": {}, "outputs": [], "source": [ - "dj.Diagram(ephys.EphysSession) + dj.Diagram(probe_element.Probe) + dj.Diagram(ephys_element.ProbeInsertion)" + "(\r\n", + " dj.Diagram(ephys.EphysSession)\r\n", + " + dj.Diagram(probe_element.Probe)\r\n", + " + dj.Diagram(ephys_element.ProbeInsertion)\r\n", + ")\r" ] }, { @@ -310,7 +314,7 @@ "metadata": {}, "outputs": [], "source": [ - "from u19_pipeline.ingest import ephys_element_ingest" + "from u19_pipeline.ingest import ephys_element_ingest\r" ] }, { @@ -319,8 +323,8 @@ "metadata": {}, "outputs": [], "source": [ - "for sess_key in ephys.EphysSession.fetch('KEY'):\r\n", - " ephys_element_ingest.process_session(sess_key)" + "for sess_key in ephys.EphysSession.fetch(\"KEY\"):\r\n", + " ephys_element_ingest.process_session(sess_key)\r" ] }, { @@ -329,7 +333,7 @@ "metadata": {}, "outputs": [], "source": [ - "probe_element.Probe()" + "probe_element.Probe()\r" ] }, { @@ -338,7 +342,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.ProbeInsertion()" + "ephys_element.ProbeInsertion()\r" ] }, { @@ -364,8 +368,12 @@ "metadata": {}, "outputs": [], "source": [ - "dj.Diagram(ephys_element.ProbeInsertion) + probe_element.ElectrodeConfig\\\r\n", - "+ ephys_element.EphysRecording + ephys_element.EphysRecording.EphysFile" + "(\r\n", + " dj.Diagram(ephys_element.ProbeInsertion)\r\n", + " + probe_element.ElectrodeConfig\r\n", + " + ephys_element.EphysRecording\r\n", + " + ephys_element.EphysRecording.EphysFile\r\n", + ")\r" ] }, { @@ -374,7 +382,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.EphysRecording()" + "ephys_element.EphysRecording()\r" ] }, { @@ -383,7 +391,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.EphysRecording.populate(display_progress=True)" + "ephys_element.EphysRecording.populate(display_progress=True)\r" ] }, { @@ -399,8 +407,8 @@ "metadata": {}, "outputs": [], "source": [ - "probe_insertion_key = ephys_element.ProbeInsertion.fetch('KEY', limit=1)[0]\r\n", - "ephys_element.EphysRecording * probe_element.ElectrodeConfig.Electrode & probe_insertion_key" + "probe_insertion_key = ephys_element.ProbeInsertion.fetch(\"KEY\", limit=1)[0]\r\n", + "ephys_element.EphysRecording * probe_element.ElectrodeConfig.Electrode & probe_insertion_key\r" ] }, { @@ -409,7 +417,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.EphysRecording.EphysFile()" + "ephys_element.EphysRecording.EphysFile()\r" ] }, { @@ -439,7 +447,11 @@ "metadata": {}, "outputs": [], "source": [ - "dj.Diagram(ephys_element.EphysRecording) + ephys_element.ClusteringParamSet + ephys_element.ClusteringTask" + "(\r\n", + " dj.Diagram(ephys_element.EphysRecording)\r\n", + " + ephys_element.ClusteringParamSet\r\n", + " + ephys_element.ClusteringTask\r\n", + ")\r" ] }, { @@ -471,11 +483,12 @@ " \"nSkipCov\": 25,\r\n", " \"scaleproc\": 200,\r\n", " \"nPCs\": 3,\r\n", - " \"useRAM\": 0\r\n", + " \"useRAM\": 0,\r\n", "}\r\n", "ephys_element.ClusteringParamSet.insert_new_params(\r\n", - " 'kilosort2', 0, 'Spike sorting using Kilosort2', params_ks)\r\n", - "ephys_element.ClusteringParamSet()" + " \"kilosort2\", 0, \"Spike sorting using Kilosort2\", params_ks\r\n", + ")\r\n", + "ephys_element.ClusteringParamSet()\r" ] }, { @@ -491,7 +504,11 @@ "metadata": {}, "outputs": [], "source": [ - "dj.Diagram(ephys_element.EphysRecording) + ephys_element.ClusteringTask + ephys_element.Clustering" + "(\r\n", + " dj.Diagram(ephys_element.EphysRecording)\r\n", + " + ephys_element.ClusteringTask\r\n", + " + ephys_element.Clustering\r\n", + ")\r" ] }, { @@ -500,7 +517,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.ClusteringTask.describe();" + "ephys_element.ClusteringTask.describe();\r" ] }, { @@ -512,8 +529,12 @@ "# Specify path and animal information\r\n", "\r\n", "ephys_key = dict(\r\n", - " subject_fullname='ms81_M004', session_date='2021-05-07', session_number=0, insertion_number=0)\r\n", - "ephys_dir = '210507_M004/towersTask_g0_imec0/'" + " subject_fullname=\"ms81_M004\",\r\n", + " session_date=\"2021-05-07\",\r\n", + " session_number=0,\r\n", + " insertion_number=0,\r\n", + ")\r\n", + "ephys_dir = \"210507_M004/towersTask_g0_imec0/\"\r" ] }, { @@ -523,8 +544,10 @@ "outputs": [], "source": [ "ephys_element.ClusteringTask.insert1(\r\n", - " dict(**ephys_key, paramset_idx=0, clustering_output_dir=ephys_dir), skip_duplicates=True)\r\n", - "ephys_element.ClusteringTask()" + " dict(**ephys_key, paramset_idx=0, clustering_output_dir=ephys_dir),\r\n", + " skip_duplicates=True,\r\n", + ")\r\n", + "ephys_element.ClusteringTask()\r" ] }, { @@ -540,7 +563,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.Clustering.populate(display_progress=True)" + "ephys_element.Clustering.populate(display_progress=True)\r" ] }, { @@ -563,8 +586,12 @@ "metadata": {}, "outputs": [], "source": [ - "dj.Diagram(ephys_element.Clustering) + ephys_element.Curation + ephys_element.CuratedClustering \\\r\n", - "+ ephys_element.CuratedClustering.Unit" + "(\r\n", + " dj.Diagram(ephys_element.Clustering)\r\n", + " + ephys_element.Curation\r\n", + " + ephys_element.CuratedClustering\r\n", + " + ephys_element.CuratedClustering.Unit\r\n", + ")\r" ] }, { @@ -573,8 +600,8 @@ "metadata": {}, "outputs": [], "source": [ - "key = (ephys_element.ClusteringTask & ephys_key).fetch1('KEY')\r\n", - "ephys_element.Curation().create1_from_clustering_task(key)" + "key = (ephys_element.ClusteringTask & ephys_key).fetch1(\"KEY\")\r\n", + "ephys_element.Curation().create1_from_clustering_task(key)\r" ] }, { @@ -583,7 +610,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.Curation()" + "ephys_element.Curation()\r" ] }, { @@ -599,7 +626,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.CuratedClustering.populate(display_progress=True)" + "ephys_element.CuratedClustering.populate(display_progress=True)\r" ] }, { @@ -615,7 +642,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.CuratedClustering.Unit()" + "ephys_element.CuratedClustering.Unit()\r" ] }, { @@ -631,7 +658,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.LFP.populate(display_progress=True)" + "ephys_element.LFP.populate(display_progress=True)\r" ] }, { @@ -640,7 +667,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_element.LFP()" + "ephys_element.LFP()\r" ] }, { @@ -661,7 +688,7 @@ "outputs": [], "source": [ "# May take a while to populate depending on data size.\r\n", - "ephys_element.WaveformSet.populate(display_progress=True)" + "ephys_element.WaveformSet.populate(display_progress=True)\r" ] }, { @@ -684,7 +711,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys.BehaviorSync()" + "ephys.BehaviorSync()\r" ] }, { @@ -693,7 +720,7 @@ "metadata": {}, "outputs": [], "source": [ - "dj.config" + "dj.config\r" ] }, { @@ -702,7 +729,7 @@ "metadata": {}, "outputs": [], "source": [ - "get_session_directory(key)" + "get_session_directory(key)\r" ] }, { @@ -711,7 +738,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys.BehaviorSync.populate(key)" + "ephys.BehaviorSync.populate(key)\r" ] }, { @@ -720,7 +747,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys.BehaviorSync()" + "ephys.BehaviorSync()\r" ] }, { @@ -757,7 +784,7 @@ } ], "source": [ - "ephys.CuratedClustersIteration.populate(key)" + "ephys.CuratedClustersIteration.populate(key)\r" ] }, { @@ -897,7 +924,7 @@ } ], "source": [ - "ephys.CuratedClustersIteration.Unit & key & 'unit = 7'" + "ephys.CuratedClustersIteration.Unit & key & \"unit = 7\"\r" ] }, { @@ -906,9 +933,14 @@ "metadata": {}, "outputs": [], "source": [ - "key = {\"subject_fullname\": \"ms81_M004\", \"insertion_number\": 0, \"session_date\": '2021-05-07', 'curation_id': 1}\n", - "\n", - "ephys.BehaviorSync() & key" + "key = {\r\n", + " \"subject_fullname\": \"ms81_M004\",\r\n", + " \"insertion_number\": 0,\r\n", + " \"session_date\": \"2021-05-07\",\r\n", + " \"curation_id\": 1,\r\n", + "}\r\n", + "\r\n", + "ephys.BehaviorSync() & key\r" ] }, { @@ -917,17 +949,19 @@ "metadata": {}, "outputs": [], "source": [ - "spikes = (ephys.CuratedClustersIteration.Unit & key & 'insertion_number = 0' & 'unit = 7').fetch1('spike_counts_iteration')\n", - "\n", - "behavior = dj.create_virtual_module('subject', 'u19_behavior')\n", - "pos, time = (behavior.TowersBlock().Trial() & key).fetch('position', 'trial_time')\n", - "\n", - "position_at_each_iteration = []\n", - "for trial in range(len(pos)):\n", - " pp = pos[trial][:,1]\n", - " snip = np.zeros(len(time[trial]))*np.NaN\n", - " snip[0:len(pp)] = pp\n", - " position_at_each_iteration = np.append(position_at_each_iteration, snip)" + "spikes = (\r\n", + " ephys.CuratedClustersIteration.Unit & key & \"insertion_number = 0\" & \"unit = 7\"\r\n", + ").fetch1(\"spike_counts_iteration\")\r\n", + "\r\n", + "behavior = dj.create_virtual_module(\"subject\", \"u19_behavior\")\r\n", + "pos, time = (behavior.TowersBlock().Trial() & key).fetch(\"position\", \"trial_time\")\r\n", + "\r\n", + "position_at_each_iteration = []\r\n", + "for trial in range(len(pos)):\r\n", + " pp = pos[trial][:, 1]\r\n", + " snip = np.zeros(len(time[trial])) * np.NaN\r\n", + " snip[0 : len(pp)] = pp\r\n", + " position_at_each_iteration = np.append(position_at_each_iteration, snip)\r" ] }, { @@ -964,15 +998,15 @@ } ], "source": [ - "pl.figure(figsize=(15,5))\n", - "pl.subplot(1,2,1)\n", - "pl.plot(spikes)\n", - "pl.plot(position_at_each_iteration/50)\n", - "pl.ylim([0,10])\n", - "pl.xlim([0,30000])\n", - "\n", - "pl.subplot(1,2,2)\n", - "pl.plot(position_at_each_iteration[1:], spikes, '.')" + "pl.figure(figsize=(15, 5))\r\n", + "pl.subplot(1, 2, 1)\r\n", + "pl.plot(spikes)\r\n", + "pl.plot(position_at_each_iteration / 50)\r\n", + "pl.ylim([0, 10])\r\n", + "pl.xlim([0, 30000])\r\n", + "\r\n", + "pl.subplot(1, 2, 2)\r\n", + "pl.plot(position_at_each_iteration[1:], spikes, \".\")\r" ] }, { diff --git a/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb b/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb index 75cade0b..8e6d73e0 100644 --- a/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb +++ b/notebooks/developer_notebooks/02-process-imaging-workflow.ipynb @@ -155,10 +155,10 @@ } ], "source": [ - "subject = 'jeremyjc_j016'\n", - "date = '2021-11-21'\n", + "subject = \"jeremyjc_j016\"\n", + "date = \"2021-11-21\"\n", "\n", - "key = (imaging.Scan & dict(session_date =date, subject_fullname=subject)).fetch1('KEY')\n", + "key = (imaging.Scan & dict(session_date=date, subject_fullname=subject)).fetch1(\"KEY\")\n", "key" ] }, @@ -177,7 +177,7 @@ "metadata": {}, "outputs": [], "source": [ - "#(scan_element.ScanInfo & key).delete()" + "# (scan_element.ScanInfo & key).delete()" ] }, { @@ -227,71 +227,76 @@ "source": [ "# ingest parameters for Suite2p\n", "pars = {\n", - " 'look_one_level_down': 0.0,\n", - " 'fast_disk': [],\n", - " 'delete_bin': False,\n", - " 'mesoscan': False,\n", - " 'h5py': [],\n", - " 'h5py_key': 'data',\n", - " 'save_path0': [],\n", - " 'subfolders': [],\n", - " 'nplanes': 1,\n", - " 'nchannels': 1,\n", - " 'functional_chan': 1,\n", - " 'tau': 1.0,\n", - " 'fs': 10.0,\n", - " 'force_sktiff': False,\n", - " 'preclassify': 0.0,\n", - " 'save_mat': False,\n", - " 'combined': True,\n", - " 'aspect': 1.0,\n", - " 'do_bidiphase': False,\n", - " 'bidiphase': 0.0,\n", - " 'do_registration': True,\n", - " 'keep_movie_raw': False,\n", - " 'nimg_init': 300,\n", - " 'batch_size': 500,\n", - " 'maxregshift': 0.1,\n", - " 'align_by_chan': 1,\n", - " 'reg_tif': False,\n", - " 'reg_tif_chan2': False,\n", - " 'subpixel': 10,\n", - " 'smooth_sigma': 1.15,\n", - " 'th_badframes': 1.0,\n", - " 'pad_fft': False,\n", - " 'nonrigid': True,\n", - " 'block_size': [128, 128],\n", - " 'snr_thresh': 1.2,\n", - " 'maxregshiftNR': 5.0,\n", - " '1Preg': False,\n", - " 'spatial_hp': 50.0,\n", - " 'pre_smooth': 2.0,\n", - " 'spatial_taper': 50.0,\n", - " 'roidetect': True,\n", - " 'sparse_mode': False,\n", - " 'diameter': 12,\n", - " 'spatial_scale': 0,\n", - " 'connected': True,\n", - " 'nbinned': 5000,\n", - " 'max_iterations': 20,\n", - " 'threshold_scaling': 1.0,\n", - " 'max_overlap': 0.75,\n", - " 'high_pass': 100.0,\n", - " 'inner_neuropil_radius': 2,\n", - " 'min_neuropil_pixels': 350,\n", - " 'allow_overlap': False,\n", - " 'chan2_thres': 0.65,\n", - " 'baseline': 'maximin',\n", - " 'win_baseline': 60.0,\n", - " 'sig_baseline': 10.0,\n", - " 'prctile_baseline': 8.0,\n", - " 'neucoeff': 0.7,\n", - " 'xrange': np.array([0, 0]),\n", - " 'yrange': np.array([0, 0])}\n", + " \"look_one_level_down\": 0.0,\n", + " \"fast_disk\": [],\n", + " \"delete_bin\": False,\n", + " \"mesoscan\": False,\n", + " \"h5py\": [],\n", + " \"h5py_key\": \"data\",\n", + " \"save_path0\": [],\n", + " \"subfolders\": [],\n", + " \"nplanes\": 1,\n", + " \"nchannels\": 1,\n", + " \"functional_chan\": 1,\n", + " \"tau\": 1.0,\n", + " \"fs\": 10.0,\n", + " \"force_sktiff\": False,\n", + " \"preclassify\": 0.0,\n", + " \"save_mat\": False,\n", + " \"combined\": True,\n", + " \"aspect\": 1.0,\n", + " \"do_bidiphase\": False,\n", + " \"bidiphase\": 0.0,\n", + " \"do_registration\": True,\n", + " \"keep_movie_raw\": False,\n", + " \"nimg_init\": 300,\n", + " \"batch_size\": 500,\n", + " \"maxregshift\": 0.1,\n", + " \"align_by_chan\": 1,\n", + " \"reg_tif\": False,\n", + " \"reg_tif_chan2\": False,\n", + " \"subpixel\": 10,\n", + " \"smooth_sigma\": 1.15,\n", + " \"th_badframes\": 1.0,\n", + " \"pad_fft\": False,\n", + " \"nonrigid\": True,\n", + " \"block_size\": [128, 128],\n", + " \"snr_thresh\": 1.2,\n", + " \"maxregshiftNR\": 5.0,\n", + " \"1Preg\": False,\n", + " \"spatial_hp\": 50.0,\n", + " \"pre_smooth\": 2.0,\n", + " \"spatial_taper\": 50.0,\n", + " \"roidetect\": True,\n", + " \"sparse_mode\": False,\n", + " \"diameter\": 12,\n", + " \"spatial_scale\": 0,\n", + " \"connected\": True,\n", + " \"nbinned\": 5000,\n", + " \"max_iterations\": 20,\n", + " \"threshold_scaling\": 1.0,\n", + " \"max_overlap\": 0.75,\n", + " \"high_pass\": 100.0,\n", + " \"inner_neuropil_radius\": 2,\n", + " \"min_neuropil_pixels\": 350,\n", + " \"allow_overlap\": False,\n", + " \"chan2_thres\": 0.65,\n", + " \"baseline\": \"maximin\",\n", + " \"win_baseline\": 60.0,\n", + " \"sig_baseline\": 10.0,\n", + " \"prctile_baseline\": 8.0,\n", + " \"neucoeff\": 0.7,\n", + " \"xrange\": np.array([0, 0]),\n", + " \"yrange\": np.array([0, 0]),\n", + "}\n", "\n", "\n", "imaging_element.ProcessingParamSet.insert_new_params(\n", - " 'suite2p', 0, 'Calcium imaging analysis with Suite2p using default Suite2p parameters', pars)" + " \"suite2p\",\n", + " 0,\n", + " \"Calcium imaging analysis with Suite2p using default Suite2p parameters\",\n", + " pars,\n", + ")" ] }, { @@ -417,7 +422,7 @@ "metadata": {}, "outputs": [], "source": [ - "#(imaging_element.ProcessingTask() & key).delete()" + "# (imaging_element.ProcessingTask() & key).delete()" ] }, { @@ -542,10 +547,13 @@ } ], "source": [ - "scan_keys = (scan_element.Scan & key).fetch('KEY')\n", + "scan_keys = (scan_element.Scan & key).fetch(\"KEY\")\n", "for scan_key in scan_keys:\n", " output_dir = get_suite2p_dir(scan_key)\n", - " imaging_element.ProcessingTask.insert1(dict(**scan_key, paramset_idx=0, processing_output_dir=output_dir), skip_duplicates=True)\n", + " imaging_element.ProcessingTask.insert1(\n", + " dict(**scan_key, paramset_idx=0, processing_output_dir=output_dir),\n", + " skip_duplicates=True,\n", + " )\n", "\n", "imaging_element.ProcessingTask() & key" ] @@ -716,7 +724,7 @@ "metadata": {}, "outputs": [], "source": [ - "processing_keys = imaging_element.Processing.fetch('KEY')\n", + "processing_keys = imaging_element.Processing.fetch(\"KEY\")\n", "for processing_key in processing_keys:\n", " imaging_element.Curation().create1_from_processing_task(processing_key)" ] @@ -1651,7 +1659,7 @@ "metadata": {}, "outputs": [], "source": [ - "scan_key = (scan_element.Scan & key).fetch('KEY')[0]" + "scan_key = (scan_element.Scan & key).fetch(\"KEY\")[0]" ] }, { @@ -1660,8 +1668,10 @@ "metadata": {}, "outputs": [], "source": [ - "px_height, px_width = (scan_element.ScanInfo().Field & scan_key).fetch1('px_height', 'px_width')\n", - "masks = (imaging_element.Segmentation.Mask & scan_key & 'curation_id=1').fetch()" + "px_height, px_width = (scan_element.ScanInfo().Field & scan_key).fetch1(\n", + " \"px_height\", \"px_width\"\n", + ")\n", + "masks = (imaging_element.Segmentation.Mask & scan_key & \"curation_id=1\").fetch()" ] }, { @@ -1680,7 +1690,7 @@ "outputs": [], "source": [ "for mask in masks:\n", - " img[mask['mask_xpix'], mask['mask_ypix']] = mask['mask_weights']" + " img[mask[\"mask_xpix\"], mask[\"mask_ypix\"]] = mask[\"mask_weights\"]" ] }, { diff --git a/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb b/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb index ab588de1..24afd8fe 100644 --- a/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb +++ b/notebooks/developer_notebooks/Creating_the_Scheduling_Table.ipynb @@ -61,7 +61,7 @@ "metadata": {}, "outputs": [], "source": [ - "db_name = 'u19_'" + "db_name = \"u19_\"" ] }, { @@ -71,9 +71,9 @@ "outputs": [], "source": [ "connect_mod = lambda x: dj.VirtualModule(x, db_name + x)\n", - "lab = connect_mod('lab')\n", - "subject = connect_mod('subject')\n", - "scheduler = connect_mod('scheduler')\n", + "lab = connect_mod(\"lab\")\n", + "subject = connect_mod(\"subject\")\n", + "scheduler = connect_mod(\"scheduler\")\n", "# lab = dj.VirtualModule('lab', db_name + 'lab')\n", "# lab = dj.VirtualModule('lab', db_name + 'lab')" ] @@ -145,7 +145,7 @@ "source": [ "# scheduler.BehaviorProfile.drop()\n", "# scheduler.RecordingProfile.drop()\n", - "scheduler.Schedule.drop()\n" + "scheduler.Schedule.drop()" ] }, { @@ -165,7 +165,7 @@ } ], "source": [ - "behavior_profile_variables = {'nested_key': 'nested_value'}\n", + "behavior_profile_variables = {\"nested_key\": \"nested_value\"}\n", "json.dumps(behavior_profile_variables)\n", "# json.dumps(behavior_profile_variables, separators=(',', \":\"))" ] @@ -180,20 +180,18 @@ "\n", "import numpy as np\n", "\n", - "json_dump = json.dumps(behavior_profile_variables, separators=(',', \":\"))\n", + "json_dump = json.dumps(behavior_profile_variables, separators=(\",\", \":\"))\n", "length = len(json_dump)\n", "# Encode the string using Latin-1\n", "# byte_array = bytearray(json_dump, 'latin1')\n", "\n", "\n", - "\n", - "\n", "behavior_dict = {\n", - " 'user_id': 'testuser',\n", - " 'date_created': '2024-10-01',\n", - " 'behavior_profile_name': 'TestUserBehavior1',\n", - " 'behavior_profile_description': 'This is a sample profile description.',\n", - " 'behavior_profile_variables': np.array(['hi']), # Example binary data\n", + " \"user_id\": \"testuser\",\n", + " \"date_created\": \"2024-10-01\",\n", + " \"behavior_profile_name\": \"TestUserBehavior1\",\n", + " \"behavior_profile_description\": \"This is a sample profile description.\",\n", + " \"behavior_profile_variables\": np.array([\"hi\"]), # Example binary data\n", " # 'behavior_profile_variables': np.array(['hi'],dtype=f'U{length}') # Example binary data\n", " # 'behavior_profile_variables': np.array([json_dump],dtype=f'U{length}') # Example binary data\n", " # 'behavior_profile_variables': # Example binary data\n", @@ -217,7 +215,7 @@ } ], "source": [ - "repr(np.array(['hi']))\n", + "repr(np.array([\"hi\"]))\n", "scheduler.BehaviorProfile.insert1()" ] }, @@ -249,7 +247,7 @@ } ], "source": [ - "(scheduler.BehaviorProfile).fetch(as_dict=True)\n" + "(scheduler.BehaviorProfile).fetch(as_dict=True)" ] }, { @@ -277,7 +275,7 @@ } ], "source": [ - "(scheduler.BehaviorProfile & 'behavior_profile_id > 1').delete()\n" + "(scheduler.BehaviorProfile & \"behavior_profile_id > 1\").delete()" ] }, { @@ -304,7 +302,6 @@ } ], "source": [ - "\n", "scheduler.BehaviorProfile.heading\n", "\n", "# scheduler.RecordingProfile.heading" @@ -316,12 +313,13 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "recording_dict = {\n", - " 'date_created': '2024-10-28',\n", - " 'recording_profile_name': '',\n", - " 'recording_profile_description': 'This is a sample profile description.',\n", - " 'recording_profile_variables': {'nested_key': 'nested_value'} # Example binary data\n", + " \"date_created\": \"2024-10-28\",\n", + " \"recording_profile_name\": \"\",\n", + " \"recording_profile_description\": \"This is a sample profile description.\",\n", + " \"recording_profile_variables\": {\n", + " \"nested_key\": \"nested_value\"\n", + " }, # Example binary data\n", "}" ] }, @@ -331,7 +329,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "scheduler.RecordingProfile.insert1(recording_dict)" ] } diff --git a/notebooks/developer_notebooks/brightness_video_analysis.ipynb b/notebooks/developer_notebooks/brightness_video_analysis.ipynb index a36b0d20..683b7215 100644 --- a/notebooks/developer_notebooks/brightness_video_analysis.ipynb +++ b/notebooks/developer_notebooks/brightness_video_analysis.ipynb @@ -29,8 +29,7 @@ "import cv2\n", "import numpy as np\n", "import pandas as pd\n", - "import pylab as plt\n", - "\n" + "import pylab as plt" ] }, { @@ -53,7 +52,7 @@ "source": [ "import json\n", "\n", - "npy_data = np.load('/Users/alvaros/Downloads/ops_mika.npy', allow_pickle=True)\n", + "npy_data = np.load(\"/Users/alvaros/Downloads/ops_mika.npy\", allow_pickle=True)\n", "dict_data = npy_data[()]\n", "with open(\"/Users/alvaros/Downloads/ops_mika.json\", \"w\") as write_file:\n", " json.dump(dict_data, write_file)\n", @@ -90,11 +89,11 @@ "from u19_pipeline.imaging_pipeline import imaging_element as ie\n", "\n", "query = dict()\n", - "query['paramset_idx'] = 1\n", + "query[\"paramset_idx\"] = 1\n", "\n", "imaging_paramset = (ie.ProcessingParamSet & query).fetch(as_dict=True)\n", "\n", - "imset = imaging_paramset[0]['params']\n" + "imset = imaging_paramset[0][\"params\"]" ] }, { @@ -158,10 +157,11 @@ " shared_keys = d1_keys.intersection(d2_keys)\n", " added = d1_keys - d2_keys\n", " removed = d2_keys - d1_keys\n", - " modified = {o : (d1[o], d2[o]) for o in shared_keys if d1[o] != d2[o]}\n", + " modified = {o: (d1[o], d2[o]) for o in shared_keys if d1[o] != d2[o]}\n", " same = set(o for o in shared_keys if d1[o] == d2[o])\n", " return added, removed, modified, same\n", "\n", + "\n", "added, removed, modified, same = dict_compare(imset, a)" ] }, @@ -185,12 +185,15 @@ "metadata": {}, "outputs": [], "source": [ - "npy_manual = np.load('/Users/alvaros/Downloads/ops_dir/ops_manual.npy', allow_pickle=True)\n", - "npy_pipeline = np.load('/Users/alvaros/Downloads/ops_dir/ops_pipeline.npy', allow_pickle=True)\n", + "npy_manual = np.load(\n", + " \"/Users/alvaros/Downloads/ops_dir/ops_manual.npy\", allow_pickle=True\n", + ")\n", + "npy_pipeline = np.load(\n", + " \"/Users/alvaros/Downloads/ops_dir/ops_pipeline.npy\", allow_pickle=True\n", + ")\n", "\n", "npy_manual_dict = npy_manual.tolist()\n", - "npy_pipeline_dict = npy_pipeline.tolist()\n", - "\n" + "npy_pipeline_dict = npy_pipeline.tolist()" ] }, { @@ -1077,17 +1080,30 @@ } ], "source": [ - "cols_trials = ['subject_fullname', 'session_date', 'session_number', 'block', 'trial_idx', 'cue_presence_left', 'cue_presence_right', 'cue_onset_left', 'cue_onset_right', 'video_path']\n", - "order_by=('block', 'trial_idx')\n", + "cols_trials = [\n", + " \"subject_fullname\",\n", + " \"session_date\",\n", + " \"session_number\",\n", + " \"block\",\n", + " \"trial_idx\",\n", + " \"cue_presence_left\",\n", + " \"cue_presence_right\",\n", + " \"cue_onset_left\",\n", + " \"cue_onset_right\",\n", + " \"video_path\",\n", + "]\n", + "order_by = (\"block\", \"trial_idx\")\n", "\n", "for i in range(1):\n", - "\n", - " key_dict = blocks_video.loc[i+10, :].to_dict()\n", - " video_df = pd.DataFrame(((behavior.TowersBlockTrialVideo * behavior.TowersBlock.Trial) & key_dict).fetch(*cols_trials, as_dict=True, order_by=order_by))\n", - "\n", + " key_dict = blocks_video.loc[i + 10, :].to_dict()\n", + " video_df = pd.DataFrame(\n", + " (\n", + " (behavior.TowersBlockTrialVideo * behavior.TowersBlock.Trial) & key_dict\n", + " ).fetch(*cols_trials, as_dict=True, order_by=order_by)\n", + " )\n", "\n", " for j in range(1):\n", - " path_video = video_df.loc[j, 'video_path']\n", + " path_video = video_df.loc[j, \"video_path\"]\n", " path_video = path_video.replace(drive_videos, root_path)\n", " print(path_video)\n", "\n", @@ -1096,21 +1112,18 @@ " cap = cv2.VideoCapture(path_video)\n", "\n", " # Check if camera opened successfully\n", - " if (cap.isOpened()== False): \n", + " if cap.isOpened() == False:\n", " print(\"Error opening video stream or file\")\n", "\n", " total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n", " mean_value = np.zeros(total_frames)\n", " for k in range(total_frames):\n", - " \n", " # Capture frame-by-frame\n", " ret, frame = cap.read()\n", " if ret == True:\n", " mean_value[k] = np.mean(frame.flatten())\n", "\n", - "\n", - " cap.release()\n", - "\n" + " cap.release()" ] }, { @@ -1271,7 +1284,7 @@ } ], "source": [ - "video_df.head()\n" + "video_df.head()" ] }, { @@ -1294,16 +1307,15 @@ } ], "source": [ - " \n", - "fig, axs = plt.subplots(1, 1, figsize=(15,12))\n", + "fig, axs = plt.subplots(1, 1, figsize=(15, 12))\n", "\n", - "plt.plot(mean_value, linewidth=3, color ='k')\n", + "plt.plot(mean_value, linewidth=3, color=\"k\")\n", "\n", "plt.xticks(fontsize=16)\n", "plt.yticks(fontsize=16)\n", "plt.xlabel(\"Iteration # \", fontsize=18)\n", "plt.ylabel(\"Mean value RGB \", fontsize=18)\n", - "plt.title('Color change across time towers task', fontsize=20);\n" + "plt.title(\"Color change across time towers task\", fontsize=20);" ] }, { diff --git a/notebooks/developer_notebooks/delete_logs_periodically.ipynb b/notebooks/developer_notebooks/delete_logs_periodically.ipynb index 8459e2e9..0ffd2076 100644 --- a/notebooks/developer_notebooks/delete_logs_periodically.ipynb +++ b/notebooks/developer_notebooks/delete_logs_periodically.ipynb @@ -36,8 +36,7 @@ "import pathlib\n", "import re\n", "\n", - "import pandas as pd\n", - "\n" + "import pandas as pd" ] }, { @@ -67,10 +66,10 @@ } ], "source": [ - "#Behavior data\n", - "directory_path = pathlib.Path('Z:\\Shared\\log')\n", + "# Behavior data\n", + "directory_path = pathlib.Path(\"Z:\\Shared\\log\")\n", "date_pattern = r\"_\\d{8}_\"\n", - "reference_date = datetime.date.today() - datetime.timedelta(days=1)\n" + "reference_date = datetime.date.today() - datetime.timedelta(days=1)" ] }, { @@ -79,8 +78,11 @@ "metadata": {}, "outputs": [], "source": [ - "files = [p for p in directory_path.rglob(\"*\") if p.is_file() and re.search(date_pattern, p.name)]\n", - "\n" + "files = [\n", + " p\n", + " for p in directory_path.rglob(\"*\")\n", + " if p.is_file() and re.search(date_pattern, p.name)\n", + "]" ] }, { @@ -89,18 +91,23 @@ "metadata": {}, "outputs": [], "source": [ - "files = [p for p in directory_path.rglob(\"*\") if p.is_file() and re.search(date_pattern, p.name)]\n", + "files = [\n", + " p\n", + " for p in directory_path.rglob(\"*\")\n", + " if p.is_file() and re.search(date_pattern, p.name)\n", + "]\n", "\n", "\n", - "files_df = pd.DataFrame(files, columns=['filepaths'])\n", - "files_df['filename'] = files_df['filepaths'].apply(extract_filename)\n", - "files_df['exctract_date'] = files_df['filename'].str.extract(r'(\\d{8,' + str(8) + r'})')\n", - "files_df['exctract_date'] = pd.to_datetime(files_df['exctract_date'], format='%Y%m%d')\n", - "files_df['before_ref_date'] = files_df['exctract_date'] <= pd.Timestamp(reference_date)\n", + "files_df = pd.DataFrame(files, columns=[\"filepaths\"])\n", + "files_df[\"filename\"] = files_df[\"filepaths\"].apply(extract_filename)\n", + "files_df[\"exctract_date\"] = files_df[\"filename\"].str.extract(r\"(\\d{8,\" + str(8) + r\"})\")\n", + "files_df[\"exctract_date\"] = pd.to_datetime(files_df[\"exctract_date\"], format=\"%Y%m%d\")\n", + "files_df[\"before_ref_date\"] = files_df[\"exctract_date\"] <= pd.Timestamp(reference_date)\n", "\n", "\n", - "\n", - "files_for_deletion = files_df.loc[files_df['before_ref_date']==True, 'filepaths'].to_list()" + "files_for_deletion = files_df.loc[\n", + " files_df[\"before_ref_date\"] == True, \"filepaths\"\n", + "].to_list()" ] }, { diff --git a/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb b/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb index ee87810a..f35bbe46 100644 --- a/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb +++ b/notebooks/developer_notebooks/delete_old_live_session_stats.ipynb @@ -43,9 +43,7 @@ } ], "source": [ - "\n", - "import datajoint as dj\n", - "\n" + "import datajoint as dj" ] }, { @@ -62,7 +60,7 @@ } ], "source": [ - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')" + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")" ] }, { @@ -79,17 +77,18 @@ } ], "source": [ - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", "\n", "old_session_stats_query = \"current_datetime < NOW() - INTERVAL 15 DAY\"\n", "\n", "order_cols = \"'subject_fullname', 'session_date', 'session_number', 'trial_idx'\"\n", "connection = acquisition.LiveSessionStats.connection\n", "with connection.transaction:\n", - " old_session_stats = (acquisition.LiveSessionStats & old_session_stats_query).fetch(as_dict=True, order_by=[order_cols])\n", + " old_session_stats = (acquisition.LiveSessionStats & old_session_stats_query).fetch(\n", + " as_dict=True, order_by=[order_cols]\n", + " )\n", " (acquisition.HistoricSessionStats).insert(old_session_stats, skip_duplicates=True)\n", - " (acquisition.LiveSessionStats & old_session_stats_query).delete(safemode=False)\n", - "\n" + " (acquisition.LiveSessionStats & old_session_stats_query).delete(safemode=False)" ] }, { diff --git a/notebooks/developer_notebooks/query_for_tech_home_based_off_alvaros_big_query.ipynb b/notebooks/developer_notebooks/query_for_tech_home_based_off_alvaros_big_query.ipynb index b1aea32f..6f72c408 100644 --- a/notebooks/developer_notebooks/query_for_tech_home_based_off_alvaros_big_query.ipynb +++ b/notebooks/developer_notebooks/query_for_tech_home_based_off_alvaros_big_query.ipynb @@ -420,12 +420,17 @@ ], "source": [ "# Get max effective_date per subject\n", - "max_dates = dj.U('subject_fullname').aggr(\n", - " action.SubjectStatus,\n", - " last_status='MAX(effective_date)'\n", + "max_dates = dj.U(\"subject_fullname\").aggr(\n", + " action.SubjectStatus, last_status=\"MAX(effective_date)\"\n", ")\n", "\n", - "sub_cag_status_exclusion = (action.SubjectStatus & max_dates.proj(effective_date='last_status')).proj('subject_status','water_per_day','schedule',unused_date ='effective_date') * subject.Subject() * subject.CagingStatus()\n", + "sub_cag_status_exclusion = (\n", + " (action.SubjectStatus & max_dates.proj(effective_date=\"last_status\")).proj(\n", + " \"subject_status\", \"water_per_day\", \"schedule\", unused_date=\"effective_date\"\n", + " )\n", + " * subject.Subject()\n", + " * subject.CagingStatus()\n", + ")\n", "display(sub_cag_status_exclusion)" ] }, @@ -1682,34 +1687,37 @@ ], "source": [ "# Obtain if training occured today\n", - "today_training_sessions = dj.U('subject_fullname').aggr(\n", + "today_training_sessions = dj.U(\"subject_fullname\").aggr(\n", " acquisition.Session() & f'DATE(session_date) = \"{today}\"',\n", - " last_session='MAX(session_date)'\n", + " last_session=\"MAX(session_date)\",\n", ")\n", "\n", - "todays_water_admin = (action.WaterAdministration & f'DATE(administration_date) = \"{today}\"').proj(\n", + "todays_water_admin = (\n", + " action.WaterAdministration & f'DATE(administration_date) = \"{today}\"'\n", + ").proj(\n", " \"earned\",\n", " \"supplement\",\n", " \"received\",\n", " \"watertype_name\",\n", - " effective_date = 'administration_date')\n", + " effective_date=\"administration_date\",\n", + ")\n", "\n", "\n", "# First, get the max weighing times for today\n", - "today_max_weighing_times = dj.U('subject_fullname').aggr(\n", + "today_max_weighing_times = dj.U(\"subject_fullname\").aggr(\n", " action.Weighing() & f'DATE(weighing_time) = \"{today}\"', # Filter for today\n", " # last_weigh='MAX(weighing_time)',\n", - " weigh='round(avg(weight),2)',\n", - " effective_date = 'date(MAX(weighing_time))'\n", + " weigh=\"round(avg(weight),2)\",\n", + " effective_date=\"date(MAX(weighing_time))\",\n", ")\n", "\n", "\n", "# Water and training. Let's add weighing\n", - "weight_water_training = todays_water_admin.join(today_training_sessions, left=True).join(\n", - " today_max_weighing_times, left=True\n", - ")\n", + "weight_water_training = todays_water_admin.join(\n", + " today_training_sessions, left=True\n", + ").join(today_max_weighing_times, left=True)\n", "\n", - "weight_water_training_fetch = weight_water_training.fetch(format='frame')\n", + "weight_water_training_fetch = weight_water_training.fetch(format=\"frame\")\n", "weight_water_training_fetch" ] }, @@ -3738,7 +3746,7 @@ " sub_cag_status_exclusion.join(weight_water_training, left=True)\n", " & 'subject_status not in (\"Dead\", \"Missing\", \"AdLibWater\")'\n", ")\n", - "all_together_fetch = all_together.fetch(format='frame')\n", + "all_together_fetch = all_together.fetch(format=\"frame\")\n", "all_together_fetch" ] }, diff --git a/notebooks/developer_notebooks/review_ephys_sync.ipynb b/notebooks/developer_notebooks/review_ephys_sync.ipynb index a1b7d53b..8afe7e0c 100644 --- a/notebooks/developer_notebooks/review_ephys_sync.ipynb +++ b/notebooks/developer_notebooks/review_ephys_sync.ipynb @@ -46,8 +46,7 @@ "\n", "import datajoint as dj\n", "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n" + "import numpy as np" ] }, { @@ -67,8 +66,7 @@ "source": [ "from u19_pipeline import acquisition, ephys_pipeline\n", "\n", - "recording = dj.create_virtual_module(\"u19_recording\",\"u19_recording\")\n", - "\n" + "recording = dj.create_virtual_module(\"u19_recording\", \"u19_recording\")" ] }, { @@ -110,11 +108,10 @@ } ], "source": [ - "key = {'subject_fullname': 'jyanar_ya008',\n", - " 'session_date': datetime.date(2024, 5, 28)}\n", + "key = {\"subject_fullname\": \"jyanar_ya008\", \"session_date\": datetime.date(2024, 5, 28)}\n", "\n", "\n", - "session_key = (acquisition.Session & key).fetch1('KEY')\n", + "session_key = (acquisition.Session & key).fetch1(\"KEY\")\n", "session_key" ] }, @@ -135,7 +132,9 @@ } ], "source": [ - "id_recording = (recording.Recording.BehaviorSession & session_key).fetch('recording_id', as_dict=True)[0]\n", + "id_recording = (recording.Recording.BehaviorSession & session_key).fetch(\n", + " \"recording_id\", as_dict=True\n", + ")[0]\n", "id_recording" ] }, @@ -198,7 +197,7 @@ } ], "source": [ - "plt.plot(behavior_sync[0]['trial_index_nidq'])" + "plt.plot(behavior_sync[0][\"trial_index_nidq\"])" ] }, { @@ -259,18 +258,22 @@ "source": [ "session_dir = pathlib.Path(get_session_directory(key))\n", "print(session_dir)\n", - "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", - "nidq_bin_full_path = list(session_dir.glob('*nidq.bin*'))[0]\n", + "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", + "nidq_bin_full_path = list(session_dir.glob(\"*nidq.bin*\"))[0]\n", "\n", - "#Nidaq file\n", - "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n", + "# Nidaq file\n", + "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n", "nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n", - "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta)\n", + "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " nidq_bin_full_path, nidq_meta\n", + ")\n", "\n", - "#Behavior data\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n", + "# Behavior data\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", "thissession = behavior.TowersBlock().Trial() & key\n", - "behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n" + "behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n", + " \"trial_time\", \"vi_start\", \"iterations\"\n", + ")" ] }, { @@ -302,7 +305,7 @@ } ], "source": [ - "plt.plot(digital_array[1,97225711-10000:97225711+1000])" + "plt.plot(digital_array[1, 97225711 - 10000 : 97225711 + 1000])" ] }, { @@ -334,8 +337,8 @@ } ], "source": [ - "plt.plot(digital_array[1,97225711-1000:97225711+1000])\n", - "plt.plot(digital_array[2,97225711-1000:97225711+1000])" + "plt.plot(digital_array[1, 97225711 - 1000 : 97225711 + 1000])\n", + "plt.plot(digital_array[2, 97225711 - 1000 : 97225711 + 1000])" ] }, { @@ -1860,21 +1863,34 @@ } ], "source": [ - "mode = None #Default for sessions before 12/01/2021\n", - "#mode = 'pulses' #Default for sessions after 12/01/2021\n", - "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[1,:], digital_array[2,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n", + "mode = None # Default for sessions before 12/01/2021\n", + "# mode = 'pulses' #Default for sessions after 12/01/2021\n", + "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n", + " digital_array[1, :],\n", + " digital_array[2, :],\n", + " nidq_sampling_rate,\n", + " behavior_time.shape[0],\n", + " behavior_time,\n", + " mode=mode,\n", + ")\n", "# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n", "\n", "\n", "# Check # of trials and iterations match\n", - "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n", + "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n", + " ephys_utils.assert_iteration_samples_count(\n", + " iteration_dict[\"iter_start_idx\"], behavior_time\n", + " )\n", + ")\n", "\n", "print(trial_count_diff)\n", "print(trials_diff_iteration_big)\n", "print(trials_diff_iteration_small)\n", "\n", "\n", - "status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small)\n" + "status = ephys_utils.evaluate_sync_process(\n", + " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small\n", + ")" ] }, { @@ -1903,7 +1919,7 @@ "metadata": {}, "outputs": [], "source": [ - "it_times = iteration_dict['iter_times_idx'][trials_diff_iteration_small[0]]\n", + "it_times = iteration_dict[\"iter_times_idx\"][trials_diff_iteration_small[0]]\n", "beh_times = behavior_time[trials_diff_iteration_small[0]].flatten()" ] }, @@ -1925,9 +1941,9 @@ ], "source": [ "if it_times.shape[0] >= beh_times.shape[0]:\n", - " print('More pulses than behavior iterations, check other method')\n", + " print(\"More pulses than behavior iterations, check other method\")\n", "else:\n", - " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n", + " diff_vector = np.diff(it_times - beh_times[: it_times.shape[0]])\n", "\n", "import scipy\n", "\n", @@ -1965,7 +1981,7 @@ "metadata": {}, "outputs": [], "source": [ - "difo_iters = np.diff(iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]])\n", + "difo_iters = np.diff(iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]])\n", "peaks2, _ = scipy.signal.find_peaks(difo_iters, height=1500, distance=20)" ] }, @@ -1986,16 +2002,16 @@ } ], "source": [ - "original_iter = iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]]\n", - "new_iter_start = iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]]\n", + "original_iter = iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]]\n", + "new_iter_start = iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]]\n", "\n", "for i in range(peaks.shape[0]):\n", - " value_insert = (original_iter[peaks[i]] + original_iter[peaks[i]+1] ) /2\n", + " value_insert = (original_iter[peaks[i]] + original_iter[peaks[i] + 1]) / 2\n", " new_iter_start = np.insert(new_iter_start, peaks[i], value_insert)\n", "\n", "\n", - "if new_iter_start.shape[0] != beh_times[:it_times.shape[0]]:\n", - " print('with peak strategy, could not find correct missing iterations')\n" + "if new_iter_start.shape[0] != beh_times[: it_times.shape[0]]:\n", + " print(\"with peak strategy, could not find correct missing iterations\")" ] }, { @@ -2048,7 +2064,7 @@ } ], "source": [ - "times = new_iter_start/nidq_sampling_rate\n", + "times = new_iter_start / nidq_sampling_rate\n", "times = times - times[0]\n", "new_diff_times = times - beh_times\n", "plt.plot(new_diff_times)" @@ -2091,9 +2107,9 @@ ], "source": [ "print(iteration_dict.keys())\n", - "so = np.where(iteration_dict['trialnumber_vector_samples'] == 12)\n", + "so = np.where(iteration_dict[\"trialnumber_vector_samples\"] == 12)\n", "\n", - "print(so[0])\n" + "print(so[0])" ] }, { @@ -2139,20 +2155,37 @@ "end_iter = 2704\n", "samp_after = 60000\n", "\n", - "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n", - "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n", + "plt.plot(\n", + " digital_array[\n", + " 1,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot][end_iter],\n", + " ]\n", + " + samp_after\n", + ")\n", + "plt.plot(\n", + " digital_array[\n", + " 2,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot][end_iter],\n", + " ]\n", + " + samp_after\n", + ")\n", "\n", "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -2190,20 +2223,37 @@ "start_iter = 2704\n", "samp_after = 600\n", "\n", - "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+samp_after])\n", - "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+samp_after])\n", + "plt.plot(\n", + " digital_array[\n", + " 2,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n", + " + samp_after,\n", + " ]\n", + ")\n", + "plt.plot(\n", + " digital_array[\n", + " 1,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n", + " + samp_after,\n", + " ]\n", + ")\n", "\n", "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -2222,8 +2272,8 @@ ], "source": [ "trial_plot = 280\n", - "print(iteration_dict['iter_start_idx'][trial_plot][2704])\n", - "print(iteration_dict['iter_start_idx'][trial_plot+1][0])" + "print(iteration_dict[\"iter_start_idx\"][trial_plot][2704])\n", + "print(iteration_dict[\"iter_start_idx\"][trial_plot + 1][0])" ] }, { @@ -2255,7 +2305,7 @@ } ], "source": [ - "plt.plot(np.diff(iteration_dict['iter_start_idx'][trial_plot]/nidq_sampling_rate))" + "plt.plot(np.diff(iteration_dict[\"iter_start_idx\"][trial_plot] / nidq_sampling_rate))" ] }, { @@ -2335,14 +2385,14 @@ "source": [ "x = np.array([])\n", "mean_x = np.array([])\n", - "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n", - " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n", + "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n", + " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n", " mean_time_trial = np.mean(s)\n", - " x = np.append(x,s, axis=0)\n", + " x = np.append(x, s, axis=0)\n", " mean_x = np.append(mean_x, mean_time_trial)\n", "\n", - "#plt.plot(x)\n", - "plt.plot(mean_x)\n" + "# plt.plot(x)\n", + "plt.plot(mean_x)" ] }, { diff --git a/notebooks/developer_notebooks/startup_time_analysis.ipynb b/notebooks/developer_notebooks/startup_time_analysis.ipynb index 65bb14c5..9313af2e 100644 --- a/notebooks/developer_notebooks/startup_time_analysis.ipynb +++ b/notebooks/developer_notebooks/startup_time_analysis.ipynb @@ -34,7 +34,7 @@ } ], "source": [ - "action = dj.create_virtual_module('action', 'u19_action')\n" + "action = dj.create_virtual_module(\"action\", \"u19_action\")" ] }, { @@ -44,7 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "startup_times = action.RigStartupTime().fetch(format='frame')\n" + "startup_times = action.RigStartupTime().fetch(format=\"frame\")" ] }, { @@ -258,30 +258,38 @@ "\n", "# --- Deduplicate: within each location, if two startups are within 45s, keep the older one ---\n", "df = startup_times.reset_index().copy()\n", - "df['startup_datetime'] = pd.to_datetime(df['startup_datetime'])\n", - "df = df.sort_values(['location', 'startup_datetime'])\n", + "df[\"startup_datetime\"] = pd.to_datetime(df[\"startup_datetime\"])\n", + "df = df.sort_values([\"location\", \"startup_datetime\"])\n", + "\n", "\n", "def drop_close_duplicates(group, threshold_sec=45):\n", - " group = group.sort_values('startup_datetime').reset_index(drop=True)\n", + " group = group.sort_values(\"startup_datetime\").reset_index(drop=True)\n", " keep = [True]\n", " for i in range(1, len(group)):\n", - " delta = (group.loc[i, 'startup_datetime'] - group.loc[i-1, 'startup_datetime']).total_seconds()\n", + " delta = (\n", + " group.loc[i, \"startup_datetime\"] - group.loc[i - 1, \"startup_datetime\"]\n", + " ).total_seconds()\n", " keep.append(delta > threshold_sec)\n", " return group[keep]\n", "\n", + "\n", "df_clean = (\n", - " df.groupby('location', group_keys=True)\n", + " df.groupby(\"location\", group_keys=True)\n", " .apply(drop_close_duplicates)\n", " .reset_index(drop=False)\n", ")\n", - "df_clean['startup_time'] = pd.to_numeric(df_clean['startup_time'], errors='coerce')\n", - "df_clean['num_subj_scheduled'] = pd.to_numeric(df_clean['num_subj_scheduled'], errors='coerce')\n", + "df_clean[\"startup_time\"] = pd.to_numeric(df_clean[\"startup_time\"], errors=\"coerce\")\n", + "df_clean[\"num_subj_scheduled\"] = pd.to_numeric(\n", + " df_clean[\"num_subj_scheduled\"], errors=\"coerce\"\n", + ")\n", "\n", "# Remove rows with startup_time > 5000s\n", "n_before = len(df_clean)\n", - "df_clean = df_clean[df_clean['startup_time'] <= 5000].copy()\n", - "print(f\"Raw rows: {len(df)}, after dedup: {n_before}, after removing >5000s: {len(df_clean)} ({n_before - len(df_clean)} dropped)\")\n", - "df_clean.head()\n" + "df_clean = df_clean[df_clean[\"startup_time\"] <= 5000].copy()\n", + "print(\n", + " f\"Raw rows: {len(df)}, after dedup: {n_before}, after removing >5000s: {len(df_clean)} ({n_before - len(df_clean)} dropped)\"\n", + ")\n", + "df_clean.head()" ] }, { @@ -334,22 +342,30 @@ "source": [ "# --- Startup time by rig (location) ---\n", "rig_stats = (\n", - " df_clean.groupby('location')['startup_time']\n", - " .agg(median='median', mean='mean', std='std', count='count', p90=lambda x: x.quantile(0.9))\n", - " .sort_values('median', ascending=False)\n", + " df_clean.groupby(\"location\")[\"startup_time\"]\n", + " .agg(\n", + " median=\"median\",\n", + " mean=\"mean\",\n", + " std=\"std\",\n", + " count=\"count\",\n", + " p90=lambda x: x.quantile(0.9),\n", + " )\n", + " .sort_values(\"median\", ascending=False)\n", ")\n", "print(rig_stats.to_string())\n", "\n", "fig, ax = plt.subplots(figsize=(max(8, len(rig_stats) * 0.6), 5))\n", "order = rig_stats.index.tolist()\n", - "data_by_rig = [df_clean[df_clean['location'] == loc]['startup_time'].dropna() for loc in order]\n", + "data_by_rig = [\n", + " df_clean[df_clean[\"location\"] == loc][\"startup_time\"].dropna() for loc in order\n", + "]\n", "ax.boxplot(data_by_rig, tick_labels=order, vert=True)\n", - "ax.set_xlabel('Rig (location)')\n", - "ax.set_ylabel('Startup time (s)')\n", - "ax.set_title('Startup time distribution by rig')\n", - "plt.xticks(rotation=45, ha='right')\n", + "ax.set_xlabel(\"Rig (location)\")\n", + "ax.set_ylabel(\"Startup time (s)\")\n", + "ax.set_title(\"Startup time distribution by rig\")\n", + "plt.xticks(rotation=45, ha=\"right\")\n", "plt.tight_layout()\n", - "plt.show()\n" + "plt.show()" ] }, { @@ -384,9 +400,15 @@ "source": [ "# --- Startup time by startup_type ---\n", "type_stats = (\n", - " df_clean.groupby('startup_type')['startup_time']\n", - " .agg(median='median', mean='mean', std='std', count='count', p90=lambda x: x.quantile(0.9))\n", - " .sort_values('median', ascending=False)\n", + " df_clean.groupby(\"startup_type\")[\"startup_time\"]\n", + " .agg(\n", + " median=\"median\",\n", + " mean=\"mean\",\n", + " std=\"std\",\n", + " count=\"count\",\n", + " p90=lambda x: x.quantile(0.9),\n", + " )\n", + " .sort_values(\"median\", ascending=False)\n", ")\n", "print(type_stats.to_string())\n", "\n", @@ -395,25 +417,25 @@ "# Boxplot by startup_type\n", "order_t = type_stats.index.tolist()\n", "axes[0].boxplot(\n", - " [df_clean[df_clean['startup_type'] == t]['startup_time'].dropna() for t in order_t],\n", + " [df_clean[df_clean[\"startup_type\"] == t][\"startup_time\"].dropna() for t in order_t],\n", " tick_labels=order_t,\n", ")\n", - "axes[0].set_xlabel('startup_type')\n", - "axes[0].set_ylabel('Startup time (s)')\n", - "axes[0].set_title('Startup time by startup_type')\n", - "axes[0].tick_params(axis='x', rotation=30)\n", + "axes[0].set_xlabel(\"startup_type\")\n", + "axes[0].set_ylabel(\"Startup time (s)\")\n", + "axes[0].set_title(\"Startup time by startup_type\")\n", + "axes[0].tick_params(axis=\"x\", rotation=30)\n", "\n", "# Mean + 95% CI bar chart\n", - "means = type_stats['mean']\n", - "sems = df_clean.groupby('startup_type')['startup_time'].sem()\n", + "means = type_stats[\"mean\"]\n", + "sems = df_clean.groupby(\"startup_type\")[\"startup_time\"].sem()\n", "axes[1].bar(order_t, means[order_t], yerr=1.96 * sems[order_t], capsize=5)\n", - "axes[1].set_xlabel('startup_type')\n", - "axes[1].set_ylabel('Mean startup time (s)')\n", - "axes[1].set_title('Mean ± 95% CI by startup_type')\n", - "axes[1].tick_params(axis='x', rotation=30)\n", + "axes[1].set_xlabel(\"startup_type\")\n", + "axes[1].set_ylabel(\"Mean startup time (s)\")\n", + "axes[1].set_title(\"Mean ± 95% CI by startup_type\")\n", + "axes[1].tick_params(axis=\"x\", rotation=30)\n", "\n", "plt.tight_layout()\n", - "plt.show()\n" + "plt.show()" ] }, { @@ -450,42 +472,46 @@ ], "source": [ "# --- Effect of num_subj_scheduled on startup_time ---\n", - "corr = df_clean[['num_subj_scheduled', 'startup_time']].corr().iloc[0, 1]\n", + "corr = df_clean[[\"num_subj_scheduled\", \"startup_time\"]].corr().iloc[0, 1]\n", "print(f\"Pearson r(num_subj_scheduled, startup_time) = {corr:.3f}\")\n", "\n", "subj_stats = (\n", - " df_clean.groupby('num_subj_scheduled')['startup_time']\n", - " .agg(median='median', mean='mean', count='count')\n", + " df_clean.groupby(\"num_subj_scheduled\")[\"startup_time\"]\n", + " .agg(median=\"median\", mean=\"mean\", count=\"count\")\n", " .reset_index()\n", - " .sort_values('num_subj_scheduled')\n", + " .sort_values(\"num_subj_scheduled\")\n", ")\n", "print(subj_stats.to_string(index=False))\n", "\n", "fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n", "\n", "# Scatter with regression line\n", - "x = df_clean['num_subj_scheduled'].dropna()\n", - "y = df_clean.loc[x.index, 'startup_time']\n", + "x = df_clean[\"num_subj_scheduled\"].dropna()\n", + "y = df_clean.loc[x.index, \"startup_time\"]\n", "m, b = np.polyfit(x, y, 1)\n", "axes[0].scatter(x, y, alpha=0.3, s=15)\n", "xs = np.linspace(x.min(), x.max(), 100)\n", - "axes[0].plot(xs, m * xs + b, 'r-', label=f'slope={m:.1f}s/subj, r={corr:.2f}')\n", - "axes[0].set_xlabel('num_subj_scheduled')\n", - "axes[0].set_ylabel('Startup time (s)')\n", - "axes[0].set_title('Startup time vs. num_subj_scheduled')\n", + "axes[0].plot(xs, m * xs + b, \"r-\", label=f\"slope={m:.1f}s/subj, r={corr:.2f}\")\n", + "axes[0].set_xlabel(\"num_subj_scheduled\")\n", + "axes[0].set_ylabel(\"Startup time (s)\")\n", + "axes[0].set_title(\"Startup time vs. num_subj_scheduled\")\n", "axes[0].legend()\n", "\n", "# Mean startup time per num_subj_scheduled (size ∝ count)\n", "sc = axes[1].scatter(\n", - " subj_stats['num_subj_scheduled'], subj_stats['mean'],\n", - " s=subj_stats['count'] * 5, alpha=0.7,\n", + " subj_stats[\"num_subj_scheduled\"],\n", + " subj_stats[\"mean\"],\n", + " s=subj_stats[\"count\"] * 5,\n", + " alpha=0.7,\n", + ")\n", + "axes[1].set_xlabel(\"num_subj_scheduled\")\n", + "axes[1].set_ylabel(\"Mean startup time (s)\")\n", + "axes[1].set_title(\n", + " \"Mean startup time per num_subj_scheduled\\n(bubble size ∝ sample count)\"\n", ")\n", - "axes[1].set_xlabel('num_subj_scheduled')\n", - "axes[1].set_ylabel('Mean startup time (s)')\n", - "axes[1].set_title('Mean startup time per num_subj_scheduled\\n(bubble size ∝ sample count)')\n", "\n", "plt.tight_layout()\n", - "plt.show()\n" + "plt.show()" ] }, { @@ -529,12 +555,14 @@ ], "source": [ "# --- Filter to \"good data\" (after May 15th) ---\n", - "cutoff_date = pd.to_datetime('2026-05-15')\n", - "df_good = df_clean[df_clean['startup_datetime'] >= cutoff_date].copy()\n", - "print(f\"Filtered to {cutoff_date.date()} onwards: {len(df_good)} rows (from {len(df_clean)})\")\n", + "cutoff_date = pd.to_datetime(\"2026-05-15\")\n", + "df_good = df_clean[df_clean[\"startup_datetime\"] >= cutoff_date].copy()\n", + "print(\n", + " f\"Filtered to {cutoff_date.date()} onwards: {len(df_good)} rows (from {len(df_clean)})\"\n", + ")\n", "\n", "print(\"\\nAll unique locations in good data:\")\n", - "print(df_good['location'].value_counts())\n" + "print(df_good[\"location\"].value_counts())" ] }, { @@ -589,27 +617,35 @@ ], "source": [ "# === ALL RIGS ANALYSIS (after May 15th) ===\n", - "print(\"\\n\" + \"=\"*70)\n", + "print(\"\\n\" + \"=\" * 70)\n", "print(\"STARTUP TIME BY RIG (after 2026-05-15)\")\n", - "print(\"=\"*70)\n", + "print(\"=\" * 70)\n", "\n", "rig_stats_good = (\n", - " df_good.groupby('location')['startup_time']\n", - " .agg(median='median', mean='mean', std='std', count='count', p95=lambda x: x.quantile(0.95))\n", - " .sort_values('median', ascending=False)\n", + " df_good.groupby(\"location\")[\"startup_time\"]\n", + " .agg(\n", + " median=\"median\",\n", + " mean=\"mean\",\n", + " std=\"std\",\n", + " count=\"count\",\n", + " p95=lambda x: x.quantile(0.95),\n", + " )\n", + " .sort_values(\"median\", ascending=False)\n", ")\n", "print(rig_stats_good.to_string())\n", "\n", "fig, ax = plt.subplots(figsize=(max(10, len(rig_stats_good) * 0.6), 5))\n", "order = rig_stats_good.index.tolist()\n", - "data_by_rig = [df_good[df_good['location'] == loc]['startup_time'].dropna() for loc in order]\n", + "data_by_rig = [\n", + " df_good[df_good[\"location\"] == loc][\"startup_time\"].dropna() for loc in order\n", + "]\n", "ax.boxplot(data_by_rig, tick_labels=order, vert=True)\n", - "ax.set_xlabel('Rig (location)')\n", - "ax.set_ylabel('Startup time (s)')\n", - "ax.set_title('Startup time by rig (after 2026-05-15)')\n", - "plt.xticks(rotation=45, ha='right')\n", + "ax.set_xlabel(\"Rig (location)\")\n", + "ax.set_ylabel(\"Startup time (s)\")\n", + "ax.set_title(\"Startup time by rig (after 2026-05-15)\")\n", + "plt.xticks(rotation=45, ha=\"right\")\n", "plt.tight_layout()\n", - "plt.show()\n" + "plt.show()" ] }, { @@ -647,14 +683,20 @@ ], "source": [ "# === STARTUP TYPE ANALYSIS (after May 15th, all rigs) ===\n", - "print(\"\\n\" + \"=\"*70)\n", + "print(\"\\n\" + \"=\" * 70)\n", "print(\"STARTUP TYPE ANALYSIS (after 2026-05-15, all rigs)\")\n", - "print(\"=\"*70)\n", + "print(\"=\" * 70)\n", "\n", "type_stats_good = (\n", - " df_good.groupby('startup_type')['startup_time']\n", - " .agg(median='median', mean='mean', std='std', count='count', p95=lambda x: x.quantile(0.95))\n", - " .sort_values('median', ascending=False)\n", + " df_good.groupby(\"startup_type\")[\"startup_time\"]\n", + " .agg(\n", + " median=\"median\",\n", + " mean=\"mean\",\n", + " std=\"std\",\n", + " count=\"count\",\n", + " p95=lambda x: x.quantile(0.95),\n", + " )\n", + " .sort_values(\"median\", ascending=False)\n", ")\n", "print(type_stats_good.to_string())\n", "\n", @@ -662,24 +704,24 @@ "\n", "order_t = type_stats_good.index.tolist()\n", "axes[0].boxplot(\n", - " [df_good[df_good['startup_type'] == t]['startup_time'].dropna() for t in order_t],\n", + " [df_good[df_good[\"startup_type\"] == t][\"startup_time\"].dropna() for t in order_t],\n", " tick_labels=order_t,\n", ")\n", - "axes[0].set_xlabel('startup_type')\n", - "axes[0].set_ylabel('Startup time (s)')\n", - "axes[0].set_title('Startup time by startup_type (after 2026-05-15)')\n", - "axes[0].tick_params(axis='x', rotation=30)\n", + "axes[0].set_xlabel(\"startup_type\")\n", + "axes[0].set_ylabel(\"Startup time (s)\")\n", + "axes[0].set_title(\"Startup time by startup_type (after 2026-05-15)\")\n", + "axes[0].tick_params(axis=\"x\", rotation=30)\n", "\n", - "means_t = type_stats_good['mean']\n", - "sems_t = df_good.groupby('startup_type')['startup_time'].sem()\n", + "means_t = type_stats_good[\"mean\"]\n", + "sems_t = df_good.groupby(\"startup_type\")[\"startup_time\"].sem()\n", "axes[1].bar(order_t, means_t[order_t], yerr=1.96 * sems_t[order_t], capsize=5)\n", - "axes[1].set_xlabel('startup_type')\n", - "axes[1].set_ylabel('Mean startup time (s)')\n", - "axes[1].set_title('Mean ± 95% CI by startup_type (after 2026-05-15)')\n", - "axes[1].tick_params(axis='x', rotation=30)\n", + "axes[1].set_xlabel(\"startup_type\")\n", + "axes[1].set_ylabel(\"Mean startup time (s)\")\n", + "axes[1].set_title(\"Mean ± 95% CI by startup_type (after 2026-05-15)\")\n", + "axes[1].tick_params(axis=\"x\", rotation=30)\n", "\n", "plt.tight_layout()\n", - "plt.show()\n" + "plt.show()" ] }, { @@ -721,18 +763,18 @@ ], "source": [ "# === NUM_SUBJ_SCHEDULED ANALYSIS (after May 15th, all rigs) ===\n", - "print(\"\\n\" + \"=\"*70)\n", + "print(\"\\n\" + \"=\" * 70)\n", "print(\"NUM_SUBJ_SCHEDULED ANALYSIS (after 2026-05-15, all rigs)\")\n", - "print(\"=\"*70)\n", + "print(\"=\" * 70)\n", "\n", - "corr_good = df_good[['num_subj_scheduled', 'startup_time']].corr().iloc[0, 1]\n", + "corr_good = df_good[[\"num_subj_scheduled\", \"startup_time\"]].corr().iloc[0, 1]\n", "print(f\"Pearson r = {corr_good:.3f}\")\n", "\n", "subj_stats_good = (\n", - " df_good.groupby('num_subj_scheduled')['startup_time']\n", - " .agg(median='median', mean='mean', count='count')\n", + " df_good.groupby(\"num_subj_scheduled\")[\"startup_time\"]\n", + " .agg(median=\"median\", mean=\"mean\", count=\"count\")\n", " .reset_index()\n", - " .sort_values('num_subj_scheduled')\n", + " .sort_values(\"num_subj_scheduled\")\n", ")\n", "print(\"\\nStartup time by num_subj_scheduled:\")\n", "print(subj_stats_good.to_string(index=False))\n", @@ -740,30 +782,40 @@ "fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n", "\n", "# Scatter with regression\n", - "x_good = df_good['num_subj_scheduled'].dropna()\n", - "y_good = df_good.loc[x_good.index, 'startup_time']\n", + "x_good = df_good[\"num_subj_scheduled\"].dropna()\n", + "y_good = df_good.loc[x_good.index, \"startup_time\"]\n", "m_good, b_good = np.polyfit(x_good, y_good, 1)\n", "\n", "axes[0].scatter(x_good, y_good, alpha=0.3, s=20)\n", "xs_good = np.linspace(x_good.min(), x_good.max(), 100)\n", - "axes[0].plot(xs_good, m_good * xs_good + b_good, 'r-', linewidth=2, label=f'slope={m_good:.2f}s/subj, r={corr_good:.2f}')\n", - "axes[0].set_xlabel('num_subj_scheduled')\n", - "axes[0].set_ylabel('Startup time (s)')\n", - "axes[0].set_title('Startup time vs. num_subj_scheduled (after 2026-05-15)')\n", + "axes[0].plot(\n", + " xs_good,\n", + " m_good * xs_good + b_good,\n", + " \"r-\",\n", + " linewidth=2,\n", + " label=f\"slope={m_good:.2f}s/subj, r={corr_good:.2f}\",\n", + ")\n", + "axes[0].set_xlabel(\"num_subj_scheduled\")\n", + "axes[0].set_ylabel(\"Startup time (s)\")\n", + "axes[0].set_title(\"Startup time vs. num_subj_scheduled (after 2026-05-15)\")\n", "axes[0].legend()\n", "axes[0].grid(True, alpha=0.3)\n", "\n", "# Bubble chart\n", "axes[1].scatter(\n", - " subj_stats_good['num_subj_scheduled'], subj_stats_good['mean'],\n", - " s=subj_stats_good['count'] * 5, alpha=0.7,\n", + " subj_stats_good[\"num_subj_scheduled\"],\n", + " subj_stats_good[\"mean\"],\n", + " s=subj_stats_good[\"count\"] * 5,\n", + " alpha=0.7,\n", + ")\n", + "axes[1].set_xlabel(\"num_subj_scheduled\")\n", + "axes[1].set_ylabel(\"Mean startup time (s)\")\n", + "axes[1].set_title(\n", + " \"Mean startup time per num_subj_scheduled\\n(bubble size ∝ sample count, after 2026-05-15)\"\n", ")\n", - "axes[1].set_xlabel('num_subj_scheduled')\n", - "axes[1].set_ylabel('Mean startup time (s)')\n", - "axes[1].set_title('Mean startup time per num_subj_scheduled\\n(bubble size ∝ sample count, after 2026-05-15)')\n", "\n", "plt.tight_layout()\n", - "plt.show()\n" + "plt.show()" ] }, { @@ -790,7 +842,7 @@ "# === Which rigs take longest: filtered by startup_type ===\n", "# Show all startup_types present in good data so we can confirm the labels\n", "print(\"Startup types in good data:\")\n", - "print(df_good['startup_type'].value_counts())\n" + "print(df_good[\"startup_type\"].value_counts())" ] }, { @@ -875,40 +927,47 @@ "source": [ "# === Rig Tester vs Training Flow GUI: which rigs take the longest? ===\n", "# Adjust these strings to match the exact startup_type values printed above\n", - "df_rig_tester = df_good[df_good['startup_type'].str.contains('Rig Tester', case=False, na=False)]\n", - "df_training_gui = df_good[df_good['startup_type'].str.contains('Training Flow GUI', case=False, na=False)]\n", + "df_rig_tester = df_good[\n", + " df_good[\"startup_type\"].str.contains(\"Rig Tester\", case=False, na=False)\n", + "]\n", + "df_training_gui = df_good[\n", + " df_good[\"startup_type\"].str.contains(\"Training Flow GUI\", case=False, na=False)\n", + "]\n", "\n", "print(f\"Rig Tester startup_type rows: {len(df_rig_tester)}\")\n", "print(f\"Training Flow GUI startup_type rows: {len(df_training_gui)}\")\n", "\n", "# --- Which rigs take longest under each startup_type ---\n", - "for label, subset in [('Rig Tester', df_rig_tester), ('Training Flow GUI', df_training_gui)]:\n", + "for label, subset in [\n", + " (\"Rig Tester\", df_rig_tester),\n", + " (\"Training Flow GUI\", df_training_gui),\n", + "]:\n", " if len(subset) == 0:\n", " print(f\"\\n[{label}] No rows found — check startup_type label above\")\n", " continue\n", "\n", " stats = (\n", - " subset.groupby('location')['startup_time']\n", - " .agg(median='median', mean='mean', std='std', count='count')\n", - " .sort_values('median', ascending=False)\n", + " subset.groupby(\"location\")[\"startup_time\"]\n", + " .agg(median=\"median\", mean=\"mean\", std=\"std\", count=\"count\")\n", + " .sort_values(\"median\", ascending=False)\n", " )\n", - " print(f\"\\n{'='*60}\")\n", + " print(f\"\\n{'=' * 60}\")\n", " print(f\" {label}: startup time by rig (sorted slowest → fastest)\")\n", - " print(f\"{'='*60}\")\n", + " print(f\"{'=' * 60}\")\n", " print(stats.to_string())\n", "\n", " fig, ax = plt.subplots(figsize=(max(8, len(stats) * 0.6), 5))\n", " locs = stats.index.tolist()\n", " ax.boxplot(\n", - " [subset[subset['location'] == loc]['startup_time'].dropna() for loc in locs],\n", + " [subset[subset[\"location\"] == loc][\"startup_time\"].dropna() for loc in locs],\n", " tick_labels=locs,\n", " )\n", - " ax.set_xlabel('Rig (location)')\n", - " ax.set_ylabel('Startup time (s)')\n", - " ax.set_title(f'Startup time by rig — {label} (after 2026-05-15)')\n", - " plt.xticks(rotation=45, ha='right')\n", + " ax.set_xlabel(\"Rig (location)\")\n", + " ax.set_ylabel(\"Startup time (s)\")\n", + " ax.set_title(f\"Startup time by rig — {label} (after 2026-05-15)\")\n", + " plt.xticks(rotation=45, ha=\"right\")\n", " plt.tight_layout()\n", - " plt.show()\n" + " plt.show()" ] }, { @@ -972,54 +1031,67 @@ ], "source": [ "# === Does num_subj_scheduled matter within each startup_type? ===\n", - "for label, subset in [('Rig Tester', df_rig_tester), ('Training Flow GUI', df_training_gui)]:\n", + "for label, subset in [\n", + " (\"Rig Tester\", df_rig_tester),\n", + " (\"Training Flow GUI\", df_training_gui),\n", + "]:\n", " if len(subset) == 0:\n", " continue\n", "\n", - " x = subset['num_subj_scheduled'].dropna()\n", - " y = subset.loc[x.index, 'startup_time']\n", + " x = subset[\"num_subj_scheduled\"].dropna()\n", + " y = subset.loc[x.index, \"startup_time\"]\n", "\n", " if x.nunique() < 2:\n", - " print(f\"\\n[{label}] Not enough variation in num_subj_scheduled to fit regression\")\n", + " print(\n", + " f\"\\n[{label}] Not enough variation in num_subj_scheduled to fit regression\"\n", + " )\n", " continue\n", "\n", - " corr = subset[['num_subj_scheduled', 'startup_time']].corr().iloc[0, 1]\n", + " corr = subset[[\"num_subj_scheduled\", \"startup_time\"]].corr().iloc[0, 1]\n", " m, b = np.polyfit(x, y, 1)\n", "\n", " subj_stats = (\n", - " subset.groupby('num_subj_scheduled')['startup_time']\n", - " .agg(median='median', mean='mean', count='count')\n", + " subset.groupby(\"num_subj_scheduled\")[\"startup_time\"]\n", + " .agg(median=\"median\", mean=\"mean\", count=\"count\")\n", " .reset_index()\n", - " .sort_values('num_subj_scheduled')\n", + " .sort_values(\"num_subj_scheduled\")\n", " )\n", "\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\" {label}: effect of num_subj_scheduled (r={corr:.2f}, slope={m:.2f}s/subj)\")\n", - " print(f\"{'='*60}\")\n", + " print(f\"\\n{'=' * 60}\")\n", + " print(\n", + " f\" {label}: effect of num_subj_scheduled (r={corr:.2f}, slope={m:.2f}s/subj)\"\n", + " )\n", + " print(f\"{'=' * 60}\")\n", " print(subj_stats.to_string(index=False))\n", "\n", " fig, axes = plt.subplots(1, 2, figsize=(14, 4))\n", "\n", " axes[0].scatter(x, y, alpha=0.4, s=20)\n", " xs = np.linspace(x.min(), x.max(), 100)\n", - " axes[0].plot(xs, m * xs + b, 'r-', linewidth=2, label=f'slope={m:.2f}s/subj, r={corr:.2f}')\n", - " axes[0].set_xlabel('num_subj_scheduled')\n", - " axes[0].set_ylabel('Startup time (s)')\n", - " axes[0].set_title(f'{label}: startup time vs. num_subj_scheduled')\n", + " axes[0].plot(\n", + " xs, m * xs + b, \"r-\", linewidth=2, label=f\"slope={m:.2f}s/subj, r={corr:.2f}\"\n", + " )\n", + " axes[0].set_xlabel(\"num_subj_scheduled\")\n", + " axes[0].set_ylabel(\"Startup time (s)\")\n", + " axes[0].set_title(f\"{label}: startup time vs. num_subj_scheduled\")\n", " axes[0].legend()\n", " axes[0].grid(True, alpha=0.3)\n", "\n", " axes[1].scatter(\n", - " subj_stats['num_subj_scheduled'], subj_stats['mean'],\n", - " s=subj_stats['count'] * 10, alpha=0.7,\n", + " subj_stats[\"num_subj_scheduled\"],\n", + " subj_stats[\"mean\"],\n", + " s=subj_stats[\"count\"] * 10,\n", + " alpha=0.7,\n", + " )\n", + " axes[1].set_xlabel(\"num_subj_scheduled\")\n", + " axes[1].set_ylabel(\"Mean startup time (s)\")\n", + " axes[1].set_title(\n", + " f\"{label}: mean startup time per num_subj_scheduled\\n(bubble size ∝ count)\"\n", " )\n", - " axes[1].set_xlabel('num_subj_scheduled')\n", - " axes[1].set_ylabel('Mean startup time (s)')\n", - " axes[1].set_title(f'{label}: mean startup time per num_subj_scheduled\\n(bubble size ∝ count)')\n", " axes[1].grid(True, alpha=0.3)\n", "\n", " plt.tight_layout()\n", - " plt.show()\n" + " plt.show()" ] }, { @@ -1095,43 +1167,57 @@ ], "source": [ "# === Startup time as a function of calendar date, per rig, split by startup_type ===\n", - "df_good['date_ordinal'] = df_good['startup_datetime'].map(pd.Timestamp.toordinal)\n", + "df_good[\"date_ordinal\"] = df_good[\"startup_datetime\"].map(pd.Timestamp.toordinal)\n", "\n", - "for label, subset in [('Rig Tester', df_rig_tester), ('Training Flow GUI', df_training_gui)]:\n", + "for label, subset in [\n", + " (\"Rig Tester\", df_rig_tester),\n", + " (\"Training Flow GUI\", df_training_gui),\n", + "]:\n", " if len(subset) == 0:\n", " continue\n", "\n", " subset = subset.copy()\n", - " subset['date_ordinal'] = subset['startup_datetime'].map(pd.Timestamp.toordinal)\n", + " subset[\"date_ordinal\"] = subset[\"startup_datetime\"].map(pd.Timestamp.toordinal)\n", "\n", - " rigs = subset['location'].unique()\n", + " rigs = subset[\"location\"].unique()\n", " n_rigs = len(rigs)\n", " cols = 3\n", " rows = int(np.ceil(n_rigs / cols))\n", "\n", " # --- Per-rig subplots with trend lines ---\n", " fig, axes = plt.subplots(rows, cols, figsize=(cols * 5, rows * 3.5), squeeze=False)\n", - " fig.suptitle(f'Startup time over time — {label} (after 2026-05-15)', fontsize=13, y=1.01)\n", + " fig.suptitle(\n", + " f\"Startup time over time — {label} (after 2026-05-15)\", fontsize=13, y=1.01\n", + " )\n", "\n", " trend_rows = []\n", " for i, rig in enumerate(sorted(rigs)):\n", " ax = axes[i // cols][i % cols]\n", - " grp = subset[subset['location'] == rig].dropna(subset=['startup_time', 'date_ordinal'])\n", + " grp = subset[subset[\"location\"] == rig].dropna(\n", + " subset=[\"startup_time\", \"date_ordinal\"]\n", + " )\n", "\n", - " ax.scatter(grp['startup_datetime'], grp['startup_time'], s=15, alpha=0.6)\n", + " ax.scatter(grp[\"startup_datetime\"], grp[\"startup_time\"], s=15, alpha=0.6)\n", "\n", " if len(grp) >= 2:\n", - " m, b = np.polyfit(grp['date_ordinal'], grp['startup_time'], 1)\n", - " ax.plot(grp['startup_datetime'], m * grp['date_ordinal'] + b, 'r-', linewidth=1.5)\n", - " corr = grp[['date_ordinal', 'startup_time']].corr().iloc[0, 1]\n", - " trend_rows.append({'location': rig, 'slope_s_per_day': m, 'r': corr, 'n': len(grp)})\n", - " ax.set_title(f'{rig}\\nslope={m:.2f}s/day, r={corr:.2f}', fontsize=8)\n", + " m, b = np.polyfit(grp[\"date_ordinal\"], grp[\"startup_time\"], 1)\n", + " ax.plot(\n", + " grp[\"startup_datetime\"],\n", + " m * grp[\"date_ordinal\"] + b,\n", + " \"r-\",\n", + " linewidth=1.5,\n", + " )\n", + " corr = grp[[\"date_ordinal\", \"startup_time\"]].corr().iloc[0, 1]\n", + " trend_rows.append(\n", + " {\"location\": rig, \"slope_s_per_day\": m, \"r\": corr, \"n\": len(grp)}\n", + " )\n", + " ax.set_title(f\"{rig}\\nslope={m:.2f}s/day, r={corr:.2f}\", fontsize=8)\n", " else:\n", - " ax.set_title(f'{rig} (n={len(grp)})', fontsize=8)\n", + " ax.set_title(f\"{rig} (n={len(grp)})\", fontsize=8)\n", "\n", - " ax.set_ylabel('Startup time (s)', fontsize=7)\n", - " ax.tick_params(axis='x', labelrotation=30, labelsize=6)\n", - " ax.tick_params(axis='y', labelsize=7)\n", + " ax.set_ylabel(\"Startup time (s)\", fontsize=7)\n", + " ax.tick_params(axis=\"x\", labelrotation=30, labelsize=6)\n", + " ax.tick_params(axis=\"y\", labelsize=7)\n", " ax.grid(True, alpha=0.3)\n", "\n", " # Hide unused subplots\n", @@ -1145,11 +1231,11 @@ " if trend_rows:\n", " trend_df = (\n", " pd.DataFrame(trend_rows)\n", - " .sort_values('slope_s_per_day', ascending=False)\n", + " .sort_values(\"slope_s_per_day\", ascending=False)\n", " .reset_index(drop=True)\n", " )\n", " print(f\"\\n{label} — trend summary (positive slope = getting slower over time):\")\n", - " print(trend_df.to_string(index=False))\n" + " print(trend_df.to_string(index=False))" ] }, { diff --git a/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb b/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb index c35557cc..6f828883 100644 --- a/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb +++ b/notebooks/ephys_element/check_catgt_parameters_lfp.ipynb @@ -56,8 +56,9 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "recording_process = dj.create_virtual_module('recording_process', 'u19_recording_process')\n" + "recording_process = dj.create_virtual_module(\n", + " \"recording_process\", \"u19_recording_process\"\n", + ")" ] }, { @@ -87,7 +88,7 @@ "outputs": [], "source": [ "key = dict()\n", - "key['job_id'] = 684\n", + "key[\"job_id\"] = 684\n", "recording_process_info = (recording_process.Processing & key).fetch(as_dict=True)" ] }, @@ -135,7 +136,7 @@ "metadata": {}, "outputs": [], "source": [ - "ephys_root = dj_config['custom']['ephys_root_data_dir'][0]" + "ephys_root = dj_config[\"custom\"][\"ephys_root_data_dir\"][0]" ] }, { @@ -155,7 +156,9 @@ } ], "source": [ - "probe_directory = pathlib.Path(ephys_root, recording_process_info[0]['recording_process_pre_path'])\n", + "probe_directory = pathlib.Path(\n", + " ephys_root, recording_process_info[0][\"recording_process_pre_path\"]\n", + ")\n", "probe_directory" ] }, diff --git a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb index c8af2023..0d152516 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_bulk.ipynb @@ -49,7 +49,7 @@ "\n", "import u19_pipeline.ephys_pipeline as ephys_pipeline\n", "import u19_pipeline.utils.ephys_fix_sync_code as ephys_fix_sync_code\n", - "import u19_pipeline.utils.ephys_utils as ephys_utils\n" + "import u19_pipeline.utils.ephys_utils as ephys_utils" ] }, { @@ -70,12 +70,16 @@ ], "source": [ "recording_query = \"recording_id > 501 and recording_id < 503\"\n", - "recording = dj.create_virtual_module('recording', 'u19_recording')\n", - "recording_keys = (recording.Recording & recording_query).fetch('recording_id', as_dict=True, order_by='recording_id')\n", + "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n", + "recording_keys = (recording.Recording & recording_query).fetch(\n", + " \"recording_id\", as_dict=True, order_by=\"recording_id\"\n", + ")\n", "\n", "\n", - "session_fields = ['subject_fullname', 'session_date', 'session_number']\n", - "session_keys = (recording.Recording.BehaviorSession & recording_query).fetch(*session_fields, as_dict=True, order_by='recording_id')\n", + "session_fields = [\"subject_fullname\", \"session_date\", \"session_number\"]\n", + "session_keys = (recording.Recording.BehaviorSession & recording_query).fetch(\n", + " *session_fields, as_dict=True, order_by=\"recording_id\"\n", + ")\n", "\n", "\n", "recording_keys" @@ -87,100 +91,128 @@ "metadata": {}, "outputs": [], "source": [ - "def main_ephys_fix_sync_code(iter_start_idx, iter_times_idx, behavior_time, nidq_sampling_rate, key):\n", + "def main_ephys_fix_sync_code(\n", + " iter_start_idx, iter_times_idx, behavior_time, nidq_sampling_rate, key\n", + "):\n", " iteration_dict = dict()\n", - " iteration_dict['iter_start_idx'] = list()\n", - " iteration_dict['iter_times_idx'] = list()\n", - " iteration_dict['trial_sync_stats'] = list()\n", + " iteration_dict[\"iter_start_idx\"] = list()\n", + " iteration_dict[\"iter_times_idx\"] = list()\n", + " iteration_dict[\"trial_sync_stats\"] = list()\n", "\n", " for i in range(len(iter_start_idx)):\n", + " trial_stats_dict = {}\n", + " trial_stats_dict[\"num_trial\"] = i + 1\n", "\n", - " trial_stats_dict= {}\n", - " trial_stats_dict['num_trial'] = i+1\n", - "\n", - " #print('fixing trial ',i)\n", + " # print('fixing trial ',i)\n", " behavior_time_vector = behavior_time[i].flatten()\n", "\n", - " #time_iteration = np.median(np.diff(behavior_time[0:5])\n", + " # time_iteration = np.median(np.diff(behavior_time[0:5])\n", "\n", - " iter_times_idx[i] = iter_times_idx[i]+behavior_time_vector[1]\n", - " #iter_times_idx[i] = iter_times_idx[i]+behavior_time_vector[1]-(behavior_time_vector[2]-iter_times_idx[i][2])\n", - " #iter_times_idx[i] = iter_times_idx[i]+behavior_time_vector[1]\n", - " #synced_time_vector = np.insert(synced_time_vector, 1, behavior_time_vector[1])\n", - " #synced_time_vector[2:] = synced_time_vector[2:]+(behavior_time_vector[2]-synced_time_vector[2])\n", - " #synced_time_vector[0] = 0\n", + " iter_times_idx[i] = iter_times_idx[i] + behavior_time_vector[1]\n", + " # iter_times_idx[i] = iter_times_idx[i]+behavior_time_vector[1]-(behavior_time_vector[2]-iter_times_idx[i][2])\n", + " # iter_times_idx[i] = iter_times_idx[i]+behavior_time_vector[1]\n", + " # synced_time_vector = np.insert(synced_time_vector, 1, behavior_time_vector[1])\n", + " # synced_time_vector[2:] = synced_time_vector[2:]+(behavior_time_vector[2]-synced_time_vector[2])\n", + " # synced_time_vector[0] = 0\n", "\n", - " synced_time_vector, shift_vec, median_vec = ephys_fix_sync_code.get_shift_vector(iter_times_idx[i],behavior_time_vector)\n", + " synced_time_vector, shift_vec, median_vec = (\n", + " ephys_fix_sync_code.get_shift_vector(\n", + " iter_times_idx[i], behavior_time_vector\n", + " )\n", + " )\n", "\n", - " #if median_vec[1] < 0:\n", + " # if median_vec[1] < 0:\n", " # print('Trial # median more', i)\n", " # raise('Median less than 0')\n", - " \n", + "\n", " # break\n", "\n", - " trial_stats_dict['max_shift'] = np.max(shift_vec)\n", - " trial_stats_dict['min_shift'] = np.min(shift_vec)\n", - " trial_stats_dict['median_diff'] = median_vec[1]\n", - " trial_stats_dict['min_diff'] = median_vec[0]\n", - " trial_stats_dict['max_diff'] = median_vec[2]\n", + " trial_stats_dict[\"max_shift\"] = np.max(shift_vec)\n", + " trial_stats_dict[\"min_shift\"] = np.min(shift_vec)\n", + " trial_stats_dict[\"median_diff\"] = median_vec[1]\n", + " trial_stats_dict[\"min_diff\"] = median_vec[0]\n", + " trial_stats_dict[\"max_diff\"] = median_vec[2]\n", "\n", - " synced_time_vector,trial_stats_dict['borrow_step2'] =\\\n", - " ephys_fix_sync_code.fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, shift_vec)\n", + " synced_time_vector, trial_stats_dict[\"borrow_step2\"] = (\n", + " ephys_fix_sync_code.fix_shifted_sync_vector(\n", + " synced_time_vector, behavior_time_vector, shift_vec\n", + " )\n", + " )\n", "\n", - " #synced_time_vector, trial_stats_dict['borrow_step3'] =\\\n", + " # synced_time_vector, trial_stats_dict['borrow_step3'] =\\\n", " # fix_sync_vector_greater(synced_time_vector, behavior_time_vector)\n", - " synced_time_vector,trial_stats_dict['borrow_step4'] =\\\n", - " ephys_fix_sync_code.complete_last_part_sync_vec(synced_time_vector, behavior_time_vector)\n", - "\n", - " synced_iteration_vector =\\\n", - " ephys_fix_sync_code.fix_iter_vector(iter_start_idx[i],synced_time_vector, iter_times_idx[i], nidq_sampling_rate)\n", - " \n", - " trial_stats_dict['num_iterations'] = synced_iteration_vector.shape[0] \n", - " \n", - " trial_stats_dict = trial_stats_dict | key\n", - " iteration_dict['trial_sync_stats'].append(trial_stats_dict)\n", + " synced_time_vector, trial_stats_dict[\"borrow_step4\"] = (\n", + " ephys_fix_sync_code.complete_last_part_sync_vec(\n", + " synced_time_vector, behavior_time_vector\n", + " )\n", + " )\n", + "\n", + " synced_iteration_vector = ephys_fix_sync_code.fix_iter_vector(\n", + " iter_start_idx[i], synced_time_vector, iter_times_idx[i], nidq_sampling_rate\n", + " )\n", + "\n", + " trial_stats_dict[\"num_iterations\"] = synced_iteration_vector.shape[0]\n", "\n", + " trial_stats_dict = trial_stats_dict | key\n", + " iteration_dict[\"trial_sync_stats\"].append(trial_stats_dict)\n", "\n", - " iteration_dict['iter_start_idx'].append(synced_iteration_vector.copy())\n", - " iteration_dict['iter_times_idx'].append(synced_time_vector.copy())\n", + " iteration_dict[\"iter_start_idx\"].append(synced_iteration_vector.copy())\n", + " iteration_dict[\"iter_times_idx\"].append(synced_time_vector.copy())\n", "\n", - " print('end fix sync code 1')\n", + " print(\"end fix sync code 1\")\n", "\n", - " iteration_dict['iter_start_idx'] = np.asarray(iteration_dict['iter_start_idx'].copy(), dtype=object)\n", - " iteration_dict['iter_times_idx'] = np.asarray(iteration_dict['iter_times_idx'].copy(), dtype=object)\n", + " iteration_dict[\"iter_start_idx\"] = np.asarray(\n", + " iteration_dict[\"iter_start_idx\"].copy(), dtype=object\n", + " )\n", + " iteration_dict[\"iter_times_idx\"] = np.asarray(\n", + " iteration_dict[\"iter_times_idx\"].copy(), dtype=object\n", + " )\n", "\n", - " print('end fix sync code')\n", + " print(\"end fix sync code\")\n", "\n", " # Check # of trials and iterations match\n", - " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n", + " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n", + " ephys_utils.assert_iteration_samples_count(\n", + " iteration_dict[\"iter_start_idx\"], behavior_time\n", + " )\n", + " )\n", "\n", - " print('after assert_iteration_samples_count fix sync code')\n", + " print(\"after assert_iteration_samples_count fix sync code\")\n", "\n", " if trial_count_diff != 0:\n", - " print('trial_count_diff', trial_count_diff)\n", + " print(\"trial_count_diff\", trial_count_diff)\n", " if len(trials_diff_iteration_big) > 0:\n", - " print('trials_diff_iteration_big', trials_diff_iteration_big)\n", + " print(\"trials_diff_iteration_big\", trials_diff_iteration_big)\n", " if len(trials_diff_iteration_small) > 0:\n", - " print('trials_diff_iteration_small', trials_diff_iteration_small)\n", - "\n", + " print(\"trials_diff_iteration_small\", trials_diff_iteration_small)\n", "\n", - " status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n", + " status = ephys_utils.evaluate_sync_process(\n", + " trial_count_diff,\n", + " trials_diff_iteration_big,\n", + " trials_diff_iteration_small,\n", + " behavior_time.shape[0],\n", + " )\n", "\n", - " print('after evaluate_sync_process fix sync code')\n", + " print(\"after evaluate_sync_process fix sync code\")\n", "\n", - " for i in range(len(iteration_dict['iter_start_idx'])):\n", - " synced_time_vector = iteration_dict['iter_times_idx'][i]\n", + " for i in range(len(iteration_dict[\"iter_start_idx\"])):\n", + " synced_time_vector = iteration_dict[\"iter_times_idx\"][i]\n", " behavior_time_vector = behavior_time[i].flatten()\n", "\n", - " status = ephys_fix_sync_code.sync_evaluation_process2(synced_time_vector, behavior_time_vector)\n", + " status = ephys_fix_sync_code.sync_evaluation_process2(\n", + " synced_time_vector, behavior_time_vector\n", + " )\n", " if status == -1:\n", " break\n", "\n", - " print('after sync_evaluation_process2', status)\n", + " print(\"after sync_evaluation_process2\", status)\n", "\n", " if status == 1:\n", - " iteration_dict['trial_start_idx'] = ephys_utils.get_index_trial_vector_from_iteration(iteration_dict['iter_start_idx'])\n", - "\n", + " iteration_dict[\"trial_start_idx\"] = (\n", + " ephys_utils.get_index_trial_vector_from_iteration(\n", + " iteration_dict[\"iter_start_idx\"]\n", + " )\n", + " )\n", "\n", " return status, iteration_dict" ] @@ -197,69 +229,108 @@ "\n", " # Get behavior key\n", " behavior_key = (recording.Recording.BehaviorSession & key).fetch1()\n", - " behavior_key.pop('recording_id')\n", + " behavior_key.pop(\"recording_id\")\n", "\n", - " if 'testuser' in behavior_key['subject_fullname']:\n", + " if \"testuser\" in behavior_key[\"subject_fullname\"]:\n", " return\n", "\n", " # If a specific block is requested, add that to our behavior_key. It should be an int referring to virmen block number.\n", " # This is useful for sessions in which the nidaq stream was interrupted due to restarting virmen\n", - " if 'block' in kwargs:\n", - " print('block: ', kwargs['block'])\n", - " behavior_key['block'] = kwargs['block']\n", + " if \"block\" in kwargs:\n", + " print(\"block: \", kwargs[\"block\"])\n", + " behavior_key[\"block\"] = kwargs[\"block\"]\n", "\n", " print(behavior_key)\n", "\n", " # And get the datajoint record\n", - " behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n", + " behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", " thissession = behavior.TowersBlock().Trial() & behavior_key\n", - " behavior_time, iterstart = thissession.fetch('trial_time', 'vi_start')\n", + " behavior_time, iterstart = thissession.fetch(\"trial_time\", \"vi_start\")\n", "\n", - " print('len iterstart', len(iterstart))\n", + " print(\"len iterstart\", len(iterstart))\n", "\n", " if len(iterstart) == 0:\n", - " raise ValueError('No behavior found')\n", + " raise ValueError(\"No behavior found\")\n", "\n", - " print('after reading behavior data')\n", + " print(\"after reading behavior data\")\n", "\n", " # 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n", - " nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate(ephys_session_fullpath)\n", + " nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate(\n", + " ephys_session_fullpath\n", + " )\n", "\n", - " trial_pulse_signal, iteration_pulse_signal = ephys_utils.load_trial_iteration_signals(ephys_session_fullpath, nidq_meta)\n", + " trial_pulse_signal, iteration_pulse_signal = (\n", + " ephys_utils.load_trial_iteration_signals(ephys_session_fullpath, nidq_meta)\n", + " )\n", "\n", - " print('after reading spikeglx data')\n", + " print(\"after reading spikeglx data\")\n", "\n", " # Synchronize between pulses and get iteration # vector for each sample\n", - " recent_recording = behavior_key['session_date'] > datetime.date(2021,6,1) # Everything past June 1 2021\n", + " recent_recording = behavior_key[\"session_date\"] > datetime.date(\n", + " 2021, 6, 1\n", + " ) # Everything past June 1 2021\n", " if recent_recording:\n", " # New synchronization method: digital_array[1,2] contain pulses for trial and frame number.\n", - " mode=None\n", - " iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal, nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode)\n", + " mode = None\n", + " iteration_dict = (\n", + " ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n", + " trial_pulse_signal,\n", + " iteration_pulse_signal,\n", + " nidq_sampling_rate,\n", + " behavior_time.shape[0],\n", + " behavior_time,\n", + " mode,\n", + " )\n", + " )\n", " else:\n", " # Old synchronization: digital_array[0:7] contain a digital word that counts the virmen frames.\n", - " raise ValueError('Old sessions < 2022 not suported anymore')\n", - " #iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_word(digital_array, behavior_time, iterstart)\n", + " raise ValueError(\"Old sessions < 2022 not suported anymore\")\n", + " # iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_word(digital_array, behavior_time, iterstart)\n", "\n", " # Check # of trials (from database record of behavior in `behavior_time`) and iterations (extracted from NIDAQ in `iter_start_idx`) match\n", - " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n", - "\n", - " print('metrics to evaluate...')\n", - " print(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n", - "\n", - " status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n", + " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n", + " ephys_utils.assert_iteration_samples_count(\n", + " iteration_dict[\"iter_start_idx\"], behavior_time\n", + " )\n", + " )\n", + "\n", + " print(\"metrics to evaluate...\")\n", + " print(\n", + " trial_count_diff,\n", + " trials_diff_iteration_big,\n", + " trials_diff_iteration_small,\n", + " behavior_time.shape[0],\n", + " )\n", + "\n", + " status = ephys_utils.evaluate_sync_process(\n", + " trial_count_diff,\n", + " trials_diff_iteration_big,\n", + " trials_diff_iteration_small,\n", + " behavior_time.shape[0],\n", + " )\n", "\n", " if status == 1:\n", - " iteration_dict['trial_start_idx'] = ephys_utils.get_index_trial_vector_from_iteration(iteration_dict['iter_start_idx'])\n", + " iteration_dict[\"trial_start_idx\"] = (\n", + " ephys_utils.get_index_trial_vector_from_iteration(\n", + " iteration_dict[\"iter_start_idx\"]\n", + " )\n", + " )\n", "\n", - " #Failed sync by a lot, error\n", + " # Failed sync by a lot, error\n", " status_regular = 1\n", " status_fix = 0\n", " if status < 1:\n", " status_regular = 0\n", - " print('Regular ephys sync failed')\n", - " \n", + " print(\"Regular ephys sync failed\")\n", + "\n", " try:\n", - " status_fix, iteration_dict2 = main_ephys_fix_sync_code(iteration_dict['iter_start_idx'], iteration_dict['iter_times_idx'], behavior_time, nidq_sampling_rate, key)\n", + " status_fix, iteration_dict2 = main_ephys_fix_sync_code(\n", + " iteration_dict[\"iter_start_idx\"],\n", + " iteration_dict[\"iter_times_idx\"],\n", + " behavior_time,\n", + " nidq_sampling_rate,\n", + " key,\n", + " )\n", " except:\n", " status_fix = 0\n", " iteration_dict2 = {}\n", @@ -368,10 +439,8 @@ } ], "source": [ - "\n", "num_session = 0\n", "for idx_session in range(len(recording_keys)):\n", - "\n", " recording_key = recording_keys[idx_session]\n", " session_key = session_keys[idx_session]\n", "\n", @@ -379,22 +448,25 @@ " if status_fix == 0:\n", " continue\n", "\n", - " print('session_key', session_key)\n", + " print(\"session_key\", session_key)\n", "\n", - " for i in range(len(iteration_dict2['trial_sync_stats'])):\n", - " iteration_dict2['trial_sync_stats'][i] = iteration_dict2['trial_sync_stats'][i] | session_key\n", + " for i in range(len(iteration_dict2[\"trial_sync_stats\"])):\n", + " iteration_dict2[\"trial_sync_stats\"][i] = (\n", + " iteration_dict2[\"trial_sync_stats\"][i] | session_key\n", + " )\n", + "\n", + " print(\"status_fix\", status_fix)\n", + " print(\"num_session\", num_session)\n", "\n", - " print('status_fix', status_fix)\n", - " print('num_session', num_session)\n", - " \n", " if num_session == 0:\n", - " trial_sync_stats_df = pd.DataFrame(iteration_dict2['trial_sync_stats'])\n", + " trial_sync_stats_df = pd.DataFrame(iteration_dict2[\"trial_sync_stats\"])\n", " else:\n", - " trial_sync_stats_df = pd.concat([trial_sync_stats_df, pd.DataFrame(iteration_dict2['trial_sync_stats'])], axis=0)\n", - "\n", - " num_session += 1\n", + " trial_sync_stats_df = pd.concat(\n", + " [trial_sync_stats_df, pd.DataFrame(iteration_dict2[\"trial_sync_stats\"])],\n", + " axis=0,\n", + " )\n", "\n", - " \n" + " num_session += 1" ] }, { @@ -406,9 +478,9 @@ "def count_borrowed_iter(borrow_list):\n", " num_borrow = 0\n", " for i in range(len(borrow_list)):\n", - " num_borrow = num_borrow+ borrow_list[i][1] - borrow_list[i][0] + 1\n", + " num_borrow = num_borrow + borrow_list[i][1] - borrow_list[i][0] + 1\n", "\n", - " return num_borrow\n" + " return num_borrow" ] }, { @@ -781,22 +853,38 @@ } ], "source": [ + "trial_sync_stats_df[\"min_shift_abs\"] = trial_sync_stats_df[\"min_shift\"].abs()\n", + "trial_sync_stats_df[\"shift\"] = trial_sync_stats_df[[\"max_shift\", \"min_shift_abs\"]].max(\n", + " axis=1\n", + ")\n", "\n", - "trial_sync_stats_df['min_shift_abs'] = trial_sync_stats_df['min_shift'].abs()\n", - "trial_sync_stats_df['shift'] = trial_sync_stats_df[['max_shift', 'min_shift_abs']].max(axis=1)\n", + "trial_sync_stats_df.loc[\n", + " trial_sync_stats_df[\"shift\"] == trial_sync_stats_df[\"min_shift_abs\"], \"shift\"\n", + "] = (\n", + " trial_sync_stats_df.loc[\n", + " trial_sync_stats_df[\"shift\"] == trial_sync_stats_df[\"min_shift_abs\"], \"shift\"\n", + " ]\n", + " * -1\n", + ")\n", "\n", - "trial_sync_stats_df.loc[trial_sync_stats_df['shift']==trial_sync_stats_df['min_shift_abs'], 'shift'] = trial_sync_stats_df.loc[trial_sync_stats_df['shift']==trial_sync_stats_df['min_shift_abs'], 'shift']*-1\n", + "trial_sync_stats_df[\"num_borrow_step2\"] = trial_sync_stats_df[\"borrow_step2\"].apply(\n", + " count_borrowed_iter\n", + ")\n", + "# trial_sync_stats_df['num_borrow_step3'] = trial_sync_stats_df['borrow_step3'].apply(count_borrowed_iter)\n", + "trial_sync_stats_df[\"num_borrow_step4\"] = trial_sync_stats_df[\"borrow_step4\"].apply(\n", + " count_borrowed_iter\n", + ")\n", "\n", - "trial_sync_stats_df['num_borrow_step2'] = trial_sync_stats_df['borrow_step2'].apply(count_borrowed_iter)\n", - "#trial_sync_stats_df['num_borrow_step3'] = trial_sync_stats_df['borrow_step3'].apply(count_borrowed_iter)\n", - "trial_sync_stats_df['num_borrow_step4'] = trial_sync_stats_df['borrow_step4'].apply(count_borrowed_iter)\n", "\n", + "# trial_sync_stats_df['total_borrow'] = trial_sync_stats_df['num_borrow_step2'] + trial_sync_stats_df['num_borrow_step3'] + trial_sync_stats_df['num_borrow_step4']\n", "\n", - "#trial_sync_stats_df['total_borrow'] = trial_sync_stats_df['num_borrow_step2'] + trial_sync_stats_df['num_borrow_step3'] + trial_sync_stats_df['num_borrow_step4'] \n", + "trial_sync_stats_df[\"total_borrow\"] = (\n", + " trial_sync_stats_df[\"num_borrow_step4\"] + trial_sync_stats_df[\"num_borrow_step2\"]\n", + ")\n", "\n", - "trial_sync_stats_df['total_borrow'] = trial_sync_stats_df['num_borrow_step4'] + trial_sync_stats_df['num_borrow_step2']\n", - "\n", - "trial_sync_stats_df['percentage_borrow'] = trial_sync_stats_df['total_borrow']*100 / trial_sync_stats_df['num_iterations']\n", + "trial_sync_stats_df[\"percentage_borrow\"] = (\n", + " trial_sync_stats_df[\"total_borrow\"] * 100 / trial_sync_stats_df[\"num_iterations\"]\n", + ")\n", "\n", "\n", "trial_sync_stats_df" @@ -1159,7 +1247,7 @@ } ], "source": [ - "trial_sync_stats_df = trial_sync_stats_df.sort_values(by='median_diff', ascending=False)\n", + "trial_sync_stats_df = trial_sync_stats_df.sort_values(by=\"median_diff\", ascending=False)\n", "trial_sync_stats_df" ] }, @@ -1227,7 +1315,7 @@ } ], "source": [ - "trial_sync_stats_df.loc[trial_sync_stats_df['median_diff'] < -0.0001, :]" + "trial_sync_stats_df.loc[trial_sync_stats_df[\"median_diff\"] < -0.0001, :]" ] }, { @@ -1247,7 +1335,7 @@ } ], "source": [ - "trial_sync_stats_df['min_shift_abs'].max()" + "trial_sync_stats_df[\"min_shift_abs\"].max()" ] }, { @@ -1279,18 +1367,18 @@ "source": [ "plt.figure(figsize=(8, 6))\n", "\n", - "shift_stats = trial_sync_stats_df['median_diff'].copy()\n", + "shift_stats = trial_sync_stats_df[\"median_diff\"].copy()\n", "num_0_shifts = (shift_stats == 0).sum()\n", "\n", "num_shifts = shift_stats[shift_stats != 0]\n", "\n", "\n", - "plt.hist(num_shifts,bins=61)\n", + "plt.hist(num_shifts, bins=61)\n", "\n", "\n", - "plt.xlabel('Time(s)')\n", - "plt.ylabel('# Trials')\n", - "plt.title('Median diff NIDAQ - ViRMEn')" + "plt.xlabel(\"Time(s)\")\n", + "plt.ylabel(\"# Trials\")\n", + "plt.title(\"Median diff NIDAQ - ViRMEn\")" ] }, { @@ -1299,11 +1387,19 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "trial_sync_stats_df['min_shift_abs'] = trial_sync_stats_df['min_shift'].abs()\n", - "trial_sync_stats_df['shift'] = trial_sync_stats_df[['max_shift', 'min_shift_abs']].max(axis=1)\n", - "\n", - "trial_sync_stats_df.loc[trial_sync_stats_df['shift']==trial_sync_stats_df['min_shift_abs'], 'shift'] = trial_sync_stats_df.loc[trial_sync_stats_df['shift']==trial_sync_stats_df['min_shift_abs'], 'shift']*-1\n" + "trial_sync_stats_df[\"min_shift_abs\"] = trial_sync_stats_df[\"min_shift\"].abs()\n", + "trial_sync_stats_df[\"shift\"] = trial_sync_stats_df[[\"max_shift\", \"min_shift_abs\"]].max(\n", + " axis=1\n", + ")\n", + "\n", + "trial_sync_stats_df.loc[\n", + " trial_sync_stats_df[\"shift\"] == trial_sync_stats_df[\"min_shift_abs\"], \"shift\"\n", + "] = (\n", + " trial_sync_stats_df.loc[\n", + " trial_sync_stats_df[\"shift\"] == trial_sync_stats_df[\"min_shift_abs\"], \"shift\"\n", + " ]\n", + " * -1\n", + ")" ] }, { @@ -1335,7 +1431,7 @@ "source": [ "plt.figure(figsize=(8, 6))\n", "\n", - "shift_stats = trial_sync_stats_df['shift'].copy()\n", + "shift_stats = trial_sync_stats_df[\"shift\"].copy()\n", "num_0_shifts = (shift_stats == 0).sum()\n", "\n", "num_shifts = shift_stats[shift_stats != 0]\n", @@ -1343,10 +1439,14 @@ "\n", "plt.hist(num_shifts)\n", "\n", - "plt.yscale('log')\n", - "plt.xlabel('# Shift iterations')\n", - "plt.ylabel('# Trials')\n", - "plt.title('\"Max\" shift in trial to preserve median: '+ str(num_0_shifts)+ ' trials without shift')" + "plt.yscale(\"log\")\n", + "plt.xlabel(\"# Shift iterations\")\n", + "plt.ylabel(\"# Trials\")\n", + "plt.title(\n", + " '\"Max\" shift in trial to preserve median: '\n", + " + str(num_0_shifts)\n", + " + \" trials without shift\"\n", + ")" ] }, { @@ -1379,13 +1479,12 @@ "plt.figure(figsize=(8, 6))\n", "\n", "\n", + "plt.hist(trial_sync_stats_df[\"percentage_borrow\"])\n", "\n", - "plt.hist(trial_sync_stats_df['percentage_borrow'])\n", - "\n", - "plt.yscale('log')\n", - "plt.xlabel('% Trial')\n", - "plt.ylabel('# Trials')\n", - "plt.title('Trial NIDAQ iterations borrowed from ViRMEn')" + "plt.yscale(\"log\")\n", + "plt.xlabel(\"% Trial\")\n", + "plt.ylabel(\"# Trials\")\n", + "plt.title(\"Trial NIDAQ iterations borrowed from ViRMEn\")" ] }, { @@ -1405,7 +1504,7 @@ } ], "source": [ - "np.max(trial_sync_stats_df['percentage_borrow'])" + "np.max(trial_sync_stats_df[\"percentage_borrow\"])" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb index 0ccda4c7..91436d0f 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_session.ipynb @@ -55,7 +55,7 @@ "from u19_pipeline.ephys_pipeline import (\n", " get_ephys_root_data_dir,\n", " get_session_directory,\n", - ")\n" + ")" ] }, { @@ -75,25 +75,26 @@ } ], "source": [ - "key = {'subject_fullname': 'jyanar_ya054',\n", - " 'session_date': datetime.date(2025, 10, 10)}\n", - "#key = {\n", + "key = {\"subject_fullname\": \"jyanar_ya054\", \"session_date\": datetime.date(2025, 10, 10)}\n", + "# key = {\n", "# 'subject_fullname': 'jk8386_jk83',\n", "# 'session_date': datetime.date(2025, 7, 31),\n", "# 'session_number': 0 }\n", "\n", - "#key2 = 'subject_fullname = \"jjulian_jj046\" and session_date = \"2022-04-19\" and block >=2'\n", - "#key = {\n", + "# key2 = 'subject_fullname = \"jjulian_jj046\" and session_date = \"2022-04-19\" and block >=2'\n", + "# key = {\n", "# 'subject_fullname': 'jjulian_jj042',\n", "# 'session_date': datetime.date(2021, 10, 31)}\n", "\n", "\n", - "key = (acquisition.Session & key).fetch1('KEY')\n", + "key = (acquisition.Session & key).fetch1(\"KEY\")\n", "key\n", "\n", - "recording = dj.create_virtual_module('recording', 'u19_recording')\n", + "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n", "\n", - "recording_key = ((acquisition.Session * recording.Recording.BehaviorSession) & key).fetch('recording_id', as_dict=True)\n", + "recording_key = (\n", + " (acquisition.Session * recording.Recording.BehaviorSession) & key\n", + ").fetch(\"recording_id\", as_dict=True)\n", "\n", "recording_key" ] @@ -116,7 +117,7 @@ ], "source": [ "session_dir = pathlib.Path(get_session_directory(recording_key[0]))\n", - "session_dir\n" + "session_dir" ] }, { @@ -134,38 +135,43 @@ } ], "source": [ - "session_dir = find_full_path(get_ephys_root_data_dir(),\n", - " get_session_directory(recording_key))\n", + "session_dir = find_full_path(\n", + " get_ephys_root_data_dir(), get_session_directory(recording_key)\n", + ")\n", "print(session_dir)\n", - "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", - "#Check if session is Nidq or OneBox\n", - "nidq_session = list(session_dir.glob('*nidq.bin*'))\n", - "obx_session = list(session_dir.glob('*obx.bin*'))\n", + "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", + "# Check if session is Nidq or OneBox\n", + "nidq_session = list(session_dir.glob(\"*nidq.bin*\"))\n", + "obx_session = list(session_dir.glob(\"*obx.bin*\"))\n", "\n", "if len(nidq_session) == 0 and len(obx_session) == 0:\n", - " print('No session found')\n", + " print(\"No session found\")\n", "elif len(nidq_session) > 0:\n", " ephys_session_fullpath = nidq_session[0]\n", "else:\n", " ephys_session_fullpath = obx_session[0]\n", "\n", - "#Nidaq file\n", - "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n", + "# Nidaq file\n", + "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n", "nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n", "\n", "\n", - "# 1: load meta data, and the content of the NIDAQ file. Its content is digital. \n", + "# 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n", "new_trial_channel = 1\n", "new_iteration_channel = 2\n", "# If PXIe card (nidq) card use for recording deduce digital channels\n", - "if nidq_meta['typeThis'] == 'nidq':\n", - " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)\n", + "if nidq_meta[\"typeThis\"] == \"nidq\":\n", + " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " ephys_session_fullpath, nidq_meta\n", + " )\n", "# If onebox card (obx) card use for recording digital channels are 0-2\n", "else:\n", - " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])\n", + " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]\n", + " )\n", " # If no sync pulse found trial and iteration signals are 0 & 1 respectively\n", - " channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]\n", - " channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]\n", + " channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]\n", + " channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]\n", "\n", " if channel0_pulses > channel1_pulses:\n", " new_trial_channel = 1\n", @@ -202,18 +208,20 @@ "metadata": {}, "outputs": [], "source": [ - "#Behavior data\n", + "# Behavior data\n", "behavior_key = key.copy()\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", "\n", "block = 0\n", "# If a specific block is requested, add that to our behavior_key. It should be an int referring to virmen block number.\n", "# This is useful for sessions in which the nidaq stream was interrupted due to restarting virmen\n", "if block > 0:\n", - " behavior_key['block'] = block\n", + " behavior_key[\"block\"] = block\n", "\n", "thissession = behavior.TowersBlock().Trial() & behavior_key\n", - "behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n" + "behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n", + " \"trial_time\", \"vi_start\", \"iterations\"\n", + ")" ] }, { @@ -290,21 +298,37 @@ } ], "source": [ - "mode = None #Default for sessions before 12/01/2021\n", - "#mode = 'pulses' #Default for sessions after 12/01/2021\n", - "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[new_trial_channel,:], digital_array[new_iteration_channel,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n", + "mode = None # Default for sessions before 12/01/2021\n", + "# mode = 'pulses' #Default for sessions after 12/01/2021\n", + "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n", + " digital_array[new_trial_channel, :],\n", + " digital_array[new_iteration_channel, :],\n", + " nidq_sampling_rate,\n", + " behavior_time.shape[0],\n", + " behavior_time,\n", + " mode=mode,\n", + ")\n", "# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n", "\n", "\n", "# Check # of trials and iterations match\n", - "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n", + "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n", + " ephys_utils.assert_iteration_samples_count(\n", + " iteration_dict[\"iter_start_idx\"], behavior_time\n", + " )\n", + ")\n", "\n", "print(trial_count_diff)\n", "print(trials_diff_iteration_big)\n", "print(trials_diff_iteration_small)\n", "\n", "\n", - "status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n" + "status = ephys_utils.evaluate_sync_process(\n", + " trial_count_diff,\n", + " trials_diff_iteration_big,\n", + " trials_diff_iteration_small,\n", + " behavior_time.shape[0],\n", + ")" ] }, { @@ -313,8 +337,11 @@ "metadata": {}, "outputs": [], "source": [ - "for i in range(len(iteration_dict['iter_start_idx'])):\n", - " if iteration_dict['iter_start_idx'][i].shape[0] > behavior_time[i].flatten().shape[0]:\n", + "for i in range(len(iteration_dict[\"iter_start_idx\"])):\n", + " if (\n", + " iteration_dict[\"iter_start_idx\"][i].shape[0]\n", + " > behavior_time[i].flatten().shape[0]\n", + " ):\n", " print(i)" ] }, @@ -334,8 +361,8 @@ ], "source": [ "trial_fix = 287\n", - "synced_iteration_vector = iteration_dict['iter_start_idx'][trial_fix]\n", - "synced_time_vector = iteration_dict['iter_times_idx'][trial_fix]\n", + "synced_iteration_vector = iteration_dict[\"iter_start_idx\"][trial_fix]\n", + "synced_time_vector = iteration_dict[\"iter_times_idx\"][trial_fix]\n", "behavior_time_vector = behavior_time[trial_fix].flatten()\n", "\n", "print(synced_iteration_vector.shape[0])\n", @@ -359,8 +386,7 @@ } ], "source": [ - "\n", - "'''\n", + "\"\"\"\n", "#synced_time_vector = synced_time_vector+behavior_time_vector[1]-(behavior_time_vector[2]-synced_time_vector[2])\n", "synced_time_vector = synced_time_vector+behavior_time_vector[1]\n", "synced_time_vector = np.insert(synced_time_vector, 1, behavior_time_vector[1])\n", @@ -373,7 +399,7 @@ "\n", "print(synced_time_vector[0:5])\n", "print(behavior_time_vector[0:5])\n", - "'''" + "\"\"\"" ] }, { @@ -406,51 +432,58 @@ "initial_sample = 0\n", "base_size = 40\n", "\n", - "baseline_diff = synced_time_vector[initial_sample:base_size] - behavior_time_vector[initial_sample:base_size] \n", + "baseline_diff = (\n", + " synced_time_vector[initial_sample:base_size]\n", + " - behavior_time_vector[initial_sample:base_size]\n", + ")\n", "\n", "count_iter_first = 0\n", - "while(1):\n", - "\n", - " count_iter_first = count_iter_first+1\n", - "\n", + "while 1:\n", + " count_iter_first = count_iter_first + 1\n", "\n", - " base_greater = np.where(baseline_diff >0)\n", + " base_greater = np.where(baseline_diff > 0)\n", " base_greater = base_greater[0]\n", "\n", - " if base_greater.shape[0] > 0 and synced_time_vector.shape[0] < behavior_time_vector.shape[0]:\n", + " if (\n", + " base_greater.shape[0] > 0\n", + " and synced_time_vector.shape[0] < behavior_time_vector.shape[0]\n", + " ):\n", " print(count_iter_first)\n", " idx_first = base_greater[0]\n", - " time_bef = behavior_time_vector[idx_first] - behavior_time_vector[idx_first-1]\n", + " time_bef = behavior_time_vector[idx_first] - behavior_time_vector[idx_first - 1]\n", "\n", + " print(\"idx_first\", idx_first)\n", + " print(\"time_bef\", time_bef)\n", + " print(\"synced_time_vector\", synced_time_vector[idx_first - 2 : idx_first + 2])\n", + " print(\n", + " \"behavior_time_vector\", behavior_time_vector[idx_first - 2 : idx_first + 2]\n", + " )\n", "\n", - " print('idx_first', idx_first)\n", - " print('time_bef', time_bef)\n", - " print('synced_time_vector', synced_time_vector[idx_first-2:idx_first+2])\n", - " print('behavior_time_vector', behavior_time_vector[idx_first-2:idx_first+2])\n", + " synced_time_vector = np.insert(\n", + " synced_time_vector, idx_first, synced_time_vector[idx_first - 1] + time_bef\n", + " )\n", "\n", - " synced_time_vector = np.insert(synced_time_vector, idx_first, synced_time_vector[idx_first-1]+time_bef)\n", + " print(\"synced_time_vector\", synced_time_vector[idx_first - 2 : idx_first + 2])\n", "\n", - " print('synced_time_vector', synced_time_vector[idx_first-2:idx_first+2])\n", - "\n", - " baseline_diff = synced_time_vector[initial_sample:base_size] - behavior_time_vector[initial_sample:base_size]\n", + " baseline_diff = (\n", + " synced_time_vector[initial_sample:base_size]\n", + " - behavior_time_vector[initial_sample:base_size]\n", + " )\n", " else:\n", - " break \n", - "\n", - "\n", - "\n", + " break\n", "\n", "\n", "median_diff = np.median(baseline_diff)\n", - "max_diff = np.median(baseline_diff)+0.007\n", - "min_diff = np.median(baseline_diff)-0.007\n", - "time_vec = np.arange(base_size-initial_sample)\n", - "#plt.plot([0, base_size-initial_sample], [median_diff, median_diff],'k')\n", + "max_diff = np.median(baseline_diff) + 0.007\n", + "min_diff = np.median(baseline_diff) - 0.007\n", + "time_vec = np.arange(base_size - initial_sample)\n", + "# plt.plot([0, base_size-initial_sample], [median_diff, median_diff],'k')\n", "plt.plot(baseline_diff)\n", - "plt.plot([0, base_size-initial_sample], [max_diff, max_diff],'r')\n", - "plt.plot([0, base_size-initial_sample], [min_diff, min_diff],'g')\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time (s)')\n", - "plt.title('Difference first iterations NIDAQ Pulse time - Virmen Behavior time')" + "plt.plot([0, base_size - initial_sample], [max_diff, max_diff], \"r\")\n", + "plt.plot([0, base_size - initial_sample], [min_diff, min_diff], \"g\")\n", + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time (s)\")\n", + "plt.title(\"Difference first iterations NIDAQ Pulse time - Virmen Behavior time\")" ] }, { @@ -470,7 +503,7 @@ "source": [ "print(synced_time_vector[0:5])\n", "\n", - "print(behavior_time_vector[0:5])\n" + "print(behavior_time_vector[0:5])" ] }, { @@ -568,20 +601,15 @@ } ], "source": [ - "\n", - "vec_shift = np.zeros((synced_time_vector.shape[0]-base_size), dtype=int)\n", - "for i in range(synced_time_vector.shape[0]-base_size):\n", - "\n", - "\n", - " idx_start = initial_sample+i+1\n", - " idx_end = base_size+i+1\n", + "vec_shift = np.zeros((synced_time_vector.shape[0] - base_size), dtype=int)\n", + "for i in range(synced_time_vector.shape[0] - base_size):\n", + " idx_start = initial_sample + i + 1\n", + " idx_end = base_size + i + 1\n", "\n", " this_bt = behavior_time_vector[idx_start:idx_end]\n", " this_iv = synced_time_vector[idx_start:idx_end]\n", "\n", - " median_ori = np.median(this_iv-this_bt)\n", - "\n", - " \n", + " median_ori = np.median(this_iv - this_bt)\n", "\n", " if median_ori >= min_diff and median_ori < max_diff:\n", " vec_shift[i] = 0\n", @@ -592,26 +620,24 @@ " else:\n", " sign = -1\n", " print(i, -1)\n", - " \n", + "\n", " new_sign = sign\n", " if sign != 0:\n", - "\n", " for j in range(1, 100):\n", - "\n", " if new_sign == 1:\n", - " if idx_end+j > synced_time_vector.shape[0]:\n", - " this_iv = synced_time_vector[idx_start+j:]\n", + " if idx_end + j > synced_time_vector.shape[0]:\n", + " this_iv = synced_time_vector[idx_start + j :]\n", " this_bt = this_bt[:-1]\n", " else:\n", - " this_iv = synced_time_vector[idx_start+j:idx_end+j]\n", + " this_iv = synced_time_vector[idx_start + j : idx_end + j]\n", " else:\n", - " this_iv = synced_time_vector[idx_start-j:idx_end-j]\n", - " \n", - " median_now = np.median(this_iv-this_bt)\n", - " #print('median_now', median_now, 'sign', sign)\n", - " \n", + " this_iv = synced_time_vector[idx_start - j : idx_end - j]\n", + "\n", + " median_now = np.median(this_iv - this_bt)\n", + " # print('median_now', median_now, 'sign', sign)\n", + "\n", " if median_now >= min_diff and median_now < max_diff:\n", - " vec_shift[i] = j*new_sign\n", + " vec_shift[i] = j * new_sign\n", " break\n", " elif median_now < min_diff:\n", " new_sign = 1\n", @@ -619,16 +645,14 @@ " new_sign = -1\n", "\n", " if new_sign != sign:\n", - " if np.abs(median_ori-median_diff) < np.abs(median_now-median_diff):\n", + " if np.abs(median_ori - median_diff) < np.abs(median_now - median_diff):\n", " vec_shift[i] = 0\n", " else:\n", - " vec_shift[i] = j*sign \n", + " vec_shift[i] = j * sign\n", " break\n", - " \n", "\n", " if j == 99:\n", - " vec_shift[i] = j*sign\n", - "\n", + " vec_shift[i] = j * sign\n", "\n", "\n", "print(vec_shift)" @@ -661,15 +685,15 @@ } ], "source": [ - "#plt.plot(np.diff(new_synced_time_vector))\n", + "# plt.plot(np.diff(new_synced_time_vector))\n", "plt.plot(np.diff(synced_time_vector))\n", "plt.plot(np.diff(behavior_time_vector))\n", - "plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n", - "#plt.plot(np.diff(new_synced_time_vector3))\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time (s)')\n", - "plt.title('Difference iterations NIDAQ Pulse time - Virmen Behavior time')\n", - "#plt.title('Iteration Duration NIDAQ')\n", + "plt.plot(synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]])\n", + "# plt.plot(np.diff(new_synced_time_vector3))\n", + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time (s)\")\n", + "plt.title(\"Difference iterations NIDAQ Pulse time - Virmen Behavior time\")\n", + "# plt.title('Iteration Duration NIDAQ')\n", "plt.ylim([-0.04, 0.04])\n", "plt.xlim([1000, 1100])" ] @@ -681,17 +705,16 @@ "outputs": [], "source": [ "new_synced_time_vector = synced_time_vector.copy()\n", - "mid_point = int(np.floor(initial_sample+base_size/2))\n", + "mid_point = int(np.floor(initial_sample + base_size / 2))\n", "for i in range(vec_shift.shape[0]):\n", - " new_synced_time_vector[mid_point+i] = synced_time_vector[mid_point+i+vec_shift[i]]\n", - " \n", - "idx_end = mid_point+vec_shift.shape[0]-1\n", - "for i in range(idx_end, new_synced_time_vector.shape[0]):\n", + " new_synced_time_vector[mid_point + i] = synced_time_vector[\n", + " mid_point + i + vec_shift[i]\n", + " ]\n", "\n", - " if i+vec_shift[-1] < new_synced_time_vector.shape[0]:\n", - " new_synced_time_vector[i] = synced_time_vector[i+vec_shift[-1]]\n", - "\n", - "\n" + "idx_end = mid_point + vec_shift.shape[0] - 1\n", + "for i in range(idx_end, new_synced_time_vector.shape[0]):\n", + " if i + vec_shift[-1] < new_synced_time_vector.shape[0]:\n", + " new_synced_time_vector[i] = synced_time_vector[i + vec_shift[-1]]" ] }, { @@ -709,18 +732,16 @@ } ], "source": [ - "\n", "diff_vec_shift = np.diff(vec_shift)\n", "where_insert_iteration = np.where(diff_vec_shift != 0)\n", "where_insert_iteration = where_insert_iteration[0]\n", - "print('where_insert_iteration', where_insert_iteration+mid_point)\n", + "print(\"where_insert_iteration\", where_insert_iteration + mid_point)\n", "\n", "find = np.full(4, 1)\n", "correlation_result = np.correlate(diff_vec_shift == 0, find)\n", "consecutive_zeros = np.flatnonzero(correlation_result == 4)\n", "\n", "\n", - "\n", "index_shift = 0\n", "index_borrow_virmen = list()\n", "for i in range(len(where_insert_iteration)):\n", @@ -728,71 +749,78 @@ " stable_parts = stable_parts[0]\n", "\n", " if stable_parts.shape[0] > 0:\n", - " index_borrow_virmen.append([where_insert_iteration[index_shift], consecutive_zeros[stable_parts[0]]])\n", - " next_borrow_start = np.where(where_insert_iteration > consecutive_zeros[stable_parts[0]])\n", + " index_borrow_virmen.append(\n", + " [where_insert_iteration[index_shift], consecutive_zeros[stable_parts[0]]]\n", + " )\n", + " next_borrow_start = np.where(\n", + " where_insert_iteration > consecutive_zeros[stable_parts[0]]\n", + " )\n", " next_borrow_start = next_borrow_start[0]\n", " if next_borrow_start.shape[0] > 0:\n", - " index_shift = next_borrow_start[0]\n", + " index_shift = next_borrow_start[0]\n", " else:\n", " break\n", " else:\n", - " print('Extreme case diff vec shift')\n", + " print(\"Extreme case diff vec shift\")\n", "\n", - " index_borrow_virmen.append([where_insert_iteration[index_shift], diff_vec_shift.shape[0]])\n", - " print('index_borrow_virmen', index_borrow_virmen)\n", + " index_borrow_virmen.append(\n", + " [where_insert_iteration[index_shift], diff_vec_shift.shape[0]]\n", + " )\n", + " print(\"index_borrow_virmen\", index_borrow_virmen)\n", " break\n", "\n", "\n", - "\n", - "print('index_borrow_virmen', index_borrow_virmen)\n", + "print(\"index_borrow_virmen\", index_borrow_virmen)\n", "\n", "\n", "new_synced_time_vector2 = new_synced_time_vector.copy()\n", "\n", - "#for i in range(where_insert_iteration.shape[0]):\n", + "# for i in range(where_insert_iteration.shape[0]):\n", "index_diff_e = -1\n", "for i in range(len(index_borrow_virmen)):\n", - "\n", - " if index_diff_e > index_borrow_virmen[i][0]+mid_point:\n", + " if index_diff_e > index_borrow_virmen[i][0] + mid_point:\n", " continue\n", "\n", " for j in range(100):\n", - "\n", - " index_diff_s = index_borrow_virmen[i][0]+mid_point\n", + " index_diff_s = index_borrow_virmen[i][0] + mid_point\n", " index_diff_vec = index_borrow_virmen[i][0]\n", - " index_diff_e = index_borrow_virmen[i][1]+mid_point\n", + " index_diff_e = index_borrow_virmen[i][1] + mid_point\n", "\n", - " if j>=1:\n", - " index_diff_s = index_diff_s-1\n", - " if j >1:\n", - " index_diff_e = index_diff_e+j-1\n", + " if j >= 1:\n", + " index_diff_s = index_diff_s - 1\n", + " if j > 1:\n", + " index_diff_e = index_diff_e + j - 1\n", "\n", " print(j)\n", "\n", - " print('index_diff_vec', index_diff_s, index_diff_e)\n", - " #print('diff_vec_shift[index_diff] ', diff_vec_shift[index_diff_vec-1:index_diff_vec+1] )\n", + " print(\"index_diff_vec\", index_diff_s, index_diff_e)\n", + " # print('diff_vec_shift[index_diff] ', diff_vec_shift[index_diff_vec-1:index_diff_vec+1] )\n", + "\n", + " # print('behavior_time_vector', behavior_time_vector[index_diff_s-1:index_diff_e+2])\n", + " # print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])\n", "\n", - " #print('behavior_time_vector', behavior_time_vector[index_diff_s-1:index_diff_e+2])\n", - " #print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])\n", + " time_iteration = (\n", + " behavior_time_vector[index_diff_s : index_diff_e + 1]\n", + " - behavior_time_vector[index_diff_s]\n", + " )\n", + " # print('time_iteration', time_iteration)\n", "\n", - " time_iteration = behavior_time_vector[index_diff_s:index_diff_e+1] - behavior_time_vector[index_diff_s]\n", - " #print('time_iteration', time_iteration)\n", + " new_synced_time_vector2[index_diff_s : index_diff_e + 1] = (\n", + " np.repeat(\n", + " new_synced_time_vector2[index_diff_s], index_diff_e - index_diff_s + 1\n", + " )\n", + " + time_iteration\n", + " )\n", "\n", - " new_synced_time_vector2[index_diff_s:index_diff_e+1] = np.repeat(new_synced_time_vector2[index_diff_s], index_diff_e-index_diff_s+1) +\\\n", - " time_iteration\n", - " \n", - " check_diff = np.diff(new_synced_time_vector2[index_diff_s:index_diff_e+2])\n", + " check_diff = np.diff(new_synced_time_vector2[index_diff_s : index_diff_e + 2])\n", " idx_back_time = np.where(check_diff <= 0.0005)\n", " idx_back_time = idx_back_time[0]\n", " if idx_back_time.shape[0] == 0:\n", " break\n", - " \n", "\n", - " \n", - " #print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])\n", - " #print('diff new_synced_time_vector2', np.diff(new_synced_time_vector2[index_diff_s-1:index_diff_e+2]))\n", - " #print('diff new_synced_time_vector2', np.diff(behavior_time_vector[index_diff_s-1:index_diff_e+2]))\n", - "\n" + " # print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2])\n", + " # print('diff new_synced_time_vector2', np.diff(new_synced_time_vector2[index_diff_s-1:index_diff_e+2]))\n", + " # print('diff new_synced_time_vector2', np.diff(behavior_time_vector[index_diff_s-1:index_diff_e+2]))" ] }, { @@ -830,10 +858,12 @@ ], "source": [ "new_synced_time_vector3 = new_synced_time_vector2.copy()\n", - "diff_vecs = (new_synced_time_vector2 - behavior_time_vector[:new_synced_time_vector2.shape[0]])\n", + "diff_vecs = (\n", + " new_synced_time_vector2 - behavior_time_vector[: new_synced_time_vector2.shape[0]]\n", + ")\n", "\n", "idx_plus = np.where(diff_vecs > 1000)\n", - "#idx_plus = np.where(diff_vecs > max_diff)\n", + "# idx_plus = np.where(diff_vecs > max_diff)\n", "idx_plus = idx_plus[0]\n", "\n", "break_points = np.where(np.diff(idx_plus) != 1)[0] + 1\n", @@ -842,17 +872,18 @@ "print(idx_plus)\n", "print(grouped_sections)\n", "for i in range(len(grouped_sections)):\n", - "\n", " if grouped_sections[i].shape[0] > 0:\n", - "\n", - " idx_start = grouped_sections[i][0]-1\n", + " idx_start = grouped_sections[i][0] - 1\n", " idx_end = grouped_sections[i][-1]\n", "\n", - " time_vector = behavior_time_vector[idx_start:idx_end+1]-behavior_time_vector[idx_start]\n", - " new_synced_time_vector3[idx_start:idx_end+1] =\\\n", - " np.repeat(new_synced_time_vector3[idx_start], idx_end-idx_start+1) + time_vector\n", - "\n", - " " + " time_vector = (\n", + " behavior_time_vector[idx_start : idx_end + 1]\n", + " - behavior_time_vector[idx_start]\n", + " )\n", + " new_synced_time_vector3[idx_start : idx_end + 1] = (\n", + " np.repeat(new_synced_time_vector3[idx_start], idx_end - idx_start + 1)\n", + " + time_vector\n", + " )" ] }, { @@ -879,19 +910,18 @@ "size_vec = new_synced_time_vector4.shape[0]\n", "diff_size = behavior_time_vector.shape[0] - new_synced_time_vector3.shape[0]\n", "\n", - "print('diff_size', diff_size)\n", + "print(\"diff_size\", diff_size)\n", "\n", "if diff_size > 0:\n", - " diff_size = diff_size+1\n", + " diff_size = diff_size + 1\n", "\n", - " last_part_bt = behavior_time_vector[-diff_size:]- behavior_time_vector[-diff_size]\n", + " last_part_bt = behavior_time_vector[-diff_size:] - behavior_time_vector[-diff_size]\n", " insert_part = np.repeat(new_synced_time_vector4[-1], diff_size) + last_part_bt\n", - " print('last_part_bt',last_part_bt)\n", + " print(\"last_part_bt\", last_part_bt)\n", "\n", - " print('insert_part',insert_part)\n", + " print(\"insert_part\", insert_part)\n", "\n", - " new_synced_time_vector4 = np.append(new_synced_time_vector4, insert_part[1:])\n", - "\n" + " new_synced_time_vector4 = np.append(new_synced_time_vector4, insert_part[1:])" ] }, { @@ -940,41 +970,44 @@ } ], "source": [ - "\n", "plt.figure(figsize=(8, 6))\n", - "#plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n", - "\n", - "\n", - "\n", - "#plt.plot(new_synced_time_vector2 - behavior_time_vector[:new_synced_time_vector2.shape[0]])\n", - "#plt.plot(np.diff(new_synced_time_vector))\n", - "#plt.plot(np.diff(new_synced_time_vector2))\n", - "#plt.plot(np.diff(behavior_time_vector))\n", - "#plt.plot(np.diff(new_synced_time_vector3))\n", - "plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n", - "#plt.plot(new_synced_time_vector - behavior_time_vector[:new_synced_time_vector.shape[0]])\n", - "#plt.plot(new_synced_time_vector2 - behavior_time_vector[:new_synced_time_vector3.shape[0]])\n", - "#plt.plot(new_synced_time_vector4 - behavior_time_vector[:new_synced_time_vector4.shape[0]], color ='#ff7f0e')\n", - "#plt.plot(new_synced_time_vector3 - behavior_time_vector[:new_synced_time_vector3.shape[0]], color='#1f77b4')\n", - "\n", - "plt.plot(new_synced_time_vector4 - behavior_time_vector[:new_synced_time_vector4.shape[0]])\n", - "\n", - "time_shift = np.arange(int(initial_sample+base_size/2), vec_shift.shape[0]+int(initial_sample+base_size/2))\n", - "#plt.plot([time_shift[0], time_shift[-1]],[min_diff, min_diff], 'g')\n", - "#plt.plot([time_shift[0], time_shift[-1]],[max_diff, max_diff], 'r')\n", - "#plt.plot([time_shift[0], 500],[0, 0], 'k')\n", - "\n", - "#plt.plot([491, 491],[min_diff, max_diff], 'k')\n", - "\n", - "#plt.plot(time_shift,vec_shift*0.01)\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time (s)')\n", - "plt.title('Difference iterations NIDAQ Pulse time - Virmen Behavior time')\n", - "#plt.title('Iteration Duration NIDAQ')\n", + "# plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n", + "\n", + "\n", + "# plt.plot(new_synced_time_vector2 - behavior_time_vector[:new_synced_time_vector2.shape[0]])\n", + "# plt.plot(np.diff(new_synced_time_vector))\n", + "# plt.plot(np.diff(new_synced_time_vector2))\n", + "# plt.plot(np.diff(behavior_time_vector))\n", + "# plt.plot(np.diff(new_synced_time_vector3))\n", + "plt.plot(synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]])\n", + "# plt.plot(new_synced_time_vector - behavior_time_vector[:new_synced_time_vector.shape[0]])\n", + "# plt.plot(new_synced_time_vector2 - behavior_time_vector[:new_synced_time_vector3.shape[0]])\n", + "# plt.plot(new_synced_time_vector4 - behavior_time_vector[:new_synced_time_vector4.shape[0]], color ='#ff7f0e')\n", + "# plt.plot(new_synced_time_vector3 - behavior_time_vector[:new_synced_time_vector3.shape[0]], color='#1f77b4')\n", + "\n", + "plt.plot(\n", + " new_synced_time_vector4 - behavior_time_vector[: new_synced_time_vector4.shape[0]]\n", + ")\n", + "\n", + "time_shift = np.arange(\n", + " int(initial_sample + base_size / 2),\n", + " vec_shift.shape[0] + int(initial_sample + base_size / 2),\n", + ")\n", + "# plt.plot([time_shift[0], time_shift[-1]],[min_diff, min_diff], 'g')\n", + "# plt.plot([time_shift[0], time_shift[-1]],[max_diff, max_diff], 'r')\n", + "# plt.plot([time_shift[0], 500],[0, 0], 'k')\n", + "\n", + "# plt.plot([491, 491],[min_diff, max_diff], 'k')\n", + "\n", + "# plt.plot(time_shift,vec_shift*0.01)\n", + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time (s)\")\n", + "plt.title(\"Difference iterations NIDAQ Pulse time - Virmen Behavior time\")\n", + "# plt.title('Iteration Duration NIDAQ')\n", "plt.ylim([-0.04, 0.2])\n", - "#plt.ylim([-0.01, 0.03])\n", - "#plt.ylim([-0.005, 0.05])\n", - "#plt.xlim([480, 500])" + "# plt.ylim([-0.01, 0.03])\n", + "# plt.ylim([-0.005, 0.05])\n", + "# plt.xlim([480, 500])" ] }, { @@ -1005,21 +1038,21 @@ ], "source": [ "plt.figure(figsize=(8, 6))\n", - "plt.plot(0,0)\n", - "plt.plot(0,0)\n", + "plt.plot(0, 0)\n", + "plt.plot(0, 0)\n", "\n", "\n", "plt.plot(np.diff(new_synced_time_vector4))\n", - "#plt.plot(np.diff(new_synced_time_vector3))\n", - "#plt.plot(np.diff(behavior_time_vector))\n", + "# plt.plot(np.diff(new_synced_time_vector3))\n", + "# plt.plot(np.diff(behavior_time_vector))\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time (s)')\n", - "plt.title('ViRMEn behavior time iteration duration')\n", + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time (s)\")\n", + "plt.title(\"ViRMEn behavior time iteration duration\")\n", "plt.ylim([-0.05, 0.05])\n", - "#plt.ylim([-0.01, 0.03])\n", - "#plt.ylim([-0.005, 0.05])\n", - "#plt.xlim([305, 320])" + "# plt.ylim([-0.01, 0.03])\n", + "# plt.ylim([-0.005, 0.05])\n", + "# plt.xlim([305, 320])" ] }, { @@ -1096,12 +1129,11 @@ "source": [ "difo = np.diff(new_synced_time_vector)\n", "\n", - "idx_r = np.where(difo <= 0 )\n", + "idx_r = np.where(difo <= 0)\n", "idx_r = idx_r[0]\n", "print(idx_r)\n", - "idx_vs = np.where(vec_shift != 0 )\n", - "idx_vs = idx_vs[0]\n", - "\n" + "idx_vs = np.where(vec_shift != 0)\n", + "idx_vs = idx_vs[0]" ] }, { @@ -1150,9 +1182,9 @@ } ], "source": [ - "plt.plot(np.diff(synced_time_vector[idx_r[0]-5:]))\n", - "plt.plot(np.diff(behavior_time_vector[idx_r[0]-5:]))\n", - "plt.ylim(-0.02,0.07)" + "plt.plot(np.diff(synced_time_vector[idx_r[0] - 5 :]))\n", + "plt.plot(np.diff(behavior_time_vector[idx_r[0] - 5 :]))\n", + "plt.ylim(-0.02, 0.07)" ] }, { @@ -1171,7 +1203,7 @@ ], "source": [ "it_times = new_synced_time_vector\n", - "#it_times = synced_time_vector\n", + "# it_times = synced_time_vector\n", "beh_times = behavior_time_vector\n", "\n", "print(it_times.shape[0])\n", @@ -1196,9 +1228,9 @@ ], "source": [ "if it_times.shape[0] >= beh_times.shape[0]:\n", - " diff_vector = np.diff(it_times[:beh_times.shape[0]] - beh_times)\n", + " diff_vector = np.diff(it_times[: beh_times.shape[0]] - beh_times)\n", "else:\n", - " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n", + " diff_vector = np.diff(it_times - beh_times[: it_times.shape[0]])\n", "\n", "import scipy\n", "\n", @@ -1223,13 +1255,13 @@ } ], "source": [ - "#plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n", - "plt.plot(new_synced_time_vector - beh_times[:it_times.shape[0]])\n", - "#plt.plot((it_times - beh_times))\n", - "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", + "# plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n", + "plt.plot(new_synced_time_vector - beh_times[: it_times.shape[0]])\n", + "# plt.plot((it_times - beh_times))\n", + "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", "plt.plot(peaks, diff_vector[peaks], \"x\")\n", - "plt.ylabel('Time diff in current iteration TTL - Virmen (s)')\n", - "plt.xlabel('Iteration')\n", + "plt.ylabel(\"Time diff in current iteration TTL - Virmen (s)\")\n", + "plt.xlabel(\"Iteration\")\n", "plt.ylim([-0.05, 0.02])\n", "plt.show()" ] @@ -1262,11 +1294,11 @@ ], "source": [ "plt.plot(np.diff(new_synced_time_vector))\n", - "plt.plot(np.diff(beh_times)+0.01)\n", - "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", - "#plt.plot(peaks, diff_vector[peaks], \"x\")\n", + "plt.plot(np.diff(beh_times) + 0.01)\n", + "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", + "# plt.plot(peaks, diff_vector[peaks], \"x\")\n", "plt.ylim([-0.01, 0.05])\n", - "plt.xlabel('Iteration')\n" + "plt.xlabel(\"Iteration\")" ] }, { @@ -1286,8 +1318,10 @@ } ], "source": [ - "difo = np.pad(np.diff(new_synced_time_vector), (1, 0), 'constant', constant_values=(0,0))\n", - "difo2 = np.pad(np.diff(beh_times), (1, 0), 'constant', constant_values=(0,0))\n", + "difo = np.pad(\n", + " np.diff(new_synced_time_vector), (1, 0), \"constant\", constant_values=(0, 0)\n", + ")\n", + "difo2 = np.pad(np.diff(beh_times), (1, 0), \"constant\", constant_values=(0, 0))\n", "\n", "peaks_ttl_pulses, _ = scipy.signal.find_peaks(difo, height=0.02, distance=20)\n", "peaks_virmen_iter, _ = scipy.signal.find_peaks(difo2, height=0.02, distance=20)\n", @@ -1354,8 +1388,8 @@ } ], "source": [ - "print(new_synced_time_vector[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n", - "print(beh_times[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n" + "print(new_synced_time_vector[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])\n", + "print(beh_times[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])" ] }, { @@ -1374,7 +1408,7 @@ ], "source": [ "print(new_synced_time_vector[-4:])\n", - "print(beh_times[-4:])\n" + "print(beh_times[-4:])" ] }, { @@ -1404,10 +1438,9 @@ } ], "source": [ - "\n", "plt.plot(np.diff(new_synced_time_vector))\n", - "#plt.plot(np.diff(synced_time_vector))\n", - "plt.plot(np.diff(behavior_time_vector)+0.01)\n", + "# plt.plot(np.diff(synced_time_vector))\n", + "plt.plot(np.diff(behavior_time_vector) + 0.01)\n", "plt.ylim(0, 0.11)" ] }, @@ -1458,8 +1491,8 @@ ], "source": [ "trial_plot = 1\n", - "plt.plot(iteration_dict['iter_times_idx'][trial_plot])\n", - "plt.plot(behavior_time[trial_plot].flatten())\n" + "plt.plot(iteration_dict[\"iter_times_idx\"][trial_plot])\n", + "plt.plot(behavior_time[trial_plot].flatten())" ] }, { @@ -1481,15 +1514,24 @@ "samp_before = 300\n", "samp_after = 300\n", "if trial_plot == 0:\n", - " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot][0]-np.int64(samp_before)\n", + " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot][0] - np.int64(\n", + " samp_before\n", + " )\n", "else:\n", - " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot-1][-1]-np.int64(samp_before)\n", - "first_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][0]+np.int64(samp_after)\n", - "second_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][1]\n", + " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1] - np.int64(\n", + " samp_before\n", + " )\n", + "first_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][0] + np.int64(\n", + " samp_after\n", + ")\n", + "second_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][1]\n", "\n", - "samp_diff = iteration_dict['iter_start_idx'][trial_plot][0] - iteration_dict['iter_start_idx'][trial_plot-1][-1]\n", - "print('last_iter_trial0', last_iter_trial0)\n", - "print('first_iter_trial1', first_iter_trial1)" + "samp_diff = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1]\n", + ")\n", + "print(\"last_iter_trial0\", last_iter_trial0)\n", + "print(\"first_iter_trial1\", first_iter_trial1)" ] }, { @@ -1519,8 +1561,8 @@ "outputs": [], "source": [ "trial_plot2 = 2\n", - "lo2 = iteration_dict['iter_times_idx'][trial_plot2][2:]\n", - "lo3 = np.append([0], lo2)\n" + "lo2 = iteration_dict[\"iter_times_idx\"][trial_plot2][2:]\n", + "lo3 = np.append([0], lo2)" ] }, { @@ -1550,11 +1592,13 @@ } ], "source": [ - "\n", "shape_beh = behavior_time[trial_plot2].shape[0]\n", - "shape_iter = iteration_dict['iter_times_idx'][trial_plot2].shape[0]\n", - "#plt.plot(iteration_dict['iter_times_idx'][trial_plot2][:shape_beh] - behavior_time[trial_plot2].flatten())\n", - "plt.plot(iteration_dict['iter_times_idx'][trial_plot2] - behavior_time[trial_plot2].flatten()[:shape_iter])\n", + "shape_iter = iteration_dict[\"iter_times_idx\"][trial_plot2].shape[0]\n", + "# plt.plot(iteration_dict['iter_times_idx'][trial_plot2][:shape_beh] - behavior_time[trial_plot2].flatten())\n", + "plt.plot(\n", + " iteration_dict[\"iter_times_idx\"][trial_plot2]\n", + " - behavior_time[trial_plot2].flatten()[:shape_iter]\n", + ")\n", "plt.ylim(-0.1, 0.11)" ] }, @@ -1608,7 +1652,7 @@ ], "source": [ "print(lo3[:5])\n", - "print(iteration_dict['iter_times_idx'][trial_plot2][:5])\n", + "print(iteration_dict[\"iter_times_idx\"][trial_plot2][:5])\n", "print(behavior_time[trial_plot2].flatten()[:5])" ] }, @@ -1652,55 +1696,64 @@ } ], "source": [ - "\n", - "\n", "start_iter = 1\n", "end_iter = 1\n", "\n", "\n", - "sample_start =(iteration_dict['iter_start_idx'][trial_plot][start_iter]-np.int64(samp_before))\n", - "sample_end = (iteration_dict['iter_start_idx'][trial_plot][end_iter]+np.int64(samp_after))\n", - "\n", - "iter_sample = digital_array[new_trial_channel,last_iter_trial0:first_iter_trial1]\n", + "sample_start = iteration_dict[\"iter_start_idx\"][trial_plot][start_iter] - np.int64(\n", + " samp_before\n", + ")\n", + "sample_end = iteration_dict[\"iter_start_idx\"][trial_plot][end_iter] + np.int64(\n", + " samp_after\n", + ")\n", "\n", - "time_vector = np.linspace(start=0, stop=iter_sample.shape[0]-1,num=iter_sample.shape[0])\n", - "time_vector = (time_vector*1000/nidq_sampling_rate)\n", - "time_vector -= ((iter_sample.shape[0]-samp_after)*1000)/nidq_sampling_rate\n", + "iter_sample = digital_array[new_trial_channel, last_iter_trial0:first_iter_trial1]\n", "\n", - "time_last_trial = time_vector[0] +(samp_before*1000)/nidq_sampling_rate\n", + "time_vector = np.linspace(\n", + " start=0, stop=iter_sample.shape[0] - 1, num=iter_sample.shape[0]\n", + ")\n", + "time_vector = time_vector * 1000 / nidq_sampling_rate\n", + "time_vector -= ((iter_sample.shape[0] - samp_after) * 1000) / nidq_sampling_rate\n", "\n", + "time_last_trial = time_vector[0] + (samp_before * 1000) / nidq_sampling_rate\n", "\n", "\n", "print(iter_sample.shape[0])\n", - "samples_after_second_pulse = (second_iter_trial1-(first_iter_trial1-samp_after))\n", - "print('samples_after_second_pulse', samples_after_second_pulse)\n", - "print(iter_sample.shape[0]-samp_after)\n", + "samples_after_second_pulse = second_iter_trial1 - (first_iter_trial1 - samp_after)\n", + "print(\"samples_after_second_pulse\", samples_after_second_pulse)\n", + "print(iter_sample.shape[0] - samp_after)\n", "\n", "idx_time_zero = np.where(time_vector >= 0)\n", "idx_time_zero = idx_time_zero[0]\n", "idx_time_zero = idx_time_zero[0]\n", "\n", "\n", - "print('new_iteration_channel', new_iteration_channel)\n", - "print(digital_array[new_iteration_channel,:].shape)\n", + "print(\"new_iteration_channel\", new_iteration_channel)\n", + "print(digital_array[new_iteration_channel, :].shape)\n", "\n", - "plt.plot(time_vector,iter_sample)\n", - "plt.plot(time_vector,digital_array[new_iteration_channel,last_iter_trial0:first_iter_trial1]+0.02)\n", - "plt.plot(0,1,\"x\")\n", - "if (idx_time_zero+samples_after_second_pulse) < time_vector.shape[0]:\n", - " plt.plot(time_vector[idx_time_zero+samples_after_second_pulse],1,\"x\")\n", - "plt.plot(time_last_trial,1,\"x\")\n", + "plt.plot(time_vector, iter_sample)\n", + "plt.plot(\n", + " time_vector,\n", + " digital_array[new_iteration_channel, last_iter_trial0:first_iter_trial1] + 0.02,\n", + ")\n", + "plt.plot(0, 1, \"x\")\n", + "if (idx_time_zero + samples_after_second_pulse) < time_vector.shape[0]:\n", + " plt.plot(time_vector[idx_time_zero + samples_after_second_pulse], 1, \"x\")\n", + "plt.plot(time_last_trial, 1, \"x\")\n", "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -1720,7 +1773,7 @@ } ], "source": [ - "iteration_dict['iter_start_idx'][trial_plot].shape" + "iteration_dict[\"iter_start_idx\"][trial_plot].shape" ] }, { @@ -1756,20 +1809,37 @@ "start_iter = 790\n", "samp_after = 600\n", "\n", - "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n", - "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n", + "plt.plot(\n", + " digital_array[\n", + " 2,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n", + " + np.int64(samp_after),\n", + " ]\n", + ")\n", + "plt.plot(\n", + " digital_array[\n", + " 1,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n", + " + np.int64(samp_after),\n", + " ]\n", + ")\n", "\n", "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -1788,8 +1858,8 @@ ], "source": [ "trial_plot = 280\n", - "print(iteration_dict['iter_start_idx'][trial_plot][2704])\n", - "print(iteration_dict['iter_start_idx'][trial_plot+1][0])" + "print(iteration_dict[\"iter_start_idx\"][trial_plot][2704])\n", + "print(iteration_dict[\"iter_start_idx\"][trial_plot + 1][0])" ] }, { @@ -1821,7 +1891,7 @@ } ], "source": [ - "plt.plot(np.diff(iteration_dict['iter_start_idx'][trial_plot]/nidq_sampling_rate))" + "plt.plot(np.diff(iteration_dict[\"iter_start_idx\"][trial_plot] / nidq_sampling_rate))" ] }, { @@ -1901,14 +1971,14 @@ "source": [ "x = np.array([])\n", "mean_x = np.array([])\n", - "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n", - " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n", + "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n", + " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n", " mean_time_trial = np.mean(s)\n", - " x = np.append(x,s, axis=0)\n", + " x = np.append(x, s, axis=0)\n", " mean_x = np.append(mean_x, mean_time_trial)\n", "\n", - "#plt.plot(x)\n", - "plt.plot(mean_x)\n" + "# plt.plot(x)\n", + "plt.plot(mean_x)" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb index 49212bef..f7e67ad1 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/check_sync_notebook-new_method_one_trial_check.ipynb @@ -50,7 +50,7 @@ "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", "import u19_pipeline.utils.ephys_fix_sync_code as ephys_fix_sync_code\n", "import u19_pipeline.utils.ephys_utils as ephys_utils\n", - "from u19_pipeline.ephys_pipeline import get_ephys_root_data_dir, get_session_directory\n" + "from u19_pipeline.ephys_pipeline import get_ephys_root_data_dir, get_session_directory" ] }, { @@ -73,12 +73,14 @@ ], "source": [ "recording_query = \"recording_id = 502\"\n", - "recording = dj.create_virtual_module('recording', 'u19_recording')\n", - "recording_key = (recording.Recording & recording_query).fetch('recording_id', as_dict=True, order_by='recording_id')\n", + "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n", + "recording_key = (recording.Recording & recording_query).fetch(\n", + " \"recording_id\", as_dict=True, order_by=\"recording_id\"\n", + ")\n", "\n", "\n", "key = (recording.Recording.BehaviorSession & recording_query).fetch1()\n", - "del key['recording_id']\n", + "del key[\"recording_id\"]\n", "\n", "key" ] @@ -101,7 +103,7 @@ ], "source": [ "session_dir = pathlib.Path(get_session_directory(recording_key[0]))\n", - "session_dir\n" + "session_dir" ] }, { @@ -119,38 +121,43 @@ } ], "source": [ - "session_dir = find_full_path(get_ephys_root_data_dir(),\n", - " get_session_directory(recording_key))\n", + "session_dir = find_full_path(\n", + " get_ephys_root_data_dir(), get_session_directory(recording_key)\n", + ")\n", "print(session_dir)\n", - "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", - "#Check if session is Nidq or OneBox\n", - "nidq_session = list(session_dir.glob('*nidq.bin*'))\n", - "obx_session = list(session_dir.glob('*obx.bin*'))\n", + "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", + "# Check if session is Nidq or OneBox\n", + "nidq_session = list(session_dir.glob(\"*nidq.bin*\"))\n", + "obx_session = list(session_dir.glob(\"*obx.bin*\"))\n", "\n", "if len(nidq_session) == 0 and len(obx_session) == 0:\n", - " print('No session found')\n", + " print(\"No session found\")\n", "elif len(nidq_session) > 0:\n", " ephys_session_fullpath = nidq_session[0]\n", "else:\n", " ephys_session_fullpath = obx_session[0]\n", "\n", - "#Nidaq file\n", - "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n", + "# Nidaq file\n", + "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n", "nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n", "\n", "\n", - "# 1: load meta data, and the content of the NIDAQ file. Its content is digital. \n", + "# 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n", "new_trial_channel = 1\n", "new_iteration_channel = 2\n", "# If PXIe card (nidq) card use for recording deduce digital channels\n", - "if nidq_meta['typeThis'] == 'nidq':\n", - " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)\n", + "if nidq_meta[\"typeThis\"] == \"nidq\":\n", + " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " ephys_session_fullpath, nidq_meta\n", + " )\n", "# If onebox card (obx) card use for recording digital channels are 0-2\n", "else:\n", - " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])\n", + " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]\n", + " )\n", " # If no sync pulse found trial and iteration signals are 0 & 1 respectively\n", - " channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]\n", - " channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]\n", + " channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]\n", + " channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]\n", "\n", " if channel0_pulses > channel1_pulses:\n", " new_trial_channel = 1\n", @@ -187,18 +194,20 @@ "metadata": {}, "outputs": [], "source": [ - "#Behavior data\n", + "# Behavior data\n", "behavior_key = key.copy()\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", "\n", "block = 0\n", "# If a specific block is requested, add that to our behavior_key. It should be an int referring to virmen block number.\n", "# This is useful for sessions in which the nidaq stream was interrupted due to restarting virmen\n", "if block > 0:\n", - " behavior_key['block'] = block\n", + " behavior_key[\"block\"] = block\n", "\n", "thissession = behavior.TowersBlock().Trial() & behavior_key\n", - "behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n" + "behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n", + " \"trial_time\", \"vi_start\", \"iterations\"\n", + ")" ] }, { @@ -207,8 +216,8 @@ "metadata": {}, "outputs": [], "source": [ - "#plt.plot(np.diff(digital_array[new_trial_channel,:5000000]))\n", - "#plt.plot(digital_array[new_trial_channel,:5000000])\n" + "# plt.plot(np.diff(digital_array[new_trial_channel,:5000000]))\n", + "# plt.plot(digital_array[new_trial_channel,:5000000])\n" ] }, { @@ -301,21 +310,37 @@ } ], "source": [ - "mode = None #Default for sessions before 12/01/2021\n", - "#mode = 'pulses' #Default for sessions after 12/01/2021\n", - "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[new_trial_channel,:], digital_array[new_iteration_channel,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n", + "mode = None # Default for sessions before 12/01/2021\n", + "# mode = 'pulses' #Default for sessions after 12/01/2021\n", + "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n", + " digital_array[new_trial_channel, :],\n", + " digital_array[new_iteration_channel, :],\n", + " nidq_sampling_rate,\n", + " behavior_time.shape[0],\n", + " behavior_time,\n", + " mode=mode,\n", + ")\n", "# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n", "\n", "\n", "# Check # of trials and iterations match\n", - "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n", + "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n", + " ephys_utils.assert_iteration_samples_count(\n", + " iteration_dict[\"iter_start_idx\"], behavior_time\n", + " )\n", + ")\n", "\n", "print(trial_count_diff)\n", "print(trials_diff_iteration_big)\n", "print(trials_diff_iteration_small)\n", "\n", "\n", - "status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n" + "status = ephys_utils.evaluate_sync_process(\n", + " trial_count_diff,\n", + " trials_diff_iteration_big,\n", + " trials_diff_iteration_small,\n", + " behavior_time.shape[0],\n", + ")" ] }, { @@ -338,8 +363,8 @@ ], "source": [ "trial_fix = 3\n", - "synced_iteration_vector = iteration_dict['iter_start_idx'][trial_fix]\n", - "synced_time_vector = (iteration_dict['iter_times_idx'][trial_fix]).copy()\n", + "synced_iteration_vector = iteration_dict[\"iter_start_idx\"][trial_fix]\n", + "synced_time_vector = (iteration_dict[\"iter_times_idx\"][trial_fix]).copy()\n", "\n", "behavior_time_vector = behavior_time[trial_fix].flatten()\n", "\n", @@ -348,10 +373,10 @@ "\n", "print(old_synced_time_vector[0:5])\n", "print(behavior_time_vector[0:5])\n", - "print('--------------------')\n", + "print(\"--------------------\")\n", "print(np.diff(old_synced_time_vector[0:6]))\n", "print(np.diff(behavior_time_vector[0:6]))\n", - "print(iteration_dict['iter_times_idx'][trial_fix][0:5])" + "print(iteration_dict[\"iter_times_idx\"][trial_fix][0:5])" ] }, { @@ -391,8 +416,7 @@ } ], "source": [ - "\n", - "'''\n", + "\"\"\"\n", "#synced_time_vector = synced_time_vector+behavior_time_vector[1]-(behavior_time_vector[2]-synced_time_vector[2])\n", "synced_time_vector = synced_time_vector+behavior_time_vector[1]\n", "synced_time_vector = np.insert(synced_time_vector, 1, behavior_time_vector[1])\n", @@ -405,7 +429,7 @@ "\n", "print(synced_time_vector[0:5])\n", "print(behavior_time_vector[0:5])\n", - "'''" + "\"\"\"" ] }, { @@ -442,29 +466,28 @@ } ], "source": [ - "\n", - "synced_time_vector2, shift_vec, median_vec = ephys_fix_sync_code.get_shift_vector(synced_time_vector,behavior_time_vector)\n", + "synced_time_vector2, shift_vec, median_vec = ephys_fix_sync_code.get_shift_vector(\n", + " synced_time_vector, behavior_time_vector\n", + ")\n", "\n", "print(median_vec)\n", "\n", "\n", + "# median_diff = np.median(baseline_diff)\n", + "# max_diff = np.median(baseline_diff)+0.007\n", + "# min_diff = np.median(baseline_diff)-0.007\n", + "# time_vec = np.arange(base_size-initial_sample)\n", + "plt.plot([0, synced_time_vector.shape[0]], [median_vec[1], median_vec[1]], \"k\")\n", + "plt.plot([0, synced_time_vector.shape[0]], [median_vec[0], median_vec[0]], \"r\")\n", + "plt.plot([0, synced_time_vector.shape[0]], [median_vec[2], median_vec[2]], \"g\")\n", "\n", - "#median_diff = np.median(baseline_diff)\n", - "#max_diff = np.median(baseline_diff)+0.007\n", - "#min_diff = np.median(baseline_diff)-0.007\n", - "#time_vec = np.arange(base_size-initial_sample)\n", - "plt.plot([0, synced_time_vector.shape[0]], [median_vec[1], median_vec[1]],'k')\n", - "plt.plot([0, synced_time_vector.shape[0]], [median_vec[0], median_vec[0]],'r')\n", - "plt.plot([0, synced_time_vector.shape[0]], [median_vec[2], median_vec[2]],'g')\n", - "\n", - "plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n", - "plt.plot(synced_time_vector2 - behavior_time_vector[:synced_time_vector2.shape[0]])\n", - "\n", + "plt.plot(synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]])\n", + "plt.plot(synced_time_vector2 - behavior_time_vector[: synced_time_vector2.shape[0]])\n", "\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time difference (s)')\n", - "plt.title('Before & After applying (get_shift_vector))')\n", + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time difference (s)\")\n", + "plt.title(\"Before & After applying (get_shift_vector))\")\n", "\n", "plt.ylim([-0.05, 0.15])" ] @@ -496,13 +519,13 @@ } ], "source": [ - "plt.plot([0, 40], [median_vec[1], median_vec[1]],'k')\n", - "plt.plot([0, 40], [median_vec[0], median_vec[0]],'r')\n", - "plt.plot([0, 40], [median_vec[2], median_vec[2]],'g')\n", + "plt.plot([0, 40], [median_vec[1], median_vec[1]], \"k\")\n", + "plt.plot([0, 40], [median_vec[0], median_vec[0]], \"r\")\n", + "plt.plot([0, 40], [median_vec[2], median_vec[2]], \"g\")\n", "\n", - "plt.plot([0, 40], [0, 0],'y')\n", + "plt.plot([0, 40], [0, 0], \"y\")\n", "\n", - "plt.plot(synced_time_vector[0:40] - behavior_time_vector[0:40])\n" + "plt.plot(synced_time_vector[0:40] - behavior_time_vector[0:40])" ] }, { @@ -524,15 +547,24 @@ "samp_before = 1000\n", "samp_after = 1600\n", "if trial_plot == 0:\n", - " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot][0]-np.int64(samp_before)\n", + " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot][0] - np.int64(\n", + " samp_before\n", + " )\n", "else:\n", - " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot-1][-1]-np.int64(samp_before)\n", - "first_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][0]+np.int64(samp_after)\n", - "second_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][1]\n", - "\n", - "samp_diff = iteration_dict['iter_start_idx'][trial_plot][0] - iteration_dict['iter_start_idx'][trial_plot-1][-1]\n", - "print('last_iter_trial0', last_iter_trial0)\n", - "print('first_iter_trial1', first_iter_trial1)" + " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1] - np.int64(\n", + " samp_before\n", + " )\n", + "first_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][0] + np.int64(\n", + " samp_after\n", + ")\n", + "second_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][1]\n", + "\n", + "samp_diff = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1]\n", + ")\n", + "print(\"last_iter_trial0\", last_iter_trial0)\n", + "print(\"first_iter_trial1\", first_iter_trial1)" ] }, { @@ -568,57 +600,70 @@ } ], "source": [ - "\n", - "\n", "start_iter = 1\n", "end_iter = 1\n", "\n", "\n", - "sample_start =(iteration_dict['iter_start_idx'][trial_plot][start_iter]-np.int64(samp_before))\n", - "sample_end = (iteration_dict['iter_start_idx'][trial_plot][end_iter]+np.int64(samp_after))\n", + "sample_start = iteration_dict[\"iter_start_idx\"][trial_plot][start_iter] - np.int64(\n", + " samp_before\n", + ")\n", + "sample_end = iteration_dict[\"iter_start_idx\"][trial_plot][end_iter] + np.int64(\n", + " samp_after\n", + ")\n", "\n", - "iter_sample = digital_array[new_trial_channel,last_iter_trial0:first_iter_trial1]\n", + "iter_sample = digital_array[new_trial_channel, last_iter_trial0:first_iter_trial1]\n", "\n", - "time_vector = np.linspace(start=0, stop=iter_sample.shape[0]-1,num=iter_sample.shape[0])\n", - "time_vector = (time_vector*1000/nidq_sampling_rate)\n", - "time_vector -= ((iter_sample.shape[0]-samp_after)*1000)/nidq_sampling_rate\n", - "\n", - "time_last_trial = time_vector[0] +(samp_before*1000)/nidq_sampling_rate\n", + "time_vector = np.linspace(\n", + " start=0, stop=iter_sample.shape[0] - 1, num=iter_sample.shape[0]\n", + ")\n", + "time_vector = time_vector * 1000 / nidq_sampling_rate\n", + "time_vector -= ((iter_sample.shape[0] - samp_after) * 1000) / nidq_sampling_rate\n", "\n", + "time_last_trial = time_vector[0] + (samp_before * 1000) / nidq_sampling_rate\n", "\n", "\n", "print(iter_sample.shape[0])\n", - "samples_after_second_pulse = (second_iter_trial1-(first_iter_trial1-samp_after))\n", - "print('samples_after_second_pulse', samples_after_second_pulse)\n", - "print(iter_sample.shape[0]-samp_after)\n", + "samples_after_second_pulse = second_iter_trial1 - (first_iter_trial1 - samp_after)\n", + "print(\"samples_after_second_pulse\", samples_after_second_pulse)\n", + "print(iter_sample.shape[0] - samp_after)\n", "\n", "idx_time_zero = np.where(time_vector >= 0)\n", "idx_time_zero = idx_time_zero[0]\n", "idx_time_zero = idx_time_zero[0]\n", "\n", "\n", - "print('new_iteration_channel', new_iteration_channel)\n", - "print(digital_array[new_iteration_channel,:].shape)\n", - "\n", - "plt.plot(time_vector,iter_sample)\n", - "plt.plot(time_vector,digital_array[new_iteration_channel,last_iter_trial0:first_iter_trial1]+0.02)\n", - "plt.plot(0,1,\"rx\",'')\n", - "if (idx_time_zero+samples_after_second_pulse) < time_vector.shape[0]:\n", - " plt.plot(time_vector[idx_time_zero+samples_after_second_pulse],1,\"gx\")\n", - "plt.plot(time_last_trial,1,\"bx\",)\n", - "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", - "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", - "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)\n", - "\n", - "#plt.xlim([-20, 20])" + "print(\"new_iteration_channel\", new_iteration_channel)\n", + "print(digital_array[new_iteration_channel, :].shape)\n", + "\n", + "plt.plot(time_vector, iter_sample)\n", + "plt.plot(\n", + " time_vector,\n", + " digital_array[new_iteration_channel, last_iter_trial0:first_iter_trial1] + 0.02,\n", + ")\n", + "plt.plot(0, 1, \"rx\", \"\")\n", + "if (idx_time_zero + samples_after_second_pulse) < time_vector.shape[0]:\n", + " plt.plot(time_vector[idx_time_zero + samples_after_second_pulse], 1, \"gx\")\n", + "plt.plot(\n", + " time_last_trial,\n", + " 1,\n", + " \"bx\",\n", + ")\n", + "\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", + "\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", + "\n", + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)\n", + "\n", + "# plt.xlim([-20, 20])" ] }, { @@ -648,18 +693,19 @@ } ], "source": [ - "synced_time_vector3,_ = ephys_fix_sync_code.fix_shifted_sync_vector(synced_time_vector2, behavior_time_vector, shift_vec)\n", + "synced_time_vector3, _ = ephys_fix_sync_code.fix_shifted_sync_vector(\n", + " synced_time_vector2, behavior_time_vector, shift_vec\n", + ")\n", "\n", - "plt.plot(synced_time_vector2 - behavior_time_vector[:synced_time_vector2.shape[0]])\n", - "plt.plot(synced_time_vector3 - behavior_time_vector[:synced_time_vector3.shape[0]])\n", + "plt.plot(synced_time_vector2 - behavior_time_vector[: synced_time_vector2.shape[0]])\n", + "plt.plot(synced_time_vector3 - behavior_time_vector[: synced_time_vector3.shape[0]])\n", "\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time difference (s)')\n", - "plt.title('Before & After applying (fix_shifted_sync_vector)')\n", + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time difference (s)\")\n", + "plt.title(\"Before & After applying (fix_shifted_sync_vector)\")\n", "\n", - "plt.ylim([-0.4,0.4])\n", - "\n" + "plt.ylim([-0.4, 0.4])" ] }, { @@ -689,16 +735,17 @@ } ], "source": [ - "synced_time_vector4,_ =\\\n", - " ephys_fix_sync_code.complete_last_part_sync_vec(synced_time_vector3, behavior_time_vector)\n", + "synced_time_vector4, _ = ephys_fix_sync_code.complete_last_part_sync_vec(\n", + " synced_time_vector3, behavior_time_vector\n", + ")\n", "\n", - "plt.plot(synced_time_vector3 - behavior_time_vector[:synced_time_vector3.shape[0]])\n", - "plt.plot(synced_time_vector4 - behavior_time_vector[:synced_time_vector4.shape[0]])\n", + "plt.plot(synced_time_vector3 - behavior_time_vector[: synced_time_vector3.shape[0]])\n", + "plt.plot(synced_time_vector4 - behavior_time_vector[: synced_time_vector4.shape[0]])\n", "\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time difference (s)')\n", - "plt.title('Before & After applying (complete_last_part_sync_vec)')\n", + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time difference (s)\")\n", + "plt.title(\"Before & After applying (complete_last_part_sync_vec)\")\n", "\n", "plt.ylim([-0.4, 0.4])" ] @@ -730,16 +777,17 @@ } ], "source": [ - "synced_time_vector4,_ =\\\n", - " ephys_fix_sync_code.complete_last_part_sync_vec(synced_time_vector3, behavior_time_vector)\n", + "synced_time_vector4, _ = ephys_fix_sync_code.complete_last_part_sync_vec(\n", + " synced_time_vector3, behavior_time_vector\n", + ")\n", "\n", - "plt.plot(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]])\n", - "plt.plot(synced_time_vector4 - behavior_time_vector[:synced_time_vector4.shape[0]])\n", + "plt.plot(synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]])\n", + "plt.plot(synced_time_vector4 - behavior_time_vector[: synced_time_vector4.shape[0]])\n", "\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time difference (s)')\n", - "plt.title('Before & After applying (complete_last_part_sync_vec)')\n", + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time difference (s)\")\n", + "plt.title(\"Before & After applying (complete_last_part_sync_vec)\")\n", "\n", "plt.ylim([-0.4, 2.2])" ] diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb index 1b560339..5b103705 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/20251119_sync_v4.ipynb @@ -60,10 +60,12 @@ "metadata": {}, "outputs": [], "source": [ - "key = {'subject_fullname': 'jyanar_ya054', 'session_date': datetime.date(2025, 10, 14)}\n", - "key = (acquisition.Session & key).fetch1('KEY')\n", - "recording = dj.create_virtual_module('recording', 'u19_recording')\n", - "recording_key = ((acquisition.Session * recording.Recording.BehaviorSession) & key).fetch('recording_id', as_dict=True)\n", + "key = {\"subject_fullname\": \"jyanar_ya054\", \"session_date\": datetime.date(2025, 10, 14)}\n", + "key = (acquisition.Session & key).fetch1(\"KEY\")\n", + "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n", + "recording_key = (\n", + " (acquisition.Session * recording.Recording.BehaviorSession) & key\n", + ").fetch(\"recording_id\", as_dict=True)\n", "session_dir = pathlib.Path(get_session_directory(recording_key[0]))" ] }, @@ -91,37 +93,42 @@ } ], "source": [ - "session_dir = find_full_path(get_ephys_root_data_dir(),\n", - " get_session_directory(recording_key))\n", + "session_dir = find_full_path(\n", + " get_ephys_root_data_dir(), get_session_directory(recording_key)\n", + ")\n", "print(session_dir)\n", - "#Check if session is Nidq or OneBox\n", - "nidq_session = list(session_dir.glob('*nidq.bin*'))\n", - "obx_session = list(session_dir.glob('*obx.bin*'))\n", + "# Check if session is Nidq or OneBox\n", + "nidq_session = list(session_dir.glob(\"*nidq.bin*\"))\n", + "obx_session = list(session_dir.glob(\"*obx.bin*\"))\n", "\n", "if len(nidq_session) == 0 and len(obx_session) == 0:\n", - " print('No session found')\n", + " print(\"No session found\")\n", "elif len(nidq_session) > 0:\n", " ephys_session_fullpath = nidq_session[0]\n", "else:\n", " ephys_session_fullpath = obx_session[0]\n", "\n", - "#Nidaq file\n", - "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n", + "# Nidaq file\n", + "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n", "nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n", "\n", "\n", - "# 1: load meta data, and the content of the NIDAQ file. Its content is digital. \n", + "# 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n", "new_trial_channel = 1\n", "new_iteration_channel = 2\n", "# If PXIe card (nidq) card use for recording deduce digital channels\n", - "if nidq_meta['typeThis'] == 'nidq':\n", - " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)\n", + "if nidq_meta[\"typeThis\"] == \"nidq\":\n", + " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " ephys_session_fullpath, nidq_meta\n", + " )\n", "# If onebox card (obx) card use for recording digital channels are 0-2\n", "else:\n", - " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])\n", + " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]\n", + " )\n", " # If no sync pulse found trial and iteration signals are 0 & 1 respectively\n", - " channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]\n", - " channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]\n", + " channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]\n", + " channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]\n", "\n", " if channel0_pulses > channel1_pulses:\n", " new_trial_channel = 1\n", @@ -130,10 +137,12 @@ " new_trial_channel = 0\n", " new_iteration_channel = 1\n", "\n", - "obx_iters_ttl = digital_array[new_iteration_channel,:]\n", - "obx_trials_ttl = digital_array[new_trial_channel,:]\n", + "obx_iters_ttl = digital_array[new_iteration_channel, :]\n", + "obx_trials_ttl = digital_array[new_trial_channel, :]\n", "obx_trial_start_idx = ephys_utils.get_idx_trial_start(obx_trials_ttl)\n", - "clock_time_s = np.arange(0, len(obx_iters_ttl)/nidq_sampling_rate, step=1/nidq_sampling_rate)\n", + "clock_time_s = np.arange(\n", + " 0, len(obx_iters_ttl) / nidq_sampling_rate, step=1 / nidq_sampling_rate\n", + ")\n", "obx_trial_start_s = clock_time_s[obx_trial_start_idx]" ] }, @@ -152,9 +161,11 @@ "metadata": {}, "outputs": [], "source": [ - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n", - "session = behavior.TowersBlock().Trial & key\n", - "virmen_time, virmen_iterstart, virmen_num_iterations = session.fetch('trial_time', 'vi_start', 'iterations')" + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", + "session = behavior.TowersBlock().Trial & key\n", + "virmen_time, virmen_iterstart, virmen_num_iterations = session.fetch(\n", + " \"trial_time\", \"vi_start\", \"iterations\"\n", + ")" ] }, { @@ -181,8 +192,8 @@ ], "source": [ "iter_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n", - " digital_array[new_trial_channel,:],\n", - " digital_array[new_iteration_channel,:],\n", + " digital_array[new_trial_channel, :],\n", + " digital_array[new_iteration_channel, :],\n", " nidq_sampling_rate,\n", " virmen_time.shape[0],\n", " virmen_time,\n", @@ -205,9 +216,11 @@ "metadata": {}, "outputs": [], "source": [ - "#spikes = nap.load_file('Z:/20251014_g0/ya054_20251014_g0/ya054_20251014_g0_imec0/job_id_882/catgt_output/kilosort4/ya054_20251014.nwb')['units']\n", + "# spikes = nap.load_file('Z:/20251014_g0/ya054_20251014_g0/ya054_20251014_g0_imec0/job_id_882/catgt_output/kilosort4/ya054_20251014.nwb')['units']\n", "\n", - "spikes = nap.load_file('Z:/Data/Processed/electrophysiology/jyanar/jyanar_ya054/20251014_g0/ya054_20251014_g0/ya054_20251014_g0_imec0/job_id_882/kilosort4_output/ya054_20251014.nwb')['units']\n", + "spikes = nap.load_file(\n", + " \"Z:/Data/Processed/electrophysiology/jyanar/jyanar_ya054/20251014_g0/ya054_20251014_g0/ya054_20251014_g0_imec0/job_id_882/kilosort4_output/ya054_20251014.nwb\"\n", + ")[\"units\"]\n", "spks_binned = spikes.count(bin_size=0.05)" ] }, @@ -274,11 +287,11 @@ "obx_trial_stop_idx = []\n", "for itrial in range(NTRIALS):\n", " trl_start_idx = obx_trial_start_idx[itrial]\n", - " trl_start_s = obx_trial_start_s[itrial]\n", + " trl_start_s = obx_trial_start_s[itrial]\n", " virmen_s = virmen_time[itrial]\n", " trl_stop_idx = np.where(virmen_s[-1] + trl_start_s <= clock_time_s)[0][0] - 1\n", " obx_trial_stop_idx.append(trl_stop_idx)\n", - " trial_index[trl_start_idx : trl_stop_idx] = itrial + 1" + " trial_index[trl_start_idx:trl_stop_idx] = itrial + 1" ] }, { @@ -289,7 +302,7 @@ "outputs": [], "source": [ "obx_trial_stop_idx = np.asarray(obx_trial_stop_idx)\n", - "obx_trial_stop_s = clock_time_s[obx_trial_stop_idx]" + "obx_trial_stop_s = clock_time_s[obx_trial_stop_idx]" ] }, { @@ -318,28 +331,28 @@ "for itrial in tqdm.tqdm(range(NTRIALS)):\n", " # Compute the trial start and stop indices\n", " trl_start_idx = obx_trial_start_idx[itrial]\n", - " trl_stop_idx = obx_trial_stop_idx[itrial]\n", - " trl_start_s = obx_trial_start_s[itrial]\n", - " trl_stop_s = clock_time_s[trl_stop_idx]\n", + " trl_stop_idx = obx_trial_stop_idx[itrial]\n", + " trl_start_s = obx_trial_start_s[itrial]\n", + " trl_stop_s = clock_time_s[trl_stop_idx]\n", " # Now iterate over virmen timestamps, filling in each frame\n", " NFRAMES = virmen_time[itrial].shape[0]\n", " iter_start_idx_thistrial = []\n", " for iframe in range(1, NFRAMES):\n", - " itr_start_s = virmen_time[itrial][iframe - 1] # Relative time to trial start\n", - " itr_stop_s = virmen_time[itrial][iframe] # Relative time to trial start\n", + " itr_start_s = virmen_time[itrial][iframe - 1] # Relative time to trial start\n", + " itr_stop_s = virmen_time[itrial][iframe] # Relative time to trial start\n", " # Compute the number of frames to get there\n", " itr_start_idx = int(np.floor(itr_start_s * nidq_sampling_rate))\n", - " itr_stop_idx = int(np.floor(itr_stop_s * nidq_sampling_rate))\n", + " itr_stop_idx = int(np.floor(itr_stop_s * nidq_sampling_rate))\n", " # Express in absolute time\n", " itr_start_idx += trl_start_idx\n", " itr_stop_idx += trl_start_idx\n", " iter_start_idx_thistrial.append(itr_start_idx)\n", " # SLOW\n", - " #itr_start_s = trl_start_s + virmen_time[itrial][iframe - 1] # Express start and stop in absolute time\n", - " #itr_stop_s = trl_start_s + virmen_time[itrial][iframe]\n", - " #itr_start_idx = np.where(itr_start_s <= clock_time_s)[0][0] - 1\n", - " #itr_stop_idx = np.where(itr_stop_s <= clock_time_s)[0][0] - 1\n", - " frame_index[itr_start_idx : itr_stop_idx] = iframe\n", + " # itr_start_s = trl_start_s + virmen_time[itrial][iframe - 1] # Express start and stop in absolute time\n", + " # itr_stop_s = trl_start_s + virmen_time[itrial][iframe]\n", + " # itr_start_idx = np.where(itr_start_s <= clock_time_s)[0][0] - 1\n", + " # itr_stop_idx = np.where(itr_stop_s <= clock_time_s)[0][0] - 1\n", + " frame_index[itr_start_idx:itr_stop_idx] = iframe\n", "\n", " last_start_rel = virmen_time[itrial][-1]\n", " last_start_idx = trl_start_idx + int(np.floor(last_start_rel * nidq_sampling_rate))\n", @@ -378,7 +391,7 @@ ], "source": [ "plt.plot(clock_time_s, frame_index)\n", - "plt.plot(clock_time_s, iter_dict['framenumber_vector_samples'])" + "plt.plot(clock_time_s, iter_dict[\"framenumber_vector_samples\"])" ] }, { @@ -410,7 +423,7 @@ ], "source": [ "plt.plot(clock_time_s, trial_index)\n", - "plt.plot(clock_time_s, iter_dict['trialnumber_vector_samples'])" + "plt.plot(clock_time_s, iter_dict[\"trialnumber_vector_samples\"])" ] }, { @@ -432,7 +445,12 @@ ], "source": [ "# Are the manually-computed trial_index and frame_index arrays different from what we already had?\n", - "plt.hist(np.diff(np.stack((trial_index, iter_dict['trialnumber_vector_samples']), axis=1), axis=1),bins=100);" + "plt.hist(\n", + " np.diff(\n", + " np.stack((trial_index, iter_dict[\"trialnumber_vector_samples\"]), axis=1), axis=1\n", + " ),\n", + " bins=100,\n", + ");" ] }, { @@ -454,7 +472,12 @@ ], "source": [ "# What about the iteration index?\n", - "plt.hist(np.diff(np.stack((frame_index, iter_dict['framenumber_vector_samples']), axis=1), axis=1), density=True);" + "plt.hist(\n", + " np.diff(\n", + " np.stack((frame_index, iter_dict[\"framenumber_vector_samples\"]), axis=1), axis=1\n", + " ),\n", + " density=True,\n", + ");" ] }, { @@ -473,7 +496,7 @@ } ], "source": [ - "lolo = frame_index - iter_dict['framenumber_vector_samples']\n", + "lolo = frame_index - iter_dict[\"framenumber_vector_samples\"]\n", "\n", "print(np.nanmax(lolo))\n", "print(np.nanmin(lolo))" @@ -505,7 +528,11 @@ } ], "source": [ - "plt.plot(np.diff(np.stack((frame_index, iter_dict['framenumber_vector_samples']), axis=1), axis=1));" + "plt.plot(\n", + " np.diff(\n", + " np.stack((frame_index, iter_dict[\"framenumber_vector_samples\"]), axis=1), axis=1\n", + " )\n", + ");" ] }, { @@ -541,15 +568,19 @@ "\n", "spk_resp = []\n", "for itrial in range(NTRIALS):\n", - " epoch = nap.IntervalSet(obx_trial_start_s[itrial] - pre_s, obx_trial_start_s[itrial] + post_s)\n", - " spk_resp.append( spikes.restrict(epoch).count(bin_size=bin_size) )\n", + " epoch = nap.IntervalSet(\n", + " obx_trial_start_s[itrial] - pre_s, obx_trial_start_s[itrial] + post_s\n", + " )\n", + " spk_resp.append(spikes.restrict(epoch).count(bin_size=bin_size))\n", "\n", - "t = spk_resp[0].t - obx_trial_start_s[0] # time axis\n", + "t = spk_resp[0].t - obx_trial_start_s[0] # time axis\n", "spk_resp = np.stack(spk_resp, axis=2)\n", "mean_resp = spk_resp.mean(axis=2).mean(axis=1).T\n", "\n", "plt.plot(t, mean_resp)\n", - "plt.gca().set(xlabel='time [s]', ylabel='spks/s', title='mean population response to trial onset');" + "plt.gca().set(\n", + " xlabel=\"time [s]\", ylabel=\"spks/s\", title=\"mean population response to trial onset\"\n", + ");" ] }, { @@ -567,13 +598,14 @@ "metadata": {}, "outputs": [], "source": [ - "np.savez('ya054_2025-10-14_manual_idx_vecs_v2.npz', \n", - " trial_index_nidq=trial_index, \n", - " iteration_index_nidq=frame_index, \n", - " obx_trial_start_idx=obx_trial_start_idx,\n", - " obx_trial_stop_idx=obx_trial_stop_idx,\n", - " iter_start_idx=np.array(iter_start_idx, dtype='object'),\n", - " sampling_rate=nidq_sampling_rate,\n", + "np.savez(\n", + " \"ya054_2025-10-14_manual_idx_vecs_v2.npz\",\n", + " trial_index_nidq=trial_index,\n", + " iteration_index_nidq=frame_index,\n", + " obx_trial_start_idx=obx_trial_start_idx,\n", + " obx_trial_stop_idx=obx_trial_stop_idx,\n", + " iter_start_idx=np.array(iter_start_idx, dtype=\"object\"),\n", + " sampling_rate=nidq_sampling_rate,\n", ")" ] }, diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb index aa8709e3..f3aa9439 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/bulk_reprocess_ephys_recordings.ipynb @@ -44,9 +44,7 @@ ], "source": [ "import datajoint as dj\n", - "import pandas as pd\n", - "\n", - "\n" + "import pandas as pd" ] }, { @@ -63,10 +61,12 @@ } ], "source": [ - "#Behavior data\n", - "recording_process = dj.create_virtual_module('recording_process', 'u19_recording_process')\n", - "recording = dj.create_virtual_module('recording', 'u19_recording')\n", - "#EphysPipeline = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')" + "# Behavior data\n", + "recording_process = dj.create_virtual_module(\n", + " \"recording_process\", \"u19_recording_process\"\n", + ")\n", + "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n", + "# EphysPipeline = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')" ] }, { @@ -75,12 +75,17 @@ "metadata": {}, "outputs": [], "source": [ - "query_subjects =\\\n", - " \"subject_fullname LIKE '%jknpx4%' OR subject_fullname LIKE '%jknpx5%' OR subject_fullname LIKE '%jk85%' OR subject_fullname LIKE '%jk86%'\"\n", - "\n", + "query_subjects = \"subject_fullname LIKE '%jknpx4%' OR subject_fullname LIKE '%jknpx5%' OR subject_fullname LIKE '%jk85%' OR subject_fullname LIKE '%jk86%'\"\n", "\n", "\n", - "selected_process = pd.DataFrame((recording_process.Processing * recording.Recording * recording.Recording.BehaviorSession & query_subjects).fetch(as_dict=True, order_by='job_id'))" + "selected_process = pd.DataFrame(\n", + " (\n", + " recording_process.Processing\n", + " * recording.Recording\n", + " * recording.Recording.BehaviorSession\n", + " & query_subjects\n", + " ).fetch(as_dict=True, order_by=\"job_id\")\n", + ")" ] }, { @@ -741,10 +746,10 @@ } ], "source": [ - "selected_process_copy['status_processing_id'] = -1\n", - "selected_process_copy['slurm_id'] = None\n", - "selected_process_copy = selected_process_copy.drop(columns={'job_id'}, axis=1)\n", - "selected_process_copy = selected_process_copy.iloc[:,0:9]\n", + "selected_process_copy[\"status_processing_id\"] = -1\n", + "selected_process_copy[\"slurm_id\"] = None\n", + "selected_process_copy = selected_process_copy.drop(columns={\"job_id\"}, axis=1)\n", + "selected_process_copy = selected_process_copy.iloc[:, 0:9]\n", "selected_process_copy" ] }, @@ -754,7 +759,7 @@ "metadata": {}, "outputs": [], "source": [ - "#recording_process.Processing.insert(selected_process_copy)" + "# recording_process.Processing.insert(selected_process_copy)" ] }, { @@ -763,11 +768,18 @@ "metadata": {}, "outputs": [], "source": [ - "query_job_id = 'job_id > 900'\n", - "\n", + "query_job_id = \"job_id > 900\"\n", "\n", "\n", - "new_process = pd.DataFrame((recording_process.Processing * recording.Recording * recording.Recording.BehaviorSession & query_subjects & query_job_id).fetch(as_dict=True, order_by='job_id'))\n" + "new_process = pd.DataFrame(\n", + " (\n", + " recording_process.Processing\n", + " * recording.Recording\n", + " * recording.Recording.BehaviorSession\n", + " & query_subjects\n", + " & query_job_id\n", + " ).fetch(as_dict=True, order_by=\"job_id\")\n", + ")" ] }, { @@ -1308,12 +1320,18 @@ } ], "source": [ - "new_process_params = new_process[['job_id', 'fragment_number','status_processing_id']].copy()\n", + "new_process_params = new_process[\n", + " [\"job_id\", \"fragment_number\", \"status_processing_id\"]\n", + "].copy()\n", "\n", - "new_process_params['fragment_number'] = 1\n", - "new_process_params['status_processing_id'] = 4\n", - "new_process_params = new_process_params.rename(columns={'fragment_number': 'precluster_param_steps_id',\\\n", - " 'status_processing_id':'paramset_idx'})\n", + "new_process_params[\"fragment_number\"] = 1\n", + "new_process_params[\"status_processing_id\"] = 4\n", + "new_process_params = new_process_params.rename(\n", + " columns={\n", + " \"fragment_number\": \"precluster_param_steps_id\",\n", + " \"status_processing_id\": \"paramset_idx\",\n", + " }\n", + ")\n", "\n", "\n", "new_process_params" diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb index a10d512c..0a4c57ee 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_daq_signals_jorge.ipynb @@ -77,22 +77,22 @@ } ], "source": [ - "session_dir = pathlib.Path('//cup.pni.princeton.edu/braininit/Data/Raw/electrophysiology/jyanar/jyanar_ya054/20251012_g0/ya054_20251012_g0/')\n", - "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", + "session_dir = pathlib.Path(\n", + " \"//cup.pni.princeton.edu/braininit/Data/Raw/electrophysiology/jyanar/jyanar_ya054/20251012_g0/ya054_20251012_g0/\"\n", + ")\n", + "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", "\n", "\n", + "nidq_bin_full_path = list(session_dir.glob(\"*obx.bin*\"))[0]\n", "\n", - "\n", - "nidq_bin_full_path = list(session_dir.glob('*obx.bin*'))[0]\n", - "\n", - "#Nidaq file\n", - "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n", + "# Nidaq file\n", + "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n", "\n", "print(nidq_meta)\n", - "#nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n", - "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta, d_line_list=[0,3])\n", - "\n", - "\n" + "# nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n", + "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " nidq_bin_full_path, nidq_meta, d_line_list=[0, 3]\n", + ")" ] }, { @@ -165,7 +165,7 @@ } ], "source": [ - "plt.plot(digital_array[0,:])" + "plt.plot(digital_array[0, :])" ] }, { @@ -185,7 +185,7 @@ } ], "source": [ - "trial_start_idx = (np.where(np.diff(digital_array[0,:]) == 1))\n", + "trial_start_idx = np.where(np.diff(digital_array[0, :]) == 1)\n", "len(trial_start_idx[0])" ] }, @@ -216,8 +216,8 @@ } ], "source": [ - "plt.plot(digital_array[0,trial_start_idx[0][0]-1000:trial_start_idx[0][0]+2000])\n", - "plt.plot(digital_array[1,trial_start_idx[0][0]-1000:trial_start_idx[0][0]+2000])" + "plt.plot(digital_array[0, trial_start_idx[0][0] - 1000 : trial_start_idx[0][0] + 2000])\n", + "plt.plot(digital_array[1, trial_start_idx[0][0] - 1000 : trial_start_idx[0][0] + 2000])" ] }, { @@ -1742,21 +1742,34 @@ } ], "source": [ - "mode = None #Default for sessions before 12/01/2021\n", - "#mode = 'pulses' #Default for sessions after 12/01/2021\n", - "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[1,:], digital_array[2,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n", + "mode = None # Default for sessions before 12/01/2021\n", + "# mode = 'pulses' #Default for sessions after 12/01/2021\n", + "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n", + " digital_array[1, :],\n", + " digital_array[2, :],\n", + " nidq_sampling_rate,\n", + " behavior_time.shape[0],\n", + " behavior_time,\n", + " mode=mode,\n", + ")\n", "# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n", "\n", "\n", "# Check # of trials and iterations match\n", - "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n", + "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n", + " ephys_utils.assert_iteration_samples_count(\n", + " iteration_dict[\"iter_start_idx\"], behavior_time\n", + " )\n", + ")\n", "\n", "print(trial_count_diff)\n", "print(trials_diff_iteration_big)\n", "print(trials_diff_iteration_small)\n", "\n", "\n", - "status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small)\n" + "status = ephys_utils.evaluate_sync_process(\n", + " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small\n", + ")" ] }, { @@ -1785,7 +1798,7 @@ "metadata": {}, "outputs": [], "source": [ - "it_times = iteration_dict['iter_times_idx'][trials_diff_iteration_small[0]]\n", + "it_times = iteration_dict[\"iter_times_idx\"][trials_diff_iteration_small[0]]\n", "beh_times = behavior_time[trials_diff_iteration_small[0]].flatten()" ] }, @@ -1807,9 +1820,9 @@ ], "source": [ "if it_times.shape[0] >= beh_times.shape[0]:\n", - " print('More pulses than behavior iterations, check other method')\n", + " print(\"More pulses than behavior iterations, check other method\")\n", "else:\n", - " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n", + " diff_vector = np.diff(it_times - beh_times[: it_times.shape[0]])\n", "\n", "import scipy\n", "\n", @@ -1847,7 +1860,7 @@ "metadata": {}, "outputs": [], "source": [ - "difo_iters = np.diff(iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]])\n", + "difo_iters = np.diff(iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]])\n", "peaks2, _ = scipy.signal.find_peaks(difo_iters, height=1500, distance=20)" ] }, @@ -1868,16 +1881,16 @@ } ], "source": [ - "original_iter = iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]]\n", - "new_iter_start = iteration_dict['iter_start_idx'][trials_diff_iteration_small[0]]\n", + "original_iter = iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]]\n", + "new_iter_start = iteration_dict[\"iter_start_idx\"][trials_diff_iteration_small[0]]\n", "\n", "for i in range(peaks.shape[0]):\n", - " value_insert = (original_iter[peaks[i]] + original_iter[peaks[i]+1] ) /2\n", + " value_insert = (original_iter[peaks[i]] + original_iter[peaks[i] + 1]) / 2\n", " new_iter_start = np.insert(new_iter_start, peaks[i], value_insert)\n", "\n", "\n", - "if new_iter_start.shape[0] != beh_times[:it_times.shape[0]]:\n", - " print('with peak strategy, could not find correct missing iterations')\n" + "if new_iter_start.shape[0] != beh_times[: it_times.shape[0]]:\n", + " print(\"with peak strategy, could not find correct missing iterations\")" ] }, { @@ -1930,7 +1943,7 @@ } ], "source": [ - "times = new_iter_start/nidq_sampling_rate\n", + "times = new_iter_start / nidq_sampling_rate\n", "times = times - times[0]\n", "new_diff_times = times - beh_times\n", "plt.plot(new_diff_times)" @@ -1973,9 +1986,9 @@ ], "source": [ "print(iteration_dict.keys())\n", - "so = np.where(iteration_dict['trialnumber_vector_samples'] == 12)\n", + "so = np.where(iteration_dict[\"trialnumber_vector_samples\"] == 12)\n", "\n", - "print(so[0])\n" + "print(so[0])" ] }, { @@ -2021,20 +2034,37 @@ "end_iter = 2704\n", "samp_after = 60000\n", "\n", - "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n", - "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n", + "plt.plot(\n", + " digital_array[\n", + " 1,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot][end_iter],\n", + " ]\n", + " + samp_after\n", + ")\n", + "plt.plot(\n", + " digital_array[\n", + " 2,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot][end_iter],\n", + " ]\n", + " + samp_after\n", + ")\n", "\n", "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -2072,20 +2102,37 @@ "start_iter = 2704\n", "samp_after = 600\n", "\n", - "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+samp_after])\n", - "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+samp_after])\n", + "plt.plot(\n", + " digital_array[\n", + " 2,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n", + " + samp_after,\n", + " ]\n", + ")\n", + "plt.plot(\n", + " digital_array[\n", + " 1,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n", + " + samp_after,\n", + " ]\n", + ")\n", "\n", "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -2104,8 +2151,8 @@ ], "source": [ "trial_plot = 280\n", - "print(iteration_dict['iter_start_idx'][trial_plot][2704])\n", - "print(iteration_dict['iter_start_idx'][trial_plot+1][0])" + "print(iteration_dict[\"iter_start_idx\"][trial_plot][2704])\n", + "print(iteration_dict[\"iter_start_idx\"][trial_plot + 1][0])" ] }, { @@ -2137,7 +2184,7 @@ } ], "source": [ - "plt.plot(np.diff(iteration_dict['iter_start_idx'][trial_plot]/nidq_sampling_rate))" + "plt.plot(np.diff(iteration_dict[\"iter_start_idx\"][trial_plot] / nidq_sampling_rate))" ] }, { @@ -2217,14 +2264,14 @@ "source": [ "x = np.array([])\n", "mean_x = np.array([])\n", - "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n", - " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n", + "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n", + " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n", " mean_time_trial = np.mean(s)\n", - " x = np.append(x,s, axis=0)\n", + " x = np.append(x, s, axis=0)\n", " mean_x = np.append(mean_x, mean_time_trial)\n", "\n", - "#plt.plot(x)\n", - "plt.plot(mean_x)\n" + "# plt.plot(x)\n", + "plt.plot(mean_x)" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb index e9fa2f5f..19e9bee3 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method.ipynb @@ -57,7 +57,7 @@ "from u19_pipeline.ephys_pipeline import (\n", " get_ephys_root_data_dir,\n", " get_session_directory,\n", - ")\n" + ")" ] }, { @@ -77,25 +77,26 @@ } ], "source": [ - "key = {'subject_fullname': 'jyanar_ya054',\n", - " 'session_date': datetime.date(2025, 10, 11)}\n", - "#key = {\n", + "key = {\"subject_fullname\": \"jyanar_ya054\", \"session_date\": datetime.date(2025, 10, 11)}\n", + "# key = {\n", "# 'subject_fullname': 'jk8386_jk83',\n", "# 'session_date': datetime.date(2025, 7, 31),\n", "# 'session_number': 0 }\n", "\n", - "#key2 = 'subject_fullname = \"jjulian_jj046\" and session_date = \"2022-04-19\" and block >=2'\n", - "#key = {\n", + "# key2 = 'subject_fullname = \"jjulian_jj046\" and session_date = \"2022-04-19\" and block >=2'\n", + "# key = {\n", "# 'subject_fullname': 'jjulian_jj042',\n", "# 'session_date': datetime.date(2021, 10, 31)}\n", "\n", "\n", - "key = (acquisition.Session & key).fetch1('KEY')\n", + "key = (acquisition.Session & key).fetch1(\"KEY\")\n", "key\n", "\n", - "recording = dj.create_virtual_module('recording', 'u19_recording')\n", + "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n", "\n", - "recording_key = ((acquisition.Session * recording.Recording.BehaviorSession) & key).fetch('recording_id', as_dict=True)\n", + "recording_key = (\n", + " (acquisition.Session * recording.Recording.BehaviorSession) & key\n", + ").fetch(\"recording_id\", as_dict=True)\n", "recording_key" ] }, @@ -117,7 +118,7 @@ ], "source": [ "session_dir = pathlib.Path(get_session_directory(recording_key[0]))\n", - "session_dir\n" + "session_dir" ] }, { @@ -135,38 +136,43 @@ } ], "source": [ - "session_dir = find_full_path(get_ephys_root_data_dir(),\n", - " get_session_directory(recording_key))\n", + "session_dir = find_full_path(\n", + " get_ephys_root_data_dir(), get_session_directory(recording_key)\n", + ")\n", "print(session_dir)\n", - "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", - "#Check if session is Nidq or OneBox\n", - "nidq_session = list(session_dir.glob('*nidq.bin*'))\n", - "obx_session = list(session_dir.glob('*obx.bin*'))\n", + "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", + "# Check if session is Nidq or OneBox\n", + "nidq_session = list(session_dir.glob(\"*nidq.bin*\"))\n", + "obx_session = list(session_dir.glob(\"*obx.bin*\"))\n", "\n", "if len(nidq_session) == 0 and len(obx_session) == 0:\n", - " print('No session found')\n", + " print(\"No session found\")\n", "elif len(nidq_session) > 0:\n", " ephys_session_fullpath = nidq_session[0]\n", "else:\n", " ephys_session_fullpath = obx_session[0]\n", "\n", - "#Nidaq file\n", - "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n", + "# Nidaq file\n", + "nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n", "nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n", "\n", "\n", - "# 1: load meta data, and the content of the NIDAQ file. Its content is digital. \n", + "# 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n", "new_trial_channel = 1\n", "new_iteration_channel = 2\n", "# If PXIe card (nidq) card use for recording deduce digital channels\n", - "if nidq_meta['typeThis'] == 'nidq':\n", - " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)\n", + "if nidq_meta[\"typeThis\"] == \"nidq\":\n", + " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " ephys_session_fullpath, nidq_meta\n", + " )\n", "# If onebox card (obx) card use for recording digital channels are 0-2\n", "else:\n", - " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])\n", + " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]\n", + " )\n", " # If no sync pulse found trial and iteration signals are 0 & 1 respectively\n", - " channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]\n", - " channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]\n", + " channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]\n", + " channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]\n", "\n", " if channel0_pulses > channel1_pulses:\n", " new_trial_channel = 1\n", @@ -203,10 +209,12 @@ "metadata": {}, "outputs": [], "source": [ - "#Behavior data\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n", + "# Behavior data\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", "thissession = behavior.TowersBlock().Trial() & key\n", - "behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n" + "behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n", + " \"trial_time\", \"vi_start\", \"iterations\"\n", + ")" ] }, { @@ -279,21 +287,37 @@ } ], "source": [ - "mode = None #Default for sessions before 12/01/2021\n", - "#mode = 'pulses' #Default for sessions after 12/01/2021\n", - "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[new_trial_channel,:], digital_array[new_iteration_channel,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n", + "mode = None # Default for sessions before 12/01/2021\n", + "# mode = 'pulses' #Default for sessions after 12/01/2021\n", + "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n", + " digital_array[new_trial_channel, :],\n", + " digital_array[new_iteration_channel, :],\n", + " nidq_sampling_rate,\n", + " behavior_time.shape[0],\n", + " behavior_time,\n", + " mode=mode,\n", + ")\n", "# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n", "\n", "\n", "# Check # of trials and iterations match\n", - "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n", + "trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n", + " ephys_utils.assert_iteration_samples_count(\n", + " iteration_dict[\"iter_start_idx\"], behavior_time\n", + " )\n", + ")\n", "\n", "print(trial_count_diff)\n", "print(trials_diff_iteration_big)\n", "print(trials_diff_iteration_small)\n", "\n", "\n", - "status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n" + "status = ephys_utils.evaluate_sync_process(\n", + " trial_count_diff,\n", + " trials_diff_iteration_big,\n", + " trials_diff_iteration_small,\n", + " behavior_time.shape[0],\n", + ")" ] }, { @@ -310,8 +334,11 @@ } ], "source": [ - "for i in range(len(iteration_dict['iter_start_idx'])):\n", - " if iteration_dict['iter_start_idx'][i].shape[0] > behavior_time[i].flatten().shape[0]:\n", + "for i in range(len(iteration_dict[\"iter_start_idx\"])):\n", + " if (\n", + " iteration_dict[\"iter_start_idx\"][i].shape[0]\n", + " > behavior_time[i].flatten().shape[0]\n", + " ):\n", " print(i)" ] }, @@ -322,9 +349,9 @@ "outputs": [], "source": [ "trial_fix = 1\n", - "synced_iteration_vector = iteration_dict['iter_start_idx'][trial_fix]\n", - "synced_time_vector = iteration_dict['iter_times_idx'][trial_fix]\n", - "behavior_time_vector = behavior_time[trial_fix].flatten()\n" + "synced_iteration_vector = iteration_dict[\"iter_start_idx\"][trial_fix]\n", + "synced_time_vector = iteration_dict[\"iter_times_idx\"][trial_fix]\n", + "behavior_time_vector = behavior_time[trial_fix].flatten()" ] }, { @@ -345,43 +372,58 @@ } ], "source": [ - "\n", "# Check where is more likely we miss an iteration pulse and insert it to iteration_vector\n", - "print('synced_iteration_vector', synced_iteration_vector.shape[0])\n", - "print('synced_time_vector', synced_time_vector.shape[0])\n", - "print('behavior_time_vector', behavior_time_vector.shape[0])\n", + "print(\"synced_iteration_vector\", synced_iteration_vector.shape[0])\n", + "print(\"synced_time_vector\", synced_time_vector.shape[0])\n", + "print(\"behavior_time_vector\", behavior_time_vector.shape[0])\n", "\n", "new_synced_iteration_vector = synced_iteration_vector.copy()\n", "new_synced_time_vector = synced_time_vector.copy()\n", "\n", "diff_time_test = new_synced_time_vector[2:50] - behavior_time_vector[1:49]\n", "pulse_greater_than_virmen = np.where(diff_time_test > 0)\n", - "pulse_greater_than_virmen = pulse_greater_than_virmen[0]+0\n", + "pulse_greater_than_virmen = pulse_greater_than_virmen[0] + 0\n", "if pulse_greater_than_virmen.shape[0] == 0:\n", - " #print('we have a special case of extra pulse beginning')\n", + " # print('we have a special case of extra pulse beginning')\n", " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, 1)\n", - " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n", + " new_synced_time_vector = (\n", + " new_synced_iteration_vector - new_synced_iteration_vector[0]\n", + " ) / nidq_sampling_rate\n", "\n", "for i in range(100):\n", " sir = 0\n", - " diff_time = new_synced_time_vector[sir:] - behavior_time_vector[sir:new_synced_time_vector.shape[0]]\n", + " diff_time = (\n", + " new_synced_time_vector[sir:]\n", + " - behavior_time_vector[sir : new_synced_time_vector.shape[0]]\n", + " )\n", " pulse_greater_than_virmen = np.where(diff_time > 0)\n", - " pulse_greater_than_virmen = pulse_greater_than_virmen[0]+sir\n", + " pulse_greater_than_virmen = pulse_greater_than_virmen[0] + sir\n", "\n", " if pulse_greater_than_virmen.shape[0] > 0:\n", " fix_trial = 1\n", - " \n", + "\n", " else:\n", " fix_trial = 0\n", " break\n", "\n", " print(i)\n", - " print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", - " print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", + " print(\n", + " new_synced_time_vector[\n", + " pulse_greater_than_virmen[0] - 2 : pulse_greater_than_virmen[0] + 4\n", + " ]\n", + " )\n", + " print(\n", + " behavior_time_vector[\n", + " pulse_greater_than_virmen[0] - 2 : pulse_greater_than_virmen[0] + 4\n", + " ]\n", + " )\n", " print(pulse_greater_than_virmen[0])\n", "\n", " if pulse_greater_than_virmen[0] > 3:\n", - " median_diff = np.median(new_synced_time_vector[0:pulse_greater_than_virmen[0]-3]-behavior_time_vector[0:pulse_greater_than_virmen[0]-3])\n", + " median_diff = np.median(\n", + " new_synced_time_vector[0 : pulse_greater_than_virmen[0] - 3]\n", + " - behavior_time_vector[0 : pulse_greater_than_virmen[0] - 3]\n", + " )\n", " else:\n", " median_diff = -0.015\n", "\n", @@ -389,63 +431,118 @@ " comp_value_l = -0.015\n", " comp_value_g = -0.003\n", " else:\n", - " comp_value_l = median_diff*1.3\n", - " comp_value_g = median_diff*0.7\n", - "\n", - " print('median_diff',median_diff, 'comp_value_l', comp_value_l,'comp_value_g', comp_value_g)\n", - "\n", - " diff_time_iter_missing_pulse = behavior_time_vector[pulse_greater_than_virmen[0]] - behavior_time_vector[pulse_greater_than_virmen[0]-1]\n", - " print('missing time in iter pulses', behavior_time_vector[pulse_greater_than_virmen[0]])\n", - " print('time_next_missing_pulse', diff_time_iter_missing_pulse)\n", - " diff_iter_add = np.floor(diff_time_iter_missing_pulse*nidq_sampling_rate)\n", - " value_insert_iteration = new_synced_iteration_vector[pulse_greater_than_virmen[0]-1] + diff_iter_add\n", - "\n", - " print('value_insert_iteration', value_insert_iteration)\n", - "\n", - " new_synced_iteration_vector = np.insert(new_synced_iteration_vector, pulse_greater_than_virmen[0], value_insert_iteration)\n", + " comp_value_l = median_diff * 1.3\n", + " comp_value_g = median_diff * 0.7\n", + "\n", + " print(\n", + " \"median_diff\",\n", + " median_diff,\n", + " \"comp_value_l\",\n", + " comp_value_l,\n", + " \"comp_value_g\",\n", + " comp_value_g,\n", + " )\n", + "\n", + " diff_time_iter_missing_pulse = (\n", + " behavior_time_vector[pulse_greater_than_virmen[0]]\n", + " - behavior_time_vector[pulse_greater_than_virmen[0] - 1]\n", + " )\n", + " print(\n", + " \"missing time in iter pulses\",\n", + " behavior_time_vector[pulse_greater_than_virmen[0]],\n", + " )\n", + " print(\"time_next_missing_pulse\", diff_time_iter_missing_pulse)\n", + " diff_iter_add = np.floor(diff_time_iter_missing_pulse * nidq_sampling_rate)\n", + " value_insert_iteration = (\n", + " new_synced_iteration_vector[pulse_greater_than_virmen[0] - 1] + diff_iter_add\n", + " )\n", + "\n", + " print(\"value_insert_iteration\", value_insert_iteration)\n", + "\n", + " new_synced_iteration_vector = np.insert(\n", + " new_synced_iteration_vector,\n", + " pulse_greater_than_virmen[0],\n", + " value_insert_iteration,\n", + " )\n", "\n", " if new_synced_iteration_vector.shape[0] > behavior_time_vector.shape[0]:\n", - " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, pulse_greater_than_virmen[0]+1)\n", - " #new_synced_iteration_vector = new_synced_iteration_vector[:-1]\n", + " new_synced_iteration_vector = np.delete(\n", + " new_synced_iteration_vector, pulse_greater_than_virmen[0] + 1\n", + " )\n", + " # new_synced_iteration_vector = new_synced_iteration_vector[:-1]\n", "\n", - " if pulse_greater_than_virmen[0] < (new_synced_time_vector.shape[0]-1):\n", + " if pulse_greater_than_virmen[0] < (new_synced_time_vector.shape[0] - 1):\n", " idx = pulse_greater_than_virmen[0]\n", - " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n", - " diff_vectorin = (new_synced_time_vector[idx+1:idx+10]- behavior_time_vector[idx+1:idx+10])\n", + " new_synced_time_vector = (\n", + " new_synced_iteration_vector - new_synced_iteration_vector[0]\n", + " ) / nidq_sampling_rate\n", + " diff_vectorin = (\n", + " new_synced_time_vector[idx + 1 : idx + 10]\n", + " - behavior_time_vector[idx + 1 : idx + 10]\n", + " )\n", " print(diff_vectorin)\n", " diff_inserted_iter = np.median(diff_vectorin)\n", - " print('diff_in new inserted iter', diff_inserted_iter)\n", + " print(\"diff_in new inserted iter\", diff_inserted_iter)\n", " if diff_inserted_iter < comp_value_l:\n", - " print('extra diff !!!!')\n", - " diff_behavior = behavior_time_vector[idx+1] - behavior_time_vector[idx]\n", - " idx_next_iter = np.where(new_synced_time_vector > behavior_time_vector[idx+1])\n", - " print('diff_behavior', diff_behavior)\n", + " print(\"extra diff !!!!\")\n", + " diff_behavior = behavior_time_vector[idx + 1] - behavior_time_vector[idx]\n", + " idx_next_iter = np.where(\n", + " new_synced_time_vector > behavior_time_vector[idx + 1]\n", + " )\n", + " print(\"diff_behavior\", diff_behavior)\n", " idx_next_iter = idx_next_iter[0][0]\n", - " print('idx_next_iter', idx_next_iter)\n", - " print('idx+1', idx+1)\n", - " if diff_behavior > 0.1 and idx_next_iter > idx+3:\n", - " print('bajale a indx_next_iter', idx_next_iter)\n", - " idx_next_iter = idx+3\n", - " if idx_next_iter > idx+2:\n", - " print('more than one to remove')\n", - " int_array_step = np.arange(idx+1, idx_next_iter-1, dtype=np.int64)\n", - " print('int_array_step', int_array_step)\n", - " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, int_array_step)\n", + " print(\"idx_next_iter\", idx_next_iter)\n", + " print(\"idx+1\", idx + 1)\n", + " if diff_behavior > 0.1 and idx_next_iter > idx + 3:\n", + " print(\"bajale a indx_next_iter\", idx_next_iter)\n", + " idx_next_iter = idx + 3\n", + " if idx_next_iter > idx + 2:\n", + " print(\"more than one to remove\")\n", + " int_array_step = np.arange(idx + 1, idx_next_iter - 1, dtype=np.int64)\n", + " print(\"int_array_step\", int_array_step)\n", + " new_synced_iteration_vector = np.delete(\n", + " new_synced_iteration_vector, int_array_step\n", + " )\n", " else:\n", - " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, [idx+1])\n", - " print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", + " new_synced_iteration_vector = np.delete(\n", + " new_synced_iteration_vector, [idx + 1]\n", + " )\n", + " print(\n", + " new_synced_time_vector[\n", + " pulse_greater_than_virmen[0] - 2 : pulse_greater_than_virmen[0] + 4\n", + " ]\n", + " )\n", " elif diff_inserted_iter > comp_value_g and diff_inserted_iter < 0:\n", - " print('less diff now, insert one iteration')\n", - " value_insert = int((new_synced_iteration_vector[pulse_greater_than_virmen[0]]+new_synced_iteration_vector[pulse_greater_than_virmen[0]+1])/2)\n", - " new_synced_iteration_vector = np.insert(new_synced_iteration_vector, pulse_greater_than_virmen[0]+1, value_insert)\n", - "\n", - "\n", - " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n", - "\n", - " print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", - " print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", - "\n", - " '''\n", + " print(\"less diff now, insert one iteration\")\n", + " value_insert = int(\n", + " (\n", + " new_synced_iteration_vector[pulse_greater_than_virmen[0]]\n", + " + new_synced_iteration_vector[pulse_greater_than_virmen[0] + 1]\n", + " )\n", + " / 2\n", + " )\n", + " new_synced_iteration_vector = np.insert(\n", + " new_synced_iteration_vector,\n", + " pulse_greater_than_virmen[0] + 1,\n", + " value_insert,\n", + " )\n", + "\n", + " new_synced_time_vector = (\n", + " new_synced_iteration_vector - new_synced_iteration_vector[0]\n", + " ) / nidq_sampling_rate\n", + "\n", + " print(\n", + " new_synced_time_vector[\n", + " pulse_greater_than_virmen[0] - 2 : pulse_greater_than_virmen[0] + 4\n", + " ]\n", + " )\n", + " print(\n", + " behavior_time_vector[\n", + " pulse_greater_than_virmen[0] - 2 : pulse_greater_than_virmen[0] + 4\n", + " ]\n", + " )\n", + "\n", + " \"\"\"\n", " diff_vector = np.diff(new_synced_time_vector - behavior_time_vector[:new_synced_time_vector.shape[0]])\n", " #In case last peak is at the end of trial (append 0 to detect it)\n", " diff_vector = np.append(diff_vector, np.array([0]))\n", @@ -475,9 +572,7 @@ "\n", " #print(new_synced_iteration_vector[peaks[0]-2:peaks[0]+5])\n", " print(new_synced_time_vector[peaks[0]-2:peaks[0]+5])\n", - " '''\n", - " \n", - "\n" + " \"\"\"" ] }, { @@ -499,7 +594,7 @@ ], "source": [ "it_times = new_synced_time_vector\n", - "#it_times = synced_time_vector\n", + "# it_times = synced_time_vector\n", "beh_times = behavior_time_vector\n", "\n", "print(it_times.shape[0])\n", @@ -525,9 +620,9 @@ ], "source": [ "if it_times.shape[0] >= beh_times.shape[0]:\n", - " diff_vector = np.diff(it_times[:beh_times.shape[0]] - beh_times)\n", + " diff_vector = np.diff(it_times[: beh_times.shape[0]] - beh_times)\n", "else:\n", - " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n", + " diff_vector = np.diff(it_times - beh_times[: it_times.shape[0]])\n", "\n", "import scipy\n", "\n", @@ -553,13 +648,13 @@ } ], "source": [ - "#plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n", - "plt.plot(new_synced_time_vector - beh_times[:it_times.shape[0]])\n", - "#plt.plot((it_times - beh_times))\n", - "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", + "# plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n", + "plt.plot(new_synced_time_vector - beh_times[: it_times.shape[0]])\n", + "# plt.plot((it_times - beh_times))\n", + "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", "plt.plot(peaks, diff_vector[peaks], \"x\")\n", - "plt.ylabel('Time diff in current iteration TTL - Virmen (s)')\n", - "plt.xlabel('Iteration')\n", + "plt.ylabel(\"Time diff in current iteration TTL - Virmen (s)\")\n", + "plt.xlabel(\"Iteration\")\n", "plt.ylim([-0.05, 0.02])\n", "plt.show()" ] @@ -583,11 +678,11 @@ ], "source": [ "plt.plot(np.diff(new_synced_time_vector))\n", - "plt.plot(np.diff(beh_times)+0.01)\n", - "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", - "#plt.plot(peaks, diff_vector[peaks], \"x\")\n", + "plt.plot(np.diff(beh_times) + 0.01)\n", + "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", + "# plt.plot(peaks, diff_vector[peaks], \"x\")\n", "plt.ylim([-0.01, 0.05])\n", - "plt.xlabel('Iteration')\n" + "plt.xlabel(\"Iteration\")" ] }, { @@ -608,8 +703,10 @@ } ], "source": [ - "difo = np.pad(np.diff(new_synced_time_vector), (1, 0), 'constant', constant_values=(0,0))\n", - "difo2 = np.pad(np.diff(beh_times), (1, 0), 'constant', constant_values=(0,0))\n", + "difo = np.pad(\n", + " np.diff(new_synced_time_vector), (1, 0), \"constant\", constant_values=(0, 0)\n", + ")\n", + "difo2 = np.pad(np.diff(beh_times), (1, 0), \"constant\", constant_values=(0, 0))\n", "\n", "peaks_ttl_pulses, _ = scipy.signal.find_peaks(difo, height=0.02, distance=20)\n", "peaks_virmen_iter, _ = scipy.signal.find_peaks(difo2, height=0.02, distance=20)\n", @@ -679,8 +776,8 @@ } ], "source": [ - "print(new_synced_time_vector[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n", - "print(beh_times[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n" + "print(new_synced_time_vector[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])\n", + "print(beh_times[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])" ] }, { @@ -702,7 +799,7 @@ ], "source": [ "print(new_synced_time_vector[-4:])\n", - "print(beh_times[-4:])\n" + "print(beh_times[-4:])" ] }, { @@ -723,10 +820,9 @@ } ], "source": [ - "\n", "plt.plot(np.diff(new_synced_time_vector))\n", - "#plt.plot(np.diff(synced_time_vector))\n", - "plt.plot(np.diff(behavior_time_vector)+0.01)\n", + "# plt.plot(np.diff(synced_time_vector))\n", + "plt.plot(np.diff(behavior_time_vector) + 0.01)\n", "plt.ylim(0, 0.11)" ] }, @@ -777,8 +873,8 @@ ], "source": [ "trial_plot = 116\n", - "plt.plot(iteration_dict['iter_times_idx'][trial_plot])\n", - "plt.plot(behavior_time[trial_plot].flatten())\n" + "plt.plot(iteration_dict[\"iter_times_idx\"][trial_plot])\n", + "plt.plot(behavior_time[trial_plot].flatten())" ] }, { @@ -800,15 +896,24 @@ "samp_before = 1000\n", "samp_after = 1600\n", "if trial_plot == 0:\n", - " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot][0]-np.int64(samp_before)\n", + " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot][0] - np.int64(\n", + " samp_before\n", + " )\n", "else:\n", - " last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot-1][-1]-np.int64(samp_before)\n", - "first_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][0]+np.int64(samp_after)\n", - "second_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][1]\n", - "\n", - "samp_diff = iteration_dict['iter_start_idx'][trial_plot][0] - iteration_dict['iter_start_idx'][trial_plot-1][-1]\n", - "print('last_iter_trial0', last_iter_trial0)\n", - "print('first_iter_trial1', first_iter_trial1)" + " last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1] - np.int64(\n", + " samp_before\n", + " )\n", + "first_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][0] + np.int64(\n", + " samp_after\n", + ")\n", + "second_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][1]\n", + "\n", + "samp_diff = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1]\n", + ")\n", + "print(\"last_iter_trial0\", last_iter_trial0)\n", + "print(\"first_iter_trial1\", first_iter_trial1)" ] }, { @@ -851,55 +956,64 @@ } ], "source": [ - "\n", - "\n", "start_iter = 1\n", "end_iter = 1\n", "\n", "\n", - "sample_start =(iteration_dict['iter_start_idx'][trial_plot][start_iter]-np.int64(samp_before))\n", - "sample_end = (iteration_dict['iter_start_idx'][trial_plot][end_iter]+np.int64(samp_after))\n", + "sample_start = iteration_dict[\"iter_start_idx\"][trial_plot][start_iter] - np.int64(\n", + " samp_before\n", + ")\n", + "sample_end = iteration_dict[\"iter_start_idx\"][trial_plot][end_iter] + np.int64(\n", + " samp_after\n", + ")\n", "\n", - "iter_sample = digital_array[new_trial_channel,last_iter_trial0:first_iter_trial1]\n", + "iter_sample = digital_array[new_trial_channel, last_iter_trial0:first_iter_trial1]\n", "\n", - "time_vector = np.linspace(start=0, stop=iter_sample.shape[0]-1,num=iter_sample.shape[0])\n", - "time_vector = (time_vector*1000/nidq_sampling_rate)\n", - "time_vector -= ((iter_sample.shape[0]-samp_after)*1000)/nidq_sampling_rate\n", - "\n", - "time_last_trial = time_vector[0] +(samp_before*1000)/nidq_sampling_rate\n", + "time_vector = np.linspace(\n", + " start=0, stop=iter_sample.shape[0] - 1, num=iter_sample.shape[0]\n", + ")\n", + "time_vector = time_vector * 1000 / nidq_sampling_rate\n", + "time_vector -= ((iter_sample.shape[0] - samp_after) * 1000) / nidq_sampling_rate\n", "\n", + "time_last_trial = time_vector[0] + (samp_before * 1000) / nidq_sampling_rate\n", "\n", "\n", "print(iter_sample.shape[0])\n", - "samples_after_second_pulse = (second_iter_trial1-(first_iter_trial1-samp_after))\n", - "print('samples_after_second_pulse', samples_after_second_pulse)\n", - "print(iter_sample.shape[0]-samp_after)\n", + "samples_after_second_pulse = second_iter_trial1 - (first_iter_trial1 - samp_after)\n", + "print(\"samples_after_second_pulse\", samples_after_second_pulse)\n", + "print(iter_sample.shape[0] - samp_after)\n", "\n", "idx_time_zero = np.where(time_vector >= 0)\n", "idx_time_zero = idx_time_zero[0]\n", "idx_time_zero = idx_time_zero[0]\n", "\n", "\n", - "print('new_iteration_channel', new_iteration_channel)\n", - "print(digital_array[new_iteration_channel,:].shape)\n", - "\n", - "plt.plot(time_vector,iter_sample)\n", - "plt.plot(time_vector,digital_array[new_iteration_channel,last_iter_trial0:first_iter_trial1]+0.02)\n", - "plt.plot(0,1,\"x\")\n", - "if (idx_time_zero+samples_after_second_pulse) < time_vector.shape[0]:\n", - " plt.plot(time_vector[idx_time_zero+samples_after_second_pulse],1,\"x\")\n", - "plt.plot(time_last_trial,1,\"x\")\n", - "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", - "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", - "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "print(\"new_iteration_channel\", new_iteration_channel)\n", + "print(digital_array[new_iteration_channel, :].shape)\n", + "\n", + "plt.plot(time_vector, iter_sample)\n", + "plt.plot(\n", + " time_vector,\n", + " digital_array[new_iteration_channel, last_iter_trial0:first_iter_trial1] + 0.02,\n", + ")\n", + "plt.plot(0, 1, \"x\")\n", + "if (idx_time_zero + samples_after_second_pulse) < time_vector.shape[0]:\n", + " plt.plot(time_vector[idx_time_zero + samples_after_second_pulse], 1, \"x\")\n", + "plt.plot(time_last_trial, 1, \"x\")\n", + "\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", + "\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", + "\n", + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -925,20 +1039,37 @@ "start_iter = 2704\n", "samp_after = 600\n", "\n", - "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n", - "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n", - "\n", - "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", - "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", - "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "plt.plot(\n", + " digital_array[\n", + " 2,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n", + " + np.int64(samp_after),\n", + " ]\n", + ")\n", + "plt.plot(\n", + " digital_array[\n", + " 1,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n", + " + np.int64(samp_after),\n", + " ]\n", + ")\n", + "\n", + "\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", + "\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", + "\n", + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -957,8 +1088,8 @@ ], "source": [ "trial_plot = 280\n", - "print(iteration_dict['iter_start_idx'][trial_plot][2704])\n", - "print(iteration_dict['iter_start_idx'][trial_plot+1][0])" + "print(iteration_dict[\"iter_start_idx\"][trial_plot][2704])\n", + "print(iteration_dict[\"iter_start_idx\"][trial_plot + 1][0])" ] }, { @@ -990,7 +1121,7 @@ } ], "source": [ - "plt.plot(np.diff(iteration_dict['iter_start_idx'][trial_plot]/nidq_sampling_rate))" + "plt.plot(np.diff(iteration_dict[\"iter_start_idx\"][trial_plot] / nidq_sampling_rate))" ] }, { @@ -1070,14 +1201,14 @@ "source": [ "x = np.array([])\n", "mean_x = np.array([])\n", - "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n", - " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n", + "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n", + " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n", " mean_time_trial = np.mean(s)\n", - " x = np.append(x,s, axis=0)\n", + " x = np.append(x, s, axis=0)\n", " mean_x = np.append(mean_x, mean_time_trial)\n", "\n", - "#plt.plot(x)\n", - "plt.plot(mean_x)\n" + "# plt.plot(x)\n", + "plt.plot(mean_x)" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb index 8f2853a1..b1582c65 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/check_sync_notebook-new_method2.ipynb @@ -50,7 +50,7 @@ "\n", "import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX\n", "import u19_pipeline.utils.ephys_utils as ephys_utils\n", - "from u19_pipeline.ephys_pipeline import get_ephys_root_data_dir, get_session_directory\n" + "from u19_pipeline.ephys_pipeline import get_ephys_root_data_dir, get_session_directory" ] }, { @@ -59,8 +59,8 @@ "metadata": {}, "outputs": [], "source": [ - "#Behavior data\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "# Behavior data\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -69,35 +69,48 @@ "metadata": {}, "outputs": [], "source": [ - "def fix_trial_times(synced_iteration_vector, synced_time_vector, behavior_time_vector, nidq_sampling_rate):\n", - "# Check where is more likely we miss an iteration pulse and insert it to iteration_vector\n", + "def fix_trial_times(\n", + " synced_iteration_vector,\n", + " synced_time_vector,\n", + " behavior_time_vector,\n", + " nidq_sampling_rate,\n", + "):\n", + " # Check where is more likely we miss an iteration pulse and insert it to iteration_vector\n", "\n", " new_synced_iteration_vector = synced_iteration_vector.copy()\n", " new_synced_time_vector = synced_time_vector.copy()\n", "\n", " diff_time_test = new_synced_time_vector[2:50] - behavior_time_vector[1:49]\n", " pulse_greater_than_virmen = np.where(diff_time_test > 0)\n", - " pulse_greater_than_virmen = pulse_greater_than_virmen[0]+0\n", + " pulse_greater_than_virmen = pulse_greater_than_virmen[0] + 0\n", " if pulse_greater_than_virmen.shape[0] == 0:\n", - " #print('we have a special case of extra pulse beginning')\n", + " # print('we have a special case of extra pulse beginning')\n", " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, 1)\n", - " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n", + " new_synced_time_vector = (\n", + " new_synced_iteration_vector - new_synced_iteration_vector[0]\n", + " ) / nidq_sampling_rate\n", "\n", " for i in range(100):\n", - " diff_time = new_synced_time_vector[0:] - behavior_time_vector[0:new_synced_time_vector.shape[0]]\n", + " diff_time = (\n", + " new_synced_time_vector[0:]\n", + " - behavior_time_vector[0 : new_synced_time_vector.shape[0]]\n", + " )\n", " pulse_greater_than_virmen = np.where(diff_time > 0)\n", - " pulse_greater_than_virmen = pulse_greater_than_virmen[0]+0\n", + " pulse_greater_than_virmen = pulse_greater_than_virmen[0] + 0\n", "\n", " if pulse_greater_than_virmen.shape[0] == 0:\n", " break\n", "\n", - " #print(i)\n", - " #print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+3])\n", - " #print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+3])\n", - " #print(pulse_greater_than_virmen[0])\n", + " # print(i)\n", + " # print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+3])\n", + " # print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+3])\n", + " # print(pulse_greater_than_virmen[0])\n", "\n", " if pulse_greater_than_virmen[0] > 3:\n", - " median_diff = np.median(new_synced_time_vector[0:pulse_greater_than_virmen[0]-3]-behavior_time_vector[0:pulse_greater_than_virmen[0]-3])\n", + " median_diff = np.median(\n", + " new_synced_time_vector[0 : pulse_greater_than_virmen[0] - 3]\n", + " - behavior_time_vector[0 : pulse_greater_than_virmen[0] - 3]\n", + " )\n", " else:\n", " median_diff = -0.015\n", "\n", @@ -105,69 +118,111 @@ " comp_value_l = -0.015\n", " comp_value_g = -0.005\n", " else:\n", - " comp_value_l = median_diff*1.3\n", - " comp_value_g = median_diff*0.7\n", - "\n", - " diff_time_iter_missing_pulse = behavior_time_vector[pulse_greater_than_virmen[0]] - behavior_time_vector[pulse_greater_than_virmen[0]-1]\n", - " #print('missing time in iter pulses', behavior_time_vector[pulse_greater_than_virmen[0]])\n", - " #print('time_next_missing_pulse', diff_time_iter_missing_pulse)\n", - " diff_iter_add = np.floor(diff_time_iter_missing_pulse*nidq_sampling_rate)\n", - " value_insert_iteration = new_synced_iteration_vector[pulse_greater_than_virmen[0]-1] + diff_iter_add\n", - "\n", - " #print('value_insert_iteration', value_insert_iteration)\n", - "\n", - " new_synced_iteration_vector = np.insert(new_synced_iteration_vector, pulse_greater_than_virmen[0], value_insert_iteration)\n", + " comp_value_l = median_diff * 1.3\n", + " comp_value_g = median_diff * 0.7\n", + "\n", + " diff_time_iter_missing_pulse = (\n", + " behavior_time_vector[pulse_greater_than_virmen[0]]\n", + " - behavior_time_vector[pulse_greater_than_virmen[0] - 1]\n", + " )\n", + " # print('missing time in iter pulses', behavior_time_vector[pulse_greater_than_virmen[0]])\n", + " # print('time_next_missing_pulse', diff_time_iter_missing_pulse)\n", + " diff_iter_add = np.floor(diff_time_iter_missing_pulse * nidq_sampling_rate)\n", + " value_insert_iteration = (\n", + " new_synced_iteration_vector[pulse_greater_than_virmen[0] - 1]\n", + " + diff_iter_add\n", + " )\n", + "\n", + " # print('value_insert_iteration', value_insert_iteration)\n", + "\n", + " new_synced_iteration_vector = np.insert(\n", + " new_synced_iteration_vector,\n", + " pulse_greater_than_virmen[0],\n", + " value_insert_iteration,\n", + " )\n", "\n", " if new_synced_iteration_vector.shape[0] > behavior_time_vector.shape[0]:\n", - " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, pulse_greater_than_virmen[0]+1)\n", - " #new_synced_iteration_vector = new_synced_iteration_vector[:-1]\n", + " new_synced_iteration_vector = np.delete(\n", + " new_synced_iteration_vector, pulse_greater_than_virmen[0] + 1\n", + " )\n", + " # new_synced_iteration_vector = new_synced_iteration_vector[:-1]\n", "\n", - " if pulse_greater_than_virmen[0] < (new_synced_time_vector.shape[0]-1):\n", + " if pulse_greater_than_virmen[0] < (new_synced_time_vector.shape[0] - 1):\n", " idx = pulse_greater_than_virmen[0]\n", - " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n", - " if idx+10 > (new_synced_time_vector.shape[0]-1):\n", - " idx_final = new_synced_time_vector.shape[0]-1\n", - " diff_vectorin = (new_synced_time_vector[idx+1:idx_final]- behavior_time_vector[idx+1:idx_final])\n", + " new_synced_time_vector = (\n", + " new_synced_iteration_vector - new_synced_iteration_vector[0]\n", + " ) / nidq_sampling_rate\n", + " if idx + 10 > (new_synced_time_vector.shape[0] - 1):\n", + " idx_final = new_synced_time_vector.shape[0] - 1\n", + " diff_vectorin = (\n", + " new_synced_time_vector[idx + 1 : idx_final]\n", + " - behavior_time_vector[idx + 1 : idx_final]\n", + " )\n", " else:\n", - " diff_vectorin = (new_synced_time_vector[idx+1:idx+10]- behavior_time_vector[idx+1:idx+10])\n", - " #print(diff_vectorin)\n", + " diff_vectorin = (\n", + " new_synced_time_vector[idx + 1 : idx + 10]\n", + " - behavior_time_vector[idx + 1 : idx + 10]\n", + " )\n", + " # print(diff_vectorin)\n", " diff_inserted_iter = np.median(diff_vectorin)\n", - " #print('diff_in new inserted iter', diff_inserted_iter)\n", + " # print('diff_in new inserted iter', diff_inserted_iter)\n", " if diff_inserted_iter < comp_value_l:\n", - " #print('extra diff !!!!')\n", - " diff_behavior = behavior_time_vector[idx+1] - behavior_time_vector[idx]\n", - " idx_next_iter = np.where(new_synced_time_vector > behavior_time_vector[idx+1])\n", - " #print('diff_behavior', diff_behavior)\n", + " # print('extra diff !!!!')\n", + " diff_behavior = (\n", + " behavior_time_vector[idx + 1] - behavior_time_vector[idx]\n", + " )\n", + " idx_next_iter = np.where(\n", + " new_synced_time_vector > behavior_time_vector[idx + 1]\n", + " )\n", + " # print('diff_behavior', diff_behavior)\n", " idx_next_iter = idx_next_iter[0][0]\n", - " #print('idx_next_iter', idx_next_iter)\n", - " #print('idx+1', idx+1)\n", - " if diff_behavior > 0.1 and idx_next_iter > idx+3:\n", - " #print('bajale a indx_next_iter', idx_next_iter)\n", - " idx_next_iter = idx+3\n", - " if idx_next_iter > idx+2:\n", - " #print('more than one to remove')\n", - " int_array_step = np.arange(idx+1, idx_next_iter-1, dtype=np.int64)\n", - " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, int_array_step)\n", + " # print('idx_next_iter', idx_next_iter)\n", + " # print('idx+1', idx+1)\n", + " if diff_behavior > 0.1 and idx_next_iter > idx + 3:\n", + " # print('bajale a indx_next_iter', idx_next_iter)\n", + " idx_next_iter = idx + 3\n", + " if idx_next_iter > idx + 2:\n", + " # print('more than one to remove')\n", + " int_array_step = np.arange(\n", + " idx + 1, idx_next_iter - 1, dtype=np.int64\n", + " )\n", + " new_synced_iteration_vector = np.delete(\n", + " new_synced_iteration_vector, int_array_step\n", + " )\n", " else:\n", - " new_synced_iteration_vector = np.delete(new_synced_iteration_vector, [idx+1])\n", - " #print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", + " new_synced_iteration_vector = np.delete(\n", + " new_synced_iteration_vector, [idx + 1]\n", + " )\n", + " # print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", " elif diff_inserted_iter > comp_value_g and diff_inserted_iter < 0:\n", " if new_synced_iteration_vector.shape[0] < behavior_time_vector.shape[0]:\n", - " #print('less diff now, insert one iteration')\n", - " #print('diff_inserted_iter', diff_inserted_iter)\n", - " #print('median_diff', median_diff, 'comp_value_g', comp_value_g)\n", - " #print('diff_vectorin', diff_vectorin)\n", - " value_insert = int((new_synced_iteration_vector[pulse_greater_than_virmen[0]]+new_synced_iteration_vector[pulse_greater_than_virmen[0]+1])/2) \n", - " new_synced_iteration_vector = np.insert(new_synced_iteration_vector, pulse_greater_than_virmen[0]+1, value_insert)\n", - "\n", - " \n", - " new_synced_time_vector = (new_synced_iteration_vector-new_synced_iteration_vector[0])/nidq_sampling_rate\n", - "\n", - " #print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", - " #print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", - "\n", - " return new_synced_iteration_vector, new_synced_time_vector\n", - "\n" + " # print('less diff now, insert one iteration')\n", + " # print('diff_inserted_iter', diff_inserted_iter)\n", + " # print('median_diff', median_diff, 'comp_value_g', comp_value_g)\n", + " # print('diff_vectorin', diff_vectorin)\n", + " value_insert = int(\n", + " (\n", + " new_synced_iteration_vector[pulse_greater_than_virmen[0]]\n", + " + new_synced_iteration_vector[\n", + " pulse_greater_than_virmen[0] + 1\n", + " ]\n", + " )\n", + " / 2\n", + " )\n", + " new_synced_iteration_vector = np.insert(\n", + " new_synced_iteration_vector,\n", + " pulse_greater_than_virmen[0] + 1,\n", + " value_insert,\n", + " )\n", + "\n", + " new_synced_time_vector = (\n", + " new_synced_iteration_vector - new_synced_iteration_vector[0]\n", + " ) / nidq_sampling_rate\n", + "\n", + " # print(new_synced_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", + " # print(behavior_time_vector[pulse_greater_than_virmen[0]-2:pulse_greater_than_virmen[0]+4])\n", + "\n", + " return new_synced_iteration_vector, new_synced_time_vector" ] }, { @@ -185,15 +240,20 @@ " max_diff = max(diff_vector)\n", " median_general = np.median(diff_vector)\n", "\n", - " num_div= 10\n", + " num_div = 10\n", " median_diff_percent = np.empty([num_div])\n", " median_diff_abs = np.empty([num_div])\n", " for j in range(num_div):\n", - " start_iter = int(j*num_iter/num_div)\n", - " end_iter = int((j+1)*num_iter/num_div)\n", - " median_diff_percent[j] = (np.median(diff_vector[start_iter:end_iter])-median_general)*100/median_general\n", - " median_diff_abs[j] = (np.median(diff_vector[start_iter:end_iter])-median_general)\n", - "\n", + " start_iter = int(j * num_iter / num_div)\n", + " end_iter = int((j + 1) * num_iter / num_div)\n", + " median_diff_percent[j] = (\n", + " (np.median(diff_vector[start_iter:end_iter]) - median_general)\n", + " * 100\n", + " / median_general\n", + " )\n", + " median_diff_abs[j] = (\n", + " np.median(diff_vector[start_iter:end_iter]) - median_general\n", + " )\n", "\n", " if max_diff <= 0 and np.max(np.abs(median_diff_abs)) < 0.005:\n", " pass\n", @@ -202,9 +262,7 @@ " print(median_general)\n", " plt.plot(median_diff_abs)\n", "\n", - " return status\n", - " \n", - "\n" + " return status" ] }, { @@ -232,12 +290,16 @@ ], "source": [ "recording_query = \"recording_id >= 500\"\n", - "recording = dj.create_virtual_module('recording', 'u19_recording')\n", - "recording_keys = (recording.Recording & recording_query).fetch('recording_id', as_dict=True, order_by='recording_id')\n", + "recording = dj.create_virtual_module(\"recording\", \"u19_recording\")\n", + "recording_keys = (recording.Recording & recording_query).fetch(\n", + " \"recording_id\", as_dict=True, order_by=\"recording_id\"\n", + ")\n", "\n", "\n", - "session_fields = ['subject_fullname', 'session_date', 'session_number']\n", - "session_keys = (recording.Recording.BehaviorSession & recording_query).fetch(*session_fields, as_dict=True, order_by='recording_id')\n", + "session_fields = [\"subject_fullname\", \"session_date\", \"session_number\"]\n", + "session_keys = (recording.Recording.BehaviorSession & recording_query).fetch(\n", + " *session_fields, as_dict=True, order_by=\"recording_id\"\n", + ")\n", "\n", "\n", "recording_keys" @@ -302,38 +364,42 @@ " recording_key = recording_keys[idx_session]\n", " session_key = session_keys[idx_session]\n", "\n", - " session_dir = find_full_path(get_ephys_root_data_dir(),\n", - " get_session_directory(recording_key))\n", + " session_dir = find_full_path(\n", + " get_ephys_root_data_dir(), get_session_directory(recording_key)\n", + " )\n", " print(session_dir)\n", - " #session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", - " #Check if session is Nidq or OneBox\n", - " nidq_session = list(session_dir.glob('*nidq.bin*'))\n", - " obx_session = list(session_dir.glob('*obx.bin*'))\n", + " # session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", + " # Check if session is Nidq or OneBox\n", + " nidq_session = list(session_dir.glob(\"*nidq.bin*\"))\n", + " obx_session = list(session_dir.glob(\"*obx.bin*\"))\n", "\n", " if len(nidq_session) == 0 and len(obx_session) == 0:\n", - " print('No session found')\n", + " print(\"No session found\")\n", " elif len(nidq_session) > 0:\n", " ephys_session_fullpath = nidq_session[0]\n", " else:\n", " ephys_session_fullpath = obx_session[0]\n", "\n", - " #Nidaq file\n", - " nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n", + " # Nidaq file\n", + " nidq_meta = readSGLX.readMeta(ephys_session_fullpath)\n", " nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n", "\n", - "\n", - " # 1: load meta data, and the content of the NIDAQ file. Its content is digital. \n", + " # 1: load meta data, and the content of the NIDAQ file. Its content is digital.\n", " new_trial_channel = 1\n", " new_iteration_channel = 2\n", " # If PXIe card (nidq) card use for recording deduce digital channels\n", - " if nidq_meta['typeThis'] == 'nidq':\n", - " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta)\n", + " if nidq_meta[\"typeThis\"] == \"nidq\":\n", + " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " ephys_session_fullpath, nidq_meta\n", + " )\n", " # If onebox card (obx) card use for recording digital channels are 0-2\n", " else:\n", - " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1])\n", + " digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " ephys_session_fullpath, nidq_meta, d_line_list=[0, 1]\n", + " )\n", " # If no sync pulse found trial and iteration signals are 0 & 1 respectively\n", - " channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0]\n", - " channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0]\n", + " channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0]\n", + " channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0]\n", "\n", " if channel0_pulses > channel1_pulses:\n", " new_trial_channel = 1\n", @@ -342,47 +408,65 @@ " new_trial_channel = 0\n", " new_iteration_channel = 1\n", "\n", - " \n", " thissession = behavior.TowersBlock().Trial() & session_key\n", - " behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n", - "\n", - " mode = None #Default for sessions before 12/01/2021\n", - " #mode = 'pulses' #Default for sessions after 12/01/2021\n", - " iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[new_trial_channel,:], digital_array[new_iteration_channel,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n", + " behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n", + " \"trial_time\", \"vi_start\", \"iterations\"\n", + " )\n", + "\n", + " mode = None # Default for sessions before 12/01/2021\n", + " # mode = 'pulses' #Default for sessions after 12/01/2021\n", + " iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n", + " digital_array[new_trial_channel, :],\n", + " digital_array[new_iteration_channel, :],\n", + " nidq_sampling_rate,\n", + " behavior_time.shape[0],\n", + " behavior_time,\n", + " mode=mode,\n", + " )\n", " # get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n", "\n", - "\n", - " for i in range(len(iteration_dict['iter_start_idx'])):\n", - " #print('fixing trial ',i)\n", - " iteration_dict['iter_start_idx'][i],iteration_dict['iter_times_idx'][i] =\\\n", - " fix_trial_times(iteration_dict['iter_start_idx'][i], iteration_dict['iter_times_idx'][i], behavior_time[i].flatten(), nidq_sampling_rate)\n", - "\n", + " for i in range(len(iteration_dict[\"iter_start_idx\"])):\n", + " # print('fixing trial ',i)\n", + " iteration_dict[\"iter_start_idx\"][i], iteration_dict[\"iter_times_idx\"][i] = (\n", + " fix_trial_times(\n", + " iteration_dict[\"iter_start_idx\"][i],\n", + " iteration_dict[\"iter_times_idx\"][i],\n", + " behavior_time[i].flatten(),\n", + " nidq_sampling_rate,\n", + " )\n", + " )\n", "\n", " # Check # of trials and iterations match\n", - " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n", - "\n", + " trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = (\n", + " ephys_utils.assert_iteration_samples_count(\n", + " iteration_dict[\"iter_start_idx\"], behavior_time\n", + " )\n", + " )\n", "\n", " print(trial_count_diff)\n", " print(trials_diff_iteration_big)\n", " print(trials_diff_iteration_small)\n", "\n", + " status = ephys_utils.evaluate_sync_process(\n", + " trial_count_diff,\n", + " trials_diff_iteration_big,\n", + " trials_diff_iteration_small,\n", + " behavior_time.shape[0],\n", + " )\n", "\n", - " status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0])\n", - "\n", - " for i in range(len(iteration_dict['iter_start_idx'])):\n", - " synced_time_vector = iteration_dict['iter_times_idx'][i]\n", + " for i in range(len(iteration_dict[\"iter_start_idx\"])):\n", + " synced_time_vector = iteration_dict[\"iter_times_idx\"][i]\n", " behavior_time_vector = behavior_time[i].flatten()\n", " status = sync_evaluation_process2(synced_time_vector, behavior_time_vector)\n", " if status == 0:\n", " print(session_key)\n", " print(recording_key)\n", " print(i)\n", - " if i!=0:\n", + " if i != 0:\n", " break\n", - " \n", + "\n", " if status == 0:\n", - " break\n", - " \n" + " break" ] }, { @@ -425,9 +509,9 @@ ], "source": [ "if it_times.shape[0] >= beh_times.shape[0]:\n", - " diff_vector = np.diff(it_times[:beh_times.shape[0]] - beh_times)\n", + " diff_vector = np.diff(it_times[: beh_times.shape[0]] - beh_times)\n", "else:\n", - " diff_vector = np.diff(it_times - beh_times[:it_times.shape[0]])\n", + " diff_vector = np.diff(it_times - beh_times[: it_times.shape[0]])\n", "\n", "import scipy\n", "\n", @@ -452,12 +536,12 @@ } ], "source": [ - "#plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n", - "plt.plot(it_times - beh_times[:it_times.shape[0]])\n", - "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", + "# plt.plot(np.diff(it_times - beh_times[:it_times.shape[0]]))\n", + "plt.plot(it_times - beh_times[: it_times.shape[0]])\n", + "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", "plt.plot(peaks, diff_vector[peaks], \"x\")\n", - "plt.ylabel('Time diff in current iteration TTL - Virmen (s)')\n", - "plt.xlabel('Iteration')\n", + "plt.ylabel(\"Time diff in current iteration TTL - Virmen (s)\")\n", + "plt.xlabel(\"Iteration\")\n", "plt.ylim([-0.05, 0.05])\n", "plt.show()" ] @@ -490,11 +574,11 @@ ], "source": [ "plt.plot(np.diff(synced_time_vector))\n", - "plt.plot(np.diff(beh_times)+0.01)\n", - "#plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", - "#plt.plot(peaks, diff_vector[peaks], \"x\")\n", + "plt.plot(np.diff(beh_times) + 0.01)\n", + "# plt.plot((np.diff(beh_times[:it_times.shape[0]])))\n", + "# plt.plot(peaks, diff_vector[peaks], \"x\")\n", "plt.ylim([-0.01, 0.05])\n", - "plt.xlabel('Iteration')\n" + "plt.xlabel(\"Iteration\")" ] }, { @@ -512,8 +596,8 @@ } ], "source": [ - "difo = np.pad(np.diff(synced_time_vector), (1, 0), 'constant', constant_values=(0,0))\n", - "difo2 = np.pad(np.diff(beh_times), (1, 0), 'constant', constant_values=(0,0))\n", + "difo = np.pad(np.diff(synced_time_vector), (1, 0), \"constant\", constant_values=(0, 0))\n", + "difo2 = np.pad(np.diff(beh_times), (1, 0), \"constant\", constant_values=(0, 0))\n", "\n", "peaks_ttl_pulses, _ = scipy.signal.find_peaks(difo, height=0.02, distance=20)\n", "peaks_virmen_iter, _ = scipy.signal.find_peaks(difo2, height=0.02, distance=20)\n", @@ -558,8 +642,8 @@ } ], "source": [ - "print(new_synced_time_vector[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n", - "print(beh_times[peaks_ttl_pulses[0]-4:peaks_ttl_pulses[0]+4])\n" + "print(new_synced_time_vector[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])\n", + "print(beh_times[peaks_ttl_pulses[0] - 4 : peaks_ttl_pulses[0] + 4])" ] }, { @@ -578,7 +662,7 @@ ], "source": [ "print(new_synced_time_vector[-4:])\n", - "print(beh_times[-4:])\n" + "print(beh_times[-4:])" ] }, { @@ -608,10 +692,9 @@ } ], "source": [ - "\n", "plt.plot(np.diff(new_synced_time_vector))\n", - "#plt.plot(np.diff(synced_time_vector))\n", - "plt.plot(np.diff(behavior_time_vector)+0.01)\n", + "# plt.plot(np.diff(synced_time_vector))\n", + "plt.plot(np.diff(behavior_time_vector) + 0.01)\n", "plt.ylim(0, 0.11)" ] }, @@ -662,8 +745,8 @@ ], "source": [ "trial_plot = 99\n", - "plt.plot(iteration_dict['iter_times_idx'][trial_plot])\n", - "plt.plot(behavior_time[trial_plot].flatten())\n" + "plt.plot(iteration_dict[\"iter_times_idx\"][trial_plot])\n", + "plt.plot(behavior_time[trial_plot].flatten())" ] }, { @@ -686,11 +769,18 @@ "trial_plot = 117\n", "samp_before = 800\n", "samp_after = 800\n", - "last_iter_trial0 = iteration_dict['iter_start_idx'][trial_plot-1][-1]-np.int64(samp_before)\n", - "first_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][0]+np.int64(samp_after)\n", - "second_iter_trial1 = iteration_dict['iter_start_idx'][trial_plot][0]\n", - "\n", - "samp_diff = iteration_dict['iter_start_idx'][trial_plot][0] - iteration_dict['iter_start_idx'][trial_plot-1][-1]\n", + "last_iter_trial0 = iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1] - np.int64(\n", + " samp_before\n", + ")\n", + "first_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][0] + np.int64(\n", + " samp_after\n", + ")\n", + "second_iter_trial1 = iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + "\n", + "samp_diff = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot - 1][-1]\n", + ")\n", "samp_diff" ] }, @@ -732,53 +822,60 @@ } ], "source": [ - "\n", - "\n", "start_iter = 1\n", "end_iter = 1\n", "\n", "\n", - "sample_start =(iteration_dict['iter_start_idx'][trial_plot][start_iter]-np.int64(samp_before))\n", - "sample_end = (iteration_dict['iter_start_idx'][trial_plot][end_iter]+np.int64(samp_after))\n", + "sample_start = iteration_dict[\"iter_start_idx\"][trial_plot][start_iter] - np.int64(\n", + " samp_before\n", + ")\n", + "sample_end = iteration_dict[\"iter_start_idx\"][trial_plot][end_iter] + np.int64(\n", + " samp_after\n", + ")\n", "\n", - "iter_sample = digital_array[new_trial_channel,last_iter_trial0:first_iter_trial1]\n", + "iter_sample = digital_array[new_trial_channel, last_iter_trial0:first_iter_trial1]\n", "\n", - "time_vector = np.linspace(start=0, stop=iter_sample.shape[0]-1,num=iter_sample.shape[0])\n", - "time_vector = (time_vector*1000/nidq_sampling_rate)\n", - "time_vector -= ((iter_sample.shape[0]-samp_after)*1000)/nidq_sampling_rate\n", - "\n", - "time_last_trial = time_vector[0] +(samp_before*1000)/nidq_sampling_rate\n", + "time_vector = np.linspace(\n", + " start=0, stop=iter_sample.shape[0] - 1, num=iter_sample.shape[0]\n", + ")\n", + "time_vector = time_vector * 1000 / nidq_sampling_rate\n", + "time_vector -= ((iter_sample.shape[0] - samp_after) * 1000) / nidq_sampling_rate\n", "\n", + "time_last_trial = time_vector[0] + (samp_before * 1000) / nidq_sampling_rate\n", "\n", "\n", "print(iter_sample.shape[0])\n", - "samples_after_second_pulse = (second_iter_trial1-(first_iter_trial1-samp_after))\n", - "print('samples_after_second_pulse', samples_after_second_pulse)\n", - "print(iter_sample.shape[0]-samp_after)\n", + "samples_after_second_pulse = second_iter_trial1 - (first_iter_trial1 - samp_after)\n", + "print(\"samples_after_second_pulse\", samples_after_second_pulse)\n", + "print(iter_sample.shape[0] - samp_after)\n", "\n", "idx_time_zero = np.where(time_vector >= 0)\n", "idx_time_zero = idx_time_zero[0]\n", "idx_time_zero = idx_time_zero[0]\n", "\n", "\n", - "\n", - "\n", - "plt.plot(time_vector,iter_sample)\n", - "plt.plot(time_vector,digital_array[new_iteration_channel,last_iter_trial0:first_iter_trial1]+0.02)\n", - "plt.plot(0,1,\"x\")\n", - "plt.plot(time_vector[idx_time_zero+samples_after_second_pulse],1,\"x\")\n", - "plt.plot(time_last_trial,1,\"x\")\n", - "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", - "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", - "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "plt.plot(time_vector, iter_sample)\n", + "plt.plot(\n", + " time_vector,\n", + " digital_array[new_iteration_channel, last_iter_trial0:first_iter_trial1] + 0.02,\n", + ")\n", + "plt.plot(0, 1, \"x\")\n", + "plt.plot(time_vector[idx_time_zero + samples_after_second_pulse], 1, \"x\")\n", + "plt.plot(time_last_trial, 1, \"x\")\n", + "\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", + "\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", + "\n", + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -804,20 +901,37 @@ "start_iter = 2704\n", "samp_after = 600\n", "\n", - "plt.plot(digital_array[2,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n", - "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][start_iter]-samp_before:iteration_dict['iter_start_idx'][trial_plot+1][0]+np.int64(samp_after)])\n", - "\n", - "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", - "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", - "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "plt.plot(\n", + " digital_array[\n", + " 2,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n", + " + np.int64(samp_after),\n", + " ]\n", + ")\n", + "plt.plot(\n", + " digital_array[\n", + " 1,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][start_iter]\n", + " - samp_before : iteration_dict[\"iter_start_idx\"][trial_plot + 1][0]\n", + " + np.int64(samp_after),\n", + " ]\n", + ")\n", + "\n", + "\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", + "\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", + "\n", + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -836,8 +950,8 @@ ], "source": [ "trial_plot = 280\n", - "print(iteration_dict['iter_start_idx'][trial_plot][2704])\n", - "print(iteration_dict['iter_start_idx'][trial_plot+1][0])" + "print(iteration_dict[\"iter_start_idx\"][trial_plot][2704])\n", + "print(iteration_dict[\"iter_start_idx\"][trial_plot + 1][0])" ] }, { @@ -869,7 +983,7 @@ } ], "source": [ - "plt.plot(np.diff(iteration_dict['iter_start_idx'][trial_plot]/nidq_sampling_rate))" + "plt.plot(np.diff(iteration_dict[\"iter_start_idx\"][trial_plot] / nidq_sampling_rate))" ] }, { @@ -949,14 +1063,14 @@ "source": [ "x = np.array([])\n", "mean_x = np.array([])\n", - "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n", - " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n", + "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n", + " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n", " mean_time_trial = np.mean(s)\n", - " x = np.append(x,s, axis=0)\n", + " x = np.append(x, s, axis=0)\n", " mean_x = np.append(mean_x, mean_time_trial)\n", "\n", - "#plt.plot(x)\n", - "plt.plot(mean_x)\n" + "# plt.plot(x)\n", + "plt.plot(mean_x)" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb index 9a5ebbdc..452c07e9 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/populate_sync_ephys_old_pipeline.ipynb @@ -42,7 +42,7 @@ "source": [ "import pandas as pd\n", "\n", - "#from u19_pipeline import ephys_pipeline as ephys\n", + "# from u19_pipeline import ephys_pipeline as ephys\n", "from u19_pipeline import ephys" ] }, @@ -91,21 +91,34 @@ "source": [ "key = \"subject_fullname like 'sbolkan%'\"\n", "\n", - "#keys_session = pd.DataFrame((ephys.EphysPipelineSession * recording.Recording.BehaviorSession & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n", - "keys_session = pd.DataFrame((ephys.EphysSession & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n", + "# keys_session = pd.DataFrame((ephys.EphysPipelineSession * recording.Recording.BehaviorSession & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n", + "keys_session = pd.DataFrame(\n", + " (ephys.EphysSession & key).fetch(\"KEY\", as_dict=True, order_by=\"session_date DESC\")\n", + ")\n", "\n", - "#already_synced = pd.DataFrame((ephys.EphysPipelineSession * recording.Recording.BehaviorSession * ephys.BehaviorSync & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n", - "already_synced = pd.DataFrame((ephys.EphysSession * ephys.BehaviorSync & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n", + "# already_synced = pd.DataFrame((ephys.EphysPipelineSession * recording.Recording.BehaviorSession * ephys.BehaviorSync & key).fetch('KEY', as_dict=True, order_by='session_date DESC'))\n", + "already_synced = pd.DataFrame(\n", + " (ephys.EphysSession * ephys.BehaviorSync & key).fetch(\n", + " \"KEY\", as_dict=True, order_by=\"session_date DESC\"\n", + " )\n", + ")\n", "\n", "\n", - "keys_session_missing = keys_session.merge(already_synced, how='left', on=['subject_fullname', 'session_date', 'session_number'], indicator=True)\n", - "#keys_session_missing = keys_session.merge(already_synced, how='left', on=['recording_id'], indicator=True)\n", + "keys_session_missing = keys_session.merge(\n", + " already_synced,\n", + " how=\"left\",\n", + " on=[\"subject_fullname\", \"session_date\", \"session_number\"],\n", + " indicator=True,\n", + ")\n", + "# keys_session_missing = keys_session.merge(already_synced, how='left', on=['recording_id'], indicator=True)\n", "\n", "\n", - "keys_session_missing = keys_session_missing.loc[keys_session_missing['_merge'] == 'left_only', :]\n", + "keys_session_missing = keys_session_missing.loc[\n", + " keys_session_missing[\"_merge\"] == \"left_only\", :\n", + "]\n", "keys_session_missing = keys_session_missing.reset_index(drop=True)\n", - "keys_session_missing = keys_session_missing.drop(['_merge'], axis=1)\n", - "keys_session_missing = keys_session_missing.to_dict(orient='records')\n", + "keys_session_missing = keys_session_missing.drop([\"_merge\"], axis=1)\n", + "keys_session_missing = keys_session_missing.to_dict(orient=\"records\")\n", "keys_session_missing" ] }, @@ -1301,8 +1314,8 @@ ], "source": [ "key = dict()\n", - "key['session_date'] = '2022-08-18'\n", - "key['subject_fullname'] = 'sbolkan_a2a_487' \n", + "key[\"session_date\"] = \"2022-08-18\"\n", + "key[\"subject_fullname\"] = \"sbolkan_a2a_487\"\n", "\n", "ephys.BehaviorSync.populate(key)" ] @@ -1475,18 +1488,22 @@ "source": [ "session_dir = pathlib.Path(get_session_directory(key))\n", "print(session_dir)\n", - "#session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", - "nidq_bin_full_path = list(session_dir.glob('*nidq.bin*'))[0]\n", + "# session_dir = pathlib.Path('/Users/alvaros/Documents/MATLAB/BrainCogsProjects/CalciumImagingData/test_g0/')\n", + "nidq_bin_full_path = list(session_dir.glob(\"*nidq.bin*\"))[0]\n", "\n", - "#Nidaq file\n", - "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n", + "# Nidaq file\n", + "nidq_meta = readSGLX.readMeta(nidq_bin_full_path)\n", "nidq_sampling_rate = readSGLX.SampRate(nidq_meta)\n", - "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta)\n", + "digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(\n", + " nidq_bin_full_path, nidq_meta\n", + ")\n", "\n", - "#Behavior data\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n", + "# Behavior data\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", "thissession = behavior.TowersBlock().Trial() & key\n", - "behavior_time, iterstart, beh_num_iterations = thissession.fetch('trial_time', 'vi_start', 'iterations')\n" + "behavior_time, iterstart, beh_num_iterations = thissession.fetch(\n", + " \"trial_time\", \"vi_start\", \"iterations\"\n", + ")" ] }, { @@ -1518,7 +1535,7 @@ } ], "source": [ - "plt.plot(digital_array[1,1292075-10000:1292075+1000])" + "plt.plot(digital_array[1, 1292075 - 10000 : 1292075 + 1000])" ] }, { @@ -1550,8 +1567,8 @@ } ], "source": [ - "plt.plot(digital_array[1,1292075-1000:1292075+1000])\n", - "plt.plot(digital_array[2,1292075-1000:1292075+1000])" + "plt.plot(digital_array[1, 1292075 - 1000 : 1292075 + 1000])\n", + "plt.plot(digital_array[2, 1292075 - 1000 : 1292075 + 1000])" ] }, { @@ -16175,15 +16192,24 @@ } ], "source": [ - "mode = 'counter_bit0' #Default for sessions before 12/01/2021\n", - "#mode = 'pulses' #Default for sessions after 12/01/2021\n", - "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[1,:], digital_array[2,:], nidq_sampling_rate, behavior_time.shape[0], behavior_time, mode=mode)\n", + "mode = \"counter_bit0\" # Default for sessions before 12/01/2021\n", + "# mode = 'pulses' #Default for sessions after 12/01/2021\n", + "iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(\n", + " digital_array[1, :],\n", + " digital_array[2, :],\n", + " nidq_sampling_rate,\n", + " behavior_time.shape[0],\n", + " behavior_time,\n", + " mode=mode,\n", + ")\n", "# get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal,\n", "\n", "print(iteration_dict)\n", "\n", - "status = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time)\n", - "status\n" + "status = ephys_utils.assert_iteration_samples_count(\n", + " iteration_dict[\"iter_start_idx\"], behavior_time\n", + ")\n", + "status" ] }, { @@ -16202,9 +16228,9 @@ ], "source": [ "print(iteration_dict.keys())\n", - "so = np.where(iteration_dict['trialnumber_vector_samples'] == 12)\n", + "so = np.where(iteration_dict[\"trialnumber_vector_samples\"] == 12)\n", "\n", - "print(so[0])\n" + "print(so[0])" ] }, { @@ -16249,20 +16275,39 @@ "end_iter = 5\n", "samp_after = 30\n", "\n", - "plt.plot(digital_array[0,iteration_dict['iter_start_idx'][trial_plot][0]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n", - "plt.plot(digital_array[1,iteration_dict['iter_start_idx'][trial_plot][0]-samp_before:iteration_dict['iter_start_idx'][trial_plot][end_iter]]+samp_after)\n", + "plt.plot(\n", + " digital_array[\n", + " 0,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][0] - samp_before : iteration_dict[\n", + " \"iter_start_idx\"\n", + " ][trial_plot][end_iter],\n", + " ]\n", + " + samp_after\n", + ")\n", + "plt.plot(\n", + " digital_array[\n", + " 1,\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][0] - samp_before : iteration_dict[\n", + " \"iter_start_idx\"\n", + " ][trial_plot][end_iter],\n", + " ]\n", + " + samp_after\n", + ")\n", "\n", "\n", - "iter_nidaq = (iteration_dict['iter_start_idx'][trial_plot][-1]-iteration_dict['iter_start_idx'][trial_plot][0])\n", - "time = iter_nidaq/nidq_sampling_rate\n", - "iter_virmen = time*120\n", + "iter_nidaq = (\n", + " iteration_dict[\"iter_start_idx\"][trial_plot][-1]\n", + " - iteration_dict[\"iter_start_idx\"][trial_plot][0]\n", + ")\n", + "time = iter_nidaq / nidq_sampling_rate\n", + "iter_virmen = time * 120\n", "\n", - "print('time from niDAQ', time)\n", - "print('time behavior', behavior_time[trial_plot][-1])\n", + "print(\"time from niDAQ\", time)\n", + "print(\"time behavior\", behavior_time[trial_plot][-1])\n", "\n", - "print('samples nidaq', iter_nidaq)\n", - "print('iter nidaq', iteration_dict['iter_start_idx'][trial_plot].shape)\n", - "print('iter_virmen',behavior_time[trial_plot].shape)" + "print(\"samples nidaq\", iter_nidaq)\n", + "print(\"iter nidaq\", iteration_dict[\"iter_start_idx\"][trial_plot].shape)\n", + "print(\"iter_virmen\", behavior_time[trial_plot].shape)" ] }, { @@ -16303,14 +16348,14 @@ "source": [ "x = np.array([])\n", "mean_x = np.array([])\n", - "for i in range(iteration_dict['iter_times_idx'].shape[0]-1):\n", - " s = behavior_time[i].flatten()-iteration_dict['iter_times_idx'][i]\n", + "for i in range(iteration_dict[\"iter_times_idx\"].shape[0] - 1):\n", + " s = behavior_time[i].flatten() - iteration_dict[\"iter_times_idx\"][i]\n", " mean_time_trial = np.mean(s)\n", - " x = np.append(x,s, axis=0)\n", + " x = np.append(x, s, axis=0)\n", " mean_x = np.append(mean_x, mean_time_trial)\n", "\n", - "#plt.plot(x)\n", - "plt.plot(mean_x)\n" + "# plt.plot(x)\n", + "plt.plot(mean_x)" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb index dbf786a9..c03cf8bc 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_missing_sync_sessions_old.ipynb @@ -36,7 +36,7 @@ "source": [ "import numpy as np\n", "\n", - "import u19_pipeline.ephys_pipeline as ep\n" + "import u19_pipeline.ephys_pipeline as ep" ] }, { @@ -45,11 +45,11 @@ "metadata": {}, "outputs": [], "source": [ - "recs2 = (ep.BehaviorSync).fetch('recording_id', as_dict=True)\n", + "recs2 = (ep.BehaviorSync).fetch(\"recording_id\", as_dict=True)\n", "bad_Sessions = list()\n", "for i in recs2:\n", " s = (ep.BehaviorSync & i).fetch(as_dict=True)\n", - " if np.all(np.isnan(s[0]['trial_index_nidq'])):\n", + " if np.all(np.isnan(s[0][\"trial_index_nidq\"])):\n", " bad_Sessions.append(i)" ] }, @@ -59,7 +59,7 @@ "metadata": {}, "outputs": [], "source": [ - "recs2 = (ep.BehaviorSync).fetch('recording_id', as_dict=True)" + "recs2 = (ep.BehaviorSync).fetch(\"recording_id\", as_dict=True)" ] }, { @@ -78,9 +78,9 @@ "outputs": [], "source": [ "missing_recs = []\n", - "for i in range(450,457):\n", + "for i in range(450, 457):\n", " dicto = {}\n", - " dicto['recording_id'] = i\n", + " dicto[\"recording_id\"] = i\n", " missing_recs.append(dicto)" ] }, diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb index 414d5ba1..0c745c04 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_nan_sync_sessions.ipynb @@ -37,7 +37,7 @@ "source": [ "import numpy as np\n", "\n", - "import u19_pipeline.ephys_pipeline as ep\n" + "import u19_pipeline.ephys_pipeline as ep" ] }, { @@ -46,11 +46,11 @@ "metadata": {}, "outputs": [], "source": [ - "recs2 = (ep.BehaviorSync & \"recording_id = 482\").fetch('recording_id', as_dict=True)\n", + "recs2 = (ep.BehaviorSync & \"recording_id = 482\").fetch(\"recording_id\", as_dict=True)\n", "bad_Sessions = list()\n", "for i in recs2:\n", " s = (ep.BehaviorSync & i).fetch(as_dict=True)\n", - " if np.all(np.isnan(s[0]['trial_index_nidq'])):\n", + " if np.all(np.isnan(s[0][\"trial_index_nidq\"])):\n", " bad_Sessions.append(i)" ] }, @@ -60,8 +60,7 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "bad_Sessions = {'recording_id = 205'}" + "bad_Sessions = {\"recording_id = 205\"}" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb index aa52df28..994afb8a 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time.ipynb @@ -39,8 +39,7 @@ "\n", "import u19_pipeline.ephys_pipeline as ep\n", "import u19_pipeline.utils.ephys_utils as eu\n", - "from u19_pipeline import recording\n", - "\n" + "from u19_pipeline import recording" ] }, { @@ -49,8 +48,8 @@ "metadata": {}, "outputs": [], "source": [ - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n", - "ephys_pipeline = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')" + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", + "ephys_pipeline = dj.create_virtual_module(\"ephys_pipeline\", \"u19_ephys_pipeline\")" ] }, { @@ -59,7 +58,9 @@ "metadata": {}, "outputs": [], "source": [ - "recording_synced = (ep.BehaviorSync & \"sync_data is null\").fetch('KEY', order_by='recording_id DESC')" + "recording_synced = (ep.BehaviorSync & \"sync_data is null\").fetch(\n", + " \"KEY\", order_by=\"recording_id DESC\"\n", + ")" ] }, { @@ -124,7 +125,7 @@ "metadata": {}, "outputs": [], "source": [ - "recording_synced = [recording_synced[0]]\n" + "recording_synced = [recording_synced[0]]" ] }, { @@ -171,105 +172,109 @@ ], "source": [ "for rec_key in recording_synced:\n", - "\n", - " print('rec_key', rec_key)\n", + " print(\"rec_key\", rec_key)\n", " behavior_key = (recording.Recording.BehaviorSession & rec_key).fetch1()\n", "\n", " full_session_path = ep.get_full_session_directory(rec_key)\n", - " \n", - " print('full_session_path', full_session_path)\n", + "\n", + " print(\"full_session_path\", full_session_path)\n", "\n", " if isinstance(full_session_path, str) and len(full_session_path) == 0:\n", " continue\n", "\n", " nidq_meta, nidq_sampling_rate = eu.read_nidq_meta_samp_rate(full_session_path)\n", "\n", - " trial_pulses_signal, _ = eu.load_trial_iteration_signals(full_session_path, nidq_meta)\n", + " trial_pulses_signal, _ = eu.load_trial_iteration_signals(\n", + " full_session_path, nidq_meta\n", + " )\n", "\n", " try:\n", " all_sync_data = (ep.BehaviorSync & rec_key).fetch1()\n", " except:\n", - " print('No sync data was sound for this session')\n", + " print(\"No sync data was sound for this session\")\n", " continue\n", "\n", - " if all_sync_data['sync_data'] != None:\n", - " print('sync_data is already inserted')\n", + " if all_sync_data[\"sync_data\"] != None:\n", + " print(\"sync_data is already inserted\")\n", " pass\n", - " #continue \n", + " # continue\n", + "\n", + " behavior_key.pop(\"recording_id\")\n", "\n", - " behavior_key.pop('recording_id')\n", - " \n", " thissession = behavior.TowersBlock().Trial() & behavior_key\n", - " behavior_time, iterstart = thissession.fetch('trial_time', 'vi_start')\n", + " behavior_time, iterstart = thissession.fetch(\"trial_time\", \"vi_start\")\n", "\n", - " if np.where(np.isnan(all_sync_data['trial_index_nidq']))[0].shape[0] == all_sync_data['trial_index_nidq'].shape[0]:\n", - " print('All nan session')\n", + " if (\n", + " np.where(np.isnan(all_sync_data[\"trial_index_nidq\"]))[0].shape[0]\n", + " == all_sync_data[\"trial_index_nidq\"].shape[0]\n", + " ):\n", + " print(\"All nan session\")\n", " continue\n", "\n", - " if int(np.nanmax(all_sync_data['trial_index_nidq'])) != behavior_time.shape[0]:\n", - " print('doing only one block')\n", - " behavior_key['block'] = 1\n", + " if int(np.nanmax(all_sync_data[\"trial_index_nidq\"])) != behavior_time.shape[0]:\n", + " print(\"doing only one block\")\n", + " behavior_key[\"block\"] = 1\n", " thissession = behavior.TowersBlock().Trial() & behavior_key\n", - " behavior_time, iterstart = thissession.fetch('trial_time', 'vi_start')\n", + " behavior_time, iterstart = thissession.fetch(\"trial_time\", \"vi_start\")\n", "\n", - " if int(np.nanmax(all_sync_data['trial_index_nidq'])) != behavior_time.shape[0]:\n", + " if int(np.nanmax(all_sync_data[\"trial_index_nidq\"])) != behavior_time.shape[0]:\n", " continue\n", "\n", - " status, trial_idx_vector, iteration_idx_vector =\\\n", - " eu.get_index_type_vectors(all_sync_data['trial_index_nidq'], all_sync_data['iteration_index_nidq'], nidq_sampling_rate)\n", - "\n", + " status, trial_idx_vector, iteration_idx_vector = eu.get_index_type_vectors(\n", + " all_sync_data[\"trial_index_nidq\"],\n", + " all_sync_data[\"iteration_index_nidq\"],\n", + " nidq_sampling_rate,\n", + " )\n", "\n", " if not status:\n", " print(behavior_key)\n", " print(rec_key)\n", - " print('Could not reproduce nidq full vector')\n", + " print(\"Could not reproduce nidq full vector\")\n", " break\n", "\n", - " trial_idx_vector_virmen, iteration_idx_vector_virmen =\\\n", - " eu.get_iteration_intertrial_from_virmen_time(trial_pulses_signal, nidq_sampling_rate, behavior_time.shape[0], behavior_time)\n", + " trial_idx_vector_virmen, iteration_idx_vector_virmen = (\n", + " eu.get_iteration_intertrial_from_virmen_time(\n", + " trial_pulses_signal,\n", + " nidq_sampling_rate,\n", + " behavior_time.shape[0],\n", + " behavior_time,\n", + " )\n", + " )\n", "\n", - " trial_index_nidq_virmen, iteration_index_nidq_virmen =\\\n", - " eu.get_full_vector_samples(iteration_idx_vector_virmen,nidq_sampling_rate,all_sync_data['iteration_index_nidq'].shape[0])\n", + " trial_index_nidq_virmen, iteration_index_nidq_virmen = eu.get_full_vector_samples(\n", + " iteration_idx_vector_virmen,\n", + " nidq_sampling_rate,\n", + " all_sync_data[\"iteration_index_nidq\"].shape[0],\n", + " )\n", "\n", - "\n", - " diff_vector = all_sync_data['iteration_index_nidq'] - iteration_index_nidq_virmen\n", + " diff_vector = all_sync_data[\"iteration_index_nidq\"] - iteration_index_nidq_virmen\n", "\n", " max_diff_virmen = np.nanmax(diff_vector)\n", - " print('max_diff_virmen', max_diff_virmen)\n", + " print(\"max_diff_virmen\", max_diff_virmen)\n", "\n", " min_diff_virmen = np.nanmin(diff_vector)\n", - " print('min_diff_virmen', min_diff_virmen)\n", + " print(\"min_diff_virmen\", min_diff_virmen)\n", "\n", " dictionary_sync_data = dict()\n", "\n", - "\n", " if max_diff_virmen > 20 or min_diff_virmen < -20:\n", - " dictionary_sync_data['virmen_sync_status'] = True\n", + " dictionary_sync_data[\"virmen_sync_status\"] = True\n", " else:\n", - " dictionary_sync_data['virmen_sync_status'] = False\n", - "\n", + " dictionary_sync_data[\"virmen_sync_status\"] = False\n", "\n", - " dictionary_sync_data['regular_sync_status'] = False\n", + " dictionary_sync_data[\"regular_sync_status\"] = False\n", "\n", - " dictionary_sync_data['trial_idx_vector'] = trial_idx_vector\n", - " dictionary_sync_data['iteration_idx_vector'] = iteration_idx_vector\n", + " dictionary_sync_data[\"trial_idx_vector\"] = trial_idx_vector\n", + " dictionary_sync_data[\"iteration_idx_vector\"] = iteration_idx_vector\n", "\n", - " dictionary_sync_data['trial_idx_vector_from_virmen'] = trial_idx_vector_virmen\n", - " dictionary_sync_data['iteration_idx_vector_from_virmen'] = iteration_idx_vector_virmen\n", + " dictionary_sync_data[\"trial_idx_vector_from_virmen\"] = trial_idx_vector_virmen\n", + " dictionary_sync_data[\"iteration_idx_vector_from_virmen\"] = (\n", + " iteration_idx_vector_virmen\n", + " )\n", "\n", " update_key = rec_key.copy()\n", - " update_key['sync_data'] = dictionary_sync_data\n", - " (ephys_pipeline.BehaviorSync & rec_key).update1(update_key)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - " \n", - "\n", - " \n", - "\n", - " " + " update_key[\"sync_data\"] = dictionary_sync_data\n", + " (ephys_pipeline.BehaviorSync & rec_key).update1(update_key)" ] }, { @@ -278,9 +283,11 @@ "metadata": {}, "outputs": [], "source": [ - "trial_index_nidq, iteration_index_nidq =\\\n", - " eu.get_full_vector_samples(iteration_idx_vector,nidq_sampling_rate,all_sync_data['iteration_index_nidq'].shape[0])\n", - "\n" + "trial_index_nidq, iteration_index_nidq = eu.get_full_vector_samples(\n", + " iteration_idx_vector,\n", + " nidq_sampling_rate,\n", + " all_sync_data[\"iteration_index_nidq\"].shape[0],\n", + ")" ] }, { @@ -289,7 +296,9 @@ "metadata": {}, "outputs": [], "source": [ - "idx_diff = np.where((all_sync_data['iteration_index_nidq'] - iteration_index_nidq) > 0 )[0]" + "idx_diff = np.where((all_sync_data[\"iteration_index_nidq\"] - iteration_index_nidq) > 0)[\n", + " 0\n", + "]" ] }, { @@ -340,7 +349,7 @@ } ], "source": [ - "all_sync_data['trial_index_nidq'][idx_diff]" + "all_sync_data[\"trial_index_nidq\"][idx_diff]" ] }, { @@ -370,7 +379,7 @@ } ], "source": [ - "plt.plot(all_sync_data['iteration_index_nidq'] - iteration_index_nidq)" + "plt.plot(all_sync_data[\"iteration_index_nidq\"] - iteration_index_nidq)" ] }, { @@ -400,7 +409,7 @@ } ], "source": [ - "plt.plot(all_sync_data['iteration_index_nidq'][idx_diff])\n", + "plt.plot(all_sync_data[\"iteration_index_nidq\"][idx_diff])\n", "plt.plot(iteration_index_nidq[idx_diff])" ] }, @@ -419,7 +428,7 @@ } ], "source": [ - "print(all_sync_data['iteration_index_nidq'][idx_diff[0:5]])\n", + "print(all_sync_data[\"iteration_index_nidq\"][idx_diff[0:5]])\n", "print(iteration_index_nidq[idx_diff[0:5]])" ] }, @@ -438,8 +447,8 @@ } ], "source": [ - "print(all_sync_data['iteration_index_nidq'][idx_diff[0:5]-3])\n", - "print(iteration_index_nidq[idx_diff[0:5]-3])" + "print(all_sync_data[\"iteration_index_nidq\"][idx_diff[0:5] - 3])\n", + "print(iteration_index_nidq[idx_diff[0:5] - 3])" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb index 0e0a8d4f..9f56ad6e 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/ephys_sync_old_notebooks/reingest_sync_all_sessions_include_virmen_time_status.ipynb @@ -37,8 +37,7 @@ "source": [ "import datajoint as dj\n", "\n", - "import u19_pipeline.ephys_pipeline as ep\n", - "\n" + "import u19_pipeline.ephys_pipeline as ep" ] }, { @@ -47,8 +46,8 @@ "metadata": {}, "outputs": [], "source": [ - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n", - "ephys_pipeline = dj.create_virtual_module('ephys_pipeline', 'u19_ephys_pipeline')" + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", + "ephys_pipeline = dj.create_virtual_module(\"ephys_pipeline\", \"u19_ephys_pipeline\")" ] }, { @@ -57,7 +56,9 @@ "metadata": {}, "outputs": [], "source": [ - "recording_synced = (ep.BehaviorSync & \"sync_data is not null\").fetch('KEY', 'sync_data', order_by='recording_id DESC', as_dict=True)" + "recording_synced = (ep.BehaviorSync & \"sync_data is not null\").fetch(\n", + " \"KEY\", \"sync_data\", order_by=\"recording_id DESC\", as_dict=True\n", + ")" ] }, { @@ -76,28 +77,26 @@ ], "source": [ "for rec_key in recording_synced:\n", - "\n", - " print('rec_key', rec_key['recording_id'])\n", + " print(\"rec_key\", rec_key[\"recording_id\"])\n", "\n", " update_key = dict()\n", - " update_key['recording_id'] = rec_key['recording_id']\n", - " if rec_key['sync_data']['regular_sync_status']:\n", - " update_key['regular_sync_status'] = 1\n", - " update_key['fixed_sync_status'] = 0\n", + " update_key[\"recording_id\"] = rec_key[\"recording_id\"]\n", + " if rec_key[\"sync_data\"][\"regular_sync_status\"]:\n", + " update_key[\"regular_sync_status\"] = 1\n", + " update_key[\"fixed_sync_status\"] = 0\n", " else:\n", - " update_key['regular_sync_status'] = 0\n", - " update_key['fixed_sync_status'] = 1\n", + " update_key[\"regular_sync_status\"] = 0\n", + " update_key[\"fixed_sync_status\"] = 1\n", "\n", - " if rec_key['sync_data']['virmen_sync_status']:\n", - " update_key['virmen_sync_status'] = 1\n", + " if rec_key[\"sync_data\"][\"virmen_sync_status\"]:\n", + " update_key[\"virmen_sync_status\"] = 1\n", " else:\n", - " update_key['regular_sync_status'] = 0\n", + " update_key[\"regular_sync_status\"] = 0\n", "\n", - " print('update_key', update_key)\n", + " print(\"update_key\", update_key)\n", " break\n", "\n", - " #(ephys_pipeline.BehaviorSync).update1(update_key)\n", - " \n" + " # (ephys_pipeline.BehaviorSync).update1(update_key)" ] }, { @@ -117,7 +116,7 @@ } ], "source": [ - "rec_key['sync_data']['trial_idx_vector'].shape" + "rec_key[\"sync_data\"][\"trial_idx_vector\"].shape" ] }, { @@ -34536,7 +34535,7 @@ } ], "source": [ - "rec_key['sync_data']['iteration_idx_vector']" + "rec_key[\"sync_data\"][\"iteration_idx_vector\"]" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb index dd172115..633f990f 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/get_ephys_sync_data.ipynb @@ -25,7 +25,7 @@ "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", "\n", - "try_find_conf_file()\n" + "try_find_conf_file()" ] }, { @@ -49,8 +49,7 @@ "import matplotlib.pyplot as plt\n", "\n", "import u19_pipeline.ephys_pipeline as ep\n", - "import u19_pipeline.recording as recording\n", - " \n" + "import u19_pipeline.recording as recording" ] }, { @@ -82,9 +81,9 @@ ], "source": [ "beh_key = dict()\n", - "beh_key['subject_fullname'] = 'emdia_nicky'\n", - "beh_key['session_date'] = '2026-03-23'\n", - "beh_key['session_number'] = 0\n", + "beh_key[\"subject_fullname\"] = \"emdia_nicky\"\n", + "beh_key[\"session_date\"] = \"2026-03-23\"\n", + "beh_key[\"session_number\"] = 0\n", "beh_key" ] }, @@ -114,13 +113,13 @@ "recording_beh_key = (recording.Recording.BehaviorSession & beh_key).fetch(as_dict=True)\n", "\n", "if len(recording_beh_key) == 0:\n", - " print('No recording data found for this session')\n", + " print(\"No recording data found for this session\")\n", "elif len(recording_beh_key) > 1:\n", - " print('Multiple recordings for this behavior key, select only one')\n", + " print(\"Multiple recordings for this behavior key, select only one\")\n", "elif len(recording_beh_key) == 1:\n", " recording_key = dict()\n", - " recording_key['recording_id'] = recording_beh_key[0]['recording_id']\n", - " print(recording_key)\n" + " recording_key[\"recording_id\"] = recording_beh_key[0][\"recording_id\"]\n", + " print(recording_key)" ] }, { @@ -151,13 +150,14 @@ } ], "source": [ - "\n", - "sync_record = (ep.BehaviorSync & recording_key).fetch1('sync_data')\n", + "sync_record = (ep.BehaviorSync & recording_key).fetch1(\"sync_data\")\n", "full_sync_record = (ep.BehaviorSync & recording_key).fetch1()\n", "\n", - "behavior = dj.create_virtual_module('behavior','u19_behavior')\n", - "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(as_dict=True)[0]\n", - "behavior_session.pop('recording_id')\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", + "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(\n", + " as_dict=True\n", + ")[0]\n", + "behavior_session.pop(\"recording_id\")\n", "behavior_time = (behavior.SpatialTimeBlobs & behavior_session).fetch(as_dict=True)\n", "\n", "behavior_session" @@ -189,8 +189,8 @@ } ], "source": [ - "print(full_sync_record['iteration_index_nidq'])\n", - "print(full_sync_record['iteration_index_nidq'])" + "print(full_sync_record[\"iteration_index_nidq\"])\n", + "print(full_sync_record[\"iteration_index_nidq\"])" ] }, { @@ -248,8 +248,8 @@ } ], "source": [ - "print(sync_record['trial_idx_vector'].shape)\n", - "sync_record['trial_idx_vector'][0:100]" + "print(sync_record[\"trial_idx_vector\"].shape)\n", + "sync_record[\"trial_idx_vector\"][0:100]" ] }, { @@ -307,11 +307,19 @@ "source": [ "import numpy as np\n", "\n", - "time_as_behavior_fullsession = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession']]), axis=0) \n", - "time_as_behavior_trial_ind = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind']]), axis=0) \n", + "time_as_behavior_fullsession = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_fullsession\"]]), axis=0\n", + ")\n", + "time_as_behavior_trial_ind = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_trial_ind\"]]), axis=0\n", + ")\n", "\n", - "time_as_behavior_fullsession_v = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession_virmen']]), axis=0) \n", - "time_as_behavior_trial_ind_v = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind_virmen']]), axis=0) \n" + "time_as_behavior_fullsession_v = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_fullsession_virmen\"]]), axis=0\n", + ")\n", + "time_as_behavior_trial_ind_v = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_trial_ind_virmen\"]]), axis=0\n", + ")" ] }, { @@ -329,11 +337,21 @@ "metadata": {}, "outputs": [], "source": [ - "diff_time_trial_ind = behavior_time[0]['trial_time'].squeeze() - time_as_behavior_trial_ind\n", - "diff_time_trial_full_session = (behavior_time[0]['cumulative_session_time'].squeeze() - behavior_time[0]['cumulative_session_time'][0]) - time_as_behavior_fullsession\n", + "diff_time_trial_ind = (\n", + " behavior_time[0][\"trial_time\"].squeeze() - time_as_behavior_trial_ind\n", + ")\n", + "diff_time_trial_full_session = (\n", + " behavior_time[0][\"cumulative_session_time\"].squeeze()\n", + " - behavior_time[0][\"cumulative_session_time\"][0]\n", + ") - time_as_behavior_fullsession\n", "\n", - "diff_time_trial_ind_v = behavior_time[0]['trial_time'].squeeze() - time_as_behavior_trial_ind_v\n", - "diff_time_trial_full_session_v = (behavior_time[0]['cumulative_session_time'].squeeze() - behavior_time[0]['cumulative_session_time'][0]) - time_as_behavior_fullsession_v\n" + "diff_time_trial_ind_v = (\n", + " behavior_time[0][\"trial_time\"].squeeze() - time_as_behavior_trial_ind_v\n", + ")\n", + "diff_time_trial_full_session_v = (\n", + " behavior_time[0][\"cumulative_session_time\"].squeeze()\n", + " - behavior_time[0][\"cumulative_session_time\"][0]\n", + ") - time_as_behavior_fullsession_v" ] }, { @@ -376,9 +394,11 @@ "plt.figure(figsize=(8, 6))\n", "plt.plot(diff_time_trial_full_session)\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time diff (s)')\n", - "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session')" + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time diff (s)\")\n", + "plt.title(\n", + " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n", + ")" ] }, { @@ -411,13 +431,14 @@ "source": [ "plt.figure(figsize=(8, 6))\n", "\n", - "plt.hist(diff_time_trial_full_session,bins=100)\n", + "plt.hist(diff_time_trial_full_session, bins=100)\n", "\n", - "plt.yscale('log')\n", - "plt.xlabel('Iteration start time difference (s)')\n", - "plt.ylabel('Counts')\n", - "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session')\n", - "\n" + "plt.yscale(\"log\")\n", + "plt.xlabel(\"Iteration start time difference (s)\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.title(\n", + " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session'\n", + ")" ] }, { @@ -451,9 +472,11 @@ "plt.figure(figsize=(8, 6))\n", "plt.plot(diff_time_trial_full_session_v)\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time diff (s)')\n", - "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session')" + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time diff (s)\")\n", + "plt.title(\n", + " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n", + ")" ] }, { @@ -486,13 +509,14 @@ "source": [ "plt.figure(figsize=(8, 6))\n", "\n", - "plt.hist(diff_time_trial_full_session_v,bins=100)\n", + "plt.hist(diff_time_trial_full_session_v, bins=100)\n", "\n", - "plt.yscale('log')\n", - "plt.xlabel('Iteration start time difference (s)')\n", - "plt.ylabel('Counts')\n", - "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session')\n", - "\n" + "plt.yscale(\"log\")\n", + "plt.xlabel(\"Iteration start time difference (s)\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.title(\n", + " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session'\n", + ")" ] } ], diff --git a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb index bccfaa32..eea28d9f 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data.ipynb @@ -17,7 +17,7 @@ "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", "\n", - "try_find_conf_file()\n" + "try_find_conf_file()" ] }, { @@ -39,8 +39,7 @@ "import matplotlib.pyplot as plt\n", "\n", "import u19_pipeline.ephys_pipeline as ep\n", - "import u19_pipeline.recording as recording\n", - " \n" + "import u19_pipeline.recording as recording" ] }, { @@ -59,7 +58,7 @@ "outputs": [], "source": [ "recording_key = dict()\n", - "recording_key['recording_id'] = 611" + "recording_key[\"recording_id\"] = 611" ] }, { @@ -90,12 +89,13 @@ } ], "source": [ + "sync_record = (ep.BehaviorSync & recording_key).fetch1(\"sync_data\")\n", "\n", - "sync_record = (ep.BehaviorSync & recording_key).fetch1('sync_data')\n", - "\n", - "behavior = dj.create_virtual_module('behavior','u19_behavior')\n", - "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(as_dict=True)[0]\n", - "behavior_session.pop('recording_id')\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", + "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(\n", + " as_dict=True\n", + ")[0]\n", + "behavior_session.pop(\"recording_id\")\n", "behavior_time = (behavior.SpatialTimeBlobs & behavior_session).fetch(as_dict=True)\n", "\n", "behavior_session" @@ -174,12 +174,19 @@ "source": [ "import numpy as np\n", "\n", - "time_as_behavior_fullsession = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession']]), axis=0) \n", - "time_as_behavior_trial_ind = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind']]), axis=0) \n", + "time_as_behavior_fullsession = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_fullsession\"]]), axis=0\n", + ")\n", + "time_as_behavior_trial_ind = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_trial_ind\"]]), axis=0\n", + ")\n", "\n", - "time_as_behavior_fullsession_v = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession_virmen']]), axis=0) \n", - "time_as_behavior_trial_ind_v = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind_virmen']]), axis=0) \n", - "\n" + "time_as_behavior_fullsession_v = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_fullsession_virmen\"]]), axis=0\n", + ")\n", + "time_as_behavior_trial_ind_v = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_trial_ind_virmen\"]]), axis=0\n", + ")" ] }, { @@ -197,11 +204,21 @@ "metadata": {}, "outputs": [], "source": [ - "diff_time_trial_ind = behavior_time[0]['trial_time'].squeeze() - time_as_behavior_trial_ind\n", - "diff_time_trial_ind_v = behavior_time[0]['trial_time'].squeeze() - time_as_behavior_trial_ind_v\n", + "diff_time_trial_ind = (\n", + " behavior_time[0][\"trial_time\"].squeeze() - time_as_behavior_trial_ind\n", + ")\n", + "diff_time_trial_ind_v = (\n", + " behavior_time[0][\"trial_time\"].squeeze() - time_as_behavior_trial_ind_v\n", + ")\n", "\n", - "diff_time_trial_full_session = (behavior_time[0]['cumulative_session_time'].squeeze() - behavior_time[0]['cumulative_session_time'][0]) - time_as_behavior_fullsession\n", - "diff_time_trial_full_session_v = (behavior_time[0]['cumulative_session_time'].squeeze() - behavior_time[0]['cumulative_session_time'][0]) - time_as_behavior_fullsession_v\n" + "diff_time_trial_full_session = (\n", + " behavior_time[0][\"cumulative_session_time\"].squeeze()\n", + " - behavior_time[0][\"cumulative_session_time\"][0]\n", + ") - time_as_behavior_fullsession\n", + "diff_time_trial_full_session_v = (\n", + " behavior_time[0][\"cumulative_session_time\"].squeeze()\n", + " - behavior_time[0][\"cumulative_session_time\"][0]\n", + ") - time_as_behavior_fullsession_v" ] }, { @@ -244,9 +261,11 @@ "plt.figure(figsize=(8, 6))\n", "plt.plot(diff_time_trial_ind)\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time diff (s)')\n", - "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Each trial ind. time')" + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time diff (s)\")\n", + "plt.title(\n", + " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Each trial ind. time'\n", + ")" ] }, { @@ -288,13 +307,14 @@ "source": [ "plt.figure(figsize=(8, 6))\n", "\n", - "plt.hist(diff_time_trial_ind,bins=100)\n", + "plt.hist(diff_time_trial_ind, bins=100)\n", "\n", - "plt.yscale('log')\n", - "plt.xlabel('Iteration start time difference (s)')\n", - "plt.ylabel('Counts')\n", - "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Each trial ind. time')\n", - "\n" + "plt.yscale(\"log\")\n", + "plt.xlabel(\"Iteration start time difference (s)\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.title(\n", + " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Each trial ind. time'\n", + ")" ] }, { @@ -366,9 +386,11 @@ "plt.figure(figsize=(8, 6))\n", "plt.plot(diff_time_trial_full_session)\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time diff (s)')\n", - "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session')" + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time diff (s)\")\n", + "plt.title(\n", + " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n", + ")" ] }, { @@ -401,13 +423,14 @@ "source": [ "plt.figure(figsize=(8, 6))\n", "\n", - "plt.hist(diff_time_trial_full_session,bins=100)\n", + "plt.hist(diff_time_trial_full_session, bins=100)\n", "\n", - "plt.yscale('log')\n", - "plt.xlabel('Iteration start time difference (s)')\n", - "plt.ylabel('Counts')\n", - "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session')\n", - "\n" + "plt.yscale(\"log\")\n", + "plt.xlabel(\"Iteration start time difference (s)\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.title(\n", + " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session'\n", + ")" ] }, { @@ -447,15 +470,14 @@ } ], "source": [ - "\n", - "\n", "plt.figure(figsize=(8, 6))\n", "plt.plot(diff_time_trial_full_session_v)\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time diff (s)')\n", - "plt.title('iteration \"start time\" sanity check comparison -- behavior file - nidaq pulse signal. Single vector time all session')\n", - "\n" + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time diff (s)\")\n", + "plt.title(\n", + " 'iteration \"start time\" sanity check comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n", + ")" ] }, { @@ -514,15 +536,14 @@ } ], "source": [ - "\n", - "\n", "plt.figure(figsize=(8, 6))\n", "plt.plot(diff_times_fullsessions)\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time diff (s)')\n", - "plt.title('iteration \"start time\" comparison -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1')\n", - "\n" + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time diff (s)\")\n", + "plt.title(\n", + " 'iteration \"start time\" comparison -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1'\n", + ")" ] }, { @@ -555,12 +576,14 @@ "source": [ "plt.figure(figsize=(8, 6))\n", "\n", - "plt.hist(diff_times_fullsessions,bins=100)\n", + "plt.hist(diff_times_fullsessions, bins=100)\n", "\n", - "plt.yscale('log')\n", - "plt.xlabel('Iteration start time difference (s)')\n", - "plt.ylabel('Counts')\n", - "plt.title('iteration \"start time\" histogram -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1')\n" + "plt.yscale(\"log\")\n", + "plt.xlabel(\"Iteration start time difference (s)\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.title(\n", + " 'iteration \"start time\" histogram -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1'\n", + ")" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb index 245a2dcb..52af6449 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/get_full_vectos_from_sync_data_w_shift.ipynb @@ -23,7 +23,6 @@ } ], "source": [ - "\n", "from scripts.conf_file_finding import try_find_conf_file\n", "\n", "try_find_conf_file()" @@ -50,9 +49,7 @@ "\n", "import u19_pipeline.ephys_pipeline as ep\n", "import u19_pipeline.recording as recording\n", - "import u19_pipeline.utils.ephys_utils as ephys_utils\n", - " \n", - "\n" + "import u19_pipeline.utils.ephys_utils as ephys_utils" ] }, { @@ -71,7 +68,7 @@ "outputs": [], "source": [ "recording_key = dict()\n", - "recording_key['recording_id'] = 446" + "recording_key[\"recording_id\"] = 446" ] }, { @@ -102,15 +99,18 @@ } ], "source": [ + "sync_record = (ep.BehaviorSync & recording_key).fetch1(\"sync_data\")\n", "\n", - "sync_record = (ep.BehaviorSync & recording_key).fetch1('sync_data')\n", - "\n", - "behavior = dj.create_virtual_module('behavior','u19_behavior')\n", - "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(as_dict=True)[0]\n", - "behavior_session.pop('recording_id')\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")\n", + "behavior_session = (recording.Recording.BehaviorSession & recording_key).fetch(\n", + " as_dict=True\n", + ")[0]\n", + "behavior_session.pop(\"recording_id\")\n", "behavior_time_old = (behavior.SpatialTimeBlobs & behavior_session).fetch(as_dict=True)\n", "\n", - "behavior_time, behavior_fullsession = ephys_utils.get_real_behavior_time(behavior_session)\n", + "behavior_time, behavior_fullsession = ephys_utils.get_real_behavior_time(\n", + " behavior_session\n", + ")\n", "behavior_time_single_vec = np.concatenate(behavior_time)\n", "\n", "behavior_session" @@ -123,7 +123,9 @@ "metadata": {}, "outputs": [], "source": [ - "comp_times = behavior_fullsession-behavior_time_old[0]['cumulative_session_time'].squeeze()" + "comp_times = (\n", + " behavior_fullsession - behavior_time_old[0][\"cumulative_session_time\"].squeeze()\n", + ")" ] }, { @@ -164,7 +166,7 @@ "metadata": {}, "outputs": [], "source": [ - "comp_times2 = behavior_time_single_vec -behavior_time_old[0]['trial_time'].squeeze()" + "comp_times2 = behavior_time_single_vec - behavior_time_old[0][\"trial_time\"].squeeze()" ] }, { @@ -215,7 +217,7 @@ } ], "source": [ - "plt.plot(comp_times2)\n" + "plt.plot(comp_times2)" ] }, { @@ -242,7 +244,9 @@ } ], "source": [ - "all_vectors = ep.get_full_vectors_from_key(recording_key, apply_behavior_shift=True, behavior_time=behavior_time)" + "all_vectors = ep.get_full_vectors_from_key(\n", + " recording_key, apply_behavior_shift=True, behavior_time=behavior_time\n", + ")" ] }, { @@ -284,7 +288,7 @@ } ], "source": [ - "all_vectors['time_as_behavior_fullsession'].shape" + "all_vectors[\"time_as_behavior_fullsession\"].shape" ] }, { @@ -302,13 +306,19 @@ "metadata": {}, "outputs": [], "source": [ + "time_as_behavior_fullsession = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_fullsession\"]]), axis=0\n", + ")\n", + "time_as_behavior_trial_ind = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_trial_ind\"]]), axis=0\n", + ")\n", "\n", - "time_as_behavior_fullsession = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession']]), axis=0) \n", - "time_as_behavior_trial_ind = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind']]), axis=0) \n", - "\n", - "time_as_behavior_fullsession_v = np.concatenate(([x for x in all_vectors['time_as_behavior_fullsession_virmen']]), axis=0) \n", - "time_as_behavior_trial_ind_v = np.concatenate(([x for x in all_vectors['time_as_behavior_trial_ind_virmen']]), axis=0) \n", - "\n" + "time_as_behavior_fullsession_v = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_fullsession_virmen\"]]), axis=0\n", + ")\n", + "time_as_behavior_trial_ind_v = np.concatenate(\n", + " ([x for x in all_vectors[\"time_as_behavior_trial_ind_virmen\"]]), axis=0\n", + ")" ] }, { @@ -329,8 +339,8 @@ "diff_time_trial_ind = time_as_behavior_trial_ind - behavior_time_single_vec\n", "diff_time_trial_ind_v = time_as_behavior_trial_ind_v - behavior_time_single_vec\n", "\n", - "diff_time_trial_full_session = time_as_behavior_fullsession - behavior_fullsession \n", - "diff_time_trial_full_session_v = time_as_behavior_fullsession_v - behavior_fullsession\n" + "diff_time_trial_full_session = time_as_behavior_fullsession - behavior_fullsession\n", + "diff_time_trial_full_session_v = time_as_behavior_fullsession_v - behavior_fullsession" ] }, { @@ -373,11 +383,13 @@ "plt.figure(figsize=(8, 6))\n", "plt.plot(diff_time_trial_ind)\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time diff (s)')\n", - "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Each trial ind. time')\n", + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time diff (s)\")\n", + "plt.title(\n", + " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Each trial ind. time'\n", + ")\n", "\n", - "#plt.ylim([-0.75,0.1])" + "# plt.ylim([-0.75,0.1])" ] }, { @@ -419,7 +431,9 @@ } ], "source": [ - "consecutive_trials = ephys_utils.get_consecutive_trials_vector_from_iteration_matrix(behavior_time_old[0]['iteration_matrix'])\n", + "consecutive_trials = ephys_utils.get_consecutive_trials_vector_from_iteration_matrix(\n", + " behavior_time_old[0][\"iteration_matrix\"]\n", + ")\n", "consecutive_trials.max()" ] }, @@ -442,12 +456,12 @@ } ], "source": [ - "print('Where minimum difference')\n", + "print(\"Where minimum difference\")\n", "idx_min_diff = diff_time_trial_ind.argmin()\n", - "print('block', behavior_time_old[0]['iteration_matrix'][idx_min_diff,0])\n", - "print('trial', behavior_time_old[0]['iteration_matrix'][idx_min_diff,1])\n", - "print('consecutive_trial', consecutive_trials[idx_min_diff])\n", - "print('iteration', behavior_time_old[0]['iteration_matrix'][idx_min_diff,2])\n" + "print(\"block\", behavior_time_old[0][\"iteration_matrix\"][idx_min_diff, 0])\n", + "print(\"trial\", behavior_time_old[0][\"iteration_matrix\"][idx_min_diff, 1])\n", + "print(\"consecutive_trial\", consecutive_trials[idx_min_diff])\n", + "print(\"iteration\", behavior_time_old[0][\"iteration_matrix\"][idx_min_diff, 2])" ] }, { @@ -490,12 +504,14 @@ } ], "source": [ - "print(time_as_behavior_trial_ind[idx_min_diff-5:idx_min_diff+5])\n", + "print(time_as_behavior_trial_ind[idx_min_diff - 5 : idx_min_diff + 5])\n", "\n", - "print(behavior_time_single_vec[idx_min_diff-5:idx_min_diff+5])\n", + "print(behavior_time_single_vec[idx_min_diff - 5 : idx_min_diff + 5])\n", "\n", "\n", - "print(behavior_time_old[0]['iteration_matrix'][idx_min_diff-5:idx_min_diff+300,2])" + "print(\n", + " behavior_time_old[0][\"iteration_matrix\"][idx_min_diff - 5 : idx_min_diff + 300, 2]\n", + ")" ] }, { @@ -556,14 +572,15 @@ "\n", "abs_diff_time_trial_ind = np.abs(diff_time_trial_ind)\n", "\n", - "plt.hist(diff_time_trial_ind,bins=100)\n", + "plt.hist(diff_time_trial_ind, bins=100)\n", "\n", - "plt.yscale('log')\n", - "#plt.xscale('log')\n", - "plt.xlabel('Iteration start time difference (s)')\n", - "plt.ylabel('Counts')\n", - "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Each trial ind. time')\n", - "\n" + "plt.yscale(\"log\")\n", + "# plt.xscale('log')\n", + "plt.xlabel(\"Iteration start time difference (s)\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.title(\n", + " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Each trial ind. time'\n", + ")" ] }, { @@ -635,9 +652,11 @@ "plt.figure(figsize=(8, 6))\n", "plt.plot(diff_time_trial_full_session)\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time diff (s)')\n", - "plt.title('iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session')" + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time diff (s)\")\n", + "plt.title(\n", + " 'iteration \"start time\" comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n", + ")" ] }, { @@ -670,13 +689,14 @@ "source": [ "plt.figure(figsize=(8, 6))\n", "\n", - "plt.hist(diff_time_trial_full_session,bins=100)\n", + "plt.hist(diff_time_trial_full_session, bins=100)\n", "\n", - "plt.yscale('log')\n", - "plt.xlabel('Iteration start time difference (s)')\n", - "plt.ylabel('Counts')\n", - "plt.title('iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session')\n", - "\n" + "plt.yscale(\"log\")\n", + "plt.xlabel(\"Iteration start time difference (s)\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.title(\n", + " 'iteration \"start time\" Histogram -- behavior file - nidaq pulse signal. Single vector time all session'\n", + ")" ] }, { @@ -716,15 +736,14 @@ } ], "source": [ - "\n", - "\n", "plt.figure(figsize=(8, 6))\n", "plt.plot(diff_time_trial_full_session_v)\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time diff (s)')\n", - "plt.title('iteration \"start time\" sanity check comparison -- behavior file - nidaq pulse signal. Single vector time all session')\n", - "\n" + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time diff (s)\")\n", + "plt.title(\n", + " 'iteration \"start time\" sanity check comparison -- behavior file - nidaq pulse signal. Single vector time all session'\n", + ")" ] }, { @@ -783,15 +802,14 @@ } ], "source": [ - "\n", - "\n", "plt.figure(figsize=(8, 6))\n", "plt.plot(diff_times_fullsessions)\n", "\n", - "plt.xlabel('Iteration #')\n", - "plt.ylabel('Time diff (s)')\n", - "plt.title('iteration \"start time\" comparison -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1')\n", - "\n" + "plt.xlabel(\"Iteration #\")\n", + "plt.ylabel(\"Time diff (s)\")\n", + "plt.title(\n", + " 'iteration \"start time\" comparison -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1'\n", + ")" ] }, { @@ -824,12 +842,14 @@ "source": [ "plt.figure(figsize=(8, 6))\n", "\n", - "plt.hist(diff_times_fullsessions,bins=100)\n", + "plt.hist(diff_times_fullsessions, bins=100)\n", "\n", - "plt.yscale('log')\n", - "plt.xlabel('Iteration start time difference (s)')\n", - "plt.ylabel('Counts')\n", - "plt.title('iteration \"start time\" histogram -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1')\n" + "plt.yscale(\"log\")\n", + "plt.xlabel(\"Iteration start time difference (s)\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.title(\n", + " 'iteration \"start time\" histogram -- \"pulse only sync\" vs \"virmen assisted sync\" single vector time for all session [0, ...] x 1'\n", + ")" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb index b5a255b9..5c289361 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/reingest_missing_sync_sessions.ipynb @@ -45,7 +45,7 @@ "import pandas as pd\n", "\n", "import u19_pipeline.ephys_pipeline as ep\n", - "import u19_pipeline.recording as recording\n" + "import u19_pipeline.recording as recording" ] }, { @@ -54,7 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "#(ep.BehaviorSync & \"recording_id=502\").delete()" + "# (ep.BehaviorSync & \"recording_id=502\").delete()" ] }, { @@ -71,12 +71,25 @@ "outputs": [], "source": [ "past_date = datetime.date.today() - datetime.timedelta(days=9000)\n", - "query_session_date = 'session_date >=\"' + past_date.strftime('%Y-%m-%d') +'\"'\n", + "query_session_date = 'session_date >=\"' + past_date.strftime(\"%Y-%m-%d\") + '\"'\n", "\n", - "all_recs = ((recording.Recording & \"recording_modality='electrophysiology'\") * (recording.Recording.BehaviorSession & query_session_date)).join(ep.BehaviorSync, left=True)\n", - "all_recs = pd.DataFrame(all_recs.fetch('recording_id','subject_fullname','session_date','session_number', as_dict=True))\n", + "all_recs = (\n", + " (recording.Recording & \"recording_modality='electrophysiology'\")\n", + " * (recording.Recording.BehaviorSession & query_session_date)\n", + ").join(ep.BehaviorSync, left=True)\n", + "all_recs = pd.DataFrame(\n", + " all_recs.fetch(\n", + " \"recording_id\",\n", + " \"subject_fullname\",\n", + " \"session_date\",\n", + " \"session_number\",\n", + " as_dict=True,\n", + " )\n", + ")\n", "\n", - "not_sync_recs = pd.DataFrame((recording.Recording * ep.BehaviorSync).fetch('KEY', as_dict=True))" + "not_sync_recs = pd.DataFrame(\n", + " (recording.Recording * ep.BehaviorSync).fetch(\"KEY\", as_dict=True)\n", + ")" ] }, { @@ -87,7 +100,7 @@ "source": [ "def get_rec_key_dict(recording_id):\n", "\n", - " return {'recording_id': recording_id}" + " return {\"recording_id\": recording_id}" ] }, { @@ -372,10 +385,10 @@ } ], "source": [ - "not_sync_recs2 = pd.merge(all_recs,not_sync_recs, how='left', indicator=True)\n", - "not_sync_recs2 = not_sync_recs2.loc[not_sync_recs2['_merge']=='left_only',:]\n", - "not_sync_recs2 = not_sync_recs2.sort_values(by='session_date', ascending=False)\n", - "not_sync_recs2['rec_key'] = not_sync_recs2['recording_id'].apply(get_rec_key_dict)\n", + "not_sync_recs2 = pd.merge(all_recs, not_sync_recs, how=\"left\", indicator=True)\n", + "not_sync_recs2 = not_sync_recs2.loc[not_sync_recs2[\"_merge\"] == \"left_only\", :]\n", + "not_sync_recs2 = not_sync_recs2.sort_values(by=\"session_date\", ascending=False)\n", + "not_sync_recs2[\"rec_key\"] = not_sync_recs2[\"recording_id\"].apply(get_rec_key_dict)\n", "not_sync_recs2 = not_sync_recs2.reset_index(drop=True)\n", "not_sync_recs2" ] @@ -482,8 +495,8 @@ } ], "source": [ - "#for i in range(2):\n", - "ep.BehaviorSync.populate(not_sync_recs2.loc[8,'rec_key'])" + "# for i in range(2):\n", + "ep.BehaviorSync.populate(not_sync_recs2.loc[8, \"rec_key\"])" ] }, { @@ -53824,7 +53837,7 @@ ], "source": [ "lo = (ep.BehaviorSync & \"recording_id=502\").fetch(as_dict=True)\n", - "lo[0]['sync_data']" + "lo[0][\"sync_data\"]" ] }, { @@ -53844,7 +53857,7 @@ } ], "source": [ - "lo[0]['sync_data'].keys()" + "lo[0][\"sync_data\"].keys()" ] }, { diff --git a/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb b/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb index 5952360b..11382e59 100644 --- a/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb +++ b/notebooks/ephys_element/ephys_sync_notebooks/reingest_sync_by_virmen_block.ipynb @@ -70,10 +70,10 @@ } ], "source": [ - "key = {'subject_fullname': \"jk8386_jknpx6\", 'session_date': \"2025-11-24\"}\n", + "key = {\"subject_fullname\": \"jk8386_jknpx6\", \"session_date\": \"2025-11-24\"}\n", "\n", "rid = (recording.Recording.BehaviorSession & key).fetch(as_dict=True)[0]\n", - "key = {'recording_id': rid['recording_id']}\n", + "key = {\"recording_id\": rid[\"recording_id\"]}\n", "key" ] }, @@ -83,7 +83,7 @@ "metadata": {}, "outputs": [], "source": [ - "key = {'recording_id': 309}" + "key = {\"recording_id\": 309}" ] }, { @@ -174,8 +174,10 @@ } ], "source": [ - "block = 1 # block numbering starts at 1. Set to whatever block you want to use as the virmen_start time.\n", - "ep.BehaviorSync.populate(key, make_kwargs={'block': block}) # when block is passed, it only starts the session at that specific block." + "block = 1 # block numbering starts at 1. Set to whatever block you want to use as the virmen_start time.\n", + "ep.BehaviorSync.populate(\n", + " key, make_kwargs={\"block\": block}\n", + ") # when block is passed, it only starts the session at that specific block." ] }, { @@ -204,7 +206,7 @@ } ], "source": [ - "np.where(np.isnan(lo[0]['iteration_index_nidq']))[0].shape" + "np.where(np.isnan(lo[0][\"iteration_index_nidq\"]))[0].shape" ] }, { diff --git a/notebooks/ephys_element/read_ephys_parameters.ipynb b/notebooks/ephys_element/read_ephys_parameters.ipynb index 850d58cc..36524143 100644 --- a/notebooks/ephys_element/read_ephys_parameters.ipynb +++ b/notebooks/ephys_element/read_ephys_parameters.ipynb @@ -181,7 +181,7 @@ } ], "source": [ - "all_params.loc[0, 'params']" + "all_params.loc[0, \"params\"]" ] } ], diff --git a/notebooks/ephys_element/test_paramspy_for_os.ipynb b/notebooks/ephys_element/test_paramspy_for_os.ipynb index 1db3d8f7..2c91b0d3 100644 --- a/notebooks/ephys_element/test_paramspy_for_os.ipynb +++ b/notebooks/ephys_element/test_paramspy_for_os.ipynb @@ -48,11 +48,13 @@ "metadata": {}, "outputs": [], "source": [ - "df_path = pd.DataFrame((lab.Path & \"global_path='/braininit'\").fetch('system', 'local_path',as_dict=True))\n", + "df_path = pd.DataFrame(\n", + " (lab.Path & \"global_path='/braininit'\").fetch(\"system\", \"local_path\", as_dict=True)\n", + ")\n", "\n", - "linux_path = df_path.loc[df_path['system']=='linux','local_path'].values[0]\n", - "windows_path = df_path.loc[df_path['system']=='windows','local_path'].values[0]\n", - "mac_path = df_path.loc[df_path['system']=='mac','local_path'].values[0]" + "linux_path = df_path.loc[df_path[\"system\"] == \"linux\", \"local_path\"].values[0]\n", + "windows_path = df_path.loc[df_path[\"system\"] == \"windows\", \"local_path\"].values[0]\n", + "mac_path = df_path.loc[df_path[\"system\"] == \"mac\", \"local_path\"].values[0]" ] }, { @@ -218,10 +220,14 @@ ], "source": [ "rec_query = dict()\n", - "rec_query['recording_modality'] = 'electrophysiology'\n", + "rec_query[\"recording_modality\"] = \"electrophysiology\"\n", "rec_query_2 = \"recording_id > 230\"\n", "\n", - "rec_data = pd.DataFrame((recording.Recording * rp.Processing & rec_query & rec_query_2).fetch('recording_process_post_path',as_dict=True))\n", + "rec_data = pd.DataFrame(\n", + " (recording.Recording * rp.Processing & rec_query & rec_query_2).fetch(\n", + " \"recording_process_post_path\", as_dict=True\n", + " )\n", + ")\n", "rec_data" ] }, @@ -240,33 +246,29 @@ } ], "source": [ - "\n", "for i in range(rec_data.shape[0]):\n", - " proc_dir = rec_data.loc[i,'recording_process_post_path']\n", - " output_dir = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][1],proc_dir)\n", + " proc_dir = rec_data.loc[i, \"recording_process_post_path\"]\n", + " output_dir = pathlib.Path(dj.config[\"custom\"][\"ephys_root_data_dir\"][1], proc_dir)\n", "\n", - " kilo_output = output_dir.glob('*kilosort*')\n", + " kilo_output = output_dir.glob(\"*kilosort*\")\n", " for kilo_dir in kilo_output:\n", - " params_file = pathlib.Path(kilo_dir,'params.py')\n", + " params_file = pathlib.Path(kilo_dir, \"params.py\")\n", " if params_file.is_file():\n", - " \n", " with open(params_file.as_posix()) as file:\n", " params_text = file.read()\n", - " \n", + "\n", " try:\n", - " windows_params = params_text.replace(linux_path,windows_path)\n", - " windows_param_file = pathlib.Path(kilo_dir,'win_params.py')\n", + " windows_params = params_text.replace(linux_path, windows_path)\n", + " windows_param_file = pathlib.Path(kilo_dir, \"win_params.py\")\n", " with open(windows_param_file, \"w\") as file:\n", " file.write(windows_params)\n", "\n", - " mac_params = params_text.replace(linux_path,mac_path)\n", - " mac_param_file = pathlib.Path(kilo_dir,'mac_params.py')\n", + " mac_params = params_text.replace(linux_path, mac_path)\n", + " mac_param_file = pathlib.Path(kilo_dir, \"mac_params.py\")\n", " with open(mac_param_file, \"w\") as file:\n", " file.write(mac_params)\n", " except Exception as e:\n", - " print(e)\n", - "\n", - "\n" + " print(e)" ] }, { diff --git a/notebooks/fetch_session_data.ipynb b/notebooks/fetch_session_data.ipynb index 986a0093..ba28308b 100644 --- a/notebooks/fetch_session_data.ipynb +++ b/notebooks/fetch_session_data.ipynb @@ -37,8 +37,8 @@ "import numpy as np\n", "import pandas as pd\n", "\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')\n" + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -1118,9 +1118,9 @@ ], "source": [ "key = dict()\n", - "key['subject_fullname'] = 'ms81_M005'\n", - "#key = \"subject_fullname like 'emdia%'\" \n", - "#key['session_date'] = '2021-05-05'\n", + "key[\"subject_fullname\"] = \"ms81_M005\"\n", + "# key = \"subject_fullname like 'emdia%'\"\n", + "# key['session_date'] = '2021-05-05'\n", "\n", "print(key)\n", "\n", @@ -1518,10 +1518,15 @@ ], "source": [ "blocktable = behavior.TowersBlock() * behavior.TowersBlock.Trial()\n", - "#Remove position, velocity fields because they take a lot to load\n", - "all_fields = pd.DataFrame.from_dict(blocktable.heading.attributes, orient='index')\n", - "sel_fields = all_fields.loc[~all_fields['name'].isin(['position', 'velocity', 'sensor_dots', 'collision', 'trial_time']), :]\n", - "sel_fields = sel_fields['name'].tolist()\n", + "# Remove position, velocity fields because they take a lot to load\n", + "all_fields = pd.DataFrame.from_dict(blocktable.heading.attributes, orient=\"index\")\n", + "sel_fields = all_fields.loc[\n", + " ~all_fields[\"name\"].isin(\n", + " [\"position\", \"velocity\", \"sensor_dots\", \"collision\", \"trial_time\"]\n", + " ),\n", + " :,\n", + "]\n", + "sel_fields = sel_fields[\"name\"].tolist()\n", "\n", "\n", "all_trials_df = pd.DataFrame((blocktable & key).fetch(*sel_fields, as_dict=True))\n", @@ -1534,14 +1539,22 @@ "metadata": {}, "outputs": [], "source": [ - "all_trials_df['towers_left'] = all_trials_df.apply(lambda x: np.sum(x['cue_presence_left'][0]),axis=1)\n", - "all_trials_df['towers_right'] = all_trials_df.apply(lambda x: np.sum(x['cue_presence_right'][0]),axis=1)\n", - "all_trials_df['stimulus'] = all_trials_df['towers_left'] - all_trials_df['towers_right']\n", - "all_trials_df['choice'] = all_trials_df['choice'].replace({'L': 0, 'R': 1, 'nil': -1})\n", - "all_trials_df['trial_type'] = all_trials_df['trial_type'].replace({'L': 0, 'R': 1, 'nil': -1})\n", + "all_trials_df[\"towers_left\"] = all_trials_df.apply(\n", + " lambda x: np.sum(x[\"cue_presence_left\"][0]), axis=1\n", + ")\n", + "all_trials_df[\"towers_right\"] = all_trials_df.apply(\n", + " lambda x: np.sum(x[\"cue_presence_right\"][0]), axis=1\n", + ")\n", + "all_trials_df[\"stimulus\"] = all_trials_df[\"towers_left\"] - all_trials_df[\"towers_right\"]\n", + "all_trials_df[\"choice\"] = all_trials_df[\"choice\"].replace({\"L\": 0, \"R\": 1, \"nil\": -1})\n", + "all_trials_df[\"trial_type\"] = all_trials_df[\"trial_type\"].replace(\n", + " {\"L\": 0, \"R\": 1, \"nil\": -1}\n", + ")\n", "\n", - "all_trials_df = all_trials_df.loc[(all_trials_df['main_level'] == 11) & (all_trials_df['level'] == 11), :]\n", - "all_trials_df = all_trials_df.reset_index(drop=True)\n" + "all_trials_df = all_trials_df.loc[\n", + " (all_trials_df[\"main_level\"] == 11) & (all_trials_df[\"level\"] == 11), :\n", + "]\n", + "all_trials_df = all_trials_df.reset_index(drop=True)" ] }, { @@ -2055,7 +2068,7 @@ } ], "source": [ - "all_trials_df[['cue_presence_left', 'cue_presence_right']]" + "all_trials_df[[\"cue_presence_left\", \"cue_presence_right\"]]" ] }, { @@ -2086,7 +2099,7 @@ } ], "source": [ - "all_trials_df['stimulus']" + "all_trials_df[\"stimulus\"]" ] }, { @@ -3106,7 +3119,7 @@ } ], "source": [ - "all_trials_df['stimulus'].tolist()" + "all_trials_df[\"stimulus\"].tolist()" ] }, { diff --git a/notebooks/imaging_element/00-Set-up-configuration.ipynb b/notebooks/imaging_element/00-Set-up-configuration.ipynb index d63a8c5d..0bea86fe 100644 --- a/notebooks/imaging_element/00-Set-up-configuration.ipynb +++ b/notebooks/imaging_element/00-Set-up-configuration.ipynb @@ -94,9 +94,11 @@ "outputs": [], "source": [ "# If there is only one root path:\n", - "dj.config['custom']['imaging_root_data_dir'] = utility.get_network_path('braininit') + '/RigData/mesoscope/imaging/'\n", + "dj.config[\"custom\"][\"imaging_root_data_dir\"] = (\n", + " utility.get_network_path(\"braininit\") + \"/RigData/mesoscope/imaging/\"\n", + ")\n", "# If there are multiple possible root paths (write them as a list):\n", - "#dj.config['custom']['imaging_root_data_dir'] = ['/mnt/bucket/braininit/RigData/mesoscope/imaging/imaging_pipe1/']" + "# dj.config['custom']['imaging_root_data_dir'] = ['/mnt/bucket/braininit/RigData/mesoscope/imaging/imaging_pipe1/']" ] }, { @@ -117,7 +119,7 @@ } ], "source": [ - "dj.config['custom']" + "dj.config[\"custom\"]" ] }, { diff --git a/notebooks/imaging_element/03-imaging-data-analysis.ipynb b/notebooks/imaging_element/03-imaging-data-analysis.ipynb index de10beda..4f4fef33 100644 --- a/notebooks/imaging_element/03-imaging-data-analysis.ipynb +++ b/notebooks/imaging_element/03-imaging-data-analysis.ipynb @@ -55,7 +55,6 @@ } ], "source": [ - "\n", "import datajoint as dj\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", @@ -64,8 +63,8 @@ "\n", "from u19_pipeline import subject\n", "\n", - "imaging = dj.create_virtual_module('imaging', 'u19_imaging')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "imaging = dj.create_virtual_module(\"imaging\", \"u19_imaging\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -81,10 +80,10 @@ "metadata": {}, "outputs": [], "source": [ - "subject = 'jeremyjc_j016'\n", - "date = '2021-11-21'\n", + "subject = \"jeremyjc_j016\"\n", + "date = \"2021-11-21\"\n", "\n", - "key = (imaging.Scan & dict(session_date =date, subject_fullname=subject)).fetch1('KEY')" + "key = (imaging.Scan & dict(session_date=date, subject_fullname=subject)).fetch1(\"KEY\")" ] }, { @@ -100,10 +99,23 @@ "metadata": {}, "outputs": [], "source": [ - "cols_sync = ['sync_im_frame', 'sync_im_frame_global', 'sync_behav_block_by_im_frame', 'sync_behav_trial_by_im_frame', 'sync_behav_iter_by_im_frame', 'sync_im_frame_span_by_behav_block'] \n", - "sync_data = pd.DataFrame((imaging.SyncImagingBehavior & key).fetch(*cols_sync, as_dict=True))\n", - "mask_data = pd.DataFrame((imaging_element.MaskClassification.MaskType & key).fetch(as_dict=True))\n", - "trace_data = pd.DataFrame((imaging_element.Fluorescence.Trace & key).fetch(as_dict=True))\n", + "cols_sync = [\n", + " \"sync_im_frame\",\n", + " \"sync_im_frame_global\",\n", + " \"sync_behav_block_by_im_frame\",\n", + " \"sync_behav_trial_by_im_frame\",\n", + " \"sync_behav_iter_by_im_frame\",\n", + " \"sync_im_frame_span_by_behav_block\",\n", + "]\n", + "sync_data = pd.DataFrame(\n", + " (imaging.SyncImagingBehavior & key).fetch(*cols_sync, as_dict=True)\n", + ")\n", + "mask_data = pd.DataFrame(\n", + " (imaging_element.MaskClassification.MaskType & key).fetch(as_dict=True)\n", + ")\n", + "trace_data = pd.DataFrame(\n", + " (imaging_element.Fluorescence.Trace & key).fetch(as_dict=True)\n", + ")\n", "behavior_data = pd.DataFrame((behavior.TowersBlock.Trial & key).fetch(as_dict=True))" ] }, @@ -144,12 +156,16 @@ } ], "source": [ - "#Which mask shall we use for plots\n", + "# Which mask shall we use for plots\n", "idx_mask = 1\n", "\n", - "mask_data = mask_data.sort_values(by=['confidence'], ascending=[False])\n", + "mask_data = mask_data.sort_values(by=[\"confidence\"], ascending=[False])\n", "mask_data = mask_data.reset_index(drop=True)\n", - "plt.plot(trace_data.loc[trace_data['mask'] == mask_data.loc[idx_mask, 'mask'], 'fluorescence'].values[0])" + "plt.plot(\n", + " trace_data.loc[\n", + " trace_data[\"mask\"] == mask_data.loc[idx_mask, \"mask\"], \"fluorescence\"\n", + " ].values[0]\n", + ")" ] }, { @@ -165,25 +181,33 @@ "metadata": {}, "outputs": [], "source": [ - "blocks_imaging_data = np.unique(sync_data.loc[0, 'sync_behav_block_by_im_frame'])\n", + "blocks_imaging_data = np.unique(sync_data.loc[0, \"sync_behav_block_by_im_frame\"])\n", "blocks_imaging_data = np.delete(blocks_imaging_data, 0)\n", "\n", - "trials_block = behavior_data[['block', 'trial_idx']].copy()\n", - "trials_block = trials_block.loc[trials_block['block'].isin(blocks_imaging_data), :]\n", - "trials_block_group = trials_block.groupby(by=['block']).count()\n", + "trials_block = behavior_data[[\"block\", \"trial_idx\"]].copy()\n", + "trials_block = trials_block.loc[trials_block[\"block\"].isin(blocks_imaging_data), :]\n", + "trials_block_group = trials_block.groupby(by=[\"block\"]).count()\n", "trials_block_group = trials_block_group.reset_index()\n", - "trials_block_group.sort_values(by=['trial_idx'], ascending=[False])\n", + "trials_block_group.sort_values(by=[\"trial_idx\"], ascending=[False])\n", "trials_block_group.reset_index(drop=True)\n", - "sel_block = trials_block_group.loc[0, 'block']\n", + "sel_block = trials_block_group.loc[0, \"block\"]\n", "\n", - "trials_sel_block = behavior_data.loc[behavior_data['block'] == sel_block, :]\n", + "trials_sel_block = behavior_data.loc[behavior_data[\"block\"] == sel_block, :]\n", "trials_sel_block = trials_sel_block.reset_index(drop=True)\n", "\n", - "#Samples imaging of the selected block\n", - "idx_samples_block = np.where(np.squeeze(sync_data.loc[0, 'sync_behav_block_by_im_frame']) == sel_block)\n", - "imaging_trial_data_idx = np.squeeze(sync_data.loc[0, 'sync_behav_trial_by_im_frame'])[idx_samples_block]\n", - "imaging_iter_data_idx = np.squeeze(sync_data.loc[0, 'sync_behav_iter_by_im_frame'])[idx_samples_block]\n", - "fluorescence_data = trace_data.loc[trace_data['mask'] == mask_data.loc[idx_mask, 'mask'], 'fluorescence'].values[0][idx_samples_block]\n" + "# Samples imaging of the selected block\n", + "idx_samples_block = np.where(\n", + " np.squeeze(sync_data.loc[0, \"sync_behav_block_by_im_frame\"]) == sel_block\n", + ")\n", + "imaging_trial_data_idx = np.squeeze(sync_data.loc[0, \"sync_behav_trial_by_im_frame\"])[\n", + " idx_samples_block\n", + "]\n", + "imaging_iter_data_idx = np.squeeze(sync_data.loc[0, \"sync_behav_iter_by_im_frame\"])[\n", + " idx_samples_block\n", + "]\n", + "fluorescence_data = trace_data.loc[\n", + " trace_data[\"mask\"] == mask_data.loc[idx_mask, \"mask\"], \"fluorescence\"\n", + "].values[0][idx_samples_block]" ] }, { @@ -199,34 +223,35 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "trace_vec = list()\n", "position_vec = list()\n", "for i in range(trials_sel_block.shape[0]):\n", - "\n", - " idx_trial = np.where(imaging_trial_data_idx == i+1)\n", + " idx_trial = np.where(imaging_trial_data_idx == i + 1)\n", " iters_trial = imaging_iter_data_idx[idx_trial]\n", " fluorescence_trial = fluorescence_data[idx_trial]\n", - " position_vec_trial = trials_sel_block.loc[trials_sel_block['trial_idx'] == i+1, 'position'].values[0]\n", - " sensor_dots = trials_sel_block.loc[trials_sel_block['trial_idx'] == i+1, 'sensor_dots'].values[0]\n", - " iter_in_trial = trials_sel_block.loc[trials_sel_block['trial_idx'] == i+1, 'iterations'].values[0]\n", + " position_vec_trial = trials_sel_block.loc[\n", + " trials_sel_block[\"trial_idx\"] == i + 1, \"position\"\n", + " ].values[0]\n", + " sensor_dots = trials_sel_block.loc[\n", + " trials_sel_block[\"trial_idx\"] == i + 1, \"sensor_dots\"\n", + " ].values[0]\n", + " iter_in_trial = trials_sel_block.loc[\n", + " trials_sel_block[\"trial_idx\"] == i + 1, \"iterations\"\n", + " ].values[0]\n", "\n", " unique_iter = np.unique(iters_trial)\n", "\n", " for idx, iter in enumerate(unique_iter):\n", - "\n", - "\n", - "\n", " if iter <= iter_in_trial:\n", " curr_iter = np.where(iters_trial == iter)\n", " trace_iter = fluorescence_trial[curr_iter[0]]\n", - " \n", - " trace_vec.append(np.mean(trace_iter)) \n", - " position_vec.append(position_vec_trial[int(iter)-1,1])\n", + "\n", + " trace_vec.append(np.mean(trace_iter))\n", + " position_vec.append(position_vec_trial[int(iter) - 1, 1])\n", "\n", "\n", - "trace_vec = np.asarray(trace_vec)\n", - "position_vec = np.asarray(position_vec)\n" + "trace_vec = np.asarray(trace_vec)\n", + "position_vec = np.asarray(position_vec)" ] }, { diff --git a/notebooks/imaging_element/getSyncInfo_python.ipynb b/notebooks/imaging_element/getSyncInfo_python.ipynb index 182075df..da60ae01 100644 --- a/notebooks/imaging_element/getSyncInfo_python.ipynb +++ b/notebooks/imaging_element/getSyncInfo_python.ipynb @@ -45,7 +45,7 @@ "metadata": {}, "outputs": [], "source": [ - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -63,7 +63,7 @@ "def get_sync_info(input_file, data_type, frame_skip):\n", " if data_type == \"uint16\":\n", " num_bytes = 2\n", - " struct_fmt = ' 0 else [])\n", " else:\n", " if len(d) != n_data:\n", - " raise ValueError(f\"Inconsistent number of I2CData entries (expected {n_data}, got {len(d)}).\")\n", + " raise ValueError(\n", + " f\"Inconsistent number of I2CData entries (expected {n_data}, got {len(d)}).\"\n", + " )\n", " final_data.append(d)\n", "\n", " epoch = np.array([epoch], dtype=np.float64)\n", @@ -186,7 +200,7 @@ "metadata": {}, "outputs": [], "source": [ - "imaging_root_dir = dj.config['custom']['imaging_root_data_dir'][0]" + "imaging_root_dir = dj.config[\"custom\"][\"imaging_root_data_dir\"][0]" ] }, { @@ -388,7 +402,11 @@ } ], "source": [ - "imaging_sessions = pd.DataFrame((recording.Recording.BehaviorSession * ip.ImagingPipelineSession).fetch(order_by='session_date desc',as_dict=True))\n", + "imaging_sessions = pd.DataFrame(\n", + " (recording.Recording.BehaviorSession * ip.ImagingPipelineSession).fetch(\n", + " order_by=\"session_date desc\", as_dict=True\n", + " )\n", + ")\n", "imaging_sessions" ] }, @@ -411,10 +429,10 @@ } ], "source": [ - "key = imaging_sessions.loc[0,:].to_dict()\n", + "key = imaging_sessions.loc[0, :].to_dict()\n", "key\n", "img_key = dict()\n", - "img_key['recording_id'] = key.pop('recording_id')\n", + "img_key[\"recording_id\"] = key.pop(\"recording_id\")\n", "session_key = key\n", "session_key" ] @@ -436,7 +454,7 @@ } ], "source": [ - "imaging_dir = (recording.Recording & img_key).fetch1('recording_directory')\n", + "imaging_dir = (recording.Recording & img_key).fetch1(\"recording_directory\")\n", "imaging_dir = pathlib.Path(imaging_root_dir, imaging_dir)\n", "imaging_dir" ] @@ -544,7 +562,9 @@ } ], "source": [ - "imaging_files = [file.as_posix() for file in imaging_dir.rglob(\"*.tif\") if file.is_file()]\n", + "imaging_files = [\n", + " file.as_posix() for file in imaging_dir.rglob(\"*.tif\") if file.is_file()\n", + "]\n", "imaging_files.sort()\n", "imaging_files" ] @@ -573,8 +593,8 @@ ], "source": [ "behavior_data = (behavior.SpatialTimeBlobs & session_key).fetch(as_dict=True)\n", - "session_time = behavior_data[0]['cumulative_session_time'].flatten()\n", - "beh_iteration_matrix = behavior_data[0]['iteration_matrix']\n", + "session_time = behavior_data[0][\"cumulative_session_time\"].flatten()\n", + "beh_iteration_matrix = behavior_data[0][\"iteration_matrix\"]\n", "beh_iteration_matrix" ] }, @@ -592,7 +612,9 @@ } ], "source": [ - "path = (acquisition.SessionStarted & session_key).fetch1('new_remote_path_behavior_file')\n", + "path = (acquisition.SessionStarted & session_key).fetch1(\n", + " \"new_remote_path_behavior_file\"\n", + ")\n", "mat_file = lab.Path().get_local_path2(path)\n", "df_behavior_file = mu.convert_behavior_file(mat_file)" ] @@ -639,7 +661,7 @@ } ], "source": [ - "data['log']" + "data[\"log\"]" ] }, { @@ -648,7 +670,7 @@ "metadata": {}, "outputs": [], "source": [ - "input_file = '/mnt/cup/braininit/Data/Raw/imaging/efonseca/efonseca_ef932_act131/20260518_g1/05182026/ef932_act131_ppc_05182026b_00003_00001.tif'" + "input_file = \"/mnt/cup/braininit/Data/Raw/imaging/efonseca/efonseca_ef932_act131/20260518_g1/05182026/ef932_act131_ppc_05182026b_00003_00001.tif\"" ] }, { diff --git a/notebooks/imaging_element/read_imaging_parameters.ipynb b/notebooks/imaging_element/read_imaging_parameters.ipynb index 4bb5363c..d9f2f28b 100644 --- a/notebooks/imaging_element/read_imaging_parameters.ipynb +++ b/notebooks/imaging_element/read_imaging_parameters.ipynb @@ -59,7 +59,7 @@ } ], "source": [ - "so = np.load('suite2p_volum_params_05162026.npy', allow_pickle=True)\n", + "so = np.load(\"suite2p_volum_params_05162026.npy\", allow_pickle=True)\n", "so" ] }, @@ -79,7 +79,7 @@ "outputs": [], "source": [ "keyo = dict()\n", - "keyo['recording_id'] = 678" + "keyo[\"recording_id\"] = 678" ] }, { @@ -326,7 +326,7 @@ } ], "source": [ - "suite2p_params = all_params.loc[6, 'params']\n", + "suite2p_params = all_params.loc[6, \"params\"]\n", "suite2p_params" ] }, @@ -336,7 +336,7 @@ "metadata": {}, "outputs": [], "source": [ - "suite2p_params['fs'] = 50\n" + "suite2p_params[\"fs\"] = 50" ] }, { @@ -345,7 +345,7 @@ "metadata": {}, "outputs": [], "source": [ - "suite2p_params['neuropil_extract'] = False\n" + "suite2p_params[\"neuropil_extract\"] = False" ] }, { @@ -354,7 +354,9 @@ "metadata": {}, "outputs": [], "source": [ - "ip.imaging_element.ProcessingParamSet.insert_new_params(all_params.loc[2, 'processing_method'],6,\"182-suite2p-params-2026-5\",lo)" + "ip.imaging_element.ProcessingParamSet.insert_new_params(\n", + " all_params.loc[2, \"processing_method\"], 6, \"182-suite2p-params-2026-5\", lo\n", + ")" ] }, { @@ -363,7 +365,12 @@ "metadata": {}, "outputs": [], "source": [ - "ip.imaging_element.ProcessingParamSet.insert_new_params(all_params.loc[2, 'processing_method'],4,\"182-suite2p-params-2026-3\",suite2p_params)" + "ip.imaging_element.ProcessingParamSet.insert_new_params(\n", + " all_params.loc[2, \"processing_method\"],\n", + " 4,\n", + " \"182-suite2p-params-2026-3\",\n", + " suite2p_params,\n", + ")" ] }, { diff --git a/notebooks/imaging_element/read_params.py b/notebooks/imaging_element/read_params.py index 7536b791..9c8c0ba3 100644 --- a/notebooks/imaging_element/read_params.py +++ b/notebooks/imaging_element/read_params.py @@ -1,4 +1,3 @@ - import os from scipy.io import savemat @@ -10,17 +9,27 @@ dj.conn() -ephys_element =dj.create_virtual_module('u19_pipeline_ephys_element','u19_pipeline_ephys_element') -imaging_element =dj.create_virtual_module('u19_pipeline_imaging_element','u19_pipeline_imaging_element') +ephys_element = dj.create_virtual_module( + "u19_pipeline_ephys_element", "u19_pipeline_ephys_element" +) +imaging_element = dj.create_virtual_module( + "u19_pipeline_imaging_element", "u19_pipeline_imaging_element" +) -modalities = ['electrophysiology', 'imaging'] +modalities = ["electrophysiology", "imaging"] -params_tables = [ephys_element.ClusteringParamSet, imaging_element.ProcessingParamSet] -preparams_steps_tables = [(ephys_element.PreClusterParamSteps * ephys_element.PreClusterParamSteps.Step * ephys_element.PreClusterParamSet)] -preparams_tables = [ephys_element.PreClusterParamSet] +params_tables = [ephys_element.ClusteringParamSet, imaging_element.ProcessingParamSet] +preparams_steps_tables = [ + ( + ephys_element.PreClusterParamSteps + * ephys_element.PreClusterParamSteps.Step + * ephys_element.PreClusterParamSet + ) +] +preparams_tables = [ephys_element.PreClusterParamSet] -#method_tables = [ephys_element.ClusteringMethod, imaging_element.ProcessingMethod] -#pre_method_tables = [ephys_element.PreClusterMethod] +# method_tables = [ephys_element.ClusteringMethod, imaging_element.ProcessingMethod] +# pre_method_tables = [ephys_element.PreClusterMethod] ###################################Fetch all params from all modalities params_dict_list = [] @@ -28,42 +37,46 @@ params_dict_list.append(table.fetch(as_dict=True)) -#Append all params in the same dictionary +# Append all params in the same dictionary params_dict_dict = {} num_params = 0 for idx, param_modality_list in enumerate(params_dict_list): for dict in param_modality_list: - dict['recording_modality'] = modalities[idx] - dict['param_set_hash'] = str(dict['param_set_hash']) - if 'clustering_method' in dict: - dict['processing_method'] = dict.pop('clustering_method') - - params_dict_dict['param_'+str(num_params)] = dict - num_params +=1 + dict["recording_modality"] = modalities[idx] + dict["param_set_hash"] = str(dict["param_set_hash"]) + if "clustering_method" in dict: + dict["processing_method"] = dict.pop("clustering_method") + + params_dict_dict["param_" + str(num_params)] = dict + num_params += 1 #################################################Fetch all preparamsStepList from all modalities preparams_steps = [] for table in preparams_steps_tables: preparams_steps.append(table.fetch(as_dict=True)) -#Append all preparams in the same dictionary +# Append all preparams in the same dictionary preparams_steps_dict_dict = {} num_preparams_steps = 0 for idx, preparam_modality_list in enumerate(preparams_steps): for dict in preparam_modality_list: - dict['param_set_hash'] = str(dict['param_set_hash']) - dict['recording_modality'] = modalities[idx] - if 'precluster_param_steps_id' in dict: - dict['preprocess_param_steps_id'] = dict.pop('precluster_param_steps_id') - if 'precluster_method' in dict: - dict['preprocess_method'] = dict.pop('precluster_method') - if 'precluster_param_steps_name' in dict: - dict['preprocess_param_steps_name'] = dict.pop('precluster_param_steps_name') - if 'precluster_param_steps_desc' in dict: - dict['preprocess_param_steps_desc'] = dict.pop('precluster_param_steps_desc') - - preparams_steps_dict_dict['param_'+str(num_preparams_steps)] = dict - num_preparams_steps +=1 + dict["param_set_hash"] = str(dict["param_set_hash"]) + dict["recording_modality"] = modalities[idx] + if "precluster_param_steps_id" in dict: + dict["preprocess_param_steps_id"] = dict.pop("precluster_param_steps_id") + if "precluster_method" in dict: + dict["preprocess_method"] = dict.pop("precluster_method") + if "precluster_param_steps_name" in dict: + dict["preprocess_param_steps_name"] = dict.pop( + "precluster_param_steps_name" + ) + if "precluster_param_steps_desc" in dict: + dict["preprocess_param_steps_desc"] = dict.pop( + "precluster_param_steps_desc" + ) + + preparams_steps_dict_dict["param_" + str(num_preparams_steps)] = dict + num_preparams_steps += 1 #################################################Fetch all preparams from all modalities preparams_dict_list = [] @@ -71,20 +84,20 @@ preparams_dict_list.append(table.fetch(as_dict=True)) -#Append all preparams in the same dictionary +# Append all preparams in the same dictionary preparams_dict_dict = {} num_preparams = 0 for idx, preparam_modality_list in enumerate(preparams_dict_list): for dict in preparam_modality_list: - dict['recording_modality'] = modalities[idx] - dict['param_set_hash'] = str(dict['param_set_hash']) - if 'precluster_method' in dict: - dict['preprocess_method'] = dict.pop('precluster_method') - - preparams_dict_dict['param_'+str(num_preparams)] = dict - num_preparams +=1 + dict["recording_modality"] = modalities[idx] + dict["param_set_hash"] = str(dict["param_set_hash"]) + if "precluster_method" in dict: + dict["preprocess_method"] = dict.pop("precluster_method") + + preparams_dict_dict["param_" + str(num_preparams)] = dict + num_preparams += 1 -''' +""" #################################################Fetch all methods from all modalities all_methods_data = [] for table in method_tables: @@ -116,14 +129,14 @@ premethods_dict['premethod_'+str(num_preparams_steps)] = dict num_premethods +=1 -''' +""" dj.conn().close() -savemat('params.mat', params_dict_dict) -savemat('preparams.mat', preparams_dict_dict) -savemat('preparams_list.mat', preparams_steps_dict_dict) +savemat("params.mat", params_dict_dict) +savemat("preparams.mat", preparams_dict_dict) +savemat("preparams_list.mat", preparams_steps_dict_dict) -#savemat('methods.mat', methods_dict) -#savemat('premethods.mat', premethods_dict) +# savemat('methods.mat', methods_dict) +# savemat('premethods.mat', premethods_dict) diff --git a/notebooks/imaging_element/test_Matlab_Python_conv.ipynb b/notebooks/imaging_element/test_Matlab_Python_conv.ipynb index 10b33689..2879c21e 100644 --- a/notebooks/imaging_element/test_Matlab_Python_conv.ipynb +++ b/notebooks/imaging_element/test_Matlab_Python_conv.ipynb @@ -40,7 +40,7 @@ "\n", "import u19_pipeline.imaging_pipeline as ip\n", "import u19_pipeline.utils.tiff_matlab_imaging_utils as tmiu\n", - "import u19_pipeline.utils.tiff_utils as tu\n" + "import u19_pipeline.utils.tiff_utils as tu" ] }, { @@ -73,7 +73,7 @@ "metadata": {}, "outputs": [], "source": [ - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -116,7 +116,7 @@ } ], "source": [ - "acquired_tiff_keys = ip.AcquiredTiff.fetch('KEY')\n", + "acquired_tiff_keys = ip.AcquiredTiff.fetch(\"KEY\")\n", "acquired_tiff_keys" ] }, @@ -138,8 +138,8 @@ ], "source": [ "imaging_key = dict()\n", - "imaging_key['recording_id'] = 684\n", - "#imaging_key = acquired_tiff_keys[16].copy()\n", + "imaging_key[\"recording_id\"] = 684\n", + "# imaging_key = acquired_tiff_keys[16].copy()\n", "imaging_key\n", "original_imaging_key = imaging_key.copy()\n", "original_imaging_key" @@ -238,7 +238,7 @@ "outputs": [], "source": [ "acquired_tiff_entry = (ip.AcquiredTiff & original_imaging_key).fetch(as_dict=True)\n", - "tiff_split_entry = (ip.TiffSplit & original_imaging_key).fetch(as_dict=True)\n", + "tiff_split_entry = (ip.TiffSplit & original_imaging_key).fetch(as_dict=True)\n", "tiff_split_file_entry = (ip.TiffSplit.File & original_imaging_key).fetch(as_dict=True)" ] }, diff --git a/notebooks/imaging_storage_calculator.ipynb b/notebooks/imaging_storage_calculator.ipynb index 72bfe6b4..8caf76ad 100644 --- a/notebooks/imaging_storage_calculator.ipynb +++ b/notebooks/imaging_storage_calculator.ipynb @@ -38,7 +38,7 @@ "source": [ "import numpy as np\n", "\n", - "(np.round(100.76 * 2)/2)" + "(np.round(100.76 * 2) / 2)" ] }, { @@ -47,7 +47,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "import datajoint as dj\n", "import matplotlib.pyplot as plt\n", "import pandas as pd\n", @@ -85,9 +84,9 @@ } ], "source": [ - "lab = dj.create_virtual_module('lab', 'u19_lab')\n", + "lab = dj.create_virtual_module(\"lab\", \"u19_lab\")\n", "\n", - "lab.SlackWebhooks.fetch(as_dict=True)\n" + "lab.SlackWebhooks.fetch(as_dict=True)" ] }, { @@ -105,11 +104,11 @@ ], "source": [ "a = dict()\n", - "a['s'] = 2\n", - "if 't' not in a:\n", - " print('not here t')\n", - "if 's' not in a:\n", - " print('not s either')" + "a[\"s\"] = 2\n", + "if \"t\" not in a:\n", + " print(\"not here t\")\n", + "if \"s\" not in a:\n", + " print(\"not s either\")" ] }, { @@ -332,9 +331,13 @@ } ], "source": [ - "imaging = dj.create_virtual_module('imaging', 'u19_imaging')\n", - "subject = dj.create_virtual_module('subject', 'u19_subject')\n", - "df_imaging = pd.DataFrame((imaging.Scan * subject.Subject.proj('subject_fullname', 'user_id')).fetch(as_dict=True))\n", + "imaging = dj.create_virtual_module(\"imaging\", \"u19_imaging\")\n", + "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n", + "df_imaging = pd.DataFrame(\n", + " (imaging.Scan * subject.Subject.proj(\"subject_fullname\", \"user_id\")).fetch(\n", + " as_dict=True\n", + " )\n", + ")\n", "df_imaging" ] }, @@ -344,9 +347,11 @@ "metadata": {}, "outputs": [], "source": [ - "#for i in range(df_imaging.shape[0])\n", - "df_imaging['size (TB)'] = df_imaging.apply(lambda x: pu.get_size_directory(x['scan_directory']),axis=1)\n", - "df_imaging['size (TB)'] = df_imaging['size (TB)']/(1024*1024*1024)" + "# for i in range(df_imaging.shape[0])\n", + "df_imaging[\"size (TB)\"] = df_imaging.apply(\n", + " lambda x: pu.get_size_directory(x[\"scan_directory\"]), axis=1\n", + ")\n", + "df_imaging[\"size (TB)\"] = df_imaging[\"size (TB)\"] / (1024 * 1024 * 1024)" ] }, { @@ -355,9 +360,24 @@ "metadata": {}, "outputs": [], "source": [ - "df_imaging['is_raw_imaging'] = df_imaging.apply(lambda x: pu.check_file_pattern_dir(x['scan_directory'], pu.file_patterns_acq['raw_imaging']),axis=1)\n", - "df_imaging['is_commpressed_imaging'] = df_imaging.apply(lambda x: pu.check_file_pattern_dir(x['scan_directory'], pu.file_patterns_acq['commpressed_imaging']),axis=1)\n", - "df_imaging['is_segmented'] = df_imaging.apply(lambda x: pu.check_file_pattern_dir(x['scan_directory'], pu.file_patterns_acq['segmented_imaging_files']),axis=1)" + "df_imaging[\"is_raw_imaging\"] = df_imaging.apply(\n", + " lambda x: pu.check_file_pattern_dir(\n", + " x[\"scan_directory\"], pu.file_patterns_acq[\"raw_imaging\"]\n", + " ),\n", + " axis=1,\n", + ")\n", + "df_imaging[\"is_commpressed_imaging\"] = df_imaging.apply(\n", + " lambda x: pu.check_file_pattern_dir(\n", + " x[\"scan_directory\"], pu.file_patterns_acq[\"commpressed_imaging\"]\n", + " ),\n", + " axis=1,\n", + ")\n", + "df_imaging[\"is_segmented\"] = df_imaging.apply(\n", + " lambda x: pu.check_file_pattern_dir(\n", + " x[\"scan_directory\"], pu.file_patterns_acq[\"segmented_imaging_files\"]\n", + " ),\n", + " axis=1,\n", + ")" ] }, { @@ -381,7 +401,12 @@ "metadata": {}, "outputs": [], "source": [ - "df_imaging['session_type'] = df_imaging.apply(lambda x: session_type(x['is_commpressed_imaging'], x['is_raw_imaging'], x['is_segmented']),axis=1)" + "df_imaging[\"session_type\"] = df_imaging.apply(\n", + " lambda x: session_type(\n", + " x[\"is_commpressed_imaging\"], x[\"is_raw_imaging\"], x[\"is_segmented\"]\n", + " ),\n", + " axis=1,\n", + ")" ] }, { @@ -390,8 +415,8 @@ "metadata": {}, "outputs": [], "source": [ - "df_imaging['session_date'] = df_imaging['session_date'].astype('str')\n", - "df_imaging['year'] = df_imaging['session_date'].str.slice(0,4).astype('int')" + "df_imaging[\"session_date\"] = df_imaging[\"session_date\"].astype(\"str\")\n", + "df_imaging[\"year\"] = df_imaging[\"session_date\"].str.slice(0, 4).astype(\"int\")" ] }, { @@ -400,7 +425,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_imaging = df_imaging.loc[df_imaging['size (TB)'] > 0.001, :]" + "df_imaging = df_imaging.loc[df_imaging[\"size (TB)\"] > 0.001, :]" ] }, { @@ -2503,10 +2528,12 @@ } ], "source": [ - "n, bins, patches = plt.hist(df_imaging['size (TB)'].values , 100, facecolor='g', alpha=0.75)\n", - "plt.xlabel('Size per session (TB)')\n", - "plt.ylabel('Count ')\n", - "plt.title('Histogram of Imaging Sessions')\n", + "n, bins, patches = plt.hist(\n", + " df_imaging[\"size (TB)\"].values, 100, facecolor=\"g\", alpha=0.75\n", + ")\n", + "plt.xlabel(\"Size per session (TB)\")\n", + "plt.ylabel(\"Count \")\n", + "plt.title(\"Histogram of Imaging Sessions\")\n", "plt.grid(True)\n", "plt.show()" ] @@ -2629,7 +2656,6 @@ } ], "source": [ - "\n", "df_imaging[[\"user_id\", \"size (TB)\", \"year\"]].groupby(by=[\"user_id\", \"year\"]).sum()" ] }, @@ -2766,7 +2792,9 @@ } ], "source": [ - "df_imaging[[\"subject_fullname\", \"user_id\", \"session_type\"]].groupby(by=[\"user_id\", \"session_type\"]).count()" + "df_imaging[[\"subject_fullname\", \"user_id\", \"session_type\"]].groupby(\n", + " by=[\"user_id\", \"session_type\"]\n", + ").count()" ] }, { @@ -2989,7 +3017,9 @@ } ], "source": [ - "df_imaging[[\"user_id\", \"session_type\", \"size (TB)\"]].groupby(by=[\"user_id\", \"session_type\"]).mean()\n" + "df_imaging[[\"user_id\", \"session_type\", \"size (TB)\"]].groupby(\n", + " by=[\"user_id\", \"session_type\"]\n", + ").mean()" ] }, { diff --git a/notebooks/live_stats_monitor.ipynb b/notebooks/live_stats_monitor.ipynb index bacd033d..bbdfec85 100644 --- a/notebooks/live_stats_monitor.ipynb +++ b/notebooks/live_stats_monitor.ipynb @@ -23,9 +23,7 @@ "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", "\n", - "try_find_conf_file()\n", - "\n", - "\n" + "try_find_conf_file()" ] }, { @@ -61,7 +59,7 @@ "dj.conn()\n", "\n", "MINUTES_ALERT = 20\n", - "SECONDS_ALERT = MINUTES_ALERT*60\n", + "SECONDS_ALERT = MINUTES_ALERT * 60\n", "MIN_SESSIONS_COMPLETED = 3" ] }, @@ -78,9 +76,9 @@ "metadata": {}, "outputs": [], "source": [ - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "lab = dj.create_virtual_module('lab', 'u19_lab')\n", - "subject = dj.create_virtual_module('subject', 'u19_subject')" + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "lab = dj.create_virtual_module(\"lab\", \"u19_lab\")\n", + "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")" ] }, { @@ -89,9 +87,7 @@ "metadata": {}, "outputs": [], "source": [ - "slack_configuration_dictionary = {\n", - " 'slack_notification_channel': ['alvaro_luna']\n", - "}" + "slack_configuration_dictionary = {\"slack_notification_channel\": [\"alvaro_luna\"]}" ] }, { @@ -100,49 +96,58 @@ "metadata": {}, "outputs": [], "source": [ - "def slack_alert_message_format_live_stats(alert_dictionary1, alert_dictionary2, time_no_response):\n", + "def slack_alert_message_format_live_stats(\n", + " alert_dictionary1, alert_dictionary2, time_no_response\n", + "):\n", "\n", " now = datetime.now()\n", - " datestr = now.strftime('%d-%b-%Y %H:%M:%S')\n", + " datestr = now.strftime(\"%d-%b-%Y %H:%M:%S\")\n", "\n", " msep = dict()\n", - " msep['type'] = \"divider\"\n", + " msep[\"type\"] = \"divider\"\n", "\n", - " #Title#\n", + " # Title#\n", " m1 = dict()\n", - " m1['type'] = 'section'\n", + " m1[\"type\"] = \"section\"\n", " m1_1 = dict()\n", " m1_1[\"type\"] = \"mrkdwn\"\n", - " m1_1[\"text\"] = ':rotating_light: * Live Monitor Alert* on ' + datestr + '\\n' +\\\n", - " 'More than ' + str(int(time_no_response/60)) + ' min without valid new trial' + '\\n'\n", - " m1['text'] = m1_1\n", - "\n", - " #Info#\n", + " m1_1[\"text\"] = (\n", + " \":rotating_light: * Live Monitor Alert* on \"\n", + " + datestr\n", + " + \"\\n\"\n", + " + \"More than \"\n", + " + str(int(time_no_response / 60))\n", + " + \" min without valid new trial\"\n", + " + \"\\n\"\n", + " )\n", + " m1[\"text\"] = m1_1\n", + "\n", + " # Info#\n", " m2 = dict()\n", - " m2['type'] = 'section'\n", + " m2[\"type\"] = \"section\"\n", " m2_1 = dict()\n", " m2_1[\"type\"] = \"mrkdwn\"\n", "\n", - " m2_1[\"text\"] = '*Session Reported:*' + '\\n'\n", + " m2_1[\"text\"] = \"*Session Reported:*\" + \"\\n\"\n", " for key in alert_dictionary1.keys():\n", - " m2_1[\"text\"] += '*' + key + '* : ' + str(alert_dictionary1[key]) + '\\n'\n", - " m2_1[\"text\"] += '\\n'\n", - " m2['text'] = m2_1\n", + " m2_1[\"text\"] += \"*\" + key + \"* : \" + str(alert_dictionary1[key]) + \"\\n\"\n", + " m2_1[\"text\"] += \"\\n\"\n", + " m2[\"text\"] = m2_1\n", "\n", " m4 = dict()\n", - " m4['type'] = 'section'\n", + " m4[\"type\"] = \"section\"\n", " m4_1 = dict()\n", " m4_1[\"type\"] = \"mrkdwn\"\n", "\n", - " m4_1[\"text\"] = '*Last Stats Reported*:' + '\\n'\n", + " m4_1[\"text\"] = \"*Last Stats Reported*:\" + \"\\n\"\n", " for key in alert_dictionary2.keys():\n", - " m4_1[\"text\"] += '*' + key + '* : ' + str(alert_dictionary2[key]) + '\\n'\n", - " m4_1[\"text\"] += '\\n'\n", - " m4['text'] = m4_1\n", + " m4_1[\"text\"] += \"*\" + key + \"* : \" + str(alert_dictionary2[key]) + \"\\n\"\n", + " m4_1[\"text\"] += \"\\n\"\n", + " m4[\"text\"] = m4_1\n", "\n", " message = dict()\n", - " message['blocks'] = [m1,msep,m2,msep,m4,msep]\n", - " message['text'] = 'Live Monitor Alert'\n", + " message[\"blocks\"] = [m1, msep, m2, msep, m4, msep]\n", + " message[\"text\"] = \"Live Monitor Alert\"\n", "\n", " return message" ] @@ -227,165 +232,253 @@ "source": [ "# Query today's started sessions that are not finished\n", "query = {}\n", - "query['session_date'] = datetime.today().strftime('%Y-%m-%d')\n", - "query['is_finished'] = 0\n", + "query[\"session_date\"] = datetime.today().strftime(\"%Y-%m-%d\")\n", + "query[\"is_finished\"] = 0\n", "\n", - "#Only look for sessions started in the last 1:30\n", - "last_time_start = datetime.now(tz=ZoneInfo('America/New_York')) - timedelta(hours=2,minutes=30)\n", - "last_time_start = last_time_start.replace(tzinfo=None).strftime('%Y-%m-%d %H:%M:%S')\n", + "# Only look for sessions started in the last 1:30\n", + "last_time_start = datetime.now(tz=ZoneInfo(\"America/New_York\")) - timedelta(\n", + " hours=2, minutes=30\n", + ")\n", + "last_time_start = last_time_start.replace(tzinfo=None).strftime(\"%Y-%m-%d %H:%M:%S\")\n", "\n", - "query_started_recently = \"session_start_time > '\" + last_time_start + \"'\" \n", - "sessions = pd.DataFrame((acquisition.SessionStarted & query & query_started_recently).fetch('KEY','session_location','session_start_time',as_dict=True))\n", - "#sessions = sessions.loc[~sessions['subject_fullname'].str.startswith('testuser'),:]\n", + "query_started_recently = \"session_start_time > '\" + last_time_start + \"'\"\n", + "sessions = pd.DataFrame(\n", + " (acquisition.SessionStarted & query & query_started_recently).fetch(\n", + " \"KEY\", \"session_location\", \"session_start_time\", as_dict=True\n", + " )\n", + ")\n", + "# sessions = sessions.loc[~sessions['subject_fullname'].str.startswith('testuser'),:]\n", "\n", - "print('sessions STARTED RECENTLY\\n', sessions)\n", + "print(\"sessions STARTED RECENTLY\\n\", sessions)\n", "\n", "if sessions.shape[0] > 0:\n", - "\n", - " #If more than one \"not finished\" session in same rig, grab the last one started\n", - " sessions2 = sessions.groupby('session_location').agg({'session_start_time': [('session_start_time', 'max')]})\n", + " # If more than one \"not finished\" session in same rig, grab the last one started\n", + " sessions2 = sessions.groupby(\"session_location\").agg(\n", + " {\"session_start_time\": [(\"session_start_time\", \"max\")]}\n", + " )\n", " sessions2.columns = sessions2.columns.droplevel()\n", " sessions2 = sessions2.reset_index()\n", - " sessions = pd.merge(sessions, sessions2, on=['session_location', 'session_start_time'])\n", - " sessions = sessions.drop(columns=['session_location'])\n", + " sessions = pd.merge(\n", + " sessions, sessions2, on=[\"session_location\", \"session_start_time\"]\n", + " )\n", + " sessions = sessions.drop(columns=[\"session_location\"])\n", " sessions = sessions.reset_index(drop=True)\n", "\n", - " print('Last session started on rig\\n', sessions)\n", + " print(\"Last session started on rig\\n\", sessions)\n", "\n", - " #Only analyze sessions that have not been reported\n", - " query_reported = {} \n", - " query_reported['session_date'] = datetime.today().strftime('%Y-%m-%d')\n", - " sessions_reported = pd.DataFrame((acquisition.ReportedLiveSessionStats & query_reported).fetch('KEY', as_dict = True))\n", + " # Only analyze sessions that have not been reported\n", + " query_reported = {}\n", + " query_reported[\"session_date\"] = datetime.today().strftime(\"%Y-%m-%d\")\n", + " sessions_reported = pd.DataFrame(\n", + " (acquisition.ReportedLiveSessionStats & query_reported).fetch(\n", + " \"KEY\", as_dict=True\n", + " )\n", + " )\n", "\n", "if sessions_reported.shape[0] > 0:\n", - "\n", - " sessions = pd.merge(sessions,sessions_reported, how='left', indicator=True)\n", - " sessions = sessions.loc[sessions['_merge'] == 'left_only']\n", - " sessions = sessions.drop(columns='_merge')\n", + " sessions = pd.merge(sessions, sessions_reported, how=\"left\", indicator=True)\n", + " sessions = sessions.loc[sessions[\"_merge\"] == \"left_only\"]\n", + " sessions = sessions.drop(columns=\"_merge\")\n", " sessions = sessions.reset_index(drop=True)\n", "\n", - "print('Sessions not reported\\n', sessions)\n", + "print(\"Sessions not reported\\n\", sessions)\n", "\n", - "#Only analyze sessions subjects > NUM_SESSIONS_COMPLETED have not been reported\n", + "# Only analyze sessions subjects > NUM_SESSIONS_COMPLETED have not been reported\n", "if sessions.shape[0] > 0:\n", + " query_subjects = (\n", + " \"subject_fullname in ('\" + \"', '\".join(sessions[\"subject_fullname\"]) + \"')\"\n", + " )\n", "\n", - " query_subjects = \"subject_fullname in ('\"+ \"', '\".join(sessions['subject_fullname']) + \"')\"\n", - "\n", - " count_sessions_table = (subject.Subject).aggr((acquisition.Session & query_subjects), num_sessions=\"count(subject_fullname)\")\n", + " count_sessions_table = (subject.Subject).aggr(\n", + " (acquisition.Session & query_subjects), num_sessions=\"count(subject_fullname)\"\n", + " )\n", " count_sessions_df = pd.DataFrame(count_sessions_table.fetch(as_dict=True))\n", "\n", - " sessions = pd.merge(sessions,count_sessions_df, how='left')\n", - " sessions['num_sessions'] = sessions['num_sessions'].fillna(0)\n", + " sessions = pd.merge(sessions, count_sessions_df, how=\"left\")\n", + " sessions[\"num_sessions\"] = sessions[\"num_sessions\"].fillna(0)\n", "\n", - " sessions = sessions.loc[sessions['num_sessions']>= MIN_SESSIONS_COMPLETED, :]\n", + " sessions = sessions.loc[sessions[\"num_sessions\"] >= MIN_SESSIONS_COMPLETED, :]\n", "\n", - "print('Subjects with completed min_num_sessions\\n', sessions)\n", + "print(\"Subjects with completed min_num_sessions\\n\", sessions)\n", "\n", "if sessions.shape[0] > 0:\n", - "\n", " # Query last live stat from the started sessions\n", - " query_live_stats = sessions.to_dict('records')\n", - " \n", - " #Last non violation trial in sessions\n", + " query_live_stats = sessions.to_dict(\"records\")\n", + "\n", + " # Last non violation trial in sessions\n", " query_no_violation_trial = dict()\n", - " query_no_violation_trial['violation_trial'] = 0\n", - " lss_nvio = acquisition.SessionStarted.aggr((acquisition.LiveSessionStats & query_no_violation_trial).proj('current_datetime'), current_datetime=\"max(current_datetime)\")\n", + " query_no_violation_trial[\"violation_trial\"] = 0\n", + " lss_nvio = acquisition.SessionStarted.aggr(\n", + " (acquisition.LiveSessionStats & query_no_violation_trial).proj(\n", + " \"current_datetime\"\n", + " ),\n", + " current_datetime=\"max(current_datetime)\",\n", + " )\n", " live_stats_nvio = pd.DataFrame((lss_nvio & query_live_stats).fetch(as_dict=True))\n", - " live_stats_nvio = live_stats_nvio.rename({'current_datetime': 'last_non_violation_trial'}, axis=1)\n", - " \n", - " #Last violation trial in sessions\n", - " query_violation_trial = dict()\n", - " query_violation_trial['violation_trial'] = 1\n", - " lss_vio = acquisition.SessionStarted.aggr((acquisition.LiveSessionStats & query_violation_trial).proj('current_datetime'), current_datetime=\"max(current_datetime)\")\n", - " live_stats_vio = pd.DataFrame((lss_vio & query_live_stats & query_violation_trial).fetch(as_dict=True))\n", - " live_stats_vio = live_stats_vio.rename({'current_datetime': 'last_violation_trial'}, axis=1)\n", + " live_stats_nvio = live_stats_nvio.rename(\n", + " {\"current_datetime\": \"last_non_violation_trial\"}, axis=1\n", + " )\n", "\n", + " # Last violation trial in sessions\n", + " query_violation_trial = dict()\n", + " query_violation_trial[\"violation_trial\"] = 1\n", + " lss_vio = acquisition.SessionStarted.aggr(\n", + " (acquisition.LiveSessionStats & query_violation_trial).proj(\"current_datetime\"),\n", + " current_datetime=\"max(current_datetime)\",\n", + " )\n", + " live_stats_vio = pd.DataFrame(\n", + " (lss_vio & query_live_stats & query_violation_trial).fetch(as_dict=True)\n", + " )\n", + " live_stats_vio = live_stats_vio.rename(\n", + " {\"current_datetime\": \"last_violation_trial\"}, axis=1\n", + " )\n", "\n", " # Merge last violation trial and last non violation Trials from sessions\n", " if live_stats_nvio.shape[0] > 0 and live_stats_vio.shape[0] > 0:\n", - " live_stats = pd.merge(live_stats_nvio,live_stats_vio, how='outer')\n", + " live_stats = pd.merge(live_stats_nvio, live_stats_vio, how=\"outer\")\n", " elif live_stats_nvio.shape[0] > 0:\n", " live_stats = live_stats_nvio.copy()\n", - " live_stats['last_violation_trial'] = pd.NaT\n", + " live_stats[\"last_violation_trial\"] = pd.NaT\n", " elif live_stats_vio.shape[0] > 0:\n", " live_stats = live_stats_vio.copy()\n", - " live_stats['last_non_violation_trial'] = pd.NaT\n", + " live_stats[\"last_non_violation_trial\"] = pd.NaT\n", " else:\n", " live_stats = pd.DataFrame()\n", "\n", - "\n", " # If there are any sessions with live stats\n", " if live_stats.shape[0] > 0:\n", - "\n", - " live_stats = pd.merge(sessions,live_stats, how='inner')\n", - " fake_date = pd.Timestamp('1900-01-01')\n", + " live_stats = pd.merge(sessions, live_stats, how=\"inner\")\n", + " fake_date = pd.Timestamp(\"1900-01-01\")\n", "\n", " # Filter sessions whose last trial info is greater than 300s\n", - " right_now_est = datetime.now(tz=ZoneInfo('America/New_York'))\n", + " right_now_est = datetime.now(tz=ZoneInfo(\"America/New_York\"))\n", " right_now_est = right_now_est.replace(tzinfo=None)\n", - " live_stats['seconds_elapsed_last_stat_nvio'] = (right_now_est- live_stats['last_non_violation_trial']).dt.total_seconds()\n", - " live_stats['alert_nvio'] = live_stats['seconds_elapsed_last_stat_nvio'] > SECONDS_ALERT\n", - "\n", - " live_stats['seconds_elapsed_session_started'] = (right_now_est- live_stats['session_start_time']).dt.total_seconds()\n", - " live_stats['alert_vio'] = live_stats['seconds_elapsed_session_started'] > SECONDS_ALERT & pd.isna(live_stats['last_non_violation_trial'].isna()) & ~pd.isna(live_stats['last_violation_trial'].isna())\n", - "\n", - "\n", - " live_stats = live_stats.loc[(live_stats['alert_nvio']==True) | (live_stats['alert_vio']==True),:]\n", + " live_stats[\"seconds_elapsed_last_stat_nvio\"] = (\n", + " right_now_est - live_stats[\"last_non_violation_trial\"]\n", + " ).dt.total_seconds()\n", + " live_stats[\"alert_nvio\"] = (\n", + " live_stats[\"seconds_elapsed_last_stat_nvio\"] > SECONDS_ALERT\n", + " )\n", + "\n", + " live_stats[\"seconds_elapsed_session_started\"] = (\n", + " right_now_est - live_stats[\"session_start_time\"]\n", + " ).dt.total_seconds()\n", + " live_stats[\"alert_vio\"] = live_stats[\n", + " \"seconds_elapsed_session_started\"\n", + " ] > SECONDS_ALERT & pd.isna(\n", + " live_stats[\"last_non_violation_trial\"].isna()\n", + " ) & ~pd.isna(live_stats[\"last_violation_trial\"].isna())\n", + "\n", + " live_stats = live_stats.loc[\n", + " (live_stats[\"alert_nvio\"] == True) | (live_stats[\"alert_vio\"] == True), :\n", + " ]\n", "\n", " print(live_stats)\n", "\n", - " #If there are any sessions to alert (more then 300s)\n", + " # If there are any sessions to alert (more then 300s)\n", " if live_stats.shape[0] > 0:\n", - "\n", - " live_stats['current_datetime'] = live_stats[['last_violation_trial', 'last_non_violation_trial']].fillna(fake_date).max(axis=1)\n", - " live_stats['seconds_elapsed_last_valid_stat'] = live_stats[['seconds_elapsed_last_stat_nvio', 'seconds_elapsed_session_started']].fillna(-np.inf).max(axis=1)\n", - "\n", - " #get_session_info to alert (plus slack researcher)\n", - " query_live_stats_sessions = live_stats[['subject_fullname', 'session_date', 'session_number']].to_dict('records')\n", - "\n", - " session_data_df = pd.DataFrame(((lab.User.proj('slack') * subject.Subject.proj('user_id') *\\\n", - " acquisition.SessionStarted.proj('session_location')) & query_live_stats_sessions).fetch(as_dict=True))\n", - " \n", - " session_data_df = session_data_df.rename({'slack': 'researcher'}, axis=1)\n", - " session_data_df['researcher'] = '<@'+ session_data_df['researcher'] + '>'\n", - " session_data_df = session_data_df[['researcher', 'subject_fullname', 'session_date', 'session_number']]\n", - "\n", - " #Query full live stat table\n", - " #session_stats = live_stats.copy()\n", - " #session_stats = session_stats.rename({'current_datetime': 'last_live_stat'}, axis=1)\n", - " query_live_stats = live_stats[['subject_fullname', 'session_date', 'session_number', 'current_datetime']].to_dict('records')\n", - " live_stats_mini = live_stats[['subject_fullname', 'session_date', 'session_number', 'seconds_elapsed_last_valid_stat']].copy()\n", - " ls_full_df = pd.DataFrame((acquisition.LiveSessionStats & query_live_stats).fetch(as_dict=True))\n", - " ls_full_df = pd.merge(ls_full_df, live_stats_mini, on=['subject_fullname', 'session_date', 'session_number'])\n", - " ls_full_df = ls_full_df.drop(columns=['subject_fullname', 'session_date', 'session_number'])\n", - " ls_full_df = ls_full_df.rename({'current_datetime': 'last_trial_time'}, axis=1)\n", - "\n", - "\n", - " mid = ls_full_df['last_trial_time']\n", - " ls_full_df = ls_full_df.drop(columns=['last_trial_time'])\n", - " ls_full_df.insert(0, 'last_trial_time', mid)\n", - "\n", - " ls_full_dict = ls_full_df.to_dict('records')\n", + " live_stats[\"current_datetime\"] = (\n", + " live_stats[[\"last_violation_trial\", \"last_non_violation_trial\"]]\n", + " .fillna(fake_date)\n", + " .max(axis=1)\n", + " )\n", + " live_stats[\"seconds_elapsed_last_valid_stat\"] = (\n", + " live_stats[\n", + " [\n", + " \"seconds_elapsed_last_stat_nvio\",\n", + " \"seconds_elapsed_session_started\",\n", + " ]\n", + " ]\n", + " .fillna(-np.inf)\n", + " .max(axis=1)\n", + " )\n", + "\n", + " # get_session_info to alert (plus slack researcher)\n", + " query_live_stats_sessions = live_stats[\n", + " [\"subject_fullname\", \"session_date\", \"session_number\"]\n", + " ].to_dict(\"records\")\n", + "\n", + " session_data_df = pd.DataFrame(\n", + " (\n", + " (\n", + " lab.User.proj(\"slack\")\n", + " * subject.Subject.proj(\"user_id\")\n", + " * acquisition.SessionStarted.proj(\"session_location\")\n", + " )\n", + " & query_live_stats_sessions\n", + " ).fetch(as_dict=True)\n", + " )\n", + "\n", + " session_data_df = session_data_df.rename({\"slack\": \"researcher\"}, axis=1)\n", + " session_data_df[\"researcher\"] = \"<@\" + session_data_df[\"researcher\"] + \">\"\n", + " session_data_df = session_data_df[\n", + " [\"researcher\", \"subject_fullname\", \"session_date\", \"session_number\"]\n", + " ]\n", + "\n", + " # Query full live stat table\n", + " # session_stats = live_stats.copy()\n", + " # session_stats = session_stats.rename({'current_datetime': 'last_live_stat'}, axis=1)\n", + " query_live_stats = live_stats[\n", + " [\n", + " \"subject_fullname\",\n", + " \"session_date\",\n", + " \"session_number\",\n", + " \"current_datetime\",\n", + " ]\n", + " ].to_dict(\"records\")\n", + " live_stats_mini = live_stats[\n", + " [\n", + " \"subject_fullname\",\n", + " \"session_date\",\n", + " \"session_number\",\n", + " \"seconds_elapsed_last_valid_stat\",\n", + " ]\n", + " ].copy()\n", + " ls_full_df = pd.DataFrame(\n", + " (acquisition.LiveSessionStats & query_live_stats).fetch(as_dict=True)\n", + " )\n", + " ls_full_df = pd.merge(\n", + " ls_full_df,\n", + " live_stats_mini,\n", + " on=[\"subject_fullname\", \"session_date\", \"session_number\"],\n", + " )\n", + " ls_full_df = ls_full_df.drop(\n", + " columns=[\"subject_fullname\", \"session_date\", \"session_number\"]\n", + " )\n", + " ls_full_df = ls_full_df.rename(\n", + " {\"current_datetime\": \"last_trial_time\"}, axis=1\n", + " )\n", + "\n", + " mid = ls_full_df[\"last_trial_time\"]\n", + " ls_full_df = ls_full_df.drop(columns=[\"last_trial_time\"])\n", + " ls_full_df.insert(0, \"last_trial_time\", mid)\n", + "\n", + " ls_full_dict = ls_full_df.to_dict(\"records\")\n", "\n", " # Send one alert per session found\n", " idx_alert = 0\n", " for this_alert_record in ls_full_dict:\n", - "\n", - " #Format message for session and live stat dictionary\n", - " this_session_stats = session_data_df.iloc[idx_alert,:]\n", - " slack_json_message = slack_alert_message_format_live_stats(this_session_stats.to_dict(), this_alert_record, int(this_alert_record['seconds_elapsed_last_valid_stat']))\n", - "\n", - " #Send alert\n", + " # Format message for session and live stat dictionary\n", + " this_session_stats = session_data_df.iloc[idx_alert, :]\n", + " slack_json_message = slack_alert_message_format_live_stats(\n", + " this_session_stats.to_dict(),\n", + " this_alert_record,\n", + " int(this_alert_record[\"seconds_elapsed_last_valid_stat\"]),\n", + " )\n", + "\n", + " # Send alert\n", " webhooks_list = su.get_webhook_list(slack_configuration_dictionary, lab)\n", " for this_webhook in webhooks_list:\n", " su.send_slack_notification(this_webhook, slack_json_message)\n", " time.sleep(1)\n", "\n", - " reported_session = this_session_stats[['subject_fullname', 'session_date', 'session_number']].copy()\n", - " reported_session['report_datetime'] = right_now_est\n", + " reported_session = this_session_stats[\n", + " [\"subject_fullname\", \"session_date\", \"session_number\"]\n", + " ].copy()\n", + " reported_session[\"report_datetime\"] = right_now_est\n", "\n", " acquisition.ReportedLiveSessionStats.insert1(reported_session.to_dict())\n", - " idx_alert += 1\n" + " idx_alert += 1" ] }, { diff --git a/notebooks/numtrials_and_bias_check.ipynb b/notebooks/numtrials_and_bias_check.ipynb index f2d36158..c78f0536 100644 --- a/notebooks/numtrials_and_bias_check.ipynb +++ b/notebooks/numtrials_and_bias_check.ipynb @@ -30,7 +30,7 @@ "import datajoint as dj\n", "import pandas as pd\n", "\n", - "from u19_pipeline import utility\n" + "from u19_pipeline import utility" ] }, { @@ -48,8 +48,8 @@ "source": [ "utility.basic_dj_configuration(dj)\n", "dj.conn()\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -65,50 +65,71 @@ "metadata": {}, "outputs": [], "source": [ - "#Filter Dates\n", + "# Filter Dates\n", "filter_dates = True\n", "num_days_check = 60\n", "current_date = date.today()\n", "first_date_to_check = current_date - timedelta(days=num_days_check)\n", - "dates = [first_date_to_check.strftime(\"%Y-%m-%d\"), current_date.strftime(\"%Y-%m-%d\")]\n", - "date_label = ' and '.join(f'\"{w}\"' for w in dates)\n", - "date_query = 'session_date between ' + date_label \n", + "dates = [first_date_to_check.strftime(\"%Y-%m-%d\"), current_date.strftime(\"%Y-%m-%d\")]\n", + "date_label = \" and \".join(f'\"{w}\"' for w in dates)\n", + "date_query = \"session_date between \" + date_label\n", "\n", "\n", "## Filter rigs\n", "filter_rigs = False\n", - "rigs = ['165I-Rig1-T',\n", - " '165I-Rig2-T',\n", - " '165I-Rig3-T',\n", - " '165I-Rig4-T',\n", - " '165A-Rig5-T',\n", - " '165A-Rig6-T',\n", - " '165A-Rig7-T',\n", - " '165A-Rig8-T', \n", - " '170b-Imaging0642',\n", - " '185f-rig1',\n", - " '188-Rig1',\n", - " '188-Rig2',\n", - " 'BezosMeso',\n", - " 'C42-Bay2-Rig1-I']\n", - "rigs_label = ', '.join(f'\"{w}\"' for w in rigs)\n", - "rigs_query = 'session_location in (' + rigs_label + ')'\n", + "rigs = [\n", + " \"165I-Rig1-T\",\n", + " \"165I-Rig2-T\",\n", + " \"165I-Rig3-T\",\n", + " \"165I-Rig4-T\",\n", + " \"165A-Rig5-T\",\n", + " \"165A-Rig6-T\",\n", + " \"165A-Rig7-T\",\n", + " \"165A-Rig8-T\",\n", + " \"170b-Imaging0642\",\n", + " \"185f-rig1\",\n", + " \"188-Rig1\",\n", + " \"188-Rig2\",\n", + " \"BezosMeso\",\n", + " \"C42-Bay2-Rig1-I\",\n", + "]\n", + "rigs_label = \", \".join(f'\"{w}\"' for w in rigs)\n", + "rigs_query = \"session_location in (\" + rigs_label + \")\"\n", "\n", "\n", - "#Filter users\n", + "# Filter users\n", "filter_user = True\n", - "users = ['efonseca_ef255_actgp002']\n", + "users = [\"efonseca_ef255_actgp002\"]\n", "users = ['subject_fullname like \"' + x + '%\"' for x in users]\n", - "user_query = ' or '.join(f'{w}' for w in users) \n", + "user_query = \" or \".join(f\"{w}\" for w in users)\n", "\n", - "#Filter extras\n", - "not_testuser = 'subject_fullname not like \"testuser%\"'\n", + "# Filter extras\n", + "not_testuser = 'subject_fullname not like \"testuser%\"'\n", "not_testsubjects = 'subject_fullname not like \"%_test\"'\n", - "not_bad_sessions = 'is_bad_session = 0'\n", - "extra_queries = [not_testuser, not_testsubjects, not_bad_sessions]\n", + "not_bad_sessions = \"is_bad_session = 0\"\n", + "extra_queries = [not_testuser, not_testsubjects, not_bad_sessions]\n", "\n", - "columns_session_table = ['subject_fullname', 'session_date', 'session_number', 'session_location', 'session_performance', 'task', 'level', 'stimulus_bank', 'session_protocol', 'num_trials']\n", - "columns_trial_table = ['subject_fullname', 'session_date', 'session_number', 'block', 'trial_idx', 'trial_type', 'choice']\n" + "columns_session_table = [\n", + " \"subject_fullname\",\n", + " \"session_date\",\n", + " \"session_number\",\n", + " \"session_location\",\n", + " \"session_performance\",\n", + " \"task\",\n", + " \"level\",\n", + " \"stimulus_bank\",\n", + " \"session_protocol\",\n", + " \"num_trials\",\n", + "]\n", + "columns_trial_table = [\n", + " \"subject_fullname\",\n", + " \"session_date\",\n", + " \"session_number\",\n", + " \"block\",\n", + " \"trial_idx\",\n", + " \"trial_type\",\n", + " \"choice\",\n", + "]" ] }, { @@ -546,23 +567,25 @@ "query = []\n", "if filter_rigs:\n", " query.append(rigs_query)\n", - " \n", + "\n", "if filter_dates:\n", " query.append(date_query)\n", - " \n", + "\n", "if filter_user:\n", " query.append(user_query)\n", "\n", - "#All extra stuff to filter\n", + "# All extra stuff to filter\n", "for i in extra_queries:\n", " query.append(i)\n", - " \n", + "\n", "# Filter by all conditions selected\n", "session_table_filtered = acquisition.Session\n", "for filter_key in query:\n", " session_table_filtered = session_table_filtered & filter_key\n", - " \n", - "session_df = pd.DataFrame(session_table_filtered.fetch(*columns_session_table, as_dict=True))\n", + "\n", + "session_df = pd.DataFrame(\n", + " session_table_filtered.fetch(*columns_session_table, as_dict=True)\n", + ")\n", "session_df" ] }, @@ -630,11 +653,12 @@ } ], "source": [ - "\n", - "protocol_df = session_df.groupby('session_protocol').agg({'subject_fullname': [('num_sessions', 'count'), ('num_subjects', 'nunique')]})\n", + "protocol_df = session_df.groupby(\"session_protocol\").agg(\n", + " {\"subject_fullname\": [(\"num_sessions\", \"count\"), (\"num_subjects\", \"nunique\")]}\n", + ")\n", "protocol_df.columns = protocol_df.columns.droplevel()\n", - "#protocol_df = protocol_df.reset_index()\n", - "protocol_df = protocol_df.sort_values(by='num_sessions', ascending=False)\n", + "# protocol_df = protocol_df.reset_index()\n", + "protocol_df = protocol_df.sort_values(by=\"num_sessions\", ascending=False)\n", "protocol_df" ] }, @@ -731,23 +755,32 @@ "source": [ "session_p_df = session_df.copy()\n", "\n", - "#Sort by subject, date and session number\n", - "session_p_df = session_p_df.sort_values(by=['subject_fullname', 'session_date', 'session_number'], ascending=[True, False, False])\n", + "# Sort by subject, date and session number\n", + "session_p_df = session_p_df.sort_values(\n", + " by=[\"subject_fullname\", \"session_date\", \"session_number\"],\n", + " ascending=[True, False, False],\n", + ")\n", "session_p_df = session_p_df.reset_index(drop=True)\n", "\n", "# Group metrics for each subject (total num sessions, mean num_trials, std num_trials)\n", - "num_sessions_df = session_p_df.groupby('subject_fullname').agg({'session_date': [('total_sessions', 'count')], \\\n", - " 'num_trials': [('avg_trials', 'mean'), ('std_trials', 'std')]})\n", + "num_sessions_df = session_p_df.groupby(\"subject_fullname\").agg(\n", + " {\n", + " \"session_date\": [(\"total_sessions\", \"count\")],\n", + " \"num_trials\": [(\"avg_trials\", \"mean\"), (\"std_trials\", \"std\")],\n", + " }\n", + ")\n", "num_sessions_df.columns = num_sessions_df.columns.droplevel()\n", "num_sessions_df = num_sessions_df.reset_index()\n", "\n", "# Merge session count to session df\n", "session_p_df = session_p_df.merge(num_sessions_df)\n", "\n", - "session_p_df['z_score_trials'] = (session_p_df['num_trials'] - session_p_df['avg_trials']) / session_p_df['std_trials']\n", + "session_p_df[\"z_score_trials\"] = (\n", + " session_p_df[\"num_trials\"] - session_p_df[\"avg_trials\"]\n", + ") / session_p_df[\"std_trials\"]\n", "\n", "# Keep only today's session\n", - "session_p_df = session_p_df.loc[session_p_df['session_date'] == current_date,:]\n", + "session_p_df = session_p_df.loc[session_p_df[\"session_date\"] == current_date, :]\n", "session_p_df = session_p_df.reset_index(drop=True)\n", "session_p_df" ] @@ -1179,33 +1212,59 @@ } ], "source": [ - "sort_columns = ['subject_fullname', 'session_date', 'session_number', 'block', 'trial_idx']\n", - "session_columns = ['subject_fullname', 'session_date', 'session_number']\n", - "trials_df = trials_df.sort_values(by=sort_columns, ascending=[True, False, False, True, True])\n", + "sort_columns = [\n", + " \"subject_fullname\",\n", + " \"session_date\",\n", + " \"session_number\",\n", + " \"block\",\n", + " \"trial_idx\",\n", + "]\n", + "session_columns = [\"subject_fullname\", \"session_date\", \"session_number\"]\n", + "trials_df = trials_df.sort_values(\n", + " by=sort_columns, ascending=[True, False, False, True, True]\n", + ")\n", "trials_df = trials_df.reset_index(drop=True)\n", "\n", "\n", "# Correct trials, correct to left and correct to right\n", - "trials_df['left_trial'] = (trials_df['trial_type'] == 'L').astype(int)\n", - "trials_df['right_trial'] = (trials_df['trial_type'] == 'R').astype(int)\n", + "trials_df[\"left_trial\"] = (trials_df[\"trial_type\"] == \"L\").astype(int)\n", + "trials_df[\"right_trial\"] = (trials_df[\"trial_type\"] == \"R\").astype(int)\n", "\n", - "trials_df['cum_left_trials'] = trials_df.groupby(['subject_fullname', 'session_date', 'session_number'])['left_trial'].cumsum()\n", - "trials_df['cum_right_trials'] = trials_df.groupby(['subject_fullname', 'session_date', 'session_number'])['right_trial'].cumsum()\n", + "trials_df[\"cum_left_trials\"] = trials_df.groupby(\n", + " [\"subject_fullname\", \"session_date\", \"session_number\"]\n", + ")[\"left_trial\"].cumsum()\n", + "trials_df[\"cum_right_trials\"] = trials_df.groupby(\n", + " [\"subject_fullname\", \"session_date\", \"session_number\"]\n", + ")[\"right_trial\"].cumsum()\n", "\n", - "trials_df['correct_trial'] = (trials_df['trial_type'] == trials_df['choice']).astype(int)\n", - "trials_df['correct_left'] = ((trials_df['correct_trial'] == 1) & (trials_df['trial_type'] == 'L')).astype(int)\n", - "trials_df['correct_right'] = ((trials_df['correct_trial'] == 1) & (trials_df['trial_type'] == 'R')).astype(int)\n", + "trials_df[\"correct_trial\"] = (trials_df[\"trial_type\"] == trials_df[\"choice\"]).astype(\n", + " int\n", + ")\n", + "trials_df[\"correct_left\"] = (\n", + " (trials_df[\"correct_trial\"] == 1) & (trials_df[\"trial_type\"] == \"L\")\n", + ").astype(int)\n", + "trials_df[\"correct_right\"] = (\n", + " (trials_df[\"correct_trial\"] == 1) & (trials_df[\"trial_type\"] == \"R\")\n", + ").astype(int)\n", "\n", - "trials_df['cum_correct_trials'] = trials_df.groupby(['subject_fullname', 'session_date', 'session_number'])['correct_trial'].cumsum()\n", - "trials_df['cum_correct_left_trials'] = trials_df.groupby(['subject_fullname', 'session_date', 'session_number'])['correct_left'].cumsum()\n", - "trials_df['cum_correct_right_trials'] = trials_df.groupby(['subject_fullname', 'session_date', 'session_number'])['correct_right'].cumsum()\n", + "trials_df[\"cum_correct_trials\"] = trials_df.groupby(\n", + " [\"subject_fullname\", \"session_date\", \"session_number\"]\n", + ")[\"correct_trial\"].cumsum()\n", + "trials_df[\"cum_correct_left_trials\"] = trials_df.groupby(\n", + " [\"subject_fullname\", \"session_date\", \"session_number\"]\n", + ")[\"correct_left\"].cumsum()\n", + "trials_df[\"cum_correct_right_trials\"] = trials_df.groupby(\n", + " [\"subject_fullname\", \"session_date\", \"session_number\"]\n", + ")[\"correct_right\"].cumsum()\n", "\n", - "trials_df = trials_df.loc[~trials_df.duplicated(subset=session_columns, keep='last'), :]\n", + "trials_df = trials_df.loc[~trials_df.duplicated(subset=session_columns, keep=\"last\"), :]\n", "trials_df = trials_df.reset_index(drop=True)\n", "\n", - "trials_df['bias'] = (trials_df['cum_correct_right_trials'] / trials_df['cum_right_trials']) - (trials_df['cum_correct_left_trials'] / trials_df['cum_left_trials'])\n", + "trials_df[\"bias\"] = (\n", + " trials_df[\"cum_correct_right_trials\"] / trials_df[\"cum_right_trials\"]\n", + ") - (trials_df[\"cum_correct_left_trials\"] / trials_df[\"cum_left_trials\"])\n", "\n", - "bias_df = trials_df[session_columns + ['bias']]\n", + "bias_df = trials_df[session_columns + [\"bias\"]]\n", "\n", "bias_df" ] diff --git a/notebooks/performance_metrics_2022.ipynb b/notebooks/performance_metrics_2022.ipynb index dc878ea7..69245199 100644 --- a/notebooks/performance_metrics_2022.ipynb +++ b/notebooks/performance_metrics_2022.ipynb @@ -29,7 +29,7 @@ "import datajoint as dj\n", "import pandas as pd\n", "\n", - "import u19_pipeline.alert_system.behavior_metrics as bm\n" + "import u19_pipeline.alert_system.behavior_metrics as bm" ] }, { @@ -54,8 +54,8 @@ ], "source": [ "dj.conn()\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -73,28 +73,28 @@ "source": [ "query_2021 = 'session_date between \"2021-01-01\" and \"2021-12-31\"'\n", "query_2022 = 'session_date between \"2022-01-01\" and \"2022-12-31\"'\n", - "not_testuser = 'subject_fullname not like \"testuser%\"'\n", + "not_testuser = 'subject_fullname not like \"testuser%\"'\n", "not_testsubjects = 'subject_fullname not like \"%_test\"'\n", - "not_bad_sessions = 'is_bad_session = 0'\n", + "not_bad_sessions = \"is_bad_session = 0\"\n", "\n", - "query_2021 = [query_2021, not_testuser, not_testsubjects, not_bad_sessions]\n", - "query_2022 = [query_2022, not_testuser, not_testsubjects, not_bad_sessions]\n", + "query_2021 = [query_2021, not_testuser, not_testsubjects, not_bad_sessions]\n", + "query_2022 = [query_2022, not_testuser, not_testsubjects, not_bad_sessions]\n", "\n", - "session_table_filtered = acquisition.Session \n", + "session_table_filtered = acquisition.Session\n", "for filter_key in query_2021:\n", " session_table_filtered = session_table_filtered & filter_key\n", - "subject_2021 = set((session_table_filtered).fetch('subject_fullname').tolist())\n", + "subject_2021 = set((session_table_filtered).fetch(\"subject_fullname\").tolist())\n", "\n", - "session_table_filtered = acquisition.Session \n", + "session_table_filtered = acquisition.Session\n", "for filter_key in query_2022:\n", " session_table_filtered = session_table_filtered & filter_key\n", - "subject_2022 = set((session_table_filtered).fetch('subject_fullname').tolist())\n", + "subject_2022 = set((session_table_filtered).fetch(\"subject_fullname\").tolist())\n", "subject_2022\n", "\n", "subjects_only_2022 = list(subject_2022 - subject_2021)\n", "len(subjects_only_2022)\n", "\n", - "subjects_only_2022query = [{'subject_fullname': i} for i in subjects_only_2022]" + "subjects_only_2022query = [{\"subject_fullname\": i} for i in subjects_only_2022]" ] }, { @@ -248,10 +248,16 @@ } ], "source": [ - "#session_protocol_q = 'session_protocol like \"%\"'\n", + "# session_protocol_q = 'session_protocol like \"%\"'\n", "session_protocol_q = 'session_protocol like \"%PoissonBlocksCondensed3m%\"'\n", - "session_df = pd.DataFrame((acquisition.Session & subjects_only_2022query & session_protocol_q).fetch('KEY','session_protocol', order_by='session_start_time', as_dict=True))\n", - "session_keys = ((acquisition.Session & subjects_only_2022query & session_protocol_q).fetch('KEY', as_dict=True))\n", + "session_df = pd.DataFrame(\n", + " (acquisition.Session & subjects_only_2022query & session_protocol_q).fetch(\n", + " \"KEY\", \"session_protocol\", order_by=\"session_start_time\", as_dict=True\n", + " )\n", + ")\n", + "session_keys = (\n", + " acquisition.Session & subjects_only_2022query & session_protocol_q\n", + ").fetch(\"KEY\", as_dict=True)\n", "session_df" ] }, @@ -379,12 +385,14 @@ } ], "source": [ - "protocol_df = session_df.groupby('session_protocol').agg({'subject_fullname': [('num_sessions', 'count'), ('num_subjects', 'nunique')]})\n", + "protocol_df = session_df.groupby(\"session_protocol\").agg(\n", + " {\"subject_fullname\": [(\"num_sessions\", \"count\"), (\"num_subjects\", \"nunique\")]}\n", + ")\n", "protocol_df.columns = protocol_df.columns.droplevel()\n", - "#protocol_df = protocol_df.reset_index()\n", - "protocol_df = protocol_df.sort_values(by='num_sessions', ascending=False)\n", - "protocol_df.loc['Total']= protocol_df.sum(numeric_only=True, axis=0)\n", - "#protocol_df = pd.concat([protocol_df, ])\n", + "# protocol_df = protocol_df.reset_index()\n", + "protocol_df = protocol_df.sort_values(by=\"num_sessions\", ascending=False)\n", + "protocol_df.loc[\"Total\"] = protocol_df.sum(numeric_only=True, axis=0)\n", + "# protocol_df = pd.concat([protocol_df, ])\n", "protocol_df.head(45)" ] }, @@ -608,8 +616,28 @@ "source": [ "block_query = \"main_level = level and level in (11,12,15,16)\"\n", "\n", - "trials_df = pd.DataFrame((acquisition.Session.proj('session_location') * behavior.TowersBlock * behavior.TowersBlock.Trial() & block_query & session_keys).fetch('KEY', 'level', 'trial_type', 'choice', 'session_location', order_by='session_date', as_dict=True))\n", - "all_sessions_df = pd.DataFrame((acquisition.Session & session_keys).fetch('KEY','session_location', order_by='session_date', as_dict=True))\n", + "trials_df = pd.DataFrame(\n", + " (\n", + " acquisition.Session.proj(\"session_location\")\n", + " * behavior.TowersBlock\n", + " * behavior.TowersBlock.Trial()\n", + " & block_query\n", + " & session_keys\n", + " ).fetch(\n", + " \"KEY\",\n", + " \"level\",\n", + " \"trial_type\",\n", + " \"choice\",\n", + " \"session_location\",\n", + " order_by=\"session_date\",\n", + " as_dict=True,\n", + " )\n", + ")\n", + "all_sessions_df = pd.DataFrame(\n", + " (acquisition.Session & session_keys).fetch(\n", + " \"KEY\", \"session_location\", order_by=\"session_date\", as_dict=True\n", + " )\n", + ")\n", "trials_df" ] }, @@ -626,7 +654,9 @@ "source": [ "all_sessions_df\n", "\n", - "summary_alls_df = all_sessions_df.groupby('session_location').agg({'subject_fullname': [('num_subjects', 'nunique'), ('num_sessions', 'count')]})\n", + "summary_alls_df = all_sessions_df.groupby(\"session_location\").agg(\n", + " {\"subject_fullname\": [(\"num_subjects\", \"nunique\"), (\"num_sessions\", \"count\")]}\n", + ")\n", "summary_alls_df.columns = summary_alls_df.columns.droplevel()\n", "summary_alls_df = summary_alls_df.reset_index()" ] @@ -1032,9 +1062,13 @@ } ], "source": [ - "metrics_df = bm.BehaviorMetrics.get_bias_from_trial_df(trials_df, return_all_metrics=True)\n", - "metrics_df['performance'] = metrics_df['cum_correct_trials']*100 / metrics_df['cum_trials']\n", - "metrics_df['bias'] = metrics_df['bias'].abs()\n", + "metrics_df = bm.BehaviorMetrics.get_bias_from_trial_df(\n", + " trials_df, return_all_metrics=True\n", + ")\n", + "metrics_df[\"performance\"] = (\n", + " metrics_df[\"cum_correct_trials\"] * 100 / metrics_df[\"cum_trials\"]\n", + ")\n", + "metrics_df[\"bias\"] = metrics_df[\"bias\"].abs()\n", "metrics_df = metrics_df.reset_index(drop=True)\n", "metrics_df" ] @@ -1052,7 +1086,7 @@ "metadata": {}, "outputs": [], "source": [ - "len(list(set(metrics_df['subject_fullname'].tolist())))" + "len(list(set(metrics_df[\"subject_fullname\"].tolist())))" ] }, { @@ -1063,9 +1097,9 @@ "source": [ "performance_min = 70\n", "perf_str = str(performance_min)\n", - "group_variable = 'session_location'\n", + "group_variable = \"session_location\"\n", "\n", - "metrics_df\n" + "metrics_df" ] }, { @@ -1421,66 +1455,107 @@ } ], "source": [ + "metrics_df_f = metrics_df.loc[\n", + " (metrics_df[\"performance\"] >= performance_min) & (metrics_df[\"cum_trials\"] > 100), :\n", + "]\n", "\n", - "\n", - "metrics_df_f = metrics_df.loc[(metrics_df['performance'] >= performance_min) & (metrics_df['cum_trials'] > 100), :]\n", - "\n", - "#Group all sessions by rig\n", - "summary_alls_df = all_sessions_df.groupby('session_location').agg({'subject_fullname': [('num_subjects', 'nunique'), ('num_sessions', 'count')]})\n", + "# Group all sessions by rig\n", + "summary_alls_df = all_sessions_df.groupby(\"session_location\").agg(\n", + " {\"subject_fullname\": [(\"num_subjects\", \"nunique\"), (\"num_sessions\", \"count\")]}\n", + ")\n", "summary_alls_df.columns = summary_alls_df.columns.droplevel()\n", "summary_alls_df = summary_alls_df.reset_index()\n", "\n", - "#Group l11 sessions by rig\n", - "summary_l11_df = metrics_df.groupby('session_location').agg({'subject_fullname': [('num_subjects_l11', 'nunique'), ('num_sessions_l11', 'count')], 'performance': [('performance_l11', 'mean')],\\\n", - " 'bias': [('bias_l11', 'mean')]})\n", + "# Group l11 sessions by rig\n", + "summary_l11_df = metrics_df.groupby(\"session_location\").agg(\n", + " {\n", + " \"subject_fullname\": [\n", + " (\"num_subjects_l11\", \"nunique\"),\n", + " (\"num_sessions_l11\", \"count\"),\n", + " ],\n", + " \"performance\": [(\"performance_l11\", \"mean\")],\n", + " \"bias\": [(\"bias_l11\", \"mean\")],\n", + " }\n", + ")\n", "summary_l11_df.columns = summary_l11_df.columns.droplevel()\n", "summary_l11_df = summary_l11_df.reset_index()\n", "\n", - "#Group l11 70 % sessions by rig\n", - "summary_l1170_df = metrics_df_f.groupby('session_location').agg({'subject_fullname': [('num_subjects_l11>'+perf_str, 'nunique'), ('num_sessions_l11>'+perf_str, 'count')], 'performance': [('performance_l11>'+perf_str, 'mean')],\\\n", - " 'bias': [('bias_l11>'+perf_str, 'mean')]})\n", + "# Group l11 70 % sessions by rig\n", + "summary_l1170_df = metrics_df_f.groupby(\"session_location\").agg(\n", + " {\n", + " \"subject_fullname\": [\n", + " (\"num_subjects_l11>\" + perf_str, \"nunique\"),\n", + " (\"num_sessions_l11>\" + perf_str, \"count\"),\n", + " ],\n", + " \"performance\": [(\"performance_l11>\" + perf_str, \"mean\")],\n", + " \"bias\": [(\"bias_l11>\" + perf_str, \"mean\")],\n", + " }\n", + ")\n", "summary_l1170_df.columns = summary_l1170_df.columns.droplevel()\n", "summary_l1170_df = summary_l1170_df.reset_index()\n", "\n", "# Merge group dataframes\n", - "summary_l11_df = summary_alls_df.merge(summary_l11_df, on='session_location', how='left')\n", - "summary_df = summary_l11_df.merge(summary_l1170_df, on='session_location', how='left')\n", + "summary_l11_df = summary_alls_df.merge(\n", + " summary_l11_df, on=\"session_location\", how=\"left\"\n", + ")\n", + "summary_df = summary_l11_df.merge(summary_l1170_df, on=\"session_location\", how=\"left\")\n", "\n", "\n", "# Fillna, new metrics\n", - "summary_df['num_subjects_l11'] = summary_df['num_subjects_l11'].fillna(0)\n", - "summary_df['num_sessions_l11'] = summary_df['num_sessions_l11'].fillna(0)\n", - "summary_df['num_sessions_l11>'+perf_str] = summary_df['num_sessions_l11>'+perf_str].fillna(0)\n", - "summary_df['num_subjects_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str].fillna(0)\n", + "summary_df[\"num_subjects_l11\"] = summary_df[\"num_subjects_l11\"].fillna(0)\n", + "summary_df[\"num_sessions_l11\"] = summary_df[\"num_sessions_l11\"].fillna(0)\n", + "summary_df[\"num_sessions_l11>\" + perf_str] = summary_df[\n", + " \"num_sessions_l11>\" + perf_str\n", + "].fillna(0)\n", + "summary_df[\"num_subjects_l11>\" + perf_str] = summary_df[\n", + " \"num_subjects_l11>\" + perf_str\n", + "].fillna(0)\n", "\n", "\n", "# Calculate Total rows\n", - "summary_df.loc['Total']= summary_df.sum(numeric_only=True, axis=0)\n", - "mean_columns = ['performance_l11', 'bias_l11', 'performance_l11>'+perf_str, 'bias_l11>'+perf_str]\n", - "summary_df.loc['Total', mean_columns]= summary_df.loc[summary_df.iloc[range(summary_df.shape[0]-1)].index, mean_columns].mean(numeric_only=True, axis=0)\n", + "summary_df.loc[\"Total\"] = summary_df.sum(numeric_only=True, axis=0)\n", + "mean_columns = [\n", + " \"performance_l11\",\n", + " \"bias_l11\",\n", + " \"performance_l11>\" + perf_str,\n", + " \"bias_l11>\" + perf_str,\n", + "]\n", + "summary_df.loc[\"Total\", mean_columns] = summary_df.loc[\n", + " summary_df.iloc[range(summary_df.shape[0] - 1)].index, mean_columns\n", + "].mean(numeric_only=True, axis=0)\n", "\n", "# Ratio sessions / session in level & performance\n", - "summary_df['ratio_l11'] = summary_df['num_subjects_l11']*100 /summary_df['num_subjects']\n", - "summary_df['ratio_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str]*100 /summary_df['num_subjects']\n", + "summary_df[\"ratio_l11\"] = (\n", + " summary_df[\"num_subjects_l11\"] * 100 / summary_df[\"num_subjects\"]\n", + ")\n", + "summary_df[\"ratio_l11>\" + perf_str] = (\n", + " summary_df[\"num_subjects_l11>\" + perf_str] * 100 / summary_df[\"num_subjects\"]\n", + ")\n", "\n", "# Make int columns pretty\n", - "summary_df['num_subjects'] = summary_df['num_subjects'].astype('Int64')\n", - "summary_df['num_sessions'] = summary_df['num_sessions'].astype('Int64')\n", - "summary_df['num_sessions_l11'] = summary_df['num_sessions_l11'].astype('Int64')\n", - "summary_df['num_subjects_l11'] = summary_df['num_subjects_l11'].astype('Int64')\n", - "summary_df['num_sessions_l11>'+perf_str] = summary_df['num_sessions_l11>'+perf_str].astype('Int64')\n", - "summary_df['num_subjects_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str].astype('Int64')\n", + "summary_df[\"num_subjects\"] = summary_df[\"num_subjects\"].astype(\"Int64\")\n", + "summary_df[\"num_sessions\"] = summary_df[\"num_sessions\"].astype(\"Int64\")\n", + "summary_df[\"num_sessions_l11\"] = summary_df[\"num_sessions_l11\"].astype(\"Int64\")\n", + "summary_df[\"num_subjects_l11\"] = summary_df[\"num_subjects_l11\"].astype(\"Int64\")\n", + "summary_df[\"num_sessions_l11>\" + perf_str] = summary_df[\n", + " \"num_sessions_l11>\" + perf_str\n", + "].astype(\"Int64\")\n", + "summary_df[\"num_subjects_l11>\" + perf_str] = summary_df[\n", + " \"num_subjects_l11>\" + perf_str\n", + "].astype(\"Int64\")\n", "\n", "# Sort properly\n", - "summary_df = summary_df.sort_values(by=['ratio_l11>'+perf_str, 'num_sessions_l11>'+perf_str], ascending=False)\n", - "summary_df = summary_df.set_index('session_location')\n", + "summary_df = summary_df.sort_values(\n", + " by=[\"ratio_l11>\" + perf_str, \"num_sessions_l11>\" + perf_str], ascending=False\n", + ")\n", + "summary_df = summary_df.set_index(\"session_location\")\n", "\n", "# Total row at the end\n", - "summary_df.index = summary_df.index.fillna('Total')\n", + "summary_df.index = summary_df.index.fillna(\"Total\")\n", "idx = summary_df.index\n", - "idx = idx.drop('Total')\n", + "idx = idx.drop(\"Total\")\n", "idx = idx.to_list()\n", - "summary_df = summary_df.reindex(idx + ['Total'])\n", + "summary_df = summary_df.reindex(idx + [\"Total\"])\n", "summary_df" ] }, @@ -1501,7 +1576,7 @@ } ], "source": [ - "96/168" + "96 / 168" ] }, { @@ -1925,7 +2000,7 @@ } ], "source": [ - "metrics_df_f = metrics_df_f.sort_values(by='performance', ascending=False)\n", + "metrics_df_f = metrics_df_f.sort_values(by=\"performance\", ascending=False)\n", "metrics_df_f" ] }, @@ -2040,8 +2115,8 @@ ], "source": [ "query_s = dict()\n", - "query_s['subject_fullname'] = 'ms81_M024'\n", - "query_s['session_date'] = '2022-10-01'\n", + "query_s[\"subject_fullname\"] = \"ms81_M024\"\n", + "query_s[\"session_date\"] = \"2022-10-01\"\n", "a = pd.DataFrame((acquisition.Session & query_s).fetch(as_dict=True))\n", "a" ] diff --git a/notebooks/psychometrics_plot.ipynb b/notebooks/psychometrics_plot.ipynb index 5c2507b3..0dc025c1 100644 --- a/notebooks/psychometrics_plot.ipynb +++ b/notebooks/psychometrics_plot.ipynb @@ -37,15 +37,11 @@ "metadata": {}, "outputs": [], "source": [ - "dj.config['database.host'] = 'datajoint00.pni.princeton.edu'\n", - "dj.config['enable_python_native_blobs'] = True\n", + "dj.config[\"database.host\"] = \"datajoint00.pni.princeton.edu\"\n", + "dj.config[\"enable_python_native_blobs\"] = True\n", "\n", - "dj.config['stores'] = {\n", - " 'extstorage':\n", - " {\n", - " 'location': '/Volumes/u19_dj/external_dj_blobs',\n", - " 'protocol': 'file'\n", - " }\n", + "dj.config[\"stores\"] = {\n", + " \"extstorage\": {\"location\": \"/Volumes/u19_dj/external_dj_blobs\", \"protocol\": \"file\"}\n", "}" ] }, @@ -73,7 +69,7 @@ ], "source": [ "dj.conn()\n", - "behavior = dj.create_virtual_module('subject', 'u19_behavior')" + "behavior = dj.create_virtual_module(\"subject\", \"u19_behavior\")" ] }, { @@ -90,7 +86,7 @@ "outputs": [], "source": [ "def psychometrics_function(x, O, A, lambd, x0):\n", - " return O + A/(1+np.exp(-(x-x0)/lambd))" + " return O + A / (1 + np.exp(-(x - x0) / lambd))" ] }, { @@ -108,95 +104,107 @@ "source": [ "def psychFit(deltaBins, numR, numL, choices):\n", "\n", - " numRight = np.zeros(len(deltaBins))\n", - " numTrials = np.zeros(len(deltaBins))\n", - " trialDelta = np.zeros(len(deltaBins))\n", - " phat = np.zeros(len(deltaBins))\n", - " pci = np.zeros((2, len(deltaBins)))\n", - " \n", - " #Evidence variable\n", - " nCues_RminusL = numR - numL\n", - " #Correct deltaBin & trialBin to produce same result as Matlab psychFit\n", - " deltaBins_search = deltaBins.astype(float) - 1.5\n", - " trialBin = np.searchsorted(deltaBins_search, nCues_RminusL, side='right')\n", + " numRight = np.zeros(len(deltaBins))\n", + " numTrials = np.zeros(len(deltaBins))\n", + " trialDelta = np.zeros(len(deltaBins))\n", + " phat = np.zeros(len(deltaBins))\n", + " pci = np.zeros((2, len(deltaBins)))\n", + "\n", + " # Evidence variable\n", + " nCues_RminusL = numR - numL\n", + " # Correct deltaBin & trialBin to produce same result as Matlab psychFit\n", + " deltaBins_search = deltaBins.astype(float) - 1.5\n", + " trialBin = np.searchsorted(deltaBins_search, nCues_RminusL, side=\"right\")\n", " trialBin -= 1\n", - " \n", - " #Put into evidence bins all Trials with corresponding choices\n", + "\n", + " # Put into evidence bins all Trials with corresponding choices\n", " for iTrial in range(len(choices)):\n", - " numTrials[trialBin[iTrial]] = numTrials[trialBin[iTrial]] + 1\n", + " numTrials[trialBin[iTrial]] = numTrials[trialBin[iTrial]] + 1\n", " if choices[iTrial] == 2:\n", - " numRight[trialBin[iTrial]] = numRight[trialBin[iTrial]] + 1\n", - " \n", - " trialDelta[trialBin[iTrial]] = trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial]\n", - " \n", - " with np.errstate(divide='ignore', invalid='ignore'):\n", - " trialDelta = np.true_divide(trialDelta, numTrials)\n", - " \n", + " numRight[trialBin[iTrial]] = numRight[trialBin[iTrial]] + 1\n", + "\n", + " trialDelta[trialBin[iTrial]] = (\n", + " trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial]\n", + " )\n", + "\n", + " with np.errstate(divide=\"ignore\", invalid=\"ignore\"):\n", + " trialDelta = np.true_divide(trialDelta, numTrials)\n", + "\n", " # Select only bins with trials\n", - " idx_zero = numTrials == 0 \n", + " idx_zero = numTrials == 0\n", " numTrials_nz = numTrials[~idx_zero]\n", - " numRight_nz = numRight[~idx_zero]\n", - " \n", - " #(Binomial proportion confidence interval given k successes, n trials)\n", - " phat_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=0, interval='jeffreys')\n", - " pci_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=1-0.1587, interval='jeffreys')\n", - " \n", - " #Correct confidence intervals and expected outcomes for bins with no trials (ci = [0 1], hat = 0.5)\n", + " numRight_nz = numRight[~idx_zero]\n", + "\n", + " # (Binomial proportion confidence interval given k successes, n trials)\n", + " phat_nz = binom_conf_interval(\n", + " numRight_nz, numTrials_nz, confidence_level=0, interval=\"jeffreys\"\n", + " )\n", + " pci_nz = binom_conf_interval(\n", + " numRight_nz, numTrials_nz, confidence_level=1 - 0.1587, interval=\"jeffreys\"\n", + " )\n", + "\n", + " # Correct confidence intervals and expected outcomes for bins with no trials (ci = [0 1], hat = 0.5)\n", " phat_nz = phat_nz[0]\n", " phat[~idx_zero] = phat_nz\n", - " phat[idx_zero] = 0.5\n", + " phat[idx_zero] = 0.5\n", " pci[0][~idx_zero] = pci_nz[0]\n", - " pci[0][idx_zero] = 0\n", + " pci[0][idx_zero] = 0\n", " pci[1][~idx_zero] = pci_nz[1]\n", - " pci[1][idx_zero] = 1\n", - " \n", + " pci[1][idx_zero] = 1\n", + "\n", " # (Logistic function fit) only valid if we have at least 5 bins with trials\n", " if np.count_nonzero(~idx_zero) < 5:\n", " is_there_psychometric = False\n", " else:\n", " is_there_psychometric = True\n", - " #Get weight matrix to \"reproduce\" Matlab fit \n", - " #https://stackoverflow.com/questions/58983113/scipy-curve-fit-vs-matlab-fit-weighted-nonlinear-least-squares\n", - " # matlab -> 'Weights' , ((pci(sel,2) - pci(sel,1))/2).^-2 \n", + " # Get weight matrix to \"reproduce\" Matlab fit\n", + " # https://stackoverflow.com/questions/58983113/scipy-curve-fit-vs-matlab-fit-weighted-nonlinear-least-squares\n", + " # matlab -> 'Weights' , ((pci(sel,2) - pci(sel,1))/2).^-2\n", " # python -> sigma = diagonal_matrix(1/weights)\n", - " \n", - " weight_array = np.power((pci[1][~idx_zero] - pci[0][~idx_zero])/2,2)\n", - " sigma_fit = np.diag(weight_array)\n", - " \n", - " psychometric, pcov = curve_fit(psychometrics_function, deltaBins[~idx_zero], phat[~idx_zero], \\\n", - " p0 = (0, 1, 3, 0),sigma = sigma_fit, maxfev=40000)\n", "\n", - " #Append a row of nans to confidence intervals . whyy ??\n", - " aux_vec = np.empty((1,pci.shape[1]))\n", + " weight_array = np.power((pci[1][~idx_zero] - pci[0][~idx_zero]) / 2, 2)\n", + " sigma_fit = np.diag(weight_array)\n", + "\n", + " psychometric, pcov = curve_fit(\n", + " psychometrics_function,\n", + " deltaBins[~idx_zero],\n", + " phat[~idx_zero],\n", + " p0=(0, 1, 3, 0),\n", + " sigma=sigma_fit,\n", + " maxfev=40000,\n", + " )\n", + "\n", + " # Append a row of nans to confidence intervals . whyy ??\n", + " aux_vec = np.empty((1, pci.shape[1]))\n", " aux_vec[:] = np.nan\n", " pci = np.vstack((pci, aux_vec))\n", - " \n", + "\n", " # x vector for plotting\n", - " delta = np.linspace(deltaBins[0]-2, deltaBins[-1]+2, num=50)\n", + " delta = np.linspace(deltaBins[0] - 2, deltaBins[-1] + 2, num=50)\n", "\n", " # Repeat trialDelta 3 times for errorX why ??\n", " errorX = np.tile(trialDelta[~idx_zero], 3)\n", - " \n", + "\n", " # Confidence intervals are errorY, as a vector\n", - " errorY = np.stack(pci[:,~idx_zero])\n", + " errorY = np.stack(pci[:, ~idx_zero])\n", " errorY = errorY.flatten()\n", - " \n", + "\n", " # Fill dictionary of results\n", " fit_results = dict()\n", - " fit_results['delta_bins'] = deltaBins[~idx_zero]\n", - " fit_results['delta_data'] = trialDelta[~idx_zero]\n", - " fit_results['pright_data'] = 100*phat[~idx_zero]\n", - " fit_results['delta_error'] = errorX\n", - " fit_results['pright_error'] = 100*errorY\n", - " \n", + " fit_results[\"delta_bins\"] = deltaBins[~idx_zero]\n", + " fit_results[\"delta_data\"] = trialDelta[~idx_zero]\n", + " fit_results[\"pright_data\"] = 100 * phat[~idx_zero]\n", + " fit_results[\"delta_error\"] = errorX\n", + " fit_results[\"pright_error\"] = 100 * errorY\n", + "\n", " if is_there_psychometric:\n", - " fit_results['delta_fit'] = delta\n", - " fit_results['pright_fit'] = psychometrics_function(delta, *psychometric)*100\n", + " fit_results[\"delta_fit\"] = delta\n", + " fit_results[\"pright_fit\"] = psychometrics_function(delta, *psychometric) * 100\n", " else:\n", - " fit_results['delta_fit'] = np.empty([0])\n", - " fit_results['pright_fit'] = np.empty([0])\n", - " \n", - " return fit_results\n" + " fit_results[\"delta_fit\"] = np.empty([0])\n", + " fit_results[\"pright_fit\"] = np.empty([0])\n", + "\n", + " return fit_results" ] }, { @@ -212,51 +220,58 @@ "metadata": {}, "outputs": [], "source": [ - "keys = [{\n", - " 'subject_fullname': 'ms81_M005',\n", - " 'session_date': datetime.date(2021, 5, 8),\n", - " 'session_number': 0\n", - "}, {\n", - " 'subject_fullname': 'ms81_M004',\n", - " 'session_date': datetime.date(2021, 5, 8),\n", - " 'session_number': 0\n", - "}, {\n", - " 'subject_fullname': 'ms81_M004',\n", - " 'session_date': datetime.date(2021, 5, 7),\n", - " 'session_number': 0\n", - "}, {\n", - " 'subject_fullname': 'ms81_M003',\n", - " 'session_date': datetime.date(2021, 5, 7),\n", - " 'session_number': 0\n", - "}, {\n", - " 'subject_fullname': 'ms81_M004',\n", - " 'session_date': datetime.date(2021, 5, 6),\n", - " 'session_number': 0\n", - "}, {\n", - " 'subject_fullname': 'ms81_M005',\n", - " 'session_date': datetime.date(2021, 5, 5),\n", - " 'session_number': 0\n", - "}, {\n", - " 'subject_fullname': 'ms81_M002',\n", - " 'session_date': datetime.date(2021, 5, 3),\n", - " 'session_number': 0\n", - "}]\n", + "keys = [\n", + " {\n", + " \"subject_fullname\": \"ms81_M005\",\n", + " \"session_date\": datetime.date(2021, 5, 8),\n", + " \"session_number\": 0,\n", + " },\n", + " {\n", + " \"subject_fullname\": \"ms81_M004\",\n", + " \"session_date\": datetime.date(2021, 5, 8),\n", + " \"session_number\": 0,\n", + " },\n", + " {\n", + " \"subject_fullname\": \"ms81_M004\",\n", + " \"session_date\": datetime.date(2021, 5, 7),\n", + " \"session_number\": 0,\n", + " },\n", + " {\n", + " \"subject_fullname\": \"ms81_M003\",\n", + " \"session_date\": datetime.date(2021, 5, 7),\n", + " \"session_number\": 0,\n", + " },\n", + " {\n", + " \"subject_fullname\": \"ms81_M004\",\n", + " \"session_date\": datetime.date(2021, 5, 6),\n", + " \"session_number\": 0,\n", + " },\n", + " {\n", + " \"subject_fullname\": \"ms81_M005\",\n", + " \"session_date\": datetime.date(2021, 5, 5),\n", + " \"session_number\": 0,\n", + " },\n", + " {\n", + " \"subject_fullname\": \"ms81_M002\",\n", + " \"session_date\": datetime.date(2021, 5, 3),\n", + " \"session_number\": 0,\n", + " },\n", + "]\n", "\n", - "#List of colors\n", - "colors = ['r', 'y', 'k', 'b']\n", + "# List of colors\n", + "colors = [\"r\", \"y\", \"k\", \"b\"]\n", "\n", - "#Assign a color to each subject\n", + "# Assign a color to each subject\n", "keys_df = pd.DataFrame(keys)\n", "subjects = keys_df.subject_fullname.unique()\n", "subjects.sort()\n", "zip_iterator = zip(subjects, colors)\n", "color_dict = dict(zip_iterator)\n", "\n", - "#Create a color patch for plot legend\n", + "# Create a color patch for plot legend\n", "patches_legend = []\n", "for key in color_dict:\n", - " patches_legend.append(mpatches.Patch(color=color_dict[key], label=key))\n", - "\n" + " patches_legend.append(mpatches.Patch(color=color_dict[key], label=key))" ] }, { @@ -328,51 +343,67 @@ } ], "source": [ - "\n", "plt.figure(figsize=(15, 12))\n", "fraction_correct = np.zeros(len(keys))\n", "for key_idx, key in enumerate(keys):\n", - "\n", " key_spec = behavior.TowersBlock() & key & \"level = 11\" & \"block_performance > 0.65\"\n", " thissession = behavior.TowersBlock().Trial() & key_spec\n", - " \n", + "\n", " # Fetch session\n", - " session_info = pd.DataFrame(thissession.fetch('choice', 'trial_type', 'cue_presence_left', 'cue_presence_right', as_dict=True))\n", - " \n", + " session_info = pd.DataFrame(\n", + " thissession.fetch(\n", + " \"choice\",\n", + " \"trial_type\",\n", + " \"cue_presence_left\",\n", + " \"cue_presence_right\",\n", + " as_dict=True,\n", + " )\n", + " )\n", + "\n", " # Choice and trial type as integer\n", - " session_info['trial_type_int'] = 0\n", - " session_info.loc[session_info['trial_type'] == 'L','trial_type_int'] = 1 \n", - " session_info.loc[session_info['trial_type'] == 'R','trial_type_int'] = 2 \n", - " session_info['choice_int'] = 0\n", - " session_info.loc[session_info['choice'] == 'L','choice_int'] = 1 \n", - " session_info.loc[session_info['choice'] == 'R','choice_int'] = 2\n", - " \n", + " session_info[\"trial_type_int\"] = 0\n", + " session_info.loc[session_info[\"trial_type\"] == \"L\", \"trial_type_int\"] = 1\n", + " session_info.loc[session_info[\"trial_type\"] == \"R\", \"trial_type_int\"] = 2\n", + " session_info[\"choice_int\"] = 0\n", + " session_info.loc[session_info[\"choice\"] == \"L\", \"choice_int\"] = 1\n", + " session_info.loc[session_info[\"choice\"] == \"R\", \"choice_int\"] = 2\n", + "\n", " # number of towers per trial\n", - " session_info['cue_presence_left'] = session_info['cue_presence_left'].apply(lambda x: np.count_nonzero(x))\n", - " session_info['cue_presence_right'] = session_info['cue_presence_right'].apply(lambda x: np.count_nonzero(x))\n", - " \n", - " \n", - " fraction_correct[key_idx] = np.sum(session_info['choice'].values == session_info['trial_type'].values)/session_info.shape[0]\n", - " \n", + " session_info[\"cue_presence_left\"] = session_info[\"cue_presence_left\"].apply(\n", + " lambda x: np.count_nonzero(x)\n", + " )\n", + " session_info[\"cue_presence_right\"] = session_info[\"cue_presence_right\"].apply(\n", + " lambda x: np.count_nonzero(x)\n", + " )\n", + "\n", + " fraction_correct[key_idx] = (\n", + " np.sum(session_info[\"choice\"].values == session_info[\"trial_type\"].values)\n", + " / session_info.shape[0]\n", + " )\n", + "\n", " # Call Fit to sigmoid function\n", - " fit_dict = psychFit(deltaBins, session_info['cue_presence_right'].values, \\\n", - " session_info['cue_presence_left'].values, session_info['choice_int'].values)\n", - " \n", - " #Select color from color dictionary\n", - " color_plot = color_dict[key['subject_fullname']]\n", - " #Plot results\n", - " plt.plot(fit_dict['delta_bins'], fit_dict['pright_data'], 'o', color = color_plot)\n", - " if fit_dict['delta_fit'].shape[0] > 0:\n", - " plt.plot(fit_dict['delta_fit'], fit_dict['pright_fit'], '--', color = color_plot)\n", - " \n", + " fit_dict = psychFit(\n", + " deltaBins,\n", + " session_info[\"cue_presence_right\"].values,\n", + " session_info[\"cue_presence_left\"].values,\n", + " session_info[\"choice_int\"].values,\n", + " )\n", + "\n", + " # Select color from color dictionary\n", + " color_plot = color_dict[key[\"subject_fullname\"]]\n", + " # Plot results\n", + " plt.plot(fit_dict[\"delta_bins\"], fit_dict[\"pright_data\"], \"o\", color=color_plot)\n", + " if fit_dict[\"delta_fit\"].shape[0] > 0:\n", + " plt.plot(fit_dict[\"delta_fit\"], fit_dict[\"pright_fit\"], \"--\", color=color_plot)\n", + "\n", "plt.xticks(fontsize=16)\n", "plt.yticks(fontsize=16)\n", "plt.xlabel(\"total evidence (L-R)\", fontsize=18)\n", "plt.ylabel(\"% Turned left\", fontsize=18)\n", "plt.ylim([-1, 101])\n", - "#pl.xlim([-11,11])\n", + "# pl.xlim([-11,11])\n", "mean_correct_string = f\"{np.mean(fraction_correct):2.2f}\"\n", - "plt.title('Fraction correct responses '+mean_correct_string, fontsize=20)\n", + "plt.title(\"Fraction correct responses \" + mean_correct_string, fontsize=20)\n", "plt.legend(handles=patches_legend, fontsize=18)\n", "print()" ] diff --git a/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb b/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb index 49cedc90..060513b6 100644 --- a/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb +++ b/notebooks/pupillometry_notebooks/grade_pupillometry_processing.ipynb @@ -62,8 +62,8 @@ ], "source": [ "dj.conn()\n", - "pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')\n", - "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')" + "pupillometry_db = dj.create_virtual_module(\"pupillometry\", \"u19_pupillometry\")\n", + "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")" ] }, { @@ -93,8 +93,8 @@ "frameNumber = 1000\n", "\n", "key = dict()\n", - "key['subject_fullname'] = 'efonseca_ef111_act111'\n", - "key['session_date'] = '2023-06-24'\n", + "key[\"subject_fullname\"] = \"efonseca_ef111_act111\"\n", + "key[\"session_date\"] = \"2023-06-24\"\n", "\n", "key" ] @@ -122,26 +122,34 @@ ], "source": [ "# Get video location\n", - "key_pupil = (acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key).fetch(as_dict=True)[0]\n", - "db_pupil_data = (pupillometry_db.PupillometrySessionModelData & key).fetch(as_dict=True)[0]\n", - "pupillometry_dir = dj.config.get('custom', {}).get('pupillometry_root_data_dir',None)\n", + "key_pupil = (\n", + " acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key\n", + ").fetch(as_dict=True)[0]\n", + "db_pupil_data = (pupillometry_db.PupillometrySessionModelData & key).fetch(\n", + " as_dict=True\n", + ")[0]\n", + "pupillometry_dir = dj.config.get(\"custom\", {}).get(\"pupillometry_root_data_dir\", None)\n", "if pupillometry_dir is None:\n", - " raise Exception('pupillometry_root_data_dir not found in config, run initial_conf.py again')\n", + " raise Exception(\n", + " \"pupillometry_root_data_dir not found in config, run initial_conf.py again\"\n", + " )\n", "pupillometry_raw_dir = pupillometry_dir[0]\n", - "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil['remote_path_video_file'])\n", - "print('videoPath', videoPath)\n", + "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil[\"remote_path_video_file\"])\n", + "print(\"videoPath\", videoPath)\n", "\n", "# Get output location\n", "pupillometry_processed_dir = pupillometry_dir[1]\n", - "output_dir = pathlib.Path(pupillometry_processed_dir,pathlib.Path(key_pupil['remote_path_video_file']).parent)\n", + "output_dir = pathlib.Path(\n", + " pupillometry_processed_dir, pathlib.Path(key_pupil[\"remote_path_video_file\"]).parent\n", + ")\n", "\n", - "#Find h5 files (output from deeplabcut)\n", - "h5_files = glob.glob(str(output_dir) + '/*.h5')\n", + "# Find h5 files (output from deeplabcut)\n", + "h5_files = glob.glob(str(output_dir) + \"/*.h5\")\n", "if len(h5_files) != 1:\n", - " raise Exception('Didn''t find any h5 files after deeplabcut analyze_video')\n", + " raise Exception(\"Didnt find any h5 files after deeplabcut analyze_video\")\n", "else:\n", " h5_files = h5_files[0]\n", - "print('h5_files', h5_files)" + "print(\"h5_files\", h5_files)" ] }, { @@ -208,7 +216,7 @@ } ], "source": [ - "labels.loc[:, (bodyparts[0],'likelihood')]" + "labels.loc[:, (bodyparts[0], \"likelihood\")]" ] }, { @@ -219,8 +227,11 @@ "source": [ "from scipy.signal import find_peaks\n", "\n", - "x = labels.loc[:, (bodyparts[2],'y')],\n", - "peaks, _ = find_peaks(x, threshold=30, )\n", + "x = (labels.loc[:, (bodyparts[2], \"y\")],)\n", + "peaks, _ = find_peaks(\n", + " x,\n", + " threshold=30,\n", + ")\n", "plt.plot(x)\n", "plt.plot(peaks, x[peaks], \"x\")\n", "plt.plot(np.zeros_like(x), \"--\", color=\"gray\")\n", @@ -246,7 +257,7 @@ } ], "source": [ - "plt.plot(labels.loc[:, (bodyparts[2],'y')], linewidth=0.5, color ='k')\n", + "plt.plot(labels.loc[:, (bodyparts[2], \"y\")], linewidth=0.5, color=\"k\")\n", "\n", "\n", "plt.xlabel(\"frame#\", fontsize=12)\n", @@ -333,14 +344,12 @@ } ], "source": [ - "\n", - "\n", "# Take a subset from the data\n", "subset = labels.loc[frameNumber]\n", "# Get the pupil markers (0 to 8) from the multiindex array obtained from deeplabcut\n", - "x = subset.xs('x', level='coords').to_numpy()[0:8]\n", - "y = subset.xs('y', level='coords').to_numpy()[0:8]\n", - "xy = np.column_stack((x,y))\n", + "x = subset.xs(\"x\", level=\"coords\").to_numpy()[0:8]\n", + "y = subset.xs(\"y\", level=\"coords\").to_numpy()[0:8]\n", + "xy = np.column_stack((x, y))\n", "\n", "# Estimate an ellipse based on the points generated around the pupil, this is enough to get an estimate of the pupil area in pixels\n", "ellipse = EllipseModel()\n", @@ -348,11 +357,17 @@ "\n", "# Validate by ploting the points and the fitting ellipse on the image from the video\n", "fig, ax = plt.subplots()\n", - "plt.rcParams['figure.dpi'] = 250\n", - "img = cv2.imread(str(frameNumber) + '.jpg')\n", - "ax.scatter(xy[:,0], xy[:,1], s=0.1, color='orange')\n", + "plt.rcParams[\"figure.dpi\"] = 250\n", + "img = cv2.imread(str(frameNumber) + \".jpg\")\n", + "ax.scatter(xy[:, 0], xy[:, 1], s=0.1, color=\"orange\")\n", "# Draw the ellipse on the original image\n", - "cy, cx = ellipse_perimeter(int(ellipse.params[1]), int(ellipse.params[0]), int(ellipse.params[3]), int(ellipse.params[2]), ellipse.params[4])\n", + "cy, cx = ellipse_perimeter(\n", + " int(ellipse.params[1]),\n", + " int(ellipse.params[0]),\n", + " int(ellipse.params[3]),\n", + " int(ellipse.params[2]),\n", + " ellipse.params[4],\n", + ")\n", "img[cy, cx] = 255\n", "ax.imshow(img)\n", "\n", @@ -366,7 +381,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(db_pupil_data['pupil_diameter'], columns=['PupilDiameter'])" + "df = pd.DataFrame(db_pupil_data[\"pupil_diameter\"], columns=[\"PupilDiameter\"])" ] }, { @@ -388,16 +403,15 @@ } ], "source": [ - "\n", "# Get a boolean array where true correspond to the frame with an outlier diameter\n", - "zscore = np.abs(stats.zscore(df, nan_policy='omit'))\n", + "zscore = np.abs(stats.zscore(df, nan_policy=\"omit\"))\n", "outlierFlags = np.abs(zscore) > 2\n", "outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: \"OutlierFlag\"})\n", "\n", "# Get the list of outlier frames\n", "from itertools import compress\n", "\n", - "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags['OutlierFlag']))\n", + "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags[\"OutlierFlag\"]))\n", "\n", "\n", "# Retrieve an outlier frames sample from the video for visual inspection\n", @@ -405,20 +419,20 @@ "fig = plt.figure(figsize=(7, 7))\n", "columns = 5\n", "rows = 5\n", - "for i in range(1, columns*rows +1):\n", + "for i in range(1, columns * rows + 1):\n", " # Get total number of frames\n", " totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n", " # Set frame position\n", - " cap.set(cv2.CAP_PROP_POS_FRAMES,outlierFrames[i])\n", + " cap.set(cv2.CAP_PROP_POS_FRAMES, outlierFrames[i])\n", " ret, frame = cap.read()\n", "\n", " fig.add_subplot(rows, columns, i)\n", " plt.title(\"Frame \" + str(outlierFrames[i]), fontsize=5)\n", " plt.imshow(frame)\n", - " plt.axis('off')\n", + " plt.axis(\"off\")\n", "plt.show()\n", "cap.release()\n", - "cv2.destroyAllWindows()\n" + "cv2.destroyAllWindows()" ] }, { @@ -472,29 +486,33 @@ "source": [ "# Concatenate outlier flags array to remove outliers from pupil diameter array\n", "temp = pd.concat([df, outlierFlags], axis=1)\n", - "temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None\n", - "pupilDiameter = temp['PupilDiameter']\n", + "temp.loc[temp[\"OutlierFlag\"] == True, \"PupilDiameter\"] = None\n", + "pupilDiameter = temp[\"PupilDiameter\"]\n", "\n", "# This might be better to leave open for the experimenter.\n", "# Replace the outlier diameter data using a cubic spline interpolator\n", "# pupilDiameter = temp['PupilDiameter'].interpolate(method='spline', order=3, s=0)\n", "\n", - "fig, (ax1, ax2) = plt.subplots(1,2)\n", + "fig, (ax1, ax2) = plt.subplots(1, 2)\n", "# plt.rcParams['figure.dpi'] = 250\n", "\n", "ax1.plot(pupilDiameter[0:3600], linewidth=0.25)\n", "# Filter by using a moving average window of 10 samples\n", - "ax1.plot(pupilDiameter[0:3600].rolling(10).mean(), linewidth=0.5, color='red')\n", - "ax1.set_xlabel('Frame')\n", - "ax1.set_ylabel('Pupil diameter [px]')\n", - "ax1.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n", + "ax1.plot(pupilDiameter[0:3600].rolling(10).mean(), linewidth=0.5, color=\"red\")\n", + "ax1.set_xlabel(\"Frame\")\n", + "ax1.set_ylabel(\"Pupil diameter [px]\")\n", + "ax1.legend(\n", + " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n", + ")\n", "\n", "ax2.plot(pupilDiameter[0:20000], linewidth=0.25)\n", "# Filter by using a moving average window of 10 samples\n", - "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color='red')\n", - "ax2.set_xlabel('Frame')\n", - "ax2.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n", - "#fig.savefig('test.eps', format='eps')" + "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color=\"red\")\n", + "ax2.set_xlabel(\"Frame\")\n", + "ax2.legend(\n", + " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n", + ")\n", + "# fig.savefig('test.eps', format='eps')" ] }, { diff --git a/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb b/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb index 19bc95d5..15fb28ad 100644 --- a/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb +++ b/notebooks/pupillometry_notebooks/insert_pupilometry_data_test.ipynb @@ -23,7 +23,7 @@ "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", "\n", - "try_find_conf_file()\n" + "try_find_conf_file()" ] }, { @@ -54,10 +54,9 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "dj.conn()\n", - "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')" + "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "pupillometry_db = dj.create_virtual_module(\"pupillometry\", \"u19_pupillometry\")" ] }, { @@ -77,13 +76,15 @@ } ], "source": [ - "root_videopath = dj.config['custom']['dlc_root_data_dir']\n", + "root_videopath = dj.config[\"custom\"][\"dlc_root_data_dir\"]\n", "\n", - "modelPath = pathlib.Path(root_videopath, 'MODELS', 'Pupillometry2-Ryan-2022-04-07').as_posix()\n", - "configPath = pathlib.Path(modelPath,'config.yaml').as_posix()\n", + "modelPath = pathlib.Path(\n", + " root_videopath, \"MODELS\", \"Pupillometry2-Ryan-2022-04-07\"\n", + ").as_posix()\n", + "configPath = pathlib.Path(modelPath, \"config.yaml\").as_posix()\n", "\n", - "proccesed_rootpath = dj.config['custom']['dlc_processed_data_dir']\n", - "proccesed_rootpath = pathlib.Path(proccesed_rootpath, 'pupillometry').as_posix()\n", + "proccesed_rootpath = dj.config[\"custom\"][\"dlc_processed_data_dir\"]\n", + "proccesed_rootpath = pathlib.Path(proccesed_rootpath, \"pupillometry\").as_posix()\n", "configPath" ] }, @@ -321,12 +322,20 @@ ], "source": [ "key = 'subject_fullname like \"jounhong%\"'\n", - "video_data_df = pd.DataFrame((pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\"KEY\", 'remote_path_video_file', as_dict=True))\n", + "video_data_df = pd.DataFrame(\n", + " (pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\n", + " \"KEY\", \"remote_path_video_file\", as_dict=True\n", + " )\n", + ")\n", "\n", - "video_data_df['remote_path_video_file'] = 'pupillometry/' + video_data_df['remote_path_video_file']\n", - "video_data_df.loc[0,'remote_path_video_file']\n", + "video_data_df[\"remote_path_video_file\"] = (\n", + " \"pupillometry/\" + video_data_df[\"remote_path_video_file\"]\n", + ")\n", + "video_data_df.loc[0, \"remote_path_video_file\"]\n", "\n", - "video_data_df['processed_path'] = video_data_df['remote_path_video_file'].apply(lambda x : pathlib.Path(x).parent.as_posix())\n", + "video_data_df[\"processed_path\"] = video_data_df[\"remote_path_video_file\"].apply(\n", + " lambda x: pathlib.Path(x).parent.as_posix()\n", + ")\n", "\n", "video_data_df" ] @@ -381,10 +390,13 @@ } ], "source": [ - "\n", "for i in range(video_data_df.shape[0]):\n", - " this_video_path = pathlib.Path(root_videopath, video_data_df.loc[i, 'remote_path_video_file']).as_posix()\n", - " this_results_path = pathlib.Path(proccesed_rootpath, video_data_df.loc[i, 'processed_path']).as_posix()\n", + " this_video_path = pathlib.Path(\n", + " root_videopath, video_data_df.loc[i, \"remote_path_video_file\"]\n", + " ).as_posix()\n", + " this_results_path = pathlib.Path(\n", + " proccesed_rootpath, video_data_df.loc[i, \"processed_path\"]\n", + " ).as_posix()\n", "\n", " pickle_file = pu.get_filepattern_paths(this_results_path, \"/*pupil_diameter.pickle\")\n", "\n", @@ -392,27 +404,26 @@ "\n", " if len(pickle_file) > 0:\n", " pickle_file = pickle_file[0]\n", - " \n", + "\n", " objects = []\n", - " with (open(pickle_file, \"rb\")) as openfile:\n", + " with open(pickle_file, \"rb\") as openfile:\n", " while True:\n", " try:\n", " object = pickle.load(openfile)\n", " except EOFError:\n", " break\n", "\n", - "\n", " pupillometry_data_array = object\n", "\n", " print(type(pupillometry_data_array))\n", " print(pupillometry_data_array)\n", "\n", " key = dict()\n", - " key['subject_fullname'] = video_data_df.loc[i, 'subject_fullname']\n", - " key['session_date'] = video_data_df.loc[i, 'session_date']\n", + " key[\"subject_fullname\"] = video_data_df.loc[i, \"subject_fullname\"]\n", + " key[\"session_date\"] = video_data_df.loc[i, \"session_date\"]\n", " new_key = (acquisition_db.Session & key).fetch1(\"KEY\")\n", - " new_key['pupil_diameter'] = pupillometry_data_array\n", - " pupillometry_db.PupillometryData.insert1(new_key, allow_direct_insert=True)\n" + " new_key[\"pupil_diameter\"] = pupillometry_data_array\n", + " pupillometry_db.PupillometryData.insert1(new_key, allow_direct_insert=True)" ] }, { @@ -420,13 +431,7 @@ "execution_count": 15, "metadata": {}, "outputs": [], - "source": [ - "\n", - "\n", - "\n", - "\n", - "\n" - ] + "source": [] } ], "metadata": { diff --git a/notebooks/pupillometry_notebooks/pupillometry.ipynb b/notebooks/pupillometry_notebooks/pupillometry.ipynb index 21357268..3924ac9d 100644 --- a/notebooks/pupillometry_notebooks/pupillometry.ipynb +++ b/notebooks/pupillometry_notebooks/pupillometry.ipynb @@ -25,12 +25,12 @@ "\n", "import deeplabcut\n", "\n", - "modelPath = 'Pupillometry2-Ryan-2022-04-07'\n", - "configPath = os.path.join(modelPath,'config.yaml')\n", - "videoPath = '/mnt/cup/braininit/Data/Raw/video_pupillometry/jounhong/jounhong_TH_276/20220401_g0/jounhong_TH_276_20220401_g0.mj2'\n", - "output_path = '/mnt/cup/braininit/Data/Processed/video_pupillometry/jounhong/jounhong_TH_276/20220401_g0/jounhong_TH_276_20220401_g0DLC_resnet101_Pupillometry2Apr7shuffle1_1030000.h5'\n", - "rawAnalysisResultPath = 'rawAnalysisResults/'\n", - "#deeplabcut.analyze_videos(configPath, videoPath, destfolder=rawAnalysisResultPath)" + "modelPath = \"Pupillometry2-Ryan-2022-04-07\"\n", + "configPath = os.path.join(modelPath, \"config.yaml\")\n", + "videoPath = \"/mnt/cup/braininit/Data/Raw/video_pupillometry/jounhong/jounhong_TH_276/20220401_g0/jounhong_TH_276_20220401_g0.mj2\"\n", + "output_path = \"/mnt/cup/braininit/Data/Processed/video_pupillometry/jounhong/jounhong_TH_276/20220401_g0/jounhong_TH_276_20220401_g0DLC_resnet101_Pupillometry2Apr7shuffle1_1030000.h5\"\n", + "rawAnalysisResultPath = \"rawAnalysisResults/\"\n", + "# deeplabcut.analyze_videos(configPath, videoPath, destfolder=rawAnalysisResultPath)" ] }, { @@ -483,10 +483,10 @@ "# Get total number of frames\n", "totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n", "# Set frame position\n", - "cap.set(cv2.CAP_PROP_POS_FRAMES,frameNumber)\n", + "cap.set(cv2.CAP_PROP_POS_FRAMES, frameNumber)\n", "\n", "ret, frame = cap.read()\n", - "name = str(frameNumber) + '.jpg'\n", + "name = str(frameNumber) + \".jpg\"\n", "cv2.imwrite(name, frame)\n", "\n", "cap.release()\n", @@ -519,9 +519,9 @@ "# Take a subset from the data\n", "subset = labels.loc[frameNumber]\n", "# Get the pupil markers (0 to 8) from the multiindex array obtained from deeplabcut\n", - "x = subset.xs('x', level='coords').to_numpy()[0:8]\n", - "y = subset.xs('y', level='coords').to_numpy()[0:8]\n", - "xy = np.column_stack((x,y))\n", + "x = subset.xs(\"x\", level=\"coords\").to_numpy()[0:8]\n", + "y = subset.xs(\"y\", level=\"coords\").to_numpy()[0:8]\n", + "xy = np.column_stack((x, y))\n", "\n", "# Estimate an ellipse based on the points generated around the pupil, this is enough to get an estimate of the pupil area in pixels\n", "ellipse = EllipseModel()\n", @@ -529,11 +529,17 @@ "\n", "# Validate by ploting the points and the fitting ellipse on the image from the video\n", "fig, ax = plt.subplots()\n", - "plt.rcParams['figure.dpi'] = 250\n", - "img = cv2.imread(str(frameNumber) + '.jpg')\n", - "ax.scatter(xy[:,0], xy[:,1], s=0.1, color='orange')\n", + "plt.rcParams[\"figure.dpi\"] = 250\n", + "img = cv2.imread(str(frameNumber) + \".jpg\")\n", + "ax.scatter(xy[:, 0], xy[:, 1], s=0.1, color=\"orange\")\n", "# Draw the ellipse on the original image\n", - "cy, cx = ellipse_perimeter(int(ellipse.params[1]), int(ellipse.params[0]), int(ellipse.params[3]), int(ellipse.params[2]), ellipse.params[4])\n", + "cy, cx = ellipse_perimeter(\n", + " int(ellipse.params[1]),\n", + " int(ellipse.params[0]),\n", + " int(ellipse.params[3]),\n", + " int(ellipse.params[2]),\n", + " ellipse.params[4],\n", + ")\n", "img[cy, cx] = 255\n", "ax.imshow(img)\n", "\n", @@ -561,20 +567,20 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(np.zeros(1), columns=['PupilDiameter'])\n", + "df = pd.DataFrame(np.zeros(1), columns=[\"PupilDiameter\"])\n", "for i in range(labels.index.size):\n", " subset = labels.loc[i]\n", - " x = subset.xs('x', level='coords').to_numpy()[0:8]\n", - " y = subset.xs('y', level='coords').to_numpy()[0:8]\n", - " xy = np.column_stack((x,y))\n", + " x = subset.xs(\"x\", level=\"coords\").to_numpy()[0:8]\n", + " y = subset.xs(\"y\", level=\"coords\").to_numpy()[0:8]\n", + " xy = np.column_stack((x, y))\n", " # Fit the points to an ellipse and get the parameters (estimate X center coordinate, estimate Y center coordinate, a, b, theta)\n", " ellipse = EllipseModel()\n", " ellipse.estimate(xy)\n", " # Calculate the area of the ellipse from the parameters a and b\n", " ellipseArea = np.pi * ellipse.params[2] * ellipse.params[3]\n", " # Get the diameter of a circle from the area of the ellipse\n", - " pupilDiameter = 2 * np.sqrt(ellipseArea/np.pi)\n", - " df.loc[i] = pupilDiameter\n" + " pupilDiameter = 2 * np.sqrt(ellipseArea / np.pi)\n", + " df.loc[i] = pupilDiameter" ] }, { @@ -635,27 +641,27 @@ "# Get the list of outlier frames\n", "from itertools import compress\n", "\n", - "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags['OutlierFlag']))\n", + "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags[\"OutlierFlag\"]))\n", "\n", "# Retrieve an outlier frames sample from the video for visual inspection\n", - "cap = cv2.VideoCapture('jounhong_TH_276_20220406_g0.mj2')\n", + "cap = cv2.VideoCapture(\"jounhong_TH_276_20220406_g0.mj2\")\n", "fig = plt.figure(figsize=(7, 7))\n", "columns = 5\n", "rows = 5\n", - "for i in range(1, columns*rows +1):\n", + "for i in range(1, columns * rows + 1):\n", " # Get total number of frames\n", " totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n", " # Set frame position\n", - " cap.set(cv2.CAP_PROP_POS_FRAMES,outlierFrames[i])\n", + " cap.set(cv2.CAP_PROP_POS_FRAMES, outlierFrames[i])\n", " ret, frame = cap.read()\n", "\n", " fig.add_subplot(rows, columns, i)\n", " plt.title(\"Frame \" + str(outlierFrames[i]), fontsize=5)\n", " plt.imshow(frame)\n", - " plt.axis('off')\n", + " plt.axis(\"off\")\n", "plt.show()\n", "cap.release()\n", - "cv2.destroyAllWindows()\n" + "cv2.destroyAllWindows()" ] }, { @@ -689,29 +695,33 @@ "source": [ "# Concatenate outlier flags array to remove outliers from pupil diameter array\n", "temp = pd.concat([df, outlierFlags], axis=1)\n", - "temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None\n", - "pupilDiameter = temp['PupilDiameter']\n", + "temp.loc[temp[\"OutlierFlag\"] == True, \"PupilDiameter\"] = None\n", + "pupilDiameter = temp[\"PupilDiameter\"]\n", "\n", "# This might be better to leave open for the experimenter.\n", "# Replace the outlier diameter data using a cubic spline interpolator\n", "# pupilDiameter = temp['PupilDiameter'].interpolate(method='spline', order=3, s=0)\n", "\n", - "fig, (ax1, ax2) = plt.subplots(1,2)\n", + "fig, (ax1, ax2) = plt.subplots(1, 2)\n", "# plt.rcParams['figure.dpi'] = 250\n", "\n", "ax1.plot(pupilDiameter[0:3600], linewidth=0.25)\n", "# Filter by using a moving average window of 10 samples\n", - "ax1.plot(pupilDiameter[0:3600].rolling(10).mean(), linewidth=0.5, color='red')\n", - "ax1.set_xlabel('Frame')\n", - "ax1.set_ylabel('Pupil diameter [px]')\n", - "ax1.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n", + "ax1.plot(pupilDiameter[0:3600].rolling(10).mean(), linewidth=0.5, color=\"red\")\n", + "ax1.set_xlabel(\"Frame\")\n", + "ax1.set_ylabel(\"Pupil diameter [px]\")\n", + "ax1.legend(\n", + " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n", + ")\n", "\n", "ax2.plot(pupilDiameter[0:20000], linewidth=0.25)\n", "# Filter by using a moving average window of 10 samples\n", - "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color='red')\n", - "ax2.set_xlabel('Frame')\n", - "ax2.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n", - "#fig.savefig('test.eps', format='eps')" + "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color=\"red\")\n", + "ax2.set_xlabel(\"Frame\")\n", + "ax2.legend(\n", + " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n", + ")\n", + "# fig.savefig('test.eps', format='eps')" ] }, { @@ -729,6 +739,7 @@ "source": [ "# Pupil diameter pipeline functions\n", "\n", + "\n", "def analyzeVideo(videoPath, modelPath, destinationFolder):\n", " \"\"\"\n", " Stores the analized video data from videoPath as h5 file in the destination folder using the DLC model in modelPath\n", @@ -739,9 +750,11 @@ " \"\"\"\n", " # Analyze the video using the selected modelPath and videoPath\n", " import os\n", - " configPath = os.path.join(modelPath,'config.yaml')\n", + "\n", + " configPath = os.path.join(modelPath, \"config.yaml\")\n", " deeplabcut.analyze_videos(configPath, videoPath, destfolder=destinationFolder)\n", "\n", + "\n", "def getPupilDiameter(analyzedVideoDataPath):\n", " \"\"\"\n", " Returns a pupil diameter numpy array from an analized video data stored in analyzedVideoDataPath\n", @@ -752,29 +765,30 @@ " \"\"\"\n", " # TODO make the function\n", " from skimage.measure import EllipseModel\n", - " \n", + "\n", " # Read the analyzed video data h5 file\n", " labels = pd.read_hdf(analyzedVideoDataPath)\n", "\n", " # Create a data frame of the same size ad the analyzed video data filled with zeros\n", - " df = pd.DataFrame(np.zeros(1), columns=['PupilDiameter'])\n", + " df = pd.DataFrame(np.zeros(1), columns=[\"PupilDiameter\"])\n", " # For each frame, get the x and y coordinates of the points around the pupil, fit an ellipse and calculate the diameter of a circle with the same area as the ellipse\n", " for i in range(labels.index.size):\n", " subset = labels.loc[i]\n", - " x = subset.xs('x', level='coords').to_numpy()[0:8]\n", - " y = subset.xs('y', level='coords').to_numpy()[0:8]\n", - " xy = np.column_stack((x,y))\n", + " x = subset.xs(\"x\", level=\"coords\").to_numpy()[0:8]\n", + " y = subset.xs(\"y\", level=\"coords\").to_numpy()[0:8]\n", + " xy = np.column_stack((x, y))\n", " # Fit the points to an ellipse and get the parameters (estimate X center coordinate, estimate Y center coordinate, a, b, theta)\n", " ellipse = EllipseModel()\n", " ellipse.estimate(xy)\n", " # Calculate the area of the ellipse from the parameters a and b\n", " ellipseArea = np.pi * ellipse.params[2] * ellipse.params[3]\n", " # Get the diameter of a circle from the area of the ellipse\n", - " pupilDiameter = 2 * np.sqrt(ellipseArea/np.pi)\n", + " pupilDiameter = 2 * np.sqrt(ellipseArea / np.pi)\n", " df.loc[i] = pupilDiameter\n", "\n", " # Get outliers (frames where either the mice have the eyes closed (blink or groom) or deeplabcut fails to track the pupil correctly)\n", " from scipy import stats\n", + "\n", " # Calculate the zscore of the data frame\n", " zscore = np.abs(stats.zscore(df))\n", " # Set a treshold for a valid zscore value (determined empirically)\n", @@ -783,11 +797,10 @@ " outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: \"OutlierFlag\"})\n", " # Concatenate outlier flags array to remove outliers from pupil diameter array\n", " temp = pd.concat([df, outlierFlags], axis=1)\n", - " temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None\n", - " pupilDiameter = temp['PupilDiameter']\n", - " \n", - " return pupilDiameter.to_numpy()\n", - " " + " temp.loc[temp[\"OutlierFlag\"] == True, \"PupilDiameter\"] = None\n", + " pupilDiameter = temp[\"PupilDiameter\"]\n", + "\n", + " return pupilDiameter.to_numpy()" ] }, { @@ -806,7 +819,7 @@ "source": [ "from scripts.conf_file_finding import try_find_conf_file\n", "\n", - "try_find_conf_file()\n" + "try_find_conf_file()" ] }, { @@ -837,8 +850,8 @@ ], "source": [ "dj.conn()\n", - "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')" + "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "pupillometry_db = dj.create_virtual_module(\"pupillometry\", \"u19_pupillometry\")" ] }, { @@ -952,7 +965,7 @@ } ], "source": [ - "dj.ERD(pupillometry_db) -1" + "dj.ERD(pupillometry_db) - 1" ] }, { @@ -972,13 +985,15 @@ } ], "source": [ - "root_videopath = dj.config['custom']['dlc_root_data_dir2']\n", + "root_videopath = dj.config[\"custom\"][\"dlc_root_data_dir2\"]\n", "\n", - "modelPath = pathlib.Path(root_videopath, 'MODELS', 'Pupillometry2-Ryan-2022-04-07').as_posix()\n", - "configPath = pathlib.Path(modelPath,'config.yaml').as_posix()\n", + "modelPath = pathlib.Path(\n", + " root_videopath, \"MODELS\", \"Pupillometry2-Ryan-2022-04-07\"\n", + ").as_posix()\n", + "configPath = pathlib.Path(modelPath, \"config.yaml\").as_posix()\n", "\n", - "proccesed_rootpath = dj.config['custom']['dlc_processed_data_dir2']\n", - "proccesed_rootpath = pathlib.Path(proccesed_rootpath, 'pupillometry').as_posix()\n", + "proccesed_rootpath = dj.config[\"custom\"][\"dlc_processed_data_dir2\"]\n", + "proccesed_rootpath = pathlib.Path(proccesed_rootpath, \"pupillometry\").as_posix()\n", "configPath" ] }, @@ -1216,12 +1231,20 @@ ], "source": [ "key = 'subject_fullname like \"jounhong%\"'\n", - "video_data_df = pd.DataFrame((pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\"KEY\", 'remote_path_video_file', as_dict=True))\n", + "video_data_df = pd.DataFrame(\n", + " (pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\n", + " \"KEY\", \"remote_path_video_file\", as_dict=True\n", + " )\n", + ")\n", "\n", - "video_data_df['remote_path_video_file'] = 'pupillometry/' + video_data_df['remote_path_video_file']\n", - "video_data_df.loc[0,'remote_path_video_file']\n", + "video_data_df[\"remote_path_video_file\"] = (\n", + " \"pupillometry/\" + video_data_df[\"remote_path_video_file\"]\n", + ")\n", + "video_data_df.loc[0, \"remote_path_video_file\"]\n", "\n", - "video_data_df['processed_path'] = video_data_df['remote_path_video_file'].apply(lambda x : pathlib.Path(x).parent.as_posix())\n", + "video_data_df[\"processed_path\"] = video_data_df[\"remote_path_video_file\"].apply(\n", + " lambda x: pathlib.Path(x).parent.as_posix()\n", + ")\n", "\n", "video_data_df" ] @@ -1243,7 +1266,7 @@ } ], "source": [ - "'''\n", + "\"\"\"\n", "for i in range(video_data_df.shape[0]-1,0,-1):\n", " this_video_path = pathlib.Path(root_videopath, video_data_df.loc[i, 'remote_path_video_file']).as_posix()\n", " this_results_path = pathlib.Path(proccesed_rootpath, video_data_df.loc[i, 'processed_path']).as_posix()\n", @@ -1253,7 +1276,7 @@ " print(this_video_path)\n", " print(this_results_path)\n", " deeplabcut.analyze_videos(configPath, this_video_path, destfolder=this_results_path)\n", - "'''" + "\"\"\"" ] }, { @@ -1273,7 +1296,7 @@ } ], "source": [ - "'''\n", + "\"\"\"\n", "import u19_pipeline.utils.dlc_process as dlc\n", "\n", "\n", @@ -1284,7 +1307,7 @@ " dlc.getPupilDiameter(destinationFolder=this_results_path)\n", " except BaseException as e:\n", " print(str(e))\n", - "'''" + "\"\"\"" ] }, { @@ -1304,7 +1327,7 @@ } ], "source": [ - "'''\n", + "\"\"\"\n", "import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n", "import u19_pipeline.automatic_job.slurm_creator as sc\n", "import subprocess\n", @@ -1373,7 +1396,7 @@ "\n", "\n", "\n", - "'''" + "\"\"\"" ] }, { @@ -1386,12 +1409,12 @@ "\n", "\n", "def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars):\n", - " '''\n", + " \"\"\"\n", " Create scp command from cluster directories and local slurm file\n", - " '''\n", + " \"\"\"\n", "\n", - " user_host = cluster_vars['user']+'@'+cluster_vars['hostname']\n", - " slurm_destination = user_host+':'+slurm_destination\n", + " user_host = cluster_vars[\"user\"] + \"@\" + cluster_vars[\"hostname\"]\n", + " slurm_destination = user_host + \":\" + slurm_destination\n", " status = ft.scp_file_transfer(slurm_file_local_path, slurm_destination)\n", "\n", " return status" @@ -1430,72 +1453,90 @@ "import u19_pipeline.automatic_job.slurm_creator as sc\n", "from u19_pipeline.utils.file_utils import write_file\n", "\n", - "repo_dir = '/scratch/gpfs/BRAINCOGS/U19-pipeline_python'\n", - "script_path = pathlib.Path(repo_dir, 'u19_pipeline/utils/dlc_process.py').as_posix()\n", + "repo_dir = \"/scratch/gpfs/BRAINCOGS/U19-pipeline_python\"\n", + "script_path = pathlib.Path(repo_dir, \"u19_pipeline/utils/dlc_process.py\").as_posix()\n", "\n", - "local_repo_dir = '/Users/alvaros/Documents/MATLAB/BrainCogsProjects/Datajoint_proj/U19-pipeline_python/'\n", + "local_repo_dir = \"/Users/alvaros/Documents/MATLAB/BrainCogsProjects/Datajoint_proj/U19-pipeline_python/\"\n", "\n", - "slurms_filepath = pathlib.Path(local_repo_dir, 'u19_pipeline/automatic_job/SlurmFiles').as_posix()\n", - "default_slurm_filename = 'slurm_real.slurm'\n", + "slurms_filepath = pathlib.Path(\n", + " local_repo_dir, \"u19_pipeline/automatic_job/SlurmFiles\"\n", + ").as_posix()\n", + "default_slurm_filename = \"slurm_real.slurm\"\n", "\n", "for i in range(9, 10):\n", - " this_video_path = pathlib.Path(root_videopath, video_data_df.loc[i, 'remote_path_video_file']).as_posix()\n", - " this_results_path = pathlib.Path(proccesed_rootpath, video_data_df.loc[i, 'processed_path']).as_posix()\n", + " this_video_path = pathlib.Path(\n", + " root_videopath, video_data_df.loc[i, \"remote_path_video_file\"]\n", + " ).as_posix()\n", + " this_results_path = pathlib.Path(\n", + " proccesed_rootpath, video_data_df.loc[i, \"processed_path\"]\n", + " ).as_posix()\n", "\n", " recording_process_id = str(i)\n", "\n", - " #Get all associated directories given the selected processing cluster\n", - " cluster_vars = ft.get_cluster_vars('tiger')\n", + " # Get all associated directories given the selected processing cluster\n", + " cluster_vars = ft.get_cluster_vars(\"tiger\")\n", "\n", " # Start with default values\n", - " slurm_dict = cluster_vars['slurm_default'].copy()\n", - " label_rec_process = 'rec'+recording_process_id\n", - " print('label_rec_process', label_rec_process)\n", - " slurm_dict['job-name'] = label_rec_process\n", - " slurm_dict['output'] = str(pathlib.Path(cluster_vars['log_files_dir'],label_rec_process+ '.log'))\n", + " slurm_dict = cluster_vars[\"slurm_default\"].copy()\n", + " label_rec_process = \"rec\" + recording_process_id\n", + " print(\"label_rec_process\", label_rec_process)\n", + " slurm_dict[\"job-name\"] = label_rec_process\n", + " slurm_dict[\"output\"] = str(\n", + " pathlib.Path(cluster_vars[\"log_files_dir\"], label_rec_process + \".log\")\n", + " )\n", " slurm_text = sc.generate_slurm_dlc2(slurm_dict)\n", "\n", " slurm_file_name = default_slurm_filename\n", - " slurm_file_local_path = pathlib.Path(slurms_filepath,slurm_file_name).as_posix()\n", + " slurm_file_local_path = pathlib.Path(slurms_filepath, slurm_file_name).as_posix()\n", "\n", " write_file(slurm_file_local_path, slurm_text)\n", "\n", - " slurm_destination = pathlib.Path(cluster_vars['slurm_files_dir'], slurm_file_name).as_posix()\n", + " slurm_destination = pathlib.Path(\n", + " cluster_vars[\"slurm_files_dir\"], slurm_file_name\n", + " ).as_posix()\n", "\n", " print(slurm_file_local_path)\n", " print(slurm_destination)\n", " print(cluster_vars)\n", "\n", " status = transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars)\n", - " \n", + "\n", " processed_data_directory = pathlib.Path(this_results_path).as_posix()\n", " repository_dir = repo_dir\n", "\n", - " command = ['ssh', cluster_vars['user']+\"@\"+cluster_vars['hostname'], 'sbatch', \n", - " \"--export=recording_process_id=\"+recording_process_id+\n", - " \",raw_data_directory=\"+this_video_path+\n", - " \",processed_data_directory=\"+processed_data_directory+\n", - " \",repository_dir='\"+repository_dir+\n", - " \"',process_script_path=\"+script_path+\n", - " \",model_path=\"+modelPath\n", - " , slurm_destination\n", + " command = [\n", + " \"ssh\",\n", + " cluster_vars[\"user\"] + \"@\" + cluster_vars[\"hostname\"],\n", + " \"sbatch\",\n", + " \"--export=recording_process_id=\"\n", + " + recording_process_id\n", + " + \",raw_data_directory=\"\n", + " + this_video_path\n", + " + \",processed_data_directory=\"\n", + " + processed_data_directory\n", + " + \",repository_dir='\"\n", + " + repository_dir\n", + " + \"',process_script_path=\"\n", + " + script_path\n", + " + \",model_path=\"\n", + " + modelPath,\n", + " slurm_destination,\n", " ]\n", "\n", " print(command)\n", " p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n", - " #p = os.popen(command_new).read()\n", + " # p = os.popen(command_new).read()\n", " p.wait()\n", " stdout, stderr = p.communicate()\n", - " print('stdout', stdout.decode('UTF-8'))\n", - " print('stderr', stderr.decode('UTF-8'))\n", - " print('p.returncode', p.returncode)\n", - "\n", - " if p.returncode == config.system_process['SUCCESS']:\n", - " batch_job_sentence = stdout.decode('UTF-8')\n", - " print('batch_job_sentence', batch_job_sentence)\n", - " id_slurm_job = batch_job_sentence.replace(\"Submitted batch job \",\"\")\n", - " id_slurm_job = re.sub(r\"[\\n\\t\\s]*\", \"\", id_slurm_job)\n", - "\n" + " print(\"stdout\", stdout.decode(\"UTF-8\"))\n", + " print(\"stderr\", stderr.decode(\"UTF-8\"))\n", + " print(\"p.returncode\", p.returncode)\n", + "\n", + " if p.returncode == config.system_process[\"SUCCESS\"]:\n", + " batch_job_sentence = stdout.decode(\"UTF-8\")\n", + " print(\"batch_job_sentence\", batch_job_sentence)\n", + " id_slurm_job = batch_job_sentence.replace(\"Submitted batch job \", \"\")\n", + " id_slurm_job = re.sub(r\"[\\n\\t\\s]*\", \"\", id_slurm_job)" ] }, { diff --git a/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb b/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb index 01951c0f..1e8ef8c0 100644 --- a/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb +++ b/notebooks/pupillometry_notebooks/pupillometry_final_notebook.ipynb @@ -82,8 +82,8 @@ ], "source": [ "dj.conn()\n", - "pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')\n", - "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')" + "pupillometry_db = dj.create_virtual_module(\"pupillometry\", \"u19_pupillometry\")\n", + "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")" ] }, { @@ -106,11 +106,15 @@ ], "source": [ "key = dict()\n", - "key['subject_fullname'] = 'efonseca_ef111_act111'\n", - "key['session_date'] = '2023-04-25'\n", + "key[\"subject_fullname\"] = \"efonseca_ef111_act111\"\n", + "key[\"session_date\"] = \"2023-04-25\"\n", "\n", - "key_pupil = (acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key).fetch(as_dict=True)[0]\n", - "model_key = (pupillometry_db.PupillometryModels & \"model_id = \"+str(key_pupil['model_id'])).fetch(as_dict=True)[0]\n", + "key_pupil = (\n", + " acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key\n", + ").fetch(as_dict=True)[0]\n", + "model_key = (\n", + " pupillometry_db.PupillometryModels & \"model_id = \" + str(key_pupil[\"model_id\"])\n", + ").fetch(as_dict=True)[0]\n", "\n", "model_key" ] @@ -158,23 +162,26 @@ } ], "source": [ - "models_dir = dj.config.get('custom', {}).get('root_data_dir',None)\n", + "models_dir = dj.config.get(\"custom\", {}).get(\"root_data_dir\", None)\n", "if models_dir is None:\n", - " raise Exception('root_data_dir in not found in config, , run initial_conf.py again')\n", + " raise Exception(\"root_data_dir in not found in config, , run initial_conf.py again\")\n", "models_dir = models_dir[0]\n", - "configPath = pathlib.Path(models_dir, model_key['model_path'],'config.yaml')\n", + "configPath = pathlib.Path(models_dir, model_key[\"model_path\"], \"config.yaml\")\n", "\n", - "pupillometry_dir = dj.config.get('custom', {}).get('pupillometry_root_data_dir',None)\n", + "pupillometry_dir = dj.config.get(\"custom\", {}).get(\"pupillometry_root_data_dir\", None)\n", "\n", "if pupillometry_dir is None:\n", - " raise Exception('pupillometry_root_data_dir not found in config, run initial_conf.py again')\n", + " raise Exception(\n", + " \"pupillometry_root_data_dir not found in config, run initial_conf.py again\"\n", + " )\n", "pupillometry_raw_dir = pupillometry_dir[0]\n", "pupillometry_processed_dir = pupillometry_dir[1]\n", "\n", - "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil['remote_path_video_file'])\n", - "output_dir = pathlib.Path(pupillometry_processed_dir,pathlib.Path(key_pupil['remote_path_video_file']).parent)\n", - "output_dir\n", - "\n" + "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil[\"remote_path_video_file\"])\n", + "output_dir = pathlib.Path(\n", + " pupillometry_processed_dir, pathlib.Path(key_pupil[\"remote_path_video_file\"]).parent\n", + ")\n", + "output_dir" ] }, { @@ -266,7 +273,7 @@ } ], "source": [ - "dj.config.get('custom', {})" + "dj.config.get(\"custom\", {})" ] }, { @@ -321,7 +328,9 @@ } ], "source": [ - "deeplabcut.analyze_videos(configPath.as_posix(), videoPath.as_posix(), destfolder=output_dir.as_posix())" + "deeplabcut.analyze_videos(\n", + " configPath.as_posix(), videoPath.as_posix(), destfolder=output_dir.as_posix()\n", + ")" ] }, { @@ -331,8 +340,8 @@ "outputs": [], "source": [ "dj.conn()\n", - "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "#pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')" + "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "# pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')" ] }, { @@ -455,7 +464,7 @@ } ], "source": [ - "dj.ERD(pupillometry_db) -1" + "dj.ERD(pupillometry_db) - 1" ] }, { @@ -475,13 +484,15 @@ } ], "source": [ - "root_videopath = dj.config['custom']['dlc_root_data_dir2']\n", + "root_videopath = dj.config[\"custom\"][\"dlc_root_data_dir2\"]\n", "\n", - "modelPath = pathlib.Path(root_videopath, 'MODELS', 'Pupillometry2-Ryan-2022-04-07').as_posix()\n", - "configPath = pathlib.Path(modelPath,'config.yaml').as_posix()\n", + "modelPath = pathlib.Path(\n", + " root_videopath, \"MODELS\", \"Pupillometry2-Ryan-2022-04-07\"\n", + ").as_posix()\n", + "configPath = pathlib.Path(modelPath, \"config.yaml\").as_posix()\n", "\n", - "proccesed_rootpath = dj.config['custom']['dlc_processed_data_dir2']\n", - "proccesed_rootpath = pathlib.Path(proccesed_rootpath, 'pupillometry').as_posix()\n", + "proccesed_rootpath = dj.config[\"custom\"][\"dlc_processed_data_dir2\"]\n", + "proccesed_rootpath = pathlib.Path(proccesed_rootpath, \"pupillometry\").as_posix()\n", "configPath" ] }, @@ -719,12 +730,20 @@ ], "source": [ "key = 'subject_fullname like \"jounhong%\"'\n", - "video_data_df = pd.DataFrame((pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\"KEY\", 'remote_path_video_file', as_dict=True))\n", + "video_data_df = pd.DataFrame(\n", + " (pupillometry_db.PupillometrySession * acquisition_db.SessionVideo & key).fetch(\n", + " \"KEY\", \"remote_path_video_file\", as_dict=True\n", + " )\n", + ")\n", "\n", - "video_data_df['remote_path_video_file'] = 'pupillometry/' + video_data_df['remote_path_video_file']\n", - "video_data_df.loc[0,'remote_path_video_file']\n", + "video_data_df[\"remote_path_video_file\"] = (\n", + " \"pupillometry/\" + video_data_df[\"remote_path_video_file\"]\n", + ")\n", + "video_data_df.loc[0, \"remote_path_video_file\"]\n", "\n", - "video_data_df['processed_path'] = video_data_df['remote_path_video_file'].apply(lambda x : pathlib.Path(x).parent.as_posix())\n", + "video_data_df[\"processed_path\"] = video_data_df[\"remote_path_video_file\"].apply(\n", + " lambda x: pathlib.Path(x).parent.as_posix()\n", + ")\n", "\n", "video_data_df" ] @@ -746,7 +765,7 @@ } ], "source": [ - "'''\n", + "\"\"\"\n", "for i in range(video_data_df.shape[0]-1,0,-1):\n", " this_video_path = pathlib.Path(root_videopath, video_data_df.loc[i, 'remote_path_video_file']).as_posix()\n", " this_results_path = pathlib.Path(proccesed_rootpath, video_data_df.loc[i, 'processed_path']).as_posix()\n", @@ -756,7 +775,7 @@ " print(this_video_path)\n", " print(this_results_path)\n", " deeplabcut.analyze_videos(configPath, this_video_path, destfolder=this_results_path)\n", - "'''" + "\"\"\"" ] }, { @@ -776,7 +795,7 @@ } ], "source": [ - "'''\n", + "\"\"\"\n", "import u19_pipeline.utils.dlc_process as dlc\n", "\n", "\n", @@ -787,7 +806,7 @@ " dlc.getPupilDiameter(destinationFolder=this_results_path)\n", " except BaseException as e:\n", " print(str(e))\n", - "'''" + "\"\"\"" ] }, { @@ -807,7 +826,7 @@ } ], "source": [ - "'''\n", + "\"\"\"\n", "import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft\n", "import u19_pipeline.automatic_job.slurm_creator as sc\n", "import subprocess\n", @@ -876,7 +895,7 @@ "\n", "\n", "\n", - "'''" + "\"\"\"" ] }, { @@ -889,12 +908,12 @@ "\n", "\n", "def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars):\n", - " '''\n", + " \"\"\"\n", " Create scp command from cluster directories and local slurm file\n", - " '''\n", + " \"\"\"\n", "\n", - " user_host = cluster_vars['user']+'@'+cluster_vars['hostname']\n", - " slurm_destination = user_host+':'+slurm_destination\n", + " user_host = cluster_vars[\"user\"] + \"@\" + cluster_vars[\"hostname\"]\n", + " slurm_destination = user_host + \":\" + slurm_destination\n", " status = ft.scp_file_transfer(slurm_file_local_path, slurm_destination)\n", "\n", " return status" @@ -933,72 +952,90 @@ "import u19_pipeline.automatic_job.slurm_creator as sc\n", "from u19_pipeline.utils.file_utils import write_file\n", "\n", - "repo_dir = '/scratch/gpfs/BRAINCOGS/U19-pipeline_python'\n", - "script_path = pathlib.Path(repo_dir, 'u19_pipeline/utils/dlc_process.py').as_posix()\n", + "repo_dir = \"/scratch/gpfs/BRAINCOGS/U19-pipeline_python\"\n", + "script_path = pathlib.Path(repo_dir, \"u19_pipeline/utils/dlc_process.py\").as_posix()\n", "\n", - "local_repo_dir = '/Users/alvaros/Documents/MATLAB/BrainCogsProjects/Datajoint_proj/U19-pipeline_python/'\n", + "local_repo_dir = \"/Users/alvaros/Documents/MATLAB/BrainCogsProjects/Datajoint_proj/U19-pipeline_python/\"\n", "\n", - "slurms_filepath = pathlib.Path(local_repo_dir, 'u19_pipeline/automatic_job/SlurmFiles').as_posix()\n", - "default_slurm_filename = 'slurm_real.slurm'\n", + "slurms_filepath = pathlib.Path(\n", + " local_repo_dir, \"u19_pipeline/automatic_job/SlurmFiles\"\n", + ").as_posix()\n", + "default_slurm_filename = \"slurm_real.slurm\"\n", "\n", "for i in range(9, 10):\n", - " this_video_path = pathlib.Path(root_videopath, video_data_df.loc[i, 'remote_path_video_file']).as_posix()\n", - " this_results_path = pathlib.Path(proccesed_rootpath, video_data_df.loc[i, 'processed_path']).as_posix()\n", + " this_video_path = pathlib.Path(\n", + " root_videopath, video_data_df.loc[i, \"remote_path_video_file\"]\n", + " ).as_posix()\n", + " this_results_path = pathlib.Path(\n", + " proccesed_rootpath, video_data_df.loc[i, \"processed_path\"]\n", + " ).as_posix()\n", "\n", " recording_process_id = str(i)\n", "\n", - " #Get all associated directories given the selected processing cluster\n", - " cluster_vars = ft.get_cluster_vars('tiger')\n", + " # Get all associated directories given the selected processing cluster\n", + " cluster_vars = ft.get_cluster_vars(\"tiger\")\n", "\n", " # Start with default values\n", - " slurm_dict = cluster_vars['slurm_default'].copy()\n", - " label_rec_process = 'rec'+recording_process_id\n", - " print('label_rec_process', label_rec_process)\n", - " slurm_dict['job-name'] = label_rec_process\n", - " slurm_dict['output'] = str(pathlib.Path(cluster_vars['log_files_dir'],label_rec_process+ '.log'))\n", + " slurm_dict = cluster_vars[\"slurm_default\"].copy()\n", + " label_rec_process = \"rec\" + recording_process_id\n", + " print(\"label_rec_process\", label_rec_process)\n", + " slurm_dict[\"job-name\"] = label_rec_process\n", + " slurm_dict[\"output\"] = str(\n", + " pathlib.Path(cluster_vars[\"log_files_dir\"], label_rec_process + \".log\")\n", + " )\n", " slurm_text = sc.generate_slurm_dlc2(slurm_dict)\n", "\n", " slurm_file_name = default_slurm_filename\n", - " slurm_file_local_path = pathlib.Path(slurms_filepath,slurm_file_name).as_posix()\n", + " slurm_file_local_path = pathlib.Path(slurms_filepath, slurm_file_name).as_posix()\n", "\n", " write_file(slurm_file_local_path, slurm_text)\n", "\n", - " slurm_destination = pathlib.Path(cluster_vars['slurm_files_dir'], slurm_file_name).as_posix()\n", + " slurm_destination = pathlib.Path(\n", + " cluster_vars[\"slurm_files_dir\"], slurm_file_name\n", + " ).as_posix()\n", "\n", " print(slurm_file_local_path)\n", " print(slurm_destination)\n", " print(cluster_vars)\n", "\n", " status = transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars)\n", - " \n", + "\n", " processed_data_directory = pathlib.Path(this_results_path).as_posix()\n", " repository_dir = repo_dir\n", "\n", - " command = ['ssh', cluster_vars['user']+\"@\"+cluster_vars['hostname'], 'sbatch', \n", - " \"--export=recording_process_id=\"+recording_process_id+\n", - " \",raw_data_directory=\"+this_video_path+\n", - " \",processed_data_directory=\"+processed_data_directory+\n", - " \",repository_dir='\"+repository_dir+\n", - " \"',process_script_path=\"+script_path+\n", - " \",model_path=\"+modelPath\n", - " , slurm_destination\n", + " command = [\n", + " \"ssh\",\n", + " cluster_vars[\"user\"] + \"@\" + cluster_vars[\"hostname\"],\n", + " \"sbatch\",\n", + " \"--export=recording_process_id=\"\n", + " + recording_process_id\n", + " + \",raw_data_directory=\"\n", + " + this_video_path\n", + " + \",processed_data_directory=\"\n", + " + processed_data_directory\n", + " + \",repository_dir='\"\n", + " + repository_dir\n", + " + \"',process_script_path=\"\n", + " + script_path\n", + " + \",model_path=\"\n", + " + modelPath,\n", + " slurm_destination,\n", " ]\n", "\n", " print(command)\n", " p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n", - " #p = os.popen(command_new).read()\n", + " # p = os.popen(command_new).read()\n", " p.wait()\n", " stdout, stderr = p.communicate()\n", - " print('stdout', stdout.decode('UTF-8'))\n", - " print('stderr', stderr.decode('UTF-8'))\n", - " print('p.returncode', p.returncode)\n", + " print(\"stdout\", stdout.decode(\"UTF-8\"))\n", + " print(\"stderr\", stderr.decode(\"UTF-8\"))\n", + " print(\"p.returncode\", p.returncode)\n", "\n", - " if p.returncode == config.system_process['SUCCESS']:\n", - " batch_job_sentence = stdout.decode('UTF-8')\n", - " print('batch_job_sentence', batch_job_sentence)\n", - " id_slurm_job = batch_job_sentence.replace(\"Submitted batch job \",\"\")\n", - " id_slurm_job = re.sub(r\"[\\n\\t\\s]*\", \"\", id_slurm_job)\n", - "\n" + " if p.returncode == config.system_process[\"SUCCESS\"]:\n", + " batch_job_sentence = stdout.decode(\"UTF-8\")\n", + " print(\"batch_job_sentence\", batch_job_sentence)\n", + " id_slurm_job = batch_job_sentence.replace(\"Submitted batch job \", \"\")\n", + " id_slurm_job = re.sub(r\"[\\n\\t\\s]*\", \"\", id_slurm_job)" ] }, { diff --git a/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb b/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb index ccc5a20e..e27eeb69 100644 --- a/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb +++ b/notebooks/pupillometry_notebooks/review_pupillometry_data.ipynb @@ -62,8 +62,8 @@ ], "source": [ "dj.conn()\n", - "pupillometry_db = dj.create_virtual_module('pupillometry', 'u19_pupillometry')\n", - "acquisition_db = dj.create_virtual_module('acquisition', 'u19_acquisition')" + "pupillometry_db = dj.create_virtual_module(\"pupillometry\", \"u19_pupillometry\")\n", + "acquisition_db = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")" ] }, { @@ -93,8 +93,8 @@ "frameNumber = 1000\n", "\n", "key = dict()\n", - "key['subject_fullname'] = 'efonseca_ef111_act111'\n", - "key['session_date'] = '2023-06-24'\n", + "key[\"subject_fullname\"] = \"efonseca_ef111_act111\"\n", + "key[\"session_date\"] = \"2023-06-24\"\n", "\n", "key" ] @@ -122,26 +122,34 @@ ], "source": [ "# Get video location\n", - "key_pupil = (acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key).fetch(as_dict=True)[0]\n", - "db_pupil_data = (pupillometry_db.PupillometrySessionModelData & key).fetch(as_dict=True)[0]\n", - "pupillometry_dir = dj.config.get('custom', {}).get('pupillometry_root_data_dir',None)\n", + "key_pupil = (\n", + " acquisition_db.SessionVideo * pupillometry_db.PupillometrySession & key\n", + ").fetch(as_dict=True)[0]\n", + "db_pupil_data = (pupillometry_db.PupillometrySessionModelData & key).fetch(\n", + " as_dict=True\n", + ")[0]\n", + "pupillometry_dir = dj.config.get(\"custom\", {}).get(\"pupillometry_root_data_dir\", None)\n", "if pupillometry_dir is None:\n", - " raise Exception('pupillometry_root_data_dir not found in config, run initial_conf.py again')\n", + " raise Exception(\n", + " \"pupillometry_root_data_dir not found in config, run initial_conf.py again\"\n", + " )\n", "pupillometry_raw_dir = pupillometry_dir[0]\n", - "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil['remote_path_video_file'])\n", - "print('videoPath', videoPath)\n", + "videoPath = pathlib.Path(pupillometry_raw_dir, key_pupil[\"remote_path_video_file\"])\n", + "print(\"videoPath\", videoPath)\n", "\n", "# Get output location\n", "pupillometry_processed_dir = pupillometry_dir[1]\n", - "output_dir = pathlib.Path(pupillometry_processed_dir,pathlib.Path(key_pupil['remote_path_video_file']).parent)\n", + "output_dir = pathlib.Path(\n", + " pupillometry_processed_dir, pathlib.Path(key_pupil[\"remote_path_video_file\"]).parent\n", + ")\n", "\n", - "#Find h5 files (output from deeplabcut)\n", - "h5_files = glob.glob(str(output_dir) + '/*.h5')\n", + "# Find h5 files (output from deeplabcut)\n", + "h5_files = glob.glob(str(output_dir) + \"/*.h5\")\n", "if len(h5_files) != 1:\n", - " raise Exception('Didn''t find any h5 files after deeplabcut analyze_video')\n", + " raise Exception(\"Didnt find any h5 files after deeplabcut analyze_video\")\n", "else:\n", " h5_files = h5_files[0]\n", - "print('h5_files', h5_files)" + "print(\"h5_files\", h5_files)" ] }, { @@ -172,13 +180,11 @@ "\n", "cap = cv2.VideoCapture(str(videoPath))\n", "\n", - "# count the number of frames \n", - "frames = cap.get(cv2.CAP_PROP_FRAME_COUNT) \n", + "# count the number of frames\n", + "frames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n", "\n", - "print('total # of frames', int(frames))\n", - "print('Length output deeplabcut', labels.shape[0])\n", - "\n", - "\n" + "print(\"total # of frames\", int(frames))\n", + "print(\"Length output deeplabcut\", labels.shape[0])" ] }, { @@ -192,10 +198,9 @@ "\n", "totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n", "# Set frame position\n", - "cap.set(cv2.CAP_PROP_POS_FRAMES,frameNumber)\n", + "cap.set(cv2.CAP_PROP_POS_FRAMES, frameNumber)\n", "\n", - "ret, frame = cap.read()\n", - "\n" + "ret, frame = cap.read()" ] }, { @@ -217,14 +222,12 @@ } ], "source": [ - "\n", - "\n", "# Take a subset from the data\n", "subset = labels.loc[frameNumber]\n", "# Get the pupil markers (0 to 8) from the multiindex array obtained from deeplabcut\n", - "x = subset.xs('x', level='coords').to_numpy()[0:8]\n", - "y = subset.xs('y', level='coords').to_numpy()[0:8]\n", - "xy = np.column_stack((x,y))\n", + "x = subset.xs(\"x\", level=\"coords\").to_numpy()[0:8]\n", + "y = subset.xs(\"y\", level=\"coords\").to_numpy()[0:8]\n", + "xy = np.column_stack((x, y))\n", "\n", "# Estimate an ellipse based on the points generated around the pupil, this is enough to get an estimate of the pupil area in pixels\n", "ellipse = EllipseModel()\n", @@ -232,11 +235,17 @@ "\n", "# Validate by ploting the points and the fitting ellipse on the image from the video\n", "fig, ax = plt.subplots()\n", - "plt.rcParams['figure.dpi'] = 250\n", + "plt.rcParams[\"figure.dpi\"] = 250\n", "img = frame\n", - "ax.scatter(xy[:,0], xy[:,1], s=0.1, color='orange')\n", + "ax.scatter(xy[:, 0], xy[:, 1], s=0.1, color=\"orange\")\n", "# Draw the ellipse on the original image\n", - "cy, cx = ellipse_perimeter(int(ellipse.params[1]), int(ellipse.params[0]), int(ellipse.params[3]), int(ellipse.params[2]), ellipse.params[4])\n", + "cy, cx = ellipse_perimeter(\n", + " int(ellipse.params[1]),\n", + " int(ellipse.params[0]),\n", + " int(ellipse.params[3]),\n", + " int(ellipse.params[2]),\n", + " ellipse.params[4],\n", + ")\n", "img[cy, cx] = 255\n", "ax.imshow(img)\n", "\n", @@ -250,7 +259,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(db_pupil_data['pupil_diameter'], columns=['PupilDiameter'])" + "df = pd.DataFrame(db_pupil_data[\"pupil_diameter\"], columns=[\"PupilDiameter\"])" ] }, { @@ -272,16 +281,15 @@ } ], "source": [ - "\n", "# Get a boolean array where true correspond to the frame with an outlier diameter\n", - "zscore = np.abs(stats.zscore(df, nan_policy='omit'))\n", + "zscore = np.abs(stats.zscore(df, nan_policy=\"omit\"))\n", "outlierFlags = np.abs(zscore) > 2\n", "outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: \"OutlierFlag\"})\n", "\n", "# Get the list of outlier frames\n", "from itertools import compress\n", "\n", - "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags['OutlierFlag']))\n", + "outlierFrames = list(compress(range(len(outlierFlags)), outlierFlags[\"OutlierFlag\"]))\n", "\n", "\n", "# Retrieve an outlier frames sample from the video for visual inspection\n", @@ -289,20 +297,20 @@ "fig = plt.figure(figsize=(7, 7))\n", "columns = 5\n", "rows = 5\n", - "for i in range(1, columns*rows +1):\n", + "for i in range(1, columns * rows + 1):\n", " # Get total number of frames\n", " totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n", " # Set frame position\n", - " cap.set(cv2.CAP_PROP_POS_FRAMES,outlierFrames[i])\n", + " cap.set(cv2.CAP_PROP_POS_FRAMES, outlierFrames[i])\n", " ret, frame = cap.read()\n", "\n", " fig.add_subplot(rows, columns, i)\n", " plt.title(\"Frame \" + str(outlierFrames[i]), fontsize=5)\n", " plt.imshow(frame)\n", - " plt.axis('off')\n", + " plt.axis(\"off\")\n", "plt.show()\n", "cap.release()\n", - "cv2.destroyAllWindows()\n" + "cv2.destroyAllWindows()" ] }, { @@ -356,29 +364,33 @@ "source": [ "# Concatenate outlier flags array to remove outliers from pupil diameter array\n", "temp = pd.concat([df, outlierFlags], axis=1)\n", - "temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None\n", - "pupilDiameter = temp['PupilDiameter']\n", + "temp.loc[temp[\"OutlierFlag\"] == True, \"PupilDiameter\"] = None\n", + "pupilDiameter = temp[\"PupilDiameter\"]\n", "\n", "# This might be better to leave open for the experimenter.\n", "# Replace the outlier diameter data using a cubic spline interpolator\n", "# pupilDiameter = temp['PupilDiameter'].interpolate(method='spline', order=3, s=0)\n", "\n", - "fig, (ax1, ax2) = plt.subplots(1,2)\n", + "fig, (ax1, ax2) = plt.subplots(1, 2)\n", "# plt.rcParams['figure.dpi'] = 250\n", "\n", "ax1.plot(pupilDiameter[500:1500], linewidth=0.25)\n", "# Filter by using a moving average window of 10 samples\n", - "ax1.plot(pupilDiameter[500:1500].rolling(10).mean(), linewidth=0.5, color='red')\n", - "ax1.set_xlabel('Frame')\n", - "ax1.set_ylabel('Pupil diameter [px]')\n", - "ax1.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n", + "ax1.plot(pupilDiameter[500:1500].rolling(10).mean(), linewidth=0.5, color=\"red\")\n", + "ax1.set_xlabel(\"Frame\")\n", + "ax1.set_ylabel(\"Pupil diameter [px]\")\n", + "ax1.legend(\n", + " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n", + ")\n", "\n", "ax2.plot(pupilDiameter[0:20000], linewidth=0.25)\n", "# Filter by using a moving average window of 10 samples\n", - "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color='red')\n", - "ax2.set_xlabel('Frame')\n", - "ax2.legend(['Original without outliers', 'Moving average window'], loc=4, prop={'size': 6})\n", - "#fig.savefig('test.eps', format='eps')" + "ax2.plot(pupilDiameter[0:20000].rolling(10).mean(), linewidth=0.5, color=\"red\")\n", + "ax2.set_xlabel(\"Frame\")\n", + "ax2.legend(\n", + " [\"Original without outliers\", \"Moving average window\"], loc=4, prop={\"size\": 6}\n", + ")\n", + "# fig.savefig('test.eps', format='eps')" ] }, { diff --git a/notebooks/read_virmen_behavior_file.ipynb b/notebooks/read_virmen_behavior_file.ipynb index 681e054f..f4075108 100644 --- a/notebooks/read_virmen_behavior_file.ipynb +++ b/notebooks/read_virmen_behavior_file.ipynb @@ -40,8 +40,8 @@ "\n", "import u19_pipeline.utils.matlab_utils as mu\n", "\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "from u19_pipeline import lab\n" + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "from u19_pipeline import lab" ] }, { @@ -57,9 +57,9 @@ "metadata": {}, "outputs": [], "source": [ - "local_matlab_path = '/Applications/MATLAB_R2020a.app/bin/'\n", - "if which('matlab') is None:\n", - " os.environ['PATH'] += ':'+local_matlab_path" + "local_matlab_path = \"/Applications/MATLAB_R2020a.app/bin/\"\n", + "if which(\"matlab\") is None:\n", + " os.environ[\"PATH\"] += \":\" + local_matlab_path" ] }, { @@ -76,10 +76,12 @@ } ], "source": [ - "if which('matlab') is None:\n", - " print('Matlab command not found, add correct path installation or proceed (Choice & TrialType data will not be read from the behavior file')\n", + "if which(\"matlab\") is None:\n", + " print(\n", + " \"Matlab command not found, add correct path installation or proceed (Choice & TrialType data will not be read from the behavior file\"\n", + " )\n", "else:\n", - " print('Matlab installation found !!')" + " print(\"Matlab installation found !!\")" ] }, { @@ -101,10 +103,12 @@ } ], "source": [ - "subject = 'efonseca_ef884_actp003'\n", - "date = '2021-12-03'\n", + "subject = \"efonseca_ef884_actp003\"\n", + "date = \"2021-12-03\"\n", "\n", - "path = (acquisition.SessionStarted & dict(session_date =date, subject_fullname=subject)).fetch1('new_remote_path_behavior_file')\n", + "path = (\n", + " acquisition.SessionStarted & dict(session_date=date, subject_fullname=subject)\n", + ").fetch1(\"new_remote_path_behavior_file\")\n", "\n", "\n", "mat_file = lab.Path().get_local_path2(path)\n", @@ -138,7 +142,7 @@ } ], "source": [ - "matin['log']['block']['trial'][12]['trialType']" + "matin[\"log\"][\"block\"][\"trial\"][12][\"trialType\"]" ] }, { @@ -654,7 +658,7 @@ } ], "source": [ - "df_behavior_file.loc[6, 'trial_type']" + "df_behavior_file.loc[6, \"trial_type\"]" ] }, { diff --git a/notebooks/storage_calculator_across_time.ipynb b/notebooks/storage_calculator_across_time.ipynb index f62d5e09..e3ecbc51 100644 --- a/notebooks/storage_calculator_across_time.ipynb +++ b/notebooks/storage_calculator_across_time.ipynb @@ -32,7 +32,7 @@ "\n", "import u19_pipeline.utils.path_utils as pu\n", "\n", - "fig_size = (15,11)" + "fig_size = (15, 11)" ] }, { @@ -64,28 +64,38 @@ } ], "source": [ - "folder_list_to_read_from = ['/mnt/cup/PNI-centers/Bezos/Ryan/', '/mnt/cup/PNI-centers/Bezos/Scott/twoP', '/mnt/cup/labs/witten/Scott/twoPhoton', '/mnt/cup/braininit/RigData/mesoscope/imaging', '/mnt/cup/PNI-centers/Bezos/RigData/scope/bay3/sakoay', '/mnt/cup/PNI-centers/Bezos/RigData/scope/bay3/edward', '/mnt/cup/braininit/RigData/training/npx/electrophysiology']\n", + "folder_list_to_read_from = [\n", + " \"/mnt/cup/PNI-centers/Bezos/Ryan/\",\n", + " \"/mnt/cup/PNI-centers/Bezos/Scott/twoP\",\n", + " \"/mnt/cup/labs/witten/Scott/twoPhoton\",\n", + " \"/mnt/cup/braininit/RigData/mesoscope/imaging\",\n", + " \"/mnt/cup/PNI-centers/Bezos/RigData/scope/bay3/sakoay\",\n", + " \"/mnt/cup/PNI-centers/Bezos/RigData/scope/bay3/edward\",\n", + " \"/mnt/cup/braininit/RigData/training/npx/electrophysiology\",\n", + "]\n", "\n", "for idx, folder in enumerate(folder_list_to_read_from):\n", " print(idx, folder)\n", " if idx == 0:\n", " df_storage = pd.DataFrame(pu.get_size_directory_time(folder))\n", - " df_storage['parent_folder'] = folder\n", + " df_storage[\"parent_folder\"] = folder\n", " else:\n", " df_storage_current = pd.DataFrame(pu.get_size_directory_time(folder))\n", - " df_storage_current['parent_folder'] = folder\n", + " df_storage_current[\"parent_folder\"] = folder\n", " df_storage = df_storage.append(df_storage_current)\n", "\n", "df_storage.reset_index(drop=True)\n", - "df_storage['size'] = df_storage['size'].astype(int)\n", - "df_storage['size'] = df_storage['size'].astype(int)\n", - "df_storage['size (TB)'] = df_storage['size']/(1024*1024*1024)\n", - "df_storage['date'] = pd.to_datetime(df_storage['date'])\n", - "df_storage['date'] = df_storage['date'].dt.normalize()\n", - "df_storage = df_storage.drop(['size', 'directory'], axis=1)\n", - "df_storage = df_storage.sort_values(by=['date'])\n", - "df_storage = df_storage.loc[df_storage['date'] > pd.to_datetime(\"'2018-01-01'\".replace(\"'\",\"\")), :]\n", - "copy_df_storage = df_storage.copy()\n" + "df_storage[\"size\"] = df_storage[\"size\"].astype(int)\n", + "df_storage[\"size\"] = df_storage[\"size\"].astype(int)\n", + "df_storage[\"size (TB)\"] = df_storage[\"size\"] / (1024 * 1024 * 1024)\n", + "df_storage[\"date\"] = pd.to_datetime(df_storage[\"date\"])\n", + "df_storage[\"date\"] = df_storage[\"date\"].dt.normalize()\n", + "df_storage = df_storage.drop([\"size\", \"directory\"], axis=1)\n", + "df_storage = df_storage.sort_values(by=[\"date\"])\n", + "df_storage = df_storage.loc[\n", + " df_storage[\"date\"] > pd.to_datetime(\"'2018-01-01'\".replace(\"'\", \"\")), :\n", + "]\n", + "copy_df_storage = df_storage.copy()" ] }, { @@ -101,11 +111,11 @@ "metadata": {}, "outputs": [], "source": [ - "df_storage_day = df_storage.groupby(by=['date']).sum()\n", + "df_storage_day = df_storage.groupby(by=[\"date\"]).sum()\n", "df_storage_day = df_storage_day.reset_index()\n", - "df_storage_day = df_storage.sort_values(by=['date'])\n", + "df_storage_day = df_storage.sort_values(by=[\"date\"])\n", "df_storage_day = df_storage_day.reset_index(drop=True)\n", - "df_storage_day['size (TB) sum'] = df_storage_day['size (TB)'].cumsum() \n" + "df_storage_day[\"size (TB) sum\"] = df_storage_day[\"size (TB)\"].cumsum()" ] }, { @@ -216,15 +226,18 @@ } ], "source": [ - "\n", - "df_storage_day_log = df_storage_day.loc[df_storage_day['size (TB) sum'] > 5, :]\n", + "df_storage_day_log = df_storage_day.loc[df_storage_day[\"size (TB) sum\"] > 5, :]\n", "df_storage_day_log.reset_index(drop=True)\n", - "df_storage_day_log['numeric_time'] = (df_storage_day_log['date'] - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta('1s')\n", + "df_storage_day_log[\"numeric_time\"] = (\n", + " df_storage_day_log[\"date\"] - pd.Timestamp(\"1970-01-01\")\n", + ") // pd.Timedelta(\"1s\")\n", "\n", - "df_storage_day_log = df_storage_day_log.drop_duplicates(subset=['numeric_time'])\n", + "df_storage_day_log = df_storage_day_log.drop_duplicates(subset=[\"numeric_time\"])\n", "\n", - "X = df_storage_day_log['numeric_time'].values\n", - "y = np.log(df_storage_day_log['size (TB) sum'].values) # Apply natural log function to the target\n", + "X = df_storage_day_log[\"numeric_time\"].values\n", + "y = np.log(\n", + " df_storage_day_log[\"size (TB) sum\"].values\n", + ") # Apply natural log function to the target\n", "\n", "X = X.reshape(-1, 1)\n", "y = y.reshape(-1, 1)\n", @@ -232,27 +245,31 @@ "model = LinearRegression()\n", "model.fit(X, y)\n", "X2 = X\n", - "ini_date = df_storage_day_log.loc[df_storage_day_log.index[-1],'date']\n", + "ini_date = df_storage_day_log.loc[df_storage_day_log.index[-1], \"date\"]\n", "for i in range(6):\n", " ini_date += pd.DateOffset(months=4)\n", - " num_time = (ini_date - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta('1s')\n", + " num_time = (ini_date - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta(\"1s\")\n", " X2 = np.append(X2, num_time)\n", "\n", "X2 = X2.reshape(-1, 1)\n", - "y_pred = np.exp(model.predict(X2)) # Apply exponential function (inverse of natural log) to the predictions\n", + "y_pred = np.exp(\n", + " model.predict(X2)\n", + ") # Apply exponential function (inverse of natural log) to the predictions\n", "\n", "\n", - "X2_date = pd.to_datetime(np.squeeze(X2),unit='s')\n", - "df_pred = pd.DataFrame(data=y_pred, index=X2_date, columns=['TB pred'])\n", - "df_pred.index.name = 'Date'\n", + "X2_date = pd.to_datetime(np.squeeze(X2), unit=\"s\")\n", + "df_pred = pd.DataFrame(data=y_pred, index=X2_date, columns=[\"TB pred\"])\n", + "df_pred.index.name = \"Date\"\n", "df_pred = df_pred.reset_index()\n", - "print('R^2', model.score(X, y))\n", - "print(df_storage_day_log.loc[df_storage_day_log.index[-1], 'size (TB) sum'])\n", - "current_TB = \"{:0.2f}\".format(df_storage_day_log.loc[df_storage_day_log.index[-1], 'size (TB) sum']) + \" TB\"\n", - "df_pred.tail(8)\n", - "\n", - "\n", - "\n" + "print(\"R^2\", model.score(X, y))\n", + "print(df_storage_day_log.loc[df_storage_day_log.index[-1], \"size (TB) sum\"])\n", + "current_TB = (\n", + " \"{:0.2f}\".format(\n", + " df_storage_day_log.loc[df_storage_day_log.index[-1], \"size (TB) sum\"]\n", + " )\n", + " + \" TB\"\n", + ")\n", + "df_pred.tail(8)" ] }, { @@ -284,15 +301,20 @@ "source": [ "fig, axs = plt.subplots(1, 1, figsize=fig_size)\n", "\n", - "plt.plot(df_storage_day_log['date'] , df_storage_day_log['size (TB) sum'], linewidth=3, color ='k')\n", - "plt.plot(X2_date, y_pred) \n", - "plt.yscale('log')\n", + "plt.plot(\n", + " df_storage_day_log[\"date\"],\n", + " df_storage_day_log[\"size (TB) sum\"],\n", + " linewidth=3,\n", + " color=\"k\",\n", + ")\n", + "plt.plot(X2_date, y_pred)\n", + "plt.yscale(\"log\")\n", "\n", "plt.xticks(fontsize=16)\n", "plt.yticks(fontsize=16)\n", "plt.xlabel(\"Date\", fontsize=18)\n", "plt.ylabel(\"Storage space (TB)\", fontsize=18)\n", - "plt.title('Storage on imaging & electrophysiology U19: ' + current_TB, fontsize=20);\n" + "plt.title(\"Storage on imaging & electrophysiology U19: \" + current_TB, fontsize=20);" ] }, { @@ -403,14 +425,20 @@ } ], "source": [ - "df_storage_day_lin = df_storage_day.loc[df_storage_day['date'] > pd.to_datetime('20210301', format='%Y%m%d'), :]\n", + "df_storage_day_lin = df_storage_day.loc[\n", + " df_storage_day[\"date\"] > pd.to_datetime(\"20210301\", format=\"%Y%m%d\"), :\n", + "]\n", "df_storage_day_lin.reset_index(drop=True)\n", - "df_storage_day_lin['numeric_time'] = (df_storage_day_lin['date'] - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta('1s')\n", + "df_storage_day_lin[\"numeric_time\"] = (\n", + " df_storage_day_lin[\"date\"] - pd.Timestamp(\"1970-01-01\")\n", + ") // pd.Timedelta(\"1s\")\n", "\n", - "df_storage_day_lin = df_storage_day_lin.drop_duplicates(subset=['numeric_time'])\n", + "df_storage_day_lin = df_storage_day_lin.drop_duplicates(subset=[\"numeric_time\"])\n", "\n", - "X = df_storage_day_lin['numeric_time'].values\n", - "y = (df_storage_day_lin['size (TB) sum'].values) # Apply natural log function to the target\n", + "X = df_storage_day_lin[\"numeric_time\"].values\n", + "y = df_storage_day_lin[\n", + " \"size (TB) sum\"\n", + "].values # Apply natural log function to the target\n", "\n", "X = X.reshape(-1, 1)\n", "y = y.reshape(-1, 1)\n", @@ -418,22 +446,24 @@ "model = LinearRegression()\n", "model.fit(X, y)\n", "X2 = X\n", - "ini_date = df_storage_day_log.loc[df_storage_day_log.index[-1],'date']\n", + "ini_date = df_storage_day_log.loc[df_storage_day_log.index[-1], \"date\"]\n", "for i in range(6):\n", " ini_date += pd.DateOffset(months=4)\n", - " num_time = (ini_date - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta('1s')\n", + " num_time = (ini_date - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta(\"1s\")\n", " X2 = np.append(X2, num_time)\n", "\n", "X2 = X2.reshape(-1, 1)\n", - "y_pred = (model.predict(X2)) # Apply exponential function (inverse of natural log) to the predictions\n", + "y_pred = model.predict(\n", + " X2\n", + ") # Apply exponential function (inverse of natural log) to the predictions\n", "\n", "\n", - "X2_date = pd.to_datetime(np.squeeze(X2),unit='s')\n", - "df_pred = pd.DataFrame(data=y_pred, index=X2_date, columns=['TB pred'])\n", - "df_pred.index.name = 'Date'\n", + "X2_date = pd.to_datetime(np.squeeze(X2), unit=\"s\")\n", + "df_pred = pd.DataFrame(data=y_pred, index=X2_date, columns=[\"TB pred\"])\n", + "df_pred.index.name = \"Date\"\n", "df_pred = df_pred.reset_index()\n", - "print('R^2', model.score(X, y))\n", - "print(df_storage_day_log.loc[df_storage_day_log.index[-1], 'size (TB) sum'])\n", + "print(\"R^2\", model.score(X, y))\n", + "print(df_storage_day_log.loc[df_storage_day_log.index[-1], \"size (TB) sum\"])\n", "df_pred.tail(8)" ] }, @@ -459,15 +489,17 @@ "source": [ "fig, axs = plt.subplots(1, 1, figsize=fig_size)\n", "\n", - "plt.plot(df_storage_day['date'] , df_storage_day['size (TB) sum'], linewidth=3, color ='k')\n", - "plt.plot(X2_date, y_pred) \n", + "plt.plot(\n", + " df_storage_day[\"date\"], df_storage_day[\"size (TB) sum\"], linewidth=3, color=\"k\"\n", + ")\n", + "plt.plot(X2_date, y_pred)\n", "\n", "\n", "plt.xticks(fontsize=16)\n", "plt.yticks(fontsize=16)\n", "plt.xlabel(\"Date\", fontsize=18)\n", "plt.ylabel(\"Storage space (TB)\", fontsize=18)\n", - "plt.title('Storage on imaging & electrophysiology U19: ' + current_TB, fontsize=20);" + "plt.title(\"Storage on imaging & electrophysiology U19: \" + current_TB, fontsize=20);" ] }, { @@ -484,11 +516,10 @@ "outputs": [], "source": [ "df_storage_year = df_storage.copy()\n", - "df_storage_year = df_storage_year.sort_values(by=['date'])\n", - "df_storage_year['year'] = df_storage_year['date'].dt.year\n", - "df_storage_year = df_storage_year.groupby(df_storage_year['year']).sum()\n", - "df_storage_year = df_storage_year.reset_index()\n", - "\n" + "df_storage_year = df_storage_year.sort_values(by=[\"date\"])\n", + "df_storage_year[\"year\"] = df_storage_year[\"date\"].dt.year\n", + "df_storage_year = df_storage_year.groupby(df_storage_year[\"year\"]).sum()\n", + "df_storage_year = df_storage_year.reset_index()" ] }, { @@ -575,18 +606,18 @@ "source": [ "fig, axs = plt.subplots(1, 1, figsize=fig_size)\n", "\n", - "#plt.scatter(df_storage_day['date'] , df_storage_day['size (TB) sum'], s=100, color = 'k')\n", - "plt.bar(df_storage_year['year'], df_storage_year['size (TB)'])\n", + "# plt.scatter(df_storage_day['date'] , df_storage_day['size (TB) sum'], s=100, color = 'k')\n", + "plt.bar(df_storage_year[\"year\"], df_storage_year[\"size (TB)\"])\n", "\n", "plt.xticks(fontsize=16)\n", "plt.yticks(fontsize=16)\n", "plt.xlabel(\"Year\", fontsize=18)\n", "plt.ylabel(\"Storage space (TB)\", fontsize=18)\n", - "plt.title('Storage on imaging & electrophysiology U19 by year', fontsize=20)\n", + "plt.title(\"Storage on imaging & electrophysiology U19 by year\", fontsize=20)\n", "\n", - "all_years = df_storage_year['year'].values\n", + "all_years = df_storage_year[\"year\"].values\n", "axs.set_xticks(all_years)\n", - "df_storage_year\n" + "df_storage_year" ] }, { @@ -683,12 +714,18 @@ } ], "source": [ - "df_size_parent_folder = df_storage.groupby(by=['parent_folder']).sum()\n", + "df_size_parent_folder = df_storage.groupby(by=[\"parent_folder\"]).sum()\n", "df_size_parent_folder = df_size_parent_folder.reset_index()\n", - "df_size_parent_folder['parent_folder'] = df_size_parent_folder['parent_folder'].str.replace('/mnt/cup/', '')\n", - "df_size_parent_folder['parent_folder'] = df_size_parent_folder['parent_folder'].str.replace('braininit/RigData/', '')\n", - "df_size_parent_folder['parent_folder'] = df_size_parent_folder['parent_folder'].str.replace('PNI-centers/Bezos/RigData/', '')\n", - "df_size_parent_folder\n" + "df_size_parent_folder[\"parent_folder\"] = df_size_parent_folder[\n", + " \"parent_folder\"\n", + "].str.replace(\"/mnt/cup/\", \"\")\n", + "df_size_parent_folder[\"parent_folder\"] = df_size_parent_folder[\n", + " \"parent_folder\"\n", + "].str.replace(\"braininit/RigData/\", \"\")\n", + "df_size_parent_folder[\"parent_folder\"] = df_size_parent_folder[\n", + " \"parent_folder\"\n", + "].str.replace(\"PNI-centers/Bezos/RigData/\", \"\")\n", + "df_size_parent_folder" ] }, { @@ -713,17 +750,17 @@ "source": [ "fig, axs = plt.subplots(1, 1, figsize=fig_size)\n", "index = np.arange(df_size_parent_folder.shape[0])\n", - "plt.bar(index, df_size_parent_folder['size (TB)'])\n", + "plt.bar(index, df_size_parent_folder[\"size (TB)\"])\n", "\n", "plt.xticks(fontsize=16)\n", "plt.yticks(fontsize=16)\n", "plt.xlabel(\"Directory\", fontsize=18)\n", "plt.ylabel(\"Storage space (TB)\", fontsize=18)\n", - "plt.title('Storage on imaging & electrophysiology U19 by main dir', fontsize=20)\n", + "plt.title(\"Storage on imaging & electrophysiology U19 by main dir\", fontsize=20)\n", "\n", - "all_folders = df_size_parent_folder['parent_folder'].values\n", + "all_folders = df_size_parent_folder[\"parent_folder\"].values\n", "axs.set_xticks(index)\n", - "axs.set_xticklabels(all_folders, rotation = 45, ha=\"right\");" + "axs.set_xticklabels(all_folders, rotation=45, ha=\"right\");" ] }, { diff --git a/notebooks/training_days_per_protocol.ipynb b/notebooks/training_days_per_protocol.ipynb index ec408b9c..7fae76ba 100644 --- a/notebooks/training_days_per_protocol.ipynb +++ b/notebooks/training_days_per_protocol.ipynb @@ -33,7 +33,7 @@ "import pandas as pd\n", "import pylab as plt\n", "\n", - "import u19_pipeline.alert_system.behavior_metrics as bm\n" + "import u19_pipeline.alert_system.behavior_metrics as bm" ] }, { @@ -50,8 +50,8 @@ "outputs": [], "source": [ "dj.conn()\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -168,12 +168,15 @@ } ], "source": [ - "\n", - "all_protocols_df = pd.DataFrame(acquisition.Session.fetch('session_protocol', as_dict=True))\n", - "all_protocols_df = all_protocols_df.groupby('session_protocol').agg({'session_protocol': [('num_sessions', 'count')]})\n", + "all_protocols_df = pd.DataFrame(\n", + " acquisition.Session.fetch(\"session_protocol\", as_dict=True)\n", + ")\n", + "all_protocols_df = all_protocols_df.groupby(\"session_protocol\").agg(\n", + " {\"session_protocol\": [(\"num_sessions\", \"count\")]}\n", + ")\n", "all_protocols_df.columns = all_protocols_df.columns.droplevel()\n", - "all_protocols_df = all_protocols_df.sort_values(by='num_sessions', ascending=False)\n", - "all_protocols_df = all_protocols_df.loc[all_protocols_df['num_sessions'] > 100, :]\n", + "all_protocols_df = all_protocols_df.sort_values(by=\"num_sessions\", ascending=False)\n", + "all_protocols_df = all_protocols_df.loc[all_protocols_df[\"num_sessions\"] > 100, :]\n", "all_protocols_df" ] }, @@ -194,7 +197,7 @@ } ], "source": [ - "session_protocol_q = 'session_protocol = \"' +all_protocols_df.index[1] + '\"'\n", + "session_protocol_q = 'session_protocol = \"' + all_protocols_df.index[1] + '\"'\n", "session_protocol_q" ] }, @@ -222,38 +225,38 @@ "source": [ "today = date.today() + timedelta(days=-1)\n", "start_week = today - timedelta(days=today.weekday())\n", - "#end_lastweek = start_week - timedelta(days=1)\n", - "start_week_str = start_week.strftime('%Y-%m-%d')\n", - "#end_lastweek_str = end_lastweek.strftime('%Y-%m-%d')\n", + "# end_lastweek = start_week - timedelta(days=1)\n", + "start_week_str = start_week.strftime(\"%Y-%m-%d\")\n", + "# end_lastweek_str = end_lastweek.strftime('%Y-%m-%d')\n", "\n", - "query_old_subjects = 'session_date <= \"'+ start_week_str + '\"'\n", - "query_new_subjects = 'session_date > \"'+ start_week_str + '\"'\n", - "not_testuser = 'subject_fullname not like \"testuser%\"'\n", - "not_testsubjects = 'subject_fullname not like \"%_test\"'\n", - "not_bad_sessions = 'is_bad_session = 0'\n", - "base_query = [session_protocol_q, not_testuser, not_testsubjects, not_bad_sessions]\n", + "query_old_subjects = 'session_date <= \"' + start_week_str + '\"'\n", + "query_new_subjects = 'session_date > \"' + start_week_str + '\"'\n", + "not_testuser = 'subject_fullname not like \"testuser%\"'\n", + "not_testsubjects = 'subject_fullname not like \"%_test\"'\n", + "not_bad_sessions = \"is_bad_session = 0\"\n", + "base_query = [session_protocol_q, not_testuser, not_testsubjects, not_bad_sessions]\n", "\n", - "query_old_subjects = [query_old_subjects] + base_query\n", - "query_new_subjects = [query_new_subjects] + base_query\n", + "query_old_subjects = [query_old_subjects] + base_query\n", + "query_new_subjects = [query_new_subjects] + base_query\n", "\n", "base_session_table_filtered = acquisition.Session & base_query\n", - "session_table_filtered = copy.deepcopy(base_session_table_filtered)\n", + "session_table_filtered = copy.deepcopy(base_session_table_filtered)\n", "for filter_key in query_old_subjects:\n", " session_table_filtered = session_table_filtered & filter_key\n", - "subject_old_sessions = set((session_table_filtered).fetch('subject_fullname').tolist())\n", + "subject_old_sessions = set((session_table_filtered).fetch(\"subject_fullname\").tolist())\n", "\n", - "session_table_filtered = copy.deepcopy(base_session_table_filtered)\n", + "session_table_filtered = copy.deepcopy(base_session_table_filtered)\n", "for filter_key in query_new_subjects:\n", " session_table_filtered = session_table_filtered & filter_key\n", - "subject_new_sessions = set((session_table_filtered).fetch('subject_fullname').tolist())\n", + "subject_new_sessions = set((session_table_filtered).fetch(\"subject_fullname\").tolist())\n", "\n", "subjects_only_old = list(subject_old_sessions - subject_new_sessions)\n", "\n", - "subjects_only_oldquery = [{'subject_fullname': i} for i in subjects_only_old]\n", - "subjects_new_sessions_query = [{'subject_fullname': i} for i in subject_new_sessions]\n", + "subjects_only_oldquery = [{\"subject_fullname\": i} for i in subjects_only_old]\n", + "subjects_new_sessions_query = [{\"subject_fullname\": i} for i in subject_new_sessions]\n", "\n", - "print('subjects this protocol did not train this week: ', len(subjects_only_oldquery))\n", - "print('subjects this protocol did train this week: ', len(subjects_new_sessions_query))" + "print(\"subjects this protocol did not train this week: \", len(subjects_only_oldquery))\n", + "print(\"subjects this protocol did train this week: \", len(subjects_new_sessions_query))" ] }, { @@ -270,10 +273,21 @@ } ], "source": [ - "session_old_df = pd.DataFrame((base_session_table_filtered & subjects_only_oldquery).fetch('KEY','session_protocol', 'level',order_by='session_start_time', as_dict=True))\n", - "session_new_df = pd.DataFrame((base_session_table_filtered & subjects_new_sessions_query).fetch('KEY','session_protocol', 'level', order_by='session_start_time', as_dict=True))\n", + "session_old_df = pd.DataFrame(\n", + " (base_session_table_filtered & subjects_only_oldquery).fetch(\n", + " \"KEY\", \"session_protocol\", \"level\", order_by=\"session_start_time\", as_dict=True\n", + " )\n", + ")\n", + "session_new_df = pd.DataFrame(\n", + " (base_session_table_filtered & subjects_new_sessions_query).fetch(\n", + " \"KEY\", \"session_protocol\", \"level\", order_by=\"session_start_time\", as_dict=True\n", + " )\n", + ")\n", "\n", - "print('Sessions this protocol from subjects did not train this week: ', session_old_df.shape[0])" + "print(\n", + " \"Sessions this protocol from subjects did not train this week: \",\n", + " session_old_df.shape[0],\n", + ")" ] }, { @@ -478,8 +492,8 @@ } ], "source": [ - "max_level_calculated = np.percentile(session_old_df['level'].values, 90)\n", - "plt.hist(session_old_df['level'].values, bins=session_old_df['level'].max()-1)\n", + "max_level_calculated = np.percentile(session_old_df[\"level\"].values, 90)\n", + "plt.hist(session_old_df[\"level\"].values, bins=session_old_df[\"level\"].max() - 1)\n", "max_level_calculated" ] }, @@ -498,13 +512,29 @@ } ], "source": [ - "level_query = 'level >= '+ str(max_level_calculated)\n", - "session_table_filtered = base_session_table_filtered & subjects_only_oldquery & level_query\n", - "subjects_only_old_level = set((session_table_filtered).fetch('subject_fullname').tolist())\n", - "subjects_only_oldquery_level = [{'subject_fullname': i} for i in subjects_only_old_level]\n", - "session_old_df = pd.DataFrame((base_session_table_filtered & subjects_only_oldquery_level).fetch('KEY', 'level', 'session_location',order_by='session_start_time', as_dict=True))\n", - "print('subjects this protocol did not train this week to get to max level: ', len(subjects_only_oldquery_level))\n", - "print('Sessions this protocol from subjects did not train this week that got to max level: ', session_old_df.shape[0])" + "level_query = \"level >= \" + str(max_level_calculated)\n", + "session_table_filtered = (\n", + " base_session_table_filtered & subjects_only_oldquery & level_query\n", + ")\n", + "subjects_only_old_level = set(\n", + " (session_table_filtered).fetch(\"subject_fullname\").tolist()\n", + ")\n", + "subjects_only_oldquery_level = [\n", + " {\"subject_fullname\": i} for i in subjects_only_old_level\n", + "]\n", + "session_old_df = pd.DataFrame(\n", + " (base_session_table_filtered & subjects_only_oldquery_level).fetch(\n", + " \"KEY\", \"level\", \"session_location\", order_by=\"session_start_time\", as_dict=True\n", + " )\n", + ")\n", + "print(\n", + " \"subjects this protocol did not train this week to get to max level: \",\n", + " len(subjects_only_oldquery_level),\n", + ")\n", + "print(\n", + " \"Sessions this protocol from subjects did not train this week that got to max level: \",\n", + " session_old_df.shape[0],\n", + ")" ] }, { @@ -695,18 +725,22 @@ ], "source": [ "# Count sessions for each subject (sequential count and total count)\n", - "session_old_df = session_old_df.sort_values(by=['subject_fullname', 'session_date', 'session_number'])\n", + "session_old_df = session_old_df.sort_values(\n", + " by=[\"subject_fullname\", \"session_date\", \"session_number\"]\n", + ")\n", "\n", - "sequential_sessions = session_old_df.groupby(['subject_fullname']).cumcount() + 1\n", - "num_sessions_df = session_old_df.groupby('subject_fullname').agg({'session_date': [('total_sessions', 'count')]})\n", + "sequential_sessions = session_old_df.groupby([\"subject_fullname\"]).cumcount() + 1\n", + "num_sessions_df = session_old_df.groupby(\"subject_fullname\").agg(\n", + " {\"session_date\": [(\"total_sessions\", \"count\")]}\n", + ")\n", "num_sessions_df.columns = num_sessions_df.columns.droplevel()\n", "num_sessions_df = num_sessions_df.reset_index()\n", "\n", - "#num_sessions_level_df = session_old_df.groupby(['subject_fullname').agg({'session_date': [('total_sessions', 'count')]})\n", - "#num_sessions_df.columns = num_sessions_df.columns.droplevel()\n", - "#num_sessions_df = num_sessions_df.reset_index()\n", + "# num_sessions_level_df = session_old_df.groupby(['subject_fullname').agg({'session_date': [('total_sessions', 'count')]})\n", + "# num_sessions_df.columns = num_sessions_df.columns.droplevel()\n", + "# num_sessions_df = num_sessions_df.reset_index()\n", "\n", - "session_old_df['seq_sessions'] = sequential_sessions\n", + "session_old_df[\"seq_sessions\"] = sequential_sessions\n", "session_old_df = session_old_df.merge(num_sessions_df)\n", "\n", "session_old_df" @@ -1388,7 +1422,7 @@ } ], "source": [ - "session_old_df.loc[session_old_df['subject_fullname'] == 'jounhong_CaMKII_tetO_7', :]" + "session_old_df.loc[session_old_df[\"subject_fullname\"] == \"jounhong_CaMKII_tetO_7\", :]" ] }, { @@ -1515,12 +1549,14 @@ } ], "source": [ - "protocol_df = session_df.groupby('session_protocol').agg({'subject_fullname': [('num_sessions', 'count'), ('num_subjects', 'nunique')]})\n", + "protocol_df = session_df.groupby(\"session_protocol\").agg(\n", + " {\"subject_fullname\": [(\"num_sessions\", \"count\"), (\"num_subjects\", \"nunique\")]}\n", + ")\n", "protocol_df.columns = protocol_df.columns.droplevel()\n", - "#protocol_df = protocol_df.reset_index()\n", - "protocol_df = protocol_df.sort_values(by='num_sessions', ascending=False)\n", - "protocol_df.loc['Total']= protocol_df.sum(numeric_only=True, axis=0)\n", - "#protocol_df = pd.concat([protocol_df, ])\n", + "# protocol_df = protocol_df.reset_index()\n", + "protocol_df = protocol_df.sort_values(by=\"num_sessions\", ascending=False)\n", + "protocol_df.loc[\"Total\"] = protocol_df.sum(numeric_only=True, axis=0)\n", + "# protocol_df = pd.concat([protocol_df, ])\n", "protocol_df.head(45)" ] }, @@ -1744,8 +1780,28 @@ "source": [ "block_query = \"main_level = level and level>=11\"\n", "\n", - "trials_df = pd.DataFrame((acquisition.Session.proj('session_location') * behavior.TowersBlock * behavior.TowersBlock.Trial() & block_query & session_keys).fetch('KEY', 'level', 'trial_type', 'choice', 'session_location', order_by='session_date', as_dict=True))\n", - "all_sessions_df = pd.DataFrame((acquisition.Session & session_keys).fetch('KEY','session_location', order_by='session_date', as_dict=True))\n", + "trials_df = pd.DataFrame(\n", + " (\n", + " acquisition.Session.proj(\"session_location\")\n", + " * behavior.TowersBlock\n", + " * behavior.TowersBlock.Trial()\n", + " & block_query\n", + " & session_keys\n", + " ).fetch(\n", + " \"KEY\",\n", + " \"level\",\n", + " \"trial_type\",\n", + " \"choice\",\n", + " \"session_location\",\n", + " order_by=\"session_date\",\n", + " as_dict=True,\n", + " )\n", + ")\n", + "all_sessions_df = pd.DataFrame(\n", + " (acquisition.Session & session_keys).fetch(\n", + " \"KEY\", \"session_location\", order_by=\"session_date\", as_dict=True\n", + " )\n", + ")\n", "trials_df" ] }, @@ -2150,9 +2206,13 @@ } ], "source": [ - "metrics_df = bm.BehaviorMetrics.get_bias_from_trial_df(trials_df, return_all_metrics=True)\n", - "metrics_df['performance'] = metrics_df['cum_correct_trials']*100 / metrics_df['cum_trials']\n", - "metrics_df['bias'] = metrics_df['bias'].abs()\n", + "metrics_df = bm.BehaviorMetrics.get_bias_from_trial_df(\n", + " trials_df, return_all_metrics=True\n", + ")\n", + "metrics_df[\"performance\"] = (\n", + " metrics_df[\"cum_correct_trials\"] * 100 / metrics_df[\"cum_trials\"]\n", + ")\n", + "metrics_df[\"bias\"] = metrics_df[\"bias\"].abs()\n", "metrics_df = metrics_df.reset_index(drop=True)\n", "metrics_df" ] @@ -2170,7 +2230,7 @@ "metadata": {}, "outputs": [], "source": [ - "len(list(set(metrics_df['subject_fullname'].tolist())))" + "len(list(set(metrics_df[\"subject_fullname\"].tolist())))" ] }, { @@ -2528,64 +2588,107 @@ "source": [ "performance_min = 70\n", "perf_str = str(performance_min)\n", - "metrics_df_f = metrics_df.loc[(metrics_df['performance'] >= performance_min) & (metrics_df['cum_trials'] > 100), :]\n", + "metrics_df_f = metrics_df.loc[\n", + " (metrics_df[\"performance\"] >= performance_min) & (metrics_df[\"cum_trials\"] > 100), :\n", + "]\n", "\n", - "#Group all sessions by rig\n", - "summary_alls_df = all_sessions_df.groupby('session_location').agg({'subject_fullname': [('num_subjects', 'nunique'), ('num_sessions', 'count')]})\n", + "# Group all sessions by rig\n", + "summary_alls_df = all_sessions_df.groupby(\"session_location\").agg(\n", + " {\"subject_fullname\": [(\"num_subjects\", \"nunique\"), (\"num_sessions\", \"count\")]}\n", + ")\n", "summary_alls_df.columns = summary_alls_df.columns.droplevel()\n", "summary_alls_df = summary_alls_df.reset_index()\n", "\n", - "#Group l11 sessions by rig\n", - "summary_l11_df = metrics_df.groupby('session_location').agg({'subject_fullname': [('num_subjects_l11', 'nunique'), ('num_sessions_l11', 'count')], 'performance': [('performance_l11', 'mean')],\\\n", - " 'bias': [('bias_l11', 'mean')]})\n", + "# Group l11 sessions by rig\n", + "summary_l11_df = metrics_df.groupby(\"session_location\").agg(\n", + " {\n", + " \"subject_fullname\": [\n", + " (\"num_subjects_l11\", \"nunique\"),\n", + " (\"num_sessions_l11\", \"count\"),\n", + " ],\n", + " \"performance\": [(\"performance_l11\", \"mean\")],\n", + " \"bias\": [(\"bias_l11\", \"mean\")],\n", + " }\n", + ")\n", "summary_l11_df.columns = summary_l11_df.columns.droplevel()\n", "summary_l11_df = summary_l11_df.reset_index()\n", "\n", - "#Group l11 70 % sessions by rig\n", - "summary_l1170_df = metrics_df_f.groupby('session_location').agg({'subject_fullname': [('num_subjects_l11>'+perf_str, 'nunique'), ('num_sessions_l11>'+perf_str, 'count')], 'performance': [('performance_l11>'+perf_str, 'mean')],\\\n", - " 'bias': [('bias_l11>'+perf_str, 'mean')]})\n", + "# Group l11 70 % sessions by rig\n", + "summary_l1170_df = metrics_df_f.groupby(\"session_location\").agg(\n", + " {\n", + " \"subject_fullname\": [\n", + " (\"num_subjects_l11>\" + perf_str, \"nunique\"),\n", + " (\"num_sessions_l11>\" + perf_str, \"count\"),\n", + " ],\n", + " \"performance\": [(\"performance_l11>\" + perf_str, \"mean\")],\n", + " \"bias\": [(\"bias_l11>\" + perf_str, \"mean\")],\n", + " }\n", + ")\n", "summary_l1170_df.columns = summary_l1170_df.columns.droplevel()\n", "summary_l1170_df = summary_l1170_df.reset_index()\n", "\n", "# Merge group dataframes\n", - "summary_l11_df = summary_alls_df.merge(summary_l11_df, on='session_location', how='left')\n", - "summary_df = summary_l11_df.merge(summary_l1170_df, on='session_location', how='left')\n", + "summary_l11_df = summary_alls_df.merge(\n", + " summary_l11_df, on=\"session_location\", how=\"left\"\n", + ")\n", + "summary_df = summary_l11_df.merge(summary_l1170_df, on=\"session_location\", how=\"left\")\n", "\n", "\n", "# Fillna, new metrics\n", - "summary_df['num_subjects_l11'] = summary_df['num_subjects_l11'].fillna(0)\n", - "summary_df['num_sessions_l11'] = summary_df['num_sessions_l11'].fillna(0)\n", - "summary_df['num_sessions_l11>'+perf_str] = summary_df['num_sessions_l11>'+perf_str].fillna(0)\n", - "summary_df['num_subjects_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str].fillna(0)\n", + "summary_df[\"num_subjects_l11\"] = summary_df[\"num_subjects_l11\"].fillna(0)\n", + "summary_df[\"num_sessions_l11\"] = summary_df[\"num_sessions_l11\"].fillna(0)\n", + "summary_df[\"num_sessions_l11>\" + perf_str] = summary_df[\n", + " \"num_sessions_l11>\" + perf_str\n", + "].fillna(0)\n", + "summary_df[\"num_subjects_l11>\" + perf_str] = summary_df[\n", + " \"num_subjects_l11>\" + perf_str\n", + "].fillna(0)\n", "\n", "\n", "# Calculate Total rows\n", - "summary_df.loc['Total']= summary_df.sum(numeric_only=True, axis=0)\n", - "mean_columns = ['performance_l11', 'bias_l11', 'performance_l11>'+perf_str, 'bias_l11>'+perf_str]\n", - "summary_df.loc['Total', mean_columns]= summary_df.loc[summary_df.iloc[range(summary_df.shape[0]-1)].index, mean_columns].mean(numeric_only=True, axis=0)\n", + "summary_df.loc[\"Total\"] = summary_df.sum(numeric_only=True, axis=0)\n", + "mean_columns = [\n", + " \"performance_l11\",\n", + " \"bias_l11\",\n", + " \"performance_l11>\" + perf_str,\n", + " \"bias_l11>\" + perf_str,\n", + "]\n", + "summary_df.loc[\"Total\", mean_columns] = summary_df.loc[\n", + " summary_df.iloc[range(summary_df.shape[0] - 1)].index, mean_columns\n", + "].mean(numeric_only=True, axis=0)\n", "\n", "# Ratio sessions / session in level & performance\n", - "summary_df['ratio_l11'] = summary_df['num_subjects_l11']*100 /summary_df['num_subjects']\n", - "summary_df['ratio_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str]*100 /summary_df['num_subjects']\n", + "summary_df[\"ratio_l11\"] = (\n", + " summary_df[\"num_subjects_l11\"] * 100 / summary_df[\"num_subjects\"]\n", + ")\n", + "summary_df[\"ratio_l11>\" + perf_str] = (\n", + " summary_df[\"num_subjects_l11>\" + perf_str] * 100 / summary_df[\"num_subjects\"]\n", + ")\n", "\n", "# Make int columns pretty\n", - "summary_df['num_subjects'] = summary_df['num_subjects'].astype('Int64')\n", - "summary_df['num_sessions'] = summary_df['num_sessions'].astype('Int64')\n", - "summary_df['num_sessions_l11'] = summary_df['num_sessions_l11'].astype('Int64')\n", - "summary_df['num_subjects_l11'] = summary_df['num_subjects_l11'].astype('Int64')\n", - "summary_df['num_sessions_l11>'+perf_str] = summary_df['num_sessions_l11>'+perf_str].astype('Int64')\n", - "summary_df['num_subjects_l11>'+perf_str] = summary_df['num_subjects_l11>'+perf_str].astype('Int64')\n", + "summary_df[\"num_subjects\"] = summary_df[\"num_subjects\"].astype(\"Int64\")\n", + "summary_df[\"num_sessions\"] = summary_df[\"num_sessions\"].astype(\"Int64\")\n", + "summary_df[\"num_sessions_l11\"] = summary_df[\"num_sessions_l11\"].astype(\"Int64\")\n", + "summary_df[\"num_subjects_l11\"] = summary_df[\"num_subjects_l11\"].astype(\"Int64\")\n", + "summary_df[\"num_sessions_l11>\" + perf_str] = summary_df[\n", + " \"num_sessions_l11>\" + perf_str\n", + "].astype(\"Int64\")\n", + "summary_df[\"num_subjects_l11>\" + perf_str] = summary_df[\n", + " \"num_subjects_l11>\" + perf_str\n", + "].astype(\"Int64\")\n", "\n", "# Sort properly\n", - "summary_df = summary_df.sort_values(by=['ratio_l11>'+perf_str, 'num_sessions_l11>'+perf_str], ascending=False)\n", - "summary_df = summary_df.set_index('session_location')\n", + "summary_df = summary_df.sort_values(\n", + " by=[\"ratio_l11>\" + perf_str, \"num_sessions_l11>\" + perf_str], ascending=False\n", + ")\n", + "summary_df = summary_df.set_index(\"session_location\")\n", "\n", "# Total row at the end\n", - "summary_df.index = summary_df.index.fillna('Total')\n", + "summary_df.index = summary_df.index.fillna(\"Total\")\n", "idx = summary_df.index\n", - "idx = idx.drop('Total')\n", + "idx = idx.drop(\"Total\")\n", "idx = idx.to_list()\n", - "summary_df = summary_df.reindex(idx + ['Total'])\n", + "summary_df = summary_df.reindex(idx + [\"Total\"])\n", "summary_df" ] }, @@ -2606,7 +2709,7 @@ } ], "source": [ - "96/168" + "96 / 168" ] }, { @@ -3030,7 +3133,7 @@ } ], "source": [ - "metrics_df_f = metrics_df_f.sort_values(by='performance', ascending=False)\n", + "metrics_df_f = metrics_df_f.sort_values(by=\"performance\", ascending=False)\n", "metrics_df_f" ] }, @@ -3145,8 +3248,8 @@ ], "source": [ "query_s = dict()\n", - "query_s['subject_fullname'] = 'jounhong_CaMKII_tetO_7'\n", - "query_s['session_date'] = '2021-07-22'\n", + "query_s[\"subject_fullname\"] = \"jounhong_CaMKII_tetO_7\"\n", + "query_s[\"session_date\"] = \"2021-07-22\"\n", "a = pd.DataFrame((acquisition.Session & query_s).fetch(as_dict=True))\n", "a" ] diff --git a/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb b/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb index 463f0a97..31982bec 100644 --- a/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb +++ b/notebooks/tutorials/1-Explore U19 data pipeline with DataJoint.ipynb @@ -503,12 +503,14 @@ "metadata": {}, "outputs": [], "source": [ - "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n", - "task = dj.create_virtual_module('task', 'u19_task') \n", - "subject = dj.create_virtual_module('subject', 'u19_subject')\n", - "action = dj.create_virtual_module('action', 'u19_action')\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "lab = dj.create_virtual_module(\n", + " \"lab\", \"u19_lab\"\n", + ") # the first argument here is the __name__ of the virtual module\n", + "task = dj.create_virtual_module(\"task\", \"u19_task\")\n", + "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n", + "action = dj.create_virtual_module(\"action\", \"u19_action\")\n", + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -1217,7 +1219,12 @@ } ], "source": [ - "dj.Diagram(subject.Subject) + dj.Diagram(subject.Death) + dj.Diagram(subject.HealthStatus) + dj.Diagram(subject.Weaning)" + "(\n", + " dj.Diagram(subject.Subject)\n", + " + dj.Diagram(subject.Death)\n", + " + dj.Diagram(subject.HealthStatus)\n", + " + dj.Diagram(subject.Weaning)\n", + ")" ] }, { @@ -2119,7 +2126,7 @@ ], "source": [ "# restrict by dictionary\n", - "subject.Subject & {'subject_nickname': 'B205'}" + "subject.Subject & {\"subject_nickname\": \"B205\"}" ] }, { @@ -2413,7 +2420,7 @@ } ], "source": [ - "subject.Subject & {'sex': 'Male'}" + "subject.Subject & {\"sex\": \"Male\"}" ] }, { @@ -2714,7 +2721,7 @@ } ], "source": [ - "subject.Subject & [{'user_id': 'hnieh'}, {'user_id': 'emanuele'}, 'line=\"DAT-IRES-CRE\"']" + "subject.Subject & [{\"user_id\": \"hnieh\"}, {\"user_id\": \"emanuele\"}, 'line=\"DAT-IRES-CRE\"']" ] }, { @@ -6325,7 +6332,7 @@ } ], "source": [ - "(subject.Subject * acquisition.Session).proj('dob', 'sex', 'location')" + "(subject.Subject * acquisition.Session).proj(\"dob\", \"sex\", \"location\")" ] }, { @@ -6461,7 +6468,7 @@ } ], "source": [ - "subject.Subject.proj(gender='sex', birth_date='dob') # 'sex->gender'" + "subject.Subject.proj(gender=\"sex\", birth_date=\"dob\") # 'sex->gender'" ] }, { @@ -6484,7 +6491,9 @@ "metadata": {}, "outputs": [], "source": [ - "weighing_with_date = action.Weighing.proj(weighing_date='date(weighing_time)') # more options, check MySQL syntax" + "weighing_with_date = action.Weighing.proj(\n", + " weighing_date=\"date(weighing_time)\"\n", + ") # more options, check MySQL syntax" ] }, { @@ -6766,7 +6775,7 @@ "source": [ "# First get the date of birth and the session date into the same query\n", "q = subject.Subject * acquisition.Session\n", - "q = q.proj('dob')\n", + "q = q.proj(\"dob\")\n", "q" ] }, @@ -6927,7 +6936,7 @@ ], "source": [ "# Then compute the age\n", - "q_with_age = q.proj('dob', age='datediff(session_date, dob)') & 'dob is not NULL'\n", + "q_with_age = q.proj(\"dob\", age=\"datediff(session_date, dob)\") & \"dob is not NULL\"\n", "q_with_age" ] }, @@ -7071,7 +7080,9 @@ } ], "source": [ - "subject.Subject.aggr(acquisition.Session, n='count(*)', lastest_session_date='max(session_date)')" + "subject.Subject.aggr(\n", + " acquisition.Session, n=\"count(*)\", lastest_session_date=\"max(session_date)\"\n", + ")" ] }, { @@ -25972,7 +25983,7 @@ } ], "source": [ - "subjs['subject_fullname']" + "subjs[\"subject_fullname\"]" ] }, { @@ -26203,7 +26214,7 @@ } ], "source": [ - "subjs['dob']" + "subjs[\"dob\"]" ] }, { @@ -52433,7 +52444,7 @@ ], "source": [ "# fetch as pandas dataframe\n", - "subjs_df = subject.Subject.fetch(format='frame').reset_index()\n", + "subjs_df = subject.Subject.fetch(format=\"frame\").reset_index()\n", "subjs_df" ] }, @@ -53012,7 +53023,7 @@ ], "source": [ "# fetch the primary key\n", - "pk = subject.Subject.fetch('KEY')\n", + "pk = subject.Subject.fetch(\"KEY\")\n", "pk" ] }, @@ -53023,7 +53034,7 @@ "outputs": [], "source": [ "# fetch specific attributes\n", - "dob, sex = subject.Subject.fetch('dob', 'sex')" + "dob, sex = subject.Subject.fetch(\"dob\", \"sex\")" ] }, { @@ -53832,7 +53843,7 @@ ], "source": [ "# fetch specific attributes as a list of dictionary\n", - "info = subject.Subject.fetch('dob', 'sex', as_dict=True)\n", + "info = subject.Subject.fetch(\"dob\", \"sex\", as_dict=True)\n", "info" ] }, @@ -53849,7 +53860,9 @@ "metadata": {}, "outputs": [], "source": [ - "B205 = (subject.Subject & {'subject_nickname': 'B205'}).fetch1() # \"fetch1()\" because we know there's only one" + "B205 = (\n", + " subject.Subject & {\"subject_nickname\": \"B205\"}\n", + ").fetch1() # \"fetch1()\" because we know there's only one" ] }, { @@ -53939,7 +53952,7 @@ "metadata": {}, "outputs": [], "source": [ - "B205_key = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('KEY')" + "B205_key = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\"KEY\")" ] }, { @@ -53968,7 +53981,9 @@ "metadata": {}, "outputs": [], "source": [ - "B205_init_weight = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('initial_weight')" + "B205_init_weight = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\n", + " \"initial_weight\"\n", + ")" ] }, { @@ -54008,7 +54023,7 @@ } ], "source": [ - "(subject.Subject & {'subject_nickname': 'B205'}).fetch('initial_weight')" + "(subject.Subject & {\"subject_nickname\": \"B205\"}).fetch(\"initial_weight\")" ] }, { diff --git a/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb b/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb index d2f72ad1..06a0de3d 100644 --- a/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb +++ b/notebooks/tutorials/2-Analyze data with U19 pipeline and save results.ipynb @@ -33,12 +33,14 @@ "import datajoint as dj\n", "import numpy as np\n", "\n", - "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n", - "task = dj.create_virtual_module('task', 'u19_task') \n", - "subject = dj.create_virtual_module('subject', 'u19_subject')\n", - "action = dj.create_virtual_module('action', 'u19_action')\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "lab = dj.create_virtual_module(\n", + " \"lab\", \"u19_lab\"\n", + ") # the first argument here is the __name__ of the virtual module\n", + "task = dj.create_virtual_module(\"task\", \"u19_task\")\n", + "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n", + "action = dj.create_virtual_module(\"action\", \"u19_action\")\n", + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -692,7 +694,7 @@ } ], "source": [ - "behavior.TowersBlock() # bracket necessary" + "behavior.TowersBlock() # bracket necessary" ] }, { @@ -843,8 +845,10 @@ } ], "source": [ - "behavior.TowersSession.aggr(behavior.TowersBlock.proj('block_performance'), \n", - " avg_performance='avg(block_performance)')" + "behavior.TowersSession.aggr(\n", + " behavior.TowersBlock.proj(\"block_performance\"),\n", + " avg_performance=\"avg(block_performance)\",\n", + ")" ] }, { @@ -875,7 +879,7 @@ "metadata": {}, "outputs": [], "source": [ - "schema = dj.schema('alvaros_tutorial2021')" + "schema = dj.schema(\"alvaros_tutorial2021\")" ] }, { @@ -1289,8 +1293,7 @@ } ], "source": [ - "behavior.TowersSession.aggr(behavior.TowersBlock.proj(),\n", - " n_sessions='count(*)')" + "behavior.TowersSession.aggr(behavior.TowersBlock.proj(), n_sessions=\"count(*)\")" ] }, { @@ -1307,9 +1310,9 @@ "outputs": [], "source": [ "key = {\n", - " 'subject_fullname': 'emanuele_B205',\n", - " 'session_date': datetime.date(2018, 7, 13),\n", - " 'session_number': 0\n", + " \"subject_fullname\": \"emanuele_B205\",\n", + " \"session_date\": datetime.date(2018, 7, 13),\n", + " \"session_number\": 0,\n", "}" ] }, @@ -1562,10 +1565,10 @@ "metadata": {}, "outputs": [], "source": [ - "performances = (behavior.TowersBlock & key).fetch('block_performance')\n", + "performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n", "\n", "# create another field in the dictionary key\n", - "key['avg_performance'] = np.mean(performances)" + "key[\"avg_performance\"] = np.mean(performances)" ] }, { @@ -1604,7 +1607,9 @@ "metadata": {}, "outputs": [], "source": [ - "SessionPerformanceManual.insert1(key, skip_duplicates=True) # insert1 only works for one entry" + "SessionPerformanceManual.insert1(\n", + " key, skip_duplicates=True\n", + ") # insert1 only works for one entry" ] }, { @@ -1821,7 +1826,9 @@ } ], "source": [ - "subject.Subject.aggr(behavior.TowersSession.proj(), n_sessions='count(*)') & 'n_sessions=100'" + "subject.Subject.aggr(\n", + " behavior.TowersSession.proj(), n_sessions=\"count(*)\"\n", + ") & \"n_sessions=100\"" ] }, { @@ -1850,14 +1857,13 @@ "\n", "start_time = time.time()\n", "\n", - "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E57\"').fetch('KEY'):\n", - " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n", + "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E57\"').fetch(\"KEY\"):\n", + " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n", " # create another field in the dictionary key\n", " avg_performance = np.mean(performances)\n", " if np.isnan(avg_performance):\n", " continue\n", - " entry = dict(**i_session, \n", - " avg_performance=avg_performance)\n", + " entry = dict(**i_session, avg_performance=avg_performance)\n", " SessionPerformanceManual.insert1(entry)\n", "\n", "print(\"--- %s seconds ---\" % (time.time() - start_time))" @@ -1881,16 +1887,15 @@ "start_time = time.time()\n", "\n", "perf_entries = []\n", - "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E77\"').fetch('KEY'):\n", - " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n", + "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E77\"').fetch(\"KEY\"):\n", + " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n", " # create another field in the dictionary key\n", " avg_performance = np.mean(performances)\n", " if np.isnan(avg_performance):\n", " continue\n", - " entry = dict(**i_session, \n", - " avg_performance=avg_performance)\n", + " entry = dict(**i_session, avg_performance=avg_performance)\n", " perf_entries.append(entry)\n", - " \n", + "\n", "SessionPerformanceManual.insert(perf_entries)\n", "print(\"--- %s seconds ---\" % (time.time() - start_time))" ] @@ -1916,14 +1921,19 @@ " ---\n", " avg_performance: float # a final product in this table\n", " \"\"\"\n", - " \n", - " key_source = acquisition.Session() & 'subject_fullname=\"hnieh_E57\"' # bracket necessary\n", - " def make(self, key): # key is one primary key of the entries in table acquisition.Session\n", + "\n", + " key_source = (\n", + " acquisition.Session() & 'subject_fullname=\"hnieh_E57\"'\n", + " ) # bracket necessary\n", + "\n", + " def make(\n", + " self, key\n", + " ): # key is one primary key of the entries in table acquisition.Session\n", " # fetch the performance for each block\n", - " performances = (behavior.TowersBlock & key).fetch('block_performance')\n", - " \n", + " performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n", + "\n", " # create another field in the dictionary key\n", - " key['avg_performance'] = np.mean(performances)\n", + " key[\"avg_performance\"] = np.mean(performances)\n", " self.insert1(key)" ] }, @@ -1948,7 +1958,7 @@ } ], "source": [ - "SessionPerformanceComputed.populate(display_progress=True) \n", + "SessionPerformanceComputed.populate(display_progress=True)\n", "# first argument could be some restrictor to control the populate" ] }, @@ -2134,21 +2144,23 @@ " ---\n", " avg_performance: float # a final product in this table\n", " \"\"\"\n", - " key_source = subject.Subject() \n", - " def make(self, key): # key is one primary key of the entries in table subject.Subject()!\n", + " key_source = subject.Subject()\n", + "\n", + " def make(\n", + " self, key\n", + " ): # key is one primary key of the entries in table subject.Subject()!\n", "\n", " perf_entries = []\n", - " for i_session in (behavior.TowersSession & key).fetch('KEY'):\n", + " for i_session in (behavior.TowersSession & key).fetch(\"KEY\"):\n", " # fetch performance of each block\n", - " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n", + " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n", " # create another field in the dictionary key\n", " avg_performance = np.mean(performances)\n", " if np.isnan(avg_performance):\n", " continue\n", - " entry = dict(**i_session, \n", - " avg_performance=avg_performance)\n", + " entry = dict(**i_session, avg_performance=avg_performance)\n", " perf_entries.append(entry)\n", - " \n", + "\n", " self.insert(perf_entries)" ] }, @@ -2176,7 +2188,12 @@ } ], "source": [ - "SessionPerformanceComputedFromSubject.populate('subject_fullname=\"hnieh_E77\"', display_progress=True, suppress_errors=True, reserve_jobs=True)" + "SessionPerformanceComputedFromSubject.populate(\n", + " 'subject_fullname=\"hnieh_E77\"',\n", + " display_progress=True,\n", + " suppress_errors=True,\n", + " reserve_jobs=True,\n", + ")" ] }, { @@ -2211,7 +2228,9 @@ } ], "source": [ - "(SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"').delete() # any restrictor would work here" + "(\n", + " SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"'\n", + ").delete() # any restrictor would work here" ] }, { diff --git a/notebooks/tutorials/3-Build a simple data pipeline.ipynb b/notebooks/tutorials/3-Build a simple data pipeline.ipynb index 235c71ac..a3eabe31 100644 --- a/notebooks/tutorials/3-Build a simple data pipeline.ipynb +++ b/notebooks/tutorials/3-Build a simple data pipeline.ipynb @@ -235,7 +235,7 @@ ], "source": [ "# create your own schema object\n", - "schema = dj.schema('alvaros_tutorial2021')" + "schema = dj.schema(\"alvaros_tutorial2021\")" ] }, { @@ -400,7 +400,7 @@ "outputs": [], "source": [ "# inserting the first mouse\n", - "Mouse.insert1((0, '2017-03-01', 'M'))" + "Mouse.insert1((0, \"2017-03-01\", \"M\"))" ] }, { @@ -513,7 +513,7 @@ "metadata": {}, "outputs": [], "source": [ - "Mouse.insert1((20, datetime.date(2020, 5, 3), 'F'))" + "Mouse.insert1((20, datetime.date(2020, 5, 3), \"F\"))" ] }, { @@ -627,11 +627,7 @@ "metadata": {}, "outputs": [], "source": [ - "data = {\n", - " 'mouse_id': 100,\n", - " 'dob': '2017-05-12',\n", - " 'sex': 'F'\n", - "}" + "data = {\"mouse_id\": 100, \"dob\": \"2017-05-12\", \"sex\": \"F\"}" ] }, { @@ -758,11 +754,7 @@ "metadata": {}, "outputs": [], "source": [ - "data = [\n", - " (1, '2016-11-19', 'M'),\n", - " (2, '2016-11-20', 'unknown'),\n", - " (5, '2016-12-25', 'F')\n", - "]" + "data = [(1, \"2016-11-19\", \"M\"), (2, \"2016-11-20\", \"unknown\"), (5, \"2016-12-25\", \"F\")]" ] }, { @@ -899,8 +891,8 @@ "outputs": [], "source": [ "data = [\n", - " {'mouse_id': 10, 'dob': '2017-01-01', 'sex': 'F'},\n", - " {'mouse_id': 11, 'dob': '2017-01-03', 'sex': 'F'},\n", + " {\"mouse_id\": 10, \"dob\": \"2017-01-01\", \"sex\": \"F\"},\n", + " {\"mouse_id\": 11, \"dob\": \"2017-01-03\", \"sex\": \"F\"},\n", "]\n", "\n", "# insert them all\n", @@ -1058,9 +1050,10 @@ ], "source": [ "Mouse.insert1(\n", - " {'mouse_id': 0,\n", - " 'dob': '2018-01-01',\n", - " 'sex': 'M',\n", + " {\n", + " \"mouse_id\": 0,\n", + " \"dob\": \"2018-01-01\",\n", + " \"sex\": \"M\",\n", " }\n", ")" ] @@ -1107,7 +1100,7 @@ } ], "source": [ - "(Mouse & 'mouse_id=11').delete()" + "(Mouse & \"mouse_id=11\").delete()" ] }, { @@ -1241,7 +1234,7 @@ } ], "source": [ - "(Mouse & 'mouse_id=0')._update('sex', 'M')" + "(Mouse & \"mouse_id=0\")._update(\"sex\", \"M\")" ] }, { @@ -1533,10 +1526,10 @@ "outputs": [], "source": [ "data = {\n", - " 'mouse_id': 0,\n", - " 'session_date': '2017-05-15',\n", - " 'experiment_setup': 0,\n", - " 'experimenter': 'Edgar Y. Walker'\n", + " \"mouse_id\": 0,\n", + " \"session_date\": \"2017-05-15\",\n", + " \"experiment_setup\": 0,\n", + " \"experimenter\": \"Edgar Y. Walker\",\n", "}\n", "\n", "Session.insert1(data)" @@ -1749,10 +1742,10 @@ ], "source": [ "data = {\n", - " 'mouse_id': 0,\n", - " 'session_date': '2018-01-15',\n", - " 'experiment_setup': 100,\n", - " 'experimenter': 'Jacob Reimer'\n", + " \"mouse_id\": 0,\n", + " \"session_date\": \"2018-01-15\",\n", + " \"experiment_setup\": 100,\n", + " \"experimenter\": \"Jacob Reimer\",\n", "}\n", "\n", "# insert them all\n", @@ -1775,10 +1768,10 @@ "outputs": [], "source": [ "data = {\n", - " 'mouse_id': 1,\n", - " 'session_date': '2018-01-15',\n", - " 'experiment_setup': 101,\n", - " 'experimenter': 'Jacob Reimer'\n", + " \"mouse_id\": 1,\n", + " \"session_date\": \"2018-01-15\",\n", + " \"experiment_setup\": 101,\n", + " \"experimenter\": \"Jacob Reimer\",\n", "}\n", "\n", "# insert them all\n", @@ -1906,10 +1899,10 @@ "outputs": [], "source": [ "bad_data = {\n", - " 'mouse_id': 9999, # this mouse doesn't exist!\n", - " 'session_date': '2017-05-15',\n", - " 'experiment_setup': 0,\n", - " 'experimenter': 'Edgar Y. Walker'\n", + " \"mouse_id\": 9999, # this mouse doesn't exist!\n", + " \"session_date\": \"2017-05-15\",\n", + " \"experiment_setup\": 0,\n", + " \"experimenter\": \"Edgar Y. Walker\",\n", "}" ] }, @@ -1987,7 +1980,7 @@ } ], "source": [ - "(Mouse & 'mouse_id = 100').delete()" + "(Mouse & \"mouse_id = 100\").delete()" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb b/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb index fed95af2..165441f5 100644 --- a/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb +++ b/notebooks/tutorials/Old_tutorials/2019/python_demo.ipynb @@ -38,7 +38,7 @@ "metadata": {}, "outputs": [], "source": [ - "dj.config['database.port'] = 6306" + "dj.config[\"database.port\"] = 6306" ] }, { @@ -47,7 +47,7 @@ "metadata": {}, "outputs": [], "source": [ - "#Overview across database schemas\n", + "# Overview across database schemas\n", "dj.list_schemas()" ] }, @@ -57,10 +57,10 @@ "metadata": {}, "outputs": [], "source": [ - "#Single entity relationship diagram. This is how the data is organized\n", - "acqu = dj.schema('pni_acquisition')\n", - "subj = dj.schema('pni_subject')\n", - "lab = dj.schema('pni_lab')\n", + "# Single entity relationship diagram. This is how the data is organized\n", + "acqu = dj.schema(\"pni_acquisition\")\n", + "subj = dj.schema(\"pni_subject\")\n", + "lab = dj.schema(\"pni_lab\")\n", "\n", "dj.ERD(subj)" ] @@ -78,11 +78,11 @@ "metadata": {}, "outputs": [], "source": [ - "acquisition = dj.create_virtual_module('acquisition', 'pni_acquisition')\n", - "subject = dj.create_virtual_module('subject', 'pni_subject')\n", - "action = dj.create_virtual_module('action', 'pni_action')\n", - "lab = dj.create_virtual_module('lab', 'pni_lab')\n", - "task = dj.create_virtual_module('task', 'pni_task')" + "acquisition = dj.create_virtual_module(\"acquisition\", \"pni_acquisition\")\n", + "subject = dj.create_virtual_module(\"subject\", \"pni_subject\")\n", + "action = dj.create_virtual_module(\"action\", \"pni_action\")\n", + "lab = dj.create_virtual_module(\"lab\", \"pni_lab\")\n", + "task = dj.create_virtual_module(\"task\", \"pni_task\")" ] }, { @@ -117,10 +117,7 @@ "outputs": [], "source": [ "new_user = dict(\n", - " user_id='testuser3',\n", - " contact_via='Slack',\n", - " presence='Away',\n", - " day_cutoff_time=' '\n", + " user_id=\"testuser3\", contact_via=\"Slack\", presence=\"Away\", day_cutoff_time=\" \"\n", ")" ] }, @@ -148,7 +145,7 @@ "metadata": {}, "outputs": [], "source": [ - "dj.Table._update(lab.User & 'user_id=\"testuser3\"', 'email', 'bla@princeton.edu')" + "dj.Table._update(lab.User & 'user_id=\"testuser3\"', \"email\", \"bla@princeton.edu\")" ] }, { @@ -203,7 +200,7 @@ "outputs": [], "source": [ "# Very fat mice\n", - "subject.Subject() & 'initial_weight > 35'" + "subject.Subject() & \"initial_weight > 35\"" ] }, { @@ -221,7 +218,9 @@ "metadata": {}, "outputs": [], "source": [ - "q = (subject.Subject & 'subject_id=\"B205\"').proj('sex', 'dob', 'initial_weight', animal_location='location')" + "q = (subject.Subject & 'subject_id=\"B205\"').proj(\n", + " \"sex\", \"dob\", \"initial_weight\", animal_location=\"location\"\n", + ")" ] }, { @@ -242,7 +241,7 @@ "# On which days of week does people like to work harder?\n", "Session = acquisition.Session()\n", "for day in range(7):\n", - " print(len(Session & 'WEEKDAY(session_date) = ' + str(day)))" + " print(len(Session & \"WEEKDAY(session_date) = \" + str(day)))" ] }, { @@ -279,7 +278,7 @@ "outputs": [], "source": [ "# some magic to get weight on the date of each session\n", - "weight_with_date = action.Weighing.proj('weight', session_date=\"DATE(weighing_time)\")" + "weight_with_date = action.Weighing.proj(\"weight\", session_date=\"DATE(weighing_time)\")" ] }, { @@ -307,7 +306,9 @@ "metadata": {}, "outputs": [], "source": [ - "weight, performance = (query & 'subject_id = \"E47\"').fetch('weight','session_performance')" + "weight, performance = (query & 'subject_id = \"E47\"').fetch(\n", + " \"weight\", \"session_performance\"\n", + ")" ] }, { @@ -326,8 +327,8 @@ "outputs": [], "source": [ "pl.scatter(weight, performance)\n", - "pl.xlabel('bodyweight [g]')\n", - "pl.ylabel('performance')" + "pl.xlabel(\"bodyweight [g]\")\n", + "pl.ylabel(\"performance\")" ] }, { @@ -343,23 +344,25 @@ "metadata": {}, "outputs": [], "source": [ - "subjects = subject.Subject.proj('sex') & query & 'user_id = \"edward\"'\n", + "subjects = subject.Subject.proj(\"sex\") & query & 'user_id = \"edward\"'\n", "corrs = []\n", "for s in subjects:\n", " # fetch all related info: weight, performance and sex\n", - " sex, weight, performance = (subject.Subject.proj('sex') * query & s).fetch('sex', 'weight','session_performance')\n", - " \n", + " sex, weight, performance = (subject.Subject.proj(\"sex\") * query & s).fetch(\n", + " \"sex\", \"weight\", \"session_performance\"\n", + " )\n", + "\n", " weight = weight[performance > 0]\n", " performance = performance[performance > 0]\n", - " if sex[0] == 'Male':\n", + " if sex[0] == \"Male\":\n", " pl.scatter(weight, performance, c=[[0, 0, 1]])\n", - " else: \n", + " else:\n", " pl.scatter(weight, performance, c=[[1, 0, 0]])\n", " corrs.append(np.corrcoef(weight, performance)[0, 1])\n", - " \n", + "\n", "pl.xlim([19, 27])\n", - "pl.xlabel('bodyweight [g]')\n", - "pl.ylabel('performance')" + "pl.xlabel(\"bodyweight [g]\")\n", + "pl.ylabel(\"performance\")" ] }, { @@ -368,10 +371,10 @@ "metadata": {}, "outputs": [], "source": [ - "subjects = (subject.Subject() & 'sex = \"Male\"' & 'user_id = \"edward\"' ).fetch('KEY')\n", - "boymean = np.mean((Session & subjects).fetch('session_performance'))\n", - "subjects = (subject.Subject() & 'sex = \"Female\"' & 'user_id = \"edward\"' ).fetch('KEY')\n", - "girlmean = np.mean((Session & subjects).fetch('session_performance'))\n", + "subjects = (subject.Subject() & 'sex = \"Male\"' & 'user_id = \"edward\"').fetch(\"KEY\")\n", + "boymean = np.mean((Session & subjects).fetch(\"session_performance\"))\n", + "subjects = (subject.Subject() & 'sex = \"Female\"' & 'user_id = \"edward\"').fetch(\"KEY\")\n", + "girlmean = np.mean((Session & subjects).fetch(\"session_performance\"))\n", "print(boymean)\n", "print(girlmean)" ] @@ -389,9 +392,9 @@ "metadata": {}, "outputs": [], "source": [ - "pl.hist(corrs,np.arange(-1,1,0.1))\n", + "pl.hist(corrs, np.arange(-1, 1, 0.1))\n", "print(np.mean(corrs))\n", - "print(np.std(corrs)/np.sqrt(len(corrs)))" + "print(np.std(corrs) / np.sqrt(len(corrs)))" ] }, { @@ -418,7 +421,7 @@ "outputs": [], "source": [ "trials_subset = acquisition.TowersBlock.Trial & sessions\n", - "acquisition.Session.aggr(trials_subset, n='count(*)')" + "acquisition.Session.aggr(trials_subset, n=\"count(*)\")" ] }, { @@ -434,7 +437,7 @@ "metadata": {}, "outputs": [], "source": [ - "pos, time = trials_subset.fetch('position', 'trial_time')" + "pos, time = trials_subset.fetch(\"position\", \"trial_time\")" ] }, { @@ -445,9 +448,9 @@ "source": [ "for trial_idx in range(len(pos)):\n", " N_samples = len(pos[trial_idx])\n", - " pl.plot(time[trial_idx][0:N_samples], pos[trial_idx][:,1])\n", - "pl.xlabel('Time [s]')\n", - "pl.ylabel('position [cm]');" + " pl.plot(time[trial_idx][0:N_samples], pos[trial_idx][:, 1])\n", + "pl.xlabel(\"Time [s]\")\n", + "pl.ylabel(\"position [cm]\");" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb b/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb index 2d936f4f..5dccf245 100644 --- a/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb +++ b/notebooks/tutorials/Old_tutorials/202001/0-Get DataJoint Ready.ipynb @@ -152,7 +152,7 @@ "metadata": {}, "outputs": [], "source": [ - "dj.config['database.host'] = 'datajoint00.pni.princeton.edu'" + "dj.config[\"database.host\"] = \"datajoint00.pni.princeton.edu\"" ] }, { @@ -296,7 +296,7 @@ " \"display.show_tuple_count\": true,\n", " \"database.use_tls\": null,\n", " \"enable_python_native_blobs\": false,\n", - " \"database.prefix\": null\n", + " \"database.prefix\": null,\n", "}" ] }, @@ -344,8 +344,8 @@ } ], "source": [ - "dj.config['database.user'] = 'shans'\n", - "dj.config['database.password'] = getpass('Type password:')" + "dj.config[\"database.user\"] = \"shans\"\n", + "dj.config[\"database.password\"] = getpass(\"Type password:\")" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202001/1-Build a simple data pipeline.ipynb b/notebooks/tutorials/Old_tutorials/202001/1-Build a simple data pipeline.ipynb index ce7bc7b3..a16fb405 100644 --- a/notebooks/tutorials/Old_tutorials/202001/1-Build a simple data pipeline.ipynb +++ b/notebooks/tutorials/Old_tutorials/202001/1-Build a simple data pipeline.ipynb @@ -216,7 +216,7 @@ "outputs": [], "source": [ "# create your own schema object\n", - "schema = dj.schema('shans_tutorial')" + "schema = dj.schema(\"shans_tutorial\")" ] }, { @@ -399,7 +399,7 @@ "outputs": [], "source": [ "# inserting the first mouse\n", - "Mouse.insert1((0, '2017-03-01', 'M'))" + "Mouse.insert1((0, \"2017-03-01\", \"M\"))" ] }, { @@ -511,11 +511,7 @@ "metadata": {}, "outputs": [], "source": [ - "data = {\n", - " 'mouse_id': 100,\n", - " 'dob': '2017-05-12',\n", - " 'sex': 'F'\n", - "}" + "data = {\"mouse_id\": 100, \"dob\": \"2017-05-12\", \"sex\": \"F\"}" ] }, { @@ -640,11 +636,7 @@ "metadata": {}, "outputs": [], "source": [ - "data = [\n", - " (1, '2016-11-19', 'M'),\n", - " (2, '2016-11-20', 'unknown'),\n", - " (5, '2016-12-25', 'F')\n", - "]" + "data = [(1, \"2016-11-19\", \"M\"), (2, \"2016-11-20\", \"unknown\"), (5, \"2016-12-25\", \"F\")]" ] }, { @@ -779,8 +771,8 @@ "outputs": [], "source": [ "data = [\n", - " {'mouse_id': 10, 'dob': '2017-01-01', 'sex': 'F'},\n", - " {'mouse_id': 11, 'dob': '2017-01-03', 'sex': 'F'},\n", + " {\"mouse_id\": 10, \"dob\": \"2017-01-01\", \"sex\": \"F\"},\n", + " {\"mouse_id\": 11, \"dob\": \"2017-01-03\", \"sex\": \"F\"},\n", "]\n", "\n", "# insert them all\n", @@ -922,10 +914,12 @@ "outputs": [], "source": [ "Mouse.insert1(\n", - "{'mouse_id': 0,\n", - " 'dob': '2018-01-01',\n", - " 'sex': 'M',\n", - "})" + " {\n", + " \"mouse_id\": 0,\n", + " \"dob\": \"2018-01-01\",\n", + " \"sex\": \"M\",\n", + " }\n", + ")" ] }, { @@ -961,7 +955,7 @@ } ], "source": [ - "(Mouse & 'mouse_id=11').delete()" + "(Mouse & \"mouse_id=11\").delete()" ] }, { @@ -1081,7 +1075,7 @@ "metadata": {}, "outputs": [], "source": [ - "dj.Table._update(Mouse & 'mouse_id=0', 'sex', 'F')" + "dj.Table._update(Mouse & \"mouse_id=0\", \"sex\", \"F\")" ] }, { @@ -1153,7 +1147,7 @@ "metadata": {}, "outputs": [], "source": [ - "schema2 = dj.schema('shans_tutorial2')" + "schema2 = dj.schema(\"shans_tutorial2\")" ] }, { @@ -1256,10 +1250,10 @@ "outputs": [], "source": [ "data = {\n", - " 'mouse_id': 0,\n", - " 'session_date': '2017-05-15',\n", - " 'experiment_setup': 0,\n", - " 'experimenter': 'Edgar Y. Walker'\n", + " \"mouse_id\": 0,\n", + " \"session_date\": \"2017-05-15\",\n", + " \"experiment_setup\": 0,\n", + " \"experimenter\": \"Edgar Y. Walker\",\n", "}\n", "\n", "Session.insert1(data)" @@ -1474,10 +1468,10 @@ ], "source": [ "data = {\n", - " 'mouse_id': 0,\n", - " 'session_date': '2018-01-15',\n", - " 'experiment_setup': 100,\n", - " 'experimenter': 'Jacob Reimer'\n", + " \"mouse_id\": 0,\n", + " \"session_date\": \"2018-01-15\",\n", + " \"experiment_setup\": 100,\n", + " \"experimenter\": \"Jacob Reimer\",\n", "}\n", "\n", "# insert them all\n", @@ -1500,10 +1494,10 @@ "outputs": [], "source": [ "data = {\n", - " 'mouse_id': 1,\n", - " 'session_date': '2018-01-15',\n", - " 'experiment_setup': 101,\n", - " 'experimenter': 'Jacob Reimer'\n", + " \"mouse_id\": 1,\n", + " \"session_date\": \"2018-01-15\",\n", + " \"experiment_setup\": 101,\n", + " \"experimenter\": \"Jacob Reimer\",\n", "}\n", "\n", "# insert them all\n", @@ -1632,10 +1626,10 @@ "outputs": [], "source": [ "bad_data = {\n", - " 'mouse_id': 9999, # this mouse doesn't exist!\n", - " 'session_date': '2017-05-15',\n", - " 'experiment_setup': 0,\n", - " 'experimenter': 'Edgar Y. Walker'\n", + " \"mouse_id\": 9999, # this mouse doesn't exist!\n", + " \"session_date\": \"2017-05-15\",\n", + " \"experiment_setup\": 0,\n", + " \"experimenter\": \"Edgar Y. Walker\",\n", "}" ] }, @@ -1704,7 +1698,7 @@ } ], "source": [ - "(Mouse & 'mouse_id = 100').delete()" + "(Mouse & \"mouse_id = 100\").delete()" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202001/2-Explore U19 data pipeline with DataJoint.ipynb b/notebooks/tutorials/Old_tutorials/202001/2-Explore U19 data pipeline with DataJoint.ipynb index 1ef28255..db743c4e 100644 --- a/notebooks/tutorials/Old_tutorials/202001/2-Explore U19 data pipeline with DataJoint.ipynb +++ b/notebooks/tutorials/Old_tutorials/202001/2-Explore U19 data pipeline with DataJoint.ipynb @@ -123,12 +123,14 @@ "metadata": {}, "outputs": [], "source": [ - "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n", - "task = dj.create_virtual_module('task', 'u19_task') \n", - "subject = dj.create_virtual_module('subject', 'u19_subject')\n", - "action = dj.create_virtual_module('action', 'u19_action')\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "lab = dj.create_virtual_module(\n", + " \"lab\", \"u19_lab\"\n", + ") # the first argument here is the __name__ of the virtual module\n", + "task = dj.create_virtual_module(\"task\", \"u19_task\")\n", + "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n", + "action = dj.create_virtual_module(\"action\", \"u19_action\")\n", + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -1069,7 +1071,12 @@ } ], "source": [ - "dj.Diagram(subject.Subject) + dj.Diagram(subject.Death) + dj.Diagram(subject.HealthStatus) + dj.Diagram(subject.Weaning)" + "(\n", + " dj.Diagram(subject.Subject)\n", + " + dj.Diagram(subject.Death)\n", + " + dj.Diagram(subject.HealthStatus)\n", + " + dj.Diagram(subject.Weaning)\n", + ")" ] }, { @@ -1861,7 +1868,7 @@ ], "source": [ "# restrict by dictionary\n", - "subject.Subject & {'subject_nickname': 'B205'}" + "subject.Subject & {\"subject_nickname\": \"B205\"}" ] }, { @@ -2126,7 +2133,7 @@ } ], "source": [ - "subject.Subject & {'sex': 'Male'}" + "subject.Subject & {\"sex\": \"Male\"}" ] }, { @@ -2398,7 +2405,7 @@ } ], "source": [ - "subject.Subject & [{'user_id': 'hnieh'}, {'user_id': 'emanuele'}]" + "subject.Subject & [{\"user_id\": \"hnieh\"}, {\"user_id\": \"emanuele\"}]" ] }, { @@ -5478,7 +5485,7 @@ } ], "source": [ - "subject.Subject.proj('dob', 'sex')" + "subject.Subject.proj(\"dob\", \"sex\")" ] }, { @@ -5615,7 +5622,7 @@ } ], "source": [ - "subject.Subject.proj(gender='sex', birth_date='dob')" + "subject.Subject.proj(gender=\"sex\", birth_date=\"dob\")" ] }, { @@ -5638,7 +5645,9 @@ "metadata": {}, "outputs": [], "source": [ - "weighing_with_date = action.Weighing.proj(weighing_date='date(weighing_time)') # more options, check MySQL syntax" + "weighing_with_date = action.Weighing.proj(\n", + " weighing_date=\"date(weighing_time)\"\n", + ") # more options, check MySQL syntax" ] }, { @@ -5922,7 +5931,7 @@ "source": [ "# First get the date of birth and the session date into the same query\n", "q = subject.Subject * acquisition.Session\n", - "q = q.proj('dob')\n", + "q = q.proj(\"dob\")\n", "q" ] }, @@ -6084,7 +6093,7 @@ ], "source": [ "# Then compute the age\n", - "q_with_age = q.proj('dob', age='datediff(session_date, dob)') & 'dob is not NULL'\n", + "q_with_age = q.proj(\"dob\", age=\"datediff(session_date, dob)\") & \"dob is not NULL\"\n", "q_with_age" ] }, @@ -6229,7 +6238,9 @@ } ], "source": [ - "subject.Subject.aggr(acquisition.Session, n='count(*)', lastest_session='min(session_date)')" + "subject.Subject.aggr(\n", + " acquisition.Session, n=\"count(*)\", lastest_session=\"min(session_date)\"\n", + ")" ] }, { @@ -14825,7 +14836,7 @@ } ], "source": [ - "subjs['subject_fullname']" + "subjs[\"subject_fullname\"]" ] }, { @@ -14902,7 +14913,7 @@ } ], "source": [ - "subjs['dob']" + "subjs[\"dob\"]" ] }, { @@ -26702,7 +26713,7 @@ ], "source": [ "# fetch as pandas dataframe\n", - "subjs_df = subject.Subject.fetch(format='frame')\n", + "subjs_df = subject.Subject.fetch(format=\"frame\")\n", "subjs_df" ] }, @@ -26914,7 +26925,7 @@ ], "source": [ "# fetch the primary key\n", - "pk = subject.Subject.fetch('KEY')\n", + "pk = subject.Subject.fetch(\"KEY\")\n", "pk" ] }, @@ -26925,7 +26936,7 @@ "outputs": [], "source": [ "# fetch specific attributes\n", - "dob, sex = subject.Subject.fetch('dob', 'sex')" + "dob, sex = subject.Subject.fetch(\"dob\", \"sex\")" ] }, { @@ -27213,7 +27224,7 @@ ], "source": [ "# fetch specific attributes as a list of dictionary\n", - "info = subject.Subject.fetch('dob', 'sex', as_dict=True)\n", + "info = subject.Subject.fetch(\"dob\", \"sex\", as_dict=True)\n", "info" ] }, @@ -27230,7 +27241,9 @@ "metadata": {}, "outputs": [], "source": [ - "B205 = (subject.Subject & {'subject_nickname': 'B205'}).fetch1() # \"fetch1()\" because we know there's only one" + "B205 = (\n", + " subject.Subject & {\"subject_nickname\": \"B205\"}\n", + ").fetch1() # \"fetch1()\" because we know there's only one" ] }, { @@ -27318,7 +27331,7 @@ "metadata": {}, "outputs": [], "source": [ - "B205_key = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('KEY')" + "B205_key = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\"KEY\")" ] }, { @@ -27347,7 +27360,9 @@ "metadata": {}, "outputs": [], "source": [ - "B205_init_weight = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('initial_weight')" + "B205_init_weight = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\n", + " \"initial_weight\"\n", + ")" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb b/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb index ebb0167e..4dfc9e6d 100644 --- a/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb +++ b/notebooks/tutorials/Old_tutorials/202001/3-Analyze data with U19 pipeline and save results.ipynb @@ -29,12 +29,14 @@ "import datajoint as dj\n", "import numpy as np\n", "\n", - "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n", - "task = dj.create_virtual_module('task', 'u19_task') \n", - "subject = dj.create_virtual_module('subject', 'u19_subject')\n", - "action = dj.create_virtual_module('action', 'u19_action')\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "lab = dj.create_virtual_module(\n", + " \"lab\", \"u19_lab\"\n", + ") # the first argument here is the __name__ of the virtual module\n", + "task = dj.create_virtual_module(\"task\", \"u19_task\")\n", + "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n", + "action = dj.create_virtual_module(\"action\", \"u19_action\")\n", + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -502,7 +504,7 @@ } ], "source": [ - "behavior.TowersBlock() # bracket necessary" + "behavior.TowersBlock() # bracket necessary" ] }, { @@ -654,8 +656,10 @@ } ], "source": [ - "behavior.TowersSession.aggr(behavior.TowersBlock.proj('block_performance'), \n", - " avg_performance='avg(block_performance)')" + "behavior.TowersSession.aggr(\n", + " behavior.TowersBlock.proj(\"block_performance\"),\n", + " avg_performance=\"avg(block_performance)\",\n", + ")" ] }, { @@ -686,7 +690,7 @@ "metadata": {}, "outputs": [], "source": [ - "schema = dj.schema('shans_tutorial')" + "schema = dj.schema(\"shans_tutorial\")" ] }, { @@ -1050,8 +1054,7 @@ } ], "source": [ - "behavior.TowersSession.aggr(behavior.TowersBlock.proj(),\n", - " n_sessions='count(*)')" + "behavior.TowersSession.aggr(behavior.TowersBlock.proj(), n_sessions=\"count(*)\")" ] }, { @@ -1068,9 +1071,9 @@ "outputs": [], "source": [ "key = {\n", - " 'subject_fullname': 'emanuele_B205',\n", - " 'session_date': datetime.date(2018, 7, 13),\n", - " 'session_number': 0\n", + " \"subject_fullname\": \"emanuele_B205\",\n", + " \"session_date\": datetime.date(2018, 7, 13),\n", + " \"session_number\": 0,\n", "}" ] }, @@ -1080,10 +1083,10 @@ "metadata": {}, "outputs": [], "source": [ - "performances = (behavior.TowersBlock & key).fetch('block_performance')\n", + "performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n", "\n", "# create another field in the dictionary key\n", - "key['avg_performance'] = np.mean(performances)" + "key[\"avg_performance\"] = np.mean(performances)" ] }, { @@ -1122,7 +1125,9 @@ "metadata": {}, "outputs": [], "source": [ - "SessionPerformanceManual.insert1(key, skip_duplicates=True) # insert1 only works for one entry" + "SessionPerformanceManual.insert1(\n", + " key, skip_duplicates=True\n", + ") # insert1 only works for one entry" ] }, { @@ -1341,7 +1346,9 @@ } ], "source": [ - "subject.Subject.aggr(behavior.TowersSession.proj(), n_sessions='count(*)') & 'n_sessions=80'" + "subject.Subject.aggr(\n", + " behavior.TowersSession.proj(), n_sessions=\"count(*)\"\n", + ") & \"n_sessions=80\"" ] }, { @@ -1370,14 +1377,13 @@ "\n", "start_time = time.time()\n", "\n", - "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E78\"').fetch('KEY'):\n", - " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n", + "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E78\"').fetch(\"KEY\"):\n", + " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n", " # create another field in the dictionary key\n", " avg_performance = np.mean(performances)\n", " if np.isnan(avg_performance):\n", " continue\n", - " entry = dict(**i_session, \n", - " avg_performance=avg_performance)\n", + " entry = dict(**i_session, avg_performance=avg_performance)\n", " SessionPerformanceManual.insert1(entry)\n", "\n", "print(\"--- %s seconds ---\" % (time.time() - start_time))" @@ -1401,16 +1407,17 @@ "start_time = time.time()\n", "\n", "perf_entries = []\n", - "for i_session in (behavior.TowersSession & 'subject_fullname=\"lpinto_vg28\"').fetch('KEY'):\n", - " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n", + "for i_session in (behavior.TowersSession & 'subject_fullname=\"lpinto_vg28\"').fetch(\n", + " \"KEY\"\n", + "):\n", + " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n", " # create another field in the dictionary key\n", " avg_performance = np.mean(performances)\n", " if np.isnan(avg_performance):\n", " continue\n", - " entry = dict(**i_session, \n", - " avg_performance=avg_performance)\n", + " entry = dict(**i_session, avg_performance=avg_performance)\n", " perf_entries.append(entry)\n", - " \n", + "\n", "SessionPerformanceManual.insert(perf_entries)\n", "print(\"--- %s seconds ---\" % (time.time() - start_time))" ] @@ -1436,13 +1443,18 @@ " ---\n", " avg_performance: float # a final product in this table\n", " \"\"\"\n", - " key_source = acquisition.Session() & 'subject_fullname=\"lpinto_vg28\"' # bracket necessary\n", - " def make(self, key): # key is one primary key of the entries in table acquisition.Session\n", + " key_source = (\n", + " acquisition.Session() & 'subject_fullname=\"lpinto_vg28\"'\n", + " ) # bracket necessary\n", + "\n", + " def make(\n", + " self, key\n", + " ): # key is one primary key of the entries in table acquisition.Session\n", " # fetch the performance for each block\n", - " performances = (behavior.TowersBlock & key).fetch('block_performance')\n", - " \n", + " performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n", + "\n", " # create another field in the dictionary key\n", - " key['avg_performance'] = np.mean(performances)\n", + " key[\"avg_performance\"] = np.mean(performances)\n", " self.insert1(key)" ] }, @@ -1467,7 +1479,7 @@ } ], "source": [ - "SessionPerformanceComputed.populate(display_progress=True) \n", + "SessionPerformanceComputed.populate(display_progress=True)\n", "# first argument could be some restrictor to control the populate" ] }, @@ -1655,20 +1667,22 @@ " avg_performance: float # a final product in this table\n", " \"\"\"\n", " key_source = subject.Subject()\n", - " def make(self, key): # key is one primary key of the entries in table subject.Subject()!\n", + "\n", + " def make(\n", + " self, key\n", + " ): # key is one primary key of the entries in table subject.Subject()!\n", "\n", " perf_entries = []\n", - " for i_session in (behavior.TowersSession & key).fetch('KEY'):\n", + " for i_session in (behavior.TowersSession & key).fetch(\"KEY\"):\n", " # fetch performance of each block\n", - " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n", + " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n", " # create another field in the dictionary key\n", " avg_performance = np.mean(performances)\n", " if np.isnan(avg_performance):\n", " continue\n", - " entry = dict(**i_session, \n", - " avg_performance=avg_performance)\n", + " entry = dict(**i_session, avg_performance=avg_performance)\n", " perf_entries.append(entry)\n", - " \n", + "\n", " self.insert(perf_entries)" ] }, @@ -9988,7 +10002,9 @@ } ], "source": [ - "SessionPerformanceComputedFromSubject.populate(display_progress=True, suppress_errors=True, reserved_jobs=True)" + "SessionPerformanceComputedFromSubject.populate(\n", + " display_progress=True, suppress_errors=True, reserved_jobs=True\n", + ")" ] }, { @@ -10004,7 +10020,9 @@ "metadata": {}, "outputs": [], "source": [ - "(SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"').delete() # any restrictor would work here" + "(\n", + " SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"'\n", + ").delete() # any restrictor would work here" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202001/subject.py b/notebooks/tutorials/Old_tutorials/202001/subject.py index 13623dde..7e463db7 100644 --- a/notebooks/tutorials/Old_tutorials/202001/subject.py +++ b/notebooks/tutorials/Old_tutorials/202001/subject.py @@ -1,12 +1,11 @@ """This module was auto-generated by datajoint from an existing schema""" - import datajoint as dj -schema = dj.schema('U19_subject') +schema = dj.schema("U19_subject") -vmod0 = dj.create_virtual_module('vmod0', 'U19_lab') +vmod0 = dj.create_virtual_module("vmod0", "U19_lab") @schema @@ -134,7 +133,6 @@ class HealthStatus(dj.Manual): comments=null : varchar(255) """ - class Action(dj.Part): definition = """ -> HealthStatus @@ -252,4 +250,4 @@ class AlleleSequence(dj.Lookup): definition = """ -> Allele -> Sequence - """ \ No newline at end of file + """ diff --git a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb index d67d0814..ae1aae44 100644 --- a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb +++ b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready-Copy1.ipynb @@ -123,7 +123,7 @@ " \"display.show_tuple_count\": true,\n", " \"database.use_tls\": null,\n", " \"enable_python_native_blobs\": false,\n", - " \"database.prefix\": null\n", + " \"database.prefix\": null,\n", "}" ] }, @@ -163,8 +163,8 @@ "metadata": {}, "outputs": [], "source": [ - "dj.config['database.user'] = 'shans'\n", - "dj.config['database.password'] = getpass('Type password:')" + "dj.config[\"database.user\"] = \"shans\"\n", + "dj.config[\"database.password\"] = getpass(\"Type password:\")" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb index 004f4769..507fdccb 100644 --- a/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb +++ b/notebooks/tutorials/Old_tutorials/202103/0-Get DataJoint Ready.ipynb @@ -162,7 +162,7 @@ " \"display.show_tuple_count\": true,\n", " \"database.use_tls\": null,\n", " \"enable_python_native_blobs\": false,\n", - " \"database.prefix\": null\n", + " \"database.prefix\": null,\n", "}" ] }, @@ -202,8 +202,8 @@ "metadata": {}, "outputs": [], "source": [ - "dj.config['database.user'] = 'shans'\n", - "dj.config['database.password'] = getpass('Type password:')" + "dj.config[\"database.user\"] = \"shans\"\n", + "dj.config[\"database.password\"] = getpass(\"Type password:\")" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb b/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb index 5a49aa5f..fdb18ad0 100644 --- a/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb +++ b/notebooks/tutorials/Old_tutorials/202103/1-Explore U19 data pipeline with DataJoint.ipynb @@ -502,12 +502,14 @@ "metadata": {}, "outputs": [], "source": [ - "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n", - "task = dj.create_virtual_module('task', 'u19_task') \n", - "subject = dj.create_virtual_module('subject', 'u19_subject')\n", - "action = dj.create_virtual_module('action', 'u19_action')\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "lab = dj.create_virtual_module(\n", + " \"lab\", \"u19_lab\"\n", + ") # the first argument here is the __name__ of the virtual module\n", + "task = dj.create_virtual_module(\"task\", \"u19_task\")\n", + "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n", + "action = dj.create_virtual_module(\"action\", \"u19_action\")\n", + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -1216,7 +1218,12 @@ } ], "source": [ - "dj.Diagram(subject.Subject) + dj.Diagram(subject.Death) + dj.Diagram(subject.HealthStatus) + dj.Diagram(subject.Weaning)" + "(\n", + " dj.Diagram(subject.Subject)\n", + " + dj.Diagram(subject.Death)\n", + " + dj.Diagram(subject.HealthStatus)\n", + " + dj.Diagram(subject.Weaning)\n", + ")" ] }, { @@ -2118,7 +2125,7 @@ ], "source": [ "# restrict by dictionary\n", - "subject.Subject & {'subject_nickname': 'B205'}" + "subject.Subject & {\"subject_nickname\": \"B205\"}" ] }, { @@ -2412,7 +2419,7 @@ } ], "source": [ - "subject.Subject & {'sex': 'Male'}" + "subject.Subject & {\"sex\": \"Male\"}" ] }, { @@ -2713,7 +2720,7 @@ } ], "source": [ - "subject.Subject & [{'user_id': 'hnieh'}, {'user_id': 'emanuele'}, 'line=\"DAT-IRES-CRE\"']" + "subject.Subject & [{\"user_id\": \"hnieh\"}, {\"user_id\": \"emanuele\"}, 'line=\"DAT-IRES-CRE\"']" ] }, { @@ -6324,7 +6331,7 @@ } ], "source": [ - "(subject.Subject * acquisition.Session).proj('dob', 'sex', 'location')" + "(subject.Subject * acquisition.Session).proj(\"dob\", \"sex\", \"location\")" ] }, { @@ -6460,7 +6467,7 @@ } ], "source": [ - "subject.Subject.proj(gender='sex', birth_date='dob') # 'sex->gender'" + "subject.Subject.proj(gender=\"sex\", birth_date=\"dob\") # 'sex->gender'" ] }, { @@ -6483,7 +6490,9 @@ "metadata": {}, "outputs": [], "source": [ - "weighing_with_date = action.Weighing.proj(weighing_date='date(weighing_time)') # more options, check MySQL syntax" + "weighing_with_date = action.Weighing.proj(\n", + " weighing_date=\"date(weighing_time)\"\n", + ") # more options, check MySQL syntax" ] }, { @@ -6765,7 +6774,7 @@ "source": [ "# First get the date of birth and the session date into the same query\n", "q = subject.Subject * acquisition.Session\n", - "q = q.proj('dob')\n", + "q = q.proj(\"dob\")\n", "q" ] }, @@ -6926,7 +6935,7 @@ ], "source": [ "# Then compute the age\n", - "q_with_age = q.proj('dob', age='datediff(session_date, dob)') & 'dob is not NULL'\n", + "q_with_age = q.proj(\"dob\", age=\"datediff(session_date, dob)\") & \"dob is not NULL\"\n", "q_with_age" ] }, @@ -7070,7 +7079,9 @@ } ], "source": [ - "subject.Subject.aggr(acquisition.Session, n='count(*)', lastest_session_date='max(session_date)')" + "subject.Subject.aggr(\n", + " acquisition.Session, n=\"count(*)\", lastest_session_date=\"max(session_date)\"\n", + ")" ] }, { @@ -25971,7 +25982,7 @@ } ], "source": [ - "subjs['subject_fullname']" + "subjs[\"subject_fullname\"]" ] }, { @@ -26202,7 +26213,7 @@ } ], "source": [ - "subjs['dob']" + "subjs[\"dob\"]" ] }, { @@ -52432,7 +52443,7 @@ ], "source": [ "# fetch as pandas dataframe\n", - "subjs_df = subject.Subject.fetch(format='frame').reset_index()\n", + "subjs_df = subject.Subject.fetch(format=\"frame\").reset_index()\n", "subjs_df" ] }, @@ -53011,7 +53022,7 @@ ], "source": [ "# fetch the primary key\n", - "pk = subject.Subject.fetch('KEY')\n", + "pk = subject.Subject.fetch(\"KEY\")\n", "pk" ] }, @@ -53022,7 +53033,7 @@ "outputs": [], "source": [ "# fetch specific attributes\n", - "dob, sex = subject.Subject.fetch('dob', 'sex')" + "dob, sex = subject.Subject.fetch(\"dob\", \"sex\")" ] }, { @@ -53831,7 +53842,7 @@ ], "source": [ "# fetch specific attributes as a list of dictionary\n", - "info = subject.Subject.fetch('dob', 'sex', as_dict=True)\n", + "info = subject.Subject.fetch(\"dob\", \"sex\", as_dict=True)\n", "info" ] }, @@ -53848,7 +53859,9 @@ "metadata": {}, "outputs": [], "source": [ - "B205 = (subject.Subject & {'subject_nickname': 'B205'}).fetch1() # \"fetch1()\" because we know there's only one" + "B205 = (\n", + " subject.Subject & {\"subject_nickname\": \"B205\"}\n", + ").fetch1() # \"fetch1()\" because we know there's only one" ] }, { @@ -53938,7 +53951,7 @@ "metadata": {}, "outputs": [], "source": [ - "B205_key = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('KEY')" + "B205_key = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\"KEY\")" ] }, { @@ -53967,7 +53980,9 @@ "metadata": {}, "outputs": [], "source": [ - "B205_init_weight = (subject.Subject & {'subject_nickname': 'B205'}).fetch1('initial_weight')" + "B205_init_weight = (subject.Subject & {\"subject_nickname\": \"B205\"}).fetch1(\n", + " \"initial_weight\"\n", + ")" ] }, { @@ -54007,7 +54022,7 @@ } ], "source": [ - "(subject.Subject & {'subject_nickname': 'B205'}).fetch('initial_weight')" + "(subject.Subject & {\"subject_nickname\": \"B205\"}).fetch(\"initial_weight\")" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb b/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb index e615f596..3b157f20 100644 --- a/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb +++ b/notebooks/tutorials/Old_tutorials/202103/2-Analyze data with U19 pipeline and save results.ipynb @@ -33,12 +33,14 @@ "import datajoint as dj\n", "import numpy as np\n", "\n", - "lab = dj.create_virtual_module('lab', 'u19_lab') # the first argument here is the __name__ of the virtual module\n", - "task = dj.create_virtual_module('task', 'u19_task') \n", - "subject = dj.create_virtual_module('subject', 'u19_subject')\n", - "action = dj.create_virtual_module('action', 'u19_action')\n", - "acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition')\n", - "behavior = dj.create_virtual_module('behavior', 'u19_behavior')" + "lab = dj.create_virtual_module(\n", + " \"lab\", \"u19_lab\"\n", + ") # the first argument here is the __name__ of the virtual module\n", + "task = dj.create_virtual_module(\"task\", \"u19_task\")\n", + "subject = dj.create_virtual_module(\"subject\", \"u19_subject\")\n", + "action = dj.create_virtual_module(\"action\", \"u19_action\")\n", + "acquisition = dj.create_virtual_module(\"acquisition\", \"u19_acquisition\")\n", + "behavior = dj.create_virtual_module(\"behavior\", \"u19_behavior\")" ] }, { @@ -692,7 +694,7 @@ } ], "source": [ - "behavior.TowersBlock() # bracket necessary" + "behavior.TowersBlock() # bracket necessary" ] }, { @@ -843,8 +845,10 @@ } ], "source": [ - "behavior.TowersSession.aggr(behavior.TowersBlock.proj('block_performance'), \n", - " avg_performance='avg(block_performance)')" + "behavior.TowersSession.aggr(\n", + " behavior.TowersBlock.proj(\"block_performance\"),\n", + " avg_performance=\"avg(block_performance)\",\n", + ")" ] }, { @@ -875,7 +879,7 @@ "metadata": {}, "outputs": [], "source": [ - "schema = dj.schema('alvaros_tutorial2021')" + "schema = dj.schema(\"alvaros_tutorial2021\")" ] }, { @@ -1289,8 +1293,7 @@ } ], "source": [ - "behavior.TowersSession.aggr(behavior.TowersBlock.proj(),\n", - " n_sessions='count(*)')" + "behavior.TowersSession.aggr(behavior.TowersBlock.proj(), n_sessions=\"count(*)\")" ] }, { @@ -1307,9 +1310,9 @@ "outputs": [], "source": [ "key = {\n", - " 'subject_fullname': 'emanuele_B205',\n", - " 'session_date': datetime.date(2018, 7, 13),\n", - " 'session_number': 0\n", + " \"subject_fullname\": \"emanuele_B205\",\n", + " \"session_date\": datetime.date(2018, 7, 13),\n", + " \"session_number\": 0,\n", "}" ] }, @@ -1562,10 +1565,10 @@ "metadata": {}, "outputs": [], "source": [ - "performances = (behavior.TowersBlock & key).fetch('block_performance')\n", + "performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n", "\n", "# create another field in the dictionary key\n", - "key['avg_performance'] = np.mean(performances)" + "key[\"avg_performance\"] = np.mean(performances)" ] }, { @@ -1604,7 +1607,9 @@ "metadata": {}, "outputs": [], "source": [ - "SessionPerformanceManual.insert1(key, skip_duplicates=True) # insert1 only works for one entry" + "SessionPerformanceManual.insert1(\n", + " key, skip_duplicates=True\n", + ") # insert1 only works for one entry" ] }, { @@ -1821,7 +1826,9 @@ } ], "source": [ - "subject.Subject.aggr(behavior.TowersSession.proj(), n_sessions='count(*)') & 'n_sessions=100'" + "subject.Subject.aggr(\n", + " behavior.TowersSession.proj(), n_sessions=\"count(*)\"\n", + ") & \"n_sessions=100\"" ] }, { @@ -1850,14 +1857,13 @@ "\n", "start_time = time.time()\n", "\n", - "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E57\"').fetch('KEY'):\n", - " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n", + "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E57\"').fetch(\"KEY\"):\n", + " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n", " # create another field in the dictionary key\n", " avg_performance = np.mean(performances)\n", " if np.isnan(avg_performance):\n", " continue\n", - " entry = dict(**i_session, \n", - " avg_performance=avg_performance)\n", + " entry = dict(**i_session, avg_performance=avg_performance)\n", " SessionPerformanceManual.insert1(entry)\n", "\n", "print(\"--- %s seconds ---\" % (time.time() - start_time))" @@ -1881,16 +1887,15 @@ "start_time = time.time()\n", "\n", "perf_entries = []\n", - "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E77\"').fetch('KEY'):\n", - " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n", + "for i_session in (behavior.TowersSession & 'subject_fullname=\"hnieh_E77\"').fetch(\"KEY\"):\n", + " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n", " # create another field in the dictionary key\n", " avg_performance = np.mean(performances)\n", " if np.isnan(avg_performance):\n", " continue\n", - " entry = dict(**i_session, \n", - " avg_performance=avg_performance)\n", + " entry = dict(**i_session, avg_performance=avg_performance)\n", " perf_entries.append(entry)\n", - " \n", + "\n", "SessionPerformanceManual.insert(perf_entries)\n", "print(\"--- %s seconds ---\" % (time.time() - start_time))" ] @@ -1916,14 +1921,19 @@ " ---\n", " avg_performance: float # a final product in this table\n", " \"\"\"\n", - " \n", - " key_source = acquisition.Session() & 'subject_fullname=\"hnieh_E57\"' # bracket necessary\n", - " def make(self, key): # key is one primary key of the entries in table acquisition.Session\n", + "\n", + " key_source = (\n", + " acquisition.Session() & 'subject_fullname=\"hnieh_E57\"'\n", + " ) # bracket necessary\n", + "\n", + " def make(\n", + " self, key\n", + " ): # key is one primary key of the entries in table acquisition.Session\n", " # fetch the performance for each block\n", - " performances = (behavior.TowersBlock & key).fetch('block_performance')\n", - " \n", + " performances = (behavior.TowersBlock & key).fetch(\"block_performance\")\n", + "\n", " # create another field in the dictionary key\n", - " key['avg_performance'] = np.mean(performances)\n", + " key[\"avg_performance\"] = np.mean(performances)\n", " self.insert1(key)" ] }, @@ -1948,7 +1958,7 @@ } ], "source": [ - "SessionPerformanceComputed.populate(display_progress=True) \n", + "SessionPerformanceComputed.populate(display_progress=True)\n", "# first argument could be some restrictor to control the populate" ] }, @@ -2134,21 +2144,23 @@ " ---\n", " avg_performance: float # a final product in this table\n", " \"\"\"\n", - " key_source = subject.Subject() \n", - " def make(self, key): # key is one primary key of the entries in table subject.Subject()!\n", + " key_source = subject.Subject()\n", + "\n", + " def make(\n", + " self, key\n", + " ): # key is one primary key of the entries in table subject.Subject()!\n", "\n", " perf_entries = []\n", - " for i_session in (behavior.TowersSession & key).fetch('KEY'):\n", + " for i_session in (behavior.TowersSession & key).fetch(\"KEY\"):\n", " # fetch performance of each block\n", - " performances = (behavior.TowersBlock & i_session).fetch('block_performance')\n", + " performances = (behavior.TowersBlock & i_session).fetch(\"block_performance\")\n", " # create another field in the dictionary key\n", " avg_performance = np.mean(performances)\n", " if np.isnan(avg_performance):\n", " continue\n", - " entry = dict(**i_session, \n", - " avg_performance=avg_performance)\n", + " entry = dict(**i_session, avg_performance=avg_performance)\n", " perf_entries.append(entry)\n", - " \n", + "\n", " self.insert(perf_entries)" ] }, @@ -2176,7 +2188,12 @@ } ], "source": [ - "SessionPerformanceComputedFromSubject.populate('subject_fullname=\"hnieh_E77\"', display_progress=True, suppress_errors=True, reserve_jobs=True)" + "SessionPerformanceComputedFromSubject.populate(\n", + " 'subject_fullname=\"hnieh_E77\"',\n", + " display_progress=True,\n", + " suppress_errors=True,\n", + " reserve_jobs=True,\n", + ")" ] }, { @@ -2211,7 +2228,9 @@ } ], "source": [ - "(SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"').delete() # any restrictor would work here" + "(\n", + " SessionPerformanceManual & 'subject_fullname=\"hnieh_E78\"'\n", + ").delete() # any restrictor would work here" ] }, { diff --git a/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb b/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb index 235c71ac..a3eabe31 100644 --- a/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb +++ b/notebooks/tutorials/Old_tutorials/202103/3-Build a simple data pipeline.ipynb @@ -235,7 +235,7 @@ ], "source": [ "# create your own schema object\n", - "schema = dj.schema('alvaros_tutorial2021')" + "schema = dj.schema(\"alvaros_tutorial2021\")" ] }, { @@ -400,7 +400,7 @@ "outputs": [], "source": [ "# inserting the first mouse\n", - "Mouse.insert1((0, '2017-03-01', 'M'))" + "Mouse.insert1((0, \"2017-03-01\", \"M\"))" ] }, { @@ -513,7 +513,7 @@ "metadata": {}, "outputs": [], "source": [ - "Mouse.insert1((20, datetime.date(2020, 5, 3), 'F'))" + "Mouse.insert1((20, datetime.date(2020, 5, 3), \"F\"))" ] }, { @@ -627,11 +627,7 @@ "metadata": {}, "outputs": [], "source": [ - "data = {\n", - " 'mouse_id': 100,\n", - " 'dob': '2017-05-12',\n", - " 'sex': 'F'\n", - "}" + "data = {\"mouse_id\": 100, \"dob\": \"2017-05-12\", \"sex\": \"F\"}" ] }, { @@ -758,11 +754,7 @@ "metadata": {}, "outputs": [], "source": [ - "data = [\n", - " (1, '2016-11-19', 'M'),\n", - " (2, '2016-11-20', 'unknown'),\n", - " (5, '2016-12-25', 'F')\n", - "]" + "data = [(1, \"2016-11-19\", \"M\"), (2, \"2016-11-20\", \"unknown\"), (5, \"2016-12-25\", \"F\")]" ] }, { @@ -899,8 +891,8 @@ "outputs": [], "source": [ "data = [\n", - " {'mouse_id': 10, 'dob': '2017-01-01', 'sex': 'F'},\n", - " {'mouse_id': 11, 'dob': '2017-01-03', 'sex': 'F'},\n", + " {\"mouse_id\": 10, \"dob\": \"2017-01-01\", \"sex\": \"F\"},\n", + " {\"mouse_id\": 11, \"dob\": \"2017-01-03\", \"sex\": \"F\"},\n", "]\n", "\n", "# insert them all\n", @@ -1058,9 +1050,10 @@ ], "source": [ "Mouse.insert1(\n", - " {'mouse_id': 0,\n", - " 'dob': '2018-01-01',\n", - " 'sex': 'M',\n", + " {\n", + " \"mouse_id\": 0,\n", + " \"dob\": \"2018-01-01\",\n", + " \"sex\": \"M\",\n", " }\n", ")" ] @@ -1107,7 +1100,7 @@ } ], "source": [ - "(Mouse & 'mouse_id=11').delete()" + "(Mouse & \"mouse_id=11\").delete()" ] }, { @@ -1241,7 +1234,7 @@ } ], "source": [ - "(Mouse & 'mouse_id=0')._update('sex', 'M')" + "(Mouse & \"mouse_id=0\")._update(\"sex\", \"M\")" ] }, { @@ -1533,10 +1526,10 @@ "outputs": [], "source": [ "data = {\n", - " 'mouse_id': 0,\n", - " 'session_date': '2017-05-15',\n", - " 'experiment_setup': 0,\n", - " 'experimenter': 'Edgar Y. Walker'\n", + " \"mouse_id\": 0,\n", + " \"session_date\": \"2017-05-15\",\n", + " \"experiment_setup\": 0,\n", + " \"experimenter\": \"Edgar Y. Walker\",\n", "}\n", "\n", "Session.insert1(data)" @@ -1749,10 +1742,10 @@ ], "source": [ "data = {\n", - " 'mouse_id': 0,\n", - " 'session_date': '2018-01-15',\n", - " 'experiment_setup': 100,\n", - " 'experimenter': 'Jacob Reimer'\n", + " \"mouse_id\": 0,\n", + " \"session_date\": \"2018-01-15\",\n", + " \"experiment_setup\": 100,\n", + " \"experimenter\": \"Jacob Reimer\",\n", "}\n", "\n", "# insert them all\n", @@ -1775,10 +1768,10 @@ "outputs": [], "source": [ "data = {\n", - " 'mouse_id': 1,\n", - " 'session_date': '2018-01-15',\n", - " 'experiment_setup': 101,\n", - " 'experimenter': 'Jacob Reimer'\n", + " \"mouse_id\": 1,\n", + " \"session_date\": \"2018-01-15\",\n", + " \"experiment_setup\": 101,\n", + " \"experimenter\": \"Jacob Reimer\",\n", "}\n", "\n", "# insert them all\n", @@ -1906,10 +1899,10 @@ "outputs": [], "source": [ "bad_data = {\n", - " 'mouse_id': 9999, # this mouse doesn't exist!\n", - " 'session_date': '2017-05-15',\n", - " 'experiment_setup': 0,\n", - " 'experimenter': 'Edgar Y. Walker'\n", + " \"mouse_id\": 9999, # this mouse doesn't exist!\n", + " \"session_date\": \"2017-05-15\",\n", + " \"experiment_setup\": 0,\n", + " \"experimenter\": \"Edgar Y. Walker\",\n", "}" ] }, @@ -1987,7 +1980,7 @@ } ], "source": [ - "(Mouse & 'mouse_id = 100').delete()" + "(Mouse & \"mouse_id = 100\").delete()" ] }, { diff --git a/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb b/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb index c633247f..61866e32 100644 --- a/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb +++ b/notebooks/tutorials/lightsheet_202010/lightsheet_datajoint_tutorial (Python).ipynb @@ -42,7 +42,7 @@ "import datajoint as dj\n", "import matplotlib\n", "\n", - "matplotlib.style.use('ggplot')\n", + "matplotlib.style.use(\"ggplot\")\n", "%matplotlib inline" ] }, @@ -63,7 +63,7 @@ "metadata": {}, "outputs": [], "source": [ - "netid = 'ahoag' # change this to your netid " + "netid = \"ahoag\" # change this to your netid" ] }, { @@ -91,8 +91,8 @@ } ], "source": [ - "dj.config['database.host'] = 'datajoint00.pni.princeton.edu'\n", - "dj.config['database.user'] = netid\n", + "dj.config[\"database.host\"] = \"datajoint00.pni.princeton.edu\"\n", + "dj.config[\"database.user\"] = netid\n", "dj.conn()" ] }, @@ -109,7 +109,9 @@ "metadata": {}, "outputs": [], "source": [ - "db_lightsheet = dj.create_virtual_module('u19lightserv_lightsheet','u19lightserv_lightsheet')" + "db_lightsheet = dj.create_virtual_module(\n", + " \"u19lightserv_lightsheet\", \"u19lightserv_lightsheet\"\n", + ")" ] }, { @@ -312,7 +314,7 @@ ], "source": [ "# Let's take a look at the schema design to get an overview of the tables and their relationships\n", - "dj.ERD(db_lightsheet.User) + 1 " + "dj.ERD(db_lightsheet.User) + 1" ] }, { @@ -596,8 +598,10 @@ } ], "source": [ - "username = 'jverpeut' # change this if you want to look up requests from a different netid\n", - "db_lightsheet.Request() & {'username':username} " + "username = (\n", + " \"jverpeut\" # change this if you want to look up requests from a different netid\n", + ")\n", + "db_lightsheet.Request() & {\"username\": username}" ] }, { @@ -730,8 +734,8 @@ } ], "source": [ - "request_name='natneuroreviews_tompisano_CTB' # change this if you want to look at a different request\n", - "db_lightsheet.Request.Sample() & {'username':username,'request_name':request_name}\n", + "request_name = \"natneuroreviews_tompisano_CTB\" # change this if you want to look at a different request\n", + "db_lightsheet.Request.Sample() & {\"username\": username, \"request_name\": request_name}\n", "# notice the syntax used. Sample is a part table of Request, hence the Request.Sample() syntax" ] }, @@ -889,7 +893,10 @@ } ], "source": [ - "db_lightsheet.Request.ClearingBatch() & {'username':username,'request_name':request_name}" + "db_lightsheet.Request.ClearingBatch() & {\n", + " \"username\": username,\n", + " \"request_name\": request_name,\n", + "}" ] }, { @@ -1978,7 +1985,10 @@ } ], "source": [ - "db_lightsheet.Request.IdiscoPlusClearing & {'username':username,'request_name':request_name}" + "db_lightsheet.Request.IdiscoPlusClearing & {\n", + " \"username\": username,\n", + " \"request_name\": request_name,\n", + "}" ] }, { @@ -3272,7 +3282,10 @@ } ], "source": [ - "db_lightsheet.Request.ImagingRequest() & {'username':username,'request_name':request_name}" + "db_lightsheet.Request.ImagingRequest() & {\n", + " \"username\": username,\n", + " \"request_name\": request_name,\n", + "}" ] }, { @@ -3407,7 +3420,10 @@ } ], "source": [ - "db_lightsheet.Request.ImagingResolutionRequest() & {'username':username,'request_name':request_name}" + "db_lightsheet.Request.ImagingResolutionRequest() & {\n", + " \"username\": username,\n", + " \"request_name\": request_name,\n", + "}" ] }, { @@ -3436,9 +3452,11 @@ } ], "source": [ - "request_contents = db_lightsheet.Request.ImagingResolutionRequest() & \\\n", - " {'username':'ejdennis','request_name':'201905_atlas00x_where_x=1:n'}\n", - "request_contents.fetch1('notes_from_imaging')" + "request_contents = db_lightsheet.Request.ImagingResolutionRequest() & {\n", + " \"username\": \"ejdennis\",\n", + " \"request_name\": \"201905_atlas00x_where_x=1:n\",\n", + "}\n", + "request_contents.fetch1(\"notes_from_imaging\")" ] }, { @@ -3725,7 +3743,10 @@ } ], "source": [ - "db_lightsheet.Request.ImagingChannel() & {'username':username,'request_name':request_name}" + "db_lightsheet.Request.ImagingChannel() & {\n", + " \"username\": username,\n", + " \"request_name\": request_name,\n", + "}" ] }, { @@ -4120,7 +4141,10 @@ } ], "source": [ - "db_lightsheet.Request.ProcessingRequest() & {'username':username,'request_name':request_name}" + "db_lightsheet.Request.ProcessingRequest() & {\n", + " \"username\": username,\n", + " \"request_name\": request_name,\n", + "}" ] }, { @@ -4280,7 +4304,10 @@ } ], "source": [ - "db_lightsheet.Request.ProcessingResolutionRequest() & {'username':username,'request_name':request_name}" + "db_lightsheet.Request.ProcessingResolutionRequest() & {\n", + " \"username\": username,\n", + " \"request_name\": request_name,\n", + "}" ] }, { @@ -4443,7 +4470,10 @@ } ], "source": [ - "db_lightsheet.Request.ProcessingChannel() & {'username':username,'request_name':request_name}" + "db_lightsheet.Request.ProcessingChannel() & {\n", + " \"username\": username,\n", + " \"request_name\": request_name,\n", + "}" ] }, { @@ -4504,14 +4534,18 @@ } ], "source": [ - "username1 = 'ejdennis' # change as needed\n", - "request_name1 = '201905_atlas00x_where_x=1:n' # change as needed\n", + "username1 = \"ejdennis\" # change as needed\n", + "request_name1 = \"201905_atlas00x_where_x=1:n\" # change as needed\n", "# Let's check the archival column of this request\n", - "request_contents = db_lightsheet.Request() & \\\n", - " {'username':username1,'request_name':request_name1}\n", - "print(\"is_archival column = \",request_contents.fetch1('is_archival'))\n", - "contents = db_lightsheet.Request.ImagingResolutionRequest() & \\\n", - " {'username':username1,'request_name':request_name1}\n", + "request_contents = db_lightsheet.Request() & {\n", + " \"username\": username1,\n", + " \"request_name\": request_name1,\n", + "}\n", + "print(\"is_archival column = \", request_contents.fetch1(\"is_archival\"))\n", + "contents = db_lightsheet.Request.ImagingResolutionRequest() & {\n", + " \"username\": username1,\n", + " \"request_name\": request_name1,\n", + "}\n", "contents.fetch(as_dict=True)" ] }, @@ -4548,24 +4582,42 @@ } ], "source": [ - "username2 = 'ejdennis' # change as needed\n", - "request_name2 = 'three_female_atlas_brains' # change as needed\n", - "sample_contents = db_lightsheet.Request.Sample() & \\\n", - " {'username':username2,'request_name':request_name2}\n", - "print(f\"Have {len(sample_contents)} samples in this request:\\n {sample_contents.fetch('sample_name')}\")\n", - "clearing_batch_contents = db_lightsheet.Request.ClearingBatch() & \\\n", - " {'username':username2,'request_name':request_name2}\n", - "print(f\"Have {len(clearing_batch_contents)} clearing batches in this request, each cleared by: {clearing_batch_contents.fetch('clearer')}\")\n", - "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & \\\n", - " {'username':username2,'request_name':request_name2}\n", - "imagers = imaging_request_contents.fetch('imager')\n", - "print(f\"Samples each imaged by: {imagers} \",)\n", - "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & \\\n", - " {'username':username2,'request_name':request_name2}\n", - "processing_request_contents = db_lightsheet.Request.ProcessingRequest() & \\\n", - " {'username':username2,'request_name':request_name2}\n", - "processors = processing_request_contents.fetch('processor')\n", - "print(f\"Samples each processed by: {processors} \",)" + "username2 = \"ejdennis\" # change as needed\n", + "request_name2 = \"three_female_atlas_brains\" # change as needed\n", + "sample_contents = db_lightsheet.Request.Sample() & {\n", + " \"username\": username2,\n", + " \"request_name\": request_name2,\n", + "}\n", + "print(\n", + " f\"Have {len(sample_contents)} samples in this request:\\n {sample_contents.fetch('sample_name')}\"\n", + ")\n", + "clearing_batch_contents = db_lightsheet.Request.ClearingBatch() & {\n", + " \"username\": username2,\n", + " \"request_name\": request_name2,\n", + "}\n", + "print(\n", + " f\"Have {len(clearing_batch_contents)} clearing batches in this request, each cleared by: {clearing_batch_contents.fetch('clearer')}\"\n", + ")\n", + "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & {\n", + " \"username\": username2,\n", + " \"request_name\": request_name2,\n", + "}\n", + "imagers = imaging_request_contents.fetch(\"imager\")\n", + "print(\n", + " f\"Samples each imaged by: {imagers} \",\n", + ")\n", + "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & {\n", + " \"username\": username2,\n", + " \"request_name\": request_name2,\n", + "}\n", + "processing_request_contents = db_lightsheet.Request.ProcessingRequest() & {\n", + " \"username\": username2,\n", + " \"request_name\": request_name2,\n", + "}\n", + "processors = processing_request_contents.fetch(\"processor\")\n", + "print(\n", + " f\"Samples each processed by: {processors} \",\n", + ")" ] }, { @@ -4595,22 +4647,32 @@ } ], "source": [ - "username3 = 'ejdennis' # change as needed\n", - "request_name3 = 'three_female_atlas_brains' # change as needed\n", - "sample_contents = db_lightsheet.Request.Sample() & \\\n", - " {'username':username3,'request_name':request_name3}\n", - "print(f\"Have {len(sample_contents)} samples in this request:\\n {sample_contents.fetch('sample_name')}\\n\")\n", - "clearing_batch_contents = db_lightsheet.Request.ClearingBatch() & \\\n", - " {'username':username3,'request_name':request_name3}\n", - "clearing_progress = clearing_batch_contents.fetch1('clearing_progress')\n", - "print(\"Clearing progress is: \",clearing_progress)\n", - "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & \\\n", - " {'username':username3,'request_name':request_name3}\n", - "imaging_progress = imaging_request_contents.fetch('imaging_progress')\n", + "username3 = \"ejdennis\" # change as needed\n", + "request_name3 = \"three_female_atlas_brains\" # change as needed\n", + "sample_contents = db_lightsheet.Request.Sample() & {\n", + " \"username\": username3,\n", + " \"request_name\": request_name3,\n", + "}\n", + "print(\n", + " f\"Have {len(sample_contents)} samples in this request:\\n {sample_contents.fetch('sample_name')}\\n\"\n", + ")\n", + "clearing_batch_contents = db_lightsheet.Request.ClearingBatch() & {\n", + " \"username\": username3,\n", + " \"request_name\": request_name3,\n", + "}\n", + "clearing_progress = clearing_batch_contents.fetch1(\"clearing_progress\")\n", + "print(\"Clearing progress is: \", clearing_progress)\n", + "imaging_request_contents = db_lightsheet.Request.ImagingRequest() & {\n", + " \"username\": username3,\n", + " \"request_name\": request_name3,\n", + "}\n", + "imaging_progress = imaging_request_contents.fetch(\"imaging_progress\")\n", "print(f\"Imaging progress is: {imaging_progress}\")\n", - "processing_request_contents = db_lightsheet.Request.ProcessingRequest() & \\\n", - " {'username':username3,'request_name':request_name3}\n", - "processing_progress = processing_request_contents.fetch('processing_progress')\n", + "processing_request_contents = db_lightsheet.Request.ProcessingRequest() & {\n", + " \"username\": username3,\n", + " \"request_name\": request_name3,\n", + "}\n", + "processing_progress = processing_request_contents.fetch(\"processing_progress\")\n", "print(f\"Processing progress is: {processing_progress}\")" ] }, @@ -4645,16 +4707,23 @@ } ], "source": [ - "username4 = 'ejdennis' # change as needed\n", - "request_name4 = 'three_female_atlas_brains' # change as needed\n", - "sample_contents = db_lightsheet.Request.Sample() & \\\n", - " {'username':username4,'request_name':request_name4}\n", + "username4 = \"ejdennis\" # change as needed\n", + "request_name4 = \"three_female_atlas_brains\" # change as needed\n", + "sample_contents = db_lightsheet.Request.Sample() & {\n", + " \"username\": username4,\n", + " \"request_name\": request_name4,\n", + "}\n", "print(f\"Have {len(sample_contents)} samples in this request\\n\")\n", "for sample_dict in sample_contents:\n", - " sample_name = sample_dict['sample_name']\n", - " imaging_channel_contents = db_lightsheet.Request.ImagingChannel() & \\\n", - " {'username':username4,'request_name':request_name4,'sample_name':sample_name}\n", - " image_resolutions,channel_names = imaging_channel_contents.fetch('image_resolution','channel_name')\n", + " sample_name = sample_dict[\"sample_name\"]\n", + " imaging_channel_contents = db_lightsheet.Request.ImagingChannel() & {\n", + " \"username\": username4,\n", + " \"request_name\": request_name4,\n", + " \"sample_name\": sample_name,\n", + " }\n", + " image_resolutions, channel_names = imaging_channel_contents.fetch(\n", + " \"image_resolution\", \"channel_name\"\n", + " )\n", " print(f\"Sample: {sample_name}:\")\n", " for ii in range(len(image_resolutions)):\n", " image_resolution = image_resolutions[ii]\n", diff --git a/notebooks/water_weight_alert.ipynb b/notebooks/water_weight_alert.ipynb index 49478118..015eb8c9 100644 --- a/notebooks/water_weight_alert.ipynb +++ b/notebooks/water_weight_alert.ipynb @@ -48,7 +48,7 @@ "import datajoint as dj\n", "import pandas as pd\n", "\n", - "import u19_pipeline.utils.slack_utils as su\n" + "import u19_pipeline.utils.slack_utils as su" ] }, { @@ -59,95 +59,132 @@ "source": [ "def get_subject_data():\n", "\n", - " #Query from file\n", + " # Query from file\n", " current_directory = pathlib.Path.cwd()\n", - " query_file = pathlib.Path(current_directory, 'useful_queries', 'get_subject_data.sql').as_posix()\n", + " query_file = pathlib.Path(\n", + " current_directory, \"useful_queries\", \"get_subject_data.sql\"\n", + " ).as_posix()\n", "\n", " with open(query_file, encoding=\"utf-8\") as file:\n", - "\n", " subject_query = file.read()\n", "\n", " conn = dj.conn()\n", " subject_data = pd.DataFrame(conn.query(subject_query, as_dict=True).fetchall())\n", "\n", - "\n", - "\n", - " #Get subject fullname only once\n", - " subject_data = subject_data.loc[:,~subject_data.columns.duplicated()]\n", - " #subject_data = subject_data.rename({'new_subject_fullname': 'subject_fullname'}, axis=1)\n", + " # Get subject fullname only once\n", + " subject_data = subject_data.loc[:, ~subject_data.columns.duplicated()]\n", + " # subject_data = subject_data.rename({'new_subject_fullname': 'subject_fullname'}, axis=1)\n", "\n", " # Get column list\n", " cols = subject_data.columns.tolist()\n", "\n", " # Remove and insert the column\n", - " cols.insert(0, cols.pop(cols.index('subject_fullname')))\n", + " cols.insert(0, cols.pop(cols.index(\"subject_fullname\")))\n", " subject_data = subject_data.reindex(columns=cols)\n", "\n", - "\n", - " #Get today's responsibility based on schedule \"Nothing/Nothing etc\" field\n", - " today_idx = (datetime.datetime.today().weekday() + 1) % 7\n", - " subject_data['schedule_today'] = subject_data['schedule'].str.split('/')\n", - " subject_data['schedule_today'] = subject_data['schedule_today'].apply(lambda x: x[today_idx])\n", - "\n", - "\n", - "\n", - " subject_data = subject_data.loc[(((subject_data['schedule_today'] != \"Nothing\") &\\\n", - " (subject_data['subject_status'] == \"InExperiments\")) |\\\n", - " (subject_data['subject_status'] == \"WaterRestrictionOnly\")), :]\n", + " # Get today's responsibility based on schedule \"Nothing/Nothing etc\" field\n", + " today_idx = (datetime.datetime.today().weekday() + 1) % 7\n", + " subject_data[\"schedule_today\"] = subject_data[\"schedule\"].str.split(\"/\")\n", + " subject_data[\"schedule_today\"] = subject_data[\"schedule_today\"].apply(\n", + " lambda x: x[today_idx]\n", + " )\n", + "\n", + " subject_data = subject_data.loc[\n", + " (\n", + " (\n", + " (subject_data[\"schedule_today\"] != \"Nothing\")\n", + " & (subject_data[\"subject_status\"] == \"InExperiments\")\n", + " )\n", + " | (subject_data[\"subject_status\"] == \"WaterRestrictionOnly\")\n", + " ),\n", + " :,\n", + " ]\n", "\n", " subject_data = subject_data.reset_index(drop=True)\n", "\n", - "\n", - "\n", " with pd.option_context(\"future.no_silent_downcasting\", True):\n", - "\n", - " subject_data['earned'] = subject_data['earned'].fillna(0).infer_objects(copy=False)\n", - " subject_data['received'] = subject_data['received'].fillna(0).infer_objects(copy=False)\n", - " subject_data['supplement'] = subject_data['supplement'].fillna(0).infer_objects(copy=False)\n", - " subject_data['prescribed_extra_supplement_amount'] = subject_data['prescribed_extra_supplement_amount'].fillna(0).infer_objects(copy=False)\n", - "\n", - "\n", - " #Calculated fields \n", - " subject_data['already_water'] = subject_data['supplement']>0\n", - " subject_data['upper_cage'] = subject_data['cage'].str.upper()\n", - " subject_data['total_water_received'] = subject_data['received']\n", - " subject_data.loc[subject_data['total_water_received'].isnull(), 'total_water_received'] = 0\n", - "\n", - " subject_data['need_supplement'] = 0\n", - " subject_data['need_supplement'] = (subject_data['total_water_received'] < (subject_data['water_per_day'] -0.05)) & ~subject_data['already_water']\n", - "\n", - " subject_data.loc[subject_data['already_received'].isnull(), 'already_received'] = 0\n", - " subject_data['need_extra_water_now'] = 0\n", - " subject_data.loc[((subject_data['prescribed_extra_supplement_amount'] > 0) & (subject_data['already_received'] == 0)),'need_extra_water_now'] = 1\n", - "\n", - " subject_data['water_status'] = \"Already Watered\"\n", - " subject_data.loc[subject_data['need_supplement'] == 1, 'water_status'] = \"Need Supplement\"\n", - " subject_data.loc[subject_data['need_extra_water_now'] == 1, 'water_status'] = \"Need Extra Supplement\"\n", - "\n", - " subject_data['need_water'] = 0\n", - " subject_data.loc[(subject_data['need_supplement'] | subject_data['need_extra_water_now']), 'need_water'] = 1\n", - "\n", - " subject_data['current_need_water'] = subject_data['suggested_water']\n", - " subject_data.loc[subject_data['need_extra_water_now']==1, 'current_need_water'] =\\\n", - " subject_data.loc[subject_data['need_extra_water_now']==1, 'prescribed_extra_supplement_amount']\n", - "\n", - "\n", - " subject_data['training_status'] = 0\n", - " subject_data.loc[~subject_data['scheduled_rig'].isnull(), 'training_status'] = 1\n", - " subject_data.loc[~subject_data['session_location'].isnull(), 'training_status'] = 2\n", - "\n", - " subject_data['training_status_label'] = \"Not scheduled\"\n", - " subject_data.loc[subject_data['training_status'] == 1, 'training_status_label'] = \"Scheduled\"\n", - " subject_data.loc[subject_data['training_status'] == 2, 'training_status_label'] = \"Training Started\"\n", - "\n", - " #Calculated fields \n", - " subject_data['already_weighted'] = ~subject_data['weight'].isnull()\n", - " subject_data['need_weight'] = ~subject_data['already_weighted'] | subject_data['need_reweight']\n", - "\n", - " subject_data['weight_status'] = \"Already Weighted\"\n", - " subject_data.loc[subject_data['already_weighted'] == 0, 'weight_status'] = \"Need Weight\"\n", - " subject_data.loc[subject_data['need_reweight'] == 1, 'weight_status'] = \"REWEIGHT\"\n", - "\n", + " subject_data[\"earned\"] = (\n", + " subject_data[\"earned\"].fillna(0).infer_objects(copy=False)\n", + " )\n", + " subject_data[\"received\"] = (\n", + " subject_data[\"received\"].fillna(0).infer_objects(copy=False)\n", + " )\n", + " subject_data[\"supplement\"] = (\n", + " subject_data[\"supplement\"].fillna(0).infer_objects(copy=False)\n", + " )\n", + " subject_data[\"prescribed_extra_supplement_amount\"] = (\n", + " subject_data[\"prescribed_extra_supplement_amount\"]\n", + " .fillna(0)\n", + " .infer_objects(copy=False)\n", + " )\n", + "\n", + " # Calculated fields\n", + " subject_data[\"already_water\"] = subject_data[\"supplement\"] > 0\n", + " subject_data[\"upper_cage\"] = subject_data[\"cage\"].str.upper()\n", + " subject_data[\"total_water_received\"] = subject_data[\"received\"]\n", + " subject_data.loc[\n", + " subject_data[\"total_water_received\"].isnull(), \"total_water_received\"\n", + " ] = 0\n", + "\n", + " subject_data[\"need_supplement\"] = 0\n", + " subject_data[\"need_supplement\"] = (\n", + " subject_data[\"total_water_received\"] < (subject_data[\"water_per_day\"] - 0.05)\n", + " ) & ~subject_data[\"already_water\"]\n", + "\n", + " subject_data.loc[subject_data[\"already_received\"].isnull(), \"already_received\"] = 0\n", + " subject_data[\"need_extra_water_now\"] = 0\n", + " subject_data.loc[\n", + " (\n", + " (subject_data[\"prescribed_extra_supplement_amount\"] > 0)\n", + " & (subject_data[\"already_received\"] == 0)\n", + " ),\n", + " \"need_extra_water_now\",\n", + " ] = 1\n", + "\n", + " subject_data[\"water_status\"] = \"Already Watered\"\n", + " subject_data.loc[subject_data[\"need_supplement\"] == 1, \"water_status\"] = (\n", + " \"Need Supplement\"\n", + " )\n", + " subject_data.loc[subject_data[\"need_extra_water_now\"] == 1, \"water_status\"] = (\n", + " \"Need Extra Supplement\"\n", + " )\n", + "\n", + " subject_data[\"need_water\"] = 0\n", + " subject_data.loc[\n", + " (subject_data[\"need_supplement\"] | subject_data[\"need_extra_water_now\"]),\n", + " \"need_water\",\n", + " ] = 1\n", + "\n", + " subject_data[\"current_need_water\"] = subject_data[\"suggested_water\"]\n", + " subject_data.loc[\n", + " subject_data[\"need_extra_water_now\"] == 1, \"current_need_water\"\n", + " ] = subject_data.loc[\n", + " subject_data[\"need_extra_water_now\"] == 1, \"prescribed_extra_supplement_amount\"\n", + " ]\n", + "\n", + " subject_data[\"training_status\"] = 0\n", + " subject_data.loc[~subject_data[\"scheduled_rig\"].isnull(), \"training_status\"] = 1\n", + " subject_data.loc[~subject_data[\"session_location\"].isnull(), \"training_status\"] = 2\n", + "\n", + " subject_data[\"training_status_label\"] = \"Not scheduled\"\n", + " subject_data.loc[subject_data[\"training_status\"] == 1, \"training_status_label\"] = (\n", + " \"Scheduled\"\n", + " )\n", + " subject_data.loc[subject_data[\"training_status\"] == 2, \"training_status_label\"] = (\n", + " \"Training Started\"\n", + " )\n", + "\n", + " # Calculated fields\n", + " subject_data[\"already_weighted\"] = ~subject_data[\"weight\"].isnull()\n", + " subject_data[\"need_weight\"] = (\n", + " ~subject_data[\"already_weighted\"] | subject_data[\"need_reweight\"]\n", + " )\n", + "\n", + " subject_data[\"weight_status\"] = \"Already Weighted\"\n", + " subject_data.loc[subject_data[\"already_weighted\"] == 0, \"weight_status\"] = (\n", + " \"Need Weight\"\n", + " )\n", + " subject_data.loc[subject_data[\"need_reweight\"] == 1, \"weight_status\"] = \"REWEIGHT\"\n", "\n", " return subject_data" ] @@ -575,7 +612,7 @@ } ], "source": [ - "pd.set_option('display.max_rows', 20)\n", + "pd.set_option(\"display.max_rows\", 20)\n", "subject_data" ] }, @@ -654,15 +691,25 @@ } ], "source": [ - "subject_data = subject_data.loc[~subject_data['subject_fullname'].str.contains('test'),:]\n", - "subjects_not_watered = subject_data.loc[subject_data['current_need_water'] > 0, ['subject_fullname', 'current_need_water']]\n", + "subject_data = subject_data.loc[\n", + " ~subject_data[\"subject_fullname\"].str.contains(\"test\"), :\n", + "]\n", + "subjects_not_watered = subject_data.loc[\n", + " subject_data[\"current_need_water\"] > 0, [\"subject_fullname\", \"current_need_water\"]\n", + "]\n", "subjects_not_watered = subjects_not_watered.reset_index(drop=True)\n", - "subjects_not_watered['current_need_water'] = subjects_not_watered['current_need_water'].apply(lambda x: f\"{x:.1f}\")\n", + "subjects_not_watered[\"current_need_water\"] = subjects_not_watered[\n", + " \"current_need_water\"\n", + "].apply(lambda x: f\"{x:.1f}\")\n", "subjects_not_watered = subjects_not_watered.head()\n", - "subjects_not_weighted = subject_data.loc[subject_data['need_weight'], ['subject_fullname', 'need_weight']]\n", + "subjects_not_weighted = subject_data.loc[\n", + " subject_data[\"need_weight\"], [\"subject_fullname\", \"need_weight\"]\n", + "]\n", "subjects_not_weighted = subjects_not_weighted.reset_index(drop=True)\n", "subjects_not_weighted = subjects_not_weighted.head()\n", - "subjects_not_trained = subject_data.loc[subject_data['training_status']==1, ['subject_fullname', 'scheduled_rig']]\n", + "subjects_not_trained = subject_data.loc[\n", + " subject_data[\"training_status\"] == 1, [\"subject_fullname\", \"scheduled_rig\"]\n", + "]\n", "subjects_not_trained = subjects_not_trained.reset_index(drop=True)\n", "subjects_not_trained = subjects_not_trained.head()\n", "subjects_not_trained" @@ -682,13 +729,16 @@ } ], "source": [ - "lab = dj.create_virtual_module('lab', 'u19_lab')\n", - "slack_configuration_dictionary = {\n", - " 'slack_notification_channel': ['alvaro_luna']\n", - "}\n", + "lab = dj.create_virtual_module(\"lab\", \"u19_lab\")\n", + "slack_configuration_dictionary = {\"slack_notification_channel\": [\"alvaro_luna\"]}\n", "webhooks_list = []\n", - "query_slack_webhooks = [{'webhook_name' : x} for x in slack_configuration_dictionary['slack_notification_channel']]\n", - "webhooks_list += (lab.SlackWebhooks & query_slack_webhooks).fetch('webhook_url').tolist()" + "query_slack_webhooks = [\n", + " {\"webhook_name\": x}\n", + " for x in slack_configuration_dictionary[\"slack_notification_channel\"]\n", + "]\n", + "webhooks_list += (\n", + " (lab.SlackWebhooks & query_slack_webhooks).fetch(\"webhook_url\").tolist()\n", + ")" ] }, { @@ -697,58 +747,72 @@ "metadata": {}, "outputs": [], "source": [ - "def slack_alert_message_format_weight_water(subjects_not_watered, subjects_not_weighted, subjects_not_trained):\n", + "def slack_alert_message_format_weight_water(\n", + " subjects_not_watered, subjects_not_weighted, subjects_not_trained\n", + "):\n", "\n", " now = datetime.datetime.now()\n", - " datestr = now.strftime('%d-%b-%Y %H:%M:%S')\n", + " datestr = now.strftime(\"%d-%b-%Y %H:%M:%S\")\n", "\n", " msep = dict()\n", - " msep['type'] = \"divider\"\n", + " msep[\"type\"] = \"divider\"\n", "\n", - " #Title#\n", + " # Title#\n", " m1 = dict()\n", - " m1['type'] = 'section'\n", + " m1[\"type\"] = \"section\"\n", " m1_1 = dict()\n", " m1_1[\"type\"] = \"mrkdwn\"\n", - " m1_1[\"text\"] = ':rotating_light: * Subjects Status Alert *'\n", - " m1['text'] = m1_1\n", + " m1_1[\"text\"] = \":rotating_light: * Subjects Status Alert *\"\n", + " m1[\"text\"] = m1_1\n", "\n", - " #Info#\n", + " # Info#\n", " m2 = dict()\n", - " m2['type'] = 'section'\n", + " m2[\"type\"] = \"section\"\n", " m2_1 = dict()\n", " m2_1[\"type\"] = \"mrkdwn\"\n", "\n", - " m2_1[\"text\"] = '*Subjects missing water:*' + '\\n'\n", + " m2_1[\"text\"] = \"*Subjects missing water:*\" + \"\\n\"\n", " for i in range(subjects_not_watered.shape[0]):\n", - " m2_1[\"text\"] += '*' + subjects_not_watered.loc[i, 'subject_fullname'] + '* : ' + str(subjects_not_watered.loc[i, 'current_need_water']) + ' ml\\n'\n", - " #m2_1[\"text\"] += '\\n'\n", - " m2['text'] = m2_1\n", + " m2_1[\"text\"] += (\n", + " \"*\"\n", + " + subjects_not_watered.loc[i, \"subject_fullname\"]\n", + " + \"* : \"\n", + " + str(subjects_not_watered.loc[i, \"current_need_water\"])\n", + " + \" ml\\n\"\n", + " )\n", + " # m2_1[\"text\"] += '\\n'\n", + " m2[\"text\"] = m2_1\n", "\n", " m4 = dict()\n", - " m4['type'] = 'section'\n", + " m4[\"type\"] = \"section\"\n", " m4_1 = dict()\n", " m4_1[\"type\"] = \"mrkdwn\"\n", "\n", - " m4_1[\"text\"] = '*Subjects missing weighing:*' + '\\n'\n", + " m4_1[\"text\"] = \"*Subjects missing weighing:*\" + \"\\n\"\n", " for i in range(subjects_not_weighted.shape[0]):\n", - " m4_1[\"text\"] += '*' + subjects_not_weighted.loc[i, 'subject_fullname'] + '*\\n' \n", - " m4['text'] = m4_1\n", + " m4_1[\"text\"] += \"*\" + subjects_not_weighted.loc[i, \"subject_fullname\"] + \"*\\n\"\n", + " m4[\"text\"] = m4_1\n", "\n", " m5 = dict()\n", - " m5['type'] = 'section'\n", + " m5[\"type\"] = \"section\"\n", " m5_1 = dict()\n", " m5_1[\"type\"] = \"mrkdwn\"\n", "\n", - " m5_1[\"text\"] = '*Subjects missing training:*' + '\\n'\n", + " m5_1[\"text\"] = \"*Subjects missing training:*\" + \"\\n\"\n", " for i in range(subjects_not_trained.shape[0]):\n", - " m5_1[\"text\"] += '*' + subjects_not_trained.loc[i, 'subject_fullname'] + '* : ' + subjects_not_trained.loc[i, 'scheduled_rig'] + '\\n'\n", + " m5_1[\"text\"] += (\n", + " \"*\"\n", + " + subjects_not_trained.loc[i, \"subject_fullname\"]\n", + " + \"* : \"\n", + " + subjects_not_trained.loc[i, \"scheduled_rig\"]\n", + " + \"\\n\"\n", + " )\n", " #\n", - " m5['text'] = m5_1\n", + " m5[\"text\"] = m5_1\n", "\n", " message = dict()\n", - " message['blocks'] = [m1,msep,m2,msep,m4,msep,m5,msep]\n", - " message['text'] = 'Subject Status Alert'\n", + " message[\"blocks\"] = [m1, msep, m2, msep, m4, msep, m5, msep]\n", + " message[\"text\"] = \"Subject Status Alert\"\n", "\n", " return message" ] @@ -759,13 +823,15 @@ "metadata": {}, "outputs": [], "source": [ - "slack_json_message = slack_alert_message_format_weight_water(subjects_not_watered, subjects_not_weighted, subjects_not_trained)\n", + "slack_json_message = slack_alert_message_format_weight_water(\n", + " subjects_not_watered, subjects_not_weighted, subjects_not_trained\n", + ")\n", "\n", "\n", - "#Send alert\n", + "# Send alert\n", "for this_webhook in webhooks_list:\n", " su.send_slack_notification(this_webhook, slack_json_message)\n", - " time.sleep(1)\n" + " time.sleep(1)" ] }, { diff --git a/notebooks/xyz_picks_notebook.ipynb b/notebooks/xyz_picks_notebook.ipynb index adab18d9..5d362a0e 100644 --- a/notebooks/xyz_picks_notebook.ipynb +++ b/notebooks/xyz_picks_notebook.ipynb @@ -20,8 +20,7 @@ "import numpy as np\n", "import pylab as pl\n", "\n", - "from u19_pipeline.utils import ephys_utils\n", - "\n" + "from u19_pipeline.utils import ephys_utils" ] }, { @@ -30,19 +29,21 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "recording_id = 81\n", - "chanmaps_jobs = ['298','299','300','301']\n", + "chanmaps_jobs = [\"298\", \"299\", \"300\", \"301\"]\n", "\n", "list_probes = list()\n", "for idx in range(4):\n", " job_id = chanmaps_jobs[idx]\n", - " procesed_data_directory = f'/Volumes/braininit/Data/Processed/electrophysiology/ms81/ms81_M019/20220611/TowersTask_g0/TowersTask_g0_imec{idx}/job_id_{job_id}'\n", - " chanmap_file = f'/Volumes/braininit//Shared/TestData/ChanMapFiles/chanmap_{job_id}.mat'\n", - " list_probes.append(ephys_utils.xyz_pick_file_creator.main_xyz_pick_file_function(recording_id, idx, chanmap_file, procesed_data_directory))\n", - "\n", - "\n", - "\n" + " procesed_data_directory = f\"/Volumes/braininit/Data/Processed/electrophysiology/ms81/ms81_M019/20220611/TowersTask_g0/TowersTask_g0_imec{idx}/job_id_{job_id}\"\n", + " chanmap_file = (\n", + " f\"/Volumes/braininit//Shared/TestData/ChanMapFiles/chanmap_{job_id}.mat\"\n", + " )\n", + " list_probes.append(\n", + " ephys_utils.xyz_pick_file_creator.main_xyz_pick_file_function(\n", + " recording_id, idx, chanmap_file, procesed_data_directory\n", + " )\n", + " )" ] }, { @@ -66,26 +67,26 @@ "source": [ "import matplotlib.pyplot as pl\n", "\n", - "fig = pl.figure(figsize = (12,12))\n", - "ax = fig.add_subplot(111,projection='3d')\n", + "fig = pl.figure(figsize=(12, 12))\n", + "ax = fig.add_subplot(111, projection=\"3d\")\n", "for i in range(4):\n", " this_probe_data = list_probes[i]\n", - " for this_shank_data in this_probe_data: \n", - " pt=np.array([np.array(xi) for xi in this_shank_data])\n", - " ax.scatter(pt[:,0], pt[:,1], pt[:,2])\n", + " for this_shank_data in this_probe_data:\n", + " pt = np.array([np.array(xi) for xi in this_shank_data])\n", + " ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2])\n", "\n", "\n", "# Just for plotting:\n", - "#xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n", - "#ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n", - "ax.scatter(xs=0, ys=0, zs=0, c='k')\n", - "ax.set_xlim(-5000,5000)\n", - "ax.set_ylim(-5000,5000)\n", - "ax.set_zlim(-5000,0)\n", + "# xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n", + "# ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n", + "ax.scatter(xs=0, ys=0, zs=0, c=\"k\")\n", + "ax.set_xlim(-5000, 5000)\n", + "ax.set_ylim(-5000, 5000)\n", + "ax.set_zlim(-5000, 0)\n", "ax.set_xlabel(\"x | ML\")\n", "ax.set_ylabel(\"y | AP\")\n", "ax.set_zlabel(\"z | DV\")\n", - "ax.view_init(elev=90., azim=90)\n", + "ax.view_init(elev=90.0, azim=90)\n", "\n", "pl.show()" ] @@ -111,22 +112,22 @@ "source": [ "import matplotlib.pyplot as pl\n", "\n", - "fig = pl.figure(figsize = (12,12))\n", - "ax = fig.add_subplot(111,projection='3d')\n", + "fig = pl.figure(figsize=(12, 12))\n", + "ax = fig.add_subplot(111, projection=\"3d\")\n", "for i in range(4):\n", " this_probe_data = list_probes[i]\n", - " for this_shank_data in this_probe_data: \n", - " pt=np.array([np.array(xi) for xi in this_shank_data])\n", - " ax.scatter(pt[:,0], pt[:,1], pt[:,2])\n", + " for this_shank_data in this_probe_data:\n", + " pt = np.array([np.array(xi) for xi in this_shank_data])\n", + " ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2])\n", "\n", "\n", "# Just for plotting:\n", - "#xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n", - "#ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n", - "ax.scatter(xs=0, ys=0, zs=0, c='k')\n", - "ax.set_xlim(-2500,2500)\n", - "ax.set_ylim(-2500,2500)\n", - "ax.set_zlim(-5000,0)\n", + "# xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n", + "# ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n", + "ax.scatter(xs=0, ys=0, zs=0, c=\"k\")\n", + "ax.set_xlim(-2500, 2500)\n", + "ax.set_ylim(-2500, 2500)\n", + "ax.set_zlim(-5000, 0)\n", "ax.set_xlabel(\"x | ML\")\n", "ax.set_ylabel(\"y | AP\")\n", "ax.set_zlabel(\"z | DV\")\n", @@ -154,22 +155,22 @@ "source": [ "import matplotlib.pyplot as pl\n", "\n", - "fig = pl.figure(figsize = (12,12))\n", - "ax = fig.add_subplot(111,projection='3d')\n", + "fig = pl.figure(figsize=(12, 12))\n", + "ax = fig.add_subplot(111, projection=\"3d\")\n", "for i in range(4):\n", " this_probe_data = list_probes[i]\n", - " for this_shank_data in this_probe_data: \n", - " pt=np.array([np.array(xi) for xi in this_shank_data])\n", - " ax.scatter(pt[:,0], pt[:,1], pt[:,2])\n", + " for this_shank_data in this_probe_data:\n", + " pt = np.array([np.array(xi) for xi in this_shank_data])\n", + " ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2])\n", "\n", "\n", "# Just for plotting:\n", - "#xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n", - "#ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n", - "ax.scatter(xs=0, ys=0, zs=0, c='k')\n", - "ax.set_xlim(-2500,2500)\n", - "ax.set_ylim(-2500,2500)\n", - "ax.set_zlim(-5000,0)\n", + "# xx, yy = np.meshgrid(range(-5000,10000), range(-5000,5000))\n", + "# ax.plot_surface(xx, yy, 0*xx, alpha=0.5)\n", + "ax.scatter(xs=0, ys=0, zs=0, c=\"k\")\n", + "ax.set_xlim(-2500, 2500)\n", + "ax.set_ylim(-2500, 2500)\n", + "ax.set_zlim(-5000, 0)\n", "ax.set_xlabel(\"x | ML\")\n", "ax.set_ylabel(\"y | AP\")\n", "ax.set_zlabel(\"z | DV\")\n", diff --git a/scripts/__init__.py b/scripts/__init__.py index e1424ed0..260c070a 100644 --- a/scripts/__init__.py +++ b/scripts/__init__.py @@ -1 +1 @@ -__version__ = '0.3.1' +__version__ = "0.3.1" diff --git a/scripts/conf_file_finding.py b/scripts/conf_file_finding.py index 677198a2..7436f241 100644 --- a/scripts/conf_file_finding.py +++ b/scripts/conf_file_finding.py @@ -1,4 +1,3 @@ - import os import pathlib @@ -9,16 +8,15 @@ def chdir_to_root(): conf_file_found = 0 starting_directory = os.getcwd() while 1: - current_dir = os.getcwd() - u19_dir = pathlib.Path(current_dir,'u19_pipeline') + u19_dir = pathlib.Path(current_dir, "u19_pipeline") if os.path.isdir(u19_dir): root_dir_found = 1 - if os.path.isfile(pathlib.Path(current_dir,'dj_local_conf.json')): + if os.path.isfile(pathlib.Path(current_dir, "dj_local_conf.json")): conf_file_found = 1 if root_dir_found: break - os.chdir('..') + os.chdir("..") new_current_dir = os.getcwd() if str(current_dir) == str(new_current_dir): os.chdir(starting_directory) @@ -26,14 +24,14 @@ def chdir_to_root(): return root_dir_found, conf_file_found + def get_root_directory(): root_dir_found = 0 current_dir = pathlib.Path(os.getcwd()) while 1: - - u19_dir = pathlib.Path(current_dir,'U19-pipeline_python') - u19_dir2 = pathlib.Path(current_dir,'U19-pipeline-python') + u19_dir = pathlib.Path(current_dir, "U19-pipeline_python") + u19_dir2 = pathlib.Path(current_dir, "U19-pipeline-python") if os.path.isdir(u19_dir) or os.path.isdir(u19_dir2): root_dir_found = 1 break @@ -49,10 +47,18 @@ def try_find_conf_file(): root_dir_found, conf_file_found = chdir_to_root() if root_dir_found and conf_file_found: - print('Local configuration file found !!, no need to run the configuration (unless configuration has changed)') + print( + "Local configuration file found !!, no need to run the configuration (unless configuration has changed)" + ) elif root_dir_found: - raise FileNotFoundError('Local configuration file not found. Ignore this if you have a global config. Run configuration script (initial_conf.py) otherwise') - elif os.path.isfile(pathlib.Path(pathlib.Path.home(),".datajoint_config.json")): - print("Global configuration file found !!, no need to run the configuration (unless configuration has changed)") + raise FileNotFoundError( + "Local configuration file not found. Ignore this if you have a global config. Run configuration script (initial_conf.py) otherwise" + ) + elif os.path.isfile(pathlib.Path(pathlib.Path.home(), ".datajoint_config.json")): + print( + "Global configuration file found !!, no need to run the configuration (unless configuration has changed)" + ) else: - raise FileNotFoundError('Root dir not found, change this notebook to the project folder') + raise FileNotFoundError( + "Root dir not found, change this notebook to the project folder" + ) diff --git a/scripts/drop_schemas.py b/scripts/drop_schemas.py index b5047991..96e4042d 100644 --- a/scripts/drop_schemas.py +++ b/scripts/drop_schemas.py @@ -28,4 +28,4 @@ subject.schema.drop() lab.schema.drop() reference.schema.drop() -task.schema.drop() \ No newline at end of file +task.schema.drop() diff --git a/scripts/ingest_hdf5_to_db.py b/scripts/ingest_hdf5_to_db.py index ea6c7afc..04fe3c0a 100644 --- a/scripts/ingest_hdf5_to_db.py +++ b/scripts/ingest_hdf5_to_db.py @@ -14,40 +14,44 @@ import numpy as np import pandas as pd -dj.config['database.host'] = 'datajoint00.pni.princeton.edu' +dj.config["database.host"] = "datajoint00.pni.princeton.edu" # load dj creds from file -credfile = '/jukebox/wang/ahoag/.djenv' +credfile = "/jukebox/wang/ahoag/.djenv" with open(credfile) as infile: cred_dict = json.load(infile) -dj.config['database.user'] = cred_dict.get('DJ_DB_USER') -dj.config['database.password'] = cred_dict.get('DJ_DB_PASS') +dj.config["database.user"] = cred_dict.get("DJ_DB_USER") +dj.config["database.password"] = cred_dict.get("DJ_DB_PASS") # Link to existing databases -u19_lab = dj.create_virtual_module('u19_lab','u19_lab') -u19_subject = dj.create_virtual_module('u19_subject','u19_subject') -u19_acq = dj.create_virtual_module('u19_acquisition','u19_acquisition') +u19_lab = dj.create_virtual_module("u19_lab", "u19_lab") +u19_subject = dj.create_virtual_module("u19_subject", "u19_subject") +u19_acq = dj.create_virtual_module("u19_acquisition", "u19_acquisition") # Link to new Puffs dbs -u19_puffs = dj.create_virtual_module('u19_puffs','u19_puffs') +u19_puffs = dj.create_virtual_module("u19_puffs", "u19_puffs") # Currently a hardcoded data folder to one of Marlies' cohorts for this example, -# but in the future will loop over folders starting from +# but in the future will loop over folders starting from # The root folder: /jukebox/braininit/puffs/ -data_folder = '/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation/cohort_10/rig0/' -h5_files = glob.glob(data_folder + '/data*.h5') -mouse_info_file = os.path.join('/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation', - 'mouse_info_0821_datajoint_format.csv') +data_folder = ( + "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation/cohort_10/rig0/" +) +h5_files = glob.glob(data_folder + "/data*.h5") +mouse_info_file = os.path.join( + "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation", + "mouse_info_0821_datajoint_format.csv", +) # first read in the mouse_info file into a pandas dataframe -df_info = pd.read_csv(mouse_info_file,keep_default_na=False) +df_info = pd.read_csv(mouse_info_file, keep_default_na=False) # # Use mouse_info file and do the u19_subject and u19_lab inserts # Tables we need to insert into (**In this order**) are: # - u19_lab.User # in case user is not already in the database # - u19_lab.Protocol # in case protocol is not already in the database # - u19_lab.Location # in case the location of each subject is not already in the database -# +# # - u19_subject.Species # in case they are NOT training on mice # - u19_subject.Strain # in case they are NOT training on a strain of mice already in the db # - u19_subject.Line # in case they are using a new genetic line @@ -55,110 +59,110 @@ # u19_lab.User() inserts -unique_netids = df_info['netid'].unique() -full_names = df_info['experimenter_fullname'] +unique_netids = df_info["netid"].unique() +full_names = df_info["experimenter_fullname"] user_insert_list = [] for netid in unique_netids: - netid_mask = df_info['netid'] == netid + netid_mask = df_info["netid"] == netid full_name = full_names[netid_mask].iloc[0] user_insert_dict = { - 'user_id': netid, - 'user_nickname': netid, - 'full_name': full_name, - 'email': f'{netid}@princeton.edu', - 'contact_via': 'Email', - 'presence': 'Available', - 'tech_responsibility': 'no', - 'day_cutoff_time': np.array([[18., 0.]]) - } + "user_id": netid, + "user_nickname": netid, + "full_name": full_name, + "email": f"{netid}@princeton.edu", + "contact_via": "Email", + "presence": "Available", + "tech_responsibility": "no", + "day_cutoff_time": np.array([[18.0, 0.0]]), + } user_insert_list.append(user_insert_dict) -u19_lab.User.insert(user_insert_list,skip_duplicates=True) +u19_lab.User.insert(user_insert_list, skip_duplicates=True) # u19_lab.Protocol() inserts -unique_protocols = df_info['protocol'].unique() +unique_protocols = df_info["protocol"].unique() protocol_insert_list = [] for protocol in unique_protocols: protocol_insert_dict = { - 'protocol': protocol, - 'protocol_description': '', - } + "protocol": protocol, + "protocol_description": "", + } protocol_insert_list.append(protocol_insert_dict) -u19_lab.Protocol.insert(protocol_insert_list,skip_duplicates=True) +u19_lab.Protocol.insert(protocol_insert_list, skip_duplicates=True) # u19_lab.Location() inserts -unique_locations = df_info['location'].unique() -location_descriptions = df_info['location_description'] +unique_locations = df_info["location"].unique() +location_descriptions = df_info["location_description"] location_insert_list = [] for location in unique_locations: - location_mask = df_info['location'] == location + location_mask = df_info["location"] == location location_description = location_descriptions[netid_mask].iloc[0] location_insert_dict = { - 'location': location, - 'location_description': location_description, - } + "location": location, + "location_description": location_description, + } location_insert_list.append(location_insert_dict) -u19_lab.Location.insert(location_insert_list,skip_duplicates=True) +u19_lab.Location.insert(location_insert_list, skip_duplicates=True) # u19_subject.Species() inserts -unique_species = df_info['binomial'].unique() +unique_species = df_info["binomial"].unique() species_insert_list = [] for species in unique_species: species_insert_dict = { - 'binomial': species, - 'species_nickname': '', - } + "binomial": species, + "species_nickname": "", + } species_insert_list.append(species_insert_dict) -u19_subject.Species.insert(species_insert_list,skip_duplicates=True) +u19_subject.Species.insert(species_insert_list, skip_duplicates=True) # u19_subject.Strain() inserts -unique_strains = df_info['strain'].unique() +unique_strains = df_info["strain"].unique() strain_insert_list = [] for strain in unique_strains: strain_insert_dict = { - 'strain_name': strain, - 'strain_description': '', - } + "strain_name": strain, + "strain_description": "", + } strain_insert_list.append(strain_insert_dict) -u19_subject.Strain.insert(strain_insert_list,skip_duplicates=True) +u19_subject.Strain.insert(strain_insert_list, skip_duplicates=True) # u19_subject.Line() inserts -unique_lines = df_info['genetic_line'].unique() -binomials = df_info['binomial'] -strains = df_info['strain'] +unique_lines = df_info["genetic_line"].unique() +binomials = df_info["binomial"] +strains = df_info["strain"] line_insert_list = [] for line in unique_lines: - line_mask = df_info['genetic_line'] == line + line_mask = df_info["genetic_line"] == line binomial = binomials[line_mask].iloc[0] strain = strains[line_mask].iloc[0] line_insert_dict = { - 'line': line, - 'binomial': binomial, - 'strain_name':strain, - 'line_description':'', - 'target_phenotype':'', - 'is_active':1, - } + "line": line, + "binomial": binomial, + "strain_name": strain, + "line_description": "", + "target_phenotype": "", + "is_active": 1, + } line_insert_list.append(line_insert_dict) -u19_subject.Line.insert(line_insert_list,skip_duplicates=True) +u19_subject.Line.insert(line_insert_list, skip_duplicates=True) # u19_subject.Subject() inserts -unique_subjects = df_info['subj'].unique() -project_names = df_info['project_name'] -subject_descriptions = df_info['subj_description'] -netids = df_info['netid'] -sexes = df_info['sex'] -DOBs = df_info['DOB'] -locations = df_info['location'] -lines = df_info['genetic_line'] -protocols = df_info['protocol'] +unique_subjects = df_info["subj"].unique() +project_names = df_info["project_name"] +subject_descriptions = df_info["subj_description"] +netids = df_info["netid"] +sexes = df_info["sex"] +DOBs = df_info["DOB"] +locations = df_info["location"] +lines = df_info["genetic_line"] +protocols = df_info["protocol"] subject_insert_list = [] for subject in unique_subjects: - subject_mask = df_info['subj'] == subject + subject_mask = df_info["subj"] == subject netid = netids[subject_mask].iloc[0] project_name = project_names[subject_mask].iloc[0] - subject_fullname = '_'.join([netid,project_name,str(subject)]) + subject_fullname = "_".join([netid, project_name, str(subject)]) sex = sexes[subject_mask].iloc[0] DOB = DOBs[subject_mask].iloc[0] location = locations[subject_mask].iloc[0] @@ -166,39 +170,40 @@ protocol = protocols[subject_mask].iloc[0] subject_description = subject_descriptions[subject_mask].iloc[0] subject_insert_dict = { - 'subject_fullname': subject_fullname, - 'subject_nickname': str(subject), - 'subject_description': subject_description, - 'user_id':netid, - 'sex': 'Male' if sex == 'M' else 'Female', - 'dob': DOB, - 'location':location, - 'line':line, - 'protocol':protocol - } + "subject_fullname": subject_fullname, + "subject_nickname": str(subject), + "subject_description": subject_description, + "user_id": netid, + "sex": "Male" if sex == "M" else "Female", + "dob": DOB, + "location": location, + "line": line, + "protocol": protocol, + } subject_insert_list.append(subject_insert_dict) -u19_subject.Subject.insert(subject_insert_list,skip_duplicates=True) +u19_subject.Subject.insert(subject_insert_list, skip_duplicates=True) # ## Now use h5 files to make inserts into u19_acquisition tables and puffs specific tables -# +# # Tables we need to insert into (**In this order**) are: # - u19_puffs.PuffsCohort # in case cohort is not already in the database # - u19_puffs.PuffsFileAcquisition # as we process the various files -# +# # - u19_acquisition.SessionStarted # as we process the session data in each h5 file # - u19_acquisition.Session # as we process the session data in each h5 file -# -# - u19_puffs.PuffsSession -# - u19_puffs.PuffsSession.Trial +# +# - u19_puffs.PuffsSession +# - u19_puffs.PuffsSession.Trial # - u19_puffs.PuffsSession.TrialPhase # - u19_puffs.PuffsSession.Puff -# +# # first find list of h5 files that are already processed so we do not # repeat the ingestion on those -already_processed_filenames = (u19_puffs.PuffsFileAcquisition() & 'ingested=1').fetch( - 'h5_filename') +already_processed_filenames = (u19_puffs.PuffsFileAcquisition() & "ingested=1").fetch( + "h5_filename" +) # Loop over h5 files in this cohort/rig folder for h5_file in h5_files: @@ -206,13 +211,13 @@ print(f"file: {h5_file} already processed!") continue print(h5_file) - - """ Grab the relevant dataframes from the hdf5 file """ + + """ Grab the relevant dataframes from the hdf5 file """ with pd.HDFStore(h5_file) as data: df_trials = data.trials df_puffs = data.trials_timing df_phases = data.phases - + """ Sometimes there are extra session metadata stored in other dataframes with the key names: 'sessions/{session_number}/notes' @@ -220,66 +225,78 @@ 'sessions/{session_number}/sync' If they exist we want to ingest their metadata, but they do not always exist """ - stored_session_keys = [x for x in data if 'sessions' in x] - + stored_session_keys = [x for x in data if "sessions" in x] + """ Figure out the information from the h5 filepath """ - rig = h5_file.split('/')[-2][-1] # just want the 0 or 1 (not "rig0" or "rig1") - cohort = h5_file.split('/')[-3].split('_')[-1] # 'want C10' or just '10' - project_name = h5_file.split('/')[-4].strip() - username = h5_file.split('/')[-5].strip() + rig = h5_file.split("/")[-2][-1] # just want the 0 or 1 (not "rig0" or "rig1") + cohort = h5_file.split("/")[-3].split("_")[-1] # 'want C10' or just '10' + project_name = h5_file.split("/")[-4].strip() + username = h5_file.split("/")[-5].strip() puffs_cohort_insert_dict = { - 'user_id':username, - 'project_name':project_name, - 'cohort':cohort + "user_id": username, + "project_name": project_name, + "cohort": cohort, } - u19_puffs.PuffsCohort().insert1(puffs_cohort_insert_dict,skip_duplicates=True) + u19_puffs.PuffsCohort().insert1(puffs_cohort_insert_dict, skip_duplicates=True) - df_trials['rig'] = rig - df_trials['cohort'] = cohort - df_trials['project_name'] = project_name + df_trials["rig"] = rig + df_trials["cohort"] = cohort + df_trials["project_name"] = project_name """ Make choice column """ - df_trials['choice'] = (df_trials['side'] == df_trials['outcome'].astype(int)) + df_trials["choice"] = df_trials["side"] == df_trials["outcome"].astype(int) """ set choice to -1 if outcome was > 1 """ - bad_outcome_mask = df_trials['outcome']>1 - df_trials.loc[bad_outcome_mask,'choice'] = -1 + bad_outcome_mask = df_trials["outcome"] > 1 + df_trials.loc[bad_outcome_mask, "choice"] = -1 """ Map choice to string values: L,R and nil """ - df_trials['choice'] = df_trials['choice'].apply( - lambda x: 'L' if x==0 else ('R' if x==1 else 'nil')) + df_trials["choice"] = df_trials["choice"].apply( + lambda x: "L" if x == 0 else ("R" if x == 1 else "nil") + ) """ Make an answered correct column from outcome column """ - df_trials['answered_correct'] = (df_trials['outcome'] == 1).astype('int') - + df_trials["answered_correct"] = (df_trials["outcome"] == 1).astype("int") + """ Turn side column into string type """ - df_trials['side'] = df_trials['side'].apply(lambda x: 'L' if x==0 else ('R' if x==1 else 'nil')) + df_trials["side"] = df_trials["side"].apply( + lambda x: "L" if x == 0 else ("R" if x == 1 else "nil") + ) """ Make a new dataframe for unique sessions, i.e. where subj and session are unique - in trials """ - df_sessions = df_trials.groupby(['subj','session']).max().reset_index().sort_values( - ['subj','session']) - sessions = df_sessions['session'] - last_levels = df_sessions['level'] - session_ends_rel = df_sessions['end'] + in trials """ + df_sessions = ( + df_trials.groupby(["subj", "session"]) + .max() + .reset_index() + .sort_values(["subj", "session"]) + ) + sessions = df_sessions["session"] + last_levels = df_sessions["level"] + session_ends_rel = df_sessions["end"] """ make a session_dates column """ session_dates = sessions.apply(lambda x: str(x)[:10]) - df_sessions['session_date'] = session_dates - subjs = df_sessions['subj'] + df_sessions["session_date"] = session_dates + subjs = df_sessions["subj"] """ look up where these subjects are """ - rigs = df_sessions['rig'].astype('int') + rigs = df_sessions["rig"].astype("int") """ make a session number column - i.e. the 0-indexed counter for the session on a day for a single subject """ - session_numbers = df_sessions.groupby(['session_date','subj']).cumcount() - df_sessions['session_number'] = session_numbers + session_numbers = df_sessions.groupby(["session_date", "subj"]).cumcount() + df_sessions["session_number"] = session_numbers """ Figure out fraction correct for each session """ - fraction_trials_correct_df = df_trials.groupby( - ['session','subj']).agg({'answered_correct': 'sum','side':'count'}) - fraction_trials_correct_df['fraction_correct'] = \ - fraction_trials_correct_df['answered_correct']/fraction_trials_correct_df['side'] - fraction_trials_correct_df = fraction_trials_correct_df.sort_values(['subj','session']).reset_index() - df_sessions['fraction_correct'] = fraction_trials_correct_df['fraction_correct'] - fractions_correct = df_sessions['fraction_correct'] - + fraction_trials_correct_df = df_trials.groupby(["session", "subj"]).agg( + {"answered_correct": "sum", "side": "count"} + ) + fraction_trials_correct_df["fraction_correct"] = ( + fraction_trials_correct_df["answered_correct"] + / fraction_trials_correct_df["side"] + ) + fraction_trials_correct_df = fraction_trials_correct_df.sort_values( + ["subj", "session"] + ).reset_index() + df_sessions["fraction_correct"] = fraction_trials_correct_df["fraction_correct"] + fractions_correct = df_sessions["fraction_correct"] + """ Initialize lists to fill and then insert into db """ session_started_insert_list = [] session_insert_list = [] @@ -289,193 +306,242 @@ n_sessions = len(df_sessions) for ii in range(len(df_sessions)): subj = str(int(subjs.iloc[ii])) - subject_fullname = '_'.join([username,project_name,subj]) + subject_fullname = "_".join([username, project_name, subj]) last_level = last_levels.iloc[ii] date = session_dates.iloc[ii] - print(f"session {ii+1}/{len(df_sessions)}: {subject_fullname}, {date}") + print(f"session {ii + 1}/{len(df_sessions)}: {subject_fullname}, {date}") session_datetime = sessions.iloc[ii] session_end_rel = session_ends_rel[ii] session_end_datetime = session_datetime + timedelta(seconds=session_end_rel) session_number = session_numbers.iloc[ii] - percentage_correct = fractions_correct.iloc[ii]*100 + percentage_correct = fractions_correct.iloc[ii] * 100 rig = rigs.iloc[ii] - + if rig == 0: location = "pni-ltl016-05" elif rig == 1: location = "wang-behavior" else: - sys.exit(f'Rig: {rig} is not one of the rigs. Check data type (needs to be integer)') - session_compressed_str = session_datetime.strftime('%Y%m%d%H%M%S') - + sys.exit( + f"Rig: {rig} is not one of the rigs. Check data type (needs to be integer)" + ) + session_compressed_str = session_datetime.strftime("%Y%m%d%H%M%S") + """ Check to see if the session metadata dataframes exist """ - this_session_params_key = f'sessions/{session_compressed_str}/params' + this_session_params_key = f"sessions/{session_compressed_str}/params" if this_session_params_key in stored_session_keys: session_param_dict = json.loads(data[this_session_params_key].iloc[0]) else: session_param_dict = None - this_session_notes_key = f'sessions/{session_compressed_str}/notes' + this_session_notes_key = f"sessions/{session_compressed_str}/notes" if this_session_notes_key in stored_session_keys: session_notes_dict = json.loads(data[this_session_notes_key].iloc[0]) - if 'notes' in session_notes_dict: - session_notes = session_notes_dict['notes'] + if "notes" in session_notes_dict: + session_notes = session_notes_dict["notes"] else: session_notes = None - if 'stdout' in session_notes_dict: - session_stdout = session_notes_dict['stdout'] + if "stdout" in session_notes_dict: + session_stdout = session_notes_dict["stdout"] else: session_stdout = None - if 'stderr' in session_notes_dict: - session_stderr = session_notes_dict['stderr'] + if "stderr" in session_notes_dict: + session_stderr = session_notes_dict["stderr"] else: session_stderr = None else: session_notes = None session_stdout = None session_stderr = None - this_session_sync_key = f'sessions/{session_compressed_str}/sync' + this_session_sync_key = f"sessions/{session_compressed_str}/sync" if this_session_sync_key in stored_session_keys: session_sync_dict = data[this_session_notes_key].todict() else: session_sync_dict = None - - """ Assemble the u19_acquistion inserts for this session """ + + """ Assemble the u19_acquistion inserts for this session """ session_started_insert_dict = { - 'subject_fullname':subject_fullname, - 'session_date':date, - 'session_number':session_number, - 'session_start_time':session_datetime.strftime('%Y-%m-%d %H:%M:%S'), - 'session_location':location, - 'task':'AirPuffs', - 'local_path_behavior_file':'', - 'remote_path_behavior_file':h5_file, - 'is_finished':1 + "subject_fullname": subject_fullname, + "session_date": date, + "session_number": session_number, + "session_start_time": session_datetime.strftime("%Y-%m-%d %H:%M:%S"), + "session_location": location, + "task": "AirPuffs", + "local_path_behavior_file": "", + "remote_path_behavior_file": h5_file, + "is_finished": 1, } session_insert_dict = { - 'subject_fullname':subject_fullname, - 'session_date':date, - 'session_number':session_number, - 'session_start_time':session_datetime.strftime('%Y-%m-%d %H:%M:%S'), - 'session_end_time':session_end_datetime.strftime('%Y-%m-%d %H:%M:%S'), - 'session_location':location, - 'task':'AirPuffs', - 'level':last_level, - 'set_id':1, - 'stimulus_bank':"", - 'stimulus_commit':"", - 'session_performance':percentage_correct, - 'session_narrative': '', - + "subject_fullname": subject_fullname, + "session_date": date, + "session_number": session_number, + "session_start_time": session_datetime.strftime("%Y-%m-%d %H:%M:%S"), + "session_end_time": session_end_datetime.strftime("%Y-%m-%d %H:%M:%S"), + "session_location": location, + "task": "AirPuffs", + "level": last_level, + "set_id": 1, + "stimulus_bank": "", + "stimulus_commit": "", + "session_performance": percentage_correct, + "session_narrative": "", } - """ Assemble the puffs inserts for this session """ + """ Assemble the puffs inserts for this session """ # PuffsSession() table puffs_session_insert_dict = { - 'subject_fullname':subject_fullname, - 'session_date':date, - 'session_number':session_number, - 'session_params':session_param_dict, - 'rig':rig, - 'notes':session_notes, - 'stdout':session_stdout, - 'stderr':session_stderr, - 'sync':session_sync_dict, - + "subject_fullname": subject_fullname, + "session_date": date, + "session_number": session_number, + "session_params": session_param_dict, + "rig": rig, + "notes": session_notes, + "stdout": session_stdout, + "stderr": session_stderr, + "sync": session_sync_dict, } # Trial() table - df_trials_this_session = df_trials[df_trials['session'] == session_datetime][[ - 'idx','level','side','choice','answered_correct', - 'draw_p','start','end','dur','nL_intended', - 'nL','nR_intended','nR','reward','reward_scale','rule']] + df_trials_this_session = df_trials[ + df_trials["session"] == session_datetime + ][ + [ + "idx", + "level", + "side", + "choice", + "answered_correct", + "draw_p", + "start", + "end", + "dur", + "nL_intended", + "nL", + "nR_intended", + "nR", + "reward", + "reward_scale", + "rule", + ] + ] num_trials_this_session = len(df_trials_this_session) - session_insert_dict['num_trials'] = num_trials_this_session - df_trials_this_session['subject_fullname'] = subject_fullname - df_trials_this_session['session_date'] = date - df_trials_this_session['session_number'] = session_number - df_trials_this_session['task'] = 'AirPuffs' - df_trials_this_session['set_id'] = 1 - df_trials_this_session['trial_duration'] = df_trials_this_session['end'] - df_trials_this_session['start'] - df_trials_this_session.rename(columns={ - 'idx': 'trial_idx', - 'draw_p': 'trial_prior_p_left', - 'start':'trial_rel_start', - 'end':'trial_rel_finish', - 'dur':'cue_period', - 'nL':'num_puffs_received_l', - 'nL_intended':'num_puffs_intended_l', - 'nR':'num_puffs_received_r', - 'nR_intended':'num_puffs_intended_r', - 'side':'trial_type', - 'reward':'reward_rel_start' - },inplace=True) - - trials_insert_list = df_trials_this_session.to_dict('records') - + session_insert_dict["num_trials"] = num_trials_this_session + df_trials_this_session["subject_fullname"] = subject_fullname + df_trials_this_session["session_date"] = date + df_trials_this_session["session_number"] = session_number + df_trials_this_session["task"] = "AirPuffs" + df_trials_this_session["set_id"] = 1 + df_trials_this_session["trial_duration"] = ( + df_trials_this_session["end"] - df_trials_this_session["start"] + ) + df_trials_this_session.rename( + columns={ + "idx": "trial_idx", + "draw_p": "trial_prior_p_left", + "start": "trial_rel_start", + "end": "trial_rel_finish", + "dur": "cue_period", + "nL": "num_puffs_received_l", + "nL_intended": "num_puffs_intended_l", + "nR": "num_puffs_received_r", + "nR_intended": "num_puffs_intended_r", + "side": "trial_type", + "reward": "reward_rel_start", + }, + inplace=True, + ) + + trials_insert_list = df_trials_this_session.to_dict("records") + # Puff() table - df_puffs_this_session = df_puffs[df_puffs['session'] == session_datetime].copy() - df_puffs_this_session['subject_fullname'] = subject_fullname - df_puffs_this_session['session_date'] = date - df_puffs_this_session['session_number'] = session_number - df_puffs_this_session['puff_idx'] = df_puffs_this_session.groupby('trial').cumcount() - df_puffs_this_session.rename(columns={ - 'time':'puff_rel_time', - 'trial':'trial_idx', - },inplace=True) - df_puffs_this_session['session_date'] = date - df_puffs_this_session['session_number'] = session_number - df_puffs_this_session = df_puffs_this_session[[ - 'subject_fullname', - 'session_date', - 'session_number', - 'trial_idx', - 'puff_idx', - 'side', - 'puff_rel_time' - ]] - puffs_insert_list = df_puffs_this_session.to_dict('records') - + df_puffs_this_session = df_puffs[ + df_puffs["session"] == session_datetime + ].copy() + df_puffs_this_session["subject_fullname"] = subject_fullname + df_puffs_this_session["session_date"] = date + df_puffs_this_session["session_number"] = session_number + df_puffs_this_session["puff_idx"] = df_puffs_this_session.groupby( + "trial" + ).cumcount() + df_puffs_this_session.rename( + columns={ + "time": "puff_rel_time", + "trial": "trial_idx", + }, + inplace=True, + ) + df_puffs_this_session["session_date"] = date + df_puffs_this_session["session_number"] = session_number + df_puffs_this_session = df_puffs_this_session[ + [ + "subject_fullname", + "session_date", + "session_number", + "trial_idx", + "puff_idx", + "side", + "puff_rel_time", + ] + ] + puffs_insert_list = df_puffs_this_session.to_dict("records") + # TrialPhase() table - df_phases_this_session = df_phases[df_phases['session'] == session_datetime].copy() - df_phases_this_session['subject_fullname'] = subject_fullname - df_phases_this_session['session_date'] = date - df_phases_this_session['session_number'] = session_number - df_phases_this_session.rename(columns={ - 'trial':'trial_idx', - 'start_time':'phase_rel_start', - 'end_time':'phase_rel_finish', - },inplace=True) - df_phases_this_session = df_phases_this_session[[ - 'subject_fullname', - 'session_date', - 'session_number', - 'trial_idx', - 'phase', - 'phase_rel_start', - 'phase_rel_finish' - ]] + df_phases_this_session = df_phases[ + df_phases["session"] == session_datetime + ].copy() + df_phases_this_session["subject_fullname"] = subject_fullname + df_phases_this_session["session_date"] = date + df_phases_this_session["session_number"] = session_number + df_phases_this_session.rename( + columns={ + "trial": "trial_idx", + "start_time": "phase_rel_start", + "end_time": "phase_rel_finish", + }, + inplace=True, + ) + df_phases_this_session = df_phases_this_session[ + [ + "subject_fullname", + "session_date", + "session_number", + "trial_idx", + "phase", + "phase_rel_start", + "phase_rel_finish", + ] + ] """ Some phases seem to reference trials that do not exist restrict phase rows to trials that exist """ - trial_idxs = df_trials_this_session['trial_idx'] - phase_trial_mask = df_phases_this_session['trial_idx'].isin(trial_idxs.values) + trial_idxs = df_trials_this_session["trial_idx"] + phase_trial_mask = df_phases_this_session["trial_idx"].isin( + trial_idxs.values + ) df_phases_this_session_goodtrials = df_phases_this_session[phase_trial_mask] - phases_insert_list = df_phases_this_session_goodtrials.to_dict('records') + phases_insert_list = df_phases_this_session_goodtrials.to_dict("records") """ Start a transaction and do the inserts for this session """ - connection = u19_acq.SessionStarted.connection + connection = u19_acq.SessionStarted.connection with connection.transaction: - u19_acq.SessionStarted().insert1(session_started_insert_dict,skip_duplicates=True) - u19_acq.Session().insert1(session_insert_dict,skip_duplicates=True) - u19_puffs.PuffsSession().insert1(puffs_session_insert_dict,skip_duplicates=True) - u19_puffs.PuffsSession.Trial().insert(trials_insert_list,skip_duplicates=True) + u19_acq.SessionStarted().insert1( + session_started_insert_dict, skip_duplicates=True + ) + u19_acq.Session().insert1(session_insert_dict, skip_duplicates=True) + u19_puffs.PuffsSession().insert1( + puffs_session_insert_dict, skip_duplicates=True + ) + u19_puffs.PuffsSession.Trial().insert( + trials_insert_list, skip_duplicates=True + ) # u19_puffs.PuffsSession.Puff().insert(puffs_insert_list,skip_duplicates=True) # u19_puffs.PuffsSession.TrialPhase().insert(phases_insert_list,skip_duplicates=True) """ If the inserts for all sessions in this hdf5 file were successful, then we need to mark this hdf5 file as processed by inserting it into the PuffsFileAcquisition() table in u19_puffs_acquisition table """ file_acq_insert_dict = { - 'user_id':username, - 'project_name':project_name, - 'cohort':cohort, - 'rig':rig, - 'h5_filename':h5_file, - 'ingested':1 + "user_id": username, + "project_name": project_name, + "cohort": cohort, + "rig": rig, + "h5_filename": h5_file, + "ingested": 1, } - u19_puffs.PuffsFileAcquisition.insert1(file_acq_insert_dict,replace=True) # want to overwrite with ingested=1 if ingested=0 for some reason \ No newline at end of file + u19_puffs.PuffsFileAcquisition.insert1( + file_acq_insert_dict, replace=True + ) # want to overwrite with ingested=1 if ingested=0 for some reason diff --git a/scripts/rig_maintenance_demo.py b/scripts/rig_maintenance_demo.py index 68ad8204..f32a10a0 100755 --- a/scripts/rig_maintenance_demo.py +++ b/scripts/rig_maintenance_demo.py @@ -15,7 +15,7 @@ from datetime import date # Add the u19_pipeline to the path -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) try: from u19_pipeline import rig_maintenance @@ -28,16 +28,16 @@ def demo_insert_maintenance_record(): """Demo: Insert a maintenance record.""" print("=== Demo: Inserting Maintenance Record ===") - + # Example maintenance record maintenance_data = { - 'location': 'Bezos2', # Must exist in lab.Location - 'maintenance_type': 'Replacing lines', # Must exist in MaintenanceType - 'maintenance_date': date.today(), - 'user': 'demo_user', # Must exist in lab.User - 'maintenance_notes': 'Replaced all water lines. System tested and working normally.' + "location": "Bezos2", # Must exist in lab.Location + "maintenance_type": "Replacing lines", # Must exist in MaintenanceType + "maintenance_date": date.today(), + "user": "demo_user", # Must exist in lab.User + "maintenance_notes": "Replaced all water lines. System tested and working normally.", } - + try: # Insert the record (commented out to prevent actual DB modifications in demo) # rig_maintenance.RigMaintenance.insert1(maintenance_data) @@ -52,15 +52,15 @@ def demo_insert_maintenance_record(): def demo_query_maintenance_history(): """Demo: Query maintenance history for a rig.""" print("\n=== Demo: Querying Maintenance History ===") - - rig_name = 'Bezos2' - + + rig_name = "Bezos2" + try: # Query maintenance history (commented out to prevent DB access in demo) # records = (rig_maintenance.RigMaintenance & {'location': rig_name}).fetch( # as_dict=True, order_by='maintenance_date DESC' # ) - + print(f"Would query maintenance history for rig: {rig_name}") print("Query would return records ordered by most recent first") print("✅ Query structure is correct") @@ -71,15 +71,15 @@ def demo_query_maintenance_history(): def demo_check_overdue_maintenance(): """Demo: Check for overdue maintenance.""" print("\n=== Demo: Checking Overdue Maintenance ===") - + print("This demo shows the logic used in check_rig_maintenance.py") - + # Show the maintenance types and their intervals print("\nMaintenance intervals defined:") for mtype in rig_maintenance.MaintenanceType.contents: maintenance_type, description, interval_days = mtype print(f" • {maintenance_type}: every {interval_days} days") - + # Show the logic for checking overdue maintenance current_date = date.today() print(f"\nCurrent date: {current_date}") @@ -95,18 +95,18 @@ def main(): """Main demo function.""" print("Rig Maintenance Schema Demo") print("=" * 50) - + print("This demo shows how to use the new rig maintenance schema.") print("The schema includes:") print("- MaintenanceType: Lookup table with maintenance types and intervals") print("- RigMaintenance: Records of maintenance performed on rigs") print() - + # Run demo functions demo_insert_maintenance_record() demo_query_maintenance_history() demo_check_overdue_maintenance() - + print("\n" + "=" * 50) print("Demo complete! To use the schema:") print("1. Configure DataJoint connection") @@ -116,4 +116,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/u19_pipeline/acquisition.py b/u19_pipeline/acquisition.py index 9349ce1b..3bdc6d83 100644 --- a/u19_pipeline/acquisition.py +++ b/u19_pipeline/acquisition.py @@ -1,6 +1,6 @@ import datajoint as dj -schema = dj.schema(dj.config['custom']['database.prefix'] + 'acquisition') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "acquisition") @schema diff --git a/u19_pipeline/alert_system/alert_code_skeleton.py b/u19_pipeline/alert_system/alert_code_skeleton.py index fab6feb9..4d6fa257 100644 --- a/u19_pipeline/alert_system/alert_code_skeleton.py +++ b/u19_pipeline/alert_system/alert_code_skeleton.py @@ -1,21 +1,15 @@ - - - - # Slack Configuration dictionary slack_configuration_dictionary = { - 'slack_notification_channel': ['custom_alerts'], - 'slack_users_channel': ['alvaros'] + "slack_notification_channel": ["custom_alerts"], + "slack_users_channel": ["alvaros"], } + def main(): - ''' + """ Main function for subject "new_customalert" alert This function should return a pandas DataFrame where each row will be a slack alert message on configured channels. You can use datajoint to get data for the alert (e.g. custom_alerts/rig_bias.py) or simply call os scripts (e.g. custom_alerts/braininit_storage.py) All columns of the dataframe will be included in the alert. (Don't add too many !!) You can check examples of some alers in the u19_pipeline/alert_system/custom_alerts directory - ''' - - - + """ diff --git a/u19_pipeline/alert_system/alert_system_utility.py b/u19_pipeline/alert_system/alert_system_utility.py index 5cd5ea65..c834330c 100644 --- a/u19_pipeline/alert_system/alert_system_utility.py +++ b/u19_pipeline/alert_system/alert_system_utility.py @@ -1,5 +1,3 @@ - - import datetime import datajoint as dj @@ -8,40 +6,47 @@ import u19_pipeline.utils.dj_shortcuts as djs -def get_acquisition_data_alert_system(type='subject_fullname', data_days=60, min_sessions=20): - ''' +def get_acquisition_data_alert_system( + type="subject_fullname", data_days=60, min_sessions=20 +): + """ Get and filter data for alert system Inputs: - type = subject_fullname/location => Data grouping by subject or by rig + type = subject_fullname/location => Data grouping by subject or by rig data_days = Amount of days to look back for sessions min_sessions = Min amount of sessions for a subject/rig to draw conclusions about it Outputs: session_df = filtered DataFrame from acquisition.Session table - key_list = list of dictionaries with primary key fields for each session (for querying behavior) - ''' + key_list = list of dictionaries with primary key fields for each session (for querying behavior) + """ - acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition') + acquisition = dj.create_virtual_module("acquisition", "u19_acquisition") # Filter sessiond for # data days only today = datetime.date.today() past_date = today - datetime.timedelta(days=data_days) - query_sessions = 'session_date >="' + past_date.strftime('%Y-%m-%d') +'"' - query_sessions += 'and session_date < "' + today.strftime('%Y-%m-%d') +'"' + query_sessions = 'session_date >="' + past_date.strftime("%Y-%m-%d") + '"' + query_sessions += 'and session_date < "' + today.strftime("%Y-%m-%d") + '"' query_sessions += ' and subject_fullname not like "testuser%"' - session_df = pd.DataFrame((acquisition.Session & query_sessions).fetch(as_dict=True)) + session_df = pd.DataFrame( + (acquisition.Session & query_sessions).fetch(as_dict=True) + ) - #Filter subjects/rigs with >= min_sessions - num_sessions_df = session_df.groupby(type).agg({'session_date': [('total_sessions', 'count')]}) + # Filter subjects/rigs with >= min_sessions + num_sessions_df = session_df.groupby(type).agg( + {"session_date": [("total_sessions", "count")]} + ) num_sessions_df.columns = num_sessions_df.columns.droplevel() num_sessions_df = num_sessions_df.reset_index() - num_sessions_df = num_sessions_df.loc[num_sessions_df['total_sessions'] >= min_sessions, :] + num_sessions_df = num_sessions_df.loc[ + num_sessions_df["total_sessions"] >= min_sessions, : + ] session_df = session_df.merge(num_sessions_df, on=type) # Get key list of sessions for further (behavior) querying key_fields = djs.get_primary_key_fields(acquisition.Session) key_list = session_df.loc[:, key_fields] - key_list = key_list.to_dict('records') - + key_list = key_list.to_dict("records") - return session_df, key_list \ No newline at end of file + return session_df, key_list diff --git a/u19_pipeline/alert_system/behavior_metrics.py b/u19_pipeline/alert_system/behavior_metrics.py index 2c259b5c..6cc64038 100644 --- a/u19_pipeline/alert_system/behavior_metrics.py +++ b/u19_pipeline/alert_system/behavior_metrics.py @@ -1,72 +1,100 @@ - - - class BehaviorMetrics: - @staticmethod def get_bias_from_trial_df(trials_df, return_all_metrics=False): - ''' + """ From a trial by trial df of multiple sessions, calculate a per session bias - ''' + """ original_columns = trials_df.columns.tolist() - sort_columns = ['subject_fullname', 'session_date', 'session_number', 'block', 'trial_idx'] - session_columns = ['subject_fullname', 'session_date', 'session_number'] - trials_df = trials_df.sort_values(by=sort_columns, ascending=[True, False, False, True, True]) + sort_columns = [ + "subject_fullname", + "session_date", + "session_number", + "block", + "trial_idx", + ] + session_columns = ["subject_fullname", "session_date", "session_number"] + trials_df = trials_df.sort_values( + by=sort_columns, ascending=[True, False, False, True, True] + ) trials_df = trials_df.reset_index(drop=True) # Left & right trials as integers - trials_df['left_trial'] = (trials_df['trial_type'] == 'L').astype(int) - trials_df['right_trial'] = (trials_df['trial_type'] == 'R').astype(int) - trials_df['trial'] = 1 + trials_df["left_trial"] = (trials_df["trial_type"] == "L").astype(int) + trials_df["right_trial"] = (trials_df["trial_type"] == "R").astype(int) + trials_df["trial"] = 1 # Cumulative sum of Left & right trials as integers per session - trials_df['cum_left_trials'] = trials_df.groupby(session_columns)['left_trial'].cumsum() - trials_df['cum_right_trials'] = trials_df.groupby(session_columns)['right_trial'].cumsum() - trials_df['cum_trials'] = trials_df.groupby(session_columns)['trial'].cumsum() + trials_df["cum_left_trials"] = trials_df.groupby(session_columns)[ + "left_trial" + ].cumsum() + trials_df["cum_right_trials"] = trials_df.groupby(session_columns)[ + "right_trial" + ].cumsum() + trials_df["cum_trials"] = trials_df.groupby(session_columns)["trial"].cumsum() # Correct trials per side as integers - trials_df['correct_trial'] = (trials_df['trial_type'] == trials_df['choice']).astype(int) - trials_df['correct_left'] = ((trials_df['correct_trial'] == 1) & (trials_df['trial_type'] == 'L')).astype(int) - trials_df['correct_right'] = ((trials_df['correct_trial'] == 1) & (trials_df['trial_type'] == 'R')).astype(int) + trials_df["correct_trial"] = ( + trials_df["trial_type"] == trials_df["choice"] + ).astype(int) + trials_df["correct_left"] = ( + (trials_df["correct_trial"] == 1) & (trials_df["trial_type"] == "L") + ).astype(int) + trials_df["correct_right"] = ( + (trials_df["correct_trial"] == 1) & (trials_df["trial_type"] == "R") + ).astype(int) # Cumulative sum of per side correct trials - trials_df['cum_correct_left_trials'] = trials_df.groupby(session_columns)['correct_left'].cumsum() - trials_df['cum_correct_right_trials'] = trials_df.groupby(session_columns)['correct_right'].cumsum() - trials_df['cum_correct_trials'] = trials_df.groupby(session_columns)['correct_trial'].cumsum() + trials_df["cum_correct_left_trials"] = trials_df.groupby(session_columns)[ + "correct_left" + ].cumsum() + trials_df["cum_correct_right_trials"] = trials_df.groupby(session_columns)[ + "correct_right" + ].cumsum() + trials_df["cum_correct_trials"] = trials_df.groupby(session_columns)[ + "correct_trial" + ].cumsum() # Get only last trial count - trials_df = trials_df.loc[~trials_df.duplicated(subset=session_columns, keep='last'), :] + trials_df = trials_df.loc[ + ~trials_df.duplicated(subset=session_columns, keep="last"), : + ] trials_df = trials_df.reset_index(drop=True) # Calculate bias - trials_df['bias'] = (trials_df['cum_correct_right_trials'] / trials_df['cum_right_trials']) - (trials_df['cum_correct_left_trials'] / trials_df['cum_left_trials']) - + trials_df["bias"] = ( + trials_df["cum_correct_right_trials"] / trials_df["cum_right_trials"] + ) - (trials_df["cum_correct_left_trials"] / trials_df["cum_left_trials"]) + if return_all_metrics: bias_df = trials_df else: - bias_df = trials_df[original_columns + ['bias']] + bias_df = trials_df[original_columns + ["bias"]] return bias_df - @staticmethod def get_zscore_metric_session_df(session_df, metric, groupby_column): - ''' + """ Get zscore from a "generic" metric of a session dataframe - ''' + """ - #Sort by subject, date and session number - session_df = session_df.sort_values(by=['subject_fullname', 'session_date', 'session_number'], ascending=[True, False, False]) + # Sort by subject, date and session number + session_df = session_df.sort_values( + by=["subject_fullname", "session_date", "session_number"], + ascending=[True, False, False], + ) session_df = session_df.reset_index(drop=True) - # Lables for avg an std metric columns - avg_metric_l = 'avg_'+ metric - std_metric_l = 'std_'+metric - z_score_metric_l = 'z_score_'+metric + # Lables for avg an std metric columns + avg_metric_l = "avg_" + metric + std_metric_l = "std_" + metric + z_score_metric_l = "z_score_" + metric # Get mean and std for the selected metric - avg_df = session_df.groupby(groupby_column).agg({metric: [(avg_metric_l, 'mean'), (std_metric_l, 'std')]}) + avg_df = session_df.groupby(groupby_column).agg( + {metric: [(avg_metric_l, "mean"), (std_metric_l, "std")]} + ) avg_df.columns = avg_df.columns.droplevel() avg_df = avg_df.reset_index() @@ -74,6 +102,8 @@ def get_zscore_metric_session_df(session_df, metric, groupby_column): session_df = session_df.merge(avg_df) # Calculate proper zscore - session_df[z_score_metric_l] = (session_df[metric] - session_df[avg_metric_l]) / session_df[std_metric_l] + session_df[z_score_metric_l] = ( + session_df[metric] - session_df[avg_metric_l] + ) / session_df[std_metric_l] - return session_df \ No newline at end of file + return session_df diff --git a/u19_pipeline/alert_system/cronjob_alert.py b/u19_pipeline/alert_system/cronjob_alert.py index 42cf1311..df38e931 100644 --- a/u19_pipeline/alert_system/cronjob_alert.py +++ b/u19_pipeline/alert_system/cronjob_alert.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file @@ -11,7 +10,7 @@ import u19_pipeline.alert_system.log_deletion.old_log_deletion as old import u19_pipeline.alert_system.noDB_backup_creation.noDB_backup_creation_script as noDBbcs -#mas.main_alert_system() +# mas.main_alert_system() old.main_old_log_deletion() lssd.main_live_session_stats_deletion() noDBbcs.main_noDB_backup() diff --git a/u19_pipeline/alert_system/custom_alerts/__init__.py b/u19_pipeline/alert_system/custom_alerts/__init__.py index eb1f6e67..f77af49c 100644 --- a/u19_pipeline/alert_system/custom_alerts/__init__.py +++ b/u19_pipeline/alert_system/custom_alerts/__init__.py @@ -1,3 +1,3 @@ import pkgutil -__path__ = pkgutil.extend_path(__path__, __name__) \ No newline at end of file +__path__ = pkgutil.extend_path(__path__, __name__) diff --git a/u19_pipeline/alert_system/custom_alerts/braininit_storage.py b/u19_pipeline/alert_system/custom_alerts/braininit_storage.py index 609e433b..641b4bc5 100644 --- a/u19_pipeline/alert_system/custom_alerts/braininit_storage.py +++ b/u19_pipeline/alert_system/custom_alerts/braininit_storage.py @@ -1,4 +1,3 @@ - import subprocess import datajoint as dj @@ -6,30 +5,32 @@ # Slack Configuration dictionary slack_configuration_dictionary = { - 'slack_notification_channel': ['custom_alerts'], - 'slack_users_channel': [] + "slack_notification_channel": ["custom_alerts"], + "slack_users_channel": [], } + def main(): conf = dj.config - braininit_path = conf['custom']['root_data_dir'][0] + braininit_path = conf["custom"]["root_data_dir"][0] - #braininit_path = lab.Path().get_local_path2('braininit/Data').as_posix() - command = "df "+ braininit_path + " | tail -1 | awk '{print $4}'" + # braininit_path = lab.Path().get_local_path2('braininit/Data').as_posix() + command = "df " + braininit_path + " | tail -1 | awk '{print $4}'" # a = os.popen(command).read() storage_left = subprocess.check_output(command, shell=True) storage_left = storage_left.decode().strip() storage_left_kb = int(storage_left) - storage_left_tb = storage_left_kb/1024/1024/1024 + storage_left_tb = storage_left_kb / 1024 / 1024 / 1024 if storage_left_tb < 4: - data = {'alert_message': ['Very little space left in braininit'], 'space(tb)': [str(storage_left_tb)]} + data = { + "alert_message": ["Very little space left in braininit"], + "space(tb)": [str(storage_left_tb)], + } alert_df = pd.DataFrame.from_dict(data) else: alert_df = pd.DataFrame([]) return alert_df - - diff --git a/u19_pipeline/alert_system/custom_alerts/rig_bias.py b/u19_pipeline/alert_system/custom_alerts/rig_bias.py index bffb7404..39656940 100644 --- a/u19_pipeline/alert_system/custom_alerts/rig_bias.py +++ b/u19_pipeline/alert_system/custom_alerts/rig_bias.py @@ -1,4 +1,3 @@ - import datetime import datajoint as dj @@ -9,63 +8,85 @@ import u19_pipeline.alert_system.behavior_metrics as bm # Slack Configuration dictionary -slack_configuration_dictionary = { - 'slack_notification_channel': ['custom_alerts'] -} +slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]} zscore_alert = 2 + + def main(): - ''' + """ Main function for subject "num trials & bias" alert - ''' + """ # Get sessions - _, rig_session_key_list = asu.get_acquisition_data_alert_system(type='session_location') + _, rig_session_key_list = asu.get_acquisition_data_alert_system( + type="session_location" + ) # Get trials - behavior = dj.create_virtual_module('behavior', dj.config['custom']['database.prefix']+'behavior') - acquisition = dj.create_virtual_module('acquisition', dj.config['custom']['database.prefix']+'acquisition') - - rig_trial_df = pd.DataFrame((behavior.TowersBlock.Trial * acquisition.SessionStarted.proj('session_location') \ - & rig_session_key_list).fetch('KEY', 'trial_type', 'choice', 'session_location', as_dict=True)) + behavior = dj.create_virtual_module( + "behavior", dj.config["custom"]["database.prefix"] + "behavior" + ) + acquisition = dj.create_virtual_module( + "acquisition", dj.config["custom"]["database.prefix"] + "acquisition" + ) + + rig_trial_df = pd.DataFrame( + ( + behavior.TowersBlock.Trial + * acquisition.SessionStarted.proj("session_location") + & rig_session_key_list + ).fetch("KEY", "trial_type", "choice", "session_location", as_dict=True) + ) # Get zscores for bias bias_df = bm.BehaviorMetrics.get_bias_from_trial_df(rig_trial_df) - bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df(bias_df, 'bias', 'subject_fullname') + bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df( + bias_df, "bias", "subject_fullname" + ) # Filter df for today today = datetime.date.today() - datetime.timedelta(days=1) - bias_df = bias_df.loc[bias_df['session_date'] == today, :] - bias_df['abs_z_score_bias'] = np.abs(bias_df['z_score_bias']) + bias_df = bias_df.loc[bias_df["session_date"] == today, :] + bias_df["abs_z_score_bias"] = np.abs(bias_df["z_score_bias"]) bias_df = bias_df.reset_index(drop=True) # Filter if today we got > 3 zscore of trials for a session - bias_df = bias_df.loc[bias_df['abs_z_score_bias'] >= zscore_alert, :] + bias_df = bias_df.loc[bias_df["abs_z_score_bias"] >= zscore_alert, :] # Get sign of bias (only group subjects with bias to same side) - bias_df['sign_bias'] = np.sign(bias_df['z_score_bias']) - bias_df['sign_bias'] = bias_df['sign_bias'].astype(int) - - #Count how many subjects were biassed by rig - bias_location = bias_df.groupby(['session_location', 'sign_bias']).agg({'session_location': [('num_subjects', 'size')],\ - 'subject_fullname': [('subject_fullnames', lambda x: ','.join(x))]}) + bias_df["sign_bias"] = np.sign(bias_df["z_score_bias"]) + bias_df["sign_bias"] = bias_df["sign_bias"].astype(int) + + # Count how many subjects were biassed by rig + bias_location = bias_df.groupby(["session_location", "sign_bias"]).agg( + { + "session_location": [("num_subjects", "size")], + "subject_fullname": [("subject_fullnames", lambda x: ",".join(x))], + } + ) bias_location.columns = bias_location.columns.droplevel() bias_location = bias_location.reset_index() - #Filter if there were subjects biased to different sides - bias_location2 = bias_location.groupby(['session_location']).agg({'session_location': [('num_bias_sides', 'size')]}) + # Filter if there were subjects biased to different sides + bias_location2 = bias_location.groupby(["session_location"]).agg( + {"session_location": [("num_bias_sides", "size")]} + ) bias_location2.columns = bias_location2.columns.droplevel() bias_location2 = bias_location2.reset_index() - bias_location2 = bias_location2.loc[bias_location2['num_bias_sides'] == 1, :] - bias_location = bias_location.merge(bias_location2, on='session_location') + bias_location2 = bias_location2.loc[bias_location2["num_bias_sides"] == 1, :] + bias_location = bias_location.merge(bias_location2, on="session_location") # Only alert if more than 1 subject was biased today - bias_location = bias_location.loc[bias_location['num_subjects'] > 1, :] + bias_location = bias_location.loc[bias_location["num_subjects"] > 1, :] # Filter columns and set message - columns_alert = ['session_location', 'sign_bias', 'num_subjects', 'subject_fullnames'] + columns_alert = [ + "session_location", + "sign_bias", + "num_subjects", + "subject_fullnames", + ] bias_location = bias_location[columns_alert] - bias_location['alert_message'] = 'Multiple subjects were biased in this rig' + bias_location["alert_message"] = "Multiple subjects were biased in this rig" return bias_location - - diff --git a/u19_pipeline/alert_system/custom_alerts/rig_trial.py b/u19_pipeline/alert_system/custom_alerts/rig_trial.py index a662cef0..1e5c186f 100644 --- a/u19_pipeline/alert_system/custom_alerts/rig_trial.py +++ b/u19_pipeline/alert_system/custom_alerts/rig_trial.py @@ -1,4 +1,3 @@ - import datetime import numpy as np @@ -7,54 +6,85 @@ import u19_pipeline.alert_system.behavior_metrics as bm # Slack Configuration dictionary -slack_configuration_dictionary = { - 'slack_notification_channel': ['custom_alerts'] -} +slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]} zscore_alert = 2 + + def main(): - ''' + """ Main function for subject "num trials & bias" alert - ''' + """ # Get sessions - subject_rig_df, _ = asu.get_acquisition_data_alert_system(type='session_location') + subject_rig_df, _ = asu.get_acquisition_data_alert_system(type="session_location") # Get zscores for each session (group by rig) - subject_rig_df = bm.BehaviorMetrics.get_zscore_metric_session_df(subject_rig_df, 'num_trials', 'session_location') - - # Get only todays data + subject_rig_df = bm.BehaviorMetrics.get_zscore_metric_session_df( + subject_rig_df, "num_trials", "session_location" + ) + + # Get only todays data today = datetime.date.today() - datetime.timedelta(days=1) - subject_rig_df = subject_rig_df.loc[subject_rig_df['session_date'] == today, :] + subject_rig_df = subject_rig_df.loc[subject_rig_df["session_date"] == today, :] # Filter if today we got > 3 zscore of trials for a session - subject_rig_df['abs_z_score_num_trials'] = np.abs(subject_rig_df['z_score_num_trials']) - num_trials_alert_df = subject_rig_df.loc[subject_rig_df['abs_z_score_num_trials'] >= zscore_alert, :].copy(deep=True) + subject_rig_df["abs_z_score_num_trials"] = np.abs( + subject_rig_df["z_score_num_trials"] + ) + num_trials_alert_df = subject_rig_df.loc[ + subject_rig_df["abs_z_score_num_trials"] >= zscore_alert, : + ].copy(deep=True) # Get sign of zscpre (only group subjects with same deviation) - num_trials_alert_df['sign_zscore'] = np.sign(num_trials_alert_df['z_score_num_trials']) - num_trials_alert_df['sign_zscore'] = num_trials_alert_df['sign_zscore'].astype(int) + num_trials_alert_df["sign_zscore"] = np.sign( + num_trials_alert_df["z_score_num_trials"] + ) + num_trials_alert_df["sign_zscore"] = num_trials_alert_df["sign_zscore"].astype(int) - #Count how many subjects were biassed by rig - num_trials_alert_location_df = num_trials_alert_df.groupby(['session_location', 'sign_zscore']).agg({'session_location': [('num_subjects', 'size')],\ - 'subject_fullname': [('subject_fullnames', lambda x: ','.join(x))]}) - num_trials_alert_location_df.columns = num_trials_alert_location_df.columns.droplevel() + # Count how many subjects were biassed by rig + num_trials_alert_location_df = num_trials_alert_df.groupby( + ["session_location", "sign_zscore"] + ).agg( + { + "session_location": [("num_subjects", "size")], + "subject_fullname": [("subject_fullnames", lambda x: ",".join(x))], + } + ) + num_trials_alert_location_df.columns = ( + num_trials_alert_location_df.columns.droplevel() + ) num_trials_alert_location_df = num_trials_alert_location_df.reset_index() - #Filter if there were subjects with a lot of trials and very little trials - num_trials_alert_location_df2 = num_trials_alert_location_df.groupby(['session_location']).agg({'session_location': [('num_bias_sides', 'size')]}) - num_trials_alert_location_df2.columns = num_trials_alert_location_df2.columns.droplevel() + # Filter if there were subjects with a lot of trials and very little trials + num_trials_alert_location_df2 = num_trials_alert_location_df.groupby( + ["session_location"] + ).agg({"session_location": [("num_bias_sides", "size")]}) + num_trials_alert_location_df2.columns = ( + num_trials_alert_location_df2.columns.droplevel() + ) num_trials_alert_location_df2 = num_trials_alert_location_df2.reset_index() - num_trials_alert_location_df2 = num_trials_alert_location_df2.loc[num_trials_alert_location_df2['num_bias_sides'] == 1, :] - num_trials_alert_location_df = num_trials_alert_location_df.merge(num_trials_alert_location_df2, on='session_location') + num_trials_alert_location_df2 = num_trials_alert_location_df2.loc[ + num_trials_alert_location_df2["num_bias_sides"] == 1, : + ] + num_trials_alert_location_df = num_trials_alert_location_df.merge( + num_trials_alert_location_df2, on="session_location" + ) # Only alert if more than 1 subject was biased today - num_trials_alert_location_df = num_trials_alert_location_df.loc[num_trials_alert_location_df['num_subjects'] > 1, :] + num_trials_alert_location_df = num_trials_alert_location_df.loc[ + num_trials_alert_location_df["num_subjects"] > 1, : + ] # Filter columns and set message - columns_alert = ['session_location', 'sign_zscore', 'num_subjects', 'subject_fullnames'] + columns_alert = [ + "session_location", + "sign_zscore", + "num_subjects", + "subject_fullnames", + ] num_trials_alert_location_df = num_trials_alert_location_df[columns_alert] - num_trials_alert_location_df['alert_message'] = 'Multiple subjects had abnormal number of trials in this rig' + num_trials_alert_location_df["alert_message"] = ( + "Multiple subjects had abnormal number of trials in this rig" + ) return num_trials_alert_location_df - - diff --git a/u19_pipeline/alert_system/custom_alerts/subject_bias.py b/u19_pipeline/alert_system/custom_alerts/subject_bias.py index 3a6037da..b768c028 100644 --- a/u19_pipeline/alert_system/custom_alerts/subject_bias.py +++ b/u19_pipeline/alert_system/custom_alerts/subject_bias.py @@ -1,4 +1,3 @@ - import datetime import datajoint as dj @@ -9,41 +8,60 @@ import u19_pipeline.alert_system.behavior_metrics as bm # Slack Configuration dictionary -slack_configuration_dictionary = { - 'slack_notification_channel': ['custom_alerts'] -} +slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]} zscore_alert = 2 + + def main(): - ''' + """ Main function for subject "num trials & bias" alert - ''' + """ # Get sessions - _, subject_session_key_list = asu.get_acquisition_data_alert_system(type='subject_fullname') + _, subject_session_key_list = asu.get_acquisition_data_alert_system( + type="subject_fullname" + ) # Get trials - behavior = dj.create_virtual_module('behavior', dj.config['custom']['database.prefix']+'behavior') - acquisition = dj.create_virtual_module('acquisition', dj.config['custom']['database.prefix']+'acquisition') + behavior = dj.create_virtual_module( + "behavior", dj.config["custom"]["database.prefix"] + "behavior" + ) + acquisition = dj.create_virtual_module( + "acquisition", dj.config["custom"]["database.prefix"] + "acquisition" + ) - subject_trial_df = pd.DataFrame((behavior.TowersBlock.Trial * acquisition.SessionStarted & subject_session_key_list).fetch('KEY', 'trial_type', 'choice', 'session_location', as_dict=True)) + subject_trial_df = pd.DataFrame( + ( + behavior.TowersBlock.Trial * acquisition.SessionStarted + & subject_session_key_list + ).fetch("KEY", "trial_type", "choice", "session_location", as_dict=True) + ) # Get zscores for bias bias_df = bm.BehaviorMetrics.get_bias_from_trial_df(subject_trial_df) - bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df(bias_df, 'bias', 'subject_fullname') + bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df( + bias_df, "bias", "subject_fullname" + ) # Filter df for todays alert today = datetime.date.today() - datetime.timedelta(days=1) - bias_df = bias_df.loc[bias_df['session_date'] == today, :] - bias_df['abs_z_score_bias'] = np.abs(bias_df['z_score_bias']) + bias_df = bias_df.loc[bias_df["session_date"] == today, :] + bias_df["abs_z_score_bias"] = np.abs(bias_df["z_score_bias"]) # Filter if today we got > 3 zscore of trials for a session - alert_bias_df = bias_df.loc[bias_df['abs_z_score_bias'] >= zscore_alert, :] + alert_bias_df = bias_df.loc[bias_df["abs_z_score_bias"] >= zscore_alert, :] - columns_alert = ['subject_fullname', 'session_date', 'session_number', 'session_location', 'avg_bias', 'bias', 'z_score_bias'] + columns_alert = [ + "subject_fullname", + "session_date", + "session_number", + "session_location", + "avg_bias", + "bias", + "z_score_bias", + ] alert_bias_df = alert_bias_df[columns_alert] - alert_bias_df['alert_message'] = 'Session had abnormal bias' + alert_bias_df["alert_message"] = "Session had abnormal bias" return alert_bias_df - - diff --git a/u19_pipeline/alert_system/custom_alerts/subject_trial.py b/u19_pipeline/alert_system/custom_alerts/subject_trial.py index b11620df..13888011 100644 --- a/u19_pipeline/alert_system/custom_alerts/subject_trial.py +++ b/u19_pipeline/alert_system/custom_alerts/subject_trial.py @@ -1,4 +1,3 @@ - import datetime import numpy as np @@ -7,34 +6,50 @@ import u19_pipeline.alert_system.behavior_metrics as bm # Slack Configuration dictionary -slack_configuration_dictionary = { - 'slack_notification_channel': ['custom_alerts'] -} +slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]} zscore_alert = 2 + + def main(): - ''' + """ Main function for subject "num trials & bias" alert - ''' + """ # Get sessions - subject_session_df, _ = asu.get_acquisition_data_alert_system(type='subject_fullname') + subject_session_df, _ = asu.get_acquisition_data_alert_system( + type="subject_fullname" + ) # Get zscores for num_trials - subject_session_df = bm.BehaviorMetrics.get_zscore_metric_session_df(subject_session_df, 'num_trials', 'subject_fullname') + subject_session_df = bm.BehaviorMetrics.get_zscore_metric_session_df( + subject_session_df, "num_trials", "subject_fullname" + ) # Filter df for todays alert today = datetime.date.today() - datetime.timedelta(days=1) - subject_session_df = subject_session_df.loc[subject_session_df['session_date'] == today, :] - subject_session_df['abs_z_score_num_trials'] = np.abs(subject_session_df['z_score_num_trials']) + subject_session_df = subject_session_df.loc[ + subject_session_df["session_date"] == today, : + ] + subject_session_df["abs_z_score_num_trials"] = np.abs( + subject_session_df["z_score_num_trials"] + ) # Filter if today we got > 3 zscore of trials for a session - alert_subjtect_trial_df = subject_session_df.loc[subject_session_df['abs_z_score_num_trials'] >= zscore_alert, :] - - columns_alert = ['subject_fullname', 'session_date', 'session_number', 'session_location', 'avg_num_trials', 'num_trials', 'z_score_num_trials'] + alert_subjtect_trial_df = subject_session_df.loc[ + subject_session_df["abs_z_score_num_trials"] >= zscore_alert, : + ] + + columns_alert = [ + "subject_fullname", + "session_date", + "session_number", + "session_location", + "avg_num_trials", + "num_trials", + "z_score_num_trials", + ] alert_subjtect_trial_df = alert_subjtect_trial_df[columns_alert] - alert_subjtect_trial_df['alert_message'] = 'Session had abnormal number of trials' + alert_subjtect_trial_df["alert_message"] = "Session had abnormal number of trials" return alert_subjtect_trial_df - - diff --git a/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py b/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py index 3fd32f10..9f7c6c4f 100644 --- a/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py +++ b/u19_pipeline/alert_system/live_monitor_alert/cronjob_live_monitor_alert.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file @@ -10,4 +9,3 @@ import u19_pipeline.alert_system.live_monitor_alert.live_monitor_alert as lma lma.main_live_monitor_alert() - diff --git a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py index 4bdd7a1a..7350af1f 100644 --- a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py +++ b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py @@ -8,58 +8,67 @@ import u19_pipeline.utils.slack_utils as su MINUTES_ALERT = 20 -SECONDS_ALERT = MINUTES_ALERT*60 +SECONDS_ALERT = MINUTES_ALERT * 60 MIN_SESSIONS_COMPLETED = 3 slack_configuration_dictionary = { - 'slack_notification_channel': ['rig_training_error_notification'] + "slack_notification_channel": ["rig_training_error_notification"] } -def slack_alert_message_format_live_stats(alert_dictionary1, alert_dictionary2, time_no_response): - 'Format dictionaries for live monitor slack alert json' +def slack_alert_message_format_live_stats( + alert_dictionary1, alert_dictionary2, time_no_response +): + "Format dictionaries for live monitor slack alert json" now = datetime.now() - datestr = now.strftime('%d-%b-%Y %H:%M:%S') + datestr = now.strftime("%d-%b-%Y %H:%M:%S") msep = dict() - msep['type'] = "divider" + msep["type"] = "divider" - #Title# + # Title# m1 = dict() - m1['type'] = 'section' + m1["type"] = "section" m1_1 = dict() m1_1["type"] = "mrkdwn" - m1_1["text"] = ':rotating_light: *Live Monitor Alert* on ' + datestr + '\n' +\ - 'More than ' + str(time_no_response) + ' min without new trial' + '\n' - m1['text'] = m1_1 - - #Info# + m1_1["text"] = ( + ":rotating_light: *Live Monitor Alert* on " + + datestr + + "\n" + + "More than " + + str(time_no_response) + + " min without new trial" + + "\n" + ) + m1["text"] = m1_1 + + # Info# m2 = dict() - m2['type'] = 'section' + m2["type"] = "section" m2_1 = dict() m2_1["type"] = "mrkdwn" - m2_1["text"] = '*Session Reported:*' + '\n' + m2_1["text"] = "*Session Reported:*" + "\n" for key in alert_dictionary1.keys(): - m2_1["text"] += '*' + key + '* : ' + str(alert_dictionary1[key]) + '\n' - m2_1["text"] += '\n' - m2['text'] = m2_1 + m2_1["text"] += "*" + key + "* : " + str(alert_dictionary1[key]) + "\n" + m2_1["text"] += "\n" + m2["text"] = m2_1 m4 = dict() - m4['type'] = 'section' + m4["type"] = "section" m4_1 = dict() m4_1["type"] = "mrkdwn" - m4_1["text"] = '*Last Stats Reported*:' + '\n' + m4_1["text"] = "*Last Stats Reported*:" + "\n" for key in alert_dictionary2.keys(): - m4_1["text"] += '*' + key + '* : ' + str(alert_dictionary2[key]) + '\n' - m4_1["text"] += '\n' - m4['text'] = m4_1 + m4_1["text"] += "*" + key + "* : " + str(alert_dictionary2[key]) + "\n" + m4_1["text"] += "\n" + m4["text"] = m4_1 message = dict() - message['blocks'] = [m1,msep,m2,msep,m4,msep] - message['text'] = 'Live Monitor Alert' + message["blocks"] = [m1, msep, m2, msep, m4, msep] + message["text"] = "Live Monitor Alert" return message @@ -68,178 +77,289 @@ def main_live_monitor_alert(): # Connect to DB and create virtual modules dj.conn() - acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition') - lab = dj.create_virtual_module('lab', 'u19_lab') - subject = dj.create_virtual_module('subject', 'u19_subject') + acquisition = dj.create_virtual_module("acquisition", "u19_acquisition") + lab = dj.create_virtual_module("lab", "u19_lab") + subject = dj.create_virtual_module("subject", "u19_subject") # Query today's started sessions that are not finished query = {} - query['session_date'] = datetime.today().strftime('%Y-%m-%d') - query['is_finished'] = 0 + query["session_date"] = datetime.today().strftime("%Y-%m-%d") + query["is_finished"] = 0 - #Only look for sessions started in the last 1:30 - last_time_start = datetime.now(tz=ZoneInfo('America/New_York')) - timedelta(hours=2,minutes=30) - last_time_start = last_time_start.replace(tzinfo=None).strftime('%Y-%m-%d %H:%M:%S') + # Only look for sessions started in the last 1:30 + last_time_start = datetime.now(tz=ZoneInfo("America/New_York")) - timedelta( + hours=2, minutes=30 + ) + last_time_start = last_time_start.replace(tzinfo=None).strftime("%Y-%m-%d %H:%M:%S") - query_started_recently = "session_start_time > '" + last_time_start + "'" - sessions = pd.DataFrame((acquisition.SessionStarted & query & query_started_recently).fetch('KEY','session_location','session_start_time',as_dict=True)) - #sessions = sessions.loc[~sessions['subject_fullname'].str.startswith('testuser'),:] + query_started_recently = "session_start_time > '" + last_time_start + "'" + sessions = pd.DataFrame( + (acquisition.SessionStarted & query & query_started_recently).fetch( + "KEY", "session_location", "session_start_time", as_dict=True + ) + ) + # sessions = sessions.loc[~sessions['subject_fullname'].str.startswith('testuser'),:] - #print('sessions STARTED RECENTLY\n', sessions) + # print('sessions STARTED RECENTLY\n', sessions) if sessions.shape[0] > 0: - - #If more than one "not finished" session in same rig, grab the last one started - sessions2 = sessions.groupby('session_location').agg({'session_start_time': [('session_start_time', 'max')]}) + # If more than one "not finished" session in same rig, grab the last one started + sessions2 = sessions.groupby("session_location").agg( + {"session_start_time": [("session_start_time", "max")]} + ) sessions2.columns = sessions2.columns.droplevel() sessions2 = sessions2.reset_index() - sessions = pd.merge(sessions, sessions2, on=['session_location', 'session_start_time']) - sessions = sessions.drop(columns=['session_location']) + sessions = pd.merge( + sessions, sessions2, on=["session_location", "session_start_time"] + ) + sessions = sessions.drop(columns=["session_location"]) sessions = sessions.reset_index(drop=True) - print('Last session started on rig\n', sessions) + print("Last session started on rig\n", sessions) - #Only analyze sessions that have not been reported - query_reported = {} - query_reported['session_date'] = datetime.today().strftime('%Y-%m-%d') - sessions_reported = pd.DataFrame((acquisition.ReportedLiveSessionStats & query_reported).fetch('KEY', as_dict = True)) + # Only analyze sessions that have not been reported + query_reported = {} + query_reported["session_date"] = datetime.today().strftime("%Y-%m-%d") + sessions_reported = pd.DataFrame( + (acquisition.ReportedLiveSessionStats & query_reported).fetch( + "KEY", as_dict=True + ) + ) if sessions_reported.shape[0] > 0: - - sessions = pd.merge(sessions,sessions_reported, how='left', indicator=True) - sessions = sessions.loc[sessions['_merge'] == 'left_only'] - sessions = sessions.drop(columns='_merge') + sessions = pd.merge(sessions, sessions_reported, how="left", indicator=True) + sessions = sessions.loc[sessions["_merge"] == "left_only"] + sessions = sessions.drop(columns="_merge") sessions = sessions.reset_index(drop=True) - #print('Sessions not reported\n', sessions) + # print('Sessions not reported\n', sessions) - #Only analyze sessions subjects > NUM_SESSIONS_COMPLETED have not been reported + # Only analyze sessions subjects > NUM_SESSIONS_COMPLETED have not been reported if sessions.shape[0] > 0: - - query_subjects = "subject_fullname in ('"+ "', '".join(sessions['subject_fullname']) + "')" - - count_sessions_table = (subject.Subject).aggr((acquisition.Session & query_subjects), num_sessions="count(subject_fullname)") + query_subjects = ( + "subject_fullname in ('" + "', '".join(sessions["subject_fullname"]) + "')" + ) + + count_sessions_table = (subject.Subject).aggr( + (acquisition.Session & query_subjects), + num_sessions="count(subject_fullname)", + ) count_sessions_df = pd.DataFrame(count_sessions_table.fetch(as_dict=True)) - sessions = pd.merge(sessions,count_sessions_df, how='left') - sessions['num_sessions'] = sessions['num_sessions'].fillna(0) - - #print('Subjects with min_num_sessions\n', sessions) + sessions = pd.merge(sessions, count_sessions_df, how="left") + sessions["num_sessions"] = sessions["num_sessions"].fillna(0) - sessions = sessions.loc[sessions['num_sessions']>= MIN_SESSIONS_COMPLETED, :] + # print('Subjects with min_num_sessions\n', sessions) + sessions = sessions.loc[sessions["num_sessions"] >= MIN_SESSIONS_COMPLETED, :] if sessions.shape[0] > 0: - # Query last live stat from the started sessions - query_live_stats = sessions.to_dict('records') - - #Last non violation trial in sessions + query_live_stats = sessions.to_dict("records") + + # Last non violation trial in sessions query_no_violation_trial = dict() - query_no_violation_trial['violation_trial'] = 0 - lss_nvio = acquisition.SessionStarted.aggr((acquisition.LiveSessionStats & query_no_violation_trial).proj('current_datetime'), current_datetime="max(current_datetime)") - live_stats_nvio = pd.DataFrame((lss_nvio & query_live_stats).fetch(as_dict=True)) - live_stats_nvio = live_stats_nvio.rename({'current_datetime': 'last_non_violation_trial'}, axis=1) - - #Last violation trial in sessions + query_no_violation_trial["violation_trial"] = 0 + lss_nvio = acquisition.SessionStarted.aggr( + (acquisition.LiveSessionStats & query_no_violation_trial).proj( + "current_datetime" + ), + current_datetime="max(current_datetime)", + ) + live_stats_nvio = pd.DataFrame( + (lss_nvio & query_live_stats).fetch(as_dict=True) + ) + live_stats_nvio = live_stats_nvio.rename( + {"current_datetime": "last_non_violation_trial"}, axis=1 + ) + + # Last violation trial in sessions query_violation_trial = dict() - query_violation_trial['violation_trial'] = 1 - lss_vio = acquisition.SessionStarted.aggr((acquisition.LiveSessionStats & query_violation_trial).proj('current_datetime'), current_datetime="max(current_datetime)") - live_stats_vio = pd.DataFrame((lss_vio & query_live_stats & query_violation_trial).fetch(as_dict=True)) - live_stats_vio = live_stats_vio.rename({'current_datetime': 'last_violation_trial'}, axis=1) - + query_violation_trial["violation_trial"] = 1 + lss_vio = acquisition.SessionStarted.aggr( + (acquisition.LiveSessionStats & query_violation_trial).proj( + "current_datetime" + ), + current_datetime="max(current_datetime)", + ) + live_stats_vio = pd.DataFrame( + (lss_vio & query_live_stats & query_violation_trial).fetch(as_dict=True) + ) + live_stats_vio = live_stats_vio.rename( + {"current_datetime": "last_violation_trial"}, axis=1 + ) # Merge last violation trial and last non violation Trials from sessions if live_stats_nvio.shape[0] > 0 and live_stats_vio.shape[0] > 0: - live_stats = pd.merge(live_stats_nvio,live_stats_vio, how='outer') + live_stats = pd.merge(live_stats_nvio, live_stats_vio, how="outer") elif live_stats_nvio.shape[0] > 0: live_stats = live_stats_nvio.copy() - live_stats['last_violation_trial'] = pd.NaT + live_stats["last_violation_trial"] = pd.NaT elif live_stats_vio.shape[0] > 0: live_stats = live_stats_vio.copy() - live_stats['last_non_violation_trial'] = pd.NaT + live_stats["last_non_violation_trial"] = pd.NaT else: live_stats = pd.DataFrame() - # If there are any sessions with live stats if live_stats.shape[0] > 0: - - live_stats = pd.merge(sessions,live_stats, how='inner') - fake_date = pd.Timestamp('1900-01-01') + live_stats = pd.merge(sessions, live_stats, how="inner") + fake_date = pd.Timestamp("1900-01-01") # Filter sessions whose last trial info is greater than 300s - right_now_est = datetime.now(tz=ZoneInfo('America/New_York')) + right_now_est = datetime.now(tz=ZoneInfo("America/New_York")) right_now_est = right_now_est.replace(tzinfo=None) - live_stats['seconds_elapsed_last_stat_nvio'] = ((right_now_est- live_stats['last_non_violation_trial']).dt.total_seconds()).astype(int) - live_stats['alert_nvio'] = live_stats['seconds_elapsed_last_stat_nvio'] > SECONDS_ALERT + live_stats["seconds_elapsed_last_stat_nvio"] = ( + ( + right_now_est - live_stats["last_non_violation_trial"] + ).dt.total_seconds() + ).astype(int) + live_stats["alert_nvio"] = ( + live_stats["seconds_elapsed_last_stat_nvio"] > SECONDS_ALERT + ) + + live_stats["seconds_elapsed_session_started"] = ( + (right_now_est - live_stats["session_start_time"]).dt.total_seconds() + ).astype(int) + live_stats["alert_vio"] = ( + live_stats["seconds_elapsed_session_started"] > SECONDS_ALERT + ) + live_stats["alert_vio"] = live_stats["alert_vio"] & ( + pd.isna(live_stats["last_non_violation_trial"]) + ) + live_stats["alert_vio"] = live_stats["alert_vio"] & ( + ~pd.isna(live_stats["last_violation_trial"]) + ) - live_stats['seconds_elapsed_session_started'] = ((right_now_est- live_stats['session_start_time']).dt.total_seconds()).astype(int) - live_stats['alert_vio'] = live_stats['seconds_elapsed_session_started'] > SECONDS_ALERT - live_stats['alert_vio'] = live_stats['alert_vio'] & (pd.isna(live_stats['last_non_violation_trial'])) - live_stats['alert_vio'] = live_stats['alert_vio'] & (~pd.isna(live_stats['last_violation_trial'])) - print(live_stats.T) - live_stats = live_stats.loc[(live_stats['alert_nvio']==True) | (live_stats['alert_vio']==True),:] + live_stats = live_stats.loc[ + (live_stats["alert_nvio"] == True) | (live_stats["alert_vio"] == True), + :, + ] - #If there are any sessions to alert (more then SECONDS_ALERT) + # If there are any sessions to alert (more then SECONDS_ALERT) if live_stats.shape[0] > 0: - - live_stats['current_datetime'] = fake_date - live_stats.loc[live_stats['alert_vio']==True, 'current_datetime'] = live_stats.loc[live_stats['alert_vio']==True, 'last_violation_trial'] - live_stats.loc[live_stats['alert_nvio']==True, 'current_datetime'] = live_stats.loc[live_stats['alert_nvio']==True, 'last_non_violation_trial'] - - live_stats['seconds_elapsed_last_valid_stat'] = 0 - live_stats.loc[live_stats['alert_vio']==True, 'seconds_elapsed_last_valid_stat'] = live_stats.loc[live_stats['alert_vio']==True, 'seconds_elapsed_session_started'] - live_stats.loc[live_stats['alert_nvio']==True, 'seconds_elapsed_last_valid_stat'] = live_stats.loc[live_stats['alert_nvio']==True, 'seconds_elapsed_last_stat_nvio'] - - #get_session_info to alert (plus slack researcher) - query_live_stats_sessions = live_stats[['subject_fullname', 'session_date', 'session_number']].to_dict('records') - - session_data_df = pd.DataFrame(((lab.User.proj('slack') * subject.Subject.proj('user_id') *\ - acquisition.SessionStarted.proj('session_location')) & query_live_stats_sessions).fetch(as_dict=True)) - - session_data_df = session_data_df.rename({'slack': 'researcher'}, axis=1) - session_data_df['researcher'] = '<@'+ session_data_df['researcher'] + '>' - session_data_df = session_data_df[['researcher', 'subject_fullname', 'session_date', 'session_number']] - - #Query full live stat table - #session_stats = live_stats.copy() - #session_stats = session_stats.rename({'current_datetime': 'last_live_stat'}, axis=1) - query_live_stats = live_stats[['subject_fullname', 'session_date', 'session_number', 'current_datetime']].to_dict('records') - live_stats_mini = live_stats[['subject_fullname', 'session_date', 'session_number', 'seconds_elapsed_last_valid_stat']].copy() - - ls_full_df = pd.DataFrame((acquisition.LiveSessionStats & query_live_stats).fetch(as_dict=True)) - - ls_full_df = pd.merge(ls_full_df, live_stats_mini, on=['subject_fullname', 'session_date', 'session_number']) - ls_full_df = ls_full_df.drop(columns=['subject_fullname', 'session_date', 'session_number']) - ls_full_df = ls_full_df.rename({'current_datetime': 'last_trial_time'}, axis=1) - - - mid = ls_full_df['last_trial_time'] - ls_full_df = ls_full_df.drop(columns=['last_trial_time']) - ls_full_df.insert(0, 'last_trial_time', mid) - - ls_full_dict = ls_full_df.to_dict('records') + live_stats["current_datetime"] = fake_date + live_stats.loc[live_stats["alert_vio"] == True, "current_datetime"] = ( + live_stats.loc[ + live_stats["alert_vio"] == True, "last_violation_trial" + ] + ) + live_stats.loc[live_stats["alert_nvio"] == True, "current_datetime"] = ( + live_stats.loc[ + live_stats["alert_nvio"] == True, "last_non_violation_trial" + ] + ) + + live_stats["seconds_elapsed_last_valid_stat"] = 0 + live_stats.loc[ + live_stats["alert_vio"] == True, "seconds_elapsed_last_valid_stat" + ] = live_stats.loc[ + live_stats["alert_vio"] == True, "seconds_elapsed_session_started" + ] + live_stats.loc[ + live_stats["alert_nvio"] == True, "seconds_elapsed_last_valid_stat" + ] = live_stats.loc[ + live_stats["alert_nvio"] == True, "seconds_elapsed_last_stat_nvio" + ] + + # get_session_info to alert (plus slack researcher) + query_live_stats_sessions = live_stats[ + ["subject_fullname", "session_date", "session_number"] + ].to_dict("records") + + session_data_df = pd.DataFrame( + ( + ( + lab.User.proj("slack") + * subject.Subject.proj("user_id") + * acquisition.SessionStarted.proj("session_location") + ) + & query_live_stats_sessions + ).fetch(as_dict=True) + ) + + session_data_df = session_data_df.rename( + {"slack": "researcher"}, axis=1 + ) + session_data_df["researcher"] = ( + "<@" + session_data_df["researcher"] + ">" + ) + session_data_df = session_data_df[ + ["researcher", "subject_fullname", "session_date", "session_number"] + ] + + # Query full live stat table + # session_stats = live_stats.copy() + # session_stats = session_stats.rename({'current_datetime': 'last_live_stat'}, axis=1) + query_live_stats = live_stats[ + [ + "subject_fullname", + "session_date", + "session_number", + "current_datetime", + ] + ].to_dict("records") + live_stats_mini = live_stats[ + [ + "subject_fullname", + "session_date", + "session_number", + "seconds_elapsed_last_valid_stat", + ] + ].copy() + + ls_full_df = pd.DataFrame( + (acquisition.LiveSessionStats & query_live_stats).fetch( + as_dict=True + ) + ) + + ls_full_df = pd.merge( + ls_full_df, + live_stats_mini, + on=["subject_fullname", "session_date", "session_number"], + ) + ls_full_df = ls_full_df.drop( + columns=["subject_fullname", "session_date", "session_number"] + ) + ls_full_df = ls_full_df.rename( + {"current_datetime": "last_trial_time"}, axis=1 + ) + + mid = ls_full_df["last_trial_time"] + ls_full_df = ls_full_df.drop(columns=["last_trial_time"]) + ls_full_df.insert(0, "last_trial_time", mid) + + ls_full_dict = ls_full_df.to_dict("records") # Send one alert per session found idx_alert = 0 for this_alert_record in ls_full_dict: - - #Format message for session and live stat dictionary - this_session_stats = session_data_df.iloc[idx_alert,:] - slack_json_message = slack_alert_message_format_live_stats(this_session_stats.to_dict(), this_alert_record, int(this_alert_record['seconds_elapsed_last_valid_stat']/60)) - - #Send alert - webhooks_list = su.get_webhook_list(slack_configuration_dictionary, lab) + # Format message for session and live stat dictionary + this_session_stats = session_data_df.iloc[idx_alert, :] + slack_json_message = slack_alert_message_format_live_stats( + this_session_stats.to_dict(), + this_alert_record, + int(this_alert_record["seconds_elapsed_last_valid_stat"] / 60), + ) + + # Send alert + webhooks_list = su.get_webhook_list( + slack_configuration_dictionary, lab + ) for this_webhook in webhooks_list: su.send_slack_notification(this_webhook, slack_json_message) time.sleep(1) - reported_session = this_session_stats[['subject_fullname', 'session_date', 'session_number']].copy() - reported_session['report_datetime'] = right_now_est + reported_session = this_session_stats[ + ["subject_fullname", "session_date", "session_number"] + ].copy() + reported_session["report_datetime"] = right_now_est - acquisition.ReportedLiveSessionStats.insert1(reported_session.to_dict()) + acquisition.ReportedLiveSessionStats.insert1( + reported_session.to_dict() + ) idx_alert += 1 diff --git a/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py b/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py index 1d31c80d..f08de720 100644 --- a/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py +++ b/u19_pipeline/alert_system/live_session_stats_deletion/cronjob_live_session_stats_deletion.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file @@ -10,4 +9,3 @@ import u19_pipeline.alert_system.live_session_stats_deletion.live_session_stats_deletion as lssd lssd.main_live_session_stats_deletion() - diff --git a/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py b/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py index 4e3793e6..24e36df3 100644 --- a/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py +++ b/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py @@ -1,17 +1,19 @@ - import datajoint as dj def main_live_session_stats_deletion(): - acquisition = dj.create_virtual_module('acquisition', 'u19_acquisition') + acquisition = dj.create_virtual_module("acquisition", "u19_acquisition") old_session_stats_query = "current_datetime < NOW() - INTERVAL 5 DAY" order_cols = "'subject_fullname', 'session_date', 'session_number', 'trial_idx'" connection = acquisition.LiveSessionStats.connection with connection.transaction: - old_session_stats = (acquisition.LiveSessionStats & old_session_stats_query).fetch(as_dict=True, order_by=[order_cols]) - (acquisition.HistoricSessionStats).insert(old_session_stats, skip_duplicates=True) + old_session_stats = ( + acquisition.LiveSessionStats & old_session_stats_query + ).fetch(as_dict=True, order_by=[order_cols]) + (acquisition.HistoricSessionStats).insert( + old_session_stats, skip_duplicates=True + ) (acquisition.LiveSessionStats & old_session_stats_query).delete(safemode=False) - diff --git a/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py b/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py index bd1d4abc..03fb6034 100644 --- a/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py +++ b/u19_pipeline/alert_system/locked_tables_alert/cronjob_locked_tables_alert.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file @@ -10,4 +9,3 @@ import u19_pipeline.alert_system.locked_tables_alert.locked_tables_alert as lta lta.main_locked_tables_alert() - diff --git a/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py b/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py index fd593574..8d61f67b 100644 --- a/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py +++ b/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py @@ -8,22 +8,22 @@ import u19_pipeline.utils.slack_utils as su # Slack Configuration dictionary -slack_configuration_dictionary = { - 'slack_notification_channel': ['dev_notifications'] -} +slack_configuration_dictionary = {"slack_notification_channel": ["dev_notifications"]} def main_locked_tables_alert(): - locked_tables_query = 'show open tables where in_use > 0' + locked_tables_query = "show open tables where in_use > 0" conn = dj.conn() - locked_tables_df = pd.DataFrame(conn.query(locked_tables_query, as_dict=True).fetchall()) + locked_tables_df = pd.DataFrame( + conn.query(locked_tables_query, as_dict=True).fetchall() + ) if locked_tables_df.shape[0] == 0: return else: locked_tables_df = locked_tables_df.head() - locked_tables_df = locked_tables_df.drop('Name_locked',axis=1) + locked_tables_df = locked_tables_df.drop("Name_locked", axis=1) locked_tables_df = su.format_df_for_slack_message(locked_tables_df) slack_json_message = slack_alert_message_format_locked_tables(locked_tables_df) diff --git a/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py b/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py index a07fef2e..f1954636 100644 --- a/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py +++ b/u19_pipeline/alert_system/log_deletion/cronjob_old_log_deletion.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file @@ -10,4 +9,3 @@ import u19_pipeline.alert_system.log_deletion.old_log_deletion as old old.main_old_log_deletion() - diff --git a/u19_pipeline/alert_system/log_deletion/old_log_deletion.py b/u19_pipeline/alert_system/log_deletion/old_log_deletion.py index 54706b45..36f9c911 100644 --- a/u19_pipeline/alert_system/log_deletion/old_log_deletion.py +++ b/u19_pipeline/alert_system/log_deletion/old_log_deletion.py @@ -1,4 +1,3 @@ - import datetime import pathlib import re @@ -12,21 +11,34 @@ def extract_filename(x): def main_old_log_deletion(): - DIRECTORY_PATH = pathlib.Path('/home/u19prod@pu.win.princeton.edu/log') + DIRECTORY_PATH = pathlib.Path("/home/u19prod@pu.win.princeton.edu/log") date_pattern = r"_\d{8}_" reference_date = datetime.date.today() - datetime.timedelta(days=10) - files = [p for p in DIRECTORY_PATH.rglob("*") if p.is_file() and re.search(date_pattern, p.name)] - - - files_df = pd.DataFrame(files, columns=['filepaths']) - files_df['filename'] = files_df['filepaths'].apply(extract_filename) - files_df['exctract_date'] = files_df['filename'].str.extract(r'(\d{8,' + str(8) + r'})') - files_df['exctract_date'] = pd.to_datetime(files_df['exctract_date'], format='%Y%m%d') - files_df['before_ref_date'] = files_df['exctract_date'] <= pd.Timestamp(reference_date) - - files_for_deletion = files_df.loc[files_df['before_ref_date']==True, 'filepaths'].to_list() + files = [ + p + for p in DIRECTORY_PATH.rglob("*") + if p.is_file() and re.search(date_pattern, p.name) + ] + + files_df = pd.DataFrame(files, columns=["filepaths"]) + files_df["filename"] = files_df["filepaths"].apply(extract_filename) + files_df["exctract_date"] = files_df["filename"].str.extract( + r"(\d{8," + str(8) + r"})" + ) + files_df["exctract_date"] = pd.to_datetime( + files_df["exctract_date"], format="%Y%m%d" + ) + files_df["before_ref_date"] = files_df["exctract_date"] <= pd.Timestamp( + reference_date + ) + + files_for_deletion = files_df.loc[ + files_df["before_ref_date"] == True, "filepaths" + ].to_list() for file_path in files_for_deletion: - if file_path.is_file(): # Check if it's actually a file before attempting to delete - file_path.unlink() \ No newline at end of file + if ( + file_path.is_file() + ): # Check if it's actually a file before attempting to delete + file_path.unlink() diff --git a/u19_pipeline/alert_system/main_alert_system.py b/u19_pipeline/alert_system/main_alert_system.py index 4c481cac..4d64efec 100644 --- a/u19_pipeline/alert_system/main_alert_system.py +++ b/u19_pipeline/alert_system/main_alert_system.py @@ -1,4 +1,3 @@ - import datetime import importlib import pkgutil @@ -10,9 +9,8 @@ import u19_pipeline.utils.slack_utils as su # Slack Configuration dictionary -slack_configuration_dictionary = { - 'slack_notification_channel': ['custom_alerts'] -} +slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]} + def main_alert_system(): 'Call main function of all alerts defined in "custom_alerts' @@ -20,32 +18,47 @@ def main_alert_system(): all_alert_submodules = pkgutil.iter_modules(ca.__path__) for this_alert_submodule in all_alert_submodules: + print("executing " + this_alert_submodule.name + " alert code") - print('executing '+ this_alert_submodule.name+ " alert code") - - my_alert_module = importlib.import_module('u19_pipeline.alert_system.custom_alerts.'+this_alert_submodule.name) + my_alert_module = importlib.import_module( + "u19_pipeline.alert_system.custom_alerts." + this_alert_submodule.name + ) try: alert_df = my_alert_module.main() - if hasattr(my_alert_module, 'slack_configuration_dictionary'): + if hasattr(my_alert_module, "slack_configuration_dictionary"): slack_dict = my_alert_module.slack_configuration_dictionary else: slack_dict = slack_configuration_dictionary if alert_df.shape[0] > 0: - - alert_dict = alert_df.to_dict('records') + alert_dict = alert_df.to_dict("records") webhooks_list = [] - if 'slack_notification_channel' in slack_dict: - query_slack_webhooks = [{'webhook_name' : x} for x in slack_dict['slack_notification_channel']] - webhooks_list += (lab.SlackWebhooks & query_slack_webhooks).fetch('webhook_url').tolist() - - if 'slack_users_channel' in slack_dict: - query_slack_user_channels = [{'user_id' : x} for x in slack_dict['slack_users_channel']] - webhooks_list += (lab.User & query_slack_user_channels).fetch('slack_webhook').tolist() + if "slack_notification_channel" in slack_dict: + query_slack_webhooks = [ + {"webhook_name": x} + for x in slack_dict["slack_notification_channel"] + ] + webhooks_list += ( + (lab.SlackWebhooks & query_slack_webhooks) + .fetch("webhook_url") + .tolist() + ) + + if "slack_users_channel" in slack_dict: + query_slack_user_channels = [ + {"user_id": x} for x in slack_dict["slack_users_channel"] + ] + webhooks_list += ( + (lab.User & query_slack_user_channels) + .fetch("slack_webhook") + .tolist() + ) for this_alert_record in alert_dict: - slack_json_message = slack_alert_message_format(this_alert_record, this_alert_submodule.name) + slack_json_message = slack_alert_message_format( + this_alert_record, this_alert_submodule.name + ) for this_webhook in webhooks_list: su.send_slack_notification(this_webhook, slack_json_message) @@ -53,44 +66,55 @@ def main_alert_system(): except Exception as e: dict_error = dict() - dict_error['message'] = 'error while executing ' + this_alert_submodule.name + " alert code" - dict_error['error_exception'] = (''.join(traceback.format_exception(type(e), value=e, tb=e.__traceback__))) - slack_json_message = slack_alert_message_format(dict_error, this_alert_submodule.name) - webhook_custom = (lab.SlackWebhooks & "webhook_name='custom_alerts'").fetch('webhook_url').tolist() + dict_error["message"] = ( + "error while executing " + this_alert_submodule.name + " alert code" + ) + dict_error["error_exception"] = "".join( + traceback.format_exception(type(e), value=e, tb=e.__traceback__) + ) + slack_json_message = slack_alert_message_format( + dict_error, this_alert_submodule.name + ) + webhook_custom = ( + (lab.SlackWebhooks & "webhook_name='custom_alerts'") + .fetch("webhook_url") + .tolist() + ) if webhook_custom: su.send_slack_notification(webhook_custom[0], slack_json_message) del my_alert_module + def slack_alert_message_format(alert_dictionaty, alert_module_name): now = datetime.datetime.now() - datestr = now.strftime('%d-%b-%Y %H:%M:%S') + datestr = now.strftime("%d-%b-%Y %H:%M:%S") msep = dict() - msep['type'] = "divider" + msep["type"] = "divider" - #Title# + # Title# m1 = dict() - m1['type'] = 'section' + m1["type"] = "section" m1_1 = dict() m1_1["type"] = "mrkdwn" - m1_1["text"] = ':rotating_light: *' +alert_module_name + '* on ' + datestr + '\n\n' - m1['text'] = m1_1 + m1_1["text"] = ":rotating_light: *" + alert_module_name + "* on " + datestr + "\n\n" + m1["text"] = m1_1 - #Info# + # Info# m2 = dict() - m2['type'] = 'section' + m2["type"] = "section" m2_1 = dict() m2_1["type"] = "mrkdwn" - m2_1["text"] = '' + m2_1["text"] = "" for key in alert_dictionaty.keys(): - m2_1["text"] += '*' + key + '* : ' + str(alert_dictionaty[key]) + '\n' - m2['text'] = m2_1 + m2_1["text"] += "*" + key + "* : " + str(alert_dictionaty[key]) + "\n" + m2["text"] = m2_1 message = dict() - message['blocks'] = [m1,msep,m2,msep,msep] - message['text'] = alert_module_name+ ' alert' + message["blocks"] = [m1, msep, m2, msep, msep] + message["text"] = alert_module_name + " alert" return message diff --git a/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py b/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py index c10a65cb..28e5bd00 100644 --- a/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py +++ b/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py @@ -1,4 +1,3 @@ - from scripts.conf_file_finding import try_find_conf_file try_find_conf_file() @@ -25,26 +24,35 @@ import u19_pipeline.scheduler as scheduler import u19_pipeline.subject as subject -DJ_CUSTOM_VARIABLES_FILENAME = 'DJCustomVariables.csv' -SLACK_WEBHOOK_FILENAME = 'SlackChannels.csv' -USER_SLACK_FILENAME = 'UserSlack.csv' -RIG_STATUS_FILENAME = 'RigStatusTable.csv' -DAY_SCHEDULE_FILENAME = 'ScheduleDay.csv' -PAST_SESSION_PERFORMANCE_FILENAME = 'PastSessions.csv' -SUBJECT_MOTOR_POSITION_FILENAME = 'SubjectMotorPosition.csv' - -WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = 'Weighing_GUI_Replacement_SpreadSheet_Template.xlsx' -WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = 'Weighing_GUI_Replacement_SpreadSheet.xlsx' +DJ_CUSTOM_VARIABLES_FILENAME = "DJCustomVariables.csv" +SLACK_WEBHOOK_FILENAME = "SlackChannels.csv" +USER_SLACK_FILENAME = "UserSlack.csv" +RIG_STATUS_FILENAME = "RigStatusTable.csv" +DAY_SCHEDULE_FILENAME = "ScheduleDay.csv" +PAST_SESSION_PERFORMANCE_FILENAME = "PastSessions.csv" +SUBJECT_MOTOR_POSITION_FILENAME = "SubjectMotorPosition.csv" + +WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = ( + "Weighing_GUI_Replacement_SpreadSheet_Template.xlsx" +) +WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = ( + "Weighing_GUI_Replacement_SpreadSheet.xlsx" +) conf = dj.config -nodb_virmen_backup_dir = pathlib.Path(pathlib.Path(conf['custom']['root_data_dir'][0]).parent.parent,'Shared','NoDBVirmenBackup') +nodb_virmen_backup_dir = pathlib.Path( + pathlib.Path(conf["custom"]["root_data_dir"][0]).parent.parent, + "Shared", + "NoDBVirmenBackup", +) MAX_SESSIONS_HISTORY = 75 + def cast_choice(choice_array): new_array = choice_array[0].copy() - new_array[new_array>2] = 127 + new_array[new_array > 2] = 127 new_array = np.array(new_array, dtype=np.uint8) @@ -53,8 +61,7 @@ def cast_choice(choice_array): def encode_webhook(webhook): - return (base64.b64encode(webhook.encode('utf-8'))).decode('utf-8') - + return (base64.b64encode(webhook.encode("utf-8"))).decode("utf-8") def write_dj_custom_vars_file(): @@ -62,46 +69,64 @@ def write_dj_custom_vars_file(): file_write = pathlib.Path(nodb_virmen_backup_dir, DJ_CUSTOM_VARIABLES_FILENAME) pd.DataFrame(lab.DjCustomVariables.fetch(as_dict=True)).to_csv(file_write) + def write_slack_webhooks_file(): slack_webhooks = pd.DataFrame(lab.SlackWebhooks.fetch(as_dict=True)) - slack_webhooks['webhook_url'] = slack_webhooks['webhook_url'].astype(str) + slack_webhooks["webhook_url"] = slack_webhooks["webhook_url"].astype(str) - slack_webhooks['webhook_url'] = slack_webhooks['webhook_url'].apply(encode_webhook) + slack_webhooks["webhook_url"] = slack_webhooks["webhook_url"].apply(encode_webhook) file_write = pathlib.Path(nodb_virmen_backup_dir, SLACK_WEBHOOK_FILENAME) slack_webhooks.to_csv(file_write, index=False) -def write_user_data_file(): +def write_user_data_file(): - user_data = pd.DataFrame(lab.User.fetch('user_id', 'slack', 'tech_responsibility', 'slack_webhook',as_dict=True)) - user_data['slack_webhook'] = user_data['slack_webhook'].astype(str) - user_data['slack_webhook'] = user_data['slack_webhook'].fillna('') + user_data = pd.DataFrame( + lab.User.fetch( + "user_id", "slack", "tech_responsibility", "slack_webhook", as_dict=True + ) + ) + user_data["slack_webhook"] = user_data["slack_webhook"].astype(str) + user_data["slack_webhook"] = user_data["slack_webhook"].fillna("") - user_data['slack_webhook'] = user_data['slack_webhook'].apply(encode_webhook) + user_data["slack_webhook"] = user_data["slack_webhook"].apply(encode_webhook) file_write = pathlib.Path(nodb_virmen_backup_dir, USER_SLACK_FILENAME) user_data.to_csv(file_write, index=False) + def write_rig_status_file(): file_write = pathlib.Path(nodb_virmen_backup_dir, RIG_STATUS_FILENAME) - df_rig_status = pd.DataFrame(scheduler.RigStatus.fetch('location', 'input_output_name','current_status',as_dict=True)) + df_rig_status = pd.DataFrame( + scheduler.RigStatus.fetch( + "location", "input_output_name", "current_status", as_dict=True + ) + ) df_rig_status.to_csv(file_write, index=False) def write_schedule_file(): schedule_query = dict() - schedule_query['date'] = datetime.date.today() - - subject_query = 'subject_fullname is not null' - - day_schedule = pd.DataFrame((scheduler.Schedule * scheduler.TrainingProfile * subject.Subject.proj(subject_user_id='user_id') & schedule_query & subject_query).fetch(as_dict=True)) - day_schedule = day_schedule.drop(columns='user_id') - day_schedule = day_schedule.rename(columns={'subject_user_id':'user_id'}) + schedule_query["date"] = datetime.date.today() + + subject_query = "subject_fullname is not null" + + day_schedule = pd.DataFrame( + ( + scheduler.Schedule + * scheduler.TrainingProfile + * subject.Subject.proj(subject_user_id="user_id") + & schedule_query + & subject_query + ).fetch(as_dict=True) + ) + day_schedule = day_schedule.drop(columns="user_id") + day_schedule = day_schedule.rename(columns={"subject_user_id": "user_id"}) file_write = pathlib.Path(nodb_virmen_backup_dir, DAY_SCHEDULE_FILENAME) day_schedule.to_csv(file_write, index=False) @@ -111,46 +136,82 @@ def write_schedule_file(): def write_past_sessions_file(day_schedule): - all_subjects_schedule = "', '".join(day_schedule['subject_fullname']) - all_subjects_schedule = "subject_fullname in ('" +all_subjects_schedule+ "')" - - ss = pd.DataFrame((behavior.TowersSession & all_subjects_schedule).fetch('KEY', order_by='subject_fullname, session_date desc, session_number desc', as_dict=True)) - ss['session_date'] = ss['session_date'].astype(str) - ss['num_sessions'] = ss.groupby(['subject_fullname'])['subject_fullname'].rank(method='first') - ss['num_sessions'] = ss['num_sessions'].astype(int) - ss = ss.loc[ss['num_sessions'] <= MAX_SESSIONS_HISTORY, :] - - max_value_indices = ss.groupby('subject_fullname')['num_sessions'].idxmax() + all_subjects_schedule = "', '".join(day_schedule["subject_fullname"]) + all_subjects_schedule = "subject_fullname in ('" + all_subjects_schedule + "')" + + ss = pd.DataFrame( + (behavior.TowersSession & all_subjects_schedule).fetch( + "KEY", + order_by="subject_fullname, session_date desc, session_number desc", + as_dict=True, + ) + ) + ss["session_date"] = ss["session_date"].astype(str) + ss["num_sessions"] = ss.groupby(["subject_fullname"])["subject_fullname"].rank( + method="first" + ) + ss["num_sessions"] = ss["num_sessions"].astype(int) + ss = ss.loc[ss["num_sessions"] <= MAX_SESSIONS_HISTORY, :] + + max_value_indices = ss.groupby("subject_fullname")["num_sessions"].idxmax() ss = ss.loc[max_value_indices] ss = ss.reset_index(drop=True) - ss['query'] = "(subject_fullname='" + ss['subject_fullname'] + "' and session_date >= '" + ss['session_date'] + "')" - - block_query = ' OR '.join(ss['query']) - - sstable=(acquisition.SessionStarted.proj('local_path_behavior_file', 'session_location')) - stable = (acquisition.Session).proj(stimulusBank='stimulus_bank') - tstable = (behavior.TowersSession.proj(trialType='rewarded_side',choice='chosen_side', stimulusSet='stimulus_set')) - tbtable = (behavior.TowersBlock.proj('first_trial','n_trials', 'sublevel', mazeID='level', mainMazeID='main_level', easyBlockFlag='easy_block',\ - duration='block_duration', rewardMil='reward_mil', medianTrialDur='trial_duration_median', start='block_start_time')) - table_fetch = sstable * stable* tstable * tbtable - - allblocks = pd.DataFrame((table_fetch & block_query).fetch(order_by='subject_fullname, session_date desc, session_number desc, block desc',as_dict=True)) - allblocks['from_DB'] = 1 - allblocks['session_date'] = allblocks['session_date'].astype(str) - - allblocks['sublevel'] = allblocks['sublevel'].astype('Int64') - allblocks['choice'] = allblocks['choice'].apply(cast_choice) - allblocks['trialType'] = allblocks['trialType'].apply(cast_choice) + ss["query"] = ( + "(subject_fullname='" + + ss["subject_fullname"] + + "' and session_date >= '" + + ss["session_date"] + + "')" + ) + + block_query = " OR ".join(ss["query"]) + + sstable = acquisition.SessionStarted.proj( + "local_path_behavior_file", "session_location" + ) + stable = (acquisition.Session).proj(stimulusBank="stimulus_bank") + tstable = behavior.TowersSession.proj( + trialType="rewarded_side", choice="chosen_side", stimulusSet="stimulus_set" + ) + tbtable = behavior.TowersBlock.proj( + "first_trial", + "n_trials", + "sublevel", + mazeID="level", + mainMazeID="main_level", + easyBlockFlag="easy_block", + duration="block_duration", + rewardMil="reward_mil", + medianTrialDur="trial_duration_median", + start="block_start_time", + ) + table_fetch = sstable * stable * tstable * tbtable + + allblocks = pd.DataFrame( + (table_fetch & block_query).fetch( + order_by="subject_fullname, session_date desc, session_number desc, block desc", + as_dict=True, + ) + ) + allblocks["from_DB"] = 1 + allblocks["session_date"] = allblocks["session_date"].astype(str) + + allblocks["sublevel"] = allblocks["sublevel"].astype("Int64") + allblocks["choice"] = allblocks["choice"].apply(cast_choice) + allblocks["trialType"] = allblocks["trialType"].apply(cast_choice) file_write = pathlib.Path(nodb_virmen_backup_dir, PAST_SESSION_PERFORMANCE_FILENAME) allblocks.to_csv(file_write, index=False) + def write_subject_motor_position(day_schedule): - all_subjects_schedule = "', '".join(day_schedule['subject_fullname']) - all_subjects_schedule = "subject_fullname in ('" +all_subjects_schedule+ "')" + all_subjects_schedule = "', '".join(day_schedule["subject_fullname"]) + all_subjects_schedule = "subject_fullname in ('" + all_subjects_schedule + "')" - sp = pd.DataFrame((subject.HeadMotorPosition & all_subjects_schedule).fetch(as_dict=True)) + sp = pd.DataFrame( + (subject.HeadMotorPosition & all_subjects_schedule).fetch(as_dict=True) + ) file_write = pathlib.Path(nodb_virmen_backup_dir, SUBJECT_MOTOR_POSITION_FILENAME) sp.to_csv(file_write, index=False) @@ -159,64 +220,75 @@ def write_subject_motor_position(day_schedule): def write_weighinig_gui_ss_file(): subject_data = wwa.get_subject_data() - subject_data = subject_data.loc[:, ['user_id', 'subject_fullname', 'cage', 'headplate_image_path', 'last_weight', 'water_per_day']] - #subject_data = subject_data.loc[subject_data['user_id'] != 'testuser',:] + subject_data = subject_data.loc[ + :, + [ + "user_id", + "subject_fullname", + "cage", + "headplate_image_path", + "last_weight", + "water_per_day", + ], + ] + # subject_data = subject_data.loc[subject_data['user_id'] != 'testuser',:] subject_data = subject_data.reset_index(drop=True) - dictionary_rename_cols =\ - {'user_id': 'owner', - 'subject_fullname': 'subject name', - 'cage': 'cage', - 'headplate_image_path': 'headplate', - 'last_weight': 'last weight', - 'water_per_day': 'water per day', + dictionary_rename_cols = { + "user_id": "owner", + "subject_fullname": "subject name", + "cage": "cage", + "headplate_image_path": "headplate", + "last_weight": "last weight", + "water_per_day": "water per day", } subject_data = subject_data.rename(columns=dictionary_rename_cols) subject_data = subject_data[list(dictionary_rename_cols.values())] - subject_data['headplate'] = subject_data['headplate'].astype(str) - - headplate_image_paths = subject_data['headplate'].copy() + subject_data["headplate"] = subject_data["headplate"].astype(str) - subject_data['last weight'] = subject_data['last weight'].round(1) - subject_data['water per day'] = subject_data['water per day'].round(1) + headplate_image_paths = subject_data["headplate"].copy() + subject_data["last weight"] = subject_data["last weight"].round(1) + subject_data["water per day"] = subject_data["water per day"].round(1) - subject_data['headplate'] = '' - subject_data['today weight'] = '' - subject_data['today water'] = '' + subject_data["headplate"] = "" + subject_data["today weight"] = "" + subject_data["today water"] = "" # Copy the source file to the destination, replacing if it exists - template_ss_file = pathlib.Path(nodb_virmen_backup_dir,WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE) - ss_file = pathlib.Path(nodb_virmen_backup_dir,WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME) + template_ss_file = pathlib.Path( + nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE + ) + ss_file = pathlib.Path( + nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME + ) shutil.copy2(template_ss_file, ss_file) book = openpyxl.load_workbook(ss_file) - sheet = book['Sheet1'] + sheet = book["Sheet1"] - - for r_idx, row in enumerate(dataframe_to_rows(subject_data, index=False, header=False), start=3): + for r_idx, row in enumerate( + dataframe_to_rows(subject_data, index=False, header=False), start=3 + ): for c_idx, value in enumerate(row, start=1): sheet.cell(row=r_idx, column=c_idx, value=value) - - column_headplate = subject_data.columns.get_loc('headplate') + 1 + column_headplate = subject_data.columns.get_loc("headplate") + 1 column_headplate = openpyxl.utils.get_column_letter(column_headplate) - for i in range(headplate_image_paths.shape[0]): - headplate_path = headplate_image_paths[i] - cell_image = column_headplate+str(i+3) + cell_image = column_headplate + str(i + 3) if pathlib.Path.is_file(pathlib.Path(headplate_path)): img = Image(headplate_path) img.width = 50 img.height = 50 sheet.add_image(img, cell_image) - sheet.row_dimensions[i+3].height = 60 + sheet.row_dimensions[i + 3].height = 60 else: - sheet.row_dimensions[i+3].height = 15 + sheet.row_dimensions[i + 3].height = 15 book.save(ss_file) @@ -235,4 +307,3 @@ def main_noDB_backup(): if __name__ == "__main__": main_noDB_backup() - diff --git a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py index efb788b3..d49e41e5 100755 --- a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py +++ b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py @@ -67,7 +67,9 @@ def setup_logging(): # File handler file_handler = logging.FileHandler(log_file, mode="a", encoding="utf-8") - formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s", datefmt="[%X]") + formatter = logging.Formatter( + "%(asctime)s - %(levelname)s - %(message)s", datefmt="[%X]" + ) console_handler.setFormatter(formatter) file_handler.setFormatter(formatter) @@ -87,8 +89,12 @@ def get_slack_webhook(): str or None: Webhook URL if found, None otherwise """ try: - slack_configuration_dictionary = {"webhook_name": "rigs_issues_and_troubleshooting"} - webhook_url = (lab.SlackWebhooks & slack_configuration_dictionary).fetch1("webhook_url") + slack_configuration_dictionary = { + "webhook_name": "rigs_issues_and_troubleshooting" + } + webhook_url = (lab.SlackWebhooks & slack_configuration_dictionary).fetch1( + "webhook_url" + ) return webhook_url except Exception as e: logger.warning(f"Failed to get Slack webhook: {e}") @@ -195,7 +201,11 @@ def check_overdue_maintenance(): # Determine which locations are active based on recent schedules (last 30 days) five_weeks_ago = date.today() - timedelta(weeks=5) - query = scheduler.Schedule & f'date >= "{five_weeks_ago}"' & 'subject_fullname not like "test%"' + query = ( + scheduler.Schedule + & f'date >= "{five_weeks_ago}"' + & 'subject_fullname not like "test%"' + ) locations = query.fetch("location") unique_locations = sorted(set(locations)) @@ -204,8 +214,12 @@ def check_overdue_maintenance(): maintenance_types = maintenance_fetch(as_dict=True) # Separate maintenance types by system type - rig_maintenance_types = [m for m in maintenance_types if m.get("number_of_lines") >= 0] - hosting_maintenance_types = [m for m in maintenance_types if m.get("number_of_lines") < 0] + rig_maintenance_types = [ + m for m in maintenance_types if m.get("number_of_lines") >= 0 + ] + hosting_maintenance_types = [ + m for m in maintenance_types if m.get("number_of_lines") < 0 + ] # Get all locations (rigs) rig_queries = [ @@ -253,7 +267,9 @@ def check_overdue_maintenance(): "maintenance_type": maintenance_type, } recent_fetch = recent_q.fetch - recent_maintenance = recent_fetch("maintenance_date", order_by="maintenance_date DESC", limit=1) + recent_maintenance = recent_fetch( + "maintenance_date", order_by="maintenance_date DESC", limit=1 + ) if len(recent_maintenance) == 0: # No maintenance record exists @@ -295,7 +311,9 @@ def check_overdue_maintenance(): "message": f"{maintenance_type} is {days_overdue} days overdue", } ) - logger.info(f" {maintenance_type:.<30} OVERDUE by {days_overdue} days ❌") + logger.info( + f" {maintenance_type:.<30} OVERDUE by {days_overdue} days ❌" + ) else: # Not yet overdue: check if within notification window if 0 < days_until_due <= notification_window: @@ -317,7 +335,9 @@ def check_overdue_maintenance(): f"{days_until_due} days (window {notification_window}) ⚠️" ) else: - logger.info(f" {maintenance_type:.<30} OK ({days_until_due} days until due) ✅") + logger.info( + f" {maintenance_type:.<30} OK ({days_until_due} days until due) ✅" + ) # Get all hosting VMs for certificate & secret updates hosting_queries = [ @@ -351,7 +371,9 @@ def check_overdue_maintenance(): "maintenance_type": maintenance_type, } recent_fetch = recent_q.fetch - recent_maintenance = recent_fetch("maintenance_date", order_by="maintenance_date DESC", limit=1) + recent_maintenance = recent_fetch( + "maintenance_date", order_by="maintenance_date DESC", limit=1 + ) if len(recent_maintenance) == 0: # No maintenance record exists @@ -386,7 +408,9 @@ def check_overdue_maintenance(): "message": f"{maintenance_type} is {days_overdue} days overdue", } ) - logger.info(f" {maintenance_type:.<30} OVERDUE by {days_overdue} days ❌") + logger.info( + f" {maintenance_type:.<30} OVERDUE by {days_overdue} days ❌" + ) else: # Not yet overdue: check if within notification window days_until_due = interval_days - days_since_last @@ -408,7 +432,9 @@ def check_overdue_maintenance(): f" {maintenance_type:.<30} DUE SOON in {days_until_due} days (window {notification_window}) ⚠️" ) else: - logger.info(f" {maintenance_type:.<30} OK ({days_until_due} days until due) ✅") + logger.info( + f" {maintenance_type:.<30} OK ({days_until_due} days until due) ✅" + ) # Return combined list; status distinguishes OVERDUE vs UPCOMING return overdue_items + upcoming_items @@ -437,14 +463,18 @@ def log_summary(overdue_items): if no_record_items: # Print a plain-text table for no-record items - logger.warning(f"📋 RIGS WITH NO MAINTENANCE RECORDS ({len(no_record_items)} items)") + logger.warning( + f"📋 RIGS WITH NO MAINTENANCE RECORDS ({len(no_record_items)} items)" + ) loc_w = 30 type_w = 40 logger.debug(f"{'.' * (loc_w + type_w + 5)}") logger.debug(f"{'Location':<{loc_w}} | {'Maintenance Type':<{type_w}}") logger.debug(f"{'.' * (loc_w + type_w + 5)}") for item in no_record_items: - logger.debug(f"{item['location']:<{loc_w}} | {item['maintenance_type']:<{type_w}}") + logger.debug( + f"{item['location']:<{loc_w}} | {item['maintenance_type']:<{type_w}}" + ) logger.debug(f"{'.' * (loc_w + type_w + 5)}") if overdue_items_list: @@ -477,7 +507,9 @@ def log_summary(overdue_items): logger.debug("=" * 60) -def dataframe_to_slack_table_block(df: pd.DataFrame, truncate_cell: int = 200, use_markdown: bool = False) -> dict: +def dataframe_to_slack_table_block( + df: pd.DataFrame, truncate_cell: int = 200, use_markdown: bool = False +) -> dict: """ Convert a DataFrame into one Slack table block. Docs: https://docs.slack.dev/reference/block-kit/blocks/table-block/ @@ -510,7 +542,9 @@ def mk_cell(val): return block -def dataframe_to_slack_table_blocks(df: pd.DataFrame, chunk_size: int = 15, **kwargs) -> list[dict]: +def dataframe_to_slack_table_blocks( + df: pd.DataFrame, chunk_size: int = 15, **kwargs +) -> list[dict]: """ Chunk a DataFrame into multiple Slack table blocks by rows. @@ -547,7 +581,9 @@ def main(): log_summary(overdue_items) records = pd.DataFrame(overdue_items) - records.columns = [" ".join(w.capitalize() for w in col.split("_")) for col in records.columns] + records.columns = [ + " ".join(w.capitalize() for w in col.split("_")) for col in records.columns + ] print(records) records = records[["Location", "Maintenance Type", "Status"]] diff --git a/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py b/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py index 1835b0d5..8eca53ca 100644 --- a/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py +++ b/u19_pipeline/alert_system/schedule_check_alert/cronjob_schedule_check.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file @@ -10,4 +9,3 @@ import u19_pipeline.alert_system.schedule_check_alert.schedule_check_alert as sca sca.main_schedule_check_alert() - diff --git a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py index 5e00eb7b..b61ea9f0 100644 --- a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py +++ b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py @@ -9,17 +9,22 @@ # Slack Configuration dictionary slack_configuration_dictionary = { - 'slack_notification_channel': ['dev_notifications'], - #"slack_users_channel": ["alvaros"] + "slack_notification_channel": ["dev_notifications"], + # "slack_users_channel": ["alvaros"] } + def get_schedule_query(): scheduler = dj.create_virtual_module("scheduler", "u19_scheduler") - tomorrow = (datetime.date.today() + datetime.timedelta(days=1)).strftime('%Y-%m-%d') - today = (datetime.date.today()).strftime('%Y-%m-%d') + tomorrow = (datetime.date.today() + datetime.timedelta(days=1)).strftime("%Y-%m-%d") + today = (datetime.date.today()).strftime("%Y-%m-%d") schedule_query = 'date >= "' + today + '" and date <= "' + tomorrow + '"' - schedule_df = pd.DataFrame((scheduler.Schedule & schedule_query).fetch('date', 'location', 'subject_fullname', as_dict=True)) + schedule_df = pd.DataFrame( + (scheduler.Schedule & schedule_query).fetch( + "date", "location", "subject_fullname", as_dict=True + ) + ) return schedule_df @@ -28,45 +33,62 @@ def main_schedule_check_alert(): alert = 0 schedule_df = get_schedule_query() - tomorrow_schedule = schedule_df.loc[schedule_df['date'] == (datetime.date.today() + datetime.timedelta(days=1)),:] + tomorrow_schedule = schedule_df.loc[ + schedule_df["date"] == (datetime.date.today() + datetime.timedelta(days=1)), : + ] if tomorrow_schedule.shape[0] == 0: alert = 1 slack_json_message = slack_alert_empty_schedule() else: - schedule_df = schedule_df.groupby(['date', 'location']).agg({'subject_fullname': [('#subj', 'count')]}) + schedule_df = schedule_df.groupby(["date", "location"]).agg( + {"subject_fullname": [("#subj", "count")]} + ) schedule_df.columns = schedule_df.columns.droplevel() schedule_df = schedule_df.reset_index() - todays_summary_df = schedule_df.loc[schedule_df['date'] == datetime.date.today(), - ['location', '#subj']].copy() - tomorrow_summary_df = schedule_df.loc[schedule_df['date'] == (datetime.date.today() + datetime.timedelta(days=1)), - ['location', '#subj']].copy() - - schedule_comp = pd.merge(todays_summary_df, tomorrow_summary_df, on=['location'], suffixes=['_today', '_tomorrow']) + todays_summary_df = schedule_df.loc[ + schedule_df["date"] == datetime.date.today(), ["location", "#subj"] + ].copy() + tomorrow_summary_df = schedule_df.loc[ + schedule_df["date"] == (datetime.date.today() + datetime.timedelta(days=1)), + ["location", "#subj"], + ].copy() + + schedule_comp = pd.merge( + todays_summary_df, + tomorrow_summary_df, + on=["location"], + suffixes=["_today", "_tomorrow"], + ) schedule_comp - schedule_comp['diff_subjects'] = schedule_comp['#subj_tomorrow'] - schedule_comp['#subj_today'] - schedule_comp['rig_less_subjects'] = schedule_comp['diff_subjects'] < -2 + schedule_comp["diff_subjects"] = ( + schedule_comp["#subj_tomorrow"] - schedule_comp["#subj_today"] + ) + schedule_comp["rig_less_subjects"] = schedule_comp["diff_subjects"] < -2 - subjects_today = schedule_comp['#subj_today'].sum() - subjects_tomorrow = schedule_comp['#subj_tomorrow'].sum() - total_rigs_less_subjects = schedule_comp['rig_less_subjects'].sum() + subjects_today = schedule_comp["#subj_today"].sum() + subjects_tomorrow = schedule_comp["#subj_tomorrow"].sum() + total_rigs_less_subjects = schedule_comp["rig_less_subjects"].sum() print(schedule_comp) - if subjects_tomorrow/subjects_today < 0.7: + if subjects_tomorrow / subjects_today < 0.7: alert = 1 if total_rigs_less_subjects >= 3: alert = 1 if alert == 1: - schedule_comp['location'] = '*'+schedule_comp['location']+'*~' - schedule_comp = schedule_comp.drop(['diff_subjects', 'rig_less_subjects'], axis=1) + schedule_comp["location"] = "*" + schedule_comp["location"] + "*~" + schedule_comp = schedule_comp.drop( + ["diff_subjects", "rig_less_subjects"], axis=1 + ) schedule_comp_df_string = su.format_df_for_slack_message(schedule_comp) - slack_json_message = slack_alert_message_format_schedule(schedule_comp_df_string) - + slack_json_message = slack_alert_message_format_schedule( + schedule_comp_df_string + ) if alert == 0: return @@ -99,7 +121,9 @@ def slack_alert_message_format_schedule(schedule_df_string): m2_1 = dict() m2_1["type"] = "mrkdwn" - m2_1["text"] = "Significantly less subjects scheduled tomorrow\nSchedule per rig:" + "\n\n" + m2_1["text"] = ( + "Significantly less subjects scheduled tomorrow\nSchedule per rig:" + "\n\n" + ) m2_1["text"] += schedule_df_string m2["text"] = m2_1 @@ -109,6 +133,7 @@ def slack_alert_message_format_schedule(schedule_df_string): return message + def slack_alert_empty_schedule(): now = datetime.datetime.now() datestr = now.strftime("%d-%b-%Y %H:%M:%S") @@ -137,4 +162,4 @@ def slack_alert_empty_schedule(): message["blocks"] = [m1, msep, m2, msep] message["text"] = "Schedule Empty Alert" - return message \ No newline at end of file + return message diff --git a/u19_pipeline/alert_system/tech_alert/tech_alert.py b/u19_pipeline/alert_system/tech_alert/tech_alert.py index cf7eb52b..197b67c8 100644 --- a/u19_pipeline/alert_system/tech_alert/tech_alert.py +++ b/u19_pipeline/alert_system/tech_alert/tech_alert.py @@ -24,8 +24,13 @@ def get_webhook_list(lab): # Get webhook lists slack_configuration_dictionary = {"slack_notification_channel": ["rigs_scheduling"]} webhooks_list = [] - query_slack_webhooks = [{"webhook_name": x} for x in slack_configuration_dictionary["slack_notification_channel"]] - webhooks_list += (lab.SlackWebhooks & query_slack_webhooks).fetch("webhook_url").tolist() + query_slack_webhooks = [ + {"webhook_name": x} + for x in slack_configuration_dictionary["slack_notification_channel"] + ] + webhooks_list += ( + (lab.SlackWebhooks & query_slack_webhooks).fetch("webhook_url").tolist() + ) return webhooks_list @@ -67,7 +72,9 @@ def slack_alert_message_format_tech_alert(schedule_data): ) else: # Ensure we don't send an empty section block to Slack (it will reject it) - m2_1["text"] = "- No technician is scheduled for today. Please assign someone for coverage" + m2_1["text"] = ( + "- No technician is scheduled for today. Please assign someone for coverage" + ) m2["text"] = m2_1 message["blocks"].append(m2) message["blocks"].append(msep) @@ -80,7 +87,11 @@ def slack_alert_message_format_tech_alert(schedule_data): # Today + next 4 days for day in range(1, 7): date_to_check = today + datetime.timedelta(days=day) - shifts_on_date = [shift for shift in upcoming_shifts if shift["date"] == date_to_check and not is_off(shift)] + shifts_on_date = [ + shift + for shift in upcoming_shifts + if shift["date"] == date_to_check and not is_off(shift) + ] if not shifts_on_date: alerts.append( @@ -101,12 +112,18 @@ def slack_alert_message_format_tech_alert(schedule_data): f"- Experimenters, {date_to_check.strftime('%A, %B %d')} will have no training due to lab cleanup." ) if shift["duties"] in ["Watering Only", "Off"]: - alerts.append("Experimenters, please make arrangements if you need to train.") + alerts.append( + "Experimenters, please make arrangements if you need to train." + ) date_to_check = datetime.date.today() + datetime.timedelta(days=day) - shifts_on_date = [shift for shift in upcoming_shifts if shift["date"] == date_to_check] + shifts_on_date = [ + shift for shift in upcoming_shifts if shift["date"] == date_to_check + ] if not shifts_on_date: - alerts.append(f"No one is scheduled for {date_to_check.strftime('%A, %B %d')}.") + alerts.append( + f"No one is scheduled for {date_to_check.strftime('%A, %B %d')}." + ) else: for shift in shifts_on_date: if shift["duties"] in ["Watering Only"]: @@ -120,7 +137,9 @@ def slack_alert_message_format_tech_alert(schedule_data): m3["type"] = "section" m3_1 = dict() m3_1["type"] = "mrkdwn" - m3_1["text"] = f"*:rotating_light: Upcoming Shifts: :rotating_light:*\n{alert_text}" + m3_1["text"] = ( + f"*:rotating_light: Upcoming Shifts: :rotating_light:*\n{alert_text}" + ) m3["text"] = m3_1 message["blocks"].append(m3) @@ -268,7 +287,12 @@ def fetch_and_parse_icalevents(weburl: str): (r"as\s*vr\s*water\s*at", "VR Water", "blue", "VR Watering only"), (r"as\s*vr\s*train\s*at", "VR Train", "orange", "VR Onboarding"), (r"as\s*vr\s*at", "Regular VR", "green", "All VR Duties"), - (r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at", "VR Brody Mice", "purple", " VR with Brody (Mice)"), + ( + r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at", + "VR Brody Mice", + "purple", + " VR with Brody (Mice)", + ), ] filtered_events: list[dict[str, str | int]] = [] @@ -276,10 +300,14 @@ def fetch_and_parse_icalevents(weburl: str): # Consolidate lab clean-up events lab_clean_up_events = [ - event for event in vr_events if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event + for event in vr_events + if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] non_lab_clean_up_events = [ - event for event in vr_events if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event + for event in vr_events + if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] if lab_clean_up_events: unique_days = sorted(set(event.start.date() for event in lab_clean_up_events)) @@ -327,4 +355,6 @@ def fetch_and_parse_icalevents(weburl: str): if sys.argv[1].lower() == "test": test_slack_alert_message_format_tech_alert() else: - raise ValueError(f"Unrecognized argument {sys.argv[1]} -- use 'test' to run in test mode.") + raise ValueError( + f"Unrecognized argument {sys.argv[1]} -- use 'test' to run in test mode." + ) diff --git a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py index 45d4dea1..850782ab 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py +++ b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py @@ -23,7 +23,9 @@ } # Query from file -QUERY_FILE = pathlib.Path(pathlib.Path(__file__).resolve().parent, "get_subject_data.sql").as_posix() +QUERY_FILE = pathlib.Path( + pathlib.Path(__file__).resolve().parent, "get_subject_data.sql" +).as_posix() def get_subject_data(): @@ -53,7 +55,10 @@ def get_subject_data(): subject_data = subject_data.loc[ ( - ((subject_data["schedule_today"] != "Nothing") & (subject_data["subject_status"] == "InExperiments")) + ( + (subject_data["schedule_today"] != "Nothing") + & (subject_data["subject_status"] == "InExperiments") + ) | (subject_data["subject_status"] == "WaterRestrictionOnly") ), :, @@ -62,18 +67,28 @@ def get_subject_data(): subject_data = subject_data.reset_index(drop=True) with pd.option_context("future.no_silent_downcasting", True): - subject_data["earned"] = subject_data["earned"].fillna(0).infer_objects(copy=False) - subject_data["received"] = subject_data["received"].fillna(0).infer_objects(copy=False) - subject_data["supplement"] = subject_data["supplement"].fillna(0).infer_objects(copy=False) + subject_data["earned"] = ( + subject_data["earned"].fillna(0).infer_objects(copy=False) + ) + subject_data["received"] = ( + subject_data["received"].fillna(0).infer_objects(copy=False) + ) + subject_data["supplement"] = ( + subject_data["supplement"].fillna(0).infer_objects(copy=False) + ) subject_data["prescribed_extra_supplement_amount"] = ( - subject_data["prescribed_extra_supplement_amount"].fillna(0).infer_objects(copy=False) + subject_data["prescribed_extra_supplement_amount"] + .fillna(0) + .infer_objects(copy=False) ) # Calculated fields subject_data["already_water"] = subject_data["supplement"] > 0 subject_data["upper_cage"] = subject_data["cage"].str.upper() subject_data["total_water_received"] = subject_data["received"] - subject_data.loc[subject_data["total_water_received"].isnull(), "total_water_received"] = 0 + subject_data.loc[ + subject_data["total_water_received"].isnull(), "total_water_received" + ] = 0 subject_data["need_supplement"] = 0 subject_data["need_supplement"] = ( @@ -83,19 +98,31 @@ def get_subject_data(): subject_data.loc[subject_data["already_received"].isnull(), "already_received"] = 0 subject_data["need_extra_water_now"] = 0 subject_data.loc[ - ((subject_data["prescribed_extra_supplement_amount"] > 0) & (subject_data["already_received"] == 0)), + ( + (subject_data["prescribed_extra_supplement_amount"] > 0) + & (subject_data["already_received"] == 0) + ), "need_extra_water_now", ] = 1 subject_data["water_status"] = "Already Watered" - subject_data.loc[subject_data["need_supplement"] == 1, "water_status"] = "Need Supplement" - subject_data.loc[subject_data["need_extra_water_now"] == 1, "water_status"] = "Need Extra Supplement" + subject_data.loc[subject_data["need_supplement"] == 1, "water_status"] = ( + "Need Supplement" + ) + subject_data.loc[subject_data["need_extra_water_now"] == 1, "water_status"] = ( + "Need Extra Supplement" + ) subject_data["need_water"] = 0 - subject_data.loc[(subject_data["need_supplement"] | subject_data["need_extra_water_now"]), "need_water"] = 1 + subject_data.loc[ + (subject_data["need_supplement"] | subject_data["need_extra_water_now"]), + "need_water", + ] = 1 subject_data["current_need_water"] = subject_data["suggested_water"] - subject_data.loc[subject_data["need_extra_water_now"] == 1, "current_need_water"] = subject_data.loc[ + subject_data.loc[ + subject_data["need_extra_water_now"] == 1, "current_need_water" + ] = subject_data.loc[ subject_data["need_extra_water_now"] == 1, "prescribed_extra_supplement_amount" ] @@ -104,15 +131,23 @@ def get_subject_data(): subject_data.loc[~subject_data["session_location"].isnull(), "training_status"] = 2 subject_data["training_status_label"] = "Not scheduled" - subject_data.loc[subject_data["training_status"] == 1, "training_status_label"] = "Scheduled" - subject_data.loc[subject_data["training_status"] == 2, "training_status_label"] = "Training Started" + subject_data.loc[subject_data["training_status"] == 1, "training_status_label"] = ( + "Scheduled" + ) + subject_data.loc[subject_data["training_status"] == 2, "training_status_label"] = ( + "Training Started" + ) # Calculated fields subject_data["already_weighted"] = ~subject_data["weight"].isnull() - subject_data["need_weight"] = ~subject_data["already_weighted"] | subject_data["need_reweight"] + subject_data["need_weight"] = ( + ~subject_data["already_weighted"] | subject_data["need_reweight"] + ) subject_data["weight_status"] = "Already Weighted" - subject_data.loc[subject_data["already_weighted"] == 0, "weight_status"] = "Need Weight" + subject_data.loc[subject_data["already_weighted"] == 0, "weight_status"] = ( + "Need Weight" + ) subject_data.loc[subject_data["need_reweight"] == 1, "weight_status"] = "REWEIGHT" return subject_data @@ -138,17 +173,26 @@ def fetch_and_parse_icalevents(weburl: str): (r"as\s*vr\s*water\s*at", "VR Water", "blue", "VR Watering only"), (r"as\s*vr\s*train\s*at", "VR Train", "orange", "VR Onboarding"), (r"as\s*vr\s*at", "Regular VR", "green", "All VR Duties"), - (r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at", "VR Brody Mice", "purple", " VR with Brody (Mice)"), + ( + r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at", + "VR Brody Mice", + "purple", + " VR with Brody (Mice)", + ), ] filtered_events: list[dict[str, str | int]] = [] # Consolidate lab clean-up events lab_clean_up_events = [ - event for event in vr_events if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event + for event in vr_events + if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] non_lab_clean_up_events = [ - event for event in vr_events if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event + for event in vr_events + if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] if lab_clean_up_events: consolidated_event = { @@ -213,9 +257,13 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: * lab.User().proj(lab_manager="user_id", manager_slack="slack") ).fetch("manager_slack", as_dict=True) print(technician_manager_slack) - technician_manager_slack = [item["manager_slack"] for item in technician_manager_slack] + technician_manager_slack = [ + item["manager_slack"] for item in technician_manager_slack + ] - people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch(format="frame") + people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch( + format="frame" + ) load_dotenv() @@ -243,7 +291,9 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: tech_slack.append(tech_uid) if tech_slack: - tech_slack.append(technician_manager_slack[0]) # Fallback to default technician manager slack + tech_slack.append( + technician_manager_slack[0] + ) # Fallback to default technician manager slack coowner_dataframe: pd.DataFrame = ( ( @@ -264,7 +314,11 @@ def resolve_responsible_slack(row): lab_name = row.get("lab") token = str(schedule_today).strip() if schedule_today is not None else None - use_coowners = (token is None) or (token.lower() == "transport") or (token.lower() == "nothing") + use_coowners = ( + (token is None) + or (token.lower() == "transport") + or (token.lower() == "nothing") + ) slack_tags = [] @@ -281,14 +335,18 @@ def resolve_responsible_slack(row): # print(temp) slack_tags += temp["manager_slack"].tolist() slack_tags += temp["user_slack"].tolist() - slack_tags += coowner_dataframe.query(f"subject_fullname == '{row['subject_fullname']}'")["slack"].tolist() + slack_tags += coowner_dataframe.query( + f"subject_fullname == '{row['subject_fullname']}'" + )["slack"].tolist() slack_tags = set(slack_tags) # Remove duplicates return " ".join(f"<@{item}>" for item in slack_tags) subject_data = subject_data.copy() - subject_data["responsible_slack_tags"] = subject_data.apply(resolve_responsible_slack, axis=1) + subject_data["responsible_slack_tags"] = subject_data.apply( + resolve_responsible_slack, axis=1 + ) return subject_data @@ -311,7 +369,9 @@ def find_unreturned_subjects() -> pd.DataFrame: .fetch(format="frame") ) - unreturned_subjects: pd.DataFrame = local_transport_data[local_transport_data["transport_in_datetime"].isnull()] + unreturned_subjects: pd.DataFrame = local_transport_data[ + local_transport_data["transport_in_datetime"].isnull() + ] return unreturned_subjects @@ -326,13 +386,17 @@ def slack_alert_message_format_weight_water( print(missing_transport) temp_missing_transport = missing_transport.copy().reset_index() - notifiable_subjects = list(set( - temp_missing_transport["subject_fullname"].tolist() - + subjects_not_watered["subject_fullname"].tolist() - + subjects_not_weighted["subject_fullname"].tolist() - )) + notifiable_subjects = list( + set( + temp_missing_transport["subject_fullname"].tolist() + + subjects_not_watered["subject_fullname"].tolist() + + subjects_not_weighted["subject_fullname"].tolist() + ) + ) - slack_handles: list[str] = fetch_slack_handles_for_lab_managers_by_subject(notifiable_subjects) + slack_handles: list[str] = fetch_slack_handles_for_lab_managers_by_subject( + notifiable_subjects + ) lab_manager_text = "\n\n" if len(slack_handles) >= 1: lab_manager_text += "Lab Manager" @@ -342,7 +406,9 @@ def slack_alert_message_format_weight_water( slack_handles_formatted = ", ".join("<@" + handle + ">" for handle in slack_handles) if slack_handles_formatted: lab_manager_text += ( - " " + slack_handles_formatted + ", please be advised that your labs' subjects are listed below." + " " + + slack_handles_formatted + + ", please be advised that your labs' subjects are listed below." ) msep = dict() @@ -457,7 +523,9 @@ def slack_alert_message_format_weight_water( # Try to get responsible slack tags for this subject tags = "" try: - tags = missing_transport.iloc[i].get("responsible_slack_tags", "") # may be string or iterable + tags = missing_transport.iloc[i].get( + "responsible_slack_tags", "" + ) # may be string or iterable except Exception: tags = "" # Normalize tags to a string @@ -514,7 +582,9 @@ def _estimate_section_size(text: str, text_type: str = "mrkdwn") -> int: return estimate_size({"blocks": [block]}) -def split_text_to_fit(text: str, max_payload_size: int, text_type: str = "mrkdwn") -> list[str]: +def split_text_to_fit( + text: str, max_payload_size: int, text_type: str = "mrkdwn" +) -> list[str]: """Split text by lines so each chunk fits in max_payload_size when JSON-encoded. Uses actual JSON size estimation rather than raw character count. @@ -574,7 +644,9 @@ def _split_large_block(block: dict, max_size: int) -> list[dict]: return [block] -def send_slack_blocks_safely(webhook: str, blocks: list[dict], max_size: int = SLACK_MAX_CHARS): +def send_slack_blocks_safely( + webhook: str, blocks: list[dict], max_size: int = SLACK_MAX_CHARS +): """Send blocks together when possible, splitting only when necessary. Strategy: @@ -646,7 +718,9 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int): smaller = _split_large_block(block, max_size) for small_block in smaller: try: - su.send_slack_notification(webhook, {"blocks": [small_block]}) + su.send_slack_notification( + webhook, {"blocks": [small_block]} + ) time.sleep(0.5) except HTTPError: print("Failed to send block even after splitting") @@ -672,16 +746,22 @@ def main_water_weigh_alert(): # print(subject_data.columns) - subject_data = subject_data.loc[~subject_data["subject_fullname"].str.contains("test"), :] + subject_data = subject_data.loc[ + ~subject_data["subject_fullname"].str.contains("test"), : + ] subjects_not_watered = subject_data.loc[ - subject_data["current_need_water"] > 0, ["subject_fullname", "current_need_water", "responsible_slack_tags"] + subject_data["current_need_water"] > 0, + ["subject_fullname", "current_need_water", "responsible_slack_tags"], ] subjects_not_watered = subjects_not_watered.reset_index(drop=True) - subjects_not_watered["current_need_water"] = subjects_not_watered["current_need_water"].apply(lambda x: f"{x:.1f}") + subjects_not_watered["current_need_water"] = subjects_not_watered[ + "current_need_water" + ].apply(lambda x: f"{x:.1f}") # subjects_not_watered = subjects_not_watered.head() subjects_not_weighted = subject_data.loc[ - subject_data["need_weight"], ["subject_fullname", "need_weight", "responsible_slack_tags"] + subject_data["need_weight"], + ["subject_fullname", "need_weight", "responsible_slack_tags"], ] subjects_not_weighted = subjects_not_weighted.reset_index(drop=True) # subjects_not_weighted = subjects_not_weighted.head() @@ -707,10 +787,12 @@ def main_water_weigh_alert(): # Send alert slack_json_messages = slack_alert_message_format_weight_water( - subjects_not_watered, subjects_not_weighted, subjects_not_trained, missing_transport=subject_not_returned + subjects_not_watered, + subjects_not_weighted, + subjects_not_trained, + missing_transport=subject_not_returned, ) - # Send each message's blocks safely (splitting large blocks as needed) for this_webhook in webhooks_list: for message in slack_json_messages: diff --git a/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py b/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py index 8aee0e7b..1321c082 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py +++ b/u19_pipeline/alert_system/water_weigh_alert/cronjob_water_weigh_alert.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file @@ -10,4 +9,3 @@ import u19_pipeline.alert_system.water_weigh_alert.water_weigh_alert as wwa wwa.main_water_weigh_alert() - diff --git a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py index 45d4dea1..850782ab 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py +++ b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py @@ -23,7 +23,9 @@ } # Query from file -QUERY_FILE = pathlib.Path(pathlib.Path(__file__).resolve().parent, "get_subject_data.sql").as_posix() +QUERY_FILE = pathlib.Path( + pathlib.Path(__file__).resolve().parent, "get_subject_data.sql" +).as_posix() def get_subject_data(): @@ -53,7 +55,10 @@ def get_subject_data(): subject_data = subject_data.loc[ ( - ((subject_data["schedule_today"] != "Nothing") & (subject_data["subject_status"] == "InExperiments")) + ( + (subject_data["schedule_today"] != "Nothing") + & (subject_data["subject_status"] == "InExperiments") + ) | (subject_data["subject_status"] == "WaterRestrictionOnly") ), :, @@ -62,18 +67,28 @@ def get_subject_data(): subject_data = subject_data.reset_index(drop=True) with pd.option_context("future.no_silent_downcasting", True): - subject_data["earned"] = subject_data["earned"].fillna(0).infer_objects(copy=False) - subject_data["received"] = subject_data["received"].fillna(0).infer_objects(copy=False) - subject_data["supplement"] = subject_data["supplement"].fillna(0).infer_objects(copy=False) + subject_data["earned"] = ( + subject_data["earned"].fillna(0).infer_objects(copy=False) + ) + subject_data["received"] = ( + subject_data["received"].fillna(0).infer_objects(copy=False) + ) + subject_data["supplement"] = ( + subject_data["supplement"].fillna(0).infer_objects(copy=False) + ) subject_data["prescribed_extra_supplement_amount"] = ( - subject_data["prescribed_extra_supplement_amount"].fillna(0).infer_objects(copy=False) + subject_data["prescribed_extra_supplement_amount"] + .fillna(0) + .infer_objects(copy=False) ) # Calculated fields subject_data["already_water"] = subject_data["supplement"] > 0 subject_data["upper_cage"] = subject_data["cage"].str.upper() subject_data["total_water_received"] = subject_data["received"] - subject_data.loc[subject_data["total_water_received"].isnull(), "total_water_received"] = 0 + subject_data.loc[ + subject_data["total_water_received"].isnull(), "total_water_received" + ] = 0 subject_data["need_supplement"] = 0 subject_data["need_supplement"] = ( @@ -83,19 +98,31 @@ def get_subject_data(): subject_data.loc[subject_data["already_received"].isnull(), "already_received"] = 0 subject_data["need_extra_water_now"] = 0 subject_data.loc[ - ((subject_data["prescribed_extra_supplement_amount"] > 0) & (subject_data["already_received"] == 0)), + ( + (subject_data["prescribed_extra_supplement_amount"] > 0) + & (subject_data["already_received"] == 0) + ), "need_extra_water_now", ] = 1 subject_data["water_status"] = "Already Watered" - subject_data.loc[subject_data["need_supplement"] == 1, "water_status"] = "Need Supplement" - subject_data.loc[subject_data["need_extra_water_now"] == 1, "water_status"] = "Need Extra Supplement" + subject_data.loc[subject_data["need_supplement"] == 1, "water_status"] = ( + "Need Supplement" + ) + subject_data.loc[subject_data["need_extra_water_now"] == 1, "water_status"] = ( + "Need Extra Supplement" + ) subject_data["need_water"] = 0 - subject_data.loc[(subject_data["need_supplement"] | subject_data["need_extra_water_now"]), "need_water"] = 1 + subject_data.loc[ + (subject_data["need_supplement"] | subject_data["need_extra_water_now"]), + "need_water", + ] = 1 subject_data["current_need_water"] = subject_data["suggested_water"] - subject_data.loc[subject_data["need_extra_water_now"] == 1, "current_need_water"] = subject_data.loc[ + subject_data.loc[ + subject_data["need_extra_water_now"] == 1, "current_need_water" + ] = subject_data.loc[ subject_data["need_extra_water_now"] == 1, "prescribed_extra_supplement_amount" ] @@ -104,15 +131,23 @@ def get_subject_data(): subject_data.loc[~subject_data["session_location"].isnull(), "training_status"] = 2 subject_data["training_status_label"] = "Not scheduled" - subject_data.loc[subject_data["training_status"] == 1, "training_status_label"] = "Scheduled" - subject_data.loc[subject_data["training_status"] == 2, "training_status_label"] = "Training Started" + subject_data.loc[subject_data["training_status"] == 1, "training_status_label"] = ( + "Scheduled" + ) + subject_data.loc[subject_data["training_status"] == 2, "training_status_label"] = ( + "Training Started" + ) # Calculated fields subject_data["already_weighted"] = ~subject_data["weight"].isnull() - subject_data["need_weight"] = ~subject_data["already_weighted"] | subject_data["need_reweight"] + subject_data["need_weight"] = ( + ~subject_data["already_weighted"] | subject_data["need_reweight"] + ) subject_data["weight_status"] = "Already Weighted" - subject_data.loc[subject_data["already_weighted"] == 0, "weight_status"] = "Need Weight" + subject_data.loc[subject_data["already_weighted"] == 0, "weight_status"] = ( + "Need Weight" + ) subject_data.loc[subject_data["need_reweight"] == 1, "weight_status"] = "REWEIGHT" return subject_data @@ -138,17 +173,26 @@ def fetch_and_parse_icalevents(weburl: str): (r"as\s*vr\s*water\s*at", "VR Water", "blue", "VR Watering only"), (r"as\s*vr\s*train\s*at", "VR Train", "orange", "VR Onboarding"), (r"as\s*vr\s*at", "Regular VR", "green", "All VR Duties"), - (r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at", "VR Brody Mice", "purple", " VR with Brody (Mice)"), + ( + r"as\s*vr(?:\s*(?:with|w)\s*)?brody(?:\s*mice)?\s*at", + "VR Brody Mice", + "purple", + " VR with Brody (Mice)", + ), ] filtered_events: list[dict[str, str | int]] = [] # Consolidate lab clean-up events lab_clean_up_events = [ - event for event in vr_events if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event + for event in vr_events + if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] non_lab_clean_up_events = [ - event for event in vr_events if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event + for event in vr_events + if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] if lab_clean_up_events: consolidated_event = { @@ -213,9 +257,13 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: * lab.User().proj(lab_manager="user_id", manager_slack="slack") ).fetch("manager_slack", as_dict=True) print(technician_manager_slack) - technician_manager_slack = [item["manager_slack"] for item in technician_manager_slack] + technician_manager_slack = [ + item["manager_slack"] for item in technician_manager_slack + ] - people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch(format="frame") + people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch( + format="frame" + ) load_dotenv() @@ -243,7 +291,9 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: tech_slack.append(tech_uid) if tech_slack: - tech_slack.append(technician_manager_slack[0]) # Fallback to default technician manager slack + tech_slack.append( + technician_manager_slack[0] + ) # Fallback to default technician manager slack coowner_dataframe: pd.DataFrame = ( ( @@ -264,7 +314,11 @@ def resolve_responsible_slack(row): lab_name = row.get("lab") token = str(schedule_today).strip() if schedule_today is not None else None - use_coowners = (token is None) or (token.lower() == "transport") or (token.lower() == "nothing") + use_coowners = ( + (token is None) + or (token.lower() == "transport") + or (token.lower() == "nothing") + ) slack_tags = [] @@ -281,14 +335,18 @@ def resolve_responsible_slack(row): # print(temp) slack_tags += temp["manager_slack"].tolist() slack_tags += temp["user_slack"].tolist() - slack_tags += coowner_dataframe.query(f"subject_fullname == '{row['subject_fullname']}'")["slack"].tolist() + slack_tags += coowner_dataframe.query( + f"subject_fullname == '{row['subject_fullname']}'" + )["slack"].tolist() slack_tags = set(slack_tags) # Remove duplicates return " ".join(f"<@{item}>" for item in slack_tags) subject_data = subject_data.copy() - subject_data["responsible_slack_tags"] = subject_data.apply(resolve_responsible_slack, axis=1) + subject_data["responsible_slack_tags"] = subject_data.apply( + resolve_responsible_slack, axis=1 + ) return subject_data @@ -311,7 +369,9 @@ def find_unreturned_subjects() -> pd.DataFrame: .fetch(format="frame") ) - unreturned_subjects: pd.DataFrame = local_transport_data[local_transport_data["transport_in_datetime"].isnull()] + unreturned_subjects: pd.DataFrame = local_transport_data[ + local_transport_data["transport_in_datetime"].isnull() + ] return unreturned_subjects @@ -326,13 +386,17 @@ def slack_alert_message_format_weight_water( print(missing_transport) temp_missing_transport = missing_transport.copy().reset_index() - notifiable_subjects = list(set( - temp_missing_transport["subject_fullname"].tolist() - + subjects_not_watered["subject_fullname"].tolist() - + subjects_not_weighted["subject_fullname"].tolist() - )) + notifiable_subjects = list( + set( + temp_missing_transport["subject_fullname"].tolist() + + subjects_not_watered["subject_fullname"].tolist() + + subjects_not_weighted["subject_fullname"].tolist() + ) + ) - slack_handles: list[str] = fetch_slack_handles_for_lab_managers_by_subject(notifiable_subjects) + slack_handles: list[str] = fetch_slack_handles_for_lab_managers_by_subject( + notifiable_subjects + ) lab_manager_text = "\n\n" if len(slack_handles) >= 1: lab_manager_text += "Lab Manager" @@ -342,7 +406,9 @@ def slack_alert_message_format_weight_water( slack_handles_formatted = ", ".join("<@" + handle + ">" for handle in slack_handles) if slack_handles_formatted: lab_manager_text += ( - " " + slack_handles_formatted + ", please be advised that your labs' subjects are listed below." + " " + + slack_handles_formatted + + ", please be advised that your labs' subjects are listed below." ) msep = dict() @@ -457,7 +523,9 @@ def slack_alert_message_format_weight_water( # Try to get responsible slack tags for this subject tags = "" try: - tags = missing_transport.iloc[i].get("responsible_slack_tags", "") # may be string or iterable + tags = missing_transport.iloc[i].get( + "responsible_slack_tags", "" + ) # may be string or iterable except Exception: tags = "" # Normalize tags to a string @@ -514,7 +582,9 @@ def _estimate_section_size(text: str, text_type: str = "mrkdwn") -> int: return estimate_size({"blocks": [block]}) -def split_text_to_fit(text: str, max_payload_size: int, text_type: str = "mrkdwn") -> list[str]: +def split_text_to_fit( + text: str, max_payload_size: int, text_type: str = "mrkdwn" +) -> list[str]: """Split text by lines so each chunk fits in max_payload_size when JSON-encoded. Uses actual JSON size estimation rather than raw character count. @@ -574,7 +644,9 @@ def _split_large_block(block: dict, max_size: int) -> list[dict]: return [block] -def send_slack_blocks_safely(webhook: str, blocks: list[dict], max_size: int = SLACK_MAX_CHARS): +def send_slack_blocks_safely( + webhook: str, blocks: list[dict], max_size: int = SLACK_MAX_CHARS +): """Send blocks together when possible, splitting only when necessary. Strategy: @@ -646,7 +718,9 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int): smaller = _split_large_block(block, max_size) for small_block in smaller: try: - su.send_slack_notification(webhook, {"blocks": [small_block]}) + su.send_slack_notification( + webhook, {"blocks": [small_block]} + ) time.sleep(0.5) except HTTPError: print("Failed to send block even after splitting") @@ -672,16 +746,22 @@ def main_water_weigh_alert(): # print(subject_data.columns) - subject_data = subject_data.loc[~subject_data["subject_fullname"].str.contains("test"), :] + subject_data = subject_data.loc[ + ~subject_data["subject_fullname"].str.contains("test"), : + ] subjects_not_watered = subject_data.loc[ - subject_data["current_need_water"] > 0, ["subject_fullname", "current_need_water", "responsible_slack_tags"] + subject_data["current_need_water"] > 0, + ["subject_fullname", "current_need_water", "responsible_slack_tags"], ] subjects_not_watered = subjects_not_watered.reset_index(drop=True) - subjects_not_watered["current_need_water"] = subjects_not_watered["current_need_water"].apply(lambda x: f"{x:.1f}") + subjects_not_watered["current_need_water"] = subjects_not_watered[ + "current_need_water" + ].apply(lambda x: f"{x:.1f}") # subjects_not_watered = subjects_not_watered.head() subjects_not_weighted = subject_data.loc[ - subject_data["need_weight"], ["subject_fullname", "need_weight", "responsible_slack_tags"] + subject_data["need_weight"], + ["subject_fullname", "need_weight", "responsible_slack_tags"], ] subjects_not_weighted = subjects_not_weighted.reset_index(drop=True) # subjects_not_weighted = subjects_not_weighted.head() @@ -707,10 +787,12 @@ def main_water_weigh_alert(): # Send alert slack_json_messages = slack_alert_message_format_weight_water( - subjects_not_watered, subjects_not_weighted, subjects_not_trained, missing_transport=subject_not_returned + subjects_not_watered, + subjects_not_weighted, + subjects_not_trained, + missing_transport=subject_not_returned, ) - # Send each message's blocks safely (splitting large blocks as needed) for this_webhook in webhooks_list: for message in slack_json_messages: diff --git a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py index cd747c13..8e8c831b 100644 --- a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py +++ b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py @@ -8,108 +8,111 @@ import u19_pipeline.automatic_job.params_config as config -#Functions to transfer files (globus, scp, smbclient) +# Functions to transfer files (globus, scp, smbclient) -#FOR PNI endpoint -#pni_ep_id = '6ce834d6-ff8a-11e6-bad1-22000b9a448b' -pni_ep_id = '005329dc-f31c-11ec-b3c1-15403b7b75ed' # pni BRAINCOGS ep points to /braininit/Data/ -pni_data_dir = '' +# FOR PNI endpoint +# pni_ep_id = '6ce834d6-ff8a-11e6-bad1-22000b9a448b' +pni_ep_id = "005329dc-f31c-11ec-b3c1-15403b7b75ed" # pni BRAINCOGS ep points to /braininit/Data/ +pni_data_dir = "" -#For tiger endpoint -public_key_location = '~/.ssh/id_ed25519.pub' -default_user = 'u19prod' # This will change to our automatic client for globus transfers -tiger_gpu_host = 'della.princeton.edu' -#tiger_ep_dir = 'a9df83d2-42f0-11e6-80cf-22000b1701d1' # tigress ep -tiger_ep_dir = '8e1bbdfe-d234-4a7c-93a5-86086a249918' # Endpoint of Della's /scratch/gpfs/. Our directory is ./BRAINCOGS/ +# For tiger endpoint +public_key_location = "~/.ssh/id_ed25519.pub" +default_user = ( + "u19prod" # This will change to our automatic client for globus transfers +) +tiger_gpu_host = "della.princeton.edu" +# tiger_ep_dir = 'a9df83d2-42f0-11e6-80cf-22000b1701d1' # tigress ep +tiger_ep_dir = "8e1bbdfe-d234-4a7c-93a5-86086a249918" # Endpoint of Della's /scratch/gpfs/. Our directory is ./BRAINCOGS/ -tiger_home_dir_globus = '/BRAINCOGS/Data/' +tiger_home_dir_globus = "/BRAINCOGS/Data/" -#Slurm default values for queue job +# Slurm default values for queue job slurm_dict_tiger_default = { - 'job-name': 'kilosort2', - 'nodes': 1, - 'ntasks': 1, - 'time': '10:00:00', - 'mem': '100G', - 'gres': 'gpu:1', - 'mail-user': 'alvaros@princeton.edu', - 'mail-type': ['END'], - 'output': 'OutputLog/job_id_${job_id}".log', - 'error': 'ErrorLog/job_id_${job_id}".log' + "job-name": "kilosort2", + "nodes": 1, + "ntasks": 1, + "time": "10:00:00", + "mem": "100G", + "gres": "gpu:1", + "mail-user": "alvaros@princeton.edu", + "mail-type": ["END"], + "output": 'OutputLog/job_id_${job_id}".log', + "error": 'ErrorLog/job_id_${job_id}".log', } slurm_dict_spockmk2_ephys = { - 'job-name': 'kilosort2', - 'nodes': 1, - 'ntasks': 1, - 'time': '30:00:00', - 'mem': '50G', - 'gres': 'gpu:2', - 'mail-user': 'alvaros@princeton.edu', - 'mail-type': ['END'], - 'output': 'OutputLog/job_id_${job_id}".log', - 'error': 'ErrorLog/job_id_${job_id}".log' + "job-name": "kilosort2", + "nodes": 1, + "ntasks": 1, + "time": "30:00:00", + "mem": "50G", + "gres": "gpu:2", + "mail-user": "alvaros@princeton.edu", + "mail-type": ["END"], + "output": 'OutputLog/job_id_${job_id}".log', + "error": 'ErrorLog/job_id_${job_id}".log', } slurm_dict_spock_default = { - 'job-name': 'kilosort2', - 'nodes': 1, - 'ntasks': 1, - 'time': '30:00:00', - 'mem': '50G', - 'gres': 'gpu:2', - 'mail-user': 'alvaros@princeton.edu', - 'mail-type': ['END'], - 'output': 'OutputLog/job_id_${job_id}".log', - 'error': 'ErrorLog/job_id_${job_id}".log' + "job-name": "kilosort2", + "nodes": 1, + "ntasks": 1, + "time": "30:00:00", + "mem": "50G", + "gres": "gpu:2", + "mail-user": "alvaros@princeton.edu", + "mail-type": ["END"], + "output": 'OutputLog/job_id_${job_id}".log', + "error": 'ErrorLog/job_id_${job_id}".log', } -#PNI directories -pni_root_data_dir = dj.config['custom']['root_data_dir'] +# PNI directories +pni_root_data_dir = dj.config["custom"]["root_data_dir"] -tiger_home_dir = '/scratch/gpfs/BRAINCOGS' -spock_home_dir = '/mnt/cup/braininit/Shared/repos/AutomaticPipelineProcessing' +tiger_home_dir = "/scratch/gpfs/BRAINCOGS" +spock_home_dir = "/mnt/cup/braininit/Shared/repos/AutomaticPipelineProcessing" -#Cluster directories +# Cluster directories cluster_vars = { "tiger": { - "home_dir": tiger_home_dir, - "root_data_dir_globus": tiger_home_dir_globus + "/Raw", - "processed_data_dir_globus": tiger_home_dir_globus + "/Processed", - "root_data_dir": tiger_home_dir + "/Data/Raw", - "processed_data_dir": tiger_home_dir + "/Data/Processed", - "slurm_files_dir": tiger_home_dir + "/SlurmFiles", - "params_files_dir": tiger_home_dir + "/ParameterFiles", - "chanmap_files_dir": tiger_home_dir + "/ChanMapFiles", - "electrophysiology_process_dir": tiger_home_dir + "/electrophysiology_processing", - "imaging_process_dir": tiger_home_dir + "/imaging_processing", - "log_files_dir": tiger_home_dir + "/OutputLog", - "error_files_dir": tiger_home_dir + "/ErrorLog", - "user": default_user, - "slurm_default": slurm_dict_tiger_default, - "hostname": "della-gpu.princeton.edu", - "script_path": "", - "conda_env": '/home/alvaros/.conda/envs/BrainCogsEphysSorters_env' - + "home_dir": tiger_home_dir, + "root_data_dir_globus": tiger_home_dir_globus + "/Raw", + "processed_data_dir_globus": tiger_home_dir_globus + "/Processed", + "root_data_dir": tiger_home_dir + "/Data/Raw", + "processed_data_dir": tiger_home_dir + "/Data/Processed", + "slurm_files_dir": tiger_home_dir + "/SlurmFiles", + "params_files_dir": tiger_home_dir + "/ParameterFiles", + "chanmap_files_dir": tiger_home_dir + "/ChanMapFiles", + "electrophysiology_process_dir": tiger_home_dir + + "/electrophysiology_processing", + "imaging_process_dir": tiger_home_dir + "/imaging_processing", + "log_files_dir": tiger_home_dir + "/OutputLog", + "error_files_dir": tiger_home_dir + "/ErrorLog", + "user": default_user, + "slurm_default": slurm_dict_tiger_default, + "hostname": "della-gpu.princeton.edu", + "script_path": "", + "conda_env": "/home/alvaros/.conda/envs/BrainCogsEphysSorters_env", }, "spock": { - "home_dir": spock_home_dir, - "root_data_dir_globus": pni_data_dir + "/Raw", - "processed_data_dir_globus": pni_data_dir + "/Processed", - "root_data_dir": spock_home_dir + "/Raw", - "processed_data_dir": spock_home_dir + "/Processed", - "slurm_files_dir": spock_home_dir + "/SlurmFiles", - "params_files_dir": spock_home_dir + "/ParameterFiles", - "chanmap_files_dir": spock_home_dir + "/ChanMapFiles", - "electrophysiology_process_dir": spock_home_dir + "/electrophysiology_processing", - "imaging_process_dir": spock_home_dir + "/imaging_processing", - "log_files_dir": spock_home_dir + "/u19_pipeline/automatic_job/OutputLog", - "error_files_dir": spock_home_dir + "/u19_pipeline/automatic_job/ErrorLog", - "user": default_user, - "slurm_default": slurm_dict_spock_default, - "hostname": "spockmk2-loginvm.pni.princeton.edu", - "script_path": "", - "conda_env": 'u19_pipeline_python_env' - } + "home_dir": spock_home_dir, + "root_data_dir_globus": pni_data_dir + "/Raw", + "processed_data_dir_globus": pni_data_dir + "/Processed", + "root_data_dir": spock_home_dir + "/Raw", + "processed_data_dir": spock_home_dir + "/Processed", + "slurm_files_dir": spock_home_dir + "/SlurmFiles", + "params_files_dir": spock_home_dir + "/ParameterFiles", + "chanmap_files_dir": spock_home_dir + "/ChanMapFiles", + "electrophysiology_process_dir": spock_home_dir + + "/electrophysiology_processing", + "imaging_process_dir": spock_home_dir + "/imaging_processing", + "log_files_dir": spock_home_dir + "/u19_pipeline/automatic_job/OutputLog", + "error_files_dir": spock_home_dir + "/u19_pipeline/automatic_job/ErrorLog", + "user": default_user, + "slurm_default": slurm_dict_spock_default, + "hostname": "spockmk2-loginvm.pni.princeton.edu", + "script_path": "", + "conda_env": "u19_pipeline_python_env", + }, } @@ -118,7 +121,8 @@ def get_cluster_vars(cluster): if cluster in cluster_vars: return cluster_vars[cluster] else: - raise('Non existing cluster') + raise ("Non existing cluster") + def scp_file_transfer(source, dest): @@ -144,87 +148,114 @@ def cp_file_transfer(source, dest): transfer_status = p.wait() return transfer_status -def request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, dest_filepath): - source_fullpath = source_ep+ ":" + source_filepath - dest_fullpath = dest_ep + ":" + dest_filepath - - globus_command = ["globus", "transfer", source_fullpath, dest_fullpath, '--label', job_id_str, '--recursive', '--format', 'json'] - print('**********************************') +def request_globus_transfer( + job_id_str, source_ep, dest_ep, source_filepath, dest_filepath +): + + source_fullpath = source_ep + ":" + source_filepath + dest_fullpath = dest_ep + ":" + dest_filepath + + globus_command = [ + "globus", + "transfer", + source_fullpath, + dest_fullpath, + "--label", + job_id_str, + "--recursive", + "--format", + "json", + ] + print("**********************************") print(globus_command) - print('**********************************') + print("**********************************") p = subprocess.run(globus_command, capture_output=True) print(p) transfer_request = dict() - print('p.stderr',p.stderr) - print('p.stdout', p.stdout) + print("p.stderr", p.stderr) + print("p.stdout", p.stdout) if len(p.stderr) == 0: - try: - dict_output = json.loads(p.stdout.decode('UTF-8')) - transfer_request['status'] = config.system_process['SUCCESS'] - transfer_request['task_id'] = dict_output['task_id'] - except Exception: - print('stdout is not a valid json, probably an error') - transfer_request['status'] = config.system_process['ERROR'] - transfer_request['error_info'] = p.stdout.decode('UTF-8') - #dict_output = translate_globus_output(p.stdout) + try: + dict_output = json.loads(p.stdout.decode("UTF-8")) + transfer_request["status"] = config.system_process["SUCCESS"] + transfer_request["task_id"] = dict_output["task_id"] + except Exception: + print("stdout is not a valid json, probably an error") + transfer_request["status"] = config.system_process["ERROR"] + transfer_request["error_info"] = p.stdout.decode("UTF-8") + # dict_output = translate_globus_output(p.stdout) else: - transfer_request['status'] = config.system_process['ERROR'] - transfer_request['error_info'] = p.stderr.decode('UTF-8') + transfer_request["status"] = config.system_process["ERROR"] + transfer_request["error_info"] = p.stderr.decode("UTF-8") return transfer_request def request_globus_transfer_status(job_id): - globus_command = ["globus", "task", "show", job_id, '--format', 'json'] + globus_command = ["globus", "task", "show", job_id, "--format", "json"] s = subprocess.run(globus_command, capture_output=True) - task_output = json.loads(s.stdout.decode('UTF-8')) + task_output = json.loads(s.stdout.decode("UTF-8")) transfer_request = dict() - if task_output['status'] == 'SUCCEEDED': - transfer_request['status'] = config.system_process['COMPLETED'] - elif task_output['status'] in ['PENDING','RETRYING', 'ACTIVE']: - transfer_request['status'] = config.system_process['SUCCESS'] + if task_output["status"] == "SUCCEEDED": + transfer_request["status"] = config.system_process["COMPLETED"] + elif task_output["status"] in ["PENDING", "RETRYING", "ACTIVE"]: + transfer_request["status"] = config.system_process["SUCCESS"] else: - transfer_request['status'] = config.system_process['ERROR'] + transfer_request["status"] = config.system_process["ERROR"] return transfer_request def globus_transfer_to_tiger(job_id, raw_rel_path, modality): - job_id_str = "job_id_"+str(job_id)+"_raw_transfer" + job_id_str = "job_id_" + str(job_id) + "_raw_transfer" source_ep = pni_ep_id - dest_ep = tiger_ep_dir + dest_ep = tiger_ep_dir - source_filepath = pathlib.Path(cluster_vars['spock']['root_data_dir_globus'], modality, raw_rel_path).as_posix() - dest_filepath = pathlib.Path(cluster_vars['tiger']['root_data_dir_globus'], modality, raw_rel_path).as_posix() + source_filepath = pathlib.Path( + cluster_vars["spock"]["root_data_dir_globus"], modality, raw_rel_path + ).as_posix() + dest_filepath = pathlib.Path( + cluster_vars["tiger"]["root_data_dir_globus"], modality, raw_rel_path + ).as_posix() - transfer_request = request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, dest_filepath) + transfer_request = request_globus_transfer( + job_id_str, source_ep, dest_ep, source_filepath, dest_filepath + ) return transfer_request + def globus_transfer_to_pni(job_id, processed_rel_path, modality): - job_id_str = "job_id_"+str(job_id)+"_processed_transfer" + job_id_str = "job_id_" + str(job_id) + "_processed_transfer" source_ep = tiger_ep_dir - dest_ep = pni_ep_id + dest_ep = pni_ep_id - dest_filepath = pathlib.Path(cluster_vars['spock']['processed_data_dir_globus'], modality, processed_rel_path).as_posix() - source_filepath = pathlib.Path(cluster_vars['tiger']['processed_data_dir_globus'], modality, processed_rel_path).as_posix() + dest_filepath = pathlib.Path( + cluster_vars["spock"]["processed_data_dir_globus"], modality, processed_rel_path + ).as_posix() + source_filepath = pathlib.Path( + cluster_vars["tiger"]["processed_data_dir_globus"], modality, processed_rel_path + ).as_posix() - transfer_request = request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, dest_filepath) + transfer_request = request_globus_transfer( + job_id_str, source_ep, dest_ep, source_filepath, dest_filepath + ) return transfer_request + def translate_globus_output(stdout_process): - u = stdout_process.decode('UTF-8') + u = stdout_process.decode("UTF-8") - n = u.split(sep='\n') - n2 = [x.split(sep=':', maxsplit=1) for x in n] + n = u.split(sep="\n") + n2 = [x.split(sep=":", maxsplit=1) for x in n] flat_list = [item for l in n2 for item in l] flat_list2 = [x.strip() for x in flat_list] @@ -233,26 +264,28 @@ def translate_globus_output(stdout_process): return d1 -def transfer_log_file(recording_process_id, program_selection_params, user_host, log_type='ERROR'): - ''' +def transfer_log_file( + recording_process_id, program_selection_params, user_host, log_type="ERROR" +): + """ Transfer and send parameter files for processing - ''' - this_cluster_vars = get_cluster_vars(program_selection_params['process_cluster']) - if log_type == 'ERROR': - cluster_log_file_dir = this_cluster_vars['error_files_dir'] - local_log_file_dir = dj.config['custom']['error_logs_dir'] + """ + this_cluster_vars = get_cluster_vars(program_selection_params["process_cluster"]) + if log_type == "ERROR": + cluster_log_file_dir = this_cluster_vars["error_files_dir"] + local_log_file_dir = dj.config["custom"]["error_logs_dir"] else: - cluster_log_file_dir = this_cluster_vars['log_files_dir'] - local_log_file_dir = dj.config['custom']['output_logs_dir'] + cluster_log_file_dir = this_cluster_vars["log_files_dir"] + local_log_file_dir = dj.config["custom"]["output_logs_dir"] - user_host = this_cluster_vars['user']+'@'+this_cluster_vars['hostname'] + user_host = this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] - default_log_filename = 'job_id_%s.log' + default_log_filename = "job_id_%s.log" log_filename = default_log_filename % (recording_process_id) - log_file_local_path = pathlib.Path(local_log_file_dir,log_filename).as_posix() - log_file_cluster_path = pathlib.Path(cluster_log_file_dir,log_filename).as_posix() - chanmap_file_full_path = user_host+':'+log_file_cluster_path + log_file_local_path = pathlib.Path(local_log_file_dir, log_filename).as_posix() + log_file_cluster_path = pathlib.Path(cluster_log_file_dir, log_filename).as_posix() + chanmap_file_full_path = user_host + ":" + log_file_cluster_path status = scp_file_transfer(chanmap_file_full_path, log_file_local_path) @@ -261,48 +294,57 @@ def transfer_log_file(recording_process_id, program_selection_params, user_host, def get_error_log_str(recording_process_id): - error_log_data = '' - default_log_filename = 'job_id_%s.log' + error_log_data = "" + default_log_filename = "job_id_%s.log" - local_log_file_dir = dj.config['custom']['error_logs_dir'] + local_log_file_dir = dj.config["custom"]["error_logs_dir"] log_filename = default_log_filename % (recording_process_id) - log_file_local_path = pathlib.Path(local_log_file_dir,log_filename).as_posix() + log_file_local_path = pathlib.Path(local_log_file_dir, log_filename).as_posix() if os.path.exists(log_file_local_path): with open(log_file_local_path) as error_log_file: - error_log_data = ' '.join(error_log_file.readlines()) + error_log_data = " ".join(error_log_file.readlines()) - error_log_data = error_log_data.replace("activate the default environment with 'conda activate' or create a new environment to customize with 'conda create'.\n",'') + error_log_data = error_log_data.replace( + "activate the default environment with 'conda activate' or create a new environment to customize with 'conda create'.\n", + "", + ) - print('error_log_data xxxxxxxxxx') - print('type(error_log_data)', type(error_log_data)) - print('len(error_log_data)', len(error_log_data)) - #if len(error_log_data) > 400: + print("error_log_data xxxxxxxxxx") + print("type(error_log_data)", type(error_log_data)) + print("len(error_log_data)", len(error_log_data)) + # if len(error_log_data) > 400: # print(error_log_data[-400:]) - #else: + # else: # print(error_log_data) - print('error_log_data xxxxxxxxxx') + print("error_log_data xxxxxxxxxx") return error_log_data -def check_directory_exists_cluster(directory, cluster, modality, type_dir='raw'): - ''' +def check_directory_exists_cluster(directory, cluster, modality, type_dir="raw"): + """ Check if directory exists in cluster, runs check_directory_script (check_directory.sh) in cluster machine Output dir_exists 1 if directory exists / 0 otherwise - ''' + """ this_cluster_vars = get_cluster_vars(cluster) - if type_dir=='raw': - final_directory = pathlib.Path(cluster_vars[cluster]['root_data_dir'], modality, directory).as_posix() + if type_dir == "raw": + final_directory = pathlib.Path( + cluster_vars[cluster]["root_data_dir"], modality, directory + ).as_posix() else: - final_directory = pathlib.Path(cluster_vars[cluster]['processed_data_dir'], modality, directory).as_posix() + final_directory = pathlib.Path( + cluster_vars[cluster]["processed_data_dir"], modality, directory + ).as_posix() - base_command = "ssh " + this_cluster_vars['user']+'@'+this_cluster_vars['hostname'] + " " + base_command = ( + "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " " + ) - command = "'if [ -d " + final_directory + " ]; " + command = "'if [ -d " + final_directory + " ]; " post_command = """then echo "1" else echo "0" @@ -318,14 +360,21 @@ def check_directory_exists_cluster(directory, cluster, modality, type_dir='raw') def delete_directory_cluster(directory, cluster): - ''' + """ Delete directory in cluster - ''' + """ this_cluster_vars = get_cluster_vars(cluster) - command = "ssh " + this_cluster_vars['user']+'@'+this_cluster_vars['hostname']\ - + " 'rm -R " + directory + " '" + command = ( + "ssh " + + this_cluster_vars["user"] + + "@" + + this_cluster_vars["hostname"] + + " 'rm -R " + + directory + + " '" + ) p = subprocess.run(command, shell=True) output = p.returncode @@ -337,36 +386,40 @@ def delete_directory_cluster(directory, cluster): def delete_directory_tiger_globus(modality, raw_rel_path): - ''' + """ Delete directory in cluster with globus command (for Raw directory globus) - ''' + """ source_ep = tiger_ep_dir - source_filepath = pathlib.Path(cluster_vars['tiger']['root_data_dir_globus'], modality, raw_rel_path).as_posix() + source_filepath = pathlib.Path( + cluster_vars["tiger"]["root_data_dir_globus"], modality, raw_rel_path + ).as_posix() - source_fullpath = source_ep+ ":" + source_filepath + source_fullpath = source_ep + ":" + source_filepath - globus_command = ["globus", "delete", source_fullpath, '--recursive'] + globus_command = ["globus", "delete", source_fullpath, "--recursive"] p = subprocess.run(globus_command, capture_output=True) output = p.returncode return output -def delete_empty_data_directory_cluster(cluster, type='raw'): +def delete_empty_data_directory_cluster(cluster, type="raw"): """ Check if directory (or its childs) contains files and if not delete them """ max_deletion = 10 this_cluster_vars = get_cluster_vars(cluster) - base_command = "ssh " + this_cluster_vars['user']+'@'+this_cluster_vars['hostname'] + ' ' + base_command = ( + "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " " + ) # Check base directory to delete - if type == 'raw': - filepath = this_cluster_vars['root_data_dir'] + if type == "raw": + filepath = this_cluster_vars["root_data_dir"] else: - filepath = this_cluster_vars['processed_data_dir'] + filepath = this_cluster_vars["processed_data_dir"] # Repeat always if we find a directory to delete total_deletion = 0 @@ -374,52 +427,49 @@ def delete_empty_data_directory_cluster(cluster, type='raw'): deleted_dirs = 0 # List all directories on base (raw/processed) directory - command_list_dir = base_command + 'find ' + filepath + ' -type d -print' + command_list_dir = base_command + "find " + filepath + " -type d -print" p = subprocess.run(command_list_dir, shell=True, capture_output=True) list_dir = p.stdout.decode() - list_dir = list_dir.split('\n') - if '.' in list_dir: - list_dir.remove('.') - if '' in list_dir: - list_dir.remove('') + list_dir = list_dir.split("\n") + if "." in list_dir: + list_dir.remove(".") + if "" in list_dir: + list_dir.remove("") for dir in list_dir: - # Check if directory has no files in it (empty) - command = base_command + 'find ' + dir + ' -type f | wc -l' + command = base_command + "find " + dir + " -type f | wc -l" num_files = subprocess.check_output(command, shell=True) num_files = int(num_files.decode().strip()) - #If directory empty, delete it + # If directory empty, delete it if num_files == 0: - if type == 'raw': - + if type == "raw": # For raw directories, delete with globus # Translate local directory to globus dir - dir_globus = dir.replace(this_cluster_vars['root_data_dir'], '') + dir_globus = dir.replace(this_cluster_vars["root_data_dir"], "") dir_globus = dir_globus[1:] index_sep = dir_globus.index("/") modality = dir_globus[0:index_sep] - dir_globus = dir_globus[index_sep+1:] + dir_globus = dir_globus[index_sep + 1 :] - #Delete with globus + # Delete with globus status = delete_directory_tiger_globus(modality, dir_globus) time.sleep(2) - if status == config.system_process['SUCCESS']: + if status == config.system_process["SUCCESS"]: deleted_dirs = 1 break - total_deletion +=1 + total_deletion += 1 else: - #Delete processed files with "normal" ssh + # Delete processed files with "normal" ssh status = delete_directory_cluster(dir, cluster) - if status == config.system_process['SUCCESS']: + if status == config.system_process["SUCCESS"]: deleted_dirs = 1 break - total_deletion +=1 + total_deletion += 1 # If in this round no directories were deleted we are done if deleted_dirs == 0 or total_deletion >= max_deletion: break - diff --git a/u19_pipeline/automatic_job/create_missing_lfps_script.py b/u19_pipeline/automatic_job/create_missing_lfps_script.py index a23be741..01c1a7c6 100644 --- a/u19_pipeline/automatic_job/create_missing_lfps_script.py +++ b/u19_pipeline/automatic_job/create_missing_lfps_script.py @@ -1,4 +1,3 @@ - import pathlib import sys @@ -10,22 +9,36 @@ def main(recording_id): rec_query = dict() - rec_query['recording_id'] = recording_id - - recording_processes = (recording_process.Processing() & rec_query).fetch('job_id', 'recording_id', 'fragment_number', 'recording_process_pre_path', as_dict=True) - #Create lfp trace if needed (neuropixel 2.0 probes) - recording_directory = (recording.Recording & rec_query).fetch1('recording_directory') - recording_directory = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][0], recording_directory).parent.as_posix() + rec_query["recording_id"] = recording_id + + recording_processes = (recording_process.Processing() & rec_query).fetch( + "job_id", + "recording_id", + "fragment_number", + "recording_process_pre_path", + as_dict=True, + ) + # Create lfp trace if needed (neuropixel 2.0 probes) + recording_directory = (recording.Recording & rec_query).fetch1( + "recording_directory" + ) + recording_directory = pathlib.Path( + dj.config["custom"]["ephys_root_data_dir"][0], recording_directory + ).parent.as_posix() for i in recording_processes: - probe_dir = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][0], i['recording_process_pre_path']).as_posix() - ephys_pipeline.create_lfp_trace(config.catgt_script, recording_directory, probe_dir) + probe_dir = pathlib.Path( + dj.config["custom"]["ephys_root_data_dir"][0], + i["recording_process_pre_path"], + ).as_posix() + ephys_pipeline.create_lfp_trace( + config.catgt_script, recording_directory, probe_dir + ) if __name__ == "__main__": - - from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() args = sys.argv[1:] print(args) - main(args[0]) \ No newline at end of file + main(args[0]) diff --git a/u19_pipeline/automatic_job/cronjob_automatic_job.py b/u19_pipeline/automatic_job/cronjob_automatic_job.py index e2804630..35ff5767 100644 --- a/u19_pipeline/automatic_job/cronjob_automatic_job.py +++ b/u19_pipeline/automatic_job/cronjob_automatic_job.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file @@ -18,7 +17,5 @@ rec_process_handler.RecProcessHandler.pipeline_handler_main() time.sleep(5) -#Check if we need to delete empty directories -#rec_process_handler.RecProcessHandler.check_job_process_deletion() - - +# Check if we need to delete empty directories +# rec_process_handler.RecProcessHandler.check_job_process_deletion() diff --git a/u19_pipeline/automatic_job/ephys_element_ingest.py b/u19_pipeline/automatic_job/ephys_element_ingest.py index a733a9c7..c85d496e 100644 --- a/u19_pipeline/automatic_job/ephys_element_ingest.py +++ b/u19_pipeline/automatic_job/ephys_element_ingest.py @@ -26,7 +26,7 @@ (for an example, see: https://github.com/ttngu207/workflow-ephys/blob/main/notebooks/run_workflow.ipynb) """ -acq_software = 'SpikeGLX' +acq_software = "SpikeGLX" def process_session(sess_key): @@ -37,36 +37,49 @@ def process_session(sess_key): :param scan_key: a `KEY` of `acquisition.Session` """ - sess_dir = pathlib.Path(find_full_path(get_ephys_root_data_dir(), - get_session_directory(sess_key))) - ephys_meta_filepaths = [fp for fp in sess_dir.rglob('*.ap.meta')] + sess_dir = pathlib.Path( + find_full_path(get_ephys_root_data_dir(), get_session_directory(sess_key)) + ) + ephys_meta_filepaths = [fp for fp in sess_dir.rglob("*.ap.meta")] - print('ephys_meta_filepaths', ephys_meta_filepaths) + print("ephys_meta_filepaths", ephys_meta_filepaths) if not len(ephys_meta_filepaths): - print(f'No SpikeGLX data found for session:{sess_key} - at {sess_dir}') + print(f"No SpikeGLX data found for session:{sess_key} - at {sess_dir}") return probe_list, probe_insertion_list = [], [] for meta_filepath in ephys_meta_filepaths: - print('meta_filepath', meta_filepath) + print("meta_filepath", meta_filepath) spikeglx_meta = spikeglx.SpikeGLXMeta(meta_filepath) - probe_key = {'probe_type': spikeglx_meta.probe_model, 'probe': spikeglx_meta.probe_SN} - if probe_key['probe'] not in [p['probe'] for p in probe_list] and probe_key not in probe_element.Probe(): + probe_key = { + "probe_type": spikeglx_meta.probe_model, + "probe": spikeglx_meta.probe_SN, + } + if ( + probe_key["probe"] not in [p["probe"] for p in probe_list] + and probe_key not in probe_element.Probe() + ): probe_list.append(probe_key) probe_dir = meta_filepath.parent - probe_number = re.search('(imec)?\d{1}$', probe_dir.name).group() - probe_number = int(probe_number.replace('imec', '')) + probe_number = re.search("(imec)?\d{1}$", probe_dir.name).group() + probe_number = int(probe_number.replace("imec", "")) - probe_insertion_list.append({**sess_key, 'probe': spikeglx_meta.probe_SN, 'insertion_number': int(probe_number)}) + probe_insertion_list.append( + { + **sess_key, + "probe": spikeglx_meta.probe_SN, + "insertion_number": int(probe_number), + } + ) print(f"{probe_list =}") probe_element.Probe.insert(probe_list, skip_duplicates=True) ephys_element.ProbeInsertion.insert(probe_insertion_list, skip_duplicates=True) -if __name__ == '__main__': - for sess_key in ephys_pipeline.EphysPipelineSession.fetch('KEY'): - process_session(sess_key) \ No newline at end of file +if __name__ == "__main__": + for sess_key in ephys_pipeline.EphysPipelineSession.fetch("KEY"): + process_session(sess_key) diff --git a/u19_pipeline/automatic_job/ephys_element_populate.py b/u19_pipeline/automatic_job/ephys_element_populate.py index 3df20a53..ae2e2318 100644 --- a/u19_pipeline/automatic_job/ephys_element_populate.py +++ b/u19_pipeline/automatic_job/ephys_element_populate.py @@ -3,64 +3,77 @@ from u19_pipeline.ephys_pipeline import ephys_element, probe_element -def populate_element_data(job_id, display_progress=True, reserve_jobs=False, suppress_errors=False): - - populate_settings = {'display_progress': display_progress, - 'reserve_jobs': reserve_jobs, - 'suppress_errors': suppress_errors} - - process_key = (recording_process.Processing * recording.Recording & - dict(recording_modality='electrophysiology', - job_id=job_id)).fetch1('KEY') - - fragment_number, recording_process_pre_path, recording_process_post_path = \ - (recording_process.Processing & process_key).fetch1( - 'fragment_number', - 'recording_process_pre_path', - 'recording_process_post_path') - - precluster_param_steps_id, paramset_idx = \ - (recording_process.Processing.EphysParams & process_key - ).fetch1('precluster_param_steps_id', - 'paramset_idx') - - precluster_paramsets = (ephys_element.PreClusterParamSteps.Step() & - dict( - precluster_param_steps_id=precluster_param_steps_id) - ).fetch('paramset_idx') - - clustering_method = (ephys_element.ClusteringParamSet & - dict(paramset_idx=paramset_idx)).fetch1( - 'clustering_method') - - if len(precluster_paramsets)==0: - task_mode = 'none' +def populate_element_data( + job_id, display_progress=True, reserve_jobs=False, suppress_errors=False +): + + populate_settings = { + "display_progress": display_progress, + "reserve_jobs": reserve_jobs, + "suppress_errors": suppress_errors, + } + + process_key = ( + recording_process.Processing * recording.Recording + & dict(recording_modality="electrophysiology", job_id=job_id) + ).fetch1("KEY") + + fragment_number, recording_process_pre_path, recording_process_post_path = ( + recording_process.Processing & process_key + ).fetch1( + "fragment_number", "recording_process_pre_path", "recording_process_post_path" + ) + + precluster_param_steps_id, paramset_idx = ( + recording_process.Processing.EphysParams & process_key + ).fetch1("precluster_param_steps_id", "paramset_idx") + + precluster_paramsets = ( + ephys_element.PreClusterParamSteps.Step() + & dict(precluster_param_steps_id=precluster_param_steps_id) + ).fetch("paramset_idx") + + clustering_method = ( + ephys_element.ClusteringParamSet & dict(paramset_idx=paramset_idx) + ).fetch1("clustering_method") + + if len(precluster_paramsets) == 0: + task_mode = "none" else: - task_mode = 'load' + task_mode = "load" - precluster_key = dict(recording_id=process_key['recording_id'], - insertion_number=fragment_number, - precluster_param_steps_id=precluster_param_steps_id) + precluster_key = dict( + recording_id=process_key["recording_id"], + insertion_number=fragment_number, + precluster_param_steps_id=precluster_param_steps_id, + ) ephys_element.PreClusterTask.insert1( - dict(**precluster_key, - precluster_output_dir=recording_process_pre_path, - task_mode=task_mode), - skip_duplicates=True) + dict( + **precluster_key, + precluster_output_dir=recording_process_pre_path, + task_mode=task_mode, + ), + skip_duplicates=True, + ) ephys_element.PreCluster.populate(precluster_key, **populate_settings) - if '1.0' in (ephys_element.ProbeInsertion * probe_element.Probe & - precluster_key).fetch1('probe_type'): + if "1.0" in ( + ephys_element.ProbeInsertion * probe_element.Probe & precluster_key + ).fetch1("probe_type"): ephys_element.LFP.populate(precluster_key, **populate_settings) - cluster_key = dict(**precluster_key, - paramset_idx=paramset_idx) + cluster_key = dict(**precluster_key, paramset_idx=paramset_idx) ephys_element.ClusteringTask.insert1( - dict(**cluster_key, - clustering_output_dir=f'{recording_process_post_path}/{clustering_method}_output', - task_mode='load'), skip_duplicates=True) + dict( + **cluster_key, + clustering_output_dir=f"{recording_process_post_path}/{clustering_method}_output", + task_mode="load", + ), + skip_duplicates=True, + ) ephys_element.Clustering.populate(cluster_key, **populate_settings) @@ -72,8 +85,8 @@ def populate_element_data(job_id, display_progress=True, reserve_jobs=False, sup # ephys_element.WaveformSet.populate(cluster_key, **populate_settings) - return config.status_update_idx['NEXT_STATUS'] - + return config.status_update_idx["NEXT_STATUS"] -if __name__ == '__main__': + +if __name__ == "__main__": populate_element_data() diff --git a/u19_pipeline/automatic_job/imaging_element.py b/u19_pipeline/automatic_job/imaging_element.py index e802e411..e9dfbc44 100644 --- a/u19_pipeline/automatic_job/imaging_element.py +++ b/u19_pipeline/automatic_job/imaging_element.py @@ -22,11 +22,11 @@ # session_date = '2021-02-27' # session_number = 0 -acq_software = 'ScanImage' -#recording_id = os.environ['recording_id'] -recording_process_id = os.environ['recording_process_id'] -#process_method = os.environ['process_method'] -#paramset_idx = os.environ['paramset_idx'] +acq_software = "ScanImage" +# recording_id = os.environ['recording_id'] +recording_process_id = os.environ["recording_process_id"] +# process_method = os.environ['process_method'] +# paramset_idx = os.environ['paramset_idx'] # parameters = { # 'look_one_level_down': 0.0, @@ -92,102 +92,114 @@ # 'yrange': np.array([0, 0])} - -# parameters_caiman = {'fr': 30, +# parameters_caiman = {'fr': 30, # 'decay_time': 0.4, -# 'strides': (48, 48), -# 'overlaps': (24, 24), -# 'max_shifts': (6,6), -# 'max_deviation_rigid': 3, -# 'pw_rigid': True, -# 'p': 1, -# 'gnb': 2, -# 'merge_thr': 0.85, -# 'rf': 15, -# 'stride_cnmf': 6, -# 'K': 4, -# 'gSig': [4, 4], -# 'method_init': 'greedy_roi', -# 'ssub': 1, -# 'tsub': 1, -# 'min_SNR': 2.0, -# 'rval_thr': 0.85, -# 'cnn_thr': 0.99, +# 'strides': (48, 48), +# 'overlaps': (24, 24), +# 'max_shifts': (6,6), +# 'max_deviation_rigid': 3, +# 'pw_rigid': True, +# 'p': 1, +# 'gnb': 2, +# 'merge_thr': 0.85, +# 'rf': 15, +# 'stride_cnmf': 6, +# 'K': 4, +# 'gSig': [4, 4], +# 'method_init': 'greedy_roi', +# 'ssub': 1, +# 'tsub': 1, +# 'min_SNR': 2.0, +# 'rval_thr': 0.85, +# 'cnn_thr': 0.99, # 'cnn_lowest': 0.1} -# imaging_element.ProcessingParamSet.insert_new_params('suite2p', 0, 'Calcium imaging analysis with Suite2p using default Suite2p parameters', parameters) -# imaging_element.ProcessingParamSet.insert_new_params('caiman', 1, 'Calcium imaging analysis with CaImAn using default CaImAn parameters', parameters_caiman) +# imaging_element.ProcessingParamSet.insert_new_params('suite2p', 0, 'Calcium imaging analysis with Suite2p using default Suite2p parameters', parameters) +# imaging_element.ProcessingParamSet.insert_new_params('caiman', 1, 'Calcium imaging analysis with CaImAn using default CaImAn parameters', parameters_caiman) -#scan_key = (imaging_rec.Scan & dict(recording_id=recording_id)).fetch1('KEY') +# scan_key = (imaging_rec.Scan & dict(recording_id=recording_id)).fetch1('KEY') -#Recording process key +# Recording process key rec_process_key = dict(recording_process_id=recording_process_id) -rec_process_str_key = 'recording_process_id_'+str(recording_process_id) +rec_process_str_key = "recording_process_id_" + str(recording_process_id) -#Get fov key +# Get fov key rec_process = (imaging_rec.ImagingProcessing & rec_process_key).fetch1() fov_key = rec_process.copy() -fov_key.pop('recording_process_id') +fov_key.pop("recording_process_id") -#Recording process extra info -recording_process_info = (lab.Location * recording.RecordingProcess * recording.Recording & rec_process_key).fetch( - 'acquisition_type', 'preprocess_paramset_idx', 'process_paramset_idx', as_dict=True) +# Recording process extra info +recording_process_info = ( + lab.Location * recording.RecordingProcess * recording.Recording & rec_process_key +).fetch( + "acquisition_type", "preprocess_paramset_idx", "process_paramset_idx", as_dict=True +) -#Paramset idx and key -paramset_idx = recording_process_info[0]['process_paramset_idx'] +# Paramset idx and key +paramset_idx = recording_process_info[0]["process_paramset_idx"] paramset_idx_key = dict() -paramset_idx_key['paramset_idx'] = paramset_idx -scanner = recording_process_info[0]['acquisition_type'] +paramset_idx_key["paramset_idx"] = paramset_idx +scanner = recording_process_info[0]["acquisition_type"] -#Scan id always 0 because we will control that on recording_process +# Scan id always 0 because we will control that on recording_process scan_id = 0 -#Get preprocess params +# Get preprocess params preprocess_params_key = dict() -preprocess_params_key['preprocess_paramset_idx'] = recording_process_info[0]['preprocess_paramset_idx'] -preprocess_params = recording.PreprocessParamSet().get_preprocess_params(preprocess_params_key) -processing_method = preprocess_params['processing_method'] -task_mode = preprocess_params['task_mode'] +preprocess_params_key["preprocess_paramset_idx"] = recording_process_info[0][ + "preprocess_paramset_idx" +] +preprocess_params = recording.PreprocessParamSet().get_preprocess_params( + preprocess_params_key +) +processing_method = preprocess_params["processing_method"] +task_mode = preprocess_params["task_mode"] -print('got processing_method', processing_method) -print('got task_mode', task_mode) +print("got processing_method", processing_method) +print("got task_mode", task_mode) -print('got paramset_idx', paramset_idx) +print("got paramset_idx", paramset_idx) -#Get directories for kov +# Get directories for kov scan_filepaths = get_scan_image_files(fov_key) scan_filepaths = scan_filepaths[:1] print(scan_filepaths) if rec_process_key not in scan_element.Scan(): - try: - #TODO: Can use tiffile function to loads - print('LOADED Scan using Scanreader') + try: + # TODO: Can use tiffile function to loads + print("LOADED Scan using Scanreader") loaded_scan = scanreader.read_scan(scan_filepaths) header = parse_scanimage_header(loaded_scan) - #scanner = header['SI_imagingSystem'].strip('\'') #TODO: If using tiffile, hardcode it to `mesoscope` + # scanner = header['SI_imagingSystem'].strip('\'') #TODO: If using tiffile, hardcode it to `mesoscope` except Exception: - print('LOADED Scan using Tifffile') - scan_filepaths = scan_filepaths # TODO load all TIFF files from session possibly using TIFFSequence + print("LOADED Scan using Tifffile") + scan_filepaths = scan_filepaths # TODO load all TIFF files from session possibly using TIFFSequence loaded_scan = tifffile.imread(scan_filepaths) - #scanner = 'mesoscope' - except: #TODO: Use except instead of else) - print('ScanImage loading error') #TODO: Modify the error message + # scanner = 'mesoscope' + except: # TODO: Use except instead of else) + print("ScanImage loading error") # TODO: Modify the error message - #Equipment.insert1({'scanner': scanner}, skip_duplicates=True) + # Equipment.insert1({'scanner': scanner}, skip_duplicates=True) scan_element.Scan.insert1( - {**rec_process_key, 'scan_id': scan_id, 'scanner': scanner, 'acq_software': acq_software}) + { + **rec_process_key, + "scan_id": scan_id, + "scanner": scanner, + "acq_software": acq_software, + } + ) scan_element.ScanInfo.populate(rec_process_key, display_progress=True) # output_dir = [x.rsplit('/', maxsplit=1)[0] for x in scan_filepaths] # output_dir = [pathlib.Path(x) for x in output_dir] # scan_folder = [x / process_method for x in output_dir] -fov_directory = (imaging_rec.FieldOfView & fov_key).fetch1('fov_directory') -#output_dir = pathlib.Path('/usr/people/gs6614/temp_output') / fov_directory / processing_method #TODO fix to possibly work with existing suite2p directories +fov_directory = (imaging_rec.FieldOfView & fov_key).fetch1("fov_directory") +# output_dir = pathlib.Path('/usr/people/gs6614/temp_output') / fov_directory / processing_method #TODO fix to possibly work with existing suite2p directories recording_process_id_folder_found = True generic_process_folder_found = True @@ -204,40 +216,41 @@ # No results to load from -if task_mode == 'load' and not generic_process_folder_found: - FileNotFoundError(processing_method + ' FOLDER NOT FOUND cannot load results!!!') +if task_mode == "load" and not generic_process_folder_found: + FileNotFoundError(processing_method + " FOLDER NOT FOUND cannot load results!!!") # Results from this specific recording_process_id already triggered -if task_mode == 'trigger' and recording_process_id_folder_found: - print('Overwritting process output from original processing folder', output_dir) +if task_mode == "trigger" and recording_process_id_folder_found: + print("Overwritting process output from original processing folder", output_dir) -#If trigger and not recording_process_id_folder_found make it -if task_mode == 'trigger': +# If trigger and not recording_process_id_folder_found make it +if task_mode == "trigger": relative_output_dir = (pathlib.Path(fov_directory) / rec_process_str_key).as_posix() output_dir = pathlib.Path(get_imaging_root_data_dir(), relative_output_dir) - output_dir.mkdir(parents=True,exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) -print('RELATIVE OUTPUT DIR') +print("RELATIVE OUTPUT DIR") print(relative_output_dir) print(output_dir) -#output_dir.mkdir(parents=True,exist_ok=True) +# output_dir.mkdir(parents=True,exist_ok=True) -#Check if found output dir is correct -if task_mode == 'load': - if processing_method == 'suite2p': - print('SUITE2P METHOD SELECTED') +# Check if found output dir is correct +if task_mode == "load": + if processing_method == "suite2p": + print("SUITE2P METHOD SELECTED") # output_dir = get_suite2p_dir(scan_key) - p = pathlib.Path(output_dir).glob('**/*') + p = pathlib.Path(output_dir).glob("**/*") plane_filepaths = [x for x in p if x.is_dir()] for plane_filepath in plane_filepaths: - ops_fp = plane_filepath / 'ops.npy' - iscell_fp = plane_filepath / 'iscell.npy' + ops_fp = plane_filepath / "ops.npy" + iscell_fp = plane_filepath / "iscell.npy" if not ops_fp.exists() or not iscell_fp.exists(): raise FileNotFoundError( - f'No "ops.npy" or "iscell.npy" found. Invalid suite2p plane folder: {plane_filepath}') + f'No "ops.npy" or "iscell.npy" found. Invalid suite2p plane folder: {plane_filepath}' + ) - elif processing_method == 'caiman': + elif processing_method == "caiman": raise ValueError("caiman not supported yet") # _required_hdf5_fields = ['/motion_correction/reference_image', # '/motion_correction/correlation_image', @@ -263,31 +276,40 @@ if paramset_idx_key not in imaging_element.ProcessingParamSet(): - - #Get all information from process params from recording.ProcessParamSet schema + # Get all information from process params from recording.ProcessParamSet schema process_params_key = dict() - process_params_key['process_paramset_idx'] = paramset_idx - process_params_info = (recording.ProcessParamSet() & process_params_key).fetch(as_dict=True) + process_params_key["process_paramset_idx"] = paramset_idx + process_params_info = (recording.ProcessParamSet() & process_params_key).fetch( + as_dict=True + ) process_params = recording.ProcessParamSet().get_process_params(process_params_key) - #print('process_params_info', process_params_info) - print('process_params', process_params) - print('description', process_params_info[0]['process_paramset_desc']) + # print('process_params_info', process_params_info) + print("process_params", process_params) + print("description", process_params_info[0]["process_paramset_desc"]) - #Insert in imaging element equivalent ProcessParamSet + # Insert in imaging element equivalent ProcessParamSet imaging_element.ProcessingParamSet.insert_new_params( - processing_method=processing_method, paramset_idx=paramset_idx+1, paramset_desc=process_params_info[0]['process_paramset_desc'], params=process_params) - -imaging_element.ProcessingTask.insert1(dict(**rec_process_key, - scan_id=scan_id, - paramset_idx=paramset_idx+1, - processing_output_dir=relative_output_dir, - task_mode=task_mode), - skip_duplicates=True) + processing_method=processing_method, + paramset_idx=paramset_idx + 1, + paramset_desc=process_params_info[0]["process_paramset_desc"], + params=process_params, + ) + +imaging_element.ProcessingTask.insert1( + dict( + **rec_process_key, + scan_id=scan_id, + paramset_idx=paramset_idx + 1, + processing_output_dir=relative_output_dir, + task_mode=task_mode, + ), + skip_duplicates=True, +) imaging_element.Processing.populate(rec_process_key, display_progress=True) -processing_keys = imaging_element.Processing.fetch('KEY') +processing_keys = imaging_element.Processing.fetch("KEY") for processing_key in processing_keys: imaging_element.Curation().create1_from_processing_task(processing_key) diff --git a/u19_pipeline/automatic_job/imaging_element_populate.py b/u19_pipeline/automatic_job/imaging_element_populate.py index bb4e6a8c..be0d009f 100644 --- a/u19_pipeline/automatic_job/imaging_element_populate.py +++ b/u19_pipeline/automatic_job/imaging_element_populate.py @@ -6,67 +6,81 @@ from u19_pipeline.imaging_pipeline import imaging_element -def populate_element_data(job_id, display_progress=True, reserve_jobs=False, suppress_errors=False): - - populate_settings = {'display_progress': display_progress, - 'reserve_jobs': reserve_jobs, - 'suppress_errors': suppress_errors} - - process_key = (recording_process.Processing * recording.Recording & - dict(job_id=job_id)).fetch1('KEY') - - if (recording.Recording & process_key).fetch1('recording_modality') != 'imaging': - warnings.warn(f'Recording modality is not `imaging` for job_id: {job_id}') +def populate_element_data( + job_id, display_progress=True, reserve_jobs=False, suppress_errors=False +): + + populate_settings = { + "display_progress": display_progress, + "reserve_jobs": reserve_jobs, + "suppress_errors": suppress_errors, + } + + process_key = ( + recording_process.Processing * recording.Recording & dict(job_id=job_id) + ).fetch1("KEY") + + if (recording.Recording & process_key).fetch1("recording_modality") != "imaging": + warnings.warn(f"Recording modality is not `imaging` for job_id: {job_id}") return - fragment_number, recording_process_pre_path, recording_process_post_path = \ - (recording_process.Processing & process_key).fetch1( - 'fragment_number', - 'recording_process_pre_path', - 'recording_process_post_path') + fragment_number, recording_process_pre_path, recording_process_post_path = ( + recording_process.Processing & process_key + ).fetch1( + "fragment_number", "recording_process_pre_path", "recording_process_post_path" + ) - preprocess_param_steps_id, paramset_idx = \ - (recording_process.Processing.ImagingParams & process_key - ).fetch1('preprocess_param_steps_id', - 'paramset_idx') + preprocess_param_steps_id, paramset_idx = ( + recording_process.Processing.ImagingParams & process_key + ).fetch1("preprocess_param_steps_id", "paramset_idx") - preprocess_paramsets = (imaging_element.PreprocessParamSteps.Step() & - dict( - preprocess_param_steps_id=preprocess_param_steps_id) - ).fetch('paramset_idx') + preprocess_paramsets = ( + imaging_element.PreprocessParamSteps.Step() + & dict(preprocess_param_steps_id=preprocess_param_steps_id) + ).fetch("paramset_idx") - processing_method = (imaging_element.ProcessingParamSet & - dict(paramset_idx=paramset_idx)).fetch1( - 'processing_method') + processing_method = ( + imaging_element.ProcessingParamSet & dict(paramset_idx=paramset_idx) + ).fetch1("processing_method") - if len(preprocess_paramsets)==0: - preprocess_task_mode = 'none' + if len(preprocess_paramsets) == 0: + preprocess_task_mode = "none" else: - preprocess_task_mode = 'load' + preprocess_task_mode = "load" - preprocess_key = dict(recording_id=process_key['recording_id'], - tiff_split=fragment_number, - scan_id=0, - preprocess_param_steps_id=preprocess_param_steps_id) + preprocess_key = dict( + recording_id=process_key["recording_id"], + tiff_split=fragment_number, + scan_id=0, + preprocess_param_steps_id=preprocess_param_steps_id, + ) imaging_element.PreprocessTask.insert1( - dict(**preprocess_key, - preprocess_output_dir=recording_process_pre_path, - task_mode=preprocess_task_mode), - skip_duplicates=True) + dict( + **preprocess_key, + preprocess_output_dir=recording_process_pre_path, + task_mode=preprocess_task_mode, + ), + skip_duplicates=True, + ) if not imaging_element.Preprocess & preprocess_key: imaging_element.Preprocess.populate(preprocess_key, **populate_settings) - process_key = dict(**preprocess_key, - paramset_idx=paramset_idx) + process_key = dict(**preprocess_key, paramset_idx=paramset_idx) - pathlib.Path(f'/mnt/cup/braininit/Data/Processed/imaging/{recording_process_post_path}/{processing_method}_output').mkdir(parents=True, exist_ok=True) + pathlib.Path( + f"/mnt/cup/braininit/Data/Processed/imaging/{recording_process_post_path}/{processing_method}_output" + ).mkdir(parents=True, exist_ok=True) imaging_element.ProcessingTask.insert1( - dict(**process_key, - processing_output_dir=f'{recording_process_post_path}/{processing_method}_output', - task_mode='trigger'), skip_duplicates=True) + dict( + **process_key, + processing_output_dir=f"{recording_process_post_path}/{processing_method}_output", + task_mode="trigger", + ), + skip_duplicates=True, + ) if not imaging_element.Processing & process_key: imaging_element.Processing.populate(process_key, **populate_settings) @@ -86,8 +100,8 @@ def populate_element_data(job_id, display_progress=True, reserve_jobs=False, sup if not imaging_element.Activity & process_key: imaging_element.Activity.populate(process_key, **populate_settings) - return config.status_update_idx['NEXT_STATUS'] + return config.status_update_idx["NEXT_STATUS"] -if __name__ == '__main__': - populate_element_data() \ No newline at end of file +if __name__ == "__main__": + populate_element_data() diff --git a/u19_pipeline/automatic_job/parameter_file_creator.py b/u19_pipeline/automatic_job/parameter_file_creator.py index ba0ac75a..ddba2287 100644 --- a/u19_pipeline/automatic_job/parameter_file_creator.py +++ b/u19_pipeline/automatic_job/parameter_file_creator.py @@ -1,7 +1,5 @@ - - -#import os -#import pathlib +# import os +# import pathlib import json import pathlib @@ -14,65 +12,87 @@ # Functions to create parameter files and send them -#parameter_files_filepath = 'u19_pipeline/automatic_job/ParameterFiles' -#default_preprocess_filename = 'preprocess_paramset_%s.json' -#default_process_filename = 'process_paramset_%s.json' +# parameter_files_filepath = 'u19_pipeline/automatic_job/ParameterFiles' +# default_preprocess_filename = 'preprocess_paramset_%s.json' +# default_process_filename = 'process_paramset_%s.json' -#chanmap_files_filepath = 'u19_pipeline/automatic_job/ChanMapFiles' -#default_chanmap_filename = 'chanmap_%s.mat' +# chanmap_files_filepath = 'u19_pipeline/automatic_job/ChanMapFiles' +# default_chanmap_filename = 'chanmap_%s.mat' default_process_script_path = "scripts/automate_imaging_element.py" -def generate_parameter_file(recording_process_id, params, type_params, program_selection_params): - ''' +def generate_parameter_file( + recording_process_id, params, type_params, program_selection_params +): + """ Generate and send parameter files for processing - ''' - if program_selection_params['local_or_cluster'] == 'cluster': - cluster_vars = ft.get_cluster_vars(program_selection_params['process_cluster']) - params_file_cluster_path = cluster_vars['params_files_dir'] - user_host = cluster_vars['user']+'@'+cluster_vars['hostname'] - - #Write preprocessing parameter file - if type_params == 'preparams': - write_parameter_file(params, recording_process_id, config.default_preprocess_filename) - if program_selection_params['local_or_cluster'] == 'cluster': - status = transfer_parameter_file(recording_process_id, config.default_preprocess_filename, params_file_cluster_path, user_host) + """ + if program_selection_params["local_or_cluster"] == "cluster": + cluster_vars = ft.get_cluster_vars(program_selection_params["process_cluster"]) + params_file_cluster_path = cluster_vars["params_files_dir"] + user_host = cluster_vars["user"] + "@" + cluster_vars["hostname"] + + # Write preprocessing parameter file + if type_params == "preparams": + write_parameter_file( + params, recording_process_id, config.default_preprocess_filename + ) + if program_selection_params["local_or_cluster"] == "cluster": + status = transfer_parameter_file( + recording_process_id, + config.default_preprocess_filename, + params_file_cluster_path, + user_host, + ) else: - status = config.system_process['SUCCESS'] + status = config.system_process["SUCCESS"] else: - write_parameter_file(params, recording_process_id, config.default_process_filename) - if program_selection_params['local_or_cluster'] == 'cluster': - status = transfer_parameter_file(recording_process_id, config.default_process_filename, params_file_cluster_path, user_host) + write_parameter_file( + params, recording_process_id, config.default_process_filename + ) + if program_selection_params["local_or_cluster"] == "cluster": + status = transfer_parameter_file( + recording_process_id, + config.default_process_filename, + params_file_cluster_path, + user_host, + ) else: - status = config.system_process['SUCCESS'] + status = config.system_process["SUCCESS"] return status def write_parameter_file(params, recording_process_id, default_param_filename): - ''' + """ Write local parameter file to send - ''' + """ str_params = json.dumps(params) param_filename = default_param_filename % (recording_process_id) - params_file_local_path = str(pathlib.Path(config.parameter_files_filepath,param_filename)) + params_file_local_path = str( + pathlib.Path(config.parameter_files_filepath, param_filename) + ) write_file(params_file_local_path, str_params) -def transfer_parameter_file(recording_process_id, default_param_filename, cluster_param_dir, user_host): - ''' +def transfer_parameter_file( + recording_process_id, default_param_filename, cluster_param_dir, user_host +): + """ Transfer parameter file to processing cluster - ''' + """ param_filename = default_param_filename % (recording_process_id) - params_file_local_path = str(pathlib.Path(config.parameter_files_filepath,param_filename)) - params_file_cluster_path = str(pathlib.Path(cluster_param_dir,param_filename)) - param_file_full_path = user_host+':'+params_file_cluster_path + params_file_local_path = str( + pathlib.Path(config.parameter_files_filepath, param_filename) + ) + params_file_cluster_path = str(pathlib.Path(cluster_param_dir, param_filename)) + param_file_full_path = user_host + ":" + params_file_cluster_path - print('transfer_parameter_file', params_file_local_path, param_file_full_path) + print("transfer_parameter_file", params_file_local_path, param_file_full_path) status = ft.scp_file_transfer(params_file_local_path, param_file_full_path) @@ -80,41 +100,52 @@ def transfer_parameter_file(recording_process_id, default_param_filename, cluste def generate_chanmap_file(recording_process_id, program_selection_params): - ''' + """ Generate and send chanmap files for processing - ''' - - cluster_vars = ft.get_cluster_vars(program_selection_params['process_cluster']) - params_file_cluster_path = cluster_vars['chanmap_files_dir'] - user_host = cluster_vars['user']+'@'+cluster_vars['hostname'] + """ + + cluster_vars = ft.get_cluster_vars(program_selection_params["process_cluster"]) + params_file_cluster_path = cluster_vars["chanmap_files_dir"] + user_host = cluster_vars["user"] + "@" + cluster_vars["hostname"] - #Write chanmap file + # Write chanmap file # write_chanmap_file(chanmap_df, recording_process_id, config.default_chanmap_filename) - status = transfer_chanmap_file(recording_process_id, config.default_chanmap_filename, params_file_cluster_path, user_host) + status = transfer_chanmap_file( + recording_process_id, + config.default_chanmap_filename, + params_file_cluster_path, + user_host, + ) return status + def write_chanmap_file(chanmap_dict, recording_process_id, default_chanmap_filename): - ''' + """ Write local chanmap file to send - ''' + """ chanmap_filename = default_chanmap_filename % (recording_process_id) - chanmap_file_local_path = str(pathlib.Path(config.chanmap_files_filepath,chanmap_filename)) + chanmap_file_local_path = str( + pathlib.Path(config.chanmap_files_filepath, chanmap_filename) + ) savemat(chanmap_file_local_path, chanmap_dict) -def transfer_chanmap_file(recording_process_id, default_chanmap_filename, cluster_chanmap_dir, user_host): - ''' +def transfer_chanmap_file( + recording_process_id, default_chanmap_filename, cluster_chanmap_dir, user_host +): + """ Transfer chanmap file to processing cluster - ''' + """ chanmap_filename = default_chanmap_filename % (recording_process_id) - chanmap_file_local_path = str(pathlib.Path(config.chanmap_files_filepath,chanmap_filename)) - chanmap_file_cluster_path = str(pathlib.Path(cluster_chanmap_dir,chanmap_filename)) - chanmap_file_full_path = user_host+':'+chanmap_file_cluster_path + chanmap_file_local_path = str( + pathlib.Path(config.chanmap_files_filepath, chanmap_filename) + ) + chanmap_file_cluster_path = str(pathlib.Path(cluster_chanmap_dir, chanmap_filename)) + chanmap_file_full_path = user_host + ":" + chanmap_file_cluster_path status = ft.scp_file_transfer(chanmap_file_local_path, chanmap_file_full_path) return status - diff --git a/u19_pipeline/automatic_job/params_config.py b/u19_pipeline/automatic_job/params_config.py index b60f8d5c..82c76f47 100644 --- a/u19_pipeline/automatic_job/params_config.py +++ b/u19_pipeline/automatic_job/params_config.py @@ -1,4 +1,3 @@ - import os import pathlib @@ -7,283 +6,271 @@ import u19_pipeline.lab as lab from scripts.conf_file_finding import get_root_directory -#Dictionary with main configuration for each modality (ephys or imaging) +# Dictionary with main configuration for each modality (ephys or imaging) recording_modality_dict = [ { - 'recording_modality': 'electrophysiology', - 'local_or_cluster': 'cluster', # Where processing will happen, locally or cluster - 'process_repository': 'BrainCogsEphysSorters', # Which repositroy will be used to process - 'process_cluster': 'spock', # Which cluster will be used to process (check clusters_paths_and_transfers.py file) - 'process_script': 'main_script.py' # Script in process_repository to + "recording_modality": "electrophysiology", + "local_or_cluster": "cluster", # Where processing will happen, locally or cluster + "process_repository": "BrainCogsEphysSorters", # Which repositroy will be used to process + "process_cluster": "spock", # Which cluster will be used to process (check clusters_paths_and_transfers.py file) + "process_script": "main_script.py", # Script in process_repository to }, { - 'recording_modality': 'imaging', - 'local_or_cluster': 'local', - 'process_repository': 'element-calcium-imaging', - 'process_cluster': 'spock', # Which cluster will be used to process (check clusters_paths_and_transfers.py file) - 'process_script': 'none' + "recording_modality": "imaging", + "local_or_cluster": "local", + "process_repository": "element-calcium-imaging", + "process_cluster": "spock", # Which cluster will be used to process (check clusters_paths_and_transfers.py file) + "process_script": "none", }, ] recording_modality_list = [list(i.values()) for i in recording_modality_dict] recording_modality_df = pd.DataFrame(recording_modality_dict) -#Dictionary with status configuration for recordings +# Dictionary with status configuration for recordings recording_status_dict = [ { - 'Value': -1, # Status id - 'Key': 'ERROR', # Status "nickname" - 'Label': 'Error in recording handling', # Status description - 'UpdateField': None, # Which field in the u19_recording.recording table will be updated - 'ProcessFunction': None, # Which function to execute in workflow for this status - 'FunctionField': None, # Which field of u19_recording.recording will be used in status function - 'SlackMessage': None # Slack notification message for this status + "Value": -1, # Status id + "Key": "ERROR", # Status "nickname" + "Label": "Error in recording handling", # Status description + "UpdateField": None, # Which field in the u19_recording.recording table will be updated + "ProcessFunction": None, # Which function to execute in workflow for this status + "FunctionField": None, # Which field of u19_recording.recording will be used in status function + "SlackMessage": None, # Slack notification message for this status }, { - 'Value': 0, - 'Key': 'NEW_RECORDING', - 'Label': 'New recording', - 'UpdateField': None, - 'ProcessFunction': None, - 'FunctionField': None, - 'SlackMessage': None + "Value": 0, + "Key": "NEW_RECORDING", + "Label": "New recording", + "UpdateField": None, + "ProcessFunction": None, + "FunctionField": None, + "SlackMessage": None, }, { - 'Value': 1, - 'Key': 'PNI_DRIVE_TRANSFER_REQUEST', - 'Label': 'Recording directory transfer to PNI requested', - 'UpdateField': 'task_copy_id_pni', - 'ProcessFunction': 'local_transfer_request', - 'FunctionField': 'recording_process_pre_path', - 'SlackMessage': None + "Value": 1, + "Key": "PNI_DRIVE_TRANSFER_REQUEST", + "Label": "Recording directory transfer to PNI requested", + "UpdateField": "task_copy_id_pni", + "ProcessFunction": "local_transfer_request", + "FunctionField": "recording_process_pre_path", + "SlackMessage": None, }, { - 'Value': 2, - 'Key': 'PNI_DRIVE_TRANSFER_END', - 'Label': 'Recording directory transferred to PNI', - 'UpdateField': None, - 'ProcessFunction': 'local_transfer_check', - 'FunctionField': 'task_copy_id_pni', - 'SlackMessage': 'Recording was transferred to braininit (cup) drive' + "Value": 2, + "Key": "PNI_DRIVE_TRANSFER_END", + "Label": "Recording directory transferred to PNI", + "UpdateField": None, + "ProcessFunction": "local_transfer_check", + "FunctionField": "task_copy_id_pni", + "SlackMessage": "Recording was transferred to braininit (cup) drive", }, { - 'Value': 3, - 'Key': 'MODALITY_PREINGESTION', - 'Label': 'modality ingestion & Syncing jobs done', - 'UpdateField': None, - 'ProcessFunction': 'modality_preingestion', - 'FunctionField': None, - 'SlackMessage': None + "Value": 3, + "Key": "MODALITY_PREINGESTION", + "Label": "modality ingestion & Syncing jobs done", + "UpdateField": None, + "ProcessFunction": "modality_preingestion", + "FunctionField": None, + "SlackMessage": None, }, - ] -recording_status_list = [[i['Value'], i['Label']] for i in recording_status_dict] +recording_status_list = [[i["Value"], i["Label"]] for i in recording_status_dict] recording_status_df = pd.DataFrame(recording_status_dict) -RECORDING_STATUS_ERROR_ID = recording_status_df.loc[recording_status_df['Key'] == 'ERROR', 'Value'].values[0] +RECORDING_STATUS_ERROR_ID = recording_status_df.loc[ + recording_status_df["Key"] == "ERROR", "Value" +].values[0] -#Dictionary with status configuration for recording process (units of each recording) +# Dictionary with status configuration for recording process (units of each recording) recording_process_status_dict = [ { - 'Value': -2, - 'Key': 'ERROR_DELETED', - 'Label': 'Error in recording process / Deleted cluster files', - 'UpdateField': None, - 'ProcessFunction': None, - 'FunctionField': None, - 'SlackMessage': None + "Value": -2, + "Key": "ERROR_DELETED", + "Label": "Error in recording process / Deleted cluster files", + "UpdateField": None, + "ProcessFunction": None, + "FunctionField": None, + "SlackMessage": None, }, { - 'Value': -1, - 'Key': 'ERROR', - 'Label': 'Error in recording process', - 'UpdateField': None, - 'ProcessFunction': None, - 'FunctionField': None, - 'SlackMessage': None + "Value": -1, + "Key": "ERROR", + "Label": "Error in recording process", + "UpdateField": None, + "ProcessFunction": None, + "FunctionField": None, + "SlackMessage": None, }, { - 'Value': 0, - 'Key': 'NEW_RECORDING_PROCESS', - 'Label': 'New recording process', - 'UpdateField': None, - 'ProcessFunction': None, - 'FunctionField': None, - 'SlackMessage': None + "Value": 0, + "Key": "NEW_RECORDING_PROCESS", + "Label": "New recording process", + "UpdateField": None, + "ProcessFunction": None, + "FunctionField": None, + "SlackMessage": None, }, { - 'Value': 1, - 'Key': 'RAW_FILE_TRANSFER_REQUEST', - 'Label': 'Raw file transfer requested', - 'UpdateField': 'task_copy_id_pre', - 'ProcessFunction': 'transfer_request', - 'FunctionField': 'recording_process_pre_path', - 'SlackMessage': None + "Value": 1, + "Key": "RAW_FILE_TRANSFER_REQUEST", + "Label": "Raw file transfer requested", + "UpdateField": "task_copy_id_pre", + "ProcessFunction": "transfer_request", + "FunctionField": "recording_process_pre_path", + "SlackMessage": None, }, { - 'Value': 2, - 'Key': 'RAW_FILE_TRANSFER_END', - 'Label': 'Raw file transferred to cluster', - 'UpdateField': None, - 'ProcessFunction': 'transfer_check', - 'FunctionField': 'task_copy_id_pre', - 'SlackMessage': None + "Value": 2, + "Key": "RAW_FILE_TRANSFER_END", + "Label": "Raw file transferred to cluster", + "UpdateField": None, + "ProcessFunction": "transfer_check", + "FunctionField": "task_copy_id_pre", + "SlackMessage": None, }, { - 'Value': 3, - 'Key': 'JOB_QUEUE', - 'Label': 'Processing job in queue', - 'UpdateField': 'slurm_id', - 'ProcessFunction': 'slurm_job_queue', - 'FunctionField': None, - 'SlackMessage': 'Job was queued to be processed in cluster' + "Value": 3, + "Key": "JOB_QUEUE", + "Label": "Processing job in queue", + "UpdateField": "slurm_id", + "ProcessFunction": "slurm_job_queue", + "FunctionField": None, + "SlackMessage": "Job was queued to be processed in cluster", }, { - 'Value': 4, - 'Key': 'JOB_FINISHED', - 'Label': 'Processing job finished', - 'UpdateField': None, - 'ProcessFunction': 'slurm_job_check', - 'FunctionField': 'slurm_id', - 'SlackMessage': None + "Value": 4, + "Key": "JOB_FINISHED", + "Label": "Processing job finished", + "UpdateField": None, + "ProcessFunction": "slurm_job_check", + "FunctionField": "slurm_id", + "SlackMessage": None, }, { - 'Value': 5, - 'Key': 'PROC_FILE_TRANSFER_REQUEST', - 'Label': 'Processed file transfer requested', - 'UpdateField': 'task_copy_id_post', - 'ProcessFunction': 'transfer_request', - 'FunctionField': 'recording_process_post_path', - 'SlackMessage': None + "Value": 5, + "Key": "PROC_FILE_TRANSFER_REQUEST", + "Label": "Processed file transfer requested", + "UpdateField": "task_copy_id_post", + "ProcessFunction": "transfer_request", + "FunctionField": "recording_process_post_path", + "SlackMessage": None, }, { - 'Value': 6, - 'Key': 'PROC_FILE_TRANSFER_END', - 'Label': 'Processed file transferred to PNI', - 'UpdateField': None, - 'ProcessFunction': 'transfer_check', - 'FunctionField': 'task_copy_id_post', - 'SlackMessage': 'Processed data was transferred to cup. Available on the GUI' + "Value": 6, + "Key": "PROC_FILE_TRANSFER_END", + "Label": "Processed file transferred to PNI", + "UpdateField": None, + "ProcessFunction": "transfer_check", + "FunctionField": "task_copy_id_post", + "SlackMessage": "Processed data was transferred to cup. Available on the GUI", }, { - 'Value': 7, - 'Key': 'JOB_FINISHED_ELEMENT_WORKFLOW', - 'Label': 'Data in element, Finished !!', - 'UpdateField': None, - 'ProcessFunction': 'populate_element', - 'FunctionField': None, - 'SlackMessage': 'Job was successfully processed. Data in element DB' + "Value": 7, + "Key": "JOB_FINISHED_ELEMENT_WORKFLOW", + "Label": "Data in element, Finished !!", + "UpdateField": None, + "ProcessFunction": "populate_element", + "FunctionField": None, + "SlackMessage": "Job was successfully processed. Data in element DB", }, { - 'Value': 8, - 'Key': 'JOB_DATA_DELETED_CLUSTER', - 'Label': 'Data in element, Finished !!', - 'UpdateField': None, - 'ProcessFunction': 'populate_element', - 'FunctionField': None, - 'SlackMessage': None + "Value": 8, + "Key": "JOB_DATA_DELETED_CLUSTER", + "Label": "Data in element, Finished !!", + "UpdateField": None, + "ProcessFunction": "populate_element", + "FunctionField": None, + "SlackMessage": None, }, ] -recording_process_status_list = [[i['Value'], i['Label']] for i in recording_process_status_dict] +recording_process_status_list = [ + [i["Value"], i["Label"]] for i in recording_process_status_dict +] recording_process_status_df = pd.DataFrame(recording_process_status_dict) # All error and success status values -JOB_STATUS_ERROR_ID = recording_process_status_df.loc[recording_process_status_df['Key'] == 'ERROR', 'Value'].values[0] -JOB_STATUS_PROCESSED = recording_process_status_df.loc[recording_process_status_df['Key'] == 'JOB_FINISHED_ELEMENT_WORKFLOW', 'Value'].values[0] -JOB_STATUS_POST_PROCESSED = recording_process_status_df.loc[recording_process_status_df['Key'] == 'JOB_DATA_DELETED_CLUSTER', 'Value'].values[0] -JOB_STATUS_ERROR_DELETED = recording_process_status_df.loc[recording_process_status_df['Key'] == 'ERROR_DELETED', 'Value'].values[0] +JOB_STATUS_ERROR_ID = recording_process_status_df.loc[ + recording_process_status_df["Key"] == "ERROR", "Value" +].values[0] +JOB_STATUS_PROCESSED = recording_process_status_df.loc[ + recording_process_status_df["Key"] == "JOB_FINISHED_ELEMENT_WORKFLOW", "Value" +].values[0] +JOB_STATUS_POST_PROCESSED = recording_process_status_df.loc[ + recording_process_status_df["Key"] == "JOB_DATA_DELETED_CLUSTER", "Value" +].values[0] +JOB_STATUS_ERROR_DELETED = recording_process_status_df.loc[ + recording_process_status_df["Key"] == "ERROR_DELETED", "Value" +].values[0] # Return code for functions for errors, next state or no status change -status_update_idx = { - 'NEXT_STATUS': 1, - 'NO_CHANGE': 0, - 'ERROR_STATUS':-1 -} +status_update_idx = {"NEXT_STATUS": 1, "NO_CHANGE": 0, "ERROR_STATUS": -1} # Degault Dictionary "skeleton" that is returned in all functions of workflow -default_update_value_dict ={ - 'value_update': None, - 'error_info': { - 'error_message': None, - 'error_exception': None, +default_update_value_dict = { + "value_update": None, + "error_info": { + "error_message": None, + "error_exception": None, }, } -# Return Codes from -system_process = { - 'COMPLETED': 1, - 'SUCCESS': 0, - 'ERROR': -1 -} +# Return Codes from +system_process = {"COMPLETED": 1, "SUCCESS": 0, "ERROR": -1} # Possible states from slurm queued jobs results and result in pipeline status (next state, no change or error) slurm_states = { - 'COMPLETED': { - 'pipeline_status': status_update_idx['NEXT_STATUS'], - 'message': '' + "COMPLETED": {"pipeline_status": status_update_idx["NEXT_STATUS"], "message": ""}, + "PENDING": {"pipeline_status": status_update_idx["NO_CHANGE"], "message": ""}, + "RUNNING": {"pipeline_status": status_update_idx["NO_CHANGE"], "message": ""}, + "FAILED": {"pipeline_status": status_update_idx["ERROR_STATUS"], "message": ""}, + "TIMEOUT": { + "pipeline_status": status_update_idx["ERROR_STATUS"], + "message": "Timeout for job has expired", }, - 'PENDING': { - 'pipeline_status': status_update_idx['NO_CHANGE'], - 'message': '' + "CANCELLED+": { + "pipeline_status": status_update_idx["ERROR_STATUS"], + "message": "Job was cancelled", }, - 'RUNNING': { - 'pipeline_status': status_update_idx['NO_CHANGE'], - 'message': '' - }, - 'FAILED': { - 'pipeline_status': status_update_idx['ERROR_STATUS'], - 'message': '' - }, - 'TIMEOUT': - { - 'pipeline_status': status_update_idx['ERROR_STATUS'], - 'message': 'Timeout for job has expired' + "CANCELLED": { + "pipeline_status": status_update_idx["ERROR_STATUS"], + "message": "Job was cancelled", }, - 'CANCELLED+': - { - 'pipeline_status': status_update_idx['ERROR_STATUS'], - 'message': 'Job was cancelled' - }, - 'CANCELLED': - { - 'pipeline_status': status_update_idx['ERROR_STATUS'], - 'message': 'Job was cancelled' - } } # Look for u19_matlab_dir (Should be present on same directory as U19-Pipeline_Python) _, u19_pipeline_python_dir = get_root_directory() datajoint_proj_dir = u19_pipeline_python_dir.parent -u19_matlab_dir = pathlib.Path(datajoint_proj_dir, 'U19-pipeline-matlab') -startup_pipeline_matlab_dir = pathlib.Path(u19_matlab_dir, 'scripts').as_posix() +u19_matlab_dir = pathlib.Path(datajoint_proj_dir, "U19-pipeline-matlab") +startup_pipeline_matlab_dir = pathlib.Path(u19_matlab_dir, "scripts").as_posix() this_dir = os.path.dirname(__file__) -ingest_scaninfo_script = pathlib.Path(this_dir, 'ingest_scaninfo_shell.sh').as_posix() +ingest_scaninfo_script = pathlib.Path(this_dir, "ingest_scaninfo_shell.sh").as_posix() # Look for CATGT directory (Should be present on same directory as U19-Pipeline_Python) -catgt_dir = pathlib.Path(datajoint_proj_dir, 'CatGT-linux') -catgt_script = pathlib.Path(catgt_dir, 'runit.sh').as_posix() +catgt_dir = pathlib.Path(datajoint_proj_dir, "CatGT-linux") +catgt_script = pathlib.Path(catgt_dir, "runit.sh").as_posix() # For parameter & channmap storing -parameter_files_filepath = pathlib.Path(this_dir, 'ParameterFiles').as_posix() -default_preprocess_filename = 'preprocess_paramset_%s.json' -default_process_filename = 'process_paramset_%s.json' +parameter_files_filepath = pathlib.Path(this_dir, "ParameterFiles").as_posix() +default_preprocess_filename = "preprocess_paramset_%s.json" +default_process_filename = "process_paramset_%s.json" # default xyzPicksFiles -xyz_picks_files_dir = pathlib.Path(this_dir, 'xyzPicksFiles') -xyz_picks_0 = pathlib.Path(xyz_picks_files_dir, 'xyz_picks.json').as_posix() -xyz_picks_n = pathlib.Path(xyz_picks_files_dir, 'xyz_picks_shank%d.json').as_posix() +xyz_picks_files_dir = pathlib.Path(this_dir, "xyzPicksFiles") +xyz_picks_0 = pathlib.Path(xyz_picks_files_dir, "xyz_picks.json").as_posix() +xyz_picks_n = pathlib.Path(xyz_picks_files_dir, "xyz_picks_shank%d.json").as_posix() -#chanmap files -chanmap_files_filepath = pathlib.Path(this_dir, 'ChanMapFiles').as_posix() -default_chanmap_filename = 'chanmap_%s.mat' +# chanmap files +chanmap_files_filepath = pathlib.Path(this_dir, "ChanMapFiles").as_posix() +default_chanmap_filename = "chanmap_%s.mat" -#Slack notification channels +# Slack notification channels slack_webhooks = lab.SlackWebhooks.fetch() slack_webhooks_dict = dict() for i in range(slack_webhooks.shape[0]): - slack_webhooks_dict[slack_webhooks[i][0]] = slack_webhooks[i][1] \ No newline at end of file + slack_webhooks_dict[slack_webhooks[i][0]] = slack_webhooks[i][1] diff --git a/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py b/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py index 49c519e1..38967474 100644 --- a/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py +++ b/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file @@ -17,25 +16,38 @@ def get_rec_key_dict(recording_id): - return {'recording_id': recording_id} + return {"recording_id": recording_id} past_date = datetime.date.today() - datetime.timedelta(days=30) -query_session_date = 'session_date >="' + past_date.strftime('%Y-%m-%d') +'"' - -all_recs = ((recording.Recording & "recording_modality='electrophysiology'") * (recording.Recording.BehaviorSession & query_session_date)).join(ep.BehaviorSync, left=True) -all_recs = pd.DataFrame(all_recs.fetch('recording_id','subject_fullname','session_date','session_number', as_dict=True)) - -not_sync_recs = pd.DataFrame((recording.Recording * ep.BehaviorSync).fetch('KEY', as_dict=True)) - -not_sync_recs2 = pd.merge(all_recs,not_sync_recs, how='left', indicator=True) -not_sync_recs2 = not_sync_recs2.loc[not_sync_recs2['_merge']=='left_only',:] -not_sync_recs2 = not_sync_recs2.sort_values(by='session_date', ascending=False) -not_sync_recs2['rec_key'] = not_sync_recs2['recording_id'].apply(get_rec_key_dict) +query_session_date = 'session_date >="' + past_date.strftime("%Y-%m-%d") + '"' + +all_recs = ( + (recording.Recording & "recording_modality='electrophysiology'") + * (recording.Recording.BehaviorSession & query_session_date) +).join(ep.BehaviorSync, left=True) +all_recs = pd.DataFrame( + all_recs.fetch( + "recording_id", + "subject_fullname", + "session_date", + "session_number", + as_dict=True, + ) +) + +not_sync_recs = pd.DataFrame( + (recording.Recording * ep.BehaviorSync).fetch("KEY", as_dict=True) +) + +not_sync_recs2 = pd.merge(all_recs, not_sync_recs, how="left", indicator=True) +not_sync_recs2 = not_sync_recs2.loc[not_sync_recs2["_merge"] == "left_only", :] +not_sync_recs2 = not_sync_recs2.sort_values(by="session_date", ascending=False) +not_sync_recs2["rec_key"] = not_sync_recs2["recording_id"].apply(get_rec_key_dict) not_sync_recs2 = not_sync_recs2.reset_index(drop=True) for i in range(not_sync_recs2.shape[0]): try: - ep.BehaviorSync.populate(not_sync_recs2.loc[i,'rec_key']) + ep.BehaviorSync.populate(not_sync_recs2.loc[i, "rec_key"]) except Exception as e: - print(e) \ No newline at end of file + print(e) diff --git a/u19_pipeline/automatic_job/pupillometry_check_handler_script.py b/u19_pipeline/automatic_job/pupillometry_check_handler_script.py index 039e774b..0597bc1d 100644 --- a/u19_pipeline/automatic_job/pupillometry_check_handler_script.py +++ b/u19_pipeline/automatic_job/pupillometry_check_handler_script.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file diff --git a/u19_pipeline/automatic_job/pupillometry_handler.py b/u19_pipeline/automatic_job/pupillometry_handler.py index cb75a02b..d5620f1f 100644 --- a/u19_pipeline/automatic_job/pupillometry_handler.py +++ b/u19_pipeline/automatic_job/pupillometry_handler.py @@ -24,51 +24,59 @@ def pupillometry_exception_handler(func): def inner_function(*args, **kwargs): try: - argout = func(*args, **kwargs) - return argout + argout = func(*args, **kwargs) + return argout except Exception as e: - print('Exception HERE ................') + print("Exception HERE ................") update_value_dict = copy.deepcopy(config.default_update_value_dict) - update_value_dict['error_info']['error_message'] = str(e) - update_value_dict['error_info']['error_exception'] = (''.join(traceback.format_exception(type(e), value=e, tb=e.__traceback__))) - - print(update_value_dict['error_info']['error_message']) - print(update_value_dict['error_info']['error_exception']) + update_value_dict["error_info"]["error_message"] = str(e) + update_value_dict["error_info"]["error_exception"] = "".join( + traceback.format_exception(type(e), value=e, tb=e.__traceback__) + ) + print(update_value_dict["error_info"]["error_message"]) + print(update_value_dict["error_info"]["error_exception"]) return (config.RECORDING_STATUS_ERROR_ID, update_value_dict) + return inner_function -class PupillometryProcessingHandler: - spock_home_dir = '/mnt/cup/braininit/Shared/repos/U19-pipeline_python/' +class PupillometryProcessingHandler: + spock_home_dir = "/mnt/cup/braininit/Shared/repos/U19-pipeline_python/" spock_log_dir = spock_home_dir + "u19_pipeline/automatic_job/OutputLog/" - process_script_path = spock_home_dir + "u19_pipeline/automatic_job/pupillometry_handler.py" + process_script_path = ( + spock_home_dir + "u19_pipeline/automatic_job/pupillometry_handler.py" + ) spock_error_dir = spock_home_dir + "u19_pipeline/automatic_job/ErrorLog/" spock_slurm_filepath = spock_home_dir + "u19_pipeline/" - spock_system_name = 'spockmk2-loginvm.pni.princeton.edu' + spock_system_name = "spockmk2-loginvm.pni.princeton.edu" - pupillometry_slurm_filepath = os.path.abspath(os.path.realpath(__file__)+ "/../") + pupillometry_slurm_filepath = os.path.abspath(os.path.realpath(__file__) + "/../") - #pupillometry_slurm_filepath = 'u19_pipeline/' - pupillometry_slurm_filename = 'slurm_pupillometry.slurm' + # pupillometry_slurm_filepath = 'u19_pipeline/' + pupillometry_slurm_filename = "slurm_pupillometry.slurm" slurm_dict_pupillometry_spock = { - 'job-name': 'dj_ingestion', - 'nodes': 1, - 'cpus-per-task': 4, - 'time': '20:00:00', - 'mem': '50G', - 'mail-type': ['END', 'FAIL'], + "job-name": "dj_ingestion", + "nodes": 1, + "cpus-per-task": 4, + "time": "20:00:00", + "mem": "50G", + "mail-type": ["END", "FAIL"], } @staticmethod def generate_slurm_pupillometry(slurm_dict): - slurm_text = '#!/bin/bash\n' - slurm_text += PupillometryProcessingHandler.create_pupillometry_slurm_params_file(slurm_dict) - slurm_text += ''' + slurm_text = "#!/bin/bash\n" + slurm_text += ( + PupillometryProcessingHandler.create_pupillometry_slurm_params_file( + slurm_dict + ) + ) + slurm_text += """ echo "SLURM_JOB_ID: ${SLURM_JOB_ID}" echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}" echo "VIDEO_DIR: ${video_dir}" @@ -84,60 +92,83 @@ def generate_slurm_pupillometry(slurm_dict): cd ${repository_dir} python ${process_script_path} ${video_dir} ${model_dir} ${output_dir} #python ${process_script_path} ${recording_process_id} - ''' + """ return slurm_text @staticmethod def create_pupillometry_slurm_params_file(slurm_dict): - text_dict = '' + text_dict = "" for slurm_param in slurm_dict.keys(): - if isinstance(slurm_dict[slurm_param], list): for list_param in slurm_dict[slurm_param]: - text_dict += '#SBATCH --' + str(slurm_param) + '=' + str(list_param) + '\n' + text_dict += ( + "#SBATCH --" + str(slurm_param) + "=" + str(list_param) + "\n" + ) else: - text_dict += '#SBATCH --' + str(slurm_param) + '=' + str(slurm_dict[slurm_param]) + '\n' + text_dict += ( + "#SBATCH --" + + str(slurm_param) + + "=" + + str(slurm_dict[slurm_param]) + + "\n" + ) return text_dict @staticmethod def generate_slurm_file(video_dir): - ''' + """ Generate and send slurm file to be queued in processing cluster - ''' + """ - #Get all associated directories given the selected processing cluster + # Get all associated directories given the selected processing cluster # Start with default values - slurm_dict = copy.deepcopy(PupillometryProcessingHandler.slurm_dict_pupillometry_spock) - label_rec_process = 'job_id_'+str(video_dir.stem) - slurm_dict['job-name'] = label_rec_process - print('PupillometryProcessingHandler.spock_log_dir') + slurm_dict = copy.deepcopy( + PupillometryProcessingHandler.slurm_dict_pupillometry_spock + ) + label_rec_process = "job_id_" + str(video_dir.stem) + slurm_dict["job-name"] = label_rec_process + print("PupillometryProcessingHandler.spock_log_dir") print(PupillometryProcessingHandler.spock_log_dir) - print('label_rec_process', label_rec_process) - print('video_dir', video_dir) - slurm_dict['output'] = PupillometryProcessingHandler.spock_log_dir+label_rec_process+'.log' - slurm_dict['error'] = PupillometryProcessingHandler.spock_error_dir+label_rec_process+'.log' - - print('slurm_dict', slurm_dict) - - slurm_text = PupillometryProcessingHandler.generate_slurm_pupillometry(slurm_dict) - - slurm_file_local_path = str(pathlib.Path(PupillometryProcessingHandler.pupillometry_slurm_filepath, - PupillometryProcessingHandler.pupillometry_slurm_filename)) + print("label_rec_process", label_rec_process) + print("video_dir", video_dir) + slurm_dict["output"] = ( + PupillometryProcessingHandler.spock_log_dir + label_rec_process + ".log" + ) + slurm_dict["error"] = ( + PupillometryProcessingHandler.spock_error_dir + label_rec_process + ".log" + ) + + print("slurm_dict", slurm_dict) + + slurm_text = PupillometryProcessingHandler.generate_slurm_pupillometry( + slurm_dict + ) + + slurm_file_local_path = str( + pathlib.Path( + PupillometryProcessingHandler.pupillometry_slurm_filepath, + PupillometryProcessingHandler.pupillometry_slurm_filename, + ) + ) print(slurm_file_local_path) - print('slurm text', slurm_text) + print("slurm text", slurm_text) write_file(slurm_file_local_path, slurm_text) - slurm_destination = pathlib.Path(PupillometryProcessingHandler.spock_slurm_filepath, - PupillometryProcessingHandler.pupillometry_slurm_filename).as_posix() + slurm_destination = pathlib.Path( + PupillometryProcessingHandler.spock_slurm_filepath, + PupillometryProcessingHandler.pupillometry_slurm_filename, + ).as_posix() - status = PupillometryProcessingHandler.transfer_pupillometry_slurm_file(slurm_file_local_path, slurm_destination) + status = PupillometryProcessingHandler.transfer_pupillometry_slurm_file( + slurm_file_local_path, slurm_destination + ) print(status) print(slurm_destination) @@ -145,23 +176,39 @@ def generate_slurm_file(video_dir): return status, slurm_destination @staticmethod - def queue_pupillometry_slurm_file(video_dir, model_dir, repository_dir, output_dir, process_script_path, slurm_location): + def queue_pupillometry_slurm_file( + video_dir, + model_dir, + repository_dir, + output_dir, + process_script_path, + slurm_location, + ): id_slurm_job = -1 - #Get all associated variables given the selected processing cluster - command = ['ssh', 'u19prod@'+PupillometryProcessingHandler.spock_system_name, 'sbatch', - "--export=video_dir='"+str(video_dir)+ - "',model_dir='"+str(model_dir)+ - "',repository_dir='"+str(repository_dir)+ - "',process_script_path='"+str(process_script_path)+ - "',output_dir='"+str(output_dir)+"'", - slurm_location + # Get all associated variables given the selected processing cluster + command = [ + "ssh", + "u19prod@" + PupillometryProcessingHandler.spock_system_name, + "sbatch", + "--export=video_dir='" + + str(video_dir) + + "',model_dir='" + + str(model_dir) + + "',repository_dir='" + + str(repository_dir) + + "',process_script_path='" + + str(process_script_path) + + "',output_dir='" + + str(output_dir) + + "'", + slurm_location, ] print(command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - #p = os.popen(command_new).read() + # p = os.popen(command_new).read() p.wait() stdout, stderr = p.communicate() @@ -170,22 +217,21 @@ def queue_pupillometry_slurm_file(video_dir, model_dir, repository_dir, output_d print(stdout) if p.returncode == 0: - error_message = '' - batch_job_sentence = stdout.decode('UTF-8') - print('batch_job_sentence', batch_job_sentence) - id_slurm_job = batch_job_sentence.replace("Submitted batch job ","") - id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job) + error_message = "" + batch_job_sentence = stdout.decode("UTF-8") + print("batch_job_sentence", batch_job_sentence) + id_slurm_job = batch_job_sentence.replace("Submitted batch job ", "") + id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job) else: - error_message = stderr.decode('UTF-8') + error_message = stderr.decode("UTF-8") return p.returncode, id_slurm_job, error_message - @staticmethod def transfer_pupillometry_slurm_file(slurm_file_local_path, slurm_destination): - ''' + """ Create scp command from cluster directories and local slurm file - ''' + """ print("cp", slurm_file_local_path, slurm_destination) @@ -208,20 +254,20 @@ def getPupilDiameter(analyzedVideoDataPath): labels = pd.read_hdf(analyzedVideoDataPath) # Create a data frame of the same size ad the analyzed video data filled with zeros - df = pd.DataFrame(np.zeros(1), columns=['PupilDiameter']) + df = pd.DataFrame(np.zeros(1), columns=["PupilDiameter"]) # For each frame, get the x and y coordinates of the points around the pupil, fit an ellipse and calculate the diameter of a circle with the same area as the ellipse for i in range(labels.index.size): subset = labels.loc[i] - x = subset.xs('x', level='coords').to_numpy()[0:8] - y = subset.xs('y', level='coords').to_numpy()[0:8] - xy = np.column_stack((x,y)) + x = subset.xs("x", level="coords").to_numpy()[0:8] + y = subset.xs("y", level="coords").to_numpy()[0:8] + xy = np.column_stack((x, y)) # Fit the points to an ellipse and get the parameters (estimate X center coordinate, estimate Y center coordinate, a, b, theta) ellipse = EllipseModel() ellipse.estimate(xy) # Calculate the area of the ellipse from the parameters a and b ellipseArea = np.pi * ellipse.params[2] * ellipse.params[3] # Get the diameter of a circle from the area of the ellipse - pupilDiameter = 2 * np.sqrt(ellipseArea/np.pi) + pupilDiameter = 2 * np.sqrt(ellipseArea / np.pi) df.loc[i] = pupilDiameter # Get outliers (frames where either the mice have the eyes closed (blink or groom) or deeplabcut fails to track the pupil correctly) @@ -231,11 +277,13 @@ def getPupilDiameter(analyzedVideoDataPath): # Set a treshold for a valid zscore value (determined empirically) outlierFlags = np.abs(zscore) > 2 # Get a boolean array where true correspond to the frame with an outlier diameter - outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: "OutlierFlag"}) + outlierFlags = outlierFlags.rename( + columns={outlierFlags.columns[0]: "OutlierFlag"} + ) # Concatenate outlier flags array to remove outliers from pupil diameter array temp = pd.concat([df, outlierFlags], axis=1) - temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None - pupilDiameter = temp['PupilDiameter'] + temp.loc[temp["OutlierFlag"] == True, "PupilDiameter"] = None + pupilDiameter = temp["PupilDiameter"] return pupilDiameter.to_numpy() @@ -243,177 +291,270 @@ def getPupilDiameter(analyzedVideoDataPath): def analyze_videos_pupillometry(configPath, videoPath, output_dir): import deeplabcut + # Analyze video and get pupil diameter data deeplabcut.analyze_videos(configPath, videoPath, destfolder=output_dir) - @staticmethod @pupillometry_exception_handler def check_pupillometry_sessions_queue(): - status_update = config.status_update_idx['NO_CHANGE'] + status_update = config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) - sessions_missing_process = (acquisition.SessionVideo * - pupillometry.PupillometrySessionModelData & 'pupillometry_job_id is NULL').fetch(as_dict=True) + sessions_missing_process = ( + acquisition.SessionVideo * pupillometry.PupillometrySessionModelData + & "pupillometry_job_id is NULL" + ).fetch(as_dict=True) print(sessions_missing_process) for pupillometry_2_process in sessions_missing_process: - # If error, job id = -1 - key_insert = dict((k, pupillometry_2_process[k]) for k in ('subject_fullname', 'session_date', 'session_number', 'model_id')) - key_insert['pupillometry_job_id'] = -1 + key_insert = dict( + (k, pupillometry_2_process[k]) + for k in ( + "subject_fullname", + "session_date", + "session_number", + "model_id", + ) + ) + key_insert["pupillometry_job_id"] = -1 # Get model location - model_key = (pupillometry.PupillometryModels & "model_id = "+str(pupillometry_2_process['model_id'])).fetch(as_dict=True)[0] - models_dir = dj.config.get('custom', {}).get('root_data_dir',None) + model_key = ( + pupillometry.PupillometryModels + & "model_id = " + str(pupillometry_2_process["model_id"]) + ).fetch(as_dict=True)[0] + models_dir = dj.config.get("custom", {}).get("root_data_dir", None) if models_dir is None: - raise Exception('root_data_dir in not found in config, , run initial_conf.py again') + raise Exception( + "root_data_dir in not found in config, , run initial_conf.py again" + ) models_dir = models_dir[0] - model_path = pathlib.Path(models_dir, model_key['model_path']) + model_path = pathlib.Path(models_dir, model_key["model_path"]) - print('model path', model_path) + print("model path", model_path) # Get video location - pupillometry_dir = dj.config.get('custom', {}).get('pupillometry_root_data_dir',None) + pupillometry_dir = dj.config.get("custom", {}).get( + "pupillometry_root_data_dir", None + ) if pupillometry_dir is None: - raise Exception('pupillometry_root_data_dir not found in config, run initial_conf.py again') + raise Exception( + "pupillometry_root_data_dir not found in config, run initial_conf.py again" + ) pupillometry_raw_dir = pupillometry_dir[0] - videoPath = pathlib.Path(pupillometry_raw_dir, pupillometry_2_process['remote_path_video_file']) + videoPath = pathlib.Path( + pupillometry_raw_dir, pupillometry_2_process["remote_path_video_file"] + ) - print('videoPath', videoPath) + print("videoPath", videoPath) # Create output location pupillometry_processed_dir = pupillometry_dir[1] - output_dir = pathlib.Path(pupillometry_processed_dir,pathlib.Path(pupillometry_2_process['remote_path_video_file']).parent) + output_dir = pathlib.Path( + pupillometry_processed_dir, + pathlib.Path(pupillometry_2_process["remote_path_video_file"]).parent, + ) if not output_dir.exists(): output_dir.mkdir(parents=True, exist_ok=True) # Generate slurm file and transfer it to spock - status, slurm_filepath = PupillometryProcessingHandler.generate_slurm_file(videoPath) - print('slurm_filepath', slurm_filepath) + status, slurm_filepath = PupillometryProcessingHandler.generate_slurm_file( + videoPath + ) + print("slurm_filepath", slurm_filepath) # Error handling (generating slurm file) - if status != config.system_process['SUCCESS']: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'Error while generating/transfering pupillometry slurm file' + if status != config.system_process["SUCCESS"]: + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "Error while generating/transfering pupillometry slurm file" + ) pupillometry.PupillometrySessionModelData.update1(key_insert) - slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\ - update_value_dict['error_info'] ,pupillometry_2_process) + slack_utils.send_slack_error_pupillometry_notification( + config.slack_webhooks_dict[ + "automation_pipeline_error_notification" + ], + update_value_dict["error_info"], + pupillometry_2_process, + ) - #return (status_update, update_value_dict) + # return (status_update, update_value_dict) continue # Queue slurm file in spock - status, slurm_jobid, error_message = PupillometryProcessingHandler.queue_pupillometry_slurm_file( - videoPath, model_path, PupillometryProcessingHandler.spock_home_dir, output_dir, - PupillometryProcessingHandler.process_script_path, slurm_filepath) + status, slurm_jobid, error_message = ( + PupillometryProcessingHandler.queue_pupillometry_slurm_file( + videoPath, + model_path, + PupillometryProcessingHandler.spock_home_dir, + output_dir, + PupillometryProcessingHandler.process_script_path, + slurm_filepath, + ) + ) # Error handling (queuing slurm file) - if status != config.system_process['SUCCESS']: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'Error to queue pupillometry slurm file' + if status != config.system_process["SUCCESS"]: + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "Error to queue pupillometry slurm file" + ) pupillometry.PupillometrySessionModelData.update1(key_insert) - slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\ - update_value_dict['error_info'] ,pupillometry_2_process) + slack_utils.send_slack_error_pupillometry_notification( + config.slack_webhooks_dict[ + "automation_pipeline_error_notification" + ], + update_value_dict["error_info"], + pupillometry_2_process, + ) - #return (status_update, update_value_dict) + # return (status_update, update_value_dict) continue # If success, store job_id - key_insert['pupillometry_job_id'] = slurm_jobid + key_insert["pupillometry_job_id"] = slurm_jobid pupillometry.PupillometrySessionModelData.update1(key_insert) - slack_utils.send_slack_pupillometry_update_notification(config.slack_webhooks_dict['automation_pipeline_update_notification'],\ - 'Pupillometry job submitted', pupillometry_2_process) + slack_utils.send_slack_pupillometry_update_notification( + config.slack_webhooks_dict["automation_pipeline_update_notification"], + "Pupillometry job submitted", + pupillometry_2_process, + ) @staticmethod @pupillometry_exception_handler def check_processed_pupillometry_sessions(): - #status_update = config.status_update_idx['NO_CHANGE'] + # status_update = config.status_update_idx['NO_CHANGE'] update_value_dict = copy.deepcopy(config.default_update_value_dict) - sessions_to_check = (acquisition.SessionVideo * pupillometry.PupillometrySessionModelData & 'pupillometry_job_id > 0 and pupil_diameter is NULL').fetch(as_dict=True) + sessions_to_check = ( + acquisition.SessionVideo * pupillometry.PupillometrySessionModelData + & "pupillometry_job_id > 0 and pupil_diameter is NULL" + ).fetch(as_dict=True) for session_check in sessions_to_check: - - key_update = dict((k, session_check[k]) for k in ('subject_fullname', 'session_date', 'session_number', 'model_id')) - print('key_update1', key_update) - - status_update, message = slurmlib.check_slurm_job('u19prod', PupillometryProcessingHandler.spock_system_name, str(session_check['pupillometry_job_id']), local_user=False) - - #If job finished copy over output and/or error log - if status_update == config.status_update_idx['ERROR_STATUS']: - - update_value_dict['error_info']['error_message'] = 'An error occured in processing (check LOG)' - update_value_dict['error_info']['error_exception'] = message - - key_update['pupillometry_job_id'] = -1 + key_update = dict( + (k, session_check[k]) + for k in ( + "subject_fullname", + "session_date", + "session_number", + "model_id", + ) + ) + print("key_update1", key_update) + + status_update, message = slurmlib.check_slurm_job( + "u19prod", + PupillometryProcessingHandler.spock_system_name, + str(session_check["pupillometry_job_id"]), + local_user=False, + ) + + # If job finished copy over output and/or error log + if status_update == config.status_update_idx["ERROR_STATUS"]: + update_value_dict["error_info"]["error_message"] = ( + "An error occured in processing (check LOG)" + ) + update_value_dict["error_info"]["error_exception"] = message + + key_update["pupillometry_job_id"] = -1 pupillometry.PupillometrySessionModelData.update1(key_update) - slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\ - update_value_dict['error_info'] ,session_check) + slack_utils.send_slack_error_pupillometry_notification( + config.slack_webhooks_dict[ + "automation_pipeline_error_notification" + ], + update_value_dict["error_info"], + session_check, + ) continue - # Get video location - if status_update == config.status_update_idx['NEXT_STATUS']: - pupillometry_dir = dj.config.get('custom', {}).get('pupillometry_root_data_dir',None) + if status_update == config.status_update_idx["NEXT_STATUS"]: + pupillometry_dir = dj.config.get("custom", {}).get( + "pupillometry_root_data_dir", None + ) if pupillometry_dir is None: - raise Exception('pupillometry_root_data_dir not found in config, run initial_conf.py again') + raise Exception( + "pupillometry_root_data_dir not found in config, run initial_conf.py again" + ) # Create output location pupillometry_processed_dir = pupillometry_dir[1] - output_dir = pathlib.Path(pupillometry_processed_dir,pathlib.Path(session_check['remote_path_video_file']).parent) + output_dir = pathlib.Path( + pupillometry_processed_dir, + pathlib.Path(session_check["remote_path_video_file"]).parent, + ) print(output_dir) - #Find h5 files - h5_files = glob.glob(str(output_dir) + '/*.h5') + # Find h5 files + h5_files = glob.glob(str(output_dir) + "/*.h5") if len(h5_files) != 1: - update_value_dict['error_info']['error_message'] = 'Didn''t find any h5 files after deeplabcut analyze_video' - slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\ - update_value_dict['error_info'] ,session_check) - key_update['pupillometry_job_id'] = -1 + update_value_dict["error_info"]["error_message"] = ( + "Didnt find any h5 files after deeplabcut analyze_video" + ) + slack_utils.send_slack_error_pupillometry_notification( + config.slack_webhooks_dict[ + "automation_pipeline_error_notification" + ], + update_value_dict["error_info"], + session_check, + ) + key_update["pupillometry_job_id"] = -1 pupillometry.PupillometrySessionModelData.update1(key_update) continue else: h5_files = h5_files[0] try: - pupil_data = PupillometryProcessingHandler.getPupilDiameter(h5_files) + pupil_data = PupillometryProcessingHandler.getPupilDiameter( + h5_files + ) except Exception: - update_value_dict['error_info']['error_message'] = 'Could not get pupil diameter (check h5 or video file)' - slack_utils.send_slack_error_pupillometry_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\ - update_value_dict['error_info'] ,session_check) - key_update['pupillometry_job_id'] = -1 + update_value_dict["error_info"]["error_message"] = ( + "Could not get pupil diameter (check h5 or video file)" + ) + slack_utils.send_slack_error_pupillometry_notification( + config.slack_webhooks_dict[ + "automation_pipeline_error_notification" + ], + update_value_dict["error_info"], + session_check, + ) + key_update["pupillometry_job_id"] = -1 pupillometry.PupillometrySessionModelData.update1(key_update) continue - key_update['pupil_diameter'] = pupil_data - print('key_update', key_update) + key_update["pupil_diameter"] = pupil_data + print("key_update", key_update) pupillometry.PupillometrySessionModelData.update1(key_update) - slack_utils.send_slack_pupillometry_update_notification(config.slack_webhooks_dict['automation_pipeline_update_notification'],\ - 'Pupillometry job finished', session_check) - + slack_utils.send_slack_pupillometry_update_notification( + config.slack_webhooks_dict[ + "automation_pipeline_update_notification" + ], + "Pupillometry job finished", + session_check, + ) -if __name__ == '__main__': - +if __name__ == "__main__": import time from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(1) args = sys.argv[1:] - args[1] = args[1] + '/config.yaml' + args[1] = args[1] + "/config.yaml" print(args) # - - PupillometryProcessingHandler.analyze_videos_pupillometry(args[1], args[0], args[2]) - diff --git a/u19_pipeline/automatic_job/pupillometry_handler_script.py b/u19_pipeline/automatic_job/pupillometry_handler_script.py index 1b595c45..42df0e1a 100644 --- a/u19_pipeline/automatic_job/pupillometry_handler_script.py +++ b/u19_pipeline/automatic_job/pupillometry_handler_script.py @@ -1,4 +1,3 @@ - import time from scripts.conf_file_finding import try_find_conf_file diff --git a/u19_pipeline/automatic_job/pupillometry_main.py b/u19_pipeline/automatic_job/pupillometry_main.py index ab882904..9bfbe20a 100644 --- a/u19_pipeline/automatic_job/pupillometry_main.py +++ b/u19_pipeline/automatic_job/pupillometry_main.py @@ -1,10 +1,10 @@ import sys -if __name__ == '__main__': - +if __name__ == "__main__": import time from scripts.conf_file_finding import try_find_conf_file + try_find_conf_file() time.sleep(1) @@ -16,4 +16,6 @@ print(args[1]) print(args[2]) - ph.PupillometryProcessingHandler.analyze_videos_pupillometry(args[0], args[1], args[2]) + ph.PupillometryProcessingHandler.analyze_videos_pupillometry( + args[0], args[1], args[2] + ) diff --git a/u19_pipeline/automatic_job/recording_handler.py b/u19_pipeline/automatic_job/recording_handler.py index e02b4f7b..536486d5 100644 --- a/u19_pipeline/automatic_job/recording_handler.py +++ b/u19_pipeline/automatic_job/recording_handler.py @@ -1,4 +1,3 @@ - import copy import pathlib import time @@ -23,88 +22,112 @@ def exception_handler(func): - ''' + """ Decorator function to get error message when a workflow manager function fails - ''' + """ + def inner_function(*args, **kwargs): try: - argout = func(*args, **kwargs) - return argout + argout = func(*args, **kwargs) + return argout except Exception as e: - print('Exception HERE ................') + print("Exception HERE ................") update_value_dict = copy.deepcopy(config.default_update_value_dict) - update_value_dict['error_info']['error_message'] = str(e) - update_value_dict['error_info']['error_exception'] = (''.join(traceback.format_exception(type(e), value=e, tb=e.__traceback__))) + update_value_dict["error_info"]["error_message"] = str(e) + update_value_dict["error_info"]["error_exception"] = "".join( + traceback.format_exception(type(e), value=e, tb=e.__traceback__) + ) return (config.RECORDING_STATUS_ERROR_ID, update_value_dict) + return inner_function -class RecordingHandler: +class RecordingHandler: @staticmethod def pipeline_handler_main(): - ''' + """ Call all processing functions according to the current status of each recording Update status of each process job accordingly - ''' + """ - #Get info from all the possible status for a processjob + # Get info from all the possible status for a processjob df_all_recordings = RecordingHandler.get_active_recordings() - #For all active process jobs + # For all active process jobs for i in range(df_all_recordings.shape[0]): - - #Filter current process job + # Filter current process job recording_series = df_all_recordings.loc[i, :] - #Filter current status info - current_status = recording_series['status_recording_id'] - next_status_series = config.recording_status_df.loc[config.recording_status_df['Value'] == current_status+1, :].squeeze() + # Filter current status info + current_status = recording_series["status_recording_id"] + next_status_series = config.recording_status_df.loc[ + config.recording_status_df["Value"] == current_status + 1, : + ].squeeze() - print('function to apply:', next_status_series['ProcessFunction']) + print("function to apply:", next_status_series["ProcessFunction"]) # Get processing function - function_status_process = getattr(RecordingHandler, next_status_series['ProcessFunction']) + function_status_process = getattr( + RecordingHandler, next_status_series["ProcessFunction"] + ) - #Trigger process, if success update recording process record + # Trigger process, if success update recording process record try: status, update_dict = function_status_process(recording_series) - #Get dictionary of record process - key = recording_series['query_key'] + # Get dictionary of record process + key = recording_series["query_key"] - if status == config.status_update_idx['NEXT_STATUS']: - #Get values to update - next_status = next_status_series['Value'] - value_update = update_dict['value_update'] - field_update = next_status_series['UpdateField'] + if status == config.status_update_idx["NEXT_STATUS"]: + # Get values to update + next_status = next_status_series["Value"] + value_update = update_dict["value_update"] + field_update = next_status_series["UpdateField"] # Update status in u19_recording.recording table (possibly other field as well) - RecordingHandler.update_status_pipeline(key, next_status, field_update, value_update) + RecordingHandler.update_status_pipeline( + key, next_status, field_update, value_update + ) # Send slack Message to webhook if slack message activated for status - if next_status_series['SlackMessage']: - slack_utils.send_slack_update_notification(config.slack_webhooks_dict['automation_pipeline_update_notification'],\ - next_status_series['SlackMessage'], recording_series) - - #An error occurred in process - if status == config.status_update_idx['ERROR_STATUS']: + if next_status_series["SlackMessage"]: + slack_utils.send_slack_update_notification( + config.slack_webhooks_dict[ + "automation_pipeline_update_notification" + ], + next_status_series["SlackMessage"], + recording_series, + ) + + # An error occurred in process + if status == config.status_update_idx["ERROR_STATUS"]: next_status = config.RECORDING_STATUS_ERROR_ID - RecordingHandler.update_status_pipeline(key,next_status, None, None) - slack_utils.send_slack_error_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\ - update_dict['error_info'] ,recording_series) - - #if success or error update status timestamps table - if status != config.status_update_idx['NO_CHANGE']: - RecordingHandler.update_recording_log(recording_series['recording_id'], current_status, next_status, update_dict['error_info']) - + RecordingHandler.update_status_pipeline( + key, next_status, None, None + ) + slack_utils.send_slack_error_notification( + config.slack_webhooks_dict[ + "automation_pipeline_error_notification" + ], + update_dict["error_info"], + recording_series, + ) + + # if success or error update status timestamps table + if status != config.status_update_idx["NO_CHANGE"]: + RecordingHandler.update_recording_log( + recording_series["recording_id"], + current_status, + next_status, + update_dict["error_info"], + ) except Exception as err: - raise(err) + raise (err) ## Send notification error, update recording to error time.sleep(2) - @staticmethod @exception_handler def local_transfer_request(rec_series): @@ -122,24 +145,30 @@ def local_transfer_request(rec_series): 'error_info': error info to be inserted if error occured } """ - status_update = config.status_update_idx['NO_CHANGE'] + status_update = config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) - data_directory = pathlib.Path(dj.config['custom']['root_data_dir'], rec_series['recording_modality'], rec_series['recording_directory']).as_posix() + data_directory = pathlib.Path( + dj.config["custom"]["root_data_dir"], + rec_series["recording_modality"], + rec_series["recording_directory"], + ).as_posix() data_directory = pathlib.Path(data_directory).parent pathlib.Path(data_directory).mkdir(parents=True, exist_ok=True) + # Encode Windows like directory for scp to work - - #Encode Windows like directory for scp to work - - status, task_id = scp_tr.call_scp_background(ip_address=rec_series['ip_address'], system_user=rec_series['system_user'], - recording_system_directory=rec_series['local_directory'], data_directory=data_directory.as_posix()) + status, task_id = scp_tr.call_scp_background( + ip_address=rec_series["ip_address"], + system_user=rec_series["system_user"], + recording_system_directory=rec_series["local_directory"], + data_directory=data_directory.as_posix(), + ) if status: - status_update = config.status_update_idx['NEXT_STATUS'] - update_value_dict['value_update'] = task_id + status_update = config.status_update_idx["NEXT_STATUS"] + update_value_dict["value_update"] = task_id return (status_update, update_value_dict) @@ -160,21 +189,23 @@ def local_transfer_check(rec_series): 'error_info': error info to be inserted if error occured } """ - status_update = config.status_update_idx['NO_CHANGE'] + status_update = config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) - id_task = rec_series['task_copy_id_pni'] + id_task = rec_series["task_copy_id_pni"] is_finished, exit_code = scp_tr.check_scp_transfer(id_task) if is_finished: if exit_code == 0: - status_update = config.status_update_idx['NEXT_STATUS'] + status_update = config.status_update_idx["NEXT_STATUS"] else: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'Return code scp not = 0' + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "Return code scp not = 0" + ) - print('is_finished', is_finished, 'status_update', status_update) + print("is_finished", is_finished, "status_update", status_update) return (status_update, update_value_dict) @@ -196,30 +227,40 @@ def modality_preingestion(in_rec_series): rec_series = in_rec_series.copy() - if rec_series['recording_modality'] == 'electrophysiology': - status_update, update_value_dict = RecordingHandler.electrophysiology_preingestion(rec_series) - - if rec_series['recording_modality'] == 'imaging': - status_update, update_value_dict = RecordingHandler.imaging_preingestion(rec_series) + if rec_series["recording_modality"] == "electrophysiology": + status_update, update_value_dict = ( + RecordingHandler.electrophysiology_preingestion(rec_series) + ) + if rec_series["recording_modality"] == "imaging": + status_update, update_value_dict = RecordingHandler.imaging_preingestion( + rec_series + ) return (status_update, update_value_dict) - @staticmethod def get_active_recordings(): - ''' + """ get all recordings that have to go through some action in the pipeline Return: df_recordings (pd.DataFrame): all recordings that are going to be processed in the pipeline - ''' + """ - status_query = 'status_recording_id > ' + str(config.recording_status_df['Value'].min()) - status_query += ' and status_recording_id < ' + str(config.recording_status_df['Value'].max()) + status_query = "status_recording_id > " + str( + config.recording_status_df["Value"].min() + ) + status_query += " and status_recording_id < " + str( + config.recording_status_df["Value"].max() + ) print(status_query) - recordings_active = recording.Recording * lab.Location.proj('ip_address', 'system_user', 'acquisition_type') & status_query + recordings_active = ( + recording.Recording + * lab.Location.proj("ip_address", "system_user", "acquisition_type") + & status_query + ) print(recordings_active) @@ -227,13 +268,16 @@ def get_active_recordings(): if df_recordings.shape[0] > 0: key_list = dj_short.get_primary_key_fields(recording.Recording) - df_recordings['query_key'] = df_recordings.loc[:, key_list].to_dict(orient='records') + df_recordings["query_key"] = df_recordings.loc[:, key_list].to_dict( + orient="records" + ) return df_recordings - @staticmethod - def update_status_pipeline(recording_key_dict, status, update_field=None, update_value=None): + def update_status_pipeline( + recording_key_dict, status, update_field=None, update_value=None + ): """ Update recording.Recording table status and optional task field Args: @@ -243,24 +287,26 @@ def update_status_pipeline(recording_key_dict, status, update_field=None, update update_value (str|int): field value to be inserted on in task_field """ - print('recording_key_dict', recording_key_dict) - print('status', status) - print('update_field', update_field) - print('update_value', update_value) + print("recording_key_dict", recording_key_dict) + print("status", status) + print("update_field", update_field) + print("update_value", update_value) if update_field is not None: update_task_id_dict = recording_key_dict.copy() update_task_id_dict[update_field] = update_value - print('update_task_id_dict', update_task_id_dict) + print("update_task_id_dict", update_task_id_dict) recording.Recording.update1(update_task_id_dict) update_status_dict = recording_key_dict.copy() - update_status_dict['status_recording_id'] = status - print('update_status_dict', update_status_dict) + update_status_dict["status_recording_id"] = status + print("update_status_dict", update_status_dict) recording.Recording.update1(update_status_dict) @staticmethod - def update_recording_log(recording_id, current_status, next_status, error_info_dict): + def update_recording_log( + recording_id, current_status, next_status, error_info_dict + ): """ Update recording.RecordingLog table status and optional task field Args: @@ -270,26 +316,33 @@ def update_recording_log(recording_id, current_status, next_status, error_info_d now = datetime.now() date_time = now.strftime("%Y-%m-%d %H:%M:%S") - print('error_info_dict', error_info_dict) + print("error_info_dict", error_info_dict) key = dict() - key['recording_id'] = recording_id - key['status_recording_id_old'] = current_status - key['status_recording_id_new'] = next_status - key['recording_status_timestamp'] = date_time - - if error_info_dict['error_message'] is not None and len(error_info_dict['error_message']) >= 256: - error_info_dict['error_message'] =error_info_dict['error_message'][:255] - - if error_info_dict['error_exception'] is not None and len(error_info_dict['error_exception']) >= 4096: - error_info_dict['error_exception'] =error_info_dict['error_exception'][:4095] - - key['recording_error_message'] = error_info_dict['error_message'] - key['recording_error_exception'] = error_info_dict['error_exception'] + key["recording_id"] = recording_id + key["status_recording_id_old"] = current_status + key["status_recording_id_new"] = next_status + key["recording_status_timestamp"] = date_time + + if ( + error_info_dict["error_message"] is not None + and len(error_info_dict["error_message"]) >= 256 + ): + error_info_dict["error_message"] = error_info_dict["error_message"][:255] + + if ( + error_info_dict["error_exception"] is not None + and len(error_info_dict["error_exception"]) >= 4096 + ): + error_info_dict["error_exception"] = error_info_dict["error_exception"][ + :4095 + ] + + key["recording_error_message"] = error_info_dict["error_message"] + key["recording_error_exception"] = error_info_dict["error_exception"] recording.LogStatus.insert1(key) - @staticmethod def electrophysiology_preingestion(rec_series): """ @@ -305,65 +358,107 @@ def electrophysiology_preingestion(rec_series): 'error_info': error info to be inserted if error occured } """ - status_update = config.status_update_idx['NO_CHANGE'] + status_update = config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) - #Insert first ephysSession and firs tables of ephys elements - ephys_pipeline.EphysPipelineSession.populate(rec_series['query_key']) - ephys_element_ingest.process_session(rec_series['query_key']) - ephys_pipeline.ephys_element.EphysRecording.populate(rec_series['query_key']) + # Insert first ephysSession and firs tables of ephys elements + ephys_pipeline.EphysPipelineSession.populate(rec_series["query_key"]) + ephys_element_ingest.process_session(rec_series["query_key"]) + ephys_pipeline.ephys_element.EphysRecording.populate(rec_series["query_key"]) - ingested_recording = (ephys_pipeline.ephys_element.EphysRecording & rec_series['query_key']).fetch("KEY", as_dict=True) + ingested_recording = ( + ephys_pipeline.ephys_element.EphysRecording & rec_series["query_key"] + ).fetch("KEY", as_dict=True) - print('before BehaviorSync') - print("rec_series['query_key']", rec_series['query_key']) + print("before BehaviorSync") + print("rec_series['query_key']", rec_series["query_key"]) - ephys_pipeline.BehaviorSync.populate(rec_series['query_key']) + ephys_pipeline.BehaviorSync.populate(rec_series["query_key"]) if len(ingested_recording) == 0: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'Ephys Recording was not ingested (probably recording not in location)' + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "Ephys Recording was not ingested (probably recording not in location)" + ) return (status_update, update_value_dict) - #Insert recording processes records - old_recording_process = (recording_process.Processing() & rec_series['query_key']).fetch("KEY", as_dict=True) + # Insert recording processes records + old_recording_process = ( + recording_process.Processing() & rec_series["query_key"] + ).fetch("KEY", as_dict=True) if len(old_recording_process) == 0: - connection = recording.Recording.connection with connection.transaction: - - probe_files = (ephys_pipeline.ephys_element.EphysRecording.EphysFile & rec_series['query_key']).fetch(as_dict=True) - probe_files = [dict(item, recording_process_pre_path=pathlib.Path(item['file_path']).parents[0].as_posix()) for item in probe_files] - - recording_process.Processing().insert_recording_process(probe_files, 'insertion_number') - - #Get parameters for recording processes - recording_processes = (recording_process.Processing() & rec_series['query_key']).fetch('job_id', 'recording_id', 'fragment_number', 'recording_process_pre_path', as_dict=True) - default_params_record_df = pd.DataFrame((recording.DefaultParams & rec_series['query_key']).fetch(as_dict=True)) - params_rec_process = recording.DefaultParams.get_default_params_rec_process(recording_processes, default_params_record_df) + probe_files = ( + ephys_pipeline.ephys_element.EphysRecording.EphysFile + & rec_series["query_key"] + ).fetch(as_dict=True) + probe_files = [ + dict( + item, + recording_process_pre_path=pathlib.Path(item["file_path"]) + .parents[0] + .as_posix(), + ) + for item in probe_files + ] + + recording_process.Processing().insert_recording_process( + probe_files, "insertion_number" + ) + + # Get parameters for recording processes + recording_processes = ( + recording_process.Processing() & rec_series["query_key"] + ).fetch( + "job_id", + "recording_id", + "fragment_number", + "recording_process_pre_path", + as_dict=True, + ) + default_params_record_df = pd.DataFrame( + (recording.DefaultParams & rec_series["query_key"]).fetch( + as_dict=True + ) + ) + params_rec_process = ( + recording.DefaultParams.get_default_params_rec_process( + recording_processes, default_params_record_df + ) + ) # Rename preprocess_param_steps_id with the electrophysiology one for i in params_rec_process: - i['precluster_param_steps_id'] = i.pop('preprocess_param_steps_id') + i["precluster_param_steps_id"] = i.pop("preprocess_param_steps_id") recording_process.Processing.EphysParams.insert(params_rec_process) - #Update recording_process_post_path - recording_process.Processing().set_recording_process_post_path(recording_processes) - - #Create lfp trace if needed (neuropixel 2.0 probes) - recording_directory = (recording.Recording & rec_series['query_key']).fetch1('recording_directory') - recording_directory = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][0], recording_directory).parent.as_posix() + # Update recording_process_post_path + recording_process.Processing().set_recording_process_post_path( + recording_processes + ) + + # Create lfp trace if needed (neuropixel 2.0 probes) + recording_directory = ( + recording.Recording & rec_series["query_key"] + ).fetch1("recording_directory") + recording_directory = pathlib.Path( + dj.config["custom"]["ephys_root_data_dir"][0], recording_directory + ).parent.as_posix() for i in recording_processes: - probe_dir = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][0], i['recording_process_pre_path']).as_posix() - ephys_pipeline.create_lfp_trace(config.catgt_script, recording_directory, probe_dir) - + probe_dir = pathlib.Path( + dj.config["custom"]["ephys_root_data_dir"][0], + i["recording_process_pre_path"], + ).as_posix() + ephys_pipeline.create_lfp_trace( + config.catgt_script, recording_directory, probe_dir + ) - status_update = config.status_update_idx['NEXT_STATUS'] + status_update = config.status_update_idx["NEXT_STATUS"] return (status_update, update_value_dict) - @staticmethod def imaging_preingestion(rec_series): """ @@ -379,75 +474,130 @@ def imaging_preingestion(rec_series): 'error_info': error info to be inserted if error occured } """ - status_update = config.status_update_idx['NO_CHANGE'] + status_update = config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) print("rec_series['query_key']") - print(rec_series['query_key']) + print(rec_series["query_key"]) - #Populate ImagingPipelineSession and call matlab script that handles TiffSplits - imaging_pipeline.ImagingPipelineSession.populate(rec_series['query_key']) - imaging_pipeline.AcquiredTiff.populate(rec_series['query_key']) + # Populate ImagingPipelineSession and call matlab script that handles TiffSplits + imaging_pipeline.ImagingPipelineSession.populate(rec_series["query_key"]) + imaging_pipeline.AcquiredTiff.populate(rec_series["query_key"]) - #Retrieve all fovs records ingested in matlab Script - fovs_ingested = (imaging_pipeline.TiffSplit & rec_series['query_key']).fetch("KEY", as_dict=True) + # Retrieve all fovs records ingested in matlab Script + fovs_ingested = (imaging_pipeline.TiffSplit & rec_series["query_key"]).fetch( + "KEY", as_dict=True + ) if len(fovs_ingested) == 0: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'Imaging TiffSplit process failed' + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "Imaging TiffSplit process failed" + ) return (status_update, update_value_dict) - #Ingest Scan for each fov from the TiffSplit process + # Ingest Scan for each fov from the TiffSplit process for this_fov in fovs_ingested: - # Scan_id always zero because TIFF splitted (FOVs) already on imaging_pipeline schema scan_id = 0 # Acquisition type will have Mesoscope or 2Photon - scanner = rec_series['acquisition_type'] + scanner = rec_series["acquisition_type"] # Hardcoded acquisition software - acq_software = 'ScanImage' + acq_software = "ScanImage" - #Insert Scan and ScanInfo + # Insert Scan and ScanInfo imaging_pipeline.scan_element.Scan.insert1( - {**this_fov, 'scan_id': 0, 'scanner': scanner, 'acq_software': acq_software}, skip_duplicates=True) - - #Populate ScanInfo for all fovs - imaging_pipeline.scan_element.ScanInfo.populate(rec_series['query_key'], display_progress=True) - - #ingested_recording = (imaging_pipeline.scan_element.Scan & rec_series['query_key']).fetch("KEY", as_dict=True) + { + **this_fov, + "scan_id": 0, + "scanner": scanner, + "acq_software": acq_software, + }, + skip_duplicates=True, + ) + + # Populate ScanInfo for all fovs + imaging_pipeline.scan_element.ScanInfo.populate( + rec_series["query_key"], display_progress=True + ) + + # ingested_recording = (imaging_pipeline.scan_element.Scan & rec_series['query_key']).fetch("KEY", as_dict=True) # Get fov directories for each recording process: - fov_files_df = pd.DataFrame((imaging_pipeline.scan_element.ScanInfo.ScanFile & rec_series['query_key']).fetch(as_dict=True)) - - fov_files = fov_files_df.groupby('tiff_split').first().reset_index() - - - #Insert recording processes records - old_recording_process = (recording_process.Processing() & rec_series['query_key']).fetch("KEY", as_dict=True) + fov_files_df = pd.DataFrame( + ( + imaging_pipeline.scan_element.ScanInfo.ScanFile + & rec_series["query_key"] + ).fetch(as_dict=True) + ) + + fov_files = fov_files_df.groupby("tiff_split").first().reset_index() + + # Insert recording processes records + old_recording_process = ( + recording_process.Processing() & rec_series["query_key"] + ).fetch("KEY", as_dict=True) if len(old_recording_process) == 0: - connection = recording.Recording.connection with connection.transaction: - # Get fov directories for each recording process: - fov_files_df = pd.DataFrame((imaging_pipeline.scan_element.ScanInfo.ScanFile & rec_series['query_key']).fetch(as_dict=True)) - fov_files = fov_files_df.groupby('tiff_split').first().reset_index().to_dict('records') - - fov_files = [dict(item, recording_process_pre_path=pathlib.Path(item['file_path']).parent.as_posix()) for item in fov_files] - - recording_process.Processing().insert_recording_process(fov_files, 'tiff_split') - - #Get parameters for recording processes - recording_processes = (recording_process.Processing() & rec_series['query_key']).fetch('job_id', 'recording_id', 'fragment_number', 'recording_process_pre_path', as_dict=True) - default_params_record_df = pd.DataFrame((recording.DefaultParams & rec_series['query_key']).fetch(as_dict=True)) - params_rec_process = recording.DefaultParams.get_default_params_rec_process(recording_processes, default_params_record_df) - - recording_process.Processing.ImagingParams.insert(params_rec_process, skip_duplicates=True) - - #Update recording_process_post_path - recording_process.Processing().set_recording_process_post_path(recording_processes) - - status_update = config.status_update_idx['NEXT_STATUS'] + fov_files_df = pd.DataFrame( + ( + imaging_pipeline.scan_element.ScanInfo.ScanFile + & rec_series["query_key"] + ).fetch(as_dict=True) + ) + fov_files = ( + fov_files_df.groupby("tiff_split") + .first() + .reset_index() + .to_dict("records") + ) + + fov_files = [ + dict( + item, + recording_process_pre_path=pathlib.Path( + item["file_path"] + ).parent.as_posix(), + ) + for item in fov_files + ] + + recording_process.Processing().insert_recording_process( + fov_files, "tiff_split" + ) + + # Get parameters for recording processes + recording_processes = ( + recording_process.Processing() & rec_series["query_key"] + ).fetch( + "job_id", + "recording_id", + "fragment_number", + "recording_process_pre_path", + as_dict=True, + ) + default_params_record_df = pd.DataFrame( + (recording.DefaultParams & rec_series["query_key"]).fetch( + as_dict=True + ) + ) + params_rec_process = ( + recording.DefaultParams.get_default_params_rec_process( + recording_processes, default_params_record_df + ) + ) + + recording_process.Processing.ImagingParams.insert( + params_rec_process, skip_duplicates=True + ) + + # Update recording_process_post_path + recording_process.Processing().set_recording_process_post_path( + recording_processes + ) + + status_update = config.status_update_idx["NEXT_STATUS"] return (status_update, update_value_dict) - diff --git a/u19_pipeline/automatic_job/recording_process_handler.py b/u19_pipeline/automatic_job/recording_process_handler.py index b0d2737a..0ad22224 100644 --- a/u19_pipeline/automatic_job/recording_process_handler.py +++ b/u19_pipeline/automatic_job/recording_process_handler.py @@ -1,4 +1,3 @@ - import copy import pathlib import time @@ -28,101 +27,142 @@ class RecProcessHandler: - @staticmethod def pipeline_handler_main(): - ''' + """ Call all processing functions according to the current status of each process job Update status of each process job accordingly - ''' + """ - #Get info from all the possible status for a processjob + # Get info from all the possible status for a processjob df_all_process_job = RecProcessHandler.get_active_process_jobs() - #For all active process jobs + # For all active process jobs for i in range(df_all_process_job.shape[0]): - - #Filter current process job + # Filter current process job rec_process_series = df_all_process_job.loc[i, :].copy() - #Filter current status info - current_status = rec_process_series['status_processing_id'] - current_status_series = config.recording_process_status_df.loc[config.recording_process_status_df['Value'] == current_status, :].squeeze() - next_status_series = config.recording_process_status_df.loc[config.recording_process_status_df['Value'] == current_status+1, :].squeeze() + # Filter current status info + current_status = rec_process_series["status_processing_id"] + current_status_series = config.recording_process_status_df.loc[ + config.recording_process_status_df["Value"] == current_status, : + ].squeeze() + next_status_series = config.recording_process_status_df.loc[ + config.recording_process_status_df["Value"] == current_status + 1, : + ].squeeze() # Get processing function - function_status_process = getattr(RecProcessHandler, next_status_series['ProcessFunction']) + function_status_process = getattr( + RecProcessHandler, next_status_series["ProcessFunction"] + ) - #Trigger process, if success update recording process record + # Trigger process, if success update recording process record try: - status, update_dict = function_status_process(rec_process_series, next_status_series) + status, update_dict = function_status_process( + rec_process_series, next_status_series + ) - #print('update_dict', update_dict) - #Get dictionary of record process - key = rec_process_series['query_key'] + # print('update_dict', update_dict) + # Get dictionary of record process + key = rec_process_series["query_key"] - print('rec_process_series') + print("rec_process_series") print(rec_process_series) - if status == config.status_update_idx['NEXT_STATUS']: - - - #Get values to update - next_status = next_status_series['Value'] - value_update = update_dict['value_update'] - field_update = next_status_series['UpdateField'] - - print('key to update', key) - print('old status', current_status, 'new status', next_status) - print('value_update', value_update, 'field_update', field_update) - print('function executed:', next_status_series['ProcessFunction']) - - - RecProcessHandler.update_status_pipeline(key, next_status, field_update, value_update) - - if next_status_series['SlackMessage']: - slack_utils.send_slack_update_notification(config.slack_webhooks_dict['automation_pipeline_update_notification'],\ - next_status_series['SlackMessage'], rec_process_series) - - #if success or error update status timestamps table - if status != config.status_update_idx['NO_CHANGE']: - if status == config.status_update_idx['ERROR_STATUS']: + if status == config.status_update_idx["NEXT_STATUS"]: + # Get values to update + next_status = next_status_series["Value"] + value_update = update_dict["value_update"] + field_update = next_status_series["UpdateField"] + + print("key to update", key) + print("old status", current_status, "new status", next_status) + print("value_update", value_update, "field_update", field_update) + print("function executed:", next_status_series["ProcessFunction"]) + + RecProcessHandler.update_status_pipeline( + key, next_status, field_update, value_update + ) + + if next_status_series["SlackMessage"]: + slack_utils.send_slack_update_notification( + config.slack_webhooks_dict[ + "automation_pipeline_update_notification" + ], + next_status_series["SlackMessage"], + rec_process_series, + ) + + # if success or error update status timestamps table + if status != config.status_update_idx["NO_CHANGE"]: + if status == config.status_update_idx["ERROR_STATUS"]: next_status = config.RECORDING_STATUS_ERROR_ID - #Crop error messages to fit in DB - if len(update_dict['error_info']['error_message']) > 255: - print('Cropping error message') - update_dict['error_info']['error_message'] = update_dict['error_info']['error_message'][-255:] - - if isinstance(update_dict['error_info']['error_exception'], str) and len(update_dict['error_info']['error_exception']) > 4095: - print('Cropping error error_exception') - update_dict['error_info']['error_exception'] = update_dict['error_info']['error_exception'][-4095:] - - RecProcessHandler.update_job_id_log(rec_process_series['job_id'], current_status, next_status, update_dict['error_info']) - - #An error occurred in process - if status == config.status_update_idx['ERROR_STATUS']: - + # Crop error messages to fit in DB + if len(update_dict["error_info"]["error_message"]) > 255: + print("Cropping error message") + update_dict["error_info"]["error_message"] = update_dict[ + "error_info" + ]["error_message"][-255:] + + if ( + isinstance( + update_dict["error_info"]["error_exception"], str + ) + and len(update_dict["error_info"]["error_exception"]) > 4095 + ): + print("Cropping error error_exception") + update_dict["error_info"]["error_exception"] = update_dict[ + "error_info" + ]["error_exception"][-4095:] + + RecProcessHandler.update_job_id_log( + rec_process_series["job_id"], + current_status, + next_status, + update_dict["error_info"], + ) + + # An error occurred in process + if status == config.status_update_idx["ERROR_STATUS"]: next_status = config.RECORDING_STATUS_ERROR_ID - RecProcessHandler.update_status_pipeline(key,next_status, None, None) - - #Crop even more error message to fit in slack notification - if isinstance(update_dict['error_info']['error_exception'], str) and len(update_dict['error_info']['error_exception']) > 1024: - print('Cropping error error_exception for slack') - update_dict['error_info']['error_exception'] = update_dict['error_info']['error_exception'][-1023:] + RecProcessHandler.update_status_pipeline( + key, next_status, None, None + ) + + # Crop even more error message to fit in slack notification + if ( + isinstance(update_dict["error_info"]["error_exception"], str) + and len(update_dict["error_info"]["error_exception"]) > 1024 + ): + print("Cropping error error_exception for slack") + update_dict["error_info"]["error_exception"] = update_dict[ + "error_info" + ]["error_exception"][-1023:] # Send slack notification with original error exception, retry if exception has invalid format try: - slack_utils.send_slack_error_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\ - update_dict['error_info'] ,rec_process_series) + slack_utils.send_slack_error_notification( + config.slack_webhooks_dict[ + "automation_pipeline_error_notification" + ], + update_dict["error_info"], + rec_process_series, + ) except: - update_dict['error_info']['error_exception'] = 'Error exception not accepted in slack, check log file for error' - slack_utils.send_slack_error_notification(config.slack_webhooks_dict['automation_pipeline_error_notification'],\ - update_dict['error_info'] ,rec_process_series) - + update_dict["error_info"]["error_exception"] = ( + "Error exception not accepted in slack, check log file for error" + ) + slack_utils.send_slack_error_notification( + config.slack_webhooks_dict[ + "automation_pipeline_error_notification" + ], + update_dict["error_info"], + rec_process_series, + ) except Exception as err: - raise(err) + raise (err) print(traceback.format_exc()) ## Send notification error, update recording to error @@ -145,38 +185,42 @@ def transfer_request(rec_series, status_series): 'error_info': error info to be inserted if error occured } """ - status_update = config.status_update_idx['NO_CHANGE'] + status_update = config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) - job_id = rec_series['job_id'] - raw_rel_path = rec_series['recording_process_pre_path'] - proc_rel_path = rec_series['recording_process_post_path'] - modality = rec_series['recording_modality'] + job_id = rec_series["job_id"] + raw_rel_path = rec_series["recording_process_pre_path"] + proc_rel_path = rec_series["recording_process_post_path"] + modality = rec_series["recording_modality"] # If tiger, we trigger globus transfer - if rec_series['program_selection_params']['process_cluster'] == "tiger": - - if status_series['Key'] == 'RAW_FILE_TRANSFER_REQUEST': + if rec_series["program_selection_params"]["process_cluster"] == "tiger": + if status_series["Key"] == "RAW_FILE_TRANSFER_REQUEST": + # status, task_id = scp_tr.call_scp_background(ip_address=rec_series['ip_address'], system_user=rec_series['system_user'], + # recording_system_directory=rec_series['local_directory'], data_directory=data_directory.as_posix()) - #status, task_id = scp_tr.call_scp_background(ip_address=rec_series['ip_address'], system_user=rec_series['system_user'], - #recording_system_directory=rec_series['local_directory'], data_directory=data_directory.as_posix()) - - transfer_request = ft.globus_transfer_to_tiger(job_id, raw_rel_path, modality) - elif status_series['Key'] == 'PROC_FILE_TRANSFER_REQUEST': - #ALS, which recording directory for processed file - transfer_request = ft.globus_transfer_to_pni(job_id, proc_rel_path, modality) + transfer_request = ft.globus_transfer_to_tiger( + job_id, raw_rel_path, modality + ) + elif status_series["Key"] == "PROC_FILE_TRANSFER_REQUEST": + # ALS, which recording directory for processed file + transfer_request = ft.globus_transfer_to_pni( + job_id, proc_rel_path, modality + ) - if transfer_request['status'] == config.system_process['SUCCESS']: - status_update = config.status_update_idx['NEXT_STATUS'] - update_value_dict['value_update'] = transfer_request['task_id'] + if transfer_request["status"] == config.system_process["SUCCESS"]: + status_update = config.status_update_idx["NEXT_STATUS"] + update_value_dict["value_update"] = transfer_request["task_id"] else: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = transfer_request['error_info'] + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = transfer_request[ + "error_info" + ] # If not tiger let's go to next status else: - status_update = config.status_update_idx['NEXT_STATUS'] + status_update = config.status_update_idx["NEXT_STATUS"] return (status_update, update_value_dict) @@ -197,27 +241,27 @@ def transfer_check(rec_series, status_series): 'error_info': error info to be inserted if error occured } """ - status_update = config.status_update_idx['NO_CHANGE'] + status_update = config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) - id_task = rec_series[status_series['FunctionField']] + id_task = rec_series[status_series["FunctionField"]] # If tiger, we trigger globus transfer - if rec_series['program_selection_params']['process_cluster'] == "tiger": - + if rec_series["program_selection_params"]["process_cluster"] == "tiger": transfer_request = ft.request_globus_transfer_status(str(id_task)) - if transfer_request['status'] == config.system_process['COMPLETED']: - status_update = config.status_update_idx['NEXT_STATUS'] - elif transfer_request['status'] == config.system_process['ERROR']: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'An error occured during globus transfer' + if transfer_request["status"] == config.system_process["COMPLETED"]: + status_update = config.status_update_idx["NEXT_STATUS"] + elif transfer_request["status"] == config.system_process["ERROR"]: + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "An error occured during globus transfer" + ) else: - status_update = config.status_update_idx['NEXT_STATUS'] + status_update = config.status_update_idx["NEXT_STATUS"] return (status_update, update_value_dict) - @staticmethod @recording_handler.exception_handler def slurm_job_queue(rec_series, status_series): @@ -235,81 +279,119 @@ def slurm_job_queue(rec_series, status_series): 'error_info': error info to be inserted if error occured } """ - status_update = config.status_update_idx['NO_CHANGE'] + status_update = config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) - print(rec_series['program_selection_params']) - + print(rec_series["program_selection_params"]) - #Create and transfer parameter file - status = paramfilelib.generate_parameter_file(rec_series['job_id'], rec_series['params'], 'params', rec_series['program_selection_params']) + # Create and transfer parameter file + status = paramfilelib.generate_parameter_file( + rec_series["job_id"], + rec_series["params"], + "params", + rec_series["program_selection_params"], + ) print(status) - #Create and transfer preparameter file - if status == config.system_process['SUCCESS']: - status = paramfilelib.generate_parameter_file(rec_series['job_id'], rec_series['preparams'], 'preparams', rec_series['program_selection_params']) + # Create and transfer preparameter file + if status == config.system_process["SUCCESS"]: + status = paramfilelib.generate_parameter_file( + rec_series["job_id"], + rec_series["preparams"], + "preparams", + rec_series["program_selection_params"], + ) else: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'Error while generating/transfering parameter file' + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "Error while generating/transfering parameter file" + ) return (status_update, update_value_dict) - #If electrophysiology, transfer chanmap as well - if status == config.system_process['SUCCESS'] and rec_series['recording_modality'] == 'electrophysiology': - recording_key = (recording_process.Processing.proj('recording_id', insertion_number='fragment_number') & rec_series['query_key']).fetch1() + # If electrophysiology, transfer chanmap as well + if ( + status == config.system_process["SUCCESS"] + and rec_series["recording_modality"] == "electrophysiology" + ): + recording_key = ( + recording_process.Processing.proj( + "recording_id", insertion_number="fragment_number" + ) + & rec_series["query_key"] + ).fetch1() del recording_key["job_id"] + chanmap_filename = config.default_chanmap_filename % (rec_series["job_id"]) + chanmap_file_local_path = pathlib.Path( + config.chanmap_files_filepath, chanmap_filename + ).as_posix() + raw_directory_for_chanmap = rec_series["recording_process_pre_path"] - chanmap_filename = config.default_chanmap_filename % (rec_series['job_id']) - chanmap_file_local_path = pathlib.Path(config.chanmap_files_filepath,chanmap_filename).as_posix() - raw_directory_for_chanmap = rec_series['recording_process_pre_path'] - - spikeglx_meta_filepath = ephys_pipeline.get_spikeglx_meta_filepath(recording_key) + spikeglx_meta_filepath = ephys_pipeline.get_spikeglx_meta_filepath( + recording_key + ) # Chanmap mat file generation - MetaToCoords(spikeglx_meta_filepath, 1, destFullPath =chanmap_file_local_path) - status = paramfilelib.generate_chanmap_file(rec_series['job_id'], rec_series['program_selection_params']) - if status != config.system_process['SUCCESS']: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'Error while generating/transfering channelmap file' + MetaToCoords( + spikeglx_meta_filepath, 1, destFullPath=chanmap_file_local_path + ) + status = paramfilelib.generate_chanmap_file( + rec_series["job_id"], rec_series["program_selection_params"] + ) + if status != config.system_process["SUCCESS"]: + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "Error while generating/transfering channelmap file" + ) return (status_update, update_value_dict) # Only queue if processing in tiger - if rec_series['program_selection_params']['local_or_cluster'] == "cluster": + if rec_series["program_selection_params"]["local_or_cluster"] == "cluster": + print("lets transfer slum file ..............xxxxxxxx............") - print('lets transfer slum file ..............xxxxxxxx............') - - #Create and transfer slurm file - if status == config.system_process['SUCCESS']: - status, slurm_filepath = slurmlib.generate_slurm_file(rec_series['job_id'], rec_series['program_selection_params']) + # Create and transfer slurm file + if status == config.system_process["SUCCESS"]: + status, slurm_filepath = slurmlib.generate_slurm_file( + rec_series["job_id"], rec_series["program_selection_params"] + ) else: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'Error while generating/transfering slurm file' + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "Error while generating/transfering slurm file" + ) return (status_update, update_value_dict) - #Queue slurm file - if status == config.system_process['SUCCESS']: - - status, slurm_jobid, error_message = slurmlib.queue_slurm_file(rec_series['job_id'], rec_series['program_selection_params'], - rec_series['recording_process_pre_path'], rec_series['recording_process_post_path'], - rec_series['recording_modality'], slurm_filepath + # Queue slurm file + if status == config.system_process["SUCCESS"]: + status, slurm_jobid, error_message = slurmlib.queue_slurm_file( + rec_series["job_id"], + rec_series["program_selection_params"], + rec_series["recording_process_pre_path"], + rec_series["recording_process_post_path"], + rec_series["recording_modality"], + slurm_filepath, ) else: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'Error while generating/transfering slurm file' + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "Error while generating/transfering slurm file" + ) return (status_update, update_value_dict) - if status == config.system_process['SUCCESS']: - status_update = config.status_update_idx['NEXT_STATUS'] - update_value_dict['value_update'] = slurm_jobid + if status == config.system_process["SUCCESS"]: + status_update = config.status_update_idx["NEXT_STATUS"] + update_value_dict["value_update"] = slurm_jobid else: - status_update = config.status_update_idx['ERROR_STATUS'] - update_value_dict['error_info']['error_message'] = 'Error while queuing job in SLURM' + status_update = config.status_update_idx["ERROR_STATUS"] + update_value_dict["error_info"]["error_message"] = ( + "Error while queuing job in SLURM" + ) return (status_update, update_value_dict) else: - #Just check we succeded with parameter files - if status == config.system_process['SUCCESS']: - status_update = config.status_update_idx['NEXT_STATUS'] + # Just check we succeded with parameter files + if status == config.system_process["SUCCESS"]: + status_update = config.status_update_idx["NEXT_STATUS"] return (status_update, update_value_dict) @@ -332,38 +414,61 @@ def slurm_job_check(rec_series, status_series): update_value_dict = copy.deepcopy(config.default_update_value_dict) # Only queue if processing in tiger - if rec_series['program_selection_params']['local_or_cluster'] == "cluster": - status_update = config.status_update_idx['NO_CHANGE'] + if rec_series["program_selection_params"]["local_or_cluster"] == "cluster": + status_update = config.status_update_idx["NO_CHANGE"] local_user = False - program_selection_params = rec_series['program_selection_params'] - if program_selection_params['process_cluster'] == 'spock' and is_this_spock(): + program_selection_params = rec_series["program_selection_params"] + if ( + program_selection_params["process_cluster"] == "spock" + and is_this_spock() + ): local_user = True - ssh_user = ft.cluster_vars[program_selection_params['process_cluster']]['user'] - ssh_host = ft.cluster_vars[program_selection_params['process_cluster']]['hostname'] - slurm_jobid = str(rec_series['slurm_id']) + ssh_user = ft.cluster_vars[program_selection_params["process_cluster"]][ + "user" + ] + ssh_host = ft.cluster_vars[program_selection_params["process_cluster"]][ + "hostname" + ] + slurm_jobid = str(rec_series["slurm_id"]) - status_update, message = slurmlib.check_slurm_job(ssh_user, ssh_host, slurm_jobid, local_user=local_user) + status_update, message = slurmlib.check_slurm_job( + ssh_user, ssh_host, slurm_jobid, local_user=local_user + ) # Get message from slurm status check - update_value_dict['error_info']['error_message'] = message + update_value_dict["error_info"]["error_message"] = message # This updates the value - #If job finished copy over output and/or error log - if status_update == config.status_update_idx['NEXT_STATUS'] or status_update == config.status_update_idx['ERROR_STATUS']: - - ft.transfer_log_file(rec_series['job_id'], program_selection_params, ssh_host, log_type='ERROR') - ft.transfer_log_file(rec_series['job_id'], program_selection_params, ssh_host, log_type='OUTPUT') - error_log = ft.get_error_log_str(rec_series['job_id']) + # If job finished copy over output and/or error log + if ( + status_update == config.status_update_idx["NEXT_STATUS"] + or status_update == config.status_update_idx["ERROR_STATUS"] + ): + ft.transfer_log_file( + rec_series["job_id"], + program_selection_params, + ssh_host, + log_type="ERROR", + ) + ft.transfer_log_file( + rec_series["job_id"], + program_selection_params, + ssh_host, + log_type="OUTPUT", + ) + error_log = ft.get_error_log_str(rec_series["job_id"]) # If the program errored, print the log. # Previous method of capturing the error_log was insufficient since Kilosort wrote to stderr - if status_update == config.status_update_idx['ERROR_STATUS']: - update_value_dict['error_info']['error_message'] = 'An error occured in processing (check LOG)' - update_value_dict['error_info']['error_exception'] = error_log + if status_update == config.status_update_idx["ERROR_STATUS"]: + update_value_dict["error_info"]["error_message"] = ( + "An error occured in processing (check LOG)" + ) + update_value_dict["error_info"]["error_exception"] = error_log else: - status_update = config.status_update_idx['NEXT_STATUS'] + status_update = config.status_update_idx["NEXT_STATUS"] return (status_update, update_value_dict) @@ -385,37 +490,46 @@ def populate_element(rec_series, status_series): """ update_value_dict = copy.deepcopy(config.default_update_value_dict) - if rec_series['recording_modality'] == 'electrophysiology': + if rec_series["recording_modality"] == "electrophysiology": # Add extra function to create xyz picks file - #chanmap_filename = config.default_chanmap_filename % (rec_series['job_id']) - #chanmap_file_local_path = pathlib.Path(config.chanmap_files_filepath,chanmap_filename).as_posix() - #ephys_utils.xyz_pick_file_creator.main_xyz_pick_file_function(rec_series['recording_id'], rec_series['fragment_number'], chanmap_file_local_path, rec_series['recording_process_post_path']) + # chanmap_filename = config.default_chanmap_filename % (rec_series['job_id']) + # chanmap_file_local_path = pathlib.Path(config.chanmap_files_filepath,chanmap_filename).as_posix() + # ephys_utils.xyz_pick_file_creator.main_xyz_pick_file_function(rec_series['recording_id'], rec_series['fragment_number'], chanmap_file_local_path, rec_series['recording_process_post_path']) - status_update = ep.populate_element_data(rec_series['job_id']) + status_update = ep.populate_element_data(rec_series["job_id"]) - elif rec_series['recording_modality'] == 'imaging': - status_update = ip.populate_element_data(rec_series['job_id']) + elif rec_series["recording_modality"] == "imaging": + status_update = ip.populate_element_data(rec_series["job_id"]) return (status_update, update_value_dict) @staticmethod def get_program_selection_params(modality): - ''' + """ get default processing variables (cluster, repository to process, etc) for a given modality Return: program_selection_params (pd.DataFrame): processing variables default dictionary - ''' + """ # Get df from config file - this_modality_program_selection_params =\ - config.recording_modality_df.loc[config.recording_modality_df['recording_modality'] == modality, :] + this_modality_program_selection_params = config.recording_modality_df.loc[ + config.recording_modality_df["recording_modality"] == modality, : + ] # Pack all features in a dictionary - this_modality_program_selection_params_dict = this_modality_program_selection_params.to_dict('records') + this_modality_program_selection_params_dict = ( + this_modality_program_selection_params.to_dict("records") + ) this_modality_program_selection_params_dict # Get two columns, (recording_modality & "packed" program_selection_params) - this_modality_program_selection_params = this_modality_program_selection_params.loc[:, 'recording_modality'].to_frame().copy() - this_modality_program_selection_params['program_selection_params'] = this_modality_program_selection_params_dict + this_modality_program_selection_params = ( + this_modality_program_selection_params.loc[:, "recording_modality"] + .to_frame() + .copy() + ) + this_modality_program_selection_params["program_selection_params"] = ( + this_modality_program_selection_params_dict + ) this_modality_program_selection_params @@ -423,55 +537,79 @@ def get_program_selection_params(modality): @staticmethod def get_active_process_jobs(active=True): - ''' + """ get all process jobs that have to go through some action in the pipeline Return: df_process_jobs (pd.DataFrame): all jobs that are going to be processed in the pipeline - ''' + """ if active: - status_query = 'status_processing_id > ' + str(config.JOB_STATUS_ERROR_ID) - status_query += ' and status_processing_id < ' + str(config.JOB_STATUS_PROCESSED) + status_query = "status_processing_id > " + str(config.JOB_STATUS_ERROR_ID) + status_query += " and status_processing_id < " + str( + config.JOB_STATUS_PROCESSED + ) else: - status_query = 'status_processing_id = ' + str(config.JOB_STATUS_ERROR_ID) - status_query += ' or status_processing_id = ' + str(config.JOB_STATUS_PROCESSED) - - jobs_active = (recording.Recording.proj('recording_modality', 'location', 'recording_directory') * \ - recording_process.Processing & status_query) + status_query = "status_processing_id = " + str(config.JOB_STATUS_ERROR_ID) + status_query += " or status_processing_id = " + str( + config.JOB_STATUS_PROCESSED + ) + + jobs_active = ( + recording.Recording.proj( + "recording_modality", "location", "recording_directory" + ) + * recording_process.Processing + & status_query + ) df_process_jobs = pd.DataFrame(jobs_active.fetch(as_dict=True)) if df_process_jobs.shape[0] > 0: key_list = dj_short.get_primary_key_fields(recording_process.Processing) - df_process_jobs['query_key'] = df_process_jobs.loc[:, key_list].to_dict(orient='records') + df_process_jobs["query_key"] = df_process_jobs.loc[:, key_list].to_dict( + orient="records" + ) # Get parameters for all modalities - all_modalities = df_process_jobs['recording_modality'].unique() + all_modalities = df_process_jobs["recording_modality"].unique() all_mods_df = list() # Get process df for each modality for this_modality in all_modalities: + this_mod_df = df_process_jobs.loc[ + df_process_jobs["recording_modality"] == this_modality, : + ].copy() + these_process_keys = this_mod_df["query_key"].to_list() - this_mod_df = df_process_jobs.loc[df_process_jobs['recording_modality'] == this_modality,:].copy() - these_process_keys = this_mod_df['query_key'].to_list() - - this_mod_program_selection_params = RecProcessHandler.get_program_selection_params(this_modality) - - if this_modality == 'electrophysiology': - params_df = RecProcessHandler.get_ephys_params_jobs(these_process_keys) - - if this_modality == 'imaging': - params_df = RecProcessHandler.get_imaging_params_jobs(these_process_keys) + this_mod_program_selection_params = ( + RecProcessHandler.get_program_selection_params(this_modality) + ) - this_mod_df = this_mod_df.merge(params_df, on='job_id', how='left') - this_mod_df = this_mod_df.merge(this_mod_program_selection_params, on='recording_modality', how='left') + if this_modality == "electrophysiology": + params_df = RecProcessHandler.get_ephys_params_jobs( + these_process_keys + ) + + if this_modality == "imaging": + params_df = RecProcessHandler.get_imaging_params_jobs( + these_process_keys + ) + + this_mod_df = this_mod_df.merge(params_df, on="job_id", how="left") + this_mod_df = this_mod_df.merge( + this_mod_program_selection_params, + on="recording_modality", + how="left", + ) all_mods_df.append(this_mod_df) - #Concatenate all process_jobs for each modality + # Concatenate all process_jobs for each modality df_process_jobs = all_mods_df[0].copy() for this_mod_df in all_mods_df[1:]: - df_process_jobs = pd.concat([df_process_jobs, this_mod_df], ignore_index=True) + df_process_jobs = pd.concat( + [df_process_jobs, this_mod_df], ignore_index=True + ) df_process_jobs = df_process_jobs.reset_index(drop=True) @@ -481,32 +619,60 @@ def get_active_process_jobs(active=True): @staticmethod def get_ephys_params_jobs(rec_process_keys): - ''' + """ get all parameters (precluster & cluster) for each of the recording process Join precluster param list into a list Args: rec_process_keys (dict): key to find recording_process records Return: params_df (pd.DataFrame): recording_process & params df - ''' + """ # Get cluster param sets - params_df = pd.DataFrame((ephys_pipeline.ephys_element.ClusteringParamSet.proj('params', 'clustering_method') * \ - recording_process.Processing.EphysParams.proj('paramset_idx') & rec_process_keys).fetch(as_dict=True)) - params_df = params_df.drop('paramset_idx', axis=1) - - #Insert clustering method in params itself (for BrainCogsEphysSorters) - params_df['params'] = params_df.apply(lambda x: {**x['params'], **{'clustering_method':x['clustering_method']}},axis=1) + params_df = pd.DataFrame( + ( + ephys_pipeline.ephys_element.ClusteringParamSet.proj( + "params", "clustering_method" + ) + * recording_process.Processing.EphysParams.proj("paramset_idx") + & rec_process_keys + ).fetch(as_dict=True) + ) + params_df = params_df.drop("paramset_idx", axis=1) + + # Insert clustering method in params itself (for BrainCogsEphysSorters) + params_df["params"] = params_df.apply( + lambda x: {**x["params"], **{"clustering_method": x["clustering_method"]}}, + axis=1, + ) # Get precluster param sets - preparams_df = pd.DataFrame((ephys_pipeline.ephys_element.PreClusterParamSteps * \ - utility.smart_dj_join(ephys_pipeline.ephys_element.PreClusterParamSteps.Step, ephys_pipeline.ephys_element.PreClusterParamSet.proj('precluster_method', 'params')) * - recording_process.Processing.EphysParams.proj('precluster_param_steps_id') & rec_process_keys).fetch(as_dict=True)) + preparams_df = pd.DataFrame( + ( + ephys_pipeline.ephys_element.PreClusterParamSteps + * utility.smart_dj_join( + ephys_pipeline.ephys_element.PreClusterParamSteps.Step, + ephys_pipeline.ephys_element.PreClusterParamSet.proj( + "precluster_method", "params" + ), + ) + * recording_process.Processing.EphysParams.proj( + "precluster_param_steps_id" + ) + & rec_process_keys + ).fetch(as_dict=True) + ) # Join precluster params for the same recording_process - preparams_df['preparams'] = preparams_df.apply(lambda x : {x['precluster_method']: x['params']}, axis=1) - preparams_df = preparams_df.sort_values(by=['job_id', 'step_number']) - preparams_df = preparams_df[['job_id', 'preparams']].groupby("job_id").agg(lambda x: list(x)) + preparams_df["preparams"] = preparams_df.apply( + lambda x: {x["precluster_method"]: x["params"]}, axis=1 + ) + preparams_df = preparams_df.sort_values(by=["job_id", "step_number"]) + preparams_df = ( + preparams_df[["job_id", "preparams"]] + .groupby("job_id") + .agg(lambda x: list(x)) + ) preparams_df = preparams_df.reset_index() params_df = params_df.merge(preparams_df) @@ -515,49 +681,84 @@ def get_ephys_params_jobs(rec_process_keys): @staticmethod def get_imaging_params_jobs(rec_process_keys): - ''' + """ get all parameters (precluster & cluster) for each of the recording process Join precluster param steps into a list Args: rec_process_keys (dict): key to find recording_process records Return: params_df (pd.DataFrame): recording_process & params df - ''' - - params_df = pd.DataFrame((imaging_pipeline.imaging_element.ProcessingParamSet.proj('params', 'processing_method') * \ - recording_process.Processing.ImagingParams.proj('paramset_idx') & rec_process_keys).fetch(as_dict=True)) - params_df = params_df.drop('paramset_idx', axis=1) + """ - #Insert processing_method in params itself (for BrainCogsImagingSorters) - params_df['params'] = params_df.apply(lambda x: {**x['params'], **{'processing_method':x['processing_method']}},axis=1) + params_df = pd.DataFrame( + ( + imaging_pipeline.imaging_element.ProcessingParamSet.proj( + "params", "processing_method" + ) + * recording_process.Processing.ImagingParams.proj("paramset_idx") + & rec_process_keys + ).fetch(as_dict=True) + ) + params_df = params_df.drop("paramset_idx", axis=1) + + # Insert processing_method in params itself (for BrainCogsImagingSorters) + params_df["params"] = params_df.apply( + lambda x: {**x["params"], **{"processing_method": x["processing_method"]}}, + axis=1, + ) # Get preprocess param sets - preparams_df = pd.DataFrame((imaging_pipeline.imaging_element.PreprocessParamSteps * \ - utility.smart_dj_join(imaging_pipeline.imaging_element.PreprocessParamSteps.Step, imaging_pipeline.imaging_element.PreprocessParamSet.proj('preprocess_method', 'params')) * - recording_process.Processing.ImagingParams.proj('preprocess_param_steps_id') & rec_process_keys).fetch(as_dict=True)) + preparams_df = pd.DataFrame( + ( + imaging_pipeline.imaging_element.PreprocessParamSteps + * utility.smart_dj_join( + imaging_pipeline.imaging_element.PreprocessParamSteps.Step, + imaging_pipeline.imaging_element.PreprocessParamSet.proj( + "preprocess_method", "params" + ), + ) + * recording_process.Processing.ImagingParams.proj( + "preprocess_param_steps_id" + ) + & rec_process_keys + ).fetch(as_dict=True) + ) - #If there is no preprocess steps for this jobs fill with empty values + # If there is no preprocess steps for this jobs fill with empty values if preparams_df.shape[0] > 0: - # Join precluster params for the same recording_process - preparams_df['preparams'] = preparams_df.apply(lambda x : {x['preprocess_method']: x['params']}, axis=1) - preparams_df = preparams_df.sort_values(by=['job_id', 'step_number']) - preparams_df = preparams_df[['job_id', 'preparams']].groupby("job_id").agg(lambda x: list(x)) + preparams_df["preparams"] = preparams_df.apply( + lambda x: {x["preprocess_method"]: x["params"]}, axis=1 + ) + preparams_df = preparams_df.sort_values(by=["job_id", "step_number"]) + preparams_df = ( + preparams_df[["job_id", "preparams"]] + .groupby("job_id") + .agg(lambda x: list(x)) + ) preparams_df = preparams_df.reset_index() else: - preparams_df = pd.DataFrame((imaging_pipeline.imaging_element.PreprocessParamSteps * \ - recording_process.Processing.ImagingParams.proj('preprocess_param_steps_id') & rec_process_keys).fetch(as_dict=True)) - preparams_df['preparams'] = None - - preparams_df = preparams_df[['job_id', 'preparams']] + preparams_df = pd.DataFrame( + ( + imaging_pipeline.imaging_element.PreprocessParamSteps + * recording_process.Processing.ImagingParams.proj( + "preprocess_param_steps_id" + ) + & rec_process_keys + ).fetch(as_dict=True) + ) + preparams_df["preparams"] = None + + preparams_df = preparams_df[["job_id", "preparams"]] params_df = params_df.merge(preparams_df) return params_df - @staticmethod - def update_status_pipeline(recording_process_key_dict, status, update_field=None, update_value=None): + def update_status_pipeline( + recording_process_key_dict, status, update_field=None, update_value=None + ): """ Update recording_process.Processing table status and optional task field Args: @@ -573,10 +774,9 @@ def update_status_pipeline(recording_process_key_dict, status, update_field=None recording_process.Processing.update1(update_task_id_dict) update_status_dict = recording_process_key_dict.copy() - update_status_dict['status_processing_id'] = status + update_status_dict["status_processing_id"] = status recording_process.Processing.update1(update_status_dict) - @staticmethod def update_job_id_log(job_id, current_status, next_status, error_info_dict): """ @@ -588,26 +788,33 @@ def update_job_id_log(job_id, current_status, next_status, error_info_dict): now = datetime.now() date_time = now.strftime("%Y-%m-%d %H:%M:%S") - print('error_info_dict', error_info_dict) + print("error_info_dict", error_info_dict) key = dict() - key['job_id'] = job_id - key['status_processing_id_old'] = current_status - key['status_processing_id_new'] = next_status - key['status_timestamp'] = date_time - - if error_info_dict['error_message'] is not None and len(error_info_dict['error_message']) >= 256: - error_info_dict['error_message'] =error_info_dict['error_message'][:255] - - if error_info_dict['error_exception'] is not None and len(error_info_dict['error_exception']) >= 4096: - error_info_dict['error_exception'] =error_info_dict['error_exception'][:4095] - - key['error_exception'] = error_info_dict['error_exception'] - key['error_message'] = error_info_dict['error_message'] + key["job_id"] = job_id + key["status_processing_id_old"] = current_status + key["status_processing_id_new"] = next_status + key["status_timestamp"] = date_time + + if ( + error_info_dict["error_message"] is not None + and len(error_info_dict["error_message"]) >= 256 + ): + error_info_dict["error_message"] = error_info_dict["error_message"][:255] + + if ( + error_info_dict["error_exception"] is not None + and len(error_info_dict["error_exception"]) >= 4096 + ): + error_info_dict["error_exception"] = error_info_dict["error_exception"][ + :4095 + ] + + key["error_exception"] = error_info_dict["error_exception"] + key["error_message"] = error_info_dict["error_message"] recording_process.LogStatus.insert1(key) - @staticmethod @recording_handler.exception_handler def check_job_process_deletion(): @@ -619,107 +826,188 @@ def check_job_process_deletion(): # Get jobs with error status or finished df_inactive_jobs = RecProcessHandler.get_active_process_jobs(active=False) - #If no inactive job, the end + # If no inactive job, the end if df_inactive_jobs.shape[0] == 0: return - jobs_ids = df_inactive_jobs['job_id'].to_frame().to_dict('records') + jobs_ids = df_inactive_jobs["job_id"].to_frame().to_dict("records") - #Get latest time they changed status + # Get latest time they changed status now = datetime.now() - timestamp_jobs = pd.DataFrame((recording_process.Processing & jobs_ids).aggr(recording_process.LogStatus, max_date="max(status_timestamp)").fetch(as_dict=True)) - df_inactive_jobs = df_inactive_jobs.merge(timestamp_jobs, on='job_id') + timestamp_jobs = pd.DataFrame( + (recording_process.Processing & jobs_ids) + .aggr(recording_process.LogStatus, max_date="max(status_timestamp)") + .fetch(as_dict=True) + ) + df_inactive_jobs = df_inactive_jobs.merge(timestamp_jobs, on="job_id") # Get jobs that were processed in cluster - df_inactive_jobs['local_or_cluster'] = df_inactive_jobs['program_selection_params'].map(lambda x: x['local_or_cluster']) - df_inactive_jobs['process_cluster'] = df_inactive_jobs['program_selection_params'].map(lambda x: x['process_cluster']) - - #Update all local jobs - df_not_cluster = df_inactive_jobs.loc[df_inactive_jobs['local_or_cluster'] == 'local'].reset_index(drop=True) + df_inactive_jobs["local_or_cluster"] = df_inactive_jobs[ + "program_selection_params" + ].map(lambda x: x["local_or_cluster"]) + df_inactive_jobs["process_cluster"] = df_inactive_jobs[ + "program_selection_params" + ].map(lambda x: x["process_cluster"]) + + # Update all local jobs + df_not_cluster = df_inactive_jobs.loc[ + df_inactive_jobs["local_or_cluster"] == "local" + ].reset_index(drop=True) if df_not_cluster.shape[0] > 0: # Update local jobs post-processed jobs - df_not_cluster_post_processed = df_not_cluster.loc[df_not_cluster['status_processing_id'] == config.JOB_STATUS_PROCESSED, :] - df_not_cluster_post_processed = df_not_cluster_post_processed.reset_index(drop=True) + df_not_cluster_post_processed = df_not_cluster.loc[ + df_not_cluster["status_processing_id"] == config.JOB_STATUS_PROCESSED, : + ] + df_not_cluster_post_processed = df_not_cluster_post_processed.reset_index( + drop=True + ) for i in range(df_not_cluster_post_processed.shape[0]): - RecProcessHandler.update_status_pipeline(df_not_cluster_post_processed.loc[i, 'query_key'], config.JOB_STATUS_POST_PROCESSED) - RecProcessHandler.update_job_id_log(df_not_cluster_post_processed.loc[i, 'job_id'], config.JOB_STATUS_PROCESSED, config.JOB_STATUS_POST_PROCESSED, update_value_dict['error_info']) + RecProcessHandler.update_status_pipeline( + df_not_cluster_post_processed.loc[i, "query_key"], + config.JOB_STATUS_POST_PROCESSED, + ) + RecProcessHandler.update_job_id_log( + df_not_cluster_post_processed.loc[i, "job_id"], + config.JOB_STATUS_PROCESSED, + config.JOB_STATUS_POST_PROCESSED, + update_value_dict["error_info"], + ) # Update also local jobs post-error jobs - df_not_cluster_post_error = df_not_cluster.loc[df_not_cluster['status_processing_id'] == config.JOB_STATUS_ERROR_ID, :] + df_not_cluster_post_error = df_not_cluster.loc[ + df_not_cluster["status_processing_id"] == config.JOB_STATUS_ERROR_ID, : + ] df_not_cluster_post_error = df_not_cluster_post_error.reset_index(drop=True) for i in range(df_not_cluster_post_error.shape[0]): - RecProcessHandler.update_status_pipeline(df_not_cluster_post_error.loc[i, 'query_key'], config.JOB_STATUS_ERROR_DELETED) - RecProcessHandler.update_job_id_log(df_not_cluster_post_error.loc[i, 'job_id'], config.JOB_STATUS_ERROR_ID, config.JOB_STATUS_ERROR_DELETED, update_value_dict['error_info']) + RecProcessHandler.update_status_pipeline( + df_not_cluster_post_error.loc[i, "query_key"], + config.JOB_STATUS_ERROR_DELETED, + ) + RecProcessHandler.update_job_id_log( + df_not_cluster_post_error.loc[i, "job_id"], + config.JOB_STATUS_ERROR_ID, + config.JOB_STATUS_ERROR_DELETED, + update_value_dict["error_info"], + ) - #If no inactive job, after locals the end - df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs['local_or_cluster'] == 'cluster'].reset_index(drop=True) + # If no inactive job, after locals the end + df_inactive_jobs = df_inactive_jobs.loc[ + df_inactive_jobs["local_or_cluster"] == "cluster" + ].reset_index(drop=True) if df_inactive_jobs.shape[0] == 0: return # Get jobs to delete (finished or with error >= 7 days) - df_inactive_jobs['days_from_last_update'] = (now - df_inactive_jobs['max_date']).dt.days - df_inactive_jobs['ready_delete'] = 1 - df_inactive_jobs.loc[(df_inactive_jobs['status_processing_id'] == config.JOB_STATUS_ERROR_ID) & (df_inactive_jobs['days_from_last_update'] < 7), 'ready_delete'] = 0 - df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs['ready_delete'] == 1,:] - - #If no inactive job, after ready delete data return now + df_inactive_jobs["days_from_last_update"] = ( + now - df_inactive_jobs["max_date"] + ).dt.days + df_inactive_jobs["ready_delete"] = 1 + df_inactive_jobs.loc[ + (df_inactive_jobs["status_processing_id"] == config.JOB_STATUS_ERROR_ID) + & (df_inactive_jobs["days_from_last_update"] < 7), + "ready_delete", + ] = 0 + df_inactive_jobs = df_inactive_jobs.loc[ + df_inactive_jobs["ready_delete"] == 1, : + ] + + # If no inactive job, after ready delete data return now if df_inactive_jobs.shape[0] == 0: return # Check if raw directories are present in cluster - df_inactive_jobs['raw_dir'] = df_inactive_jobs.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_pre_path'],\ - x['process_cluster'], x['recording_modality'], type_dir='raw'), axis=1) - df_inactive_jobs['raw_dir'] = df_inactive_jobs['raw_dir'].astype(str) - df_inactive_jobs['raw_dir_deleted'] = 0 - df_inactive_jobs.loc[df_inactive_jobs['raw_dir'] == "0", 'raw_dir_deleted'] = 1 + df_inactive_jobs["raw_dir"] = df_inactive_jobs.apply( + lambda x: ft.check_directory_exists_cluster( + x["recording_process_pre_path"], + x["process_cluster"], + x["recording_modality"], + type_dir="raw", + ), + axis=1, + ) + df_inactive_jobs["raw_dir"] = df_inactive_jobs["raw_dir"].astype(str) + df_inactive_jobs["raw_dir_deleted"] = 0 + df_inactive_jobs.loc[df_inactive_jobs["raw_dir"] == "0", "raw_dir_deleted"] = 1 # Check if processed directories are present in cluster - df_inactive_jobs['processed_dir'] = df_inactive_jobs.apply(lambda x: ft.check_directory_exists_cluster(x['recording_process_post_path'],\ - x['process_cluster'], x['recording_modality'], type_dir='processed'), axis=1) - df_inactive_jobs['processed_dir'] = df_inactive_jobs['processed_dir'].astype(str) - df_inactive_jobs['processed_dir_deleted'] = 0 - df_inactive_jobs.loc[df_inactive_jobs['processed_dir'] == "0", 'processed_dir_deleted'] = 1 + df_inactive_jobs["processed_dir"] = df_inactive_jobs.apply( + lambda x: ft.check_directory_exists_cluster( + x["recording_process_post_path"], + x["process_cluster"], + x["recording_modality"], + type_dir="processed", + ), + axis=1, + ) + df_inactive_jobs["processed_dir"] = df_inactive_jobs["processed_dir"].astype( + str + ) + df_inactive_jobs["processed_dir_deleted"] = 0 + df_inactive_jobs.loc[ + df_inactive_jobs["processed_dir"] == "0", "processed_dir_deleted" + ] = 1 df_inactive_jobs = df_inactive_jobs.reset_index(drop=True) - #Delete raw directories tiger + # Delete raw directories tiger for i in range(df_inactive_jobs.shape[0]): - modality = df_inactive_jobs.loc[i, 'recording_modality'] - dir_delete = df_inactive_jobs.loc[i, 'recording_process_pre_path'] - to_delete = df_inactive_jobs.loc[i, 'raw_dir'] + modality = df_inactive_jobs.loc[i, "recording_modality"] + dir_delete = df_inactive_jobs.loc[i, "recording_process_pre_path"] + to_delete = df_inactive_jobs.loc[i, "raw_dir"] if str(to_delete) != "0": status = ft.delete_directory_tiger_globus(modality, dir_delete) - if status == config.system_process['SUCCESS']: - df_inactive_jobs.loc[i, 'raw_dir_deleted'] = 1 + if status == config.system_process["SUCCESS"]: + df_inactive_jobs.loc[i, "raw_dir_deleted"] = 1 - #Delete processed directories tiger + # Delete processed directories tiger for i in range(df_inactive_jobs.shape[0]): - cluster = df_inactive_jobs.loc[i, 'process_cluster'] - dir_delete = df_inactive_jobs.loc[i, 'processed_dir'] + cluster = df_inactive_jobs.loc[i, "process_cluster"] + dir_delete = df_inactive_jobs.loc[i, "processed_dir"] if str(dir_delete) != "0": status = ft.delete_directory_cluster(dir_delete, cluster) - if status == config.system_process['SUCCESS']: - df_inactive_jobs.loc[i, 'processed_dir_deleted'] = 1 + if status == config.system_process["SUCCESS"]: + df_inactive_jobs.loc[i, "processed_dir_deleted"] = 1 - #Delete empty directories in clusters - clusters = df_inactive_jobs['process_cluster'].unique().tolist() + # Delete empty directories in clusters + clusters = df_inactive_jobs["process_cluster"].unique().tolist() for this_cluster in clusters: ft.delete_empty_data_directory_cluster(this_cluster, type="raw") ft.delete_empty_data_directory_cluster(this_cluster, type="processed") - # Update status for post-processed jobs - df_post_processed = df_inactive_jobs.loc[(df_inactive_jobs['raw_dir_deleted'] == 1) & (df_inactive_jobs['processed_dir_deleted'] == 1) & (df_inactive_jobs['status_processing_id'] == config.JOB_STATUS_PROCESSED)] + df_post_processed = df_inactive_jobs.loc[ + (df_inactive_jobs["raw_dir_deleted"] == 1) + & (df_inactive_jobs["processed_dir_deleted"] == 1) + & (df_inactive_jobs["status_processing_id"] == config.JOB_STATUS_PROCESSED) + ] df_post_processed = df_post_processed.reset_index(drop=True) for i in range(df_post_processed.shape[0]): - RecProcessHandler.update_status_pipeline(df_post_processed.loc[i, 'query_key'], config.JOB_STATUS_POST_PROCESSED) - RecProcessHandler.update_job_id_log(df_post_processed.loc[i, 'job_id'], config.JOB_STATUS_PROCESSED, config.JOB_STATUS_POST_PROCESSED, update_value_dict['error_info']) + RecProcessHandler.update_status_pipeline( + df_post_processed.loc[i, "query_key"], config.JOB_STATUS_POST_PROCESSED + ) + RecProcessHandler.update_job_id_log( + df_post_processed.loc[i, "job_id"], + config.JOB_STATUS_PROCESSED, + config.JOB_STATUS_POST_PROCESSED, + update_value_dict["error_info"], + ) # Update status for post-error jobs - df_post_error = df_inactive_jobs.loc[(df_inactive_jobs['raw_dir_deleted'] == 1) & (df_inactive_jobs['processed_dir_deleted'] == 1) & (df_inactive_jobs['status_processing_id'] == config.JOB_STATUS_ERROR_ID)] + df_post_error = df_inactive_jobs.loc[ + (df_inactive_jobs["raw_dir_deleted"] == 1) + & (df_inactive_jobs["processed_dir_deleted"] == 1) + & (df_inactive_jobs["status_processing_id"] == config.JOB_STATUS_ERROR_ID) + ] df_post_error = df_post_error.reset_index(drop=True) for i in range(df_post_error.shape[0]): - RecProcessHandler.update_status_pipeline(df_post_error.loc[i, 'query_key'], config.JOB_STATUS_ERROR_DELETED) - RecProcessHandler.update_job_id_log(df_post_error.loc[i, 'job_id'], config.JOB_STATUS_ERROR_ID, config.JOB_STATUS_ERROR_DELETED, update_value_dict['error_info']) + RecProcessHandler.update_status_pipeline( + df_post_error.loc[i, "query_key"], config.JOB_STATUS_ERROR_DELETED + ) + RecProcessHandler.update_job_id_log( + df_post_error.loc[i, "job_id"], + config.JOB_STATUS_ERROR_ID, + config.JOB_STATUS_ERROR_DELETED, + update_value_dict["error_info"], + ) return @@ -738,4 +1026,3 @@ def filter_session_status(df_rec_process, status): df_rec_process_status = df_rec_process.loc[df_sessions['status_processing_id'] == status, :] df_rec_process_status = df_rec_process_status.reset_index(drop=True) ''' - diff --git a/u19_pipeline/automatic_job/slurm_creator.py b/u19_pipeline/automatic_job/slurm_creator.py index 2dc81797..c8a88300 100644 --- a/u19_pipeline/automatic_job/slurm_creator.py +++ b/u19_pipeline/automatic_job/slurm_creator.py @@ -1,5 +1,3 @@ - - import copy import pathlib import re @@ -12,52 +10,61 @@ # Functions to create slurm jobs -slurms_filepath = 'u19_pipeline/automatic_job/SlurmFiles' -default_slurm_filename = 'slurm_real.slurm' +slurms_filepath = "u19_pipeline/automatic_job/SlurmFiles" +default_slurm_filename = "slurm_real.slurm" default_process_script_path = "scripts/automate_imaging_element.py" -default_preprocessing_tool = 'kilosort2' -default_matlab_ver = 'R2020b' +default_preprocessing_tool = "kilosort2" +default_matlab_ver = "R2020b" + def generate_slurm_file(job_id, program_selection_params): - ''' + """ Generate and send slurm file to be queued in processing cluster - ''' + """ - #Get all associated directories given the selected processing cluster - cluster_vars = ft.get_cluster_vars(program_selection_params['process_cluster']) + # Get all associated directories given the selected processing cluster + cluster_vars = ft.get_cluster_vars(program_selection_params["process_cluster"]) # Start with default values - slurm_dict = copy.deepcopy(cluster_vars['slurm_default']) - label_rec_process = 'job_id_'+str(job_id) - slurm_dict['job-name'] = label_rec_process + slurm_dict = copy.deepcopy(cluster_vars["slurm_default"]) + label_rec_process = "job_id_" + str(job_id) + slurm_dict["job-name"] = label_rec_process - slurm_dict['output'] = str(pathlib.Path(cluster_vars['log_files_dir'],label_rec_process+ '.log')) - slurm_dict['error'] = str(pathlib.Path(cluster_vars['error_files_dir'],label_rec_process+ '.log')) + slurm_dict["output"] = str( + pathlib.Path(cluster_vars["log_files_dir"], label_rec_process + ".log") + ) + slurm_dict["error"] = str( + pathlib.Path(cluster_vars["error_files_dir"], label_rec_process + ".log") + ) - print('slurm_dict', slurm_dict) + print("slurm_dict", slurm_dict) - if program_selection_params['process_cluster'] == 'spock': + if program_selection_params["process_cluster"] == "spock": slurm_text = generate_slurm_spock(slurm_dict) else: slurm_text = generate_slurm_tiger(slurm_dict) slurm_file_name = default_slurm_filename - slurm_file_local_path = str(pathlib.Path(slurms_filepath,slurm_file_name)) + slurm_file_local_path = str(pathlib.Path(slurms_filepath, slurm_file_name)) print(slurm_file_local_path) - print(cluster_vars['slurm_files_dir']) + print(cluster_vars["slurm_files_dir"]) print(slurm_file_name) write_file(slurm_file_local_path, slurm_text) - if program_selection_params['process_cluster'] == 'spock' and is_this_spock(): - status = config.system_process['SUCCESS'] + if program_selection_params["process_cluster"] == "spock" and is_this_spock(): + status = config.system_process["SUCCESS"] slurm_destination = slurm_file_local_path else: - slurm_destination = pathlib.Path(cluster_vars['slurm_files_dir'], slurm_file_name).as_posix() - status = transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars) + slurm_destination = pathlib.Path( + cluster_vars["slurm_files_dir"], slurm_file_name + ).as_posix() + status = transfer_slurm_file( + slurm_file_local_path, slurm_destination, cluster_vars + ) print(status) print(slurm_destination) @@ -66,49 +73,66 @@ def generate_slurm_file(job_id, program_selection_params): return status, slurm_destination -def queue_slurm_file(job_id, program_selection_params, raw_directory, proc_directory, modality, slurm_location): +def queue_slurm_file( + job_id, + program_selection_params, + raw_directory, + proc_directory, + modality, + slurm_location, +): id_slurm_job = -1 job_id = str(job_id) - #Get all associated variables given the selected processing cluster - cluster_vars = ft.get_cluster_vars(program_selection_params['process_cluster']) - - print('queue_slurm_file **********************************') - - - processing_repository = program_selection_params['process_repository'] - repository_dir = pathlib.Path(cluster_vars[modality+'_process_dir'],processing_repository).as_posix() - - command = ['ssh', cluster_vars['user']+"@"+cluster_vars['hostname'], 'sbatch', - "--export=recording_process_id="+job_id+ - ",raw_data_directory='"+raw_directory+ - "',processed_data_directory='"+proc_directory+ - "',repository_dir='"+repository_dir+ - "',process_script_path='"+program_selection_params['process_script']+"'" - , slurm_location + # Get all associated variables given the selected processing cluster + cluster_vars = ft.get_cluster_vars(program_selection_params["process_cluster"]) + + print("queue_slurm_file **********************************") + + processing_repository = program_selection_params["process_repository"] + repository_dir = pathlib.Path( + cluster_vars[modality + "_process_dir"], processing_repository + ).as_posix() + + command = [ + "ssh", + cluster_vars["user"] + "@" + cluster_vars["hostname"], + "sbatch", + "--export=recording_process_id=" + + job_id + + ",raw_data_directory='" + + raw_directory + + "',processed_data_directory='" + + proc_directory + + "',repository_dir='" + + repository_dir + + "',process_script_path='" + + program_selection_params["process_script"] + + "'", + slurm_location, ] - if program_selection_params['process_cluster'] == 'spock' and is_this_spock(): + if program_selection_params["process_cluster"] == "spock" and is_this_spock(): command = command[2:] print(command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - #p = os.popen(command_new).read() + # p = os.popen(command_new).read() p.wait() stdout, stderr = p.communicate() print(stdout) print(stderr) - if p.returncode == config.system_process['SUCCESS']: - error_message = '' - batch_job_sentence = stdout.decode('UTF-8') - print('batch_job_sentence', batch_job_sentence) - id_slurm_job = batch_job_sentence.replace("Submitted batch job ","") - id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job) + if p.returncode == config.system_process["SUCCESS"]: + error_message = "" + batch_job_sentence = stdout.decode("UTF-8") + print("batch_job_sentence", batch_job_sentence) + id_slurm_job = batch_job_sentence.replace("Submitted batch job ", "") + id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job) else: - error_message = stderr.decode('UTF-8') + error_message = stderr.decode("UTF-8") return p.returncode, id_slurm_job, error_message @@ -116,41 +140,48 @@ def queue_slurm_file(job_id, program_selection_params, raw_directory, proc_direc def check_slurm_job(ssh_user, host, jobid, local_user=False): if local_user: - command = ['sacct', '--job', jobid, '--format=state'] + command = ["sacct", "--job", jobid, "--format=state"] else: - command = ['ssh', ssh_user+'@'+host, 'sacct', '--job', jobid, '--format=state'] + command = [ + "ssh", + ssh_user + "@" + host, + "sacct", + "--job", + jobid, + "--format=state", + ] p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.wait() stdout, stderr = p.communicate() - stdout = stdout.decode('UTF-8') + stdout = stdout.decode("UTF-8") - print('p.returncode !!!!!!!!!!!!', p.returncode) - print("config.system_process['SUCCESS']", config.system_process['SUCCESS']) + print("p.returncode !!!!!!!!!!!!", p.returncode) + print("config.system_process['SUCCESS']", config.system_process["SUCCESS"]) - if p.returncode == config.system_process['SUCCESS']: + if p.returncode == config.system_process["SUCCESS"]: state_slurm_job = stdout.split("\n")[2].strip() - state_pipeline = config.slurm_states[state_slurm_job]['pipeline_status'] - error_message = config.slurm_states[state_slurm_job]['message'] + state_pipeline = config.slurm_states[state_slurm_job]["pipeline_status"] + error_message = config.slurm_states[state_slurm_job]["message"] - print('state_pipeline ....', state_pipeline) - print('error_message', error_message) + print("state_pipeline ....", state_pipeline) + print("error_message", error_message) else: - state_pipeline = config.status_update_idx['ERROR_STATUS'] - error_message = 'Failed to retrieve slurm job status' + state_pipeline = config.status_update_idx["ERROR_STATUS"] + error_message = "Failed to retrieve slurm job status" return state_pipeline, error_message def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars): - ''' + """ Create scp command from cluster directories and local slurm file - ''' + """ - user_host = cluster_vars['user']+'@'+cluster_vars['hostname'] - slurm_destination = user_host+':'+slurm_destination + user_host = cluster_vars["user"] + "@" + cluster_vars["hostname"] + slurm_destination = user_host + ":" + slurm_destination status = ft.scp_file_transfer(slurm_file_local_path, slurm_destination) return status @@ -158,23 +189,31 @@ def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars): def create_slurm_params_file(slurm_dict): - text_dict = '' + text_dict = "" for slurm_param in slurm_dict.keys(): - if isinstance(slurm_dict[slurm_param], list): for list_param in slurm_dict[slurm_param]: - text_dict += '#SBATCH --' + str(slurm_param) + '=' + str(list_param) + '\n' + text_dict += ( + "#SBATCH --" + str(slurm_param) + "=" + str(list_param) + "\n" + ) else: - text_dict += '#SBATCH --' + str(slurm_param) + '=' + str(slurm_dict[slurm_param]) + '\n' + text_dict += ( + "#SBATCH --" + + str(slurm_param) + + "=" + + str(slurm_dict[slurm_param]) + + "\n" + ) return text_dict + def generate_slurm_spock(slurm_dict): - slurm_text = '#!/bin/bash\n' - #slurm_text += module_defininition_text() + slurm_text = "#!/bin/bash\n" + # slurm_text += module_defininition_text() slurm_text += create_slurm_params_file(slurm_dict) - slurm_text += ''' + slurm_text += """ echo "SLURM_JOB_ID: ${SLURM_JOB_ID}" echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}" echo "RECORDING_PROCESS_ID: ${recording_process_id}" @@ -192,13 +231,14 @@ def generate_slurm_spock(slurm_dict): cd ${repository_dir} python -u ${process_script_path} #python ${process_script_path} ${recording_process_id} - ''' + """ return slurm_text + def module_defininition_text(): - return ''' + return """ module () { local _mlredir=1; @@ -225,14 +265,15 @@ def module_defininition_text(): _module_raw "$@" 2>&1; fi } - ''' + """ + def generate_slurm_spockmk2_ephys(slurm_dict): - slurm_text = '#!/bin/bash\n' - slurm_text += 'source ~/.bashrc' + slurm_text = "#!/bin/bash\n" + slurm_text += "source ~/.bashrc" slurm_text += create_slurm_params_file(slurm_dict) - slurm_text += ''' + slurm_text += """ echo "SLURM_JOB_ID: ${SLURM_JOB_ID}" echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}" echo "RECORDING_PROCESS_ID: ${recording_process_id}" @@ -249,16 +290,16 @@ def generate_slurm_spockmk2_ephys(slurm_dict): cd ${repository_dir} python -u ${process_script_path} #python ${process_script_path} ${recording_process_id} - ''' + """ return slurm_text def generate_slurm_tiger(slurm_dict): - slurm_text = '#!/bin/bash\n' + slurm_text = "#!/bin/bash\n" slurm_text += create_slurm_params_file(slurm_dict) - slurm_text += ''' + slurm_text += """ echo "SLURM_JOB_ID: ${SLURM_JOB_ID}" echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}" echo "RECORDING_PROCESS_ID: ${recording_process_id}" @@ -274,16 +315,16 @@ def generate_slurm_tiger(slurm_dict): cd ${repository_dir} python -u ${process_script_path} - ''' + """ return slurm_text def generate_slurm_dlc(slurm_dict): - slurm_text = '#!/bin/bash\n' + slurm_text = "#!/bin/bash\n" slurm_text += create_slurm_params_file(slurm_dict) - slurm_text += ''' + slurm_text += """ echo "SLURM_JOB_ID: ${SLURM_JOB_ID}" echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}" echo "RAW_DATA_DIRECTORY: ${raw_data_directory}" @@ -296,16 +337,16 @@ def generate_slurm_dlc(slurm_dict): conda activate /usr/people/alvaros/.conda/envs/u19_datajoint_py39_env python -u ${process_script_path} ${raw_data_directory} ${model_path} ${processed_data_directory} - ''' + """ return slurm_text def generate_slurm_dlc2(slurm_dict): - slurm_text = '#!/bin/bash\n' + slurm_text = "#!/bin/bash\n" slurm_text += create_slurm_params_file(slurm_dict) - slurm_text += ''' + slurm_text += """ echo "SLURM_JOB_ID: ${SLURM_JOB_ID}" echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}" echo "RAW_DATA_DIRECTORY: ${raw_data_directory}" @@ -318,7 +359,6 @@ def generate_slurm_dlc2(slurm_dict): conda activate /home/alvaros/.conda/envs/u19_datajoint_py39_env python -u ${process_script_path} ${raw_data_directory} ${model_path} ${processed_data_directory} - ''' + """ return slurm_text - diff --git a/u19_pipeline/automatic_job/tigress2pni.py b/u19_pipeline/automatic_job/tigress2pni.py index 92130afe..4ebbae98 100755 --- a/u19_pipeline/automatic_job/tigress2pni.py +++ b/u19_pipeline/automatic_job/tigress2pni.py @@ -29,22 +29,22 @@ from globus_sdk.services.transfer.errors import TransferAPIError # Princeton TIGRESS -SOURCE_ENDPOINT = 'a9df83d2-42f0-11e6-80cf-22000b1701d1' +SOURCE_ENDPOINT = "a9df83d2-42f0-11e6-80cf-22000b1701d1" # SOURCE_ENDPOINT = 'ef3a4e74-e742-11ec-9912-3b4cfda38030' -DESTINATION_PATH = '/mnt/cup/braininit/Shared/TestGlobusTransfer' -SOURCE_PATH = '/Data/Processed/imaging/' -SOURCE_PATH = '/tiger/scratch/gpfs/BRAINCOGS/Data/Processed/imaging/' +DESTINATION_PATH = "/mnt/cup/braininit/Shared/TestGlobusTransfer" +SOURCE_PATH = "/Data/Processed/imaging/" +SOURCE_PATH = "/tiger/scratch/gpfs/BRAINCOGS/Data/Processed/imaging/" # Princeton PNI -DESTINATION_ENDPOINT = '6ce834d6-ff8a-11e6-bad1-22000b9a448b' +DESTINATION_ENDPOINT = "6ce834d6-ff8a-11e6-bad1-22000b9a448b" # Copy data off of the endpoint share # SOURCE_PATH = '/tigress/bdsinger/data/' # Destination Path -- The directory will be created if it doesn't exist # DESTINATION_PATH = '/mnt/cup/people/bdsinger/data/' -TRANSFER_LABEL = 'FolderSyncTest' +TRANSFER_LABEL = "FolderSyncTest" # You will need to register a *Native App* at https://developers.globus.org/ # Your app should include the following: @@ -53,24 +53,23 @@ # - "Native App" should be checked # For more information: # https://docs.globus.org/api/auth/developer-guide/#register-app -CLIENT_ID = '71286d4e-778d-45f7-9a30-813305f9e1f3' -DATA_FILE = 'transfer-data.json' -REDIRECT_URI = 'https://auth.globus.org/v2/web/auth-code' -SCOPES = ('openid email profile ' - 'urn:globus:auth:scope:transfer.api.globus.org:all') +CLIENT_ID = "71286d4e-778d-45f7-9a30-813305f9e1f3" +DATA_FILE = "transfer-data.json" +REDIRECT_URI = "https://auth.globus.org/v2/web/auth-code" +SCOPES = "openid email profile urn:globus:auth:scope:transfer.api.globus.org:all" -APP_NAME = 'FolderSyncTest' +APP_NAME = "FolderSyncTest" # ONLY run new tasks if there was a previous task and it exited with one of the # following statuses. This is ignored if there was no previous task. # The previous task is queried from the DATA_FILE -PREVIOUS_TASK_RUN_CASES = ['SUCCEEDED', 'FAILED'] +PREVIOUS_TASK_RUN_CASES = ["SUCCEEDED", "FAILED"] # Create the destination folder if it does not already exist CREATE_DESTINATION_FOLDER = True -get_input = getattr(__builtins__, 'raw_input', input) +get_input = getattr(__builtins__, "raw_input", input) def load_data_from_file(filepath): @@ -91,17 +90,18 @@ def save_data_to_file(filepath, key, data): store = {} if len(store) > 0: store[key] = data - with open(filepath, 'w') as f: + with open(filepath, "w") as f: json.dump(store, f) def setup_transfer_client(transfer_tokens): authorizer = RefreshTokenAuthorizer( - transfer_tokens['refresh_token'], + transfer_tokens["refresh_token"], NativeAppAuthClient(client_id=CLIENT_ID), - access_token=transfer_tokens['access_token'], - expires_at=transfer_tokens['expires_at_seconds']) + access_token=transfer_tokens["access_token"], + expires_at=transfer_tokens["expires_at_seconds"], + ) transfer_client = TransferClient(authorizer=authorizer) @@ -110,9 +110,11 @@ def setup_transfer_client(transfer_tokens): r = transfer_client.endpoint_autoactivate(DESTINATION_ENDPOINT) except GlobusAPIError as ex: if ex.http_status == 401: - sys.exit('Refresh token has expired. ' - 'Please delete the `tokens` object from ' - f'{DATA_FILE} and try again.') + sys.exit( + "Refresh token has expired. " + "Please delete the `tokens` object from " + f"{DATA_FILE} and try again." + ) else: raise ex return transfer_client @@ -134,9 +136,9 @@ def create_destination_directory(transfer_client, dest_ep, dest_path): except TransferAPIError: try: transfer_client.operation_mkdir(dest_ep, dest_path) - print(f'Created directory: {dest_path}') + print(f"Created directory: {dest_path}") except TransferAPIError as tapie: - print(f'Failed to start transfer: {tapie.message}') + print(f"Failed to start transfer: {tapie.message}") sys.exit(1) @@ -153,39 +155,40 @@ def main(): if not tokens: # if we need to get tokens, start the Native App authentication process # need to specify that we want refresh tokens - print('login in') + print("login in") tokens = client.login(requested_scopes=SCOPES, refresh_tokens=True) try: client.save_tokens(tokens) except: pass - transfer = setup_transfer_client(tokens['transfer.api.globus.org']) + transfer = setup_transfer_client(tokens["transfer.api.globus.org"]) try: data = load_data_from_file(DATA_FILE) if len(data) > 0: - task_data = data['task'] - task = transfer.get_task(task_data['task_id']) - if task['status'] not in PREVIOUS_TASK_RUN_CASES: - print('The last transfer status is {}, skipping run...'.format( - task['status'] - )) + task_data = data["task"] + task = transfer.get_task(task_data["task_id"]) + if task["status"] not in PREVIOUS_TASK_RUN_CASES: + print( + "The last transfer status is {}, skipping run...".format( + task["status"] + ) + ) sys.exit(1) except KeyError: # Ignore if there is no previous task pass if len(sys.argv) < 2: - print(f'Usage: {sys.argv[0]} tigress_dir pni_dir') + print(f"Usage: {sys.argv[0]} tigress_dir pni_dir") sys.exit(1) SOURCE_PATH = sys.argv[1] DESTINATION_PATH = sys.argv[2] check_endpoint_path(transfer, SOURCE_ENDPOINT, SOURCE_PATH) if CREATE_DESTINATION_FOLDER: - create_destination_directory(transfer, DESTINATION_ENDPOINT, - DESTINATION_PATH) + create_destination_directory(transfer, DESTINATION_ENDPOINT, DESTINATION_PATH) else: check_endpoint_path(transfer, DESTINATION_ENDPOINT, DESTINATION_PATH) @@ -194,22 +197,25 @@ def main(): SOURCE_ENDPOINT, DESTINATION_ENDPOINT, label=TRANSFER_LABEL, - sync_level="checksum" + sync_level="checksum", ) tdata.add_item(SOURCE_PATH, DESTINATION_PATH, recursive=True) task = transfer.submit_transfer(tdata) - save_data_to_file(DATA_FILE, 'task', task.data) - print(f'Transfer has been started from\n {SOURCE_ENDPOINT}:{SOURCE_PATH}\nto\n {DESTINATION_ENDPOINT}:{DESTINATION_PATH}') - url_string = 'https://globus.org/app/transfer?' + \ - six.moves.urllib.parse.urlencode({ - 'origin_id': SOURCE_ENDPOINT, - 'origin_path': SOURCE_PATH, - 'destination_id': DESTINATION_ENDPOINT, - 'destination_path': DESTINATION_PATH - }) - print(f'Visit the link below to see the changes:\n{url_string}') - - -if __name__ == '__main__': + save_data_to_file(DATA_FILE, "task", task.data) + print( + f"Transfer has been started from\n {SOURCE_ENDPOINT}:{SOURCE_PATH}\nto\n {DESTINATION_ENDPOINT}:{DESTINATION_PATH}" + ) + url_string = "https://globus.org/app/transfer?" + six.moves.urllib.parse.urlencode( + { + "origin_id": SOURCE_ENDPOINT, + "origin_path": SOURCE_PATH, + "destination_id": DESTINATION_ENDPOINT, + "destination_path": DESTINATION_PATH, + } + ) + print(f"Visit the link below to see the changes:\n{url_string}") + + +if __name__ == "__main__": main() diff --git a/u19_pipeline/behavior.py b/u19_pipeline/behavior.py index 770d0897..9164be9f 100644 --- a/u19_pipeline/behavior.py +++ b/u19_pipeline/behavior.py @@ -1,6 +1,7 @@ import datajoint as dj -schema = dj.schema(dj.config['custom']['database.prefix'] + 'behavior') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "behavior") + @schema class DataDirectory(dj.Computed): @@ -115,4 +116,4 @@ class TowersSessionPsych(dj.Computed): session_pright_error=null : blob # confidence interval for precentage went right of data session_delta_fit=null : blob # num of right - num of left, x ticks for fitting results session_pright_fit=null : blob # fitting results for percent went right - """ \ No newline at end of file + """ diff --git a/u19_pipeline/copy_table.py b/u19_pipeline/copy_table.py index 14d3348d..22d0ed5a 100644 --- a/u19_pipeline/copy_table.py +++ b/u19_pipeline/copy_table.py @@ -7,29 +7,24 @@ from u19_pipeline.temp import acquisition, behavior, imaging, meso, meso_analysis acquisition_original = dj.create_virtual_module( - 'acquisition_original', 'u19_acquisition' + "acquisition_original", "u19_acquisition" ) -behavior_original = dj.create_virtual_module( - 'behavior_original', 'u19_behavior') +behavior_original = dj.create_virtual_module("behavior_original", "u19_behavior") -meso_original = dj.create_virtual_module( - 'meso_original', 'u19_meso' -) +meso_original = dj.create_virtual_module("meso_original", "u19_meso") meso_analysis_original = dj.create_virtual_module( - 'meso_analysis_original', 'u19_meso_analysis' + "meso_analysis_original", "u19_meso_analysis" ) -imaging_original = dj.create_virtual_module( - 'imaging_original', 'u19_imaging' -) +imaging_original = dj.create_virtual_module("imaging_original", "u19_imaging") def copy_table(target_schema, src_schema, table_name, **kwargs): - if '.' in table_name: - attrs = table_name.split('.') + if "." in table_name: + attrs = table_name.split(".") target_table = target_schema src_table = src_schema @@ -41,9 +36,8 @@ def copy_table(target_schema, src_schema, table_name, **kwargs): src_table = getattr(src_schema, table_name) pk = src_table.heading.primary_key - if 'session_number' in pk: - q_insert = (src_table & acquisition.SessionStarted.proj()) - \ - target_table.proj() + if "session_number" in pk: + q_insert = (src_table & acquisition.SessionStarted.proj()) - target_table.proj() else: q_insert = src_table - target_table.proj() @@ -62,167 +56,171 @@ def copy_table(target_schema, src_schema, table_name, **kwargs): def copy_acquisition_tables(): acquisition.SessionTemp.insert( - (acquisition_original.Session - acquisition.SessionTemp.proj()) & - acquisition.SessionStarted.proj(), - skip_duplicates=True) + (acquisition_original.Session - acquisition.SessionTemp.proj()) + & acquisition.SessionStarted.proj(), + skip_duplicates=True, + ) acquisition.DataDirectoryTemp.insert( - (acquisition_original.DataDirectory - acquisition.DataDirectoryTemp.proj()) & - acquisition.SessionStarted.proj(), - allow_direct_insert=True + (acquisition_original.DataDirectory - acquisition.DataDirectoryTemp.proj()) + & acquisition.SessionStarted.proj(), + allow_direct_insert=True, ) def copy_behavior_tables(): BEHAVIOR_TABLES = [ - 'DataDirectory', - 'TowersSession', - 'TowersBlock', - 'TowersBlock.Trial', - 'TowersBlockTrialVideo', - 'TowersSubjectCumulativePsych', - 'TowersSessionPsych', + "DataDirectory", + "TowersSession", + "TowersBlock", + "TowersBlock.Trial", + "TowersBlockTrialVideo", + "TowersSubjectCumulativePsych", + "TowersSessionPsych", ] for table in BEHAVIOR_TABLES: - - print(f'Copying table {table}') - if '.' in table: - if table == 'TowersBlock.Trial': - for subj in tqdm((subject.Subject & behavior.TowersBlock).fetch('KEY')): + print(f"Copying table {table}") + if "." in table: + if table == "TowersBlock.Trial": + for subj in tqdm((subject.Subject & behavior.TowersBlock).fetch("KEY")): behavior.TowersBlock.Trial.insert( - behavior_original.TowersBlock.Trial & subj, - skip_duplicates=True) + behavior_original.TowersBlock.Trial & subj, skip_duplicates=True + ) else: - if table == 'TowersBlockVideo': - for subj in tqdm((subject.Subject & behavior.TowersBlock).fetch('KEY')): + if table == "TowersBlockVideo": + for subj in tqdm( + (subject.Subject & behavior.TowersBlock).fetch("KEY") + ): behavior.TowersBlockTrialVideo.insert( - behavior_original.TowersBlockTrialVideo & subj & behavior.TowersBlock, - skip_duplicates=True) + behavior_original.TowersBlockTrialVideo + & subj + & behavior.TowersBlock, + skip_duplicates=True, + ) else: copy_table(behavior, behavior_original, table) else: - copy_table(behavior, behavior_original, table, - allow_direct_insert=True) + copy_table(behavior, behavior_original, table, allow_direct_insert=True) def copy_imaging_tables(): IMAGING_TABLES = [ - 'Scan', - 'ScanInfo', - 'FieldOfView', - 'FieldOfView.File', - 'McMethod', - 'McParameter', - 'McParameterSet', - 'McParameterSet.Parameter', - 'MotionCorrection', - 'MotionCorrection.WithinFile', - 'MotionCorrection.AcrossFiles', - 'SegmentationMethod', - 'SegParameter', - 'SegParameterSet', - 'SegParameterSet.Parameter', - 'Segmentation', - 'Segmentation.Roi', - 'Segmentation.RoiMorphologyAuto', - 'Segmentation.Chunks', - 'Segmentation.Background', - 'Trace' + "Scan", + "ScanInfo", + "FieldOfView", + "FieldOfView.File", + "McMethod", + "McParameter", + "McParameterSet", + "McParameterSet.Parameter", + "MotionCorrection", + "MotionCorrection.WithinFile", + "MotionCorrection.AcrossFiles", + "SegmentationMethod", + "SegParameter", + "SegParameterSet", + "SegParameterSet.Parameter", + "Segmentation", + "Segmentation.Roi", + "Segmentation.RoiMorphologyAuto", + "Segmentation.Chunks", + "Segmentation.Background", + "Trace", ] for table in IMAGING_TABLES: + print(f"Copying table {table}...") - print(f'Copying table {table}...') - - if '.' in table: + if "." in table: copy_table(imaging, imaging_original, table) else: temp_table = getattr(imaging, table) - if isinstance(temp_table, dj.Lookup) or \ - isinstance(temp_table, dj.Manual): - copy_table(imaging, imaging_original, table, - skip_duplicates=True) + if isinstance(temp_table, dj.Lookup) or isinstance(temp_table, dj.Manual): + copy_table(imaging, imaging_original, table, skip_duplicates=True) else: - copy_table(imaging, imaging_original, table, - allow_direct_insert=True, - skip_duplicates=True) + copy_table( + imaging, + imaging_original, + table, + allow_direct_insert=True, + skip_duplicates=True, + ) def copy_meso_tables(): MESO_TABLES = [ - 'Scan', - 'ScanInfo', - 'FieldOfView', - 'FieldOfView.File', - 'SyncImagingBehavior', - 'MotionCorrectionMethod', - 'McParameter', - 'McParameterSet', - 'McParameterSet.Parameter', - 'MotionCorrectionWithinFile', - 'MotionCorrectionAcrossFiles', - 'MotionCorrection', - 'SegmentationMethod', - 'SegParameter', - 'SegParameterSet', - 'SegParameterSet.Parameter', - 'Segmentation', - 'Segmentation.Roi', - 'Segmentation.RoiMorphologyAuto', - 'Segmentation.Chunks', - 'Segmentation.Background', - 'SegmentationRoiMorphologyManual' + "Scan", + "ScanInfo", + "FieldOfView", + "FieldOfView.File", + "SyncImagingBehavior", + "MotionCorrectionMethod", + "McParameter", + "McParameterSet", + "McParameterSet.Parameter", + "MotionCorrectionWithinFile", + "MotionCorrectionAcrossFiles", + "MotionCorrection", + "SegmentationMethod", + "SegParameter", + "SegParameterSet", + "SegParameterSet.Parameter", + "Segmentation", + "Segmentation.Roi", + "Segmentation.RoiMorphologyAuto", + "Segmentation.Chunks", + "Segmentation.Background", + "SegmentationRoiMorphologyManual", ] for table in MESO_TABLES: + print(f"Copying table {table}...") - print(f'Copying table {table}...') - - if '.' in table: + if "." in table: copy_table(meso, meso_original, table) else: temp_table = getattr(meso, table) - if isinstance(temp_table, dj.Lookup) or \ - isinstance(temp_table, dj.Manual): + if isinstance(temp_table, dj.Lookup) or isinstance(temp_table, dj.Manual): copy_table(meso, meso_original, table) else: - copy_table(meso, meso_original, table, - allow_direct_insert=True) + copy_table(meso, meso_original, table, allow_direct_insert=True) def copy_meso_analysis_tables(): MESO_ANALYSIS_TABLES = [ - 'TrialSelectionParams', - 'BinningParameters', - 'StandardizedTime', - 'Trialstats', - 'BinnedBehavior', - 'TrialSelectionParameters', - 'BinnedTrace' + "TrialSelectionParams", + "BinningParameters", + "StandardizedTime", + "Trialstats", + "BinnedBehavior", + "TrialSelectionParameters", + "BinnedTrace", ] for table in MESO_ANALYSIS_TABLES: + print(f"Copying table {table}...") - print(f'Copying table {table}...') - - if '.' in table: + if "." in table: copy_table(meso_analysis, meso_analysis_original, table) else: temp_table = getattr(meso_analysis, table) - if isinstance(temp_table, dj.Lookup) or \ - isinstance(temp_table, dj.Manual): + if isinstance(temp_table, dj.Lookup) or isinstance(temp_table, dj.Manual): copy_table(meso_analysis, meso_analysis_original, table) else: - copy_table(meso_analysis, meso_analysis_original, table, - allow_direct_insert=True) + copy_table( + meso_analysis, + meso_analysis_original, + table, + allow_direct_insert=True, + ) def main(): @@ -234,5 +232,5 @@ def main(): copy_meso_analysis_tables() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/u19_pipeline/ephys_pipeline.py b/u19_pipeline/ephys_pipeline.py index dd6a52be..60bbc87f 100644 --- a/u19_pipeline/ephys_pipeline.py +++ b/u19_pipeline/ephys_pipeline.py @@ -84,14 +84,18 @@ def get_ephys_root_data_dir(): def get_session_directory(session_key): # root_dir = get_ephys_root_data_dir() - session_dir = pathlib.Path((recording.Recording & session_key).fetch1("recording_directory")).as_posix() + session_dir = pathlib.Path( + (recording.Recording & session_key).fetch1("recording_directory") + ).as_posix() # session_dir = pathlib.Path(root_dir, session_dir).as_posix() return session_dir def get_full_session_directory(recording_key): - session_dir = find_full_path([get_ephys_root_data_dir()[0]], get_session_directory(recording_key)) + session_dir = find_full_path( + [get_ephys_root_data_dir()[0]], get_session_directory(recording_key) + ) print("ephys dir:", session_dir) nidq_session = list(session_dir.glob("*nidq.bin*")) @@ -165,8 +169,8 @@ def create_lfp_trace(cat_gt_script, recording_directory, probe_directory): p.wait() stdout, stderr = p.communicate() - print('stdout', stdout) - print('stderr', stderr.decode("UTF-8")) + print("stdout", stdout) + print("stderr", stderr.decode("UTF-8")) if stderr: error = json.loads(stderr.decode("UTF-8")) @@ -185,26 +189,35 @@ def create_lfp_trace(cat_gt_script, recording_directory, probe_directory): def get_spikeglx_meta_filepath(ephys_recording_key): # attempt to retrieve from EphysRecording.EphysFile spikeglx_meta_filepath = ( - ephys_element.EphysRecording.EphysFile & ephys_recording_key & 'file_path LIKE "%.ap.meta"' + ephys_element.EphysRecording.EphysFile + & ephys_recording_key + & 'file_path LIKE "%.ap.meta"' ).fetch1("file_path") print("ephys_recording_key", ephys_recording_key) print("spikeglx_meta_filepath", spikeglx_meta_filepath) print("get_ephys_root_data_dir", get_ephys_root_data_dir()) - spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(), spikeglx_meta_filepath) + spikeglx_meta_filepath = find_full_path( + get_ephys_root_data_dir(), spikeglx_meta_filepath + ) print("spikeglx_meta_filepath", spikeglx_meta_filepath) try: - spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(), spikeglx_meta_filepath) + spikeglx_meta_filepath = find_full_path( + get_ephys_root_data_dir(), spikeglx_meta_filepath + ) print("spikeglx_meta_filepath", spikeglx_meta_filepath) except FileNotFoundError: # if not found, search in session_dir again if not spikeglx_meta_filepath.exists(): - session_dir = find_full_path(get_ephys_root_data_dir(), get_session_directory(ephys_recording_key)) + session_dir = find_full_path( + get_ephys_root_data_dir(), get_session_directory(ephys_recording_key) + ) inserted_probe_serial_number = ( - ephys_element.ProbeInsertion * ephys_element.probe.Probe & ephys_recording_key + ephys_element.ProbeInsertion * ephys_element.probe.Probe + & ephys_recording_key ).fetch1("probe") spikeglx_meta_filepaths = [fp for fp in session_dir.rglob("*.ap.meta")] @@ -214,10 +227,13 @@ def get_spikeglx_meta_filepath(ephys_recording_key): spikeglx_meta_filepath = meta_filepath break else: - raise FileNotFoundError(f"No SpikeGLX data found for probe insertion: {ephys_recording_key}") + raise FileNotFoundError( + f"No SpikeGLX data found for probe insertion: {ephys_recording_key}" + ) return spikeglx_meta_filepath + def get_full_vectors_from_key(rec_key, single_vec_mode=False): """ Get time and "old school" synchronization vectors from a recording key. @@ -250,7 +266,9 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): return # Get sampling rate and calculate channels and samples - nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate(full_session_path) + nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate( + full_session_path + ) nChan = int(nidq_meta["nSavedChans"]) num_samples = int(int(nidq_meta["fileSizeBytes"]) / (2 * nChan)) @@ -261,38 +279,47 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): print("No sync data was sound for this session") return + trial_index_nidq_virmen, iteration_index_nidq_virmen = ( + ephys_utils.get_full_vector_samples( + sync_data["iteration_idx_vector_from_virmen"], + nidq_sampling_rate, + num_samples, + ) + ) - trial_index_nidq_virmen, iteration_index_nidq_virmen =\ - ephys_utils.get_full_vector_samples(sync_data['iteration_idx_vector_from_virmen'],nidq_sampling_rate,num_samples) - - trial_index_nidq, iteration_index_nidq =\ - ephys_utils.get_full_vector_samples(sync_data['iteration_idx_vector'],nidq_sampling_rate,num_samples) - + trial_index_nidq, iteration_index_nidq = ephys_utils.get_full_vector_samples( + sync_data["iteration_idx_vector"], nidq_sampling_rate, num_samples + ) time_vector = ephys_utils.get_time_vector(trial_index_nidq, nidq_sampling_rate) - #Calculate time for iteration start samples - trial_times_ind, trial_times_full =\ - ephys_utils.get_time_vector_as_behavior(sync_data['iteration_idx_vector'], nidq_sampling_rate) + # Calculate time for iteration start samples + trial_times_ind, trial_times_full = ephys_utils.get_time_vector_as_behavior( + sync_data["iteration_idx_vector"], nidq_sampling_rate + ) - trial_times_ind_virmen, trial_times_full_virmen =\ - ephys_utils.get_time_vector_as_behavior(sync_data['iteration_idx_vector_from_virmen'], nidq_sampling_rate) + trial_times_ind_virmen, trial_times_full_virmen = ( + ephys_utils.get_time_vector_as_behavior( + sync_data["iteration_idx_vector_from_virmen"], nidq_sampling_rate + ) + ) - - #Store data + # Store data all_vectors = dict() - all_vectors['time_as_behavior_trial_ind'] = trial_times_ind - all_vectors['time_as_behavior_fullsession'] = trial_times_full + all_vectors["time_as_behavior_trial_ind"] = trial_times_ind + all_vectors["time_as_behavior_fullsession"] = trial_times_full - all_vectors['time_as_behavior_trial_ind_virmen'] = trial_times_ind_virmen - all_vectors['time_as_behavior_fullsession_virmen'] = trial_times_full_virmen + all_vectors["time_as_behavior_trial_ind_virmen"] = trial_times_ind_virmen + all_vectors["time_as_behavior_fullsession_virmen"] = trial_times_full_virmen if single_vec_mode: for this_key in all_vectors: - print('this_key', this_key) + print("this_key", this_key) print(all_vectors[this_key].shape) - all_vectors[this_key] = np.concatenate(([x for x in all_vectors[this_key]]), axis=0) + all_vectors[this_key] = np.concatenate( + ([x for x in all_vectors[this_key]]), axis=0 + ) print(all_vectors[this_key].shape) all_vectors["trial_index_nidq_virmen"] = trial_index_nidq_virmen @@ -302,7 +329,6 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): all_vectors["iteration_index_nidq"] = iteration_index_nidq all_vectors["time_vector"] = time_vector - return all_vectors @@ -360,7 +386,7 @@ def make(self, key, **kwargs): thissession = behavior.TowersBlock().Trial() & behavior_key behavior_time, iterstart = thissession.fetch("trial_time", "vi_start") - if key['recording_id']==530: + if key["recording_id"] == 530: print(type(behavior_time)) print(len(behavior_time)) behavior_time = behavior_time[:-2] @@ -373,26 +399,34 @@ def make(self, key, **kwargs): print("after reading behavior data") # 1: load meta data, and the content of the NIDAQ file. Its content is digital. - nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate(ephys_session_fullpath) + nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate( + ephys_session_fullpath + ) - trial_pulse_signal, iteration_pulse_signal = ephys_utils.load_trial_iteration_signals( - ephys_session_fullpath, nidq_meta + trial_pulse_signal, iteration_pulse_signal = ( + ephys_utils.load_trial_iteration_signals( + ephys_session_fullpath, nidq_meta + ) ) print("after reading spikeglx data") # Synchronize between pulses and get iteration # vector for each sample - recent_recording = behavior_key["session_date"] > datetime.date(2021, 6, 1) # Everything past June 1 2021 + recent_recording = behavior_key["session_date"] > datetime.date( + 2021, 6, 1 + ) # Everything past June 1 2021 if recent_recording: # New synchronization method: digital_array[1,2] contain pulses for trial and frame number. mode = None - iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses( - trial_pulse_signal, - iteration_pulse_signal, - nidq_sampling_rate, - behavior_time.shape[0], - behavior_time, - mode, + iteration_dict = ( + ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses( + trial_pulse_signal, + iteration_pulse_signal, + nidq_sampling_rate, + behavior_time.shape[0], + behavior_time, + mode, + ) ) else: # Old synchronization: digital_array[0:7] contain a digital word that counts the virmen frames. @@ -401,19 +435,31 @@ def make(self, key, **kwargs): # Check # of trials (from database record of behavior in `behavior_time`) and iterations (extracted from NIDAQ in `iter_start_idx`) match trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ( - ephys_utils.assert_iteration_samples_count(iteration_dict["iter_start_idx"], behavior_time) + ephys_utils.assert_iteration_samples_count( + iteration_dict["iter_start_idx"], behavior_time + ) ) print("metrics to evaluate...") - print(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0]) + print( + trial_count_diff, + trials_diff_iteration_big, + trials_diff_iteration_small, + behavior_time.shape[0], + ) status = ephys_utils.evaluate_sync_process( - trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0] + trial_count_diff, + trials_diff_iteration_big, + trials_diff_iteration_small, + behavior_time.shape[0], ) if status == 1: - iteration_dict["trial_start_idx"] = ephys_utils.get_index_trial_vector_from_iteration( - iteration_dict["iter_start_idx"] + iteration_dict["trial_start_idx"] = ( + ephys_utils.get_index_trial_vector_from_iteration( + iteration_dict["iter_start_idx"] + ) ) # Failed sync by a lot, error @@ -434,20 +480,32 @@ def make(self, key, **kwargs): print("after all main ehpys fix sync code", status_regular, status_fix) if status_regular > 0 or status_fix > 0: - dictionary_sync_data["trial_idx_vector"] = iteration_dict["trial_start_idx"] - dictionary_sync_data["iteration_idx_vector"] = iteration_dict["iter_start_idx"] + dictionary_sync_data["trial_idx_vector"] = iteration_dict[ + "trial_start_idx" + ] + dictionary_sync_data["iteration_idx_vector"] = iteration_dict[ + "iter_start_idx" + ] else: dictionary_sync_data["trial_idx_vector"] = [] dictionary_sync_data["iteration_idx_vector"] = [] - iteration_dict["trial_start_idx_virmen"], iteration_dict["iter_start_idx_virmen"] = ( - ephys_utils.get_iteration_intertrial_from_virmen_time( - trial_pulse_signal, nidq_sampling_rate, behavior_time.shape[0], behavior_time - ) + ( + iteration_dict["trial_start_idx_virmen"], + iteration_dict["iter_start_idx_virmen"], + ) = ephys_utils.get_iteration_intertrial_from_virmen_time( + trial_pulse_signal, + nidq_sampling_rate, + behavior_time.shape[0], + behavior_time, ) - dictionary_sync_data["trial_idx_vector_from_virmen"] = iteration_dict["trial_start_idx_virmen"] - dictionary_sync_data["iteration_idx_vector_from_virmen"] = iteration_dict["iter_start_idx_virmen"] + dictionary_sync_data["trial_idx_vector_from_virmen"] = iteration_dict[ + "trial_start_idx_virmen" + ] + dictionary_sync_data["iteration_idx_vector_from_virmen"] = iteration_dict[ + "iter_start_idx_virmen" + ] final_key = dict( key, @@ -460,13 +518,13 @@ def make(self, key, **kwargs): virmen_sync_status=1, ) - print('ephys_session_fullpath', ephys_session_fullpath) - print('sync code executed sucessfully !!!!!!!!!!!!!!!!!!!!!!!!!!!') + print("ephys_session_fullpath", ephys_session_fullpath) + print("sync code executed sucessfully !!!!!!!!!!!!!!!!!!!!!!!!!!!") print(kwargs) - if 'populate' not in kwargs or kwargs['populate'] == True: - BehaviorSync.insert1(final_key,allow_direct_insert=True) + if "populate" not in kwargs or kwargs["populate"] == True: + BehaviorSync.insert1(final_key, allow_direct_insert=True) self.insert_imec_sampling_rate(key, ephys_session_fullpath.parent) else: return final_key @@ -479,13 +537,19 @@ def insert_imec_sampling_rate(self, key, session_dir): here = ephys_element.ProbeInsertion & key for probe_insertion in here.fetch("KEY"): # imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.bin'.format(probe_insertion['insertion_number']))) - imec_bin_filepath = list(session_dir.glob("*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]))) + imec_bin_filepath = list( + session_dir.glob( + "*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]) + ) + ) if len(imec_bin_filepath) == 1: # find the binary file to get meta data imec_bin_filepath = imec_bin_filepath[0] else: # if this fails, get the ap.meta file. imec_bin_filepath = list( - session_dir.glob("*imec{}/*.ap.meta".format(probe_insertion["insertion_number"])) + session_dir.glob( + "*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]) + ) ) if len(imec_bin_filepath) == 1: s = str(imec_bin_filepath[0]) @@ -494,4 +558,6 @@ def insert_imec_sampling_rate(self, key, session_dir): raise NameError("No imec meta file found.") imec_meta = readSGLX.readMeta(imec_bin_filepath) - self.ImecSamplingRate.insert1(dict(probe_insertion, ephys_sampling_rate=imec_meta["imSampRate"])) + self.ImecSamplingRate.insert1( + dict(probe_insertion, ephys_sampling_rate=imec_meta["imSampRate"]) + ) diff --git a/u19_pipeline/ephys_sync.py b/u19_pipeline/ephys_sync.py index 3d18492e..eb6e11b7 100644 --- a/u19_pipeline/ephys_sync.py +++ b/u19_pipeline/ephys_sync.py @@ -9,7 +9,8 @@ from u19_pipeline.ephys_pipeline import ephys_element, get_session_directory # Tables downstream from `ephys_pipeline` module --------------------------------------- -schema = dj.schema(dj.config['custom']['database.prefix'] + 'ephys_sync') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "ephys_sync") + @schema class BehaviorSync(dj.Imported): @@ -32,67 +33,93 @@ class ImecSamplingRate(dj.Part): def make(self, key): # Pull the Nidaq file/record session_dir = pathlib.Path(get_session_directory(key)) - nidq_bin_full_path = list(session_dir.glob('*nidq.bin*'))[0] + nidq_bin_full_path = list(session_dir.glob("*nidq.bin*"))[0] # And get the datajoint record thissession = behavior.TowersBlock().Trial() & key - behavior_time, iterstart = thissession.fetch('trial_time', 'vi_start') + behavior_time, iterstart = thissession.fetch("trial_time", "vi_start") # 1: load meta data, and the content of the NIDAQ file. Its content is digital. - nidq_meta = readSGLX.readMeta(nidq_bin_full_path) + nidq_meta = readSGLX.readMeta(nidq_bin_full_path) nidq_sampling_rate = readSGLX.SampRate(nidq_meta) - if nidq_meta['typeThis'] == 'nidq': - digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta) + if nidq_meta["typeThis"] == "nidq": + digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file( + nidq_bin_full_path, nidq_meta + ) else: - digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta, d_line_list=[1,2]) + digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file( + nidq_bin_full_path, nidq_meta, d_line_list=[1, 2] + ) # Synchronize between pulses and get iteration # vector for each sample - mode='counter_bit0' - iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[1,:], digital_array[2,:], nidq_sampling_rate, behavior_time.shape[0], mode) + mode = "counter_bit0" + iteration_dict = ( + ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses( + digital_array[1, :], + digital_array[2, :], + nidq_sampling_rate, + behavior_time.shape[0], + mode, + ) + ) # Check # of trials and iterations match - status = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time) + status = ephys_utils.assert_iteration_samples_count( + iteration_dict["iter_start_idx"], behavior_time + ) - #They didn't match, try counter method (if available) + # They didn't match, try counter method (if available) if (not status) and (digital_array.shape[0] > 3): - [framenumber_in_trial, trialnumber] = ephys_utils.behavior_sync_frame_counter_method(digital_array, behavior_time, thissession, nidq_sampling_rate, 3, 5) - iteration_dict['framenumber_vector_samples'] = framenumber_in_trial - iteration_dict['trialnumber_vector_samples'] = trialnumber - + [framenumber_in_trial, trialnumber] = ( + ephys_utils.behavior_sync_frame_counter_method( + digital_array, behavior_time, thissession, nidq_sampling_rate, 3, 5 + ) + ) + iteration_dict["framenumber_vector_samples"] = framenumber_in_trial + iteration_dict["trialnumber_vector_samples"] = trialnumber - final_key = dict(key, nidq_sampling_rate = nidq_sampling_rate, - iteration_index_nidq = iteration_dict['framenumber_vector_samples'], - trial_index_nidq = iteration_dict['trialnumber_vector_samples']) + final_key = dict( + key, + nidq_sampling_rate=nidq_sampling_rate, + iteration_index_nidq=iteration_dict["framenumber_vector_samples"], + trial_index_nidq=iteration_dict["trialnumber_vector_samples"], + ) print(final_key) - self.insert1(final_key,allow_direct_insert=True) + self.insert1(final_key, allow_direct_insert=True) self.insert_imec_sampling_rate(key, session_dir) - - def insert_imec_sampling_rate(self, key, session_dir): # get the imec sampling rate for a particular probe here = ephys_element.ProbeInsertion & key - for probe_insertion in here.fetch('KEY'): - #imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.bin'.format(probe_insertion['insertion_number']))) - imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.meta'.format(probe_insertion['insertion_number']))) + for probe_insertion in here.fetch("KEY"): + # imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.bin'.format(probe_insertion['insertion_number']))) + imec_bin_filepath = list( + session_dir.glob( + "*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]) + ) + ) - if len(imec_bin_filepath) == 1: # find the binary file to get meta data + if len(imec_bin_filepath) == 1: # find the binary file to get meta data imec_bin_filepath = imec_bin_filepath[0] - else: # if this fails, get the ap.meta file. - imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.meta'.format(probe_insertion['insertion_number']))) + else: # if this fails, get the ap.meta file. + imec_bin_filepath = list( + session_dir.glob( + "*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]) + ) + ) if len(imec_bin_filepath) == 1: s = str(imec_bin_filepath[0]) imec_bin_filepath = pathlib.Path(s.replace(".meta", ".bin")) - else: # If this fails too, no imec file exists at the path. + else: # If this fails too, no imec file exists at the path. raise NameError("No imec meta file found.") imec_meta = readSGLX.readMeta(imec_bin_filepath) self.ImecSamplingRate.insert1( - dict(probe_insertion, - ephys_sampling_rate=imec_meta['imSampRate'])) + dict(probe_insertion, ephys_sampling_rate=imec_meta["imSampRate"]) + ) @schema @@ -116,58 +143,70 @@ def make(self, key): self.insert1(key) - nidq_sampling_rate, iteration_index_nidq = \ - (BehaviorSync * BehaviorSync.ImecSamplingRate & key).fetch1( - 'nidq_sampling_rate', 'iteration_index_nidq') + nidq_sampling_rate, iteration_index_nidq = ( + BehaviorSync * BehaviorSync.ImecSamplingRate & key + ).fetch1("nidq_sampling_rate", "iteration_index_nidq") key_session = key.copy() del key_session["insertion_number"] thissession = behavior.TowersBlock().Trial() & key - iterstart = thissession.fetch('vi_start') + iterstart = thissession.fetch("vi_start") first_vr_iteration = iterstart[0] # Obtain the precise times when the frames transition. # This is obtained from iteration_index_nidq ls = np.diff(iteration_index_nidq) - ls[ls<0] = 1 # These are the trial transitions (see definition above). To get total number of frames, we define this as a transition like all others. + ls[ls < 0] = ( + 1 # These are the trial transitions (see definition above). To get total number of frames, we define this as a transition like all others. + ) ls[np.isnan(ls)] = 0 iteration_transition_indexes = np.where(ls)[0] # First iterations captured not in virmen because vr was not started yet - #for i in range(first_vr_iteration): + # for i in range(first_vr_iteration): # if iteration_index_nidq[iteration_transition_indexes[i]] <= first_vr_iteration: # ls[iteration_transition_indexes[i]] = 0 - print('sum_iterationtrans', np.sum(ls)) + print("sum_iterationtrans", np.sum(ls)) - iteration_times = np.where(ls)[0]/nidq_sampling_rate + iteration_times = np.where(ls)[0] / nidq_sampling_rate # get end of time from nidq metadata session_dir = pathlib.Path(get_session_directory(key)) - nidq_bin_full_path = list(session_dir.glob('*nidq.bin*'))[0] + nidq_bin_full_path = list(session_dir.glob("*nidq.bin*"))[0] nidq_meta = readSGLX.readMeta(nidq_bin_full_path) - t_end = np.float(nidq_meta['fileTimeSecs']) + t_end = np.float(nidq_meta["fileTimeSecs"]) unit_spike_counts = [] - for unit_key in (ephys_element.CuratedClustering.Unit & key).fetch('KEY'): - spike_times = (ephys_element.CuratedClustering.Unit & unit_key).fetch1('spike_times') + for unit_key in (ephys_element.CuratedClustering.Unit & key).fetch("KEY"): + spike_times = (ephys_element.CuratedClustering.Unit & unit_key).fetch1( + "spike_times" + ) # vector with length n_iterations + 1 - spike_counts_iteration = np.bincount(np.digitize(spike_times, iteration_times)) + spike_counts_iteration = np.bincount( + np.digitize(spike_times, iteration_times) + ) - firing_rate_before_first_iteration = spike_counts_iteration[0]/iteration_times[0] - firing_rate_after_last_iteration = spike_counts_iteration[-1]/(t_end - iteration_times[-1]) + firing_rate_before_first_iteration = ( + spike_counts_iteration[0] / iteration_times[0] + ) + firing_rate_after_last_iteration = spike_counts_iteration[-1] / ( + t_end - iteration_times[-1] + ) # remove the first and last entries spike_counts_iteration = np.delete(spike_counts_iteration, [0, -1]) unit_spike_counts.append( - dict(unit_key, - spike_counts_iteration=spike_counts_iteration, - firing_rate_before_first_iteration=firing_rate_before_first_iteration, - firing_rate_after_last_iteration=firing_rate_after_last_iteration) + dict( + unit_key, + spike_counts_iteration=spike_counts_iteration, + firing_rate_before_first_iteration=firing_rate_before_first_iteration, + firing_rate_after_last_iteration=firing_rate_after_last_iteration, + ) ) self.Unit.insert(unit_spike_counts) diff --git a/u19_pipeline/imaging.py b/u19_pipeline/imaging.py index 2c3d2e6e..1973a52e 100644 --- a/u19_pipeline/imaging.py +++ b/u19_pipeline/imaging.py @@ -1,6 +1,6 @@ import datajoint as dj -schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "imaging") @schema diff --git a/u19_pipeline/imaging_pipeline.py b/u19_pipeline/imaging_pipeline.py index 9012bef5..adf7ae22 100644 --- a/u19_pipeline/imaging_pipeline.py +++ b/u19_pipeline/imaging_pipeline.py @@ -1,4 +1,3 @@ - import pathlib import subprocess from pathlib import Path @@ -12,7 +11,8 @@ import u19_pipeline.utils.tiff_utils as tu from u19_pipeline import lab, recording, subject -schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging_pipeline') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "imaging_pipeline") + # Declare upstream imaging tables ------------------------------------------------------ @schema @@ -21,18 +21,17 @@ class ImagingPipelineSession(dj.Computed): # General information of an imaging session -> recording.Recording """ + @property def key_source(self): - return recording.Recording & {'recording_modality': 'imaging'} + return recording.Recording & {"recording_modality": "imaging"} def make(self, key): self.insert1(key) - @schema class AcquiredTiff(dj.Imported): - definition = """ -> imaging_pipeline.ImagingPipelineSession --- @@ -68,31 +67,28 @@ class AcquiredTiff(dj.Imported): stack_definition="N/A" : varchar(64) """ - photon_micro_acq = ['2photon', '3photon'] - mesoscope_acq = ['mesoscope'] + photon_micro_acq = ["2photon", "3photon"] + mesoscope_acq = ["mesoscope"] def make(self, key, test_mode=False): scan_info = ( - ImagingPipelineSession - * recording.Recording - * lab.Location - & key + ImagingPipelineSession * recording.Recording * lab.Location & key ).fetch1() - imaging_root = dj.config['custom']['imaging_root_data_dir'][0] - scan_directory = Path(imaging_root) / scan_info['recording_directory'] - acq_type = scan_info['acquisition_type'] + imaging_root = dj.config["custom"]["imaging_root_data_dir"][0] + scan_directory = Path(imaging_root) / scan_info["recording_directory"] + acq_type = scan_info["acquisition_type"] is_mesoscope = acq_type in self.mesoscope_acq is_2photon = acq_type in self.photon_micro_acq - print(f'Preparing {scan_directory}') + print(f"Preparing {scan_directory}") if is_mesoscope: - original_stacks_dir = scan_directory / 'originalStacks' + original_stacks_dir = scan_directory / "originalStacks" - tif_files = list(scan_directory.glob('*tif*')) + tif_files = list(scan_directory.glob("*tif*")) if not tif_files and original_stacks_dir.exists(): tif_dir = original_stacks_dir @@ -106,90 +102,96 @@ def make(self, key, test_mode=False): fl, basename, is_compressed = tu.check_tif_files(tif_dir) - fl = [Path(tif_dir,x).as_posix() for x in fl] + fl = [Path(tif_dir, x).as_posix() for x in fl] if is_mesoscope: imheader, parsed_info = tu.get_parsed_info_mesoscope(fl) else: imheader, parsed_info = tu.get_parsed_info_2photon(fl) - rec_info, frames_per_file = tu.get_recording_info( - fl, - imheader, - parsed_info - ) + rec_info, frames_per_file = tu.get_recording_info(fl, imheader, parsed_info) - rec_info['nfovs'] = tu.get_nfovs(rec_info, is_mesoscope) + rec_info["nfovs"] = tu.get_nfovs(rec_info, is_mesoscope) last_good_file, cumulative_frames = tu.get_last_good_frame( - frames_per_file, - tif_dir + frames_per_file, tif_dir ) - rec_info['nframes_good'] = cumulative_frames[last_good_file] - rec_info['last_good_file'] = last_good_file+1 + rec_info["nframes_good"] = cumulative_frames[last_good_file] + rec_info["last_good_file"] = last_good_file + 1 - rec_info['AcqTime'] = tu.check_acqtime( - rec_info['AcqTime'], - scan_directory - ) + rec_info["AcqTime"] = tu.check_acqtime(rec_info["AcqTime"], scan_directory) if is_compressed: tu.remove_compressed_videos(fl, scan_directory) scan_info_key = tu.create_scan_info_key( - key, - rec_info, - scan_info['recording_directory'] + key, rec_info, scan_info["recording_directory"] ) if not test_mode: - self.insert1(scan_info_key,allow_direct_insert=True) + self.insert1(scan_info_key, allow_direct_insert=True) if is_mesoscope: - tiffsplit_mesoscope_keys,tiff_splitfiles_mesoscope_keys = tu.get_fov_mesoscope( - fl, - key, - skip_parsing, - imheader, - rec_info, - basename, - cumulative_frames, - scan_info, - imaging_root + tiffsplit_mesoscope_keys, tiff_splitfiles_mesoscope_keys = ( + tu.get_fov_mesoscope( + fl, + key, + skip_parsing, + imheader, + rec_info, + basename, + cumulative_frames, + scan_info, + imaging_root, + ) ) for i in range(len(tiffsplit_mesoscope_keys)): if not test_mode: - TiffSplit.insert(tiffsplit_mesoscope_keys[i],allow_direct_insert=True) - TiffSplit.File.insert(tiff_splitfiles_mesoscope_keys[i],allow_direct_insert=True) + TiffSplit.insert( + tiffsplit_mesoscope_keys[i], allow_direct_insert=True + ) + TiffSplit.File.insert( + tiff_splitfiles_mesoscope_keys[i], allow_direct_insert=True + ) if test_mode: - return scan_info_key, tiffsplit_mesoscope_keys, tiff_splitfiles_mesoscope_keys + return ( + scan_info_key, + tiffsplit_mesoscope_keys, + tiff_splitfiles_mesoscope_keys, + ) elif is_2photon: tiffsplit_2photon_key = tu.get_fov_photonmicro(key, rec_info, scan_info) tiffsplitfile_2photon_key = tu.get_fovfile_photonmicro(key, fl, imheader) if not test_mode: - TiffSplit.insert([tiffsplit_2photon_key],allow_direct_insert=True) - TiffSplit.File.insert(tiffsplitfile_2photon_key,allow_direct_insert=True) + TiffSplit.insert([tiffsplit_2photon_key], allow_direct_insert=True) + TiffSplit.File.insert( + tiffsplitfile_2photon_key, allow_direct_insert=True + ) if test_mode: return scan_info_key, tiffsplit_2photon_key, tiffsplitfile_2photon_key else: raise ValueError("Invalid acquisition type") - + def old_make(self, key): str_key = dj_short.get_string_key(key) - command = [config.ingest_scaninfo_script, config.startup_pipeline_matlab_dir, str_key] + command = [ + config.ingest_scaninfo_script, + config.startup_pipeline_matlab_dir, + str_key, + ] print(command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.wait() - print('aftercommand before comm') + print("aftercommand before comm") stdout, stderr = p.communicate() - print('aftercommand after comm') - print(stdout.decode('UTF-8')) - print(stderr.decode('UTF-8')) + print("aftercommand after comm") + print(stdout.decode("UTF-8")) + print(stderr.decode("UTF-8")) @schema @@ -246,13 +248,16 @@ class File(dj.Part): """ # 1. Schema names ---------------------------------------------------------------------- -scan_schema_name = dj.config['custom']['database.prefix'] + 'pipeline_scan_element' -imaging_schema_name = dj.config['custom']['database.prefix'] + 'pipeline_imaging_element' +scan_schema_name = dj.config["custom"]["database.prefix"] + "pipeline_scan_element" +imaging_schema_name = ( + dj.config["custom"]["database.prefix"] + "pipeline_imaging_element" +) # 2. Upstream tables ------------------------------------------------------------------- Session = TiffSplit + @lab.schema class Equipment(dj.Manual): definition = """ @@ -262,68 +267,93 @@ class Equipment(dj.Manual): description=null : varchar(256) """ + # 3. Utility functions ----------------------------------------------------------------- + def get_imaging_root_data_dir(): - return dj.config.get('custom', {}).get('imaging_root_data_dir', None) + return dj.config.get("custom", {}).get("imaging_root_data_dir", None) + def get_scan_image_files(job_id): - #scan_key = (TiffSplit * recording_process.Processing.proj('recording_id', tiff_split='fragment_number') & job_id).fetch1('KEY') + # scan_key = (TiffSplit * recording_process.Processing.proj('recording_id', tiff_split='fragment_number') & job_id).fetch1('KEY') - scan_key = (TiffSplit & job_id).fetch1('KEY') + scan_key = (TiffSplit & job_id).fetch1("KEY") - filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch('tiff_split_directory', 'tiff_split_filename', as_dict=True) + filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch( + "tiff_split_directory", "tiff_split_filename", as_dict=True + ) - tiff_filepaths = [find_full_path(get_imaging_root_data_dir(), - pathlib.Path(file['tiff_split_directory']) / - file['tiff_split_filename']).as_posix() - for file in filepaths] + tiff_filepaths = [ + find_full_path( + get_imaging_root_data_dir(), + pathlib.Path(file["tiff_split_directory"]) / file["tiff_split_filename"], + ).as_posix() + for file in filepaths + ] return tiff_filepaths + def get_calcium_imaging_files(scan_key, acq_software): - filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch('tiff_split_directory', 'tiff_split_filename', as_dict=True) + filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch( + "tiff_split_directory", "tiff_split_filename", as_dict=True + ) - tiff_filepaths = [find_full_path(get_imaging_root_data_dir(), - pathlib.Path(file['tiff_split_directory']) / - file['tiff_split_filename']).as_posix() - for file in filepaths] + tiff_filepaths = [ + find_full_path( + get_imaging_root_data_dir(), + pathlib.Path(file["tiff_split_directory"]) / file["tiff_split_filename"], + ).as_posix() + for file in filepaths + ] return tiff_filepaths - + + def get_calcium_imaging_files(scan_key, acq_software): - filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch('tiff_split_directory', 'tiff_split_filename', as_dict=True) + filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch( + "tiff_split_directory", "tiff_split_filename", as_dict=True + ) - tiff_filepaths = [find_full_path(get_imaging_root_data_dir(), - pathlib.Path(file['tiff_split_directory']) / - file['tiff_split_filename']).as_posix() - for file in filepaths] + tiff_filepaths = [ + find_full_path( + get_imaging_root_data_dir(), + pathlib.Path(file["tiff_split_directory"]) / file["tiff_split_filename"], + ).as_posix() + for file in filepaths + ] return tiff_filepaths + def get_processed_dir(processing_task_key, process_method): - sess_key = (ImagingPipelineSession & processing_task_key).fetch1('KEY') - bucket_scan_dir = (TiffSplit & sess_key & - {'tiff_split': processing_task_key['scan_id']}).fetch1('tiff_split_directory') - user_id = (subject.Subject & processing_task_key).fetch1('user_id') + sess_key = (ImagingPipelineSession & processing_task_key).fetch1("KEY") + bucket_scan_dir = ( + TiffSplit & sess_key & {"tiff_split": processing_task_key["scan_id"]} + ).fetch1("tiff_split_directory") + user_id = (subject.Subject & processing_task_key).fetch1("user_id") sess_dir = find_full_path(get_imaging_root_data_dir(), bucket_scan_dir) - relative_suite2p_dir = (pathlib.Path(bucket_scan_dir) / process_method).as_posix() + relative_suite2p_dir = (pathlib.Path(bucket_scan_dir) / process_method).as_posix() if not sess_dir.exists(): - raise FileNotFoundError(f'Session directory not found ({sess_dir})') + raise FileNotFoundError(f"Session directory not found ({sess_dir})") - if process_method == 'suite2p': + if process_method == "suite2p": # Check if ops.npy is inside suite2p_dir - suite2p_dirs = set([fp.parent.parent for fp in sess_dir.rglob('*ops.npy')]) + suite2p_dirs = set([fp.parent.parent for fp in sess_dir.rglob("*ops.npy")]) if len(suite2p_dirs) != 1: - raise FileNotFoundError(f'Error searching for Suite2p output directory in {bucket_scan_dir} - Found {suite2p_dirs}') - elif process_method == 'caiman': - raise NotImplementedError('CaImAn is not currented implemented.') + raise FileNotFoundError( + f"Error searching for Suite2p output directory in {bucket_scan_dir} - Found {suite2p_dirs}" + ) + elif process_method == "caiman": + raise NotImplementedError("CaImAn is not currented implemented.") return sess_dir + # 4. Activate imaging schema ----------------------------------------------------------- -imaging_element.activate(imaging_schema_name, scan_schema_name, linking_module=__name__) \ No newline at end of file +imaging_element.activate(imaging_schema_name, scan_schema_name, linking_module=__name__) diff --git a/u19_pipeline/lab.py b/u19_pipeline/lab.py index 855e9b47..40c1dde5 100644 --- a/u19_pipeline/lab.py +++ b/u19_pipeline/lab.py @@ -229,11 +229,35 @@ class Path(dj.Lookup): contents = [ ["/Bezos-center", "windows", "Y:", r"\\cup.pni.princeton.edu\Bezos-center", ""], - ["/Bezos-center", "mac", "/Volumes/Bezos-center", "//cup.pni.princeton.edu/Bezos-center", ""], - ["/Bezos-center", "linux", "/mnt/Bezos-center", "//cup.pni.princeton.edu/Bezos-center", ""], + [ + "/Bezos-center", + "mac", + "/Volumes/Bezos-center", + "//cup.pni.princeton.edu/Bezos-center", + "", + ], + [ + "/Bezos-center", + "linux", + "/mnt/Bezos-center", + "//cup.pni.princeton.edu/Bezos-center", + "", + ], ["/braininit", "windows", "Z:", r"\\cup.pni.princeton.edu\braininit", ""], - ["/braininit", "mac", "/Volumes/braininit", "//cup.pni.princeton.edu/Bezos-center", ""], - ["/braininit", "linux", "/mnt/braininit", "//cup.pni.princeton.edu/Bezos-center", ""], + [ + "/braininit", + "mac", + "/Volumes/braininit", + "//cup.pni.princeton.edu/Bezos-center", + "", + ], + [ + "/braininit", + "linux", + "/mnt/braininit", + "//cup.pni.princeton.edu/Bezos-center", + "", + ], ] def get_local_path(self, path, local_os=None): @@ -271,7 +295,9 @@ def get_local_path(self, path, local_os=None): for iglob, glob in enumerate(globs.fetch("KEY")): mapping[iglob].append(glob["global_path"]) for system in systems: - mapping[iglob].append((self & glob & {"system": system}).fetch1("local_path")) + mapping[iglob].append( + (self & glob & {"system": system}).fetch1("local_path") + ) mapping = np.asarray(mapping) @@ -306,9 +332,13 @@ def get_local_path2(self, bucket_path): path_df = path_df[path_df["system"] == system] # Search in path which of the main buckets we are referring from - path_df["idx_global_path"] = path_df["global_path"].apply(lambda x: bucket_path.find(x)) + path_df["idx_global_path"] = path_df["global_path"].apply( + lambda x: bucket_path.find(x) + ) path_df = path_df[path_df["idx_global_path"] != -1] - path_df = path_df[path_df["idx_global_path"] == path_df["idx_global_path"].min()].squeeze() + path_df = path_df[ + path_df["idx_global_path"] == path_df["idx_global_path"].min() + ].squeeze() path_df = path_df.to_dict() # Remove bucket "base" dir from path diff --git a/u19_pipeline/lightsheet.py b/u19_pipeline/lightsheet.py index 154eef7f..1efacefe 100644 --- a/u19_pipeline/lightsheet.py +++ b/u19_pipeline/lightsheet.py @@ -1,6 +1,6 @@ import datajoint as dj -schema = dj.schema('U19_lightsheet') +schema = dj.schema("U19_lightsheet") @schema @@ -11,7 +11,8 @@ class User(dj.Lookup): --- princeton_email : varchar(50) """ - + + @schema class Experiment(dj.Manual): definition = """ # Experiments performed using the light sheet microscope @@ -33,10 +34,13 @@ class Experiment(dj.Manual): channel555 : enum("","registration","cell_detection","probe_detection","injection_detection") channel647 : enum("","registration","cell_detection","probe_detection","injection_detection") channel790 : enum("","registration","cell_detection","probe_detection","injection_detection") - """ + """ + @schema -class IdiscoPlusClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data +class IdiscoPlusClearing( + dj.Manual +): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data definition = """ # Periodic calibration data of the light sheet microscope -> Experiment # experiment_id, the primary key from the Experiment() table ---- @@ -147,7 +151,9 @@ class IdiscoPlusClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table @schema -class IdiscoAbbreviatedClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data +class IdiscoAbbreviatedClearing( + dj.Manual +): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data definition = """ # Periodic calibration data of the light sheet microscope -> Experiment # experiment_id, the primary key from the Experiment() table ---- @@ -186,8 +192,11 @@ class IdiscoAbbreviatedClearing(dj.Manual): # dj.Manual is one of the 4 datajoin clearing_notes = "" : varchar(500) """ + @schema -class IdiscoAbbreviatedRatClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data +class IdiscoAbbreviatedRatClearing( + dj.Manual +): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data definition = """ # Abbreviated Rat clearing table -> Experiment # experiment_id, the primary key from the Experiment() table ---- @@ -240,8 +249,11 @@ class IdiscoAbbreviatedRatClearing(dj.Manual): # dj.Manual is one of the 4 dataj clearing_notes = "" : varchar(500) """ + @schema -class UdiscoClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data +class UdiscoClearing( + dj.Manual +): # dj.Manual is one of the 4 datajoint table types - Manual corresponds to externally inputted data definition = """ # uDISCO clearing table -> Experiment # experiment_id, the primary key from the Experiment() table ---- @@ -270,4 +282,4 @@ class UdiscoClearing(dj.Manual): # dj.Manual is one of the 4 datajoint table typ time_clearing_babb_wash1 = NULL : datetime clearing_babb_wash1_notes = "" : varchar(250) clearing_notes = "" : varchar(500) - """ \ No newline at end of file + """ diff --git a/u19_pipeline/meso.py b/u19_pipeline/meso.py index 2ded474c..3b9449ef 100644 --- a/u19_pipeline/meso.py +++ b/u19_pipeline/meso.py @@ -1,7 +1,7 @@ import datajoint as dj import numpy as np -schema = dj.schema('u19_meso') +schema = dj.schema("u19_meso") @schema @@ -107,8 +107,8 @@ class MotionCorrectionMethod(dj.Lookup): """ contents = [ - ['LinearNormalized', 'Normalized', 'Linear'], - ['NonLinearNormalized', 'Normalized', 'Nonlinear'] + ["LinearNormalized", "Normalized", "Linear"], + ["NonLinearNormalized", "Normalized", "Nonlinear"], ] @@ -122,11 +122,11 @@ class McParameter(dj.Lookup): """ contents = [ - ['LinearNormalized', 'mc_max_shift', ''], - ['LinearNormalized', 'mc_max_iter', ''], - ['LinearNormalized', 'mc_stop_below_shift', ''], - ['LinearNormalized', 'mc_black_tolerance', ''], - ['LinearNormalized', 'mc_median_rebin', ''] + ["LinearNormalized", "mc_max_shift", ""], + ["LinearNormalized", "mc_max_iter", ""], + ["LinearNormalized", "mc_stop_below_shift", ""], + ["LinearNormalized", "mc_black_tolerance", ""], + ["LinearNormalized", "mc_median_rebin", ""], ] @@ -138,7 +138,7 @@ class McParameterSet(dj.Lookup): mc_parameter_set_id : int # parameter set id """ - contents = [['LinearNormalized', 1]] + contents = [["LinearNormalized", 1]] class Parameter(dj.Part): definition = """ @@ -353,23 +353,19 @@ class Trace(dj.Imported): """ - - -if __name__ == '__main__': - - key = {'mcorr_method': 'LinearNormalized', - 'mc_parameter_set_id': 1} +if __name__ == "__main__": + key = {"mcorr_method": "LinearNormalized", "mc_parameter_set_id": 1} parameters = [ - dict(**key, mc_parameter_name='mc_max_shift', - mc_parameter_value=np.array([15.])), - dict(**key, mc_parameter_name='mc_max_iter', - mc_parameter_value=np.array([5.])), - dict(**key, mc_parameter_name='mc_stop_below_shift', - mc_parameter_value=0.3), - dict(**key, mc_parameter_name='mc_black_tolerance', - mc_parameter_value=-1.), - dict(**key, mc_parameter_name='mc_median_rebin', - mc_parameter_value=10.)] + dict( + **key, mc_parameter_name="mc_max_shift", mc_parameter_value=np.array([15.0]) + ), + dict( + **key, mc_parameter_name="mc_max_iter", mc_parameter_value=np.array([5.0]) + ), + dict(**key, mc_parameter_name="mc_stop_below_shift", mc_parameter_value=0.3), + dict(**key, mc_parameter_name="mc_black_tolerance", mc_parameter_value=-1.0), + dict(**key, mc_parameter_name="mc_median_rebin", mc_parameter_value=10.0), + ] McParameterSet.Parameter.insert(parameters) diff --git a/u19_pipeline/meso_analysis.py b/u19_pipeline/meso_analysis.py index 419423ea..7da59d5d 100644 --- a/u19_pipeline/meso_analysis.py +++ b/u19_pipeline/meso_analysis.py @@ -1,6 +1,6 @@ import datajoint as dj -schema = dj.schema('u19_meso_analysis') +schema = dj.schema("u19_meso_analysis") @schema diff --git a/u19_pipeline/microsocope.py b/u19_pipeline/microsocope.py index d6b2a5e4..00d52050 100644 --- a/u19_pipeline/microsocope.py +++ b/u19_pipeline/microsocope.py @@ -1,6 +1,7 @@ import datajoint as dj -schema = dj.schema(dj.config['custom']['database.prefix'] + 'microscope') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "microscope") + @schema class Center(dj.Lookup): @@ -9,9 +10,8 @@ class Center(dj.Lookup): --- description = "" : varchar(500) """ - contents = [ - ["Bezos Center",""],["McDonnell Center",""] - ] + contents = [["Bezos Center", ""], ["McDonnell Center", ""]] + @schema class Microscope(dj.Lookup): @@ -25,6 +25,7 @@ class Microscope(dj.Lookup): microscope_description='' : varchar(2047) """ + @schema class Laser(dj.Manual): definition = """ @@ -138,9 +139,9 @@ class DaqSystemType(dj.Lookup): daq_notes='': varchar(255) """ contents = [ - ['National Instrument PXI', ''], - ['National Instrument PCI', ''], - ['Vidrio hardware', ''] + ["National Instrument PXI", ""], + ["National Instrument PCI", ""], + ["Vidrio hardware", ""], ] @@ -256,6 +257,7 @@ class AcquisitionSoftwareStatus(dj.Manual): -> AcquisitionSoftware """ + @schema class PsfStackData(dj.Manual): definition = """ diff --git a/u19_pipeline/puffs.py b/u19_pipeline/puffs.py index 3394d129..d52df1c8 100644 --- a/u19_pipeline/puffs.py +++ b/u19_pipeline/puffs.py @@ -2,7 +2,7 @@ import datajoint as dj -schema = dj.schema(dj.config['custom']['database.prefix'] + 'puffs') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "puffs") @schema @@ -13,10 +13,7 @@ class Rig(dj.Lookup): -> lab.location """ # "wang-behavior" - contents = [ - [0, "pni-ltl016-05"], - [1, "wang-behavior"] - ] + contents = [[0, "pni-ltl016-05"], [1, "wang-behavior"]] @schema diff --git a/u19_pipeline/pupillometry.py b/u19_pipeline/pupillometry.py index 40efaa0a..b5606e17 100644 --- a/u19_pipeline/pupillometry.py +++ b/u19_pipeline/pupillometry.py @@ -2,7 +2,7 @@ from u19_pipeline import acquisition -schema = dj.schema(dj.config['custom']['database.prefix'] + 'pupillometry') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "pupillometry") @schema @@ -26,10 +26,10 @@ class PupillometrySession(dj.Imported): -> acquisition.Session --- """ - + @property def key_source(self): - return acquisition.SessionVideo & {'video_type': 'pupillometry'} + return acquisition.SessionVideo & {"video_type": "pupillometry"} def make(self, key): pass @@ -46,7 +46,7 @@ class PupillometrySyncBehavior(dj.Imported): """ def make(self, key): - pass + pass @schema @@ -69,4 +69,4 @@ class PupillometrySessionModelData(dj.Imported): """ def make(self, key): - pass \ No newline at end of file + pass diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index ddcbdaf6..eb6c2719 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -1,14 +1,14 @@ - import datajoint as dj import u19_pipeline.automatic_job.params_config as config -schema = dj.schema(dj.config['custom']['database.prefix'] + 'recording') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "recording") + # Declare recording tables ------------------------------------------------------------- @schema class Modality(dj.Lookup): - definition = """ + definition = """ recording_modality: varchar(64) # recording modalities # (ephys, imaging, video_recording, etc.) --- @@ -26,22 +26,22 @@ class Modality(dj.Lookup): process_repository: varchar(64) # Name of the repository that handles the # processing of these modality """ - contents = config.recording_modality_list + contents = config.recording_modality_list @schema class Status(dj.Lookup): - definition = """ + definition = """ status_recording_id: TINYINT(1) # Status in the automatic processing pipeline --- status_recording_definition: VARCHAR(256) # Status definition """ - contents = config.recording_status_list + contents = config.recording_status_list @schema class Recording(dj.Manual): - definition = """ + definition = """ recording_id: INT(11) AUTO_INCREMENT # Unique number assigned to recording --- -> Modality @@ -50,17 +50,17 @@ class Recording(dj.Manual): task_copy_id_pni=null: int(11) # globus transfer task raw file local->cup recording_directory: varchar(255) # relative directory on cup local_directory: varchar(255) # local directory where the recording is stored on system - """ + """ - class BehaviorSession(dj.Part): - definition = """ + class BehaviorSession(dj.Part): + definition = """ -> master --- -> acquisition.Session """ - - class RecordingSession(dj.Part): - definition = """ + + class RecordingSession(dj.Part): + definition = """ -> master --- -> subject.Subject @@ -70,7 +70,7 @@ class RecordingSession(dj.Part): @schema class LogStatus(dj.Manual): - definition = """ + definition = """ recording_log_id: INT(11) AUTO_INCREMENT # Unique number assigned to each change # of status for all recordings --- @@ -85,7 +85,7 @@ class LogStatus(dj.Manual): @schema class DefaultParams(dj.Manual): - definition = """ + definition = """ -> Recording fragment_number: TINYINT(1) # probe/field_of_view # if not always the same ----- @@ -95,50 +95,70 @@ class DefaultParams(dj.Manual): paramset_idx: INT(11) # params index for recording (could be imaging/ephys) """ - @staticmethod - def get_default_params_rec_process(recording_processes, default_params_record_df): - 'Get associated params from DefaultParams record and recording processes (jobs) of recording' - - params_rec_process = list() - for i in recording_processes: - - this_params_rec_process = dict() - this_params_rec_process['job_id'] = i['job_id'] - this_fragment = i['fragment_number'] - - this_params_rec_process['preprocess_param_steps_id'] = \ - DefaultParams.get_corresponding_param(default_params_record_df, this_fragment, 'default_same_preparams_all', 'preprocess_param_steps_id') - - this_params_rec_process['paramset_idx'] = \ - DefaultParams.get_corresponding_param(default_params_record_df, this_fragment, 'default_same_params_all', 'paramset_idx') - - params_rec_process.append(this_params_rec_process) - - return params_rec_process - - - @staticmethod - def get_corresponding_param(default_params_record_df, this_fragment, default_label, param_label): - 'Get corresponding param (preprocess_param_steps_id or paramset_idx) for this fragment' - - 'default_label = default_same_preparams_all / default_same_params_all ' - 'param_label = preprocess_param_steps_id / paramset_idx ' - - #If there is no default params for this recording, get default ones (0 id) - if default_params_record_df.shape[0] == 0: - this_fragment_preprocess_param_steps_id = 0 - return this_fragment_preprocess_param_steps_id - - else: - if default_params_record_df.loc[0, default_label] == 1: - this_fragment_preprocess_param_steps_id = default_params_record_df.loc[0, param_label] - else: - this_fragment_preprocess_param_steps_id = \ - default_params_record_df.loc[default_params_record_df['fragment_number'] == this_fragment, param_label] - #If there is no list id for this specific fragment, get default one - if this_fragment_preprocess_param_steps_id.shape[0] == 0: - this_fragment_preprocess_param_steps_id = default_params_record_df.loc[0, param_label] - else: - this_fragment_preprocess_param_steps_id = this_fragment_preprocess_param_steps_id.values[0] - - return this_fragment_preprocess_param_steps_id + @staticmethod + def get_default_params_rec_process(recording_processes, default_params_record_df): + "Get associated params from DefaultParams record and recording processes (jobs) of recording" + + params_rec_process = list() + for i in recording_processes: + this_params_rec_process = dict() + this_params_rec_process["job_id"] = i["job_id"] + this_fragment = i["fragment_number"] + + this_params_rec_process["preprocess_param_steps_id"] = ( + DefaultParams.get_corresponding_param( + default_params_record_df, + this_fragment, + "default_same_preparams_all", + "preprocess_param_steps_id", + ) + ) + + this_params_rec_process["paramset_idx"] = ( + DefaultParams.get_corresponding_param( + default_params_record_df, + this_fragment, + "default_same_params_all", + "paramset_idx", + ) + ) + + params_rec_process.append(this_params_rec_process) + + return params_rec_process + + @staticmethod + def get_corresponding_param( + default_params_record_df, this_fragment, default_label, param_label + ): + "Get corresponding param (preprocess_param_steps_id or paramset_idx) for this fragment" + + "default_label = default_same_preparams_all / default_same_params_all " + "param_label = preprocess_param_steps_id / paramset_idx " + + # If there is no default params for this recording, get default ones (0 id) + if default_params_record_df.shape[0] == 0: + this_fragment_preprocess_param_steps_id = 0 + return this_fragment_preprocess_param_steps_id + + else: + if default_params_record_df.loc[0, default_label] == 1: + this_fragment_preprocess_param_steps_id = default_params_record_df.loc[ + 0, param_label + ] + else: + this_fragment_preprocess_param_steps_id = default_params_record_df.loc[ + default_params_record_df["fragment_number"] == this_fragment, + param_label, + ] + # If there is no list id for this specific fragment, get default one + if this_fragment_preprocess_param_steps_id.shape[0] == 0: + this_fragment_preprocess_param_steps_id = ( + default_params_record_df.loc[0, param_label] + ) + else: + this_fragment_preprocess_param_steps_id = ( + this_fragment_preprocess_param_steps_id.values[0] + ) + + return this_fragment_preprocess_param_steps_id diff --git a/u19_pipeline/recording_process.py b/u19_pipeline/recording_process.py index bb60c46d..22fe38f0 100644 --- a/u19_pipeline/recording_process.py +++ b/u19_pipeline/recording_process.py @@ -1,26 +1,26 @@ - import pathlib import datajoint as dj import u19_pipeline.automatic_job.params_config as config -schema = dj.schema(dj.config['custom']['database.prefix'] + 'recording_process') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "recording_process") + # Declare recording processing tables -------------------------------------------------- @schema class Status(dj.Lookup): - definition = """ + definition = """ status_processing_id: TINYINT(1) # Status in the automatic processing pipeline --- status_processing_definition: VARCHAR(256) # Status definition """ - contents = config.recording_process_status_list + contents = config.recording_process_status_list @schema class Processing(dj.Manual): - definition = """ + definition = """ job_id: INT(11) AUTO_INCREMENT # Unique number assigned to each # processing job for a recording @@ -41,66 +41,72 @@ class Processing(dj.Manual): task_copy_id_post=null: UUID # id for globus transfer task # sorted file tiger->cup slurm_id=null: VARCHAR(16) # id for slurm process in tiger - """ + """ - class EphysParams(dj.Part): - definition=""" + class EphysParams(dj.Part): + definition = """ -> master --- -> ephys_element.PreClusterParamSteps -> ephys_element.ClusteringParamSet.proj(cluster_paramset_idx='paramset_idx') """ - class ImagingParams(dj.Part): - definition=""" + class ImagingParams(dj.Part): + definition = """ -> master --- -> imaging_element.PreProcessParamSteps -> imaging_element.ProcessingParamSet - """ + """ - # This table would control ingestion of PreClusteringTask - def insert_recording_process(self, fragment_keys, fragment_fieldname): - ''' + # This table would control ingestion of PreClusteringTask + def insert_recording_process(self, fragment_keys, fragment_fieldname): + """ # Insert RecordingTask(s) from recording. # For each processing "unit" of a recording add a new RecordingTask (imaging ->field of view, electrophysiology->probe) Input: recording_key (dict) = Dictionary with recording record rec_unit (dict) = Dictionary of recording "unit" to be processed unit_fieldname (str) = Unit fieldname to be read (ephys-> probe_, imaging->fov) - ''' + """ # Get directory fieldname for specific modality (probe_directory, fov_directory, etc.) # Append data for the unit to insert this_recprocess_key = list() for idx, fragment_key in enumerate(fragment_keys): - this_recprocess_key.append(dict()) - this_recprocess_key[idx]['recording_id'] = fragment_key['recording_id'] - this_recprocess_key[idx]['fragment_number'] = fragment_key[fragment_fieldname] - this_recprocess_key[idx]['status_processing_id'] = 0 - this_recprocess_key[idx]['recording_process_pre_path'] = fragment_key['recording_process_pre_path'] + this_recprocess_key.append(dict()) + this_recprocess_key[idx]["recording_id"] = fragment_key["recording_id"] + this_recprocess_key[idx]["fragment_number"] = fragment_key[ + fragment_fieldname + ] + this_recprocess_key[idx]["status_processing_id"] = 0 + this_recprocess_key[idx]["recording_process_pre_path"] = fragment_key[ + "recording_process_pre_path" + ] self.insert(this_recprocess_key) - def set_recording_process_post_path(self, recprocess_keys): - ''' - # Update recording_process_post_path from recording_process keys - Input: - recprocess_keys (dict) = Dictionary with job_ids & recording_process_pre_path - ''' + def set_recording_process_post_path(self, recprocess_keys): + """ + # Update recording_process_post_path from recording_process keys + Input: + recprocess_keys (dict) = Dictionary with job_ids & recording_process_pre_path + """ - for rec_process in recprocess_keys: - this_key_dict = dict() - this_key_dict['job_id'] = rec_process['job_id'] - this_key_dict['recording_process_post_path'] = \ - pathlib.Path(rec_process['recording_process_pre_path'], 'job_id_' + str(rec_process['job_id'])).as_posix() - self.update1(this_key_dict) + for rec_process in recprocess_keys: + this_key_dict = dict() + this_key_dict["job_id"] = rec_process["job_id"] + this_key_dict["recording_process_post_path"] = pathlib.Path( + rec_process["recording_process_pre_path"], + "job_id_" + str(rec_process["job_id"]), + ).as_posix() + self.update1(this_key_dict) @schema class LogStatus(dj.Manual): - definition = """ + definition = """ log_id: INT(11) AUTO_INCREMENT # Unique number assigned to each change # of status for all processing jobs --- diff --git a/u19_pipeline/reference.py b/u19_pipeline/reference.py index 5539a19a..d72b574f 100644 --- a/u19_pipeline/reference.py +++ b/u19_pipeline/reference.py @@ -1,9 +1,8 @@ """This module defines tables in the schema U19_reference.""" - import datajoint as dj -schema = dj.schema(dj.config['custom']['database.prefix'] + 'reference') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "reference") @schema @@ -14,10 +13,10 @@ class BrainArea(dj.Lookup): area_full_name="" : varchar(128) """ contents = [ - ['Hippocampus', ''], - ['Striatum', ''], - ['PPC', 'Posterior Parietal Cortex'], - ['EC', 'Entorhinal Cortex'] + ["Hippocampus", ""], + ["Striatum", ""], + ["PPC", "Posterior Parietal Cortex"], + ["EC", "Entorhinal Cortex"], ] @@ -26,15 +25,9 @@ class VirusSource(dj.Lookup): definition = """ virus_source : varchar(32) """ - contents = zip([ - 'UNC', - 'UPenn', - 'Addgene', - 'MIT', - 'Stanford', - 'Princeton', - 'custom' - ]) + contents = zip( + ["UNC", "UPenn", "Addgene", "MIT", "Stanford", "Princeton", "custom"] + ) @schema @@ -42,12 +35,7 @@ class VirusType(dj.Lookup): definition = """ virus_type : varchar(64) """ - contents = zip([ - 'AAV', - 'Rabies', - 'Psedotyped rabies', - 'Lenti' - ]) + contents = zip(["AAV", "Rabies", "Psedotyped rabies", "Lenti"]) @schema @@ -65,14 +53,74 @@ class Virus(dj.Lookup): virus_description="" : varchar(512) """ contents = [ - ['jRCaMP1a', 1, 'AAV1.Syn.NES.jRCaMP1a.WPRE.SV40', 'AAV', 'Upenn', '', 33.6, None, ''], - ['GCaMP6f', 1, 'AAV1.Syn.GCaMP6f.WPRE.SV40', 'AAV', 'UPenn', '', 26.5, None, ''], - ['Syn.RFP', 1, 'AAV5.hSyn.TurboRFP.WPRE.rBG', 'AAV', 'UPenn', '', 44, None, ''], - ['GFAP.GFP', 1, 'AAV5.GFAP.eGFP.WPRE.hGH', 'AAV', 'UPenn', '', 10.6, None, ''], - ['CamKII.GFP', 1, 'AAV9.CamKII0.4.eGFP.WPRE.rBG', 'AAV', 'UPenn', '', 34.9, None, ''], - ['Syn.RFP', 1, 'AAV9.hSyn.TurboRFP.WPRE.rBG', 'AAV', 'UPenn', '', 66.4, None, ''], - ['ChR2(H134R)-YFP', 1, 'AAV9.hSyn.hChR2(H134R)-eYFP.WPRE.hGH', 'AAV', 'UPenn', '', 33.9, None, ''], - ['Chronos-GFP', 1, 'AAV9.Syn.Chronos-GFP.WPRE.bGH', 'AAV', 'UPenn', '', 35.1, None, ''] + [ + "jRCaMP1a", + 1, + "AAV1.Syn.NES.jRCaMP1a.WPRE.SV40", + "AAV", + "Upenn", + "", + 33.6, + None, + "", + ], + [ + "GCaMP6f", + 1, + "AAV1.Syn.GCaMP6f.WPRE.SV40", + "AAV", + "UPenn", + "", + 26.5, + None, + "", + ], + ["Syn.RFP", 1, "AAV5.hSyn.TurboRFP.WPRE.rBG", "AAV", "UPenn", "", 44, None, ""], + ["GFAP.GFP", 1, "AAV5.GFAP.eGFP.WPRE.hGH", "AAV", "UPenn", "", 10.6, None, ""], + [ + "CamKII.GFP", + 1, + "AAV9.CamKII0.4.eGFP.WPRE.rBG", + "AAV", + "UPenn", + "", + 34.9, + None, + "", + ], + [ + "Syn.RFP", + 1, + "AAV9.hSyn.TurboRFP.WPRE.rBG", + "AAV", + "UPenn", + "", + 66.4, + None, + "", + ], + [ + "ChR2(H134R)-YFP", + 1, + "AAV9.hSyn.hChR2(H134R)-eYFP.WPRE.hGH", + "AAV", + "UPenn", + "", + 33.9, + None, + "", + ], + [ + "Chronos-GFP", + 1, + "AAV9.Syn.Chronos-GFP.WPRE.bGH", + "AAV", + "UPenn", + "", + 35.1, + None, + "", + ], ] @@ -86,7 +134,6 @@ class Template(dj.Lookup): database_field_names : longblob # cell array for database field names """ - class RightNow(dj.Part): definition = """ # ActionItems template for the information of each column @@ -96,7 +143,6 @@ class RightNow(dj.Part): plot_index : int # index of where to plot in GUI """ - class Genotype(dj.Part): definition = """ # ActionItems template for the information of each column @@ -106,7 +152,6 @@ class Genotype(dj.Part): plot_index : int # index of where to plot in GUI """ - class DailyInfo(dj.Part): definition = """ # Daily Info template for the information of each column @@ -124,7 +169,6 @@ class DailyInfo(dj.Part): plot_index : int # index of where to plot in GUI """ - class Animal(dj.Part): definition = """ # Animal template for the information of each column @@ -141,7 +185,6 @@ class Animal(dj.Part): plot_index : int # index of where to plot in GUI """ - class ActionItems(dj.Part): definition = """ # ActionItems template for the information of each column diff --git a/u19_pipeline/rig_maintenance.py b/u19_pipeline/rig_maintenance.py index 504f5d1c..c93f2b0b 100644 --- a/u19_pipeline/rig_maintenance.py +++ b/u19_pipeline/rig_maintenance.py @@ -28,10 +28,22 @@ class MaintenanceType(dj.Lookup): contents = [ ["Replacing lines", "Replacing water/reward lines", 30, 7, 1], ["Replacing spheres", "Replacing spheres in the rig", 60, 7, 0], - ["General Calibration", "General calibration (always after changing lines or solenoids)", 30, 7, 0], + [ + "General Calibration", + "General calibration (always after changing lines or solenoids)", + 30, + 7, + 0, + ], ["Replacing solenoid valve", "Replacing solenoid valve", 365, 30, 1], ["Deep Clean solenoid valve", "Deep cleaning of solenoid valve", 30, 7, 1], - ["Replacing reward lines connectors", "Replacing reward lines connectors", 180, 14, 0], + [ + "Replacing reward lines connectors", + "Replacing reward lines connectors", + 180, + 14, + 0, + ], ] diff --git a/u19_pipeline/scheduler.py b/u19_pipeline/scheduler.py index ea794671..aa89aa2a 100644 --- a/u19_pipeline/scheduler.py +++ b/u19_pipeline/scheduler.py @@ -125,6 +125,7 @@ class InputOutputProfileList(dj.Manual): check_type : enum('Mandatory','Optional') # Prevent training if missing this input/output """ + @schema class RigStatus(dj.Manual): definition = """ @@ -137,6 +138,7 @@ class RigStatus(dj.Manual): last_status_update : datetime # at what time status changed """ + @schema class Shift(dj.Lookup): definition = """ diff --git a/u19_pipeline/subject.py b/u19_pipeline/subject.py index bf07ba7a..dbb6a1ee 100644 --- a/u19_pipeline/subject.py +++ b/u19_pipeline/subject.py @@ -1,6 +1,5 @@ """This module defines tables in the schema U19_subject""" - import datajoint as dj prefix = dj.config["custom"]["database.prefix"] @@ -22,9 +21,7 @@ class Species(dj.Lookup): --- species_nickname : varchar(32) # nickname """ - contents = [ - ['Mus musculus', 'Laboratory mouse'] - ] + contents = [["Mus musculus", "Laboratory mouse"]] @schema @@ -34,11 +31,7 @@ class Source(dj.Lookup): --- source_description="" : varchar(255) """ - contents = [ - ['Jax Lab', ''], - ['Princeton', ''], - ['Allen Institute', ''] - ] + contents = [["Jax Lab", ""], ["Princeton", ""], ["Allen Institute", ""]] @schema @@ -48,9 +41,7 @@ class Strain(dj.Lookup): --- strain_description="" : varchar(255) # description """ - contents = [ - ['C57BL6/J', ''] - ] + contents = [["C57BL6/J", ""]] @schema @@ -64,16 +55,40 @@ class Allele(dj.Lookup): allele_description="" : varchar(1023) """ contents = [ - ['Thy1-GP5.3', 'Thy1-GCaMP6f', 'Jax Lab', 'David W. Tank, Princeton University', ''], - ['Ai93', '(TITL-GCaMP6f)-D', 'Jax Lab', 'Allen Institute', ''], - ['Ai148', '(TIT2L-GC6f-ICL-tTA2)-D', 'Jax Lab', 'Allen Institute', ''], - ['Thy1-YFP', '', 'Jax Lab', 'Joshua R Sanes, Harvard University', ''], - ['VGAT-ChR2-EYFP', 'Slc32a1-COP4*H134R/EYFP', 'Jax Lab', 'Guoping Feng, Massachusetts Institute of Technology', ''], - ['Emx1-Cre', 'Emx-IRES-Cre', 'Jax Lab', 'Kevin R. Jones, University of Colorado- Boulder', ''], - ['D1-Cre', 'Drd1-cre', 'Jax Lab', 'Ming Xu, University of Chicago', ''], - ['D2-Cre', 'Drd2-cre', 'Jax Lab', 'Unknown', ''], - ['DAT-IRES-Cre', 'Slc6a3tm1.1(cre)Bkmn', 'Jax Lab', 'Cristina M Backman, National Institute on Drug Abuse (NIH)', ''], - ['Slc17a7-IRES2-Cre', '', 'Jax Lab', 'Allen Institute', ''] + [ + "Thy1-GP5.3", + "Thy1-GCaMP6f", + "Jax Lab", + "David W. Tank, Princeton University", + "", + ], + ["Ai93", "(TITL-GCaMP6f)-D", "Jax Lab", "Allen Institute", ""], + ["Ai148", "(TIT2L-GC6f-ICL-tTA2)-D", "Jax Lab", "Allen Institute", ""], + ["Thy1-YFP", "", "Jax Lab", "Joshua R Sanes, Harvard University", ""], + [ + "VGAT-ChR2-EYFP", + "Slc32a1-COP4*H134R/EYFP", + "Jax Lab", + "Guoping Feng, Massachusetts Institute of Technology", + "", + ], + [ + "Emx1-Cre", + "Emx-IRES-Cre", + "Jax Lab", + "Kevin R. Jones, University of Colorado- Boulder", + "", + ], + ["D1-Cre", "Drd1-cre", "Jax Lab", "Ming Xu, University of Chicago", ""], + ["D2-Cre", "Drd2-cre", "Jax Lab", "Unknown", ""], + [ + "DAT-IRES-Cre", + "Slc6a3tm1.1(cre)Bkmn", + "Jax Lab", + "Cristina M Backman, National Institute on Drug Abuse (NIH)", + "", + ], + ["Slc17a7-IRES2-Cre", "", "Jax Lab", "Allen Institute", ""], ] @@ -85,11 +100,11 @@ class SequenceType(dj.Lookup): seq_type_description="" : varchar(255) """ contents = [ - ['calcium sensor', ''], - ['optogenetics', ''], - ['promoter', ''], - ['recombinase', ''], - ['fluorescent protein', ''] + ["calcium sensor", ""], + ["optogenetics", ""], + ["promoter", ""], + ["recombinase", ""], + ["fluorescent protein", ""], ] @@ -103,15 +118,15 @@ class Sequence(dj.Lookup): sequence_description="" : varchar(255) """ contents = [ - ['GCaMP6f', 'calcium sensor', '', ''], - ['GCaMP6s', 'calcium sensor', '', ''], - ['ChR2', 'optogenetics', '', ''], - ['EYFP', 'fluorescent protein', '', ''], - ['Thy1', 'promoter', '', ''], - ['Emx1', 'promoter', '', ''], - ['Cre', 'recombinase', '', ''], - ['D1', 'promoter', '', 'dopamine receptor type 1'], - ['D2', 'promoter', '', 'dopamine receptor type 2'] + ["GCaMP6f", "calcium sensor", "", ""], + ["GCaMP6s", "calcium sensor", "", ""], + ["ChR2", "optogenetics", "", ""], + ["EYFP", "fluorescent protein", "", ""], + ["Thy1", "promoter", "", ""], + ["Emx1", "promoter", "", ""], + ["Cre", "recombinase", "", ""], + ["D1", "promoter", "", "dopamine receptor type 1"], + ["D2", "promoter", "", "dopamine receptor type 2"], ] @@ -122,19 +137,19 @@ class AlleleSequence(dj.Lookup): -> Sequence """ contents = [ - ['Thy1-YFP', 'Thy1'], - ['Thy1-YFP', 'EYFP'], - ['Thy1-GP5.3', 'GCaMP6f'], - ['VGAT-ChR2-EYFP', 'EYFP'], - ['VGAT-ChR2-EYFP', 'ChR2'], - ['Emx1-Cre', 'Emx1'], - ['Emx1-Cre', 'Cre'], - ['D1-Cre', 'D1'], - ['D1-Cre', 'Cre'], - ['D2-Cre', 'D2'], - ['D2-Cre', 'Cre'], - ['DAT-IRES-Cre', 'Cre'], - ['Slc17a7-IRES2-Cre', 'Cre'] + ["Thy1-YFP", "Thy1"], + ["Thy1-YFP", "EYFP"], + ["Thy1-GP5.3", "GCaMP6f"], + ["VGAT-ChR2-EYFP", "EYFP"], + ["VGAT-ChR2-EYFP", "ChR2"], + ["Emx1-Cre", "Emx1"], + ["Emx1-Cre", "Cre"], + ["D1-Cre", "D1"], + ["D1-Cre", "Cre"], + ["D2-Cre", "D2"], + ["D2-Cre", "Cre"], + ["DAT-IRES-Cre", "Cre"], + ["Slc17a7-IRES2-Cre", "Cre"], ] @@ -150,17 +165,17 @@ class Line(dj.Lookup): is_active=1 : tinyint # is active """ contents = [ - ['Unknown', 'Mus musculus', 'C57BL6/J', '', '', 1], - ['C57BL6/J', 'Mus musculus', 'C57BL6/J', 'wild-type mice', '', 1], - ['VGAT-ChR2-EYFP', 'Mus musculus', 'C57BL6/J', '', '', 1], - ['Ai93-Emx1', 'Mus musculus', 'C57BL6/J', '', '', 1], - ['Thy1-GP5.3', 'Mus musculus', 'C57BL6/J', '', '', 1], - ['Thy1-YFP', 'Mus musculus', 'C57BL6/J', '', '', 1], - ['DAT-IRES-CRE', 'Mus musculus', 'C57BL6/J', '', '', 1], - ['DAT-Ai148', 'Mus musculus', 'C57BL6/J', '', '', 1], - ['Slc17a7-Ai148', 'Mus musculus', 'C57BL6/J', '', '', 1], - ['D1-CRE', 'Mus musculus', 'C57BL6/J', '', '', 1], - ['D2-CRE', 'Mus musculus', 'C57BL6/J', '', '', 1] + ["Unknown", "Mus musculus", "C57BL6/J", "", "", 1], + ["C57BL6/J", "Mus musculus", "C57BL6/J", "wild-type mice", "", 1], + ["VGAT-ChR2-EYFP", "Mus musculus", "C57BL6/J", "", "", 1], + ["Ai93-Emx1", "Mus musculus", "C57BL6/J", "", "", 1], + ["Thy1-GP5.3", "Mus musculus", "C57BL6/J", "", "", 1], + ["Thy1-YFP", "Mus musculus", "C57BL6/J", "", "", 1], + ["DAT-IRES-CRE", "Mus musculus", "C57BL6/J", "", "", 1], + ["DAT-Ai148", "Mus musculus", "C57BL6/J", "", "", 1], + ["Slc17a7-Ai148", "Mus musculus", "C57BL6/J", "", "", 1], + ["D1-CRE", "Mus musculus", "C57BL6/J", "", "", 1], + ["D2-CRE", "Mus musculus", "C57BL6/J", "", "", 1], ] @@ -171,18 +186,18 @@ class LineAllele(dj.Lookup): -> Allele """ contents = [ - ['VGAT-ChR2-EYFP', 'VGAT-ChR2-EYFP'], - ['Ai93-Emx1', 'Ai93'], - ['Ai93-Emx1', 'Emx1-Cre'], - ['Thy1-GP5.3', 'Thy1-GP5.3'], - ['Thy1-YFP', 'Thy1-YFP'], - ['DAT-IRES-CRE', 'DAT-IRES-Cre'], - ['DAT-Ai148', 'DAT-IRES-Cre'], - ['DAT-Ai148', 'Ai148'], - ['Slc17a7-Ai148', 'Ai148'], - ['Slc17a7-Ai148', 'Slc17a7-IRES2-Cre'], - ['D1-CRE', 'D1-Cre'], - ['D2-CRE', 'D2-Cre'] + ["VGAT-ChR2-EYFP", "VGAT-ChR2-EYFP"], + ["Ai93-Emx1", "Ai93"], + ["Ai93-Emx1", "Emx1-Cre"], + ["Thy1-GP5.3", "Thy1-GP5.3"], + ["Thy1-YFP", "Thy1-YFP"], + ["DAT-IRES-CRE", "DAT-IRES-Cre"], + ["DAT-Ai148", "DAT-IRES-Cre"], + ["DAT-Ai148", "Ai148"], + ["Slc17a7-Ai148", "Ai148"], + ["Slc17a7-Ai148", "Slc17a7-IRES2-Cre"], + ["D1-CRE", "D1-Cre"], + ["D2-CRE", "D2-Cre"], ] @@ -210,11 +225,15 @@ class ActItem(dj.Lookup): definition = """ act_item : varchar(64) # possible act item """ - contents = zip(['Post-op painkillers', - 'Post-op monitoring', - 'Acclimatize to humans', - 'Topical antibiotics for headplate scab', - 'Needs fattening']) + contents = zip( + [ + "Post-op painkillers", + "Post-op monitoring", + "Acclimatize to humans", + "Topical antibiotics for headplate scab", + "Needs fattening", + ] + ) @schema @@ -224,6 +243,7 @@ class SubjectActionManual(dj.Manual): -> ActItem """ + @schema class SubjectActionAutomatic(dj.Manual): definition = """ @@ -233,6 +253,7 @@ class SubjectActionAutomatic(dj.Manual): notification_message : varchar(255) # Notification message e.g. low bodyweight warning """ + @schema class SubjectProject(dj.Manual): definition = """ @@ -295,6 +316,7 @@ class CagingStatus(dj.Manual): -> Cage """ + @schema class HealthStatus(dj.Manual): definition = """ @@ -310,7 +332,6 @@ class HealthStatus(dj.Manual): comments=null : varchar(255) """ - class Action(dj.Part): definition = """ -> HealthStatus @@ -375,6 +396,7 @@ class Zygosity(dj.Manual): zygosity : enum('Present','Absent','Homozygous','Heterozygous') """ + @schema class HeadMotorPosition(dj.Manual): definition = """ diff --git a/u19_pipeline/task.py b/u19_pipeline/task.py index 9a422388..ae421e1f 100644 --- a/u19_pipeline/task.py +++ b/u19_pipeline/task.py @@ -2,7 +2,7 @@ import datajoint as dj -schema = dj.schema(dj.config['custom']['database.prefix'] + 'task') +schema = dj.schema(dj.config["custom"]["database.prefix"] + "task") @schema @@ -12,12 +12,7 @@ class Task(dj.Lookup): --- task_description="" : varchar(512) """ - contents = [ - ['AirPuffs', ''], - ['Towers', ''], - ['Clicks', ''], - ['LinearTrack', ''] - ] + contents = [["AirPuffs", ""], ["Towers", ""], ["Clicks", ""], ["LinearTrack", ""]] @schema diff --git a/u19_pipeline/temp/acquisition.py b/u19_pipeline/temp/acquisition.py index 4d14f722..1169de95 100644 --- a/u19_pipeline/temp/acquisition.py +++ b/u19_pipeline/temp/acquisition.py @@ -1,6 +1,6 @@ import datajoint as dj -schema = dj.schema('u19_acquisition') +schema = dj.schema("u19_acquisition") @schema diff --git a/u19_pipeline/temp/behavior.py b/u19_pipeline/temp/behavior.py index 47354b82..a7c0cd4f 100644 --- a/u19_pipeline/temp/behavior.py +++ b/u19_pipeline/temp/behavior.py @@ -1,14 +1,13 @@ """This module was auto-generated by datajoint from an existing schema""" - import datajoint as dj -schema = dj.schema('u19_behavior_temp') +schema = dj.schema("u19_behavior_temp") -vmod0 = dj.create_virtual_module('vmod0', 'u19_acquisition') -vmod1 = dj.create_virtual_module('vmod1', 'u19_task') -vmod2 = dj.create_virtual_module('vmod2', 'u19_subject') +vmod0 = dj.create_virtual_module("vmod0", "u19_acquisition") +vmod1 = dj.create_virtual_module("vmod1", "u19_task") +vmod2 = dj.create_virtual_module("vmod2", "u19_subject") @schema diff --git a/u19_pipeline/temp/imaging.py b/u19_pipeline/temp/imaging.py index a644d9e6..bef752e7 100644 --- a/u19_pipeline/temp/imaging.py +++ b/u19_pipeline/temp/imaging.py @@ -1,10 +1,9 @@ """This module was auto-generated by datajoint from an existing schema""" - import datajoint as dj -schema = dj.schema('u19_imaging_temp') -vmod0 = dj.create_virtual_module('vmod0', 'u19_acquisition') +schema = dj.schema("u19_imaging_temp") +vmod0 = dj.create_virtual_module("vmod0", "u19_acquisition") @schema diff --git a/u19_pipeline/temp/meso.py b/u19_pipeline/temp/meso.py index b7d94ef6..3a912f58 100644 --- a/u19_pipeline/temp/meso.py +++ b/u19_pipeline/temp/meso.py @@ -1,12 +1,11 @@ """This module was auto-generated by datajoint from an existing schema""" - import datajoint as dj -schema = dj.schema('u19_meso_temp') +schema = dj.schema("u19_meso_temp") -vmod0 = dj.create_virtual_module('vmod0', 'u19_acquisition') +vmod0 = dj.create_virtual_module("vmod0", "u19_acquisition") @schema diff --git a/u19_pipeline/temp/meso_analysis.py b/u19_pipeline/temp/meso_analysis.py index a1c7c5c8..d17fbde4 100644 --- a/u19_pipeline/temp/meso_analysis.py +++ b/u19_pipeline/temp/meso_analysis.py @@ -1,12 +1,11 @@ """This module was auto-generated by datajoint from an existing schema""" - import datajoint as dj -schema = dj.schema('u19_meso_analysis_temp') +schema = dj.schema("u19_meso_analysis_temp") -vmod0 = dj.create_virtual_module('vmod0', 'u19_meso') +vmod0 = dj.create_virtual_module("vmod0", "u19_meso") @schema diff --git a/u19_pipeline/utility.py b/u19_pipeline/utility.py index e82a1ae2..818b2cf0 100644 --- a/u19_pipeline/utility.py +++ b/u19_pipeline/utility.py @@ -1,5 +1,3 @@ - - import os import subprocess import sys @@ -23,8 +21,8 @@ def is_this_spock(): stdout, stderr = p.communicate() if p.returncode == 0: - hostname = stdout.decode('UTF-8') - if 'spock' in hostname or 'scotty' in hostname: + hostname = stdout.decode("UTF-8") + if "spock" in hostname or "scotty" in hostname: isSpock = True return isSpock @@ -35,24 +33,20 @@ def basic_dj_configuration(dj): Configure host and external storage locations for datajoint """ - dj.config['database.host'] = 'datajoint00.pni.princeton.edu' - dj.config['enable_python_native_blobs'] = True + dj.config["database.host"] = "datajoint00.pni.princeton.edu" + dj.config["enable_python_native_blobs"] = True if is_this_spock(): - ext_storage_location = '/mnt/bucket/u19_dj/external_dj_blobs' + ext_storage_location = "/mnt/bucket/u19_dj/external_dj_blobs" elif sys.platform == "darwin": - ext_storage_location = '/Volumes/u19_dj/external_dj_blobs' + ext_storage_location = "/Volumes/u19_dj/external_dj_blobs" elif sys.platform == "win32": - ext_storage_location = '//bucket.pni.princeton.edu/u19_dj/external_dj_blobs' + ext_storage_location = "//bucket.pni.princeton.edu/u19_dj/external_dj_blobs" elif sys.platform == "linux" or sys.platform == "linux2": - ext_storage_location = '/mnt/u19_dj/external_dj_blobs' - - dj.config['stores'] = { - 'extstorage': - { - 'location': ext_storage_location, - 'protocol': 'file' - } + ext_storage_location = "/mnt/u19_dj/external_dj_blobs" + + dj.config["stores"] = { + "extstorage": {"location": ext_storage_location, "protocol": "file"} } @@ -66,28 +60,31 @@ def get_network_path(path_name): """ key = dict() - # Check if path name to search starts with needed / at start - if path_name[0] != '/': - key['global_path'] = '/' + path_name + # Check if path name to search starts with needed / at start + if path_name[0] != "/": + key["global_path"] = "/" + path_name else: - key['global_path'] = path_name - - field_get = ['local_path'] + key["global_path"] = path_name + + field_get = ["local_path"] if is_this_spock(): - field_get = ['bucket_path'] - key['system'] = 'linux' + field_get = ["bucket_path"] + key["system"] = "linux" elif sys.platform == "darwin": - key['system'] = 'mac' - elif os.name == 'nt': - key['system'] = 'windows' + key["system"] = "mac" + elif os.name == "nt": + key["system"] = "windows" else: - key['system'] = 'linux' + key["system"] = "linux" - lab = dj.create_virtual_module('lab', dj.config['custom']['database.prefix']+'lab') + lab = dj.create_virtual_module( + "lab", dj.config["custom"]["database.prefix"] + "lab" + ) network_path = (lab.Path & key).fetch1(*field_get) return network_path + # Function copied in dj_shorts def smart_dj_join(t1, t2): """ @@ -96,12 +93,12 @@ def smart_dj_join(t1, t2): """ # Get all fields from tables - fields_t1 = pd.DataFrame.from_dict(t1.heading.attributes, orient='index') - fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient='index') + fields_t1 = pd.DataFrame.from_dict(t1.heading.attributes, orient="index") + fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient="index") # Get only secondary fields and check matches - fields_t1_list = set(fields_t1.loc[fields_t1['in_key'] == False].index.to_list()) - fields_t2_list = set(fields_t2.loc[fields_t2['in_key'] == False].index.to_list()) + fields_t1_list = set(fields_t1.loc[fields_t1["in_key"] == False].index.to_list()) + fields_t2_list = set(fields_t2.loc[fields_t2["in_key"] == False].index.to_list()) intersected_fields = fields_t2_list.intersection(fields_t1_list) # If there are: @@ -110,7 +107,7 @@ def smart_dj_join(t1, t2): suffix = t2.table_name new_name_attr_dict = dict() for i in intersected_fields: - new_name_attr_dict[suffix + '_' + i] = i + new_name_attr_dict[suffix + "_" + i] = i # List non matching ones non_intersected_fields = list(fields_t2_list - intersected_fields) @@ -128,7 +125,7 @@ def psychometrics_function(x, O, A, lambd, x0): """ Standard sigmoid function """ - return O + A/(1+np.exp(-(x-x0)/lambd)) + return O + A / (1 + np.exp(-(x - x0) / lambd)) def psychFit(deltaBins, numR, numL, choices): @@ -146,7 +143,7 @@ def psychFit(deltaBins, numR, numL, choices): nCues_RminusL = numR - numL # Correct deltaBin & trialBin to produce same result as Matlab psychFit deltaBins_search = deltaBins.astype(float) - 1.5 - trialBin = np.searchsorted(deltaBins_search, nCues_RminusL, side='right') + trialBin = np.searchsorted(deltaBins_search, nCues_RminusL, side="right") trialBin -= 1 # Put into evidence bins all Trials with corresponding choices @@ -155,9 +152,11 @@ def psychFit(deltaBins, numR, numL, choices): if choices[iTrial] == 2: numRight[trialBin[iTrial]] = numRight[trialBin[iTrial]] + 1 - trialDelta[trialBin[iTrial]] = trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial] + trialDelta[trialBin[iTrial]] = ( + trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial] + ) - with np.errstate(divide='ignore', invalid='ignore'): + with np.errstate(divide="ignore", invalid="ignore"): trialDelta = np.true_divide(trialDelta, numTrials) # Select only bins with trials @@ -166,8 +165,12 @@ def psychFit(deltaBins, numR, numL, choices): numRight_nz = numRight[~idx_zero] # (Binomial proportion confidence interval given k successes, n trials) - phat_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=0, interval='jeffreys') - pci_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=1 - 0.1587, interval='jeffreys') + phat_nz = binom_conf_interval( + numRight_nz, numTrials_nz, confidence_level=0, interval="jeffreys" + ) + pci_nz = binom_conf_interval( + numRight_nz, numTrials_nz, confidence_level=1 - 0.1587, interval="jeffreys" + ) # Correct confidence intervals and expected outcomes for bins with no trials (ci = [0 1], hat = 0.5) phat_nz = phat_nz[0] @@ -191,8 +194,14 @@ def psychFit(deltaBins, numR, numL, choices): weight_array = np.power((pci[1][~idx_zero] - pci[0][~idx_zero]) / 2, 2) sigma_fit = np.diag(weight_array) - psychometric, pcov = curve_fit(psychometrics_function, deltaBins[~idx_zero], phat[~idx_zero], \ - p0=(0, 1, 3, 0), sigma=sigma_fit, maxfev=40000) + psychometric, pcov = curve_fit( + psychometrics_function, + deltaBins[~idx_zero], + phat[~idx_zero], + p0=(0, 1, 3, 0), + sigma=sigma_fit, + maxfev=40000, + ) # Append a row of nans to confidence intervals . whyy ?? aux_vec = np.empty((1, pci.shape[1])) @@ -211,18 +220,18 @@ def psychFit(deltaBins, numR, numL, choices): # Fill dictionary of results fit_results = dict() - fit_results['delta_bins'] = deltaBins[~idx_zero] - fit_results['delta_data'] = trialDelta[~idx_zero] - fit_results['pright_data'] = 100 * phat[~idx_zero] - fit_results['delta_error'] = errorX - fit_results['pright_error'] = 100 * errorY + fit_results["delta_bins"] = deltaBins[~idx_zero] + fit_results["delta_data"] = trialDelta[~idx_zero] + fit_results["pright_data"] = 100 * phat[~idx_zero] + fit_results["delta_error"] = errorX + fit_results["pright_error"] = 100 * errorY if is_there_psychometric: - fit_results['delta_fit'] = delta - fit_results['pright_fit'] = psychometrics_function(delta, *psychometric) * 100 + fit_results["delta_fit"] = delta + fit_results["pright_fit"] = psychometrics_function(delta, *psychometric) * 100 else: - fit_results['delta_fit'] = np.empty([0]) - fit_results['pright_fit'] = np.empty([0]) + fit_results["delta_fit"] = np.empty([0]) + fit_results["pright_fit"] = np.empty([0]) return fit_results @@ -232,21 +241,25 @@ def translate_choice_trials_cues(session_df): Transform to numeric values trialtype, choice and left & right cues """ - session_df['trial_type_int'] = 0 - session_df.loc[session_df['trial_type'] == 'L','trial_type_int'] = 1 - session_df.loc[session_df['trial_type'] == 'R','trial_type_int'] = 2 - session_df['choice_int'] = 0 - session_df.loc[session_df['choice'] == 'L','choice_int'] = 1 - session_df.loc[session_df['choice'] == 'R','choice_int'] = 2 + session_df["trial_type_int"] = 0 + session_df.loc[session_df["trial_type"] == "L", "trial_type_int"] = 1 + session_df.loc[session_df["trial_type"] == "R", "trial_type_int"] = 2 + session_df["choice_int"] = 0 + session_df.loc[session_df["choice"] == "L", "choice_int"] = 1 + session_df.loc[session_df["choice"] == "R", "choice_int"] = 2 # If we are translating towers task cues - if 'cue_presence_left' in session_df.columns: - session_df['cue_presence_left'] = session_df['cue_presence_left'].apply(lambda x: np.count_nonzero(x)) - session_df['cue_presence_right'] = session_df['cue_presence_right'].apply(lambda x: np.count_nonzero(x)) + if "cue_presence_left" in session_df.columns: + session_df["cue_presence_left"] = session_df["cue_presence_left"].apply( + lambda x: np.count_nonzero(x) + ) + session_df["cue_presence_right"] = session_df["cue_presence_right"].apply( + lambda x: np.count_nonzero(x) + ) # If we are translating puff task cues - elif 'num_puffs_received_r' in session_df.columns: - session_df['cue_presence_left'] = session_df['num_puffs_received_l'] - session_df['cue_presence_right'] = session_df['num_puffs_received_r'] + elif "num_puffs_received_r" in session_df.columns: + session_df["cue_presence_left"] = session_df["num_puffs_received_l"] + session_df["cue_presence_right"] = session_df["num_puffs_received_r"] return session_df @@ -259,11 +272,10 @@ def get_cols_rows_plot(num_plots, fig_size): # Check width vs height relation fig_rel = fig_size[1] / fig_size[0] - #Let's start with square root + # Let's start with square root num_rows = 1 num_cols = 1 while 1: - # Add rows and columns to closely match desired width height relationship and surpass number of desired plots ac_rel = num_cols / num_rows @@ -280,17 +292,17 @@ def get_cols_rows_plot(num_plots, fig_size): def create_str_from_dict(key_dict): - slurm_file_name = '' + slurm_file_name = "" for i in key_dict.keys(): - slurm_file_name += str(i) + '_' + str(key_dict[i]) + slurm_file_name += str(i) + "_" + str(key_dict[i]) return slurm_file_name def numpy_array_to_dict(np_array, as_int=True): - ''' + """ Transform a numpy array to dictionary: (numpy array are stored when saving Blobs in MATLAB Datajoint, normally a dictionary will be the fit) - ''' + """ # Transform numpy array to DF out_dict = pd.DataFrame(np_array.flatten()) @@ -298,27 +310,31 @@ def numpy_array_to_dict(np_array, as_int=True): # Flatten each column and get the "real value of it" out_dict = out_dict.applymap(lambda x: x.flatten()) - #Get not empty columns to extract fist value of only those columns + # Get not empty columns to extract fist value of only those columns s = out_dict.applymap(lambda x: x.shape[0]) not_empty_columns = s.loc[:, ~(s == 0).any()].columns.to_list() - out_dict = out_dict.apply(lambda x: x[0].flatten() if x.name in not_empty_columns else x) - + out_dict = out_dict.apply( + lambda x: x[0].flatten() if x.name in not_empty_columns else x + ) + if not isinstance(out_dict, pd.DataFrame): out_dict = out_dict.to_frame() - #Get columns that are "real" arrays not unique values disguised + # Get columns that are "real" arrays not unique values disguised s = out_dict.applymap(lambda x: x.size).T real_array_columns = s.loc[:, (s > 1).any()].columns.to_list() out_dict = out_dict.T - out_dict = out_dict.apply(lambda x: x[0] if x.name not in real_array_columns else x, axis=0) - + out_dict = out_dict.apply( + lambda x: x[0] if x.name not in real_array_columns else x, axis=0 + ) + columns = out_dict.columns.copy() out_dict = out_dict.squeeze() - #Transform numeric columns to int (normally for params) + # Transform numeric columns to int (normally for params) if as_int: for i in columns: - if (isinstance(out_dict[i],np.float64) and out_dict[i].is_integer()): - out_dict[i] = out_dict[i].astype('int') + if isinstance(out_dict[i], np.float64) and out_dict[i].is_integer(): + out_dict[i] = out_dict[i].astype("int") out_dict = out_dict.to_dict() diff --git a/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py b/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py index bcc85579..c05754ce 100644 --- a/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py +++ b/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py @@ -14,6 +14,7 @@ much easier! """ + from pathlib import Path from tkinter import Tk, filedialog @@ -39,15 +40,15 @@ def readMeta(binFullPath): mdatList = f.read().splitlines() # convert the list entries into key value pairs for m in mdatList: - csList = m.split(sep='=') - if csList[0][0] == '~': - currKey = csList[0][1:len(csList[0])] + csList = m.split(sep="=") + if csList[0][0] == "~": + currKey = csList[0][1 : len(csList[0])] else: currKey = csList[0] metaDict.update({currKey: csList[1]}) else: print("no meta file") - return(metaDict) + return metaDict # Return sample rate as python float. @@ -55,17 +56,17 @@ def readMeta(binFullPath): # Use python command sys.float_info to get properties of float on your system. # def SampRate(meta): - if meta['typeThis'] == 'imec': - srate = float(meta['imSampRate']) - elif meta['typeThis'] == 'nidq': - srate = float(meta['niSampRate']) - elif meta['typeThis'] == 'obx': - srate = float(meta['obSampRate']) + if meta["typeThis"] == "imec": + srate = float(meta["imSampRate"]) + elif meta["typeThis"] == "nidq": + srate = float(meta["niSampRate"]) + elif meta["typeThis"] == "obx": + srate = float(meta["obSampRate"]) else: - print('Error: unknown stream type') + print("Error: unknown stream type") srate = 1 - - return(srate) + + return srate # Return a multiplicative factor for converting 16-bit file data @@ -75,23 +76,23 @@ def SampRate(meta): # Note that each channel may have its own gain. # def Int2Volts(meta): - if meta['typeThis'] == 'imec': - if 'imMaxInt' in meta: - maxInt = int(meta['imMaxInt']) + if meta["typeThis"] == "imec": + if "imMaxInt" in meta: + maxInt = int(meta["imMaxInt"]) else: maxInt = 512 - fI2V = float(meta['imAiRangeMax'])/maxInt - elif meta['typeThis'] == 'nidq': - maxInt = int(meta['niMaxInt']) - fI2V = float(meta['niAiRangeMax'])/maxInt - elif meta['typeThis'] == 'obx': - maxInt = int(meta['obMaxInt']) - fI2V = float(meta['obAiRangeMax'])/maxInt + fI2V = float(meta["imAiRangeMax"]) / maxInt + elif meta["typeThis"] == "nidq": + maxInt = int(meta["niMaxInt"]) + fI2V = float(meta["niAiRangeMax"]) / maxInt + elif meta["typeThis"] == "obx": + maxInt = int(meta["obMaxInt"]) + fI2V = float(meta["obAiRangeMax"]) / maxInt else: - print('Error: unknown stream type') + print("Error: unknown stream type") fI2V = 1 - - return(fI2V) + + return fI2V # Return array of original channel IDs. As an example, suppose we want the @@ -103,57 +104,58 @@ def Int2Volts(meta): # Note that the SpikeGLX channels are 0 based. # def OriginalChans(meta): - if meta['snsSaveChanSubset'] == 'all': + if meta["snsSaveChanSubset"] == "all": # output = int32, 0 to nSavedChans - 1 - chans = np.arange(0, int(meta['nSavedChans'])) + chans = np.arange(0, int(meta["nSavedChans"])) else: # parse the snsSaveChanSubset string # split at commas - chStrList = meta['snsSaveChanSubset'].split(sep=',') + chStrList = meta["snsSaveChanSubset"].split(sep=",") chans = np.arange(0, 0) # creates an empty array of int32 for sL in chStrList: - currList = sL.split(sep=':') + currList = sL.split(sep=":") if len(currList) > 1: # each set of contiguous channels specified by # chan1:chan2 inclusive - newChans = np.arange(int(currList[0]), int(currList[1])+1) + newChans = np.arange(int(currList[0]), int(currList[1]) + 1) else: - newChans = np.arange(int(currList[0]), int(currList[0])+1) + newChans = np.arange(int(currList[0]), int(currList[0]) + 1) chans = np.append(chans, newChans) - return(chans) + return chans # Return counts of each nidq channel type that composes the timepoints # stored in the binary file. # def ChannelCountsNI(meta): - chanCountList = meta['snsMnMaXaDw'].split(sep=',') + chanCountList = meta["snsMnMaXaDw"].split(sep=",") MN = int(chanCountList[0]) MA = int(chanCountList[1]) XA = int(chanCountList[2]) DW = int(chanCountList[3]) - return(MN, MA, XA, DW) + return (MN, MA, XA, DW) # Return counts of each imec channel type that composes the timepoints # stored in the binary files. # def ChannelCountsIM(meta): - chanCountList = meta['snsApLfSy'].split(sep=',') + chanCountList = meta["snsApLfSy"].split(sep=",") AP = int(chanCountList[0]) LF = int(chanCountList[1]) SY = int(chanCountList[2]) - return(AP, LF, SY) + return (AP, LF, SY) + # Return counts of each obx channel type that composes the timepoints # stored in the binary files. # def ChannelCountsOBX(meta): - chanCountList = meta['snsXaDwSy'].split(sep=',') + chanCountList = meta["snsXaDwSy"].split(sep=",") XA = int(chanCountList[0]) DW = int(chanCountList[1]) SY = int(chanCountList[2]) - return(XA, DW, SY) + return (XA, DW, SY) # Return gain for ith channel stored in nidq file. @@ -162,12 +164,12 @@ def ChannelCountsOBX(meta): # Note: there is nomatching function for OBX, where the gain is fixed = 1 def ChanGainNI(ichan, savedMN, savedMA, meta): if ichan < savedMN: - gain = float(meta['niMNGain']) + gain = float(meta["niMNGain"]) elif ichan < (savedMN + savedMA): - gain = float(meta['niMAGain']) + gain = float(meta["niMAGain"]) else: - gain = 1 # non multiplexed channels have no extra gain - return(gain) + gain = 1 # non multiplexed channels have no extra gain + return gain # Return gain for imec channels. @@ -175,56 +177,56 @@ def ChanGainNI(ichan, savedMN, savedMA, meta): # def ChanGainsIM(meta): # list of probe types with NP 1.0 imro format - np1_imro = [0,1020,1030,1200,1100,1120,1121,1122,1123,1300] + np1_imro = [0, 1020, 1030, 1200, 1100, 1120, 1121, 1122, 1123, 1300] # number of channels acquired - acqCountList = meta['acqApLfSy'].split(sep=',') - APgain = np.zeros(int(acqCountList[0])) # default type = float64 - LFgain = np.zeros(int(acqCountList[1])) # empty array for 2.0 - - if 'imDatPrb_type' in meta: - probeType = int(meta['imDatPrb_type']) + acqCountList = meta["acqApLfSy"].split(sep=",") + APgain = np.zeros(int(acqCountList[0])) # default type = float64 + LFgain = np.zeros(int(acqCountList[1])) # empty array for 2.0 + + if "imDatPrb_type" in meta: + probeType = int(meta["imDatPrb_type"]) else: probeType = 0 - + if sum(np.isin(np1_imro, probeType)): # imro + probe allows setting gain independently for each channel - imroList = meta['imroTbl'].split(sep=')') + imroList = meta["imroTbl"].split(sep=")") # One entry for each channel plus header entry, - # plus a final empty entry following the last ')' + # plus a final empty entry following the last ')' for i in range(0, int(acqCountList[0])): - currList = imroList[i+1].split(sep=' ') + currList = imroList[i + 1].split(sep=" ") APgain[i] = float(currList[3]) LFgain[i] = float(currList[4]) - else: + else: # get gain from imChan0apGain - if 'imChan0apGain' in meta: - APgain = APgain + float(meta['imChan0apGain']) - if int(acqCountList[1]) > 0: - LFgain = LFgain + float(meta['imChan0lfGain']) - elif (probeType == 1110): + if "imChan0apGain" in meta: + APgain = APgain + float(meta["imChan0apGain"]) + if int(acqCountList[1]) > 0: + LFgain = LFgain + float(meta["imChan0lfGain"]) + elif probeType == 1110: # active UHD, for metadata lacking imChan0apGain, get gain from # imro table header - imroList = meta['imroTbl'].split(sep=')') - currList = imroList[0].split(sep=',') + imroList = meta["imroTbl"].split(sep=")") + currList = imroList[0].split(sep=",") APgain = APgain + float(currList[3]) LFgain = LFgain + float(currList[4]) elif (probeType == 21) or (probeType == 24): # development NP 2.0; APGain = 80 for all AP # return 0 for LFgain (no LF channels) - APgain = APgain + 80 - elif (probeType == 2013): + APgain = APgain + 80 + elif probeType == 2013: # commercial NP 2.0; APGain = 100 for all AP APgain = APgain + 100 else: - print('unknown gain, setting APgain to 1') + print("unknown gain, setting APgain to 1") APgain = APgain + 1 fI2V = Int2Volts(meta) - APChan0_to_uV = 1e6*fI2V/APgain[0] + APChan0_to_uV = 1e6 * fI2V / APgain[0] if LFgain.size > 0: - LFChan0_to_uV = 1e6*fI2V/LFgain[0] + LFChan0_to_uV = 1e6 * fI2V / LFgain[0] else: LFChan0_to_uV = 0 - return(APgain, LFgain, APChan0_to_uV, LFChan0_to_uV) + return (APgain, LFgain, APChan0_to_uV, LFChan0_to_uV) # Having accessed a block of raw nidq data using makeMemMapRaw, convert @@ -246,11 +248,12 @@ def GainCorrectNI(dataArray, chanList, meta): # in chanList, so output matches that shape convArray = np.zeros(dataArray.shape, dtype=float) for i in range(0, len(chanList)): - j = chanList[i] # index in saved data - conv = fI2V/ChanGainNI(j, MN, MA, meta) + j = chanList[i] # index in saved data + conv = fI2V / ChanGainNI(j, MN, MA, meta) # dataArray contains only the channels in chanList convArray[i, :] = dataArray[i, :] * conv - return(convArray) + return convArray + # Having accessed a block of raw obx data using makeMemMapRaw, convert # values to volts. The conversion is only applied to the @@ -268,7 +271,7 @@ def GainCorrectOBX(dataArray, chanList, meta): for i in range(0, len(chanList)): # dataArray contains only the channels in chanList convArray[i, :] = dataArray[i, :] * fI2V - return(convArray) + return convArray # Having accessed a block of raw imec data using makeMemMapRaw, convert @@ -293,10 +296,10 @@ def GainCorrectIM(dataArray, chanList, meta): # make array of floats to return. dataArray contains only the channels # in chanList, so output matches that shape - convArray = np.zeros(dataArray.shape, dtype='float') + convArray = np.zeros(dataArray.shape, dtype="float") for i in range(0, len(chanList)): - j = chanList[i] # index into timepoint - k = chans[j] # acquisition index + j = chanList[i] # index into timepoint + k = chans[j] # acquisition index if k < nAP: conv = fI2V / APgain[k] elif k < nNu: @@ -304,20 +307,27 @@ def GainCorrectIM(dataArray, chanList, meta): else: conv = 1 # The dataArray contains only the channels in chanList - convArray[i, :] = dataArray[i, :]*conv - return(convArray) + convArray[i, :] = dataArray[i, :] * conv + return convArray + # Return memmap for the raw data # Fortran ordering is used to match the MATLAB version # of these tools. # def makeMemMapRaw(binFullPath, meta): - nChan = int(meta['nSavedChans']) - nFileSamp = int(int(meta['fileSizeBytes'])/(2*nChan)) + nChan = int(meta["nSavedChans"]) + nFileSamp = int(int(meta["fileSizeBytes"]) / (2 * nChan)) print("nChan: %d, nFileSamp: %d" % (nChan, nFileSamp)) - rawData = np.memmap(binFullPath, dtype='int16', mode='r', - shape=(nChan, nFileSamp), offset=0, order='F') - return(rawData) + rawData = np.memmap( + binFullPath, + dtype="int16", + mode="r", + shape=(nChan, nFileSamp), + offset=0, + order="F", + ) + return rawData # Return an array [lines X timepoints] of uint8 values for a @@ -330,49 +340,49 @@ def makeMemMapRaw(binFullPath, meta): # def ExtractDigital(rawData, firstSamp, lastSamp, dwReq, dLineList, meta): # Get channel index of requested digital word dwReq - if meta['typeThis'] == 'imec': + if meta["typeThis"] == "imec": AP, LF, SY = ChannelCountsIM(meta) if SY == 0: print("No imec sync channel saved.") - digArray = np.zeros((0), 'uint8') - return(digArray) + digArray = np.zeros((0), "uint8") + return digArray else: digCh = AP + LF + dwReq - elif meta['typeThis'] == 'nidq': + elif meta["typeThis"] == "nidq": MN, MA, XA, DW = ChannelCountsNI(meta) - if dwReq > DW-1: - print("Maximum digital word in file = %d" % (DW-1)) - digArray = np.zeros((0), 'uint8') - return(digArray) + if dwReq > DW - 1: + print("Maximum digital word in file = %d" % (DW - 1)) + digArray = np.zeros((0), "uint8") + return digArray else: digCh = MN + MA + XA + dwReq - elif meta['typeThis'] == 'obx': + elif meta["typeThis"] == "obx": XA, DW, SY = ChannelCountsOBX(meta) - if dwReq > DW-1: - print("Maximum digital word in file = %d" % (DW-1)) - digArray = np.zeros((0), 'uint8') - return(digArray) + if dwReq > DW - 1: + print("Maximum digital word in file = %d" % (DW - 1)) + digArray = np.zeros((0), "uint8") + return digArray else: digCh = XA + dwReq else: - print('unknown data stream') + print("unknown data stream") - selectData = np.ascontiguousarray(rawData[digCh, firstSamp:lastSamp+1], 'int16') - nSamp = lastSamp-firstSamp + 1 + selectData = np.ascontiguousarray(rawData[digCh, firstSamp : lastSamp + 1], "int16") + nSamp = lastSamp - firstSamp + 1 # unpack bits of selectData; unpack bits works with uint8 # original data is int16 - bitWiseData = np.unpackbits(selectData.view(dtype='uint8')) + bitWiseData = np.unpackbits(selectData.view(dtype="uint8")) # output is 1-D array, nSamp*16. Reshape and transpose bitWiseData = np.transpose(np.reshape(bitWiseData, (nSamp, 16))) nLine = len(dLineList) - digArray = np.zeros((nLine, nSamp), 'uint8') + digArray = np.zeros((nLine, nSamp), "uint8") for i in range(0, nLine): byteN, bitN = np.divmod(dLineList[i], 8) - targI = byteN*8 + (7 - bitN) + targI = byteN * 8 + (7 - bitN) digArray[i, :] = bitWiseData[targI, :] - return(digArray) + return digArray # Sample calling program to get a file from the user, @@ -385,19 +395,19 @@ def ExtractDigital(rawData, firstSamp, lastSamp, dwReq, dLineList, meta): def main(): # Get file from user - root = Tk() # create the Tkinter widget - root.withdraw() # hide the Tkinter root window + root = Tk() # create the Tkinter widget + root.withdraw() # hide the Tkinter root window # Windows specific; forces the window to appear in front root.attributes("-topmost", True) binFullPath = Path(filedialog.askopenfilename(title="Select binary file")) - root.destroy() # destroy the Tkinter widget + root.destroy() # destroy the Tkinter widget # Other parameters about what data to read tStart = 0 tEnd = 2 - dataType = 'A' # 'A' for analog, 'D' for digital data + dataType = "A" # 'A' for analog, 'D' for digital data # For analog channels: zero-based index of a channel to extract, # gain correct and plot (plots first channel only) @@ -416,44 +426,43 @@ def main(): # parameters common to NI and imec data sRate = SampRate(meta) - firstSamp = int(sRate*tStart) - lastSamp = int(sRate*tEnd) + firstSamp = int(sRate * tStart) + lastSamp = int(sRate * tEnd) # array of times for plot - tDat = np.arange(firstSamp, lastSamp+1, dtype='uint64') - tDat = 1000*tDat/sRate # plot time axis in msec + tDat = np.arange(firstSamp, lastSamp + 1, dtype="uint64") + tDat = 1000 * tDat / sRate # plot time axis in msec rawData = makeMemMapRaw(binFullPath, meta) - if dataType == 'A': - selectData = rawData[chanList, firstSamp:lastSamp+1] - if meta['typeThis'] == 'imec': + if dataType == "A": + selectData = rawData[chanList, firstSamp : lastSamp + 1] + if meta["typeThis"] == "imec": # apply gain correction and convert to uV - convData = 1e6*GainCorrectIM(selectData, chanList, meta) - elif meta['typeThis'] == 'nidq': + convData = 1e6 * GainCorrectIM(selectData, chanList, meta) + elif meta["typeThis"] == "nidq": MN, MA, XA, DW = ChannelCountsNI(meta) # print("NI channel counts: %d, %d, %d, %d" % (MN, MA, XA, DW)) # apply gain correction and convert to mV - convData = 1e3*GainCorrectNI(selectData, chanList, meta) - elif meta['typeThis'] == 'obx': - # Gain correct is just conversion to volts - convData = 1e3*GainCorrectOBX(selectData, chanList, meta) - + convData = 1e3 * GainCorrectNI(selectData, chanList, meta) + elif meta["typeThis"] == "obx": + # Gain correct is just conversion to volts + convData = 1e3 * GainCorrectOBX(selectData, chanList, meta) + # Plot the first of the extracted channels fig, ax = plt.subplots() ax.plot(tDat, convData[0, :]) plt.show() else: - digArray = ExtractDigital(rawData, firstSamp, lastSamp, dw, - dLineList, meta) + digArray = ExtractDigital(rawData, firstSamp, lastSamp, dw, dLineList, meta) # Plot the first of the extracted channels fig, ax = plt.subplots() for i in range(0, len(dLineList)): - ax.plot(tDat, digArray[i, :]) + ax.plot(tDat, digArray[i, :]) plt.show() if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/u19_pipeline/utils/dj_shortcuts.py b/u19_pipeline/utils/dj_shortcuts.py index ff007fd6..62c184f4 100644 --- a/u19_pipeline/utils/dj_shortcuts.py +++ b/u19_pipeline/utils/dj_shortcuts.py @@ -5,14 +5,14 @@ def get_primary_key_fields(t): """ Get list of all fields that compose primary key Args: - t (Dj table): Instance of a table in datajoint + t (Dj table): Instance of a table in datajoint Returns: primary_field_list: (list): List of all fields that make primary key """ - fields_t = pd.DataFrame.from_dict(t.heading.attributes, orient='index') - primary_field_list = fields_t.loc[fields_t['in_key'] == True].index.to_list() - + fields_t = pd.DataFrame.from_dict(t.heading.attributes, orient="index") + primary_field_list = fields_t.loc[fields_t["in_key"] == True].index.to_list() + return primary_field_list @@ -24,12 +24,12 @@ def smart_dj_join(t1, t2): """ # Get all fields from tables - fields_t1 = pd.DataFrame.from_dict(t1.heading.attributes, orient='index') - fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient='index') + fields_t1 = pd.DataFrame.from_dict(t1.heading.attributes, orient="index") + fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient="index") # Get only secondary fields and check matches - fields_t1_list = set(fields_t1.loc[fields_t1['in_key'] == False].index.to_list()) - fields_t2_list = set(fields_t2.loc[fields_t2['in_key'] == False].index.to_list()) + fields_t1_list = set(fields_t1.loc[fields_t1["in_key"] == False].index.to_list()) + fields_t2_list = set(fields_t2.loc[fields_t2["in_key"] == False].index.to_list()) intersected_fields = fields_t2_list.intersection(fields_t1_list) # If there are: @@ -38,7 +38,7 @@ def smart_dj_join(t1, t2): suffix = t2.table_name new_name_attr_dict = dict() for i in intersected_fields: - new_name_attr_dict[suffix + '_' + i] = i + new_name_attr_dict[suffix + "_" + i] = i # List non matching ones non_intersected_fields = list(fields_t2_list - intersected_fields) @@ -57,15 +57,15 @@ def get_string_key(key): Translate list or dict key to string """ - str_key = '' + str_key = "" if isinstance(key, list): - str_key = [[k + '=' + str(v) for k,v in x.items()] for x in key] - str_key = ['('+' and '.join(sublist)+')' for sublist in str_key] - str_key = ' or '.join(str_key) + str_key = [[k + "=" + str(v) for k, v in x.items()] for x in key] + str_key = ["(" + " and ".join(sublist) + ")" for sublist in str_key] + str_key = " or ".join(str_key) elif isinstance(key, dict): - str_key = [k + '=' + str(v) for k,v in key.items()] - str_key = ' and '.join(str_key) - elif isinstance(key,str): + str_key = [k + "=" + str(v) for k, v in key.items()] + str_key = " and ".join(str_key) + elif isinstance(key, str): str_key = key - - return str_key \ No newline at end of file + + return str_key diff --git a/u19_pipeline/utils/dlc_process.py b/u19_pipeline/utils/dlc_process.py index 073872ee..7df9dc23 100644 --- a/u19_pipeline/utils/dlc_process.py +++ b/u19_pipeline/utils/dlc_process.py @@ -1,4 +1,3 @@ - import os import pathlib import pickle @@ -14,6 +13,7 @@ # Pupil diameter pipeline functions + def analyzeVideo(videoPath=None, modelPath=None, destinationFolder=None): """ Stores the analized video data from videoPath as h5 file in the destination folder using the DLC model in modelPath @@ -24,9 +24,10 @@ def analyzeVideo(videoPath=None, modelPath=None, destinationFolder=None): """ # Analyze the video using the selected modelPath and videoPath - configPath = os.path.join(modelPath,'config.yaml') + configPath = os.path.join(modelPath, "config.yaml") deeplabcut.analyze_videos(configPath, videoPath, destfolder=destinationFolder) + def getPupilDiameter(destinationFolder=None): """ Returns a pupil diameter numpy array from an analized video data stored in analyzedVideoDataPath @@ -36,35 +37,35 @@ def getPupilDiameter(destinationFolder=None): An array that contains the pupil diameter (index is the video frame) [numpy Array] """ # TODO make the function - + # Read the analyzed video data h5 file h5_file = pu.get_filepattern_paths(destinationFolder, "/*.h5") print(h5_file) if len(h5_file) == 0: - raise Exception('No h5 file in directory: '+ destinationFolder) + raise Exception("No h5 file in directory: " + destinationFolder) if len(h5_file) > 1: - raise Exception('To many h5 files in directory: '+ destinationFolder) - + raise Exception("To many h5 files in directory: " + destinationFolder) + h5_file = h5_file[0] labels = pd.read_hdf(h5_file) # Create a data frame of the same size ad the analyzed video data filled with zeros - df = pd.DataFrame(np.zeros(1), columns=['PupilDiameter']) + df = pd.DataFrame(np.zeros(1), columns=["PupilDiameter"]) # For each frame, get the x and y coordinates of the points around the pupil, fit an ellipse and calculate the diameter of a circle with the same area as the ellipse for i in range(labels.index.size): subset = labels.loc[i] - x = subset.xs('x', level='coords').to_numpy()[0:8] - y = subset.xs('y', level='coords').to_numpy()[0:8] - xy = np.column_stack((x,y)) + x = subset.xs("x", level="coords").to_numpy()[0:8] + y = subset.xs("y", level="coords").to_numpy()[0:8] + xy = np.column_stack((x, y)) # Fit the points to an ellipse and get the parameters (estimate X center coordinate, estimate Y center coordinate, a, b, theta) ellipse = EllipseModel() ellipse.estimate(xy) # Calculate the area of the ellipse from the parameters a and b ellipseArea = np.pi * ellipse.params[2] * ellipse.params[3] # Get the diameter of a circle from the area of the ellipse - pupilDiameter = 2 * np.sqrt(ellipseArea/np.pi) + pupilDiameter = 2 * np.sqrt(ellipseArea / np.pi) df.loc[i] = pupilDiameter # Get outliers (frames where either the mice have the eyes closed (blink or groom) or deeplabcut fails to track the pupil correctly) @@ -77,8 +78,8 @@ def getPupilDiameter(destinationFolder=None): outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: "OutlierFlag"}) # Concatenate outlier flags array to remove outliers from pupil diameter array temp = pd.concat([df, outlierFlags], axis=1) - temp.loc[temp['OutlierFlag']==True, 'PupilDiameter'] = None - pupilDiameter = temp['PupilDiameter'].to_numpy() + temp.loc[temp["OutlierFlag"] == True, "PupilDiameter"] = None + pupilDiameter = temp["PupilDiameter"].to_numpy() filename = pathlib.Path(destinationFolder, "pupil_diameter.pickle").as_posix() @@ -91,4 +92,4 @@ def getPupilDiameter(destinationFolder=None): args = sys.argv[1:] print(args) analyzeVideo(videoPath=args[0], modelPath=args[1], destinationFolder=args[2]) - getPupilDiameter(destinationFolder=args[2]) \ No newline at end of file + getPupilDiameter(destinationFolder=args[2]) diff --git a/u19_pipeline/utils/ephys_fix_sync_code.py b/u19_pipeline/utils/ephys_fix_sync_code.py index a024f2d5..b7a51cf3 100644 --- a/u19_pipeline/utils/ephys_fix_sync_code.py +++ b/u19_pipeline/utils/ephys_fix_sync_code.py @@ -1,17 +1,24 @@ - import numpy as np import u19_pipeline.utils.ephys_utils as ephys_utils -def get_shift_vector(synced_time_vector, behavior_time_vector, base_size=40,initial_sample=0, samples_shift=100): - +def get_shift_vector( + synced_time_vector, + behavior_time_vector, + base_size=40, + initial_sample=0, + samples_shift=100, +): diff_size = np.abs(synced_time_vector.shape[0] - behavior_time_vector.shape[0]) - baseline_diff = synced_time_vector[initial_sample:base_size] - behavior_time_vector[initial_sample:base_size] + baseline_diff = ( + synced_time_vector[initial_sample:base_size] + - behavior_time_vector[initial_sample:base_size] + ) - ''' + """ while(1): base_greater = np.where(baseline_diff >0) @@ -26,25 +33,21 @@ def get_shift_vector(synced_time_vector, behavior_time_vector, base_size=40,init baseline_diff = synced_time_vector[initial_sample:base_size] - behavior_time_vector[initial_sample:base_size] else: break - ''' - + """ median_diff = np.median(baseline_diff) - max_diff = np.median(baseline_diff)+0.007 - min_diff = np.median(baseline_diff)-0.007 - - - vec_shift = np.zeros((synced_time_vector.shape[0]-base_size), dtype=int) - for i in range(synced_time_vector.shape[0]-base_size): + max_diff = np.median(baseline_diff) + 0.007 + min_diff = np.median(baseline_diff) - 0.007 - - idx_start = initial_sample+i+1 - idx_end = base_size+i+1 + vec_shift = np.zeros((synced_time_vector.shape[0] - base_size), dtype=int) + for i in range(synced_time_vector.shape[0] - base_size): + idx_start = initial_sample + i + 1 + idx_end = base_size + i + 1 this_bt = behavior_time_vector[idx_start:idx_end] this_iv = synced_time_vector[idx_start:idx_end] - median_ori = np.median(this_iv-this_bt) + median_ori = np.median(this_iv - this_bt) if median_ori >= min_diff and median_ori < max_diff: vec_shift[i] = 0 @@ -53,25 +56,23 @@ def get_shift_vector(synced_time_vector, behavior_time_vector, base_size=40,init sign = 1 else: sign = -1 - + new_sign = sign if sign != 0: - for j in range(1, samples_shift): - if new_sign == 1: - if idx_end+j > synced_time_vector.shape[0]: - this_iv = synced_time_vector[idx_start+j:] + if idx_end + j > synced_time_vector.shape[0]: + this_iv = synced_time_vector[idx_start + j :] this_bt = this_bt[:-1] else: - this_iv = synced_time_vector[idx_start+j:idx_end+j] + this_iv = synced_time_vector[idx_start + j : idx_end + j] else: - this_iv = synced_time_vector[idx_start-j:idx_end-j] - - median_now = np.median(this_iv-this_bt) - + this_iv = synced_time_vector[idx_start - j : idx_end - j] + + median_now = np.median(this_iv - this_bt) + if median_now >= min_diff and median_now < max_diff: - vec_shift[i] = j*new_sign + vec_shift[i] = j * new_sign break elif median_now < min_diff: new_sign = 1 @@ -79,48 +80,53 @@ def get_shift_vector(synced_time_vector, behavior_time_vector, base_size=40,init new_sign = -1 if new_sign != sign: - if np.abs(median_ori-median_diff) < np.abs(median_now-median_diff): + if np.abs(median_ori - median_diff) < np.abs( + median_now - median_diff + ): vec_shift[i] = 0 else: - vec_shift[i] = j*sign + vec_shift[i] = j * sign break - - if j == samples_shift-1: - #print('Extreme case !!!') - vec_shift[i] = j*sign + + if j == samples_shift - 1: + # print('Extreme case !!!') + vec_shift[i] = j * sign new_synced_time_vector = synced_time_vector.copy() - mid_point = int(initial_sample+base_size/2) + mid_point = int(initial_sample + base_size / 2) for i in range(vec_shift.shape[0]): - new_synced_time_vector[mid_point+i] = synced_time_vector[mid_point+i+vec_shift[i]] - - idx_end = mid_point+vec_shift.shape[0]-1 + new_synced_time_vector[mid_point + i] = synced_time_vector[ + mid_point + i + vec_shift[i] + ] + + idx_end = mid_point + vec_shift.shape[0] - 1 for i in range(idx_end, new_synced_time_vector.shape[0]): - if i+vec_shift[-1] < new_synced_time_vector.shape[0]: - new_synced_time_vector[i] = synced_time_vector[i+vec_shift[-1]] + if i + vec_shift[-1] < new_synced_time_vector.shape[0]: + new_synced_time_vector[i] = synced_time_vector[i + vec_shift[-1]] max_shift = np.max(np.abs(vec_shift)) - if max_shift > diff_size and diff_size != 1: pass - #print('max_shift', max_shift) - #print('diff_size', diff_size) - #print('walaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') - #raise ValueError('more max shift than diff size') + # print('max_shift', max_shift) + # print('diff_size', diff_size) + # print('walaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + # raise ValueError('more max shift than diff size') return new_synced_time_vector, vec_shift, [min_diff, median_diff, max_diff] -def fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, vec_shift, initial_sample=0, base_size=40): +def fix_shifted_sync_vector( + synced_time_vector, behavior_time_vector, vec_shift, initial_sample=0, base_size=40 +): - mid_point = int(initial_sample+base_size/2) + mid_point = int(initial_sample + base_size / 2) new_synced_time_vector = synced_time_vector.copy() diff_vec_shift = np.diff(vec_shift) where_insert_iteration = np.where(diff_vec_shift != 0) where_insert_iteration = where_insert_iteration[0] - #print('where_insert_iteration', where_insert_iteration) + # print('where_insert_iteration', where_insert_iteration) find = np.full(4, 1) correlation_result = np.correlate(diff_vec_shift == 0, find) @@ -133,67 +139,79 @@ def fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, vec_shift, stable_parts = stable_parts[0] if stable_parts.shape[0] > 0: - index_borrow_virmen.append([where_insert_iteration[index_shift], consecutive_zeros[stable_parts[0]]]) - next_borrow_start = np.where(where_insert_iteration > consecutive_zeros[stable_parts[0]]) + index_borrow_virmen.append( + [ + where_insert_iteration[index_shift], + consecutive_zeros[stable_parts[0]], + ] + ) + next_borrow_start = np.where( + where_insert_iteration > consecutive_zeros[stable_parts[0]] + ) next_borrow_start = next_borrow_start[0] if next_borrow_start.shape[0] > 0: index_shift = next_borrow_start[0] else: break else: - print('Extreme case diff vec shift') + print("Extreme case diff vec shift") - index_borrow_virmen.append([where_insert_iteration[index_shift], diff_vec_shift.shape[0]]) - print('index_borrow_virmen', index_borrow_virmen) + index_borrow_virmen.append( + [where_insert_iteration[index_shift], diff_vec_shift.shape[0]] + ) + print("index_borrow_virmen", index_borrow_virmen) break + # print('index_borrow_virmen', index_borrow_virmen) - - #print('index_borrow_virmen', index_borrow_virmen) - - #for i in range(where_insert_iteration.shape[0]): - index_diff_e = -1 + # for i in range(where_insert_iteration.shape[0]): + index_diff_e = -1 borrowed_indexes = list() for i in range(len(index_borrow_virmen)): - - if index_diff_e > index_borrow_virmen[i][0]+mid_point: + if index_diff_e > index_borrow_virmen[i][0] + mid_point: continue for j in range(100): - - index_diff_s = index_borrow_virmen[i][0]+mid_point-1 - index_diff_e = index_borrow_virmen[i][1]+mid_point+1 - - if j>=1: - index_diff_s = index_diff_s-1 - if j >1: - index_diff_e = index_diff_e+j-1 - - #print('index_diff_vec', index_diff_s, index_diff_e) - #print('diff_vec_shift[index_diff] ', diff_vec_shift[index_diff_vec-1:index_diff_vec+1] ) - - #print('behavior_time_vector', behavior_time_vector[index_diff_s-1:index_diff_e+2]) - #print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2]) - - time_iteration = behavior_time_vector[index_diff_s:index_diff_e+1] - behavior_time_vector[index_diff_s] - #print('time_iteration', time_iteration) - - new_synced_time_vector[index_diff_s:index_diff_e+1] = np.repeat(new_synced_time_vector[index_diff_s], index_diff_e-index_diff_s+1) +\ - time_iteration - - check_diff = np.diff(new_synced_time_vector[index_diff_s:index_diff_e+2]) + index_diff_s = index_borrow_virmen[i][0] + mid_point - 1 + index_diff_e = index_borrow_virmen[i][1] + mid_point + 1 + + if j >= 1: + index_diff_s = index_diff_s - 1 + if j > 1: + index_diff_e = index_diff_e + j - 1 + + # print('index_diff_vec', index_diff_s, index_diff_e) + # print('diff_vec_shift[index_diff] ', diff_vec_shift[index_diff_vec-1:index_diff_vec+1] ) + + # print('behavior_time_vector', behavior_time_vector[index_diff_s-1:index_diff_e+2]) + # print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2]) + + time_iteration = ( + behavior_time_vector[index_diff_s : index_diff_e + 1] + - behavior_time_vector[index_diff_s] + ) + # print('time_iteration', time_iteration) + + new_synced_time_vector[index_diff_s : index_diff_e + 1] = ( + np.repeat( + new_synced_time_vector[index_diff_s], + index_diff_e - index_diff_s + 1, + ) + + time_iteration + ) + + check_diff = np.diff( + new_synced_time_vector[index_diff_s : index_diff_e + 2] + ) idx_back_time = np.where(check_diff <= 0.0005) idx_back_time = idx_back_time[0] if idx_back_time.shape[0] == 0: break - - - borrowed_indexes.append([index_diff_s,index_diff_e+1]) - #print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2]) - #print('diff new_synced_time_vector2', np.diff(new_synced_time_vector2[index_diff_s-1:index_diff_e+2])) - #print('diff new_synced_time_vector2', np.diff(behavior_time_vector[index_diff_s-1:index_diff_e+2])) - + borrowed_indexes.append([index_diff_s, index_diff_e + 1]) + # print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2]) + # print('diff new_synced_time_vector2', np.diff(new_synced_time_vector2[index_diff_s-1:index_diff_e+2])) + # print('diff new_synced_time_vector2', np.diff(behavior_time_vector[index_diff_s-1:index_diff_e+2])) return new_synced_time_vector, borrowed_indexes @@ -202,7 +220,9 @@ def fix_sync_vector_greater(sync_time_vector, behavior_time_vector): new_sync_time_vector = sync_time_vector.copy() - diff_vecs = (new_sync_time_vector - behavior_time_vector[:new_sync_time_vector.shape[0]]) + diff_vecs = ( + new_sync_time_vector - behavior_time_vector[: new_sync_time_vector.shape[0]] + ) idx_plus = np.where(diff_vecs > 0) idx_plus = idx_plus[0] @@ -212,45 +232,59 @@ def fix_sync_vector_greater(sync_time_vector, behavior_time_vector): borrowed_indexes = [] for i in range(len(grouped_sections)): - if grouped_sections[i].shape[0] > 0: - - idx_start = grouped_sections[i][0]-1 + idx_start = grouped_sections[i][0] - 1 idx_end = grouped_sections[i][-1] - time_vector = behavior_time_vector[idx_start:idx_end+1]-behavior_time_vector[idx_start] - new_sync_time_vector[idx_start:idx_end+1] =\ - np.repeat(new_sync_time_vector[idx_start], idx_end-idx_start+1) + time_vector - + time_vector = ( + behavior_time_vector[idx_start : idx_end + 1] + - behavior_time_vector[idx_start] + ) + new_sync_time_vector[idx_start : idx_end + 1] = ( + np.repeat(new_sync_time_vector[idx_start], idx_end - idx_start + 1) + + time_vector + ) + borrowed_indexes.append([idx_start, idx_end]) - return new_sync_time_vector, borrowed_indexes def complete_last_part_sync_vec(sync_time_vector, behavior_time_vector): new_sync_time_vector = sync_time_vector.copy() - + diff_size = behavior_time_vector.shape[0] - new_sync_time_vector.shape[0] borrowed_indexes = [] if diff_size > 0: - diff_size = diff_size+1 + diff_size = diff_size + 1 - last_part_bt = behavior_time_vector[-diff_size:]- behavior_time_vector[-diff_size] + last_part_bt = ( + behavior_time_vector[-diff_size:] - behavior_time_vector[-diff_size] + ) insert_part = np.repeat(new_sync_time_vector[-1], diff_size) + last_part_bt - #print('last_part_bt',last_part_bt) + # print('last_part_bt',last_part_bt) - #print('insert_part',insert_part) + # print('insert_part',insert_part) new_sync_time_vector = np.append(new_sync_time_vector, insert_part[1:]) - borrowed_indexes.append([behavior_time_vector.shape[0]-diff_size, behavior_time_vector.shape[0]-1]) - + borrowed_indexes.append( + [ + behavior_time_vector.shape[0] - diff_size, + behavior_time_vector.shape[0] - 1, + ] + ) + return new_sync_time_vector, borrowed_indexes - -def fix_iter_vector(synced_iteration_vector, synced_time_vector, original_time_vector, nidq_sampling_rate): + +def fix_iter_vector( + synced_iteration_vector, + synced_time_vector, + original_time_vector, + nidq_sampling_rate, +): new_synced_iteration_vector = synced_iteration_vector.copy() @@ -261,11 +295,17 @@ def fix_iter_vector(synced_iteration_vector, synced_time_vector, original_time_v first_iter = synced_iteration_vector[0] for i in range(diff_iter_times_idx.shape[0]): if diff_iter_times_idx[i] != 0: - new_synced_iteration_vector[i] = first_iter+int(synced_time_vector[i]*nidq_sampling_rate) + new_synced_iteration_vector[i] = first_iter + int( + synced_time_vector[i] * nidq_sampling_rate + ) if ori_shape != new_shape: - last_iterations = (np.round(first_iter+synced_time_vector[ori_shape:]*nidq_sampling_rate)) - new_synced_iteration_vector = np.append(new_synced_iteration_vector, last_iterations.astype(np.int64)) + last_iterations = np.round( + first_iter + synced_time_vector[ori_shape:] * nidq_sampling_rate + ) + new_synced_iteration_vector = np.append( + new_synced_iteration_vector, last_iterations.astype(np.int64) + ) return new_synced_iteration_vector @@ -273,21 +313,28 @@ def fix_iter_vector(synced_iteration_vector, synced_time_vector, original_time_v def sync_evaluation_process2(synced_time_vector, behavior_time_vector): status = 1 - diff_vector = synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]] + diff_vector = ( + synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]] + ) num_iter = diff_vector.shape[0] - #max_diff = max(diff_vector) + # max_diff = max(diff_vector) median_general = np.median(diff_vector) - num_div= 10 + num_div = 10 median_diff_percent = np.empty([num_div]) median_diff_abs = np.empty([num_div]) for j in range(num_div): - start_iter = int(j*num_iter/num_div) - end_iter = int((j+1)*num_iter/num_div) - median_diff_percent[j] = (np.median(diff_vector[start_iter:end_iter])-median_general)*100/median_general - median_diff_abs[j] = (np.median(diff_vector[start_iter:end_iter])-median_general) - + start_iter = int(j * num_iter / num_div) + end_iter = int((j + 1) * num_iter / num_div) + median_diff_percent[j] = ( + (np.median(diff_vector[start_iter:end_iter]) - median_general) + * 100 + / median_general + ) + median_diff_abs[j] = ( + np.median(diff_vector[start_iter:end_iter]) - median_general + ) if np.max(np.abs(median_diff_abs)) < 0.005: pass @@ -298,74 +345,92 @@ def sync_evaluation_process2(synced_time_vector, behavior_time_vector): return status -def main_ephys_fix_sync_code(iter_start_idx, iter_times_idx, behavior_time, nidq_sampling_rate): +def main_ephys_fix_sync_code( + iter_start_idx, iter_times_idx, behavior_time, nidq_sampling_rate +): iteration_dict = dict() - iteration_dict['iter_start_idx'] = list() - iteration_dict['iter_times_idx'] = list() + iteration_dict["iter_start_idx"] = list() + iteration_dict["iter_times_idx"] = list() for i in range(len(iter_start_idx)): - - #print('fixing trial ',i) + # print('fixing trial ',i) behavior_time_vector = behavior_time[i].flatten() - synced_time_vector, shift_vec, median_vec = get_shift_vector(iter_times_idx[i],behavior_time_vector) - + synced_time_vector, shift_vec, median_vec = get_shift_vector( + iter_times_idx[i], behavior_time_vector + ) - synced_time_vector,_ =\ - fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, shift_vec) + synced_time_vector, _ = fix_shifted_sync_vector( + synced_time_vector, behavior_time_vector, shift_vec + ) - #synced_time_vector, trial_stats_dict['borrow_step3'] =\ + # synced_time_vector, trial_stats_dict['borrow_step3'] =\ # fix_sync_vector_greater(synced_time_vector, behavior_time_vector) - synced_time_vector,_ =\ - complete_last_part_sync_vec(synced_time_vector, behavior_time_vector) + synced_time_vector, _ = complete_last_part_sync_vec( + synced_time_vector, behavior_time_vector + ) - synced_iteration_vector =\ - fix_iter_vector(iter_start_idx[i],synced_time_vector, iter_times_idx[i], nidq_sampling_rate) - + synced_iteration_vector = fix_iter_vector( + iter_start_idx[i], synced_time_vector, iter_times_idx[i], nidq_sampling_rate + ) - iteration_dict['iter_start_idx'].append(synced_iteration_vector.copy()) - iteration_dict['iter_times_idx'].append(synced_time_vector.copy()) + iteration_dict["iter_start_idx"].append(synced_iteration_vector.copy()) + iteration_dict["iter_times_idx"].append(synced_time_vector.copy()) - print('end fix sync code 1') + print("end fix sync code 1") - iteration_dict['iter_start_idx'] = np.asarray(iteration_dict['iter_start_idx'].copy(), dtype=object) - iteration_dict['iter_times_idx'] = np.asarray(iteration_dict['iter_times_idx'].copy(), dtype=object) + iteration_dict["iter_start_idx"] = np.asarray( + iteration_dict["iter_start_idx"].copy(), dtype=object + ) + iteration_dict["iter_times_idx"] = np.asarray( + iteration_dict["iter_times_idx"].copy(), dtype=object + ) - print('end fix sync code') + print("end fix sync code") # Check # of trials and iterations match - trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time) + trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ( + ephys_utils.assert_iteration_samples_count( + iteration_dict["iter_start_idx"], behavior_time + ) + ) - print('after assert_iteration_samples_count fix sync code') + print("after assert_iteration_samples_count fix sync code") if trial_count_diff != 0: - print('trial_count_diff', trial_count_diff) + print("trial_count_diff", trial_count_diff) if len(trials_diff_iteration_big) > 0: - print('trials_diff_iteration_big', trials_diff_iteration_big) + print("trials_diff_iteration_big", trials_diff_iteration_big) if len(trials_diff_iteration_small) > 0: - print('trials_diff_iteration_small', trials_diff_iteration_small) + print("trials_diff_iteration_small", trials_diff_iteration_small) + status = ephys_utils.evaluate_sync_process( + trial_count_diff, + trials_diff_iteration_big, + trials_diff_iteration_small, + behavior_time.shape[0], + ) + print("after evaluate_sync_process fix sync code") - status = ephys_utils.evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, behavior_time.shape[0]) - - print('after evaluate_sync_process fix sync code') - - for i in range(len(iteration_dict['iter_start_idx'])): - synced_time_vector = iteration_dict['iter_times_idx'][i] + for i in range(len(iteration_dict["iter_start_idx"])): + synced_time_vector = iteration_dict["iter_times_idx"][i] behavior_time_vector = behavior_time[i].flatten() status = sync_evaluation_process2(synced_time_vector, behavior_time_vector) if status == -1: break - print('after sync_evaluation_process2', status) - + print("after sync_evaluation_process2", status) + if status == 1: - iteration_dict['trial_start_idx'] = ephys_utils.get_index_trial_vector_from_iteration(iteration_dict['iter_start_idx']) + iteration_dict["trial_start_idx"] = ( + ephys_utils.get_index_trial_vector_from_iteration( + iteration_dict["iter_start_idx"] + ) + ) - print('after get_index_trial_vector_from_iteration') + print("after get_index_trial_vector_from_iteration") return status, iteration_dict - diff --git a/u19_pipeline/utils/ephys_utils.py b/u19_pipeline/utils/ephys_utils.py index d9ade41a..f9f10432 100644 --- a/u19_pipeline/utils/ephys_utils.py +++ b/u19_pipeline/utils/ephys_utils.py @@ -1,4 +1,3 @@ - import json import pathlib import warnings @@ -13,7 +12,6 @@ class spice_glx_utility: - @staticmethod def load_spice_glx_digital_file(file_path, nidq_meta, d_line_list=None): # Read NIDAQ digital file. @@ -24,80 +22,96 @@ def load_spice_glx_digital_file(file_path, nidq_meta, d_line_list=None): # Get all digitial channels if not provided if d_line_list is None: - list_start_end_chan = nidq_meta['niXDChans1'].split(sep=':') + list_start_end_chan = nidq_meta["niXDChans1"].split(sep=":") print(list_start_end_chan) - if len(list_start_end_chan) == 2: - d_line_list = list(range(int(list_start_end_chan[0]), int(list_start_end_chan[1])+1)) + if len(list_start_end_chan) == 2: + d_line_list = list( + range(int(list_start_end_chan[0]), int(list_start_end_chan[1]) + 1) + ) else: - raise ValueError('Could not infer channel list from nidq_meta["niXDChans1"] ') + raise ValueError( + 'Could not infer channel list from nidq_meta["niXDChans1"] ' + ) nidq_sampling_rate = readSGLX.SampRate(nidq_meta) - #Get first and last sample idx from the file + # Get first and last sample idx from the file t_start = 0 - t_end = np.float64(nidq_meta['fileTimeSecs']) + t_end = np.float64(nidq_meta["fileTimeSecs"]) dw = 0 first_sample_index = int(nidq_sampling_rate * t_start) last_sample_index = int(nidq_sampling_rate * t_end) - 1 - #Read binary and digital - nidq_raw_data = readSGLX.makeMemMapRaw(file_path, nidq_meta) # Pull raw bin data - digital_array = readSGLX.ExtractDigital( # extract interation index - nidq_raw_data, first_sample_index, last_sample_index, - dw, d_line_list, nidq_meta) + # Read binary and digital + nidq_raw_data = readSGLX.makeMemMapRaw( + file_path, nidq_meta + ) # Pull raw bin data + digital_array = readSGLX.ExtractDigital( # extract interation index + nidq_raw_data, + first_sample_index, + last_sample_index, + dw, + d_line_list, + nidq_meta, + ) return digital_array - + def read_nidq_meta_samp_rate(ephys_session_fullpath): - #Nidaq file - nidq_meta = readSGLX.readMeta(ephys_session_fullpath) + # Nidaq file + nidq_meta = readSGLX.readMeta(ephys_session_fullpath) nidq_sampling_rate = readSGLX.SampRate(nidq_meta) return nidq_meta, nidq_sampling_rate + def load_trial_iteration_signals(ephys_session_fullpath, nidq_meta): - # 1: load meta data, and the content of the NIDAQ file. Its content is digital. + # 1: load meta data, and the content of the NIDAQ file. Its content is digital. new_trial_channel = 1 new_iteration_channel = 2 # If PXIe card (nidq) card use for recording deduce digital channels - if nidq_meta['typeThis'] == 'nidq': - digital_array = spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta) + if nidq_meta["typeThis"] == "nidq": + digital_array = spice_glx_utility.load_spice_glx_digital_file( + ephys_session_fullpath, nidq_meta + ) # If onebox card (obx) card use for recording digital channels are 0-2 else: - digital_array = spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta, d_line_list=[0,1]) + digital_array = spice_glx_utility.load_spice_glx_digital_file( + ephys_session_fullpath, nidq_meta, d_line_list=[0, 1] + ) # If no sync pulse found trial and iteration signals are 0 & 1 respectively - channel0_pulses = np.where(np.diff(digital_array[0])==1)[0].shape[0] - channel1_pulses = np.where(np.diff(digital_array[1])==1)[0].shape[0] + channel0_pulses = np.where(np.diff(digital_array[0]) == 1)[0].shape[0] + channel1_pulses = np.where(np.diff(digital_array[1]) == 1)[0].shape[0] if channel0_pulses > channel1_pulses: new_trial_channel = 1 new_iteration_channel = 0 else: new_trial_channel = 0 - new_iteration_channel = 1 + new_iteration_channel = 1 - return digital_array[new_trial_channel,:], digital_array[new_iteration_channel,:] + return digital_array[new_trial_channel, :], digital_array[new_iteration_channel, :] def get_idx_trial_start(trial_pulse_signal): - #Get index of samples when trial has started based on a pulse signal + # Get index of samples when trial has started based on a pulse signal - #Get idx samples trial starts + # Get idx samples trial starts trial_start_idx = np.where(np.diff(trial_pulse_signal) == 1) trial_start_idx = trial_start_idx[0] - #Get idx samples trial pulse start end + # Get idx samples trial pulse start end trial_start_pulse_end_idx = np.where(np.diff(trial_pulse_signal) == 255) trial_start_pulse_end_idx = trial_start_pulse_end_idx[0] - #Detect fake trial init pulses (a single sample in 1 instead of 5ms signal) + # Detect fake trial init pulses (a single sample in 1 instead of 5ms signal) fake_trial_init = [] for idx, sample in enumerate(trial_start_idx): - #Get the mean value of next samples after rising edge detected - mean_pulse = np.mean(trial_pulse_signal[sample+1:sample+10]) + # Get the mean value of next samples after rising edge detected + mean_pulse = np.mean(trial_pulse_signal[sample + 1 : sample + 10]) # Average value should be 1 if mean_pulse < 0.9: fake_trial_init.append(idx) @@ -107,10 +121,16 @@ def get_idx_trial_start(trial_pulse_signal): return trial_start_idx, trial_start_pulse_end_idx -def get_idx_iter_start_pulsesignal(iteration_pulse_signal_trial, iteration_pulse_start_signal,trial_start_idx, samples_before_pulse_start, behavior_iterations): - #Get index of iteration starts on a trial based on a pulse start signal +def get_idx_iter_start_pulsesignal( + iteration_pulse_signal_trial, + iteration_pulse_start_signal, + trial_start_idx, + samples_before_pulse_start, + behavior_iterations, +): + # Get index of iteration starts on a trial based on a pulse start signal - #Get idx of iteration start during trial (pulse signal after trial start signal) + # Get idx of iteration start during trial (pulse signal after trial start signal) iter_samples = np.where(np.diff(iteration_pulse_signal_trial) == 1) iter_samples = np.squeeze(iter_samples) # First iteration is at trial start, just align first trial start @@ -119,12 +139,12 @@ def get_idx_iter_start_pulsesignal(iteration_pulse_signal_trial, iteration_pulse iter_start_samples = np.where(np.diff(iteration_pulse_start_signal) == 1) iter_start_samples = np.squeeze(iter_start_samples) - #print('iter_start_samples', iter_start_samples) + # print('iter_start_samples', iter_start_samples) - #print('iter_start_samples.size', iter_start_samples.size) - #print('type(iter_start_samples)', type(iter_start_samples)) + # print('iter_start_samples.size', iter_start_samples.size) + # print('type(iter_start_samples)', type(iter_start_samples)) - # If we find some kind of pulse at the start of + # If we find some kind of pulse at the start of if iter_start_samples.size > 0: iter_samples = np.insert(iter_samples, 0, trial_start_idx) else: @@ -137,15 +157,14 @@ def get_idx_iter_start_pulsesignal(iteration_pulse_signal_trial, iteration_pulse def get_idx_iter_start_counterbit(iteration_pulse_signal_trial, trial_start_idx): - #Get index of iteration starts on a trial based on a iteration bit0 counter - + # Get index of iteration starts on a trial based on a iteration bit0 counter - #Get idx of odd iteration during trial + # Get idx of odd iteration during trial iter_samples = np.where(np.diff(iteration_pulse_signal_trial) == 1) iter_samples = iter_samples + trial_start_idx if iteration_pulse_signal_trial[0] == 1: - #If last iteration was odd, insert a iteration at start + # If last iteration was odd, insert a iteration at start iter_samples = np.insert(iter_samples, 0, trial_start_idx) else: # First iteration is at trial start, just align first trial start @@ -153,7 +172,7 @@ def get_idx_iter_start_counterbit(iteration_pulse_signal_trial, trial_start_idx) iter_samples = np.squeeze(iter_samples) - #Get idx of even iteration during trial + # Get idx of even iteration during trial iter_samples2 = np.where(np.diff(iteration_pulse_signal_trial) == 255) iter_samples2 = iter_samples2 + trial_start_idx iter_samples2 = np.squeeze(iter_samples2) @@ -169,43 +188,55 @@ def get_trial_signal_mode(iteration_pulse_signal_trial, behavior_time_vector_tri # If iterations in trial are less than the ones in behavior, the mode was the counterbit iter_samples = np.where(np.diff(iteration_pulse_signal_trial) == 1) - if iter_samples[0].shape[0] < (behavior_time_vector_trial.shape[0]*3/4): - mode = 'counter_bit0' + if iter_samples[0].shape[0] < (behavior_time_vector_trial.shape[0] * 3 / 4): + mode = "counter_bit0" else: - mode = 'pulse_signal' + mode = "pulse_signal" - print('mode deduction: ', mode) + print("mode deduction: ", mode) return mode -def get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, iteration_pulse_signal, nidq_sampling_rate, num_behavior_trials, behavior_time_vector, mode=None) -> dict: +def get_iteration_sample_vector_from_digital_lines_pulses( + trial_pulse_signal, + iteration_pulse_signal, + nidq_sampling_rate, + num_behavior_trials, + behavior_time_vector, + mode=None, +) -> dict: - #Output as a dictionary + # Output as a dictionary iteration_vector_output = dict() - #Vectors that will contain trial # and iter # for each sample on file - #iteration_vector_output['framenumber_vector_samples'] = np.zeros(trial_pulse_signal.shape[0])*np.nan - #iteration_vector_output['trialnumber_vector_samples'] = np.zeros(trial_pulse_signal.shape[0])*np.nan + # Vectors that will contain trial # and iter # for each sample on file + # iteration_vector_output['framenumber_vector_samples'] = np.zeros(trial_pulse_signal.shape[0])*np.nan + # iteration_vector_output['trialnumber_vector_samples'] = np.zeros(trial_pulse_signal.shape[0])*np.nan - #Get idx samples trial starts + # Get idx samples trial starts trial_start_idx, trial_end_pulse_idx = get_idx_trial_start(trial_pulse_signal) - print('len trial_start_idx', trial_start_idx.shape) + print("len trial_start_idx", trial_start_idx.shape) if mode is None: - mode = get_trial_signal_mode(iteration_pulse_signal[trial_start_idx[0]:trial_start_idx[1]], behavior_time_vector[0]) + mode = get_trial_signal_mode( + iteration_pulse_signal[trial_start_idx[0] : trial_start_idx[1]], + behavior_time_vector[0], + ) # Just to make sure we get corresponding iter pulse (trial and iter pulse at same time !!) - if mode == 'counter_bit0': + if mode == "counter_bit0": ms_after_trial_start_pulse = 1 ms_before_trial_end = 1 else: ms_after_trial_start_pulse = -4 ms_before_trial_end = 4 - samples_after_pulse_start = int(nidq_sampling_rate*(ms_after_trial_start_pulse/1000)) - samples_before_pulse_end = int(nidq_sampling_rate*(ms_before_trial_end/1000)) + samples_after_pulse_start = int( + nidq_sampling_rate * (ms_after_trial_start_pulse / 1000) + ) + samples_before_pulse_end = int(nidq_sampling_rate * (ms_before_trial_end / 1000)) # num Trials to sync (if behavior stopped before last trial was saved) num_trials_sync = min([trial_start_idx.shape[0], num_behavior_trials]) @@ -213,44 +244,54 @@ def get_iteration_sample_vector_from_digital_lines_pulses(trial_pulse_signal, it iter_start_idx = [] iter_times_idx = [] for i in range(num_trials_sync): - #Trial starts for iteration signal when trial pulse ends () + # Trial starts for iteration signal when trial pulse ends () idx_start = trial_end_pulse_idx[i] idx_start_before = trial_start_idx[i] + samples_after_pulse_start - if i < trial_start_idx.shape[0]-1: - idx_end = trial_start_idx[i+1] -samples_before_pulse_end + if i < trial_start_idx.shape[0] - 1: + idx_end = trial_start_idx[i + 1] - samples_before_pulse_end else: idx_end = trial_pulse_signal.shape[0] - samples_before_pulse_end - - - #Get idx of iteration start of current trial - if mode == 'counter_bit0': - iter_samples = get_idx_iter_start_counterbit(iteration_pulse_signal[idx_start_before:idx_end], trial_start_idx[i]) + # Get idx of iteration start of current trial + if mode == "counter_bit0": + iter_samples = get_idx_iter_start_counterbit( + iteration_pulse_signal[idx_start_before:idx_end], trial_start_idx[i] + ) else: - iter_samples = get_idx_iter_start_pulsesignal(iteration_pulse_signal[idx_start:idx_end], iteration_pulse_signal[idx_start_before:idx_start], trial_start_idx[i], idx_start, behavior_time_vector[i].shape[0]) - - #Append as an array of arrays (each trial is an array with idx of iterations) + iter_samples = get_idx_iter_start_pulsesignal( + iteration_pulse_signal[idx_start:idx_end], + iteration_pulse_signal[idx_start_before:idx_start], + trial_start_idx[i], + idx_start, + behavior_time_vector[i].shape[0], + ) + + # Append as an array of arrays (each trial is an array with idx of iterations) iter_start_idx.append(iter_samples) - #Calculate time for each iteration start - times = iter_samples/nidq_sampling_rate + # Calculate time for each iteration start + times = iter_samples / nidq_sampling_rate times = times - times[0] iter_times_idx.append(times) - #Fill vector samples - #for j in range(iter_samples.shape[0]-1): + # Fill vector samples + # for j in range(iter_samples.shape[0]-1): # iteration_vector_output['framenumber_vector_samples'][iter_samples[j]:iter_samples[j+1]] = j+1 - #Last iteration # is from start of iteration to end of trial - #if i < trial_start_idx.shape[0]-1: + # Last iteration # is from start of iteration to end of trial + # if i < trial_start_idx.shape[0]-1: # iteration_vector_output['trialnumber_vector_samples'][trial_start_idx[i]:trial_start_idx[i+1]] = i+1 # iteration_vector_output['framenumber_vector_samples'][iter_samples[-1]:trial_start_idx[i+1]] = iter_samples.shape[0] # For last trial, lets finish it 1s after last iteration detected - #else: + # else: # iteration_vector_output['trialnumber_vector_samples'][trial_start_idx[i]:iter_samples[-1]+int(nidq_sampling_rate)] = i+1 # iteration_vector_output['framenumber_vector_samples'][iter_samples[-1]:iter_samples[-1]+int(nidq_sampling_rate)] = iter_samples.shape[0] - iteration_vector_output['iter_start_idx'] = np.asarray(iter_start_idx.copy(), dtype=object) - iteration_vector_output['iter_times_idx'] = np.asarray(iter_times_idx.copy(), dtype=object) + iteration_vector_output["iter_start_idx"] = np.asarray( + iter_start_idx.copy(), dtype=object + ) + iteration_vector_output["iter_times_idx"] = np.asarray( + iter_times_idx.copy(), dtype=object + ) return iteration_vector_output @@ -261,35 +302,54 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite # ... and also get start and end time framenumber = np.zeros(digital_array.shape[1]) for i in range(digital_array.shape[1]): - a = BitArray(np.flip(digital_array[1:, i])) # ignore 0-bit, as this is the NPX sync puls, and not virmen. + a = BitArray( + np.flip(digital_array[1:, i]) + ) # ignore 0-bit, as this is the NPX sync puls, and not virmen. framenumber[i] = a.uint - iterations_raw = np.array(framenumber, dtype=np.int32) # Transform frames into integer - recording_start = np.min(np.where(iterations_raw>0)) #first chane of testlist - recording_end = np.where(np.abs(np.diff(iterations_raw))>0)[0][-1] + 200 # Adding a random 200 measurements, so ~40ms at our usual 5kHz sampling rate. + iterations_raw = np.array( + framenumber, dtype=np.int32 + ) # Transform frames into integer + recording_start = np.min(np.where(iterations_raw > 0)) # first chane of testlist + recording_end = ( + np.where(np.abs(np.diff(iterations_raw)) > 0)[0][-1] + 200 + ) # Adding a random 200 measurements, so ~40ms at our usual 5kHz sampling rate. # Second, transform `iterations_raw` into `framenumber_in_trial` and `trialnumber` - framenumber_in_trial = np.zeros(len(iterations_raw))*np.nan - trialnumber = np.zeros(len(iterations_raw))*np.nan + framenumber_in_trial = np.zeros(len(iterations_raw)) * np.nan + trialnumber = np.zeros(len(iterations_raw)) * np.nan current_trial = 0 overflow = 0 iter_start_idx = [] for idx, frame_number in enumerate(iterations_raw): - if (idx>recording_start) & (idx recording_start) & (idx < recording_end): + if (frame_number == 0) & ( + iterations_raw[idx - 1] == 127 + ): # At the reset, add 128 overflow = overflow + 1 - if (frame_number==0) & (iterations_raw[idx-1]!=127) & (iterations_raw[idx-1]!=0) & (iterations_raw[idx-2]==127): # Unlucky reset if happened to be sampled at the wrong time + if ( + (frame_number == 0) + & (iterations_raw[idx - 1] != 127) + & (iterations_raw[idx - 1] != 0) + & (iterations_raw[idx - 2] == 127) + ): # Unlucky reset if happened to be sampled at the wrong time overflow = overflow + 1 - framenumber_in_trial[idx-1] = frame_number + overflow*128 - 1 # In case this happened, the previous sample has to be corrected + framenumber_in_trial[idx - 1] = ( + frame_number + overflow * 128 - 1 + ) # In case this happened, the previous sample has to be corrected # Keep track of trial number - endflag = framenumber_in_trial[idx-1] == (len(time[current_trial])) #Trial end has been reached. - transitionflag = frame_number < 3 # Next trial should start at zero again - if endflag & transitionflag: # Only at the transitions + endflag = framenumber_in_trial[idx - 1] == ( + len(time[current_trial]) + ) # Trial end has been reached. + transitionflag = frame_number < 3 # Next trial should start at zero again + if endflag & transitionflag: # Only at the transitions current_trial = current_trial + 1 # Increases trial count - overflow = 0 # Reset the 7 bit counter - iter_start_idx.append(idx) # Make a note when this happened - framenumber_in_trial[idx] = frame_number + overflow*128 - 1 + overflow = 0 # Reset the 7 bit counter + iter_start_idx.append(idx) # Make a note when this happened + framenumber_in_trial[idx] = frame_number + overflow * 128 - 1 trialnumber[idx] = current_trial - trial_list = np.array(np.unique(trialnumber[np.isfinite(trialnumber)]), dtype = np.int32) + trial_list = np.array( + np.unique(trialnumber[np.isfinite(trialnumber)]), dtype=np.int32 + ) # Fourth, find and remove the nidaq glitches # These are single samples where the iteration number is corrupted @@ -300,62 +360,87 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite din = np.diff(framenumber_in_trial) trial_transitions = np.where(np.diff(trialnumber)) glitches = [] - for candidate in np.where( np.logical_or(din>1, din<0) )[0]: # skipped frames or counting down + for candidate in np.where(np.logical_or(din > 1, din < 0))[ + 0 + ]: # skipped frames or counting down if np.sum(candidate == trial_transitions) == 0: glitches = np.append(glitches, candidate) - glitches = np.array(glitches, dtype = np.int32) + glitches = np.array(glitches, dtype=np.int32) # Attempt to remove them skipped_frames = 0 for g in glitches: - if framenumber_in_trial[g] < framenumber_in_trial[g+2]: - if framenumber_in_trial[g+2] - framenumber_in_trial[g] == 2: # skipped frame, should be very rare - framenumber_in_trial[g+1] = framenumber_in_trial[g]+1 + if framenumber_in_trial[g] < framenumber_in_trial[g + 2]: + if ( + framenumber_in_trial[g + 2] - framenumber_in_trial[g] == 2 + ): # skipped frame, should be very rare + framenumber_in_trial[g + 1] = framenumber_in_trial[g] + 1 skipped_frames = skipped_frames + 1 - else: # If random number, nidaq sample in the middle of update. - framenumber_in_trial[g+1] = framenumber_in_trial[g] + else: # If random number, nidaq sample in the middle of update. + framenumber_in_trial[g + 1] = framenumber_in_trial[g] # This point we have framenumber_in_trial and trialnumber. Now just some refactoring to fit into the usual data structure iteration_vector_output = dict() - iteration_vector_output['trialnumber_vector_samples'] = trialnumber - iteration_vector_output['framenumber_vector_samples'] = framenumber_in_trial + iteration_vector_output["trialnumber_vector_samples"] = trialnumber + iteration_vector_output["framenumber_vector_samples"] = framenumber_in_trial iter_start_idx = [] for t in trial_list: - iter_start_idx.append( np.arange(0, framenumber_in_trial[trialnumber==t][-1], 1)) - iteration_vector_output['iter_start_idx'] = np.asarray(iter_start_idx.copy(), dtype=object) + iter_start_idx.append( + np.arange(0, framenumber_in_trial[trialnumber == t][-1], 1) + ) + iteration_vector_output["iter_start_idx"] = np.asarray( + iter_start_idx.copy(), dtype=object + ) return iteration_vector_output + def assert_iteration_samples_count(iteration_sample_idx_output, behavior_time_vector): - #Assert that vector sync pulses match behavior time vector + # Assert that vector sync pulses match behavior time vector # Count trial count differences - trial_count_diff = np.abs(iteration_sample_idx_output.shape[0] - (behavior_time_vector.shape[0])) + trial_count_diff = np.abs( + iteration_sample_idx_output.shape[0] - (behavior_time_vector.shape[0]) + ) count = 0 trials_diff_iteration_small = list() trials_diff_iteration_big = list() for idx_trial, iter_trials in enumerate(iteration_sample_idx_output): if iter_trials.shape[0] != behavior_time_vector[idx_trial].shape[0]: - print('trial#', count, - 'iterPulses:', iter_trials.shape[0], - 'IterBeh:', behavior_time_vector[idx_trial].shape[0], - 'Difference (', behavior_time_vector[idx_trial].shape[0]-iter_trials.shape[0], ')') + print( + "trial#", + count, + "iterPulses:", + iter_trials.shape[0], + "IterBeh:", + behavior_time_vector[idx_trial].shape[0], + "Difference (", + behavior_time_vector[idx_trial].shape[0] - iter_trials.shape[0], + ")", + ) count += 1 # For each trial iteration # should be equal to the behavioral file iterations if iter_trials.shape[0] != behavior_time_vector[idx_trial].shape[0]: - if np.abs(iter_trials.shape[0] - behavior_time_vector[idx_trial].shape[0]) < 6: + if ( + np.abs(iter_trials.shape[0] - behavior_time_vector[idx_trial].shape[0]) + < 6 + ): trials_diff_iteration_small.append(idx_trial) else: trials_diff_iteration_big.append(idx_trial) - return trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small -def evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, total_trials): +def evaluate_sync_process( + trial_count_diff, + trials_diff_iteration_big, + trials_diff_iteration_small, + total_trials, +): """ Check if all sync process ran smoothly, we need to redo some trials or it's not worth it @@ -366,144 +451,179 @@ def evaluate_sync_process(trial_count_diff, trials_diff_iteration_big, trials_di """ if len(trials_diff_iteration_big) > 1: - print('Missed by a lot some trials: ', trials_diff_iteration_big) + print("Missed by a lot some trials: ", trials_diff_iteration_big) status = -1 return status - if len(trials_diff_iteration_big) == 1 and trials_diff_iteration_big[0] == total_trials-1: - print('Missed by a lot last trial: (Assume recording stopped earlier) ', trials_diff_iteration_big) + if ( + len(trials_diff_iteration_big) == 1 + and trials_diff_iteration_big[0] == total_trials - 1 + ): + print( + "Missed by a lot last trial: (Assume recording stopped earlier) ", + trials_diff_iteration_big, + ) status = 1 return status # All trials synced perfectly - if trial_count_diff ==0 and len(trials_diff_iteration_small) == 0: - print('Synced perfectly xxxxxxxxxxxxxx') + if trial_count_diff == 0 and len(trials_diff_iteration_small) == 0: + print("Synced perfectly xxxxxxxxxxxxxx") status = 1 return status # We miss last trial (surely recording was stop before behavior) if trial_count_diff < 2 and len(trials_diff_iteration_small) == 0: - print('Missed one trial signal xxxxxxxxxxxxxx') + print("Missed one trial signal xxxxxxxxxxxxxx") status = -1 return status # Iterations differ in more than two trials if len(trials_diff_iteration_small) > 2: - print('Missed iteration count on many trials: ', len(trials_diff_iteration_small)) + print( + "Missed iteration count on many trials: ", len(trials_diff_iteration_small) + ) status = -1 return status if trial_count_diff < 2 and len(trials_diff_iteration_small) <= 2: - print('Missed num trials: ', trial_count_diff) - print('Missed iteration count in how many trials: ', len(trials_diff_iteration_small)) - print('Trying to fix trials') + print("Missed num trials: ", trial_count_diff) + print( + "Missed iteration count in how many trials: ", + len(trials_diff_iteration_small), + ) + print("Trying to fix trials") status = 0 return status else: status = -1 - print('Missed by a lot of trials, everything different or missing') + print("Missed by a lot of trials, everything different or missing") return status - -def fix_missing_iteration_trials(trials_diff_iteration_small, iteration_dict, behavior_times, nidq_sampling_rate): +def fix_missing_iteration_trials( + trials_diff_iteration_small, iteration_dict, behavior_times, nidq_sampling_rate +): # Fix and insert missing synced iteration vectors - print('trials_diff_iteration_small', trials_diff_iteration_small) + print("trials_diff_iteration_small", trials_diff_iteration_small) print(type(trials_diff_iteration_small)) - # For each bad synced trial (Should be only a few) for i in range(len(trials_diff_iteration_small)): - - #Insert missing iterations on synced vector + # Insert missing iterations on synced vector idx_trial = trials_diff_iteration_small[i] - status, new_iter_start = insert_missing_synced_iteration(iteration_dict['iter_start_idx'][idx_trial],\ - iteration_dict['iter_times_idx'][idx_trial], behavior_times[idx_trial].flatten()) + status, new_iter_start = insert_missing_synced_iteration( + iteration_dict["iter_start_idx"][idx_trial], + iteration_dict["iter_times_idx"][idx_trial], + behavior_times[idx_trial].flatten(), + ) if not status: raise ValueError("Coud not find missing iteration in trial") - iteration_dict['iter_start_idx'][idx_trial] = new_iter_start + iteration_dict["iter_start_idx"][idx_trial] = new_iter_start # Get new synced time vector for trial - new_times = new_iter_start/nidq_sampling_rate + new_times = new_iter_start / nidq_sampling_rate new_times = new_times - new_times[0] - iteration_dict['iter_times_idx'][idx_trial] = new_times + iteration_dict["iter_times_idx"][idx_trial] = new_times # Fix framenumber in the sample vector - #for j in range(new_iter_start.shape[0]-1): + # for j in range(new_iter_start.shape[0]-1): # iteration_dict['framenumber_vector_samples'][new_iter_start[j]:new_iter_start[j+1]] = j+1 - #Last iteration fixed as well - #next_trial = idx_trial+1 - #start_iteration_next_trial = iteration_dict['iter_start_idx'][next_trial][0] + # Last iteration fixed as well + # next_trial = idx_trial+1 + # start_iteration_next_trial = iteration_dict['iter_start_idx'][next_trial][0] - #print('next_trial ........', next_trial) - #print('last iteration of this trial so far', new_iter_start[-1]) - #print('start next trial iteration', start_iteration_next_trial) + # print('next_trial ........', next_trial) + # print('last iteration of this trial so far', new_iter_start[-1]) + # print('start next trial iteration', start_iteration_next_trial) - #iteration_dict['framenumber_vector_samples'][new_iter_start[-1]:start_iteration_next_trial] = new_iter_start.shape[0] + # iteration_dict['framenumber_vector_samples'][new_iter_start[-1]:start_iteration_next_trial] = new_iter_start.shape[0] return iteration_dict -def insert_missing_synced_iteration(synced_iteration_vector, synced_time_vector, behavior_time_vector): +def insert_missing_synced_iteration( + synced_iteration_vector, synced_time_vector, behavior_time_vector +): # Check where is more likely we miss an iteration pulse and insert it to iteration_vector status = 1 - print('synced_iteration_vector', synced_iteration_vector.shape[0]) - print('synced_time_vector', synced_time_vector.shape[0]) - print('behavior_time_vector', behavior_time_vector.shape[0]) + print("synced_iteration_vector", synced_iteration_vector.shape[0]) + print("synced_time_vector", synced_time_vector.shape[0]) + print("behavior_time_vector", behavior_time_vector.shape[0]) # Get in which indexes we get a "peak" of non matching times... if synced_time_vector.shape[0] >= behavior_time_vector.shape[0]: - print('More pulses than behavior iterations, check other method') # Christian: What is the other method? + print( + "More pulses than behavior iterations, check other method" + ) # Christian: What is the other method? status = -1 return status, np.empty(0) else: - diff_vector = np.diff(synced_time_vector - behavior_time_vector[:synced_time_vector.shape[0]]) - #In case last peak is at the end of trial (append 0 to detect it) + diff_vector = np.diff( + synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]] + ) + # In case last peak is at the end of trial (append 0 to detect it) diff_vector = np.append(diff_vector, np.array([0])) peaks, _ = sp.find_peaks(diff_vector, height=0.05, distance=20) - print('peaks here .............', peaks) + print("peaks here .............", peaks) print(peaks.shape) - # Insert extra iterations as a new "iteration" start to match behavior iterations new_synced_iteration_vector = synced_iteration_vector.copy() for i in range(peaks.shape[0]): - value_insert = (synced_iteration_vector[peaks[i]] + synced_iteration_vector[peaks[i]+1] ) /2 - new_synced_iteration_vector = np.insert(new_synced_iteration_vector, peaks[i], value_insert) - + value_insert = ( + synced_iteration_vector[peaks[i]] + synced_iteration_vector[peaks[i] + 1] + ) / 2 + new_synced_iteration_vector = np.insert( + new_synced_iteration_vector, peaks[i], value_insert + ) if new_synced_iteration_vector.shape[0] != behavior_time_vector.shape[0]: - print('with peak strategy, could not find correct missing iterations') + print("with peak strategy, could not find correct missing iterations") status = -1 return status, np.empty(0) return status, new_synced_iteration_vector - # Deprecated -def behavior_sync_frame_counter_method(digital_array, behavior_time_vector, session_trial_keys, nidq_sampling_rate, bit_start, number_bits): - - max_count = np.power(2, number_bits)-1 - - # 2: transform the digital lines into a number, save in an array of integers +def behavior_sync_frame_counter_method( + digital_array, + behavior_time_vector, + session_trial_keys, + nidq_sampling_rate, + bit_start, + number_bits, +): + + max_count = np.power(2, number_bits) - 1 + + # 2: transform the digital lines into a number, save in an array of integers # ... and also get start and end time framenumber = np.zeros(digital_array.shape[1]) for i in range(digital_array.shape[1]): - a = BitArray(np.flip(digital_array[bit_start:, i])) # ignore 0-bit, as this is the NPX sync puls, and not virmen. + a = BitArray( + np.flip(digital_array[bit_start:, i]) + ) # ignore 0-bit, as this is the NPX sync puls, and not virmen. framenumber[i] = a.uint - iterations_raw = np.array(framenumber, dtype=np.int) # Transform frames into integer - recording_start = np.min(np.where(iterations_raw>0)) #Get start of recording: first change of testlist - dt = np.int(0.04*nidq_sampling_rate) - recording_end = np.where(np.abs(np.diff(iterations_raw))>0)[0][-1] + dt #Get end of recording + iterations_raw = np.array( + framenumber, dtype=np.int + ) # Transform frames into integer + recording_start = np.min( + np.where(iterations_raw > 0) + ) # Get start of recording: first change of testlist + dt = np.int(0.04 * nidq_sampling_rate) + recording_end = ( + np.where(np.abs(np.diff(iterations_raw)) > 0)[0][-1] + dt + ) # Get end of recording # 3: transform `iterations_raw` into `framenumber_in_trial` and `trialnumber` # iterations_raw is just a number between 0 and max_count+1. Some math has to be done to obtain: @@ -511,34 +631,52 @@ def behavior_sync_frame_counter_method(digital_array, behavior_time_vector, sess # trialnumber has the length of the number of samples of the NIDAQ card, and every entry is the current trial. # # NOTE: some minor glitches have to be catched, if a NIDAQ sample happenes to be recorded while the VR System updates the iteration number. - framenumber_in_trial = np.zeros(len(iterations_raw))*np.nan - trialnumber = np.zeros(len(iterations_raw))*np.nan + framenumber_in_trial = np.zeros(len(iterations_raw)) * np.nan + trialnumber = np.zeros(len(iterations_raw)) * np.nan current_trial = 0 - overflow = 0 # This variable keep track whenever the reset from max_count to 0 happens. + overflow = ( + 0 # This variable keep track whenever the reset from max_count to 0 happens. + ) for idx, frame_number in enumerate(iterations_raw): - if (idx>recording_start) & (idx recording_start) & (idx < recording_end): + # print(iterations_raw2[idx], frame_number) + if (frame_number == 0) & ( + iterations_raw[idx - 1] == max_count + ): # At the reset, add max_count+1 overflow = overflow + 1 - if (frame_number==0) & (iterations_raw[idx-1]!=max_count) & (iterations_raw[idx-1]!=0) & (iterations_raw[idx-2]==max_count): # Unlucky reset if happened to be sampled at the wrong time + if ( + (frame_number == 0) + & (iterations_raw[idx - 1] != max_count) + & (iterations_raw[idx - 1] != 0) + & (iterations_raw[idx - 2] == max_count) + ): # Unlucky reset if happened to be sampled at the wrong time overflow = overflow + 1 - framenumber_in_trial[idx-1] = frame_number + overflow*(max_count+1) - 1 # In case this happened, the previous sample has to be corrected + framenumber_in_trial[idx - 1] = ( + frame_number + overflow * (max_count + 1) - 1 + ) # In case this happened, the previous sample has to be corrected # Keep track of trial number - endflag = framenumber_in_trial[idx-1] == (len(behavior_time_vector[current_trial])) # Trial end has been reached. - - transitionflag = frame_number == 2 # Next trial should start at zero again (it starts with two ??) - if endflag & transitionflag: # Only at the transitions - + endflag = framenumber_in_trial[idx - 1] == ( + len(behavior_time_vector[current_trial]) + ) # Trial end has been reached. + + transitionflag = ( + frame_number == 2 + ) # Next trial should start at zero again (it starts with two ??) + if endflag & transitionflag: # Only at the transitions current_trial = current_trial + 1 # Increases trial count - overflow = 0 # Reset the 7 bit counter for the next trial + overflow = 0 # Reset the 7 bit counter for the next trial if overflow == 0: framenumber_in_trial[idx] = frame_number else: - framenumber_in_trial[idx] = frame_number + overflow*(max_count+1) -1 + framenumber_in_trial[idx] = ( + frame_number + overflow * (max_count + 1) - 1 + ) trialnumber[idx] = current_trial - trial_list = np.array(np.unique(trialnumber[np.isfinite(trialnumber)]), dtype = np.int64) + trial_list = np.array( + np.unique(trialnumber[np.isfinite(trialnumber)]), dtype=np.int64 + ) # 4: find and remove additional NIDAQ glitches of two types: # a) single samples where the iteration number is corrupted because sampling happened faster than output of the behevior PC. @@ -547,45 +685,60 @@ def behavior_sync_frame_counter_method(digital_array, behavior_time_vector, sess din = np.diff(framenumber_in_trial) trial_transitions = np.where(np.diff(trialnumber)) glitches = [] - for candidate in np.where( np.logical_or(din>1, din<0) )[0]: # skipped frames or counting down + for candidate in np.where(np.logical_or(din > 1, din < 0))[ + 0 + ]: # skipped frames or counting down if np.sum(candidate == trial_transitions) == 0: glitches = np.append(glitches, candidate) - glitches = np.array(glitches, dtype = np.int) + glitches = np.array(glitches, dtype=np.int) # Attempt to remove them skipped_frames = 0 for g in glitches: - if framenumber_in_trial[g] < framenumber_in_trial[g+2]: - if framenumber_in_trial[g+2] - framenumber_in_trial[g] == 2: # skipped frame, should be very rare + if framenumber_in_trial[g] < framenumber_in_trial[g + 2]: + if ( + framenumber_in_trial[g + 2] - framenumber_in_trial[g] == 2 + ): # skipped frame, should be very rare pass - framenumber_in_trial[g+1] = framenumber_in_trial[g]+1 + framenumber_in_trial[g + 1] = framenumber_in_trial[g] + 1 skipped_frames = skipped_frames + 1 - else: # If random number, nidaq sample in the middle of update. - framenumber_in_trial[g+1] = framenumber_in_trial[g] + else: # If random number, nidaq sample in the middle of update. + framenumber_in_trial[g + 1] = framenumber_in_trial[g] # A set of final asserts, making sure that the code worked as intended - assert len(trial_list) == len(session_trial_keys) # Make sure the trial number is correct. - assert np.sum(np.diff(framenumber_in_trial)>1) == 0 # No frames should be skipped - assert np.sum(np.diff(framenumber_in_trial)<0) 1) == 0 # No frames should be skipped + assert np.sum(np.diff(framenumber_in_trial) < 0) < len( + trial_list + ) # Negative iterations only at trial transitions iterations_test = 0 for t in trial_list: - iterations_test = iterations_test + framenumber_in_trial[trialnumber==t][-1] # Integrate number of iterations - assert framenumber_in_trial[trialnumber==t][-1] == len(behavior_time_vector[t]) # Make sure number of nidaq-frames in each trial is identical to dj record: - nidaqtime = np.sum(trialnumber == t)/nidq_sampling_rate + iterations_test = ( + iterations_test + framenumber_in_trial[trialnumber == t][-1] + ) # Integrate number of iterations + assert framenumber_in_trial[trialnumber == t][-1] == len( + behavior_time_vector[t] + ) # Make sure number of nidaq-frames in each trial is identical to dj record: + nidaqtime = np.sum(trialnumber == t) / nidq_sampling_rate matlabtime = np.max(behavior_time_vector[t]) - assert ((nidaqtime - matlabtime) / matlabtime) < 0.1 # # Make sure the nidaq-trial-duration and dj records are consistent; 10% arbitrarily chosen + assert ( + ((nidaqtime - matlabtime) / matlabtime) < 0.1 + ) # # Make sure the nidaq-trial-duration and dj records are consistent; 10% arbitrarily chosen nidaq_duration = iterations_test + skipped_frames - #dj_duration = iterstart[-1] + len(behavior_time_vector[-1]) - #assert np.abs(nidaq_duration - dj_duration) < 3 # at most two frames off - sometimes this happens at the beginning/end of the recording + # dj_duration = iterstart[-1] + len(behavior_time_vector[-1]) + # assert np.abs(nidaq_duration - dj_duration) < 3 # at most two frames off - sometimes this happens at the beginning/end of the recording # If this is done, and the asserts are passed, insert the data into the database return (framenumber_in_trial, trialnumber) - def future_counter_get_signal(): pass -''' + + +""" # Cleaner way to get iteration number from counter, still need debugging, if necessary framenumber = np.zeros(idx_end-idx_start) @@ -611,22 +764,26 @@ def future_counter_get_signal(): framenumber_in_trial[idx] = frame_number + overflow*(max_count+1) print(np.max(framenumber_in_trial)) -''' +""" + def load_open_ephys_digital_file(file_path): pass -def get_iteration_intertrial_from_virmen_time(trial_pulse_signal, nidq_sampling_rate, num_behavior_trials, behavior_time_vector): +def get_iteration_intertrial_from_virmen_time( + trial_pulse_signal, nidq_sampling_rate, num_behavior_trials, behavior_time_vector +): - #Get idx samples trial starts - trial_start_idx,_ = get_idx_trial_start(trial_pulse_signal) + # Get idx samples trial starts + trial_start_idx, _ = get_idx_trial_start(trial_pulse_signal) iter_start_idx = [] for i in range(num_behavior_trials): - - new_synced_iteration_vector = trial_start_idx[i]+np.int64(behavior_time_vector[i]*nidq_sampling_rate) + new_synced_iteration_vector = trial_start_idx[i] + np.int64( + behavior_time_vector[i] * nidq_sampling_rate + ) iter_start_idx.append(new_synced_iteration_vector.squeeze()) iter_start_idx = np.asarray(iter_start_idx.copy(), dtype=object) @@ -636,62 +793,73 @@ def get_iteration_intertrial_from_virmen_time(trial_pulse_signal, nidq_sampling_ def get_full_vector_samples(iter_start_idx_vectors, nidq_sampling_rate, total_samples): - framenumber_vector_samples = np.zeros(total_samples)*np.nan - trialnumber_vector_samples = np.zeros(total_samples)*np.nan + framenumber_vector_samples = np.zeros(total_samples) * np.nan + trialnumber_vector_samples = np.zeros(total_samples) * np.nan for i in range(len(iter_start_idx_vectors)): - this_trial_iter_vector = iter_start_idx_vectors[i] - if i < len(iter_start_idx_vectors)-1: - next_trial_iter_vector = iter_start_idx_vectors[i+1] - - #Fill vector samples - for j in range(this_trial_iter_vector.shape[0]-1): - framenumber_vector_samples[this_trial_iter_vector[j]:this_trial_iter_vector[j+1]] = j+1 - - #Last iteration # is from start of iteration to end of trial - if i < len(iter_start_idx_vectors)-1: - trialnumber_vector_samples[this_trial_iter_vector[0]:next_trial_iter_vector[0]] = i+1 - framenumber_vector_samples[this_trial_iter_vector[-1]:next_trial_iter_vector[0]] = this_trial_iter_vector.shape[0] + if i < len(iter_start_idx_vectors) - 1: + next_trial_iter_vector = iter_start_idx_vectors[i + 1] + + # Fill vector samples + for j in range(this_trial_iter_vector.shape[0] - 1): + framenumber_vector_samples[ + this_trial_iter_vector[j] : this_trial_iter_vector[j + 1] + ] = j + 1 + + # Last iteration # is from start of iteration to end of trial + if i < len(iter_start_idx_vectors) - 1: + trialnumber_vector_samples[ + this_trial_iter_vector[0] : next_trial_iter_vector[0] + ] = i + 1 + framenumber_vector_samples[ + this_trial_iter_vector[-1] : next_trial_iter_vector[0] + ] = this_trial_iter_vector.shape[0] # For last trial, lets finish it 1s after last iteration detected else: - #print('total_samples', total_samples) - #print('this_trial_iter_vector[-1]', this_trial_iter_vector[-1]) - #print('this_trial_iter_vector[0]', this_trial_iter_vector[0]) - #print('this_trial_iter_vector[-1]+int(nidq_sampling_rate)', this_trial_iter_vector[-1]+int(nidq_sampling_rate)) - trialnumber_vector_samples[this_trial_iter_vector[0]:this_trial_iter_vector[-1]+int(nidq_sampling_rate)] = i+1 - framenumber_vector_samples[this_trial_iter_vector[-1]:this_trial_iter_vector[-1]+int(nidq_sampling_rate)] = this_trial_iter_vector.shape[0] + # print('total_samples', total_samples) + # print('this_trial_iter_vector[-1]', this_trial_iter_vector[-1]) + # print('this_trial_iter_vector[0]', this_trial_iter_vector[0]) + # print('this_trial_iter_vector[-1]+int(nidq_sampling_rate)', this_trial_iter_vector[-1]+int(nidq_sampling_rate)) + trialnumber_vector_samples[ + this_trial_iter_vector[0] : this_trial_iter_vector[-1] + + int(nidq_sampling_rate) + ] = i + 1 + framenumber_vector_samples[ + this_trial_iter_vector[-1] : this_trial_iter_vector[-1] + + int(nidq_sampling_rate) + ] = this_trial_iter_vector.shape[0] return trialnumber_vector_samples, framenumber_vector_samples + def get_time_vector_as_behavior(trial_idx_vector, nidq_sampling_rate): first_iter_session = trial_idx_vector[0][0] - time_0 = first_iter_session/nidq_sampling_rate + time_0 = first_iter_session / nidq_sampling_rate trial_times_ind = [] trial_times_full = [] for i in range(trial_idx_vector.shape[0]): - - time_vector = trial_idx_vector[i]/nidq_sampling_rate + time_vector = trial_idx_vector[i] / nidq_sampling_rate time_vector_ind = time_vector - time_vector[0] time_vector_full = time_vector - time_0 trial_times_ind.append(time_vector_ind) trial_times_full.append(time_vector_full) - trial_times_ind= np.asarray(trial_times_ind.copy(), dtype=object) + trial_times_ind = np.asarray(trial_times_ind.copy(), dtype=object) trial_times_full = np.asarray(trial_times_full.copy(), dtype=object) - return trial_times_ind, trial_times_full + def get_time_vector(trial_index_nidq, nidq_sampling_rate): time_vector = np.arange(0, trial_index_nidq.shape[0]) time_vector = time_vector.astype(np.float64) - time_vector = time_vector/nidq_sampling_rate + time_vector = time_vector / nidq_sampling_rate session_start_index = np.where(trial_index_nidq == 1)[0][0] time_vector -= time_vector[session_start_index] @@ -699,7 +867,6 @@ def get_time_vector(trial_index_nidq, nidq_sampling_rate): return time_vector - def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling_rate): status = False @@ -712,14 +879,15 @@ def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling trial_index_nidq[:first_non_nan] = 0 iteration_index_nidq[:first_non_nan] = -1 - idx_trial = np.where((np.diff(trial_index_nidq)) == 1 )[0] + 1 + idx_trial = np.where((np.diff(trial_index_nidq)) == 1)[0] + 1 idx_iteration = np.where((np.diff(iteration_index_nidq)) == 1)[0] + 1 idx_iteration_final = [] for i in range(idx_trial.shape[0]): - - if i < idx_trial.shape[0]-1: - iterations_this_trial = idx_iteration[(idx_iteration > idx_trial[i]) & (idx_iteration idx_trial[i]) & (idx_iteration < idx_trial[i + 1]) + ] else: iterations_this_trial = idx_iteration[(idx_iteration > idx_trial[i])] idx_this_trial_iterations = np.insert(iterations_this_trial, 0, idx_trial[i]) @@ -729,17 +897,26 @@ def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling trial_index_nidq[:first_non_nan] = np.nan iteration_index_nidq[:first_non_nan] = np.nan - trial_index_nidq2, iteration_index_nidq2 = get_full_vector_samples(idx_iteration_final, nidq_sampling_rate, trial_index_nidq.shape[0]) - + trial_index_nidq2, iteration_index_nidq2 = get_full_vector_samples( + idx_iteration_final, nidq_sampling_rate, trial_index_nidq.shape[0] + ) - iteration_equal = np.allclose(iteration_index_nidq, iteration_index_nidq2, equal_nan=True) + iteration_equal = np.allclose( + iteration_index_nidq, iteration_index_nidq2, equal_nan=True + ) trial_equal = np.allclose(trial_index_nidq, trial_index_nidq2, equal_nan=True) - print('iteration_equal', iteration_equal) - print('trial_equal', trial_equal) + print("iteration_equal", iteration_equal) + print("trial_equal", trial_equal) - print('np.where(np.isnan(iteration_index_nidq))[0].shape', np.where(np.isnan(iteration_index_nidq))[0].shape) - print('np.where(np.isnan(iteration_index_nidq2))[0].shape', np.where(np.isnan(iteration_index_nidq2))[0].shape) + print( + "np.where(np.isnan(iteration_index_nidq))[0].shape", + np.where(np.isnan(iteration_index_nidq))[0].shape, + ) + print( + "np.where(np.isnan(iteration_index_nidq2))[0].shape", + np.where(np.isnan(iteration_index_nidq2))[0].shape, + ) if trial_equal and iteration_equal: status = True @@ -747,25 +924,25 @@ def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling return status, idx_trial, idx_iteration_final - def get_index_trial_vector_from_iteration(iteration_start_idx): trial_start_idx = np.zeros(len(iteration_start_idx), dtype=np.int64) for i in range(len(iteration_start_idx)): - trial_start_idx[i] = iteration_start_idx[i][0] return trial_start_idx class xyz_pick_file_creator: - ''' + """ Class that handles probe coordinates locations given initial isertion coordinates & shank coordinates - ''' + """ @staticmethod - def main_xyz_pick_file_function(recording_id, fragment_number, chanmap_file, processed_data_directory): + def main_xyz_pick_file_function( + recording_id, fragment_number, chanmap_file, processed_data_directory + ): """ Stores xyz_pick_files on ibl_postprocess directory for ibl_atlas_gui Input: @@ -776,23 +953,33 @@ def main_xyz_pick_file_function(recording_id, fragment_number, chanmap_file, pro """ # Check existance of ibl output path - ibl_output_dir = pathlib.Path(dj.config['custom']['ephys_root_data_dir'][1], processed_data_directory, 'ibl_data') + ibl_output_dir = pathlib.Path( + dj.config["custom"]["ephys_root_data_dir"][1], + processed_data_directory, + "ibl_data", + ) if not ibl_output_dir.is_dir(): pathlib.Path.mkdir(ibl_output_dir) # Get recording id - probe_location = xyz_pick_file_creator.get_probe_insertion_coordinates(recording_id, fragment_number) + probe_location = xyz_pick_file_creator.get_probe_insertion_coordinates( + recording_id, fragment_number + ) print(probe_location) - #Load channelmap and check how many probes there are + # Load channelmap and check how many probes there are chanmap = loadmat(chanmap_file) - max_shank = int(np.max(chanmap['kcoords'])) + max_shank = int(np.max(chanmap["kcoords"])) # Calculate probe coordinates and store files all_shanks = list() for i in range(max_shank): - probe_track = xyz_pick_file_creator.get_probetrack(chanmap, shank=i+1, **probe_location) - xyz_pick_file_creator.save_xyz_pick_file(ibl_output_dir, probe_track, shank=i) + probe_track = xyz_pick_file_creator.get_probetrack( + chanmap, shank=i + 1, **probe_location + ) + xyz_pick_file_creator.save_xyz_pick_file( + ibl_output_dir, probe_track, shank=i + ) all_shanks.append(probe_track) return all_shanks @@ -806,39 +993,67 @@ def get_probe_insertion_coordinates(recording_id, probe_num): probe_num (int) = Reference to probe# of current recording """ - coordinates_columns = ['real_ap_coordinates', 'real_depth_coordinates', 'real_ml_coordinates', 'phi_angle', 'theta_angle', 'rho_angle'] + coordinates_columns = [ + "real_ap_coordinates", + "real_depth_coordinates", + "real_ml_coordinates", + "phi_angle", + "theta_angle", + "rho_angle", + ] probes_not_found = False # Create virtual modules of needed DBs - action_db = dj.create_virtual_module('action', 'u19_action') - recording_db = dj.create_virtual_module('recording', 'u19_recording') + action_db = dj.create_virtual_module("action", "u19_action") + recording_db = dj.create_virtual_module("recording", "u19_recording") # Query subject of recording - query = {'recording_id': recording_id} - subject_recording = (recording_db.Recording.BehaviorSession & query).fetch('subject_fullname') + query = {"recording_id": recording_id} + subject_recording = (recording_db.Recording.BehaviorSession & query).fetch( + "subject_fullname" + ) if subject_recording.shape[0] != 0: # Query probe insertion table - query_surgery = {'subject_fullname': subject_recording[0], 'device_idx': probe_num} - probe_location = (action_db.SurgeryLocation & query_surgery).fetch(*coordinates_columns, as_dict=True) + query_surgery = { + "subject_fullname": subject_recording[0], + "device_idx": probe_num, + } + probe_location = (action_db.SurgeryLocation & query_surgery).fetch( + *coordinates_columns, as_dict=True + ) if len(probe_location) == 0: probes_not_found = True else: probe_location = probe_location[0] # Convert to float - probe_location = {k:float(v)for (k,v) in probe_location.items()} + probe_location = {k: float(v) for (k, v) in probe_location.items()} else: probes_not_found = True # If insertion decive is not found, create a dummy one but raise a warning if probes_not_found: - warnings.warn("Warning probe location was not found on DB for recording_id: " + str(recording_id) + " & probe# " + str(probe_num) ) + warnings.warn( + "Warning probe location was not found on DB for recording_id: " + + str(recording_id) + + " & probe# " + + str(probe_num) + ) probe_location = dict.fromkeys(coordinates_columns, 0) return probe_location @staticmethod - def get_probetrack(chanmap, shank=1, real_ml_coordinates=0, real_ap_coordinates=0, real_depth_coordinates=0, phi_angle=0, theta_angle=0, rho_angle=0): + def get_probetrack( + chanmap, + shank=1, + real_ml_coordinates=0, + real_ap_coordinates=0, + real_depth_coordinates=0, + phi_angle=0, + theta_angle=0, + rho_angle=0, + ): """ Build numpy array with "brain" coordinates from insertion device and probe features Input: @@ -853,30 +1068,44 @@ def get_probetrack(chanmap, shank=1, real_ml_coordinates=0, real_ap_coordinates= """ # Step 1: Convert degrees to radiants and reformat chanmap - phi = phi_angle*np.pi/180 - theta = theta_angle*np.pi/180 - roll = rho_angle*np.pi/180 - x = np.array([i[0] for i in chanmap['xcoords']]) - y = np.array([i[0] for i in chanmap['ycoords']]) - k = np.array([i[0] for i in chanmap['kcoords']]) + phi = phi_angle * np.pi / 180 + theta = theta_angle * np.pi / 180 + roll = rho_angle * np.pi / 180 + x = np.array([i[0] for i in chanmap["xcoords"]]) + y = np.array([i[0] for i in chanmap["ycoords"]]) + k = np.array([i[0] for i in chanmap["kcoords"]]) # Step 2: Transform them into 3D, assuming the Probe is perpendicular to x|y plane - avx = np.mean(x[k==shank]) # Center "X" on middle of probe - probe_x0 = np.array([np.cos(roll)*avx, -np.sin(roll)*avx]) # coordinates of the shank after "roll" around probe axis - probe_length = np.max(y[k==shank]) - np.min(y[k==shank]) # Length off the probe per chanmap - probe_unitVec = np.array([np.sin(theta)*np.cos(phi), np.sin(theta)*np.sin(phi), np.cos(theta)]) # Unit vector point along insertion direction - - probe_length = np.arange(-real_depth_coordinates*1000, -real_depth_coordinates*1000 + probe_length, 10) # Resulution of future xyz_pick.json file + avx = np.mean(x[k == shank]) # Center "X" on middle of probe + probe_x0 = np.array( + [np.cos(roll) * avx, -np.sin(roll) * avx] + ) # coordinates of the shank after "roll" around probe axis + probe_length = np.max(y[k == shank]) - np.min( + y[k == shank] + ) # Length off the probe per chanmap + probe_unitVec = np.array( + [np.sin(theta) * np.cos(phi), np.sin(theta) * np.sin(phi), np.cos(theta)] + ) # Unit vector point along insertion direction + + probe_length = np.arange( + -real_depth_coordinates * 1000, + -real_depth_coordinates * 1000 + probe_length, + 10, + ) # Resulution of future xyz_pick.json file # Step 3: Produce the 3D coordinates along the probe track - probe_track = np.zeros((len(probe_length),3)) + probe_track = np.zeros((len(probe_length), 3)) for i in range(len(probe_length)): - probe_track[i,:] = probe_length[i]*probe_unitVec + np.array([probe_x0[0], probe_x0[1], 0]) + probe_track[i, :] = probe_length[i] * probe_unitVec + np.array( + [probe_x0[0], probe_x0[1], 0] + ) # Step 4: Shift probe my ML|AP insertion coordinates probe_track_shifted = np.zeros(probe_track.shape) for i in range(len(probe_track_shifted)): - probe_track_shifted[i,:] = probe_track[i,:] + np.array([real_ml_coordinates*1000, real_ap_coordinates*1000, 0]) + probe_track_shifted[i, :] = probe_track[i, :] + np.array( + [real_ml_coordinates * 1000, real_ap_coordinates * 1000, 0] + ) return probe_track_shifted.tolist() @@ -889,11 +1118,16 @@ def save_xyz_pick_file(save_directory, probe_coord_data, shank=0): shank (int) = Shank # (0 index based) (different filename depending on it) """ - filenames = ['xyz_picks.json', 'xyz_picks_shank1.json', 'xyz_picks_shank2.json', 'xyz_picks_shank3.json'] + filenames = [ + "xyz_picks.json", + "xyz_picks_shank1.json", + "xyz_picks_shank2.json", + "xyz_picks_shank3.json", + ] final_filename = pathlib.Path(save_directory, filenames[shank]).as_posix() dict_coord = dict() dict_coord["xyz_picks"] = probe_coord_data - with open(final_filename, 'w') as fp: - json.dump(dict_coord, fp) \ No newline at end of file + with open(final_filename, "w") as fp: + json.dump(dict_coord, fp) diff --git a/u19_pipeline/utils/file_utils.py b/u19_pipeline/utils/file_utils.py index 901a4529..11added6 100644 --- a/u19_pipeline/utils/file_utils.py +++ b/u19_pipeline/utils/file_utils.py @@ -5,14 +5,14 @@ def write_file(path, text): os.umask(0) descriptor = os.open( - path=path, - flags=( - os.O_WRONLY # access mode: write only - | os.O_CREAT # create if not exists - | os.O_TRUNC # truncate the file to zero - ), - mode=0o664 + path=path, + flags=( + os.O_WRONLY # access mode: write only + | os.O_CREAT # create if not exists + | os.O_TRUNC # truncate the file to zero + ), + mode=0o664, ) - with open(descriptor, 'w') as fh: - fh.write(text) \ No newline at end of file + with open(descriptor, "w") as fh: + fh.write(text) diff --git a/u19_pipeline/utils/insert_miscelaneous_db.py b/u19_pipeline/utils/insert_miscelaneous_db.py index b34eb875..351aa7d1 100644 --- a/u19_pipeline/utils/insert_miscelaneous_db.py +++ b/u19_pipeline/utils/insert_miscelaneous_db.py @@ -1,4 +1,3 @@ - import os import pathlib @@ -10,26 +9,28 @@ def add_researcher_user_table(user_id, full_name, email, phone): # Values sent by function caller new_key = dict() - new_key['user_id'] = user_id - new_key['user_nickname'] = user_id - new_key['full_name'] = full_name - new_key['email'] = email - new_key['phone'] = phone + new_key["user_id"] = user_id + new_key["user_nickname"] = user_id + new_key["full_name"] = full_name + new_key["email"] = email + new_key["phone"] = phone # Write default values for User - new_key['mobile_carrier'] = 'none' - new_key['slack'] = full_name - new_key['contact_via'] = 'Slack' - new_key['primary_tech'] = 'N/A' - new_key['tech_responsibility']='yes' - new_key['day_cutoff_time']= np.array([18, 0]) + new_key["mobile_carrier"] = "none" + new_key["slack"] = full_name + new_key["contact_via"] = "Slack" + new_key["primary_tech"] = "N/A" + new_key["tech_responsibility"] = "yes" + new_key["day_cutoff_time"] = np.array([18, 0]) # Find conf file (to load it even if we are in different path) - repository_dir = os.path.abspath(os.path.realpath(__file__)+ "/../../..") - conf_file = pathlib.Path(repository_dir, 'dj_local_conf.json') + repository_dir = os.path.abspath(os.path.realpath(__file__) + "/../../..") + conf_file = pathlib.Path(repository_dir, "dj_local_conf.json") if not pathlib.Path.exists(conf_file): - raise Exception("No configuration file found, configure repository first: https://braincogs.github.io/software/db_access.html#db-access-for-python-repository") + raise Exception( + "No configuration file found, configure repository first: https://braincogs.github.io/software/db_access.html#db-access-for-python-repository" + ) config = dj.settings.Config() config.load(str(conf_file)) @@ -37,4 +38,5 @@ def add_researcher_user_table(user_id, full_name, email, phone): # Connect and insert record dj.conn() import u19_pipeline.lab as lab + lab.User.insert1(new_key) diff --git a/u19_pipeline/utils/matlab_utils.py b/u19_pipeline/utils/matlab_utils.py index dc7d7045..cb3efabf 100644 --- a/u19_pipeline/utils/matlab_utils.py +++ b/u19_pipeline/utils/matlab_utils.py @@ -1,4 +1,5 @@ """Authors: Ben Dichter, Cody Baker.""" + import os import sys from collections.abc import Iterable @@ -60,13 +61,15 @@ def find_discontinuities(tt, factor=10000): def mat_obj_to_dict(mat_struct): """Recursive function to convert nested matlab struct objects to dictionaries.""" dict_from_struct = {} - for field_name in mat_struct.__dict__['_fieldnames']: + for field_name in mat_struct.__dict__["_fieldnames"]: dict_from_struct[field_name] = mat_struct.__dict__[field_name] if isinstance(dict_from_struct[field_name], matlab.mio5_params.mat_struct): dict_from_struct[field_name] = mat_obj_to_dict(dict_from_struct[field_name]) elif isinstance(dict_from_struct[field_name], np.ndarray): try: - dict_from_struct[field_name] = mat_obj_to_array(dict_from_struct[field_name]) + dict_from_struct[field_name] = mat_obj_to_array( + dict_from_struct[field_name] + ) except TypeError: continue return dict_from_struct @@ -76,7 +79,9 @@ def mat_obj_to_array(mat_struct_array): """Construct array from matlab cell arrays. Recursively converts array elements if they contain mat objects.""" if has_struct(mat_struct_array): - array_from_cell = [mat_obj_to_dict(mat_struct) for mat_struct in mat_struct_array] + array_from_cell = [ + mat_obj_to_dict(mat_struct) for mat_struct in mat_struct_array + ] array_from_cell = np.array(array_from_cell) else: array_from_cell = mat_struct_array @@ -87,7 +92,9 @@ def mat_obj_to_array(mat_struct_array): def has_struct(mat_struct_array): """Determines if a matlab cell array contains any mat objects.""" return any( - isinstance(mat_struct, matlab.mio5_params.mat_struct) for mat_struct in mat_struct_array) + isinstance(mat_struct, matlab.mio5_params.mat_struct) + for mat_struct in mat_struct_array + ) def convert_mat_file_to_dict(mat_file_name): @@ -145,8 +152,8 @@ def flatten_nested_dict(nested_dict): def convert_function_handle_to_str(mat_file_path): """Executes a matlab script which converts function handle values to str - if matlab is installed on the system.""" - matlab_class = ''' + if matlab is installed on the system.""" + matlab_class = """ classdef Choice < uint32 enumeration @@ -180,8 +187,8 @@ def convert_function_handle_to_str(mat_file_path): end end - ''' - matlab_code = r''' + """ + matlab_code = r""" %Support when behavioral files has 2 "versions" if length(log.version) > 1 @@ -231,33 +238,33 @@ def convert_function_handle_to_str(mat_file_path): fclose(fid); quit; - ''' + """ - with Path('Choice.m').open('w') as f: + with Path("Choice.m").open("w") as f: f.write(matlab_class) metadata = {} convert_script_code = f"filePath = '{mat_file_path}';\nload(filePath);{matlab_code}" convert_script_path = Path("convert_function_to_txt.m") - with convert_script_path.open('w') as f: + with convert_script_path.open("w") as f: f.write(convert_script_code) - if 'win' in sys.platform and sys.platform != 'darwin': - matlab_cmd = ''' + if "win" in sys.platform and sys.platform != "darwin": + matlab_cmd = """ #!/bin/bash matlab -nosplash -wait -log -r convert_function_to_txt - ''' + """ else: - matlab_cmd = ''' + matlab_cmd = """ #!/bin/bash matlab -nosplash -nodisplay -log -r convert_function_to_txt - ''' + """ - if which('matlab') is not None: + if which("matlab") is not None: try: os.system(matlab_cmd) - + with open("trial_choice.txt") as f: trial_choice = f.read().splitlines() with open("trial_type.txt") as f: @@ -269,11 +276,11 @@ def convert_function_handle_to_str(mat_file_path): with open("protocol.txt") as f: protocol = f.readline() - metadata['experiment_name'] = version - metadata['protocol_name'] = protocol - metadata['trial_choice'] = trial_choice - metadata['trial_type'] = trial_type - metadata['shaping_protocol'] = shaping_protocol + metadata["experiment_name"] = version + metadata["protocol_name"] = protocol + metadata["trial_choice"] = trial_choice + metadata["trial_type"] = trial_type + metadata["shaping_protocol"] = shaping_protocol os.remove("code_version.txt") os.remove("protocol.txt") @@ -282,16 +289,21 @@ def convert_function_handle_to_str(mat_file_path): os.remove("shaping_protocol.txt") except Exception as e: - print(f"There was an error while trying to execute {convert_script_path}:\n{e}") + print( + f"There was an error while trying to execute {convert_script_path}:\n{e}" + ) else: - print("A working matlab version was not found. " - "Code version, animal protocol, type of trial, and choice could not be saved to NWB.") + print( + "A working matlab version was not found. " + "Code version, animal protocol, type of trial, and choice could not be saved to NWB." + ) os.remove("Choice.m") os.remove("convert_function_to_txt.m") return metadata + def convert_towers_block_trial_2_df(current_block_trial, block_num): """ Convert block trial data from matlab file to a Pandas DataFrame @@ -303,26 +315,29 @@ def convert_towers_block_trial_2_df(current_block_trial, block_num): ------- pandas.DataFrame """ - + valid_block = 0 # "Normal" blocks are stored as numpy arrays and its length is greater than 0 if isinstance(current_block_trial, np.ndarray) and current_block_trial.shape[0] > 0: current_block_trial = current_block_trial.tolist() - valid_block = 1 + valid_block = 1 # One trial blocks are stored as dictionaries if isinstance(current_block_trial, dict): current_block_trial = [current_block_trial] - valid_block = 1 + valid_block = 1 if valid_block: block_trial_df = pd.DataFrame(current_block_trial) - block_trial_df.insert(loc=0, column='trial_idx', value=np.arange(len(block_trial_df))+1) - block_trial_df.insert(loc=0, column='block', value=block_num) + block_trial_df.insert( + loc=0, column="trial_idx", value=np.arange(len(block_trial_df)) + 1 + ) + block_trial_df.insert(loc=0, column="block", value=block_num) else: block_trial_df = pd.DataFrame() return valid_block, block_trial_df + def convert_towers_block_2_df(current_block, num_block): """ Convert block data from matlab file to a Pandas DataFrame @@ -339,16 +354,16 @@ def convert_towers_block_2_df(current_block, num_block): # "Normal" blocks are stored as numpy arrays and its length is greater than 0 if isinstance(current_block, np.ndarray) and current_block_trial.shape[0] > 0: current_block = current_block.tolist() - valid_block = 1 + valid_block = 1 # One trial blocks are stored as dictionaries if isinstance(current_block, dict): current_block = [current_block] - valid_block = 1 + valid_block = 1 if valid_block: block_df = pd.DataFrame(current_block) - block_df.insert(loc=0, column='block', value=num_block) - block_df = block_df.drop(['trial'], axis=1) + block_df.insert(loc=0, column="block", value=num_block) + block_df = block_df.drop(["trial"], axis=1) else: block_df = pd.DataFrame() @@ -375,46 +390,51 @@ def convert_behavior_file(mat_file): session_block_trial_df = pd.DataFrame() - #Convert all blocks trials to dataframe and append them + # Convert all blocks trials to dataframe and append them - #For a single block sessions - if isinstance(matin['log']['block'], dict): + # For a single block sessions + if isinstance(matin["log"]["block"], dict): length_blocks = 1 dict_block = 1 - #For multiple block sessions + # For multiple block sessions else: - length_blocks = matin['log']['block'].shape[0] + length_blocks = matin["log"]["block"].shape[0] dict_block = 0 - #Convert all blocks of the session + # Convert all blocks of the session num_blocks_conv = 0 for i in range(length_blocks): - if dict_block: - block = matin['log']['block'] + block = matin["log"]["block"] else: - block = matin['log']['block'][i] - - #Convert trial df and block df - valid_block, block_trial_df = convert_towers_block_trial_2_df(block['trial'],i+1) - valid_blocks, block_df = convert_towers_block_2_df(block, i+1) - #Write string of block level Protocol (from matlab obscured data) + block = matin["log"]["block"][i] + + # Convert trial df and block df + valid_block, block_trial_df = convert_towers_block_trial_2_df( + block["trial"], i + 1 + ) + valid_blocks, block_df = convert_towers_block_2_df(block, i + 1) + # Write string of block level Protocol (from matlab obscured data) if metadata_read: - block_df['shapingProtocol'] = converted_metadata['shaping_protocol'][i] + block_df["shapingProtocol"] = converted_metadata["shaping_protocol"][i] if valid_block and valid_blocks: - session_current_block_trial_df = block_trial_df.merge(block_df, on='block', suffixes=['_block', '_trial']) + session_current_block_trial_df = block_trial_df.merge( + block_df, on="block", suffixes=["_block", "_trial"] + ) if num_blocks_conv == 0: session_block_trial_df = session_current_block_trial_df.copy() else: - session_block_trial_df = pd.concat([session_block_trial_df, session_current_block_trial_df],ignore_index=True) - num_blocks_conv +=1 - + session_block_trial_df = pd.concat( + [session_block_trial_df, session_current_block_trial_df], + ignore_index=True, + ) + num_blocks_conv += 1 - #Write choice and trial type of each trial (from matlab obscured data) + # Write choice and trial type of each trial (from matlab obscured data) if metadata_read: - session_block_trial_df['choice'] = converted_metadata['trial_choice'] - session_block_trial_df['trialType'] = converted_metadata['trial_type'] - + session_block_trial_df["choice"] = converted_metadata["trial_choice"] + session_block_trial_df["trialType"] = converted_metadata["trial_type"] + session_block_trial_df = session_block_trial_df.reset_index(drop=True) return session_block_trial_df diff --git a/u19_pipeline/utils/path_utils.py b/u19_pipeline/utils/path_utils.py index 9cce7bdf..8fddb387 100644 --- a/u19_pipeline/utils/path_utils.py +++ b/u19_pipeline/utils/path_utils.py @@ -1,16 +1,16 @@ - import glob import os import subprocess file_patterns_acq = { - "raw_imaging": ['/*.tiff', '/*.tif'], - "commpressed_imaging": ['/*.tiff.gz', '/*.tif.gz'], - "segmented_imaging_files": ['/*.modeling.mat', '/*suite2p/'], - "raw_np_files": ['/*ap.bin', '/*ap.meta'], - "sorted_np_files": ['/*.npy'] + "raw_imaging": ["/*.tiff", "/*.tif"], + "commpressed_imaging": ["/*.tiff.gz", "/*.tif.gz"], + "segmented_imaging_files": ["/*.modeling.mat", "/*suite2p/"], + "raw_np_files": ["/*ap.bin", "/*ap.meta"], + "sorted_np_files": ["/*.npy"], } + def check_file_pattern_dir(filepath, file_patterns): """ Check if directory (or its childs) contains some files with specific pattern names @@ -20,7 +20,7 @@ def check_file_pattern_dir(filepath, file_patterns): patterns_found = 0 for dir in child_dirs: for pat in file_patterns: - found_file = glob.glob(dir+pat) + found_file = glob.glob(dir + pat) if len(found_file) > 0: patterns_found = 1 break @@ -33,23 +33,25 @@ def check_file_pattern_dir(filepath, file_patterns): else: return 0 + def get_filepattern_paths(filepath, file_pattern): """ Check directory/files that correspond to file pattern in filepath """ - found_patterns = glob.glob(filepath+file_pattern) + found_patterns = glob.glob(filepath + file_pattern) return found_patterns + def get_size_directory(path): """ get directory size of a folder for linux systems """ - command = ["du", path, '-s'] + command = ["du", path, "-s"] s = subprocess.run(command, capture_output=True) - output = s.stdout.decode('UTF-8') + output = s.stdout.decode("UTF-8") if len(output) != 0: - kbytes = int(output.split('\t')[0]) + kbytes = int(output.split("\t")[0]) else: kbytes = -1 return kbytes @@ -62,18 +64,16 @@ def get_size_directory_time(path): command = ["du", "--separate-dirs", "--time", path] output = subprocess.check_output(command) - output = output.decode('UTF-8') + output = output.decode("UTF-8") list_values = output.split("\n") list_return = [] for line in list_values: line_values = line.split("\t") dict_line = dict() if len(line_values) == 3: - dict_line['size'] = line_values[0] - dict_line['date'] = line_values[1] - dict_line['directory'] = line_values[2] + dict_line["size"] = line_values[0] + dict_line["date"] = line_values[1] + dict_line["directory"] = line_values[2] list_return.append(dict_line) - - return list_return - + return list_return diff --git a/u19_pipeline/utils/scp_transfers.py b/u19_pipeline/utils/scp_transfers.py index 8191f77d..c27e9324 100644 --- a/u19_pipeline/utils/scp_transfers.py +++ b/u19_pipeline/utils/scp_transfers.py @@ -1,4 +1,3 @@ - import os import subprocess import sys @@ -12,7 +11,7 @@ public_key_location as public_key_location, ) -#Steps on windows machine +# Steps on windows machine # https://thesysadminchannel.com/solved-add-windowscapability-failed-error-code-0x800f0954-rsat-fix/ # PowerShell # Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 @@ -30,6 +29,7 @@ # PowerShell # restart-service sshd + class RemoteClient: """Client to interact with a remote host via SSH & SCP.""" @@ -40,21 +40,17 @@ def __init__(self, host, user, ssh_key_filepath, remote_path): self.remote_path = remote_path def _get_ssh_key(self): - """ Fetch locally stored SSH key.""" + """Fetch locally stored SSH key.""" try: - self.ssh_key = RSAKey.from_private_key_file( - self.ssh_key_filepath - ) - print( - f"Found SSH key at self {self.ssh_key_filepath}" - ) + self.ssh_key = RSAKey.from_private_key_file(self.ssh_key_filepath) + print(f"Found SSH key at self {self.ssh_key_filepath}") return self.ssh_key except SSHException as e: print(e) @property def connection(self): - """Open connection to remote host. """ + """Open connection to remote host.""" try: client = SSHClient() client.load_system_host_keys() @@ -67,9 +63,7 @@ def connection(self): ) return client except AuthenticationException as e: - print( - f"Authentication failed: did you remember to create an SSH key? {e}" - ) + print(f"Authentication failed: did you remember to create an SSH key? {e}") raise e @property @@ -102,25 +96,39 @@ def transfer_scp(host=None, username=None, remote_path=None, local_path=None): rc.download_folder(remote_path=remote_path, local_path=local_path) rc.disconnect() -def call_scp_background(ip_address=None, system_user=None, recording_system_directory=None, data_directory=None): + +def call_scp_background( + ip_address=None, + system_user=None, + recording_system_directory=None, + data_directory=None, +): print(ip_address, system_user, data_directory, recording_system_directory) - #transfer_scp(rec_series['ip_address'], rec_series['system_user'], rec_series['local_directory'], full_remote_path) + # transfer_scp(rec_series['ip_address'], rec_series['system_user'], rec_series['local_directory'], full_remote_path) this_file = os.path.realpath(__file__) - - p = subprocess.Popen(["nohup", "python", this_file, ip_address, system_user, recording_system_directory, data_directory, "&"]) + p = subprocess.Popen( + [ + "nohup", + "python", + this_file, + ip_address, + system_user, + recording_system_directory, + data_directory, + "&", + ] + ) # To test without nohup - #p = subprocess.run(["python", this_file, ip_address, system_user, recording_system_directory, data_directory], capture_output=True) - #print('stderr', p.stderr.decode('UTF-8')) - #print('stdout', p.stdout.decode('UTF-8')) - #print('p.returncode', p.returncode) - - return True,p.pid - + # p = subprocess.run(["python", this_file, ip_address, system_user, recording_system_directory, data_directory], capture_output=True) + # print('stderr', p.stderr.decode('UTF-8')) + # print('stdout', p.stdout.decode('UTF-8')) + # print('p.returncode', p.returncode) + return True, p.pid def check_scp_transfer(pid): @@ -144,6 +152,7 @@ def check_scp_transfer(pid): return finished, exit_code + def check_directory_copied_correctly(): pass # diff -r -q /path/to/dir1 /path/to/dir2 @@ -153,7 +162,6 @@ def check_directory_copied_correctly(): args = sys.argv[1:] print(args) - transfer_scp(host=args[0], username=args[1], remote_path=args[2], local_path=args[3]) - - - + transfer_scp( + host=args[0], username=args[1], remote_path=args[2], local_path=args[3] + ) diff --git a/u19_pipeline/utils/slack_utils.py b/u19_pipeline/utils/slack_utils.py index d78f1c9f..3c37557c 100644 --- a/u19_pipeline/utils/slack_utils.py +++ b/u19_pipeline/utils/slack_utils.py @@ -12,12 +12,20 @@ def get_webhook_list(slack_dict, lab): webhooks_list = [] if "slack_notification_channel" in slack_dict: - query_slack_webhooks = [{"webhook_name": x} for x in slack_dict["slack_notification_channel"]] - webhooks_list += (lab.SlackWebhooks & query_slack_webhooks).fetch("webhook_url").tolist() + query_slack_webhooks = [ + {"webhook_name": x} for x in slack_dict["slack_notification_channel"] + ] + webhooks_list += ( + (lab.SlackWebhooks & query_slack_webhooks).fetch("webhook_url").tolist() + ) if "slack_users_channel" in slack_dict: - query_slack_user_channels = [{"user_id": x} for x in slack_dict["slack_users_channel"]] - webhooks_list += (lab.User & query_slack_user_channels).fetch("slack_webhook").tolist() + query_slack_user_channels = [ + {"user_id": x} for x in slack_dict["slack_users_channel"] + ] + webhooks_list += ( + (lab.User & query_slack_user_channels).fetch("slack_webhook").tolist() + ) return webhooks_list @@ -30,9 +38,9 @@ def format_df_for_slack_message(df): for i in column_list: df[i] = df[i].astype(str) max_len = max([df[i].str.len().max(), len(i)]) - df[i] = df[i].str.pad(width=max_len, side='right', fillchar='_') - column_pad = i.ljust(max_len, '_') - df = df.rename(columns={i:column_pad}) + df[i] = df[i].str.pad(width=max_len, side="right", fillchar="_") + column_pad = i.ljust(max_len, "_") + df = df.rename(columns={i: column_pad}) df = df.to_string(index=False) df = df.replace(" ", " --- ") @@ -43,7 +51,9 @@ def format_df_for_slack_message(df): def send_slack_notification(webhook_url, slack_json_message): byte_length = str(sys.getsizeof(slack_json_message)) headers = {"Content-Type": "application/json", "Content-Length": byte_length} - response = requests.post(webhook_url, data=json.dumps(slack_json_message), headers=headers) + response = requests.post( + webhook_url, data=json.dumps(slack_json_message), headers=headers + ) response.raise_for_status() @@ -59,7 +69,9 @@ def send_slack_update_notification(webhook_url, base_message, session_info): m1["type"] = "section" m1_1 = dict() m1_1["type"] = "mrkdwn" - m1_1["text"] = ":white_check_mark: *Automation pipeline update* on " + datestr + "\n\n" + m1_1["text"] = ( + ":white_check_mark: *Automation pipeline update* on " + datestr + "\n\n" + ) m1["text"] = m1_1 # Info # @@ -90,7 +102,9 @@ def send_slack_update_notification(webhook_url, base_message, session_info): message = dict() message["blocks"] = [m1, msep, m2] - message["text"] = "Automation pipeline update recording:" + str(session_info["recording_id"]) + message["text"] = "Automation pipeline update recording:" + str( + session_info["recording_id"] + ) logging.info("Slack payload: %s", json.dumps(message, default=str)) @@ -156,7 +170,9 @@ def send_slack_error_notification(webhook_url, error_info, session_info): message = dict() message["blocks"] = [m1, msep, m2, msep, m3] - message["text"] = "Automation pipeline error in recording:" + str(session_info["recording_id"]) + message["text"] = "Automation pipeline error in recording:" + str( + session_info["recording_id"] + ) logging.info("Slack payload: %s", json.dumps(message, default=str)) @@ -226,7 +242,9 @@ def send_slack_error_pupillometry_notification(webhook_url, error_info, session_ send_slack_notification(webhook_url, message) -def send_slack_pupillometry_update_notification(webhook_url, base_message, session_info): +def send_slack_pupillometry_update_notification( + webhook_url, base_message, session_info +): now = datetime.now() datestr = now.strftime("%d-%b-%Y %H:%M:%S") @@ -238,7 +256,9 @@ def send_slack_pupillometry_update_notification(webhook_url, base_message, sessi m1["type"] = "section" m1_1 = dict() m1_1["type"] = "mrkdwn" - m1_1["text"] = ":white_check_mark: *Pupillometry pipeline update* on " + datestr + "\n\n" + m1_1["text"] = ( + ":white_check_mark: *Pupillometry pipeline update* on " + datestr + "\n\n" + ) m1["text"] = m1_1 # Info # diff --git a/u19_pipeline/utils/subject_metadata.py b/u19_pipeline/utils/subject_metadata.py index 1fc0e6e3..4ff21f1b 100644 --- a/u19_pipeline/utils/subject_metadata.py +++ b/u19_pipeline/utils/subject_metadata.py @@ -3,22 +3,34 @@ from u19_pipeline import lab, subject -def fetch_slack_handles_for_lab_managers_by_subject(subjects: list[str] | str) -> list[str]: +def fetch_slack_handles_for_lab_managers_by_subject( + subjects: list[str] | str, +) -> list[str]: """Returns the Slack handles of lab managers for the labs associated with the given subjects.""" if isinstance(subjects, str): subjects = [subjects] - sql_formatted_names = ", ".join([f"'{name}'" for name in subjects]) if subjects else "''" - expanded_animal = (subject.Subject & f"subject_fullname in ({sql_formatted_names})") * lab.User() * lab.UserLab() + sql_formatted_names = ( + ", ".join([f"'{name}'" for name in subjects]) if subjects else "''" + ) + expanded_animal = ( + (subject.Subject & f"subject_fullname in ({sql_formatted_names})") + * lab.User() + * lab.UserLab() + ) labs = set(expanded_animal.fetch("lab")) sql_formatted_labs = ", ".join([f"'{lab}'" for lab in labs]) if labs else "''" - lab_managers = (lab.LabManager() & f"lab in ({sql_formatted_labs})").proj(user_id="lab_manager") * lab.User() + lab_managers = (lab.LabManager() & f"lab in ({sql_formatted_labs})").proj( + user_id="lab_manager" + ) * lab.User() slack_tags = lab_managers.fetch("slack") return slack_tags -def fetch_slack_handles_for_lab_managers_by_user(user_ids: Iterable[str] | str) -> list[str]: +def fetch_slack_handles_for_lab_managers_by_user( + user_ids: Iterable[str] | str, +) -> list[str]: """Returns the Slack handles of lab managers for the given user IDs.""" if isinstance(user_ids, str): diff --git a/u19_pipeline/utils/tiff_matlab_imaging_utils.py b/u19_pipeline/utils/tiff_matlab_imaging_utils.py index 2a7c71d9..076f85c5 100644 --- a/u19_pipeline/utils/tiff_matlab_imaging_utils.py +++ b/u19_pipeline/utils/tiff_matlab_imaging_utils.py @@ -7,14 +7,11 @@ import tifffile as tiff from sklearn.linear_model import HuberRegressor -xySizeFactor = 1.05 # images are this much larger than nominal size -zFactor = 1.45 # actual displacement in z vs command +xySizeFactor = 1.05 # images are this much larger than nominal size +zFactor = 1.45 # actual displacement in z vs command -def select_files_from_mean_f( - scan_directory, - f_decrease_threshold=15 -): +def select_files_from_mean_f(scan_directory, f_decrease_threshold=15): """ Estimate bleaching by loading only the first frame from each TIFF file. @@ -47,9 +44,7 @@ def select_files_from_mean_f( n_files = len(tif_files) if n_files == 0: - raise FileNotFoundError( - f"No TIFF files found in {scan_directory}" - ) + raise FileNotFoundError(f"No TIFF files found in {scan_directory}") mean_f = np.zeros(n_files) @@ -62,11 +57,9 @@ def select_files_from_mean_f( # --------------------------------------------------------- for i, tif_path in enumerate(tif_files): - print(".", end="", flush=True) with tiff.TiffFile(tif_path) as tif_obj: - n_frames = len(tif_obj.pages) first_frame = tif_obj.pages[0].asarray() @@ -91,7 +84,6 @@ def select_files_from_mean_f( x = frame_id.reshape(-1, 1) if len(mean_f) > 1: - try: # Robust regression equivalent to MATLAB robustfit model = HuberRegressor() @@ -101,16 +93,12 @@ def select_files_from_mean_f( yhat = model.predict(x) except Exception: - # fallback to standard linear fit coeffs = np.polyfit(frame_id, mean_f, 1) yhat = np.polyval(coeffs, frame_id) - threshold_value = ( - yhat[0] - - (f_decrease_threshold / 100.0) * yhat[0] - ) + threshold_value = yhat[0] - (f_decrease_threshold / 100.0) * yhat[0] valid_idx = np.where(yhat > threshold_value)[0] @@ -120,7 +108,6 @@ def select_files_from_mean_f( last_good_file = n_files - 1 else: - last_good_file = 0 # --------------------------------------------------------- @@ -166,7 +153,6 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): # --------------------------------------------------------- with tiff.TiffFile(tif_fn) as tif: - pages = tif.pages header = pages @@ -177,7 +163,7 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): image_description = first_page.description # Some ScanImage versions also use Software tag - software = first_page.tags.get('Software') + software = first_page.tags.get("Software") if software is not None: scope_str = str(software.value) @@ -190,95 +176,66 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): # General image info # ----------------------------------------------------- - parsed_info['Filename'] = str(tif_fn) + parsed_info["Filename"] = str(tif_fn) - parsed_info['Width'] = first_page.imagewidth + parsed_info["Width"] = first_page.imagewidth - parsed_info['Height'] = first_page.imagelength + parsed_info["Height"] = first_page.imagelength # Acquisition time - acq_match = re.search( - r'epoch = \[([0-9].+?)\]', - image_description - ) + acq_match = re.search(r"epoch = \[([0-9].+?)\]", image_description) - parsed_info['AcqTime'] = ( - acq_match.group(1) - if acq_match - else '' - ) + parsed_info["AcqTime"] = acq_match.group(1) if acq_match else "" # Number of depths - depth_match = re.search( - r'SI\.hFastZ\.numFramesPerVolume = (\d+)', - scope_str - ) + depth_match = re.search(r"SI\.hFastZ\.numFramesPerVolume = (\d+)", scope_str) if depth_match: - parsed_info['nDepths'] = int(depth_match.group(1)) + parsed_info["nDepths"] = int(depth_match.group(1)) else: - parsed_info['nDepths'] = 0 + parsed_info["nDepths"] = 0 - parsed_info['Zs'] = -1 + parsed_info["Zs"] = -1 # Frame rate frame_rate_match = re.search( - r'SI\.hRoiManager\.scanVolumeRate = ([0-9]+\.[0-9]+)', - scope_str + r"SI\.hRoiManager\.scanVolumeRate = ([0-9]+\.[0-9]+)", scope_str ) - parsed_info['frameRate'] = ( - float(frame_rate_match.group(1)) - if frame_rate_match - else 0 + parsed_info["frameRate"] = ( + float(frame_rate_match.group(1)) if frame_rate_match else 0 ) # ROI lag lag_match = re.search( - r'SI\.hScan2D\.flytoTimePerScanfield = ([0-9]+\.[0-9]+)', - scope_str + r"SI\.hScan2D\.flytoTimePerScanfield = ([0-9]+\.[0-9]+)", scope_str ) - parsed_info['interROIlag_sec'] = ( - float(lag_match.group(1)) - if lag_match - else 0 - ) + parsed_info["interROIlag_sec"] = float(lag_match.group(1)) if lag_match else 0 # ----------------------------------------------------- # Timing / behavioral sync # ----------------------------------------------------- if not skip_behav_sync: - - parsed_info['Timing'] = { - 'Frame_ts_sec': np.zeros(len(pages)), - 'BehavFrames': [] + parsed_info["Timing"] = { + "Frame_ts_sec": np.zeros(len(pages)), + "BehavFrames": [], } for i, page in enumerate(pages): - desc = page.description # Frame timestamps - ts_match = re.search( - r'frameTimestamps_sec = ([0-9]+\.[0-9]+)', - desc - ) + ts_match = re.search(r"frameTimestamps_sec = ([0-9]+\.[0-9]+)", desc) if ts_match: - parsed_info['Timing']['Frame_ts_sec'][i] = ( - float(ts_match.group(1)) - ) + parsed_info["Timing"]["Frame_ts_sec"][i] = float(ts_match.group(1)) # I2C behavioral sync - i2c_match = re.search( - r'I2CData = ({.+})', - desc - ) + i2c_match = re.search(r"I2CData = ({.+})", desc) if i2c_match: - raw_data = i2c_match.group(1) try: @@ -291,73 +248,49 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): else: behav_data = [] - parsed_info['Timing']['BehavFrames'].append( - behav_data - ) + parsed_info["Timing"]["BehavFrames"].append(behav_data) # ----------------------------------------------------- # Microscope / ScanImage metadata # ----------------------------------------------------- - parsed_info['Scope'] = {} + parsed_info["Scope"] = {} # Resolution scaling - if 'objectiveResolution' in scope_str: - + if "objectiveResolution" in scope_str: try: - res_match = re.search( - r'SI\.objectiveResolution = ([0-9]+\.[0-9]+)', - scope_str + r"SI\.objectiveResolution = ([0-9]+\.[0-9]+)", scope_str ) - resolution_factor = ( - xySizeFactor * float(res_match.group(1)) - ) + resolution_factor = xySizeFactor * float(res_match.group(1)) except Exception: + res_match = re.search(r"SI\.objectiveResolution = (\d+)", scope_str) - res_match = re.search( - r'SI\.objectiveResolution = (\d+)', - scope_str - ) - - resolution_factor = ( - xySizeFactor * float(res_match.group(1)) - ) + resolution_factor = xySizeFactor * float(res_match.group(1)) else: - resolution_factor = 1 # ----------------------------------------------------- # Power # ----------------------------------------------------- - power_match = re.search( - r'SI\.hBeams\.powers = (\d+)', - scope_str - ) + power_match = re.search(r"SI\.hBeams\.powers = (\d+)", scope_str) - parsed_info['Scope']['Power_percent'] = ( - float(power_match.group(1)) - if power_match - else 0 + parsed_info["Scope"]["Power_percent"] = ( + float(power_match.group(1)) if power_match else 0 ) # ----------------------------------------------------- # Channels # ----------------------------------------------------- - channel_match = re.search( - r'SI\.hChannels\.channelSave = (\d+)', - scope_str - ) + channel_match = re.search(r"SI\.hChannels\.channelSave = (\d+)", scope_str) - parsed_info['Scope']['Channels'] = ( - int(channel_match.group(1)) - if channel_match - else 0 + parsed_info["Scope"]["Channels"] = ( + int(channel_match.group(1)) if channel_match else 0 ) # ----------------------------------------------------- @@ -365,40 +298,29 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): # ----------------------------------------------------- cfg_match = re.search( - r'SI\.hConfigurationSaver\.cfgFilename = (.+cfg)', - scope_str + r"SI\.hConfigurationSaver\.cfgFilename = (.+cfg)", scope_str ) - parsed_info['Scope']['cfgFilename'] = ( - cfg_match.group(1).strip() - if cfg_match - else '' + parsed_info["Scope"]["cfgFilename"] = ( + cfg_match.group(1).strip() if cfg_match else "" ) usr_match = re.search( - r'SI\.hConfigurationSaver\.usrFilename = (.+usr)', - scope_str + r"SI\.hConfigurationSaver\.usrFilename = (.+usr)", scope_str ) - parsed_info['Scope']['usrFilename'] = ( - usr_match.group(1).strip() - if usr_match - else '' + parsed_info["Scope"]["usrFilename"] = ( + usr_match.group(1).strip() if usr_match else "" ) # ----------------------------------------------------- # Fast-Z lag # ----------------------------------------------------- - lag_match = re.search( - r'SI\.hFastZ\.actuatorLag = ([0-9eE\.\-]+)', - scope_str - ) + lag_match = re.search(r"SI\.hFastZ\.actuatorLag = ([0-9eE\.\-]+)", scope_str) - parsed_info['Scope']['fastZ_lag'] = ( - float(lag_match.group(1)) - if lag_match - else 0 + parsed_info["Scope"]["fastZ_lag"] = ( + float(lag_match.group(1)) if lag_match else 0 ) # ----------------------------------------------------- @@ -406,14 +328,11 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): # ----------------------------------------------------- flyback_match = re.search( - r'SI\.hFastZ\.flybackTime = ([0-9]+\.[0-9]+)', - scope_str + r"SI\.hFastZ\.flybackTime = ([0-9]+\.[0-9]+)", scope_str ) - parsed_info['Scope']['fastZ_flybackTime'] = ( - float(flyback_match.group(1)) - if flyback_match - else 0 + parsed_info["Scope"]["fastZ_flybackTime"] = ( + float(flyback_match.group(1)) if flyback_match else 0 ) # ----------------------------------------------------- @@ -426,77 +345,64 @@ def extract_float(pattern, default=0): return float(match.group(1)) if match else default - parsed_info['Scope']['linePeriod'] = extract_float( - r'SI\.hRoiManager\.linePeriod = ([0-9.eE\-]+)' + parsed_info["Scope"]["linePeriod"] = extract_float( + r"SI\.hRoiManager\.linePeriod = ([0-9.eE\-]+)" ) - parsed_info['Scope']['scanFramePeriod'] = extract_float( - r'SI\.hRoiManager\.scanFramePeriod = ([0-9.eE\-]+)' + parsed_info["Scope"]["scanFramePeriod"] = extract_float( + r"SI\.hRoiManager\.scanFramePeriod = ([0-9.eE\-]+)" ) - parsed_info['Scope']['scanFrameRate'] = extract_float( - r'SI\.hRoiManager\.scanFrameRate = ([0-9.eE\-]+)' + parsed_info["Scope"]["scanFrameRate"] = extract_float( + r"SI\.hRoiManager\.scanFrameRate = ([0-9.eE\-]+)" ) - parsed_info['Scope']['scanVolumeRate'] = extract_float( - r'SI\.hRoiManager\.scanVolumeRate = ([0-9.eE\-]+)' + parsed_info["Scope"]["scanVolumeRate"] = extract_float( + r"SI\.hRoiManager\.scanVolumeRate = ([0-9.eE\-]+)" ) - parsed_info['Scope']['flybackTimePerFrame'] = extract_float( - r'SI\.hScan2D\.flybackTimePerFrame = ([0-9.eE\-]+)' + parsed_info["Scope"]["flybackTimePerFrame"] = extract_float( + r"SI\.hScan2D\.flybackTimePerFrame = ([0-9.eE\-]+)" ) - parsed_info['Scope']['flytoTimePerScanfield'] = extract_float( - r'SI\.hScan2D\.flytoTimePerScanfield = ([0-9.eE\-]+)' + parsed_info["Scope"]["flytoTimePerScanfield"] = extract_float( + r"SI\.hScan2D\.flytoTimePerScanfield = ([0-9.eE\-]+)" ) # ----------------------------------------------------- # FOV corner points # ----------------------------------------------------- - if 'fovCornerPoints' in scope_str: - + if "fovCornerPoints" in scope_str: fov_match = re.search( - r'SI\.hScan2D\.fovCornerPoints = (\[.+?\])', - scope_str, - re.DOTALL + r"SI\.hScan2D\.fovCornerPoints = (\[.+?\])", scope_str, re.DOTALL ) if fov_match: - try: - - new_fov_match = fov_match.group(1).replace(' ',',') - new_fov_match = new_fov_match.replace(';',',') + new_fov_match = fov_match.group(1).replace(" ", ",") + new_fov_match = new_fov_match.replace(";", ",") fov_points = np.array(ast.literal_eval(new_fov_match)) - fov_points = fov_points.reshape(4,2) + fov_points = fov_points.reshape(4, 2) - parsed_info['Scope']['fovCornerPoints'] = ( + parsed_info["Scope"]["fovCornerPoints"] = ( resolution_factor * fov_points ) except Exception: - - parsed_info['Scope']['fovCornerPoints'] = 0 + parsed_info["Scope"]["fovCornerPoints"] = 0 else: - - parsed_info['Scope']['fovCornerPoints'] = 0 + parsed_info["Scope"]["fovCornerPoints"] = 0 else: - - parsed_info['Scope']['fovCornerPoints'] = 0 + parsed_info["Scope"]["fovCornerPoints"] = 0 return header, parsed_info - - -def parse_tif_header_mesoscope( - tif_fn, - skip_behav_sync=False -): +def parse_tif_header_mesoscope(tif_fn, skip_behav_sync=False): """ Parse ScanImage mesoscope TIFF metadata. @@ -524,7 +430,6 @@ def parse_tif_header_mesoscope( # --------------------------------------------------------- with tiff.TiffFile(tif_fn) as tif: - pages = tif.pages header = pages @@ -533,21 +438,15 @@ def parse_tif_header_mesoscope( image_description = first_page.description - software_tag = first_page.tags.get('Software') + software_tag = first_page.tags.get("Software") - artist_tag = first_page.tags.get('Artist') + artist_tag = first_page.tags.get("Artist") scope_str = ( - str(software_tag.value) - if software_tag is not None - else image_description + str(software_tag.value) if software_tag is not None else image_description ) - roi_info = ( - str(artist_tag.value) - if artist_tag is not None - else "" - ) + roi_info = str(artist_tag.value) if artist_tag is not None else "" parsed_info = {} @@ -555,41 +454,28 @@ def parse_tif_header_mesoscope( # General image info # ----------------------------------------------------- - parsed_info['Filename'] = str(tif_fn) + parsed_info["Filename"] = str(tif_fn) - parsed_info['Width'] = first_page.imagewidth + parsed_info["Width"] = first_page.imagewidth - parsed_info['Height'] = first_page.imagelength + parsed_info["Height"] = first_page.imagelength - acq_match = re.search( - r'(?<=epoch = )\[.+?]', - image_description - ) + acq_match = re.search(r"(?<=epoch = )\[.+?]", image_description) - parsed_info['AcqTime'] = ( - acq_match.group(0) - if acq_match - else "" - ) + parsed_info["AcqTime"] = acq_match.group(0) if acq_match else "" def extract_float(pattern, text, default=0): match = re.search(pattern, text) - return ( - float(match.group(0)) - if match - else default - ) + return float(match.group(0)) if match else default - parsed_info['frameRate'] = extract_float( - r'(?<=SI\.hRoiManager\.scanVolumeRate = )\d+\.\d+', - scope_str + parsed_info["frameRate"] = extract_float( + r"(?<=SI\.hRoiManager\.scanVolumeRate = )\d+\.\d+", scope_str ) - parsed_info['interROIlag_sec'] = extract_float( - r'(?<=SI\.hScan2D\.flytoTimePerScanfield = )\d+\.\d+', - scope_str + parsed_info["interROIlag_sec"] = extract_float( + r"(?<=SI\.hScan2D\.flytoTimePerScanfield = )\d+\.\d+", scope_str ) # ----------------------------------------------------- @@ -597,33 +483,22 @@ def extract_float(pattern, text, default=0): # ----------------------------------------------------- if not skip_behav_sync: - - parsed_info['Timing'] = { - 'Frame_ts_sec': np.zeros(len(pages)), - 'BehavFrames': [] + parsed_info["Timing"] = { + "Frame_ts_sec": np.zeros(len(pages)), + "BehavFrames": [], } for i, page in enumerate(pages): - desc = page.description - ts_match = re.search( - r'(?<=frameTimestamps_sec = )\d+\.\d+', - desc - ) + ts_match = re.search(r"(?<=frameTimestamps_sec = )\d+\.\d+", desc) if ts_match: - parsed_info['Timing']['Frame_ts_sec'][i] = ( - float(ts_match.group(0)) - ) + parsed_info["Timing"]["Frame_ts_sec"][i] = float(ts_match.group(0)) - i2c_match = re.search( - r'(?<=I2CData = ){.+}', - desc - ) + i2c_match = re.search(r"(?<=I2CData = ){.+}", desc) if i2c_match: - raw_data = i2c_match.group(0) try: @@ -633,56 +508,39 @@ def extract_float(pattern, text, default=0): behav_data = np.nan else: - behav_data = [] - parsed_info['Timing']['BehavFrames'].append( - behav_data - ) + parsed_info["Timing"]["BehavFrames"].append(behav_data) # ----------------------------------------------------- # ROI parsing # ----------------------------------------------------- roi_marks = [ - m.start() - for m in re.finditer( - r'"scanimage\.mroi\.Roi"', - roi_info - ) + m.start() for m in re.finditer(r'"scanimage\.mroi\.Roi"', roi_info) ] - parsed_info['nROIs'] = len(roi_marks) + parsed_info["nROIs"] = len(roi_marks) - parsed_info['ROI'] = [] + parsed_info["ROI"] = [] # Resolution scaling resolution_match = re.search( - r'(?<=SI\.objectiveResolution = )\d+\.\d+', - scope_str + r"(?<=SI\.objectiveResolution = )\d+\.\d+", scope_str ) if resolution_match: - - resolution_factor = ( - xySizeFactor * float(resolution_match.group(0)) - ) + resolution_factor = xySizeFactor * float(resolution_match.group(0)) else: - resolution_factor = 1 for i_roi in range(len(roi_marks)): - if i_roi != len(roi_marks) - 1: - - this_roi = roi_info[ - roi_marks[i_roi]:roi_marks[i_roi + 1] - ] + this_roi = roi_info[roi_marks[i_roi] : roi_marks[i_roi + 1]] else: - - this_roi = roi_info[roi_marks[i_roi]:] + this_roi = roi_info[roi_marks[i_roi] :] roi_dict = {} @@ -690,31 +548,17 @@ def extract_float(pattern, text, default=0): # ROI name # ------------------------------------------------- - name_match = re.search( - r'(?<="name": ")\w+\d*', - this_roi - ) + name_match = re.search(r'(?<="name": ")\w+\d*', this_roi) - roi_dict['name'] = ( - name_match.group(0) - if name_match - else '' - ) + roi_dict["name"] = name_match.group(0) if name_match else "" # ------------------------------------------------- # Z position # ------------------------------------------------- - z_match = re.search( - r'(?<="zs": )(|-)\d+', - this_roi - ) + z_match = re.search(r'(?<="zs": )(|-)\d+', this_roi) - roi_dict['Zs'] = ( - float(z_match.group(0)) - if z_match - else 0 - ) + roi_dict["Zs"] = float(z_match.group(0)) if z_match else 0 # ------------------------------------------------- # Helper for vector parsing @@ -728,9 +572,7 @@ def parse_array(pattern, text): return np.array([]) try: - return np.array( - ast.literal_eval(match.group(0)) - ) + return np.array(ast.literal_eval(match.group(0))) except Exception: return np.array([]) @@ -739,348 +581,217 @@ def parse_array(pattern, text): # ROI geometry # ------------------------------------------------- - roi_dict['centerXY'] = ( - resolution_factor - * parse_array( - r'(?<="centerXY": )\[.+?]', - this_roi - ) + roi_dict["centerXY"] = resolution_factor * parse_array( + r'(?<="centerXY": )\[.+?]', this_roi ) - roi_dict['sizeXY'] = ( - resolution_factor - * parse_array( - r'(?<="sizeXY": )\[.+?]', - this_roi - ) + roi_dict["sizeXY"] = resolution_factor * parse_array( + r'(?<="sizeXY": )\[.+?]', this_roi ) rotation_match = re.search( - r'(?<="rotationDegrees":) (\d+|\d+\.\d+)', - this_roi + r'(?<="rotationDegrees":) (\d+|\d+\.\d+)', this_roi ) - roi_dict['rotationDegrees'] = ( - float(rotation_match.group(0)) - if rotation_match - else 0 + roi_dict["rotationDegrees"] = ( + float(rotation_match.group(0)) if rotation_match else 0 ) - roi_dict['pixelResolutionXY'] = parse_array( - r'(?<="pixelResolutionXY": )\[.+?]', - this_roi + roi_dict["pixelResolutionXY"] = parse_array( + r'(?<="pixelResolutionXY": )\[.+?]', this_roi ) - discrete_match = re.search( - r'(?<="discretePlaneMode":) \d', - this_roi - ) + discrete_match = re.search(r'(?<="discretePlaneMode":) \d', this_roi) - roi_dict['discretePlaneMode'] = bool( - int(discrete_match.group(0)) - ) if discrete_match else False - - power_match = re.search( - r'(?<="powers":) \d+', - this_roi + roi_dict["discretePlaneMode"] = ( + bool(int(discrete_match.group(0))) if discrete_match else False ) - if power_match: + power_match = re.search(r'(?<="powers":) \d+', this_roi) - roi_dict['Power_percent'] = float( - power_match.group(0) - ) + if power_match: + roi_dict["Power_percent"] = float(power_match.group(0)) - parsed_info['ROI'].append(roi_dict) + parsed_info["ROI"].append(roi_dict) # ----------------------------------------------------- # Scope metadata # ----------------------------------------------------- - parsed_info['Scope'] = {} + parsed_info["Scope"] = {} - if not re.search( - r'(?<="powers":) \d+', - roi_info - ): + if not re.search(r'(?<="powers":) \d+', roi_info): + power_match = re.search(r"(?<=SI\.hBeams\.powers = )\d+", scope_str) - power_match = re.search( - r'(?<=SI\.hBeams\.powers = )\d+', - scope_str - ) - - parsed_info['Scope']['Power_percent'] = ( - float(power_match.group(0)) - if power_match - else 0 + parsed_info["Scope"]["Power_percent"] = ( + float(power_match.group(0)) if power_match else 0 ) else: - - parsed_info['Scope']['Power_percent'] = ( - 'discrete powers per ROI' - ) + parsed_info["Scope"]["Power_percent"] = "discrete powers per ROI" # Channels - channel_match = re.search( - r'(?<=SI\.hChannels\.channelSave = )\d+', - scope_str - ) + channel_match = re.search(r"(?<=SI\.hChannels\.channelSave = )\d+", scope_str) - parsed_info['Scope']['Channels'] = ( - int(channel_match.group(0)) - if channel_match - else 0 + parsed_info["Scope"]["Channels"] = ( + int(channel_match.group(0)) if channel_match else 0 ) # Config filenames cfg_match = re.search( - r"(?<=SI\.hConfigurationSaver\.cfgFilename = ').+cfg", - scope_str + r"(?<=SI\.hConfigurationSaver\.cfgFilename = ').+cfg", scope_str ) - parsed_info['Scope']['cfgFilename'] = ( - cfg_match.group(0) - if cfg_match - else '' - ) + parsed_info["Scope"]["cfgFilename"] = cfg_match.group(0) if cfg_match else "" usr_match = re.search( - r"(?<=SI\.hConfigurationSaver\.usrFilename = ').+usr", - scope_str + r"(?<=SI\.hConfigurationSaver\.usrFilename = ').+usr", scope_str ) - parsed_info['Scope']['usrFilename'] = ( - usr_match.group(0) - if usr_match - else '' - ) + parsed_info["Scope"]["usrFilename"] = usr_match.group(0) if usr_match else "" # ----------------------------------------------------- # Timing metadata # ----------------------------------------------------- timing_fields = { - 'fastZ_lag': - r'(?<=SI\.hFastZ\.actuatorLag = )\d+\.\d+', - - 'fastZ_flybackTime': - r'(?<=SI\.hFastZ\.flybackTime = )\d+\.\d+', - - 'linePeriod': - r'(?<=SI\.hRoiManager\.linePeriod = )\d+\.\d+e-[0-9]+', - - 'scanFramePeriod': - r'(?<=SI\.hRoiManager\.scanFramePeriod = )\d+\.\d+', - - 'scanFrameRate': - r'(?<=SI\.hRoiManager\.scanFrameRate = )\d+\.\d+', - - 'scanVolumeRate': - r'(?<=SI\.hRoiManager\.scanVolumeRate = )\d+\.\d+', - - 'flybackTimePerFrame': - r'(?<=SI\.hScan2D\.flybackTimePerFrame = )\d+\.\d+', - - 'flytoTimePerScanfield': - r'(?<=SI\.hScan2D\.flytoTimePerScanfield = )\d+\.\d+' + "fastZ_lag": r"(?<=SI\.hFastZ\.actuatorLag = )\d+\.\d+", + "fastZ_flybackTime": r"(?<=SI\.hFastZ\.flybackTime = )\d+\.\d+", + "linePeriod": r"(?<=SI\.hRoiManager\.linePeriod = )\d+\.\d+e-[0-9]+", + "scanFramePeriod": r"(?<=SI\.hRoiManager\.scanFramePeriod = )\d+\.\d+", + "scanFrameRate": r"(?<=SI\.hRoiManager\.scanFrameRate = )\d+\.\d+", + "scanVolumeRate": r"(?<=SI\.hRoiManager\.scanVolumeRate = )\d+\.\d+", + "flybackTimePerFrame": r"(?<=SI\.hScan2D\.flybackTimePerFrame = )\d+\.\d+", + "flytoTimePerScanfield": r"(?<=SI\.hScan2D\.flytoTimePerScanfield = )\d+\.\d+", } for key, pattern in timing_fields.items(): - - parsed_info['Scope'][key] = extract_float( - pattern, - scope_str - ) + parsed_info["Scope"][key] = extract_float(pattern, scope_str) # ----------------------------------------------------- # FOV corner points # ----------------------------------------------------- - fov_match = re.search( - r'(?<=SI\.hScan2D\.fovCornerPoints = )\[.+?]', - scope_str - ) + fov_match = re.search(r"(?<=SI\.hScan2D\.fovCornerPoints = )\[.+?]", scope_str) if fov_match: - try: - - new_fov_match = fov_match.group(0).replace(' ',',') - new_fov_match = new_fov_match.replace(';',',') + new_fov_match = fov_match.group(0).replace(" ", ",") + new_fov_match = new_fov_match.replace(";", ",") fov_points = np.array(ast.literal_eval(new_fov_match)) - fov_points = fov_points.reshape(4,2) + fov_points = fov_points.reshape(4, 2) - parsed_info['Scope']['fovCornerPoints'] = ( - resolution_factor * fov_points - ) + parsed_info["Scope"]["fovCornerPoints"] = resolution_factor * fov_points except Exception: - - parsed_info['Scope']['fovCornerPoints'] = 0 + parsed_info["Scope"]["fovCornerPoints"] = 0 else: - - parsed_info['Scope']['fovCornerPoints'] = 0 + parsed_info["Scope"]["fovCornerPoints"] = 0 # ----------------------------------------------------- # Stack metadata # ----------------------------------------------------- - stack_match = re.search( - r'(?<=SI\.hStackManager\.enable = )\w+', - scope_str - ) + stack_match = re.search(r"(?<=SI\.hStackManager\.enable = )\w+", scope_str) - stacks_enabled = ( - stack_match.group(0) - if stack_match - else 'false' - ) + stacks_enabled = stack_match.group(0) if stack_match else "false" - parsed_info['Scope']['stacks_enabled'] = ( - 1 if stacks_enabled == 'true' else 0 - ) - - if stacks_enabled == 'true': + parsed_info["Scope"]["stacks_enabled"] = 1 if stacks_enabled == "true" else 0 + if stacks_enabled == "true": actuator_match = re.search( - r"(?<=SI\.hStackManager\.stackActuator = ')\w+", - scope_str + r"(?<=SI\.hStackManager\.stackActuator = ')\w+", scope_str ) definition_match = re.search( - r"(?<=SI\.hStackManager\.stackDefinition = ')\w+", - scope_str + r"(?<=SI\.hStackManager\.stackDefinition = ')\w+", scope_str ) - parsed_info['Scope']['stackActuator'] = ( - actuator_match.group(0) - if actuator_match - else '' + parsed_info["Scope"]["stackActuator"] = ( + actuator_match.group(0) if actuator_match else "" ) - parsed_info['Scope']['stackDefinition'] = ( - definition_match.group(0) - if definition_match - else '' + parsed_info["Scope"]["stackDefinition"] = ( + definition_match.group(0) if definition_match else "" ) # ----------------------------------------------------- # Motion correction # ----------------------------------------------------- - motion_match = re.search( - r'(?<=SI\.hMotionManager\.enable = )\w+', - scope_str - ) + motion_match = re.search(r"(?<=SI\.hMotionManager\.enable = )\w+", scope_str) - motion_enabled = ( - motion_match.group(0) - if motion_match - else 'false' - ) + motion_enabled = motion_match.group(0) if motion_match else "false" - parsed_info['Scope']['motionCorrection_enabled'] = ( - 1 if motion_enabled == 'true' else 0 + parsed_info["Scope"]["motionCorrection_enabled"] = ( + 1 if motion_enabled == "true" else 0 ) - if motion_enabled == 'true': - + if motion_enabled == "true": correction_z_match = re.search( - r'(?<=SI\.hMotionManager\.correctionEnableZ = )\w+', - scope_str + r"(?<=SI\.hMotionManager\.correctionEnableZ = )\w+", scope_str ) - if ( - correction_z_match - and correction_z_match.group(0) == 'true' - ): - - parsed_info['Scope']['motionCorMode'] = ( - 'automated' - ) + if correction_z_match and correction_z_match.group(0) == "true": + parsed_info["Scope"]["motionCorMode"] = "automated" else: - - parsed_info['Scope']['motionCorMode'] = ( - 'manual' - ) + parsed_info["Scope"]["motionCorMode"] = "manual" # ----------------------------------------------------- # Depths / Z planes # ----------------------------------------------------- - depth_match = re.search( - r'SI\.hFastZ\.numFramesPerVolume = \d+', - scope_str - ) + depth_match = re.search(r"SI\.hFastZ\.numFramesPerVolume = \d+", scope_str) if depth_match: - - parsed_info['nDepths'] = int( - re.search(r'\d+', depth_match.group(0)).group(0) + parsed_info["nDepths"] = int( + re.search(r"\d+", depth_match.group(0)).group(0) ) else: - - if stacks_enabled == 'true': - + if stacks_enabled == "true": slices_match = re.search( - r'(?<=SI\.hStackManager\.actualNumSlices = )\d+', - scope_str + r"(?<=SI\.hStackManager\.actualNumSlices = )\d+", scope_str ) - parsed_info['nDepths'] = ( - int(slices_match.group(0)) - if slices_match - else 1 + parsed_info["nDepths"] = ( + int(slices_match.group(0)) if slices_match else 1 ) else: - - parsed_info['nDepths'] = 1 + parsed_info["nDepths"] = 1 # ----------------------------------------------------- # Z positions # ----------------------------------------------------- - parsed_info['Zs'] = None + parsed_info["Zs"] = None z_patterns = [ - r'(?<=SI\.hFastZ\.userZs = )\[.+?]', - r'(?<=SI\.hStackManager\.zs = )\[.+?]', - r'(?<=SI\.hFastZ\.position = )(|-)\d+' + r"(?<=SI\.hFastZ\.userZs = )\[.+?]", + r"(?<=SI\.hStackManager\.zs = )\[.+?]", + r"(?<=SI\.hFastZ\.position = )(|-)\d+", ] for pattern in z_patterns: - z_match = re.search(pattern, scope_str) if z_match: - try: + z_val = ast.literal_eval(z_match.group(0)) - z_val = ast.literal_eval( - z_match.group(0) - ) - - parsed_info['Zs'] = ( - zFactor * np.array(z_val) - ) + parsed_info["Zs"] = zFactor * np.array(z_val) except Exception: - try: - - parsed_info['Zs'] = float( - z_match.group(0) - ) + parsed_info["Zs"] = float(z_match.group(0)) except Exception: pass break - return header, parsed_info \ No newline at end of file + return header, parsed_info diff --git a/u19_pipeline/utils/tiff_utils.py b/u19_pipeline/utils/tiff_utils.py index 764f9648..9e8947e8 100644 --- a/u19_pipeline/utils/tiff_utils.py +++ b/u19_pipeline/utils/tiff_utils.py @@ -11,11 +11,11 @@ import u19_pipeline.utils.tiff_matlab_imaging_utils as tmiu -tif_number_fmt = r'_[0-9]{5}\.tif' -tif_gz_number_fmt = r'_[0-9]{5}\.tif\.gz' +tif_number_fmt = r"_[0-9]{5}\.tif" +tif_gz_number_fmt = r"_[0-9]{5}\.tif\.gz" -patt_acq_number = r'_[0-9]{5}_' -patt_file_number = r'_[0-9]{5}\.' +patt_acq_number = r"_[0-9]{5}_" +patt_file_number = r"_[0-9]{5}\." def check_tif_files(tif_dir): @@ -24,38 +24,32 @@ def check_tif_files(tif_dir): is_compressed = False - fl = sorted(tif_dir.glob('*.tif')) + fl = sorted(tif_dir.glob("*.tif")) if not fl: - - gz_files = sorted(tif_dir.glob('*.tif.gz')) + gz_files = sorted(tif_dir.glob("*.tif.gz")) if gz_files: - is_compressed = True for gz_file in gz_files: - with gzip.open(gz_file, 'rb') as f_in: - with open(gz_file.with_suffix(''), 'wb') as f_out: + with gzip.open(gz_file, "rb") as f_in: + with open(gz_file.with_suffix(""), "wb") as f_out: shutil.copyfileobj(f_in, f_out) - fl = sorted(tif_dir.glob('*.tif')) + fl = sorted(tif_dir.glob("*.tif")) else: - raise FileNotFoundError( - "No tif or tif.gz files found" - ) + raise FileNotFoundError("No tif or tif.gz files found") fl = [f.name for f in fl] match = re.search(tif_number_fmt, fl[0]) if match is None: - raise ValueError( - f'Invalid tif naming format: {fl[0]}' - ) + raise ValueError(f"Invalid tif naming format: {fl[0]}") - basename = fl[0][:match.start()] + basename = fl[0][: match.start()] return fl, basename, is_compressed @@ -63,8 +57,7 @@ def check_tif_files(tif_dir): def get_parsed_info_2photon(fl): with ProcessPoolExecutor(max_workers=16) as exe: - - results = list(exe.map(tmiu.parse_tif_header_2photon,fl)) + results = list(exe.map(tmiu.parse_tif_header_2photon, fl)) imheader = [r[0] for r in results] parsed_info = [r[1] for r in results] @@ -75,8 +68,7 @@ def get_parsed_info_2photon(fl): def get_parsed_info_mesoscope(fl): with ProcessPoolExecutor(max_workers=16) as exe: - - results = list(exe.map(tmiu.parse_tif_header_mesoscope,fl)) + results = list(exe.map(tmiu.parse_tif_header_mesoscope, fl)) imheader = [r[0] for r in results] parsed_info = [r[1] for r in results] @@ -89,39 +81,36 @@ def get_recording_info(fl, imheader, parsed_info): frames_per_file = np.zeros(len(fl), dtype=int) for i, file in enumerate(fl): - if i == 0: rec_info = parsed_info[i] - rec_info['Timing']['BehavFrames'] = np.array(rec_info['Timing']['BehavFrames'], dtype=object) + rec_info["Timing"]["BehavFrames"] = np.array( + rec_info["Timing"]["BehavFrames"], dtype=object + ) else: - - if parsed_info[i]['Timing']['Frame_ts_sec'][0] == 0: - - parsed_info[i]['Timing']['Frame_ts_sec'] += ( - rec_info['Timing']['Frame_ts_sec'][-1] - + 1 / rec_info['frameRate'] + if parsed_info[i]["Timing"]["Frame_ts_sec"][0] == 0: + parsed_info[i]["Timing"]["Frame_ts_sec"] += ( + rec_info["Timing"]["Frame_ts_sec"][-1] + 1 / rec_info["frameRate"] ) - rec_info['Timing']['Frame_ts_sec'] = np.concatenate([ - rec_info['Timing']['Frame_ts_sec'], - parsed_info[i]['Timing']['Frame_ts_sec'] - ]) + rec_info["Timing"]["Frame_ts_sec"] = np.concatenate( + [ + rec_info["Timing"]["Frame_ts_sec"], + parsed_info[i]["Timing"]["Frame_ts_sec"], + ] + ) - aux = np.array(parsed_info[i]['Timing']['BehavFrames'], dtype=object) + aux = np.array(parsed_info[i]["Timing"]["BehavFrames"], dtype=object) aux = np.squeeze(aux) if aux.size > 0: - rec_info['Timing']['BehavFrames'] = np.concatenate([ - rec_info['Timing']['BehavFrames'], - aux - ]) + rec_info["Timing"]["BehavFrames"] = np.concatenate( + [rec_info["Timing"]["BehavFrames"], aux] + ) frames_per_file[i] = len(imheader[i]) - rec_info['nFrames'] = len( - rec_info['Timing']['Frame_ts_sec'] - ) + rec_info["nFrames"] = len(rec_info["Timing"]["Frame_ts_sec"]) return rec_info, frames_per_file @@ -129,103 +118,96 @@ def get_recording_info(fl, imheader, parsed_info): def get_nfovs(rec_info, is_mesoscope): if is_mesoscope: - return len(rec_info['ROI']) + return len(rec_info["ROI"]) return 1 + def check_acqtime(acq_time, scan_directory): try: - dt = datetime.strptime(acq_time,'%Y %m %d %H %M %S.%f' - ) + dt = datetime.strptime(acq_time, "%Y %m %d %H %M %S.%f") - return dt.strftime('%Y-%m-%d %H:%M:%S') + return dt.strftime("%Y-%m-%d %H:%M:%S") except Exception: - dirname = Path(scan_directory).name - date_match = re.search(r'(\d{8})', dirname) + date_match = re.search(r"(\d{8})", dirname) if date_match: - d = date_match.group(1) - return ( - f'{d[:4]}-{d[4:6]}-{d[6:8]} 00:00:00' - ) + return f"{d[:4]}-{d[4:6]}-{d[6:8]} 00:00:00" + + return "1000-01-01 00:00:00" - return '1000-01-01 00:00:00' - def get_last_good_frame(frames_per_file, scan_directory): last_good_file = tmiu.select_files_from_mean_f(scan_directory) cumulative_frames = np.cumsum(frames_per_file) - return last_good_file, cumulative_frames def create_scan_info_key(key, rec_info, bucket_dir): - filename = Path(rec_info['Filename']).name + filename = Path(rec_info["Filename"]).name scan_info_key = key.copy() - scan_info_key['file_name_base'] = str( - Path('/') / bucket_dir / filename - ) + scan_info_key["file_name_base"] = str(Path("/") / bucket_dir / filename) - scan_info_key['scan_width'] = rec_info['Width'] - scan_info_key['scan_height'] = rec_info['Height'] + scan_info_key["scan_width"] = rec_info["Width"] + scan_info_key["scan_height"] = rec_info["Height"] try: - datetime.strptime(rec_info['AcqTime'],'%Y-%m-%d %H:%M:%S') - scan_info_key['acq_time'] = rec_info['AcqTime'] + datetime.strptime(rec_info["AcqTime"], "%Y-%m-%d %H:%M:%S") + scan_info_key["acq_time"] = rec_info["AcqTime"] except Exception: - scan_info_key['acq_time'] = '1000-01-01 00:00:00' - - scan_info_key['n_depths'] = rec_info['nDepths'] - scan_info_key['scan_depths'] = rec_info['Zs'] - scan_info_key['frame_rate'] = rec_info['frameRate'] - scan_info_key['inter_fov_lag_sec'] = rec_info['interROIlag_sec'] - scan_info_key['frame_ts_sec'] = rec_info['Timing']['Frame_ts_sec'] - - scope = rec_info['Scope'] - scan_info_key['power_percent'] = scope.get('Power_percent', 0) - scan_info_key['channels'] = scope['Channels'] - scan_info_key['cfg_filename'] = scope['cfgFilename'] - scan_info_key['usr_filename'] = scope['usrFilename'] - scan_info_key['fast_z_lag'] = scope['fastZ_lag'] - scan_info_key['fast_z_flyback_time'] = scope['fastZ_flybackTime'] - scan_info_key['line_period'] = scope['linePeriod'] - scan_info_key['scan_frame_period'] = scope['scanFramePeriod'] - scan_info_key['scan_volume_rate'] = scope['scanVolumeRate'] - scan_info_key['flyback_time_per_frame'] = scope['flybackTimePerFrame'] - scan_info_key['flyto_time_per_scan_field'] = scope['flytoTimePerScanfield'] - scan_info_key['fov_corner_points'] = scope['fovCornerPoints'] - - scan_info_key['nfovs'] = rec_info['nfovs'] - scan_info_key['nframes'] = rec_info['nFrames'] - scan_info_key['nframes_good'] = rec_info['nframes_good'] - scan_info_key['last_good_file'] = rec_info['last_good_file'] - - if 'stacks_enabled' in scope: - scan_info_key['stacks_enabled'] = scope['stacks_enabled'] - - if 'stackActuator' in scope: - scan_info_key['stack_actuator'] = scope['stackActuator'] - - if 'stackDefinition' in scope: - scan_info_key['stack_definition'] = scope['stackDefinition'] - - if 'motionCorrection_enabled' in scope: - scan_info_key['motion_correction_enabled'] = scope['motionCorrection_enabled'] - - if 'motionCorMode' in scope: - scan_info_key['motion_correction_mode'] = scope['motionCorMode'] - + scan_info_key["acq_time"] = "1000-01-01 00:00:00" + + scan_info_key["n_depths"] = rec_info["nDepths"] + scan_info_key["scan_depths"] = rec_info["Zs"] + scan_info_key["frame_rate"] = rec_info["frameRate"] + scan_info_key["inter_fov_lag_sec"] = rec_info["interROIlag_sec"] + scan_info_key["frame_ts_sec"] = rec_info["Timing"]["Frame_ts_sec"] + + scope = rec_info["Scope"] + scan_info_key["power_percent"] = scope.get("Power_percent", 0) + scan_info_key["channels"] = scope["Channels"] + scan_info_key["cfg_filename"] = scope["cfgFilename"] + scan_info_key["usr_filename"] = scope["usrFilename"] + scan_info_key["fast_z_lag"] = scope["fastZ_lag"] + scan_info_key["fast_z_flyback_time"] = scope["fastZ_flybackTime"] + scan_info_key["line_period"] = scope["linePeriod"] + scan_info_key["scan_frame_period"] = scope["scanFramePeriod"] + scan_info_key["scan_volume_rate"] = scope["scanVolumeRate"] + scan_info_key["flyback_time_per_frame"] = scope["flybackTimePerFrame"] + scan_info_key["flyto_time_per_scan_field"] = scope["flytoTimePerScanfield"] + scan_info_key["fov_corner_points"] = scope["fovCornerPoints"] + + scan_info_key["nfovs"] = rec_info["nfovs"] + scan_info_key["nframes"] = rec_info["nFrames"] + scan_info_key["nframes_good"] = rec_info["nframes_good"] + scan_info_key["last_good_file"] = rec_info["last_good_file"] + + if "stacks_enabled" in scope: + scan_info_key["stacks_enabled"] = scope["stacks_enabled"] + + if "stackActuator" in scope: + scan_info_key["stack_actuator"] = scope["stackActuator"] + + if "stackDefinition" in scope: + scan_info_key["stack_definition"] = scope["stackDefinition"] + + if "motionCorrection_enabled" in scope: + scan_info_key["motion_correction_enabled"] = scope["motionCorrection_enabled"] + + if "motionCorMode" in scope: + scan_info_key["motion_correction_mode"] = scope["motionCorMode"] + return scan_info_key @@ -246,34 +228,29 @@ def remove_compressed_videos(fl, directory): directory = Path(directory) for tif_file in fl: - file_base = directory / tif_file - gz_file = Path(str(file_base) + '.gz') + gz_file = Path(str(file_base) + ".gz") if gz_file.exists() and file_base.exists(): - print(f"Removing {gz_file}") gz_file.unlink() else: - - print( - f"Could not find compressed pair {file_base}" - ) - + print(f"Could not find compressed pair {file_base}") # ----------------------------------------------------------------------------- # Helper functions # ----------------------------------------------------------------------------- + def replace_frame_timestamp(imdescription, lag): """ Replace frameTimestamps_sec value with lag-adjusted value. """ - pattern = r'(?<=frameTimestamps_sec = )([0-9]+\.[0-9]+)' + pattern = r"(?<=frameTimestamps_sec = )([0-9]+\.[0-9]+)" match = re.search(pattern, imdescription) if match: @@ -292,15 +269,14 @@ def clean_software_string(software): return "" software = software.replace( - "hRoiManager.mroiEnable = 1", - "hRoiManager.mroiEnable = 0" + "hRoiManager.mroiEnable = 1", "hRoiManager.mroiEnable = 0" ) idx = software.find("SI.hRoiManager.imagingFovUm") if idx != -1: newline_idx = software.find("\n", idx) if newline_idx != -1: - software = software[:idx] + software[newline_idx + 1:] + software = software[:idx] + software[newline_idx + 1 :] return software @@ -309,6 +285,7 @@ def clean_software_string(software): # Mesoscope ROI splitting # ----------------------------------------------------------------------------- + def get_fov_mesoscope( fl, key_data, @@ -318,7 +295,7 @@ def get_fov_mesoscope( basename, cumulative_frames, scan_dirs_db, - imaging_root + imaging_root, ): """ Python translation of MATLAB insert_fov_mesoscope. @@ -329,7 +306,6 @@ def get_fov_mesoscope( n_roi = rec_info["nROIs"] if not skip_parsing: - print("\tparsing ROIs...") roi_nr = [roi["pixelResolutionXY"][1] for roi in rec_info["ROI"]] @@ -338,23 +314,21 @@ def get_fov_mesoscope( inter_roi_lag = rec_info["interROIlag_sec"] depths = rec_info["nDepths"] - which_depths = sorted( - list(set([roi["Zs"] for roi in rec_info["ROI"]])) - ) + which_depths = sorted(list(set([roi["Zs"] for roi in rec_info["ROI"]]))) # --------------------------------------------------------------------- # Create directories # --------------------------------------------------------------------- for idepth in range(depths): - which_roi = [ - i for i, roi in enumerate(rec_info["ROI"]) + i + for i, roi in enumerate(rec_info["ROI"]) if roi["Zs"] == which_depths[idepth] ] for iroi in which_roi: - dirname = f"ROI{iroi+1:02d}_z{idepth+1}" + dirname = f"ROI{iroi + 1:02d}_z{idepth + 1}" os.makedirs(dirname, exist_ok=True) # --------------------------------------------------------------------- @@ -362,9 +336,7 @@ def get_fov_mesoscope( # --------------------------------------------------------------------- for iF, tif_file in enumerate(fl): - with tifffile.TiffFile(tif_file) as tif: - pages = tif.pages current_header = pages[0].tags @@ -376,28 +348,19 @@ def get_fov_mesoscope( first_page = pages[0].asarray() thisstack = np.zeros( - ( - imheader[iF][0]["Height"], - 512, - len(imheader[iF]) - ), - dtype=np.uint16 + (imheader[iF][0]["Height"], 512, len(imheader[iF])), dtype=np.uint16 ) pixel2sum = 1 for iframe, page in enumerate(pages): - temp_stack = page.asarray() if temp_stack.shape[1] != thisstack.shape[1]: - pixel2sum = imheader[iF][0]["Width"] // 512 reshaped = temp_stack.reshape( - temp_stack.shape[0], - pixel2sum, - 512 + temp_stack.shape[0], pixel2sum, 512 ) temp_stack = reshaped.sum(axis=1) @@ -414,36 +377,33 @@ def get_fov_mesoscope( # ------------------------------------------------------------- for idepth in range(depths): - ilag = 0 rowct = 0 which_roi = [ - i for i, roi in enumerate(rec_info["ROI"]) + i + for i, roi in enumerate(rec_info["ROI"]) if roi["Zs"] == which_depths[idepth] ] for iroi in which_roi: - zidx = list(range(idepth, thisstack.shape[2], depths)) - substack = thisstack[ - rowct:rowct + roi_nr[iroi], - :nc, - : - ][:, :, zidx] + substack = thisstack[rowct : rowct + roi_nr[iroi], :nc, :][ + :, :, zidx + ] # ----------------------------------------------------- # Output filename # ----------------------------------------------------- - match = re.search(r'_[0-9]{5}\.tif', tif_file) + match = re.search(r"_[0-9]{5}\.tif", tif_file) thisfn = ( - f"./ROI{iroi+1:02d}_z{idepth+1}/" - f"{tif_file[:match.start()]}" - f"ROI{iroi+1:02d}_z{idepth+1}_" - f"{tif_file[match.start()+1:]}" + f"./ROI{iroi + 1:02d}_z{idepth + 1}/" + f"{tif_file[: match.start()]}" + f"ROI{iroi + 1:02d}_z{idepth + 1}_" + f"{tif_file[match.start() + 1 :]}" ) # ----------------------------------------------------- @@ -453,24 +413,25 @@ def get_fov_mesoscope( first_desc = imheader[iF][zidx[0]]["ImageDescription"] first_desc = replace_frame_timestamp( - first_desc, - inter_roi_lag * ilag + first_desc, inter_roi_lag * ilag ) software = clean_software_string( current_header.get("Software", None).value - if "Software" in current_header else "" + if "Software" in current_header + else "" ) artist = ( current_header.get("Artist", None).value - if "Artist" in current_header else "" + if "Artist" in current_header + else "" ) metadata = { "ImageDescription": first_desc, "Software": software, - "Artist": artist + "Artist": artist, } # ----------------------------------------------------- @@ -478,23 +439,18 @@ def get_fov_mesoscope( # ----------------------------------------------------- with tifffile.TiffWriter(thisfn, bigtiff=True) as writer: - for iz in range(substack.shape[2]): - - desc = imheader[iF][zidx[iz]][ - "ImageDescription" - ] + desc = imheader[iF][zidx[iz]]["ImageDescription"] desc = replace_frame_timestamp( - desc, - inter_roi_lag * ilag + desc, inter_roi_lag * ilag ) writer.write( substack[:, :, iz], description=desc, metadata=None, - extratags=[] + extratags=[], ) ilag += 1 @@ -504,10 +460,9 @@ def get_fov_mesoscope( # ----------------------------------------------------- if len(which_roi) > 1: - - padsize = ( - nr - sum([roi_nr[r] for r in which_roi]) - ) / (len(which_roi) - 1) + padsize = (nr - sum([roi_nr[r] for r in which_roi])) / ( + len(which_roi) - 1 + ) rowct += int(padsize + roi_nr[iroi]) @@ -517,40 +472,31 @@ def get_fov_mesoscope( os.makedirs("originalStacks", exist_ok=True) - shutil.move( - tif_file, - os.path.join("originalStacks", tif_file) - ) - + shutil.move(tif_file, os.path.join("originalStacks", tif_file)) ct = 1 - cumulative_frames = np.concatenate( - ([0], cumulative_frames) - ) + cumulative_frames = np.concatenate(([0], cumulative_frames)) fov_keys = [] tiff_files_entries = [] for iroi in range(n_roi): - ndepths = len(np.atleast_1d(rec_info["ROI"][iroi]["Zs"])) for iz in range(ndepths): - fov_key = dict(key_data) fov_key["tiff_split"] = ct fov_key["tiff_split_directory"] = ( - f"{scan_dirs_db['recording_directory']}/" - f"ROI{iroi+1:02d}_z{iz+1}/" + f"{scan_dirs_db['recording_directory']}/ROI{iroi + 1:02d}_z{iz + 1}/" ) roi_name = rec_info["ROI"][iroi]["name"] if roi_name: - thisname = f"{roi_name}_z{iz+1}" + thisname = f"{roi_name}_z{iz + 1}" else: - thisname = f"ROI{iroi+1:02d}_z{iz+1}" + thisname = f"ROI{iroi + 1:02d}_z{iz + 1}" fov_key["tiff_split_name"] = thisname @@ -561,52 +507,39 @@ def get_fov_mesoscope( else 0 ) - fov_key["fov_center_xy"] = ( - rec_info["ROI"][iroi].get("centerXY", -1) - ) + fov_key["fov_center_xy"] = rec_info["ROI"][iroi].get("centerXY", -1) - fov_key["fov_size_xy"] = ( - rec_info["ROI"][iroi].get("sizeXY", -1) - ) + fov_key["fov_size_xy"] = rec_info["ROI"][iroi].get("sizeXY", -1) - fov_key["fov_rotation_degrees"] = ( - rec_info["ROI"][iroi].get("rotationDegrees", -1) + fov_key["fov_rotation_degrees"] = rec_info["ROI"][iroi].get( + "rotationDegrees", -1 ) - fov_key["fov_pixel_resolution_xy"] = ( - rec_info["ROI"][iroi].get("pixelResolutionXY", -1) + fov_key["fov_pixel_resolution_xy"] = rec_info["ROI"][iroi].get( + "pixelResolutionXY", -1 ) - fov_key["fov_discrete_plane_mode"] = ( - rec_info["ROI"][iroi].get("discretePlaneMode", -1) + fov_key["fov_discrete_plane_mode"] = rec_info["ROI"][iroi].get( + "discretePlaneMode", -1 ) if not fov_key["fov_discrete_plane_mode"]: fov_key["fov_discrete_plane_mode"] = 0 - fov_key["power_percent"] = ( - rec_info["ROI"][iroi].get( - "Power_percent", - rec_info["Scope"]["Power_percent"] - ) + fov_key["power_percent"] = rec_info["ROI"][iroi].get( + "Power_percent", rec_info["Scope"]["Power_percent"] ) fov_keys.append(fov_key) ct += 1 - tiff_split_directory = ( - Path(imaging_root) - / fov_key["tiff_split_directory"] - ) + tiff_split_directory = Path(imaging_root) / fov_key["tiff_split_directory"] - tif_files = sorted( - tiff_split_directory.glob("*.tif") - ) + tif_files = sorted(tiff_split_directory.glob("*.tif")) file_entries = [] for iF, tif_file in enumerate(tif_files): - entry = dict(key_data) entry["tiff_split"] = fov_key["tiff_split"] @@ -615,13 +548,12 @@ def get_fov_mesoscope( entry["file_frame_range"] = [ cumulative_frames[iF] + 1, - cumulative_frames[iF + 1] + cumulative_frames[iF + 1], ] file_entries.append(entry) - - tiff_files_entries += file_entries + tiff_files_entries += file_entries return fov_keys, tiff_files_entries @@ -630,6 +562,7 @@ def get_fov_mesoscope( # 2-photon FOV insert # ----------------------------------------------------------------------------- + def get_fov_photonmicro(key, rec_info, scan_dirs_db): """ Insert FOV metadata for 2-photon imaging. @@ -638,9 +571,7 @@ def get_fov_photonmicro(key, rec_info, scan_dirs_db): fovkey = dict(key) fovkey["tiff_split"] = 1 - fovkey["tiff_split_directory"] = ( - scan_dirs_db["recording_directory"] - ) + fovkey["tiff_split_directory"] = scan_dirs_db["recording_directory"] fovkey["fov_depth"] = 0 fovkey["fov_center_xy"] = 0 @@ -649,9 +580,7 @@ def get_fov_photonmicro(key, rec_info, scan_dirs_db): fovkey["fov_pixel_resolution_xy"] = 0 fovkey["fov_discrete_plane_mode"] = 0 - fovkey["power_percent"] = ( - rec_info["Scope"]["Power_percent"] - ) + fovkey["power_percent"] = rec_info["Scope"]["Power_percent"] return fovkey @@ -660,12 +589,9 @@ def get_fov_photonmicro(key, rec_info, scan_dirs_db): # 2-photon file insertion # ----------------------------------------------------------------------------- + def get_fovfile_photonmicro( - key, - fl, - imheader, - patt_acq_number=r'_[0-9]{5}_', - patt_file_number=r'_[0-9]{5}\.' + key, fl, imheader, patt_acq_number=r"_[0-9]{5}_", patt_file_number=r"_[0-9]{5}\." ): """ Insert TIFF split file entries for 2-photon imaging. @@ -674,38 +600,22 @@ def get_fovfile_photonmicro( filekeys = [] if len(fl) > 0: - prefile_frame_range = 0 for iF, filename in enumerate(fl): + acq_string = re.findall(patt_acq_number, filename) - acq_string = re.findall( - patt_acq_number, - filename - ) - - number_string = re.findall( - patt_file_number, - filename - ) + number_string = re.findall(patt_file_number, filename) if len(acq_string) == 1 and len(number_string) == 1: - - file_number = int( - number_string[0][1:-1] - ) - acq_number = int( - acq_string[0][1:-1] - ) + file_number = int(number_string[0][1:-1]) + acq_number = int(acq_string[0][1:-1]) if acq_number != 1: - file_number = file_number-1+(acq_number-1)*100 + file_number = file_number - 1 + (acq_number - 1) * 100 nframes = len(imheader[iF]) - frame_range = [ - prefile_frame_range + 1, - prefile_frame_range + nframes - ] + frame_range = [prefile_frame_range + 1, prefile_frame_range + nframes] prefile_frame_range = frame_range[1] @@ -718,4 +628,4 @@ def get_fovfile_photonmicro( filekeys.append(entry) - return filekeys \ No newline at end of file + return filekeys From adae9bb1390da577b6333c23a703ddc7c29a3e80 Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Thu, 11 Jun 2026 22:03:44 -0400 Subject: [PATCH 3/6] consolidate the ruff details into pyproject.toml --- pyproject.toml | 14 +++++++------- ruff.toml | 20 -------------------- 2 files changed, 7 insertions(+), 27 deletions(-) delete mode 100644 ruff.toml diff --git a/pyproject.toml b/pyproject.toml index ea9caee4..3d8081fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,12 +75,12 @@ extend-exclude = ["*.ipynb", "**/*.ipynb"] # https://docs.astral.sh/ruff/rules/ # here's a couple to start with select = [ - "A", # warn about shadowing built-ins - "E", # style stuff, whitespaces - "F", # important pyflakes lints - "I", # import sorting - "N", # naming - "T100", # breakpoints (probably don't want these in prod!) + "A", # warn about shadowing built-ins + "E", # style stuff, whitespaces + "F", # important pyflakes lints + "I", # import sorting + "N", # naming + "T100", # breakpoints (probably don't want these in prod!) "B", # flake8-bugbear: likely bugs and design problems "SIM", # flake8-simplify: code that can be simplified "UP", # pyupgrade: syntax that can be modernized (e.g., old Python idioms) @@ -89,7 +89,7 @@ select = [ # if you're feeling confident you can do: # select = ["ALL"] # and then manually ignore annoying ones: -ignore = ["N815","A002", "N806"] +ignore = ["N815", "A002", "N806", "E501", "SIM108"] [dependency-groups] dev = [ diff --git a/ruff.toml b/ruff.toml deleted file mode 100644 index 6510f6fe..00000000 --- a/ruff.toml +++ /dev/null @@ -1,20 +0,0 @@ -[lint] -select = [ - # pycodestyle - "E", - # Pyflakes - "F", - # pyupgrade - "UP", - # flake8-bugbear - "B", - # flake8-simplify - "SIM", - # isort - "I", -] - -ignore = [ - "E501" -] - From 0125fca58edaf7277ca41e55fb752321320bb335 Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Thu, 11 Jun 2026 22:04:47 -0400 Subject: [PATCH 4/6] ruff format --- initial_conf.py | 17 +- notebooks/imaging_element/read_params.py | 22 +- scripts/conf_file_finding.py | 12 +- scripts/ingest_hdf5_to_db.py | 70 +---- .../alert_system/alert_system_utility.py | 16 +- u19_pipeline/alert_system/behavior_metrics.py | 54 +--- .../alert_system/custom_alerts/rig_bias.py | 28 +- .../alert_system/custom_alerts/rig_trial.py | 40 +-- .../custom_alerts/subject_bias.py | 23 +- .../custom_alerts/subject_trial.py | 16 +- .../live_monitor_alert/live_monitor_alert.py | 140 +++------ .../live_session_stats_deletion.py | 8 +- .../locked_tables_alert.py | 4 +- .../log_deletion/old_log_deletion.py | 28 +- .../alert_system/main_alert_system.py | 43 +-- .../noDB_backup_creation_script.py | 58 +--- .../rig_maintenance/check_rig_maintenance.py | 74 ++--- .../schedule_check_alert.py | 32 +- .../alert_system/tech_alert/tech_alert.py | 47 +-- .../create_weighing_gui_ss.py | 135 ++------ .../water_weigh_alert/water_weigh_alert.py | 135 ++------ .../clusters_paths_and_transfers.py | 68 ++-- .../create_missing_lfps_script.py | 8 +- .../automatic_job/ephys_element_ingest.py | 9 +- .../automatic_job/ephys_element_populate.py | 28 +- u19_pipeline/automatic_job/imaging_element.py | 16 +- .../automatic_job/imaging_element_populate.py | 27 +- .../automatic_job/parameter_file_creator.py | 36 +-- u19_pipeline/automatic_job/params_config.py | 12 +- .../populate_missing_syncbehavior_ephys.py | 4 +- .../automatic_job/pupillometry_handler.py | 128 ++------ .../automatic_job/pupillometry_main.py | 4 +- .../automatic_job/recording_handler.py | 184 +++-------- .../recording_process_handler.py | 292 +++++------------- u19_pipeline/automatic_job/slurm_creator.py | 32 +- u19_pipeline/automatic_job/tigress2pni.py | 12 +- u19_pipeline/copy_table.py | 26 +- u19_pipeline/ephys_pipeline.py | 125 +++----- u19_pipeline/ephys_sync.py | 64 ++-- u19_pipeline/imaging_pipeline.py | 56 ++-- u19_pipeline/lab.py | 12 +- u19_pipeline/meso.py | 8 +- u19_pipeline/recording.py | 40 +-- u19_pipeline/recording_process.py | 8 +- u19_pipeline/reference.py | 4 +- u19_pipeline/utility.py | 36 +-- u19_pipeline/utils/ephys_fix_sync_code.py | 110 ++----- u19_pipeline/utils/ephys_utils.py | 226 ++++---------- u19_pipeline/utils/matlab_utils.py | 29 +- u19_pipeline/utils/scp_transfers.py | 4 +- u19_pipeline/utils/slack_utils.py | 40 +-- u19_pipeline/utils/subject_metadata.py | 14 +- .../utils/tiff_matlab_imaging_utils.py | 176 +++-------- u19_pipeline/utils/tiff_utils.py | 88 ++---- 54 files changed, 769 insertions(+), 2159 deletions(-) diff --git a/initial_conf.py b/initial_conf.py index 9adcb382..a85e059f 100755 --- a/initial_conf.py +++ b/initial_conf.py @@ -44,10 +44,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True): host = "datajoint00.pni.princeton.edu" user_already = False - if ( - "database.user" in dj.config - and dj.config.instance._conf["database.user"] is not None - ): + if "database.user" in dj.config and dj.config.instance._conf["database.user"] is not None: user_already = True if replace_user or not user_already: @@ -76,9 +73,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True): # Transform variables to list and path if applicable for custom_var in custom_vars_names: - this_var = custom_vars.loc[ - custom_vars["custom_variable"] == custom_var, "value" - ].tolist() + this_var = custom_vars.loc[custom_vars["custom_variable"] == custom_var, "value"].tolist() # If custom variables are directories, get local path for this system if "dir" in custom_var: @@ -100,9 +95,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True): for i in dj_stores: store_name = i.pop("store_name") dj_stores_dict[store_name] = i - dj_stores_dict[store_name]["location"] = ( - lab.Path().get_local_path2(i["location"]).as_posix() - ) + dj_stores_dict[store_name]["location"] = lab.Path().get_local_path2(i["location"]).as_posix() dj.config["stores"] = dj_stores_dict @@ -123,9 +116,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True): help="prevent to save user into conf file", action="store_true", ) - parser.add_argument( - "--replace_user", "-r", help="replace user in conf file", action="store_true" - ) + parser.add_argument("--replace_user", "-r", help="replace user in conf file", action="store_true") parser.add_argument( "--not_global_config", "-ng", diff --git a/notebooks/imaging_element/read_params.py b/notebooks/imaging_element/read_params.py index 9c8c0ba3..c0e8e8f8 100644 --- a/notebooks/imaging_element/read_params.py +++ b/notebooks/imaging_element/read_params.py @@ -9,22 +9,14 @@ dj.conn() -ephys_element = dj.create_virtual_module( - "u19_pipeline_ephys_element", "u19_pipeline_ephys_element" -) -imaging_element = dj.create_virtual_module( - "u19_pipeline_imaging_element", "u19_pipeline_imaging_element" -) +ephys_element = dj.create_virtual_module("u19_pipeline_ephys_element", "u19_pipeline_ephys_element") +imaging_element = dj.create_virtual_module("u19_pipeline_imaging_element", "u19_pipeline_imaging_element") modalities = ["electrophysiology", "imaging"] params_tables = [ephys_element.ClusteringParamSet, imaging_element.ProcessingParamSet] preparams_steps_tables = [ - ( - ephys_element.PreClusterParamSteps - * ephys_element.PreClusterParamSteps.Step - * ephys_element.PreClusterParamSet - ) + (ephys_element.PreClusterParamSteps * ephys_element.PreClusterParamSteps.Step * ephys_element.PreClusterParamSet) ] preparams_tables = [ephys_element.PreClusterParamSet] @@ -67,13 +59,9 @@ if "precluster_method" in dict: dict["preprocess_method"] = dict.pop("precluster_method") if "precluster_param_steps_name" in dict: - dict["preprocess_param_steps_name"] = dict.pop( - "precluster_param_steps_name" - ) + dict["preprocess_param_steps_name"] = dict.pop("precluster_param_steps_name") if "precluster_param_steps_desc" in dict: - dict["preprocess_param_steps_desc"] = dict.pop( - "precluster_param_steps_desc" - ) + dict["preprocess_param_steps_desc"] = dict.pop("precluster_param_steps_desc") preparams_steps_dict_dict["param_" + str(num_preparams_steps)] = dict num_preparams_steps += 1 diff --git a/scripts/conf_file_finding.py b/scripts/conf_file_finding.py index 7436f241..7f94b548 100644 --- a/scripts/conf_file_finding.py +++ b/scripts/conf_file_finding.py @@ -47,18 +47,12 @@ def try_find_conf_file(): root_dir_found, conf_file_found = chdir_to_root() if root_dir_found and conf_file_found: - print( - "Local configuration file found !!, no need to run the configuration (unless configuration has changed)" - ) + print("Local configuration file found !!, no need to run the configuration (unless configuration has changed)") elif root_dir_found: raise FileNotFoundError( "Local configuration file not found. Ignore this if you have a global config. Run configuration script (initial_conf.py) otherwise" ) elif os.path.isfile(pathlib.Path(pathlib.Path.home(), ".datajoint_config.json")): - print( - "Global configuration file found !!, no need to run the configuration (unless configuration has changed)" - ) + print("Global configuration file found !!, no need to run the configuration (unless configuration has changed)") else: - raise FileNotFoundError( - "Root dir not found, change this notebook to the project folder" - ) + raise FileNotFoundError("Root dir not found, change this notebook to the project folder") diff --git a/scripts/ingest_hdf5_to_db.py b/scripts/ingest_hdf5_to_db.py index 04fe3c0a..89b51a9f 100644 --- a/scripts/ingest_hdf5_to_db.py +++ b/scripts/ingest_hdf5_to_db.py @@ -34,9 +34,7 @@ # Currently a hardcoded data folder to one of Marlies' cohorts for this example, # but in the future will loop over folders starting from # The root folder: /jukebox/braininit/puffs/ -data_folder = ( - "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation/cohort_10/rig0/" -) +data_folder = "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation/cohort_10/rig0/" h5_files = glob.glob(data_folder + "/data*.h5") mouse_info_file = os.path.join( "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation", @@ -201,9 +199,7 @@ # first find list of h5 files that are already processed so we do not # repeat the ingestion on those -already_processed_filenames = (u19_puffs.PuffsFileAcquisition() & "ingested=1").fetch( - "h5_filename" -) +already_processed_filenames = (u19_puffs.PuffsFileAcquisition() & "ingested=1").fetch("h5_filename") # Loop over h5 files in this cohort/rig folder for h5_file in h5_files: @@ -249,26 +245,17 @@ bad_outcome_mask = df_trials["outcome"] > 1 df_trials.loc[bad_outcome_mask, "choice"] = -1 """ Map choice to string values: L,R and nil """ - df_trials["choice"] = df_trials["choice"].apply( - lambda x: "L" if x == 0 else ("R" if x == 1 else "nil") - ) + df_trials["choice"] = df_trials["choice"].apply(lambda x: "L" if x == 0 else ("R" if x == 1 else "nil")) """ Make an answered correct column from outcome column """ df_trials["answered_correct"] = (df_trials["outcome"] == 1).astype("int") """ Turn side column into string type """ - df_trials["side"] = df_trials["side"].apply( - lambda x: "L" if x == 0 else ("R" if x == 1 else "nil") - ) + df_trials["side"] = df_trials["side"].apply(lambda x: "L" if x == 0 else ("R" if x == 1 else "nil")) """ Make a new dataframe for unique sessions, i.e. where subj and session are unique in trials """ - df_sessions = ( - df_trials.groupby(["subj", "session"]) - .max() - .reset_index() - .sort_values(["subj", "session"]) - ) + df_sessions = df_trials.groupby(["subj", "session"]).max().reset_index().sort_values(["subj", "session"]) sessions = df_sessions["session"] last_levels = df_sessions["level"] session_ends_rel = df_sessions["end"] @@ -288,12 +275,9 @@ {"answered_correct": "sum", "side": "count"} ) fraction_trials_correct_df["fraction_correct"] = ( - fraction_trials_correct_df["answered_correct"] - / fraction_trials_correct_df["side"] + fraction_trials_correct_df["answered_correct"] / fraction_trials_correct_df["side"] ) - fraction_trials_correct_df = fraction_trials_correct_df.sort_values( - ["subj", "session"] - ).reset_index() + fraction_trials_correct_df = fraction_trials_correct_df.sort_values(["subj", "session"]).reset_index() df_sessions["fraction_correct"] = fraction_trials_correct_df["fraction_correct"] fractions_correct = df_sessions["fraction_correct"] @@ -322,9 +306,7 @@ elif rig == 1: location = "wang-behavior" else: - sys.exit( - f"Rig: {rig} is not one of the rigs. Check data type (needs to be integer)" - ) + sys.exit(f"Rig: {rig} is not one of the rigs. Check data type (needs to be integer)") session_compressed_str = session_datetime.strftime("%Y%m%d%H%M%S") """ Check to see if the session metadata dataframes exist """ @@ -399,9 +381,7 @@ "sync": session_sync_dict, } # Trial() table - df_trials_this_session = df_trials[ - df_trials["session"] == session_datetime - ][ + df_trials_this_session = df_trials[df_trials["session"] == session_datetime][ [ "idx", "level", @@ -428,9 +408,7 @@ df_trials_this_session["session_number"] = session_number df_trials_this_session["task"] = "AirPuffs" df_trials_this_session["set_id"] = 1 - df_trials_this_session["trial_duration"] = ( - df_trials_this_session["end"] - df_trials_this_session["start"] - ) + df_trials_this_session["trial_duration"] = df_trials_this_session["end"] - df_trials_this_session["start"] df_trials_this_session.rename( columns={ "idx": "trial_idx", @@ -451,15 +429,11 @@ trials_insert_list = df_trials_this_session.to_dict("records") # Puff() table - df_puffs_this_session = df_puffs[ - df_puffs["session"] == session_datetime - ].copy() + df_puffs_this_session = df_puffs[df_puffs["session"] == session_datetime].copy() df_puffs_this_session["subject_fullname"] = subject_fullname df_puffs_this_session["session_date"] = date df_puffs_this_session["session_number"] = session_number - df_puffs_this_session["puff_idx"] = df_puffs_this_session.groupby( - "trial" - ).cumcount() + df_puffs_this_session["puff_idx"] = df_puffs_this_session.groupby("trial").cumcount() df_puffs_this_session.rename( columns={ "time": "puff_rel_time", @@ -483,9 +457,7 @@ puffs_insert_list = df_puffs_this_session.to_dict("records") # TrialPhase() table - df_phases_this_session = df_phases[ - df_phases["session"] == session_datetime - ].copy() + df_phases_this_session = df_phases[df_phases["session"] == session_datetime].copy() df_phases_this_session["subject_fullname"] = subject_fullname df_phases_this_session["session_date"] = date df_phases_this_session["session_number"] = session_number @@ -511,24 +483,16 @@ """ Some phases seem to reference trials that do not exist restrict phase rows to trials that exist """ trial_idxs = df_trials_this_session["trial_idx"] - phase_trial_mask = df_phases_this_session["trial_idx"].isin( - trial_idxs.values - ) + phase_trial_mask = df_phases_this_session["trial_idx"].isin(trial_idxs.values) df_phases_this_session_goodtrials = df_phases_this_session[phase_trial_mask] phases_insert_list = df_phases_this_session_goodtrials.to_dict("records") """ Start a transaction and do the inserts for this session """ connection = u19_acq.SessionStarted.connection with connection.transaction: - u19_acq.SessionStarted().insert1( - session_started_insert_dict, skip_duplicates=True - ) + u19_acq.SessionStarted().insert1(session_started_insert_dict, skip_duplicates=True) u19_acq.Session().insert1(session_insert_dict, skip_duplicates=True) - u19_puffs.PuffsSession().insert1( - puffs_session_insert_dict, skip_duplicates=True - ) - u19_puffs.PuffsSession.Trial().insert( - trials_insert_list, skip_duplicates=True - ) + u19_puffs.PuffsSession().insert1(puffs_session_insert_dict, skip_duplicates=True) + u19_puffs.PuffsSession.Trial().insert(trials_insert_list, skip_duplicates=True) # u19_puffs.PuffsSession.Puff().insert(puffs_insert_list,skip_duplicates=True) # u19_puffs.PuffsSession.TrialPhase().insert(phases_insert_list,skip_duplicates=True) """ If the inserts for all sessions in this hdf5 file were successful, diff --git a/u19_pipeline/alert_system/alert_system_utility.py b/u19_pipeline/alert_system/alert_system_utility.py index c834330c..a0bb93d8 100644 --- a/u19_pipeline/alert_system/alert_system_utility.py +++ b/u19_pipeline/alert_system/alert_system_utility.py @@ -6,9 +6,7 @@ import u19_pipeline.utils.dj_shortcuts as djs -def get_acquisition_data_alert_system( - type="subject_fullname", data_days=60, min_sessions=20 -): +def get_acquisition_data_alert_system(type="subject_fullname", data_days=60, min_sessions=20): """ Get and filter data for alert system Inputs: @@ -29,19 +27,13 @@ def get_acquisition_data_alert_system( query_sessions += 'and session_date < "' + today.strftime("%Y-%m-%d") + '"' query_sessions += ' and subject_fullname not like "testuser%"' - session_df = pd.DataFrame( - (acquisition.Session & query_sessions).fetch(as_dict=True) - ) + session_df = pd.DataFrame((acquisition.Session & query_sessions).fetch(as_dict=True)) # Filter subjects/rigs with >= min_sessions - num_sessions_df = session_df.groupby(type).agg( - {"session_date": [("total_sessions", "count")]} - ) + num_sessions_df = session_df.groupby(type).agg({"session_date": [("total_sessions", "count")]}) num_sessions_df.columns = num_sessions_df.columns.droplevel() num_sessions_df = num_sessions_df.reset_index() - num_sessions_df = num_sessions_df.loc[ - num_sessions_df["total_sessions"] >= min_sessions, : - ] + num_sessions_df = num_sessions_df.loc[num_sessions_df["total_sessions"] >= min_sessions, :] session_df = session_df.merge(num_sessions_df, on=type) # Get key list of sessions for further (behavior) querying diff --git a/u19_pipeline/alert_system/behavior_metrics.py b/u19_pipeline/alert_system/behavior_metrics.py index 6cc64038..17b4ecce 100644 --- a/u19_pipeline/alert_system/behavior_metrics.py +++ b/u19_pipeline/alert_system/behavior_metrics.py @@ -14,9 +14,7 @@ def get_bias_from_trial_df(trials_df, return_all_metrics=False): "trial_idx", ] session_columns = ["subject_fullname", "session_date", "session_number"] - trials_df = trials_df.sort_values( - by=sort_columns, ascending=[True, False, False, True, True] - ) + trials_df = trials_df.sort_values(by=sort_columns, ascending=[True, False, False, True, True]) trials_df = trials_df.reset_index(drop=True) # Left & right trials as integers @@ -25,46 +23,28 @@ def get_bias_from_trial_df(trials_df, return_all_metrics=False): trials_df["trial"] = 1 # Cumulative sum of Left & right trials as integers per session - trials_df["cum_left_trials"] = trials_df.groupby(session_columns)[ - "left_trial" - ].cumsum() - trials_df["cum_right_trials"] = trials_df.groupby(session_columns)[ - "right_trial" - ].cumsum() + trials_df["cum_left_trials"] = trials_df.groupby(session_columns)["left_trial"].cumsum() + trials_df["cum_right_trials"] = trials_df.groupby(session_columns)["right_trial"].cumsum() trials_df["cum_trials"] = trials_df.groupby(session_columns)["trial"].cumsum() # Correct trials per side as integers - trials_df["correct_trial"] = ( - trials_df["trial_type"] == trials_df["choice"] - ).astype(int) - trials_df["correct_left"] = ( - (trials_df["correct_trial"] == 1) & (trials_df["trial_type"] == "L") - ).astype(int) - trials_df["correct_right"] = ( - (trials_df["correct_trial"] == 1) & (trials_df["trial_type"] == "R") - ).astype(int) + trials_df["correct_trial"] = (trials_df["trial_type"] == trials_df["choice"]).astype(int) + trials_df["correct_left"] = ((trials_df["correct_trial"] == 1) & (trials_df["trial_type"] == "L")).astype(int) + trials_df["correct_right"] = ((trials_df["correct_trial"] == 1) & (trials_df["trial_type"] == "R")).astype(int) # Cumulative sum of per side correct trials - trials_df["cum_correct_left_trials"] = trials_df.groupby(session_columns)[ - "correct_left" - ].cumsum() - trials_df["cum_correct_right_trials"] = trials_df.groupby(session_columns)[ - "correct_right" - ].cumsum() - trials_df["cum_correct_trials"] = trials_df.groupby(session_columns)[ - "correct_trial" - ].cumsum() + trials_df["cum_correct_left_trials"] = trials_df.groupby(session_columns)["correct_left"].cumsum() + trials_df["cum_correct_right_trials"] = trials_df.groupby(session_columns)["correct_right"].cumsum() + trials_df["cum_correct_trials"] = trials_df.groupby(session_columns)["correct_trial"].cumsum() # Get only last trial count - trials_df = trials_df.loc[ - ~trials_df.duplicated(subset=session_columns, keep="last"), : - ] + trials_df = trials_df.loc[~trials_df.duplicated(subset=session_columns, keep="last"), :] trials_df = trials_df.reset_index(drop=True) # Calculate bias - trials_df["bias"] = ( - trials_df["cum_correct_right_trials"] / trials_df["cum_right_trials"] - ) - (trials_df["cum_correct_left_trials"] / trials_df["cum_left_trials"]) + trials_df["bias"] = (trials_df["cum_correct_right_trials"] / trials_df["cum_right_trials"]) - ( + trials_df["cum_correct_left_trials"] / trials_df["cum_left_trials"] + ) if return_all_metrics: bias_df = trials_df @@ -92,9 +72,7 @@ def get_zscore_metric_session_df(session_df, metric, groupby_column): z_score_metric_l = "z_score_" + metric # Get mean and std for the selected metric - avg_df = session_df.groupby(groupby_column).agg( - {metric: [(avg_metric_l, "mean"), (std_metric_l, "std")]} - ) + avg_df = session_df.groupby(groupby_column).agg({metric: [(avg_metric_l, "mean"), (std_metric_l, "std")]}) avg_df.columns = avg_df.columns.droplevel() avg_df = avg_df.reset_index() @@ -102,8 +80,6 @@ def get_zscore_metric_session_df(session_df, metric, groupby_column): session_df = session_df.merge(avg_df) # Calculate proper zscore - session_df[z_score_metric_l] = ( - session_df[metric] - session_df[avg_metric_l] - ) / session_df[std_metric_l] + session_df[z_score_metric_l] = (session_df[metric] - session_df[avg_metric_l]) / session_df[std_metric_l] return session_df diff --git a/u19_pipeline/alert_system/custom_alerts/rig_bias.py b/u19_pipeline/alert_system/custom_alerts/rig_bias.py index 39656940..ef77fbf0 100644 --- a/u19_pipeline/alert_system/custom_alerts/rig_bias.py +++ b/u19_pipeline/alert_system/custom_alerts/rig_bias.py @@ -18,31 +18,21 @@ def main(): Main function for subject "num trials & bias" alert """ # Get sessions - _, rig_session_key_list = asu.get_acquisition_data_alert_system( - type="session_location" - ) + _, rig_session_key_list = asu.get_acquisition_data_alert_system(type="session_location") # Get trials - behavior = dj.create_virtual_module( - "behavior", dj.config["custom"]["database.prefix"] + "behavior" - ) - acquisition = dj.create_virtual_module( - "acquisition", dj.config["custom"]["database.prefix"] + "acquisition" - ) + behavior = dj.create_virtual_module("behavior", dj.config["custom"]["database.prefix"] + "behavior") + acquisition = dj.create_virtual_module("acquisition", dj.config["custom"]["database.prefix"] + "acquisition") rig_trial_df = pd.DataFrame( - ( - behavior.TowersBlock.Trial - * acquisition.SessionStarted.proj("session_location") - & rig_session_key_list - ).fetch("KEY", "trial_type", "choice", "session_location", as_dict=True) + (behavior.TowersBlock.Trial * acquisition.SessionStarted.proj("session_location") & rig_session_key_list).fetch( + "KEY", "trial_type", "choice", "session_location", as_dict=True + ) ) # Get zscores for bias bias_df = bm.BehaviorMetrics.get_bias_from_trial_df(rig_trial_df) - bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df( - bias_df, "bias", "subject_fullname" - ) + bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df(bias_df, "bias", "subject_fullname") # Filter df for today today = datetime.date.today() - datetime.timedelta(days=1) @@ -68,9 +58,7 @@ def main(): bias_location = bias_location.reset_index() # Filter if there were subjects biased to different sides - bias_location2 = bias_location.groupby(["session_location"]).agg( - {"session_location": [("num_bias_sides", "size")]} - ) + bias_location2 = bias_location.groupby(["session_location"]).agg({"session_location": [("num_bias_sides", "size")]}) bias_location2.columns = bias_location2.columns.droplevel() bias_location2 = bias_location2.reset_index() bias_location2 = bias_location2.loc[bias_location2["num_bias_sides"] == 1, :] diff --git a/u19_pipeline/alert_system/custom_alerts/rig_trial.py b/u19_pipeline/alert_system/custom_alerts/rig_trial.py index 1e5c186f..3ad79122 100644 --- a/u19_pipeline/alert_system/custom_alerts/rig_trial.py +++ b/u19_pipeline/alert_system/custom_alerts/rig_trial.py @@ -19,49 +19,37 @@ def main(): subject_rig_df, _ = asu.get_acquisition_data_alert_system(type="session_location") # Get zscores for each session (group by rig) - subject_rig_df = bm.BehaviorMetrics.get_zscore_metric_session_df( - subject_rig_df, "num_trials", "session_location" - ) + subject_rig_df = bm.BehaviorMetrics.get_zscore_metric_session_df(subject_rig_df, "num_trials", "session_location") # Get only todays data today = datetime.date.today() - datetime.timedelta(days=1) subject_rig_df = subject_rig_df.loc[subject_rig_df["session_date"] == today, :] # Filter if today we got > 3 zscore of trials for a session - subject_rig_df["abs_z_score_num_trials"] = np.abs( - subject_rig_df["z_score_num_trials"] + subject_rig_df["abs_z_score_num_trials"] = np.abs(subject_rig_df["z_score_num_trials"]) + num_trials_alert_df = subject_rig_df.loc[subject_rig_df["abs_z_score_num_trials"] >= zscore_alert, :].copy( + deep=True ) - num_trials_alert_df = subject_rig_df.loc[ - subject_rig_df["abs_z_score_num_trials"] >= zscore_alert, : - ].copy(deep=True) # Get sign of zscpre (only group subjects with same deviation) - num_trials_alert_df["sign_zscore"] = np.sign( - num_trials_alert_df["z_score_num_trials"] - ) + num_trials_alert_df["sign_zscore"] = np.sign(num_trials_alert_df["z_score_num_trials"]) num_trials_alert_df["sign_zscore"] = num_trials_alert_df["sign_zscore"].astype(int) # Count how many subjects were biassed by rig - num_trials_alert_location_df = num_trials_alert_df.groupby( - ["session_location", "sign_zscore"] - ).agg( + num_trials_alert_location_df = num_trials_alert_df.groupby(["session_location", "sign_zscore"]).agg( { "session_location": [("num_subjects", "size")], "subject_fullname": [("subject_fullnames", lambda x: ",".join(x))], } ) - num_trials_alert_location_df.columns = ( - num_trials_alert_location_df.columns.droplevel() - ) + num_trials_alert_location_df.columns = num_trials_alert_location_df.columns.droplevel() num_trials_alert_location_df = num_trials_alert_location_df.reset_index() # Filter if there were subjects with a lot of trials and very little trials - num_trials_alert_location_df2 = num_trials_alert_location_df.groupby( - ["session_location"] - ).agg({"session_location": [("num_bias_sides", "size")]}) - num_trials_alert_location_df2.columns = ( - num_trials_alert_location_df2.columns.droplevel() + num_trials_alert_location_df2 = num_trials_alert_location_df.groupby(["session_location"]).agg( + {"session_location": [("num_bias_sides", "size")]} ) + num_trials_alert_location_df2.columns = num_trials_alert_location_df2.columns.droplevel() num_trials_alert_location_df2 = num_trials_alert_location_df2.reset_index() num_trials_alert_location_df2 = num_trials_alert_location_df2.loc[ num_trials_alert_location_df2["num_bias_sides"] == 1, : @@ -71,9 +59,7 @@ def main(): ) # Only alert if more than 1 subject was biased today - num_trials_alert_location_df = num_trials_alert_location_df.loc[ - num_trials_alert_location_df["num_subjects"] > 1, : - ] + num_trials_alert_location_df = num_trials_alert_location_df.loc[num_trials_alert_location_df["num_subjects"] > 1, :] # Filter columns and set message columns_alert = [ @@ -83,8 +69,6 @@ def main(): "subject_fullnames", ] num_trials_alert_location_df = num_trials_alert_location_df[columns_alert] - num_trials_alert_location_df["alert_message"] = ( - "Multiple subjects had abnormal number of trials in this rig" - ) + num_trials_alert_location_df["alert_message"] = "Multiple subjects had abnormal number of trials in this rig" return num_trials_alert_location_df diff --git a/u19_pipeline/alert_system/custom_alerts/subject_bias.py b/u19_pipeline/alert_system/custom_alerts/subject_bias.py index b768c028..0bd7d7d7 100644 --- a/u19_pipeline/alert_system/custom_alerts/subject_bias.py +++ b/u19_pipeline/alert_system/custom_alerts/subject_bias.py @@ -19,30 +19,21 @@ def main(): """ # Get sessions - _, subject_session_key_list = asu.get_acquisition_data_alert_system( - type="subject_fullname" - ) + _, subject_session_key_list = asu.get_acquisition_data_alert_system(type="subject_fullname") # Get trials - behavior = dj.create_virtual_module( - "behavior", dj.config["custom"]["database.prefix"] + "behavior" - ) - acquisition = dj.create_virtual_module( - "acquisition", dj.config["custom"]["database.prefix"] + "acquisition" - ) + behavior = dj.create_virtual_module("behavior", dj.config["custom"]["database.prefix"] + "behavior") + acquisition = dj.create_virtual_module("acquisition", dj.config["custom"]["database.prefix"] + "acquisition") subject_trial_df = pd.DataFrame( - ( - behavior.TowersBlock.Trial * acquisition.SessionStarted - & subject_session_key_list - ).fetch("KEY", "trial_type", "choice", "session_location", as_dict=True) + (behavior.TowersBlock.Trial * acquisition.SessionStarted & subject_session_key_list).fetch( + "KEY", "trial_type", "choice", "session_location", as_dict=True + ) ) # Get zscores for bias bias_df = bm.BehaviorMetrics.get_bias_from_trial_df(subject_trial_df) - bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df( - bias_df, "bias", "subject_fullname" - ) + bias_df = bm.BehaviorMetrics.get_zscore_metric_session_df(bias_df, "bias", "subject_fullname") # Filter df for todays alert today = datetime.date.today() - datetime.timedelta(days=1) diff --git a/u19_pipeline/alert_system/custom_alerts/subject_trial.py b/u19_pipeline/alert_system/custom_alerts/subject_trial.py index 13888011..00858029 100644 --- a/u19_pipeline/alert_system/custom_alerts/subject_trial.py +++ b/u19_pipeline/alert_system/custom_alerts/subject_trial.py @@ -17,9 +17,7 @@ def main(): """ # Get sessions - subject_session_df, _ = asu.get_acquisition_data_alert_system( - type="subject_fullname" - ) + subject_session_df, _ = asu.get_acquisition_data_alert_system(type="subject_fullname") # Get zscores for num_trials subject_session_df = bm.BehaviorMetrics.get_zscore_metric_session_df( @@ -28,17 +26,11 @@ def main(): # Filter df for todays alert today = datetime.date.today() - datetime.timedelta(days=1) - subject_session_df = subject_session_df.loc[ - subject_session_df["session_date"] == today, : - ] - subject_session_df["abs_z_score_num_trials"] = np.abs( - subject_session_df["z_score_num_trials"] - ) + subject_session_df = subject_session_df.loc[subject_session_df["session_date"] == today, :] + subject_session_df["abs_z_score_num_trials"] = np.abs(subject_session_df["z_score_num_trials"]) # Filter if today we got > 3 zscore of trials for a session - alert_subjtect_trial_df = subject_session_df.loc[ - subject_session_df["abs_z_score_num_trials"] >= zscore_alert, : - ] + alert_subjtect_trial_df = subject_session_df.loc[subject_session_df["abs_z_score_num_trials"] >= zscore_alert, :] columns_alert = [ "subject_fullname", diff --git a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py index 7350af1f..eb7910ea 100644 --- a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py +++ b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py @@ -11,14 +11,10 @@ SECONDS_ALERT = MINUTES_ALERT * 60 MIN_SESSIONS_COMPLETED = 3 -slack_configuration_dictionary = { - "slack_notification_channel": ["rig_training_error_notification"] -} +slack_configuration_dictionary = {"slack_notification_channel": ["rig_training_error_notification"]} -def slack_alert_message_format_live_stats( - alert_dictionary1, alert_dictionary2, time_no_response -): +def slack_alert_message_format_live_stats(alert_dictionary1, alert_dictionary2, time_no_response): "Format dictionaries for live monitor slack alert json" now = datetime.now() @@ -87,9 +83,7 @@ def main_live_monitor_alert(): query["is_finished"] = 0 # Only look for sessions started in the last 1:30 - last_time_start = datetime.now(tz=ZoneInfo("America/New_York")) - timedelta( - hours=2, minutes=30 - ) + last_time_start = datetime.now(tz=ZoneInfo("America/New_York")) - timedelta(hours=2, minutes=30) last_time_start = last_time_start.replace(tzinfo=None).strftime("%Y-%m-%d %H:%M:%S") query_started_recently = "session_start_time > '" + last_time_start + "'" @@ -104,14 +98,10 @@ def main_live_monitor_alert(): if sessions.shape[0] > 0: # If more than one "not finished" session in same rig, grab the last one started - sessions2 = sessions.groupby("session_location").agg( - {"session_start_time": [("session_start_time", "max")]} - ) + sessions2 = sessions.groupby("session_location").agg({"session_start_time": [("session_start_time", "max")]}) sessions2.columns = sessions2.columns.droplevel() sessions2 = sessions2.reset_index() - sessions = pd.merge( - sessions, sessions2, on=["session_location", "session_start_time"] - ) + sessions = pd.merge(sessions, sessions2, on=["session_location", "session_start_time"]) sessions = sessions.drop(columns=["session_location"]) sessions = sessions.reset_index(drop=True) @@ -121,9 +111,7 @@ def main_live_monitor_alert(): query_reported = {} query_reported["session_date"] = datetime.today().strftime("%Y-%m-%d") sessions_reported = pd.DataFrame( - (acquisition.ReportedLiveSessionStats & query_reported).fetch( - "KEY", as_dict=True - ) + (acquisition.ReportedLiveSessionStats & query_reported).fetch("KEY", as_dict=True) ) if sessions_reported.shape[0] > 0: @@ -136,9 +124,7 @@ def main_live_monitor_alert(): # Only analyze sessions subjects > NUM_SESSIONS_COMPLETED have not been reported if sessions.shape[0] > 0: - query_subjects = ( - "subject_fullname in ('" + "', '".join(sessions["subject_fullname"]) + "')" - ) + query_subjects = "subject_fullname in ('" + "', '".join(sessions["subject_fullname"]) + "')" count_sessions_table = (subject.Subject).aggr( (acquisition.Session & query_subjects), @@ -161,33 +147,21 @@ def main_live_monitor_alert(): query_no_violation_trial = dict() query_no_violation_trial["violation_trial"] = 0 lss_nvio = acquisition.SessionStarted.aggr( - (acquisition.LiveSessionStats & query_no_violation_trial).proj( - "current_datetime" - ), + (acquisition.LiveSessionStats & query_no_violation_trial).proj("current_datetime"), current_datetime="max(current_datetime)", ) - live_stats_nvio = pd.DataFrame( - (lss_nvio & query_live_stats).fetch(as_dict=True) - ) - live_stats_nvio = live_stats_nvio.rename( - {"current_datetime": "last_non_violation_trial"}, axis=1 - ) + live_stats_nvio = pd.DataFrame((lss_nvio & query_live_stats).fetch(as_dict=True)) + live_stats_nvio = live_stats_nvio.rename({"current_datetime": "last_non_violation_trial"}, axis=1) # Last violation trial in sessions query_violation_trial = dict() query_violation_trial["violation_trial"] = 1 lss_vio = acquisition.SessionStarted.aggr( - (acquisition.LiveSessionStats & query_violation_trial).proj( - "current_datetime" - ), + (acquisition.LiveSessionStats & query_violation_trial).proj("current_datetime"), current_datetime="max(current_datetime)", ) - live_stats_vio = pd.DataFrame( - (lss_vio & query_live_stats & query_violation_trial).fetch(as_dict=True) - ) - live_stats_vio = live_stats_vio.rename( - {"current_datetime": "last_violation_trial"}, axis=1 - ) + live_stats_vio = pd.DataFrame((lss_vio & query_live_stats & query_violation_trial).fetch(as_dict=True)) + live_stats_vio = live_stats_vio.rename({"current_datetime": "last_violation_trial"}, axis=1) # Merge last violation trial and last non violation Trials from sessions if live_stats_nvio.shape[0] > 0 and live_stats_vio.shape[0] > 0: @@ -211,26 +185,16 @@ def main_live_monitor_alert(): right_now_est = right_now_est.replace(tzinfo=None) live_stats["seconds_elapsed_last_stat_nvio"] = ( - ( - right_now_est - live_stats["last_non_violation_trial"] - ).dt.total_seconds() + (right_now_est - live_stats["last_non_violation_trial"]).dt.total_seconds() ).astype(int) - live_stats["alert_nvio"] = ( - live_stats["seconds_elapsed_last_stat_nvio"] > SECONDS_ALERT - ) + live_stats["alert_nvio"] = live_stats["seconds_elapsed_last_stat_nvio"] > SECONDS_ALERT live_stats["seconds_elapsed_session_started"] = ( (right_now_est - live_stats["session_start_time"]).dt.total_seconds() ).astype(int) - live_stats["alert_vio"] = ( - live_stats["seconds_elapsed_session_started"] > SECONDS_ALERT - ) - live_stats["alert_vio"] = live_stats["alert_vio"] & ( - pd.isna(live_stats["last_non_violation_trial"]) - ) - live_stats["alert_vio"] = live_stats["alert_vio"] & ( - ~pd.isna(live_stats["last_violation_trial"]) - ) + live_stats["alert_vio"] = live_stats["seconds_elapsed_session_started"] > SECONDS_ALERT + live_stats["alert_vio"] = live_stats["alert_vio"] & (pd.isna(live_stats["last_non_violation_trial"])) + live_stats["alert_vio"] = live_stats["alert_vio"] & (~pd.isna(live_stats["last_violation_trial"])) print(live_stats.T) @@ -242,33 +206,25 @@ def main_live_monitor_alert(): # If there are any sessions to alert (more then SECONDS_ALERT) if live_stats.shape[0] > 0: live_stats["current_datetime"] = fake_date - live_stats.loc[live_stats["alert_vio"] == True, "current_datetime"] = ( - live_stats.loc[ - live_stats["alert_vio"] == True, "last_violation_trial" - ] - ) - live_stats.loc[live_stats["alert_nvio"] == True, "current_datetime"] = ( - live_stats.loc[ - live_stats["alert_nvio"] == True, "last_non_violation_trial" - ] - ) + live_stats.loc[live_stats["alert_vio"] == True, "current_datetime"] = live_stats.loc[ + live_stats["alert_vio"] == True, "last_violation_trial" + ] + live_stats.loc[live_stats["alert_nvio"] == True, "current_datetime"] = live_stats.loc[ + live_stats["alert_nvio"] == True, "last_non_violation_trial" + ] live_stats["seconds_elapsed_last_valid_stat"] = 0 - live_stats.loc[ - live_stats["alert_vio"] == True, "seconds_elapsed_last_valid_stat" - ] = live_stats.loc[ + live_stats.loc[live_stats["alert_vio"] == True, "seconds_elapsed_last_valid_stat"] = live_stats.loc[ live_stats["alert_vio"] == True, "seconds_elapsed_session_started" ] - live_stats.loc[ - live_stats["alert_nvio"] == True, "seconds_elapsed_last_valid_stat" - ] = live_stats.loc[ + live_stats.loc[live_stats["alert_nvio"] == True, "seconds_elapsed_last_valid_stat"] = live_stats.loc[ live_stats["alert_nvio"] == True, "seconds_elapsed_last_stat_nvio" ] # get_session_info to alert (plus slack researcher) - query_live_stats_sessions = live_stats[ - ["subject_fullname", "session_date", "session_number"] - ].to_dict("records") + query_live_stats_sessions = live_stats[["subject_fullname", "session_date", "session_number"]].to_dict( + "records" + ) session_data_df = pd.DataFrame( ( @@ -281,15 +237,9 @@ def main_live_monitor_alert(): ).fetch(as_dict=True) ) - session_data_df = session_data_df.rename( - {"slack": "researcher"}, axis=1 - ) - session_data_df["researcher"] = ( - "<@" + session_data_df["researcher"] + ">" - ) - session_data_df = session_data_df[ - ["researcher", "subject_fullname", "session_date", "session_number"] - ] + session_data_df = session_data_df.rename({"slack": "researcher"}, axis=1) + session_data_df["researcher"] = "<@" + session_data_df["researcher"] + ">" + session_data_df = session_data_df[["researcher", "subject_fullname", "session_date", "session_number"]] # Query full live stat table # session_stats = live_stats.copy() @@ -311,23 +261,15 @@ def main_live_monitor_alert(): ] ].copy() - ls_full_df = pd.DataFrame( - (acquisition.LiveSessionStats & query_live_stats).fetch( - as_dict=True - ) - ) + ls_full_df = pd.DataFrame((acquisition.LiveSessionStats & query_live_stats).fetch(as_dict=True)) ls_full_df = pd.merge( ls_full_df, live_stats_mini, on=["subject_fullname", "session_date", "session_number"], ) - ls_full_df = ls_full_df.drop( - columns=["subject_fullname", "session_date", "session_number"] - ) - ls_full_df = ls_full_df.rename( - {"current_datetime": "last_trial_time"}, axis=1 - ) + ls_full_df = ls_full_df.drop(columns=["subject_fullname", "session_date", "session_number"]) + ls_full_df = ls_full_df.rename({"current_datetime": "last_trial_time"}, axis=1) mid = ls_full_df["last_trial_time"] ls_full_df = ls_full_df.drop(columns=["last_trial_time"]) @@ -347,19 +289,13 @@ def main_live_monitor_alert(): ) # Send alert - webhooks_list = su.get_webhook_list( - slack_configuration_dictionary, lab - ) + webhooks_list = su.get_webhook_list(slack_configuration_dictionary, lab) for this_webhook in webhooks_list: su.send_slack_notification(this_webhook, slack_json_message) time.sleep(1) - reported_session = this_session_stats[ - ["subject_fullname", "session_date", "session_number"] - ].copy() + reported_session = this_session_stats[["subject_fullname", "session_date", "session_number"]].copy() reported_session["report_datetime"] = right_now_est - acquisition.ReportedLiveSessionStats.insert1( - reported_session.to_dict() - ) + acquisition.ReportedLiveSessionStats.insert1(reported_session.to_dict()) idx_alert += 1 diff --git a/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py b/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py index 24e36df3..7a900943 100644 --- a/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py +++ b/u19_pipeline/alert_system/live_session_stats_deletion/live_session_stats_deletion.py @@ -10,10 +10,8 @@ def main_live_session_stats_deletion(): order_cols = "'subject_fullname', 'session_date', 'session_number', 'trial_idx'" connection = acquisition.LiveSessionStats.connection with connection.transaction: - old_session_stats = ( - acquisition.LiveSessionStats & old_session_stats_query - ).fetch(as_dict=True, order_by=[order_cols]) - (acquisition.HistoricSessionStats).insert( - old_session_stats, skip_duplicates=True + old_session_stats = (acquisition.LiveSessionStats & old_session_stats_query).fetch( + as_dict=True, order_by=[order_cols] ) + (acquisition.HistoricSessionStats).insert(old_session_stats, skip_duplicates=True) (acquisition.LiveSessionStats & old_session_stats_query).delete(safemode=False) diff --git a/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py b/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py index 8d61f67b..503a0f13 100644 --- a/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py +++ b/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py @@ -15,9 +15,7 @@ def main_locked_tables_alert(): locked_tables_query = "show open tables where in_use > 0" conn = dj.conn() - locked_tables_df = pd.DataFrame( - conn.query(locked_tables_query, as_dict=True).fetchall() - ) + locked_tables_df = pd.DataFrame(conn.query(locked_tables_query, as_dict=True).fetchall()) if locked_tables_df.shape[0] == 0: return diff --git a/u19_pipeline/alert_system/log_deletion/old_log_deletion.py b/u19_pipeline/alert_system/log_deletion/old_log_deletion.py index 36f9c911..a9224176 100644 --- a/u19_pipeline/alert_system/log_deletion/old_log_deletion.py +++ b/u19_pipeline/alert_system/log_deletion/old_log_deletion.py @@ -15,30 +15,16 @@ def main_old_log_deletion(): date_pattern = r"_\d{8}_" reference_date = datetime.date.today() - datetime.timedelta(days=10) - files = [ - p - for p in DIRECTORY_PATH.rglob("*") - if p.is_file() and re.search(date_pattern, p.name) - ] + files = [p for p in DIRECTORY_PATH.rglob("*") if p.is_file() and re.search(date_pattern, p.name)] files_df = pd.DataFrame(files, columns=["filepaths"]) files_df["filename"] = files_df["filepaths"].apply(extract_filename) - files_df["exctract_date"] = files_df["filename"].str.extract( - r"(\d{8," + str(8) + r"})" - ) - files_df["exctract_date"] = pd.to_datetime( - files_df["exctract_date"], format="%Y%m%d" - ) - files_df["before_ref_date"] = files_df["exctract_date"] <= pd.Timestamp( - reference_date - ) - - files_for_deletion = files_df.loc[ - files_df["before_ref_date"] == True, "filepaths" - ].to_list() + files_df["exctract_date"] = files_df["filename"].str.extract(r"(\d{8," + str(8) + r"})") + files_df["exctract_date"] = pd.to_datetime(files_df["exctract_date"], format="%Y%m%d") + files_df["before_ref_date"] = files_df["exctract_date"] <= pd.Timestamp(reference_date) + + files_for_deletion = files_df.loc[files_df["before_ref_date"] == True, "filepaths"].to_list() for file_path in files_for_deletion: - if ( - file_path.is_file() - ): # Check if it's actually a file before attempting to delete + if file_path.is_file(): # Check if it's actually a file before attempting to delete file_path.unlink() diff --git a/u19_pipeline/alert_system/main_alert_system.py b/u19_pipeline/alert_system/main_alert_system.py index 4d64efec..c261cd78 100644 --- a/u19_pipeline/alert_system/main_alert_system.py +++ b/u19_pipeline/alert_system/main_alert_system.py @@ -35,30 +35,15 @@ def main_alert_system(): webhooks_list = [] if "slack_notification_channel" in slack_dict: - query_slack_webhooks = [ - {"webhook_name": x} - for x in slack_dict["slack_notification_channel"] - ] - webhooks_list += ( - (lab.SlackWebhooks & query_slack_webhooks) - .fetch("webhook_url") - .tolist() - ) + query_slack_webhooks = [{"webhook_name": x} for x in slack_dict["slack_notification_channel"]] + webhooks_list += (lab.SlackWebhooks & query_slack_webhooks).fetch("webhook_url").tolist() if "slack_users_channel" in slack_dict: - query_slack_user_channels = [ - {"user_id": x} for x in slack_dict["slack_users_channel"] - ] - webhooks_list += ( - (lab.User & query_slack_user_channels) - .fetch("slack_webhook") - .tolist() - ) + query_slack_user_channels = [{"user_id": x} for x in slack_dict["slack_users_channel"]] + webhooks_list += (lab.User & query_slack_user_channels).fetch("slack_webhook").tolist() for this_alert_record in alert_dict: - slack_json_message = slack_alert_message_format( - this_alert_record, this_alert_submodule.name - ) + slack_json_message = slack_alert_message_format(this_alert_record, this_alert_submodule.name) for this_webhook in webhooks_list: su.send_slack_notification(this_webhook, slack_json_message) @@ -66,20 +51,10 @@ def main_alert_system(): except Exception as e: dict_error = dict() - dict_error["message"] = ( - "error while executing " + this_alert_submodule.name + " alert code" - ) - dict_error["error_exception"] = "".join( - traceback.format_exception(type(e), value=e, tb=e.__traceback__) - ) - slack_json_message = slack_alert_message_format( - dict_error, this_alert_submodule.name - ) - webhook_custom = ( - (lab.SlackWebhooks & "webhook_name='custom_alerts'") - .fetch("webhook_url") - .tolist() - ) + dict_error["message"] = "error while executing " + this_alert_submodule.name + " alert code" + dict_error["error_exception"] = "".join(traceback.format_exception(type(e), value=e, tb=e.__traceback__)) + slack_json_message = slack_alert_message_format(dict_error, this_alert_submodule.name) + webhook_custom = (lab.SlackWebhooks & "webhook_name='custom_alerts'").fetch("webhook_url").tolist() if webhook_custom: su.send_slack_notification(webhook_custom[0], slack_json_message) diff --git a/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py b/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py index 28e5bd00..cdd7092f 100644 --- a/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py +++ b/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py @@ -32,12 +32,8 @@ PAST_SESSION_PERFORMANCE_FILENAME = "PastSessions.csv" SUBJECT_MOTOR_POSITION_FILENAME = "SubjectMotorPosition.csv" -WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = ( - "Weighing_GUI_Replacement_SpreadSheet_Template.xlsx" -) -WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = ( - "Weighing_GUI_Replacement_SpreadSheet.xlsx" -) +WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE = "Weighing_GUI_Replacement_SpreadSheet_Template.xlsx" +WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME = "Weighing_GUI_Replacement_SpreadSheet.xlsx" conf = dj.config nodb_virmen_backup_dir = pathlib.Path( @@ -83,11 +79,7 @@ def write_slack_webhooks_file(): def write_user_data_file(): - user_data = pd.DataFrame( - lab.User.fetch( - "user_id", "slack", "tech_responsibility", "slack_webhook", as_dict=True - ) - ) + user_data = pd.DataFrame(lab.User.fetch("user_id", "slack", "tech_responsibility", "slack_webhook", as_dict=True)) user_data["slack_webhook"] = user_data["slack_webhook"].astype(str) user_data["slack_webhook"] = user_data["slack_webhook"].fillna("") @@ -102,9 +94,7 @@ def write_rig_status_file(): file_write = pathlib.Path(nodb_virmen_backup_dir, RIG_STATUS_FILENAME) df_rig_status = pd.DataFrame( - scheduler.RigStatus.fetch( - "location", "input_output_name", "current_status", as_dict=True - ) + scheduler.RigStatus.fetch("location", "input_output_name", "current_status", as_dict=True) ) df_rig_status.to_csv(file_write, index=False) @@ -118,9 +108,7 @@ def write_schedule_file(): day_schedule = pd.DataFrame( ( - scheduler.Schedule - * scheduler.TrainingProfile - * subject.Subject.proj(subject_user_id="user_id") + scheduler.Schedule * scheduler.TrainingProfile * subject.Subject.proj(subject_user_id="user_id") & schedule_query & subject_query ).fetch(as_dict=True) @@ -147,32 +135,20 @@ def write_past_sessions_file(day_schedule): ) ) ss["session_date"] = ss["session_date"].astype(str) - ss["num_sessions"] = ss.groupby(["subject_fullname"])["subject_fullname"].rank( - method="first" - ) + ss["num_sessions"] = ss.groupby(["subject_fullname"])["subject_fullname"].rank(method="first") ss["num_sessions"] = ss["num_sessions"].astype(int) ss = ss.loc[ss["num_sessions"] <= MAX_SESSIONS_HISTORY, :] max_value_indices = ss.groupby("subject_fullname")["num_sessions"].idxmax() ss = ss.loc[max_value_indices] ss = ss.reset_index(drop=True) - ss["query"] = ( - "(subject_fullname='" - + ss["subject_fullname"] - + "' and session_date >= '" - + ss["session_date"] - + "')" - ) + ss["query"] = "(subject_fullname='" + ss["subject_fullname"] + "' and session_date >= '" + ss["session_date"] + "')" block_query = " OR ".join(ss["query"]) - sstable = acquisition.SessionStarted.proj( - "local_path_behavior_file", "session_location" - ) + sstable = acquisition.SessionStarted.proj("local_path_behavior_file", "session_location") stable = (acquisition.Session).proj(stimulusBank="stimulus_bank") - tstable = behavior.TowersSession.proj( - trialType="rewarded_side", choice="chosen_side", stimulusSet="stimulus_set" - ) + tstable = behavior.TowersSession.proj(trialType="rewarded_side", choice="chosen_side", stimulusSet="stimulus_set") tbtable = behavior.TowersBlock.proj( "first_trial", "n_trials", @@ -209,9 +185,7 @@ def write_subject_motor_position(day_schedule): all_subjects_schedule = "', '".join(day_schedule["subject_fullname"]) all_subjects_schedule = "subject_fullname in ('" + all_subjects_schedule + "')" - sp = pd.DataFrame( - (subject.HeadMotorPosition & all_subjects_schedule).fetch(as_dict=True) - ) + sp = pd.DataFrame((subject.HeadMotorPosition & all_subjects_schedule).fetch(as_dict=True)) file_write = pathlib.Path(nodb_virmen_backup_dir, SUBJECT_MOTOR_POSITION_FILENAME) sp.to_csv(file_write, index=False) @@ -257,20 +231,14 @@ def write_weighinig_gui_ss_file(): subject_data["today water"] = "" # Copy the source file to the destination, replacing if it exists - template_ss_file = pathlib.Path( - nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE - ) - ss_file = pathlib.Path( - nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME - ) + template_ss_file = pathlib.Path(nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME_TEMPLATE) + ss_file = pathlib.Path(nodb_virmen_backup_dir, WEIGHING_GUI_REPLACEMENT_SPREADHSHEET_FILENAME) shutil.copy2(template_ss_file, ss_file) book = openpyxl.load_workbook(ss_file) sheet = book["Sheet1"] - for r_idx, row in enumerate( - dataframe_to_rows(subject_data, index=False, header=False), start=3 - ): + for r_idx, row in enumerate(dataframe_to_rows(subject_data, index=False, header=False), start=3): for c_idx, value in enumerate(row, start=1): sheet.cell(row=r_idx, column=c_idx, value=value) diff --git a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py index d49e41e5..6a7da4fb 100755 --- a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py +++ b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py @@ -67,9 +67,7 @@ def setup_logging(): # File handler file_handler = logging.FileHandler(log_file, mode="a", encoding="utf-8") - formatter = logging.Formatter( - "%(asctime)s - %(levelname)s - %(message)s", datefmt="[%X]" - ) + formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s", datefmt="[%X]") console_handler.setFormatter(formatter) file_handler.setFormatter(formatter) @@ -89,12 +87,8 @@ def get_slack_webhook(): str or None: Webhook URL if found, None otherwise """ try: - slack_configuration_dictionary = { - "webhook_name": "rigs_issues_and_troubleshooting" - } - webhook_url = (lab.SlackWebhooks & slack_configuration_dictionary).fetch1( - "webhook_url" - ) + slack_configuration_dictionary = {"webhook_name": "rigs_issues_and_troubleshooting"} + webhook_url = (lab.SlackWebhooks & slack_configuration_dictionary).fetch1("webhook_url") return webhook_url except Exception as e: logger.warning(f"Failed to get Slack webhook: {e}") @@ -201,11 +195,7 @@ def check_overdue_maintenance(): # Determine which locations are active based on recent schedules (last 30 days) five_weeks_ago = date.today() - timedelta(weeks=5) - query = ( - scheduler.Schedule - & f'date >= "{five_weeks_ago}"' - & 'subject_fullname not like "test%"' - ) + query = scheduler.Schedule & f'date >= "{five_weeks_ago}"' & 'subject_fullname not like "test%"' locations = query.fetch("location") unique_locations = sorted(set(locations)) @@ -214,12 +204,8 @@ def check_overdue_maintenance(): maintenance_types = maintenance_fetch(as_dict=True) # Separate maintenance types by system type - rig_maintenance_types = [ - m for m in maintenance_types if m.get("number_of_lines") >= 0 - ] - hosting_maintenance_types = [ - m for m in maintenance_types if m.get("number_of_lines") < 0 - ] + rig_maintenance_types = [m for m in maintenance_types if m.get("number_of_lines") >= 0] + hosting_maintenance_types = [m for m in maintenance_types if m.get("number_of_lines") < 0] # Get all locations (rigs) rig_queries = [ @@ -267,9 +253,7 @@ def check_overdue_maintenance(): "maintenance_type": maintenance_type, } recent_fetch = recent_q.fetch - recent_maintenance = recent_fetch( - "maintenance_date", order_by="maintenance_date DESC", limit=1 - ) + recent_maintenance = recent_fetch("maintenance_date", order_by="maintenance_date DESC", limit=1) if len(recent_maintenance) == 0: # No maintenance record exists @@ -293,9 +277,7 @@ def check_overdue_maintenance(): if interval_days <= 0: # If the interval is 0 or negative, it is an as-needed maintenance type, # so we only log if there is no record - logger.info( - f" {maintenance_type:.<30} AS-NEEDED (record exists, no interval) ✅" - ) + logger.info(f" {maintenance_type:.<30} AS-NEEDED (record exists, no interval) ✅") elif days_since_last > interval_days: # Maintenance is overdue days_overdue = days_since_last - interval_days @@ -311,9 +293,7 @@ def check_overdue_maintenance(): "message": f"{maintenance_type} is {days_overdue} days overdue", } ) - logger.info( - f" {maintenance_type:.<30} OVERDUE by {days_overdue} days ❌" - ) + logger.info(f" {maintenance_type:.<30} OVERDUE by {days_overdue} days ❌") else: # Not yet overdue: check if within notification window if 0 < days_until_due <= notification_window: @@ -335,9 +315,7 @@ def check_overdue_maintenance(): f"{days_until_due} days (window {notification_window}) ⚠️" ) else: - logger.info( - f" {maintenance_type:.<30} OK ({days_until_due} days until due) ✅" - ) + logger.info(f" {maintenance_type:.<30} OK ({days_until_due} days until due) ✅") # Get all hosting VMs for certificate & secret updates hosting_queries = [ @@ -371,9 +349,7 @@ def check_overdue_maintenance(): "maintenance_type": maintenance_type, } recent_fetch = recent_q.fetch - recent_maintenance = recent_fetch( - "maintenance_date", order_by="maintenance_date DESC", limit=1 - ) + recent_maintenance = recent_fetch("maintenance_date", order_by="maintenance_date DESC", limit=1) if len(recent_maintenance) == 0: # No maintenance record exists @@ -408,9 +384,7 @@ def check_overdue_maintenance(): "message": f"{maintenance_type} is {days_overdue} days overdue", } ) - logger.info( - f" {maintenance_type:.<30} OVERDUE by {days_overdue} days ❌" - ) + logger.info(f" {maintenance_type:.<30} OVERDUE by {days_overdue} days ❌") else: # Not yet overdue: check if within notification window days_until_due = interval_days - days_since_last @@ -432,9 +406,7 @@ def check_overdue_maintenance(): f" {maintenance_type:.<30} DUE SOON in {days_until_due} days (window {notification_window}) ⚠️" ) else: - logger.info( - f" {maintenance_type:.<30} OK ({days_until_due} days until due) ✅" - ) + logger.info(f" {maintenance_type:.<30} OK ({days_until_due} days until due) ✅") # Return combined list; status distinguishes OVERDUE vs UPCOMING return overdue_items + upcoming_items @@ -463,18 +435,14 @@ def log_summary(overdue_items): if no_record_items: # Print a plain-text table for no-record items - logger.warning( - f"📋 RIGS WITH NO MAINTENANCE RECORDS ({len(no_record_items)} items)" - ) + logger.warning(f"📋 RIGS WITH NO MAINTENANCE RECORDS ({len(no_record_items)} items)") loc_w = 30 type_w = 40 logger.debug(f"{'.' * (loc_w + type_w + 5)}") logger.debug(f"{'Location':<{loc_w}} | {'Maintenance Type':<{type_w}}") logger.debug(f"{'.' * (loc_w + type_w + 5)}") for item in no_record_items: - logger.debug( - f"{item['location']:<{loc_w}} | {item['maintenance_type']:<{type_w}}" - ) + logger.debug(f"{item['location']:<{loc_w}} | {item['maintenance_type']:<{type_w}}") logger.debug(f"{'.' * (loc_w + type_w + 5)}") if overdue_items_list: @@ -507,9 +475,7 @@ def log_summary(overdue_items): logger.debug("=" * 60) -def dataframe_to_slack_table_block( - df: pd.DataFrame, truncate_cell: int = 200, use_markdown: bool = False -) -> dict: +def dataframe_to_slack_table_block(df: pd.DataFrame, truncate_cell: int = 200, use_markdown: bool = False) -> dict: """ Convert a DataFrame into one Slack table block. Docs: https://docs.slack.dev/reference/block-kit/blocks/table-block/ @@ -542,9 +508,7 @@ def mk_cell(val): return block -def dataframe_to_slack_table_blocks( - df: pd.DataFrame, chunk_size: int = 15, **kwargs -) -> list[dict]: +def dataframe_to_slack_table_blocks(df: pd.DataFrame, chunk_size: int = 15, **kwargs) -> list[dict]: """ Chunk a DataFrame into multiple Slack table blocks by rows. @@ -581,9 +545,7 @@ def main(): log_summary(overdue_items) records = pd.DataFrame(overdue_items) - records.columns = [ - " ".join(w.capitalize() for w in col.split("_")) for col in records.columns - ] + records.columns = [" ".join(w.capitalize() for w in col.split("_")) for col in records.columns] print(records) records = records[["Location", "Maintenance Type", "Status"]] diff --git a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py index b61ea9f0..80e27bd4 100644 --- a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py +++ b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py @@ -21,9 +21,7 @@ def get_schedule_query(): today = (datetime.date.today()).strftime("%Y-%m-%d") schedule_query = 'date >= "' + today + '" and date <= "' + tomorrow + '"' schedule_df = pd.DataFrame( - (scheduler.Schedule & schedule_query).fetch( - "date", "location", "subject_fullname", as_dict=True - ) + (scheduler.Schedule & schedule_query).fetch("date", "location", "subject_fullname", as_dict=True) ) return schedule_df @@ -33,23 +31,17 @@ def main_schedule_check_alert(): alert = 0 schedule_df = get_schedule_query() - tomorrow_schedule = schedule_df.loc[ - schedule_df["date"] == (datetime.date.today() + datetime.timedelta(days=1)), : - ] + tomorrow_schedule = schedule_df.loc[schedule_df["date"] == (datetime.date.today() + datetime.timedelta(days=1)), :] if tomorrow_schedule.shape[0] == 0: alert = 1 slack_json_message = slack_alert_empty_schedule() else: - schedule_df = schedule_df.groupby(["date", "location"]).agg( - {"subject_fullname": [("#subj", "count")]} - ) + schedule_df = schedule_df.groupby(["date", "location"]).agg({"subject_fullname": [("#subj", "count")]}) schedule_df.columns = schedule_df.columns.droplevel() schedule_df = schedule_df.reset_index() - todays_summary_df = schedule_df.loc[ - schedule_df["date"] == datetime.date.today(), ["location", "#subj"] - ].copy() + todays_summary_df = schedule_df.loc[schedule_df["date"] == datetime.date.today(), ["location", "#subj"]].copy() tomorrow_summary_df = schedule_df.loc[ schedule_df["date"] == (datetime.date.today() + datetime.timedelta(days=1)), ["location", "#subj"], @@ -63,9 +55,7 @@ def main_schedule_check_alert(): ) schedule_comp - schedule_comp["diff_subjects"] = ( - schedule_comp["#subj_tomorrow"] - schedule_comp["#subj_today"] - ) + schedule_comp["diff_subjects"] = schedule_comp["#subj_tomorrow"] - schedule_comp["#subj_today"] schedule_comp["rig_less_subjects"] = schedule_comp["diff_subjects"] < -2 subjects_today = schedule_comp["#subj_today"].sum() @@ -82,13 +72,9 @@ def main_schedule_check_alert(): if alert == 1: schedule_comp["location"] = "*" + schedule_comp["location"] + "*~" - schedule_comp = schedule_comp.drop( - ["diff_subjects", "rig_less_subjects"], axis=1 - ) + schedule_comp = schedule_comp.drop(["diff_subjects", "rig_less_subjects"], axis=1) schedule_comp_df_string = su.format_df_for_slack_message(schedule_comp) - slack_json_message = slack_alert_message_format_schedule( - schedule_comp_df_string - ) + slack_json_message = slack_alert_message_format_schedule(schedule_comp_df_string) if alert == 0: return @@ -121,9 +107,7 @@ def slack_alert_message_format_schedule(schedule_df_string): m2_1 = dict() m2_1["type"] = "mrkdwn" - m2_1["text"] = ( - "Significantly less subjects scheduled tomorrow\nSchedule per rig:" + "\n\n" - ) + m2_1["text"] = "Significantly less subjects scheduled tomorrow\nSchedule per rig:" + "\n\n" m2_1["text"] += schedule_df_string m2["text"] = m2_1 diff --git a/u19_pipeline/alert_system/tech_alert/tech_alert.py b/u19_pipeline/alert_system/tech_alert/tech_alert.py index 197b67c8..302735ab 100644 --- a/u19_pipeline/alert_system/tech_alert/tech_alert.py +++ b/u19_pipeline/alert_system/tech_alert/tech_alert.py @@ -24,13 +24,8 @@ def get_webhook_list(lab): # Get webhook lists slack_configuration_dictionary = {"slack_notification_channel": ["rigs_scheduling"]} webhooks_list = [] - query_slack_webhooks = [ - {"webhook_name": x} - for x in slack_configuration_dictionary["slack_notification_channel"] - ] - webhooks_list += ( - (lab.SlackWebhooks & query_slack_webhooks).fetch("webhook_url").tolist() - ) + query_slack_webhooks = [{"webhook_name": x} for x in slack_configuration_dictionary["slack_notification_channel"]] + webhooks_list += (lab.SlackWebhooks & query_slack_webhooks).fetch("webhook_url").tolist() return webhooks_list @@ -72,9 +67,7 @@ def slack_alert_message_format_tech_alert(schedule_data): ) else: # Ensure we don't send an empty section block to Slack (it will reject it) - m2_1["text"] = ( - "- No technician is scheduled for today. Please assign someone for coverage" - ) + m2_1["text"] = "- No technician is scheduled for today. Please assign someone for coverage" m2["text"] = m2_1 message["blocks"].append(m2) message["blocks"].append(msep) @@ -87,11 +80,7 @@ def slack_alert_message_format_tech_alert(schedule_data): # Today + next 4 days for day in range(1, 7): date_to_check = today + datetime.timedelta(days=day) - shifts_on_date = [ - shift - for shift in upcoming_shifts - if shift["date"] == date_to_check and not is_off(shift) - ] + shifts_on_date = [shift for shift in upcoming_shifts if shift["date"] == date_to_check and not is_off(shift)] if not shifts_on_date: alerts.append( @@ -112,18 +101,12 @@ def slack_alert_message_format_tech_alert(schedule_data): f"- Experimenters, {date_to_check.strftime('%A, %B %d')} will have no training due to lab cleanup." ) if shift["duties"] in ["Watering Only", "Off"]: - alerts.append( - "Experimenters, please make arrangements if you need to train." - ) + alerts.append("Experimenters, please make arrangements if you need to train.") date_to_check = datetime.date.today() + datetime.timedelta(days=day) - shifts_on_date = [ - shift for shift in upcoming_shifts if shift["date"] == date_to_check - ] + shifts_on_date = [shift for shift in upcoming_shifts if shift["date"] == date_to_check] if not shifts_on_date: - alerts.append( - f"No one is scheduled for {date_to_check.strftime('%A, %B %d')}." - ) + alerts.append(f"No one is scheduled for {date_to_check.strftime('%A, %B %d')}.") else: for shift in shifts_on_date: if shift["duties"] in ["Watering Only"]: @@ -137,9 +120,7 @@ def slack_alert_message_format_tech_alert(schedule_data): m3["type"] = "section" m3_1 = dict() m3_1["type"] = "mrkdwn" - m3_1["text"] = ( - f"*:rotating_light: Upcoming Shifts: :rotating_light:*\n{alert_text}" - ) + m3_1["text"] = f"*:rotating_light: Upcoming Shifts: :rotating_light:*\n{alert_text}" m3["text"] = m3_1 message["blocks"].append(m3) @@ -300,14 +281,10 @@ def fetch_and_parse_icalevents(weburl: str): # Consolidate lab clean-up events lab_clean_up_events = [ - event - for event in vr_events - if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event for event in vr_events if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] non_lab_clean_up_events = [ - event - for event in vr_events - if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event for event in vr_events if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] if lab_clean_up_events: unique_days = sorted(set(event.start.date() for event in lab_clean_up_events)) @@ -355,6 +332,4 @@ def fetch_and_parse_icalevents(weburl: str): if sys.argv[1].lower() == "test": test_slack_alert_message_format_tech_alert() else: - raise ValueError( - f"Unrecognized argument {sys.argv[1]} -- use 'test' to run in test mode." - ) + raise ValueError(f"Unrecognized argument {sys.argv[1]} -- use 'test' to run in test mode.") diff --git a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py index 850782ab..bd552082 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py +++ b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py @@ -23,9 +23,7 @@ } # Query from file -QUERY_FILE = pathlib.Path( - pathlib.Path(__file__).resolve().parent, "get_subject_data.sql" -).as_posix() +QUERY_FILE = pathlib.Path(pathlib.Path(__file__).resolve().parent, "get_subject_data.sql").as_posix() def get_subject_data(): @@ -55,10 +53,7 @@ def get_subject_data(): subject_data = subject_data.loc[ ( - ( - (subject_data["schedule_today"] != "Nothing") - & (subject_data["subject_status"] == "InExperiments") - ) + ((subject_data["schedule_today"] != "Nothing") & (subject_data["subject_status"] == "InExperiments")) | (subject_data["subject_status"] == "WaterRestrictionOnly") ), :, @@ -67,28 +62,18 @@ def get_subject_data(): subject_data = subject_data.reset_index(drop=True) with pd.option_context("future.no_silent_downcasting", True): - subject_data["earned"] = ( - subject_data["earned"].fillna(0).infer_objects(copy=False) - ) - subject_data["received"] = ( - subject_data["received"].fillna(0).infer_objects(copy=False) - ) - subject_data["supplement"] = ( - subject_data["supplement"].fillna(0).infer_objects(copy=False) - ) + subject_data["earned"] = subject_data["earned"].fillna(0).infer_objects(copy=False) + subject_data["received"] = subject_data["received"].fillna(0).infer_objects(copy=False) + subject_data["supplement"] = subject_data["supplement"].fillna(0).infer_objects(copy=False) subject_data["prescribed_extra_supplement_amount"] = ( - subject_data["prescribed_extra_supplement_amount"] - .fillna(0) - .infer_objects(copy=False) + subject_data["prescribed_extra_supplement_amount"].fillna(0).infer_objects(copy=False) ) # Calculated fields subject_data["already_water"] = subject_data["supplement"] > 0 subject_data["upper_cage"] = subject_data["cage"].str.upper() subject_data["total_water_received"] = subject_data["received"] - subject_data.loc[ - subject_data["total_water_received"].isnull(), "total_water_received" - ] = 0 + subject_data.loc[subject_data["total_water_received"].isnull(), "total_water_received"] = 0 subject_data["need_supplement"] = 0 subject_data["need_supplement"] = ( @@ -98,20 +83,13 @@ def get_subject_data(): subject_data.loc[subject_data["already_received"].isnull(), "already_received"] = 0 subject_data["need_extra_water_now"] = 0 subject_data.loc[ - ( - (subject_data["prescribed_extra_supplement_amount"] > 0) - & (subject_data["already_received"] == 0) - ), + ((subject_data["prescribed_extra_supplement_amount"] > 0) & (subject_data["already_received"] == 0)), "need_extra_water_now", ] = 1 subject_data["water_status"] = "Already Watered" - subject_data.loc[subject_data["need_supplement"] == 1, "water_status"] = ( - "Need Supplement" - ) - subject_data.loc[subject_data["need_extra_water_now"] == 1, "water_status"] = ( - "Need Extra Supplement" - ) + subject_data.loc[subject_data["need_supplement"] == 1, "water_status"] = "Need Supplement" + subject_data.loc[subject_data["need_extra_water_now"] == 1, "water_status"] = "Need Extra Supplement" subject_data["need_water"] = 0 subject_data.loc[ @@ -120,9 +98,7 @@ def get_subject_data(): ] = 1 subject_data["current_need_water"] = subject_data["suggested_water"] - subject_data.loc[ - subject_data["need_extra_water_now"] == 1, "current_need_water" - ] = subject_data.loc[ + subject_data.loc[subject_data["need_extra_water_now"] == 1, "current_need_water"] = subject_data.loc[ subject_data["need_extra_water_now"] == 1, "prescribed_extra_supplement_amount" ] @@ -131,23 +107,15 @@ def get_subject_data(): subject_data.loc[~subject_data["session_location"].isnull(), "training_status"] = 2 subject_data["training_status_label"] = "Not scheduled" - subject_data.loc[subject_data["training_status"] == 1, "training_status_label"] = ( - "Scheduled" - ) - subject_data.loc[subject_data["training_status"] == 2, "training_status_label"] = ( - "Training Started" - ) + subject_data.loc[subject_data["training_status"] == 1, "training_status_label"] = "Scheduled" + subject_data.loc[subject_data["training_status"] == 2, "training_status_label"] = "Training Started" # Calculated fields subject_data["already_weighted"] = ~subject_data["weight"].isnull() - subject_data["need_weight"] = ( - ~subject_data["already_weighted"] | subject_data["need_reweight"] - ) + subject_data["need_weight"] = ~subject_data["already_weighted"] | subject_data["need_reweight"] subject_data["weight_status"] = "Already Weighted" - subject_data.loc[subject_data["already_weighted"] == 0, "weight_status"] = ( - "Need Weight" - ) + subject_data.loc[subject_data["already_weighted"] == 0, "weight_status"] = "Need Weight" subject_data.loc[subject_data["need_reweight"] == 1, "weight_status"] = "REWEIGHT" return subject_data @@ -185,14 +153,10 @@ def fetch_and_parse_icalevents(weburl: str): # Consolidate lab clean-up events lab_clean_up_events = [ - event - for event in vr_events - if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event for event in vr_events if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] non_lab_clean_up_events = [ - event - for event in vr_events - if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event for event in vr_events if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] if lab_clean_up_events: consolidated_event = { @@ -257,13 +221,9 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: * lab.User().proj(lab_manager="user_id", manager_slack="slack") ).fetch("manager_slack", as_dict=True) print(technician_manager_slack) - technician_manager_slack = [ - item["manager_slack"] for item in technician_manager_slack - ] + technician_manager_slack = [item["manager_slack"] for item in technician_manager_slack] - people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch( - format="frame" - ) + people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch(format="frame") load_dotenv() @@ -291,9 +251,7 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: tech_slack.append(tech_uid) if tech_slack: - tech_slack.append( - technician_manager_slack[0] - ) # Fallback to default technician manager slack + tech_slack.append(technician_manager_slack[0]) # Fallback to default technician manager slack coowner_dataframe: pd.DataFrame = ( ( @@ -314,11 +272,7 @@ def resolve_responsible_slack(row): lab_name = row.get("lab") token = str(schedule_today).strip() if schedule_today is not None else None - use_coowners = ( - (token is None) - or (token.lower() == "transport") - or (token.lower() == "nothing") - ) + use_coowners = (token is None) or (token.lower() == "transport") or (token.lower() == "nothing") slack_tags = [] @@ -335,18 +289,14 @@ def resolve_responsible_slack(row): # print(temp) slack_tags += temp["manager_slack"].tolist() slack_tags += temp["user_slack"].tolist() - slack_tags += coowner_dataframe.query( - f"subject_fullname == '{row['subject_fullname']}'" - )["slack"].tolist() + slack_tags += coowner_dataframe.query(f"subject_fullname == '{row['subject_fullname']}'")["slack"].tolist() slack_tags = set(slack_tags) # Remove duplicates return " ".join(f"<@{item}>" for item in slack_tags) subject_data = subject_data.copy() - subject_data["responsible_slack_tags"] = subject_data.apply( - resolve_responsible_slack, axis=1 - ) + subject_data["responsible_slack_tags"] = subject_data.apply(resolve_responsible_slack, axis=1) return subject_data @@ -369,9 +319,7 @@ def find_unreturned_subjects() -> pd.DataFrame: .fetch(format="frame") ) - unreturned_subjects: pd.DataFrame = local_transport_data[ - local_transport_data["transport_in_datetime"].isnull() - ] + unreturned_subjects: pd.DataFrame = local_transport_data[local_transport_data["transport_in_datetime"].isnull()] return unreturned_subjects @@ -394,9 +342,7 @@ def slack_alert_message_format_weight_water( ) ) - slack_handles: list[str] = fetch_slack_handles_for_lab_managers_by_subject( - notifiable_subjects - ) + slack_handles: list[str] = fetch_slack_handles_for_lab_managers_by_subject(notifiable_subjects) lab_manager_text = "\n\n" if len(slack_handles) >= 1: lab_manager_text += "Lab Manager" @@ -406,9 +352,7 @@ def slack_alert_message_format_weight_water( slack_handles_formatted = ", ".join("<@" + handle + ">" for handle in slack_handles) if slack_handles_formatted: lab_manager_text += ( - " " - + slack_handles_formatted - + ", please be advised that your labs' subjects are listed below." + " " + slack_handles_formatted + ", please be advised that your labs' subjects are listed below." ) msep = dict() @@ -523,9 +467,7 @@ def slack_alert_message_format_weight_water( # Try to get responsible slack tags for this subject tags = "" try: - tags = missing_transport.iloc[i].get( - "responsible_slack_tags", "" - ) # may be string or iterable + tags = missing_transport.iloc[i].get("responsible_slack_tags", "") # may be string or iterable except Exception: tags = "" # Normalize tags to a string @@ -582,9 +524,7 @@ def _estimate_section_size(text: str, text_type: str = "mrkdwn") -> int: return estimate_size({"blocks": [block]}) -def split_text_to_fit( - text: str, max_payload_size: int, text_type: str = "mrkdwn" -) -> list[str]: +def split_text_to_fit(text: str, max_payload_size: int, text_type: str = "mrkdwn") -> list[str]: """Split text by lines so each chunk fits in max_payload_size when JSON-encoded. Uses actual JSON size estimation rather than raw character count. @@ -644,9 +584,7 @@ def _split_large_block(block: dict, max_size: int) -> list[dict]: return [block] -def send_slack_blocks_safely( - webhook: str, blocks: list[dict], max_size: int = SLACK_MAX_CHARS -): +def send_slack_blocks_safely(webhook: str, blocks: list[dict], max_size: int = SLACK_MAX_CHARS): """Send blocks together when possible, splitting only when necessary. Strategy: @@ -718,9 +656,7 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int): smaller = _split_large_block(block, max_size) for small_block in smaller: try: - su.send_slack_notification( - webhook, {"blocks": [small_block]} - ) + su.send_slack_notification(webhook, {"blocks": [small_block]}) time.sleep(0.5) except HTTPError: print("Failed to send block even after splitting") @@ -746,17 +682,13 @@ def main_water_weigh_alert(): # print(subject_data.columns) - subject_data = subject_data.loc[ - ~subject_data["subject_fullname"].str.contains("test"), : - ] + subject_data = subject_data.loc[~subject_data["subject_fullname"].str.contains("test"), :] subjects_not_watered = subject_data.loc[ subject_data["current_need_water"] > 0, ["subject_fullname", "current_need_water", "responsible_slack_tags"], ] subjects_not_watered = subjects_not_watered.reset_index(drop=True) - subjects_not_watered["current_need_water"] = subjects_not_watered[ - "current_need_water" - ].apply(lambda x: f"{x:.1f}") + subjects_not_watered["current_need_water"] = subjects_not_watered["current_need_water"].apply(lambda x: f"{x:.1f}") # subjects_not_watered = subjects_not_watered.head() subjects_not_weighted = subject_data.loc[ @@ -767,8 +699,7 @@ def main_water_weigh_alert(): # subjects_not_weighted = subjects_not_weighted.head() subjects_not_trained = subject_data.loc[ - (subject_data["training_status"] == 1) - & (subject_data["schedule_today"].str.lower() != "water"), + (subject_data["training_status"] == 1) & (subject_data["schedule_today"].str.lower() != "water"), ["subject_fullname", "scheduled_rig"], ] subjects_not_trained = subjects_not_trained.reset_index(drop=True) diff --git a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py index 850782ab..bd552082 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py +++ b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py @@ -23,9 +23,7 @@ } # Query from file -QUERY_FILE = pathlib.Path( - pathlib.Path(__file__).resolve().parent, "get_subject_data.sql" -).as_posix() +QUERY_FILE = pathlib.Path(pathlib.Path(__file__).resolve().parent, "get_subject_data.sql").as_posix() def get_subject_data(): @@ -55,10 +53,7 @@ def get_subject_data(): subject_data = subject_data.loc[ ( - ( - (subject_data["schedule_today"] != "Nothing") - & (subject_data["subject_status"] == "InExperiments") - ) + ((subject_data["schedule_today"] != "Nothing") & (subject_data["subject_status"] == "InExperiments")) | (subject_data["subject_status"] == "WaterRestrictionOnly") ), :, @@ -67,28 +62,18 @@ def get_subject_data(): subject_data = subject_data.reset_index(drop=True) with pd.option_context("future.no_silent_downcasting", True): - subject_data["earned"] = ( - subject_data["earned"].fillna(0).infer_objects(copy=False) - ) - subject_data["received"] = ( - subject_data["received"].fillna(0).infer_objects(copy=False) - ) - subject_data["supplement"] = ( - subject_data["supplement"].fillna(0).infer_objects(copy=False) - ) + subject_data["earned"] = subject_data["earned"].fillna(0).infer_objects(copy=False) + subject_data["received"] = subject_data["received"].fillna(0).infer_objects(copy=False) + subject_data["supplement"] = subject_data["supplement"].fillna(0).infer_objects(copy=False) subject_data["prescribed_extra_supplement_amount"] = ( - subject_data["prescribed_extra_supplement_amount"] - .fillna(0) - .infer_objects(copy=False) + subject_data["prescribed_extra_supplement_amount"].fillna(0).infer_objects(copy=False) ) # Calculated fields subject_data["already_water"] = subject_data["supplement"] > 0 subject_data["upper_cage"] = subject_data["cage"].str.upper() subject_data["total_water_received"] = subject_data["received"] - subject_data.loc[ - subject_data["total_water_received"].isnull(), "total_water_received" - ] = 0 + subject_data.loc[subject_data["total_water_received"].isnull(), "total_water_received"] = 0 subject_data["need_supplement"] = 0 subject_data["need_supplement"] = ( @@ -98,20 +83,13 @@ def get_subject_data(): subject_data.loc[subject_data["already_received"].isnull(), "already_received"] = 0 subject_data["need_extra_water_now"] = 0 subject_data.loc[ - ( - (subject_data["prescribed_extra_supplement_amount"] > 0) - & (subject_data["already_received"] == 0) - ), + ((subject_data["prescribed_extra_supplement_amount"] > 0) & (subject_data["already_received"] == 0)), "need_extra_water_now", ] = 1 subject_data["water_status"] = "Already Watered" - subject_data.loc[subject_data["need_supplement"] == 1, "water_status"] = ( - "Need Supplement" - ) - subject_data.loc[subject_data["need_extra_water_now"] == 1, "water_status"] = ( - "Need Extra Supplement" - ) + subject_data.loc[subject_data["need_supplement"] == 1, "water_status"] = "Need Supplement" + subject_data.loc[subject_data["need_extra_water_now"] == 1, "water_status"] = "Need Extra Supplement" subject_data["need_water"] = 0 subject_data.loc[ @@ -120,9 +98,7 @@ def get_subject_data(): ] = 1 subject_data["current_need_water"] = subject_data["suggested_water"] - subject_data.loc[ - subject_data["need_extra_water_now"] == 1, "current_need_water" - ] = subject_data.loc[ + subject_data.loc[subject_data["need_extra_water_now"] == 1, "current_need_water"] = subject_data.loc[ subject_data["need_extra_water_now"] == 1, "prescribed_extra_supplement_amount" ] @@ -131,23 +107,15 @@ def get_subject_data(): subject_data.loc[~subject_data["session_location"].isnull(), "training_status"] = 2 subject_data["training_status_label"] = "Not scheduled" - subject_data.loc[subject_data["training_status"] == 1, "training_status_label"] = ( - "Scheduled" - ) - subject_data.loc[subject_data["training_status"] == 2, "training_status_label"] = ( - "Training Started" - ) + subject_data.loc[subject_data["training_status"] == 1, "training_status_label"] = "Scheduled" + subject_data.loc[subject_data["training_status"] == 2, "training_status_label"] = "Training Started" # Calculated fields subject_data["already_weighted"] = ~subject_data["weight"].isnull() - subject_data["need_weight"] = ( - ~subject_data["already_weighted"] | subject_data["need_reweight"] - ) + subject_data["need_weight"] = ~subject_data["already_weighted"] | subject_data["need_reweight"] subject_data["weight_status"] = "Already Weighted" - subject_data.loc[subject_data["already_weighted"] == 0, "weight_status"] = ( - "Need Weight" - ) + subject_data.loc[subject_data["already_weighted"] == 0, "weight_status"] = "Need Weight" subject_data.loc[subject_data["need_reweight"] == 1, "weight_status"] = "REWEIGHT" return subject_data @@ -185,14 +153,10 @@ def fetch_and_parse_icalevents(weburl: str): # Consolidate lab clean-up events lab_clean_up_events = [ - event - for event in vr_events - if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event for event in vr_events if re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] non_lab_clean_up_events = [ - event - for event in vr_events - if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) + event for event in vr_events if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] if lab_clean_up_events: consolidated_event = { @@ -257,13 +221,9 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: * lab.User().proj(lab_manager="user_id", manager_slack="slack") ).fetch("manager_slack", as_dict=True) print(technician_manager_slack) - technician_manager_slack = [ - item["manager_slack"] for item in technician_manager_slack - ] + technician_manager_slack = [item["manager_slack"] for item in technician_manager_slack] - people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch( - format="frame" - ) + people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch(format="frame") load_dotenv() @@ -291,9 +251,7 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: tech_slack.append(tech_uid) if tech_slack: - tech_slack.append( - technician_manager_slack[0] - ) # Fallback to default technician manager slack + tech_slack.append(technician_manager_slack[0]) # Fallback to default technician manager slack coowner_dataframe: pd.DataFrame = ( ( @@ -314,11 +272,7 @@ def resolve_responsible_slack(row): lab_name = row.get("lab") token = str(schedule_today).strip() if schedule_today is not None else None - use_coowners = ( - (token is None) - or (token.lower() == "transport") - or (token.lower() == "nothing") - ) + use_coowners = (token is None) or (token.lower() == "transport") or (token.lower() == "nothing") slack_tags = [] @@ -335,18 +289,14 @@ def resolve_responsible_slack(row): # print(temp) slack_tags += temp["manager_slack"].tolist() slack_tags += temp["user_slack"].tolist() - slack_tags += coowner_dataframe.query( - f"subject_fullname == '{row['subject_fullname']}'" - )["slack"].tolist() + slack_tags += coowner_dataframe.query(f"subject_fullname == '{row['subject_fullname']}'")["slack"].tolist() slack_tags = set(slack_tags) # Remove duplicates return " ".join(f"<@{item}>" for item in slack_tags) subject_data = subject_data.copy() - subject_data["responsible_slack_tags"] = subject_data.apply( - resolve_responsible_slack, axis=1 - ) + subject_data["responsible_slack_tags"] = subject_data.apply(resolve_responsible_slack, axis=1) return subject_data @@ -369,9 +319,7 @@ def find_unreturned_subjects() -> pd.DataFrame: .fetch(format="frame") ) - unreturned_subjects: pd.DataFrame = local_transport_data[ - local_transport_data["transport_in_datetime"].isnull() - ] + unreturned_subjects: pd.DataFrame = local_transport_data[local_transport_data["transport_in_datetime"].isnull()] return unreturned_subjects @@ -394,9 +342,7 @@ def slack_alert_message_format_weight_water( ) ) - slack_handles: list[str] = fetch_slack_handles_for_lab_managers_by_subject( - notifiable_subjects - ) + slack_handles: list[str] = fetch_slack_handles_for_lab_managers_by_subject(notifiable_subjects) lab_manager_text = "\n\n" if len(slack_handles) >= 1: lab_manager_text += "Lab Manager" @@ -406,9 +352,7 @@ def slack_alert_message_format_weight_water( slack_handles_formatted = ", ".join("<@" + handle + ">" for handle in slack_handles) if slack_handles_formatted: lab_manager_text += ( - " " - + slack_handles_formatted - + ", please be advised that your labs' subjects are listed below." + " " + slack_handles_formatted + ", please be advised that your labs' subjects are listed below." ) msep = dict() @@ -523,9 +467,7 @@ def slack_alert_message_format_weight_water( # Try to get responsible slack tags for this subject tags = "" try: - tags = missing_transport.iloc[i].get( - "responsible_slack_tags", "" - ) # may be string or iterable + tags = missing_transport.iloc[i].get("responsible_slack_tags", "") # may be string or iterable except Exception: tags = "" # Normalize tags to a string @@ -582,9 +524,7 @@ def _estimate_section_size(text: str, text_type: str = "mrkdwn") -> int: return estimate_size({"blocks": [block]}) -def split_text_to_fit( - text: str, max_payload_size: int, text_type: str = "mrkdwn" -) -> list[str]: +def split_text_to_fit(text: str, max_payload_size: int, text_type: str = "mrkdwn") -> list[str]: """Split text by lines so each chunk fits in max_payload_size when JSON-encoded. Uses actual JSON size estimation rather than raw character count. @@ -644,9 +584,7 @@ def _split_large_block(block: dict, max_size: int) -> list[dict]: return [block] -def send_slack_blocks_safely( - webhook: str, blocks: list[dict], max_size: int = SLACK_MAX_CHARS -): +def send_slack_blocks_safely(webhook: str, blocks: list[dict], max_size: int = SLACK_MAX_CHARS): """Send blocks together when possible, splitting only when necessary. Strategy: @@ -718,9 +656,7 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int): smaller = _split_large_block(block, max_size) for small_block in smaller: try: - su.send_slack_notification( - webhook, {"blocks": [small_block]} - ) + su.send_slack_notification(webhook, {"blocks": [small_block]}) time.sleep(0.5) except HTTPError: print("Failed to send block even after splitting") @@ -746,17 +682,13 @@ def main_water_weigh_alert(): # print(subject_data.columns) - subject_data = subject_data.loc[ - ~subject_data["subject_fullname"].str.contains("test"), : - ] + subject_data = subject_data.loc[~subject_data["subject_fullname"].str.contains("test"), :] subjects_not_watered = subject_data.loc[ subject_data["current_need_water"] > 0, ["subject_fullname", "current_need_water", "responsible_slack_tags"], ] subjects_not_watered = subjects_not_watered.reset_index(drop=True) - subjects_not_watered["current_need_water"] = subjects_not_watered[ - "current_need_water" - ].apply(lambda x: f"{x:.1f}") + subjects_not_watered["current_need_water"] = subjects_not_watered["current_need_water"].apply(lambda x: f"{x:.1f}") # subjects_not_watered = subjects_not_watered.head() subjects_not_weighted = subject_data.loc[ @@ -767,8 +699,7 @@ def main_water_weigh_alert(): # subjects_not_weighted = subjects_not_weighted.head() subjects_not_trained = subject_data.loc[ - (subject_data["training_status"] == 1) - & (subject_data["schedule_today"].str.lower() != "water"), + (subject_data["training_status"] == 1) & (subject_data["schedule_today"].str.lower() != "water"), ["subject_fullname", "scheduled_rig"], ] subjects_not_trained = subjects_not_trained.reset_index(drop=True) diff --git a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py index 8e8c831b..060e0342 100644 --- a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py +++ b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py @@ -17,12 +17,12 @@ # For tiger endpoint public_key_location = "~/.ssh/id_ed25519.pub" -default_user = ( - "u19prod" # This will change to our automatic client for globus transfers -) +default_user = "u19prod" # This will change to our automatic client for globus transfers tiger_gpu_host = "della.princeton.edu" # tiger_ep_dir = 'a9df83d2-42f0-11e6-80cf-22000b1701d1' # tigress ep -tiger_ep_dir = "8e1bbdfe-d234-4a7c-93a5-86086a249918" # Endpoint of Della's /scratch/gpfs/. Our directory is ./BRAINCOGS/ +tiger_ep_dir = ( + "8e1bbdfe-d234-4a7c-93a5-86086a249918" # Endpoint of Della's /scratch/gpfs/. Our directory is ./BRAINCOGS/ +) tiger_home_dir_globus = "/BRAINCOGS/Data/" @@ -82,8 +82,7 @@ "slurm_files_dir": tiger_home_dir + "/SlurmFiles", "params_files_dir": tiger_home_dir + "/ParameterFiles", "chanmap_files_dir": tiger_home_dir + "/ChanMapFiles", - "electrophysiology_process_dir": tiger_home_dir - + "/electrophysiology_processing", + "electrophysiology_process_dir": tiger_home_dir + "/electrophysiology_processing", "imaging_process_dir": tiger_home_dir + "/imaging_processing", "log_files_dir": tiger_home_dir + "/OutputLog", "error_files_dir": tiger_home_dir + "/ErrorLog", @@ -102,8 +101,7 @@ "slurm_files_dir": spock_home_dir + "/SlurmFiles", "params_files_dir": spock_home_dir + "/ParameterFiles", "chanmap_files_dir": spock_home_dir + "/ChanMapFiles", - "electrophysiology_process_dir": spock_home_dir - + "/electrophysiology_processing", + "electrophysiology_process_dir": spock_home_dir + "/electrophysiology_processing", "imaging_process_dir": spock_home_dir + "/imaging_processing", "log_files_dir": spock_home_dir + "/u19_pipeline/automatic_job/OutputLog", "error_files_dir": spock_home_dir + "/u19_pipeline/automatic_job/ErrorLog", @@ -149,9 +147,7 @@ def cp_file_transfer(source, dest): return transfer_status -def request_globus_transfer( - job_id_str, source_ep, dest_ep, source_filepath, dest_filepath -): +def request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, dest_filepath): source_fullpath = source_ep + ":" + source_filepath dest_fullpath = dest_ep + ":" + dest_filepath @@ -216,16 +212,10 @@ def globus_transfer_to_tiger(job_id, raw_rel_path, modality): source_ep = pni_ep_id dest_ep = tiger_ep_dir - source_filepath = pathlib.Path( - cluster_vars["spock"]["root_data_dir_globus"], modality, raw_rel_path - ).as_posix() - dest_filepath = pathlib.Path( - cluster_vars["tiger"]["root_data_dir_globus"], modality, raw_rel_path - ).as_posix() + source_filepath = pathlib.Path(cluster_vars["spock"]["root_data_dir_globus"], modality, raw_rel_path).as_posix() + dest_filepath = pathlib.Path(cluster_vars["tiger"]["root_data_dir_globus"], modality, raw_rel_path).as_posix() - transfer_request = request_globus_transfer( - job_id_str, source_ep, dest_ep, source_filepath, dest_filepath - ) + transfer_request = request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, dest_filepath) return transfer_request @@ -243,9 +233,7 @@ def globus_transfer_to_pni(job_id, processed_rel_path, modality): cluster_vars["tiger"]["processed_data_dir_globus"], modality, processed_rel_path ).as_posix() - transfer_request = request_globus_transfer( - job_id_str, source_ep, dest_ep, source_filepath, dest_filepath - ) + transfer_request = request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, dest_filepath) return transfer_request @@ -264,9 +252,7 @@ def translate_globus_output(stdout_process): return d1 -def transfer_log_file( - recording_process_id, program_selection_params, user_host, log_type="ERROR" -): +def transfer_log_file(recording_process_id, program_selection_params, user_host, log_type="ERROR"): """ Transfer and send parameter files for processing """ @@ -332,17 +318,11 @@ def check_directory_exists_cluster(directory, cluster, modality, type_dir="raw") this_cluster_vars = get_cluster_vars(cluster) if type_dir == "raw": - final_directory = pathlib.Path( - cluster_vars[cluster]["root_data_dir"], modality, directory - ).as_posix() + final_directory = pathlib.Path(cluster_vars[cluster]["root_data_dir"], modality, directory).as_posix() else: - final_directory = pathlib.Path( - cluster_vars[cluster]["processed_data_dir"], modality, directory - ).as_posix() + final_directory = pathlib.Path(cluster_vars[cluster]["processed_data_dir"], modality, directory).as_posix() - base_command = ( - "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " " - ) + base_command = "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " " command = "'if [ -d " + final_directory + " ]; " post_command = """then @@ -366,15 +346,7 @@ def delete_directory_cluster(directory, cluster): this_cluster_vars = get_cluster_vars(cluster) - command = ( - "ssh " - + this_cluster_vars["user"] - + "@" - + this_cluster_vars["hostname"] - + " 'rm -R " - + directory - + " '" - ) + command = "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " 'rm -R " + directory + " '" p = subprocess.run(command, shell=True) output = p.returncode @@ -391,9 +363,7 @@ def delete_directory_tiger_globus(modality, raw_rel_path): """ source_ep = tiger_ep_dir - source_filepath = pathlib.Path( - cluster_vars["tiger"]["root_data_dir_globus"], modality, raw_rel_path - ).as_posix() + source_filepath = pathlib.Path(cluster_vars["tiger"]["root_data_dir_globus"], modality, raw_rel_path).as_posix() source_fullpath = source_ep + ":" + source_filepath @@ -411,9 +381,7 @@ def delete_empty_data_directory_cluster(cluster, type="raw"): max_deletion = 10 this_cluster_vars = get_cluster_vars(cluster) - base_command = ( - "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " " - ) + base_command = "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " " # Check base directory to delete if type == "raw": diff --git a/u19_pipeline/automatic_job/create_missing_lfps_script.py b/u19_pipeline/automatic_job/create_missing_lfps_script.py index 01c1a7c6..062745b6 100644 --- a/u19_pipeline/automatic_job/create_missing_lfps_script.py +++ b/u19_pipeline/automatic_job/create_missing_lfps_script.py @@ -19,9 +19,7 @@ def main(recording_id): as_dict=True, ) # Create lfp trace if needed (neuropixel 2.0 probes) - recording_directory = (recording.Recording & rec_query).fetch1( - "recording_directory" - ) + recording_directory = (recording.Recording & rec_query).fetch1("recording_directory") recording_directory = pathlib.Path( dj.config["custom"]["ephys_root_data_dir"][0], recording_directory ).parent.as_posix() @@ -30,9 +28,7 @@ def main(recording_id): dj.config["custom"]["ephys_root_data_dir"][0], i["recording_process_pre_path"], ).as_posix() - ephys_pipeline.create_lfp_trace( - config.catgt_script, recording_directory, probe_dir - ) + ephys_pipeline.create_lfp_trace(config.catgt_script, recording_directory, probe_dir) if __name__ == "__main__": diff --git a/u19_pipeline/automatic_job/ephys_element_ingest.py b/u19_pipeline/automatic_job/ephys_element_ingest.py index c85d496e..4b102d2d 100644 --- a/u19_pipeline/automatic_job/ephys_element_ingest.py +++ b/u19_pipeline/automatic_job/ephys_element_ingest.py @@ -37,9 +37,7 @@ def process_session(sess_key): :param scan_key: a `KEY` of `acquisition.Session` """ - sess_dir = pathlib.Path( - find_full_path(get_ephys_root_data_dir(), get_session_directory(sess_key)) - ) + sess_dir = pathlib.Path(find_full_path(get_ephys_root_data_dir(), get_session_directory(sess_key))) ephys_meta_filepaths = [fp for fp in sess_dir.rglob("*.ap.meta")] print("ephys_meta_filepaths", ephys_meta_filepaths) @@ -57,10 +55,7 @@ def process_session(sess_key): "probe_type": spikeglx_meta.probe_model, "probe": spikeglx_meta.probe_SN, } - if ( - probe_key["probe"] not in [p["probe"] for p in probe_list] - and probe_key not in probe_element.Probe() - ): + if probe_key["probe"] not in [p["probe"] for p in probe_list] and probe_key not in probe_element.Probe(): probe_list.append(probe_key) probe_dir = meta_filepath.parent diff --git a/u19_pipeline/automatic_job/ephys_element_populate.py b/u19_pipeline/automatic_job/ephys_element_populate.py index ae2e2318..432a0391 100644 --- a/u19_pipeline/automatic_job/ephys_element_populate.py +++ b/u19_pipeline/automatic_job/ephys_element_populate.py @@ -3,9 +3,7 @@ from u19_pipeline.ephys_pipeline import ephys_element, probe_element -def populate_element_data( - job_id, display_progress=True, reserve_jobs=False, suppress_errors=False -): +def populate_element_data(job_id, display_progress=True, reserve_jobs=False, suppress_errors=False): populate_settings = { "display_progress": display_progress, @@ -14,28 +12,22 @@ def populate_element_data( } process_key = ( - recording_process.Processing * recording.Recording - & dict(recording_modality="electrophysiology", job_id=job_id) + recording_process.Processing * recording.Recording & dict(recording_modality="electrophysiology", job_id=job_id) ).fetch1("KEY") fragment_number, recording_process_pre_path, recording_process_post_path = ( recording_process.Processing & process_key - ).fetch1( - "fragment_number", "recording_process_pre_path", "recording_process_post_path" - ) + ).fetch1("fragment_number", "recording_process_pre_path", "recording_process_post_path") - precluster_param_steps_id, paramset_idx = ( - recording_process.Processing.EphysParams & process_key - ).fetch1("precluster_param_steps_id", "paramset_idx") + precluster_param_steps_id, paramset_idx = (recording_process.Processing.EphysParams & process_key).fetch1( + "precluster_param_steps_id", "paramset_idx" + ) precluster_paramsets = ( - ephys_element.PreClusterParamSteps.Step() - & dict(precluster_param_steps_id=precluster_param_steps_id) + ephys_element.PreClusterParamSteps.Step() & dict(precluster_param_steps_id=precluster_param_steps_id) ).fetch("paramset_idx") - clustering_method = ( - ephys_element.ClusteringParamSet & dict(paramset_idx=paramset_idx) - ).fetch1("clustering_method") + clustering_method = (ephys_element.ClusteringParamSet & dict(paramset_idx=paramset_idx)).fetch1("clustering_method") if len(precluster_paramsets) == 0: task_mode = "none" @@ -59,9 +51,7 @@ def populate_element_data( ephys_element.PreCluster.populate(precluster_key, **populate_settings) - if "1.0" in ( - ephys_element.ProbeInsertion * probe_element.Probe & precluster_key - ).fetch1("probe_type"): + if "1.0" in (ephys_element.ProbeInsertion * probe_element.Probe & precluster_key).fetch1("probe_type"): ephys_element.LFP.populate(precluster_key, **populate_settings) cluster_key = dict(**precluster_key, paramset_idx=paramset_idx) diff --git a/u19_pipeline/automatic_job/imaging_element.py b/u19_pipeline/automatic_job/imaging_element.py index e9dfbc44..e405952b 100644 --- a/u19_pipeline/automatic_job/imaging_element.py +++ b/u19_pipeline/automatic_job/imaging_element.py @@ -129,9 +129,7 @@ fov_key.pop("recording_process_id") # Recording process extra info -recording_process_info = ( - lab.Location * recording.RecordingProcess * recording.Recording & rec_process_key -).fetch( +recording_process_info = (lab.Location * recording.RecordingProcess * recording.Recording & rec_process_key).fetch( "acquisition_type", "preprocess_paramset_idx", "process_paramset_idx", as_dict=True ) @@ -148,12 +146,8 @@ # Get preprocess params preprocess_params_key = dict() -preprocess_params_key["preprocess_paramset_idx"] = recording_process_info[0][ - "preprocess_paramset_idx" -] -preprocess_params = recording.PreprocessParamSet().get_preprocess_params( - preprocess_params_key -) +preprocess_params_key["preprocess_paramset_idx"] = recording_process_info[0]["preprocess_paramset_idx"] +preprocess_params = recording.PreprocessParamSet().get_preprocess_params(preprocess_params_key) processing_method = preprocess_params["processing_method"] task_mode = preprocess_params["task_mode"] @@ -279,9 +273,7 @@ # Get all information from process params from recording.ProcessParamSet schema process_params_key = dict() process_params_key["process_paramset_idx"] = paramset_idx - process_params_info = (recording.ProcessParamSet() & process_params_key).fetch( - as_dict=True - ) + process_params_info = (recording.ProcessParamSet() & process_params_key).fetch(as_dict=True) process_params = recording.ProcessParamSet().get_process_params(process_params_key) # print('process_params_info', process_params_info) diff --git a/u19_pipeline/automatic_job/imaging_element_populate.py b/u19_pipeline/automatic_job/imaging_element_populate.py index be0d009f..675d6384 100644 --- a/u19_pipeline/automatic_job/imaging_element_populate.py +++ b/u19_pipeline/automatic_job/imaging_element_populate.py @@ -6,9 +6,7 @@ from u19_pipeline.imaging_pipeline import imaging_element -def populate_element_data( - job_id, display_progress=True, reserve_jobs=False, suppress_errors=False -): +def populate_element_data(job_id, display_progress=True, reserve_jobs=False, suppress_errors=False): populate_settings = { "display_progress": display_progress, @@ -16,9 +14,7 @@ def populate_element_data( "suppress_errors": suppress_errors, } - process_key = ( - recording_process.Processing * recording.Recording & dict(job_id=job_id) - ).fetch1("KEY") + process_key = (recording_process.Processing * recording.Recording & dict(job_id=job_id)).fetch1("KEY") if (recording.Recording & process_key).fetch1("recording_modality") != "imaging": warnings.warn(f"Recording modality is not `imaging` for job_id: {job_id}") @@ -26,22 +22,19 @@ def populate_element_data( fragment_number, recording_process_pre_path, recording_process_post_path = ( recording_process.Processing & process_key - ).fetch1( - "fragment_number", "recording_process_pre_path", "recording_process_post_path" - ) + ).fetch1("fragment_number", "recording_process_pre_path", "recording_process_post_path") - preprocess_param_steps_id, paramset_idx = ( - recording_process.Processing.ImagingParams & process_key - ).fetch1("preprocess_param_steps_id", "paramset_idx") + preprocess_param_steps_id, paramset_idx = (recording_process.Processing.ImagingParams & process_key).fetch1( + "preprocess_param_steps_id", "paramset_idx" + ) preprocess_paramsets = ( - imaging_element.PreprocessParamSteps.Step() - & dict(preprocess_param_steps_id=preprocess_param_steps_id) + imaging_element.PreprocessParamSteps.Step() & dict(preprocess_param_steps_id=preprocess_param_steps_id) ).fetch("paramset_idx") - processing_method = ( - imaging_element.ProcessingParamSet & dict(paramset_idx=paramset_idx) - ).fetch1("processing_method") + processing_method = (imaging_element.ProcessingParamSet & dict(paramset_idx=paramset_idx)).fetch1( + "processing_method" + ) if len(preprocess_paramsets) == 0: preprocess_task_mode = "none" diff --git a/u19_pipeline/automatic_job/parameter_file_creator.py b/u19_pipeline/automatic_job/parameter_file_creator.py index ddba2287..6551b54d 100644 --- a/u19_pipeline/automatic_job/parameter_file_creator.py +++ b/u19_pipeline/automatic_job/parameter_file_creator.py @@ -22,9 +22,7 @@ default_process_script_path = "scripts/automate_imaging_element.py" -def generate_parameter_file( - recording_process_id, params, type_params, program_selection_params -): +def generate_parameter_file(recording_process_id, params, type_params, program_selection_params): """ Generate and send parameter files for processing """ @@ -35,9 +33,7 @@ def generate_parameter_file( # Write preprocessing parameter file if type_params == "preparams": - write_parameter_file( - params, recording_process_id, config.default_preprocess_filename - ) + write_parameter_file(params, recording_process_id, config.default_preprocess_filename) if program_selection_params["local_or_cluster"] == "cluster": status = transfer_parameter_file( recording_process_id, @@ -48,9 +44,7 @@ def generate_parameter_file( else: status = config.system_process["SUCCESS"] else: - write_parameter_file( - params, recording_process_id, config.default_process_filename - ) + write_parameter_file(params, recording_process_id, config.default_process_filename) if program_selection_params["local_or_cluster"] == "cluster": status = transfer_parameter_file( recording_process_id, @@ -71,24 +65,18 @@ def write_parameter_file(params, recording_process_id, default_param_filename): str_params = json.dumps(params) param_filename = default_param_filename % (recording_process_id) - params_file_local_path = str( - pathlib.Path(config.parameter_files_filepath, param_filename) - ) + params_file_local_path = str(pathlib.Path(config.parameter_files_filepath, param_filename)) write_file(params_file_local_path, str_params) -def transfer_parameter_file( - recording_process_id, default_param_filename, cluster_param_dir, user_host -): +def transfer_parameter_file(recording_process_id, default_param_filename, cluster_param_dir, user_host): """ Transfer parameter file to processing cluster """ param_filename = default_param_filename % (recording_process_id) - params_file_local_path = str( - pathlib.Path(config.parameter_files_filepath, param_filename) - ) + params_file_local_path = str(pathlib.Path(config.parameter_files_filepath, param_filename)) params_file_cluster_path = str(pathlib.Path(cluster_param_dir, param_filename)) param_file_full_path = user_host + ":" + params_file_cluster_path @@ -125,24 +113,18 @@ def write_chanmap_file(chanmap_dict, recording_process_id, default_chanmap_filen Write local chanmap file to send """ chanmap_filename = default_chanmap_filename % (recording_process_id) - chanmap_file_local_path = str( - pathlib.Path(config.chanmap_files_filepath, chanmap_filename) - ) + chanmap_file_local_path = str(pathlib.Path(config.chanmap_files_filepath, chanmap_filename)) savemat(chanmap_file_local_path, chanmap_dict) -def transfer_chanmap_file( - recording_process_id, default_chanmap_filename, cluster_chanmap_dir, user_host -): +def transfer_chanmap_file(recording_process_id, default_chanmap_filename, cluster_chanmap_dir, user_host): """ Transfer chanmap file to processing cluster """ chanmap_filename = default_chanmap_filename % (recording_process_id) - chanmap_file_local_path = str( - pathlib.Path(config.chanmap_files_filepath, chanmap_filename) - ) + chanmap_file_local_path = str(pathlib.Path(config.chanmap_files_filepath, chanmap_filename)) chanmap_file_cluster_path = str(pathlib.Path(cluster_chanmap_dir, chanmap_filename)) chanmap_file_full_path = user_host + ":" + chanmap_file_cluster_path diff --git a/u19_pipeline/automatic_job/params_config.py b/u19_pipeline/automatic_job/params_config.py index 82c76f47..30a55f6d 100644 --- a/u19_pipeline/automatic_job/params_config.py +++ b/u19_pipeline/automatic_job/params_config.py @@ -78,9 +78,7 @@ recording_status_list = [[i["Value"], i["Label"]] for i in recording_status_dict] recording_status_df = pd.DataFrame(recording_status_dict) -RECORDING_STATUS_ERROR_ID = recording_status_df.loc[ - recording_status_df["Key"] == "ERROR", "Value" -].values[0] +RECORDING_STATUS_ERROR_ID = recording_status_df.loc[recording_status_df["Key"] == "ERROR", "Value"].values[0] # Dictionary with status configuration for recording process (units of each recording) recording_process_status_dict = [ @@ -185,15 +183,11 @@ }, ] -recording_process_status_list = [ - [i["Value"], i["Label"]] for i in recording_process_status_dict -] +recording_process_status_list = [[i["Value"], i["Label"]] for i in recording_process_status_dict] recording_process_status_df = pd.DataFrame(recording_process_status_dict) # All error and success status values -JOB_STATUS_ERROR_ID = recording_process_status_df.loc[ - recording_process_status_df["Key"] == "ERROR", "Value" -].values[0] +JOB_STATUS_ERROR_ID = recording_process_status_df.loc[recording_process_status_df["Key"] == "ERROR", "Value"].values[0] JOB_STATUS_PROCESSED = recording_process_status_df.loc[ recording_process_status_df["Key"] == "JOB_FINISHED_ELEMENT_WORKFLOW", "Value" ].values[0] diff --git a/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py b/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py index 38967474..68df166b 100644 --- a/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py +++ b/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py @@ -36,9 +36,7 @@ def get_rec_key_dict(recording_id): ) ) -not_sync_recs = pd.DataFrame( - (recording.Recording * ep.BehaviorSync).fetch("KEY", as_dict=True) -) +not_sync_recs = pd.DataFrame((recording.Recording * ep.BehaviorSync).fetch("KEY", as_dict=True)) not_sync_recs2 = pd.merge(all_recs, not_sync_recs, how="left", indicator=True) not_sync_recs2 = not_sync_recs2.loc[not_sync_recs2["_merge"] == "left_only", :] diff --git a/u19_pipeline/automatic_job/pupillometry_handler.py b/u19_pipeline/automatic_job/pupillometry_handler.py index d5620f1f..f51dbff1 100644 --- a/u19_pipeline/automatic_job/pupillometry_handler.py +++ b/u19_pipeline/automatic_job/pupillometry_handler.py @@ -46,9 +46,7 @@ def inner_function(*args, **kwargs): class PupillometryProcessingHandler: spock_home_dir = "/mnt/cup/braininit/Shared/repos/U19-pipeline_python/" spock_log_dir = spock_home_dir + "u19_pipeline/automatic_job/OutputLog/" - process_script_path = ( - spock_home_dir + "u19_pipeline/automatic_job/pupillometry_handler.py" - ) + process_script_path = spock_home_dir + "u19_pipeline/automatic_job/pupillometry_handler.py" spock_error_dir = spock_home_dir + "u19_pipeline/automatic_job/ErrorLog/" spock_slurm_filepath = spock_home_dir + "u19_pipeline/" spock_system_name = "spockmk2-loginvm.pni.princeton.edu" @@ -71,11 +69,7 @@ class PupillometryProcessingHandler: def generate_slurm_pupillometry(slurm_dict): slurm_text = "#!/bin/bash\n" - slurm_text += ( - PupillometryProcessingHandler.create_pupillometry_slurm_params_file( - slurm_dict - ) - ) + slurm_text += PupillometryProcessingHandler.create_pupillometry_slurm_params_file(slurm_dict) slurm_text += """ echo "SLURM_JOB_ID: ${SLURM_JOB_ID}" echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}" @@ -103,17 +97,9 @@ def create_pupillometry_slurm_params_file(slurm_dict): for slurm_param in slurm_dict.keys(): if isinstance(slurm_dict[slurm_param], list): for list_param in slurm_dict[slurm_param]: - text_dict += ( - "#SBATCH --" + str(slurm_param) + "=" + str(list_param) + "\n" - ) + text_dict += "#SBATCH --" + str(slurm_param) + "=" + str(list_param) + "\n" else: - text_dict += ( - "#SBATCH --" - + str(slurm_param) - + "=" - + str(slurm_dict[slurm_param]) - + "\n" - ) + text_dict += "#SBATCH --" + str(slurm_param) + "=" + str(slurm_dict[slurm_param]) + "\n" return text_dict @@ -126,27 +112,19 @@ def generate_slurm_file(video_dir): # Get all associated directories given the selected processing cluster # Start with default values - slurm_dict = copy.deepcopy( - PupillometryProcessingHandler.slurm_dict_pupillometry_spock - ) + slurm_dict = copy.deepcopy(PupillometryProcessingHandler.slurm_dict_pupillometry_spock) label_rec_process = "job_id_" + str(video_dir.stem) slurm_dict["job-name"] = label_rec_process print("PupillometryProcessingHandler.spock_log_dir") print(PupillometryProcessingHandler.spock_log_dir) print("label_rec_process", label_rec_process) print("video_dir", video_dir) - slurm_dict["output"] = ( - PupillometryProcessingHandler.spock_log_dir + label_rec_process + ".log" - ) - slurm_dict["error"] = ( - PupillometryProcessingHandler.spock_error_dir + label_rec_process + ".log" - ) + slurm_dict["output"] = PupillometryProcessingHandler.spock_log_dir + label_rec_process + ".log" + slurm_dict["error"] = PupillometryProcessingHandler.spock_error_dir + label_rec_process + ".log" print("slurm_dict", slurm_dict) - slurm_text = PupillometryProcessingHandler.generate_slurm_pupillometry( - slurm_dict - ) + slurm_text = PupillometryProcessingHandler.generate_slurm_pupillometry(slurm_dict) slurm_file_local_path = str( pathlib.Path( @@ -277,9 +255,7 @@ def getPupilDiameter(analyzedVideoDataPath): # Set a treshold for a valid zscore value (determined empirically) outlierFlags = np.abs(zscore) > 2 # Get a boolean array where true correspond to the frame with an outlier diameter - outlierFlags = outlierFlags.rename( - columns={outlierFlags.columns[0]: "OutlierFlag"} - ) + outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: "OutlierFlag"}) # Concatenate outlier flags array to remove outliers from pupil diameter array temp = pd.concat([df, outlierFlags], axis=1) temp.loc[temp["OutlierFlag"] == True, "PupilDiameter"] = None @@ -303,8 +279,7 @@ def check_pupillometry_sessions_queue(): update_value_dict = copy.deepcopy(config.default_update_value_dict) sessions_missing_process = ( - acquisition.SessionVideo * pupillometry.PupillometrySessionModelData - & "pupillometry_job_id is NULL" + acquisition.SessionVideo * pupillometry.PupillometrySessionModelData & "pupillometry_job_id is NULL" ).fetch(as_dict=True) print(sessions_missing_process) @@ -324,31 +299,22 @@ def check_pupillometry_sessions_queue(): # Get model location model_key = ( - pupillometry.PupillometryModels - & "model_id = " + str(pupillometry_2_process["model_id"]) + pupillometry.PupillometryModels & "model_id = " + str(pupillometry_2_process["model_id"]) ).fetch(as_dict=True)[0] models_dir = dj.config.get("custom", {}).get("root_data_dir", None) if models_dir is None: - raise Exception( - "root_data_dir in not found in config, , run initial_conf.py again" - ) + raise Exception("root_data_dir in not found in config, , run initial_conf.py again") models_dir = models_dir[0] model_path = pathlib.Path(models_dir, model_key["model_path"]) print("model path", model_path) # Get video location - pupillometry_dir = dj.config.get("custom", {}).get( - "pupillometry_root_data_dir", None - ) + pupillometry_dir = dj.config.get("custom", {}).get("pupillometry_root_data_dir", None) if pupillometry_dir is None: - raise Exception( - "pupillometry_root_data_dir not found in config, run initial_conf.py again" - ) + raise Exception("pupillometry_root_data_dir not found in config, run initial_conf.py again") pupillometry_raw_dir = pupillometry_dir[0] - videoPath = pathlib.Path( - pupillometry_raw_dir, pupillometry_2_process["remote_path_video_file"] - ) + videoPath = pathlib.Path(pupillometry_raw_dir, pupillometry_2_process["remote_path_video_file"]) print("videoPath", videoPath) @@ -362,9 +328,7 @@ def check_pupillometry_sessions_queue(): output_dir.mkdir(parents=True, exist_ok=True) # Generate slurm file and transfer it to spock - status, slurm_filepath = PupillometryProcessingHandler.generate_slurm_file( - videoPath - ) + status, slurm_filepath = PupillometryProcessingHandler.generate_slurm_file(videoPath) print("slurm_filepath", slurm_filepath) # Error handling (generating slurm file) @@ -376,9 +340,7 @@ def check_pupillometry_sessions_queue(): pupillometry.PupillometrySessionModelData.update1(key_insert) slack_utils.send_slack_error_pupillometry_notification( - config.slack_webhooks_dict[ - "automation_pipeline_error_notification" - ], + config.slack_webhooks_dict["automation_pipeline_error_notification"], update_value_dict["error_info"], pupillometry_2_process, ) @@ -387,29 +349,23 @@ def check_pupillometry_sessions_queue(): continue # Queue slurm file in spock - status, slurm_jobid, error_message = ( - PupillometryProcessingHandler.queue_pupillometry_slurm_file( - videoPath, - model_path, - PupillometryProcessingHandler.spock_home_dir, - output_dir, - PupillometryProcessingHandler.process_script_path, - slurm_filepath, - ) + status, slurm_jobid, error_message = PupillometryProcessingHandler.queue_pupillometry_slurm_file( + videoPath, + model_path, + PupillometryProcessingHandler.spock_home_dir, + output_dir, + PupillometryProcessingHandler.process_script_path, + slurm_filepath, ) # Error handling (queuing slurm file) if status != config.system_process["SUCCESS"]: status_update = config.status_update_idx["ERROR_STATUS"] - update_value_dict["error_info"]["error_message"] = ( - "Error to queue pupillometry slurm file" - ) + update_value_dict["error_info"]["error_message"] = "Error to queue pupillometry slurm file" pupillometry.PupillometrySessionModelData.update1(key_insert) slack_utils.send_slack_error_pupillometry_notification( - config.slack_webhooks_dict[ - "automation_pipeline_error_notification" - ], + config.slack_webhooks_dict["automation_pipeline_error_notification"], update_value_dict["error_info"], pupillometry_2_process, ) @@ -459,17 +415,13 @@ def check_processed_pupillometry_sessions(): # If job finished copy over output and/or error log if status_update == config.status_update_idx["ERROR_STATUS"]: - update_value_dict["error_info"]["error_message"] = ( - "An error occured in processing (check LOG)" - ) + update_value_dict["error_info"]["error_message"] = "An error occured in processing (check LOG)" update_value_dict["error_info"]["error_exception"] = message key_update["pupillometry_job_id"] = -1 pupillometry.PupillometrySessionModelData.update1(key_update) slack_utils.send_slack_error_pupillometry_notification( - config.slack_webhooks_dict[ - "automation_pipeline_error_notification" - ], + config.slack_webhooks_dict["automation_pipeline_error_notification"], update_value_dict["error_info"], session_check, ) @@ -477,13 +429,9 @@ def check_processed_pupillometry_sessions(): # Get video location if status_update == config.status_update_idx["NEXT_STATUS"]: - pupillometry_dir = dj.config.get("custom", {}).get( - "pupillometry_root_data_dir", None - ) + pupillometry_dir = dj.config.get("custom", {}).get("pupillometry_root_data_dir", None) if pupillometry_dir is None: - raise Exception( - "pupillometry_root_data_dir not found in config, run initial_conf.py again" - ) + raise Exception("pupillometry_root_data_dir not found in config, run initial_conf.py again") # Create output location pupillometry_processed_dir = pupillometry_dir[1] @@ -501,9 +449,7 @@ def check_processed_pupillometry_sessions(): "Didnt find any h5 files after deeplabcut analyze_video" ) slack_utils.send_slack_error_pupillometry_notification( - config.slack_webhooks_dict[ - "automation_pipeline_error_notification" - ], + config.slack_webhooks_dict["automation_pipeline_error_notification"], update_value_dict["error_info"], session_check, ) @@ -514,17 +460,13 @@ def check_processed_pupillometry_sessions(): h5_files = h5_files[0] try: - pupil_data = PupillometryProcessingHandler.getPupilDiameter( - h5_files - ) + pupil_data = PupillometryProcessingHandler.getPupilDiameter(h5_files) except Exception: update_value_dict["error_info"]["error_message"] = ( "Could not get pupil diameter (check h5 or video file)" ) slack_utils.send_slack_error_pupillometry_notification( - config.slack_webhooks_dict[ - "automation_pipeline_error_notification" - ], + config.slack_webhooks_dict["automation_pipeline_error_notification"], update_value_dict["error_info"], session_check, ) @@ -536,9 +478,7 @@ def check_processed_pupillometry_sessions(): print("key_update", key_update) pupillometry.PupillometrySessionModelData.update1(key_update) slack_utils.send_slack_pupillometry_update_notification( - config.slack_webhooks_dict[ - "automation_pipeline_update_notification" - ], + config.slack_webhooks_dict["automation_pipeline_update_notification"], "Pupillometry job finished", session_check, ) diff --git a/u19_pipeline/automatic_job/pupillometry_main.py b/u19_pipeline/automatic_job/pupillometry_main.py index 9bfbe20a..5bd37607 100644 --- a/u19_pipeline/automatic_job/pupillometry_main.py +++ b/u19_pipeline/automatic_job/pupillometry_main.py @@ -16,6 +16,4 @@ print(args[1]) print(args[2]) - ph.PupillometryProcessingHandler.analyze_videos_pupillometry( - args[0], args[1], args[2] - ) + ph.PupillometryProcessingHandler.analyze_videos_pupillometry(args[0], args[1], args[2]) diff --git a/u19_pipeline/automatic_job/recording_handler.py b/u19_pipeline/automatic_job/recording_handler.py index 536486d5..881865cb 100644 --- a/u19_pipeline/automatic_job/recording_handler.py +++ b/u19_pipeline/automatic_job/recording_handler.py @@ -67,9 +67,7 @@ def pipeline_handler_main(): print("function to apply:", next_status_series["ProcessFunction"]) # Get processing function - function_status_process = getattr( - RecordingHandler, next_status_series["ProcessFunction"] - ) + function_status_process = getattr(RecordingHandler, next_status_series["ProcessFunction"]) # Trigger process, if success update recording process record try: @@ -85,16 +83,12 @@ def pipeline_handler_main(): field_update = next_status_series["UpdateField"] # Update status in u19_recording.recording table (possibly other field as well) - RecordingHandler.update_status_pipeline( - key, next_status, field_update, value_update - ) + RecordingHandler.update_status_pipeline(key, next_status, field_update, value_update) # Send slack Message to webhook if slack message activated for status if next_status_series["SlackMessage"]: slack_utils.send_slack_update_notification( - config.slack_webhooks_dict[ - "automation_pipeline_update_notification" - ], + config.slack_webhooks_dict["automation_pipeline_update_notification"], next_status_series["SlackMessage"], recording_series, ) @@ -102,13 +96,9 @@ def pipeline_handler_main(): # An error occurred in process if status == config.status_update_idx["ERROR_STATUS"]: next_status = config.RECORDING_STATUS_ERROR_ID - RecordingHandler.update_status_pipeline( - key, next_status, None, None - ) + RecordingHandler.update_status_pipeline(key, next_status, None, None) slack_utils.send_slack_error_notification( - config.slack_webhooks_dict[ - "automation_pipeline_error_notification" - ], + config.slack_webhooks_dict["automation_pipeline_error_notification"], update_dict["error_info"], recording_series, ) @@ -201,9 +191,7 @@ def local_transfer_check(rec_series): status_update = config.status_update_idx["NEXT_STATUS"] else: status_update = config.status_update_idx["ERROR_STATUS"] - update_value_dict["error_info"]["error_message"] = ( - "Return code scp not = 0" - ) + update_value_dict["error_info"]["error_message"] = "Return code scp not = 0" print("is_finished", is_finished, "status_update", status_update) @@ -228,14 +216,10 @@ def modality_preingestion(in_rec_series): rec_series = in_rec_series.copy() if rec_series["recording_modality"] == "electrophysiology": - status_update, update_value_dict = ( - RecordingHandler.electrophysiology_preingestion(rec_series) - ) + status_update, update_value_dict = RecordingHandler.electrophysiology_preingestion(rec_series) if rec_series["recording_modality"] == "imaging": - status_update, update_value_dict = RecordingHandler.imaging_preingestion( - rec_series - ) + status_update, update_value_dict = RecordingHandler.imaging_preingestion(rec_series) return (status_update, update_value_dict) @@ -247,19 +231,13 @@ def get_active_recordings(): df_recordings (pd.DataFrame): all recordings that are going to be processed in the pipeline """ - status_query = "status_recording_id > " + str( - config.recording_status_df["Value"].min() - ) - status_query += " and status_recording_id < " + str( - config.recording_status_df["Value"].max() - ) + status_query = "status_recording_id > " + str(config.recording_status_df["Value"].min()) + status_query += " and status_recording_id < " + str(config.recording_status_df["Value"].max()) print(status_query) recordings_active = ( - recording.Recording - * lab.Location.proj("ip_address", "system_user", "acquisition_type") - & status_query + recording.Recording * lab.Location.proj("ip_address", "system_user", "acquisition_type") & status_query ) print(recordings_active) @@ -268,16 +246,12 @@ def get_active_recordings(): if df_recordings.shape[0] > 0: key_list = dj_short.get_primary_key_fields(recording.Recording) - df_recordings["query_key"] = df_recordings.loc[:, key_list].to_dict( - orient="records" - ) + df_recordings["query_key"] = df_recordings.loc[:, key_list].to_dict(orient="records") return df_recordings @staticmethod - def update_status_pipeline( - recording_key_dict, status, update_field=None, update_value=None - ): + def update_status_pipeline(recording_key_dict, status, update_field=None, update_value=None): """ Update recording.Recording table status and optional task field Args: @@ -304,9 +278,7 @@ def update_status_pipeline( recording.Recording.update1(update_status_dict) @staticmethod - def update_recording_log( - recording_id, current_status, next_status, error_info_dict - ): + def update_recording_log(recording_id, current_status, next_status, error_info_dict): """ Update recording.RecordingLog table status and optional task field Args: @@ -324,19 +296,11 @@ def update_recording_log( key["status_recording_id_new"] = next_status key["recording_status_timestamp"] = date_time - if ( - error_info_dict["error_message"] is not None - and len(error_info_dict["error_message"]) >= 256 - ): + if error_info_dict["error_message"] is not None and len(error_info_dict["error_message"]) >= 256: error_info_dict["error_message"] = error_info_dict["error_message"][:255] - if ( - error_info_dict["error_exception"] is not None - and len(error_info_dict["error_exception"]) >= 4096 - ): - error_info_dict["error_exception"] = error_info_dict["error_exception"][ - :4095 - ] + if error_info_dict["error_exception"] is not None and len(error_info_dict["error_exception"]) >= 4096: + error_info_dict["error_exception"] = error_info_dict["error_exception"][:4095] key["recording_error_message"] = error_info_dict["error_message"] key["recording_error_exception"] = error_info_dict["error_exception"] @@ -366,9 +330,9 @@ def electrophysiology_preingestion(rec_series): ephys_element_ingest.process_session(rec_series["query_key"]) ephys_pipeline.ephys_element.EphysRecording.populate(rec_series["query_key"]) - ingested_recording = ( - ephys_pipeline.ephys_element.EphysRecording & rec_series["query_key"] - ).fetch("KEY", as_dict=True) + ingested_recording = (ephys_pipeline.ephys_element.EphysRecording & rec_series["query_key"]).fetch( + "KEY", as_dict=True + ) print("before BehaviorSync") print("rec_series['query_key']", rec_series["query_key"]) @@ -383,34 +347,25 @@ def electrophysiology_preingestion(rec_series): return (status_update, update_value_dict) # Insert recording processes records - old_recording_process = ( - recording_process.Processing() & rec_series["query_key"] - ).fetch("KEY", as_dict=True) + old_recording_process = (recording_process.Processing() & rec_series["query_key"]).fetch("KEY", as_dict=True) if len(old_recording_process) == 0: connection = recording.Recording.connection with connection.transaction: - probe_files = ( - ephys_pipeline.ephys_element.EphysRecording.EphysFile - & rec_series["query_key"] - ).fetch(as_dict=True) + probe_files = (ephys_pipeline.ephys_element.EphysRecording.EphysFile & rec_series["query_key"]).fetch( + as_dict=True + ) probe_files = [ dict( item, - recording_process_pre_path=pathlib.Path(item["file_path"]) - .parents[0] - .as_posix(), + recording_process_pre_path=pathlib.Path(item["file_path"]).parents[0].as_posix(), ) for item in probe_files ] - recording_process.Processing().insert_recording_process( - probe_files, "insertion_number" - ) + recording_process.Processing().insert_recording_process(probe_files, "insertion_number") # Get parameters for recording processes - recording_processes = ( - recording_process.Processing() & rec_series["query_key"] - ).fetch( + recording_processes = (recording_process.Processing() & rec_series["query_key"]).fetch( "job_id", "recording_id", "fragment_number", @@ -418,14 +373,10 @@ def electrophysiology_preingestion(rec_series): as_dict=True, ) default_params_record_df = pd.DataFrame( - (recording.DefaultParams & rec_series["query_key"]).fetch( - as_dict=True - ) + (recording.DefaultParams & rec_series["query_key"]).fetch(as_dict=True) ) - params_rec_process = ( - recording.DefaultParams.get_default_params_rec_process( - recording_processes, default_params_record_df - ) + params_rec_process = recording.DefaultParams.get_default_params_rec_process( + recording_processes, default_params_record_df ) # Rename preprocess_param_steps_id with the electrophysiology one @@ -435,14 +386,10 @@ def electrophysiology_preingestion(rec_series): recording_process.Processing.EphysParams.insert(params_rec_process) # Update recording_process_post_path - recording_process.Processing().set_recording_process_post_path( - recording_processes - ) + recording_process.Processing().set_recording_process_post_path(recording_processes) # Create lfp trace if needed (neuropixel 2.0 probes) - recording_directory = ( - recording.Recording & rec_series["query_key"] - ).fetch1("recording_directory") + recording_directory = (recording.Recording & rec_series["query_key"]).fetch1("recording_directory") recording_directory = pathlib.Path( dj.config["custom"]["ephys_root_data_dir"][0], recording_directory ).parent.as_posix() @@ -451,9 +398,7 @@ def electrophysiology_preingestion(rec_series): dj.config["custom"]["ephys_root_data_dir"][0], i["recording_process_pre_path"], ).as_posix() - ephys_pipeline.create_lfp_trace( - config.catgt_script, recording_directory, probe_dir - ) + ephys_pipeline.create_lfp_trace(config.catgt_script, recording_directory, probe_dir) status_update = config.status_update_idx["NEXT_STATUS"] @@ -485,15 +430,11 @@ def imaging_preingestion(rec_series): imaging_pipeline.AcquiredTiff.populate(rec_series["query_key"]) # Retrieve all fovs records ingested in matlab Script - fovs_ingested = (imaging_pipeline.TiffSplit & rec_series["query_key"]).fetch( - "KEY", as_dict=True - ) + fovs_ingested = (imaging_pipeline.TiffSplit & rec_series["query_key"]).fetch("KEY", as_dict=True) if len(fovs_ingested) == 0: status_update = config.status_update_idx["ERROR_STATUS"] - update_value_dict["error_info"]["error_message"] = ( - "Imaging TiffSplit process failed" - ) + update_value_dict["error_info"]["error_message"] = "Imaging TiffSplit process failed" return (status_update, update_value_dict) # Ingest Scan for each fov from the TiffSplit process @@ -517,61 +458,40 @@ def imaging_preingestion(rec_series): ) # Populate ScanInfo for all fovs - imaging_pipeline.scan_element.ScanInfo.populate( - rec_series["query_key"], display_progress=True - ) + imaging_pipeline.scan_element.ScanInfo.populate(rec_series["query_key"], display_progress=True) # ingested_recording = (imaging_pipeline.scan_element.Scan & rec_series['query_key']).fetch("KEY", as_dict=True) # Get fov directories for each recording process: fov_files_df = pd.DataFrame( - ( - imaging_pipeline.scan_element.ScanInfo.ScanFile - & rec_series["query_key"] - ).fetch(as_dict=True) + (imaging_pipeline.scan_element.ScanInfo.ScanFile & rec_series["query_key"]).fetch(as_dict=True) ) fov_files = fov_files_df.groupby("tiff_split").first().reset_index() # Insert recording processes records - old_recording_process = ( - recording_process.Processing() & rec_series["query_key"] - ).fetch("KEY", as_dict=True) + old_recording_process = (recording_process.Processing() & rec_series["query_key"]).fetch("KEY", as_dict=True) if len(old_recording_process) == 0: connection = recording.Recording.connection with connection.transaction: # Get fov directories for each recording process: fov_files_df = pd.DataFrame( - ( - imaging_pipeline.scan_element.ScanInfo.ScanFile - & rec_series["query_key"] - ).fetch(as_dict=True) - ) - fov_files = ( - fov_files_df.groupby("tiff_split") - .first() - .reset_index() - .to_dict("records") + (imaging_pipeline.scan_element.ScanInfo.ScanFile & rec_series["query_key"]).fetch(as_dict=True) ) + fov_files = fov_files_df.groupby("tiff_split").first().reset_index().to_dict("records") fov_files = [ dict( item, - recording_process_pre_path=pathlib.Path( - item["file_path"] - ).parent.as_posix(), + recording_process_pre_path=pathlib.Path(item["file_path"]).parent.as_posix(), ) for item in fov_files ] - recording_process.Processing().insert_recording_process( - fov_files, "tiff_split" - ) + recording_process.Processing().insert_recording_process(fov_files, "tiff_split") # Get parameters for recording processes - recording_processes = ( - recording_process.Processing() & rec_series["query_key"] - ).fetch( + recording_processes = (recording_process.Processing() & rec_series["query_key"]).fetch( "job_id", "recording_id", "fragment_number", @@ -579,24 +499,16 @@ def imaging_preingestion(rec_series): as_dict=True, ) default_params_record_df = pd.DataFrame( - (recording.DefaultParams & rec_series["query_key"]).fetch( - as_dict=True - ) + (recording.DefaultParams & rec_series["query_key"]).fetch(as_dict=True) ) - params_rec_process = ( - recording.DefaultParams.get_default_params_rec_process( - recording_processes, default_params_record_df - ) + params_rec_process = recording.DefaultParams.get_default_params_rec_process( + recording_processes, default_params_record_df ) - recording_process.Processing.ImagingParams.insert( - params_rec_process, skip_duplicates=True - ) + recording_process.Processing.ImagingParams.insert(params_rec_process, skip_duplicates=True) # Update recording_process_post_path - recording_process.Processing().set_recording_process_post_path( - recording_processes - ) + recording_process.Processing().set_recording_process_post_path(recording_processes) status_update = config.status_update_idx["NEXT_STATUS"] diff --git a/u19_pipeline/automatic_job/recording_process_handler.py b/u19_pipeline/automatic_job/recording_process_handler.py index 0ad22224..1d63206c 100644 --- a/u19_pipeline/automatic_job/recording_process_handler.py +++ b/u19_pipeline/automatic_job/recording_process_handler.py @@ -52,15 +52,11 @@ def pipeline_handler_main(): ].squeeze() # Get processing function - function_status_process = getattr( - RecProcessHandler, next_status_series["ProcessFunction"] - ) + function_status_process = getattr(RecProcessHandler, next_status_series["ProcessFunction"]) # Trigger process, if success update recording process record try: - status, update_dict = function_status_process( - rec_process_series, next_status_series - ) + status, update_dict = function_status_process(rec_process_series, next_status_series) # print('update_dict', update_dict) # Get dictionary of record process @@ -80,15 +76,11 @@ def pipeline_handler_main(): print("value_update", value_update, "field_update", field_update) print("function executed:", next_status_series["ProcessFunction"]) - RecProcessHandler.update_status_pipeline( - key, next_status, field_update, value_update - ) + RecProcessHandler.update_status_pipeline(key, next_status, field_update, value_update) if next_status_series["SlackMessage"]: slack_utils.send_slack_update_notification( - config.slack_webhooks_dict[ - "automation_pipeline_update_notification" - ], + config.slack_webhooks_dict["automation_pipeline_update_notification"], next_status_series["SlackMessage"], rec_process_series, ) @@ -101,20 +93,18 @@ def pipeline_handler_main(): # Crop error messages to fit in DB if len(update_dict["error_info"]["error_message"]) > 255: print("Cropping error message") - update_dict["error_info"]["error_message"] = update_dict[ - "error_info" - ]["error_message"][-255:] + update_dict["error_info"]["error_message"] = update_dict["error_info"]["error_message"][ + -255: + ] if ( - isinstance( - update_dict["error_info"]["error_exception"], str - ) + isinstance(update_dict["error_info"]["error_exception"], str) and len(update_dict["error_info"]["error_exception"]) > 4095 ): print("Cropping error error_exception") - update_dict["error_info"]["error_exception"] = update_dict[ - "error_info" - ]["error_exception"][-4095:] + update_dict["error_info"]["error_exception"] = update_dict["error_info"]["error_exception"][ + -4095: + ] RecProcessHandler.update_job_id_log( rec_process_series["job_id"], @@ -126,9 +116,7 @@ def pipeline_handler_main(): # An error occurred in process if status == config.status_update_idx["ERROR_STATUS"]: next_status = config.RECORDING_STATUS_ERROR_ID - RecProcessHandler.update_status_pipeline( - key, next_status, None, None - ) + RecProcessHandler.update_status_pipeline(key, next_status, None, None) # Crop even more error message to fit in slack notification if ( @@ -136,16 +124,14 @@ def pipeline_handler_main(): and len(update_dict["error_info"]["error_exception"]) > 1024 ): print("Cropping error error_exception for slack") - update_dict["error_info"]["error_exception"] = update_dict[ - "error_info" - ]["error_exception"][-1023:] + update_dict["error_info"]["error_exception"] = update_dict["error_info"]["error_exception"][ + -1023: + ] # Send slack notification with original error exception, retry if exception has invalid format try: slack_utils.send_slack_error_notification( - config.slack_webhooks_dict[ - "automation_pipeline_error_notification" - ], + config.slack_webhooks_dict["automation_pipeline_error_notification"], update_dict["error_info"], rec_process_series, ) @@ -154,9 +140,7 @@ def pipeline_handler_main(): "Error exception not accepted in slack, check log file for error" ) slack_utils.send_slack_error_notification( - config.slack_webhooks_dict[ - "automation_pipeline_error_notification" - ], + config.slack_webhooks_dict["automation_pipeline_error_notification"], update_dict["error_info"], rec_process_series, ) @@ -199,14 +183,10 @@ def transfer_request(rec_series, status_series): # status, task_id = scp_tr.call_scp_background(ip_address=rec_series['ip_address'], system_user=rec_series['system_user'], # recording_system_directory=rec_series['local_directory'], data_directory=data_directory.as_posix()) - transfer_request = ft.globus_transfer_to_tiger( - job_id, raw_rel_path, modality - ) + transfer_request = ft.globus_transfer_to_tiger(job_id, raw_rel_path, modality) elif status_series["Key"] == "PROC_FILE_TRANSFER_REQUEST": # ALS, which recording directory for processed file - transfer_request = ft.globus_transfer_to_pni( - job_id, proc_rel_path, modality - ) + transfer_request = ft.globus_transfer_to_pni(job_id, proc_rel_path, modality) if transfer_request["status"] == config.system_process["SUCCESS"]: status_update = config.status_update_idx["NEXT_STATUS"] @@ -214,9 +194,7 @@ def transfer_request(rec_series, status_series): else: status_update = config.status_update_idx["ERROR_STATUS"] - update_value_dict["error_info"]["error_message"] = transfer_request[ - "error_info" - ] + update_value_dict["error_info"]["error_message"] = transfer_request["error_info"] # If not tiger let's go to next status else: @@ -253,9 +231,7 @@ def transfer_check(rec_series, status_series): status_update = config.status_update_idx["NEXT_STATUS"] elif transfer_request["status"] == config.system_process["ERROR"]: status_update = config.status_update_idx["ERROR_STATUS"] - update_value_dict["error_info"]["error_message"] = ( - "An error occured during globus transfer" - ) + update_value_dict["error_info"]["error_message"] = "An error occured during globus transfer" else: status_update = config.status_update_idx["NEXT_STATUS"] @@ -304,45 +280,28 @@ def slurm_job_queue(rec_series, status_series): ) else: status_update = config.status_update_idx["ERROR_STATUS"] - update_value_dict["error_info"]["error_message"] = ( - "Error while generating/transfering parameter file" - ) + update_value_dict["error_info"]["error_message"] = "Error while generating/transfering parameter file" return (status_update, update_value_dict) # If electrophysiology, transfer chanmap as well - if ( - status == config.system_process["SUCCESS"] - and rec_series["recording_modality"] == "electrophysiology" - ): + if status == config.system_process["SUCCESS"] and rec_series["recording_modality"] == "electrophysiology": recording_key = ( - recording_process.Processing.proj( - "recording_id", insertion_number="fragment_number" - ) + recording_process.Processing.proj("recording_id", insertion_number="fragment_number") & rec_series["query_key"] ).fetch1() del recording_key["job_id"] chanmap_filename = config.default_chanmap_filename % (rec_series["job_id"]) - chanmap_file_local_path = pathlib.Path( - config.chanmap_files_filepath, chanmap_filename - ).as_posix() + chanmap_file_local_path = pathlib.Path(config.chanmap_files_filepath, chanmap_filename).as_posix() raw_directory_for_chanmap = rec_series["recording_process_pre_path"] - spikeglx_meta_filepath = ephys_pipeline.get_spikeglx_meta_filepath( - recording_key - ) + spikeglx_meta_filepath = ephys_pipeline.get_spikeglx_meta_filepath(recording_key) # Chanmap mat file generation - MetaToCoords( - spikeglx_meta_filepath, 1, destFullPath=chanmap_file_local_path - ) - status = paramfilelib.generate_chanmap_file( - rec_series["job_id"], rec_series["program_selection_params"] - ) + MetaToCoords(spikeglx_meta_filepath, 1, destFullPath=chanmap_file_local_path) + status = paramfilelib.generate_chanmap_file(rec_series["job_id"], rec_series["program_selection_params"]) if status != config.system_process["SUCCESS"]: status_update = config.status_update_idx["ERROR_STATUS"] - update_value_dict["error_info"]["error_message"] = ( - "Error while generating/transfering channelmap file" - ) + update_value_dict["error_info"]["error_message"] = "Error while generating/transfering channelmap file" return (status_update, update_value_dict) # Only queue if processing in tiger @@ -356,9 +315,7 @@ def slurm_job_queue(rec_series, status_series): ) else: status_update = config.status_update_idx["ERROR_STATUS"] - update_value_dict["error_info"]["error_message"] = ( - "Error while generating/transfering slurm file" - ) + update_value_dict["error_info"]["error_message"] = "Error while generating/transfering slurm file" return (status_update, update_value_dict) # Queue slurm file @@ -373,9 +330,7 @@ def slurm_job_queue(rec_series, status_series): ) else: status_update = config.status_update_idx["ERROR_STATUS"] - update_value_dict["error_info"]["error_message"] = ( - "Error while generating/transfering slurm file" - ) + update_value_dict["error_info"]["error_message"] = "Error while generating/transfering slurm file" return (status_update, update_value_dict) if status == config.system_process["SUCCESS"]: @@ -383,9 +338,7 @@ def slurm_job_queue(rec_series, status_series): update_value_dict["value_update"] = slurm_jobid else: status_update = config.status_update_idx["ERROR_STATUS"] - update_value_dict["error_info"]["error_message"] = ( - "Error while queuing job in SLURM" - ) + update_value_dict["error_info"]["error_message"] = "Error while queuing job in SLURM" return (status_update, update_value_dict) else: @@ -419,23 +372,14 @@ def slurm_job_check(rec_series, status_series): local_user = False program_selection_params = rec_series["program_selection_params"] - if ( - program_selection_params["process_cluster"] == "spock" - and is_this_spock() - ): + if program_selection_params["process_cluster"] == "spock" and is_this_spock(): local_user = True - ssh_user = ft.cluster_vars[program_selection_params["process_cluster"]][ - "user" - ] - ssh_host = ft.cluster_vars[program_selection_params["process_cluster"]][ - "hostname" - ] + ssh_user = ft.cluster_vars[program_selection_params["process_cluster"]]["user"] + ssh_host = ft.cluster_vars[program_selection_params["process_cluster"]]["hostname"] slurm_jobid = str(rec_series["slurm_id"]) - status_update, message = slurmlib.check_slurm_job( - ssh_user, ssh_host, slurm_jobid, local_user=local_user - ) + status_update, message = slurmlib.check_slurm_job(ssh_user, ssh_host, slurm_jobid, local_user=local_user) # Get message from slurm status check update_value_dict["error_info"]["error_message"] = message @@ -463,9 +407,7 @@ def slurm_job_check(rec_series, status_series): # If the program errored, print the log. # Previous method of capturing the error_log was insufficient since Kilosort wrote to stderr if status_update == config.status_update_idx["ERROR_STATUS"]: - update_value_dict["error_info"]["error_message"] = ( - "An error occured in processing (check LOG)" - ) + update_value_dict["error_info"]["error_message"] = "An error occured in processing (check LOG)" update_value_dict["error_info"]["error_exception"] = error_log else: status_update = config.status_update_idx["NEXT_STATUS"] @@ -516,20 +458,14 @@ def get_program_selection_params(modality): ] # Pack all features in a dictionary - this_modality_program_selection_params_dict = ( - this_modality_program_selection_params.to_dict("records") - ) + this_modality_program_selection_params_dict = this_modality_program_selection_params.to_dict("records") this_modality_program_selection_params_dict # Get two columns, (recording_modality & "packed" program_selection_params) this_modality_program_selection_params = ( - this_modality_program_selection_params.loc[:, "recording_modality"] - .to_frame() - .copy() - ) - this_modality_program_selection_params["program_selection_params"] = ( - this_modality_program_selection_params_dict + this_modality_program_selection_params.loc[:, "recording_modality"].to_frame().copy() ) + this_modality_program_selection_params["program_selection_params"] = this_modality_program_selection_params_dict this_modality_program_selection_params @@ -545,19 +481,13 @@ def get_active_process_jobs(active=True): if active: status_query = "status_processing_id > " + str(config.JOB_STATUS_ERROR_ID) - status_query += " and status_processing_id < " + str( - config.JOB_STATUS_PROCESSED - ) + status_query += " and status_processing_id < " + str(config.JOB_STATUS_PROCESSED) else: status_query = "status_processing_id = " + str(config.JOB_STATUS_ERROR_ID) - status_query += " or status_processing_id = " + str( - config.JOB_STATUS_PROCESSED - ) + status_query += " or status_processing_id = " + str(config.JOB_STATUS_PROCESSED) jobs_active = ( - recording.Recording.proj( - "recording_modality", "location", "recording_directory" - ) + recording.Recording.proj("recording_modality", "location", "recording_directory") * recording_process.Processing & status_query ) @@ -565,9 +495,7 @@ def get_active_process_jobs(active=True): if df_process_jobs.shape[0] > 0: key_list = dj_short.get_primary_key_fields(recording_process.Processing) - df_process_jobs["query_key"] = df_process_jobs.loc[:, key_list].to_dict( - orient="records" - ) + df_process_jobs["query_key"] = df_process_jobs.loc[:, key_list].to_dict(orient="records") # Get parameters for all modalities all_modalities = df_process_jobs["recording_modality"].unique() @@ -575,24 +503,16 @@ def get_active_process_jobs(active=True): all_mods_df = list() # Get process df for each modality for this_modality in all_modalities: - this_mod_df = df_process_jobs.loc[ - df_process_jobs["recording_modality"] == this_modality, : - ].copy() + this_mod_df = df_process_jobs.loc[df_process_jobs["recording_modality"] == this_modality, :].copy() these_process_keys = this_mod_df["query_key"].to_list() - this_mod_program_selection_params = ( - RecProcessHandler.get_program_selection_params(this_modality) - ) + this_mod_program_selection_params = RecProcessHandler.get_program_selection_params(this_modality) if this_modality == "electrophysiology": - params_df = RecProcessHandler.get_ephys_params_jobs( - these_process_keys - ) + params_df = RecProcessHandler.get_ephys_params_jobs(these_process_keys) if this_modality == "imaging": - params_df = RecProcessHandler.get_imaging_params_jobs( - these_process_keys - ) + params_df = RecProcessHandler.get_imaging_params_jobs(these_process_keys) this_mod_df = this_mod_df.merge(params_df, on="job_id", how="left") this_mod_df = this_mod_df.merge( @@ -607,9 +527,7 @@ def get_active_process_jobs(active=True): df_process_jobs = all_mods_df[0].copy() for this_mod_df in all_mods_df[1:]: - df_process_jobs = pd.concat( - [df_process_jobs, this_mod_df], ignore_index=True - ) + df_process_jobs = pd.concat([df_process_jobs, this_mod_df], ignore_index=True) df_process_jobs = df_process_jobs.reset_index(drop=True) @@ -631,9 +549,7 @@ def get_ephys_params_jobs(rec_process_keys): # Get cluster param sets params_df = pd.DataFrame( ( - ephys_pipeline.ephys_element.ClusteringParamSet.proj( - "params", "clustering_method" - ) + ephys_pipeline.ephys_element.ClusteringParamSet.proj("params", "clustering_method") * recording_process.Processing.EphysParams.proj("paramset_idx") & rec_process_keys ).fetch(as_dict=True) @@ -652,27 +568,17 @@ def get_ephys_params_jobs(rec_process_keys): ephys_pipeline.ephys_element.PreClusterParamSteps * utility.smart_dj_join( ephys_pipeline.ephys_element.PreClusterParamSteps.Step, - ephys_pipeline.ephys_element.PreClusterParamSet.proj( - "precluster_method", "params" - ), - ) - * recording_process.Processing.EphysParams.proj( - "precluster_param_steps_id" + ephys_pipeline.ephys_element.PreClusterParamSet.proj("precluster_method", "params"), ) + * recording_process.Processing.EphysParams.proj("precluster_param_steps_id") & rec_process_keys ).fetch(as_dict=True) ) # Join precluster params for the same recording_process - preparams_df["preparams"] = preparams_df.apply( - lambda x: {x["precluster_method"]: x["params"]}, axis=1 - ) + preparams_df["preparams"] = preparams_df.apply(lambda x: {x["precluster_method"]: x["params"]}, axis=1) preparams_df = preparams_df.sort_values(by=["job_id", "step_number"]) - preparams_df = ( - preparams_df[["job_id", "preparams"]] - .groupby("job_id") - .agg(lambda x: list(x)) - ) + preparams_df = preparams_df[["job_id", "preparams"]].groupby("job_id").agg(lambda x: list(x)) preparams_df = preparams_df.reset_index() params_df = params_df.merge(preparams_df) @@ -692,9 +598,7 @@ def get_imaging_params_jobs(rec_process_keys): params_df = pd.DataFrame( ( - imaging_pipeline.imaging_element.ProcessingParamSet.proj( - "params", "processing_method" - ) + imaging_pipeline.imaging_element.ProcessingParamSet.proj("params", "processing_method") * recording_process.Processing.ImagingParams.proj("paramset_idx") & rec_process_keys ).fetch(as_dict=True) @@ -713,13 +617,9 @@ def get_imaging_params_jobs(rec_process_keys): imaging_pipeline.imaging_element.PreprocessParamSteps * utility.smart_dj_join( imaging_pipeline.imaging_element.PreprocessParamSteps.Step, - imaging_pipeline.imaging_element.PreprocessParamSet.proj( - "preprocess_method", "params" - ), - ) - * recording_process.Processing.ImagingParams.proj( - "preprocess_param_steps_id" + imaging_pipeline.imaging_element.PreprocessParamSet.proj("preprocess_method", "params"), ) + * recording_process.Processing.ImagingParams.proj("preprocess_param_steps_id") & rec_process_keys ).fetch(as_dict=True) ) @@ -727,24 +627,16 @@ def get_imaging_params_jobs(rec_process_keys): # If there is no preprocess steps for this jobs fill with empty values if preparams_df.shape[0] > 0: # Join precluster params for the same recording_process - preparams_df["preparams"] = preparams_df.apply( - lambda x: {x["preprocess_method"]: x["params"]}, axis=1 - ) + preparams_df["preparams"] = preparams_df.apply(lambda x: {x["preprocess_method"]: x["params"]}, axis=1) preparams_df = preparams_df.sort_values(by=["job_id", "step_number"]) - preparams_df = ( - preparams_df[["job_id", "preparams"]] - .groupby("job_id") - .agg(lambda x: list(x)) - ) + preparams_df = preparams_df[["job_id", "preparams"]].groupby("job_id").agg(lambda x: list(x)) preparams_df = preparams_df.reset_index() else: preparams_df = pd.DataFrame( ( imaging_pipeline.imaging_element.PreprocessParamSteps - * recording_process.Processing.ImagingParams.proj( - "preprocess_param_steps_id" - ) + * recording_process.Processing.ImagingParams.proj("preprocess_param_steps_id") & rec_process_keys ).fetch(as_dict=True) ) @@ -756,9 +648,7 @@ def get_imaging_params_jobs(rec_process_keys): return params_df @staticmethod - def update_status_pipeline( - recording_process_key_dict, status, update_field=None, update_value=None - ): + def update_status_pipeline(recording_process_key_dict, status, update_field=None, update_value=None): """ Update recording_process.Processing table status and optional task field Args: @@ -796,19 +686,11 @@ def update_job_id_log(job_id, current_status, next_status, error_info_dict): key["status_processing_id_new"] = next_status key["status_timestamp"] = date_time - if ( - error_info_dict["error_message"] is not None - and len(error_info_dict["error_message"]) >= 256 - ): + if error_info_dict["error_message"] is not None and len(error_info_dict["error_message"]) >= 256: error_info_dict["error_message"] = error_info_dict["error_message"][:255] - if ( - error_info_dict["error_exception"] is not None - and len(error_info_dict["error_exception"]) >= 4096 - ): - error_info_dict["error_exception"] = error_info_dict["error_exception"][ - :4095 - ] + if error_info_dict["error_exception"] is not None and len(error_info_dict["error_exception"]) >= 4096: + error_info_dict["error_exception"] = error_info_dict["error_exception"][:4095] key["error_exception"] = error_info_dict["error_exception"] key["error_message"] = error_info_dict["error_message"] @@ -842,25 +724,21 @@ def check_job_process_deletion(): df_inactive_jobs = df_inactive_jobs.merge(timestamp_jobs, on="job_id") # Get jobs that were processed in cluster - df_inactive_jobs["local_or_cluster"] = df_inactive_jobs[ - "program_selection_params" - ].map(lambda x: x["local_or_cluster"]) - df_inactive_jobs["process_cluster"] = df_inactive_jobs[ - "program_selection_params" - ].map(lambda x: x["process_cluster"]) + df_inactive_jobs["local_or_cluster"] = df_inactive_jobs["program_selection_params"].map( + lambda x: x["local_or_cluster"] + ) + df_inactive_jobs["process_cluster"] = df_inactive_jobs["program_selection_params"].map( + lambda x: x["process_cluster"] + ) # Update all local jobs - df_not_cluster = df_inactive_jobs.loc[ - df_inactive_jobs["local_or_cluster"] == "local" - ].reset_index(drop=True) + df_not_cluster = df_inactive_jobs.loc[df_inactive_jobs["local_or_cluster"] == "local"].reset_index(drop=True) if df_not_cluster.shape[0] > 0: # Update local jobs post-processed jobs df_not_cluster_post_processed = df_not_cluster.loc[ df_not_cluster["status_processing_id"] == config.JOB_STATUS_PROCESSED, : ] - df_not_cluster_post_processed = df_not_cluster_post_processed.reset_index( - drop=True - ) + df_not_cluster_post_processed = df_not_cluster_post_processed.reset_index(drop=True) for i in range(df_not_cluster_post_processed.shape[0]): RecProcessHandler.update_status_pipeline( df_not_cluster_post_processed.loc[i, "query_key"], @@ -891,25 +769,21 @@ def check_job_process_deletion(): ) # If no inactive job, after locals the end - df_inactive_jobs = df_inactive_jobs.loc[ - df_inactive_jobs["local_or_cluster"] == "cluster" - ].reset_index(drop=True) + df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs["local_or_cluster"] == "cluster"].reset_index( + drop=True + ) if df_inactive_jobs.shape[0] == 0: return # Get jobs to delete (finished or with error >= 7 days) - df_inactive_jobs["days_from_last_update"] = ( - now - df_inactive_jobs["max_date"] - ).dt.days + df_inactive_jobs["days_from_last_update"] = (now - df_inactive_jobs["max_date"]).dt.days df_inactive_jobs["ready_delete"] = 1 df_inactive_jobs.loc[ (df_inactive_jobs["status_processing_id"] == config.JOB_STATUS_ERROR_ID) & (df_inactive_jobs["days_from_last_update"] < 7), "ready_delete", ] = 0 - df_inactive_jobs = df_inactive_jobs.loc[ - df_inactive_jobs["ready_delete"] == 1, : - ] + df_inactive_jobs = df_inactive_jobs.loc[df_inactive_jobs["ready_delete"] == 1, :] # If no inactive job, after ready delete data return now if df_inactive_jobs.shape[0] == 0: @@ -939,13 +813,9 @@ def check_job_process_deletion(): ), axis=1, ) - df_inactive_jobs["processed_dir"] = df_inactive_jobs["processed_dir"].astype( - str - ) + df_inactive_jobs["processed_dir"] = df_inactive_jobs["processed_dir"].astype(str) df_inactive_jobs["processed_dir_deleted"] = 0 - df_inactive_jobs.loc[ - df_inactive_jobs["processed_dir"] == "0", "processed_dir_deleted" - ] = 1 + df_inactive_jobs.loc[df_inactive_jobs["processed_dir"] == "0", "processed_dir_deleted"] = 1 df_inactive_jobs = df_inactive_jobs.reset_index(drop=True) # Delete raw directories tiger @@ -999,9 +869,7 @@ def check_job_process_deletion(): ] df_post_error = df_post_error.reset_index(drop=True) for i in range(df_post_error.shape[0]): - RecProcessHandler.update_status_pipeline( - df_post_error.loc[i, "query_key"], config.JOB_STATUS_ERROR_DELETED - ) + RecProcessHandler.update_status_pipeline(df_post_error.loc[i, "query_key"], config.JOB_STATUS_ERROR_DELETED) RecProcessHandler.update_job_id_log( df_post_error.loc[i, "job_id"], config.JOB_STATUS_ERROR_ID, diff --git a/u19_pipeline/automatic_job/slurm_creator.py b/u19_pipeline/automatic_job/slurm_creator.py index c8a88300..9fd8c6b5 100644 --- a/u19_pipeline/automatic_job/slurm_creator.py +++ b/u19_pipeline/automatic_job/slurm_creator.py @@ -32,12 +32,8 @@ def generate_slurm_file(job_id, program_selection_params): label_rec_process = "job_id_" + str(job_id) slurm_dict["job-name"] = label_rec_process - slurm_dict["output"] = str( - pathlib.Path(cluster_vars["log_files_dir"], label_rec_process + ".log") - ) - slurm_dict["error"] = str( - pathlib.Path(cluster_vars["error_files_dir"], label_rec_process + ".log") - ) + slurm_dict["output"] = str(pathlib.Path(cluster_vars["log_files_dir"], label_rec_process + ".log")) + slurm_dict["error"] = str(pathlib.Path(cluster_vars["error_files_dir"], label_rec_process + ".log")) print("slurm_dict", slurm_dict) @@ -59,12 +55,8 @@ def generate_slurm_file(job_id, program_selection_params): status = config.system_process["SUCCESS"] slurm_destination = slurm_file_local_path else: - slurm_destination = pathlib.Path( - cluster_vars["slurm_files_dir"], slurm_file_name - ).as_posix() - status = transfer_slurm_file( - slurm_file_local_path, slurm_destination, cluster_vars - ) + slurm_destination = pathlib.Path(cluster_vars["slurm_files_dir"], slurm_file_name).as_posix() + status = transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars) print(status) print(slurm_destination) @@ -91,9 +83,7 @@ def queue_slurm_file( print("queue_slurm_file **********************************") processing_repository = program_selection_params["process_repository"] - repository_dir = pathlib.Path( - cluster_vars[modality + "_process_dir"], processing_repository - ).as_posix() + repository_dir = pathlib.Path(cluster_vars[modality + "_process_dir"], processing_repository).as_posix() command = [ "ssh", @@ -193,17 +183,9 @@ def create_slurm_params_file(slurm_dict): for slurm_param in slurm_dict.keys(): if isinstance(slurm_dict[slurm_param], list): for list_param in slurm_dict[slurm_param]: - text_dict += ( - "#SBATCH --" + str(slurm_param) + "=" + str(list_param) + "\n" - ) + text_dict += "#SBATCH --" + str(slurm_param) + "=" + str(list_param) + "\n" else: - text_dict += ( - "#SBATCH --" - + str(slurm_param) - + "=" - + str(slurm_dict[slurm_param]) - + "\n" - ) + text_dict += "#SBATCH --" + str(slurm_param) + "=" + str(slurm_dict[slurm_param]) + "\n" return text_dict diff --git a/u19_pipeline/automatic_job/tigress2pni.py b/u19_pipeline/automatic_job/tigress2pni.py index 4ebbae98..e8f1722e 100755 --- a/u19_pipeline/automatic_job/tigress2pni.py +++ b/u19_pipeline/automatic_job/tigress2pni.py @@ -110,11 +110,7 @@ def setup_transfer_client(transfer_tokens): r = transfer_client.endpoint_autoactivate(DESTINATION_ENDPOINT) except GlobusAPIError as ex: if ex.http_status == 401: - sys.exit( - "Refresh token has expired. " - "Please delete the `tokens` object from " - f"{DATA_FILE} and try again." - ) + sys.exit(f"Refresh token has expired. Please delete the `tokens` object from {DATA_FILE} and try again.") else: raise ex return transfer_client @@ -170,11 +166,7 @@ def main(): task_data = data["task"] task = transfer.get_task(task_data["task_id"]) if task["status"] not in PREVIOUS_TASK_RUN_CASES: - print( - "The last transfer status is {}, skipping run...".format( - task["status"] - ) - ) + print("The last transfer status is {}, skipping run...".format(task["status"])) sys.exit(1) except KeyError: # Ignore if there is no previous task diff --git a/u19_pipeline/copy_table.py b/u19_pipeline/copy_table.py index 22d0ed5a..3b9b8d41 100644 --- a/u19_pipeline/copy_table.py +++ b/u19_pipeline/copy_table.py @@ -6,17 +6,13 @@ from u19_pipeline import subject from u19_pipeline.temp import acquisition, behavior, imaging, meso, meso_analysis -acquisition_original = dj.create_virtual_module( - "acquisition_original", "u19_acquisition" -) +acquisition_original = dj.create_virtual_module("acquisition_original", "u19_acquisition") behavior_original = dj.create_virtual_module("behavior_original", "u19_behavior") meso_original = dj.create_virtual_module("meso_original", "u19_meso") -meso_analysis_original = dj.create_virtual_module( - "meso_analysis_original", "u19_meso_analysis" -) +meso_analysis_original = dj.create_virtual_module("meso_analysis_original", "u19_meso_analysis") imaging_original = dj.create_virtual_module("imaging_original", "u19_imaging") @@ -56,14 +52,12 @@ def copy_table(target_schema, src_schema, table_name, **kwargs): def copy_acquisition_tables(): acquisition.SessionTemp.insert( - (acquisition_original.Session - acquisition.SessionTemp.proj()) - & acquisition.SessionStarted.proj(), + (acquisition_original.Session - acquisition.SessionTemp.proj()) & acquisition.SessionStarted.proj(), skip_duplicates=True, ) acquisition.DataDirectoryTemp.insert( - (acquisition_original.DataDirectory - acquisition.DataDirectoryTemp.proj()) - & acquisition.SessionStarted.proj(), + (acquisition_original.DataDirectory - acquisition.DataDirectoryTemp.proj()) & acquisition.SessionStarted.proj(), allow_direct_insert=True, ) @@ -85,18 +79,12 @@ def copy_behavior_tables(): if "." in table: if table == "TowersBlock.Trial": for subj in tqdm((subject.Subject & behavior.TowersBlock).fetch("KEY")): - behavior.TowersBlock.Trial.insert( - behavior_original.TowersBlock.Trial & subj, skip_duplicates=True - ) + behavior.TowersBlock.Trial.insert(behavior_original.TowersBlock.Trial & subj, skip_duplicates=True) else: if table == "TowersBlockVideo": - for subj in tqdm( - (subject.Subject & behavior.TowersBlock).fetch("KEY") - ): + for subj in tqdm((subject.Subject & behavior.TowersBlock).fetch("KEY")): behavior.TowersBlockTrialVideo.insert( - behavior_original.TowersBlockTrialVideo - & subj - & behavior.TowersBlock, + behavior_original.TowersBlockTrialVideo & subj & behavior.TowersBlock, skip_duplicates=True, ) else: diff --git a/u19_pipeline/ephys_pipeline.py b/u19_pipeline/ephys_pipeline.py index 60bbc87f..ecb90659 100644 --- a/u19_pipeline/ephys_pipeline.py +++ b/u19_pipeline/ephys_pipeline.py @@ -84,18 +84,14 @@ def get_ephys_root_data_dir(): def get_session_directory(session_key): # root_dir = get_ephys_root_data_dir() - session_dir = pathlib.Path( - (recording.Recording & session_key).fetch1("recording_directory") - ).as_posix() + session_dir = pathlib.Path((recording.Recording & session_key).fetch1("recording_directory")).as_posix() # session_dir = pathlib.Path(root_dir, session_dir).as_posix() return session_dir def get_full_session_directory(recording_key): - session_dir = find_full_path( - [get_ephys_root_data_dir()[0]], get_session_directory(recording_key) - ) + session_dir = find_full_path([get_ephys_root_data_dir()[0]], get_session_directory(recording_key)) print("ephys dir:", session_dir) nidq_session = list(session_dir.glob("*nidq.bin*")) @@ -189,35 +185,26 @@ def create_lfp_trace(cat_gt_script, recording_directory, probe_directory): def get_spikeglx_meta_filepath(ephys_recording_key): # attempt to retrieve from EphysRecording.EphysFile spikeglx_meta_filepath = ( - ephys_element.EphysRecording.EphysFile - & ephys_recording_key - & 'file_path LIKE "%.ap.meta"' + ephys_element.EphysRecording.EphysFile & ephys_recording_key & 'file_path LIKE "%.ap.meta"' ).fetch1("file_path") print("ephys_recording_key", ephys_recording_key) print("spikeglx_meta_filepath", spikeglx_meta_filepath) print("get_ephys_root_data_dir", get_ephys_root_data_dir()) - spikeglx_meta_filepath = find_full_path( - get_ephys_root_data_dir(), spikeglx_meta_filepath - ) + spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(), spikeglx_meta_filepath) print("spikeglx_meta_filepath", spikeglx_meta_filepath) try: - spikeglx_meta_filepath = find_full_path( - get_ephys_root_data_dir(), spikeglx_meta_filepath - ) + spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(), spikeglx_meta_filepath) print("spikeglx_meta_filepath", spikeglx_meta_filepath) except FileNotFoundError: # if not found, search in session_dir again if not spikeglx_meta_filepath.exists(): - session_dir = find_full_path( - get_ephys_root_data_dir(), get_session_directory(ephys_recording_key) - ) + session_dir = find_full_path(get_ephys_root_data_dir(), get_session_directory(ephys_recording_key)) inserted_probe_serial_number = ( - ephys_element.ProbeInsertion * ephys_element.probe.Probe - & ephys_recording_key + ephys_element.ProbeInsertion * ephys_element.probe.Probe & ephys_recording_key ).fetch1("probe") spikeglx_meta_filepaths = [fp for fp in session_dir.rglob("*.ap.meta")] @@ -227,9 +214,7 @@ def get_spikeglx_meta_filepath(ephys_recording_key): spikeglx_meta_filepath = meta_filepath break else: - raise FileNotFoundError( - f"No SpikeGLX data found for probe insertion: {ephys_recording_key}" - ) + raise FileNotFoundError(f"No SpikeGLX data found for probe insertion: {ephys_recording_key}") return spikeglx_meta_filepath @@ -266,9 +251,7 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): return # Get sampling rate and calculate channels and samples - nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate( - full_session_path - ) + nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate(full_session_path) nChan = int(nidq_meta["nSavedChans"]) num_samples = int(int(nidq_meta["fileSizeBytes"]) / (2 * nChan)) @@ -279,12 +262,10 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): print("No sync data was sound for this session") return - trial_index_nidq_virmen, iteration_index_nidq_virmen = ( - ephys_utils.get_full_vector_samples( - sync_data["iteration_idx_vector_from_virmen"], - nidq_sampling_rate, - num_samples, - ) + trial_index_nidq_virmen, iteration_index_nidq_virmen = ephys_utils.get_full_vector_samples( + sync_data["iteration_idx_vector_from_virmen"], + nidq_sampling_rate, + num_samples, ) trial_index_nidq, iteration_index_nidq = ephys_utils.get_full_vector_samples( @@ -298,10 +279,8 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): sync_data["iteration_idx_vector"], nidq_sampling_rate ) - trial_times_ind_virmen, trial_times_full_virmen = ( - ephys_utils.get_time_vector_as_behavior( - sync_data["iteration_idx_vector_from_virmen"], nidq_sampling_rate - ) + trial_times_ind_virmen, trial_times_full_virmen = ephys_utils.get_time_vector_as_behavior( + sync_data["iteration_idx_vector_from_virmen"], nidq_sampling_rate ) # Store data @@ -317,9 +296,7 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): for this_key in all_vectors: print("this_key", this_key) print(all_vectors[this_key].shape) - all_vectors[this_key] = np.concatenate( - ([x for x in all_vectors[this_key]]), axis=0 - ) + all_vectors[this_key] = np.concatenate(([x for x in all_vectors[this_key]]), axis=0) print(all_vectors[this_key].shape) all_vectors["trial_index_nidq_virmen"] = trial_index_nidq_virmen @@ -399,34 +376,26 @@ def make(self, key, **kwargs): print("after reading behavior data") # 1: load meta data, and the content of the NIDAQ file. Its content is digital. - nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate( - ephys_session_fullpath - ) + nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate(ephys_session_fullpath) - trial_pulse_signal, iteration_pulse_signal = ( - ephys_utils.load_trial_iteration_signals( - ephys_session_fullpath, nidq_meta - ) + trial_pulse_signal, iteration_pulse_signal = ephys_utils.load_trial_iteration_signals( + ephys_session_fullpath, nidq_meta ) print("after reading spikeglx data") # Synchronize between pulses and get iteration # vector for each sample - recent_recording = behavior_key["session_date"] > datetime.date( - 2021, 6, 1 - ) # Everything past June 1 2021 + recent_recording = behavior_key["session_date"] > datetime.date(2021, 6, 1) # Everything past June 1 2021 if recent_recording: # New synchronization method: digital_array[1,2] contain pulses for trial and frame number. mode = None - iteration_dict = ( - ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses( - trial_pulse_signal, - iteration_pulse_signal, - nidq_sampling_rate, - behavior_time.shape[0], - behavior_time, - mode, - ) + iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses( + trial_pulse_signal, + iteration_pulse_signal, + nidq_sampling_rate, + behavior_time.shape[0], + behavior_time, + mode, ) else: # Old synchronization: digital_array[0:7] contain a digital word that counts the virmen frames. @@ -435,9 +404,7 @@ def make(self, key, **kwargs): # Check # of trials (from database record of behavior in `behavior_time`) and iterations (extracted from NIDAQ in `iter_start_idx`) match trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ( - ephys_utils.assert_iteration_samples_count( - iteration_dict["iter_start_idx"], behavior_time - ) + ephys_utils.assert_iteration_samples_count(iteration_dict["iter_start_idx"], behavior_time) ) print("metrics to evaluate...") @@ -456,10 +423,8 @@ def make(self, key, **kwargs): ) if status == 1: - iteration_dict["trial_start_idx"] = ( - ephys_utils.get_index_trial_vector_from_iteration( - iteration_dict["iter_start_idx"] - ) + iteration_dict["trial_start_idx"] = ephys_utils.get_index_trial_vector_from_iteration( + iteration_dict["iter_start_idx"] ) # Failed sync by a lot, error @@ -480,12 +445,8 @@ def make(self, key, **kwargs): print("after all main ehpys fix sync code", status_regular, status_fix) if status_regular > 0 or status_fix > 0: - dictionary_sync_data["trial_idx_vector"] = iteration_dict[ - "trial_start_idx" - ] - dictionary_sync_data["iteration_idx_vector"] = iteration_dict[ - "iter_start_idx" - ] + dictionary_sync_data["trial_idx_vector"] = iteration_dict["trial_start_idx"] + dictionary_sync_data["iteration_idx_vector"] = iteration_dict["iter_start_idx"] else: dictionary_sync_data["trial_idx_vector"] = [] dictionary_sync_data["iteration_idx_vector"] = [] @@ -500,12 +461,8 @@ def make(self, key, **kwargs): behavior_time, ) - dictionary_sync_data["trial_idx_vector_from_virmen"] = iteration_dict[ - "trial_start_idx_virmen" - ] - dictionary_sync_data["iteration_idx_vector_from_virmen"] = iteration_dict[ - "iter_start_idx_virmen" - ] + dictionary_sync_data["trial_idx_vector_from_virmen"] = iteration_dict["trial_start_idx_virmen"] + dictionary_sync_data["iteration_idx_vector_from_virmen"] = iteration_dict["iter_start_idx_virmen"] final_key = dict( key, @@ -537,19 +494,13 @@ def insert_imec_sampling_rate(self, key, session_dir): here = ephys_element.ProbeInsertion & key for probe_insertion in here.fetch("KEY"): # imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.bin'.format(probe_insertion['insertion_number']))) - imec_bin_filepath = list( - session_dir.glob( - "*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]) - ) - ) + imec_bin_filepath = list(session_dir.glob("*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]))) if len(imec_bin_filepath) == 1: # find the binary file to get meta data imec_bin_filepath = imec_bin_filepath[0] else: # if this fails, get the ap.meta file. imec_bin_filepath = list( - session_dir.glob( - "*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]) - ) + session_dir.glob("*imec{}/*.ap.meta".format(probe_insertion["insertion_number"])) ) if len(imec_bin_filepath) == 1: s = str(imec_bin_filepath[0]) @@ -558,6 +509,4 @@ def insert_imec_sampling_rate(self, key, session_dir): raise NameError("No imec meta file found.") imec_meta = readSGLX.readMeta(imec_bin_filepath) - self.ImecSamplingRate.insert1( - dict(probe_insertion, ephys_sampling_rate=imec_meta["imSampRate"]) - ) + self.ImecSamplingRate.insert1(dict(probe_insertion, ephys_sampling_rate=imec_meta["imSampRate"])) diff --git a/u19_pipeline/ephys_sync.py b/u19_pipeline/ephys_sync.py index eb6e11b7..327e46b3 100644 --- a/u19_pipeline/ephys_sync.py +++ b/u19_pipeline/ephys_sync.py @@ -43,9 +43,7 @@ def make(self, key): nidq_sampling_rate = readSGLX.SampRate(nidq_meta) if nidq_meta["typeThis"] == "nidq": - digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file( - nidq_bin_full_path, nidq_meta - ) + digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta) else: digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file( nidq_bin_full_path, nidq_meta, d_line_list=[1, 2] @@ -53,26 +51,20 @@ def make(self, key): # Synchronize between pulses and get iteration # vector for each sample mode = "counter_bit0" - iteration_dict = ( - ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses( - digital_array[1, :], - digital_array[2, :], - nidq_sampling_rate, - behavior_time.shape[0], - mode, - ) + iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses( + digital_array[1, :], + digital_array[2, :], + nidq_sampling_rate, + behavior_time.shape[0], + mode, ) # Check # of trials and iterations match - status = ephys_utils.assert_iteration_samples_count( - iteration_dict["iter_start_idx"], behavior_time - ) + status = ephys_utils.assert_iteration_samples_count(iteration_dict["iter_start_idx"], behavior_time) # They didn't match, try counter method (if available) if (not status) and (digital_array.shape[0] > 3): - [framenumber_in_trial, trialnumber] = ( - ephys_utils.behavior_sync_frame_counter_method( - digital_array, behavior_time, thissession, nidq_sampling_rate, 3, 5 - ) + [framenumber_in_trial, trialnumber] = ephys_utils.behavior_sync_frame_counter_method( + digital_array, behavior_time, thissession, nidq_sampling_rate, 3, 5 ) iteration_dict["framenumber_vector_samples"] = framenumber_in_trial iteration_dict["trialnumber_vector_samples"] = trialnumber @@ -96,19 +88,13 @@ def insert_imec_sampling_rate(self, key, session_dir): here = ephys_element.ProbeInsertion & key for probe_insertion in here.fetch("KEY"): # imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.bin'.format(probe_insertion['insertion_number']))) - imec_bin_filepath = list( - session_dir.glob( - "*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]) - ) - ) + imec_bin_filepath = list(session_dir.glob("*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]))) if len(imec_bin_filepath) == 1: # find the binary file to get meta data imec_bin_filepath = imec_bin_filepath[0] else: # if this fails, get the ap.meta file. imec_bin_filepath = list( - session_dir.glob( - "*imec{}/*.ap.meta".format(probe_insertion["insertion_number"]) - ) + session_dir.glob("*imec{}/*.ap.meta".format(probe_insertion["insertion_number"])) ) if len(imec_bin_filepath) == 1: s = str(imec_bin_filepath[0]) @@ -117,9 +103,7 @@ def insert_imec_sampling_rate(self, key, session_dir): raise NameError("No imec meta file found.") imec_meta = readSGLX.readMeta(imec_bin_filepath) - self.ImecSamplingRate.insert1( - dict(probe_insertion, ephys_sampling_rate=imec_meta["imSampRate"]) - ) + self.ImecSamplingRate.insert1(dict(probe_insertion, ephys_sampling_rate=imec_meta["imSampRate"])) @schema @@ -143,9 +127,9 @@ def make(self, key): self.insert1(key) - nidq_sampling_rate, iteration_index_nidq = ( - BehaviorSync * BehaviorSync.ImecSamplingRate & key - ).fetch1("nidq_sampling_rate", "iteration_index_nidq") + nidq_sampling_rate, iteration_index_nidq = (BehaviorSync * BehaviorSync.ImecSamplingRate & key).fetch1( + "nidq_sampling_rate", "iteration_index_nidq" + ) key_session = key.copy() del key_session["insertion_number"] @@ -183,20 +167,12 @@ def make(self, key): unit_spike_counts = [] for unit_key in (ephys_element.CuratedClustering.Unit & key).fetch("KEY"): - spike_times = (ephys_element.CuratedClustering.Unit & unit_key).fetch1( - "spike_times" - ) + spike_times = (ephys_element.CuratedClustering.Unit & unit_key).fetch1("spike_times") # vector with length n_iterations + 1 - spike_counts_iteration = np.bincount( - np.digitize(spike_times, iteration_times) - ) + spike_counts_iteration = np.bincount(np.digitize(spike_times, iteration_times)) - firing_rate_before_first_iteration = ( - spike_counts_iteration[0] / iteration_times[0] - ) - firing_rate_after_last_iteration = spike_counts_iteration[-1] / ( - t_end - iteration_times[-1] - ) + firing_rate_before_first_iteration = spike_counts_iteration[0] / iteration_times[0] + firing_rate_after_last_iteration = spike_counts_iteration[-1] / (t_end - iteration_times[-1]) # remove the first and last entries spike_counts_iteration = np.delete(spike_counts_iteration, [0, -1]) diff --git a/u19_pipeline/imaging_pipeline.py b/u19_pipeline/imaging_pipeline.py index adf7ae22..4f6ad245 100644 --- a/u19_pipeline/imaging_pipeline.py +++ b/u19_pipeline/imaging_pipeline.py @@ -72,9 +72,7 @@ class AcquiredTiff(dj.Imported): def make(self, key, test_mode=False): - scan_info = ( - ImagingPipelineSession * recording.Recording * lab.Location & key - ).fetch1() + scan_info = (ImagingPipelineSession * recording.Recording * lab.Location & key).fetch1() imaging_root = dj.config["custom"]["imaging_root_data_dir"][0] scan_directory = Path(imaging_root) / scan_info["recording_directory"] @@ -113,9 +111,7 @@ def make(self, key, test_mode=False): rec_info["nfovs"] = tu.get_nfovs(rec_info, is_mesoscope) - last_good_file, cumulative_frames = tu.get_last_good_frame( - frames_per_file, tif_dir - ) + last_good_file, cumulative_frames = tu.get_last_good_frame(frames_per_file, tif_dir) rec_info["nframes_good"] = cumulative_frames[last_good_file] rec_info["last_good_file"] = last_good_file + 1 @@ -125,34 +121,26 @@ def make(self, key, test_mode=False): if is_compressed: tu.remove_compressed_videos(fl, scan_directory) - scan_info_key = tu.create_scan_info_key( - key, rec_info, scan_info["recording_directory"] - ) + scan_info_key = tu.create_scan_info_key(key, rec_info, scan_info["recording_directory"]) if not test_mode: self.insert1(scan_info_key, allow_direct_insert=True) if is_mesoscope: - tiffsplit_mesoscope_keys, tiff_splitfiles_mesoscope_keys = ( - tu.get_fov_mesoscope( - fl, - key, - skip_parsing, - imheader, - rec_info, - basename, - cumulative_frames, - scan_info, - imaging_root, - ) + tiffsplit_mesoscope_keys, tiff_splitfiles_mesoscope_keys = tu.get_fov_mesoscope( + fl, + key, + skip_parsing, + imheader, + rec_info, + basename, + cumulative_frames, + scan_info, + imaging_root, ) for i in range(len(tiffsplit_mesoscope_keys)): if not test_mode: - TiffSplit.insert( - tiffsplit_mesoscope_keys[i], allow_direct_insert=True - ) - TiffSplit.File.insert( - tiff_splitfiles_mesoscope_keys[i], allow_direct_insert=True - ) + TiffSplit.insert(tiffsplit_mesoscope_keys[i], allow_direct_insert=True) + TiffSplit.File.insert(tiff_splitfiles_mesoscope_keys[i], allow_direct_insert=True) if test_mode: return ( @@ -166,9 +154,7 @@ def make(self, key, test_mode=False): tiffsplitfile_2photon_key = tu.get_fovfile_photonmicro(key, fl, imheader) if not test_mode: TiffSplit.insert([tiffsplit_2photon_key], allow_direct_insert=True) - TiffSplit.File.insert( - tiffsplitfile_2photon_key, allow_direct_insert=True - ) + TiffSplit.File.insert(tiffsplitfile_2photon_key, allow_direct_insert=True) if test_mode: return scan_info_key, tiffsplit_2photon_key, tiffsplitfile_2photon_key @@ -249,9 +235,7 @@ class File(dj.Part): # 1. Schema names ---------------------------------------------------------------------- scan_schema_name = dj.config["custom"]["database.prefix"] + "pipeline_scan_element" -imaging_schema_name = ( - dj.config["custom"]["database.prefix"] + "pipeline_imaging_element" -) +imaging_schema_name = dj.config["custom"]["database.prefix"] + "pipeline_imaging_element" # 2. Upstream tables ------------------------------------------------------------------- @@ -331,9 +315,9 @@ def get_calcium_imaging_files(scan_key, acq_software): def get_processed_dir(processing_task_key, process_method): sess_key = (ImagingPipelineSession & processing_task_key).fetch1("KEY") - bucket_scan_dir = ( - TiffSplit & sess_key & {"tiff_split": processing_task_key["scan_id"]} - ).fetch1("tiff_split_directory") + bucket_scan_dir = (TiffSplit & sess_key & {"tiff_split": processing_task_key["scan_id"]}).fetch1( + "tiff_split_directory" + ) user_id = (subject.Subject & processing_task_key).fetch1("user_id") sess_dir = find_full_path(get_imaging_root_data_dir(), bucket_scan_dir) diff --git a/u19_pipeline/lab.py b/u19_pipeline/lab.py index 40c1dde5..3cfd2f02 100644 --- a/u19_pipeline/lab.py +++ b/u19_pipeline/lab.py @@ -295,9 +295,7 @@ def get_local_path(self, path, local_os=None): for iglob, glob in enumerate(globs.fetch("KEY")): mapping[iglob].append(glob["global_path"]) for system in systems: - mapping[iglob].append( - (self & glob & {"system": system}).fetch1("local_path") - ) + mapping[iglob].append((self & glob & {"system": system}).fetch1("local_path")) mapping = np.asarray(mapping) @@ -332,13 +330,9 @@ def get_local_path2(self, bucket_path): path_df = path_df[path_df["system"] == system] # Search in path which of the main buckets we are referring from - path_df["idx_global_path"] = path_df["global_path"].apply( - lambda x: bucket_path.find(x) - ) + path_df["idx_global_path"] = path_df["global_path"].apply(lambda x: bucket_path.find(x)) path_df = path_df[path_df["idx_global_path"] != -1] - path_df = path_df[ - path_df["idx_global_path"] == path_df["idx_global_path"].min() - ].squeeze() + path_df = path_df[path_df["idx_global_path"] == path_df["idx_global_path"].min()].squeeze() path_df = path_df.to_dict() # Remove bucket "base" dir from path diff --git a/u19_pipeline/meso.py b/u19_pipeline/meso.py index 3b9449ef..ece2a913 100644 --- a/u19_pipeline/meso.py +++ b/u19_pipeline/meso.py @@ -357,12 +357,8 @@ class Trace(dj.Imported): key = {"mcorr_method": "LinearNormalized", "mc_parameter_set_id": 1} parameters = [ - dict( - **key, mc_parameter_name="mc_max_shift", mc_parameter_value=np.array([15.0]) - ), - dict( - **key, mc_parameter_name="mc_max_iter", mc_parameter_value=np.array([5.0]) - ), + dict(**key, mc_parameter_name="mc_max_shift", mc_parameter_value=np.array([15.0])), + dict(**key, mc_parameter_name="mc_max_iter", mc_parameter_value=np.array([5.0])), dict(**key, mc_parameter_name="mc_stop_below_shift", mc_parameter_value=0.3), dict(**key, mc_parameter_name="mc_black_tolerance", mc_parameter_value=-1.0), dict(**key, mc_parameter_name="mc_median_rebin", mc_parameter_value=10.0), diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index eb6c2719..57a7aa1d 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -105,22 +105,18 @@ def get_default_params_rec_process(recording_processes, default_params_record_df this_params_rec_process["job_id"] = i["job_id"] this_fragment = i["fragment_number"] - this_params_rec_process["preprocess_param_steps_id"] = ( - DefaultParams.get_corresponding_param( - default_params_record_df, - this_fragment, - "default_same_preparams_all", - "preprocess_param_steps_id", - ) + this_params_rec_process["preprocess_param_steps_id"] = DefaultParams.get_corresponding_param( + default_params_record_df, + this_fragment, + "default_same_preparams_all", + "preprocess_param_steps_id", ) - this_params_rec_process["paramset_idx"] = ( - DefaultParams.get_corresponding_param( - default_params_record_df, - this_fragment, - "default_same_params_all", - "paramset_idx", - ) + this_params_rec_process["paramset_idx"] = DefaultParams.get_corresponding_param( + default_params_record_df, + this_fragment, + "default_same_params_all", + "paramset_idx", ) params_rec_process.append(this_params_rec_process) @@ -128,9 +124,7 @@ def get_default_params_rec_process(recording_processes, default_params_record_df return params_rec_process @staticmethod - def get_corresponding_param( - default_params_record_df, this_fragment, default_label, param_label - ): + def get_corresponding_param(default_params_record_df, this_fragment, default_label, param_label): "Get corresponding param (preprocess_param_steps_id or paramset_idx) for this fragment" "default_label = default_same_preparams_all / default_same_params_all " @@ -143,9 +137,7 @@ def get_corresponding_param( else: if default_params_record_df.loc[0, default_label] == 1: - this_fragment_preprocess_param_steps_id = default_params_record_df.loc[ - 0, param_label - ] + this_fragment_preprocess_param_steps_id = default_params_record_df.loc[0, param_label] else: this_fragment_preprocess_param_steps_id = default_params_record_df.loc[ default_params_record_df["fragment_number"] == this_fragment, @@ -153,12 +145,8 @@ def get_corresponding_param( ] # If there is no list id for this specific fragment, get default one if this_fragment_preprocess_param_steps_id.shape[0] == 0: - this_fragment_preprocess_param_steps_id = ( - default_params_record_df.loc[0, param_label] - ) + this_fragment_preprocess_param_steps_id = default_params_record_df.loc[0, param_label] else: - this_fragment_preprocess_param_steps_id = ( - this_fragment_preprocess_param_steps_id.values[0] - ) + this_fragment_preprocess_param_steps_id = this_fragment_preprocess_param_steps_id.values[0] return this_fragment_preprocess_param_steps_id diff --git a/u19_pipeline/recording_process.py b/u19_pipeline/recording_process.py index 22fe38f0..864ceb07 100644 --- a/u19_pipeline/recording_process.py +++ b/u19_pipeline/recording_process.py @@ -77,13 +77,9 @@ def insert_recording_process(self, fragment_keys, fragment_fieldname): for idx, fragment_key in enumerate(fragment_keys): this_recprocess_key.append(dict()) this_recprocess_key[idx]["recording_id"] = fragment_key["recording_id"] - this_recprocess_key[idx]["fragment_number"] = fragment_key[ - fragment_fieldname - ] + this_recprocess_key[idx]["fragment_number"] = fragment_key[fragment_fieldname] this_recprocess_key[idx]["status_processing_id"] = 0 - this_recprocess_key[idx]["recording_process_pre_path"] = fragment_key[ - "recording_process_pre_path" - ] + this_recprocess_key[idx]["recording_process_pre_path"] = fragment_key["recording_process_pre_path"] self.insert(this_recprocess_key) diff --git a/u19_pipeline/reference.py b/u19_pipeline/reference.py index d72b574f..1285dd4d 100644 --- a/u19_pipeline/reference.py +++ b/u19_pipeline/reference.py @@ -25,9 +25,7 @@ class VirusSource(dj.Lookup): definition = """ virus_source : varchar(32) """ - contents = zip( - ["UNC", "UPenn", "Addgene", "MIT", "Stanford", "Princeton", "custom"] - ) + contents = zip(["UNC", "UPenn", "Addgene", "MIT", "Stanford", "Princeton", "custom"]) @schema diff --git a/u19_pipeline/utility.py b/u19_pipeline/utility.py index 818b2cf0..14391c03 100644 --- a/u19_pipeline/utility.py +++ b/u19_pipeline/utility.py @@ -45,9 +45,7 @@ def basic_dj_configuration(dj): elif sys.platform == "linux" or sys.platform == "linux2": ext_storage_location = "/mnt/u19_dj/external_dj_blobs" - dj.config["stores"] = { - "extstorage": {"location": ext_storage_location, "protocol": "file"} - } + dj.config["stores"] = {"extstorage": {"location": ext_storage_location, "protocol": "file"}} def get_network_path(path_name): @@ -78,9 +76,7 @@ def get_network_path(path_name): else: key["system"] = "linux" - lab = dj.create_virtual_module( - "lab", dj.config["custom"]["database.prefix"] + "lab" - ) + lab = dj.create_virtual_module("lab", dj.config["custom"]["database.prefix"] + "lab") network_path = (lab.Path & key).fetch1(*field_get) return network_path @@ -152,9 +148,7 @@ def psychFit(deltaBins, numR, numL, choices): if choices[iTrial] == 2: numRight[trialBin[iTrial]] = numRight[trialBin[iTrial]] + 1 - trialDelta[trialBin[iTrial]] = ( - trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial] - ) + trialDelta[trialBin[iTrial]] = trialDelta[trialBin[iTrial]] + nCues_RminusL[iTrial] with np.errstate(divide="ignore", invalid="ignore"): trialDelta = np.true_divide(trialDelta, numTrials) @@ -165,12 +159,8 @@ def psychFit(deltaBins, numR, numL, choices): numRight_nz = numRight[~idx_zero] # (Binomial proportion confidence interval given k successes, n trials) - phat_nz = binom_conf_interval( - numRight_nz, numTrials_nz, confidence_level=0, interval="jeffreys" - ) - pci_nz = binom_conf_interval( - numRight_nz, numTrials_nz, confidence_level=1 - 0.1587, interval="jeffreys" - ) + phat_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=0, interval="jeffreys") + pci_nz = binom_conf_interval(numRight_nz, numTrials_nz, confidence_level=1 - 0.1587, interval="jeffreys") # Correct confidence intervals and expected outcomes for bins with no trials (ci = [0 1], hat = 0.5) phat_nz = phat_nz[0] @@ -250,12 +240,8 @@ def translate_choice_trials_cues(session_df): # If we are translating towers task cues if "cue_presence_left" in session_df.columns: - session_df["cue_presence_left"] = session_df["cue_presence_left"].apply( - lambda x: np.count_nonzero(x) - ) - session_df["cue_presence_right"] = session_df["cue_presence_right"].apply( - lambda x: np.count_nonzero(x) - ) + session_df["cue_presence_left"] = session_df["cue_presence_left"].apply(lambda x: np.count_nonzero(x)) + session_df["cue_presence_right"] = session_df["cue_presence_right"].apply(lambda x: np.count_nonzero(x)) # If we are translating puff task cues elif "num_puffs_received_r" in session_df.columns: session_df["cue_presence_left"] = session_df["num_puffs_received_l"] @@ -313,9 +299,7 @@ def numpy_array_to_dict(np_array, as_int=True): # Get not empty columns to extract fist value of only those columns s = out_dict.applymap(lambda x: x.shape[0]) not_empty_columns = s.loc[:, ~(s == 0).any()].columns.to_list() - out_dict = out_dict.apply( - lambda x: x[0].flatten() if x.name in not_empty_columns else x - ) + out_dict = out_dict.apply(lambda x: x[0].flatten() if x.name in not_empty_columns else x) if not isinstance(out_dict, pd.DataFrame): out_dict = out_dict.to_frame() @@ -323,9 +307,7 @@ def numpy_array_to_dict(np_array, as_int=True): s = out_dict.applymap(lambda x: x.size).T real_array_columns = s.loc[:, (s > 1).any()].columns.to_list() out_dict = out_dict.T - out_dict = out_dict.apply( - lambda x: x[0] if x.name not in real_array_columns else x, axis=0 - ) + out_dict = out_dict.apply(lambda x: x[0] if x.name not in real_array_columns else x, axis=0) columns = out_dict.columns.copy() out_dict = out_dict.squeeze() diff --git a/u19_pipeline/utils/ephys_fix_sync_code.py b/u19_pipeline/utils/ephys_fix_sync_code.py index b7a51cf3..82c13970 100644 --- a/u19_pipeline/utils/ephys_fix_sync_code.py +++ b/u19_pipeline/utils/ephys_fix_sync_code.py @@ -13,10 +13,7 @@ def get_shift_vector( diff_size = np.abs(synced_time_vector.shape[0] - behavior_time_vector.shape[0]) - baseline_diff = ( - synced_time_vector[initial_sample:base_size] - - behavior_time_vector[initial_sample:base_size] - ) + baseline_diff = synced_time_vector[initial_sample:base_size] - behavior_time_vector[initial_sample:base_size] """ while(1): @@ -80,9 +77,7 @@ def get_shift_vector( new_sign = -1 if new_sign != sign: - if np.abs(median_ori - median_diff) < np.abs( - median_now - median_diff - ): + if np.abs(median_ori - median_diff) < np.abs(median_now - median_diff): vec_shift[i] = 0 else: vec_shift[i] = j * sign @@ -95,9 +90,7 @@ def get_shift_vector( new_synced_time_vector = synced_time_vector.copy() mid_point = int(initial_sample + base_size / 2) for i in range(vec_shift.shape[0]): - new_synced_time_vector[mid_point + i] = synced_time_vector[ - mid_point + i + vec_shift[i] - ] + new_synced_time_vector[mid_point + i] = synced_time_vector[mid_point + i + vec_shift[i]] idx_end = mid_point + vec_shift.shape[0] - 1 for i in range(idx_end, new_synced_time_vector.shape[0]): @@ -116,9 +109,7 @@ def get_shift_vector( return new_synced_time_vector, vec_shift, [min_diff, median_diff, max_diff] -def fix_shifted_sync_vector( - synced_time_vector, behavior_time_vector, vec_shift, initial_sample=0, base_size=40 -): +def fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, vec_shift, initial_sample=0, base_size=40): mid_point = int(initial_sample + base_size / 2) new_synced_time_vector = synced_time_vector.copy() @@ -145,9 +136,7 @@ def fix_shifted_sync_vector( consecutive_zeros[stable_parts[0]], ] ) - next_borrow_start = np.where( - where_insert_iteration > consecutive_zeros[stable_parts[0]] - ) + next_borrow_start = np.where(where_insert_iteration > consecutive_zeros[stable_parts[0]]) next_borrow_start = next_borrow_start[0] if next_borrow_start.shape[0] > 0: index_shift = next_borrow_start[0] @@ -156,9 +145,7 @@ def fix_shifted_sync_vector( else: print("Extreme case diff vec shift") - index_borrow_virmen.append( - [where_insert_iteration[index_shift], diff_vec_shift.shape[0]] - ) + index_borrow_virmen.append([where_insert_iteration[index_shift], diff_vec_shift.shape[0]]) print("index_borrow_virmen", index_borrow_virmen) break @@ -186,10 +173,7 @@ def fix_shifted_sync_vector( # print('behavior_time_vector', behavior_time_vector[index_diff_s-1:index_diff_e+2]) # print('new_synced_time_vector2', new_synced_time_vector2[index_diff_s-1:index_diff_e+2]) - time_iteration = ( - behavior_time_vector[index_diff_s : index_diff_e + 1] - - behavior_time_vector[index_diff_s] - ) + time_iteration = behavior_time_vector[index_diff_s : index_diff_e + 1] - behavior_time_vector[index_diff_s] # print('time_iteration', time_iteration) new_synced_time_vector[index_diff_s : index_diff_e + 1] = ( @@ -200,9 +184,7 @@ def fix_shifted_sync_vector( + time_iteration ) - check_diff = np.diff( - new_synced_time_vector[index_diff_s : index_diff_e + 2] - ) + check_diff = np.diff(new_synced_time_vector[index_diff_s : index_diff_e + 2]) idx_back_time = np.where(check_diff <= 0.0005) idx_back_time = idx_back_time[0] if idx_back_time.shape[0] == 0: @@ -220,9 +202,7 @@ def fix_sync_vector_greater(sync_time_vector, behavior_time_vector): new_sync_time_vector = sync_time_vector.copy() - diff_vecs = ( - new_sync_time_vector - behavior_time_vector[: new_sync_time_vector.shape[0]] - ) + diff_vecs = new_sync_time_vector - behavior_time_vector[: new_sync_time_vector.shape[0]] idx_plus = np.where(diff_vecs > 0) idx_plus = idx_plus[0] @@ -236,13 +216,9 @@ def fix_sync_vector_greater(sync_time_vector, behavior_time_vector): idx_start = grouped_sections[i][0] - 1 idx_end = grouped_sections[i][-1] - time_vector = ( - behavior_time_vector[idx_start : idx_end + 1] - - behavior_time_vector[idx_start] - ) + time_vector = behavior_time_vector[idx_start : idx_end + 1] - behavior_time_vector[idx_start] new_sync_time_vector[idx_start : idx_end + 1] = ( - np.repeat(new_sync_time_vector[idx_start], idx_end - idx_start + 1) - + time_vector + np.repeat(new_sync_time_vector[idx_start], idx_end - idx_start + 1) + time_vector ) borrowed_indexes.append([idx_start, idx_end]) @@ -260,9 +236,7 @@ def complete_last_part_sync_vec(sync_time_vector, behavior_time_vector): if diff_size > 0: diff_size = diff_size + 1 - last_part_bt = ( - behavior_time_vector[-diff_size:] - behavior_time_vector[-diff_size] - ) + last_part_bt = behavior_time_vector[-diff_size:] - behavior_time_vector[-diff_size] insert_part = np.repeat(new_sync_time_vector[-1], diff_size) + last_part_bt # print('last_part_bt',last_part_bt) @@ -295,17 +269,11 @@ def fix_iter_vector( first_iter = synced_iteration_vector[0] for i in range(diff_iter_times_idx.shape[0]): if diff_iter_times_idx[i] != 0: - new_synced_iteration_vector[i] = first_iter + int( - synced_time_vector[i] * nidq_sampling_rate - ) + new_synced_iteration_vector[i] = first_iter + int(synced_time_vector[i] * nidq_sampling_rate) if ori_shape != new_shape: - last_iterations = np.round( - first_iter + synced_time_vector[ori_shape:] * nidq_sampling_rate - ) - new_synced_iteration_vector = np.append( - new_synced_iteration_vector, last_iterations.astype(np.int64) - ) + last_iterations = np.round(first_iter + synced_time_vector[ori_shape:] * nidq_sampling_rate) + new_synced_iteration_vector = np.append(new_synced_iteration_vector, last_iterations.astype(np.int64)) return new_synced_iteration_vector @@ -313,9 +281,7 @@ def fix_iter_vector( def sync_evaluation_process2(synced_time_vector, behavior_time_vector): status = 1 - diff_vector = ( - synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]] - ) + diff_vector = synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]] num_iter = diff_vector.shape[0] # max_diff = max(diff_vector) @@ -327,14 +293,8 @@ def sync_evaluation_process2(synced_time_vector, behavior_time_vector): for j in range(num_div): start_iter = int(j * num_iter / num_div) end_iter = int((j + 1) * num_iter / num_div) - median_diff_percent[j] = ( - (np.median(diff_vector[start_iter:end_iter]) - median_general) - * 100 - / median_general - ) - median_diff_abs[j] = ( - np.median(diff_vector[start_iter:end_iter]) - median_general - ) + median_diff_percent[j] = (np.median(diff_vector[start_iter:end_iter]) - median_general) * 100 / median_general + median_diff_abs[j] = np.median(diff_vector[start_iter:end_iter]) - median_general if np.max(np.abs(median_diff_abs)) < 0.005: pass @@ -345,9 +305,7 @@ def sync_evaluation_process2(synced_time_vector, behavior_time_vector): return status -def main_ephys_fix_sync_code( - iter_start_idx, iter_times_idx, behavior_time, nidq_sampling_rate -): +def main_ephys_fix_sync_code(iter_start_idx, iter_times_idx, behavior_time, nidq_sampling_rate): iteration_dict = dict() iteration_dict["iter_start_idx"] = list() @@ -357,19 +315,13 @@ def main_ephys_fix_sync_code( # print('fixing trial ',i) behavior_time_vector = behavior_time[i].flatten() - synced_time_vector, shift_vec, median_vec = get_shift_vector( - iter_times_idx[i], behavior_time_vector - ) + synced_time_vector, shift_vec, median_vec = get_shift_vector(iter_times_idx[i], behavior_time_vector) - synced_time_vector, _ = fix_shifted_sync_vector( - synced_time_vector, behavior_time_vector, shift_vec - ) + synced_time_vector, _ = fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, shift_vec) # synced_time_vector, trial_stats_dict['borrow_step3'] =\ # fix_sync_vector_greater(synced_time_vector, behavior_time_vector) - synced_time_vector, _ = complete_last_part_sync_vec( - synced_time_vector, behavior_time_vector - ) + synced_time_vector, _ = complete_last_part_sync_vec(synced_time_vector, behavior_time_vector) synced_iteration_vector = fix_iter_vector( iter_start_idx[i], synced_time_vector, iter_times_idx[i], nidq_sampling_rate @@ -380,20 +332,14 @@ def main_ephys_fix_sync_code( print("end fix sync code 1") - iteration_dict["iter_start_idx"] = np.asarray( - iteration_dict["iter_start_idx"].copy(), dtype=object - ) - iteration_dict["iter_times_idx"] = np.asarray( - iteration_dict["iter_times_idx"].copy(), dtype=object - ) + iteration_dict["iter_start_idx"] = np.asarray(iteration_dict["iter_start_idx"].copy(), dtype=object) + iteration_dict["iter_times_idx"] = np.asarray(iteration_dict["iter_times_idx"].copy(), dtype=object) print("end fix sync code") # Check # of trials and iterations match trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ( - ephys_utils.assert_iteration_samples_count( - iteration_dict["iter_start_idx"], behavior_time - ) + ephys_utils.assert_iteration_samples_count(iteration_dict["iter_start_idx"], behavior_time) ) print("after assert_iteration_samples_count fix sync code") @@ -425,10 +371,8 @@ def main_ephys_fix_sync_code( print("after sync_evaluation_process2", status) if status == 1: - iteration_dict["trial_start_idx"] = ( - ephys_utils.get_index_trial_vector_from_iteration( - iteration_dict["iter_start_idx"] - ) + iteration_dict["trial_start_idx"] = ephys_utils.get_index_trial_vector_from_iteration( + iteration_dict["iter_start_idx"] ) print("after get_index_trial_vector_from_iteration") diff --git a/u19_pipeline/utils/ephys_utils.py b/u19_pipeline/utils/ephys_utils.py index f9f10432..79567024 100644 --- a/u19_pipeline/utils/ephys_utils.py +++ b/u19_pipeline/utils/ephys_utils.py @@ -25,13 +25,9 @@ def load_spice_glx_digital_file(file_path, nidq_meta, d_line_list=None): list_start_end_chan = nidq_meta["niXDChans1"].split(sep=":") print(list_start_end_chan) if len(list_start_end_chan) == 2: - d_line_list = list( - range(int(list_start_end_chan[0]), int(list_start_end_chan[1]) + 1) - ) + d_line_list = list(range(int(list_start_end_chan[0]), int(list_start_end_chan[1]) + 1)) else: - raise ValueError( - 'Could not infer channel list from nidq_meta["niXDChans1"] ' - ) + raise ValueError('Could not infer channel list from nidq_meta["niXDChans1"] ') nidq_sampling_rate = readSGLX.SampRate(nidq_meta) @@ -43,9 +39,7 @@ def load_spice_glx_digital_file(file_path, nidq_meta, d_line_list=None): last_sample_index = int(nidq_sampling_rate * t_end) - 1 # Read binary and digital - nidq_raw_data = readSGLX.makeMemMapRaw( - file_path, nidq_meta - ) # Pull raw bin data + nidq_raw_data = readSGLX.makeMemMapRaw(file_path, nidq_meta) # Pull raw bin data digital_array = readSGLX.ExtractDigital( # extract interation index nidq_raw_data, first_sample_index, @@ -74,9 +68,7 @@ def load_trial_iteration_signals(ephys_session_fullpath, nidq_meta): new_iteration_channel = 2 # If PXIe card (nidq) card use for recording deduce digital channels if nidq_meta["typeThis"] == "nidq": - digital_array = spice_glx_utility.load_spice_glx_digital_file( - ephys_session_fullpath, nidq_meta - ) + digital_array = spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta) # If onebox card (obx) card use for recording digital channels are 0-2 else: digital_array = spice_glx_utility.load_spice_glx_digital_file( @@ -233,9 +225,7 @@ def get_iteration_sample_vector_from_digital_lines_pulses( ms_after_trial_start_pulse = -4 ms_before_trial_end = 4 - samples_after_pulse_start = int( - nidq_sampling_rate * (ms_after_trial_start_pulse / 1000) - ) + samples_after_pulse_start = int(nidq_sampling_rate * (ms_after_trial_start_pulse / 1000)) samples_before_pulse_end = int(nidq_sampling_rate * (ms_before_trial_end / 1000)) # num Trials to sync (if behavior stopped before last trial was saved) @@ -286,12 +276,8 @@ def get_iteration_sample_vector_from_digital_lines_pulses( # iteration_vector_output['trialnumber_vector_samples'][trial_start_idx[i]:iter_samples[-1]+int(nidq_sampling_rate)] = i+1 # iteration_vector_output['framenumber_vector_samples'][iter_samples[-1]:iter_samples[-1]+int(nidq_sampling_rate)] = iter_samples.shape[0] - iteration_vector_output["iter_start_idx"] = np.asarray( - iter_start_idx.copy(), dtype=object - ) - iteration_vector_output["iter_times_idx"] = np.asarray( - iter_times_idx.copy(), dtype=object - ) + iteration_vector_output["iter_start_idx"] = np.asarray(iter_start_idx.copy(), dtype=object) + iteration_vector_output["iter_times_idx"] = np.asarray(iter_times_idx.copy(), dtype=object) return iteration_vector_output @@ -302,13 +288,9 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite # ... and also get start and end time framenumber = np.zeros(digital_array.shape[1]) for i in range(digital_array.shape[1]): - a = BitArray( - np.flip(digital_array[1:, i]) - ) # ignore 0-bit, as this is the NPX sync puls, and not virmen. + a = BitArray(np.flip(digital_array[1:, i])) # ignore 0-bit, as this is the NPX sync puls, and not virmen. framenumber[i] = a.uint - iterations_raw = np.array( - framenumber, dtype=np.int32 - ) # Transform frames into integer + iterations_raw = np.array(framenumber, dtype=np.int32) # Transform frames into integer recording_start = np.min(np.where(iterations_raw > 0)) # first chane of testlist recording_end = ( np.where(np.abs(np.diff(iterations_raw)) > 0)[0][-1] + 200 @@ -322,9 +304,7 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite iter_start_idx = [] for idx, frame_number in enumerate(iterations_raw): if (idx > recording_start) & (idx < recording_end): - if (frame_number == 0) & ( - iterations_raw[idx - 1] == 127 - ): # At the reset, add 128 + if (frame_number == 0) & (iterations_raw[idx - 1] == 127): # At the reset, add 128 overflow = overflow + 1 if ( (frame_number == 0) @@ -337,9 +317,7 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite frame_number + overflow * 128 - 1 ) # In case this happened, the previous sample has to be corrected # Keep track of trial number - endflag = framenumber_in_trial[idx - 1] == ( - len(time[current_trial]) - ) # Trial end has been reached. + endflag = framenumber_in_trial[idx - 1] == (len(time[current_trial])) # Trial end has been reached. transitionflag = frame_number < 3 # Next trial should start at zero again if endflag & transitionflag: # Only at the transitions current_trial = current_trial + 1 # Increases trial count @@ -347,9 +325,7 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite iter_start_idx.append(idx) # Make a note when this happened framenumber_in_trial[idx] = frame_number + overflow * 128 - 1 trialnumber[idx] = current_trial - trial_list = np.array( - np.unique(trialnumber[np.isfinite(trialnumber)]), dtype=np.int32 - ) + trial_list = np.array(np.unique(trialnumber[np.isfinite(trialnumber)]), dtype=np.int32) # Fourth, find and remove the nidaq glitches # These are single samples where the iteration number is corrupted @@ -360,9 +336,7 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite din = np.diff(framenumber_in_trial) trial_transitions = np.where(np.diff(trialnumber)) glitches = [] - for candidate in np.where(np.logical_or(din > 1, din < 0))[ - 0 - ]: # skipped frames or counting down + for candidate in np.where(np.logical_or(din > 1, din < 0))[0]: # skipped frames or counting down if np.sum(candidate == trial_transitions) == 0: glitches = np.append(glitches, candidate) glitches = np.array(glitches, dtype=np.int32) @@ -371,9 +345,7 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite skipped_frames = 0 for g in glitches: if framenumber_in_trial[g] < framenumber_in_trial[g + 2]: - if ( - framenumber_in_trial[g + 2] - framenumber_in_trial[g] == 2 - ): # skipped frame, should be very rare + if framenumber_in_trial[g + 2] - framenumber_in_trial[g] == 2: # skipped frame, should be very rare framenumber_in_trial[g + 1] = framenumber_in_trial[g] + 1 skipped_frames = skipped_frames + 1 else: # If random number, nidaq sample in the middle of update. @@ -387,12 +359,8 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite iter_start_idx = [] for t in trial_list: - iter_start_idx.append( - np.arange(0, framenumber_in_trial[trialnumber == t][-1], 1) - ) - iteration_vector_output["iter_start_idx"] = np.asarray( - iter_start_idx.copy(), dtype=object - ) + iter_start_idx.append(np.arange(0, framenumber_in_trial[trialnumber == t][-1], 1)) + iteration_vector_output["iter_start_idx"] = np.asarray(iter_start_idx.copy(), dtype=object) return iteration_vector_output @@ -401,9 +369,7 @@ def assert_iteration_samples_count(iteration_sample_idx_output, behavior_time_ve # Assert that vector sync pulses match behavior time vector # Count trial count differences - trial_count_diff = np.abs( - iteration_sample_idx_output.shape[0] - (behavior_time_vector.shape[0]) - ) + trial_count_diff = np.abs(iteration_sample_idx_output.shape[0] - (behavior_time_vector.shape[0])) count = 0 trials_diff_iteration_small = list() @@ -424,10 +390,7 @@ def assert_iteration_samples_count(iteration_sample_idx_output, behavior_time_ve count += 1 # For each trial iteration # should be equal to the behavioral file iterations if iter_trials.shape[0] != behavior_time_vector[idx_trial].shape[0]: - if ( - np.abs(iter_trials.shape[0] - behavior_time_vector[idx_trial].shape[0]) - < 6 - ): + if np.abs(iter_trials.shape[0] - behavior_time_vector[idx_trial].shape[0]) < 6: trials_diff_iteration_small.append(idx_trial) else: trials_diff_iteration_big.append(idx_trial) @@ -455,10 +418,7 @@ def evaluate_sync_process( status = -1 return status - if ( - len(trials_diff_iteration_big) == 1 - and trials_diff_iteration_big[0] == total_trials - 1 - ): + if len(trials_diff_iteration_big) == 1 and trials_diff_iteration_big[0] == total_trials - 1: print( "Missed by a lot last trial: (Assume recording stopped earlier) ", trials_diff_iteration_big, @@ -480,9 +440,7 @@ def evaluate_sync_process( # Iterations differ in more than two trials if len(trials_diff_iteration_small) > 2: - print( - "Missed iteration count on many trials: ", len(trials_diff_iteration_small) - ) + print("Missed iteration count on many trials: ", len(trials_diff_iteration_small)) status = -1 return status @@ -502,9 +460,7 @@ def evaluate_sync_process( return status -def fix_missing_iteration_trials( - trials_diff_iteration_small, iteration_dict, behavior_times, nidq_sampling_rate -): +def fix_missing_iteration_trials(trials_diff_iteration_small, iteration_dict, behavior_times, nidq_sampling_rate): # Fix and insert missing synced iteration vectors print("trials_diff_iteration_small", trials_diff_iteration_small) @@ -547,9 +503,7 @@ def fix_missing_iteration_trials( return iteration_dict -def insert_missing_synced_iteration( - synced_iteration_vector, synced_time_vector, behavior_time_vector -): +def insert_missing_synced_iteration(synced_iteration_vector, synced_time_vector, behavior_time_vector): # Check where is more likely we miss an iteration pulse and insert it to iteration_vector status = 1 @@ -559,15 +513,11 @@ def insert_missing_synced_iteration( # Get in which indexes we get a "peak" of non matching times... if synced_time_vector.shape[0] >= behavior_time_vector.shape[0]: - print( - "More pulses than behavior iterations, check other method" - ) # Christian: What is the other method? + print("More pulses than behavior iterations, check other method") # Christian: What is the other method? status = -1 return status, np.empty(0) else: - diff_vector = np.diff( - synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]] - ) + diff_vector = np.diff(synced_time_vector - behavior_time_vector[: synced_time_vector.shape[0]]) # In case last peak is at the end of trial (append 0 to detect it) diff_vector = np.append(diff_vector, np.array([0])) @@ -579,12 +529,8 @@ def insert_missing_synced_iteration( # Insert extra iterations as a new "iteration" start to match behavior iterations new_synced_iteration_vector = synced_iteration_vector.copy() for i in range(peaks.shape[0]): - value_insert = ( - synced_iteration_vector[peaks[i]] + synced_iteration_vector[peaks[i] + 1] - ) / 2 - new_synced_iteration_vector = np.insert( - new_synced_iteration_vector, peaks[i], value_insert - ) + value_insert = (synced_iteration_vector[peaks[i]] + synced_iteration_vector[peaks[i] + 1]) / 2 + new_synced_iteration_vector = np.insert(new_synced_iteration_vector, peaks[i], value_insert) if new_synced_iteration_vector.shape[0] != behavior_time_vector.shape[0]: print("with peak strategy, could not find correct missing iterations") @@ -614,16 +560,10 @@ def behavior_sync_frame_counter_method( np.flip(digital_array[bit_start:, i]) ) # ignore 0-bit, as this is the NPX sync puls, and not virmen. framenumber[i] = a.uint - iterations_raw = np.array( - framenumber, dtype=np.int - ) # Transform frames into integer - recording_start = np.min( - np.where(iterations_raw > 0) - ) # Get start of recording: first change of testlist + iterations_raw = np.array(framenumber, dtype=np.int) # Transform frames into integer + recording_start = np.min(np.where(iterations_raw > 0)) # Get start of recording: first change of testlist dt = np.int(0.04 * nidq_sampling_rate) - recording_end = ( - np.where(np.abs(np.diff(iterations_raw)) > 0)[0][-1] + dt - ) # Get end of recording + recording_end = np.where(np.abs(np.diff(iterations_raw)) > 0)[0][-1] + dt # Get end of recording # 3: transform `iterations_raw` into `framenumber_in_trial` and `trialnumber` # iterations_raw is just a number between 0 and max_count+1. Some math has to be done to obtain: @@ -634,15 +574,11 @@ def behavior_sync_frame_counter_method( framenumber_in_trial = np.zeros(len(iterations_raw)) * np.nan trialnumber = np.zeros(len(iterations_raw)) * np.nan current_trial = 0 - overflow = ( - 0 # This variable keep track whenever the reset from max_count to 0 happens. - ) + overflow = 0 # This variable keep track whenever the reset from max_count to 0 happens. for idx, frame_number in enumerate(iterations_raw): if (idx > recording_start) & (idx < recording_end): # print(iterations_raw2[idx], frame_number) - if (frame_number == 0) & ( - iterations_raw[idx - 1] == max_count - ): # At the reset, add max_count+1 + if (frame_number == 0) & (iterations_raw[idx - 1] == max_count): # At the reset, add max_count+1 overflow = overflow + 1 if ( (frame_number == 0) @@ -659,9 +595,7 @@ def behavior_sync_frame_counter_method( len(behavior_time_vector[current_trial]) ) # Trial end has been reached. - transitionflag = ( - frame_number == 2 - ) # Next trial should start at zero again (it starts with two ??) + transitionflag = frame_number == 2 # Next trial should start at zero again (it starts with two ??) if endflag & transitionflag: # Only at the transitions current_trial = current_trial + 1 # Increases trial count overflow = 0 # Reset the 7 bit counter for the next trial @@ -669,14 +603,10 @@ def behavior_sync_frame_counter_method( if overflow == 0: framenumber_in_trial[idx] = frame_number else: - framenumber_in_trial[idx] = ( - frame_number + overflow * (max_count + 1) - 1 - ) + framenumber_in_trial[idx] = frame_number + overflow * (max_count + 1) - 1 trialnumber[idx] = current_trial - trial_list = np.array( - np.unique(trialnumber[np.isfinite(trialnumber)]), dtype=np.int64 - ) + trial_list = np.array(np.unique(trialnumber[np.isfinite(trialnumber)]), dtype=np.int64) # 4: find and remove additional NIDAQ glitches of two types: # a) single samples where the iteration number is corrupted because sampling happened faster than output of the behevior PC. @@ -685,9 +615,7 @@ def behavior_sync_frame_counter_method( din = np.diff(framenumber_in_trial) trial_transitions = np.where(np.diff(trialnumber)) glitches = [] - for candidate in np.where(np.logical_or(din > 1, din < 0))[ - 0 - ]: # skipped frames or counting down + for candidate in np.where(np.logical_or(din > 1, din < 0))[0]: # skipped frames or counting down if np.sum(candidate == trial_transitions) == 0: glitches = np.append(glitches, candidate) glitches = np.array(glitches, dtype=np.int) @@ -695,9 +623,7 @@ def behavior_sync_frame_counter_method( skipped_frames = 0 for g in glitches: if framenumber_in_trial[g] < framenumber_in_trial[g + 2]: - if ( - framenumber_in_trial[g + 2] - framenumber_in_trial[g] == 2 - ): # skipped frame, should be very rare + if framenumber_in_trial[g + 2] - framenumber_in_trial[g] == 2: # skipped frame, should be very rare pass framenumber_in_trial[g + 1] = framenumber_in_trial[g] + 1 skipped_frames = skipped_frames + 1 @@ -705,26 +631,20 @@ def behavior_sync_frame_counter_method( framenumber_in_trial[g + 1] = framenumber_in_trial[g] # A set of final asserts, making sure that the code worked as intended - assert len(trial_list) == len( - session_trial_keys - ) # Make sure the trial number is correct. + assert len(trial_list) == len(session_trial_keys) # Make sure the trial number is correct. assert np.sum(np.diff(framenumber_in_trial) > 1) == 0 # No frames should be skipped - assert np.sum(np.diff(framenumber_in_trial) < 0) < len( - trial_list - ) # Negative iterations only at trial transitions + assert np.sum(np.diff(framenumber_in_trial) < 0) < len(trial_list) # Negative iterations only at trial transitions iterations_test = 0 for t in trial_list: - iterations_test = ( - iterations_test + framenumber_in_trial[trialnumber == t][-1] - ) # Integrate number of iterations + iterations_test = iterations_test + framenumber_in_trial[trialnumber == t][-1] # Integrate number of iterations assert framenumber_in_trial[trialnumber == t][-1] == len( behavior_time_vector[t] ) # Make sure number of nidaq-frames in each trial is identical to dj record: nidaqtime = np.sum(trialnumber == t) / nidq_sampling_rate matlabtime = np.max(behavior_time_vector[t]) assert ( - ((nidaqtime - matlabtime) / matlabtime) < 0.1 - ) # # Make sure the nidaq-trial-duration and dj records are consistent; 10% arbitrarily chosen + (nidaqtime - matlabtime) / matlabtime + ) < 0.1 # # Make sure the nidaq-trial-duration and dj records are consistent; 10% arbitrarily chosen nidaq_duration = iterations_test + skipped_frames # dj_duration = iterstart[-1] + len(behavior_time_vector[-1]) # assert np.abs(nidaq_duration - dj_duration) < 3 # at most two frames off - sometimes this happens at the beginning/end of the recording @@ -781,9 +701,7 @@ def get_iteration_intertrial_from_virmen_time( iter_start_idx = [] for i in range(num_behavior_trials): - new_synced_iteration_vector = trial_start_idx[i] + np.int64( - behavior_time_vector[i] * nidq_sampling_rate - ) + new_synced_iteration_vector = trial_start_idx[i] + np.int64(behavior_time_vector[i] * nidq_sampling_rate) iter_start_idx.append(new_synced_iteration_vector.squeeze()) iter_start_idx = np.asarray(iter_start_idx.copy(), dtype=object) @@ -803,18 +721,14 @@ def get_full_vector_samples(iter_start_idx_vectors, nidq_sampling_rate, total_sa # Fill vector samples for j in range(this_trial_iter_vector.shape[0] - 1): - framenumber_vector_samples[ - this_trial_iter_vector[j] : this_trial_iter_vector[j + 1] - ] = j + 1 + framenumber_vector_samples[this_trial_iter_vector[j] : this_trial_iter_vector[j + 1]] = j + 1 # Last iteration # is from start of iteration to end of trial if i < len(iter_start_idx_vectors) - 1: - trialnumber_vector_samples[ - this_trial_iter_vector[0] : next_trial_iter_vector[0] - ] = i + 1 - framenumber_vector_samples[ - this_trial_iter_vector[-1] : next_trial_iter_vector[0] - ] = this_trial_iter_vector.shape[0] + trialnumber_vector_samples[this_trial_iter_vector[0] : next_trial_iter_vector[0]] = i + 1 + framenumber_vector_samples[this_trial_iter_vector[-1] : next_trial_iter_vector[0]] = ( + this_trial_iter_vector.shape[0] + ) # For last trial, lets finish it 1s after last iteration detected else: # print('total_samples', total_samples) @@ -822,12 +736,10 @@ def get_full_vector_samples(iter_start_idx_vectors, nidq_sampling_rate, total_sa # print('this_trial_iter_vector[0]', this_trial_iter_vector[0]) # print('this_trial_iter_vector[-1]+int(nidq_sampling_rate)', this_trial_iter_vector[-1]+int(nidq_sampling_rate)) trialnumber_vector_samples[ - this_trial_iter_vector[0] : this_trial_iter_vector[-1] - + int(nidq_sampling_rate) + this_trial_iter_vector[0] : this_trial_iter_vector[-1] + int(nidq_sampling_rate) ] = i + 1 framenumber_vector_samples[ - this_trial_iter_vector[-1] : this_trial_iter_vector[-1] - + int(nidq_sampling_rate) + this_trial_iter_vector[-1] : this_trial_iter_vector[-1] + int(nidq_sampling_rate) ] = this_trial_iter_vector.shape[0] return trialnumber_vector_samples, framenumber_vector_samples @@ -885,9 +797,7 @@ def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling idx_iteration_final = [] for i in range(idx_trial.shape[0]): if i < idx_trial.shape[0] - 1: - iterations_this_trial = idx_iteration[ - (idx_iteration > idx_trial[i]) & (idx_iteration < idx_trial[i + 1]) - ] + iterations_this_trial = idx_iteration[(idx_iteration > idx_trial[i]) & (idx_iteration < idx_trial[i + 1])] else: iterations_this_trial = idx_iteration[(idx_iteration > idx_trial[i])] idx_this_trial_iterations = np.insert(iterations_this_trial, 0, idx_trial[i]) @@ -901,9 +811,7 @@ def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling idx_iteration_final, nidq_sampling_rate, trial_index_nidq.shape[0] ) - iteration_equal = np.allclose( - iteration_index_nidq, iteration_index_nidq2, equal_nan=True - ) + iteration_equal = np.allclose(iteration_index_nidq, iteration_index_nidq2, equal_nan=True) trial_equal = np.allclose(trial_index_nidq, trial_index_nidq2, equal_nan=True) print("iteration_equal", iteration_equal) @@ -940,9 +848,7 @@ class xyz_pick_file_creator: """ @staticmethod - def main_xyz_pick_file_function( - recording_id, fragment_number, chanmap_file, processed_data_directory - ): + def main_xyz_pick_file_function(recording_id, fragment_number, chanmap_file, processed_data_directory): """ Stores xyz_pick_files on ibl_postprocess directory for ibl_atlas_gui Input: @@ -962,9 +868,7 @@ def main_xyz_pick_file_function( pathlib.Path.mkdir(ibl_output_dir) # Get recording id - probe_location = xyz_pick_file_creator.get_probe_insertion_coordinates( - recording_id, fragment_number - ) + probe_location = xyz_pick_file_creator.get_probe_insertion_coordinates(recording_id, fragment_number) print(probe_location) # Load channelmap and check how many probes there are @@ -974,12 +878,8 @@ def main_xyz_pick_file_function( # Calculate probe coordinates and store files all_shanks = list() for i in range(max_shank): - probe_track = xyz_pick_file_creator.get_probetrack( - chanmap, shank=i + 1, **probe_location - ) - xyz_pick_file_creator.save_xyz_pick_file( - ibl_output_dir, probe_track, shank=i - ) + probe_track = xyz_pick_file_creator.get_probetrack(chanmap, shank=i + 1, **probe_location) + xyz_pick_file_creator.save_xyz_pick_file(ibl_output_dir, probe_track, shank=i) all_shanks.append(probe_track) return all_shanks @@ -1009,9 +909,7 @@ def get_probe_insertion_coordinates(recording_id, probe_num): # Query subject of recording query = {"recording_id": recording_id} - subject_recording = (recording_db.Recording.BehaviorSession & query).fetch( - "subject_fullname" - ) + subject_recording = (recording_db.Recording.BehaviorSession & query).fetch("subject_fullname") if subject_recording.shape[0] != 0: # Query probe insertion table @@ -1019,9 +917,7 @@ def get_probe_insertion_coordinates(recording_id, probe_num): "subject_fullname": subject_recording[0], "device_idx": probe_num, } - probe_location = (action_db.SurgeryLocation & query_surgery).fetch( - *coordinates_columns, as_dict=True - ) + probe_location = (action_db.SurgeryLocation & query_surgery).fetch(*coordinates_columns, as_dict=True) if len(probe_location) == 0: probes_not_found = True else: @@ -1080,9 +976,7 @@ def get_probetrack( probe_x0 = np.array( [np.cos(roll) * avx, -np.sin(roll) * avx] ) # coordinates of the shank after "roll" around probe axis - probe_length = np.max(y[k == shank]) - np.min( - y[k == shank] - ) # Length off the probe per chanmap + probe_length = np.max(y[k == shank]) - np.min(y[k == shank]) # Length off the probe per chanmap probe_unitVec = np.array( [np.sin(theta) * np.cos(phi), np.sin(theta) * np.sin(phi), np.cos(theta)] ) # Unit vector point along insertion direction @@ -1096,9 +990,7 @@ def get_probetrack( # Step 3: Produce the 3D coordinates along the probe track probe_track = np.zeros((len(probe_length), 3)) for i in range(len(probe_length)): - probe_track[i, :] = probe_length[i] * probe_unitVec + np.array( - [probe_x0[0], probe_x0[1], 0] - ) + probe_track[i, :] = probe_length[i] * probe_unitVec + np.array([probe_x0[0], probe_x0[1], 0]) # Step 4: Shift probe my ML|AP insertion coordinates probe_track_shifted = np.zeros(probe_track.shape) diff --git a/u19_pipeline/utils/matlab_utils.py b/u19_pipeline/utils/matlab_utils.py index cb3efabf..39906cd2 100644 --- a/u19_pipeline/utils/matlab_utils.py +++ b/u19_pipeline/utils/matlab_utils.py @@ -67,9 +67,7 @@ def mat_obj_to_dict(mat_struct): dict_from_struct[field_name] = mat_obj_to_dict(dict_from_struct[field_name]) elif isinstance(dict_from_struct[field_name], np.ndarray): try: - dict_from_struct[field_name] = mat_obj_to_array( - dict_from_struct[field_name] - ) + dict_from_struct[field_name] = mat_obj_to_array(dict_from_struct[field_name]) except TypeError: continue return dict_from_struct @@ -79,9 +77,7 @@ def mat_obj_to_array(mat_struct_array): """Construct array from matlab cell arrays. Recursively converts array elements if they contain mat objects.""" if has_struct(mat_struct_array): - array_from_cell = [ - mat_obj_to_dict(mat_struct) for mat_struct in mat_struct_array - ] + array_from_cell = [mat_obj_to_dict(mat_struct) for mat_struct in mat_struct_array] array_from_cell = np.array(array_from_cell) else: array_from_cell = mat_struct_array @@ -91,10 +87,7 @@ def mat_obj_to_array(mat_struct_array): def has_struct(mat_struct_array): """Determines if a matlab cell array contains any mat objects.""" - return any( - isinstance(mat_struct, matlab.mio5_params.mat_struct) - for mat_struct in mat_struct_array - ) + return any(isinstance(mat_struct, matlab.mio5_params.mat_struct) for mat_struct in mat_struct_array) def convert_mat_file_to_dict(mat_file_name): @@ -289,9 +282,7 @@ def convert_function_handle_to_str(mat_file_path): os.remove("shaping_protocol.txt") except Exception as e: - print( - f"There was an error while trying to execute {convert_script_path}:\n{e}" - ) + print(f"There was an error while trying to execute {convert_script_path}:\n{e}") else: print( "A working matlab version was not found. " @@ -328,9 +319,7 @@ def convert_towers_block_trial_2_df(current_block_trial, block_num): if valid_block: block_trial_df = pd.DataFrame(current_block_trial) - block_trial_df.insert( - loc=0, column="trial_idx", value=np.arange(len(block_trial_df)) + 1 - ) + block_trial_df.insert(loc=0, column="trial_idx", value=np.arange(len(block_trial_df)) + 1) block_trial_df.insert(loc=0, column="block", value=block_num) else: block_trial_df = pd.DataFrame() @@ -410,18 +399,14 @@ def convert_behavior_file(mat_file): block = matin["log"]["block"][i] # Convert trial df and block df - valid_block, block_trial_df = convert_towers_block_trial_2_df( - block["trial"], i + 1 - ) + valid_block, block_trial_df = convert_towers_block_trial_2_df(block["trial"], i + 1) valid_blocks, block_df = convert_towers_block_2_df(block, i + 1) # Write string of block level Protocol (from matlab obscured data) if metadata_read: block_df["shapingProtocol"] = converted_metadata["shaping_protocol"][i] if valid_block and valid_blocks: - session_current_block_trial_df = block_trial_df.merge( - block_df, on="block", suffixes=["_block", "_trial"] - ) + session_current_block_trial_df = block_trial_df.merge(block_df, on="block", suffixes=["_block", "_trial"]) if num_blocks_conv == 0: session_block_trial_df = session_current_block_trial_df.copy() else: diff --git a/u19_pipeline/utils/scp_transfers.py b/u19_pipeline/utils/scp_transfers.py index c27e9324..d50fb364 100644 --- a/u19_pipeline/utils/scp_transfers.py +++ b/u19_pipeline/utils/scp_transfers.py @@ -162,6 +162,4 @@ def check_directory_copied_correctly(): args = sys.argv[1:] print(args) - transfer_scp( - host=args[0], username=args[1], remote_path=args[2], local_path=args[3] - ) + transfer_scp(host=args[0], username=args[1], remote_path=args[2], local_path=args[3]) diff --git a/u19_pipeline/utils/slack_utils.py b/u19_pipeline/utils/slack_utils.py index 3c37557c..a4462f57 100644 --- a/u19_pipeline/utils/slack_utils.py +++ b/u19_pipeline/utils/slack_utils.py @@ -12,20 +12,12 @@ def get_webhook_list(slack_dict, lab): webhooks_list = [] if "slack_notification_channel" in slack_dict: - query_slack_webhooks = [ - {"webhook_name": x} for x in slack_dict["slack_notification_channel"] - ] - webhooks_list += ( - (lab.SlackWebhooks & query_slack_webhooks).fetch("webhook_url").tolist() - ) + query_slack_webhooks = [{"webhook_name": x} for x in slack_dict["slack_notification_channel"]] + webhooks_list += (lab.SlackWebhooks & query_slack_webhooks).fetch("webhook_url").tolist() if "slack_users_channel" in slack_dict: - query_slack_user_channels = [ - {"user_id": x} for x in slack_dict["slack_users_channel"] - ] - webhooks_list += ( - (lab.User & query_slack_user_channels).fetch("slack_webhook").tolist() - ) + query_slack_user_channels = [{"user_id": x} for x in slack_dict["slack_users_channel"]] + webhooks_list += (lab.User & query_slack_user_channels).fetch("slack_webhook").tolist() return webhooks_list @@ -51,9 +43,7 @@ def format_df_for_slack_message(df): def send_slack_notification(webhook_url, slack_json_message): byte_length = str(sys.getsizeof(slack_json_message)) headers = {"Content-Type": "application/json", "Content-Length": byte_length} - response = requests.post( - webhook_url, data=json.dumps(slack_json_message), headers=headers - ) + response = requests.post(webhook_url, data=json.dumps(slack_json_message), headers=headers) response.raise_for_status() @@ -69,9 +59,7 @@ def send_slack_update_notification(webhook_url, base_message, session_info): m1["type"] = "section" m1_1 = dict() m1_1["type"] = "mrkdwn" - m1_1["text"] = ( - ":white_check_mark: *Automation pipeline update* on " + datestr + "\n\n" - ) + m1_1["text"] = ":white_check_mark: *Automation pipeline update* on " + datestr + "\n\n" m1["text"] = m1_1 # Info # @@ -102,9 +90,7 @@ def send_slack_update_notification(webhook_url, base_message, session_info): message = dict() message["blocks"] = [m1, msep, m2] - message["text"] = "Automation pipeline update recording:" + str( - session_info["recording_id"] - ) + message["text"] = "Automation pipeline update recording:" + str(session_info["recording_id"]) logging.info("Slack payload: %s", json.dumps(message, default=str)) @@ -170,9 +156,7 @@ def send_slack_error_notification(webhook_url, error_info, session_info): message = dict() message["blocks"] = [m1, msep, m2, msep, m3] - message["text"] = "Automation pipeline error in recording:" + str( - session_info["recording_id"] - ) + message["text"] = "Automation pipeline error in recording:" + str(session_info["recording_id"]) logging.info("Slack payload: %s", json.dumps(message, default=str)) @@ -242,9 +226,7 @@ def send_slack_error_pupillometry_notification(webhook_url, error_info, session_ send_slack_notification(webhook_url, message) -def send_slack_pupillometry_update_notification( - webhook_url, base_message, session_info -): +def send_slack_pupillometry_update_notification(webhook_url, base_message, session_info): now = datetime.now() datestr = now.strftime("%d-%b-%Y %H:%M:%S") @@ -256,9 +238,7 @@ def send_slack_pupillometry_update_notification( m1["type"] = "section" m1_1 = dict() m1_1["type"] = "mrkdwn" - m1_1["text"] = ( - ":white_check_mark: *Pupillometry pipeline update* on " + datestr + "\n\n" - ) + m1_1["text"] = ":white_check_mark: *Pupillometry pipeline update* on " + datestr + "\n\n" m1["text"] = m1_1 # Info # diff --git a/u19_pipeline/utils/subject_metadata.py b/u19_pipeline/utils/subject_metadata.py index 4ff21f1b..2d4449f2 100644 --- a/u19_pipeline/utils/subject_metadata.py +++ b/u19_pipeline/utils/subject_metadata.py @@ -9,20 +9,12 @@ def fetch_slack_handles_for_lab_managers_by_subject( """Returns the Slack handles of lab managers for the labs associated with the given subjects.""" if isinstance(subjects, str): subjects = [subjects] - sql_formatted_names = ( - ", ".join([f"'{name}'" for name in subjects]) if subjects else "''" - ) - expanded_animal = ( - (subject.Subject & f"subject_fullname in ({sql_formatted_names})") - * lab.User() - * lab.UserLab() - ) + sql_formatted_names = ", ".join([f"'{name}'" for name in subjects]) if subjects else "''" + expanded_animal = (subject.Subject & f"subject_fullname in ({sql_formatted_names})") * lab.User() * lab.UserLab() labs = set(expanded_animal.fetch("lab")) sql_formatted_labs = ", ".join([f"'{lab}'" for lab in labs]) if labs else "''" - lab_managers = (lab.LabManager() & f"lab in ({sql_formatted_labs})").proj( - user_id="lab_manager" - ) * lab.User() + lab_managers = (lab.LabManager() & f"lab in ({sql_formatted_labs})").proj(user_id="lab_manager") * lab.User() slack_tags = lab_managers.fetch("slack") return slack_tags diff --git a/u19_pipeline/utils/tiff_matlab_imaging_utils.py b/u19_pipeline/utils/tiff_matlab_imaging_utils.py index 076f85c5..c12577d7 100644 --- a/u19_pipeline/utils/tiff_matlab_imaging_utils.py +++ b/u19_pipeline/utils/tiff_matlab_imaging_utils.py @@ -198,18 +198,12 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): parsed_info["Zs"] = -1 # Frame rate - frame_rate_match = re.search( - r"SI\.hRoiManager\.scanVolumeRate = ([0-9]+\.[0-9]+)", scope_str - ) + frame_rate_match = re.search(r"SI\.hRoiManager\.scanVolumeRate = ([0-9]+\.[0-9]+)", scope_str) - parsed_info["frameRate"] = ( - float(frame_rate_match.group(1)) if frame_rate_match else 0 - ) + parsed_info["frameRate"] = float(frame_rate_match.group(1)) if frame_rate_match else 0 # ROI lag - lag_match = re.search( - r"SI\.hScan2D\.flytoTimePerScanfield = ([0-9]+\.[0-9]+)", scope_str - ) + lag_match = re.search(r"SI\.hScan2D\.flytoTimePerScanfield = ([0-9]+\.[0-9]+)", scope_str) parsed_info["interROIlag_sec"] = float(lag_match.group(1)) if lag_match else 0 @@ -259,9 +253,7 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): # Resolution scaling if "objectiveResolution" in scope_str: try: - res_match = re.search( - r"SI\.objectiveResolution = ([0-9]+\.[0-9]+)", scope_str - ) + res_match = re.search(r"SI\.objectiveResolution = ([0-9]+\.[0-9]+)", scope_str) resolution_factor = xySizeFactor * float(res_match.group(1)) @@ -279,9 +271,7 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): power_match = re.search(r"SI\.hBeams\.powers = (\d+)", scope_str) - parsed_info["Scope"]["Power_percent"] = ( - float(power_match.group(1)) if power_match else 0 - ) + parsed_info["Scope"]["Power_percent"] = float(power_match.group(1)) if power_match else 0 # ----------------------------------------------------- # Channels @@ -289,29 +279,19 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): channel_match = re.search(r"SI\.hChannels\.channelSave = (\d+)", scope_str) - parsed_info["Scope"]["Channels"] = ( - int(channel_match.group(1)) if channel_match else 0 - ) + parsed_info["Scope"]["Channels"] = int(channel_match.group(1)) if channel_match else 0 # ----------------------------------------------------- # Config / user filenames # ----------------------------------------------------- - cfg_match = re.search( - r"SI\.hConfigurationSaver\.cfgFilename = (.+cfg)", scope_str - ) + cfg_match = re.search(r"SI\.hConfigurationSaver\.cfgFilename = (.+cfg)", scope_str) - parsed_info["Scope"]["cfgFilename"] = ( - cfg_match.group(1).strip() if cfg_match else "" - ) + parsed_info["Scope"]["cfgFilename"] = cfg_match.group(1).strip() if cfg_match else "" - usr_match = re.search( - r"SI\.hConfigurationSaver\.usrFilename = (.+usr)", scope_str - ) + usr_match = re.search(r"SI\.hConfigurationSaver\.usrFilename = (.+usr)", scope_str) - parsed_info["Scope"]["usrFilename"] = ( - usr_match.group(1).strip() if usr_match else "" - ) + parsed_info["Scope"]["usrFilename"] = usr_match.group(1).strip() if usr_match else "" # ----------------------------------------------------- # Fast-Z lag @@ -319,21 +299,15 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): lag_match = re.search(r"SI\.hFastZ\.actuatorLag = ([0-9eE\.\-]+)", scope_str) - parsed_info["Scope"]["fastZ_lag"] = ( - float(lag_match.group(1)) if lag_match else 0 - ) + parsed_info["Scope"]["fastZ_lag"] = float(lag_match.group(1)) if lag_match else 0 # ----------------------------------------------------- # Fast-Z flyback # ----------------------------------------------------- - flyback_match = re.search( - r"SI\.hFastZ\.flybackTime = ([0-9]+\.[0-9]+)", scope_str - ) + flyback_match = re.search(r"SI\.hFastZ\.flybackTime = ([0-9]+\.[0-9]+)", scope_str) - parsed_info["Scope"]["fastZ_flybackTime"] = ( - float(flyback_match.group(1)) if flyback_match else 0 - ) + parsed_info["Scope"]["fastZ_flybackTime"] = float(flyback_match.group(1)) if flyback_match else 0 # ----------------------------------------------------- # Timing-related ScanImage parameters @@ -345,25 +319,15 @@ def extract_float(pattern, default=0): return float(match.group(1)) if match else default - parsed_info["Scope"]["linePeriod"] = extract_float( - r"SI\.hRoiManager\.linePeriod = ([0-9.eE\-]+)" - ) + parsed_info["Scope"]["linePeriod"] = extract_float(r"SI\.hRoiManager\.linePeriod = ([0-9.eE\-]+)") - parsed_info["Scope"]["scanFramePeriod"] = extract_float( - r"SI\.hRoiManager\.scanFramePeriod = ([0-9.eE\-]+)" - ) + parsed_info["Scope"]["scanFramePeriod"] = extract_float(r"SI\.hRoiManager\.scanFramePeriod = ([0-9.eE\-]+)") - parsed_info["Scope"]["scanFrameRate"] = extract_float( - r"SI\.hRoiManager\.scanFrameRate = ([0-9.eE\-]+)" - ) + parsed_info["Scope"]["scanFrameRate"] = extract_float(r"SI\.hRoiManager\.scanFrameRate = ([0-9.eE\-]+)") - parsed_info["Scope"]["scanVolumeRate"] = extract_float( - r"SI\.hRoiManager\.scanVolumeRate = ([0-9.eE\-]+)" - ) + parsed_info["Scope"]["scanVolumeRate"] = extract_float(r"SI\.hRoiManager\.scanVolumeRate = ([0-9.eE\-]+)") - parsed_info["Scope"]["flybackTimePerFrame"] = extract_float( - r"SI\.hScan2D\.flybackTimePerFrame = ([0-9.eE\-]+)" - ) + parsed_info["Scope"]["flybackTimePerFrame"] = extract_float(r"SI\.hScan2D\.flybackTimePerFrame = ([0-9.eE\-]+)") parsed_info["Scope"]["flytoTimePerScanfield"] = extract_float( r"SI\.hScan2D\.flytoTimePerScanfield = ([0-9.eE\-]+)" @@ -374,9 +338,7 @@ def extract_float(pattern, default=0): # ----------------------------------------------------- if "fovCornerPoints" in scope_str: - fov_match = re.search( - r"SI\.hScan2D\.fovCornerPoints = (\[.+?\])", scope_str, re.DOTALL - ) + fov_match = re.search(r"SI\.hScan2D\.fovCornerPoints = (\[.+?\])", scope_str, re.DOTALL) if fov_match: try: @@ -386,9 +348,7 @@ def extract_float(pattern, default=0): fov_points = np.array(ast.literal_eval(new_fov_match)) fov_points = fov_points.reshape(4, 2) - parsed_info["Scope"]["fovCornerPoints"] = ( - resolution_factor * fov_points - ) + parsed_info["Scope"]["fovCornerPoints"] = resolution_factor * fov_points except Exception: parsed_info["Scope"]["fovCornerPoints"] = 0 @@ -442,9 +402,7 @@ def parse_tif_header_mesoscope(tif_fn, skip_behav_sync=False): artist_tag = first_page.tags.get("Artist") - scope_str = ( - str(software_tag.value) if software_tag is not None else image_description - ) + scope_str = str(software_tag.value) if software_tag is not None else image_description roi_info = str(artist_tag.value) if artist_tag is not None else "" @@ -470,13 +428,9 @@ def extract_float(pattern, text, default=0): return float(match.group(0)) if match else default - parsed_info["frameRate"] = extract_float( - r"(?<=SI\.hRoiManager\.scanVolumeRate = )\d+\.\d+", scope_str - ) + parsed_info["frameRate"] = extract_float(r"(?<=SI\.hRoiManager\.scanVolumeRate = )\d+\.\d+", scope_str) - parsed_info["interROIlag_sec"] = extract_float( - r"(?<=SI\.hScan2D\.flytoTimePerScanfield = )\d+\.\d+", scope_str - ) + parsed_info["interROIlag_sec"] = extract_float(r"(?<=SI\.hScan2D\.flytoTimePerScanfield = )\d+\.\d+", scope_str) # ----------------------------------------------------- # Timing / behavior sync @@ -516,18 +470,14 @@ def extract_float(pattern, text, default=0): # ROI parsing # ----------------------------------------------------- - roi_marks = [ - m.start() for m in re.finditer(r'"scanimage\.mroi\.Roi"', roi_info) - ] + roi_marks = [m.start() for m in re.finditer(r'"scanimage\.mroi\.Roi"', roi_info)] parsed_info["nROIs"] = len(roi_marks) parsed_info["ROI"] = [] # Resolution scaling - resolution_match = re.search( - r"(?<=SI\.objectiveResolution = )\d+\.\d+", scope_str - ) + resolution_match = re.search(r"(?<=SI\.objectiveResolution = )\d+\.\d+", scope_str) if resolution_match: resolution_factor = xySizeFactor * float(resolution_match.group(0)) @@ -581,31 +531,19 @@ def parse_array(pattern, text): # ROI geometry # ------------------------------------------------- - roi_dict["centerXY"] = resolution_factor * parse_array( - r'(?<="centerXY": )\[.+?]', this_roi - ) + roi_dict["centerXY"] = resolution_factor * parse_array(r'(?<="centerXY": )\[.+?]', this_roi) - roi_dict["sizeXY"] = resolution_factor * parse_array( - r'(?<="sizeXY": )\[.+?]', this_roi - ) + roi_dict["sizeXY"] = resolution_factor * parse_array(r'(?<="sizeXY": )\[.+?]', this_roi) - rotation_match = re.search( - r'(?<="rotationDegrees":) (\d+|\d+\.\d+)', this_roi - ) + rotation_match = re.search(r'(?<="rotationDegrees":) (\d+|\d+\.\d+)', this_roi) - roi_dict["rotationDegrees"] = ( - float(rotation_match.group(0)) if rotation_match else 0 - ) + roi_dict["rotationDegrees"] = float(rotation_match.group(0)) if rotation_match else 0 - roi_dict["pixelResolutionXY"] = parse_array( - r'(?<="pixelResolutionXY": )\[.+?]', this_roi - ) + roi_dict["pixelResolutionXY"] = parse_array(r'(?<="pixelResolutionXY": )\[.+?]', this_roi) discrete_match = re.search(r'(?<="discretePlaneMode":) \d', this_roi) - roi_dict["discretePlaneMode"] = ( - bool(int(discrete_match.group(0))) if discrete_match else False - ) + roi_dict["discretePlaneMode"] = bool(int(discrete_match.group(0))) if discrete_match else False power_match = re.search(r'(?<="powers":) \d+', this_roi) @@ -623,9 +561,7 @@ def parse_array(pattern, text): if not re.search(r'(?<="powers":) \d+', roi_info): power_match = re.search(r"(?<=SI\.hBeams\.powers = )\d+", scope_str) - parsed_info["Scope"]["Power_percent"] = ( - float(power_match.group(0)) if power_match else 0 - ) + parsed_info["Scope"]["Power_percent"] = float(power_match.group(0)) if power_match else 0 else: parsed_info["Scope"]["Power_percent"] = "discrete powers per ROI" @@ -633,20 +569,14 @@ def parse_array(pattern, text): # Channels channel_match = re.search(r"(?<=SI\.hChannels\.channelSave = )\d+", scope_str) - parsed_info["Scope"]["Channels"] = ( - int(channel_match.group(0)) if channel_match else 0 - ) + parsed_info["Scope"]["Channels"] = int(channel_match.group(0)) if channel_match else 0 # Config filenames - cfg_match = re.search( - r"(?<=SI\.hConfigurationSaver\.cfgFilename = ').+cfg", scope_str - ) + cfg_match = re.search(r"(?<=SI\.hConfigurationSaver\.cfgFilename = ').+cfg", scope_str) parsed_info["Scope"]["cfgFilename"] = cfg_match.group(0) if cfg_match else "" - usr_match = re.search( - r"(?<=SI\.hConfigurationSaver\.usrFilename = ').+usr", scope_str - ) + usr_match = re.search(r"(?<=SI\.hConfigurationSaver\.usrFilename = ').+usr", scope_str) parsed_info["Scope"]["usrFilename"] = usr_match.group(0) if usr_match else "" @@ -701,21 +631,13 @@ def parse_array(pattern, text): parsed_info["Scope"]["stacks_enabled"] = 1 if stacks_enabled == "true" else 0 if stacks_enabled == "true": - actuator_match = re.search( - r"(?<=SI\.hStackManager\.stackActuator = ')\w+", scope_str - ) + actuator_match = re.search(r"(?<=SI\.hStackManager\.stackActuator = ')\w+", scope_str) - definition_match = re.search( - r"(?<=SI\.hStackManager\.stackDefinition = ')\w+", scope_str - ) + definition_match = re.search(r"(?<=SI\.hStackManager\.stackDefinition = ')\w+", scope_str) - parsed_info["Scope"]["stackActuator"] = ( - actuator_match.group(0) if actuator_match else "" - ) + parsed_info["Scope"]["stackActuator"] = actuator_match.group(0) if actuator_match else "" - parsed_info["Scope"]["stackDefinition"] = ( - definition_match.group(0) if definition_match else "" - ) + parsed_info["Scope"]["stackDefinition"] = definition_match.group(0) if definition_match else "" # ----------------------------------------------------- # Motion correction @@ -725,14 +647,10 @@ def parse_array(pattern, text): motion_enabled = motion_match.group(0) if motion_match else "false" - parsed_info["Scope"]["motionCorrection_enabled"] = ( - 1 if motion_enabled == "true" else 0 - ) + parsed_info["Scope"]["motionCorrection_enabled"] = 1 if motion_enabled == "true" else 0 if motion_enabled == "true": - correction_z_match = re.search( - r"(?<=SI\.hMotionManager\.correctionEnableZ = )\w+", scope_str - ) + correction_z_match = re.search(r"(?<=SI\.hMotionManager\.correctionEnableZ = )\w+", scope_str) if correction_z_match and correction_z_match.group(0) == "true": parsed_info["Scope"]["motionCorMode"] = "automated" @@ -747,19 +665,13 @@ def parse_array(pattern, text): depth_match = re.search(r"SI\.hFastZ\.numFramesPerVolume = \d+", scope_str) if depth_match: - parsed_info["nDepths"] = int( - re.search(r"\d+", depth_match.group(0)).group(0) - ) + parsed_info["nDepths"] = int(re.search(r"\d+", depth_match.group(0)).group(0)) else: if stacks_enabled == "true": - slices_match = re.search( - r"(?<=SI\.hStackManager\.actualNumSlices = )\d+", scope_str - ) + slices_match = re.search(r"(?<=SI\.hStackManager\.actualNumSlices = )\d+", scope_str) - parsed_info["nDepths"] = ( - int(slices_match.group(0)) if slices_match else 1 - ) + parsed_info["nDepths"] = int(slices_match.group(0)) if slices_match else 1 else: parsed_info["nDepths"] = 1 diff --git a/u19_pipeline/utils/tiff_utils.py b/u19_pipeline/utils/tiff_utils.py index 9e8947e8..b1b7872c 100644 --- a/u19_pipeline/utils/tiff_utils.py +++ b/u19_pipeline/utils/tiff_utils.py @@ -83,9 +83,7 @@ def get_recording_info(fl, imheader, parsed_info): for i, file in enumerate(fl): if i == 0: rec_info = parsed_info[i] - rec_info["Timing"]["BehavFrames"] = np.array( - rec_info["Timing"]["BehavFrames"], dtype=object - ) + rec_info["Timing"]["BehavFrames"] = np.array(rec_info["Timing"]["BehavFrames"], dtype=object) else: if parsed_info[i]["Timing"]["Frame_ts_sec"][0] == 0: @@ -104,9 +102,7 @@ def get_recording_info(fl, imheader, parsed_info): aux = np.squeeze(aux) if aux.size > 0: - rec_info["Timing"]["BehavFrames"] = np.concatenate( - [rec_info["Timing"]["BehavFrames"], aux] - ) + rec_info["Timing"]["BehavFrames"] = np.concatenate([rec_info["Timing"]["BehavFrames"], aux]) frames_per_file[i] = len(imheader[i]) @@ -268,9 +264,7 @@ def clean_software_string(software): if software is None: return "" - software = software.replace( - "hRoiManager.mroiEnable = 1", "hRoiManager.mroiEnable = 0" - ) + software = software.replace("hRoiManager.mroiEnable = 1", "hRoiManager.mroiEnable = 0") idx = software.find("SI.hRoiManager.imagingFovUm") if idx != -1: @@ -321,11 +315,7 @@ def get_fov_mesoscope( # --------------------------------------------------------------------- for idepth in range(depths): - which_roi = [ - i - for i, roi in enumerate(rec_info["ROI"]) - if roi["Zs"] == which_depths[idepth] - ] + which_roi = [i for i, roi in enumerate(rec_info["ROI"]) if roi["Zs"] == which_depths[idepth]] for iroi in which_roi: dirname = f"ROI{iroi + 1:02d}_z{idepth + 1}" @@ -347,9 +337,7 @@ def get_fov_mesoscope( first_page = pages[0].asarray() - thisstack = np.zeros( - (imheader[iF][0]["Height"], 512, len(imheader[iF])), dtype=np.uint16 - ) + thisstack = np.zeros((imheader[iF][0]["Height"], 512, len(imheader[iF])), dtype=np.uint16) pixel2sum = 1 @@ -359,9 +347,7 @@ def get_fov_mesoscope( if temp_stack.shape[1] != thisstack.shape[1]: pixel2sum = imheader[iF][0]["Width"] // 512 - reshaped = temp_stack.reshape( - temp_stack.shape[0], pixel2sum, 512 - ) + reshaped = temp_stack.reshape(temp_stack.shape[0], pixel2sum, 512) temp_stack = reshaped.sum(axis=1) @@ -380,18 +366,12 @@ def get_fov_mesoscope( ilag = 0 rowct = 0 - which_roi = [ - i - for i, roi in enumerate(rec_info["ROI"]) - if roi["Zs"] == which_depths[idepth] - ] + which_roi = [i for i, roi in enumerate(rec_info["ROI"]) if roi["Zs"] == which_depths[idepth]] for iroi in which_roi: zidx = list(range(idepth, thisstack.shape[2], depths)) - substack = thisstack[rowct : rowct + roi_nr[iroi], :nc, :][ - :, :, zidx - ] + substack = thisstack[rowct : rowct + roi_nr[iroi], :nc, :][:, :, zidx] # ----------------------------------------------------- # Output filename @@ -412,21 +392,13 @@ def get_fov_mesoscope( first_desc = imheader[iF][zidx[0]]["ImageDescription"] - first_desc = replace_frame_timestamp( - first_desc, inter_roi_lag * ilag - ) + first_desc = replace_frame_timestamp(first_desc, inter_roi_lag * ilag) software = clean_software_string( - current_header.get("Software", None).value - if "Software" in current_header - else "" + current_header.get("Software", None).value if "Software" in current_header else "" ) - artist = ( - current_header.get("Artist", None).value - if "Artist" in current_header - else "" - ) + artist = current_header.get("Artist", None).value if "Artist" in current_header else "" metadata = { "ImageDescription": first_desc, @@ -442,9 +414,7 @@ def get_fov_mesoscope( for iz in range(substack.shape[2]): desc = imheader[iF][zidx[iz]]["ImageDescription"] - desc = replace_frame_timestamp( - desc, inter_roi_lag * ilag - ) + desc = replace_frame_timestamp(desc, inter_roi_lag * ilag) writer.write( substack[:, :, iz], @@ -460,9 +430,7 @@ def get_fov_mesoscope( # ----------------------------------------------------- if len(which_roi) > 1: - padsize = (nr - sum([roi_nr[r] for r in which_roi])) / ( - len(which_roi) - 1 - ) + padsize = (nr - sum([roi_nr[r] for r in which_roi])) / (len(which_roi) - 1) rowct += int(padsize + roi_nr[iroi]) @@ -487,9 +455,7 @@ def get_fov_mesoscope( fov_key["tiff_split"] = ct - fov_key["tiff_split_directory"] = ( - f"{scan_dirs_db['recording_directory']}/ROI{iroi + 1:02d}_z{iz + 1}/" - ) + fov_key["tiff_split_directory"] = f"{scan_dirs_db['recording_directory']}/ROI{iroi + 1:02d}_z{iz + 1}/" roi_name = rec_info["ROI"][iroi]["name"] @@ -501,33 +467,21 @@ def get_fov_mesoscope( fov_key["tiff_split_name"] = thisname # Safe assignments - fov_key["fov_depth"] = ( - rec_info["ROI"][iroi]["Zs"] - if rec_info["ROI"][iroi]["Zs"] is not None - else 0 - ) + fov_key["fov_depth"] = rec_info["ROI"][iroi]["Zs"] if rec_info["ROI"][iroi]["Zs"] is not None else 0 fov_key["fov_center_xy"] = rec_info["ROI"][iroi].get("centerXY", -1) fov_key["fov_size_xy"] = rec_info["ROI"][iroi].get("sizeXY", -1) - fov_key["fov_rotation_degrees"] = rec_info["ROI"][iroi].get( - "rotationDegrees", -1 - ) + fov_key["fov_rotation_degrees"] = rec_info["ROI"][iroi].get("rotationDegrees", -1) - fov_key["fov_pixel_resolution_xy"] = rec_info["ROI"][iroi].get( - "pixelResolutionXY", -1 - ) + fov_key["fov_pixel_resolution_xy"] = rec_info["ROI"][iroi].get("pixelResolutionXY", -1) - fov_key["fov_discrete_plane_mode"] = rec_info["ROI"][iroi].get( - "discretePlaneMode", -1 - ) + fov_key["fov_discrete_plane_mode"] = rec_info["ROI"][iroi].get("discretePlaneMode", -1) if not fov_key["fov_discrete_plane_mode"]: fov_key["fov_discrete_plane_mode"] = 0 - fov_key["power_percent"] = rec_info["ROI"][iroi].get( - "Power_percent", rec_info["Scope"]["Power_percent"] - ) + fov_key["power_percent"] = rec_info["ROI"][iroi].get("Power_percent", rec_info["Scope"]["Power_percent"]) fov_keys.append(fov_key) @@ -590,9 +544,7 @@ def get_fov_photonmicro(key, rec_info, scan_dirs_db): # ----------------------------------------------------------------------------- -def get_fovfile_photonmicro( - key, fl, imheader, patt_acq_number=r"_[0-9]{5}_", patt_file_number=r"_[0-9]{5}\." -): +def get_fovfile_photonmicro(key, fl, imheader, patt_acq_number=r"_[0-9]{5}_", patt_file_number=r"_[0-9]{5}\."): """ Insert TIFF split file entries for 2-photon imaging. """ From ec32e390fa0e42d82a687a36aa9a5f6f426245d9 Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Thu, 11 Jun 2026 22:34:16 -0400 Subject: [PATCH 5/6] exclude selected ruff rules; then fix more files --- initial_conf.py | 8 +-- notebooks/imaging_element/read_params.py | 50 ++++++++--------- pyproject.toml | 2 +- scripts/ingest_hdf5_to_db.py | 15 +---- scripts/populate_imaging.py | 2 +- u19_pipeline/__init__.py | 4 -- u19_pipeline/alert_system/behavior_metrics.py | 5 +- u19_pipeline/alert_system/cronjob_alert.py | 4 +- .../live_monitor_alert/live_monitor_alert.py | 46 ++++++++------- .../locked_tables_alert.py | 14 ++--- .../log_deletion/old_log_deletion.py | 2 +- .../alert_system/main_alert_system.py | 16 +++--- ...ipt.py => no_db_backup_creation_script.py} | 2 +- .../rig_maintenance/check_rig_maintenance.py | 24 +++----- .../schedule_check_alert.py | 30 +++++----- .../alert_system/tech_alert/tech_alert.py | 18 +++--- .../create_weighing_gui_ss.py | 26 ++++----- .../water_weigh_alert/water_weigh_alert.py | 26 ++++----- .../clusters_paths_and_transfers.py | 19 +++---- .../create_missing_lfps_script.py | 2 +- .../automatic_job/ephys_element_ingest.py | 2 +- .../automatic_job/ephys_element_populate.py | 21 +++---- u19_pipeline/automatic_job/imaging_element.py | 12 ++-- .../automatic_job/imaging_element_populate.py | 25 ++++----- u19_pipeline/automatic_job/params_config.py | 2 +- .../automatic_job/pupillometry_handler.py | 22 ++++---- .../automatic_job/recording_handler.py | 3 +- .../recording_process_handler.py | 14 ++--- u19_pipeline/automatic_job/slurm_creator.py | 2 +- u19_pipeline/automatic_job/tigress2pni.py | 11 ++-- u19_pipeline/copy_table.py | 6 +- u19_pipeline/ephys_pipeline.py | 14 ++--- u19_pipeline/ephys_sync.py | 8 +-- u19_pipeline/imaging_pipeline.py | 23 +------- u19_pipeline/lab.py | 6 +- u19_pipeline/recording.py | 4 +- u19_pipeline/recording_process.py | 6 +- u19_pipeline/utility.py | 16 +++--- .../{readSGLX.py => read_sglx.py} | 26 +++------ u19_pipeline/utils/dj_shortcuts.py | 8 +-- u19_pipeline/utils/dlc_process.py | 7 +-- u19_pipeline/utils/ephys_fix_sync_code.py | 12 ++-- u19_pipeline/utils/ephys_utils.py | 53 ++++++++---------- u19_pipeline/utils/insert_miscelaneous_db.py | 2 +- u19_pipeline/utils/matlab_utils.py | 30 +++------- u19_pipeline/utils/path_utils.py | 12 ++-- u19_pipeline/utils/scp_transfers.py | 2 +- u19_pipeline/utils/slack_utils.py | 56 +++++++++---------- .../utils/tiff_matlab_imaging_utils.py | 33 ++++------- u19_pipeline/utils/tiff_utils.py | 32 ++++------- 50 files changed, 334 insertions(+), 451 deletions(-) rename u19_pipeline/alert_system/noDB_backup_creation/{noDB_backup_creation_script.py => no_db_backup_creation_script.py} (99%) rename u19_pipeline/utils/DemoReadSGLXData/{readSGLX.py => read_sglx.py} (95%) diff --git a/initial_conf.py b/initial_conf.py index a85e059f..31897693 100755 --- a/initial_conf.py +++ b/initial_conf.py @@ -62,7 +62,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True): dj.config["database.host"] = host if "custom" not in dj.config: - dj.config["custom"] = dict() + dj.config["custom"] = {} dj.config["custom"]["database.prefix"] = "u19_" import u19_pipeline.lab as lab @@ -87,11 +87,11 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True): # Get store info if "stores" not in dj.config: - dj.config["stores"] = dict() + dj.config["stores"] = {} dj_stores = lab.DjStores.fetch(as_dict=True) - dj_stores_dict = dict() + dj_stores_dict = {} for i in dj_stores: store_name = i.pop("store_name") dj_stores_dict[store_name] = i @@ -124,7 +124,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True): action="store_true", ) - default_args = dict() + default_args = {} default_args["save_user"] = True default_args["replace_user"] = False default_args["global_config"] = False diff --git a/notebooks/imaging_element/read_params.py b/notebooks/imaging_element/read_params.py index c0e8e8f8..88dee707 100644 --- a/notebooks/imaging_element/read_params.py +++ b/notebooks/imaging_element/read_params.py @@ -33,13 +33,13 @@ params_dict_dict = {} num_params = 0 for idx, param_modality_list in enumerate(params_dict_list): - for dict in param_modality_list: - dict["recording_modality"] = modalities[idx] - dict["param_set_hash"] = str(dict["param_set_hash"]) - if "clustering_method" in dict: - dict["processing_method"] = dict.pop("clustering_method") + for entry in param_modality_list: + entry["recording_modality"] = modalities[idx] + entry["param_set_hash"] = str(entry["param_set_hash"]) + if "clustering_method" in entry: + entry["processing_method"] = entry.pop("clustering_method") - params_dict_dict["param_" + str(num_params)] = dict + params_dict_dict["param_" + str(num_params)] = entry num_params += 1 #################################################Fetch all preparamsStepList from all modalities @@ -51,19 +51,19 @@ preparams_steps_dict_dict = {} num_preparams_steps = 0 for idx, preparam_modality_list in enumerate(preparams_steps): - for dict in preparam_modality_list: - dict["param_set_hash"] = str(dict["param_set_hash"]) - dict["recording_modality"] = modalities[idx] - if "precluster_param_steps_id" in dict: - dict["preprocess_param_steps_id"] = dict.pop("precluster_param_steps_id") - if "precluster_method" in dict: - dict["preprocess_method"] = dict.pop("precluster_method") - if "precluster_param_steps_name" in dict: - dict["preprocess_param_steps_name"] = dict.pop("precluster_param_steps_name") - if "precluster_param_steps_desc" in dict: - dict["preprocess_param_steps_desc"] = dict.pop("precluster_param_steps_desc") - - preparams_steps_dict_dict["param_" + str(num_preparams_steps)] = dict + for entry in preparam_modality_list: + entry["param_set_hash"] = str(entry["param_set_hash"]) + entry["recording_modality"] = modalities[idx] + if "precluster_param_steps_id" in entry: + entry["preprocess_param_steps_id"] = entry.pop("precluster_param_steps_id") + if "precluster_method" in entry: + entry["preprocess_method"] = entry.pop("precluster_method") + if "precluster_param_steps_name" in entry: + entry["preprocess_param_steps_name"] = entry.pop("precluster_param_steps_name") + if "precluster_param_steps_desc" in entry: + entry["preprocess_param_steps_desc"] = entry.pop("precluster_param_steps_desc") + + preparams_steps_dict_dict["param_" + str(num_preparams_steps)] = entry num_preparams_steps += 1 #################################################Fetch all preparams from all modalities @@ -76,13 +76,13 @@ preparams_dict_dict = {} num_preparams = 0 for idx, preparam_modality_list in enumerate(preparams_dict_list): - for dict in preparam_modality_list: - dict["recording_modality"] = modalities[idx] - dict["param_set_hash"] = str(dict["param_set_hash"]) - if "precluster_method" in dict: - dict["preprocess_method"] = dict.pop("precluster_method") + for entry in preparam_modality_list: + entry["recording_modality"] = modalities[idx] + entry["param_set_hash"] = str(entry["param_set_hash"]) + if "precluster_method" in entry: + entry["preprocess_method"] = entry.pop("precluster_method") - preparams_dict_dict["param_" + str(num_preparams)] = dict + preparams_dict_dict["param_" + str(num_preparams)] = entry num_preparams += 1 """ diff --git a/pyproject.toml b/pyproject.toml index 3d8081fb..ba3d3f9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ select = [ # if you're feeling confident you can do: # select = ["ALL"] # and then manually ignore annoying ones: -ignore = ["N815", "A002", "N806", "E501", "SIM108"] +ignore = ["N815", "A002", "N802", "N803", "N806", "E402", "E501", "SIM108"] [dependency-groups] dev = [ diff --git a/scripts/ingest_hdf5_to_db.py b/scripts/ingest_hdf5_to_db.py index 89b51a9f..c21f73f4 100644 --- a/scripts/ingest_hdf5_to_db.py +++ b/scripts/ingest_hdf5_to_db.py @@ -318,18 +318,9 @@ this_session_notes_key = f"sessions/{session_compressed_str}/notes" if this_session_notes_key in stored_session_keys: session_notes_dict = json.loads(data[this_session_notes_key].iloc[0]) - if "notes" in session_notes_dict: - session_notes = session_notes_dict["notes"] - else: - session_notes = None - if "stdout" in session_notes_dict: - session_stdout = session_notes_dict["stdout"] - else: - session_stdout = None - if "stderr" in session_notes_dict: - session_stderr = session_notes_dict["stderr"] - else: - session_stderr = None + session_notes = session_notes_dict.get("notes", None) + session_stdout = session_notes_dict.get("stdout", None) + session_stderr = session_notes_dict.get("stderr", None) else: session_notes = None session_stdout = None diff --git a/scripts/populate_imaging.py b/scripts/populate_imaging.py index ce6e5ae1..5845def6 100644 --- a/scripts/populate_imaging.py +++ b/scripts/populate_imaging.py @@ -1,4 +1,4 @@ from u19_pipeline_python import acquisition -kargs = dict(supress_errors=True, display_progress=True) +kargs = {"supress_errors": True, "display_progress": True} acquisition.Scan.populate(**kargs) diff --git a/u19_pipeline/__init__.py b/u19_pipeline/__init__.py index 4c64b43a..e69de29b 100644 --- a/u19_pipeline/__init__.py +++ b/u19_pipeline/__init__.py @@ -1,4 +0,0 @@ -import os -from os.path import dirname, join - -import datajoint as dj diff --git a/u19_pipeline/alert_system/behavior_metrics.py b/u19_pipeline/alert_system/behavior_metrics.py index 17b4ecce..75ac5e96 100644 --- a/u19_pipeline/alert_system/behavior_metrics.py +++ b/u19_pipeline/alert_system/behavior_metrics.py @@ -46,10 +46,7 @@ def get_bias_from_trial_df(trials_df, return_all_metrics=False): trials_df["cum_correct_left_trials"] / trials_df["cum_left_trials"] ) - if return_all_metrics: - bias_df = trials_df - else: - bias_df = trials_df[original_columns + ["bias"]] + bias_df = trials_df if return_all_metrics else trials_df[original_columns + ["bias"]] return bias_df diff --git a/u19_pipeline/alert_system/cronjob_alert.py b/u19_pipeline/alert_system/cronjob_alert.py index df38e931..84b76a0f 100644 --- a/u19_pipeline/alert_system/cronjob_alert.py +++ b/u19_pipeline/alert_system/cronjob_alert.py @@ -8,9 +8,9 @@ import u19_pipeline.alert_system.live_session_stats_deletion.live_session_stats_deletion as lssd import u19_pipeline.alert_system.log_deletion.old_log_deletion as old -import u19_pipeline.alert_system.noDB_backup_creation.noDB_backup_creation_script as noDBbcs +import u19_pipeline.alert_system.noDB_backup_creation.no_db_backup_creation_script as no_db_bcs # mas.main_alert_system() old.main_old_log_deletion() lssd.main_live_session_stats_deletion() -noDBbcs.main_noDB_backup() +no_db_bcs.main_noDB_backup() diff --git a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py index eb7910ea..fedf67fa 100644 --- a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py +++ b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py @@ -20,13 +20,13 @@ def slack_alert_message_format_live_stats(alert_dictionary1, alert_dictionary2, now = datetime.now() datestr = now.strftime("%d-%b-%Y %H:%M:%S") - msep = dict() + msep = {} msep["type"] = "divider" # Title# - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ( ":rotating_light: *Live Monitor Alert* on " @@ -40,29 +40,29 @@ def slack_alert_message_format_live_stats(alert_dictionary1, alert_dictionary2, m1["text"] = m1_1 # Info# - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" m2_1["text"] = "*Session Reported:*" + "\n" - for key in alert_dictionary1.keys(): + for key in alert_dictionary1: m2_1["text"] += "*" + key + "* : " + str(alert_dictionary1[key]) + "\n" m2_1["text"] += "\n" m2["text"] = m2_1 - m4 = dict() + m4 = {} m4["type"] = "section" - m4_1 = dict() + m4_1 = {} m4_1["type"] = "mrkdwn" m4_1["text"] = "*Last Stats Reported*:" + "\n" - for key in alert_dictionary2.keys(): + for key in alert_dictionary2: m4_1["text"] += "*" + key + "* : " + str(alert_dictionary2[key]) + "\n" m4_1["text"] += "\n" m4["text"] = m4_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2, msep, m4, msep] message["text"] = "Live Monitor Alert" @@ -144,7 +144,7 @@ def main_live_monitor_alert(): query_live_stats = sessions.to_dict("records") # Last non violation trial in sessions - query_no_violation_trial = dict() + query_no_violation_trial = {} query_no_violation_trial["violation_trial"] = 0 lss_nvio = acquisition.SessionStarted.aggr( (acquisition.LiveSessionStats & query_no_violation_trial).proj("current_datetime"), @@ -154,7 +154,7 @@ def main_live_monitor_alert(): live_stats_nvio = live_stats_nvio.rename({"current_datetime": "last_non_violation_trial"}, axis=1) # Last violation trial in sessions - query_violation_trial = dict() + query_violation_trial = {} query_violation_trial["violation_trial"] = 1 lss_vio = acquisition.SessionStarted.aggr( (acquisition.LiveSessionStats & query_violation_trial).proj("current_datetime"), @@ -199,26 +199,26 @@ def main_live_monitor_alert(): print(live_stats.T) live_stats = live_stats.loc[ - (live_stats["alert_nvio"] == True) | (live_stats["alert_vio"] == True), + (live_stats["alert_nvio"]) | (live_stats["alert_vio"]), :, ] # If there are any sessions to alert (more then SECONDS_ALERT) if live_stats.shape[0] > 0: live_stats["current_datetime"] = fake_date - live_stats.loc[live_stats["alert_vio"] == True, "current_datetime"] = live_stats.loc[ - live_stats["alert_vio"] == True, "last_violation_trial" + live_stats.loc[live_stats["alert_vio"], "current_datetime"] = live_stats.loc[ + live_stats["alert_vio"], "last_violation_trial" ] - live_stats.loc[live_stats["alert_nvio"] == True, "current_datetime"] = live_stats.loc[ - live_stats["alert_nvio"] == True, "last_non_violation_trial" + live_stats.loc[live_stats["alert_nvio"], "current_datetime"] = live_stats.loc[ + live_stats["alert_nvio"], "last_non_violation_trial" ] live_stats["seconds_elapsed_last_valid_stat"] = 0 - live_stats.loc[live_stats["alert_vio"] == True, "seconds_elapsed_last_valid_stat"] = live_stats.loc[ - live_stats["alert_vio"] == True, "seconds_elapsed_session_started" + live_stats.loc[live_stats["alert_vio"], "seconds_elapsed_last_valid_stat"] = live_stats.loc[ + live_stats["alert_vio"], "seconds_elapsed_session_started" ] - live_stats.loc[live_stats["alert_nvio"] == True, "seconds_elapsed_last_valid_stat"] = live_stats.loc[ - live_stats["alert_nvio"] == True, "seconds_elapsed_last_stat_nvio" + live_stats.loc[live_stats["alert_nvio"], "seconds_elapsed_last_valid_stat"] = live_stats.loc[ + live_stats["alert_nvio"], "seconds_elapsed_last_stat_nvio" ] # get_session_info to alert (plus slack researcher) @@ -278,8 +278,7 @@ def main_live_monitor_alert(): ls_full_dict = ls_full_df.to_dict("records") # Send one alert per session found - idx_alert = 0 - for this_alert_record in ls_full_dict: + for idx_alert, this_alert_record in enumerate(ls_full_dict): # Format message for session and live stat dictionary this_session_stats = session_data_df.iloc[idx_alert, :] slack_json_message = slack_alert_message_format_live_stats( @@ -298,4 +297,3 @@ def main_live_monitor_alert(): reported_session["report_datetime"] = right_now_est acquisition.ReportedLiveSessionStats.insert1(reported_session.to_dict()) - idx_alert += 1 diff --git a/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py b/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py index 503a0f13..6772acdd 100644 --- a/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py +++ b/u19_pipeline/alert_system/locked_tables_alert/locked_tables_alert.py @@ -34,30 +34,30 @@ def main_locked_tables_alert(): def slack_alert_message_format_locked_tables(locked_tables_df): now = datetime.datetime.now() - datestr = now.strftime("%d-%b-%Y %H:%M:%S") + now.strftime("%d-%b-%Y %H:%M:%S") - msep = dict() + msep = {} msep["type"] = "divider" # Title# - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ":rotating_light: *Locked Tables Alert *" m1["text"] = m1_1 # Info for subjects missing water - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" m2_1["text"] = "*Locked tables:*" + "\n" m2_1["text"] += locked_tables_df m2["text"] = m2_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2, msep] message["text"] = "Locked Tables Alert" diff --git a/u19_pipeline/alert_system/log_deletion/old_log_deletion.py b/u19_pipeline/alert_system/log_deletion/old_log_deletion.py index a9224176..395208e1 100644 --- a/u19_pipeline/alert_system/log_deletion/old_log_deletion.py +++ b/u19_pipeline/alert_system/log_deletion/old_log_deletion.py @@ -23,7 +23,7 @@ def main_old_log_deletion(): files_df["exctract_date"] = pd.to_datetime(files_df["exctract_date"], format="%Y%m%d") files_df["before_ref_date"] = files_df["exctract_date"] <= pd.Timestamp(reference_date) - files_for_deletion = files_df.loc[files_df["before_ref_date"] == True, "filepaths"].to_list() + files_for_deletion = files_df.loc[files_df["before_ref_date"], "filepaths"].to_list() for file_path in files_for_deletion: if file_path.is_file(): # Check if it's actually a file before attempting to delete diff --git a/u19_pipeline/alert_system/main_alert_system.py b/u19_pipeline/alert_system/main_alert_system.py index c261cd78..d3892abe 100644 --- a/u19_pipeline/alert_system/main_alert_system.py +++ b/u19_pipeline/alert_system/main_alert_system.py @@ -50,7 +50,7 @@ def main_alert_system(): time.sleep(1) except Exception as e: - dict_error = dict() + dict_error = {} dict_error["message"] = "error while executing " + this_alert_submodule.name + " alert code" dict_error["error_exception"] = "".join(traceback.format_exception(type(e), value=e, tb=e.__traceback__)) slack_json_message = slack_alert_message_format(dict_error, this_alert_submodule.name) @@ -66,29 +66,29 @@ def slack_alert_message_format(alert_dictionaty, alert_module_name): now = datetime.datetime.now() datestr = now.strftime("%d-%b-%Y %H:%M:%S") - msep = dict() + msep = {} msep["type"] = "divider" # Title# - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ":rotating_light: *" + alert_module_name + "* on " + datestr + "\n\n" m1["text"] = m1_1 # Info# - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" m2_1["text"] = "" - for key in alert_dictionaty.keys(): + for key in alert_dictionaty: m2_1["text"] += "*" + key + "* : " + str(alert_dictionaty[key]) + "\n" m2["text"] = m2_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2, msep, msep] message["text"] = alert_module_name + " alert" diff --git a/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py b/u19_pipeline/alert_system/noDB_backup_creation/no_db_backup_creation_script.py similarity index 99% rename from u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py rename to u19_pipeline/alert_system/noDB_backup_creation/no_db_backup_creation_script.py index cdd7092f..35be59ff 100644 --- a/u19_pipeline/alert_system/noDB_backup_creation/noDB_backup_creation_script.py +++ b/u19_pipeline/alert_system/noDB_backup_creation/no_db_backup_creation_script.py @@ -101,7 +101,7 @@ def write_rig_status_file(): def write_schedule_file(): - schedule_query = dict() + schedule_query = {} schedule_query["date"] = datetime.date.today() subject_query = "subject_fullname is not null" diff --git a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py index 6a7da4fb..e2a530d7 100755 --- a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py +++ b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py @@ -12,6 +12,7 @@ - Uses the rigs_issues_and_troubleshooting webhook for notifications """ +import contextlib import json import logging import os @@ -43,19 +44,13 @@ def setup_logging(): # Create logs directory in the user's home directory (allow override via env var) # Default: ~/.u19_pipeline/logs env_log_dir = os.getenv("U19_PIPELINE_LOG_DIR") - if env_log_dir: - log_dir = Path(env_log_dir) - else: - log_dir = Path.home() / "u19_pipeline_logs" + log_dir = Path(env_log_dir) if env_log_dir else Path.home() / "u19_pipeline_logs" log_dir.mkdir(parents=True, exist_ok=True) # Create log file with timestamp timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") log_file = log_dir / f"rig_maintenance_check_{timestamp}.log" - # Open log file handle - log_file_handle = open(log_file, "a", encoding="utf-8") - # Configure standard logging: StreamHandler (console) + FileHandler (file) logger = logging.getLogger("rig_maintenance") logger.setLevel(logging.INFO) @@ -76,7 +71,7 @@ def setup_logging(): # Prevent messages from also being handled by the root logger logger.propagate = False - return logger, log_file, log_file_handle + return logger, log_file def get_slack_webhook(): @@ -529,10 +524,9 @@ def dataframe_to_slack_table_blocks(df: pd.DataFrame, chunk_size: int = 15, **kw def main(): """Main function to run the maintenance check.""" - log_file_handle = None try: # Set up logging - _, log_file, log_file_handle = setup_logging() + _, log_file = setup_logging() logger.info("🔧 Rig Maintenance Status Checker") logger.info("=" * 60) @@ -606,12 +600,10 @@ def main(): logger.exception(f"❌ Error during maintenance check: {e}") sys.exit(1) finally: - # Close the log file handle - if log_file_handle: - try: - log_file_handle.close() - except Exception: - pass + # Close any file handlers to flush logs + for handler in logging.getLogger("rig_maintenance").handlers[:]: + with contextlib.suppress(Exception): + handler.close() if __name__ == "__main__": diff --git a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py index 80e27bd4..a64fffbf 100644 --- a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py +++ b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py @@ -53,7 +53,7 @@ def main_schedule_check_alert(): on=["location"], suffixes=["_today", "_tomorrow"], ) - schedule_comp + print(schedule_comp) schedule_comp["diff_subjects"] = schedule_comp["#subj_tomorrow"] - schedule_comp["#subj_today"] schedule_comp["rig_less_subjects"] = schedule_comp["diff_subjects"] < -2 @@ -88,30 +88,30 @@ def main_schedule_check_alert(): def slack_alert_message_format_schedule(schedule_df_string): now = datetime.datetime.now() - datestr = now.strftime("%d-%b-%Y %H:%M:%S") + now.strftime("%d-%b-%Y %H:%M:%S") - msep = dict() + msep = {} msep["type"] = "divider" # Title# - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ":rotating_light: *Schedule Alert *" m1["text"] = m1_1 # Info for subjects missing water - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" m2_1["text"] = "Significantly less subjects scheduled tomorrow\nSchedule per rig:" + "\n\n" m2_1["text"] += schedule_df_string m2["text"] = m2_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2, msep] message["text"] = "Suspicious Schedule Alert" @@ -120,29 +120,29 @@ def slack_alert_message_format_schedule(schedule_df_string): def slack_alert_empty_schedule(): now = datetime.datetime.now() - datestr = now.strftime("%d-%b-%Y %H:%M:%S") + now.strftime("%d-%b-%Y %H:%M:%S") - msep = dict() + msep = {} msep["type"] = "divider" # Title# - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ":rotating_light: *Schedule Alert *" m1["text"] = m1_1 # Info for subjects missing water - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" m2_1["text"] = "*No Schedule found for tomorrow:*" + "\n" m2["text"] = m2_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2, msep] message["text"] = "Schedule Empty Alert" diff --git a/u19_pipeline/alert_system/tech_alert/tech_alert.py b/u19_pipeline/alert_system/tech_alert/tech_alert.py index 302735ab..f20e3e82 100644 --- a/u19_pipeline/alert_system/tech_alert/tech_alert.py +++ b/u19_pipeline/alert_system/tech_alert/tech_alert.py @@ -36,25 +36,25 @@ def slack_alert_message_format_tech_alert(schedule_data): # Divider # - msep = dict() + msep = {} msep["type"] = "divider" # Title# - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = "🗓 *Today's Tech Schedule:*" m1["text"] = m1_1 - message = dict() + message = {} message["blocks"] = [m1, msep] message["text"] = "🗓 *Today's Tech Schedule:*" # Info# - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" # If there are shifts today, list them; otherwise provide a short fallback @@ -116,9 +116,9 @@ def slack_alert_message_format_tech_alert(schedule_data): if alerts: alert_text = "\n".join(alerts) - m3 = dict() + m3 = {} m3["type"] = "section" - m3_1 = dict() + m3_1 = {} m3_1["type"] = "mrkdwn" m3_1["text"] = f"*:rotating_light: Upcoming Shifts: :rotating_light:*\n{alert_text}" m3["text"] = m3_1 @@ -287,7 +287,7 @@ def fetch_and_parse_icalevents(weburl: str): event for event in vr_events if not re.search(r"as\s*lab*\s*clean*\s*up*\s*at", event.summary.lower()) ] if lab_clean_up_events: - unique_days = sorted(set(event.start.date() for event in lab_clean_up_events)) + unique_days = sorted({event.start.date() for event in lab_clean_up_events}) for day in unique_days: filtered_events.append( { diff --git a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py index bd552082..c0ceb80f 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py +++ b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py @@ -269,7 +269,7 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: def resolve_responsible_slack(row): schedule_today = row.get("schedule_today") - lab_name = row.get("lab") + row.get("lab") token = str(schedule_today).strip() if schedule_today is not None else None use_coowners = (token is None) or (token.lower() == "transport") or (token.lower() == "nothing") @@ -355,22 +355,22 @@ def slack_alert_message_format_weight_water( " " + slack_handles_formatted + ", please be advised that your labs' subjects are listed below." ) - msep = dict() + msep = {} msep["type"] = "divider" # Title# - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ":rotating_light: *Subjects Status Alert *" + lab_manager_text m1["text"] = m1_1 # Info for subjects missing water - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" if subjects_not_watered.empty: @@ -400,9 +400,9 @@ def slack_alert_message_format_weight_water( m2["text"] = m2_1 # Info for subjects missing weighing - m4 = dict() + m4 = {} m4["type"] = "section" - m4_1 = dict() + m4_1 = {} m4_1["type"] = "mrkdwn" if subjects_not_weighted.empty: @@ -433,9 +433,9 @@ def slack_alert_message_format_weight_water( m4["text"] = m4_1 # Info for subjects missing training - m5 = dict() + m5 = {} m5["type"] = "section" - m5_1 = dict() + m5_1 = {} m5_1["type"] = "mrkdwn" if subjects_not_trained.empty: @@ -453,9 +453,9 @@ def slack_alert_message_format_weight_water( m5["text"] = m5_1 # Info for missing transport - m6 = dict() + m6 = {} m6["type"] = "section" - m6_1 = dict() + m6_1 = {} m6_1["type"] = "mrkdwn" if missing_transport.empty: @@ -484,7 +484,7 @@ def slack_alert_message_format_weight_water( m6_1["text"] += line + "\n" m6["text"] = m6_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2, msep, m4, msep, m5, msep, m6, msep] # msg_groups = [m1, m2, m4, m5, m6] msg_groups = [m2] diff --git a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py index bd552082..c0ceb80f 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py +++ b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py @@ -269,7 +269,7 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: def resolve_responsible_slack(row): schedule_today = row.get("schedule_today") - lab_name = row.get("lab") + row.get("lab") token = str(schedule_today).strip() if schedule_today is not None else None use_coowners = (token is None) or (token.lower() == "transport") or (token.lower() == "nothing") @@ -355,22 +355,22 @@ def slack_alert_message_format_weight_water( " " + slack_handles_formatted + ", please be advised that your labs' subjects are listed below." ) - msep = dict() + msep = {} msep["type"] = "divider" # Title# - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ":rotating_light: *Subjects Status Alert *" + lab_manager_text m1["text"] = m1_1 # Info for subjects missing water - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" if subjects_not_watered.empty: @@ -400,9 +400,9 @@ def slack_alert_message_format_weight_water( m2["text"] = m2_1 # Info for subjects missing weighing - m4 = dict() + m4 = {} m4["type"] = "section" - m4_1 = dict() + m4_1 = {} m4_1["type"] = "mrkdwn" if subjects_not_weighted.empty: @@ -433,9 +433,9 @@ def slack_alert_message_format_weight_water( m4["text"] = m4_1 # Info for subjects missing training - m5 = dict() + m5 = {} m5["type"] = "section" - m5_1 = dict() + m5_1 = {} m5_1["type"] = "mrkdwn" if subjects_not_trained.empty: @@ -453,9 +453,9 @@ def slack_alert_message_format_weight_water( m5["text"] = m5_1 # Info for missing transport - m6 = dict() + m6 = {} m6["type"] = "section" - m6_1 = dict() + m6_1 = {} m6_1["type"] = "mrkdwn" if missing_transport.empty: @@ -484,7 +484,7 @@ def slack_alert_message_format_weight_water( m6_1["text"] += line + "\n" m6["text"] = m6_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2, msep, m4, msep, m5, msep, m6, msep] # msg_groups = [m1, m2, m4, m5, m6] msg_groups = [m2] diff --git a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py index 060e0342..13813f97 100644 --- a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py +++ b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py @@ -119,7 +119,7 @@ def get_cluster_vars(cluster): if cluster in cluster_vars: return cluster_vars[cluster] else: - raise ("Non existing cluster") + raise ValueError("Non existing cluster") def scp_file_transfer(source, dest): @@ -168,7 +168,7 @@ def request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, des print("**********************************") p = subprocess.run(globus_command, capture_output=True) print(p) - transfer_request = dict() + transfer_request = {} print("p.stderr", p.stderr) print("p.stdout", p.stdout) @@ -195,7 +195,7 @@ def request_globus_transfer_status(job_id): s = subprocess.run(globus_command, capture_output=True) task_output = json.loads(s.stdout.decode("UTF-8")) - transfer_request = dict() + transfer_request = {} if task_output["status"] == "SUCCEEDED": transfer_request["status"] = config.system_process["COMPLETED"] elif task_output["status"] in ["PENDING", "RETRYING", "ACTIVE"]: @@ -245,10 +245,10 @@ def translate_globus_output(stdout_process): n = u.split(sep="\n") n2 = [x.split(sep=":", maxsplit=1) for x in n] - flat_list = [item for l in n2 for item in l] + flat_list = [item for sublist in n2 for item in sublist] flat_list2 = [x.strip() for x in flat_list] - d1 = dict(zip(flat_list2[::2], flat_list2[1::2])) + d1 = dict(zip(flat_list2[::2], flat_list2[1::2], strict=False)) return d1 @@ -384,10 +384,7 @@ def delete_empty_data_directory_cluster(cluster, type="raw"): base_command = "ssh " + this_cluster_vars["user"] + "@" + this_cluster_vars["hostname"] + " " # Check base directory to delete - if type == "raw": - filepath = this_cluster_vars["root_data_dir"] - else: - filepath = this_cluster_vars["processed_data_dir"] + filepath = this_cluster_vars["root_data_dir"] if type == "raw" else this_cluster_vars["processed_data_dir"] # Repeat always if we find a directory to delete total_deletion = 0 @@ -405,9 +402,9 @@ def delete_empty_data_directory_cluster(cluster, type="raw"): if "" in list_dir: list_dir.remove("") - for dir in list_dir: + for directory in list_dir: # Check if directory has no files in it (empty) - command = base_command + "find " + dir + " -type f | wc -l" + command = base_command + "find " + directory + " -type f | wc -l" num_files = subprocess.check_output(command, shell=True) num_files = int(num_files.decode().strip()) diff --git a/u19_pipeline/automatic_job/create_missing_lfps_script.py b/u19_pipeline/automatic_job/create_missing_lfps_script.py index 062745b6..16e22f93 100644 --- a/u19_pipeline/automatic_job/create_missing_lfps_script.py +++ b/u19_pipeline/automatic_job/create_missing_lfps_script.py @@ -8,7 +8,7 @@ def main(recording_id): - rec_query = dict() + rec_query = {} rec_query["recording_id"] = recording_id recording_processes = (recording_process.Processing() & rec_query).fetch( diff --git a/u19_pipeline/automatic_job/ephys_element_ingest.py b/u19_pipeline/automatic_job/ephys_element_ingest.py index 4b102d2d..18bb05d7 100644 --- a/u19_pipeline/automatic_job/ephys_element_ingest.py +++ b/u19_pipeline/automatic_job/ephys_element_ingest.py @@ -38,7 +38,7 @@ def process_session(sess_key): :param scan_key: a `KEY` of `acquisition.Session` """ sess_dir = pathlib.Path(find_full_path(get_ephys_root_data_dir(), get_session_directory(sess_key))) - ephys_meta_filepaths = [fp for fp in sess_dir.rglob("*.ap.meta")] + ephys_meta_filepaths = list(sess_dir.rglob("*.ap.meta")) print("ephys_meta_filepaths", ephys_meta_filepaths) diff --git a/u19_pipeline/automatic_job/ephys_element_populate.py b/u19_pipeline/automatic_job/ephys_element_populate.py index 432a0391..65f19ece 100644 --- a/u19_pipeline/automatic_job/ephys_element_populate.py +++ b/u19_pipeline/automatic_job/ephys_element_populate.py @@ -12,7 +12,7 @@ def populate_element_data(job_id, display_progress=True, reserve_jobs=False, sup } process_key = ( - recording_process.Processing * recording.Recording & dict(recording_modality="electrophysiology", job_id=job_id) + recording_process.Processing * recording.Recording & {"recording_modality": "electrophysiology", "job_id": job_id} ).fetch1("KEY") fragment_number, recording_process_pre_path, recording_process_post_path = ( @@ -24,21 +24,18 @@ def populate_element_data(job_id, display_progress=True, reserve_jobs=False, sup ) precluster_paramsets = ( - ephys_element.PreClusterParamSteps.Step() & dict(precluster_param_steps_id=precluster_param_steps_id) + ephys_element.PreClusterParamSteps.Step() & {"precluster_param_steps_id": precluster_param_steps_id} ).fetch("paramset_idx") - clustering_method = (ephys_element.ClusteringParamSet & dict(paramset_idx=paramset_idx)).fetch1("clustering_method") + clustering_method = (ephys_element.ClusteringParamSet & {"paramset_idx": paramset_idx}).fetch1("clustering_method") - if len(precluster_paramsets) == 0: - task_mode = "none" - else: - task_mode = "load" + task_mode = "none" if len(precluster_paramsets) == 0 else "load" - precluster_key = dict( - recording_id=process_key["recording_id"], - insertion_number=fragment_number, - precluster_param_steps_id=precluster_param_steps_id, - ) + precluster_key = { + "recording_id": process_key["recording_id"], + "insertion_number": fragment_number, + "precluster_param_steps_id": precluster_param_steps_id, + } ephys_element.PreClusterTask.insert1( dict( diff --git a/u19_pipeline/automatic_job/imaging_element.py b/u19_pipeline/automatic_job/imaging_element.py index e405952b..28599162 100644 --- a/u19_pipeline/automatic_job/imaging_element.py +++ b/u19_pipeline/automatic_job/imaging_element.py @@ -24,7 +24,7 @@ acq_software = "ScanImage" # recording_id = os.environ['recording_id'] -recording_process_id = os.environ["recording_process_id"] +recording_process_id = os.environ["RECORDING_PROCESS_ID"] # process_method = os.environ['process_method'] # paramset_idx = os.environ['paramset_idx'] @@ -120,7 +120,7 @@ # scan_key = (imaging_rec.Scan & dict(recording_id=recording_id)).fetch1('KEY') # Recording process key -rec_process_key = dict(recording_process_id=recording_process_id) +rec_process_key = {"recording_process_id": recording_process_id} rec_process_str_key = "recording_process_id_" + str(recording_process_id) # Get fov key @@ -136,7 +136,7 @@ # Paramset idx and key paramset_idx = recording_process_info[0]["process_paramset_idx"] -paramset_idx_key = dict() +paramset_idx_key = {} paramset_idx_key["paramset_idx"] = paramset_idx scanner = recording_process_info[0]["acquisition_type"] @@ -145,7 +145,7 @@ scan_id = 0 # Get preprocess params -preprocess_params_key = dict() +preprocess_params_key = {} preprocess_params_key["preprocess_paramset_idx"] = recording_process_info[0]["preprocess_paramset_idx"] preprocess_params = recording.PreprocessParamSet().get_preprocess_params(preprocess_params_key) processing_method = preprocess_params["processing_method"] @@ -174,7 +174,7 @@ scan_filepaths = scan_filepaths # TODO load all TIFF files from session possibly using TIFFSequence loaded_scan = tifffile.imread(scan_filepaths) # scanner = 'mesoscope' - except: # TODO: Use except instead of else) + except BaseException: # TODO: Use except instead of else) print("ScanImage loading error") # TODO: Modify the error message # Equipment.insert1({'scanner': scanner}, skip_duplicates=True) @@ -271,7 +271,7 @@ if paramset_idx_key not in imaging_element.ProcessingParamSet(): # Get all information from process params from recording.ProcessParamSet schema - process_params_key = dict() + process_params_key = {} process_params_key["process_paramset_idx"] = paramset_idx process_params_info = (recording.ProcessParamSet() & process_params_key).fetch(as_dict=True) process_params = recording.ProcessParamSet().get_process_params(process_params_key) diff --git a/u19_pipeline/automatic_job/imaging_element_populate.py b/u19_pipeline/automatic_job/imaging_element_populate.py index 675d6384..989a6602 100644 --- a/u19_pipeline/automatic_job/imaging_element_populate.py +++ b/u19_pipeline/automatic_job/imaging_element_populate.py @@ -14,10 +14,10 @@ def populate_element_data(job_id, display_progress=True, reserve_jobs=False, sup "suppress_errors": suppress_errors, } - process_key = (recording_process.Processing * recording.Recording & dict(job_id=job_id)).fetch1("KEY") + process_key = (recording_process.Processing * recording.Recording & {"job_id": job_id}).fetch1("KEY") if (recording.Recording & process_key).fetch1("recording_modality") != "imaging": - warnings.warn(f"Recording modality is not `imaging` for job_id: {job_id}") + warnings.warn(f"Recording modality is not `imaging` for job_id: {job_id}", stacklevel=2) return fragment_number, recording_process_pre_path, recording_process_post_path = ( @@ -29,24 +29,21 @@ def populate_element_data(job_id, display_progress=True, reserve_jobs=False, sup ) preprocess_paramsets = ( - imaging_element.PreprocessParamSteps.Step() & dict(preprocess_param_steps_id=preprocess_param_steps_id) + imaging_element.PreprocessParamSteps.Step() & {"preprocess_param_steps_id": preprocess_param_steps_id} ).fetch("paramset_idx") - processing_method = (imaging_element.ProcessingParamSet & dict(paramset_idx=paramset_idx)).fetch1( + processing_method = (imaging_element.ProcessingParamSet & {"paramset_idx": paramset_idx}).fetch1( "processing_method" ) - if len(preprocess_paramsets) == 0: - preprocess_task_mode = "none" - else: - preprocess_task_mode = "load" + preprocess_task_mode = "none" if len(preprocess_paramsets) == 0 else "load" - preprocess_key = dict( - recording_id=process_key["recording_id"], - tiff_split=fragment_number, - scan_id=0, - preprocess_param_steps_id=preprocess_param_steps_id, - ) + preprocess_key = { + "recording_id": process_key["recording_id"], + "tiff_split": fragment_number, + "scan_id": 0, + "preprocess_param_steps_id": preprocess_param_steps_id, + } imaging_element.PreprocessTask.insert1( dict( diff --git a/u19_pipeline/automatic_job/params_config.py b/u19_pipeline/automatic_job/params_config.py index 30a55f6d..0a5260cd 100644 --- a/u19_pipeline/automatic_job/params_config.py +++ b/u19_pipeline/automatic_job/params_config.py @@ -265,6 +265,6 @@ # Slack notification channels slack_webhooks = lab.SlackWebhooks.fetch() -slack_webhooks_dict = dict() +slack_webhooks_dict = {} for i in range(slack_webhooks.shape[0]): slack_webhooks_dict[slack_webhooks[i][0]] = slack_webhooks[i][1] diff --git a/u19_pipeline/automatic_job/pupillometry_handler.py b/u19_pipeline/automatic_job/pupillometry_handler.py index f51dbff1..63cd2395 100644 --- a/u19_pipeline/automatic_job/pupillometry_handler.py +++ b/u19_pipeline/automatic_job/pupillometry_handler.py @@ -94,7 +94,7 @@ def generate_slurm_pupillometry(slurm_dict): def create_pupillometry_slurm_params_file(slurm_dict): text_dict = "" - for slurm_param in slurm_dict.keys(): + for slurm_param in slurm_dict: if isinstance(slurm_dict[slurm_param], list): for list_param in slurm_dict[slurm_param]: text_dict += "#SBATCH --" + str(slurm_param) + "=" + str(list_param) + "\n" @@ -258,7 +258,7 @@ def getPupilDiameter(analyzedVideoDataPath): outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: "OutlierFlag"}) # Concatenate outlier flags array to remove outliers from pupil diameter array temp = pd.concat([df, outlierFlags], axis=1) - temp.loc[temp["OutlierFlag"] == True, "PupilDiameter"] = None + temp.loc[temp["OutlierFlag"], "PupilDiameter"] = None pupilDiameter = temp["PupilDiameter"] return pupilDiameter.to_numpy() @@ -275,7 +275,7 @@ def analyze_videos_pupillometry(configPath, videoPath, output_dir): @pupillometry_exception_handler def check_pupillometry_sessions_queue(): - status_update = config.status_update_idx["NO_CHANGE"] + config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) sessions_missing_process = ( @@ -286,15 +286,15 @@ def check_pupillometry_sessions_queue(): for pupillometry_2_process in sessions_missing_process: # If error, job id = -1 - key_insert = dict( - (k, pupillometry_2_process[k]) + key_insert = { + k: pupillometry_2_process[k] for k in ( "subject_fullname", "session_date", "session_number", "model_id", ) - ) + } key_insert["pupillometry_job_id"] = -1 # Get model location @@ -333,7 +333,7 @@ def check_pupillometry_sessions_queue(): # Error handling (generating slurm file) if status != config.system_process["SUCCESS"]: - status_update = config.status_update_idx["ERROR_STATUS"] + config.status_update_idx["ERROR_STATUS"] update_value_dict["error_info"]["error_message"] = ( "Error while generating/transfering pupillometry slurm file" ) @@ -360,7 +360,7 @@ def check_pupillometry_sessions_queue(): # Error handling (queuing slurm file) if status != config.system_process["SUCCESS"]: - status_update = config.status_update_idx["ERROR_STATUS"] + config.status_update_idx["ERROR_STATUS"] update_value_dict["error_info"]["error_message"] = "Error to queue pupillometry slurm file" pupillometry.PupillometrySessionModelData.update1(key_insert) @@ -395,15 +395,15 @@ def check_processed_pupillometry_sessions(): ).fetch(as_dict=True) for session_check in sessions_to_check: - key_update = dict( - (k, session_check[k]) + key_update = { + k: session_check[k] for k in ( "subject_fullname", "session_date", "session_number", "model_id", ) - ) + } print("key_update1", key_update) status_update, message = slurmlib.check_slurm_job( diff --git a/u19_pipeline/automatic_job/recording_handler.py b/u19_pipeline/automatic_job/recording_handler.py index 881865cb..14e93dc3 100644 --- a/u19_pipeline/automatic_job/recording_handler.py +++ b/u19_pipeline/automatic_job/recording_handler.py @@ -290,7 +290,7 @@ def update_recording_log(recording_id, current_status, next_status, error_info_d print("error_info_dict", error_info_dict) - key = dict() + key = {} key["recording_id"] = recording_id key["status_recording_id_old"] = current_status key["status_recording_id_new"] = next_status @@ -440,7 +440,6 @@ def imaging_preingestion(rec_series): # Ingest Scan for each fov from the TiffSplit process for this_fov in fovs_ingested: # Scan_id always zero because TIFF splitted (FOVs) already on imaging_pipeline schema - scan_id = 0 # Acquisition type will have Mesoscope or 2Photon scanner = rec_series["acquisition_type"] # Hardcoded acquisition software diff --git a/u19_pipeline/automatic_job/recording_process_handler.py b/u19_pipeline/automatic_job/recording_process_handler.py index 1d63206c..0f8b9270 100644 --- a/u19_pipeline/automatic_job/recording_process_handler.py +++ b/u19_pipeline/automatic_job/recording_process_handler.py @@ -44,7 +44,7 @@ def pipeline_handler_main(): # Filter current status info current_status = rec_process_series["status_processing_id"] - current_status_series = config.recording_process_status_df.loc[ + config.recording_process_status_df.loc[ config.recording_process_status_df["Value"] == current_status, : ].squeeze() next_status_series = config.recording_process_status_df.loc[ @@ -135,7 +135,7 @@ def pipeline_handler_main(): update_dict["error_info"], rec_process_series, ) - except: + except Exception: update_dict["error_info"]["error_exception"] = ( "Error exception not accepted in slack, check log file for error" ) @@ -293,7 +293,7 @@ def slurm_job_queue(rec_series, status_series): chanmap_filename = config.default_chanmap_filename % (rec_series["job_id"]) chanmap_file_local_path = pathlib.Path(config.chanmap_files_filepath, chanmap_filename).as_posix() - raw_directory_for_chanmap = rec_series["recording_process_pre_path"] + rec_series["recording_process_pre_path"] spikeglx_meta_filepath = ephys_pipeline.get_spikeglx_meta_filepath(recording_key) # Chanmap mat file generation @@ -459,7 +459,7 @@ def get_program_selection_params(modality): # Pack all features in a dictionary this_modality_program_selection_params_dict = this_modality_program_selection_params.to_dict("records") - this_modality_program_selection_params_dict + print(this_modality_program_selection_params_dict) # Get two columns, (recording_modality & "packed" program_selection_params) this_modality_program_selection_params = ( @@ -467,7 +467,7 @@ def get_program_selection_params(modality): ) this_modality_program_selection_params["program_selection_params"] = this_modality_program_selection_params_dict - this_modality_program_selection_params + print(this_modality_program_selection_params) return this_modality_program_selection_params @@ -500,7 +500,7 @@ def get_active_process_jobs(active=True): # Get parameters for all modalities all_modalities = df_process_jobs["recording_modality"].unique() - all_mods_df = list() + all_mods_df = [] # Get process df for each modality for this_modality in all_modalities: this_mod_df = df_process_jobs.loc[df_process_jobs["recording_modality"] == this_modality, :].copy() @@ -680,7 +680,7 @@ def update_job_id_log(job_id, current_status, next_status, error_info_dict): print("error_info_dict", error_info_dict) - key = dict() + key = {} key["job_id"] = job_id key["status_processing_id_old"] = current_status key["status_processing_id_new"] = next_status diff --git a/u19_pipeline/automatic_job/slurm_creator.py b/u19_pipeline/automatic_job/slurm_creator.py index 9fd8c6b5..fb303e82 100644 --- a/u19_pipeline/automatic_job/slurm_creator.py +++ b/u19_pipeline/automatic_job/slurm_creator.py @@ -180,7 +180,7 @@ def transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars): def create_slurm_params_file(slurm_dict): text_dict = "" - for slurm_param in slurm_dict.keys(): + for slurm_param in slurm_dict: if isinstance(slurm_dict[slurm_param], list): for list_param in slurm_dict[slurm_param]: text_dict += "#SBATCH --" + str(slurm_param) + "=" + str(list_param) + "\n" diff --git a/u19_pipeline/automatic_job/tigress2pni.py b/u19_pipeline/automatic_job/tigress2pni.py index e8f1722e..1db6316b 100755 --- a/u19_pipeline/automatic_job/tigress2pni.py +++ b/u19_pipeline/automatic_job/tigress2pni.py @@ -13,6 +13,7 @@ previous transfers, so if this script is run twice in quick succession, the second run won't queue a duplicate transfer.""" +import contextlib import json import os import sys @@ -86,7 +87,7 @@ def save_data_to_file(filepath, key, data): """Save data to a file""" try: store = load_data_from_file(filepath) - except: + except Exception: store = {} if len(store) > 0: store[key] = data @@ -107,7 +108,7 @@ def setup_transfer_client(transfer_tokens): try: transfer_client.endpoint_autoactivate(SOURCE_ENDPOINT) - r = transfer_client.endpoint_autoactivate(DESTINATION_ENDPOINT) + transfer_client.endpoint_autoactivate(DESTINATION_ENDPOINT) except GlobusAPIError as ex: if ex.http_status == 401: sys.exit(f"Refresh token has expired. Please delete the `tokens` object from {DATA_FILE} and try again.") @@ -145,7 +146,7 @@ def main(): # if we already have tokens, load and use the tokens = client.load_tokens(requested_scopes=SCOPES) print(tokens) - except: + except Exception: pass if not tokens: @@ -153,10 +154,8 @@ def main(): # need to specify that we want refresh tokens print("login in") tokens = client.login(requested_scopes=SCOPES, refresh_tokens=True) - try: + with contextlib.suppress(Exception): client.save_tokens(tokens) - except: - pass transfer = setup_transfer_client(tokens["transfer.api.globus.org"]) diff --git a/u19_pipeline/copy_table.py b/u19_pipeline/copy_table.py index 3b9b8d41..9d244316 100644 --- a/u19_pipeline/copy_table.py +++ b/u19_pipeline/copy_table.py @@ -127,7 +127,7 @@ def copy_imaging_tables(): else: temp_table = getattr(imaging, table) - if isinstance(temp_table, dj.Lookup) or isinstance(temp_table, dj.Manual): + if isinstance(temp_table, (dj.Lookup, dj.Manual)): copy_table(imaging, imaging_original, table, skip_duplicates=True) else: copy_table( @@ -174,7 +174,7 @@ def copy_meso_tables(): else: temp_table = getattr(meso, table) - if isinstance(temp_table, dj.Lookup) or isinstance(temp_table, dj.Manual): + if isinstance(temp_table, (dj.Lookup, dj.Manual)): copy_table(meso, meso_original, table) else: copy_table(meso, meso_original, table, allow_direct_insert=True) @@ -200,7 +200,7 @@ def copy_meso_analysis_tables(): else: temp_table = getattr(meso_analysis, table) - if isinstance(temp_table, dj.Lookup) or isinstance(temp_table, dj.Manual): + if isinstance(temp_table, (dj.Lookup, dj.Manual)): copy_table(meso_analysis, meso_analysis_original, table) else: copy_table( diff --git a/u19_pipeline/ephys_pipeline.py b/u19_pipeline/ephys_pipeline.py index ecb90659..aa3d6075 100644 --- a/u19_pipeline/ephys_pipeline.py +++ b/u19_pipeline/ephys_pipeline.py @@ -109,7 +109,7 @@ def get_full_session_directory(recording_key): def append_cat_gt_params_from_probedir(probe_dirname): - extra_cat_gt_params = dict() + extra_cat_gt_params = {} probe_match = re.search("_imec[0-9]$", probe_dirname) if probe_match: @@ -207,7 +207,7 @@ def get_spikeglx_meta_filepath(ephys_recording_key): ephys_element.ProbeInsertion * ephys_element.probe.Probe & ephys_recording_key ).fetch1("probe") - spikeglx_meta_filepaths = [fp for fp in session_dir.rglob("*.ap.meta")] + spikeglx_meta_filepaths = list(session_dir.rglob("*.ap.meta")) for meta_filepath in spikeglx_meta_filepaths: spikeglx_meta = spikeglx.SpikeGLXMeta(meta_filepath) if str(spikeglx_meta.probe_SN) == inserted_probe_serial_number: @@ -258,7 +258,7 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): # Read behavior sync record try: sync_data = (BehaviorSync & rec_key).fetch1("sync_data") - except: + except Exception: print("No sync data was sound for this session") return @@ -284,7 +284,7 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): ) # Store data - all_vectors = dict() + all_vectors = {} all_vectors["time_as_behavior_trial_ind"] = trial_times_ind all_vectors["time_as_behavior_fullsession"] = trial_times_full @@ -296,7 +296,7 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): for this_key in all_vectors: print("this_key", this_key) print(all_vectors[this_key].shape) - all_vectors[this_key] = np.concatenate(([x for x in all_vectors[this_key]]), axis=0) + all_vectors[this_key] = np.concatenate((list(all_vectors[this_key])), axis=0) print(all_vectors[this_key].shape) all_vectors["trial_index_nidq_virmen"] = trial_index_nidq_virmen @@ -440,7 +440,7 @@ def make(self, key, **kwargs): nidq_sampling_rate, ) - dictionary_sync_data = dict() + dictionary_sync_data = {} print("after all main ehpys fix sync code", status_regular, status_fix) @@ -480,7 +480,7 @@ def make(self, key, **kwargs): print(kwargs) - if "populate" not in kwargs or kwargs["populate"] == True: + if "populate" not in kwargs or kwargs["populate"]: BehaviorSync.insert1(final_key, allow_direct_insert=True) self.insert_imec_sampling_rate(key, ephys_session_fullpath.parent) else: diff --git a/u19_pipeline/ephys_sync.py b/u19_pipeline/ephys_sync.py index 327e46b3..a022d0c4 100644 --- a/u19_pipeline/ephys_sync.py +++ b/u19_pipeline/ephys_sync.py @@ -43,9 +43,9 @@ def make(self, key): nidq_sampling_rate = readSGLX.SampRate(nidq_meta) if nidq_meta["typeThis"] == "nidq": - digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta) + digital_array = ephys_utils.SpiceGlxUtility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta) else: - digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file( + digital_array = ephys_utils.SpiceGlxUtility.load_spice_glx_digital_file( nidq_bin_full_path, nidq_meta, d_line_list=[1, 2] ) @@ -137,7 +137,7 @@ def make(self, key): thissession = behavior.TowersBlock().Trial() & key iterstart = thissession.fetch("vi_start") - first_vr_iteration = iterstart[0] + iterstart[0] # Obtain the precise times when the frames transition. # This is obtained from iteration_index_nidq @@ -146,7 +146,7 @@ def make(self, key): 1 # These are the trial transitions (see definition above). To get total number of frames, we define this as a transition like all others. ) ls[np.isnan(ls)] = 0 - iteration_transition_indexes = np.where(ls)[0] + np.where(ls)[0] # First iterations captured not in virmen because vr was not started yet # for i in range(first_vr_iteration): diff --git a/u19_pipeline/imaging_pipeline.py b/u19_pipeline/imaging_pipeline.py index 4f6ad245..c45546b7 100644 --- a/u19_pipeline/imaging_pipeline.py +++ b/u19_pipeline/imaging_pipeline.py @@ -296,39 +296,22 @@ def get_calcium_imaging_files(scan_key, acq_software): return tiff_filepaths -def get_calcium_imaging_files(scan_key, acq_software): - - filepaths = (TiffSplit.File * TiffSplit & scan_key).fetch( - "tiff_split_directory", "tiff_split_filename", as_dict=True - ) - - tiff_filepaths = [ - find_full_path( - get_imaging_root_data_dir(), - pathlib.Path(file["tiff_split_directory"]) / file["tiff_split_filename"], - ).as_posix() - for file in filepaths - ] - - return tiff_filepaths - - def get_processed_dir(processing_task_key, process_method): sess_key = (ImagingPipelineSession & processing_task_key).fetch1("KEY") bucket_scan_dir = (TiffSplit & sess_key & {"tiff_split": processing_task_key["scan_id"]}).fetch1( "tiff_split_directory" ) - user_id = (subject.Subject & processing_task_key).fetch1("user_id") + (subject.Subject & processing_task_key).fetch1("user_id") sess_dir = find_full_path(get_imaging_root_data_dir(), bucket_scan_dir) - relative_suite2p_dir = (pathlib.Path(bucket_scan_dir) / process_method).as_posix() + (pathlib.Path(bucket_scan_dir) / process_method).as_posix() if not sess_dir.exists(): raise FileNotFoundError(f"Session directory not found ({sess_dir})") if process_method == "suite2p": # Check if ops.npy is inside suite2p_dir - suite2p_dirs = set([fp.parent.parent for fp in sess_dir.rglob("*ops.npy")]) + suite2p_dirs = {fp.parent.parent for fp in sess_dir.rglob("*ops.npy")} if len(suite2p_dirs) != 1: raise FileNotFoundError( f"Error searching for Suite2p output directory in {bucket_scan_dir} - Found {suite2p_dirs}" diff --git a/u19_pipeline/lab.py b/u19_pipeline/lab.py index 3cfd2f02..71ef7a86 100644 --- a/u19_pipeline/lab.py +++ b/u19_pipeline/lab.py @@ -306,11 +306,7 @@ def get_local_path(self, path, local_os=None): path = os.path.join(mapping[i, local], path[n + 1 :]) break - if os.path.sep == "\\" and local_os.lower() != "glo": - path = path.replace("/", "\\") - - else: - path = path.replace("\\", "/") + path = path.replace("/", "\\") if os.path.sep == "\\" and local_os.lower() != "glo" else path.replace("\\", "/") return path diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 57a7aa1d..b23a2776 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -99,9 +99,9 @@ class DefaultParams(dj.Manual): def get_default_params_rec_process(recording_processes, default_params_record_df): "Get associated params from DefaultParams record and recording processes (jobs) of recording" - params_rec_process = list() + params_rec_process = [] for i in recording_processes: - this_params_rec_process = dict() + this_params_rec_process = {} this_params_rec_process["job_id"] = i["job_id"] this_fragment = i["fragment_number"] diff --git a/u19_pipeline/recording_process.py b/u19_pipeline/recording_process.py index 864ceb07..b30275e7 100644 --- a/u19_pipeline/recording_process.py +++ b/u19_pipeline/recording_process.py @@ -73,9 +73,9 @@ def insert_recording_process(self, fragment_keys, fragment_fieldname): # Get directory fieldname for specific modality (probe_directory, fov_directory, etc.) # Append data for the unit to insert - this_recprocess_key = list() + this_recprocess_key = [] for idx, fragment_key in enumerate(fragment_keys): - this_recprocess_key.append(dict()) + this_recprocess_key.append({}) this_recprocess_key[idx]["recording_id"] = fragment_key["recording_id"] this_recprocess_key[idx]["fragment_number"] = fragment_key[fragment_fieldname] this_recprocess_key[idx]["status_processing_id"] = 0 @@ -91,7 +91,7 @@ def set_recording_process_post_path(self, recprocess_keys): """ for rec_process in recprocess_keys: - this_key_dict = dict() + this_key_dict = {} this_key_dict["job_id"] = rec_process["job_id"] this_key_dict["recording_process_post_path"] = pathlib.Path( rec_process["recording_process_pre_path"], diff --git a/u19_pipeline/utility.py b/u19_pipeline/utility.py index 14391c03..bf8f80c3 100644 --- a/u19_pipeline/utility.py +++ b/u19_pipeline/utility.py @@ -57,7 +57,7 @@ def get_network_path(path_name): network_path: (str): String with network path as mounted by the corresponding os """ - key = dict() + key = {} # Check if path name to search starts with needed / at start if path_name[0] != "/": key["global_path"] = "/" + path_name @@ -93,15 +93,15 @@ def smart_dj_join(t1, t2): fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient="index") # Get only secondary fields and check matches - fields_t1_list = set(fields_t1.loc[fields_t1["in_key"] == False].index.to_list()) - fields_t2_list = set(fields_t2.loc[fields_t2["in_key"] == False].index.to_list()) + fields_t1_list = set(fields_t1.loc[not fields_t1["in_key"]].index.to_list()) + fields_t2_list = set(fields_t2.loc[not fields_t2["in_key"]].index.to_list()) intersected_fields = fields_t2_list.intersection(fields_t1_list) # If there are: if len(intersected_fields) > 0: # Create a dictionary to rename matching ones suffix = t2.table_name - new_name_attr_dict = dict() + new_name_attr_dict = {} for i in intersected_fields: new_name_attr_dict[suffix + "_" + i] = i @@ -117,11 +117,11 @@ def smart_dj_join(t1, t2): return t -def psychometrics_function(x, O, A, lambd, x0): +def psychometrics_function(x, offset, A, lambd, x0): """ Standard sigmoid function """ - return O + A / (1 + np.exp(-(x - x0) / lambd)) + return offset + A / (1 + np.exp(-(x - x0) / lambd)) def psychFit(deltaBins, numR, numL, choices): @@ -209,7 +209,7 @@ def psychFit(deltaBins, numR, numL, choices): errorY = errorY.flatten() # Fill dictionary of results - fit_results = dict() + fit_results = {} fit_results["delta_bins"] = deltaBins[~idx_zero] fit_results["delta_data"] = trialDelta[~idx_zero] fit_results["pright_data"] = 100 * phat[~idx_zero] @@ -279,7 +279,7 @@ def get_cols_rows_plot(num_plots, fig_size): def create_str_from_dict(key_dict): slurm_file_name = "" - for i in key_dict.keys(): + for i in key_dict: slurm_file_name += str(i) + "_" + str(key_dict[i]) return slurm_file_name diff --git a/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py b/u19_pipeline/utils/DemoReadSGLXData/read_sglx.py similarity index 95% rename from u19_pipeline/utils/DemoReadSGLXData/readSGLX.py rename to u19_pipeline/utils/DemoReadSGLXData/read_sglx.py index c05754ce..e1eae308 100644 --- a/u19_pipeline/utils/DemoReadSGLXData/readSGLX.py +++ b/u19_pipeline/utils/DemoReadSGLXData/read_sglx.py @@ -41,10 +41,7 @@ def readMeta(binFullPath): # convert the list entries into key value pairs for m in mdatList: csList = m.split(sep="=") - if csList[0][0] == "~": - currKey = csList[0][1 : len(csList[0])] - else: - currKey = csList[0] + currKey = csList[0][1:len(csList[0])] if csList[0][0] == "~" else csList[0] metaDict.update({currKey: csList[1]}) else: print("no meta file") @@ -77,10 +74,7 @@ def SampRate(meta): # def Int2Volts(meta): if meta["typeThis"] == "imec": - if "imMaxInt" in meta: - maxInt = int(meta["imMaxInt"]) - else: - maxInt = 512 + maxInt = int(meta["imMaxInt"]) if "imMaxInt" in meta else 512 fI2V = float(meta["imAiRangeMax"]) / maxInt elif meta["typeThis"] == "nidq": maxInt = int(meta["niMaxInt"]) @@ -183,10 +177,7 @@ def ChanGainsIM(meta): APgain = np.zeros(int(acqCountList[0])) # default type = float64 LFgain = np.zeros(int(acqCountList[1])) # empty array for 2.0 - if "imDatPrb_type" in meta: - probeType = int(meta["imDatPrb_type"]) - else: - probeType = 0 + probeType = int(meta["imDatPrb_type"]) if "imDatPrb_type" in meta else 0 if sum(np.isin(np1_imro, probeType)): # imro + probe allows setting gain independently for each channel @@ -222,10 +213,7 @@ def ChanGainsIM(meta): APgain = APgain + 1 fI2V = Int2Volts(meta) APChan0_to_uV = 1e6 * fI2V / APgain[0] - if LFgain.size > 0: - LFChan0_to_uV = 1e6 * fI2V / LFgain[0] - else: - LFChan0_to_uV = 0 + LFChan0_to_uV = 1000000.0 * fI2V / LFgain[0] if LFgain.size > 0 else 0 return (APgain, LFgain, APChan0_to_uV, LFChan0_to_uV) @@ -318,7 +306,7 @@ def GainCorrectIM(dataArray, chanList, meta): def makeMemMapRaw(binFullPath, meta): nChan = int(meta["nSavedChans"]) nFileSamp = int(int(meta["fileSizeBytes"]) / (2 * nChan)) - print("nChan: %d, nFileSamp: %d" % (nChan, nFileSamp)) + print(f"nChan: {nChan:d}, nFileSamp: {nFileSamp:d}") rawData = np.memmap( binFullPath, dtype="int16", @@ -351,7 +339,7 @@ def ExtractDigital(rawData, firstSamp, lastSamp, dwReq, dLineList, meta): elif meta["typeThis"] == "nidq": MN, MA, XA, DW = ChannelCountsNI(meta) if dwReq > DW - 1: - print("Maximum digital word in file = %d" % (DW - 1)) + print(f"Maximum digital word in file = {DW - 1:d}") digArray = np.zeros((0), "uint8") return digArray else: @@ -359,7 +347,7 @@ def ExtractDigital(rawData, firstSamp, lastSamp, dwReq, dLineList, meta): elif meta["typeThis"] == "obx": XA, DW, SY = ChannelCountsOBX(meta) if dwReq > DW - 1: - print("Maximum digital word in file = %d" % (DW - 1)) + print(f"Maximum digital word in file = {DW - 1:d}") digArray = np.zeros((0), "uint8") return digArray else: diff --git a/u19_pipeline/utils/dj_shortcuts.py b/u19_pipeline/utils/dj_shortcuts.py index 62c184f4..9d1b9764 100644 --- a/u19_pipeline/utils/dj_shortcuts.py +++ b/u19_pipeline/utils/dj_shortcuts.py @@ -11,7 +11,7 @@ def get_primary_key_fields(t): """ fields_t = pd.DataFrame.from_dict(t.heading.attributes, orient="index") - primary_field_list = fields_t.loc[fields_t["in_key"] == True].index.to_list() + primary_field_list = fields_t.loc[fields_t["in_key"]].index.to_list() return primary_field_list @@ -28,15 +28,15 @@ def smart_dj_join(t1, t2): fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient="index") # Get only secondary fields and check matches - fields_t1_list = set(fields_t1.loc[fields_t1["in_key"] == False].index.to_list()) - fields_t2_list = set(fields_t2.loc[fields_t2["in_key"] == False].index.to_list()) + fields_t1_list = set(fields_t1.loc[not fields_t1["in_key"]].index.to_list()) + fields_t2_list = set(fields_t2.loc[not fields_t2["in_key"]].index.to_list()) intersected_fields = fields_t2_list.intersection(fields_t1_list) # If there are: if len(intersected_fields) > 0: # Create a dictionary to rename matching ones suffix = t2.table_name - new_name_attr_dict = dict() + new_name_attr_dict = {} for i in intersected_fields: new_name_attr_dict[suffix + "_" + i] = i diff --git a/u19_pipeline/utils/dlc_process.py b/u19_pipeline/utils/dlc_process.py index 7df9dc23..0ee6e706 100644 --- a/u19_pipeline/utils/dlc_process.py +++ b/u19_pipeline/utils/dlc_process.py @@ -78,14 +78,13 @@ def getPupilDiameter(destinationFolder=None): outlierFlags = outlierFlags.rename(columns={outlierFlags.columns[0]: "OutlierFlag"}) # Concatenate outlier flags array to remove outliers from pupil diameter array temp = pd.concat([df, outlierFlags], axis=1) - temp.loc[temp["OutlierFlag"] == True, "PupilDiameter"] = None + temp.loc[temp["OutlierFlag"], "PupilDiameter"] = None pupilDiameter = temp["PupilDiameter"].to_numpy() filename = pathlib.Path(destinationFolder, "pupil_diameter.pickle").as_posix() - file_to_store = open(filename, "wb") - pickle.dump(pupilDiameter, file_to_store) - file_to_store.close() + with open(filename, "wb") as file_to_store: + pickle.dump(pupilDiameter, file_to_store) if __name__ == "__main__": diff --git a/u19_pipeline/utils/ephys_fix_sync_code.py b/u19_pipeline/utils/ephys_fix_sync_code.py index 82c13970..2d763933 100644 --- a/u19_pipeline/utils/ephys_fix_sync_code.py +++ b/u19_pipeline/utils/ephys_fix_sync_code.py @@ -124,8 +124,8 @@ def fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, vec_shift, consecutive_zeros = np.flatnonzero(correlation_result == 4) index_shift = 0 - index_borrow_virmen = list() - for i in range(len(where_insert_iteration)): + index_borrow_virmen = [] + for _i in range(len(where_insert_iteration)): stable_parts = np.where(consecutive_zeros > where_insert_iteration[index_shift]) stable_parts = stable_parts[0] @@ -153,7 +153,7 @@ def fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, vec_shift, # for i in range(where_insert_iteration.shape[0]): index_diff_e = -1 - borrowed_indexes = list() + borrowed_indexes = [] for i in range(len(index_borrow_virmen)): if index_diff_e > index_borrow_virmen[i][0] + mid_point: continue @@ -307,9 +307,9 @@ def sync_evaluation_process2(synced_time_vector, behavior_time_vector): def main_ephys_fix_sync_code(iter_start_idx, iter_times_idx, behavior_time, nidq_sampling_rate): - iteration_dict = dict() - iteration_dict["iter_start_idx"] = list() - iteration_dict["iter_times_idx"] = list() + iteration_dict = {} + iteration_dict["iter_start_idx"] = [] + iteration_dict["iter_times_idx"] = [] for i in range(len(iter_start_idx)): # print('fixing trial ',i) diff --git a/u19_pipeline/utils/ephys_utils.py b/u19_pipeline/utils/ephys_utils.py index 79567024..4a07ca70 100644 --- a/u19_pipeline/utils/ephys_utils.py +++ b/u19_pipeline/utils/ephys_utils.py @@ -11,7 +11,7 @@ import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX -class spice_glx_utility: +class SpiceGlxUtility: @staticmethod def load_spice_glx_digital_file(file_path, nidq_meta, d_line_list=None): # Read NIDAQ digital file. @@ -68,10 +68,10 @@ def load_trial_iteration_signals(ephys_session_fullpath, nidq_meta): new_iteration_channel = 2 # If PXIe card (nidq) card use for recording deduce digital channels if nidq_meta["typeThis"] == "nidq": - digital_array = spice_glx_utility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta) + digital_array = SpiceGlxUtility.load_spice_glx_digital_file(ephys_session_fullpath, nidq_meta) # If onebox card (obx) card use for recording digital channels are 0-2 else: - digital_array = spice_glx_utility.load_spice_glx_digital_file( + digital_array = SpiceGlxUtility.load_spice_glx_digital_file( ephys_session_fullpath, nidq_meta, d_line_list=[0, 1] ) # If no sync pulse found trial and iteration signals are 0 & 1 respectively @@ -180,10 +180,7 @@ def get_trial_signal_mode(iteration_pulse_signal_trial, behavior_time_vector_tri # If iterations in trial are less than the ones in behavior, the mode was the counterbit iter_samples = np.where(np.diff(iteration_pulse_signal_trial) == 1) - if iter_samples[0].shape[0] < (behavior_time_vector_trial.shape[0] * 3 / 4): - mode = "counter_bit0" - else: - mode = "pulse_signal" + mode = "counter_bit0" if iter_samples[0].shape[0] < behavior_time_vector_trial.shape[0] * 3 / 4 else "pulse_signal" print("mode deduction: ", mode) @@ -200,7 +197,7 @@ def get_iteration_sample_vector_from_digital_lines_pulses( ) -> dict: # Output as a dictionary - iteration_vector_output = dict() + iteration_vector_output = {} # Vectors that will contain trial # and iter # for each sample on file # iteration_vector_output['framenumber_vector_samples'] = np.zeros(trial_pulse_signal.shape[0])*np.nan @@ -352,7 +349,7 @@ def get_iteration_sample_vector_from_digital_lines_word(digital_array, time, ite framenumber_in_trial[g + 1] = framenumber_in_trial[g] # This point we have framenumber_in_trial and trialnumber. Now just some refactoring to fit into the usual data structure - iteration_vector_output = dict() + iteration_vector_output = {} iteration_vector_output["trialnumber_vector_samples"] = trialnumber iteration_vector_output["framenumber_vector_samples"] = framenumber_in_trial @@ -371,29 +368,27 @@ def assert_iteration_samples_count(iteration_sample_idx_output, behavior_time_ve # Count trial count differences trial_count_diff = np.abs(iteration_sample_idx_output.shape[0] - (behavior_time_vector.shape[0])) - count = 0 - trials_diff_iteration_small = list() - trials_diff_iteration_big = list() - for idx_trial, iter_trials in enumerate(iteration_sample_idx_output): - if iter_trials.shape[0] != behavior_time_vector[idx_trial].shape[0]: + trials_diff_iteration_small = [] + trials_diff_iteration_big = [] + for count, iter_trials in enumerate(iteration_sample_idx_output): + if iter_trials.shape[0] != behavior_time_vector[count].shape[0]: print( "trial#", count, "iterPulses:", iter_trials.shape[0], "IterBeh:", - behavior_time_vector[idx_trial].shape[0], + behavior_time_vector[count].shape[0], "Difference (", - behavior_time_vector[idx_trial].shape[0] - iter_trials.shape[0], + behavior_time_vector[count].shape[0] - iter_trials.shape[0], ")", ) - count += 1 # For each trial iteration # should be equal to the behavioral file iterations - if iter_trials.shape[0] != behavior_time_vector[idx_trial].shape[0]: - if np.abs(iter_trials.shape[0] - behavior_time_vector[idx_trial].shape[0]) < 6: - trials_diff_iteration_small.append(idx_trial) + if iter_trials.shape[0] != behavior_time_vector[count].shape[0]: + if np.abs(iter_trials.shape[0] - behavior_time_vector[count].shape[0]) < 6: + trials_diff_iteration_small.append(count) else: - trials_diff_iteration_big.append(idx_trial) + trials_diff_iteration_big.append(count) return trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small @@ -645,7 +640,7 @@ def behavior_sync_frame_counter_method( assert ( (nidaqtime - matlabtime) / matlabtime ) < 0.1 # # Make sure the nidaq-trial-duration and dj records are consistent; 10% arbitrarily chosen - nidaq_duration = iterations_test + skipped_frames + iterations_test + skipped_frames # dj_duration = iterstart[-1] + len(behavior_time_vector[-1]) # assert np.abs(nidaq_duration - dj_duration) < 3 # at most two frames off - sometimes this happens at the beginning/end of the recording @@ -842,7 +837,7 @@ def get_index_trial_vector_from_iteration(iteration_start_idx): return trial_start_idx -class xyz_pick_file_creator: +class XyzPickFileCreator: """ Class that handles probe coordinates locations given initial isertion coordinates & shank coordinates """ @@ -868,7 +863,7 @@ def main_xyz_pick_file_function(recording_id, fragment_number, chanmap_file, pro pathlib.Path.mkdir(ibl_output_dir) # Get recording id - probe_location = xyz_pick_file_creator.get_probe_insertion_coordinates(recording_id, fragment_number) + probe_location = XyzPickFileCreator.get_probe_insertion_coordinates(recording_id, fragment_number) print(probe_location) # Load channelmap and check how many probes there are @@ -876,10 +871,10 @@ def main_xyz_pick_file_function(recording_id, fragment_number, chanmap_file, pro max_shank = int(np.max(chanmap["kcoords"])) # Calculate probe coordinates and store files - all_shanks = list() + all_shanks = [] for i in range(max_shank): - probe_track = xyz_pick_file_creator.get_probetrack(chanmap, shank=i + 1, **probe_location) - xyz_pick_file_creator.save_xyz_pick_file(ibl_output_dir, probe_track, shank=i) + probe_track = XyzPickFileCreator.get_probetrack(chanmap, shank=i + 1, **probe_location) + XyzPickFileCreator.save_xyz_pick_file(ibl_output_dir, probe_track, shank=i) all_shanks.append(probe_track) return all_shanks @@ -933,7 +928,7 @@ def get_probe_insertion_coordinates(recording_id, probe_num): "Warning probe location was not found on DB for recording_id: " + str(recording_id) + " & probe# " - + str(probe_num) + + str(probe_num), stacklevel=2 ) probe_location = dict.fromkeys(coordinates_columns, 0) @@ -1018,7 +1013,7 @@ def save_xyz_pick_file(save_directory, probe_coord_data, shank=0): ] final_filename = pathlib.Path(save_directory, filenames[shank]).as_posix() - dict_coord = dict() + dict_coord = {} dict_coord["xyz_picks"] = probe_coord_data with open(final_filename, "w") as fp: diff --git a/u19_pipeline/utils/insert_miscelaneous_db.py b/u19_pipeline/utils/insert_miscelaneous_db.py index 351aa7d1..2f550cae 100644 --- a/u19_pipeline/utils/insert_miscelaneous_db.py +++ b/u19_pipeline/utils/insert_miscelaneous_db.py @@ -8,7 +8,7 @@ def add_researcher_user_table(user_id, full_name, email, phone): # Values sent by function caller - new_key = dict() + new_key = {} new_key["user_id"] = user_id new_key["user_nickname"] = user_id new_key["full_name"] = full_name diff --git a/u19_pipeline/utils/matlab_utils.py b/u19_pipeline/utils/matlab_utils.py index 39906cd2..97f97ee6 100644 --- a/u19_pipeline/utils/matlab_utils.py +++ b/u19_pipeline/utils/matlab_utils.py @@ -2,25 +2,17 @@ import os import sys -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from datetime import datetime from pathlib import Path from shutil import which import numpy as np import pandas as pd +from numpy import ndarray from scipy.io import loadmat, matlab -try: - from typing import ArrayLike -except ImportError: - from collections.abc import Sequence - from typing import Union - - from numpy import ndarray - - # adapted from numpy typing - ArrayLike = Union[bool, int, float, complex, list, ndarray, Sequence] +ArrayLike = bool | int | float | complex | list | ndarray | Sequence def check_module(nwbfile, name, description=None): @@ -50,7 +42,7 @@ def find_discontinuities(tt, factor=10000): if len(before_jumps): out = np.array([tt[0], tt[before_jumps[0]]]) - for i, j in zip(before_jumps, before_jumps[1:]): + for i, j in zip(before_jumps, before_jumps[1:], strict=False): out = np.vstack((out, [tt[i + 1], tt[j]])) out = np.vstack((out, [tt[before_jumps[-1] + 1], tt[-1]])) return out @@ -134,7 +126,7 @@ def flatten_nested_dict(nested_dict): if isinstance(v, dict): if v: flatten_sub_dict = flatten_nested_dict(v).items() - flatten_dict.update({k2: v2 for k2, v2 in flatten_sub_dict}) + flatten_dict.update(dict(flatten_sub_dict)) else: flatten_dict[k] = np.array([]) else: @@ -341,7 +333,7 @@ def convert_towers_block_2_df(current_block, num_block): valid_block = 0 # "Normal" blocks are stored as numpy arrays and its length is greater than 0 - if isinstance(current_block, np.ndarray) and current_block_trial.shape[0] > 0: + if isinstance(current_block, np.ndarray) and current_block.shape[0] > 0: current_block = current_block.tolist() valid_block = 1 # One trial blocks are stored as dictionaries @@ -372,10 +364,7 @@ def convert_behavior_file(mat_file): matin = convert_mat_file_to_dict(mat_file) converted_metadata = convert_function_handle_to_str(mat_file_path=mat_file) - if bool(converted_metadata): - metadata_read = True - else: - metadata_read = False + metadata_read = bool(bool(converted_metadata)) session_block_trial_df = pd.DataFrame() @@ -393,10 +382,7 @@ def convert_behavior_file(mat_file): # Convert all blocks of the session num_blocks_conv = 0 for i in range(length_blocks): - if dict_block: - block = matin["log"]["block"] - else: - block = matin["log"]["block"][i] + block = matin["log"]["block"] if dict_block else matin["log"]["block"][i] # Convert trial df and block df valid_block, block_trial_df = convert_towers_block_trial_2_df(block["trial"], i + 1) diff --git a/u19_pipeline/utils/path_utils.py b/u19_pipeline/utils/path_utils.py index 8fddb387..43f28fde 100644 --- a/u19_pipeline/utils/path_utils.py +++ b/u19_pipeline/utils/path_utils.py @@ -15,12 +15,11 @@ def check_file_pattern_dir(filepath, file_patterns): """ Check if directory (or its childs) contains some files with specific pattern names """ - dirs_with_session_files = [] child_dirs = [x[0] for x in os.walk(filepath)] patterns_found = 0 - for dir in child_dirs: + for directory in child_dirs: for pat in file_patterns: - found_file = glob.glob(dir + pat) + found_file = glob.glob(directory + pat) if len(found_file) > 0: patterns_found = 1 break @@ -50,10 +49,7 @@ def get_size_directory(path): command = ["du", path, "-s"] s = subprocess.run(command, capture_output=True) output = s.stdout.decode("UTF-8") - if len(output) != 0: - kbytes = int(output.split("\t")[0]) - else: - kbytes = -1 + kbytes = int(output.split("\t")[0]) if len(output) != 0 else -1 return kbytes @@ -69,7 +65,7 @@ def get_size_directory_time(path): list_return = [] for line in list_values: line_values = line.split("\t") - dict_line = dict() + dict_line = {} if len(line_values) == 3: dict_line["size"] = line_values[0] dict_line["date"] = line_values[1] diff --git a/u19_pipeline/utils/scp_transfers.py b/u19_pipeline/utils/scp_transfers.py index d50fb364..aee5ed4b 100644 --- a/u19_pipeline/utils/scp_transfers.py +++ b/u19_pipeline/utils/scp_transfers.py @@ -92,7 +92,7 @@ def transfer_scp(host=None, username=None, remote_path=None, local_path=None): print(remote_path) print(local_path) rc._get_ssh_key() - rc.scp + print(rc.scp) rc.download_folder(remote_path=remote_path, local_path=local_path) rc.disconnect() diff --git a/u19_pipeline/utils/slack_utils.py b/u19_pipeline/utils/slack_utils.py index a4462f57..e514654c 100644 --- a/u19_pipeline/utils/slack_utils.py +++ b/u19_pipeline/utils/slack_utils.py @@ -51,21 +51,21 @@ def send_slack_update_notification(webhook_url, base_message, session_info): now = datetime.now() datestr = now.strftime("%d-%b-%Y %H:%M:%S") - msep = dict() + msep = {} msep["type"] = "divider" # Title # - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ":white_check_mark: *Automation pipeline update* on " + datestr + "\n\n" m1["text"] = m1_1 # Info # - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" m2_1["text"] = ( "*" @@ -88,7 +88,7 @@ def send_slack_update_notification(webhook_url, base_message, session_info): ) m2["text"] = m2_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2] message["text"] = "Automation pipeline update recording:" + str(session_info["recording_id"]) @@ -104,21 +104,21 @@ def send_slack_error_notification(webhook_url, error_info, session_info): if "job_id" not in session_info: session_info["job_id"] = "Not a job" - msep = dict() + msep = {} msep["type"] = "divider" # Title # - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ":rotating_light: *Automation pipeline error* on " + datestr + "\n\n" m1["text"] = m1_1 # Info # - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" m2_1["text"] = ( "*Automation pipeline failed for:* \n" @@ -140,9 +140,9 @@ def send_slack_error_notification(webhook_url, error_info, session_info): m2["text"] = m2_1 # Error # - m3 = dict() + m3 = {} m3["type"] = "section" - m3_1 = dict() + m3_1 = {} m3_1["type"] = "mrkdwn" m3_1["text"] = ( "*Error info* \n" @@ -154,7 +154,7 @@ def send_slack_error_notification(webhook_url, error_info, session_info): ) m3["text"] = m3_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2, msep, m3] message["text"] = "Automation pipeline error in recording:" + str(session_info["recording_id"]) @@ -170,21 +170,21 @@ def send_slack_error_pupillometry_notification(webhook_url, error_info, session_ if "job_id" not in session_info: session_info["job_id"] = "Not a job" - msep = dict() + msep = {} msep["type"] = "divider" # Title # - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ":rotating_light: *Automation pipeline error* on " + datestr + "\n\n" m1["text"] = m1_1 # Info # - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" m2_1["text"] = ( "*Pupillometry pipeline failed for:* \n" @@ -203,9 +203,9 @@ def send_slack_error_pupillometry_notification(webhook_url, error_info, session_ m2["text"] = m2_1 # Error # - m3 = dict() + m3 = {} m3["type"] = "section" - m3_1 = dict() + m3_1 = {} m3_1["type"] = "mrkdwn" m3_1["text"] = ( "*Error info* \n" @@ -217,7 +217,7 @@ def send_slack_error_pupillometry_notification(webhook_url, error_info, session_ ) m3["text"] = m3_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2, msep, m3] message["text"] = "Pupillometry pipeline error in session" @@ -230,21 +230,21 @@ def send_slack_pupillometry_update_notification(webhook_url, base_message, sessi now = datetime.now() datestr = now.strftime("%d-%b-%Y %H:%M:%S") - msep = dict() + msep = {} msep["type"] = "divider" # Title # - m1 = dict() + m1 = {} m1["type"] = "section" - m1_1 = dict() + m1_1 = {} m1_1["type"] = "mrkdwn" m1_1["text"] = ":white_check_mark: *Pupillometry pipeline update* on " + datestr + "\n\n" m1["text"] = m1_1 # Info # - m2 = dict() + m2 = {} m2["type"] = "section" - m2_1 = dict() + m2_1 = {} m2_1["type"] = "mrkdwn" m2_1["text"] = ( "*" @@ -264,7 +264,7 @@ def send_slack_pupillometry_update_notification(webhook_url, base_message, sessi ) m2["text"] = m2_1 - message = dict() + message = {} message["blocks"] = [m1, msep, m2] message["text"] = "Pupillometry pipeline update" diff --git a/u19_pipeline/utils/tiff_matlab_imaging_utils.py b/u19_pipeline/utils/tiff_matlab_imaging_utils.py index c12577d7..669def71 100644 --- a/u19_pipeline/utils/tiff_matlab_imaging_utils.py +++ b/u19_pipeline/utils/tiff_matlab_imaging_utils.py @@ -1,4 +1,5 @@ import ast +import contextlib import re import time from pathlib import Path @@ -7,8 +8,8 @@ import tifffile as tiff from sklearn.linear_model import HuberRegressor -xySizeFactor = 1.05 # images are this much larger than nominal size -zFactor = 1.45 # actual displacement in z vs command +xy_size_factor = 1.05 # images are this much larger than nominal size +z_factor = 1.45 # actual displacement in z vs command def select_files_from_mean_f(scan_directory, f_decrease_threshold=15): @@ -102,10 +103,7 @@ def select_files_from_mean_f(scan_directory, f_decrease_threshold=15): valid_idx = np.where(yhat > threshold_value)[0] - if len(valid_idx) > 0: - last_good_file = valid_idx[-1] - else: - last_good_file = n_files - 1 + last_good_file = valid_idx[-1] if len(valid_idx) > 0 else n_files - 1 else: last_good_file = 0 @@ -116,7 +114,7 @@ def select_files_from_mean_f(scan_directory, f_decrease_threshold=15): cumulative_frames = np.cumsum(frame_id) - last_good_frame = cumulative_frames[last_good_file] + cumulative_frames[last_good_file] elapsed_minutes = (time.time() - start_time) / 60 @@ -165,10 +163,7 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): # Some ScanImage versions also use Software tag software = first_page.tags.get("Software") - if software is not None: - scope_str = str(software.value) - else: - scope_str = image_description + scope_str = str(software.value) if software is not None else image_description parsed_info = {} @@ -255,12 +250,12 @@ def parse_tif_header_2photon(tif_fn, skip_behav_sync=False): try: res_match = re.search(r"SI\.objectiveResolution = ([0-9]+\.[0-9]+)", scope_str) - resolution_factor = xySizeFactor * float(res_match.group(1)) + resolution_factor = xy_size_factor * float(res_match.group(1)) except Exception: res_match = re.search(r"SI\.objectiveResolution = (\d+)", scope_str) - resolution_factor = xySizeFactor * float(res_match.group(1)) + resolution_factor = xy_size_factor * float(res_match.group(1)) else: resolution_factor = 1 @@ -479,11 +474,7 @@ def extract_float(pattern, text, default=0): # Resolution scaling resolution_match = re.search(r"(?<=SI\.objectiveResolution = )\d+\.\d+", scope_str) - if resolution_match: - resolution_factor = xySizeFactor * float(resolution_match.group(0)) - - else: - resolution_factor = 1 + resolution_factor = xy_size_factor * float(resolution_match.group(0)) if resolution_match else 1 for i_roi in range(len(roi_marks)): if i_roi != len(roi_marks) - 1: @@ -695,14 +686,12 @@ def parse_array(pattern, text): try: z_val = ast.literal_eval(z_match.group(0)) - parsed_info["Zs"] = zFactor * np.array(z_val) + parsed_info["Zs"] = z_factor * np.array(z_val) except Exception: - try: + with contextlib.suppress(Exception): parsed_info["Zs"] = float(z_match.group(0)) - except Exception: - pass break diff --git a/u19_pipeline/utils/tiff_utils.py b/u19_pipeline/utils/tiff_utils.py index b1b7872c..f5971e16 100644 --- a/u19_pipeline/utils/tiff_utils.py +++ b/u19_pipeline/utils/tiff_utils.py @@ -33,9 +33,8 @@ def check_tif_files(tif_dir): is_compressed = True for gz_file in gz_files: - with gzip.open(gz_file, "rb") as f_in: - with open(gz_file.with_suffix(""), "wb") as f_out: - shutil.copyfileobj(f_in, f_out) + with gzip.open(gz_file, "rb") as f_in, open(gz_file.with_suffix(""), "wb") as f_out: + shutil.copyfileobj(f_in, f_out) fl = sorted(tif_dir.glob("*.tif")) @@ -80,7 +79,7 @@ def get_recording_info(fl, imheader, parsed_info): frames_per_file = np.zeros(len(fl), dtype=int) - for i, file in enumerate(fl): + for i, _file in enumerate(fl): if i == 0: rec_info = parsed_info[i] rec_info["Timing"]["BehavFrames"] = np.array(rec_info["Timing"]["BehavFrames"], dtype=object) @@ -303,12 +302,12 @@ def get_fov_mesoscope( print("\tparsing ROIs...") roi_nr = [roi["pixelResolutionXY"][1] for roi in rec_info["ROI"]] - roi_nc = [roi["pixelResolutionXY"][0] for roi in rec_info["ROI"]] + [roi["pixelResolutionXY"][0] for roi in rec_info["ROI"]] inter_roi_lag = rec_info["interROIlag_sec"] depths = rec_info["nDepths"] - which_depths = sorted(list(set([roi["Zs"] for roi in rec_info["ROI"]]))) + which_depths = sorted({roi["Zs"] for roi in rec_info["ROI"]}) # --------------------------------------------------------------------- # Create directories @@ -335,7 +334,7 @@ def get_fov_mesoscope( # Read stack # ------------------------------------------------------------- - first_page = pages[0].asarray() + pages[0].asarray() thisstack = np.zeros((imheader[iF][0]["Height"], 512, len(imheader[iF])), dtype=np.uint16) @@ -363,12 +362,11 @@ def get_fov_mesoscope( # ------------------------------------------------------------- for idepth in range(depths): - ilag = 0 rowct = 0 which_roi = [i for i, roi in enumerate(rec_info["ROI"]) if roi["Zs"] == which_depths[idepth]] - for iroi in which_roi: + for ilag, iroi in enumerate(which_roi): zidx = list(range(idepth, thisstack.shape[2], depths)) substack = thisstack[rowct : rowct + roi_nr[iroi], :nc, :][:, :, zidx] @@ -394,17 +392,12 @@ def get_fov_mesoscope( first_desc = replace_frame_timestamp(first_desc, inter_roi_lag * ilag) - software = clean_software_string( + clean_software_string( current_header.get("Software", None).value if "Software" in current_header else "" ) - artist = current_header.get("Artist", None).value if "Artist" in current_header else "" + current_header.get("Artist", None).value if "Artist" in current_header else "" - metadata = { - "ImageDescription": first_desc, - "Software": software, - "Artist": artist, - } # ----------------------------------------------------- # Write TIFF @@ -423,8 +416,6 @@ def get_fov_mesoscope( extratags=[], ) - ilag += 1 - # ----------------------------------------------------- # Update row counter # ----------------------------------------------------- @@ -459,10 +450,7 @@ def get_fov_mesoscope( roi_name = rec_info["ROI"][iroi]["name"] - if roi_name: - thisname = f"{roi_name}_z{iz + 1}" - else: - thisname = f"ROI{iroi + 1:02d}_z{iz + 1}" + thisname = f"{roi_name}_z{iz + 1}" if roi_name else f"ROI{iroi + 1:02d}_z{iz + 1}" fov_key["tiff_split_name"] = thisname From c4622bd73b00dce7a0fdf23423e05bcaa4fb0070 Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Fri, 12 Jun 2026 06:42:07 -0400 Subject: [PATCH 6/6] replace print() with structured logging via rich throughout codebase - Add rich>=15.0.0 to dependencies - Add u19_pipeline/utils/logging_config.py with get_logger() (library-safe, no handler setup) and setup_logging() (Rich handler for script entrypoints) - Root logger gets a NullHandler so library imports are silent by default - Convert all active print() calls to logger.debug/info/warning/error/exception across ~35 files in u19_pipeline/ and scripts/ - Wrap scripts/ingest_hdf5_to_db.py in main() + __main__ guard so it is safe to import; setup_logging() called only at script entry - scripts/rig_maintenance_demo.py: setup_logging() in __main__ block - scripts/conf_file_finding.py: pure utility module, no setup_logging() call Co-Authored-By: Claude Sonnet 4.6 --- initial_conf.py | 13 +- pyproject.toml | 3 +- scripts/conf_file_finding.py | 7 +- scripts/ingest_hdf5_to_db.py | 973 +++++++++--------- scripts/rig_maintenance_demo.py | 77 +- .../live_monitor_alert/live_monitor_alert.py | 7 +- .../alert_system/main_alert_system.py | 5 +- .../rig_maintenance/check_rig_maintenance.py | 11 +- .../schedule_check_alert.py | 7 +- .../alert_system/tech_alert/tech_alert.py | 7 +- .../create_weighing_gui_ss.py | 13 +- .../water_weigh_alert/water_weigh_alert.py | 13 +- .../clusters_paths_and_transfers.py | 38 +- .../create_missing_lfps_script.py | 5 +- .../automatic_job/ephys_element_ingest.py | 11 +- u19_pipeline/automatic_job/imaging_element.py | 31 +- .../automatic_job/parameter_file_creator.py | 5 +- .../populate_missing_syncbehavior_ephys.py | 5 +- .../automatic_job/pupillometry_handler.py | 59 +- .../automatic_job/pupillometry_main.py | 12 +- .../automatic_job/recording_handler.py | 34 +- .../recording_process_handler.py | 36 +- u19_pipeline/automatic_job/slurm_creator.py | 35 +- u19_pipeline/automatic_job/tigress2pni.py | 25 +- u19_pipeline/copy_table.py | 13 +- u19_pipeline/ephys_pipeline.py | 65 +- u19_pipeline/ephys_sync.py | 7 +- u19_pipeline/imaging_pipeline.py | 15 +- .../utils/DemoReadSGLXData/read_sglx.py | 22 +- u19_pipeline/utils/dlc_process.py | 7 +- u19_pipeline/utils/ephys_fix_sync_code.py | 27 +- u19_pipeline/utils/ephys_utils.py | 72 +- u19_pipeline/utils/logging_config.py | 46 + u19_pipeline/utils/matlab_utils.py | 8 +- u19_pipeline/utils/scp_transfers.py | 29 +- .../utils/tiff_matlab_imaging_utils.py | 10 +- u19_pipeline/utils/tiff_utils.py | 9 +- uv.lock | 36 + 38 files changed, 990 insertions(+), 808 deletions(-) create mode 100644 u19_pipeline/utils/logging_config.py diff --git a/initial_conf.py b/initial_conf.py index 31897693..efbcc879 100755 --- a/initial_conf.py +++ b/initial_conf.py @@ -16,6 +16,9 @@ from getpass import getpass import pandas as pd +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) def initial_conf(save_user=True, replace_user=False, global_config_flag=True): @@ -29,13 +32,13 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True): try_find_conf_file() except FileNotFoundError: pass - print("DataJoint configuration file not found. Running configuration script...") + logger.info("DataJoint configuration file not found. Running configuration script...") if global_config_flag: - print( + logger.info( "Global configuration flag is set to True. The configuration will be saved in the global configuration file." ) else: - print( + logger.info( "Global configuration flag is set to False. The configuration will be saved in the local configuration file." ) @@ -48,7 +51,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True): user_already = True if replace_user or not user_already: - print("Enter your username (Princeton NETID):") + logger.info("Enter your username (Princeton NETID):") user = input() password = getpass() dj.conn(host=host, user=user, password=password) @@ -100,7 +103,7 @@ def initial_conf(save_user=True, replace_user=False, global_config_flag=True): dj.config["stores"] = dj_stores_dict if global_config_flag: - print( + logger.info( "Global configuration flag is set to True. The configuration will be saved in the global configuration file." ) dj.config.save_global() diff --git a/pyproject.toml b/pyproject.toml index ba3d3f9e..958dec98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,8 @@ dependencies = [ "scipy", "tables>=3.8", "astropy>=6.0", - "openpyxl" + "openpyxl", + "rich>=15.0.0", ] requires-python = ">=3.12" diff --git a/scripts/conf_file_finding.py b/scripts/conf_file_finding.py index 7f94b548..61fe31e5 100644 --- a/scripts/conf_file_finding.py +++ b/scripts/conf_file_finding.py @@ -1,5 +1,8 @@ import os import pathlib +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) def chdir_to_root(): @@ -47,12 +50,12 @@ def try_find_conf_file(): root_dir_found, conf_file_found = chdir_to_root() if root_dir_found and conf_file_found: - print("Local configuration file found !!, no need to run the configuration (unless configuration has changed)") + logger.info("Local configuration file found !!, no need to run the configuration (unless configuration has changed)") elif root_dir_found: raise FileNotFoundError( "Local configuration file not found. Ignore this if you have a global config. Run configuration script (initial_conf.py) otherwise" ) elif os.path.isfile(pathlib.Path(pathlib.Path.home(), ".datajoint_config.json")): - print("Global configuration file found !!, no need to run the configuration (unless configuration has changed)") + logger.info("Global configuration file found !!, no need to run the configuration (unless configuration has changed)") else: raise FileNotFoundError("Root dir not found, change this notebook to the project folder") diff --git a/scripts/ingest_hdf5_to_db.py b/scripts/ingest_hdf5_to_db.py index c21f73f4..acdffb15 100644 --- a/scripts/ingest_hdf5_to_db.py +++ b/scripts/ingest_hdf5_to_db.py @@ -13,490 +13,501 @@ import datajoint as dj import numpy as np import pandas as pd - -dj.config["database.host"] = "datajoint00.pni.princeton.edu" -# load dj creds from file -credfile = "/jukebox/wang/ahoag/.djenv" -with open(credfile) as infile: - cred_dict = json.load(infile) - -dj.config["database.user"] = cred_dict.get("DJ_DB_USER") -dj.config["database.password"] = cred_dict.get("DJ_DB_PASS") - -# Link to existing databases -u19_lab = dj.create_virtual_module("u19_lab", "u19_lab") -u19_subject = dj.create_virtual_module("u19_subject", "u19_subject") -u19_acq = dj.create_virtual_module("u19_acquisition", "u19_acquisition") - -# Link to new Puffs dbs -u19_puffs = dj.create_virtual_module("u19_puffs", "u19_puffs") - -# Currently a hardcoded data folder to one of Marlies' cohorts for this example, -# but in the future will loop over folders starting from -# The root folder: /jukebox/braininit/puffs/ -data_folder = "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation/cohort_10/rig0/" -h5_files = glob.glob(data_folder + "/data*.h5") -mouse_info_file = os.path.join( - "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation", - "mouse_info_0821_datajoint_format.csv", -) - -# first read in the mouse_info file into a pandas dataframe -df_info = pd.read_csv(mouse_info_file, keep_default_na=False) - -# # Use mouse_info file and do the u19_subject and u19_lab inserts -# Tables we need to insert into (**In this order**) are: -# - u19_lab.User # in case user is not already in the database -# - u19_lab.Protocol # in case protocol is not already in the database -# - u19_lab.Location # in case the location of each subject is not already in the database -# -# - u19_subject.Species # in case they are NOT training on mice -# - u19_subject.Strain # in case they are NOT training on a strain of mice already in the db -# - u19_subject.Line # in case they are using a new genetic line -# - u19_subject.Subject # in case they are using new subjects - - -# u19_lab.User() inserts -unique_netids = df_info["netid"].unique() -full_names = df_info["experimenter_fullname"] -user_insert_list = [] -for netid in unique_netids: - netid_mask = df_info["netid"] == netid - full_name = full_names[netid_mask].iloc[0] - user_insert_dict = { - "user_id": netid, - "user_nickname": netid, - "full_name": full_name, - "email": f"{netid}@princeton.edu", - "contact_via": "Email", - "presence": "Available", - "tech_responsibility": "no", - "day_cutoff_time": np.array([[18.0, 0.0]]), - } - user_insert_list.append(user_insert_dict) -u19_lab.User.insert(user_insert_list, skip_duplicates=True) - -# u19_lab.Protocol() inserts -unique_protocols = df_info["protocol"].unique() -protocol_insert_list = [] -for protocol in unique_protocols: - protocol_insert_dict = { - "protocol": protocol, - "protocol_description": "", - } - protocol_insert_list.append(protocol_insert_dict) -u19_lab.Protocol.insert(protocol_insert_list, skip_duplicates=True) - - -# u19_lab.Location() inserts -unique_locations = df_info["location"].unique() -location_descriptions = df_info["location_description"] -location_insert_list = [] -for location in unique_locations: - location_mask = df_info["location"] == location - location_description = location_descriptions[netid_mask].iloc[0] - location_insert_dict = { - "location": location, - "location_description": location_description, - } - location_insert_list.append(location_insert_dict) -u19_lab.Location.insert(location_insert_list, skip_duplicates=True) - -# u19_subject.Species() inserts -unique_species = df_info["binomial"].unique() -species_insert_list = [] -for species in unique_species: - species_insert_dict = { - "binomial": species, - "species_nickname": "", - } - species_insert_list.append(species_insert_dict) -u19_subject.Species.insert(species_insert_list, skip_duplicates=True) - -# u19_subject.Strain() inserts -unique_strains = df_info["strain"].unique() -strain_insert_list = [] -for strain in unique_strains: - strain_insert_dict = { - "strain_name": strain, - "strain_description": "", - } - strain_insert_list.append(strain_insert_dict) -u19_subject.Strain.insert(strain_insert_list, skip_duplicates=True) - -# u19_subject.Line() inserts -unique_lines = df_info["genetic_line"].unique() -binomials = df_info["binomial"] -strains = df_info["strain"] -line_insert_list = [] -for line in unique_lines: - line_mask = df_info["genetic_line"] == line - binomial = binomials[line_mask].iloc[0] - strain = strains[line_mask].iloc[0] - line_insert_dict = { - "line": line, - "binomial": binomial, - "strain_name": strain, - "line_description": "", - "target_phenotype": "", - "is_active": 1, - } - line_insert_list.append(line_insert_dict) -u19_subject.Line.insert(line_insert_list, skip_duplicates=True) - - -# u19_subject.Subject() inserts -unique_subjects = df_info["subj"].unique() -project_names = df_info["project_name"] -subject_descriptions = df_info["subj_description"] -netids = df_info["netid"] -sexes = df_info["sex"] -DOBs = df_info["DOB"] -locations = df_info["location"] -lines = df_info["genetic_line"] -protocols = df_info["protocol"] -subject_insert_list = [] -for subject in unique_subjects: - subject_mask = df_info["subj"] == subject - netid = netids[subject_mask].iloc[0] - project_name = project_names[subject_mask].iloc[0] - subject_fullname = "_".join([netid, project_name, str(subject)]) - sex = sexes[subject_mask].iloc[0] - DOB = DOBs[subject_mask].iloc[0] - location = locations[subject_mask].iloc[0] - line = lines[subject_mask].iloc[0] - protocol = protocols[subject_mask].iloc[0] - subject_description = subject_descriptions[subject_mask].iloc[0] - subject_insert_dict = { - "subject_fullname": subject_fullname, - "subject_nickname": str(subject), - "subject_description": subject_description, - "user_id": netid, - "sex": "Male" if sex == "M" else "Female", - "dob": DOB, - "location": location, - "line": line, - "protocol": protocol, - } - subject_insert_list.append(subject_insert_dict) -u19_subject.Subject.insert(subject_insert_list, skip_duplicates=True) - - -# ## Now use h5 files to make inserts into u19_acquisition tables and puffs specific tables -# -# Tables we need to insert into (**In this order**) are: -# - u19_puffs.PuffsCohort # in case cohort is not already in the database -# - u19_puffs.PuffsFileAcquisition # as we process the various files -# -# - u19_acquisition.SessionStarted # as we process the session data in each h5 file -# - u19_acquisition.Session # as we process the session data in each h5 file -# -# - u19_puffs.PuffsSession -# - u19_puffs.PuffsSession.Trial -# - u19_puffs.PuffsSession.TrialPhase -# - u19_puffs.PuffsSession.Puff -# - -# first find list of h5 files that are already processed so we do not -# repeat the ingestion on those -already_processed_filenames = (u19_puffs.PuffsFileAcquisition() & "ingested=1").fetch("h5_filename") - -# Loop over h5 files in this cohort/rig folder -for h5_file in h5_files: - if h5_file in already_processed_filenames: - print(f"file: {h5_file} already processed!") - continue - print(h5_file) - - """ Grab the relevant dataframes from the hdf5 file """ - with pd.HDFStore(h5_file) as data: - df_trials = data.trials - df_puffs = data.trials_timing - df_phases = data.phases - - """ Sometimes there are extra session metadata stored in other dataframes - with the key names: - 'sessions/{session_number}/notes' - 'sessions/{session_number}/params' - 'sessions/{session_number}/sync' - If they exist we want to ingest their metadata, but they do not always exist - """ - stored_session_keys = [x for x in data if "sessions" in x] - - """ Figure out the information from the h5 filepath """ - rig = h5_file.split("/")[-2][-1] # just want the 0 or 1 (not "rig0" or "rig1") - cohort = h5_file.split("/")[-3].split("_")[-1] # 'want C10' or just '10' - project_name = h5_file.split("/")[-4].strip() - username = h5_file.split("/")[-5].strip() - puffs_cohort_insert_dict = { - "user_id": username, - "project_name": project_name, - "cohort": cohort, +from u19_pipeline.utils.logging_config import get_logger, setup_logging + +logger = get_logger(__name__) + + +def main(): + setup_logging() + + dj.config["database.host"] = "datajoint00.pni.princeton.edu" + # load dj creds from file + credfile = "/jukebox/wang/ahoag/.djenv" + with open(credfile) as infile: + cred_dict = json.load(infile) + + dj.config["database.user"] = cred_dict.get("DJ_DB_USER") + dj.config["database.password"] = cred_dict.get("DJ_DB_PASS") + + # Link to existing databases + u19_lab = dj.create_virtual_module("u19_lab", "u19_lab") + u19_subject = dj.create_virtual_module("u19_subject", "u19_subject") + u19_acq = dj.create_virtual_module("u19_acquisition", "u19_acquisition") + + # Link to new Puffs dbs + u19_puffs = dj.create_virtual_module("u19_puffs", "u19_puffs") + + # Currently a hardcoded data folder to one of Marlies' cohorts for this example, + # but in the future will loop over folders starting from + # The root folder: /jukebox/braininit/puffs/ + data_folder = "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation/cohort_10/rig0/" + h5_files = glob.glob(data_folder + "/data*.h5") + mouse_info_file = os.path.join( + "/jukebox/braininit/puffs/oostland/Tsc1_evidence_accumulation", + "mouse_info_0821_datajoint_format.csv", + ) + + # first read in the mouse_info file into a pandas dataframe + df_info = pd.read_csv(mouse_info_file, keep_default_na=False) + + # # Use mouse_info file and do the u19_subject and u19_lab inserts + # Tables we need to insert into (**In this order**) are: + # - u19_lab.User # in case user is not already in the database + # - u19_lab.Protocol # in case protocol is not already in the database + # - u19_lab.Location # in case the location of each subject is not already in the database + # + # - u19_subject.Species # in case they are NOT training on mice + # - u19_subject.Strain # in case they are NOT training on a strain of mice already in the db + # - u19_subject.Line # in case they are using a new genetic line + # - u19_subject.Subject # in case they are using new subjects + + + # u19_lab.User() inserts + unique_netids = df_info["netid"].unique() + full_names = df_info["experimenter_fullname"] + user_insert_list = [] + for netid in unique_netids: + netid_mask = df_info["netid"] == netid + full_name = full_names[netid_mask].iloc[0] + user_insert_dict = { + "user_id": netid, + "user_nickname": netid, + "full_name": full_name, + "email": f"{netid}@princeton.edu", + "contact_via": "Email", + "presence": "Available", + "tech_responsibility": "no", + "day_cutoff_time": np.array([[18.0, 0.0]]), } - u19_puffs.PuffsCohort().insert1(puffs_cohort_insert_dict, skip_duplicates=True) - - df_trials["rig"] = rig - df_trials["cohort"] = cohort - df_trials["project_name"] = project_name - - """ Make choice column """ - df_trials["choice"] = df_trials["side"] == df_trials["outcome"].astype(int) - """ set choice to -1 if outcome was > 1 """ - bad_outcome_mask = df_trials["outcome"] > 1 - df_trials.loc[bad_outcome_mask, "choice"] = -1 - """ Map choice to string values: L,R and nil """ - df_trials["choice"] = df_trials["choice"].apply(lambda x: "L" if x == 0 else ("R" if x == 1 else "nil")) - - """ Make an answered correct column from outcome column """ - df_trials["answered_correct"] = (df_trials["outcome"] == 1).astype("int") - - """ Turn side column into string type """ - df_trials["side"] = df_trials["side"].apply(lambda x: "L" if x == 0 else ("R" if x == 1 else "nil")) - - """ Make a new dataframe for unique sessions, i.e. where subj and session are unique - in trials """ - df_sessions = df_trials.groupby(["subj", "session"]).max().reset_index().sort_values(["subj", "session"]) - sessions = df_sessions["session"] - last_levels = df_sessions["level"] - session_ends_rel = df_sessions["end"] - """ make a session_dates column """ - session_dates = sessions.apply(lambda x: str(x)[:10]) - df_sessions["session_date"] = session_dates - subjs = df_sessions["subj"] - """ look up where these subjects are """ - rigs = df_sessions["rig"].astype("int") - """ make a session number column - i.e. - the 0-indexed counter for the session on a day for a single subject """ - session_numbers = df_sessions.groupby(["session_date", "subj"]).cumcount() - df_sessions["session_number"] = session_numbers - - """ Figure out fraction correct for each session """ - fraction_trials_correct_df = df_trials.groupby(["session", "subj"]).agg( - {"answered_correct": "sum", "side": "count"} - ) - fraction_trials_correct_df["fraction_correct"] = ( - fraction_trials_correct_df["answered_correct"] / fraction_trials_correct_df["side"] - ) - fraction_trials_correct_df = fraction_trials_correct_df.sort_values(["subj", "session"]).reset_index() - df_sessions["fraction_correct"] = fraction_trials_correct_df["fraction_correct"] - fractions_correct = df_sessions["fraction_correct"] - - """ Initialize lists to fill and then insert into db """ - session_started_insert_list = [] - session_insert_list = [] - puffs_session_insert_list = [] - - """ Loop over sessions and assemble the inserts """ - n_sessions = len(df_sessions) - for ii in range(len(df_sessions)): - subj = str(int(subjs.iloc[ii])) - subject_fullname = "_".join([username, project_name, subj]) - last_level = last_levels.iloc[ii] - date = session_dates.iloc[ii] - print(f"session {ii + 1}/{len(df_sessions)}: {subject_fullname}, {date}") - session_datetime = sessions.iloc[ii] - session_end_rel = session_ends_rel[ii] - session_end_datetime = session_datetime + timedelta(seconds=session_end_rel) - session_number = session_numbers.iloc[ii] - percentage_correct = fractions_correct.iloc[ii] * 100 - rig = rigs.iloc[ii] - - if rig == 0: - location = "pni-ltl016-05" - elif rig == 1: - location = "wang-behavior" - else: - sys.exit(f"Rig: {rig} is not one of the rigs. Check data type (needs to be integer)") - session_compressed_str = session_datetime.strftime("%Y%m%d%H%M%S") - - """ Check to see if the session metadata dataframes exist """ - this_session_params_key = f"sessions/{session_compressed_str}/params" - if this_session_params_key in stored_session_keys: - session_param_dict = json.loads(data[this_session_params_key].iloc[0]) - else: - session_param_dict = None - this_session_notes_key = f"sessions/{session_compressed_str}/notes" - if this_session_notes_key in stored_session_keys: - session_notes_dict = json.loads(data[this_session_notes_key].iloc[0]) - session_notes = session_notes_dict.get("notes", None) - session_stdout = session_notes_dict.get("stdout", None) - session_stderr = session_notes_dict.get("stderr", None) - else: - session_notes = None - session_stdout = None - session_stderr = None - this_session_sync_key = f"sessions/{session_compressed_str}/sync" - if this_session_sync_key in stored_session_keys: - session_sync_dict = data[this_session_notes_key].todict() - else: - session_sync_dict = None - - """ Assemble the u19_acquistion inserts for this session """ - session_started_insert_dict = { - "subject_fullname": subject_fullname, - "session_date": date, - "session_number": session_number, - "session_start_time": session_datetime.strftime("%Y-%m-%d %H:%M:%S"), - "session_location": location, - "task": "AirPuffs", - "local_path_behavior_file": "", - "remote_path_behavior_file": h5_file, - "is_finished": 1, - } - session_insert_dict = { - "subject_fullname": subject_fullname, - "session_date": date, - "session_number": session_number, - "session_start_time": session_datetime.strftime("%Y-%m-%d %H:%M:%S"), - "session_end_time": session_end_datetime.strftime("%Y-%m-%d %H:%M:%S"), - "session_location": location, - "task": "AirPuffs", - "level": last_level, - "set_id": 1, - "stimulus_bank": "", - "stimulus_commit": "", - "session_performance": percentage_correct, - "session_narrative": "", - } - """ Assemble the puffs inserts for this session """ - # PuffsSession() table - puffs_session_insert_dict = { - "subject_fullname": subject_fullname, - "session_date": date, - "session_number": session_number, - "session_params": session_param_dict, - "rig": rig, - "notes": session_notes, - "stdout": session_stdout, - "stderr": session_stderr, - "sync": session_sync_dict, + user_insert_list.append(user_insert_dict) + u19_lab.User.insert(user_insert_list, skip_duplicates=True) + + # u19_lab.Protocol() inserts + unique_protocols = df_info["protocol"].unique() + protocol_insert_list = [] + for protocol in unique_protocols: + protocol_insert_dict = { + "protocol": protocol, + "protocol_description": "", + } + protocol_insert_list.append(protocol_insert_dict) + u19_lab.Protocol.insert(protocol_insert_list, skip_duplicates=True) + + + # u19_lab.Location() inserts + unique_locations = df_info["location"].unique() + location_descriptions = df_info["location_description"] + location_insert_list = [] + for location in unique_locations: + location_mask = df_info["location"] == location + location_description = location_descriptions[netid_mask].iloc[0] + location_insert_dict = { + "location": location, + "location_description": location_description, + } + location_insert_list.append(location_insert_dict) + u19_lab.Location.insert(location_insert_list, skip_duplicates=True) + + # u19_subject.Species() inserts + unique_species = df_info["binomial"].unique() + species_insert_list = [] + for species in unique_species: + species_insert_dict = { + "binomial": species, + "species_nickname": "", + } + species_insert_list.append(species_insert_dict) + u19_subject.Species.insert(species_insert_list, skip_duplicates=True) + + # u19_subject.Strain() inserts + unique_strains = df_info["strain"].unique() + strain_insert_list = [] + for strain in unique_strains: + strain_insert_dict = { + "strain_name": strain, + "strain_description": "", + } + strain_insert_list.append(strain_insert_dict) + u19_subject.Strain.insert(strain_insert_list, skip_duplicates=True) + + # u19_subject.Line() inserts + unique_lines = df_info["genetic_line"].unique() + binomials = df_info["binomial"] + strains = df_info["strain"] + line_insert_list = [] + for line in unique_lines: + line_mask = df_info["genetic_line"] == line + binomial = binomials[line_mask].iloc[0] + strain = strains[line_mask].iloc[0] + line_insert_dict = { + "line": line, + "binomial": binomial, + "strain_name": strain, + "line_description": "", + "target_phenotype": "", + "is_active": 1, + } + line_insert_list.append(line_insert_dict) + u19_subject.Line.insert(line_insert_list, skip_duplicates=True) + + + # u19_subject.Subject() inserts + unique_subjects = df_info["subj"].unique() + project_names = df_info["project_name"] + subject_descriptions = df_info["subj_description"] + netids = df_info["netid"] + sexes = df_info["sex"] + DOBs = df_info["DOB"] + locations = df_info["location"] + lines = df_info["genetic_line"] + protocols = df_info["protocol"] + subject_insert_list = [] + for subject in unique_subjects: + subject_mask = df_info["subj"] == subject + netid = netids[subject_mask].iloc[0] + project_name = project_names[subject_mask].iloc[0] + subject_fullname = "_".join([netid, project_name, str(subject)]) + sex = sexes[subject_mask].iloc[0] + DOB = DOBs[subject_mask].iloc[0] + location = locations[subject_mask].iloc[0] + line = lines[subject_mask].iloc[0] + protocol = protocols[subject_mask].iloc[0] + subject_description = subject_descriptions[subject_mask].iloc[0] + subject_insert_dict = { + "subject_fullname": subject_fullname, + "subject_nickname": str(subject), + "subject_description": subject_description, + "user_id": netid, + "sex": "Male" if sex == "M" else "Female", + "dob": DOB, + "location": location, + "line": line, + "protocol": protocol, + } + subject_insert_list.append(subject_insert_dict) + u19_subject.Subject.insert(subject_insert_list, skip_duplicates=True) + + + # ## Now use h5 files to make inserts into u19_acquisition tables and puffs specific tables + # + # Tables we need to insert into (**In this order**) are: + # - u19_puffs.PuffsCohort # in case cohort is not already in the database + # - u19_puffs.PuffsFileAcquisition # as we process the various files + # + # - u19_acquisition.SessionStarted # as we process the session data in each h5 file + # - u19_acquisition.Session # as we process the session data in each h5 file + # + # - u19_puffs.PuffsSession + # - u19_puffs.PuffsSession.Trial + # - u19_puffs.PuffsSession.TrialPhase + # - u19_puffs.PuffsSession.Puff + # + + # first find list of h5 files that are already processed so we do not + # repeat the ingestion on those + already_processed_filenames = (u19_puffs.PuffsFileAcquisition() & "ingested=1").fetch("h5_filename") + + # Loop over h5 files in this cohort/rig folder + for h5_file in h5_files: + if h5_file in already_processed_filenames: + logger.info("file: %s already processed!", h5_file) + continue + logger.info("processing file %s", h5_file) + + """ Grab the relevant dataframes from the hdf5 file """ + with pd.HDFStore(h5_file) as data: + df_trials = data.trials + df_puffs = data.trials_timing + df_phases = data.phases + + """ Sometimes there are extra session metadata stored in other dataframes + with the key names: + 'sessions/{session_number}/notes' + 'sessions/{session_number}/params' + 'sessions/{session_number}/sync' + If they exist we want to ingest their metadata, but they do not always exist + """ + stored_session_keys = [x for x in data if "sessions" in x] + + """ Figure out the information from the h5 filepath """ + rig = h5_file.split("/")[-2][-1] # just want the 0 or 1 (not "rig0" or "rig1") + cohort = h5_file.split("/")[-3].split("_")[-1] # 'want C10' or just '10' + project_name = h5_file.split("/")[-4].strip() + username = h5_file.split("/")[-5].strip() + puffs_cohort_insert_dict = { + "user_id": username, + "project_name": project_name, + "cohort": cohort, } - # Trial() table - df_trials_this_session = df_trials[df_trials["session"] == session_datetime][ - [ - "idx", - "level", - "side", - "choice", - "answered_correct", - "draw_p", - "start", - "end", - "dur", - "nL_intended", - "nL", - "nR_intended", - "nR", - "reward", - "reward_scale", - "rule", - ] - ] - num_trials_this_session = len(df_trials_this_session) - session_insert_dict["num_trials"] = num_trials_this_session - df_trials_this_session["subject_fullname"] = subject_fullname - df_trials_this_session["session_date"] = date - df_trials_this_session["session_number"] = session_number - df_trials_this_session["task"] = "AirPuffs" - df_trials_this_session["set_id"] = 1 - df_trials_this_session["trial_duration"] = df_trials_this_session["end"] - df_trials_this_session["start"] - df_trials_this_session.rename( - columns={ - "idx": "trial_idx", - "draw_p": "trial_prior_p_left", - "start": "trial_rel_start", - "end": "trial_rel_finish", - "dur": "cue_period", - "nL": "num_puffs_received_l", - "nL_intended": "num_puffs_intended_l", - "nR": "num_puffs_received_r", - "nR_intended": "num_puffs_intended_r", - "side": "trial_type", - "reward": "reward_rel_start", - }, - inplace=True, + u19_puffs.PuffsCohort().insert1(puffs_cohort_insert_dict, skip_duplicates=True) + + df_trials["rig"] = rig + df_trials["cohort"] = cohort + df_trials["project_name"] = project_name + + """ Make choice column """ + df_trials["choice"] = df_trials["side"] == df_trials["outcome"].astype(int) + """ set choice to -1 if outcome was > 1 """ + bad_outcome_mask = df_trials["outcome"] > 1 + df_trials.loc[bad_outcome_mask, "choice"] = -1 + """ Map choice to string values: L,R and nil """ + df_trials["choice"] = df_trials["choice"].apply(lambda x: "L" if x == 0 else ("R" if x == 1 else "nil")) + + """ Make an answered correct column from outcome column """ + df_trials["answered_correct"] = (df_trials["outcome"] == 1).astype("int") + + """ Turn side column into string type """ + df_trials["side"] = df_trials["side"].apply(lambda x: "L" if x == 0 else ("R" if x == 1 else "nil")) + + """ Make a new dataframe for unique sessions, i.e. where subj and session are unique + in trials """ + df_sessions = df_trials.groupby(["subj", "session"]).max().reset_index().sort_values(["subj", "session"]) + sessions = df_sessions["session"] + last_levels = df_sessions["level"] + session_ends_rel = df_sessions["end"] + """ make a session_dates column """ + session_dates = sessions.apply(lambda x: str(x)[:10]) + df_sessions["session_date"] = session_dates + subjs = df_sessions["subj"] + """ look up where these subjects are """ + rigs = df_sessions["rig"].astype("int") + """ make a session number column - i.e. + the 0-indexed counter for the session on a day for a single subject """ + session_numbers = df_sessions.groupby(["session_date", "subj"]).cumcount() + df_sessions["session_number"] = session_numbers + + """ Figure out fraction correct for each session """ + fraction_trials_correct_df = df_trials.groupby(["session", "subj"]).agg( + {"answered_correct": "sum", "side": "count"} ) - - trials_insert_list = df_trials_this_session.to_dict("records") - - # Puff() table - df_puffs_this_session = df_puffs[df_puffs["session"] == session_datetime].copy() - df_puffs_this_session["subject_fullname"] = subject_fullname - df_puffs_this_session["session_date"] = date - df_puffs_this_session["session_number"] = session_number - df_puffs_this_session["puff_idx"] = df_puffs_this_session.groupby("trial").cumcount() - df_puffs_this_session.rename( - columns={ - "time": "puff_rel_time", - "trial": "trial_idx", - }, - inplace=True, + fraction_trials_correct_df["fraction_correct"] = ( + fraction_trials_correct_df["answered_correct"] / fraction_trials_correct_df["side"] ) - df_puffs_this_session["session_date"] = date - df_puffs_this_session["session_number"] = session_number - df_puffs_this_session = df_puffs_this_session[ - [ - "subject_fullname", - "session_date", - "session_number", - "trial_idx", - "puff_idx", - "side", - "puff_rel_time", + fraction_trials_correct_df = fraction_trials_correct_df.sort_values(["subj", "session"]).reset_index() + df_sessions["fraction_correct"] = fraction_trials_correct_df["fraction_correct"] + fractions_correct = df_sessions["fraction_correct"] + + """ Initialize lists to fill and then insert into db """ + session_started_insert_list = [] + session_insert_list = [] + puffs_session_insert_list = [] + + """ Loop over sessions and assemble the inserts """ + n_sessions = len(df_sessions) + for ii in range(len(df_sessions)): + subj = str(int(subjs.iloc[ii])) + subject_fullname = "_".join([username, project_name, subj]) + last_level = last_levels.iloc[ii] + date = session_dates.iloc[ii] + logger.info("session %s/%s: %s, %s", ii + 1, len(df_sessions), subject_fullname, date) + session_datetime = sessions.iloc[ii] + session_end_rel = session_ends_rel[ii] + session_end_datetime = session_datetime + timedelta(seconds=session_end_rel) + session_number = session_numbers.iloc[ii] + percentage_correct = fractions_correct.iloc[ii] * 100 + rig = rigs.iloc[ii] + + if rig == 0: + location = "pni-ltl016-05" + elif rig == 1: + location = "wang-behavior" + else: + sys.exit(f"Rig: {rig} is not one of the rigs. Check data type (needs to be integer)") + session_compressed_str = session_datetime.strftime("%Y%m%d%H%M%S") + + """ Check to see if the session metadata dataframes exist """ + this_session_params_key = f"sessions/{session_compressed_str}/params" + if this_session_params_key in stored_session_keys: + session_param_dict = json.loads(data[this_session_params_key].iloc[0]) + else: + session_param_dict = None + this_session_notes_key = f"sessions/{session_compressed_str}/notes" + if this_session_notes_key in stored_session_keys: + session_notes_dict = json.loads(data[this_session_notes_key].iloc[0]) + session_notes = session_notes_dict.get("notes", None) + session_stdout = session_notes_dict.get("stdout", None) + session_stderr = session_notes_dict.get("stderr", None) + else: + session_notes = None + session_stdout = None + session_stderr = None + this_session_sync_key = f"sessions/{session_compressed_str}/sync" + if this_session_sync_key in stored_session_keys: + session_sync_dict = data[this_session_notes_key].todict() + else: + session_sync_dict = None + + """ Assemble the u19_acquistion inserts for this session """ + session_started_insert_dict = { + "subject_fullname": subject_fullname, + "session_date": date, + "session_number": session_number, + "session_start_time": session_datetime.strftime("%Y-%m-%d %H:%M:%S"), + "session_location": location, + "task": "AirPuffs", + "local_path_behavior_file": "", + "remote_path_behavior_file": h5_file, + "is_finished": 1, + } + session_insert_dict = { + "subject_fullname": subject_fullname, + "session_date": date, + "session_number": session_number, + "session_start_time": session_datetime.strftime("%Y-%m-%d %H:%M:%S"), + "session_end_time": session_end_datetime.strftime("%Y-%m-%d %H:%M:%S"), + "session_location": location, + "task": "AirPuffs", + "level": last_level, + "set_id": 1, + "stimulus_bank": "", + "stimulus_commit": "", + "session_performance": percentage_correct, + "session_narrative": "", + } + """ Assemble the puffs inserts for this session """ + # PuffsSession() table + puffs_session_insert_dict = { + "subject_fullname": subject_fullname, + "session_date": date, + "session_number": session_number, + "session_params": session_param_dict, + "rig": rig, + "notes": session_notes, + "stdout": session_stdout, + "stderr": session_stderr, + "sync": session_sync_dict, + } + # Trial() table + df_trials_this_session = df_trials[df_trials["session"] == session_datetime][ + [ + "idx", + "level", + "side", + "choice", + "answered_correct", + "draw_p", + "start", + "end", + "dur", + "nL_intended", + "nL", + "nR_intended", + "nR", + "reward", + "reward_scale", + "rule", + ] ] - ] - puffs_insert_list = df_puffs_this_session.to_dict("records") - - # TrialPhase() table - df_phases_this_session = df_phases[df_phases["session"] == session_datetime].copy() - df_phases_this_session["subject_fullname"] = subject_fullname - df_phases_this_session["session_date"] = date - df_phases_this_session["session_number"] = session_number - df_phases_this_session.rename( - columns={ - "trial": "trial_idx", - "start_time": "phase_rel_start", - "end_time": "phase_rel_finish", - }, - inplace=True, - ) - df_phases_this_session = df_phases_this_session[ - [ - "subject_fullname", - "session_date", - "session_number", - "trial_idx", - "phase", - "phase_rel_start", - "phase_rel_finish", + num_trials_this_session = len(df_trials_this_session) + session_insert_dict["num_trials"] = num_trials_this_session + df_trials_this_session["subject_fullname"] = subject_fullname + df_trials_this_session["session_date"] = date + df_trials_this_session["session_number"] = session_number + df_trials_this_session["task"] = "AirPuffs" + df_trials_this_session["set_id"] = 1 + df_trials_this_session["trial_duration"] = df_trials_this_session["end"] - df_trials_this_session["start"] + df_trials_this_session.rename( + columns={ + "idx": "trial_idx", + "draw_p": "trial_prior_p_left", + "start": "trial_rel_start", + "end": "trial_rel_finish", + "dur": "cue_period", + "nL": "num_puffs_received_l", + "nL_intended": "num_puffs_intended_l", + "nR": "num_puffs_received_r", + "nR_intended": "num_puffs_intended_r", + "side": "trial_type", + "reward": "reward_rel_start", + }, + inplace=True, + ) + + trials_insert_list = df_trials_this_session.to_dict("records") + + # Puff() table + df_puffs_this_session = df_puffs[df_puffs["session"] == session_datetime].copy() + df_puffs_this_session["subject_fullname"] = subject_fullname + df_puffs_this_session["session_date"] = date + df_puffs_this_session["session_number"] = session_number + df_puffs_this_session["puff_idx"] = df_puffs_this_session.groupby("trial").cumcount() + df_puffs_this_session.rename( + columns={ + "time": "puff_rel_time", + "trial": "trial_idx", + }, + inplace=True, + ) + df_puffs_this_session["session_date"] = date + df_puffs_this_session["session_number"] = session_number + df_puffs_this_session = df_puffs_this_session[ + [ + "subject_fullname", + "session_date", + "session_number", + "trial_idx", + "puff_idx", + "side", + "puff_rel_time", + ] + ] + puffs_insert_list = df_puffs_this_session.to_dict("records") + + # TrialPhase() table + df_phases_this_session = df_phases[df_phases["session"] == session_datetime].copy() + df_phases_this_session["subject_fullname"] = subject_fullname + df_phases_this_session["session_date"] = date + df_phases_this_session["session_number"] = session_number + df_phases_this_session.rename( + columns={ + "trial": "trial_idx", + "start_time": "phase_rel_start", + "end_time": "phase_rel_finish", + }, + inplace=True, + ) + df_phases_this_session = df_phases_this_session[ + [ + "subject_fullname", + "session_date", + "session_number", + "trial_idx", + "phase", + "phase_rel_start", + "phase_rel_finish", + ] ] - ] - """ Some phases seem to reference trials that do not exist - restrict phase rows to trials that exist """ - trial_idxs = df_trials_this_session["trial_idx"] - phase_trial_mask = df_phases_this_session["trial_idx"].isin(trial_idxs.values) - df_phases_this_session_goodtrials = df_phases_this_session[phase_trial_mask] - phases_insert_list = df_phases_this_session_goodtrials.to_dict("records") - """ Start a transaction and do the inserts for this session """ - connection = u19_acq.SessionStarted.connection - with connection.transaction: - u19_acq.SessionStarted().insert1(session_started_insert_dict, skip_duplicates=True) - u19_acq.Session().insert1(session_insert_dict, skip_duplicates=True) - u19_puffs.PuffsSession().insert1(puffs_session_insert_dict, skip_duplicates=True) - u19_puffs.PuffsSession.Trial().insert(trials_insert_list, skip_duplicates=True) - # u19_puffs.PuffsSession.Puff().insert(puffs_insert_list,skip_duplicates=True) - # u19_puffs.PuffsSession.TrialPhase().insert(phases_insert_list,skip_duplicates=True) - """ If the inserts for all sessions in this hdf5 file were successful, - then we need to mark this hdf5 file as processed by inserting it - into the PuffsFileAcquisition() table in u19_puffs_acquisition table """ - file_acq_insert_dict = { - "user_id": username, - "project_name": project_name, - "cohort": cohort, - "rig": rig, - "h5_filename": h5_file, - "ingested": 1, - } - u19_puffs.PuffsFileAcquisition.insert1( - file_acq_insert_dict, replace=True - ) # want to overwrite with ingested=1 if ingested=0 for some reason + """ Some phases seem to reference trials that do not exist + restrict phase rows to trials that exist """ + trial_idxs = df_trials_this_session["trial_idx"] + phase_trial_mask = df_phases_this_session["trial_idx"].isin(trial_idxs.values) + df_phases_this_session_goodtrials = df_phases_this_session[phase_trial_mask] + phases_insert_list = df_phases_this_session_goodtrials.to_dict("records") + """ Start a transaction and do the inserts for this session """ + connection = u19_acq.SessionStarted.connection + with connection.transaction: + u19_acq.SessionStarted().insert1(session_started_insert_dict, skip_duplicates=True) + u19_acq.Session().insert1(session_insert_dict, skip_duplicates=True) + u19_puffs.PuffsSession().insert1(puffs_session_insert_dict, skip_duplicates=True) + u19_puffs.PuffsSession.Trial().insert(trials_insert_list, skip_duplicates=True) + # u19_puffs.PuffsSession.Puff().insert(puffs_insert_list,skip_duplicates=True) + # u19_puffs.PuffsSession.TrialPhase().insert(phases_insert_list,skip_duplicates=True) + """ If the inserts for all sessions in this hdf5 file were successful, + then we need to mark this hdf5 file as processed by inserting it + into the PuffsFileAcquisition() table in u19_puffs_acquisition table """ + file_acq_insert_dict = { + "user_id": username, + "project_name": project_name, + "cohort": cohort, + "rig": rig, + "h5_filename": h5_file, + "ingested": 1, + } + u19_puffs.PuffsFileAcquisition.insert1( + file_acq_insert_dict, replace=True + ) # want to overwrite with ingested=1 if ingested=0 for some reason + + +if __name__ == "__main__": + main() diff --git a/scripts/rig_maintenance_demo.py b/scripts/rig_maintenance_demo.py index f32a10a0..ebc07313 100755 --- a/scripts/rig_maintenance_demo.py +++ b/scripts/rig_maintenance_demo.py @@ -19,15 +19,19 @@ try: from u19_pipeline import rig_maintenance + from u19_pipeline.utils.logging_config import get_logger, setup_logging except ImportError as e: - print(f"Error importing modules: {e}") - print("Make sure u19_pipeline is properly installed and configured.") + import sys as _sys + _sys.stderr.write(f"Error importing modules: {e}\n") + _sys.stderr.write("Make sure u19_pipeline is properly installed and configured.\n") sys.exit(1) +logger = get_logger(__name__) + def demo_insert_maintenance_record(): """Demo: Insert a maintenance record.""" - print("=== Demo: Inserting Maintenance Record ===") + logger.info("=== Demo: Inserting Maintenance Record ===") # Example maintenance record maintenance_data = { @@ -41,17 +45,17 @@ def demo_insert_maintenance_record(): try: # Insert the record (commented out to prevent actual DB modifications in demo) # rig_maintenance.RigMaintenance.insert1(maintenance_data) - print("Would insert maintenance record:") + logger.info("Would insert maintenance record:") for key, value in maintenance_data.items(): - print(f" {key}: {value}") - print("✅ Maintenance record ready for insertion") + logger.info(" %s: %s", key, value) + logger.info("Maintenance record ready for insertion") except Exception as e: - print(f"❌ Error inserting maintenance record: {e}") + logger.error("Error inserting maintenance record: %s", e) def demo_query_maintenance_history(): """Demo: Query maintenance history for a rig.""" - print("\n=== Demo: Querying Maintenance History ===") + logger.info("=== Demo: Querying Maintenance History ===") rig_name = "Bezos2" @@ -61,59 +65,58 @@ def demo_query_maintenance_history(): # as_dict=True, order_by='maintenance_date DESC' # ) - print(f"Would query maintenance history for rig: {rig_name}") - print("Query would return records ordered by most recent first") - print("✅ Query structure is correct") + logger.info("Would query maintenance history for rig: %s", rig_name) + logger.info("Query would return records ordered by most recent first") + logger.info("Query structure is correct") except Exception as e: - print(f"❌ Error querying maintenance history: {e}") + logger.error("Error querying maintenance history: %s", e) def demo_check_overdue_maintenance(): """Demo: Check for overdue maintenance.""" - print("\n=== Demo: Checking Overdue Maintenance ===") + logger.info("=== Demo: Checking Overdue Maintenance ===") - print("This demo shows the logic used in check_rig_maintenance.py") + logger.info("This demo shows the logic used in check_rig_maintenance.py") # Show the maintenance types and their intervals - print("\nMaintenance intervals defined:") + logger.info("Maintenance intervals defined:") for mtype in rig_maintenance.MaintenanceType.contents: maintenance_type, description, interval_days = mtype - print(f" • {maintenance_type}: every {interval_days} days") + logger.info(" %s: every %s days", maintenance_type, interval_days) # Show the logic for checking overdue maintenance current_date = date.today() - print(f"\nCurrent date: {current_date}") - print("For each rig and maintenance type combination:") - print("1. Find most recent maintenance record") - print("2. Calculate days since last maintenance") - print("3. Compare against required interval") - print("4. Flag as overdue if interval exceeded") - print("✅ Overdue checking logic is sound") + logger.info("Current date: %s", current_date) + logger.info("For each rig and maintenance type combination:") + logger.info("1. Find most recent maintenance record") + logger.info("2. Calculate days since last maintenance") + logger.info("3. Compare against required interval") + logger.info("4. Flag as overdue if interval exceeded") + logger.info("Overdue checking logic is sound") def main(): """Main demo function.""" - print("Rig Maintenance Schema Demo") - print("=" * 50) - - print("This demo shows how to use the new rig maintenance schema.") - print("The schema includes:") - print("- MaintenanceType: Lookup table with maintenance types and intervals") - print("- RigMaintenance: Records of maintenance performed on rigs") - print() + logger.info("Rig Maintenance Schema Demo") + logger.info("=" * 50) + logger.info("This demo shows how to use the new rig maintenance schema.") + logger.info("The schema includes:") + logger.info("- MaintenanceType: Lookup table with maintenance types and intervals") + logger.info("- RigMaintenance: Records of maintenance performed on rigs") # Run demo functions demo_insert_maintenance_record() demo_query_maintenance_history() demo_check_overdue_maintenance() - print("\n" + "=" * 50) - print("Demo complete! To use the schema:") - print("1. Configure DataJoint connection") - print("2. Create the schema tables in your database") - print("3. Use the check_rig_maintenance.py script to monitor maintenance") - print("4. Insert maintenance records as work is completed") + logger.info("=" * 50) + logger.info("Demo complete! To use the schema:") + logger.info("1. Configure DataJoint connection") + logger.info("2. Create the schema tables in your database") + logger.info("3. Use the check_rig_maintenance.py script to monitor maintenance") + logger.info("4. Insert maintenance records as work is completed") if __name__ == "__main__": + setup_logging() main() diff --git a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py index fedf67fa..47dd628a 100644 --- a/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py +++ b/u19_pipeline/alert_system/live_monitor_alert/live_monitor_alert.py @@ -6,6 +6,9 @@ import pandas as pd import u19_pipeline.utils.slack_utils as su +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) MINUTES_ALERT = 20 SECONDS_ALERT = MINUTES_ALERT * 60 @@ -105,7 +108,7 @@ def main_live_monitor_alert(): sessions = sessions.drop(columns=["session_location"]) sessions = sessions.reset_index(drop=True) - print("Last session started on rig\n", sessions) + logger.info("Last session started on rig\n%s", sessions) # Only analyze sessions that have not been reported query_reported = {} @@ -196,7 +199,7 @@ def main_live_monitor_alert(): live_stats["alert_vio"] = live_stats["alert_vio"] & (pd.isna(live_stats["last_non_violation_trial"])) live_stats["alert_vio"] = live_stats["alert_vio"] & (~pd.isna(live_stats["last_violation_trial"])) - print(live_stats.T) + logger.debug("live_stats %s", live_stats.T) live_stats = live_stats.loc[ (live_stats["alert_nvio"]) | (live_stats["alert_vio"]), diff --git a/u19_pipeline/alert_system/main_alert_system.py b/u19_pipeline/alert_system/main_alert_system.py index d3892abe..f3f12883 100644 --- a/u19_pipeline/alert_system/main_alert_system.py +++ b/u19_pipeline/alert_system/main_alert_system.py @@ -7,6 +7,9 @@ import u19_pipeline.alert_system.custom_alerts as ca import u19_pipeline.lab as lab import u19_pipeline.utils.slack_utils as su +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) # Slack Configuration dictionary slack_configuration_dictionary = {"slack_notification_channel": ["custom_alerts"]} @@ -18,7 +21,7 @@ def main_alert_system(): all_alert_submodules = pkgutil.iter_modules(ca.__path__) for this_alert_submodule in all_alert_submodules: - print("executing " + this_alert_submodule.name + " alert code") + logger.info("executing %s alert code", this_alert_submodule.name) my_alert_module = importlib.import_module( "u19_pipeline.alert_system.custom_alerts." + this_alert_submodule.name diff --git a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py index e2a530d7..0aad616c 100755 --- a/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py +++ b/u19_pipeline/alert_system/rig_maintenance/check_rig_maintenance.py @@ -17,6 +17,7 @@ import logging import os import sys +from u19_pipeline.utils.logging_config import get_logger from datetime import date, datetime, timedelta from pathlib import Path @@ -27,11 +28,13 @@ from u19_pipeline import lab, rig_maintenance, scheduler from u19_pipeline.utils import slack_utils as su except Exception as e: # pragma: no cover - only happens when package not available - print(f"Error importing modules: {e}") - print("Make sure u19_pipeline is properly installed and configured.") + # Cannot use logger here since get_logger may not be importable + import sys as _sys + _sys.stderr.write(f"Error importing modules: {e}\n") + _sys.stderr.write("Make sure u19_pipeline is properly installed and configured.\n") sys.exit(1) -logger = logging.getLogger("rig_maintenance") +logger = get_logger(__name__) def setup_logging(): @@ -540,7 +543,7 @@ def main(): records = pd.DataFrame(overdue_items) records.columns = [" ".join(w.capitalize() for w in col.split("_")) for col in records.columns] - print(records) + logger.info("records\n%s", records) records = records[["Location", "Maintenance Type", "Status"]] diff --git a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py index a64fffbf..ce610f68 100644 --- a/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py +++ b/u19_pipeline/alert_system/schedule_check_alert/schedule_check_alert.py @@ -6,6 +6,9 @@ import u19_pipeline.lab as lab import u19_pipeline.utils.slack_utils as su +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) # Slack Configuration dictionary slack_configuration_dictionary = { @@ -53,7 +56,7 @@ def main_schedule_check_alert(): on=["location"], suffixes=["_today", "_tomorrow"], ) - print(schedule_comp) + logger.debug("schedule_comp %s", schedule_comp) schedule_comp["diff_subjects"] = schedule_comp["#subj_tomorrow"] - schedule_comp["#subj_today"] schedule_comp["rig_less_subjects"] = schedule_comp["diff_subjects"] < -2 @@ -62,7 +65,7 @@ def main_schedule_check_alert(): subjects_tomorrow = schedule_comp["#subj_tomorrow"].sum() total_rigs_less_subjects = schedule_comp["rig_less_subjects"].sum() - print(schedule_comp) + logger.debug("schedule_comp %s", schedule_comp) if subjects_tomorrow / subjects_today < 0.7: alert = 1 diff --git a/u19_pipeline/alert_system/tech_alert/tech_alert.py b/u19_pipeline/alert_system/tech_alert/tech_alert.py index f20e3e82..97d82a8f 100644 --- a/u19_pipeline/alert_system/tech_alert/tech_alert.py +++ b/u19_pipeline/alert_system/tech_alert/tech_alert.py @@ -8,6 +8,9 @@ from icalevents.icalevents import events from u19_pipeline.utils import slack_utils as su +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) def tech_schedule(): @@ -260,7 +263,7 @@ def fetch_and_parse_icalevents(weburl: str): try: vr_events = events(url=weburl, start=start_date, end=end_date) except Exception as e: - print(f"Error fetching events from WhenIWork: {e}") + logger.error("Error fetching events from WhenIWork: %s", e) return [] # Define mapping of substrings to event types and their corresponding colors @@ -325,7 +328,7 @@ def fetch_and_parse_icalevents(weburl: str): if __name__ == "__main__": - print("Command-line arguments:", sys.argv) + logger.info("Command-line arguments: %s", sys.argv) if len(sys.argv) < 2: main_loop() else: diff --git a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py index c0ceb80f..fd0003ba 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py +++ b/u19_pipeline/alert_system/water_weigh_alert/create_weighing_gui_ss.py @@ -17,6 +17,9 @@ from u19_pipeline.utils.subject_metadata import ( fetch_slack_handles_for_lab_managers_by_subject, ) +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) slack_configuration_dictionary = { "slack_notification_channel": ["subject_health"], @@ -220,7 +223,7 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: (lab.LabManager().proj("lab", "lab_manager") & 'lab = "technician"') * lab.User().proj(lab_manager="user_id", manager_slack="slack") ).fetch("manager_slack", as_dict=True) - print(technician_manager_slack) + logger.debug("technician_manager_slack %s", technician_manager_slack) technician_manager_slack = [item["manager_slack"] for item in technician_manager_slack] people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch(format="frame") @@ -332,7 +335,7 @@ def slack_alert_message_format_weight_water( individual_alert: bool = False, ): - print(missing_transport) + logger.debug("missing_transport %s", missing_transport) temp_missing_transport = missing_transport.copy().reset_index() notifiable_subjects = list( set( @@ -551,7 +554,7 @@ def split_text_to_fit(text: str, max_payload_size: int, text_type: str = "mrkdwn if _estimate_section_size(line, text_type) > max_payload_size: # Single line too big - this shouldn't happen often, but handle it # by truncating (better than crashing) - print("Warning: Single line too large, may be truncated") + logger.warning("Warning: Single line too large, may be truncated") else: # Even first line doesn't fit - add it anyway (will be handled later) current_lines = [line] @@ -659,7 +662,7 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int): su.send_slack_notification(webhook, {"blocks": [small_block]}) time.sleep(0.5) except HTTPError: - print("Failed to send block even after splitting") + logger.error("Failed to send block even after splitting") else: raise else: @@ -670,7 +673,7 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int): su.send_slack_notification(webhook, {"blocks": [small_block]}) time.sleep(0.5) except HTTPError: - print("Failed to send small block") + logger.error("Failed to send small block") def main_water_weigh_alert(): diff --git a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py index c0ceb80f..fd0003ba 100644 --- a/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py +++ b/u19_pipeline/alert_system/water_weigh_alert/water_weigh_alert.py @@ -17,6 +17,9 @@ from u19_pipeline.utils.subject_metadata import ( fetch_slack_handles_for_lab_managers_by_subject, ) +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) slack_configuration_dictionary = { "slack_notification_channel": ["subject_health"], @@ -220,7 +223,7 @@ def get_responsible_user_slack(subject_data: pd.DataFrame) -> pd.DataFrame: (lab.LabManager().proj("lab", "lab_manager") & 'lab = "technician"') * lab.User().proj(lab_manager="user_id", manager_slack="slack") ).fetch("manager_slack", as_dict=True) - print(technician_manager_slack) + logger.debug("technician_manager_slack %s", technician_manager_slack) technician_manager_slack = [item["manager_slack"] for item in technician_manager_slack] people_and_their_managers: pd.DataFrame = people_and_their_managers_query.fetch(format="frame") @@ -332,7 +335,7 @@ def slack_alert_message_format_weight_water( individual_alert: bool = False, ): - print(missing_transport) + logger.debug("missing_transport %s", missing_transport) temp_missing_transport = missing_transport.copy().reset_index() notifiable_subjects = list( set( @@ -551,7 +554,7 @@ def split_text_to_fit(text: str, max_payload_size: int, text_type: str = "mrkdwn if _estimate_section_size(line, text_type) > max_payload_size: # Single line too big - this shouldn't happen often, but handle it # by truncating (better than crashing) - print("Warning: Single line too large, may be truncated") + logger.warning("Warning: Single line too large, may be truncated") else: # Even first line doesn't fit - add it anyway (will be handled later) current_lines = [line] @@ -659,7 +662,7 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int): su.send_slack_notification(webhook, {"blocks": [small_block]}) time.sleep(0.5) except HTTPError: - print("Failed to send block even after splitting") + logger.error("Failed to send block even after splitting") else: raise else: @@ -670,7 +673,7 @@ def _send_blocks_individually(webhook: str, blocks: list[dict], max_size: int): su.send_slack_notification(webhook, {"blocks": [small_block]}) time.sleep(0.5) except HTTPError: - print("Failed to send small block") + logger.error("Failed to send small block") def main_water_weigh_alert(): diff --git a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py index 13813f97..f58888ee 100644 --- a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py +++ b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py @@ -7,6 +7,9 @@ import datajoint as dj import u19_pipeline.automatic_job.params_config as config +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) # Functions to transfer files (globus, scp, smbclient) @@ -124,9 +127,8 @@ def get_cluster_vars(cluster): def scp_file_transfer(source, dest): - print("scp", source, dest) - - print(["scp", "-i", public_key_location, source, dest]) + logger.debug("scp %s %s", source, dest) + logger.debug("scp command %s", ["scp", "-i", public_key_location, source, dest]) p = subprocess.Popen(["scp", "-i", public_key_location, source, dest]) transfer_status = p.wait() @@ -135,13 +137,13 @@ def scp_file_transfer(source, dest): def scp_file_transfer_big_files(source, dest, source_filepath, dest_filepath): - print("scp", source, dest) + logger.debug("scp big files %s %s", source, dest) # IMPLEMENT HERE def cp_file_transfer(source, dest): - print("cp", source, dest) + logger.debug("cp %s %s", source, dest) p = subprocess.Popen(["cp", source, dest]) transfer_status = p.wait() return transfer_status @@ -163,14 +165,14 @@ def request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, des "--format", "json", ] - print("**********************************") - print(globus_command) - print("**********************************") + logger.debug("**********************************") + logger.debug("globus_command %s", globus_command) + logger.debug("**********************************") p = subprocess.run(globus_command, capture_output=True) - print(p) + logger.debug("globus run result %s", p) transfer_request = {} - print("p.stderr", p.stderr) - print("p.stdout", p.stdout) + logger.debug("p.stderr %s", p.stderr) + logger.debug("p.stdout %s", p.stdout) if len(p.stderr) == 0: try: @@ -178,7 +180,7 @@ def request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, des transfer_request["status"] = config.system_process["SUCCESS"] transfer_request["task_id"] = dict_output["task_id"] except Exception: - print("stdout is not a valid json, probably an error") + logger.warning("stdout is not a valid json, probably an error") transfer_request["status"] = config.system_process["ERROR"] transfer_request["error_info"] = p.stdout.decode("UTF-8") # dict_output = translate_globus_output(p.stdout) @@ -296,14 +298,10 @@ def get_error_log_str(recording_process_id): "", ) - print("error_log_data xxxxxxxxxx") - print("type(error_log_data)", type(error_log_data)) - print("len(error_log_data)", len(error_log_data)) - # if len(error_log_data) > 400: - # print(error_log_data[-400:]) - # else: - # print(error_log_data) - print("error_log_data xxxxxxxxxx") + logger.debug("error_log_data xxxxxxxxxx") + logger.debug("type(error_log_data) %s", type(error_log_data)) + logger.debug("len(error_log_data) %s", len(error_log_data)) + logger.debug("error_log_data xxxxxxxxxx") return error_log_data diff --git a/u19_pipeline/automatic_job/create_missing_lfps_script.py b/u19_pipeline/automatic_job/create_missing_lfps_script.py index 16e22f93..887b19c1 100644 --- a/u19_pipeline/automatic_job/create_missing_lfps_script.py +++ b/u19_pipeline/automatic_job/create_missing_lfps_script.py @@ -5,6 +5,9 @@ import u19_pipeline.automatic_job.params_config as config from u19_pipeline import ephys_pipeline, recording, recording_process +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) def main(recording_id): @@ -36,5 +39,5 @@ def main(recording_id): try_find_conf_file() args = sys.argv[1:] - print(args) + logger.debug("args %s", args) main(args[0]) diff --git a/u19_pipeline/automatic_job/ephys_element_ingest.py b/u19_pipeline/automatic_job/ephys_element_ingest.py index 18bb05d7..cd39a825 100644 --- a/u19_pipeline/automatic_job/ephys_element_ingest.py +++ b/u19_pipeline/automatic_job/ephys_element_ingest.py @@ -11,6 +11,9 @@ get_session_directory, probe_element, ) +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) """ The ingestion routine for imaging element includes: @@ -40,15 +43,15 @@ def process_session(sess_key): sess_dir = pathlib.Path(find_full_path(get_ephys_root_data_dir(), get_session_directory(sess_key))) ephys_meta_filepaths = list(sess_dir.rglob("*.ap.meta")) - print("ephys_meta_filepaths", ephys_meta_filepaths) + logger.debug("ephys_meta_filepaths %s", ephys_meta_filepaths) if not len(ephys_meta_filepaths): - print(f"No SpikeGLX data found for session:{sess_key} - at {sess_dir}") + logger.warning("No SpikeGLX data found for session:%s - at %s", sess_key, sess_dir) return probe_list, probe_insertion_list = [], [] for meta_filepath in ephys_meta_filepaths: - print("meta_filepath", meta_filepath) + logger.debug("meta_filepath %s", meta_filepath) spikeglx_meta = spikeglx.SpikeGLXMeta(meta_filepath) probe_key = { @@ -70,7 +73,7 @@ def process_session(sess_key): } ) - print(f"{probe_list =}") + logger.debug("probe_list %s", probe_list) probe_element.Probe.insert(probe_list, skip_duplicates=True) ephys_element.ProbeInsertion.insert(probe_insertion_list, skip_duplicates=True) diff --git a/u19_pipeline/automatic_job/imaging_element.py b/u19_pipeline/automatic_job/imaging_element.py index 28599162..9a7a1ea4 100644 --- a/u19_pipeline/automatic_job/imaging_element.py +++ b/u19_pipeline/automatic_job/imaging_element.py @@ -14,6 +14,9 @@ imaging_element, scan_element, ) +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) # subject_fullname = 'koay_K65' # session_date = '2018-02-02' @@ -151,31 +154,31 @@ processing_method = preprocess_params["processing_method"] task_mode = preprocess_params["task_mode"] -print("got processing_method", processing_method) -print("got task_mode", task_mode) +logger.info("got processing_method %s", processing_method) +logger.info("got task_mode %s", task_mode) -print("got paramset_idx", paramset_idx) +logger.info("got paramset_idx %s", paramset_idx) # Get directories for kov scan_filepaths = get_scan_image_files(fov_key) scan_filepaths = scan_filepaths[:1] -print(scan_filepaths) +logger.debug("scan_filepaths %s", scan_filepaths) if rec_process_key not in scan_element.Scan(): try: # TODO: Can use tiffile function to loads - print("LOADED Scan using Scanreader") + logger.info("LOADED Scan using Scanreader") loaded_scan = scanreader.read_scan(scan_filepaths) header = parse_scanimage_header(loaded_scan) # scanner = header['SI_imagingSystem'].strip('\'') #TODO: If using tiffile, hardcode it to `mesoscope` except Exception: - print("LOADED Scan using Tifffile") + logger.info("LOADED Scan using Tifffile") scan_filepaths = scan_filepaths # TODO load all TIFF files from session possibly using TIFFSequence loaded_scan = tifffile.imread(scan_filepaths) # scanner = 'mesoscope' except BaseException: # TODO: Use except instead of else) - print("ScanImage loading error") # TODO: Modify the error message + logger.error("ScanImage loading error") # TODO: Modify the error message # Equipment.insert1({'scanner': scanner}, skip_duplicates=True) scan_element.Scan.insert1( @@ -215,7 +218,7 @@ # Results from this specific recording_process_id already triggered if task_mode == "trigger" and recording_process_id_folder_found: - print("Overwritting process output from original processing folder", output_dir) + logger.warning("Overwritting process output from original processing folder %s", output_dir) # If trigger and not recording_process_id_folder_found make it if task_mode == "trigger": @@ -224,15 +227,15 @@ output_dir.mkdir(parents=True, exist_ok=True) -print("RELATIVE OUTPUT DIR") -print(relative_output_dir) -print(output_dir) +logger.info("RELATIVE OUTPUT DIR") +logger.info("relative_output_dir %s", relative_output_dir) +logger.info("output_dir %s", output_dir) # output_dir.mkdir(parents=True,exist_ok=True) # Check if found output dir is correct if task_mode == "load": if processing_method == "suite2p": - print("SUITE2P METHOD SELECTED") + logger.info("SUITE2P METHOD SELECTED") # output_dir = get_suite2p_dir(scan_key) p = pathlib.Path(output_dir).glob("**/*") plane_filepaths = [x for x in p if x.is_dir()] @@ -277,8 +280,8 @@ process_params = recording.ProcessParamSet().get_process_params(process_params_key) # print('process_params_info', process_params_info) - print("process_params", process_params) - print("description", process_params_info[0]["process_paramset_desc"]) + logger.info("process_params %s", process_params) + logger.info("description %s", process_params_info[0]["process_paramset_desc"]) # Insert in imaging element equivalent ProcessParamSet imaging_element.ProcessingParamSet.insert_new_params( diff --git a/u19_pipeline/automatic_job/parameter_file_creator.py b/u19_pipeline/automatic_job/parameter_file_creator.py index 6551b54d..4e7e5df8 100644 --- a/u19_pipeline/automatic_job/parameter_file_creator.py +++ b/u19_pipeline/automatic_job/parameter_file_creator.py @@ -8,6 +8,9 @@ import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft import u19_pipeline.automatic_job.params_config as config from u19_pipeline.utils.file_utils import write_file +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) # Functions to create parameter files and send them @@ -80,7 +83,7 @@ def transfer_parameter_file(recording_process_id, default_param_filename, cluste params_file_cluster_path = str(pathlib.Path(cluster_param_dir, param_filename)) param_file_full_path = user_host + ":" + params_file_cluster_path - print("transfer_parameter_file", params_file_local_path, param_file_full_path) + logger.debug("transfer_parameter_file %s %s", params_file_local_path, param_file_full_path) status = ft.scp_file_transfer(params_file_local_path, param_file_full_path) diff --git a/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py b/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py index 68df166b..d7dbc112 100644 --- a/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py +++ b/u19_pipeline/automatic_job/populate_missing_syncbehavior_ephys.py @@ -12,6 +12,9 @@ import u19_pipeline.ephys_pipeline as ep import u19_pipeline.recording as recording +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) def get_rec_key_dict(recording_id): @@ -48,4 +51,4 @@ def get_rec_key_dict(recording_id): try: ep.BehaviorSync.populate(not_sync_recs2.loc[i, "rec_key"]) except Exception as e: - print(e) + logger.exception("Error populating BehaviorSync: %s", e) diff --git a/u19_pipeline/automatic_job/pupillometry_handler.py b/u19_pipeline/automatic_job/pupillometry_handler.py index 63cd2395..c1cb7f9a 100644 --- a/u19_pipeline/automatic_job/pupillometry_handler.py +++ b/u19_pipeline/automatic_job/pupillometry_handler.py @@ -19,6 +19,9 @@ import u19_pipeline.pupillometry as pupillometry import u19_pipeline.utils.slack_utils as slack_utils from u19_pipeline.utils.file_utils import write_file +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) def pupillometry_exception_handler(func): @@ -27,7 +30,7 @@ def inner_function(*args, **kwargs): argout = func(*args, **kwargs) return argout except Exception as e: - print("Exception HERE ................") + logger.exception("Exception HERE ................") update_value_dict = copy.deepcopy(config.default_update_value_dict) update_value_dict["error_info"]["error_message"] = str(e) @@ -35,8 +38,8 @@ def inner_function(*args, **kwargs): traceback.format_exception(type(e), value=e, tb=e.__traceback__) ) - print(update_value_dict["error_info"]["error_message"]) - print(update_value_dict["error_info"]["error_exception"]) + logger.error("error_message %s", update_value_dict["error_info"]["error_message"]) + logger.error("error_exception %s", update_value_dict["error_info"]["error_exception"]) return (config.RECORDING_STATUS_ERROR_ID, update_value_dict) @@ -115,14 +118,13 @@ def generate_slurm_file(video_dir): slurm_dict = copy.deepcopy(PupillometryProcessingHandler.slurm_dict_pupillometry_spock) label_rec_process = "job_id_" + str(video_dir.stem) slurm_dict["job-name"] = label_rec_process - print("PupillometryProcessingHandler.spock_log_dir") - print(PupillometryProcessingHandler.spock_log_dir) - print("label_rec_process", label_rec_process) - print("video_dir", video_dir) + logger.debug("PupillometryProcessingHandler.spock_log_dir %s", PupillometryProcessingHandler.spock_log_dir) + logger.debug("label_rec_process %s", label_rec_process) + logger.debug("video_dir %s", video_dir) slurm_dict["output"] = PupillometryProcessingHandler.spock_log_dir + label_rec_process + ".log" slurm_dict["error"] = PupillometryProcessingHandler.spock_error_dir + label_rec_process + ".log" - print("slurm_dict", slurm_dict) + logger.debug("slurm_dict %s", slurm_dict) slurm_text = PupillometryProcessingHandler.generate_slurm_pupillometry(slurm_dict) @@ -133,9 +135,8 @@ def generate_slurm_file(video_dir): ) ) - print(slurm_file_local_path) - - print("slurm text", slurm_text) + logger.debug("slurm_file_local_path %s", slurm_file_local_path) + logger.debug("slurm text %s", slurm_text) write_file(slurm_file_local_path, slurm_text) @@ -148,8 +149,8 @@ def generate_slurm_file(video_dir): slurm_file_local_path, slurm_destination ) - print(status) - print(slurm_destination) + logger.debug("status %s", status) + logger.debug("slurm_destination %s", slurm_destination) return status, slurm_destination @@ -184,20 +185,20 @@ def queue_pupillometry_slurm_file( slurm_location, ] - print(command) + logger.debug("command %s", command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # p = os.popen(command_new).read() p.wait() stdout, stderr = p.communicate() - print(p.returncode) - print(stderr) - print(stdout) + logger.debug("returncode %s", p.returncode) + logger.debug("stderr %s", stderr) + logger.debug("stdout %s", stdout) if p.returncode == 0: error_message = "" batch_job_sentence = stdout.decode("UTF-8") - print("batch_job_sentence", batch_job_sentence) + logger.debug("batch_job_sentence %s", batch_job_sentence) id_slurm_job = batch_job_sentence.replace("Submitted batch job ", "") id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job) else: @@ -211,9 +212,8 @@ def transfer_pupillometry_slurm_file(slurm_file_local_path, slurm_destination): Create scp command from cluster directories and local slurm file """ - print("cp", slurm_file_local_path, slurm_destination) - - print(["cp", slurm_file_local_path, slurm_destination]) + logger.debug("cp %s %s", slurm_file_local_path, slurm_destination) + logger.debug("cp command %s", ["cp", slurm_file_local_path, slurm_destination]) p = subprocess.Popen(["cp", slurm_file_local_path, slurm_destination]) transfer_status = p.wait() @@ -282,7 +282,7 @@ def check_pupillometry_sessions_queue(): acquisition.SessionVideo * pupillometry.PupillometrySessionModelData & "pupillometry_job_id is NULL" ).fetch(as_dict=True) - print(sessions_missing_process) + logger.debug("sessions_missing_process %s", sessions_missing_process) for pupillometry_2_process in sessions_missing_process: # If error, job id = -1 @@ -307,7 +307,7 @@ def check_pupillometry_sessions_queue(): models_dir = models_dir[0] model_path = pathlib.Path(models_dir, model_key["model_path"]) - print("model path", model_path) + logger.debug("model path %s", model_path) # Get video location pupillometry_dir = dj.config.get("custom", {}).get("pupillometry_root_data_dir", None) @@ -316,7 +316,7 @@ def check_pupillometry_sessions_queue(): pupillometry_raw_dir = pupillometry_dir[0] videoPath = pathlib.Path(pupillometry_raw_dir, pupillometry_2_process["remote_path_video_file"]) - print("videoPath", videoPath) + logger.debug("videoPath %s", videoPath) # Create output location pupillometry_processed_dir = pupillometry_dir[1] @@ -329,7 +329,7 @@ def check_pupillometry_sessions_queue(): # Generate slurm file and transfer it to spock status, slurm_filepath = PupillometryProcessingHandler.generate_slurm_file(videoPath) - print("slurm_filepath", slurm_filepath) + logger.debug("slurm_filepath %s", slurm_filepath) # Error handling (generating slurm file) if status != config.system_process["SUCCESS"]: @@ -404,7 +404,7 @@ def check_processed_pupillometry_sessions(): "model_id", ) } - print("key_update1", key_update) + logger.debug("key_update1 %s", key_update) status_update, message = slurmlib.check_slurm_job( "u19prod", @@ -440,7 +440,7 @@ def check_processed_pupillometry_sessions(): pathlib.Path(session_check["remote_path_video_file"]).parent, ) - print(output_dir) + logger.debug("output_dir %s", output_dir) # Find h5 files h5_files = glob.glob(str(output_dir) + "/*.h5") @@ -475,7 +475,7 @@ def check_processed_pupillometry_sessions(): continue key_update["pupil_diameter"] = pupil_data - print("key_update", key_update) + logger.debug("key_update %s", key_update) pupillometry.PupillometrySessionModelData.update1(key_update) slack_utils.send_slack_pupillometry_update_notification( config.slack_webhooks_dict["automation_pipeline_update_notification"], @@ -494,7 +494,6 @@ def check_processed_pupillometry_sessions(): args = sys.argv[1:] args[1] = args[1] + "/config.yaml" - print(args) - # + logger.debug("args %s", args) PupillometryProcessingHandler.analyze_videos_pupillometry(args[1], args[0], args[2]) diff --git a/u19_pipeline/automatic_job/pupillometry_main.py b/u19_pipeline/automatic_job/pupillometry_main.py index 5bd37607..b931f0da 100644 --- a/u19_pipeline/automatic_job/pupillometry_main.py +++ b/u19_pipeline/automatic_job/pupillometry_main.py @@ -1,5 +1,9 @@ import sys +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) + if __name__ == "__main__": import time @@ -11,9 +15,9 @@ import u19_pipeline.automatic_job.pupillometry_handler as ph args = sys.argv[1:] - print(args) - print(args[0]) - print(args[1]) - print(args[2]) + logger.debug("args %s", args) + logger.debug("args[0] %s", args[0]) + logger.debug("args[1] %s", args[1]) + logger.debug("args[2] %s", args[2]) ph.PupillometryProcessingHandler.analyze_videos_pupillometry(args[0], args[1], args[2]) diff --git a/u19_pipeline/automatic_job/recording_handler.py b/u19_pipeline/automatic_job/recording_handler.py index 14e93dc3..291a0953 100644 --- a/u19_pipeline/automatic_job/recording_handler.py +++ b/u19_pipeline/automatic_job/recording_handler.py @@ -19,6 +19,9 @@ recording_process, ) from u19_pipeline.automatic_job import ephys_element_ingest +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) def exception_handler(func): @@ -31,7 +34,7 @@ def inner_function(*args, **kwargs): argout = func(*args, **kwargs) return argout except Exception as e: - print("Exception HERE ................") + logger.exception("Exception HERE ................") update_value_dict = copy.deepcopy(config.default_update_value_dict) update_value_dict["error_info"]["error_message"] = str(e) update_value_dict["error_info"]["error_exception"] = "".join( @@ -64,7 +67,7 @@ def pipeline_handler_main(): config.recording_status_df["Value"] == current_status + 1, : ].squeeze() - print("function to apply:", next_status_series["ProcessFunction"]) + logger.info("function to apply: %s", next_status_series["ProcessFunction"]) # Get processing function function_status_process = getattr(RecordingHandler, next_status_series["ProcessFunction"]) @@ -193,7 +196,7 @@ def local_transfer_check(rec_series): status_update = config.status_update_idx["ERROR_STATUS"] update_value_dict["error_info"]["error_message"] = "Return code scp not = 0" - print("is_finished", is_finished, "status_update", status_update) + logger.debug("is_finished %s status_update %s", is_finished, status_update) return (status_update, update_value_dict) @@ -234,13 +237,13 @@ def get_active_recordings(): status_query = "status_recording_id > " + str(config.recording_status_df["Value"].min()) status_query += " and status_recording_id < " + str(config.recording_status_df["Value"].max()) - print(status_query) + logger.debug("status_query %s", status_query) recordings_active = ( recording.Recording * lab.Location.proj("ip_address", "system_user", "acquisition_type") & status_query ) - print(recordings_active) + logger.debug("recordings_active %s", recordings_active) df_recordings = pd.DataFrame(recordings_active.fetch(as_dict=True)) @@ -261,20 +264,20 @@ def update_status_pipeline(recording_key_dict, status, update_field=None, update update_value (str|int): field value to be inserted on in task_field """ - print("recording_key_dict", recording_key_dict) - print("status", status) - print("update_field", update_field) - print("update_value", update_value) + logger.debug("recording_key_dict %s", recording_key_dict) + logger.debug("status %s", status) + logger.debug("update_field %s", update_field) + logger.debug("update_value %s", update_value) if update_field is not None: update_task_id_dict = recording_key_dict.copy() update_task_id_dict[update_field] = update_value - print("update_task_id_dict", update_task_id_dict) + logger.debug("update_task_id_dict %s", update_task_id_dict) recording.Recording.update1(update_task_id_dict) update_status_dict = recording_key_dict.copy() update_status_dict["status_recording_id"] = status - print("update_status_dict", update_status_dict) + logger.debug("update_status_dict %s", update_status_dict) recording.Recording.update1(update_status_dict) @staticmethod @@ -288,7 +291,7 @@ def update_recording_log(recording_id, current_status, next_status, error_info_d now = datetime.now() date_time = now.strftime("%Y-%m-%d %H:%M:%S") - print("error_info_dict", error_info_dict) + logger.debug("error_info_dict %s", error_info_dict) key = {} key["recording_id"] = recording_id @@ -334,8 +337,8 @@ def electrophysiology_preingestion(rec_series): "KEY", as_dict=True ) - print("before BehaviorSync") - print("rec_series['query_key']", rec_series["query_key"]) + logger.debug("before BehaviorSync") + logger.debug("rec_series['query_key'] %s", rec_series["query_key"]) ephys_pipeline.BehaviorSync.populate(rec_series["query_key"]) @@ -422,8 +425,7 @@ def imaging_preingestion(rec_series): status_update = config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) - print("rec_series['query_key']") - print(rec_series["query_key"]) + logger.debug("rec_series['query_key'] %s", rec_series["query_key"]) # Populate ImagingPipelineSession and call matlab script that handles TiffSplits imaging_pipeline.ImagingPipelineSession.populate(rec_series["query_key"]) diff --git a/u19_pipeline/automatic_job/recording_process_handler.py b/u19_pipeline/automatic_job/recording_process_handler.py index 0f8b9270..f83100ad 100644 --- a/u19_pipeline/automatic_job/recording_process_handler.py +++ b/u19_pipeline/automatic_job/recording_process_handler.py @@ -23,6 +23,9 @@ utility, ) from u19_pipeline.automatic_job import recording_handler +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) from u19_pipeline.utility import is_this_spock @@ -62,8 +65,7 @@ def pipeline_handler_main(): # Get dictionary of record process key = rec_process_series["query_key"] - print("rec_process_series") - print(rec_process_series) + logger.debug("rec_process_series %s", rec_process_series) if status == config.status_update_idx["NEXT_STATUS"]: # Get values to update @@ -71,10 +73,10 @@ def pipeline_handler_main(): value_update = update_dict["value_update"] field_update = next_status_series["UpdateField"] - print("key to update", key) - print("old status", current_status, "new status", next_status) - print("value_update", value_update, "field_update", field_update) - print("function executed:", next_status_series["ProcessFunction"]) + logger.info("key to update %s", key) + logger.info("old status %s new status %s", current_status, next_status) + logger.info("value_update %s field_update %s", value_update, field_update) + logger.info("function executed: %s", next_status_series["ProcessFunction"]) RecProcessHandler.update_status_pipeline(key, next_status, field_update, value_update) @@ -92,7 +94,7 @@ def pipeline_handler_main(): # Crop error messages to fit in DB if len(update_dict["error_info"]["error_message"]) > 255: - print("Cropping error message") + logger.warning("Cropping error message") update_dict["error_info"]["error_message"] = update_dict["error_info"]["error_message"][ -255: ] @@ -101,7 +103,7 @@ def pipeline_handler_main(): isinstance(update_dict["error_info"]["error_exception"], str) and len(update_dict["error_info"]["error_exception"]) > 4095 ): - print("Cropping error error_exception") + logger.warning("Cropping error error_exception") update_dict["error_info"]["error_exception"] = update_dict["error_info"]["error_exception"][ -4095: ] @@ -123,7 +125,7 @@ def pipeline_handler_main(): isinstance(update_dict["error_info"]["error_exception"], str) and len(update_dict["error_info"]["error_exception"]) > 1024 ): - print("Cropping error error_exception for slack") + logger.warning("Cropping error error_exception for slack") update_dict["error_info"]["error_exception"] = update_dict["error_info"]["error_exception"][ -1023: ] @@ -147,7 +149,7 @@ def pipeline_handler_main(): except Exception as err: raise (err) - print(traceback.format_exc()) + logger.exception("Unhandled exception: %s", traceback.format_exc()) ## Send notification error, update recording to error time.sleep(2) @@ -258,7 +260,7 @@ def slurm_job_queue(rec_series, status_series): status_update = config.status_update_idx["NO_CHANGE"] update_value_dict = copy.deepcopy(config.default_update_value_dict) - print(rec_series["program_selection_params"]) + logger.debug("program_selection_params %s", rec_series["program_selection_params"]) # Create and transfer parameter file status = paramfilelib.generate_parameter_file( @@ -268,7 +270,7 @@ def slurm_job_queue(rec_series, status_series): rec_series["program_selection_params"], ) - print(status) + logger.debug("status %s", status) # Create and transfer preparameter file if status == config.system_process["SUCCESS"]: @@ -306,7 +308,7 @@ def slurm_job_queue(rec_series, status_series): # Only queue if processing in tiger if rec_series["program_selection_params"]["local_or_cluster"] == "cluster": - print("lets transfer slum file ..............xxxxxxxx............") + logger.info("lets transfer slum file ..............xxxxxxxx............") # Create and transfer slurm file if status == config.system_process["SUCCESS"]: @@ -459,7 +461,7 @@ def get_program_selection_params(modality): # Pack all features in a dictionary this_modality_program_selection_params_dict = this_modality_program_selection_params.to_dict("records") - print(this_modality_program_selection_params_dict) + logger.debug("program_selection_params_dict %s", this_modality_program_selection_params_dict) # Get two columns, (recording_modality & "packed" program_selection_params) this_modality_program_selection_params = ( @@ -467,7 +469,7 @@ def get_program_selection_params(modality): ) this_modality_program_selection_params["program_selection_params"] = this_modality_program_selection_params_dict - print(this_modality_program_selection_params) + logger.debug("program_selection_params %s", this_modality_program_selection_params) return this_modality_program_selection_params @@ -531,7 +533,7 @@ def get_active_process_jobs(active=True): df_process_jobs = df_process_jobs.reset_index(drop=True) - print(df_process_jobs) + logger.debug("df_process_jobs %s", df_process_jobs) return df_process_jobs @@ -678,7 +680,7 @@ def update_job_id_log(job_id, current_status, next_status, error_info_dict): now = datetime.now() date_time = now.strftime("%Y-%m-%d %H:%M:%S") - print("error_info_dict", error_info_dict) + logger.debug("error_info_dict %s", error_info_dict) key = {} key["job_id"] = job_id diff --git a/u19_pipeline/automatic_job/slurm_creator.py b/u19_pipeline/automatic_job/slurm_creator.py index fb303e82..290048fa 100644 --- a/u19_pipeline/automatic_job/slurm_creator.py +++ b/u19_pipeline/automatic_job/slurm_creator.py @@ -7,6 +7,9 @@ import u19_pipeline.automatic_job.params_config as config from u19_pipeline.utility import is_this_spock from u19_pipeline.utils.file_utils import write_file +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) # Functions to create slurm jobs @@ -35,7 +38,7 @@ def generate_slurm_file(job_id, program_selection_params): slurm_dict["output"] = str(pathlib.Path(cluster_vars["log_files_dir"], label_rec_process + ".log")) slurm_dict["error"] = str(pathlib.Path(cluster_vars["error_files_dir"], label_rec_process + ".log")) - print("slurm_dict", slurm_dict) + logger.debug("slurm_dict %s", slurm_dict) if program_selection_params["process_cluster"] == "spock": slurm_text = generate_slurm_spock(slurm_dict) @@ -45,9 +48,9 @@ def generate_slurm_file(job_id, program_selection_params): slurm_file_name = default_slurm_filename slurm_file_local_path = str(pathlib.Path(slurms_filepath, slurm_file_name)) - print(slurm_file_local_path) - print(cluster_vars["slurm_files_dir"]) - print(slurm_file_name) + logger.debug("slurm_file_local_path %s", slurm_file_local_path) + logger.debug("slurm_files_dir %s", cluster_vars["slurm_files_dir"]) + logger.debug("slurm_file_name %s", slurm_file_name) write_file(slurm_file_local_path, slurm_text) @@ -58,9 +61,9 @@ def generate_slurm_file(job_id, program_selection_params): slurm_destination = pathlib.Path(cluster_vars["slurm_files_dir"], slurm_file_name).as_posix() status = transfer_slurm_file(slurm_file_local_path, slurm_destination, cluster_vars) - print(status) - print(slurm_destination) - print(cluster_vars) + logger.debug("status %s", status) + logger.debug("slurm_destination %s", slurm_destination) + logger.debug("cluster_vars %s", cluster_vars) return status, slurm_destination @@ -80,7 +83,7 @@ def queue_slurm_file( # Get all associated variables given the selected processing cluster cluster_vars = ft.get_cluster_vars(program_selection_params["process_cluster"]) - print("queue_slurm_file **********************************") + logger.debug("queue_slurm_file **********************************") processing_repository = program_selection_params["process_repository"] repository_dir = pathlib.Path(cluster_vars[modality + "_process_dir"], processing_repository).as_posix() @@ -106,19 +109,19 @@ def queue_slurm_file( if program_selection_params["process_cluster"] == "spock" and is_this_spock(): command = command[2:] - print(command) + logger.debug("command %s", command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # p = os.popen(command_new).read() p.wait() stdout, stderr = p.communicate() - print(stdout) - print(stderr) + logger.debug("stdout %s", stdout) + logger.debug("stderr %s", stderr) if p.returncode == config.system_process["SUCCESS"]: error_message = "" batch_job_sentence = stdout.decode("UTF-8") - print("batch_job_sentence", batch_job_sentence) + logger.debug("batch_job_sentence %s", batch_job_sentence) id_slurm_job = batch_job_sentence.replace("Submitted batch job ", "") id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job) else: @@ -146,8 +149,8 @@ def check_slurm_job(ssh_user, host, jobid, local_user=False): stdout, stderr = p.communicate() stdout = stdout.decode("UTF-8") - print("p.returncode !!!!!!!!!!!!", p.returncode) - print("config.system_process['SUCCESS']", config.system_process["SUCCESS"]) + logger.debug("p.returncode %s", p.returncode) + logger.debug("config.system_process['SUCCESS'] %s", config.system_process["SUCCESS"]) if p.returncode == config.system_process["SUCCESS"]: state_slurm_job = stdout.split("\n")[2].strip() @@ -155,8 +158,8 @@ def check_slurm_job(ssh_user, host, jobid, local_user=False): state_pipeline = config.slurm_states[state_slurm_job]["pipeline_status"] error_message = config.slurm_states[state_slurm_job]["message"] - print("state_pipeline ....", state_pipeline) - print("error_message", error_message) + logger.debug("state_pipeline %s", state_pipeline) + logger.debug("error_message %s", error_message) else: state_pipeline = config.status_update_idx["ERROR_STATUS"] diff --git a/u19_pipeline/automatic_job/tigress2pni.py b/u19_pipeline/automatic_job/tigress2pni.py index 1db6316b..bbf0acb0 100755 --- a/u19_pipeline/automatic_job/tigress2pni.py +++ b/u19_pipeline/automatic_job/tigress2pni.py @@ -29,6 +29,10 @@ from globus_sdk.exc import GlobusAPIError from globus_sdk.services.transfer.errors import TransferAPIError +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) + # Princeton TIGRESS SOURCE_ENDPOINT = "a9df83d2-42f0-11e6-80cf-22000b1701d1" # SOURCE_ENDPOINT = 'ef3a4e74-e742-11ec-9912-3b4cfda38030' @@ -122,7 +126,7 @@ def check_endpoint_path(transfer_client, endpoint, path): try: transfer_client.operation_ls(endpoint, path=path) except TransferAPIError as tapie: - print(f'Failed to query endpoint "{endpoint}": {tapie.message}') + logger.error('Failed to query endpoint "%s": %s', endpoint, tapie.message) sys.exit(1) @@ -133,9 +137,9 @@ def create_destination_directory(transfer_client, dest_ep, dest_path): except TransferAPIError: try: transfer_client.operation_mkdir(dest_ep, dest_path) - print(f"Created directory: {dest_path}") + logger.info("Created directory: %s", dest_path) except TransferAPIError as tapie: - print(f"Failed to start transfer: {tapie.message}") + logger.error("Failed to start transfer: %s", tapie.message) sys.exit(1) @@ -145,14 +149,14 @@ def main(): try: # if we already have tokens, load and use the tokens = client.load_tokens(requested_scopes=SCOPES) - print(tokens) + logger.debug("tokens %s", tokens) except Exception: pass if not tokens: # if we need to get tokens, start the Native App authentication process # need to specify that we want refresh tokens - print("login in") + logger.info("login in") tokens = client.login(requested_scopes=SCOPES, refresh_tokens=True) with contextlib.suppress(Exception): client.save_tokens(tokens) @@ -165,14 +169,14 @@ def main(): task_data = data["task"] task = transfer.get_task(task_data["task_id"]) if task["status"] not in PREVIOUS_TASK_RUN_CASES: - print("The last transfer status is {}, skipping run...".format(task["status"])) + logger.info("The last transfer status is %s, skipping run...", task["status"]) sys.exit(1) except KeyError: # Ignore if there is no previous task pass if len(sys.argv) < 2: - print(f"Usage: {sys.argv[0]} tigress_dir pni_dir") + logger.error("Usage: %s tigress_dir pni_dir", sys.argv[0]) sys.exit(1) SOURCE_PATH = sys.argv[1] @@ -194,8 +198,9 @@ def main(): task = transfer.submit_transfer(tdata) save_data_to_file(DATA_FILE, "task", task.data) - print( - f"Transfer has been started from\n {SOURCE_ENDPOINT}:{SOURCE_PATH}\nto\n {DESTINATION_ENDPOINT}:{DESTINATION_PATH}" + logger.info( + "Transfer has been started from\n %s:%s\nto\n %s:%s", + SOURCE_ENDPOINT, SOURCE_PATH, DESTINATION_ENDPOINT, DESTINATION_PATH, ) url_string = "https://globus.org/app/transfer?" + six.moves.urllib.parse.urlencode( { @@ -205,7 +210,7 @@ def main(): "destination_path": DESTINATION_PATH, } ) - print(f"Visit the link below to see the changes:\n{url_string}") + logger.info("Visit the link below to see the changes:\n%s", url_string) if __name__ == "__main__": diff --git a/u19_pipeline/copy_table.py b/u19_pipeline/copy_table.py index 9d244316..d6c6779a 100644 --- a/u19_pipeline/copy_table.py +++ b/u19_pipeline/copy_table.py @@ -5,6 +5,9 @@ from u19_pipeline import subject from u19_pipeline.temp import acquisition, behavior, imaging, meso, meso_analysis +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) acquisition_original = dj.create_virtual_module("acquisition_original", "u19_acquisition") @@ -45,7 +48,7 @@ def copy_table(target_schema, src_schema, table_name, **kwargs): try: target_table.insert1(t, skip_duplicates=True, **kwargs) except Exception: - print(f"Error when inserting {t}") + logger.error(f"Error when inserting {t}") traceback.print_exc() @@ -75,7 +78,7 @@ def copy_behavior_tables(): ] for table in BEHAVIOR_TABLES: - print(f"Copying table {table}") + logger.info(f"Copying table {table}") if "." in table: if table == "TowersBlock.Trial": for subj in tqdm((subject.Subject & behavior.TowersBlock).fetch("KEY")): @@ -120,7 +123,7 @@ def copy_imaging_tables(): ] for table in IMAGING_TABLES: - print(f"Copying table {table}...") + logger.info(f"Copying table {table}...") if "." in table: copy_table(imaging, imaging_original, table) @@ -167,7 +170,7 @@ def copy_meso_tables(): ] for table in MESO_TABLES: - print(f"Copying table {table}...") + logger.info(f"Copying table {table}...") if "." in table: copy_table(meso, meso_original, table) @@ -193,7 +196,7 @@ def copy_meso_analysis_tables(): ] for table in MESO_ANALYSIS_TABLES: - print(f"Copying table {table}...") + logger.info(f"Copying table {table}...") if "." in table: copy_table(meso_analysis, meso_analysis_original, table) diff --git a/u19_pipeline/ephys_pipeline.py b/u19_pipeline/ephys_pipeline.py index aa3d6075..7ad5e5e5 100644 --- a/u19_pipeline/ephys_pipeline.py +++ b/u19_pipeline/ephys_pipeline.py @@ -16,6 +16,9 @@ import u19_pipeline.utils.ephys_fix_sync_code as efsc import u19_pipeline.utils.ephys_utils as ephys_utils from u19_pipeline import recording +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) schema = dj.schema(dj.config["custom"]["database.prefix"] + "ephys_pipeline") @@ -93,12 +96,12 @@ def get_session_directory(session_key): def get_full_session_directory(recording_key): session_dir = find_full_path([get_ephys_root_data_dir()[0]], get_session_directory(recording_key)) - print("ephys dir:", session_dir) + logger.info("ephys dir: %s", session_dir) nidq_session = list(session_dir.glob("*nidq.bin*")) obx_session = list(session_dir.glob("*obx.bin*")) if len(nidq_session) == 0 and len(obx_session) == 0: - print("No ephys session found") + logger.warning("No ephys session found") ephys_session_fullpath = "" elif len(nidq_session) > 0: ephys_session_fullpath = nidq_session[0] @@ -160,13 +163,13 @@ def create_lfp_trace(cat_gt_script, recording_directory, probe_directory): "-lffilter=" + lfp_filter_params, ] - print(cat_gt_command) + logger.debug("cat_gt_command: %s", cat_gt_command) p = subprocess.Popen(cat_gt_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.wait() stdout, stderr = p.communicate() - print("stdout", stdout) - print("stderr", stderr.decode("UTF-8")) + logger.debug("stdout: %s", stdout) + logger.debug("stderr: %s", stderr.decode("UTF-8")) if stderr: error = json.loads(stderr.decode("UTF-8")) @@ -188,17 +191,17 @@ def get_spikeglx_meta_filepath(ephys_recording_key): ephys_element.EphysRecording.EphysFile & ephys_recording_key & 'file_path LIKE "%.ap.meta"' ).fetch1("file_path") - print("ephys_recording_key", ephys_recording_key) - print("spikeglx_meta_filepath", spikeglx_meta_filepath) + logger.debug("ephys_recording_key: %s", ephys_recording_key) + logger.debug("spikeglx_meta_filepath: %s", spikeglx_meta_filepath) - print("get_ephys_root_data_dir", get_ephys_root_data_dir()) + logger.debug("get_ephys_root_data_dir: %s", get_ephys_root_data_dir()) spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(), spikeglx_meta_filepath) - print("spikeglx_meta_filepath", spikeglx_meta_filepath) + logger.debug("spikeglx_meta_filepath: %s", spikeglx_meta_filepath) try: spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(), spikeglx_meta_filepath) - print("spikeglx_meta_filepath", spikeglx_meta_filepath) + logger.debug("spikeglx_meta_filepath: %s", spikeglx_meta_filepath) except FileNotFoundError: # if not found, search in session_dir again if not spikeglx_meta_filepath.exists(): @@ -247,7 +250,7 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): full_session_path = get_full_session_directory(rec_key) if isinstance(full_session_path, str) and len(full_session_path) == 0: - print("No session found for this key") + logger.warning("No session found for this key") return # Get sampling rate and calculate channels and samples @@ -259,7 +262,7 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): try: sync_data = (BehaviorSync & rec_key).fetch1("sync_data") except Exception: - print("No sync data was sound for this session") + logger.warning("No sync data was found for this session") return trial_index_nidq_virmen, iteration_index_nidq_virmen = ephys_utils.get_full_vector_samples( @@ -294,10 +297,10 @@ def get_full_vectors_from_key(rec_key, single_vec_mode=False): if single_vec_mode: for this_key in all_vectors: - print("this_key", this_key) - print(all_vectors[this_key].shape) + logger.debug("this_key: %s", this_key) + logger.debug(all_vectors[this_key].shape) all_vectors[this_key] = np.concatenate((list(all_vectors[this_key])), axis=0) - print(all_vectors[this_key].shape) + logger.debug(all_vectors[this_key].shape) all_vectors["trial_index_nidq_virmen"] = trial_index_nidq_virmen all_vectors["iteration_index_nidq_virmen"] = iteration_index_nidq_virmen @@ -339,7 +342,7 @@ class ImecSamplingRate(dj.Part): def make(self, key, **kwargs): # Pull the Nidaq file/record - print(key) + logger.debug("key: %s", key) try: ephys_session_fullpath = get_full_session_directory(key) @@ -353,10 +356,10 @@ def make(self, key, **kwargs): # If a specific block is requested, add that to our behavior_key. It should be an int referring to virmen block number. # This is useful for sessions in which the nidaq stream was interrupted due to restarting virmen if "block" in kwargs: - print("block: ", kwargs["block"]) + logger.debug("block: %s", kwargs["block"]) behavior_key["block"] = kwargs["block"] - print(behavior_key) + logger.debug("behavior_key: %s", behavior_key) # And get the datajoint record behavior = dj.create_virtual_module("behavior", "u19_behavior") @@ -364,16 +367,16 @@ def make(self, key, **kwargs): behavior_time, iterstart = thissession.fetch("trial_time", "vi_start") if key["recording_id"] == 530: - print(type(behavior_time)) - print(len(behavior_time)) + logger.debug("behavior_time type: %s", type(behavior_time)) + logger.debug("behavior_time len: %d", len(behavior_time)) behavior_time = behavior_time[:-2] - print("len iterstart", len(iterstart)) + logger.debug("len iterstart: %d", len(iterstart)) if len(iterstart) == 0: raise ValueError("No behavior found") - print("after reading behavior data") + logger.debug("after reading behavior data") # 1: load meta data, and the content of the NIDAQ file. Its content is digital. nidq_meta, nidq_sampling_rate = ephys_utils.read_nidq_meta_samp_rate(ephys_session_fullpath) @@ -382,7 +385,7 @@ def make(self, key, **kwargs): ephys_session_fullpath, nidq_meta ) - print("after reading spikeglx data") + logger.debug("after reading spikeglx data") # Synchronize between pulses and get iteration # vector for each sample recent_recording = behavior_key["session_date"] > datetime.date(2021, 6, 1) # Everything past June 1 2021 @@ -407,8 +410,8 @@ def make(self, key, **kwargs): ephys_utils.assert_iteration_samples_count(iteration_dict["iter_start_idx"], behavior_time) ) - print("metrics to evaluate...") - print( + logger.debug( + "metrics to evaluate: trial_count_diff=%s, trials_diff_iteration_big=%s, trials_diff_iteration_small=%s, n_trials=%d", trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small, @@ -432,7 +435,7 @@ def make(self, key, **kwargs): status_fix = 0 if status < 1: status_regular = 0 - print("Regular ephys sync failed") + logger.warning("Regular ephys sync failed") status_fix, iteration_dict = efsc.main_ephys_fix_sync_code( iteration_dict["iter_start_idx"], iteration_dict["iter_times_idx"], @@ -442,7 +445,7 @@ def make(self, key, **kwargs): dictionary_sync_data = {} - print("after all main ehpys fix sync code", status_regular, status_fix) + logger.debug("after all main ephys fix sync code: status_regular=%s, status_fix=%s", status_regular, status_fix) if status_regular > 0 or status_fix > 0: dictionary_sync_data["trial_idx_vector"] = iteration_dict["trial_start_idx"] @@ -475,10 +478,10 @@ def make(self, key, **kwargs): virmen_sync_status=1, ) - print("ephys_session_fullpath", ephys_session_fullpath) - print("sync code executed sucessfully !!!!!!!!!!!!!!!!!!!!!!!!!!!") + logger.info("ephys_session_fullpath: %s", ephys_session_fullpath) + logger.info("sync code executed successfully") - print(kwargs) + logger.debug("kwargs: %s", kwargs) if "populate" not in kwargs or kwargs["populate"]: BehaviorSync.insert1(final_key, allow_direct_insert=True) @@ -487,7 +490,7 @@ def make(self, key, **kwargs): return final_key except Exception as e: - print(e) + logger.exception("Error in BehaviorSync.make: %s", e) def insert_imec_sampling_rate(self, key, session_dir): # get the imec sampling rate for a particular probe diff --git a/u19_pipeline/ephys_sync.py b/u19_pipeline/ephys_sync.py index a022d0c4..65bee7d1 100644 --- a/u19_pipeline/ephys_sync.py +++ b/u19_pipeline/ephys_sync.py @@ -7,6 +7,9 @@ import u19_pipeline.utils.ephys_utils as ephys_utils from u19_pipeline import behavior from u19_pipeline.ephys_pipeline import ephys_element, get_session_directory +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) # Tables downstream from `ephys_pipeline` module --------------------------------------- schema = dj.schema(dj.config["custom"]["database.prefix"] + "ephys_sync") @@ -76,7 +79,7 @@ def make(self, key): trial_index_nidq=iteration_dict["trialnumber_vector_samples"], ) - print(final_key) + logger.debug("final_key: %s", final_key) self.insert1(final_key, allow_direct_insert=True) @@ -154,7 +157,7 @@ def make(self, key): # if iteration_index_nidq[iteration_transition_indexes[i]] <= first_vr_iteration: # ls[iteration_transition_indexes[i]] = 0 - print("sum_iterationtrans", np.sum(ls)) + logger.debug("sum_iterationtrans: %s", np.sum(ls)) iteration_times = np.where(ls)[0] / nidq_sampling_rate diff --git a/u19_pipeline/imaging_pipeline.py b/u19_pipeline/imaging_pipeline.py index c45546b7..ea358756 100644 --- a/u19_pipeline/imaging_pipeline.py +++ b/u19_pipeline/imaging_pipeline.py @@ -10,6 +10,9 @@ import u19_pipeline.utils.dj_shortcuts as dj_short import u19_pipeline.utils.tiff_utils as tu from u19_pipeline import lab, recording, subject +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) schema = dj.schema(dj.config["custom"]["database.prefix"] + "imaging_pipeline") @@ -81,7 +84,7 @@ def make(self, key, test_mode=False): is_mesoscope = acq_type in self.mesoscope_acq is_2photon = acq_type in self.photon_micro_acq - print(f"Preparing {scan_directory}") + logger.info(f"Preparing {scan_directory}") if is_mesoscope: original_stacks_dir = scan_directory / "originalStacks" @@ -170,14 +173,14 @@ def old_make(self, key): config.startup_pipeline_matlab_dir, str_key, ] - print(command) + logger.debug("command: %s", command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.wait() - print("aftercommand before comm") + logger.debug("aftercommand before comm") stdout, stderr = p.communicate() - print("aftercommand after comm") - print(stdout.decode("UTF-8")) - print(stderr.decode("UTF-8")) + logger.debug("aftercommand after comm") + logger.debug(stdout.decode("UTF-8")) + logger.debug(stderr.decode("UTF-8")) @schema diff --git a/u19_pipeline/utils/DemoReadSGLXData/read_sglx.py b/u19_pipeline/utils/DemoReadSGLXData/read_sglx.py index e1eae308..d736ddc8 100644 --- a/u19_pipeline/utils/DemoReadSGLXData/read_sglx.py +++ b/u19_pipeline/utils/DemoReadSGLXData/read_sglx.py @@ -21,6 +21,10 @@ import matplotlib.pyplot as plt import numpy as np +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) + # Parse ini file returning a dictionary whose keys are the metadata # left-hand-side-tags, and values are string versions of the right-hand-side @@ -44,7 +48,7 @@ def readMeta(binFullPath): currKey = csList[0][1:len(csList[0])] if csList[0][0] == "~" else csList[0] metaDict.update({currKey: csList[1]}) else: - print("no meta file") + logger.error("no meta file") return metaDict @@ -60,7 +64,7 @@ def SampRate(meta): elif meta["typeThis"] == "obx": srate = float(meta["obSampRate"]) else: - print("Error: unknown stream type") + logger.error("Error: unknown stream type") srate = 1 return srate @@ -83,7 +87,7 @@ def Int2Volts(meta): maxInt = int(meta["obMaxInt"]) fI2V = float(meta["obAiRangeMax"]) / maxInt else: - print("Error: unknown stream type") + logger.error("Error: unknown stream type") fI2V = 1 return fI2V @@ -209,7 +213,7 @@ def ChanGainsIM(meta): # commercial NP 2.0; APGain = 100 for all AP APgain = APgain + 100 else: - print("unknown gain, setting APgain to 1") + logger.warning("unknown gain, setting APgain to 1") APgain = APgain + 1 fI2V = Int2Volts(meta) APChan0_to_uV = 1e6 * fI2V / APgain[0] @@ -306,7 +310,7 @@ def GainCorrectIM(dataArray, chanList, meta): def makeMemMapRaw(binFullPath, meta): nChan = int(meta["nSavedChans"]) nFileSamp = int(int(meta["fileSizeBytes"]) / (2 * nChan)) - print(f"nChan: {nChan:d}, nFileSamp: {nFileSamp:d}") + logger.info("nChan: %d, nFileSamp: %d", nChan, nFileSamp) rawData = np.memmap( binFullPath, dtype="int16", @@ -331,7 +335,7 @@ def ExtractDigital(rawData, firstSamp, lastSamp, dwReq, dLineList, meta): if meta["typeThis"] == "imec": AP, LF, SY = ChannelCountsIM(meta) if SY == 0: - print("No imec sync channel saved.") + logger.warning("No imec sync channel saved.") digArray = np.zeros((0), "uint8") return digArray else: @@ -339,7 +343,7 @@ def ExtractDigital(rawData, firstSamp, lastSamp, dwReq, dLineList, meta): elif meta["typeThis"] == "nidq": MN, MA, XA, DW = ChannelCountsNI(meta) if dwReq > DW - 1: - print(f"Maximum digital word in file = {DW - 1:d}") + logger.info("Maximum digital word in file = %d", DW - 1) digArray = np.zeros((0), "uint8") return digArray else: @@ -347,13 +351,13 @@ def ExtractDigital(rawData, firstSamp, lastSamp, dwReq, dLineList, meta): elif meta["typeThis"] == "obx": XA, DW, SY = ChannelCountsOBX(meta) if dwReq > DW - 1: - print(f"Maximum digital word in file = {DW - 1:d}") + logger.info("Maximum digital word in file = %d", DW - 1) digArray = np.zeros((0), "uint8") return digArray else: digCh = XA + dwReq else: - print("unknown data stream") + logger.error("unknown data stream") selectData = np.ascontiguousarray(rawData[digCh, firstSamp : lastSamp + 1], "int16") nSamp = lastSamp - firstSamp + 1 diff --git a/u19_pipeline/utils/dlc_process.py b/u19_pipeline/utils/dlc_process.py index 0ee6e706..4e5658b2 100644 --- a/u19_pipeline/utils/dlc_process.py +++ b/u19_pipeline/utils/dlc_process.py @@ -10,6 +10,9 @@ from skimage.measure import EllipseModel import u19_pipeline.utils.path_utils as pu +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) # Pupil diameter pipeline functions @@ -41,7 +44,7 @@ def getPupilDiameter(destinationFolder=None): # Read the analyzed video data h5 file h5_file = pu.get_filepattern_paths(destinationFolder, "/*.h5") - print(h5_file) + logger.debug("h5_file: %s", h5_file) if len(h5_file) == 0: raise Exception("No h5 file in directory: " + destinationFolder) @@ -89,6 +92,6 @@ def getPupilDiameter(destinationFolder=None): if __name__ == "__main__": args = sys.argv[1:] - print(args) + logger.debug("args: %s", args) analyzeVideo(videoPath=args[0], modelPath=args[1], destinationFolder=args[2]) getPupilDiameter(destinationFolder=args[2]) diff --git a/u19_pipeline/utils/ephys_fix_sync_code.py b/u19_pipeline/utils/ephys_fix_sync_code.py index 2d763933..684b1d2c 100644 --- a/u19_pipeline/utils/ephys_fix_sync_code.py +++ b/u19_pipeline/utils/ephys_fix_sync_code.py @@ -1,6 +1,9 @@ import numpy as np import u19_pipeline.utils.ephys_utils as ephys_utils +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) def get_shift_vector( @@ -143,10 +146,10 @@ def fix_shifted_sync_vector(synced_time_vector, behavior_time_vector, vec_shift, else: break else: - print("Extreme case diff vec shift") + logger.warning("Extreme case diff vec shift") index_borrow_virmen.append([where_insert_iteration[index_shift], diff_vec_shift.shape[0]]) - print("index_borrow_virmen", index_borrow_virmen) + logger.debug("index_borrow_virmen: %s", index_borrow_virmen) break # print('index_borrow_virmen', index_borrow_virmen) @@ -300,7 +303,7 @@ def sync_evaluation_process2(synced_time_vector, behavior_time_vector): pass else: status = -1 - print(median_general) + logger.debug("median_general: %s", median_general) return status @@ -330,26 +333,26 @@ def main_ephys_fix_sync_code(iter_start_idx, iter_times_idx, behavior_time, nidq iteration_dict["iter_start_idx"].append(synced_iteration_vector.copy()) iteration_dict["iter_times_idx"].append(synced_time_vector.copy()) - print("end fix sync code 1") + logger.debug("end fix sync code 1") iteration_dict["iter_start_idx"] = np.asarray(iteration_dict["iter_start_idx"].copy(), dtype=object) iteration_dict["iter_times_idx"] = np.asarray(iteration_dict["iter_times_idx"].copy(), dtype=object) - print("end fix sync code") + logger.debug("end fix sync code") # Check # of trials and iterations match trial_count_diff, trials_diff_iteration_big, trials_diff_iteration_small = ( ephys_utils.assert_iteration_samples_count(iteration_dict["iter_start_idx"], behavior_time) ) - print("after assert_iteration_samples_count fix sync code") + logger.debug("after assert_iteration_samples_count fix sync code") if trial_count_diff != 0: - print("trial_count_diff", trial_count_diff) + logger.debug("trial_count_diff: %s", trial_count_diff) if len(trials_diff_iteration_big) > 0: - print("trials_diff_iteration_big", trials_diff_iteration_big) + logger.debug("trials_diff_iteration_big: %s", trials_diff_iteration_big) if len(trials_diff_iteration_small) > 0: - print("trials_diff_iteration_small", trials_diff_iteration_small) + logger.debug("trials_diff_iteration_small: %s", trials_diff_iteration_small) status = ephys_utils.evaluate_sync_process( trial_count_diff, @@ -358,7 +361,7 @@ def main_ephys_fix_sync_code(iter_start_idx, iter_times_idx, behavior_time, nidq behavior_time.shape[0], ) - print("after evaluate_sync_process fix sync code") + logger.debug("after evaluate_sync_process fix sync code") for i in range(len(iteration_dict["iter_start_idx"])): synced_time_vector = iteration_dict["iter_times_idx"][i] @@ -368,13 +371,13 @@ def main_ephys_fix_sync_code(iter_start_idx, iter_times_idx, behavior_time, nidq if status == -1: break - print("after sync_evaluation_process2", status) + logger.debug("after sync_evaluation_process2: status=%s", status) if status == 1: iteration_dict["trial_start_idx"] = ephys_utils.get_index_trial_vector_from_iteration( iteration_dict["iter_start_idx"] ) - print("after get_index_trial_vector_from_iteration") + logger.debug("after get_index_trial_vector_from_iteration") return status, iteration_dict diff --git a/u19_pipeline/utils/ephys_utils.py b/u19_pipeline/utils/ephys_utils.py index 4a07ca70..8e9daa8d 100644 --- a/u19_pipeline/utils/ephys_utils.py +++ b/u19_pipeline/utils/ephys_utils.py @@ -9,6 +9,9 @@ from scipy.io import loadmat import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) class SpiceGlxUtility: @@ -23,7 +26,7 @@ def load_spice_glx_digital_file(file_path, nidq_meta, d_line_list=None): # Get all digitial channels if not provided if d_line_list is None: list_start_end_chan = nidq_meta["niXDChans1"].split(sep=":") - print(list_start_end_chan) + logger.debug("list_start_end_chan: %s", list_start_end_chan) if len(list_start_end_chan) == 2: d_line_list = list(range(int(list_start_end_chan[0]), int(list_start_end_chan[1]) + 1)) else: @@ -182,7 +185,7 @@ def get_trial_signal_mode(iteration_pulse_signal_trial, behavior_time_vector_tri mode = "counter_bit0" if iter_samples[0].shape[0] < behavior_time_vector_trial.shape[0] * 3 / 4 else "pulse_signal" - print("mode deduction: ", mode) + logger.debug("mode deduction: %s", mode) return mode @@ -206,7 +209,7 @@ def get_iteration_sample_vector_from_digital_lines_pulses( # Get idx samples trial starts trial_start_idx, trial_end_pulse_idx = get_idx_trial_start(trial_pulse_signal) - print("len trial_start_idx", trial_start_idx.shape) + logger.debug("len trial_start_idx: %s", trial_start_idx.shape) if mode is None: mode = get_trial_signal_mode( @@ -372,16 +375,12 @@ def assert_iteration_samples_count(iteration_sample_idx_output, behavior_time_ve trials_diff_iteration_big = [] for count, iter_trials in enumerate(iteration_sample_idx_output): if iter_trials.shape[0] != behavior_time_vector[count].shape[0]: - print( - "trial#", + logger.debug( + "trial# %d iterPulses: %d IterBeh: %d Difference (%d)", count, - "iterPulses:", iter_trials.shape[0], - "IterBeh:", behavior_time_vector[count].shape[0], - "Difference (", behavior_time_vector[count].shape[0] - iter_trials.shape[0], - ")", ) # For each trial iteration # should be equal to the behavioral file iterations if iter_trials.shape[0] != behavior_time_vector[count].shape[0]: @@ -409,13 +408,13 @@ def evaluate_sync_process( """ if len(trials_diff_iteration_big) > 1: - print("Missed by a lot some trials: ", trials_diff_iteration_big) + logger.warning("Missed by a lot some trials: %s", trials_diff_iteration_big) status = -1 return status if len(trials_diff_iteration_big) == 1 and trials_diff_iteration_big[0] == total_trials - 1: - print( - "Missed by a lot last trial: (Assume recording stopped earlier) ", + logger.warning( + "Missed by a lot last trial: (Assume recording stopped earlier) %s", trials_diff_iteration_big, ) status = 1 @@ -423,43 +422,39 @@ def evaluate_sync_process( # All trials synced perfectly if trial_count_diff == 0 and len(trials_diff_iteration_small) == 0: - print("Synced perfectly xxxxxxxxxxxxxx") + logger.info("Synced perfectly") status = 1 return status # We miss last trial (surely recording was stop before behavior) if trial_count_diff < 2 and len(trials_diff_iteration_small) == 0: - print("Missed one trial signal xxxxxxxxxxxxxx") + logger.warning("Missed one trial signal") status = -1 return status # Iterations differ in more than two trials if len(trials_diff_iteration_small) > 2: - print("Missed iteration count on many trials: ", len(trials_diff_iteration_small)) + logger.warning("Missed iteration count on many trials: %d", len(trials_diff_iteration_small)) status = -1 return status if trial_count_diff < 2 and len(trials_diff_iteration_small) <= 2: - print("Missed num trials: ", trial_count_diff) - print( - "Missed iteration count in how many trials: ", - len(trials_diff_iteration_small), - ) - print("Trying to fix trials") + logger.warning("Missed num trials: %d", trial_count_diff) + logger.warning("Missed iteration count in how many trials: %d", len(trials_diff_iteration_small)) + logger.info("Trying to fix trials") status = 0 return status else: status = -1 - print("Missed by a lot of trials, everything different or missing") + logger.warning("Missed by a lot of trials, everything different or missing") return status def fix_missing_iteration_trials(trials_diff_iteration_small, iteration_dict, behavior_times, nidq_sampling_rate): # Fix and insert missing synced iteration vectors - print("trials_diff_iteration_small", trials_diff_iteration_small) - print(type(trials_diff_iteration_small)) + logger.debug("trials_diff_iteration_small: %s (type: %s)", trials_diff_iteration_small, type(trials_diff_iteration_small)) # For each bad synced trial (Should be only a few) for i in range(len(trials_diff_iteration_small)): @@ -502,13 +497,13 @@ def insert_missing_synced_iteration(synced_iteration_vector, synced_time_vector, # Check where is more likely we miss an iteration pulse and insert it to iteration_vector status = 1 - print("synced_iteration_vector", synced_iteration_vector.shape[0]) - print("synced_time_vector", synced_time_vector.shape[0]) - print("behavior_time_vector", behavior_time_vector.shape[0]) + logger.debug("synced_iteration_vector: %d", synced_iteration_vector.shape[0]) + logger.debug("synced_time_vector: %d", synced_time_vector.shape[0]) + logger.debug("behavior_time_vector: %d", behavior_time_vector.shape[0]) # Get in which indexes we get a "peak" of non matching times... if synced_time_vector.shape[0] >= behavior_time_vector.shape[0]: - print("More pulses than behavior iterations, check other method") # Christian: What is the other method? + logger.warning("More pulses than behavior iterations, check other method") status = -1 return status, np.empty(0) else: @@ -518,8 +513,7 @@ def insert_missing_synced_iteration(synced_iteration_vector, synced_time_vector, peaks, _ = sp.find_peaks(diff_vector, height=0.05, distance=20) - print("peaks here .............", peaks) - print(peaks.shape) + logger.debug("peaks: %s (shape: %s)", peaks, peaks.shape) # Insert extra iterations as a new "iteration" start to match behavior iterations new_synced_iteration_vector = synced_iteration_vector.copy() @@ -528,7 +522,7 @@ def insert_missing_synced_iteration(synced_iteration_vector, synced_time_vector, new_synced_iteration_vector = np.insert(new_synced_iteration_vector, peaks[i], value_insert) if new_synced_iteration_vector.shape[0] != behavior_time_vector.shape[0]: - print("with peak strategy, could not find correct missing iterations") + logger.warning("with peak strategy, could not find correct missing iterations") status = -1 return status, np.empty(0) @@ -678,7 +672,7 @@ def future_counter_get_signal(): framenumber_in_trial[idx] = frame_number + overflow*(max_count+1) - print(np.max(framenumber_in_trial)) + logger.debug("max framenumber_in_trial: %s", np.max(framenumber_in_trial)) """ @@ -809,15 +803,15 @@ def get_index_type_vectors(trial_index_nidq, iteration_index_nidq, nidq_sampling iteration_equal = np.allclose(iteration_index_nidq, iteration_index_nidq2, equal_nan=True) trial_equal = np.allclose(trial_index_nidq, trial_index_nidq2, equal_nan=True) - print("iteration_equal", iteration_equal) - print("trial_equal", trial_equal) + logger.debug("iteration_equal: %s", iteration_equal) + logger.debug("trial_equal: %s", trial_equal) - print( - "np.where(np.isnan(iteration_index_nidq))[0].shape", + logger.debug( + "np.where(np.isnan(iteration_index_nidq))[0].shape: %s", np.where(np.isnan(iteration_index_nidq))[0].shape, ) - print( - "np.where(np.isnan(iteration_index_nidq2))[0].shape", + logger.debug( + "np.where(np.isnan(iteration_index_nidq2))[0].shape: %s", np.where(np.isnan(iteration_index_nidq2))[0].shape, ) @@ -864,7 +858,7 @@ def main_xyz_pick_file_function(recording_id, fragment_number, chanmap_file, pro # Get recording id probe_location = XyzPickFileCreator.get_probe_insertion_coordinates(recording_id, fragment_number) - print(probe_location) + logger.debug("probe_location: %s", probe_location) # Load channelmap and check how many probes there are chanmap = loadmat(chanmap_file) diff --git a/u19_pipeline/utils/logging_config.py b/u19_pipeline/utils/logging_config.py new file mode 100644 index 00000000..f0acf2a6 --- /dev/null +++ b/u19_pipeline/utils/logging_config.py @@ -0,0 +1,46 @@ +"""Logging helpers for u19_pipeline. + +Library usage (u19_pipeline/*): + Call get_logger(__name__) at module level. No handlers are attached here; + the caller (application or script) is responsible for configuring them. + +Script usage (scripts/*): + Call setup_logging() once at the top of the __main__ block. This installs + a Rich handler on the root logger so all u19_pipeline loggers emit + formatted output without disturbing any downstream caller that configures + their own handlers. +""" + +import logging + +# Silence the library's own loggers by default (PEP 396 / logging HOWTO). +# Downstream code or scripts call setup_logging() to attach real handlers. +logging.getLogger("u19_pipeline").addHandler(logging.NullHandler()) + + +def get_logger(name: str) -> logging.Logger: + """Return a named logger. Never configures handlers — safe for library use.""" + return logging.getLogger(name) + + +def setup_logging(level: int = logging.INFO) -> None: + """Configure the root logger with a Rich handler. + + Call this exactly once at the entry point of a script or application. + Safe to call multiple times (idempotent via basicConfig semantics). + """ + from rich.console import Console + from rich.logging import RichHandler + + logging.basicConfig( + level=level, + format="%(message)s", + datefmt="[%X]", + handlers=[ + RichHandler( + console=Console(stderr=True), + rich_tracebacks=True, + show_path=True, + ) + ], + ) diff --git a/u19_pipeline/utils/matlab_utils.py b/u19_pipeline/utils/matlab_utils.py index 97f97ee6..9df8db86 100644 --- a/u19_pipeline/utils/matlab_utils.py +++ b/u19_pipeline/utils/matlab_utils.py @@ -8,6 +8,10 @@ from shutil import which import numpy as np + +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) import pandas as pd from numpy import ndarray from scipy.io import loadmat, matlab @@ -274,9 +278,9 @@ def convert_function_handle_to_str(mat_file_path): os.remove("shaping_protocol.txt") except Exception as e: - print(f"There was an error while trying to execute {convert_script_path}:\n{e}") + logger.error("There was an error while trying to execute %s: %s", convert_script_path, e) else: - print( + logger.warning( "A working matlab version was not found. " "Code version, animal protocol, type of trial, and choice could not be saved to NWB." ) diff --git a/u19_pipeline/utils/scp_transfers.py b/u19_pipeline/utils/scp_transfers.py index aee5ed4b..237143f7 100644 --- a/u19_pipeline/utils/scp_transfers.py +++ b/u19_pipeline/utils/scp_transfers.py @@ -10,6 +10,9 @@ from u19_pipeline.automatic_job.clusters_paths_and_transfers import ( public_key_location as public_key_location, ) +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) # Steps on windows machine # https://thesysadminchannel.com/solved-add-windowscapability-failed-error-code-0x800f0954-rsat-fix/ @@ -43,10 +46,10 @@ def _get_ssh_key(self): """Fetch locally stored SSH key.""" try: self.ssh_key = RSAKey.from_private_key_file(self.ssh_key_filepath) - print(f"Found SSH key at self {self.ssh_key_filepath}") + logger.info("Found SSH key at %s", self.ssh_key_filepath) return self.ssh_key except SSHException as e: - print(e) + logger.error("SSHException: %s", e) @property def connection(self): @@ -63,7 +66,7 @@ def connection(self): ) return client except AuthenticationException as e: - print(f"Authentication failed: did you remember to create an SSH key? {e}") + logger.error("Authentication failed: did you remember to create an SSH key? %s", e) raise e @property @@ -86,13 +89,13 @@ def download_folder(self, remote_path: str, local_path: str): def transfer_scp(host=None, username=None, remote_path=None, local_path=None): rc = RemoteClient(host, username, public_key_location, remote_path) - print(host) - print(username) - print(public_key_location) - print(remote_path) - print(local_path) + logger.debug("host: %s", host) + logger.debug("username: %s", username) + logger.debug("public_key_location: %s", public_key_location) + logger.debug("remote_path: %s", remote_path) + logger.debug("local_path: %s", local_path) rc._get_ssh_key() - print(rc.scp) + logger.debug("scp: %s", rc.scp) rc.download_folder(remote_path=remote_path, local_path=local_path) rc.disconnect() @@ -104,7 +107,7 @@ def call_scp_background( data_directory=None, ): - print(ip_address, system_user, data_directory, recording_system_directory) + logger.debug("ip_address=%s, system_user=%s, data_directory=%s, recording_system_directory=%s", ip_address, system_user, data_directory, recording_system_directory) # transfer_scp(rec_series['ip_address'], rec_series['system_user'], rec_series['local_directory'], full_remote_path) this_file = os.path.realpath(__file__) @@ -138,9 +141,9 @@ def check_scp_transfer(pid): if psutil.pid_exists(pid): pr = psutil.Process(pid=pid) - print(pr) + logger.debug("process: %s", pr) gone, _ = psutil.wait_procs([pr], timeout=3) - print(gone) + logger.debug("gone: %s", gone) if len(gone) > 0: finished = True @@ -160,6 +163,6 @@ def check_directory_copied_correctly(): if __name__ == "__main__": args = sys.argv[1:] - print(args) + logger.debug("args: %s", args) transfer_scp(host=args[0], username=args[1], remote_path=args[2], local_path=args[3]) diff --git a/u19_pipeline/utils/tiff_matlab_imaging_utils.py b/u19_pipeline/utils/tiff_matlab_imaging_utils.py index 669def71..92d12dc1 100644 --- a/u19_pipeline/utils/tiff_matlab_imaging_utils.py +++ b/u19_pipeline/utils/tiff_matlab_imaging_utils.py @@ -8,6 +8,10 @@ import tifffile as tiff from sklearn.linear_model import HuberRegressor +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) + xy_size_factor = 1.05 # images are this much larger than nominal size z_factor = 1.45 # actual displacement in z vs command @@ -36,7 +40,7 @@ def select_files_from_mean_f(scan_directory, f_decrease_threshold=15): start_time = time.time() - print("Estimating bleaching...", end="", flush=True) + logger.info("Estimating bleaching...") scan_directory = Path(scan_directory) @@ -58,7 +62,7 @@ def select_files_from_mean_f(scan_directory, f_decrease_threshold=15): # --------------------------------------------------------- for i, tif_path in enumerate(tif_files): - print(".", end="", flush=True) + logger.debug("processing file %d/%d", i + 1, n_files) with tiff.TiffFile(tif_path) as tif_obj: n_frames = len(tif_obj.pages) @@ -118,7 +122,7 @@ def select_files_from_mean_f(scan_directory, f_decrease_threshold=15): elapsed_minutes = (time.time() - start_time) / 60 - print(f" done after {elapsed_minutes:.1f} min") + logger.info("done after %.1f min", elapsed_minutes) return last_good_file diff --git a/u19_pipeline/utils/tiff_utils.py b/u19_pipeline/utils/tiff_utils.py index f5971e16..41d478e6 100644 --- a/u19_pipeline/utils/tiff_utils.py +++ b/u19_pipeline/utils/tiff_utils.py @@ -10,6 +10,9 @@ import tifffile import u19_pipeline.utils.tiff_matlab_imaging_utils as tmiu +from u19_pipeline.utils.logging_config import get_logger + +logger = get_logger(__name__) tif_number_fmt = r"_[0-9]{5}\.tif" tif_gz_number_fmt = r"_[0-9]{5}\.tif\.gz" @@ -228,12 +231,12 @@ def remove_compressed_videos(fl, directory): gz_file = Path(str(file_base) + ".gz") if gz_file.exists() and file_base.exists(): - print(f"Removing {gz_file}") + logger.info("Removing %s", gz_file) gz_file.unlink() else: - print(f"Could not find compressed pair {file_base}") + logger.warning("Could not find compressed pair %s", file_base) # ----------------------------------------------------------------------------- @@ -299,7 +302,7 @@ def get_fov_mesoscope( n_roi = rec_info["nROIs"] if not skip_parsing: - print("\tparsing ROIs...") + logger.debug("parsing ROIs...") roi_nr = [roi["pixelResolutionXY"][1] for roi in rec_info["ROI"]] [roi["pixelResolutionXY"][0] for roi in rec_info["ROI"]] diff --git a/uv.lock b/uv.lock index e684b35f..4f035520 100644 --- a/uv.lock +++ b/uv.lock @@ -2543,6 +2543,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/79/d3bbab197e86e0ff4f9c07122895b66a3e0d024247fcff7f12c473cb36d9/llvmlite-0.47.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6842cf6f707ec4be3d985a385ad03f72b2d724439e118fcbe99b2929964f0453", size = 39153839, upload-time = "2026-03-31T18:29:51.004Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -2684,6 +2696,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "minio" version = "7.2.20" @@ -4418,6 +4439,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/71/44ce230e1b7fadd372515a97e32a83011f906ddded8d03e3c6aafbdedbb7/rfc3987_syntax-1.1.0-py3-none-any.whl", hash = "sha256:6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f", size = 8046, upload-time = "2025-07-18T01:05:03.843Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "roifile" version = "2026.2.10" @@ -5219,6 +5253,7 @@ dependencies = [ { name = "openpyxl" }, { name = "pandas" }, { name = "python-dotenv" }, + { name = "rich" }, { name = "scikit-image" }, { name = "scipy" }, { name = "tables" }, @@ -5277,6 +5312,7 @@ requires-dist = [ { name = "plotly", marker = "extra == 'analysis'", specifier = ">=5.14" }, { name = "psutil", marker = "extra == 'pipeline'" }, { name = "python-dotenv", specifier = ">=0.20" }, + { name = "rich", specifier = ">=15.0.0" }, { name = "scanreader", marker = "extra == 'pipeline'", git = "https://github.com/BrainCOGS/scanreader.git" }, { name = "scikit-image" }, { name = "scipy" },