From d4fb6e10d56019fde5ca4245c51b12c72827b199 Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Mon, 2 Feb 2026 11:57:15 -0700 Subject: [PATCH 01/13] add generic upload script --- python/custom_file_uploads/generic_upload.py | 240 +++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 python/custom_file_uploads/generic_upload.py diff --git a/python/custom_file_uploads/generic_upload.py b/python/custom_file_uploads/generic_upload.py new file mode 100644 index 0000000..f36843c --- /dev/null +++ b/python/custom_file_uploads/generic_upload.py @@ -0,0 +1,240 @@ +"""Lightweight generalized file upload script for Civis Platform. + +Uploads a CSV file to a database table based on user's group and +dropdown selection. +The script: +1. Determines user's schema from metadata table based on primary_group_id +2. Maps dropdown selection to table name via UPLOAD_OPTIONS config +3. Drops existing table if present and imports CSV +4. Sends email notification on completion + +In order to function correctly, the following environment variables must be + set: +- UPLOAD_OPTION: Dropdown selection indicating type of data being uploaded + (configured in this script) +- DATABASE: Target Civis database name +- METADATA_TABLE: Civis table containing mapping of primary_group_id + to schema_name +- SCRIPT_ID: Civis script ID used to send notification emails + - this should just be a blank script configured to send success emails + +The template script will also need to accept the following parameters: +- FILE: Civis file parameter +- EMAIL: Optional email address for notification +- UPLOAD_OPTION: Dropdown selection for upload type +- TESTING: Optional boolean to indicate testing mode (skip email) +""" + +import os +import tempfile +from distutils.util import strtobool + +import civis +import pandas as pd + +LOG = civis.loggers.civis_logger() +# Configuration +UPLOAD_OPTIONS = { + "People Data": "people_data", + "Sales Data": "sales_data", + "Inventory Data": "inventory_data", +} + + +def get_schema(metadata_table: str, database: str, client=None) -> str: + client = client or civis.APIClient() + + # Get user info + user = client.users.list_me() + user_id = user["id"] + user_email = user["email"] + primary_group_id = client.users.get(user_id)["primary_group_id"] + + # Lookup schema from metadata table + LOG.info(f"Looking up schema for primary_group_id {primary_group_id}") + schema_query = f""" + SELECT schema_name + FROM {metadata_table} + WHERE primary_group_id = {primary_group_id} + """ + schema_result = civis.io.read_civis_sql( + schema_query, database=database, use_pandas=True + ) + + if schema_result.empty: + raise ValueError( + f"No schema mapping found for primary_group_id {primary_group_id}." + f"Please add a row to {metadata_table}." + ) + + schema = schema_result["schema_name"].iloc[0] + LOG.info(f"Using schema: {schema}") + + # Create schema if needed + LOG.info(f"Creating schema {schema} if needed") + civis.io.query_civis( + f"CREATE SCHEMA IF NOT EXISTS {schema};", + database=database, + ).result() + + return schema, user_email + + +def get_table(upload_option: str): + # Get table name from dropdown selection + if upload_option not in UPLOAD_OPTIONS: + raise ValueError( + f"Invalid upload option '{upload_option}'. " + f"Valid options: {list(UPLOAD_OPTIONS.keys())}" + ) + return UPLOAD_OPTIONS[upload_option] + + +def download_data_create_table( + file_id: int, full_table: str, database: str, client=None +): + client = client or civis.APIClient() + file_obj = client.files.get(file_id) + LOG.info(f"Downloading file: {file_obj['name']}") + + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = os.path.join(tmpdir, file_obj["name"]) + civis.io.civis_to_file(file_id, tmp_path) + df = pd.read_csv(tmp_path) + + LOG.info(f"Read CSV with {len(df)} rows and {len(df.columns)} columns") + + # Drop existing table + LOG.info(f"Dropping table if exists: {full_table}") + civis.io.query_civis( + f"DROP TABLE IF EXISTS {full_table};", + database=database, + ).result() + + # Import data to table + LOG.info(f"Importing data to {full_table}") + civis.io.dataframe_to_civis( + df=df, + database=database, + table=full_table, + existing_table_rows="fail", + ) + LOG.info(f"Successfully uploaded {len(df)} rows to {full_table}") + + +def send_email_notification( + email_address: str, + table_name: str, + schema: str, + full_table: str, + database: str, + user_email: str, + upload_option: str, + file_obj: dict, + testing: bool = False, + client=None, +): + client = client or civis.APIClient() + recipient_email = email_address if email_address else user_email + email_subject = f"Data Upload Complete: {upload_option}" + email_body = f"""Your data upload has been completed successfully. + +File: {file_obj['name']} +Upload Type: {upload_option} +Database: {database} +Schema: {schema} +Table: {table_name} +User: {user_email} + +The data is now available at: {database}.{full_table} +""" + + if not testing: + LOG.info(f"Sending notification email to {recipient_email}") + # Use a blank script on platform to trigger email notification + # NOTE: This requires an existing script ID that you can configure + # to send success notification emails + email_script_id = int(os.getenv("EMAIL_SCRIPT_ID")) + + client.scripts.patch_python3( + id=email_script_id, + name=f"Upload notification for {user_email}", + notifications={ + "success_email_subject": email_subject, + "success_email_body": email_body, + "success_email_addresses": [recipient_email], + }, + ) + future = civis.utils.run_job(email_script_id, client=client).result() + + if not future.succeeded(): + LOG.warning("Unable to send notification email") + else: + LOG.info(f"Email sent to {recipient_email}") + else: + LOG.info(f"Testing mode: skipping email to {recipient_email}") + + +def main( + file_id: int, + upload_option: str, + database: str, + metadata_table: str, + email_address: str = None, + testing: bool = False, +): + """Main function to upload CSV file to Civis database table.""" + + client = civis.APIClient() + + schema, user_email = get_schema( + metadata_table=metadata_table, + database=database, + client=client, + ) + + table_name = get_table(upload_option) + + full_table = f"{schema}.{table_name}" + LOG.info(f"Target table: {full_table}") + + download_data_create_table( + file_id=file_id, + full_table=full_table, + database=database, + client=client, + ) + + send_email_notification( + email_address=email_address, + table_name=table_name, + schema=schema, + full_table=full_table, + database=database, + user_email=user_email, + upload_option=upload_option, + file_obj=client.files.get(file_id), + testing=testing, + client=client, + ) + + LOG.info("Upload process completed successfully") + + +if __name__ == "__main__": + # Get environment variables + file_id = int(os.environ["FILE_ID"]) + upload_option = os.environ["UPLOAD_OPTION"] + database = os.environ["DATABASE"] + metadata_table = os.environ["METADATA_TABLE"] + email_address = os.getenv("EMAIL") + testing = strtobool(os.getenv("TESTING", "false")) + + main( + file_id=file_id, + upload_option=upload_option, + database=database, + metadata_table=metadata_table, + email_address=email_address, + testing=testing, + ) From 5629ac6fc94b735a4a181eb842e228d49a86d91d Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Mon, 2 Feb 2026 12:14:37 -0700 Subject: [PATCH 02/13] update script to use table name --- python/custom_file_uploads/generic_upload.py | 34 +++++--------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/python/custom_file_uploads/generic_upload.py b/python/custom_file_uploads/generic_upload.py index f36843c..b0655d0 100644 --- a/python/custom_file_uploads/generic_upload.py +++ b/python/custom_file_uploads/generic_upload.py @@ -4,14 +4,12 @@ dropdown selection. The script: 1. Determines user's schema from metadata table based on primary_group_id -2. Maps dropdown selection to table name via UPLOAD_OPTIONS config -3. Drops existing table if present and imports CSV -4. Sends email notification on completion +2. Drops existing table if present and imports CSV +3. Sends email notification on completion In order to function correctly, the following environment variables must be set: -- UPLOAD_OPTION: Dropdown selection indicating type of data being uploaded - (configured in this script) +- TABLE_NAME: Platform dropdown selection that maps to the table name - DATABASE: Target Civis database name - METADATA_TABLE: Civis table containing mapping of primary_group_id to schema_name @@ -21,7 +19,7 @@ The template script will also need to accept the following parameters: - FILE: Civis file parameter - EMAIL: Optional email address for notification -- UPLOAD_OPTION: Dropdown selection for upload type +- TABLE_NAME: Platform dropdown selection that maps to the table name - TESTING: Optional boolean to indicate testing mode (skip email) """ @@ -33,12 +31,7 @@ import pandas as pd LOG = civis.loggers.civis_logger() -# Configuration -UPLOAD_OPTIONS = { - "People Data": "people_data", - "Sales Data": "sales_data", - "Inventory Data": "inventory_data", -} + def get_schema(metadata_table: str, database: str, client=None) -> str: @@ -80,15 +73,6 @@ def get_schema(metadata_table: str, database: str, client=None) -> str: return schema, user_email -def get_table(upload_option: str): - # Get table name from dropdown selection - if upload_option not in UPLOAD_OPTIONS: - raise ValueError( - f"Invalid upload option '{upload_option}'. " - f"Valid options: {list(UPLOAD_OPTIONS.keys())}" - ) - return UPLOAD_OPTIONS[upload_option] - def download_data_create_table( file_id: int, full_table: str, database: str, client=None @@ -177,7 +161,7 @@ def send_email_notification( def main( file_id: int, - upload_option: str, + table_name: str, database: str, metadata_table: str, email_address: str = None, @@ -193,8 +177,6 @@ def main( client=client, ) - table_name = get_table(upload_option) - full_table = f"{schema}.{table_name}" LOG.info(f"Target table: {full_table}") @@ -224,7 +206,7 @@ def main( if __name__ == "__main__": # Get environment variables file_id = int(os.environ["FILE_ID"]) - upload_option = os.environ["UPLOAD_OPTION"] + table_name = os.environ["TABLE_NAME"] database = os.environ["DATABASE"] metadata_table = os.environ["METADATA_TABLE"] email_address = os.getenv("EMAIL") @@ -232,7 +214,7 @@ def main( main( file_id=file_id, - upload_option=upload_option, + table_name=table_name, database=database, metadata_table=metadata_table, email_address=email_address, From 11765cf860ed2535a384001b70742e611b81378d Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Mon, 2 Feb 2026 12:25:22 -0700 Subject: [PATCH 03/13] update boolean handling --- python/custom_file_uploads/generic_upload.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/python/custom_file_uploads/generic_upload.py b/python/custom_file_uploads/generic_upload.py index b0655d0..6b392fc 100644 --- a/python/custom_file_uploads/generic_upload.py +++ b/python/custom_file_uploads/generic_upload.py @@ -25,7 +25,6 @@ import os import tempfile -from distutils.util import strtobool import civis import pandas as pd @@ -113,18 +112,16 @@ def send_email_notification( full_table: str, database: str, user_email: str, - upload_option: str, file_obj: dict, testing: bool = False, client=None, ): client = client or civis.APIClient() recipient_email = email_address if email_address else user_email - email_subject = f"Data Upload Complete: {upload_option}" + email_subject = f"Data Upload Complete" email_body = f"""Your data upload has been completed successfully. File: {file_obj['name']} -Upload Type: {upload_option} Database: {database} Schema: {schema} Table: {table_name} @@ -194,7 +191,6 @@ def main( full_table=full_table, database=database, user_email=user_email, - upload_option=upload_option, file_obj=client.files.get(file_id), testing=testing, client=client, @@ -210,7 +206,8 @@ def main( database = os.environ["DATABASE"] metadata_table = os.environ["METADATA_TABLE"] email_address = os.getenv("EMAIL") - testing = strtobool(os.getenv("TESTING", "false")) + testing = os.getenv("TESTING", "false") + print(testing) main( file_id=file_id, From b7de0058d4036f9bf886c8b6cb02ebe074580299 Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Mon, 2 Feb 2026 12:30:26 -0700 Subject: [PATCH 04/13] update boolean handling again --- python/custom_file_uploads/generic_upload.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/custom_file_uploads/generic_upload.py b/python/custom_file_uploads/generic_upload.py index 6b392fc..3cf769d 100644 --- a/python/custom_file_uploads/generic_upload.py +++ b/python/custom_file_uploads/generic_upload.py @@ -32,7 +32,6 @@ LOG = civis.loggers.civis_logger() - def get_schema(metadata_table: str, database: str, client=None) -> str: client = client or civis.APIClient() @@ -206,9 +205,7 @@ def main( database = os.environ["DATABASE"] metadata_table = os.environ["METADATA_TABLE"] email_address = os.getenv("EMAIL") - testing = os.getenv("TESTING", "false") - print(testing) - + testing = int(os.getenv("TESTING", 0)) == 1 main( file_id=file_id, table_name=table_name, From df88c66de2911bc919fa01e260d6ec8320609018 Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Mon, 2 Feb 2026 12:39:21 -0700 Subject: [PATCH 05/13] handle permisions better --- python/custom_file_uploads/generic_upload.py | 33 +++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/python/custom_file_uploads/generic_upload.py b/python/custom_file_uploads/generic_upload.py index 3cf769d..f688962 100644 --- a/python/custom_file_uploads/generic_upload.py +++ b/python/custom_file_uploads/generic_upload.py @@ -63,15 +63,19 @@ def get_schema(metadata_table: str, database: str, client=None) -> str: # Create schema if needed LOG.info(f"Creating schema {schema} if needed") - civis.io.query_civis( - f"CREATE SCHEMA IF NOT EXISTS {schema};", - database=database, - ).result() + try: + civis.io.query_civis( + f"CREATE SCHEMA IF NOT EXISTS {schema};", + database=database, + ).result() + except Exception: + LOG.warning("""You do not have permissions to create schemas. + Script will continue and raise an error later + if the schema doesn't exist""") return schema, user_email - def download_data_create_table( file_id: int, full_table: str, database: str, client=None ): @@ -95,13 +99,18 @@ def download_data_create_table( # Import data to table LOG.info(f"Importing data to {full_table}") - civis.io.dataframe_to_civis( - df=df, - database=database, - table=full_table, - existing_table_rows="fail", - ) - LOG.info(f"Successfully uploaded {len(df)} rows to {full_table}") + try: + civis.io.dataframe_to_civis( + df=df, + database=database, + table=full_table, + existing_table_rows="fail", + ) + LOG.info(f"Successfully uploaded {len(df)} rows to {full_table}") + except Exception as e: + LOG.error(f"""Failed to upload data to {full_table}: {e} + Please check that the schema exists and you have + permissions to write to it.""") def send_email_notification( From 827c7d833a5cfe0544a5a94600fc8463c973da73 Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Mon, 2 Feb 2026 12:42:05 -0700 Subject: [PATCH 06/13] add a readme --- python/custom_file_uploads/README.md | 122 +++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 python/custom_file_uploads/README.md diff --git a/python/custom_file_uploads/README.md b/python/custom_file_uploads/README.md new file mode 100644 index 0000000..b3b07d2 --- /dev/null +++ b/python/custom_file_uploads/README.md @@ -0,0 +1,122 @@ +# Generic File Upload Script + +This script provides a lightweight, generalized CSV upload solution for Civis Platform. It handles uploading CSV files to database tables based on a user's primary group ID and a metadata table that defines schema mappings. + +## How It Works + +The script performs the following operations: + +1. **Schema Determination**: Looks up the user's schema from a metadata table based on their `primary_group_id` +2. **Table Creation**: Drops the existing table if present and imports the CSV data +3. **Email Notification**: Sends an email notification upon successful completion + +## Key Features + +- **Automatic Schema Assignment**: The script automatically determines which schema to use based on the user's primary group ID +- **Metadata-Driven Configuration**: Schema mappings are stored in a metadata table, making it easy to configure without code changes +- **Table Name Configuration**: Table names are set directly in Civis Platform using the "parameters" feature +- **Email Notifications**: Optional email notifications upon successful upload + +## Setup Requirements + +### 1. Create a Metadata Table + +Create a metadata table in your Civis database with the following structure: + +```sql +CREATE TABLE your_database.your_schema.metadata_data_upload_mmk ( + primary_group_id INTEGER, + schema_name VARCHAR +); +``` + +Populate this table with mappings of primary group IDs to their corresponding schemas: + +```sql +INSERT INTO your_database.your_schema.metadata_data_upload_mmk +VALUES + (123, 'team_a_schema'), + (456, 'team_b_schema'), + (789, 'team_c_schema'); +``` + +### 2. Create a Notification Script + +Create a blank Python script in Civis Platform that will be used to send notification emails. This script doesn't need to contain any code - it's only used to trigger the email notification system. Note the script ID for use in the configuration. + +### 3. Set Up the Container Script in Platform + +When setting up this script in Civis Platform, create a container script with the following configuration: + +```bash +cd /app; +export DATABASE='redshift-general' +export TESTING=0 +export EMAIL_RECIPIENTS="" +export METADATA_TABLE="metadata_data_upload_mmk" +export EMAIL_SCRIPT_ID="340695845" +python python/custom_file_uploads/generic_upload.py +``` + +**Configuration Variables:** + +- `DATABASE`: The name of your Civis database (e.g., 'redshift-general') +- `TESTING`: Set to `1` to skip email notifications (for testing), `0` for production +- `EMAIL_RECIPIENTS`: Optional email addresses for notifications (defaults to user's email) +- `METADATA_TABLE`: The full table name (including schema) of your metadata table +- `EMAIL_SCRIPT_ID`: The ID of the blank notification script you created in step 2 + +### 4. Configure Script Parameters + +In the Civis Platform script configuration, add the following parameters: + +- **FILE**: File parameter (required) - Users will upload their CSV file here +- **TABLE_NAME**: Dropdown or text parameter (required) - The name of the target table +- **EMAIL**: Text parameter (optional) - Email address for notification +- **TESTING**: Boolean parameter (optional) - Set to true to skip sending emails + +## Usage + +Once configured, users can run the script by: + +1. Uploading a CSV file via the FILE parameter +2. Selecting or entering the target table name via the TABLE_NAME parameter +3. Optionally providing an email address for notification +4. Running the script + +The script will: +- Automatically determine the correct schema based on the user's primary group +- Drop and recreate the table with the uploaded CSV data +- Send a notification email upon completion + +## Example Metadata Table Setup + +Here's a complete example for setting up your metadata table: + +```sql +-- Create the metadata table +CREATE TABLE your_database.your_schema.metadata_data_upload_mmk ( + primary_group_id INTEGER, + schema_name VARCHAR +); + +-- Add your group mappings +INSERT INTO your_database.your_schema.metadata_data_upload_mmk + (primary_group_id, schema_name) +VALUES + (123, 'analytics_team'), + (456, 'marketing_team'), + (789, 'operations_team'); +``` + +## Troubleshooting + +- **"No schema mapping found"**: Ensure your primary group ID is added to the metadata table +- **Schema creation errors**: You may need database permissions to create schemas, or ensure the schema already exists +- **Email not received**: Check that the SCRIPT_ID points to a valid script and TESTING is set to 0 + +## Notes + +- The script will drop and recreate the table on each run, so existing data will be replaced +- Users must have appropriate database permissions to write to their assigned schema +- The metadata table must be readable by all users who will run this script From 0a6a8238c060475b779514659196df29db5e6992 Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Mon, 2 Feb 2026 12:44:43 -0700 Subject: [PATCH 07/13] remove email testing logs --- python/custom_file_uploads/generic_upload.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/custom_file_uploads/generic_upload.py b/python/custom_file_uploads/generic_upload.py index f688962..7361267 100644 --- a/python/custom_file_uploads/generic_upload.py +++ b/python/custom_file_uploads/generic_upload.py @@ -155,11 +155,6 @@ def send_email_notification( }, ) future = civis.utils.run_job(email_script_id, client=client).result() - - if not future.succeeded(): - LOG.warning("Unable to send notification email") - else: - LOG.info(f"Email sent to {recipient_email}") else: LOG.info(f"Testing mode: skipping email to {recipient_email}") From c009a41eded0d27854b6641ef097c2f928b8f98d Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Mon, 2 Feb 2026 12:52:42 -0700 Subject: [PATCH 08/13] updaet the read me --- python/custom_file_uploads/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/custom_file_uploads/README.md b/python/custom_file_uploads/README.md index b3b07d2..0193579 100644 --- a/python/custom_file_uploads/README.md +++ b/python/custom_file_uploads/README.md @@ -14,7 +14,7 @@ The script performs the following operations: - **Automatic Schema Assignment**: The script automatically determines which schema to use based on the user's primary group ID - **Metadata-Driven Configuration**: Schema mappings are stored in a metadata table, making it easy to configure without code changes -- **Table Name Configuration**: Table names are set directly in Civis Platform using the "parameters" feature +- **Table Name Configuration**: Table names are set directly in Civis Platform using the "parameters" feature, if you set up a dropdown parameter then the "value name" will be used as the table name. - **Email Notifications**: Optional email notifications upon successful upload ## Setup Requirements @@ -62,7 +62,6 @@ python python/custom_file_uploads/generic_upload.py - `DATABASE`: The name of your Civis database (e.g., 'redshift-general') - `TESTING`: Set to `1` to skip email notifications (for testing), `0` for production -- `EMAIL_RECIPIENTS`: Optional email addresses for notifications (defaults to user's email) - `METADATA_TABLE`: The full table name (including schema) of your metadata table - `EMAIL_SCRIPT_ID`: The ID of the blank notification script you created in step 2 From f9e005160630f9e332f75b46f3079bd71fc55f73 Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Tue, 3 Mar 2026 08:52:11 -0700 Subject: [PATCH 09/13] linting --- python/custom_file_uploads/README.md | 7 ------- python/custom_file_uploads/generic_upload.py | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/python/custom_file_uploads/README.md b/python/custom_file_uploads/README.md index 0193579..9347a13 100644 --- a/python/custom_file_uploads/README.md +++ b/python/custom_file_uploads/README.md @@ -10,13 +10,6 @@ The script performs the following operations: 2. **Table Creation**: Drops the existing table if present and imports the CSV data 3. **Email Notification**: Sends an email notification upon successful completion -## Key Features - -- **Automatic Schema Assignment**: The script automatically determines which schema to use based on the user's primary group ID -- **Metadata-Driven Configuration**: Schema mappings are stored in a metadata table, making it easy to configure without code changes -- **Table Name Configuration**: Table names are set directly in Civis Platform using the "parameters" feature, if you set up a dropdown parameter then the "value name" will be used as the table name. -- **Email Notifications**: Optional email notifications upon successful upload - ## Setup Requirements ### 1. Create a Metadata Table diff --git a/python/custom_file_uploads/generic_upload.py b/python/custom_file_uploads/generic_upload.py index 7361267..172f520 100644 --- a/python/custom_file_uploads/generic_upload.py +++ b/python/custom_file_uploads/generic_upload.py @@ -126,7 +126,7 @@ def send_email_notification( ): client = client or civis.APIClient() recipient_email = email_address if email_address else user_email - email_subject = f"Data Upload Complete" + email_subject = "Data Upload Complete" email_body = f"""Your data upload has been completed successfully. File: {file_obj['name']} @@ -154,7 +154,7 @@ def send_email_notification( "success_email_addresses": [recipient_email], }, ) - future = civis.utils.run_job(email_script_id, client=client).result() + civis.utils.run_job(email_script_id, client=client).result() else: LOG.info(f"Testing mode: skipping email to {recipient_email}") From f4f90c81f3a1c41a9b59322259dcddfca7e7b958 Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Tue, 3 Mar 2026 15:46:37 -0700 Subject: [PATCH 10/13] update send email to only send on success --- python/custom_file_uploads/generic_upload.py | 29 +++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/python/custom_file_uploads/generic_upload.py b/python/custom_file_uploads/generic_upload.py index 172f520..f3bc45b 100644 --- a/python/custom_file_uploads/generic_upload.py +++ b/python/custom_file_uploads/generic_upload.py @@ -107,10 +107,12 @@ def download_data_create_table( existing_table_rows="fail", ) LOG.info(f"Successfully uploaded {len(df)} rows to {full_table}") + return 'scuccess' except Exception as e: LOG.error(f"""Failed to upload data to {full_table}: {e} Please check that the schema exists and you have permissions to write to it.""") + return 'failure' def send_email_notification( @@ -180,26 +182,27 @@ def main( full_table = f"{schema}.{table_name}" LOG.info(f"Target table: {full_table}") - download_data_create_table( + status = download_data_create_table( file_id=file_id, full_table=full_table, database=database, client=client, ) - send_email_notification( - email_address=email_address, - table_name=table_name, - schema=schema, - full_table=full_table, - database=database, - user_email=user_email, - file_obj=client.files.get(file_id), - testing=testing, - client=client, - ) + if status == 'success': + send_email_notification( + email_address=email_address, + table_name=table_name, + schema=schema, + full_table=full_table, + database=database, + user_email=user_email, + file_obj=client.files.get(file_id), + testing=testing, + client=client, + ) - LOG.info("Upload process completed successfully") + LOG.info("Upload process completed successfully") if __name__ == "__main__": From 74e6593aa3b6c605f90512121a515c0d356d7264 Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Tue, 3 Mar 2026 15:49:47 -0700 Subject: [PATCH 11/13] update readme --- python/custom_file_uploads/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/custom_file_uploads/README.md b/python/custom_file_uploads/README.md index 9347a13..6e041fc 100644 --- a/python/custom_file_uploads/README.md +++ b/python/custom_file_uploads/README.md @@ -45,7 +45,7 @@ When setting up this script in Civis Platform, create a container script with th cd /app; export DATABASE='redshift-general' export TESTING=0 -export EMAIL_RECIPIENTS="" +export EMAIL="" export METADATA_TABLE="metadata_data_upload_mmk" export EMAIL_SCRIPT_ID="340695845" python python/custom_file_uploads/generic_upload.py @@ -65,7 +65,7 @@ In the Civis Platform script configuration, add the following parameters: - **FILE**: File parameter (required) - Users will upload their CSV file here - **TABLE_NAME**: Dropdown or text parameter (required) - The name of the target table - **EMAIL**: Text parameter (optional) - Email address for notification -- **TESTING**: Boolean parameter (optional) - Set to true to skip sending emails +- **TESTING**: Numeric 0/1 (optional) - Set to true to skip sending emails ## Usage From c832866b29eab9819f669f2a873e315aed3f0d13 Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Tue, 3 Mar 2026 15:52:36 -0700 Subject: [PATCH 12/13] update per copilot review --- python/custom_file_uploads/README.md | 2 +- python/custom_file_uploads/generic_upload.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/python/custom_file_uploads/README.md b/python/custom_file_uploads/README.md index 6e041fc..0ff546c 100644 --- a/python/custom_file_uploads/README.md +++ b/python/custom_file_uploads/README.md @@ -105,7 +105,7 @@ VALUES - **"No schema mapping found"**: Ensure your primary group ID is added to the metadata table - **Schema creation errors**: You may need database permissions to create schemas, or ensure the schema already exists -- **Email not received**: Check that the SCRIPT_ID points to a valid script and TESTING is set to 0 +- **Email not received**: Check that the EMAIL_SCRIPT_ID points to a valid script and TESTING is set to 0 ## Notes diff --git a/python/custom_file_uploads/generic_upload.py b/python/custom_file_uploads/generic_upload.py index f3bc45b..71a2569 100644 --- a/python/custom_file_uploads/generic_upload.py +++ b/python/custom_file_uploads/generic_upload.py @@ -32,7 +32,7 @@ LOG = civis.loggers.civis_logger() -def get_schema(metadata_table: str, database: str, client=None) -> str: +def get_schema(metadata_table: str, database: str, client=None) -> tuple[str, str]: client = client or civis.APIClient() # Get user info @@ -69,9 +69,11 @@ def get_schema(metadata_table: str, database: str, client=None) -> str: database=database, ).result() except Exception: - LOG.warning("""You do not have permissions to create schemas. + LOG.warning( + """You do not have permissions to create schemas. Script will continue and raise an error later - if the schema doesn't exist""") + if the schema doesn't exist""" + ) return schema, user_email @@ -107,12 +109,14 @@ def download_data_create_table( existing_table_rows="fail", ) LOG.info(f"Successfully uploaded {len(df)} rows to {full_table}") - return 'scuccess' + return "scuccess" except Exception as e: - LOG.error(f"""Failed to upload data to {full_table}: {e} + LOG.error( + f"""Failed to upload data to {full_table}: {e} Please check that the schema exists and you have - permissions to write to it.""") - return 'failure' + permissions to write to it.""" + ) + return "failure" def send_email_notification( @@ -189,7 +193,7 @@ def main( client=client, ) - if status == 'success': + if status == "success": send_email_notification( email_address=email_address, table_name=table_name, From 6aff09da1d0d167de548a43c1a0c08708685b618 Mon Sep 17 00:00:00 2001 From: Maya Mallaby-Kay Date: Mon, 9 Mar 2026 08:55:25 -0600 Subject: [PATCH 13/13] update readme --- python/custom_file_uploads/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/custom_file_uploads/README.md b/python/custom_file_uploads/README.md index 0ff546c..080c408 100644 --- a/python/custom_file_uploads/README.md +++ b/python/custom_file_uploads/README.md @@ -17,7 +17,7 @@ The script performs the following operations: Create a metadata table in your Civis database with the following structure: ```sql -CREATE TABLE your_database.your_schema.metadata_data_upload_mmk ( +CREATE TABLE your_database.your_schema.metadata_data_upload( primary_group_id INTEGER, schema_name VARCHAR ); @@ -26,7 +26,7 @@ CREATE TABLE your_database.your_schema.metadata_data_upload_mmk ( Populate this table with mappings of primary group IDs to their corresponding schemas: ```sql -INSERT INTO your_database.your_schema.metadata_data_upload_mmk +INSERT INTO your_database.your_schema.metadata_data_upload VALUES (123, 'team_a_schema'), (456, 'team_b_schema'), @@ -46,7 +46,7 @@ cd /app; export DATABASE='redshift-general' export TESTING=0 export EMAIL="" -export METADATA_TABLE="metadata_data_upload_mmk" +export METADATA_TABLE="metadata_data_upload" export EMAIL_SCRIPT_ID="340695845" python python/custom_file_uploads/generic_upload.py ``` @@ -87,13 +87,13 @@ Here's a complete example for setting up your metadata table: ```sql -- Create the metadata table -CREATE TABLE your_database.your_schema.metadata_data_upload_mmk ( +CREATE TABLE your_database.your_schema.metadata_data_upload ( primary_group_id INTEGER, schema_name VARCHAR ); -- Add your group mappings -INSERT INTO your_database.your_schema.metadata_data_upload_mmk +INSERT INTO your_database.your_schema.metadata_data_upload (primary_group_id, schema_name) VALUES (123, 'analytics_team'),