From 0e567a864f229963d102e379f79ef197ba04e607 Mon Sep 17 00:00:00 2001 From: Cole Aydelotte Date: Mon, 6 Apr 2026 15:19:29 -0400 Subject: [PATCH 1/2] Added frdocnum field to query. :) --- postgres/IngestDocument.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/postgres/IngestDocument.py b/postgres/IngestDocument.py index 87f92bb..805bd54 100644 --- a/postgres/IngestDocument.py +++ b/postgres/IngestDocument.py @@ -69,6 +69,7 @@ def insert_document(conn, json_data): attributes.get("topics"), attributes.get("withdrawn"), attributes.get("zip"), + attributes.get("frDocNum"), ) # Insert into the database @@ -85,8 +86,8 @@ def insert_document(conn, json_data): submitter_org, phone, posted_date, postmark_date, reason_withdrawn, receive_date, reg_writer_instruction, restriction_reason, restriction_reason_type, state_province_region, subtype, - document_title, topics, is_withdrawn, postal_code - ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s); + document_title, topics, is_withdrawn, postal_code, frdocnum + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s); """ cursor.execute(insert_query, values) print(f"Document {document_id} inserted successfully.") @@ -94,7 +95,6 @@ def insert_document(conn, json_data): print(f"An error occurred: {e}") -# Main function to read JSON from file and insert into the database # Main function to read JSON from file and insert into the database def main(): if len(sys.argv) != 2: @@ -129,6 +129,5 @@ def main(): except Exception as e: print(f"An error occurred: {e}") - if __name__ == "__main__": main() From 3268e3cbdece99f6d31b45ce2b1179a7962a1a4f Mon Sep 17 00:00:00 2001 From: Cole Aydelotte Date: Mon, 6 Apr 2026 15:36:53 -0400 Subject: [PATCH 2/2] Added missing fields present on production schema. --- postgres/IngestDocument.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/postgres/IngestDocument.py b/postgres/IngestDocument.py index 805bd54..73333bf 100644 --- a/postgres/IngestDocument.py +++ b/postgres/IngestDocument.py @@ -70,8 +70,18 @@ def insert_document(conn, json_data): attributes.get("withdrawn"), attributes.get("zip"), attributes.get("frDocNum"), + attributes.get("relationships").get("attachments").get("links").get("self"), + attributes.get("relationships").get("attachments").get("links").get("related"), + attributes.get("fileFormats"), + attributes.get("displayProperties"), ) + # attachments_self_link VARCHAR(2000), + # attachments_related_link VARCHAR(2000), + # file_formats JSONB, + # display_properties JSONB + + # Insert into the database try: with conn.cursor() as cursor: @@ -86,8 +96,9 @@ def insert_document(conn, json_data): submitter_org, phone, posted_date, postmark_date, reason_withdrawn, receive_date, reg_writer_instruction, restriction_reason, restriction_reason_type, state_province_region, subtype, - document_title, topics, is_withdrawn, postal_code, frdocnum - ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s); + document_title, topics, is_withdrawn, postal_code, frdocnum, + attachments_self_link, attachments_related_link, file_formats, display_properties + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s); """ cursor.execute(insert_query, values) print(f"Document {document_id} inserted successfully.")