event notification file change

Author: networkandcode

structured-kb-demo/.env.example

@@ -1,11 +1,18 @@
-ACCOUNT_ID=
+AWS_ACCOUNT_ID=
 AWS_ACCESS_KEY_ID=
+AWS_REGION=
 AWS_SECRET_ACCESS_KEY=
-BUCKET=
-CRAWLER_IAM_POLICY=
-CRAWLER_IAM_ROLE=
-CRAWLER=
-DB=
-FOLDER=
-QUEUE=
-REGION=
+
+GLUE_CRAWLER=
+GLUE_CRAWLER_IAM_POLICY=
+GLUE_CRAWLER_IAM_ROLE=
+GLUE_DB=
+
+REDSHIFT_IAM_ROLE=
+REDSHIFT_NAMESPACE=
+REDSHIFT_WORKGROUP=
+
+S3_BUCKET=
+S3_BUCKET_FOLDER=
+
+SQS_QUEUE=

structured-kb-demo/arns.py

@@ -1,17 +1,17 @@
 from vars import (
-    ACCOUNT_ID,
-    BUCKET,
-    CRAWLER_IAM_POLICY,
-    CRAWLER_IAM_ROLE,
+    AWS_ACCOUNT_ID,
+    S3_BUCKET,
+    GLUE_CRAWLER_IAM_POLICY,
+    GLUE_CRAWLER_IAM_ROLE,
     REDSHIFT_IAM_ROLE,
-    REGION,
-    QUEUE,
+    AWS_REGION,
+    SQS_QUEUE,
 )
 
 AWS_MANAGED_GLUE_IAM_POLICY_ARN = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
 AWS_MANAGED_REDSHIFT_IAM_POLICY_ARN = "arn:aws:iam::aws:policy/AmazonRedshiftAllCommandsFullAccess"
-BUCKET_ARN = f"arn:aws:s3:::{BUCKET}"
-CRAWLER_IAM_POLICY_ARN = f"arn:aws:iam::{ACCOUNT_ID}:policy/{CRAWLER_IAM_POLICY}"
-CRAWLER_IAM_ROLE_ARN = f"arn:aws:iam::{ACCOUNT_ID}:role/{CRAWLER_IAM_ROLE}"
-QUEUE_ARN = f"arn:aws:sqs:{REGION}:{ACCOUNT_ID}:{QUEUE}"
-REDSHIFT_IAM_ROLE_ARN = f"arn:aws:iam::{ACCOUNT_ID}:role/{REDSHIFT_IAM_ROLE}"
+S3_BUCKET_ARN = f"arn:aws:s3:::{S3_BUCKET}"
+GLUE_CRAWLER_IAM_POLICY_ARN = f"arn:aws:iam::{AWS_ACCOUNT_ID}:policy/{GLUE_CRAWLER_IAM_POLICY}"
+GLUE_CRAWLER_IAM_ROLE_ARN = f"arn:aws:iam::{AWS_ACCOUNT_ID}:role/{GLUE_CRAWLER_IAM_ROLE}"
+SQS_QUEUE_ARN = f"arn:aws:sqs:{AWS_REGION}:{AWS_ACCOUNT_ID}:{SQS_QUEUE}"
+REDSHIFT_IAM_ROLE_ARN = f"arn:aws:iam::{AWS_ACCOUNT_ID}:role/{REDSHIFT_IAM_ROLE}"

structured-kb-demo/setup_glue_crawler.py

@@ -1,6 +1,6 @@
 import boto3
 
-from arns import CRAWLER_IAM_ROLE_ARN, QUEUE_ARN
+from arns import GLUE_CRAWLER_IAM_ROLE_ARN, QUEUE_ARN
 from logger import logger
 from vars import BUCKET, CRAWLER, DB, FOLDER
 
@@ -9,7 +9,7 @@
 glue = boto3.client("glue", region_name="us-east-1")
 
 CRAWLER_CONFIG = {
-    "Role": CRAWLER_IAM_ROLE_ARN,
+    "Role": GLUE_CRAWLER_IAM_ROLE_ARN,
     "DatabaseName": DB,
     "Description": "Crawler for inventory data triggered by SQS events",
     "Targets": {

structured-kb-demo/setup_glue_crawler_iam_policy.py

@@ -2,10 +2,10 @@
 
 import json
 
-from arns import BUCKET_ARN, CRAWLER_IAM_POLICY_ARN, QUEUE_ARN
+from arns import BUCKET_ARN, GLUE_CRAWLER_IAM_POLICY_ARN, QUEUE_ARN
 
 from logger import logger
-from vars import CRAWLER_IAM_POLICY
+from vars import GLUE_CRAWLER_IAM_POLICY
 
 iam = boto3.client('iam')
 
@@ -39,7 +39,7 @@
 
 try:
     iam.create_policy(
-        PolicyName=CRAWLER_IAM_POLICY,
+        PolicyName=GLUE_CRAWLER_IAM_POLICY,
         PolicyDocument=json.dumps(policy_document),
         Description='Permissions for Glue Crawler to crawl S3 and use SQS Events'
     )
@@ -50,7 +50,7 @@
     try:
         # Create a new policy version
         iam.create_policy_version(
-            PolicyArn=CRAWLER_IAM_POLICY_ARN,
+            PolicyArn=GLUE_CRAWLER_IAM_POLICY_ARN,
             PolicyDocument=json.dumps(policy_document),
             SetAsDefault=True
         )

structured-kb-demo/setup_glue_crawler_iam_role.py

@@ -2,9 +2,9 @@
 
 import json
 
-from arns import AWS_MANAGED_GLUE_IAM_POLICY_ARN, CRAWLER_IAM_POLICY_ARN
+from arns import AWS_MANAGED_GLUE_IAM_POLICY_ARN, GLUE_CRAWLER_IAM_POLICY_ARN
 from logger import logger
-from vars import ACCOUNT_ID, CRAWLER_IAM_ROLE
+from vars import ACCOUNT_ID, GLUE_CRAWLER_IAM_ROLE
 
 iam = boto3.client("iam")
 
@@ -28,7 +28,7 @@
 
 try:
     iam.create_role(
-        RoleName=CRAWLER_IAM_ROLE,
+        RoleName=GLUE_CRAWLER_IAM_ROLE,
         AssumeRolePolicyDocument=json.dumps(trust_policy)
     )
     logger.info(f"Created role")
@@ -37,22 +37,22 @@
 
 
 # Get existing attached policies
-response = iam.list_attached_role_policies(RoleName=CRAWLER_IAM_ROLE)
+response = iam.list_attached_role_policies(RoleName=GLUE_CRAWLER_IAM_ROLE)
 attached_policies = [p['PolicyArn'] for p in response.get('AttachedPolicies', [])]
 
 if AWS_MANAGED_GLUE_IAM_POLICY_ARN not in attached_policies:
     iam.attach_role_policy(
-        RoleName=CRAWLER_IAM_ROLE,
+        RoleName=GLUE_CRAWLER_IAM_ROLE,
         PolicyArn=AWS_MANAGED_GLUE_IAM_POLICY_ARN
     )
     logger.info("AWS Glue Service Role policy attached.")
 else:
     logger.info("AWS Glue Service Role policy already attached.")
 
-if CRAWLER_IAM_POLICY_ARN not in attached_policies:
+if GLUE_CRAWLER_IAM_POLICY_ARN not in attached_policies:
     iam.attach_role_policy(
-        RoleName=CRAWLER_IAM_ROLE,
-        PolicyArn=CRAWLER_IAM_POLICY_ARN
+        RoleName=GLUE_CRAWLER_IAM_ROLE,
+        PolicyArn=GLUE_CRAWLER_IAM_POLICY_ARN
     )
     logger.info("Custom Glue Crawler policy attached.")
 else:

structured-kb-demo/setup_s3_bucket.py

@@ -3,16 +3,17 @@
 import boto3
 from botocore.exceptions import ClientError
 
-from vars import BUCKET, REGION
+from vars import S3_BUCKET, AWS_REGION
 
-s3_client = boto3.client('s3', region_name=REGION)
+s3_client = boto3.client('s3', region_name=AWS_REGION)
 
 try:
-    s3_client.head_bucket(Bucket=BUCKET)
-    logger.info(f"Bucket already exists")
+    s3_client.head_bucket(Bucket=S3_BUCKET)
+    logger.info(f"Bucket {S3_BUCKET} already exists")
 except ClientError as e:
-    logger.error(f"Error accessing bucket {BUCKET}: {e}")
+    logger.error(f"Error accessing bucket {S3_BUCKET}: {e}")
     s3_client.create_bucket(
-        Bucket=BUCKET
+        Bucket=S3_BUCKET,
+        CreateBucketConfiguration={'LocationConstraint': AWS_REGION}
     )
-    logger.info(f"Bucket created successfully")
+    logger.info(f"Bucket {S3_BUCKET} created successfully")

…ured-kb-demo/setup_event_notification.py …mo/setup_s3_to_sqs_event_notification.pystructured-kb-demo/setup_event_notification.py renamed to structured-kb-demo/setup_s3_to_sqs_event_notification.py structured-kb-demo/setup_event_notification.py renamed to structured-kb-demo/setup_s3_to_sqs_event_notification.py

@@ -2,11 +2,11 @@
 
 import boto3
 
-from arns import BUCKET_ARN, QUEUE_ARN
+from arns import S3_BUCKET_ARN, SQS_QUEUE_ARN
 from logger import logger
-from vars import QUEUE, REGION
+from vars import SQS_QUEUE, AWS_REGION
 
-sqs = boto3.client('sqs', region_name=REGION)
+sqs = boto3.client('sqs', region_name=AWS_REGION)
 
 access_policy = {
     "Statement": [
@@ -16,10 +16,10 @@
                 "Service": "s3.amazonaws.com"
             },
             "Action": "sqs:SendMessage",
-            "Resource": QUEUE_ARN,
+            "Resource": SQS_QUEUE_ARN,
             "Condition": {
                 "ArnLike": {
-                    "aws:SourceArn": BUCKET_ARN
+                    "aws:SourceArn": S3_BUCKET_ARN
                 }
             }
         }
@@ -28,12 +28,12 @@
 
 try:
     # Check if queue exists
-    sqs.get_queue_url(QueueName=QUEUE)
+    sqs.get_queue_url(QueueName=SQS_QUEUE)
     logger.info(f"Queue already exists.")
 except sqs.exceptions.QueueDoesNotExist:
     # Create queue if it doesn't exist
     sqs.create_queue(
-        QueueName=QUEUE,
+        QueueName=SQS_QUEUE,
         Attributes={
             'Policy': json.dumps(access_policy)
         }

structured-kb-demo/setup_sqs_queue.py

@@ -3,6 +3,7 @@
 
 from dotenv import load_dotenv
 
+
 @lru_cache(maxsize=1)
 def _load_env_once() -> os._Environ[str]:
 	load_dotenv()
@@ -11,15 +12,18 @@ def _load_env_once() -> os._Environ[str]:
 # load all env vars at once as a dictionary
 env_vars = _load_env_once()
 
-ACCOUNT_ID = env_vars["ACCOUNT_ID"]
-BUCKET = env_vars["BUCKET"]
-CRAWLER_IAM_POLICY = env_vars["CRAWLER_IAM_POLICY"]
-CRAWLER_IAM_ROLE = env_vars["CRAWLER_IAM_ROLE"]
-CRAWLER = env_vars["CRAWLER"]
-DB = env_vars["DB"]
-FOLDER = env_vars["FOLDER"]
-QUEUE = env_vars["QUEUE"]
+AWS_ACCOUNT_ID = env_vars["AWS_ACCOUNT_ID"]
+AWS_REGION = env_vars["AWS_REGION"]
+
+GLUE_CRAWLER_IAM_POLICY = env_vars["GLUE_CRAWLER_IAM_POLICY"]
+GLUE_CRAWLER_IAM_ROLE = env_vars["GLUE_CRAWLER_IAM_ROLE"]
+GLUE_CRAWLER = env_vars["GLUE_CRAWLER"]
+GLUE_DB = env_vars["GLUE_DB"]
+
+S3_BUCKET = env_vars["S3_BUCKET"]
+S3_BUCKET_FOLDER = env_vars["S3_BUCKET_FOLDER"]
+SQS_QUEUE = env_vars["SQS_QUEUE"]
+
 REDSHIFT_IAM_ROLE = env_vars["REDSHIFT_IAM_ROLE"]
 REDSHIFT_NAMESPACE = env_vars["REDSHIFT_NAMESPACE"]
 REDSHIFT_WORKGROUP = env_vars["REDSHIFT_WORKGROUP"]
-REGION = env_vars["REGION"]