Upsert rather than clear collections every time we update the db

init_ratings.py

@@ -5,8 +5,15 @@
 from src.scraper.ratinglists.updater import update_cfc_rating_list, update_fide_rating_list, update_uscf_rating_list
 from src.scraper.ratinglists.db import reset_collections
 
-# Get download flag from command line argument
+# Parse command line arguments
 DOWNLOAD_LATEST = False
+RESET_DB = False
+
+for arg in sys.argv[1:]:
+    if arg == "--download":
+        DOWNLOAD_LATEST = True
+    elif arg == "--reset":
+        RESET_DB = True
 
 # Configure logging
 logging.basicConfig(
@@ -18,11 +25,15 @@
 try:
     logger.info("Starting rating list initialization...")
     logger.info(f"Download latest ratings: {DOWNLOAD_LATEST}")
+    logger.info(f"Reset database: {RESET_DB}")
 
-    # First reset collections to ensure clean state
-    logger.info("Resetting database collections...")
-    reset_result = reset_collections()
-    logger.info(f"Reset result: {'Successful' if reset_result else 'Failed'}")
+    # Only reset collections if explicitly requested
+    if RESET_DB:
+        logger.info("Resetting database collections (--reset flag detected)...")
+        reset_result = reset_collections()
+        logger.info(f"Reset result: {'Successful' if reset_result else 'Failed'}")
+    else:
+        logger.info("Using incremental update mode - existing records will be updated, not replaced")
 
     if DOWNLOAD_LATEST:
         # Download and parse the latest rating lists

initialize_rating_lists.sh

@@ -2,20 +2,26 @@
 # This script initializes both FIDE and CFC rating lists
 # to run this script, make sure you have Python 3 and pip installed
 # and that you have the required packages installed in your Python environment
-# Usage: ./initialize_rating_lists.sh [--no-download]
+# Usage: ./initialize_rating_lists.sh [--no-download] [--reset]
 
 # Parse command line arguments
 DOWNLOAD_LATEST=True
+RESET_DB=False
 
 while [[ $# -gt 0 ]]; do
     case $1 in
         --no-download)
             DOWNLOAD_LATEST=False
             shift
             ;;
+        --reset)
+            RESET_DB=True
+            shift
+            ;;
         -h|--help)
-            echo "Usage: $0 [--no-download]"
+            echo "Usage: $0 [--no-download] [--reset]"
             echo "  --no-download    Skip downloading latest rating lists, use existing data"
+            echo "  --reset          Drop and recreate collections before loading data"
             echo "  -h, --help       Show this help message"
             exit 0
             ;;
@@ -34,6 +40,12 @@ else
     echo "Skipping download, using existing rating lists..."
 fi
 
+if [ "$RESET_DB" = True ]; then
+    echo "Will reset database collections..."
+else
+    echo "Using incremental update mode - existing data will be preserved..."
+fi
+
 # Make sure we're in the project directory
 cd "$(dirname "$0")"
 
@@ -46,8 +58,9 @@ from src.scraper.ratinglists.parsers import parse_fide_rating_list, parse_cfc_ra
 from src.scraper.ratinglists.updater import update_cfc_rating_list, update_fide_rating_list, update_uscf_rating_list
 from src.scraper.ratinglists.db import reset_collections
 
-# Get download flag from command line argument
+# Parse command line arguments
 DOWNLOAD_LATEST = ${DOWNLOAD_LATEST}
+RESET_DB = ${RESET_DB}
 
 # Configure logging
 logging.basicConfig(
@@ -59,11 +72,15 @@ logger = logging.getLogger('rating_list_init')
 try:
     logger.info("Starting rating list initialization...")
     logger.info(f"Download latest ratings: {DOWNLOAD_LATEST}")
+    logger.info(f"Reset database: {RESET_DB}")
 
-    # First reset collections to ensure clean state
-    logger.info("Resetting database collections...")
-    reset_result = reset_collections()
-    logger.info(f"Reset result: {'Successful' if reset_result else 'Failed'}")
+    # Only reset collections if explicitly requested
+    if RESET_DB:
+        logger.info("Resetting database collections (--reset flag detected)...")
+        reset_result = reset_collections()
+        logger.info(f"Reset result: {'Successful' if reset_result else 'Failed'}")
+    else:
+        logger.info("Using incremental update mode - existing records will be updated, not replaced")
 
     if DOWNLOAD_LATEST:
         # Download and parse the latest rating lists