v0.237.004

application/single_app/config.py

@@ -88,7 +88,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.237.003"
+VERSION = "0.237.004"
 
 
 SECRET_KEY = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production')

application/single_app/functions_control_center.py

@@ -0,0 +1,138 @@
+# functions_control_center.py
+"""
+Functions for Control Center operations including scheduled auto-refresh.
+Version: 0.237.004
+"""
+
+from datetime import datetime, timezone, timedelta
+from config import debug_print, cosmos_user_settings_container, cosmos_groups_container
+from functions_settings import get_settings, update_settings
+from functions_appinsights import log_event
+
+
+def execute_control_center_refresh(manual_execution=False):
+    """
+    Execute Control Center data refresh operation.
+    Refreshes user and group metrics data.
+
+    Args:
+        manual_execution: True if triggered manually, False if scheduled
+
+    Returns:
+        dict: Results containing success status and refresh counts
+    """
+    results = {
+        'success': True,
+        'refreshed_users': 0,
+        'failed_users': 0,
+        'refreshed_groups': 0,
+        'failed_groups': 0,
+        'error': None,
+        'manual_execution': manual_execution
+    }
+
+    try:
+        debug_print(f"🔄 [AUTO-REFRESH] Starting Control Center {'manual' if manual_execution else 'scheduled'} refresh...")
+
+        # Import enhance functions from route module
+        from route_backend_control_center import enhance_user_with_activity, enhance_group_with_activity
+
+        # Get all users to refresh their metrics
+        debug_print("🔄 [AUTO-REFRESH] Querying all users...")
+        users_query = "SELECT c.id, c.email, c.display_name, c.lastUpdated, c.settings FROM c"
+        all_users = list(cosmos_user_settings_container.query_items(
+            query=users_query,
+            enable_cross_partition_query=True
+        ))
+        debug_print(f"🔄 [AUTO-REFRESH] Found {len(all_users)} users to process")
+
+        # Refresh metrics for each user
+        for user in all_users:
+            try:
+                user_id = user.get('id')
+                debug_print(f"🔄 [AUTO-REFRESH] Processing user {user_id}")
+
+                # Force refresh of metrics for this user
+                enhanced_user = enhance_user_with_activity(user, force_refresh=True)
+                results['refreshed_users'] += 1
+
+            except Exception as user_error:
+                results['failed_users'] += 1
+                debug_print(f"❌ [AUTO-REFRESH] Failed to refresh user {user.get('id')}: {user_error}")
+
+        debug_print(f"🔄 [AUTO-REFRESH] User refresh completed. Refreshed: {results['refreshed_users']}, Failed: {results['failed_users']}")
+
+        # Refresh metrics for all groups
+        debug_print("🔄 [AUTO-REFRESH] Starting group refresh...")
+
+        try:
+            groups_query = "SELECT * FROM c"
+            all_groups = list(cosmos_groups_container.query_items(
+                query=groups_query,
+                enable_cross_partition_query=True
+            ))
+            debug_print(f"🔄 [AUTO-REFRESH] Found {len(all_groups)} groups to process")
+
+            # Refresh metrics for each group
+            for group in all_groups:
+                try:
+                    group_id = group.get('id')
+                    debug_print(f"🔄 [AUTO-REFRESH] Processing group {group_id}")
+
+                    # Force refresh of metrics for this group
+                    enhanced_group = enhance_group_with_activity(group, force_refresh=True)
+                    results['refreshed_groups'] += 1
+
+                except Exception as group_error:
+                    results['failed_groups'] += 1
+                    debug_print(f"❌ [AUTO-REFRESH] Failed to refresh group {group.get('id')}: {group_error}")
+
+        except Exception as groups_error:
+            debug_print(f"❌ [AUTO-REFRESH] Error querying groups: {groups_error}")
+
+        debug_print(f"🔄 [AUTO-REFRESH] Group refresh completed. Refreshed: {results['refreshed_groups']}, Failed: {results['failed_groups']}")
+
+        # Update admin settings with refresh timestamp and calculate next run time
+        try:
+            settings = get_settings()
+            if settings:
+                current_time = datetime.now(timezone.utc)
+                settings['control_center_last_refresh'] = current_time.isoformat()
+
+                # Calculate next scheduled auto-refresh time if enabled
+                if settings.get('control_center_auto_refresh_enabled', False):
+                    execution_hour = settings.get('control_center_auto_refresh_hour', 2)
+                    next_run = current_time.replace(hour=execution_hour, minute=0, second=0, microsecond=0)
+                    if next_run <= current_time:
+                        next_run += timedelta(days=1)
+                    settings['control_center_auto_refresh_next_run'] = next_run.isoformat()
+
+                update_success = update_settings(settings)
+
+                if update_success:
+                    debug_print("✅ [AUTO-REFRESH] Admin settings updated with refresh timestamp")
+                else:
+                    debug_print("⚠️ [AUTO-REFRESH] Failed to update admin settings")
+
+        except Exception as settings_error:
+            debug_print(f"❌ [AUTO-REFRESH] Admin settings update failed: {settings_error}")
+
+        # Log the activity
+        log_event("control_center_refresh", {
+            "manual_execution": manual_execution,
+            "refreshed_users": results['refreshed_users'],
+            "failed_users": results['failed_users'],
+            "refreshed_groups": results['refreshed_groups'],
+            "failed_groups": results['failed_groups']
+        })
+
+        debug_print(f"🎉 [AUTO-REFRESH] Refresh completed! Users: {results['refreshed_users']} refreshed, {results['failed_users']} failed. "
+                   f"Groups: {results['refreshed_groups']} refreshed, {results['failed_groups']} failed")
+
+        return results
+
+    except Exception as e:
+        debug_print(f"💥 [AUTO-REFRESH] Error executing Control Center refresh: {e}")
+        results['success'] = False
+        results['error'] = str(e)
+        return results

application/single_app/functions_retention_policy.py

@@ -6,9 +6,10 @@
 This module handles automated deletion of aged conversations and documents
 based on configurable retention policies for personal, group, and public workspaces.
 
-Version: 0.236.012
+Version: 0.237.004
 Implemented in: 0.234.067
 Updated in: 0.236.012 - Fixed race condition handling for NotFound errors during deletion
+Updated in: 0.237.004 - Fixed critical bug where conversations with null/undefined last_activity_at were deleted regardless of age
 """
 
 from config import *
@@ -449,20 +450,11 @@ def process_public_retention():
                 'document_details': []
             }
 
-            # Process conversations
-            if conversation_retention_days != 'none':
-                try:
-                    conv_results = delete_aged_conversations(
-                        public_workspace_id=workspace_id,
-                        retention_days=int(conversation_retention_days),
-                        workspace_type='public'
-                    )
-                    workspace_deletion_summary['conversations_deleted'] = conv_results['count']
-                    workspace_deletion_summary['conversation_details'] = conv_results['details']
-                    results['conversations'] += conv_results['count']
-                except Exception as e:
-                    log_event("process_public_retention_conversations_error", {"error": str(e), "public_workspace_id": workspace_id})
-                    debug_print(f"Error processing conversations for public workspace {workspace_id}: {e}")
+            # Note: Public workspaces do not have a separate conversations container.
+            # Conversations are only stored in personal (cosmos_conversations_container) or 
+            # group (cosmos_group_conversations_container) workspaces.
+            # Therefore, we skip conversation processing for public workspaces.
+            # Only documents are processed for public workspace retention.
 
             # Process documents
             if document_retention_days != 'none':
@@ -529,14 +521,16 @@ def delete_aged_conversations(retention_days, workspace_type='personal', user_id
     cutoff_iso = cutoff_date.isoformat()
 
     # Query for aged conversations
-    # Check for null/undefined FIRST to avoid comparing null values with dates
+    # ONLY delete conversations that have a valid last_activity_at that is older than the cutoff
+    # Conversations with null/undefined last_activity_at should be SKIPPED (not deleted)
+    # This prevents accidentally deleting new conversations that haven't had activity tracked yet
     query = f"""
         SELECT c.id, c.title, c.last_activity_at, c.{partition_field}
         FROM c
         WHERE c.{partition_field} = @partition_value
-        AND (NOT IS_DEFINED(c.last_activity_at) 
-             OR IS_NULL(c.last_activity_at)
-             OR (IS_DEFINED(c.last_activity_at) AND NOT IS_NULL(c.last_activity_at) AND c.last_activity_at < @cutoff_date))
+        AND IS_DEFINED(c.last_activity_at) 
+        AND NOT IS_NULL(c.last_activity_at)
+        AND c.last_activity_at < @cutoff_date
     """
 
     parameters = [

application/single_app/static/js/control-center.js

@@ -3639,6 +3639,32 @@ async function loadRefreshStatus() {
             lastRefreshElement.textContent = 'Error loading';
         }
     }
+
+    // Load and display auto-refresh schedule info
+    try {
+        const response = await fetch('/api/admin/control-center/refresh-status');
+        if (response.ok) {
+            const result = await response.json();
+            const autoRefreshInfoElement = document.getElementById('autoRefreshInfo');
+            const autoRefreshStatusElement = document.getElementById('autoRefreshStatus');
+
+            if (autoRefreshInfoElement && autoRefreshStatusElement) {
+                if (result.auto_refresh_enabled) {
+                    // Build status text
+                    let statusText = `Auto-refresh: daily at ${result.auto_refresh_hour_formatted || result.auto_refresh_hour + ':00 UTC'}`;
+                    if (result.auto_refresh_next_run_formatted) {
+                        statusText += ` (next: ${result.auto_refresh_next_run_formatted})`;
+                    }
+                    autoRefreshStatusElement.textContent = statusText;
+                    autoRefreshInfoElement.classList.remove('d-none');
+                } else {
+                    autoRefreshInfoElement.classList.add('d-none');
+                }
+            }
+        }
+    } catch (autoRefreshError) {
+        console.error('Error loading auto-refresh status:', autoRefreshError);
+    }
 }
 
 async function refreshActiveTabContent() {

application/single_app/templates/control_center.html

@@ -443,12 +443,18 @@ <h2>Control Center</h2>
             <p class="text-muted mb-1">Manage users and their workspaces, groups and their workspaces, and public workspaces.</p>
         </div>
         <div class="text-end">
-            <div class="mb-2">
+            <div class="mb-1">
                 <small class="text-muted" id="lastRefreshInfo">
                     <i class="bi bi-clock-history me-1"></i>
                     Data last refreshed: <span id="lastRefreshTime">Loading...</span>
                 </small>
             </div>
+            <div class="mb-2 d-none" id="autoRefreshInfo">
+                <small class="text-info">
+                    <i class="bi bi-calendar-check me-1"></i>
+                    <span id="autoRefreshStatus">Auto-refresh scheduled</span>
+                </small>
+            </div>
             <button type="button" class="btn btn-outline-primary btn-sm" id="refreshDataBtn" onclick="refreshControlCenterData()">
                 <i class="bi bi-arrow-clockwise me-1"></i>
                 <span id="refreshBtnText">Refresh user, group, public workspace data</span>

docs/explanation/fixes/v0.237.004/RETENTION_POLICY_NULL_LAST_ACTIVITY_FIX.md

@@ -0,0 +1,133 @@
+# Retention Policy Null Last Activity Fix
+
+## Version: 0.237.004
+
+## Problem Statement
+
+The retention policy execution was incorrectly deleting brand new conversations that had null or undefined `last_activity_at` fields. Users reported that conversations created just minutes or hours ago were being deleted when the retention policy ran, even with a 730-day (2 year) retention period configured.
+
+### Symptoms
+- New conversations deleted unexpectedly after retention policy execution
+- Deleted conversations had `last_activity_at: None` in the archived records
+- Error logs showed: `name 'cosmos_public_conversations_container' is not defined` for public workspaces
+
+### Example
+A conversation created at `2026-01-26T20:16:50` was deleted despite a 730-day retention period because it had `last_activity_at: None`:
+
+```json
+{
+    "id": "3e137eed-cfe0-4ce4-a011-cf285e20fbc7",
+    "last_updated": "2026-01-26T20:16:50.590604",
+    "last_activity_at": null,
+    "title": "sumamrize https://academy.faa....",
+    "archived_at": "2026-01-26T21:22:47.746434+00:00",
+    "archived_by_retention_policy": true
+}
+```
+
+## Root Cause Analysis
+
+### Issue 1: Flawed Query Logic
+
+The original SQL query in `delete_aged_conversations()` had this logic:
+
+```sql
+WHERE c.{partition_field} = @partition_value
+AND (NOT IS_DEFINED(c.last_activity_at) 
+     OR IS_NULL(c.last_activity_at)
+     OR (IS_DEFINED(c.last_activity_at) AND NOT IS_NULL(c.last_activity_at) AND c.last_activity_at < @cutoff_date))
+```
+
+This query translated to: **Delete if `last_activity_at` is undefined OR null OR older than cutoff.**
+
+The problem: Conversations with null/undefined `last_activity_at` were being deleted **regardless of their actual age**. The intent was likely to handle edge cases, but it had the opposite effect—treating "no activity date" as "infinitely old."
+
+### Issue 2: Missing Public Conversations Container
+
+The code referenced `cosmos_public_conversations_container` for public workspace retention, but this container doesn't exist. Public workspaces only have:
+- `cosmos_public_documents_container`
+- `cosmos_public_prompts_container`
+- `cosmos_public_workspaces_container`
+
+There is no separate conversations container for public workspaces.
+
+## Solution Implementation
+
+### Fix 1: Corrected Query Logic
+
+**File Modified**: `functions_retention_policy.py`
+
+Changed the query to only delete conversations that have a **valid, non-null** `last_activity_at` that is older than the cutoff:
+
+```python
+# Query for aged conversations
+# ONLY delete conversations that have a valid last_activity_at that is older than the cutoff
+# Conversations with null/undefined last_activity_at should be SKIPPED (not deleted)
+# This prevents accidentally deleting new conversations that haven't had activity tracked yet
+query = f"""
+    SELECT c.id, c.title, c.last_activity_at, c.{partition_field}
+    FROM c
+    WHERE c.{partition_field} = @partition_value
+    AND IS_DEFINED(c.last_activity_at) 
+    AND NOT IS_NULL(c.last_activity_at)
+    AND c.last_activity_at < @cutoff_date
+"""
+```
+
+**Logic Change**:
+- Before: Delete if null/undefined OR older than cutoff
+- After: Delete ONLY if valid date AND older than cutoff
+
+### Fix 2: Removed Public Workspace Conversation Processing
+
+**File Modified**: `functions_retention_policy.py`
+
+Replaced the conversation processing block for public workspaces with a comment explaining why it's skipped:
+
+```python
+# Note: Public workspaces do not have a separate conversations container.
+# Conversations are only stored in personal (cosmos_conversations_container) or 
+# group (cosmos_group_conversations_container) workspaces.
+# Therefore, we skip conversation processing for public workspaces.
+# Only documents are processed for public workspace retention.
+```
+
+## Files Modified
+
+| File | Changes |
+|------|---------|
+| `config.py` | Version updated to `0.237.004` |
+| `functions_retention_policy.py` | Fixed query logic (line ~528), removed public workspace conversation processing (line ~453) |
+
+## Testing & Validation
+
+After the fix, retention policy execution should:
+
+1. ✅ Skip conversations with null/undefined `last_activity_at` instead of deleting them
+2. ✅ Only delete conversations with valid `last_activity_at` dates older than the retention period
+3. ✅ Not show `cosmos_public_conversations_container` errors for public workspaces
+4. ✅ Successfully process document retention for public workspaces
+
+### Test Scenarios
+
+| Scenario | Before Fix | After Fix |
+|----------|------------|-----------|
+| Conversation with `last_activity_at: null` | ❌ Deleted | ✅ Skipped |
+| Conversation with `last_activity_at: undefined` | ❌ Deleted | ✅ Skipped |
+| Conversation with valid old date | ✅ Deleted | ✅ Deleted |
+| Conversation with valid recent date | ✅ Kept | ✅ Kept |
+| Public workspace conversation retention | ❌ Error | ✅ Skipped (documents only) |
+
+## Recommendations
+
+1. **Backfill `last_activity_at`**: Consider running a migration to populate `last_activity_at` for existing conversations that have null values, using `last_updated` as a fallback.
+
+2. **Ensure New Conversations Set `last_activity_at`**: Verify that all conversation creation and update paths properly set the `last_activity_at` field.
+
+3. **Monitor Retention Execution**: After deploying this fix, monitor the next retention policy execution to confirm no unexpected deletions occur.
+
+## Related Documentation
+
+- [Retention Policy Feature Documentation](../../features/RETENTION_POLICY.md)
+- [v0.236.012 NotFound Error Fix](../v0.236.012/RETENTION_POLICY_NOTFOUND_FIX.md)
+- [v0.235.022 Document Deletion Fix](../v0.235.022/RETENTION_POLICY_DOCUMENT_DELETION_FIX.md)