diff --git a/.github/instructions/javascript-lang.instructions.md b/.github/instructions/javascript-lang.instructions.md new file mode 100644 index 00000000..43d67240 --- /dev/null +++ b/.github/instructions/javascript-lang.instructions.md @@ -0,0 +1,21 @@ +--- +applyTo: '**/*.js' +--- + +# JavaScript Language Guide + +- Files should start with a comment of the file name. Ex: `// functions_personal_agents.js` + +- Imports should be grouped at the top of the document after the module docstring, unless otherwise indicated by the user or for performance reasons in which case the import should be as close as possible to the usage with a documented note as to why the import is not at the top of the file. + +- Use 4 spaces per indentation level. No tabs. + +- Code and definitions should occur after the imports block. + +- Use camelCase for variable and function names. Ex: `myVariable`, `getUserData()` + +- Use PascalCase for class names. Ex: `MyClass` + +- Do not use display:none. Instead add and remove the d-none class when hiding or showing elements. + +- Prefer inline html notifications or toast messages using Bootstrap alert classes over browser alert() calls. \ No newline at end of file diff --git a/.github/instructions/python-lang.instructions.md b/.github/instructions/python-lang.instructions.md index c37b99c7..eff15aef 100644 --- a/.github/instructions/python-lang.instructions.md +++ b/.github/instructions/python-lang.instructions.md @@ -1,5 +1,5 @@ --- -applyTo: '**' +applyTo: '**/*.py' --- # Python Language Guide @@ -10,4 +10,6 @@ applyTo: '**' - Use 4 spaces per indentation level. No tabs. -- Code and definitions should occur after the imports block. \ No newline at end of file +- Code and definitions should occur after the imports block. + +- Prefer log_event from functions_appinsights.py for logging activites. \ No newline at end of file diff --git a/.github/instructions/santize_settings_for_frontend_routes.instructions.md b/.github/instructions/santize_settings_for_frontend_routes.instructions.md index d21d469b..bb10fcf0 100644 --- a/.github/instructions/santize_settings_for_frontend_routes.instructions.md +++ b/.github/instructions/santize_settings_for_frontend_routes.instructions.md @@ -20,6 +20,8 @@ When building or working with Python frontend routes (Flask routes that render t ## Required Pattern +### Exception: Admin Routes should NEVER be sanitized as it breaks many admin features. + ### āœ… CORRECT - Sanitize Before Sending ```python from functions_settings import get_settings, sanitize_settings_for_user diff --git a/.github/workflows/docker_image_publish.yml b/.github/workflows/docker_image_publish.yml index 94255a6e..ef8732c3 100644 --- a/.github/workflows/docker_image_publish.yml +++ b/.github/workflows/docker_image_publish.yml @@ -1,4 +1,3 @@ - name: SimpleChat Docker Image Publish on: @@ -8,9 +7,7 @@ on: workflow_dispatch: jobs: - build: - runs-on: ubuntu-latest steps: @@ -18,16 +15,25 @@ jobs: uses: Azure/docker-login@v2 with: # Container registry username - username: ${{ secrets.ACR_USERNAME }} + username: ${{ secrets.MAIN_ACR_USERNAME }} # Container registry password - password: ${{ secrets.ACR_PASSWORD }} + password: ${{ secrets.MAIN_ACR_PASSWORD }} # Container registry server url - login-server: ${{ secrets.ACR_LOGIN_SERVER }} + login-server: ${{ secrets.MAIN_ACR_LOGIN_SERVER }} + - name: Normalize branch name for tag + run: | + REF="${GITHUB_REF_NAME}" + SAFE=$(echo "$REF" \ + | tr '[:upper:]' '[:lower:]' \ + | sed 's#[^a-z0-9._-]#-#g' \ + | sed 's/^-*//;s/-*$//' \ + | cut -c1-128) + echo "BRANCH_TAG=$SAFE" >> "$GITHUB_ENV" - uses: actions/checkout@v3 - name: Build the Docker image run: - docker build . --file application/single_app/Dockerfile --tag ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER; - docker tag ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat:latest; - docker push ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER; - docker push ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat:latest; + docker build . --file application/single_app/Dockerfile --tag ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat:$(date +'%Y-%m-%d')_${BRANCH_TAG}_$GITHUB_RUN_NUMBER; + docker tag ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat:$(date +'%Y-%m-%d')_${BRANCH_TAG}_$GITHUB_RUN_NUMBER ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat:latest; + docker push ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat:$(date +'%Y-%m-%d')_${BRANCH_TAG}_$GITHUB_RUN_NUMBER; + docker push ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat:latest; \ No newline at end of file diff --git a/.github/workflows/docker_image_publish_dev.yml b/.github/workflows/docker_image_publish_dev.yml index e5fb31a0..a78e0c71 100644 --- a/.github/workflows/docker_image_publish_dev.yml +++ b/.github/workflows/docker_image_publish_dev.yml @@ -1,17 +1,16 @@ -name: SimpleChat Docker Image Publish (dev branch) +name: SimpleChat Docker Image Publish (development/staging branch) on: push: branches: - Development + - Staging workflow_dispatch: jobs: - - build: - + build-tomain: runs-on: ubuntu-latest steps: @@ -25,10 +24,53 @@ jobs: # Container registry server url login-server: ${{ secrets.ACR_LOGIN_SERVER }} + - name: Normalize branch name for tag + run: | + REF="${GITHUB_REF_NAME}" + SAFE=$(echo "$REF" \ + | tr '[:upper:]' '[:lower:]' \ + | sed 's#[^a-z0-9._-]#-#g' \ + | sed 's/^-*//;s/-*$//' \ + | cut -c1-128) + echo "BRANCH_TAG=$SAFE" >> "$GITHUB_ENV" + - uses: actions/checkout@v3 - name: Build the Docker image run: - docker build . --file application/single_app/Dockerfile --tag ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER; - docker tag ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:latest; - docker push ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER; + docker build . --file application/single_app/Dockerfile --tag ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:$(date +'%Y-%m-%d')_${BRANCH_TAG}_$GITHUB_RUN_NUMBER; + docker tag ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:$(date +'%Y-%m-%d')_${BRANCH_TAG}_$GITHUB_RUN_NUMBER ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:latest; + docker push ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:$(date +'%Y-%m-%d')_${BRANCH_TAG}_$GITHUB_RUN_NUMBER; docker push ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:latest; + + build-nadoyle: + runs-on: ubuntu-latest + + steps: + - name: Azure Container Registry Login + uses: Azure/docker-login@v2 + with: + # Container registry username + username: ${{ secrets.ACR_USERNAME_NADOYLE }} + # Container registry password + password: ${{ secrets.ACR_PASSWORD_NADOYLE }} + # Container registry server url + login-server: ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }} + + - name: Normalize branch name for tag + run: | + REF="${GITHUB_REF_NAME}" + SAFE=$(echo "$REF" \ + | tr '[:upper:]' '[:lower:]' \ + | sed 's#[^a-z0-9._-]#-#g' \ + | sed 's/^-*//;s/-*$//' \ + | cut -c1-128) + echo "BRANCH_TAG=$SAFE" >> "$GITHUB_ENV" + + - uses: actions/checkout@v3 + - name: Build the Docker image + run: + docker build . --file application/single_app/Dockerfile --tag ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }}/simple-chat-dev:$(date +'%Y-%m-%d')_${BRANCH_TAG}_$GITHUB_RUN_NUMBER; + docker tag ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }}/simple-chat-dev:$(date +'%Y-%m-%d')_${BRANCH_TAG}_$GITHUB_RUN_NUMBER ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }}/simple-chat-dev:latest; + docker push ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }}/simple-chat-dev:$(date +'%Y-%m-%d')_${BRANCH_TAG}_$GITHUB_RUN_NUMBER; + docker push ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }}/simple-chat-dev:latest; + diff --git a/.github/workflows/docker_image_publish_nadoyle.yml b/.github/workflows/docker_image_publish_nadoyle.yml index 4aa90f7b..0dd56e09 100644 --- a/.github/workflows/docker_image_publish_nadoyle.yml +++ b/.github/workflows/docker_image_publish_nadoyle.yml @@ -5,10 +5,7 @@ on: push: branches: - nadoyle - - feature/group-agents-actions - - security/containerBuild - feature/aifoundryagents - - azureBillingPlugin workflow_dispatch: diff --git a/.github/workflows/release-notes-check.yml b/.github/workflows/release-notes-check.yml new file mode 100644 index 00000000..2eb7cee1 --- /dev/null +++ b/.github/workflows/release-notes-check.yml @@ -0,0 +1,205 @@ +name: Release Notes Check + +on: + pull_request: + branches: + - Development + types: + - opened + - reopened + - synchronize + - edited + +jobs: + check-release-notes: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v44 + with: + files_yaml: | + code: + - 'application/single_app/**/*.py' + - 'application/single_app/**/*.js' + - 'application/single_app/**/*.html' + - 'application/single_app/**/*.css' + release_notes: + - 'docs/explanation/release_notes.md' + config: + - 'application/single_app/config.py' + + - name: Check for feature/fix keywords in PR + id: check-keywords + env: + PR_TITLE: ${{ github.event.pull_request.title }} + PR_BODY: ${{ github.event.pull_request.body }} + run: | + echo "šŸ” Analyzing PR title and body for feature/fix indicators..." + + # Convert to lowercase for case-insensitive matching + title_lower=$(echo "$PR_TITLE" | tr '[:upper:]' '[:lower:]') + body_lower=$(echo "$PR_BODY" | tr '[:upper:]' '[:lower:]') + + # Check for feature indicators + if echo "$title_lower $body_lower" | grep -qE "(feat|feature|add|new|implement|introduce|enhancement|improve)"; then + echo "has_feature=true" >> $GITHUB_OUTPUT + echo "šŸ“¦ Feature-related keywords detected" + else + echo "has_feature=false" >> $GITHUB_OUTPUT + fi + + # Check for fix indicators + if echo "$title_lower $body_lower" | grep -qE "(fix|bug|patch|resolve|correct|repair|hotfix|issue)"; then + echo "has_fix=true" >> $GITHUB_OUTPUT + echo "šŸ› Fix-related keywords detected" + else + echo "has_fix=false" >> $GITHUB_OUTPUT + fi + + - name: Determine if release notes update is required + id: require-notes + env: + CODE_CHANGED: ${{ steps.changed-files.outputs.code_any_changed }} + CONFIG_CHANGED: ${{ steps.changed-files.outputs.config_any_changed }} + RELEASE_NOTES_CHANGED: ${{ steps.changed-files.outputs.release_notes_any_changed }} + HAS_FEATURE: ${{ steps.check-keywords.outputs.has_feature }} + HAS_FIX: ${{ steps.check-keywords.outputs.has_fix }} + run: | + echo "" + echo "================================" + echo "šŸ“‹ PR Analysis Summary" + echo "================================" + echo "Code files changed: $CODE_CHANGED" + echo "Config changed: $CONFIG_CHANGED" + echo "Release notes updated: $RELEASE_NOTES_CHANGED" + echo "Feature keywords found: $HAS_FEATURE" + echo "Fix keywords found: $HAS_FIX" + echo "================================" + echo "" + + # Determine if this PR likely needs release notes + needs_notes="false" + reason="" + + if [[ "$HAS_FEATURE" == "true" ]]; then + needs_notes="true" + reason="Feature-related keywords detected in PR title/body" + elif [[ "$HAS_FIX" == "true" ]]; then + needs_notes="true" + reason="Fix-related keywords detected in PR title/body" + elif [[ "$CODE_CHANGED" == "true" && "$CONFIG_CHANGED" == "true" ]]; then + needs_notes="true" + reason="Both code and config.py were modified" + fi + + echo "needs_notes=$needs_notes" >> $GITHUB_OUTPUT + echo "reason=$reason" >> $GITHUB_OUTPUT + + - name: Validate release notes update + env: + CODE_CHANGED: ${{ steps.changed-files.outputs.code_any_changed }} + RELEASE_NOTES_CHANGED: ${{ steps.changed-files.outputs.release_notes_any_changed }} + NEEDS_NOTES: ${{ steps.require-notes.outputs.needs_notes }} + REASON: ${{ steps.require-notes.outputs.reason }} + CODE_FILES: ${{ steps.changed-files.outputs.code_all_changed_files }} + run: | + echo "" + + if [[ "$NEEDS_NOTES" == "true" && "$RELEASE_NOTES_CHANGED" != "true" ]]; then + echo "āš ļø ==============================================" + echo "āš ļø RELEASE NOTES UPDATE RECOMMENDED" + echo "āš ļø ==============================================" + echo "" + echo "šŸ“ Reason: $REASON" + echo "" + echo "This PR appears to contain changes that should be documented" + echo "in the release notes (docs/explanation/release_notes.md)." + echo "" + echo "šŸ“ Code files changed:" + echo "$CODE_FILES" | tr ' ' '\n' | sed 's/^/ - /' + echo "" + echo "šŸ’” Please consider adding an entry to release_notes.md describing:" + echo " ā€¢ New features added" + echo " ā€¢ Bug fixes implemented" + echo " ā€¢ Breaking changes (if any)" + echo " ā€¢ Files modified" + echo "" + echo "šŸ“– Follow the existing format in release_notes.md" + echo "" + # Exit with warning (non-zero) to flag the PR but not block it + # Change 'exit 0' to 'exit 1' below to make this a hard requirement + exit 0 + elif [[ "$RELEASE_NOTES_CHANGED" == "true" ]]; then + echo "āœ… Release notes have been updated - great job!" + elif [[ "$CODE_CHANGED" != "true" ]]; then + echo "ā„¹ļø No significant code changes detected - release notes update not required." + else + echo "ā„¹ļø Changes appear to be minor - release notes update optional." + fi + + echo "" + echo "āœ… Release notes check completed successfully." + + - name: Post PR comment (when notes needed but missing) + if: steps.require-notes.outputs.needs_notes == 'true' && steps.changed-files.outputs.release_notes_any_changed != 'true' + uses: actions/github-script@v7 + with: + script: | + const reason = '${{ steps.require-notes.outputs.reason }}'; + + // Check if we already commented + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('šŸ“‹ Release Notes Reminder') + ); + + if (!botComment) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `## šŸ“‹ Release Notes Reminder + + This PR appears to contain changes that should be documented in the release notes. + + **Reason:** ${reason} + + ### šŸ“ Please consider updating: + \`docs/explanation/release_notes.md\` + + ### Template for new features: + \`\`\`markdown + * **Feature Name** + * Brief description of the feature. + * **Key Details**: Important implementation notes. + * **Files Modified**: \`file1.py\`, \`file2.js\`. + * (Ref: related components, patterns) + \`\`\` + + ### Template for bug fixes: + \`\`\`markdown + * **Bug Fix Title** + * Description of what was fixed. + * **Root Cause**: What caused the issue. + * **Solution**: How it was resolved. + * **Files Modified**: \`file.py\`. + * (Ref: related issue numbers, components) + \`\`\` + + --- + *This is an automated reminder. If this PR doesn't require release notes (e.g., internal refactoring, documentation-only changes), you can ignore this message.*` + }); + } diff --git a/application/single_app/static/json/schemas/azure_billing_plugin.additional_settings.schema.json b/application/community_customizations/actions/azure_billing_retriever/azure_billing_plugin.additional_settings.schema.json similarity index 100% rename from application/single_app/static/json/schemas/azure_billing_plugin.additional_settings.schema.json rename to application/community_customizations/actions/azure_billing_retriever/azure_billing_plugin.additional_settings.schema.json diff --git a/application/single_app/static/json/schemas/azure_billing_plugin.definition.json b/application/community_customizations/actions/azure_billing_retriever/azure_billing_plugin.definition.json similarity index 100% rename from application/single_app/static/json/schemas/azure_billing_plugin.definition.json rename to application/community_customizations/actions/azure_billing_retriever/azure_billing_plugin.definition.json diff --git a/application/community_customizations/actions/azure_billing_retriever/readme.md b/application/community_customizations/actions/azure_billing_retriever/readme.md index 0c9b365e..982ec8ab 100644 --- a/application/community_customizations/actions/azure_billing_retriever/readme.md +++ b/application/community_customizations/actions/azure_billing_retriever/readme.md @@ -3,7 +3,7 @@ # Azure Billing Action Instructions ## Overview -The Azure Billing action is an experimental Semantic Kernel plugin that helps agents explore Azure Cost Management data, generate CSV outputs, and render server-side charts for conversational reporting. It stitches together Azure REST APIs, matplotlib rendering, and Cosmos DB persistence so prototype agents can investigate subscriptions, budgets, alerts, and forecasts without touching the production portal. It leverages message injection (direct cosmos_messages_container access) to store chart images as conversation artifacts in lieu of embedding binary data in chat responses. +The Azure Billing action is an experimental Semantic Kernel plugin that helps agents explore Azure Cost Management data, generate CSV outputs, and render server-side charts for conversational reporting. It stitches together Azure REST APIs, matplotlib rendering, and Cosmos DB persistence so prototype agents can investigate subscriptions, budgets, alerts, and forecasts without touching the production portal. It leverages message injection (direct cosmos_messages_container access) to store chart images as conversation artifacts in lieu of embedding binary data in chat responses. You will need to move the ```azure_billing_plugin.py``` to the [semantic-kernel-plugins](../../../single_app/semantic_kernel_plugins/) folder, and move the ```schema.json``` and ```definition.json``` to the [schemas](../../../single_app/static/json/schemas) folder. ## Core capabilities - Enumerate subscriptions and resource groups via `list_subscriptions*` helpers for quick scope discovery. @@ -48,6 +48,5 @@ The Azure Billing action is an experimental Semantic Kernel plugin that helps ag ## Additional resources - Review `instructions.md` in the same directory for the autonomous agent persona tailored to this action. -- Inspect `abd_proto.py` for prompt experimentation tied to Azure Billing dialogues. - Leverage the sample CSV files to validate plotting offline before wiring the plugin into a notebook or agent loop. diff --git a/application/single_app/app.py b/application/single_app/app.py index 54336100..53f2ff5c 100644 --- a/application/single_app/app.py +++ b/application/single_app/app.py @@ -55,9 +55,11 @@ from route_backend_retention_policy import * from route_backend_plugins import bpap as admin_plugins_bp, bpdp as dynamic_plugins_bp from route_backend_agents import bpa as admin_agents_bp +from route_backend_agent_templates import bp_agent_templates from route_backend_public_workspaces import * from route_backend_public_documents import * from route_backend_public_prompts import * +from route_backend_user_agreement import register_route_backend_user_agreement from route_backend_speech import register_route_backend_speech from route_backend_tts import register_route_backend_tts from route_enhanced_citations import register_enhanced_citations_routes @@ -97,6 +99,7 @@ app.register_blueprint(admin_plugins_bp) app.register_blueprint(dynamic_plugins_bp) app.register_blueprint(admin_agents_bp) +app.register_blueprint(bp_agent_templates) app.register_blueprint(plugin_validation_bp) app.register_blueprint(bp_migration) app.register_blueprint(plugin_logging_bp) @@ -617,6 +620,9 @@ def list_semantic_kernel_plugins(): # ------------------- API Public Prompts Routes ---------- register_route_backend_public_prompts(app) +# ------------------- API User Agreement Routes ---------- +register_route_backend_user_agreement(app) + # ------------------- Extenral Health Routes ---------- register_route_external_health(app) diff --git a/application/single_app/config.py b/application/single_app/config.py index 2224a49e..caf09fc8 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -88,7 +88,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.235.025" +VERSION = "0.236.012" SECRET_KEY = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production') @@ -377,6 +377,12 @@ def get_redis_cache_infrastructure_endpoint(redis_hostname: str) -> str: partition_key=PartitionKey(path="/id") ) +cosmos_agent_templates_container_name = "agent_templates" +cosmos_agent_templates_container = cosmos_database.create_container_if_not_exists( + id=cosmos_agent_templates_container_name, + partition_key=PartitionKey(path="/id") +) + cosmos_agent_facts_container_name = "agent_facts" cosmos_agent_facts_container = cosmos_database.create_container_if_not_exists( id=cosmos_agent_facts_container_name, @@ -645,7 +651,7 @@ def initialize_clients(settings): azure_apim_content_safety_endpoint = settings.get("azure_apim_content_safety_endpoint") azure_apim_content_safety_subscription_key = settings.get("azure_apim_content_safety_subscription_key") - if safety_endpoint and safety_key: + if safety_endpoint: try: if enable_content_safety_apim: content_safety_client = ContentSafetyClient( diff --git a/application/single_app/foundry_agent_runtime.py b/application/single_app/foundry_agent_runtime.py new file mode 100644 index 00000000..36a99ec3 --- /dev/null +++ b/application/single_app/foundry_agent_runtime.py @@ -0,0 +1,355 @@ +# foundry_agent_runtime.py +"""Azure AI Foundry agent execution helpers.""" + +import asyncio +import logging +import os +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Optional + +from azure.identity import AzureAuthorityHosts +from azure.identity.aio import ( # type: ignore + ClientSecretCredential, + DefaultAzureCredential, +) +from semantic_kernel.agents import AzureAIAgent +from semantic_kernel.contents.chat_message_content import ChatMessageContent + +from functions_appinsights import log_event +from functions_debug import debug_print +from functions_keyvault import ( + retrieve_secret_from_key_vault_by_full_name, + validate_secret_name_dynamic, +) + +_logger = logging.getLogger("foundry_agent_runtime") + + +@dataclass +class FoundryAgentInvocationResult: + """Represents the outcome from a Foundry agent run.""" + + message: str + model: Optional[str] + citations: List[Dict[str, Any]] + metadata: Dict[str, Any] + + +class FoundryAgentInvocationError(RuntimeError): + """Raised when the Foundry agent invocation cannot be completed.""" + + +class AzureAIFoundryChatCompletionAgent: + """Lightweight wrapper so Foundry agents behave like SK chat agents.""" + + agent_type = "aifoundry" + + def __init__(self, agent_config: Dict[str, Any], settings: Dict[str, Any]): + self.name = agent_config.get("name") + self.display_name = agent_config.get("display_name") or self.name + self.description = agent_config.get("description", "") + self.id = agent_config.get("id") + self.default_agent = agent_config.get("default_agent", False) + self.is_global = agent_config.get("is_global", False) + self.is_group = agent_config.get("is_group", False) + self.group_id = agent_config.get("group_id") + self.group_name = agent_config.get("group_name") + self.max_completion_tokens = agent_config.get("max_completion_tokens", -1) + self.last_run_citations: List[Dict[str, Any]] = [] + self.last_run_model: Optional[str] = None + self._foundry_settings = ( + (agent_config.get("other_settings") or {}).get("azure_ai_foundry") or {} + ) + self._global_settings = settings or {} + + def invoke( + self, + agent_message_history: Iterable[ChatMessageContent], + metadata: Optional[Dict[str, Any]] = None, + ) -> str: + """Synchronously invoke the Foundry agent and return the final message text.""" + + metadata = metadata or {} + history = list(agent_message_history) + debug_print( + f"[FoundryAgent] Invoking agent '{self.name}' with {len(history)} messages" + ) + + try: + result = asyncio.run( + execute_foundry_agent( + foundry_settings=self._foundry_settings, + global_settings=self._global_settings, + message_history=history, + metadata=metadata, + ) + ) + except RuntimeError: + log_event( + "[FoundryAgent] Invocation runtime error", + extra={ + "agent_id": self.id, + "agent_name": self.name, + }, + level=logging.ERROR, + ) + raise + except Exception as exc: # pragma: no cover - defensive logging + log_event( + "[FoundryAgent] Invocation error", + extra={ + "agent_id": self.id, + "agent_name": self.name, + }, + level=logging.ERROR, + ) + raise + + self.last_run_citations = result.citations + self.last_run_model = result.model + return result.message + + +async def execute_foundry_agent( + *, + foundry_settings: Dict[str, Any], + global_settings: Dict[str, Any], + message_history: List[ChatMessageContent], + metadata: Dict[str, Any], +) -> FoundryAgentInvocationResult: + """Invoke a Foundry agent using Semantic Kernel's AzureAIAgent abstraction.""" + + agent_id = (foundry_settings.get("agent_id") or "").strip() + if not agent_id: + raise FoundryAgentInvocationError( + "Azure AI Foundry agents require an agent_id in other_settings.azure_ai_foundry." + ) + + endpoint = _resolve_endpoint(foundry_settings, global_settings) + api_version = foundry_settings.get("api_version") or global_settings.get( + "azure_ai_foundry_api_version" + ) + + credential = _build_async_credential(foundry_settings, global_settings) + client = AzureAIAgent.create_client( + credential=credential, + endpoint=endpoint, + api_version=api_version, + ) + + try: + definition = await client.agents.get_agent(agent_id) + azure_agent = AzureAIAgent(client=client, definition=definition) + responses = [] + async for response in azure_agent.invoke( + messages=message_history, + metadata={k: str(v) for k, v in metadata.items() if v is not None}, + ): + responses.append(response) + + if not responses: + raise FoundryAgentInvocationError("Foundry agent returned no messages.") + + last_response = responses[-1] + + thread_id = None + if last_response.thread is not None: + thread_id = getattr(last_response.thread, "id", None) + + message_obj = last_response.message + + if not thread_id: + metadata_thread_id = None + if isinstance(message_obj.metadata, dict): + metadata_thread_id = message_obj.metadata.get("thread_id") + thread_id = metadata_thread_id or metadata.get("thread_id") + + if thread_id: + try: + if last_response.thread is not None and hasattr(last_response.thread, "delete"): + await last_response.thread.delete() + elif hasattr(client, "agents") and hasattr(client.agents, "delete_thread"): + await client.agents.delete_thread(thread_id) + except Exception as cleanup_error: # pragma: no cover - best effort cleanup + _logger.warning("Failed to delete Foundry thread: %s", cleanup_error) + text = _extract_message_text(message_obj) + citations = _extract_citations(message_obj) + model_name = getattr(definition, "model", None) + if isinstance(model_name, dict): + model_value = model_name.get("id") + else: + model_value = getattr(model_name, "id", None) + + log_event( + "[FoundryAgent] Invocation complete", + extra={ + "agent_id": agent_id, + "endpoint": endpoint, + "model": model_value, + "message_length": len(text or ""), + }, + ) + + return FoundryAgentInvocationResult( + message=text, + model=model_value, + citations=citations, + metadata=message_obj.metadata or {}, + ) + finally: + try: + await client.close() + finally: + await credential.close() + + +def _resolve_endpoint(foundry_settings: Dict[str, Any], global_settings: Dict[str, Any]) -> str: + endpoint = ( + foundry_settings.get("endpoint") + or global_settings.get("azure_ai_foundry_endpoint") + or os.getenv("AZURE_AI_AGENT_ENDPOINT") + ) + if endpoint: + return endpoint.rstrip("/") + + raise FoundryAgentInvocationError( + "Azure AI Foundry endpoint is not configured. Provide an endpoint in the agent's other_settings.azure_ai_foundry or global settings." + ) + + +def _build_async_credential( + foundry_settings: Dict[str, Any], + global_settings: Dict[str, Any], +): + auth_type = ( + foundry_settings.get("authentication_type") + or foundry_settings.get("auth_type") + or global_settings.get("azure_ai_foundry_authentication_type") + ) + managed_identity_type = ( + foundry_settings.get("managed_identity_type") + or global_settings.get("azure_ai_foundry_managed_identity_type") + ) + managed_identity_client_id = ( + foundry_settings.get("managed_identity_client_id") + or global_settings.get("azure_ai_foundry_managed_identity_client_id") + ) + + authority = ( + foundry_settings.get("authority") + or global_settings.get("azure_ai_foundry_authority") + or _authority_from_cloud(foundry_settings.get("cloud") or global_settings.get("azure_ai_foundry_cloud")) + ) + + tenant_id = foundry_settings.get("tenant_id") or global_settings.get( + "azure_ai_foundry_tenant_id" + ) + client_id = foundry_settings.get("client_id") or global_settings.get( + "azure_ai_foundry_client_id" + ) + client_secret = foundry_settings.get("client_secret") or global_settings.get( + "azure_ai_foundry_client_secret" + ) + + if auth_type == "service_principal": + if not client_secret: + raise FoundryAgentInvocationError( + "Foundry service principals require client_secret value." + ) + resolved_secret = _resolve_secret_value(client_secret) + if not tenant_id or not client_id: + raise FoundryAgentInvocationError( + "Foundry service principals require tenant_id and client_id values." + ) + return ClientSecretCredential( + tenant_id=tenant_id, + client_id=client_id, + client_secret=resolved_secret, + authority=authority, + ) + + if client_secret and auth_type != "managed_identity": + resolved_secret = _resolve_secret_value(client_secret) + if not tenant_id or not client_id: + raise FoundryAgentInvocationError( + "Foundry service principals require tenant_id and client_id values." + ) + return ClientSecretCredential( + tenant_id=tenant_id, + client_id=client_id, + client_secret=resolved_secret, + authority=authority, + ) + + if auth_type == "managed_identity": + if managed_identity_type == "user_assigned" and managed_identity_client_id: + return DefaultAzureCredential( + authority=authority, + managed_identity_client_id=managed_identity_client_id, + ) + return DefaultAzureCredential(authority=authority) + + # Fall back to default chained credentials (managed identity, CLI, etc.) + return DefaultAzureCredential(authority=authority) + + +def _resolve_secret_value(value: str) -> str: + if validate_secret_name_dynamic(value): + resolved = retrieve_secret_from_key_vault_by_full_name(value) + if not resolved: + raise FoundryAgentInvocationError( + f"Unable to resolve Key Vault secret '{value}' for Foundry credentials." + ) + return resolved + return value + + +def _authority_from_cloud(cloud_value: Optional[str]) -> str: + if not cloud_value: + return AzureAuthorityHosts.AZURE_PUBLIC_CLOUD + + normalized = cloud_value.lower() + if normalized in ("usgov", "usgovernment", "gcc"): + return AzureAuthorityHosts.AZURE_GOVERNMENT + return AzureAuthorityHosts.AZURE_PUBLIC_CLOUD + + +def _extract_message_text(message: ChatMessageContent) -> str: + if message.content: + if isinstance(message.content, str): + return message.content + try: + return "".join(str(chunk) for chunk in message.content) + except TypeError: + return str(message.content) + return "" + + +def _extract_citations(message: ChatMessageContent) -> List[Dict[str, Any]]: + metadata = message.metadata or {} + citations = metadata.get("citations") + if isinstance(citations, list): + return [c for c in citations if isinstance(c, dict)] + items = getattr(message, "items", None) + if isinstance(items, list): + extracted: List[Dict[str, Any]] = [] + for item in items: + content_type = getattr(item, "content_type", None) + if content_type != "annotation": + continue + url = getattr(item, "url", None) + title = getattr(item, "title", None) + quote = getattr(item, "quote", None) + if not url: + continue + extracted.append( + { + "url": url, + "title": title, + "quote": quote, + "citation_type": getattr(item, "citation_type", None), + } + ) + if extracted: + return extracted + return [] diff --git a/application/single_app/functions_activity_logging.py b/application/single_app/functions_activity_logging.py index fb005f06..df9cabf3 100644 --- a/application/single_app/functions_activity_logging.py +++ b/application/single_app/functions_activity_logging.py @@ -118,6 +118,58 @@ def log_user_activity( debug_print(f"Error logging user activity for user {user_id}: {str(e)}") +def log_web_search_consent_acceptance( + user_id: str, + admin_email: str, + consent_text: str, + source: str = 'admin_settings' +) -> None: + """ + Log web search consent acceptance to activity_logs and App Insights. + + Args: + user_id (str): Admin user ID who accepted the consent. + admin_email (str): Admin email who accepted the consent. + consent_text (str): Consent message accepted by the admin. + source (str, optional): Origin of the consent action. + """ + try: + activity_record = { + 'id': str(uuid.uuid4()), + 'activity_type': 'web_search_consent_acceptance', + 'user_id': user_id, + 'timestamp': datetime.utcnow().isoformat(), + 'created_at': datetime.utcnow().isoformat(), + 'accepted_by': { + 'user_id': user_id, + 'email': admin_email + }, + 'source': source, + 'description': consent_text + } + + cosmos_activity_logs_container.create_item(body=activity_record) + + log_event( + message=consent_text, + extra=activity_record, + level=logging.INFO + ) + debug_print(f"Logged web search consent acceptance for user {user_id}") + + except Exception as e: + log_event( + message=f"Error logging web search consent acceptance: {str(e)}", + extra={ + 'user_id': user_id, + 'admin_email': admin_email, + 'error': str(e) + }, + level=logging.ERROR + ) + debug_print(f"Error logging web search consent acceptance for user {user_id}: {str(e)}") + + def log_document_upload( user_id: str, container_type: str, @@ -1080,3 +1132,210 @@ def log_public_workspace_status_change( level=logging.ERROR ) debug_print(f"āš ļø Warning: Failed to log public workspace status change: {str(e)}") + + +def log_user_agreement_accepted( + user_id: str, + workspace_type: str, + workspace_id: str, + workspace_name: Optional[str] = None, + action_context: Optional[str] = None +) -> None: + """ + Log when a user accepts a user agreement in a workspace. + This record is used to track acceptance and support daily acceptance features. + + Args: + user_id (str): The ID of the user who accepted the agreement + workspace_type (str): Type of workspace ('personal', 'group', 'public') + workspace_id (str): The ID of the workspace + workspace_name (str, optional): The name of the workspace + action_context (str, optional): The context/action that triggered the agreement + (e.g., 'file_upload', 'chat') + """ + + try: + import uuid + + # Create user agreement acceptance record + acceptance_record = { + 'id': str(uuid.uuid4()), + 'user_id': user_id, + 'activity_type': 'user_agreement_accepted', + 'timestamp': datetime.utcnow().isoformat(), + 'created_at': datetime.utcnow().isoformat(), + 'accepted_date': datetime.utcnow().strftime('%Y-%m-%d'), # Date only for daily lookup + 'workspace_type': workspace_type, + 'workspace_context': { + f'{workspace_type}_workspace_id': workspace_id, + 'workspace_name': workspace_name + }, + 'action_context': action_context + } + + # Save to activity_logs container + cosmos_activity_logs_container.create_item(body=acceptance_record) + + # Also log to Application Insights for monitoring + log_event( + message=f"User agreement accepted: user {user_id} in {workspace_type} workspace {workspace_id}", + extra=acceptance_record, + level=logging.INFO + ) + + debug_print(f"āœ… Logged user agreement acceptance: user {user_id} in {workspace_type} workspace {workspace_id}") + + except Exception as e: + # Log error but don't fail the operation + log_event( + message=f"Error logging user agreement acceptance: {str(e)}", + extra={ + 'user_id': user_id, + 'workspace_type': workspace_type, + 'workspace_id': workspace_id, + 'error': str(e) + }, + level=logging.ERROR + ) + debug_print(f"āš ļø Warning: Failed to log user agreement acceptance: {str(e)}") + + +def has_user_accepted_agreement_today( + user_id: str, + workspace_type: str, + workspace_id: str +) -> bool: + """ + Check if a user has already accepted the user agreement today for a given workspace. + Used to implement the "accept once per day" feature. + + Args: + user_id (str): The ID of the user + workspace_type (str): Type of workspace ('personal', 'group', 'public') + workspace_id (str): The ID of the workspace + + Returns: + bool: True if user has accepted today, False otherwise + """ + + try: + today_date = datetime.utcnow().strftime('%Y-%m-%d') + + # Query for today's acceptance record + query = """ + SELECT VALUE COUNT(1) FROM c + WHERE c.user_id = @user_id + AND c.activity_type = 'user_agreement_accepted' + AND c.accepted_date = @today_date + AND c.workspace_type = @workspace_type + AND c.workspace_context[@workspace_id_key] = @workspace_id + """ + + workspace_id_key = f'{workspace_type}_workspace_id' + + params = [ + {"name": "@user_id", "value": user_id}, + {"name": "@today_date", "value": today_date}, + {"name": "@workspace_type", "value": workspace_type}, + {"name": "@workspace_id_key", "value": workspace_id_key}, + {"name": "@workspace_id", "value": workspace_id} + ] + + results = list(cosmos_activity_logs_container.query_items( + query=query, + parameters=params, + enable_cross_partition_query=False # Query by partition key (user_id) + )) + + count = results[0] if results else 0 + + debug_print(f"šŸ” User agreement check: user {user_id}, workspace {workspace_id}, today={today_date}, accepted={count > 0}") + + return count > 0 + + except Exception as e: + # Log error and return False (require re-acceptance on error) + log_event( + message=f"Error checking user agreement acceptance: {str(e)}", + extra={ + 'user_id': user_id, + 'workspace_type': workspace_type, + 'workspace_id': workspace_id, + 'error': str(e) + }, + level=logging.ERROR + ) + debug_print(f"āš ļø Error checking user agreement acceptance: {str(e)}") + return False + + +def log_retention_policy_force_push( + admin_user_id: str, + admin_email: str, + scopes: list, + results: dict, + total_updated: int +) -> None: + """ + Log retention policy force push action to activity_logs container. + + This creates a permanent audit record when an admin forces organization + default retention policies to be applied to all workspaces. + + Args: + admin_user_id (str): User ID of the admin performing the force push + admin_email (str): Email of the admin performing the force push + scopes (list): List of workspace types affected (e.g., ['personal', 'group', 'public']) + results (dict): Breakdown of updates per workspace type + total_updated (int): Total number of workspaces/users updated + """ + + try: + # Create force push activity record + force_push_activity = { + 'id': str(uuid.uuid4()), + 'user_id': admin_user_id, # Partition key + 'activity_type': 'retention_policy_force_push', + 'timestamp': datetime.utcnow().isoformat(), + 'created_at': datetime.utcnow().isoformat(), + 'admin': { + 'user_id': admin_user_id, + 'email': admin_email + }, + 'force_push_details': { + 'scopes': scopes, + 'results': results, + 'total_updated': total_updated, + 'executed_at': datetime.utcnow().isoformat() + }, + 'workspace_type': 'admin', + 'workspace_context': { + 'action': 'retention_policy_force_push' + } + } + + # Save to activity_logs container for permanent audit trail + cosmos_activity_logs_container.create_item(body=force_push_activity) + + # Also log to Application Insights for monitoring + log_event( + message=f"Retention policy force push executed by {admin_email} for scopes: {', '.join(scopes)}. Total updated: {total_updated}", + extra=force_push_activity, + level=logging.INFO + ) + + debug_print(f"āœ… Retention policy force push logged: {scopes} by {admin_email}, updated {total_updated}") + + except Exception as e: + # Log error but don't break the force push flow + log_event( + message=f"Error logging retention policy force push: {str(e)}", + extra={ + 'admin_user_id': admin_user_id, + 'scopes': scopes, + 'total_updated': total_updated, + 'error': str(e) + }, + level=logging.ERROR + ) + debug_print(f"āš ļø Warning: Failed to log retention policy force push: {str(e)}") diff --git a/application/single_app/functions_agent_payload.py b/application/single_app/functions_agent_payload.py new file mode 100644 index 00000000..09f1f343 --- /dev/null +++ b/application/single_app/functions_agent_payload.py @@ -0,0 +1,206 @@ +# functions_agent_payload.py +"""Utility helpers for normalizing agent payloads before validation and storage.""" + +from copy import deepcopy +from typing import Any, Dict, List + +_SUPPORTED_AGENT_TYPES = {"local", "aifoundry"} +_APIM_FIELDS = [ + "azure_agent_apim_gpt_endpoint", + "azure_agent_apim_gpt_subscription_key", + "azure_agent_apim_gpt_deployment", + "azure_agent_apim_gpt_api_version", +] +_GPT_FIELDS = [ + "azure_openai_gpt_endpoint", + "azure_openai_gpt_key", + "azure_openai_gpt_deployment", + "azure_openai_gpt_api_version", +] +_FREE_FORM_TEXT = [ + "name", + "display_name", + "description", + "instructions", +] +_TEXT_FIELDS = [ + "name", + "display_name", + "description", + "instructions", + "azure_openai_gpt_endpoint", + "azure_openai_gpt_deployment", + "azure_openai_gpt_api_version", + "azure_agent_apim_gpt_endpoint", + "azure_agent_apim_gpt_deployment", + "azure_agent_apim_gpt_api_version", +] +_STRING_DEFAULT_FIELDS = [ + "azure_openai_gpt_endpoint", + "azure_openai_gpt_key", + "azure_openai_gpt_deployment", + "azure_openai_gpt_api_version", + "azure_agent_apim_gpt_endpoint", + "azure_agent_apim_gpt_subscription_key", + "azure_agent_apim_gpt_deployment", + "azure_agent_apim_gpt_api_version", +] + +_MAX_FIELD_LENGTHS = { + "name": 100, + "display_name": 200, + "description": 2000, + "instructions": 30000, + "azure_openai_gpt_endpoint": 2048, + "azure_openai_gpt_key": 1024, + "azure_openai_gpt_deployment": 256, + "azure_openai_gpt_api_version": 64, + "azure_agent_apim_gpt_endpoint": 2048, + "azure_agent_apim_gpt_subscription_key": 1024, + "azure_agent_apim_gpt_deployment": 256, + "azure_agent_apim_gpt_api_version": 64, +} +_FOUNDRY_FIELD_LENGTHS = { + "agent_id": 128, + "endpoint": 2048, + "api_version": 64, + "authority": 2048, + "tenant_id": 64, + "client_id": 64, + "client_secret": 1024, + "managed_identity_client_id": 64, +} + + +class AgentPayloadError(ValueError): + """Raised when an agent payload violates backend requirements.""" + + +def is_azure_ai_foundry_agent(agent: Dict[str, Any]) -> bool: + """Return True when the agent type is Azure AI Foundry.""" + agent_type = (agent or {}).get("agent_type", "local") + if isinstance(agent_type, str): + return agent_type.strip().lower() == "aifoundry" + return False + + +def _normalize_text_fields(payload: Dict[str, Any]) -> None: + for field in _TEXT_FIELDS: + value = payload.get(field) + if isinstance(value, str): + payload[field] = value.strip() + + +def _coerce_actions(actions: Any) -> List[str]: + if actions is None or actions == "": + return [] + if not isinstance(actions, list): + raise AgentPayloadError("actions_to_load must be an array of strings.") + cleaned: List[str] = [] + for item in actions: + if isinstance(item, str): + trimmed = item.strip() + if trimmed: + cleaned.append(trimmed) + else: + raise AgentPayloadError("actions_to_load entries must be strings.") + return cleaned + + +def _coerce_other_settings(settings: Any) -> Dict[str, Any]: + if settings in (None, ""): + return {} + if not isinstance(settings, dict): + raise AgentPayloadError("other_settings must be an object.") + return settings + + +def _coerce_agent_type(agent_type: Any) -> str: + if isinstance(agent_type, str): + agent_type = agent_type.strip().lower() + else: + agent_type = "local" + if agent_type not in _SUPPORTED_AGENT_TYPES: + return "local" + return agent_type + + +def _coerce_completion_tokens(value: Any) -> int: + if value in (None, "", " "): + return -1 + try: + return int(value) + except (TypeError, ValueError) as exc: + raise AgentPayloadError("max_completion_tokens must be an integer.") from exc + +def _validate_field_lengths(payload: Dict[str, Any]) -> None: + for field, max_len in _MAX_FIELD_LENGTHS.items(): + value = payload.get(field, "") + if isinstance(value, str) and len(value) > max_len: + raise AgentPayloadError(f"{field} exceeds maximum length of {max_len}.") + + +def _validate_foundry_field_lengths(foundry_settings: Dict[str, Any]) -> None: + for field, max_len in _FOUNDRY_FIELD_LENGTHS.items(): + value = foundry_settings.get(field, "") + if isinstance(value, str) and len(value) > max_len: + raise AgentPayloadError(f"azure_ai_foundry.{field} exceeds maximum length of {max_len}.") + +def sanitize_agent_payload(agent: Dict[str, Any]) -> Dict[str, Any]: + """Return a sanitized copy of the agent payload or raise AgentPayloadError.""" + if not isinstance(agent, dict): + raise AgentPayloadError("Agent payload must be an object.") + + sanitized = deepcopy(agent) + _normalize_text_fields(sanitized) + + for field in _STRING_DEFAULT_FIELDS: + value = sanitized.get(field) + if value is None: + sanitized[field] = "" + + _validate_field_lengths(sanitized) + + agent_type = _coerce_agent_type(sanitized.get("agent_type")) + sanitized["agent_type"] = agent_type + + sanitized["other_settings"] = _coerce_other_settings(sanitized.get("other_settings")) + sanitized["actions_to_load"] = _coerce_actions(sanitized.get("actions_to_load")) + sanitized["max_completion_tokens"] = _coerce_completion_tokens( + sanitized.get("max_completion_tokens") + ) + + sanitized["enable_agent_gpt_apim"] = bool( + sanitized.get("enable_agent_gpt_apim", False) + ) + sanitized.setdefault("is_global", False) + sanitized.setdefault("is_group", False) + + if agent_type == "aifoundry": + sanitized["enable_agent_gpt_apim"] = False + for field in _APIM_FIELDS: + sanitized.pop(field, None) + sanitized["actions_to_load"] = [] + + foundry_settings = sanitized["other_settings"].get("azure_ai_foundry") + if not isinstance(foundry_settings, dict): + raise AgentPayloadError( + "Azure AI Foundry agents require other_settings.azure_ai_foundry." + ) + agent_id = str(foundry_settings.get("agent_id", "")).strip() + if not agent_id: + raise AgentPayloadError( + "Azure AI Foundry agents require other_settings.azure_ai_foundry.agent_id." + ) + foundry_settings["agent_id"] = agent_id + _validate_foundry_field_lengths(foundry_settings) + sanitized["other_settings"]["azure_ai_foundry"] = foundry_settings + else: + # Remove stale foundry metadata when toggling back to local agents. + azure_foundry = sanitized["other_settings"].get("azure_ai_foundry") + if azure_foundry is not None and not isinstance(azure_foundry, dict): + raise AgentPayloadError("azure_ai_foundry must be an object when provided.") + if azure_foundry: + sanitized["other_settings"].pop("azure_ai_foundry", None) + + return sanitized \ No newline at end of file diff --git a/application/single_app/functions_agent_templates.py b/application/single_app/functions_agent_templates.py new file mode 100644 index 00000000..10838fd6 --- /dev/null +++ b/application/single_app/functions_agent_templates.py @@ -0,0 +1,349 @@ +# functions_agent_templates.py +"""Agent template helper functions. + +This module centralizes CRUD operations for agent templates stored in the +Cosmos DB `agent_templates` container. Templates are surfaced as reusable +starting points inside the agent builder UI. +""" + +from __future__ import annotations + +import json +import uuid +from datetime import datetime +from typing import Any, Dict, List, Optional + +from azure.cosmos import exceptions +from flask import current_app + +from config import cosmos_agent_templates_container +from functions_appinsights import log_event + +STATUS_PENDING = "pending" +STATUS_APPROVED = "approved" +STATUS_REJECTED = "rejected" +STATUS_ARCHIVED = "archived" +ALLOWED_STATUSES = {STATUS_PENDING, STATUS_APPROVED, STATUS_REJECTED, STATUS_ARCHIVED} + +_MAX_TEMPLATE_FIELD_LENGTHS = { + "title": 200, + "display_name": 200, + "helper_text": 140, + "description": 2000, + "instructions": 30000, + "template_key": 128, +} + +_MAX_TEMPLATE_LIST_ITEM_LENGTHS = { + "tags": 64, + "actions_to_load": 128, +} + + +def _utc_now() -> str: + return datetime.utcnow().isoformat() + + +def _slugify(text: str) -> str: + if not text: + return "template" + slug = text.strip().lower() + allowed = "abcdefghijklmnopqrstuvwxyz0123456789-_" + slug = slug.replace(" ", "-") + slug = ''.join(ch for ch in slug if ch in allowed) + slug = slug.strip('-') + return slug or "template" + + +def _normalize_helper_text(description: str, explicit_helper: Optional[str]) -> str: + helper = explicit_helper or description or "" + helper = helper.strip() + if len(helper) <= 140: + return helper + return helper[:137].rstrip() + "..." + + +def _parse_additional_settings(value: Any) -> Dict[str, Any]: + if not value: + return {} + if isinstance(value, dict): + return value + if isinstance(value, str): + trimmed = value.strip() + if not trimmed: + return {} + try: + return json.loads(trimmed) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON for additional_settings: {exc}") from exc + raise ValueError("additional_settings must be a JSON string or object") + + +def _strip_metadata(doc: Dict[str, Any]) -> Dict[str, Any]: + return {k: v for k, v in doc.items() if not k.startswith('_')} + + +def _serialize_additional_settings(raw: Any) -> str: + try: + parsed = _parse_additional_settings(raw) + except ValueError: + return raw if isinstance(raw, str) else "" + if not parsed: + return "" + return json.dumps(parsed, indent=2, sort_keys=True) + + +def _sanitize_template(doc: Dict[str, Any], include_internal: bool = False) -> Dict[str, Any]: + cleaned = _strip_metadata(doc) + cleaned.setdefault('actions_to_load', []) + cleaned['actions_to_load'] = [a for a in cleaned['actions_to_load'] if a] + cleaned.setdefault('tags', []) + cleaned['tags'] = [str(tag)[:64] for tag in cleaned['tags']] + cleaned['helper_text'] = _normalize_helper_text( + cleaned.get('description', ''), + cleaned.get('helper_text') + ) + cleaned['additional_settings'] = _serialize_additional_settings(cleaned.get('additional_settings')) + cleaned.setdefault('status', STATUS_PENDING) + cleaned.setdefault('title', cleaned.get('display_name') or 'Agent Template') + cleaned.setdefault('template_key', _slugify(cleaned['title'])) + + if not include_internal: + for field in ['submission_notes', 'review_notes', 'rejection_reason', 'created_by', 'created_by_email']: + cleaned.pop(field, None) + + return cleaned + + +def _validate_template_lengths(payload: Dict[str, Any]) -> None: + for field, max_len in _MAX_TEMPLATE_FIELD_LENGTHS.items(): + value = payload.get(field, "") + if isinstance(value, str) and len(value) > max_len: + raise ValueError(f"{field} exceeds maximum length of {max_len}.") + + for field, max_len in _MAX_TEMPLATE_LIST_ITEM_LENGTHS.items(): + values = payload.get(field) or [] + if not isinstance(values, list): + continue + for item in values: + if isinstance(item, str) and len(item) > max_len: + raise ValueError(f"{field} entries exceed maximum length of {max_len}.") + + +def validate_template_payload(payload: Dict[str, Any]) -> Optional[str]: + if not isinstance(payload, dict): + return "Template payload must be an object" + if not (payload.get('display_name') or payload.get('title')): + return "Display name is required" + if not payload.get('description'): + return "Description is required" + if not payload.get('instructions'): + return "Instructions are required" + if payload.get('additional_settings'): + try: + _parse_additional_settings(payload['additional_settings']) + except ValueError as exc: + return str(exc) + # Return false if valid to keep with consistency of returning bools or values because we return the error. + return False + + +def list_agent_templates(status: Optional[str] = None, include_internal: bool = False) -> List[Dict[str, Any]]: + query = "SELECT * FROM c" + parameters = [] + if status: + query += " WHERE c.status = @status" + parameters.append({"name": "@status", "value": status}) + + try: + items = list( + cosmos_agent_templates_container.query_items( + query=query, + parameters=parameters or None, + enable_cross_partition_query=True, + ) + ) + except Exception as exc: + current_app.logger.error("Failed to list agent templates: %s", exc) + return [] + + sanitized = [_sanitize_template(item, include_internal) for item in items] + sanitized.sort(key=lambda tpl: tpl.get('title', '').lower()) + return sanitized + + +def get_agent_template(template_id: str) -> Optional[Dict[str, Any]]: + try: + doc = cosmos_agent_templates_container.read_item(item=template_id, partition_key=template_id) + return _sanitize_template(doc, include_internal=True) + except exceptions.CosmosResourceNotFoundError: + return None + except Exception as exc: + current_app.logger.error("Failed to fetch agent template %s: %s", template_id, exc) + return None + + +def _base_template_from_payload(payload: Dict[str, Any], user_info: Optional[Dict[str, Any]], auto_approve: bool) -> Dict[str, Any]: + now = _utc_now() + title = payload.get('title') or payload.get('display_name') or 'Agent Template' + helper_text = _normalize_helper_text(payload.get('description', ''), payload.get('helper_text')) + additional_settings = _parse_additional_settings(payload.get('additional_settings')) + tags = payload.get('tags') or [] + tags = [str(tag)[:64] for tag in tags] + + actions = [str(action) for action in (payload.get('actions_to_load') or []) if action] + + template = { + 'id': payload.get('id') or str(uuid.uuid4()), + 'template_key': payload.get('template_key') or f"{_slugify(title)}-{uuid.uuid4().hex[:6]}", + 'title': title, + 'display_name': payload.get('display_name') or title, + 'helper_text': helper_text, + 'description': payload.get('description', ''), + 'instructions': payload.get('instructions', ''), + 'additional_settings': additional_settings, + 'actions_to_load': actions, + 'tags': tags, + 'status': STATUS_APPROVED if auto_approve else STATUS_PENDING, + 'created_at': now, + 'updated_at': now, + 'created_by': user_info.get('userId') if user_info else None, + 'created_by_name': user_info.get('displayName') if user_info else None, + 'created_by_email': user_info.get('email') if user_info else None, + 'submission_notes': payload.get('submission_notes'), + 'source_agent_id': payload.get('source_agent_id'), + 'source_scope': payload.get('source_scope') or 'personal', + 'approved_by': user_info.get('userId') if auto_approve and user_info else None, + 'approved_at': now if auto_approve else None, + 'review_notes': payload.get('review_notes'), + 'rejection_reason': None, + } + return template + + +def create_agent_template(payload: Dict[str, Any], user_info: Optional[Dict[str, Any]], auto_approve: bool = False) -> Dict[str, Any]: + template = _base_template_from_payload(payload, user_info, auto_approve) + try: + cosmos_agent_templates_container.upsert_item(template) + except Exception as exc: + current_app.logger.error("Failed to save agent template: %s", exc) + raise + + log_event( + "Agent template submitted", + extra={ + "template_id": template['id'], + "status": template['status'], + "created_by": template.get('created_by'), + }, + ) + return _sanitize_template(template, include_internal=True) + + +def update_agent_template(template_id: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]: + doc = get_agent_template(template_id) + if not doc: + return None + + mutable_fields = { + 'title', 'display_name', 'helper_text', 'description', 'instructions', + 'additional_settings', 'actions_to_load', 'tags', 'status' + } + payload = {k: v for k, v in updates.items() if k in mutable_fields} + + if 'additional_settings' in payload: + payload['additional_settings'] = _parse_additional_settings(payload['additional_settings']) + else: + payload['additional_settings'] = _parse_additional_settings(doc.get('additional_settings')) + + if 'tags' in payload: + payload['tags'] = [str(tag)[:64] for tag in payload['tags']] + + if 'status' in payload: + status = payload['status'] + if status not in ALLOWED_STATUSES: + raise ValueError("Invalid template status") + else: + payload['status'] = doc.get('status', STATUS_PENDING) + + template = { + **doc, + **payload, + } + template['helper_text'] = _normalize_helper_text( + template.get('description', ''), + template.get('helper_text') + ) + template['updated_at'] = _utc_now() + template['additional_settings'] = payload['additional_settings'] + _validate_template_lengths(template) + + try: + cosmos_agent_templates_container.upsert_item(template) + except Exception as exc: + current_app.logger.error("Failed to update agent template %s: %s", template_id, exc) + raise + + return _sanitize_template(template, include_internal=True) + + +def approve_agent_template(template_id: str, approver_info: Dict[str, Any], notes: Optional[str] = None) -> Optional[Dict[str, Any]]: + doc = get_agent_template(template_id) + if not doc: + return None + doc['additional_settings'] = _parse_additional_settings(doc.get('additional_settings')) + doc['status'] = STATUS_APPROVED + doc['approved_by'] = approver_info.get('userId') + doc['approved_at'] = _utc_now() + doc['review_notes'] = notes + doc['rejection_reason'] = None + doc['updated_at'] = doc['approved_at'] + + try: + cosmos_agent_templates_container.upsert_item(doc) + except Exception as exc: + current_app.logger.error("Failed to approve agent template %s: %s", template_id, exc) + raise + + log_event( + "Agent template approved", + extra={"template_id": template_id, "approved_by": doc['approved_by']}, + ) + return _sanitize_template(doc, include_internal=True) + + +def reject_agent_template(template_id: str, approver_info: Dict[str, Any], reason: str, notes: Optional[str] = None) -> Optional[Dict[str, Any]]: + doc = get_agent_template(template_id) + if not doc: + return None + doc['additional_settings'] = _parse_additional_settings(doc.get('additional_settings')) + doc['status'] = STATUS_REJECTED + doc['approved_by'] = approver_info.get('userId') + doc['approved_at'] = _utc_now() + doc['review_notes'] = notes + doc['rejection_reason'] = reason + doc['updated_at'] = doc['approved_at'] + + try: + cosmos_agent_templates_container.upsert_item(doc) + except Exception as exc: + current_app.logger.error("Failed to reject agent template %s: %s", template_id, exc) + raise + + log_event( + "Agent template rejected", + extra={"template_id": template_id, "approved_by": doc['approved_by']}, + ) + return _sanitize_template(doc, include_internal=True) + + +def delete_agent_template(template_id: str) -> bool: + try: + cosmos_agent_templates_container.delete_item(item=template_id, partition_key=template_id) + log_event("Agent template deleted", extra={"template_id": template_id}) + return True + except exceptions.CosmosResourceNotFoundError: + return False + except Exception as exc: + current_app.logger.error("Failed to delete agent template %s: %s", template_id, exc) + raise diff --git a/application/single_app/functions_global_agents.py b/application/single_app/functions_global_agents.py index 2ffd9d8f..5cf6a3d4 100644 --- a/application/single_app/functions_global_agents.py +++ b/application/single_app/functions_global_agents.py @@ -16,6 +16,7 @@ from config import cosmos_global_agents_container from functions_keyvault import keyvault_agent_save_helper, keyvault_agent_get_helper, keyvault_agent_delete_helper from functions_settings import * +from functions_agent_payload import sanitize_agent_payload, AgentPayloadError def ensure_default_global_agent_exists(): @@ -173,21 +174,19 @@ def save_global_agent(agent_data): dict: Saved agent data or None if failed """ try: - # Ensure required fields user_id = get_current_user_id() - if 'id' not in agent_data: - agent_data['id'] = str(uuid.uuid4()) - # Add metadata - agent_data['is_global'] = True - agent_data['is_group'] = False - agent_data.setdefault('agent_type', 'local') - agent_data['created_at'] = datetime.utcnow().isoformat() - agent_data['updated_at'] = datetime.utcnow().isoformat() + cleaned_agent = sanitize_agent_payload(agent_data) + if 'id' not in cleaned_agent: + cleaned_agent['id'] = str(uuid.uuid4()) + cleaned_agent['is_global'] = True + cleaned_agent['is_group'] = False + cleaned_agent['created_at'] = datetime.utcnow().isoformat() + cleaned_agent['updated_at'] = datetime.utcnow().isoformat() log_event( "Saving global agent.", - extra={"agent_name": agent_data.get('name', 'Unknown')}, + extra={"agent_name": cleaned_agent.get('name', 'Unknown')}, ) - print(f"Saving global agent: {agent_data.get('name', 'Unknown')}") + print(f"Saving global agent: {cleaned_agent.get('name', 'Unknown')}") # Use the new helper to store sensitive agent keys in Key Vault agent_data = keyvault_agent_save_helper(agent_data, agent_data['id'], scope="global") @@ -198,7 +197,7 @@ def save_global_agent(agent_data): if agent_data.get('reasoning_effort') == '': agent_data.pop('reasoning_effort', None) - result = cosmos_global_agents_container.upsert_item(body=agent_data) + result = cosmos_global_agents_container.upsert_item(body=cleaned_agent) log_event( "Global agent saved successfully.", extra={"agent_id": result['id'], "user_id": user_id}, diff --git a/application/single_app/functions_group_agents.py b/application/single_app/functions_group_agents.py index 76448098..8bf6f87c 100644 --- a/application/single_app/functions_group_agents.py +++ b/application/single_app/functions_group_agents.py @@ -16,6 +16,7 @@ keyvault_agent_get_helper, keyvault_agent_save_helper, ) +from functions_agent_payload import sanitize_agent_payload _NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$") @@ -64,8 +65,8 @@ def get_group_agent(group_id: str, agent_id: str) -> Optional[Dict[str, Any]]: def save_group_agent(group_id: str, agent_data: Dict[str, Any]) -> Dict[str, Any]: """Create or update a group agent entry.""" - agent_id = agent_data.get("id") or str(uuid.uuid4()) - payload = dict(agent_data) + payload = sanitize_agent_payload(agent_data) + agent_id = payload.get("id") or str(uuid.uuid4()) payload["id"] = agent_id payload["group_id"] = group_id payload["last_updated"] = datetime.utcnow().isoformat() diff --git a/application/single_app/functions_personal_agents.py b/application/single_app/functions_personal_agents.py index 7462d1b4..bf721842 100644 --- a/application/single_app/functions_personal_agents.py +++ b/application/single_app/functions_personal_agents.py @@ -18,6 +18,7 @@ from config import cosmos_personal_agents_container from functions_settings import get_settings, get_user_settings, update_user_settings from functions_keyvault import keyvault_agent_save_helper, keyvault_agent_get_helper, keyvault_agent_delete_helper +from functions_agent_payload import sanitize_agent_payload from functions_debug import debug_print def get_personal_agents(user_id): @@ -111,12 +112,27 @@ def save_personal_agent(user_id, agent_data): dict: Saved agent data with ID """ try: - # Ensure required fields - if 'id' not in agent_data: - agent_data['id'] = str(f"{user_id}_{agent_data.get('name', 'default')}") + cleaned_agent = sanitize_agent_payload(agent_data) + for field in ['name', 'display_name', 'description', 'instructions']: + cleaned_agent.setdefault(field, '') + for field in [ + 'azure_openai_gpt_endpoint', + 'azure_openai_gpt_key', + 'azure_openai_gpt_deployment', + 'azure_openai_gpt_api_version', + 'azure_agent_apim_gpt_endpoint', + 'azure_agent_apim_gpt_subscription_key', + 'azure_agent_apim_gpt_deployment', + 'azure_agent_apim_gpt_api_version' + ]: + cleaned_agent.setdefault(field, '') + if 'id' not in cleaned_agent: + cleaned_agent['id'] = str(f"{user_id}_{cleaned_agent.get('name', 'default')}") - agent_data['user_id'] = user_id - agent_data['last_updated'] = datetime.utcnow().isoformat() + cleaned_agent['user_id'] = user_id + cleaned_agent['last_updated'] = datetime.utcnow().isoformat() + cleaned_agent['is_global'] = False + cleaned_agent['is_group'] = False # Validate required fields required_fields = ['name', 'display_name', 'description', 'instructions'] diff --git a/application/single_app/functions_retention_policy.py b/application/single_app/functions_retention_policy.py index 6c59ef64..56167fa1 100644 --- a/application/single_app/functions_retention_policy.py +++ b/application/single_app/functions_retention_policy.py @@ -6,8 +6,9 @@ This module handles automated deletion of aged conversations and documents based on configurable retention policies for personal, group, and public workspaces. -Version: 0.234.067 +Version: 0.236.012 Implemented in: 0.234.067 +Updated in: 0.236.012 - Fixed race condition handling for NotFound errors during deletion """ from config import * @@ -82,6 +83,47 @@ def get_all_public_workspaces(): return [] +def resolve_retention_value(value, workspace_type, retention_type, settings=None): + """ + Resolve a retention value, handling 'default' by looking up organization defaults. + + Args: + value: The retention value ('none', 'default', or a number/string of days) + workspace_type: 'personal', 'group', or 'public' + retention_type: 'conversation' or 'document' + settings: Optional pre-loaded settings dict (to avoid repeated lookups) + + Returns: + str or int: 'none' if no deletion, or the number of days as int + """ + if value is None or value == 'default' or value == '': + # Look up the organization default + if settings is None: + settings = get_settings() + + setting_key = f'default_retention_{retention_type}_{workspace_type}' + default_value = settings.get(setting_key, 'none') + + # If the org default is also 'none', return 'none' + if default_value == 'none' or default_value is None: + return 'none' + + # Return the org default as the effective value + try: + return int(default_value) + except (ValueError, TypeError): + return 'none' + + # User/workspace has their own explicit value + if value == 'none': + return 'none' + + try: + return int(value) + except (ValueError, TypeError): + return 'none' + + def execute_retention_policy(workspace_scopes=None, manual_execution=False): """ Execute retention policy for specified workspace scopes. @@ -185,6 +227,9 @@ def process_personal_retention(): # Get all user settings all_users = get_all_user_settings() + # Pre-load settings once for efficiency + settings = get_settings() + for user in all_users: user_id = user.get('id') if not user_id: @@ -194,10 +239,15 @@ def process_personal_retention(): user_settings = user.get('settings', {}) retention_settings = user_settings.get('retention_policy', {}) - conversation_retention_days = retention_settings.get('conversation_retention_days', 'none') - document_retention_days = retention_settings.get('document_retention_days', 'none') + # Get raw values (may be 'default', 'none', or a number) + raw_conversation_days = retention_settings.get('conversation_retention_days') + raw_document_days = retention_settings.get('document_retention_days') - # Skip if both are set to "none" + # Resolve to effective values (handles 'default' -> org default lookup) + conversation_retention_days = resolve_retention_value(raw_conversation_days, 'personal', 'conversation', settings) + document_retention_days = resolve_retention_value(raw_document_days, 'personal', 'document', settings) + + # Skip if both resolve to "none" if conversation_retention_days == 'none' and document_retention_days == 'none': continue @@ -273,6 +323,9 @@ def process_group_retention(): # Get all groups all_groups = get_all_groups() + # Pre-load settings once for efficiency + settings = get_settings() + for group in all_groups: group_id = group.get('id') if not group_id: @@ -281,10 +334,15 @@ def process_group_retention(): # Get group's retention settings retention_settings = group.get('retention_policy', {}) - conversation_retention_days = retention_settings.get('conversation_retention_days', 'none') - document_retention_days = retention_settings.get('document_retention_days', 'none') + # Get raw values (may be 'default', 'none', or a number) + raw_conversation_days = retention_settings.get('conversation_retention_days') + raw_document_days = retention_settings.get('document_retention_days') - # Skip if both are set to "none" + # Resolve to effective values (handles 'default' -> org default lookup) + conversation_retention_days = resolve_retention_value(raw_conversation_days, 'group', 'conversation', settings) + document_retention_days = resolve_retention_value(raw_document_days, 'group', 'document', settings) + + # Skip if both resolve to "none" if conversation_retention_days == 'none' and document_retention_days == 'none': continue @@ -359,6 +417,9 @@ def process_public_retention(): # Get all public workspaces all_workspaces = get_all_public_workspaces() + # Pre-load settings once for efficiency + settings = get_settings() + for workspace in all_workspaces: workspace_id = workspace.get('id') if not workspace_id: @@ -367,10 +428,15 @@ def process_public_retention(): # Get workspace's retention settings retention_settings = workspace.get('retention_policy', {}) - conversation_retention_days = retention_settings.get('conversation_retention_days', 'none') - document_retention_days = retention_settings.get('document_retention_days', 'none') + # Get raw values (may be 'default', 'none', or a number) + raw_conversation_days = retention_settings.get('conversation_retention_days') + raw_document_days = retention_settings.get('document_retention_days') - # Skip if both are set to "none" + # Resolve to effective values (handles 'default' -> org default lookup) + conversation_retention_days = resolve_retention_value(raw_conversation_days, 'public', 'conversation', settings) + document_retention_days = resolve_retention_value(raw_document_days, 'public', 'document', settings) + + # Skip if both resolve to "none" if conversation_retention_days == 'none' and document_retention_days == 'none': continue @@ -500,10 +566,21 @@ def delete_aged_conversations(retention_days, workspace_type='personal', user_id conversation_title = conv.get('title', 'Untitled') # Read full conversation for archiving/logging - conversation_item = container.read_item( - item=conversation_id, - partition_key=conversation_id - ) + try: + conversation_item = container.read_item( + item=conversation_id, + partition_key=conversation_id + ) + except CosmosResourceNotFoundError: + # Conversation was already deleted (race condition) - this is fine, skip to next + debug_print(f"Conversation {conversation_id} already deleted (not found during read), skipping") + deleted_details.append({ + 'id': conversation_id, + 'title': conversation_title, + 'last_activity_at': conv.get('last_activity_at'), + 'already_deleted': True + }) + continue # Archive if enabled if archiving_enabled: @@ -548,7 +625,11 @@ def delete_aged_conversations(retention_days, workspace_type='personal', user_id archived_msg["archived_by_retention_policy"] = True cosmos_archived_messages_container.upsert_item(archived_msg) - messages_container.delete_item(msg['id'], partition_key=conversation_id) + try: + messages_container.delete_item(msg['id'], partition_key=conversation_id) + except CosmosResourceNotFoundError: + # Message was already deleted - this is fine, continue + debug_print(f"Message {msg['id']} already deleted (not found), skipping") # Log deletion log_conversation_deletion( @@ -566,10 +647,14 @@ def delete_aged_conversations(retention_days, workspace_type='personal', user_id ) # Delete conversation - container.delete_item( - item=conversation_id, - partition_key=conversation_id - ) + try: + container.delete_item( + item=conversation_id, + partition_key=conversation_id + ) + except CosmosResourceNotFoundError: + # Conversation was already deleted after we read it (race condition) - this is fine + debug_print(f"Conversation {conversation_id} already deleted (not found during delete)") deleted_details.append({ 'id': conversation_id, @@ -665,10 +750,21 @@ def delete_aged_documents(retention_days, workspace_type='personal', user_id=Non doc_user_id = doc.get('user_id') or deletion_user_id # Delete document chunks from search index - delete_document_chunks(document_id, group_id, public_workspace_id) + try: + delete_document_chunks(document_id, group_id, public_workspace_id) + except CosmosResourceNotFoundError: + # Document chunks already deleted - this is fine + debug_print(f"Document chunks for {document_id} already deleted (not found)") + except Exception as chunk_error: + # Log chunk deletion errors but continue with document deletion + debug_print(f"Error deleting chunks for document {document_id}: {chunk_error}") # Delete document from Cosmos DB and blob storage - delete_document(doc_user_id, document_id, group_id, public_workspace_id) + try: + delete_document(doc_user_id, document_id, group_id, public_workspace_id) + except CosmosResourceNotFoundError: + # Document was already deleted (race condition) - this is fine + debug_print(f"Document {document_id} already deleted (not found)") deleted_details.append({ 'id': document_id, @@ -679,6 +775,17 @@ def delete_aged_documents(retention_days, workspace_type='personal', user_id=Non debug_print(f"Deleted document {document_id} ({file_name}) due to retention policy") + except CosmosResourceNotFoundError: + # Document was already deleted - count as success + doc_id = doc.get('id', 'unknown') if doc else 'unknown' + debug_print(f"Document {doc_id} already deleted (not found)") + deleted_details.append({ + 'id': doc_id, + 'file_name': doc.get('file_name', 'Unknown'), + 'title': doc.get('title', doc.get('file_name', 'Unknown')), + 'last_updated': doc.get('last_updated'), + 'already_deleted': True + }) except Exception as e: doc_id = doc.get('id', 'unknown') if doc else 'unknown' log_event("delete_aged_documents_deletion_error", {"error": str(e), "document_id": doc_id, "workspace_type": workspace_type}) diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index a0575f54..7a411064 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -40,6 +40,12 @@ def get_settings(use_cosmos=False): 'allow_user_plugins': False, 'allow_group_agents': False, 'allow_group_custom_agent_endpoints': False, + 'allow_ai_foundry_agents': False, + 'allow_group_ai_foundry_agents': False, + 'allow_personal_ai_foundry_agents': False, + 'enable_agent_template_gallery': True, + 'agent_templates_allow_user_submission': True, + 'agent_templates_require_approval': True, 'allow_group_plugins': False, 'id': 'app_settings', # Control Center settings @@ -216,6 +222,34 @@ def get_settings(use_cosmos=False): 'azure_apim_document_intelligence_endpoint': '', 'azure_apim_document_intelligence_subscription_key': '', + # Web search (via Azure AI Foundry agent) + 'enable_web_search': False, + 'web_search_consent_accepted': False, + 'enable_web_search_user_notice': False, # Show popup to users explaining their message will be sent to Bing + 'web_search_user_notice_text': 'Your message will be sent to Microsoft Bing for web search. Only your current message is sent, not your conversation history.', + 'web_search_agent': { + 'agent_type': 'aifoundry', + 'azure_openai_gpt_endpoint': '', + 'azure_openai_gpt_api_version': '', + 'azure_openai_gpt_deployment': '', + 'other_settings': { + 'azure_ai_foundry': { + 'agent_id': '', + 'endpoint': '', + 'api_version': 'v1', + 'authentication_type': 'managed_identity', + 'managed_identity_type': 'system_assigned', + 'managed_identity_client_id': '', + 'tenant_id': '', + 'client_id': '', + 'client_secret': '', + 'cloud': '', + 'authority': '', + 'notes': '' + } + } + }, + # Authentication & Redirect Settings 'enable_front_door': False, 'front_door_url': '', @@ -276,6 +310,15 @@ def get_settings(use_cosmos=False): 'retention_conversation_max_days': 3650, # ~10 years 'retention_document_min_days': 1, 'retention_document_max_days': 3650, # ~10 years + # Default retention policies for each workspace type + # 'none' means no automatic deletion (users can still set their own) + # Numeric values (e.g., 30, 60, 90, 180, 365, 730) represent days + 'default_retention_conversation_personal': 'none', + 'default_retention_document_personal': 'none', + 'default_retention_conversation_group': 'none', + 'default_retention_document_group': 'none', + 'default_retention_conversation_public': 'none', + 'default_retention_document_public': 'none', } try: @@ -732,9 +775,26 @@ def wrapper(*args, **kwargs): return decorator def sanitize_settings_for_user(full_settings: dict) -> dict: - # Exclude any key containing "key", "base64", "storage_account_url" - return {k: v for k, v in full_settings.items() - if "key" not in k.lower() and "storage_account_url" not in k.lower()} + if not isinstance(full_settings, dict): + return full_settings + + sensitive_terms = ("key", "secret", "password", "connection", "base64", "storage_account_url") + sanitized = {} + + for k, v in full_settings.items(): + if any(term in k.lower() for term in sensitive_terms): + continue + if isinstance(v, dict): + sanitized[k] = sanitize_settings_for_user(v) + elif isinstance(v, list): + sanitized[k] = [ + sanitize_settings_for_user(item) if isinstance(item, dict) else item + for item in v + ] + else: + sanitized[k] = v + + return sanitized def sanitize_settings_for_logging(full_settings: dict) -> dict: """ diff --git a/application/single_app/requirements.txt b/application/single_app/requirements.txt index 187b4a98..6a738388 100644 --- a/application/single_app/requirements.txt +++ b/application/single_app/requirements.txt @@ -4,9 +4,9 @@ azure-monitor-query==1.4.1 Flask==2.2.5 Flask-WTF==1.2.1 gunicorn -Werkzeug==3.1.4 +Werkzeug==3.1.5 requests==2.32.4 -openai==1.67 +openai>=1.98.0,<2.0.0 docx2txt==0.8 Markdown==3.3.4 bleach==6.1.0 @@ -41,7 +41,7 @@ xlrd==2.0.1 pillow==11.1.0 ffmpeg-binaries-compat==1.0.1 ffmpeg-python==0.2.0 -semantic-kernel>=1.32.1 +semantic-kernel>=1.39.2 redis>=5.0,<6.0 pyodbc>=4.0.0 PyMySQL>=1.0.0 @@ -49,7 +49,7 @@ azure-monitor-opentelemetry==1.6.13 psycopg2-binary==2.9.10 cython pyyaml==6.0.2 -aiohttp==3.12.15 +aiohttp==3.13.3 html2text==2025.4.15 matplotlib==3.10.7 azure-cognitiveservices-speech==1.47.0 \ No newline at end of file diff --git a/application/single_app/route_backend_agent_templates.py b/application/single_app/route_backend_agent_templates.py new file mode 100644 index 00000000..282b157c --- /dev/null +++ b/application/single_app/route_backend_agent_templates.py @@ -0,0 +1,188 @@ +"""Backend routes for agent template management.""" + +from flask import Blueprint, jsonify, request, session +from swagger_wrapper import swagger_route, get_auth_security + +from functions_authentication import ( + admin_required, + login_required, + get_current_user_info, +) +from functions_agent_templates import ( + STATUS_APPROVED, + validate_template_payload, + list_agent_templates, + create_agent_template, + update_agent_template, + approve_agent_template, + reject_agent_template, + delete_agent_template, + get_agent_template, +) +from functions_settings import get_settings + +bp_agent_templates = Blueprint('agent_templates', __name__) + + +def _feature_flags(): + settings = get_settings() + enabled = settings.get('enable_agent_template_gallery', False) + allow_submissions = settings.get('agent_templates_allow_user_submission', True) + require_approval = settings.get('agent_templates_require_approval', True) + return enabled, allow_submissions, require_approval, settings + + +def _is_admin() -> bool: + user = session.get('user') or {} + return 'Admin' in (user.get('roles') or []) + + +@bp_agent_templates.route('/api/agent-templates', methods=['GET']) +@login_required +@swagger_route(security=get_auth_security()) +def list_public_agent_templates(): + enabled, _, _, _ = _feature_flags() + if not enabled: + return jsonify({'templates': []}) + templates = list_agent_templates(status=STATUS_APPROVED, include_internal=False) + return jsonify({'templates': templates}) + + +@bp_agent_templates.route('/api/agent-templates', methods=['POST']) +@login_required +@swagger_route(security=get_auth_security()) +def submit_agent_template(): + enabled, allow_submissions, require_approval, settings = _feature_flags() + if not enabled: + return jsonify({'error': 'Agent template gallery is disabled.'}), 403 + if not settings.get('allow_user_agents') and not _is_admin(): + return jsonify({'error': 'Agent creation is disabled for your workspace.'}), 403 + if not allow_submissions and not _is_admin(): + return jsonify({'error': 'Template submissions are disabled for users.'}), 403 + + data = request.get_json(silent=True) or {} + payload = data.get('template') or data + validation_error = validate_template_payload(payload) + # validate_template_payload returns false if valid, returns the simple error otherwise. + if validation_error: + return jsonify({'error': validation_error}), 400 + + is_admin_user = _is_admin() + payload['source_agent_id'] = payload.get('source_agent_id') or data.get('source_agent_id') + submission_scope = ( + payload.get('source_scope') + or data.get('source_scope') + or ('global' if is_admin_user else 'personal') + ) + submission_scope = str(submission_scope).lower() + payload['source_scope'] = submission_scope + + admin_context_submission = is_admin_user and submission_scope == 'global' + auto_approve = admin_context_submission or not require_approval + + try: + template = create_agent_template(payload, get_current_user_info(), auto_approve=auto_approve) + except ValueError as exc: + return jsonify({'error': str(exc)}), 400 + except Exception: + return jsonify({'error': 'Failed to submit template.'}), 500 + + if not is_admin_user: + for field in ('submission_notes', 'review_notes', 'rejection_reason', 'created_by_email'): + template.pop(field, None) + + status_code = 201 if template.get('status') == STATUS_APPROVED else 202 + return jsonify({'template': template}), status_code + + +@bp_agent_templates.route('/api/admin/agent-templates', methods=['GET']) +@login_required +@admin_required +@swagger_route(security=get_auth_security()) +def admin_list_agent_templates(): + status = request.args.get('status') + if status == 'all': + status = None + templates = list_agent_templates(status=status, include_internal=True) + return jsonify({'templates': templates}) + + +@bp_agent_templates.route('/api/admin/agent-templates/', methods=['GET']) +@login_required +@admin_required +@swagger_route(security=get_auth_security()) +def admin_get_agent_template(template_id): + template = get_agent_template(template_id) + if not template: + return jsonify({'error': 'Template not found.'}), 404 + return jsonify({'template': template}) + + +@bp_agent_templates.route('/api/admin/agent-templates/', methods=['PATCH']) +@login_required +@admin_required +@swagger_route(security=get_auth_security()) +def admin_update_agent_template(template_id): + payload = request.get_json(silent=True) or {} + try: + template = update_agent_template(template_id, payload) + except ValueError as exc: + return jsonify({'error': str(exc)}), 400 + except Exception: + return jsonify({'error': 'Failed to update template.'}), 500 + + if not template: + return jsonify({'error': 'Template not found.'}), 404 + return jsonify({'template': template}) + + +@bp_agent_templates.route('/api/admin/agent-templates//approve', methods=['POST']) +@login_required +@admin_required +@swagger_route(security=get_auth_security()) +def admin_approve_agent_template(template_id): + data = request.get_json(silent=True) or {} + notes = data.get('notes') + try: + template = approve_agent_template(template_id, get_current_user_info(), notes) + except Exception: + return jsonify({'error': 'Failed to approve template.'}), 500 + + if not template: + return jsonify({'error': 'Template not found.'}), 404 + return jsonify({'template': template}) + + +@bp_agent_templates.route('/api/admin/agent-templates//reject', methods=['POST']) +@login_required +@admin_required +@swagger_route(security=get_auth_security()) +def admin_reject_agent_template(template_id): + data = request.get_json(silent=True) or {} + reason = (data.get('reason') or '').strip() + if not reason: + return jsonify({'error': 'A rejection reason is required.'}), 400 + notes = data.get('notes') + try: + template = reject_agent_template(template_id, get_current_user_info(), reason, notes) + except Exception: + return jsonify({'error': 'Failed to reject template.'}), 500 + + if not template: + return jsonify({'error': 'Template not found.'}), 404 + return jsonify({'template': template}) + + +@bp_agent_templates.route('/api/admin/agent-templates/', methods=['DELETE']) +@login_required +@admin_required +@swagger_route(security=get_auth_security()) +def admin_delete_agent_template(template_id): + try: + deleted = delete_agent_template(template_id) + except Exception: + return jsonify({'error': 'Failed to delete template.'}), 500 + + if not deleted: + return jsonify({'error': 'Template not found.'}), 404 + return jsonify({'success': True}) diff --git a/application/single_app/route_backend_agents.py b/application/single_app/route_backend_agents.py index 5032ebec..b3a8220a 100644 --- a/application/single_app/route_backend_agents.py +++ b/application/single_app/route_backend_agents.py @@ -10,6 +10,7 @@ from functions_global_agents import get_global_agents, save_global_agent, delete_global_agent from functions_personal_agents import get_personal_agents, ensure_migration_complete, save_personal_agent, delete_personal_agent from functions_group import require_active_group, assert_group_role +from functions_agent_payload import sanitize_agent_payload, AgentPayloadError from functions_group_agents import ( get_group_agents, get_group_agent, @@ -111,15 +112,16 @@ def set_user_agents(): for agent in agents: if agent.get('is_global', False): continue # Skip global agents - agent['is_global'] = False # Ensure user agents are not global - agent['is_group'] = False - # --- Require at least one deployment field --- - #if not (agent.get('azure_openai_gpt_deployment') or agent.get('azure_agent_apim_gpt_deployment')): - # return jsonify({'error': f'Agent "{agent.get("name", "(unnamed)")}" must have either azure_openai_gpt_deployment or azure_agent_apim_gpt_deployment set.'}), 400 - validation_error = validate_agent(agent) + try: + cleaned_agent = sanitize_agent_payload(agent) + except AgentPayloadError as exc: + return jsonify({'error': str(exc)}), 400 + cleaned_agent['is_global'] = False + cleaned_agent['is_group'] = False + validation_error = validate_agent(cleaned_agent) if validation_error: return jsonify({'error': f'Agent validation failed: {validation_error}'}), 400 - filtered_agents.append(agent) + filtered_agents.append(cleaned_agent) # Enforce global agent only if per_user_semantic_kernel is False per_user_semantic_kernel = settings.get('per_user_semantic_kernel', False) @@ -258,14 +260,15 @@ def create_group_agent_route(): payload = request.get_json(silent=True) or {} try: validate_group_agent_payload(payload, partial=False) - except ValueError as exc: + cleaned_payload = sanitize_agent_payload(payload) + except (ValueError, AgentPayloadError) as exc: return jsonify({'error': str(exc)}), 400 for key in ('group_id', 'last_updated', 'is_global', 'is_group'): - payload.pop(key, None) + cleaned_payload.pop(key, None) try: - saved = save_group_agent(active_group, payload) + saved = save_group_agent(active_group, cleaned_payload) except Exception as exc: debug_print('Failed to save group agent: %s', exc) return jsonify({'error': 'Unable to save agent'}), 500 @@ -313,7 +316,12 @@ def update_group_agent_route(agent_id): return jsonify({'error': str(exc)}), 400 try: - saved = save_group_agent(active_group, merged) + cleaned_payload = sanitize_agent_payload(merged) + except AgentPayloadError as exc: + return jsonify({'error': str(exc)}), 400 + + try: + saved = save_group_agent(active_group, cleaned_payload) except Exception as exc: debug_print('Failed to update group agent %s: %s', agent_id, exc) return jsonify({'error': 'Unable to update agent'}), 500 @@ -466,26 +474,31 @@ def add_agent(): try: agents = get_global_agents() new_agent = request.json.copy() if hasattr(request.json, 'copy') else dict(request.json) - new_agent['is_global'] = True - new_agent['is_group'] = False - validation_error = validate_agent(new_agent) + try: + cleaned_agent = sanitize_agent_payload(new_agent) + except AgentPayloadError as exc: + log_event("Add agent failed: payload error", level=logging.WARNING, extra={"action": "add", "error": str(exc)}) + return jsonify({'error': str(exc)}), 400 + cleaned_agent['is_global'] = True + cleaned_agent['is_group'] = False + validation_error = validate_agent(cleaned_agent) if validation_error: - log_event("Add agent failed: validation error", level=logging.WARNING, extra={"action": "add", "agent": new_agent, "error": validation_error}) + log_event("Add agent failed: validation error", level=logging.WARNING, extra={"action": "add", "agent": cleaned_agent, "error": validation_error}) return jsonify({'error': validation_error}), 400 # Prevent duplicate names (case-insensitive) - if any(a['name'].lower() == new_agent['name'].lower() for a in agents): - log_event("Add agent failed: duplicate name", level=logging.WARNING, extra={"action": "add", "agent": new_agent}) + if any(a['name'].lower() == cleaned_agent['name'].lower() for a in agents): + log_event("Add agent failed: duplicate name", level=logging.WARNING, extra={"action": "add", "agent": cleaned_agent}) return jsonify({'error': 'Agent with this name already exists.'}), 400 # Assign a new GUID as id unless this is the default agent (which should have a static GUID) - if not new_agent.get('default_agent', False): - new_agent['id'] = str(uuid.uuid4()) + if not cleaned_agent.get('default_agent', False): + cleaned_agent['id'] = str(uuid.uuid4()) else: # If default_agent, ensure the static GUID is present (do not overwrite if already set) - if not new_agent.get('id'): - new_agent['id'] = '15b0c92a-741d-42ff-ba0b-367c7ee0c848' + if not cleaned_agent.get('id'): + cleaned_agent['id'] = '15b0c92a-741d-42ff-ba0b-367c7ee0c848' # Save to global agents container - result = save_global_agent(new_agent) + result = save_global_agent(cleaned_agent) if not result: return jsonify({'error': 'Failed to save agent.'}), 500 @@ -499,7 +512,7 @@ def add_agent(): if not found: return jsonify({'error': 'There must be at least one agent matching the global_selected_agent.'}), 400 - log_event("Agent added", extra={"action": "add", "agent": {k: v for k, v in new_agent.items() if k != 'id'}, "user": str(get_current_user_id())}) + log_event("Agent added", extra={"action": "add", "agent": {k: v for k, v in cleaned_agent.items() if k != 'id'}, "user": str(get_current_user_id())}) # --- HOT RELOAD TRIGGER --- setattr(builtins, "kernel_reload_needed", True) return jsonify({'success': True}) @@ -576,15 +589,20 @@ def edit_agent(agent_name): try: agents = get_global_agents() updated_agent = request.json.copy() if hasattr(request.json, 'copy') else dict(request.json) - updated_agent['is_global'] = True - updated_agent['is_group'] = False - validation_error = validate_agent(updated_agent) + try: + cleaned_agent = sanitize_agent_payload(updated_agent) + except AgentPayloadError as exc: + log_event("Edit agent failed: payload error", level=logging.WARNING, extra={"action": "edit", "agent_name": agent_name, "error": str(exc)}) + return jsonify({'error': str(exc)}), 400 + cleaned_agent['is_global'] = True + cleaned_agent['is_group'] = False + validation_error = validate_agent(cleaned_agent) if validation_error: - log_event("Edit agent failed: validation error", level=logging.WARNING, extra={"action": "edit", "agent": updated_agent, "error": validation_error}) + log_event("Edit agent failed: validation error", level=logging.WARNING, extra={"action": "edit", "agent": cleaned_agent, "error": validation_error}) return jsonify({'error': validation_error}), 400 # --- Require at least one deployment field --- - if not (updated_agent.get('azure_openai_gpt_deployment') or updated_agent.get('azure_agent_apim_gpt_deployment')): - log_event("Edit agent failed: missing deployment field", level=logging.WARNING, extra={"action": "edit", "agent": updated_agent}) + if not (cleaned_agent.get('azure_openai_gpt_deployment') or cleaned_agent.get('azure_agent_apim_gpt_deployment')): + log_event("Edit agent failed: missing deployment field", level=logging.WARNING, extra={"action": "edit", "agent": cleaned_agent}) return jsonify({'error': 'Agent must have either azure_openai_gpt_deployment or azure_agent_apim_gpt_deployment set.'}), 400 # Find the agent to update @@ -592,7 +610,7 @@ def edit_agent(agent_name): for a in agents: if a['name'] == agent_name: # Preserve the existing id - updated_agent['id'] = a.get('id') + cleaned_agent['id'] = a.get('id') agent_found = True break @@ -601,7 +619,7 @@ def edit_agent(agent_name): return jsonify({'error': 'Agent not found.'}), 404 # Save the updated agent - result = save_global_agent(updated_agent) + result = save_global_agent(cleaned_agent) if not result: return jsonify({'error': 'Failed to save agent.'}), 500 @@ -619,7 +637,7 @@ def edit_agent(agent_name): f"Agent {agent_name} edited", extra={ "action": "edit", - "agent": {k: v for k, v in updated_agent.items() if k != 'id'}, + "agent": {k: v for k, v in cleaned_agent.items() if k != 'id'}, "user": str(get_current_user_id()), } ) diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 27c30e9c..ad514e6f 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -8,10 +8,13 @@ from semantic_kernel_fact_memory_store import FactMemoryStore from semantic_kernel_loader import initialize_semantic_kernel from semantic_kernel_plugins.plugin_invocation_logger import get_plugin_logger +from foundry_agent_runtime import FoundryAgentInvocationError, execute_foundry_agent import builtins import asyncio, types +import ast import json -from typing import Any, Dict, List +import re +from typing import Any, Dict, List, Mapping, Optional from config import * from flask import g from functions_authentication import * @@ -22,7 +25,7 @@ from functions_chat import * from functions_conversation_metadata import collect_conversation_metadata, update_conversation_with_metadata from functions_debug import debug_print -from functions_activity_logging import log_chat_activity, log_conversation_creation +from functions_activity_logging import log_chat_activity, log_conversation_creation, log_token_usage from flask import current_app from swagger_wrapper import swagger_route, get_auth_security @@ -55,6 +58,7 @@ def chat_api(): user_message = data.get('message', '') conversation_id = data.get('conversation_id') hybrid_search_enabled = data.get('hybrid_search') + web_search_enabled = data.get('web_search_enabled') selected_document_id = data.get('selected_document_id') image_gen_enabled = data.get('image_generation') document_scope = data.get('doc_scope') @@ -153,6 +157,7 @@ def result_requires_message_reload(result: Any) -> bool: search_query = user_message # <--- ADD THIS LINE (Initialize search_query) hybrid_citations_list = [] # <--- ADD THIS LINE (Initialize hybrid list) agent_citations_list = [] # <--- ADD THIS LINE (Initialize agent citations list) + web_search_citations_list = [] system_messages_for_augmentation = [] # Collect system messages from search search_results = [] selected_agent = None # Initialize selected_agent early to prevent NameError @@ -172,6 +177,8 @@ def result_requires_message_reload(result: Any) -> bool: # Convert toggles from string -> bool if needed if isinstance(hybrid_search_enabled, str): hybrid_search_enabled = hybrid_search_enabled.lower() == 'true' + if isinstance(web_search_enabled, str): + web_search_enabled = web_search_enabled.lower() == 'true' if isinstance(image_gen_enabled, str): image_gen_enabled = image_gen_enabled.lower() == 'true' @@ -262,7 +269,7 @@ def result_requires_message_reload(result: Any) -> bool: debug_print(f"Error initializing GPT client/model: {e}") # Handle error appropriately - maybe return 500 or default behavior return jsonify({'error': f'Failed to initialize AI model: {str(e)}'}), 500 - + # region 1 - Load or Create Conversation # --------------------------------------------------------------------- # 1) Load or create conversation # --------------------------------------------------------------------- @@ -356,7 +363,7 @@ def result_requires_message_reload(result: Any) -> bool: elif document_scope == 'public': actual_chat_type = 'public' debug_print(f"New conversation - using legacy logic: {actual_chat_type}") - + # region 2 - Append User Message # --------------------------------------------------------------------- # 2) Append the user message to conversation immediately (or use existing for retry) # --------------------------------------------------------------------- @@ -406,7 +413,8 @@ def result_requires_message_reload(result: Any) -> bool: # Button states and selections user_metadata['button_states'] = { 'image_generation': image_gen_enabled, - 'document_search': hybrid_search_enabled + 'document_search': hybrid_search_enabled, + 'web_search': bool(web_search_enabled) } # Document search scope and selections @@ -635,7 +643,7 @@ def result_requires_message_reload(result: Any) -> bool: conversation_item['last_updated'] = datetime.utcnow().isoformat() cosmos_conversations_container.upsert_item(conversation_item) # Update timestamp and potentially title - + # region 3 - Content Safety # --------------------------------------------------------------------- # 3) Check Content Safety (but DO NOT return 403). # If blocked, add a "safety" role message & skip GPT. @@ -741,7 +749,7 @@ def result_requires_message_reload(result: Any) -> bool: debug_print(f"[Content Safety Error] {e}") except Exception as ex: debug_print(f"[Content Safety] Unexpected error: {ex}") - + # region 4 - Augmentation # --------------------------------------------------------------------- # 4) Augmentation (Search, etc.) - Run *before* final history prep # --------------------------------------------------------------------- @@ -1449,6 +1457,24 @@ def result_requires_message_reload(result: Any) -> bool: 'error': user_friendly_message }), status_code + if web_search_enabled: + perform_web_search( + settings=settings, + conversation_id=conversation_id, + user_id=user_id, + user_message=user_message, + user_message_id=user_message_id, + chat_type=chat_type, + document_scope=document_scope, + active_group_id=active_group_id, + active_public_workspace_id=active_public_workspace_id, + search_query=search_query, + system_messages_for_augmentation=system_messages_for_augmentation, + agent_citations_list=agent_citations_list, + web_search_citations_list=web_search_citations_list, + ) + + # region 5 - FINAL conversation history preparation # --------------------------------------------------------------------- # 5) Prepare FINAL conversation history for GPT (including summarization) # --------------------------------------------------------------------- @@ -1728,6 +1754,7 @@ def result_requires_message_reload(result: Any) -> bool: debug_print(f"Error preparing conversation history: {e}") return jsonify({'error': f'Error preparing conversation history: {str(e)}'}), 500 + # region 6 - Final GPT Call # --------------------------------------------------------------------- # 6) Final GPT Call # --------------------------------------------------------------------- @@ -2153,12 +2180,101 @@ def agent_error(e): level=logging.ERROR, exceptionTraceback=True ) - fallback_steps.append({ - 'name': 'agent', - 'func': invoke_selected_agent, - 'on_success': agent_success, - 'on_error': agent_error - }) + + selected_agent_type = getattr(selected_agent, 'agent_type', 'local') or 'local' + if isinstance(selected_agent_type, str): + selected_agent_type = selected_agent_type.lower() + + if selected_agent_type == 'aifoundry': + def invoke_foundry_agent(): + foundry_metadata = { + 'conversation_id': conversation_id, + 'user_id': user_id, + 'message_id': user_message_id, + 'chat_type': chat_type, + 'document_scope': document_scope, + 'group_id': active_group_id if chat_type == 'group' else None, + 'hybrid_search_enabled': hybrid_search_enabled, + 'selected_document_id': selected_document_id, + 'search_query': search_query, + } + return selected_agent.invoke( + agent_message_history, + metadata={k: v for k, v in foundry_metadata.items() if v is not None} + ) + + def foundry_agent_success(result): + msg = str(result) + notice = None + agent_used = getattr(selected_agent, 'name', 'Azure AI Foundry Agent') + actual_model_deployment = ( + getattr(selected_agent, 'last_run_model', None) + or getattr(selected_agent, 'deployment_name', None) + or agent_used + ) + + foundry_citations = getattr(selected_agent, 'last_run_citations', []) or [] + if foundry_citations: + for citation in foundry_citations: + try: + serializable = json.loads(json.dumps(citation, default=str)) + except (TypeError, ValueError): + serializable = {'value': str(citation)} + agent_citations_list.append({ + 'tool_name': agent_used, + 'function_name': 'azure_ai_foundry_citation', + 'plugin_name': 'azure_ai_foundry', + 'function_arguments': serializable, + 'function_result': serializable, + 'timestamp': datetime.utcnow().isoformat(), + 'success': True + }) + + if enable_multi_agent_orchestration and not per_user_semantic_kernel: + notice = ( + "[SK Fallback]: The AI assistant is running in single agent fallback mode. " + "Some advanced features may not be available. " + "Please contact your administrator to configure Semantic Kernel for richer responses." + ) + + log_event( + f"[Foundry Agent] Invocation complete for {agent_used}", + extra={ + 'conversation_id': conversation_id, + 'user_id': user_id, + 'agent_id': getattr(selected_agent, 'id', None), + 'model_used': actual_model_deployment, + 'citation_count': len(foundry_citations), + } + ) + + return (msg, actual_model_deployment, 'agent', notice) + + def foundry_agent_error(e): + log_event( + f"Error during Azure AI Foundry agent invocation: {str(e)}", + extra={ + 'conversation_id': conversation_id, + 'user_id': user_id, + 'agent_id': getattr(selected_agent, 'id', None) + }, + level=logging.ERROR, + exceptionTraceback=True + ) + + fallback_steps.append({ + 'name': 'foundry_agent', + 'func': invoke_foundry_agent, + 'on_success': foundry_agent_success, + 'on_error': foundry_agent_error + }) + else: + fallback_steps.append({ + 'name': 'agent', + 'func': invoke_selected_agent, + 'on_success': agent_success, + 'on_error': agent_error + }) if kernel: def invoke_kernel(): @@ -2342,7 +2458,7 @@ def gpt_error(e): exceptionTraceback=True ) - + # region 7 - Save GPT Response # --------------------------------------------------------------------- # 7) Save GPT response (or error message) # --------------------------------------------------------------------- @@ -2390,6 +2506,7 @@ def gpt_error(e): 'timestamp': datetime.utcnow().isoformat(), 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, # <--- SIMPLIFIED: Directly use the list + 'web_search_citations': web_search_citations_list, 'hybridsearch_query': search_query if hybrid_search_enabled and search_results else None, # Log query only if hybrid search ran and found results 'agent_citations': agent_citations_list, # <--- NEW: Store agent tool invocation results 'user_message': user_message, @@ -2521,6 +2638,7 @@ def gpt_error(e): 'blocked': False, # Explicitly false if we got this far 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, + 'web_search_citations': web_search_citations_list, 'agent_citations': agent_citations_list, 'reload_messages': reload_messages_required, 'kernel_fallback_notice': kernel_fallback_notice @@ -2580,6 +2698,7 @@ def generate(): user_message = data.get('message', '') conversation_id = data.get('conversation_id') hybrid_search_enabled = data.get('hybrid_search') + web_search_enabled = data.get('web_search_enabled') selected_document_id = data.get('selected_document_id') image_gen_enabled = data.get('image_generation') document_scope = data.get('doc_scope') @@ -2657,6 +2776,7 @@ def generate(): search_query = user_message hybrid_citations_list = [] agent_citations_list = [] + web_search_citations_list = [] system_messages_for_augmentation = [] search_results = [] selected_agent = None @@ -2670,6 +2790,8 @@ def generate(): # Convert toggles if isinstance(hybrid_search_enabled, str): hybrid_search_enabled = hybrid_search_enabled.lower() == 'true' + if isinstance(web_search_enabled, str): + web_search_enabled = web_search_enabled.lower() == 'true' # Initialize GPT client (simplified version) gpt_model = "" @@ -2716,7 +2838,7 @@ def generate(): credential = DefaultAzureCredential() token_provider = get_bearer_token_provider( credential, - "https://cognitiveservices.azure.com/.default" + cognitive_services_scope ) gpt_client = AzureOpenAI( api_version=api_version, @@ -2789,7 +2911,8 @@ def generate(): user_metadata['button_states'] = { 'image_generation': False, - 'document_search': hybrid_search_enabled + 'document_search': hybrid_search_enabled, + 'web_search': bool(web_search_enabled) } # Document search scope and selections @@ -3126,16 +3249,15 @@ def generate(): retrieved_content = "\n\n".join(retrieved_texts) system_prompt_search = f"""You are an AI assistant. Use the following retrieved document excerpts to answer the user's question. Cite sources using the format (Source: filename, Page: page number). + Retrieved Excerpts: + {retrieved_content} -Retrieved Excerpts: -{retrieved_content} - -Based *only* on the information provided above, answer the user's query. If the answer isn't in the excerpts, say so. + Based *only* on the information provided above, answer the user's query. If the answer isn't in the excerpts, say so. -Example -User: What is the policy on double dipping? -Assistant: The policy prohibits entities from using federal funds received through one program to apply for additional funds through another program, commonly known as 'double dipping' (Source: PolicyDocument.pdf, Page: 12) -""" + Example + User: What is the policy on double dipping? + Assistant: The policy prohibits entities from using federal funds received through one program to apply for additional funds through another program, commonly known as 'double dipping' (Source: PolicyDocument.pdf, Page: 12) + """ system_messages_for_augmentation.append({ 'role': 'system', @@ -3146,6 +3268,23 @@ def generate(): # Reorder hybrid citations list in descending order based on page_number hybrid_citations_list.sort(key=lambda x: x.get('page_number', 0), reverse=True) + if web_search_enabled: + perform_web_search( + settings=settings, + conversation_id=conversation_id, + user_id=user_id, + user_message=user_message, + user_message_id=user_message_id, + chat_type=chat_type, + document_scope=document_scope, + active_group_id=active_group_id, + active_public_workspace_id=active_public_workspace_id, + search_query=search_query, + system_messages_for_augmentation=system_messages_for_augmentation, + agent_citations_list=agent_citations_list, + web_search_citations_list=web_search_citations_list, + ) + # Update message chat type message_chat_type = None if hybrid_search_enabled and search_results and len(search_results) > 0: @@ -3529,6 +3668,7 @@ def make_json_serializable(obj): 'timestamp': datetime.utcnow().isoformat(), 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, + 'web_search_citations': web_search_citations_list, 'hybridsearch_query': search_query if hybrid_search_enabled and search_results else None, 'agent_citations': agent_citations_list, 'user_message': user_message, @@ -3619,6 +3759,7 @@ def make_json_serializable(obj): 'user_message_id': user_message_id, 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, + 'web_search_citations': web_search_citations_list, 'agent_citations': agent_citations_list, 'agent_display_name': agent_display_name_used if use_agent_streaming else None, 'agent_name': agent_name_used if use_agent_streaming else None, @@ -3642,6 +3783,7 @@ def make_json_serializable(obj): 'timestamp': datetime.utcnow().isoformat(), 'augmented': bool(system_messages_for_augmentation), 'hybrid_citations': hybrid_citations_list, + 'web_search_citations': web_search_citations_list, 'hybridsearch_query': search_query if hybrid_search_enabled and search_results else None, 'agent_citations': agent_citations_list, 'user_message': user_message, @@ -3889,4 +4031,414 @@ def remove_masked_content(content, masked_ranges): if start < end: result = result[:start] + result[end:] - return result \ No newline at end of file + return result + + +def _extract_web_search_citations_from_content(content: str) -> List[Dict[str, str]]: + if not content: + return [] + debug_print(f"[Citation Extraction] Extracting citations from:\n{content}\n") + + citations: List[Dict[str, str]] = [] + + markdown_pattern = re.compile(r"\[([^\]]+)\]\((https?://[^\s\)]+)(?:\s+\"([^\"]+)\")?\)") + html_pattern = re.compile( + r"]+href=\"(https?://[^\"]+)\"([^>]*)>(.*?)", + re.IGNORECASE | re.DOTALL, + ) + title_pattern = re.compile(r"title=\"([^\"]+)\"", re.IGNORECASE) + url_pattern = re.compile(r"https?://[^\s\)\]\">]+") + + occupied_spans: List[range] = [] + + for match in markdown_pattern.finditer(content): + text, url, title = match.groups() + url = (url or "").strip().rstrip(".,)") + if not url: + continue + display_title = (title or text or url).strip() + citations.append({"url": url, "title": display_title}) + occupied_spans.append(range(match.start(), match.end())) + + for match in html_pattern.finditer(content): + url, attrs, inner = match.groups() + url = (url or "").strip().rstrip(".,)") + if not url: + continue + title_match = title_pattern.search(attrs or "") + title = title_match.group(1) if title_match else None + inner_text = re.sub(r"<[^>]+>", "", inner or "").strip() + display_title = (title or inner_text or url).strip() + citations.append({"url": url, "title": display_title}) + occupied_spans.append(range(match.start(), match.end())) + + for match in url_pattern.finditer(content): + if any(match.start() in span for span in occupied_spans): + continue + url = (match.group(0) or "").strip().rstrip(".,)") + if not url: + continue + citations.append({"url": url, "title": url}) + debug_print(f"[Citation Extraction] Extracted {len(citations)} citations. - {citations}\n") + + return citations + + +def _extract_token_usage_from_metadata(metadata: Dict[str, Any]) -> Dict[str, int]: + if not isinstance(metadata, Mapping): + debug_print( + "[Web Search][Token Usage Extraction] Metadata is not a mapping. " + f"type={type(metadata)}" + ) + return {} + + usage = metadata.get("usage") + if not usage: + debug_print("[Web Search][Token Usage Extraction] No usage field found in metadata.") + return {} + + if isinstance(usage, str): + raw_usage = usage.strip() + if not raw_usage: + debug_print("[Web Search][Token Usage Extraction] Usage string was empty.") + return {} + try: + usage = json.loads(raw_usage) + except json.JSONDecodeError: + try: + usage = ast.literal_eval(raw_usage) + except (ValueError, SyntaxError): + debug_print( + "[Web Search][Token Usage Extraction] Failed to parse usage string." + ) + return {} + + if not isinstance(usage, Mapping): + debug_print( + "[Web Search][Token Usage Extraction] Usage is not a mapping. " + f"type={type(usage)}" + ) + return {} + + def to_int(value: Any) -> Optional[int]: + try: + return int(float(value)) + except (TypeError, ValueError): + return None + + total_tokens = to_int(usage.get("total_tokens")) + if total_tokens is None: + debug_print( + "[Web Search][Token Usage Extraction] total_tokens missing or invalid. " + f"usage={usage}" + ) + return {} + + prompt_tokens = to_int(usage.get("prompt_tokens")) or 0 + completion_tokens = to_int(usage.get("completion_tokens")) or 0 + debug_print( + "[Web Search][Token Usage Extraction] Extracted token usage - " + f"prompt: {prompt_tokens}, completion: {completion_tokens}, total: {total_tokens}" + ) + + return { + "total_tokens": int(total_tokens), + "prompt_tokens": int(prompt_tokens), + "completion_tokens": int(completion_tokens), + } + +def perform_web_search( + *, + settings, + conversation_id, + user_id, + user_message, + user_message_id, + chat_type, + document_scope, + active_group_id, + active_public_workspace_id, + search_query, + system_messages_for_augmentation, + agent_citations_list, + web_search_citations_list, +): + debug_print("[WebSearch] ========== ENTERING perform_web_search ==========") + debug_print(f"[WebSearch] Parameters received:") + debug_print(f"[WebSearch] conversation_id: {conversation_id}") + debug_print(f"[WebSearch] user_id: {user_id}") + debug_print(f"[WebSearch] user_message: {user_message[:100] if user_message else None}...") + debug_print(f"[WebSearch] user_message_id: {user_message_id}") + debug_print(f"[WebSearch] chat_type: {chat_type}") + debug_print(f"[WebSearch] document_scope: {document_scope}") + debug_print(f"[WebSearch] active_group_id: {active_group_id}") + debug_print(f"[WebSearch] active_public_workspace_id: {active_public_workspace_id}") + debug_print(f"[WebSearch] search_query: {search_query[:100] if search_query else None}...") + + enable_web_search = settings.get("enable_web_search") + debug_print(f"[WebSearch] enable_web_search setting: {enable_web_search}") + + if not enable_web_search: + debug_print("[WebSearch] Web search is DISABLED in settings, returning early") + return True # Not an error, just disabled + + debug_print("[WebSearch] Web search is ENABLED, proceeding...") + + web_search_agent = settings.get("web_search_agent") or {} + debug_print(f"[WebSearch] web_search_agent config present: {bool(web_search_agent)}") + if web_search_agent: + # Avoid logging sensitive data, just log structure + debug_print(f"[WebSearch] web_search_agent keys: {list(web_search_agent.keys())}") + + other_settings = web_search_agent.get("other_settings") or {} + debug_print(f"[WebSearch] other_settings keys: {list(other_settings.keys()) if other_settings else ''}") + + foundry_settings = other_settings.get("azure_ai_foundry") or {} + debug_print(f"[WebSearch] foundry_settings present: {bool(foundry_settings)}") + if foundry_settings: + # Log only non-sensitive keys + safe_keys = ['agent_id', 'project_id', 'endpoint'] + safe_info = {k: foundry_settings.get(k, '') for k in safe_keys} + debug_print(f"[WebSearch] foundry_settings (safe keys): {safe_info}") + + agent_id = (foundry_settings.get("agent_id") or "").strip() + debug_print(f"[WebSearch] Extracted agent_id: '{agent_id}'") + + if not agent_id: + log_event( + "[WebSearch] Skipping Foundry web search: agent_id is not configured", + extra={ + "conversation_id": conversation_id, + "user_id": user_id, + }, + level=logging.WARNING, + ) + debug_print("[WebSearch] Foundry agent_id not configured, skipping web search.") + # Add failure message so the model knows search was requested but not configured + system_messages_for_augmentation.append({ + "role": "system", + "content": "Web search was requested but is not properly configured. Please inform the user that web search is currently unavailable and you cannot provide real-time information. Do not attempt to answer questions requiring current information from your training data.", + }) + return False # Configuration error + + debug_print(f"[WebSearch] Agent ID is configured: {agent_id}") + + query_text = None + try: + query_text = search_query + debug_print(f"[WebSearch] Using search_query as query_text: {query_text[:100] if query_text else None}...") + except NameError: + query_text = None + debug_print("[WebSearch] search_query not defined, query_text is None") + + query_text = (query_text or user_message or "").strip() + debug_print(f"[WebSearch] Final query_text after fallback: '{query_text[:100] if query_text else ''}'") + + if not query_text: + debug_print("[WebSearch] Query text is EMPTY after processing, skipping web search") + log_event( + "[WebSearch] Skipping Foundry web search: empty query", + extra={ + "conversation_id": conversation_id, + "user_id": user_id, + }, + level=logging.WARNING, + ) + return True # Not an error, just empty query + + debug_print(f"[WebSearch] Building message history with query: {query_text[:100]}...") + message_history = [ + ChatMessageContent(role="user", content=query_text) + ] + debug_print(f"[WebSearch] Message history created with {len(message_history)} message(s)") + + try: + foundry_metadata = { + "conversation_id": conversation_id, + "user_id": user_id, + "message_id": user_message_id, + "chat_type": chat_type, + "document_scope": document_scope, + "group_id": active_group_id if chat_type == "group" else None, + "public_workspace_id": active_public_workspace_id, + "search_query": query_text, + } + debug_print(f"[WebSearch] Foundry metadata prepared: {json.dumps(foundry_metadata, default=str)}") + + debug_print("[WebSearch] Calling execute_foundry_agent...") + debug_print(f"[WebSearch] foundry_settings keys: {list(foundry_settings.keys())}") + debug_print(f"[WebSearch] global_settings type: {type(settings)}") + + result = asyncio.run( + execute_foundry_agent( + foundry_settings=foundry_settings, + global_settings=settings, + message_history=message_history, + metadata={k: v for k, v in foundry_metadata.items() if v is not None}, + ) + ) + except FoundryAgentInvocationError as exc: + log_event( + f"[WebSearch] Foundry agent invocation failed: {exc}", + extra={ + "conversation_id": conversation_id, + "user_id": user_id, + "agent_id": agent_id, + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + # Add failure message so the model informs the user + system_messages_for_augmentation.append({ + "role": "system", + "content": f"Web search failed with error: {exc}. Please inform the user that the web search encountered an error and you cannot provide real-time information for this query. Do not attempt to answer questions requiring current information from your training data - instead, acknowledge the search failure and suggest the user try again.", + }) + return False # Search failed + except Exception as exc: + log_event( + f"[WebSearch] Unexpected error invoking Foundry agent: {exc}", + extra={ + "conversation_id": conversation_id, + "user_id": user_id, + "agent_id": agent_id, + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + # Add failure message so the model informs the user + system_messages_for_augmentation.append({ + "role": "system", + "content": f"Web search failed with an unexpected error: {exc}. Please inform the user that the web search encountered an error and you cannot provide real-time information for this query. Do not attempt to answer questions requiring current information from your training data - instead, acknowledge the search failure and suggest the user try again.", + }) + return False # Search failed + + debug_print("[WebSearch] ========== FOUNDRY AGENT RESULT ==========") + debug_print(f"[WebSearch] Result type: {type(result)}") + debug_print(f"[WebSearch] Result has message: {bool(result.message)}") + debug_print(f"[WebSearch] Result has citations: {bool(result.citations)}") + debug_print(f"[WebSearch] Result has metadata: {bool(result.metadata)}") + debug_print(f"[WebSearch] Result model: {getattr(result, 'model', 'N/A')}") + + if result.message: + debug_print(f"[WebSearch] Result message length: {len(result.message)} chars") + debug_print(f"[WebSearch] Result message preview: {result.message[:500] if len(result.message) > 500 else result.message}") + else: + debug_print("[WebSearch] Result message is EMPTY or None") + + if result.citations: + debug_print(f"[WebSearch] Result citations count: {len(result.citations)}") + for i, cit in enumerate(result.citations[:3]): + debug_print(f"[WebSearch] Citation {i}: {json.dumps(cit, default=str)[:200]}...") + else: + debug_print("[WebSearch] Result citations is EMPTY or None") + + if result.metadata: + try: + metadata_payload = json.dumps(result.metadata, default=str) + except (TypeError, ValueError): + metadata_payload = str(result.metadata) + debug_print(f"[WebSearch] Foundry metadata: {metadata_payload}") + else: + debug_print("[WebSearch] Foundry metadata: ") + + if result.message: + debug_print("[WebSearch] Adding result message to system_messages_for_augmentation") + system_messages_for_augmentation.append({ + "role": "system", + "content": f"Web search results:\n{result.message}", + }) + debug_print(f"[WebSearch] Added system message to augmentation list. Total augmentation messages: {len(system_messages_for_augmentation)}") + + debug_print("[WebSearch] Extracting web citations from result message...") + web_citations = _extract_web_search_citations_from_content(result.message) + debug_print(f"[WebSearch] Extracted {len(web_citations)} web citations from message content") + if web_citations: + web_search_citations_list.extend(web_citations) + debug_print(f"[WebSearch] Total web_search_citations_list now has {len(web_search_citations_list)} citations") + else: + debug_print("[WebSearch] No web citations extracted from message content") + else: + debug_print("[WebSearch] No result.message to process for augmentation") + + citations = result.citations or [] + debug_print(f"[WebSearch] Processing {len(citations)} citations from result.citations") + if citations: + for i, citation in enumerate(citations): + debug_print(f"[WebSearch] Processing citation {i}: {json.dumps(citation, default=str)[:200]}...") + try: + serializable = json.loads(json.dumps(citation, default=str)) + except (TypeError, ValueError): + serializable = {"value": str(citation)} + citation_title = serializable.get("title") or serializable.get("url") or "Web search source" + debug_print(f"[WebSearch] Adding agent citation with title: {citation_title}") + agent_citations_list.append({ + "tool_name": citation_title, + "function_name": "azure_ai_foundry_web_search", + "plugin_name": "azure_ai_foundry", + "function_arguments": serializable, + "function_result": serializable, + "timestamp": datetime.utcnow().isoformat(), + "success": True, + }) + debug_print(f"[WebSearch] Total agent_citations_list now has {len(agent_citations_list)} citations") + else: + debug_print("[WebSearch] No citations in result.citations to process") + + debug_print(f"[WebSearch] Starting token usage extraction from Foundry metadata. Metadata: {result.metadata}") + token_usage = _extract_token_usage_from_metadata(result.metadata or {}) + if token_usage.get("total_tokens"): + try: + workspace_type = 'personal' + if active_public_workspace_id: + workspace_type = 'public' + elif active_group_id: + workspace_type = 'group' + + log_token_usage( + user_id=user_id, + token_type='web_search', + total_tokens=token_usage.get('total_tokens', 0), + model=result.model or 'azure-ai-foundry-web-search', + workspace_type=workspace_type, + prompt_tokens=token_usage.get('prompt_tokens'), + completion_tokens=token_usage.get('completion_tokens'), + conversation_id=conversation_id, + message_id=user_message_id, + group_id=active_group_id, + public_workspace_id=active_public_workspace_id, + additional_context={ + 'agent_id': agent_id, + 'search_query': query_text, + 'token_source': 'foundry_metadata' + } + ) + except Exception as log_error: + log_event( + f"[WebSearch] Failed to log web search token usage: {log_error}", + extra={ + "conversation_id": conversation_id, + "user_id": user_id, + "agent_id": agent_id, + }, + level=logging.WARNING, + ) + + debug_print("[WebSearch] ========== FINAL SUMMARY ==========") + debug_print(f"[WebSearch] system_messages_for_augmentation count: {len(system_messages_for_augmentation)}") + debug_print(f"[WebSearch] agent_citations_list count: {len(agent_citations_list)}") + debug_print(f"[WebSearch] web_search_citations_list count: {len(web_search_citations_list)}") + debug_print(f"[WebSearch] Token usage extracted: {token_usage}") + debug_print("[WebSearch] ========== EXITING perform_web_search ==========") + + log_event( + "[WebSearch] Foundry web search invocation complete", + extra={ + "conversation_id": conversation_id, + "user_id": user_id, + "agent_id": agent_id, + "citation_count": len(citations), + }, + level=logging.INFO, + ) + + return True # Search succeeded \ No newline at end of file diff --git a/application/single_app/route_backend_control_center.py b/application/single_app/route_backend_control_center.py index 0e5bcc29..2c3952f1 100644 --- a/application/single_app/route_backend_control_center.py +++ b/application/single_app/route_backend_control_center.py @@ -1355,7 +1355,8 @@ def get_activity_trends_data(start_date, end_date): date_key = current_date.strftime('%Y-%m-%d') token_daily_data[date_key] = { 'embedding': 0, - 'chat': 0 + 'chat': 0, + 'web_search': 0 } current_date += timedelta(days=1) @@ -1364,7 +1365,7 @@ def get_activity_trends_data(start_date, end_date): token_type = token_record.get('token_type', '') token_count = token_record.get('token_count', 0) - if timestamp and token_type in ['embedding', 'chat']: + if timestamp and token_type in ['embedding', 'chat', 'web_search']: try: if isinstance(timestamp, str): token_date = datetime.fromisoformat(timestamp.replace('Z', '+00:00') if 'Z' in timestamp else timestamp) @@ -1387,7 +1388,7 @@ def get_activity_trends_data(start_date, end_date): current_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) while current_date <= end_date: date_key = current_date.strftime('%Y-%m-%d') - token_daily_data[date_key] = {'embedding': 0, 'chat': 0} + token_daily_data[date_key] = {'embedding': 0, 'chat': 0, 'web_search': 0} current_date += timedelta(days=1) # Calculate totals for each day diff --git a/application/single_app/route_backend_plugins.py b/application/single_app/route_backend_plugins.py index edd53dbd..01d448b5 100644 --- a/application/single_app/route_backend_plugins.py +++ b/application/single_app/route_backend_plugins.py @@ -2,6 +2,7 @@ import re import builtins +import json from flask import Blueprint, jsonify, request, current_app from semantic_kernel_plugins.plugin_loader import get_all_plugin_metadata from semantic_kernel_plugins.plugin_health_checker import PluginHealthChecker, PluginErrorRecovery @@ -802,6 +803,58 @@ def merge_plugin_settings(plugin_type): merged = get_merged_plugin_settings(plugin_type, current_settings, schema_dir) return jsonify(merged) + +@bpap.route('/api/plugins//auth-types', methods=['GET']) +@swagger_route(security=get_auth_security()) +@login_required +@user_required +def get_plugin_auth_types(plugin_type): + """ + Returns allowed auth types for a plugin type. Uses definition file if present, + otherwise falls back to AuthType enum in plugin.schema.json. + """ + schema_dir = os.path.join(current_app.root_path, 'static', 'json', 'schemas') + safe_type = re.sub(r'[^a-zA-Z0-9_]', '_', plugin_type).lower() + + definition_path = os.path.join(schema_dir, f'{safe_type}.definition.json') + schema_path = os.path.join(schema_dir, 'plugin.schema.json') + + allowed_auth_types = [] + source = "schema" + + try: + with open(schema_path, 'r', encoding='utf-8') as schema_file: + schema = json.load(schema_file) + allowed_auth_types = ( + schema + .get('definitions', {}) + .get('AuthType', {}) + .get('enum', []) + ) + except Exception as exc: + debug_print(f"Failed to read plugin.schema.json: {exc}") + allowed_auth_types = [] + + if os.path.exists(definition_path): + try: + with open(definition_path, 'r', encoding='utf-8') as definition_file: + definition = json.load(definition_file) + allowed_from_definition = definition.get('allowedAuthTypes') + if isinstance(allowed_from_definition, list) and allowed_from_definition: + allowed_auth_types = allowed_from_definition + source = "definition" + except Exception as exc: + debug_print(f"Failed to read {definition_path}: {exc}") + + if not allowed_auth_types: + allowed_auth_types = [] + source = "schema" + + return jsonify({ + "allowedAuthTypes": allowed_auth_types, + "source": source + }) + ########################################################################################################## # Dynamic Plugin Metadata Endpoint diff --git a/application/single_app/route_backend_retention_policy.py b/application/single_app/route_backend_retention_policy.py index 70d5cc76..60935f60 100644 --- a/application/single_app/route_backend_retention_policy.py +++ b/application/single_app/route_backend_retention_policy.py @@ -3,7 +3,8 @@ from config import * from functions_authentication import * from functions_settings import * -from functions_retention_policy import execute_retention_policy +from functions_retention_policy import execute_retention_policy, get_all_user_settings, get_all_groups, get_all_public_workspaces +from functions_activity_logging import log_retention_policy_force_push from swagger_wrapper import swagger_route, get_auth_security from functions_debug import debug_print @@ -106,6 +107,75 @@ def update_retention_policy_settings(): }), 500 + @app.route('/api/retention-policy/defaults/', methods=['GET']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + def get_retention_policy_defaults(workspace_type): + """ + Get organization default retention policy settings for a specific workspace type. + + Args: + workspace_type: One of 'personal', 'group', or 'public' + + Returns: + JSON with default_conversation_days and default_document_days for the workspace type + """ + try: + # Validate workspace type + if workspace_type not in ['personal', 'group', 'public']: + return jsonify({ + 'success': False, + 'error': f'Invalid workspace type: {workspace_type}' + }), 400 + + settings = get_settings() + + # Get the default values for the specified workspace type + default_conversation = settings.get(f'default_retention_conversation_{workspace_type}', 'none') + default_document = settings.get(f'default_retention_document_{workspace_type}', 'none') + + # Get human-readable labels for the values + def get_retention_label(value): + if value == 'none' or value is None: + return 'No automatic deletion' + try: + days = int(value) + if days == 1: + return '1 day' + elif days == 21: + return '21 days (3 weeks)' + elif days == 90: + return '90 days (3 months)' + elif days == 180: + return '180 days (6 months)' + elif days == 365: + return '365 days (1 year)' + elif days == 730: + return '730 days (2 years)' + else: + return f'{days} days' + except (ValueError, TypeError): + return 'No automatic deletion' + + return jsonify({ + 'success': True, + 'workspace_type': workspace_type, + 'default_conversation_days': default_conversation, + 'default_document_days': default_document, + 'default_conversation_label': get_retention_label(default_conversation), + 'default_document_label': get_retention_label(default_document) + }) + + except Exception as e: + debug_print(f"Error fetching retention policy defaults: {e}") + log_event(f"Fetching retention policy defaults failed: {e}", level=logging.ERROR) + return jsonify({ + 'success': False, + 'error': 'Failed to fetch retention policy defaults' + }), 500 + + @app.route('/api/admin/retention-policy/execute', methods=['POST']) @swagger_route(security=get_auth_security()) @login_required @@ -155,6 +225,165 @@ def manual_execute_retention_policy(): }), 500 + @app.route('/api/admin/retention-policy/force-push', methods=['POST']) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def force_push_retention_defaults(): + """ + Force push organization default retention policies to all users/groups/workspaces. + This resets all custom retention policies to use the organization default ('default' value). + + Body: + scopes (list): List of workspace types to push defaults to: 'personal', 'group', 'public' + """ + try: + data = request.get_json() + scopes = data.get('scopes', []) + + if not scopes: + return jsonify({ + 'success': False, + 'error': 'No workspace scopes provided' + }), 400 + + # Validate scopes + valid_scopes = ['personal', 'group', 'public'] + invalid_scopes = [s for s in scopes if s not in valid_scopes] + if invalid_scopes: + return jsonify({ + 'success': False, + 'error': f'Invalid workspace scopes: {", ".join(invalid_scopes)}' + }), 400 + + details = {} + total_updated = 0 + + # Force push to personal workspaces (user settings) + if 'personal' in scopes: + debug_print("Force pushing retention defaults to personal workspaces...") + all_users = get_all_user_settings() + personal_count = 0 + + for user in all_users: + user_id = user.get('id') + if not user_id: + continue + + try: + # Update user's retention policy to use 'default' + user_settings = user.get('settings', {}) + user_settings['retention_policy'] = { + 'conversation_retention_days': 'default', + 'document_retention_days': 'default' + } + user['settings'] = user_settings + + cosmos_user_settings_container.upsert_item(user) + personal_count += 1 + except Exception as e: + debug_print(f"Error updating user {user_id}: {e}") + log_event(f"Error updating user {user_id} during force push: {e}", level=logging.ERROR) + continue + + details['personal'] = personal_count + total_updated += personal_count + debug_print(f"Updated {personal_count} personal workspaces") + + # Force push to group workspaces + if 'group' in scopes: + debug_print("Force pushing retention defaults to group workspaces...") + from functions_group import cosmos_groups_container + all_groups = get_all_groups() + group_count = 0 + + for group in all_groups: + group_id = group.get('id') + if not group_id: + continue + + try: + # Update group's retention policy to use 'default' + group['retention_policy'] = { + 'conversation_retention_days': 'default', + 'document_retention_days': 'default' + } + + cosmos_groups_container.upsert_item(group) + group_count += 1 + except Exception as e: + debug_print(f"Error updating group {group_id}: {e}") + log_event(f"Error updating group {group_id} during force push: {e}", level=logging.ERROR) + continue + + details['group'] = group_count + total_updated += group_count + debug_print(f"Updated {group_count} group workspaces") + + # Force push to public workspaces + if 'public' in scopes: + debug_print("Force pushing retention defaults to public workspaces...") + from functions_public_workspaces import cosmos_public_workspaces_container + all_workspaces = get_all_public_workspaces() + public_count = 0 + + for workspace in all_workspaces: + workspace_id = workspace.get('id') + if not workspace_id: + continue + + try: + # Update workspace's retention policy to use 'default' + workspace['retention_policy'] = { + 'conversation_retention_days': 'default', + 'document_retention_days': 'default' + } + + cosmos_public_workspaces_container.upsert_item(workspace) + public_count += 1 + except Exception as e: + debug_print(f"Error updating public workspace {workspace_id}: {e}") + log_event(f"Error updating public workspace {workspace_id} during force push: {e}", level=logging.ERROR) + continue + + details['public'] = public_count + total_updated += public_count + debug_print(f"Updated {public_count} public workspaces") + + # Log to activity logs for audit trail + admin_user_id = session.get('user', {}).get('oid', 'unknown') + admin_email = session.get('user', {}).get('preferred_username', session.get('user', {}).get('email', 'unknown')) + log_retention_policy_force_push( + admin_user_id=admin_user_id, + admin_email=admin_email, + scopes=scopes, + results=details, + total_updated=total_updated + ) + + log_event("retention_policy_force_push", { + "scopes": scopes, + "updated_count": total_updated, + "details": details + }) + + return jsonify({ + 'success': True, + 'message': f'Defaults pushed to {total_updated} items', + 'updated_count': total_updated, + 'scopes': scopes, + 'details': details + }) + + except Exception as e: + debug_print(f"Error force pushing retention defaults: {e}") + log_event(f"Force push retention defaults failed: {e}", level=logging.ERROR) + return jsonify({ + 'success': False, + 'error': f'Failed to push retention defaults' + }), 500 + + @app.route('/api/retention-policy/user', methods=['POST']) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/single_app/route_backend_user_agreement.py b/application/single_app/route_backend_user_agreement.py new file mode 100644 index 00000000..f46559ff --- /dev/null +++ b/application/single_app/route_backend_user_agreement.py @@ -0,0 +1,167 @@ +# route_backend_user_agreement.py + +from config import * +from functions_authentication import * +from functions_settings import get_settings +from functions_public_workspaces import find_public_workspace_by_id +from functions_activity_logging import log_user_agreement_accepted, has_user_accepted_agreement_today +from swagger_wrapper import swagger_route, get_auth_security +from functions_debug import debug_print + + +def register_route_backend_user_agreement(app): + """ + Register user agreement API endpoints under '/api/user_agreement/...' + These endpoints handle checking and recording user agreement acceptance. + """ + + @app.route("/api/user_agreement/check", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + def api_check_user_agreement(): + """ + GET /api/user_agreement/check + Check if the current user needs to accept a user agreement for a workspace. + + Query params: + workspace_id: The workspace ID + workspace_type: The workspace type ('personal', 'group', 'public', 'chat') + action_context: The action context ('file_upload', 'chat') - optional + + Returns: + { + needsAgreement: bool, + agreementText: str (if needs agreement), + enableDailyAcceptance: bool + } + """ + info = get_current_user_info() + user_id = info["userId"] + + workspace_id = request.args.get("workspace_id") + workspace_type = request.args.get("workspace_type") + action_context = request.args.get("action_context", "file_upload") + + if not workspace_id or not workspace_type: + return jsonify({"error": "workspace_id and workspace_type are required"}), 400 + + # Validate workspace type + valid_types = ["personal", "group", "public", "chat"] + if workspace_type not in valid_types: + return jsonify({"error": f"Invalid workspace_type. Must be one of: {', '.join(valid_types)}"}), 400 + + # Get global user agreement settings from app settings + settings = get_settings() + + # Check if user agreement is enabled globally + if not settings.get("enable_user_agreement", False): + return jsonify({ + "needsAgreement": False, + "agreementText": "", + "enableDailyAcceptance": False + }), 200 + + apply_to = settings.get("user_agreement_apply_to", []) + + # Check if the agreement applies to this workspace type or action + applies = False + if workspace_type in apply_to: + applies = True + elif action_context == "chat" and "chat" in apply_to: + applies = True + + if not applies: + return jsonify({ + "needsAgreement": False, + "agreementText": "", + "enableDailyAcceptance": False + }), 200 + + # Check if daily acceptance is enabled and user already accepted today + enable_daily_acceptance = settings.get("enable_user_agreement_daily", False) + + if enable_daily_acceptance: + already_accepted = has_user_accepted_agreement_today(user_id, workspace_type, workspace_id) + if already_accepted: + debug_print(f"[USER_AGREEMENT] User {user_id} already accepted today for {workspace_type} workspace {workspace_id}") + return jsonify({ + "needsAgreement": False, + "agreementText": "", + "enableDailyAcceptance": True, + "alreadyAcceptedToday": True + }), 200 + + # User needs to accept the agreement + return jsonify({ + "needsAgreement": True, + "agreementText": settings.get("user_agreement_text", ""), + "enableDailyAcceptance": enable_daily_acceptance + }), 200 + + @app.route("/api/user_agreement/accept", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + def api_accept_user_agreement(): + """ + POST /api/user_agreement/accept + Record that a user has accepted the user agreement for a workspace. + + Body JSON: + { + workspace_id: str, + workspace_type: str ('personal', 'group', 'public'), + action_context: str (optional, e.g., 'file_upload', 'chat') + } + + Returns: + { success: bool, message: str } + """ + info = get_current_user_info() + user_id = info["userId"] + + data = request.get_json() or {} + workspace_id = data.get("workspace_id") + workspace_type = data.get("workspace_type") + action_context = data.get("action_context", "file_upload") + + if not workspace_id or not workspace_type: + return jsonify({"error": "workspace_id and workspace_type are required"}), 400 + + # Validate workspace type + valid_types = ["personal", "group", "public"] + if workspace_type not in valid_types: + return jsonify({"error": f"Invalid workspace_type. Must be one of: {', '.join(valid_types)}"}), 400 + + # Get workspace name for logging + workspace_name = None + if workspace_type == "public": + ws = find_public_workspace_by_id(workspace_id) + if ws: + workspace_name = ws.get("name", "") + + # Log the acceptance + try: + log_user_agreement_accepted( + user_id=user_id, + workspace_type=workspace_type, + workspace_id=workspace_id, + workspace_name=workspace_name, + action_context=action_context + ) + + debug_print(f"[USER_AGREEMENT] Recorded acceptance: user {user_id}, {workspace_type} workspace {workspace_id}") + + return jsonify({ + "success": True, + "message": "User agreement acceptance recorded" + }), 200 + + except Exception as e: + debug_print(f"[USER_AGREEMENT] Error recording acceptance: {str(e)}") + log_event(f"Error recording user agreement acceptance: {str(e)}", level=logging.ERROR) + return jsonify({ + "success": False, + "error": f"Failed to record acceptance: {str(e)}" + }), 500 diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index da45c965..ae361984 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -4,6 +4,7 @@ from functions_documents import * from functions_authentication import * from functions_settings import * +from functions_activity_logging import log_web_search_consent_acceptance from functions_logging import * from swagger_wrapper import swagger_route, get_auth_security from datetime import datetime, timedelta @@ -78,6 +79,9 @@ def admin_settings(): settings['per_user_semantic_kernel'] = False if 'enable_semantic_kernel' not in settings: settings['enable_semantic_kernel'] = False + + if 'web_search_consent_accepted' not in settings: + settings['web_search_consent_accepted'] = False # --- Add default for swagger documentation --- if 'enable_swagger' not in settings: @@ -135,6 +139,9 @@ def admin_settings(): 'name': 'default_agent', 'is_global': True } + log_event("Error retrieving global agents for default selection.", level=logging.ERROR) + debug_print("Error retrieving global agents for default selection.") + if 'allow_user_agents' not in settings: settings['allow_user_agents'] = False if 'allow_user_custom_agent_endpoints' not in settings: @@ -147,6 +154,12 @@ def admin_settings(): settings['allow_group_custom_agent_endpoints'] = False if 'allow_group_plugins' not in settings: settings['allow_group_plugins'] = False + if 'enable_agent_template_gallery' not in settings: + settings['enable_agent_template_gallery'] = True + if 'agent_templates_allow_user_submission' not in settings: + settings['agent_templates_allow_user_submission'] = True + if 'agent_templates_require_approval' not in settings: + settings['agent_templates_require_approval'] = True # --- Add defaults for classification banner --- if 'classification_banner_enabled' not in settings: @@ -158,6 +171,16 @@ def admin_settings(): if 'classification_banner_text_color' not in settings: settings['classification_banner_text_color'] = '#ffffff' # White text by default + # --- Add defaults for user agreement --- + if 'enable_user_agreement' not in settings: + settings['enable_user_agreement'] = False + if 'user_agreement_text' not in settings: + settings['user_agreement_text'] = '' + if 'user_agreement_apply_to' not in settings: + settings['user_agreement_apply_to'] = [] + if 'enable_user_agreement_daily' not in settings: + settings['enable_user_agreement_daily'] = False + # --- Add defaults for key vault if 'enable_key_vault_secret_storage' not in settings: settings['enable_key_vault_secret_storage'] = False @@ -190,7 +213,7 @@ def admin_settings(): pass # Replace with actual logic except Exception as e: print(f"Error retrieving GPT deployments: {e}") - # ... similar try/except for embedding and image models ... + log_event(f"Error retrieving GPT deployments: {e}", level=logging.ERROR) # Check for application updates current_version = app.config['VERSION'] @@ -233,6 +256,7 @@ def admin_settings(): settings.update(new_settings) except Exception as e: print(f"Error checking for updates: {e}") + log_event(f"Error checking for updates: {e}", level=logging.ERROR) # Get the persisted values for template rendering update_available = settings.get('update_available', False) @@ -258,6 +282,7 @@ def admin_settings(): if request.method == 'POST': form_data = request.form # Use a variable for easier access + user_id = get_current_user_id() # --- Fetch all other form data as before --- app_title = form_data.get('app_title', 'AI Chat Application') @@ -279,6 +304,33 @@ def admin_settings(): require_member_of_control_center_dashboard_reader = form_data.get('require_member_of_control_center_dashboard_reader') == 'on' require_member_of_feedback_admin = form_data.get('require_member_of_feedback_admin') == 'on' + web_search_consent_message = ( + "When you use Grounding with Bing Search, your customer data is transferred " + "outside of the Azure compliance boundary to the Grounding with Bing Search service. " + "Grounding with Bing Search is not subject to the same data processing terms " + "(including location of processing) and does not have the same compliance standards " + "and certifications as the Azure AI Agent Service, as described in the " + "Grounding with Bing Search TOU (https://www.microsoft.com/en-us/bing/apis/grounding-legal). " + "It is your responsibility to assess whether use of Grounding with Bing Search in your agent " + "meets your needs and requirements." + ) + web_search_consent_accepted = form_data.get('web_search_consent_accepted') == 'true' + requested_enable_web_search = form_data.get('enable_web_search') == 'on' + enable_web_search = requested_enable_web_search and web_search_consent_accepted + + if requested_enable_web_search and not web_search_consent_accepted: + flash('Web search requires consent before it can be enabled.', 'warning') + + if enable_web_search and web_search_consent_accepted and not settings.get('web_search_consent_accepted'): + admin_user = session.get('user', {}) + admin_email = admin_user.get('preferred_username', admin_user.get('email', 'unknown')) + log_web_search_consent_acceptance( + user_id=user_id, + admin_email=admin_email, + consent_text=web_search_consent_message, + source='admin_settings' + ) + # --- Handle Document Classification Toggle --- enable_document_classification = form_data.get('enable_document_classification') == 'on' @@ -367,19 +419,22 @@ def admin_settings(): except Exception as e: print(f"Error parsing gpt_model_json: {e}") flash('Error parsing GPT model data. Changes may not be saved.', 'warning') + log_event(f"Error parsing GPT model data: {e}", level=logging.ERROR) gpt_model_obj = settings.get('gpt_model', {'selected': [], 'all': []}) # Fallback - # ... similar try/except for embedding and image models ... + try: embedding_model_obj = json.loads(embedding_model_json) if embedding_model_json else {'selected': [], 'all': []} except Exception as e: print(f"Error parsing embedding_model_json: {e}") flash('Error parsing Embedding model data. Changes may not be saved.', 'warning') + log_event(f"Error parsing Embedding model data: {e}", level=logging.ERROR) embedding_model_obj = settings.get('embedding_model', {'selected': [], 'all': []}) # Fallback try: image_gen_model_obj = json.loads(image_gen_model_json) if image_gen_model_json else {'selected': [], 'all': []} except Exception as e: print(f"Error parsing image_gen_model_json: {e}") flash('Error parsing Image Gen model data. Changes may not be saved.', 'warning') + log_event(f"Error parsing Image Gen model data: {e}", level=logging.ERROR) image_gen_model_obj = settings.get('image_gen_model', {'selected': [], 'all': []}) # Fallback # --- Extract banner fields from form_data --- @@ -520,6 +575,14 @@ def admin_settings(): enable_retention_policy_public = form_data.get('enable_retention_policy_public') == 'on' retention_policy_execution_hour = int(form_data.get('retention_policy_execution_hour', 2)) + # Default retention policy values for each workspace type + default_retention_conversation_personal = form_data.get('default_retention_conversation_personal', 'none') + default_retention_document_personal = form_data.get('default_retention_document_personal', 'none') + default_retention_conversation_group = form_data.get('default_retention_conversation_group', 'none') + default_retention_document_group = form_data.get('default_retention_document_group', 'none') + default_retention_conversation_public = form_data.get('default_retention_conversation_public', 'none') + default_retention_document_public = form_data.get('default_retention_document_public', 'none') + # Validate execution hour (0-23) if retention_policy_execution_hour < 0 or retention_policy_execution_hour > 23: retention_policy_execution_hour = 2 # Default to 2 AM @@ -537,6 +600,28 @@ def admin_settings(): retention_policy_next_run = next_run.isoformat() + # --- User Agreement Settings --- + enable_user_agreement = form_data.get('enable_user_agreement') == 'on' + user_agreement_text = form_data.get('user_agreement_text', '').strip() + enable_user_agreement_daily = form_data.get('enable_user_agreement_daily') == 'on' + + # Build apply_to list from checkboxes + user_agreement_apply_to = [] + if form_data.get('user_agreement_apply_personal') == 'on': + user_agreement_apply_to.append('personal') + if form_data.get('user_agreement_apply_group') == 'on': + user_agreement_apply_to.append('group') + if form_data.get('user_agreement_apply_public') == 'on': + user_agreement_apply_to.append('public') + if form_data.get('user_agreement_apply_chat') == 'on': + user_agreement_apply_to.append('chat') + + # Validate word count (max 200 words) + if enable_user_agreement and user_agreement_text: + word_count = len(user_agreement_text.split()) + if word_count > 200: + flash('User Agreement text exceeds 200 word limit. Please shorten the text.', 'warning') + # --- Authentication & Redirect Settings --- enable_front_door = form_data.get('enable_front_door') == 'on' front_door_url = form_data.get('front_door_url', '').strip() @@ -586,6 +671,9 @@ def is_valid_url(url): 'enable_swagger': form_data.get('enable_swagger') == 'on', 'enable_semantic_kernel': form_data.get('enable_semantic_kernel') == 'on', 'per_user_semantic_kernel': form_data.get('per_user_semantic_kernel') == 'on', + 'enable_agent_template_gallery': form_data.get('enable_agent_template_gallery') == 'on', + 'agent_templates_allow_user_submission': form_data.get('agent_templates_allow_user_submission') == 'on', + 'agent_templates_require_approval': form_data.get('agent_templates_require_approval') == 'on', # GPT (Direct & APIM) 'enable_gpt_apim': form_data.get('enable_gpt_apim') == 'on', @@ -657,6 +745,18 @@ def is_valid_url(url): 'enable_retention_policy_public': enable_retention_policy_public, 'retention_policy_execution_hour': retention_policy_execution_hour, 'retention_policy_next_run': retention_policy_next_run, + 'default_retention_conversation_personal': default_retention_conversation_personal, + 'default_retention_document_personal': default_retention_document_personal, + 'default_retention_conversation_group': default_retention_conversation_group, + 'default_retention_document_group': default_retention_document_group, + 'default_retention_conversation_public': default_retention_conversation_public, + 'default_retention_document_public': default_retention_document_public, + + # User Agreement + 'enable_user_agreement': enable_user_agreement, + 'user_agreement_text': user_agreement_text, + 'user_agreement_apply_to': user_agreement_apply_to, + 'enable_user_agreement_daily': enable_user_agreement_daily, # Multimedia & Metadata 'enable_video_file_support': enable_video_file_support, @@ -706,11 +806,33 @@ def is_valid_url(url): 'enable_user_feedback': form_data.get('enable_user_feedback') == 'on', 'enable_conversation_archiving': form_data.get('enable_conversation_archiving') == 'on', - # Search (Web Search Direct & APIM) - 'enable_web_search': form_data.get('enable_web_search') == 'on', - 'enable_web_search_apim': form_data.get('enable_web_search_apim') == 'on', - 'azure_apim_web_search_endpoint': form_data.get('azure_apim_web_search_endpoint', '').strip(), - 'azure_apim_web_search_subscription_key': form_data.get('azure_apim_web_search_subscription_key', '').strip(), + # Search (Web Search via Azure AI Foundry agent) + 'enable_web_search': enable_web_search, + 'web_search_consent_accepted': web_search_consent_accepted, + 'enable_web_search_user_notice': form_data.get('enable_web_search_user_notice') == 'on', + 'web_search_user_notice_text': form_data.get('web_search_user_notice_text', 'Your message will be sent to Microsoft Bing for web search. Only your current message is sent, not your conversation history.').strip(), + 'web_search_agent': { + 'agent_type': 'aifoundry', + 'azure_openai_gpt_endpoint': form_data.get('web_search_foundry_endpoint', '').strip(), + 'azure_openai_gpt_api_version': form_data.get('web_search_foundry_api_version', '').strip(), + 'azure_openai_gpt_deployment': '', + 'other_settings': { + 'azure_ai_foundry': { + 'agent_id': form_data.get('web_search_foundry_agent_id', '').strip(), + 'endpoint': form_data.get('web_search_foundry_endpoint', '').strip(), + 'api_version': form_data.get('web_search_foundry_api_version', '').strip(), + 'authentication_type': form_data.get('web_search_foundry_auth_type', 'managed_identity').strip(), + 'managed_identity_type': form_data.get('web_search_foundry_managed_identity_type', 'system_assigned').strip(), + 'managed_identity_client_id': form_data.get('web_search_foundry_managed_identity_client_id', '').strip(), + 'tenant_id': form_data.get('web_search_foundry_tenant_id', '').strip(), + 'client_id': form_data.get('web_search_foundry_client_id', '').strip(), + 'client_secret': form_data.get('web_search_foundry_client_secret', '').strip(), + 'cloud': form_data.get('web_search_foundry_cloud', '').strip(), + 'authority': form_data.get('web_search_foundry_authority', '').strip(), + 'notes': form_data.get('web_search_foundry_notes', '').strip() + } + } + }, # Search (AI Search Direct & APIM) 'azure_ai_search_endpoint': form_data.get('azure_ai_search_endpoint', '').strip(), @@ -786,6 +908,16 @@ def is_valid_url(url): del new_settings['semantic_kernel_agents'] if 'semantic_kernel_plugins' in new_settings: del new_settings['semantic_kernel_plugins'] + + # Remove legacy web search keys if present + for legacy_key in [ + 'bing_search_key', + 'enable_web_search_apim', + 'azure_apim_web_search_endpoint', + 'azure_apim_web_search_subscription_key' + ]: + if legacy_key in new_settings: + del new_settings[legacy_key] logo_file = request.files.get('logo_file') if logo_file and allowed_file(logo_file.filename, ALLOWED_EXTENSIONS_IMG): @@ -866,7 +998,7 @@ def is_valid_url(url): except Exception as e: print(f"Error processing logo file: {e}") # Log the error for debugging flash(f"Error processing logo file: {e}. Existing logo preserved.", "danger") - # On error, new_settings['custom_logo_base64'] keeps its initial value (the old logo) + log_event(f"Error processing logo file: {e}", level=logging.ERROR) # Process dark mode logo file upload logo_dark_file = request.files.get('logo_dark_file') @@ -949,7 +1081,7 @@ def is_valid_url(url): except Exception as e: print(f"Error processing dark mode logo file: {e}") # Log the error for debugging flash(f"Error processing dark mode logo file: {e}. Existing dark mode logo preserved.", "danger") - # On error, new_settings['custom_logo_dark_base64'] keeps its initial value (the old logo) + log_event(f"Error processing dark mode logo file: {e}", level=logging.ERROR) # Process favicon file upload favicon_file = request.files.get('favicon_file') @@ -1023,7 +1155,7 @@ def is_valid_url(url): except Exception as e: print(f"Error processing favicon file: {e}") # Log the error for debugging flash(f"Error processing favicon file: {e}. Existing favicon preserved.", "danger") - # On error, new_settings['custom_favicon_base64'] keeps its initial value (the old favicon) + log_event(f"Error processing favicon file: {e}", level=logging.ERROR) # --- Update settings in DB --- # new_settings now contains either the new logo/favicon base64 or the original ones diff --git a/application/single_app/semantic_kernel_loader.py b/application/single_app/semantic_kernel_loader.py index 0874fa20..35d35965 100644 --- a/application/single_app/semantic_kernel_loader.py +++ b/application/single_app/semantic_kernel_loader.py @@ -20,6 +20,7 @@ from semantic_kernel_plugins.embedding_model_plugin import EmbeddingModelPlugin from semantic_kernel_plugins.fact_memory_plugin import FactMemoryPlugin from functions_settings import get_settings, get_user_settings +from foundry_agent_runtime import AzureAIFoundryChatCompletionAgent from functions_appinsights import log_event, get_appinsights_logger from functions_authentication import get_current_user_id from semantic_kernel_plugins.plugin_health_checker import PluginHealthChecker, PluginErrorRecovery @@ -106,6 +107,7 @@ def resolve_agent_config(agent, settings): debug_print(f"[SK Loader] Agent is_group flag: {agent.get('is_group')}") agent_type = (agent.get('agent_type') or 'local').lower() agent['agent_type'] = agent_type + other_settings = agent.get("other_settings", {}) or {} gpt_model_obj = settings.get('gpt_model', {}) selected_model = gpt_model_obj.get('selected', [{}])[0] if gpt_model_obj.get('selected') else {} @@ -231,6 +233,22 @@ def merge_fields(primary, fallback): return tuple(p if p not in [None, ""] else f for p, f in zip(primary, fallback)) # If per-user mode is not enabled, ignore all user/agent-specific config fields + if agent_type == "aifoundry": + return { + "name": agent.get("name"), + "display_name": agent.get("display_name", agent.get("name")), + "description": agent.get("description", ""), + "id": agent.get("id", ""), + "default_agent": agent.get("default_agent", False), + "is_global": agent.get("is_global", False), + "is_group": agent.get("is_group", False), + "group_id": agent.get("group_id"), + "group_name": agent.get("group_name"), + "agent_type": "aifoundry", + "other_settings": other_settings, + "max_completion_tokens": agent.get("max_completion_tokens", -1), + } + if not per_user_enabled: try: if global_apim_enabled: @@ -258,7 +276,8 @@ def merge_fields(primary, fallback): "group_name": agent.get("group_name"), "enable_agent_gpt_apim": agent.get("enable_agent_gpt_apim", False), "max_completion_tokens": agent.get("max_completion_tokens", -1), - "agent_type": agent_type or "local" + "agent_type": agent_type or "local", + "other_settings": other_settings, } except Exception as e: log_event(f"[SK Loader] Error resolving agent config: {e}", level=logging.ERROR, exceptionTraceback=True) @@ -317,6 +336,7 @@ def merge_fields(primary, fallback): "enable_agent_gpt_apim": agent.get("enable_agent_gpt_apim", False), # Use this to check if APIM is enabled for the agent "max_completion_tokens": agent.get("max_completion_tokens", -1), # -1 meant use model default determined by the service, 35-trubo is 4096, 4o is 16384, 4.1 is at least 32768 "agent_type": agent_type or "local", + "other_settings": other_settings, } print(f"[SK Loader] Final resolved config for {agent.get('name')}: endpoint={bool(endpoint)}, key={bool(key)}, deployment={deployment}") @@ -722,6 +742,20 @@ def load_single_agent_for_kernel(kernel, agent_cfg, settings, context_obj, redis chat_service = None apim_enabled = settings.get("enable_gpt_apim", False) + if agent_type == "aifoundry": + foundry_agent = AzureAIFoundryChatCompletionAgent(agent_config, settings) + agent_objs[agent_config["name"]] = foundry_agent + log_event( + f"[SK Loader] Registered Foundry agent: {agent_config['name']} ({mode_label})", + { + "agent_name": agent_config["name"], + "agent_id": agent_config.get("id"), + "is_global": agent_config.get("is_global", False), + }, + level=logging.INFO, + ) + return kernel, agent_objs + log_event(f"[SK Loader] Agent config resolved for {agent_cfg.get('name')} - endpoint: {bool(agent_config.get('endpoint'))}, key: {bool(agent_config.get('key'))}, deployment: {agent_config.get('deployment')}, max_completion_tokens: {agent_config.get('max_completion_tokens')}", level=logging.INFO) if AzureChatCompletion and agent_config["endpoint"] and agent_config["key"] and agent_config["deployment"]: diff --git a/application/single_app/semantic_kernel_plugins/smart_http_plugin.py b/application/single_app/semantic_kernel_plugins/smart_http_plugin.py index f5209685..2292e7bc 100644 --- a/application/single_app/semantic_kernel_plugins/smart_http_plugin.py +++ b/application/single_app/semantic_kernel_plugins/smart_http_plugin.py @@ -560,6 +560,7 @@ async def _summarize_large_content(self, content: str, uri: str, page_count: int from functions_settings import get_settings from openai import AzureOpenAI from azure.identity import DefaultAzureCredential, get_bearer_token_provider + from config import cognitive_services_scope settings = get_settings() @@ -580,7 +581,6 @@ async def _summarize_large_content(self, content: str, uri: str, page_count: int ) else: if settings.get('azure_openai_gpt_authentication_type') == 'managed_identity': - cognitive_services_scope = "https://cognitiveservices.azure.com/.default" token_provider = get_bearer_token_provider( DefaultAzureCredential(), cognitive_services_scope diff --git a/application/single_app/static/js/admin/admin_agent_templates.js b/application/single_app/static/js/admin/admin_agent_templates.js new file mode 100644 index 00000000..4bea4924 --- /dev/null +++ b/application/single_app/static/js/admin/admin_agent_templates.js @@ -0,0 +1,515 @@ +// admin_agent_templates.js +// Admin UI logic for reviewing, approving, and deleting agent template submissions + +import { showToast } from "../chat/chat-toast.js"; + +const panel = document.getElementById("agent-templates-admin-panel"); +const tableBody = document.getElementById("agent-template-table-body"); +const statusFilters = document.getElementById("agent-template-status-filters"); +const disabledAlert = document.getElementById("agent-templates-disabled-alert"); +const searchInput = document.getElementById("agent-template-search"); +const paginationEl = document.getElementById("agent-template-pagination"); +const paginationSummary = document.getElementById("agent-template-pagination-summary"); +const paginationNav = document.getElementById("agent-template-pagination-nav"); +const modalEl = document.getElementById("agentTemplateReviewModal"); +const approveBtn = document.getElementById("agent-template-approve-btn"); +const rejectBtn = document.getElementById("agent-template-reject-btn"); +const deleteBtn = document.getElementById("agent-template-delete-btn"); +const notesInput = document.getElementById("agent-template-review-notes"); +const rejectReasonInput = document.getElementById("agent-template-reject-reason"); +const errorAlert = document.getElementById("agent-template-review-error"); +const statusBadge = document.getElementById("agent-template-review-status"); +const helperEl = document.getElementById("agent-template-review-helper"); +const descriptionEl = document.getElementById("agent-template-review-description"); +const instructionsEl = document.getElementById("agent-template-review-instructions"); +const actionsWrapper = document.getElementById("agent-template-review-actions-wrapper"); +const actionsList = document.getElementById("agent-template-review-actions"); +const settingsWrapper = document.getElementById("agent-template-review-settings-wrapper"); +const settingsEl = document.getElementById("agent-template-review-settings"); +const tagsContainer = document.getElementById("agent-template-review-tags"); +const subtitleEl = document.getElementById("agent-template-review-subtitle"); +const metaEl = document.getElementById("agent-template-review-meta"); +const titleEl = document.getElementById("agentTemplateReviewModalLabel"); + +let currentFilter = "pending"; +let templates = []; +let selectedTemplate = null; +let reviewModal = null; +let currentPage = 1; +let searchQuery = ""; +const PAGE_SIZE = 10; + +function init() { + if (!panel) { + return; + } + + if (modalEl && window.bootstrap) { + reviewModal = bootstrap.Modal.getOrCreateInstance(modalEl); + } + + if (!window.appSettings?.enable_agent_template_gallery) { + if (disabledAlert) disabledAlert.classList.remove("d-none"); + renderEmptyState("Template gallery is disabled."); + return; + } + + attachFilterHandlers(); + attachTableHandlers(); + attachSearchHandler(); + attachModalHandlers(); + loadTemplatesForFilter(currentFilter); +} + +function attachFilterHandlers() { + if (!statusFilters) { + return; + } + statusFilters.addEventListener("click", (event) => { + const button = event.target.closest("button[data-status]"); + if (!button) { + return; + } + const { status } = button.dataset; + if (!status || status === currentFilter) { + return; + } + currentFilter = status; + statusFilters.querySelectorAll("button").forEach((btn) => btn.classList.remove("active")); + button.classList.add("active"); + currentPage = 1; + loadTemplatesForFilter(currentFilter); + }); +} + +function attachTableHandlers() { + if (!tableBody) { + return; + } + tableBody.addEventListener("click", (event) => { + const reviewBtn = event.target.closest(".agent-template-review-btn"); + if (reviewBtn) { + const templateId = reviewBtn.dataset.templateId; + openReviewModal(templateId); + return; + } + const deleteBtn = event.target.closest(".agent-template-inline-delete"); + if (deleteBtn) { + const templateId = deleteBtn.dataset.templateId; + confirmAndDelete(templateId); + } + }); +} + +function attachModalHandlers() { + if (!approveBtn || !rejectBtn || !deleteBtn) { + return; + } + + approveBtn.addEventListener("click", () => handleApproval()); + rejectBtn.addEventListener("click", () => handleRejection()); + deleteBtn.addEventListener("click", () => { + if (selectedTemplate?.id) { + confirmAndDelete(selectedTemplate.id, true); + } + }); +} + +function attachSearchHandler() { + if (!searchInput) { + return; + } + searchInput.addEventListener("input", (event) => { + searchQuery = event.target.value?.trim().toLowerCase() || ""; + currentPage = 1; + renderTemplates(); + }); +} + +async function loadTemplatesForFilter(status) { + renderLoadingRow(); + try { + const query = status && status !== "all" ? `?status=${encodeURIComponent(status)}` : "?status=all"; + const response = await fetch(`/api/admin/agent-templates${query}`); + if (!response.ok) { + throw new Error("Failed to load templates."); + } + const data = await response.json(); + templates = data.templates || []; + currentPage = 1; + renderTemplates(); + } catch (error) { + console.error("Error loading agent templates", error); + renderEmptyState(error.message || "Unable to load templates."); + } +} + +function renderLoadingRow() { + if (!tableBody) return; + tableBody.innerHTML = ` + + Loading templates... + `; + setSummaryMessage("Loading templates..."); + renderPaginationControls(0); +} + +function renderEmptyState(message) { + if (!tableBody) return; + tableBody.innerHTML = `${message}`; + setSummaryMessage(message); + renderPaginationControls(0); +} + +function renderTemplates() { + if (!tableBody) { + return; + } + const filtered = getFilteredTemplates(); + if (!filtered.length) { + const emptyMessage = searchQuery ? "No templates match your search." : "No templates found for this filter."; + renderEmptyState(emptyMessage); + return; + } + + const totalItems = filtered.length; + const totalPages = Math.ceil(totalItems / PAGE_SIZE) || 1; + if (currentPage > totalPages) { + currentPage = totalPages; + } + const startIndex = (currentPage - 1) * PAGE_SIZE; + const pageItems = filtered.slice(startIndex, startIndex + PAGE_SIZE); + const endIndex = startIndex + pageItems.length; + + tableBody.innerHTML = ""; + pageItems.forEach((template) => { + const row = document.createElement("tr"); + row.innerHTML = ` + +
${escapeHtml(template.title || template.display_name || "Template")}
+
${escapeHtml(template.helper_text || template.description || "")}
+ + ${renderStatusBadge(template.status)} + +
${escapeHtml(template.created_by_name || 'Unknown')}
+
${escapeHtml(template.created_by_email || '')}
+ + ${formatDate(template.updated_at || template.created_at)} + +
+ + +
+ + `; + tableBody.appendChild(row); + }); + + setSummaryMessage(`Showing ${startIndex + 1}-${endIndex} of ${totalItems} (page ${currentPage} of ${totalPages})`); + renderPaginationControls(totalPages); +} + +function getFilteredTemplates() { + if (!searchQuery) { + return templates; + } + return templates.filter((template) => { + return [ + template.title, + template.display_name, + template.created_by_name, + template.created_by_email + ].some((value) => value && value.toString().toLowerCase().includes(searchQuery)); + }); +} + +function renderStatusBadge(status) { + const normalized = (status || "pending").toLowerCase(); + const variants = { + approved: "success", + rejected: "danger", + archived: "secondary", + pending: "warning", + }; + const badgeClass = variants[normalized] || "secondary"; + return `${normalized}`; +} + +function setSummaryMessage(message = "") { + if (paginationSummary) { + paginationSummary.textContent = message; + } +} + +function renderPaginationControls(totalPages) { + if (!paginationEl) { + return; + } + + if (paginationNav) { + if (totalPages <= 1) { + paginationNav.classList.add("d-none"); + } else { + paginationNav.classList.remove("d-none"); + } + } + + if (totalPages <= 1) { + paginationEl.innerHTML = ""; + return; + } + + const maxButtons = 5; + let startPage = Math.max(1, currentPage - Math.floor(maxButtons / 2)); + let endPage = startPage + maxButtons - 1; + if (endPage > totalPages) { + endPage = totalPages; + startPage = Math.max(1, endPage - maxButtons + 1); + } + + const fragment = document.createDocumentFragment(); + fragment.appendChild(createPageItem("Previous", currentPage - 1, currentPage === 1)); + + for (let page = startPage; page <= endPage; page += 1) { + fragment.appendChild(createPageItem(page, page, false, page === currentPage)); + } + + fragment.appendChild(createPageItem("Next", currentPage + 1, currentPage === totalPages)); + + paginationEl.innerHTML = ""; + paginationEl.appendChild(fragment); +} + +function createPageItem(label, targetPage, disabled, active = false) { + const li = document.createElement("li"); + li.className = "page-item"; + if (disabled) li.classList.add("disabled"); + if (active) li.classList.add("active"); + + const button = document.createElement("button"); + button.type = "button"; + button.className = "page-link"; + button.textContent = label.toString(); + button.disabled = disabled; + button.addEventListener("click", () => { + if (disabled || targetPage === currentPage) { + return; + } + currentPage = Math.min(Math.max(targetPage, 1), Math.ceil(getFilteredTemplates().length / PAGE_SIZE) || 1); + renderTemplates(); + }); + + li.appendChild(button); + return li; +} + +function formatDate(value) { + if (!value) { + return "-"; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return value; + } + return date.toLocaleString(); +} + +async function openReviewModal(templateId) { + if (!templateId || !reviewModal) { + return; + } + try { + const response = await fetch(`/api/admin/agent-templates/${templateId}`); + if (!response.ok) { + throw new Error('Failed to load template.'); + } + const data = await response.json(); + selectedTemplate = data.template; + populateReviewModal(selectedTemplate); + reviewModal.show(); + } catch (error) { + console.error('Failed to open template modal', error); + showToast(error.message || 'Unable to load template.', 'danger'); + } +} + +function populateReviewModal(template) { + if (!template) { + return; + } + titleEl.textContent = template.title || template.display_name || 'Agent Template'; + helperEl.textContent = template.helper_text || template.description || '-'; + descriptionEl.textContent = template.description || '-'; + instructionsEl.textContent = template.instructions || ''; + notesInput.value = template.review_notes || ''; + rejectReasonInput.value = template.rejection_reason || ''; + updateStatusBadge(template.status); + + const submittedBy = template.created_by_name || 'Unknown submitter'; + const submittedAt = formatDate(template.created_at); + subtitleEl.textContent = `Submitted by ${submittedBy} on ${submittedAt}`; + metaEl.textContent = `Updated ${formatDate(template.updated_at)}`; + + if (Array.isArray(template.actions_to_load) && template.actions_to_load.length) { + actionsWrapper.classList.remove('d-none'); + actionsList.innerHTML = ''; + template.actions_to_load.forEach((action) => { + const badge = document.createElement('span'); + badge.className = 'badge bg-info text-dark me-1 mb-1'; + badge.textContent = action; + actionsList.appendChild(badge); + }); + } else { + actionsWrapper.classList.add('d-none'); + actionsList.innerHTML = ''; + } + + if (template.additional_settings) { + settingsWrapper.classList.remove('d-none'); + settingsEl.textContent = template.additional_settings; + } else { + settingsWrapper.classList.add('d-none'); + settingsEl.textContent = ''; + } + + if (Array.isArray(template.tags) && template.tags.length) { + tagsContainer.classList.remove('d-none'); + tagsContainer.innerHTML = ''; + template.tags.slice(0, 8).forEach((tag) => { + const badge = document.createElement('span'); + badge.className = 'badge bg-secondary-subtle text-secondary-emphasis'; + badge.textContent = tag; + tagsContainer.appendChild(badge); + }); + } else { + tagsContainer.classList.add('d-none'); + tagsContainer.innerHTML = ''; + } + + hideModalError(); +} + +function updateStatusBadge(status) { + const normalized = (status || 'pending').toLowerCase(); + statusBadge.textContent = normalized; + statusBadge.className = 'badge'; + statusBadge.classList.add(`bg-${{ + approved: 'success', + rejected: 'danger', + archived: 'secondary', + pending: 'warning' + }[normalized] || 'secondary'}`); +} + +function hideModalError() { + if (errorAlert) { + errorAlert.classList.add('d-none'); + errorAlert.textContent = ''; + } +} + +function showModalError(message) { + if (!errorAlert) { + showToast(message, 'danger'); + return; + } + errorAlert.classList.remove('d-none'); + errorAlert.textContent = message; +} + +async function handleApproval() { + if (!selectedTemplate?.id) { + return; + } + await submitTemplateDecision(`/api/admin/agent-templates/${selectedTemplate.id}/approve`, { + notes: notesInput.value?.trim() || undefined + }, 'Template approved!'); +} + +async function handleRejection() { + if (!selectedTemplate?.id) { + return; + } + const reason = rejectReasonInput.value?.trim(); + if (!reason) { + showModalError('A rejection reason is required.'); + rejectReasonInput.focus(); + return; + } + await submitTemplateDecision(`/api/admin/agent-templates/${selectedTemplate.id}/reject`, { + reason, + notes: notesInput.value?.trim() || undefined + }, 'Template rejected.'); +} + +async function submitTemplateDecision(url, payload, successMessage) { + try { + setModalButtonsDisabled(true); + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + const data = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(data.error || 'Failed to update template.'); + } + showToast(successMessage, 'success'); + hideModalError(); + reviewModal?.hide(); + loadTemplatesForFilter(currentFilter); + } catch (error) { + console.error('Template decision failed', error); + showModalError(error.message || 'Failed to update template.'); + } finally { + setModalButtonsDisabled(false); + } +} + +function setModalButtonsDisabled(disabled) { + [approveBtn, rejectBtn, deleteBtn].forEach((btn) => { + if (btn) btn.disabled = disabled; + }); +} + +async function confirmAndDelete(templateId, closeModal = false) { + if (!templateId) { + return; + } + if (!confirm('Delete this template? This action cannot be undone.')) { + return; + } + try { + const response = await fetch(`/api/admin/agent-templates/${templateId}`, { + method: 'DELETE' + }); + const data = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(data.error || 'Failed to delete template.'); + } + showToast('Template deleted.', 'success'); + if (closeModal) { + reviewModal?.hide(); + } + loadTemplatesForFilter(currentFilter); + } catch (error) { + console.error('Failed to delete template', error); + showToast(error.message || 'Failed to delete template.', 'danger'); + } +} + +function escapeHtml(value) { + const div = document.createElement('div'); + div.textContent = value || ''; + return div.innerHTML; +} + +if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', init); +} else { + init(); +} diff --git a/application/single_app/static/js/admin/admin_settings.js b/application/single_app/static/js/admin/admin_settings.js index 6b3ed8c2..81f80f9e 100644 --- a/application/single_app/static/js/admin/admin_settings.js +++ b/application/single_app/static/js/admin/admin_settings.js @@ -1575,22 +1575,158 @@ function setupToggles() { } const enableWebSearch = document.getElementById('enable_web_search'); - if (enableWebSearch) { + const webSearchFoundrySettings = document.getElementById('web_search_foundry_settings'); + const webSearchConsentInput = document.getElementById('web_search_consent_accepted'); + const webSearchConsentModalEl = document.getElementById('web-search-consent-modal'); + const webSearchConsentAcceptBtn = document.getElementById('web-search-consent-accept'); + const webSearchConsentDeclineBtn = document.getElementById('web-search-consent-decline'); + let webSearchConsentModal = null; + const toggleVisibility = (element, isVisible) => { + if (!element) { + return; + } + element.classList.toggle('d-none', !isVisible); + }; + if (enableWebSearch && webSearchFoundrySettings) { + const setConsentAccepted = (value) => { + if (webSearchConsentInput) { + webSearchConsentInput.value = value ? 'true' : 'false'; + } + }; + + const showConsentModal = () => { + if (!webSearchConsentModalEl) { + showToast('Consent modal could not be loaded.', 'warning'); + return; + } + + if (!webSearchConsentModal) { + webSearchConsentModal = new bootstrap.Modal(webSearchConsentModalEl, { + backdrop: 'static', + keyboard: false + }); + } + + webSearchConsentModal.show(); + }; + + const hasConsent = () => webSearchConsentInput?.value === 'true'; + + if (enableWebSearch.checked && !hasConsent()) { + enableWebSearch.checked = false; + } + toggleVisibility(webSearchFoundrySettings, enableWebSearch.checked && hasConsent()); + enableWebSearch.addEventListener('change', function () { - document.getElementById('web_search_settings').style.display = this.checked ? 'block' : 'none'; + if (this.checked && !hasConsent()) { + this.checked = false; + toggleVisibility(webSearchFoundrySettings, false); + showConsentModal(); + return; + } + + toggleVisibility(webSearchFoundrySettings, this.checked); + markFormAsModified(); + }); + + if (webSearchConsentAcceptBtn) { + webSearchConsentAcceptBtn.addEventListener('click', () => { + setConsentAccepted(true); + enableWebSearch.checked = true; + toggleVisibility(webSearchFoundrySettings, true); + markFormAsModified(); + if (webSearchConsentModal) { + webSearchConsentModal.hide(); + } + }); + } + + if (webSearchConsentDeclineBtn) { + webSearchConsentDeclineBtn.addEventListener('click', () => { + setConsentAccepted(false); + enableWebSearch.checked = false; + toggleVisibility(webSearchFoundrySettings, false); + markFormAsModified(); + if (webSearchConsentModal) { + webSearchConsentModal.hide(); + } + }); + } + } + + // Web Search User Notice toggle + const enableWebSearchUserNotice = document.getElementById('enable_web_search_user_notice'); + const webSearchUserNoticeSettings = document.getElementById('web_search_user_notice_settings'); + if (enableWebSearchUserNotice && webSearchUserNoticeSettings) { + enableWebSearchUserNotice.addEventListener('change', function() { + toggleVisibility(webSearchUserNoticeSettings, this.checked); + markFormAsModified(); + }); + } + + const foundryAuthType = document.getElementById('web_search_foundry_auth_type'); + const foundryMiType = document.getElementById('web_search_foundry_managed_identity_type'); + const foundryCloud = document.getElementById('web_search_foundry_cloud'); + const foundrySpFields = document.getElementById('web_search_foundry_service_principal_fields'); + const foundryMiTypeContainer = document.getElementById('web_search_foundry_managed_identity_type_container'); + const foundryMiClientIdContainer = document.getElementById('web_search_foundry_managed_identity_client_id_container'); + const foundryCloudContainer = document.getElementById('web_search_foundry_cloud_container'); + const foundryAuthorityContainer = document.getElementById('web_search_foundry_authority_container'); + + function updateFoundryAuthVisibility() { + const authType = foundryAuthType?.value || 'managed_identity'; + const cloudValue = foundryCloud?.value || ''; + + toggleVisibility(foundrySpFields, authType === 'service_principal'); + toggleVisibility(foundryCloudContainer, authType === 'service_principal'); + toggleVisibility( + foundryAuthorityContainer, + authType === 'service_principal' && cloudValue === 'custom' + ); + toggleVisibility(foundryMiTypeContainer, authType === 'managed_identity'); + if (foundryMiClientIdContainer) { + const miType = foundryMiType?.value || 'system_assigned'; + toggleVisibility( + foundryMiClientIdContainer, + authType === 'managed_identity' && miType === 'user_assigned' + ); + } + } + + if (foundryAuthType || foundryMiType || foundryCloud) { + updateFoundryAuthVisibility(); + } + + if (foundryMiType) { + foundryMiType.addEventListener('change', () => { + updateFoundryAuthVisibility(); markFormAsModified(); }); } - const enableWebSearchApim = document.getElementById('enable_web_search_apim'); - if (enableWebSearchApim) { - enableWebSearchApim.addEventListener('change', function () { - document.getElementById('non_apim_web_search_settings').style.display = this.checked ? 'none' : 'block'; - document.getElementById('apim_web_search_settings').style.display = this.checked ? 'block' : 'none'; + if (foundryCloud) { + foundryCloud.addEventListener('change', () => { + updateFoundryAuthVisibility(); markFormAsModified(); }); } + if (foundryAuthType) { + foundryAuthType.addEventListener('change', () => { + updateFoundryAuthVisibility(); + markFormAsModified(); + }); + } + + const toggleFoundrySecret = document.getElementById('toggle_web_search_foundry_client_secret'); + const foundrySecretInput = document.getElementById('web_search_foundry_client_secret'); + if (toggleFoundrySecret && foundrySecretInput) { + toggleFoundrySecret.addEventListener('click', () => { + foundrySecretInput.type = foundrySecretInput.type === 'password' ? 'text' : 'password'; + toggleFoundrySecret.textContent = foundrySecretInput.type === 'password' ? 'Show' : 'Hide'; + }); + } + const enableAiSearchApim = document.getElementById('enable_ai_search_apim'); if (enableAiSearchApim) { enableAiSearchApim.addEventListener('change', function () { diff --git a/application/single_app/static/js/admin/admin_sidebar_nav.js b/application/single_app/static/js/admin/admin_sidebar_nav.js index 72965781..3f1bb667 100644 --- a/application/single_app/static/js/admin/admin_sidebar_nav.js +++ b/application/single_app/static/js/admin/admin_sidebar_nav.js @@ -206,6 +206,7 @@ function scrollToSection(sectionId) { // Security tab sections 'keyvault-section': 'keyvault-section', // Search & Extract tab sections + 'web-search-section': 'web-search-foundry-section', 'azure-ai-search-section': 'azure-ai-search-section', 'document-intelligence-section': 'document-intelligence-section', 'multimedia-support-section': 'multimedia-support-section' diff --git a/application/single_app/static/js/agent_modal_stepper.js b/application/single_app/static/js/agent_modal_stepper.js index 30cf31fc..800751be 100644 --- a/application/single_app/static/js/agent_modal_stepper.js +++ b/application/single_app/static/js/agent_modal_stepper.js @@ -10,11 +10,18 @@ export class AgentModalStepper { this.maxSteps = 6; this.isEditMode = false; this.isAdmin = isAdmin; // Track if this is admin context + this.currentAgentType = 'local'; this.originalAgent = null; // Track original state for change detection this.actionsToSelect = null; // Store actions to select when they're loaded this.updateStepIndicatorTimeout = null; // For debouncing step indicator updates + this.templateSubmitButton = document.getElementById('agent-modal-submit-template-btn'); + this.foundryPlaceholderInstructions = 'Placeholder instructions: Azure AI Foundry agent manages its own prompt.'; this.bindEvents(); + + if (this.templateSubmitButton) { + this.templateSubmitButton.addEventListener('click', () => this.submitTemplate()); + } } bindEvents() { @@ -24,6 +31,7 @@ export class AgentModalStepper { const saveBtn = document.getElementById('agent-modal-save-btn'); const skipBtn = document.getElementById('agent-modal-skip'); const powerUserToggle = document.getElementById('agent-power-user-toggle'); + const agentTypeRadios = document.querySelectorAll('input[name="agent-type"]'); if (nextBtn) { nextBtn.addEventListener('click', () => this.nextStep()); @@ -40,6 +48,12 @@ export class AgentModalStepper { if (powerUserToggle) { powerUserToggle.addEventListener('change', (e) => this.togglePowerUserMode(e.target.checked)); } + + if (agentTypeRadios && agentTypeRadios.length) { + agentTypeRadios.forEach(r => { + r.addEventListener('change', (e) => this.handleAgentTypeChange(e.target.value)); + }); + } // Set up display name to generated name conversion this.setupNameGeneration(); @@ -70,6 +84,90 @@ export class AgentModalStepper { } } + handleAgentTypeChange(agentType) { + this.currentAgentType = agentType || 'local'; + this.applyAgentTypeVisibility(); + // Clear actions if switching to foundry + if (this.currentAgentType === 'aifoundry') { + this.clearSelectedActions(); + } + this.populateSummary(); + } + + applyAgentTypeVisibility() { + const isFoundry = this.currentAgentType === 'aifoundry'; + const foundryFields = document.getElementById('agent-foundry-fields'); + const modelGroup = document.getElementById('agent-global-model-group'); + const customToggle = document.getElementById('agent-custom-connection-toggle'); + const customFields = document.getElementById('agent-custom-connection-fields'); + const actionsSection = document.getElementById('agent-step-4'); + const actionsDisabled = document.getElementById('agent-actions-disabled'); + const actionsContainer = document.getElementById('agent-actions-container'); + const actionsHeader = actionsSection?.querySelector('.card'); + const summaryActionsSection = document.getElementById('summary-actions-section'); + const instructionsContainer = document.getElementById('agent-instructions-container'); + const instructionsFoundryNote = document.getElementById('agent-instructions-foundry-note'); + const instructionsInput = document.getElementById('agent-instructions'); + + if (foundryFields) foundryFields.classList.toggle('d-none', !isFoundry); + if (modelGroup) modelGroup.classList.toggle('d-none', isFoundry); + if (customToggle) customToggle.classList.toggle('d-none', isFoundry); + if (customFields) customFields.classList.toggle('d-none', isFoundry); + + if (instructionsContainer) instructionsContainer.classList.toggle('d-none', isFoundry); + if (instructionsFoundryNote) instructionsFoundryNote.classList.toggle('d-none', !isFoundry); + if (instructionsInput) { + if (isFoundry) { + instructionsInput.value = this.foundryPlaceholderInstructions; + } + } + + if (actionsSection) { + // Hide interactive actions when foundry + if (actionsDisabled) actionsDisabled.classList.toggle('d-none', !isFoundry); + if (actionsHeader) actionsHeader.classList.toggle('d-none', isFoundry); + if (actionsContainer) actionsContainer.classList.toggle('d-none', isFoundry); + const noActionsMsg = document.getElementById('agent-no-actions-message'); + if (noActionsMsg) noActionsMsg.classList.toggle('d-none', isFoundry); + const selectedSummary = document.getElementById('agent-selected-actions-summary'); + if (selectedSummary) selectedSummary.classList.toggle('d-none', isFoundry); + } + + if (summaryActionsSection) { + summaryActionsSection.classList.toggle('d-none', isFoundry); + } + + // Update helper text + const helper = document.getElementById('agent-type-helper'); + if (helper) { + helper.textContent = isFoundry + ? 'Foundry agents use Azure-managed tools. Actions step is disabled.' + : 'Local agents can attach actions and use SK plugins.'; + } + } + + updateAgentTypeLock() { + const radios = document.querySelectorAll('input[name="agent-type"]'); + if (!radios || !radios.length) { + return; + } + + const shouldDisable = this.isEditMode || this.currentStep > 1; + + radios.forEach(radio => { + radio.disabled = shouldDisable; + const wrapper = radio.closest('.form-check'); + if (wrapper) { + wrapper.classList.toggle('opacity-50', shouldDisable); + } + }); + + const selector = document.getElementById('agent-type-selector'); + if (selector) { + selector.classList.toggle('pe-none', shouldDisable); + } + } + updateReasoningEffortForModel() { const globalModelSelect = document.getElementById('agent-global-model-select'); const reasoningEffortSelect = document.getElementById('agent-reasoning-effort'); @@ -147,6 +245,7 @@ export class AgentModalStepper { showModal(agent = null) { this.isEditMode = !!agent; + this.currentAgentType = (agent && agent.agent_type) || 'local'; // Store original state for change detection this.originalAgent = agent ? JSON.parse(JSON.stringify(agent)) : null; @@ -179,6 +278,9 @@ export class AgentModalStepper { // Ensure generated name is populated for both new and existing agents this.updateGeneratedName(); + this.syncAgentTypeSelector(); + this.applyAgentTypeVisibility(); + this.updateAgentTypeLock(); // Load models for the modal this.loadModelsForModal(); @@ -197,6 +299,7 @@ export class AgentModalStepper { this.updateStepIndicator(); this.showStep(1); this.updateNavigationButtons(); + this.updateTemplateButtonVisibility(); console.log('Step indicators initialized'); } else { // Modal not ready yet, try again @@ -225,6 +328,14 @@ export class AgentModalStepper { } } + syncAgentTypeSelector() { + const radios = document.querySelectorAll('input[name="agent-type"]'); + if (!radios || !radios.length) return; + radios.forEach(r => { + r.checked = r.value === this.currentAgentType; + }); + } + clearFields() { // Clear all form fields const displayName = document.getElementById('agent-display-name'); @@ -282,6 +393,11 @@ export class AgentModalStepper { customConnection.checked = agentsCommon.shouldEnableCustomConnection(agent); } + // Agent type selection + this.currentAgentType = agent.agent_type || 'local'; + this.syncAgentTypeSelector(); + this.applyAgentTypeVisibility(); + // Use shared function to populate all fields if (agentsCommon && typeof agentsCommon.setAgentModalFields === 'function') { agentsCommon.setAgentModalFields(agent); @@ -327,6 +443,24 @@ export class AgentModalStepper { if (agent.actions_to_load && Array.isArray(agent.actions_to_load)) { this.actionsToSelect = agent.actions_to_load; } + + // Foundry-specific fields + if (agent.agent_type === 'aifoundry') { + const other = agent.other_settings || {}; + const foundry = (other && other.azure_ai_foundry) || {}; + const endpointEl = document.getElementById('agent-foundry-endpoint'); + const apiEl = document.getElementById('agent-foundry-api-version'); + const depEl = document.getElementById('agent-foundry-deployment'); + const idEl = document.getElementById('agent-foundry-agent-id'); + const notesEl = document.getElementById('agent-foundry-notes'); + if (endpointEl) endpointEl.value = agent.azure_openai_gpt_endpoint || ''; + if (apiEl) apiEl.value = agent.azure_openai_gpt_api_version || ''; + if (depEl) depEl.value = agent.azure_openai_gpt_deployment || ''; + if (idEl) idEl.value = foundry.agent_id || ''; + if (notesEl) notesEl.value = foundry.notes || ''; + // ensure actions cleared for UI + this.clearSelectedActions(); + } } nextStep() { @@ -357,7 +491,9 @@ export class AgentModalStepper { skipBtn.innerHTML = `Skipping...`; } try { - await this.loadAvailableActions(); + if (this.currentAgentType !== 'aifoundry') { + await this.loadAvailableActions(); + } this.goToStep(this.maxSteps); } catch (error) { console.error('Error loading actions:', error); @@ -380,6 +516,8 @@ export class AgentModalStepper { this.showStep(stepNumber); this.updateStepIndicator(); this.updateNavigationButtons(); + this.updateTemplateButtonVisibility(); + this.updateAgentTypeLock(); } showStep(stepNumber) { @@ -398,22 +536,31 @@ export class AgentModalStepper { } if (stepNumber === 2) { - if (!this.isAdmin) { - const customConnectionToggle = document.getElementById('agent-custom-connection-toggle'); - if (customConnectionToggle) { + const isFoundry = this.currentAgentType === 'aifoundry'; + const customConnectionToggle = document.getElementById('agent-custom-connection-toggle'); + const modelGroup = document.getElementById('agent-global-model-group'); + + if (customConnectionToggle) { + if (isFoundry) { + customConnectionToggle.classList.add('d-none'); + } else if (!this.isAdmin) { const allowUserCustom = appSettings?.allow_user_custom_agent_endpoints; - if (!allowUserCustom) { - customConnectionToggle.classList.add('d-none'); - } else { - customConnectionToggle.classList.remove('d-none'); - } + customConnectionToggle.classList.toggle('d-none', !allowUserCustom); + } else { + customConnectionToggle.classList.remove('d-none'); } } + + if (modelGroup) { + modelGroup.classList.toggle('d-none', isFoundry); + } } // Load actions when reaching step 4 if (stepNumber === 4) { - this.loadAvailableActions(); + if (this.currentAgentType !== 'aifoundry') { + this.loadAvailableActions(); + } } // Populate summary when reaching step 6 @@ -511,6 +658,27 @@ export class AgentModalStepper { } } + canSubmitTemplate() { + if (!window.appSettings || !window.appSettings.enable_agent_template_gallery) { + return false; + } + if (this.isAdmin) { + return true; + } + if (window.appSettings.allow_user_agents === false) { + return false; + } + return window.appSettings.agent_templates_allow_user_submission !== false; + } + + updateTemplateButtonVisibility() { + if (!this.templateSubmitButton) { + return; + } + const shouldShow = this.canSubmitTemplate() && this.currentStep === this.maxSteps; + this.templateSubmitButton.classList.toggle('d-none', !shouldShow); + } + validateCurrentStep() { switch (this.currentStep) { case 1: // Basic Info @@ -531,20 +699,54 @@ export class AgentModalStepper { break; case 2: // Model & Connection - // Model validation would go here + if (this.currentAgentType === 'aifoundry') { + const endpoint = document.getElementById('agent-foundry-endpoint'); + const apiVersion = document.getElementById('agent-foundry-api-version'); + const deployment = document.getElementById('agent-foundry-deployment'); + const agentId = document.getElementById('agent-foundry-agent-id'); + if (!endpoint || !endpoint.value.trim()) { + this.showError('Azure AI Foundry endpoint is required.'); + endpoint?.focus(); + return false; + } + if (!apiVersion || !apiVersion.value.trim()) { + this.showError('Azure AI Foundry API version is required.'); + apiVersion?.focus(); + return false; + } + if (!deployment || !deployment.value.trim()) { + this.showError('Foundry deployment/project is required.'); + deployment?.focus(); + return false; + } + if (!agentId || !agentId.value.trim()) { + this.showError('Foundry agent ID is required.'); + agentId?.focus(); + return false; + } + } break; case 3: // Instructions const instructions = document.getElementById('agent-instructions'); - if (!instructions || !instructions.value.trim()) { - this.showError('Please provide instructions for the agent.'); - if (instructions) instructions.focus(); - return false; - } + if (this.currentAgentType !== 'aifoundry') { + if (!instructions || !instructions.value.trim()) { + this.showError('Please provide instructions for the agent.'); + if (instructions) instructions.focus(); + return false; + } + } else { + // Ensure placeholder present + if (instructions && !instructions.value.trim()) { + instructions.value = this.foundryPlaceholderInstructions; + } + } break; case 4: // Actions - // Actions validation would go here if needed + if (this.currentAgentType !== 'aifoundry') { + // Actions validation would go here if needed + } break; case 5: // Advanced @@ -648,6 +850,10 @@ export class AgentModalStepper { } getFormModelName() { + if (this.currentAgentType === 'aifoundry') { + const foundryDeployment = document.getElementById('agent-foundry-deployment'); + return foundryDeployment?.value?.trim() || '-'; + } const customConnection = document.getElementById('agent-custom-connection')?.checked || false; let modelName = '-'; if (customConnection) { @@ -671,6 +877,7 @@ export class AgentModalStepper { const displayName = document.getElementById('agent-display-name')?.value || '-'; const generatedName = document.getElementById('agent-name')?.value || '-'; const description = document.getElementById('agent-description')?.value || '-'; + const agentType = this.currentAgentType || 'local'; // Model & Connection const customConnection = document.getElementById('agent-custom-connection')?.checked ? 'Yes' : 'No'; @@ -691,6 +898,11 @@ export class AgentModalStepper { // Update configuration document.getElementById('summary-model').textContent = modelName; document.getElementById('summary-custom-connection').textContent = customConnection; + const typeBadge = document.getElementById('summary-agent-type-badge'); + if (typeBadge) { + typeBadge.textContent = agentType === 'aifoundry' ? 'Azure AI Foundry' : 'Local (Semantic Kernel)'; + typeBadge.className = agentType === 'aifoundry' ? 'badge bg-warning text-dark' : 'badge bg-info'; + } // Update instructions document.getElementById('summary-instructions').textContent = instructions; @@ -705,10 +917,16 @@ export class AgentModalStepper { const actionsListContainer = document.getElementById('summary-actions-list'); const actionsEmptyContainer = document.getElementById('summary-actions-empty'); - if (actionsCount > 0) { + if (this.currentAgentType === 'aifoundry') { + // Hide actions entirely for Foundry + const actionsSection = document.getElementById('summary-actions-section'); + if (actionsSection) actionsSection.style.display = 'none'; + } else if (actionsCount > 0) { // Show actions list, hide empty message actionsListContainer.style.display = 'block'; actionsEmptyContainer.style.display = 'none'; + const actionsSection = document.getElementById('summary-actions-section'); + if (actionsSection) actionsSection.style.display = ''; // Clear existing content actionsListContainer.innerHTML = ''; @@ -751,6 +969,8 @@ export class AgentModalStepper { // Hide actions list, show empty message actionsListContainer.style.display = 'none'; actionsEmptyContainer.style.display = 'block'; + const actionsSection = document.getElementById('summary-actions-section'); + if (actionsSection) actionsSection.style.display = ''; } // Update creation date @@ -1247,8 +1467,12 @@ export class AgentModalStepper { } } - // Add selected actions - agentData.actions_to_load = this.getSelectedActionIds(); + // Add selected actions (skip for Foundry) + if (agentData.agent_type === 'aifoundry') { + agentData.actions_to_load = []; + } else { + agentData.actions_to_load = this.getSelectedActionIds(); + } agentData.is_global = this.isAdmin; // Set based on admin context // Ensure required schema fields are present @@ -1304,6 +1528,9 @@ export class AgentModalStepper { } getAgentFormData() { + const agentTypeInput = document.querySelector('input[name="agent-type"]:checked'); + const selectedAgentType = agentTypeInput ? agentTypeInput.value : 'local'; + const formData = { display_name: document.getElementById('agent-display-name')?.value || '', name: document.getElementById('agent-name')?.value || '', @@ -1314,8 +1541,37 @@ export class AgentModalStepper { other_settings: document.getElementById('agent-additional-settings')?.value || '{}', max_completion_tokens: parseInt(document.getElementById('agent-max-completion-tokens')?.value.trim()) || null, reasoning_effort: document.getElementById('agent-reasoning-effort')?.value || '', - agent_type: 'local' + agent_type: selectedAgentType }; + + if (selectedAgentType === 'aifoundry') { + // Foundry required fields + formData.azure_openai_gpt_endpoint = document.getElementById('agent-foundry-endpoint')?.value?.trim() || ''; + formData.azure_openai_gpt_deployment = document.getElementById('agent-foundry-deployment')?.value?.trim() || ''; + formData.azure_openai_gpt_api_version = document.getElementById('agent-foundry-api-version')?.value?.trim() || ''; + formData.instructions = document.getElementById('agent-instructions')?.value?.trim() || this.foundryPlaceholderInstructions; + + // other_settings for foundry + let otherSettingsObj = {}; + try { + otherSettingsObj = JSON.parse(formData.other_settings || '{}'); + } catch (e) { + otherSettingsObj = {}; + } + otherSettingsObj = otherSettingsObj || {}; + const notesVal = document.getElementById('agent-foundry-notes')?.value || ''; + otherSettingsObj.azure_ai_foundry = { + ...(otherSettingsObj.azure_ai_foundry || {}), + agent_id: document.getElementById('agent-foundry-agent-id')?.value?.trim() || '', + ...(notesVal ? { notes: notesVal } : {}) + }; + formData.other_settings = JSON.stringify(otherSettingsObj); + + // Foundry agents cannot have actions + formData.actions_to_load = []; + formData.enable_agent_gpt_apim = false; + return formData; + } // Handle model and deployment configuration if (formData.custom_connection) { @@ -1472,6 +1728,100 @@ export class AgentModalStepper { window.showToast(`Agent ${this.isEditMode ? 'updated' : 'created'} successfully!`, 'success'); } } + + validateTemplateRequirements() { + const displayName = document.getElementById('agent-display-name'); + const description = document.getElementById('agent-description'); + const instructions = document.getElementById('agent-instructions'); + + if (!displayName || !displayName.value.trim()) { + this.showError('Please add a display name before submitting a template.'); + displayName?.focus(); + return false; + } + + if (!description || !description.value.trim()) { + this.showError('Please add a description before submitting a template.'); + description?.focus(); + return false; + } + + if (!instructions || !instructions.value.trim()) { + this.showError('Instructions are required before submitting a template.'); + instructions?.focus(); + return false; + } + + this.hideError(); + return true; + } + + buildTemplatePayload() { + const displayName = document.getElementById('agent-display-name')?.value?.trim() || ''; + const description = document.getElementById('agent-description')?.value?.trim() || ''; + const instructions = document.getElementById('agent-instructions')?.value || ''; + const additionalSettings = document.getElementById('agent-additional-settings')?.value || ''; + + return { + title: displayName || 'Agent Template', + display_name: displayName || 'Agent Template', + description, + helper_text: description, + instructions, + additional_settings: additionalSettings, + actions_to_load: this.getSelectedActionIds(), + source_agent_id: this.originalAgent?.id, + source_scope: this.isAdmin ? 'global' : 'personal' + }; + } + + async submitTemplate() { + if (!this.canSubmitTemplate()) { + showToast('Template submissions are disabled right now.', 'warning'); + return; + } + + if (!this.validateTemplateRequirements()) { + return; + } + + const button = this.templateSubmitButton; + if (!button) { + return; + } + + const originalHtml = button.innerHTML; + button.disabled = true; + button.innerHTML = 'Submitting...'; + + try { + const payload = { template: this.buildTemplatePayload() }; + const response = await fetch('/api/agent-templates', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + + const data = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(data.error || 'Failed to submit agent template.'); + } + + const status = data.template?.status; + const successMessage = (this.isAdmin && status === 'approved') + ? 'Template published to the gallery!' + : 'Template submitted for review.'; + showToast(successMessage, 'success'); + this.hideError(); + } catch (error) { + console.error('Template submission failed:', error); + this.showError(error.message || 'Failed to submit template.'); + showToast(error.message || 'Failed to submit template.', 'error'); + } finally { + button.disabled = false; + button.innerHTML = originalHtml; + } + } } // Global instance will be created contextually by the calling code diff --git a/application/single_app/static/js/agent_templates_gallery.js b/application/single_app/static/js/agent_templates_gallery.js new file mode 100644 index 00000000..428ebf70 --- /dev/null +++ b/application/single_app/static/js/agent_templates_gallery.js @@ -0,0 +1,278 @@ +// agent_templates_gallery.js +// Dynamically renders the agent template gallery within the agent builder + +import { showToast } from "./chat/chat-toast.js"; + +const gallerySelector = ".agent-template-gallery"; +let cachedTemplates = null; +let loadingPromise = null; + +function getGalleryElements(container) { + return { + spinner: container.querySelector(".agent-template-gallery-loading"), + emptyState: container.querySelector(".agent-template-gallery-empty"), + disabledState: container.querySelector(".agent-template-gallery-disabled"), + errorState: container.querySelector(".agent-template-gallery-error"), + errorText: container.querySelector(".agent-template-gallery-error-text"), + accordion: container.querySelector(".accordion"), + }; +} + +async function fetchTemplates() { + if (cachedTemplates) { + return cachedTemplates; + } + if (loadingPromise) { + return loadingPromise; + } + loadingPromise = fetch("/api/agent-templates") + .then(async (response) => { + if (!response.ok) { + throw new Error("Failed to load templates."); + } + const data = await response.json(); + cachedTemplates = data.templates || []; + return cachedTemplates; + }) + .catch((error) => { + cachedTemplates = []; + throw error; + }) + .finally(() => { + loadingPromise = null; + }); + return loadingPromise; +} + +function renderAccordion(accordion, templates, options = {}) { + const accordionId = options.accordionId || "agentTemplates"; + const showCopy = options.showCopy !== "false"; + const showCreate = options.showCreate !== "false"; + + accordion.innerHTML = ""; + + templates.forEach((template, index) => { + const collapseId = `${accordionId}-collapse-${index}`; + const headingId = `${accordionId}-heading-${index}`; + const instructionsId = `${accordionId}-instructions-${index}`; + + const accordionItem = document.createElement("div"); + accordionItem.className = "accordion-item"; + + const header = document.createElement("h2"); + header.className = "accordion-header"; + header.id = headingId; + + const headerButton = document.createElement("button"); + headerButton.className = `accordion-button${index === 0 ? "" : " collapsed"}`; + headerButton.type = "button"; + headerButton.setAttribute("data-bs-toggle", "collapse"); + headerButton.setAttribute("data-bs-target", `#${collapseId}`); + headerButton.textContent = template.title || template.display_name || "Agent Template"; + header.appendChild(headerButton); + + const collapse = document.createElement("div"); + collapse.id = collapseId; + collapse.className = `accordion-collapse collapse${index === 0 ? " show" : ""}`; + collapse.setAttribute("aria-labelledby", headingId); + collapse.setAttribute("data-bs-parent", `#${accordionId}`); + + const body = document.createElement("div"); + body.className = "accordion-body"; + + const headerRow = document.createElement("div"); + headerRow.className = "d-flex flex-wrap justify-content-between align-items-start gap-2 mb-3"; + + const helper = document.createElement("div"); + helper.className = "small text-muted"; + helper.textContent = template.helper_text || template.description || "Reusable agent template"; + headerRow.appendChild(helper); + + const buttonGroup = document.createElement("div"); + buttonGroup.className = "d-flex gap-2 flex-wrap"; + + if (showCopy) { + const copyBtn = document.createElement("button"); + copyBtn.type = "button"; + copyBtn.className = "btn btn-sm btn-outline-secondary"; + copyBtn.innerHTML = ' Copy'; + copyBtn.addEventListener("click", () => copyInstructions(instructionsId)); + buttonGroup.appendChild(copyBtn); + } + + if (showCreate) { + const createBtn = document.createElement("button"); + createBtn.type = "button"; + createBtn.className = "btn btn-sm btn-success agent-example-create-btn"; + createBtn.innerHTML = ' Use Template'; + const payload = { + display_name: template.display_name || template.title || "Agent Template", + description: template.description || template.helper_text || "", + instructions: template.instructions || "", + additional_settings: template.additional_settings || "", + actions_to_load: template.actions_to_load || [], + }; + createBtn.dataset.agentExample = JSON.stringify(payload); + buttonGroup.appendChild(createBtn); + } + + headerRow.appendChild(buttonGroup); + body.appendChild(headerRow); + + const metaList = document.createElement("div"); + metaList.className = "mb-3"; + + const helperLine = document.createElement("p"); + helperLine.className = "mb-1 text-muted small"; + helperLine.innerHTML = `Suggested display name: ${escapeHtml(template.display_name || template.title || "Agent Template")}`; + metaList.appendChild(helperLine); + + if (Array.isArray(template.tags) && template.tags.length) { + const tagList = document.createElement("div"); + tagList.className = "mb-1"; + template.tags.slice(0, 5).forEach((tag) => { + const badge = document.createElement("span"); + badge.className = "badge bg-secondary-subtle text-secondary-emphasis me-1 mb-1"; + badge.textContent = tag; + tagList.appendChild(badge); + }); + metaList.appendChild(tagList); + } + + if (Array.isArray(template.actions_to_load) && template.actions_to_load.length) { + const actionLine = document.createElement("p"); + actionLine.className = "mb-0 text-muted small"; + actionLine.innerHTML = `Recommended actions: ${template.actions_to_load.join(", ")}`; + metaList.appendChild(actionLine); + } + + body.appendChild(metaList); + + const description = document.createElement("p"); + description.className = "mb-3"; + description.textContent = template.description || template.helper_text || "No description provided."; + body.appendChild(description); + + const instructions = document.createElement("pre"); + instructions.className = "bg-dark text-white p-3 rounded"; + instructions.id = instructionsId; + instructions.textContent = template.instructions || ""; + body.appendChild(instructions); + + if (template.additional_settings) { + const advancedBlock = document.createElement("pre"); + advancedBlock.className = "bg-light border rounded p-3 mt-3"; + advancedBlock.textContent = template.additional_settings; + const advancedLabel = document.createElement("p"); + advancedLabel.className = "text-muted small mb-1"; + advancedLabel.textContent = "Additional settings"; + body.appendChild(advancedLabel); + body.appendChild(advancedBlock); + } + + collapse.appendChild(body); + accordionItem.appendChild(header); + accordionItem.appendChild(collapse); + accordion.appendChild(accordionItem); + }); +} + +function escapeHtml(value) { + const div = document.createElement("div"); + div.textContent = value || ""; + return div.innerHTML; +} + +function copyInstructions(instructionsId) { + const target = document.getElementById(instructionsId); + if (!target) { + return; + } + if (typeof window.copyAgentInstructionSample === "function") { + window.copyAgentInstructionSample(instructionsId); + return; + } + const text = target.textContent || ""; + if (navigator.clipboard?.writeText) { + navigator.clipboard.writeText(text).then(() => { + showToast("Instructions copied to clipboard", "success"); + }).catch(() => { + fallbackCopyText(text); + }); + } else { + fallbackCopyText(text); + } +} + +function fallbackCopyText(text) { + const textarea = document.createElement("textarea"); + textarea.value = text; + textarea.style.position = "fixed"; + textarea.style.top = "-1000px"; + document.body.appendChild(textarea); + textarea.focus(); + textarea.select(); + try { + document.execCommand("copy"); + showToast("Instructions copied to clipboard", "success"); + } catch (err) { + console.error("Clipboard copy failed", err); + showToast("Unable to copy instructions", "error"); + } finally { + document.body.removeChild(textarea); + } +} + +async function initializeGallery(container) { + const elements = getGalleryElements(container); + + if (!window.appSettings?.enable_agent_template_gallery) { + if (elements.spinner) elements.spinner.classList.add("d-none"); + if (elements.disabledState) elements.disabledState.classList.remove("d-none"); + return; + } + + try { + const templates = await fetchTemplates(); + if (elements.spinner) elements.spinner.classList.add("d-none"); + + if (!templates.length) { + if (elements.emptyState) elements.emptyState.classList.remove("d-none"); + return; + } + + if (elements.accordion) { + elements.accordion.classList.remove("d-none"); + renderAccordion(elements.accordion, templates, { + accordionId: container.dataset.accordionId, + showCopy: container.dataset.showCopy, + showCreate: container.dataset.showCreate, + }); + } + } catch (error) { + console.error("Failed to render agent templates", error); + if (elements.spinner) elements.spinner.classList.add("d-none"); + if (elements.errorState) { + elements.errorState.classList.remove("d-none"); + if (elements.errorText) { + elements.errorText.textContent = error.message || "Unexpected error"; + } + } + } +} + +function initAgentTemplateGalleries() { + const containers = document.querySelectorAll(gallerySelector); + if (!containers.length) { + return; + } + containers.forEach((container) => { + initializeGallery(container); + }); +} + +if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", initAgentTemplateGalleries); +} else { + initAgentTemplateGalleries(); +} diff --git a/application/single_app/static/js/chat/chat-citations.js b/application/single_app/static/js/chat/chat-citations.js index a69619c9..abad0af0 100644 --- a/application/single_app/static/js/chat/chat-citations.js +++ b/application/single_app/static/js/chat/chat-citations.js @@ -306,6 +306,13 @@ export function showAgentCitationModal(toolName, toolArgs, toolResult) {
Tool Name:
+
+
Source:
+
+ +
+
+
Function Arguments:

@@ -325,17 +332,20 @@ export function showAgentCitationModal(toolName, toolArgs, toolResult) {
   const toolNameEl = document.getElementById("agent-tool-name");
   const toolArgsEl = document.getElementById("agent-tool-args");
   const toolResultEl = document.getElementById("agent-tool-result");
+  const toolSourceEl = document.getElementById("agent-tool-source");
+  const toolUrlEl = document.getElementById("agent-tool-url");
+  const toolUrlMetaEl = document.getElementById("agent-tool-url-meta");
 
   if (toolNameEl) {
     toolNameEl.textContent = toolName || "Unknown";
   }
   
+  let parsedArgs = null;
   if (toolArgsEl) {
     // Handle empty or no parameters more gracefully
     let argsContent = "";
     
     try {
-      let parsedArgs;
       if (!toolArgs || toolArgs === "" || toolArgs === "{}") {
         argsContent = "No parameters required";
       } else {
@@ -379,9 +389,9 @@ export function showAgentCitationModal(toolName, toolArgs, toolResult) {
   if (toolResultEl) {
     // Handle result formatting and truncation with expand/collapse
     let resultContent = "";
+    let parsedResult = null;
     
     try {
-      let parsedResult;
       if (!toolResult || toolResult === "" || toolResult === "{}") {
         resultContent = "No result";
       } else if (toolResult === "[object Object]") {
@@ -399,6 +409,9 @@ export function showAgentCitationModal(toolName, toolArgs, toolResult) {
     } catch (e) {
       resultContent = toolResult || "No result";
     }
+
+    const citationDetails = extractAgentCitationDetails(parsedResult || parsedArgs);
+    updateAgentCitationSource(toolSourceEl, toolUrlEl, toolUrlMetaEl, citationDetails);
     
     // Add truncation with expand/collapse if content is long
     if (resultContent.length > 300) {
@@ -424,6 +437,63 @@ export function showAgentCitationModal(toolName, toolArgs, toolResult) {
   modal.show();
 }
 
+function extractAgentCitationDetails(source) {
+  if (!source || typeof source !== "object") {
+    return null;
+  }
+
+  const url = source.url;
+  if (!isValidHttpUrl(url)) {
+    return null;
+  }
+
+  return {
+    url,
+    title: source.title || null,
+    quote: source.quote || null,
+    citationType: source.citation_type || null,
+  };
+}
+
+function updateAgentCitationSource(containerEl, linkEl, metaEl, details) {
+  if (!containerEl || !linkEl || !metaEl) {
+    return;
+  }
+
+  if (!details || !details.url) {
+    containerEl.classList.add("d-none");
+    linkEl.textContent = "";
+    linkEl.removeAttribute("href");
+    metaEl.textContent = "";
+    return;
+  }
+
+  containerEl.classList.remove("d-none");
+  linkEl.href = details.url;
+  linkEl.textContent = details.title || details.url;
+
+  const metaParts = [];
+  if (details.citationType) {
+    metaParts.push(`Type: ${details.citationType}`);
+  }
+  if (details.quote) {
+    metaParts.push(`Quote: ${details.quote}`);
+  }
+  metaEl.textContent = metaParts.join(" ā€¢ ");
+}
+
+function isValidHttpUrl(value) {
+  if (!value || typeof value !== "string") {
+    return false;
+  }
+  try {
+    const parsed = new URL(value);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch (error) {
+    return false;
+  }
+}
+
 // --- MODIFIED: Added citationId parameter and fallback in catch ---
 export function showPdfModal(docId, pageNumber, citationId) {
   const fetchUrl = `/view_pdf?doc_id=${encodeURIComponent(docId)}&page=${encodeURIComponent(pageNumber)}`;
diff --git a/application/single_app/static/js/chat/chat-conversations.js b/application/single_app/static/js/chat/chat-conversations.js
index 7d1990cf..9eb3e61f 100644
--- a/application/single_app/static/js/chat/chat-conversations.js
+++ b/application/single_app/static/js/chat/chat-conversations.js
@@ -242,7 +242,7 @@ export function loadConversations() {
   isLoadingConversations = true;
   conversationsList.innerHTML = '
Loading conversations...
'; // Loading state
 
-  fetch("/api/get_conversations")
+  return fetch("/api/get_conversations")
     .then(response => response.ok ? response.json() : response.json().then(err => Promise.reject(err)))
     .then(data => {
       conversationsList.innerHTML = ""; // Clear loading state
@@ -310,6 +310,48 @@ export function loadConversations() {
     });
 }
 
+// Ensure a conversation exists in the list; fetch metadata if missing
+export async function ensureConversationPresent(conversationId) {
+  if (!conversationId) throw new Error('No conversationId provided');
+
+  // Already in list
+  const existing = document.querySelector(`.conversation-item[data-conversation-id="${conversationId}"]`);
+  if (existing) return existing;
+
+  // Fetch metadata to validate ownership and get details
+  const res = await fetch(`/api/conversations/${conversationId}/metadata`);
+  if (!res.ok) {
+    const err = await res.json().catch(() => ({}));
+    throw new Error(err.error || `Failed to load conversation ${conversationId}`);
+  }
+  const metadata = await res.json();
+
+  // Build a conversation object compatible with createConversationItem
+  const convo = {
+    id: conversationId,
+    title: metadata.title || 'Conversation',
+    last_updated: metadata.last_updated || new Date().toISOString(),
+    classification: metadata.classification || [],
+    context: metadata.context || [],
+    chat_type: metadata.chat_type || null,
+    is_pinned: metadata.is_pinned || false,
+    is_hidden: metadata.is_hidden || false,
+  };
+
+  // Keep allConversations in sync
+  allConversations = [convo, ...allConversations.filter(c => c.id !== conversationId)];
+
+  const convoItem = createConversationItem(convo);
+  conversationsList.prepend(convoItem);
+
+  // Refresh sidebar so it appears there too
+  if (window.chatSidebarConversations && window.chatSidebarConversations.loadSidebarConversations) {
+    window.chatSidebarConversations.loadSidebarConversations();
+  }
+
+  return convoItem;
+}
+
 export function createConversationItem(convo) {
   const convoItem = document.createElement("div"); // Changed from  to 
 for better semantics with checkboxes
   convoItem.classList.add("list-group-item", "list-group-item-action", "conversation-item", "d-flex", "align-items-center"); // Use action class
@@ -922,6 +964,8 @@ export async function selectConversation(conversationId) {
     setSidebarActiveConversation(conversationId);
   }
 
+  updateConversationUrl(conversationId);
+
   // Clear any "edit mode" state if switching conversations
   if (currentlyEditingId && currentlyEditingId !== conversationId) {
       const editingItem = document.querySelector(`.conversation-item[data-conversation-id="${currentlyEditingId}"]`);
@@ -1030,6 +1074,7 @@ export async function createNewConversation(callback) {
     if (titleEl) {
       titleEl.textContent = data.title || "New Conversation";
     }
+    updateConversationUrl(data.conversation_id);
     console.log('[createNewConversation] Created conversation without reload:', data.conversation_id);
 
     // Execute callback if provided (e.g., to send the first message)
@@ -1567,4 +1612,16 @@ function addChatTypeBadges(convoItem, classificationsEl) {
     // If chatType is unknown/null or model-only, don't add any workspace badges
     console.log(`addChatTypeBadges: No badges added for chatType="${chatType}" (likely model-only conversation)`);
   }
+}
+
+function updateConversationUrl(conversationId) {
+  if (!conversationId) return;
+
+  try {
+    const url = new URL(window.location.href);
+    url.searchParams.set('conversationId', conversationId);
+    window.history.replaceState({}, '', url.toString());
+  } catch (error) {
+    console.warn('Failed to update conversation URL:', error);
+  }
 }
\ No newline at end of file
diff --git a/application/single_app/static/js/chat/chat-input-actions.js b/application/single_app/static/js/chat/chat-input-actions.js
index 0325812f..77851319 100644
--- a/application/single_app/static/js/chat/chat-input-actions.js
+++ b/application/single_app/static/js/chat/chat-input-actions.js
@@ -347,8 +347,38 @@ if (imageGenBtn) {
 }
 
 if (webSearchBtn) {
+  const webSearchNoticeContainer = document.getElementById("web-search-notice-container");
+  const webSearchNoticeDismiss = document.getElementById("web-search-notice-dismiss");
+  const webSearchNoticeSessionKey = "webSearchNoticeDismissed";
+  
+  // Check if notice was dismissed this session
+  const isNoticeDismissed = () => sessionStorage.getItem(webSearchNoticeSessionKey) === "true";
+  
+  // Show/hide notice based on web search state
+  const updateWebSearchNotice = (isActive) => {
+    if (webSearchNoticeContainer && window.appSettings?.enable_web_search_user_notice) {
+      if (isActive && !isNoticeDismissed()) {
+        webSearchNoticeContainer.style.display = "block";
+      } else {
+        webSearchNoticeContainer.style.display = "none";
+      }
+    }
+  };
+  
+  // Dismiss button handler
+  if (webSearchNoticeDismiss) {
+    webSearchNoticeDismiss.addEventListener("click", function() {
+      sessionStorage.setItem(webSearchNoticeSessionKey, "true");
+      if (webSearchNoticeContainer) {
+        webSearchNoticeContainer.style.display = "none";
+      }
+    });
+  }
+  
   webSearchBtn.addEventListener("click", function () {
     this.classList.toggle("active");
+    const isActive = this.classList.contains("active");
+    updateWebSearchNotice(isActive);
   });
 }
 
@@ -374,13 +404,29 @@ if (fileInputEl) {
       // Hide the upload button since we're auto-uploading
       uploadBtn.style.display = "none";
       
-      // Automatically upload the file
-      if (!currentConversationId) {
-        createNewConversation(() => {
+      // Check for user agreement before uploading
+      const doUpload = () => {
+        if (!currentConversationId) {
+          createNewConversation(() => {
+            uploadFileToConversation(file);
+          });
+        } else {
           uploadFileToConversation(file);
-        });
+        }
+      };
+      
+      // Check if UserAgreementManager exists and check for agreement
+      if (window.UserAgreementManager) {
+        window.UserAgreementManager.checkBeforeUpload(
+          fileInputEl.files,
+          'chat',
+          'default',
+          function(files) {
+            doUpload();
+          }
+        );
       } else {
-        uploadFileToConversation(file);
+        doUpload();
       }
     } else {
       resetFileButton();
@@ -407,12 +453,29 @@ if (uploadBtn) {
       return;
     }
 
-    if (!currentConversationId) {
-      createNewConversation(() => {
+    // Check for user agreement before uploading
+    const doUpload = () => {
+      if (!currentConversationId) {
+        createNewConversation(() => {
+          uploadFileToConversation(file);
+        });
+      } else {
         uploadFileToConversation(file);
-      });
+      }
+    };
+    
+    // Check if UserAgreementManager exists and check for agreement
+    if (window.UserAgreementManager) {
+      window.UserAgreementManager.checkBeforeUpload(
+        fileInput.files,
+        'chat',
+        'default',
+        function(files) {
+          doUpload();
+        }
+      );
     } else {
-      uploadFileToConversation(file);
+      doUpload();
     }
   });
 }
diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js
index 48fc6166..45dbf6f3 100644
--- a/application/single_app/static/js/chat/chat-messages.js
+++ b/application/single_app/static/js/chat/chat-messages.js
@@ -1460,6 +1460,8 @@ export function actuallySendMessage(finalMessageToSend) {
 
   // Fallback: if group_id is null/empty, use window.activeGroupId
   const finalGroupId = group_id || window.activeGroupId || null;
+  const webSearchToggle = document.getElementById("search-web-btn");
+  const webSearchEnabled = webSearchToggle ? webSearchToggle.classList.contains("active") : false;
   
   // Prepare message data object
   // Get active public workspace ID from user settings (similar to active_group_id)
@@ -1469,6 +1471,7 @@ export function actuallySendMessage(finalMessageToSend) {
     message: finalMessageToSend,
     conversation_id: currentConversationId,
     hybrid_search: hybridSearchEnabled,
+    web_search_enabled: webSearchEnabled,
     selected_document_id: selectedDocumentId,
     classifications: classificationsToSend,
     image_generation: imageGenEnabled,
diff --git a/application/single_app/static/js/chat/chat-onload.js b/application/single_app/static/js/chat/chat-onload.js
index 2a83b20b..e20f7240 100644
--- a/application/single_app/static/js/chat/chat-onload.js
+++ b/application/single_app/static/js/chat/chat-onload.js
@@ -1,6 +1,6 @@
 // chat-onload.js
 
-import { loadConversations } from "./chat-conversations.js";
+import { loadConversations, selectConversation, ensureConversationPresent } from "./chat-conversations.js";
 // Import handleDocumentSelectChange
 import { loadAllDocs, populateDocumentSelectScope, handleDocumentSelectChange } from "./chat-documents.js";
 import { getUrlParameter } from "./chat-utils.js"; // Assuming getUrlParameter is in chat-utils.js now
@@ -12,10 +12,11 @@ import { initializeStreamingToggle } from "./chat-streaming.js";
 import { initializeReasoningToggle } from "./chat-reasoning.js";
 import { initializeSpeechInput } from "./chat-speech-input.js";
 
-window.addEventListener('DOMContentLoaded', () => {
+window.addEventListener('DOMContentLoaded', async () => {
   console.log("DOM Content Loaded. Starting initializations."); // Log start
 
-  loadConversations(); // Load conversations immediately
+    // Load conversations immediately (awaitable so deep-link can run after)
+    await loadConversations();
 
   // Initialize the conversation info button
   initConversationInfoButton();
@@ -78,13 +79,13 @@ window.addEventListener('DOMContentLoaded', () => {
   }
 
   // Load documents, prompts, and user settings
-  Promise.all([
-      loadAllDocs(),
-      loadUserPrompts(),
-      loadGroupPrompts(),
-      loadUserSettings()
-  ])
-  .then(([docsResult, userPromptsResult, groupPromptsResult, userSettings]) => {
+  try {
+      const [docsResult, userPromptsResult, groupPromptsResult, userSettings] = await Promise.all([
+          loadAllDocs(),
+          loadUserPrompts(),
+          loadGroupPrompts(),
+          loadUserSettings()
+      ]);
       console.log("Initial data (Docs, Prompts, Settings) loaded successfully."); // Log success
       
       // Set the preferred model if available
@@ -199,13 +200,24 @@ window.addEventListener('DOMContentLoaded', () => {
       initializePromptInteractions();
 
 
+      // Deep-link: conversationId query param
+      const conversationId = getUrlParameter("conversationId") || getUrlParameter("conversation_id");
+      if (conversationId) {
+          try {
+              await ensureConversationPresent(conversationId);
+              await selectConversation(conversationId);
+          } catch (err) {
+              console.error('Failed to load conversation from URL param:', err);
+              showToast('Could not open that conversation.', 'danger');
+          }
+      }
+
       console.log("All initializations complete."); // Log end
 
-  })
-  .catch((err) => {
+  } catch (err) {
       console.error("Error during initial data loading or setup:", err);
       // Maybe try to initialize prompts even if doc loading fails? Depends on requirements.
       // console.log("Attempting to initialize prompts despite data load error...");
       // initializePromptInteractions();
-  });
+  }
 });
diff --git a/application/single_app/static/js/control-center.js b/application/single_app/static/js/control-center.js
index e804865b..bf155fe7 100644
--- a/application/single_app/static/js/control-center.js
+++ b/application/single_app/static/js/control-center.js
@@ -1,9 +1,28 @@
-
+// control-center.js
 // Control Center JavaScript functionality
 // Handles user management, pagination, modals, and API interactions
 
 import { showToast } from "./chat/chat-toast.js";
 
+function parseDateKey(dateStr) {
+    if (!dateStr) {
+        return null;
+    }
+
+    const parts = dateStr.split("-");
+    if (parts.length === 3) {
+        const year = Number(parts[0]);
+        const month = Number(parts[1]);
+        const day = Number(parts[2]);
+        if (Number.isFinite(year) && Number.isFinite(month) && Number.isFinite(day)) {
+            return new Date(year, month - 1, day);
+        }
+    }
+
+    const parsed = new Date(dateStr);
+    return Number.isNaN(parsed.getTime()) ? null : parsed;
+}
+
 // Group Table Sorter - similar to user table but for groups
 class GroupTableSorter {
     constructor(tableId) {
@@ -1423,8 +1442,10 @@ class ControlCenter {
         const allDates = [...new Set([...Object.keys(createdData), ...Object.keys(deletedData)])].sort();
         
         const labels = allDates.map(date => {
-            const dateObj = new Date(date);
-            return dateObj.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
+            const dateObj = parseDateKey(date);
+            return dateObj
+                ? dateObj.toLocaleDateString('en-US', { month: 'short', day: 'numeric' })
+                : date;
         });
         
         const createdValues = allDates.map(date => createdData[date] || 0);
@@ -1553,8 +1574,10 @@ class ControlCenter {
         console.log(`šŸ” [Frontend Debug] Documents date range:`, allDates);
         
         const labels = allDates.map(date => {
-            const dateObj = new Date(date);
-            return dateObj.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
+            const dateObj = parseDateKey(date);
+            return dateObj
+                ? dateObj.toLocaleDateString('en-US', { month: 'short', day: 'numeric' })
+                : date;
         });
         
         // Prepare datasets - lines for creations, bars for deletions
@@ -1661,7 +1684,10 @@ class ControlCenter {
                                 title: function(context) {
                                     const dataIndex = context[0].dataIndex;
                                     const dateStr = allDates[dataIndex];
-                                    const date = new Date(dateStr);
+                                    const date = parseDateKey(dateStr);
+                                    if (!date) {
+                                        return dateStr;
+                                    }
                                     return date.toLocaleDateString('en-US', { 
                                         weekday: 'long', 
                                         year: 'numeric', 
@@ -1710,7 +1736,7 @@ class ControlCenter {
             console.log('šŸ” [Frontend Debug] Rendering tokens chart with data:', activityData.tokens);
         }
         
-        // Render combined chart with embedding and chat tokens
+        // Render combined chart with embedding, chat, and web search tokens
         this.renderCombinedTokensChart('tokensChart', activityData.tokens || {});
     }
 
@@ -1745,7 +1771,7 @@ class ControlCenter {
             this.tokensChart.destroy();
         }
         
-        // Prepare data from tokens object (format: { "YYYY-MM-DD": { "embedding": count, "chat": count } })
+        // Prepare data from tokens object (format: { "YYYY-MM-DD": { "embedding": count, "chat": count, "web_search": count } })
         const allDates = Object.keys(tokensData).sort();
         if (appSettings?.enable_debug_logging) {
             console.log('šŸ” [Frontend Debug] Token dates:', allDates);
@@ -1753,17 +1779,21 @@ class ControlCenter {
         
         // Format labels for display
         const labels = allDates.map(dateStr => {
-            const date = new Date(dateStr);
-            return date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
+            const date = parseDateKey(dateStr);
+            return date
+                ? date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' })
+                : dateStr;
         });
         
-        // Extract embedding and chat token counts
+        // Extract embedding, chat, and web search token counts
         const embeddingTokens = allDates.map(date => tokensData[date]?.embedding || 0);
         const chatTokens = allDates.map(date => tokensData[date]?.chat || 0);
+        const webSearchTokens = allDates.map(date => tokensData[date]?.web_search || 0);
         
         if (appSettings?.enable_debug_logging) {
             console.log('šŸ” [Frontend Debug] Embedding tokens:', embeddingTokens);
             console.log('šŸ” [Frontend Debug] Chat tokens:', chatTokens);
+            console.log('šŸ” [Frontend Debug] Web search tokens:', webSearchTokens);
         }
         
         // Create datasets
@@ -1791,6 +1821,18 @@ class ControlCenter {
                 pointRadius: 3,
                 pointHoverRadius: 5,
                 pointBackgroundColor: '#0dcaf0'
+            },
+            {
+                label: 'Web Search Tokens',
+                data: webSearchTokens,
+                backgroundColor: 'rgba(32, 201, 151, 0.2)',
+                borderColor: '#20c997',
+                borderWidth: 2,
+                fill: false,
+                tension: 0.4,
+                pointRadius: 3,
+                pointHoverRadius: 5,
+                pointBackgroundColor: '#20c997'
             }
         ];
         
@@ -1823,7 +1865,10 @@ class ControlCenter {
                                 title: function(context) {
                                     const dataIndex = context[0].dataIndex;
                                     const dateStr = allDates[dataIndex];
-                                    const date = new Date(dateStr);
+                                    const date = parseDateKey(dateStr);
+                                    if (!date) {
+                                        return dateStr;
+                                    }
                                     return date.toLocaleDateString('en-US', { 
                                         weekday: 'long', 
                                         year: 'numeric', 
@@ -1919,8 +1964,10 @@ class ControlCenter {
         console.log(`šŸ” [Frontend Debug] ${chartType} date range:`, dates);
         
         const labels = dates.map(date => {
-            const dateObj = new Date(date);
-            return dateObj.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
+            const dateObj = parseDateKey(date);
+            return dateObj
+                ? dateObj.toLocaleDateString('en-US', { month: 'short', day: 'numeric' })
+                : date;
         });
         
         const data = dates.map(date => chartData[date] || 0);
@@ -1962,7 +2009,10 @@ class ControlCenter {
                                 title: function(context) {
                                     const dataIndex = context[0].dataIndex;
                                     const dateStr = dates[dataIndex];
-                                    const date = new Date(dateStr);
+                                    const date = parseDateKey(dateStr);
+                                    if (!date) {
+                                        return dateStr;
+                                    }
                                     return date.toLocaleDateString('en-US', { 
                                         weekday: 'long', 
                                         year: 'numeric', 
diff --git a/application/single_app/static/js/group/manage_group.js b/application/single_app/static/js/group/manage_group.js
index 87a12b58..2de03e43 100644
--- a/application/single_app/static/js/group/manage_group.js
+++ b/application/single_app/static/js/group/manage_group.js
@@ -92,6 +92,39 @@ $(document).ready(function () {
     rejectRequest(requestId);
   });
 
+  // Add event delegation for select user button in search results
+  $(document).on("click", ".select-user-btn", function () {
+    const id = $(this).data("user-id");
+    const name = $(this).data("user-name");
+    const email = $(this).data("user-email");
+    selectUserForAdd(id, name, email);
+  });
+
+  // Add event delegation for remove member button
+  $(document).on("click", ".remove-member-btn", function () {
+    const userId = $(this).data("user-id");
+    removeMember(userId);
+  });
+
+  // Add event delegation for change role button
+  $(document).on("click", ".change-role-btn", function () {
+    const userId = $(this).data("user-id");
+    const currentRole = $(this).data("user-role");
+    openChangeRoleModal(userId, currentRole);
+    $("#changeRoleModal").modal("show");
+  });
+
+  // Add event delegation for approve/reject request buttons
+  $(document).on("click", ".approve-request-btn", function () {
+    const requestId = $(this).data("request-id");
+    approveRequest(requestId);
+  });
+
+  $(document).on("click", ".reject-request-btn", function () {
+    const requestId = $(this).data("request-id");
+    rejectRequest(requestId);
+  });
+
   // CSV Bulk Upload Events
   $("#addBulkMemberBtn").on("click", function () {
     $("#csvBulkUploadModal").modal("show");
@@ -440,6 +473,8 @@ function renderMemberActions(member) {
     } else {
       return `
         
       `;
@@ -508,6 +546,10 @@ function loadPendingRequests() {
                     data-request-id="${u.userId}">Approve
             
+            
+            
           
         
       `;
@@ -555,40 +597,69 @@ function rejectRequest(requestId) {
   });
 }
 
+// Search users for manual add
 // Search users for manual add
 function searchUsers() {
   const term = $("#userSearchTerm").val().trim();
   if (!term) {
-    alert("Enter a name or email to search.");
+    // Show inline validation error
+    $("#searchStatus").text("āš ļø Please enter a name or email to search");
+    $("#searchStatus").removeClass("text-muted text-success").addClass("text-warning");
+    $("#userSearchTerm").addClass("is-invalid");
     return;
   }
+  
+  // Clear any previous validation states
+  $("#userSearchTerm").removeClass("is-invalid");
+  $("#searchStatus").removeClass("text-warning text-danger text-success").addClass("text-muted");
   $("#searchStatus").text("Searching...");
   $("#searchUsersBtn").prop("disabled", true);
 
   $.get("/api/userSearch", { query: term })
-    .done(renderUserSearchResults)
+    .done(function(users) {
+      renderUserSearchResults(users);
+      // Show success status
+      if (users && users.length > 0) {
+        $("#searchStatus").text(`āœ“ Found ${users.length} user(s)`);
+        $("#searchStatus").removeClass("text-muted text-warning text-danger").addClass("text-success");
+      } else {
+        $("#searchStatus").text("No users found");
+        $("#searchStatus").removeClass("text-muted text-warning text-success").addClass("text-muted");
+      }
+    })
     .fail(function (jq) {
       const err = jq.responseJSON?.error || jq.statusText;
-      alert("User search failed: " + err);
+      // Show inline error
+      $("#searchStatus").text(`āŒ Search failed: ${err}`);
+      $("#searchStatus").removeClass("text-muted text-warning text-success").addClass("text-danger");
+      // Also show toast for critical errors
+      showToast("User search failed: " + err, "danger");
     })
     .always(function () {
-      $("#searchStatus").text("");
       $("#searchUsersBtn").prop("disabled", false);
     });
 }
 
+// Render user-search results in add-member modal
 // Render user-search results in add-member modal
 function renderUserSearchResults(users) {
   let html = "";
+  if (!users || !users.length) {
+    html = `No results.`;
   if (!users || !users.length) {
     html = `No results.`;
   } else {
+    users.forEach(u => {
     users.forEach(u => {
       html += `
         
           ${u.displayName || "(no name)"}
           ${u.email || ""}
           
+