fix(mcp): detect OAuth in test-connection and surface structured signal

Test-connection used to attempt an unauthenticated McpClient.connect and
surface the resulting transport error to the modal. For spec-compliant
OAuth servers (Semrush, Linear, Notion, Atlassian) this produced
misleading "Streamable HTTP error: Error POSTing to endpoint:" messages
and aborted the create flow before the OAuth dance could start. The form
path had a string-regex escape hatch; the JSON path didn't.

Replaces the regex with a structured signal — the same `detectMcpAuthType`
probe the create flow uses. When the probe returns 'oauth', the route
short-circuits with `{ success: false, authRequired: true, authType:
'oauth' }`. Both modal submit paths check `authRequired` explicitly to
continue into the normal create + OAuth start flow.

Matches the pattern Claude Code / Cursor's MCP clients use: see the
OAuth challenge, skip the unauth attempt, run the OAuth handshake.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: waleedlatif1

apps/sim/app/api/mcp/servers/test-connection/route.ts

@@ -12,6 +12,7 @@ import {
   validateMcpServerSsrf,
 } from '@/lib/mcp/domain-check'
 import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
+import { detectMcpAuthType } from '@/lib/mcp/oauth'
 import { resolveMcpConfigEnvVars } from '@/lib/mcp/resolve-config'
 import type { McpTransport } from '@/lib/mcp/types'
 import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
@@ -31,6 +32,13 @@ function isUrlBasedTransport(transport: McpTransport): boolean {
 interface TestConnectionResult {
   success: boolean
   error?: string
+  /**
+   * Set when the URL is reachable but responds with an OAuth challenge
+   * (RFC 9728). The modal uses this to continue into the create + OAuth
+   * start flow instead of treating the probe failure as fatal.
+   */
+  authRequired?: boolean
+  authType?: 'none' | 'headers' | 'oauth'
   serverInfo?: {
     name: string
     version: string
@@ -163,6 +171,24 @@ export const POST = withRouteHandler(
       }
 
       const result: TestConnectionResult = { success: false }
+
+      /**
+       * Detect OAuth-protected servers (RFC 9728) before attempting an
+       * unauthenticated connection — for those, an unauthenticated McpClient
+       * connect will fail with a generic HTTP error, which is misleading.
+       * The modal uses `authRequired` to continue into the OAuth flow.
+       */
+      const detectedAuthType = await detectMcpAuthType(testConfig.url)
+      if (detectedAuthType === 'oauth') {
+        logger.info(`[${requestId}] OAuth challenge detected, skipping unauth connect:`, {
+          name: body.name,
+        })
+        result.authRequired = true
+        result.authType = 'oauth'
+        return createMcpSuccessResponse(result, 200)
+      }
+      result.authType = detectedAuthType
+
       let client: McpClient | null = null
 
       try {

apps/sim/app/workspace/[workspaceId]/settings/components/mcp/components/mcp-server-form-modal/mcp-server-form-modal.tsx

@@ -67,22 +67,6 @@ export interface McpServerFormModalProps {
 
 const ENV_VAR_PATTERN = /\{\{[^}]+\}\}/
 
-/**
- * Treats a failed test-connection as a soft failure when the response looks like
- * an OAuth/auth challenge — the create flow can then run the probe and kick off
- * the OAuth handshake. Without this, OAuth-protected servers like Semrush get
- * rejected at the modal before the probe ever runs.
- */
-function isAuthRequiredError(errorMessage: string | undefined): boolean {
-  const text = (errorMessage || '').toLowerCase()
-  return (
-    /\b401\b/.test(text) ||
-    text.includes('unauthorized') ||
-    text.includes('oauth') ||
-    text.includes('authentication')
-  )
-}
-
 function hasEnvVarInHostname(url: string): boolean {
   const globalPattern = new RegExp(ENV_VAR_PATTERN.source, 'g')
   if (url.trim().replace(globalPattern, '').trim() === '') return true
@@ -125,11 +109,12 @@ interface EnvVarDropdownConfig {
 }
 
 function getTestButtonLabel(
-  testResult: { success: boolean; error?: string } | null,
+  testResult: { success: boolean; error?: string; authRequired?: boolean } | null,
   isTestingConnection: boolean
 ): string {
   if (isTestingConnection) return 'Testing...'
   if (testResult?.success) return 'Connection success'
+  if (testResult?.authRequired) return 'Requires OAuth'
   if (testResult && !testResult.success) return 'No connection: retry'
   return 'Test Connection'
 }
@@ -533,7 +518,7 @@ export function McpServerFormModal({
         workspaceId,
       })
 
-      if (!connectionResult.success && !isAuthRequiredError(connectionResult.error)) {
+      if (!connectionResult.success && !connectionResult.authRequired) {
         setSubmitError(
           connectionResult.error || 'Connection test failed. Please check the URL and try again.'
         )
@@ -595,7 +580,7 @@ export function McpServerFormModal({
         workspaceId,
       })
 
-      if (!connectionResult.success && !isAuthRequiredError(connectionResult.error)) {
+      if (!connectionResult.success && !connectionResult.authRequired) {
         setSubmitError(
           connectionResult.error || 'Connection test failed. Please check the URL and try again.'
         )

apps/sim/lib/api/contracts/mcp.ts

@@ -252,6 +252,13 @@ export const mcpServerTestResultSchema = z.object({
   success: z.boolean(),
   message: z.string().optional(),
   error: z.string().optional(),
+  /**
+   * True when the probe detected an OAuth-protected resource (RFC 9728).
+   * The modal uses this to skip the failure path and continue into the
+   * normal create + OAuth start flow.
+   */
+  authRequired: z.boolean().optional(),
+  authType: mcpAuthTypeSchema.optional(),
   serverInfo: z
     .object({
       name: z.string(),