Replace Linkup web search with Serper (#717)

Author: jahooma

.env.example

@@ -28,7 +28,7 @@ STRIPE_SUBSCRIPTION_200_PRICE_ID=price_dummy_subscription_200_id
 STRIPE_SUBSCRIPTION_500_PRICE_ID=price_dummy_subscription_500_id
 
 # External Services
-LINKUP_API_KEY=dummy_linkup_key
+SERPER_API_KEY=dummy_serper_key
 LOOPS_API_KEY=dummy_loops_key
 ZEROCLICK_API_KEY=dummy_zeroclick_key
 

agents/researcher/researcher-web.ts

@@ -1,6 +1,5 @@
 import { publisher } from '../constants'
 
-import type { ToolCall } from '../types/agent-definition'
 import type { SecretAgentDefinition } from '../types/secret-agent-definition'
 
 const definition: SecretAgentDefinition = {
@@ -17,36 +16,18 @@ const definition: SecretAgentDefinition = {
   },
   outputMode: 'last_message',
   includeMessageHistory: false,
-  toolNames: ['web_search'],
+  toolNames: ['web_search', 'run_terminal_command'],
   spawnableAgents: [],
 
-  systemPrompt: `You are an expert researcher who can search the web to find relevant information. Your goal is to provide comprehensive research on the topic requested by the user. Use web_search to find current information.`,
+  systemPrompt: `You are an expert researcher who can search the web to find relevant information. Your goal is to answer the user's question from current search results and any useful source pages. Use web_search to get Serper JSON search results. Use run_terminal_command with tools like curl to fetch web pages that would help answer the user's question.`,
   instructionsPrompt: `Provide comprehensive research on the user's prompt.
 
-Use web_search to find current information. Repeat the web_search tool call until you have gathered all the relevant information.
+Use web_search to find current information. The tool returns JSON search results, so inspect the titles, links, snippets, answer boxes, and related results before deciding what to fetch next.
 
-Then, write up a concise report that includes key findings for the user's prompt.
-`.trim(),
-
-  handleSteps: function* ({ agentState, prompt, params }) {
-    const { toolResult } = yield {
-      toolName: 'web_search' as const,
-      input: { query: prompt || '', depth: 'standard' as const },
-      includeToolCall: false,
-    } satisfies ToolCall<'web_search'>
-
-    const results = (toolResult
-      ?.filter((r) => r.type === 'json')
-      ?.map((r) => r.value)?.[0] ?? {}) as {
-        result: string | undefined
-        errorMessage: string | undefined
-      }
+Use run_terminal_command to fetch any web page that would help answer the user's question. Prefer targeted, relevant pages from the search results. Avoid fetching pages that are unlikely to add useful evidence.
 
-    yield {
-      type: 'STEP_TEXT',
-      text: results.result ?? results.errorMessage ?? '',
-    }
-  },
+Then, write up a concise answer that includes key findings for the user's prompt and cites source URLs when useful.
+`.trim(),
 }
 
 export default definition

agents/types/tools.ts

@@ -398,7 +398,7 @@ export interface ThinkDeeplyParams {
 }
 
 /**
- * Search the web for current information using Linkup API.
+ * Search the web for current information using Serper API.
  */
 export interface WebSearchParams {
   /** The search query to find relevant web content */

cli/src/__tests__/test-utils.ts

@@ -70,7 +70,7 @@ const TEST_SERVER_ENV_DEFAULTS: Record<string, string> = {
   OPEN_ROUTER_API_KEY: 'test',
   OPENAI_API_KEY: 'test',
   ANTHROPIC_API_KEY: 'test',
-  LINKUP_API_KEY: 'test',
+  SERPER_API_KEY: 'test',
   GRAVITY_API_KEY: 'test',
   PORT: '4242',
   DATABASE_URL: 'postgres://user:pass@localhost:5432/db',

common/src/templates/initial-agents-dir/types/tools.ts

@@ -398,7 +398,7 @@ export interface ThinkDeeplyParams {
 }
 
 /**
- * Search the web for current information using Linkup API.
+ * Search the web for current information using Serper API.
  */
 export interface WebSearchParams {
   /** The search query to find relevant web content */

common/src/tools/params/tool/web-search.ts

@@ -20,9 +20,9 @@ const inputSchema = z
         `Search depth - 'standard' for quick results, 'deep' for more comprehensive search. Default is 'standard'.`,
       ),
   })
-  .describe(`Search the web for current information using Linkup API.`)
+  .describe(`Search the web for current information using Serper API.`)
 const description = `
-Purpose: Search the web for current, up-to-date information on any topic. This tool uses Linkup's web search API to find relevant content from across the internet.
+Purpose: Search the web for current, up-to-date information on any topic. This tool uses Serper's Google Search API to find relevant content from across the internet.
 
 Use cases:
 - Finding current information about technologies, libraries, or frameworks
@@ -31,7 +31,7 @@ Use cases:
 - Finding examples and tutorials
 - Checking current status of services or APIs
 
-The tool will return search results with titles, URLs, and content snippets.
+The tool will return JSON search results with titles, URLs, content snippets, and other available SERP fields such as answer boxes or related questions.
 
 Example:
 ${$getNativeToolCallExampleString({

packages/agent-runtime/src/__tests__/web-search-tool.test.ts

@@ -5,7 +5,6 @@ import { getInitialSessionState } from '@codebuff/common/types/session-state'
 import { promptSuccess, success } from '@codebuff/common/util/error'
 import {
   afterEach,
-
   beforeEach,
   describe,
   expect,
@@ -243,7 +242,7 @@ describe('web_search tool with researcher agent (via web API facade)', () => {
 
   test('should handle API errors gracefully', async () => {
     spyOn(webApi, 'callWebSearchAPI').mockResolvedValue({
-      error: 'Linkup API timeout',
+      error: 'Serper API timeout',
     })
 
     mockAgentStream([
@@ -275,7 +274,7 @@ describe('web_search tool with researcher agent (via web API facade)', () => {
     expect(toolMsgs.length).toBeGreaterThan(0)
     const last = JSON.stringify(toolMsgs[toolMsgs.length - 1].content)
     expect(last).toContain('errorMessage')
-    expect(last).toContain('Linkup API timeout')
+    expect(last).toContain('Serper API timeout')
   })
 
   test('should handle non-Error exceptions from facade', async () => {

…src/llm-api/__tests__/linkup-api.test.ts …src/llm-api/__tests__/serper-api.test.tspackages/agent-runtime/src/llm-api/__tests__/linkup-api.test.ts renamed to packages/agent-runtime/src/llm-api/__tests__/serper-api.test.ts packages/agent-runtime/src/llm-api/__tests__/linkup-api.test.ts renamed to packages/agent-runtime/src/llm-api/__tests__/serper-api.test.ts

@@ -14,18 +14,16 @@ import {
   test,
 } from 'bun:test'
 
-import { searchWeb } from '../linkup-api'
+import { searchWeb } from '../serper-api'
 
 import type { AgentRuntimeDeps } from '@codebuff/common/types/contracts/agent-runtime'
 
-// Test server env for Linkup API
-const testServerEnv = { LINKUP_API_KEY: 'test-api-key' }
+const testServerEnv = { SERPER_API_KEY: 'test-api-key' }
 
-describe('Linkup API', () => {
+describe('Serper API', () => {
   let agentRuntimeImpl: AgentRuntimeDeps & { serverEnv: typeof testServerEnv }
 
   beforeAll(async () => {
-    // Mock withTimeout utility
     await mockModule('@codebuff/common/util/promise', () => ({
       withTimeout: async (promise: Promise<any>, timeout: number) => promise,
     }))
@@ -48,14 +46,14 @@ describe('Linkup API', () => {
 
   test('should successfully search with basic query', async () => {
     const mockResponse = {
-      answer:
-        'React is a JavaScript library for building user interfaces. You can learn how to build your first React application by following the official documentation.',
-      sources: [
+      searchParameters: { q: 'React tutorial', type: 'search', num: 10 },
+      organic: [
         {
-          name: 'React Documentation',
-          url: 'https://react.dev',
+          title: 'React Documentation',
+          link: 'https://react.dev',
           snippet:
             'React is a JavaScript library for building user interfaces.',
+          position: 1,
         },
       ],
     }
@@ -74,37 +72,32 @@ describe('Linkup API', () => {
       query: 'React tutorial',
     })
 
-    expect(result).toBe(
-      'React is a JavaScript library for building user interfaces. You can learn how to build your first React application by following the official documentation.',
-    )
-
-    // Verify fetch was called with correct parameters
+    expect(JSON.parse(result!)).toEqual(mockResponse)
     expect(agentRuntimeImpl.fetch).toHaveBeenCalledWith(
-      'https://api.linkup.so/v1/search',
+      'https://google.serper.dev/search',
       expect.objectContaining({
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
-          Authorization: 'Bearer test-api-key',
+          'X-API-KEY': 'test-api-key',
         },
         body: JSON.stringify({
           q: 'React tutorial',
-          depth: 'standard',
-          outputType: 'sourcedAnswer',
+          num: 10,
         }),
       }),
     )
   })
 
   test('should handle custom depth', async () => {
     const mockResponse = {
-      answer:
-        'Advanced React patterns include render props, higher-order components, and custom hooks for building reusable and maintainable components.',
-      sources: [
+      searchParameters: { q: 'React patterns', type: 'search', num: 20 },
+      organic: [
         {
-          name: 'Advanced React Patterns',
-          url: 'https://example.com/advanced-react',
+          title: 'Advanced React Patterns',
+          link: 'https://example.com/advanced-react',
           snippet: 'Deep dive into React patterns and best practices.',
+          position: 1,
         },
       ],
     }
@@ -124,18 +117,13 @@ describe('Linkup API', () => {
       depth: 'deep',
     })
 
-    expect(result).toBe(
-      'Advanced React patterns include render props, higher-order components, and custom hooks for building reusable and maintainable components.',
-    )
-
-    // Verify fetch was called with correct parameters
+    expect(JSON.parse(result!)).toEqual(mockResponse)
     expect(agentRuntimeImpl.fetch).toHaveBeenCalledWith(
-      'https://api.linkup.so/v1/search',
+      'https://google.serper.dev/search',
       expect.objectContaining({
         body: JSON.stringify({
           q: 'React patterns',
-          depth: 'deep',
-          outputType: 'sourcedAnswer',
+          num: 20,
         }),
       }),
     )
@@ -169,7 +157,7 @@ describe('Linkup API', () => {
   test('should handle invalid response format', async () => {
     agentRuntimeImpl.fetch = mock(() => {
       return Promise.resolve(
-        new Response(JSON.stringify({ invalid: 'format' }), {
+        new Response(JSON.stringify(['invalid']), {
           status: 200,
           headers: { 'Content-Type': 'application/json' },
         }),
@@ -181,10 +169,21 @@ describe('Linkup API', () => {
     expect(result).toBeNull()
   })
 
-  test('should handle missing answer field', async () => {
+  test('should return JSON search results without an answer field', async () => {
+    const mockResponse = {
+      organic: [
+        {
+          title: 'Test result',
+          link: 'https://example.com',
+          snippet: 'Test snippet',
+          position: 1,
+        },
+      ],
+    }
+
     agentRuntimeImpl.fetch = mock(() => {
       return Promise.resolve(
-        new Response(JSON.stringify({ sources: [] }), {
+        new Response(JSON.stringify(mockResponse), {
           status: 200,
           headers: { 'Content-Type': 'application/json' },
         }),
@@ -196,12 +195,13 @@ describe('Linkup API', () => {
       query: 'test query',
     })
 
-    expect(result).toBeNull()
+    expect(JSON.parse(result!)).toEqual(mockResponse)
   })
-  test('should handle empty answer', async () => {
+
+  test('should return sparse JSON search results', async () => {
     const mockResponse = {
-      answer: '',
-      sources: [],
+      searchParameters: { q: 'test query', type: 'search' },
+      organic: [],
     }
 
     agentRuntimeImpl.fetch = mock(() => {
@@ -215,14 +215,13 @@ describe('Linkup API', () => {
 
     const result = await searchWeb({ ...agentRuntimeImpl, query: 'test query' })
 
-    expect(result).toBeNull()
+    expect(JSON.parse(result!)).toEqual(mockResponse)
   })
 
   test('should use default options when none provided', async () => {
     const mockResponse = {
-      answer: 'Test answer content',
-      sources: [
-        { name: 'Test', url: 'https://example.com', snippet: 'Test content' },
+      organic: [
+        { title: 'Test', link: 'https://example.com', snippet: 'Test content' },
       ],
     }
 
@@ -237,14 +236,12 @@ describe('Linkup API', () => {
 
     await searchWeb({ ...agentRuntimeImpl, query: 'test query' })
 
-    // Verify fetch was called with default parameters
     expect(agentRuntimeImpl.fetch).toHaveBeenCalledWith(
-      'https://api.linkup.so/v1/search',
+      'https://google.serper.dev/search',
       expect.objectContaining({
         body: JSON.stringify({
           q: 'test query',
-          depth: 'standard',
-          outputType: 'sourcedAnswer',
+          num: 10,
         }),
       }),
     )
@@ -264,7 +261,6 @@ describe('Linkup API', () => {
     const result = await searchWeb({ ...agentRuntimeImpl, query: 'test query' })
 
     expect(result).toBeNull()
-    // Verify that error logging was called
     expect(agentRuntimeImpl.logger.error).toHaveBeenCalled()
   })
 
@@ -287,13 +283,12 @@ describe('Linkup API', () => {
     })
 
     expect(result).toBeNull()
-    // Verify that detailed error logging was called with 404 info
     expect(agentRuntimeImpl.logger.error).toHaveBeenCalledWith(
       expect.objectContaining({
         status: 404,
         statusText: 'Not Found',
         responseBody: mockErrorResponse,
-        requestUrl: 'https://api.linkup.so/v1/search',
+        requestUrl: 'https://google.serper.dev/search',
         query: 'test query for 404',
       }),
       expect.stringContaining('404'),