diff --git a/devin.js b/devin.js deleted file mode 100755 index 0fa801b..0000000 --- a/devin.js +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env node -// Combined Devin CLI provider and grader for promptfoo -// -// Auto-detects mode based on prompt format: -// - Grader mode: Prompt is JSON array [{role, content}, ...] -// - Provider mode: Prompt is plain text -// -// Grader: Parses JSON chat array, concatenates system + user messages (no --system-prompt) -// Provider: Passes prompt directly to Devin CLI - -const { spawnSync } = require('child_process'); - -const prompt = process.argv[2]; -const options = process.argv[3]; -const context = process.argv[4]; - -// Detect mode: if prompt looks like a JSON array, use grader mode -let isGraderMode = false; -try { - const parsed = JSON.parse(prompt); - if (Array.isArray(parsed) && parsed.length > 0 && parsed[0].role) { - isGraderMode = true; - } -} catch (e) { - // Not JSON, so provider mode -} - -if (isGraderMode) { - // ===== GRADER MODE ===== - // Parse OPTIONS to get model from config - let model = 'SWE-1.6'; // Default model - if (options && options !== '{}') { - try { - const optionsObj = JSON.parse(options); - if (optionsObj.config && optionsObj.config.model) { - model = optionsObj.config.model; - } - } catch (e) { - // If JSON parsing fails, use default - model = 'SWE-1.6'; - } - } - - // Parse the JSON chat message array that promptfoo sends to graders - let systemMsg, userMsg; - try { - const messages = JSON.parse(prompt); - const systemMessage = messages.find(m => m.role === 'system'); - const userMessage = messages.find(m => m.role === 'user'); - - if (systemMessage && userMessage) { - systemMsg = systemMessage.content; - userMsg = userMessage.content; - } else { - throw new Error('Missing system or user message'); - } - } catch (e) { - // Fallback: treat the whole thing as a user message - systemMsg = 'You are an evaluator. Respond with only valid JSON: {"pass": bool, "score": 0.0-1.0, "reason": "string"}'; - userMsg = prompt; - } - - // Combine system and user into one prompt (Devin has no --system-prompt) - const fullPrompt = `${systemMsg}\n\n${userMsg}`; - - const result = spawnSync('devin', ['-p', '--model', model, '--', fullPrompt], { - encoding: 'utf8', - stdio: ['pipe', 'pipe', 'pipe'] - }); - - if (result.error) { - console.error(result.error.message); - process.exit(1); - } - - if (result.status !== 0) { - console.error(result.stdout || result.stderr); - process.exit(result.status || 1); - } - - let output = result.stdout; - const jsonMatch = output.match(/```json\s*([\s\S]*?)\s*```/); - if (jsonMatch) { - output = jsonMatch[1].trim(); - } - console.log(output); - - - // console.log(result.stdout); -} else { - // ===== PROVIDER MODE ===== - // Parse OPTIONS to get model from config - let model = 'SWE-1.6'; // Default model - if (options && options !== '{}') { - try { - const optionsObj = JSON.parse(options); - if (optionsObj.config && optionsObj.config.model) { - model = optionsObj.config.model; - } - } catch (e) { - // If JSON parsing fails, use default - model = 'SWE-1.6'; - } - } - - // Call devin cli with single-turn mode and specified model - const result = spawnSync('devin', ['-p', '--permission-mode', 'auto', '--model', model, '--', prompt], { - encoding: 'utf8', - stdio: ['pipe', 'pipe', 'pipe'] - }); - - if (result.error) { - console.error(result.error.message); - process.exit(1); - } - - if (result.status !== 0) { - console.error(result.stdout || result.stderr); - process.exit(result.status || 1); - } - - console.log(result.stdout); -} diff --git a/devinProvider.js b/devinProvider.js new file mode 100644 index 0000000..af66819 --- /dev/null +++ b/devinProvider.js @@ -0,0 +1,150 @@ +const { spawn } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +class DevinProvider { + constructor(options) { + this.providerId = options.id || 'devin-custom'; + this.config = options.config || {}; + } + + id() { + return this.providerId; + } + + async callApi(prompt, context, options) { + const model = this.config.model || 'SWE-1.6'; + const forcePromptFile = this.config.forcePromptFile || false; + const trackTokens = this.config.trackTokens !== false; // default true + const promptFileSizeThreshold = this.config.promptFileSizeThreshold || 32000; + + let isGraderMode = false; + try { + const parsed = JSON.parse(prompt); + if (Array.isArray(parsed) && parsed.length > 0 && parsed[0].role) { + isGraderMode = true; + } + } catch (e) { + // Not JSON, provider mode + } + + let fullPrompt; + + if (isGraderMode) { + let systemMsg, userMsg; + try { + const messages = JSON.parse(prompt); + const systemMessage = messages.find(m => m.role === 'system'); + const userMessage = messages.find(m => m.role === 'user'); + + if (!systemMessage || !userMessage) { + throw new Error('Missing system or user message in grader mode'); + } + + systemMsg = systemMessage.content; + userMsg = userMessage.content; + } catch (e) { + systemMsg = 'You are an evaluator. Respond with only valid JSON: {"pass": bool, "score": 0.0-1.0, "reason": "string"}'; + userMsg = prompt; + } + fullPrompt = `${systemMsg}\n\n${userMsg}`; + } else { + fullPrompt = prompt; + } + + const shouldUseFile = forcePromptFile || fullPrompt.length > promptFileSizeThreshold; + let tmpFile, exportFile; + + if (shouldUseFile) { + tmpFile = path.join(os.tmpdir(), `devin-prompt-${Date.now()}-${process.pid}.txt`); + fs.writeFileSync(tmpFile, fullPrompt, 'utf8'); + } + + if (trackTokens) { + exportFile = path.join(os.tmpdir(), `devin-export-${Date.now()}-${process.pid}.json`); + } + + try { + const args = ['-p', '--model', model]; + + + if (trackTokens) { + args.push('--export', exportFile); + } + + if (!isGraderMode) { + args.push('--permission-mode', 'auto'); + } + + if (shouldUseFile) { + args.push('--prompt-file', tmpFile); + } else { + args.push('--', fullPrompt); + } + + const result = await new Promise((resolve, reject) => { + const child = spawn('devin', args, { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { stdout += data; }); + child.stderr.on('data', (data) => { stderr += data; }); + + child.on('error', (err) => { + resolve({ error: err.message, tokenUsage: trackTokens ? parseTokenUsage(exportFile) : undefined }); + }); + + child.on('close', (code) => { + if (code !== 0) { + resolve({ error: stdout || stderr || `devin exited with code ${code}`, tokenUsage: trackTokens ? parseTokenUsage(exportFile) : undefined }); + } else { + + let output = stdout; + if (isGraderMode) { + const jsonMatch = output.match(/```json\s*([\s\S]*?)\s*```/); + if (jsonMatch) { + output = jsonMatch[1].trim(); + } + } + + resolve({ output, tokenUsage: trackTokens ? parseTokenUsage(exportFile) : undefined }); + } + }); + }); + + return result; + } finally { + try { + if (tmpFile) fs.unlinkSync(tmpFile); + if (exportFile) fs.unlinkSync(exportFile); + } catch (e) { + // Ignore cleanup errors + } + } + } +} + +function parseTokenUsage(exportFilePath) { + try { + const data = JSON.parse(fs.readFileSync(exportFilePath, 'utf8')); + const agentSteps = (data.steps || []).filter(s => s.source === 'agent'); + let promptTokens = 0, completionTokens = 0; + for (const step of agentSteps) { + const m = step.metadata?.metrics; + if (m) { + promptTokens += (m.input_tokens || 0) + (m.cache_read_tokens || 0) + (m.cache_creation_tokens || 0); + completionTokens += (m.output_tokens || 0); + } + } + return { total: promptTokens + completionTokens, prompt: promptTokens, completion: completionTokens }; + } catch { + return { total: 0, prompt: 0, completion: 0 }; + } +} + +module.exports = DevinProvider; diff --git a/docs/install-devin.md b/docs/install-devin.md index de4154b..08c2e4a 100644 --- a/docs/install-devin.md +++ b/docs/install-devin.md @@ -88,7 +88,7 @@ devin -p -- "What is 2+2?" ## Usage in This Project -This project uses `devin.js` which calls: +This project uses `devinProvider.js` which calls: ```bash # Provider mode @@ -96,6 +96,9 @@ devin -p --permission-mode auto --model SWE-1.6 -- "your prompt" # Grader mode devin -p --model SWE-1.6 -- "your prompt" + +# Provider Mode, with temp File prompting +devin -p --permission-mode auto --model SWE-1.6 --file /path/to/prompt.txt ``` --- diff --git a/promptfooconfig.yaml b/promptfooconfig.yaml index f686701..6f1120c 100644 --- a/promptfooconfig.yaml +++ b/promptfooconfig.yaml @@ -18,7 +18,7 @@ defaultTest: # SPECIFY the Provider by the Label from the providers section. # # Devin CLI grader — ACTIVE - provider: 'SWE-1.6' + provider: 'Devin SWE-1.6' # # GitHub Copilot CLI grader # provider: 'GitHub Haiku' @@ -43,18 +43,27 @@ providers: # Grader mode: prompt is JSON array [{role, content}, ...] # Provider mode: prompt is plain text # - # Devin CLI provider - - id: 'exec: node ./devin.js' + # Devin CLI provider — optimized with conditional file handling and token tracking + # - Uses direct CLI args for small prompts (<32k) for speed + # - Auto-switches to temp files for large prompts to avoid ENAMETOOLONG on Windows + # - Tracks token usage by default (prompt, completion, total) + - id: 'file://devinProvider.js' label: 'Devin SWE-1.6' config: model: 'SWE-1.6' + # forcePromptFile: false # Set to true to always use temp files (default: false) + # trackTokens: true # Enable token usage tracking via --export (default: true) + promptFileSizeThreshold: 15000 # Prompt size threshold in bytes to auto-use temp file (default: 32000 characters) # Devin CLI provider — For Testing a Second Model # MUST Use Label to differentiate providers with same ID (first in wins otherwise) - # - id: 'exec: node ./devin.js' - # label: 'Haiku' # MUST provide label when using same provider with different config! + # - id: 'file://devinProvider.js' + # label: 'Devin Haiku' # config: # model: 'claude-haiku-4.5' + # forcePromptFile: false # Set to true to always use temp files (default: false) + # trackTokens: true # Enable token usage tracking via --export (default: true) + # promptFileSizeThreshold: 32000 # defaults to 32000 characters # GitHub Copilot CLI provider # - id: 'exec: node ./gh_copilot.js' diff --git a/trust-ai.config.yaml b/trust-ai.config.yaml index d20edd8..f1d97a4 100644 --- a/trust-ai.config.yaml +++ b/trust-ai.config.yaml @@ -44,17 +44,23 @@ providers: # Provider mode: prompt is plain text # # Devin CLI provider - - id: 'exec: node ./devin.js' + - id: 'file://devinProvider.js' label: 'Devin SWE-1.6' config: model: 'SWE-1.6' + # forcePromptFile: false # Set to true to always use temp files (default: false) + # trackTokens: true # Enable token usage tracking via --export (default: true) + promptFileSizeThreshold: 15000 # Prompt size threshold in bytes to auto-use temp file (default: 32000 characters) # Devin CLI provider — For Testing a Second Model # MUST Use Label to differentiate providers with same ID (first in wins otherwise) - # - id: 'exec: node ./devin.js' - # label: 'Devin Kimi' # MUST provide label when using same provider with different config! + # - id: 'file://devinProvider.js' + # label: 'Devin Kimi' # config: # model: 'kimi-k2.7' + # # forcePromptFile: false # Set to true to always use temp files (default: false) + # # trackTokens: true # Enable token usage tracking via --export (default: true) + # promptFileSizeThreshold: 15000 # Prompt size threshold in bytes to auto-use temp file (default: 32000 characters) # GitHub Copilot CLI provider # - id: 'exec: node ./gh_copilot.js'