From 7f8195b0eba63e0a0c867c8821e161d81a92446f Mon Sep 17 00:00:00 2001 From: Justin Butterfield Date: Thu, 2 Jul 2026 11:44:48 -0400 Subject: [PATCH 1/7] feat: add file-based Devin provider to fix ENAMETOOLONG on Windows Add devinProvider.js as a promptfoo custom provider that writes prompts to a temp file and passes them via --prompt-file instead of as a CLI argument. This avoids the Windows ENAMETOOLONG error when prompts exceed the 32k character command-line limit. The provider also uses async spawn and parses token usage from Devin's --export output. The provider entry in promptfooconfig.yaml is added but commented out, ready to be enabled when needed. --- devinProvider.js | 119 +++++++++++++++++++++++++++++++++++++++++++ promptfooconfig.yaml | 6 +++ 2 files changed, 125 insertions(+) create mode 100644 devinProvider.js diff --git a/devinProvider.js b/devinProvider.js new file mode 100644 index 0000000..30b1239 --- /dev/null +++ b/devinProvider.js @@ -0,0 +1,119 @@ +const { spawn } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +class DevinProvider { + constructor(options) { + this.providerId = options.id || 'devin-custom'; + this.config = options.config || {}; + } + + id() { + return this.providerId; + } + + async callApi(prompt, context, options) { + const model = this.config.model || 'SWE-1.6'; + + let isGraderMode = false; + try { + const parsed = JSON.parse(prompt); + if (Array.isArray(parsed) && parsed.length > 0 && parsed[0].role) { + isGraderMode = true; + } + } catch (e) { + // Not JSON, provider mode + } + + let fullPrompt; + + if (isGraderMode) { + let systemMsg, userMsg; + try { + const messages = JSON.parse(prompt); + const systemMessage = messages.find(m => m.role === 'system'); + const userMessage = messages.find(m => m.role === 'user'); + + if (systemMessage && userMessage) { + systemMsg = systemMessage.content; + userMsg = userMessage.content; + } else { + throw new Error('Missing system or user message'); + } + } catch (e) { + systemMsg = 'You are an evaluator. Respond with only valid JSON: {"pass": bool, "score": 0.0-1.0, "reason": "string"}'; + userMsg = prompt; + } + fullPrompt = `${systemMsg}\n\n${userMsg}`; + } else { + fullPrompt = prompt; + } + + const tmpFile = path.join(os.tmpdir(), `devin-prompt-${Date.now()}-${process.pid}.txt`); + const exportFile = path.join(os.tmpdir(), `devin-export-${Date.now()}-${process.pid}.json`); + + try { + fs.writeFileSync(tmpFile, fullPrompt, 'utf8'); + + const args = ['-p', '--prompt-file', tmpFile, '--model', model, '--export', exportFile]; + if (!isGraderMode) { + args.push('--permission-mode', 'auto'); + } + + const result = await new Promise((resolve, reject) => { + const child = spawn('devin', args, { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { stdout += data; }); + child.stderr.on('data', (data) => { stderr += data; }); + + child.on('error', (err) => { + resolve({ error: err.message, tokenUsage: parseTokenUsage(exportFile) }); + }); + + child.on('close', (code) => { + if (code !== 0) { + resolve({ error: stdout || stderr || `devin exited with code ${code}`, tokenUsage: parseTokenUsage(exportFile) }); + } else { + resolve({ output: stdout, tokenUsage: parseTokenUsage(exportFile) }); + } + }); + }); + + return result; + } finally { + try { + fs.unlinkSync(tmpFile); + fs.unlinkSync(exportFile); + } catch (e) { + // Ignore cleanup errors + } + } + } +} + +function parseTokenUsage(exportFilePath) { + try { + const data = JSON.parse(fs.readFileSync(exportFilePath, 'utf8')); + const agentSteps = (data.steps || []).filter(s => s.source === 'agent'); + let promptTokens = 0, completionTokens = 0; + for (const step of agentSteps) { + const m = step.metadata?.metrics; + if (m) { + promptTokens += (m.input_tokens || 0) + (m.cache_read_tokens || 0) + (m.cache_creation_tokens || 0); + completionTokens += (m.output_tokens || 0); + } + } + return { total: promptTokens + completionTokens, prompt: promptTokens, completion: completionTokens }; + } catch { + return { total: 0, prompt: 0, completion: 0 }; + } +} + +module.exports = DevinProvider; diff --git a/promptfooconfig.yaml b/promptfooconfig.yaml index f686701..c5deb9e 100644 --- a/promptfooconfig.yaml +++ b/promptfooconfig.yaml @@ -49,6 +49,12 @@ providers: config: model: 'SWE-1.6' + # Devin CLI provider — file-based (fixes ENAMETOOLONG on Windows for prompts >32k) + #- id: 'file://devinProvider.js' + # label: 'Devin SWE-1.6 (file provider)' + # config: + # model: 'SWE-1.6' + # Devin CLI provider — For Testing a Second Model # MUST Use Label to differentiate providers with same ID (first in wins otherwise) # - id: 'exec: node ./devin.js' From 137552dedbe0831debaa8d4e7f66bb1defdde883 Mon Sep 17 00:00:00 2001 From: Daniel Scheufler Date: Thu, 16 Jul 2026 10:29:21 -0500 Subject: [PATCH 2/7] feat(add config for forced prompt file, toggle token tracking, and configured prompt length limits) --- devinProvider.js | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/devinProvider.js b/devinProvider.js index 30b1239..c2d1164 100644 --- a/devinProvider.js +++ b/devinProvider.js @@ -15,6 +15,9 @@ class DevinProvider { async callApi(prompt, context, options) { const model = this.config.model || 'SWE-1.6'; + const forcePromptFile = this.config.forcePromptFile || false; + const trackTokens = this.config.trackTokens !== false; // default true + const promptFileSizeThreshold = this.config.promptFileSizeThreshold || 32000; let isGraderMode = false; try { @@ -50,13 +53,31 @@ class DevinProvider { fullPrompt = prompt; } - const tmpFile = path.join(os.tmpdir(), `devin-prompt-${Date.now()}-${process.pid}.txt`); - const exportFile = path.join(os.tmpdir(), `devin-export-${Date.now()}-${process.pid}.json`); + const shouldUseFile = forcePromptFile || fullPrompt.length > promptFileSizeThreshold; + let tmpFile, exportFile; + + if (shouldUseFile) { + tmpFile = path.join(os.tmpdir(), `devin-prompt-${Date.now()}-${process.pid}.txt`); + fs.writeFileSync(tmpFile, fullPrompt, 'utf8'); + } + + if (trackTokens) { + exportFile = path.join(os.tmpdir(), `devin-export-${Date.now()}-${process.pid}.json`); + } try { - fs.writeFileSync(tmpFile, fullPrompt, 'utf8'); + const args = ['-p', '--model', model]; + + if (shouldUseFile) { + args.push('--prompt-file', tmpFile); + } else { + args.push('--', fullPrompt); + } + + if (trackTokens) { + args.push('--export', exportFile); + } - const args = ['-p', '--prompt-file', tmpFile, '--model', model, '--export', exportFile]; if (!isGraderMode) { args.push('--permission-mode', 'auto'); } @@ -74,14 +95,14 @@ class DevinProvider { child.stderr.on('data', (data) => { stderr += data; }); child.on('error', (err) => { - resolve({ error: err.message, tokenUsage: parseTokenUsage(exportFile) }); + resolve({ error: err.message, tokenUsage: trackTokens ? parseTokenUsage(exportFile) : undefined }); }); child.on('close', (code) => { if (code !== 0) { - resolve({ error: stdout || stderr || `devin exited with code ${code}`, tokenUsage: parseTokenUsage(exportFile) }); + resolve({ error: stdout || stderr || `devin exited with code ${code}`, tokenUsage: trackTokens ? parseTokenUsage(exportFile) : undefined }); } else { - resolve({ output: stdout, tokenUsage: parseTokenUsage(exportFile) }); + resolve({ output: stdout, tokenUsage: trackTokens ? parseTokenUsage(exportFile) : undefined }); } }); }); @@ -89,8 +110,8 @@ class DevinProvider { return result; } finally { try { - fs.unlinkSync(tmpFile); - fs.unlinkSync(exportFile); + if (tmpFile) fs.unlinkSync(tmpFile); + if (exportFile) fs.unlinkSync(exportFile); } catch (e) { // Ignore cleanup errors } From e541438bfde36a2b15c1090d13a5aa1290bd25e6 Mon Sep 17 00:00:00 2001 From: Daniel Scheufler Date: Thu, 16 Jul 2026 10:54:48 -0500 Subject: [PATCH 3/7] fix(moved prompt to end of devin args)/feat(added grader json guard, improves stability)/clean(simplified system /user prompt requirement check) --- devinProvider.js | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/devinProvider.js b/devinProvider.js index c2d1164..af66819 100644 --- a/devinProvider.js +++ b/devinProvider.js @@ -38,12 +38,12 @@ class DevinProvider { const systemMessage = messages.find(m => m.role === 'system'); const userMessage = messages.find(m => m.role === 'user'); - if (systemMessage && userMessage) { - systemMsg = systemMessage.content; - userMsg = userMessage.content; - } else { - throw new Error('Missing system or user message'); + if (!systemMessage || !userMessage) { + throw new Error('Missing system or user message in grader mode'); } + + systemMsg = systemMessage.content; + userMsg = userMessage.content; } catch (e) { systemMsg = 'You are an evaluator. Respond with only valid JSON: {"pass": bool, "score": 0.0-1.0, "reason": "string"}'; userMsg = prompt; @@ -68,11 +68,6 @@ class DevinProvider { try { const args = ['-p', '--model', model]; - if (shouldUseFile) { - args.push('--prompt-file', tmpFile); - } else { - args.push('--', fullPrompt); - } if (trackTokens) { args.push('--export', exportFile); @@ -82,6 +77,12 @@ class DevinProvider { args.push('--permission-mode', 'auto'); } + if (shouldUseFile) { + args.push('--prompt-file', tmpFile); + } else { + args.push('--', fullPrompt); + } + const result = await new Promise((resolve, reject) => { const child = spawn('devin', args, { encoding: 'utf8', @@ -102,7 +103,16 @@ class DevinProvider { if (code !== 0) { resolve({ error: stdout || stderr || `devin exited with code ${code}`, tokenUsage: trackTokens ? parseTokenUsage(exportFile) : undefined }); } else { - resolve({ output: stdout, tokenUsage: trackTokens ? parseTokenUsage(exportFile) : undefined }); + + let output = stdout; + if (isGraderMode) { + const jsonMatch = output.match(/```json\s*([\s\S]*?)\s*```/); + if (jsonMatch) { + output = jsonMatch[1].trim(); + } + } + + resolve({ output, tokenUsage: trackTokens ? parseTokenUsage(exportFile) : undefined }); } }); }); From b58c06fa9987cc10bb643b3095d3cbef9e61f4ff Mon Sep 17 00:00:00 2001 From: Daniel Scheufler Date: Thu, 16 Jul 2026 10:56:45 -0500 Subject: [PATCH 4/7] feat(update config to demonstrate improved Devin ProvideR) --- promptfooconfig.yaml | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/promptfooconfig.yaml b/promptfooconfig.yaml index c5deb9e..6f1120c 100644 --- a/promptfooconfig.yaml +++ b/promptfooconfig.yaml @@ -18,7 +18,7 @@ defaultTest: # SPECIFY the Provider by the Label from the providers section. # # Devin CLI grader — ACTIVE - provider: 'SWE-1.6' + provider: 'Devin SWE-1.6' # # GitHub Copilot CLI grader # provider: 'GitHub Haiku' @@ -43,24 +43,27 @@ providers: # Grader mode: prompt is JSON array [{role, content}, ...] # Provider mode: prompt is plain text # - # Devin CLI provider - - id: 'exec: node ./devin.js' + # Devin CLI provider — optimized with conditional file handling and token tracking + # - Uses direct CLI args for small prompts (<32k) for speed + # - Auto-switches to temp files for large prompts to avoid ENAMETOOLONG on Windows + # - Tracks token usage by default (prompt, completion, total) + - id: 'file://devinProvider.js' label: 'Devin SWE-1.6' config: model: 'SWE-1.6' - - # Devin CLI provider — file-based (fixes ENAMETOOLONG on Windows for prompts >32k) - #- id: 'file://devinProvider.js' - # label: 'Devin SWE-1.6 (file provider)' - # config: - # model: 'SWE-1.6' + # forcePromptFile: false # Set to true to always use temp files (default: false) + # trackTokens: true # Enable token usage tracking via --export (default: true) + promptFileSizeThreshold: 15000 # Prompt size threshold in bytes to auto-use temp file (default: 32000 characters) # Devin CLI provider — For Testing a Second Model # MUST Use Label to differentiate providers with same ID (first in wins otherwise) - # - id: 'exec: node ./devin.js' - # label: 'Haiku' # MUST provide label when using same provider with different config! + # - id: 'file://devinProvider.js' + # label: 'Devin Haiku' # config: # model: 'claude-haiku-4.5' + # forcePromptFile: false # Set to true to always use temp files (default: false) + # trackTokens: true # Enable token usage tracking via --export (default: true) + # promptFileSizeThreshold: 32000 # defaults to 32000 characters # GitHub Copilot CLI provider # - id: 'exec: node ./gh_copilot.js' From 7b6eca61fc86e4df40ec0039a103c5908f12e598 Mon Sep 17 00:00:00 2001 From: Daniel Scheufler Date: Thu, 16 Jul 2026 11:00:33 -0500 Subject: [PATCH 5/7] feat(update trust AI config with better devin provider) --- trust-ai.config.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/trust-ai.config.yaml b/trust-ai.config.yaml index d20edd8..f1d97a4 100644 --- a/trust-ai.config.yaml +++ b/trust-ai.config.yaml @@ -44,17 +44,23 @@ providers: # Provider mode: prompt is plain text # # Devin CLI provider - - id: 'exec: node ./devin.js' + - id: 'file://devinProvider.js' label: 'Devin SWE-1.6' config: model: 'SWE-1.6' + # forcePromptFile: false # Set to true to always use temp files (default: false) + # trackTokens: true # Enable token usage tracking via --export (default: true) + promptFileSizeThreshold: 15000 # Prompt size threshold in bytes to auto-use temp file (default: 32000 characters) # Devin CLI provider — For Testing a Second Model # MUST Use Label to differentiate providers with same ID (first in wins otherwise) - # - id: 'exec: node ./devin.js' - # label: 'Devin Kimi' # MUST provide label when using same provider with different config! + # - id: 'file://devinProvider.js' + # label: 'Devin Kimi' # config: # model: 'kimi-k2.7' + # # forcePromptFile: false # Set to true to always use temp files (default: false) + # # trackTokens: true # Enable token usage tracking via --export (default: true) + # promptFileSizeThreshold: 15000 # Prompt size threshold in bytes to auto-use temp file (default: 32000 characters) # GitHub Copilot CLI provider # - id: 'exec: node ./gh_copilot.js' From 601149bfaac001c3d5fe430fd0a13abd9f0f1df5 Mon Sep 17 00:00:00 2001 From: Daniel Scheufler Date: Thu, 16 Jul 2026 11:01:49 -0500 Subject: [PATCH 6/7] doc(update devin docs with for new approach): indicates wrapped temp file usage --- docs/install-devin.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/install-devin.md b/docs/install-devin.md index de4154b..08c2e4a 100644 --- a/docs/install-devin.md +++ b/docs/install-devin.md @@ -88,7 +88,7 @@ devin -p -- "What is 2+2?" ## Usage in This Project -This project uses `devin.js` which calls: +This project uses `devinProvider.js` which calls: ```bash # Provider mode @@ -96,6 +96,9 @@ devin -p --permission-mode auto --model SWE-1.6 -- "your prompt" # Grader mode devin -p --model SWE-1.6 -- "your prompt" + +# Provider Mode, with temp File prompting +devin -p --permission-mode auto --model SWE-1.6 --file /path/to/prompt.txt ``` --- From 75f8b31327707c040c7c3cd6680d7c14c3733e67 Mon Sep 17 00:00:00 2001 From: Daniel Scheufler Date: Thu, 16 Jul 2026 11:02:00 -0500 Subject: [PATCH 7/7] clean(remove former devin wrapper)_ --- devin.js | 123 ------------------------------------------------------- 1 file changed, 123 deletions(-) delete mode 100755 devin.js diff --git a/devin.js b/devin.js deleted file mode 100755 index 0fa801b..0000000 --- a/devin.js +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env node -// Combined Devin CLI provider and grader for promptfoo -// -// Auto-detects mode based on prompt format: -// - Grader mode: Prompt is JSON array [{role, content}, ...] -// - Provider mode: Prompt is plain text -// -// Grader: Parses JSON chat array, concatenates system + user messages (no --system-prompt) -// Provider: Passes prompt directly to Devin CLI - -const { spawnSync } = require('child_process'); - -const prompt = process.argv[2]; -const options = process.argv[3]; -const context = process.argv[4]; - -// Detect mode: if prompt looks like a JSON array, use grader mode -let isGraderMode = false; -try { - const parsed = JSON.parse(prompt); - if (Array.isArray(parsed) && parsed.length > 0 && parsed[0].role) { - isGraderMode = true; - } -} catch (e) { - // Not JSON, so provider mode -} - -if (isGraderMode) { - // ===== GRADER MODE ===== - // Parse OPTIONS to get model from config - let model = 'SWE-1.6'; // Default model - if (options && options !== '{}') { - try { - const optionsObj = JSON.parse(options); - if (optionsObj.config && optionsObj.config.model) { - model = optionsObj.config.model; - } - } catch (e) { - // If JSON parsing fails, use default - model = 'SWE-1.6'; - } - } - - // Parse the JSON chat message array that promptfoo sends to graders - let systemMsg, userMsg; - try { - const messages = JSON.parse(prompt); - const systemMessage = messages.find(m => m.role === 'system'); - const userMessage = messages.find(m => m.role === 'user'); - - if (systemMessage && userMessage) { - systemMsg = systemMessage.content; - userMsg = userMessage.content; - } else { - throw new Error('Missing system or user message'); - } - } catch (e) { - // Fallback: treat the whole thing as a user message - systemMsg = 'You are an evaluator. Respond with only valid JSON: {"pass": bool, "score": 0.0-1.0, "reason": "string"}'; - userMsg = prompt; - } - - // Combine system and user into one prompt (Devin has no --system-prompt) - const fullPrompt = `${systemMsg}\n\n${userMsg}`; - - const result = spawnSync('devin', ['-p', '--model', model, '--', fullPrompt], { - encoding: 'utf8', - stdio: ['pipe', 'pipe', 'pipe'] - }); - - if (result.error) { - console.error(result.error.message); - process.exit(1); - } - - if (result.status !== 0) { - console.error(result.stdout || result.stderr); - process.exit(result.status || 1); - } - - let output = result.stdout; - const jsonMatch = output.match(/```json\s*([\s\S]*?)\s*```/); - if (jsonMatch) { - output = jsonMatch[1].trim(); - } - console.log(output); - - - // console.log(result.stdout); -} else { - // ===== PROVIDER MODE ===== - // Parse OPTIONS to get model from config - let model = 'SWE-1.6'; // Default model - if (options && options !== '{}') { - try { - const optionsObj = JSON.parse(options); - if (optionsObj.config && optionsObj.config.model) { - model = optionsObj.config.model; - } - } catch (e) { - // If JSON parsing fails, use default - model = 'SWE-1.6'; - } - } - - // Call devin cli with single-turn mode and specified model - const result = spawnSync('devin', ['-p', '--permission-mode', 'auto', '--model', model, '--', prompt], { - encoding: 'utf8', - stdio: ['pipe', 'pipe', 'pipe'] - }); - - if (result.error) { - console.error(result.error.message); - process.exit(1); - } - - if (result.status !== 0) { - console.error(result.stdout || result.stderr); - process.exit(result.status || 1); - } - - console.log(result.stdout); -}