From 62fae4a8cee2ad88589d0db09dc148c9d8781348 Mon Sep 17 00:00:00 2001 From: alichherawalla Date: Thu, 23 Jul 2026 19:13:41 +0530 Subject: [PATCH 1/7] test(A1): prove ggml-org mmproj on-disk name is refused, across live HF vision repos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hits real Hugging Face (no mocks) for every curated vision repo + popular families, runs the real pair→rename→belongs pipeline, and asserts the projector belongs after rename. Red today for the ggml-org `mmproj--` shape (SmolVLM, Qwen2.5-VL, pixtral, InternVL). Also locks the separate no-pairing bug set as a tracked ledger. --- .../mmProjNamingAcrossRepos.network.test.ts | 117 ++++++++++++++++++ .../unit/services/parallelMmproj.test.ts | 18 +++ 2 files changed, 135 insertions(+) create mode 100644 __tests__/integration/models/mmProjNamingAcrossRepos.network.test.ts diff --git a/__tests__/integration/models/mmProjNamingAcrossRepos.network.test.ts b/__tests__/integration/models/mmProjNamingAcrossRepos.network.test.ts new file mode 100644 index 000000000..531fc508d --- /dev/null +++ b/__tests__/integration/models/mmProjNamingAcrossRepos.network.test.ts @@ -0,0 +1,117 @@ +/** + * A1 — iOS "Multimodal support not enabled. Call initMultimodal first." + * + * The chain a vision model must survive to actually initialize multimodal on device: + * (1) PAIR — from the LIVE HF listing, `pickMmProjForDownload` picks the projector that belongs to the + * model (huggingFaceService.getModelFiles → ModelFile.mmProjFile). If it returns none, the + * app never even downloads a projector → text-only → the device crash. + * (2) RENAME — on download the projector is renamed via `mmProjLocalName` to a shared on-disk name. + * (3) BELONGS — the strict on-disk matcher (`mmProjBelongsToModel`) must still accept the renamed file, or + * `linkOrphanMmProj` clears the link on the next scan → text-only → the device crash. + * + * This test hits the REAL Hugging Face listing for every vision repo the app curates plus popular families, + * runs the REAL product pipeline over whatever HF publishes (no hardcoded filenames, no mocks of our code), + * and reports at WHICH stage each repo breaks. Requires network. + */ +import { huggingFaceService } from '../../../src/services/huggingface'; +import { mmProjLocalName } from '../../../src/services/modelManager/download'; +import { mmProjBelongsToModel } from '../../../src/services/mmproj'; +import { RECOMMENDED_MODELS } from '../../../src/constants/models'; + +const CURATED_VISION_REPOS = RECOMMENDED_MODELS.filter(m => m.type === 'vision').map(m => m.id); + +// Popular vision families in the wild (real repos confirmed via HF API during research). +const POPULAR_VISION_REPOS = [ + 'ggml-org/SmolVLM-256M-Instruct-GGUF', // the exact device report (Christophe) + 'ggml-org/SmolVLM-500M-Instruct-GGUF', + 'ggml-org/Qwen2.5-VL-3B-Instruct-GGUF', + 'ggml-org/Qwen2.5-VL-7B-Instruct-GGUF', + 'unsloth/Qwen2.5-VL-7B-Instruct-GGUF', + 'ggml-org/gemma-3-4b-it-GGUF', + 'unsloth/gemma-3-4b-it-GGUF', + 'ggml-org/pixtral-12b-GGUF', + 'ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF', + 'ggml-org/InternVL3-2B-Instruct-GGUF', + 'openbmb/MiniCPM-V-2_6-gguf', + 'moondream/moondream2-gguf', + 'abetlen/Phi-3.5-vision-instruct-gguf', + 'leafspark/Llama-3.2-11B-Vision-Instruct-GGUF', +]; + +const ALL_REPOS = [...new Set([...CURATED_VISION_REPOS, ...POPULAR_VISION_REPOS])]; + +interface RepoResult { + repoId: string; + curated: boolean; + exists: boolean; + modelCount: number; + pairedCount: number; + sampleModel?: string; + sampleProjectorHF?: string; + sampleOnDisk?: string; + belongsAll: boolean; + stage: 'ok' | 'no-repo' | 'no-pairing' | 'rename-mismatch'; +} + +async function examineRepo(repoId: string): Promise { + const curated = CURATED_VISION_REPOS.includes(repoId); + let files; + try { + files = await huggingFaceService.getModelFiles(repoId); + } catch { + return { repoId, curated, exists: false, modelCount: 0, pairedCount: 0, belongsAll: false, stage: 'no-repo' }; + } + if (files.length === 0) { + return { repoId, curated, exists: false, modelCount: 0, pairedCount: 0, belongsAll: false, stage: 'no-repo' }; + } + const paired = files.filter(f => f.mmProjFile); + if (paired.length === 0) { + return { repoId, curated, exists: true, modelCount: files.length, pairedCount: 0, belongsAll: false, stage: 'no-pairing' }; + } + const sample = paired[0]; + const sampleOnDisk = mmProjLocalName(sample.name, sample.mmProjFile!.name); + const belongsAll = paired.every(f => mmProjBelongsToModel(f.name, mmProjLocalName(f.name, f.mmProjFile!.name))); + return { + repoId, curated, exists: true, modelCount: files.length, pairedCount: paired.length, + sampleModel: sample.name, sampleProjectorHF: sample.mmProjFile!.name, sampleOnDisk, + belongsAll, stage: belongsAll ? 'ok' : 'rename-mismatch', + }; +} + +describe('mmproj naming — live HF pairing→rename→belongs pipeline across every vision repo', () => { + const results: RepoResult[] = []; + + it('examines every repo and records the stage it reaches', async () => { + for (const repoId of ALL_REPOS) { + results.push(await examineRepo(repoId)); + } + // Print the full matrix so the examination is on the record regardless of pass/fail. + const lines = results.map(r => { + const head = `${r.stage === 'ok' ? 'OK ' : 'FAIL'} ${r.curated ? '*' : ' '} ${r.repoId} stage=${r.stage} paired=${r.pairedCount}/${r.modelCount}`; + const detail = r.sampleModel ? `\n hfProj=${r.sampleProjectorHF} onDisk=${r.sampleOnDisk} belongs=${r.belongsAll}` : ''; + return `${head}${detail}`; + }); + process.stdout.write(`\n[A1 vision-projector matrix]\n${lines.join('\n')}\n`); + expect(results.length).toBe(ALL_REPOS.length); + }, 180000); + + it('CORE INVARIANT: every curated vision repo pairs a projector AND its renamed on-disk name belongs', () => { + const curated = results.filter(r => r.curated); + const broken = curated.filter(r => r.stage !== 'ok'); + expect(broken.map(r => `${r.repoId} [${r.stage}]`)).toEqual([]); + }); + + it('KNOWN-OPEN (separate, non-curated bug): exactly these wild repos still fail at PAIRING — tracked, not silently green', () => { + // pickMmProjForDownload returns no projector for these shapes: `mmproj-model-` literal token + // (ggml-org gemma-3, openbmb MiniCPM), a `UD-`packaged model quant (Mistral-Small), and an infixed + // `mmproj` on a `-text-model-` weights name (moondream). None are curated. Fixing that is a SEPARATE + // backlog item — this assertion is a ledger: a NEW pairing break, or a fix, forces this list to change. + const noPairing = results.filter(r => r.stage === 'no-pairing').map(r => r.repoId).sort(); + expect(noPairing).toEqual([ + 'ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF', + 'ggml-org/gemma-3-4b-it-GGUF', + 'moondream/moondream2-gguf', + 'openbmb/MiniCPM-V-2_6-gguf', + ]); + }); +}); diff --git a/__tests__/unit/services/parallelMmproj.test.ts b/__tests__/unit/services/parallelMmproj.test.ts index f140067fc..5e4c66141 100644 --- a/__tests__/unit/services/parallelMmproj.test.ts +++ b/__tests__/unit/services/parallelMmproj.test.ts @@ -1276,4 +1276,22 @@ describe('mmProjLocalName — one shared projector per model (dedup across quant expect(mmProjBelongsToModel('gemma-4-E2B-it-Q8_0.gguf', name)).toBe(true); expect(mmProjBelongsToModel('gemma-4-E4B-it-Q8_0.gguf', name)).toBe(false); }); + + // A1 (device 2026-07-23, iOS): ggml-org names projectors `mmproj--.gguf` — the model name + // sits AFTER the "mmproj" token. The old `/mmproj.*$/` kept that model name, doubling the on-disk stem so + // mmProjBelongsToModel refused it → link cleared → "Multimodal support not enabled". Offline guard so the + // regression is caught without the network (the live-HF proof is the .network integration test). + it('ggml-org shape: model name AFTER the mmproj token is dropped, on-disk name still belongs (SmolVLM/Qwen-VL)', () => { + const smol = mmProjLocalName('SmolVLM-256M-Instruct-Q8_0.gguf', 'mmproj-SmolVLM-256M-Instruct-Q8_0.gguf'); + expect(smol).toBe('smolvlm-256m-instruct-mmproj-Q8_0.gguf'); + expect(mmProjBelongsToModel('SmolVLM-256M-Instruct-Q8_0.gguf', smol)).toBe(true); + + const qwen = mmProjLocalName('Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen2.5-VL-3B-Instruct-f16.gguf'); + expect(qwen).toBe('qwen2.5-vl-3b-instruct-mmproj-f16.gguf'); + expect(mmProjBelongsToModel('Qwen2.5-VL-3B-Instruct-Q8_0.gguf', qwen)).toBe(true); + + // every quant of the ggml-org model still shares ONE projector file (quant-independent dedup preserved) + expect(mmProjLocalName('SmolVLM-256M-Instruct-Q8_0.gguf', 'mmproj-SmolVLM-256M-Instruct-f16.gguf')) + .toBe(mmProjLocalName('SmolVLM-256M-Instruct-Q4_K_M.gguf', 'mmproj-SmolVLM-256M-Instruct-f16.gguf')); + }); }); From e4380b30e49e7fa25ed8fcf26a5dd92e2d85b74e Mon Sep 17 00:00:00 2001 From: alichherawalla Date: Thu, 23 Jul 2026 19:13:41 +0530 Subject: [PATCH 2/7] fix(A1): keep only the projector precision in the on-disk mmproj name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ggml-org names projectors `mmproj--.gguf` (model name AFTER the mmproj token). The old `/mmproj.*$/` kept that model name, doubling the on-disk identity stem, so mmProjBelongsToModel refused it, linkOrphanMmProj cleared the link, and the model loaded text-only → initMultimodal never ran → 'Multimodal support not enabled' on a vision send (device 2026-07-23, iPhone 12, SmolVLM). Extract only the precision token so the on-disk stem always matches the model. Verified against live HF: 8 rename-broken repos now pass, the 6 already-working stay working. --- src/services/modelManager/download.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/services/modelManager/download.ts b/src/services/modelManager/download.ts index 02fcbb153..23b39cf80 100644 --- a/src/services/modelManager/download.ts +++ b/src/services/modelManager/download.ts @@ -24,11 +24,16 @@ import { extractBaseName } from './scan'; */ export function mmProjLocalName(ggufFileName: string, mmProjSourceName?: string): string { const base = extractBaseName(ggufFileName); - // Keep the projector's OWN precision (mmproj-F16 / mmproj-BF16) from its source filename, but strip any - // model prefix the repo added — so the on-disk name is `-mmproj-.gguf`: - // - model-prefixed → two repos that both ship `mmproj-F16.gguf` can't collide on disk (the original loss), + // Build the on-disk name as `-mmproj-.gguf`, keeping ONLY the projector's own + // precision token from its source name and dropping every model-name token wherever it sits. + // - `` prefix → two repos that both ship `mmproj-F16.gguf` can't collide on disk. // - MODEL-quant-independent (extractBaseName drops the model quant) → every quant shares ONE projector file. - const proj = mmProjSourceName?.match(/mmproj.*$/i)?.[0] ?? 'mmproj.gguf'; + // - precision-only projector portion → the on-disk stem always matches the model, so mmProjBelongsToModel + // keeps the link. The old `/mmproj.*$/` swallowed a model name the repo put AFTER "mmproj" + // (ggml-org ships `mmproj--.gguf`), doubling the stem → the link was cleared and vision + // never initialized ("Multimodal support not enabled" — device 2026-07-23, SmolVLM/Qwen2.5-VL on iOS). + const precision = mmProjSourceName?.match(/\b(iq\d+[a-z0-9_]*|q\d+[a-z0-9_]*|bf16|fp16|f16|f32)\b/i)?.[0]; + const proj = precision ? `mmproj-${precision}.gguf` : 'mmproj.gguf'; return `${base}-${proj}`; } From 0e29da907faf3867fcf4737d0e32e697c780c234 Mon Sep 17 00:00:00 2001 From: alichherawalla Date: Thu, 23 Jul 2026 19:22:14 +0530 Subject: [PATCH 3/7] test(A1): capture-once HF fixture + fast offline vision mmproj pipeline test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the always-network tests (flaky, slow) with the capture-once/reuse pattern: UPDATE_HF_FIXTURES=1 downloads the real HF listing + real GGUF header bytes for every vision repo into a committed fixture; normal runs replay the real pair→rename→belongs pipeline over that captured real data in ~0.5s, offline, deterministic. Reverting the fix flips the ggml-org curated repos red. Tracks the separate no-pairing bug as a ledger. --- __tests__/fixtures/hf/vision-repos.json | 544 ++++++++++++++++++ .../mmProjNamingAcrossRepos.network.test.ts | 117 ---- .../models/visionMmprojFromHF.test.ts | 157 +++++ 3 files changed, 701 insertions(+), 117 deletions(-) create mode 100644 __tests__/fixtures/hf/vision-repos.json delete mode 100644 __tests__/integration/models/mmProjNamingAcrossRepos.network.test.ts create mode 100644 __tests__/integration/models/visionMmprojFromHF.test.ts diff --git a/__tests__/fixtures/hf/vision-repos.json b/__tests__/fixtures/hf/vision-repos.json new file mode 100644 index 000000000..a4b1c245f --- /dev/null +++ b/__tests__/fixtures/hf/vision-repos.json @@ -0,0 +1,544 @@ +{ + "unsloth/gemma-4-E2B-it-GGUF": { + "exists": true, + "ggufFiles": [ + "MTP/mtp-gemma-4-E2B-it-BF16.gguf", + "MTP/mtp-gemma-4-E2B-it-F16.gguf", + "MTP/mtp-gemma-4-E2B-it-Q8_0.gguf", + "gemma-4-E2B-it-BF16.gguf", + "gemma-4-E2B-it-IQ4_NL.gguf", + "gemma-4-E2B-it-IQ4_XS.gguf", + "gemma-4-E2B-it-Q3_K_M.gguf", + "gemma-4-E2B-it-Q3_K_S.gguf", + "gemma-4-E2B-it-Q4_0.gguf", + "gemma-4-E2B-it-Q4_1.gguf", + "gemma-4-E2B-it-Q4_K_M.gguf", + "gemma-4-E2B-it-Q4_K_S.gguf", + "gemma-4-E2B-it-Q5_K_M.gguf", + "gemma-4-E2B-it-Q5_K_S.gguf", + "gemma-4-E2B-it-Q6_K.gguf", + "gemma-4-E2B-it-Q8_0.gguf", + "gemma-4-E2B-it-UD-IQ2_M.gguf", + "gemma-4-E2B-it-UD-IQ3_XXS.gguf", + "gemma-4-E2B-it-UD-Q2_K_XL.gguf", + "gemma-4-E2B-it-UD-Q3_K_XL.gguf", + "gemma-4-E2B-it-UD-Q4_K_XL.gguf", + "gemma-4-E2B-it-UD-Q5_K_XL.gguf", + "gemma-4-E2B-it-UD-Q6_K_XL.gguf", + "gemma-4-E2B-it-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "mmproj-F16.gguf", + "mmproj-F32.gguf", + "mtp-gemma-4-E2B-it.gguf" + ], + "probe": { + "model": { + "name": "mtp-gemma-4-E2B-it.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "unsloth/gemma-4-E4B-it-GGUF": { + "exists": true, + "ggufFiles": [ + "MTP/mtp-gemma-4-E4B-it-BF16.gguf", + "MTP/mtp-gemma-4-E4B-it-F16.gguf", + "MTP/mtp-gemma-4-E4B-it-Q8_0.gguf", + "gemma-4-E4B-it-BF16.gguf", + "gemma-4-E4B-it-IQ4_NL.gguf", + "gemma-4-E4B-it-IQ4_XS.gguf", + "gemma-4-E4B-it-Q3_K_M.gguf", + "gemma-4-E4B-it-Q3_K_S.gguf", + "gemma-4-E4B-it-Q4_0.gguf", + "gemma-4-E4B-it-Q4_1.gguf", + "gemma-4-E4B-it-Q4_K_M.gguf", + "gemma-4-E4B-it-Q4_K_S.gguf", + "gemma-4-E4B-it-Q5_K_M.gguf", + "gemma-4-E4B-it-Q5_K_S.gguf", + "gemma-4-E4B-it-Q6_K.gguf", + "gemma-4-E4B-it-Q8_0.gguf", + "gemma-4-E4B-it-UD-IQ2_M.gguf", + "gemma-4-E4B-it-UD-IQ3_XXS.gguf", + "gemma-4-E4B-it-UD-Q2_K_XL.gguf", + "gemma-4-E4B-it-UD-Q3_K_XL.gguf", + "gemma-4-E4B-it-UD-Q4_K_XL.gguf", + "gemma-4-E4B-it-UD-Q5_K_XL.gguf", + "gemma-4-E4B-it-UD-Q6_K_XL.gguf", + "gemma-4-E4B-it-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "mmproj-F16.gguf", + "mmproj-F32.gguf", + "mtp-gemma-4-E4B-it.gguf" + ], + "probe": { + "model": { + "name": "mtp-gemma-4-E4B-it.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "unsloth/Qwen3.5-0.8B-GGUF": { + "exists": true, + "ggufFiles": [ + "Qwen3.5-0.8B-BF16.gguf", + "Qwen3.5-0.8B-IQ4_NL.gguf", + "Qwen3.5-0.8B-IQ4_XS.gguf", + "Qwen3.5-0.8B-Q3_K_M.gguf", + "Qwen3.5-0.8B-Q3_K_S.gguf", + "Qwen3.5-0.8B-Q4_0.gguf", + "Qwen3.5-0.8B-Q4_1.gguf", + "Qwen3.5-0.8B-Q4_K_M.gguf", + "Qwen3.5-0.8B-Q4_K_S.gguf", + "Qwen3.5-0.8B-Q5_K_M.gguf", + "Qwen3.5-0.8B-Q5_K_S.gguf", + "Qwen3.5-0.8B-Q6_K.gguf", + "Qwen3.5-0.8B-Q8_0.gguf", + "Qwen3.5-0.8B-UD-IQ2_M.gguf", + "Qwen3.5-0.8B-UD-IQ2_XXS.gguf", + "Qwen3.5-0.8B-UD-IQ3_XXS.gguf", + "Qwen3.5-0.8B-UD-Q2_K_XL.gguf", + "Qwen3.5-0.8B-UD-Q3_K_XL.gguf", + "Qwen3.5-0.8B-UD-Q4_K_XL.gguf", + "Qwen3.5-0.8B-UD-Q5_K_XL.gguf", + "Qwen3.5-0.8B-UD-Q6_K_XL.gguf", + "Qwen3.5-0.8B-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "mmproj-F16.gguf", + "mmproj-F32.gguf" + ], + "probe": { + "model": { + "name": "Qwen3.5-0.8B-BF16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/SmolVLM2-500M-Video-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "SmolVLM2-500M-Video-Instruct-Q8_0.gguf", + "SmolVLM2-500M-Video-Instruct-f16.gguf", + "mmproj-SmolVLM2-500M-Video-Instruct-Q8_0.gguf", + "mmproj-SmolVLM2-500M-Video-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "SmolVLM2-500M-Video-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-SmolVLM2-500M-Video-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/SmolVLM-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "SmolVLM-Instruct-Q4_K_M.gguf", + "SmolVLM-Instruct-Q8_0.gguf", + "SmolVLM-Instruct-f16.gguf", + "mmproj-SmolVLM-Instruct-Q8_0.gguf", + "mmproj-SmolVLM-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "SmolVLM-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-SmolVLM-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/SmolVLM2-2.2B-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "SmolVLM2-2.2B-Instruct-Q4_K_M.gguf", + "SmolVLM2-2.2B-Instruct-Q8_0.gguf", + "SmolVLM2-2.2B-Instruct-f16.gguf", + "mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf", + "mmproj-SmolVLM2-2.2B-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "SmolVLM2-2.2B-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/SmolVLM-256M-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "SmolVLM-256M-Instruct-Q8_0.gguf", + "SmolVLM-256M-Instruct-f16.gguf", + "mmproj-SmolVLM-256M-Instruct-Q8_0.gguf", + "mmproj-SmolVLM-256M-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "SmolVLM-256M-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-SmolVLM-256M-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/SmolVLM-500M-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "SmolVLM-500M-Instruct-Q8_0.gguf", + "SmolVLM-500M-Instruct-f16.gguf", + "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf", + "mmproj-SmolVLM-500M-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "SmolVLM-500M-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/Qwen2.5-VL-3B-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf", + "Qwen2.5-VL-3B-Instruct-Q8_0.gguf", + "Qwen2.5-VL-3B-Instruct-f16.gguf", + "mmproj-Qwen2.5-VL-3B-Instruct-Q8_0.gguf", + "mmproj-Qwen2.5-VL-3B-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "Qwen2.5-VL-3B-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-Qwen2.5-VL-3B-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/Qwen2.5-VL-7B-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", + "Qwen2.5-VL-7B-Instruct-Q8_0.gguf", + "Qwen2.5-VL-7B-Instruct-f16.gguf", + "mmproj-Qwen2.5-VL-7B-Instruct-Q8_0.gguf", + "mmproj-Qwen2.5-VL-7B-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "Qwen2.5-VL-7B-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-Qwen2.5-VL-7B-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "unsloth/Qwen2.5-VL-7B-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "Qwen2.5-VL-7B-Instruct-BF16.gguf", + "Qwen2.5-VL-7B-Instruct-IQ4_NL.gguf", + "Qwen2.5-VL-7B-Instruct-IQ4_XS.gguf", + "Qwen2.5-VL-7B-Instruct-Q2_K.gguf", + "Qwen2.5-VL-7B-Instruct-Q2_K_L.gguf", + "Qwen2.5-VL-7B-Instruct-Q3_K_M.gguf", + "Qwen2.5-VL-7B-Instruct-Q3_K_S.gguf", + "Qwen2.5-VL-7B-Instruct-Q4_0.gguf", + "Qwen2.5-VL-7B-Instruct-Q4_1.gguf", + "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", + "Qwen2.5-VL-7B-Instruct-Q4_K_S.gguf", + "Qwen2.5-VL-7B-Instruct-Q5_K_M.gguf", + "Qwen2.5-VL-7B-Instruct-Q5_K_S.gguf", + "Qwen2.5-VL-7B-Instruct-Q6_K.gguf", + "Qwen2.5-VL-7B-Instruct-Q8_0.gguf", + "Qwen2.5-VL-7B-Instruct-UD-IQ1_M.gguf", + "Qwen2.5-VL-7B-Instruct-UD-IQ1_S.gguf", + "Qwen2.5-VL-7B-Instruct-UD-IQ2_M.gguf", + "Qwen2.5-VL-7B-Instruct-UD-IQ2_XXS.gguf", + "Qwen2.5-VL-7B-Instruct-UD-IQ3_XXS.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q2_K_XL.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q3_K_XL.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q4_K_XL.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q5_K_XL.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q6_K_XL.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "mmproj-F16.gguf", + "mmproj-F32.gguf" + ], + "probe": { + "model": { + "name": "Qwen2.5-VL-7B-Instruct-BF16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/pixtral-12b-GGUF": { + "exists": true, + "ggufFiles": [ + "mmproj-pixtral-12b-Q8_0.gguf", + "mmproj-pixtral-12b-f16.gguf", + "pixtral-12b-Q2_K.gguf", + "pixtral-12b-Q4_K_M.gguf", + "pixtral-12b-Q8_0.gguf", + "pixtral-12b-f16.gguf" + ], + "probe": { + "model": { + "name": "pixtral-12b-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-pixtral-12b-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/InternVL3-2B-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "InternVL3-2B-Instruct-Q4_K_M.gguf", + "InternVL3-2B-Instruct-Q8_0.gguf", + "InternVL3-2B-Instruct-f16.gguf", + "mmproj-InternVL3-2B-Instruct-Q8_0.gguf", + "mmproj-InternVL3-2B-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "InternVL3-2B-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-InternVL3-2B-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/gemma-3-4b-it-GGUF": { + "exists": true, + "ggufFiles": [ + "gemma-3-4b-it-Q4_K_M.gguf", + "gemma-3-4b-it-Q8_0.gguf", + "gemma-3-4b-it-f16.gguf", + "mmproj-model-f16.gguf" + ], + "probe": { + "model": { + "name": "gemma-3-4b-it-f16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "unsloth/gemma-3-4b-it-GGUF": { + "exists": true, + "ggufFiles": [ + "gemma-3-4b-it-BF16.gguf", + "gemma-3-4b-it-IQ4_NL.gguf", + "gemma-3-4b-it-IQ4_XS.gguf", + "gemma-3-4b-it-Q2_K.gguf", + "gemma-3-4b-it-Q2_K_L.gguf", + "gemma-3-4b-it-Q3_K_M.gguf", + "gemma-3-4b-it-Q3_K_S.gguf", + "gemma-3-4b-it-Q4_0.gguf", + "gemma-3-4b-it-Q4_1.gguf", + "gemma-3-4b-it-Q4_K_M.gguf", + "gemma-3-4b-it-Q4_K_S.gguf", + "gemma-3-4b-it-Q5_K_M.gguf", + "gemma-3-4b-it-Q5_K_S.gguf", + "gemma-3-4b-it-Q6_K.gguf", + "gemma-3-4b-it-Q8_0.gguf", + "gemma-3-4b-it-UD-IQ1_M.gguf", + "gemma-3-4b-it-UD-IQ1_S.gguf", + "gemma-3-4b-it-UD-IQ2_M.gguf", + "gemma-3-4b-it-UD-IQ2_XXS.gguf", + "gemma-3-4b-it-UD-IQ3_XXS.gguf", + "gemma-3-4b-it-UD-Q2_K_XL.gguf", + "gemma-3-4b-it-UD-Q3_K_XL.gguf", + "gemma-3-4b-it-UD-Q4_K_XL.gguf", + "gemma-3-4b-it-UD-Q5_K_XL.gguf", + "gemma-3-4b-it-UD-Q6_K_XL.gguf", + "gemma-3-4b-it-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "mmproj-F16.gguf", + "mmproj-F32.gguf" + ], + "probe": { + "model": { + "name": "gemma-3-4b-it-BF16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF": { + "exists": true, + "ggufFiles": [ + "Mistral-Small-3.1-24B-Instruct-2503-UD-IQ2_M.gguf", + "mmproj-Mistral-Small-3.1-24B-Instruct-2503-Q8_0.gguf", + "mmproj-Mistral-Small-3.1-24B-Instruct-2503-f16.gguf" + ], + "probe": { + "model": { + "name": "Mistral-Small-3.1-24B-Instruct-2503-UD-IQ2_M.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "openbmb/MiniCPM-V-2_6-gguf": { + "exists": true, + "ggufFiles": [ + "ggml-model-IQ3_M.gguf", + "ggml-model-IQ3_S.gguf", + "ggml-model-IQ3_XS.gguf", + "ggml-model-IQ4_NL.gguf", + "ggml-model-IQ4_XS.gguf", + "ggml-model-Q2_K.gguf", + "ggml-model-Q3_K.gguf", + "ggml-model-Q3_K_L.gguf", + "ggml-model-Q3_K_M.gguf", + "ggml-model-Q3_K_S.gguf", + "ggml-model-Q4_0.gguf", + "ggml-model-Q4_1.gguf", + "ggml-model-Q4_K.gguf", + "ggml-model-Q4_K_M.gguf", + "ggml-model-Q4_K_S.gguf", + "ggml-model-Q5_0.gguf", + "ggml-model-Q5_1.gguf", + "ggml-model-Q5_K.gguf", + "ggml-model-Q5_K_M.gguf", + "ggml-model-Q5_K_S.gguf", + "ggml-model-Q6_K.gguf", + "ggml-model-Q8_0.gguf", + "ggml-model-f16.gguf", + "mmproj-model-f16.gguf" + ], + "probe": { + "model": { + "name": "ggml-model-f16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "moondream/moondream2-gguf": { + "exists": true, + "ggufFiles": [ + "moondream2-mmproj-f16.gguf", + "moondream2-text-model-f16.gguf" + ], + "probe": { + "model": { + "name": "moondream2-text-model-f16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "abetlen/Phi-3.5-vision-instruct-gguf": { + "exists": true, + "ggufFiles": [ + "Phi-3.5-3.8B-vision-instruct-F16.gguf", + "Phi-3.5-3.8B-vision-instruct-Q8_0.gguf", + "Phi-3.5-3.8B-vision-instruct-mmproj-F16.gguf" + ], + "probe": { + "model": { + "name": "Phi-3.5-3.8B-vision-instruct-F16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "Phi-3.5-3.8B-vision-instruct-mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "leafspark/Llama-3.2-11B-Vision-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "Llama-3.2-11B-Vision-Instruct-mmproj.f16.gguf", + "Llama-3.2-11B-Vision-Instruct.Q4_K_M.gguf", + "Llama-3.2-11B-Vision-Instruct.Q8_0.gguf", + "Llama-3.2-11B-Vision-Instruct.f16.gguf" + ], + "probe": { + "model": { + "name": "Llama-3.2-11B-Vision-Instruct.f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "Llama-3.2-11B-Vision-Instruct-mmproj.f16.gguf", + "status": 206, + "magic": "GGUF" + } + } + } +} diff --git a/__tests__/integration/models/mmProjNamingAcrossRepos.network.test.ts b/__tests__/integration/models/mmProjNamingAcrossRepos.network.test.ts deleted file mode 100644 index 531fc508d..000000000 --- a/__tests__/integration/models/mmProjNamingAcrossRepos.network.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -/** - * A1 — iOS "Multimodal support not enabled. Call initMultimodal first." - * - * The chain a vision model must survive to actually initialize multimodal on device: - * (1) PAIR — from the LIVE HF listing, `pickMmProjForDownload` picks the projector that belongs to the - * model (huggingFaceService.getModelFiles → ModelFile.mmProjFile). If it returns none, the - * app never even downloads a projector → text-only → the device crash. - * (2) RENAME — on download the projector is renamed via `mmProjLocalName` to a shared on-disk name. - * (3) BELONGS — the strict on-disk matcher (`mmProjBelongsToModel`) must still accept the renamed file, or - * `linkOrphanMmProj` clears the link on the next scan → text-only → the device crash. - * - * This test hits the REAL Hugging Face listing for every vision repo the app curates plus popular families, - * runs the REAL product pipeline over whatever HF publishes (no hardcoded filenames, no mocks of our code), - * and reports at WHICH stage each repo breaks. Requires network. - */ -import { huggingFaceService } from '../../../src/services/huggingface'; -import { mmProjLocalName } from '../../../src/services/modelManager/download'; -import { mmProjBelongsToModel } from '../../../src/services/mmproj'; -import { RECOMMENDED_MODELS } from '../../../src/constants/models'; - -const CURATED_VISION_REPOS = RECOMMENDED_MODELS.filter(m => m.type === 'vision').map(m => m.id); - -// Popular vision families in the wild (real repos confirmed via HF API during research). -const POPULAR_VISION_REPOS = [ - 'ggml-org/SmolVLM-256M-Instruct-GGUF', // the exact device report (Christophe) - 'ggml-org/SmolVLM-500M-Instruct-GGUF', - 'ggml-org/Qwen2.5-VL-3B-Instruct-GGUF', - 'ggml-org/Qwen2.5-VL-7B-Instruct-GGUF', - 'unsloth/Qwen2.5-VL-7B-Instruct-GGUF', - 'ggml-org/gemma-3-4b-it-GGUF', - 'unsloth/gemma-3-4b-it-GGUF', - 'ggml-org/pixtral-12b-GGUF', - 'ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF', - 'ggml-org/InternVL3-2B-Instruct-GGUF', - 'openbmb/MiniCPM-V-2_6-gguf', - 'moondream/moondream2-gguf', - 'abetlen/Phi-3.5-vision-instruct-gguf', - 'leafspark/Llama-3.2-11B-Vision-Instruct-GGUF', -]; - -const ALL_REPOS = [...new Set([...CURATED_VISION_REPOS, ...POPULAR_VISION_REPOS])]; - -interface RepoResult { - repoId: string; - curated: boolean; - exists: boolean; - modelCount: number; - pairedCount: number; - sampleModel?: string; - sampleProjectorHF?: string; - sampleOnDisk?: string; - belongsAll: boolean; - stage: 'ok' | 'no-repo' | 'no-pairing' | 'rename-mismatch'; -} - -async function examineRepo(repoId: string): Promise { - const curated = CURATED_VISION_REPOS.includes(repoId); - let files; - try { - files = await huggingFaceService.getModelFiles(repoId); - } catch { - return { repoId, curated, exists: false, modelCount: 0, pairedCount: 0, belongsAll: false, stage: 'no-repo' }; - } - if (files.length === 0) { - return { repoId, curated, exists: false, modelCount: 0, pairedCount: 0, belongsAll: false, stage: 'no-repo' }; - } - const paired = files.filter(f => f.mmProjFile); - if (paired.length === 0) { - return { repoId, curated, exists: true, modelCount: files.length, pairedCount: 0, belongsAll: false, stage: 'no-pairing' }; - } - const sample = paired[0]; - const sampleOnDisk = mmProjLocalName(sample.name, sample.mmProjFile!.name); - const belongsAll = paired.every(f => mmProjBelongsToModel(f.name, mmProjLocalName(f.name, f.mmProjFile!.name))); - return { - repoId, curated, exists: true, modelCount: files.length, pairedCount: paired.length, - sampleModel: sample.name, sampleProjectorHF: sample.mmProjFile!.name, sampleOnDisk, - belongsAll, stage: belongsAll ? 'ok' : 'rename-mismatch', - }; -} - -describe('mmproj naming — live HF pairing→rename→belongs pipeline across every vision repo', () => { - const results: RepoResult[] = []; - - it('examines every repo and records the stage it reaches', async () => { - for (const repoId of ALL_REPOS) { - results.push(await examineRepo(repoId)); - } - // Print the full matrix so the examination is on the record regardless of pass/fail. - const lines = results.map(r => { - const head = `${r.stage === 'ok' ? 'OK ' : 'FAIL'} ${r.curated ? '*' : ' '} ${r.repoId} stage=${r.stage} paired=${r.pairedCount}/${r.modelCount}`; - const detail = r.sampleModel ? `\n hfProj=${r.sampleProjectorHF} onDisk=${r.sampleOnDisk} belongs=${r.belongsAll}` : ''; - return `${head}${detail}`; - }); - process.stdout.write(`\n[A1 vision-projector matrix]\n${lines.join('\n')}\n`); - expect(results.length).toBe(ALL_REPOS.length); - }, 180000); - - it('CORE INVARIANT: every curated vision repo pairs a projector AND its renamed on-disk name belongs', () => { - const curated = results.filter(r => r.curated); - const broken = curated.filter(r => r.stage !== 'ok'); - expect(broken.map(r => `${r.repoId} [${r.stage}]`)).toEqual([]); - }); - - it('KNOWN-OPEN (separate, non-curated bug): exactly these wild repos still fail at PAIRING — tracked, not silently green', () => { - // pickMmProjForDownload returns no projector for these shapes: `mmproj-model-` literal token - // (ggml-org gemma-3, openbmb MiniCPM), a `UD-`packaged model quant (Mistral-Small), and an infixed - // `mmproj` on a `-text-model-` weights name (moondream). None are curated. Fixing that is a SEPARATE - // backlog item — this assertion is a ledger: a NEW pairing break, or a fix, forces this list to change. - const noPairing = results.filter(r => r.stage === 'no-pairing').map(r => r.repoId).sort(); - expect(noPairing).toEqual([ - 'ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF', - 'ggml-org/gemma-3-4b-it-GGUF', - 'moondream/moondream2-gguf', - 'openbmb/MiniCPM-V-2_6-gguf', - ]); - }); -}); diff --git a/__tests__/integration/models/visionMmprojFromHF.test.ts b/__tests__/integration/models/visionMmprojFromHF.test.ts new file mode 100644 index 000000000..085cfbb99 --- /dev/null +++ b/__tests__/integration/models/visionMmprojFromHF.test.ts @@ -0,0 +1,157 @@ +/** + * A1 — vision projector pairing/rename/download, verified against REAL Hugging Face data. + * + * Design: capture once, reuse. The real HF listing + a real download probe (opening bytes of the model and + * its paired mmproj) are captured into a committed fixture (`__tests__/fixtures/hf/vision-repos.json`). Normal + * runs are FAST and OFFLINE: they replay the real product pipeline (pickMmProjForDownload → mmProjLocalName → + * mmProjBelongsToModel) over the captured real filenames, and assert the captured download probe shows a + * genuine GGUF. No per-run network, no flakiness. + * + * To refresh from live HF (the "download once" step — re-hits huggingface.co, downloads the header bytes of + * every sampled file, rewrites the fixture): + * + * UPDATE_HF_FIXTURES=1 npx jest visionMmprojFromHF + * + * The pipeline under test is the exact code that broke on device (SmolVLM/Qwen2.5-VL, iOS, 2026-07-23). + */ +import fs from 'fs'; +import path from 'path'; +import { isMMProjFile, pickMmProjForDownload, mmProjBelongsToModel } from '../../../src/services/mmproj'; +import { mmProjLocalName } from '../../../src/services/modelManager/download'; +import { RECOMMENDED_MODELS } from '../../../src/constants/models'; + +const CURATED_VISION_REPOS = RECOMMENDED_MODELS.filter(m => m.type === 'vision').map(m => m.id); +const POPULAR_VISION_REPOS = [ + 'ggml-org/SmolVLM-256M-Instruct-GGUF', // the exact device report + 'ggml-org/SmolVLM-500M-Instruct-GGUF', + 'ggml-org/Qwen2.5-VL-3B-Instruct-GGUF', + 'ggml-org/Qwen2.5-VL-7B-Instruct-GGUF', + 'unsloth/Qwen2.5-VL-7B-Instruct-GGUF', + 'ggml-org/pixtral-12b-GGUF', + 'ggml-org/InternVL3-2B-Instruct-GGUF', + 'ggml-org/gemma-3-4b-it-GGUF', + 'unsloth/gemma-3-4b-it-GGUF', + 'ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF', + 'openbmb/MiniCPM-V-2_6-gguf', + 'moondream/moondream2-gguf', + 'abetlen/Phi-3.5-vision-instruct-gguf', + 'leafspark/Llama-3.2-11B-Vision-Instruct-GGUF', +]; +const ALL_REPOS = [...new Set([...CURATED_VISION_REPOS, ...POPULAR_VISION_REPOS])]; + +const FIXTURE_PATH = path.join(__dirname, '../../fixtures/hf/vision-repos.json'); +const REFRESH = !!process.env.UPDATE_HF_FIXTURES; + +interface DownloadProbe { name: string; status: number; magic: string } +interface RepoFixture { + exists: boolean; + ggufFiles: string[]; // every .gguf filename HF publishes (real) + probe: { model?: DownloadProbe; mmproj?: DownloadProbe } | null; // real opening-bytes download +} +type Fixture = Record; + +// ---- capture (live network; only when UPDATE_HF_FIXTURES=1) ---------------------------------------- + +async function fetchTreeGgufFiles(repoId: string): Promise { + const res = await fetch(`https://huggingface.co/api/models/${repoId}/tree/main?recursive=true`, { + headers: { Accept: 'application/json' }, + }); + if (!res.ok) return []; + const entries: Array<{ type: string; path: string }> = await res.json(); + return entries.filter(e => e.type === 'file' && e.path.endsWith('.gguf')).map(e => e.path); +} + +async function downloadHead(repoId: string, fileName: string): Promise { + const url = `https://huggingface.co/${repoId}/resolve/main/${fileName}`; + const controller = new AbortController(); + const res = await fetch(url, { headers: { Range: 'bytes=0-3' }, signal: controller.signal }); + let magic = ''; + if (res.body) { + const { value } = await res.body.getReader().read(); + if (value) magic = Buffer.from(value).subarray(0, 4).toString('latin1'); + } + controller.abort(); // never pull the whole (multi-GB) file + return { name: fileName, status: res.status, magic }; +} + +async function captureFixture(): Promise { + const out: Fixture = {}; + for (const repoId of ALL_REPOS) { + const ggufFiles = await fetchTreeGgufFiles(repoId); + const projectors = ggufFiles.filter(isMMProjFile); + const models = ggufFiles.filter(f => !isMMProjFile(f)).sort((a, b) => a.length - b.length); + let probe: RepoFixture['probe'] = null; + const sampleModel = models.find(m => pickMmProjForDownload(m, projectors)) ?? models[0]; + if (sampleModel) { + const chosen = pickMmProjForDownload(sampleModel, projectors); + const model = await downloadHead(repoId, sampleModel); + const mmproj = chosen ? await downloadHead(repoId, chosen) : undefined; + probe = { model, mmproj }; + } + out[repoId] = { exists: ggufFiles.length > 0, ggufFiles, probe }; + } + return out; +} + +// ---- the fixture (captured real HF data) ----------------------------------------------------------- + +function loadFixture(): Fixture | null { + return fs.existsSync(FIXTURE_PATH) ? JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')) : null; +} + +// A vision repo passes when the app pairs a projector to at least one model AND the on-disk rename of that +// projector still belongs to the model (so the link survives → initMultimodal runs). +function repoPairsAndBelongs(fx: RepoFixture): { pairs: boolean; belongs: boolean } { + const projectors = fx.ggufFiles.filter(isMMProjFile); + const models = fx.ggufFiles.filter(f => !isMMProjFile(f)); + const pairedModels = models.filter(m => pickMmProjForDownload(m, projectors)); + if (pairedModels.length === 0) return { pairs: false, belongs: false }; + const belongs = pairedModels.every(m => { + const chosen = pickMmProjForDownload(m, projectors)!; + return mmProjBelongsToModel(m, mmProjLocalName(m, chosen)); + }); + return { pairs: true, belongs }; +} + +(REFRESH ? describe : describe.skip)('refresh HF vision fixtures (live network — UPDATE_HF_FIXTURES=1)', () => { + it('captures the real listing + a real download probe for every vision repo', async () => { + const fixture = await captureFixture(); + fs.mkdirSync(path.dirname(FIXTURE_PATH), { recursive: true }); + fs.writeFileSync(FIXTURE_PATH, `${JSON.stringify(fixture, null, 2)}\n`); + expect(Object.keys(fixture)).toEqual(ALL_REPOS); + }, 300000); +}); + +describe('vision mmproj pairing/rename/download over cached REAL Hugging Face data', () => { + const fx = loadFixture(); + + it('the HF fixture exists (run UPDATE_HF_FIXTURES=1 to capture it)', () => { + expect(fx).not.toBeNull(); + }); + + it.each(CURATED_VISION_REPOS)( + 'curated %s: pairs a projector, its renamed on-disk name belongs, and both files are real GGUF on HF', + (repoId) => { + const repo = fx?.[repoId]; + expect(repo?.exists).toBe(true); + const { pairs, belongs } = repoPairsAndBelongs(repo!); + expect({ repoId, pairs, belongs }).toEqual({ repoId, pairs: true, belongs: true }); + // the captured real download probe proves the model + its projector are genuine, present GGUF files + expect(repo!.probe?.model?.magic).toBe('GGUF'); + expect(repo!.probe?.mmproj?.magic).toBe('GGUF'); + }, + ); + + it('KNOWN-OPEN (separate, non-curated pairing bug): exactly these wild repos still fail to pair — tracked', () => { + const noPairing = ALL_REPOS.filter(id => { + const repo = fx?.[id]; + return repo?.exists && !repoPairsAndBelongs(repo).pairs; + }).sort(); + expect(noPairing).toEqual([ + 'ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF', + 'ggml-org/gemma-3-4b-it-GGUF', + 'moondream/moondream2-gguf', + 'openbmb/MiniCPM-V-2_6-gguf', + ]); + }); +}); From 087d9cd8e664e14f361ad962ec939db3ffc433fb Mon Sep 17 00:00:00 2001 From: alichherawalla Date: Fri, 24 Jul 2026 12:18:36 +0530 Subject: [PATCH 4/7] fix(A1): pair the remaining ggml-org/wild mmproj shapes (no-pairing seam) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the second mmproj bug class found in the live-HF sweep, so ALL vision-projector shapes pair + belong: - `mmproj-model-` literal placeholder → treated as a generic repo-single-model projector (ggml-org gemma-3, openbmb MiniCPM). - unsloth `UD-` dynamic-quant packaging stripped from the identity stem so a UD-quant model matches its non-UD projector (Mistral-Small-3.1). - moondream `-text-model` weights marker normalized so the weights and `-mmproj` file share a stem. Strict matching preserved — E2B still refuses an E4B projector (unit test). Verified over live-HF fixture data: every curated + popular vision repo now pairs and its on-disk name belongs. --- .../models/visionMmprojFromHF.test.ts | 18 ++++++------ .../unit/services/parallelMmproj.test.ts | 28 ++++++++++++++++++- src/services/mmproj.ts | 23 +++++++++++---- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/__tests__/integration/models/visionMmprojFromHF.test.ts b/__tests__/integration/models/visionMmprojFromHF.test.ts index 085cfbb99..38ea2551f 100644 --- a/__tests__/integration/models/visionMmprojFromHF.test.ts +++ b/__tests__/integration/models/visionMmprojFromHF.test.ts @@ -142,16 +142,16 @@ describe('vision mmproj pairing/rename/download over cached REAL Hugging Face da }, ); - it('KNOWN-OPEN (separate, non-curated pairing bug): exactly these wild repos still fail to pair — tracked', () => { - const noPairing = ALL_REPOS.filter(id => { + it('EVERY examined vision repo (curated + popular, all naming shapes) pairs a projector that belongs', () => { + // Was a "known-open" ledger for the no-pairing shapes (mmproj-model literal, moondream infixed + // mmproj + -text-model weights, unsloth UD- quant packaging). All are fixed in mmproj.ts now, so the + // invariant is stronger: NO real vision repo is left unpaired or with a mismatched on-disk name. + const broken = ALL_REPOS.filter(id => { const repo = fx?.[id]; - return repo?.exists && !repoPairsAndBelongs(repo).pairs; + if (!repo?.exists) return false; + const { pairs, belongs } = repoPairsAndBelongs(repo); + return !pairs || !belongs; }).sort(); - expect(noPairing).toEqual([ - 'ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF', - 'ggml-org/gemma-3-4b-it-GGUF', - 'moondream/moondream2-gguf', - 'openbmb/MiniCPM-V-2_6-gguf', - ]); + expect(broken).toEqual([]); }); }); diff --git a/__tests__/unit/services/parallelMmproj.test.ts b/__tests__/unit/services/parallelMmproj.test.ts index 5e4c66141..e8b313079 100644 --- a/__tests__/unit/services/parallelMmproj.test.ts +++ b/__tests__/unit/services/parallelMmproj.test.ts @@ -17,7 +17,7 @@ import { syncCompletedBackgroundDownloads, mmProjLocalName, } from '../../../src/services/modelManager/download'; -import { mmProjBelongsToModel } from '../../../src/services/mmproj'; +import { mmProjBelongsToModel, pickMmProjForDownload } from '../../../src/services/mmproj'; import { restoreInProgressDownloads } from '../../../src/services/modelManager/restore'; import { backgroundDownloadService } from '../../../src/services/backgroundDownloadService'; import { BackgroundDownloadContext } from '../../../src/services/modelManager/types'; @@ -1295,3 +1295,29 @@ describe('mmProjLocalName — one shared projector per model (dedup across quant .toBe(mmProjLocalName('SmolVLM-256M-Instruct-Q4_K_M.gguf', 'mmproj-SmolVLM-256M-Instruct-f16.gguf')); }); }); + +// A1 no-pairing shapes (found in the live-HF sweep) — pickMmProjForDownload must pair these too. +describe('pickMmProjForDownload — additional real-world projector shapes', () => { + it('literal `mmproj-model-` placeholder is generic (ggml-org gemma-3, openbmb MiniCPM)', () => { + expect(pickMmProjForDownload('gemma-3-4b-it-Q4_K_M.gguf', ['mmproj-model-f16.gguf'])) + .toBe('mmproj-model-f16.gguf'); + expect(pickMmProjForDownload('ggml-model-Q4_K_M.gguf', ['mmproj-model-f16.gguf'])) + .toBe('mmproj-model-f16.gguf'); + }); + + it('unsloth UD- dynamic-quant packaging is stripped so the model matches its non-UD projector', () => { + expect(mmProjBelongsToModel( + 'Mistral-Small-3.1-24B-Instruct-2503-UD-IQ2_M.gguf', + 'mmproj-Mistral-Small-3.1-24B-Instruct-2503-Q8_0.gguf', + )).toBe(true); + }); + + it('moondream `-text-model` weights match the `-mmproj` projector of the same base', () => { + expect(mmProjBelongsToModel('moondream2-text-model-f16.gguf', 'moondream2-mmproj-f16.gguf')).toBe(true); + }); + + it('still REFUSES a genuinely different model+variant (no over-loosening: E2B ≠ E4B)', () => { + expect(pickMmProjForDownload('gemma-4-E2B-it-Q4_K_M.gguf', ['gemma-4-E4B-it-mmproj-F16.gguf'])) + .toBeUndefined(); + }); +}); diff --git a/src/services/mmproj.ts b/src/services/mmproj.ts index 11064bb22..6c39c31f9 100644 --- a/src/services/mmproj.ts +++ b/src/services/mmproj.ts @@ -24,13 +24,21 @@ export function isMMProjFile(fileName: string): boolean { * The model-identity stem of a gguf/mmproj filename: lowercased, with the extension, any `mmproj` marker, * and the QUANTIZATION token removed — so name + variant remain and quant is ignored. Both * gemma-4-E2B-it-Q4_K_M.gguf and gemma-4-E2B-it-Q8_0-mmproj.gguf reduce to `gemma4e2bit`; E4B reduces to - * `gemma4e4bit` (distinct). Packaging/variant suffixes (UD, instruct, …) are kept, so they must match too. + * `gemma4e4bit` (distinct). Model-VARIANT tokens (E2B/E4B, instruct, …) are kept, so they must match too; + * only quant packaging (the `UD-` dynamic-quant prefix) and the `-text-model` role marker are normalized out. */ function modelIdentityStem(fileName: string): string { return fileName .toLowerCase() .replace(/\.gguf$/, '') .replace(/[-_.]?mmproj/g, '') + // moondream ships its LLM weights as `-text-model-` alongside `-mmproj-`; + // drop the `-text-model` role marker so the weights and projector reduce to the same stem. + .replace(/[-_.]text[-_]?model/g, '') + // unsloth-DYNAMIC quant packaging is `UD-` (e.g. `-UD-IQ2_M`). The `UD` is quant packaging, + // NOT a model variant — strip it (only when it directly precedes a quant token) so a UD-quant model + // reduces to the same stem as its (non-UD) projector. + .replace(/[-_.]ud(?=[-_.](?:iq\d|q\d|f\d|fp16|bf16))/gi, '') // quant / precision tokens: Q4_K_M, Q8_0, Q5_K_S, Q6_K, IQ4_XS, F16, FP16, F32, BF16, … .replace(/[-_.]?(iq\d+[a-z0-9_]*|q\d+[a-z0-9_]*|fp16|f16|f32|bf16)/gi, '') .replace(/[^a-z0-9]+/g, ''); @@ -85,10 +93,15 @@ export function pickMmProjForDownload( const exact = candidateNames.find(name => modelIdentityStem(name) === modelStem); if (exact) return exact; - // (A) Projectors with NO model-name token (empty identity stem) are generic to the repo's single model. - // Prefer F16 (not BF16), else the first. A NAMED-but-different projector is excluded here, so it can - // never be chosen as the generic fallback (case B refusal). - const generic = candidateNames.filter(name => modelIdentityStem(name) === ''); + // (A) GENERIC projectors are generic to the repo's single model: either NO model-name token (empty + // identity stem, e.g. unsloth `mmproj-F16`) OR the literal placeholder token `model` (ggml-org gemma-3 / + // openbmb MiniCPM ship `mmproj-model-`, whose stem reduces to `model` — a placeholder, not a real + // model name). Prefer F16 (not BF16), else the first. A NAMED-but-different projector is excluded here, so + // it can never be chosen as the generic fallback (case B refusal). + const generic = candidateNames.filter(name => { + const stem = modelIdentityStem(name); + return stem === '' || stem === 'model'; + }); if (generic.length > 0) { const f16 = generic.find(name => { const lower = name.toLowerCase(); From 85510b42d1b38af69419069c7a2eaee3c87c3462 Mon Sep 17 00:00:00 2001 From: alichherawalla Date: Fri, 24 Jul 2026 14:21:43 +0530 Subject: [PATCH 5/7] fix(A1): anchor mmproj precision match to \.gguf$ (linear, trailing token) Addresses SonarCloud (super-linear regex backtracking, download.ts:35) and CodeRabbit (take the projector's OWN trailing precision, not the first precision-like token in the name) in one change. Also: fixture refresh now throws on a HF 429/5xx instead of silently persisting an empty fixture. --- __tests__/integration/models/visionMmprojFromHF.test.ts | 5 ++++- src/services/modelManager/download.ts | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/__tests__/integration/models/visionMmprojFromHF.test.ts b/__tests__/integration/models/visionMmprojFromHF.test.ts index 38ea2551f..14792f782 100644 --- a/__tests__/integration/models/visionMmprojFromHF.test.ts +++ b/__tests__/integration/models/visionMmprojFromHF.test.ts @@ -56,7 +56,10 @@ async function fetchTreeGgufFiles(repoId: string): Promise { const res = await fetch(`https://huggingface.co/api/models/${repoId}/tree/main?recursive=true`, { headers: { Accept: 'application/json' }, }); - if (!res.ok) return []; + // Fail the refresh LOUDLY on a HF error (429/5xx) — silently returning [] would persist an empty/degraded + // fixture and mask a real regression on the next offline run. A genuinely non-existent repo returns 200 + // with an empty tree, which is handled below (empty list), not an error. + if (!res.ok) throw new Error(`HF tree fetch failed for ${repoId}: ${res.status}`); const entries: Array<{ type: string; path: string }> = await res.json(); return entries.filter(e => e.type === 'file' && e.path.endsWith('.gguf')).map(e => e.path); } diff --git a/src/services/modelManager/download.ts b/src/services/modelManager/download.ts index 23b39cf80..bad01613f 100644 --- a/src/services/modelManager/download.ts +++ b/src/services/modelManager/download.ts @@ -32,7 +32,10 @@ export function mmProjLocalName(ggufFileName: string, mmProjSourceName?: string) // keeps the link. The old `/mmproj.*$/` swallowed a model name the repo put AFTER "mmproj" // (ggml-org ships `mmproj--.gguf`), doubling the stem → the link was cleared and vision // never initialized ("Multimodal support not enabled" — device 2026-07-23, SmolVLM/Qwen2.5-VL on iOS). - const precision = mmProjSourceName?.match(/\b(iq\d+[a-z0-9_]*|q\d+[a-z0-9_]*|bf16|fp16|f16|f32)\b/i)?.[0]; + // The projector's own precision is the token immediately before `.gguf`. Anchoring to `\.gguf$` makes the + // match LINEAR (no super-linear backtracking) and inherently takes the TRAILING token, so a precision-like + // token buried in a model name can never win. + const precision = mmProjSourceName?.match(/[-_.](iq\d[a-z0-9_]*|q\d[a-z0-9_]*|bf16|fp16|f16|f32)\.gguf$/i)?.[1]; const proj = precision ? `mmproj-${precision}.gguf` : 'mmproj.gguf'; return `${base}-${proj}`; } From 8f4bfa161284981b191e89bca2da7297e0221ce9 Mon Sep 17 00:00:00 2001 From: alichherawalla Date: Fri, 24 Jul 2026 15:25:40 +0530 Subject: [PATCH 6/7] =?UTF-8?q?docs(gaps):=20A1=20fix=20verified=20iOS=20b?= =?UTF-8?q?ut=20reproduces=20on=20Android=20=E2=80=94=20merge=20held,=20ro?= =?UTF-8?q?ot-cause=20pending=20device=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/GAPS_BACKLOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/GAPS_BACKLOG.md b/docs/GAPS_BACKLOG.md index 83d1db41c..0dc4b7e23 100644 --- a/docs/GAPS_BACKLOG.md +++ b/docs/GAPS_BACKLOG.md @@ -373,3 +373,28 @@ fileExceedsBudget's boundary (size == budget: `>` vs `>=`) has no test straddlin the verifier's `>`↔`>=` mutant survived. Off-by-one-byte at the budget edge; no user-visible impact (a model exactly at the budget is a measure-zero case). Add a boundary test if fileExceedsBudget is touched again. Not fixed now (marginal, near release). + +## A1 mmproj fix — VERIFIED iOS, REPRODUCES on Android (2026-07-24) + +**Verdict: instrument-and-revisit — merge of PR #605 HELD until root-caused on Android.** + +The vision-projector fix (mmProjLocalName precision-only rename + pickMmProjForDownload shape +handling) is proven on iOS device (ggml-org SmolVLM-256M → real vision answer; log +`[WIRE-VISION] initialized:true`) and by the live-HF fixture test. But the SAME model on Android +(OnePlus CPH2707, Android 16, the fixed JS served via Metro) still throws **"Multimodal support +not enabled. Call initMultimodal first."** at generation — Christophe's exact error. + +Observed: the model downloaded (model+mmproj), showed the **Vision badge (no repair wrench)** in the +picker and a downloaded/trash state in the detail — so the app believes the mmproj is linked — yet +`initMultimodal` did not take on the loaded Android context. Points to an Android load/init or +Swift↔Kotlin download-path parity gap, NOT the shared naming logic. + +BLOCKER to root-cause: the installed `.dev` build is not debuggable (`run-as` refused), Settings has +no Debug Logs entry, logcat is empty (file-sink only) — no device trace obtainable. Per the "pull the +log, don't guess" rule, the Android fix is NOT being written speculatively. + +NEXT: install a debuggable `__DEV__` Android build (so `offgrid-debug.log` + Debug Logs exist), +reproduce, grep `[DL-SM]`/`[WIRE-VISION]`/mmProjLocalPath to see whether (a) the mmproj file is at the +`mmProjLocalName` path on disk, (b) mmProjPath is passed to the Android load, (c) initMultimodal +returns false natively. Then fix the identified seam. Decide then whether it belongs in #605 or a +separate Android-parity PR. From 02ae6cb5a1bf941e5cab94f8b12998dc28c5be16 Mon Sep 17 00:00:00 2001 From: alichherawalla Date: Fri, 24 Jul 2026 16:31:40 +0530 Subject: [PATCH 7/7] docs(gaps): A1 vision VERIFIED on both iOS + Android device; earlier Android fail was transient (load-timing watch-item, not a blocker) --- docs/GAPS_BACKLOG.md | 43 +++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/docs/GAPS_BACKLOG.md b/docs/GAPS_BACKLOG.md index 0dc4b7e23..83d8d6d45 100644 --- a/docs/GAPS_BACKLOG.md +++ b/docs/GAPS_BACKLOG.md @@ -374,27 +374,22 @@ the verifier's `>`↔`>=` mutant survived. Off-by-one-byte at the budget edge; n (a model exactly at the budget is a measure-zero case). Add a boundary test if fileExceedsBudget is touched again. Not fixed now (marginal, near release). -## A1 mmproj fix — VERIFIED iOS, REPRODUCES on Android (2026-07-24) - -**Verdict: instrument-and-revisit — merge of PR #605 HELD until root-caused on Android.** - -The vision-projector fix (mmProjLocalName precision-only rename + pickMmProjForDownload shape -handling) is proven on iOS device (ggml-org SmolVLM-256M → real vision answer; log -`[WIRE-VISION] initialized:true`) and by the live-HF fixture test. But the SAME model on Android -(OnePlus CPH2707, Android 16, the fixed JS served via Metro) still throws **"Multimodal support -not enabled. Call initMultimodal first."** at generation — Christophe's exact error. - -Observed: the model downloaded (model+mmproj), showed the **Vision badge (no repair wrench)** in the -picker and a downloaded/trash state in the detail — so the app believes the mmproj is linked — yet -`initMultimodal` did not take on the loaded Android context. Points to an Android load/init or -Swift↔Kotlin download-path parity gap, NOT the shared naming logic. - -BLOCKER to root-cause: the installed `.dev` build is not debuggable (`run-as` refused), Settings has -no Debug Logs entry, logcat is empty (file-sink only) — no device trace obtainable. Per the "pull the -log, don't guess" rule, the Android fix is NOT being written speculatively. - -NEXT: install a debuggable `__DEV__` Android build (so `offgrid-debug.log` + Debug Logs exist), -reproduce, grep `[DL-SM]`/`[WIRE-VISION]`/mmProjLocalPath to see whether (a) the mmproj file is at the -`mmProjLocalName` path on disk, (b) mmProjPath is passed to the Android load, (c) initMultimodal -returns false natively. Then fix the identified seam. Decide then whether it belongs in #605 or a -separate Android-parity PR. +## A1 mmproj fix — possible load-immediately-after-download vision race (2026-07-24) + +**Verdict: instrument-and-revisit (LOW — unconfirmed). NOT a merge blocker; PR #605 verified on both +platforms.** + +A1's fix is verified on-device on BOTH iOS and Android (ggml-org SmolVLM-256M → real vision answer; +`[WIRE-VISION] initialized:true, vision:true` on both; mmproj on disk as the correct +`smolvlm-256m-instruct-mmproj-Q8_0.gguf`, `mmProjFileExists:true`). So the naming/matching fix is +correct and complete on both platforms. + +Watch-item: during the FIRST Android attempt — a messy sequence right after download, on a +non-debuggable build, with a system notification-permission dialog intercepting taps — a vision send +threw "Multimodal support not enabled". It did NOT reproduce on the clean debuggable run (model +downloaded+linked, loaded fresh → vision:true → correct description). Possible unconfirmed cause: a +model loaded text-only if loaded in the window before its mmProjPath is persisted on the record +(load-immediately-after-download race), OR just the permission-dialog interference. If a real user +reports vision failing right after a first download, instrument the load path: assert the model +record's mmProjPath is set before the first load, and re-derive multimodal if a mmproj is linked after +a text-only load.