diff --git a/__tests__/fixtures/hf/vision-repos.json b/__tests__/fixtures/hf/vision-repos.json new file mode 100644 index 000000000..a4b1c245f --- /dev/null +++ b/__tests__/fixtures/hf/vision-repos.json @@ -0,0 +1,544 @@ +{ + "unsloth/gemma-4-E2B-it-GGUF": { + "exists": true, + "ggufFiles": [ + "MTP/mtp-gemma-4-E2B-it-BF16.gguf", + "MTP/mtp-gemma-4-E2B-it-F16.gguf", + "MTP/mtp-gemma-4-E2B-it-Q8_0.gguf", + "gemma-4-E2B-it-BF16.gguf", + "gemma-4-E2B-it-IQ4_NL.gguf", + "gemma-4-E2B-it-IQ4_XS.gguf", + "gemma-4-E2B-it-Q3_K_M.gguf", + "gemma-4-E2B-it-Q3_K_S.gguf", + "gemma-4-E2B-it-Q4_0.gguf", + "gemma-4-E2B-it-Q4_1.gguf", + "gemma-4-E2B-it-Q4_K_M.gguf", + "gemma-4-E2B-it-Q4_K_S.gguf", + "gemma-4-E2B-it-Q5_K_M.gguf", + "gemma-4-E2B-it-Q5_K_S.gguf", + "gemma-4-E2B-it-Q6_K.gguf", + "gemma-4-E2B-it-Q8_0.gguf", + "gemma-4-E2B-it-UD-IQ2_M.gguf", + "gemma-4-E2B-it-UD-IQ3_XXS.gguf", + "gemma-4-E2B-it-UD-Q2_K_XL.gguf", + "gemma-4-E2B-it-UD-Q3_K_XL.gguf", + "gemma-4-E2B-it-UD-Q4_K_XL.gguf", + "gemma-4-E2B-it-UD-Q5_K_XL.gguf", + "gemma-4-E2B-it-UD-Q6_K_XL.gguf", + "gemma-4-E2B-it-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "mmproj-F16.gguf", + "mmproj-F32.gguf", + "mtp-gemma-4-E2B-it.gguf" + ], + "probe": { + "model": { + "name": "mtp-gemma-4-E2B-it.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "unsloth/gemma-4-E4B-it-GGUF": { + "exists": true, + "ggufFiles": [ + "MTP/mtp-gemma-4-E4B-it-BF16.gguf", + "MTP/mtp-gemma-4-E4B-it-F16.gguf", + "MTP/mtp-gemma-4-E4B-it-Q8_0.gguf", + "gemma-4-E4B-it-BF16.gguf", + "gemma-4-E4B-it-IQ4_NL.gguf", + "gemma-4-E4B-it-IQ4_XS.gguf", + "gemma-4-E4B-it-Q3_K_M.gguf", + "gemma-4-E4B-it-Q3_K_S.gguf", + "gemma-4-E4B-it-Q4_0.gguf", + "gemma-4-E4B-it-Q4_1.gguf", + "gemma-4-E4B-it-Q4_K_M.gguf", + "gemma-4-E4B-it-Q4_K_S.gguf", + "gemma-4-E4B-it-Q5_K_M.gguf", + "gemma-4-E4B-it-Q5_K_S.gguf", + "gemma-4-E4B-it-Q6_K.gguf", + "gemma-4-E4B-it-Q8_0.gguf", + "gemma-4-E4B-it-UD-IQ2_M.gguf", + "gemma-4-E4B-it-UD-IQ3_XXS.gguf", + "gemma-4-E4B-it-UD-Q2_K_XL.gguf", + "gemma-4-E4B-it-UD-Q3_K_XL.gguf", + "gemma-4-E4B-it-UD-Q4_K_XL.gguf", + "gemma-4-E4B-it-UD-Q5_K_XL.gguf", + "gemma-4-E4B-it-UD-Q6_K_XL.gguf", + "gemma-4-E4B-it-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "mmproj-F16.gguf", + "mmproj-F32.gguf", + "mtp-gemma-4-E4B-it.gguf" + ], + "probe": { + "model": { + "name": "mtp-gemma-4-E4B-it.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "unsloth/Qwen3.5-0.8B-GGUF": { + "exists": true, + "ggufFiles": [ + "Qwen3.5-0.8B-BF16.gguf", + "Qwen3.5-0.8B-IQ4_NL.gguf", + "Qwen3.5-0.8B-IQ4_XS.gguf", + "Qwen3.5-0.8B-Q3_K_M.gguf", + "Qwen3.5-0.8B-Q3_K_S.gguf", + "Qwen3.5-0.8B-Q4_0.gguf", + "Qwen3.5-0.8B-Q4_1.gguf", + "Qwen3.5-0.8B-Q4_K_M.gguf", + "Qwen3.5-0.8B-Q4_K_S.gguf", + "Qwen3.5-0.8B-Q5_K_M.gguf", + "Qwen3.5-0.8B-Q5_K_S.gguf", + "Qwen3.5-0.8B-Q6_K.gguf", + "Qwen3.5-0.8B-Q8_0.gguf", + "Qwen3.5-0.8B-UD-IQ2_M.gguf", + "Qwen3.5-0.8B-UD-IQ2_XXS.gguf", + "Qwen3.5-0.8B-UD-IQ3_XXS.gguf", + "Qwen3.5-0.8B-UD-Q2_K_XL.gguf", + "Qwen3.5-0.8B-UD-Q3_K_XL.gguf", + "Qwen3.5-0.8B-UD-Q4_K_XL.gguf", + "Qwen3.5-0.8B-UD-Q5_K_XL.gguf", + "Qwen3.5-0.8B-UD-Q6_K_XL.gguf", + "Qwen3.5-0.8B-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "mmproj-F16.gguf", + "mmproj-F32.gguf" + ], + "probe": { + "model": { + "name": "Qwen3.5-0.8B-BF16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/SmolVLM2-500M-Video-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "SmolVLM2-500M-Video-Instruct-Q8_0.gguf", + "SmolVLM2-500M-Video-Instruct-f16.gguf", + "mmproj-SmolVLM2-500M-Video-Instruct-Q8_0.gguf", + "mmproj-SmolVLM2-500M-Video-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "SmolVLM2-500M-Video-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-SmolVLM2-500M-Video-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/SmolVLM-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "SmolVLM-Instruct-Q4_K_M.gguf", + "SmolVLM-Instruct-Q8_0.gguf", + "SmolVLM-Instruct-f16.gguf", + "mmproj-SmolVLM-Instruct-Q8_0.gguf", + "mmproj-SmolVLM-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "SmolVLM-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-SmolVLM-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/SmolVLM2-2.2B-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "SmolVLM2-2.2B-Instruct-Q4_K_M.gguf", + "SmolVLM2-2.2B-Instruct-Q8_0.gguf", + "SmolVLM2-2.2B-Instruct-f16.gguf", + "mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf", + "mmproj-SmolVLM2-2.2B-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "SmolVLM2-2.2B-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/SmolVLM-256M-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "SmolVLM-256M-Instruct-Q8_0.gguf", + "SmolVLM-256M-Instruct-f16.gguf", + "mmproj-SmolVLM-256M-Instruct-Q8_0.gguf", + "mmproj-SmolVLM-256M-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "SmolVLM-256M-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-SmolVLM-256M-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/SmolVLM-500M-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "SmolVLM-500M-Instruct-Q8_0.gguf", + "SmolVLM-500M-Instruct-f16.gguf", + "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf", + "mmproj-SmolVLM-500M-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "SmolVLM-500M-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/Qwen2.5-VL-3B-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf", + "Qwen2.5-VL-3B-Instruct-Q8_0.gguf", + "Qwen2.5-VL-3B-Instruct-f16.gguf", + "mmproj-Qwen2.5-VL-3B-Instruct-Q8_0.gguf", + "mmproj-Qwen2.5-VL-3B-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "Qwen2.5-VL-3B-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-Qwen2.5-VL-3B-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/Qwen2.5-VL-7B-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", + "Qwen2.5-VL-7B-Instruct-Q8_0.gguf", + "Qwen2.5-VL-7B-Instruct-f16.gguf", + "mmproj-Qwen2.5-VL-7B-Instruct-Q8_0.gguf", + "mmproj-Qwen2.5-VL-7B-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "Qwen2.5-VL-7B-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-Qwen2.5-VL-7B-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "unsloth/Qwen2.5-VL-7B-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "Qwen2.5-VL-7B-Instruct-BF16.gguf", + "Qwen2.5-VL-7B-Instruct-IQ4_NL.gguf", + "Qwen2.5-VL-7B-Instruct-IQ4_XS.gguf", + "Qwen2.5-VL-7B-Instruct-Q2_K.gguf", + "Qwen2.5-VL-7B-Instruct-Q2_K_L.gguf", + "Qwen2.5-VL-7B-Instruct-Q3_K_M.gguf", + "Qwen2.5-VL-7B-Instruct-Q3_K_S.gguf", + "Qwen2.5-VL-7B-Instruct-Q4_0.gguf", + "Qwen2.5-VL-7B-Instruct-Q4_1.gguf", + "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", + "Qwen2.5-VL-7B-Instruct-Q4_K_S.gguf", + "Qwen2.5-VL-7B-Instruct-Q5_K_M.gguf", + "Qwen2.5-VL-7B-Instruct-Q5_K_S.gguf", + "Qwen2.5-VL-7B-Instruct-Q6_K.gguf", + "Qwen2.5-VL-7B-Instruct-Q8_0.gguf", + "Qwen2.5-VL-7B-Instruct-UD-IQ1_M.gguf", + "Qwen2.5-VL-7B-Instruct-UD-IQ1_S.gguf", + "Qwen2.5-VL-7B-Instruct-UD-IQ2_M.gguf", + "Qwen2.5-VL-7B-Instruct-UD-IQ2_XXS.gguf", + "Qwen2.5-VL-7B-Instruct-UD-IQ3_XXS.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q2_K_XL.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q3_K_XL.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q4_K_XL.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q5_K_XL.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q6_K_XL.gguf", + "Qwen2.5-VL-7B-Instruct-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "mmproj-F16.gguf", + "mmproj-F32.gguf" + ], + "probe": { + "model": { + "name": "Qwen2.5-VL-7B-Instruct-BF16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/pixtral-12b-GGUF": { + "exists": true, + "ggufFiles": [ + "mmproj-pixtral-12b-Q8_0.gguf", + "mmproj-pixtral-12b-f16.gguf", + "pixtral-12b-Q2_K.gguf", + "pixtral-12b-Q4_K_M.gguf", + "pixtral-12b-Q8_0.gguf", + "pixtral-12b-f16.gguf" + ], + "probe": { + "model": { + "name": "pixtral-12b-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-pixtral-12b-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/InternVL3-2B-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "InternVL3-2B-Instruct-Q4_K_M.gguf", + "InternVL3-2B-Instruct-Q8_0.gguf", + "InternVL3-2B-Instruct-f16.gguf", + "mmproj-InternVL3-2B-Instruct-Q8_0.gguf", + "mmproj-InternVL3-2B-Instruct-f16.gguf" + ], + "probe": { + "model": { + "name": "InternVL3-2B-Instruct-f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-InternVL3-2B-Instruct-Q8_0.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/gemma-3-4b-it-GGUF": { + "exists": true, + "ggufFiles": [ + "gemma-3-4b-it-Q4_K_M.gguf", + "gemma-3-4b-it-Q8_0.gguf", + "gemma-3-4b-it-f16.gguf", + "mmproj-model-f16.gguf" + ], + "probe": { + "model": { + "name": "gemma-3-4b-it-f16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "unsloth/gemma-3-4b-it-GGUF": { + "exists": true, + "ggufFiles": [ + "gemma-3-4b-it-BF16.gguf", + "gemma-3-4b-it-IQ4_NL.gguf", + "gemma-3-4b-it-IQ4_XS.gguf", + "gemma-3-4b-it-Q2_K.gguf", + "gemma-3-4b-it-Q2_K_L.gguf", + "gemma-3-4b-it-Q3_K_M.gguf", + "gemma-3-4b-it-Q3_K_S.gguf", + "gemma-3-4b-it-Q4_0.gguf", + "gemma-3-4b-it-Q4_1.gguf", + "gemma-3-4b-it-Q4_K_M.gguf", + "gemma-3-4b-it-Q4_K_S.gguf", + "gemma-3-4b-it-Q5_K_M.gguf", + "gemma-3-4b-it-Q5_K_S.gguf", + "gemma-3-4b-it-Q6_K.gguf", + "gemma-3-4b-it-Q8_0.gguf", + "gemma-3-4b-it-UD-IQ1_M.gguf", + "gemma-3-4b-it-UD-IQ1_S.gguf", + "gemma-3-4b-it-UD-IQ2_M.gguf", + "gemma-3-4b-it-UD-IQ2_XXS.gguf", + "gemma-3-4b-it-UD-IQ3_XXS.gguf", + "gemma-3-4b-it-UD-Q2_K_XL.gguf", + "gemma-3-4b-it-UD-Q3_K_XL.gguf", + "gemma-3-4b-it-UD-Q4_K_XL.gguf", + "gemma-3-4b-it-UD-Q5_K_XL.gguf", + "gemma-3-4b-it-UD-Q6_K_XL.gguf", + "gemma-3-4b-it-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "mmproj-F16.gguf", + "mmproj-F32.gguf" + ], + "probe": { + "model": { + "name": "gemma-3-4b-it-BF16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF": { + "exists": true, + "ggufFiles": [ + "Mistral-Small-3.1-24B-Instruct-2503-UD-IQ2_M.gguf", + "mmproj-Mistral-Small-3.1-24B-Instruct-2503-Q8_0.gguf", + "mmproj-Mistral-Small-3.1-24B-Instruct-2503-f16.gguf" + ], + "probe": { + "model": { + "name": "Mistral-Small-3.1-24B-Instruct-2503-UD-IQ2_M.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "openbmb/MiniCPM-V-2_6-gguf": { + "exists": true, + "ggufFiles": [ + "ggml-model-IQ3_M.gguf", + "ggml-model-IQ3_S.gguf", + "ggml-model-IQ3_XS.gguf", + "ggml-model-IQ4_NL.gguf", + "ggml-model-IQ4_XS.gguf", + "ggml-model-Q2_K.gguf", + "ggml-model-Q3_K.gguf", + "ggml-model-Q3_K_L.gguf", + "ggml-model-Q3_K_M.gguf", + "ggml-model-Q3_K_S.gguf", + "ggml-model-Q4_0.gguf", + "ggml-model-Q4_1.gguf", + "ggml-model-Q4_K.gguf", + "ggml-model-Q4_K_M.gguf", + "ggml-model-Q4_K_S.gguf", + "ggml-model-Q5_0.gguf", + "ggml-model-Q5_1.gguf", + "ggml-model-Q5_K.gguf", + "ggml-model-Q5_K_M.gguf", + "ggml-model-Q5_K_S.gguf", + "ggml-model-Q6_K.gguf", + "ggml-model-Q8_0.gguf", + "ggml-model-f16.gguf", + "mmproj-model-f16.gguf" + ], + "probe": { + "model": { + "name": "ggml-model-f16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "moondream/moondream2-gguf": { + "exists": true, + "ggufFiles": [ + "moondream2-mmproj-f16.gguf", + "moondream2-text-model-f16.gguf" + ], + "probe": { + "model": { + "name": "moondream2-text-model-f16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "abetlen/Phi-3.5-vision-instruct-gguf": { + "exists": true, + "ggufFiles": [ + "Phi-3.5-3.8B-vision-instruct-F16.gguf", + "Phi-3.5-3.8B-vision-instruct-Q8_0.gguf", + "Phi-3.5-3.8B-vision-instruct-mmproj-F16.gguf" + ], + "probe": { + "model": { + "name": "Phi-3.5-3.8B-vision-instruct-F16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "Phi-3.5-3.8B-vision-instruct-mmproj-F16.gguf", + "status": 206, + "magic": "GGUF" + } + } + }, + "leafspark/Llama-3.2-11B-Vision-Instruct-GGUF": { + "exists": true, + "ggufFiles": [ + "Llama-3.2-11B-Vision-Instruct-mmproj.f16.gguf", + "Llama-3.2-11B-Vision-Instruct.Q4_K_M.gguf", + "Llama-3.2-11B-Vision-Instruct.Q8_0.gguf", + "Llama-3.2-11B-Vision-Instruct.f16.gguf" + ], + "probe": { + "model": { + "name": "Llama-3.2-11B-Vision-Instruct.f16.gguf", + "status": 206, + "magic": "GGUF" + }, + "mmproj": { + "name": "Llama-3.2-11B-Vision-Instruct-mmproj.f16.gguf", + "status": 206, + "magic": "GGUF" + } + } + } +} diff --git a/__tests__/integration/models/visionMmprojFromHF.test.ts b/__tests__/integration/models/visionMmprojFromHF.test.ts new file mode 100644 index 000000000..14792f782 --- /dev/null +++ b/__tests__/integration/models/visionMmprojFromHF.test.ts @@ -0,0 +1,160 @@ +/** + * A1 — vision projector pairing/rename/download, verified against REAL Hugging Face data. + * + * Design: capture once, reuse. The real HF listing + a real download probe (opening bytes of the model and + * its paired mmproj) are captured into a committed fixture (`__tests__/fixtures/hf/vision-repos.json`). Normal + * runs are FAST and OFFLINE: they replay the real product pipeline (pickMmProjForDownload → mmProjLocalName → + * mmProjBelongsToModel) over the captured real filenames, and assert the captured download probe shows a + * genuine GGUF. No per-run network, no flakiness. + * + * To refresh from live HF (the "download once" step — re-hits huggingface.co, downloads the header bytes of + * every sampled file, rewrites the fixture): + * + * UPDATE_HF_FIXTURES=1 npx jest visionMmprojFromHF + * + * The pipeline under test is the exact code that broke on device (SmolVLM/Qwen2.5-VL, iOS, 2026-07-23). + */ +import fs from 'fs'; +import path from 'path'; +import { isMMProjFile, pickMmProjForDownload, mmProjBelongsToModel } from '../../../src/services/mmproj'; +import { mmProjLocalName } from '../../../src/services/modelManager/download'; +import { RECOMMENDED_MODELS } from '../../../src/constants/models'; + +const CURATED_VISION_REPOS = RECOMMENDED_MODELS.filter(m => m.type === 'vision').map(m => m.id); +const POPULAR_VISION_REPOS = [ + 'ggml-org/SmolVLM-256M-Instruct-GGUF', // the exact device report + 'ggml-org/SmolVLM-500M-Instruct-GGUF', + 'ggml-org/Qwen2.5-VL-3B-Instruct-GGUF', + 'ggml-org/Qwen2.5-VL-7B-Instruct-GGUF', + 'unsloth/Qwen2.5-VL-7B-Instruct-GGUF', + 'ggml-org/pixtral-12b-GGUF', + 'ggml-org/InternVL3-2B-Instruct-GGUF', + 'ggml-org/gemma-3-4b-it-GGUF', + 'unsloth/gemma-3-4b-it-GGUF', + 'ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF', + 'openbmb/MiniCPM-V-2_6-gguf', + 'moondream/moondream2-gguf', + 'abetlen/Phi-3.5-vision-instruct-gguf', + 'leafspark/Llama-3.2-11B-Vision-Instruct-GGUF', +]; +const ALL_REPOS = [...new Set([...CURATED_VISION_REPOS, ...POPULAR_VISION_REPOS])]; + +const FIXTURE_PATH = path.join(__dirname, '../../fixtures/hf/vision-repos.json'); +const REFRESH = !!process.env.UPDATE_HF_FIXTURES; + +interface DownloadProbe { name: string; status: number; magic: string } +interface RepoFixture { + exists: boolean; + ggufFiles: string[]; // every .gguf filename HF publishes (real) + probe: { model?: DownloadProbe; mmproj?: DownloadProbe } | null; // real opening-bytes download +} +type Fixture = Record; + +// ---- capture (live network; only when UPDATE_HF_FIXTURES=1) ---------------------------------------- + +async function fetchTreeGgufFiles(repoId: string): Promise { + const res = await fetch(`https://huggingface.co/api/models/${repoId}/tree/main?recursive=true`, { + headers: { Accept: 'application/json' }, + }); + // Fail the refresh LOUDLY on a HF error (429/5xx) — silently returning [] would persist an empty/degraded + // fixture and mask a real regression on the next offline run. A genuinely non-existent repo returns 200 + // with an empty tree, which is handled below (empty list), not an error. + if (!res.ok) throw new Error(`HF tree fetch failed for ${repoId}: ${res.status}`); + const entries: Array<{ type: string; path: string }> = await res.json(); + return entries.filter(e => e.type === 'file' && e.path.endsWith('.gguf')).map(e => e.path); +} + +async function downloadHead(repoId: string, fileName: string): Promise { + const url = `https://huggingface.co/${repoId}/resolve/main/${fileName}`; + const controller = new AbortController(); + const res = await fetch(url, { headers: { Range: 'bytes=0-3' }, signal: controller.signal }); + let magic = ''; + if (res.body) { + const { value } = await res.body.getReader().read(); + if (value) magic = Buffer.from(value).subarray(0, 4).toString('latin1'); + } + controller.abort(); // never pull the whole (multi-GB) file + return { name: fileName, status: res.status, magic }; +} + +async function captureFixture(): Promise { + const out: Fixture = {}; + for (const repoId of ALL_REPOS) { + const ggufFiles = await fetchTreeGgufFiles(repoId); + const projectors = ggufFiles.filter(isMMProjFile); + const models = ggufFiles.filter(f => !isMMProjFile(f)).sort((a, b) => a.length - b.length); + let probe: RepoFixture['probe'] = null; + const sampleModel = models.find(m => pickMmProjForDownload(m, projectors)) ?? models[0]; + if (sampleModel) { + const chosen = pickMmProjForDownload(sampleModel, projectors); + const model = await downloadHead(repoId, sampleModel); + const mmproj = chosen ? await downloadHead(repoId, chosen) : undefined; + probe = { model, mmproj }; + } + out[repoId] = { exists: ggufFiles.length > 0, ggufFiles, probe }; + } + return out; +} + +// ---- the fixture (captured real HF data) ----------------------------------------------------------- + +function loadFixture(): Fixture | null { + return fs.existsSync(FIXTURE_PATH) ? JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')) : null; +} + +// A vision repo passes when the app pairs a projector to at least one model AND the on-disk rename of that +// projector still belongs to the model (so the link survives → initMultimodal runs). +function repoPairsAndBelongs(fx: RepoFixture): { pairs: boolean; belongs: boolean } { + const projectors = fx.ggufFiles.filter(isMMProjFile); + const models = fx.ggufFiles.filter(f => !isMMProjFile(f)); + const pairedModels = models.filter(m => pickMmProjForDownload(m, projectors)); + if (pairedModels.length === 0) return { pairs: false, belongs: false }; + const belongs = pairedModels.every(m => { + const chosen = pickMmProjForDownload(m, projectors)!; + return mmProjBelongsToModel(m, mmProjLocalName(m, chosen)); + }); + return { pairs: true, belongs }; +} + +(REFRESH ? describe : describe.skip)('refresh HF vision fixtures (live network — UPDATE_HF_FIXTURES=1)', () => { + it('captures the real listing + a real download probe for every vision repo', async () => { + const fixture = await captureFixture(); + fs.mkdirSync(path.dirname(FIXTURE_PATH), { recursive: true }); + fs.writeFileSync(FIXTURE_PATH, `${JSON.stringify(fixture, null, 2)}\n`); + expect(Object.keys(fixture)).toEqual(ALL_REPOS); + }, 300000); +}); + +describe('vision mmproj pairing/rename/download over cached REAL Hugging Face data', () => { + const fx = loadFixture(); + + it('the HF fixture exists (run UPDATE_HF_FIXTURES=1 to capture it)', () => { + expect(fx).not.toBeNull(); + }); + + it.each(CURATED_VISION_REPOS)( + 'curated %s: pairs a projector, its renamed on-disk name belongs, and both files are real GGUF on HF', + (repoId) => { + const repo = fx?.[repoId]; + expect(repo?.exists).toBe(true); + const { pairs, belongs } = repoPairsAndBelongs(repo!); + expect({ repoId, pairs, belongs }).toEqual({ repoId, pairs: true, belongs: true }); + // the captured real download probe proves the model + its projector are genuine, present GGUF files + expect(repo!.probe?.model?.magic).toBe('GGUF'); + expect(repo!.probe?.mmproj?.magic).toBe('GGUF'); + }, + ); + + it('EVERY examined vision repo (curated + popular, all naming shapes) pairs a projector that belongs', () => { + // Was a "known-open" ledger for the no-pairing shapes (mmproj-model literal, moondream infixed + // mmproj + -text-model weights, unsloth UD- quant packaging). All are fixed in mmproj.ts now, so the + // invariant is stronger: NO real vision repo is left unpaired or with a mismatched on-disk name. + const broken = ALL_REPOS.filter(id => { + const repo = fx?.[id]; + if (!repo?.exists) return false; + const { pairs, belongs } = repoPairsAndBelongs(repo); + return !pairs || !belongs; + }).sort(); + expect(broken).toEqual([]); + }); +}); diff --git a/__tests__/unit/services/parallelMmproj.test.ts b/__tests__/unit/services/parallelMmproj.test.ts index f140067fc..e8b313079 100644 --- a/__tests__/unit/services/parallelMmproj.test.ts +++ b/__tests__/unit/services/parallelMmproj.test.ts @@ -17,7 +17,7 @@ import { syncCompletedBackgroundDownloads, mmProjLocalName, } from '../../../src/services/modelManager/download'; -import { mmProjBelongsToModel } from '../../../src/services/mmproj'; +import { mmProjBelongsToModel, pickMmProjForDownload } from '../../../src/services/mmproj'; import { restoreInProgressDownloads } from '../../../src/services/modelManager/restore'; import { backgroundDownloadService } from '../../../src/services/backgroundDownloadService'; import { BackgroundDownloadContext } from '../../../src/services/modelManager/types'; @@ -1276,4 +1276,48 @@ describe('mmProjLocalName — one shared projector per model (dedup across quant expect(mmProjBelongsToModel('gemma-4-E2B-it-Q8_0.gguf', name)).toBe(true); expect(mmProjBelongsToModel('gemma-4-E4B-it-Q8_0.gguf', name)).toBe(false); }); + + // A1 (device 2026-07-23, iOS): ggml-org names projectors `mmproj--.gguf` — the model name + // sits AFTER the "mmproj" token. The old `/mmproj.*$/` kept that model name, doubling the on-disk stem so + // mmProjBelongsToModel refused it → link cleared → "Multimodal support not enabled". Offline guard so the + // regression is caught without the network (the live-HF proof is the .network integration test). + it('ggml-org shape: model name AFTER the mmproj token is dropped, on-disk name still belongs (SmolVLM/Qwen-VL)', () => { + const smol = mmProjLocalName('SmolVLM-256M-Instruct-Q8_0.gguf', 'mmproj-SmolVLM-256M-Instruct-Q8_0.gguf'); + expect(smol).toBe('smolvlm-256m-instruct-mmproj-Q8_0.gguf'); + expect(mmProjBelongsToModel('SmolVLM-256M-Instruct-Q8_0.gguf', smol)).toBe(true); + + const qwen = mmProjLocalName('Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen2.5-VL-3B-Instruct-f16.gguf'); + expect(qwen).toBe('qwen2.5-vl-3b-instruct-mmproj-f16.gguf'); + expect(mmProjBelongsToModel('Qwen2.5-VL-3B-Instruct-Q8_0.gguf', qwen)).toBe(true); + + // every quant of the ggml-org model still shares ONE projector file (quant-independent dedup preserved) + expect(mmProjLocalName('SmolVLM-256M-Instruct-Q8_0.gguf', 'mmproj-SmolVLM-256M-Instruct-f16.gguf')) + .toBe(mmProjLocalName('SmolVLM-256M-Instruct-Q4_K_M.gguf', 'mmproj-SmolVLM-256M-Instruct-f16.gguf')); + }); +}); + +// A1 no-pairing shapes (found in the live-HF sweep) — pickMmProjForDownload must pair these too. +describe('pickMmProjForDownload — additional real-world projector shapes', () => { + it('literal `mmproj-model-` placeholder is generic (ggml-org gemma-3, openbmb MiniCPM)', () => { + expect(pickMmProjForDownload('gemma-3-4b-it-Q4_K_M.gguf', ['mmproj-model-f16.gguf'])) + .toBe('mmproj-model-f16.gguf'); + expect(pickMmProjForDownload('ggml-model-Q4_K_M.gguf', ['mmproj-model-f16.gguf'])) + .toBe('mmproj-model-f16.gguf'); + }); + + it('unsloth UD- dynamic-quant packaging is stripped so the model matches its non-UD projector', () => { + expect(mmProjBelongsToModel( + 'Mistral-Small-3.1-24B-Instruct-2503-UD-IQ2_M.gguf', + 'mmproj-Mistral-Small-3.1-24B-Instruct-2503-Q8_0.gguf', + )).toBe(true); + }); + + it('moondream `-text-model` weights match the `-mmproj` projector of the same base', () => { + expect(mmProjBelongsToModel('moondream2-text-model-f16.gguf', 'moondream2-mmproj-f16.gguf')).toBe(true); + }); + + it('still REFUSES a genuinely different model+variant (no over-loosening: E2B ≠ E4B)', () => { + expect(pickMmProjForDownload('gemma-4-E2B-it-Q4_K_M.gguf', ['gemma-4-E4B-it-mmproj-F16.gguf'])) + .toBeUndefined(); + }); }); diff --git a/docs/GAPS_BACKLOG.md b/docs/GAPS_BACKLOG.md index 83d1db41c..83d8d6d45 100644 --- a/docs/GAPS_BACKLOG.md +++ b/docs/GAPS_BACKLOG.md @@ -373,3 +373,23 @@ fileExceedsBudget's boundary (size == budget: `>` vs `>=`) has no test straddlin the verifier's `>`↔`>=` mutant survived. Off-by-one-byte at the budget edge; no user-visible impact (a model exactly at the budget is a measure-zero case). Add a boundary test if fileExceedsBudget is touched again. Not fixed now (marginal, near release). + +## A1 mmproj fix — possible load-immediately-after-download vision race (2026-07-24) + +**Verdict: instrument-and-revisit (LOW — unconfirmed). NOT a merge blocker; PR #605 verified on both +platforms.** + +A1's fix is verified on-device on BOTH iOS and Android (ggml-org SmolVLM-256M → real vision answer; +`[WIRE-VISION] initialized:true, vision:true` on both; mmproj on disk as the correct +`smolvlm-256m-instruct-mmproj-Q8_0.gguf`, `mmProjFileExists:true`). So the naming/matching fix is +correct and complete on both platforms. + +Watch-item: during the FIRST Android attempt — a messy sequence right after download, on a +non-debuggable build, with a system notification-permission dialog intercepting taps — a vision send +threw "Multimodal support not enabled". It did NOT reproduce on the clean debuggable run (model +downloaded+linked, loaded fresh → vision:true → correct description). Possible unconfirmed cause: a +model loaded text-only if loaded in the window before its mmProjPath is persisted on the record +(load-immediately-after-download race), OR just the permission-dialog interference. If a real user +reports vision failing right after a first download, instrument the load path: assert the model +record's mmProjPath is set before the first load, and re-derive multimodal if a mmproj is linked after +a text-only load. diff --git a/src/services/mmproj.ts b/src/services/mmproj.ts index 11064bb22..6c39c31f9 100644 --- a/src/services/mmproj.ts +++ b/src/services/mmproj.ts @@ -24,13 +24,21 @@ export function isMMProjFile(fileName: string): boolean { * The model-identity stem of a gguf/mmproj filename: lowercased, with the extension, any `mmproj` marker, * and the QUANTIZATION token removed — so name + variant remain and quant is ignored. Both * gemma-4-E2B-it-Q4_K_M.gguf and gemma-4-E2B-it-Q8_0-mmproj.gguf reduce to `gemma4e2bit`; E4B reduces to - * `gemma4e4bit` (distinct). Packaging/variant suffixes (UD, instruct, …) are kept, so they must match too. + * `gemma4e4bit` (distinct). Model-VARIANT tokens (E2B/E4B, instruct, …) are kept, so they must match too; + * only quant packaging (the `UD-` dynamic-quant prefix) and the `-text-model` role marker are normalized out. */ function modelIdentityStem(fileName: string): string { return fileName .toLowerCase() .replace(/\.gguf$/, '') .replace(/[-_.]?mmproj/g, '') + // moondream ships its LLM weights as `-text-model-` alongside `-mmproj-`; + // drop the `-text-model` role marker so the weights and projector reduce to the same stem. + .replace(/[-_.]text[-_]?model/g, '') + // unsloth-DYNAMIC quant packaging is `UD-` (e.g. `-UD-IQ2_M`). The `UD` is quant packaging, + // NOT a model variant — strip it (only when it directly precedes a quant token) so a UD-quant model + // reduces to the same stem as its (non-UD) projector. + .replace(/[-_.]ud(?=[-_.](?:iq\d|q\d|f\d|fp16|bf16))/gi, '') // quant / precision tokens: Q4_K_M, Q8_0, Q5_K_S, Q6_K, IQ4_XS, F16, FP16, F32, BF16, … .replace(/[-_.]?(iq\d+[a-z0-9_]*|q\d+[a-z0-9_]*|fp16|f16|f32|bf16)/gi, '') .replace(/[^a-z0-9]+/g, ''); @@ -85,10 +93,15 @@ export function pickMmProjForDownload( const exact = candidateNames.find(name => modelIdentityStem(name) === modelStem); if (exact) return exact; - // (A) Projectors with NO model-name token (empty identity stem) are generic to the repo's single model. - // Prefer F16 (not BF16), else the first. A NAMED-but-different projector is excluded here, so it can - // never be chosen as the generic fallback (case B refusal). - const generic = candidateNames.filter(name => modelIdentityStem(name) === ''); + // (A) GENERIC projectors are generic to the repo's single model: either NO model-name token (empty + // identity stem, e.g. unsloth `mmproj-F16`) OR the literal placeholder token `model` (ggml-org gemma-3 / + // openbmb MiniCPM ship `mmproj-model-`, whose stem reduces to `model` — a placeholder, not a real + // model name). Prefer F16 (not BF16), else the first. A NAMED-but-different projector is excluded here, so + // it can never be chosen as the generic fallback (case B refusal). + const generic = candidateNames.filter(name => { + const stem = modelIdentityStem(name); + return stem === '' || stem === 'model'; + }); if (generic.length > 0) { const f16 = generic.find(name => { const lower = name.toLowerCase(); diff --git a/src/services/modelManager/download.ts b/src/services/modelManager/download.ts index 02fcbb153..bad01613f 100644 --- a/src/services/modelManager/download.ts +++ b/src/services/modelManager/download.ts @@ -24,11 +24,19 @@ import { extractBaseName } from './scan'; */ export function mmProjLocalName(ggufFileName: string, mmProjSourceName?: string): string { const base = extractBaseName(ggufFileName); - // Keep the projector's OWN precision (mmproj-F16 / mmproj-BF16) from its source filename, but strip any - // model prefix the repo added — so the on-disk name is `-mmproj-.gguf`: - // - model-prefixed → two repos that both ship `mmproj-F16.gguf` can't collide on disk (the original loss), + // Build the on-disk name as `-mmproj-.gguf`, keeping ONLY the projector's own + // precision token from its source name and dropping every model-name token wherever it sits. + // - `` prefix → two repos that both ship `mmproj-F16.gguf` can't collide on disk. // - MODEL-quant-independent (extractBaseName drops the model quant) → every quant shares ONE projector file. - const proj = mmProjSourceName?.match(/mmproj.*$/i)?.[0] ?? 'mmproj.gguf'; + // - precision-only projector portion → the on-disk stem always matches the model, so mmProjBelongsToModel + // keeps the link. The old `/mmproj.*$/` swallowed a model name the repo put AFTER "mmproj" + // (ggml-org ships `mmproj--.gguf`), doubling the stem → the link was cleared and vision + // never initialized ("Multimodal support not enabled" — device 2026-07-23, SmolVLM/Qwen2.5-VL on iOS). + // The projector's own precision is the token immediately before `.gguf`. Anchoring to `\.gguf$` makes the + // match LINEAR (no super-linear backtracking) and inherently takes the TRAILING token, so a precision-like + // token buried in a model name can never win. + const precision = mmProjSourceName?.match(/[-_.](iq\d[a-z0-9_]*|q\d[a-z0-9_]*|bf16|fp16|f16|f32)\.gguf$/i)?.[1]; + const proj = precision ? `mmproj-${precision}.gguf` : 'mmproj.gguf'; return `${base}-${proj}`; }