From 624f13f4aec9be83290ef6a7ef2c7751634fc2dd Mon Sep 17 00:00:00 2001 From: Ivan Pleshkov Date: Thu, 2 Apr 2026 14:05:08 +0200 Subject: [PATCH 01/31] turbo quant rest experiments --- .../configurations/qdrant-turbo-quant.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 experiments/configurations/qdrant-turbo-quant.json diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json new file mode 100644 index 000000000..c8dba8945 --- /dev/null +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -0,0 +1,17 @@ +[ + { + "name": "qdrant-turbo-quant-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { "turbo_quant": { "always_ram": true } } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + } +] From 69eb6d53b5bb0059c8f8406629625e683f80b50d Mon Sep 17 00:00:00 2001 From: Ivan Pleshkov Date: Sun, 5 Apr 2026 17:50:39 +0200 Subject: [PATCH 02/31] cascade tq experiments --- .../configurations/qdrant-turbo-quant.json | 215 +++++++++++++++++- 1 file changed, 211 insertions(+), 4 deletions(-) diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json index c8dba8945..118f933d4 100644 --- a/experiments/configurations/qdrant-turbo-quant.json +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -1,16 +1,223 @@ [ { - "name": "qdrant-turbo-quant-m-16-ef-128", + "name": "qdrant-tq4-m-16-ef-128", "engine": "qdrant_native", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, "hnsw_config": { "m": 16, "ef_construct": 128 }, - "quantization_config": { "turbo_quant": { "always_ram": true } } + "quantization_config": { + "turbo_quant": { + "correction": "normalization", + "bits": 4, + "plus": false, + "always_ram": true + } + } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-tq4-plus-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "turbo_quant": { + "correction": "normalization", + "bits": 4, + "plus": true, + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-tq2-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "turbo_quant": { + "correction": "normalization", + "bits": 2, + "plus": false, + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-tq2-plus-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "turbo_quant": { + "correction": "normalization", + "bits": 2, + "plus": true, + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-tq1-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "turbo_quant": { + "correction": "normalization", + "bits": 1, + "plus": false, + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-tq1-plus-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "turbo_quant": { + "correction": "normalization", + "bits": 1, + "plus": true, + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-sq-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "scalar": { + "type": "int8", + "quantile": 0.99, + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-bq-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "binary": { + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-bq2-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "binary": { + "encoding": "two_bits", + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-bq-asym-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "binary": { + "query_encoding": "scalar8bits", + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } } From a01260ee7388ce7aded872dad25c612a72577c46 Mon Sep 17 00:00:00 2001 From: Ivan Pleshkov Date: Sun, 5 Apr 2026 23:53:41 +0200 Subject: [PATCH 03/31] one more test --- experiments/configurations/qdrant-turbo-quant.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json index 118f933d4..75b4bd674 100644 --- a/experiments/configurations/qdrant-turbo-quant.json +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -220,5 +220,18 @@ { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64 } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } } ] From 3967876118cb60cfd389b87a231d6765e41fad0b Mon Sep 17 00:00:00 2001 From: tellet-q Date: Mon, 6 Apr 2026 00:26:00 +0200 Subject: [PATCH 04/31] Mount experiments into container --- tools/run_client_script.sh | 2 ++ tools/run_experiment.sh | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/tools/run_client_script.sh b/tools/run_client_script.sh index f48461362..1fe7ac2e6 100644 --- a/tools/run_client_script.sh +++ b/tools/run_client_script.sh @@ -36,6 +36,7 @@ GHCR_USERNAME=${GHCR_USERNAME:-""} if [[ "$EXPERIMENT_MODE" == "snapshot" ]]; then scp_with_retry "${SCRIPT_PATH}/run_experiment.sh" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/run_experiment_snapshot.sh" scp_with_retry "${SCRIPT_PATH}/../datasets/datasets.json" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/datasets.json" + scp_with_retry -r "${SCRIPT_PATH}/../experiments/configurations" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/configurations" RUN_EXPERIMENT="ENGINE_NAME=${ENGINE_NAME} \ DATASETS=${DATASETS} \ @@ -52,6 +53,7 @@ if [[ "$EXPERIMENT_MODE" == "snapshot" ]]; then else scp_with_retry "${SCRIPT_PATH}/run_experiment.sh" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/run_experiment.sh" scp_with_retry "${SCRIPT_PATH}/../datasets/datasets.json" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/datasets.json" + scp_with_retry -r "${SCRIPT_PATH}/../experiments/configurations" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/configurations" RUN_EXPERIMENT="ENGINE_NAME=${ENGINE_NAME} \ DATASETS=${DATASETS} \ diff --git a/tools/run_experiment.sh b/tools/run_experiment.sh index 52e08620a..21fff4c87 100644 --- a/tools/run_experiment.sh +++ b/tools/run_experiment.sh @@ -57,6 +57,13 @@ if [[ "$EXPERIMENT_MODE" != "snapshot" ]]; then docker rmi --force "${VECTOR_DB_BENCHMARK_IMAGE}" || true fi +# Mount custom configurations if available +CONFIGURATIONS_MOUNT="" +if [[ -d "$HOME/configurations" ]]; then + echo "Found custom configurations directory, will mount into container" + CONFIGURATIONS_MOUNT="$HOME/configurations:/code/experiments/configurations" +fi + echo "Ensure datasets volume exists and contains latest datasets.json" docker volume create ci-datasets if [[ -f "$HOME/datasets.json" ]]; then @@ -74,6 +81,7 @@ if [[ "$EXPERIMENT_MODE" == "full" ]] || [[ "$EXPERIMENT_MODE" == "upload" ]]; t --name ci-benchmark-upload \ -v "$HOME/results:/code/results" \ -v "ci-datasets:/code/datasets" \ + ${CONFIGURATIONS_MOUNT:+"-v" "$CONFIGURATIONS_MOUNT"} \ "${VECTOR_DB_BENCHMARK_IMAGE}" \ python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-search fi @@ -93,6 +101,7 @@ if [[ "$EXPERIMENT_MODE" == "full" ]] || [[ "$EXPERIMENT_MODE" == "search" ]]; t --name ci-benchmark-search \ -v "$HOME/results:/code/results" \ -v "ci-datasets:/code/datasets" \ + ${CONFIGURATIONS_MOUNT:+"-v" "$CONFIGURATIONS_MOUNT"} \ "${VECTOR_DB_BENCHMARK_IMAGE}" \ python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-upload fi @@ -109,6 +118,7 @@ if [[ "$EXPERIMENT_MODE" == "parallel" ]]; then --name ci-benchmark-upload \ -v "$HOME/results/parallel:/code/results" \ -v "ci-datasets:/code/datasets" \ + ${CONFIGURATIONS_MOUNT:+"-v" "$CONFIGURATIONS_MOUNT"} \ "${VECTOR_DB_BENCHMARK_IMAGE}" \ python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-search --skip-configure & UPLOAD_PID=$! @@ -119,6 +129,7 @@ if [[ "$EXPERIMENT_MODE" == "parallel" ]]; then --name ci-benchmark-search \ -v "$HOME/results/parallel:/code/results" \ -v "ci-datasets:/code/datasets" \ + ${CONFIGURATIONS_MOUNT:+"-v" "$CONFIGURATIONS_MOUNT"} \ "${VECTOR_DB_BENCHMARK_IMAGE}" \ python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-upload & SEARCH_PID=$! From 8729b6a90cc029fa9c8326eb2bdba057989473f1 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Tue, 7 Apr 2026 08:51:16 +0200 Subject: [PATCH 05/31] Fix config name and jupyter notebook --- .../configurations/qdrant-turbo-quant.json | 2 +- scripts/process-benchmarks.ipynb | 155 ++---------------- 2 files changed, 11 insertions(+), 146 deletions(-) diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json index 75b4bd674..63c998361 100644 --- a/experiments/configurations/qdrant-turbo-quant.json +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -222,7 +222,7 @@ "upload_params": { "parallel": 16, "batch_size": 1024 } }, { - "name": "qdrant-m-16-ef-128", + "name": "qdrant-turbo-m-16-ef-128", "engine": "qdrant_native", "connection_params": { "timeout": 30 }, "collection_params": { diff --git a/scripts/process-benchmarks.ipynb b/scripts/process-benchmarks.ipynb index 8727f8f84..48e60910f 100644 --- a/scripts/process-benchmarks.ipynb +++ b/scripts/process-benchmarks.ipynb @@ -62,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-08-05T10:03:51.482299Z", @@ -73,21 +73,11 @@ } }, "outputs": [], - "source": [ - "PATH_REGEX = re.compile(r\"(?P(\"\n", - " r\"?P[a-z\\-]+)\"\n", - " r\"\\-m\\-(?P[0-9]+)\"\n", - " r\"\\-ef\\-(?P[0-9]+)\"\n", - " r\")\"\n", - " r\"\\-(?P[a-zA-Z0-9\\-]+)\"\n", - " r\"\\-(?P(search)|(upload))\"\n", - " r\"(\\-(?P[0-9]{1,2})\\-)?\"\n", - " r\"\\-?(?P.*)\\.json\")" - ] + "source": "PATH_REGEX = re.compile(r\"(?P(\"\n r\"?P[a-z0-9\\-]+)\"\n r\"\\-m\\-(?P[0-9]+)\"\n r\"\\-ef\\-(?P[0-9]+)\"\n r\")\"\n r\"\\-(?P[a-zA-Z0-9\\-]+)\"\n r\"\\-(?P(search)|(upload))\"\n r\"(\\-(?P[0-9]{1,2})\\-)?\"\n r\"\\-?(?P.*)\\.json\")" }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-08-05T10:03:54.150582Z", @@ -97,55 +87,8 @@ "name": "#%%\n" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "(0, 0)" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "upload_results, search_results = [], []\n", - "\n", - "for path in DATA_DIR.glob(\"*.json\"):\n", - " match = PATH_REGEX.match(path.name)\n", - " if match is None:\n", - " continue\n", - "\n", - " experiment = match.groupdict()\n", - "\n", - " with open(path, \"r\") as fp:\n", - " stats = json.load(fp)\n", - "\n", - " params = stats[\"params\"]\n", - " dataset = params.pop(\"dataset\")\n", - " engine = params.pop(\"engine\")\n", - "\n", - " entry = {\n", - " \"dataset\": dataset,\n", - " \"engine\": engine,\n", - " \"m\": match[\"m\"],\n", - " \"ef\": match[\"ef\"],\n", - " \"date\": match[\"date\"],\n", - " \"params\": params,\n", - " \"results\": stats[\"results\"],\n", - " }\n", - "\n", - " if experiment[\"operation\"] == \"search\":\n", - " entry.update({\"search_index\": match[\"search_index\"]})\n", - " search_results.append(entry)\n", - " elif experiment[\"operation\"] == \"upload\":\n", - " upload_results.append(entry)\n", - " else:\n", - " raise Exception(\"Unknown operation\")\n", - "\n", - "len(upload_results), len(search_results)" - ] + "outputs": [], + "source": "upload_results, search_results = [], []\n\nfor path in DATA_DIR.glob(\"*.json\"):\n match = PATH_REGEX.match(path.name)\n if match is None:\n continue\n\n experiment = match.groupdict()\n\n with open(path, \"r\") as fp:\n stats = json.load(fp)\n\n params = stats[\"params\"]\n dataset = params.pop(\"dataset\")\n engine = params.pop(\"engine\")\n\n entry = {\n \"dataset\": dataset,\n \"engine\": engine,\n \"engine_name\": match[\"engine_name\"],\n \"m\": match[\"m\"],\n \"ef\": match[\"ef\"],\n \"date\": match[\"date\"],\n \"params\": params,\n \"results\": stats[\"results\"],\n }\n\n if experiment[\"operation\"] == \"search\":\n entry.update({\"search_index\": match[\"search_index\"]})\n search_results.append(entry)\n elif experiment[\"operation\"] == \"upload\":\n upload_results.append(entry)\n else:\n raise Exception(\"Unknown operation\")\n\nlen(upload_results), len(search_results)" }, { "cell_type": "code", @@ -181,17 +124,7 @@ } }, "outputs": [], - "source": [ - "upload_df = pd.DataFrame(upload_results)\n", - "upload_df[\"date\"] = pd.to_datetime(upload_df[\"date\"], format=\"%Y-%m-%d-%H-%M-%S\")\n", - "upload_df = upload_df.sort_values(\"date\", ascending=False) \\\n", - " .groupby([\"engine\", \"m\", \"ef\", \"dataset\"]) \\\n", - " .first()\n", - "\n", - "temp_df = upload_df.copy()\n", - "temp_df[\"total_time\"] = temp_df[\"results\"].apply(lambda x: x[\"total_time\"])\n", - "temp_df.sort_values(\"total_time\", ascending=True).head(n=5)" - ] + "source": "upload_df = pd.DataFrame(upload_results)\nupload_df[\"date\"] = pd.to_datetime(upload_df[\"date\"], format=\"%Y-%m-%d-%H-%M-%S\")\nupload_df = upload_df.sort_values(\"date\", ascending=False) \\\n .groupby([\"engine_name\", \"engine\", \"m\", \"ef\", \"dataset\"]) \\\n .first()\n\ntemp_df = upload_df.copy()\ntemp_df[\"total_time\"] = temp_df[\"results\"].apply(lambda x: x[\"total_time\"])\ntemp_df.sort_values(\"total_time\", ascending=True).head(n=5)" }, { "cell_type": "code", @@ -206,89 +139,21 @@ } }, "outputs": [], - "source": [ - "search_df = pd.DataFrame(search_results)\n", - "search_df[\"date\"] = pd.to_datetime(search_df[\"date\"], format=\"%Y-%m-%d-%H-%M-%S\")\n", - "search_df = search_df.sort_values(\"date\", ascending=False) \\\n", - " .groupby([\"engine\", \"m\", \"ef\", \"dataset\", \"search_index\"]) \\\n", - " .first()\n", - "\n", - "temp_df = search_df.copy()\n", - "temp_df['rps'] = temp_df['results'].apply(lambda x: x[\"rps\"])\n", - "temp_df.sort_values(\"rps\", ascending=False).head(n=10)" - ] + "source": "search_df = pd.DataFrame(search_results)\nsearch_df[\"date\"] = pd.to_datetime(search_df[\"date\"], format=\"%Y-%m-%d-%H-%M-%S\")\nsearch_df = search_df.sort_values(\"date\", ascending=False) \\\n .groupby([\"engine_name\", \"engine\", \"m\", \"ef\", \"dataset\", \"search_index\"]) \\\n .first()\n\ntemp_df = search_df.copy()\ntemp_df['rps'] = temp_df['results'].apply(lambda x: x[\"rps\"])\ntemp_df.sort_values(\"rps\", ascending=False).head(n=10)" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "_search = search_df.reset_index()\n", - "_upload = upload_df.reset_index()\n", - "\n", - "joined_df = _search.merge(_upload, on=[\"engine\", \"m\", \"ef\", \"dataset\"], how=\"left\", suffixes=(\"_search\", \"_upload\"))\n", - "print(len(joined_df))\n", - "joined_df" - ] + "source": "_search = search_df.reset_index()\n_upload = upload_df.reset_index()\n\njoined_df = _search.merge(_upload, on=[\"engine_name\", \"engine\", \"m\", \"ef\", \"dataset\"], how=\"left\", suffixes=(\"_search\", \"_upload\"))\nprint(len(joined_df))\njoined_df" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "json_results = []\n", - "\n", - "for index, row in joined_df.reset_index().iterrows():\n", - " engine_params = {}\n", - " \n", - " if isinstance(row['params_upload'], dict):\n", - " engine_params.update(row['params_upload'])\n", - " if isinstance(row['params_search'], dict):\n", - " search_params = row['params_search']\n", - " engine_params.update(search_params.get('config', {}))\n", - " engine_params.update(search_params.get('params', {}))\n", - " engine_params.update(search_params.get('search_params', {}))\n", - " engine_params.update(search_params.get('vectorIndexConfig', {}))\n", - "\n", - " engine_params.pop('experiment')\n", - " engine_params.pop('parallel')\n", - "\n", - " engine_name = row['engine']\n", - "\n", - " if engine_name.startswith(\"qdrant-\"):\n", - " engine_name = \"qdrant\"\n", - "\n", - " json_object = {\n", - " \"engine_name\": engine_name,\n", - " \"setup_name\": f\"{row['params_search']['experiment']}\",\n", - " \"dataset_name\": row['dataset'],\n", - " \"search_idx\": row['search_index'],\n", - " \"upload_time\": row['results_upload']['upload_time'],\n", - " \"total_upload_time\": row['results_upload']['total_time'],\n", - " \"p95_time\": row['results_search']['p95_time'],\n", - " \"rps\": row['results_search']['rps'],\n", - " \"parallel\": row['params_search']['parallel'],\n", - " \"p99_time\": row['results_search']['p99_time'],\n", - " \"mean_time\": row['results_search']['mean_time'],\n", - " \"mean_precisions\": row['results_search']['mean_precisions'],\n", - " \"engine_params\": engine_params,\n", - " }\n", - " json_results.append(json_object)\n", - "\n", - "format = '%Y-%M-%dT%H:%M:%S'\n", - "now = datetime.now().replace(tzinfo=timezone.utc).strftime(format)\n", - "\n", - "Path(\"results.json\").write_text(json.dumps(json_results, indent=2))\n", - "Path(f\"results-{now}.json\").write_text(json.dumps(json_results, indent=2))\n", - "\n", - "print(json_results[-1], len(json_results))\n", - "\n", - "results_df = pd.DataFrame(json_results).sort_values(\"p99_time\", ascending=True)\n", - "# results_df.to_csv('results.csv')\n", - "results_df" - ] + "source": "json_results = []\n\nfor index, row in joined_df.reset_index().iterrows():\n engine_params = {}\n \n if isinstance(row['params_upload'], dict):\n engine_params.update(row['params_upload'])\n if isinstance(row['params_search'], dict):\n search_params = row['params_search']\n engine_params.update(search_params.get('config', {}))\n engine_params.update(search_params.get('params', {}))\n engine_params.update(search_params.get('search_params', {}))\n engine_params.update(search_params.get('vectorIndexConfig', {}))\n\n engine_params.pop('experiment')\n engine_params.pop('parallel')\n\n engine_name = row['engine']\n\n if engine_name.startswith(\"qdrant-\"):\n engine_name = \"qdrant\"\n\n json_object = {\n \"engine_name\": engine_name,\n \"setup_name\": f\"{row['params_search']['experiment']}\",\n \"dataset_name\": row['dataset'],\n \"search_idx\": row['search_index'],\n \"upload_time\": row['results_upload']['upload_time'],\n \"total_upload_time\": row['results_upload']['total_time'],\n \"p95_time\": row['results_search']['p95_time'],\n \"rps\": row['results_search']['rps'],\n \"parallel\": row['params_search']['parallel'],\n \"p99_time\": row['results_search']['p99_time'],\n \"mean_time\": row['results_search']['mean_time'],\n \"mean_precisions\": row['results_search']['mean_precisions'],\n \"engine_params\": engine_params,\n }\n json_results.append(json_object)\n\nformat = '%Y-%M-%dT%H:%M:%S'\nnow = datetime.now().replace(tzinfo=timezone.utc).strftime(format)\n\nPath(\"results.json\").write_text(json.dumps(json_results, indent=2))\nPath(f\"results-{now}.json\").write_text(json.dumps(json_results, indent=2))\n\nprint(json_results[-1], len(json_results))\n\nresults_df = pd.DataFrame(json_results).sort_values(\"p99_time\", ascending=True)\n# results_df.to_csv('results.csv')\nresults_df" } ], "metadata": { @@ -312,4 +177,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file From b211735dcabadc322bdc11d157e90cc81d7d664a Mon Sep 17 00:00:00 2001 From: tellet-q Date: Tue, 7 Apr 2026 10:23:07 +0200 Subject: [PATCH 06/31] Add option to split by dataset to plots.html --- scripts/plots.html | 250 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 243 insertions(+), 7 deletions(-) diff --git a/scripts/plots.html b/scripts/plots.html index c0190a270..a35b12b50 100644 --- a/scripts/plots.html +++ b/scripts/plots.html @@ -129,7 +129,7 @@

Data Summary:

-
+
+ +
+ + +
@@ -192,14 +203,14 @@

Filters

- -
+ +

Performance Chart

- 🔍 Scroll to zoom • Drag to select area • Alt+Drag to pan + Scroll to zoom | Drag to select area | Alt+Drag to pan
+ + +
From 043cd31a84514f50de9f149597c212a9ba2e966f Mon Sep 17 00:00:00 2001 From: Ivan Pleshkov Date: Tue, 7 Apr 2026 14:44:29 +0200 Subject: [PATCH 07/31] no rescoring --- .../configurations/qdrant-turbo-quant.json | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json index 63c998361..8d3da71ee 100644 --- a/experiments/configurations/qdrant-turbo-quant.json +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -16,8 +16,6 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } @@ -39,8 +37,6 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } @@ -62,8 +58,6 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } @@ -85,8 +79,6 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } @@ -108,8 +100,6 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } @@ -131,8 +121,6 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } @@ -153,8 +141,6 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } @@ -173,8 +159,6 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } @@ -194,8 +178,6 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } @@ -215,14 +197,12 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, { - "name": "qdrant-turbo-m-16-ef-128", + "name": "qdrant-no-quant-m-16-ef-128", "engine": "qdrant_native", "connection_params": { "timeout": 30 }, "collection_params": { From 363f897927e76bc0b1ae493235bc0090aaa17d44 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Tue, 7 Apr 2026 16:58:51 +0200 Subject: [PATCH 08/31] More improvements --- scripts/plots.html | 69 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/scripts/plots.html b/scripts/plots.html index a35b12b50..b255d8cc9 100644 --- a/scripts/plots.html +++ b/scripts/plots.html @@ -367,6 +367,12 @@

if (name.startsWith('results-')) { name = name.substring(8); // 'results-' is 8 characters } + // Remove 'ghcr-' prefix if present + if (name.startsWith('ghcr-')) { + name = name.substring(5); + } + // Trim commit hash to first 4 characters (pattern: 40-char hex at end) + name = name.replace(/([0-9a-f]{4})[0-9a-f]{36,}$/, '$1'); return name; } @@ -1055,9 +1061,7 @@

${dataset.label} return setupPointStyles[index % setupPointStyles.length]; } - function buildSplitChartConfig(dataForDataset, datasetName, setupColorMap, yMetric, xAxis, chartType) { - const groupBy = groupBySelect.value; - + function buildSplitChartConfig(dataForDataset, datasetName, seriesColorMap, yMetric, xAxis, chartType) { function sortDataByXAxis(data) { return data.sort((a, b) => { const aValue = xAxis === 'mean_precisions' ? a[xAxis] * 100 : a[xAxis]; @@ -1070,21 +1074,24 @@

${dataset.label} } let chartDatasets = []; + const multipleFiles = new Set(dataForDataset.map(d => d.datasetLabel)).size > 1; - // In split mode, always group by setup_name for meaningful comparison - const groups = [...new Set(dataForDataset.map(d => d.setup_name))].sort(); + // Group by (file label, setup_name) so same setup from different files are separate series + const seriesKeys = [...new Set(dataForDataset.map(d => `${d.datasetLabel}\0${d.setup_name}`))].sort(); - groups.forEach((setupName) => { - let groupData = dataForDataset.filter(d => d.setup_name === setupName); + seriesKeys.forEach((key) => { + const [fileLabel, setupName] = key.split('\0'); + let groupData = dataForDataset.filter(d => d.datasetLabel === fileLabel && d.setup_name === setupName); groupData = applyParetoFilter(groupData, yMetric); groupData = sortDataByXAxis(groupData); if (groupData.length === 0) return; - const colorInfo = setupColorMap[setupName]; + const colorInfo = seriesColorMap[key]; + const label = multipleFiles ? `${fileLabel}: ${setupName}` : setupName; chartDatasets.push({ - label: setupName, + label: label, data: groupData.map(row => ({ x: xAxis === 'mean_precisions' ? row[xAxis] * 100 : row[xAxis], y: yMetric.includes('time') && yMetric !== 'total_upload_time' ? row[yMetric] * 1000 : row[yMetric], @@ -1097,7 +1104,8 @@

${dataset.label} tension: 0.4, pointRadius: 7, pointHoverRadius: 10, - pointStyle: colorInfo.pointStyle + pointStyle: colorInfo.pointStyle, + borderDash: colorInfo.borderDash }); }); @@ -1230,14 +1238,39 @@

${dataset.label} if (filteredData.length === 0) return; - // Build a consistent color map across all datasets + // Build a consistent color map keyed by (fileLabel, setup_name) + // Use file color for each file's series, different point styles per setup const allSetups = [...new Set(filteredData.map(d => d.setup_name))].sort(); - const setupColorMap = {}; - allSetups.forEach((setup, i) => { - setupColorMap[setup] = { - color: getSetupColor(i), - pointStyle: getSetupPointStyle(i) - }; + const fileLabels = [...new Set(filteredData.map(d => d.datasetLabel))].sort(); + const multipleFiles = fileLabels.length > 1; + const seriesColorMap = {}; + + // Map each file to its dataset color, each setup to a point style + const fileLabelToColor = {}; + fileLabels.forEach((label, i) => { + const ds = datasets.find(d => d.label === label); + fileLabelToColor[label] = ds ? ds.color : getSetupColor(i); + }); + + let seriesIdx = 0; + fileLabels.forEach((fileLabel, fileIdx) => { + allSetups.forEach((setup, setupIdx) => { + const key = `${fileLabel}\0${setup}`; + if (multipleFiles) { + seriesColorMap[key] = { + color: fileLabelToColor[fileLabel], + pointStyle: getSetupPointStyle(setupIdx), + borderDash: fileIdx > 0 ? [5, 5] : undefined + }; + } else { + seriesColorMap[key] = { + color: getSetupColor(setupIdx), + pointStyle: getSetupPointStyle(setupIdx), + borderDash: undefined + }; + } + seriesIdx++; + }); }); // Get unique dataset names @@ -1261,7 +1294,7 @@

${dataset.label} wrapper.appendChild(canvasContainer); multiChartSection.appendChild(wrapper); - const config = buildSplitChartConfig(dataForDs, dsName, setupColorMap, yMetric, xAxis, chartType); + const config = buildSplitChartConfig(dataForDs, dsName, seriesColorMap, yMetric, xAxis, chartType); const chartInstance = new Chart(canvas.getContext('2d'), config); multiCharts.push(chartInstance); }); From ace005b066c058ef750532f6ca00d6fd4b0845f6 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Tue, 7 Apr 2026 17:24:53 +0200 Subject: [PATCH 09/31] Set log_lvl to error --- engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml b/engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml index 83a5c900c..55d31d752 100644 --- a/engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml +++ b/engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml @@ -18,3 +18,4 @@ services: memory: 25Gb environment: - QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL:-false} + - QDRANT__LOG_LEVEL=ERROR From 19e7bd3cb702fd27cbc2723e3dcab8c9084ab9a9 Mon Sep 17 00:00:00 2001 From: Ivan Pleshkov Date: Tue, 7 Apr 2026 19:56:13 +0200 Subject: [PATCH 10/31] qdrant_hybrid --- .../configurations/qdrant-turbo-quant.json | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json index 8d3da71ee..bbd7c18c5 100644 --- a/experiments/configurations/qdrant-turbo-quant.json +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -1,7 +1,7 @@ [ { "name": "qdrant-tq4-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -22,7 +22,7 @@ }, { "name": "qdrant-tq4-plus-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -43,7 +43,7 @@ }, { "name": "qdrant-tq2-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -64,7 +64,7 @@ }, { "name": "qdrant-tq2-plus-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -85,7 +85,7 @@ }, { "name": "qdrant-tq1-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -106,7 +106,7 @@ }, { "name": "qdrant-tq1-plus-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -127,7 +127,7 @@ }, { "name": "qdrant-sq-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -147,7 +147,7 @@ }, { "name": "qdrant-bq-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -165,7 +165,7 @@ }, { "name": "qdrant-bq2-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -184,7 +184,7 @@ }, { "name": "qdrant-bq-asym-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -203,7 +203,7 @@ }, { "name": "qdrant-no-quant-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, From 2f27bbf73999a3c265297ced0eeda3fc6bd7f0b0 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Wed, 8 Apr 2026 11:10:58 +0200 Subject: [PATCH 11/31] Run server and client on the same machine --- .github/workflows/manual-benchmarks-cascade.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual-benchmarks-cascade.yaml b/.github/workflows/manual-benchmarks-cascade.yaml index 52a21e385..0be800966 100644 --- a/.github/workflows/manual-benchmarks-cascade.yaml +++ b/.github/workflows/manual-benchmarks-cascade.yaml @@ -184,7 +184,7 @@ jobs: MACHINES_FIRST=false SERVER_NAME="benchmark-cascade-server-${i}" - CLIENT_NAME="benchmark-cascade-client-${i}" + CLIENT_NAME="benchmark-cascade-server-${i}" MACHINES_INFO="${MACHINES_INFO}{\"server_name\":\"$SERVER_NAME\",\"client_name\":\"$CLIENT_NAME\"}" done @@ -273,7 +273,7 @@ jobs: max_retries: 5 - name: Create Client - if: ${{ inputs.machines_per_bench == true || inputs.current_batch == 0 }} + if: ${{ (inputs.machines_per_bench == true || inputs.current_batch == 0) && steps.extract_names.outputs.client_name != steps.extract_names.outputs.server_name }} uses: ./.github/workflows/actions/create-server-with-retry with: server_name: ${{ steps.extract_names.outputs.client_name }} From 1afe9134ef63851c37880cbae97bc1433839c405 Mon Sep 17 00:00:00 2001 From: Ivan Pleshkov Date: Wed, 8 Apr 2026 13:54:13 +0200 Subject: [PATCH 12/31] 6bit experiments --- .../configurations/qdrant-turbo-quant.json | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json index bbd7c18c5..54307c0f0 100644 --- a/experiments/configurations/qdrant-turbo-quant.json +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -1,4 +1,46 @@ [ + { + "name": "qdrant-tq6-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "turbo_quant": { + "correction": "normalization", + "bits": 6, + "plus": false, + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-tq6-plus-m-16-ef-128", + "engine": "qdrant_native", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { + "turbo_quant": { + "correction": "normalization", + "bits": 6, + "plus": true, + "always_ram": true + } + } + }, + "search_params": [ + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, { "name": "qdrant-tq4-m-16-ef-128", "engine": "qdrant_hybrid", From 1ed2cfa84459e031c5a01a60e5471abfe36937c7 Mon Sep 17 00:00:00 2001 From: Ivan Pleshkov Date: Wed, 8 Apr 2026 15:03:46 +0200 Subject: [PATCH 13/31] more search params --- .../configurations/qdrant-turbo-quant.json | 44 ++++++++++++++----- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json index 54307c0f0..7d855bbd2 100644 --- a/experiments/configurations/qdrant-turbo-quant.json +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -16,7 +16,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, @@ -37,7 +39,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, @@ -79,7 +83,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, @@ -100,7 +106,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, @@ -121,7 +129,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, @@ -142,7 +152,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, @@ -163,7 +175,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, @@ -183,7 +197,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, @@ -201,7 +217,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, @@ -220,7 +238,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, @@ -239,7 +259,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, From 283b0776a8db6d8d884075411e9e10d6e478de89 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Wed, 8 Apr 2026 16:06:50 +0200 Subject: [PATCH 14/31] Change serach and add another viz --- engine/base_client/search.py | 27 +- scripts/plots.html | 706 ++++++++++++++++++++----------- scripts/process-benchmarks.ipynb | 2 +- scripts/recall-at-k.html | 386 +++++++++++++++++ 4 files changed, 863 insertions(+), 258 deletions(-) create mode 100644 scripts/recall-at-k.html diff --git a/engine/base_client/search.py b/engine/base_client/search.py index 3626191e0..a14925474 100644 --- a/engine/base_client/search.py +++ b/engine/base_client/search.py @@ -9,6 +9,7 @@ from dataset_reader.base_reader import Query DEFAULT_TOP = 10 +RECALL_AT_K_VALUES = [1, 2, 4, 8, 16, 32, 64] class BaseSearcher: @@ -47,11 +48,23 @@ def _search_one(cls, query: Query, top: Optional[int] = None): end = time.perf_counter() precision = 1.0 + recall_at_1_at_k = {} if query.expected_result: ids = set(x[0] for x in search_res) precision = len(ids.intersection(query.expected_result[:top])) / top - return precision, end - start + # Compute recall@1@k: is the true nearest neighbor in the top-k results? + true_nn = query.expected_result[0] + result_ids = [x[0] for x in search_res] + try: + nn_pos = result_ids.index(true_nn) + 1 # 1-based position + except ValueError: + nn_pos = float('inf') # not found at all + for k in RECALL_AT_K_VALUES: + if k <= len(result_ids): + recall_at_1_at_k[k] = 1.0 if nn_pos <= k else 0.0 + + return precision, end - start, recall_at_1_at_k def search_all( self, @@ -71,7 +84,7 @@ def search_all( if parallel == 1: start = time.perf_counter() - precisions, latencies = list( + precisions, latencies, recall_at_1_at_k_list = list( zip(*[search_one(query) for query in tqdm.tqdm(queries)]) ) else: @@ -90,7 +103,7 @@ def search_all( if parallel > 10: time.sleep(15) # Wait for all processes to start start = time.perf_counter() - precisions, latencies = list( + precisions, latencies, recall_at_1_at_k_list = list( zip(*pool.imap_unordered(search_one, iterable=tqdm.tqdm(queries))) ) @@ -98,6 +111,13 @@ def search_all( self.__class__.delete_client() + # Aggregate recall@1@k across all queries + mean_recall_at_1_at_k = {} + for k in RECALL_AT_K_VALUES: + values = [r[k] for r in recall_at_1_at_k_list if k in r] + if values: + mean_recall_at_1_at_k[k] = float(np.mean(values)) + return { "total_time": total_time, "mean_time": np.mean(latencies), @@ -110,6 +130,7 @@ def search_all( "p99_time": np.percentile(latencies, 99), "precisions": precisions, "latencies": latencies, + "recall_at_1_at_k": mean_recall_at_1_at_k, } def setup_search(self): diff --git a/scripts/plots.html b/scripts/plots.html index b255d8cc9..4ba1f8e72 100644 --- a/scripts/plots.html +++ b/scripts/plots.html @@ -10,6 +10,7 @@ + @@ -190,6 +179,20 @@

Data Summary:

> + + +

+ +
+ +
@@ -354,9 +357,6 @@

'#84CC16' // Lime ]; - // Legacy colors array for backward compatibility - const colors = datasetColors; - // Utility function to get filename without extension and 'results-' prefix function getFilenameWithoutExtension(filename) { if (!filename) return ''; @@ -401,9 +401,26 @@

const filtersSection = document.getElementById('filtersSection'); const filterControls = document.getElementById('filterControls'); const splitBySelect = document.getElementById('splitBySelect'); + const quantGroupSelect = document.getElementById('quantGroupSelect'); const singleChartSection = document.getElementById('singleChartSection'); const multiChartSection = document.getElementById('multiChartSection'); const resetZoomBtn = document.getElementById('resetZoomBtn'); + + // Quantization group definitions: patterns matched against setup_name + const quantGroups = { + group1: ['-no-quant-', '-sq-', '-tq4-', '-tq4-plus-'], + group2: ['-bq2-', '-tq2-', '-tq2-plus-'], + group3: ['-bq-', '-bq-asym-', '-tq1-', '-tq1-plus-'] + }; + + function matchesQuantGroup(setupName, group) { + if (group === 'all') return true; + const patterns = quantGroups[group]; + if (!patterns) return true; + // Pad with dashes so patterns match reliably + const padded = '-' + setupName + '-'; + return patterns.some(p => padded.includes(p)); + } let multiCharts = []; // Track chart instances for split view // Initialize with one empty dataset input @@ -444,80 +461,42 @@

} }); - // Pareto frontier filtering function function applyParetoFilter(data, yMetric) { - // Always apply Pareto filter for all metrics - if (data.length === 0) { - return data; - } - + if (data.length === 0) return data; - // Group data by dataset and parallel thread count - const datasetParallelGroups = {}; + // Group by dataset + parallel thread count + const groups = {}; data.forEach(point => { - const dataset = point.dataset_name || 'Unknown'; - const parallel = point.parallel || 1; - const groupKey = `${dataset}__${parallel}`; - - if (!datasetParallelGroups[groupKey]) { - datasetParallelGroups[groupKey] = []; - } - datasetParallelGroups[groupKey].push(point); + const key = `${point.dataset_name || 'Unknown'}__${point.parallel || 1}`; + (groups[key] = groups[key] || []).push(point); }); - const allFiltered = []; + const isHigherBetter = yMetric === 'rps'; + + Object.values(groups).forEach(groupData => { + // Sort by precision descending, deduplicate + const sorted = [...groupData].sort((a, b) => b.mean_precisions - a.mean_precisions); + const seen = new Set(); + const unique = sorted.filter(p => { + const prec = p.mean_precisions || 0; + if (seen.has(prec)) return false; + seen.add(prec); + return true; + }); - // Apply Pareto filter to each dataset-parallel group separately - Object.keys(datasetParallelGroups).forEach(groupKey => { - const [dataset, parallel] = groupKey.split('__'); - const groupData = datasetParallelGroups[groupKey]; - - - // Sort by precision descending (highest first) within this group - const sortedByPrecision = [...groupData].sort((a, b) => b.mean_precisions - a.mean_precisions); - - // Remove duplicates based on precision value - const seenPrecisions = new Set(); - const uniqueByPrecision = []; - for (const point of sortedByPrecision) { - const precision = point.mean_precisions || 0; - if (!seenPrecisions.has(precision)) { - seenPrecisions.add(precision); - uniqueByPrecision.push(point); - } - } - const sortedByPrecisionUnique = uniqueByPrecision; - - if (sortedByPrecisionUnique.length === 0) return; - - const filtered = []; - - // Determine if higher or lower values are better for this metric - const isHigherBetter = yMetric === 'rps'; + if (unique.length === 0) return; - // Start with the highest precision point - let bestPerformance = sortedByPrecisionUnique[0][yMetric]; - if (bestPerformance === null || bestPerformance === undefined) { - return; - } - filtered.push(sortedByPrecisionUnique[0]); + let bestPerformance = unique[0][yMetric]; + if (bestPerformance == null) return; + const filtered = [unique[0]]; - - // For each subsequent point (lower precision), only include if it improves performance - for (let i = 1; i < sortedByPrecisionUnique.length; i++) { - const currentValue = sortedByPrecisionUnique[i][yMetric]; - if (currentValue === null || currentValue === undefined) { - continue; // Skip points with missing data - } - - const isImprovement = isHigherBetter ? - currentValue > bestPerformance : - currentValue < bestPerformance; - - if (isImprovement) { - filtered.push(sortedByPrecisionUnique[i]); - bestPerformance = currentValue; + for (let i = 1; i < unique.length; i++) { + const val = unique[i][yMetric]; + if (val == null) continue; + if (isHigherBetter ? val > bestPerformance : val < bestPerformance) { + filtered.push(unique[i]); + bestPerformance = val; } } @@ -660,13 +639,25 @@

${dataset.label} } function applyFilters() { + const selectedGroup = quantGroupSelect.value; filteredData = currentData.filter(row => { + // Apply quantization group filter + if (!matchesQuantGroup(row.setup_name || '', selectedGroup)) { + return false; + } + return Object.entries(filters).every(([field, value]) => { if (!value) return true; @@ -826,6 +824,56 @@

${dataset.label} updateChart(); } + function sortDataByXAxis(data, xAxis) { + return data.sort((a, b) => { + const aValue = xAxis === 'mean_precisions' ? a[xAxis] * 100 : a[xAxis]; + const bValue = xAxis === 'mean_precisions' ? b[xAxis] * 100 : b[xAxis]; + if (typeof aValue === 'number' && typeof bValue === 'number') return aValue - bValue; + return String(aValue).localeCompare(String(bValue)); + }); + } + + function getAxisLabels(yMetric, xAxis) { + const isTimeMetric = yMetric.includes('time'); + const yLabel = yMetric === 'total_upload_time' ? 'Index Time (s)' : + isTimeMetric ? yMetric.replace('_', ' ').toUpperCase() + ' (ms)' : + yMetric.replace('_', ' ').toUpperCase(); + const xLabel = xAxis === 'mean_precisions' ? 'Precision (%)' : + xAxis.replace('_', ' ').toUpperCase(); + return { yLabel, xLabel, isTimeMetric }; + } + + function baseZoomOptions() { + return { + pan: { enabled: true, mode: 'xy', threshold: 10 }, + zoom: { + wheel: { enabled: true, speed: 0.1 }, + pinch: { enabled: true }, + drag: { + enabled: true, + backgroundColor: 'rgba(59, 130, 246, 0.1)', + borderColor: 'rgba(59, 130, 246, 0.5)', + borderWidth: 1 + }, + mode: 'xy' + } + }; + } + + function baseScales(xLabel, yLabel, xAxis) { + return { + x: { + title: { display: true, text: xLabel }, + grid: { color: '#E5E7EB' }, + type: xAxis === 'setup_name' ? 'category' : 'linear' + }, + y: { + title: { display: true, text: yLabel }, + grid: { color: '#E5E7EB' } + } + }; + } + function getChartConfig() { const yMetric = yMetricSelect.value; const xAxis = xAxisSelect.value; @@ -834,21 +882,6 @@

${dataset.label} let chartDatasets = []; - // Helper function to sort data by x-axis value - function sortDataByXAxis(data) { - return data.sort((a, b) => { - const aValue = xAxis === 'mean_precisions' ? a[xAxis] * 100 : a[xAxis]; - const bValue = xAxis === 'mean_precisions' ? b[xAxis] * 100 : b[xAxis]; - - // Handle numeric vs string sorting - if (typeof aValue === 'number' && typeof bValue === 'number') { - return aValue - bValue; - } else { - return String(aValue).localeCompare(String(bValue)); - } - }); - } - // Process each dataset datasets.forEach((dataset) => { if (dataset.data.length === 0) return; @@ -856,13 +889,11 @@

${dataset.label} const datasetData = filteredData.filter(row => row.datasetId === dataset.id); if (datasetData.length === 0) return; - const filenameShort = getFilenameWithoutExtension(dataset.filename) || dataset.label; - if (groupBy === 'none') { // No grouping - one line per dataset let dataToPlot = [...datasetData]; dataToPlot = applyParetoFilter(dataToPlot, yMetric); - const sortedData = sortDataByXAxis(dataToPlot); + const sortedData = sortDataByXAxis(dataToPlot, xAxis); chartDatasets.push({ label: dataset.label, @@ -886,7 +917,7 @@

${dataset.label} groups.forEach((group, groupIndex) => { let groupData = datasetData.filter(d => d[groupBy] === group); groupData = applyParetoFilter(groupData, yMetric); - groupData = sortDataByXAxis(groupData); + groupData = sortDataByXAxis(groupData, xAxis); chartDatasets.push({ label: `${dataset.label}: ${group}`, @@ -908,13 +939,7 @@

${dataset.label} } }); - const isTimeMetric = yMetric.includes('time'); - const yLabel = yMetric === 'total_upload_time' ? 'Index Time (s)' : - isTimeMetric ? yMetric.replace('_', ' ').toUpperCase() + ' (ms)' : - yMetric.replace('_', ' ').toUpperCase(); - - const xLabel = xAxis === 'mean_precisions' ? 'Precision (%)' : - xAxis.replace('_', ' ').toUpperCase(); + const { yLabel, xLabel } = getAxisLabels(yMetric, xAxis); return { type: chartType === 'scatter' ? 'scatter' : chartType, @@ -923,29 +948,7 @@

${dataset.label} responsive: true, maintainAspectRatio: false, plugins: { - zoom: { - pan: { - enabled: true, - mode: 'xy', - threshold: 10 - }, - zoom: { - wheel: { - enabled: true, - speed: 0.1 - }, - pinch: { - enabled: true - }, - drag: { - enabled: true, - backgroundColor: 'rgba(59, 130, 246, 0.1)', - borderColor: 'rgba(59, 130, 246, 0.5)', - borderWidth: 1 - }, - mode: 'xy' - } - }, + zoom: baseZoomOptions(), legend: { position: 'top', labels: { @@ -1025,21 +1028,117 @@

${dataset.label} footerColor: '#ccc' } }, - scales: { - x: { - title: { display: true, text: xLabel }, - grid: { color: '#E5E7EB' }, - type: xAxis === 'setup_name' ? 'category' : 'linear' - }, - y: { - title: { display: true, text: yLabel }, - grid: { color: '#E5E7EB' } - } - } + scales: baseScales(xLabel, yLabel, xAxis) } }; } + // Register and disable datalabels globally; enable per-chart in split view + Chart.register(ChartDataLabels); + Chart.defaults.plugins.datalabels = { display: false }; + + // Extract short label from setup_name, e.g. "qdrant-tq1-plus-m-16-ef-128" → "tq1-plus" + function getShortSetupLabel(setupName) { + if (!setupName) return ''; + let short = setupName + .replace(/^qdrant-/, '') + .replace(/-m-\d+-ef-\d+$/, '') + .replace(/-rps$/, ''); + return short || setupName; + } + + // Custom plugin: draw lines connecting matching setup_name across files + const connectorLinesPlugin = { + id: 'connectorLines', + afterDatasetsDraw(chart) { + const meta = chart.config._config._connectorMeta; + if (!meta || !meta.enabled) return; + + const ctx = chart.ctx; + const datasetsData = chart.data.datasets; + + // Group points by setup_name + const pointsBySetup = {}; + datasetsData.forEach((ds, dsIndex) => { + const metaDs = chart.getDatasetMeta(dsIndex); + if (metaDs.hidden) return; + ds.data.forEach((pt, ptIndex) => { + const setup = pt.originalData?.setup_name; + if (!setup) return; + if (!pointsBySetup[setup]) pointsBySetup[setup] = []; + const element = metaDs.data[ptIndex]; + if (element) { + pointsBySetup[setup].push({ x: element.x, y: element.y }); + } + }); + }); + + // Draw lines for setups that appear in multiple series (i.e. across files) + ctx.save(); + ctx.setLineDash([4, 4]); + ctx.lineWidth = 1; + ctx.strokeStyle = 'rgba(150, 150, 150, 0.5)'; + + Object.values(pointsBySetup).forEach(points => { + if (points.length < 2) return; + ctx.beginPath(); + ctx.moveTo(points[0].x, points[0].y); + for (let i = 1; i < points.length; i++) { + ctx.lineTo(points[i].x, points[i].y); + } + ctx.stroke(); + }); + + ctx.restore(); + } + }; + Chart.register(connectorLinesPlugin); + + // Custom plugin: draw drop lines from each point to both axes + const dropLinesPlugin = { + id: 'dropLines', + afterDatasetsDraw(chart) { + if (!chart.config._config._dropLinesEnabled) return; + + const ctx = chart.ctx; + const xAxis = chart.scales.x; + const yAxis = chart.scales.y; + + ctx.save(); + ctx.setLineDash([2, 3]); + ctx.lineWidth = 0.85; + + chart.data.datasets.forEach((ds, dsIndex) => { + const metaDs = chart.getDatasetMeta(dsIndex); + if (metaDs.hidden) return; + const color = ds.borderColor || 'rgba(150,150,150,0.3)'; + + metaDs.data.forEach((element) => { + const px = element.x; + const py = element.y; + + ctx.strokeStyle = color; + ctx.globalAlpha = 0.36; + + // Vertical line: point → x-axis + ctx.beginPath(); + ctx.moveTo(px, py); + ctx.lineTo(px, yAxis.bottom); + ctx.stroke(); + + // Horizontal line: point → y-axis + ctx.beginPath(); + ctx.moveTo(px, py); + ctx.lineTo(xAxis.left, py); + ctx.stroke(); + }); + }); + + ctx.restore(); + } + }; + Chart.register(dropLinesPlugin); + // Distinct colors for setup names in split view const setupColors = [ '#1E40AF', '#EA580C', '#059669', '#7C3AED', '#DC2626', @@ -1062,17 +1161,6 @@

${dataset.label} } function buildSplitChartConfig(dataForDataset, datasetName, seriesColorMap, yMetric, xAxis, chartType) { - function sortDataByXAxis(data) { - return data.sort((a, b) => { - const aValue = xAxis === 'mean_precisions' ? a[xAxis] * 100 : a[xAxis]; - const bValue = xAxis === 'mean_precisions' ? b[xAxis] * 100 : b[xAxis]; - if (typeof aValue === 'number' && typeof bValue === 'number') { - return aValue - bValue; - } - return String(aValue).localeCompare(String(bValue)); - }); - } - let chartDatasets = []; const multipleFiles = new Set(dataForDataset.map(d => d.datasetLabel)).size > 1; @@ -1083,7 +1171,7 @@

${dataset.label} const [fileLabel, setupName] = key.split('\0'); let groupData = dataForDataset.filter(d => d.datasetLabel === fileLabel && d.setup_name === setupName); groupData = applyParetoFilter(groupData, yMetric); - groupData = sortDataByXAxis(groupData); + groupData = sortDataByXAxis(groupData, xAxis); if (groupData.length === 0) return; @@ -1105,100 +1193,184 @@

${dataset.label} pointRadius: 7, pointHoverRadius: 10, pointStyle: colorInfo.pointStyle, - borderDash: colorInfo.borderDash + borderDash: colorInfo.borderDash, + datalabels: { + display: true, + color: colorInfo.color, + font: { size: 10, weight: 'bold' }, + anchor: 'end', + align: 'top', + offset: 4, + formatter: function(value, context) { + const row = context.dataset.data[context.dataIndex]; + return getShortSetupLabel(row.originalData?.setup_name); + } + } }); }); - const isTimeMetric = yMetric.includes('time'); - const yLabel = yMetric === 'total_upload_time' ? 'Index Time (s)' : - isTimeMetric ? yMetric.replace('_', ' ').toUpperCase() + ' (ms)' : - yMetric.replace('_', ' ').toUpperCase(); - const xLabel = xAxis === 'mean_precisions' ? 'Precision (%)' : - xAxis.replace('_', ' ').toUpperCase(); + const { yLabel, xLabel } = getAxisLabels(yMetric, xAxis); return { type: chartType === 'scatter' ? 'scatter' : chartType, data: { datasets: chartDatasets }, + _connectorMeta: { enabled: multipleFiles }, + _dropLinesEnabled: true, options: { responsive: true, maintainAspectRatio: false, plugins: { - zoom: { - pan: { enabled: true, mode: 'xy', threshold: 10 }, - zoom: { - wheel: { enabled: true, speed: 0.1 }, - pinch: { enabled: true }, - drag: { - enabled: true, - backgroundColor: 'rgba(59, 130, 246, 0.1)', - borderColor: 'rgba(59, 130, 246, 0.5)', - borderWidth: 1 - }, - mode: 'xy' - } - }, - legend: { - position: 'top', - labels: { - usePointStyle: true, - padding: 15, - font: { size: 11 } - } - }, - title: { - display: true, - text: datasetName, - font: { size: 16, weight: 'bold' }, - padding: { bottom: 10 } - }, - tooltip: { - callbacks: { - title: function(tooltipItems) { - const originalData = tooltipItems[0].raw.originalData; - return originalData.setup_name || 'Unknown Setup'; - }, - label: function(context) { - const originalData = context.raw.originalData; - let labels = []; - labels.push(`${xLabel}: ${context.parsed.x.toFixed(2)}${xAxis === 'mean_precisions' ? '%' : ''}`); - labels.push(`${yLabel}: ${context.parsed.y.toFixed(2)}`); - if (originalData.parallel !== undefined) { - labels.push(`Parallel: ${originalData.parallel} threads`); - } - // Show quantization info - if (originalData.engine_params) { - const ep = originalData.engine_params; - if (ep.quantization_config) { - labels.push(`Quantization: ${JSON.stringify(ep.quantization_config)}`); - } - if (ep.quantization) { - labels.push(`Search quantization: ${JSON.stringify(ep.quantization)}`); - } - } - return labels; - } - }, - displayColors: true, - titleFont: { weight: 'bold', size: 13 }, - bodyFont: { size: 11 }, - padding: 10, - cornerRadius: 6, - backgroundColor: 'rgba(0, 0, 0, 0.85)', - titleColor: '#fff', - bodyColor: '#fff' - } + zoom: baseZoomOptions(), + legend: { position: 'top', labels: { usePointStyle: true, padding: 15, font: { size: 11 } } }, + title: { display: true, text: datasetName, font: { size: 16, weight: 'bold' }, padding: { bottom: 10 } }, + tooltip: splitTooltip(xLabel, yLabel, xAxis) }, - scales: { - x: { - title: { display: true, text: xLabel }, - grid: { color: '#E5E7EB' }, - type: xAxis === 'setup_name' ? 'category' : 'linear' - }, - y: { - title: { display: true, text: yLabel }, - grid: { color: '#E5E7EB' } + scales: baseScales(xLabel, yLabel, xAxis) + } + }; + } + + function splitTooltip(xLabel, yLabel, xAxis) { + return { + callbacks: { + title: function(tooltipItems) { + return tooltipItems[0].raw.originalData?.setup_name || 'Unknown Setup'; + }, + label: function(context) { + const od = context.raw.originalData; + let labels = []; + labels.push(`${xLabel}: ${context.parsed.x.toFixed(2)}${xAxis === 'mean_precisions' ? '%' : ''}`); + labels.push(`${yLabel}: ${context.parsed.y.toFixed(2)}`); + if (od?.parallel !== undefined) labels.push(`Parallel: ${od.parallel} threads`); + if (od?.engine_params?.quantization_config) { + labels.push(`Quantization: ${JSON.stringify(od.engine_params.quantization_config)}`); + } + if (od?.engine_params?.quantization) { + labels.push(`Search quantization: ${JSON.stringify(od.engine_params.quantization)}`); } + return labels; } + }, + displayColors: true, + titleFont: { weight: 'bold', size: 13 }, + bodyFont: { size: 11 }, + padding: 10, + cornerRadius: 6, + backgroundColor: 'rgba(0, 0, 0, 0.85)', + titleColor: '#fff', + bodyColor: '#fff' + }; + } + + // Get oversampling label for a data point + function getOversamplingLabel(row) { + const q = row.engine_params?.quantization; + if (!q || q.rescore === false || q.rescore === undefined) return 'no rescore'; + if (q.oversampling) return `oversample ${q.oversampling}x`; + return 'rescore only'; + } + + // Get numeric sort key for oversampling (for ordering points on the line) + function getOversamplingSort(row) { + const q = row.engine_params?.quantization; + if (!q || q.rescore === false || q.rescore === undefined) return 0; + return q.oversampling || 0.5; + } + + // Quantization group labels + const quantGroupLabels = { + group1: '4-bit+: orig, SQ, TQ4, TQ4+', + group2: '2-bit: BQ2, TQ2, TQ2+', + group3: '1-bit: BQ, BQ-asym, TQ1, TQ1+' + }; + + function buildQuantGroupChartConfig(data, title, yMetric, xAxis, chartType) { + const { yLabel, xLabel, isTimeMetric } = getAxisLabels(yMetric, xAxis); + + // Group by (fileLabel, setup_name) — each becomes a line + const fileLabels = [...new Set(data.map(d => d.datasetLabel))].sort(); + const multipleFiles = fileLabels.length > 1; + const allSetups = [...new Set(data.map(d => d.setup_name))].sort(); + + let chartDatasets = []; + let setupIdx = 0; + + fileLabels.forEach((fileLabel, fileIdx) => { + allSetups.forEach((setup) => { + let points = data.filter(d => d.datasetLabel === fileLabel && d.setup_name === setup); + if (points.length === 0) return; + + // Sort by oversampling value so the line goes: no rescore → oversample 2 → oversample 4 + points.sort((a, b) => getOversamplingSort(a) - getOversamplingSort(b)); + + const color = multipleFiles + ? (datasets.find(d => d.label === fileLabel)?.color || getSetupColor(setupIdx)) + : getSetupColor(setupIdx); + const pointStyle = getSetupPointStyle(setupIdx); + const shortName = getShortSetupLabel(setup); + const label = multipleFiles ? `${fileLabel}: ${shortName}` : shortName; + + const oversamplingLabels = points.map(row => { + const short = getShortSetupLabel(row.setup_name); + return `${short} (${row._oversamplingLabel})`; + }); + + chartDatasets.push({ + label: label, + data: points.map(row => ({ + x: xAxis === 'mean_precisions' ? row[xAxis] * 100 : row[xAxis], + y: isTimeMetric && yMetric !== 'total_upload_time' ? row[yMetric] * 1000 : row[yMetric], + originalData: row + })), + _oversamplingLabels: oversamplingLabels, + borderColor: color, + backgroundColor: color + '40', + borderWidth: 2, + fill: false, + tension: 0, + pointRadius: 7, + pointHoverRadius: 10, + pointStyle: pointStyle, + borderDash: multipleFiles && fileIdx > 0 ? [5, 5] : undefined, + datalabels: { + display: true, + color: color, + font: { size: 9, weight: 'bold' }, + anchor: 'end', + align: 'top', + offset: 4, + formatter: function(value, context) { + return context.dataset._oversamplingLabels?.[context.dataIndex] || ''; + } + } + }); + + setupIdx++; + }); + }); + + const baseTooltip = splitTooltip(xLabel, yLabel, xAxis); + // Override title to include oversampling info + baseTooltip.callbacks.title = function(tooltipItems) { + const od = tooltipItems[0].raw.originalData; + return `${od?.setup_name} — ${getOversamplingLabel(od)}`; + }; + + return { + type: chartType === 'scatter' ? 'scatter' : chartType, + data: { datasets: chartDatasets }, + _connectorMeta: { enabled: false }, + _dropLinesEnabled: true, + options: { + responsive: true, + maintainAspectRatio: false, + plugins: { + zoom: baseZoomOptions(), + legend: { position: 'top', labels: { usePointStyle: true, padding: 15, font: { size: 11 } } }, + title: { display: true, text: title, font: { size: 16, weight: 'bold' }, padding: { bottom: 10 } }, + tooltip: baseTooltip + }, + scales: baseScales(xLabel, yLabel, xAxis) } }; } @@ -1230,6 +1402,44 @@

${dataset.label} const ctx = document.getElementById('benchmarkChart').getContext('2d'); chart = new Chart(ctx, getChartConfig()); } + } else if (splitBy === 'quant_group') { + // Split by quantization group per dataset + singleChartSection.classList.add('hidden'); + multiChartSection.classList.remove('hidden'); + multiChartSection.innerHTML = ''; + + if (filteredData.length === 0) return; + + const datasetNames = [...new Set(filteredData.map(d => d.dataset_name))].sort(); + const chartType = chartTypeSelect.value; + const groupKeys = ['group1', 'group2', 'group3']; + + datasetNames.forEach((dsName) => { + const dataForDs = filteredData.filter(d => d.dataset_name === dsName); + if (dataForDs.length === 0) return; + + groupKeys.forEach((groupKey) => { + const groupData = dataForDs.filter(d => matchesQuantGroup(d.setup_name || '', groupKey)); + if (groupData.length === 0) return; + + const title = `${dsName} — ${quantGroupLabels[groupKey]}`; + + const wrapper = document.createElement('div'); + wrapper.className = 'bg-white border border-gray-200 rounded-lg p-6 shadow-sm'; + + const canvasContainer = document.createElement('div'); + canvasContainer.className = 'chart-container'; + + const canvas = document.createElement('canvas'); + canvasContainer.appendChild(canvas); + wrapper.appendChild(canvasContainer); + multiChartSection.appendChild(wrapper); + + const config = buildQuantGroupChartConfig(groupData, title, yMetric, xAxis, chartType); + const chartInstance = new Chart(canvas.getContext('2d'), config); + multiCharts.push(chartInstance); + }); + }); } else { // Split by dataset mode singleChartSection.classList.add('hidden'); @@ -1252,24 +1462,12 @@

${dataset.label} fileLabelToColor[label] = ds ? ds.color : getSetupColor(i); }); - let seriesIdx = 0; fileLabels.forEach((fileLabel, fileIdx) => { allSetups.forEach((setup, setupIdx) => { const key = `${fileLabel}\0${setup}`; - if (multipleFiles) { - seriesColorMap[key] = { - color: fileLabelToColor[fileLabel], - pointStyle: getSetupPointStyle(setupIdx), - borderDash: fileIdx > 0 ? [5, 5] : undefined - }; - } else { - seriesColorMap[key] = { - color: getSetupColor(setupIdx), - pointStyle: getSetupPointStyle(setupIdx), - borderDash: undefined - }; - } - seriesIdx++; + seriesColorMap[key] = multipleFiles + ? { color: fileLabelToColor[fileLabel], pointStyle: getSetupPointStyle(setupIdx), borderDash: fileIdx > 0 ? [5, 5] : undefined } + : { color: getSetupColor(setupIdx), pointStyle: getSetupPointStyle(setupIdx), borderDash: undefined }; }); }); diff --git a/scripts/process-benchmarks.ipynb b/scripts/process-benchmarks.ipynb index 48e60910f..d0a9a237d 100644 --- a/scripts/process-benchmarks.ipynb +++ b/scripts/process-benchmarks.ipynb @@ -153,7 +153,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "json_results = []\n\nfor index, row in joined_df.reset_index().iterrows():\n engine_params = {}\n \n if isinstance(row['params_upload'], dict):\n engine_params.update(row['params_upload'])\n if isinstance(row['params_search'], dict):\n search_params = row['params_search']\n engine_params.update(search_params.get('config', {}))\n engine_params.update(search_params.get('params', {}))\n engine_params.update(search_params.get('search_params', {}))\n engine_params.update(search_params.get('vectorIndexConfig', {}))\n\n engine_params.pop('experiment')\n engine_params.pop('parallel')\n\n engine_name = row['engine']\n\n if engine_name.startswith(\"qdrant-\"):\n engine_name = \"qdrant\"\n\n json_object = {\n \"engine_name\": engine_name,\n \"setup_name\": f\"{row['params_search']['experiment']}\",\n \"dataset_name\": row['dataset'],\n \"search_idx\": row['search_index'],\n \"upload_time\": row['results_upload']['upload_time'],\n \"total_upload_time\": row['results_upload']['total_time'],\n \"p95_time\": row['results_search']['p95_time'],\n \"rps\": row['results_search']['rps'],\n \"parallel\": row['params_search']['parallel'],\n \"p99_time\": row['results_search']['p99_time'],\n \"mean_time\": row['results_search']['mean_time'],\n \"mean_precisions\": row['results_search']['mean_precisions'],\n \"engine_params\": engine_params,\n }\n json_results.append(json_object)\n\nformat = '%Y-%M-%dT%H:%M:%S'\nnow = datetime.now().replace(tzinfo=timezone.utc).strftime(format)\n\nPath(\"results.json\").write_text(json.dumps(json_results, indent=2))\nPath(f\"results-{now}.json\").write_text(json.dumps(json_results, indent=2))\n\nprint(json_results[-1], len(json_results))\n\nresults_df = pd.DataFrame(json_results).sort_values(\"p99_time\", ascending=True)\n# results_df.to_csv('results.csv')\nresults_df" + "source": "json_results = []\n\nfor index, row in joined_df.reset_index().iterrows():\n engine_params = {}\n \n if isinstance(row['params_upload'], dict):\n engine_params.update(row['params_upload'])\n if isinstance(row['params_search'], dict):\n search_params = row['params_search']\n engine_params.update(search_params.get('config', {}))\n engine_params.update(search_params.get('params', {}))\n engine_params.update(search_params.get('search_params', {}))\n engine_params.update(search_params.get('vectorIndexConfig', {}))\n\n engine_params.pop('experiment')\n engine_params.pop('parallel')\n\n engine_name = row['engine']\n\n if engine_name.startswith(\"qdrant-\"):\n engine_name = \"qdrant\"\n\n json_object = {\n \"engine_name\": engine_name,\n \"setup_name\": f\"{row['params_search']['experiment']}\",\n \"dataset_name\": row['dataset'],\n \"search_idx\": row['search_index'],\n \"upload_time\": row['results_upload']['upload_time'],\n \"total_upload_time\": row['results_upload']['total_time'],\n \"p95_time\": row['results_search']['p95_time'],\n \"rps\": row['results_search']['rps'],\n \"parallel\": row['params_search']['parallel'],\n \"p99_time\": row['results_search']['p99_time'],\n \"mean_time\": row['results_search']['mean_time'],\n \"mean_precisions\": row['results_search']['mean_precisions'],\n \"engine_params\": engine_params,\n }\n\n # Include recall@1@k if available\n if 'recall_at_1_at_k' in row['results_search']:\n json_object['recall_at_1_at_k'] = row['results_search']['recall_at_1_at_k']\n\n json_results.append(json_object)\n\nformat = '%Y-%M-%dT%H:%M:%S'\nnow = datetime.now().replace(tzinfo=timezone.utc).strftime(format)\n\nPath(\"results.json\").write_text(json.dumps(json_results, indent=2))\nPath(f\"results-{now}.json\").write_text(json.dumps(json_results, indent=2))\n\nprint(json_results[-1], len(json_results))\n\nresults_df = pd.DataFrame(json_results).sort_values(\"p99_time\", ascending=True)\n# results_df.to_csv('results.csv')\nresults_df" } ], "metadata": { diff --git a/scripts/recall-at-k.html b/scripts/recall-at-k.html new file mode 100644 index 000000000..db05b7c86 --- /dev/null +++ b/scripts/recall-at-k.html @@ -0,0 +1,386 @@ + + + + + + Recall@1@k Visualization + + + + + + + + +
+
+

Recall@1@k Visualization

+

Upload benchmark JSON results containing recall_at_1_at_k data. Shows the probability of finding the true nearest neighbor in the top-k results.

+
+ + +
+
+
+

Data Input

+
+ + +
+
+
+ + +
+ +
+

Expected Data Format

+
+ 
[
+  {
+    "setup_name": "qdrant-tq4-m-16-ef-128",
+    "dataset_name": "glove-100-angular",
+    "parallel": 100,
+    "recall_at_1_at_k": {
+      "1": 0.86, "2": 0.95, "4": 0.98,
+      "8": 1.0, "16": 1.0, "32": 1.0, "64": 1.0
+    },
+    ...
+  }
+]
+
+
+
+ + + + + +
+
+ + + + \ No newline at end of file From 8d4580602755b50abdfde1a163c65c6300eb06c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 14:07:03 +0000 Subject: [PATCH 15/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- engine/base_client/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/base_client/search.py b/engine/base_client/search.py index a14925474..ff2e736b9 100644 --- a/engine/base_client/search.py +++ b/engine/base_client/search.py @@ -59,7 +59,7 @@ def _search_one(cls, query: Query, top: Optional[int] = None): try: nn_pos = result_ids.index(true_nn) + 1 # 1-based position except ValueError: - nn_pos = float('inf') # not found at all + nn_pos = float("inf") # not found at all for k in RECALL_AT_K_VALUES: if k <= len(result_ids): recall_at_1_at_k[k] = 1.0 if nn_pos <= k else 0.0 From 09c213aee7f8e1411c6d8b669706de23273e1c57 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Wed, 8 Apr 2026 16:13:16 +0200 Subject: [PATCH 16/31] Add script to merge final results into one --- scripts/merge_results.py | 103 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 scripts/merge_results.py diff --git a/scripts/merge_results.py b/scripts/merge_results.py new file mode 100644 index 000000000..156af8336 --- /dev/null +++ b/scripts/merge_results.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Merge multiple benchmark result JSON files by averaging numeric metrics. + +Usage: + python scripts/merge_results.py results-run1.json results-run2.json results-run3.json -o merged-results.json + +Records are matched by (setup_name, dataset_name, search_idx, parallel). +Numeric fields are averaged. Non-numeric fields (engine_params, engine_name) are taken from the first occurrence. +""" + +import argparse +import json +import sys +from collections import defaultdict +from pathlib import Path + +NUMERIC_FIELDS = [ + "rps", "mean_time", "p95_time", "p99_time", + "mean_precisions", "upload_time", "total_upload_time", +] + +KEY_FIELDS = ("setup_name", "dataset_name", "search_idx", "parallel") + + +def record_key(record): + return tuple(record.get(f) for f in KEY_FIELDS) + + +def average(values): + return sum(values) / len(values) + + +def merge_recall_at_1_at_k(records): + """Average recall_at_1_at_k dicts across records.""" + all_k = defaultdict(list) + for r in records: + rat = r.get("recall_at_1_at_k") + if not rat: + continue + for k, v in rat.items(): + all_k[k].append(v) + if not all_k: + return None + return {k: average(vs) for k, vs in sorted(all_k.items(), key=lambda x: int(x[0]))} + + +def merge(files): + grouped = defaultdict(list) + + for path in files: + data = json.loads(Path(path).read_text()) + for record in data: + grouped[record_key(record)].append(record) + + merged = [] + for key, records in grouped.items(): + # Start from first record as base + base = dict(records[0]) + n = len(records) + + # Average numeric fields + for field in NUMERIC_FIELDS: + values = [r[field] for r in records if field in r and r[field] is not None] + if values: + base[field] = average(values) + + # Average recall_at_1_at_k + rat = merge_recall_at_1_at_k(records) + if rat is not None: + base["recall_at_1_at_k"] = rat + + base["_merged_from"] = n + merged.append(base) + + return merged + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("files", nargs="+", help="Result JSON files to merge") + parser.add_argument("-o", "--output", default="merged-results.json", help="Output file (default: merged-results.json)") + args = parser.parse_args() + + for f in args.files: + if not Path(f).exists(): + print(f"Error: {f} not found", file=sys.stderr) + sys.exit(1) + + merged = merge(args.files) + Path(args.output).write_text(json.dumps(merged, indent=2)) + + # Summary + counts = defaultdict(int) + for r in merged: + counts[r.get("_merged_from", 1)] += 1 + print(f"Merged {sum(len(json.loads(Path(f).read_text())) for f in args.files)} records from {len(args.files)} files into {len(merged)} entries") + for n, c in sorted(counts.items()): + print(f" {c} entries averaged from {n} run(s)") + print(f"Written to {args.output}") + + +if __name__ == "__main__": + main() \ No newline at end of file From 1ef92a12aa46dbc8f194abef3c2fca35061004cb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 14:13:30 +0000 Subject: [PATCH 17/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/merge_results.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/merge_results.py b/scripts/merge_results.py index 156af8336..07358fba4 100644 --- a/scripts/merge_results.py +++ b/scripts/merge_results.py @@ -15,8 +15,13 @@ from pathlib import Path NUMERIC_FIELDS = [ - "rps", "mean_time", "p95_time", "p99_time", - "mean_precisions", "upload_time", "total_upload_time", + "rps", + "mean_time", + "p95_time", + "p99_time", + "mean_precisions", + "upload_time", + "total_upload_time", ] KEY_FIELDS = ("setup_name", "dataset_name", "search_idx", "parallel") @@ -76,9 +81,16 @@ def merge(files): def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) parser.add_argument("files", nargs="+", help="Result JSON files to merge") - parser.add_argument("-o", "--output", default="merged-results.json", help="Output file (default: merged-results.json)") + parser.add_argument( + "-o", + "--output", + default="merged-results.json", + help="Output file (default: merged-results.json)", + ) args = parser.parse_args() for f in args.files: @@ -93,11 +105,13 @@ def main(): counts = defaultdict(int) for r in merged: counts[r.get("_merged_from", 1)] += 1 - print(f"Merged {sum(len(json.loads(Path(f).read_text())) for f in args.files)} records from {len(args.files)} files into {len(merged)} entries") + print( + f"Merged {sum(len(json.loads(Path(f).read_text())) for f in args.files)} records from {len(args.files)} files into {len(merged)} entries" + ) for n, c in sorted(counts.items()): print(f" {c} entries averaged from {n} run(s)") print(f"Written to {args.output}") if __name__ == "__main__": - main() \ No newline at end of file + main() From 889e58505c7f6fd36ae509b339efe80c7a06b351 Mon Sep 17 00:00:00 2001 From: Ivan Pleshkov Date: Wed, 8 Apr 2026 19:56:58 +0200 Subject: [PATCH 18/31] params to start --- params.json | 24 ++++++++++++++++++++++++ params_dev.json | 18 ++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 params.json create mode 100644 params_dev.json diff --git a/params.json b/params.json new file mode 100644 index 000000000..561adb515 --- /dev/null +++ b/params.json @@ -0,0 +1,24 @@ +{ + "params": { + "qdrant_version": ["ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564"], + "dataset": [ + "dbpedia-openai-100K-1536-angular", + "laion-small-clip", + "glove-100-angular", + "cohere-wiki-100k-no-filters" + ], + "engine_config": [ + "qdrant-no-quant-m-16-ef-128", + "qdrant-sq-m-16-ef-128", + "qdrant-tq4-m-16-ef-128", + "qdrant-tq4-plus-m-16-ef-128", + "qdrant-bq2-m-16-ef-128", + "qdrant-tq2-m-16-ef-128", + "qdrant-tq2-plus-m-16-ef-128", + "qdrant-tq1-m-16-ef-128", + "qdrant-tq1-plus-m-16-ef-128", + "qdrant-bq-asym-m-16-ef-128", + "qdrant-bq-m-16-ef-128" + ] + } +} \ No newline at end of file diff --git a/params_dev.json b/params_dev.json new file mode 100644 index 000000000..2b3abe7b5 --- /dev/null +++ b/params_dev.json @@ -0,0 +1,18 @@ +{ + "params": { + "qdrant_version": ["ghcr/dev-c9c4227691b4d1dbeb6dea8f23783266cdcb6db6"], + "dataset": [ + "dbpedia-openai-100K-1536-angular", + "laion-small-clip", + "glove-100-angular", + "cohere-wiki-100k-no-filters" + ], + "engine_config": [ + "qdrant-no-quant-m-16-ef-128", + "qdrant-sq-m-16-ef-128", + "qdrant-bq2-m-16-ef-128", + "qdrant-bq-asym-m-16-ef-128", + "qdrant-bq-m-16-ef-128" + ] + } +} \ No newline at end of file From c2980a00927def2d4797e06c9bf1af254bdf278f Mon Sep 17 00:00:00 2001 From: tellet-q Date: Wed, 8 Apr 2026 20:06:57 +0200 Subject: [PATCH 19/31] Add custom config --- benchmark_cascade/benchmark-configs.json | 2166 ++++++++++++++++++++++ scripts/process-benchmarks.ipynb | 8 +- 2 files changed, 2170 insertions(+), 4 deletions(-) diff --git a/benchmark_cascade/benchmark-configs.json b/benchmark_cascade/benchmark-configs.json index 4378e7905..c330b907d 100644 --- a/benchmark_cascade/benchmark-configs.json +++ b/benchmark_cascade/benchmark-configs.json @@ -301,6 +301,2172 @@ "engine_config": "qdrant-sq-rps-m-64-ef-512", "feature_flags_all": true } + ], + "turbo_quant": [ + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/dev", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true + }, + + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true + } ] } } \ No newline at end of file diff --git a/scripts/process-benchmarks.ipynb b/scripts/process-benchmarks.ipynb index d0a9a237d..d055ff30e 100644 --- a/scripts/process-benchmarks.ipynb +++ b/scripts/process-benchmarks.ipynb @@ -124,7 +124,7 @@ } }, "outputs": [], - "source": "upload_df = pd.DataFrame(upload_results)\nupload_df[\"date\"] = pd.to_datetime(upload_df[\"date\"], format=\"%Y-%m-%d-%H-%M-%S\")\nupload_df = upload_df.sort_values(\"date\", ascending=False) \\\n .groupby([\"engine_name\", \"engine\", \"m\", \"ef\", \"dataset\"]) \\\n .first()\n\ntemp_df = upload_df.copy()\ntemp_df[\"total_time\"] = temp_df[\"results\"].apply(lambda x: x[\"total_time\"])\ntemp_df.sort_values(\"total_time\", ascending=True).head(n=5)" + "source": "upload_df = pd.DataFrame(upload_results)\nupload_df[\"date\"] = pd.to_datetime(upload_df[\"date\"], format=\"%Y-%m-%d-%H-%M-%S\")\n\ntemp_df = upload_df.copy()\ntemp_df[\"total_time\"] = temp_df[\"results\"].apply(lambda x: x[\"total_time\"])\ntemp_df.sort_values(\"total_time\", ascending=True).head(n=5)" }, { "cell_type": "code", @@ -139,21 +139,21 @@ } }, "outputs": [], - "source": "search_df = pd.DataFrame(search_results)\nsearch_df[\"date\"] = pd.to_datetime(search_df[\"date\"], format=\"%Y-%m-%d-%H-%M-%S\")\nsearch_df = search_df.sort_values(\"date\", ascending=False) \\\n .groupby([\"engine_name\", \"engine\", \"m\", \"ef\", \"dataset\", \"search_index\"]) \\\n .first()\n\ntemp_df = search_df.copy()\ntemp_df['rps'] = temp_df['results'].apply(lambda x: x[\"rps\"])\ntemp_df.sort_values(\"rps\", ascending=False).head(n=10)" + "source": "search_df = pd.DataFrame(search_results)\nsearch_df[\"date\"] = pd.to_datetime(search_df[\"date\"], format=\"%Y-%m-%d-%H-%M-%S\")\n\ntemp_df = search_df.copy()\ntemp_df['rps'] = temp_df['results'].apply(lambda x: x[\"rps\"])\ntemp_df.sort_values(\"rps\", ascending=False).head(n=10)" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "_search = search_df.reset_index()\n_upload = upload_df.reset_index()\n\njoined_df = _search.merge(_upload, on=[\"engine_name\", \"engine\", \"m\", \"ef\", \"dataset\"], how=\"left\", suffixes=(\"_search\", \"_upload\"))\nprint(len(joined_df))\njoined_df" + "source": "_search = search_df.reset_index(drop=True)\n_upload = upload_df.reset_index(drop=True)\n\njoined_df = _search.merge(_upload, on=[\"engine_name\", \"engine\", \"m\", \"ef\", \"dataset\"], how=\"left\", suffixes=(\"_search\", \"_upload\"))\nprint(len(joined_df))\njoined_df" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "json_results = []\n\nfor index, row in joined_df.reset_index().iterrows():\n engine_params = {}\n \n if isinstance(row['params_upload'], dict):\n engine_params.update(row['params_upload'])\n if isinstance(row['params_search'], dict):\n search_params = row['params_search']\n engine_params.update(search_params.get('config', {}))\n engine_params.update(search_params.get('params', {}))\n engine_params.update(search_params.get('search_params', {}))\n engine_params.update(search_params.get('vectorIndexConfig', {}))\n\n engine_params.pop('experiment')\n engine_params.pop('parallel')\n\n engine_name = row['engine']\n\n if engine_name.startswith(\"qdrant-\"):\n engine_name = \"qdrant\"\n\n json_object = {\n \"engine_name\": engine_name,\n \"setup_name\": f\"{row['params_search']['experiment']}\",\n \"dataset_name\": row['dataset'],\n \"search_idx\": row['search_index'],\n \"upload_time\": row['results_upload']['upload_time'],\n \"total_upload_time\": row['results_upload']['total_time'],\n \"p95_time\": row['results_search']['p95_time'],\n \"rps\": row['results_search']['rps'],\n \"parallel\": row['params_search']['parallel'],\n \"p99_time\": row['results_search']['p99_time'],\n \"mean_time\": row['results_search']['mean_time'],\n \"mean_precisions\": row['results_search']['mean_precisions'],\n \"engine_params\": engine_params,\n }\n\n # Include recall@1@k if available\n if 'recall_at_1_at_k' in row['results_search']:\n json_object['recall_at_1_at_k'] = row['results_search']['recall_at_1_at_k']\n\n json_results.append(json_object)\n\nformat = '%Y-%M-%dT%H:%M:%S'\nnow = datetime.now().replace(tzinfo=timezone.utc).strftime(format)\n\nPath(\"results.json\").write_text(json.dumps(json_results, indent=2))\nPath(f\"results-{now}.json\").write_text(json.dumps(json_results, indent=2))\n\nprint(json_results[-1], len(json_results))\n\nresults_df = pd.DataFrame(json_results).sort_values(\"p99_time\", ascending=True)\n# results_df.to_csv('results.csv')\nresults_df" + "source": "rows = []\n\nfor index, row in joined_df.iterrows():\n engine_params = {}\n\n if isinstance(row['params_upload'], dict):\n engine_params.update(row['params_upload'])\n if isinstance(row['params_search'], dict):\n search_params = row['params_search']\n engine_params.update(search_params.get('config', {}))\n engine_params.update(search_params.get('params', {}))\n engine_params.update(search_params.get('search_params', {}))\n engine_params.update(search_params.get('vectorIndexConfig', {}))\n\n engine_params.pop('experiment')\n engine_params.pop('parallel')\n\n engine_name = row['engine']\n if engine_name.startswith(\"qdrant-\"):\n engine_name = \"qdrant\"\n\n entry = {\n \"engine_name\": engine_name,\n \"setup_name\": row['params_search']['experiment'],\n \"dataset_name\": row['dataset'],\n \"search_idx\": row['search_index'],\n \"parallel\": row['params_search']['parallel'],\n \"engine_params\": json.dumps(engine_params, sort_keys=True),\n # metrics\n \"upload_time\": row['results_upload']['upload_time'],\n \"total_upload_time\": row['results_upload']['total_time'],\n \"p95_time\": row['results_search']['p95_time'],\n \"rps\": row['results_search']['rps'],\n \"p99_time\": row['results_search']['p99_time'],\n \"mean_time\": row['results_search']['mean_time'],\n \"mean_precisions\": row['results_search']['mean_precisions'],\n }\n\n if 'recall_at_1_at_k' in row['results_search']:\n entry['recall_at_1_at_k'] = row['results_search']['recall_at_1_at_k']\n\n rows.append(entry)\n\nexpanded_df = pd.DataFrame(rows)\n\n# Group by experiment identity and compute median of numeric metrics\ngroup_cols = [\"engine_name\", \"setup_name\", \"dataset_name\", \"search_idx\", \"parallel\", \"engine_params\"]\nmetric_cols = [\"upload_time\", \"total_upload_time\", \"p95_time\", \"rps\", \"p99_time\", \"mean_time\", \"mean_precisions\"]\nif \"recall_at_1_at_k\" in expanded_df.columns:\n metric_cols.append(\"recall_at_1_at_k\")\n\nagg_df = expanded_df.groupby(group_cols, dropna=False)[metric_cols].median().reset_index()\n\n# Build final JSON results\njson_results = []\nfor _, row in agg_df.iterrows():\n json_object = {\n \"engine_name\": row[\"engine_name\"],\n \"setup_name\": row[\"setup_name\"],\n \"dataset_name\": row[\"dataset_name\"],\n \"search_idx\": row[\"search_idx\"],\n \"upload_time\": row[\"upload_time\"],\n \"total_upload_time\": row[\"total_upload_time\"],\n \"p95_time\": row[\"p95_time\"],\n \"rps\": row[\"rps\"],\n \"parallel\": row[\"parallel\"],\n \"p99_time\": row[\"p99_time\"],\n \"mean_time\": row[\"mean_time\"],\n \"mean_precisions\": row[\"mean_precisions\"],\n \"engine_params\": json.loads(row[\"engine_params\"]),\n }\n\n if \"recall_at_1_at_k\" in row and pd.notna(row.get(\"recall_at_1_at_k\")):\n json_object[\"recall_at_1_at_k\"] = row[\"recall_at_1_at_k\"]\n\n json_results.append(json_object)\n\nformat = '%Y-%M-%dT%H:%M:%S'\nnow = datetime.now().replace(tzinfo=timezone.utc).strftime(format)\n\nPath(\"results.json\").write_text(json.dumps(json_results, indent=2))\nPath(f\"results-{now}.json\").write_text(json.dumps(json_results, indent=2))\n\nprint(json_results[-1], len(json_results))\n\nresults_df = pd.DataFrame(json_results).sort_values(\"p99_time\", ascending=True)\nresults_df" } ], "metadata": { From 63e3400c2c856d3107adff375285dada68c47085 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Wed, 8 Apr 2026 23:34:10 +0200 Subject: [PATCH 20/31] Allow 20 machines at once --- .github/workflows/manual-benchmarks-cascade.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual-benchmarks-cascade.yaml b/.github/workflows/manual-benchmarks-cascade.yaml index 0be800966..56d2e96c4 100644 --- a/.github/workflows/manual-benchmarks-cascade.yaml +++ b/.github/workflows/manual-benchmarks-cascade.yaml @@ -76,7 +76,7 @@ jobs: CONFIG_FILE="benchmark_cascade/benchmark-configs.json" BENCHMARK_SET="${{ inputs.benchmark_set }}" CURRENT_BATCH=${{ inputs.current_batch || 0 }} - BATCH_SIZE=10 + BATCH_SIZE=20 echo "benchmark_set=$BENCHMARK_SET" >> $GITHUB_OUTPUT @@ -234,7 +234,7 @@ jobs: if: inputs.process_results == false runs-on: ubuntu-latest strategy: - max-parallel: 10 + max-parallel: 20 fail-fast: false matrix: config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }} @@ -289,7 +289,7 @@ jobs: contents: read packages: read strategy: - max-parallel: 10 + max-parallel: 20 fail-fast: false matrix: config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }} From a658f128421417b60e58f74aaae9cfa6938b6d76 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Thu, 9 Apr 2026 10:11:08 +0200 Subject: [PATCH 21/31] Freeze dev version and fix engine name --- benchmark_cascade/benchmark-configs.json | 200 +++++++++--------- .../configurations/qdrant-turbo-quant.json | 4 +- 2 files changed, 102 insertions(+), 102 deletions(-) diff --git a/benchmark_cascade/benchmark-configs.json b/benchmark_cascade/benchmark-configs.json index c330b907d..12103ef8a 100644 --- a/benchmark_cascade/benchmark-configs.json +++ b/benchmark_cascade/benchmark-configs.json @@ -304,601 +304,601 @@ ], "turbo_quant": [ { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-no-quant-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-sq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq2-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq-asym-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, { - "qdrant_version": "ghcr/dev", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json index 7d855bbd2..c76649170 100644 --- a/experiments/configurations/qdrant-turbo-quant.json +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -1,7 +1,7 @@ [ { "name": "qdrant-tq6-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, @@ -24,7 +24,7 @@ }, { "name": "qdrant-tq6-plus-m-16-ef-128", - "engine": "qdrant_native", + "engine": "qdrant_hybrid", "connection_params": { "timeout": 30 }, "collection_params": { "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold": 10000000, "default_segment_number": 2 }, From 147abaa3d7cd6b78c499f93d01fc21a0232857bf Mon Sep 17 00:00:00 2001 From: tellet-q Date: Thu, 9 Apr 2026 10:14:43 +0200 Subject: [PATCH 22/31] Add tq6 cascade set --- benchmark_cascade/benchmark-configs.json | 242 +++++++++++++++++++++++ 1 file changed, 242 insertions(+) diff --git a/benchmark_cascade/benchmark-configs.json b/benchmark_cascade/benchmark-configs.json index 12103ef8a..4b02c28ad 100644 --- a/benchmark_cascade/benchmark-configs.json +++ b/benchmark_cascade/benchmark-configs.json @@ -2467,6 +2467,248 @@ "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true } + ], + "turbo_quant_tq6": [ + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + }, + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq6-plus-m-16-ef-128", + "feature_flags_all": true + } ] } } \ No newline at end of file From 358c2cc668530685166ed44cc54847cc7aff1587 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Thu, 9 Apr 2026 10:25:32 +0200 Subject: [PATCH 23/31] Remove tq6 from turbo_quant --- benchmark_cascade/benchmark-configs.json | 240 ----------------------- 1 file changed, 240 deletions(-) diff --git a/benchmark_cascade/benchmark-configs.json b/benchmark_cascade/benchmark-configs.json index 4b02c28ad..bac2533c3 100644 --- a/benchmark_cascade/benchmark-configs.json +++ b/benchmark_cascade/benchmark-configs.json @@ -903,66 +903,6 @@ "engine_config": "qdrant-bq-m-16-ef-128", "feature_flags_all": true }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "cohere-wiki-100k-no-filters", @@ -1144,66 +1084,6 @@ "engine_config": "qdrant-tq1-plus-m-16-ef-128", "feature_flags_all": true }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "glove-100-angular", @@ -1385,66 +1265,6 @@ "engine_config": "qdrant-tq1-plus-m-16-ef-128", "feature_flags_all": true }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "laion-small-clip", @@ -1626,66 +1446,6 @@ "engine_config": "qdrant-tq1-plus-m-16-ef-128", "feature_flags_all": true }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true - }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "dbpedia-openai-100K-1536-angular", From c67cd34310d919c347e204e2d2f1412f4256b527 Mon Sep 17 00:00:00 2001 From: Ivan Pleshkov Date: Thu, 9 Apr 2026 11:21:06 +0200 Subject: [PATCH 24/31] line for orig vectors --- experiments/configurations/qdrant-turbo-quant.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json index c76649170..349b6906d 100644 --- a/experiments/configurations/qdrant-turbo-quant.json +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -274,7 +274,9 @@ "hnsw_config": { "m": 16, "ef_construct": 128 } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64 } } + { "parallel": 100, "config": { "hnsw_ef": 64 } }, + { "parallel": 100, "config": { "hnsw_ef": 128 } }, + { "parallel": 100, "config": { "hnsw_ef": 256 } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } } From 3bdac8bea9605a5570a26e3beae7b8d9c59761b0 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Thu, 9 Apr 2026 11:23:40 +0200 Subject: [PATCH 25/31] Enable repeats and clean results --- .../workflows/manual-benchmarks-cascade.yaml | 12 + benchmark_cascade/benchmark-configs.json | 2161 ++--------------- 2 files changed, 263 insertions(+), 1910 deletions(-) diff --git a/.github/workflows/manual-benchmarks-cascade.yaml b/.github/workflows/manual-benchmarks-cascade.yaml index 56d2e96c4..fd59a456e 100644 --- a/.github/workflows/manual-benchmarks-cascade.yaml +++ b/.github/workflows/manual-benchmarks-cascade.yaml @@ -102,6 +102,9 @@ jobs: exit 1 fi + # Expand per-item "repeat" field: each item with "repeat": N becomes N copies (default 1) + BENCHMARK_CONFIGS=$(jq '[.[] | . as $c | range($c.repeat // 1) | $c | del(.repeat)]' <<< "$BENCHMARK_CONFIGS") + # Apply params_override if provided PARAMS_OVERRIDE='${{ inputs.params_override }}' if [ "$PARAMS_OVERRIDE" != "{}" ]; then @@ -311,6 +314,15 @@ jobs: job_index: ${{ strategy.job-index }} machines_info: ${{ (inputs.current_batch != 0 && inputs.machines_info) || needs.prepareBenchmarks.outputs.machines_info }} + - name: Clear previous results on remote client + continue-on-error: true + run: | + source tools/ssh.sh + CLOUD_NAME=${CLOUD_NAME:-"hetzner"} + CLIENT_NAME=${{ steps.extract_names.outputs.client_name }} + IP_OF_THE_CLIENT=$(bash "tools/${CLOUD_NAME}/get_public_ip.sh" "$CLIENT_NAME") + ssh_with_retry "root@${IP_OF_THE_CLIENT}" "rm -rf ~/results" + - name: Run benchmark run: | export QDRANT_VERSION=${{ matrix.config.qdrant_version }} diff --git a/benchmark_cascade/benchmark-configs.json b/benchmark_cascade/benchmark-configs.json index bac2533c3..c25d01769 100644 --- a/benchmark_cascade/benchmark-configs.json +++ b/benchmark_cascade/benchmark-configs.json @@ -307,2167 +307,508 @@ "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "laion-small-clip", "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq4-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq4-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq2-m-16-ef-128", - "feature_flags_all": true - }, - - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq2-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq1-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq1-plus-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "qdrant_version": "ghcr/dev-22cf04367be1c13b0dfb668fd20bf74fabb3e8de", "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true - } - ], - "turbo_quant_tq6": [ - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq4-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, + { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq2-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-tq1-plus-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "dbpedia-openai-100K-1536-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "laion-small-clip", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-no-quant-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-sq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq2-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-asym-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-bq-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 + } + ], + "turbo_quant_tq6": [ + { + "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", + "dataset": "cohere-wiki-100k-no-filters", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "dataset": "glove-100-angular", + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "laion-small-clip", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "engine_config": "qdrant-tq6-m-16-ef-128", + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", + "dataset": "cohere-wiki-100k-no-filters", "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", + "dataset": "glove-100-angular", "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", + "dataset": "laion-small-clip", "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 }, { "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", "dataset": "dbpedia-openai-100K-1536-angular", "engine_config": "qdrant-tq6-plus-m-16-ef-128", - "feature_flags_all": true + "feature_flags_all": true, + "repeat": 9 } ] } From 751bba78e0483c40e2e5ce79911d92bfdb5ab6e1 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Thu, 9 Apr 2026 11:40:51 +0200 Subject: [PATCH 26/31] Use 12 machines --- .github/workflows/manual-benchmarks-cascade.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual-benchmarks-cascade.yaml b/.github/workflows/manual-benchmarks-cascade.yaml index fd59a456e..d9c2beb5d 100644 --- a/.github/workflows/manual-benchmarks-cascade.yaml +++ b/.github/workflows/manual-benchmarks-cascade.yaml @@ -76,7 +76,7 @@ jobs: CONFIG_FILE="benchmark_cascade/benchmark-configs.json" BENCHMARK_SET="${{ inputs.benchmark_set }}" CURRENT_BATCH=${{ inputs.current_batch || 0 }} - BATCH_SIZE=20 + BATCH_SIZE=12 echo "benchmark_set=$BENCHMARK_SET" >> $GITHUB_OUTPUT @@ -237,7 +237,7 @@ jobs: if: inputs.process_results == false runs-on: ubuntu-latest strategy: - max-parallel: 20 + max-parallel: 12 fail-fast: false matrix: config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }} @@ -292,7 +292,7 @@ jobs: contents: read packages: read strategy: - max-parallel: 20 + max-parallel: 12 fail-fast: false matrix: config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }} From ba78022b9868a44c0a4422828ebe7308798cd9b8 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Thu, 9 Apr 2026 11:44:51 +0200 Subject: [PATCH 27/31] Fix cleanup --- .github/workflows/manual-benchmarks-cascade.yaml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/manual-benchmarks-cascade.yaml b/.github/workflows/manual-benchmarks-cascade.yaml index d9c2beb5d..f20f6b552 100644 --- a/.github/workflows/manual-benchmarks-cascade.yaml +++ b/.github/workflows/manual-benchmarks-cascade.yaml @@ -314,15 +314,6 @@ jobs: job_index: ${{ strategy.job-index }} machines_info: ${{ (inputs.current_batch != 0 && inputs.machines_info) || needs.prepareBenchmarks.outputs.machines_info }} - - name: Clear previous results on remote client - continue-on-error: true - run: | - source tools/ssh.sh - CLOUD_NAME=${CLOUD_NAME:-"hetzner"} - CLIENT_NAME=${{ steps.extract_names.outputs.client_name }} - IP_OF_THE_CLIENT=$(bash "tools/${CLOUD_NAME}/get_public_ip.sh" "$CLIENT_NAME") - ssh_with_retry "root@${IP_OF_THE_CLIENT}" "rm -rf ~/results" - - name: Run benchmark run: | export QDRANT_VERSION=${{ matrix.config.qdrant_version }} @@ -341,6 +332,13 @@ jobs: fi bash -x tools/setup_ci.sh + + # Clear previous results on remote client to avoid mixing data from different runs + source tools/ssh.sh + CLOUD_NAME=${CLOUD_NAME:-"hetzner"} + IP_OF_THE_CLIENT=$(bash "tools/${CLOUD_NAME}/get_public_ip.sh" "$CLIENT_NAME") + ssh_with_retry "root@${IP_OF_THE_CLIENT}" "rm -rf ~/results" || true + bash -x tools/run_ci.sh - name: Upload benchmark results From 7010d34404184232debecabafd5419e421b6f476 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Thu, 9 Apr 2026 11:50:18 +0200 Subject: [PATCH 28/31] Fix cleanup --- .github/workflows/manual-benchmarks-cascade.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual-benchmarks-cascade.yaml b/.github/workflows/manual-benchmarks-cascade.yaml index f20f6b552..f17312246 100644 --- a/.github/workflows/manual-benchmarks-cascade.yaml +++ b/.github/workflows/manual-benchmarks-cascade.yaml @@ -337,7 +337,7 @@ jobs: source tools/ssh.sh CLOUD_NAME=${CLOUD_NAME:-"hetzner"} IP_OF_THE_CLIENT=$(bash "tools/${CLOUD_NAME}/get_public_ip.sh" "$CLIENT_NAME") - ssh_with_retry "root@${IP_OF_THE_CLIENT}" "rm -rf ~/results" || true + ssh_with_retry -o StrictHostKeyChecking=no "root@${IP_OF_THE_CLIENT}" "rm -rf ~/results" || true bash -x tools/run_ci.sh From 46115a8998e0d46eff38443b78e747e5bf80d47a Mon Sep 17 00:00:00 2001 From: Ivan Pleshkov Date: Fri, 10 Apr 2026 08:49:57 +0200 Subject: [PATCH 29/31] less tests, fix line --- benchmark_cascade/benchmark-configs.json | 140 ------------------ .../configurations/qdrant-turbo-quant.json | 4 +- 2 files changed, 3 insertions(+), 141 deletions(-) diff --git a/benchmark_cascade/benchmark-configs.json b/benchmark_cascade/benchmark-configs.json index c25d01769..e3ecf81a8 100644 --- a/benchmark_cascade/benchmark-configs.json +++ b/benchmark_cascade/benchmark-configs.json @@ -611,146 +611,6 @@ "engine_config": "qdrant-tq1-plus-m-16-ef-128", "feature_flags_all": true, "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "dbpedia-openai-100K-1536-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "laion-small-clip", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "glove-100-angular", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-no-quant-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-sq-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq2-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-asym-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 - }, - { - "qdrant_version": "ghcr/llama-turbo-quant-6bf64369f479f2c06298351db6e34ac6a142e564", - "dataset": "cohere-wiki-100k-no-filters", - "engine_config": "qdrant-bq-m-16-ef-128", - "feature_flags_all": true, - "repeat": 9 } ], "turbo_quant_tq6": [ diff --git a/experiments/configurations/qdrant-turbo-quant.json b/experiments/configurations/qdrant-turbo-quant.json index 349b6906d..845cb554f 100644 --- a/experiments/configurations/qdrant-turbo-quant.json +++ b/experiments/configurations/qdrant-turbo-quant.json @@ -62,7 +62,9 @@ } }, "search_params": [ - { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } } + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": false } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "config": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } } ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, From 02697794018cec7fd280423518a404a2c8722e8c Mon Sep 17 00:00:00 2001 From: tellet-q Date: Fri, 10 Apr 2026 08:52:50 +0200 Subject: [PATCH 30/31] Shuffle --- .../workflows/manual-benchmarks-cascade.yaml | 21 +++++++++++++--- scripts/plots.html | 24 ++++++++++++------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.github/workflows/manual-benchmarks-cascade.yaml b/.github/workflows/manual-benchmarks-cascade.yaml index f17312246..05ef65eb1 100644 --- a/.github/workflows/manual-benchmarks-cascade.yaml +++ b/.github/workflows/manual-benchmarks-cascade.yaml @@ -76,7 +76,7 @@ jobs: CONFIG_FILE="benchmark_cascade/benchmark-configs.json" BENCHMARK_SET="${{ inputs.benchmark_set }}" CURRENT_BATCH=${{ inputs.current_batch || 0 }} - BATCH_SIZE=12 + BATCH_SIZE=20 echo "benchmark_set=$BENCHMARK_SET" >> $GITHUB_OUTPUT @@ -114,6 +114,10 @@ jobs: OVERRIDE_PARAMS=$(echo "$PARAMS_OVERRIDE" | jq -r '.params // {}') if [ "$OVERRIDE_PARAMS" != "{}" ]; then + # Extract and remove repeat before passing to generate_configs.py + OVERRIDE_REPEAT=$(echo "$OVERRIDE_PARAMS" | jq -r '.repeat // 1') + OVERRIDE_PARAMS=$(echo "$OVERRIDE_PARAMS" | jq 'del(.repeat)') + # Generate all parameter combinations using separate Python script if ! BENCHMARK_CONFIGS=$(python3 benchmark_cascade/generate_configs.py "$OVERRIDE_PARAMS" "$BENCHMARK_CONFIGS"); then echo "Error: Failed to generate benchmark configurations" @@ -121,9 +125,20 @@ jobs: fi echo "Generated $(echo "$BENCHMARK_CONFIGS" | jq length) configurations after applying params_override" + + # Apply override repeat to all generated configs + if [ "$OVERRIDE_REPEAT" -gt 1 ]; then + BENCHMARK_CONFIGS=$(jq --argjson n "$OVERRIDE_REPEAT" '[.[] as $c | range($n) | $c]' <<< "$BENCHMARK_CONFIGS") + echo "Applied repeat=$OVERRIDE_REPEAT, total configs: $(echo "$BENCHMARK_CONFIGS" | jq length)" + fi fi fi + # Shuffle so that repeated configs land on different machines/batches + # Use the first workflow run ID as seed for deterministic shuffle across batches + SHUFFLE_SEED=$(echo "$WORKFLOW_RUN_IDS" | cut -d',' -f1) + BENCHMARK_CONFIGS=$(python3 -c "import json, random, sys; d=json.load(sys.stdin); random.seed(int(sys.argv[1])); random.shuffle(d); print(json.dumps(d))" "$SHUFFLE_SEED" <<< "$BENCHMARK_CONFIGS") + TOTAL_CONFIGS=$(echo "$BENCHMARK_CONFIGS" | jq length) START_INDEX=$((CURRENT_BATCH * BATCH_SIZE)) END_INDEX=$((START_INDEX + BATCH_SIZE - 1)) @@ -237,7 +252,7 @@ jobs: if: inputs.process_results == false runs-on: ubuntu-latest strategy: - max-parallel: 12 + max-parallel: 20 fail-fast: false matrix: config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }} @@ -292,7 +307,7 @@ jobs: contents: read packages: read strategy: - max-parallel: 12 + max-parallel: 20 fail-fast: false matrix: config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }} diff --git a/scripts/plots.html b/scripts/plots.html index 4ba1f8e72..62f708365 100644 --- a/scripts/plots.html +++ b/scripts/plots.html @@ -358,8 +358,8 @@

]; // Utility function to get filename without extension and 'results-' prefix - function getFilenameWithoutExtension(filename) { - if (!filename) return ''; + function parseFilename(filename) { + if (!filename) return { version: '', commit: '' }; // Remove extension const lastDotIndex = filename.lastIndexOf('.'); let name = lastDotIndex === -1 ? filename : filename.substring(0, lastDotIndex); @@ -371,9 +371,12 @@

if (name.startsWith('ghcr-')) { name = name.substring(5); } - // Trim commit hash to first 4 characters (pattern: 40-char hex at end) - name = name.replace(/([0-9a-f]{4})[0-9a-f]{36,}$/, '$1'); - return name; + // Extract commit hash (40-char hex at end) and version (everything before it) + const match = name.match(/^(.+?)-([0-9a-f]{40})$/); + if (match) { + return { version: match[1], commit: match[2].substring(0, 10) }; + } + return { version: name, commit: '' }; } // State @@ -518,6 +521,7 @@

data: [], filename: '', label: '', + commit: '', color: color }; datasets.push(dataset); @@ -636,8 +640,10 @@

${dataset.label}:

File: ${dataset.filename || 'No file'}
-
+
Engines: ${engines.size}
Datasets: ${datasetNames.size}
Records: ${dataset.data.length}
+ ${commitInfo}
`; datasetSummariesContainer.appendChild(summaryDiv); From d5883ad29ae32d0fdf4d7858fac43d056fdb90e8 Mon Sep 17 00:00:00 2001 From: tellet-q Date: Fri, 10 Apr 2026 08:56:49 +0200 Subject: [PATCH 31/31] Reduce number of machines --- .github/workflows/manual-benchmarks-cascade.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual-benchmarks-cascade.yaml b/.github/workflows/manual-benchmarks-cascade.yaml index 05ef65eb1..016962c15 100644 --- a/.github/workflows/manual-benchmarks-cascade.yaml +++ b/.github/workflows/manual-benchmarks-cascade.yaml @@ -76,7 +76,7 @@ jobs: CONFIG_FILE="benchmark_cascade/benchmark-configs.json" BENCHMARK_SET="${{ inputs.benchmark_set }}" CURRENT_BATCH=${{ inputs.current_batch || 0 }} - BATCH_SIZE=20 + BATCH_SIZE=12 echo "benchmark_set=$BENCHMARK_SET" >> $GITHUB_OUTPUT @@ -252,7 +252,7 @@ jobs: if: inputs.process_results == false runs-on: ubuntu-latest strategy: - max-parallel: 20 + max-parallel: 12 fail-fast: false matrix: config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }} @@ -307,7 +307,7 @@ jobs: contents: read packages: read strategy: - max-parallel: 20 + max-parallel: 12 fail-fast: false matrix: config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }}