From fdcce8b5d6f6d31e408d24f0805612891e425f97 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 24 Mar 2025 09:50:38 +0000 Subject: [PATCH 1/7] Added FastAPI router for Rust package repository endpoints; added endpoint to download config.json file from --- src/murfey/server/api/bootstrap.py | 43 +++++++++++++++++++++++++++++- src/murfey/server/main.py | 1 + 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/murfey/server/api/bootstrap.py b/src/murfey/server/api/bootstrap.py index 5a9661b34..86a471d37 100644 --- a/src/murfey/server/api/bootstrap.py +++ b/src/murfey/server/api/bootstrap.py @@ -15,6 +15,8 @@ from __future__ import annotations import functools +import io +import json import logging import random import re @@ -23,7 +25,7 @@ import packaging.version import requests from fastapi import APIRouter, HTTPException, Request, Response -from fastapi.responses import FileResponse, HTMLResponse +from fastapi.responses import FileResponse, HTMLResponse, StreamingResponse import murfey from murfey.server import get_machine_config, respond_with_template @@ -43,6 +45,7 @@ bootstrap = APIRouter(prefix="/bootstrap", tags=["bootstrap"]) cygwin = APIRouter(prefix="/cygwin", tags=["bootstrap"]) msys2 = APIRouter(prefix="/msys2", tags=["bootstrap"]) +rust = APIRouter(prefix="/rust", tags=["bootstrap"]) pypi = APIRouter(prefix="/pypi", tags=["bootstrap"]) plugins = APIRouter(prefix="/plugins", tags=["bootstrap"]) @@ -565,6 +568,44 @@ def get_msys2_package_file( raise HTTPException(status_code=package_file.status_code) +""" +======================================================================================= +RUST-RELATED FUNCTIONS AND ENDPOINTS +======================================================================================= +""" + +rust_dl = "https://static.crates.io/crates" +rust_api = "https://crates.io" + + +@rust.get("/cargo/config.json", response_class=StreamingResponse) +def get_maturin_config(request: Request): + """ + Download a config.json file used by Maturin that is used when integrating Rust + backends for Python packages. This file will determine where Maturin sources + Rust backend packages from. + + This config is to be saved in ~/.cargo/registry/config.json + """ + + # Construct config file with the necessary endpoints + config = { + "dl": f"{request.url.scheme}://{request.url.netloc}/crates", + "api": f"{request.url.scheme}://{request.url.netloc}/api/crates", + # "proxy": f"{request.url.scheme}://{request.url.netloc}", + } + + # Save it as a JSON and return it as part of the response + json_data = json.dumps(config, indent=4) + json_bytes = io.BytesIO(json_data.encode("utf-8")) + + return StreamingResponse( + json_bytes, + media_type="application/json", + headers={"Content-Disposition": "attachment; filename=config.json"}, + ) + + """ ======================================================================================= PYPI-RELATED FUNCTIONS AND ENDPOINTS diff --git a/src/murfey/server/main.py b/src/murfey/server/main.py index 96533fd7d..d589bc174 100644 --- a/src/murfey/server/main.py +++ b/src/murfey/server/main.py @@ -65,6 +65,7 @@ class Settings(BaseSettings): app.include_router(murfey.server.api.bootstrap.bootstrap) app.include_router(murfey.server.api.bootstrap.cygwin) app.include_router(murfey.server.api.bootstrap.msys2) +app.include_router(murfey.server.api.bootstrap.rust) app.include_router(murfey.server.api.bootstrap.pypi) app.include_router(murfey.server.api.bootstrap.plugins) app.include_router(murfey.server.api.clem.router) From c3f5db887b1840c9794b4a1ed341efb8c5c9854d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 26 Mar 2025 17:16:47 +0000 Subject: [PATCH 2/7] Added API endpoints to facilitate setting up and download Rust packages from our server --- src/murfey/server/api/bootstrap.py | 369 ++++++++++++++++++++++++++++- 1 file changed, 359 insertions(+), 10 deletions(-) diff --git a/src/murfey/server/api/bootstrap.py b/src/murfey/server/api/bootstrap.py index 86a471d37..3952340e3 100644 --- a/src/murfey/server/api/bootstrap.py +++ b/src/murfey/server/api/bootstrap.py @@ -24,7 +24,7 @@ import packaging.version import requests -from fastapi import APIRouter, HTTPException, Request, Response +from fastapi import APIRouter, HTTPException, Query, Request, Response from fastapi.responses import FileResponse, HTMLResponse, StreamingResponse import murfey @@ -51,6 +51,8 @@ logger = logging.getLogger("murfey.server.api.bootstrap") +# Create a reusable HTTP session to avoid spamming external endpoints +http_session = requests.Session() """ ======================================================================================= @@ -574,25 +576,86 @@ def get_msys2_package_file( ======================================================================================= """ +# Base URLs to use +rust_index = "https://index.crates.io" rust_dl = "https://static.crates.io/crates" rust_api = "https://crates.io" -@rust.get("/cargo/config.json", response_class=StreamingResponse) -def get_maturin_config(request: Request): +@rust.get("/cargo/config.toml", response_class=StreamingResponse) +def get_cargo_config(request: Request): """ - Download a config.json file used by Maturin that is used when integrating Rust - backends for Python packages. This file will determine where Maturin sources - Rust backend packages from. + Returns a properly configured Cargo config that sets it to look ONLY at the + crates.io mirror. - This config is to be saved in ~/.cargo/registry/config.json + This file is saved as ~/.cargo/config.toml on Linux devices by default, but + will be saved as %USERPROFILE%\\.cargo\\config.toml on Windows ones. """ + # Construct URL to our mirror of the Rust sparse index + index_mirror = ( + f"{request.url.scheme}://{request.url.netloc}/{rust.prefix.strip('/')}/index/" + ) + + # Construct and return the config.toml file + config_data = "\n".join( + [ + "[registries.murfey-crates]", + f'index = "sparse+{index_mirror}"', # sparse+ to use sparse index logic + "", + "[registry]", + 'default = "murfey-crates"', # Use our registry as default + "", + "[registries.crates-io]", + 'index = "false"', # Disables using crates.io + ] + ) + config_bytes = io.BytesIO(config_data.encode("utf-8")) + + return StreamingResponse( + config_bytes, + media_type="application/toml+json", + headers={"Content-Disposition": "attachment; filename=config.toml"}, + ) + + +@rust.get("/index") +def get_index_page(): + """ + Returns a mirror of the https://index.crates.io landing page. + """ + + response = http_session.get(rust_index) + if response.status_code != 200: + raise HTTPException(status_code=response.status_code) + return Response( + content=response.content, + media_type=response.headers.get("Content-Type"), + status_code=response.status_code, + ) + + +@rust.get("/index/config.json", response_class=StreamingResponse) +def get_index_config(request: Request): + """ + Download a config.json file used by Cargo to navigate sparse index registries + with. + + The 'dl' key points to our mirror of the static crates.io repository, while + the 'api' key points to an API version of that same registry. Both will be + used by Cargo when searching for and downloading packages. + """ + + print(f"Received request to access {str(request.url)}") + + # Construct URL for Rust router + base_url = f"{request.url.scheme}://{request.url.netloc}" + rust.prefix + print(f"Base URL is {base_url}") + # Construct config file with the necessary endpoints config = { - "dl": f"{request.url.scheme}://{request.url.netloc}/crates", - "api": f"{request.url.scheme}://{request.url.netloc}/api/crates", - # "proxy": f"{request.url.scheme}://{request.url.netloc}", + "dl": f"{base_url}/crates", + "api": f"{base_url}", } # Save it as a JSON and return it as part of the response @@ -606,6 +669,292 @@ def get_maturin_config(request: Request): ) +@rust.get("/index/{c1}/{c2}/{package}", response_class=StreamingResponse) +def get_index_package_metadata( + request: Request, + c1: str, + c2: str, + package: str, +): + """ + Download the metadata for a given package from the crates.io sparse index. + The path to the metadata file on the server side takes the following form: + /{c1}/{c2}/{package} + + c1 and c2 are 2 characters-long strings that are taken from the first 4 + characters of the package name (a-z, A-Z, 0-9, -, _). For 3-letter packages, + c1 = 3, and c2 is the first character of the package. + """ + + print(f"Received request to access {str(request.url)}") + + # Validate path to the package metadata + if not all(bool(re.fullmatch(r"[\w\-]{1,2}", char)) for char in (c1, c2)): + raise ValueError("Invalid path to package metadata") + + if len(c1) == 1 and not c1 == "3": + raise ValueError("Invalid path to package metadata") + if c1 == "3" and not len(c2) == 1: + raise ValueError("Invalid path to package metadata") + + if not bool(re.fullmatch(r"[\w\-]+", package)): + raise ValueError("Invalid package name") + + # Request and return the metadata as a JSON file + url = f"{rust_index}/{c1}/{c2}/{package}" + response = http_session.get(url) + if response.status_code != 200: + raise HTTPException(status_code=response.status_code) + return StreamingResponse( + response.iter_content(chunk_size=8192), + media_type="application/json", + ) + + +@rust.get("/index/{n}/{package}", response_class=StreamingResponse) +def get_index_package_metadata_for_short_package_names( + request: Request, + n: str, + package: str, +): + """ + The Rust sparse index' naming scheme for packages with 1-2 characters is + different from the standard path convention. They are stored under + /1/{package} or /2/{package}. + """ + + print(f"Received request to access {str(request.url)}") + + # Validate path to crate + if n not in ("1", "2"): + raise ValueError("Invalid path to package metadata") + if not bool(re.fullmatch(r"[\w\-]{1,2}", package)): + raise ValueError("Invalid package name") + + # Request and return the metadata as a JSON file + url = f"{rust_index}/{n}/{package}" + response = http_session.get(url) + if response.status_code != 200: + raise HTTPException(status_code=response.status_code) + return StreamingResponse( + response.iter_content(chunk_size=8192), + media_type="application/json", + ) + + +@rust.get("/api/v1/crates") +def get_rust_api_package_index( + request: Request, + package: str = Query(None, alias="q"), + per_page: int = Query(10), + cursor: str = Query(None, alias="seek"), +): + """ + Displays the Rust API package index, which returns names of available packages + in a JSON object based on the search query given. + """ + + print(f"Received request to access {str(request.url)}") + + # Validate package name + if package and not bool(re.fullmatch(r"[\w\-]+", package)): + raise ValueError("Invalid package name") + if cursor and not bool(re.fullmatch(r"[a-zA-Z0-9]+", cursor)): + raise ValueError("Invalid cursor") + + # Formulate the search query to pass to the crates page + search_params: dict[str, str | int] = {} + if package: + search_params["q"] = package + if per_page: + search_params["per_page"] = per_page + if cursor: + search_params["seek"] = cursor + + # Submit request and return response + url = f"{rust_api}/api/v1/crates" + response = http_session.get(url, params=search_params) + if response.status_code != 200: + raise HTTPException(status_code=response.status_code) + return response.json() + + +@rust.get("/api/v1/crates/{package}") +def get_rust_api_package_info( + request: Request, + package: str, +): + """ + Displays general information for a given Rust package, as a JSON object. + Contains both version information and download information, in addition + to other types of metadata. + """ + + print(f"Received request to access {str(request.url)}") + + # Validate package name + if not bool(re.fullmatch(r"[\w\-]+", package)): + raise ValueError("Invalid package name") + + # Return JSON of the package's page + url = f"{rust_api}/api/v1/crates/{package}" + response = http_session.get(url) + if response.status_code != 200: + raise HTTPException(status_code=response.status_code) + return response.json() + + +@rust.get("/api/v1/crates/{package}/versions") +def get_rust_api_package_versions( + request: Request, + package: str, +): + """ + Displays all available versions for a particular Rust package, along with download + links for said versions. + """ + + print(f"Received request to access {str(request.url)}") + + # Validate crate name + if not bool(re.fullmatch(r"[\w\-]+", package)): + raise ValueError("Invalid package name") + + # Return JSON of the package's version information + url = f"{rust_api}/api/v1/crates/{package}/versions" + response = http_session.get(url) + if response.status_code != 200: + raise HTTPException(status_code=response.status_code) + return response.json() + + +@rust.get( + "/api/v1/crates/{package}/{version}/download", response_class=StreamingResponse +) +def get_rust_api_package_download( + request: Request, + package: str, + version: str, +): + """ + Obtain and pass through a crate download request for a specific Rust package. + """ + + print(f"Received request to access {str(request.url)}") + + # Validate package name + if not bool(re.fullmatch(r"[\w\-]+", package)): + raise ValueError("Invalid package name") + # Validate version number + # Not all developers adhere to guidelines when versioning their packages, so + # '-', '_', '+', as well as letters can also be present in this field. + if not bool(re.fullmatch(r"[\w\-\.\+]+", version)): + raise ValueError("Invalid version number") + + # Request and return package + url = f"{rust_api}/api/v1/crates/{package}/{version}/download" + response = http_session.get(url) + file_name = f"{package}-{version}.crate" # Construct crate name to save as + if response.status_code != 200: + raise HTTPException(status_code=response.status_code) + return StreamingResponse( + content=response.iter_content(chunk_size=8192), + headers={ + "Content-Disposition": f'attachment; filename="{file_name}"', + "Content-Type": response.headers.get( + "Content-Type", "application/octet-stream" + ), + "Content-Length": response.headers.get("Content-Length"), + }, + status_code=response.status_code, + ) + + +@rust.get("/crates/{package}/{version}/download", response_class=StreamingResponse) +def get_rust_package_download( + request: Request, + package: str, + version: str, +): + """ + Obtain and pass through a crate download request for a Rust package. + """ + + print(f"Received request to access {str(request.url)}") + + # Validate package and version + if not bool(re.fullmatch(r"[\w\-]+", package)): + raise ValueError("Invalid package name") + if not bool(re.fullmatch(r"[\w\-\.\+]+", version)): + raise ValueError("Invalid version number") + + # Request and return crate from https://static.crates.io + url = f"{rust_dl}/{package}/{version}/download" + response = http_session.get(url) + file_name = f"{package}-{version}.crate" # Construct file name to save package as + if response.status_code != 200: + raise HTTPException(status_code=response.status_code) + return StreamingResponse( + content=response.iter_content(chunk_size=8192), + headers={ + "Content-Disposition": f'attachment; filename="{file_name}"', + "Content-Type": response.headers.get( + "Content-Type", "application/octet-stream" + ), + "Content-Length": response.headers.get("Content-Length"), + }, + status_code=response.status_code, + ) + + +@rust.get("/crates/{package}/{crate}", response_class=StreamingResponse) +def get_rust_package_crate( + request: Request, + package: str, + crate: str, +): + """ + Obtain and pass through a download for a specific Rust crate. The Rust API + download request actually redirects to the static crate repository, so this + endpoint covers cases where the static crate download link is requested. + + The static Rust package repository has been configured such that only requests + for a specific crate are accepted and handled. + (e.g. https://static.crates.io/crates/anyhow/anyhow-1.0.97.crate will pass) + + A request for any other part of the URL path will be denied. + (e.g. https://static.crates.io/crates/anyhow will fail) + """ + + print(f"Received request to access {str(request.url)}") + + # Validate crate and package names + if not bool(re.fullmatch(r"[\w\-]+", package)): + raise ValueError("Invalid package name") + if not crate.endswith(".crate"): + raise ValueError("This is a not a Rust crate") + # Rust crates follow a '{crate}-{version}.crate' structure + if not bool(re.fullmatch(r"[\w\-]+\-[0-9\.]+\.crate", crate)): + raise ValueError("Invalid crate name") + + # Request and return package + url = f"{rust_dl}/{package}/{crate}" + response = http_session.get(url) + if response.status_code != 200: + raise HTTPException(status_code=response.status_code) + return StreamingResponse( + content=response.iter_content(), + headers={ + "Content-Disposition": f'attachment; filename="{crate}"', + "Content-Type": response.headers.get( + "Content-Type", "application/octet-stream" + ), + "Content-Length": response.headers.get("Content-Length"), + }, + status_code=response.status_code, + ) + + """ ======================================================================================= PYPI-RELATED FUNCTIONS AND ENDPOINTS From fa11f393469f84da6386604fa8655c25b5c72ca8 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 26 Mar 2025 17:41:16 +0000 Subject: [PATCH 3/7] Converted print statements to debug logs for Rust endpoints that require validation --- src/murfey/server/api/bootstrap.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/murfey/server/api/bootstrap.py b/src/murfey/server/api/bootstrap.py index 3952340e3..6075fc05a 100644 --- a/src/murfey/server/api/bootstrap.py +++ b/src/murfey/server/api/bootstrap.py @@ -646,11 +646,8 @@ def get_index_config(request: Request): used by Cargo when searching for and downloading packages. """ - print(f"Received request to access {str(request.url)}") - # Construct URL for Rust router base_url = f"{request.url.scheme}://{request.url.netloc}" + rust.prefix - print(f"Base URL is {base_url}") # Construct config file with the necessary endpoints config = { @@ -686,7 +683,7 @@ def get_index_package_metadata( c1 = 3, and c2 is the first character of the package. """ - print(f"Received request to access {str(request.url)}") + logger.debug(f"Received request to access {str(request.url)}") # Validate path to the package metadata if not all(bool(re.fullmatch(r"[\w\-]{1,2}", char)) for char in (c1, c2)): @@ -723,7 +720,7 @@ def get_index_package_metadata_for_short_package_names( /1/{package} or /2/{package}. """ - print(f"Received request to access {str(request.url)}") + logger.debug(f"Received request to access {str(request.url)}") # Validate path to crate if n not in ("1", "2"): @@ -754,7 +751,7 @@ def get_rust_api_package_index( in a JSON object based on the search query given. """ - print(f"Received request to access {str(request.url)}") + logger.debug(f"Received request to access {str(request.url)}") # Validate package name if package and not bool(re.fullmatch(r"[\w\-]+", package)): @@ -790,7 +787,7 @@ def get_rust_api_package_info( to other types of metadata. """ - print(f"Received request to access {str(request.url)}") + logger.debug(f"Received request to access {str(request.url)}") # Validate package name if not bool(re.fullmatch(r"[\w\-]+", package)): @@ -814,7 +811,7 @@ def get_rust_api_package_versions( links for said versions. """ - print(f"Received request to access {str(request.url)}") + logger.debug(f"Received request to access {str(request.url)}") # Validate crate name if not bool(re.fullmatch(r"[\w\-]+", package)): @@ -840,7 +837,7 @@ def get_rust_api_package_download( Obtain and pass through a crate download request for a specific Rust package. """ - print(f"Received request to access {str(request.url)}") + logger.debug(f"Received request to access {str(request.url)}") # Validate package name if not bool(re.fullmatch(r"[\w\-]+", package)): @@ -880,7 +877,7 @@ def get_rust_package_download( Obtain and pass through a crate download request for a Rust package. """ - print(f"Received request to access {str(request.url)}") + logger.debug(f"Received request to access {str(request.url)}") # Validate package and version if not bool(re.fullmatch(r"[\w\-]+", package)): @@ -926,7 +923,7 @@ def get_rust_package_crate( (e.g. https://static.crates.io/crates/anyhow will fail) """ - print(f"Received request to access {str(request.url)}") + logger.debug(f"Received request to access {str(request.url)}") # Validate crate and package names if not bool(re.fullmatch(r"[\w\-]+", package)): From 09951868a7a4e60a3976ea268874c200feba7a9a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 27 Mar 2025 11:16:48 +0000 Subject: [PATCH 4/7] bool() cast during URL path validation not needed --- src/murfey/server/api/bootstrap.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/murfey/server/api/bootstrap.py b/src/murfey/server/api/bootstrap.py index 6075fc05a..5164a33ca 100644 --- a/src/murfey/server/api/bootstrap.py +++ b/src/murfey/server/api/bootstrap.py @@ -686,7 +686,7 @@ def get_index_package_metadata( logger.debug(f"Received request to access {str(request.url)}") # Validate path to the package metadata - if not all(bool(re.fullmatch(r"[\w\-]{1,2}", char)) for char in (c1, c2)): + if any(not re.fullmatch(r"[\w\-]{1,2}", char) for char in (c1, c2)): raise ValueError("Invalid path to package metadata") if len(c1) == 1 and not c1 == "3": @@ -694,7 +694,7 @@ def get_index_package_metadata( if c1 == "3" and not len(c2) == 1: raise ValueError("Invalid path to package metadata") - if not bool(re.fullmatch(r"[\w\-]+", package)): + if not re.fullmatch(r"[\w\-]+", package): raise ValueError("Invalid package name") # Request and return the metadata as a JSON file @@ -725,7 +725,7 @@ def get_index_package_metadata_for_short_package_names( # Validate path to crate if n not in ("1", "2"): raise ValueError("Invalid path to package metadata") - if not bool(re.fullmatch(r"[\w\-]{1,2}", package)): + if not re.fullmatch(r"[\w\-]{1,2}", package): raise ValueError("Invalid package name") # Request and return the metadata as a JSON file @@ -754,9 +754,9 @@ def get_rust_api_package_index( logger.debug(f"Received request to access {str(request.url)}") # Validate package name - if package and not bool(re.fullmatch(r"[\w\-]+", package)): + if package and not re.fullmatch(r"[\w\-]+", package): raise ValueError("Invalid package name") - if cursor and not bool(re.fullmatch(r"[a-zA-Z0-9]+", cursor)): + if cursor and not re.fullmatch(r"[a-zA-Z0-9]+", cursor): raise ValueError("Invalid cursor") # Formulate the search query to pass to the crates page @@ -790,7 +790,7 @@ def get_rust_api_package_info( logger.debug(f"Received request to access {str(request.url)}") # Validate package name - if not bool(re.fullmatch(r"[\w\-]+", package)): + if not re.fullmatch(r"[\w\-]+", package): raise ValueError("Invalid package name") # Return JSON of the package's page @@ -814,7 +814,7 @@ def get_rust_api_package_versions( logger.debug(f"Received request to access {str(request.url)}") # Validate crate name - if not bool(re.fullmatch(r"[\w\-]+", package)): + if not re.fullmatch(r"[\w\-]+", package): raise ValueError("Invalid package name") # Return JSON of the package's version information @@ -840,12 +840,12 @@ def get_rust_api_package_download( logger.debug(f"Received request to access {str(request.url)}") # Validate package name - if not bool(re.fullmatch(r"[\w\-]+", package)): + if not re.fullmatch(r"[\w\-]+", package): raise ValueError("Invalid package name") # Validate version number # Not all developers adhere to guidelines when versioning their packages, so # '-', '_', '+', as well as letters can also be present in this field. - if not bool(re.fullmatch(r"[\w\-\.\+]+", version)): + if not re.fullmatch(r"[\w\-\.\+]+", version): raise ValueError("Invalid version number") # Request and return package @@ -880,9 +880,9 @@ def get_rust_package_download( logger.debug(f"Received request to access {str(request.url)}") # Validate package and version - if not bool(re.fullmatch(r"[\w\-]+", package)): + if not re.fullmatch(r"[\w\-]+", package): raise ValueError("Invalid package name") - if not bool(re.fullmatch(r"[\w\-\.\+]+", version)): + if not re.fullmatch(r"[\w\-\.\+]+", version): raise ValueError("Invalid version number") # Request and return crate from https://static.crates.io @@ -926,12 +926,12 @@ def get_rust_package_crate( logger.debug(f"Received request to access {str(request.url)}") # Validate crate and package names - if not bool(re.fullmatch(r"[\w\-]+", package)): + if not re.fullmatch(r"[\w\-]+", package): raise ValueError("Invalid package name") if not crate.endswith(".crate"): raise ValueError("This is a not a Rust crate") # Rust crates follow a '{crate}-{version}.crate' structure - if not bool(re.fullmatch(r"[\w\-]+\-[0-9\.]+\.crate", crate)): + if not re.fullmatch(r"[\w\-]+\-[0-9\.]+\.crate", crate): raise ValueError("Invalid crate name") # Request and return package From 482582ef0cd3b6c273367950aaa959665a84d38f Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 27 Mar 2025 11:23:13 +0000 Subject: [PATCH 5/7] Loosened crate name validation, since it turns out some devs version packages unconventionally --- src/murfey/server/api/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/server/api/bootstrap.py b/src/murfey/server/api/bootstrap.py index 5164a33ca..7c3f4905c 100644 --- a/src/murfey/server/api/bootstrap.py +++ b/src/murfey/server/api/bootstrap.py @@ -931,7 +931,7 @@ def get_rust_package_crate( if not crate.endswith(".crate"): raise ValueError("This is a not a Rust crate") # Rust crates follow a '{crate}-{version}.crate' structure - if not re.fullmatch(r"[\w\-]+\-[0-9\.]+\.crate", crate): + if not re.fullmatch(r"[\w\-\.]+\.crate", crate): raise ValueError("Invalid crate name") # Request and return package From 5fa0d76c42c16913a2840611637385ea6078ef2c Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 27 Mar 2025 17:36:13 +0000 Subject: [PATCH 6/7] Corrected keys in constructed config.toml file; rearranged endpoints based whether they are used by the sparse index or the API --- src/murfey/server/api/bootstrap.py | 99 ++++++++++++++++-------------- 1 file changed, 54 insertions(+), 45 deletions(-) diff --git a/src/murfey/server/api/bootstrap.py b/src/murfey/server/api/bootstrap.py index 7c3f4905c..466b13140 100644 --- a/src/murfey/server/api/bootstrap.py +++ b/src/murfey/server/api/bootstrap.py @@ -588,8 +588,8 @@ def get_cargo_config(request: Request): Returns a properly configured Cargo config that sets it to look ONLY at the crates.io mirror. - This file is saved as ~/.cargo/config.toml on Linux devices by default, but - will be saved as %USERPROFILE%\\.cargo\\config.toml on Windows ones. + The default path for this config on Linux devices is ~/.cargo/config.toml, + and its default path on Windows is %USERPROFILE%\\.cargo\\config.toml. """ # Construct URL to our mirror of the Rust sparse index @@ -600,14 +600,12 @@ def get_cargo_config(request: Request): # Construct and return the config.toml file config_data = "\n".join( [ - "[registries.murfey-crates]", - f'index = "sparse+{index_mirror}"', # sparse+ to use sparse index logic + "[source.crates-io]", + 'replace-with = "murfey-crates"', # Redirect to our mirror "", - "[registry]", - 'default = "murfey-crates"', # Use our registry as default + "[source.murfey-crates]", + f'registry = "sparse+{index_mirror}"', # sparse+ to use sparse protocol "", - "[registries.crates-io]", - 'index = "false"', # Disables using crates.io ] ) config_bytes = io.BytesIO(config_data.encode("utf-8")) @@ -619,6 +617,11 @@ def get_cargo_config(request: Request): ) +""" +crates.io Sparse Index Registry Key Endpoints +""" + + @rust.get("/index") def get_index_page(): """ @@ -739,6 +742,49 @@ def get_index_package_metadata_for_short_package_names( ) +@rust.get("/crates/{package}/{version}/download", response_class=StreamingResponse) +def get_rust_package_download( + request: Request, + package: str, + version: str, +): + """ + Obtain and pass through a crate download request for a Rust package via the + sparse index registry. + """ + + logger.debug(f"Received request to access {str(request.url)}") + + # Validate package and version + if not re.fullmatch(r"[\w\-]+", package): + raise ValueError("Invalid package name") + if not re.fullmatch(r"[\w\-\.\+]+", version): + raise ValueError("Invalid version number") + + # Request and return crate from https://static.crates.io + url = f"{rust_dl}/{package}/{version}/download" + response = http_session.get(url) + file_name = f"{package}-{version}.crate" # Construct file name to save package as + if response.status_code != 200: + raise HTTPException(status_code=response.status_code) + return StreamingResponse( + content=response.iter_content(chunk_size=8192), + headers={ + "Content-Disposition": f'attachment; filename="{file_name}"', + "Content-Type": response.headers.get( + "Content-Type", "application/octet-stream" + ), + "Content-Length": response.headers.get("Content-Length"), + }, + status_code=response.status_code, + ) + + +""" +crates.io API Key Endpoints +""" + + @rust.get("/api/v1/crates") def get_rust_api_package_index( request: Request, @@ -867,43 +913,6 @@ def get_rust_api_package_download( ) -@rust.get("/crates/{package}/{version}/download", response_class=StreamingResponse) -def get_rust_package_download( - request: Request, - package: str, - version: str, -): - """ - Obtain and pass through a crate download request for a Rust package. - """ - - logger.debug(f"Received request to access {str(request.url)}") - - # Validate package and version - if not re.fullmatch(r"[\w\-]+", package): - raise ValueError("Invalid package name") - if not re.fullmatch(r"[\w\-\.\+]+", version): - raise ValueError("Invalid version number") - - # Request and return crate from https://static.crates.io - url = f"{rust_dl}/{package}/{version}/download" - response = http_session.get(url) - file_name = f"{package}-{version}.crate" # Construct file name to save package as - if response.status_code != 200: - raise HTTPException(status_code=response.status_code) - return StreamingResponse( - content=response.iter_content(chunk_size=8192), - headers={ - "Content-Disposition": f'attachment; filename="{file_name}"', - "Content-Type": response.headers.get( - "Content-Type", "application/octet-stream" - ), - "Content-Length": response.headers.get("Content-Length"), - }, - status_code=response.status_code, - ) - - @rust.get("/crates/{package}/{crate}", response_class=StreamingResponse) def get_rust_package_crate( request: Request, From c832bf6543c8bef5eeace53a934b6ed1c6a08aa6 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 27 Mar 2025 17:57:05 +0000 Subject: [PATCH 7/7] Registry keys still needed in config.toml to allow cargo's other functions to work --- src/murfey/server/api/bootstrap.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/murfey/server/api/bootstrap.py b/src/murfey/server/api/bootstrap.py index 466b13140..991650dd9 100644 --- a/src/murfey/server/api/bootstrap.py +++ b/src/murfey/server/api/bootstrap.py @@ -606,6 +606,12 @@ def get_cargo_config(request: Request): "[source.murfey-crates]", f'registry = "sparse+{index_mirror}"', # sparse+ to use sparse protocol "", + "[registries.murfey-crates]", + f'index = "sparse+{index_mirror}"', # sparse+ to use sparse protocol + "", + "[registry]", + 'default = "murfey-crates"', # Redirect to our mirror + "", ] ) config_bytes = io.BytesIO(config_data.encode("utf-8"))