diff --git a/Dockerfile b/Dockerfile index a01a7e0..140329b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG PARENT_IMAGE=kernai/refinery-parent-images:v3.0.0-torch-cpu +ARG PARENT_IMAGE=kernai/refinery-parent-images:v3.1.0-torch-cpu ARG DHI_PYTHON_BUILD=dhi.io/python:3.11-debian12-dev FROM ${PARENT_IMAGE} AS venv-source diff --git a/gpu-requirements.txt b/gpu-requirements.txt index 9f1668c..e5fdb49 100644 --- a/gpu-requirements.txt +++ b/gpu-requirements.txt @@ -15,7 +15,7 @@ annotated-types==0.7.0 # via # -r requirements/torch-cuda-requirements.txt # pydantic -anyio==4.13.0 +anyio==4.14.0 # via # -r requirements/torch-cuda-requirements.txt # httpx @@ -47,7 +47,7 @@ catalogue==2.0.10 # spacy # srsly # thinc -certifi==2026.4.22 +certifi==2026.5.20 # via # -r requirements/torch-cuda-requirements.txt # httpcore @@ -62,9 +62,10 @@ charset-normalizer==3.4.7 # via # -r requirements/torch-cuda-requirements.txt # requests -click==8.4.0 +click==8.4.1 # via # -r requirements/torch-cuda-requirements.txt + # huggingface-hub # typer # uvicorn cloudpathlib==0.24.0 @@ -74,6 +75,12 @@ confection==1.3.3 # spacy # thinc # weasel +cuda-bindings==13.3.1 + # via torch +cuda-pathfinder==1.5.5 + # via cuda-bindings +cuda-toolkit[cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2 + # via torch cymem==2.0.13 # via # preshed @@ -81,14 +88,14 @@ cymem==2.0.13 # thinc distro==1.9.0 # via openai -fastapi==0.120.1 +fastapi==0.136.3 # via -r requirements/torch-cuda-requirements.txt -filelock==3.20.3 +filelock==3.29.4 # via # -r requirements/torch-cuda-requirements.txt # huggingface-hub # torch -fsspec==2026.4.0 +fsspec==2026.6.0 # via # -r requirements/torch-cuda-requirements.txt # huggingface-hub @@ -97,9 +104,9 @@ googleapis-common-protos==1.75.0 # via # -r requirements/torch-cuda-requirements.txt # opentelemetry-exporter-otlp-proto-grpc -greenlet==3.5.0 +greenlet==3.5.1 # via sqlalchemy -grpcio==1.80.0 +grpcio==1.81.1 # via # -r requirements/torch-cuda-requirements.txt # opentelemetry-exporter-otlp-proto-grpc @@ -108,7 +115,7 @@ h11==0.16.0 # -r requirements/torch-cuda-requirements.txt # httpcore # uvicorn -hf-xet==1.5.0 +hf-xet==1.5.1 # via # -r requirements/torch-cuda-requirements.txt # huggingface-hub @@ -122,13 +129,13 @@ httpx==0.28.1 # huggingface-hub # openai # weasel -huggingface-hub==1.15.0 +huggingface-hub==1.19.0 # via # -r requirements/torch-cuda-requirements.txt # sentence-transformers # tokenizers # transformers -idna==3.15 +idna==3.18 # via # -r requirements/torch-cuda-requirements.txt # anyio @@ -143,7 +150,7 @@ jinja2==3.1.6 # -r requirements/torch-cuda-requirements.txt # spacy # torch -jiter==0.14.0 +jiter==0.15.0 # via openai jmespath==1.1.0 # via @@ -178,6 +185,10 @@ murmurhash==1.0.15 # preshed # spacy # thinc +narwhals==2.22.1 + # via + # -r requirements/torch-cuda-requirements.txt + # scikit-learn networkx==3.6.1 # via # -r requirements/torch-cuda-requirements.txt @@ -195,43 +206,47 @@ numpy==1.26.2 # thinc # torchvision # transformers -nvidia-cublas-cu12==12.8.4.1 +nvidia-cublas==13.1.1.3 # via - # nvidia-cudnn-cu12 - # nvidia-cusolver-cu12 + # nvidia-cudnn-cu13 + # nvidia-cusolver # torch -nvidia-cuda-cupti-cu12==12.8.90 - # via torch -nvidia-cuda-nvrtc-cu12==12.8.93 - # via torch -nvidia-cuda-runtime-cu12==12.8.90 - # via torch -nvidia-cudnn-cu12==9.10.2.21 +nvidia-cuda-cupti==13.0.85 + # via cuda-toolkit +nvidia-cuda-nvrtc==13.0.88 + # via + # cuda-toolkit + # nvidia-cublas +nvidia-cuda-runtime==13.0.96 + # via cuda-toolkit +nvidia-cudnn-cu13==9.20.0.48 # via torch -nvidia-cufft-cu12==11.3.3.83 +nvidia-cufft==12.0.0.61 + # via cuda-toolkit +nvidia-cufile==1.15.1.6 + # via cuda-toolkit +nvidia-curand==10.4.0.35 + # via cuda-toolkit +nvidia-cusolver==12.0.4.66 + # via cuda-toolkit +nvidia-cusparse==12.6.3.3 + # via + # cuda-toolkit + # nvidia-cusolver +nvidia-cusparselt-cu13==0.8.1 # via torch -nvidia-cufile-cu12==1.13.1.3 +nvidia-nccl-cu13==2.29.7 # via torch -nvidia-curand-cu12==10.3.9.90 - # via torch -nvidia-cusolver-cu12==11.7.3.90 - # via torch -nvidia-cusparse-cu12==12.5.8.93 +nvidia-nvjitlink==13.0.88 # via - # nvidia-cusolver-cu12 - # torch -nvidia-cusparselt-cu12==0.7.1 - # via torch -nvidia-nccl-cu12==2.27.3 - # via torch -nvidia-nvjitlink-cu12==12.8.93 - # via - # nvidia-cufft-cu12 - # nvidia-cusolver-cu12 - # nvidia-cusparse-cu12 - # torch -nvidia-nvtx-cu12==12.8.90 + # cuda-toolkit + # nvidia-cufft + # nvidia-cusolver + # nvidia-cusparse +nvidia-nvshmem-cu13==3.4.5 # via torch +nvidia-nvtx==13.0.85 + # via cuda-toolkit openai==1.97.1 # via -r requirements/gpu-requirements.in opentelemetry-api==1.39.1 @@ -290,6 +305,7 @@ packaging==26.2 # -r requirements/torch-cuda-requirements.txt # huggingface-hub # opentelemetry-instrumentation + # scikit-optimize # spacy # thinc # transformers @@ -372,16 +388,16 @@ s3transfer==0.13.1 # via # -r requirements/torch-cuda-requirements.txt # boto3 -safetensors==0.7.0 +safetensors==0.8.0 # via # -r requirements/torch-cuda-requirements.txt # transformers -scikit-learn==1.5.2 +scikit-learn==1.9.0 # via # -r requirements/torch-cuda-requirements.txt # scikit-optimize # sentence-transformers -scikit-optimize==0.9.0 +scikit-optimize==0.10.2 # via -r requirements/torch-cuda-requirements.txt scipy==1.16.3 # via @@ -389,7 +405,7 @@ scipy==1.16.3 # scikit-learn # scikit-optimize # sentence-transformers -sentence-transformers==5.3.0 +sentence-transformers==5.5.1 # via -r requirements/gpu-requirements.in shellingham==1.5.4 # via @@ -416,7 +432,7 @@ srsly==2.5.3 # spacy # thinc # weasel -starlette==0.49.3 +starlette==1.3.1 # via # -r requirements/torch-cuda-requirements.txt # fastapi @@ -436,14 +452,14 @@ tokenizers==0.22.2 # via # -r requirements/torch-cuda-requirements.txt # transformers -torch==2.8.0 +torch==2.12.0 # via # -r requirements/torch-cuda-requirements.txt # sentence-transformers # torchvision -torchvision==0.23.0 +torchvision==0.27.0 # via -r requirements/gpu-requirements.in -tqdm==4.67.3 +tqdm==4.68.2 # via # -r requirements/torch-cuda-requirements.txt # huggingface-hub @@ -451,11 +467,11 @@ tqdm==4.67.3 # sentence-transformers # spacy # transformers -transformers==5.5.0 +transformers==5.12.1 # via # -r requirements/torch-cuda-requirements.txt # sentence-transformers -triton==3.4.0 +triton==3.7.0 # via torch typer==0.25.1 # via @@ -486,6 +502,7 @@ typing-extensions==4.15.0 typing-inspection==0.4.2 # via # -r requirements/torch-cuda-requirements.txt + # fastapi # pydantic urllib3==2.7.0 # via @@ -507,7 +524,7 @@ wrapt==1.17.3 # -r requirements/torch-cuda-requirements.txt # opentelemetry-instrumentation # smart-open -zipp==3.23.1 +zipp==4.1.0 # via # -r requirements/torch-cuda-requirements.txt # importlib-metadata diff --git a/gpu.Dockerfile b/gpu.Dockerfile index 5ceefeb..6b79de8 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -1,4 +1,4 @@ -ARG PARENT_IMAGE=kernai/refinery-parent-images:v2.6.0-torch-cuda +ARG PARENT_IMAGE=kernai/refinery-parent-images:v3.1.0-torch-cuda FROM ${PARENT_IMAGE} AS builder diff --git a/requirements.txt b/requirements.txt index 13315c0..e676799 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ annotated-types==0.7.0 # via # -r requirements/torch-cpu-requirements.txt # pydantic -anyio==4.13.0 +anyio==4.14.0 # via # -r requirements/torch-cpu-requirements.txt # httpx @@ -47,7 +47,7 @@ catalogue==2.0.10 # spacy # srsly # thinc -certifi==2026.4.22 +certifi==2026.5.20 # via # -r requirements/torch-cpu-requirements.txt # httpcore @@ -62,9 +62,10 @@ charset-normalizer==3.4.7 # via # -r requirements/torch-cpu-requirements.txt # requests -click==8.4.0 +click==8.4.1 # via # -r requirements/torch-cpu-requirements.txt + # huggingface-hub # typer # uvicorn cloudpathlib==0.24.0 @@ -81,14 +82,14 @@ cymem==2.0.13 # thinc distro==1.9.0 # via openai -fastapi==0.120.1 +fastapi==0.136.3 # via -r requirements/torch-cpu-requirements.txt -filelock==3.20.3 +filelock==3.29.4 # via # -r requirements/torch-cpu-requirements.txt # huggingface-hub # torch -fsspec==2026.4.0 +fsspec==2026.6.0 # via # -r requirements/torch-cpu-requirements.txt # huggingface-hub @@ -97,9 +98,9 @@ googleapis-common-protos==1.75.0 # via # -r requirements/torch-cpu-requirements.txt # opentelemetry-exporter-otlp-proto-grpc -greenlet==3.5.0 +greenlet==3.5.1 # via sqlalchemy -grpcio==1.80.0 +grpcio==1.81.1 # via # -r requirements/torch-cpu-requirements.txt # opentelemetry-exporter-otlp-proto-grpc @@ -108,7 +109,7 @@ h11==0.16.0 # -r requirements/torch-cpu-requirements.txt # httpcore # uvicorn -hf-xet==1.5.0 +hf-xet==1.5.1 # via # -r requirements/torch-cpu-requirements.txt # huggingface-hub @@ -122,13 +123,13 @@ httpx==0.28.1 # huggingface-hub # openai # weasel -huggingface-hub==1.15.0 +huggingface-hub==1.19.0 # via # -r requirements/torch-cpu-requirements.txt # sentence-transformers # tokenizers # transformers -idna==3.15 +idna==3.18 # via # -r requirements/torch-cpu-requirements.txt # anyio @@ -143,7 +144,7 @@ jinja2==3.1.6 # -r requirements/torch-cpu-requirements.txt # spacy # torch -jiter==0.14.0 +jiter==0.15.0 # via openai jmespath==1.1.0 # via @@ -178,6 +179,10 @@ murmurhash==1.0.15 # preshed # spacy # thinc +narwhals==2.22.1 + # via + # -r requirements/torch-cpu-requirements.txt + # scikit-learn networkx==3.6.1 # via # -r requirements/torch-cpu-requirements.txt @@ -253,6 +258,7 @@ packaging==26.2 # -r requirements/torch-cpu-requirements.txt # huggingface-hub # opentelemetry-instrumentation + # scikit-optimize # spacy # thinc # transformers @@ -335,16 +341,16 @@ s3transfer==0.13.1 # via # -r requirements/torch-cpu-requirements.txt # boto3 -safetensors==0.7.0 +safetensors==0.8.0 # via # -r requirements/torch-cpu-requirements.txt # transformers -scikit-learn==1.5.2 +scikit-learn==1.9.0 # via # -r requirements/torch-cpu-requirements.txt # scikit-optimize # sentence-transformers -scikit-optimize==0.9.0 +scikit-optimize==0.10.2 # via -r requirements/torch-cpu-requirements.txt scipy==1.16.3 # via @@ -352,7 +358,7 @@ scipy==1.16.3 # scikit-learn # scikit-optimize # sentence-transformers -sentence-transformers==5.3.0 +sentence-transformers==5.5.1 # via -r requirements/requirements.in shellingham==1.5.4 # via @@ -379,7 +385,7 @@ srsly==2.5.3 # spacy # thinc # weasel -starlette==0.49.3 +starlette==1.3.1 # via # -r requirements/torch-cpu-requirements.txt # fastapi @@ -399,14 +405,14 @@ tokenizers==0.22.2 # via # -r requirements/torch-cpu-requirements.txt # transformers -torch==2.8.0 +torch==2.12.0 # via # -r requirements/torch-cpu-requirements.txt # sentence-transformers # torchvision -torchvision==0.23.0 +torchvision==0.27.0 # via -r requirements/requirements.in -tqdm==4.67.3 +tqdm==4.68.2 # via # -r requirements/torch-cpu-requirements.txt # huggingface-hub @@ -414,7 +420,7 @@ tqdm==4.67.3 # sentence-transformers # spacy # transformers -transformers==5.5.0 +transformers==5.12.1 # via # -r requirements/torch-cpu-requirements.txt # sentence-transformers @@ -447,6 +453,7 @@ typing-extensions==4.15.0 typing-inspection==0.4.2 # via # -r requirements/torch-cpu-requirements.txt + # fastapi # pydantic urllib3==2.7.0 # via @@ -468,7 +475,7 @@ wrapt==1.17.3 # -r requirements/torch-cpu-requirements.txt # opentelemetry-instrumentation # smart-open -zipp==3.23.1 +zipp==4.1.0 # via # -r requirements/torch-cpu-requirements.txt # importlib-metadata diff --git a/requirements/gpu-requirements.in b/requirements/gpu-requirements.in index 52f3769..1b61271 100644 --- a/requirements/gpu-requirements.in +++ b/requirements/gpu-requirements.in @@ -1,6 +1,6 @@ -r torch-cuda-requirements.txt openai==1.97.1 tiktoken>=0.5.0 -sentence-transformers==5.3.0 +sentence-transformers==5.5.1 spacy==3.8.14 -torchvision==0.23.0 \ No newline at end of file +torchvision==0.27.0 \ No newline at end of file diff --git a/requirements/requirements.in b/requirements/requirements.in index ee3f1a8..1d40669 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -1,6 +1,6 @@ -r torch-cpu-requirements.txt openai==1.97.1 tiktoken>=0.5.0 -sentence-transformers==5.3.0 +sentence-transformers==5.5.1 spacy==3.8.14 -torchvision==0.23.0 \ No newline at end of file +torchvision==0.27.0 \ No newline at end of file diff --git a/requirements/torch-cpu-requirements.txt b/requirements/torch-cpu-requirements.txt index 7f21ca7..a332c62 100644 --- a/requirements/torch-cpu-requirements.txt +++ b/requirements/torch-cpu-requirements.txt @@ -12,7 +12,7 @@ annotated-doc==0.0.4 # typer annotated-types==0.7.0 # via pydantic -anyio==4.13.0 +anyio==4.14.0 # via # httpx # starlette @@ -28,7 +28,7 @@ botocore==1.39.17 # via # boto3 # s3transfer -certifi==2026.4.22 +certifi==2026.5.20 # via # httpcore # httpx @@ -38,40 +38,41 @@ cffi==2.0.0 # via argon2-cffi-bindings charset-normalizer==3.4.7 # via requests -click==8.4.0 +click==8.4.1 # via + # huggingface-hub # typer # uvicorn -fastapi==0.120.1 +fastapi==0.136.3 # via -r mini-requirements.in -filelock==3.20.3 +filelock==3.29.4 # via # -r torch-cpu-requirements.in # huggingface-hub # torch -fsspec==2026.4.0 +fsspec==2026.6.0 # via # huggingface-hub # torch googleapis-common-protos==1.75.0 # via opentelemetry-exporter-otlp-proto-grpc -grpcio==1.80.0 +grpcio==1.81.1 # via opentelemetry-exporter-otlp-proto-grpc h11==0.16.0 # via # httpcore # uvicorn -hf-xet==1.5.0 +hf-xet==1.5.1 # via huggingface-hub httpcore==1.0.9 # via httpx httpx==0.28.1 # via huggingface-hub -huggingface-hub==1.15.0 +huggingface-hub==1.19.0 # via # tokenizers # transformers -idna==3.15 +idna==3.18 # via # anyio # httpx @@ -98,6 +99,8 @@ minio==7.2.15 # via -r common-requirements.in mpmath==1.3.0 # via sympy +narwhals==2.22.1 + # via scikit-learn networkx==3.6.1 # via torch numpy==1.26.2 @@ -156,6 +159,7 @@ packaging==26.2 # via # huggingface-hub # opentelemetry-instrumentation + # scikit-optimize # transformers pandas==1.5.1 # via -r common-requirements.in @@ -201,13 +205,13 @@ rich==15.0.0 # via typer s3transfer==0.13.1 # via boto3 -safetensors==0.7.0 +safetensors==0.8.0 # via transformers -scikit-learn==1.5.2 +scikit-learn==1.9.0 # via # -r torch-cpu-requirements.in # scikit-optimize -scikit-optimize==0.9.0 +scikit-optimize==0.10.2 # via -r torch-cpu-requirements.in scipy==1.16.3 # via @@ -219,7 +223,7 @@ six==1.17.0 # via python-dateutil sqlalchemy==1.4.42 # via -r common-requirements.in -starlette==0.49.3 +starlette==1.3.1 # via fastapi sympy==1.14.0 # via torch @@ -227,13 +231,13 @@ threadpoolctl==3.6.0 # via scikit-learn tokenizers==0.22.2 # via transformers -torch==2.8.0 +torch==2.12.0 # via -r torch-cpu-requirements.in -tqdm==4.67.3 +tqdm==4.68.2 # via # huggingface-hub # transformers -transformers==5.5.0 +transformers==5.12.1 # via -r torch-cpu-requirements.in typer==0.25.1 # via @@ -256,7 +260,9 @@ typing-extensions==4.15.0 # torch # typing-inspection typing-inspection==0.4.2 - # via pydantic + # via + # fastapi + # pydantic urllib3==2.7.0 # via # botocore @@ -266,5 +272,8 @@ uvicorn==0.35.0 # via -r mini-requirements.in wrapt==1.17.3 # via opentelemetry-instrumentation -zipp==3.23.1 +zipp==4.1.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/torch-cuda-requirements.txt b/requirements/torch-cuda-requirements.txt index 2d9ce84..b317ad3 100644 --- a/requirements/torch-cuda-requirements.txt +++ b/requirements/torch-cuda-requirements.txt @@ -12,7 +12,7 @@ annotated-doc==0.0.4 # typer annotated-types==0.7.0 # via pydantic -anyio==4.13.0 +anyio==4.14.0 # via # httpx # starlette @@ -28,7 +28,7 @@ botocore==1.39.17 # via # boto3 # s3transfer -certifi==2026.4.22 +certifi==2026.5.20 # via # httpcore # httpx @@ -38,40 +38,41 @@ cffi==2.0.0 # via argon2-cffi-bindings charset-normalizer==3.4.7 # via requests -click==8.4.0 +click==8.4.1 # via + # huggingface-hub # typer # uvicorn -fastapi==0.120.1 +fastapi==0.136.3 # via -r mini-requirements.in -filelock==3.20.3 +filelock==3.29.4 # via # -r torch-cuda-requirements.in # huggingface-hub # torch -fsspec==2026.4.0 +fsspec==2026.6.0 # via # huggingface-hub # torch googleapis-common-protos==1.75.0 # via opentelemetry-exporter-otlp-proto-grpc -grpcio==1.80.0 +grpcio==1.81.1 # via opentelemetry-exporter-otlp-proto-grpc h11==0.16.0 # via # httpcore # uvicorn -hf-xet==1.5.0 +hf-xet==1.5.1 # via huggingface-hub httpcore==1.0.9 # via httpx httpx==0.28.1 # via huggingface-hub -huggingface-hub==1.15.0 +huggingface-hub==1.19.0 # via # tokenizers # transformers -idna==3.15 +idna==3.18 # via # anyio # httpx @@ -98,6 +99,8 @@ minio==7.2.15 # via -r common-requirements.in mpmath==1.3.0 # via sympy +narwhals==2.22.1 + # via scikit-learn networkx==3.6.1 # via torch numpy==1.26.2 @@ -156,6 +159,7 @@ packaging==26.2 # via # huggingface-hub # opentelemetry-instrumentation + # scikit-optimize # transformers pandas==1.5.1 # via -r common-requirements.in @@ -201,13 +205,13 @@ rich==15.0.0 # via typer s3transfer==0.13.1 # via boto3 -safetensors==0.7.0 +safetensors==0.8.0 # via transformers -scikit-learn==1.5.2 +scikit-learn==1.9.0 # via # -r torch-cuda-requirements.in # scikit-optimize -scikit-optimize==0.9.0 +scikit-optimize==0.10.2 # via -r torch-cuda-requirements.in scipy==1.16.3 # via @@ -219,7 +223,7 @@ six==1.17.0 # via python-dateutil sqlalchemy==1.4.42 # via -r common-requirements.in -starlette==0.49.3 +starlette==1.3.1 # via fastapi sympy==1.14.0 # via torch @@ -227,13 +231,13 @@ threadpoolctl==3.6.0 # via scikit-learn tokenizers==0.22.2 # via transformers -torch==2.8.0 +torch==2.12.0 # via -r torch-cuda-requirements.in -tqdm==4.67.3 +tqdm==4.68.2 # via # huggingface-hub # transformers -transformers==5.5.0 +transformers==5.12.1 # via -r torch-cuda-requirements.in typer==0.25.1 # via @@ -256,7 +260,9 @@ typing-extensions==4.15.0 # torch # typing-inspection typing-inspection==0.4.2 - # via pydantic + # via + # fastapi + # pydantic urllib3==2.7.0 # via # botocore @@ -266,5 +272,8 @@ uvicorn==0.35.0 # via -r mini-requirements.in wrapt==1.17.3 # via opentelemetry-instrumentation -zipp==3.23.1 +zipp==4.1.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools