From 43759bea9c1aa902a2ca67234654bff72f74b054 Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos <34061419+stratika@users.noreply.github.com> Date: Wed, 10 Dec 2025 11:55:17 +0200 Subject: [PATCH 1/7] Revise TornadoVM SDK installation instructions Updated installation instructions for TornadoVM SDK on Linux, macOS, and Windows. Removed unnecessary commands and added wget commands for downloading SDKs. --- README.md | 82 ++++++++++++++++++++----------------------------------- 1 file changed, 30 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index f407d927..28a70834 100644 --- a/README.md +++ b/README.md @@ -106,79 +106,57 @@ When cloning this repository, use the `--recursive` flag to ensure that TornadoV ```bash # Clone the repository with all submodules -git clone --recursive https://github.com/beehive-lab/GPULlama3.java.git - -# Navigate to the project directory -cd GPULlama3.java - -# Update the submodules to match the exact commit point recorded in this repository -git submodule update --recursive +git clone https://github.com/beehive-lab/GPULlama3.java.git ``` -#### On Linux or macOS -```bash -# Enter the TornadoVM submodule directory -cd external/tornadovm - -# Optional: Create and activate a Python virtual environment if needed -python3 -m venv venv -source ./venv/bin/activate - -# Install TornadoVM with a supported JDK 21 and select the backends (--backend opencl,ptx). -# To see the compatible JDKs run: ./bin/tornadovm-installer --listJDKs -# For example, to install with OpenJDK 21 and build the OpenCL backend, run: -./bin/tornadovm-installer --jdk jdk21 --backend opencl +#### Install the TornadoVM SDK on Linux or macOS -# Source the TornadoVM environment variables -source setvars.sh +Ensure that your JAVA_HOME points to a supported JDK before using the SDK. Download an SDK package matching your OS, architecture, and accelerator backend (opencl, ptx). +All pre-built SDKs are available on the TornadoVM [Releases Page](https://github.com/beehive-lab/TornadoVM/releases). +#After extracting the SDK, add its bin/ directory to your PATH so the `tornado` command becomes available. -# Navigate back to the project root directory -cd ../../ +##### Linux (x86_64) -# Source the project-specific environment paths -> this will ensure the correct paths are set for the project and the TornadoVM SDK -# Expect to see: [INFO] Environment configured for Llama3 with TornadoVM at: /home/YOUR_PATH_TO_TORNADOVM -source set_paths - -# Build the project using Maven (skip tests for faster build) -# mvn clean package -DskipTests or just make -make +```bash +wget https://github.com/beehive-lab/TornadoVM/releases/download/v2.1.0/tornadovm-2.1.0-opencl-linux-amd64.zip +unzip tornadovm-2.1.0-opencl-linux-amd64.zip +# Replace manually with the absolute path of the extracted folder +export TORNADO_SDK="/tornadovm-2.1.0-opencl" -# Run the model (make sure you have downloaded the model file first - see below) -./llama-tornado --gpu --verbose-init --opencl --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "tell me a joke" +tornado --devices +tornado --version ``` -#### On Windows -```bash -# Enter the TornadoVM submodule directory -cd external/tornadovm +##### macOS (Apple Silicon) -# Optional: Create and activate a Python virtual environment if needed -python -m venv .venv -.venv\Scripts\activate.bat -.\bin\windowsMicrosoftStudioTools2022.cmd +```bash +wget https://github.com/beehive-lab/TornadoVM/releases/download/v2.1.0/tornadovm-2.1.0-opencl-mac-aarch64.zip +unzip tornadovm-2.1.0-opencl-mac-aarch64.zip +# Replace manually with the absolute path of the extracted folder +export TORNADO_SDK="/tornadovm-2.1.0-opencl" -# Install TornadoVM with a supported JDK 21 and select the backends (--backend opencl,ptx). -# To see the compatible JDKs run: ./bin/tornadovm-installer --listJDKs -# For example, to install with OpenJDK 21 and build the OpenCL backend, run: -python bin\tornadovm-installer --jdk jdk21 --backend opencl +tornado --devices +tornado --version +``` -# Source the TornadoVM environment variables -setvars.cmd +#### Build the GPULlama3.java -# Navigate back to the project root directory -cd ../../ +```bash +# Navigate to the project directory +cd GPULlama3.java # Source the project-specific environment paths -> this will ensure the correct paths are set for the project and the TornadoVM SDK -# Expect to see: [INFO] Environment configured for Llama3 with TornadoVM at: C:\Users\YOUR_PATH_TO_TORNADOVM -set_paths.cmd +# Expect to see: [INFO] Environment configured for Llama3 with TornadoVM at: $TORNADO_SDK +source set_paths # Build the project using Maven (skip tests for faster build) # mvn clean package -DskipTests or just make make # Run the model (make sure you have downloaded the model file first - see below) -python llama-tornado --gpu --verbose-init --opencl --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "tell me a joke" +./llama-tornado --gpu --verbose-init --opencl --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "tell me a joke" ``` + ----------- ## 📦 Maven Dependency From 7b14fb3da563c3b8772b2ab0edbcf3f689da7189 Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos Date: Wed, 10 Dec 2025 11:56:06 +0200 Subject: [PATCH 2/7] Update set_paths script to be synced with TORNADO_SDK installation --- set_paths | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/set_paths b/set_paths index fd807c5e..0f356cc8 100644 --- a/set_paths +++ b/set_paths @@ -6,19 +6,15 @@ # Resolve root of this project (LLaMA3) and TornadoVM export LLAMA_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -export TORNADO_ROOT="${LLAMA_ROOT}/external/tornadovm" - -# Set the path to TornadoVM SDK binaries -export TORNADO_SDK="${TORNADO_ROOT}/bin/sdk" # Add TornadoVM and LLaMA bin directories to PATH -export PATH="${PATH}:${TORNADO_SDK}:${LLAMA_ROOT}" +export PATH="${PATH}:${TORNADO_SDK}/bin:${LLAMA_ROOT}" # Optional: Set JAVA_HOME if required # export JAVA_HOME=/path/to/graalvm # export PATH="${JAVA_HOME}/bin:${PATH}" -echo "[INFO] Environment configured for LLaMA3 with TornadoVM at: $TORNADO_ROOT" +echo "[INFO] Environment configured for LLaMA3 with TornadoVM at: $TORNADO_SDK" # ===== Notes ===== # After sourcing this script: # 1. TornadoVM will be available for GPU computation From ce2d71f43f9afbe1efea61bca9c546418ffd6620 Mon Sep 17 00:00:00 2001 From: Michalis Papadimitriou Date: Wed, 10 Dec 2025 13:33:31 +0200 Subject: [PATCH 3/7] Revise model download section in README.md Updated model download links and improved organization in README. --- README.md | 83 ++++++------------------------------------------------- 1 file changed, 9 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 28a70834..4f560398 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,6 @@ Ensure you have the following installed and configured: - **Java 21**: Required for Vector API support & TornadoVM. - [TornadoVM](https://github.com/beehive-lab/TornadoVM) with OpenCL or PTX backends. -- [Maven](https://maven.apache.org/): For building the Java project. ### Install, Build, and Run @@ -239,83 +238,19 @@ llama-tornado --gpu --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "te The above model can we swapped with one of the other models, such as `beehive-llama-3.2-3b-instruct-fp16.gguf` or `beehive-llama-3.2-8b-instruct-fp16.gguf`, depending on your needs. Check models below. -## Download Model Files +## Collection of Tested Models -Download `FP16` quantized `Llama-3` .gguf files from: -- https://huggingface.co/beehive-lab/Llama-3.2-1B-Instruct-GGUF-FP16 -- https://huggingface.co/beehive-lab/Llama-3.2-3B-Instruct-GGUF-FP16 -- https://huggingface.co/beehive-lab/Llama-3.2-8B-Instruct-GGUF-FP16 +### Llama3.2 Collection +https://huggingface.co/collections/beehive-lab/llama3-gpullama3java -Download `FP16` quantized `Mistral` .gguf files from: -- https://huggingface.co/collections/beehive-lab/mistral-gpullama3java-684afabb206136d2e9cd47e0 +### Llama3.2 Collection +https://huggingface.co/collections/beehive-lab/llama3-gpullama3java -Download `FP16` quantized `Qwen3` .gguf files from: -- https://huggingface.co/ggml-org/Qwen3-0.6B-GGUF -- https://huggingface.co/ggml-org/Qwen3-1.7B-GGUF -- https://huggingface.co/ggml-org/Qwen3-4B-GGUF -- https://huggingface.co/ggml-org/Qwen3-8B-GGUF +### Llama3.2 Collection +https://huggingface.co/collections/beehive-lab/llama3-gpullama3java -Download `FP16` quantized `Qwen2.5` .gguf files from: -- https://huggingface.co/bartowski/Qwen2.5-0.5B-Instruct-GGUF -- https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF - -Download `FP16` quantized `DeepSeek-R1-Distill-Qwen` .gguf files from: -- https://huggingface.co/hdnh2006/DeepSeek-R1-Distill-Qwen-1.5B-GGUF - -Please be gentle with [huggingface.co](https://huggingface.co) servers: - -**Note** FP16 models are first-class citizens for the current version. -``` -# Llama 3.2 (1B) - FP16 -wget https://huggingface.co/beehive-lab/Llama-3.2-1B-Instruct-GGUF-FP16/resolve/main/beehive-llama-3.2-1b-instruct-fp16.gguf - -# Llama 3.2 (3B) - FP16 -wget https://huggingface.co/beehive-lab/Llama-3.2-3B-Instruct-GGUF-FP16/resolve/main/beehive-llama-3.2-3b-instruct-fp16.gguf - -# Llama 3 (8B) - FP16 -wget https://huggingface.co/beehive-lab/Llama-3.2-8B-Instruct-GGUF-FP16/resolve/main/beehive-llama-3.2-8b-instruct-fp16.gguf - -# Mistral (7B) - FP16 -wget https://huggingface.co/MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF/resolve/main/Mistral-7B-Instruct-v0.3.fp16.gguf - -# Qwen3 (0.6B) - FP16 -wget https://huggingface.co/ggml-org/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-f16.gguf - -# Qwen3 (1.7B) - FP16 -wget https://huggingface.co/ggml-org/Qwen3-0.6B-GGUF/resolve/main/Qwen3-1.7B-f16.gguf - -# Qwen3 (4B) - FP16 -wget https://huggingface.co/ggml-org/Qwen3-0.6B-GGUF/resolve/main/Qwen3-4B-f16.gguf - -# Qwen3 (8B) - FP16 -wget https://huggingface.co/ggml-org/Qwen3-0.6B-GGUF/resolve/main/Qwen3-8B-f16.gguf - -# Phi-3-mini-4k - FP16 -wget https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-fp16.gguf - -# Qwen2.5 (0.5B) -wget https://huggingface.co/bartowski/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/Qwen2.5-0.5B-Instruct-f16.gguf - -# Qwen2.5 (1.5B) -wget https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/qwen2.5-1.5b-instruct-fp16.gguf - -# DeepSeek-R1-Distill-Qwen (1.5B) -wget https://huggingface.co/hdnh2006/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-F16.gguf -``` - -**[Experimental]** you can download the Q8 and Q4 used in the original implementation of Llama3.java, but for now are going to be dequanted to FP16 for TornadoVM support: -``` -# Llama 3.2 (1B) - Q4_0 -curl -L -O https://huggingface.co/mukel/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf -# Llama 3.2 (3B) - Q4_0 -curl -L -O https://huggingface.co/mukel/Llama-3.2-3B-Instruct-GGUF/resolve/main/Llama-3.2-3B-Instruct-Q4_0.gguf -# Llama 3 (8B) - Q4_0 -curl -L -O https://huggingface.co/mukel/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct-Q4_0.gguf -# Llama 3.2 (1B) - Q8_0 -curl -L -O https://huggingface.co/mukel/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q8_0.gguf -# Llama 3.1 (8B) - Q8_0 -curl -L -O https://huggingface.co/mukel/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q4_0.gguf -``` +### Llama3.2 Collection +https://huggingface.co/collections/beehive-lab/llama3-gpullama3java ----------- From cf282ef394630165fb32a44d90a04c29e30bad82 Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos Date: Wed, 10 Dec 2025 13:35:46 +0200 Subject: [PATCH 4/7] Remove TornadoVM submodule --- .gitmodules | 5 ----- external/tornadovm | 1 - 2 files changed, 6 deletions(-) delete mode 160000 external/tornadovm diff --git a/.gitmodules b/.gitmodules index 48d4a7fc..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +0,0 @@ -[submodule "external/tornadovm"] - path = external/tornadovm - url = https://github.com/beehive-lab/TornadoVM.git - branch = master - diff --git a/external/tornadovm b/external/tornadovm deleted file mode 160000 index f6de88c1..00000000 --- a/external/tornadovm +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f6de88c150117d17ddc04a749e34f7f4ac4d0429 From 9a05519a72356f06d8d250b8e6e1f14e9712e623 Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos <34061419+stratika@users.noreply.github.com> Date: Wed, 10 Dec 2025 13:40:52 +0200 Subject: [PATCH 5/7] Update model collections with new links --- README.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4f560398..6ae96741 100644 --- a/README.md +++ b/README.md @@ -241,16 +241,22 @@ Check models below. ## Collection of Tested Models ### Llama3.2 Collection -https://huggingface.co/collections/beehive-lab/llama3-gpullama3java +[https://huggingface.co/collections/beehive-lab/llama3-gpullama3java](https://huggingface.co/collections/beehive-lab/llama3-gpullama3java) -### Llama3.2 Collection -https://huggingface.co/collections/beehive-lab/llama3-gpullama3java +### Qwen 2.5 Collection +[https://huggingface.co/collections/beehive-lab/qwen-25-gpullama3java](https://huggingface.co/collections/beehive-lab/qwen-25-gpullama3java) -### Llama3.2 Collection -https://huggingface.co/collections/beehive-lab/llama3-gpullama3java +### Qwen 3 Collection +[https://huggingface.co/collections/beehive-lab/llama3-gpullama3java](https://huggingface.co/collections/beehive-lab/qwen-3-gpullama3java) -### Llama3.2 Collection -https://huggingface.co/collections/beehive-lab/llama3-gpullama3java +### Phi-3 Collection +[https://huggingface.co/collections/beehive-lab/llama3-gpullama3java](https://huggingface.co/collections/beehive-lab/phi-3-gpullama3java) + +### Mistral Collection +[https://huggingface.co/collections/beehive-lab/llama3-gpullama3java](https://huggingface.co/collections/beehive-lab/mistral-gpullama3java) + +### DeepSeek-R1-Distill-Qwen Collection +[https://huggingface.co/collections/beehive-lab/deepseek-r1-distill-qwen-gpullama3java](https://huggingface.co/collections/beehive-lab/deepseek-r1-distill-qwen-gpullama3java) ----------- From aec0c527b1541963791938c87de60665ded587b7 Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos <34061419+stratika@users.noreply.github.com> Date: Wed, 10 Dec 2025 13:48:38 +0200 Subject: [PATCH 6/7] Update README with PATH export for TornadoVM Added PATH export command for TornadoVM SDK setup. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6ae96741..ab14b82f 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ wget https://github.com/beehive-lab/TornadoVM/releases/download/v2.1.0/tornadovm unzip tornadovm-2.1.0-opencl-linux-amd64.zip # Replace manually with the absolute path of the extracted folder export TORNADO_SDK="/tornadovm-2.1.0-opencl" +export PATH=$TORNADO_SDK/bin:$PATH tornado --devices tornado --version @@ -133,6 +134,7 @@ wget https://github.com/beehive-lab/TornadoVM/releases/download/v2.1.0/tornadovm unzip tornadovm-2.1.0-opencl-mac-aarch64.zip # Replace manually with the absolute path of the extracted folder export TORNADO_SDK="/tornadovm-2.1.0-opencl" +export PATH=$TORNADO_SDK/bin:$PATH tornado --devices tornado --version From 6d1457e7f38d1f27354a3aef1d27812002765e21 Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos <34061419+stratika@users.noreply.github.com> Date: Thu, 11 Dec 2025 11:16:32 +0200 Subject: [PATCH 7/7] Clarify GCC/G++ version requirement in README Updated the README to clarify GCC/G++ version requirement. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ab14b82f..1e714b77 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ Ensure you have the following installed and configured: - **Java 21**: Required for Vector API support & TornadoVM. - [TornadoVM](https://github.com/beehive-lab/TornadoVM) with OpenCL or PTX backends. +- GCC/G++ 13 or newer: Required to build and run TornadoVM native components. ### Install, Build, and Run