From 47642151cd9863695bb9ad1d1f02d0995e6575e3 Mon Sep 17 00:00:00 2001 From: Hammad Usmani Date: Thu, 25 Sep 2025 19:36:26 +0000 Subject: [PATCH 01/10] Refactor to upgrade to orchestrate multiple containers --- docker-compose.yml | 0 docker/{ => llm}/Dockerfile | 0 docker/{ => llm}/entrypoint.sh | 0 start.sh | 0 stop.sh | 0 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 docker-compose.yml rename docker/{ => llm}/Dockerfile (100%) rename docker/{ => llm}/entrypoint.sh (100%) create mode 100644 start.sh create mode 100644 stop.sh diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e69de29 diff --git a/docker/Dockerfile b/docker/llm/Dockerfile similarity index 100% rename from docker/Dockerfile rename to docker/llm/Dockerfile diff --git a/docker/entrypoint.sh b/docker/llm/entrypoint.sh similarity index 100% rename from docker/entrypoint.sh rename to docker/llm/entrypoint.sh diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..e69de29 diff --git a/stop.sh b/stop.sh new file mode 100644 index 0000000..e69de29 From 0d7f5b0d10ab2321602a7c9e60542bcd48fe8375 Mon Sep 17 00:00:00 2001 From: Hammad Usmani Date: Thu, 25 Sep 2025 20:07:36 +0000 Subject: [PATCH 02/10] Update initial prototype of orchestrator and documentation --- README.md | 7 +++++++ docker-compose.yml | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/README.md b/README.md index 970b089..8af3daa 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,14 @@ # PAM Personal Assistance Machine +# Large Language Model (LLM) Files (GGUF) +The system is locally hosted. The prerequisite are the associated GGUF files. Please create a directory called `symlink` in the root that directs the orchestrator to find the LLM files. + # Quickstart 1. `docker build -t pam docker/` 2. `sudo docker run -d -v .:/data/ -p 10000:10000 pam` + +# Networking Quickstart + +The reverse proxy is configured externally. For example, [this configuration file](https://github.com/hammad93/hurricane-server/blob/main/docker/proxy/conf.d/open-webui.conf) is an example of how to configure it. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index e69de29..2bed783 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +services: + llm: + build: + context: ./docker/llm + ports: + - "10000:10000" + volumes: + - ./symlink:/data + restart: unless-stopped + open-webui: + image: ghcr.io/open-webui/open-webui:main + container_name: open-webui + ports: + - "8080:8080" + volumes: + - /var/lib/docker/volumes/open-webui/_data:/app/backend/data + restart: unless-stopped \ No newline at end of file From 10284a8bfd6d56e6647afadb351dde74b7450081 Mon Sep 17 00:00:00 2001 From: Hammad Usmani Date: Thu, 25 Sep 2025 20:28:33 +0000 Subject: [PATCH 03/10] Update repo with TPM functionality and symlink documentation --- .gitignore | 1 + README.md | 9 ++++++++- docker/llm/Dockerfile | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0edb700 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +symlink/ \ No newline at end of file diff --git a/README.md b/README.md index 8af3daa..18ffe0f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,14 @@ Personal Assistance Machine # Large Language Model (LLM) Files (GGUF) -The system is locally hosted. The prerequisite are the associated GGUF files. Please create a directory called `symlink` in the root that directs the orchestrator to find the LLM files. +The system is locally hosted. The prerequisite are the associated GGUF files. Please create a directory called `symlink` in the root that directs the orchestrator to find the LLM files. To create the symbolic link, use the following command by replacing the `/path/to/ggufs` with the path to the LLM files. + +`ln -s /path/to/ggufs ./symlink` + +# Trusted Platform Module (TPM) +The TPM can be configured such that the application is compliant with rigorous security standards and for data science purposes such as random number generation. The following command describes how to configure a TPM chip to work with a container. + +`docker run --device /dev/tpm0:/dev/tpm0 --device /dev/tpmrm0:/dev/tpmrm0` # Quickstart diff --git a/docker/llm/Dockerfile b/docker/llm/Dockerfile index 1ad5d5d..d370c45 100644 --- a/docker/llm/Dockerfile +++ b/docker/llm/Dockerfile @@ -1,5 +1,6 @@ FROM continuumio/anaconda3 RUN apt-get update && apt-get install -y build-essential +# RUN apt-get install -y tpm2tools libtss2-dev # Install llama.cpp RUN git clone https://github.com/ggerganov/llama.cpp.git From 30af055f12a3f366f10c49d627b306d5e2cd4947 Mon Sep 17 00:00:00 2001 From: Hammad Usmani Date: Thu, 25 Sep 2025 20:34:07 +0000 Subject: [PATCH 04/10] Change LLAMA.cpp server configuration to use available RAM and set to offline mode --- docker/llm/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/llm/entrypoint.sh b/docker/llm/entrypoint.sh index 2607f05..3527123 100644 --- a/docker/llm/entrypoint.sh +++ b/docker/llm/entrypoint.sh @@ -4,4 +4,6 @@ cd /llama.cpp/build/bin/ --port 10000 \ --ctx-size 8192 \ --threads 64 \ - --host 0.0.0.0 + --host 0.0.0.0 \ + --no-nmap \ + --offline From 85206faf3f5112dbd5d55b8dcb3972cbddefe60b Mon Sep 17 00:00:00 2001 From: Hammad Usmani Date: Thu, 25 Sep 2025 20:38:24 +0000 Subject: [PATCH 05/10] Update docs with instructions on how to update the model --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 18ffe0f..9a5ded4 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ The system is locally hosted. The prerequisite are the associated GGUF files. Pl `ln -s /path/to/ggufs ./symlink` +## Updating Model +If we are switching the LLM or otherwise updating it, we need to ensure that the llama.cpp server configuration reflects it. In the `docker/llm` directory there is a `entrypoint.sh` file that contains the initial configuration. Please update the GGUF filename with the first in the sequence for the `--model` flag. + # Trusted Platform Module (TPM) The TPM can be configured such that the application is compliant with rigorous security standards and for data science purposes such as random number generation. The following command describes how to configure a TPM chip to work with a container. From 52e7cb70a827bda406daa3e58e05bb383237a4e2 Mon Sep 17 00:00:00 2001 From: Hammad Usmani Date: Thu, 25 Sep 2025 20:40:35 +0000 Subject: [PATCH 06/10] Initialize start and stop scripts for orchestration --- start.sh | 2 ++ stop.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/start.sh b/start.sh index e69de29..e878b75 100644 --- a/start.sh +++ b/start.sh @@ -0,0 +1,2 @@ +docker compose build --no-cache +docker compose up -d \ No newline at end of file diff --git a/stop.sh b/stop.sh index e69de29..1c3e9ca 100644 --- a/stop.sh +++ b/stop.sh @@ -0,0 +1 @@ +docker compose down \ No newline at end of file From 4946e356897b0338ec0a0b8e7f2dd0e346168130 Mon Sep 17 00:00:00 2001 From: Hammad Usmani Date: Thu, 25 Sep 2025 21:16:16 +0000 Subject: [PATCH 07/10] Include documentation on how to download models locally --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 9a5ded4..b8420ae 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,13 @@ The system is locally hosted. The prerequisite are the associated GGUF files. Pl ## Updating Model If we are switching the LLM or otherwise updating it, we need to ensure that the llama.cpp server configuration reflects it. In the `docker/llm` directory there is a `entrypoint.sh` file that contains the initial configuration. Please update the GGUF filename with the first in the sequence for the `--model` flag. +## Downloading Models +Currently, the design supports command line interfaces such as the [huggingface-cli](https://huggingface.co/docs/huggingface_hub/en/guides/cli). Please reference the following command to understand how to download a LLM to a locally specified directory. + +``` +sudo huggingface-cli download DevQuasar/swiss-ai.Apertus-70B-Instruct-2509-GGUF --include "*Q8*" --local-dir . +``` + # Trusted Platform Module (TPM) The TPM can be configured such that the application is compliant with rigorous security standards and for data science purposes such as random number generation. The following command describes how to configure a TPM chip to work with a container. From 08f8eb1e1b6a4bf42a99f2861ce6e50c6063bf84 Mon Sep 17 00:00:00 2001 From: Hammad Usmani Date: Thu, 25 Sep 2025 21:35:09 +0000 Subject: [PATCH 08/10] Change model to Apertus and update docs --- README.md | 1 + docker/llm/entrypoint.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8420ae..8b63e64 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # PAM Personal Assistance Machine +This artificial intelligence consists of a large language model for tropical storm simulations. # Large Language Model (LLM) Files (GGUF) The system is locally hosted. The prerequisite are the associated GGUF files. Please create a directory called `symlink` in the root that directs the orchestrator to find the LLM files. To create the symbolic link, use the following command by replacing the `/path/to/ggufs` with the path to the LLM files. diff --git a/docker/llm/entrypoint.sh b/docker/llm/entrypoint.sh index 3527123..0836b7d 100644 --- a/docker/llm/entrypoint.sh +++ b/docker/llm/entrypoint.sh @@ -1,8 +1,8 @@ cd /llama.cpp/build/bin/ ./llama-server \ - --model /data/DeepSeek-R1-GGUF/DeepSeek-R1-UD-Q2_K_XL/DeepSeek-R1-UD-Q2_K_XL-00001-of-00005.gguf \ + --model /data/swiss-ai.Apertus-70B-Instruct-2509.Q8_0-00001-of-00006.gguf \ --port 10000 \ - --ctx-size 8192 \ + --ctx-size 4096 \ --threads 64 \ --host 0.0.0.0 \ --no-nmap \ From 2705530b9fe41c5205a52aacd2892ce010d750cd Mon Sep 17 00:00:00 2001 From: Hammad Usmani Date: Thu, 25 Sep 2025 21:39:13 +0000 Subject: [PATCH 09/10] Include check for symbolic link and print files in startup script --- start.sh | 10 ++++++++++ stop.sh | 1 + 2 files changed, 11 insertions(+) diff --git a/start.sh b/start.sh index e878b75..8712062 100644 --- a/start.sh +++ b/start.sh @@ -1,2 +1,12 @@ +#!/bin/bash +# Check if the symbolic link exists +if [[ -L "./symlink" ]]; then + echo "Symbolic link exists: $link_path with files:" + ls -la ./symlink +else + echo "Error: Symbolic link does not exist: ./symlink" >&2 + exit 1 +fi + docker compose build --no-cache docker compose up -d \ No newline at end of file diff --git a/stop.sh b/stop.sh index 1c3e9ca..dd327ba 100644 --- a/stop.sh +++ b/stop.sh @@ -1 +1,2 @@ +#!/bin/bash docker compose down \ No newline at end of file From 3cfa8b824a1b1f3c4beadc310fe59fc715d3f1b6 Mon Sep 17 00:00:00 2001 From: Hammad Usmani Date: Fri, 26 Sep 2025 03:38:15 +0000 Subject: [PATCH 10/10] Include all changes after debugging --- .gitignore | 3 ++- docker-compose.yml | 1 + docker/llm/Dockerfile | 4 ++-- docker/llm/entrypoint.sh | 3 ++- start.sh | 4 ++-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 0edb700..110b8b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -symlink/ \ No newline at end of file +symlink/ +symlink \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 2bed783..7443f2f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,4 +14,5 @@ services: - "8080:8080" volumes: - /var/lib/docker/volumes/open-webui/_data:/app/backend/data + network_mode: host restart: unless-stopped \ No newline at end of file diff --git a/docker/llm/Dockerfile b/docker/llm/Dockerfile index d370c45..9c09ae6 100644 --- a/docker/llm/Dockerfile +++ b/docker/llm/Dockerfile @@ -3,10 +3,10 @@ RUN apt-get update && apt-get install -y build-essential # RUN apt-get install -y tpm2tools libtss2-dev # Install llama.cpp -RUN git clone https://github.com/ggerganov/llama.cpp.git +RUN git clone -b apertus-implementation https://github.com/pwilkin/llama.cpp.git WORKDIR llama.cpp RUN apt install cmake -y -RUN cmake -B build +RUN cmake -B build -DLLAMA_CURL=OFF RUN cmake --build build --config Release # Entrypoint diff --git a/docker/llm/entrypoint.sh b/docker/llm/entrypoint.sh index 0836b7d..d32236f 100644 --- a/docker/llm/entrypoint.sh +++ b/docker/llm/entrypoint.sh @@ -5,5 +5,6 @@ cd /llama.cpp/build/bin/ --ctx-size 4096 \ --threads 64 \ --host 0.0.0.0 \ - --no-nmap \ + --mlock \ + --no-mmap \ --offline diff --git a/start.sh b/start.sh index 8712062..4c0fc0c 100644 --- a/start.sh +++ b/start.sh @@ -1,8 +1,8 @@ #!/bin/bash # Check if the symbolic link exists if [[ -L "./symlink" ]]; then - echo "Symbolic link exists: $link_path with files:" - ls -la ./symlink + echo "Symbolic link exists: ./symlink with files:" + ls -la symlink/ else echo "Error: Symbolic link does not exist: ./symlink" >&2 exit 1