add jwt and model dependent queue

add JSON Web Token as a user login service.
This makes it easier to make the proxy distributed and does not need any
user text file. And users can be added without restarting the server.
It also make it more secure in a non https env.

Extracted the queue part of the code to a subset of classes so different
queues can easily be implemented and used. As long as they conform to an
API.

Add a model depended queue, it prioritize the servers which have a
certain model already loaded. It also can have black list and white list of
which model it can handle due to cpu/ram/gpu. It conforms to the
shortest queue principle of the simple queue

Add logging instead of prints to the stderr output.

Add github-ci

Signed-off-by: Robert Marklund <robbelibobban@gmail.com>

Author: trollkarlen

.github/workflows/github-actions-build-wheel.yml

@@ -0,0 +1,50 @@
+---
+name: build wheel
+
+"on":
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  build-wheel:
+    retry: 2
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.13
+
+      - name: Set up the cache
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: cache-python-packages
+
+      - name: Set up the project
+        run: |
+          pip install poetry
+          # poetry config virtualenvs.in-project true
+
+      # build for all versions of python ?
+      - name: build
+        run: poetry build
+
+      - name: Archive build and wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-and-wheel
+          path: dist

.github/workflows/github-actions-docker.yml

@@ -0,0 +1,42 @@
+---
+name: build docker images
+
+"on":
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  docker:
+    retry: 2
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # - name: Login to Docker Hub
+      #  uses: docker/login-action@v3
+      #  with:
+      #    username: ${{ vars.DOCKERHUB_USERNAME }}
+      #    password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: false
+          tags: ollama_proxy_server:latest

.github/workflows/github-actions-quality.yml

@@ -0,0 +1,81 @@
+---
+name: run quality tests
+
+"on":
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  quality:
+    retry: 2
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.13
+
+      - name: Set up the cache
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: cache-python-packages
+
+      - name: Set up the project
+        run: |
+          pip install poetry safety tox
+          # poetry config virtualenvs.in-project true
+
+      - name: Run ruff lint
+        run: tox -e lint
+
+      - name: Run pylint
+        run: tox -e pylint
+
+  code-saftey:
+    retry: 2
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out master
+        uses: actions/checkout@v4
+
+      - name: Assign variable
+        id: safety_api_key
+        run: echo '::set-output name=secret::${{secrets.SAFETY_API_KEY}}'
+
+      - name: Run Safety CLI to check for vulnerabilities
+        if: steps.secret.outputs.safety_api_key
+        uses: pyupio/safety-action@v1
+        with:
+          # api-key: ${{ secrets.SAFETY_API_KEY }}
+          # To always see detailed output from this action
+          args: --detailed-output
+        env:  # Or as an environment variable
+          SAFETY_API_KEY: ${{ secrets.SAFETY_API_KEY }}
+
+  shell-check:
+    retry: 2
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out master
+        uses: actions/checkout@v4
+
+      - name: install shellcheck
+        run: apt install shellcheck
+
+      - name: Run shellcheck
+        run: |
+          ./scripts/run-shellcheck.sh

.github/workflows/github-actions-tests.yml

@@ -0,0 +1,59 @@
+---
+name: run tox test
+
+"on":
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  tests:
+    retry: 2
+    strategy:
+      max-parallel: 6
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]  # , "3.14"]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up the cache
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-python-packages
+        with:
+          path: .venv
+          key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ env.cache-name }}
+          restore-keys: |
+            ${{ matrix.os }}-${{ matrix.python-version }}-
+            ${{ matrix.os }}-
+
+      - name: Set up the project
+        run: |
+          pip install poetry tox
+          # poetry config virtualenvs.in-project true
+
+      - name: Run the test suite
+        run: tox -f test -l | tr '\n' ',' | xargs tox -e
+
+      - name: Archive coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: code-coverage-report
+          path: coverage.xml

.yamllint.yml

@@ -0,0 +1,7 @@
+---
+extends: default
+ignore: ".tox*\n.git*"
+rules:
+    line-length:
+      max: 120
+      level: warning

Dockerfile

@@ -1,28 +1,67 @@
-FROM python:3.11
+FROM python:3.13 AS python-base
 
-# Update packagtes, install necessary tools into the base image, clean up and clone git repository
-RUN apt update \
-    && apt install -y --no-install-recommends --no-install-suggests git apache2 \
-    && apt autoremove -y --purge \
-    && apt clean \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    && git clone https://github.com/ParisNeo/ollama_proxy_server.git
+# https://python-poetry.org/docs#ci-recommendations
+ENV POETRY_VERSION=2.1.3
+ENV POETRY_HOME=/opt/poetry
+ENV POETRY_VENV=/opt/poetry-venv
 
-# Change working directory to cloned git repository
-WORKDIR ollama_proxy_server
+# Tell Poetry where to place its cache and virtual environment
+ENV POETRY_CACHE_DIR=/opt/.cache
 
-# Install all needed requirements
-RUN pip3 install -e .
+# Create stage for Poetry installation
+FROM python-base AS poetry-base
 
-# Copy config.ini and authorized_users.txt into project working directory
-COPY config.ini .
-COPY authorized_users.txt .
+# Creating a virtual environment just for poetry and install it with pip
+RUN python3 -m venv $POETRY_VENV \
+	&& $POETRY_VENV/bin/pip install -U pip setuptools \
+	&& $POETRY_VENV/bin/pip install poetry==${POETRY_VERSION}
+
+# Copy Poetry to app image
+# COPY --from=poetry-base ${POETRY_VENV} ${POETRY_VENV}
+
+# Add Poetry to PATH
+ENV PATH="${PATH}:${POETRY_VENV}/bin"
+
+# temp dir
+WORKDIR /app-tmp
+
+# Copy Dependencies
+COPY --chown=worker:worker poetry.lock pyproject.toml ./
+
+# [OPTIONAL] Validate the project is properly configured
+RUN poetry check
+
+# Copy Application
+COPY --chown=worker:worker . ./
+
+# build and install the app
+RUN poetry build
 
-# Start the proxy server as entrypoint
-ENTRYPOINT ["ollama_proxy_server"]
+# Create a new stage from the base python image
+FROM python-base AS ollama-proxy-server
+
+RUN adduser worker
+WORKDIR /home/worker
+
+# copy wheel and tgz
+COPY --from=poetry-base /app-tmp/dist ./dist
+
+RUN pip install dist/*.whl
+
+RUN rm -fr ./dist
+
+# copy entry point
+COPY --from=poetry-base /app-tmp/entry-point.sh /
+
+USER worker
+
+# Copy config.ini and authorized_users.txt into project working directory
+COPY --chown=worker:worker config.ini .
+COPY --chown=worker:worker authorized_users.txt .
 
 # Do not buffer output, e.g. logs to stdout
 ENV PYTHONUNBUFFERED=1
 
-# Set command line parameters
-CMD ["--config", "./config.ini", "--users_list", "./authorized_users.txt", "--port", "8080"]
+# Run Application
+EXPOSE 8080
+ENTRYPOINT ["/entry-point.sh"]

config.ini

@@ -1,8 +1,11 @@
 [DefaultServer]
 url = http://localhost:11434
+model_white_list = ["llama3.2:1b"]
+model_black_list = ["llama3.2:800b"]
 
 [SecondaryServer]
 url = http://localhost:3002
+model_black_list = ["llama3.2:200b"]
 
 # Add more servers as you need.
 

entry-point.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+[[ -n "$DEBUG" ]] && set -x
+
+JWT_KEY="${JWT_KEY:-}"
+JWT_KEY_FILE="$HOME/.jwt-key"
+
+set -Euo pipefail
+set -e
+
+if [[ -z "$JWT_KEY" ]] || [[ ! -s "$JWT_KEY_FILE" ]]; then
+    >&2 echo "No jwt key file or env set creating key and saving in $JWT_KEY_FILE"
+    JWT_KEY="$(openssl rand -hex 64)"
+    echo -ne "$JWT_KEY" > "$HOME/.jwt-key"
+elif [[ -s "$JWT_KEY_FILE" ]]; then
+    >&2 echo "found jwt key file '$JWT_KEY_FILE' setting to env"
+    JWT_KEY="$(cat "$JWT_KEY_FILE")"
+fi
+
+if (( ${#@} )) && [ -t 0 ]; then
+    # user whants interactive
+    "${@}"
+else
+    ollama_proxy_server "--config=./config.ini" "--users_list=./authorized_users.txt" "--port=8080" "${@}"
+fi