From 84a0ed89ed2ebc74f25e11e8506d614e90f8eb4d Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 25 Mar 2026 10:51:53 +0100 Subject: [PATCH 1/7] Use service names instead of container names To allow for multiple deployments on the same machine. --- .github/workflows/tests.yml | 4 ++-- docker-compose.yaml | 4 ---- docs/contributing/contributing.md | 28 ++++++++++++++-------------- docs/installation.md | 4 ++-- src/config.toml | 2 +- tests/conftest.py | 2 +- 6 files changed, 20 insertions(+), 24 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 18f61a49..99de4b57 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,9 +37,9 @@ jobs: - name: Run tests run: | marker="${{ matrix.php_api == true && 'php_api' || 'not php_api' }} and ${{ matrix.mutations == true && 'mut' || 'not mut' }}" - docker exec openml-python-rest-api coverage run -m pytest -n auto -v -m "$marker" + docker compose exec python-api coverage run -m pytest -n auto -v -m "$marker" - name: Produce coverage report - run: docker exec openml-python-rest-api coverage xml + run: docker compose exec python-api coverage xml - name: Upload results to Codecov uses: codecov/codecov-action@v4 with: diff --git a/docker-compose.yaml b/docker-compose.yaml index e93b548b..46e259fe 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,7 +2,6 @@ services: database: profiles: ["python", "php", "all"] image: "openml/test-database:v0.1.20260204" - container_name: "openml-test-database" environment: MYSQL_ROOT_PASSWORD: ok ports: @@ -28,7 +27,6 @@ services: elasticsearch: profiles: ["php", "all"] image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 - container_name: "openml-elasticsearch" platform: "linux/amd64" ports: - "9200:9200" # also known as /es (nginx) @@ -52,7 +50,6 @@ services: php-api: profiles: ["php", "all"] image: "openml/php-rest-api:v1.2.4" - container_name: "openml-php-rest-api" env_file: docker/php/.env ports: - "8002:80" @@ -70,7 +67,6 @@ services: python-api: profiles: ["python", "all"] - container_name: "openml-python-rest-api" build: context: . dockerfile: docker/python/Dockerfile diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md index 98f125df..09dafc75 100644 --- a/docs/contributing/contributing.md +++ b/docs/contributing/contributing.md @@ -34,19 +34,19 @@ required services for development is through [`docker compose`](https://docs.doc docker compose --profile all up -d ``` -This will spin up 5 containers, as defined in the `docker-compose.yaml` file: +This will spin up 5 services, as defined in the `docker-compose.yaml` file: - - `openml-test-database`: this is a mysql database prepopulated with test data. + - `database`: this is a mysql database prepopulated with test data. It is reachable from the host machine with port `3306`, by default it is configured to have a root user with password `"ok"`. - - `server-api-docs-1`: this container serves project documentation at `localhost:8000`. + - `docs`: this container serves project documentation at `localhost:8000`. These pages are built from the documents in the `docs/` directory of this repository, whenever you edit and save a file there, the page will immediately be updated. - - `openml-php-rest-api`: this container serves the old PHP REST API at `localhost:8002`. + - `php-api`: this container serves the old PHP REST API at `localhost:8002`. For example, visit [http://localhost:8002/api/v1/json/data/1](http://localhost:8002/api/v1/json/data/1) to fetch a JSON description of dataset 1. - - `openml-elasticsearch`: Elasticsearch, required for the PHP REST API to function. - - `openml-python-rest-api`: this container serves the new Python-based REST API at `localhost:8001`. + - `elasticsearch`: Elasticsearch, required for the PHP REST API to function. + - `python-api`: this container serves the new Python-based REST API at `localhost:8001`. For example, visit [http://localhost:8001/docs](http://localhost:8001/docs) to see the REST API documentation. Changes to the code in `src/` will be reflected in this container. @@ -57,7 +57,7 @@ This will spin up 5 containers, as defined in the `docker-compose.yaml` file: We can now run the full test suite, which takes about 4 minutes: ```bash -docker exec openml-python-rest-api python -m pytest tests +docker compose exec python-api python -m pytest tests ``` There are three important [test markers](https://docs.pytest.org/en/7.1.x/example/markers.html) to be aware of: @@ -69,8 +69,8 @@ In many cases during development it's sufficient to either run with `not php_api The `not slow` is only needed if a slow test would be included in your test selection. In many cases, you might prefer to only run the specific tests (or test modules) that you are working on and excluding it through markers may be unnecessary. Examples: - - `docker exec openml-python-rest-api python -m pytest tests -m "not php_api and not slow"`, here the test selection is made primarily through markers. This command takes a few seconds. - - `docker exec openml-python-rest-api python -m pytest tests/routers/openml/dataset_tag_test.py`, here the test selection is made through specifying the file with tests. Since this test file naturally includes neither migration tests (in `tests/routers/openml/migration`) nor the slow test (at `tests/routers/openml/datasets_list_datasets_test.py`), excluding tests through markers is unnecessary. This command takes a few seconds. + - `docker compose exec python-api python -m pytest tests -m "not php_api and not slow"`, here the test selection is made primarily through markers. This command takes a few seconds. + - `docker compose exec python-api python -m pytest tests/routers/openml/dataset_tag_test.py`, here the test selection is made through specifying the file with tests. Since this test file naturally includes neither migration tests (in `tests/routers/openml/migration`) nor the slow test (at `tests/routers/openml/datasets_list_datasets_test.py`), excluding tests through markers is unnecessary. This command takes a few seconds. You don't always need every container, often just having a database and the Python-based @@ -87,18 +87,18 @@ Refer to the `docker compose` documentation for more uses. ### Connecting to containers -To connect to a container, run: +To connect to a container of a service, run: ```bash -docker exec -it CONTAINER_NAME /bin/bash +docker compose exec SERVICE_NAME /bin/bash ``` -where `CONTAINER_NAME` is the name of the container. If you are unsure of your container -name, then `docker container ls` may help you find it. Assuming the default container +where `SERVICE_NAME` is the name of the service. If you are unsure of the service +name, then `docker compose ps` may help you find it. Assuming the default service names are used, you may connect to the Python-based REST API container using: ```bash -docker exec -it openml-python-rest-api /bin/bash +docker compose exec python-api /bin/bash ``` This is useful, for example, to run unit tests in the container: diff --git a/docs/installation.md b/docs/installation.md index 20943788..6ef8795e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -11,8 +11,8 @@ By default, the current code is also mounted into the Python REST API container For development, it should suffice to run the services from a fresh clone by running `docker compose --profile "python" up -d`. The REST API will be exposed on port 8001 on the host machine. To visit the Swagger Docs, visit http://localhost:8001/docs. -Once the containers are started, you can run tests with `docker exec -it openml-python-rest-api python -m pytest -m "not php_api" tests`. -For migration testing, which compares output of the Python-based REST API with the old PHP-based one, also start the PHP server (`docker compose --profile "php" --profile "python" up -d`) and include tests with the `php_api` marker/fixture: `docker exec -it openml-python-rest-api python -m pytest tests`. +Once the containers are started, you can run tests with `docker compose exec python-api python -m pytest -m "not php_api" tests`. +For migration testing, which compares output of the Python-based REST API with the old PHP-based one, also start the PHP server (`docker compose --profile "php" --profile "python" up -d`) and include tests with the `php_api` marker/fixture: `docker compose exec python-api python -m pytest tests`. !!! note diff --git a/src/config.toml b/src/config.toml index 79f7c557..f819ec94 100644 --- a/src/config.toml +++ b/src/config.toml @@ -8,7 +8,7 @@ allow_test_api_keys=true root_path="" [databases.defaults] -host="openml-test-database" +host="database" port="3306" # SQLAlchemy `dialect` and `driver`: https://docs.sqlalchemy.org/en/20/dialects/index.html drivername="mysql+aiomysql" diff --git a/tests/conftest.py b/tests/conftest.py index 32845cbc..89a7bcb9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,7 +16,7 @@ from main import create_api from routers.dependencies import expdb_connection, userdb_connection -PHP_API_URL = "http://openml-php-rest-api:80/api/v1/json" +PHP_API_URL = "http://php-api:80/api/v1/json" @contextlib.asynccontextmanager From 84c47d1c71ff1a47d960944414e1e28246a1a24b Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 25 Mar 2026 11:06:19 +0100 Subject: [PATCH 2/7] Do not expose ports to host network --- docker-compose.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 46e259fe..d7e16203 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,8 +4,6 @@ services: image: "openml/test-database:v0.1.20260204" environment: MYSQL_ROOT_PASSWORD: ok - ports: - - "3306:3306" healthcheck: test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] start_period: 30s @@ -19,8 +17,6 @@ services: build: context: . dockerfile: docker/docs/Dockerfile - ports: - - "8000:8000" volumes: - .:/docs @@ -28,9 +24,6 @@ services: profiles: ["php", "all"] image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 platform: "linux/amd64" - ports: - - "9200:9200" # also known as /es (nginx) - - "9300:9300" env_file: docker/elasticsearch/.env healthcheck: test: curl 127.0.0.1:9200/_cluster/health | grep -e "green" @@ -51,8 +44,6 @@ services: profiles: ["php", "all"] image: "openml/php-rest-api:v1.2.4" env_file: docker/php/.env - ports: - - "8002:80" depends_on: elasticsearch: condition: service_healthy @@ -70,8 +61,6 @@ services: build: context: . dockerfile: docker/python/Dockerfile - ports: - - "8001:8000" volumes: - .:/app - ./src/config.toml:/config/config.toml From d8822bd260616851cab0a9cdaf5f274e5e7fca72 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 25 Mar 2026 11:20:59 +0100 Subject: [PATCH 3/7] Specify a separate file for exposing container ports to host network --- compose.ports.yaml | 21 +++++++++++++++++++++ docker-compose.yaml => compose.yaml | 2 ++ docs/contributing/contributing.md | 13 +++++++++---- docs/installation.md | 4 ++-- 4 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 compose.ports.yaml rename docker-compose.yaml => compose.yaml (98%) diff --git a/compose.ports.yaml b/compose.ports.yaml new file mode 100644 index 00000000..cecae80c --- /dev/null +++ b/compose.ports.yaml @@ -0,0 +1,21 @@ +services: + database: + ports: + - "3306:3306" + + docs: + ports: + - "8000:8000" + + elasticsearch: + ports: + - "9200:9200" + - "9300:9300" + + php-api: + ports: + - "8002:80" + + python-api: + ports: + - "8001:8000" diff --git a/docker-compose.yaml b/compose.yaml similarity index 98% rename from docker-compose.yaml rename to compose.yaml index d7e16203..e1987182 100644 --- a/docker-compose.yaml +++ b/compose.yaml @@ -14,6 +14,8 @@ services: docs: profiles: ["all"] + ports: + - "8000:8000" build: context: . dockerfile: docker/docs/Dockerfile diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md index 09dafc75..84961e6f 100644 --- a/docs/contributing/contributing.md +++ b/docs/contributing/contributing.md @@ -33,24 +33,29 @@ required services for development is through [`docker compose`](https://docs.doc ```bash docker compose --profile all up -d ``` +or, to expose the services to the host network at the same time: +```bash +docker compose -f compose.yaml -f compose.ports.yaml --profile all up -d +``` -This will spin up 5 services, as defined in the `docker-compose.yaml` file: +This will spin up 5 services, as defined in the `compose.yaml` file: - `database`: this is a mysql database prepopulated with test data. - It is reachable from the host machine with port `3306`, by default it is configured - to have a root user with password `"ok"`. + By default it is configured to have a root user with password `"ok"`. - `docs`: this container serves project documentation at `localhost:8000`. These pages are built from the documents in the `docs/` directory of this repository, whenever you edit and save a file there, the page will immediately be updated. + - `elasticsearch`: Elasticsearch, required for the PHP REST API to function. - `php-api`: this container serves the old PHP REST API at `localhost:8002`. For example, visit [http://localhost:8002/api/v1/json/data/1](http://localhost:8002/api/v1/json/data/1) to fetch a JSON description of dataset 1. - - `elasticsearch`: Elasticsearch, required for the PHP REST API to function. - `python-api`: this container serves the new Python-based REST API at `localhost:8001`. For example, visit [http://localhost:8001/docs](http://localhost:8001/docs) to see the REST API documentation. Changes to the code in `src/` will be reflected in this container. +Exposing ports to the host network isn't needed for development, but may be useful to inspect responses directly from the host machine. + !!! note On arm-based Macs, you need to enable Rosetta emulation for Docker for the Elasticsearch container to work. diff --git a/docs/installation.md b/docs/installation.md index 6ef8795e..dbda92b3 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -4,7 +4,7 @@ See also ["Contributing"](contributing/contributing.md). The primary way to run this service is through a Docker container. The REST API needs to be able to connect to a MySQL database with the OpenML "openml" and "openml_expdb" databases. -The `docker-compose.yaml` file of this project defines these together out of the box. +The `compose.yaml` file of this project defines these together out of the box. This is useful for development purposes, but the database does not persist between restarts in the current configuration. By default, the current code is also mounted into the Python REST API container (again, for development purposes). @@ -21,4 +21,4 @@ For migration testing, which compares output of the Python-based REST API with t When we start testing more upload functionality, for which the PHP API needs built indices, we'll work on an ES image with prebuilt indices. Information for a production deployment will follow, in a nutshell you need to configure the REST API to connect to a persistent database, -which can be the one defined in `docker-compose.yaml` if has an appropriately mounted volume. +which can be the one defined in `compose.yaml` if has an appropriately mounted volume. From f19e30a650b1c9ed6dc2c3c4c715f7ed36a491de Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 25 Mar 2026 11:30:11 +0100 Subject: [PATCH 4/7] Remove stray port bind --- compose.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/compose.yaml b/compose.yaml index e1987182..d7e16203 100644 --- a/compose.yaml +++ b/compose.yaml @@ -14,8 +14,6 @@ services: docs: profiles: ["all"] - ports: - - "8000:8000" build: context: . dockerfile: docker/docs/Dockerfile From 63bc988736d50154e69b31d74341a1e2d9dabff2 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 25 Mar 2026 11:31:44 +0100 Subject: [PATCH 5/7] Fix typo --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index dbda92b3..39467a5a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -21,4 +21,4 @@ For migration testing, which compares output of the Python-based REST API with t When we start testing more upload functionality, for which the PHP API needs built indices, we'll work on an ES image with prebuilt indices. Information for a production deployment will follow, in a nutshell you need to configure the REST API to connect to a persistent database, -which can be the one defined in `compose.yaml` if has an appropriately mounted volume. +which can be the one defined in `compose.yaml` if it has an appropriately mounted volume. From 26be59783a3f546573d33f4a88fbb27d392d39b5 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 25 Mar 2026 11:42:34 +0100 Subject: [PATCH 6/7] Revise compose profiles --- compose.yaml | 6 ++---- docs/contributing/contributing.md | 2 +- docs/installation.md | 7 ++++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/compose.yaml b/compose.yaml index d7e16203..c51a4afc 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,5 @@ services: database: - profiles: ["python", "php", "all"] image: "openml/test-database:v0.1.20260204" environment: MYSQL_ROOT_PASSWORD: ok @@ -21,7 +20,6 @@ services: - .:/docs elasticsearch: - profiles: ["php", "all"] image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 platform: "linux/amd64" env_file: docker/elasticsearch/.env @@ -41,7 +39,7 @@ services: memory: 250M php-api: - profiles: ["php", "all"] + profiles: ["apis", "all"] image: "openml/php-rest-api:v1.2.4" env_file: docker/php/.env depends_on: @@ -57,7 +55,7 @@ services: interval: 1m python-api: - profiles: ["python", "all"] + profiles: ["apis", "all"] build: context: . dockerfile: docker/python/Dockerfile diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md index 84961e6f..f2e1eb6c 100644 --- a/docs/contributing/contributing.md +++ b/docs/contributing/contributing.md @@ -82,7 +82,7 @@ You don't always need every container, often just having a database and the Pyth REST API may be enough. In that case, only specify those services: ```bash -docker compose --profile python up -d +docker compose up python-api -d ``` Refer to the `docker compose` documentation for more uses. diff --git a/docs/installation.md b/docs/installation.md index 39467a5a..e0a7c449 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -8,11 +8,12 @@ The `compose.yaml` file of this project defines these together out of the box. This is useful for development purposes, but the database does not persist between restarts in the current configuration. By default, the current code is also mounted into the Python REST API container (again, for development purposes). -For development, it should suffice to run the services from a fresh clone by running `docker compose --profile "python" up -d`. -The REST API will be exposed on port 8001 on the host machine. To visit the Swagger Docs, visit http://localhost:8001/docs. +For development, it should suffice to run the services from a fresh clone by running `docker compose up python-api -d`. +If you want to make sure to bind the exposed container ports to the host machine then you will need to use the `compose.ports.yaml` file too (`docker compose -f compose.yaml -f compose.ports.yaml up python-api -d`). +The REST API will then be exposed on port 8001 on the host machine. To visit the Swagger Docs, visit http://localhost:8001/docs. Once the containers are started, you can run tests with `docker compose exec python-api python -m pytest -m "not php_api" tests`. -For migration testing, which compares output of the Python-based REST API with the old PHP-based one, also start the PHP server (`docker compose --profile "php" --profile "python" up -d`) and include tests with the `php_api` marker/fixture: `docker compose exec python-api python -m pytest tests`. +For migration testing, which compares output of the Python-based REST API with the old PHP-based one, also start the PHP server (`docker compose --profile "apis" up -d`) and include tests with the `php_api` marker/fixture: `docker compose exec python-api python -m pytest tests`. !!! note From 9a857cc4667a715d360ae33bfd96fd781005887e Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 25 Mar 2026 11:45:57 +0100 Subject: [PATCH 7/7] Call services directly instead of through profile --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 99de4b57..0b04dfba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,11 +28,11 @@ jobs: # https://github.com/docker/compose/issues/10596 - name: Start services run: | - profiles="--profile python" + services="python-api" if [ "${{ matrix.php_api }}" = "true" ]; then - profiles="$profiles --profile php" + services="$services php-api" fi - docker compose $profiles up --detach --wait --remove-orphans || exit $(docker compose ps -q | xargs docker inspect -f '{{.State.ExitCode}}' | grep -v '^0' | wc -l) + docker compose up $services --detach --wait --remove-orphans || exit $(docker compose ps -q | xargs docker inspect -f '{{.State.ExitCode}}' | grep -v '^0' | wc -l) - name: Run tests run: |