CI Fixups

.devcontainer/Dockerfile

@@ -1,7 +1,11 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-FROM mcr.microsoft.com/vscode/devcontainers/base AS base
+# TODO: FIXME: Temporarily pin to bookworm until latest (trixie) base image is fully supported.
+# See Also:
+# - <https://github.com/microsoft/MLOS/issues/1002>
+# - .devcontainer/devcontainer.json
+FROM mcr.microsoft.com/vscode/devcontainers/base:bookworm AS base
 
 # Add some additional packages for the devcontainer terminal environment.
 USER root
@@ -80,6 +84,8 @@ RUN echo "Setup miniconda" \
     && /opt/conda/bin/conda init \
     && /opt/conda/bin/conda config --set channel_priority strict \
     && /opt/conda/bin/conda info \
+    && /opt/conda/bin/conda config --show \
+    && /opt/conda/bin/conda tos accept --override-channels --channel defaults \
     && /opt/conda/bin/conda update -v -y -n base -c conda-forge -c defaults --all \
     && /opt/conda/bin/conda list -n base \
     && /opt/conda/bin/conda install -v -y -n base -c conda-forge -c defaults conda-libmamba-solver \

.devcontainer/devcontainer.json

@@ -31,11 +31,22 @@
     "features": {
         "ghcr.io/devcontainers/features/azure-cli:1": {},
         //"ghcr.io/devcontainers/features/conda:1": {},
-        "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
+        "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
+            // Note: this may be required when moving to Debian Trixie base image.
+            // See Also:
+            // - <https://github.com/microsoft/MLOS/issues/1002>
+            // - .devcontainer/Dockerfile
+            //"moby": false
+        },
         "ghcr.io/devcontainers/features/github-cli:1": {},
         "ghcr.io/stuartleeks/dev-container-features/azure-cli-persistence:0": {},
         "ghcr.io/stuartleeks/dev-container-features/shell-history:0": {}
     },
+    "forwardPorts": [
+        // Make the nginx instance started as a part of `make doc` available to view
+        // the coverage results.
+        8080
+    ],
     // Use 'postCreateCommand' to run commands after the container is created.
     // "postCreateCommand": "python --version",
     // Configure tool-specific properties.

.devcontainer/scripts/run-devcontainer.sh

@@ -50,5 +50,6 @@ docker run -it --rm \
     --env http_proxy="${http_proxy:-}" \
     --env https_proxy="${https_proxy:-}" \
     --env no_proxy="${no_proxy:-}" \
+    --add-host host.docker.internal:host-gateway \
     mlos-devcontainer \
     $*

.github/workflows/devcontainer.yml

@@ -108,7 +108,18 @@ jobs:
       timeout-minutes: 3
       run: |
         set -x
+        gid=`id -g`
+        docker_sock_gid=`stat -c '%g' /var/run/docker.sock`
+        docker_gid=`getent group docker | cut -d: -f3 || echo 999`
+        if [ -n "$docker_sock_gid" ]; then
+          docker_gid=$docker_sock_gid
+          echo "Using docker gid: $docker_gid"
+        fi
+        if [ "$docker_sock_gid" != "$docker_gid" ] && [ -n "$docker_sock_gid" != "$gid" ] && [ "$docker_sock_gid" != 0 ]; then
+          echo "WARNING: docker.sock gid ($docker_sock_gid) does not match host docker group gid ($docker_gid) or user gid ($gid)." >&2
+        fi
         docker run -d --rm --user root \
+          --group-add $docker_sock_gid \
           --volume /var/run/docker.sock:/var/run/docker.sock \
           --env DOCKER_BUILDKIT=$DOCKER_BUILDKIT \
           --volume $(pwd):/workspaces/MLOS \
@@ -129,6 +140,28 @@ jobs:
     - name: Print some debug info from inside the container
       run: |
         docker exec --user vscode --env USER=vscode mlos-devcontainer printenv
+        docker exec --user vscode --env USER=vscode mlos-devcontainer id
+        docker exec --user vscode --env USER=vscode mlos-devcontainer ls -l /var/run/docker.sock
+        docker exec --user vscode --env USER=vscode mlos-devcontainer stat /var/run/docker.sock
+
+    - name: Check that docker is usable inside the devcontainer
+      timeout-minutes: 2
+      run: |
+        set -x
+        docker_sock_gid=`stat -c '%g' /var/run/docker.sock`
+        if ! [ -w /var/run/docker.sock ] \
+        || ! docker exec --user vscode --env USER=vscode mlos-devcontainer id -G | sed 's/ /\n/' | grep -q -w $docker_sock_gid
+        then
+          echo "ERROR: vscode user in container does not have access to docker.sock group (gid: $docker_sock_gid)" >&2
+          exit 1
+        fi
+        docker exec --user vscode --env USER=vscode mlos-devcontainer docker version
+        docker exec --user vscode --env USER=vscode mlos-devcontainer docker info
+        docker exec --user vscode --env USER=vscode mlos-devcontainer docker buildx version
+        if ! docker exec --user vscode --env USER=vscode mlos-devcontainer docker buildx ls | grep linux/; then
+          echo "ERROR: docker buildx does not have linux builder available" >&2
+          exit 1
+        fi
 
     - name: Check that github.com is in the ssh known_hosts file
       run: |
@@ -170,21 +203,26 @@ jobs:
         test_count=$(docker exec --user vscode --env USER=vscode mlos-devcontainer \
           conda run -n mlos python -m pytest -svxl -n auto --collect-only --rootdir /workspaces/MLOS -s --cache-clear \
           | grep -c '<Function ')
-        if [ "${test_count:-0}" -lt 725 ]; then echo "Expected at least 725 tests, got '$test_count'" >&2; exit 1; fi
+        # if [ "${test_count:-0}" -lt 800 ]; then echo "Expected at least 800 tests, got '$test_count'" >&2; exit 1; fi
 
         # Now actually run the tests.
         docker exec --user vscode --env USER=vscode mlos-devcontainer make CONDA_INFO_LEVEL=-v test
 
+    - name: Upload coverage report as build artifact
+      if: always() # ensures it runs even if tests fail
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-report
+        path: coverage.xml
+
     - name: Generate and test binary distribution files
       timeout-minutes: 10
       run: |
         set -x
         docker exec --user vscode --env USER=vscode mlos-devcontainer make CONDA_INFO_LEVEL=-v dist dist-test
 
     - name: Test rebuilding the devcontainer in the devcontainer
-      # FIXME:
-      # timeout-minutes: 3
-      timeout-minutes: 10
+      timeout-minutes: 5
       run: |
         set -x
         git --no-pager diff --exit-code

.vscode/settings.json

@@ -176,5 +176,17 @@
     "githubPullRequests.experimental.chat": true,
     "github.copilot.chat.codesearch.enabled": true,
     "github.copilot.chat.copilotDebugCommand.enabled": true,
-    "github.copilot.chat.reviewSelection.enabled": true
+    "github.copilot.chat.reviewSelection.enabled": true,
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda",
+    // Make the nginx instance started as a part of `make doc` available to view
+    // the coverage results.
+    "remote.localPortHost": "localhost",
+    "remote.SSH.defaultForwardedPorts": [
+        {
+            "name": "nginx for doc viewing",
+            "localPort": 8080,
+            "remotePort": 8080
+        }
+    ]
 }

conda-envs/mlos-3.10.yml

@@ -20,6 +20,9 @@ dependencies:
   - python=3.10
   # See comments in mlos.yml.
   #- gcc_linux-64
+  # Install an SMAC requirement pre-compiled from conda-forge.
+  # See Also: https://github.com/microsoft/MLOS/issues/1001
+  - pyrfr>=0.9.0
   - pip:
     - bump2version
     - check-jsonschema
@@ -30,7 +33,7 @@ dependencies:
     - pylint==3.3.7
     - tomlkit
     - mypy==1.15.0
-    - pyright==1.1.400
+    - pyright==1.1.406
     - pandas-stubs
     - types-beautifulsoup4
     - types-colorama

conda-envs/mlos-3.11.yml

@@ -20,6 +20,9 @@ dependencies:
   - python=3.11
   # See comments in mlos.yml.
   #- gcc_linux-64
+  # Install an SMAC requirement pre-compiled from conda-forge.
+  # See Also: https://github.com/microsoft/MLOS/issues/1001
+  - pyrfr>=0.9.0
   - pip:
     - bump2version
     - check-jsonschema
@@ -30,7 +33,7 @@ dependencies:
     - pylint==3.3.7
     - tomlkit
     - mypy==1.15.0
-    - pyright==1.1.400
+    - pyright==1.1.406
     - pandas-stubs
     - types-beautifulsoup4
     - types-colorama

conda-envs/mlos-3.12.yml

@@ -22,6 +22,9 @@ dependencies:
   - python=3.12
   # See comments in mlos.yml.
   #- gcc_linux-64
+  # Install an SMAC requirement pre-compiled from conda-forge.
+  # See Also: https://github.com/microsoft/MLOS/issues/1001
+  - pyrfr>=0.9.0
   - pip:
     - bump2version
     - check-jsonschema
@@ -32,7 +35,7 @@ dependencies:
     - pylint==3.3.7
     - tomlkit
     - mypy==1.15.0
-    - pyright==1.1.400
+    - pyright==1.1.406
     - pandas-stubs
     - types-beautifulsoup4
     - types-colorama

conda-envs/mlos-3.13.yml

@@ -22,6 +22,9 @@ dependencies:
   - python=3.13
   # See comments in mlos.yml.
   #- gcc_linux-64
+  # Install an SMAC requirement pre-compiled from conda-forge.
+  # See Also: https://github.com/microsoft/MLOS/issues/1001
+  - pyrfr>=0.9.0
   - pip:
     - bump2version
     - check-jsonschema
@@ -32,7 +35,7 @@ dependencies:
     - pylint==3.3.7
     - tomlkit
     - mypy==1.15.0
-    - pyright==1.1.400
+    - pyright==1.1.406
     - pandas-stubs
     - types-beautifulsoup4
     - types-colorama

conda-envs/mlos-windows.yml

@@ -33,7 +33,7 @@ dependencies:
     - pylint==3.3.7
     - tomlkit
     - mypy==1.15.0
-    - pyright==1.1.400
+    - pyright==1.1.406
     - pandas-stubs
     - types-beautifulsoup4
     - types-colorama

conda-envs/mlos.yml

@@ -18,6 +18,9 @@ dependencies:
   # FIXME: Temporarily avoid broken libpq that's missing client headers.
   - libpq<17.0
   - python
+  # Install an SMAC requirement pre-compiled from conda-forge.
+  # See Also: https://github.com/microsoft/MLOS/issues/1001
+  - pyrfr>=0.9.0
   - pip:
     - bump2version
     - check-jsonschema
@@ -28,7 +31,7 @@ dependencies:
     - pylint==3.3.7
     - tomlkit
     - mypy==1.15.0
-    - pyright==1.1.400
+    - pyright==1.1.406
     - pandas-stubs
     - types-beautifulsoup4
     - types-colorama

mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py

@@ -285,7 +285,7 @@ def bulk_register(
 
         if status is not None:
             # Select only the completed trials, set scores for failed trials to +inf.
-            df_status = pd.Series(status)
+            df_status = pd.Series(list(status), dtype=object)
             # TODO: Be more flexible with values used for failed trials (not just +inf).
             # Issue: https://github.com/microsoft/MLOS/issues/523
             df_scores[df_status != Status.SUCCEEDED] = float("inf")

mlos_bench/mlos_bench/storage/sql/alembic/versions/b61aa446e724_support_fractional_seconds_with_mysql.py

@@ -34,8 +34,8 @@ def _mysql_datetime(*, with_fsp: bool = False) -> mysql.DATETIME:
     See <https://github.com/sqlalchemy/sqlalchemy/pull/12164> for details.
     """
     if with_fsp:
-        return mysql.DATETIME(fsp=6)  # type: ignore[no-untyped-call]
-    return mysql.DATETIME()  # type: ignore[no-untyped-call]
+        return mysql.DATETIME(fsp=6)
+    return mysql.DATETIME()
 
 
 def upgrade() -> None:

mlos_bench/mlos_bench/storage/sql/schema.py

@@ -56,7 +56,7 @@ def _mysql_datetime_with_fsp() -> mysql.DATETIME:
     Split out to allow single mypy ignore.
     See <https://github.com/sqlalchemy/sqlalchemy/pull/12164> for details.
     """
-    return mysql.DATETIME(fsp=6)  # type: ignore[no-untyped-call]
+    return mysql.DATETIME(fsp=6)
 
 
 class _DDL:

mlos_bench/mlos_bench/tests/__init__.py

@@ -12,8 +12,9 @@
 import os
 import shutil
 import socket
+import stat
 from datetime import tzinfo
-from logging import debug, warning
+from logging import warning
 from subprocess import run
 
 import pytest
@@ -38,22 +39,56 @@
     "trial_runner_id": None,
 }
 
-# A decorator for tests that require docker.
-# Use with @requires_docker above a test_...() function.
+
 DOCKER = shutil.which("docker")
 if DOCKER:
+    # Gathering info about Github CI docker.sock permissions for debugging purposes.
+    DOCKER_SOCK_PATH = "/var/run/docker.sock"
+    mode: str | None = None
+    uid: int | None = None
+    gid: int | None = None
+    current_uid: int | None = None
+    current_gid: int | None = None
+    gids: list[int] | None = None
+    try:
+        st = os.stat(DOCKER_SOCK_PATH)
+        mode = stat.filemode(st.st_mode)
+        uid = st.st_uid
+        gid = st.st_gid
+    except Exception as e:  # pylint: disable=broad-except
+        warning(f"Could not stat {DOCKER_SOCK_PATH}: {e}")
+    try:
+        current_uid = os.getuid()
+        current_gid = os.getgid()
+        gids = os.getgroups()
+    except Exception as e:  # pylint: disable=broad-except
+        warning(f"Could not get current user info: {e}")
+
     cmd = run(
         "docker builder inspect default || docker buildx inspect default",
         shell=True,
         check=False,
         capture_output=True,
     )
     stdout = cmd.stdout.decode()
+    stderr = cmd.stderr.decode()
     if cmd.returncode != 0 or not any(
         line for line in stdout.splitlines() if "Platform" in line and "linux" in line
     ):
-        debug("Docker is available but missing support for targeting linux platform.")
         DOCKER = None
+        warning(
+            "Docker is available but missing buildx support for targeting linux platform:\n"
+            + f"stdout:\n{stdout}\n"
+            + f"stderr:\n{stderr}\n"
+            + f"sock_path: {DOCKER_SOCK_PATH} sock mode: {mode} sock uid: {uid} gid: {gid}\n"
+            + f"current_uid: {current_uid} groups: {gids}\n"
+        )
+
+if not DOCKER:
+    warning("Docker is not available on this system. Some tests will be skipped.")
+
+# A decorator for tests that require docker.
+# Use with @requires_docker above a test_...() function.
 requires_docker = pytest.mark.skipif(
     not DOCKER,
     reason="Docker with Linux support is not available on this system.",
@@ -62,6 +97,8 @@
 # A decorator for tests that require ssh.
 # Use with @requires_ssh above a test_...() function.
 SSH = shutil.which("ssh")
+if not SSH:
+    warning("ssh is not available on this system.  Some tests will be skipped.")
 requires_ssh = pytest.mark.skipif(not SSH, reason="ssh is not available on this system.")
 
 # A common seed to use to avoid tracking down race conditions and intermingling
@@ -112,7 +149,7 @@ def wait_docker_service_healthy(
     docker_services: DockerServices,
     project_name: str,
     service_name: str,
-    timeout: float = 30.0,
+    timeout: float = 60.0,
 ) -> None:
     """Wait until a docker service is healthy."""
     docker_services.wait_until_responsive(
@@ -126,7 +163,7 @@ def wait_docker_service_socket(docker_services: DockerServices, hostname: str, p
     """Wait until a docker service is ready."""
     docker_services.wait_until_responsive(
         check=lambda: check_socket(hostname, port),
-        timeout=30.0,
+        timeout=60.0,
         pause=0.5,
     )
 

mlos_bench/mlos_bench/tests/conftest.py

@@ -104,6 +104,8 @@ def docker_compose_file(pytestconfig: pytest.Config) -> list[str]:
         Path to the docker-compose file.
     """
     _ = pytestconfig  # unused
+    # TODO: move this closer to the necessary submodules so that different
+    # docker tests can run independently.
     return [
         os.path.join(os.path.dirname(__file__), "services", "remote", "ssh", "docker-compose.yml"),
         os.path.join(os.path.dirname(__file__), "storage", "sql", "docker-compose.yml"),