From 9945a200e89d667a46752a0a80262d838f067bd0 Mon Sep 17 00:00:00 2001 From: brianlball Date: Thu, 16 Jul 2026 16:55:42 -0400 Subject: [PATCH 1/2] Pin R package installs to a dated CRAN snapshot (Posit Package Manager) install_and_verify pins the packages it names, but their dependencies resolved from live CRAN. A mid-day CRAN publish on 2026-07-16 (ggrepel requiring ggplot2 >= 3.5.2/gtable >= 0.3.6) broke rserve image rebuilds that had passed hours earlier, plus transient cloud.r-project.org download failures. Resolve everything from the 2026-07-15 snapshot: same inputs every build until the date is bumped deliberately. The __linux__/jammy path serves prebuilt binaries for the base image (Ubuntu 22.04 + R 4.4) - install_packages.R drops from ~480s to ~100s - with the plain source snapshot as fallback. Verified locally: docker build of docker/R completes, 13/13 packages install and load. Co-Authored-By: Claude Fable 5 --- docker/R/install_packages.R | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docker/R/install_packages.R b/docker/R/install_packages.R index 03e492fe1..bdba072c4 100644 --- a/docker/R/install_packages.R +++ b/docker/R/install_packages.R @@ -7,8 +7,24 @@ # OpenStudio-R image here: # https://raw.githubusercontent.com/NREL/docker-openstudio-r/master/base_packages.R +# Install from a date-frozen CRAN snapshot (Posit Package Manager) so the UNPINNED +# dependencies of the pinned packages below also resolve deterministically. On +# 2026-07-16 a mid-day CRAN publish (ggrepel requiring ggplot2 >= 3.5.2/gtable >= +# 0.3.6) broke image rebuilds that had passed hours earlier. The __linux__/jammy +# path serves prebuilt binaries for the base image's Ubuntu 22.04 + R 4.4 (much +# faster than source compiles); the plain snapshot path is the source fallback. +# To take newer packages, bump the snapshot date deliberately. +snapshot_repos = c( + 'https://packagemanager.posit.co/cran/__linux__/jammy/2026-07-15', + 'https://packagemanager.posit.co/cran/2026-07-15' +) +# Posit Package Manager only serves Linux binaries to clients whose User-Agent +# announces the R version; set it explicitly so install.packages gets binaries. +options(HTTPUserAgent = sprintf('R/%s R (%s)', getRversion(), + paste(getRversion(), R.version$platform, R.version$arch, R.version$os))) + # Function for installing and verifying that the package was installed correctly (i.e. can be loaded) -install_and_verify = function(package_name, version=NULL, configure.args=c(), repos=c('http://cloud.r-project.org', 'http://cran.r-project.org')){ +install_and_verify = function(package_name, version=NULL, configure.args=c(), repos=snapshot_repos){ if (!is.null(version)) { print(paste("Installing package", package_name, "version", version)) remotes::install_version(package_name, version=version, repos=repos) From c922034159dd4907aa97280cc606e209912a0cdd Mon Sep 17 00:00:00 2001 From: brianlball Date: Thu, 16 Jul 2026 16:55:42 -0400 Subject: [PATCH 2/2] CI: skip the rserve rebuild on PRs that do not touch docker/R Every docker job rebuilt the rserve image from scratch: BuildKit (Docker 28 default) does not use pulled images as layer cache, so the docker-compose pull that used to make the build a cache no-op under the classic builder stopped helping. That cost ~10 min per run and exposed every PR to CRAN availability (see 2026-07-16 failure on #845's run). - PR runs build only web unless the PR changes docker/R (two-commit tree diff against the fetched base tip - works on the shallow merge-ref clone; any fetch/diff failure falls back to building everything). The pulled nrel/openstudio-rserve:latest, kept fresh by docker-upload on develop/master pushes, backs the stack instead - push runs still build everything (docker-upload deploys those images) - compose build config gains cache_from + BUILDKIT_INLINE_CACHE so images pushed from develop carry layer-cache metadata and can seed the builds that do still happen Co-Authored-By: Claude Fable 5 --- .github/workflows/openstudio-server-tests.yml | 16 +++++++++++++- docker-compose.test.yml | 22 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.github/workflows/openstudio-server-tests.yml b/.github/workflows/openstudio-server-tests.yml index 50aee4909..4c61a5a26 100644 --- a/.github/workflows/openstudio-server-tests.yml +++ b/.github/workflows/openstudio-server-tests.yml @@ -120,7 +120,21 @@ jobs: docker --version docker-compose --version docker-compose -f docker-compose.test.yml pull - docker-compose -f docker-compose.test.yml build --build-arg OPENSTUDIO_VERSION=$OPENSTUDIO_TAG + # Rebuild the rserve image only when its inputs changed. PR runs otherwise reuse + # the pulled nrel/openstudio-rserve:latest (kept fresh by docker-upload on + # develop/master pushes): rebuilding R packages on every run cost ~10 min and + # failed whenever CRAN had a transient outage or mid-day version drift. + # Push runs always build everything - docker-upload deploys those images. + # Two-commit tree diff (not A...B) so it works on the shallow PR merge-ref clone; + # any fetch/diff failure falls back to building everything. + BUILD_SERVICES="web rserve" + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ] \ + && git fetch --no-tags --quiet --depth=1 origin "${GITHUB_BASE_REF}" \ + && git diff --quiet FETCH_HEAD HEAD -- docker/R; then + echo "docker/R is unchanged by this PR: reusing pulled nrel/openstudio-rserve:latest instead of rebuilding it" + BUILD_SERVICES="web" + fi + docker-compose -f docker-compose.test.yml build --build-arg OPENSTUDIO_VERSION=$OPENSTUDIO_TAG $BUILD_SERVICES docker-compose -f docker-compose.test.yml up -d #set SKIP_URBANOPT_ALGO=true to skip UrbanOpt algo tests docker-compose exec -e SKIP_URBANOPT_ALGO=true -T web /usr/local/bin/run-server-tests diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 40c04a464..d3e075bf3 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -22,6 +22,11 @@ services: args: rails_env: docker bundle_args: '' + # embed layer-cache metadata so pushed images can seed later builds (BuildKit + # no longer uses pulled images as cache without it, unlike the classic builder) + BUILDKIT_INLINE_CACHE: '1' + cache_from: + - nrel/openstudio-server:latest environment: - CI - COVERALLS_REPO_TOKEN @@ -53,6 +58,11 @@ services: args: rails_env: docker bundle_args: '' + # embed layer-cache metadata so pushed images can seed later builds (BuildKit + # no longer uses pulled images as cache without it, unlike the classic builder) + BUILDKIT_INLINE_CACHE: '1' + cache_from: + - nrel/openstudio-server:latest environment: - OS_SERVER_NUMBER_OF_WORKERS=1 - QUEUES=background,analyses @@ -75,6 +85,11 @@ services: args: rails_env: docker bundle_args: '' + # embed layer-cache metadata so pushed images can seed later builds (BuildKit + # no longer uses pulled images as cache without it, unlike the classic builder) + BUILDKIT_INLINE_CACHE: '1' + cache_from: + - nrel/openstudio-server:latest environment: - OS_SERVER_NUMBER_OF_WORKERS=1 - QUEUES=requeued,simulations @@ -92,7 +107,12 @@ services: rserve: image: nrel/openstudio-rserve:latest platform: ${DOCKER_PLATFORM:-linux/amd64} - build: ./docker/R + build: + context: ./docker/R + args: + BUILDKIT_INLINE_CACHE: '1' + cache_from: + - nrel/openstudio-rserve:latest environment: - OS_SERVER_NUMBER_OF_WORKERS=1 - REDIS_URL=${REDIS_URL}