From 33cc658456d4eaa9caf88482e137f97d769fe766 Mon Sep 17 00:00:00 2001
From: Nathan Chancellor <nathan@kernel.org>
Date: Wed, 25 May 2022 15:42:37 -0700
Subject: [PATCH 01/10] Test statically linked clang outside of Alpine Linux

We should verify that we can use the toolchain produced in Alpine Linux
on other distributions as part of the continuous integration pipeline.

We download the toolchain out of the container, mount it into the
containers that we want to test it in, and test dynamically and
statically linking hello.{c,cpp} with musl. Eventually, glibc should be
tested but there are issues during the link phase.

Initially, the only container that is being tested is Fedora. However, I
intentionally wrote this in such a way that we should be able to use
this across multiple distributions and test more than just
hello.{c,cpp}.

This was tested on a self-hosted GitHub Actions runner:

https://github.com/nathanchance/containers/actions/runs/2387476728

Signed-off-by: Nathan Chancellor <nathan@kernel.org>
---
 .github/workflows/llvm-project-epoch-one.yml |  2 +
 .github/workflows/llvm-project-epoch-two.yml |  2 +
 ci/test-clang-docker.sh                      | 58 ++++++++++++++++++++
 ci/test-clang.sh                             | 31 +++++++++++
 4 files changed, 93 insertions(+)
 create mode 100755 ci/test-clang-docker.sh
 create mode 100755 ci/test-clang.sh

diff --git a/.github/workflows/llvm-project-epoch-one.yml b/.github/workflows/llvm-project-epoch-one.yml
index 5de49d3..52e5c05 100644
--- a/.github/workflows/llvm-project-epoch-one.yml
+++ b/.github/workflows/llvm-project-epoch-one.yml
@@ -28,3 +28,5 @@ jobs:
           push: ${{ github.repository_owner == 'ClangBuiltLinux' && github.event_name != 'pull_request' && github.ref_name == 'main' }}
           tags: |
             ghcr.io/clangbuiltlinux/llvm-project:stage2
+      - name: Test statically linked clang
+        run: bash ci/test-clang.sh
diff --git a/.github/workflows/llvm-project-epoch-two.yml b/.github/workflows/llvm-project-epoch-two.yml
index b2dd52b..05920f0 100644
--- a/.github/workflows/llvm-project-epoch-two.yml
+++ b/.github/workflows/llvm-project-epoch-two.yml
@@ -28,3 +28,5 @@ jobs:
           push: ${{ github.repository_owner == 'ClangBuiltLinux' && github.event_name != 'pull_request' && github.ref_name == 'main' }}
           tags: |
             ghcr.io/clangbuiltlinux/llvm-project:stage2
+      - name: Test statically linked clang
+        run: bash ci/test-clang.sh
diff --git a/ci/test-clang-docker.sh b/ci/test-clang-docker.sh
new file mode 100755
index 0000000..45d9c37
--- /dev/null
+++ b/ci/test-clang-docker.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+set -eu
+
+if [[ ! -d /repo ]]; then
+    echo "[-] This script needs to be run within Docker! Use test-clang.sh instead."
+    exit 1
+fi
+
+clang=/repo/toolchain/bin/clang
+hello_c=/repo/llvm-project/hello.c
+hello_cpp=/repo/llvm-project/hello.cpp
+hello_exe=/tmp/hello
+host_arch=$(uname -m)
+
+# Print clang information, which makes sure it can run
+echo "[+] Testing 'clang --version'"
+"$clang" --version
+
+case "$(source /usr/lib/os-release; echo "$ID")" in
+    fedora)
+        musl_cc_flags=(
+            -B /usr/"$host_arch"-linux-musl/lib64   # Scrt1.o, crti.o, crtn.o
+            -I /usr/"$host_arch"-linux-musl/include # stdio.h
+            -L /usr/"$host_arch"-linux-musl/lib64   # -lc
+        )
+
+        echo "[+] Updating OS"
+        dnf update -qy
+
+        echo "[+] Installing musl"
+        dnf install -qy \
+            musl-devel \
+            musl-libc \
+            musl-libc-static
+        ;;
+esac
+
+########
+# MUSL #
+########
+
+echo "[+] Testing dynamically linking hello.c against musl"
+"$clang" "${musl_cc_flags[@]}" -o "$hello_exe" "$hello_c"
+"$hello_exe"
+
+echo "[+] Testing statically linking hello.c against musl"
+"$clang" "${musl_cc_flags[@]}" -static -o "$hello_exe" "$hello_c"
+"$hello_exe"
+
+echo "[+] Testing dynamically linking hello.cpp against musl"
+# --rpath to avoid having to pull in libc++ and libunwind from distribution
+"$clang"++ "${musl_cc_flags[@]}" -Wl,--rpath=/repo/toolchain/lib/"$host_arch"-alpine-linux-musl -o "$hello_exe" "$hello_cpp"
+"$hello_exe"
+
+echo "[+] Testing statically linking hello.cpp against musl"
+"$clang"++ "${musl_cc_flags[@]}" -static -lc++abi -o "$hello_exe" "$hello_cpp"
+"$hello_exe"
diff --git a/ci/test-clang.sh b/ci/test-clang.sh
new file mode 100755
index 0000000..f73fbcf
--- /dev/null
+++ b/ci/test-clang.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -eu
+
+trap 'rm -fr "$toolchain"' EXIT
+
+# Set up workspace
+rootdir=${GITHUB_WORKSPACE:-$(dirname "$(dirname "$(realpath "$0")")")}
+toolchain=$rootdir/toolchain
+
+# Pull toolchain out of container
+echo "[+] Downloading toolchain from container"
+docker create --name llvm-project ghcr.io/clangbuiltlinux/llvm-project:stage2
+mkdir -p "$toolchain"/{bin,include,lib}
+docker cp llvm-project:/usr/local/bin/. "$toolchain"/bin
+docker cp llvm-project:/usr/local/include/. "$toolchain"/include
+docker cp llvm-project:/usr/local/lib/. "$toolchain"/lib
+echo "[+] Cleaning up container"
+docker rm llvm-project
+
+# Test toolchain in Docker images so that the environment is consistent,
+# regardless of runners.
+docker_images=(
+    docker.io/fedora:latest
+)
+for docker_image in "${docker_images[@]}"; do
+    echo "[+] Updating '$docker_image'"
+    docker pull "$docker_image"
+    echo "[+] Testing clang in '$docker_image' container"
+    docker run --rm -v "$rootdir":/repo:ro "$docker_image" bash /repo/ci/test-clang-docker.sh
+done

From 22c734e153fa0f87dba1ff5a6de2725cb21fad56 Mon Sep 17 00:00:00 2001
From: Nathan Chancellor <nathan@kernel.org>
Date: Thu, 26 May 2022 09:46:42 -0700
Subject: [PATCH 02/10] ci/test-clang.sh: Use long flag for volume

This makes it clearer what '"$rootdir":/repo:ro' is doing.

Signed-off-by: Nathan Chancellor <nathan@kernel.org>
---
 ci/test-clang.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/test-clang.sh b/ci/test-clang.sh
index f73fbcf..44d8d14 100755
--- a/ci/test-clang.sh
+++ b/ci/test-clang.sh
@@ -27,5 +27,5 @@ for docker_image in "${docker_images[@]}"; do
     echo "[+] Updating '$docker_image'"
     docker pull "$docker_image"
     echo "[+] Testing clang in '$docker_image' container"
-    docker run --rm -v "$rootdir":/repo:ro "$docker_image" bash /repo/ci/test-clang-docker.sh
+    docker run --rm --volume "$rootdir":/repo:ro "$docker_image" bash /repo/ci/test-clang-docker.sh
 done

From 5cc84afddae54ec7c09798603a53528ad89dc4fc Mon Sep 17 00:00:00 2001
From: Nathan Chancellor <nathan@kernel.org>
Date: Thu, 26 May 2022 09:47:39 -0700
Subject: [PATCH 03/10] ci/test-clang.sh: Split docker run command over
 multiple lines

I find this a little easier to parse when multiple arguments are
involved.

Signed-off-by: Nathan Chancellor <nathan@kernel.org>
---
 ci/test-clang.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ci/test-clang.sh b/ci/test-clang.sh
index 44d8d14..d42bb01 100755
--- a/ci/test-clang.sh
+++ b/ci/test-clang.sh
@@ -27,5 +27,9 @@ for docker_image in "${docker_images[@]}"; do
     echo "[+] Updating '$docker_image'"
     docker pull "$docker_image"
     echo "[+] Testing clang in '$docker_image' container"
-    docker run --rm --volume "$rootdir":/repo:ro "$docker_image" bash /repo/ci/test-clang-docker.sh
+    docker run \
+        --rm \
+        --volume "$rootdir":/repo:ro \
+        "$docker_image" \
+        bash /repo/ci/test-clang-docker.sh
 done

From 55125a5086cd353263a06b1917736fe2ead0d194 Mon Sep 17 00:00:00 2001
From: Nathan Chancellor <nathan@kernel.org>
Date: Thu, 26 May 2022 10:30:28 -0700
Subject: [PATCH 04/10] ci/test-clang-docker.sh: Define CC and CXX explicitly

Signed-off-by: Nathan Chancellor <nathan@kernel.org>
---
 ci/test-clang-docker.sh | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/ci/test-clang-docker.sh b/ci/test-clang-docker.sh
index 45d9c37..e7ebd56 100755
--- a/ci/test-clang-docker.sh
+++ b/ci/test-clang-docker.sh
@@ -7,7 +7,8 @@ if [[ ! -d /repo ]]; then
     exit 1
 fi
 
-clang=/repo/toolchain/bin/clang
+CC=/repo/toolchain/bin/clang
+CXX=/repo/toolchain/bin/clang++
 hello_c=/repo/llvm-project/hello.c
 hello_cpp=/repo/llvm-project/hello.cpp
 hello_exe=/tmp/hello
@@ -15,7 +16,7 @@ host_arch=$(uname -m)
 
 # Print clang information, which makes sure it can run
 echo "[+] Testing 'clang --version'"
-"$clang" --version
+"$CC" --version
 
 case "$(source /usr/lib/os-release; echo "$ID")" in
     fedora)
@@ -41,18 +42,18 @@ esac
 ########
 
 echo "[+] Testing dynamically linking hello.c against musl"
-"$clang" "${musl_cc_flags[@]}" -o "$hello_exe" "$hello_c"
+"$CC" "${musl_cc_flags[@]}" -o "$hello_exe" "$hello_c"
 "$hello_exe"
 
 echo "[+] Testing statically linking hello.c against musl"
-"$clang" "${musl_cc_flags[@]}" -static -o "$hello_exe" "$hello_c"
+"$CC" "${musl_cc_flags[@]}" -static -o "$hello_exe" "$hello_c"
 "$hello_exe"
 
 echo "[+] Testing dynamically linking hello.cpp against musl"
 # --rpath to avoid having to pull in libc++ and libunwind from distribution
-"$clang"++ "${musl_cc_flags[@]}" -Wl,--rpath=/repo/toolchain/lib/"$host_arch"-alpine-linux-musl -o "$hello_exe" "$hello_cpp"
+"$CXX" "${musl_cc_flags[@]}" -Wl,--rpath=/repo/toolchain/lib/"$host_arch"-alpine-linux-musl -o "$hello_exe" "$hello_cpp"
 "$hello_exe"
 
 echo "[+] Testing statically linking hello.cpp against musl"
-"$clang"++ "${musl_cc_flags[@]}" -static -lc++abi -o "$hello_exe" "$hello_cpp"
+"$CXX" "${musl_cc_flags[@]}" -static -lc++abi -o "$hello_exe" "$hello_cpp"
 "$hello_exe"

From b8a02d204216d65c3c86d8e0774828ac2e2e2752 Mon Sep 17 00:00:00 2001
From: Nathan Chancellor <nathan@kernel.org>
Date: Thu, 26 May 2022 10:41:11 -0700
Subject: [PATCH 05/10] ci/test-clang.sh: Let docker cp create bin, include,
 and lib directories

Per the 'docker cp' documentation, if SRC_PATH specifies a directory
that does not end with '/.' and DEST_PATH is a directory that exists,
SRC_PATH is copied into DEST_PATH.

'docker cp' does NOT create parent directories so we still need the
intial 'mkdir' call but we do not need the '-p' flag as '$toolchain' is
a subfolder within the repo, which has to exist if we are running this
command.

Link: https://docs.docker.com/engine/reference/commandline/cp/
Signed-off-by: Nathan Chancellor <nathan@kernel.org>
---
 ci/test-clang.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/test-clang.sh b/ci/test-clang.sh
index d42bb01..0a3c030 100755
--- a/ci/test-clang.sh
+++ b/ci/test-clang.sh
@@ -11,10 +11,10 @@ toolchain=$rootdir/toolchain
 # Pull toolchain out of container
 echo "[+] Downloading toolchain from container"
 docker create --name llvm-project ghcr.io/clangbuiltlinux/llvm-project:stage2
-mkdir -p "$toolchain"/{bin,include,lib}
-docker cp llvm-project:/usr/local/bin/. "$toolchain"/bin
-docker cp llvm-project:/usr/local/include/. "$toolchain"/include
-docker cp llvm-project:/usr/local/lib/. "$toolchain"/lib
+mkdir "$toolchain"
+docker cp llvm-project:/usr/local/bin "$toolchain"
+docker cp llvm-project:/usr/local/include "$toolchain"
+docker cp llvm-project:/usr/local/lib "$toolchain"
 echo "[+] Cleaning up container"
 docker rm llvm-project
 

From 179aaa1399e18c519f66fb0eca20ee849a3e97b0 Mon Sep 17 00:00:00 2001
From: Nathan Chancellor <nathan@kernel.org>
Date: Thu, 26 May 2022 10:50:12 -0700
Subject: [PATCH 06/10] ci/test-clang-docker.sh: Add LD_LIBRARY_PATH test

Signed-off-by: Nathan Chancellor <nathan@kernel.org>
---
 ci/test-clang-docker.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ci/test-clang-docker.sh b/ci/test-clang-docker.sh
index e7ebd56..4f9de16 100755
--- a/ci/test-clang-docker.sh
+++ b/ci/test-clang-docker.sh
@@ -49,11 +49,15 @@ echo "[+] Testing statically linking hello.c against musl"
 "$CC" "${musl_cc_flags[@]}" -static -o "$hello_exe" "$hello_c"
 "$hello_exe"
 
-echo "[+] Testing dynamically linking hello.cpp against musl"
+echo "[+] Testing dynamically linking hello.cpp against musl ('--rpath')"
 # --rpath to avoid having to pull in libc++ and libunwind from distribution
 "$CXX" "${musl_cc_flags[@]}" -Wl,--rpath=/repo/toolchain/lib/"$host_arch"-alpine-linux-musl -o "$hello_exe" "$hello_cpp"
 "$hello_exe"
 
+echo "[+] Testing dynamically linking hello.cpp against musl ('LD_LIBRARY_PATH')"
+"$CXX" "${musl_cc_flags[@]}" -o "$hello_exe" "$hello_cpp"
+LD_LIBRARY_PATH=/lib:/repo/toolchain/lib/"$host_arch"-alpine-linux-musl "$hello_exe"
+
 echo "[+] Testing statically linking hello.cpp against musl"
 "$CXX" "${musl_cc_flags[@]}" -static -lc++abi -o "$hello_exe" "$hello_cpp"
 "$hello_exe"

From 5f2bbadc55a717f16a372efb98b6f5b772fd7553 Mon Sep 17 00:00:00 2001
From: Nathan Chancellor <nathan@kernel.org>
Date: Thu, 26 May 2022 11:28:25 -0700
Subject: [PATCH 07/10] ci: Test statically linked clang in Arch Linux
 containers

Signed-off-by: Nathan Chancellor <nathan@kernel.org>
---
 ci/test-clang-docker.sh | 11 +++++++++++
 ci/test-clang.sh        |  2 ++
 2 files changed, 13 insertions(+)

diff --git a/ci/test-clang-docker.sh b/ci/test-clang-docker.sh
index 4f9de16..3a492ed 100755
--- a/ci/test-clang-docker.sh
+++ b/ci/test-clang-docker.sh
@@ -19,6 +19,17 @@ echo "[+] Testing 'clang --version'"
 "$CC" --version
 
 case "$(source /usr/lib/os-release; echo "$ID")" in
+    arch)
+        musl_cc_flags=(
+            -B /usr/lib/musl/lib     # Scrt1.o, crti.o, crtn.o
+            -I /usr/lib/musl/include # stdio.h
+            -L /usr/lib/musl/lib     # -lc
+        )
+
+        echo "[+] Updating OS and installing musl"
+        pacman -Syyu --noconfirm musl
+        ;;
+
     fedora)
         musl_cc_flags=(
             -B /usr/"$host_arch"-linux-musl/lib64   # Scrt1.o, crti.o, crtn.o
diff --git a/ci/test-clang.sh b/ci/test-clang.sh
index 0a3c030..0c6ff90 100755
--- a/ci/test-clang.sh
+++ b/ci/test-clang.sh
@@ -23,6 +23,8 @@ docker rm llvm-project
 docker_images=(
     docker.io/fedora:latest
 )
+# Arch Linux is x86_64-only
+[[ $(uname -m) = "x86_64" ]] && docker_images+=(docker.io/archlinux:latest)
 for docker_image in "${docker_images[@]}"; do
     echo "[+] Updating '$docker_image'"
     docker pull "$docker_image"

From a6a0b028bc0c6d13b290ea19f7142c6406bd35af Mon Sep 17 00:00:00 2001
From: Nathan Chancellor <nathan@kernel.org>
Date: Thu, 26 May 2022 16:51:31 -0700
Subject: [PATCH 08/10] github: workflows: Run 'docker push' in a separate step

Currently, we are not actually testing the image that is being built
during CI because there is no '--output' flag to 'docker buildx':

  WARNING: No output specified for docker-container driver. Build result
  will only remain in the build cache. To push result image into registry
  use --push or to load image into docker use --load

The docker/build-push-action supports "load" and "push" as shorthands
for different '--output' values but they are mutually exclusive, meaning
that if we want to test the image before we push it, we need to use
"load: true" to make the image available locally, test it, then run a
simple 'docker push' once it passes.

Signed-off-by: Nathan Chancellor <nathan@kernel.org>
---
 .github/workflows/llvm-project-epoch-one.yml | 6 +++++-
 .github/workflows/llvm-project-epoch-two.yml | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/llvm-project-epoch-one.yml b/.github/workflows/llvm-project-epoch-one.yml
index 52e5c05..2917a59 100644
--- a/.github/workflows/llvm-project-epoch-one.yml
+++ b/.github/workflows/llvm-project-epoch-one.yml
@@ -24,9 +24,13 @@ jobs:
         with:
           context: ./llvm-project
           file: ./llvm-project/Dockerfile.epoch1
+          load: true
           platforms: linux/amd64
-          push: ${{ github.repository_owner == 'ClangBuiltLinux' && github.event_name != 'pull_request' && github.ref_name == 'main' }}
           tags: |
             ghcr.io/clangbuiltlinux/llvm-project:stage2
       - name: Test statically linked clang
         run: bash ci/test-clang.sh
+      - name: Push image to ghcr.io
+        if: ${{ github.repository_owner == 'ClangBuiltLinux' && github.event_name != 'pull_request' && github.ref_name == 'main' }}
+        run: |
+          docker push ghcr.io/clangbuiltlinux/llvm-project:stage2
diff --git a/.github/workflows/llvm-project-epoch-two.yml b/.github/workflows/llvm-project-epoch-two.yml
index 05920f0..20163dc 100644
--- a/.github/workflows/llvm-project-epoch-two.yml
+++ b/.github/workflows/llvm-project-epoch-two.yml
@@ -24,9 +24,13 @@ jobs:
         with:
           context: ./llvm-project
           file: ./llvm-project/Dockerfile.epoch2
+          load: true
           platforms: linux/amd64
-          push: ${{ github.repository_owner == 'ClangBuiltLinux' && github.event_name != 'pull_request' && github.ref_name == 'main' }}
           tags: |
             ghcr.io/clangbuiltlinux/llvm-project:stage2
       - name: Test statically linked clang
         run: bash ci/test-clang.sh
+      - name: Push image to ghcr.io
+        if: ${{ github.repository_owner == 'ClangBuiltLinux' && github.event_name != 'pull_request' && github.ref_name == 'main' }}
+        run: |
+          docker push ghcr.io/clangbuiltlinux/llvm-project:stage2

From 162cc51a36827e94e2d538a35916171a156c871f Mon Sep 17 00:00:00 2001
From: Nathan Chancellor <nathan@kernel.org>
Date: Fri, 27 May 2022 11:04:03 -0700
Subject: [PATCH 09/10] ci/test-clang.sh: Clarify Arch Linux comment

Signed-off-by: Nathan Chancellor <nathan@kernel.org>
---
 ci/test-clang.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/test-clang.sh b/ci/test-clang.sh
index 0c6ff90..354717c 100755
--- a/ci/test-clang.sh
+++ b/ci/test-clang.sh
@@ -23,7 +23,7 @@ docker rm llvm-project
 docker_images=(
     docker.io/fedora:latest
 )
-# Arch Linux is x86_64-only
+# Arch Linux is an x86_64-only distribution
 [[ $(uname -m) = "x86_64" ]] && docker_images+=(docker.io/archlinux:latest)
 for docker_image in "${docker_images[@]}"; do
     echo "[+] Updating '$docker_image'"

From 50fab280c153fab864a8455f2e4e7cba93d1dc62 Mon Sep 17 00:00:00 2001
From: Nathan Chancellor <nathan@kernel.org>
Date: Fri, 27 May 2022 11:39:29 -0700
Subject: [PATCH 10/10] ci/test-clang-docker.sh: Use '--sysroot' for Arch Linux

Signed-off-by: Nathan Chancellor <nathan@kernel.org>
---
 ci/test-clang-docker.sh | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/ci/test-clang-docker.sh b/ci/test-clang-docker.sh
index 3a492ed..d975ae1 100755
--- a/ci/test-clang-docker.sh
+++ b/ci/test-clang-docker.sh
@@ -20,11 +20,7 @@ echo "[+] Testing 'clang --version'"
 
 case "$(source /usr/lib/os-release; echo "$ID")" in
     arch)
-        musl_cc_flags=(
-            -B /usr/lib/musl/lib     # Scrt1.o, crti.o, crtn.o
-            -I /usr/lib/musl/include # stdio.h
-            -L /usr/lib/musl/lib     # -lc
-        )
+        musl_cc_flags=(--sysroot=/usr/lib/musl)
 
         echo "[+] Updating OS and installing musl"
         pacman -Syyu --noconfirm musl