diff --git a/.github/workflows/diff-rendered-charts.yml b/.github/workflows/diff-rendered-charts.yml
index 26ab54e..9745460 100644
--- a/.github/workflows/diff-rendered-charts.yml
+++ b/.github/workflows/diff-rendered-charts.yml
@@ -240,12 +240,19 @@ jobs:
           shopt -s nullglob
 
           for chart in ${CHARTS}; do
+            # When adding or removing a values file the `render_charts` job will only render one side of the chart to be compared
+            # Create the directory to be compared if it doesn't exist so `find` succeeds
+            mkdir -p "shared/base-charts/${chart}" "shared/head-charts/${chart}"
+
             CONFIGS=`find "shared/base-charts/${chart}" "shared/head-charts/${chart}" -mindepth 1 -maxdepth 1 -type d -print0 | xargs --null basename -a | sort | uniq`
 
             mkdir -p diff/${chart}
 
             for config in ${CONFIGS}; do
               for dir in "shared/base-charts/${chart}/${config}" "shared/head-charts/${chart}/${config}"; do
+                # Create the directory to be compared if it doesn't exist so `find` succeeds
+                mkdir -p "${dir}"
+
                 OUTPUT_FILE="${dir}.yaml"
 
                 touch "$OUTPUT_FILE"
@@ -268,7 +275,7 @@ jobs:
         run: |
           set -uo pipefail
 
-          CONFTEST_OUTPUT=$(conftest test --no-color --no-fail --strict --update https://raw.githubusercontent.com/mozilla/helm-charts/main/policy/helm-automerge.rego diff)
+          CONFTEST_OUTPUT=$(conftest test --no-color --strict --update https://raw.githubusercontent.com/mozilla/helm-charts/main/policy/helm-automerge.rego diff)
           CONFTEST_EXIT_CODE=$?
           STATUS_DESCRIPTION=$(echo "$CONFTEST_OUTPUT" | tail -1)