diff --git a/.github/workflows/run-smoke-test.yml b/.github/workflows/run-smoke-test.yml
index 80b0aad3d..d30ff01a4 100644
--- a/.github/workflows/run-smoke-test.yml
+++ b/.github/workflows/run-smoke-test.yml
@@ -10,6 +10,7 @@ jobs:
     smoke-test:
         name: Build SDK and run smoke test
         runs-on: ubuntu-latest
+        if: github.ref == 'refs/heads/main'
         steps:
             - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
@@ -45,7 +46,7 @@ jobs:
                   If any step fails, exit with a non-zero exit code. Do not silently fix errors.
                   PROMPT_EOF
 
-                  copilot --prompt "$(cat /tmp/smoke-test-prompt.txt)"
+                  copilot --yolo --prompt "$(cat /tmp/smoke-test-prompt.txt)"
 
             - name: Run smoke test jar
               env: