From be8a030a977a154d3f74e91bcc0d6ee483c645d6 Mon Sep 17 00:00:00 2001 From: Rory Abraham Date: Sat, 9 Jul 2022 01:27:08 -0700 Subject: [PATCH 1/2] Make schema validations async --- .../scripts/validateActionsAndWorkflows.sh | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/.github/scripts/validateActionsAndWorkflows.sh b/.github/scripts/validateActionsAndWorkflows.sh index a91b76c9fdd3..258fa4ebc967 100755 --- a/.github/scripts/validateActionsAndWorkflows.sh +++ b/.github/scripts/validateActionsAndWorkflows.sh @@ -2,25 +2,48 @@ echo 'Validates the Github Actions and workflows using the json schemas provided by (https://www.schemastore.org/json/)' -# Track exit codes separately so we can run a full validation, report errors, and exit with the correct code -declare EXIT_CODE=0 - # Download the up-to-date json schemas for github actions and workflows cd ./.github && mkdir ./tempSchemas || exit 1 curl https://json.schemastore.org/github-action.json --output ./tempSchemas/github-action.json --silent || exit 1 curl https://json.schemastore.org/github-workflow.json --output ./tempSchemas/github-workflow.json --silent || exit 1 +# Track exit codes separately so we can run a full validation, report errors, and exit with the correct code +declare EXIT_CODE=0 + +# This stores all the process IDs of the ajv commands so they can run in parallel +declare ASYNC_PROCESSES + +# Arrays of actions and workflows +declare -r ACTIONS=(./actions/*/*/action.yml) +declare -r WORKFLOWS=(./workflows/*.yml) + # Validate the actions and workflows using the JSON schemas and ajv https://github.com/ajv-validator/ajv-cli -find ./actions -type f -name "*.yml" -print0 | xargs -0 -I file ajv -s ./tempSchemas/github-action.json -d file --strict=false || EXIT_CODE=1 -find ./workflows -type f -name "*.yml" -print0 | xargs -0 -I file ajv -s ./tempSchemas/github-workflow.json -d file --strict=false || EXIT_CODE=1 +for ((i=0; i < ${#ACTIONS[@]}; i++)); do + ACTION=${ACTIONS[$i]} + ajv -s ./tempSchemas/github-action.json -d "$ACTION" --strict=false & + ASYNC_PROCESSES[$i]=$! +done -if (( "$EXIT_CODE" != 0 )); then - exit $EXIT_CODE -fi +for ((i=0; i < ${#WORKFLOWS[@]}; i++)); do + WORKFLOW=${WORKFLOWS[$i]} + ajv -s ./tempSchemas/github-workflow.json -d "$WORKFLOW" --strict=false & + ASYNC_PROCESSES[${#ACTIONS[@]} + $i]=$! +done + +# Wait for the background builds to finish +for PID in ${ASYNC_PROCESSES[*]}; do + if ! wait $PID; then + EXIT_CODE=$? + fi +done # Cleanup after ourselves and delete the schemas rm -rf ./tempSchemas +if (( "$EXIT_CODE" != 0 )); then + exit $EXIT_CODE +fi + echo echo 'Lint Github Actions via actionlint (https://github.com/rhysd/actionlint)' From 598fb59ac89afcb87ddb650dfefde43b83ad824d Mon Sep 17 00:00:00 2001 From: Rory Abraham Date: Tue, 12 Jul 2022 10:08:43 -0700 Subject: [PATCH 2/2] Fix exit code and use shellUtils --- .../scripts/validateActionsAndWorkflows.sh | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/.github/scripts/validateActionsAndWorkflows.sh b/.github/scripts/validateActionsAndWorkflows.sh index 258fa4ebc967..2d93cc97695d 100755 --- a/.github/scripts/validateActionsAndWorkflows.sh +++ b/.github/scripts/validateActionsAndWorkflows.sh @@ -1,11 +1,24 @@ #!/bin/bash -echo 'Validates the Github Actions and workflows using the json schemas provided by (https://www.schemastore.org/json/)' +source ./scripts/shellUtils.sh + +title 'Validating the Github Actions and workflows using the json schemas provided by (https://www.schemastore.org/json/)' + +function downloadSchema { + [[ $1 = 'github-action.json' ]] && SCHEMA_NAME='GitHub Action' || SCHEMA_NAME='GitHub Workflow' + info "Downloading $SCHEMA_NAME schema..." + if curl "https://json.schemastore.org/$1" --output "./tempSchemas/$1" --silent; then + success "Successfully downloaded $SCHEMA_NAME schema!" + else + error "Failed downloading $SCHEMA_NAME schema" + exit 1 + fi +} # Download the up-to-date json schemas for github actions and workflows cd ./.github && mkdir ./tempSchemas || exit 1 -curl https://json.schemastore.org/github-action.json --output ./tempSchemas/github-action.json --silent || exit 1 -curl https://json.schemastore.org/github-workflow.json --output ./tempSchemas/github-workflow.json --silent || exit 1 +downloadSchema 'github-action.json' || exit 1 +downloadSchema 'github-workflow.json' || exit 1 # Track exit codes separately so we can run a full validation, report errors, and exit with the correct code declare EXIT_CODE=0 @@ -17,6 +30,8 @@ declare ASYNC_PROCESSES declare -r ACTIONS=(./actions/*/*/action.yml) declare -r WORKFLOWS=(./workflows/*.yml) +info 'Validating actions and workflows against their JSON schemas...' + # Validate the actions and workflows using the JSON schemas and ajv https://github.com/ajv-validator/ajv-cli for ((i=0; i < ${#ACTIONS[@]}; i++)); do ACTION=${ACTIONS[$i]} @@ -32,20 +47,17 @@ done # Wait for the background builds to finish for PID in ${ASYNC_PROCESSES[*]}; do - if ! wait $PID; then - EXIT_CODE=$? + wait $PID + RESULT=$? + if [[ $RESULT != 0 ]]; then + EXIT_CODE=$RESULT fi done # Cleanup after ourselves and delete the schemas rm -rf ./tempSchemas -if (( "$EXIT_CODE" != 0 )); then - exit $EXIT_CODE -fi - -echo -echo 'Lint Github Actions via actionlint (https://github.com/rhysd/actionlint)' +title 'Lint Github Actions via actionlint (https://github.com/rhysd/actionlint)' # If we are running this on a non-CI machine (e.g. locally), install shellcheck if [[ -z "${CI}" && -z "$(command -v shellcheck)" ]]; then @@ -57,7 +69,15 @@ if [[ -z "${CI}" && -z "$(command -v shellcheck)" ]]; then brew install shellcheck fi -curl -s curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash | bash -s -- 1.6.13 +info 'Downloading actionlint...' +if bash <(curl --silent https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash); then + success 'Successfully downloaded actionlint!' +else + error 'Error downloading actionlint' + exit 1 +fi + +info 'Linting workflows...' ./actionlint -color || EXIT_CODE=1 # Cleanup after ourselves and delete actionlint