From 9bad41b3f6f9f157c52ed6a3a34f87b8cad8447c Mon Sep 17 00:00:00 2001 From: Samiya Akhtar Date: Tue, 17 Mar 2020 14:20:53 -0700 Subject: [PATCH 1/2] Improve auto generated introspection scripts before 1165 --- src/lib/fileutils.ts | 56 +++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/lib/fileutils.ts b/src/lib/fileutils.ts index 70d7b9bd5..9b40ecc5d 100644 --- a/src/lib/fileutils.ts +++ b/src/lib/fileutils.ts @@ -222,6 +222,17 @@ export const serviceBuildAndUpdatePipeline = ( ]), displayName: "Azure Login" }, + { + script: generateYamlScript([ + `# Download build.sh`, + `curl $BEDROCK_BUILD_SCRIPT > build.sh`, + `chmod +x ./build.sh` + ]), + displayName: "Download bedrock bash scripts", + env: { + BEDROCK_BUILD_SCRIPT: "$(BUILD_SCRIPT_URL)" + } + }, { script: generateYamlScript([ `export BUILD_REPO_NAME=${BUILD_REPO_NAME(serviceName)}`, @@ -232,9 +243,6 @@ export const serviceBuildAndUpdatePipeline = ( `service=\${service##*/}`, `url=$(git remote --verbose | grep origin | grep fetch | cut -f2 | cut -d' ' -f1)`, `repourl=\${url##*@}`, - `echo "Downloading SPK"`, - `curl https://raw.githubusercontent.com/Microsoft/bedrock/master/gitops/azure-devops/build.sh > build.sh`, - `chmod +x build.sh`, `. ./build.sh --source-only`, `get_spk_version`, `download_spk`, @@ -347,7 +355,6 @@ export const serviceBuildAndUpdatePipeline = ( `response=$(az repos pr create --description "Updating $SERVICE_NAME_LOWER to ${IMAGE_TAG}." "PR created by: $(Build.DefinitionName) with buildId: $(Build.BuildId) and buildNumber: $(Build.BuildNumber)")`, `pr_id=$(echo $response | jq -r '.pullRequestId')`, ``, - ``, `# Update introspection storage with this information, if applicable`, `if [ -z "$(INTROSPECTION_ACCOUNT_NAME)" -o -z "$(INTROSPECTION_ACCOUNT_KEY)" -o -z "$(INTROSPECTION_TABLE_NAME)" -o -z "$(INTROSPECTION_PARTITION_KEY)" ]; then`, `echo "Introspection variables are not defined. Skipping..."`, @@ -356,10 +363,6 @@ export const serviceBuildAndUpdatePipeline = ( `tag_name="$BUILD_REPO_NAME:$(Build.SourceBranchName)-$(Build.BuildNumber)"`, `url=$(git remote --verbose | grep origin | grep fetch | cut -f2 | cut -d' ' -f1)`, `repourl=\${url##*@}`, - `echo "Downloading SPK"`, - `curl https://raw.githubusercontent.com/Microsoft/bedrock/master/gitops/azure-devops/build.sh > build.sh`, - `chmod +x build.sh`, - `. ./build.sh --source-only`, `get_spk_version`, `download_spk`, `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p2 $(Build.BuildId) --hld-commit-id $latest_commit --env $BRANCH_NAME --image-tag $tag_name --pr $pr_id --repository $repourl`, @@ -580,6 +583,26 @@ const manifestGenerationPipelineYaml = (): string => { BEDROCK_BUILD_SCRIPT: "$(BUILD_SCRIPT_URL)" } }, + { + script: generateYamlScript([ + `commitId=$(Build.SourceVersion)`, + `commitId=$(echo "\${commitId:0:7}")`, + `. ./build.sh --source-only`, + `get_spk_version`, + `download_spk`, + `message="$(Build.SourceVersionMessage)"`, + `if [[ $message == *"Merged PR"* ]]; then`, + `pr_id=$(echo $message | grep -oE '[0-9]+' | head -1 | sed -e 's/^0\\+//')`, + `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId --pr $pr_id`, + `else`, + `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId`, + `fi` + ]), + displayName: + "If configured, update manifest pipeline details in Spektate db before manifest generation", + condition: + "and(ne(variables['INTROSPECTION_ACCOUNT_NAME'], ''), ne(variables['INTROSPECTION_ACCOUNT_KEY'], ''),ne(variables['INTROSPECTION_TABLE_NAME'], ''),ne(variables['INTROSPECTION_PARTITION_KEY'], ''))" + }, { task: "ShellScript@2", displayName: "Validate fabrikate definitions", @@ -608,28 +631,17 @@ const manifestGenerationPipelineYaml = (): string => { }, { script: generateYamlScript([ + `. ./build.sh --source-only`, `cd "$HOME"/\${MANIFEST_REPO##*/}`, - `commitId=$(Build.SourceVersion)`, - `commitId=$(echo "\${commitId:0:7}")`, `latest_commit=$(git rev-parse --short HEAD)`, `url=$(git remote --verbose | grep origin | grep fetch | cut -f2 | cut -d' ' -f1)`, `repourl=\${url##*@}`, - `echo "Downloading SPK"`, - `curl https://raw.githubusercontent.com/Microsoft/bedrock/master/gitops/azure-devops/build.sh > build.sh`, - `chmod +x build.sh`, - `. ./build.sh --source-only`, `get_spk_version`, `download_spk`, - `message="$(Build.SourceVersionMessage)"`, - `if [[ $message == *"Merged PR"* ]]; then`, - `pr_id=$(echo $message | grep -oE '[0-9]+' | head -1 | sed -e 's/^0\\+//')`, - `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId --manifest-commit-id $latest_commit --pr $pr_id --repository $repourl`, - `else`, - `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId --manifest-commit-id $latest_commit --repository $repourl`, - `fi` + `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --manifest-commit-id $latest_commit --repository $repourl` ]), displayName: - "If configured, update manifest pipeline details in Spektate db", + "If configured, update manifest pipeline details in Spektate db after manifest generation", condition: "and(ne(variables['INTROSPECTION_ACCOUNT_NAME'], ''), ne(variables['INTROSPECTION_ACCOUNT_KEY'], ''),ne(variables['INTROSPECTION_TABLE_NAME'], ''),ne(variables['INTROSPECTION_PARTITION_KEY'], ''))" } From 96417f07abbcfb9f2ccc7bfad2af1904467c2e83 Mon Sep 17 00:00:00 2001 From: Samiya Akhtar Date: Tue, 17 Mar 2020 14:23:52 -0700 Subject: [PATCH 2/2] fix unit test --- src/test/mockFactory.ts | 56 +++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/test/mockFactory.ts b/src/test/mockFactory.ts index c612afcf6..65894c928 100644 --- a/src/test/mockFactory.ts +++ b/src/test/mockFactory.ts @@ -53,6 +53,17 @@ export const createTestServiceBuildAndUpdatePipelineYaml = ( ]), displayName: "Azure Login" }, + { + script: generateYamlScript([ + `# Download build.sh`, + `curl $BEDROCK_BUILD_SCRIPT > build.sh`, + `chmod +x ./build.sh` + ]), + displayName: "Download bedrock bash scripts", + env: { + BEDROCK_BUILD_SCRIPT: "$(BUILD_SCRIPT_URL)" + } + }, { script: generateYamlScript([ `export BUILD_REPO_NAME=${BUILD_REPO_NAME(serviceName)}`, @@ -63,9 +74,6 @@ export const createTestServiceBuildAndUpdatePipelineYaml = ( `service=\${service##*/}`, `url=$(git remote --verbose | grep origin | grep fetch | cut -f2 | cut -d' ' -f1)`, `repourl=\${url##*@}`, - `echo "Downloading SPK"`, - `curl https://raw.githubusercontent.com/Microsoft/bedrock/master/gitops/azure-devops/build.sh > build.sh`, - `chmod +x build.sh`, `. ./build.sh --source-only`, `get_spk_version`, `download_spk`, @@ -178,7 +186,6 @@ export const createTestServiceBuildAndUpdatePipelineYaml = ( `response=$(az repos pr create --description "Updating $SERVICE_NAME_LOWER to ${IMAGE_TAG}." "PR created by: $(Build.DefinitionName) with buildId: $(Build.BuildId) and buildNumber: $(Build.BuildNumber)")`, `pr_id=$(echo $response | jq -r '.pullRequestId')`, ``, - ``, `# Update introspection storage with this information, if applicable`, `if [ -z "$(INTROSPECTION_ACCOUNT_NAME)" -o -z "$(INTROSPECTION_ACCOUNT_KEY)" -o -z "$(INTROSPECTION_TABLE_NAME)" -o -z "$(INTROSPECTION_PARTITION_KEY)" ]; then`, `echo "Introspection variables are not defined. Skipping..."`, @@ -187,10 +194,6 @@ export const createTestServiceBuildAndUpdatePipelineYaml = ( `tag_name="$BUILD_REPO_NAME:$(Build.SourceBranchName)-$(Build.BuildNumber)"`, `url=$(git remote --verbose | grep origin | grep fetch | cut -f2 | cut -d' ' -f1)`, `repourl=\${url##*@}`, - `echo "Downloading SPK"`, - `curl https://raw.githubusercontent.com/Microsoft/bedrock/master/gitops/azure-devops/build.sh > build.sh`, - `chmod +x build.sh`, - `. ./build.sh --source-only`, `get_spk_version`, `download_spk`, `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p2 $(Build.BuildId) --hld-commit-id $latest_commit --env $BRANCH_NAME --image-tag $tag_name --pr $pr_id --repository $repourl`, @@ -430,6 +433,26 @@ export const createTestHldAzurePipelinesYaml = ( BEDROCK_BUILD_SCRIPT: "$(BUILD_SCRIPT_URL)" } }, + { + script: generateYamlScript([ + `commitId=$(Build.SourceVersion)`, + `commitId=$(echo "\${commitId:0:7}")`, + `. ./build.sh --source-only`, + `get_spk_version`, + `download_spk`, + `message="$(Build.SourceVersionMessage)"`, + `if [[ $message == *"Merged PR"* ]]; then`, + `pr_id=$(echo $message | grep -oE '[0-9]+' | head -1 | sed -e 's/^0\\+//')`, + `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId --pr $pr_id`, + `else`, + `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId`, + `fi` + ]), + displayName: + "If configured, update manifest pipeline details in Spektate db before manifest generation", + condition: + "and(ne(variables['INTROSPECTION_ACCOUNT_NAME'], ''), ne(variables['INTROSPECTION_ACCOUNT_KEY'], ''),ne(variables['INTROSPECTION_TABLE_NAME'], ''),ne(variables['INTROSPECTION_PARTITION_KEY'], ''))" + }, { task: "ShellScript@2", displayName: "Validate fabrikate definitions", @@ -458,28 +481,17 @@ export const createTestHldAzurePipelinesYaml = ( }, { script: generateYamlScript([ + `. ./build.sh --source-only`, `cd "$HOME"/\${MANIFEST_REPO##*/}`, - `commitId=$(Build.SourceVersion)`, - `commitId=$(echo "\${commitId:0:7}")`, `latest_commit=$(git rev-parse --short HEAD)`, `url=$(git remote --verbose | grep origin | grep fetch | cut -f2 | cut -d' ' -f1)`, `repourl=\${url##*@}`, - `echo "Downloading SPK"`, - `curl https://raw.githubusercontent.com/Microsoft/bedrock/master/gitops/azure-devops/build.sh > build.sh`, - `chmod +x build.sh`, - `. ./build.sh --source-only`, `get_spk_version`, `download_spk`, - `message="$(Build.SourceVersionMessage)"`, - `if [[ $message == *"Merged PR"* ]]; then`, - `pr_id=$(echo $message | grep -oE '[0-9]+' | head -1 | sed -e 's/^0\\+//')`, - `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId --manifest-commit-id $latest_commit --pr $pr_id --repository $repourl`, - `else`, - `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId --manifest-commit-id $latest_commit --repository $repourl`, - `fi` + `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --manifest-commit-id $latest_commit --repository $repourl` ]), displayName: - "If configured, update manifest pipeline details in Spektate db", + "If configured, update manifest pipeline details in Spektate db after manifest generation", condition: "and(ne(variables['INTROSPECTION_ACCOUNT_NAME'], ''), ne(variables['INTROSPECTION_ACCOUNT_KEY'], ''),ne(variables['INTROSPECTION_TABLE_NAME'], ''),ne(variables['INTROSPECTION_PARTITION_KEY'], ''))" }