From fc921f9179bb3d9b8fba3ba227a6b6bf7fc42276 Mon Sep 17 00:00:00 2001 From: Edaena Salinas Date: Wed, 25 Mar 2020 17:00:14 -0700 Subject: [PATCH 1/7] Add variable group to existing manifest-generation.yaml --- src/commands/hld/index.ts | 2 +- src/lib/fileutils.test.ts | 40 ++++++++++++++ src/lib/fileutils.ts | 25 +++++++++ src/test/mockFactory.ts | 107 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 173 insertions(+), 1 deletion(-) diff --git a/src/commands/hld/index.ts b/src/commands/hld/index.ts index 9bde8bdea..8fda88d79 100644 --- a/src/commands/hld/index.ts +++ b/src/commands/hld/index.ts @@ -1,6 +1,6 @@ import { Command } from "../command"; -const subfolders = ["init", "pipeline", "reconcile"]; +const subfolders = ["init", "pipeline", "reconcile", "append-variable-group"]; export const commandDecorator = Command( "hld", diff --git a/src/lib/fileutils.test.ts b/src/lib/fileutils.test.ts index b978088cb..2b33c5b1e 100644 --- a/src/lib/fileutils.test.ts +++ b/src/lib/fileutils.test.ts @@ -28,6 +28,7 @@ import { disableVerboseLogging, enableVerboseLogging } from "../logger"; import { createTestComponentYaml, createTestHldAzurePipelinesYaml, + createTestHldAzurePipelinesYamlWithVariableGroup, createTestHldLifecyclePipelineYaml, createTestMaintainersYaml, createTestServiceBuildAndUpdatePipelineYaml, @@ -35,6 +36,7 @@ import { import { AccessYaml, AzurePipelinesYaml, MaintainersFile } from "../types"; import { addNewServiceToMaintainersFile, + appendVariableGroupToPipelineYaml, generateAccessYaml, generateDefaultHldComponentYaml, generateDockerfile, @@ -423,6 +425,44 @@ describe("generateHldAzurePipelinesYaml", () => { }); }); +describe("appendVariableGroupToHldAzurePipelinesYaml", () => { + const targetDirectory = "hld-repository"; + const writeSpy = jest.spyOn(fs, "writeFileSync"); + const appendSpy = jest.spyOn(fs, "appendFileSync"); + beforeEach(() => { + mockFs({ + "hld-repository": {}, + }); + }); + afterEach(() => { + mockFs.restore(); + }); + it("should append a variable group", () => { + const absTargetPath = path.resolve(targetDirectory); + const expectedFilePath = `${absTargetPath}/${RENDER_HLD_PIPELINE_FILENAME}`; + + const mockFsOptions = { + [`${targetDirectory}/${RENDER_HLD_PIPELINE_FILENAME}`]: createTestHldAzurePipelinesYaml() as string, + }; + mockFs(mockFsOptions); + + appendVariableGroupToPipelineYaml( + absTargetPath, + RENDER_HLD_PIPELINE_FILENAME, + "my-vg" + ); + expect(writeSpy).toBeCalledWith( + expectedFilePath, + `${getVersionMessage()}\n`, + "utf8" + ); + expect(appendSpy).toBeCalledWith( + expectedFilePath, + createTestHldAzurePipelinesYamlWithVariableGroup() + ); + }); +}); + describe("generateDefaultHldComponentYaml", () => { const targetDirectory = "hld-repository"; const writeSpy = jest.spyOn(fs, "writeFileSync"); diff --git a/src/lib/fileutils.ts b/src/lib/fileutils.ts index 06965ec15..9297255b1 100644 --- a/src/lib/fileutils.ts +++ b/src/lib/fileutils.ts @@ -19,6 +19,7 @@ import { MaintainersFile, User, } from "../types"; +import { readYaml, write } from "../config"; /** * Read given pipeline file as json object. @@ -470,6 +471,29 @@ export const updateTriggerBranchesForServiceBuildAndUpdatePipeline = ( ); }; +/** + * Appends a variable group an Azure pipeline yaml + * @param dir The directory where the pipeline yaml file is + * @param pipelineFile The name of the pipeline yaml file + * @param variableGroupName The name of the variable group to be added + */ +export const appendVariableGroupToPipelineYaml = ( + dir: string, + fileName: string, + variableGroupName: string +): void => { + const pipelineFile = readYaml(path.join(dir, fileName)) as AzurePipelinesYaml; + + if (!pipelineFile.variables) { + pipelineFile.variables = []; + } + + pipelineFile.variables.push({ group: variableGroupName }); + + logger.info(`Updating '${dir}/${fileName}'.`); + write(pipelineFile, dir, fileName); +}; + /** * Returns a the Manifest Generation Pipeline as defined here: https://github.com/microsoft/bedrock/blob/master/gitops/azure-devops/ManifestGeneration.md#add-azure-pipelines-build-yaml */ @@ -481,6 +505,7 @@ const manifestGenerationPipelineYaml = (): string => { include: ["master"], }, }, + variables: [], pool: { vmImage: VM_IMAGE, }, diff --git a/src/test/mockFactory.ts b/src/test/mockFactory.ts index 33eedc53f..8b5fa5e41 100644 --- a/src/test/mockFactory.ts +++ b/src/test/mockFactory.ts @@ -413,6 +413,113 @@ export const createTestHldAzurePipelinesYaml = ( include: ["master"], }, }, + variables: [], + pool: { + vmImage: VM_IMAGE, + }, + steps: [ + { + checkout: "self", + persistCredentials: true, + clean: true, + }, + { + task: "HelmInstaller@1", + inputs: { + helmVersionToInstall: HELM_VERSION, + }, + }, + { + script: generateYamlScript([ + `# Download build.sh`, + `curl $BEDROCK_BUILD_SCRIPT > build.sh`, + `chmod +x ./build.sh`, + ]), + displayName: "Download bedrock bash scripts", + env: { + BEDROCK_BUILD_SCRIPT: "$(BUILD_SCRIPT_URL)", + }, + }, + { + script: generateYamlScript([ + `commitId=$(Build.SourceVersion)`, + `commitId=$(echo "\${commitId:0:7}")`, + `. ./build.sh --source-only`, + `get_spk_version`, + `download_spk`, + `message="$(Build.SourceVersionMessage)"`, + `if [[ $message == *"Merge"* ]]; then`, + `pr_id=$(echo $message | grep -oE '[0-9]+' | head -1 | sed -e 's/^0\\+//')`, + `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId --pr $pr_id`, + `else`, + `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId`, + `fi`, + ]), + displayName: + "If configured, update manifest pipeline details in Spektate db before manifest generation", + condition: + "and(ne(variables['INTROSPECTION_ACCOUNT_NAME'], ''), ne(variables['INTROSPECTION_ACCOUNT_KEY'], ''),ne(variables['INTROSPECTION_TABLE_NAME'], ''),ne(variables['INTROSPECTION_PARTITION_KEY'], ''))", + }, + { + task: "ShellScript@2", + displayName: "Validate fabrikate definitions", + inputs: { + scriptPath: "build.sh", + }, + condition: `eq(variables['Build.Reason'], 'PullRequest')`, + env: { + VERIFY_ONLY: 1, + }, + }, + { + task: "ShellScript@2", + displayName: + "Transform fabrikate definitions and publish to YAML manifests to repo", + inputs: { + scriptPath: "build.sh", + }, + condition: `ne(variables['Build.Reason'], 'PullRequest')`, + env: { + ACCESS_TOKEN_SECRET: "$(PAT)", + COMMIT_MESSAGE: "$(Build.SourceVersionMessage)", + REPO: "$(MANIFEST_REPO)", + BRANCH_NAME: "$(Build.SourceBranchName)", + }, + }, + { + script: generateYamlScript([ + `. ./build.sh --source-only`, + `cd "$HOME"/\${MANIFEST_REPO##*/}`, + `latest_commit=$(git rev-parse --short HEAD)`, + `url=$(git remote --verbose | grep origin | grep fetch | cut -f2 | cut -d' ' -f1)`, + `repourl=\${url##*@}`, + `get_spk_version`, + `download_spk`, + `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --manifest-commit-id $latest_commit --repository $repourl`, + ]), + displayName: + "If configured, update manifest pipeline details in Spektate db after manifest generation", + condition: + "and(ne(variables['INTROSPECTION_ACCOUNT_NAME'], ''), ne(variables['INTROSPECTION_ACCOUNT_KEY'], ''),ne(variables['INTROSPECTION_TABLE_NAME'], ''),ne(variables['INTROSPECTION_PARTITION_KEY'], ''))", + }, + ], + }; + + return asString + ? yaml.safeDump(data, { lineWidth: Number.MAX_SAFE_INTEGER }) + : data; +}; + +export const createTestHldAzurePipelinesYamlWithVariableGroup = ( + asString = true +): AzurePipelinesYaml | string => { + const data: AzurePipelinesYaml = { + trigger: { + branches: { + include: ["master"], + }, + }, + variables: [{ group: "my-vg" }], pool: { vmImage: VM_IMAGE, }, From 91b63c64e76cc59e3322065078cef47a195ac970 Mon Sep 17 00:00:00 2001 From: Edaena Salinas Date: Wed, 25 Mar 2020 17:13:50 -0700 Subject: [PATCH 2/7] Update integration tests --- tests/validations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/validations.sh b/tests/validations.sh index d9818b882..d81bacfef 100644 --- a/tests/validations.sh +++ b/tests/validations.sh @@ -153,7 +153,7 @@ spk hld install-manifest-pipeline --org-name $AZDO_ORG -d $AZDO_PROJECT --person # Will no longer be needed once install-manifest-pipeline supports adding a VG ################################## cd $hld_dir -printf "variables:\n - group: $vg_name\n" | cat - manifest-generation.yaml > temp && mv temp manifest-generation.yaml +spk hld append-variable-group $vg_name git add . git commit -m "Adding variable group $vg_name to pipeline" git push origin master From 3f2f4e7b5a2a401298b7422a47fd495fb2a3f769 Mon Sep 17 00:00:00 2001 From: Edaena Salinas Date: Thu, 26 Mar 2020 10:09:33 -0700 Subject: [PATCH 3/7] Adding files for append variable group --- .../hld/append-variable-group.decorator.json | 5 ++ src/commands/hld/append-variable-group.ts | 60 +++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 src/commands/hld/append-variable-group.decorator.json create mode 100644 src/commands/hld/append-variable-group.ts diff --git a/src/commands/hld/append-variable-group.decorator.json b/src/commands/hld/append-variable-group.decorator.json new file mode 100644 index 000000000..f685f134f --- /dev/null +++ b/src/commands/hld/append-variable-group.decorator.json @@ -0,0 +1,5 @@ +{ + "command": "append-variable-group ", + "alias": "av", + "description": "Appends the name of an existing variable group to the current manifest-generation.yaml file." +} diff --git a/src/commands/hld/append-variable-group.ts b/src/commands/hld/append-variable-group.ts new file mode 100644 index 000000000..8dedc9185 --- /dev/null +++ b/src/commands/hld/append-variable-group.ts @@ -0,0 +1,60 @@ +import commander from "commander"; +import { build as buildCmd, exit as exitCmd } from "../../lib/commandBuilder"; +import { RENDER_HLD_PIPELINE_FILENAME } from "../../lib/constants"; +import { appendVariableGroupToPipelineYaml } from "../../lib/fileutils"; +import { logger } from "../../logger"; +import decorator from "./append-variable-group.decorator.json"; +import { Logger } from "azure-storage"; +import path from "path"; + +interface CommandOptions { + file: string | undefined; + interactive: boolean; +} + +/** + * Executes the command, can all exit function with 0 or 1 + * when command completed successfully or failed respectively. + * + * @param opts option value from commander + * @param exitFn exit function + */ +export const execute = async ( + hldRepoPath: string, + variableGroupName: string, + exitFn: (status: number) => Promise +): Promise => { + if (!variableGroupName) { + logger.error("Service name is missing"); + await exitFn(1); + return; + } + + try { + appendVariableGroupToPipelineYaml( + hldRepoPath, + RENDER_HLD_PIPELINE_FILENAME, + variableGroupName + ); + await exitFn(0); + } catch (err) { + logger.error(`Error occurred while appending variable group.`); + logger.error(err); + await exitFn(1); + } + + await exitFn(0); +}; + +/** + * Adds the init command to the commander command object + * @param command Commander command object to decorate + */ +export const commandDecorator = (command: commander.Command): void => { + buildCmd(command, decorator).action(async (variableGroupName: string) => { + const hldRepoPath = process.cwd(); + await execute(hldRepoPath, variableGroupName, async (status: number) => { + await exitCmd(logger, process.exit, status); + }); + }); +}; From 686a52645a3154ebb27fa30a9ee24e5b41d65ba8 Mon Sep 17 00:00:00 2001 From: Edaena Salinas Date: Thu, 26 Mar 2020 13:31:36 -0700 Subject: [PATCH 4/7] Add unit test file --- src/commands/hld/append-variable-group.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/commands/hld/append-variable-group.ts b/src/commands/hld/append-variable-group.ts index 8dedc9185..c48f4735a 100644 --- a/src/commands/hld/append-variable-group.ts +++ b/src/commands/hld/append-variable-group.ts @@ -4,8 +4,6 @@ import { RENDER_HLD_PIPELINE_FILENAME } from "../../lib/constants"; import { appendVariableGroupToPipelineYaml } from "../../lib/fileutils"; import { logger } from "../../logger"; import decorator from "./append-variable-group.decorator.json"; -import { Logger } from "azure-storage"; -import path from "path"; interface CommandOptions { file: string | undefined; @@ -25,7 +23,7 @@ export const execute = async ( exitFn: (status: number) => Promise ): Promise => { if (!variableGroupName) { - logger.error("Service name is missing"); + logger.error("Variable group name is missing."); await exitFn(1); return; } @@ -42,8 +40,6 @@ export const execute = async ( logger.error(err); await exitFn(1); } - - await exitFn(0); }; /** From 4278e27335dea76c1abac05747ed195ad7c911fb Mon Sep 17 00:00:00 2001 From: Edaena Salinas Date: Thu, 26 Mar 2020 13:41:38 -0700 Subject: [PATCH 5/7] Update test mock function --- src/test/mockFactory.ts | 101 ++-------------------------------------- 1 file changed, 4 insertions(+), 97 deletions(-) diff --git a/src/test/mockFactory.ts b/src/test/mockFactory.ts index 8b5fa5e41..313cb7eb2 100644 --- a/src/test/mockFactory.ts +++ b/src/test/mockFactory.ts @@ -513,103 +513,10 @@ export const createTestHldAzurePipelinesYaml = ( export const createTestHldAzurePipelinesYamlWithVariableGroup = ( asString = true ): AzurePipelinesYaml | string => { - const data: AzurePipelinesYaml = { - trigger: { - branches: { - include: ["master"], - }, - }, - variables: [{ group: "my-vg" }], - pool: { - vmImage: VM_IMAGE, - }, - steps: [ - { - checkout: "self", - persistCredentials: true, - clean: true, - }, - { - task: "HelmInstaller@1", - inputs: { - helmVersionToInstall: HELM_VERSION, - }, - }, - { - script: generateYamlScript([ - `# Download build.sh`, - `curl $BEDROCK_BUILD_SCRIPT > build.sh`, - `chmod +x ./build.sh`, - ]), - displayName: "Download bedrock bash scripts", - env: { - BEDROCK_BUILD_SCRIPT: "$(BUILD_SCRIPT_URL)", - }, - }, - { - script: generateYamlScript([ - `commitId=$(Build.SourceVersion)`, - `commitId=$(echo "\${commitId:0:7}")`, - `. ./build.sh --source-only`, - `get_spk_version`, - `download_spk`, - `message="$(Build.SourceVersionMessage)"`, - `if [[ $message == *"Merge"* ]]; then`, - `pr_id=$(echo $message | grep -oE '[0-9]+' | head -1 | sed -e 's/^0\\+//')`, - `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId --pr $pr_id`, - `else`, - `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId`, - `fi`, - ]), - displayName: - "If configured, update manifest pipeline details in Spektate db before manifest generation", - condition: - "and(ne(variables['INTROSPECTION_ACCOUNT_NAME'], ''), ne(variables['INTROSPECTION_ACCOUNT_KEY'], ''),ne(variables['INTROSPECTION_TABLE_NAME'], ''),ne(variables['INTROSPECTION_PARTITION_KEY'], ''))", - }, - { - task: "ShellScript@2", - displayName: "Validate fabrikate definitions", - inputs: { - scriptPath: "build.sh", - }, - condition: `eq(variables['Build.Reason'], 'PullRequest')`, - env: { - VERIFY_ONLY: 1, - }, - }, - { - task: "ShellScript@2", - displayName: - "Transform fabrikate definitions and publish to YAML manifests to repo", - inputs: { - scriptPath: "build.sh", - }, - condition: `ne(variables['Build.Reason'], 'PullRequest')`, - env: { - ACCESS_TOKEN_SECRET: "$(PAT)", - COMMIT_MESSAGE: "$(Build.SourceVersionMessage)", - REPO: "$(MANIFEST_REPO)", - BRANCH_NAME: "$(Build.SourceBranchName)", - }, - }, - { - script: generateYamlScript([ - `. ./build.sh --source-only`, - `cd "$HOME"/\${MANIFEST_REPO##*/}`, - `latest_commit=$(git rev-parse --short HEAD)`, - `url=$(git remote --verbose | grep origin | grep fetch | cut -f2 | cut -d' ' -f1)`, - `repourl=\${url##*@}`, - `get_spk_version`, - `download_spk`, - `./spk/spk deployment create -n $(INTROSPECTION_ACCOUNT_NAME) -k $(INTROSPECTION_ACCOUNT_KEY) -t $(INTROSPECTION_TABLE_NAME) -p $(INTROSPECTION_PARTITION_KEY) --p3 $(Build.BuildId) --manifest-commit-id $latest_commit --repository $repourl`, - ]), - displayName: - "If configured, update manifest pipeline details in Spektate db after manifest generation", - condition: - "and(ne(variables['INTROSPECTION_ACCOUNT_NAME'], ''), ne(variables['INTROSPECTION_ACCOUNT_KEY'], ''),ne(variables['INTROSPECTION_TABLE_NAME'], ''),ne(variables['INTROSPECTION_PARTITION_KEY'], ''))", - }, - ], - }; + const data: AzurePipelinesYaml = createTestHldAzurePipelinesYaml( + false + ) as AzurePipelinesYaml; + data.variables = [{ group: "my-vg" }]; return asString ? yaml.safeDump(data, { lineWidth: Number.MAX_SAFE_INTEGER }) From b41130bb54c81678d36d419102e145925a1dc26c Mon Sep 17 00:00:00 2001 From: Edaena Salinas Date: Thu, 26 Mar 2020 15:00:53 -0700 Subject: [PATCH 6/7] Update unit test --- src/commands/hld/append-variable-group.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/commands/hld/append-variable-group.test.ts diff --git a/src/commands/hld/append-variable-group.test.ts b/src/commands/hld/append-variable-group.test.ts new file mode 100644 index 000000000..1303d8556 --- /dev/null +++ b/src/commands/hld/append-variable-group.test.ts @@ -0,0 +1,18 @@ +import { execute } from "./append-variable-group"; +import * as fileutils from "../../lib/fileutils"; + +describe("Test execute function", () => { + it("missing variable group name", async () => { + const exitFn = jest.fn(); + await execute("my-path", "", exitFn); + expect(exitFn).toBeCalledTimes(1); + expect(exitFn.mock.calls).toEqual([[1]]); + }); + it("appends variable group", async () => { + const exitFn = jest.fn(); + spyOn(fileutils, "appendVariableGroupToPipelineYaml"); + await execute("my-path", "my-vg", exitFn); + expect(exitFn).toBeCalledTimes(1); + expect(exitFn.mock.calls).toEqual([[0]]); + }); +}); From 3dacea929bb48599f4fe25ed066aea4cdfed8996 Mon Sep 17 00:00:00 2001 From: Edaena Salinas Date: Fri, 27 Mar 2020 15:03:22 -0700 Subject: [PATCH 7/7] updates --- docs/commands/data.json | 5 +++++ src/commands/hld/append-variable-group.decorator.json | 2 +- src/commands/hld/append-variable-group.ts | 10 +++------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/commands/data.json b/docs/commands/data.json index 38057dc8f..7bb3e0918 100644 --- a/docs/commands/data.json +++ b/docs/commands/data.json @@ -226,6 +226,11 @@ } ] }, + "hld append-variable-group": { + "command": "append-variable-group ", + "alias": "avg", + "description": "Appends the name of an existing variable group to the current manifest-generation.yaml file." + }, "hld init": { "command": "init", "alias": "i", diff --git a/src/commands/hld/append-variable-group.decorator.json b/src/commands/hld/append-variable-group.decorator.json index f685f134f..8ee25be23 100644 --- a/src/commands/hld/append-variable-group.decorator.json +++ b/src/commands/hld/append-variable-group.decorator.json @@ -1,5 +1,5 @@ { "command": "append-variable-group ", - "alias": "av", + "alias": "avg", "description": "Appends the name of an existing variable group to the current manifest-generation.yaml file." } diff --git a/src/commands/hld/append-variable-group.ts b/src/commands/hld/append-variable-group.ts index c48f4735a..004a05d29 100644 --- a/src/commands/hld/append-variable-group.ts +++ b/src/commands/hld/append-variable-group.ts @@ -5,16 +5,12 @@ import { appendVariableGroupToPipelineYaml } from "../../lib/fileutils"; import { logger } from "../../logger"; import decorator from "./append-variable-group.decorator.json"; -interface CommandOptions { - file: string | undefined; - interactive: boolean; -} - /** - * Executes the command, can all exit function with 0 or 1 + * Executes the command, can call exit function with 0 or 1 * when command completed successfully or failed respectively. * - * @param opts option value from commander + * @param hldRepoPath The hld repository path + * @param variableGroupName The variable group name * @param exitFn exit function */ export const execute = async (