From 83fc54ae9ddb08f4dee1f7042a65e3d390ff337c Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Thu, 25 Mar 2021 12:10:16 -0700 Subject: [PATCH 01/31] two-stages-pipeline plugin --- mypy.ini | 2 +- samcli/commands/_utils/template.py | 27 +- samcli/lib/pipeline/init/plugins/__init__.py | 0 .../plugins/two_stages_pipeline/__init__.py | 0 .../cfn_templates/.stage_resources.yaml.swp | Bin 0 -> 16384 bytes .../cfn_templates/deployer.yaml | 36 ++ .../cfn_templates/stage_resources.yaml | 222 +++++++++ .../plugins/two_stages_pipeline/config.py | 2 + .../plugins/two_stages_pipeline/context.py | 92 ++++ .../two_stages_pipeline/postprocessor.py | 104 +++++ .../two_stages_pipeline/preprocessor.py | 238 ++++++++++ .../two_stages_pipeline/questions.json | 114 +++++ .../plugins/two_stages_pipeline/resource.py | 26 ++ .../init/plugins/two_stages_pipeline/stage.py | 155 ++++++ .../lib/utils/managed_cloudformation_stack.py | 82 +++- samcli/local/common/runtime_template.py | 15 + tests/unit/commands/_utils/test_template.py | 17 + tests/unit/lib/pipeline/__init__.py | 0 tests/unit/lib/pipeline/init/__init__.py | 0 .../lib/pipeline/init/plugins/__init__.py | 0 .../plugins/two_stages_pipeline/__init__.py | 0 .../two_stages_pipeline/test_context.py | 161 +++++++ .../two_stages_pipeline/test_postprocessor.py | 101 ++++ .../two_stages_pipeline/test_preprocessor.py | 440 ++++++++++++++++++ .../two_stages_pipeline/test_resource.py | 49 ++ .../plugins/two_stages_pipeline/test_stage.py | 70 +++ .../test_managed_cloudformation_stack.py | 21 +- 27 files changed, 1951 insertions(+), 23 deletions(-) create mode 100644 samcli/lib/pipeline/init/plugins/__init__.py create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/.stage_resources.yaml.swp create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/deployer.yaml create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/stage_resources.yaml create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/config.py create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/context.py create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/questions.json create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/resource.py create mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/stage.py create mode 100644 tests/unit/lib/pipeline/__init__.py create mode 100644 tests/unit/lib/pipeline/init/__init__.py create mode 100644 tests/unit/lib/pipeline/init/plugins/__init__.py create mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py create mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_context.py create mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py create mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_preprocessor.py create mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_resource.py create mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_stage.py diff --git a/mypy.ini b/mypy.ini index 30040750a02..02bf97b5f9d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -59,6 +59,6 @@ ignore_missing_imports=True ignore_missing_imports=True # progressive add typechecks and these modules already complete the process, let's keep them clean -[mypy-samcli.commands.build,samcli.lib.build.*,samcli.commands.local.cli_common.invoke_context,samcli.commands.local.lib.local_lambda,samcli.lib.providers.*] +[mypy-samcli.commands.build,samcli.lib.build.*,samcli.commands.local.cli_common.invoke_context,samcli.commands.local.lib.local_lambda,samcli.lib.providers.*,samcli.lib.cookiecutter.*,samcli.lib.pipeline.*] disallow_untyped_defs=True disallow_incomplete_defs=True \ No newline at end of file diff --git a/samcli/commands/_utils/template.py b/samcli/commands/_utils/template.py index bd9658b55b6..4eeef7326e9 100644 --- a/samcli/commands/_utils/template.py +++ b/samcli/commands/_utils/template.py @@ -4,14 +4,12 @@ import itertools import os import pathlib +from typing import List import jmespath import yaml from botocore.utils import set_value_from_jmespath -from samcli.commands.exceptions import UserException -from samcli.lib.utils.packagetype import ZIP -from samcli.yamlhelper import yaml_parse, yaml_dump from samcli.commands._utils.resources import ( METADATA_WITH_LOCAL_PATHS, RESOURCES_WITH_LOCAL_PATHS, @@ -19,6 +17,9 @@ AWS_LAMBDA_FUNCTION, get_packageable_resource_paths, ) +from samcli.commands.exceptions import UserException +from samcli.lib.utils.packagetype import ZIP +from samcli.yamlhelper import yaml_parse, yaml_dump class TemplateNotFoundException(UserException): @@ -300,3 +301,23 @@ def get_template_function_resource_ids(template_file, artifact): ]: _function_resource_ids.append(resource_id) return _function_resource_ids + + +def get_template_function_runtimes(template_file: str) -> List[str]: + """ + Get a list of function runtimes from template file. + Function resource types include + AWS::Lambda::Function + AWS::Serverless::Function + :param template_file: template file location. + :return: list of runtimes + """ + + template_dict = get_template_data(template_file=template_file) + _function_runtimes = set() + for _, resource in template_dict.get("Resources", {}).items(): + if resource.get("Type") in [AWS_SERVERLESS_FUNCTION, AWS_LAMBDA_FUNCTION]: + runtime = resource.get("Properties", {}).get("Runtime") + if runtime: # IMAGE functions don't define a runtime + _function_runtimes.add(runtime) + return list(_function_runtimes) diff --git a/samcli/lib/pipeline/init/plugins/__init__.py b/samcli/lib/pipeline/init/plugins/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/.stage_resources.yaml.swp b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/.stage_resources.yaml.swp new file mode 100644 index 0000000000000000000000000000000000000000..e7b96679f04dd5a034634f10da0c32593b28b08d GIT binary patch literal 16384 zcmeI2ZHy#E8OM9MLlEu`BBu!`m%4#CcgM}n?rk`@4u%Xnv&VYu-DYPw?)Gw7YkF$- zn(gTxx~p#)Q8XG6jUOc93xSxJs3GCQ5i|-RYWQHH;fvx+G%9Mq`0~Y(2gmGE>h}_0-e#JY7%Cw5E>FFOXX&r!`zxYuf6Glka`BAv4yZjCc39nT+;y)<4OXQaUd5x` zieZ^myowjgY*l+^k6EV8s-|u7YR~f5OxvsS4QJKkbd7ndk$};$S2^qUEXtWz4O&-S z<~hD=FmJL?yVigW#hbzd7q17lYcmI?r^NhkDD5WKyzTCb*A>Og3l9_?C_GSjpzuK9 zfx-iY2MP~dgdX6%OSBJR-j}Gk->+ULQmC_GSj zpzuK9fx-iY2MP}q9w!1w=Q{r~f;H0^8P8Squ`dGHW83Fg4f z;CiqPyuL%zeg|F!KLcL`E8seCE!YA6xn0wK1D*v>f`@?xPJ-LP9`M>W$bj#FbKn6$ zK@IE!+rVG0)U=nt55WuIN$@e?gS){Y@NTdT{QU~Zfp38)z^4EMN5D;BCwTp>n)Why z9()OW9Gn0*g4ZtBw4Z`+gHM1Z@GkJmWv~xyfZM<>a6PyT{P``K_I>ai_$Xjt85{)f z09(KxFU1_d=fDR*3)~5&!7i{B{BkSC01p8R+zz&bU*jO=MQ{#$25@i;RKeT9PA~y} zkNf!o_%e7D_~0~X0CBIc1?Rtrx|+;A%iN~fW)0hL`#o+t_N-%f%r)Po;*G9J$vb>w zo$**G{G5;`r`-2+(y(dEVr?aVoVgy0#B`htGj3``*lrstu@8dN*f;GQ%pkXr8dm`-Zqw7&iN>Cq2jQ+C zb|#}J$S&}l3SYvalv8cX5P%*ry6Zuo=_E0B9yqdqWFX@<98zTre3(iE1lREUCt5{9 zC>fUHw>ys8l~W9p1DtEDhw3tGNtN(_RuzG_tbW+m^t)Z^_D9nj7}EPMbQ))|2u4(F z+|LZ3v)MD$USmzh#~W@;wON;8iJnI(Kkc0_B4hWDlNQCD9yc)xosm-`8JZfyH53PF zI0Z34S!9Mq>UknzhLIQxjPIYEq#(MB0zZy>EgnY3ecdtq!E{m~vPI+&1i67bahZ*w#JId%b8#bFn}Af6RTbS_MybG~KC#d0B*h}hqn2G-+X z=2kyq#?vA_nfyK1kad$0*0nIXVABm(=kKMlBWT>`Q_CZ#*p+&~t5kCPT7J=Zylx=I#Asgc7D1IreviiVOKLs8E zZEyf=1zW%?sNKH@z5+fAY;XeH2KIuhz`szt{{lP@9tR%=_kv|0YWr7FzkeOv2O3}+ zTmk-s`u!#FH25Uwf%kxSf*s&*sNw$#{s4Xi9tWQW9|5c27H|#N0)Bv6{=49t;4yF( z+ym-B)cRL~e~Vfkd;>fR9t4x%av*&EE8+LzDm+kl-~xC+JhhKg%g56_+^(k*KkB^b ztRQ-3=tCZTcy17EB*k%0Sg0-?B~FL%GmOkG93j5P+61XNe!P%Ya4MEx$fCcjIq}9` zoq?o8@IVOc5VpisNJ=Q0la$JTlB;f#hcMDvLBqg2#g09kXf%|X8>FL;{ z>{#>dPfFw)pOhHG@>+a{uxTd3f<4Ku_M)%k`;6^*TL3kF@&;tY@@J6|dxKNS^e5(% z`B2x6`Ml?&e2Nb@b57WlEy72TBm-9khaVla<2dRB>m(&1#D^}(WSk}Y4R@UB;bdo} z)cNX=(*-FNoRDTE@(aWGOrR9(($acSt)5!iT5S9X$CoMPRy zPpR{^oK1DPy01)56S1>SUqjtKun0q~?2zh=H)XC+-cN=OJA#YEh{^O4S}^RciyD7M zzsvM^&RM%?f}t|S3~68V`LWi$IDH+`4*lHr>>`O5$lx(ps5c?&oh^cjWR^(Qn& zJ_#Ek%97LD%)+GXX2vCDwa_);>oB{-iPL~^?rDMOC5n3y_GPM$_OZxyy}UoKv_72< z{jqzH(VXh%|0)>M**X)B`5~wFCArBpxLcctTPisQT8WdD_$$RI>{1#-;;q?i)T}kf yHTfA-^y1qcJkEmNu}0K|? None: + + testing_stage: Stage = Stage( + name=Context.TESTING_STAGE_NAME, + aws_profile=context.get("testing_profile"), + aws_region=context.get("testing_region"), + stack_name=context.get("testing_stack_name"), + deployer_role_arn=context.get("testing_deployer_role"), + cfn_deployment_role_arn=context.get("testing_cfn_deployment_role"), + artifacts_bucket_arn=context.get("testing_artifacts_bucket"), + ) + + prod_stage: Stage = Stage( + name=Context.PROD_STAGE_NAME, + aws_profile=context.get("prod_profile"), + aws_region=context.get("prod_region"), + stack_name=context.get("prod_stack_name"), + deployer_role_arn=context.get("prod_deployer_role"), + cfn_deployment_role_arn=context.get("prod_cfn_deployment_role"), + artifacts_bucket_arn=context.get("prod_artifacts_bucket"), + ) + + self.stages: List[Stage] = [testing_stage, prod_stage] + self.deployer: Deployer = Deployer(arn=context.get("deployer_arn")) + self.deployer_aws_access_key_id_variable_name: str = context.get("deployer_aws_access_key_id_variable_name") + self.deployer_aws_secret_access_key_variable_name: str = context.get( + "deployer_aws_secret_access_key_variable_name" + ) + self.build_image: Optional[str] = None + + def get_stage(self, stage_name: str) -> Stage: + """ + returns a stage by name. + + Parameters + ---------- + stage_name: str + The name of the stage to return + """ + return next((stage for stage in self.stages if stage.name == stage_name), None) + + def deployer_permissions(self) -> str: + """ + returns a string representing the required IAM policies for the deployer IAM user + to be able to deploy the pipeline + """ + deployer_roles = ", ".join(list(filter(None, map(lambda stage: stage.deployer_role.arn, self.stages)))) + permissions = f""" +{{ + "Effect": "Allow", + "Action": ["sts:AssumeRole"], + "Resource": [{deployer_roles}] +}} + """ + return permissions diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py new file mode 100644 index 00000000000..b77fa344a6e --- /dev/null +++ b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py @@ -0,0 +1,104 @@ +""" +The plugin's postprocessor prints information about the created and reused AWS resources and the required permissions +""" +import logging +from typing import Dict, List + +import click + +from samcli.lib.cookiecutter.processor import Processor +from .config import PLUGIN_NAME +from .context import Context as PluginContext +from .resource import Deployer, Resource + +LOG = logging.getLogger(__name__) + + +class Postprocessor(Processor): + """ + prints information about the created and reused AWS resources and the required permissions + + Attributes + ---------- + resources_reused: + A list of the required AWS resources that got provided by the user. + resources_reused: + A list of the required AWS resources that the plugin created on behalf of the user. + """ + + def __init__(self) -> None: + self.resources_reused: List[Dict[str, str]] = [] + self.resources_created: List[Dict[str, str]] = [] + + def run(self, context: Dict) -> Dict: + """ + iterates through the pipeline's AWS resources and categorize them into two categories: + 1. Resources created by the plugin + 2. Resources provided by the user + It then prints to the user the ARNs of the resources created by the plugin. And for each resource provided + by the user, it prints instructions about the required IAM policies for this resource to operate as expected, + so that the user can ensure it already has this permissions. + + Parameters + ---------- + context: Dict + A dictionary of the whole context of the cookiecutter template, it contains a key, context[PLUGIN_NAME], + that contains this plugin's context(object of type PluginContext) where the method extracts its required + information from. + """ + + context = context.copy() + plugin_context: PluginContext = context[PLUGIN_NAME] + deployer: Deployer = plugin_context.deployer + self._categorize_resource(deployer, plugin_context.deployer_permissions()) + + for stage in plugin_context.stages: + self._categorize_resource(stage.deployer_role, stage.deployer_role_permissions(deployer.arn)) + self._categorize_resource(stage.cfn_deployment_role, stage.cfn_deployment_role_permissions()) + self._categorize_resource(stage.artifacts_bucket, stage.artifacts_bucket_permissions()) + + if self.resources_created: + click.secho("\nWe have created the following resources:", fg="yellow") + for resource in self.resources_created: + click.secho(f"\t{resource['arn']}", fg="yellow") + + if self.resources_reused: + click.secho( + "\nWe have reused the following resources, please make sure it has the required permissions:", + fg="yellow", + ) + for resource in self.resources_reused: + click.secho(f"\n{resource['arn']}", fg="yellow") + click.secho(f"Required Permissions: {resource['required_permissions']}", fg="yellow") + click.echo("\n") + + if not deployer.is_user_provided: + click.secho( + "Please set the following variables of the IAM user credentials to your CICD project:", fg="green" + ) + click.secho( + f"{plugin_context.deployer_aws_access_key_id_variable_name}: {deployer.access_key_id}", fg="green" + ) + click.secho( + f"{plugin_context.deployer_aws_secret_access_key_variable_name}: {deployer.secret_access_key}", + fg="green", + ) + + return context + + def _categorize_resource(self, resource: Resource, required_permissions: str) -> None: + """ + add the resource to the corresponding category; reused or created. And store a reference for the resource's + required permissions in case if it is not created by the plugin + + Parameters + ---------- + resource: Resource + The resource to categorized + required_permissions: str + the IAM policies required for the resource to operate correctly. + """ + if resource.is_user_provided: + self.resources_reused.append({"arn": resource.arn, "required_permissions": required_permissions}) + else: + self.resources_created.append({"arn": resource.arn}) diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py new file mode 100644 index 00000000000..c12409a2d9e --- /dev/null +++ b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py @@ -0,0 +1,238 @@ +""" +The plugin's preprocessor creates the required AWS resources for this pipeline if not already provided by the user. +""" +import logging +import os +import pathlib +from typing import Dict, List, Optional, Tuple + +import click + +from samcli.commands._utils.template import get_template_function_runtimes +from samcli.lib.cookiecutter.processor import Processor +from samcli.lib.utils.managed_cloudformation_stack import manage_stack as manage_cloudformation_stack +from samcli.local.common.runtime_template import RUNTIME_TO_BUILD_IMAGE +from .config import PLUGIN_NAME +from .context import Context as PluginContext +from .resource import Deployer +from .stage import Stage + +ROOT_PATH = str(pathlib.Path(os.path.dirname(__file__))) +CFN_TEMPLATE_PATH = os.path.join(ROOT_PATH, "cfn_templates") +STACK_NAME_PREFIX = "aws-sam-cli-managed" +DEPLOYER_STACK_NAME_SUFFIX = "pipeline-deployer" +STAGE_RESOURCES_STACK_NAME_SUFFIX = "pipeline-resources" + +LOG = logging.getLogger(__name__) + + +class Preprocessor(Processor): + """ + 1. Find the appropriate docker build image for the SAM temolate + 2. Creates the required AWS resources for this pipeline if not already provided by the user. + + Methods + ------- + _get_build_image(sam_template_file) -> Optional[str]: + scan the SAM template for the ZIP functions, extract the runtimes and return the appropriate SA< build image + for the runtime if the template contains one and exactly one supported runtime, otherwise, asks the user + to provide an alternative build-image + run(context: Dict) -> Dict: + Creates the missed required AWS resources and updated the passed cookiecutter context with its ARNs + _create_deployer_at(stage: Stage): + deploys the CFN template(./cfn_templates/deployer.yaml) to the given stage + _create_missing_stage_resources(stage: Stage, deployer_arn: str): + deploys the CFN template(./cfn_templates/resource_stages.yaml) to the given stage + """ + + BASIC_PROVIDED_BUILD_IMAGE: str = "public.ecr.aws/sam/build-provided" + + def run(self, context: Dict) -> Dict: + """ + searches the passed cookiecutter context for the pipeline's required AWS resources, identifies which resources + are missing, create them through a CFN stack. + This method create and add to the context a plugin-explicit context that contains all of the required + AWS resources and additional plugin explicit attribute. This plugin-explicit context is not used in the + cookiecutter template itself, instead, it is used by the postprocessor of this plugin. + The method returns a mutated copy of the cookiecutter context that updates the context with the ARNs of the + created resources. + + Parameters + ---------- + context: Dict + The cookiecutter context to look for the resources from. + """ + context = context.copy() + plugin_context: PluginContext = PluginContext(context) + context[PLUGIN_NAME] = plugin_context + + context["build_image"] = plugin_context.build_image = Preprocessor._get_build_image(context["sam_template"]) + + deployer: Deployer = plugin_context.deployer + if deployer.is_user_provided: + deployer_arn = deployer.arn + else: + # create deployer(IAM user) in the testing stage + testing_stage: Stage = plugin_context.get_stage(PluginContext.TESTING_STAGE_NAME) + deployer_arn, access_key_id, secret_access_key = Preprocessor._create_deployer_at(stage=testing_stage) + context["deployer_arn"] = deployer.arn = deployer_arn + deployer.access_key_id = access_key_id + deployer.secret_access_key = secret_access_key + + for stage in plugin_context.stages: + ( + deployer_role_arn, + cfn_deployment_role_arn, + artifacts_bucket_arn, + kms_key_arn, + ) = Preprocessor._create_missing_stage_resources(stage=stage, deployer_arn=deployer_arn) + context[f"{stage.name}_deployer_role"] = stage.deployer_role.arn = deployer_role_arn + context[f"{stage.name}_cfn_deployment_role"] = stage.cfn_deployment_role.arn = cfn_deployment_role_arn + stage.artifacts_bucket.arn = artifacts_bucket_arn + stage.artifacts_bucket.kms_key_arn = kms_key_arn + # The cookiecutter context requires the name of the artifacts bucket instead of ots ARN + context[f"{stage.name}_artifacts_bucket"] = stage.artifacts_bucket.name() + + return context + + # This method is a first iteration only to support the major usecase of having a SAM template with one supported + # runtime + # todo improve the experience to support Iamge lambda functions and lambda functions with different runtimes + @staticmethod + def _get_build_image(sam_template_file: str) -> str: + """ + Scans the SAM template for lambda runtimes, and if it contains only one supported runtime, it returns + the corresponding SAM build-image, otherwise, it asks the user to provide one + + Parameters + ---------- + sam_template_file: str + the path of the SAM template to scan for function's runtimes + + Returns: a docker build-image to use for the CICD pipeline + """ + runtimes: List[str] = get_template_function_runtimes(template_file=sam_template_file) + if not runtimes: + return Preprocessor.BASIC_PROVIDED_BUILD_IMAGE + elif len(runtimes) > 1: + click.echo( + "The SAM template defines multiple functions with different runtimes\n" + "SAM doesn't have an appropriate docker build image for that, please provide one" + ) + return click.prompt("Docker Build image") + else: + runtime = runtimes[0] + build_image = RUNTIME_TO_BUILD_IMAGE.get(runtime) + if not build_image: + click.echo( + f"The SAM template defines functions of runtime {runtime} but SAM doesn't have a docker " + f"build-image for {runtime}, please provide one" + ) + build_image = click.prompt("Docker Build image") + return build_image + + @staticmethod + def _create_deployer_at(stage: Stage) -> Tuple[str, str, str]: + """ + Deploys the CFN template(./cfn_templates/deployer.yaml) which defines a deployer IAM user and credentials + to the AWS account and region associated with the given stage. It will not redeploy the stack if already exists. + + Parameters + ---------- + stage: Stage + The pipeline stage to deploy the CFN template to its associated AWS account and region. + + Returns + ------- + ARN, access_key_id and secret_access_key of the IAM user identified by the template + """ + + profile: str = stage.aws_profile + region: str = stage.aws_region + stack_name: str = f"{STACK_NAME_PREFIX}-{stage.stack_name}-{DEPLOYER_STACK_NAME_SUFFIX}" + deployer_template_path: str = os.path.join(CFN_TEMPLATE_PATH, "deployer.yaml") + with open(deployer_template_path, "r") as fp: + deployer_template_body = fp.read() + click.echo(f"Creating an IAM user for pipeline Deployment. Account: '{profile}' Region: '{region}'") + outputs: List[Dict[str, str]] = manage_cloudformation_stack( + stack_name=stack_name, profile=profile, region=region, template_body=deployer_template_body + ) + deployer_arn: str = next(o for o in outputs if o.get("OutputKey") == "Deployer").get("OutputValue") + access_key_id_arn: str = next(o for o in outputs if o.get("OutputKey") == "AccessKeyId").get("OutputValue") + secret_access_key_arn: str = next(o for o in outputs if o.get("OutputKey") == "SecretAccessKey").get( + "OutputValue" + ) + return deployer_arn, access_key_id_arn, secret_access_key_arn + + @staticmethod + def _create_missing_stage_resources(stage: Stage, deployer_arn: str) -> Tuple[str, str, str, Optional[str]]: + """ + Deploys the CFN template(./cfn_templates/stage_resources.yaml) which defines: + * Deployer execution IAM role + * CloudFormation execution IAM role + * Artifacts' S3 Bucket along with KMS encryption key + to the AWS account and region associated with the given stage. It will not redeploy the stack if already exists. + This CFN template accepts the ARNs of the resources as parameters and will not create a resource if already + provided, this way we can conditionally create a resource only if the user didn't provide it + + Parameters + ---------- + stage: Stage + The pipeline stage to deploy the CFN template to its associated AWS account and region. + deployer_arn: str + The ARN of the deployer IAM user. This is used by the CFN template to give this IAM user permissions to + assume the IAM roles. + + Returns + ------- + ARNs of the deployer execution role, CLoudFormation execution role, artifacts S3 bucket and bucket KMS key. + """ + + if stage.did_user_provide_all_required_resources(): + LOG.info(f"All required resources for the {stage.name} stage exist, skipping creation.") + return ( + stage.deployer_role.arn, + stage.cfn_deployment_role.arn, + stage.artifacts_bucket.arn, + stage.artifacts_bucket.kms_key_arn, + ) + missing_resources: str = "" + if not stage.deployer_role.is_user_provided: + missing_resources += "\n\tDeployer role." + if not stage.cfn_deployment_role.is_user_provided: + missing_resources += "\n\tCloudFormation deployment role." + if not stage.artifacts_bucket.is_user_provided: + missing_resources += "\n\tArtifacts bucket." + LOG.info(f"Creating missing required resources for the {stage.name} stage: {missing_resources}") + stage_resources_template_path: str = os.path.join(CFN_TEMPLATE_PATH, "stage_resources.yaml") + stack_name: str = f"{STACK_NAME_PREFIX}-{stage.stack_name}-{STAGE_RESOURCES_STACK_NAME_SUFFIX}" + with open(stage_resources_template_path, "r") as fp: + stage_resources_template_body = fp.read() + output: List[Dict[str, str]] = manage_cloudformation_stack( + stack_name=stack_name, + region=stage.aws_region, + profile=stage.aws_profile, + template_body=stage_resources_template_body, + parameter_overrides={ + "DeployerArn": deployer_arn, + "DeployerRoleArn": stage.deployer_role.arn, + "CFNDeploymentRoleArn": stage.cfn_deployment_role.arn, + "ArtifactsBucketArn": stage.artifacts_bucket.arn, + }, + ) + + deployer_role_arn: str = next(o for o in output if o.get("OutputKey") == "DeployerRole").get("OutputValue") + cfn_deployment_role_arn: str = next(o for o in output if o.get("OutputKey") == "CFNDeploymentRole").get( + "OutputValue" + ) + artifacts_bucket_arn: str = next(o for o in output if o.get("OutputKey") == "ArtifactsBucket").get( + "OutputValue" + ) + try: + artifacts_bucket_key_arn: str = next(o for o in output if o.get("OutputKey") == "ArtifactsBucketKey").get( + "OutputValue" + ) + except StopIteration: + artifacts_bucket_key_arn = None + + return deployer_role_arn, cfn_deployment_role_arn, artifacts_bucket_arn, artifacts_bucket_key_arn diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/questions.json b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/questions.json new file mode 100644 index 00000000000..75ccc6a9636 --- /dev/null +++ b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/questions.json @@ -0,0 +1,114 @@ +{ + "questions": [ + { + "key": "sam_template", + "question": "SAM template", + "default": "template.yaml" + }, + { + "key": "main_git_branch", + "question": "git branch to trigger the pipeline", + "isRequired": true + }, + { + "key": "testing_intro", + "question": "Let us configure your testing stage first.", + "kind": "info" + }, + { + "key": "testing_profile", + "question": "Which AWS profile will you deploy the 'Testing' stage to?", + "isRequired": true, + "options": %(aws_profiles)s + }, + { + "key": "deployer_provided", + "question": "We need an IAM user which will be used to initiate deployment.", + "isRequired": true, + "options": [ + "I already have IAM user configured in %(provider)s", + "I need help creating one" + ], + "nextQuestion": { + "I already have IAM user configured in %(provider)s": "deployer_arn", + "I need help creating one": "creating_deployer_message" + } + }, + { + "key": "deployer_arn", + "question": "IAM user ARN?", + "isRequired": true, + "defaultNextQuestion": "deployer_aws_access_key_id_variable_name" + }, + { + "key": "creating_deployer_message", + "question": "We are going to create an IAM user for you.", + "kind": "info" + }, + { + "key": "deployer_aws_access_key_id_variable_name", + "question": "%(provider)s variable name for AWS_ACCESS_KEY_ID", + "isRequired": true + }, + { + "key": "deployer_aws_secret_access_key_variable_name", + "question": "%(provider)s variable name for AWS_SECRET_ACCESS_KEY", + "isRequired": true + }, + { + "key": "testing_region", + "question": "Enter region for 'Testing' stage", + "default": "us-east-1" + }, + { + "key": "testing_stack_name", + "question": "Enter stack name for 'Testing' stage", + "default": "testing-stack", + }, + { + "key": "testing_artifacts_bucket", + "question": "We need an S3 bucket in testing account to upload artifacts.\nS3 Bucket ARN [leave blank to create one]" + }, + { + "key": "testing_deployer_role", + "question": "Deployer Execution Role [leave blank to create one]" + }, + { + "key": "testing_cfn_deployment_role", + "question": "Cloudformation Deployment Role [leave blank to create one]" + }, + { + "key": "prod_intro", + "question": "NICE!! Let us configure your prod stage as well.", + "kind": "info" + }, + { + "key": "prod_profile", + "question": "Which AWS profile will you deploy the 'Prod' stage to?", + "options": %(aws_profiles)s, + "isRequired": true + }, + { + "key": "prod_region", + "question": "Enter region for 'Prod' stage", + "default": "us-east-1", + }, + { + "key": "prod_stack_name", + "question": "Enter stack name for 'Prod' stage", + "default": "prod-stack", + }, + { + "key": "prod_artifacts_bucket", + "question": "S3 Bucket ARN [leave blank to create one]" + }, + { + "key": "prod_deployer_role", + "question": "Deployer Execution Role [leave blank to create one]" + }, + { + "key": "prod_cfn_deployment_role", + "question": "Cloudformation Deployment Role [leave blank to create one]" + } + ] +} diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/resource.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/resource.py new file mode 100644 index 00000000000..25e119e6477 --- /dev/null +++ b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/resource.py @@ -0,0 +1,26 @@ +""" AWS resource represented by ARN""" +from typing import Optional + + +class Resource: + def __init__(self, arn: str) -> None: + self.arn: str = arn + self.is_user_provided: bool = bool(arn) + + def name(self) -> Optional[str]: + if self.arn: + return self.arn.split(":")[-1] + return None + + +class Deployer(Resource): + def __init__(self, arn: str, access_key_id: Optional[str] = None, secret_access_key: Optional[str] = None) -> None: + self.access_key_id: Optional[str] = access_key_id + self.secret_access_key: Optional[str] = secret_access_key + super().__init__(arn=arn) + + +class S3Bucket(Resource): + def __init__(self, arn: str, kms_key_arn: Optional[str] = None) -> None: + self.kms_key_arn: Optional[str] = kms_key_arn + super().__init__(arn=arn) diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/stage.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/stage.py new file mode 100644 index 00000000000..bba33956ffb --- /dev/null +++ b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/stage.py @@ -0,0 +1,155 @@ +""" Pipeline stage""" +from .resource import Resource, S3Bucket + + +class Stage: + """ + Represents a pipeline stage + + Attributes + ---------- + name: str + The name of the stage + aws_profile: str + The named AWS profile(in user's machine) of the AWS account to deploy this stage to. + aws_region: str + The AWS region to deploy this stage to. + stack_name: str + The stack-name to be used for deploying the application's CFN template to this stage. + deployer_role: Resource + The IAM role assumed by the pipeline's deployer IAM user to get access to the AWS account and executes the + CloudFormation stack. + cfn_deployment_role: Resource + The IAM role assumed by the CloudFormation service to executes the CloudFormation stack. + artifacts_bucket: S3Bucket + The S3 bucket to hold the SAM build artifacts of the application's CFN template. + + Methods: + did_user_provide_all_required_resources() -> bool: + checks if all of the stage requires resources(deployer_role, cfn_deployment_role, artifacts_bucket) are provided + by the user. + deployer_role_permissions(deployer_arn): + returns a string of the permissions(IAM policies) required for the deployer_role to operate as expected. + cfn_deployment_role_permissions(): + returns a string of the permissions(IAM policies) required for the cfn_deployment_role to operate as expected. + artifacts_bucket_permissions(): + returns a string of the permissions(IAM policies) required for the artifacts_bucket to operate as expected. + + """ + + def __init__( + self, + name: str, + aws_profile: str, + aws_region: str, + stack_name: str, + deployer_role_arn: str, + cfn_deployment_role_arn: str, + artifacts_bucket_arn: str, + ) -> None: + self.name: str = name + self.aws_profile: str = aws_profile + self.aws_region: str = aws_region + self.stack_name: str = stack_name + self.deployer_role: Resource = Resource(arn=deployer_role_arn) + self.cfn_deployment_role: Resource = Resource(arn=cfn_deployment_role_arn) + self.artifacts_bucket: S3Bucket = S3Bucket(arn=artifacts_bucket_arn) + + def did_user_provide_all_required_resources(self) -> bool: + return ( + self.artifacts_bucket.is_user_provided + and self.deployer_role.is_user_provided + and self.cfn_deployment_role.is_user_provided + ) + + def deployer_role_permissions(self, deployer_arn: str) -> str: + permissions: str = f""" +AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + AWS: {deployer_arn} + Action: + - 'sts:AssumeRole' +Policies: + - PolicyName: AccessRolePolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'iam:PassRole' + Resource: + - "{self.cfn_deployment_role.arn}" + - Effect: Allow + Action: + - "cloudformation:CreateChangeSet" + - "cloudformation:DescribeChangeSet" + - "cloudformation:ExecuteChangeSet" + - "cloudformation:DescribeStackEvents" + - "cloudformation:DescribeStacks" + - "cloudformation:GetTemplateSummary" + - "cloudformation:DescribeStackResource" + Resource: '*' + - Effect: Allow + Action: + - 's3:GetObject*' + - 's3:PutObject*' + - 's3:GetBucket*' + - 's3:List*' + Resource: + - {self.artifacts_bucket.arn} + - {self.artifacts_bucket.arn}/* + """ + if not self.artifacts_bucket.is_user_provided: + permissions += f""" + - Effect: Allow + Action: + - "kms:Decrypt" + - "kms:DescribeKey" + Resource: + - {self.artifacts_bucket.kms_key_arn} + """ + return permissions + + def cfn_deployment_role_permissions(self) -> str: + permissions: str = """ +AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: cloudformation.amazonaws.com + Action: + - 'sts:AssumeRole' +Policies: + - PolicyName: GrantCloudFormationFullAccess + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: '*' + Resource: '*' + """ + return permissions + + def artifacts_bucket_permissions(self) -> str: + permissions: str = f""" +PolicyDocument: + Statement: + - Effect: "Allow" + Action: + - 's3:GetObject*' + - 's3:PutObject*' + - 's3:GetBucket*' + - 's3:List*' + Resource: + - {self.artifacts_bucket.arn} + - {self.artifacts_bucket.arn}/* + Principal: + AWS: + - {self.deployer_role.arn} + - {self.cfn_deployment_role.arn} + """ + return permissions diff --git a/samcli/lib/utils/managed_cloudformation_stack.py b/samcli/lib/utils/managed_cloudformation_stack.py index 25973fbc8b5..d85e16c7a0b 100644 --- a/samcli/lib/utils/managed_cloudformation_stack.py +++ b/samcli/lib/utils/managed_cloudformation_stack.py @@ -1,20 +1,17 @@ """ Bootstrap's user's development environment by creating cloud resources required by SAM CLI """ - import logging +from collections.abc import Collection +from typing import cast, Dict, List, Optional, Union import boto3 - import click - from botocore.config import Config from botocore.exceptions import ClientError, BotoCoreError, NoRegionError, NoCredentialsError, ProfileNotFound from samcli.commands.exceptions import UserException, CredentialsError, RegionError - -SAM_CLI_STACK_PREFIX = "aws-sam-cli-managed-" LOG = logging.getLogger(__name__) @@ -25,10 +22,34 @@ def __init__(self, ex): super().__init__(message=message_fmt.format(ex=self.ex)) -def manage_stack(profile, region, stack_name, template_body): +def manage_stack( + region: str, + stack_name: str, + template_body: str, + profile: Optional[str] = None, + parameter_overrides: Optional[Dict[str, Union[str, List[str]]]] = None, +) -> List[Dict[str, str]]: + """ + get or create a CloudFormation stack + + Parameters + ---------- + region: str + AWS region for the CloudFormation stack + stack_name: str + CloudFormation stack name + template_body: str + CloudFormation template's content + profile: Optional[str] + AWS named profile for the AWS account + parameter_overrides: Optional[Dict[str, Union[str, List[str]]]] + Values of template parameters, if any. + + Returns: Stack output section(list of OutputKey, OutputValue pairs) + """ try: if profile: - session = boto3.Session(profile_name=profile, region_name=region if region else None) + session = boto3.Session(profile_name=profile, region_name=region if region else None) # type: ignore cloudformation_client = session.client("cloudformation") else: cloudformation_client = boto3.client( @@ -51,32 +72,41 @@ def manage_stack(profile, region, stack_name, template_body): "Error Setting Up Managed Stack Client: Unable to resolve a region. " "Please provide a region via the --region parameter or by the AWS_REGION environment variable." ) from ex - return _create_or_get_stack(cloudformation_client, stack_name, template_body) + return _create_or_get_stack(cloudformation_client, stack_name, template_body, parameter_overrides) -def _create_or_get_stack(cloudformation_client, stack_name, template_body): +# Todo Add _update_stack to handle the case when the values of the stack parameter got changed +def _create_or_get_stack( + cloudformation_client, + stack_name: str, + template_body: str, + parameter_overrides: Optional[Dict[str, Union[str, List[str]]]] = None, +) -> List[Dict[str, str]]: try: ds_resp = cloudformation_client.describe_stacks(StackName=stack_name) stacks = ds_resp["Stacks"] stack = stacks[0] click.echo("\n\tLooking for resources needed for deployment: Found!") - _check_sanity_of_stack(stack, stack_name) - return stack["Outputs"] + _check_sanity_of_stack(stack) + stack_outputs = cast(List[Dict[str, str]], stack["Outputs"]) + return stack_outputs except ClientError: click.echo("\n\tLooking for resources needed for deployment: Not found.") try: stack = _create_stack( - cloudformation_client, stack_name, template_body + cloudformation_client, stack_name, template_body, parameter_overrides ) # exceptions are not captured from subcommands - _check_sanity_of_stack(stack, stack_name) - return stack["Outputs"] + _check_sanity_of_stack(stack) + stack_outputs = cast(List[Dict[str, str]], stack["Outputs"]) + return stack_outputs except (ClientError, BotoCoreError) as ex: LOG.debug("Failed to create managed resources", exc_info=ex) raise ManagedStackError(str(ex)) from ex -def _check_sanity_of_stack(stack, stack_name): +def _check_sanity_of_stack(stack): + stack_name = stack.get("StackName") tags = stack.get("Tags", None) outputs = stack.get("Outputs", None) @@ -112,15 +142,23 @@ def _check_sanity_of_stack(stack, stack_name): raise UserException(msg) from ex -def _create_stack(cloudformation_client, stack_name, template_body): +def _create_stack( + cloudformation_client, + stack_name: str, + template_body: str, + parameter_overrides: Optional[Dict[str, Union[str, List[str]]]] = None, +): click.echo("\tCreating the required resources...") change_set_name = "InitialCreation" + parameters = _generate_stack_parameters(parameter_overrides) change_set_resp = cloudformation_client.create_change_set( StackName=stack_name, TemplateBody=template_body, Tags=[{"Key": "ManagedStackSource", "Value": "AwsSamCli"}], ChangeSetType="CREATE", ChangeSetName=change_set_name, # this must be unique for the stack, but we only create so that's fine + Capabilities=["CAPABILITY_IAM"], + Parameters=parameters, ) stack_id = change_set_resp["StackId"] change_waiter = cloudformation_client.get_waiter("change_set_create_complete") @@ -134,3 +172,15 @@ def _create_stack(cloudformation_client, stack_name, template_body): stacks = ds_resp["Stacks"] click.echo("\tSuccessfully created!") return stacks[0] + + +def _generate_stack_parameters( + parameter_overrides: Optional[Dict[str, Union[str, List[str]]]] = None +) -> List[Dict[str, str]]: + parameters = [] + if parameter_overrides: + for key, value in parameter_overrides.items(): + if isinstance(value, Collection) and not isinstance(value, str): + value = ",".join(value) + parameters.append({"ParameterKey": key, "ParameterValue": value}) + return parameters diff --git a/samcli/local/common/runtime_template.py b/samcli/local/common/runtime_template.py index 2924a5aa9f8..8c30e48e6a9 100644 --- a/samcli/local/common/runtime_template.py +++ b/samcli/local/common/runtime_template.py @@ -96,6 +96,21 @@ def get_local_lambda_images_location(mapping, runtime): "java8.al2": ["maven", "gradle"], } +RUNTIME_TO_BUILD_IMAGE = { + "python3.8": "public.ecr.aws/sam/build-python3.8", + "python3.7": "public.ecr.aws/sam/build-python3.7", + "python3.6": "public.ecr.aws/sam/build-python3.6", + "python2.7": "public.ecr.aws/sam/build-python2.7", + "ruby2.5": "public.ecr.aws/sam/build-ruby2.5", + "ruby2.7": "public.ecr.aws/sam/build-ruby2.7", + "nodejs14.x": "public.ecr.aws/sam/build-nodejs14.x", + "nodejs12.x": "public.ecr.aws/sam/build-nodejs12.x", + "nodejs10.x": "public.ecr.aws/sam/build-nodejs10.x", + "java8": "public.ecr.aws/sam/build-java8", + "java11": "public.ecr.aws/sam/build-java11", + "java8.al2": "public.ecr.aws/sam/build-java8.al2", +} + SUPPORTED_DEP_MANAGERS: Set[str] = { c["dependency_manager"] # type: ignore for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))) diff --git a/tests/unit/commands/_utils/test_template.py b/tests/unit/commands/_utils/test_template.py index be4001be684..55330531ed6 100644 --- a/tests/unit/commands/_utils/test_template.py +++ b/tests/unit/commands/_utils/test_template.py @@ -21,6 +21,7 @@ TemplateFailedParsingException, get_template_artifacts_format, get_template_function_resource_ids, + get_template_function_runtimes, ) from samcli.lib.utils.packagetype import IMAGE, ZIP @@ -336,3 +337,19 @@ def test_get_template_function_resouce_ids(self, mock_get_template_data): } } self.assertEqual(get_template_function_resource_ids(MagicMock(), IMAGE), ["HelloWorldFunction1"]) + + +class Test_get_template_function_runtimes(TestCase): + @patch("samcli.commands._utils.template.get_template_data") + def test_get_template_function_runtimes(self, mock_get_template_data): + mock_get_template_data.return_value = { + "Resources": { + "HelloWorldFunction1": {"Type": "AWS::Lambda::Function", "Properties": {"PackageType": "Image"}}, + "HelloWorldFunction2": {"Type": "AWS::Lambda::Function", "Properties": {"Runtime": "python3.8"}}, + "HelloWorldFunction3": {"Type": "AWS::Serverless::Function", "Properties": {"Runtime": "python3.8"}}, + "HelloWorldFunction4": {"Type": "AWS::Serverless::Function", "Properties": {"Runtime": "python3.8"}}, + "HelloWorldFunction5": {"Type": "AWS::Serverless::Function", "Properties": {"Key": "value"}}, + "HelloWorldApi": {"Type": "AWS::Serverless::Api", "Properties": {"Runtime": "Java8"}}, + } + } + self.assertEqual(get_template_function_runtimes(MagicMock()), ["python3.8"]) diff --git a/tests/unit/lib/pipeline/__init__.py b/tests/unit/lib/pipeline/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/lib/pipeline/init/__init__.py b/tests/unit/lib/pipeline/init/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/lib/pipeline/init/plugins/__init__.py b/tests/unit/lib/pipeline/init/plugins/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_context.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_context.py new file mode 100644 index 00000000000..5fadc296e63 --- /dev/null +++ b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_context.py @@ -0,0 +1,161 @@ +from unittest import TestCase +from samcli.lib.pipeline.init.plugins.two_stages_pipeline.context import Context # type: ignore + + +ANY_TESTING_PROFILE = "ANY_TESTING_PROFILE" +ANY_TESTING_REGION = "ANY_TESTING_REGION" +ANY_TESTING_STACK_NAME = "ANY_TESTING_STACK_NAME" +ANY_TESTING_DEPLOYER_ROLE = "ANY:TESTING:DEPLOYER_ROLE" +ANY_TESTING_CFN_DEPLOYMENT_ROLE = "ANY:TESTING:CFN_DEPLOYMENT_ROLE" +ANY_TESTING_ARTIFACTS_BUCKET = "ANY:TESTING:ARTIFACTS_BUCKET" +ANY_PROD_PROFILE = "ANY_PROD_PROFILE" +ANY_PROD_REGION = "ANY_PROD_REGION" +ANY_PROD_STACK_NAME = "ANY_PROD_STACK_NAME" +ANY_PROD_DEPLOYER_ROLE = "ANY:PROD:DEPLOYER_ROLE" +ANY_PROD_CFN_DEPLOYMENT_ROLE = "ANY:PROD:CFN_DEPLOYMENT_ROLE" +ANY_PROD_ARTIFACTS_BUCKET = "ANY:PROD:ARTIFACTS_BUCKET" +ANY_DEPLOYER_ARN = "ANY:DEPLOYER_ARN" +ANY_DEPLOYER_AWS_ACCESS_KEY_ID_VARIABLE_NAME = "ANY_DEPLOYER_AWS_ACCESS_KEY_ID_VARIABLE_NAME" +ANY_DEPLOYER_AWS_SECRET_ACCESS_KEY_VARIABLE_NAME = "ANY_DEPLOYER_AWS_SECRET_ACCESS_KEY_VARIABLE_NAME" + +ANY_CONTEXT = { + "testing_profile": ANY_TESTING_PROFILE, + "testing_region": ANY_TESTING_REGION, + "testing_stack_name": ANY_TESTING_STACK_NAME, + "testing_deployer_role": ANY_TESTING_DEPLOYER_ROLE, + "testing_cfn_deployment_role": ANY_TESTING_CFN_DEPLOYMENT_ROLE, + "testing_artifacts_bucket": ANY_TESTING_ARTIFACTS_BUCKET, + "prod_profile": ANY_PROD_PROFILE, + "prod_region": ANY_PROD_REGION, + "prod_stack_name": ANY_PROD_STACK_NAME, + "prod_deployer_role": ANY_PROD_DEPLOYER_ROLE, + "prod_cfn_deployment_role": ANY_PROD_CFN_DEPLOYMENT_ROLE, + "prod_artifacts_bucket": ANY_PROD_ARTIFACTS_BUCKET, + "deployer_arn": ANY_DEPLOYER_ARN, + "deployer_aws_access_key_id_variable_name": ANY_DEPLOYER_AWS_ACCESS_KEY_ID_VARIABLE_NAME, + "deployer_aws_secret_access_key_variable_name": ANY_DEPLOYER_AWS_SECRET_ACCESS_KEY_VARIABLE_NAME, +} + + +class TestContext(TestCase): + def test_init_with_all_keys_provided(self): + context: Context = Context(ANY_CONTEXT) + testing_stage = context.get_stage(Context.TESTING_STAGE_NAME) + prod_stage = context.get_stage(Context.PROD_STAGE_NAME) + + self.assertEqual(testing_stage.name, Context.TESTING_STAGE_NAME) + self.assertEqual(testing_stage.aws_profile, ANY_TESTING_PROFILE) + self.assertEqual(testing_stage.aws_region, ANY_TESTING_REGION) + self.assertEqual(testing_stage.stack_name, ANY_TESTING_STACK_NAME) + self.assertEqual(testing_stage.deployer_role.arn, ANY_TESTING_DEPLOYER_ROLE) + self.assertEqual(testing_stage.deployer_role.name(), "DEPLOYER_ROLE") + self.assertTrue(testing_stage.deployer_role.is_user_provided) + self.assertEqual(testing_stage.cfn_deployment_role.arn, ANY_TESTING_CFN_DEPLOYMENT_ROLE) + self.assertEqual(testing_stage.cfn_deployment_role.name(), "CFN_DEPLOYMENT_ROLE") + self.assertTrue(testing_stage.cfn_deployment_role.is_user_provided) + self.assertEqual(testing_stage.artifacts_bucket.arn, ANY_TESTING_ARTIFACTS_BUCKET) + self.assertEqual(testing_stage.artifacts_bucket.name(), "ARTIFACTS_BUCKET") + self.assertTrue(testing_stage.artifacts_bucket.is_user_provided) + self.assertIsNone(testing_stage.artifacts_bucket.kms_key_arn) + + self.assertEqual(prod_stage.name, Context.PROD_STAGE_NAME) + self.assertEqual(prod_stage.aws_profile, ANY_PROD_PROFILE) + self.assertEqual(prod_stage.aws_region, ANY_PROD_REGION) + self.assertEqual(prod_stage.stack_name, ANY_PROD_STACK_NAME) + self.assertEqual(prod_stage.deployer_role.arn, ANY_PROD_DEPLOYER_ROLE) + self.assertEqual(prod_stage.deployer_role.name(), "DEPLOYER_ROLE") + self.assertTrue(prod_stage.deployer_role.is_user_provided) + self.assertEqual(prod_stage.cfn_deployment_role.arn, ANY_PROD_CFN_DEPLOYMENT_ROLE) + self.assertEqual(prod_stage.cfn_deployment_role.name(), "CFN_DEPLOYMENT_ROLE") + self.assertTrue(prod_stage.cfn_deployment_role.is_user_provided) + self.assertEqual(prod_stage.artifacts_bucket.arn, ANY_PROD_ARTIFACTS_BUCKET) + self.assertEqual(prod_stage.artifacts_bucket.name(), "ARTIFACTS_BUCKET") + self.assertTrue(prod_stage.artifacts_bucket.is_user_provided) + self.assertIsNone(prod_stage.artifacts_bucket.kms_key_arn) + + self.assertEqual(context.deployer.arn, ANY_DEPLOYER_ARN) + self.assertEqual(context.deployer.name(), "DEPLOYER_ARN") + self.assertTrue(context.deployer.is_user_provided) + self.assertIsNone(context.deployer.access_key_id) + self.assertIsNone(context.deployer.secret_access_key) + + self.assertEqual(context.deployer_aws_access_key_id_variable_name, ANY_DEPLOYER_AWS_ACCESS_KEY_ID_VARIABLE_NAME) + self.assertEqual( + context.deployer_aws_secret_access_key_variable_name, ANY_DEPLOYER_AWS_SECRET_ACCESS_KEY_VARIABLE_NAME + ) + self.assertIsNone(context.build_image) + + def test_init_with_no_keys_provided(self): + context: Context = Context({}) + testing_stage = context.get_stage(Context.TESTING_STAGE_NAME) + prod_stage = context.get_stage(Context.PROD_STAGE_NAME) + + self.assertEqual(testing_stage.name, Context.TESTING_STAGE_NAME) + self.assertIsNone(testing_stage.aws_profile) + self.assertIsNone(testing_stage.aws_region) + self.assertIsNone(testing_stage.stack_name) + self.assertIsNone(testing_stage.deployer_role.arn) + self.assertIsNone(testing_stage.deployer_role.name()) + self.assertFalse(testing_stage.deployer_role.is_user_provided) + self.assertIsNone(testing_stage.cfn_deployment_role.arn) + self.assertIsNone(testing_stage.cfn_deployment_role.name()) + self.assertFalse(testing_stage.cfn_deployment_role.is_user_provided) + self.assertIsNone(testing_stage.artifacts_bucket.arn) + self.assertIsNone(testing_stage.artifacts_bucket.name()) + self.assertFalse(testing_stage.artifacts_bucket.is_user_provided) + self.assertIsNone(testing_stage.artifacts_bucket.kms_key_arn) + + self.assertEqual(prod_stage.name, Context.PROD_STAGE_NAME) + self.assertIsNone(prod_stage.aws_profile) + self.assertIsNone(prod_stage.aws_region) + self.assertIsNone(prod_stage.stack_name) + self.assertIsNone(prod_stage.deployer_role.arn) + self.assertIsNone(prod_stage.deployer_role.name()) + self.assertFalse(prod_stage.deployer_role.is_user_provided) + self.assertIsNone(prod_stage.cfn_deployment_role.arn) + self.assertIsNone(prod_stage.cfn_deployment_role.name()) + self.assertFalse(prod_stage.cfn_deployment_role.is_user_provided) + self.assertIsNone(prod_stage.artifacts_bucket.arn) + self.assertIsNone(prod_stage.artifacts_bucket.name()) + self.assertFalse(prod_stage.artifacts_bucket.is_user_provided) + self.assertIsNone(prod_stage.artifacts_bucket.kms_key_arn) + + self.assertIsNone(context.deployer.arn) + self.assertIsNone(context.deployer.name()) + self.assertFalse(context.deployer.is_user_provided) + self.assertIsNone(context.deployer.access_key_id) + self.assertIsNone(context.deployer.secret_access_key) + + self.assertIsNone(context.deployer_aws_access_key_id_variable_name) + self.assertIsNone(context.deployer_aws_secret_access_key_variable_name) + self.assertIsNone(context.build_image) + + def test_init_with_none_context_raises_exception(self): + with self.assertRaises(AttributeError): + context: Context = Context(None) + + def test_get_stage(self): + context: Context = Context({}) + testing_stage = context.get_stage(Context.TESTING_STAGE_NAME) + prod_stage = context.get_stage(Context.PROD_STAGE_NAME) + none_stage1 = context.get_stage("non-existing-stage") + none_stage2 = context.get_stage(None) + self.assertEqual(testing_stage.name, Context.TESTING_STAGE_NAME) + self.assertEqual(prod_stage.name, Context.PROD_STAGE_NAME) + self.assertIsNone(none_stage1) + self.assertIsNone(none_stage2) + + def test_deployer_permissions(self): + context: Context = Context(ANY_CONTEXT) + permissions: str = context.deployer_permissions() + # assert the deployer has assume-rule access to the deployer roles + self.assertIn("sts:AssumeRole", permissions) + self.assertIn(ANY_TESTING_DEPLOYER_ROLE, permissions) + self.assertIn(ANY_PROD_DEPLOYER_ROLE, permissions) + + def test_deployer_permissions_with_empty_context(self): + context: Context = Context({}) + permissions: str = context.deployer_permissions() + self.assertIn("sts:AssumeRole", permissions) + self.assertNotIn(ANY_TESTING_DEPLOYER_ROLE, permissions) + self.assertNotIn(ANY_PROD_DEPLOYER_ROLE, permissions) diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py new file mode 100644 index 00000000000..388cb64f863 --- /dev/null +++ b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py @@ -0,0 +1,101 @@ +from unittest import TestCase +from unittest.mock import Mock +from samcli.lib.pipeline.init.plugins.two_stages_pipeline.config import PLUGIN_NAME # type: ignore +from samcli.lib.pipeline.init.plugins.two_stages_pipeline.context import Context as PluginContext # type: ignore +from samcli.lib.pipeline.init.plugins.two_stages_pipeline.postprocessor import Postprocessor # type: ignore +from .test_context import ( + ANY_DEPLOYER_ARN, + ANY_TESTING_DEPLOYER_ROLE, + ANY_TESTING_CFN_DEPLOYMENT_ROLE, + ANY_TESTING_ARTIFACTS_BUCKET, + ANY_PROD_DEPLOYER_ROLE, + ANY_PROD_CFN_DEPLOYMENT_ROLE, + ANY_PROD_ARTIFACTS_BUCKET, +) + + +class TestPostprocessor(TestCase): + def test_the_postprocessoer_correctly_categorize_the_resources_into_reused_and_created_resources(self): + # setup + deployer = Mock() + deployer.is_user_provided = False + deployer.arn = ANY_DEPLOYER_ARN + + testing_deployer_role = Mock() + testing_deployer_role.is_user_provided = True + testing_deployer_role.arn = ANY_TESTING_DEPLOYER_ROLE + + testing_cfn_deployment_role = Mock() + testing_cfn_deployment_role.is_user_provided = True + testing_cfn_deployment_role.arn = ANY_TESTING_CFN_DEPLOYMENT_ROLE + + testing_artifacts_bucket = Mock() + testing_artifacts_bucket.is_user_provided = True + testing_artifacts_bucket.arn = ANY_TESTING_ARTIFACTS_BUCKET + + testing_stage = Mock() + testing_stage.deployer_role = testing_deployer_role + testing_stage.cfn_deployment_role = testing_cfn_deployment_role + testing_stage.artifacts_bucket = testing_artifacts_bucket + + prod_deployer_role = Mock() + prod_deployer_role.is_user_provided = False + prod_deployer_role.arn = ANY_PROD_DEPLOYER_ROLE + + prod_cfn_deployment_role = Mock() + prod_cfn_deployment_role.is_user_provided = False + prod_cfn_deployment_role.arn = ANY_PROD_CFN_DEPLOYMENT_ROLE + + prod_artifacts_bucket = Mock() + prod_artifacts_bucket.is_user_provided = False + prod_artifacts_bucket.arn = ANY_PROD_ARTIFACTS_BUCKET + + prod_stage = Mock() + prod_stage.deployer_role = prod_deployer_role + prod_stage.cfn_deployment_role = prod_cfn_deployment_role + prod_stage.artifacts_bucket = prod_artifacts_bucket + + plugin_context = Mock() + plugin_context.deployer = deployer + plugin_context.stages = [testing_stage, prod_stage] + + context = {PLUGIN_NAME: plugin_context} + postprocessor = Postprocessor() + + self.assertEqual(0, len(postprocessor.resources_reused)) + self.assertEqual(0, len(postprocessor.resources_created)) + + # trigger + mutated_context = postprocessor.run(context=context) + + # verify + reused_resources_arns = list(map(lambda r: r["arn"], postprocessor.resources_reused)) + self.assertEqual(3, len(reused_resources_arns)) + self.assertIn(ANY_TESTING_DEPLOYER_ROLE, reused_resources_arns) + self.assertIn(ANY_TESTING_CFN_DEPLOYMENT_ROLE, reused_resources_arns) + self.assertIn(ANY_TESTING_ARTIFACTS_BUCKET, reused_resources_arns) + + created_resources_arns = list(map(lambda r: r["arn"], postprocessor.resources_created)) + self.assertEqual(4, len(created_resources_arns)) + self.assertIn(ANY_DEPLOYER_ARN, created_resources_arns) + self.assertIn(ANY_PROD_DEPLOYER_ROLE, created_resources_arns) + self.assertIn(ANY_PROD_CFN_DEPLOYMENT_ROLE, created_resources_arns) + self.assertIn(ANY_PROD_ARTIFACTS_BUCKET, created_resources_arns) + + def test_the_plugin_context_is_required(self): + postprocessor: Postprocessor = Postprocessor() + with (self.assertRaises(KeyError)): + postprocessor.run(context={}) + + def test_the_postprocessoer_return_a_new_copy_of_the_context_without_modifications(self): + # setup + plugin_context = PluginContext({}) + context = {PLUGIN_NAME: plugin_context} + postprocessor: Postprocessor = Postprocessor() + + # trigger + mutated_context = postprocessor.run(context=context) + + # verify + self.assertIsNot(context, mutated_context) + self.assertEqual(context, mutated_context) diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_preprocessor.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_preprocessor.py new file mode 100644 index 00000000000..8dbc1acc8da --- /dev/null +++ b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_preprocessor.py @@ -0,0 +1,440 @@ +from unittest import TestCase +from unittest.mock import ANY, Mock, patch +from samcli.local.common.runtime_template import RUNTIME_TO_BUILD_IMAGE +from samcli.lib.pipeline.init.plugins.two_stages_pipeline.config import PLUGIN_NAME # type: ignore +from samcli.lib.pipeline.init.plugins.two_stages_pipeline.context import Context # type: ignore +from samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor import Preprocessor # type: ignore +from samcli.lib.pipeline.init.plugins.two_stages_pipeline.stage import Stage # type: ignore +from .test_context import ( + ANY_TESTING_DEPLOYER_ROLE, + ANY_TESTING_CFN_DEPLOYMENT_ROLE, + ANY_TESTING_ARTIFACTS_BUCKET, + ANY_DEPLOYER_ARN, +) + +# Let the user provide the deployer and the testing-stage resources and the plugin creates the prod-resources +plugin_context_with_deployer_and_testing_resources_but_no_prod_resources = { + "testing_deployer_role": ANY_TESTING_DEPLOYER_ROLE, + "testing_cfn_deployment_role": ANY_TESTING_CFN_DEPLOYMENT_ROLE, + "testing_artifacts_bucket": ANY_TESTING_ARTIFACTS_BUCKET, + "deployer_arn": ANY_DEPLOYER_ARN, +} +ANY_SAM_TEMPLATE = "any/sam/template.yaml" +context = {"sam_template": ANY_SAM_TEMPLATE} +context.update(plugin_context_with_deployer_and_testing_resources_but_no_prod_resources) +preprocessor: Preprocessor = Preprocessor() + + +class TestPreprocessor(TestCase): + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.get_template_function_runtimes") + def test_used_build_image_when_sam_template_has_no_runtimes(self, get_template_function_runtimes_mock): + # setup + get_template_function_runtimes_mock.return_value = [] + # trigger + build_image = preprocessor._get_build_image(sam_template_file=ANY_SAM_TEMPLATE) + # verify + self.assertEqual(build_image, Preprocessor.BASIC_PROVIDED_BUILD_IMAGE) + + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.get_template_function_runtimes") + def test_used_build_image_when_sam_template_has_one_supported_runtime(self, get_template_function_runtimes_mock): + # setup + get_template_function_runtimes_mock.return_value = ["python3.8"] + # trigger + build_image = preprocessor._get_build_image(sam_template_file=ANY_SAM_TEMPLATE) + # verify + self.assertEqual(build_image, RUNTIME_TO_BUILD_IMAGE["python3.8"]) + + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.get_template_function_runtimes") + def test_used_build_image_when_sam_template_has_one_unsupported_runtime( + self, get_template_function_runtimes_mock, click_mock + ): + # setup + click_mock.prompt.return_value = "user-provided-build-image" + get_template_function_runtimes_mock.return_value = ["any-unsupported-runtime"] + # trigger + build_image = preprocessor._get_build_image(sam_template_file=ANY_SAM_TEMPLATE) + # verify + click_mock.prompt.assert_called_once() + self.assertEqual(build_image, "user-provided-build-image") + + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.get_template_function_runtimes") + def test_used_build_image_when_sam_template_has_multiple_runtimes( + self, get_template_function_runtimes_mock, click_mock + ): + # setup + click_mock.prompt.return_value = "user-provided-build-image" + get_template_function_runtimes_mock.return_value = ["python3.8", "python3.7", "any-unsupported-runtime"] + # trigger + build_image = preprocessor._get_build_image(sam_template_file=ANY_SAM_TEMPLATE) + # verify + click_mock.prompt.assert_called_once() + self.assertEqual(build_image, "user-provided-build-image") + + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") + def test_create_deployer(self, manage_cloudformation_stack_mock, click_mock): + # setup + stage_mock = Mock() + stage_mock.profile.return_value = "any-profile" + stage_mock.region.return_value = "any-region" + stage_mock.stack_name.return_value = "any-stack-name" + manage_cloudformation_stack_mock.return_value = [ + {"OutputKey": "Deployer", "OutputValue": "any-deployer-arn"}, + {"OutputKey": "AccessKeyId", "OutputValue": "any-access-key-id"}, + {"OutputKey": "SecretAccessKey", "OutputValue": "any-secret-access-key"}, + ] + + # trigger + ( + deployer_arn, + access_key_id_arn, + secret_access_key_arn, + ) = preprocessor._create_deployer_at(stage=stage_mock) + + # verify + self.assertEqual(deployer_arn, "any-deployer-arn") + self.assertEqual(access_key_id_arn, "any-access-key-id") + self.assertEqual(secret_access_key_arn, "any-secret-access-key") + + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") + def test_create_missing_stage_resources_when_all_resources_are_missed( + self, manage_cloudformation_stack_mock, click_mock + ): + # setup + stage = Stage( + name="any-name", + aws_profile="any-profile", + aws_region="any-region", + stack_name="any-stack-name", + deployer_role_arn=None, + cfn_deployment_role_arn=None, + artifacts_bucket_arn=None, + ) + + manage_cloudformation_stack_mock.return_value = [ + {"OutputKey": "DeployerRole", "OutputValue": "new-deployer-role-arn"}, + {"OutputKey": "CFNDeploymentRole", "OutputValue": "new-cfn-deployment-role-arn"}, + {"OutputKey": "ArtifactsBucket", "OutputValue": "new-artifacts-bucket-arn"}, + {"OutputKey": "ArtifactsBucketKey", "OutputValue": "new-artifacts-bucket-key-arn"}, + ] + + # trigger + ( + deployer_role_arn, + cfn_deployment_role_arn, + artifacts_bucket_arn, + artifacts_bucket_key_arn, + ) = preprocessor._create_missing_stage_resources(stage=stage, deployer_arn="any-deployer-arn") + + # verify + self.assertEqual(deployer_role_arn, "new-deployer-role-arn") + self.assertEqual(cfn_deployment_role_arn, "new-cfn-deployment-role-arn") + self.assertEqual(artifacts_bucket_arn, "new-artifacts-bucket-arn") + self.assertEqual(artifacts_bucket_key_arn, "new-artifacts-bucket-key-arn") + manage_cloudformation_stack_mock.assert_called_once_with( + stack_name="aws-sam-cli-managed-any-stack-name-pipeline-resources", + region="any-region", + profile="any-profile", + template_body=ANY, + parameter_overrides={ + "DeployerArn": "any-deployer-arn", + "DeployerRoleArn": None, + "CFNDeploymentRoleArn": None, + "ArtifactsBucketArn": None, + }, + ) + + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") + def test_create_missing_stage_resources_when_only_deployer_role_is_missed( + self, manage_cloudformation_stack_mock, click_mock + ): + # setup + stage = Stage( + name="any-name", + aws_profile="any-profile", + aws_region="any-region", + stack_name="any-stack-name", + deployer_role_arn=None, + cfn_deployment_role_arn="existing-cfn-deployment-role-arn", + artifacts_bucket_arn="existing-artifacts-bucket-arn", + ) + + manage_cloudformation_stack_mock.return_value = [ + {"OutputKey": "DeployerRole", "OutputValue": "new-deployer-role-arn"}, + {"OutputKey": "CFNDeploymentRole", "OutputValue": "existing-cfn-deployment-role-arn"}, + {"OutputKey": "ArtifactsBucket", "OutputValue": "existing-artifacts-bucket-arn"}, + ] + + # trigger + ( + deployer_role_arn, + cfn_deployment_role_arn, + artifacts_bucket_arn, + artifacts_bucket_key_arn, + ) = preprocessor._create_missing_stage_resources(stage=stage, deployer_arn="any-deployer-arn") + + # verify + self.assertEqual(deployer_role_arn, "new-deployer-role-arn") + self.assertEqual(cfn_deployment_role_arn, "existing-cfn-deployment-role-arn") + self.assertEqual(artifacts_bucket_arn, "existing-artifacts-bucket-arn") + # if we didn't create the artifacts bucket then we don't know about the user's bucket encryption method + self.assertIsNone(artifacts_bucket_key_arn) + manage_cloudformation_stack_mock.assert_called_once_with( + stack_name="aws-sam-cli-managed-any-stack-name-pipeline-resources", + region="any-region", + profile="any-profile", + template_body=ANY, + parameter_overrides={ + "DeployerArn": "any-deployer-arn", + "DeployerRoleArn": None, + "CFNDeploymentRoleArn": "existing-cfn-deployment-role-arn", + "ArtifactsBucketArn": "existing-artifacts-bucket-arn", + }, + ) + + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") + def test_create_missing_stage_resources_when_only_cfn_deployment_role_is_missed( + self, manage_cloudformation_stack_mock, click_mock + ): + # setup + stage = Stage( + name="any-name", + aws_profile="any-profile", + aws_region="any-region", + stack_name="any-stack-name", + deployer_role_arn="existing-deployer-role-arn", + cfn_deployment_role_arn=None, + artifacts_bucket_arn="existing-artifacts-bucket-arn", + ) + + manage_cloudformation_stack_mock.return_value = [ + {"OutputKey": "DeployerRole", "OutputValue": "existing-deployer-role-arn"}, + {"OutputKey": "CFNDeploymentRole", "OutputValue": "new-cfn-deployment-role-arn"}, + {"OutputKey": "ArtifactsBucket", "OutputValue": "existing-artifacts-bucket-arn"}, + ] + + # trigger + ( + deployer_role_arn, + cfn_deployment_role_arn, + artifacts_bucket_arn, + artifacts_bucket_key_arn, + ) = preprocessor._create_missing_stage_resources(stage=stage, deployer_arn="any-deployer-arn") + + # verify + self.assertEqual(deployer_role_arn, "existing-deployer-role-arn") + self.assertEqual(cfn_deployment_role_arn, "new-cfn-deployment-role-arn") + self.assertEqual(artifacts_bucket_arn, "existing-artifacts-bucket-arn") + # if we didn't create the artifacts bucket then we don't know about the user's bucket encryption method + self.assertIsNone(artifacts_bucket_key_arn) + manage_cloudformation_stack_mock.assert_called_once_with( + stack_name="aws-sam-cli-managed-any-stack-name-pipeline-resources", + region="any-region", + profile="any-profile", + template_body=ANY, + parameter_overrides={ + "DeployerArn": "any-deployer-arn", + "DeployerRoleArn": "existing-deployer-role-arn", + "CFNDeploymentRoleArn": None, + "ArtifactsBucketArn": "existing-artifacts-bucket-arn", + }, + ) + + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") + def test_create_missing_stage_resources_when_only_artifacts_bucket_is_missed( + self, manage_cloudformation_stack_mock, click_mock + ): + # setup + stage = Stage( + name="any-name", + aws_profile="any-profile", + aws_region="any-region", + stack_name="any-stack-name", + deployer_role_arn="existing-deployer-role-arn", + cfn_deployment_role_arn="existing-cfn-deployment-role-arn", + artifacts_bucket_arn=None, + ) + + manage_cloudformation_stack_mock.return_value = [ + {"OutputKey": "DeployerRole", "OutputValue": "existing-deployer-role-arn"}, + {"OutputKey": "CFNDeploymentRole", "OutputValue": "existing-cfn-deployment-role-arn"}, + {"OutputKey": "ArtifactsBucket", "OutputValue": "new-artifacts-bucket-arn"}, + {"OutputKey": "ArtifactsBucketKey", "OutputValue": "new-artifacts-bucket-key-arn"}, + ] + + # trigger + ( + deployer_role_arn, + cfn_deployment_role_arn, + artifacts_bucket_arn, + artifacts_bucket_key_arn, + ) = preprocessor._create_missing_stage_resources(stage=stage, deployer_arn="any-deployer-arn") + + # verify + self.assertEqual(deployer_role_arn, "existing-deployer-role-arn") + self.assertEqual(cfn_deployment_role_arn, "existing-cfn-deployment-role-arn") + self.assertEqual(artifacts_bucket_arn, "new-artifacts-bucket-arn") + self.assertEqual(artifacts_bucket_key_arn, "new-artifacts-bucket-key-arn") + manage_cloudformation_stack_mock.assert_called_once_with( + stack_name="aws-sam-cli-managed-any-stack-name-pipeline-resources", + region="any-region", + profile="any-profile", + template_body=ANY, + parameter_overrides={ + "DeployerArn": "any-deployer-arn", + "DeployerRoleArn": "existing-deployer-role-arn", + "CFNDeploymentRoleArn": "existing-cfn-deployment-role-arn", + "ArtifactsBucketArn": None, + }, + ) + + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") + def test_create_missing_stage_resources_when_all_resources_are_provided( + self, manage_cloudformation_stack_mock, click_mock + ): + # setup + stage = Stage( + name="any-name", + aws_profile="any-profile", + aws_region="any-region", + stack_name="any-stack-name", + deployer_role_arn="existing-deployer-role-arn", + cfn_deployment_role_arn="existing-cfn-deployment-role-arn", + artifacts_bucket_arn="existing-artifacts-bucket-arn", + ) + + # trigger + ( + deployer_role_arn, + cfn_deployment_role_arn, + artifacts_bucket_arn, + artifacts_bucket_key_arn, + ) = preprocessor._create_missing_stage_resources(stage=stage, deployer_arn="any-deployer-arn") + + # verify + manage_cloudformation_stack_mock.assert_not_called() + self.assertEqual(deployer_role_arn, "existing-deployer-role-arn") + self.assertEqual(cfn_deployment_role_arn, "existing-cfn-deployment-role-arn") + self.assertEqual(artifacts_bucket_arn, "existing-artifacts-bucket-arn") + # if we didn't create the artifacts bucket then we don't know about the user's bucket encryption method + self.assertIsNone(artifacts_bucket_key_arn) + + @patch.object(Preprocessor, "_get_build_image") + @patch.object(Preprocessor, "_create_deployer_at") + @patch.object(Preprocessor, "_create_missing_stage_resources") + def test_run_creates_the_missing_deployer_at_the_testing_stage_aws_account( + self, create_missing_stage_resources_mock, create_deployer_at_mock, get_build_image_mock + ): + # setup + context_without_deployer = context.copy() + del context_without_deployer["deployer_arn"] + create_deployer_at_mock.return_value = ["ANY", "ANY", "ANY"] + create_missing_stage_resources_mock.return_value = ["ANY", "ANY", "ANY", "ANY"] + + # trigger + preprocessor.run(context=context_without_deployer) + + # verify + create_deployer_at_mock.assert_called_once() + _, kwargs = create_deployer_at_mock.call_args + stage = kwargs["stage"] + self.assertEqual(Context.TESTING_STAGE_NAME, stage.name) + + @patch.object(Preprocessor, "_get_build_image") + @patch.object(Preprocessor, "_create_deployer_at") + @patch.object(Preprocessor, "_create_missing_stage_resources") + def test_run_will_not_create_deployer_if_already_provided( + self, create_missing_stage_resources_mock, create_deployer_at_mock, get_build_image_mock + ): + # setup + create_deployer_at_mock.return_value = ["ANY", "ANY", "ANY"] + create_missing_stage_resources_mock.return_value = ["ANY", "ANY", "ANY", "ANY"] + + # trigger + self.assertIn("deployer_arn", context) + preprocessor.run(context=context) + + # verify + create_deployer_at_mock.assert_not_called() + + @patch.object(Preprocessor, "_get_build_image") + @patch.object(Preprocessor, "_create_deployer_at") + @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") + def test_run_will_not_recreate_provided_resources( + self, manage_cloudformation_stack_mock, create_deployer_at_mock, get_build_image_mock + ): + # setup + context["prod_stack_name"] = "PROD-STACK" + create_deployer_at_mock.return_value = ["ANY", "ANY", "ANY"] + manage_cloudformation_stack_mock.return_value = [ + {"OutputKey": "DeployerRole", "OutputValue": "new-deployer-role-arn"}, + {"OutputKey": "CFNDeploymentRole", "OutputValue": "new-cfn-deployment-role-arn"}, + {"OutputKey": "ArtifactsBucket", "OutputValue": "new-artifacts-bucket-arn"}, + {"OutputKey": "ArtifactsBucketKey", "OutputValue": "new-artifacts-bucket-key-arn"}, + ] + + # trigger + self.assertIn("deployer_arn", context) + self.assertIn("testing_deployer_role", context) + self.assertIn("testing_cfn_deployment_role", context) + self.assertIn("testing_artifacts_bucket", context) + self.assertNotIn("prod_deployer_role", context) + self.assertNotIn("prod_cfn_deployment_role", context) + self.assertNotIn("prod_artifacts_bucket", context) + preprocessor.run(context=context) + + # verify + create_deployer_at_mock.assert_not_called() + manage_cloudformation_stack_mock.assert_called_once() # for prod stage only + _, kwargs = manage_cloudformation_stack_mock.call_args + self.assertEqual(kwargs["stack_name"], "aws-sam-cli-managed-PROD-STACK-pipeline-resources") + actual_parameter_overrides = kwargs["parameter_overrides"] + expected_parameter_overrides = { + "DeployerArn": ANY_DEPLOYER_ARN, + "DeployerRoleArn": None, + "CFNDeploymentRoleArn": None, + "ArtifactsBucketArn": None, + } + self.assertEqual(actual_parameter_overrides, expected_parameter_overrides) + + @patch.object(Preprocessor, "_get_build_image") + @patch.object(Preprocessor, "_create_deployer_at") + @patch.object(Preprocessor, "_create_missing_stage_resources") + def test_run_creates_and_mutate_new_copy_of_the_context( + self, create_missing_stage_resources_mock, create_deployer_at_mock, get_build_image_mock + ): + # setup + context = {"sam_template": ANY_SAM_TEMPLATE} + create_missing_stage_resources_mock.return_value = ["ANY", "ANY", "ANY", "ANY"] + create_deployer_at_mock.return_value = ["ANY", "ANY", "ANY"] + + # triggert + mutated_context = preprocessor.run(context=context) + + # verify + self.assertIsNot(context, mutated_context) + + self.assertNotIn(PLUGIN_NAME, context) + self.assertNotIn("build_image", context) + self.assertNotIn("testing_deployer_role", context) + self.assertNotIn("testing_cfn_deployment_role", context) + self.assertNotIn("testing_artifacts_bucket", context) + self.assertNotIn("prod_deployer_role", context) + self.assertNotIn("prod_cfn_deployment_role", context) + self.assertNotIn("prod_artifacts_bucket", context) + + self.assertIn(PLUGIN_NAME, mutated_context) + self.assertIn("build_image", mutated_context) + self.assertIn("testing_deployer_role", mutated_context) + self.assertIn("testing_cfn_deployment_role", mutated_context) + self.assertIn("testing_artifacts_bucket", mutated_context) + self.assertIn("prod_deployer_role", mutated_context) + self.assertIn("prod_cfn_deployment_role", mutated_context) + self.assertIn("prod_artifacts_bucket", mutated_context) diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_resource.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_resource.py new file mode 100644 index 00000000000..d246bc7f9de --- /dev/null +++ b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_resource.py @@ -0,0 +1,49 @@ +from unittest import TestCase +from samcli.lib.pipeline.init.plugins.two_stages_pipeline.resource import Resource, Deployer, S3Bucket # type: ignore + +VALID_AWS_ARN = "arn:partition:service:region:account-id:resource-id" +INVALID_AWS_ARN = "ARN" + + +class TestResource(TestCase): + def test_resource(self): + resource = Resource(arn=VALID_AWS_ARN) + self.assertEqual(resource.arn, VALID_AWS_ARN) + self.assertTrue(resource.is_user_provided) + self.assertEqual(resource.name(), "resource-id") + + resource = Resource(arn=INVALID_AWS_ARN) + self.assertEqual(resource.arn, INVALID_AWS_ARN) + self.assertTrue(resource.is_user_provided) + self.assertEqual(resource.name(), "ARN") + + resource = Resource(arn=None) + self.assertIsNone(resource.arn) + self.assertFalse(resource.is_user_provided) + self.assertIsNone(resource.name()) + + +class TestDeployerResource(TestCase): + def test_deployer_resource(self): + deployer = Deployer(arn=VALID_AWS_ARN) + self.assertEqual(deployer.arn, VALID_AWS_ARN) + self.assertTrue(deployer.is_user_provided) + self.assertEqual(deployer.name(), "resource-id") + self.assertIsNone(deployer.access_key_id) + self.assertIsNone(deployer.secret_access_key) + + deployer = Deployer(arn=VALID_AWS_ARN, access_key_id="access_key_id", secret_access_key="secret_access_key") + self.assertEqual("access_key_id", deployer.access_key_id) + self.assertEqual("secret_access_key", deployer.secret_access_key) + + +class TestS3BucketResource(TestCase): + def test_s3bucket_resource(self): + bucket = S3Bucket(arn=VALID_AWS_ARN) + self.assertEqual(bucket.arn, VALID_AWS_ARN) + self.assertTrue(bucket.is_user_provided) + self.assertEqual(bucket.name(), "resource-id") + self.assertIsNone(bucket.kms_key_arn) + + bucket = S3Bucket(arn=VALID_AWS_ARN, kms_key_arn="kms_key_arn") + self.assertEqual("kms_key_arn", bucket.kms_key_arn) diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_stage.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_stage.py new file mode 100644 index 00000000000..63859de3ad6 --- /dev/null +++ b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_stage.py @@ -0,0 +1,70 @@ +from unittest import TestCase +from samcli.lib.pipeline.init.plugins.two_stages_pipeline.stage import Stage # type: ignore + + +class TestStage(TestCase): + def test_init_stage(self): + stage = Stage( + name="any-name", + aws_profile="any-profile", + aws_region="any-region", + stack_name="any-stack-name", + deployer_role_arn="any-deployer-role-arn", + cfn_deployment_role_arn=None, + artifacts_bucket_arn=None, + ) + self.assertEqual(stage.name, "any-name") + self.assertEqual(stage.aws_profile, "any-profile") + self.assertEqual(stage.aws_region, "any-region") + self.assertEqual(stage.stack_name, "any-stack-name") + self.assertEqual(stage.deployer_role.arn, "any-deployer-role-arn") + self.assertTrue(stage.deployer_role.is_user_provided) + self.assertIsNone(stage.cfn_deployment_role.arn) + self.assertFalse(stage.cfn_deployment_role.is_user_provided) + self.assertIsNone(stage.artifacts_bucket.arn) + self.assertFalse(stage.artifacts_bucket.is_user_provided) + + def test_did_user_provide_all_required_resources(self): + stage = Stage( + name="any", + aws_profile="any", + aws_region="any", + stack_name="any", + deployer_role_arn=None, + cfn_deployment_role_arn=None, + artifacts_bucket_arn=None, + ) + self.assertFalse(stage.did_user_provide_all_required_resources()) + + stage = Stage( + name="any", + aws_profile="any", + aws_region="any", + stack_name="any", + deployer_role_arn="any-deployer-role-arn", + cfn_deployment_role_arn=None, + artifacts_bucket_arn=None, + ) + self.assertFalse(stage.did_user_provide_all_required_resources()) + + stage = Stage( + name="any", + aws_profile="any", + aws_region="any", + stack_name="any", + deployer_role_arn="any-deployer-role-arn", + cfn_deployment_role_arn="any-cfn-deployment-role-arn", + artifacts_bucket_arn=None, + ) + self.assertFalse(stage.did_user_provide_all_required_resources()) + + stage = Stage( + name="any", + aws_profile="any", + aws_region="any", + stack_name="any", + deployer_role_arn="any-deployer-role-arn", + cfn_deployment_role_arn="any-cfn-deployment-role-arn", + artifacts_bucket_arn="any-artifacts-bucket-arn", + ) + self.assertTrue(stage.did_user_provide_all_required_resources()) diff --git a/tests/unit/lib/utils/test_managed_cloudformation_stack.py b/tests/unit/lib/utils/test_managed_cloudformation_stack.py index 9f1ea0915ae..fd21b792f17 100644 --- a/tests/unit/lib/utils/test_managed_cloudformation_stack.py +++ b/tests/unit/lib/utils/test_managed_cloudformation_stack.py @@ -21,19 +21,28 @@ def _stubbed_cf_client(self): def test_session_missing_profile(self, boto_mock): boto_mock.side_effect = ProfileNotFound(profile="test-profile") with self.assertRaises(CredentialsError): - manage_stack("test-profile", "fake-region", SAM_CLI_STACK_NAME, _get_stack_template()) + manage_stack( + profile="test-profile", + region="fake-region", + stack_name=SAM_CLI_STACK_NAME, + template_body=_get_stack_template(), + ) @patch("boto3.client") def test_client_missing_credentials(self, boto_mock): boto_mock.side_effect = NoCredentialsError() with self.assertRaises(CredentialsError): - manage_stack(None, "fake-region", SAM_CLI_STACK_NAME, _get_stack_template()) + manage_stack( + profile=None, region="fake-region", stack_name=SAM_CLI_STACK_NAME, template_body=_get_stack_template() + ) @patch("boto3.client") def test_client_missing_region(self, boto_mock): boto_mock.side_effect = NoRegionError() with self.assertRaises(RegionError): - manage_stack(None, "fake-region", SAM_CLI_STACK_NAME, _get_stack_template()) + manage_stack( + profile=None, region="fake-region", stack_name=SAM_CLI_STACK_NAME, template_body=_get_stack_template() + ) def test_new_stack(self): stub_cf, stubber = self._stubbed_cf_client() @@ -47,6 +56,8 @@ def test_new_stack(self): "Tags": [{"Key": "ManagedStackSource", "Value": "AwsSamCli"}], "ChangeSetType": "CREATE", "ChangeSetName": "InitialCreation", + "Capabilities": ["CAPABILITY_IAM"], + "Parameters": [], } ccs_resp = {"Id": "id", "StackId": "aws-sam-cli-managed-default"} stubber.add_response("create_change_set", ccs_resp, ccs_params) @@ -151,6 +162,8 @@ def test_change_set_creation_fails(self): "Tags": [{"Key": "ManagedStackSource", "Value": "AwsSamCli"}], "ChangeSetType": "CREATE", "ChangeSetName": "InitialCreation", + "Capabilities": ["CAPABILITY_IAM"], + "Parameters": [], } stubber.add_client_error("create_change_set", service_error_code="ClientError", expected_params=ccs_params) stubber.activate() @@ -171,6 +184,8 @@ def test_change_set_execution_fails(self): "Tags": [{"Key": "ManagedStackSource", "Value": "AwsSamCli"}], "ChangeSetType": "CREATE", "ChangeSetName": "InitialCreation", + "Capabilities": ["CAPABILITY_IAM"], + "Parameters": [], } ccs_resp = {"Id": "id", "StackId": "aws-sam-cli-managed-default"} stubber.add_response("create_change_set", ccs_resp, ccs_params) From 9f0147add9bc27cf30c498ebd3f5e4f064017545 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 30 Mar 2021 19:48:05 -0700 Subject: [PATCH 02/31] typos --- .../plugins/two_stages_pipeline/postprocessor.py | 12 ++++++------ .../init/plugins/two_stages_pipeline/preprocessor.py | 2 +- .../two_stages_pipeline/test_postprocessor.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py index b77fa344a6e..2564d0fcadb 100644 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py +++ b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py @@ -20,14 +20,14 @@ class Postprocessor(Processor): Attributes ---------- - resources_reused: + resources_provided: A list of the required AWS resources that got provided by the user. - resources_reused: + resources_created: A list of the required AWS resources that the plugin created on behalf of the user. """ def __init__(self) -> None: - self.resources_reused: List[Dict[str, str]] = [] + self.resources_provided: List[Dict[str, str]] = [] self.resources_created: List[Dict[str, str]] = [] def run(self, context: Dict) -> Dict: @@ -62,12 +62,12 @@ def run(self, context: Dict) -> Dict: for resource in self.resources_created: click.secho(f"\t{resource['arn']}", fg="yellow") - if self.resources_reused: + if self.resources_provided: click.secho( "\nWe have reused the following resources, please make sure it has the required permissions:", fg="yellow", ) - for resource in self.resources_reused: + for resource in self.resources_provided: click.secho(f"\n{resource['arn']}", fg="yellow") click.secho(f"Required Permissions: {resource['required_permissions']}", fg="yellow") click.echo("\n") @@ -99,6 +99,6 @@ def _categorize_resource(self, resource: Resource, required_permissions: str) -> the IAM policies required for the resource to operate correctly. """ if resource.is_user_provided: - self.resources_reused.append({"arn": resource.arn, "required_permissions": required_permissions}) + self.resources_provided.append({"arn": resource.arn, "required_permissions": required_permissions}) else: self.resources_created.append({"arn": resource.arn}) diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py index c12409a2d9e..a1ae9b3f4c0 100644 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py +++ b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py @@ -28,7 +28,7 @@ class Preprocessor(Processor): """ - 1. Find the appropriate docker build image for the SAM temolate + 1. Find the appropriate docker build image for the SAM template 2. Creates the required AWS resources for this pipeline if not already provided by the user. Methods diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py index 388cb64f863..4b80611f17b 100644 --- a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py +++ b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py @@ -62,14 +62,14 @@ def test_the_postprocessoer_correctly_categorize_the_resources_into_reused_and_c context = {PLUGIN_NAME: plugin_context} postprocessor = Postprocessor() - self.assertEqual(0, len(postprocessor.resources_reused)) + self.assertEqual(0, len(postprocessor.resources_provided)) self.assertEqual(0, len(postprocessor.resources_created)) # trigger mutated_context = postprocessor.run(context=context) # verify - reused_resources_arns = list(map(lambda r: r["arn"], postprocessor.resources_reused)) + reused_resources_arns = list(map(lambda r: r["arn"], postprocessor.resources_provided)) self.assertEqual(3, len(reused_resources_arns)) self.assertIn(ANY_TESTING_DEPLOYER_ROLE, reused_resources_arns) self.assertIn(ANY_TESTING_CFN_DEPLOYMENT_ROLE, reused_resources_arns) From 3946009555779f4e513513386ac0b4fc53927ee2 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 30 Mar 2021 19:52:36 -0700 Subject: [PATCH 03/31] add docstring --- .../pipeline/init/plugins/two_stages_pipeline/context.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/context.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/context.py index 2591d997f8d..a73efb17818 100644 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/context.py +++ b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/context.py @@ -36,6 +36,13 @@ class Context: PROD_STAGE_NAME: str = "prod" def __init__(self, context: Dict[str, str]) -> None: + """ + + Parameters + ---------- + context: dictionary of user's response to the questions defined in the ./questions.json file where + keys are questions' keys and values are user's answers + """ testing_stage: Stage = Stage( name=Context.TESTING_STAGE_NAME, From d3db05bc535beec1ec5a4fe5098ba45f5e2ced65 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 30 Mar 2021 20:29:06 -0700 Subject: [PATCH 04/31] make mypy happy --- samcli/lib/cookiecutter/exceptions.py | 4 ++-- .../lib/cookiecutter/interactive_flow_creator.py | 10 +++++----- samcli/lib/cookiecutter/processor.py | 2 +- samcli/lib/cookiecutter/question.py | 14 +++++++------- samcli/lib/cookiecutter/template.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/samcli/lib/cookiecutter/exceptions.py b/samcli/lib/cookiecutter/exceptions.py index af193648114..5d379228d81 100644 --- a/samcli/lib/cookiecutter/exceptions.py +++ b/samcli/lib/cookiecutter/exceptions.py @@ -4,8 +4,8 @@ class CookiecutterErrorException(Exception): fmt = "An unspecified error occurred" - def __init__(self, **kwargs): - msg = self.fmt.format(**kwargs) + def __init__(self, **kwargs): # type: ignore + msg: str = self.fmt.format(**kwargs) Exception.__init__(self, msg) self.kwargs = kwargs diff --git a/samcli/lib/cookiecutter/interactive_flow_creator.py b/samcli/lib/cookiecutter/interactive_flow_creator.py index d1a227f1c81..9ec0dbfa79d 100644 --- a/samcli/lib/cookiecutter/interactive_flow_creator.py +++ b/samcli/lib/cookiecutter/interactive_flow_creator.py @@ -1,5 +1,5 @@ """ This module parses a json/yaml file that defines a flow of questions to fulfill the cookiecutter context""" -from typing import Dict, Optional, Tuple +from typing import cast, Dict, Optional, Tuple import yaml from samcli.commands.exceptions import UserException from samcli.yamlhelper import parse_yaml_file @@ -17,7 +17,7 @@ class QuestionsFailedParsingException(UserException): class InteractiveFlowCreator: @staticmethod - def create_flow(flow_definition_path: str, extra_context: Optional[Dict] = None): + def create_flow(flow_definition_path: str, extra_context: Optional[Dict] = None) -> InteractiveFlow: """ This method parses the given json/yaml file to create an InteractiveFLow. It expects the file to define a list of questions. It parses the questions and add it to the flow in the same order they are defined @@ -63,7 +63,7 @@ def _load_questions( questions: Dict[str, Question] = {} questions_definition = InteractiveFlowCreator._parse_questions_definition(flow_definition_path, extra_context) - for question in questions_definition.get("questions"): + for question in questions_definition.get("questions", []): q = QuestionFactory.create_question_from_json(question) if not first_question_key: first_question_key = q.key @@ -74,7 +74,7 @@ def _load_questions( return questions, first_question_key @staticmethod - def _parse_questions_definition(file_path, extra_context: Optional[Dict] = None): + def _parse_questions_definition(file_path: str, extra_context: Optional[Dict] = None) -> Dict: """ Read the questions definition file, do variable substitution, parse it as JSON/YAML @@ -96,7 +96,7 @@ def _parse_questions_definition(file_path, extra_context: Optional[Dict] = None) """ try: - return parse_yaml_file(file_path=file_path, extra_context=extra_context) + return cast(Dict, parse_yaml_file(file_path=file_path, extra_context=extra_context)) except FileNotFoundError as ex: raise QuestionsNotFoundException(f"questions definition file not found at {file_path}") from ex except (KeyError, ValueError, yaml.YAMLError) as ex: diff --git a/samcli/lib/cookiecutter/processor.py b/samcli/lib/cookiecutter/processor.py index 5994c779495..4f34df06f8a 100644 --- a/samcli/lib/cookiecutter/processor.py +++ b/samcli/lib/cookiecutter/processor.py @@ -9,7 +9,7 @@ class Processor(ABC): """ @abstractmethod - def run(self, context: Dict): + def run(self, context: Dict) -> Dict: """ the processing logic of this processor diff --git a/samcli/lib/cookiecutter/question.py b/samcli/lib/cookiecutter/question.py index 71c30d98da7..4a9039d0b90 100644 --- a/samcli/lib/cookiecutter/question.py +++ b/samcli/lib/cookiecutter/question.py @@ -64,27 +64,27 @@ def __init__( self._default_next_question_key = default_next_question_key @property - def key(self): + def key(self) -> str: return self._key @property - def text(self): + def text(self) -> str: return self._text @property - def default_answer(self): + def default_answer(self) -> Optional[str]: return self._default_answer @property - def required(self): + def required(self) -> Optional[bool]: return self._required @property - def next_question_map(self): + def next_question_map(self) -> Dict[str, str]: return self._next_question_map @property - def default_next_question_key(self): + def default_next_question_key(self) -> Optional[str]: return self._default_next_question_key def ask(self) -> Any: @@ -96,7 +96,7 @@ def get_next_question_key(self, answer: Any) -> Optional[str]: answer = str(answer) return self._next_question_map.get(answer, self._default_next_question_key) - def set_default_next_question_key(self, next_question_key): + def set_default_next_question_key(self, next_question_key: str) -> None: self._default_next_question_key = next_question_key diff --git a/samcli/lib/cookiecutter/template.py b/samcli/lib/cookiecutter/template.py index c7d643bb439..a9642cd9f87 100644 --- a/samcli/lib/cookiecutter/template.py +++ b/samcli/lib/cookiecutter/template.py @@ -119,7 +119,7 @@ def run_interactive_flows(self) -> Dict: except Exception as e: raise UserException(str(e), wrapped_from=e.__class__.__name__) from e - def generate_project(self, context: Dict): + def generate_project(self, context: Dict) -> None: """ Generates a project based on this cookiecutter template and the given context. The context is first processed and manipulated by series of preprocessors(if any) then the project is generated and finally From 201de2eb0e9874483b1a95a67d422107bae7c670 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 30 Mar 2021 20:44:24 -0700 Subject: [PATCH 05/31] removing swap file --- .../cfn_templates/.stage_resources.yaml.swp | Bin 16384 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/.stage_resources.yaml.swp diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/.stage_resources.yaml.swp b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/.stage_resources.yaml.swp deleted file mode 100644 index e7b96679f04dd5a034634f10da0c32593b28b08d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI2ZHy#E8OM9MLlEu`BBu!`m%4#CcgM}n?rk`@4u%Xnv&VYu-DYPw?)Gw7YkF$- zn(gTxx~p#)Q8XG6jUOc93xSxJs3GCQ5i|-RYWQHH;fvx+G%9Mq`0~Y(2gmGE>h}_0-e#JY7%Cw5E>FFOXX&r!`zxYuf6Glka`BAv4yZjCc39nT+;y)<4OXQaUd5x` zieZ^myowjgY*l+^k6EV8s-|u7YR~f5OxvsS4QJKkbd7ndk$};$S2^qUEXtWz4O&-S z<~hD=FmJL?yVigW#hbzd7q17lYcmI?r^NhkDD5WKyzTCb*A>Og3l9_?C_GSjpzuK9 zfx-iY2MP~dgdX6%OSBJR-j}Gk->+ULQmC_GSj zpzuK9fx-iY2MP}q9w!1w=Q{r~f;H0^8P8Squ`dGHW83Fg4f z;CiqPyuL%zeg|F!KLcL`E8seCE!YA6xn0wK1D*v>f`@?xPJ-LP9`M>W$bj#FbKn6$ zK@IE!+rVG0)U=nt55WuIN$@e?gS){Y@NTdT{QU~Zfp38)z^4EMN5D;BCwTp>n)Why z9()OW9Gn0*g4ZtBw4Z`+gHM1Z@GkJmWv~xyfZM<>a6PyT{P``K_I>ai_$Xjt85{)f z09(KxFU1_d=fDR*3)~5&!7i{B{BkSC01p8R+zz&bU*jO=MQ{#$25@i;RKeT9PA~y} zkNf!o_%e7D_~0~X0CBIc1?Rtrx|+;A%iN~fW)0hL`#o+t_N-%f%r)Po;*G9J$vb>w zo$**G{G5;`r`-2+(y(dEVr?aVoVgy0#B`htGj3``*lrstu@8dN*f;GQ%pkXr8dm`-Zqw7&iN>Cq2jQ+C zb|#}J$S&}l3SYvalv8cX5P%*ry6Zuo=_E0B9yqdqWFX@<98zTre3(iE1lREUCt5{9 zC>fUHw>ys8l~W9p1DtEDhw3tGNtN(_RuzG_tbW+m^t)Z^_D9nj7}EPMbQ))|2u4(F z+|LZ3v)MD$USmzh#~W@;wON;8iJnI(Kkc0_B4hWDlNQCD9yc)xosm-`8JZfyH53PF zI0Z34S!9Mq>UknzhLIQxjPIYEq#(MB0zZy>EgnY3ecdtq!E{m~vPI+&1i67bahZ*w#JId%b8#bFn}Af6RTbS_MybG~KC#d0B*h}hqn2G-+X z=2kyq#?vA_nfyK1kad$0*0nIXVABm(=kKMlBWT>`Q_CZ#*p+&~t5kCPT7J=Zylx=I#Asgc7D1IreviiVOKLs8E zZEyf=1zW%?sNKH@z5+fAY;XeH2KIuhz`szt{{lP@9tR%=_kv|0YWr7FzkeOv2O3}+ zTmk-s`u!#FH25Uwf%kxSf*s&*sNw$#{s4Xi9tWQW9|5c27H|#N0)Bv6{=49t;4yF( z+ym-B)cRL~e~Vfkd;>fR9t4x%av*&EE8+LzDm+kl-~xC+JhhKg%g56_+^(k*KkB^b ztRQ-3=tCZTcy17EB*k%0Sg0-?B~FL%GmOkG93j5P+61XNe!P%Ya4MEx$fCcjIq}9` zoq?o8@IVOc5VpisNJ=Q0la$JTlB;f#hcMDvLBqg2#g09kXf%|X8>FL;{ z>{#>dPfFw)pOhHG@>+a{uxTd3f<4Ku_M)%k`;6^*TL3kF@&;tY@@J6|dxKNS^e5(% z`B2x6`Ml?&e2Nb@b57WlEy72TBm-9khaVla<2dRB>m(&1#D^}(WSk}Y4R@UB;bdo} z)cNX=(*-FNoRDTE@(aWGOrR9(($acSt)5!iT5S9X$CoMPRy zPpR{^oK1DPy01)56S1>SUqjtKun0q~?2zh=H)XC+-cN=OJA#YEh{^O4S}^RciyD7M zzsvM^&RM%?f}t|S3~68V`LWi$IDH+`4*lHr>>`O5$lx(ps5c?&oh^cjWR^(Qn& zJ_#Ek%97LD%)+GXX2vCDwa_);>oB{-iPL~^?rDMOC5n3y_GPM$_OZxyy}UoKv_72< z{jqzH(VXh%|0)>M**X)B`5~wFCArBpxLcctTPisQT8WdD_$$RI>{1#-;;q?i)T}kf yHTfA-^y1qcJkEmNu}0K|? Date: Tue, 13 Apr 2021 17:20:22 -0700 Subject: [PATCH 06/31] delete the two_stages_pipeline plugin as the pipeline-bootstrap command took over its responsibility --- samcli/lib/pipeline/init/plugins/__init__.py | 0 .../plugins/two_stages_pipeline/__init__.py | 0 .../cfn_templates/deployer.yaml | 36 -- .../plugins/two_stages_pipeline/config.py | 2 - .../plugins/two_stages_pipeline/context.py | 99 ---- .../two_stages_pipeline/postprocessor.py | 104 ----- .../two_stages_pipeline/preprocessor.py | 238 ---------- .../two_stages_pipeline/questions.json | 114 ----- .../plugins/two_stages_pipeline/resource.py | 26 -- .../init/plugins/two_stages_pipeline/stage.py | 155 ------ tests/unit/lib/pipeline/init/__init__.py | 0 .../lib/pipeline/init/plugins/__init__.py | 0 .../plugins/two_stages_pipeline/__init__.py | 0 .../two_stages_pipeline/test_context.py | 161 ------- .../two_stages_pipeline/test_postprocessor.py | 101 ---- .../two_stages_pipeline/test_preprocessor.py | 440 ------------------ .../two_stages_pipeline/test_resource.py | 49 -- .../plugins/two_stages_pipeline/test_stage.py | 70 --- 18 files changed, 1595 deletions(-) delete mode 100644 samcli/lib/pipeline/init/plugins/__init__.py delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/deployer.yaml delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/config.py delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/context.py delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/questions.json delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/resource.py delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/stage.py delete mode 100644 tests/unit/lib/pipeline/init/__init__.py delete mode 100644 tests/unit/lib/pipeline/init/plugins/__init__.py delete mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py delete mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_context.py delete mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py delete mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_preprocessor.py delete mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_resource.py delete mode 100644 tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_stage.py diff --git a/samcli/lib/pipeline/init/plugins/__init__.py b/samcli/lib/pipeline/init/plugins/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/deployer.yaml b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/deployer.yaml deleted file mode 100644 index c5f82b0404e..00000000000 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/deployer.yaml +++ /dev/null @@ -1,36 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Transform: AWS::Serverless-2016-10-31 -Resources: - Deployer: - Type: AWS::IAM::User - Properties: - Tags: - - Key: ManagedStackSource - Value: AwsSamCli - Policies: - - PolicyName: AssumeRoles - PolicyDocument: | - { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": ["sts:AssumeRole"], - "Resource": "*" - }] - } - DeployerAccessKey: - Type: AWS::IAM::AccessKey - Properties: - Serial: 1 - Status: Active - UserName: !Ref Deployer -Outputs: - Deployer: - Description: ARN of the Deployer IAM User - Value: !GetAtt Deployer.Arn - AccessKeyId: - Description: AccessKeyId of the Deployer IAM User - Value: !Ref DeployerAccessKey - SecretAccessKey: - Description: SecretAccessKey of the Deployer IAM User - Value: !GetAtt DeployerAccessKey.SecretAccessKey diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/config.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/config.py deleted file mode 100644 index 26a2c297654..00000000000 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/config.py +++ /dev/null @@ -1,2 +0,0 @@ -""" plugin configurations""" -PLUGIN_NAME = "TWO_STAGES_PIPELINE" diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/context.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/context.py deleted file mode 100644 index a73efb17818..00000000000 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/context.py +++ /dev/null @@ -1,99 +0,0 @@ -""" The plugin context """ -from typing import Dict, List, Optional - -from .resource import Deployer -from .stage import Stage - - -class Context: - """ - The context of the plugin. it defines two pipeline stages; testing and prod, - the deployer IAM user and additional required context. - - Attributes - ---------- - stages: List[Stage] - The stages of the pipeline; testing and prod - deployer: Deployer - Represents the IAM User that deploys the pipeline. The credentials(AWS_ACCESS_KEY_ID, - AWS_SECRET_ACCESS_KEY) of this IAM user must be shared with the CICD provider. - deployer_aws_access_key_id_variable_name: str - The name of the CICD env variable that holds the value of the AWS_ACCESS_KEY_ID - deployer_aws_secret_access_key_variable_name: str - The name of the CICD env variable that holds the value of the AWS_SECRET_ACCESS_KEY - - - Methods - ------- - get_stage(stage_name: str): Stage: - returns a stage by name - deployer_permissions(): str - returns a string representing the required IAM policies for the deployer IAM user - to be able to deploy the pipeline - """ - - TESTING_STAGE_NAME: str = "testing" - PROD_STAGE_NAME: str = "prod" - - def __init__(self, context: Dict[str, str]) -> None: - """ - - Parameters - ---------- - context: dictionary of user's response to the questions defined in the ./questions.json file where - keys are questions' keys and values are user's answers - """ - - testing_stage: Stage = Stage( - name=Context.TESTING_STAGE_NAME, - aws_profile=context.get("testing_profile"), - aws_region=context.get("testing_region"), - stack_name=context.get("testing_stack_name"), - deployer_role_arn=context.get("testing_deployer_role"), - cfn_deployment_role_arn=context.get("testing_cfn_deployment_role"), - artifacts_bucket_arn=context.get("testing_artifacts_bucket"), - ) - - prod_stage: Stage = Stage( - name=Context.PROD_STAGE_NAME, - aws_profile=context.get("prod_profile"), - aws_region=context.get("prod_region"), - stack_name=context.get("prod_stack_name"), - deployer_role_arn=context.get("prod_deployer_role"), - cfn_deployment_role_arn=context.get("prod_cfn_deployment_role"), - artifacts_bucket_arn=context.get("prod_artifacts_bucket"), - ) - - self.stages: List[Stage] = [testing_stage, prod_stage] - self.deployer: Deployer = Deployer(arn=context.get("deployer_arn")) - self.deployer_aws_access_key_id_variable_name: str = context.get("deployer_aws_access_key_id_variable_name") - self.deployer_aws_secret_access_key_variable_name: str = context.get( - "deployer_aws_secret_access_key_variable_name" - ) - self.build_image: Optional[str] = None - - def get_stage(self, stage_name: str) -> Stage: - """ - returns a stage by name. - - Parameters - ---------- - stage_name: str - The name of the stage to return - """ - return next((stage for stage in self.stages if stage.name == stage_name), None) - - def deployer_permissions(self) -> str: - """ - returns a string representing the required IAM policies for the deployer IAM user - to be able to deploy the pipeline - """ - deployer_roles = ", ".join(list(filter(None, map(lambda stage: stage.deployer_role.arn, self.stages)))) - permissions = f""" -{{ - "Effect": "Allow", - "Action": ["sts:AssumeRole"], - "Resource": [{deployer_roles}] -}} - """ - return permissions diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py deleted file mode 100644 index 2564d0fcadb..00000000000 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/postprocessor.py +++ /dev/null @@ -1,104 +0,0 @@ -""" -The plugin's postprocessor prints information about the created and reused AWS resources and the required permissions -""" -import logging -from typing import Dict, List - -import click - -from samcli.lib.cookiecutter.processor import Processor -from .config import PLUGIN_NAME -from .context import Context as PluginContext -from .resource import Deployer, Resource - -LOG = logging.getLogger(__name__) - - -class Postprocessor(Processor): - """ - prints information about the created and reused AWS resources and the required permissions - - Attributes - ---------- - resources_provided: - A list of the required AWS resources that got provided by the user. - resources_created: - A list of the required AWS resources that the plugin created on behalf of the user. - """ - - def __init__(self) -> None: - self.resources_provided: List[Dict[str, str]] = [] - self.resources_created: List[Dict[str, str]] = [] - - def run(self, context: Dict) -> Dict: - """ - iterates through the pipeline's AWS resources and categorize them into two categories: - 1. Resources created by the plugin - 2. Resources provided by the user - It then prints to the user the ARNs of the resources created by the plugin. And for each resource provided - by the user, it prints instructions about the required IAM policies for this resource to operate as expected, - so that the user can ensure it already has this permissions. - - Parameters - ---------- - context: Dict - A dictionary of the whole context of the cookiecutter template, it contains a key, context[PLUGIN_NAME], - that contains this plugin's context(object of type PluginContext) where the method extracts its required - information from. - """ - - context = context.copy() - plugin_context: PluginContext = context[PLUGIN_NAME] - deployer: Deployer = plugin_context.deployer - self._categorize_resource(deployer, plugin_context.deployer_permissions()) - - for stage in plugin_context.stages: - self._categorize_resource(stage.deployer_role, stage.deployer_role_permissions(deployer.arn)) - self._categorize_resource(stage.cfn_deployment_role, stage.cfn_deployment_role_permissions()) - self._categorize_resource(stage.artifacts_bucket, stage.artifacts_bucket_permissions()) - - if self.resources_created: - click.secho("\nWe have created the following resources:", fg="yellow") - for resource in self.resources_created: - click.secho(f"\t{resource['arn']}", fg="yellow") - - if self.resources_provided: - click.secho( - "\nWe have reused the following resources, please make sure it has the required permissions:", - fg="yellow", - ) - for resource in self.resources_provided: - click.secho(f"\n{resource['arn']}", fg="yellow") - click.secho(f"Required Permissions: {resource['required_permissions']}", fg="yellow") - click.echo("\n") - - if not deployer.is_user_provided: - click.secho( - "Please set the following variables of the IAM user credentials to your CICD project:", fg="green" - ) - click.secho( - f"{plugin_context.deployer_aws_access_key_id_variable_name}: {deployer.access_key_id}", fg="green" - ) - click.secho( - f"{plugin_context.deployer_aws_secret_access_key_variable_name}: {deployer.secret_access_key}", - fg="green", - ) - - return context - - def _categorize_resource(self, resource: Resource, required_permissions: str) -> None: - """ - add the resource to the corresponding category; reused or created. And store a reference for the resource's - required permissions in case if it is not created by the plugin - - Parameters - ---------- - resource: Resource - The resource to categorized - required_permissions: str - the IAM policies required for the resource to operate correctly. - """ - if resource.is_user_provided: - self.resources_provided.append({"arn": resource.arn, "required_permissions": required_permissions}) - else: - self.resources_created.append({"arn": resource.arn}) diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py deleted file mode 100644 index a1ae9b3f4c0..00000000000 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/preprocessor.py +++ /dev/null @@ -1,238 +0,0 @@ -""" -The plugin's preprocessor creates the required AWS resources for this pipeline if not already provided by the user. -""" -import logging -import os -import pathlib -from typing import Dict, List, Optional, Tuple - -import click - -from samcli.commands._utils.template import get_template_function_runtimes -from samcli.lib.cookiecutter.processor import Processor -from samcli.lib.utils.managed_cloudformation_stack import manage_stack as manage_cloudformation_stack -from samcli.local.common.runtime_template import RUNTIME_TO_BUILD_IMAGE -from .config import PLUGIN_NAME -from .context import Context as PluginContext -from .resource import Deployer -from .stage import Stage - -ROOT_PATH = str(pathlib.Path(os.path.dirname(__file__))) -CFN_TEMPLATE_PATH = os.path.join(ROOT_PATH, "cfn_templates") -STACK_NAME_PREFIX = "aws-sam-cli-managed" -DEPLOYER_STACK_NAME_SUFFIX = "pipeline-deployer" -STAGE_RESOURCES_STACK_NAME_SUFFIX = "pipeline-resources" - -LOG = logging.getLogger(__name__) - - -class Preprocessor(Processor): - """ - 1. Find the appropriate docker build image for the SAM template - 2. Creates the required AWS resources for this pipeline if not already provided by the user. - - Methods - ------- - _get_build_image(sam_template_file) -> Optional[str]: - scan the SAM template for the ZIP functions, extract the runtimes and return the appropriate SA< build image - for the runtime if the template contains one and exactly one supported runtime, otherwise, asks the user - to provide an alternative build-image - run(context: Dict) -> Dict: - Creates the missed required AWS resources and updated the passed cookiecutter context with its ARNs - _create_deployer_at(stage: Stage): - deploys the CFN template(./cfn_templates/deployer.yaml) to the given stage - _create_missing_stage_resources(stage: Stage, deployer_arn: str): - deploys the CFN template(./cfn_templates/resource_stages.yaml) to the given stage - """ - - BASIC_PROVIDED_BUILD_IMAGE: str = "public.ecr.aws/sam/build-provided" - - def run(self, context: Dict) -> Dict: - """ - searches the passed cookiecutter context for the pipeline's required AWS resources, identifies which resources - are missing, create them through a CFN stack. - This method create and add to the context a plugin-explicit context that contains all of the required - AWS resources and additional plugin explicit attribute. This plugin-explicit context is not used in the - cookiecutter template itself, instead, it is used by the postprocessor of this plugin. - The method returns a mutated copy of the cookiecutter context that updates the context with the ARNs of the - created resources. - - Parameters - ---------- - context: Dict - The cookiecutter context to look for the resources from. - """ - context = context.copy() - plugin_context: PluginContext = PluginContext(context) - context[PLUGIN_NAME] = plugin_context - - context["build_image"] = plugin_context.build_image = Preprocessor._get_build_image(context["sam_template"]) - - deployer: Deployer = plugin_context.deployer - if deployer.is_user_provided: - deployer_arn = deployer.arn - else: - # create deployer(IAM user) in the testing stage - testing_stage: Stage = plugin_context.get_stage(PluginContext.TESTING_STAGE_NAME) - deployer_arn, access_key_id, secret_access_key = Preprocessor._create_deployer_at(stage=testing_stage) - context["deployer_arn"] = deployer.arn = deployer_arn - deployer.access_key_id = access_key_id - deployer.secret_access_key = secret_access_key - - for stage in plugin_context.stages: - ( - deployer_role_arn, - cfn_deployment_role_arn, - artifacts_bucket_arn, - kms_key_arn, - ) = Preprocessor._create_missing_stage_resources(stage=stage, deployer_arn=deployer_arn) - context[f"{stage.name}_deployer_role"] = stage.deployer_role.arn = deployer_role_arn - context[f"{stage.name}_cfn_deployment_role"] = stage.cfn_deployment_role.arn = cfn_deployment_role_arn - stage.artifacts_bucket.arn = artifacts_bucket_arn - stage.artifacts_bucket.kms_key_arn = kms_key_arn - # The cookiecutter context requires the name of the artifacts bucket instead of ots ARN - context[f"{stage.name}_artifacts_bucket"] = stage.artifacts_bucket.name() - - return context - - # This method is a first iteration only to support the major usecase of having a SAM template with one supported - # runtime - # todo improve the experience to support Iamge lambda functions and lambda functions with different runtimes - @staticmethod - def _get_build_image(sam_template_file: str) -> str: - """ - Scans the SAM template for lambda runtimes, and if it contains only one supported runtime, it returns - the corresponding SAM build-image, otherwise, it asks the user to provide one - - Parameters - ---------- - sam_template_file: str - the path of the SAM template to scan for function's runtimes - - Returns: a docker build-image to use for the CICD pipeline - """ - runtimes: List[str] = get_template_function_runtimes(template_file=sam_template_file) - if not runtimes: - return Preprocessor.BASIC_PROVIDED_BUILD_IMAGE - elif len(runtimes) > 1: - click.echo( - "The SAM template defines multiple functions with different runtimes\n" - "SAM doesn't have an appropriate docker build image for that, please provide one" - ) - return click.prompt("Docker Build image") - else: - runtime = runtimes[0] - build_image = RUNTIME_TO_BUILD_IMAGE.get(runtime) - if not build_image: - click.echo( - f"The SAM template defines functions of runtime {runtime} but SAM doesn't have a docker " - f"build-image for {runtime}, please provide one" - ) - build_image = click.prompt("Docker Build image") - return build_image - - @staticmethod - def _create_deployer_at(stage: Stage) -> Tuple[str, str, str]: - """ - Deploys the CFN template(./cfn_templates/deployer.yaml) which defines a deployer IAM user and credentials - to the AWS account and region associated with the given stage. It will not redeploy the stack if already exists. - - Parameters - ---------- - stage: Stage - The pipeline stage to deploy the CFN template to its associated AWS account and region. - - Returns - ------- - ARN, access_key_id and secret_access_key of the IAM user identified by the template - """ - - profile: str = stage.aws_profile - region: str = stage.aws_region - stack_name: str = f"{STACK_NAME_PREFIX}-{stage.stack_name}-{DEPLOYER_STACK_NAME_SUFFIX}" - deployer_template_path: str = os.path.join(CFN_TEMPLATE_PATH, "deployer.yaml") - with open(deployer_template_path, "r") as fp: - deployer_template_body = fp.read() - click.echo(f"Creating an IAM user for pipeline Deployment. Account: '{profile}' Region: '{region}'") - outputs: List[Dict[str, str]] = manage_cloudformation_stack( - stack_name=stack_name, profile=profile, region=region, template_body=deployer_template_body - ) - deployer_arn: str = next(o for o in outputs if o.get("OutputKey") == "Deployer").get("OutputValue") - access_key_id_arn: str = next(o for o in outputs if o.get("OutputKey") == "AccessKeyId").get("OutputValue") - secret_access_key_arn: str = next(o for o in outputs if o.get("OutputKey") == "SecretAccessKey").get( - "OutputValue" - ) - return deployer_arn, access_key_id_arn, secret_access_key_arn - - @staticmethod - def _create_missing_stage_resources(stage: Stage, deployer_arn: str) -> Tuple[str, str, str, Optional[str]]: - """ - Deploys the CFN template(./cfn_templates/stage_resources.yaml) which defines: - * Deployer execution IAM role - * CloudFormation execution IAM role - * Artifacts' S3 Bucket along with KMS encryption key - to the AWS account and region associated with the given stage. It will not redeploy the stack if already exists. - This CFN template accepts the ARNs of the resources as parameters and will not create a resource if already - provided, this way we can conditionally create a resource only if the user didn't provide it - - Parameters - ---------- - stage: Stage - The pipeline stage to deploy the CFN template to its associated AWS account and region. - deployer_arn: str - The ARN of the deployer IAM user. This is used by the CFN template to give this IAM user permissions to - assume the IAM roles. - - Returns - ------- - ARNs of the deployer execution role, CLoudFormation execution role, artifacts S3 bucket and bucket KMS key. - """ - - if stage.did_user_provide_all_required_resources(): - LOG.info(f"All required resources for the {stage.name} stage exist, skipping creation.") - return ( - stage.deployer_role.arn, - stage.cfn_deployment_role.arn, - stage.artifacts_bucket.arn, - stage.artifacts_bucket.kms_key_arn, - ) - missing_resources: str = "" - if not stage.deployer_role.is_user_provided: - missing_resources += "\n\tDeployer role." - if not stage.cfn_deployment_role.is_user_provided: - missing_resources += "\n\tCloudFormation deployment role." - if not stage.artifacts_bucket.is_user_provided: - missing_resources += "\n\tArtifacts bucket." - LOG.info(f"Creating missing required resources for the {stage.name} stage: {missing_resources}") - stage_resources_template_path: str = os.path.join(CFN_TEMPLATE_PATH, "stage_resources.yaml") - stack_name: str = f"{STACK_NAME_PREFIX}-{stage.stack_name}-{STAGE_RESOURCES_STACK_NAME_SUFFIX}" - with open(stage_resources_template_path, "r") as fp: - stage_resources_template_body = fp.read() - output: List[Dict[str, str]] = manage_cloudformation_stack( - stack_name=stack_name, - region=stage.aws_region, - profile=stage.aws_profile, - template_body=stage_resources_template_body, - parameter_overrides={ - "DeployerArn": deployer_arn, - "DeployerRoleArn": stage.deployer_role.arn, - "CFNDeploymentRoleArn": stage.cfn_deployment_role.arn, - "ArtifactsBucketArn": stage.artifacts_bucket.arn, - }, - ) - - deployer_role_arn: str = next(o for o in output if o.get("OutputKey") == "DeployerRole").get("OutputValue") - cfn_deployment_role_arn: str = next(o for o in output if o.get("OutputKey") == "CFNDeploymentRole").get( - "OutputValue" - ) - artifacts_bucket_arn: str = next(o for o in output if o.get("OutputKey") == "ArtifactsBucket").get( - "OutputValue" - ) - try: - artifacts_bucket_key_arn: str = next(o for o in output if o.get("OutputKey") == "ArtifactsBucketKey").get( - "OutputValue" - ) - except StopIteration: - artifacts_bucket_key_arn = None - - return deployer_role_arn, cfn_deployment_role_arn, artifacts_bucket_arn, artifacts_bucket_key_arn diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/questions.json b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/questions.json deleted file mode 100644 index 75ccc6a9636..00000000000 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/questions.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "questions": [ - { - "key": "sam_template", - "question": "SAM template", - "default": "template.yaml" - }, - { - "key": "main_git_branch", - "question": "git branch to trigger the pipeline", - "isRequired": true - }, - { - "key": "testing_intro", - "question": "Let us configure your testing stage first.", - "kind": "info" - }, - { - "key": "testing_profile", - "question": "Which AWS profile will you deploy the 'Testing' stage to?", - "isRequired": true, - "options": %(aws_profiles)s - }, - { - "key": "deployer_provided", - "question": "We need an IAM user which will be used to initiate deployment.", - "isRequired": true, - "options": [ - "I already have IAM user configured in %(provider)s", - "I need help creating one" - ], - "nextQuestion": { - "I already have IAM user configured in %(provider)s": "deployer_arn", - "I need help creating one": "creating_deployer_message" - } - }, - { - "key": "deployer_arn", - "question": "IAM user ARN?", - "isRequired": true, - "defaultNextQuestion": "deployer_aws_access_key_id_variable_name" - }, - { - "key": "creating_deployer_message", - "question": "We are going to create an IAM user for you.", - "kind": "info" - }, - { - "key": "deployer_aws_access_key_id_variable_name", - "question": "%(provider)s variable name for AWS_ACCESS_KEY_ID", - "isRequired": true - }, - { - "key": "deployer_aws_secret_access_key_variable_name", - "question": "%(provider)s variable name for AWS_SECRET_ACCESS_KEY", - "isRequired": true - }, - { - "key": "testing_region", - "question": "Enter region for 'Testing' stage", - "default": "us-east-1" - }, - { - "key": "testing_stack_name", - "question": "Enter stack name for 'Testing' stage", - "default": "testing-stack", - }, - { - "key": "testing_artifacts_bucket", - "question": "We need an S3 bucket in testing account to upload artifacts.\nS3 Bucket ARN [leave blank to create one]" - }, - { - "key": "testing_deployer_role", - "question": "Deployer Execution Role [leave blank to create one]" - }, - { - "key": "testing_cfn_deployment_role", - "question": "Cloudformation Deployment Role [leave blank to create one]" - }, - { - "key": "prod_intro", - "question": "NICE!! Let us configure your prod stage as well.", - "kind": "info" - }, - { - "key": "prod_profile", - "question": "Which AWS profile will you deploy the 'Prod' stage to?", - "options": %(aws_profiles)s, - "isRequired": true - }, - { - "key": "prod_region", - "question": "Enter region for 'Prod' stage", - "default": "us-east-1", - }, - { - "key": "prod_stack_name", - "question": "Enter stack name for 'Prod' stage", - "default": "prod-stack", - }, - { - "key": "prod_artifacts_bucket", - "question": "S3 Bucket ARN [leave blank to create one]" - }, - { - "key": "prod_deployer_role", - "question": "Deployer Execution Role [leave blank to create one]" - }, - { - "key": "prod_cfn_deployment_role", - "question": "Cloudformation Deployment Role [leave blank to create one]" - } - ] -} diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/resource.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/resource.py deleted file mode 100644 index 25e119e6477..00000000000 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/resource.py +++ /dev/null @@ -1,26 +0,0 @@ -""" AWS resource represented by ARN""" -from typing import Optional - - -class Resource: - def __init__(self, arn: str) -> None: - self.arn: str = arn - self.is_user_provided: bool = bool(arn) - - def name(self) -> Optional[str]: - if self.arn: - return self.arn.split(":")[-1] - return None - - -class Deployer(Resource): - def __init__(self, arn: str, access_key_id: Optional[str] = None, secret_access_key: Optional[str] = None) -> None: - self.access_key_id: Optional[str] = access_key_id - self.secret_access_key: Optional[str] = secret_access_key - super().__init__(arn=arn) - - -class S3Bucket(Resource): - def __init__(self, arn: str, kms_key_arn: Optional[str] = None) -> None: - self.kms_key_arn: Optional[str] = kms_key_arn - super().__init__(arn=arn) diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/stage.py b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/stage.py deleted file mode 100644 index bba33956ffb..00000000000 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/stage.py +++ /dev/null @@ -1,155 +0,0 @@ -""" Pipeline stage""" -from .resource import Resource, S3Bucket - - -class Stage: - """ - Represents a pipeline stage - - Attributes - ---------- - name: str - The name of the stage - aws_profile: str - The named AWS profile(in user's machine) of the AWS account to deploy this stage to. - aws_region: str - The AWS region to deploy this stage to. - stack_name: str - The stack-name to be used for deploying the application's CFN template to this stage. - deployer_role: Resource - The IAM role assumed by the pipeline's deployer IAM user to get access to the AWS account and executes the - CloudFormation stack. - cfn_deployment_role: Resource - The IAM role assumed by the CloudFormation service to executes the CloudFormation stack. - artifacts_bucket: S3Bucket - The S3 bucket to hold the SAM build artifacts of the application's CFN template. - - Methods: - did_user_provide_all_required_resources() -> bool: - checks if all of the stage requires resources(deployer_role, cfn_deployment_role, artifacts_bucket) are provided - by the user. - deployer_role_permissions(deployer_arn): - returns a string of the permissions(IAM policies) required for the deployer_role to operate as expected. - cfn_deployment_role_permissions(): - returns a string of the permissions(IAM policies) required for the cfn_deployment_role to operate as expected. - artifacts_bucket_permissions(): - returns a string of the permissions(IAM policies) required for the artifacts_bucket to operate as expected. - - """ - - def __init__( - self, - name: str, - aws_profile: str, - aws_region: str, - stack_name: str, - deployer_role_arn: str, - cfn_deployment_role_arn: str, - artifacts_bucket_arn: str, - ) -> None: - self.name: str = name - self.aws_profile: str = aws_profile - self.aws_region: str = aws_region - self.stack_name: str = stack_name - self.deployer_role: Resource = Resource(arn=deployer_role_arn) - self.cfn_deployment_role: Resource = Resource(arn=cfn_deployment_role_arn) - self.artifacts_bucket: S3Bucket = S3Bucket(arn=artifacts_bucket_arn) - - def did_user_provide_all_required_resources(self) -> bool: - return ( - self.artifacts_bucket.is_user_provided - and self.deployer_role.is_user_provided - and self.cfn_deployment_role.is_user_provided - ) - - def deployer_role_permissions(self, deployer_arn: str) -> str: - permissions: str = f""" -AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - AWS: {deployer_arn} - Action: - - 'sts:AssumeRole' -Policies: - - PolicyName: AccessRolePolicy - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - 'iam:PassRole' - Resource: - - "{self.cfn_deployment_role.arn}" - - Effect: Allow - Action: - - "cloudformation:CreateChangeSet" - - "cloudformation:DescribeChangeSet" - - "cloudformation:ExecuteChangeSet" - - "cloudformation:DescribeStackEvents" - - "cloudformation:DescribeStacks" - - "cloudformation:GetTemplateSummary" - - "cloudformation:DescribeStackResource" - Resource: '*' - - Effect: Allow - Action: - - 's3:GetObject*' - - 's3:PutObject*' - - 's3:GetBucket*' - - 's3:List*' - Resource: - - {self.artifacts_bucket.arn} - - {self.artifacts_bucket.arn}/* - """ - if not self.artifacts_bucket.is_user_provided: - permissions += f""" - - Effect: Allow - Action: - - "kms:Decrypt" - - "kms:DescribeKey" - Resource: - - {self.artifacts_bucket.kms_key_arn} - """ - return permissions - - def cfn_deployment_role_permissions(self) -> str: - permissions: str = """ -AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: cloudformation.amazonaws.com - Action: - - 'sts:AssumeRole' -Policies: - - PolicyName: GrantCloudFormationFullAccess - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: '*' - Resource: '*' - """ - return permissions - - def artifacts_bucket_permissions(self) -> str: - permissions: str = f""" -PolicyDocument: - Statement: - - Effect: "Allow" - Action: - - 's3:GetObject*' - - 's3:PutObject*' - - 's3:GetBucket*' - - 's3:List*' - Resource: - - {self.artifacts_bucket.arn} - - {self.artifacts_bucket.arn}/* - Principal: - AWS: - - {self.deployer_role.arn} - - {self.cfn_deployment_role.arn} - """ - return permissions diff --git a/tests/unit/lib/pipeline/init/__init__.py b/tests/unit/lib/pipeline/init/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/unit/lib/pipeline/init/plugins/__init__.py b/tests/unit/lib/pipeline/init/plugins/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_context.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_context.py deleted file mode 100644 index 5fadc296e63..00000000000 --- a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_context.py +++ /dev/null @@ -1,161 +0,0 @@ -from unittest import TestCase -from samcli.lib.pipeline.init.plugins.two_stages_pipeline.context import Context # type: ignore - - -ANY_TESTING_PROFILE = "ANY_TESTING_PROFILE" -ANY_TESTING_REGION = "ANY_TESTING_REGION" -ANY_TESTING_STACK_NAME = "ANY_TESTING_STACK_NAME" -ANY_TESTING_DEPLOYER_ROLE = "ANY:TESTING:DEPLOYER_ROLE" -ANY_TESTING_CFN_DEPLOYMENT_ROLE = "ANY:TESTING:CFN_DEPLOYMENT_ROLE" -ANY_TESTING_ARTIFACTS_BUCKET = "ANY:TESTING:ARTIFACTS_BUCKET" -ANY_PROD_PROFILE = "ANY_PROD_PROFILE" -ANY_PROD_REGION = "ANY_PROD_REGION" -ANY_PROD_STACK_NAME = "ANY_PROD_STACK_NAME" -ANY_PROD_DEPLOYER_ROLE = "ANY:PROD:DEPLOYER_ROLE" -ANY_PROD_CFN_DEPLOYMENT_ROLE = "ANY:PROD:CFN_DEPLOYMENT_ROLE" -ANY_PROD_ARTIFACTS_BUCKET = "ANY:PROD:ARTIFACTS_BUCKET" -ANY_DEPLOYER_ARN = "ANY:DEPLOYER_ARN" -ANY_DEPLOYER_AWS_ACCESS_KEY_ID_VARIABLE_NAME = "ANY_DEPLOYER_AWS_ACCESS_KEY_ID_VARIABLE_NAME" -ANY_DEPLOYER_AWS_SECRET_ACCESS_KEY_VARIABLE_NAME = "ANY_DEPLOYER_AWS_SECRET_ACCESS_KEY_VARIABLE_NAME" - -ANY_CONTEXT = { - "testing_profile": ANY_TESTING_PROFILE, - "testing_region": ANY_TESTING_REGION, - "testing_stack_name": ANY_TESTING_STACK_NAME, - "testing_deployer_role": ANY_TESTING_DEPLOYER_ROLE, - "testing_cfn_deployment_role": ANY_TESTING_CFN_DEPLOYMENT_ROLE, - "testing_artifacts_bucket": ANY_TESTING_ARTIFACTS_BUCKET, - "prod_profile": ANY_PROD_PROFILE, - "prod_region": ANY_PROD_REGION, - "prod_stack_name": ANY_PROD_STACK_NAME, - "prod_deployer_role": ANY_PROD_DEPLOYER_ROLE, - "prod_cfn_deployment_role": ANY_PROD_CFN_DEPLOYMENT_ROLE, - "prod_artifacts_bucket": ANY_PROD_ARTIFACTS_BUCKET, - "deployer_arn": ANY_DEPLOYER_ARN, - "deployer_aws_access_key_id_variable_name": ANY_DEPLOYER_AWS_ACCESS_KEY_ID_VARIABLE_NAME, - "deployer_aws_secret_access_key_variable_name": ANY_DEPLOYER_AWS_SECRET_ACCESS_KEY_VARIABLE_NAME, -} - - -class TestContext(TestCase): - def test_init_with_all_keys_provided(self): - context: Context = Context(ANY_CONTEXT) - testing_stage = context.get_stage(Context.TESTING_STAGE_NAME) - prod_stage = context.get_stage(Context.PROD_STAGE_NAME) - - self.assertEqual(testing_stage.name, Context.TESTING_STAGE_NAME) - self.assertEqual(testing_stage.aws_profile, ANY_TESTING_PROFILE) - self.assertEqual(testing_stage.aws_region, ANY_TESTING_REGION) - self.assertEqual(testing_stage.stack_name, ANY_TESTING_STACK_NAME) - self.assertEqual(testing_stage.deployer_role.arn, ANY_TESTING_DEPLOYER_ROLE) - self.assertEqual(testing_stage.deployer_role.name(), "DEPLOYER_ROLE") - self.assertTrue(testing_stage.deployer_role.is_user_provided) - self.assertEqual(testing_stage.cfn_deployment_role.arn, ANY_TESTING_CFN_DEPLOYMENT_ROLE) - self.assertEqual(testing_stage.cfn_deployment_role.name(), "CFN_DEPLOYMENT_ROLE") - self.assertTrue(testing_stage.cfn_deployment_role.is_user_provided) - self.assertEqual(testing_stage.artifacts_bucket.arn, ANY_TESTING_ARTIFACTS_BUCKET) - self.assertEqual(testing_stage.artifacts_bucket.name(), "ARTIFACTS_BUCKET") - self.assertTrue(testing_stage.artifacts_bucket.is_user_provided) - self.assertIsNone(testing_stage.artifacts_bucket.kms_key_arn) - - self.assertEqual(prod_stage.name, Context.PROD_STAGE_NAME) - self.assertEqual(prod_stage.aws_profile, ANY_PROD_PROFILE) - self.assertEqual(prod_stage.aws_region, ANY_PROD_REGION) - self.assertEqual(prod_stage.stack_name, ANY_PROD_STACK_NAME) - self.assertEqual(prod_stage.deployer_role.arn, ANY_PROD_DEPLOYER_ROLE) - self.assertEqual(prod_stage.deployer_role.name(), "DEPLOYER_ROLE") - self.assertTrue(prod_stage.deployer_role.is_user_provided) - self.assertEqual(prod_stage.cfn_deployment_role.arn, ANY_PROD_CFN_DEPLOYMENT_ROLE) - self.assertEqual(prod_stage.cfn_deployment_role.name(), "CFN_DEPLOYMENT_ROLE") - self.assertTrue(prod_stage.cfn_deployment_role.is_user_provided) - self.assertEqual(prod_stage.artifacts_bucket.arn, ANY_PROD_ARTIFACTS_BUCKET) - self.assertEqual(prod_stage.artifacts_bucket.name(), "ARTIFACTS_BUCKET") - self.assertTrue(prod_stage.artifacts_bucket.is_user_provided) - self.assertIsNone(prod_stage.artifacts_bucket.kms_key_arn) - - self.assertEqual(context.deployer.arn, ANY_DEPLOYER_ARN) - self.assertEqual(context.deployer.name(), "DEPLOYER_ARN") - self.assertTrue(context.deployer.is_user_provided) - self.assertIsNone(context.deployer.access_key_id) - self.assertIsNone(context.deployer.secret_access_key) - - self.assertEqual(context.deployer_aws_access_key_id_variable_name, ANY_DEPLOYER_AWS_ACCESS_KEY_ID_VARIABLE_NAME) - self.assertEqual( - context.deployer_aws_secret_access_key_variable_name, ANY_DEPLOYER_AWS_SECRET_ACCESS_KEY_VARIABLE_NAME - ) - self.assertIsNone(context.build_image) - - def test_init_with_no_keys_provided(self): - context: Context = Context({}) - testing_stage = context.get_stage(Context.TESTING_STAGE_NAME) - prod_stage = context.get_stage(Context.PROD_STAGE_NAME) - - self.assertEqual(testing_stage.name, Context.TESTING_STAGE_NAME) - self.assertIsNone(testing_stage.aws_profile) - self.assertIsNone(testing_stage.aws_region) - self.assertIsNone(testing_stage.stack_name) - self.assertIsNone(testing_stage.deployer_role.arn) - self.assertIsNone(testing_stage.deployer_role.name()) - self.assertFalse(testing_stage.deployer_role.is_user_provided) - self.assertIsNone(testing_stage.cfn_deployment_role.arn) - self.assertIsNone(testing_stage.cfn_deployment_role.name()) - self.assertFalse(testing_stage.cfn_deployment_role.is_user_provided) - self.assertIsNone(testing_stage.artifacts_bucket.arn) - self.assertIsNone(testing_stage.artifacts_bucket.name()) - self.assertFalse(testing_stage.artifacts_bucket.is_user_provided) - self.assertIsNone(testing_stage.artifacts_bucket.kms_key_arn) - - self.assertEqual(prod_stage.name, Context.PROD_STAGE_NAME) - self.assertIsNone(prod_stage.aws_profile) - self.assertIsNone(prod_stage.aws_region) - self.assertIsNone(prod_stage.stack_name) - self.assertIsNone(prod_stage.deployer_role.arn) - self.assertIsNone(prod_stage.deployer_role.name()) - self.assertFalse(prod_stage.deployer_role.is_user_provided) - self.assertIsNone(prod_stage.cfn_deployment_role.arn) - self.assertIsNone(prod_stage.cfn_deployment_role.name()) - self.assertFalse(prod_stage.cfn_deployment_role.is_user_provided) - self.assertIsNone(prod_stage.artifacts_bucket.arn) - self.assertIsNone(prod_stage.artifacts_bucket.name()) - self.assertFalse(prod_stage.artifacts_bucket.is_user_provided) - self.assertIsNone(prod_stage.artifacts_bucket.kms_key_arn) - - self.assertIsNone(context.deployer.arn) - self.assertIsNone(context.deployer.name()) - self.assertFalse(context.deployer.is_user_provided) - self.assertIsNone(context.deployer.access_key_id) - self.assertIsNone(context.deployer.secret_access_key) - - self.assertIsNone(context.deployer_aws_access_key_id_variable_name) - self.assertIsNone(context.deployer_aws_secret_access_key_variable_name) - self.assertIsNone(context.build_image) - - def test_init_with_none_context_raises_exception(self): - with self.assertRaises(AttributeError): - context: Context = Context(None) - - def test_get_stage(self): - context: Context = Context({}) - testing_stage = context.get_stage(Context.TESTING_STAGE_NAME) - prod_stage = context.get_stage(Context.PROD_STAGE_NAME) - none_stage1 = context.get_stage("non-existing-stage") - none_stage2 = context.get_stage(None) - self.assertEqual(testing_stage.name, Context.TESTING_STAGE_NAME) - self.assertEqual(prod_stage.name, Context.PROD_STAGE_NAME) - self.assertIsNone(none_stage1) - self.assertIsNone(none_stage2) - - def test_deployer_permissions(self): - context: Context = Context(ANY_CONTEXT) - permissions: str = context.deployer_permissions() - # assert the deployer has assume-rule access to the deployer roles - self.assertIn("sts:AssumeRole", permissions) - self.assertIn(ANY_TESTING_DEPLOYER_ROLE, permissions) - self.assertIn(ANY_PROD_DEPLOYER_ROLE, permissions) - - def test_deployer_permissions_with_empty_context(self): - context: Context = Context({}) - permissions: str = context.deployer_permissions() - self.assertIn("sts:AssumeRole", permissions) - self.assertNotIn(ANY_TESTING_DEPLOYER_ROLE, permissions) - self.assertNotIn(ANY_PROD_DEPLOYER_ROLE, permissions) diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py deleted file mode 100644 index 4b80611f17b..00000000000 --- a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_postprocessor.py +++ /dev/null @@ -1,101 +0,0 @@ -from unittest import TestCase -from unittest.mock import Mock -from samcli.lib.pipeline.init.plugins.two_stages_pipeline.config import PLUGIN_NAME # type: ignore -from samcli.lib.pipeline.init.plugins.two_stages_pipeline.context import Context as PluginContext # type: ignore -from samcli.lib.pipeline.init.plugins.two_stages_pipeline.postprocessor import Postprocessor # type: ignore -from .test_context import ( - ANY_DEPLOYER_ARN, - ANY_TESTING_DEPLOYER_ROLE, - ANY_TESTING_CFN_DEPLOYMENT_ROLE, - ANY_TESTING_ARTIFACTS_BUCKET, - ANY_PROD_DEPLOYER_ROLE, - ANY_PROD_CFN_DEPLOYMENT_ROLE, - ANY_PROD_ARTIFACTS_BUCKET, -) - - -class TestPostprocessor(TestCase): - def test_the_postprocessoer_correctly_categorize_the_resources_into_reused_and_created_resources(self): - # setup - deployer = Mock() - deployer.is_user_provided = False - deployer.arn = ANY_DEPLOYER_ARN - - testing_deployer_role = Mock() - testing_deployer_role.is_user_provided = True - testing_deployer_role.arn = ANY_TESTING_DEPLOYER_ROLE - - testing_cfn_deployment_role = Mock() - testing_cfn_deployment_role.is_user_provided = True - testing_cfn_deployment_role.arn = ANY_TESTING_CFN_DEPLOYMENT_ROLE - - testing_artifacts_bucket = Mock() - testing_artifacts_bucket.is_user_provided = True - testing_artifacts_bucket.arn = ANY_TESTING_ARTIFACTS_BUCKET - - testing_stage = Mock() - testing_stage.deployer_role = testing_deployer_role - testing_stage.cfn_deployment_role = testing_cfn_deployment_role - testing_stage.artifacts_bucket = testing_artifacts_bucket - - prod_deployer_role = Mock() - prod_deployer_role.is_user_provided = False - prod_deployer_role.arn = ANY_PROD_DEPLOYER_ROLE - - prod_cfn_deployment_role = Mock() - prod_cfn_deployment_role.is_user_provided = False - prod_cfn_deployment_role.arn = ANY_PROD_CFN_DEPLOYMENT_ROLE - - prod_artifacts_bucket = Mock() - prod_artifacts_bucket.is_user_provided = False - prod_artifacts_bucket.arn = ANY_PROD_ARTIFACTS_BUCKET - - prod_stage = Mock() - prod_stage.deployer_role = prod_deployer_role - prod_stage.cfn_deployment_role = prod_cfn_deployment_role - prod_stage.artifacts_bucket = prod_artifacts_bucket - - plugin_context = Mock() - plugin_context.deployer = deployer - plugin_context.stages = [testing_stage, prod_stage] - - context = {PLUGIN_NAME: plugin_context} - postprocessor = Postprocessor() - - self.assertEqual(0, len(postprocessor.resources_provided)) - self.assertEqual(0, len(postprocessor.resources_created)) - - # trigger - mutated_context = postprocessor.run(context=context) - - # verify - reused_resources_arns = list(map(lambda r: r["arn"], postprocessor.resources_provided)) - self.assertEqual(3, len(reused_resources_arns)) - self.assertIn(ANY_TESTING_DEPLOYER_ROLE, reused_resources_arns) - self.assertIn(ANY_TESTING_CFN_DEPLOYMENT_ROLE, reused_resources_arns) - self.assertIn(ANY_TESTING_ARTIFACTS_BUCKET, reused_resources_arns) - - created_resources_arns = list(map(lambda r: r["arn"], postprocessor.resources_created)) - self.assertEqual(4, len(created_resources_arns)) - self.assertIn(ANY_DEPLOYER_ARN, created_resources_arns) - self.assertIn(ANY_PROD_DEPLOYER_ROLE, created_resources_arns) - self.assertIn(ANY_PROD_CFN_DEPLOYMENT_ROLE, created_resources_arns) - self.assertIn(ANY_PROD_ARTIFACTS_BUCKET, created_resources_arns) - - def test_the_plugin_context_is_required(self): - postprocessor: Postprocessor = Postprocessor() - with (self.assertRaises(KeyError)): - postprocessor.run(context={}) - - def test_the_postprocessoer_return_a_new_copy_of_the_context_without_modifications(self): - # setup - plugin_context = PluginContext({}) - context = {PLUGIN_NAME: plugin_context} - postprocessor: Postprocessor = Postprocessor() - - # trigger - mutated_context = postprocessor.run(context=context) - - # verify - self.assertIsNot(context, mutated_context) - self.assertEqual(context, mutated_context) diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_preprocessor.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_preprocessor.py deleted file mode 100644 index 8dbc1acc8da..00000000000 --- a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_preprocessor.py +++ /dev/null @@ -1,440 +0,0 @@ -from unittest import TestCase -from unittest.mock import ANY, Mock, patch -from samcli.local.common.runtime_template import RUNTIME_TO_BUILD_IMAGE -from samcli.lib.pipeline.init.plugins.two_stages_pipeline.config import PLUGIN_NAME # type: ignore -from samcli.lib.pipeline.init.plugins.two_stages_pipeline.context import Context # type: ignore -from samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor import Preprocessor # type: ignore -from samcli.lib.pipeline.init.plugins.two_stages_pipeline.stage import Stage # type: ignore -from .test_context import ( - ANY_TESTING_DEPLOYER_ROLE, - ANY_TESTING_CFN_DEPLOYMENT_ROLE, - ANY_TESTING_ARTIFACTS_BUCKET, - ANY_DEPLOYER_ARN, -) - -# Let the user provide the deployer and the testing-stage resources and the plugin creates the prod-resources -plugin_context_with_deployer_and_testing_resources_but_no_prod_resources = { - "testing_deployer_role": ANY_TESTING_DEPLOYER_ROLE, - "testing_cfn_deployment_role": ANY_TESTING_CFN_DEPLOYMENT_ROLE, - "testing_artifacts_bucket": ANY_TESTING_ARTIFACTS_BUCKET, - "deployer_arn": ANY_DEPLOYER_ARN, -} -ANY_SAM_TEMPLATE = "any/sam/template.yaml" -context = {"sam_template": ANY_SAM_TEMPLATE} -context.update(plugin_context_with_deployer_and_testing_resources_but_no_prod_resources) -preprocessor: Preprocessor = Preprocessor() - - -class TestPreprocessor(TestCase): - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.get_template_function_runtimes") - def test_used_build_image_when_sam_template_has_no_runtimes(self, get_template_function_runtimes_mock): - # setup - get_template_function_runtimes_mock.return_value = [] - # trigger - build_image = preprocessor._get_build_image(sam_template_file=ANY_SAM_TEMPLATE) - # verify - self.assertEqual(build_image, Preprocessor.BASIC_PROVIDED_BUILD_IMAGE) - - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.get_template_function_runtimes") - def test_used_build_image_when_sam_template_has_one_supported_runtime(self, get_template_function_runtimes_mock): - # setup - get_template_function_runtimes_mock.return_value = ["python3.8"] - # trigger - build_image = preprocessor._get_build_image(sam_template_file=ANY_SAM_TEMPLATE) - # verify - self.assertEqual(build_image, RUNTIME_TO_BUILD_IMAGE["python3.8"]) - - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.get_template_function_runtimes") - def test_used_build_image_when_sam_template_has_one_unsupported_runtime( - self, get_template_function_runtimes_mock, click_mock - ): - # setup - click_mock.prompt.return_value = "user-provided-build-image" - get_template_function_runtimes_mock.return_value = ["any-unsupported-runtime"] - # trigger - build_image = preprocessor._get_build_image(sam_template_file=ANY_SAM_TEMPLATE) - # verify - click_mock.prompt.assert_called_once() - self.assertEqual(build_image, "user-provided-build-image") - - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.get_template_function_runtimes") - def test_used_build_image_when_sam_template_has_multiple_runtimes( - self, get_template_function_runtimes_mock, click_mock - ): - # setup - click_mock.prompt.return_value = "user-provided-build-image" - get_template_function_runtimes_mock.return_value = ["python3.8", "python3.7", "any-unsupported-runtime"] - # trigger - build_image = preprocessor._get_build_image(sam_template_file=ANY_SAM_TEMPLATE) - # verify - click_mock.prompt.assert_called_once() - self.assertEqual(build_image, "user-provided-build-image") - - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") - def test_create_deployer(self, manage_cloudformation_stack_mock, click_mock): - # setup - stage_mock = Mock() - stage_mock.profile.return_value = "any-profile" - stage_mock.region.return_value = "any-region" - stage_mock.stack_name.return_value = "any-stack-name" - manage_cloudformation_stack_mock.return_value = [ - {"OutputKey": "Deployer", "OutputValue": "any-deployer-arn"}, - {"OutputKey": "AccessKeyId", "OutputValue": "any-access-key-id"}, - {"OutputKey": "SecretAccessKey", "OutputValue": "any-secret-access-key"}, - ] - - # trigger - ( - deployer_arn, - access_key_id_arn, - secret_access_key_arn, - ) = preprocessor._create_deployer_at(stage=stage_mock) - - # verify - self.assertEqual(deployer_arn, "any-deployer-arn") - self.assertEqual(access_key_id_arn, "any-access-key-id") - self.assertEqual(secret_access_key_arn, "any-secret-access-key") - - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") - def test_create_missing_stage_resources_when_all_resources_are_missed( - self, manage_cloudformation_stack_mock, click_mock - ): - # setup - stage = Stage( - name="any-name", - aws_profile="any-profile", - aws_region="any-region", - stack_name="any-stack-name", - deployer_role_arn=None, - cfn_deployment_role_arn=None, - artifacts_bucket_arn=None, - ) - - manage_cloudformation_stack_mock.return_value = [ - {"OutputKey": "DeployerRole", "OutputValue": "new-deployer-role-arn"}, - {"OutputKey": "CFNDeploymentRole", "OutputValue": "new-cfn-deployment-role-arn"}, - {"OutputKey": "ArtifactsBucket", "OutputValue": "new-artifacts-bucket-arn"}, - {"OutputKey": "ArtifactsBucketKey", "OutputValue": "new-artifacts-bucket-key-arn"}, - ] - - # trigger - ( - deployer_role_arn, - cfn_deployment_role_arn, - artifacts_bucket_arn, - artifacts_bucket_key_arn, - ) = preprocessor._create_missing_stage_resources(stage=stage, deployer_arn="any-deployer-arn") - - # verify - self.assertEqual(deployer_role_arn, "new-deployer-role-arn") - self.assertEqual(cfn_deployment_role_arn, "new-cfn-deployment-role-arn") - self.assertEqual(artifacts_bucket_arn, "new-artifacts-bucket-arn") - self.assertEqual(artifacts_bucket_key_arn, "new-artifacts-bucket-key-arn") - manage_cloudformation_stack_mock.assert_called_once_with( - stack_name="aws-sam-cli-managed-any-stack-name-pipeline-resources", - region="any-region", - profile="any-profile", - template_body=ANY, - parameter_overrides={ - "DeployerArn": "any-deployer-arn", - "DeployerRoleArn": None, - "CFNDeploymentRoleArn": None, - "ArtifactsBucketArn": None, - }, - ) - - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") - def test_create_missing_stage_resources_when_only_deployer_role_is_missed( - self, manage_cloudformation_stack_mock, click_mock - ): - # setup - stage = Stage( - name="any-name", - aws_profile="any-profile", - aws_region="any-region", - stack_name="any-stack-name", - deployer_role_arn=None, - cfn_deployment_role_arn="existing-cfn-deployment-role-arn", - artifacts_bucket_arn="existing-artifacts-bucket-arn", - ) - - manage_cloudformation_stack_mock.return_value = [ - {"OutputKey": "DeployerRole", "OutputValue": "new-deployer-role-arn"}, - {"OutputKey": "CFNDeploymentRole", "OutputValue": "existing-cfn-deployment-role-arn"}, - {"OutputKey": "ArtifactsBucket", "OutputValue": "existing-artifacts-bucket-arn"}, - ] - - # trigger - ( - deployer_role_arn, - cfn_deployment_role_arn, - artifacts_bucket_arn, - artifacts_bucket_key_arn, - ) = preprocessor._create_missing_stage_resources(stage=stage, deployer_arn="any-deployer-arn") - - # verify - self.assertEqual(deployer_role_arn, "new-deployer-role-arn") - self.assertEqual(cfn_deployment_role_arn, "existing-cfn-deployment-role-arn") - self.assertEqual(artifacts_bucket_arn, "existing-artifacts-bucket-arn") - # if we didn't create the artifacts bucket then we don't know about the user's bucket encryption method - self.assertIsNone(artifacts_bucket_key_arn) - manage_cloudformation_stack_mock.assert_called_once_with( - stack_name="aws-sam-cli-managed-any-stack-name-pipeline-resources", - region="any-region", - profile="any-profile", - template_body=ANY, - parameter_overrides={ - "DeployerArn": "any-deployer-arn", - "DeployerRoleArn": None, - "CFNDeploymentRoleArn": "existing-cfn-deployment-role-arn", - "ArtifactsBucketArn": "existing-artifacts-bucket-arn", - }, - ) - - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") - def test_create_missing_stage_resources_when_only_cfn_deployment_role_is_missed( - self, manage_cloudformation_stack_mock, click_mock - ): - # setup - stage = Stage( - name="any-name", - aws_profile="any-profile", - aws_region="any-region", - stack_name="any-stack-name", - deployer_role_arn="existing-deployer-role-arn", - cfn_deployment_role_arn=None, - artifacts_bucket_arn="existing-artifacts-bucket-arn", - ) - - manage_cloudformation_stack_mock.return_value = [ - {"OutputKey": "DeployerRole", "OutputValue": "existing-deployer-role-arn"}, - {"OutputKey": "CFNDeploymentRole", "OutputValue": "new-cfn-deployment-role-arn"}, - {"OutputKey": "ArtifactsBucket", "OutputValue": "existing-artifacts-bucket-arn"}, - ] - - # trigger - ( - deployer_role_arn, - cfn_deployment_role_arn, - artifacts_bucket_arn, - artifacts_bucket_key_arn, - ) = preprocessor._create_missing_stage_resources(stage=stage, deployer_arn="any-deployer-arn") - - # verify - self.assertEqual(deployer_role_arn, "existing-deployer-role-arn") - self.assertEqual(cfn_deployment_role_arn, "new-cfn-deployment-role-arn") - self.assertEqual(artifacts_bucket_arn, "existing-artifacts-bucket-arn") - # if we didn't create the artifacts bucket then we don't know about the user's bucket encryption method - self.assertIsNone(artifacts_bucket_key_arn) - manage_cloudformation_stack_mock.assert_called_once_with( - stack_name="aws-sam-cli-managed-any-stack-name-pipeline-resources", - region="any-region", - profile="any-profile", - template_body=ANY, - parameter_overrides={ - "DeployerArn": "any-deployer-arn", - "DeployerRoleArn": "existing-deployer-role-arn", - "CFNDeploymentRoleArn": None, - "ArtifactsBucketArn": "existing-artifacts-bucket-arn", - }, - ) - - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") - def test_create_missing_stage_resources_when_only_artifacts_bucket_is_missed( - self, manage_cloudformation_stack_mock, click_mock - ): - # setup - stage = Stage( - name="any-name", - aws_profile="any-profile", - aws_region="any-region", - stack_name="any-stack-name", - deployer_role_arn="existing-deployer-role-arn", - cfn_deployment_role_arn="existing-cfn-deployment-role-arn", - artifacts_bucket_arn=None, - ) - - manage_cloudformation_stack_mock.return_value = [ - {"OutputKey": "DeployerRole", "OutputValue": "existing-deployer-role-arn"}, - {"OutputKey": "CFNDeploymentRole", "OutputValue": "existing-cfn-deployment-role-arn"}, - {"OutputKey": "ArtifactsBucket", "OutputValue": "new-artifacts-bucket-arn"}, - {"OutputKey": "ArtifactsBucketKey", "OutputValue": "new-artifacts-bucket-key-arn"}, - ] - - # trigger - ( - deployer_role_arn, - cfn_deployment_role_arn, - artifacts_bucket_arn, - artifacts_bucket_key_arn, - ) = preprocessor._create_missing_stage_resources(stage=stage, deployer_arn="any-deployer-arn") - - # verify - self.assertEqual(deployer_role_arn, "existing-deployer-role-arn") - self.assertEqual(cfn_deployment_role_arn, "existing-cfn-deployment-role-arn") - self.assertEqual(artifacts_bucket_arn, "new-artifacts-bucket-arn") - self.assertEqual(artifacts_bucket_key_arn, "new-artifacts-bucket-key-arn") - manage_cloudformation_stack_mock.assert_called_once_with( - stack_name="aws-sam-cli-managed-any-stack-name-pipeline-resources", - region="any-region", - profile="any-profile", - template_body=ANY, - parameter_overrides={ - "DeployerArn": "any-deployer-arn", - "DeployerRoleArn": "existing-deployer-role-arn", - "CFNDeploymentRoleArn": "existing-cfn-deployment-role-arn", - "ArtifactsBucketArn": None, - }, - ) - - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.click") - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") - def test_create_missing_stage_resources_when_all_resources_are_provided( - self, manage_cloudformation_stack_mock, click_mock - ): - # setup - stage = Stage( - name="any-name", - aws_profile="any-profile", - aws_region="any-region", - stack_name="any-stack-name", - deployer_role_arn="existing-deployer-role-arn", - cfn_deployment_role_arn="existing-cfn-deployment-role-arn", - artifacts_bucket_arn="existing-artifacts-bucket-arn", - ) - - # trigger - ( - deployer_role_arn, - cfn_deployment_role_arn, - artifacts_bucket_arn, - artifacts_bucket_key_arn, - ) = preprocessor._create_missing_stage_resources(stage=stage, deployer_arn="any-deployer-arn") - - # verify - manage_cloudformation_stack_mock.assert_not_called() - self.assertEqual(deployer_role_arn, "existing-deployer-role-arn") - self.assertEqual(cfn_deployment_role_arn, "existing-cfn-deployment-role-arn") - self.assertEqual(artifacts_bucket_arn, "existing-artifacts-bucket-arn") - # if we didn't create the artifacts bucket then we don't know about the user's bucket encryption method - self.assertIsNone(artifacts_bucket_key_arn) - - @patch.object(Preprocessor, "_get_build_image") - @patch.object(Preprocessor, "_create_deployer_at") - @patch.object(Preprocessor, "_create_missing_stage_resources") - def test_run_creates_the_missing_deployer_at_the_testing_stage_aws_account( - self, create_missing_stage_resources_mock, create_deployer_at_mock, get_build_image_mock - ): - # setup - context_without_deployer = context.copy() - del context_without_deployer["deployer_arn"] - create_deployer_at_mock.return_value = ["ANY", "ANY", "ANY"] - create_missing_stage_resources_mock.return_value = ["ANY", "ANY", "ANY", "ANY"] - - # trigger - preprocessor.run(context=context_without_deployer) - - # verify - create_deployer_at_mock.assert_called_once() - _, kwargs = create_deployer_at_mock.call_args - stage = kwargs["stage"] - self.assertEqual(Context.TESTING_STAGE_NAME, stage.name) - - @patch.object(Preprocessor, "_get_build_image") - @patch.object(Preprocessor, "_create_deployer_at") - @patch.object(Preprocessor, "_create_missing_stage_resources") - def test_run_will_not_create_deployer_if_already_provided( - self, create_missing_stage_resources_mock, create_deployer_at_mock, get_build_image_mock - ): - # setup - create_deployer_at_mock.return_value = ["ANY", "ANY", "ANY"] - create_missing_stage_resources_mock.return_value = ["ANY", "ANY", "ANY", "ANY"] - - # trigger - self.assertIn("deployer_arn", context) - preprocessor.run(context=context) - - # verify - create_deployer_at_mock.assert_not_called() - - @patch.object(Preprocessor, "_get_build_image") - @patch.object(Preprocessor, "_create_deployer_at") - @patch("samcli.lib.pipeline.init.plugins.two_stages_pipeline.preprocessor.manage_cloudformation_stack") - def test_run_will_not_recreate_provided_resources( - self, manage_cloudformation_stack_mock, create_deployer_at_mock, get_build_image_mock - ): - # setup - context["prod_stack_name"] = "PROD-STACK" - create_deployer_at_mock.return_value = ["ANY", "ANY", "ANY"] - manage_cloudformation_stack_mock.return_value = [ - {"OutputKey": "DeployerRole", "OutputValue": "new-deployer-role-arn"}, - {"OutputKey": "CFNDeploymentRole", "OutputValue": "new-cfn-deployment-role-arn"}, - {"OutputKey": "ArtifactsBucket", "OutputValue": "new-artifacts-bucket-arn"}, - {"OutputKey": "ArtifactsBucketKey", "OutputValue": "new-artifacts-bucket-key-arn"}, - ] - - # trigger - self.assertIn("deployer_arn", context) - self.assertIn("testing_deployer_role", context) - self.assertIn("testing_cfn_deployment_role", context) - self.assertIn("testing_artifacts_bucket", context) - self.assertNotIn("prod_deployer_role", context) - self.assertNotIn("prod_cfn_deployment_role", context) - self.assertNotIn("prod_artifacts_bucket", context) - preprocessor.run(context=context) - - # verify - create_deployer_at_mock.assert_not_called() - manage_cloudformation_stack_mock.assert_called_once() # for prod stage only - _, kwargs = manage_cloudformation_stack_mock.call_args - self.assertEqual(kwargs["stack_name"], "aws-sam-cli-managed-PROD-STACK-pipeline-resources") - actual_parameter_overrides = kwargs["parameter_overrides"] - expected_parameter_overrides = { - "DeployerArn": ANY_DEPLOYER_ARN, - "DeployerRoleArn": None, - "CFNDeploymentRoleArn": None, - "ArtifactsBucketArn": None, - } - self.assertEqual(actual_parameter_overrides, expected_parameter_overrides) - - @patch.object(Preprocessor, "_get_build_image") - @patch.object(Preprocessor, "_create_deployer_at") - @patch.object(Preprocessor, "_create_missing_stage_resources") - def test_run_creates_and_mutate_new_copy_of_the_context( - self, create_missing_stage_resources_mock, create_deployer_at_mock, get_build_image_mock - ): - # setup - context = {"sam_template": ANY_SAM_TEMPLATE} - create_missing_stage_resources_mock.return_value = ["ANY", "ANY", "ANY", "ANY"] - create_deployer_at_mock.return_value = ["ANY", "ANY", "ANY"] - - # triggert - mutated_context = preprocessor.run(context=context) - - # verify - self.assertIsNot(context, mutated_context) - - self.assertNotIn(PLUGIN_NAME, context) - self.assertNotIn("build_image", context) - self.assertNotIn("testing_deployer_role", context) - self.assertNotIn("testing_cfn_deployment_role", context) - self.assertNotIn("testing_artifacts_bucket", context) - self.assertNotIn("prod_deployer_role", context) - self.assertNotIn("prod_cfn_deployment_role", context) - self.assertNotIn("prod_artifacts_bucket", context) - - self.assertIn(PLUGIN_NAME, mutated_context) - self.assertIn("build_image", mutated_context) - self.assertIn("testing_deployer_role", mutated_context) - self.assertIn("testing_cfn_deployment_role", mutated_context) - self.assertIn("testing_artifacts_bucket", mutated_context) - self.assertIn("prod_deployer_role", mutated_context) - self.assertIn("prod_cfn_deployment_role", mutated_context) - self.assertIn("prod_artifacts_bucket", mutated_context) diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_resource.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_resource.py deleted file mode 100644 index d246bc7f9de..00000000000 --- a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_resource.py +++ /dev/null @@ -1,49 +0,0 @@ -from unittest import TestCase -from samcli.lib.pipeline.init.plugins.two_stages_pipeline.resource import Resource, Deployer, S3Bucket # type: ignore - -VALID_AWS_ARN = "arn:partition:service:region:account-id:resource-id" -INVALID_AWS_ARN = "ARN" - - -class TestResource(TestCase): - def test_resource(self): - resource = Resource(arn=VALID_AWS_ARN) - self.assertEqual(resource.arn, VALID_AWS_ARN) - self.assertTrue(resource.is_user_provided) - self.assertEqual(resource.name(), "resource-id") - - resource = Resource(arn=INVALID_AWS_ARN) - self.assertEqual(resource.arn, INVALID_AWS_ARN) - self.assertTrue(resource.is_user_provided) - self.assertEqual(resource.name(), "ARN") - - resource = Resource(arn=None) - self.assertIsNone(resource.arn) - self.assertFalse(resource.is_user_provided) - self.assertIsNone(resource.name()) - - -class TestDeployerResource(TestCase): - def test_deployer_resource(self): - deployer = Deployer(arn=VALID_AWS_ARN) - self.assertEqual(deployer.arn, VALID_AWS_ARN) - self.assertTrue(deployer.is_user_provided) - self.assertEqual(deployer.name(), "resource-id") - self.assertIsNone(deployer.access_key_id) - self.assertIsNone(deployer.secret_access_key) - - deployer = Deployer(arn=VALID_AWS_ARN, access_key_id="access_key_id", secret_access_key="secret_access_key") - self.assertEqual("access_key_id", deployer.access_key_id) - self.assertEqual("secret_access_key", deployer.secret_access_key) - - -class TestS3BucketResource(TestCase): - def test_s3bucket_resource(self): - bucket = S3Bucket(arn=VALID_AWS_ARN) - self.assertEqual(bucket.arn, VALID_AWS_ARN) - self.assertTrue(bucket.is_user_provided) - self.assertEqual(bucket.name(), "resource-id") - self.assertIsNone(bucket.kms_key_arn) - - bucket = S3Bucket(arn=VALID_AWS_ARN, kms_key_arn="kms_key_arn") - self.assertEqual("kms_key_arn", bucket.kms_key_arn) diff --git a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_stage.py b/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_stage.py deleted file mode 100644 index 63859de3ad6..00000000000 --- a/tests/unit/lib/pipeline/init/plugins/two_stages_pipeline/test_stage.py +++ /dev/null @@ -1,70 +0,0 @@ -from unittest import TestCase -from samcli.lib.pipeline.init.plugins.two_stages_pipeline.stage import Stage # type: ignore - - -class TestStage(TestCase): - def test_init_stage(self): - stage = Stage( - name="any-name", - aws_profile="any-profile", - aws_region="any-region", - stack_name="any-stack-name", - deployer_role_arn="any-deployer-role-arn", - cfn_deployment_role_arn=None, - artifacts_bucket_arn=None, - ) - self.assertEqual(stage.name, "any-name") - self.assertEqual(stage.aws_profile, "any-profile") - self.assertEqual(stage.aws_region, "any-region") - self.assertEqual(stage.stack_name, "any-stack-name") - self.assertEqual(stage.deployer_role.arn, "any-deployer-role-arn") - self.assertTrue(stage.deployer_role.is_user_provided) - self.assertIsNone(stage.cfn_deployment_role.arn) - self.assertFalse(stage.cfn_deployment_role.is_user_provided) - self.assertIsNone(stage.artifacts_bucket.arn) - self.assertFalse(stage.artifacts_bucket.is_user_provided) - - def test_did_user_provide_all_required_resources(self): - stage = Stage( - name="any", - aws_profile="any", - aws_region="any", - stack_name="any", - deployer_role_arn=None, - cfn_deployment_role_arn=None, - artifacts_bucket_arn=None, - ) - self.assertFalse(stage.did_user_provide_all_required_resources()) - - stage = Stage( - name="any", - aws_profile="any", - aws_region="any", - stack_name="any", - deployer_role_arn="any-deployer-role-arn", - cfn_deployment_role_arn=None, - artifacts_bucket_arn=None, - ) - self.assertFalse(stage.did_user_provide_all_required_resources()) - - stage = Stage( - name="any", - aws_profile="any", - aws_region="any", - stack_name="any", - deployer_role_arn="any-deployer-role-arn", - cfn_deployment_role_arn="any-cfn-deployment-role-arn", - artifacts_bucket_arn=None, - ) - self.assertFalse(stage.did_user_provide_all_required_resources()) - - stage = Stage( - name="any", - aws_profile="any", - aws_region="any", - stack_name="any", - deployer_role_arn="any-deployer-role-arn", - cfn_deployment_role_arn="any-cfn-deployment-role-arn", - artifacts_bucket_arn="any-artifacts-bucket-arn", - ) - self.assertTrue(stage.did_user_provide_all_required_resources()) From 588d4abcbe71b6b3ea5a72bb9927bfc0b49ab2be Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 13 Apr 2021 17:27:39 -0700 Subject: [PATCH 07/31] remove 'get_template_function_runtimes' function as the decision is made to not process the SAM template during pipeline init which was the only place we use the function --- samcli/commands/_utils/template.py | 21 ---------------- samcli/local/common/runtime_template.py | 15 ------------ tests/unit/commands/_utils/test_template.py | 27 +++------------------ 3 files changed, 4 insertions(+), 59 deletions(-) diff --git a/samcli/commands/_utils/template.py b/samcli/commands/_utils/template.py index 4eeef7326e9..08c02836dad 100644 --- a/samcli/commands/_utils/template.py +++ b/samcli/commands/_utils/template.py @@ -4,7 +4,6 @@ import itertools import os import pathlib -from typing import List import jmespath import yaml @@ -301,23 +300,3 @@ def get_template_function_resource_ids(template_file, artifact): ]: _function_resource_ids.append(resource_id) return _function_resource_ids - - -def get_template_function_runtimes(template_file: str) -> List[str]: - """ - Get a list of function runtimes from template file. - Function resource types include - AWS::Lambda::Function - AWS::Serverless::Function - :param template_file: template file location. - :return: list of runtimes - """ - - template_dict = get_template_data(template_file=template_file) - _function_runtimes = set() - for _, resource in template_dict.get("Resources", {}).items(): - if resource.get("Type") in [AWS_SERVERLESS_FUNCTION, AWS_LAMBDA_FUNCTION]: - runtime = resource.get("Properties", {}).get("Runtime") - if runtime: # IMAGE functions don't define a runtime - _function_runtimes.add(runtime) - return list(_function_runtimes) diff --git a/samcli/local/common/runtime_template.py b/samcli/local/common/runtime_template.py index 8c30e48e6a9..2924a5aa9f8 100644 --- a/samcli/local/common/runtime_template.py +++ b/samcli/local/common/runtime_template.py @@ -96,21 +96,6 @@ def get_local_lambda_images_location(mapping, runtime): "java8.al2": ["maven", "gradle"], } -RUNTIME_TO_BUILD_IMAGE = { - "python3.8": "public.ecr.aws/sam/build-python3.8", - "python3.7": "public.ecr.aws/sam/build-python3.7", - "python3.6": "public.ecr.aws/sam/build-python3.6", - "python2.7": "public.ecr.aws/sam/build-python2.7", - "ruby2.5": "public.ecr.aws/sam/build-ruby2.5", - "ruby2.7": "public.ecr.aws/sam/build-ruby2.7", - "nodejs14.x": "public.ecr.aws/sam/build-nodejs14.x", - "nodejs12.x": "public.ecr.aws/sam/build-nodejs12.x", - "nodejs10.x": "public.ecr.aws/sam/build-nodejs10.x", - "java8": "public.ecr.aws/sam/build-java8", - "java11": "public.ecr.aws/sam/build-java11", - "java8.al2": "public.ecr.aws/sam/build-java8.al2", -} - SUPPORTED_DEP_MANAGERS: Set[str] = { c["dependency_manager"] # type: ignore for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))) diff --git a/tests/unit/commands/_utils/test_template.py b/tests/unit/commands/_utils/test_template.py index 55330531ed6..5c84dedb036 100644 --- a/tests/unit/commands/_utils/test_template.py +++ b/tests/unit/commands/_utils/test_template.py @@ -1,12 +1,10 @@ -import os import copy +import os +from unittest import TestCase +from unittest.mock import patch, mock_open, MagicMock -import jmespath import yaml from botocore.utils import set_value_from_jmespath - -from unittest import TestCase -from unittest.mock import patch, mock_open, MagicMock from parameterized import parameterized, param from samcli.commands._utils.resources import AWS_SERVERLESS_FUNCTION, AWS_SERVERLESS_API @@ -20,8 +18,7 @@ TemplateNotFoundException, TemplateFailedParsingException, get_template_artifacts_format, - get_template_function_resource_ids, - get_template_function_runtimes, + get_template_function_resource_ids ) from samcli.lib.utils.packagetype import IMAGE, ZIP @@ -337,19 +334,3 @@ def test_get_template_function_resouce_ids(self, mock_get_template_data): } } self.assertEqual(get_template_function_resource_ids(MagicMock(), IMAGE), ["HelloWorldFunction1"]) - - -class Test_get_template_function_runtimes(TestCase): - @patch("samcli.commands._utils.template.get_template_data") - def test_get_template_function_runtimes(self, mock_get_template_data): - mock_get_template_data.return_value = { - "Resources": { - "HelloWorldFunction1": {"Type": "AWS::Lambda::Function", "Properties": {"PackageType": "Image"}}, - "HelloWorldFunction2": {"Type": "AWS::Lambda::Function", "Properties": {"Runtime": "python3.8"}}, - "HelloWorldFunction3": {"Type": "AWS::Serverless::Function", "Properties": {"Runtime": "python3.8"}}, - "HelloWorldFunction4": {"Type": "AWS::Serverless::Function", "Properties": {"Runtime": "python3.8"}}, - "HelloWorldFunction5": {"Type": "AWS::Serverless::Function", "Properties": {"Key": "value"}}, - "HelloWorldApi": {"Type": "AWS::Serverless::Api", "Properties": {"Runtime": "Java8"}}, - } - } - self.assertEqual(get_template_function_runtimes(MagicMock()), ["python3.8"]) From 46402f0e0de7e3125c61e575227653080024fbca Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Sat, 10 Apr 2021 19:40:15 -0700 Subject: [PATCH 08/31] sam pipeline bootstrap command --- mypy.ini | 2 +- samcli/cli/command.py | 1 + samcli/commands/pipeline/__init__.py | 0 .../commands/pipeline/bootstrap/__init__.py | 0 samcli/commands/pipeline/bootstrap/cli.py | 221 +++++++++++ .../pipeline/bootstrap/guided_context.py | 94 +++++ samcli/commands/pipeline/pipeline.py | 19 + samcli/lib/bootstrap/bootstrap.py | 11 +- samcli/lib/pipeline/__init__.py | 0 samcli/lib/pipeline/bootstrap/__init__.py | 0 samcli/lib/pipeline/bootstrap/resource.py | 133 +++++++ samcli/lib/pipeline/bootstrap/stage.py | 274 +++++++++++++ .../pipeline/bootstrap/stage_resources.yaml | 359 ++++++++++++++++++ .../cfn_templates/stage_resources.yaml | 222 ----------- .../lib/utils/managed_cloudformation_stack.py | 21 +- tests/unit/commands/_utils/test_template.py | 2 +- tests/unit/commands/pipeline/__init__.py | 0 .../commands/pipeline/bootstrap/__init__.py | 0 .../commands/pipeline/bootstrap/test_cli.py | 331 ++++++++++++++++ .../pipeline/bootstrap/test_guided_context.py | 92 +++++ tests/unit/lib/bootstrap/test_bootstrap.py | 12 +- tests/unit/lib/pipeline/bootstrap/__init__.py | 0 .../lib/pipeline/bootstrap/test_resource.py | 93 +++++ .../unit/lib/pipeline/bootstrap/test_stage.py | 355 +++++++++++++++++ 24 files changed, 2003 insertions(+), 239 deletions(-) create mode 100644 samcli/commands/pipeline/__init__.py create mode 100644 samcli/commands/pipeline/bootstrap/__init__.py create mode 100644 samcli/commands/pipeline/bootstrap/cli.py create mode 100644 samcli/commands/pipeline/bootstrap/guided_context.py create mode 100644 samcli/commands/pipeline/pipeline.py create mode 100644 samcli/lib/pipeline/__init__.py create mode 100644 samcli/lib/pipeline/bootstrap/__init__.py create mode 100644 samcli/lib/pipeline/bootstrap/resource.py create mode 100644 samcli/lib/pipeline/bootstrap/stage.py create mode 100644 samcli/lib/pipeline/bootstrap/stage_resources.yaml delete mode 100644 samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/stage_resources.yaml create mode 100644 tests/unit/commands/pipeline/__init__.py create mode 100644 tests/unit/commands/pipeline/bootstrap/__init__.py create mode 100644 tests/unit/commands/pipeline/bootstrap/test_cli.py create mode 100644 tests/unit/commands/pipeline/bootstrap/test_guided_context.py create mode 100644 tests/unit/lib/pipeline/bootstrap/__init__.py create mode 100644 tests/unit/lib/pipeline/bootstrap/test_resource.py create mode 100644 tests/unit/lib/pipeline/bootstrap/test_stage.py diff --git a/mypy.ini b/mypy.ini index 02bf97b5f9d..2b069faa2e2 100644 --- a/mypy.ini +++ b/mypy.ini @@ -59,6 +59,6 @@ ignore_missing_imports=True ignore_missing_imports=True # progressive add typechecks and these modules already complete the process, let's keep them clean -[mypy-samcli.commands.build,samcli.lib.build.*,samcli.commands.local.cli_common.invoke_context,samcli.commands.local.lib.local_lambda,samcli.lib.providers.*,samcli.lib.cookiecutter.*,samcli.lib.pipeline.*] +[mypy-samcli.commands.build,samcli.lib.build.*,samcli.commands.local.cli_common.invoke_context,samcli.commands.local.lib.local_lambda,samcli.lib.providers.*,samcli.lib.cookiecutter.*,samcli.lib.pipeline.*,samcli.commands.pipeline.*] disallow_untyped_defs=True disallow_incomplete_defs=True \ No newline at end of file diff --git a/samcli/cli/command.py b/samcli/cli/command.py index 384529f78ba..c135400586d 100644 --- a/samcli/cli/command.py +++ b/samcli/cli/command.py @@ -21,6 +21,7 @@ "samcli.commands.deploy", "samcli.commands.logs", "samcli.commands.publish", + "samcli.commands.pipeline.pipeline", # We intentionally do not expose the `bootstrap` command for now. We might open it up later # "samcli.commands.bootstrap", ] diff --git a/samcli/commands/pipeline/__init__.py b/samcli/commands/pipeline/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/samcli/commands/pipeline/bootstrap/__init__.py b/samcli/commands/pipeline/bootstrap/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py new file mode 100644 index 00000000000..7d34051389e --- /dev/null +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -0,0 +1,221 @@ +""" +CLI command for "pipeline bootstrap", which sets up the require pipeline infrastructure resources +""" +from typing import Any, Dict, Optional, Tuple + +import click + +from samcli.cli.cli_config_file import configuration_option, TomlProvider +from samcli.cli.context import get_cmd_names +from samcli.cli.main import pass_context, common_options, aws_creds_options, print_cmdline_args +from samcli.lib.config.samconfig import SamConfig +from samcli.lib.pipeline.bootstrap.stage import Stage +from samcli.lib.telemetry.metric import track_command +from samcli.lib.utils.version_checker import check_newer_version +from .guided_context import GuidedContext + +SHORT_HELP = "Sets up infrastructure resources for AWS SAM CI/CD pipelines." + +HELP_TEXT = """Sets up the following infrastructure resources for AWS SAM CI/CD pipelines: +\n\t - Pipeline user(IAM User) with AccessKeyId and SecretAccessKey credentials to be shared with the CI/CD provider +\n\t - Pipeline Execution Role(IAM Role) that is assumed by the Pipeline user to obtain access to the AWS account +\n\t - CloudFormation Execution Role(IAM Role) that is assumed by CloudFormation to deploy the SAM application +\n\t - Artifacts bucket(S3 bucket) to store the sam build artifacts +\n\t - ECR repo for the container images of Lambda functions having PackageType property set to Image +""" + +DEFAULT_SAMCONFIG_DIR = "samconfig_dir" +PIPELINE_SAMCONFIG_FILENAME = "pipelineconfig.toml" + + +@click.command("bootstrap", short_help=SHORT_HELP, help=HELP_TEXT, context_settings=dict(max_content_width=120)) +@configuration_option(provider=TomlProvider(section="parameters")) +@click.option( + "--interactive/--no-interactive", + is_flag=True, + default=True, + help="Disable interactive prompting for bootstrap parameters, and fail if any required arguments are missing.", +) +@click.option( + "--stage-name", + help="The name of the corresponding pipeline stage. It is used as a suffix for the created resources.", + required=False, +) +@click.option( + "--pipeline-user", + help="The ARN of the IAM user having its AccessKeyId and SecretAccessKey shared with the CI/CD provider." + "It is used to grant this IAM user the permissions to access the corresponding AWS account." + "If not provided, the command will create one along with AccessKeyId and SecretAccessKey credentials.", + required=False, +) +@click.option( + "--pipeline-execution-role", + help="The ARN of an IAM Role to be assumed by the pipeline-user to operate on this stage. " + "Provide it only if you want to user your own role, otherwise, the command will create one", + required=False, +) +@click.option( + "--cloudformation-execution-role", + help="The ARN of an IAM Role to be assumed by the CloudFormation service while deploying the application's stack " + "Provide it only if you want to user your own role, otherwise, the command will create one", + required=False, +) +@click.option( + "--artifacts-bucket", + help="The ARN of a S3 bucket to hold the sam build artifacts. " + "Provide it only if you want to user your own S3 bucket, otherwise, the command will create one", + required=False, +) +@click.option( + "--create-ecr-repo/--no-create-ecr-repo", + is_flag=True, + default=False, + help="If set to true and no ecr-repo is provided this command will create an ECR repo to hold the image container " + "of the lambda functions having Image package type.", +) +@click.option( + "--ecr-repo", + help="The ARN of an ECR repo to hold the image containers of the lambda functions of image package type. " + "If Provided, the create-ecr-repo argument is ignored. If not provided and create-ecr-repo is set to true " + "The command will create one.", + required=False, +) +@click.option( + "--pipeline-ip-range", + help="If provided, all requests coming from outside of the given range(s) are denied.", + required=False, +) +@click.option( + "--confirm-changeset/--no-confirm-changeset", + default=True, + is_flag=True, + help="Prompt to confirm if the resources is to be deployed by SAM CLI.", +) +@common_options +@aws_creds_options +@pass_context +@track_command +@check_newer_version +@print_cmdline_args +def cli( + ctx: Any, + interactive: bool, + stage_name: Optional[str], + pipeline_user: Optional[str], + pipeline_execution_role: Optional[str], + cloudformation_execution_role: Optional[str], + artifacts_bucket: Optional[str], + create_ecr_repo: bool, + ecr_repo: Optional[str], + pipeline_ip_range: Optional[str], + confirm_changeset: bool, + config_file: Optional[str], + config_env: Optional[str], +) -> None: + """ + `sam pipeline bootstrap` command entry point + """ + do_cli( + region=ctx.region, + profile=ctx.profile, + interactive=interactive, + stage_name=stage_name, + pipeline_user_arn=pipeline_user, + pipeline_execution_role_arn=pipeline_execution_role, + cloudformation_execution_role_arn=cloudformation_execution_role, + artifacts_bucket_arn=artifacts_bucket, + create_ecr_repo=create_ecr_repo, + ecr_repo_arn=ecr_repo, + pipeline_ip_range=pipeline_ip_range, + confirm_changeset=confirm_changeset, + config_file=config_env, + config_env=config_file, + ) # pragma: no cover + + +def do_cli( + region: Optional[str], + profile: Optional[str], + interactive: bool, + stage_name: Optional[str], + pipeline_user_arn: Optional[str], + pipeline_execution_role_arn: Optional[str], + cloudformation_execution_role_arn: Optional[str], + artifacts_bucket_arn: Optional[str], + create_ecr_repo: bool, + ecr_repo_arn: Optional[str], + pipeline_ip_range: Optional[str], + confirm_changeset: bool, + config_file: Optional[str], + config_env: Optional[str], +) -> None: + """ + implementation of `sam pipeline bootstrap` command + """ + if not pipeline_user_arn: + pipeline_user_arn = _load_saved_pipeline_user() + + if interactive: + guided_context = GuidedContext( + stage_name=stage_name, + pipeline_user_arn=pipeline_user_arn, + pipeline_execution_role_arn=pipeline_execution_role_arn, + cloudformation_execution_role_arn=cloudformation_execution_role_arn, + artifacts_bucket_arn=artifacts_bucket_arn, + create_ecr_repo=create_ecr_repo, + ecr_repo_arn=ecr_repo_arn, + pipeline_ip_range=pipeline_ip_range, + ) + guided_context.run() + stage_name = guided_context.stage_name + pipeline_user_arn = guided_context.pipeline_user_arn + pipeline_execution_role_arn = guided_context.pipeline_execution_role_arn + pipeline_ip_range = guided_context.pipeline_ip_range + cloudformation_execution_role_arn = guided_context.cloudformation_execution_role_arn + artifacts_bucket_arn = guided_context.artifacts_bucket_arn + create_ecr_repo = guided_context.create_ecr_repo + ecr_repo_arn = guided_context.ecr_repo_arn + + if not stage_name: + raise click.UsageError("Missing required parameter '--stage-name'") + + stage: Stage = Stage( + name=stage_name, + aws_profile=profile, + aws_region=region, + pipeline_user_arn=pipeline_user_arn, + pipeline_execution_role_arn=pipeline_execution_role_arn, + pipeline_ip_range=pipeline_ip_range, + cloudformation_execution_role_arn=cloudformation_execution_role_arn, + artifacts_bucket_arn=artifacts_bucket_arn, + create_ecr_repo=create_ecr_repo, + ecr_repo_arn=ecr_repo_arn, + ) + + stage.bootstrap(confirm_changeset=confirm_changeset) + + stage.print_resources_summary() + + try: + samconfig_dir, filename, cmd_names = _get_toml_file_metadata() + stage.save_config(config_dir=samconfig_dir, filename=filename, cmd_names=cmd_names) + except Exception: + # Swallow saving exceptions, if any, as the resources are already bootstrapped and the ARNs are already + # printed out in the screen. + pass + + +def _load_saved_pipeline_user() -> Optional[str]: + samconfig_dir, filename, cmd_names = _get_toml_file_metadata() + samconfig: SamConfig = SamConfig(config_dir=samconfig_dir, filename=filename) + if not samconfig.exists(): + return None + config: Dict[str, str] = samconfig.get_all(cmd_names=cmd_names, section="parameters") + return config.get("pipeline_user") + + +def _get_toml_file_metadata() -> Tuple: + ctx = click.get_current_context() + samconfig_dir: str = getattr(ctx, DEFAULT_SAMCONFIG_DIR, SamConfig.config_dir()) + cmd_names = get_cmd_names(ctx.info_name, ctx) # ["pipeline", "bootstrap"] + return samconfig_dir, PIPELINE_SAMCONFIG_FILENAME, cmd_names diff --git a/samcli/commands/pipeline/bootstrap/guided_context.py b/samcli/commands/pipeline/bootstrap/guided_context.py new file mode 100644 index 00000000000..096aa85df56 --- /dev/null +++ b/samcli/commands/pipeline/bootstrap/guided_context.py @@ -0,0 +1,94 @@ +""" +An interactive flow that prompt the user for required information to bootstrap the AWS account of a pipeline stage +with the required infrastructure +""" +from typing import Optional + +import click + + +class GuidedContext: + def __init__( + self, + stage_name: Optional[str] = None, + pipeline_user_arn: Optional[str] = None, + pipeline_execution_role_arn: Optional[str] = None, + cloudformation_execution_role_arn: Optional[str] = None, + artifacts_bucket_arn: Optional[str] = None, + create_ecr_repo: bool = False, + ecr_repo_arn: Optional[str] = None, + pipeline_ip_range: Optional[str] = None, + ) -> None: + self.stage_name = stage_name + self.pipeline_user_arn = pipeline_user_arn + self.pipeline_execution_role_arn = pipeline_execution_role_arn + self.cloudformation_execution_role_arn = cloudformation_execution_role_arn + self.artifacts_bucket_arn = artifacts_bucket_arn + self.create_ecr_repo = create_ecr_repo + self.ecr_repo_arn = ecr_repo_arn + self.pipeline_ip_range = pipeline_ip_range + + def run(self) -> None: + """ + Runs an interactive questionnaire to prompt the user for the ARNs of the AWS resources(infrastructure) required + for the pipeline to work. Users can provide all, none or some resources' ARNs and leave the remaining empty + and it will be created by the bootstrap command + """ + if not self.stage_name: + self.stage_name = click.prompt("Stage Name", type=click.STRING) + + if not self.pipeline_user_arn: + click.echo( + "\nThere must be exactly one pipeline user across all of the pipeline stages. " + "If you have ran this command before to bootstrap a previous pipeline stage, please " + "provide the ARN of the created pipeline user, otherwise, we will create a new user for you, " + "please make sure to configure user's AccessKeyId and SecretAccessKey for the CI/CD provider." + ) + self.pipeline_user_arn = click.prompt( + "Pipeline user [leave blank to create one]", default="", type=click.STRING + ) + + if not self.pipeline_execution_role_arn: + self.pipeline_execution_role_arn = click.prompt( + "\nPipeline execution role(an IAM Role to be assumed by the pipeline-user to operate on this stage.) " + "[leave blank to create one]", + default="", + type=click.STRING, + ) + + if not self.cloudformation_execution_role_arn: + self.cloudformation_execution_role_arn = click.prompt( + "\nCloudFormation execution role(an IAM Role to be assumed by the CloudFormation service to deploy " + "the application's stack) [leave blank to create one]", + default="", + type=click.STRING, + ) + + if not self.artifacts_bucket_arn: + self.artifacts_bucket_arn = click.prompt( + "\nArtifacts bucket(S3 bucket to hold the sam build artifacts) " "[leave blank to create one]", + default="", + type=click.STRING, + ) + if not self.ecr_repo_arn: + click.echo( + "\nIf your SAM template will include lambda functions of Image package-type, " + "then an ECR repo is required, should we create one?" + ) + click.echo("\t1 - No, My SAM Template won't include lambda functions of Image package-type") + click.echo("\t2 - Yes, I need a help creating one") + click.echo("\t3 - I already have an ECR repo") + choice = click.prompt(text="Choice", show_choices=False, type=click.Choice(["1", "2", "3"])) + if choice == "1": + self.create_ecr_repo = False + elif choice == "2": + self.create_ecr_repo = True + else: # choice == "3" + self.create_ecr_repo = False + self.ecr_repo_arn = click.prompt("ECR repo", type=click.STRING) + + if not self.pipeline_ip_range: + click.echo("\nWe can deny requests if not coming from a recognized IP address.") + self.pipeline_ip_range = click.prompt( + "Pipeline IP address range [leave blank if you don't know]", default="", type=click.STRING + ) diff --git a/samcli/commands/pipeline/pipeline.py b/samcli/commands/pipeline/pipeline.py new file mode 100644 index 00000000000..bb1247d770d --- /dev/null +++ b/samcli/commands/pipeline/pipeline.py @@ -0,0 +1,19 @@ +""" +Command group for "pipeline" suite for commands. It provides common CLI arguments, template parsing capabilities, +setting up stdin/stdout etc +""" + +import click + +from .bootstrap.cli import cli as bootstrap_cli + + +@click.group() +def cli() -> None: + """ + Manage the continuous delivery of the application + """ + + +# Add individual commands under this group +cli.add_command(bootstrap_cli) diff --git a/samcli/lib/bootstrap/bootstrap.py b/samcli/lib/bootstrap/bootstrap.py index 81c30c7748d..3596dd7917b 100644 --- a/samcli/lib/bootstrap/bootstrap.py +++ b/samcli/lib/bootstrap/bootstrap.py @@ -7,25 +7,24 @@ from samcli import __version__ from samcli.cli.global_config import GlobalConfig from samcli.commands.exceptions import UserException -from samcli.lib.utils.managed_cloudformation_stack import manage_stack as manage_cloudformation_stack +from samcli.lib.utils.managed_cloudformation_stack import StackOutput, manage_stack as manage_cloudformation_stack SAM_CLI_STACK_NAME = "aws-sam-cli-managed-default" LOG = logging.getLogger(__name__) def manage_stack(profile, region): - outputs = manage_cloudformation_stack( + outputs: StackOutput = manage_cloudformation_stack( profile=None, region=region, stack_name=SAM_CLI_STACK_NAME, template_body=_get_stack_template() ) - try: - bucket_name = next(o for o in outputs if o["OutputKey"] == "SourceBucket")["OutputValue"] - except StopIteration as ex: + bucket_name = outputs.get("SourceBucket") + if bucket_name is None: msg = ( "Stack " + SAM_CLI_STACK_NAME + " exists, but is missing the managed source bucket key. " "Failing as this stack was likely not created by the AWS SAM CLI." ) - raise UserException(msg) from ex + raise UserException(msg) # This bucket name is what we would write to a config file return bucket_name diff --git a/samcli/lib/pipeline/__init__.py b/samcli/lib/pipeline/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/samcli/lib/pipeline/bootstrap/__init__.py b/samcli/lib/pipeline/bootstrap/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/samcli/lib/pipeline/bootstrap/resource.py b/samcli/lib/pipeline/bootstrap/resource.py new file mode 100644 index 00000000000..a7b6668514b --- /dev/null +++ b/samcli/lib/pipeline/bootstrap/resource.py @@ -0,0 +1,133 @@ +""" Represents AWS resource""" +from typing import List, Optional + + +class ARNParts: + """ + Decompose a given ARN into its parts + + Attributes + ---------- + partition: str + the partition part(AWS, aws-cn or aws-us-gov) of the ARN + service: str + the service part(S3, IAM, ECR, ...etc) of the ARN + region: str + the AWS region part(us-east-1, eu-west-1, ...etc) of the ARN + account-id: str + the account-id part of the ARN + resource-id: str + the resource-id part of the ARN + resource-type: str + the resource-type part of the ARN + """ + + def __init__(self, arn: str) -> None: + if not isinstance(arn, str): + raise ValueError(f"Invalid ARN ({arn}) is not a String") + + parts = arn.split(":") + try: + self.partition: str = parts[1] + self.service: str = parts[2] + self.region: str = parts[3] + self.account_id: str = parts[4] + self.resource_id: str = parts[5] + self.resource_type: Optional[str] = None + except IndexError as ex: + raise ValueError(f"Invalid ARN ({arn})") from ex + + if "/" in self.resource_id: + resource_type_and_id: List[str] = self.resource_id.split("/") + self.resource_type = resource_type_and_id[0] + self.resource_id = resource_type_and_id[1] + + +class Resource: + """ + Represents an AWS resource + + Attributes + ---------- + arn: str + the ARN of the resource + is_user_provided: bool + True if the user provided the ARN of the resource during the initialization. It indicates whether this pipeline- + resource is provided by the user or created by SAM during `sam pipeline bootstrap` + + Methods + ------- + name(self) -> Optional[str]: + extracts and returns the resource name from its ARN + """ + + def __init__(self, arn: Optional[str]) -> None: + self.arn: Optional[str] = arn + self.is_user_provided: bool = bool(arn) + + def _get_arn_parts(self) -> Optional[ARNParts]: + return ARNParts(self.arn) if self.arn else None + + def name(self) -> Optional[str]: + """ + extracts and returns the resource name from its ARN + Raises + ------ + ValueError if the ARN is invalid + """ + arn_parts: Optional[ARNParts] = self._get_arn_parts() + return arn_parts.resource_id if arn_parts else None + + +class IamUser(Resource): + """ + Represents an AWS IamUser resource + Attributes + ---------- + access_key_id: Optional[str] + holds the AccessKeyId of the credential of this IAM user, if any. + secret_access_key: Optional[str] + holds the SecretAccessKey of the credential of this IAM user, if any. + """ + + def __init__( + self, arn: Optional[str], access_key_id: Optional[str] = None, secret_access_key: Optional[str] = None + ) -> None: + self.access_key_id: Optional[str] = access_key_id + self.secret_access_key: Optional[str] = secret_access_key + super().__init__(arn=arn) + + +class S3Bucket(Resource): + """ + Represents an AWS S3Bucket resource + Attributes + ---------- + kms_key_arn: Optional[str] + The ARN of the KMS key used in encrypting this S3Bucket, if any. + """ + + def __init__(self, arn: Optional[str], kms_key_arn: Optional[str] = None) -> None: + self.kms_key_arn: Optional[str] = kms_key_arn + super().__init__(arn=arn) + + +class EcrRepo(Resource): + """ Represents an AWS EcrRepo resource """ + + def __init__(self, arn: Optional[str]) -> None: + super().__init__(arn=arn) + + def get_uri(self) -> Optional[str]: + """ + extracts and returns the URI of the given ECR repo from its ARN + Raises + ------ + ValueError if the ARN is invalid + """ + arn_parts: Optional[ARNParts] = self._get_arn_parts() + return ( + f"{arn_parts.account_id}.dkr.ecr.{arn_parts.region}.amazonaws.com/{arn_parts.resource_id}" + if arn_parts + else None + ) diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py new file mode 100644 index 00000000000..bdf73565f22 --- /dev/null +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -0,0 +1,274 @@ +""" Pipeline stage""" +import os +import pathlib +import re +from typing import List, Optional + +import click + +from samcli.lib.config.samconfig import SamConfig +from samcli.lib.utils.managed_cloudformation_stack import manage_stack, StackOutput +from .resource import Resource, IamUser, S3Bucket, EcrRepo + +CFN_TEMPLATE_PATH = str(pathlib.Path(os.path.dirname(__file__))) +STACK_NAME_PREFIX = "aws-sam-cli-managed" +DEPLOYER_STACK_NAME_SUFFIX = "pipeline-deployer" +STAGE_RESOURCES_STACK_NAME_SUFFIX = "pipeline-resources" + + +class Stage: + """ + Represents a pipeline stage + + Attributes + ---------- + name: str + The name of the stage + aws_profile: str + The named AWS profile(in user's machine) of the AWS account to deploy this stage to. + aws_region: str + The AWS region to deploy this stage to. + pipeline_user: str + The IAM User having its AccessKeyId and SecretAccessKey credentials shared with the CI/CD provider + pipeline_execution_role: Resource + The IAM role assumed by the pipeline-user to get access to the AWS account and executes the + CloudFormation stack. + cloudformation_execution_role: Resource + The IAM role assumed by the CloudFormation service to executes the CloudFormation stack. + artifacts_bucket: S3Bucket + The S3 bucket to hold the SAM build artifacts of the application's CFN template. + ecr_repo: Resource + The ECR repo to hold the image container of lambda functions with Image package-type + + Methods: + -------- + did_user_provide_all_required_resources(self) -> bool: + checks if all of the stage required resources(pipeline_user, pipeline_execution_role, + cloudformation_execution_role, artifacts_bucket and ecr_repo) are provided by the user. + bootstrap(self, confirm_changeset: bool = True) -> None: + deploys the CFN template ./stage_resources.yaml to the AWS account identified by aws_profile and aws_region + member fields. if aws_profile is not provided, it will fallback to default boto3 credentials' resolving. + Note that ./stage_resources.yaml template accepts the ARNs of already existing resources(if any) as parameters + and it will skip the creation of those resources but will use the ARNs to set the proper permissions of other + missing resources(resources created by the template) + save_config(self, config_dir: str, filename: str, cmd_names: List[str]): + save the Artifacts bucket name, ECR repo URI and ARNs of pipeline_user, pipeline_execution_role and + cloudformation_execution_role to the "pipelineconfig.toml" file so that it can be auto-filled during + the `sam pipeline init` command. + print_resources_summary(self) -> None: + prints to the screen(console) the ARNs of the created and provided resources. + """ + + def __init__( + self, + name: str, + aws_profile: Optional[str] = None, + aws_region: Optional[str] = None, + pipeline_user_arn: Optional[str] = None, + pipeline_execution_role_arn: Optional[str] = None, + pipeline_ip_range: Optional[str] = None, + cloudformation_execution_role_arn: Optional[str] = None, + artifacts_bucket_arn: Optional[str] = None, + create_ecr_repo: bool = False, + ecr_repo_arn: Optional[str] = None, + ) -> None: + self.name: str = name + self.aws_profile: Optional[str] = aws_profile + self.aws_region: Optional[str] = aws_region + self.pipeline_user: IamUser = IamUser(arn=pipeline_user_arn) + self.pipeline_execution_role: Resource = Resource(arn=pipeline_execution_role_arn) + self.pipeline_ip_range = pipeline_ip_range + self.cloudformation_execution_role: Resource = Resource(arn=cloudformation_execution_role_arn) + self.artifacts_bucket: S3Bucket = S3Bucket(arn=artifacts_bucket_arn) + self.create_ecr_repo: bool = create_ecr_repo + self.ecr_repo: EcrRepo = EcrRepo(arn=ecr_repo_arn) + + def did_user_provide_all_required_resources(self) -> bool: + """Check if the user provided all of the stage resources or not""" + return ( + self.pipeline_user.is_user_provided + and self.pipeline_execution_role.is_user_provided + and self.cloudformation_execution_role.is_user_provided + and self.artifacts_bucket.is_user_provided + and (not self.create_ecr_repo or self.ecr_repo.is_user_provided) + ) + + def _get_non_user_provided_resources_msg(self) -> str: + missing_resources_msg = "" + if not self.pipeline_user.is_user_provided: + missing_resources_msg += "\n\tPipeline user" + if not self.pipeline_execution_role.is_user_provided: + missing_resources_msg += "\n\tPipeline execution role." + if not self.cloudformation_execution_role.is_user_provided: + missing_resources_msg += "\n\tCloudFormation execution role." + if not self.artifacts_bucket.is_user_provided: + missing_resources_msg += "\n\tArtifacts bucket." + if self.create_ecr_repo and not self.ecr_repo.is_user_provided: + missing_resources_msg += "\n\tECR repo." + return missing_resources_msg + + def bootstrap(self, confirm_changeset: bool = True) -> None: + """ + Deploys the CFN template(./stage_resources.yaml) which deploys: + * Pipeline IAM User + * Pipeline execution IAM role + * CloudFormation execution IAM role + * Artifacts' S3 Bucket along with KMS encryption key + * ECR Repo + to the AWS account associated with the given stage. It will not redeploy the stack if already exists. + This CFN template accepts the ARNs of the resources as parameters and will not create a resource if already + provided, this way we can conditionally create a resource only if the user didn't provide it + + THIS METHOD UPDATES THE STATE OF THE CALLING INSTANCE(self) IT WILL SET THE VALUES OF THE RESOURCES ATTRIBUTES + + Parameters + ---------- + confirm_changeset: bool + if set to false, the stage_resources.yaml CFN template will directly be deployed, otherwise, the user will + be prompted for confirmation + """ + + if self.did_user_provide_all_required_resources(): + click.secho(f"\nAll required resources for the {self.name} stage exist, skipping creation.", fg="yellow") + return + + missing_resources: str = self._get_non_user_provided_resources_msg() + click.echo(f"This will create the following required resources for the {self.name} stage: {missing_resources}") + if confirm_changeset: + confirmed: bool = click.confirm("Should we proceed with the creation?") + if not confirmed: + return + + stage_name_suffix: str = re.sub("[^0-9a-zA-Z]+", "-", self.name) + stack_name: str = f"{STACK_NAME_PREFIX}-{stage_name_suffix}-{STAGE_RESOURCES_STACK_NAME_SUFFIX}" + stage_resources_template_body = Stage._read_template("stage_resources.yaml") + output: StackOutput = manage_stack( + stack_name=stack_name, + region=self.aws_region, + profile=self.aws_profile, + template_body=stage_resources_template_body, + parameter_overrides={ + "PipelineUserArn": self.pipeline_user.arn or "", + "PipelineExecutionRoleArn": self.pipeline_execution_role.arn or "", + "PipelineIpRange": self.pipeline_ip_range or "", + "CloudFormationExecutionRoleArn": self.cloudformation_execution_role.arn or "", + "ArtifactsBucketArn": self.artifacts_bucket.arn or "", + "CreateEcrRepo": "true" if self.create_ecr_repo else "false", + "EcrRepoArn": self.ecr_repo.arn or "", + }, + ) + + self.pipeline_user.arn = output.get("PipelineUser") + self.pipeline_user.access_key_id = output.get("PipelineUserAccessKeyId") + self.pipeline_user.secret_access_key = output.get("PipelineUserSecretAccessKey") + self.pipeline_execution_role.arn = output.get("PipelineExecutionRole") + self.cloudformation_execution_role.arn = output.get("CloudFormationExecutionRole") + self.artifacts_bucket.arn = output.get("ArtifactsBucket") + self.artifacts_bucket.kms_key_arn = output.get("ArtifactsBucketKMS") + self.ecr_repo.arn = output.get("EcrRepo") + + @staticmethod + def _read_template(template_file_name: str) -> str: + template_path: str = os.path.join(CFN_TEMPLATE_PATH, template_file_name) + with open(template_path, "r") as fp: + template_body = fp.read() + return template_body + + def save_config(self, config_dir: str, filename: str, cmd_names: List[str]) -> None: + """ + save the Artifacts bucket name, ECR repo URI and ARNs of pipeline_user, pipeline_execution_role and + cloudformation_execution_role to the given filename and directory. + + Parameters + ---------- + config_dir: str + the directory of the toml file to save to + filename: str + the name of the toml file to save to + cmd_names: List[str] + nested command name to scope the saved configs to inside the toml file + + Raises + ------ + ValueError: if the artifacts_bucket or ecr_repo ARNs are invalid + """ + + samconfig: SamConfig = SamConfig(config_dir=config_dir, filename=filename) + + if self.pipeline_user.arn: + samconfig.put(cmd_names=cmd_names, section="parameters", key="pipeline_user", value=self.pipeline_user.arn) + + if self.pipeline_execution_role.arn: + samconfig.put( + cmd_names=cmd_names, + section="parameters", + key="pipeline_execution_role", + value=self.pipeline_execution_role.arn, + env=self.name, + ) + + if self.cloudformation_execution_role.arn: + samconfig.put( + cmd_names=cmd_names, + section="parameters", + key="cloudformation_execution_role", + value=self.cloudformation_execution_role.arn, + env=self.name, + ) + + if self.artifacts_bucket.name(): + samconfig.put( + cmd_names=cmd_names, + section="parameters", + key="artifacts_bucket", + value=self.artifacts_bucket.name(), + env=self.name, + ) + + if self.ecr_repo.get_uri(): + samconfig.put( + cmd_names=cmd_names, section="parameters", key="ecr_repo", value=self.ecr_repo.get_uri(), env=self.name + ) + + samconfig.flush() + + def _get_resources(self) -> List[Resource]: + resources = [ + self.pipeline_user, + self.pipeline_execution_role, + self.cloudformation_execution_role, + self.artifacts_bucket, + ] + if self.create_ecr_repo: # ECR Repo is optional + resources.append(self.ecr_repo) + return resources + + def print_resources_summary(self) -> None: + """prints to the screen(console) the ARNs of the created and provided resources.""" + + provided_resources = [] + created_resources = [] + for resource in self._get_resources(): + if resource.is_user_provided: + provided_resources.append(resource) + else: + created_resources.append(resource) + + if created_resources: + click.secho("\nWe have created the following resources:", fg="green") + for resource in created_resources: + click.secho(f"\t{resource.arn}", fg="green") + + if provided_resources: + click.secho( + "\nYou provided the following resources. Please make sure it has the required permissions as shown at " + "https://github.com/aws/aws-sam-cli/blob/develop/samcli/lib/pipeline/bootstrap/stage_resources.yaml", + fg="green", + ) + for resource in provided_resources: + click.secho(f"\t{resource.arn}", fg="green") + + if not self.pipeline_user.is_user_provided: + click.secho("Please configure your CICD project with the following pipeline-user credentials:", fg="green") + click.secho(f"\tACCESS_KEY_ID: {self.pipeline_user.access_key_id}", fg="green") + click.secho(f"\tSECRET_ACCESS_KEY: {self.pipeline_user.secret_access_key}", fg="green") diff --git a/samcli/lib/pipeline/bootstrap/stage_resources.yaml b/samcli/lib/pipeline/bootstrap/stage_resources.yaml new file mode 100644 index 00000000000..7134f3c8507 --- /dev/null +++ b/samcli/lib/pipeline/bootstrap/stage_resources.yaml @@ -0,0 +1,359 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Parameters: + PipelineUserArn: + Type: String + PipelineExecutionRoleArn: + Type: String + PipelineIpRange: + Type: String + CloudFormationExecutionRoleArn: + Type: String + ArtifactsBucketArn: + Type: String + CreateEcrRepo: + Type: String + Default: false + AllowedValues: [true, false] + EcrRepoArn: + Type: String + +Conditions: + MissingPipelineUser: !Equals [!Ref PipelineUserArn, ""] + MissingPipelineExecutionRole: !Equals [!Ref PipelineExecutionRoleArn, ""] + HasPipelineIpRange: !Not [!Equals [!Ref PipelineIpRange, ""]] + MissingCloudFormationExecutionRole: !Equals [!Ref CloudFormationExecutionRoleArn, ""] + MissingArtifactsBucket: !Equals [!Ref ArtifactsBucketArn, ""] + EcrRepoRequired: !Equals [!Ref CreateEcrRepo, "true"] + MissingEcrRepo: !And [!Condition EcrRepoRequired, !Equals [!Ref EcrRepoArn, ""]] + +Resources: + PipelineUser: + Type: AWS::IAM::User + Condition: MissingPipelineUser + Properties: + Tags: + - Key: ManagedStackSource + Value: AwsSamCli + Policies: + - PolicyName: AssumeRoles + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "sts:AssumeRole" + Resource: "*" + - Fn::If: + - HasPipelineIpRange + - Effect: Deny + Action: + - 'sts:AssumeRole' + Resource: "*" + Condition: + NotIpAddress: + aws:SourceIp: !Ref PipelineIpRange + - !Ref AWS::NoValue + + PipelineUserAccessKey: + Type: AWS::IAM::AccessKey + Condition: MissingPipelineUser + Properties: + Serial: 1 + Status: Active + UserName: !Ref PipelineUser + + CloudFormationExecutionRole: + Type: AWS::IAM::Role + Condition: MissingCloudFormationExecutionRole + Properties: + Tags: + - Key: ManagedStackSource + Value: AwsSamCli + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: cloudformation.amazonaws.com + Action: + - 'sts:AssumeRole' + Policies: + - PolicyName: GrantCloudFormationFullAccess + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: '*' + Resource: '*' + + PipelineExecutionRole: + Type: AWS::IAM::Role + Condition: MissingPipelineExecutionRole + Properties: + Tags: + - Key: ManagedStackSource + Value: AwsSamCli + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + AWS: + - Fn::If: + - MissingPipelineUser + - !GetAtt PipelineUser.Arn + - !Ref PipelineUserArn + Action: + - 'sts:AssumeRole' + - Fn::If: + - HasPipelineIpRange + - Effect: Deny + Principal: + AWS: + - Fn::If: + - MissingPipelineUser + - !GetAtt PipelineUser.Arn + - !Ref PipelineUserArn + Action: + - 'sts:AssumeRole' + Condition: + NotIpAddress: + aws:SourceIp: + - !Ref PipelineIpRange + - !Ref AWS::NoValue + + ArtifactsBucketKey: + Type: AWS::KMS::Key + Condition: MissingArtifactsBucket + Properties: + Tags: + - Key: ManagedStackSource + Value: AwsSamCli + Description: Artifact encryption/decryption cmk + EnableKeyRotation: true + KeyPolicy: + Version: '2012-10-17' + Id: !Ref AWS::StackName + Statement: + - Effect: Allow + Principal: + AWS: !Sub arn:aws:iam::${AWS::AccountId}:root + Action: kms:* + Resource: '*' + - Effect: Allow + Principal: + AWS: + - Fn::If: + - MissingPipelineExecutionRole + - !GetAtt PipelineExecutionRole.Arn + - !Ref PipelineExecutionRoleArn + - Fn::If: + - MissingCloudFormationExecutionRole + - !GetAtt CloudFormationExecutionRole.Arn + - !Ref CloudFormationExecutionRoleArn + Action: + - kms:Encrypt + - kms:Decrypt + - kms:ReEncrypt* + - kms:GenerateDataKey* + - kms:DescribeKey + Resource: '*' + + ArtifactsBucket: + Type: AWS::S3::Bucket + Condition: MissingArtifactsBucket + DeletionPolicy: "Retain" + Properties: + Tags: + - Key: ManagedStackSource + Value: AwsSamCli + VersioningConfiguration: + Status: Enabled + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + KMSMasterKeyID: !GetAtt ArtifactsBucketKey.Arn + SSEAlgorithm: aws:kms + + ArtifactsBucketPolicy: + Type: AWS::S3::BucketPolicy + Condition: MissingArtifactsBucket + Properties: + Bucket: !Ref ArtifactsBucket + PolicyDocument: + Statement: + - Effect: "Allow" + Action: + - 's3:GetObject*' + - 's3:PutObject*' + - 's3:GetBucket*' + - 's3:List*' + Resource: + - !Join ['',[!GetAtt ArtifactsBucket.Arn, '/*']] + - !GetAtt ArtifactsBucket.Arn + Principal: + AWS: + - Fn::If: + - MissingPipelineExecutionRole + - !GetAtt PipelineExecutionRole.Arn + - !Ref PipelineExecutionRoleArn + - Fn::If: + - MissingCloudFormationExecutionRole + - !GetAtt CloudFormationExecutionRole.Arn + - !Ref CloudFormationExecutionRoleArn + + PipelineExecutionRolePermissionPolicy: + Type: AWS::IAM::Policy + Condition: MissingPipelineExecutionRole + Properties: + PolicyName: PipelineExecutionRolePermissions + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: 'iam:PassRole' + Resource: + Fn::If: + - MissingCloudFormationExecutionRole + - !GetAtt CloudFormationExecutionRole.Arn + - !Ref CloudFormationExecutionRoleArn + - Effect: Allow + Action: + - "cloudformation:CreateChangeSet" + - "cloudformation:DescribeChangeSet" + - "cloudformation:ExecuteChangeSet" + - "cloudformation:DescribeStackEvents" + - "cloudformation:DescribeStacks" + - "cloudformation:GetTemplateSummary" + - "cloudformation:DescribeStackResource" + Resource: '*' + - Effect: Allow + Action: + - 's3:GetObject*' + - 's3:PutObject*' + - 's3:GetBucket*' + - 's3:List*' + Resource: + Fn::If: + - MissingArtifactsBucket + - - !Join [ '',[ !GetAtt ArtifactsBucket.Arn, '/*' ] ] + - !GetAtt ArtifactsBucket.Arn + - - !Join [ '',[ !Ref ArtifactsBucketArn, '/*' ] ] + - !Ref ArtifactsBucketArn + - Fn::If: + - MissingArtifactsBucket + - Effect: "Allow" + Action: + - "kms:Decrypt" + - "kms:DescribeKey" + Resource: + - !GetAtt ArtifactsBucketKey.Arn + - !Ref AWS::NoValue + - Fn::If: + - EcrRepoRequired + - Effect: "Allow" + Action: "ecr:GetAuthorizationToken" + Resource: "*" + - !Ref AWS::NoValue + - Fn::If: + - EcrRepoRequired + - Effect: "Allow" + Action: + - "ecr:GetDownloadUrlForLayer" + - "ecr:BatchGetImage" + - "ecr:BatchCheckLayerAvailability" + - "ecr:PutImage" + - "ecr:InitiateLayerUpload" + - "ecr:UploadLayerPart" + - "ecr:CompleteLayerUpload" + Resource: + Fn::If: + - MissingEcrRepo + - !GetAtt EcrRepo.Arn + - !Ref EcrRepoArn + - !Ref AWS::NoValue + Roles: + - !Ref PipelineExecutionRole + + EcrRepo: + Type: AWS::ECR::Repository + Condition: MissingEcrRepo + Properties: + RepositoryPolicyText: + Version: "2012-10-17" + Statement: + - Sid: AllowPushPull + Effect: Allow + Principal: + AWS: + - Fn::If: + - MissingPipelineExecutionRole + - !GetAtt PipelineExecutionRole.Arn + - !Ref PipelineExecutionRoleArn + - Fn::If: + - MissingCloudFormationExecutionRole + - !GetAtt CloudFormationExecutionRole.Arn + - !Ref CloudFormationExecutionRoleArn + Action: + - "ecr:GetDownloadUrlForLayer" + - "ecr:BatchGetImage" + - "ecr:BatchCheckLayerAvailability" + - "ecr:PutImage" + - "ecr:InitiateLayerUpload" + - "ecr:UploadLayerPart" + - "ecr:CompleteLayerUpload" + +Outputs: + PipelineUser: + Description: ARN of the Pipeline IAM User + Value: + Fn::If: + - MissingPipelineUser + - !GetAtt PipelineUser.Arn + - !Ref PipelineUserArn + + PipelineUserAccessKeyId: + Condition: MissingPipelineUser + Description: AccessKeyId of the Pipeline IAM User + Value: !Ref PipelineUserAccessKey + + PipelineUserSecretAccessKey: + Condition: MissingPipelineUser + Description: SecretAccessKey of the Pipeline IAM User + Value: !GetAtt PipelineUserAccessKey.SecretAccessKey + + CloudFormationExecutionRole: + Description: ARN of the IAM Role(CloudFormationExecutionRole) + Value: + Fn::If: + - MissingCloudFormationExecutionRole + - !GetAtt CloudFormationExecutionRole.Arn + - !Ref CloudFormationExecutionRoleArn + + PipelineExecutionRole: + Description: ARN of the IAM Role(PipelineExecutionRole) + Value: + Fn::If: + - MissingPipelineExecutionRole + - !GetAtt PipelineExecutionRole.Arn + - !Ref PipelineExecutionRoleArn + + ArtifactsBucket: + Description: ARN of the Artifacts bucket + Value: + Fn::If: + - MissingArtifactsBucket + - !GetAtt ArtifactsBucket.Arn + - !Ref ArtifactsBucketArn + + ArtifactsBucketKey: + Description: ARN of the CMK used for Artifacts bucket encryption/decryption + Condition: MissingArtifactsBucket + Value: !GetAtt ArtifactsBucketKey.Arn + + EcrRepo: + Description: ARN of the ECR repo + Condition: MissingEcrRepo + Value: !GetAtt EcrRepo.Arn diff --git a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/stage_resources.yaml b/samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/stage_resources.yaml deleted file mode 100644 index 3c7cbef5481..00000000000 --- a/samcli/lib/pipeline/init/plugins/two_stages_pipeline/cfn_templates/stage_resources.yaml +++ /dev/null @@ -1,222 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Transform: AWS::Serverless-2016-10-31 - -Parameters: - DeployerArn: - Type: String - AllowedPattern : ".+" # Must be provided - DeployerRoleArn: - Type: String - CFNDeploymentRoleArn: - Type: String - ArtifactsBucketArn: - Type: String - -Conditions: - MissingDeployerRole: !Equals [!Ref DeployerRoleArn, ""] - MissingCFNDeploymentRole: !Equals [!Ref CFNDeploymentRoleArn, ""] - MissingArtifactsBucket: !Equals [!Ref ArtifactsBucketArn, ""] - -Resources: - CFNDeploymentRole: - Type: AWS::IAM::Role - Condition: MissingCFNDeploymentRole - Properties: - Tags: - - Key: ManagedStackSource - Value: AwsSamCli - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: cloudformation.amazonaws.com - Action: - - 'sts:AssumeRole' - Policies: - - PolicyName: GrantCloudFormationFullAccess - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: '*' - Resource: '*' - - DeployerRole: - Type: AWS::IAM::Role - Condition: MissingDeployerRole - Properties: - Tags: - - Key: ManagedStackSource - Value: AwsSamCli - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - AWS: !Ref DeployerArn - Action: - - 'sts:AssumeRole' - Policies: - - PolicyName: AccessRolePolicy - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - 'iam:PassRole' - Resource: - Fn::If: - - MissingCFNDeploymentRole - - !GetAtt CFNDeploymentRole.Arn - - !Ref CFNDeploymentRoleArn - - Effect: Allow - Action: - - "cloudformation:CreateChangeSet" - - "cloudformation:DescribeChangeSet" - - "cloudformation:ExecuteChangeSet" - - "cloudformation:DescribeStackEvents" - - "cloudformation:DescribeStacks" - - "cloudformation:GetTemplateSummary" - - "cloudformation:DescribeStackResource" - Resource: '*' - - ArtifactsBucketKey: - Type: AWS::KMS::Key - Condition: MissingArtifactsBucket - Properties: - Tags: - - Key: ManagedStackSource - Value: AwsSamCli - Description: Artifact encryption/decryption cmk - EnableKeyRotation: true - KeyPolicy: - Version: '2012-10-17' - Id: !Ref AWS::StackName - Statement: - - Effect: Allow - Principal: - AWS: !Sub arn:aws:iam::${AWS::AccountId}:root - Action: kms:* - Resource: '*' - - Effect: Allow - Principal: - AWS: - - Fn::If: - - MissingDeployerRole - - !GetAtt DeployerRole.Arn - - !Ref DeployerRoleArn - - Fn::If: - - MissingCFNDeploymentRole - - !GetAtt CFNDeploymentRole.Arn - - !Ref CFNDeploymentRoleArn - Action: - - kms:Encrypt - - kms:Decrypt - - kms:ReEncrypt* - - kms:GenerateDataKey* - - kms:DescribeKey - Resource: '*' - - ArtifactsBucket: - Type: AWS::S3::Bucket - Condition: MissingArtifactsBucket - DeletionPolicy: "Retain" - Properties: - Tags: - - Key: ManagedStackSource - Value: AwsSamCli - VersioningConfiguration: - Status: Enabled - BucketEncryption: - ServerSideEncryptionConfiguration: - - ServerSideEncryptionByDefault: - KMSMasterKeyID: !GetAtt ArtifactsBucketKey.Arn - SSEAlgorithm: aws:kms - - ArtifactsBucketPolicy: - Type: AWS::S3::BucketPolicy - Condition: MissingArtifactsBucket - Properties: - Bucket: !Ref ArtifactsBucket - PolicyDocument: - Statement: - - Effect: "Allow" - Action: - - 's3:GetObject*' - - 's3:PutObject*' - - 's3:GetBucket*' - - 's3:List*' - Resource: - - !Join ['',['arn:', !Ref AWS::Partition, ':s3:::',!Ref ArtifactsBucket, '/*']] - - !Join ['',['arn:', !Ref AWS::Partition, ':s3:::',!Ref ArtifactsBucket]] - Principal: - AWS: - - Fn::If: - - MissingDeployerRole - - !GetAtt DeployerRole.Arn - - !Ref DeployerRoleArn - - Fn::If: - - MissingCFNDeploymentRole - - !GetAtt CFNDeploymentRole.Arn - - !Ref CFNDeploymentRoleArn - - DeployerRolePermissionPolicy: - Type: AWS::IAM::Policy - Condition: MissingDeployerRole - Properties: - PolicyName: ArtifactsPermissions - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - 's3:GetObject*' - - 's3:PutObject*' - - 's3:GetBucket*' - - 's3:List*' - Resource: - Fn::If: - - MissingArtifactsBucket - - - !Join [ '',[ !GetAtt ArtifactsBucket.Arn, '/*' ] ] - - !GetAtt ArtifactsBucket.Arn - - - !Join [ '',[ !Ref ArtifactsBucketArn, '/*' ] ] - - !Ref ArtifactsBucketArn - - Fn::If: - - MissingArtifactsBucket - - Effect: "Allow" - Action: - - "kms:Decrypt" - - "kms:DescribeKey" - Resource: - - !GetAtt ArtifactsBucketKey.Arn - - Ref: AWS::NoValue - Roles: - - !Ref DeployerRole - -Outputs: - CFNDeploymentRole: - Description: ARN of the IAM Role(CFNDeploymentRole) - Value: - Fn::If: - - MissingCFNDeploymentRole - - !GetAtt CFNDeploymentRole.Arn - - !Ref CFNDeploymentRoleArn - DeployerRole: - Description: ARN of the IAM Role(DeployerRole) - Value: - Fn::If: - - MissingDeployerRole - - !GetAtt DeployerRole.Arn - - !Ref DeployerRoleArn - ArtifactsBucket: - Description: Name of the Artifacts bucket - Value: - Fn::If: - - MissingArtifactsBucket - - !GetAtt ArtifactsBucket.Arn - - !Ref ArtifactsBucketArn - ArtifactsBucketKey: - Description: ARN of the CMK used for Artifacts bucket encryption/decryption - Condition: MissingArtifactsBucket - Value: !GetAtt ArtifactsBucketKey.Arn diff --git a/samcli/lib/utils/managed_cloudformation_stack.py b/samcli/lib/utils/managed_cloudformation_stack.py index d85e16c7a0b..84ed1031ba4 100644 --- a/samcli/lib/utils/managed_cloudformation_stack.py +++ b/samcli/lib/utils/managed_cloudformation_stack.py @@ -22,13 +22,24 @@ def __init__(self, ex): super().__init__(message=message_fmt.format(ex=self.ex)) +class StackOutput: + def __init__(self, stack_output: List[Dict[str, str]]): + self._stack_output: List[Dict[str, str]] = stack_output + + def get(self, key) -> Optional[str]: + try: + return next(o for o in self._stack_output if o.get("OutputKey") == key).get("OutputValue") + except StopIteration: + return None + + def manage_stack( - region: str, + region: Optional[str], stack_name: str, template_body: str, profile: Optional[str] = None, parameter_overrides: Optional[Dict[str, Union[str, List[str]]]] = None, -) -> List[Dict[str, str]]: +) -> StackOutput: """ get or create a CloudFormation stack @@ -81,7 +92,7 @@ def _create_or_get_stack( stack_name: str, template_body: str, parameter_overrides: Optional[Dict[str, Union[str, List[str]]]] = None, -) -> List[Dict[str, str]]: +) -> StackOutput: try: ds_resp = cloudformation_client.describe_stacks(StackName=stack_name) stacks = ds_resp["Stacks"] @@ -89,7 +100,7 @@ def _create_or_get_stack( click.echo("\n\tLooking for resources needed for deployment: Found!") _check_sanity_of_stack(stack) stack_outputs = cast(List[Dict[str, str]], stack["Outputs"]) - return stack_outputs + return StackOutput(stack_outputs) except ClientError: click.echo("\n\tLooking for resources needed for deployment: Not found.") @@ -99,7 +110,7 @@ def _create_or_get_stack( ) # exceptions are not captured from subcommands _check_sanity_of_stack(stack) stack_outputs = cast(List[Dict[str, str]], stack["Outputs"]) - return stack_outputs + return StackOutput(stack_outputs) except (ClientError, BotoCoreError) as ex: LOG.debug("Failed to create managed resources", exc_info=ex) raise ManagedStackError(str(ex)) from ex diff --git a/tests/unit/commands/_utils/test_template.py b/tests/unit/commands/_utils/test_template.py index 5c84dedb036..1de707ec381 100644 --- a/tests/unit/commands/_utils/test_template.py +++ b/tests/unit/commands/_utils/test_template.py @@ -18,7 +18,7 @@ TemplateNotFoundException, TemplateFailedParsingException, get_template_artifacts_format, - get_template_function_resource_ids + get_template_function_resource_ids, ) from samcli.lib.utils.packagetype import IMAGE, ZIP diff --git a/tests/unit/commands/pipeline/__init__.py b/tests/unit/commands/pipeline/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/commands/pipeline/bootstrap/__init__.py b/tests/unit/commands/pipeline/bootstrap/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/commands/pipeline/bootstrap/test_cli.py b/tests/unit/commands/pipeline/bootstrap/test_cli.py new file mode 100644 index 00000000000..da8b501200d --- /dev/null +++ b/tests/unit/commands/pipeline/bootstrap/test_cli.py @@ -0,0 +1,331 @@ +import os +from unittest import TestCase +from unittest.mock import patch, Mock + +import click +from click.testing import CliRunner + +from samcli.commands.pipeline.bootstrap.cli import _load_saved_pipeline_user, _get_toml_file_metadata +from samcli.commands.pipeline.bootstrap.cli import cli as bootstrap_cmd +from samcli.commands.pipeline.bootstrap.cli import do_cli as bootstrap_cli + +ANY_REGION = "ANY_REGION" +ANY_PROFILE = "ANY_PROFILE" +ANY_STAGE_NAME = "ANY_STAGE_NAME" +ANY_PIPELINE_USER_ARN = "ANY_PIPELINE_USER_ARN" +ANY_PIPELINE_EXECUTION_ROLE_ARN = "ANY_PIPELINE_EXECUTION_ROLE_ARN" +ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN = "ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN" +ANY_ARTIFACTS_BUCKET_ARN = "ANY_ARTIFACTS_BUCKET_ARN" +ANY_ECR_REPO_ARN = "ANY_ECR_REPO_ARN" +ANY_ARN = "ANY_ARN" +ANY_PIPELINE_IP_RANGE = "111.222.333.0/24" +ANY_CONFIG_FILE = "ANY_CONFIG_FILE" +ANY_CONFIG_ENV = "ANY_CONFIG_ENV" +ANY_CONFIG_DIR = "ANY_CONFIG_DIR" +PIPELINE_TOML_FILE = "PIPELINE_TOML_FILE" +PIPELINE_BOOTSTRAP_COMMAND_NAMES = ["pipeline", "bootstrap"] + + +class TestCli(TestCase): + def setUp(self) -> None: + self.cli_context = { + "region": ANY_REGION, + "profile": ANY_PROFILE, + "interactive": True, + "stage_name": ANY_STAGE_NAME, + "pipeline_user_arn": ANY_PIPELINE_USER_ARN, + "pipeline_execution_role_arn": ANY_PIPELINE_EXECUTION_ROLE_ARN, + "cloudformation_execution_role_arn": ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN, + "artifacts_bucket_arn": ANY_ARTIFACTS_BUCKET_ARN, + "create_ecr_repo": True, + "ecr_repo_arn": ANY_ECR_REPO_ARN, + "pipeline_ip_range": ANY_PIPELINE_IP_RANGE, + "confirm_changeset": True, + "config_file": ANY_CONFIG_FILE, + "config_env": ANY_CONFIG_ENV, + } + + @patch("samcli.commands.pipeline.bootstrap.cli.do_cli") + def test_bootstrap_command_default_argument_values(self, do_cli_mock): + runner: CliRunner = CliRunner() + runner.invoke(bootstrap_cmd) + # Test the defaults are as following: + # interactive -> True + # create_ecr_repo -> False + # confirm_changeset -> True + # region, profile, stage_name and all ARNs are None + do_cli_mock.assert_called_once_with( + region=None, + profile=None, + interactive=True, + stage_name=None, + pipeline_user_arn=None, + pipeline_execution_role_arn=None, + cloudformation_execution_role_arn=None, + artifacts_bucket_arn=None, + create_ecr_repo=False, + ecr_repo_arn=None, + pipeline_ip_range=None, + confirm_changeset=True, + config_file="default", + config_env="samconfig.toml", + ) + + @patch("samcli.commands.pipeline.bootstrap.cli.do_cli") + def test_bootstrap_command_flag_arguments(self, do_cli_mock): + runner: CliRunner = CliRunner() + runner.invoke(bootstrap_cmd, args=["--interactive", "--no-create-ecr-repo", "--confirm-changeset"]) + args, kwargs = do_cli_mock.call_args + self.assertTrue(kwargs["interactive"]) + self.assertFalse(kwargs["create_ecr_repo"]) + self.assertTrue(kwargs["confirm_changeset"]) + + runner.invoke(bootstrap_cmd, args=["--no-interactive", "--create-ecr-repo", "--no-confirm-changeset"]) + args, kwargs = do_cli_mock.call_args + self.assertFalse(kwargs["interactive"]) + self.assertTrue(kwargs["create_ecr_repo"]) + self.assertFalse(kwargs["confirm_changeset"]) + + @patch("samcli.commands.pipeline.bootstrap.cli.do_cli") + def test_bootstrap_command_with_different_arguments_combination(self, do_cli_mock): + runner: CliRunner = CliRunner() + runner.invoke( + bootstrap_cmd, args=["--no-interactive", "--stage-name", "stage1", "--artifacts-bucket", "bucketARN"] + ) + args, kwargs = do_cli_mock.call_args + self.assertFalse(kwargs["interactive"]) + self.assertEqual(kwargs["stage_name"], "stage1") + self.assertEqual(kwargs["artifacts_bucket_arn"], "bucketARN") + + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli.Stage") + @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") + def test_bootstrapping_normal_interactive_flow( + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + ): + # setup + gc_instance = Mock() + guided_context_mock.return_value = gc_instance + stage_instance = Mock() + stage_mock.return_value = stage_instance + load_saved_pipeline_user_mock.return_value = ANY_PIPELINE_USER_ARN + self.cli_context["interactive"] = True + self.cli_context["pipeline_user_arn"] = None + get_toml_file_metadata_mock.return_value = ( + ANY_CONFIG_DIR, + PIPELINE_TOML_FILE, + PIPELINE_BOOTSTRAP_COMMAND_NAMES, + ) + + # trigger + bootstrap_cli(**self.cli_context) + + # verify + load_saved_pipeline_user_mock.assert_called_once() + gc_instance.run.assert_called_once() + stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) + stage_instance.print_resources_summary.assert_called_once() + stage_instance.save_config.assert_called_once_with( + config_dir=ANY_CONFIG_DIR, filename=PIPELINE_TOML_FILE, cmd_names=PIPELINE_BOOTSTRAP_COMMAND_NAMES + ) + + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli.Stage") + @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") + def test_bootstrap_will_not_try_loading_pipeline_user_if_already_provided( + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + ): + bootstrap_cli(**self.cli_context) + load_saved_pipeline_user_mock.assert_not_called() + + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli.Stage") + @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") + def test_bootstrap_will_try_loading_pipeline_user_if_not_provided( + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + ): + self.cli_context["pipeline_user_arn"] = None + bootstrap_cli(**self.cli_context) + load_saved_pipeline_user_mock.assert_called_once() + + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli.Stage") + @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") + def test_stage_name_is_required_to_be_provided_in_case_of_non_interactive_mode( + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + ): + self.cli_context["interactive"] = False + self.cli_context["stage_name"] = None + with self.assertRaises(click.UsageError): + bootstrap_cli(**self.cli_context) + + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli.Stage") + @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") + def test_stage_name_is_not_required_to_be_provided_in_case_of_interactive_mode( + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + ): + self.cli_context["interactive"] = True + self.cli_context["stage_name"] = None + bootstrap_cli(**self.cli_context) # No exception is thrown + + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli.Stage") + @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") + def test_guided_context_will_be_enabled_or_disabled_based_on_the_interactive_mode( + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + ): + gc_instance = Mock() + guided_context_mock.return_value = gc_instance + self.cli_context["interactive"] = False + bootstrap_cli(**self.cli_context) + gc_instance.run.assert_not_called() + self.cli_context["interactive"] = True + bootstrap_cli(**self.cli_context) + gc_instance.run.assert_called_once() + + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli.Stage") + @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") + def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_user_choose_not_to( + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + ): + stage_instance = Mock() + stage_mock.return_value = stage_instance + self.cli_context["confirm_changeset"] = False + bootstrap_cli(**self.cli_context) + stage_instance.bootstrap.assert_called_once_with(confirm_changeset=False) + stage_instance.bootstrap.reset_mock() + self.cli_context["confirm_changeset"] = True + bootstrap_cli(**self.cli_context) + stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) + + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli.Stage") + @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") + def test_bootstrapping_will_not_fail_if_saving_resources_arns_to_local_toml_file_failed( + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + ): + # setup + stage_instance = Mock() + stage_mock.return_value = stage_instance + self.cli_context["interactive"] = False + stage_instance.save_config.side_effect = Exception + get_toml_file_metadata_mock.return_value = ( + ANY_CONFIG_DIR, + PIPELINE_TOML_FILE, + PIPELINE_BOOTSTRAP_COMMAND_NAMES, + ) + + # trigger + bootstrap_cli(**self.cli_context) + + # verify + stage_instance.save_config.assert_called_once() # called and the thrown exception got swallowed + + @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + def test_load_saved_pipeline_user_will_return_non_if_the_pipeline_toml_file_is_not_found( + self, get_toml_file_metadata_mock, sam_config_mock + ): + # setup + get_toml_file_metadata_mock.return_value = ( + ANY_CONFIG_DIR, + PIPELINE_TOML_FILE, + PIPELINE_BOOTSTRAP_COMMAND_NAMES, + ) + sam_config_instance_mock = Mock() + sam_config_mock.return_value = sam_config_instance_mock + sam_config_instance_mock.exists.return_value = False + + # trigger + pipeline_user_arn = _load_saved_pipeline_user() + + # verify + self.assertIsNone(pipeline_user_arn) + + @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + def test_load_saved_pipeline_user_will_return_non_if_the_pipeline_toml_file_does_not_contain_pipeline_user( + self, get_toml_file_metadata_mock, sam_config_mock + ): + # setup + get_toml_file_metadata_mock.return_value = ( + ANY_CONFIG_DIR, + PIPELINE_TOML_FILE, + PIPELINE_BOOTSTRAP_COMMAND_NAMES, + ) + sam_config_instance_mock = Mock() + sam_config_mock.return_value = sam_config_instance_mock + sam_config_instance_mock.exists.return_value = True + sam_config_instance_mock.get_all.return_value = {"non-pipeline_user-key": "any_value"} + + # trigger + pipeline_user_arn = _load_saved_pipeline_user() + + # verify + self.assertIsNone(pipeline_user_arn) + + @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") + @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + def test_load_saved_pipeline_user_returns_the_pipeline_user_arn_from_the_pipeline_toml_file( + self, get_toml_file_metadata_mock, sam_config_mock + ): + # setup + get_toml_file_metadata_mock.return_value = ( + ANY_CONFIG_DIR, + PIPELINE_TOML_FILE, + PIPELINE_BOOTSTRAP_COMMAND_NAMES, + ) + sam_config_instance_mock = Mock() + sam_config_mock.return_value = sam_config_instance_mock + sam_config_instance_mock.exists.return_value = True + sam_config_instance_mock.get_all.return_value = {"pipeline_user": ANY_PIPELINE_USER_ARN} + + # trigger + pipeline_user_arn = _load_saved_pipeline_user() + + # verify + self.assertEqual(pipeline_user_arn, ANY_PIPELINE_USER_ARN) + + @patch("samcli.commands.pipeline.bootstrap.cli.click") + @patch("samcli.commands.pipeline.bootstrap.cli.get_cmd_names") + def test_get_toml_file_metadata_when_click_context_defines_samconfig_dir(self, get_cmd_names_mock, click_mock): + # setup + get_cmd_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES + ctx_mock = Mock(samconfig_dir=ANY_CONFIG_DIR) + click_mock.get_current_context.return_value = ctx_mock + + # trigger + samconfig_dir, pipeline_samconfig_filename, cmd_names = _get_toml_file_metadata() + + # verify + self.assertEqual(samconfig_dir, ANY_CONFIG_DIR) + self.assertEqual(pipeline_samconfig_filename, "pipelineconfig.toml") # Hardcoded + self.assertEqual(cmd_names, PIPELINE_BOOTSTRAP_COMMAND_NAMES) + + @patch("samcli.commands.pipeline.bootstrap.cli.click") + @patch("samcli.commands.pipeline.bootstrap.cli.get_cmd_names") + def test_get_toml_file_metadata_when_click_context_does_not_define_samconfig_dir( + self, get_cmd_names_mock, click_mock + ): + # setup + get_cmd_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES + ctx_mock = Mock(spec=["info_name"]) + click_mock.get_current_context.return_value = ctx_mock + + # trigger + samconfig_dir, pipeline_samconfig_filename, cmd_names = _get_toml_file_metadata() + + # verify + self.assertEqual(samconfig_dir, os.getcwd()) + self.assertEqual(pipeline_samconfig_filename, "pipelineconfig.toml") # Hardcoded + self.assertEqual(cmd_names, PIPELINE_BOOTSTRAP_COMMAND_NAMES) diff --git a/tests/unit/commands/pipeline/bootstrap/test_guided_context.py b/tests/unit/commands/pipeline/bootstrap/test_guided_context.py new file mode 100644 index 00000000000..dfdff1976b7 --- /dev/null +++ b/tests/unit/commands/pipeline/bootstrap/test_guided_context.py @@ -0,0 +1,92 @@ +from unittest import TestCase +from unittest.mock import patch + +from samcli.commands.pipeline.bootstrap.guided_context import GuidedContext + +ANY_STAGE_NAME = "ANY_STAGE_NAME" +ANY_PIPELINE_USER_ARN = "ANY_PIPELINE_USER_ARN" +ANY_PIPELINE_EXECUTION_ROLE_ARN = "ANY_PIPELINE_EXECUTION_ROLE_ARN" +ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN = "ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN" +ANY_ARTIFACTS_BUCKET_ARN = "ANY_ARTIFACTS_BUCKET_ARN" +ANY_ECR_REPO_ARN = "ANY_ECR_REPO_ARN" +ANY_ARN = "ANY_ARN" +ANY_PIPELINE_IP_RANGE = "111.222.333.0/24" + + +class TestGuidedContext(TestCase): + @patch("samcli.commands.pipeline.bootstrap.guided_context.click") + def test_guided_context_will_not_prompt_for_fields_that_are_already_provided(self, click_mock): + gc: GuidedContext = GuidedContext( + stage_name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + pipeline_execution_role_arn=ANY_PIPELINE_EXECUTION_ROLE_ARN, + cloudformation_execution_role_arn=ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN, + artifacts_bucket_arn=ANY_ARTIFACTS_BUCKET_ARN, + create_ecr_repo=True, + ecr_repo_arn=ANY_ECR_REPO_ARN, + pipeline_ip_range=ANY_PIPELINE_IP_RANGE, + ) + gc.run() + click_mock.prompt.assert_not_called() + + @patch("samcli.commands.pipeline.bootstrap.guided_context.click") + def test_guided_context_will_prompt_for_fields_that_are_not_provided(self, click_mock): + gc: GuidedContext = GuidedContext( + ecr_repo_arn=ANY_ECR_REPO_ARN # Exclude ECR repo, it has its own detailed test below + ) + gc.run() + self.assertTrue(self.did_prompt_text_like("Stage Name", click_mock.prompt)) + self.assertTrue(self.did_prompt_text_like("Pipeline user", click_mock.prompt)) + self.assertTrue(self.did_prompt_text_like("Pipeline execution role", click_mock.prompt)) + self.assertTrue(self.did_prompt_text_like("CloudFormation execution role", click_mock.prompt)) + self.assertTrue(self.did_prompt_text_like("Artifacts bucket", click_mock.prompt)) + self.assertTrue(self.did_prompt_text_like("Artifacts bucket", click_mock.prompt)) + self.assertTrue(self.did_prompt_text_like("Pipeline IP address range", click_mock.prompt)) + + @patch("samcli.commands.pipeline.bootstrap.guided_context.click") + def test_guided_context_will_not_prompt_for_not_provided_ecr_repo_if_no_ecr_repo_is_required(self, click_mock): + # ECR Repo choices: + # 1 - No, My SAM Template won't include lambda functions of Image package-type + # 2 - Yes, I need a help creating one + # 3 - I already have an ECR repo + gc_without_ecr_info: GuidedContext = GuidedContext( + stage_name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + pipeline_execution_role_arn=ANY_PIPELINE_EXECUTION_ROLE_ARN, + cloudformation_execution_role_arn=ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN, + artifacts_bucket_arn=ANY_ARTIFACTS_BUCKET_ARN, + pipeline_ip_range=ANY_PIPELINE_IP_RANGE, + ) + + self.assertIsNone(gc_without_ecr_info.ecr_repo_arn) + + click_mock.prompt.return_value = "1" # the user chose to not CREATE an ECR repo + gc_without_ecr_info.run() + self.assertIsNone(gc_without_ecr_info.ecr_repo_arn) + self.assertFalse(gc_without_ecr_info.create_ecr_repo) + self.assertFalse(self.did_prompt_text_like("ECR repo", click_mock.prompt)) + + click_mock.prompt.return_value = "2" # the user chose to CREATE an ECR repo + gc_without_ecr_info.run() + self.assertIsNone(gc_without_ecr_info.ecr_repo_arn) + self.assertTrue(gc_without_ecr_info.create_ecr_repo) + self.assertFalse(self.did_prompt_text_like("ECR repo", click_mock.prompt)) + + click_mock.prompt.side_effect = ["3", ANY_ECR_REPO_ARN] # the user already has a repo + gc_without_ecr_info.run() + self.assertFalse(gc_without_ecr_info.create_ecr_repo) + self.assertTrue(self.did_prompt_text_like("ECR repo", click_mock.prompt)) # we've asked about it + self.assertEqual(gc_without_ecr_info.ecr_repo_arn, ANY_ECR_REPO_ARN) + + @staticmethod + def did_prompt_text_like(txt, click_prompt_mock): + txt = txt.lower() + for kall in click_prompt_mock.call_args_list: + args, kwargs = kall + if args: + text = args[0].lower() + else: + text = kwargs.get("text", "").lower() + if txt in text: + return True + return False diff --git a/tests/unit/lib/bootstrap/test_bootstrap.py b/tests/unit/lib/bootstrap/test_bootstrap.py index 8094a404c09..7abfc522cea 100644 --- a/tests/unit/lib/bootstrap/test_bootstrap.py +++ b/tests/unit/lib/bootstrap/test_bootstrap.py @@ -2,22 +2,26 @@ from unittest.mock import patch from samcli.commands.exceptions import UserException -from samcli.lib.bootstrap.bootstrap import manage_stack +from samcli.lib.bootstrap.bootstrap import manage_stack, StackOutput class TestBootstrapManagedStack(TestCase): @patch("samcli.lib.bootstrap.bootstrap.manage_cloudformation_stack") def test_stack_missing_bucket(self, manage_cfn_stack_mock): - manage_cfn_stack_mock.return_value = [] + manage_cfn_stack_mock.return_value = StackOutput(stack_output=[]) with self.assertRaises(UserException): manage_stack("testProfile", "fakeRegion") - manage_cfn_stack_mock.return_value = [{"OutputKey": "NotSourceBucket", "OutputValue": "AnyValue"}] + manage_cfn_stack_mock.return_value = StackOutput( + stack_output=[{"OutputKey": "NotSourceBucket", "OutputValue": "AnyValue"}] + ) with self.assertRaises(UserException): manage_stack("testProfile", "fakeRegion") @patch("samcli.lib.bootstrap.bootstrap.manage_cloudformation_stack") def test_manage_stack_happy_case(self, manage_cfn_stack_mock): expected_bucket_name = "BucketName" - manage_cfn_stack_mock.return_value = [{"OutputKey": "SourceBucket", "OutputValue": expected_bucket_name}] + manage_cfn_stack_mock.return_value = StackOutput( + stack_output=[{"OutputKey": "SourceBucket", "OutputValue": expected_bucket_name}] + ) actual_bucket_name = manage_stack("testProfile", "fakeRegion") self.assertEqual(actual_bucket_name, expected_bucket_name) diff --git a/tests/unit/lib/pipeline/bootstrap/__init__.py b/tests/unit/lib/pipeline/bootstrap/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/lib/pipeline/bootstrap/test_resource.py b/tests/unit/lib/pipeline/bootstrap/test_resource.py new file mode 100644 index 00000000000..243380a3c75 --- /dev/null +++ b/tests/unit/lib/pipeline/bootstrap/test_resource.py @@ -0,0 +1,93 @@ +from unittest import TestCase +from unittest.mock import Mock, patch, call + +from samcli.lib.pipeline.bootstrap.resource import ARNParts, Resource, S3Bucket, IamUser, EcrRepo + +VALID_ARN = "arn:partition:service:region:account-id:resource-id" +VALID_ARN_WITH_RESOURCE_TYPE = "arn:partition:service:region:account-id:resource-type/resource-id" +INVALID_ARN = "ARN" + + +class TestArnParts(TestCase): + def test_arn_parts_of_valid_arn_without_resource_type(self): + arn_parts: ARNParts = ARNParts(arn=VALID_ARN) + self.assertEqual(arn_parts.partition, "partition") + self.assertEqual(arn_parts.service, "service") + self.assertEqual(arn_parts.region, "region") + self.assertEqual(arn_parts.account_id, "account-id") + self.assertEqual(arn_parts.resource_id, "resource-id") + self.assertIsNone(arn_parts.resource_type) + + def test_arn_parts_of_valid_arn_including_resource_type(self): + arn_parts: ARNParts = ARNParts(arn=VALID_ARN_WITH_RESOURCE_TYPE) + self.assertEqual(arn_parts.partition, "partition") + self.assertEqual(arn_parts.service, "service") + self.assertEqual(arn_parts.region, "region") + self.assertEqual(arn_parts.account_id, "account-id") + self.assertEqual(arn_parts.resource_id, "resource-id") + self.assertEqual(arn_parts.resource_type, "resource-type") + + def test_arn_parts_of_none_arn_is_invalid(self): + with self.assertRaises(ValueError): + ARNParts(arn=None) + + with self.assertRaises(ValueError): + any_non_string = 1 + ARNParts(arn=any_non_string) + + with self.assertRaises(ValueError): + invalid_arn = "invalid_arn" + ARNParts(arn=invalid_arn) + + +class TestResource(TestCase): + def test_resource(self): + resource = Resource(arn=VALID_ARN) + self.assertEqual(resource.arn, VALID_ARN) + self.assertTrue(resource.is_user_provided) + self.assertEqual(resource.name(), "resource-id") + + resource = Resource(arn=INVALID_ARN) + self.assertEqual(resource.arn, INVALID_ARN) + self.assertTrue(resource.is_user_provided) + with self.assertRaises(ValueError): + resource.name() + + resource = Resource(arn=None) + self.assertIsNone(resource.arn) + self.assertFalse(resource.is_user_provided) + self.assertIsNone(resource.name()) + + +class TestIamUser(TestCase): + def test_create_iam_user(self): + user: IamUser = IamUser(arn=VALID_ARN) + self.assertEquals(user.arn, VALID_ARN) + self.assertIsNone(user.access_key_id) + self.assertIsNone(user.secret_access_key) + + user = IamUser(arn=INVALID_ARN, access_key_id="any_access_key_id", secret_access_key="any_secret_access_key") + self.assertEquals(user.arn, INVALID_ARN) + self.assertEquals(user.access_key_id, "any_access_key_id") + self.assertEquals(user.secret_access_key, "any_secret_access_key") + + +class TestS3Bucket(TestCase): + def test_create_s3_bucket(self): + bucket: S3Bucket = S3Bucket(arn=VALID_ARN) + self.assertEquals(bucket.arn, VALID_ARN) + self.assertIsNone(bucket.kms_key_arn) + + bucket = S3Bucket(arn=INVALID_ARN, kms_key_arn="any_kms_key_arn") + self.assertEquals(bucket.arn, INVALID_ARN) + self.assertEquals(bucket.kms_key_arn, "any_kms_key_arn") + + +class TestEcrRepo(TestCase): + def test_get_uri(self): + repo: EcrRepo = EcrRepo(arn=VALID_ARN) + self.assertEqual(repo.get_uri(), "account-id.dkr.ecr.region.amazonaws.com/resource-id") + + repo = EcrRepo(arn=INVALID_ARN) + with self.assertRaises(ValueError): + repo.get_uri() diff --git a/tests/unit/lib/pipeline/bootstrap/test_stage.py b/tests/unit/lib/pipeline/bootstrap/test_stage.py new file mode 100644 index 00000000000..162d933c659 --- /dev/null +++ b/tests/unit/lib/pipeline/bootstrap/test_stage.py @@ -0,0 +1,355 @@ +from unittest import TestCase +from unittest.mock import Mock, patch, call + +from samcli.lib.pipeline.bootstrap.stage import Stage + +ANY_STAGE_NAME = "ANY_STAGE_NAME" +ANY_PIPELINE_USER_ARN = "ANY_PIPELINE_USER_ARN" +ANY_PIPELINE_EXECUTION_ROLE_ARN = "ANY_PIPELINE_EXECUTION_ROLE_ARN" +ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN = "ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN" +ANY_ARTIFACTS_BUCKET_ARN = "ANY_ARTIFACTS_BUCKET_ARN" +ANY_ECR_REPO_ARN = "ANY_ECR_REPO_ARN" +ANY_ARN = "ANY_ARN" + + +class TestStage(TestCase): + def test_stage_name_is_the_only_required_field_to_initialize_a_stage(self): + stage: Stage = Stage(name=ANY_STAGE_NAME) + self.assertEqual(stage.name, ANY_STAGE_NAME) + self.assertIsNone(stage.aws_profile) + self.assertIsNone(stage.aws_region) + self.assertIsNotNone(stage.pipeline_user) + self.assertIsNotNone(stage.pipeline_execution_role) + self.assertIsNotNone(stage.cloudformation_execution_role) + self.assertIsNotNone(stage.artifacts_bucket) + self.assertIsNotNone(stage.ecr_repo) + + with self.assertRaises(TypeError): + stage = Stage() + + def test_did_user_provide_all_required_resources_when_not_all_resources_are_provided(self): + stage: Stage = Stage(name=ANY_STAGE_NAME) + self.assertFalse(stage.did_user_provide_all_required_resources()) + stage: Stage = Stage(name=ANY_STAGE_NAME, pipeline_user_arn=ANY_PIPELINE_USER_ARN) + self.assertFalse(stage.did_user_provide_all_required_resources()) + stage: Stage = Stage( + name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + pipeline_execution_role_arn=ANY_PIPELINE_EXECUTION_ROLE_ARN, + ) + self.assertFalse(stage.did_user_provide_all_required_resources()) + stage: Stage = Stage( + name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + pipeline_execution_role_arn=ANY_PIPELINE_EXECUTION_ROLE_ARN, + cloudformation_execution_role_arn=ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN, + ) + self.assertFalse(stage.did_user_provide_all_required_resources()) + stage: Stage = Stage( + name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + pipeline_execution_role_arn=ANY_PIPELINE_EXECUTION_ROLE_ARN, + cloudformation_execution_role_arn=ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN, + artifacts_bucket_arn=ANY_ARTIFACTS_BUCKET_ARN, + create_ecr_repo=True, + ) + self.assertFalse(stage.did_user_provide_all_required_resources()) + + def test_did_user_provide_all_required_resources_ignore_ecr_repo_if_it_is_not_required(self): + stage: Stage = Stage( + name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + pipeline_execution_role_arn=ANY_PIPELINE_EXECUTION_ROLE_ARN, + cloudformation_execution_role_arn=ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN, + artifacts_bucket_arn=ANY_ARTIFACTS_BUCKET_ARN, + create_ecr_repo=False, + ) + self.assertTrue(stage.did_user_provide_all_required_resources()) + + def test_did_user_provide_all_required_resources_when_ecr_repo_is_required(self): + stage: Stage = Stage( + name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + pipeline_execution_role_arn=ANY_PIPELINE_EXECUTION_ROLE_ARN, + cloudformation_execution_role_arn=ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN, + artifacts_bucket_arn=ANY_ARTIFACTS_BUCKET_ARN, + create_ecr_repo=True, + ) + self.assertFalse(stage.did_user_provide_all_required_resources()) + stage: Stage = Stage( + name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + pipeline_execution_role_arn=ANY_PIPELINE_EXECUTION_ROLE_ARN, + cloudformation_execution_role_arn=ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN, + artifacts_bucket_arn=ANY_ARTIFACTS_BUCKET_ARN, + create_ecr_repo=True, + ecr_repo_arn=ANY_ECR_REPO_ARN, + ) + self.assertTrue(stage.did_user_provide_all_required_resources()) + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + @patch("samcli.lib.pipeline.bootstrap.stage.manage_stack") + def test_did_user_provide_all_required_resources_returns_false_if_the_stage_was_initialized_without_any_of_the_resources_even_if_fulfilled_after_bootstrap( + self, manage_stack_mock, click_mock + ): + # setup + stack_output = Mock() + stack_output.get.return_value = ANY_ARN + manage_stack_mock.return_value = stack_output + stage: Stage = Stage(name=ANY_STAGE_NAME) + + self.assertFalse(stage.did_user_provide_all_required_resources()) + + stage.bootstrap(confirm_changeset=False) + # After bootstrapping, all the resources should be fulfilled + self.assertEqual(ANY_ARN, stage.pipeline_user.arn) + self.assertEqual(ANY_ARN, stage.pipeline_execution_role.arn) + self.assertEqual(ANY_ARN, stage.cloudformation_execution_role.arn) + self.assertEqual(ANY_ARN, stage.artifacts_bucket.arn) + self.assertEqual(ANY_ARN, stage.ecr_repo.arn) + + # although all of the resources got fulfilled, `did_user_provide_all_required_resources` should return false + # as these resources are not provided by the user + self.assertFalse(stage.did_user_provide_all_required_resources()) + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + @patch("samcli.lib.pipeline.bootstrap.stage.manage_stack") + @patch.object(Stage, "did_user_provide_all_required_resources") + def test_bootstrap_will_not_deploy_the_cfn_template_if_all_resources_are_already_provided( + self, did_user_provide_all_required_resources_mock, manage_stack_mock, click_mock + ): + did_user_provide_all_required_resources_mock.return_value = True + stage: Stage = Stage(name=ANY_STAGE_NAME) + stage.bootstrap(confirm_changeset=False) + manage_stack_mock.assert_not_called() + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + @patch("samcli.lib.pipeline.bootstrap.stage.manage_stack") + def test_bootstrap_will_confirm_before_deploying_unless_confirm_changeset_is_disabled( + self, manage_stack_mock, click_mock + ): + click_mock.confirm.return_value = False + stage: Stage = Stage(name=ANY_STAGE_NAME) + stage.bootstrap(confirm_changeset=False) + click_mock.confirm.assert_not_called() + manage_stack_mock.assert_called_once() + manage_stack_mock.reset_mock() + stage.bootstrap(confirm_changeset=True) + click_mock.confirm.assert_called_once() + manage_stack_mock.assert_not_called() # As the user choose to not confirm + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + @patch("samcli.lib.pipeline.bootstrap.stage.manage_stack") + def test_bootstrap_will_not_deploy_the_cfn_template_if_the_user_did_not_confirm( + self, manage_stack_mock, click_mock + ): + click_mock.confirm.return_value = False + stage: Stage = Stage(name=ANY_STAGE_NAME) + stage.bootstrap(confirm_changeset=True) + manage_stack_mock.assert_not_called() + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + @patch("samcli.lib.pipeline.bootstrap.stage.manage_stack") + def test_bootstrap_will_deploy_the_cfn_template_if_the_user_did_confirm(self, manage_stack_mock, click_mock): + click_mock.confirm.return_value = True + stage: Stage = Stage(name=ANY_STAGE_NAME) + stage.bootstrap(confirm_changeset=True) + manage_stack_mock.assert_called_once() + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + @patch("samcli.lib.pipeline.bootstrap.stage.manage_stack") + def test_bootstrap_will_pass_arns_of_all_user_provided_resources_any_empty_strings_for_other_resources_to_the_cfn_stack( + self, manage_stack_mock, click_mock + ): + click_mock.confirm.return_value = True + stage: Stage = Stage( + name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + artifacts_bucket_arn=ANY_ARTIFACTS_BUCKET_ARN, + create_ecr_repo=True, + ecr_repo_arn=ANY_ECR_REPO_ARN, + ) + stage.bootstrap() + manage_stack_mock.assert_called_once() + args, kwargs = manage_stack_mock.call_args_list[0] + expected_parameter_overrides = { + "PipelineUserArn": ANY_PIPELINE_USER_ARN, + "PipelineExecutionRoleArn": "", + "PipelineIpRange": "", + "CloudFormationExecutionRoleArn": "", + "ArtifactsBucketArn": ANY_ARTIFACTS_BUCKET_ARN, + "CreateEcrRepo": "true", + "EcrRepoArn": ANY_ECR_REPO_ARN, + } + self.assertEqual(expected_parameter_overrides, kwargs["parameter_overrides"]) + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + @patch("samcli.lib.pipeline.bootstrap.stage.manage_stack") + def test_bootstrap_will_fullfill_all_resource_arns(self, manage_stack_mock, click_mock): + # setup + stack_output = Mock() + stack_output.get.return_value = ANY_ARN + manage_stack_mock.return_value = stack_output + stage: Stage = Stage(name=ANY_STAGE_NAME) + click_mock.confirm.return_value = True + + # verify resources' ARNS are empty + self.assertIsNone(stage.pipeline_user.arn) + self.assertIsNone(stage.pipeline_execution_role.arn) + self.assertIsNone(stage.cloudformation_execution_role.arn) + self.assertIsNone(stage.artifacts_bucket.arn) + + # trigger + stage.bootstrap() + + # verify + manage_stack_mock.assert_called_once() + self.assertEqual(ANY_ARN, stage.pipeline_user.arn) + self.assertEqual(ANY_ARN, stage.pipeline_execution_role.arn) + self.assertEqual(ANY_ARN, stage.cloudformation_execution_role.arn) + self.assertEqual(ANY_ARN, stage.artifacts_bucket.arn) + + @patch("samcli.lib.pipeline.bootstrap.stage.SamConfig") + def test_save_config_escapes_non_resources(self, samconfig_mock): + cmd_names = ["any", "commands"] + samconfig_instance_mock = Mock() + samconfig_mock.return_value = samconfig_instance_mock + stage: Stage = Stage(name=ANY_STAGE_NAME) + + expected_calls = [] + self.trigger_and_assert_save_config_calls(stage, cmd_names, expected_calls, samconfig_instance_mock.put) + + stage.pipeline_user.arn = ANY_PIPELINE_USER_ARN + expected_calls.append( + call(cmd_names=cmd_names, section="parameters", key="pipeline_user", value=ANY_PIPELINE_USER_ARN) + ) + self.trigger_and_assert_save_config_calls(stage, cmd_names, expected_calls, samconfig_instance_mock.put) + + stage.pipeline_execution_role.arn = ANY_PIPELINE_EXECUTION_ROLE_ARN + expected_calls.append( + call( + cmd_names=cmd_names, + section="parameters", + env="ANY_STAGE_NAME", + key="pipeline_execution_role", + value=ANY_PIPELINE_EXECUTION_ROLE_ARN, + ) + ) + self.trigger_and_assert_save_config_calls(stage, cmd_names, expected_calls, samconfig_instance_mock.put) + + stage.cloudformation_execution_role.arn = ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN + expected_calls.append( + call( + cmd_names=cmd_names, + section="parameters", + env="ANY_STAGE_NAME", + key="cloudformation_execution_role", + value=ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN, + ) + ) + self.trigger_and_assert_save_config_calls(stage, cmd_names, expected_calls, samconfig_instance_mock.put) + + stage.artifacts_bucket.arn = "arn:aws:s3:::artifact_bucket_name" + expected_calls.append( + call( + cmd_names=cmd_names, + section="parameters", + env="ANY_STAGE_NAME", + key="artifacts_bucket", + value="artifact_bucket_name", + ) + ) + self.trigger_and_assert_save_config_calls(stage, cmd_names, expected_calls, samconfig_instance_mock.put) + + stage.ecr_repo.arn = "arn:aws:ecr:us-east-2:111111111111:ecr_repo_name" + expected_calls.append( + call( + cmd_names=cmd_names, + section="parameters", + env="ANY_STAGE_NAME", + key="ecr_repo", + value="111111111111.dkr.ecr.us-east-2.amazonaws.com/ecr_repo_name", + ) + ) + self.trigger_and_assert_save_config_calls(stage, cmd_names, expected_calls, samconfig_instance_mock.put) + + def trigger_and_assert_save_config_calls(self, stage, cmd_names, expected_calls, samconfig_put_mock): + stage.save_config(config_dir="any_config_dir", filename="any_pipeline.toml", cmd_names=cmd_names) + self.assertEqual(len(expected_calls), samconfig_put_mock.call_count) + samconfig_put_mock.assert_has_calls(expected_calls) + samconfig_put_mock.reset_mock() + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + def test_print_resources_summary_when_no_resources_provided_by_the_user(self, click_mock): + stage: Stage = Stage(name=ANY_STAGE_NAME) + stage.print_resources_summary() + self.assert_summary_has_a_message_like("We have created the following resources", click_mock.secho) + self.assert_summary_does_not_have_a_message_like("You provided the following resources", click_mock.secho) + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + def test_print_resources_summary_when_all_resources_are_provided_by_the_user(self, click_mock): + stage: Stage = Stage( + name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + pipeline_execution_role_arn=ANY_PIPELINE_EXECUTION_ROLE_ARN, + cloudformation_execution_role_arn=ANY_CLOUDFORMATION_EXECUTION_ROLE_ARN, + artifacts_bucket_arn=ANY_ARTIFACTS_BUCKET_ARN, + create_ecr_repo=True, + ecr_repo_arn=ANY_ECR_REPO_ARN, + ) + stage.print_resources_summary() + self.assert_summary_does_not_have_a_message_like("We have created the following resources", click_mock.secho) + self.assert_summary_has_a_message_like("You provided the following resources", click_mock.secho) + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + def test_print_resources_summary_when_some_resources_are_provided_by_the_user(self, click_mock): + stage: Stage = Stage( + name=ANY_STAGE_NAME, + pipeline_user_arn=ANY_PIPELINE_USER_ARN, + artifacts_bucket_arn=ANY_ARTIFACTS_BUCKET_ARN, + create_ecr_repo=True, + ecr_repo_arn=ANY_ECR_REPO_ARN, + ) + stage.print_resources_summary() + self.assert_summary_has_a_message_like("We have created the following resources", click_mock.secho) + self.assert_summary_has_a_message_like("You provided the following resources", click_mock.secho) + + @patch("samcli.lib.pipeline.bootstrap.stage.click") + def test_print_resources_summary_prints_the_credentials_of_the_pipeline_user_iff_not_provided_by_the_user( + self, click_mock + ): + stage_with_provided_pipeline_user: Stage = Stage(name=ANY_STAGE_NAME, pipeline_user_arn=ANY_PIPELINE_USER_ARN) + stage_with_provided_pipeline_user.print_resources_summary() + self.assert_summary_does_not_have_a_message_like("ACCESS_KEY_ID", click_mock.secho) + self.assert_summary_does_not_have_a_message_like("SECRET_ACCESS_KEY", click_mock.secho) + click_mock.secho.reset_mock() + + stage_without_provided_pipeline_user: Stage = Stage(name=ANY_STAGE_NAME) + stage_without_provided_pipeline_user.print_resources_summary() + self.assert_summary_has_a_message_like("ACCESS_KEY_ID", click_mock.secho) + self.assert_summary_has_a_message_like("SECRET_ACCESS_KEY", click_mock.secho) + + def assert_summary_has_a_message_like(self, msg, click_secho_mock): + self.assertTrue( + self.does_summary_have_a_message_like(msg, click_secho_mock), + msg=f'stage resources summary does not include "{msg}" which is unexpected', + ) + + def assert_summary_does_not_have_a_message_like(self, msg, click_secho_mock): + self.assertFalse( + self.does_summary_have_a_message_like(msg, click_secho_mock), + msg=f'stage resources summary includes "{msg}" which is unexpected', + ) + + @staticmethod + def does_summary_have_a_message_like(msg, click_secho_mock): + msg = msg.lower() + for kall in click_secho_mock.call_args_list: + args, kwargs = kall + if args: + message = args[0].lower() + else: + message = kwargs.get("message", "").lower() + if msg in message: + return True + return False From 97474133d71a11c52d4742af1789932fb09bc78d Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Thu, 15 Apr 2021 14:16:43 -0700 Subject: [PATCH 09/31] move the pipelineconfig.toml file to .aws-sam --- samcli/commands/pipeline/bootstrap/cli.py | 24 ++-- samcli/lib/config/samconfig.py | 9 +- samcli/lib/pipeline/bootstrap/stage.py | 1 - .../commands/pipeline/bootstrap/test_cli.py | 132 ++++++++---------- tests/unit/lib/samconfig/test_samconfig.py | 20 ++- 5 files changed, 94 insertions(+), 92 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index 7d34051389e..33da3b55cd8 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -1,7 +1,8 @@ """ CLI command for "pipeline bootstrap", which sets up the require pipeline infrastructure resources """ -from typing import Any, Dict, Optional, Tuple +import os +from typing import Any, cast, Dict, List, Optional import click @@ -24,8 +25,8 @@ \n\t - ECR repo for the container images of Lambda functions having PackageType property set to Image """ -DEFAULT_SAMCONFIG_DIR = "samconfig_dir" -PIPELINE_SAMCONFIG_FILENAME = "pipelineconfig.toml" +PIPELINE_CONFIG_DIR = os.path.join(".aws-sam", "pipeline") +PIPELINE_CONFIG_FILENAME = "pipelineconfig.toml" @click.command("bootstrap", short_help=SHORT_HELP, help=HELP_TEXT, context_settings=dict(max_content_width=120)) @@ -197,8 +198,9 @@ def do_cli( stage.print_resources_summary() try: - samconfig_dir, filename, cmd_names = _get_toml_file_metadata() - stage.save_config(config_dir=samconfig_dir, filename=filename, cmd_names=cmd_names) + stage.save_config( + config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=_get_command_name() + ) except Exception: # Swallow saving exceptions, if any, as the resources are already bootstrapped and the ARNs are already # printed out in the screen. @@ -206,16 +208,14 @@ def do_cli( def _load_saved_pipeline_user() -> Optional[str]: - samconfig_dir, filename, cmd_names = _get_toml_file_metadata() - samconfig: SamConfig = SamConfig(config_dir=samconfig_dir, filename=filename) + samconfig: SamConfig = SamConfig(config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME) if not samconfig.exists(): return None - config: Dict[str, str] = samconfig.get_all(cmd_names=cmd_names, section="parameters") + config: Dict[str, str] = samconfig.get_all(cmd_names=_get_command_name(), section="parameters") return config.get("pipeline_user") -def _get_toml_file_metadata() -> Tuple: +def _get_command_name() -> List[str]: ctx = click.get_current_context() - samconfig_dir: str = getattr(ctx, DEFAULT_SAMCONFIG_DIR, SamConfig.config_dir()) - cmd_names = get_cmd_names(ctx.info_name, ctx) # ["pipeline", "bootstrap"] - return samconfig_dir, PIPELINE_SAMCONFIG_FILENAME, cmd_names + cmd_names: List[str] = cast(List[str], get_cmd_names(ctx.info_name, ctx)) # ["pipeline", "bootstrap"] + return cmd_names diff --git a/samcli/lib/config/samconfig.py b/samcli/lib/config/samconfig.py index 996ac5f6488..6cda8a075b4 100644 --- a/samcli/lib/config/samconfig.py +++ b/samcli/lib/config/samconfig.py @@ -153,6 +153,11 @@ def sanity_check(self): def exists(self): return self.filepath.exists() + def _ensure_exists(self): + if not self.exists(): + self.filepath.parent.mkdir(parents=True, exist_ok=True) + open(self.filepath, "a+").close() + def path(self): return str(self.filepath) @@ -183,8 +188,8 @@ def _read(self): def _write(self): if not self.document: return - if not self.exists(): - open(self.filepath, "a+").close() + + self._ensure_exists() current_version = self._version() if self._version() else SAM_CONFIG_VERSION try: diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index bdf73565f22..a1814111ad9 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -12,7 +12,6 @@ CFN_TEMPLATE_PATH = str(pathlib.Path(os.path.dirname(__file__))) STACK_NAME_PREFIX = "aws-sam-cli-managed" -DEPLOYER_STACK_NAME_SUFFIX = "pipeline-deployer" STAGE_RESOURCES_STACK_NAME_SUFFIX = "pipeline-resources" diff --git a/tests/unit/commands/pipeline/bootstrap/test_cli.py b/tests/unit/commands/pipeline/bootstrap/test_cli.py index da8b501200d..9c7549b0f81 100644 --- a/tests/unit/commands/pipeline/bootstrap/test_cli.py +++ b/tests/unit/commands/pipeline/bootstrap/test_cli.py @@ -5,7 +5,12 @@ import click from click.testing import CliRunner -from samcli.commands.pipeline.bootstrap.cli import _load_saved_pipeline_user, _get_toml_file_metadata +from samcli.commands.pipeline.bootstrap.cli import ( + _load_saved_pipeline_user, + _get_command_name, + PIPELINE_CONFIG_FILENAME, + PIPELINE_CONFIG_DIR, +) from samcli.commands.pipeline.bootstrap.cli import cli as bootstrap_cmd from samcli.commands.pipeline.bootstrap.cli import do_cli as bootstrap_cli @@ -21,8 +26,6 @@ ANY_PIPELINE_IP_RANGE = "111.222.333.0/24" ANY_CONFIG_FILE = "ANY_CONFIG_FILE" ANY_CONFIG_ENV = "ANY_CONFIG_ENV" -ANY_CONFIG_DIR = "ANY_CONFIG_DIR" -PIPELINE_TOML_FILE = "PIPELINE_TOML_FILE" PIPELINE_BOOTSTRAP_COMMAND_NAMES = ["pipeline", "bootstrap"] @@ -97,12 +100,12 @@ def test_bootstrap_command_with_different_arguments_combination(self, do_cli_moc self.assertEqual(kwargs["stage_name"], "stage1") self.assertEqual(kwargs["artifacts_bucket_arn"], "bucketARN") - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrapping_normal_interactive_flow( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock ): # setup gc_instance = Mock() @@ -112,11 +115,7 @@ def test_bootstrapping_normal_interactive_flow( load_saved_pipeline_user_mock.return_value = ANY_PIPELINE_USER_ARN self.cli_context["interactive"] = True self.cli_context["pipeline_user_arn"] = None - get_toml_file_metadata_mock.return_value = ( - ANY_CONFIG_DIR, - PIPELINE_TOML_FILE, - PIPELINE_BOOTSTRAP_COMMAND_NAMES, - ) + get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES # trigger bootstrap_cli(**self.cli_context) @@ -127,59 +126,61 @@ def test_bootstrapping_normal_interactive_flow( stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) stage_instance.print_resources_summary.assert_called_once() stage_instance.save_config.assert_called_once_with( - config_dir=ANY_CONFIG_DIR, filename=PIPELINE_TOML_FILE, cmd_names=PIPELINE_BOOTSTRAP_COMMAND_NAMES + config_dir=PIPELINE_CONFIG_DIR, + filename=PIPELINE_CONFIG_FILENAME, + cmd_names=PIPELINE_BOOTSTRAP_COMMAND_NAMES, ) - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrap_will_not_try_loading_pipeline_user_if_already_provided( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock ): bootstrap_cli(**self.cli_context) load_saved_pipeline_user_mock.assert_not_called() - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrap_will_try_loading_pipeline_user_if_not_provided( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock ): self.cli_context["pipeline_user_arn"] = None bootstrap_cli(**self.cli_context) load_saved_pipeline_user_mock.assert_called_once() - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_stage_name_is_required_to_be_provided_in_case_of_non_interactive_mode( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock ): self.cli_context["interactive"] = False self.cli_context["stage_name"] = None with self.assertRaises(click.UsageError): bootstrap_cli(**self.cli_context) - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_stage_name_is_not_required_to_be_provided_in_case_of_interactive_mode( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock ): self.cli_context["interactive"] = True self.cli_context["stage_name"] = None bootstrap_cli(**self.cli_context) # No exception is thrown - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_guided_context_will_be_enabled_or_disabled_based_on_the_interactive_mode( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock ): gc_instance = Mock() guided_context_mock.return_value = gc_instance @@ -190,12 +191,12 @@ def test_guided_context_will_be_enabled_or_disabled_based_on_the_interactive_mod bootstrap_cli(**self.cli_context) gc_instance.run.assert_called_once() - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_user_choose_not_to( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock ): stage_instance = Mock() stage_mock.return_value = stage_instance @@ -207,23 +208,19 @@ def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_use bootstrap_cli(**self.cli_context) stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrapping_will_not_fail_if_saving_resources_arns_to_local_toml_file_failed( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_toml_file_metadata_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock ): # setup stage_instance = Mock() stage_mock.return_value = stage_instance self.cli_context["interactive"] = False stage_instance.save_config.side_effect = Exception - get_toml_file_metadata_mock.return_value = ( - ANY_CONFIG_DIR, - PIPELINE_TOML_FILE, - PIPELINE_BOOTSTRAP_COMMAND_NAMES, - ) + get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES # trigger bootstrap_cli(**self.cli_context) @@ -232,16 +229,27 @@ def test_bootstrapping_will_not_fail_if_saving_resources_arns_to_local_toml_file stage_instance.save_config.assert_called_once() # called and the thrown exception got swallowed @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + def test_load_saved_pipeline_user_will_read_from_the_correct_file(self, get_command_name_mock, sam_config_mock): + # setup + get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES + sam_config_instance_mock = Mock() + sam_config_mock.return_value = sam_config_instance_mock + sam_config_instance_mock.exists.return_value = False + + # trigger + _load_saved_pipeline_user() + + # verify + sam_config_mock.assert_called_once_with(config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME) + + @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") def test_load_saved_pipeline_user_will_return_non_if_the_pipeline_toml_file_is_not_found( - self, get_toml_file_metadata_mock, sam_config_mock + self, get_command_name_mock, sam_config_mock ): # setup - get_toml_file_metadata_mock.return_value = ( - ANY_CONFIG_DIR, - PIPELINE_TOML_FILE, - PIPELINE_BOOTSTRAP_COMMAND_NAMES, - ) + get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() sam_config_mock.return_value = sam_config_instance_mock sam_config_instance_mock.exists.return_value = False @@ -253,16 +261,12 @@ def test_load_saved_pipeline_user_will_return_non_if_the_pipeline_toml_file_is_n self.assertIsNone(pipeline_user_arn) @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") def test_load_saved_pipeline_user_will_return_non_if_the_pipeline_toml_file_does_not_contain_pipeline_user( - self, get_toml_file_metadata_mock, sam_config_mock + self, get_command_name_mock, sam_config_mock ): # setup - get_toml_file_metadata_mock.return_value = ( - ANY_CONFIG_DIR, - PIPELINE_TOML_FILE, - PIPELINE_BOOTSTRAP_COMMAND_NAMES, - ) + get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() sam_config_mock.return_value = sam_config_instance_mock sam_config_instance_mock.exists.return_value = True @@ -275,16 +279,12 @@ def test_load_saved_pipeline_user_will_return_non_if_the_pipeline_toml_file_does self.assertIsNone(pipeline_user_arn) @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") - @patch("samcli.commands.pipeline.bootstrap.cli._get_toml_file_metadata") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") def test_load_saved_pipeline_user_returns_the_pipeline_user_arn_from_the_pipeline_toml_file( - self, get_toml_file_metadata_mock, sam_config_mock + self, get_command_name_mock, sam_config_mock ): # setup - get_toml_file_metadata_mock.return_value = ( - ANY_CONFIG_DIR, - PIPELINE_TOML_FILE, - PIPELINE_BOOTSTRAP_COMMAND_NAMES, - ) + get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() sam_config_mock.return_value = sam_config_instance_mock sam_config_instance_mock.exists.return_value = True @@ -296,36 +296,18 @@ def test_load_saved_pipeline_user_returns_the_pipeline_user_arn_from_the_pipelin # verify self.assertEqual(pipeline_user_arn, ANY_PIPELINE_USER_ARN) - @patch("samcli.commands.pipeline.bootstrap.cli.click") @patch("samcli.commands.pipeline.bootstrap.cli.get_cmd_names") - def test_get_toml_file_metadata_when_click_context_defines_samconfig_dir(self, get_cmd_names_mock, click_mock): - # setup - get_cmd_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES - ctx_mock = Mock(samconfig_dir=ANY_CONFIG_DIR) - click_mock.get_current_context.return_value = ctx_mock - - # trigger - samconfig_dir, pipeline_samconfig_filename, cmd_names = _get_toml_file_metadata() - - # verify - self.assertEqual(samconfig_dir, ANY_CONFIG_DIR) - self.assertEqual(pipeline_samconfig_filename, "pipelineconfig.toml") # Hardcoded - self.assertEqual(cmd_names, PIPELINE_BOOTSTRAP_COMMAND_NAMES) - @patch("samcli.commands.pipeline.bootstrap.cli.click") - @patch("samcli.commands.pipeline.bootstrap.cli.get_cmd_names") - def test_get_toml_file_metadata_when_click_context_does_not_define_samconfig_dir( - self, get_cmd_names_mock, click_mock - ): + def test_get_command_name(self, click_mock, get_cmd_names_mock): # setup - get_cmd_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES - ctx_mock = Mock(spec=["info_name"]) + ctx_mock = Mock(spec=["info_name"], info_name="bootstrap") click_mock.get_current_context.return_value = ctx_mock + get_cmd_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES # trigger - samconfig_dir, pipeline_samconfig_filename, cmd_names = _get_toml_file_metadata() + cmd_names = _get_command_name() # verify - self.assertEqual(samconfig_dir, os.getcwd()) - self.assertEqual(pipeline_samconfig_filename, "pipelineconfig.toml") # Hardcoded self.assertEqual(cmd_names, PIPELINE_BOOTSTRAP_COMMAND_NAMES) + click_mock.get_current_context.assert_called_once() + get_cmd_names_mock.assert_called_once_with("bootstrap", ctx_mock) diff --git a/tests/unit/lib/samconfig/test_samconfig.py b/tests/unit/lib/samconfig/test_samconfig.py index 74c9ee96619..b5a0ada6c37 100644 --- a/tests/unit/lib/samconfig/test_samconfig.py +++ b/tests/unit/lib/samconfig/test_samconfig.py @@ -1,11 +1,12 @@ import os +import shutil from pathlib import Path - from unittest import TestCase from samcli.lib.config.exceptions import SamConfigVersionException -from samcli.lib.config.version import VERSION_KEY, SAM_CONFIG_VERSION from samcli.lib.config.samconfig import SamConfig, DEFAULT_CONFIG_FILE_NAME, DEFAULT_GLOBAL_CMDNAME +from samcli.lib.config.version import VERSION_KEY, SAM_CONFIG_VERSION +from samcli.lib.utils import osutils class TestSamConfig(TestCase): @@ -195,3 +196,18 @@ def test_write_config_file_non_standard_version(self): self.samconfig.put(cmd_names=["local", "start", "api"], section="parameters", key="skip_pull_image", value=True) self.samconfig.sanity_check() self.assertEqual(self.samconfig.document.get(VERSION_KEY), 0.2) + + def test_write_config_file_will_create_the_file_if_not_exist(self): + with osutils.mkdir_temp(ignore_errors=True) as tempdir: + non_existing_dir = os.path.join(tempdir, "non-existing-dir") + non_existing_file = "non-existing-file" + samconfig = SamConfig(config_dir=non_existing_dir, filename=non_existing_file) + + self.assertFalse(samconfig.exists()) + + samconfig.flush() + self.assertFalse(samconfig.exists()) # nothing to write, no need to create the file + + samconfig.put(cmd_names=["any", "command"], section="any-section", key="any-key", value="any-value") + samconfig.flush() + self.assertTrue(samconfig.exists()) From 8a7404e00dc6aaceb35f30a962e47016bdc5ac37 Mon Sep 17 00:00:00 2001 From: elbayaaa <72949274+elbayaaa@users.noreply.github.com> Date: Fri, 16 Apr 2021 14:07:57 -0700 Subject: [PATCH 10/31] UX - rewriting Co-authored-by: Chris Rehn --- samcli/commands/pipeline/bootstrap/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index 33da3b55cd8..d8ea0e7d742 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -18,11 +18,11 @@ SHORT_HELP = "Sets up infrastructure resources for AWS SAM CI/CD pipelines." HELP_TEXT = """Sets up the following infrastructure resources for AWS SAM CI/CD pipelines: -\n\t - Pipeline user(IAM User) with AccessKeyId and SecretAccessKey credentials to be shared with the CI/CD provider -\n\t - Pipeline Execution Role(IAM Role) that is assumed by the Pipeline user to obtain access to the AWS account -\n\t - CloudFormation Execution Role(IAM Role) that is assumed by CloudFormation to deploy the SAM application -\n\t - Artifacts bucket(S3 bucket) to store the sam build artifacts -\n\t - ECR repo for the container images of Lambda functions having PackageType property set to Image +\n\t - Pipeline IAM user with access key ID and secret access key credentials to be shared with the CI/CD provider +\n\t - Pipeline execution IAM role assumed by the pipeline user to obtain access to the AWS account +\n\t - CloudFormation execution IAM role assumed by CloudFormation to deploy the AWS SAM application +\n\t - Artifacts S3 bucket to store the AWS SAM build artifacts +\n\t - Optionally, an ECR repository to store container image Lambda deployment packages """ PIPELINE_CONFIG_DIR = os.path.join(".aws-sam", "pipeline") From 6e3f21dfbeb5d8e586d2e59876570d56304bcf58 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Fri, 16 Apr 2021 14:38:13 -0700 Subject: [PATCH 11/31] UX improvements --- samcli/commands/pipeline/bootstrap/cli.py | 43 ++++++++++++----------- samcli/lib/pipeline/bootstrap/resource.py | 2 +- samcli/lib/pipeline/bootstrap/stage.py | 9 +++-- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index d8ea0e7d742..3e38706e367 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -21,8 +21,8 @@ \n\t - Pipeline IAM user with access key ID and secret access key credentials to be shared with the CI/CD provider \n\t - Pipeline execution IAM role assumed by the pipeline user to obtain access to the AWS account \n\t - CloudFormation execution IAM role assumed by CloudFormation to deploy the AWS SAM application -\n\t - Artifacts S3 bucket to store the AWS SAM build artifacts -\n\t - Optionally, an ECR repository to store container image Lambda deployment packages +\n\t - Artifacts S3 bucket to hold the AWS SAM build artifacts +\n\t - Optionally, an ECR repository to hold container image Lambda deployment packages """ PIPELINE_CONFIG_DIR = os.path.join(".aws-sam", "pipeline") @@ -44,20 +44,20 @@ ) @click.option( "--pipeline-user", - help="The ARN of the IAM user having its AccessKeyId and SecretAccessKey shared with the CI/CD provider." + help="The ARN of the IAM user having its access key ID and secret access key shared with the CI/CD provider." "It is used to grant this IAM user the permissions to access the corresponding AWS account." - "If not provided, the command will create one along with AccessKeyId and SecretAccessKey credentials.", + "If not provided, the command will create one along with access key ID and secret access key credentials.", required=False, ) @click.option( "--pipeline-execution-role", - help="The ARN of an IAM Role to be assumed by the pipeline-user to operate on this stage. " + help="The ARN of an IAM Role to be assumed by the pipeline user to operate on this stage. " "Provide it only if you want to user your own role, otherwise, the command will create one", required=False, ) @click.option( "--cloudformation-execution-role", - help="The ARN of an IAM Role to be assumed by the CloudFormation service while deploying the application's stack " + help="The ARN of an IAM Role to be assumed by the CloudFormation service while deploying the application's stack. " "Provide it only if you want to user your own role, otherwise, the command will create one", required=False, ) @@ -71,19 +71,19 @@ "--create-ecr-repo/--no-create-ecr-repo", is_flag=True, default=False, - help="If set to true and no ecr-repo is provided this command will create an ECR repo to hold the image container " - "of the lambda functions having Image package type.", + help="If set to true and no ecr-repo is provided this command will create an ECR repository to hold the image " + "container of the lambda functions having Image package type.", ) @click.option( "--ecr-repo", - help="The ARN of an ECR repo to hold the image containers of the lambda functions of image package type. " + help="The ARN of an ECR repository to hold the image containers of the lambda functions of image package type. " "If Provided, the create-ecr-repo argument is ignored. If not provided and create-ecr-repo is set to true " - "The command will create one.", + "the command will create one.", required=False, ) @click.option( "--pipeline-ip-range", - help="If provided, all requests coming from outside of the given range(s) are denied.", + help="If provided, all requests coming from outside of the given range are denied. Example: 10.24.34.0/24", required=False, ) @click.option( @@ -193,18 +193,19 @@ def do_cli( ecr_repo_arn=ecr_repo_arn, ) - stage.bootstrap(confirm_changeset=confirm_changeset) + bootstrapped: bool = stage.bootstrap(confirm_changeset=confirm_changeset) - stage.print_resources_summary() + if bootstrapped: + stage.print_resources_summary() - try: - stage.save_config( - config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=_get_command_name() - ) - except Exception: - # Swallow saving exceptions, if any, as the resources are already bootstrapped and the ARNs are already - # printed out in the screen. - pass + try: + stage.save_config( + config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=_get_command_name() + ) + except Exception: + # Swallow saving exceptions, if any, as the resources are already bootstrapped and the ARNs are already + # printed out in the screen. + pass def _load_saved_pipeline_user() -> Optional[str]: diff --git a/samcli/lib/pipeline/bootstrap/resource.py b/samcli/lib/pipeline/bootstrap/resource.py index a7b6668514b..960e6c675a4 100644 --- a/samcli/lib/pipeline/bootstrap/resource.py +++ b/samcli/lib/pipeline/bootstrap/resource.py @@ -4,7 +4,7 @@ class ARNParts: """ - Decompose a given ARN into its parts + Decompose a given ARN into its parts https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html Attributes ---------- diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index a1814111ad9..0b66345a59d 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -106,7 +106,7 @@ def _get_non_user_provided_resources_msg(self) -> str: missing_resources_msg += "\n\tECR repo." return missing_resources_msg - def bootstrap(self, confirm_changeset: bool = True) -> None: + def bootstrap(self, confirm_changeset: bool = True) -> bool: """ Deploys the CFN template(./stage_resources.yaml) which deploys: * Pipeline IAM User @@ -125,18 +125,20 @@ def bootstrap(self, confirm_changeset: bool = True) -> None: confirm_changeset: bool if set to false, the stage_resources.yaml CFN template will directly be deployed, otherwise, the user will be prompted for confirmation + + Returns True if bootstrapped, otherwise False """ if self.did_user_provide_all_required_resources(): click.secho(f"\nAll required resources for the {self.name} stage exist, skipping creation.", fg="yellow") - return + return True missing_resources: str = self._get_non_user_provided_resources_msg() click.echo(f"This will create the following required resources for the {self.name} stage: {missing_resources}") if confirm_changeset: confirmed: bool = click.confirm("Should we proceed with the creation?") if not confirmed: - return + return False stage_name_suffix: str = re.sub("[^0-9a-zA-Z]+", "-", self.name) stack_name: str = f"{STACK_NAME_PREFIX}-{stage_name_suffix}-{STAGE_RESOURCES_STACK_NAME_SUFFIX}" @@ -165,6 +167,7 @@ def bootstrap(self, confirm_changeset: bool = True) -> None: self.artifacts_bucket.arn = output.get("ArtifactsBucket") self.artifacts_bucket.kms_key_arn = output.get("ArtifactsBucketKMS") self.ecr_repo.arn = output.get("EcrRepo") + return True @staticmethod def _read_template(template_file_name: str) -> str: From 58e48a2d913e220d3f3ef5401a46c17edd632d99 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Fri, 16 Apr 2021 17:15:01 -0700 Subject: [PATCH 12/31] make black happy --- samcli/commands/pipeline/bootstrap/cli.py | 2 +- samcli/lib/pipeline/bootstrap/stage.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index 3e38706e367..cf8b0e04aa1 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -72,7 +72,7 @@ is_flag=True, default=False, help="If set to true and no ecr-repo is provided this command will create an ECR repository to hold the image " - "container of the lambda functions having Image package type.", + "container of the lambda functions having Image package type.", ) @click.option( "--ecr-repo", diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index 0b66345a59d..2d53041095a 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -133,8 +133,10 @@ def bootstrap(self, confirm_changeset: bool = True) -> bool: click.secho(f"\nAll required resources for the {self.name} stage exist, skipping creation.", fg="yellow") return True - missing_resources: str = self._get_non_user_provided_resources_msg() - click.echo(f"This will create the following required resources for the {self.name} stage: {missing_resources}") + missing_resources_msg: str = self._get_non_user_provided_resources_msg() + click.echo( + "This will create the following required resources for the {self.name} stage: " f"{missing_resources_msg}" + ) if confirm_changeset: confirmed: bool = click.confirm("Should we proceed with the creation?") if not confirmed: From 206b7e3ece574956a7cd360a00bc8498eec9ccc6 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Sun, 18 Apr 2021 21:04:14 -0700 Subject: [PATCH 13/31] apply review comments --- samcli/lib/config/samconfig.py | 5 ++-- samcli/lib/pipeline/bootstrap/resource.py | 24 ++++++++--------- samcli/lib/pipeline/bootstrap/stage.py | 16 +++++++----- .../lib/pipeline/bootstrap/test_resource.py | 26 +++++++++---------- .../unit/lib/pipeline/bootstrap/test_stage.py | 2 +- 5 files changed, 37 insertions(+), 36 deletions(-) diff --git a/samcli/lib/config/samconfig.py b/samcli/lib/config/samconfig.py index 6cda8a075b4..be9ebb40278 100644 --- a/samcli/lib/config/samconfig.py +++ b/samcli/lib/config/samconfig.py @@ -154,9 +154,8 @@ def exists(self): return self.filepath.exists() def _ensure_exists(self): - if not self.exists(): - self.filepath.parent.mkdir(parents=True, exist_ok=True) - open(self.filepath, "a+").close() + self.filepath.parent.mkdir(parents=True, exist_ok=True) + self.filepath.touch() def path(self): return str(self.filepath) diff --git a/samcli/lib/pipeline/bootstrap/resource.py b/samcli/lib/pipeline/bootstrap/resource.py index 960e6c675a4..11e3bb4990a 100644 --- a/samcli/lib/pipeline/bootstrap/resource.py +++ b/samcli/lib/pipeline/bootstrap/resource.py @@ -1,5 +1,5 @@ """ Represents AWS resource""" -from typing import List, Optional +from typing import Optional class ARNParts: @@ -33,15 +33,9 @@ def __init__(self, arn: str) -> None: self.region: str = parts[3] self.account_id: str = parts[4] self.resource_id: str = parts[5] - self.resource_type: Optional[str] = None except IndexError as ex: raise ValueError(f"Invalid ARN ({arn})") from ex - if "/" in self.resource_id: - resource_type_and_id: List[str] = self.resource_id.split("/") - self.resource_type = resource_type_and_id[0] - self.resource_id = resource_type_and_id[1] - class Resource: """ @@ -121,13 +115,19 @@ def __init__(self, arn: Optional[str]) -> None: def get_uri(self) -> Optional[str]: """ extracts and returns the URI of the given ECR repo from its ARN + see https://docs.aws.amazon.com/AmazonECR/latest/userguide/Registries.html Raises ------ ValueError if the ARN is invalid """ arn_parts: Optional[ARNParts] = self._get_arn_parts() - return ( - f"{arn_parts.account_id}.dkr.ecr.{arn_parts.region}.amazonaws.com/{arn_parts.resource_id}" - if arn_parts - else None - ) + if not arn_parts: + return None + # ECR's resource_id contains the resource-type("resource") which is excluded from the URL + # from docs: https://docs.aws.amazon.com/AmazonECR/latest/userguide/security_iam_service-with-iam.html + # ECR's ARN: arn:${Partition}:ecr:${Region}:${Account}:repository/${Repository-name} + if not "repository/" in arn_parts.resource_id: + raise ValueError(f"Invalid ECR ARN: {self.arn}") + i = len("repository/") + repo_name = arn_parts.resource_id[i:] + return f"{arn_parts.account_id}.dkr.ecr.{arn_parts.region}.amazonaws.com/{repo_name}" diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index 2d53041095a..5e1a474e166 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -13,7 +13,11 @@ CFN_TEMPLATE_PATH = str(pathlib.Path(os.path.dirname(__file__))) STACK_NAME_PREFIX = "aws-sam-cli-managed" STAGE_RESOURCES_STACK_NAME_SUFFIX = "pipeline-resources" - +STAGE_RESOURCES_CFN_TEMPLATE = "stage_resources.yaml" +PIPELINE_EXECUTION_ROLE = "pipeline_execution_role" +CLOUDFORMATION_EXECUTION_ROLE = "cloudformation_execution_role" +ARTIFACTS_BUCKET = "artifacts_bucket" +ECR_REPO = "ecr_repo" class Stage: """ @@ -144,7 +148,7 @@ def bootstrap(self, confirm_changeset: bool = True) -> bool: stage_name_suffix: str = re.sub("[^0-9a-zA-Z]+", "-", self.name) stack_name: str = f"{STACK_NAME_PREFIX}-{stage_name_suffix}-{STAGE_RESOURCES_STACK_NAME_SUFFIX}" - stage_resources_template_body = Stage._read_template("stage_resources.yaml") + stage_resources_template_body = Stage._read_template(STAGE_RESOURCES_CFN_TEMPLATE) output: StackOutput = manage_stack( stack_name=stack_name, region=self.aws_region, @@ -206,7 +210,7 @@ def save_config(self, config_dir: str, filename: str, cmd_names: List[str]) -> N samconfig.put( cmd_names=cmd_names, section="parameters", - key="pipeline_execution_role", + key=PIPELINE_EXECUTION_ROLE, value=self.pipeline_execution_role.arn, env=self.name, ) @@ -215,7 +219,7 @@ def save_config(self, config_dir: str, filename: str, cmd_names: List[str]) -> N samconfig.put( cmd_names=cmd_names, section="parameters", - key="cloudformation_execution_role", + key=CLOUDFORMATION_EXECUTION_ROLE, value=self.cloudformation_execution_role.arn, env=self.name, ) @@ -224,14 +228,14 @@ def save_config(self, config_dir: str, filename: str, cmd_names: List[str]) -> N samconfig.put( cmd_names=cmd_names, section="parameters", - key="artifacts_bucket", + key=ARTIFACTS_BUCKET, value=self.artifacts_bucket.name(), env=self.name, ) if self.ecr_repo.get_uri(): samconfig.put( - cmd_names=cmd_names, section="parameters", key="ecr_repo", value=self.ecr_repo.get_uri(), env=self.name + cmd_names=cmd_names, section="parameters", key=ECR_REPO, value=self.ecr_repo.get_uri(), env=self.name ) samconfig.flush() diff --git a/tests/unit/lib/pipeline/bootstrap/test_resource.py b/tests/unit/lib/pipeline/bootstrap/test_resource.py index 243380a3c75..f27dbd57ad3 100644 --- a/tests/unit/lib/pipeline/bootstrap/test_resource.py +++ b/tests/unit/lib/pipeline/bootstrap/test_resource.py @@ -4,37 +4,28 @@ from samcli.lib.pipeline.bootstrap.resource import ARNParts, Resource, S3Bucket, IamUser, EcrRepo VALID_ARN = "arn:partition:service:region:account-id:resource-id" -VALID_ARN_WITH_RESOURCE_TYPE = "arn:partition:service:region:account-id:resource-type/resource-id" INVALID_ARN = "ARN" class TestArnParts(TestCase): - def test_arn_parts_of_valid_arn_without_resource_type(self): + def test_arn_parts_of_valid_arn(self): arn_parts: ARNParts = ARNParts(arn=VALID_ARN) self.assertEqual(arn_parts.partition, "partition") self.assertEqual(arn_parts.service, "service") self.assertEqual(arn_parts.region, "region") self.assertEqual(arn_parts.account_id, "account-id") self.assertEqual(arn_parts.resource_id, "resource-id") - self.assertIsNone(arn_parts.resource_type) - - def test_arn_parts_of_valid_arn_including_resource_type(self): - arn_parts: ARNParts = ARNParts(arn=VALID_ARN_WITH_RESOURCE_TYPE) - self.assertEqual(arn_parts.partition, "partition") - self.assertEqual(arn_parts.service, "service") - self.assertEqual(arn_parts.region, "region") - self.assertEqual(arn_parts.account_id, "account-id") - self.assertEqual(arn_parts.resource_id, "resource-id") - self.assertEqual(arn_parts.resource_type, "resource-type") def test_arn_parts_of_none_arn_is_invalid(self): with self.assertRaises(ValueError): ARNParts(arn=None) + def test_arn_parts_of_none_string_arn_is_invalid(self): with self.assertRaises(ValueError): any_non_string = 1 ARNParts(arn=any_non_string) + def test_arn_parts_of_invalid_arn(self): with self.assertRaises(ValueError): invalid_arn = "invalid_arn" ARNParts(arn=invalid_arn) @@ -85,9 +76,16 @@ def test_create_s3_bucket(self): class TestEcrRepo(TestCase): def test_get_uri(self): - repo: EcrRepo = EcrRepo(arn=VALID_ARN) - self.assertEqual(repo.get_uri(), "account-id.dkr.ecr.region.amazonaws.com/resource-id") + valid_ecr_arn = "arn:partition:service:region:account-id:repository/repository-name" + repo: EcrRepo = EcrRepo(arn=valid_ecr_arn) + self.assertEqual(repo.get_uri(), "account-id.dkr.ecr.region.amazonaws.com/repository-name") + valid_ecr_arn = "arn:partition:service:region:account-id:repository/repository-name" repo = EcrRepo(arn=INVALID_ARN) with self.assertRaises(ValueError): repo.get_uri() + + ecr_arn_missing_repository_prefix = "arn:partition:service:region:account-id:repository-name-not-prefixed-with-repository/" + repo = EcrRepo(arn=ecr_arn_missing_repository_prefix) + with self.assertRaises(ValueError): + repo.get_uri() diff --git a/tests/unit/lib/pipeline/bootstrap/test_stage.py b/tests/unit/lib/pipeline/bootstrap/test_stage.py index 162d933c659..2d40500f056 100644 --- a/tests/unit/lib/pipeline/bootstrap/test_stage.py +++ b/tests/unit/lib/pipeline/bootstrap/test_stage.py @@ -261,7 +261,7 @@ def test_save_config_escapes_non_resources(self, samconfig_mock): ) self.trigger_and_assert_save_config_calls(stage, cmd_names, expected_calls, samconfig_instance_mock.put) - stage.ecr_repo.arn = "arn:aws:ecr:us-east-2:111111111111:ecr_repo_name" + stage.ecr_repo.arn = "arn:aws:ecr:us-east-2:111111111111:repository/ecr_repo_name" expected_calls.append( call( cmd_names=cmd_names, From 23148fffab7f9b452ef2aaa15718744bd3730a81 Mon Sep 17 00:00:00 2001 From: elbayaaa <72949274+elbayaaa@users.noreply.github.com> Date: Mon, 19 Apr 2021 12:21:46 -0700 Subject: [PATCH 14/31] UX - rewriting Co-authored-by: Chris Rehn --- samcli/commands/pipeline/bootstrap/cli.py | 20 +++++++++---------- .../pipeline/bootstrap/guided_context.py | 18 ++++++++--------- samcli/lib/pipeline/bootstrap/stage.py | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index cf8b0e04aa1..2fbc35ccf10 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -44,40 +44,40 @@ ) @click.option( "--pipeline-user", - help="The ARN of the IAM user having its access key ID and secret access key shared with the CI/CD provider." - "It is used to grant this IAM user the permissions to access the corresponding AWS account." + help="The ARN of the IAM user having its access key ID and secret access key shared with the CI/CD provider. " + "It is used to grant this IAM user the permissions to access the corresponding AWS account. " "If not provided, the command will create one along with access key ID and secret access key credentials.", required=False, ) @click.option( "--pipeline-execution-role", - help="The ARN of an IAM Role to be assumed by the pipeline user to operate on this stage. " + help="The ARN of an IAM role to be assumed by the pipeline user to operate on this stage. " "Provide it only if you want to user your own role, otherwise, the command will create one", required=False, ) @click.option( "--cloudformation-execution-role", - help="The ARN of an IAM Role to be assumed by the CloudFormation service while deploying the application's stack. " - "Provide it only if you want to user your own role, otherwise, the command will create one", + help="The ARN of an IAM role to be assumed by the CloudFormation service while deploying the application's stack. " + "Provide it only if you want to user your own role, otherwise, the command will create one.", required=False, ) @click.option( "--artifacts-bucket", help="The ARN of a S3 bucket to hold the sam build artifacts. " - "Provide it only if you want to user your own S3 bucket, otherwise, the command will create one", + "Provide it only if you want to user your own S3 bucket, otherwise, the command will create one.", required=False, ) @click.option( "--create-ecr-repo/--no-create-ecr-repo", is_flag=True, default=False, - help="If set to true and no ecr-repo is provided this command will create an ECR repository to hold the image " - "container of the lambda functions having Image package type.", + help="If set to true and no ECR repository is provided, this command will create an ECR repository to hold the container " + "images of Lambda functions having an Image package type.", ) @click.option( "--ecr-repo", - help="The ARN of an ECR repository to hold the image containers of the lambda functions of image package type. " - "If Provided, the create-ecr-repo argument is ignored. If not provided and create-ecr-repo is set to true " + help="The ARN of an ECR repository to hold the containers images of Lambda functions of Image package type. " + "If provided, the --create-ecr-repo argument is ignored. If not provided and --create-ecr-repo is set to true, " "the command will create one.", required=False, ) diff --git a/samcli/commands/pipeline/bootstrap/guided_context.py b/samcli/commands/pipeline/bootstrap/guided_context.py index 096aa85df56..9895d0e1d8f 100644 --- a/samcli/commands/pipeline/bootstrap/guided_context.py +++ b/samcli/commands/pipeline/bootstrap/guided_context.py @@ -50,7 +50,7 @@ def run(self) -> None: if not self.pipeline_execution_role_arn: self.pipeline_execution_role_arn = click.prompt( - "\nPipeline execution role(an IAM Role to be assumed by the pipeline-user to operate on this stage.) " + "\nPipeline execution role (an IAM role assumed by the pipeline user to operate on this stage) " "[leave blank to create one]", default="", type=click.STRING, @@ -58,7 +58,7 @@ def run(self) -> None: if not self.cloudformation_execution_role_arn: self.cloudformation_execution_role_arn = click.prompt( - "\nCloudFormation execution role(an IAM Role to be assumed by the CloudFormation service to deploy " + "\nCloudFormation execution role (an IAM role assumed by CloudFormation to deploy " "the application's stack) [leave blank to create one]", default="", type=click.STRING, @@ -66,18 +66,18 @@ def run(self) -> None: if not self.artifacts_bucket_arn: self.artifacts_bucket_arn = click.prompt( - "\nArtifacts bucket(S3 bucket to hold the sam build artifacts) " "[leave blank to create one]", + "\nArtifacts bucket (S3 bucket to hold the AWS SAM build artifacts) [leave blank to create one]", default="", type=click.STRING, ) if not self.ecr_repo_arn: click.echo( - "\nIf your SAM template will include lambda functions of Image package-type, " - "then an ECR repo is required, should we create one?" + "\nIf your SAM template will include Lambda functions of Image package type, " + "then an ECR repository is required. Should we create one?" ) - click.echo("\t1 - No, My SAM Template won't include lambda functions of Image package-type") - click.echo("\t2 - Yes, I need a help creating one") - click.echo("\t3 - I already have an ECR repo") + click.echo("\t1 - No, My SAM template won't include Lambda functions of Image package type") + click.echo("\t2 - Yes, I need help creating one") + click.echo("\t3 - I already have an ECR repository") choice = click.prompt(text="Choice", show_choices=False, type=click.Choice(["1", "2", "3"])) if choice == "1": self.create_ecr_repo = False @@ -88,7 +88,7 @@ def run(self) -> None: self.ecr_repo_arn = click.prompt("ECR repo", type=click.STRING) if not self.pipeline_ip_range: - click.echo("\nWe can deny requests if not coming from a recognized IP address.") + click.echo("\nWe can deny requests not coming from a recognized IP address range.") self.pipeline_ip_range = click.prompt( "Pipeline IP address range [leave blank if you don't know]", default="", type=click.STRING ) diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index 5e1a474e166..c50561e27f9 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -277,6 +277,6 @@ def print_resources_summary(self) -> None: click.secho(f"\t{resource.arn}", fg="green") if not self.pipeline_user.is_user_provided: - click.secho("Please configure your CICD project with the following pipeline-user credentials:", fg="green") + click.secho("Please configure your CI/CD project with the following pipeline user credentials:", fg="green") click.secho(f"\tACCESS_KEY_ID: {self.pipeline_user.access_key_id}", fg="green") click.secho(f"\tSECRET_ACCESS_KEY: {self.pipeline_user.secret_access_key}", fg="green") From 37a4f5f102c5ef3a676cb71fa58aced7ef7cecdc Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Mon, 19 Apr 2021 12:41:12 -0700 Subject: [PATCH 15/31] refactor --- samcli/lib/pipeline/bootstrap/stage.py | 54 ++++++++++---------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index c50561e27f9..1e5d8bfb768 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -2,7 +2,7 @@ import os import pathlib import re -from typing import List, Optional +from typing import Dict, List, Optional import click @@ -14,6 +14,7 @@ STACK_NAME_PREFIX = "aws-sam-cli-managed" STAGE_RESOURCES_STACK_NAME_SUFFIX = "pipeline-resources" STAGE_RESOURCES_CFN_TEMPLATE = "stage_resources.yaml" +PIPELINE_USER = "pipeline_user" PIPELINE_EXECUTION_ROLE = "pipeline_execution_role" CLOUDFORMATION_EXECUTION_ROLE = "cloudformation_execution_role" ARTIFACTS_BUCKET = "artifacts_bucket" @@ -204,39 +205,24 @@ def save_config(self, config_dir: str, filename: str, cmd_names: List[str]) -> N samconfig: SamConfig = SamConfig(config_dir=config_dir, filename=filename) if self.pipeline_user.arn: - samconfig.put(cmd_names=cmd_names, section="parameters", key="pipeline_user", value=self.pipeline_user.arn) - - if self.pipeline_execution_role.arn: - samconfig.put( - cmd_names=cmd_names, - section="parameters", - key=PIPELINE_EXECUTION_ROLE, - value=self.pipeline_execution_role.arn, - env=self.name, - ) - - if self.cloudformation_execution_role.arn: - samconfig.put( - cmd_names=cmd_names, - section="parameters", - key=CLOUDFORMATION_EXECUTION_ROLE, - value=self.cloudformation_execution_role.arn, - env=self.name, - ) - - if self.artifacts_bucket.name(): - samconfig.put( - cmd_names=cmd_names, - section="parameters", - key=ARTIFACTS_BUCKET, - value=self.artifacts_bucket.name(), - env=self.name, - ) - - if self.ecr_repo.get_uri(): - samconfig.put( - cmd_names=cmd_names, section="parameters", key=ECR_REPO, value=self.ecr_repo.get_uri(), env=self.name - ) + samconfig.put(cmd_names=cmd_names, section="parameters", key=PIPELINE_USER, value=self.pipeline_user.arn) + + stage_specific_configs: Dict[str, str] = { + PIPELINE_EXECUTION_ROLE: self.pipeline_execution_role.arn, + CLOUDFORMATION_EXECUTION_ROLE: self.cloudformation_execution_role.arn, + ARTIFACTS_BUCKET: self.artifacts_bucket.name(), + ECR_REPO: self.ecr_repo.get_uri(), + } + + for key, value in enumerate(stage_specific_configs): + if value: + samconfig.put( + cmd_names=cmd_names, + section="parameters", + key=key, + value=value, + env=self.name, + ) samconfig.flush() From 8ea4153a09b857b1fa475f8a66524304876a1fbc Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Mon, 19 Apr 2021 15:34:53 -0700 Subject: [PATCH 16/31] Apply review comments --- samcli/commands/pipeline/bootstrap/cli.py | 10 ++-- .../pipeline/bootstrap/guided_context.py | 8 ++- samcli/lib/pipeline/bootstrap/resource.py | 4 +- samcli/lib/pipeline/bootstrap/stage.py | 20 +++++-- .../commands/pipeline/bootstrap/test_cli.py | 58 +++++++++---------- .../lib/pipeline/bootstrap/test_resource.py | 10 ++-- .../unit/lib/pipeline/bootstrap/test_stage.py | 20 ++++++- 7 files changed, 82 insertions(+), 48 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index 2fbc35ccf10..f3647695525 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -63,7 +63,7 @@ ) @click.option( "--artifacts-bucket", - help="The ARN of a S3 bucket to hold the sam build artifacts. " + help="The ARN of an S3 bucket to hold the AWS SAM build artifacts. " "Provide it only if you want to user your own S3 bucket, otherwise, the command will create one.", required=False, ) @@ -71,8 +71,8 @@ "--create-ecr-repo/--no-create-ecr-repo", is_flag=True, default=False, - help="If set to true and no ECR repository is provided, this command will create an ECR repository to hold the container " - "images of Lambda functions having an Image package type.", + help="If set to true and no ECR repository is provided, this command will create an ECR repository to hold the" + " container images of Lambda functions having an Image package type.", ) @click.option( "--ecr-repo", @@ -154,7 +154,7 @@ def do_cli( implementation of `sam pipeline bootstrap` command """ if not pipeline_user_arn: - pipeline_user_arn = _load_saved_pipeline_user() + pipeline_user_arn = _load_saved_pipeline_user_arn() if interactive: guided_context = GuidedContext( @@ -208,7 +208,7 @@ def do_cli( pass -def _load_saved_pipeline_user() -> Optional[str]: +def _load_saved_pipeline_user_arn() -> Optional[str]: samconfig: SamConfig = SamConfig(config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME) if not samconfig.exists(): return None diff --git a/samcli/commands/pipeline/bootstrap/guided_context.py b/samcli/commands/pipeline/bootstrap/guided_context.py index 9895d0e1d8f..572021adbe6 100644 --- a/samcli/commands/pipeline/bootstrap/guided_context.py +++ b/samcli/commands/pipeline/bootstrap/guided_context.py @@ -41,8 +41,8 @@ def run(self) -> None: click.echo( "\nThere must be exactly one pipeline user across all of the pipeline stages. " "If you have ran this command before to bootstrap a previous pipeline stage, please " - "provide the ARN of the created pipeline user, otherwise, we will create a new user for you, " - "please make sure to configure user's AccessKeyId and SecretAccessKey for the CI/CD provider." + "provide the ARN of the created pipeline user, otherwise, we will create a new user for you. " + "Please make sure to store the credentials safely with the CI/CD provider." ) self.pipeline_user_arn = click.prompt( "Pipeline user [leave blank to create one]", default="", type=click.STRING @@ -90,5 +90,7 @@ def run(self) -> None: if not self.pipeline_ip_range: click.echo("\nWe can deny requests not coming from a recognized IP address range.") self.pipeline_ip_range = click.prompt( - "Pipeline IP address range [leave blank if you don't know]", default="", type=click.STRING + "Pipeline IP address range (using CIDR notation) [leave blank if you don't know]", + default="", + type=click.STRING, ) diff --git a/samcli/lib/pipeline/bootstrap/resource.py b/samcli/lib/pipeline/bootstrap/resource.py index 11e3bb4990a..2b1a079ccd1 100644 --- a/samcli/lib/pipeline/bootstrap/resource.py +++ b/samcli/lib/pipeline/bootstrap/resource.py @@ -126,8 +126,8 @@ def get_uri(self) -> Optional[str]: # ECR's resource_id contains the resource-type("resource") which is excluded from the URL # from docs: https://docs.aws.amazon.com/AmazonECR/latest/userguide/security_iam_service-with-iam.html # ECR's ARN: arn:${Partition}:ecr:${Region}:${Account}:repository/${Repository-name} - if not "repository/" in arn_parts.resource_id: - raise ValueError(f"Invalid ECR ARN: {self.arn}") + if not arn_parts.resource_id.startswith("repository/"): + raise ValueError(f"Invalid ECR ARN ({self.arn}), can't extract the URL from it.") i = len("repository/") repo_name = arn_parts.resource_id[i:] return f"{arn_parts.account_id}.dkr.ecr.{arn_parts.region}.amazonaws.com/{repo_name}" diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index 1e5d8bfb768..b12436ecd80 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -20,6 +20,7 @@ ARTIFACTS_BUCKET = "artifacts_bucket" ECR_REPO = "ecr_repo" + class Stage: """ Represents a pipeline stage @@ -207,14 +208,25 @@ def save_config(self, config_dir: str, filename: str, cmd_names: List[str]) -> N if self.pipeline_user.arn: samconfig.put(cmd_names=cmd_names, section="parameters", key=PIPELINE_USER, value=self.pipeline_user.arn) - stage_specific_configs: Dict[str, str] = { + # Computing Artifacts bucket name and ECR repo URL may through an exception if the ARNs are wrong + # Let's swallow such an exception to be able to save the remaining resources + try: + artifacts_bucket_name: Optional[str] = self.artifacts_bucket.name() + except ValueError: + artifacts_bucket_name = "" + try: + ecr_repo_uri: Optional[str] = self.ecr_repo.get_uri() + except ValueError: + ecr_repo_uri = "" + + stage_specific_configs: Dict[str, Optional[str]] = { PIPELINE_EXECUTION_ROLE: self.pipeline_execution_role.arn, CLOUDFORMATION_EXECUTION_ROLE: self.cloudformation_execution_role.arn, - ARTIFACTS_BUCKET: self.artifacts_bucket.name(), - ECR_REPO: self.ecr_repo.get_uri(), + ARTIFACTS_BUCKET: artifacts_bucket_name, + ECR_REPO: ecr_repo_uri, } - for key, value in enumerate(stage_specific_configs): + for key, value in stage_specific_configs.items(): if value: samconfig.put( cmd_names=cmd_names, diff --git a/tests/unit/commands/pipeline/bootstrap/test_cli.py b/tests/unit/commands/pipeline/bootstrap/test_cli.py index 9c7549b0f81..f1e1e771609 100644 --- a/tests/unit/commands/pipeline/bootstrap/test_cli.py +++ b/tests/unit/commands/pipeline/bootstrap/test_cli.py @@ -6,7 +6,7 @@ from click.testing import CliRunner from samcli.commands.pipeline.bootstrap.cli import ( - _load_saved_pipeline_user, + _load_saved_pipeline_user_arn, _get_command_name, PIPELINE_CONFIG_FILENAME, PIPELINE_CONFIG_DIR, @@ -101,18 +101,18 @@ def test_bootstrap_command_with_different_arguments_combination(self, do_cli_moc self.assertEqual(kwargs["artifacts_bucket_arn"], "bucketARN") @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrapping_normal_interactive_flow( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock ): # setup gc_instance = Mock() guided_context_mock.return_value = gc_instance stage_instance = Mock() stage_mock.return_value = stage_instance - load_saved_pipeline_user_mock.return_value = ANY_PIPELINE_USER_ARN + load_saved_pipeline_user_arn_mock.return_value = ANY_PIPELINE_USER_ARN self.cli_context["interactive"] = True self.cli_context["pipeline_user_arn"] = None get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES @@ -121,7 +121,7 @@ def test_bootstrapping_normal_interactive_flow( bootstrap_cli(**self.cli_context) # verify - load_saved_pipeline_user_mock.assert_called_once() + load_saved_pipeline_user_arn_mock.assert_called_once() gc_instance.run.assert_called_once() stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) stage_instance.print_resources_summary.assert_called_once() @@ -132,32 +132,32 @@ def test_bootstrapping_normal_interactive_flow( ) @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrap_will_not_try_loading_pipeline_user_if_already_provided( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock ): bootstrap_cli(**self.cli_context) - load_saved_pipeline_user_mock.assert_not_called() + load_saved_pipeline_user_arn_mock.assert_not_called() @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrap_will_try_loading_pipeline_user_if_not_provided( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock ): self.cli_context["pipeline_user_arn"] = None bootstrap_cli(**self.cli_context) - load_saved_pipeline_user_mock.assert_called_once() + load_saved_pipeline_user_arn_mock.assert_called_once() @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_stage_name_is_required_to_be_provided_in_case_of_non_interactive_mode( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock ): self.cli_context["interactive"] = False self.cli_context["stage_name"] = None @@ -165,22 +165,22 @@ def test_stage_name_is_required_to_be_provided_in_case_of_non_interactive_mode( bootstrap_cli(**self.cli_context) @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_stage_name_is_not_required_to_be_provided_in_case_of_interactive_mode( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock ): self.cli_context["interactive"] = True self.cli_context["stage_name"] = None bootstrap_cli(**self.cli_context) # No exception is thrown @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_guided_context_will_be_enabled_or_disabled_based_on_the_interactive_mode( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock ): gc_instance = Mock() guided_context_mock.return_value = gc_instance @@ -192,11 +192,11 @@ def test_guided_context_will_be_enabled_or_disabled_based_on_the_interactive_mod gc_instance.run.assert_called_once() @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_user_choose_not_to( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock ): stage_instance = Mock() stage_mock.return_value = stage_instance @@ -209,11 +209,11 @@ def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_use stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user") + @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrapping_will_not_fail_if_saving_resources_arns_to_local_toml_file_failed( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock ): # setup stage_instance = Mock() @@ -230,7 +230,7 @@ def test_bootstrapping_will_not_fail_if_saving_resources_arns_to_local_toml_file @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - def test_load_saved_pipeline_user_will_read_from_the_correct_file(self, get_command_name_mock, sam_config_mock): + def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file(self, get_command_name_mock, sam_config_mock): # setup get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() @@ -238,14 +238,14 @@ def test_load_saved_pipeline_user_will_read_from_the_correct_file(self, get_comm sam_config_instance_mock.exists.return_value = False # trigger - _load_saved_pipeline_user() + _load_saved_pipeline_user_arn() # verify sam_config_mock.assert_called_once_with(config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME) @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - def test_load_saved_pipeline_user_will_return_non_if_the_pipeline_toml_file_is_not_found( + def test_load_saved_pipeline_user_arn_will_return_non_if_the_pipeline_toml_file_is_not_found( self, get_command_name_mock, sam_config_mock ): # setup @@ -255,14 +255,14 @@ def test_load_saved_pipeline_user_will_return_non_if_the_pipeline_toml_file_is_n sam_config_instance_mock.exists.return_value = False # trigger - pipeline_user_arn = _load_saved_pipeline_user() + pipeline_user_arn = _load_saved_pipeline_user_arn() # verify self.assertIsNone(pipeline_user_arn) @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - def test_load_saved_pipeline_user_will_return_non_if_the_pipeline_toml_file_does_not_contain_pipeline_user( + def test_load_saved_pipeline_user_arn_will_return_non_if_the_pipeline_toml_file_does_not_contain_pipeline_user( self, get_command_name_mock, sam_config_mock ): # setup @@ -273,14 +273,14 @@ def test_load_saved_pipeline_user_will_return_non_if_the_pipeline_toml_file_does sam_config_instance_mock.get_all.return_value = {"non-pipeline_user-key": "any_value"} # trigger - pipeline_user_arn = _load_saved_pipeline_user() + pipeline_user_arn = _load_saved_pipeline_user_arn() # verify self.assertIsNone(pipeline_user_arn) @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - def test_load_saved_pipeline_user_returns_the_pipeline_user_arn_from_the_pipeline_toml_file( + def test_load_saved_pipeline_user_arn_returns_the_pipeline_user_arn_from_the_pipeline_toml_file( self, get_command_name_mock, sam_config_mock ): # setup @@ -291,7 +291,7 @@ def test_load_saved_pipeline_user_returns_the_pipeline_user_arn_from_the_pipelin sam_config_instance_mock.get_all.return_value = {"pipeline_user": ANY_PIPELINE_USER_ARN} # trigger - pipeline_user_arn = _load_saved_pipeline_user() + pipeline_user_arn = _load_saved_pipeline_user_arn() # verify self.assertEqual(pipeline_user_arn, ANY_PIPELINE_USER_ARN) diff --git a/tests/unit/lib/pipeline/bootstrap/test_resource.py b/tests/unit/lib/pipeline/bootstrap/test_resource.py index f27dbd57ad3..5e5d20c8b35 100644 --- a/tests/unit/lib/pipeline/bootstrap/test_resource.py +++ b/tests/unit/lib/pipeline/bootstrap/test_resource.py @@ -1,5 +1,4 @@ from unittest import TestCase -from unittest.mock import Mock, patch, call from samcli.lib.pipeline.bootstrap.resource import ARNParts, Resource, S3Bucket, IamUser, EcrRepo @@ -75,17 +74,20 @@ def test_create_s3_bucket(self): class TestEcrRepo(TestCase): - def test_get_uri(self): + def test_get_uri_with_valid_ecr_arn(self): valid_ecr_arn = "arn:partition:service:region:account-id:repository/repository-name" repo: EcrRepo = EcrRepo(arn=valid_ecr_arn) self.assertEqual(repo.get_uri(), "account-id.dkr.ecr.region.amazonaws.com/repository-name") - valid_ecr_arn = "arn:partition:service:region:account-id:repository/repository-name" + def test_get_uri_with_invalid_ecr_arn(self): repo = EcrRepo(arn=INVALID_ARN) with self.assertRaises(ValueError): repo.get_uri() - ecr_arn_missing_repository_prefix = "arn:partition:service:region:account-id:repository-name-not-prefixed-with-repository/" + def test_get_uri_with_valid_aws_arn_that_is_invalid_ecr_arn(self): + ecr_arn_missing_repository_prefix = ( + "arn:partition:service:region:account-id:repository-name-without-repository/-prefix" + ) repo = EcrRepo(arn=ecr_arn_missing_repository_prefix) with self.assertRaises(ValueError): repo.get_uri() diff --git a/tests/unit/lib/pipeline/bootstrap/test_stage.py b/tests/unit/lib/pipeline/bootstrap/test_stage.py index 2d40500f056..0dc7b286221 100644 --- a/tests/unit/lib/pipeline/bootstrap/test_stage.py +++ b/tests/unit/lib/pipeline/bootstrap/test_stage.py @@ -210,7 +210,7 @@ def test_bootstrap_will_fullfill_all_resource_arns(self, manage_stack_mock, clic self.assertEqual(ANY_ARN, stage.artifacts_bucket.arn) @patch("samcli.lib.pipeline.bootstrap.stage.SamConfig") - def test_save_config_escapes_non_resources(self, samconfig_mock): + def test_save_config_escapes_none_resources(self, samconfig_mock): cmd_names = ["any", "commands"] samconfig_instance_mock = Mock() samconfig_mock.return_value = samconfig_instance_mock @@ -279,6 +279,24 @@ def trigger_and_assert_save_config_calls(self, stage, cmd_names, expected_calls, samconfig_put_mock.assert_has_calls(expected_calls) samconfig_put_mock.reset_mock() + @patch("samcli.lib.pipeline.bootstrap.stage.SamConfig") + def test_save_config_ignores_exceptions_thrown_while_calculating_artifacts_bucket_name(self, samconfig_mock): + samconfig_instance_mock = Mock() + samconfig_mock.return_value = samconfig_instance_mock + stage: Stage = Stage(name=ANY_STAGE_NAME, artifacts_bucket_arn="invalid ARN") + # calling artifacts_bucket.name() during save_config() will raise a ValueError exception, we need to make sure + # this exception is swallowed so that other configs can be safely saved to the pipelineconfig.toml file + stage.save_config(config_dir="any_config_dir", filename="any_pipeline.toml", cmd_names=["any", "commands"]) + + @patch("samcli.lib.pipeline.bootstrap.stage.SamConfig") + def test_save_config_ignores_exceptions_thrown_while_calculating_ecr_repo_uri(self, samconfig_mock): + samconfig_instance_mock = Mock() + samconfig_mock.return_value = samconfig_instance_mock + stage: Stage = Stage(name=ANY_STAGE_NAME, ecr_repo_arn="invalid ARN") + # calling ecr_repo.get_uri() during save_config() will raise a ValueError exception, we need to make sure + # this exception is swallowed so that other configs can be safely saved to the pipelineconfig.toml file + stage.save_config(config_dir="any_config_dir", filename="any_pipeline.toml", cmd_names=["any", "commands"]) + @patch("samcli.lib.pipeline.bootstrap.stage.click") def test_print_resources_summary_when_no_resources_provided_by_the_user(self, click_mock): stage: Stage = Stage(name=ANY_STAGE_NAME) From 99c91c82f61f90e0412ee6b5abc3edee894a1b3b Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Mon, 19 Apr 2021 15:52:09 -0700 Subject: [PATCH 17/31] use python way of array elements assignments --- samcli/lib/pipeline/bootstrap/resource.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/samcli/lib/pipeline/bootstrap/resource.py b/samcli/lib/pipeline/bootstrap/resource.py index 2b1a079ccd1..e2471bfe4bd 100644 --- a/samcli/lib/pipeline/bootstrap/resource.py +++ b/samcli/lib/pipeline/bootstrap/resource.py @@ -21,19 +21,19 @@ class ARNParts: resource-type: str the resource-type part of the ARN """ + partition: str + service: str + region: str + account_id: str + resource_id: str def __init__(self, arn: str) -> None: if not isinstance(arn, str): raise ValueError(f"Invalid ARN ({arn}) is not a String") - parts = arn.split(":") try: - self.partition: str = parts[1] - self.service: str = parts[2] - self.region: str = parts[3] - self.account_id: str = parts[4] - self.resource_id: str = parts[5] - except IndexError as ex: + [_, self.partition, self.service, self.region, self.account_id, self.resource_id] = parts + except ValueError as ex: raise ValueError(f"Invalid ARN ({arn})") from ex From b4c248f1dff81b4b52ce793d6c82146e13b8832b Mon Sep 17 00:00:00 2001 From: elbayaaa <72949274+elbayaaa@users.noreply.github.com> Date: Tue, 20 Apr 2021 11:21:19 -0700 Subject: [PATCH 18/31] Update samcli/lib/pipeline/bootstrap/stage.py Co-authored-by: _sam <3804518+aahung@users.noreply.github.com> --- samcli/lib/pipeline/bootstrap/stage.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index b12436ecd80..c7804d555a6 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -90,13 +90,7 @@ def __init__( def did_user_provide_all_required_resources(self) -> bool: """Check if the user provided all of the stage resources or not""" - return ( - self.pipeline_user.is_user_provided - and self.pipeline_execution_role.is_user_provided - and self.cloudformation_execution_role.is_user_provided - and self.artifacts_bucket.is_user_provided - and (not self.create_ecr_repo or self.ecr_repo.is_user_provided) - ) + return all(resource.is_user_provided for resource in self._get_resources()) def _get_non_user_provided_resources_msg(self) -> str: missing_resources_msg = "" From 3acea8cb4b26ad519fd26b7e378d9c06ca74563a Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 11:58:56 -0700 Subject: [PATCH 19/31] apply review comments --- samcli/cli/context.py | 4 ++-- samcli/commands/pipeline/bootstrap/cli.py | 5 ++--- samcli/lib/pipeline/bootstrap/resource.py | 1 + samcli/lib/pipeline/bootstrap/stage.py | 20 +++++++++++++------- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/samcli/cli/context.py b/samcli/cli/context.py index a69ebb9ff2c..74c35155a14 100644 --- a/samcli/cli/context.py +++ b/samcli/cli/context.py @@ -4,7 +4,7 @@ import logging import uuid -from typing import Optional, cast +from typing import Optional, cast, List import boto3 import botocore @@ -186,7 +186,7 @@ def _refresh_session(self): raise CredentialsError(str(ex)) from ex -def get_cmd_names(cmd_name, ctx): +def get_cmd_names(cmd_name, ctx) -> List[str]: """ Given the click core context, return a list representing all the subcommands passed to the CLI diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index f3647695525..c783f39ab4b 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -2,7 +2,7 @@ CLI command for "pipeline bootstrap", which sets up the require pipeline infrastructure resources """ import os -from typing import Any, cast, Dict, List, Optional +from typing import Any, Dict, List, Optional import click @@ -218,5 +218,4 @@ def _load_saved_pipeline_user_arn() -> Optional[str]: def _get_command_name() -> List[str]: ctx = click.get_current_context() - cmd_names: List[str] = cast(List[str], get_cmd_names(ctx.info_name, ctx)) # ["pipeline", "bootstrap"] - return cmd_names + return get_cmd_names(ctx.info_name, ctx) # ["pipeline", "bootstrap"] diff --git a/samcli/lib/pipeline/bootstrap/resource.py b/samcli/lib/pipeline/bootstrap/resource.py index e2471bfe4bd..badf35860b2 100644 --- a/samcli/lib/pipeline/bootstrap/resource.py +++ b/samcli/lib/pipeline/bootstrap/resource.py @@ -21,6 +21,7 @@ class ARNParts: resource-type: str the resource-type part of the ARN """ + partition: str service: str region: str diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index c7804d555a6..83f267dfb00 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -29,20 +29,26 @@ class Stage: ---------- name: str The name of the stage - aws_profile: str + aws_profile: Optional[str] The named AWS profile(in user's machine) of the AWS account to deploy this stage to. - aws_region: str + aws_region: Optional[str] The AWS region to deploy this stage to. - pipeline_user: str + pipeline_user: IamUser The IAM User having its AccessKeyId and SecretAccessKey credentials shared with the CI/CD provider pipeline_execution_role: Resource The IAM role assumed by the pipeline-user to get access to the AWS account and executes the CloudFormation stack. + pipeline_ip_range: Optional[str] + The IP range (in CIDR format) of the machine running the pipeline instance. If provided, IAM will deny requests + not coming from this IP range. + https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_aws_deny-ip.html cloudformation_execution_role: Resource The IAM role assumed by the CloudFormation service to executes the CloudFormation stack. artifacts_bucket: S3Bucket The S3 bucket to hold the SAM build artifacts of the application's CFN template. - ecr_repo: Resource + create_ecr_repo: bool + A boolean flag that determins whether the user wants to create an ECR repository or not + ecr_repo: EcrRepo The ECR repo to hold the image container of lambda functions with Image package-type Methods: @@ -82,7 +88,7 @@ def __init__( self.aws_region: Optional[str] = aws_region self.pipeline_user: IamUser = IamUser(arn=pipeline_user_arn) self.pipeline_execution_role: Resource = Resource(arn=pipeline_execution_role_arn) - self.pipeline_ip_range = pipeline_ip_range + self.pipeline_ip_range: Optional[str] = pipeline_ip_range self.cloudformation_execution_role: Resource = Resource(arn=cloudformation_execution_role_arn) self.artifacts_bucket: S3Bucket = S3Bucket(arn=artifacts_bucket_arn) self.create_ecr_repo: bool = create_ecr_repo @@ -142,8 +148,8 @@ def bootstrap(self, confirm_changeset: bool = True) -> bool: if not confirmed: return False - stage_name_suffix: str = re.sub("[^0-9a-zA-Z]+", "-", self.name) - stack_name: str = f"{STACK_NAME_PREFIX}-{stage_name_suffix}-{STAGE_RESOURCES_STACK_NAME_SUFFIX}" + sanitized_stage_name: str = re.sub("[^0-9a-zA-Z]+", "-", self.name) + stack_name: str = f"{STACK_NAME_PREFIX}-{sanitized_stage_name}-{STAGE_RESOURCES_STACK_NAME_SUFFIX}" stage_resources_template_body = Stage._read_template(STAGE_RESOURCES_CFN_TEMPLATE) output: StackOutput = manage_stack( stack_name=stack_name, From 99bef0beedfef93bf8c2fc1a1a7c5492013c8b2a Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 12:17:05 -0700 Subject: [PATCH 20/31] typo --- samcli/lib/pipeline/bootstrap/stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index 83f267dfb00..77ded532ec7 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -141,7 +141,7 @@ def bootstrap(self, confirm_changeset: bool = True) -> bool: missing_resources_msg: str = self._get_non_user_provided_resources_msg() click.echo( - "This will create the following required resources for the {self.name} stage: " f"{missing_resources_msg}" + f"This will create the following required resources for the {self.name} stage: {missing_resources_msg}" ) if confirm_changeset: confirmed: bool = click.confirm("Should we proceed with the creation?") From eacfe9c0c3d77fdc67be17740a47f8c1c75a3e23 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 12:21:04 -0700 Subject: [PATCH 21/31] read using utf-8 --- samcli/lib/pipeline/bootstrap/stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index 77ded532ec7..0a3c4f229b0 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -180,7 +180,7 @@ def bootstrap(self, confirm_changeset: bool = True) -> bool: @staticmethod def _read_template(template_file_name: str) -> str: template_path: str = os.path.join(CFN_TEMPLATE_PATH, template_file_name) - with open(template_path, "r") as fp: + with open(template_path, "r", encoding="utf-8") as fp: template_body = fp.read() return template_body From 5fdd32a55b90dff213f1f75602a1a5bcfa3355e3 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 12:41:55 -0700 Subject: [PATCH 22/31] create and user a safe version of the save_config method --- samcli/commands/pipeline/bootstrap/cli.py | 11 +++------- samcli/lib/pipeline/bootstrap/stage.py | 9 ++++++++ .../commands/pipeline/bootstrap/test_cli.py | 22 +------------------ .../unit/lib/pipeline/bootstrap/test_stage.py | 7 ++++++ 4 files changed, 20 insertions(+), 29 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index c783f39ab4b..393a6236169 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -198,14 +198,9 @@ def do_cli( if bootstrapped: stage.print_resources_summary() - try: - stage.save_config( - config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=_get_command_name() - ) - except Exception: - # Swallow saving exceptions, if any, as the resources are already bootstrapped and the ARNs are already - # printed out in the screen. - pass + stage.save_config_safe( + config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=_get_command_name() + ) def _load_saved_pipeline_user_arn() -> Optional[str]: diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index 0a3c4f229b0..8391e179584 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -238,6 +238,15 @@ def save_config(self, config_dir: str, filename: str, cmd_names: List[str]) -> N samconfig.flush() + def save_config_safe(self, config_dir: str, filename: str, cmd_names: List[str]) -> None: + """ + A safe version of save_config method that doesn't raise any exception + """ + try: + self.save_config(config_dir, filename, cmd_names) + except Exception: + pass + def _get_resources(self) -> List[Resource]: resources = [ self.pipeline_user, diff --git a/tests/unit/commands/pipeline/bootstrap/test_cli.py b/tests/unit/commands/pipeline/bootstrap/test_cli.py index f1e1e771609..9160d6bc8f8 100644 --- a/tests/unit/commands/pipeline/bootstrap/test_cli.py +++ b/tests/unit/commands/pipeline/bootstrap/test_cli.py @@ -125,7 +125,7 @@ def test_bootstrapping_normal_interactive_flow( gc_instance.run.assert_called_once() stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) stage_instance.print_resources_summary.assert_called_once() - stage_instance.save_config.assert_called_once_with( + stage_instance.save_config_safe.assert_called_once_with( config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=PIPELINE_BOOTSTRAP_COMMAND_NAMES, @@ -208,26 +208,6 @@ def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_use bootstrap_cli(**self.cli_context) stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") - @patch("samcli.commands.pipeline.bootstrap.cli.Stage") - @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") - def test_bootstrapping_will_not_fail_if_saving_resources_arns_to_local_toml_file_failed( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock - ): - # setup - stage_instance = Mock() - stage_mock.return_value = stage_instance - self.cli_context["interactive"] = False - stage_instance.save_config.side_effect = Exception - get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES - - # trigger - bootstrap_cli(**self.cli_context) - - # verify - stage_instance.save_config.assert_called_once() # called and the thrown exception got swallowed - @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file(self, get_command_name_mock, sam_config_mock): diff --git a/tests/unit/lib/pipeline/bootstrap/test_stage.py b/tests/unit/lib/pipeline/bootstrap/test_stage.py index 0dc7b286221..9c0975a5ade 100644 --- a/tests/unit/lib/pipeline/bootstrap/test_stage.py +++ b/tests/unit/lib/pipeline/bootstrap/test_stage.py @@ -297,6 +297,13 @@ def test_save_config_ignores_exceptions_thrown_while_calculating_ecr_repo_uri(se # this exception is swallowed so that other configs can be safely saved to the pipelineconfig.toml file stage.save_config(config_dir="any_config_dir", filename="any_pipeline.toml", cmd_names=["any", "commands"]) + @patch.object(Stage, "save_config") + def test_save_config_safe(self, save_config_mock): + save_config_mock.side_effect = Exception + stage: Stage = Stage(name=ANY_STAGE_NAME) + stage.save_config_safe(config_dir="any_config_dir", filename="any_pipeline.toml", cmd_names=["commands"]) + save_config_mock.assert_called_once_with("any_config_dir", "any_pipeline.toml", ["commands"]) + @patch("samcli.lib.pipeline.bootstrap.stage.click") def test_print_resources_summary_when_no_resources_provided_by_the_user(self, click_mock): stage: Stage = Stage(name=ANY_STAGE_NAME) From 83712cd14f2c37464c17ff5dafe57962ab4a9e3f Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 14:52:50 -0700 Subject: [PATCH 23/31] apply review comments --- samcli/lib/pipeline/bootstrap/resource.py | 15 ++++++--------- samcli/lib/utils/managed_cloudformation_stack.py | 1 + .../unit/lib/pipeline/bootstrap/test_resource.py | 9 --------- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/samcli/lib/pipeline/bootstrap/resource.py b/samcli/lib/pipeline/bootstrap/resource.py index badf35860b2..2a537cccd40 100644 --- a/samcli/lib/pipeline/bootstrap/resource.py +++ b/samcli/lib/pipeline/bootstrap/resource.py @@ -29,8 +29,6 @@ class ARNParts: resource_id: str def __init__(self, arn: str) -> None: - if not isinstance(arn, str): - raise ValueError(f"Invalid ARN ({arn}) is not a String") parts = arn.split(":") try: [_, self.partition, self.service, self.region, self.account_id, self.resource_id] = parts @@ -60,9 +58,6 @@ def __init__(self, arn: Optional[str]) -> None: self.arn: Optional[str] = arn self.is_user_provided: bool = bool(arn) - def _get_arn_parts(self) -> Optional[ARNParts]: - return ARNParts(self.arn) if self.arn else None - def name(self) -> Optional[str]: """ extracts and returns the resource name from its ARN @@ -70,8 +65,10 @@ def name(self) -> Optional[str]: ------ ValueError if the ARN is invalid """ - arn_parts: Optional[ARNParts] = self._get_arn_parts() - return arn_parts.resource_id if arn_parts else None + if not self.arn: + return None + arn_parts: ARNParts = ARNParts(arn=self.arn) + return arn_parts.resource_id class IamUser(Resource): @@ -121,9 +118,9 @@ def get_uri(self) -> Optional[str]: ------ ValueError if the ARN is invalid """ - arn_parts: Optional[ARNParts] = self._get_arn_parts() - if not arn_parts: + if not self.arn: return None + arn_parts: ARNParts = ARNParts(self.arn) # ECR's resource_id contains the resource-type("resource") which is excluded from the URL # from docs: https://docs.aws.amazon.com/AmazonECR/latest/userguide/security_iam_service-with-iam.html # ECR's ARN: arn:${Partition}:ecr:${Region}:${Account}:repository/${Repository-name} diff --git a/samcli/lib/utils/managed_cloudformation_stack.py b/samcli/lib/utils/managed_cloudformation_stack.py index 84ed1031ba4..29d148a7d99 100644 --- a/samcli/lib/utils/managed_cloudformation_stack.py +++ b/samcli/lib/utils/managed_cloudformation_stack.py @@ -192,6 +192,7 @@ def _generate_stack_parameters( if parameter_overrides: for key, value in parameter_overrides.items(): if isinstance(value, Collection) and not isinstance(value, str): + # Assumption: values don't include commas or spaces. Need to refactor to handle such a case if needed. value = ",".join(value) parameters.append({"ParameterKey": key, "ParameterValue": value}) return parameters diff --git a/tests/unit/lib/pipeline/bootstrap/test_resource.py b/tests/unit/lib/pipeline/bootstrap/test_resource.py index 5e5d20c8b35..06c96a2a5f2 100644 --- a/tests/unit/lib/pipeline/bootstrap/test_resource.py +++ b/tests/unit/lib/pipeline/bootstrap/test_resource.py @@ -15,15 +15,6 @@ def test_arn_parts_of_valid_arn(self): self.assertEqual(arn_parts.account_id, "account-id") self.assertEqual(arn_parts.resource_id, "resource-id") - def test_arn_parts_of_none_arn_is_invalid(self): - with self.assertRaises(ValueError): - ARNParts(arn=None) - - def test_arn_parts_of_none_string_arn_is_invalid(self): - with self.assertRaises(ValueError): - any_non_string = 1 - ARNParts(arn=any_non_string) - def test_arn_parts_of_invalid_arn(self): with self.assertRaises(ValueError): invalid_arn = "invalid_arn" From 2c3ad8b2f95fcc16fd79713a884e2aa8ff6da001 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 15:00:37 -0700 Subject: [PATCH 24/31] rename _get_command_name to _get_command_names --- samcli/commands/pipeline/bootstrap/cli.py | 6 +- .../commands/pipeline/bootstrap/test_cli.py | 59 +++++++++---------- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index 393a6236169..1b2c7093729 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -199,7 +199,7 @@ def do_cli( stage.print_resources_summary() stage.save_config_safe( - config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=_get_command_name() + config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=_get_command_names() ) @@ -207,10 +207,10 @@ def _load_saved_pipeline_user_arn() -> Optional[str]: samconfig: SamConfig = SamConfig(config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME) if not samconfig.exists(): return None - config: Dict[str, str] = samconfig.get_all(cmd_names=_get_command_name(), section="parameters") + config: Dict[str, str] = samconfig.get_all(cmd_names=_get_command_names(), section="parameters") return config.get("pipeline_user") -def _get_command_name() -> List[str]: +def _get_command_names() -> List[str]: ctx = click.get_current_context() return get_cmd_names(ctx.info_name, ctx) # ["pipeline", "bootstrap"] diff --git a/tests/unit/commands/pipeline/bootstrap/test_cli.py b/tests/unit/commands/pipeline/bootstrap/test_cli.py index 9160d6bc8f8..4a293f2a69e 100644 --- a/tests/unit/commands/pipeline/bootstrap/test_cli.py +++ b/tests/unit/commands/pipeline/bootstrap/test_cli.py @@ -1,4 +1,3 @@ -import os from unittest import TestCase from unittest.mock import patch, Mock @@ -7,7 +6,7 @@ from samcli.commands.pipeline.bootstrap.cli import ( _load_saved_pipeline_user_arn, - _get_command_name, + _get_command_names, PIPELINE_CONFIG_FILENAME, PIPELINE_CONFIG_DIR, ) @@ -100,12 +99,12 @@ def test_bootstrap_command_with_different_arguments_combination(self, do_cli_moc self.assertEqual(kwargs["stage_name"], "stage1") self.assertEqual(kwargs["artifacts_bucket_arn"], "bucketARN") - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrapping_normal_interactive_flow( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_names_mock ): # setup gc_instance = Mock() @@ -115,7 +114,7 @@ def test_bootstrapping_normal_interactive_flow( load_saved_pipeline_user_arn_mock.return_value = ANY_PIPELINE_USER_ARN self.cli_context["interactive"] = True self.cli_context["pipeline_user_arn"] = None - get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES + get_command_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES # trigger bootstrap_cli(**self.cli_context) @@ -131,56 +130,56 @@ def test_bootstrapping_normal_interactive_flow( cmd_names=PIPELINE_BOOTSTRAP_COMMAND_NAMES, ) - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrap_will_not_try_loading_pipeline_user_if_already_provided( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_names_mock ): bootstrap_cli(**self.cli_context) load_saved_pipeline_user_arn_mock.assert_not_called() - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrap_will_try_loading_pipeline_user_if_not_provided( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_names_mock ): self.cli_context["pipeline_user_arn"] = None bootstrap_cli(**self.cli_context) load_saved_pipeline_user_arn_mock.assert_called_once() - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_stage_name_is_required_to_be_provided_in_case_of_non_interactive_mode( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_names_mock ): self.cli_context["interactive"] = False self.cli_context["stage_name"] = None with self.assertRaises(click.UsageError): bootstrap_cli(**self.cli_context) - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_stage_name_is_not_required_to_be_provided_in_case_of_interactive_mode( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_names_mock ): self.cli_context["interactive"] = True self.cli_context["stage_name"] = None bootstrap_cli(**self.cli_context) # No exception is thrown - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_guided_context_will_be_enabled_or_disabled_based_on_the_interactive_mode( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_names_mock ): gc_instance = Mock() guided_context_mock.return_value = gc_instance @@ -191,12 +190,12 @@ def test_guided_context_will_be_enabled_or_disabled_based_on_the_interactive_mod bootstrap_cli(**self.cli_context) gc_instance.run.assert_called_once() - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @patch("samcli.commands.pipeline.bootstrap.cli.Stage") @patch("samcli.commands.pipeline.bootstrap.cli.GuidedContext") def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_user_choose_not_to( - self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_name_mock + self, guided_context_mock, stage_mock, load_saved_pipeline_user_arn_mock, get_command_names_mock ): stage_instance = Mock() stage_mock.return_value = stage_instance @@ -209,10 +208,10 @@ def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_use stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") - def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file(self, get_command_name_mock, sam_config_mock): + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") + def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file(self, get_command_names_mock, sam_config_mock): # setup - get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES + get_command_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() sam_config_mock.return_value = sam_config_instance_mock sam_config_instance_mock.exists.return_value = False @@ -224,12 +223,12 @@ def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file(self, get_ sam_config_mock.assert_called_once_with(config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME) @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") def test_load_saved_pipeline_user_arn_will_return_non_if_the_pipeline_toml_file_is_not_found( - self, get_command_name_mock, sam_config_mock + self, get_command_names_mock, sam_config_mock ): # setup - get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES + get_command_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() sam_config_mock.return_value = sam_config_instance_mock sam_config_instance_mock.exists.return_value = False @@ -241,12 +240,12 @@ def test_load_saved_pipeline_user_arn_will_return_non_if_the_pipeline_toml_file_ self.assertIsNone(pipeline_user_arn) @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") def test_load_saved_pipeline_user_arn_will_return_non_if_the_pipeline_toml_file_does_not_contain_pipeline_user( - self, get_command_name_mock, sam_config_mock + self, get_command_names_mock, sam_config_mock ): # setup - get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES + get_command_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() sam_config_mock.return_value = sam_config_instance_mock sam_config_instance_mock.exists.return_value = True @@ -259,12 +258,12 @@ def test_load_saved_pipeline_user_arn_will_return_non_if_the_pipeline_toml_file_ self.assertIsNone(pipeline_user_arn) @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") - @patch("samcli.commands.pipeline.bootstrap.cli._get_command_name") + @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") def test_load_saved_pipeline_user_arn_returns_the_pipeline_user_arn_from_the_pipeline_toml_file( - self, get_command_name_mock, sam_config_mock + self, get_command_names_mock, sam_config_mock ): # setup - get_command_name_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES + get_command_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() sam_config_mock.return_value = sam_config_instance_mock sam_config_instance_mock.exists.return_value = True @@ -285,7 +284,7 @@ def test_get_command_name(self, click_mock, get_cmd_names_mock): get_cmd_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES # trigger - cmd_names = _get_command_name() + cmd_names = _get_command_names() # verify self.assertEqual(cmd_names, PIPELINE_BOOTSTRAP_COMMAND_NAMES) From d184e164022d9560131c62a826436edbc93da189 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 15:27:08 -0700 Subject: [PATCH 25/31] don't save generated ARNs for now, will save during init --- samcli/commands/pipeline/bootstrap/cli.py | 4 ---- tests/unit/commands/pipeline/bootstrap/test_cli.py | 9 +++------ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index 1b2c7093729..46a19720f42 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -198,10 +198,6 @@ def do_cli( if bootstrapped: stage.print_resources_summary() - stage.save_config_safe( - config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=_get_command_names() - ) - def _load_saved_pipeline_user_arn() -> Optional[str]: samconfig: SamConfig = SamConfig(config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME) diff --git a/tests/unit/commands/pipeline/bootstrap/test_cli.py b/tests/unit/commands/pipeline/bootstrap/test_cli.py index 4a293f2a69e..4a79b93a779 100644 --- a/tests/unit/commands/pipeline/bootstrap/test_cli.py +++ b/tests/unit/commands/pipeline/bootstrap/test_cli.py @@ -124,11 +124,6 @@ def test_bootstrapping_normal_interactive_flow( gc_instance.run.assert_called_once() stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) stage_instance.print_resources_summary.assert_called_once() - stage_instance.save_config_safe.assert_called_once_with( - config_dir=PIPELINE_CONFIG_DIR, - filename=PIPELINE_CONFIG_FILENAME, - cmd_names=PIPELINE_BOOTSTRAP_COMMAND_NAMES, - ) @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @@ -209,7 +204,9 @@ def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_use @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") - def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file(self, get_command_names_mock, sam_config_mock): + def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file( + self, get_command_names_mock, sam_config_mock + ): # setup get_command_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() From 6d9fb34744697f65337681f6367dd76c1c6c0370 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 15:47:02 -0700 Subject: [PATCH 26/31] Revert "don't save generated ARNs for now, will save during init" This reverts commit d184e164022d9560131c62a826436edbc93da189. --- samcli/commands/pipeline/bootstrap/cli.py | 4 ++++ tests/unit/commands/pipeline/bootstrap/test_cli.py | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index 46a19720f42..1b2c7093729 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -198,6 +198,10 @@ def do_cli( if bootstrapped: stage.print_resources_summary() + stage.save_config_safe( + config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME, cmd_names=_get_command_names() + ) + def _load_saved_pipeline_user_arn() -> Optional[str]: samconfig: SamConfig = SamConfig(config_dir=PIPELINE_CONFIG_DIR, filename=PIPELINE_CONFIG_FILENAME) diff --git a/tests/unit/commands/pipeline/bootstrap/test_cli.py b/tests/unit/commands/pipeline/bootstrap/test_cli.py index 4a79b93a779..4a293f2a69e 100644 --- a/tests/unit/commands/pipeline/bootstrap/test_cli.py +++ b/tests/unit/commands/pipeline/bootstrap/test_cli.py @@ -124,6 +124,11 @@ def test_bootstrapping_normal_interactive_flow( gc_instance.run.assert_called_once() stage_instance.bootstrap.assert_called_once_with(confirm_changeset=True) stage_instance.print_resources_summary.assert_called_once() + stage_instance.save_config_safe.assert_called_once_with( + config_dir=PIPELINE_CONFIG_DIR, + filename=PIPELINE_CONFIG_FILENAME, + cmd_names=PIPELINE_BOOTSTRAP_COMMAND_NAMES, + ) @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") @patch("samcli.commands.pipeline.bootstrap.cli._load_saved_pipeline_user_arn") @@ -204,9 +209,7 @@ def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_use @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") - def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file( - self, get_command_names_mock, sam_config_mock - ): + def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file(self, get_command_names_mock, sam_config_mock): # setup get_command_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() From 0f1152c1a8cd321d282db5fd61d6e02a893e72b8 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 16:03:28 -0700 Subject: [PATCH 27/31] Notify the user to rotate periodically rotate the IAM credentials --- samcli/lib/pipeline/bootstrap/stage.py | 6 +++++- tests/unit/commands/pipeline/bootstrap/test_cli.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index 8391e179584..b5772f98344 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -284,6 +284,10 @@ def print_resources_summary(self) -> None: click.secho(f"\t{resource.arn}", fg="green") if not self.pipeline_user.is_user_provided: - click.secho("Please configure your CI/CD project with the following pipeline user credentials:", fg="green") + click.secho( + "Please configure your CI/CD project with the following pipeline user credentials and " + "make sure to periodically rotate it: ", + fg="green", + ) click.secho(f"\tACCESS_KEY_ID: {self.pipeline_user.access_key_id}", fg="green") click.secho(f"\tSECRET_ACCESS_KEY: {self.pipeline_user.secret_access_key}", fg="green") diff --git a/tests/unit/commands/pipeline/bootstrap/test_cli.py b/tests/unit/commands/pipeline/bootstrap/test_cli.py index 4a293f2a69e..093e440e029 100644 --- a/tests/unit/commands/pipeline/bootstrap/test_cli.py +++ b/tests/unit/commands/pipeline/bootstrap/test_cli.py @@ -209,7 +209,9 @@ def test_bootstrapping_will_confirm_before_creating_the_resources_unless_the_use @patch("samcli.commands.pipeline.bootstrap.cli.SamConfig") @patch("samcli.commands.pipeline.bootstrap.cli._get_command_names") - def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file(self, get_command_names_mock, sam_config_mock): + def test_load_saved_pipeline_user_arn_will_read_from_the_correct_file( + self, get_command_names_mock, sam_config_mock + ): # setup get_command_names_mock.return_value = PIPELINE_BOOTSTRAP_COMMAND_NAMES sam_config_instance_mock = Mock() From 1721c3570175bf5ca7fbe466476279eaa23eb036 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 16:04:23 -0700 Subject: [PATCH 28/31] typo --- samcli/lib/pipeline/bootstrap/stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index b5772f98344..8ac722ce644 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -286,7 +286,7 @@ def print_resources_summary(self) -> None: if not self.pipeline_user.is_user_provided: click.secho( "Please configure your CI/CD project with the following pipeline user credentials and " - "make sure to periodically rotate it: ", + "make sure to periodically rotate it:", fg="green", ) click.secho(f"\tACCESS_KEY_ID: {self.pipeline_user.access_key_id}", fg="green") From 15b64d9f08f12053780cc119e10a96ed2ad7a4d4 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Tue, 20 Apr 2021 17:57:54 -0700 Subject: [PATCH 29/31] Use AES instead of KMS for S3 SSE --- .../pipeline/bootstrap/guided_context.py | 2 +- samcli/lib/pipeline/bootstrap/stage.py | 9 ++-- .../pipeline/bootstrap/stage_resources.yaml | 54 +------------------ .../lib/pipeline/bootstrap/test_resource.py | 13 +---- 4 files changed, 7 insertions(+), 71 deletions(-) diff --git a/samcli/commands/pipeline/bootstrap/guided_context.py b/samcli/commands/pipeline/bootstrap/guided_context.py index 572021adbe6..13f2cb5cb0c 100644 --- a/samcli/commands/pipeline/bootstrap/guided_context.py +++ b/samcli/commands/pipeline/bootstrap/guided_context.py @@ -72,7 +72,7 @@ def run(self) -> None: ) if not self.ecr_repo_arn: click.echo( - "\nIf your SAM template will include Lambda functions of Image package type, " + "\nIf your SAM template includes (or going to include) Lambda functions of Image package type, " "then an ECR repository is required. Should we create one?" ) click.echo("\t1 - No, My SAM template won't include Lambda functions of Image package type") diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index 8ac722ce644..cd9de3e885b 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -8,7 +8,7 @@ from samcli.lib.config.samconfig import SamConfig from samcli.lib.utils.managed_cloudformation_stack import manage_stack, StackOutput -from .resource import Resource, IamUser, S3Bucket, EcrRepo +from .resource import Resource, IamUser, EcrRepo CFN_TEMPLATE_PATH = str(pathlib.Path(os.path.dirname(__file__))) STACK_NAME_PREFIX = "aws-sam-cli-managed" @@ -44,7 +44,7 @@ class Stage: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_aws_deny-ip.html cloudformation_execution_role: Resource The IAM role assumed by the CloudFormation service to executes the CloudFormation stack. - artifacts_bucket: S3Bucket + artifacts_bucket: Resource The S3 bucket to hold the SAM build artifacts of the application's CFN template. create_ecr_repo: bool A boolean flag that determins whether the user wants to create an ECR repository or not @@ -90,7 +90,7 @@ def __init__( self.pipeline_execution_role: Resource = Resource(arn=pipeline_execution_role_arn) self.pipeline_ip_range: Optional[str] = pipeline_ip_range self.cloudformation_execution_role: Resource = Resource(arn=cloudformation_execution_role_arn) - self.artifacts_bucket: S3Bucket = S3Bucket(arn=artifacts_bucket_arn) + self.artifacts_bucket: Resource = Resource(arn=artifacts_bucket_arn) self.create_ecr_repo: bool = create_ecr_repo self.ecr_repo: EcrRepo = EcrRepo(arn=ecr_repo_arn) @@ -118,7 +118,7 @@ def bootstrap(self, confirm_changeset: bool = True) -> bool: * Pipeline IAM User * Pipeline execution IAM role * CloudFormation execution IAM role - * Artifacts' S3 Bucket along with KMS encryption key + * Artifacts' S3 Bucket * ECR Repo to the AWS account associated with the given stage. It will not redeploy the stack if already exists. This CFN template accepts the ARNs of the resources as parameters and will not create a resource if already @@ -173,7 +173,6 @@ def bootstrap(self, confirm_changeset: bool = True) -> bool: self.pipeline_execution_role.arn = output.get("PipelineExecutionRole") self.cloudformation_execution_role.arn = output.get("CloudFormationExecutionRole") self.artifacts_bucket.arn = output.get("ArtifactsBucket") - self.artifacts_bucket.kms_key_arn = output.get("ArtifactsBucketKMS") self.ecr_repo.arn = output.get("EcrRepo") return True diff --git a/samcli/lib/pipeline/bootstrap/stage_resources.yaml b/samcli/lib/pipeline/bootstrap/stage_resources.yaml index 7134f3c8507..1154f831fc0 100644 --- a/samcli/lib/pipeline/bootstrap/stage_resources.yaml +++ b/samcli/lib/pipeline/bootstrap/stage_resources.yaml @@ -124,43 +124,6 @@ Resources: - !Ref PipelineIpRange - !Ref AWS::NoValue - ArtifactsBucketKey: - Type: AWS::KMS::Key - Condition: MissingArtifactsBucket - Properties: - Tags: - - Key: ManagedStackSource - Value: AwsSamCli - Description: Artifact encryption/decryption cmk - EnableKeyRotation: true - KeyPolicy: - Version: '2012-10-17' - Id: !Ref AWS::StackName - Statement: - - Effect: Allow - Principal: - AWS: !Sub arn:aws:iam::${AWS::AccountId}:root - Action: kms:* - Resource: '*' - - Effect: Allow - Principal: - AWS: - - Fn::If: - - MissingPipelineExecutionRole - - !GetAtt PipelineExecutionRole.Arn - - !Ref PipelineExecutionRoleArn - - Fn::If: - - MissingCloudFormationExecutionRole - - !GetAtt CloudFormationExecutionRole.Arn - - !Ref CloudFormationExecutionRoleArn - Action: - - kms:Encrypt - - kms:Decrypt - - kms:ReEncrypt* - - kms:GenerateDataKey* - - kms:DescribeKey - Resource: '*' - ArtifactsBucket: Type: AWS::S3::Bucket Condition: MissingArtifactsBucket @@ -174,8 +137,7 @@ Resources: BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: - KMSMasterKeyID: !GetAtt ArtifactsBucketKey.Arn - SSEAlgorithm: aws:kms + SSEAlgorithm: AES256 ArtifactsBucketPolicy: Type: AWS::S3::BucketPolicy @@ -242,15 +204,6 @@ Resources: - !GetAtt ArtifactsBucket.Arn - - !Join [ '',[ !Ref ArtifactsBucketArn, '/*' ] ] - !Ref ArtifactsBucketArn - - Fn::If: - - MissingArtifactsBucket - - Effect: "Allow" - Action: - - "kms:Decrypt" - - "kms:DescribeKey" - Resource: - - !GetAtt ArtifactsBucketKey.Arn - - !Ref AWS::NoValue - Fn::If: - EcrRepoRequired - Effect: "Allow" @@ -348,11 +301,6 @@ Outputs: - !GetAtt ArtifactsBucket.Arn - !Ref ArtifactsBucketArn - ArtifactsBucketKey: - Description: ARN of the CMK used for Artifacts bucket encryption/decryption - Condition: MissingArtifactsBucket - Value: !GetAtt ArtifactsBucketKey.Arn - EcrRepo: Description: ARN of the ECR repo Condition: MissingEcrRepo diff --git a/tests/unit/lib/pipeline/bootstrap/test_resource.py b/tests/unit/lib/pipeline/bootstrap/test_resource.py index 06c96a2a5f2..e118270f73b 100644 --- a/tests/unit/lib/pipeline/bootstrap/test_resource.py +++ b/tests/unit/lib/pipeline/bootstrap/test_resource.py @@ -1,6 +1,6 @@ from unittest import TestCase -from samcli.lib.pipeline.bootstrap.resource import ARNParts, Resource, S3Bucket, IamUser, EcrRepo +from samcli.lib.pipeline.bootstrap.resource import ARNParts, Resource, IamUser, EcrRepo VALID_ARN = "arn:partition:service:region:account-id:resource-id" INVALID_ARN = "ARN" @@ -53,17 +53,6 @@ def test_create_iam_user(self): self.assertEquals(user.secret_access_key, "any_secret_access_key") -class TestS3Bucket(TestCase): - def test_create_s3_bucket(self): - bucket: S3Bucket = S3Bucket(arn=VALID_ARN) - self.assertEquals(bucket.arn, VALID_ARN) - self.assertIsNone(bucket.kms_key_arn) - - bucket = S3Bucket(arn=INVALID_ARN, kms_key_arn="any_kms_key_arn") - self.assertEquals(bucket.arn, INVALID_ARN) - self.assertEquals(bucket.kms_key_arn, "any_kms_key_arn") - - class TestEcrRepo(TestCase): def test_get_uri_with_valid_ecr_arn(self): valid_ecr_arn = "arn:partition:service:region:account-id:repository/repository-name" From b1e31c05f931d433866d24dbe2b1e9f8839d1523 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Thu, 22 Apr 2021 18:12:59 -0700 Subject: [PATCH 30/31] rename Ecr to ECR and Iam to IAM --- samcli/lib/pipeline/bootstrap/resource.py | 8 +++--- samcli/lib/pipeline/bootstrap/stage.py | 16 +++++------ .../pipeline/bootstrap/stage_resources.yaml | 28 +++++++++---------- .../lib/pipeline/bootstrap/test_resource.py | 16 +++++------ .../unit/lib/pipeline/bootstrap/test_stage.py | 4 +-- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/samcli/lib/pipeline/bootstrap/resource.py b/samcli/lib/pipeline/bootstrap/resource.py index 2a537cccd40..8dd2462b4be 100644 --- a/samcli/lib/pipeline/bootstrap/resource.py +++ b/samcli/lib/pipeline/bootstrap/resource.py @@ -71,9 +71,9 @@ def name(self) -> Optional[str]: return arn_parts.resource_id -class IamUser(Resource): +class IAMUser(Resource): """ - Represents an AWS IamUser resource + Represents an AWS IAM User resource Attributes ---------- access_key_id: Optional[str] @@ -104,8 +104,8 @@ def __init__(self, arn: Optional[str], kms_key_arn: Optional[str] = None) -> Non super().__init__(arn=arn) -class EcrRepo(Resource): - """ Represents an AWS EcrRepo resource """ +class ECRRepo(Resource): + """ Represents an AWS ECR repo resource """ def __init__(self, arn: Optional[str]) -> None: super().__init__(arn=arn) diff --git a/samcli/lib/pipeline/bootstrap/stage.py b/samcli/lib/pipeline/bootstrap/stage.py index cd9de3e885b..a0071c3bba5 100644 --- a/samcli/lib/pipeline/bootstrap/stage.py +++ b/samcli/lib/pipeline/bootstrap/stage.py @@ -8,7 +8,7 @@ from samcli.lib.config.samconfig import SamConfig from samcli.lib.utils.managed_cloudformation_stack import manage_stack, StackOutput -from .resource import Resource, IamUser, EcrRepo +from .resource import Resource, IAMUser, ECRRepo CFN_TEMPLATE_PATH = str(pathlib.Path(os.path.dirname(__file__))) STACK_NAME_PREFIX = "aws-sam-cli-managed" @@ -33,7 +33,7 @@ class Stage: The named AWS profile(in user's machine) of the AWS account to deploy this stage to. aws_region: Optional[str] The AWS region to deploy this stage to. - pipeline_user: IamUser + pipeline_user: IAMUser The IAM User having its AccessKeyId and SecretAccessKey credentials shared with the CI/CD provider pipeline_execution_role: Resource The IAM role assumed by the pipeline-user to get access to the AWS account and executes the @@ -48,7 +48,7 @@ class Stage: The S3 bucket to hold the SAM build artifacts of the application's CFN template. create_ecr_repo: bool A boolean flag that determins whether the user wants to create an ECR repository or not - ecr_repo: EcrRepo + ecr_repo: ECRRepo The ECR repo to hold the image container of lambda functions with Image package-type Methods: @@ -86,13 +86,13 @@ def __init__( self.name: str = name self.aws_profile: Optional[str] = aws_profile self.aws_region: Optional[str] = aws_region - self.pipeline_user: IamUser = IamUser(arn=pipeline_user_arn) + self.pipeline_user: IAMUser = IAMUser(arn=pipeline_user_arn) self.pipeline_execution_role: Resource = Resource(arn=pipeline_execution_role_arn) self.pipeline_ip_range: Optional[str] = pipeline_ip_range self.cloudformation_execution_role: Resource = Resource(arn=cloudformation_execution_role_arn) self.artifacts_bucket: Resource = Resource(arn=artifacts_bucket_arn) self.create_ecr_repo: bool = create_ecr_repo - self.ecr_repo: EcrRepo = EcrRepo(arn=ecr_repo_arn) + self.ecr_repo: ECRRepo = ECRRepo(arn=ecr_repo_arn) def did_user_provide_all_required_resources(self) -> bool: """Check if the user provided all of the stage resources or not""" @@ -162,8 +162,8 @@ def bootstrap(self, confirm_changeset: bool = True) -> bool: "PipelineIpRange": self.pipeline_ip_range or "", "CloudFormationExecutionRoleArn": self.cloudformation_execution_role.arn or "", "ArtifactsBucketArn": self.artifacts_bucket.arn or "", - "CreateEcrRepo": "true" if self.create_ecr_repo else "false", - "EcrRepoArn": self.ecr_repo.arn or "", + "CreateECRRepo": "true" if self.create_ecr_repo else "false", + "ECRRepoArn": self.ecr_repo.arn or "", }, ) @@ -173,7 +173,7 @@ def bootstrap(self, confirm_changeset: bool = True) -> bool: self.pipeline_execution_role.arn = output.get("PipelineExecutionRole") self.cloudformation_execution_role.arn = output.get("CloudFormationExecutionRole") self.artifacts_bucket.arn = output.get("ArtifactsBucket") - self.ecr_repo.arn = output.get("EcrRepo") + self.ecr_repo.arn = output.get("ECRRepo") return True @staticmethod diff --git a/samcli/lib/pipeline/bootstrap/stage_resources.yaml b/samcli/lib/pipeline/bootstrap/stage_resources.yaml index 1154f831fc0..6ef83dfa809 100644 --- a/samcli/lib/pipeline/bootstrap/stage_resources.yaml +++ b/samcli/lib/pipeline/bootstrap/stage_resources.yaml @@ -12,11 +12,11 @@ Parameters: Type: String ArtifactsBucketArn: Type: String - CreateEcrRepo: + CreateECRRepo: Type: String Default: false AllowedValues: [true, false] - EcrRepoArn: + ECRRepoArn: Type: String Conditions: @@ -25,8 +25,8 @@ Conditions: HasPipelineIpRange: !Not [!Equals [!Ref PipelineIpRange, ""]] MissingCloudFormationExecutionRole: !Equals [!Ref CloudFormationExecutionRoleArn, ""] MissingArtifactsBucket: !Equals [!Ref ArtifactsBucketArn, ""] - EcrRepoRequired: !Equals [!Ref CreateEcrRepo, "true"] - MissingEcrRepo: !And [!Condition EcrRepoRequired, !Equals [!Ref EcrRepoArn, ""]] + ECRRepoRequired: !Equals [!Ref CreateECRRepo, "true"] + MissingECRRepo: !And [!Condition ECRRepoRequired, !Equals [!Ref ECRRepoArn, ""]] Resources: PipelineUser: @@ -205,13 +205,13 @@ Resources: - - !Join [ '',[ !Ref ArtifactsBucketArn, '/*' ] ] - !Ref ArtifactsBucketArn - Fn::If: - - EcrRepoRequired + - ECRRepoRequired - Effect: "Allow" Action: "ecr:GetAuthorizationToken" Resource: "*" - !Ref AWS::NoValue - Fn::If: - - EcrRepoRequired + - ECRRepoRequired - Effect: "Allow" Action: - "ecr:GetDownloadUrlForLayer" @@ -223,16 +223,16 @@ Resources: - "ecr:CompleteLayerUpload" Resource: Fn::If: - - MissingEcrRepo - - !GetAtt EcrRepo.Arn - - !Ref EcrRepoArn + - MissingECRRepo + - !GetAtt ECRRepo.Arn + - !Ref ECRRepoArn - !Ref AWS::NoValue Roles: - !Ref PipelineExecutionRole - EcrRepo: + ECRRepo: Type: AWS::ECR::Repository - Condition: MissingEcrRepo + Condition: MissingECRRepo Properties: RepositoryPolicyText: Version: "2012-10-17" @@ -301,7 +301,7 @@ Outputs: - !GetAtt ArtifactsBucket.Arn - !Ref ArtifactsBucketArn - EcrRepo: + ECRRepo: Description: ARN of the ECR repo - Condition: MissingEcrRepo - Value: !GetAtt EcrRepo.Arn + Condition: MissingECRRepo + Value: !GetAtt ECRRepo.Arn diff --git a/tests/unit/lib/pipeline/bootstrap/test_resource.py b/tests/unit/lib/pipeline/bootstrap/test_resource.py index e118270f73b..be7a9d3c313 100644 --- a/tests/unit/lib/pipeline/bootstrap/test_resource.py +++ b/tests/unit/lib/pipeline/bootstrap/test_resource.py @@ -1,6 +1,6 @@ from unittest import TestCase -from samcli.lib.pipeline.bootstrap.resource import ARNParts, Resource, IamUser, EcrRepo +from samcli.lib.pipeline.bootstrap.resource import ARNParts, Resource, IAMUser, ECRRepo VALID_ARN = "arn:partition:service:region:account-id:resource-id" INVALID_ARN = "ARN" @@ -40,27 +40,27 @@ def test_resource(self): self.assertIsNone(resource.name()) -class TestIamUser(TestCase): +class TestIAMUser(TestCase): def test_create_iam_user(self): - user: IamUser = IamUser(arn=VALID_ARN) + user: IAMUser = IAMUser(arn=VALID_ARN) self.assertEquals(user.arn, VALID_ARN) self.assertIsNone(user.access_key_id) self.assertIsNone(user.secret_access_key) - user = IamUser(arn=INVALID_ARN, access_key_id="any_access_key_id", secret_access_key="any_secret_access_key") + user = IAMUser(arn=INVALID_ARN, access_key_id="any_access_key_id", secret_access_key="any_secret_access_key") self.assertEquals(user.arn, INVALID_ARN) self.assertEquals(user.access_key_id, "any_access_key_id") self.assertEquals(user.secret_access_key, "any_secret_access_key") -class TestEcrRepo(TestCase): +class TestECRRepo(TestCase): def test_get_uri_with_valid_ecr_arn(self): valid_ecr_arn = "arn:partition:service:region:account-id:repository/repository-name" - repo: EcrRepo = EcrRepo(arn=valid_ecr_arn) + repo: ECRRepo = ECRRepo(arn=valid_ecr_arn) self.assertEqual(repo.get_uri(), "account-id.dkr.ecr.region.amazonaws.com/repository-name") def test_get_uri_with_invalid_ecr_arn(self): - repo = EcrRepo(arn=INVALID_ARN) + repo = ECRRepo(arn=INVALID_ARN) with self.assertRaises(ValueError): repo.get_uri() @@ -68,6 +68,6 @@ def test_get_uri_with_valid_aws_arn_that_is_invalid_ecr_arn(self): ecr_arn_missing_repository_prefix = ( "arn:partition:service:region:account-id:repository-name-without-repository/-prefix" ) - repo = EcrRepo(arn=ecr_arn_missing_repository_prefix) + repo = ECRRepo(arn=ecr_arn_missing_repository_prefix) with self.assertRaises(ValueError): repo.get_uri() diff --git a/tests/unit/lib/pipeline/bootstrap/test_stage.py b/tests/unit/lib/pipeline/bootstrap/test_stage.py index 9c0975a5ade..348b5acd88d 100644 --- a/tests/unit/lib/pipeline/bootstrap/test_stage.py +++ b/tests/unit/lib/pipeline/bootstrap/test_stage.py @@ -178,8 +178,8 @@ def test_bootstrap_will_pass_arns_of_all_user_provided_resources_any_empty_strin "PipelineIpRange": "", "CloudFormationExecutionRoleArn": "", "ArtifactsBucketArn": ANY_ARTIFACTS_BUCKET_ARN, - "CreateEcrRepo": "true", - "EcrRepoArn": ANY_ECR_REPO_ARN, + "CreateECRRepo": "true", + "ECRRepoArn": ANY_ECR_REPO_ARN, } self.assertEqual(expected_parameter_overrides, kwargs["parameter_overrides"]) From 050827aa351083fbf4f921f1a42c3345f633f63d Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Fri, 23 Apr 2021 13:01:21 -0700 Subject: [PATCH 31/31] Grant lambda service explicit permissions to thhe ECR instead of relying on giving this permissions on ad-hoc while creating the container images --- samcli/lib/pipeline/bootstrap/stage_resources.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/samcli/lib/pipeline/bootstrap/stage_resources.yaml b/samcli/lib/pipeline/bootstrap/stage_resources.yaml index 6ef83dfa809..61b5e723556 100644 --- a/samcli/lib/pipeline/bootstrap/stage_resources.yaml +++ b/samcli/lib/pipeline/bootstrap/stage_resources.yaml @@ -237,6 +237,16 @@ Resources: RepositoryPolicyText: Version: "2012-10-17" Statement: + - Sid: LambdaECRImageRetrievalPolicy + Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: + - "ecr:GetDownloadUrlForLayer" + - "ecr:BatchGetImage" + - "ecr:GetRepositoryPolicy" + - "ecr:SetRepositoryPolicy" + - "ecr:DeleteRepositoryPolicy" - Sid: AllowPushPull Effect: Allow Principal: