From a9ed249d8055f78cbd274e8b404a7b028a3d60b2 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sat, 5 Mar 2022 14:19:20 -0800 Subject: [PATCH 01/19] feat(backend): Internet gateway --- backend/src/terraform/AwsInternetGateway.ts | 22 +++++++++++++++++++ backend/src/terraform/awsSubnet.ts | 2 +- backend/src/terraform/awsVpc.ts | 24 ++++++++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 backend/src/terraform/AwsInternetGateway.ts diff --git a/backend/src/terraform/AwsInternetGateway.ts b/backend/src/terraform/AwsInternetGateway.ts new file mode 100644 index 0000000..db577de --- /dev/null +++ b/backend/src/terraform/AwsInternetGateway.ts @@ -0,0 +1,22 @@ +import {jsonRoot} from "./util"; +import {Resource} from "./resource"; + +export interface AwsInternetGateway { + vpc: string; +} +export class AwsInternetGateway + extends Resource + implements AwsInternetGateway +{ + constructor(id: string, vpc: string, name?: string) { + super(id, "AwsInternetGateway", false, name); + this.vpc = vpc; + } + + //Returns a resource block + toJSON() { + return jsonRoot("aws_internet_gateway", this.id, { + vpc_id: `\${aws_vpc.${this.vpc}.id}` + }); + } +} diff --git a/backend/src/terraform/awsSubnet.ts b/backend/src/terraform/awsSubnet.ts index f5c250e..c2ea3a9 100644 --- a/backend/src/terraform/awsSubnet.ts +++ b/backend/src/terraform/awsSubnet.ts @@ -29,7 +29,7 @@ export class AwsSubnet extends Resource implements AwsSubnet { toJSON() { return [ jsonRoot("aws_subnet", this.id, { - vpc: this.vpc, + vpc_id: `\${aws_vpc.${this.vpc}.id}`, cidr_block: this.cidr_block, map_public_ip_on_launch: this.map_public_ip_on_launch, availability_zone: this.availability_zone diff --git a/backend/src/terraform/awsVpc.ts b/backend/src/terraform/awsVpc.ts index a033956..08d74d5 100644 --- a/backend/src/terraform/awsVpc.ts +++ b/backend/src/terraform/awsVpc.ts @@ -1,27 +1,49 @@ import {jsonRoot} from "./util"; import {Resource} from "./resource"; +import {AwsSubnet} from "./awsSubnet"; +import {AwsInternetGateway} from "./AwsInternetGateway"; export interface AwsVpc { cidr_block: string; + internet: boolean; } export class AwsVpc extends Resource implements AwsVpc { constructor( cidr_block: string, + internetGateway: boolean, id: string, autoIam?: boolean, name?: string ) { super(id, "awsVpc", autoIam, name); this.cidr_block = cidr_block; + this.internet = internetGateway; } //Returns an array of resource blocks toJSON() { - return [ + let json = [ + new AwsSubnet( + this.id, + this.cidr_block, + false, + `${this.id}_subnet` + ).toJSON(), jsonRoot("aws_vpc", this.id, { cidr_block: this.cidr_block }) ]; + if (this.internet) { + json = [ + ...json, + new AwsInternetGateway( + `${this.id}_internetgateway`, + this.id + ).toJSON() + ]; + } + + return json; } } From 2cdf1e89b87c393dc6de69ee9bef52a5e7a2ba2a Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sat, 5 Mar 2022 14:40:14 -0800 Subject: [PATCH 02/19] feat(backend): Route table --- backend/src/terraform/AwsRouteTable.ts | 31 ++++++++++++++++++++++++++ backend/src/types/terraform.ts | 8 +++++++ 2 files changed, 39 insertions(+) create mode 100644 backend/src/terraform/AwsRouteTable.ts diff --git a/backend/src/terraform/AwsRouteTable.ts b/backend/src/terraform/AwsRouteTable.ts new file mode 100644 index 0000000..f7facca --- /dev/null +++ b/backend/src/terraform/AwsRouteTable.ts @@ -0,0 +1,31 @@ +import {jsonRoot} from "./util"; +import {Resource} from "./resource"; +import {AwsRoute} from "../types/terraform"; + +export interface AwsRouteTable { + vpc: string; + routes: AwsRoute[]; +} +export class AwsRouteTable + extends Resource + implements AwsRouteTable +{ + constructor( + id: string, + vpc: string, + routes: AwsRoute[] = [], + name?: string + ) { + super(id, "AwsRouteTable", false, name); + this.vpc = vpc; + this.routes = routes; + } + + //Returns a resource block + toJSON() { + return jsonRoot("aws_default_route_table", this.id, { + default_route_table_id: `\${aws_vpc.${this.vpc}.default_route_table_id}`, + route: this.routes + }); + } +} diff --git a/backend/src/types/terraform.ts b/backend/src/types/terraform.ts index 3e27c64..45a133c 100644 --- a/backend/src/types/terraform.ts +++ b/backend/src/types/terraform.ts @@ -247,6 +247,14 @@ export interface TerraformJson { resource: Record[]; } +export interface AwsRoute { + gateway_id: string; + cidr_block: string; + ipv6_cidr_block?: string; + egress_only_gateway_id?: string; + instance_id?: string; //ec2 +} + // ----------------------------Terraform Root-------------------------------- // export interface Terraform { From 0e501d205b8ab874b7cd0a93beed8e16cb7c65c4 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sat, 5 Mar 2022 15:31:39 -0800 Subject: [PATCH 03/19] feat(backend): Security Group --- backend/src/terraform/AwsSecurityGroup.ts | 33 +++++++++++++++++++++++ backend/src/terraform/awsVpc.ts | 4 ++- backend/src/types/terraform.ts | 11 +++++--- 3 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 backend/src/terraform/AwsSecurityGroup.ts diff --git a/backend/src/terraform/AwsSecurityGroup.ts b/backend/src/terraform/AwsSecurityGroup.ts new file mode 100644 index 0000000..7fbe490 --- /dev/null +++ b/backend/src/terraform/AwsSecurityGroup.ts @@ -0,0 +1,33 @@ +import {jsonRoot} from "./util"; +import {Resource} from "./resource"; +import {Firewall} from "../types/terraform"; + +export interface AwsSecurityGroup { + vpc: string; + firewalls: Firewall[]; +} +export class AwsSecurityGroup + extends Resource + implements AwsSecurityGroup +{ + constructor( + id: string, + vpc: string, + firewalls: Firewall[] = [], + name?: string + ) { + super(id, "AwsSecurityGroup", false, name); + this.vpc = vpc; + this.firewalls = firewalls; + } + + //Returns a resource block + toJSON() { + return jsonRoot("aws_security_group", this.id, { + vpc_id: `\${aws_vpc.${this.vpc}.id}`, + name: this.name, + ingress: this.firewalls.filter(f => f.type === "ingress"), + egress: this.firewalls.filter(f => f.type === "egress") + }); + } +} diff --git a/backend/src/terraform/awsVpc.ts b/backend/src/terraform/awsVpc.ts index 08d74d5..939d8c1 100644 --- a/backend/src/terraform/awsVpc.ts +++ b/backend/src/terraform/awsVpc.ts @@ -2,6 +2,7 @@ import {jsonRoot} from "./util"; import {Resource} from "./resource"; import {AwsSubnet} from "./awsSubnet"; import {AwsInternetGateway} from "./AwsInternetGateway"; +import {AwsRouteTable} from "./AwsRouteTable"; export interface AwsVpc { cidr_block: string; @@ -40,7 +41,8 @@ export class AwsVpc extends Resource implements AwsVpc { new AwsInternetGateway( `${this.id}_internetgateway`, this.id - ).toJSON() + ).toJSON(), + new AwsRouteTable(`${this.id}_routetable`, this.id).toJSON() ]; } diff --git a/backend/src/types/terraform.ts b/backend/src/types/terraform.ts index 45a133c..c4d8a09 100644 --- a/backend/src/types/terraform.ts +++ b/backend/src/types/terraform.ts @@ -250,9 +250,14 @@ export interface TerraformJson { export interface AwsRoute { gateway_id: string; cidr_block: string; - ipv6_cidr_block?: string; - egress_only_gateway_id?: string; - instance_id?: string; //ec2 +} + +export interface Firewall { + type: "egress" | "ingress"; + from_port: number | "icmp"; + to_port: number | "icmp"; + protocol: string; + cidr_blocks?: string[]; } // ----------------------------Terraform Root-------------------------------- // From 504d4d4607f4394c9ed76b91b3b548d2ae0ec083 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sat, 5 Mar 2022 16:11:08 -0800 Subject: [PATCH 04/19] feat(backend): Ec2 security and networking adjustments --- backend/src/terraform/AwsRouteTable.ts | 13 ++++++++++++- backend/src/terraform/ec2.ts | 26 ++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/backend/src/terraform/AwsRouteTable.ts b/backend/src/terraform/AwsRouteTable.ts index f7facca..30c4110 100644 --- a/backend/src/terraform/AwsRouteTable.ts +++ b/backend/src/terraform/AwsRouteTable.ts @@ -13,11 +13,22 @@ export class AwsRouteTable constructor( id: string, vpc: string, - routes: AwsRoute[] = [], + routes: AwsRoute[] | string, name?: string ) { super(id, "AwsRouteTable", false, name); this.vpc = vpc; + + if (!Array.isArray(routes)) { + //If an internet gateway is provided, + //map it to a global cidr block route + routes = [ + { + cidr_block: "0.0.0.0/0", + gateway_id: `\${aws_internet_gateway.${routes}.id}` + } + ]; + } this.routes = routes; } diff --git a/backend/src/terraform/ec2.ts b/backend/src/terraform/ec2.ts index 102e48f..fc7c841 100644 --- a/backend/src/terraform/ec2.ts +++ b/backend/src/terraform/ec2.ts @@ -1,21 +1,28 @@ import {ec2InstanceType, amiType, TerraformJson} from "../types/terraform"; import {jsonRoot} from "./util"; import {ResourceWithIam} from "./resource"; +import {Eip} from "./Eip"; export interface Ec2 { ami: amiType; instance_type: ec2InstanceType; + eip: boolean; + subnet?: string; } export class Ec2 extends ResourceWithIam implements Ec2 { constructor( ami: amiType, instance_type: ec2InstanceType, id: string, - autoIam?: boolean + autoIam?: boolean, + eip?: boolean, + subnet?: string ) { super(id, "Ec2", autoIam); this.ami = ami; this.instance_type = instance_type; + this.eip = eip ?? false; + this.subnet = subnet; } //Returns a resource block @@ -38,7 +45,22 @@ export class Ec2 extends ResourceWithIam implements Ec2 { ]; } - return jsonRoot("aws_instance", this.id, json); + if (this.subnet) { + json.subnet_id = `\${aws_subnet.${this.subnet}.id}`; + } + + let output = [jsonRoot("aws_instance", this.id, json)]; + + if (this.eip) { + output = [ + ...output, + + //TODO: Check for VPC + new Eip(`${this.id}_eip`, this.id, false) + ]; + } + + return output; } static latestAmiMap: Record = { From 25da783cea0fb3a098fa7d96b52810116a71c413 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sat, 5 Mar 2022 16:11:29 -0800 Subject: [PATCH 05/19] feat(backend): Elastic IP support --- backend/src/terraform/Eip.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 backend/src/terraform/Eip.ts diff --git a/backend/src/terraform/Eip.ts b/backend/src/terraform/Eip.ts new file mode 100644 index 0000000..4345d88 --- /dev/null +++ b/backend/src/terraform/Eip.ts @@ -0,0 +1,22 @@ +import {jsonRoot} from "./util"; +import {Resource} from "./resource"; + +export interface Eip { + instance: string; + vpc: boolean; +} +export class Eip extends Resource implements Eip { + constructor(id: string, instance: string, vpc: boolean) { + super(id, "Eip"); + this.instance = instance; + this.vpc = vpc; + } + + //Returns a resource block + toJSON() { + return jsonRoot("aws_eip", this.id, { + instance: `\${aws_instance.${this.instance}.id}`, + vpc: this.vpc + }); + } +} From fab1d646a509cc20344c3f3635655abeff081852 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sat, 5 Mar 2022 16:12:43 -0800 Subject: [PATCH 06/19] chore(backend): Linting --- backend/src/terraform/iamRole.ts | 1 - backend/src/terraform/s3.ts | 1 - backend/src/terraform/terraform.ts | 1 - backend/src/types/terraform.ts | 1 - 4 files changed, 4 deletions(-) diff --git a/backend/src/terraform/iamRole.ts b/backend/src/terraform/iamRole.ts index 61cc64e..f4d49cb 100644 --- a/backend/src/terraform/iamRole.ts +++ b/backend/src/terraform/iamRole.ts @@ -1,6 +1,5 @@ import {jsonRoot} from "./util"; import {Resource} from "./resource"; -import {arr} from "../util"; export interface IamRole {} export class IamRole extends Resource implements IamRole { diff --git a/backend/src/terraform/s3.ts b/backend/src/terraform/s3.ts index fa4d485..42d3883 100644 --- a/backend/src/terraform/s3.ts +++ b/backend/src/terraform/s3.ts @@ -1,4 +1,3 @@ -import {acl} from "../types/terraform"; import {jsonRoot} from "./util"; import {ResourceWithIam} from "./resource"; diff --git a/backend/src/terraform/terraform.ts b/backend/src/terraform/terraform.ts index 706509c..87d52e1 100644 --- a/backend/src/terraform/terraform.ts +++ b/backend/src/terraform/terraform.ts @@ -13,7 +13,6 @@ import {GoogleProvider} from "./googleProvider"; import {namedDestructure} from "./util"; import {arr} from "../util"; import {IamUserForId} from "./awsIamUser"; -import {ResourceWithIam} from "./resource"; export const terraformBlock = ( providers: NamedRequiredProvider[] | NamedRequiredProvider, diff --git a/backend/src/types/terraform.ts b/backend/src/types/terraform.ts index c4d8a09..d4e0dce 100644 --- a/backend/src/types/terraform.ts +++ b/backend/src/types/terraform.ts @@ -10,7 +10,6 @@ import {DatabaseModel, generateSchemaInternals} from "./database"; import {arr} from "../util"; import {IamUser} from "../terraform/awsIamUser"; import {GlacierVault} from "../terraform/glacierVault"; -import {Resource} from "../terraform/resource"; import {lambdaFunction} from "../terraform/lambdaFunction"; // ---------------------------------Variable---------------------------------- // From 0ec7fab7be4676161b660b75adbb398e5cf2aba5 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 07:52:29 -0800 Subject: [PATCH 07/19] chore: Small tweaks and fixes --- backend/src/index.ts | 40 +++++++++++++++++--- backend/src/terraform/AwsRouteTable.ts | 18 +-------- backend/src/terraform/awsSubnet.ts | 2 +- backend/src/terraform/awsVpc.ts | 21 +++++++---- backend/src/terraform/resource.ts | 2 +- backend/src/types/terraform.ts | 52 +++++++++++++++++++++++++- 6 files changed, 102 insertions(+), 33 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index a84c01b..c79fbe3 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -15,11 +15,41 @@ server.serve("/about"); server.serve("/contact"); server.route("/", mainRouter); -// import {testToFileAws} from "./util"; -// import {Ec2} from "./terraform/ec2"; -// testToFileAws("/home/brennan/aws_test/devxp.tf.json", [ -// new Ec2("AUTO_UBUNTU", "t2.medium", "myinstance", true) -// ]); +import {testToFileAws} from "./util"; +import {Ec2} from "./terraform/ec2"; +import {AwsVpc} from "./terraform/awsVpc"; +import {AwsSecurityGroup} from "./terraform/AwsSecurityGroup"; + +const vpc = "my_vpc_for_devxp"; +const cidr = "10.0.0.0/24"; + +testToFileAws("/home/brennan/aws_test/devxp.tf.json", [ + new Ec2("AUTO_UBUNTU", "t2.medium", "myinstance", true, true), + new AwsVpc(cidr, true, vpc), + new AwsSecurityGroup(`securitygroup_for_devp`, vpc, [ + { + type: "ingress", + from_port: 433, + to_port: 433, + protocol: "tcp", + cidr_blocks: [cidr] + }, + { + type: "ingress", + from_port: 80, + to_port: 80, + protocol: "tcp", + cidr_blocks: [cidr] + }, + { + type: "egress", + from_port: 0, + to_port: 0, + protocol: "-1", + cidr_blocks: ["0.0.0.0/0"] + } + ]) +]); mongoose.connection.on( "error", diff --git a/backend/src/terraform/AwsRouteTable.ts b/backend/src/terraform/AwsRouteTable.ts index 30c4110..c2051bb 100644 --- a/backend/src/terraform/AwsRouteTable.ts +++ b/backend/src/terraform/AwsRouteTable.ts @@ -10,25 +10,9 @@ export class AwsRouteTable extends Resource implements AwsRouteTable { - constructor( - id: string, - vpc: string, - routes: AwsRoute[] | string, - name?: string - ) { + constructor(id: string, vpc: string, routes: AwsRoute[], name?: string) { super(id, "AwsRouteTable", false, name); this.vpc = vpc; - - if (!Array.isArray(routes)) { - //If an internet gateway is provided, - //map it to a global cidr block route - routes = [ - { - cidr_block: "0.0.0.0/0", - gateway_id: `\${aws_internet_gateway.${routes}.id}` - } - ]; - } this.routes = routes; } diff --git a/backend/src/terraform/awsSubnet.ts b/backend/src/terraform/awsSubnet.ts index c2ea3a9..443c9ab 100644 --- a/backend/src/terraform/awsSubnet.ts +++ b/backend/src/terraform/awsSubnet.ts @@ -14,7 +14,7 @@ export class AwsSubnet extends Resource implements AwsSubnet { cidr_block: string, map_public_ip_on_launch: boolean, id: string, - availability_zone = "us-west-2", + availability_zone = "us-west-2a", autoIam?: boolean, name?: string ) { diff --git a/backend/src/terraform/awsVpc.ts b/backend/src/terraform/awsVpc.ts index 939d8c1..ac8501d 100644 --- a/backend/src/terraform/awsVpc.ts +++ b/backend/src/terraform/awsVpc.ts @@ -3,6 +3,7 @@ import {Resource} from "./resource"; import {AwsSubnet} from "./awsSubnet"; import {AwsInternetGateway} from "./AwsInternetGateway"; import {AwsRouteTable} from "./AwsRouteTable"; +import {AwsRoute} from "../types/terraform"; export interface AwsVpc { cidr_block: string; @@ -12,14 +13,14 @@ export interface AwsVpc { export class AwsVpc extends Resource implements AwsVpc { constructor( cidr_block: string, - internetGateway: boolean, + internet: boolean, id: string, autoIam?: boolean, name?: string ) { super(id, "awsVpc", autoIam, name); this.cidr_block = cidr_block; - this.internet = internetGateway; + this.internet = internet; } //Returns an array of resource blocks @@ -36,16 +37,20 @@ export class AwsVpc extends Resource implements AwsVpc { }) ]; if (this.internet) { + const gatewayId = `${this.id}_internetgateway`; + json = [ ...json, - new AwsInternetGateway( - `${this.id}_internetgateway`, - this.id - ).toJSON(), - new AwsRouteTable(`${this.id}_routetable`, this.id).toJSON() + new AwsInternetGateway(gatewayId, this.id).toJSON(), + new AwsRouteTable(`${this.id}_routetable`, this.id, [ + { + cidr_block: this.cidr_block, + gateway_id: gatewayId + } + ]).toJSON() ]; } - return json; + return json.flat(); } } diff --git a/backend/src/terraform/resource.ts b/backend/src/terraform/resource.ts index 346ba3d..9c274ca 100644 --- a/backend/src/terraform/resource.ts +++ b/backend/src/terraform/resource.ts @@ -60,7 +60,7 @@ export abstract class ResourceWithIam extends Resource { json = super.postProcess(json); if (this.autoIam) { - json.data = [...json.data, this.toPolicyDocument()]; + json.data = [...json.data, this.toPolicyDocument()].flat(); } return json; } diff --git a/backend/src/types/terraform.ts b/backend/src/types/terraform.ts index d4e0dce..63c0f7d 100644 --- a/backend/src/types/terraform.ts +++ b/backend/src/types/terraform.ts @@ -11,6 +11,13 @@ import {arr} from "../util"; import {IamUser} from "../terraform/awsIamUser"; import {GlacierVault} from "../terraform/glacierVault"; import {lambdaFunction} from "../terraform/lambdaFunction"; +import {AwsVpc} from "../terraform/awsVpc"; +import {AwsInternetGateway} from "../terraform/AwsInternetGateway"; +import {AwsRouteTable} from "../terraform/AwsRouteTable"; +import {AwsSecurityGroup} from "../terraform/AwsSecurityGroup"; +import {Eip} from "../terraform/Eip"; +import {SnsTopic} from "../terraform/awsSnsTopic"; +import {AwsSubnet} from "../terraform/awsSubnet"; // ---------------------------------Variable---------------------------------- // export type VariableType = @@ -231,7 +238,14 @@ export type TerraformResource = | S3 | IamUser | GlacierVault - | lambdaFunction; + | lambdaFunction + | AwsVpc + | AwsInternetGateway + | AwsRouteTable + | AwsSecurityGroup + | SnsTopic + | AwsSubnet + | Eip; export interface PolicyStatement { actions: string[]; @@ -259,6 +273,42 @@ export interface Firewall { cidr_blocks?: string[]; } +/* + +//TODO: Refactor away from using attributes as blocks +//https://stackoverflow.com/questions/69079945/terraform-inappropriate-value-for-attribute-ingress-while-creating-sg +export const AwsRouteWithDefaults = ( + cidr_block: string, + id: string +) => ({ + gateway_id: id, + cidr_block: cidr_block, + egress_only_gateway_id: "", + instance_id: "", + ipv6_cidr_block: "", + nat_gateway_id: "", + network_interface_id: "", + transit_gateway_id: "", + vpc_endpoint_id: "", + vpc_peering_connection_id: "", + destination_prefix_list_id: "" +}) + +//TODO: Refactor away from using attributes as blocks +//https://stackoverflow.com/questions/69079945/terraform-inappropriate-value-for-attribute-ingress-while-creating-sg +export const FirewallWithDefaults = (firewall: Firewall) => ({ + from_port: firewall.from_port, + to_port: firewall.to_port, + protocol: firewall.protocol, + cidr_blocks: firewall.cidr_blocks, + description: "", + ipv6_cidr_blocks: ["::/0"], + prefix_list_ids: [], + security_groups: [], + self: false +}) +*/ + // ----------------------------Terraform Root-------------------------------- // export interface Terraform { From e60a2da75224f9c4b7b2c7d2aafd9b9295714601 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 11:26:00 -0800 Subject: [PATCH 08/19] feat(backend): Support for output to .tf --- backend/src/index.ts | 2 +- backend/src/terraform/AwsSecurityGroup.ts | 13 +++- backend/src/terraform/awsVpc.ts | 10 ++- backend/src/util.ts | 87 ++++++++++++++++++++++- 4 files changed, 106 insertions(+), 6 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index c79fbe3..cd7da5c 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -23,7 +23,7 @@ import {AwsSecurityGroup} from "./terraform/AwsSecurityGroup"; const vpc = "my_vpc_for_devxp"; const cidr = "10.0.0.0/24"; -testToFileAws("/home/brennan/aws_test/devxp.tf.json", [ +testToFileAws("/home/brennan/aws_test/devxp.tf", [ new Ec2("AUTO_UBUNTU", "t2.medium", "myinstance", true, true), new AwsVpc(cidr, true, vpc), new AwsSecurityGroup(`securitygroup_for_devp`, vpc, [ diff --git a/backend/src/terraform/AwsSecurityGroup.ts b/backend/src/terraform/AwsSecurityGroup.ts index 7fbe490..aa29773 100644 --- a/backend/src/terraform/AwsSecurityGroup.ts +++ b/backend/src/terraform/AwsSecurityGroup.ts @@ -2,6 +2,11 @@ import {jsonRoot} from "./util"; import {Resource} from "./resource"; import {Firewall} from "../types/terraform"; +const removeType = (f: any) => { + delete f.type; + return f; +}; + export interface AwsSecurityGroup { vpc: string; firewalls: Firewall[]; @@ -26,8 +31,12 @@ export class AwsSecurityGroup return jsonRoot("aws_security_group", this.id, { vpc_id: `\${aws_vpc.${this.vpc}.id}`, name: this.name, - ingress: this.firewalls.filter(f => f.type === "ingress"), - egress: this.firewalls.filter(f => f.type === "egress") + ingress: this.firewalls + .filter(f => f.type === "ingress") + .map(removeType), + egress: this.firewalls + .filter(f => f.type === "egress") + .map(removeType) }); } } diff --git a/backend/src/terraform/awsVpc.ts b/backend/src/terraform/awsVpc.ts index ac8501d..75d8f5c 100644 --- a/backend/src/terraform/awsVpc.ts +++ b/backend/src/terraform/awsVpc.ts @@ -44,8 +44,14 @@ export class AwsVpc extends Resource implements AwsVpc { new AwsInternetGateway(gatewayId, this.id).toJSON(), new AwsRouteTable(`${this.id}_routetable`, this.id, [ { - cidr_block: this.cidr_block, - gateway_id: gatewayId + //cidr_block: this.cidr_block, + //TODO: ALlow this to be configurable + //This is the collection of EXTERNAL ips which + //our INTERNAL resources are allowed to access + //ex a database should only be able to access an Ec2 + //instance, not the internet + cidr_block: "0.0.0.0/0", + gateway_id: `\${aws_internet_gateway.${gatewayId}.id}` } ]).toJSON() ]; diff --git a/backend/src/util.ts b/backend/src/util.ts index d07b25f..11dd467 100644 --- a/backend/src/util.ts +++ b/backend/src/util.ts @@ -8,6 +8,9 @@ import { TerraformResource } from "./types/terraform"; +// @ts-ignore +import HCL from "js-hcl-parser"; + export const arr = (data: T | T[]) => (Array.isArray(data) ? data : [data]); export const testToFile = ( @@ -22,7 +25,12 @@ export const testToFile = ( resources ); - fs.writeFileSync(filename, JSON.stringify(root, null, 2), { + /* + fs.writeFileSync(`${filename}.json`, JSON.stringify(root, null, 2), { + flag: "w" + }); + */ + fs.writeFileSync(filename, jsonToHcl(root), { flag: "w" }); }; @@ -30,3 +38,80 @@ export const testToFileAws = ( filename: string, resources: TerraformResource[] = [] ) => testToFile(filename, new AwsProvider(), new NamedAwsBackend(), resources); + +export const jsonToHcl = (json: string | Record) => { + if (typeof json !== "string") { + json = JSON.stringify(json, null, 2); + } + let hcl: string = HCL.stringify(json); + + //Unpack the opening resource and data block + hcl = hcl.replace( + /"(resource|data)" = {\n {2}"([^"]+)" = {\n {4}"([^"]+)" = {/g, + (_match, $1, $2, $3) => `${$1} "${$2}" "${$3}" {` + ); + + //Remove the hanging closing tags + hcl = hcl.replace(/ {4}}\n {2}}/g, ""); + + //Unpack the opening provider + hcl = hcl.replace( + /"provider" = {\n {2}"([^"]+)" = {/g, + (_match, $1) => `provider "${$1}" {` + ); + + //Remove the hanging closing tags + hcl = hcl.replace(/ {2}}\n}/g, "}"); + + //Formatting + hcl = hcl.replace(/\n\n/g, "\n"); + hcl = hcl.replace(/^}$/gm, "}\n"); + + //Unpack references + hcl = hcl.replace(/"\${([^}]+)}"/g, (_match, $1) => $1); + + //Remove variable quotes + hcl = hcl.replace(/"([^"]+)" = /g, (_match, $1) => `${$1} = `); + + //Fix up required providers block + hcl = hcl.replace( + /terraform = {\n {2}"([^"]+)" "([^"]+)"([^}]+)}/g, + (_match, $1, $2, $3) => + `terraform {\n ${$1} {\n ${$2} = ${$3}}\n}\n}` + ); + + //Remove incorrect block as attribute styles + hcl = hcl.replace( + /(lifecycle|ingress|egress|statement|filter|route) = {/g, + (_match, $1) => `${$1} {` + ); + + //Remove incorrect ignore quotes + hcl = hcl.replace(/(ignore_changes = \[[^\]]+\])/g, (_match, $1) => + $1.replace(/"/g, "") + ); + + /* + //Merge duplicate blocks into arrays + let matches: Record = {} + hcl = hcl.replace(/([a-zA-Z0-9_-]+) = ({[^}]+})/g, (_match, $1, $2) => { + + matches[$1] = [...(matches[$1] ?? []), $2]; + return `MARKER_${$1}`;//`${$1} = [${$2}]` + }); + + Object.keys(matches).forEach(key => { + let json; + if(matches[key].length === 1){ + json = `${key} = ${matches[key]}`; + } + else{ + json = `${key} = [${matches[key].reduce((acc, obj) => `${acc}, ${obj}`)}]`; + } + hcl = hcl.replace(new RegExp(`MARKER_${key}`), json); + hcl = hcl.replace(new RegExp(`MARKER_${key}`, "g"), ""); + }); + */ + + return hcl; +}; From 26032676d066e8c7997226b2db23a4663371b85e Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 11:26:11 -0800 Subject: [PATCH 09/19] fix: Further networking bugfixes --- backend/src/index.ts | 13 +++++++++++-- backend/src/terraform/ec2.ts | 13 ++++++++++++- backend/src/util.ts | 1 + 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index cd7da5c..55f03f5 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -21,12 +21,21 @@ import {AwsVpc} from "./terraform/awsVpc"; import {AwsSecurityGroup} from "./terraform/AwsSecurityGroup"; const vpc = "my_vpc_for_devxp"; +const securityGroup = "securitygroup_for_devp"; const cidr = "10.0.0.0/24"; testToFileAws("/home/brennan/aws_test/devxp.tf", [ - new Ec2("AUTO_UBUNTU", "t2.medium", "myinstance", true, true), + new Ec2( + "AUTO_UBUNTU", + "t2.medium", + "myinstance", + false, + true, + `${vpc}_subnet`, + securityGroup + ), new AwsVpc(cidr, true, vpc), - new AwsSecurityGroup(`securitygroup_for_devp`, vpc, [ + new AwsSecurityGroup(securityGroup, vpc, [ { type: "ingress", from_port: 433, diff --git a/backend/src/terraform/ec2.ts b/backend/src/terraform/ec2.ts index fc7c841..d6d3917 100644 --- a/backend/src/terraform/ec2.ts +++ b/backend/src/terraform/ec2.ts @@ -2,12 +2,14 @@ import {ec2InstanceType, amiType, TerraformJson} from "../types/terraform"; import {jsonRoot} from "./util"; import {ResourceWithIam} from "./resource"; import {Eip} from "./Eip"; +import {arr} from "../util"; export interface Ec2 { ami: amiType; instance_type: ec2InstanceType; eip: boolean; subnet?: string; + securityGroups?: string[] | string; } export class Ec2 extends ResourceWithIam implements Ec2 { constructor( @@ -16,13 +18,15 @@ export class Ec2 extends ResourceWithIam implements Ec2 { id: string, autoIam?: boolean, eip?: boolean, - subnet?: string + subnet?: string, + securityGroups?: string[] | string ) { super(id, "Ec2", autoIam); this.ami = ami; this.instance_type = instance_type; this.eip = eip ?? false; this.subnet = subnet; + this.securityGroups = securityGroups; } //Returns a resource block @@ -47,6 +51,13 @@ export class Ec2 extends ResourceWithIam implements Ec2 { if (this.subnet) { json.subnet_id = `\${aws_subnet.${this.subnet}.id}`; + json.associate_public_ip_address = true; + } + + if (this.securityGroups) { + json.vpc_security_group_ids = arr(this.securityGroups).map( + g => `\${aws_security_group.${g}.id}` + ); } let output = [jsonRoot("aws_instance", this.id, json)]; diff --git a/backend/src/util.ts b/backend/src/util.ts index 11dd467..2c47b5e 100644 --- a/backend/src/util.ts +++ b/backend/src/util.ts @@ -8,6 +8,7 @@ import { TerraformResource } from "./types/terraform"; +/* eslint-disable @typescript-eslint/ban-ts-comment */ // @ts-ignore import HCL from "js-hcl-parser"; From f62d9cf4510f431d42c8a816ca0f7a6bf7368a5c Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 12:45:47 -0800 Subject: [PATCH 10/19] feat(backend): SSH Networking prefab for ec2 --- backend/src/index.ts | 51 ++++------------- backend/src/terraform/README.md | 48 ++++++++++++++++ backend/src/terraform/awsVpc.ts | 1 - backend/src/terraform/ec2.ts | 22 +++++--- backend/src/terraform/prefab.ts | 98 +++++++++++++++++++++++++++++++++ backend/src/util.ts | 3 + 6 files changed, 176 insertions(+), 47 deletions(-) create mode 100644 backend/src/terraform/prefab.ts diff --git a/backend/src/index.ts b/backend/src/index.ts index 55f03f5..855b2d7 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -17,48 +17,21 @@ server.route("/", mainRouter); import {testToFileAws} from "./util"; import {Ec2} from "./terraform/ec2"; -import {AwsVpc} from "./terraform/awsVpc"; -import {AwsSecurityGroup} from "./terraform/AwsSecurityGroup"; +import {prefabNetwork} from "./terraform/prefab"; -const vpc = "my_vpc_for_devxp"; -const securityGroup = "securitygroup_for_devp"; -const cidr = "10.0.0.0/24"; - -testToFileAws("/home/brennan/aws_test/devxp.tf", [ - new Ec2( - "AUTO_UBUNTU", - "t2.medium", - "myinstance", - false, - true, - `${vpc}_subnet`, - securityGroup - ), - new AwsVpc(cidr, true, vpc), - new AwsSecurityGroup(securityGroup, vpc, [ - { - type: "ingress", - from_port: 433, - to_port: 433, - protocol: "tcp", - cidr_blocks: [cidr] - }, +testToFileAws( + "/home/brennan/aws_test/devxp.tf", + prefabNetwork( + [ + new Ec2("AUTO_UBUNTU", "t2.small", "instance_a", true), + new Ec2("AUTO_UBUNTU", "t2.micro", "instance_b", true) + ], { - type: "ingress", - from_port: 80, - to_port: 80, - protocol: "tcp", - cidr_blocks: [cidr] - }, - { - type: "egress", - from_port: 0, - to_port: 0, - protocol: "-1", - cidr_blocks: ["0.0.0.0/0"] + ssh: true, + allEgress: true } - ]) -]); + ) +); mongoose.connection.on( "error", diff --git a/backend/src/terraform/README.md b/backend/src/terraform/README.md index e7ed645..ef78942 100644 --- a/backend/src/terraform/README.md +++ b/backend/src/terraform/README.md @@ -1,3 +1,51 @@ +```ts +import {testToFileAws} from "./util"; +import {Ec2} from "./terraform/ec2"; +import {AwsVpc} from "./terraform/awsVpc"; +import {AwsSecurityGroup} from "./terraform/AwsSecurityGroup"; + +const vpc = "my_vpc_for_devxp"; +const securityGroup = "securitygroup_for_devp"; +const cidr = "10.0.0.0/24"; + +testToFileAws("/home/brennan/aws_test/devxp.tf", [ + new Ec2( + "AUTO_UBUNTU", + "t2.medium", + "myinstance", + false, + 2, + `${vpc}_subnet`, + securityGroup + ), + new AwsVpc(cidr, true, vpc), + new AwsSecurityGroup(securityGroup, vpc, [ + { + type: "ingress", + from_port: 433, + to_port: 433, + protocol: "tcp", + cidr_blocks: [cidr] + }, + { + type: "ingress", + from_port: 80, + to_port: 80, + protocol: "tcp", + cidr_blocks: [cidr] + }, + { + type: "egress", + from_port: 0, + to_port: 0, + protocol: "-1", + cidr_blocks: ["0.0.0.0/0"] + } + ]) +]); +``` + + ```hcl terraform { diff --git a/backend/src/terraform/awsVpc.ts b/backend/src/terraform/awsVpc.ts index 75d8f5c..d657c0f 100644 --- a/backend/src/terraform/awsVpc.ts +++ b/backend/src/terraform/awsVpc.ts @@ -3,7 +3,6 @@ import {Resource} from "./resource"; import {AwsSubnet} from "./awsSubnet"; import {AwsInternetGateway} from "./AwsInternetGateway"; import {AwsRouteTable} from "./AwsRouteTable"; -import {AwsRoute} from "../types/terraform"; export interface AwsVpc { cidr_block: string; diff --git a/backend/src/terraform/ec2.ts b/backend/src/terraform/ec2.ts index d6d3917..3dc2e9e 100644 --- a/backend/src/terraform/ec2.ts +++ b/backend/src/terraform/ec2.ts @@ -7,7 +7,7 @@ import {arr} from "../util"; export interface Ec2 { ami: amiType; instance_type: ec2InstanceType; - eip: boolean; + eip: number; subnet?: string; securityGroups?: string[] | string; } @@ -17,14 +17,14 @@ export class Ec2 extends ResourceWithIam implements Ec2 { instance_type: ec2InstanceType, id: string, autoIam?: boolean, - eip?: boolean, + eip?: number, subnet?: string, securityGroups?: string[] | string ) { super(id, "Ec2", autoIam); this.ami = ami; this.instance_type = instance_type; - this.eip = eip ?? false; + this.eip = eip ?? 0; this.subnet = subnet; this.securityGroups = securityGroups; } @@ -62,12 +62,11 @@ export class Ec2 extends ResourceWithIam implements Ec2 { let output = [jsonRoot("aws_instance", this.id, json)]; - if (this.eip) { + if (this.eip > 0) { output = [ ...output, - //TODO: Check for VPC - new Eip(`${this.id}_eip`, this.id, false) + new Eip(`${this.id}_eip`, this.id, this.eip === 2) ]; } @@ -88,8 +87,17 @@ export class Ec2 extends ResourceWithIam implements Ec2 { if (/^AUTO_(UBUNTU|WINDOWS|AMAZON)$/.test(this.ami)) { const os = this.ami.slice(5).toLowerCase(); - if ("data" in json && Array.isArray(json.data)) { + for (let i = 0; i < json.data.length; i++) { + if ( + "aws_ami" in json.data[i] && + Array.isArray(json.data[i].aws_ami) && + json.data[i].aws_ami.length > 0 && + `${os}_latest` in json.data[i].aws_ami[0] + ) { + return json; + } + } json.data = [ ...json.data, jsonRoot("aws_ami", `${os}_latest`, { diff --git a/backend/src/terraform/prefab.ts b/backend/src/terraform/prefab.ts new file mode 100644 index 0000000..8931987 --- /dev/null +++ b/backend/src/terraform/prefab.ts @@ -0,0 +1,98 @@ +import {Firewall, TerraformResource} from "../types/terraform"; +import {arr} from "../util"; +import {AwsSecurityGroup} from "./AwsSecurityGroup"; +import {AwsVpc} from "./awsVpc"; +import {Ec2} from "./ec2"; + +export const prefabNetwork = ( + instances: Ec2 | Ec2[], + rules: { + ssh?: boolean; + sshCidr?: string[]; + allEgress?: boolean; + web?: boolean; + webCidr?: string[]; + }, + cidr = "10.0.0.0/24", + vpc = "devxp_allow_ssh_vpc", + securityGroup = "devxp_allow_ssh_security_group" +): TerraformResource[] => { + instances = arr(instances); + + let firewalls: Firewall[] = []; + if (rules.allEgress) { + firewalls = [ + ...firewalls, + { + type: "egress", + from_port: 0, + to_port: 0, + protocol: "-1", + cidr_blocks: ["0.0.0.0/0"] + } + ]; + } + if (rules.ssh) { + firewalls = [ + ...firewalls, + { + type: "ingress", + from_port: 22, + to_port: 22, + protocol: "tcp", + cidr_blocks: rules.sshCidr ?? ["0.0.0.0/0"] + } + ]; + } + if (rules.web) { + firewalls = [ + ...firewalls, + { + type: "ingress", + from_port: 80, + to_port: 80, + protocol: "tcp", + cidr_blocks: rules.webCidr ?? ["0.0.0.0/0"] + }, + { + type: "ingress", + from_port: 433, + to_port: 433, + protocol: "tcp", + cidr_blocks: rules.webCidr ?? ["0.0.0.0/0"] + }, + { + type: "egress", + from_port: 80, + to_port: 80, + protocol: "tcp", + cidr_blocks: rules.webCidr ?? ["0.0.0.0/0"] + }, + { + type: "egress", + from_port: 433, + to_port: 433, + protocol: "tcp", + cidr_blocks: rules.webCidr ?? ["0.0.0.0/0"] + } + ]; + } + + return [ + ...instances.map( + (ec2: Ec2) => + new Ec2( + ec2.ami, + ec2.instance_type, + ec2.id, + ec2.autoIam, + 2, + `${vpc}_subnet`, + securityGroup + ) + ), + + new AwsVpc(cidr, true, vpc), + new AwsSecurityGroup(securityGroup, vpc, firewalls) + ]; +}; diff --git a/backend/src/util.ts b/backend/src/util.ts index 2c47b5e..be244bc 100644 --- a/backend/src/util.ts +++ b/backend/src/util.ts @@ -92,6 +92,9 @@ export const jsonToHcl = (json: string | Record) => { $1.replace(/"/g, "") ); + //Cleanup + hcl = hcl.replace(/}\n}\n}/g, " }\n }\n}"); + /* //Merge duplicate blocks into arrays let matches: Record = {} From cbca676b317029f883bb77c51db9c31f40b3e67d Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 15:11:34 -0800 Subject: [PATCH 11/19] feat(backend): Linking IAM perms from s3 to ec2 --- backend/src/index.ts | 17 +++-- .../src/terraform/AwsIamInstanceProfile.ts | 23 +++++++ .../terraform/awsIamRolePolicyAttachment.ts | 29 ++++++++ backend/src/terraform/ec2.ts | 9 ++- backend/src/terraform/iamRole.ts | 30 ++++++-- backend/src/terraform/lambdaFunction.ts | 5 +- backend/src/terraform/prefab.ts | 68 +++++++++++++++---- backend/src/types/terraform.ts | 8 ++- 8 files changed, 160 insertions(+), 29 deletions(-) create mode 100644 backend/src/terraform/AwsIamInstanceProfile.ts create mode 100644 backend/src/terraform/awsIamRolePolicyAttachment.ts diff --git a/backend/src/index.ts b/backend/src/index.ts index 855b2d7..19e3855 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -18,17 +18,24 @@ server.route("/", mainRouter); import {testToFileAws} from "./util"; import {Ec2} from "./terraform/ec2"; import {prefabNetwork} from "./terraform/prefab"; +import {S3} from "./terraform/s3"; testToFileAws( "/home/brennan/aws_test/devxp.tf", prefabNetwork( - [ - new Ec2("AUTO_UBUNTU", "t2.small", "instance_a", true), - new Ec2("AUTO_UBUNTU", "t2.micro", "instance_b", true) - ], + { + ec2: [ + new Ec2("AUTO_UBUNTU", "t2.micro", "instance_a", false), + new Ec2("AUTO_UBUNTU", "t2.micro", "instance_b", false) + ], + s3: [ + new S3("devxp_test_bucket_a", false, "devxp-test-bucket-a"), + new S3("devxp_test_bucket_b", false, "devxp-test-bucket-b") + ] + }, { ssh: true, - allEgress: true + web: true } ) ); diff --git a/backend/src/terraform/AwsIamInstanceProfile.ts b/backend/src/terraform/AwsIamInstanceProfile.ts new file mode 100644 index 0000000..4a01aca --- /dev/null +++ b/backend/src/terraform/AwsIamInstanceProfile.ts @@ -0,0 +1,23 @@ +import {jsonRoot} from "./util"; +import {Resource} from "./resource"; + +export interface AwsIamInstanceProfile { + role: string; +} +export class AwsIamInstanceProfile + extends Resource + implements AwsIamInstanceProfile +{ + constructor(id: string, role: string, name?: string) { + super(id, "AwsIamInstanceProfile", false, name); + this.role = role; + } + + //Returns an array of resource blocks + toJSON() { + return jsonRoot("aws_iam_instance_profile", this.id, { + name: this.name, + role: `\${aws_iam_role.${this.role}.name}` + }); + } +} diff --git a/backend/src/terraform/awsIamRolePolicyAttachment.ts b/backend/src/terraform/awsIamRolePolicyAttachment.ts new file mode 100644 index 0000000..d6f2cba --- /dev/null +++ b/backend/src/terraform/awsIamRolePolicyAttachment.ts @@ -0,0 +1,29 @@ +import {jsonRoot} from "./util"; +import {Resource} from "./resource"; + +export interface AwsIamRolePolicyAttachment { + role: string; + policy: string; +} +export class AwsIamRolePolicyAttachment + extends Resource + implements AwsIamRolePolicyAttachment +{ + constructor(id: string, policy: string, role: string, name?: string) { + super(id, "AwsIamRolePolicyAttachment", false, name); + + this.policy = policy; + this.role = role; + } + + //Returns an array of resource blocks + toJSON() { + return [ + //The iam user block itself + jsonRoot("aws_iam_role_policy_attachment", this.id, { + policy_arn: `\${aws_iam_policy.${this.policy}.arn}`, + role: `\${aws_iam_role.${this.role}.name}` + }) + ]; + } +} diff --git a/backend/src/terraform/ec2.ts b/backend/src/terraform/ec2.ts index 3dc2e9e..56a62d0 100644 --- a/backend/src/terraform/ec2.ts +++ b/backend/src/terraform/ec2.ts @@ -10,6 +10,7 @@ export interface Ec2 { eip: number; subnet?: string; securityGroups?: string[] | string; + iam_instance_profile?: string; } export class Ec2 extends ResourceWithIam implements Ec2 { constructor( @@ -19,7 +20,8 @@ export class Ec2 extends ResourceWithIam implements Ec2 { autoIam?: boolean, eip?: number, subnet?: string, - securityGroups?: string[] | string + securityGroups?: string[] | string, + iam_instance_profile?: string ) { super(id, "Ec2", autoIam); this.ami = ami; @@ -27,6 +29,7 @@ export class Ec2 extends ResourceWithIam implements Ec2 { this.eip = eip ?? 0; this.subnet = subnet; this.securityGroups = securityGroups; + this.iam_instance_profile = iam_instance_profile; } //Returns a resource block @@ -60,6 +63,10 @@ export class Ec2 extends ResourceWithIam implements Ec2 { ); } + if (this.iam_instance_profile) { + json.iam_instance_profile = `\${aws_iam_instance_profile.${this.iam_instance_profile}.name}`; + } + let output = [jsonRoot("aws_instance", this.id, json)]; if (this.eip > 0) { diff --git a/backend/src/terraform/iamRole.ts b/backend/src/terraform/iamRole.ts index f4d49cb..70eaa09 100644 --- a/backend/src/terraform/iamRole.ts +++ b/backend/src/terraform/iamRole.ts @@ -1,18 +1,36 @@ import {jsonRoot} from "./util"; import {Resource} from "./resource"; -export interface IamRole {} +export interface IamRole { + service: string; +} export class IamRole extends Resource implements IamRole { - constructor(id: string) { - super(id, "IamRole"); + constructor(id: string, service: string, name?: string) { + super(id, "IamRole", false, name); + this.service = service; } //Returns an array of resource blocks toJSON() { return jsonRoot("aws_iam_role", this.id, { - name: this.id, - assume_role_policy: - '{\n "Version": "2012-10-17",\n "Statement": [\n {\n "Action": "sts:AssumeRole",\n "Principal": {\n "Service": "lambda.amazonaws.com"\n },\n "Effect": "Allow",\n "Sid": ""\n }\n ]\n}\n' + name: this.name, + assume_role_policy: JSON.stringify( + { + Version: "2012-10-17", + Statement: [ + { + Action: "sts:AssumeRole", + Principal: { + Service: this.service + }, + Effect: "Allow", + Sid: "" + } + ] + }, + null, + 2 + ) }); } } diff --git a/backend/src/terraform/lambdaFunction.ts b/backend/src/terraform/lambdaFunction.ts index 06b4907..f5d5f23 100644 --- a/backend/src/terraform/lambdaFunction.ts +++ b/backend/src/terraform/lambdaFunction.ts @@ -27,7 +27,10 @@ export class lambdaFunction //Returns an array of resource blocks toJSON() { - const iamRole = new IamRole("iam_for_lambda_" + this.functionName); + const iamRole = new IamRole( + "iam_for_lambda_" + this.functionName, + "lambda.amazonaws.com" + ); return [ iamRole.toJSON(), jsonRoot("aws_lambda_function", this.id, { diff --git a/backend/src/terraform/prefab.ts b/backend/src/terraform/prefab.ts index 8931987..4eb87f5 100644 --- a/backend/src/terraform/prefab.ts +++ b/backend/src/terraform/prefab.ts @@ -1,11 +1,18 @@ import {Firewall, TerraformResource} from "../types/terraform"; import {arr} from "../util"; +import {AwsIamInstanceProfile} from "./AwsIamInstanceProfile"; +import {AwsIamRolePolicyAttachment} from "./awsIamRolePolicyAttachment"; import {AwsSecurityGroup} from "./AwsSecurityGroup"; import {AwsVpc} from "./awsVpc"; import {Ec2} from "./ec2"; +import {IamRole} from "./iamRole"; +import {S3} from "./s3"; export const prefabNetwork = ( - instances: Ec2 | Ec2[], + resources: { + ec2?: Ec2[] | Ec2; + s3?: S3[] | S3; + }, rules: { ssh?: boolean; sshCidr?: string[]; @@ -17,7 +24,46 @@ export const prefabNetwork = ( vpc = "devxp_allow_ssh_vpc", securityGroup = "devxp_allow_ssh_security_group" ): TerraformResource[] => { - instances = arr(instances); + const instances = arr(resources.ec2 ?? []).map( + (ec2: Ec2) => + new Ec2( + ec2.ami, + ec2.instance_type, + ec2.id, + ec2.autoIam, + 2, + `${vpc}_subnet`, + securityGroup + ) + ); + const buckets = arr(resources.s3 ?? []).map( + (bucket: S3) => new S3(bucket.id, true, bucket.name) + ); + const policies = buckets.map(bucket => `${bucket.id}_iam_policy0`); + const iamRoles = instances.map( + ec2 => new IamRole(`${ec2.id}_iam_role`, "ec2.amazonaws.com") + ); + + let attachments: AwsIamRolePolicyAttachment[] = []; + policies.forEach(p => { + iamRoles.forEach(r => { + attachments = [ + ...attachments, + new AwsIamRolePolicyAttachment( + `${r.id}_${p}_attachment`, + p, + r.id + ) + ]; + }); + }); + + const instanceProfiles = iamRoles.map( + r => new AwsIamInstanceProfile(`${r.id}_instance_profile`, r.id) + ); + instanceProfiles.forEach((p, i) => { + instances[i].iam_instance_profile = p.id; + }); let firewalls: Firewall[] = []; if (rules.allEgress) { @@ -79,19 +125,11 @@ export const prefabNetwork = ( } return [ - ...instances.map( - (ec2: Ec2) => - new Ec2( - ec2.ami, - ec2.instance_type, - ec2.id, - ec2.autoIam, - 2, - `${vpc}_subnet`, - securityGroup - ) - ), - + ...instances, + ...buckets, + ...instanceProfiles, + ...iamRoles, + ...attachments, new AwsVpc(cidr, true, vpc), new AwsSecurityGroup(securityGroup, vpc, firewalls) ]; diff --git a/backend/src/types/terraform.ts b/backend/src/types/terraform.ts index 63c0f7d..7bb6ea2 100644 --- a/backend/src/types/terraform.ts +++ b/backend/src/types/terraform.ts @@ -18,6 +18,9 @@ import {AwsSecurityGroup} from "../terraform/AwsSecurityGroup"; import {Eip} from "../terraform/Eip"; import {SnsTopic} from "../terraform/awsSnsTopic"; import {AwsSubnet} from "../terraform/awsSubnet"; +import {AwsIamInstanceProfile} from "../terraform/AwsIamInstanceProfile"; +import {AwsIamRolePolicyAttachment} from "../terraform/awsIamRolePolicyAttachment"; +import {IamRole} from "../terraform/iamRole"; // ---------------------------------Variable---------------------------------- // export type VariableType = @@ -245,7 +248,10 @@ export type TerraformResource = | AwsSecurityGroup | SnsTopic | AwsSubnet - | Eip; + | Eip + | AwsIamInstanceProfile + | AwsIamRolePolicyAttachment + | IamRole; export interface PolicyStatement { actions: string[]; From 3801bc88aee4627ed2b547319bde3a229803a71f Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 19:56:16 -0800 Subject: [PATCH 12/19] feat(backend): Basic prefab network --- backend/src/index.ts | 16 +++-- backend/src/terraform/AwsRoute.ts | 30 ++++++++ backend/src/terraform/AwsRouteTable.ts | 31 ++++++-- backend/src/terraform/AwsVpcEndpoint.ts | 84 ++++++++++++++++++++++ backend/src/terraform/awsVpc.ts | 94 ++++++++++++++++++------- backend/src/terraform/prefab.ts | 61 ++++++++++++---- backend/src/terraform/s3.ts | 36 ++++++++-- backend/src/types/terraform.ts | 6 +- 8 files changed, 300 insertions(+), 58 deletions(-) create mode 100644 backend/src/terraform/AwsRoute.ts create mode 100644 backend/src/terraform/AwsVpcEndpoint.ts diff --git a/backend/src/index.ts b/backend/src/index.ts index 19e3855..6279c61 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -24,18 +24,20 @@ testToFileAws( "/home/brennan/aws_test/devxp.tf", prefabNetwork( { - ec2: [ - new Ec2("AUTO_UBUNTU", "t2.micro", "instance_a", false), - new Ec2("AUTO_UBUNTU", "t2.micro", "instance_b", false) - ], + ec2: [new Ec2("AUTO_UBUNTU", "t2.micro", "instance_a", true)], s3: [ - new S3("devxp_test_bucket_a", false, "devxp-test-bucket-a"), - new S3("devxp_test_bucket_b", false, "devxp-test-bucket-b") + new S3( + "devxp_test_bucket_a", + false, + false, + "devxp-test-bucket-a" + ) ] }, { ssh: true, - web: true + webEgress: true, + webIngress: true } ) ); diff --git a/backend/src/terraform/AwsRoute.ts b/backend/src/terraform/AwsRoute.ts new file mode 100644 index 0000000..d3597da --- /dev/null +++ b/backend/src/terraform/AwsRoute.ts @@ -0,0 +1,30 @@ +import {jsonRoot} from "./util"; +import {Resource} from "./resource"; + +export interface AwsRoute { + route_table_id: string; + cidr_block: string; + gateway_id: string; +} +export class AwsRoute extends Resource implements AwsRoute { + constructor( + id: string, + route_table_id: string, + cidr_block: string, + gateway_id: string + ) { + super(id, "AwsRoute", false); + this.route_table_id = route_table_id; + this.cidr_block = cidr_block; + this.gateway_id = gateway_id; + } + + //Returns a resource block + toJSON() { + return jsonRoot("aws_route", this.id, { + route_table_id: `\${aws_route_table.${this.route_table_id}.id}`, + destination_cidr_block: this.cidr_block, + gateway_id: `\${aws_internet_gateway.${this.gateway_id}.id}` + }); + } +} diff --git a/backend/src/terraform/AwsRouteTable.ts b/backend/src/terraform/AwsRouteTable.ts index c2051bb..d369c5d 100644 --- a/backend/src/terraform/AwsRouteTable.ts +++ b/backend/src/terraform/AwsRouteTable.ts @@ -5,22 +5,43 @@ import {AwsRoute} from "../types/terraform"; export interface AwsRouteTable { vpc: string; routes: AwsRoute[]; + defaultTable: boolean; } export class AwsRouteTable extends Resource implements AwsRouteTable { - constructor(id: string, vpc: string, routes: AwsRoute[], name?: string) { + constructor( + id: string, + vpc: string, + routes: AwsRoute[], + defaultTable = false, + name?: string + ) { super(id, "AwsRouteTable", false, name); this.vpc = vpc; this.routes = routes; + this.defaultTable = defaultTable; } //Returns a resource block toJSON() { - return jsonRoot("aws_default_route_table", this.id, { - default_route_table_id: `\${aws_vpc.${this.vpc}.default_route_table_id}`, - route: this.routes - }); + const resource = this.defaultTable + ? "aws_default_route_table" + : "aws_route_table"; + + const json: Record = {}; + + if (this.routes.length > 0) { + json.route = this.routes; + } + + if (this.defaultTable) { + json.default_route_table_id = `\${aws_vpc.${this.vpc}.default_route_table_id}`; + } else { + json.vpc_id = `\${aws_vpc.${this.vpc}.id}`; + } + + return jsonRoot(resource, this.id, json); } } diff --git a/backend/src/terraform/AwsVpcEndpoint.ts b/backend/src/terraform/AwsVpcEndpoint.ts new file mode 100644 index 0000000..f69e0a3 --- /dev/null +++ b/backend/src/terraform/AwsVpcEndpoint.ts @@ -0,0 +1,84 @@ +import {jsonRoot} from "./util"; +import {Resource} from "./resource"; + +export interface AwsVpcEndpoint { + vpc: string; + service: string; + security_group_ids: string[]; + vpc_endpoint_type: string; + route_table: { + id?: string; + isDefault: boolean; + }; + privateDns: boolean; +} +export class AwsVpcEndpoint + extends Resource + implements AwsVpcEndpoint +{ + constructor( + id: string, + vpc: string, + service: string, + security_group_ids: string[] = [], + route_table: { + id?: string; + isDefault: boolean; + }, + vpc_endpoint_type = "Gateway", + privateDns = false + ) { + super(id, "AwsVpcEndpoint"); + this.vpc = vpc; + this.service = service; + this.security_group_ids = security_group_ids ?? []; + this.vpc_endpoint_type = vpc_endpoint_type; + this.route_table = route_table; + this.privateDns = privateDns; + } + + //Returns a resource block + toJSON() { + const internal: Record = { + vpc_id: `\${aws_vpc.${this.vpc}.id}`, + service_name: this.service, + vpc_endpoint_type: this.vpc_endpoint_type, + private_dns_enabled: this.privateDns ?? false + }; + + if (this.security_group_ids && this.security_group_ids.length > 0) { + internal.security_group_ids = this.security_group_ids.map( + id => `\${aws_security_group.${id}.id}` + ); + } + + let json = [jsonRoot("aws_vpc_endpoint", this.id, internal)]; + + const tableParent = this.route_table.isDefault + ? "aws_default_route_table" + : "aws_route_table"; + if (!this.route_table.id) { + json = [ + ...json, + jsonRoot(tableParent, `${this.id}_route_table`, { + vpc_id: `\${aws_vpc.${this.vpc}.id}` + }) + ]; + this.route_table.id = `${this.id}_route_table`; + } + + json = [ + ...json, + jsonRoot( + "aws_vpc_endpoint_route_table_association", + `${this.id}_route_association`, + { + route_table_id: `\${${tableParent}.${this.route_table.id}.id}`, + vpc_endpoint_id: `\${aws_vpc_endpoint.${this.id}.id}` + } + ) + ]; + + return json; + } +} diff --git a/backend/src/terraform/awsVpc.ts b/backend/src/terraform/awsVpc.ts index d657c0f..d383413 100644 --- a/backend/src/terraform/awsVpc.ts +++ b/backend/src/terraform/awsVpc.ts @@ -3,59 +3,99 @@ import {Resource} from "./resource"; import {AwsSubnet} from "./awsSubnet"; import {AwsInternetGateway} from "./AwsInternetGateway"; import {AwsRouteTable} from "./AwsRouteTable"; +import {AwsRoute} from "./AwsRoute"; export interface AwsVpc { cidr_block: string; - internet: boolean; + private_cidr: string; + public_cidr: string; + privateDns: boolean; } export class AwsVpc extends Resource implements AwsVpc { constructor( cidr_block: string, - internet: boolean, + private_cidr: string, + public_cidr: string, id: string, autoIam?: boolean, - name?: string + name?: string, + privateDns = false ) { super(id, "awsVpc", autoIam, name); this.cidr_block = cidr_block; - this.internet = internet; + this.private_cidr = private_cidr; + this.public_cidr = public_cidr; + this.privateDns = privateDns; } //Returns an array of resource blocks toJSON() { - let json = [ + const gatewayId = `${this.id}_internetgateway`; + const publicRouteTableId = `${this.id}_routetable_pub`; + const privateRouteTableId = `${this.id}_routetable_priv`; + const publicSubetId = `${this.id}_subnet_public`; + const privateSubnetId = `${this.id}_subnet_private`; + + return [ + //PRIVATE new AwsSubnet( this.id, - this.cidr_block, + this.private_cidr, false, - `${this.id}_subnet` + privateSubnetId ).toJSON(), - jsonRoot("aws_vpc", this.id, { - cidr_block: this.cidr_block - }) - ]; - if (this.internet) { - const gatewayId = `${this.id}_internetgateway`; + new AwsRouteTable(privateRouteTableId, this.id, [], false).toJSON(), + jsonRoot( + "aws_route_table_association", + `${this.id}_subnet_private_assoc`, + { + subnet_id: `\${aws_subnet.${privateSubnetId}.id}`, + route_table_id: `\${aws_route_table.${privateRouteTableId}.id}` + } + ), - json = [ - ...json, - new AwsInternetGateway(gatewayId, this.id).toJSON(), - new AwsRouteTable(`${this.id}_routetable`, this.id, [ + //----------------------------------------------------------------// + + //PUBLIC + new AwsSubnet( + this.id, + this.public_cidr, + true, + publicSubetId + ).toJSON(), + new AwsInternetGateway(gatewayId, this.id).toJSON(), + new AwsRouteTable( + publicRouteTableId, + this.id, + [ { - //cidr_block: this.cidr_block, - //TODO: ALlow this to be configurable - //This is the collection of EXTERNAL ips which - //our INTERNAL resources are allowed to access - //ex a database should only be able to access an Ec2 - //instance, not the internet cidr_block: "0.0.0.0/0", gateway_id: `\${aws_internet_gateway.${gatewayId}.id}` } - ]).toJSON() - ]; - } + ], + false + ).toJSON(), + new AwsRoute( + `${this.id}_internet_route`, + publicRouteTableId, + "0.0.0.0/0", + gatewayId + ).toJSON(), + jsonRoot( + "aws_route_table_association", + `${this.id}_subnet_public_assoc`, + { + subnet_id: `\${aws_subnet.${publicSubetId}.id}`, + route_table_id: `\${aws_route_table.${publicRouteTableId}.id}` + } + ), - return json.flat(); + jsonRoot("aws_vpc", this.id, { + cidr_block: this.cidr_block, + enable_dns_support: this.privateDns, + enable_dns_hostnames: this.privateDns + }) + ].flat(); } } diff --git a/backend/src/terraform/prefab.ts b/backend/src/terraform/prefab.ts index 4eb87f5..1fc1922 100644 --- a/backend/src/terraform/prefab.ts +++ b/backend/src/terraform/prefab.ts @@ -4,6 +4,7 @@ import {AwsIamInstanceProfile} from "./AwsIamInstanceProfile"; import {AwsIamRolePolicyAttachment} from "./awsIamRolePolicyAttachment"; import {AwsSecurityGroup} from "./AwsSecurityGroup"; import {AwsVpc} from "./awsVpc"; +import {AwsVpcEndpoint} from "./AwsVpcEndpoint"; import {Ec2} from "./ec2"; import {IamRole} from "./iamRole"; import {S3} from "./s3"; @@ -17,12 +18,16 @@ export const prefabNetwork = ( ssh?: boolean; sshCidr?: string[]; allEgress?: boolean; - web?: boolean; + allIngress?: boolean; + webEgress?: boolean; + webIngress?: boolean; webCidr?: string[]; }, - cidr = "10.0.0.0/24", - vpc = "devxp_allow_ssh_vpc", - securityGroup = "devxp_allow_ssh_security_group" + vpc_cidr = "10.0.0.0/16", + public_cidr = "10.0.0.0/24", + private_cidr = "10.0.128.0/24", + vpc = "devxp_vpc", + securityGroup = "devxp_security_group" ): TerraformResource[] => { const instances = arr(resources.ec2 ?? []).map( (ec2: Ec2) => @@ -32,12 +37,13 @@ export const prefabNetwork = ( ec2.id, ec2.autoIam, 2, - `${vpc}_subnet`, + `${vpc}_subnet_public`, + //`${vpc}_subnet_private`, securityGroup ) ); const buckets = arr(resources.s3 ?? []).map( - (bucket: S3) => new S3(bucket.id, true, bucket.name) + (bucket: S3) => new S3(bucket.id, true, true, bucket.name) ); const policies = buckets.map(bucket => `${bucket.id}_iam_policy0`); const iamRoles = instances.map( @@ -78,6 +84,18 @@ export const prefabNetwork = ( } ]; } + if (rules.allIngress) { + firewalls = [ + ...firewalls, + { + type: "ingress", + from_port: 0, + to_port: 0, + protocol: "-1", + cidr_blocks: ["0.0.0.0/0"] + } + ]; + } if (rules.ssh) { firewalls = [ ...firewalls, @@ -90,7 +108,7 @@ export const prefabNetwork = ( } ]; } - if (rules.web) { + if (rules.webIngress) { firewalls = [ ...firewalls, { @@ -102,11 +120,16 @@ export const prefabNetwork = ( }, { type: "ingress", - from_port: 433, - to_port: 433, + from_port: 443, + to_port: 443, protocol: "tcp", cidr_blocks: rules.webCidr ?? ["0.0.0.0/0"] - }, + } + ]; + } + if (rules.webEgress) { + firewalls = [ + ...firewalls, { type: "egress", from_port: 80, @@ -116,8 +139,8 @@ export const prefabNetwork = ( }, { type: "egress", - from_port: 433, - to_port: 433, + from_port: 443, + to_port: 443, protocol: "tcp", cidr_blocks: rules.webCidr ?? ["0.0.0.0/0"] } @@ -130,7 +153,19 @@ export const prefabNetwork = ( ...instanceProfiles, ...iamRoles, ...attachments, - new AwsVpc(cidr, true, vpc), + // new AwsVpcEndpoint(`${vpc}_endpoint`, vpc, `com.amazonaws.us-west-2.s3`, [], { + // isDefault: false, + // id: `${vpc}_routetable_pub` + // }), + new AwsVpc( + vpc_cidr, + private_cidr, + public_cidr, + vpc, + false, + undefined, + true + ), new AwsSecurityGroup(securityGroup, vpc, firewalls) ]; }; diff --git a/backend/src/terraform/s3.ts b/backend/src/terraform/s3.ts index 42d3883..d5ff320 100644 --- a/backend/src/terraform/s3.ts +++ b/backend/src/terraform/s3.ts @@ -1,17 +1,43 @@ import {jsonRoot} from "./util"; import {ResourceWithIam} from "./resource"; -export interface S3 {} +export interface S3 { + isPrivate: boolean; +} export class S3 extends ResourceWithIam implements S3 { - constructor(id: string, autoIam?: boolean, name?: string) { + constructor( + id: string, + isPrivate = false, + autoIam?: boolean, + name?: string + ) { super(id, "S3", autoIam, name); + this.isPrivate = isPrivate; } //Returns a resource block toJSON() { - return jsonRoot("aws_s3_bucket", this.id, { - bucket: this.name - }); + let json = [ + jsonRoot("aws_s3_bucket", this.id, { + bucket: this.name + }) + ]; + + if (this.isPrivate) { + json = [ + ...json, + jsonRoot( + "aws_s3_bucket_public_access_block", + `${this.id}_access`, + { + bucket: `\${aws_s3_bucket.${this.id}.id}`, + block_public_acls: true, + block_public_policy: true + } + ) + ]; + } + return json; } //An array of policy statements for IAM diff --git a/backend/src/types/terraform.ts b/backend/src/types/terraform.ts index 7bb6ea2..64a0d39 100644 --- a/backend/src/types/terraform.ts +++ b/backend/src/types/terraform.ts @@ -21,6 +21,8 @@ import {AwsSubnet} from "../terraform/awsSubnet"; import {AwsIamInstanceProfile} from "../terraform/AwsIamInstanceProfile"; import {AwsIamRolePolicyAttachment} from "../terraform/awsIamRolePolicyAttachment"; import {IamRole} from "../terraform/iamRole"; +import {AwsRoute as AwsRouteResource} from "../terraform/AwsRoute"; +import {AwsVpcEndpoint} from "../terraform/AwsVpcEndpoint"; // ---------------------------------Variable---------------------------------- // export type VariableType = @@ -251,7 +253,9 @@ export type TerraformResource = | Eip | AwsIamInstanceProfile | AwsIamRolePolicyAttachment - | IamRole; + | IamRole + | AwsVpcEndpoint + | AwsRouteResource; export interface PolicyStatement { actions: string[]; From 529140e8390d302c9a38433ab3f4e82a367609ff Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 20:21:16 -0800 Subject: [PATCH 13/19] feat(backend): Added glacier support to network prefab --- backend/src/index.ts | 8 +++++++- backend/src/terraform/glacierVault.ts | 26 +++++++++++++++----------- backend/src/terraform/prefab.ts | 11 ++++++++++- backend/src/util.ts | 2 +- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 6279c61..74d89ef 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -19,6 +19,7 @@ import {testToFileAws} from "./util"; import {Ec2} from "./terraform/ec2"; import {prefabNetwork} from "./terraform/prefab"; import {S3} from "./terraform/s3"; +import {GlacierVault} from "./terraform/glacierVault"; testToFileAws( "/home/brennan/aws_test/devxp.tf", @@ -32,7 +33,12 @@ testToFileAws( false, "devxp-test-bucket-a" ) - ] + ], + glacier: new GlacierVault( + "devxp_test_vault", + false, + "devxp-test-vault" + ) }, { ssh: true, diff --git a/backend/src/terraform/glacierVault.ts b/backend/src/terraform/glacierVault.ts index f732808..b4eca8e 100644 --- a/backend/src/terraform/glacierVault.ts +++ b/backend/src/terraform/glacierVault.ts @@ -34,16 +34,20 @@ export class GlacierVault //These need to be researched from //https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_examples.html getPolicyDocument() { - return ResourceWithIam.policyStatement( - [ - "glacier:InitiateJob", - "glacier:GetJobOutput", - "glacier:UploadArchive", - "glacier:InitiateMultipartUpload", - "glacier:AbortMultipartUpload", - "glacier:CompleteMultipartUpload" - ], - `\${aws_glacier_vault.${this.id}.arn}` - ); + return [ + ResourceWithIam.policyStatement( + [ + "glacier:InitiateJob", + "glacier:GetJobOutput", + "glacier:UploadArchive", + "glacier:InitiateMultipartUpload", + "glacier:AbortMultipartUpload", + "glacier:CompleteMultipartUpload", + "glacier:DescribeVault" + ], + `\${aws_glacier_vault.${this.id}.arn}` + ), + ResourceWithIam.policyStatement(["glacier:ListVaults"], "*") + ]; } } diff --git a/backend/src/terraform/prefab.ts b/backend/src/terraform/prefab.ts index 1fc1922..02f0783 100644 --- a/backend/src/terraform/prefab.ts +++ b/backend/src/terraform/prefab.ts @@ -6,6 +6,7 @@ import {AwsSecurityGroup} from "./AwsSecurityGroup"; import {AwsVpc} from "./awsVpc"; import {AwsVpcEndpoint} from "./AwsVpcEndpoint"; import {Ec2} from "./ec2"; +import {GlacierVault} from "./glacierVault"; import {IamRole} from "./iamRole"; import {S3} from "./s3"; @@ -13,6 +14,7 @@ export const prefabNetwork = ( resources: { ec2?: Ec2[] | Ec2; s3?: S3[] | S3; + glacier?: GlacierVault[] | GlacierVault; }, rules: { ssh?: boolean; @@ -45,7 +47,13 @@ export const prefabNetwork = ( const buckets = arr(resources.s3 ?? []).map( (bucket: S3) => new S3(bucket.id, true, true, bucket.name) ); - const policies = buckets.map(bucket => `${bucket.id}_iam_policy0`); + const vaults = arr(resources.glacier ?? []).map( + (vault: GlacierVault) => new GlacierVault(vault.id, true) + ); + + const policies = [...buckets, ...vaults].map( + bucket => `${bucket.id}_iam_policy0` + ); const iamRoles = instances.map( ec2 => new IamRole(`${ec2.id}_iam_role`, "ec2.amazonaws.com") ); @@ -150,6 +158,7 @@ export const prefabNetwork = ( return [ ...instances, ...buckets, + ...vaults, ...instanceProfiles, ...iamRoles, ...attachments, diff --git a/backend/src/util.ts b/backend/src/util.ts index be244bc..3e4b000 100644 --- a/backend/src/util.ts +++ b/backend/src/util.ts @@ -83,7 +83,7 @@ export const jsonToHcl = (json: string | Record) => { //Remove incorrect block as attribute styles hcl = hcl.replace( - /(lifecycle|ingress|egress|statement|filter|route) = {/g, + /(lifecycle|ingress|egress|statement|filter|route|notification) = {/g, (_match, $1) => `${$1} {` ); From d0c187c54ca75ff2e7e5c1801c8cf15290cd2686 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 20:46:02 -0800 Subject: [PATCH 14/19] feat(backend): Support for dynamo DB --- backend/src/controllers/terraform.ts | 16 ++++++- backend/src/terraform/DynamoDb.ts | 43 +++++++++++++++++ backend/src/types/terraform.ts | 51 ++++++--------------- backend/src/validators/resourceValidator.ts | 23 +++++++++- 4 files changed, 93 insertions(+), 40 deletions(-) create mode 100644 backend/src/terraform/DynamoDb.ts diff --git a/backend/src/controllers/terraform.ts b/backend/src/controllers/terraform.ts index c281b7b..57dec3e 100644 --- a/backend/src/controllers/terraform.ts +++ b/backend/src/controllers/terraform.ts @@ -9,6 +9,7 @@ import updateHead from "../githubapi/updateHead"; import {getModeNumber} from "../githubapi/util"; import {NamedAwsBackend} from "../terraform/awsBackend"; import {AwsProvider} from "../terraform/awsProvider"; +import {DynamoDb} from "../terraform/DynamoDb"; import {Ec2} from "../terraform/ec2"; import {Gce} from "../terraform/gce"; import {GlacierVault} from "../terraform/glacierVault"; @@ -28,7 +29,13 @@ export const createTerraformSettings = (req: Request, res: Response): void => { provider === "google" ? (req.body.settings?.project as string) : ""; const resourcesRaw = req.body.settings?.resources as (TerraformResource & { - type: "ec2" | "gce" | "s3" | "glacierVault" | "lambdaFunction"; + type: + | "ec2" + | "gce" + | "s3" + | "glacierVault" + | "lambdaFunction" + | "dynamoDb"; })[]; const repo = req.body.repo as string; const token = req.headers?.token as string; @@ -48,6 +55,13 @@ export const createTerraformSettings = (req: Request, res: Response): void => { } else if (resource.type === "glacierVault") { const glacierVault: GlacierVault = resource as GlacierVault; return new GlacierVault(glacierVault.id, glacierVault.autoIam); + } else if (resource.type === "dynamoDb") { + const dynamoDb: DynamoDb = resource as DynamoDb; + return new DynamoDb( + dynamoDb.id, + dynamoDb.attributes, + dynamoDb.autoIam + ); } else if (resource.type === "lambdaFunction") { const lambdaFunc: lambdaFunction = resource as lambdaFunction; return new lambdaFunction( diff --git a/backend/src/terraform/DynamoDb.ts b/backend/src/terraform/DynamoDb.ts new file mode 100644 index 0000000..6f6c297 --- /dev/null +++ b/backend/src/terraform/DynamoDb.ts @@ -0,0 +1,43 @@ +import {jsonRoot} from "./util"; +import {ResourceWithIam} from "./resource"; +import {db_attribute} from "../types/terraform"; + +export interface DynamoDb { + attributes: db_attribute[]; +} +export class DynamoDb extends ResourceWithIam implements DynamoDb { + constructor( + id: string, + attributes: db_attribute[], + autoIam?: boolean, + name?: string + ) { + super(id, "DynamoDb", autoIam, name); + this.attributes = attributes; + } + + //Returns a resource block + toJSON() { + return jsonRoot("aws_dynamodb_table", this.id, { + name: this.name, + hash_key: this.attributes.filter(a => a.isHash)[0]?.name, + ttl: { + attribute_name: "TimeToExist", + enabled: false + }, + attributes: this.attributes.map(a => { + if ("isHash" in a) { + delete a.isHash; + } + return a; + }) + }); + } + + getPolicyDocument() { + return ResourceWithIam.policyStatement( + ["dynamodb:DescribeTable", "dynamodb:Query", "dynamodb:Scan"], + `\${aws_dynamodb_table.${this.id}.arn}` + ); + } +} diff --git a/backend/src/types/terraform.ts b/backend/src/types/terraform.ts index 64a0d39..7ba01c9 100644 --- a/backend/src/types/terraform.ts +++ b/backend/src/types/terraform.ts @@ -23,6 +23,7 @@ import {AwsIamRolePolicyAttachment} from "../terraform/awsIamRolePolicyAttachmen import {IamRole} from "../terraform/iamRole"; import {AwsRoute as AwsRouteResource} from "../terraform/AwsRoute"; import {AwsVpcEndpoint} from "../terraform/AwsVpcEndpoint"; +import {DynamoDb} from "../terraform/DynamoDb"; // ---------------------------------Variable---------------------------------- // export type VariableType = @@ -174,7 +175,16 @@ export type source_image = export type acl = "private" | "public-read" | "public-read-write"; -// ----------------------------------LambdaFunction-------------------------------------- // +// -------------------------------DynamoDb----------------------------------- // + +export type billing_mode = "PROVISIONED" | "PAY_PER_REQUEST"; +export interface db_attribute { + name: string; + type: "S" | "N" | "B"; + isHash?: boolean; +} + +// ----------------------------LambdaFunction-------------------------------- // export type runtime = | "nodejs" @@ -255,7 +265,8 @@ export type TerraformResource = | AwsIamRolePolicyAttachment | IamRole | AwsVpcEndpoint - | AwsRouteResource; + | AwsRouteResource + | DynamoDb; export interface PolicyStatement { actions: string[]; @@ -283,42 +294,6 @@ export interface Firewall { cidr_blocks?: string[]; } -/* - -//TODO: Refactor away from using attributes as blocks -//https://stackoverflow.com/questions/69079945/terraform-inappropriate-value-for-attribute-ingress-while-creating-sg -export const AwsRouteWithDefaults = ( - cidr_block: string, - id: string -) => ({ - gateway_id: id, - cidr_block: cidr_block, - egress_only_gateway_id: "", - instance_id: "", - ipv6_cidr_block: "", - nat_gateway_id: "", - network_interface_id: "", - transit_gateway_id: "", - vpc_endpoint_id: "", - vpc_peering_connection_id: "", - destination_prefix_list_id: "" -}) - -//TODO: Refactor away from using attributes as blocks -//https://stackoverflow.com/questions/69079945/terraform-inappropriate-value-for-attribute-ingress-while-creating-sg -export const FirewallWithDefaults = (firewall: Firewall) => ({ - from_port: firewall.from_port, - to_port: firewall.to_port, - protocol: firewall.protocol, - cidr_blocks: firewall.cidr_blocks, - description: "", - ipv6_cidr_blocks: ["::/0"], - prefix_list_ids: [], - security_groups: [], - self: false -}) -*/ - // ----------------------------Terraform Root-------------------------------- // export interface Terraform { diff --git a/backend/src/validators/resourceValidator.ts b/backend/src/validators/resourceValidator.ts index cee58f6..edb917d 100644 --- a/backend/src/validators/resourceValidator.ts +++ b/backend/src/validators/resourceValidator.ts @@ -1,7 +1,7 @@ import {CustomValidator} from "express-validator"; import {isRuntime} from "../types/terraform"; -export const resourceTypes = /^(ec2|gce|s3|lambdaFunc)$/; +export const resourceTypes = /^(ec2|gce|s3|lambdaFunc|glacierVault|dynamoDb)$/; const validId = (id: string): boolean => { return /^[a-z]([-a-z0-9]*[a-z0-9])?$/.test(id); @@ -108,6 +108,27 @@ const resourceValidator: CustomValidator = (resource: any) => { if (!/^[a-z][-a-z0-9]*[a-z0-9]$/.test(resource.id)) { return false; } + } else if (resource.type === "dynamoDb") { + if (!hasAllKeys(resource, ["id", "attributes"])) { + return false; + } + if (!/^[a-z][-a-z0-9]*[a-z0-9]$/.test(resource.id)) { + return false; + } + if (!Array.isArray(resource.attributes)) { + return false; + } + for (let i = 0; i < resource.attributes.length; i++) { + if (!hasAllKeys(resource.attributes[i], ["name", "type"])) { + return false; + } + if (typeof resource.attributes[i].name !== "string") { + return false; + } + if (!/^(S|N|B)$/.test(resource.attributes[i].type)) { + return false; + } + } } return true; From 189f9d668d03d97fcd0b2dd2bb4bdfb4d175001d Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 21:04:48 -0800 Subject: [PATCH 15/19] feat(backend): Dynamodb support with networking --- backend/src/index.ts | 10 +++++- backend/src/terraform/DynamoDb.ts | 40 +++++++++++++++++---- backend/src/terraform/ec2.ts | 1 + backend/src/terraform/glacierVault.ts | 1 + backend/src/terraform/prefab.ts | 12 +++++-- backend/src/terraform/s3.ts | 1 + backend/src/types/terraform.ts | 2 ++ backend/src/util.ts | 2 +- backend/src/validators/resourceValidator.ts | 7 +++- 9 files changed, 65 insertions(+), 11 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 74d89ef..254da70 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -20,6 +20,7 @@ import {Ec2} from "./terraform/ec2"; import {prefabNetwork} from "./terraform/prefab"; import {S3} from "./terraform/s3"; import {GlacierVault} from "./terraform/glacierVault"; +import {DynamoDb} from "./terraform/DynamoDb"; testToFileAws( "/home/brennan/aws_test/devxp.tf", @@ -38,7 +39,14 @@ testToFileAws( "devxp_test_vault", false, "devxp-test-vault" - ) + ), + dynamo: new DynamoDb("devxp_test_dynamo_db", [ + { + name: "field1", + type: "S", + isHash: true + } + ]) }, { ssh: true, diff --git a/backend/src/terraform/DynamoDb.ts b/backend/src/terraform/DynamoDb.ts index 6f6c297..acc08a2 100644 --- a/backend/src/terraform/DynamoDb.ts +++ b/backend/src/terraform/DynamoDb.ts @@ -21,11 +21,12 @@ export class DynamoDb extends ResourceWithIam implements DynamoDb { return jsonRoot("aws_dynamodb_table", this.id, { name: this.name, hash_key: this.attributes.filter(a => a.isHash)[0]?.name, + billing_mode: "PAY_PER_REQUEST", ttl: { attribute_name: "TimeToExist", - enabled: false + enabled: true }, - attributes: this.attributes.map(a => { + attribute: this.attributes.map(a => { if ("isHash" in a) { delete a.isHash; } @@ -34,10 +35,37 @@ export class DynamoDb extends ResourceWithIam implements DynamoDb { }); } + //https://asecure.cloud/l/iam/ getPolicyDocument() { - return ResourceWithIam.policyStatement( - ["dynamodb:DescribeTable", "dynamodb:Query", "dynamodb:Scan"], - `\${aws_dynamodb_table.${this.id}.arn}` - ); + return [ + ResourceWithIam.policyStatement( + [ + "dynamodb:DescribeTable", + "dynamodb:Query", + "dynamodb:Scan", + "dynamodb:BatchGet*", + "dynamodb:DescribeStream", + "dynamodb:DescribeTable", + "dynamodb:Get*", + "dynamodb:Query", + "dynamodb:Scan", + "dynamodb:BatchWrite*", + "dynamodb:CreateTable", + "dynamodb:Delete*", + "dynamodb:Update*", + "dynamodb:PutItem" + ], + `\${aws_dynamodb_table.${this.id}.arn}` + ), + ResourceWithIam.policyStatement( + [ + "dynamodb:List*", + "dynamodb:DescribeReservedCapacity*", + "dynamodb:DescribeLimits", + "dynamodb:DescribeTimeToLive" + ], + `*` + ) + ]; } } diff --git a/backend/src/terraform/ec2.ts b/backend/src/terraform/ec2.ts index 56a62d0..9b0e283 100644 --- a/backend/src/terraform/ec2.ts +++ b/backend/src/terraform/ec2.ts @@ -130,6 +130,7 @@ export class Ec2 extends ResourceWithIam implements Ec2 { //An array of policy statements for IAM //These need to be researched from //https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_examples.html + //https://asecure.cloud/l/iam/ getPolicyDocument() { return [ ResourceWithIam.policyStatement( diff --git a/backend/src/terraform/glacierVault.ts b/backend/src/terraform/glacierVault.ts index b4eca8e..4b03f24 100644 --- a/backend/src/terraform/glacierVault.ts +++ b/backend/src/terraform/glacierVault.ts @@ -33,6 +33,7 @@ export class GlacierVault //An array of policy statements for IAM //These need to be researched from //https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_examples.html + //https://asecure.cloud/l/iam/ getPolicyDocument() { return [ ResourceWithIam.policyStatement( diff --git a/backend/src/terraform/prefab.ts b/backend/src/terraform/prefab.ts index 02f0783..377aa40 100644 --- a/backend/src/terraform/prefab.ts +++ b/backend/src/terraform/prefab.ts @@ -4,7 +4,8 @@ import {AwsIamInstanceProfile} from "./AwsIamInstanceProfile"; import {AwsIamRolePolicyAttachment} from "./awsIamRolePolicyAttachment"; import {AwsSecurityGroup} from "./AwsSecurityGroup"; import {AwsVpc} from "./awsVpc"; -import {AwsVpcEndpoint} from "./AwsVpcEndpoint"; +//import {AwsVpcEndpoint} from "./AwsVpcEndpoint"; +import {DynamoDb} from "./DynamoDb"; import {Ec2} from "./ec2"; import {GlacierVault} from "./glacierVault"; import {IamRole} from "./iamRole"; @@ -15,6 +16,7 @@ export const prefabNetwork = ( ec2?: Ec2[] | Ec2; s3?: S3[] | S3; glacier?: GlacierVault[] | GlacierVault; + dynamo?: DynamoDb[] | DynamoDb; }, rules: { ssh?: boolean; @@ -39,6 +41,8 @@ export const prefabNetwork = ( ec2.id, ec2.autoIam, 2, + + //TODO: Find a way to put this in the private subnet `${vpc}_subnet_public`, //`${vpc}_subnet_private`, securityGroup @@ -50,8 +54,11 @@ export const prefabNetwork = ( const vaults = arr(resources.glacier ?? []).map( (vault: GlacierVault) => new GlacierVault(vault.id, true) ); + const dbs = arr(resources.dynamo ?? []).map( + (db: DynamoDb) => new DynamoDb(db.id, db.attributes, true, db.name) + ); - const policies = [...buckets, ...vaults].map( + const policies = [...buckets, ...vaults, ...dbs].map( bucket => `${bucket.id}_iam_policy0` ); const iamRoles = instances.map( @@ -159,6 +166,7 @@ export const prefabNetwork = ( ...instances, ...buckets, ...vaults, + ...dbs, ...instanceProfiles, ...iamRoles, ...attachments, diff --git a/backend/src/terraform/s3.ts b/backend/src/terraform/s3.ts index d5ff320..085b2fe 100644 --- a/backend/src/terraform/s3.ts +++ b/backend/src/terraform/s3.ts @@ -43,6 +43,7 @@ export class S3 extends ResourceWithIam implements S3 { //An array of policy statements for IAM //These need to be researched from //https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_examples.html + //https://asecure.cloud/l/iam/ getPolicyDocument() { return [ ResourceWithIam.policyStatement( diff --git a/backend/src/types/terraform.ts b/backend/src/types/terraform.ts index 7ba01c9..bd615fc 100644 --- a/backend/src/types/terraform.ts +++ b/backend/src/types/terraform.ts @@ -181,6 +181,8 @@ export type billing_mode = "PROVISIONED" | "PAY_PER_REQUEST"; export interface db_attribute { name: string; type: "S" | "N" | "B"; + + //TODO: Add sort key isHash?: boolean; } diff --git a/backend/src/util.ts b/backend/src/util.ts index 3e4b000..1f5beb6 100644 --- a/backend/src/util.ts +++ b/backend/src/util.ts @@ -83,7 +83,7 @@ export const jsonToHcl = (json: string | Record) => { //Remove incorrect block as attribute styles hcl = hcl.replace( - /(lifecycle|ingress|egress|statement|filter|route|notification) = {/g, + /(lifecycle|ingress|egress|statement|filter|route|notification|ttl|attribute) = {/g, (_match, $1) => `${$1} {` ); diff --git a/backend/src/validators/resourceValidator.ts b/backend/src/validators/resourceValidator.ts index edb917d..a3b2895 100644 --- a/backend/src/validators/resourceValidator.ts +++ b/backend/src/validators/resourceValidator.ts @@ -1,5 +1,5 @@ import {CustomValidator} from "express-validator"; -import {isRuntime} from "../types/terraform"; +import {db_attribute, isRuntime} from "../types/terraform"; export const resourceTypes = /^(ec2|gce|s3|lambdaFunc|glacierVault|dynamoDb)$/; @@ -129,6 +129,11 @@ const resourceValidator: CustomValidator = (resource: any) => { return false; } } + if ( + resource.attributes.filter((a: db_attribute) => a.isHash).length < 1 + ) { + return false; + } } return true; From aa0f3b378bc131c02303432ae808a7fdb93762d6 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 21:25:07 -0800 Subject: [PATCH 16/19] feat(backend): Hooked up prefab network to controller --- backend/src/controllers/terraform.ts | 17 +++++- backend/src/index.ts | 78 ++++++++++++++-------------- backend/src/terraform/prefab.ts | 58 +++++++++++++++++++++ 3 files changed, 113 insertions(+), 40 deletions(-) diff --git a/backend/src/controllers/terraform.ts b/backend/src/controllers/terraform.ts index 57dec3e..f38bd69 100644 --- a/backend/src/controllers/terraform.ts +++ b/backend/src/controllers/terraform.ts @@ -16,6 +16,11 @@ import {GlacierVault} from "../terraform/glacierVault"; import {NamedGoogleBackend} from "../terraform/googleBackend"; import {GoogleProvider} from "../terraform/googleProvider"; import {lambdaFunction} from "../terraform/lambdaFunction"; +import { + prefabNetworkFromArr, + PrefabSupports, + splitForPrefab +} from "../terraform/prefab"; import {S3} from "../terraform/s3"; import {rootBlockSplitBackend} from "../terraform/terraform"; import {internalErrorHandler} from "../types/errorHandler"; @@ -80,12 +85,22 @@ export const createTerraformSettings = (req: Request, res: Response): void => { return; } + const [gce, lambda, networkedResources] = splitForPrefab(resources); + + const network = + networkedResources.length > 0 && provider === "aws" + ? prefabNetworkFromArr(networkedResources, { + allEgress: true, + allIngress: true + }) + : networkedResources; + const [root, backend] = rootBlockSplitBackend( provider === "aws" ? new AwsProvider() : new GoogleProvider(project), provider === "aws" ? new NamedAwsBackend() : new NamedGoogleBackend(project), - resources + [...gce, ...lambda, ...network] ); getHead(token, repo, "main") diff --git a/backend/src/index.ts b/backend/src/index.ts index 254da70..eb03af9 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -15,46 +15,46 @@ server.serve("/about"); server.serve("/contact"); server.route("/", mainRouter); -import {testToFileAws} from "./util"; -import {Ec2} from "./terraform/ec2"; -import {prefabNetwork} from "./terraform/prefab"; -import {S3} from "./terraform/s3"; -import {GlacierVault} from "./terraform/glacierVault"; -import {DynamoDb} from "./terraform/DynamoDb"; +// import {testToFileAws} from "./util"; +// import {Ec2} from "./terraform/ec2"; +// import {prefabNetwork} from "./terraform/prefab"; +// import {S3} from "./terraform/s3"; +// import {GlacierVault} from "./terraform/glacierVault"; +// import {DynamoDb} from "./terraform/DynamoDb"; -testToFileAws( - "/home/brennan/aws_test/devxp.tf", - prefabNetwork( - { - ec2: [new Ec2("AUTO_UBUNTU", "t2.micro", "instance_a", true)], - s3: [ - new S3( - "devxp_test_bucket_a", - false, - false, - "devxp-test-bucket-a" - ) - ], - glacier: new GlacierVault( - "devxp_test_vault", - false, - "devxp-test-vault" - ), - dynamo: new DynamoDb("devxp_test_dynamo_db", [ - { - name: "field1", - type: "S", - isHash: true - } - ]) - }, - { - ssh: true, - webEgress: true, - webIngress: true - } - ) -); +// testToFileAws( +// "/home/brennan/aws_test/devxp.tf", +// prefabNetwork( +// { +// ec2: [new Ec2("AUTO_UBUNTU", "t2.micro", "instance_a", true)], +// s3: [ +// new S3( +// "devxp_test_bucket_a", +// false, +// false, +// "devxp-test-bucket-a" +// ) +// ], +// glacier: new GlacierVault( +// "devxp_test_vault", +// false, +// "devxp-test-vault" +// ), +// dynamo: new DynamoDb("devxp_test_dynamo_db", [ +// { +// name: "field1", +// type: "S", +// isHash: true +// } +// ]) +// }, +// { +// ssh: true, +// webEgress: true, +// webIngress: true +// } +// ) +// ); mongoose.connection.on( "error", diff --git a/backend/src/terraform/prefab.ts b/backend/src/terraform/prefab.ts index 377aa40..1919ea0 100644 --- a/backend/src/terraform/prefab.ts +++ b/backend/src/terraform/prefab.ts @@ -7,10 +7,68 @@ import {AwsVpc} from "./awsVpc"; //import {AwsVpcEndpoint} from "./AwsVpcEndpoint"; import {DynamoDb} from "./DynamoDb"; import {Ec2} from "./ec2"; +import {Gce} from "./gce"; import {GlacierVault} from "./glacierVault"; import {IamRole} from "./iamRole"; +import {lambdaFunction} from "./lambdaFunction"; import {S3} from "./s3"; +export type PrefabSupports = Ec2 | S3 | GlacierVault | DynamoDb; + +export const splitForPrefab = ( + resources: TerraformResource[] +): [Gce[], lambdaFunction[], PrefabSupports[]] => { + let gce: Gce[] = []; + let lambda: lambdaFunction[] = []; + let prefabSupports: PrefabSupports[] = []; + + resources.forEach(r => { + if (r.type === "gce") { + gce = [...gce, r as Gce]; + } else if (r.type === "lambdaFunction") { + lambda = [...lambda, r as lambdaFunction]; + } else { + prefabSupports = [...prefabSupports, r as PrefabSupports]; + } + }); + + return [gce, lambda, prefabSupports]; +}; + +export const prefabNetworkFromArr = ( + resources: PrefabSupports[], + rules: { + ssh?: boolean; + sshCidr?: string[]; + allEgress?: boolean; + allIngress?: boolean; + webEgress?: boolean; + webIngress?: boolean; + webCidr?: string[]; + }, + vpc_cidr = "10.0.0.0/16", + public_cidr = "10.0.0.0/24", + private_cidr = "10.0.128.0/24", + vpc = "devxp_vpc", + securityGroup = "devxp_security_group" +) => + prefabNetwork( + { + ec2: resources.filter(r => r.type === "ec2") as Ec2[], + s3: resources.filter(r => r.type === "s3") as S3[], + dynamo: resources.filter(r => r.type === "dynamoDb") as DynamoDb[], + glacier: resources.filter( + r => r.type === "glacierVault" + ) as GlacierVault[] + }, + rules, + vpc_cidr, + public_cidr, + private_cidr, + vpc, + securityGroup + ); + export const prefabNetwork = ( resources: { ec2?: Ec2[] | Ec2; From 08931f51abe07f193bebf3ba4400c4656b0765c2 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 21:32:59 -0800 Subject: [PATCH 17/19] feat(backend): Hooked up secure mode to controller --- backend/src/controllers/terraform.ts | 18 +++++++++------ backend/src/validators/terraformValidator.ts | 24 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/backend/src/controllers/terraform.ts b/backend/src/controllers/terraform.ts index f38bd69..8932560 100644 --- a/backend/src/controllers/terraform.ts +++ b/backend/src/controllers/terraform.ts @@ -16,11 +16,7 @@ import {GlacierVault} from "../terraform/glacierVault"; import {NamedGoogleBackend} from "../terraform/googleBackend"; import {GoogleProvider} from "../terraform/googleProvider"; import {lambdaFunction} from "../terraform/lambdaFunction"; -import { - prefabNetworkFromArr, - PrefabSupports, - splitForPrefab -} from "../terraform/prefab"; +import {prefabNetworkFromArr, splitForPrefab} from "../terraform/prefab"; import {S3} from "../terraform/s3"; import {rootBlockSplitBackend} from "../terraform/terraform"; import {internalErrorHandler} from "../types/errorHandler"; @@ -29,6 +25,11 @@ import {TerraformResource} from "../types/terraform"; export const createTerraformSettings = (req: Request, res: Response): void => { const provider = req.body.settings?.provider as "aws" | "google" | "azure"; + const secure = req.body.secure ?? false; + const allowSsh = req.body.allowSsh ?? false; + const allowEgressWeb = req.body.allowEgressWeb ?? false; + const allowIngressWeb = req.body.allowIngressWeb ?? false; + //Only needed for google const project = provider === "google" ? (req.body.settings?.project as string) : ""; @@ -90,8 +91,11 @@ export const createTerraformSettings = (req: Request, res: Response): void => { const network = networkedResources.length > 0 && provider === "aws" ? prefabNetworkFromArr(networkedResources, { - allEgress: true, - allIngress: true + allEgress: !secure, + allIngress: !secure, + ssh: secure && allowSsh, + webEgress: secure && allowEgressWeb, + webIngress: secure && allowIngressWeb }) : networkedResources; diff --git a/backend/src/validators/terraformValidator.ts b/backend/src/validators/terraformValidator.ts index 016dce0..59a05a0 100644 --- a/backend/src/validators/terraformValidator.ts +++ b/backend/src/validators/terraformValidator.ts @@ -31,6 +31,30 @@ export const settingsValidator = [ .isLength({min: 1}) .matches(/^(aws|google|azure)$/) .withMessage("Provider must be aws, google, or azure at this time"), + body("settings.secure") + .if(body("tool").equals("terraform")) + .optional() + .isBoolean() + .default(false) + .withMessage("secure flag must be boolean"), + body("settings.allowSsh") + .if(body("tool").equals("terraform")) + .optional() + .isBoolean() + .default(false) + .withMessage("allowSsh flag must be boolean"), + body("settings.allowIngressWeb") + .if(body("tool").equals("terraform")) + .optional() + .isBoolean() + .default(false) + .withMessage("allowIngressWeb flag must be boolean"), + body("settings.allowEgressWeb") + .if(body("tool").equals("terraform")) + .optional() + .isBoolean() + .default(false) + .withMessage("allowEgressWeb flag must be boolean"), body("settings.project") .if(body("tool").equals("terraform")) .if(body("settings.provider").equals("google")) From d623daf0c65d40473d9146cfe2b0ee78e9f84980 Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 21:42:19 -0800 Subject: [PATCH 18/19] feat(backend): Write to .tf file --- backend/src/controllers/terraform.ts | 17 +++++++++-------- backend/src/terraform/prefab.ts | 14 ++++++++------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/backend/src/controllers/terraform.ts b/backend/src/controllers/terraform.ts index 8932560..712ea23 100644 --- a/backend/src/controllers/terraform.ts +++ b/backend/src/controllers/terraform.ts @@ -21,14 +21,15 @@ import {S3} from "../terraform/s3"; import {rootBlockSplitBackend} from "../terraform/terraform"; import {internalErrorHandler} from "../types/errorHandler"; import {TerraformResource} from "../types/terraform"; +import {jsonToHcl} from "../util"; export const createTerraformSettings = (req: Request, res: Response): void => { const provider = req.body.settings?.provider as "aws" | "google" | "azure"; - const secure = req.body.secure ?? false; - const allowSsh = req.body.allowSsh ?? false; - const allowEgressWeb = req.body.allowEgressWeb ?? false; - const allowIngressWeb = req.body.allowIngressWeb ?? false; + const secure = req.body.settings.secure ?? false; + const allowSsh = req.body.settings.allowSsh ?? false; + const allowEgressWeb = req.body.settings.allowEgressWeb ?? false; + const allowIngressWeb = req.body.settings.allowIngressWeb ?? false; //Only needed for google const project = @@ -113,7 +114,7 @@ export const createTerraformSettings = (req: Request, res: Response): void => { const blobRoot = await postBlob( token, repo, - JSON.stringify(root, null, 2) + "\n" + jsonToHcl(root) + "\n" ); /* @@ -122,7 +123,7 @@ export const createTerraformSettings = (req: Request, res: Response): void => { const blobBackend = await postBlob( token, repo, - JSON.stringify(backend, null, 2) + "\n" + jsonToHcl(backend) + "\n" ); */ @@ -135,7 +136,7 @@ export const createTerraformSettings = (req: Request, res: Response): void => { //Create a new tree within that one const newTree = await createTree(token, repo, tree.sha, [ { - path: "terraform.tf.json", + path: "terraform.tf", mode: getModeNumber("blob"), type: "blob", sha: blobRoot.sha, @@ -145,7 +146,7 @@ export const createTerraformSettings = (req: Request, res: Response): void => { /* Removed for M1 presentation. We'll solve the chicken and egg for milestone 2 { - path: "backend.tf.json", + path: "backend.tf", mode: getModeNumber("blob"), type: "blob", sha: blobBackend.sha, diff --git a/backend/src/terraform/prefab.ts b/backend/src/terraform/prefab.ts index 1919ea0..693fac9 100644 --- a/backend/src/terraform/prefab.ts +++ b/backend/src/terraform/prefab.ts @@ -23,9 +23,9 @@ export const splitForPrefab = ( let prefabSupports: PrefabSupports[] = []; resources.forEach(r => { - if (r.type === "gce") { + if (r.type.toLowerCase() === "gce") { gce = [...gce, r as Gce]; - } else if (r.type === "lambdaFunction") { + } else if (r.type.toLowerCase() === "lambdafunction") { lambda = [...lambda, r as lambdaFunction]; } else { prefabSupports = [...prefabSupports, r as PrefabSupports]; @@ -54,11 +54,13 @@ export const prefabNetworkFromArr = ( ) => prefabNetwork( { - ec2: resources.filter(r => r.type === "ec2") as Ec2[], - s3: resources.filter(r => r.type === "s3") as S3[], - dynamo: resources.filter(r => r.type === "dynamoDb") as DynamoDb[], + ec2: resources.filter(r => r.type.toLowerCase() === "ec2") as Ec2[], + s3: resources.filter(r => r.type.toLowerCase() === "s3") as S3[], + dynamo: resources.filter( + r => r.type.toLowerCase() === "dynamodb" + ) as DynamoDb[], glacier: resources.filter( - r => r.type === "glacierVault" + r => r.type.toLowerCase() === "glaciervault" ) as GlacierVault[] }, rules, From 801580b00ff0ae468467f7cc5c7c931577b78bba Mon Sep 17 00:00:00 2001 From: Brennan Wilkes Date: Sun, 6 Mar 2022 21:52:00 -0800 Subject: [PATCH 19/19] fix(backend): Resolved branch already exists bug --- backend/src/controllers/terraform.ts | 2 +- backend/src/githubapi/createBranch.ts | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/backend/src/controllers/terraform.ts b/backend/src/controllers/terraform.ts index 6b57e38..ea9924d 100644 --- a/backend/src/controllers/terraform.ts +++ b/backend/src/controllers/terraform.ts @@ -131,7 +131,7 @@ export const createTerraformSettings = (req: Request, res: Response): void => { // Create a new branch to post our commit to const branchName = "DevXP-Configuration"; const newBranch = await createBranch( - `refs/heads/${branchName}`, + branchName, token, repo, head.sha diff --git a/backend/src/githubapi/createBranch.ts b/backend/src/githubapi/createBranch.ts index c792b7b..cca6591 100644 --- a/backend/src/githubapi/createBranch.ts +++ b/backend/src/githubapi/createBranch.ts @@ -1,21 +1,24 @@ /* eslint-disable prettier/prettier */ import axios from "axios"; import {GithubBranch, isGithubBranch} from "../types/github"; +import getHead from "./getHead"; import {GITHUB_BASE_URL, createGithubHeader} from "./util"; export default ( - ref: string, + branchName: string, token: string, repo: string, treeSha: string ): Promise => new Promise((resolve, reject) => { + let errCache: any; + axios .post( `${GITHUB_BASE_URL}/repos/${repo}/git/refs`, { sha: treeSha, - ref: ref + ref: `refs/heads/${branchName}` }, createGithubHeader(token) ) @@ -31,5 +34,15 @@ export default ( reject(new Error("Invalid response from github")); } }) - .catch(reject); + .catch(err => { + errCache = err; + //Maybe the branch exists so we should try to retrieve it + return getHead(token, repo, branchName); + + //TODO: Refactor this + }) + .then(head => resolve(head as GithubBranch)) + .catch(() => { + reject(errCache); + }); });