From 9df1c6311f1611bc11e5f2de6958269d6c9588bf Mon Sep 17 00:00:00 2001 From: Andrew Hobson Date: Tue, 31 Aug 2021 13:03:36 -0400 Subject: [PATCH 1/8] Update Dockerfile for ECS --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd4444c3..9a4fae44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8 +FROM python:3.9.6 ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 @@ -11,7 +11,10 @@ RUN pip install -r /app/requirements.txt # Copy over everything else for the app: COPY docker.__init__.py /app/__init__.py -COPY . /app +COPY locustfiles /app/locustfiles +COPY static /app/static +COPY tasks /app/tasks +COPY utils /app/utils EXPOSE 8089 5557 From 9d076a246ec6c8dc5573db3ca4db240510c96c86 Mon Sep 17 00:00:00 2001 From: Andrew Hobson Date: Tue, 31 Aug 2021 13:04:22 -0400 Subject: [PATCH 2/8] Nix updates for ECS / python 3.9 --- nix/default.nix | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/nix/default.nix b/nix/default.nix index 0f938eb3..0e5164e6 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -16,19 +16,43 @@ in buildEnv { (import (builtins.fetchGit { # Descriptive name to make the store path easier to identify - name = "python3-3.8.8"; + name = "editorconfig-core-c-0.12.1"; url = "https://github.com/NixOS/nixpkgs/"; ref = "refs/heads/nixpkgs-unstable"; - rev = "89dacdd7a518c14d3e824d4c6cc0cc93002367f6"; + rev = "c8ff5bc6f74a2960fab5ae417cd2bb055eab1002"; + }) {}).editorconfig-core-c + + (import (builtins.fetchGit { + # Descriptive name to make the store path easier to identify + name = "python3-3.9.6"; + url = "https://github.com/NixOS/nixpkgs/"; + ref = "refs/heads/nixpkgs-unstable"; + rev = "c8ff5bc6f74a2960fab5ae417cd2bb055eab1002"; }) {}).python3 (import (builtins.fetchGit { # Descriptive name to make the store path easier to identify - name = "pre-commit-2.7.1"; + name = "pre-commit-2.13.0"; url = "https://github.com/NixOS/nixpkgs/"; ref = "refs/heads/nixpkgs-unstable"; - rev = "559cf76fa3642106d9f23c9e845baf4d354be682"; + rev = "f62847d5655a6954fc946e5d9c81424171cc281e"; }) {}).pre-commit + (import (builtins.fetchGit { + # Descriptive name to make the store path easier to identify + name = "aws-vault-6.3.1"; + url = "https://github.com/NixOS/nixpkgs/"; + ref = "refs/heads/nixpkgs-unstable"; + rev = "253aecf69ed7595aaefabde779aa6449195bebb7"; + }) {}).aws-vault + + (import (builtins.fetchGit { + # Descriptive name to make the store path easier to identify + name = "awscli2-2.2.14"; + url = "https://github.com/NixOS/nixpkgs/"; + ref = "refs/heads/nixpkgs-unstable"; + rev = "14b0f20fa1f56438b74100513c9b1f7c072cf789"; + }) {}).awscli2 + ]; } From 4cda7cc9c7792da38048c210a0ed39e90c792096 Mon Sep 17 00:00:00 2001 From: Andrew Hobson Date: Tue, 31 Aug 2021 13:05:14 -0400 Subject: [PATCH 3/8] Add ECS task definition, plus script for port forwarding --- ecs/milmove_load_testing_ecs.json.tmpl | 72 ++++++++++++++++++++++++ scripts/aws-session-port-forward.py | 77 ++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 ecs/milmove_load_testing_ecs.json.tmpl create mode 100755 scripts/aws-session-port-forward.py diff --git a/ecs/milmove_load_testing_ecs.json.tmpl b/ecs/milmove_load_testing_ecs.json.tmpl new file mode 100644 index 00000000..02154b0a --- /dev/null +++ b/ecs/milmove_load_testing_ecs.json.tmpl @@ -0,0 +1,72 @@ +{ + "containerDefinitions": [ + { + "command": [ + "-f", + "/app/locustfiles/prime.py", + "--host", + "exp", + "--web-port", + "4000" + ], + "essential": true, + "image": "__ACCOUNTID__.dkr.ecr.__REGION__.amazonaws.com/loadtesting:exp", + "readonlyRootFilesystem": false, + "linuxParameters": { + "initProcessEnabled": true + }, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group" : "/ecs/loadtesting/exp-web", + "awslogs-region": "__REGION__", + "awslogs-stream-prefix": "exp-web" + } + }, + "name": "exp-web-loadtesting", + "portMappings": [ + { + "containerPort": 4000, + "hostPort": 4000, + "protocol": "tcp" + } + ], + "environment": [ + { + "name": "DEVLOCAL_AUTH", + "value": "true" + }, + { + "name": "STATIC_TLS_FILE_PATH", + "value": "/tmp" + } + ], + "secrets": [ + { + "name": "MOVE_MIL_EXP_TLS_CERT", + "valueFrom": "arn:__PARTITION__:ssm:__REGION__:__ACCOUNTID__:parameter/load-testing/exp-tls-cert" + }, + { + "name": "MOVE_MIL_EXP_TLS_KEY", + "valueFrom": "arn:__PARTITION__:ssm:__REGION__:__ACCOUNTID__:parameter/load-testing/exp-tls-key" + } + ], + "ulimits": [ + { + "name":"nofile", + "softLimit": 10000, + "hardLimit": 10000 + } + ] + } + ], + "cpu": "256", + "taskRoleArn": "arn:__PARTITION__:iam::__ACCOUNTID__:role/ecs-task-role-exp-web-loadtesting", + "executionRoleArn": "arn:__PARTITION__:iam::__ACCOUNTID__:role/ecs-task-execution-role-exp-web-loadtesting", + "family": "exp-web-loadtesting", + "memory": "512", + "networkMode": "awsvpc", + "requiresCompatibilities": [ + "FARGATE" + ] +} diff --git a/scripts/aws-session-port-forward.py b/scripts/aws-session-port-forward.py new file mode 100755 index 00000000..3ff69202 --- /dev/null +++ b/scripts/aws-session-port-forward.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import json +import os +import subprocess +import sys + +cluster = "loadtesting" +port_number = "4000" + +result = subprocess.run(["aws", "ecs", "list-tasks", "--cluster", cluster], stdout=subprocess.PIPE) + +tasks = json.loads(result.stdout) +if "taskArns" not in tasks or len(tasks["taskArns"]) == 0: + print("Cannot find any running tasks in cluster: {}", cluster, file=sys.stderr) + sys.exit(1) + +if len(tasks["taskArns"]) != 1: + print("More than one task running in cluster: {}", cluster, file=sys.stderr) + sys.exit(1) + +task_arn = tasks["taskArns"][0] + +result = subprocess.run( + ["aws", "ecs", "describe-tasks", "--cluster", cluster, "--tasks", task_arn], stdout=subprocess.PIPE +) + +try: + task_details = json.loads(result.stdout) +except Exception as e: + print("Cannot parse describe tasks for taskArn: {}", task_arn, file=sys.stderr) + print(sys.exc_info()[0], file=sys.stderr) + print(e, file=sys.stderr) + sys.exit(1) + +if "tasks" not in task_details or len(task_details["tasks"]) == 0: + print("Cannot find task detail for taskArn: {}", task_arn, file=sys.stderr) + sys.exit(1) + +if len(task_details["tasks"]) != 1: + print("More than one task detail for taskArn: {}", task_arn, file=sys.stderr) + sys.exit(1) + +task = task_details["tasks"][0] + +if "containers" not in task or len(task["containers"]) == 0: + print("Cannot find containers for taskArn: {}", task_arn, file=sys.stderr) + sys.exit(1) + +if len(task["containers"]) != 1: + print("More than one container for taskArn: {}", task_arn, file=sys.stderr) + sys.exit(1) + +container = task["containers"][0] + +task_id = task_arn.split("/")[-1] +container_id = container["runtimeId"] + +target = "ecs:{cluster}_{task_id}_{container_id}".format(cluster=cluster, task_id=task_id, container_id=container_id) + +parameters = json.dumps({"portNumber": [port_number], "localPortNumber": [port_number]}) + +print("Hit 'Control-C' to exit") +# replace this process with the session +os.execlp( + "aws", + "aws", + "ssm", + "start-session", + "--target", + target, + "--document-name", + "AWS-StartPortForwardingSession", + "--parameters", + parameters, +) From 29aac83f7ca1073c2a1ae7ab31d22c51344a8fd1 Mon Sep 17 00:00:00 2001 From: Andrew Hobson Date: Thu, 2 Sep 2021 11:34:58 -0400 Subject: [PATCH 4/8] Add codebuild config --- buildspec.yml | 7 +++++++ scripts/codebuild | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 buildspec.yml create mode 100755 scripts/codebuild diff --git a/buildspec.yml b/buildspec.yml new file mode 100644 index 00000000..7e067bee --- /dev/null +++ b/buildspec.yml @@ -0,0 +1,7 @@ +version: 0.2 + +phases: + build: + on-failure: ABORT + commands: + - ./scripts/codebuild diff --git a/scripts/codebuild b/scripts/codebuild new file mode 100755 index 00000000..a0e58972 --- /dev/null +++ b/scripts/codebuild @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail +set -x + +echo "${DOCKER_PASSWORD}" | \ + docker login --username "${DOCKER_USERNAME}" --password-stdin + +docker build -t "${ECR_REPO}:exp" . + +aws ecr get-login-password --region $AWS_REGION \ + | docker login --username AWS --password-stdin "${ECR_REPO}" + +docker push "${ECR_REPO}:exp" + +aws ecs update-service \ + --cluster loadtesting \ + --service exp-web \ + --task-definition exp-web-loadtesting:3 \ + --desired-count 1 + +exit 0 From d09db885ed301ce16bde7af46bb6b881dac4e5b0 Mon Sep 17 00:00:00 2001 From: Ronak Patel Date: Tue, 12 Oct 2021 17:19:45 -0400 Subject: [PATCH 5/8] Fix spelling errors --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 332676d6..06340c41 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ your virtual environment and use the command: pytest ``` -To see verbose output and any print statments in the tests, use: +To see verbose output and any print statements in the tests, use: ```shell pytest -v -s @@ -446,7 +446,7 @@ Ex: locust -f locustfiles/prime.py --host local ``` -For mor information on running custom tests, refer to the [wiki](https://github.com/transcom/milmove_load_testing/wiki/Running-Load-Tests-Against-MyMove) +For more information on running custom tests, refer to the [wiki](https://github.com/transcom/milmove_load_testing/wiki/Running-Load-Tests-Against-MyMove) ### Running Tests for Reporting @@ -624,7 +624,7 @@ in this repo. If you need to do something different, however, Locust's documenta ### Adding tasks to existing load tests -Adding a task to an existing load test is thankfully a fairly straight-forward processs that requires just a bit of +Adding a task to an existing load test is thankfully a fairly straight-forward process that requires just a bit of research and just a bit of coding. Here are the general steps: * Locate the locust file your test needs to be run from. From da5dc86ddcba89601cc95d0bebbbc7ab2a3b1d13 Mon Sep 17 00:00:00 2001 From: Ronak Patel Date: Tue, 12 Oct 2021 18:03:49 -0400 Subject: [PATCH 6/8] Add instructions to run load tests in AWS --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 06340c41..d205fc25 100644 --- a/README.md +++ b/README.md @@ -406,6 +406,22 @@ Back in `milmove_load_testing`, make sure you activate your virtual environment pyenv activate locust-venv ``` +### Setting up Tests in AWS + +Run the port-forwarding script. + + ```sh + aws-vault exec $AWS_PROFILE -- ./scripts/aws-session-port-forward.py + ``` + +You may see the following error: + +* `SessionManagerPlugin is not found`. If you do please follow the link and the instructions to install the Session Manager plugin or reference [this](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html#install-plugin-macos) directly. + +* You may see an error mentioning `credentials missing` and an additional reference to a specific profile. If this is the case please add the corresponding entry from [this template](https://dp3.atlassian.net/wiki/spaces/MT/pages/1348927493/AWS+GovCloud+Config+Templates) to your `~/.aws/config` file. + +You can then visit and run tests from AWS. + ### Running preset tests Default tests commands for each locustfile are added to the Makefile to make rerunning common preset tests either. These From 69f9ef79d3219b6fa3637a374c51805017b7f3ef Mon Sep 17 00:00:00 2001 From: Ronak Patel Date: Wed, 13 Oct 2021 11:18:50 -0400 Subject: [PATCH 7/8] Update table of contents --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d205fc25..ba627940 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ in the [LICENSE.txt](./LICENSE.txt) file in this repository. * [Testing](#testing) * [Running Load Tests](#running-load-tests) * [Setting up the local environment](#setting-up-the-local-environment) + * [Setting up Tests in AWS](#setting-up-tests-in-aws) * [Running preset tests](#running-preset-tests) * [Running custom tests](#running-custom-tests) * [Running Tests for Reporting](#running-tests-for-reporting) @@ -52,7 +53,7 @@ in the [LICENSE.txt](./LICENSE.txt) file in this repository. * [Metrics](#metrics) * [References](#references) - + From ed88148925e23a7426fac40a851035e0c3011cee Mon Sep 17 00:00:00 2001 From: Ronak Patel Date: Wed, 13 Oct 2021 11:22:10 -0400 Subject: [PATCH 8/8] Add additional context for missing credentials error --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba627940..1fe6554f 100644 --- a/README.md +++ b/README.md @@ -419,7 +419,7 @@ You may see the following error: * `SessionManagerPlugin is not found`. If you do please follow the link and the instructions to install the Session Manager plugin or reference [this](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html#install-plugin-macos) directly. -* You may see an error mentioning `credentials missing` and an additional reference to a specific profile. If this is the case please add the corresponding entry from [this template](https://dp3.atlassian.net/wiki/spaces/MT/pages/1348927493/AWS+GovCloud+Config+Templates) to your `~/.aws/config` file. +* You may see an error mentioning `credentials missing` and an additional reference to a specific profile. If this is the case please add the corresponding entry from [this template](https://dp3.atlassian.net/wiki/spaces/MT/pages/1348927493/AWS+GovCloud+Config+Templates) to your `~/.aws/config` file. If you only see an error mentioning `credentials missing` without an additional reference to a specific profile, please ensure that your $AWS_PROFILE variable is not blank. This can be set by running `direnv allow`. You can then visit and run tests from AWS.