MB-10413: Convert to using pipenv to manage dependencies

.circleci/config.yml

@@ -14,7 +14,9 @@ executors:
   primary:
     resource_class: small
     docker:
-      - image: trussworks/circleci-docker-primary:c3a4d876a5681cceef9f927392732c259308d158
+      - image: milmove/circleci-docker:86f3163991b707d1351cc01d2c9cdc2dbaae4e93
+    environment:
+      PIPENV_VENV_IN_PROJECT: true
 
 jobs:
 
@@ -28,14 +30,33 @@ jobs:
       - restore_cache:
           keys:
             - v1-pre-commit-dot-cache-{{ checksum ".pre-commit-config.yaml" }}
+      - restore_cache:  # ensure this step occurs *before* installing dependencies
+          key: v1-pipenv-{{ checksum "Pipfile.lock" }}
       - run: pre-commit install-hooks
       - run:
           name: Run pre-commit tests
           command: pre-commit run --all-files
+      - run:
+          name: install dependencies
+          command: |  # use pipenv to install dependencies
+            pipenv sync -d
+      - run:
+          name: run tests
+          command: |
+            pipenv run pytest --junit-xml=junit/report.xml --cov-report html:coverage_html --cov=utils --cov=tasks
+      - store_test_results:
+          path: ~/project/junit/report.xml
+      - store_artifacts:
+          path: ~/project/coverage_html
+          destination: coverage_html
       - save_cache:
           key: v1-pre-commit-dot-cache-{{ checksum ".pre-commit-config.yaml" }}
           paths:
             - ~/.cache/pre-commit
+      - save_cache:
+          key: v1-pipenv-{{ checksum "Pipfile.lock" }}
+          paths:
+            - ".venv"
 
 workflows:
   version: 2

.envrc

@@ -47,8 +47,6 @@ if [ ! -r .nix-disable  ] && has nix-env; then
   # is available on the path
   PATH_add ${NIX_PROFILE}/bin
 
-  # layout after setting PATH so we get the right python version
-  layout python3
 fi
 
 # Loads secrets from chamber instead of requiring them to be listed in .envrc.local
@@ -85,3 +83,6 @@ if [ -e .envrc.local ]
 then
   source_env .envrc.local
 fi
+
+# layout after setting PATH so we get the right python version
+layout pipenv

.pre-commit-config.yaml

@@ -1,18 +1,13 @@
 repos:
 
   - repo: https://github.com/ambv/black
-    rev: 20.8b1
+    rev: 21.10b0
     hooks:
       - id: black
-        language_version: python3.8
-        exclude: >
-          (?x)^(
-            scripts/gen-docs-index|
-          )$
-        args: ['--line-length', '120']
+        language_version: python3.9
 
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.0
+    rev: 4.0.1
     hooks:
       - id: flake8
 

Brewfile.local

@@ -4,6 +4,6 @@ brew 'direnv'
 brew 'libev'
 brew 'pre-commit'
 brew 'pyenv'
-brew 'pyenv-virtualenv'
+brew 'pipenv'
 
 cask 'aws-vault'

Dockerfile

@@ -2,6 +2,8 @@ FROM python:3.9.6
 
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
+# we do not want pipenv to create a virtualenv inside the container
+ENV PIPENV_SITE_PACKAGES 1
 
 ARG locustfile
 ENV LOCUSTFILE=$locustfile
@@ -10,9 +12,14 @@ RUN echo $LOCUSTFILE
 
 WORKDIR /app
 
-# Copy over and install requirements:
-COPY requirements.txt /app/requirements.txt
-RUN pip install -r /app/requirements.txt
+RUN pip install pipenv
+
+# Copy over and install Pipfiles and use pip to install requirements
+# outside of virtualenv
+COPY Pipfile Pipfile.lock /app/
+RUN set -x \
+    && pipenv install --system --deploy --site-packages \
+    && rm -rf /root/.local/share/virtualenv /root/.local/share/virtualenvs
 
 # Copy over everything else for the app:
 COPY docker.__init__.py /app/__init__.py

Makefile

@@ -18,17 +18,9 @@ help:  ## Print the help documentation
 install_tools:  ## Install tools needed for project.
 	scripts/install_tools
 
-.PHONY: ensure_venv
-ensure_venv:  ## Ensure that the virtualenv is activated.
-ifndef VIRTUAL_ENV
-	@echo "Virtual env not defined. If you are using nix, make sure it's not disabled. If you aren't using nix, run 'make setup'."
-	false
-endif
-
 .PHONY: install_python_deps
-install_python_deps: ensure_venv ## Install all python dependencies/requirements
-	pip install -r requirements.txt
-	pip install -r requirements-dev.txt
+install_python_deps: ## Install all python dependencies/requirements
+	pipenv install
 
 .PHONY: ensure_pre_commit
 ensure_pre_commit: .git/hooks/pre-commit  ## Ensure pre-commit is installed
@@ -53,11 +45,11 @@ clean:  ## Clean all generated files
 	find ./ -type f -name '*.pyc' -delete
 
 .PHONY: pretty
-pretty: ensure_venv  ## Prettify the code
+pretty: ## Prettify the code
 	black .
 
 .PHONY: lint
-lint: ensure_venv  ## Run linting tests
+lint: ## Run linting tests
 	flake8 .
 
 .PHONY: generate_readme_toc

Pipfile

@@ -0,0 +1,24 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+requests = ">=2.22.0"
+locust = ">=1"
+prance = {version = "==0.19.0", extras = ["cli", "osv"]}
+Faker = "*"
+
+[dev-packages]
+# pipenv doesn't resolve pre-release versions, so manually pin
+black = "==21.10b0"
+flake8 = "*"
+pytest = "*"
+pytest-mock = "*"
+cryptography = "*"
+responses = "*"
+python-lsp-server = "*"
+pytest-cov = "*"
+
+[requires]
+python_version = "3.9"