Add docker support, start supporting views on relations

Author: gsoltis

.gitignore

@@ -2,4 +2,7 @@
 __pycache__/
 *.pyc
 /sample_queries
-requirements.txt
+/requirements.txt
+/pg_data
+/gf
+dist/

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.7-slim
+
+COPY requirements.txt /app/
+WORKDIR /app/
+RUN pip install -r requirements.txt
+EXPOSE 5000/tcp
+COPY goldfig.py /app/
+COPY goldfig /app/goldfig
+LABEL goldfig-cli=0.0.1
+
+ENTRYPOINT ["/app/goldfig.py", "serve"]

Pipfile

@@ -25,6 +25,7 @@ tabulate = "*"
 jsonschema = "*"
 click = "*"
 flask = "*"
+requests = "*"
 
 [requires]
 python_version = "3.7"

Pipfile.lock

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+PACKAGE=`basename $DIR`
+
+echo "Building package ${PACKAGE}"
+
+pipenv lock -r > requirements.txt
+
+GOLDFIG_DOCKER_REPO=${DOCKER_REPO:-goldfig}
+IMAGE="${GOLDFIG_DOCKER_REPO}/${PACKAGE}"
+docker build -t ${IMAGE} .
+
+echo "Building launcher"
+launcher/build.sh
+
+mkdir -p dist
+
+ESCAPED_IMAGE=$(printf '%s\n' "$IMAGE" | sed -e 's/[\/&]/\\&/g')
+sed "s/build: ./image: ${ESCAPED_IMAGE}:latest/g" docker-compose.yml > dist/docker-compose.yml
+cp launcher/dist/* dist/
+
+cd dist
+# Build linux package
+ln gf_linux gf
+zip goldfig_linux.zip gf docker-compose.yml
+unlink gf
+
+# Build osx package
+ln gf_osx gf
+zip goldfig_osx.zip gf docker-compose.yml
+unlink gf
+
+echo "To publish"
+echo "docker push ${IMAGE}:latest"

build.sh

@@ -0,0 +1,24 @@
+version: "3.8"
+services:
+  goldfig:
+    build: .
+    init: true
+    environment:
+      - GOLDFIG_DB_HOST=db
+      - GOLDFIG_DB_SU_USER=postgres
+      - GOLDFIG_DB_SU_PASSWORD=postgres
+    depends_on:
+      - db
+  db:
+    image: postgres
+    environment:
+      - POSTGRES_DB=postgres
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+volumes:
+  pg_data:
+    driver: local

docker-compose.yml

@@ -3,7 +3,7 @@
 import logging
 import os
 import textwrap
-from typing import Callable, Dict, Generator, List, Optional, Tuple
+from typing import Any, Callable, Dict, Generator, List, Optional, Tuple
 
 from botocore.loaders import create_loader
 from botocore.regions import EndpointResolver
@@ -162,7 +162,7 @@ def build_graph(parent_id: str, path: List[str]):
   }
 
 
-def load_boto_session_from_config(config: Dict) -> boto.Session:
+def load_boto_session_from_config(config: Dict[str, Any]) -> boto.Session:
   if config.get('from_environment', False):
     session = boto.get_session()
   else:

gf

@@ -73,7 +73,7 @@ resources:
           id:
             path: Ebs.VolumeId
         attributes:
-          DeleteOnTermiation:
+          DeleteOnTermination:
             path: Ebs.DeleteOnTermination
           AttachTime:
             path: Ebs.AttachTime

goldfig/aws/__init__.py

@@ -4,15 +4,19 @@
 from typing import List, Tuple
 
 # import psycopg2.errors
+import psycopg2
+from psycopg2 import sql
 from sqlalchemy import create_engine, text
+from sqlalchemy.exc import OperationalError
 from sqlalchemy.orm import sessionmaker, Session
 
 from goldfig import models
 
 _log = logging.getLogger(__name__)
 
 DBNAME = 'goldfig'
-HOST = 'localhost'
+HOST = os.environ.get('GOLDFIG_DB_HOST', 'localhost')
+PORT = int(os.environ.get('GOLDFIG_DB_PORT', 5432))
 
 
 @dataclass
@@ -29,12 +33,12 @@ def connection_string(self) -> str:
 _ImportCredential = DbCredential(db_name=DBNAME,
                                  user='goldfig',
                                  password='goldfig',
-                                 host=HOST)
+                                 host=f'{HOST}:5432')
 
 _ReadonlyCredential = DbCredential(db_name=DBNAME,
                                    user='goldfig_ro',
                                    password='goldfig_ro',
-                                   host=HOST)
+                                   host=f'{HOST}:5432')
 
 _import_engine = None
 _readonly_engine = None
@@ -43,8 +47,7 @@ def connection_string(self) -> str:
 
 
 def _view_files() -> Tuple[str, List[str]]:
-  path = os.path.realpath(
-      os.path.join(os.path.dirname(__file__), 'views'))
+  path = os.path.realpath(os.path.join(os.path.dirname(__file__), 'views'))
   files = [
       f for f in os.listdir(path)
       if os.path.isfile(os.path.join(path, f)) and f[-4:] == '.sql'
@@ -76,9 +79,7 @@ def _install_views(db: Session):
         raise
 
 
-# TODO: switch to something like alembic?
-def init_db():
-  db = import_session()
+def _install_schema(db: Session) -> None:
   version = None
   try:
     version = db.query(models.SchemaVersion).one_or_none()
@@ -100,10 +101,79 @@ def init_db():
   db.commit()
 
 
+def _install_db_and_roles():
+  print('creating goldfig database and installing roles')
+  cred = DbCredential(db_name='postgres',
+                      host=HOST,
+                      user=os.environ.get('GOLDFIG_DB_SU_USER', 'postgres'),
+                      password=os.environ.get('GOLDFIG_DB_SU_PASSWORD',
+                                              'postgres'))
+  su_conn = psycopg2.connect(dbname=cred.db_name,
+                             user=cred.user,
+                             password=cred.password,
+                             host=cred.host)
+  su_conn.autocommit = True
+  cursor = su_conn.cursor()
+  cursor.execute('CREATE DATABASE goldfig')
+  cursor.close()
+  su_conn.autocommit = False
+  cursor = su_conn.cursor()
+  for user in (_ImportCredential, _ReadonlyCredential):
+    cursor.execute(
+        sql.SQL('CREATE USER {} WITH ENCRYPTED PASSWORD %s').format(
+            sql.Identifier(user.user)), (user.password, ))
+
+  su_conn.commit()
+  su_conn.close()
+
+  cred = DbCredential(db_name='goldfig',
+                      host=cred.host,
+                      user=cred.user,
+                      password=cred.password)
+  su_conn = psycopg2.connect(dbname=cred.db_name,
+                             user=cred.user,
+                             password=cred.password,
+                             host=cred.host)
+  su_conn.autocommit = False
+  cursor = su_conn.cursor()
+  import_user = sql.Identifier(_ImportCredential.user)
+  ro_user = sql.Identifier(_ReadonlyCredential.user)
+  cursor.execute('revoke create on schema public from public')
+  cursor.execute(
+      sql.SQL('grant all privileges on schema public to {}').format(
+          import_user))
+  cursor.execute(
+      sql.SQL('grant select on all tables in schema public to {}').format(
+          ro_user))
+  cursor.execute(
+      sql.SQL(
+          'alter default privileges for role {} in schema public grant select on tables to {}'
+      ).format(import_user, ro_user))
+  cursor.close()
+  su_conn.commit()
+
+
+# TODO: switch to something like alembic?
+def init_db():
+  try:
+    db = import_session()
+  except OperationalError as e:
+    if 'password authentication failed' in str(e):
+      # need to set up db
+      _install_db_and_roles()
+      db = import_session()
+    else:
+      raise
+  _install_schema(db)
+
+
 def import_session() -> Session:
   global _import_engine
   if _import_engine is None:
-    _import_engine = create_engine(_ImportCredential.connection_string())
+    _import_engine = create_engine(_ImportCredential.connection_string(),
+                                   connect_args={'connect_timeout': 3})
+    # Force connection errors early
+    _import_engine.connect()
   return sessionmaker(bind=_import_engine)()