Add service column to raw_import

ec2 imports now pass region in context, relation mapping takes a context, add support for ec2 security groups

ec2 instances now have a relation to security groups

Added Key Pairs for AWS EC2.

Move transforms into service folders

Move keypairs into ec2 folder

Fixed KeyPair view.

Filter disabled regions.

Adding Lamba support.

Merge and add db clusters

Add async rds support

Fix aws organizations mapping, uncomment all resources for debug mode

Remove old compose file, we don't need it anymore. Instead, we use sed in build.sh to construct it

Fix aws arn function

Refactor transform specs partially into dataclasses

Image from lightkeeper

Added support to serve schema docs.

Drop PsaswordPolicy from account in organizations

Completed lambda support.

ship aws view and transforms from lightkeeper

light typing

Fix up some typing

Added sample query for checking RDS backups disabled.

Added VPC generated files.

Add relations for instance, add vpcs

Make certain correct db session is used

Drop prints outside cli

Author: gsoltis

Dockerfile

@@ -6,6 +6,7 @@ RUN pip install -r requirements.txt
 EXPOSE 5000/tcp
 COPY goldfig.py /app/
 COPY goldfig /app/goldfig
+COPY schema-docs /app/schema-docs
 LABEL goldfig-cli=0.0.1
 
 ENTRYPOINT ["/app/goldfig.py", "serve"]

Pipfile.lock

@@ -9,6 +9,8 @@ services:
       - GOLDFIG_DB_SU_PASSWORD=postgres
     depends_on:
       - db
+    ports:
+      - "5000:5000"
   db:
     image: postgres
     environment:

docker-compose.yml

@@ -38,10 +38,12 @@ def tail(self) -> str:
 
 
 class ImportWriter:
-  def __init__(self, db: Session, import_job_id: int, phase: int):
+  def __init__(self, db: Session, import_job_id: int, service: str,
+               phase: int):
     self._db = db
     self._import_job_id = import_job_id
     self._phase = phase
+    self._service = service
 
   def __call__(self,
                path: Union[PathStack, str],
@@ -53,16 +55,17 @@ def __call__(self,
     _log.info(f'writing {path} - {resource_name}')
     model = RawImport(import_job_id=self._import_job_id,
                       path=path,
+                      service=self._service,
                       resource_name=resource_name,
                       raw=raw,
                       context=context,
                       phase=self._phase)
     self._db.add(model)
 
 
-def db_import_writer(db: Session, import_job_id: int,
+def db_import_writer(db: Session, import_job_id: int, service: str,
                      phase: int) -> ImportWriter:
-  return ImportWriter(db, import_job_id, phase)
+  return ImportWriter(db, import_job_id, service, phase)
 
 
 def collect_exceptions(results: List[f.Future]) -> List[str]:

goldfig/__init__.py

@@ -1,15 +1,17 @@
 import concurrent.futures as f
 import logging
 import os
-from typing import Any, Callable, Dict, Generator, List, Optional, Tuple
+from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Tuple
 
 from botocore.loaders import create_loader
 from botocore.regions import EndpointResolver
+from botocore.exceptions import ClientError
 import botocore.session as boto
 from sqlalchemy.orm import Session
 
 from goldfig import collect_exceptions, PathStack
 from goldfig.aws.fetch import Proxy
+from goldfig.error import GFInternal
 from goldfig.models import ImportJob, ProviderAccount, ProviderCredential
 
 ProxyBuilder = Callable[[boto.Session], Proxy]
@@ -99,21 +101,29 @@ def build_aws_import_job(db: Session, proxy_builder: ProxyBuilder,
   }
 
 
+def _require_resp(tup: Optional[Tuple[str, Dict[str, Any]]]) -> Dict[str, Any]:
+  if tup is None:
+    raise GFInternal(f'Missing response')
+  else:
+    return tup[1]
+
+
 def _build_org_graph(proxy: Proxy):
   org = proxy.service('organizations')
   org_resp = org.get('describe_organization')['Organization']
-  _, roots_resp = org.list('list_roots')
+  roots_resp = _require_resp(org.list('list_roots'))
   roots = roots_resp['Roots']
   accounts = {}
   organizational_units = {}
 
   def build_graph(parent_id: str, path: List[str]):
-    _, accounts_resp = org.list('list_accounts_for_parent', ParentId=parent_id)
+    accounts_resp = _require_resp(
+        org.list('list_accounts_for_parent', ParentId=parent_id))
     next_path = [*path, parent_id]
     path_str = '/'.join(next_path)
     accounts[path_str] = accounts_resp['Accounts']
-    _, organizational_units_resp = org.list(
-        'list_organizational_units_for_parent', ParentId=parent_id)
+    organizational_units_resp = _require_resp(
+        org.list('list_organizational_units_for_parent', ParentId=parent_id))
     ous = organizational_units_resp['OrganizationalUnits']
     if len(ous) > 0:
       organizational_units[path_str] = ous
@@ -146,16 +156,29 @@ def load_boto_session(provider_credential: ProviderCredential) -> boto.Session:
   return load_boto_session_from_config(config)
 
 
-def get_region_fn() -> Callable[[str], List[str]]:
+def region_is_available(region):
+  # https://www.cloudar.be/awsblog/checking-if-a-region-is-enabled-using-the-aws-api/
+  # https://stackoverflow.com/a/56184952
+  regional_sts = get_boto_session().create_client('sts', region_name=region)
+  try:
+    regional_sts.get_caller_identity()
+    return True
+  except ClientError:
+    return False
+
+
+def get_region_fn() -> Callable[[str], Iterator[str]]:
   data_dir = os.path.join(os.path.dirname(boto.__file__), 'data')
   loader = create_loader(data_dir)
   endpoint_data = loader.load_data('endpoints')
   endpoints = EndpointResolver(endpoint_data)
 
-  def get_regions_for_service(service: str) -> List[str]:
+  def get_regions_for_service(service: str) -> Iterator[str]:
     sm = loader.load_service_model(service, type_name='service-2')
     prefix = sm['metadata'].get('endpointPrefix', service)
-    return endpoints.get_available_endpoints(prefix, partition_name='aws')
+    return filter(
+        lambda region: region_is_available(region),
+        endpoints.get_available_endpoints(prefix, partition_name='aws'))
 
   return get_regions_for_service
 
@@ -166,8 +189,15 @@ def run_parallel_session(accounts: List[Tuple[str, ProviderCredential]],
   from goldfig.aws.iam import import_account_iam_with_pool
   from goldfig.aws.ec2 import import_account_ec2_region_with_pool
   from goldfig.aws.elb import import_account_elb_region_with_pool
+  from goldfig.aws.rds import import_account_rds_region_with_pool
   from goldfig.aws.s3 import import_account_s3_with_pool
-  workers = max(1, os.cpu_count() - 1)
+  from goldfig.aws.lambdax import import_account_lambda_region_with_pool
+
+  cpu_count = os.cpu_count()
+  if cpu_count is not None:
+    workers = max(1, cpu_count - 1)
+  else:
+    workers = 1
   ps = PathStack.from_import_job(import_job)
   get_region_for_service = get_region_fn()
   with f.ProcessPoolExecutor(max_workers=workers) as pool:
@@ -183,6 +213,13 @@ def run_parallel_session(accounts: List[Tuple[str, ProviderCredential]],
                                                      accounts)
     results += import_account_s3_with_pool(pool, proxy_builder_args,
                                            import_job.id, ps, accounts)
+    for region in get_region_for_service('rds'):
+      results += import_account_rds_region_with_pool(pool, proxy_builder_args,
+                                                     import_job.id, region, ps,
+                                                     accounts)
+    for region in get_region_for_service('lambda'):
+      results += import_account_lambda_region_with_pool(
+          pool, proxy_builder_args, import_job.id, region, ps, accounts)
     f.wait(results)
     # raise any exceptions
     return collect_exceptions(results)
@@ -194,6 +231,9 @@ def run_single_session(db: Session, import_job_id: int,
   from goldfig.aws.ec2 import import_account_ec2_region_to_db
   from goldfig.aws.elb import import_account_elb_region_to_db
   from goldfig.aws.s3 import import_account_s3_to_db
+  from goldfig.aws.lambdax import import_account_lambda_region_to_db
+  from goldfig.aws.rds import import_account_rds_region_to_db
+
   import_account_iam_to_db(db, import_job_id, proxy_builder)
   db.flush()
   get_region_for_service = get_region_fn()
@@ -203,6 +243,14 @@ def run_single_session(db: Session, import_job_id: int,
   for region in get_region_for_service('elb'):
     import_account_elb_region_to_db(db, import_job_id, region, proxy_builder)
     db.flush()
+  for region in get_region_for_service('lambda'):
+    import_account_lambda_region_to_db(db, import_job_id, region,
+                                       proxy_builder)
+    db.flush()
+  for region in get_region_for_service('rds'):
+    import_account_rds_region_to_db(db, import_job_id, region, proxy_builder)
+    db.flush()
+
   import_account_s3_to_db(db, import_job_id, proxy_builder)
 
 
@@ -222,15 +270,15 @@ def account_paths_for_import(
   def find_credential(account_id: str) -> ProviderCredential:
     return next(cred for cred in creds if cred.scope == account_id)
 
-  account_paths = import_job.configuration['aws_graph']['accounts']
+  account_paths = import_job.aws_config.graph.accounts
   results = []
   for path, accounts in account_paths.items():
     for account in accounts:
       try:
-        cred = find_credential(account['Id'])
+        cred = find_credential(account.id)
         results.append((f'{path}/{cred.scope}', cred))
       except StopIteration:
         # If we don't have credentials, don't import it
-        _log.info(f'Skipping account id {account["Id"]}, no credentials')
+        _log.info(f'Skipping account id {account.id}, no credentials')
         continue
   return results

goldfig/aws/__init__.py

@@ -6,8 +6,8 @@
 
 from goldfig import ImportWriter, db_import_writer, PathStack
 from goldfig.aws import (account_paths_for_import, load_boto_session,
-                            ProxyBuilder, make_proxy_builder,
-                            load_boto_session_from_config)
+                         ProxyBuilder, make_proxy_builder,
+                         load_boto_session_from_config)
 from goldfig.aws.fetch import Proxy, ServiceProxy
 from goldfig.bootstrap_db import import_session
 from goldfig.models import ImportJob, ProviderCredential
@@ -28,7 +28,7 @@ def _import_ec2_region(
 def import_account_ec2_region_to_db(db: Session, import_job_id: int,
                                     region: str, proxy_builder: ProxyBuilder):
   job: ImportJob = db.query(ImportJob).get(import_job_id)
-  writer = db_import_writer(db, job.id, phase=0)
+  writer = db_import_writer(db, job.id, 'ec2', phase=0)
   for path, account in account_paths_for_import(db, job):
     boto = load_boto_session(account)
     proxy = proxy_builder(boto)
@@ -41,7 +41,7 @@ def _import_ec2_region_to_db(proxy: Proxy, writer: ImportWriter, ps: PathStack,
   service_proxy = proxy.service('ec2', region)
   ps = ps.scope(region)
   for resource_name, raw_resources in _import_ec2_region(service_proxy):
-    writer(ps, resource_name, raw_resources)
+    writer(ps, resource_name, raw_resources, {'region': region})
 
 
 def _async_proxy(ps: PathStack, proxy_builder_args, import_job_id: int,
@@ -50,7 +50,7 @@ def _async_proxy(ps: PathStack, proxy_builder_args, import_job_id: int,
   proxy_builder = make_proxy_builder(*proxy_builder_args)
   boto = load_boto_session_from_config(config)
   proxy = proxy_builder(boto)
-  writer = db_import_writer(db, import_job_id, phase=0)
+  writer = db_import_writer(db, import_job_id, 'ec2', phase=0)
   f(proxy, writer, ps, region)
   db.commit()
 

goldfig/aws/ec2.py

@@ -5,8 +5,8 @@
 
 from goldfig import ImportWriter, db_import_writer, PathStack
 from goldfig.aws import (load_boto_session, account_paths_for_import,
-                            ProxyBuilder, make_proxy_builder,
-                            load_boto_session_from_config)
+                         ProxyBuilder, make_proxy_builder,
+                         load_boto_session_from_config)
 from goldfig.aws.fetch import Proxy, ServiceProxy
 from goldfig.bootstrap_db import import_session
 from goldfig.models import ImportJob, ProviderCredential
@@ -54,7 +54,7 @@ def _async_proxy(ps: PathStack, proxy_builder_args, import_job_id: int,
   proxy_builder = make_proxy_builder(*proxy_builder_args)
   boto = load_boto_session_from_config(config)
   proxy = proxy_builder(boto)
-  writer = db_import_writer(db, import_job_id, phase=0)
+  writer = db_import_writer(db, import_job_id, 'elb', phase=0)
   f(proxy, writer, ps, region)
   db.commit()
 
@@ -82,7 +82,7 @@ def queue_job(fn, path: str, account: ProviderCredential):
 def import_account_elb_region_to_db(db: Session, import_job_id: int,
                                     region: str, proxy_builder: ProxyBuilder):
   job: ImportJob = db.query(ImportJob).get(import_job_id)
-  writer = db_import_writer(db, job.id, phase=0)
+  writer = db_import_writer(db, job.id, 'elb', phase=0)
   for path, account in account_paths_for_import(db, job):
     boto = load_boto_session(account)
     proxy = proxy_builder(boto)