Add support for new resources:

 - apigatewayv2
 - cloudfront distribution
 - ACM
 - redshift cluster
 - SQS
 - ECS
Fix view bugs, update data types to be more useful

Author: gsoltis

goldfig/aws/acm.py

@@ -0,0 +1,38 @@
+from dis import dis
+import logging
+from typing import Any, Dict, Generator, Tuple
+
+from goldfig.aws.fetch import ServiceProxy
+from goldfig.aws.svc import make_import_to_db, make_import_with_pool
+
+_log = logging.getLogger(__name__)
+
+
+def _import_certificate(proxy: ServiceProxy, summary: Dict) -> Dict[str, Any]:
+  arn = summary['CertificateArn']
+  certificate = proxy.get('describe_certificate',
+                          CertificateArn=arn)['Certificate']
+  tags_resp = proxy.list('list_tags_for_certificate', CertificateArn=arn)
+  if tags_resp is not None:
+    certificate['Tags'] = tags_resp[1]['Tags']
+  return certificate
+
+
+def _import_certificates(proxy: ServiceProxy, region: str):
+  certificates_resp = proxy.list('list_certificates')
+  if certificates_resp is not None:
+    certificates = certificates_resp[1]['CertificateSummaryList']
+    for certificate in certificates:
+      yield 'Certificate', _import_certificate(proxy, certificate)
+
+
+def _import_acm_region(proxy: ServiceProxy,
+                       region: str) -> Generator[Tuple[str, Any], None, None]:
+  _log.info(f'importing Certificates')
+  yield from _import_certificates(proxy, region)
+
+
+import_account_acm_region_to_db = make_import_to_db('acm', _import_acm_region)
+
+import_account_acm_region_with_pool = make_import_with_pool(
+    'acm', _import_acm_region)

goldfig/aws/apigatewayv2.py

@@ -0,0 +1,41 @@
+from dis import dis
+import logging
+from typing import Any, Dict, Generator, Tuple
+
+from goldfig.aws.fetch import ServiceProxy
+from goldfig.aws.svc import make_import_to_db, make_import_with_pool
+
+_log = logging.getLogger(__name__)
+
+ApiResources = ['stages', 'routes', 'integrations', 'models']
+
+
+def _import_api(proxy: ServiceProxy, api: Dict) -> Dict[str, Any]:
+  api_id = api['ApiId']
+  for resource in ApiResources:
+    resp = proxy.list('get_' + resource, ApiId=api_id)
+    if resp is not None:
+      api[resource.capitalize()] = resp[1]['Items']
+  return api
+
+
+def _import_apis(proxy: ServiceProxy, region: str):
+  apis_resp = proxy.list('get_apis')
+  if apis_resp is not None:
+    apis = apis_resp[1]['Items']
+    for api in apis:
+      yield 'Api', _import_api(proxy, api)
+
+
+def _import_apigatewayv2_region(
+    proxy: ServiceProxy,
+    region: str) -> Generator[Tuple[str, Any], None, None]:
+  _log.info(f'importing Apis')
+  yield from _import_apis(proxy, region)
+
+
+import_account_apigatewayv2_region_to_db = make_import_to_db(
+    'apigatewayv2', _import_apigatewayv2_region)
+
+import_account_apigatewayv2_region_with_pool = make_import_with_pool(
+    'apigatewayv2', _import_apigatewayv2_region)

goldfig/aws/cloudfront.py

@@ -0,0 +1,42 @@
+from dis import dis
+import logging
+from typing import Any, Dict, Generator, Tuple
+
+from goldfig.aws.fetch import ServiceProxy
+from goldfig.aws.svc import make_import_to_db, make_import_with_pool
+
+_log = logging.getLogger(__name__)
+
+
+def _import_distribution(proxy: ServiceProxy, summary: Dict) -> Dict[str, Any]:
+  arn = summary['ARN']
+  distribution_id = summary['Id']
+  distribution = proxy.get('get_distribution', Id=distribution_id)
+  config = proxy.get('get_distribution_config', Id=distribution_id)
+  distribution.update(config)
+  tags_resp = proxy.list('list_tags_for_resource', Resource=arn)
+  if tags_resp is not None:
+    distribution['Tags'] = tags_resp[1]['Tags']
+  return distribution
+
+
+def _import_distributions(proxy: ServiceProxy, region: str):
+  distributions_resp = proxy.list('list_distributions')
+  if distributions_resp is not None:
+    distributions = distributions_resp[1]['DistributionList']['Items']
+    for summary in distributions:
+      yield 'Distribution', _import_distribution(proxy, summary)
+
+
+def _import_cloudfront_region(
+    proxy: ServiceProxy,
+    region: str) -> Generator[Tuple[str, Any], None, None]:
+  _log.info(f'importing distributions')
+  yield from _import_distributions(proxy, region)
+
+
+import_account_cloudfront_region_to_db = make_import_to_db(
+    'cloudfront', _import_cloudfront_region)
+
+import_account_cloudfront_region_with_pool = make_import_with_pool(
+    'cloudfront', _import_cloudfront_region)

goldfig/aws/ecs.py

@@ -0,0 +1,106 @@
+import logging
+from typing import Any, Dict, Generator, Tuple
+
+from goldfig.aws.fetch import ServiceProxy
+from goldfig.aws.svc import make_import_to_db, make_import_with_pool
+from goldfig.error import GFInternal
+
+_log = logging.getLogger(__name__)
+
+
+def _import_cluster(proxy: ServiceProxy, cluster_arn: str) -> Dict[str, Any]:
+  clusters_resp = proxy.list(
+      'describe_clusters',
+      clusters=[cluster_arn],
+      include=["ATTACHMENTS", "SETTINGS", "STATISTICS", "TAGS"])
+  if clusters_resp is None:
+    raise GFInternal(f'Failed to fetch ecs cluster {cluster_arn}')
+  cluster_list = clusters_resp[1].get('clusters', [])
+  if len(cluster_list) != 1:
+    raise GFInternal(
+        f'Wrong number of clusters for {cluster_arn} {clusters_resp}')
+  cluster = cluster_list[0]
+  return cluster
+
+
+def _import_service(proxy: ServiceProxy, cluster_arn: str, service_arn: str):
+  services_resp = proxy.list('describe_services',
+                             cluster=cluster_arn,
+                             services=[service_arn],
+                             include=['TAGS'])
+  if services_resp is None:
+    raise GFInternal(f'Failed to fetch ecs service {service_arn}')
+  service_list = services_resp[1].get('services', [])
+  if len(service_list) != 1:
+    raise GFInternal(
+        f'Wrong number of services for {service_arn} {services_resp}')
+  service = service_list[0]
+  return service
+
+
+def _import_services(proxy: ServiceProxy, cluster_arn: str):
+  services_resp = proxy.list('list_services', cluster=cluster_arn)
+  if services_resp is not None:
+    service_arns = services_resp[1].get('serviceArns', [])
+    for service_arn in service_arns:
+      yield 'Service', _import_service(proxy, cluster_arn, service_arn)
+
+
+def _import_task(proxy: ServiceProxy, cluster_arn: str, task_arn: str):
+  tasks_resp = proxy.list('describe_tasks',
+                          cluster=cluster_arn,
+                          tasks=[task_arn],
+                          include=['TAGS'])
+  if tasks_resp is None:
+    raise GFInternal(f'Failed to fetch ecs task {task_arn}')
+  task_list = tasks_resp[1].get('tasks', [])
+  if len(task_list) != 1:
+    raise GFInternal(f'Wrong number of tasks for {task_arn} {tasks_resp}')
+  task = task_list[0]
+  return task
+
+
+def _import_tasks(proxy: ServiceProxy, cluster_arn: str):
+  tasks_resp = proxy.list('list_tasks', cluster=cluster_arn)
+  if tasks_resp is not None:
+    task_arns = tasks_resp[1].get('taskArns', [])
+    for task_arn in task_arns:
+      yield 'Task', _import_task(proxy, cluster_arn, task_arn)
+
+
+def _import_clusters(proxy: ServiceProxy, region: str):
+  clusters_resp = proxy.list('list_clusters')
+  if clusters_resp is not None:
+    cluster_arns = clusters_resp[1].get('clusterArns', [])
+    for cluster_arn in cluster_arns:
+      yield 'Cluster', _import_cluster(proxy, cluster_arn)
+      yield from _import_services(proxy, cluster_arn)
+      yield from _import_tasks(proxy, cluster_arn)
+
+
+def _import_task_definition(proxy: ServiceProxy, definition_arn: str):
+  definition = proxy.get('describe_task_definition',
+                         taskDefinition=definition_arn,
+                         include=['TAGS'])['taskDefinition']
+  return definition
+
+
+def _import_task_definitions(proxy: ServiceProxy):
+  definitions_resp = proxy.list('list_task_definitions')
+  if definitions_resp is not None:
+    definition_arns = definitions_resp[1].get('taskDefinitionArns', [])
+    for definition_arn in definition_arns:
+      yield 'TaskDefinition', _import_task_definition(proxy, definition_arn)
+
+
+def _import_ecs_region(proxy: ServiceProxy,
+                       region: str) -> Generator[Tuple[str, Any], None, None]:
+  _log.info(f'importing ecs clusters')
+  yield from _import_clusters(proxy, region)
+  yield from _import_task_definitions(proxy)
+
+
+import_account_ecs_region_to_db = make_import_to_db('ecs', _import_ecs_region)
+
+import_account_ecs_region_with_pool = make_import_with_pool(
+    'ecs', _import_ecs_region)

goldfig/aws/fetch.py

@@ -159,6 +159,7 @@ def list(self, key: str, kwargs) -> Optional[Tuple[str, Any]]:
         result = {
             result_key: full_result[result_key]
             for result_key in output.members.keys()
+            if result_key in full_result
         }
       return resource_name, result
     except KeyError as e:

goldfig/aws/importer.py

@@ -6,19 +6,25 @@
 
 from goldfig import collect_exceptions, PathStack
 from goldfig.aws import ProxyBuilder
+from goldfig.aws.acm import import_account_acm_region_to_db, import_account_acm_region_with_pool
+from goldfig.aws.apigatewayv2 import import_account_apigatewayv2_region_to_db, import_account_apigatewayv2_region_with_pool
 from goldfig.aws.iam import import_account_iam_to_db, import_account_iam_with_pool
 from goldfig.aws.ec2 import import_account_ec2_region_to_db, import_account_ec2_region_with_pool
+from goldfig.aws.ecs import import_account_ecs_region_to_db, import_account_ecs_region_with_pool
 from goldfig.aws.elb import import_account_elb_region_to_db, import_account_elb_region_with_pool
 from goldfig.aws.s3 import import_account_s3_to_db, import_account_s3_with_pool
 from goldfig.aws.kms import import_account_kms_region_to_db, import_account_kms_region_with_pool
 from goldfig.aws.lambdax import import_account_lambda_region_to_db, import_account_lambda_region_with_pool
 from goldfig.aws.logs import import_account_logs_region_to_db, import_account_logs_region_with_pool
+from goldfig.aws.cloudfront import import_account_cloudfront_region_to_db, import_account_cloudfront_region_with_pool
 from goldfig.aws.cloudtrail import import_account_cloudtrail_region_to_db, import_account_cloudtrail_region_with_pool
 from goldfig.aws.cloudwatch import import_account_cloudwatch_region_to_db, import_account_cloudwatch_region_with_pool
 from goldfig.aws.config import import_account_config_region_to_db, import_account_config_region_with_pool
 from goldfig.aws.region import RegionCache
 from goldfig.aws.rds import import_account_rds_region_to_db, import_account_rds_region_with_pool
+from goldfig.aws.redshift import import_account_redshift_region_to_db, import_account_redshift_region_with_pool
 from goldfig.aws.sns import import_account_sns_region_to_db, import_account_sns_region_with_pool
+from goldfig.aws.sqs import import_account_sqs_region_to_db, import_account_sqs_region_with_pool
 from goldfig.models import ImportJob, ProviderCredential
 
 
@@ -32,6 +38,10 @@ def run_single_session(db: Session, import_job_id: int,
     import_account_ec2_region_to_db(db, import_job_id, region, proxy_builder)
     db.flush()
 
+  for region in region_cache.regions_for_service('ecs'):
+    import_account_ecs_region_to_db(db, import_job_id, region, proxy_builder)
+    db.flush()
+
   for region in region_cache.regions_for_service('kms'):
     import_account_kms_region_to_db(db, import_job_id, region, proxy_builder)
     db.flush()
@@ -70,6 +80,29 @@ def run_single_session(db: Session, import_job_id: int,
     import_account_sns_region_to_db(db, import_job_id, region, proxy_builder)
     db.flush()
 
+  for region in region_cache.regions_for_service('sqs'):
+    import_account_sqs_region_to_db(db, import_job_id, region, proxy_builder)
+    db.flush()
+
+  for region in region_cache.regions_for_service('cloudfront'):
+    import_account_cloudfront_region_to_db(db, import_job_id, region,
+                                           proxy_builder)
+    db.flush()
+
+  for region in region_cache.regions_for_service('apigatewayv2'):
+    import_account_apigatewayv2_region_to_db(db, import_job_id, region,
+                                             proxy_builder)
+    db.flush()
+
+  for region in region_cache.regions_for_service('acm'):
+    import_account_acm_region_to_db(db, import_job_id, region, proxy_builder)
+    db.flush()
+
+  for region in region_cache.regions_for_service('redshift'):
+    import_account_redshift_region_to_db(db, import_job_id, region,
+                                         proxy_builder)
+    db.flush()
+
   import_account_s3_to_db(db, import_job_id, proxy_builder)
 
 
@@ -92,6 +125,11 @@ def run_parallel_session(region_cache: RegionCache,
                                                      import_job.id, region, ps,
                                                      accounts)
 
+    for region in region_cache.regions_for_service('ecs'):
+      results += import_account_ecs_region_with_pool(pool, proxy_builder_args,
+                                                     import_job.id, region, ps,
+                                                     accounts)
+
     for region in region_cache.regions_for_service('kms'):
       results += import_account_kms_region_with_pool(pool, proxy_builder_args,
                                                      import_job.id, region, ps,
@@ -133,6 +171,27 @@ def run_parallel_session(region_cache: RegionCache,
                                                      import_job.id, region, ps,
                                                      accounts)
 
+    for region in region_cache.regions_for_service('sqs'):
+      results += import_account_sqs_region_with_pool(pool, proxy_builder_args,
+                                                     import_job.id, region, ps,
+                                                     accounts)
+
+    for region in region_cache.regions_for_service('cloudfront'):
+      results += import_account_cloudfront_region_with_pool(
+          pool, proxy_builder_args, import_job.id, region, ps, accounts)
+
+    for region in region_cache.regions_for_service('apigatewayv2'):
+      results += import_account_apigatewayv2_region_with_pool(
+          pool, proxy_builder_args, import_job.id, region, ps, accounts)
+
+    for region in region_cache.regions_for_service('acm'):
+      results += import_account_acm_region_with_pool(pool, proxy_builder_args,
+                                                     import_job.id, region, ps,
+                                                     accounts)
+
+    for region in region_cache.regions_for_service('redshift'):
+      results += import_account_redshift_region_with_pool(
+          pool, proxy_builder_args, import_job.id, region, ps, accounts)
     f.wait(results)
     # raise any exceptions
     return collect_exceptions(results)

goldfig/aws/redshift.py

@@ -0,0 +1,34 @@
+from dis import dis
+import logging
+from typing import Any, Dict, Generator, Tuple
+
+from goldfig.aws.fetch import ServiceProxy
+from goldfig.aws.svc import make_import_to_db, make_import_with_pool
+
+_log = logging.getLogger(__name__)
+
+
+def _import_cluster(proxy: ServiceProxy, cluster: Dict) -> Dict[str, Any]:
+  return cluster
+
+
+def _import_clusters(proxy: ServiceProxy, region: str):
+  clusters_resp = proxy.list('describe_clusters')
+  if clusters_resp is not None:
+    clusters = clusters_resp[1]['Clusters']
+    for cluster in clusters:
+      yield 'Cluster', _import_cluster(proxy, cluster)
+
+
+def _import_redshift_region(
+    proxy: ServiceProxy,
+    region: str) -> Generator[Tuple[str, Any], None, None]:
+  _log.info(f'importing Redshift Clusters')
+  yield from _import_clusters(proxy, region)
+
+
+import_account_redshift_region_to_db = make_import_to_db(
+    'redshift', _import_redshift_region)
+
+import_account_redshift_region_with_pool = make_import_with_pool(
+    'redshift', _import_redshift_region)