feat: migrate aws billing lambdas to use external packaging (#158)

edersonbrilhante · web-flow · commit e847cac2da1b · 2025-10-13T17:41:40.000+03:00
* feat: migrate lambda cur_per_resource_process to use external packaging

* feat: migrate lambda cur_per_resource to use external packaging

* feat: migrate lambda cur_per_service to use external packaging
diff --git a/modules/integrations/splunk_aws_billing/README.md b/modules/integrations/splunk_aws_billing/README.md
@@ -15,11 +15,14 @@
 | Name | Version |
 |------|---------|
 | <a name="provider_aws"></a> [aws](#provider\_aws) | 6.16.0 |
-| <a name="provider_null"></a> [null](#provider\_null) | 3.2.4 |
 
 ## Modules
 
-No modules.
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_cur_per_resource"></a> [cur\_per\_resource](#module\_cur\_per\_resource) | terraform-aws-modules/lambda/aws | 8.1.0 |
+| <a name="module_cur_per_resource_process"></a> [cur\_per\_resource\_process](#module\_cur\_per\_resource\_process) | terraform-aws-modules/lambda/aws | 8.1.0 |
+| <a name="module_cur_per_service"></a> [cur\_per\_service](#module\_cur\_per\_service) | terraform-aws-modules/lambda/aws | 8.1.0 |
 
 ## Resources
 
@@ -33,9 +36,6 @@ No modules.
 | [aws_iam_policy.lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.lambda_exec_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role_policy_attachment.attach_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
-| [aws_lambda_function.cur_per_resource](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
-| [aws_lambda_function.cur_per_resource_process](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
-| [aws_lambda_function.cur_per_service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
 | [aws_lambda_permission.cur_per_resource](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
 | [aws_lambda_permission.cur_per_resource_process](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
 | [aws_lambda_permission.cur_per_service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
@@ -47,14 +47,11 @@ No modules.
 | [aws_s3_bucket_public_access_block.aws_billing_report](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
 | [aws_s3_bucket_server_side_encryption_configuration.aws_billing_report_settings](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
 | [aws_s3_bucket_versioning.aws_billing_report](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
-| [aws_s3_object.cur_per_resource](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
-| [aws_s3_object.cur_per_resource_process](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
-| [aws_s3_object.cur_per_service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
-| [null_resource.install_dependencies_per_resource](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
-| [null_resource.install_dependencies_per_resource_process](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
-| [null_resource.install_dependencies_per_service](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_iam_policy_document.cur_bucket_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.cur_per_resource](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.cur_per_resource_process](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.cur_per_service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.lambda_policy_document](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_secretsmanager_secret.secrets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/secretsmanager_secret) | data source |
 | [aws_secretsmanager_secret_version.secrets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/secretsmanager_secret_version) | data source |
diff --git a/modules/integrations/splunk_aws_billing/billing_per_resource.tf b/modules/integrations/splunk_aws_billing/billing_per_resource.tf
@@ -1,51 +1,84 @@
-resource "null_resource" "install_dependencies_per_resource" {
-  provisioner "local-exec" {
-    command = "bash ${path.module}/scripts/requirements.sh /tmp/aws_billing/per_resource/${var.aws_profile} handler_per_resource"
-  }
-
-  triggers = {
-    lambda_source_hash        = filesha256("${path.module}/lambda/handler_per_resource.py")
-    common_source_hash        = filesha256("${path.module}/lambda/common.py")
-    requirements_hash         = filesha256("${path.module}/lambda/requirements.txt")
-    requirements_handler_hash = filesha256("${path.module}/scripts/requirements.sh")
-  }
+locals {
+  cur_per_resource_lambda_name = "forge-aws-billing-per-resource"
 }
 
-resource "aws_s3_object" "cur_per_resource" {
-  bucket = aws_s3_bucket.aws_billing_report.id
-  key    = "lambda/aws_billing_lambda_function_per_resource-${null_resource.install_dependencies_per_resource.id}.zip"
-  source = "/tmp/aws_billing/per_resource/${var.aws_profile}/lambda_package/lambda.zip"
+module "cur_per_resource" {
+  source  = "terraform-aws-modules/lambda/aws"
+  version = "8.1.0"
 
-  depends_on = [null_resource.install_dependencies_per_resource]
-}
+  function_name = local.cur_per_resource_lambda_name
+  description   = "Processes AWS Billing CUR reports per resource and sends data to Splunk"
 
-resource "aws_lambda_function" "cur_per_resource" {
-  function_name = "forge-aws-billing-per-resource"
-  s3_bucket     = aws_s3_object.cur_per_resource.bucket
-  s3_key        = aws_s3_object.cur_per_resource.key
   handler       = "handler_per_resource.lambda_handler"
-  architectures = ["x86_64"]
   runtime       = "python3.12"
-  role          = aws_iam_role.lambda_exec_role.arn
-  timeout       = 900
+  architectures = ["x86_64"]
   memory_size   = 10240
+  timeout       = 900
+  publish       = true
+
+  source_path = [{
+    path             = "${path.module}/lambda"
+    pip_requirements = "${path.module}/lambda/requirements.txt"
+  }]
+
+  logging_log_group                 = aws_cloudwatch_log_group.cur_per_resource.name
+  use_existing_cloudwatch_log_group = true
+
+  trigger_on_package_timestamp = false
+
+  environment_variables = {
+    SPLUNK_HEC_URL       = var.splunk_aws_billing_config.splunk_hec_url
+    SPLUNK_HEC_TOKEN     = data.aws_secretsmanager_secret_version.secrets["splunk_cloud_hec_token_aws_billing"].secret_string
+    SPLUNK_INDEX         = var.splunk_aws_billing_config.splunk_index
+    SPLUNK_METRICS_TOKEN = data.aws_secretsmanager_secret_version.secrets["splunk_o11y_ingest_token_aws_billing"].secret_string
+    SPLUNK_METRICS_URL   = var.splunk_aws_billing_config.splunk_metrics_url
+  }
+
+  attach_policy_jsons = true
+  policy_jsons = [
+    data.aws_iam_policy_document.cur_per_resource.json,
+    data.aws_iam_policy_document.lambda_policy_document.json,
+  ]
+  number_of_policy_jsons = 2
+
+  tags = local.all_security_tags
+
+  store_on_s3         = true
+  s3_object_tags_only = true
+  s3_object_tags      = var.default_tags
+  s3_bucket           = aws_s3_bucket.aws_billing_report.id
+  s3_prefix           = "lambda/aws_billing_lambda_function_per_resource_process"
 }
 
 resource "aws_lambda_permission" "cur_per_resource" {
   statement_id  = "AllowS3Invoke"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.cur_per_resource.function_name
+  function_name = local.cur_per_resource_lambda_name
   principal     = "s3.amazonaws.com"
   source_arn    = "arn:aws:s3:::${aws_s3_bucket.aws_billing_report.id}"
 }
 
 resource "aws_cloudwatch_log_group" "cur_per_resource" {
-  name              = "/aws/lambda/${aws_lambda_function.cur_per_resource.function_name}"
+  name              = "/aws/lambda/${local.cur_per_resource_lambda_name}"
   retention_in_days = 3
   tags              = local.all_security_tags
   tags_all          = local.all_security_tags
 }
 
+data "aws_iam_policy_document" "cur_per_resource" {
+  statement {
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+    effect = "Allow"
+
+    resources = [
+      "arn:aws:logs:*:${data.aws_caller_identity.current.account_id}:log-group:${aws_cloudwatch_log_group.cur_per_resource.name}*:*"
+    ]
+  }
+}
+
 resource "aws_bcmdataexports_export" "cur_per_resource" {
   export {
     name = "aws-billing-report-per-resource"
diff --git a/modules/integrations/splunk_aws_billing/billing_per_resource_process.tf b/modules/integrations/splunk_aws_billing/billing_per_resource_process.tf
@@ -1,57 +1,80 @@
-resource "null_resource" "install_dependencies_per_resource_process" {
-  provisioner "local-exec" {
-    command = "bash ${path.module}/scripts/requirements.sh /tmp/aws_billing/per_resource_process/${var.aws_profile} handler_per_resource_process"
-  }
-
-  triggers = {
-    lambda_source_hash        = filesha256("${path.module}/lambda/handler_per_resource_process.py")
-    common_source_hash        = filesha256("${path.module}/lambda/common.py")
-    requirements_hash         = filesha256("${path.module}/lambda/requirements.txt")
-    requirements_handler_hash = filesha256("${path.module}/scripts/requirements.sh")
-  }
+locals {
+  cur_per_resource_process_lambda_name = "forge-aws-billing-per-resource-process"
 }
 
-resource "aws_s3_object" "cur_per_resource_process" {
-  bucket = aws_s3_bucket.aws_billing_report.id
-  key    = "lambda/aws_billing_lambda_function_per_resource_process-${null_resource.install_dependencies_per_resource_process.id}.zip"
-  source = "/tmp/aws_billing/per_resource_process/${var.aws_profile}/lambda_package/lambda.zip"
+module "cur_per_resource_process" {
+  source  = "terraform-aws-modules/lambda/aws"
+  version = "8.1.0"
 
-  depends_on = [null_resource.install_dependencies_per_resource_process]
-}
+  function_name = local.cur_per_resource_process_lambda_name
+  description   = "Processes AWS billing data and sends to Splunk"
 
-resource "aws_lambda_function" "cur_per_resource_process" {
-  function_name = "forge-aws-billing-per-resource-process"
-  s3_bucket     = aws_s3_object.cur_per_resource_process.bucket
-  s3_key        = aws_s3_object.cur_per_resource_process.key
   handler       = "handler_per_resource_process.lambda_handler"
-  architectures = ["x86_64"]
   runtime       = "python3.12"
-  role          = aws_iam_role.lambda_exec_role.arn
-  timeout       = 900
+  architectures = ["x86_64"]
   memory_size   = 10240
+  timeout       = 900
+  publish       = true
+
+  source_path = [{
+    path             = "${path.module}/lambda"
+    pip_requirements = "${path.module}/lambda/requirements.txt"
+  }]
 
-  environment {
-    variables = {
-      SPLUNK_HEC_URL       = var.splunk_aws_billing_config.splunk_hec_url
-      SPLUNK_HEC_TOKEN     = data.aws_secretsmanager_secret_version.secrets["splunk_cloud_hec_token_aws_billing"].secret_string
-      SPLUNK_INDEX         = var.splunk_aws_billing_config.splunk_index
-      SPLUNK_METRICS_TOKEN = data.aws_secretsmanager_secret_version.secrets["splunk_o11y_ingest_token_aws_billing"].secret_string
-      SPLUNK_METRICS_URL   = var.splunk_aws_billing_config.splunk_metrics_url
-    }
+  logging_log_group                 = aws_cloudwatch_log_group.cur_per_resource_process.name
+  use_existing_cloudwatch_log_group = true
+
+  trigger_on_package_timestamp = false
+
+  environment_variables = {
+    SPLUNK_HEC_URL       = var.splunk_aws_billing_config.splunk_hec_url
+    SPLUNK_HEC_TOKEN     = data.aws_secretsmanager_secret_version.secrets["splunk_cloud_hec_token_aws_billing"].secret_string
+    SPLUNK_INDEX         = var.splunk_aws_billing_config.splunk_index
+    SPLUNK_METRICS_TOKEN = data.aws_secretsmanager_secret_version.secrets["splunk_o11y_ingest_token_aws_billing"].secret_string
+    SPLUNK_METRICS_URL   = var.splunk_aws_billing_config.splunk_metrics_url
   }
+
+  attach_policy_jsons = true
+  policy_jsons = [
+    data.aws_iam_policy_document.cur_per_resource_process.json,
+    data.aws_iam_policy_document.lambda_policy_document.json,
+  ]
+  number_of_policy_jsons = 2
+
+  tags = local.all_security_tags
+
+  store_on_s3         = true
+  s3_object_tags_only = true
+  s3_object_tags      = var.default_tags
+  s3_bucket           = aws_s3_bucket.aws_billing_report.id
+  s3_prefix           = "lambda/aws_billing_lambda_function_per_resource_process"
 }
 
 resource "aws_lambda_permission" "cur_per_resource_process" {
   statement_id  = "AllowS3Invoke"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.cur_per_resource_process.function_name
+  function_name = local.cur_per_resource_process_lambda_name
   principal     = "s3.amazonaws.com"
   source_arn    = "arn:aws:s3:::${aws_s3_bucket.aws_billing_report.id}"
 }
 
 resource "aws_cloudwatch_log_group" "cur_per_resource_process" {
-  name              = "/aws/lambda/${aws_lambda_function.cur_per_resource_process.function_name}"
+  name              = "/aws/lambda/${local.cur_per_resource_process_lambda_name}"
   retention_in_days = 3
   tags              = local.all_security_tags
   tags_all          = local.all_security_tags
 }
+
+data "aws_iam_policy_document" "cur_per_resource_process" {
+  statement {
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+    effect = "Allow"
+
+    resources = [
+      "arn:aws:logs:*:${data.aws_caller_identity.current.account_id}:log-group:${aws_cloudwatch_log_group.cur_per_resource_process.name}*:*"
+    ]
+  }
+}
diff --git a/modules/integrations/splunk_aws_billing/billing_per_service.tf b/modules/integrations/splunk_aws_billing/billing_per_service.tf
@@ -1,62 +1,84 @@
-resource "null_resource" "install_dependencies_per_service" {
-
-  provisioner "local-exec" {
-    command = "bash ${path.module}/scripts/requirements.sh /tmp/aws_billing/per_service/${var.aws_profile} handler_per_service"
-  }
-
-  triggers = {
-    lambda_source_hash        = filesha256("${path.module}/lambda/handler_per_service.py")
-    common_source_hash        = filesha256("${path.module}/lambda/common.py")
-    requirements_hash         = filesha256("${path.module}/lambda/requirements.txt")
-    requirements_handler_hash = filesha256("${path.module}/scripts/requirements.sh")
-  }
+locals {
+  cur_per_service_lambda_name = "forge-aws-billing-per-service"
 }
 
-resource "aws_s3_object" "cur_per_service" {
-  bucket = aws_s3_bucket.aws_billing_report.id
-  key    = "lambda/aws_billing_lambda_function_per_service-${null_resource.install_dependencies_per_service.id}.zip"
-  source = "/tmp/aws_billing/per_service/${var.aws_profile}/lambda_package/lambda.zip"
+module "cur_per_service" {
+  source  = "terraform-aws-modules/lambda/aws"
+  version = "8.1.0"
 
-  depends_on = [null_resource.install_dependencies_per_service]
-}
+  function_name = local.cur_per_service_lambda_name
+  description   = "Processes AWS Billing CUR reports per service and sends data to Splunk"
 
-resource "aws_lambda_function" "cur_per_service" {
-  function_name = "forge-aws-billing-per-service"
-  s3_bucket     = aws_s3_object.cur_per_service.bucket
-  s3_key        = aws_s3_object.cur_per_service.key
   handler       = "handler_per_service.lambda_handler"
-  architectures = ["x86_64"]
   runtime       = "python3.12"
-  role          = aws_iam_role.lambda_exec_role.arn
-  timeout       = 900
+  architectures = ["x86_64"]
   memory_size   = 10240
+  timeout       = 900
+  publish       = true
 
-  environment {
-    variables = {
-      SPLUNK_HEC_URL       = var.splunk_aws_billing_config.splunk_hec_url
-      SPLUNK_HEC_TOKEN     = data.aws_secretsmanager_secret_version.secrets["splunk_cloud_hec_token_aws_billing"].secret_string
-      SPLUNK_INDEX         = var.splunk_aws_billing_config.splunk_index
-      SPLUNK_METRICS_TOKEN = data.aws_secretsmanager_secret_version.secrets["splunk_o11y_ingest_token_aws_billing"].secret_string
-      SPLUNK_METRICS_URL   = var.splunk_aws_billing_config.splunk_metrics_url
-    }
+  source_path = [{
+    path             = "${path.module}/lambda"
+    pip_requirements = "${path.module}/lambda/requirements.txt"
+  }]
+
+  logging_log_group                 = aws_cloudwatch_log_group.cur_per_service.name
+  use_existing_cloudwatch_log_group = true
+
+  trigger_on_package_timestamp = false
+
+  environment_variables = {
+    SPLUNK_HEC_URL       = var.splunk_aws_billing_config.splunk_hec_url
+    SPLUNK_HEC_TOKEN     = data.aws_secretsmanager_secret_version.secrets["splunk_cloud_hec_token_aws_billing"].secret_string
+    SPLUNK_INDEX         = var.splunk_aws_billing_config.splunk_index
+    SPLUNK_METRICS_TOKEN = data.aws_secretsmanager_secret_version.secrets["splunk_o11y_ingest_token_aws_billing"].secret_string
+    SPLUNK_METRICS_URL   = var.splunk_aws_billing_config.splunk_metrics_url
   }
+
+  attach_policy_jsons = true
+  policy_jsons = [
+    data.aws_iam_policy_document.cur_per_service.json,
+    data.aws_iam_policy_document.lambda_policy_document.json,
+  ]
+  number_of_policy_jsons = 2
+
+  tags = local.all_security_tags
+
+  store_on_s3         = true
+  s3_object_tags_only = true
+  s3_object_tags      = var.default_tags
+  s3_bucket           = aws_s3_bucket.aws_billing_report.id
+  s3_prefix           = "lambda/aws_billing_lambda_function_per_resource_process"
 }
 
 resource "aws_lambda_permission" "cur_per_service" {
   statement_id  = "AllowS3Invoke"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.cur_per_service.function_name
+  function_name = local.cur_per_service_lambda_name
   principal     = "s3.amazonaws.com"
   source_arn    = "arn:aws:s3:::${aws_s3_bucket.aws_billing_report.id}"
 }
 
 resource "aws_cloudwatch_log_group" "cur_per_service" {
-  name              = "/aws/lambda/${aws_lambda_function.cur_per_service.function_name}"
+  name              = "/aws/lambda/${local.cur_per_service_lambda_name}"
   retention_in_days = 3
   tags              = local.all_security_tags
   tags_all          = local.all_security_tags
 }
 
+data "aws_iam_policy_document" "cur_per_service" {
+  statement {
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+    effect = "Allow"
+
+    resources = [
+      "arn:aws:logs:*:${data.aws_caller_identity.current.account_id}:log-group:${aws_cloudwatch_log_group.cur_per_service.name}*:*"
+    ]
+  }
+}
+
 resource "aws_bcmdataexports_export" "cur_per_service" {
   export {
     name = "aws-billing-report-per-service"
diff --git a/modules/integrations/splunk_aws_billing/lambda/requirements.txt b/modules/integrations/splunk_aws_billing/lambda/requirements.txt
@@ -1,3 +1,2 @@
-boto3
 pandas[pyarrow]
 requests
diff --git a/modules/integrations/splunk_aws_billing/s3.tf b/modules/integrations/splunk_aws_billing/s3.tf

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,2 @@`
`1`		`-boto3`
`2`	`1`	`pandas[pyarrow]`
`3`	`2`	`requests`