refactor(forge): create submodule for github features (#180)

* refactor: create submodule github_global_lock

* refactor: create submodule github_webhook_relay

* refactor: create submodule github_app_runner_group

* refactor: delete old files

Author: edersonbrilhante

modules/platform/forge_runners/github_app_runner_group.tf

@@ -0,0 +1,16 @@
+module "github_app_runner_group" {
+  source = "./github_app_runner_group"
+
+  providers = {
+    aws = aws
+  }
+
+  prefix                    = var.deployment_config.prefix
+  secrets_prefix            = local.cicd_secrets_prefix
+  logging_retention_in_days = var.logging_retention_in_days
+  tags                      = local.all_security_tags
+  github_api                = local.github_api
+  ghes_org                  = var.ghes_org
+  runner_group_name         = var.runner_group_name
+  repository_selection      = var.repository_selection
+}

modules/platform/forge_runners/github_app_runner_group/data.tf

@@ -0,0 +1 @@
+data "aws_region" "current" {}

…unners/lambda/github_app_runner_group.py …_group/lambda/github_app_runner_group.pymodules/platform/forge_runners/lambda/github_app_runner_group.py renamed to modules/platform/forge_runners/github_app_runner_group/lambda/github_app_runner_group.py modules/platform/forge_runners/lambda/github_app_runner_group.py renamed to modules/platform/forge_runners/github_app_runner_group/lambda/github_app_runner_group.py

@@ -2,19 +2,20 @@ module "register_github_app_runner_group_lambda" {
   source  = "terraform-aws-modules/lambda/aws"
   version = "8.1.0"
 
-  function_name = "${var.deployment_config.prefix}-register-github-app-runner-group"
+  function_name = "${var.prefix}-register-github-app-runner-group"
   handler       = "github_app_runner_group.lambda_handler"
   runtime       = "python3.12"
   timeout       = 900
   architectures = ["x86_64"]
 
   source_path = [{
-    path             = "${path.module}/lambda"
-    pip_requirements = "${path.module}/lambda/requirements.txt"
+    path = "${path.module}/lambda"
   }]
 
   layers = [
-    "arn:aws:lambda:${var.aws_region}:770693421928:layer:Klayers-p312-cryptography:17"
+    "arn:aws:lambda:${data.aws_region.current.region}:770693421928:layer:Klayers-p312-cryptography:17",
+    "arn:aws:lambda:${data.aws_region.current.region}:770693421928:layer:Klayers-p312-requests:17",
+    "arn:aws:lambda:${data.aws_region.current.region}:770693421928:layer:Klayers-p312-PyJWT:1",
   ]
 
   logging_log_group                 = aws_cloudwatch_log_group.register_github_app_runner_group_lambda.name
@@ -23,22 +24,22 @@ module "register_github_app_runner_group_lambda" {
   trigger_on_package_timestamp = false
 
   environment_variables = {
-    GITHUB_API                  = local.github_api
+    GITHUB_API                  = var.github_api
     ORGANIZATION                = var.ghes_org
     RUNNER_GROUP_NAME           = var.runner_group_name
     REPOSITORY_SELECTION        = var.repository_selection
-    SECRET_NAME_APP_ID          = "${local.cicd_secrets_prefix}github_actions_runners_app_id"
-    SECRET_NAME_PRIVATE_KEY     = "${local.cicd_secrets_prefix}github_actions_runners_app_key"
-    SECRET_NAME_INSTALLATION_ID = "${local.cicd_secrets_prefix}github_actions_runners_app_installation_id"
+    SECRET_NAME_APP_ID          = local.secrets.github_actions_runners_app_id.name
+    SECRET_NAME_PRIVATE_KEY     = local.secrets.github_actions_runners_app_key.name
+    SECRET_NAME_INSTALLATION_ID = local.secrets.github_actions_runners_app_installation_id.name
   }
 
   attach_policy_json = true
 
   policy_json = data.aws_iam_policy_document.register_github_app_runner_group_lambda.json
 
-  function_tags = local.all_security_tags
-  role_tags     = local.all_security_tags
-  tags          = local.all_security_tags
+  function_tags = var.tags
+  role_tags     = var.tags
+  tags          = var.tags
 
   depends_on = [aws_cloudwatch_log_group.register_github_app_runner_group_lambda]
 }
@@ -51,27 +52,27 @@ data "aws_iam_policy_document" "register_github_app_runner_group_lambda" {
     ]
     effect = "Allow"
     resources = [
-      data.aws_secretsmanager_secret_version.data_cicd_secrets["${local.cicd_secrets_prefix}github_actions_runners_app_key"].arn,
-      data.aws_secretsmanager_secret_version.data_cicd_secrets["${local.cicd_secrets_prefix}github_actions_runners_app_id"].arn,
-      data.aws_secretsmanager_secret_version.data_cicd_secrets["${local.cicd_secrets_prefix}github_actions_runners_app_installation_id"].arn,
+      data.aws_secretsmanager_secret_version.secrets["github_actions_runners_app_key"].arn,
+      data.aws_secretsmanager_secret_version.secrets["github_actions_runners_app_id"].arn,
+      data.aws_secretsmanager_secret_version.secrets["github_actions_runners_app_installation_id"].arn,
     ]
   }
 }
 
 resource "aws_cloudwatch_log_group" "register_github_app_runner_group_lambda" {
-  name              = "/aws/lambda/${var.deployment_config.prefix}-register-github-app-runner-group"
+  name              = "/aws/lambda/${var.prefix}-register-github-app-runner-group"
   retention_in_days = var.logging_retention_in_days
-  tags              = local.all_security_tags
-  tags_all          = local.all_security_tags
+  tags              = var.tags
+  tags_all          = var.tags
 }
 
 resource "aws_cloudwatch_event_rule" "register_github_app_runner_group_lambda" {
-  name                = "${var.deployment_config.prefix}-register-github-app-runner-group"
+  name                = "${var.prefix}-register-github-app-runner-group"
   description         = "Trigger Lambda every 10 minutes"
   schedule_expression = "cron(*/10 * * * ? *)"
 
-  tags     = local.all_security_tags
-  tags_all = local.all_security_tags
+  tags     = var.tags
+  tags_all = var.tags
 
   depends_on = [module.register_github_app_runner_group_lambda]
 }
@@ -85,7 +86,7 @@ resource "aws_cloudwatch_event_target" "register_github_app_runner_group_lambda"
 
 resource "aws_lambda_permission" "register_github_app_runner_group_lambda" {
   action        = "lambda:InvokeFunction"
-  function_name = "${var.deployment_config.prefix}-register-github-app-runner-group"
+  function_name = "${var.prefix}-register-github-app-runner-group"
   principal     = "events.amazonaws.com"
   statement_id  = "AllowExecutionFromCloudWatch"
   source_arn    = aws_cloudwatch_event_rule.register_github_app_runner_group_lambda.arn

…ge_runners/register_repo_runner_group.tf …_runners/github_app_runner_group/main.tfmodules/platform/forge_runners/register_repo_runner_group.tf renamed to modules/platform/forge_runners/github_app_runner_group/main.tf modules/platform/forge_runners/register_repo_runner_group.tf renamed to modules/platform/forge_runners/github_app_runner_group/main.tf

@@ -0,0 +1,23 @@
+locals {
+  secrets = {
+    github_actions_runners_app_key = {
+      name = "${var.secrets_prefix}github_actions_runners_app_key"
+    }
+    github_actions_runners_app_id = {
+      name = "${var.secrets_prefix}github_actions_runners_app_id"
+    }
+    github_actions_runners_app_installation_id = {
+      name = "${var.secrets_prefix}github_actions_runners_app_installation_id"
+    }
+  }
+}
+
+data "aws_secretsmanager_secret" "secrets" {
+  for_each = local.secrets
+  name     = each.value.name
+}
+
+data "aws_secretsmanager_secret_version" "secrets" {
+  for_each  = data.aws_secretsmanager_secret.secrets
+  secret_id = each.value.id
+}

modules/platform/forge_runners/github_app_runner_group/secrets.tf

@@ -0,0 +1,46 @@
+variable "prefix" {
+  description = "Prefix for all resources"
+  type        = string
+}
+
+variable "tags" {
+  description = "Tags to apply to created resources."
+  type        = map(string)
+  default     = {}
+}
+
+variable "secrets_prefix" {
+  description = "Prefix for all secrets"
+  type        = string
+}
+
+variable "logging_retention_in_days" {
+  description = "Retention in days for CloudWatch Log Group for the Lambdas."
+  type        = number
+  default     = 30
+}
+
+variable "github_api" {
+  description = "Base URL for the GitHub API (set to GHES API endpoint if using Enterprise)."
+  type        = string
+  default     = "https://api.github.com"
+}
+
+variable "ghes_org" {
+  description = "GitHub organization (GHES or GitHub.com)."
+  type        = string
+}
+
+variable "runner_group_name" {
+  description = "Name of the GitHub Actions runner group to create/update and attach repositories to."
+  type        = string
+}
+
+variable "repository_selection" {
+  description = "Repository selection type: 'all' or 'selected'."
+  type        = string
+  validation {
+    condition     = contains(["all", "selected"], var.repository_selection)
+    error_message = "repository_selection must be 'all' or 'selected'."
+  }
+}

modules/platform/forge_runners/github_app_runner_group/variables.tf

@@ -0,0 +1,11 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.27"
+    }
+  }
+
+  # OpenTofu version.
+  required_version = ">= 1.9.1"
+}

modules/platform/forge_runners/github_app_runner_group/versions.tf

@@ -0,0 +1,12 @@
+module "github_global_lock" {
+  source = "./github_global_lock"
+
+  providers = {
+    aws = aws
+  }
+
+  prefix                    = var.deployment_config.prefix
+  secrets_prefix            = local.cicd_secrets_prefix
+  logging_retention_in_days = var.logging_retention_in_days
+  tags                      = local.all_security_tags
+}

modules/platform/forge_runners/github_global_lock.tf

@@ -0,0 +1 @@
+data "aws_region" "current" {}

modules/platform/forge_runners/github_global_lock/data.tf

@@ -0,0 +1,119 @@
+# GitHub Actions Job Log Archiver Module
+
+Archives completed GitHub Actions workflow job logs into per-tenant S3 buckets for audit, troubleshooting, and retention requirements.
+
+## Features
+- Per-tenant buckets (auto-created or externally provided)
+- Two processing modes:
+  - Direct (legacy): EventBridge -> downloader Lambda -> S3
+  - Queue pipeline (default): EventBridge -> dispatcher Lambda -> SQS -> downloader Lambda (better isolation, retry & backpressure)
+- GitHub App authentication (JWT + installation token) to fetch job logs
+- EventBridge rule listening for `workflow_job.completed` events (via existing webhook relay -> EventBridge integration)
+- Optional KMS encryption
+- Shared read/list access for platform/observability roles
+- Versioning & basic lifecycle management
+
+## How It Works
+### Queue Pipeline (default)
+1. GitHub webhook (workflow_job events) reaches your existing relay (not part of this module) which forwards events onto EventBridge as detail-type `GitHub Webhook`.
+2. EventBridge rule filters for `workflow_job` with `action=completed`.
+3. Dispatcher Lambda validates mapping & completion status, then enqueues a concise message to SQS.
+4. Downloader Lambda (SQS trigger) consumes batches, performs GitHub API log download, and writes to S3.
+5. Object stored at: `s3://<bucket>/<workflow_name>/<run_attempt>/<job_id>.zip`.
+
+### Direct Mode (fallback)
+Set `enable_queue_pipeline = false` to revert to the original single-Lambda flow (less durable, fewer moving parts for very low volume environments).
+
+## Inputs
+See `variables.tf` for full list. Key inputs:
+- `repo_tenant_map` (map) : Maps `org/repo` -> tenant id.
+- `github_app_id`, `github_app_private_key_secret_arn` : GitHub App credentials.
+- `github_app_installation_id` : Optional fixed installation id (skip lookup per repo).
+- `kms_key_arn` : Optional SSE-KMS key for encryption.
+- `shared_role_arns` : Optional map of role ARNs granted read-list across all buckets.
+- `enable_queue_pipeline` : Toggle queue pipeline (default true).
+- `sqs_visibility_timeout_seconds`, `sqs_max_receive_count` : Queue configuration.
+- `downloader_batch_size`, `downloader_max_concurrency` : SQS batch processing tuning.
+
+## Outputs
+- `bucket_names` : tenant -> bucket mapping.
+- `lambda_function_name` / `lambda_function_arn`.
+
+## Bucket Naming
+Uses `bucket_name_format` replacing `{{tenant}}` with tenant id. Provide pre-created buckets by setting `create_buckets = false` and ensuring the names exist.
+
+## Example (Queue Pipeline)
+```hcl
+module "gha_job_log_archiver" {
+  source = "../modules/integrations/github_actions_job_log_archiver"
+
+  repo_tenant_map = {
+    "myorg/app-service" = "tenant-a"
+    "myorg/api-gateway" = "tenant-b"
+  }
+
+  github_app_id                      = var.github_app_id
+  github_app_private_key_secret_arn  = var.github_app_private_key_secret_arn
+  github_app_installation_id         = var.github_app_installation_id # optional
+  kms_key_arn                        = var.logs_kms_key_arn           # optional
+  shared_role_arns = {
+    observability = aws_iam_role.observability.arn
+  }
+  tags = var.tags
+}
+```
+
+## Example (Direct Mode)
+```hcl
+module "gha_job_log_archiver" {
+  source = "../modules/integrations/github_actions_job_log_archiver"
+  enable_queue_pipeline = false
+  repo_tenant_map = { "myorg/app-service" = "tenant-a" }
+  github_app_id                     = var.github_app_id
+  github_app_private_key_secret_arn = var.github_app_private_key_secret_arn
+}
+```
+
+## Required GitHub App Permissions
+- Actions: Read
+- Metadata: Read
+
+## IAM / Security Notes
+- Ensure the provided KMS key allows the Lambda role to `Encrypt/Decrypt/GenerateDataKey`.
+- Bucket policies (shared access) are not yet explicitly created; consider future enhancement if central read roles need cross-account access.
+
+## Future Enhancements
+- Add optional CloudWatch metric/log filters for download errors.
+- Retry & backoff for GitHub API secondary rate limits / 403 abuse detection.
+- Support artifact / log size tagging in object metadata.
+- Add SQS FIFO option for strict ordering per repository.
+
+## Event Shape Assumption
+Expecting EventBridge detail of form (subset):
+```json
+{
+  "detail": {
+    "repository": { "full_name": "org/repo" },
+    "action": "completed",
+    "workflow_job": {
+      "id": 123456789,
+      "run_id": 987654321,
+      "run_attempt": 1,
+      "name": "build",
+      "workflow_name": "CI",
+      "status": "completed",
+      "conclusion": "success"
+    }
+  }
+}
+```
+
+## Local Development
+Package the Lambda:
+```bash
+(cd modules/integrations/github_actions_job_log_archiver/lambda && zip job_log_archiver.zip job_log_archiver.py)
+```
+Then re-run Terraform apply so the updated `source_code_hash` triggers deployment.
+
+## License
+See parent repository license.