From 7a53b90d380fdfd6ca783d62623d1c65ea3ef8ed Mon Sep 17 00:00:00 2001 From: Ryan VanGundy <85766511+rmvangun@users.noreply.github.com> Date: Mon, 8 Dec 2025 08:45:06 -0500 Subject: [PATCH 1/4] feat(aks): Enforce Active Directory RBAC Enforces the use of AD RBAC for k8s api connectivity. Automatically sets the active terraform identity as an administrator. Allows for restricting to specific IPs for further restriction. Signed-off-by: Ryan VanGundy <85766511+rmvangun@users.noreply.github.com> --- aqua.yaml | 1 + terraform/cluster/azure-aks/main.tf | 21 +++++++++++++++++++++ terraform/cluster/azure-aks/variables.tf | 14 +++++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/aqua.yaml b/aqua.yaml index 0ef3586b3..e67f0605f 100644 --- a/aqua.yaml +++ b/aqua.yaml @@ -32,3 +32,4 @@ packages: - name: evilmartians/lefthook@v2.0.8 - name: bridgecrewio/checkov@3.2.495 - name: kubernetes-sigs/krew@v0.4.5 + - name: Azure/kubelogin@v0.2.13 diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index cc729630a..f4f4d582d 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -34,6 +34,8 @@ provider "azurerm" { data "azurerm_client_config" "current" {} +data "azurerm_subscription" "current" {} + data "azurerm_virtual_network" "vnet" { name = "${var.vnet_module_name}-${var.context_id}" resource_group_name = "${var.vnet_module_name}-${var.context_id}" @@ -236,6 +238,11 @@ resource "azurerm_kubernetes_cluster" "main" { # checkov:skip=CKV_AZURE_141: We are setting this to false to avoid the creation of an AD local_account_disabled = var.local_account_disabled + azure_active_directory_role_based_access_control { + azure_rbac_enabled = true + admin_group_object_ids = var.admin_object_ids + } + key_vault_secrets_provider { secret_rotation_enabled = true } @@ -334,3 +341,17 @@ resource "local_file" "kube_config" { content = azurerm_kubernetes_cluster.main.kube_config_raw filename = local.kubeconfig_path } + +# Automatically assign "Azure Kubernetes Service RBAC Cluster Admin" to the +# identity running Terraform (the deployer) and any additional admins provided. +# This ensures immediate access when local_account_disabled is set to true. +resource "azurerm_role_assignment" "aks_rbac_admin" { + for_each = toset(concat( + [data.azurerm_client_config.current.object_id], + var.admin_object_ids + )) + + scope = azurerm_kubernetes_cluster.main.id + role_definition_name = "Azure Kubernetes Service RBAC Cluster Admin" + principal_id = each.value +} diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index 44e0a5947..a90965144 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -2,6 +2,12 @@ # Variables #----------------------------------------------------------------------------------------------------------------------- +variable "admin_object_ids" { + type = list(string) + description = "List of Azure AD Object IDs (User or Group) to assign 'Azure Kubernetes Service RBAC Cluster Admin' role. Required when local_account_disabled is true to ensure access." + default = [] +} + variable "context_path" { type = string description = "The path to the context folder, where kubeconfig is stored" @@ -184,7 +190,13 @@ variable "azure_policy_enabled" { variable "local_account_disabled" { type = bool description = "Whether to disable local accounts for the AKS cluster" - default = false + default = true +} + +variable "authorized_ip_ranges" { + type = set(string) + description = "Set of authorized IP ranges to allow access to the API server. If null, allows all (0.0.0.0/0)." + default = null } variable "public_network_access_enabled" { From 33c76fd3364efbaa893f7ac40c1f614c01c05ea5 Mon Sep 17 00:00:00 2001 From: Ryan VanGundy <85766511+rmvangun@users.noreply.github.com> Date: Mon, 8 Dec 2025 08:53:33 -0500 Subject: [PATCH 2/4] Add authorized IP block Signed-off-by: Ryan VanGundy <85766511+rmvangun@users.noreply.github.com> --- terraform/cluster/azure-aks/main.tf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index f4f4d582d..cfb5d59f9 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -227,8 +227,12 @@ resource "azurerm_kubernetes_cluster" "main" { role_based_access_control_enabled = var.role_based_access_control_enabled automatic_upgrade_channel = var.automatic_upgrade_channel sku_tier = var.sku_tier - # checkov:skip=CKV_AZURE_6: This feature is in preview, we are using a public cluster for testing - # api_server_authorized_ip_ranges = [0.0.0.0/0] + + # checkov:skip=CKV_AZURE_6: We allow user to restrict IPs or default to open (null) + api_server_access_profile { + authorized_ip_ranges = var.authorized_ip_ranges + } + # checkov:skip=CKV_AZURE_115: We are using a public cluster for testing # private clusters are encouraged for production private_cluster_enabled = var.private_cluster_enabled From 12aa294402ae11f262cd5240910f85f7def0e4c1 Mon Sep 17 00:00:00 2001 From: Ryan VanGundy <85766511+rmvangun@users.noreply.github.com> Date: Mon, 8 Dec 2025 08:59:18 -0500 Subject: [PATCH 3/4] Add tests Signed-off-by: Ryan VanGundy <85766511+rmvangun@users.noreply.github.com> --- terraform/cluster/azure-aks/test.tftest.hcl | 151 +++++++++++++++++++- 1 file changed, 149 insertions(+), 2 deletions(-) diff --git a/terraform/cluster/azure-aks/test.tftest.hcl b/terraform/cluster/azure-aks/test.tftest.hcl index 3a3a6885e..c372b647b 100644 --- a/terraform/cluster/azure-aks/test.tftest.hcl +++ b/terraform/cluster/azure-aks/test.tftest.hcl @@ -5,6 +5,11 @@ mock_provider "azurerm" { object_id = "22222222-2222-2222-2222-222222222222" } } + mock_data "azurerm_subscription" { + defaults = { + subscription_id = "12345678-1234-9876-4563-123456789012" + } + } mock_data "azurerm_virtual_network" { defaults = { subnets = ["private-1-test", "private-2-test", "private-3-test", "public-1-test", "public-2-test", "isolated-1-test", "isolated-2-test"] @@ -76,14 +81,39 @@ run "minimal_configuration" { } assert { - condition = azurerm_kubernetes_cluster.main.local_account_disabled == false - error_message = "Local accounts should be enabled by default" + condition = azurerm_kubernetes_cluster.main.local_account_disabled == true + error_message = "Local accounts should be disabled by default" } assert { condition = azurerm_kubernetes_cluster.main.identity[0].type == "SystemAssigned" error_message = "Cluster should use system-assigned identity by default" } + + assert { + condition = azurerm_kubernetes_cluster.main.azure_active_directory_role_based_access_control[0].azure_rbac_enabled == true + error_message = "Azure RBAC should be enabled by default" + } + + assert { + condition = length(azurerm_kubernetes_cluster.main.azure_active_directory_role_based_access_control[0].admin_group_object_ids) == 0 + error_message = "Admin group object IDs should be empty by default" + } + + assert { + condition = azurerm_kubernetes_cluster.main.api_server_access_profile[0].authorized_ip_ranges == null + error_message = "Authorized IP ranges should be null by default (allowing all)" + } + + assert { + condition = length(azurerm_role_assignment.aks_rbac_admin) == 1 + error_message = "Role assignment should be created for the deployer identity by default" + } + + assert { + condition = azurerm_role_assignment.aks_rbac_admin["22222222-2222-2222-2222-222222222222"].role_definition_name == "Azure Kubernetes Service RBAC Cluster Admin" + error_message = "Role assignment should use 'Azure Kubernetes Service RBAC Cluster Admin' role" + } } # Tests a full configuration with all optional variables explicitly set, @@ -130,6 +160,8 @@ run "full_configuration" { private_cluster_enabled = false azure_policy_enabled = true local_account_disabled = false + authorized_ip_ranges = ["10.0.0.0/8"] + admin_object_ids = ["55555555-5555-5555-5555-555555555555"] } assert { @@ -231,6 +263,36 @@ run "full_configuration" { condition = azurerm_kubernetes_cluster.main.kubelet_identity[0].user_assigned_identity_id == "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity-1" error_message = "Kubelet user-assigned identity ID should match input" } + + assert { + condition = length(azurerm_kubernetes_cluster.main.api_server_access_profile[0].authorized_ip_ranges) == 1 + error_message = "Authorized IP ranges should contain 1 entry" + } + + assert { + condition = contains(azurerm_kubernetes_cluster.main.api_server_access_profile[0].authorized_ip_ranges, "10.0.0.0/8") + error_message = "Authorized IP ranges should include 10.0.0.0/8" + } + + assert { + condition = azurerm_kubernetes_cluster.main.azure_active_directory_role_based_access_control[0].azure_rbac_enabled == true + error_message = "Azure RBAC should be enabled" + } + + assert { + condition = length(azurerm_kubernetes_cluster.main.azure_active_directory_role_based_access_control[0].admin_group_object_ids) == 1 + error_message = "Admin group object IDs should contain 1 entry" + } + + assert { + condition = contains(azurerm_kubernetes_cluster.main.azure_active_directory_role_based_access_control[0].admin_group_object_ids, "55555555-5555-5555-5555-555555555555") + error_message = "Admin group object IDs should include the specified object ID" + } + + assert { + condition = length(azurerm_role_assignment.aks_rbac_admin) == 2 + error_message = "Role assignments should be created for deployer plus 1 admin object ID (2 total)" + } } # Tests the private cluster configuration, ensuring that enabling the private_cluster_enabled @@ -290,6 +352,91 @@ run "network_configuration" { } } +# Tests the authorized IP ranges configuration, ensuring that setting authorized_ip_ranges +# results in the API server access profile being configured with the specified IP ranges. +run "authorized_ip_ranges" { + command = plan + + variables { + context_id = "test" + name = "windsor-aks" + cluster_name = "test-cluster" + kubernetes_version = "1.32" + authorized_ip_ranges = ["10.0.0.0/8", "192.168.0.0/16"] + } + + assert { + condition = length(azurerm_kubernetes_cluster.main.api_server_access_profile[0].authorized_ip_ranges) == 2 + error_message = "Authorized IP ranges should contain 2 entries" + } + + assert { + condition = contains(azurerm_kubernetes_cluster.main.api_server_access_profile[0].authorized_ip_ranges, "10.0.0.0/8") + error_message = "Authorized IP ranges should include 10.0.0.0/8" + } + + assert { + condition = contains(azurerm_kubernetes_cluster.main.api_server_access_profile[0].authorized_ip_ranges, "192.168.0.0/16") + error_message = "Authorized IP ranges should include 192.168.0.0/16" + } +} + +# Tests the Azure RBAC configuration with admin object IDs, ensuring that the +# azure_active_directory_role_based_access_control block is configured correctly and +# role assignments are created for all specified admin object IDs plus the deployer. +run "azure_rbac_with_admin_object_ids" { + command = plan + + variables { + context_id = "test" + name = "windsor-aks" + cluster_name = "test-cluster" + kubernetes_version = "1.32" + local_account_disabled = true + admin_object_ids = ["33333333-3333-3333-3333-333333333333", "44444444-4444-4444-4444-444444444444"] + } + + assert { + condition = azurerm_kubernetes_cluster.main.azure_active_directory_role_based_access_control[0].azure_rbac_enabled == true + error_message = "Azure RBAC should be enabled" + } + + assert { + condition = length(azurerm_kubernetes_cluster.main.azure_active_directory_role_based_access_control[0].admin_group_object_ids) == 2 + error_message = "Admin group object IDs should contain 2 entries" + } + + assert { + condition = contains(azurerm_kubernetes_cluster.main.azure_active_directory_role_based_access_control[0].admin_group_object_ids, "33333333-3333-3333-3333-333333333333") + error_message = "Admin group object IDs should include the first specified object ID" + } + + assert { + condition = contains(azurerm_kubernetes_cluster.main.azure_active_directory_role_based_access_control[0].admin_group_object_ids, "44444444-4444-4444-4444-444444444444") + error_message = "Admin group object IDs should include the second specified object ID" + } + + assert { + condition = length(azurerm_role_assignment.aks_rbac_admin) == 3 + error_message = "Role assignments should be created for deployer plus 2 admin object IDs (3 total)" + } + + assert { + condition = azurerm_role_assignment.aks_rbac_admin["22222222-2222-2222-2222-222222222222"].role_definition_name == "Azure Kubernetes Service RBAC Cluster Admin" + error_message = "Role assignment for deployer should use 'Azure Kubernetes Service RBAC Cluster Admin' role" + } + + assert { + condition = azurerm_role_assignment.aks_rbac_admin["33333333-3333-3333-3333-333333333333"].role_definition_name == "Azure Kubernetes Service RBAC Cluster Admin" + error_message = "Role assignment for first admin should use 'Azure Kubernetes Service RBAC Cluster Admin' role" + } + + assert { + condition = azurerm_role_assignment.aks_rbac_admin["44444444-4444-4444-4444-444444444444"].role_definition_name == "Azure Kubernetes Service RBAC Cluster Admin" + error_message = "Role assignment for second admin should use 'Azure Kubernetes Service RBAC Cluster Admin' role" + } +} + run "multiple_invalid_inputs" { command = plan expect_failures = [ From 5ed356a2e2a14f88fa9e55335aa43cc481b898e9 Mon Sep 17 00:00:00 2001 From: Ryan VanGundy <85766511+rmvangun@users.noreply.github.com> Date: Mon, 8 Dec 2025 10:12:04 -0500 Subject: [PATCH 4/4] fmt Signed-off-by: Ryan VanGundy <85766511+rmvangun@users.noreply.github.com> --- terraform/cluster/azure-aks/test.tftest.hcl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/terraform/cluster/azure-aks/test.tftest.hcl b/terraform/cluster/azure-aks/test.tftest.hcl index c372b647b..4244c3680 100644 --- a/terraform/cluster/azure-aks/test.tftest.hcl +++ b/terraform/cluster/azure-aks/test.tftest.hcl @@ -358,10 +358,10 @@ run "authorized_ip_ranges" { command = plan variables { - context_id = "test" - name = "windsor-aks" - cluster_name = "test-cluster" - kubernetes_version = "1.32" + context_id = "test" + name = "windsor-aks" + cluster_name = "test-cluster" + kubernetes_version = "1.32" authorized_ip_ranges = ["10.0.0.0/8", "192.168.0.0/16"] } @@ -388,12 +388,12 @@ run "azure_rbac_with_admin_object_ids" { command = plan variables { - context_id = "test" - name = "windsor-aks" - cluster_name = "test-cluster" - kubernetes_version = "1.32" + context_id = "test" + name = "windsor-aks" + cluster_name = "test-cluster" + kubernetes_version = "1.32" local_account_disabled = true - admin_object_ids = ["33333333-3333-3333-3333-333333333333", "44444444-4444-4444-4444-444444444444"] + admin_object_ids = ["33333333-3333-3333-3333-333333333333", "44444444-4444-4444-4444-444444444444"] } assert {