[pull] main from GoogleCloudPlatform:main

tools/custom-module-security-health-analytics-library/Makefile

@@ -1,15 +1,17 @@
-# Make will use bash instead of sh
 SHELL 	:= /usr/bin/env bash
-CONFIG := -f build/config/schema.yaml -f build/config/services/ -f values.yaml -f build/ytt_lib/
-OUTPUT_GCLOUD 	:= samples/gcloud
-OUTPUT_TF := samples/tf
+
+INPUT_VALUES    := $(or $(CUSTOM_SHA_INPUT_VALUES), values.yaml)
+OUTPUT_GCLOUD   := $(or $(CUSTOM_SHA_OUTPUT_GCLOUD), samples/gcloud)
+OUTPUT_TF       := $(or $(CUSTOM_SHA_OUTPUT_TF), samples/tf)
+
+CONFIG := -f build/config/schema.yaml -f build/config/services/ -f $(INPUT_VALUES) -f build/ytt_lib/
 
 YQ := $(shell command -v yq 2> /dev/null)
 ifndef YQ
     $(error "yq is not installed or not in PATH. Please install yq: https://github.com/mikefarah/yq/#install")
 endif
 
-ORGANIZATION_ID := $(shell $(YQ) '.organization' values.yaml)
+ORGANIZATION_ID := $(shell $(YQ) '.organization' ${INPUT_VALUES})
 
 .PHONY: sha
 sha: 

tools/custom-organization-policy-library/Makefile

@@ -1,7 +1,10 @@
-SHELL 	:= /usr/bin/env bash	# Make will use bash instead of sh
-CONFIG 	:= -f build/config/schema.yaml -f build/config/services/ -f values.yaml -f build/ytt_lib/
-OUTPUT_GCLOUD 	:= samples/gcloud
-OUTPUT_TF := samples/tf
+SHELL 	:= /usr/bin/env bash
+
+INPUT_VALUES    := $(or $(CUSTOM_OP_INPUT_VALUES), values.yaml)
+OUTPUT_GCLOUD   := $(or $(CUSTOM_OP_OUTPUT_GCLOUD), samples/gcloud)
+OUTPUT_TF       := $(or $(CUSTOM_OP_OUTPUT_TF), samples/tf)
+
+CONFIG 	:= -f build/config/schema.yaml -f build/config/services/ -f $(INPUT_VALUES) -f build/ytt_lib/
 
 # By default, generating all services 
 # For testing, can be set SERVICE := "cloudsql"

tools/custom-organization-policy-library/build/custom-constraints/iam/iamDisableAdminServiceAccount.yaml

@@ -10,7 +10,8 @@ methodTypes:
 - UPDATE
 condition: >- 
   resource.bindings.exists(binding,
-    binding.members.exists(member,MemberSubjectStartsWith(member, ['serviceAccount:'])) &&
+    binding.members.exists(member, !MemberSubjectEndsWith(member, ['@cloudservices.gserviceaccount.com']) &&
+    MemberSubjectStartsWith(member, ['serviceAccount:'])) &&
     (
       RoleNameMatches(binding.role, ["roles/editor"]) ||
       RoleNameMatches(binding.role, ["roles/owner"]) ||

tools/custom-organization-policy-library/build/custom-constraints/iam/iamDisableBasicRoles.yaml

@@ -8,7 +8,15 @@ resourceTypes:
 methodTypes:
 - CREATE
 - UPDATE
-condition: resource.bindings.exists(bind, RoleNameMatches(bind.role, ["roles/viewer"]) || RoleNameMatches(bind.role, ["roles/editor"]) || RoleNameMatches(bind.role, ["roles/owner"]))
+condition: >- 
+  resource.bindings.exists(binding,
+    binding.members.exists(member,MemberSubjectStartsWith(member, ['user:', 'group'])) &&
+    (
+        RoleNameMatches(binding.role, ["roles/editor"]) ||
+        RoleNameMatches(binding.role, ["roles/owner"]) ||
+        RoleNameContains(binding.role, ["roles/viewer"])
+    )
+  )
 actionType: DENY
 displayName: Deny use of the basic roles
 description: Ensure no use of the basic roles (viewer, editor and owner)

tools/custom-organization-policy-library/samples/gcloud/constraints/iam/iamDisableAdminServiceAccount.yaml

@@ -6,7 +6,8 @@ methodTypes:
 - UPDATE
 condition: |-
   resource.bindings.exists(binding,
-    binding.members.exists(member,MemberSubjectStartsWith(member, ['serviceAccount:'])) &&
+    binding.members.exists(member, !MemberSubjectEndsWith(member, ['@cloudservices.gserviceaccount.com']) &&
+    MemberSubjectStartsWith(member, ['serviceAccount:'])) &&
     (
       RoleNameMatches(binding.role, ["roles/editor"]) ||
       RoleNameMatches(binding.role, ["roles/owner"]) ||

tools/custom-organization-policy-library/samples/gcloud/constraints/iam/iamDisableBasicRoles.yaml

@@ -4,7 +4,15 @@ resourceTypes:
 methodTypes:
 - CREATE
 - UPDATE
-condition: resource.bindings.exists(bind, RoleNameMatches(bind.role, ["roles/viewer"]) || RoleNameMatches(bind.role, ["roles/editor"]) || RoleNameMatches(bind.role, ["roles/owner"]))
+condition: |-
+  resource.bindings.exists(binding,
+    binding.members.exists(member,MemberSubjectStartsWith(member, ['user:', 'group'])) &&
+    (
+        RoleNameMatches(binding.role, ["roles/editor"]) ||
+        RoleNameMatches(binding.role, ["roles/owner"]) ||
+        RoleNameContains(binding.role, ["roles/viewer"])
+    )
+  )
 actionType: DENY
 displayName: Deny use of the basic roles
 description: Ensure no use of the basic roles (viewer, editor and owner)

tools/custom-organization-policy-library/samples/tf/custom-constraints/custom.iamDisableAdminServiceAccount.yaml

@@ -2,7 +2,8 @@ custom.iamDisableAdminServiceAccount:
   action_type: DENY
   condition: |-
     resource.bindings.exists(binding,
-      binding.members.exists(member,MemberSubjectStartsWith(member, ['serviceAccount:'])) &&
+      binding.members.exists(member, !MemberSubjectEndsWith(member, ['@cloudservices.gserviceaccount.com']) &&
+      MemberSubjectStartsWith(member, ['serviceAccount:'])) &&
       (
         RoleNameMatches(binding.role, ["roles/editor"]) ||
         RoleNameMatches(binding.role, ["roles/owner"]) ||

tools/custom-organization-policy-library/samples/tf/custom-constraints/custom.iamDisableBasicRoles.yaml

@@ -1,7 +1,14 @@
 custom.iamDisableBasicRoles:
   action_type: DENY
   condition: |-
-    resource.bindings.exists(bind, RoleNameMatches(bind.role, ["roles/viewer"]) || RoleNameMatches(bind.role, ["roles/editor"]) || RoleNameMatches(bind.role, ["roles/owner"]))
+    resource.bindings.exists(binding,
+      binding.members.exists(member,MemberSubjectStartsWith(member, ['user:', 'group'])) &&
+      (
+          RoleNameMatches(binding.role, ["roles/editor"]) ||
+          RoleNameMatches(binding.role, ["roles/owner"]) ||
+          RoleNameContains(binding.role, ["roles/viewer"])
+      )
+    )
   description: Ensure no use of the basic roles (viewer, editor and owner)
   display_name: Deny use of the basic roles
   method_types:

tools/monitoring-alert-library/Makefile

@@ -1,26 +1,42 @@
-# Make will use bash instead of sh
 SHELL 	:= /usr/bin/env bash
-CONFIG := -f build/config/schema.yaml -f values.yaml -f build/ytt_lib/
-OUTPUT_GCLOUD 	:= samples/gcloud
+
+INPUT_VALUES    := $(or $(MONITORING_ALERT_INPUT_VALUES), values.yaml)
+OUTPUT_GCLOUD   := $(or $(MONITORING_ALERT_OUTPUT_GCLOUD), samples/gcloud)
+OUTPUT_TF       := $(or $(MONITORING_ALERT_OUTPUT_TF), samples/tf)
+CONFIG := -f build/config/schema.yaml -f $(INPUT_VALUES) -f build/ytt_lib/
+
 
 YQ := $(shell command -v yq 2> /dev/null)
 ifndef YQ
     $(error "yq is not installed or not in PATH. Please install yq: https://github.com/mikefarah/yq/#install")
 endif
 
-PROJECT_ID := $(shell $(YQ) '.project' values.yaml)
+PROJECT_ID := $(shell $(YQ) '.project' ${INPUT_VALUES})
 
 .PHONY: alerts
 alerts:
+	rm -rf $(OUTPUT_GCLOUD)/log-based-metrics	
 	rm -rf $(OUTPUT_GCLOUD)/alerts
+	ytt $(CONFIG) -f build/log-based-metrics/  --output-files $(OUTPUT_GCLOUD)/log-based-metrics
 	ytt $(CONFIG) -f build/alerts/  --output-files $(OUTPUT_GCLOUD)/alerts
 
+.PHONY: alerts-tf
+alerts-tf: alerts
+	python3 scripts/cvt-tf-alerts.py $(OUTPUT_GCLOUD)/log-based-metrics $(OUTPUT_TF) logging_metrics
+	python3 scripts/cvt-tf-alerts.py $(OUTPUT_GCLOUD)/alerts $(OUTPUT_TF) alerts
+
 .PHONY: build
 build: clean alerts
 
+.PHONY: build-tf
+build-tf: clean alerts-tf
+
+.PHONY: all
+all: clean build build-tf
+
 .PHONY: deploy-alerts
 deploy-alerts:
-	sh scripts/deploy.sh  $(PROJECT_ID)  $(OUTPUT_GCLOUD)/alerts
+	sh scripts/deploy.sh  $(PROJECT_ID)  $(OUTPUT_GCLOUD)/log-based-metrics $(OUTPUT_GCLOUD)/alerts
 
 .PHONY: deploy
 deploy: build deploy-alerts

tools/monitoring-alert-library/build/alerts/auditConfigChanges.yaml

@@ -16,23 +16,27 @@ documentation:
     ```
   mimeType: text/markdown
 conditions:
-  - displayName: 'Log match condition: audit configuration changes'
-    conditionMatchedLog:
-      filter: >-
-        protoPayload.methodName="SetIamPolicy" AND
-        protoPayload.serviceData.policyDelta.auditConfigDeltas:*
-      labelExtractors:
-        principal: "EXTRACT(protoPayload.authenticationInfo.principalEmail)"
-        method_name: "EXTRACT(protoPayload.methodName)"
-        organization_id: "EXTRACT(labels.organization_id)"
-        folder_id: "EXTRACT(labels.folder_id)"
-        project_id: "EXTRACT(labels.project_id)"
+- displayName: 'Log match condition: Audit Configuration Changes'
+  conditionThreshold:
+    filter: >-
+      resource.type = "logging_bucket" AND metric.type = "logging.googleapis.com/user/auditConfigChanges"
+    aggregations:
+    - perSeriesAligner: ALIGN_SUM
+      alignmentPeriod: 60s
+      crossSeriesReducer: REDUCE_SUM
+      groupByFields:
+      - metric.label.principal
+      - metric.label.method_name
+      - metric.label.organization_id
+      - metric.label.folder_id
+      - metric.label.project_id
+    comparison: COMPARISON_GT
+    thresholdValue: 0
+    duration: 0s
+    trigger:
+      count: 1
 combiner: OR
 #@ if alert.has_notification_channels():
 _: #@ template.replace(generate_notification_channels())
 #@ end
-alertStrategy:
-  notificationRateLimit:
-    period: 300s
-  autoClose: 604800s
 #@ end

tools/monitoring-alert-library/build/alerts/binaryAuthorizationPolicyChanges.yaml

@@ -18,22 +18,25 @@ documentation:
     ```
   mimeType: text/markdown
 conditions:
-  - displayName: 'Log match condition: Cloud SQL instance configuration changes'
-    conditionMatchedLog:
-      filter: >-
-        protoPayload.methodName="google.cloud.binaryauthorization.v1.BinauthzManagementServiceV1.UpdatePolicy" AND 
-        protoPayload.serviceName="binaryauthorization.googleapis.com" AND 
-        ("ALWAYS_ALLOW" OR "DRYRUN_AUDIT_LOG_ONLY")
-      labelExtractors:
-        principal: "EXTRACT(protoPayload.authenticationInfo.principalEmail)"
-        method_name: "EXTRACT(protoPayload.methodName)"
-        project_id: "EXTRACT(labels.project_id)"
+- displayName: 'Log match condition: Cloud SQL instance configuration changes'
+  conditionThreshold:
+    filter: >-
+      resource.type = "logging_bucket" AND metric.type = "logging.googleapis.com/user/binaryAuthorizationPolicyChanges"
+    aggregations:
+    - perSeriesAligner: ALIGN_SUM
+      alignmentPeriod: 60s
+      crossSeriesReducer: REDUCE_SUM
+      groupByFields:
+      - metric.label.principal
+      - metric.label.method_name
+      - metric.label.project_id
+    comparison: COMPARISON_GT
+    thresholdValue: 0
+    duration: 0s
+    trigger:
+      count: 1
 combiner: OR
 #@ if alert.has_notification_channels():
 _: #@ template.replace(generate_notification_channels())
 #@ end
-alertStrategy:
-  notificationRateLimit:
-    period: 300s
-  autoClose: 604800s
-#@ end
+#@ end

tools/monitoring-alert-library/build/alerts/cloudsqlInstanceChanges.yaml

@@ -15,21 +15,26 @@ documentation:
     ```
   mimeType: text/markdown
 conditions:
-  - displayName: 'Log match condition: Cloud SQL instance configuration changes'
-    conditionMatchedLog:
-      filter: >-
-        protoPayload.methodName="cloudsql.instances.update"
-      labelExtractors:
-        principal: "EXTRACT(protoPayload.authenticationInfo.principalEmail)"
-        method_name: "EXTRACT(protoPayload.methodName)"
-        project_id: "EXTRACT(labels.project_id)"
-        database_id: "EXTRACT(labels.database_id)"
+- displayName: 'Log match condition: Cloud SQL instance configuration changes'
+  conditionThreshold:
+    filter: >-
+      resource.type = "logging_bucket" AND metric.type = "logging.googleapis.com/user/cloudsqlInstanceChanges"
+    aggregations:
+    - perSeriesAligner: ALIGN_SUM
+      alignmentPeriod: 60s
+      crossSeriesReducer: REDUCE_SUM
+      groupByFields:
+      - metric.label.principal
+      - metric.label.method_name
+      - metric.label.project_id
+      - metric.label.database_id
+    comparison: COMPARISON_GT
+    thresholdValue: 0
+    duration: 0s
+    trigger:
+      count: 1
 combiner: OR
 #@ if alert.has_notification_channels():
 _: #@ template.replace(generate_notification_channels())
 #@ end
-alertStrategy:
-  notificationRateLimit:
-    period: 300s
-  autoClose: 604800s
-#@ end
+#@ end

tools/monitoring-alert-library/build/alerts/customRoleChanges.yaml

@@ -20,27 +20,27 @@ documentation:
     ```
   mimeType: text/markdown
 conditions:
-  - displayName: 'Log match condition: custom role changes'
-    conditionMatchedLog:
-      filter: >-
-        resource.type="iam_role" AND 
-        (
-          protoPayload.methodName="google.iam.admin.v1.CreateRole" OR 
-          protoPayload.methodName="google.iam.admin.v1.UpdateRole" OR 
-          protoPayload.methodName="google.iam.admin.v1.DeleteRole"
-        )
-      labelExtractors:
-        principal: "EXTRACT(protoPayload.authenticationInfo.principalEmail)"
-        method_name: "EXTRACT(protoPayload.methodName)"
-        organization_id: "EXTRACT(labels.organization_id)"
-        project_id: "EXTRACT(labels.project_id)"
-        role_name: "EXTRACT(labels.role_name)"
+- displayName: 'Log match condition: custom role changes'
+  conditionThreshold:
+    filter: >-
+      resource.type = "logging_bucket" AND metric.type = "logging.googleapis.com/user/customRoleChanges"
+    aggregations:
+    - perSeriesAligner: ALIGN_SUM
+      alignmentPeriod: 60s
+      crossSeriesReducer: REDUCE_SUM
+      groupByFields:
+      - metric.label.principal
+      - metric.label.method_name
+      - metric.label.organization_id
+      - metric.label.project_id
+      - metric.label.role_name
+    comparison: COMPARISON_GT
+    thresholdValue: 0
+    duration: 0s
+    trigger:
+      count: 1
 combiner: OR
 #@ if alert.has_notification_channels():
 _: #@ template.replace(generate_notification_channels())
 #@ end
-alertStrategy:
-  notificationRateLimit:
-    period: 300s
-  autoClose: 604800s
-#@ end
+#@ end

tools/monitoring-alert-library/build/alerts/dnsMalformedQuery.yaml

@@ -15,19 +15,24 @@ documentation:
     ```
   mimeType: text/markdown
 conditions:
-  - displayName: 'Log match condition: dns malformed queries'
-    conditionMatchedLog:
-      filter: >-
-        resource.type="dns_query" AND jsonPayload.responseCode="FORMERR"
-      labelExtractors:
-        principal: "EXTRACT(protoPayload.authenticationInfo.principalEmail)"
-        project_id: "EXTRACT(labels.project_id)"
+- displayName: 'Log match condition: DNS Malformed Queries'
+  conditionThreshold:
+    filter: >-
+      resource.type = "logging_bucket" AND metric.type = "logging.googleapis.com/user/dnsMalformedQuery"
+    aggregations:
+    - perSeriesAligner: ALIGN_SUM
+      alignmentPeriod: 60s
+      crossSeriesReducer: REDUCE_SUM
+      groupByFields:
+      - metric.label.principal
+      - metric.label.project_id
+    comparison: COMPARISON_GT
+    thresholdValue: 0
+    duration: 0s
+    trigger:
+      count: 1
 combiner: OR
 #@ if alert.has_notification_channels():
 _: #@ template.replace(generate_notification_channels())
 #@ end
-alertStrategy:
-  notificationRateLimit:
-    period: 300s
-  autoClose: 604800s
-#@ end
+#@ end