Merge pull request #2425 from confluentinc/pr_merge_from_8_1_x_to_8_2_x

Merge Conflict Resolution (from 8.1.x to 8.2.x)

Author: rkunwar-28

docs/VARIABLES.md

@@ -0,0 +1,87 @@
+---
+##
+## The following is an example inventory file for deploying Confluent Platform
+## using AWS Systems Manager (SSM) as the connection method with hostname aliasing enabled.
+##
+## For full SSM setup details, prerequisites, and configuration, see:
+##   https://docs.confluent.io/ansible/7.9/ansible-prepare.html#manage-connections-to-cp-hosts
+##
+## When using SSM, ansible_host must contain the EC2 Instance ID for SSM to connect.
+## Since hostname_aliasing_enabled uses ansible_host as a fallback for resolving
+## the internal hostname, you must explicitly set the "hostname" variable on each host
+## to the actual network address (FQDN or IP) so that Confluent Platform components
+## can communicate with each other correctly.
+##
+## Resolution order when hostname_aliasing_enabled is true:
+##   1. hostname (host variable)
+##   2. ansible_host (fallback - but this is the Instance ID with SSM, so don't rely on it)
+##   3. inventory_hostname (final fallback)
+##
+## Usage:
+##   ansible-playbook -i docs/sample_inventories/ssm/ssm_hostname_aliasing.yml confluent.platform.all
+
+all:
+  vars:
+    ## SSM Connection Settings
+    ansible_connection: amazon.aws.aws_ssm
+    ansible_aws_ssm_region: "us-east-2"
+    ansible_aws_ssm_bucket_name: "cp-ansible-ssm-staging-bucket"
+    ansible_become: true
+
+    ## Enable hostname aliasing so that the "hostname" variable is used
+    ## for internal CP component addressing instead of ansible_host (which holds the Instance ID).
+    hostname_aliasing_enabled: true
+
+kafka_controller:
+  hosts:
+    kc1:
+      ansible_host: i-0a1b2c3d4e5f60001
+      hostname: ip-172-31-10-101.us-east-2.compute.internal
+    kc2:
+      ansible_host: i-0a1b2c3d4e5f60002
+      hostname: ip-172-31-10-102.us-east-2.compute.internal
+    kc3:
+      ansible_host: i-0a1b2c3d4e5f60003
+      hostname: ip-172-31-10-103.us-east-2.compute.internal
+
+kafka_broker:
+  hosts:
+    kb1:
+      ansible_host: i-0a1b2c3d4e5f60004
+      hostname: ip-172-31-10-104.us-east-2.compute.internal
+    kb2:
+      ansible_host: i-0a1b2c3d4e5f60005
+      hostname: ip-172-31-10-105.us-east-2.compute.internal
+    kb3:
+      ansible_host: i-0a1b2c3d4e5f60006
+      hostname: ip-172-31-10-106.us-east-2.compute.internal
+
+schema_registry:
+  hosts:
+    sr1:
+      ansible_host: i-0a1b2c3d4e5f60007
+      hostname: ip-172-31-10-107.us-east-2.compute.internal
+
+kafka_connect:
+  hosts:
+    kc-connect1:
+      ansible_host: i-0a1b2c3d4e5f60008
+      hostname: ip-172-31-10-108.us-east-2.compute.internal
+
+kafka_rest:
+  hosts:
+    kr1:
+      ansible_host: i-0a1b2c3d4e5f60009
+      hostname: ip-172-31-10-109.us-east-2.compute.internal
+
+ksql:
+  hosts:
+    ksql1:
+      ansible_host: i-0a1b2c3d4e5f60010
+      hostname: ip-172-31-10-110.us-east-2.compute.internal
+
+control_center_next_gen:
+  hosts:
+    c3ng1:
+      ansible_host: i-0a1b2c3d4e5f60011
+      hostname: ip-172-31-10-111.us-east-2.compute.internal

docs/sample_inventories/ssm/ssm_hostname_aliasing.yml

@@ -0,0 +1,66 @@
+---
+##
+## The following is an example inventory file for deploying Confluent Platform
+## using AWS Systems Manager (SSM) as the connection method without hostname aliasing.
+##
+## For full SSM setup details, prerequisites, and configuration, see:
+##   https://docs.confluent.io/ansible/7.9/ansible-prepare.html#manage-connections-to-cp-hosts
+##
+## When hostname_aliasing_enabled is false (default), the resolve_hostname filter
+## returns inventory_hostname. Therefore, the inventory hostname must be the actual
+## network address (FQDN) of the EC2 instance, and ansible_host holds the Instance ID
+## for SSM to connect.
+##
+## Usage:
+##   ansible-playbook -i docs/sample_inventories/ssm/ssm_without_hostname_aliasing.yml confluent.platform.all
+
+all:
+  vars:
+    ## SSM Connection Settings
+    ansible_connection: amazon.aws.aws_ssm
+    ansible_aws_ssm_region: "us-east-2"
+    ansible_aws_ssm_bucket_name: "cp-ansible-ssm-staging-bucket"
+    ansible_become: true
+
+kafka_controller:
+  hosts:
+    ip-172-31-10-101.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60001
+    ip-172-31-10-102.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60002
+    ip-172-31-10-103.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60003
+
+kafka_broker:
+  hosts:
+    ip-172-31-10-104.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60004
+    ip-172-31-10-105.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60005
+    ip-172-31-10-106.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60006
+
+schema_registry:
+  hosts:
+    ip-172-31-10-107.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60007
+
+kafka_connect:
+  hosts:
+    ip-172-31-10-108.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60008
+
+kafka_rest:
+  hosts:
+    ip-172-31-10-109.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60009
+
+ksql:
+  hosts:
+    ip-172-31-10-110.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60010
+
+control_center_next_gen:
+  hosts:
+    ip-172-31-10-111.us-east-2.compute.internal:
+      ansible_host: i-0a1b2c3d4e5f60011

docs/sample_inventories/ssm/ssm_without_hostname_aliasing.yml

@@ -6,7 +6,6 @@
   # Extract DNAME from line
   # Remove spaces after commas
   shell: |
-    set -o pipefail
     keytool -list -keystore {{keystore_path}} \
         -storepass {{keystore_storepass}} \
         {% if fips_enabled|bool and ('kafka_controller' in group_names or 'kafka_broker' in group_names) %}

roles/common/tasks/cert_principal_extract.yml

@@ -17,7 +17,7 @@
 
 - name: Retrieve SSL public key hash from private key on Local Host
   shell:
-    cmd: set -o pipefail; openssl pkey -pubout | openssl sha256
+    cmd: openssl pkey -pubout | openssl sha256
     stdin: "{{ lookup('file', ssl_key_filepath) }}"
     executable: "{{ shell_executable }}"
   register: key_hash_local
@@ -45,7 +45,7 @@
 
 - name: Retrieve SSL public key Hash from private key on Remote Host
   shell:
-    cmd: set -o pipefail; openssl pkey -pubout | openssl sha256
+    cmd: openssl pkey -pubout | openssl sha256
     stdin: "{{ remote_key['results'][group_idx].content | b64decode }}"
     executable: "{{ shell_executable }}"
   register: key_hash_remote
@@ -62,7 +62,7 @@
 
 - name: Retrieve SSL public key hash from X509 certificate on Local Host
   shell:
-    cmd: set -o pipefail; openssl x509 -noout -pubkey | openssl sha256
+    cmd: openssl x509 -noout -pubkey | openssl sha256
     stdin: "{{ lookup('file', ssl_signed_cert_filepath) }}"
     executable: "{{ shell_executable }}"
   register: cert_hash_local
@@ -90,7 +90,7 @@
 
 - name: Retrieve SSL public key hash from X509 certificate on Remote Host
   shell:
-    cmd: set -o pipefail; openssl x509 -noout -pubkey | openssl sha256
+    cmd: openssl x509 -noout -pubkey | openssl sha256
     stdin: "{{ remote_cert['results'][group_idx].content | b64decode }}"
     executable: "{{ shell_executable }}"
   register: cert_hash_remote

roles/common/tasks/config_validations.yml

@@ -8,7 +8,6 @@
 - name: Generate Master Encryption Key and File
   tags: masterkey
   shell: |
-    set -o pipefail
     {{ confluent_cli_path }} secret master-key generate \
       --local-secrets-file /tmp/security.properties \
       --passphrase @/tmp/passphrase.txt | awk '/Master/{print $5}'

roles/common/tasks/masterkey.yml

@@ -145,9 +145,7 @@
     - validation
 
 - name: Validate Cron Jobs for Logrotate
-  shell: |
-    set -o pipefail
-    crontab -l | grep -c "{{ item }}"
+  shell: crontab -l | grep -c "{{ item }}"
   args:
     executable: "{{ shell_executable }}"
   register: logrotate_cron_validation

roles/control_center_next_gen/tasks/health_check.yml

@@ -26,7 +26,6 @@
   # Extract DNAME from line
   # Remove spaces after commas
   shell: |
-    set -o pipefail
     keytool -list -keystore {{kb_keystore_path}} \
         -storepass {{kb_keystore_storepass}} \
         {% if fips_enabled|bool %}

roles/kafka_broker/tasks/set_principal.yml

@@ -39,7 +39,6 @@
 #Registers LEO of controllers only if Metadata Quorum passed in the above task
 - name: Register LogEndOffset
   shell: |
-    set -o pipefail
     {{ binary_base_path }}/bin/kafka-metadata-quorum --bootstrap-{{bootstrap_server_or_controller}} {{server_hostname}}:{{server_port}} \
       --command-config {{kafka_controller.client_config_file}} describe --replication |  grep -v Observer | awk '{print $3}'
   args:

roles/kafka_controller/tasks/health_check.yml

@@ -26,7 +26,6 @@
   # Extract DNAME from line
   # Remove spaces after commas
   shell: |
-    set -o pipefail
     keytool -list -keystore {{kc_keystore_path}} \
         -storepass {{kc_keystore_storepass}} \
         {% if fips_enabled|bool %}