Support CPMS test role in shiftstack-qa automation

collection/stages/roles/cpms_test/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+# defaults file for cpms_test
+cpms_test_name: cluster-control-plane-machine-set-operator
+cpms_test_dir: "{{ artifacts_dir }}/{{ cpms_test_name }}"
+cpms_test_results_dir: "{{ artifacts_dir }}/cpms_test-results"
+cpms_test_testsuite_name: cpms
+cpms_tests_go_version: "{{ tests.default_go_version_target }}"
+cpms_test_junit_filename: junit_control_plane_machine_set_operator.xml

collection/stages/roles/cpms_test/meta/main.yml

@@ -0,0 +1,3 @@
+---
+collections:
+  - shiftstack.tools

collection/stages/roles/cpms_test/tasks/main.yml

@@ -0,0 +1,25 @@
+---
+# tasks file for cpms_test
+- name: Prepare CPMS test
+  ansible.builtin.include_role:
+    name: tools_openshift_tests
+    tasks_from: prepare_openshift_tests.yml
+  vars:
+    repo_name: "{{ cpms_test_name }}"
+    results_dir: "{{ cpms_test_results_dir }}"
+    go_version_target: "{{ cpms_tests_go_version }}"
+
+- name: Run CPMS e2e-presubmit tests
+  ansible.builtin.include_tasks: run_cpms_test.yml
+  vars:
+    cpms_tests_type: e2e-presubmit
+
+- name: Run CPMS e2e-periodic tests
+  ansible.builtin.include_tasks: run_cpms_test.yml
+  vars:
+    cpms_tests_type: e2e-periodic
+
+- name: Remove the source directory after tests complete
+  ansible.builtin.file:
+    path: "{{ cpms_test_dir }}"
+    state: absent

collection/stages/roles/cpms_test/tasks/run_cpms_test.yml

@@ -0,0 +1,57 @@
+---
+- name: Set result paths for {{ cpms_tests_type }}
+  ansible.builtin.set_fact:
+    cpms_results_subdir: "{{ cpms_test_results_dir }}/{{ cpms_tests_type }}"
+
+- name: Create results directory for {{ cpms_tests_type }}
+  ansible.builtin.file:
+    path: "{{ cpms_results_subdir }}"
+    state: directory
+    mode: u=rwx,g=rw,o=r
+
+- name: Run {{ cpms_test_name }} {{ cpms_tests_type }} tests
+  block:
+    - name: Run make {{ cpms_tests_type }}
+      ansible.builtin.shell: |
+        source {{ home_dir }}/.bashrc
+        make {{ cpms_tests_type }}
+      args:
+        chdir: "{{ cpms_test_dir }}"
+      environment:
+        KUBECONFIG: "{{ kubeconfig }}"
+        OS_CLOUD: "{{ user_cloud }}"
+        OPENSHIFT_CI: "true"
+        ARTIFACT_DIR: "{{ cpms_results_subdir }}"
+      changed_when: true
+
+    - name: Rename JUnit XML to match expected prefix
+      ansible.builtin.copy:
+        src: "{{ cpms_results_subdir }}/{{ cpms_test_junit_filename }}"
+        dest: "{{ cpms_results_subdir }}/junit_e2e_{{ cpms_tests_type }}.xml"
+        remote_src: yes
+        mode: u=rw,g=rw,o=r
+
+    - name: Post openshift-test
+      ansible.builtin.include_role:
+        name: tools_openshift_tests
+        tasks_from: post_openshift_tests.yml
+      vars:
+        testsuite_name: "{{ cpms_test_testsuite_name }}_{{ cpms_tests_type }}"
+        key_for_filtering_results: "cpms"
+        test_name: "{{ cpms_test_name }}-{{ cpms_tests_type }}"
+        results_dir: "{{ cpms_results_subdir }}"
+
+  rescue:
+    - name: Mark the CPMS {{ cpms_tests_type }} tests as UNSTABLE
+      ansible.builtin.include_role:
+        name: tools_stage_results
+        tasks_from: mark_stage_unstable.yml
+      vars:
+        unstable_msg: >-
+          The {{ cpms_test_name }} {{ cpms_tests_type }} test suite failed.
+
+    - name: Run must-gather
+      ansible.builtin.include_role:
+        name: tools_must-gather
+      vars:
+        must_gather_suffix: "cpms-{{ cpms_tests_type }}"

collection/stages/roles/day2ops/tasks/procedures/cpms_replace_attrs.yml

@@ -0,0 +1,228 @@
+---
+# Procedure that updates the CPMS object by adding a fake extra network/subnet and SG to the
+# masters and replacing failureDomain params on master-0 to have the same attributes as master-1.
+# After reconciliation, master-0 and master-1 will have same nova AZ, cinderAZ, and volumeType.
+- name: Set log directory for cpms_replace_attrs procedure
+  ansible.builtin.set_fact:
+    cpms_log_directory: "{{ artifacts_dir }}/cpms_replace_attrs"
+
+- name: Run cpms_replace_attrs procedure
+  block:
+    - name: Create {{ cpms_log_directory }} directory
+      ansible.builtin.file:
+        path: "{{ cpms_log_directory }}"
+        state: directory
+        mode: u=rwx,g=rw,o=r
+
+    - name: Set artifact paths
+      ansible.builtin.set_fact:
+        cpms_orig_path: "{{ cpms_log_directory }}/orig_cpms.yaml"
+        cpms_patch_to_apply_path: "{{ cpms_log_directory }}/cpms_patch_to_apply.yaml"
+        cpms_applied_path: "{{ cpms_log_directory }}/applied_cpms.yaml"
+        cpms_patch_to_restore_path: "{{ cpms_log_directory }}/cpms_patch_to_restore_NOTAPPLIED.yaml"
+
+    - name: Get original CPMS definition
+      kubernetes.core.k8s_info:
+        kubeconfig: "{{ kubeconfig }}"
+        api_version: machine.openshift.io/v1
+        kind: ControlPlaneMachineSet
+        namespace: openshift-machine-api
+      register: original_cpms
+
+    - name: Save the original CPMS definition
+      ansible.builtin.copy:
+        content: "{{ original_cpms.resources[0] | to_nice_yaml(indent=2) }}"
+        dest: "{{ cpms_orig_path }}"
+        mode: u=rw,g=rw,o=r
+
+    - name: Create network {{ cpms_replacements.network_name }}
+      openstack.cloud.network:
+        cloud: "{{ user_cloud }}"
+        name: "{{ cpms_replacements.network_name }}"
+        state: present
+      register: network_to_add
+
+    - name: Create subnet {{ cpms_replacements.subnet_name }}
+      openstack.cloud.subnet:
+        cloud: "{{ user_cloud }}"
+        name: "{{ cpms_replacements.subnet_name }}"
+        cidr: "{{ cpms_replacements.cidr }}"
+        network_name: "{{ cpms_replacements.network_name }}"
+        enable_dhcp: false
+        state: present
+
+    - name: Create security group {{ cpms_replacements.sg_name }}
+      openstack.cloud.security_group:
+        cloud: "{{ user_cloud }}"
+        name: "{{ cpms_replacements.sg_name }}"
+        state: present
+      register: sg_to_add
+
+    - name: Store original CPMS attributes that will be updated
+      ansible.builtin.set_fact:
+        orig_failuredomains: >-
+          {{ original_cpms.resources[0].spec.template.machines_v1beta1_machine_openshift_io.failureDomains.openstack | list }}
+        orig_networks: >-
+          {{ original_cpms.resources[0].spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value.networks | list }}
+        orig_sgs: >-
+          {{ original_cpms.resources[0].spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value.securityGroups | list }}
+
+    - name: Create patch for CPMS
+      ansible.builtin.template:
+        src: control-plane-machine-set-patch.yaml.j2
+        dest: "{{ cpms_patch_to_apply_path }}"
+        mode: u=rw,g=rw,o=r
+      vars:
+        cpms_failuredomains: >-
+          {{ orig_failuredomains |
+            replace(orig_failuredomains[0].availabilityZone, orig_failuredomains[1].availabilityZone) |
+            replace(orig_failuredomains[0].rootVolume.availabilityZone, orig_failuredomains[1].rootVolume.availabilityZone) |
+            replace(orig_failuredomains[0].rootVolume.volumeType, orig_failuredomains[1].rootVolume.volumeType) }}
+        cpms_networks: "{{ orig_networks + [{'filter': {}, 'uuid': network_to_add.id}] }}"
+        cpms_sgs: "{{ orig_sgs + [{'filter': {}, 'name': cpms_replacements.sg_name}] }}"
+
+    - name: Apply patch for CPMS
+      kubernetes.core.k8s:
+        kubeconfig: "{{ kubeconfig }}"
+        state: present
+        apply: true
+        src: "{{ cpms_patch_to_apply_path }}"
+
+    - name: Wait for CPMS reconciliation to complete
+      ansible.builtin.include_role:
+        name: tools_cluster_checks
+        tasks_from: wait_cpms_updated.yml
+      vars:
+        wait_retries: 180
+        wait_delay: 60
+
+    - name: Check cluster health after CPMS patch application
+      block:
+        - name: Check the Control Plane MachineSet is healthy
+          ansible.builtin.include_role:
+            name: tools_cluster_checks
+            tasks_from: check_controlplane_machinesets.yml
+
+        - name: Wait until OCP cluster is healthy
+          ansible.builtin.include_role:
+            name: tools_cluster_checks
+            tasks_from: wait_until_cluster_is_healthy.yml
+
+    - name: Get resulting CPMS definition after application
+      kubernetes.core.k8s_info:
+        kubeconfig: "{{ kubeconfig }}"
+        api_version: machine.openshift.io/v1
+        kind: ControlPlaneMachineSet
+        namespace: openshift-machine-api
+      register: applied_cpms
+
+    - name: Save the resulting CPMS after application
+      ansible.builtin.copy:
+        content: "{{ applied_cpms.resources[0] | to_nice_yaml(indent=2) }}"
+        dest: "{{ cpms_applied_path }}"
+        mode: u=rw,g=rw,o=r
+
+    - name: Get all the master VMs after procedure
+      openstack.cloud.server_info:
+        cloud: "{{ user_cloud }}"
+        server: "*master*"
+      register: result
+
+    - name: Store the master VMs after procedure
+      ansible.builtin.set_fact:
+        master_after: "{{ result.servers | sort(attribute='name') | list }}"
+
+    - name: Get root volumes info
+      ansible.builtin.shell: >
+        openstack volume show {{ item }} -c type -c availability_zone -f shell
+      environment:
+        OS_CLOUD: "{{ user_cloud }}"
+      register: root_volumes_after
+      changed_when: false
+      with_items:
+        - "{{ master_after[0].volumes[0].id }}"
+        - "{{ master_after[1].volumes[0].id }}"
+        - "{{ master_after[2].volumes[0].id }}"
+
+    - name: Confirm that master VMs reflect FailureDomain changes after procedure
+      ansible.builtin.assert:
+        that:
+          - master_after | length == 3
+          - master_after | selectattr('location.zone', 'eq', expected_fd_az) | list | length == 2
+          - root_volumes_after.results | selectattr('stdout', 'eq',
+             'availability_zone="' + expected_fd_cinderaz + '"\ntype="' + expected_fd_volumeType + '"') | list | length == 2
+      vars:
+        expected_fd_az: "{{ orig_failuredomains[1].availabilityZone }}"
+        expected_fd_cinderaz: "{{ orig_failuredomains[1].rootVolume.availabilityZone }}"
+        expected_fd_volumeType: "{{ orig_failuredomains[1].rootVolume.volumeType }}"
+
+    - name: Confirm that the VMs include the added network and SG
+      ansible.builtin.assert:
+        that:
+          - cpms_replacements.network_name in item.addresses.keys()
+          - cpms_replacements.sg_name in item.security_groups | json_query('[*].name')
+      with_items: "{{ master_after }}"
+
+  rescue:
+    - name: Run must-gather after cpms_replace_attrs failure
+      ansible.builtin.include_role:
+        name: tools_must-gather
+      vars:
+        must_gather_suffix: "cpms-replace-attrs"
+
+    - name: Fail inside rescue block
+      ansible.builtin.fail:
+        msg: "The cpms_replace_attrs procedure failed"
+
+  always:
+    - name: Create restore patch for CPMS
+      ansible.builtin.template:
+        src: control-plane-machine-set-patch.yaml.j2
+        dest: "{{ cpms_patch_to_restore_path }}"
+        mode: u=rw,g=rw,o=r
+      vars:
+        cpms_failuredomains: "{{ orig_failuredomains }}"
+        cpms_networks: "{{ orig_networks }}"
+        cpms_sgs: "{{ orig_sgs }}"
+
+    - name: Apply restore patch for CPMS
+      kubernetes.core.k8s:
+        kubeconfig: "{{ kubeconfig }}"
+        state: present
+        apply: true
+        src: "{{ cpms_patch_to_restore_path }}"
+
+    - name: Wait for CPMS restore reconciliation to complete
+      ansible.builtin.include_role:
+        name: tools_cluster_checks
+        tasks_from: wait_cpms_updated.yml
+      vars:
+        wait_retries: 180
+        wait_delay: 60
+
+    - name: Check cluster health after restore
+      block:
+        - name: Check the Control Plane MachineSet is healthy
+          ansible.builtin.include_role:
+            name: tools_cluster_checks
+            tasks_from: check_controlplane_machinesets.yml
+
+        - name: Wait until OCP cluster is healthy
+          ansible.builtin.include_role:
+            name: tools_cluster_checks
+            tasks_from: wait_until_cluster_is_healthy.yml
+
+    - name: Clean up test OpenStack resources
+      block:
+        - name: Delete test network
+          openstack.cloud.network:
+            cloud: "{{ user_cloud }}"
+            name: "{{ cpms_replacements.network_name }}"
+            state: absent
+
+        - name: Delete test security group
+          openstack.cloud.security_group:
+            cloud: "{{ user_cloud }}"
+            name: "{{ cpms_replacements.sg_name }}"
+            state: absent
+      ignore_errors: true

collection/stages/roles/day2ops/templates/control-plane-machine-set-patch.yaml.j2

@@ -0,0 +1,19 @@
+apiVersion: machine.openshift.io/v1
+metadata:
+  name: cluster
+  namespace: openshift-machine-api
+kind: ControlPlaneMachineSet
+spec:
+  template:
+    machines_v1beta1_machine_openshift_io:
+      failureDomains:
+        openstack:
+        {{cpms_failuredomains| to_nice_yaml| indent(8, false)}}
+      spec:
+        providerSpec:
+          value:
+            networks:
+            {{cpms_networks| to_nice_yaml| indent(12, false)}}
+            securityGroups:
+            {{cpms_sgs| to_nice_yaml| indent(12, false)}}
+status: {}

jobs_definitions/4.17_ovnkubernetes_ipi.yaml

@@ -10,13 +10,21 @@ stages:
   - post
   - verification
   - day2ops
+  - cpms_test
   - openstack_test
   - conformance_test
   - cinder_csi_tests
   - manila_csi_tests
 
 day2ops_procedures:
   - moving-etcd-to-ephemeral
+  - cpms_replace_attrs
+
+cpms_replacements:
+  network_name: cpms-test-network
+  subnet_name: cpms-test-subnet
+  cidr: 192.168.240.0/24
+  sg_name: cpms-test-sg
 
 ocp_deployment_topology:
   network_type: OVNKubernetes

jobs_definitions/osp_verification.yaml

@@ -19,6 +19,7 @@ stages:
   - install
   - post
   - verification
+  - cpms_test
   - openstack_test
   - lb_tests
 

playbooks/ocp_testing.yaml

@@ -96,6 +96,10 @@
   ansible.builtin.import_playbook: plays/conformance_test.yaml
   when: "'conformance_test' in stages"
 
+- name: Run CPMS e2e tests on OpenShift
+  ansible.builtin.import_playbook: plays/cpms_test.yaml
+  when: "'cpms_test' in stages"
+
 - name: Run Openshift Cinder CSI Tests on OpenShift
   ansible.builtin.import_playbook: plays/cinder_csi.yaml
   when: "'cinder_csi_tests' in stages"