From 80d7292ca24599ec3b61d55fe9a1742ff8863471 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Tue, 4 Jul 2023 12:49:50 +0200 Subject: [PATCH] backing up the data is top prio --- test/bin/download_images.sh | 1 + test/resources/TestAgent.py | 57 +++++++++++ test/resources/ostree-data.resource | 50 ++++++++++ test/resources/ostree-health.resource | 37 ++++++++ test/resources/ostree.resource | 115 ++++++++++------------- test/suites-ostree/backup-restore.robot | 2 +- test/suites-ostree/failed-upgrade.robot | 44 +++++++++ test/suites-ostree/fdo.robot | 75 ++------------- test/suites-ostree/healthy-upgrade.robot | 14 +-- 9 files changed, 248 insertions(+), 147 deletions(-) create mode 100644 test/resources/TestAgent.py create mode 100644 test/resources/ostree-data.resource create mode 100644 test/resources/ostree-health.resource create mode 100644 test/suites-ostree/failed-upgrade.robot diff --git a/test/bin/download_images.sh b/test/bin/download_images.sh index 8ff8df7d55..6a75157c15 100755 --- a/test/bin/download_images.sh +++ b/test/bin/download_images.sh @@ -55,5 +55,6 @@ done echo "Updating references" cd "${IMAGEDIR}" +ostree refs --force --create "rhel-9.2-microshift-failing" "rhel-9.2-microshift-source" ostree summary --update --repo=repo ostree summary --view --repo=repo diff --git a/test/resources/TestAgent.py b/test/resources/TestAgent.py new file mode 100644 index 0000000000..016885ffd8 --- /dev/null +++ b/test/resources/TestAgent.py @@ -0,0 +1,57 @@ +import json +import libostree +from robot.libraries.BuiltIn import BuiltIn + +_log = BuiltIn().log + +CONFIG_PATH = "/var/lib/microshift-test-agent.json" + +# Example config +# { +# "deploy-id": { +# "every": [ "prevent_backup" ], +# "1": [ "fail_greenboot" ], +# "2": [ "..." ], +# "3": [ "..." ] +# } +# } + + +class TestAgent: + def __init__(self): + self.cfg = dict() + + def add_action(self, deployment: str, boot: str, action: str) -> None: + if deployment not in self.cfg: + self.cfg[deployment] = {boot: [action]} + + elif boot not in self.cfg[deployment]: + self.cfg[deployment][boot] = [action] + + elif action not in self.cfg[deployment][boot]: + self.cfg[deployment][boot].append(action) + + _log(f"TestAgent Config: {self.cfg}") + + def add_action_for_next_deployment(self, boot: str, action: str) -> None: + self.add_action("next", boot, action) + + def substitute_staged(self) -> None: + if "next" not in self.cfg: + _log("'next' deployment not found") + return + id = libostree.get_staged_deployment_id() + self.cfg[id] = self.cfg["next"] + del self.cfg["next"] + + def write(self) -> None: + self.substitute_staged() + j = json.dumps(self.cfg) + BuiltIn().should_not_be_empty(j) + libostree.remote_sudo( + f"echo '{j}' | sudo tee /var/lib/microshift-test-agent.json" + ) + + def remove(self) -> None: + self.cfg = {} + libostree.remote_sudo("rm /var/lib/microshift-test-agent.json") diff --git a/test/resources/ostree-data.resource b/test/resources/ostree-data.resource new file mode 100644 index 0000000000..d4ba50b968 --- /dev/null +++ b/test/resources/ostree-data.resource @@ -0,0 +1,50 @@ +*** Settings *** +Documentation Keywords for OSTree-based systems + +Resource systemd.resource +Resource microshift-process.resource +Library libostree.py + + +*** Variables *** +${DATA_DIR} "/var/lib/microshift" +${BACKUP_STORAGE} "/var/lib/microshift-backups" + + +*** Keywords *** +Get Future Backup Name For Current Boot + [Documentation] Provides a name for future backup: + ... "4.13" for MicroShift 4.13, otherwise deployment ID + boot ID + + # When system is upgrading from 4.13, there is no metadata regarding previous boot, + # therefore, when naming backup for existing data, "4.13" will be used as deployment ID, + # i.e. it should reside in `/var/lib/microshift-backups/4.13` + ${version}= MicroShift Version + IF ${version.minor} == 13 RETURN 4.13 + + ${deploy_id}= Get Booted Deployment ID + ${boot_id}= Get Current Boot Id + ${backup_name}= Catenate SEPARATOR=_ ${deploy_id} ${boot_id} + RETURN ${backup_name} + +Backup Should Exist + [Documentation] Checks if backup identified by deployment ID + ... and (optionally) boot ID exists + [Arguments] ${deployment_id} ${boot_id}=${EMPTY} + ${exists}= Does Backup Exist ${deployment_id} ${boot_id} + Should Be True ${exists} + +Remove Existing Backup For Current Deployment + [Documentation] Remove any existing backup for currently running deployment + ${deploy_id}= Get Booted Deployment Id + Remove Backups For Deployment ${deploy_id} + +MicroShift 413 Should Not Have Upgrade Artifacts + [Documentation] Verifies that host running MicroShift 4.13 + ... does not have upgrade related artifacts. + + ${version}= MicroShift Version + IF ${version.minor} == 13 + Path Should Not Exist ${BACKUP_STORAGE} + Path Should Not Exist /var/lib/microshift/version + END diff --git a/test/resources/ostree-health.resource b/test/resources/ostree-health.resource new file mode 100644 index 0000000000..aee93e4733 --- /dev/null +++ b/test/resources/ostree-health.resource @@ -0,0 +1,37 @@ +*** Settings *** +Documentation Keywords for OSTree-based systems + +Resource systemd.resource +Resource microshift-process.resource +Resource ostree-data.resource + + +*** Keywords *** +Wait Until Greenboot Health Check Exited + [Documentation] Wait until greenboot healthchecks are done + + Wait Until Keyword Succeeds 10m 15s + ... Greenboot Health Check Exited + +Greenboot Health Check Exited + [Documentation] Checks if greenboot-healthcheck finished running successfully (exited) + + ${value}= Get Systemd Setting greenboot-healthcheck.service SubState + Should Be Equal As Strings ${value} exited + +Wait For Healthy System + [Documentation] Waits for greenboot healthchecks and checks health.json + Wait Until Greenboot Health Check Exited + System Should Be Healthy + +System Should Be Healthy + [Documentation] Verifies if system is healthy by looking at health.json. + ... If this runs after greenboot-healthcheck finishes, it verifies + ... health of current boot. + + # Health of system with MicroShift 4.13 is implicitly checked by "Greenboot Health Check Exited" + ${version}= MicroShift Version + IF ${version.minor} == 13 RETURN + + ${health}= Get Persisted System Health + Should Be Equal As Strings ${health} healthy diff --git a/test/resources/ostree.resource b/test/resources/ostree.resource index 31b81b4adb..93fee38415 100644 --- a/test/resources/ostree.resource +++ b/test/resources/ostree.resource @@ -3,86 +3,69 @@ Documentation Keywords for OSTree-based systems Resource systemd.resource Resource microshift-process.resource +Resource ostree-data.resource +Resource ostree-health.resource Library libostree.py +Library ../resources/TestAgent.py -*** Variables *** -${DATA_DIR} "/var/lib/microshift" -${BACKUP_STORAGE} "/var/lib/microshift-backups" +*** Keywords *** +Make New SSH Connection + [Documentation] Closes all SSH connections and makes a new one. + # Staging deployments potentially introduces multiple reboots + # which could break existing SSH connection + SSHLibrary.Close All Connections + Login MicroShift Host -*** Keywords *** -Wait Until Greenboot Health Check Exited - [Documentation] Wait until greenboot healthchecks are done +Current Deployment Should Be + [Documentation] Checks if currently booted deployment is as expected + [Arguments] ${expected_deploy} - Wait Until Keyword Succeeds 10m 15s - ... Greenboot Health Check Exited - -Greenboot Health Check Exited - [Documentation] Checks if greenboot-healthcheck finished running successfully (exited) - - ${value}= Get Systemd Setting greenboot-healthcheck.service SubState - Should Be Equal As Strings ${value} exited - -Get Future Backup Name For Current Boot - [Documentation] Provides a name for future backup: - ... "4.13" for MicroShift 4.13, otherwise deployment ID + boot ID - - # When system is upgrading from 4.13, there is no metadata regarding previous boot, - # therefore, when naming backup for existing data, "4.13" will be used as deployment ID, - # i.e. it should reside in `/var/lib/microshift-backups/4.13` - ${version}= MicroShift Version - IF ${version.minor} == 13 RETURN 4.13 - - ${deploy_id}= Get Booted Deployment ID - ${boot_id}= Get Current Boot Id - ${backup_name}= Catenate SEPARATOR=_ ${deploy_id} ${boot_id} - RETURN ${backup_name} - -Backup Should Exist - [Documentation] Checks if backup identified by deployment ID - ... and (optionally) boot ID exists - [Arguments] ${deployment_id} ${boot_id}=${EMPTY} - ${exists}= Does Backup Exist ${deployment_id} ${boot_id} - Should Be True ${exists} - -Remove Existing Backup For Current Deployment - [Documentation] Remove any existing backup for currently running deployment - ${deploy_id}= Get Booted Deployment Id - Remove Backups For Deployment ${deploy_id} - -Rebase System And Verify - [Documentation] Deploys given ostree ref on the host, reboots it, - ... and checks if the host is healthy. + Make New SSH Connection + + ${current_deploy}= libostree.Get Booted Deployment Id + Should Be Equal As Strings ${expected_deploy} ${current_deploy} + +Deploy Commit Expecting A Rollback + [Documentation] Deploys given ref and configures test agent for failing greenboot. + ... It expects the system to roll back. [Arguments] ${ref} - Rebase System ${ref} + ${initial_deploy_id}= Get Booted Deployment Id + ${deploy_id}= Rebase System ${ref} + TestAgent.Write Reboot MicroShift Host - Wait Until Greenboot Health Check Exited - System Should Be Healthy + Log To Console "Failing ref deployed - waiting for system to roll back" + Wait Until Keyword Succeeds 20m 15s + ... Current Deployment Should Be ${initial_deploy_id} + Log To Console "System rolled back - deploying target ref" - ${ref_after_reboot}= Get Current Ref - Should Be Equal As Strings ${ref_after_reboot} ${ref} +Deploy Commit Not Expecting A Rollback + [Documentation] Deploys given ref and configures test agent for failing greenboot. + ... It expects the system to roll back. + [Arguments] ${target_ref} -System Should Be Healthy - [Documentation] Verifies if system is healthy by looking at health.json. - ... If this runs after greenboot-healthcheck finishes, it verifies - ... health of current boot. + ${initial_deploy_id}= Get Booted Deployment Id + ${deploy_id}= Rebase System ${target_ref} + Reboot MicroShift Host - # Health of system with MicroShift 4.13 is implicitly checked by "Greenboot Health Check Exited" - ${version}= MicroShift Version - IF ${version.minor} == 13 RETURN + Log To Console "Target ref deployed - starting health checking" + Wait Until Keyword Succeeds 10m 15s + ... System Is Running Right Ref And Healthy ${deploy_id} ${initial_deploy_id} - ${health}= Get Persisted System Health - Should Be Equal As Strings ${health} healthy +System Is Running Right Ref And Healthy + [Documentation] Checks if system is running right reference and is healthy + [Arguments] ${expected_deploy} ${initial_deploy} -MicroShift 413 Should Not Have Upgrade Artifacts - [Documentation] Verifies that host running MicroShift 4.13 - ... does not have upgrade related artifacts. + Make New SSH Connection - ${version}= MicroShift Version - IF ${version.minor} == 13 - Path Should Not Exist ${BACKUP_STORAGE} - Path Should Not Exist /var/lib/microshift/version + ${current_deploy}= libostree.Get Booted Deployment Id + IF "${current_deploy}" == "${initial_deploy}" + Fatal Error "System rolled back to initial deployment" END + + Should Be Equal As Strings ${expected_deploy} ${current_deploy} + Greenboot Health Check Exited + System Should Be Healthy diff --git a/test/suites-ostree/backup-restore.robot b/test/suites-ostree/backup-restore.robot index e8a1b4a2c6..b33be47ac7 100644 --- a/test/suites-ostree/backup-restore.robot +++ b/test/suites-ostree/backup-restore.robot @@ -76,7 +76,7 @@ Make Masquerading Backup Mark System As Unhealthy [Documentation] Marks systems as unhealthy by executing microshift's red script ${stdout} ${stderr} ${rc}= Execute Command - ... /etc/greenboot/red.d/40_microshift_set_unhealthy.sh + ... FORCE=1 /etc/greenboot/red.d/40_microshift_set_unhealthy.sh ... sudo=True return_stderr=True return_rc=True Should Be Equal As Integers 0 ${rc} diff --git a/test/suites-ostree/failed-upgrade.robot b/test/suites-ostree/failed-upgrade.robot new file mode 100644 index 0000000000..65ca6f3440 --- /dev/null +++ b/test/suites-ostree/failed-upgrade.robot @@ -0,0 +1,44 @@ +*** Settings *** +Documentation Tests related to upgrading MicroShift + +Resource ../resources/common.resource +Resource ../resources/ostree.resource +Library Collections + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags ostree + + +*** Variables *** +${USHIFT_HOST} ${EMPTY} +${USHIFT_USER} ${EMPTY} + +${FAILING_REF} ${EMPTY} + + +*** Test Cases *** +Staged Deployment Consistently Fails To Back Up The Data + [Documentation] Verifies that instructions to back up the data + ... ("healthy" system) is not lost if staged deployment fails + ... to back up and it is performed after rolling back. + + Wait For Healthy System + ${backup}= Get Future Backup Name For Current Boot + + TestAgent.Add Action For Next Deployment every prevent_backup + Deploy Commit Expecting A Rollback ${FAILING_REF} + Backup Should Exist ${backup} + + +*** Keywords *** +Setup + [Documentation] Test suite setup + Check Required Env Variables + Should Not Be Empty ${FAILING_REF} FAILING_REF variable is required + Login MicroShift Host + +Teardown + [Documentation] Test suite teardown + Logout MicroShift Host diff --git a/test/suites-ostree/fdo.robot b/test/suites-ostree/fdo.robot index 4c8c9ec429..597f87425c 100644 --- a/test/suites-ostree/fdo.robot +++ b/test/suites-ostree/fdo.robot @@ -26,24 +26,20 @@ FIDO Onboarding Device ... ... It is expected that final deployment will gracefully handle existence of: ... MicroShift data, unhealthy stored in health file, - ... and a deployment gap (no-microshift rollback ). + ... and a deployment gap (no-microshift rollback). System Should Not Feature MicroShift - ${initial_deploy_id}= Get Booted Deployment Id - Deploy Commit Expecting A Rollback ${initial_deploy_id} - Deploy Commit Not Expecting A Rollback ${initial_deploy_id} + TestAgent.Add Action For Next Deployment every fail_greenboot + Deploy Commit Expecting A Rollback ${FAILING_REF} + Deploy Commit Not Expecting A Rollback ${TARGET_REF} *** Keywords *** Setup [Documentation] Test suite setup Check Required Env Variables - IF "${TARGET_REF}"=="${EMPTY}" - Fatal Error TARGET_REF variable is required - END - IF "${FAILING_REF}"=="${EMPTY}" - Fatal Error FAILING_REF variable is required - END + Should Not Be Empty ${TARGET_REF} TARGET_REF variable is required + Should Not Be Empty ${FAILING_REF} FAILING_REF variable is required Login MicroShift Host Teardown @@ -55,62 +51,3 @@ System Should Not Feature MicroShift SSHLibrary.Directory Should Not Exist ${BACKUP_STORAGE} SSHLibrary.Directory Should Not Exist ${DATA_DIR} SSHLibrary.File Should Not Exist /usr/bin/microshift - -Deploy Commit Expecting A Rollback - [Documentation] Deploys given ref and configures test agent for failing greenboot. - ... It expects the system to roll back. - [Arguments] ${initial_deploy_id} - - ${deploy_id}= Rebase System ${FAILING_REF} - ${cfg}= Evaluate json.dumps({"${deploy_id}" : {"every": ["fail_greenboot"]}}) - Create Agent Config ${cfg} - Reboot MicroShift Host - - Log To Console "Failing ref deployed - waiting for system to roll back" - Wait Until Keyword Succeeds 20m 15s - ... Check If Current Deployment Is ${initial_deploy_id} - Log To Console "System rolled back - deploying target ref" - -Deploy Commit Not Expecting A Rollback - [Documentation] Deploys given ref and configures test agent for failing greenboot. - ... It expects the system to roll back. - [Arguments] ${initial_deploy_id} - - ${deploy_id}= Rebase System ${TARGET_REF} - Reboot MicroShift Host - - Log To Console "Target ref deployed - starting health checking" - Wait Until Keyword Succeeds 10m 15s - ... System Is Running Right Ref And Healthy ${deploy_id} ${initial_deploy_id} - -Make New SSH Connection - [Documentation] Closes all SSH connections and makes a new one. - # Staging deployments potentially introduces multiple reboots - # which could break existing SSH connection - - SSHLibrary.Close All Connections - Login MicroShift Host - -Check If Current Deployment Is - [Documentation] Checks if currently booted deployment is as expected - [Arguments] ${expected_deploy} - - Make New SSH Connection - - ${current_deploy}= libostree.Get Booted Deployment Id - Should Be Equal As Strings ${expected_deploy} ${current_deploy} - -System Is Running Right Ref And Healthy - [Documentation] Checks if system is running right reference and is healthy - [Arguments] ${expected_deploy} ${initial_deploy} - - Make New SSH Connection - - ${current_deploy}= libostree.Get Booted Deployment Id - IF "${current_deploy}" == "${initial_deploy}" - Fatal Error "System rolled back to initial deployment" - END - - Should Be Equal As Strings ${expected_deploy} ${current_deploy} - Wait Until Greenboot Health Check Exited - System Should Be Healthy diff --git a/test/suites-ostree/healthy-upgrade.robot b/test/suites-ostree/healthy-upgrade.robot index 4f410f175e..8e19ef3ba6 100644 --- a/test/suites-ostree/healthy-upgrade.robot +++ b/test/suites-ostree/healthy-upgrade.robot @@ -24,16 +24,10 @@ Upgrade ... and verifies if it was successful MicroShift 413 Should Not Have Upgrade Artifacts - Wait Until Greenboot Health Check Exited - System Should Be Healthy + Wait For Healthy System ${future_backup}= Get Future Backup Name For Current Boot - ${old_deploy_id}= Get Booted Deployment Id - - Rebase System And Verify ${TARGET_REF} - ${new_deploy_id}= Get Booted Deployment ID - Should Not Be Equal As Strings ${old_deploy_id} ${new_deploy_id} - + Deploy Commit Not Expecting A Rollback ${TARGET_REF} Backup Should Exist ${future_backup} @@ -41,9 +35,7 @@ Upgrade Setup [Documentation] Test suite setup Check Required Env Variables - IF "${TARGET_REF}"=="${EMPTY}" - Fatal Error TARGET_REF variable is required - END + Should Not Be Empty ${TARGET_REF} TARGET_REF variable is required Login MicroShift Host Teardown