USHIFT-1372: Backing up the data is top priority

test/bin/download_images.sh

@@ -55,5 +55,6 @@ done
 
 echo "Updating references"
 cd "${IMAGEDIR}"
+ostree refs --force --create "rhel-9.2-microshift-failing" "rhel-9.2-microshift-source"
 ostree summary --update --repo=repo
 ostree summary --view --repo=repo

test/resources/TestAgent.py

@@ -0,0 +1,57 @@
+import json
+import libostree
+from robot.libraries.BuiltIn import BuiltIn
+
+_log = BuiltIn().log
+
+CONFIG_PATH = "/var/lib/microshift-test-agent.json"
+
+# Example config
+# {
+#     "deploy-id": {
+#         "every": [ "prevent_backup" ],
+#         "1": [ "fail_greenboot" ],
+#         "2": [ "..." ],
+#         "3": [ "..." ]
+#     }
+# }
+
+
+class TestAgent:
+    def __init__(self):
+        self.cfg = dict()
+
+    def add_action(self, deployment: str, boot: str, action: str) -> None:
+        if deployment not in self.cfg:
+            self.cfg[deployment] = {boot: [action]}
+
+        elif boot not in self.cfg[deployment]:
+            self.cfg[deployment][boot] = [action]
+
+        elif action not in self.cfg[deployment][boot]:
+            self.cfg[deployment][boot].append(action)
+
+        _log(f"TestAgent Config: {self.cfg}")
+
+    def add_action_for_next_deployment(self, boot: str, action: str) -> None:
+        self.add_action("next", boot, action)
+
+    def substitute_staged(self) -> None:
+        if "next" not in self.cfg:
+            _log("'next' deployment not found")
+            return
+        id = libostree.get_staged_deployment_id()
+        self.cfg[id] = self.cfg["next"]
+        del self.cfg["next"]
+
+    def write(self) -> None:
+        self.substitute_staged()
+        j = json.dumps(self.cfg)
+        BuiltIn().should_not_be_empty(j)
+        libostree.remote_sudo(
+            f"echo '{j}' | sudo tee /var/lib/microshift-test-agent.json"
+        )
+
+    def remove(self) -> None:
+        self.cfg = {}
+        libostree.remote_sudo("rm /var/lib/microshift-test-agent.json")

test/resources/ostree-data.resource

@@ -0,0 +1,50 @@
+*** Settings ***
+Documentation       Keywords for OSTree-based systems
+
+Resource            systemd.resource
+Resource            microshift-process.resource
+Library             libostree.py
+
+
+*** Variables ***
+${DATA_DIR}             "/var/lib/microshift"
+${BACKUP_STORAGE}       "/var/lib/microshift-backups"
+
+
+*** Keywords ***
+Get Future Backup Name For Current Boot
+    [Documentation]    Provides a name for future backup:
+    ...    "4.13" for MicroShift 4.13, otherwise deployment ID + boot ID
+
+    # When system is upgrading from 4.13, there is no metadata regarding previous boot,
+    # therefore, when naming backup for existing data, "4.13" will be used as deployment ID,
+    # i.e. it should reside in `/var/lib/microshift-backups/4.13`
+    ${version}=    MicroShift Version
+    IF    ${version.minor} == 13    RETURN    4.13
+
+    ${deploy_id}=    Get Booted Deployment ID
+    ${boot_id}=    Get Current Boot Id
+    ${backup_name}=    Catenate    SEPARATOR=_    ${deploy_id}    ${boot_id}
+    RETURN    ${backup_name}
+
+Backup Should Exist
+    [Documentation]    Checks if backup identified by deployment ID
+    ...    and (optionally) boot ID exists
+    [Arguments]    ${deployment_id}    ${boot_id}=${EMPTY}
+    ${exists}=    Does Backup Exist    ${deployment_id}    ${boot_id}
+    Should Be True    ${exists}
+
+Remove Existing Backup For Current Deployment
+    [Documentation]    Remove any existing backup for currently running deployment
+    ${deploy_id}=    Get Booted Deployment Id
+    Remove Backups For Deployment    ${deploy_id}
+
+MicroShift 413 Should Not Have Upgrade Artifacts
+    [Documentation]    Verifies that host running MicroShift 4.13
+    ...    does not have upgrade related artifacts.
+
+    ${version}=    MicroShift Version
+    IF    ${version.minor} == 13
+        Path Should Not Exist    ${BACKUP_STORAGE}
+        Path Should Not Exist    /var/lib/microshift/version
+    END

test/resources/ostree-health.resource

@@ -0,0 +1,37 @@
+*** Settings ***
+Documentation       Keywords for OSTree-based systems
+
+Resource            systemd.resource
+Resource            microshift-process.resource
+Resource            ostree-data.resource
+
+
+*** Keywords ***
+Wait Until Greenboot Health Check Exited
+    [Documentation]    Wait until greenboot healthchecks are done
+
+    Wait Until Keyword Succeeds    10m    15s
+    ...    Greenboot Health Check Exited
+
+Greenboot Health Check Exited
+    [Documentation]    Checks if greenboot-healthcheck finished running successfully (exited)
+
+    ${value}=    Get Systemd Setting    greenboot-healthcheck.service    SubState
+    Should Be Equal As Strings    ${value}    exited
+
+Wait For Healthy System
+    [Documentation]    Waits for greenboot healthchecks and checks health.json
+    Wait Until Greenboot Health Check Exited
+    System Should Be Healthy
+
+System Should Be Healthy
+    [Documentation]    Verifies if system is healthy by looking at health.json.
+    ...    If this runs after greenboot-healthcheck finishes, it verifies
+    ...    health of current boot.
+
+    # Health of system with MicroShift 4.13 is implicitly checked by "Greenboot Health Check Exited"
+    ${version}=    MicroShift Version
+    IF    ${version.minor} == 13    RETURN
+
+    ${health}=    Get Persisted System Health
+    Should Be Equal As Strings    ${health}    healthy

test/resources/ostree.resource

@@ -3,86 +3,69 @@ Documentation       Keywords for OSTree-based systems
 
 Resource            systemd.resource
 Resource            microshift-process.resource
+Resource            ostree-data.resource
+Resource            ostree-health.resource
 Library             libostree.py
+Library             ../resources/TestAgent.py
 
 
-*** Variables ***
-${DATA_DIR}             "/var/lib/microshift"
-${BACKUP_STORAGE}       "/var/lib/microshift-backups"
+*** Keywords ***
+Make New SSH Connection
+    [Documentation]    Closes all SSH connections and makes a new one.
+    # Staging deployments potentially introduces multiple reboots
+    # which could break existing SSH connection
 
+    SSHLibrary.Close All Connections
+    Login MicroShift Host
 
-*** Keywords ***
-Wait Until Greenboot Health Check Exited
-    [Documentation]    Wait until greenboot healthchecks are done
+Current Deployment Should Be
+    [Documentation]    Checks if currently booted deployment is as expected
+    [Arguments]    ${expected_deploy}
 
-    Wait Until Keyword Succeeds    10m    15s
-    ...    Greenboot Health Check Exited
-
-Greenboot Health Check Exited
-    [Documentation]    Checks if greenboot-healthcheck finished running successfully (exited)
-
-    ${value}=    Get Systemd Setting    greenboot-healthcheck.service    SubState
-    Should Be Equal As Strings    ${value}    exited
-
-Get Future Backup Name For Current Boot
-    [Documentation]    Provides a name for future backup:
-    ...    "4.13" for MicroShift 4.13, otherwise deployment ID + boot ID
-
-    # When system is upgrading from 4.13, there is no metadata regarding previous boot,
-    # therefore, when naming backup for existing data, "4.13" will be used as deployment ID,
-    # i.e. it should reside in `/var/lib/microshift-backups/4.13`
-    ${version}=    MicroShift Version
-    IF    ${version.minor} == 13    RETURN    4.13
-
-    ${deploy_id}=    Get Booted Deployment ID
-    ${boot_id}=    Get Current Boot Id
-    ${backup_name}=    Catenate    SEPARATOR=_    ${deploy_id}    ${boot_id}
-    RETURN    ${backup_name}
-
-Backup Should Exist
-    [Documentation]    Checks if backup identified by deployment ID
-    ...    and (optionally) boot ID exists
-    [Arguments]    ${deployment_id}    ${boot_id}=${EMPTY}
-    ${exists}=    Does Backup Exist    ${deployment_id}    ${boot_id}
-    Should Be True    ${exists}
-
-Remove Existing Backup For Current Deployment
-    [Documentation]    Remove any existing backup for currently running deployment
-    ${deploy_id}=    Get Booted Deployment Id
-    Remove Backups For Deployment    ${deploy_id}
-
-Rebase System And Verify
-    [Documentation]    Deploys given ostree ref on the host, reboots it,
-    ...    and checks if the host is healthy.
+    Make New SSH Connection
+
+    ${current_deploy}=    libostree.Get Booted Deployment Id
+    Should Be Equal As Strings    ${expected_deploy}    ${current_deploy}
+
+Deploy Commit Expecting A Rollback
+    [Documentation]    Deploys given ref and configures test agent for failing greenboot.
+    ...    It expects the system to roll back.
     [Arguments]    ${ref}
 
-    Rebase System    ${ref}
+    ${initial_deploy_id}=    Get Booted Deployment Id
+    ${deploy_id}=    Rebase System    ${ref}
+    TestAgent.Write
     Reboot MicroShift Host
 
-    Wait Until Greenboot Health Check Exited
-    System Should Be Healthy
+    Log To Console    "Failing ref deployed - waiting for system to roll back"
+    Wait Until Keyword Succeeds    20m    15s
+    ...    Current Deployment Should Be    ${initial_deploy_id}
+    Log To Console    "System rolled back - deploying target ref"
 
-    ${ref_after_reboot}=    Get Current Ref
-    Should Be Equal As Strings    ${ref_after_reboot}    ${ref}
+Deploy Commit Not Expecting A Rollback
+    [Documentation]    Deploys given ref and configures test agent for failing greenboot.
+    ...    It expects the system to roll back.
+    [Arguments]    ${target_ref}
 
-System Should Be Healthy
-    [Documentation]    Verifies if system is healthy by looking at health.json.
-    ...    If this runs after greenboot-healthcheck finishes, it verifies
-    ...    health of current boot.
+    ${initial_deploy_id}=    Get Booted Deployment Id
+    ${deploy_id}=    Rebase System    ${target_ref}
+    Reboot MicroShift Host
 
-    # Health of system with MicroShift 4.13 is implicitly checked by "Greenboot Health Check Exited"
-    ${version}=    MicroShift Version
-    IF    ${version.minor} == 13    RETURN
+    Log To Console    "Target ref deployed - starting health checking"
+    Wait Until Keyword Succeeds    10m    15s
+    ...    System Is Running Right Ref And Healthy    ${deploy_id}    ${initial_deploy_id}
 
-    ${health}=    Get Persisted System Health
-    Should Be Equal As Strings    ${health}    healthy
+System Is Running Right Ref And Healthy
+    [Documentation]    Checks if system is running right reference and is healthy
+    [Arguments]    ${expected_deploy}    ${initial_deploy}
 
-MicroShift 413 Should Not Have Upgrade Artifacts
-    [Documentation]    Verifies that host running MicroShift 4.13
-    ...    does not have upgrade related artifacts.
+    Make New SSH Connection
 
-    ${version}=    MicroShift Version
-    IF    ${version.minor} == 13
-        Path Should Not Exist    ${BACKUP_STORAGE}
-        Path Should Not Exist    /var/lib/microshift/version
+    ${current_deploy}=    libostree.Get Booted Deployment Id
+    IF    "${current_deploy}" == "${initial_deploy}"
+        Fatal Error    "System rolled back to initial deployment"
     END
+
+    Should Be Equal As Strings    ${expected_deploy}    ${current_deploy}
+    Greenboot Health Check Exited
+    System Should Be Healthy

test/suites-ostree/backup-restore.robot

@@ -76,7 +76,7 @@ Make Masquerading Backup
 Mark System As Unhealthy
     [Documentation]    Marks systems as unhealthy by executing microshift's red script
     ${stdout}    ${stderr}    ${rc}=    Execute Command
-    ...    /etc/greenboot/red.d/40_microshift_set_unhealthy.sh
+    ...    FORCE=1 /etc/greenboot/red.d/40_microshift_set_unhealthy.sh
     ...    sudo=True    return_stderr=True    return_rc=True
     Should Be Equal As Integers    0    ${rc}
 

test/suites-ostree/failed-upgrade.robot

@@ -0,0 +1,44 @@
+*** Settings ***
+Documentation       Tests related to upgrading MicroShift
+
+Resource            ../resources/common.resource
+Resource            ../resources/ostree.resource
+Library             Collections
+
+Suite Setup         Setup
+Suite Teardown      Teardown
+
+Test Tags           ostree
+
+
+*** Variables ***
+${USHIFT_HOST}      ${EMPTY}
+${USHIFT_USER}      ${EMPTY}
+
+${FAILING_REF}      ${EMPTY}
+
+
+*** Test Cases ***
+Staged Deployment Consistently Fails To Back Up The Data
+    [Documentation]    Verifies that instructions to back up the data
+    ...    ("healthy" system) is not lost if staged deployment fails
+    ...    to back up and it is performed after rolling back.
+
+    Wait For Healthy System
+    ${backup}=    Get Future Backup Name For Current Boot
+
+    TestAgent.Add Action For Next Deployment    every    prevent_backup
+    Deploy Commit Expecting A Rollback    ${FAILING_REF}
+    Backup Should Exist    ${backup}
+
+
+*** Keywords ***
+Setup
+    [Documentation]    Test suite setup
+    Check Required Env Variables
+    Should Not Be Empty    ${FAILING_REF}    FAILING_REF variable is required
+    Login MicroShift Host
+
+Teardown
+    [Documentation]    Test suite teardown
+    Logout MicroShift Host