Test Fixes to Run Xcode 15 in CI

.buildkite/cache-builder.yml

@@ -14,7 +14,7 @@ common_params:
   # Common environment values to use with the `env` key.
   - &common_env
     # Be sure to also update the `.xcode-version` file when updating the Xcode image/version here
-    IMAGE_ID: xcode-14.3.1
+    IMAGE_ID: xcode-15.1
 
 steps:
 

.buildkite/pipeline.yml

@@ -3,13 +3,10 @@ common_params:
   # Common plugin settings to use with the `plugins` key.
   - &common_plugins
     - automattic/a8c-ci-toolkit#2.18.1
-    - automattic/git-s3-cache#1.1.4:
-        bucket: "a8c-repo-mirrors"
-        repo: "automattic/wordpress-ios/"
   # Common environment values to use with the `env` key.
   - &common_env
     # Be sure to also update the `.xcode-version` file when updating the Xcode image/version here
-    IMAGE_ID: xcode-14.3.1
+    IMAGE_ID: xcode-15.1
 
 # This is the default pipeline – it will build and test the app
 steps:
@@ -80,7 +77,7 @@ steps:
   - group: "🔬 UI Tests"
     steps:
       - label: "🔬 :jetpack: UI Tests (iPhone)"
-        command: .buildkite/commands/run-ui-tests.sh 'iPhone SE (3rd generation)'
+        command: .buildkite/commands/run-ui-tests.sh 'iPhone 15'
         depends_on: "build_jetpack"
         env: *common_env
         plugins: *common_plugins
@@ -92,7 +89,7 @@ steps:
               context: "UI Tests (iPhone)"
 
       - label: "🔬 :jetpack: UI Tests (iPad)"
-        command: .buildkite/commands/run-ui-tests.sh 'iPad Air (5th generation)'
+        command: .buildkite/commands/run-ui-tests.sh 'iPad Pro (12.9-inch) (6th generation)'
         depends_on: "build_jetpack"
         env: *common_env
         plugins: *common_plugins

.buildkite/release-builds.yml

@@ -11,7 +11,7 @@ common_params:
   # Common environment values to use with the `env` key.
   - &common_env
     # Be sure to also update the `.xcode-version` file when updating the Xcode image/version here
-    IMAGE_ID: xcode-14.3.1
+    IMAGE_ID: xcode-15.1
 
 steps:
 

.xcode-version

@@ -1 +1 @@
-14.3.1
+15.1

API-Mocks/WordPressMocks/src/main/assets/mocks/mappings/wpcom/stats/stats_visits-year.json

@@ -20,7 +20,7 @@
   "response": {
     "status": 200,
     "jsonBody": {
-      "date": "2019-07-16",
+      "date": "{{now format='yyyy-MM-dd'}}",      
       "unit": "year",
       "fields": [
         "period",
@@ -33,7 +33,7 @@
       ],
       "data": [
         [
-          "2005-01-01",
+          "{{now offset='-7 years' format='yyyy-MM-dd'}}",
           0,
           0,
           0,
@@ -42,7 +42,7 @@
           0
         ],
         [
-          "2006-01-01",
+          "{{now offset='-6 years' format='yyyy-MM-dd'}}",
           0,
           0,
           0,
@@ -51,7 +51,7 @@
           0
         ],
         [
-          "2007-01-01",
+          "{{now offset='-5 years' format='yyyy-MM-dd'}}",
           0,
           0,
           0,
@@ -60,7 +60,7 @@
           0
         ],
         [
-          "2008-01-01",
+          "{{now offset='-4 years' format='yyyy-MM-dd'}}",
           0,
           0,
           0,
@@ -69,70 +69,7 @@
           0
         ],
         [
-          "2009-01-01",
-          0,
-          0,
-          0,
-          0,
-          0,
-          0
-        ],
-        [
-          "2010-01-01",
-          0,
-          0,
-          0,
-          0,
-          0,
-          0
-        ],
-        [
-          "2011-01-01",
-          0,
-          0,
-          0,
-          0,
-          0,
-          0
-        ],
-        [
-          "2012-01-01",
-          0,
-          0,
-          0,
-          0,
-          0,
-          0
-        ],
-        [
-          "2013-01-01",
-          0,
-          0,
-          0,
-          0,
-          0,
-          0
-        ],
-        [
-          "2014-01-01",
-          0,
-          0,
-          0,
-          0,
-          0,
-          0
-        ],
-        [
-          "2015-01-01",
-          0,
-          0,
-          0,
-          0,
-          0,
-          0
-        ],
-        [
-          "2016-01-01",
+          "{{now offset='-3 years' format='yyyy-MM-dd'}}",
           48,
           12,
           0,
@@ -141,7 +78,7 @@
           13
         ],
         [
-          "2017-01-01",
+          "{{now offset='-2 years' format='yyyy-MM-dd'}}",
           788,
           465,
           0,
@@ -150,7 +87,7 @@
           3
         ],
         [
-          "2018-01-01",
+          "{{now offset='-1 years' format='yyyy-MM-dd'}}",
           1215,
           632,
           0,
@@ -159,7 +96,7 @@
           3
         ],
         [
-          "2019-01-01",
+          "{{now format='yyyy-MM-dd'}}",
           9148,
           4216,
           1351,

Gemfile

@@ -6,7 +6,8 @@ gem 'cocoapods', '~> 1.14'
 gem 'commonmarker'
 gem 'danger-dangermattic', git: 'https://github.com/Automattic/dangermattic'
 gem 'dotenv'
-gem 'fastlane', '~> 2.216'
+# 2.217.0 includes a fix for Xcode 15 test results parsing in CI
+gem 'fastlane', '~> 2.217'
 gem 'fastlane-plugin-appcenter', '~> 2.1'
 gem 'fastlane-plugin-sentry'
 # This comment avoids typing to switch to a development version for testing.

Gemfile.lock

@@ -405,7 +405,7 @@ DEPENDENCIES
   commonmarker
   danger-dangermattic!
   dotenv
-  fastlane (~> 2.216)
+  fastlane (~> 2.217)
   fastlane-plugin-appcenter (~> 2.1)
   fastlane-plugin-sentry
   fastlane-plugin-wpmreleasetoolkit (~> 9.1)

WordPress/UITests/JetpackUITests.xctestplan

@@ -21,7 +21,6 @@
   },
   "testTargets" : [
     {
-      "parallelizable" : true,
       "skippedTests" : [
         "EditorAztecTests",
         "LoginTests\/testEmailMagicLinkLogin()",

WordPress/UITests/Tests/LoginTests.swift

@@ -24,7 +24,6 @@ class LoginTests: XCTestCase {
                 siteUrl: WPUITestCredentials.testWPcomPaidSite
             )
             .continueWithSelectedSite()
-            .dismissNotificationAlertIfNeeded()
         try TabNavComponent()
             .goToMeScreen()
             .logoutToPrologue()

WordPress/UITests/Tests/StatsTests.swift

@@ -22,42 +22,43 @@ class StatsTests: XCTestCase {
         takeScreenshotOfFailedTest()
     }
 
-    let insightsStats: [String] = [
-        "Your views in the last 7-days are -9 (-82%) lower than the previous 7-days. ",
-        "Thursday",
-        "34% of views",
-        "Best Hour",
-        "4 AM",
-        "25% of views"
-    ]
-
-    let yearsStats: [String] = [
-        "9,148",
-        "+7,933 (653%)",
-        "United States, 60",
-        "Canada, 44",
-        "Germany, 15",
-        "France, 14",
-        "United Kingdom, 12",
-        "India, 121"
-    ]
-
-    let yearsChartBars: [String] = [
-        "Views,  2019: 9148",
-        "Visitors,  2019: 4216",
-        "Views,  2018: 1215",
-        "Visitors,  2018: 632",
-        "Views,  2017: 788",
-        "Visitors,  2017: 465"
-    ]
-
     func testInsightsStatsLoadProperly() throws {
+        let insightsStats: [String] = [
+            "Your views in the last 7-days are -9 (-82%) lower than the previous 7-days. ",
+            "Thursday",
+            "34% of views",
+            "Best Hour",
+            "4 AM",
+            "25% of views"
+        ]
+
         try StatsScreen()
             .switchTo(mode: "insights")
             .assertStatsAreLoaded(insightsStats)
     }
 
     func testYearsStatsLoadProperly() throws {
+        let yearsStats: [String] = [
+            "9,148",
+            "+7,933 (653%)",
+            "United States, 60",
+            "Canada, 44",
+            "Germany, 15",
+            "France, 14",
+            "United Kingdom, 12",
+            "India, 121"
+        ]
+
+        let currentYear = Calendar.current.component(.year, from: Date())
+        let yearsChartBars: [String] = [
+            "Views,  \(currentYear): 9148",
+            "Visitors,  \(currentYear): 4216",
+            "Views,  \(currentYear - 1): 1215",
+            "Visitors,  \(currentYear - 1): 632",
+            "Views,  \(currentYear - 2): 788",
+            "Visitors,  \(currentYear - 2): 465"
+        ]
+
         try StatsScreen()
             .switchTo(mode: "years")
             .assertStatsAreLoaded(yearsStats)

WordPress/UITestsFoundation/Globals.swift

@@ -62,23 +62,6 @@ public func waitAndTap( _ element: XCUIElement, maxRetries: Int = 20) {
     }
 }
 
-public func tapUntilCondition(element: XCUIElement, condition: Bool, description: String, maxRetries: Int = 10) {
-    var retries = 0
-    while retries < maxRetries {
-        if !condition {
-            element.tap()
-            break
-        }
-
-        usleep(500000) // a 0.5 second delay before retrying
-        retries += 1
-    }
-
-    if retries == maxRetries {
-        XCTFail("Condition \(description) still not met after \(maxRetries) tries.")
-    }
-}
-
 public func waitForElementToDisappear( _ element: XCUIElement, maxRetries: Int = 10) {
     var retries = 0
     while retries < maxRetries {

WordPress/UITestsFoundation/Screens/Editor/EditorPostSettings.swift

@@ -152,7 +152,7 @@ public class EditorPostSettings: ScreenObject {
         // To ensure that the day tap happens on the correct month
         let nextMonth = monthLabel.value as! String
         if nextMonth != currentMonth {
-            tapUntilCondition(element: firstCalendarDayButton, condition: firstCalendarDayButton.isSelected, description: "First Day button selected")
+            firstCalendarDayButton.tapUntil(.selected, failureMessage: "First Day button not selected!")
         }
 
         doneButton.tap()

WordPress/UITestsFoundation/Screens/Login/LoginUsernamePasswordScreen.swift

@@ -40,13 +40,11 @@ public class LoginUsernamePasswordScreen: ScreenObject {
 
     public func proceedWithSelfHostedSiteAddedFromSitesList(username: String, password: String) throws -> MySitesScreen {
         fill(username: username, password: password)
-
         return try MySitesScreen()
     }
 
     public func proceedWithSelfHosted(username: String, password: String) throws -> MySiteScreen {
         fill(username: username, password: password)
-
         return try MySiteScreen()
     }
 
@@ -66,6 +64,10 @@ public class LoginUsernamePasswordScreen: ScreenObject {
             passwordTextField.typeText(password)
         }
         nextButton.tap()
+
+        if #available(iOS 17.2, *) {
+            app.dismissSavePasswordPrompt()
+        }
     }
 
     private func dismissQuickStartPromptIfNeeded() throws {

WordPress/UITestsFoundation/Screens/Login/Unified/PasswordScreen.swift

@@ -29,7 +29,6 @@ public class PasswordScreen: ScreenObject {
     @discardableResult
     public func proceedWithValidPassword() throws -> LoginEpilogueScreen {
         try tryProceed(password: "pw")
-
         return try LoginEpilogueScreen()
     }
 
@@ -59,6 +58,15 @@ public class PasswordScreen: ScreenObject {
 
         passwordTextField.typeText(password)
         continueButton.tap()
+
+        // iOS 16.4 introduced a prompt to save passwords in the keychain.
+        // Prior to iOS 17.2, we used a test observer (see TestObserver.swift) to disable storing passwords before the tests started.
+        // Xcode 15.1 and iOS 17.2 have what at the time of writing looks like a bug in the Settings app which breaks that approach.
+        // As soon as the passwords screen is pushed in the Settings navigation stack, it's immediately popped back.
+        // For the time being, let's manually dismiss the prompt on demand.
+        if #available(iOS 17.2, *) {
+            app.dismissSavePasswordPrompt()
+        }
     }
 
     @discardableResult