Make the database portable and encryptable (#3848)

.gitattributes

@@ -0,0 +1,8 @@
+# Vendored third-party sources. Marking them keeps them out of language
+# statistics and collapses them in diffs, so a routine engine update does not
+# read as a 400,000 line change and does not push review tooling past its
+# size limits.
+vm/ByteCodeTranslator/src/cn1_sqlite3_amalgamation.h linguist-vendored linguist-generated -diff
+vm/ByteCodeTranslator/src/cn1_sqlite3.h             linguist-vendored linguist-generated -diff
+Ports/JavaScriptPort/src/main/webapp/js/sqlite3mc.js              linguist-vendored -diff
+Ports/JavaScriptPort/src/main/webapp/js/sqlite3-opfs-async-proxy.js linguist-vendored -diff

.github/codeql/codeql-config.yml

@@ -9,3 +9,10 @@ name: "Codename One CodeQL configuration"
 # String sinks across the rest of the codebase).
 paths-ignore:
   - 'scripts/cn1playground/**/bsh/cn1/gen/**'
+  # Vendored SQLite engine (SQLite3 Multiple Ciphers), 13MB of upstream C carried
+  # verbatim. Analysing it reports upstream's findings as ours, and its sheer size
+  # pushes CodeQL off incremental analysis so it re-reports pre-existing alerts
+  # across unrelated files. Provenance and update instructions are in
+  # vm/ByteCodeTranslator/src/cn1_sqlite3_README.md.
+  - 'vm/ByteCodeTranslator/src/cn1_sqlite3_amalgamation.h'
+  - 'vm/ByteCodeTranslator/src/cn1_sqlite3.h'

.github/workflows/ios-packaging.yml

@@ -227,7 +227,15 @@ jobs:
           ./scripts/run-ios-native-tests.sh \
             "${{ steps.build_ios_app.outputs.workspace }}" \
             "${{ steps.build_ios_app.outputs.scheme }}"
-        timeout-minutes: 20
+        # The reuse above is partial: run-ios-native-tests.sh injects the test target into the
+        # pbxproj before building, and Xcode rebuilds the app target after a project change even
+        # though the objects are there. So this step still pays a near-full native build --
+        # 6m55s of the 8m13s it takes on a fast runner, leaving little of a 20-minute budget for
+        # a slow one. Measured on the same commit, a slow runner spent 4m31s on simulator boot
+        # alone (a fast one: 1m15s) and 13 minutes on that build, and the cap fired while the
+        # tests had not started. The real fix is to inject before the UI-smoke step so this build
+        # is genuinely incremental; until then the budget has to cover the build it actually does.
+        timeout-minutes: 40
 
       - name: Upload packaging artifacts
         if: always()

.github/workflows/linux-build-run.yml

@@ -88,9 +88,14 @@ jobs:
         with: { distribution: 'temurin', java-version: '8' }
 
       - name: Build plugin + cn1-ads-mock (JDK 8)
+        # Maven Central intermittently serves 403/429 from the runner CDN edge and Maven treats it
+        # as a permanent resolution failure, which killed this job at dependency download before
+        # anything was compiled. See scripts/ci/retry.sh; these steps only build, so a blanket
+        # retry cannot turn a flaky test into a pass.
         run: |
           cd maven
-          JAVA_HOME="$JDK_8_HOME" mvn -B -pl codenameone-maven-plugin,cn1-ads-mock -am \
+          JAVA_HOME="$JDK_8_HOME" bash ../scripts/ci/retry.sh \
+            mvn -B -pl codenameone-maven-plugin,cn1-ads-mock -am \
             -DskipTests '-Dmaven.javadoc.skip=true' '-Plocal-dev-javase' install
 
       - name: Build hellocodenameone-common (JDK 17)
@@ -102,7 +107,8 @@ jobs:
           sleep 2
           mkdir -p "$HOME/.codenameone"
           [ -f "$HOME/.codenameone/guibuilder.jar" ] || : > "$HOME/.codenameone/guibuilder.jar"
-          mvn -B -f scripts/hellocodenameone/pom.xml -pl common -am install -DskipTests '-Dmaven.javadoc.skip=true'
+          bash scripts/ci/retry.sh \
+            mvn -B -f scripts/hellocodenameone/pom.xml -pl common -am install -DskipTests '-Dmaven.javadoc.skip=true'
 
       - name: Package suite classes
         run: |
@@ -181,9 +187,15 @@ jobs:
       # plugin -- so this step works on arm64 too. The suite classes the translator
       # also needs come from the prepare-suite artifact below.
       - name: Build core + Linux port (JDK 8)
+        # Retried for the same reason as the two steps above, which this one was left out of:
+        # Maven Central answers 429 from the runner CDN edge and Maven treats it as a permanent
+        # resolution failure, so the job dies at plugin download with nothing built. See
+        # scripts/ci/retry.sh; this step only builds, so a blanket retry cannot turn a flaky test
+        # into a pass.
         run: |
           cd maven
-          JAVA_HOME="$JDK_8_HOME" mvn -B -pl linux -am \
+          JAVA_HOME="$JDK_8_HOME" bash ../scripts/ci/retry.sh \
+            mvn -B -pl linux -am \
             -DskipTests '-Dmaven.javadoc.skip=true' '-Plocal-dev-javase' install
 
       - name: Download suite classes
@@ -265,7 +277,7 @@ jobs:
           echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope >/dev/null 2>&1 || true
           mkdir -p /tmp/cn1-cores
           rc=0
-          mvn -B clean package -pl JavaAPI -am -DskipTests
+          bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B clean package -pl JavaAPI -am -DskipTests
           mvn -B test -pl tests -am \
             '-Dtest=CleanTargetLinuxIntegrationTest#capturesHelloSuiteOverWebSocketLinux' \
             '-Dsurefire.failIfNoSpecifiedTests=false' || rc=$?

.github/workflows/parparvm-tests-windows.yml

@@ -203,8 +203,13 @@ jobs:
           # than with a function and splatting, because that shape is already proven
           # on these runners and this is not the place to find out about a quoting
           # difference.
+          # Five attempts out to five minutes rather than three out to ninety seconds: a run
+          # that took all three still ended on "status code: 429, reason phrase: Too Many
+          # Requests" two minutes after it started, which is a rate limit that had not lifted
+          # yet rather than a blip. This step only downloads and builds, so waiting longer costs
+          # time and cannot turn a failing test green.
           $ok = $false
-          foreach ($delay in 0, 30, 90) {
+          foreach ($delay in 0, 30, 90, 180, 300) {
             if ($delay -gt 0) {
               Write-Host "mvn failed; retrying in $delay s..."
               Start-Sleep -Seconds $delay
@@ -219,7 +224,7 @@ jobs:
           # a gate is for, and worse than the flake it was hiding.
           $resolutionFailure = 'status: (403|429|50[0-9])|Could not transfer artifact|Unresolveable build extension|Non-resolvable import POM|Could not resolve dependencies'
           $ok = $false
-          foreach ($delay in 0, 30, 90) {
+          foreach ($delay in 0, 30, 90, 180, 300) {
             if ($delay -gt 0) {
               Write-Host "mvn failed on a dependency-resolution error; retrying in $delay s..."
               Start-Sleep -Seconds $delay

.github/workflows/pr.yml

@@ -5,33 +5,55 @@ on:
   pull_request:
     branches:
       - master
-    paths-ignore:
-      - '.github/workflows/scripts-android.yml'
-      - '.github/workflows/scripts-ios.yml'
-      - 'scripts/**'
-      - '!scripts/setup-workspace.sh'
-      - 'docs/**'
-      - '**/*.md'
-      - '.github/workflows/developer-guide-docs.yml'
-      - '.github/workflows/website-docs.yml'
-      - '.github/workflows/initializr-cn1-version-pr.yml'
-      - 'CodenameOneDesigner/**'
-      - 'Ports/IOSPort/**'
+    # paths rather than paths-ignore, because this list both excludes and re-includes and only
+    # this form supports that: a "!" entry in paths-ignore is not a re-inclusion, so the entries
+    # for setup-workspace.sh and the database check scripts below never took effect there. The
+    # last matching pattern decides, so this reads as "everything, except these, except these".
+    paths:
+      - '**'
+      - '!.github/workflows/scripts-android.yml'
+      - '!.github/workflows/scripts-ios.yml'
+      - '!scripts/**'
+      - 'scripts/setup-workspace.sh'
+      # Run from this workflow and nowhere else, so ignoring the whole directory meant a change
+      # to one of them -- including one that breaks the check itself -- merged without it ever
+      # running.
+      - 'scripts/ci/db-cipher-interop.sh'
+      - 'scripts/ci/check-android-cipher-package-compiles.sh'
+      - 'scripts/ci/check-js-sqlite-wasm.sh'
+      - '!docs/**'
+      - '!**/*.md'
+      - '!.github/workflows/developer-guide-docs.yml'
+      - '!.github/workflows/website-docs.yml'
+      - '!.github/workflows/initializr-cn1-version-pr.yml'
+      - '!CodenameOneDesigner/**'
+      - '!Ports/IOSPort/**'
   push:
     branches:
       - master
-    paths-ignore:
-      - '.github/workflows/scripts-android.yml'
-      - '.github/workflows/scripts-ios.yml'
-      - 'scripts/**'
-      - '!scripts/setup-workspace.sh'
-      - 'docs/**'
-      - '**/*.md'
-      - '.github/workflows/developer-guide-docs.yml'
-      - '.github/workflows/website-docs.yml'
-      - '.github/workflows/initializr-cn1-version-pr.yml'
-      - 'CodenameOneDesigner/**'
-      - 'Ports/IOSPort/**'
+    # paths rather than paths-ignore, because this list both excludes and re-includes and only
+    # this form supports that: a "!" entry in paths-ignore is not a re-inclusion, so the entries
+    # for setup-workspace.sh and the database check scripts below never took effect there. The
+    # last matching pattern decides, so this reads as "everything, except these, except these".
+    paths:
+      - '**'
+      - '!.github/workflows/scripts-android.yml'
+      - '!.github/workflows/scripts-ios.yml'
+      - '!scripts/**'
+      - 'scripts/setup-workspace.sh'
+      # Run from this workflow and nowhere else, so ignoring the whole directory meant a change
+      # to one of them -- including one that breaks the check itself -- merged without it ever
+      # running.
+      - 'scripts/ci/db-cipher-interop.sh'
+      - 'scripts/ci/check-android-cipher-package-compiles.sh'
+      - 'scripts/ci/check-js-sqlite-wasm.sh'
+      - '!docs/**'
+      - '!**/*.md'
+      - '!.github/workflows/developer-guide-docs.yml'
+      - '!.github/workflows/website-docs.yml'
+      - '!.github/workflows/initializr-cn1-version-pr.yml'
+      - '!CodenameOneDesigner/**'
+      - '!Ports/IOSPort/**'
 
 permissions:
   contents: write
@@ -181,6 +203,15 @@ jobs:
             -Dcodename1.platform=javase \
             -Dcn1.binaries="${CN1_BINARIES}" \
             -pl javase -Plocal-dev-javase test
+      - name: Verify encrypted databases interoperate with reference SQLCipher
+        if: ${{ matrix.java-version == 8 }}
+        # The portability claim is that every port writes one on-disk format. A cipher
+        # misconfiguration produces files each platform reads happily and nothing else can
+        # touch, which no single-platform test would catch, so check ours against the stock
+        # sqlcipher client in both directions.
+        run: |
+          apt-get update -qq && apt-get install -y -qq sqlcipher
+          scripts/ci/db-cipher-interop.sh
       - name: Determine AI/ad cn1libs to build
         id: cn1libs
         if: ${{ matrix.java-version == 8 }}
@@ -344,6 +375,20 @@ jobs:
             -Dcn1.binaries="${CN1_BINARIES}" \
             -Pcompile-android \
             -pl android,ios,codenameone-maven-plugin -am verify
+      - name: Type-check the SQLCipher-backed database package
+        if: ${{ matrix.java-version == 8 }}
+        env:
+          CN1_BINARIES: ${{ github.workspace }}/maven/target/cn1-binaries
+        # That package is excluded from the port build on purpose, so nothing above compiles it
+        # and a member of com.codename1.impl.android it reaches can stop being public without
+        # anything noticing until an Android app build forty minutes later. Runs after the port
+        # is compiled, which is what it type-checks against.
+        run: scripts/ci/check-android-cipher-package-compiles.sh
+      - name: Compile the bundled SQLite WebAssembly module
+        if: ${{ matrix.java-version == 8 }}
+        # A module no engine accepts does not fail anything on its own: the JavaScript port just
+        # reports the database as unavailable and its tests skip.
+        run: scripts/ci/check-js-sqlite-wasm.sh
       - name: Run SpotBugs for ByteCodeTranslator
         if: ${{ matrix.java-version == 8 }}
         run: mvn -B -DskipTests=true -f vm/ByteCodeTranslator/pom.xml verify

.github/workflows/scripts-android.yml

@@ -12,6 +12,7 @@ name: Test Android build scripts
     paths:
       - '.github/workflows/scripts-android.yml'
       - 'scripts/setup-workspace.sh'
+      - 'scripts/ci/retry.sh'
       - 'scripts/build-android-port.sh'
       - 'scripts/build-android-app.sh'
       - 'scripts/run-android-instrumentation-tests.sh'
@@ -41,6 +42,7 @@ name: Test Android build scripts
     paths:
       - '.github/workflows/scripts-android.yml'
       - 'scripts/setup-workspace.sh'
+      - 'scripts/ci/retry.sh'
       - 'scripts/build-android-port.sh'
       - 'scripts/build-android-app.sh'
       - 'scripts/run-android-instrumentation-tests.sh'
@@ -180,9 +182,14 @@ jobs:
             ${{ runner.os }}-gradle-${{ matrix.id }}-
             ${{ runner.os }}-gradle-
       - name: Setup workspace
-        run: ./scripts/setup-workspace.sh -q -DskipTests
+        # Maven Central intermittently serves 403 from the runner CDN edge and Maven treats it as
+        # a permanent resolution failure, so this step died at plugin download with nothing built
+        # (central-publishing-maven-plugin and junit-bom, both 403 on the same attempt). See
+        # scripts/ci/retry.sh; this step only downloads and builds, so a blanket retry cannot turn
+        # a flaky test into a pass.
+        run: bash scripts/ci/retry.sh ./scripts/setup-workspace.sh -q -DskipTests
       - name: Build Android port
-        run: ./scripts/build-android-port.sh -q -DskipTests
+        run: bash scripts/ci/retry.sh ./scripts/build-android-port.sh -q -DskipTests
       - name: Build Hello Codename One Android app
         id: build-android-app
         run: |