fix(core): create all tables upfront when creating the database (#33843)

Create all the tables upfront when creating the database. This prevents
some issues in some scenarios where a missing table is reported.

(cherry picked from commit ed09ee1)

Author: leosvelperez

packages/nx/src/native/cache/cache.rs

@@ -14,6 +14,15 @@ use crate::native::cache::file_ops::_copy;
 use crate::native::db::connection::NxDbConnection;
 use crate::native::utils::Normalize;
 
+pub const SCHEMA: &str = "CREATE TABLE IF NOT EXISTS cache_outputs (
+    hash    TEXT PRIMARY KEY NOT NULL,
+    code   INTEGER NOT NULL,
+    size   INTEGER NOT NULL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    accessed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (hash) REFERENCES task_details (hash)
+);";
+
 #[napi(object)]
 #[derive(Default, Clone, Debug)]
 pub struct CachedResult {
@@ -29,7 +38,6 @@ pub struct NxCache {
     workspace_root: PathBuf,
     cache_path: PathBuf,
     db: External<NxDbConnection>,
-    link_task_details: bool,
     max_cache_size: i64,
 }
 
@@ -40,7 +48,6 @@ impl NxCache {
         workspace_root: String,
         cache_path: String,
         db_connection: External<NxDbConnection>,
-        link_task_details: Option<bool>,
         max_cache_size: Option<i64>,
     ) -> anyhow::Result<Self> {
         let cache_path = PathBuf::from(&cache_path);
@@ -50,44 +57,13 @@ impl NxCache {
 
         let max_cache_size = max_cache_size.unwrap_or(0);
 
-        let r = Self {
+        Ok(Self {
             db: db_connection,
             workspace_root: PathBuf::from(workspace_root),
             cache_directory: cache_path.to_normalized_string(),
             cache_path,
-            link_task_details: link_task_details.unwrap_or(true),
             max_cache_size,
-        };
-
-        r.setup()?;
-
-        Ok(r)
-    }
-
-    fn setup(&self) -> anyhow::Result<()> {
-        let query = if self.link_task_details {
-            "CREATE TABLE IF NOT EXISTS cache_outputs (
-                    hash    TEXT PRIMARY KEY NOT NULL,
-                    code   INTEGER NOT NULL,
-                    size   INTEGER NOT NULL,
-                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-                    accessed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-                    FOREIGN KEY (hash) REFERENCES task_details (hash)
-              );
-            "
-        } else {
-            "CREATE TABLE IF NOT EXISTS cache_outputs (
-                    hash    TEXT PRIMARY KEY NOT NULL,
-                    code   INTEGER NOT NULL,
-                    size   INTEGER NOT NULL,
-                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-                    accessed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-                );
-                "
-        };
-
-        self.db.execute(query, []).map_err(anyhow::Error::from)?;
-        Ok(())
+        })
     }
 
     #[napi]

packages/nx/src/native/db/initialize.rs

@@ -1,4 +1,8 @@
+use crate::native::cache::cache::SCHEMA as CACHE_SCHEMA;
 use crate::native::db::connection::NxDbConnection;
+use crate::native::tasks::details::SCHEMA as TASK_DETAILS_SCHEMA;
+use crate::native::tasks::running_tasks_service::SCHEMA as RUNNING_TASKS_SCHEMA;
+use crate::native::tasks::task_history::SCHEMA as TASK_HISTORY_SCHEMA;
 use rusqlite::vtab::array;
 use rusqlite::{Connection, OpenFlags};
 use std::fs::{File, remove_file};
@@ -57,6 +61,7 @@ pub(super) fn initialize_db(nx_version: String, db_path: &Path) -> anyhow::Resul
                 Err(s) if s.to_string().contains("metadata") => {
                     configure_database(&c)?;
                     create_metadata_table(&mut c, &nx_version)?;
+                    create_all_tables(&mut c)?;
                     c
                 }
                 reason => {
@@ -108,6 +113,24 @@ fn create_metadata_table(c: &mut NxDbConnection, nx_version: &str) -> anyhow::Re
     Ok(())
 }
 
+fn create_all_tables(c: &mut NxDbConnection) -> anyhow::Result<()> {
+    debug!("Creating all database tables");
+
+    c.transaction(|conn| {
+        // Order matters: tables with no FK dependencies first
+        conn.execute_batch(TASK_DETAILS_SCHEMA)?;
+        conn.execute_batch(RUNNING_TASKS_SCHEMA)?;
+
+        // Tables with FK dependencies
+        conn.execute_batch(TASK_HISTORY_SCHEMA)?;
+        conn.execute_batch(CACHE_SCHEMA)?;
+
+        Ok(())
+    })?;
+
+    Ok(())
+}
+
 fn open_database_connection(db_path: &Path) -> anyhow::Result<NxDbConnection> {
     let conn = Connection::open_with_flags(
         db_path,

packages/nx/src/native/index.d.ts

@@ -69,7 +69,7 @@ export declare class ImportResult {
 
 export declare class NxCache {
   cacheDirectory: string
-  constructor(workspaceRoot: string, cachePath: string, dbConnection: ExternalObject<NxDbConnection>, linkTaskDetails?: boolean | undefined | null, maxCacheSize?: number | undefined | null)
+  constructor(workspaceRoot: string, cachePath: string, dbConnection: ExternalObject<NxDbConnection>, maxCacheSize?: number | undefined | null)
   get(hash: string): CachedResult | null
   put(hash: string, terminalOutput: string, outputs: Array<string>, code: number): void
   applyRemoteCacheResults(hash: string, result: CachedResult, outputs?: Array<string> | undefined | null): void

packages/nx/src/native/tasks/details.rs

@@ -3,6 +3,13 @@ use napi::bindgen_prelude::*;
 use rusqlite::params;
 use tracing::trace;
 
+pub const SCHEMA: &str = "CREATE TABLE IF NOT EXISTS task_details (
+    hash    TEXT PRIMARY KEY NOT NULL,
+    project  TEXT NOT NULL,
+    target  TEXT NOT NULL,
+    configuration  TEXT
+);";
+
 #[napi(object)]
 #[derive(Default, Clone, Debug)]
 pub struct HashedTask {
@@ -21,25 +28,7 @@ pub struct TaskDetails {
 impl TaskDetails {
     #[napi(constructor)]
     pub fn new(db: External<NxDbConnection>) -> anyhow::Result<Self> {
-        let r = Self { db };
-
-        r.setup()?;
-
-        Ok(r)
-    }
-
-    fn setup(&self) -> anyhow::Result<()> {
-        self.db.execute(
-            "CREATE TABLE IF NOT EXISTS task_details (
-                hash    TEXT PRIMARY KEY NOT NULL,
-                project  TEXT NOT NULL,
-                target  TEXT NOT NULL,
-                configuration  TEXT
-            );",
-            params![],
-        )?;
-
-        Ok(())
+        Ok(Self { db })
     }
 
     #[napi]

packages/nx/src/native/tasks/running_tasks_service.rs

@@ -7,6 +7,13 @@ use std::ffi::OsString;
 use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
 use tracing::debug;
 
+pub const SCHEMA: &str = "CREATE TABLE IF NOT EXISTS running_tasks (
+    task_id TEXT PRIMARY KEY NOT NULL,
+    pid INTEGER NOT NULL,
+    command TEXT NOT NULL,
+    cwd TEXT NOT NULL
+);";
+
 #[napi]
 struct RunningTasksService {
     db: External<NxDbConnection>,
@@ -17,14 +24,10 @@ struct RunningTasksService {
 impl RunningTasksService {
     #[napi(constructor)]
     pub fn new(db: External<NxDbConnection>) -> anyhow::Result<Self> {
-        let s = Self {
+        Ok(Self {
             db,
             added_tasks: Default::default(),
-        };
-
-        s.setup()?;
-
-        Ok(s)
+        })
     }
 
     #[napi]
@@ -115,21 +118,6 @@ impl RunningTasksService {
         debug!("Removed {} from running tasks", task_id);
         Ok(())
     }
-
-    fn setup(&self) -> anyhow::Result<()> {
-        self.db.execute_batch(
-            "
-            CREATE TABLE IF NOT EXISTS running_tasks (
-                task_id TEXT PRIMARY KEY NOT NULL,
-                pid INTEGER NOT NULL,
-                command TEXT NOT NULL,
-                cwd TEXT NOT NULL
-            );
-            ",
-        )?;
-        debug!("Setup running tasks service");
-        Ok(())
-    }
 }
 
 impl Drop for RunningTasksService {

packages/nx/src/native/tasks/task_history.rs

@@ -6,6 +6,18 @@ use std::collections::HashMap;
 use std::rc::Rc;
 use tracing::trace;
 
+pub const SCHEMA: &str = "CREATE TABLE IF NOT EXISTS task_history (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    hash TEXT NOT NULL,
+    status TEXT NOT NULL,
+    code INTEGER NOT NULL,
+    start TIMESTAMP NOT NULL,
+    end TIMESTAMP NOT NULL,
+    FOREIGN KEY (hash) REFERENCES task_details (hash)
+);
+CREATE INDEX IF NOT EXISTS hash_idx ON task_history (hash);
+CREATE INDEX IF NOT EXISTS status_idx ON task_history (status);";
+
 #[napi(object)]
 pub struct TaskRun {
     pub hash: String,
@@ -24,33 +36,7 @@ pub struct NxTaskHistory {
 impl NxTaskHistory {
     #[napi(constructor)]
     pub fn new(db: External<NxDbConnection>) -> anyhow::Result<Self> {
-        let s = Self { db };
-
-        s.setup()?;
-
-        Ok(s)
-    }
-
-    fn setup(&self) -> anyhow::Result<()> {
-        self.db
-            .execute_batch(
-                "
-            BEGIN IMMEDIATE;
-            CREATE TABLE IF NOT EXISTS task_history (
-                id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-                hash TEXT NOT NULL,
-                status TEXT NOT NULL,
-                code INTEGER NOT NULL,
-                start TIMESTAMP NOT NULL,
-                end TIMESTAMP NOT NULL,
-                FOREIGN KEY (hash) REFERENCES task_details (hash)
-            );
-            CREATE INDEX IF NOT EXISTS hash_idx ON task_history (hash);
-            CREATE INDEX IF NOT EXISTS status_idx ON task_history (status);
-            COMMIT;
-            ",
-            )
-            .map_err(anyhow::Error::from)
+        Ok(Self { db })
     }
 
     #[napi]

packages/nx/src/tasks-runner/cache.ts

@@ -78,7 +78,6 @@ export class DbCache {
     workspaceRoot,
     cacheDir,
     getDbConnection(),
-    undefined,
     resolveMaxCacheSize(this.nxJson)
   );