diff --git a/common/src/config.rs b/common/src/config.rs index 10ad4f449..319245864 100644 --- a/common/src/config.rs +++ b/common/src/config.rs @@ -48,6 +48,8 @@ impl Default for LogConfig { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct DbConfig { + pub db_type: String, + pub db_path: String, pub db_url: String, pub max_connection: u32, pub min_connection: u32, @@ -57,6 +59,8 @@ pub struct DbConfig { impl Default for DbConfig { fn default() -> Self { Self { + db_type: String::new(), + db_path: String::new(), db_url: String::new(), max_connection: 32, min_connection: 16, diff --git a/docs/development.md b/docs/development.md index 6f7e6c837..237660881 100644 --- a/docs/development.md +++ b/docs/development.md @@ -23,7 +23,7 @@ $ cargo build ``` -3. Install PostgreSQL and init database. +3. Install PostgreSQL and init database. (You can skip this step if using SQLite in `config.toml`) 1. Install PostgreSQL 16 with `brew` command. @@ -90,8 +90,15 @@ [database] + # "sqlite" | "postgres" + # "sqlite" will use `db_path` and ignore `db_url` + db_type = "sqlite" + + # used for sqlite + db_path = "/tmp/.mega/mega.db" + # database connection url - db_url = "postgres://postgres:postgres@localhost:5432/mega" + db_url = "postgres://mega:mega@localhost:5432/mega" # db max connection, setting it to twice the number of CPU cores would be appropriate. max_connection = 32 @@ -188,7 +195,7 @@ $ cargo build ``` -3. Install PostgreSQL and initialize database. +3. Install PostgreSQL and initialize database. (You can skip this step if using SQLite in `config.toml`) 1.Install PostgreSQL. @@ -261,6 +268,13 @@ [database] + # "sqlite" | "postgres" + # "sqlite" will use `db_path` and ignore `db_url` + db_type = "sqlite" + + # used for sqlite + db_path = "/tmp/.mega/mega.db" + # database connection url db_url = "postgres://mega:mega@localhost:5432/mega" @@ -356,6 +370,8 @@ If you are using GitHub codespaces, you can follow the steps below to set up the Mega project. When you create a new Codespace, the Mega project will be cloned automatically. You can then follow the steps below to set up the project. +You can skip this step (PostgreSQL setup) if using SQLite in `config.toml`. + When the codespace is ready, the PostgreSQL will be installed and started automatically. You can then follow the steps below to set up the database with below steps. ```bash @@ -390,6 +406,13 @@ Config `confg.toml` file for the Mega project. [database] + # "sqlite" | "postgres" + # "sqlite" will use `db_path` and ignore `db_url` + db_type = "sqlite" + + # used for sqlite + db_path = "/tmp/.mega/mega.db" + # database connection url db_url = "postgres://mega:mega@localhost:5432/mega" @@ -448,6 +471,14 @@ Config `confg.toml` file for the Mega project. # Size of each file chunk when splitting is enabled, in bytes. Ignored if splitting is disabled. split_size = 20971520 # Default size is 20MB (20971520 bytes) ``` +## Database maintenance +Currently, the tables of database are created by `.sql` file. + +If you want to add a new table or modify the existing table, you need to update the `.sql` files which are located in the `sql` directory. + +### Attention +- Each database corresponds to one `.sql` file, you must modify all of them if you want to update the tables in order to keep the consistency of the database. +- DO NOT use `Array` Type in PostgreSQL but use `JSON` instead, for compatibility with SQLite & MySQL. (`JSON` <==> `serde_json::Value`) ## Comment Guideline diff --git a/jupiter/Cargo.toml b/jupiter/Cargo.toml index 476e6f916..e8cab5109 100644 --- a/jupiter/Cargo.toml +++ b/jupiter/Cargo.toml @@ -18,6 +18,7 @@ mercury = { workspace = true } sea-orm = { workspace = true, features = [ "sqlx-postgres", "sqlx-mysql", + "sqlx-sqlite", "runtime-tokio-rustls", "macros", ] } diff --git a/jupiter/callisto/Cargo.toml b/jupiter/callisto/Cargo.toml index a4f9d17cb..d1c75148e 100644 --- a/jupiter/callisto/Cargo.toml +++ b/jupiter/callisto/Cargo.toml @@ -12,5 +12,6 @@ path = "src/lib.rs" common = { path = "../../common" } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } chrono = { workspace = true } -sea-orm = { workspace = true, features = ["sqlx-postgres", "sqlx-mysql"] } +sea-orm = { workspace = true, features = ["sqlx-postgres", "sqlx-mysql", "sqlx-sqlite"] } diff --git a/jupiter/callisto/src/git_commit.rs b/jupiter/callisto/src/git_commit.rs index bf7741fef..89b4aa5f2 100644 --- a/jupiter/callisto/src/git_commit.rs +++ b/jupiter/callisto/src/git_commit.rs @@ -1,6 +1,7 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 use sea_orm::entity::prelude::*; +use serde_json::Value; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "git_commit")] @@ -10,7 +11,7 @@ pub struct Model { pub repo_id: i64, pub commit_id: String, pub tree: String, - pub parents_id: Vec, + pub parents_id: Value, #[sea_orm(column_type = "Text", nullable)] pub author: Option, #[sea_orm(column_type = "Text", nullable)] diff --git a/jupiter/callisto/src/mega_commit.rs b/jupiter/callisto/src/mega_commit.rs index 4006c566e..9c54ef085 100644 --- a/jupiter/callisto/src/mega_commit.rs +++ b/jupiter/callisto/src/mega_commit.rs @@ -1,6 +1,7 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 use sea_orm::entity::prelude::*; +use serde_json::Value; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "mega_commit")] @@ -10,7 +11,7 @@ pub struct Model { #[sea_orm(unique)] pub commit_id: String, pub tree: String, - pub parents_id: Vec, + pub parents_id: Value, #[sea_orm(column_type = "Text", nullable)] pub author: Option, #[sea_orm(column_type = "Text", nullable)] diff --git a/jupiter/src/storage/init.rs b/jupiter/src/storage/init.rs index fb9116de2..e5d905578 100644 --- a/jupiter/src/storage/init.rs +++ b/jupiter/src/storage/init.rs @@ -1,6 +1,6 @@ -use std::time::Duration; +use std::{path::Path, time::Duration}; -use sea_orm::{ConnectOptions, Database, DatabaseConnection}; +use sea_orm::{ConnectOptions, ConnectionTrait, Database, DatabaseConnection, DbErr, Statement, TransactionError, TransactionTrait}; use tracing::log; use common::config::DbConfig; @@ -9,7 +9,21 @@ use crate::utils::id_generator; pub async fn database_connection(db_config: &DbConfig) -> DatabaseConnection { id_generator::set_up_options().unwrap(); - let mut opt = ConnectOptions::new(db_config.db_url.to_owned()); + + let is_sqlite = db_config.db_type == "sqlite"; + let db_path = &db_config.db_path; + let db_url = if is_sqlite { + if !Path::new(db_path).exists() { + log::info!("Creating new sqlite database: {}", db_path); + std::fs::File::create(db_path).expect("Failed to create sqlite database"); + } + &format!("sqlite://{}", db_path) + } else { + &db_config.db_url + }; + log::info!("Connecting to database: {}", db_url); + + let mut opt = ConnectOptions::new(db_url.to_owned()); opt.max_connections(db_config.max_connection) .min_connections(db_config.min_connection) .acquire_timeout(Duration::from_secs(30)) @@ -18,7 +32,34 @@ pub async fn database_connection(db_config: &DbConfig) -> DatabaseConnection { .max_lifetime(Duration::from_secs(8)) .sqlx_logging(db_config.sqlx_logging) .sqlx_logging_level(log::LevelFilter::Debug); - Database::connect(opt) + let conn = Database::connect(opt) .await - .expect("Database connection failed") + .expect("Database connection failed"); + + // setup sqlite database (execute .sql) + if is_sqlite && is_file_empty(db_path) { + log::info!("Setting up sqlite database"); + setup_sql(&conn).await.expect("Failed to setup sqlite database"); + } + conn } + +/// create table from .sql file +async fn setup_sql(conn: &DatabaseConnection) -> Result<(), TransactionError> { + conn.transaction::<_, _, DbErr>(|txn| { + Box::pin(async move { + let backend = txn.get_database_backend(); + + // `include_str!` will expand the file while compiling, so `.sql` is not needed after that + const SETUP_SQL: &str = include_str!("../../../sql/sqlite/sqlite_20240711_init.sql"); + txn.execute(Statement::from_string(backend, SETUP_SQL)).await?; + Ok(()) + }) + }) + .await +} + +fn is_file_empty(path: &str) -> bool { + let metadata = std::fs::metadata(path).unwrap(); + metadata.len() == 0 +} \ No newline at end of file diff --git a/mega/config.toml b/mega/config.toml index 007acc16a..e30d25f86 100644 --- a/mega/config.toml +++ b/mega/config.toml @@ -12,6 +12,13 @@ print_std = true [database] +# "sqlite" | "postgres" +# "sqlite" will use `db_path` and ignore `db_url` +db_type = "sqlite" + +# used for sqlite +db_path = "/tmp/.mega/mega.db" + # database connection url db_url = "postgres://mega:mega@localhost:5432/mega" diff --git a/mercury/src/internal/model/commit.rs b/mercury/src/internal/model/commit.rs index a5cf278bc..76c268a89 100644 --- a/mercury/src/internal/model/commit.rs +++ b/mercury/src/internal/model/commit.rs @@ -15,8 +15,10 @@ impl From for Commit { tree_id: SHA1::from_str(&value.tree).unwrap(), parent_commit_ids: value .parents_id - .into_iter() - .map(|id| SHA1::from_str(&id).unwrap()) + .as_array() + .unwrap() + .iter() + .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap()) .collect(), author: Signature::from_data(value.author.unwrap().into()).unwrap(), committer: Signature::from_data(value.committer.unwrap().into()).unwrap(), @@ -32,8 +34,10 @@ impl From for Commit { tree_id: SHA1::from_str(&value.tree).unwrap(), parent_commit_ids: value .parents_id - .into_iter() - .map(|id| SHA1::from_str(&id).unwrap()) + .as_array() + .unwrap() + .iter() + .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap()) .collect(), author: Signature::from_data(value.author.unwrap().into()).unwrap(), committer: Signature::from_data(value.committer.unwrap().into()).unwrap(), diff --git a/sql/postgres/pg_20240205__init.sql b/sql/postgres/pg_20240205__init.sql index 084227256..0d73841cf 100644 --- a/sql/postgres/pg_20240205__init.sql +++ b/sql/postgres/pg_20240205__init.sql @@ -2,7 +2,7 @@ CREATE TABLE IF NOT EXISTS "mega_commit" ( "id" BIGINT PRIMARY KEY, "commit_id" VARCHAR(40) NOT NULL, "tree" VARCHAR(40) NOT NULL, - "parents_id" TEXT [] NOT NULL, + "parents_id" JSON NOT NULL, -- for compatibility with sqlite, DO NOT use Array Type "author" TEXT, "committer" TEXT, "content" TEXT, @@ -117,7 +117,7 @@ CREATE TABLE IF NOT EXISTS "git_commit" ( "repo_id" BIGINT NOT NULL, "commit_id" VARCHAR(40) NOT NULL, "tree" VARCHAR(40) NOT NULL, - "parents_id" TEXT [] NOT NULL, + "parents_id" JSON NOT NULL, "author" TEXT, "committer" TEXT, "content" TEXT, diff --git a/sql/sqlite/sqlite_20240711_init.sql b/sql/sqlite/sqlite_20240711_init.sql new file mode 100644 index 000000000..106e88992 --- /dev/null +++ b/sql/sqlite/sqlite_20240711_init.sql @@ -0,0 +1,246 @@ +CREATE TABLE IF NOT EXISTS "mega_commit" ( + "id" INTEGER PRIMARY KEY, + "commit_id" TEXT NOT NULL, + "tree" TEXT NOT NULL, + "parents_id" TEXT NOT NULL, -- Use JSON to store array + "author" TEXT, + "committer" TEXT, + "content" TEXT, + "created_at" TEXT NOT NULL, + CONSTRAINT uniq_mc_git_id UNIQUE (commit_id) +); +CREATE INDEX "idx_mc_git_id" ON "mega_commit" ("commit_id"); +CREATE TABLE IF NOT EXISTS "mega_tree" ( + "id" INTEGER PRIMARY KEY, + "tree_id" TEXT NOT NULL, + "sub_trees" BLOB NOT NULL, + "size" INTEGER NOT NULL, + "commit_id" TEXT NOT NULL, + "created_at" TEXT NOT NULL +); +CREATE INDEX "idx_mt_git_id" ON "mega_tree" ("tree_id"); +CREATE TABLE IF NOT EXISTS "mega_blob" ( + "id" INTEGER PRIMARY KEY, + "blob_id" TEXT NOT NULL, + "commit_id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "size" INTEGER NOT NULL, + "created_at" TEXT NOT NULL +); +CREATE INDEX "idx_mb_git_id" ON "mega_blob" ("blob_id"); +CREATE TABLE IF NOT EXISTS "mega_tag" ( + "id" INTEGER PRIMARY KEY, + "tag_id" TEXT NOT NULL, + "object_id" TEXT NOT NULL, + "object_type" TEXT NOT NULL, + "tag_name" TEXT NOT NULL, + "tagger" TEXT NOT NULL, + "message" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + CONSTRAINT uniq_mtag_tag_id UNIQUE (tag_id) +); +CREATE TABLE IF NOT EXISTS "mega_mr" ( + "id" INTEGER PRIMARY KEY, + "mr_link" TEXT NOT NULL, + "merge_date" TEXT, + "status" TEXT NOT NULL, + "path" TEXT NOT NULL, + "from_hash" TEXT NOT NULL, + "to_hash" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + "updated_at" TEXT NOT NULL +); +CREATE INDEX "idx_mr_path" ON "mega_mr" ("path"); + +CREATE TABLE IF NOT EXISTS "mega_mr_conv" ( + "id" INTEGER PRIMARY KEY, + "mr_id" INTEGER NOT NULL, + "user_id" INTEGER NOT NULL, + "conv_type" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + "updated_at" TEXT NOT NULL +); +CREATE INDEX "idx_conversation" ON "mega_mr_conv" ("mr_id"); + + +CREATE TABLE IF NOT EXISTS "mega_mr_comment" ( + "id" INTEGER PRIMARY KEY, + "conv_id" INTEGER NOT NULL, + "comment" TEXT, + "edited" INTEGER NOT NULL +); +CREATE INDEX "idx_comment_id" ON "mega_mr_comment" ("conv_id"); + +CREATE TABLE IF NOT EXISTS "mega_issue" ( + "id" INTEGER PRIMARY KEY, + "number" INTEGER NOT NULL, + "title" TEXT NOT NULL, + "sender_name" TEXT NOT NULL, + "sender_id" INTEGER NOT NULL, + "state" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + "updated_at" TEXT NOT NULL, + "closed_at" TEXT DEFAULT NULL +); +CREATE TABLE IF NOT EXISTS "mega_refs" ( + "id" INTEGER PRIMARY KEY, + "path" TEXT NOT NULL, + "ref_commit_hash" TEXT NOT NULL, + "ref_tree_hash" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + "updated_at" TEXT NOT NULL, + CONSTRAINT uniq_mref_path UNIQUE (path) +); +CREATE TABLE IF NOT EXISTS "import_refs" ( + "id" INTEGER PRIMARY KEY, + "repo_id" INTEGER NOT NULL, + "ref_name" TEXT NOT NULL, + "ref_git_id" TEXT NOT NULL, + "ref_type" TEXT NOT NULL, + "default_branch" INTEGER NOT NULL, + "created_at" TEXT NOT NULL, + "updated_at" TEXT NOT NULL, + CONSTRAINT uniq_ref_path_name UNIQUE (repo_id, ref_name) +); +CREATE INDEX "idx_refs_repo_id" ON "import_refs" ("repo_id"); +CREATE TABLE IF NOT EXISTS "git_repo" ( + "id" INTEGER PRIMARY KEY, + "repo_path" TEXT NOT NULL, + "repo_name" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + "updated_at" TEXT NOT NULL, + CONSTRAINT uniq_ir_path UNIQUE (repo_path) +); +CREATE INDEX "idx_ir_repo_path" ON "git_repo" ("repo_path"); +CREATE TABLE IF NOT EXISTS "git_commit" ( + "id" INTEGER PRIMARY KEY, + "repo_id" INTEGER NOT NULL, + "commit_id" TEXT NOT NULL, + "tree" TEXT NOT NULL, + "parents_id" TEXT NOT NULL, -- Use JSON to store array + "author" TEXT, + "committer" TEXT, + "content" TEXT, + "created_at" TEXT NOT NULL, + CONSTRAINT uniq_c_git_repo_id UNIQUE (repo_id, commit_id) +); +CREATE INDEX "idx_ic_git_id" ON "git_commit" ("commit_id"); +CREATE INDEX "idx_ic_repo_id" ON "git_commit" ("repo_id"); +CREATE TABLE IF NOT EXISTS "git_tree" ( + "id" INTEGER PRIMARY KEY, + "repo_id" INTEGER NOT NULL, + "tree_id" TEXT NOT NULL, + "sub_trees" BLOB NOT NULL, + "size" INTEGER NOT NULL, + "commit_id" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + CONSTRAINT uniq_t_git_repo UNIQUE (repo_id, tree_id) +); +CREATE INDEX "idx_t_git_id" ON "git_tree" ("tree_id"); +CREATE INDEX "idx_t_repo_id" ON "git_tree" ("repo_id"); +CREATE TABLE IF NOT EXISTS "git_blob" ( + "id" INTEGER PRIMARY KEY, + "repo_id" INTEGER NOT NULL, + "blob_id" TEXT NOT NULL, + "name" TEXT, + "size" INTEGER NOT NULL, + "commit_id" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + CONSTRAINT uniq_b_git_repo UNIQUE (repo_id, blob_id) +); +CREATE INDEX "idx_b_git_id" ON "git_blob" ("blob_id"); +CREATE TABLE IF NOT EXISTS "git_tag" ( + "id" INTEGER PRIMARY KEY, + "repo_id" INTEGER NOT NULL, + "tag_id" TEXT NOT NULL, + "object_id" TEXT NOT NULL, + "object_type" TEXT NOT NULL, + "tag_name" TEXT NOT NULL, + "tagger" TEXT NOT NULL, + "message" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + CONSTRAINT uniq_gtag_tag_id UNIQUE (tag_id) +); +CREATE TABLE IF NOT EXISTS "raw_blob" ( + "id" INTEGER PRIMARY KEY, + "sha1" TEXT NOT NULL, + "content" TEXT, + "file_type" TEXT, + "storage_type" TEXT NOT NULL, + "data" BLOB, + "local_path" TEXT, + "remote_url" TEXT, + "created_at" TEXT NOT NULL, + CONSTRAINT uniq_rb_sha1 UNIQUE (sha1) +); +CREATE INDEX "idx_rb_sha1" ON "raw_blob" ("sha1"); +CREATE TABLE IF NOT EXISTS "git_pr" ( + "id" INTEGER PRIMARY KEY, + "number" INTEGER NOT NULL, + "title" TEXT NOT NULL, + "state" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + "updated_at" TEXT NOT NULL, + "closed_at" TEXT DEFAULT NULL, + "merged_at" TEXT DEFAULT NULL, + "merge_commit_sha" TEXT DEFAULT NULL, + "repo_id" INTEGER NOT NULL, + "sender_name" TEXT NOT NULL, + "sender_id" INTEGER NOT NULL, + "user_name" TEXT NOT NULL, + "user_id" INTEGER NOT NULL, + "commits_url" TEXT NOT NULL, + "patch_url" TEXT NOT NULL, + "head_label" TEXT NOT NULL, + "head_ref" TEXT NOT NULL, + "base_label" TEXT NOT NULL, + "base_ref" TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS "git_issue" ( + "id" INTEGER PRIMARY KEY, + "number" INTEGER NOT NULL, + "title" TEXT NOT NULL, + "sender_name" TEXT NOT NULL, + "sender_id" INTEGER NOT NULL, + "state" TEXT NOT NULL, + "created_at" TEXT NOT NULL, + "updated_at" TEXT NOT NULL, + "closed_at" TEXT DEFAULT NULL, + "repo_id" INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS "lfs_locks" ( + "id" TEXT PRIMARY KEY, + "data" TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS "lfs_objects" ( + "oid" TEXT PRIMARY KEY, + "size" INTEGER NOT NULL, + "exist" INTEGER NOT NULL, + "splited" INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS "lfs_split_relations" ( + "ori_oid" TEXT NOT NULL, + "sub_oid" TEXT NOT NULL, + "offset" INTEGER NOT NULL, + "size" INTEGER NOT NULL, + PRIMARY KEY ("ori_oid", "sub_oid", "offset") +); + + +CREATE TABLE IF NOT EXISTS "ztm_node" ( + "peer_id" TEXT PRIMARY KEY, + "hub" TEXT, + "agent_name" TEXT, + "service_name" TEXT, + "type" TEXT, + "online" INTEGER NOT NULL, + "last_online_time" INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS "ztm_repo_info" ( + "identifier" TEXT PRIMARY KEY, + "name" TEXT, + "origin" TEXT, + "update_time" INTEGER NOT NULL, + "commit" TEXT +); \ No newline at end of file