From 46d68abca1c56e58071d4417e370a0a0c76acee8 Mon Sep 17 00:00:00 2001 From: "benjamin.747" Date: Thu, 15 May 2025 18:35:17 +0800 Subject: [PATCH] fix(jupiter): support sqlite migration --- ceres/src/api_service/import_api_service.rs | 15 ++-- ceres/src/api_service/mod.rs | 54 ++++++++----- ceres/src/api_service/mono_api_service.rs | 11 ++- ceres/src/model/git.rs | 28 +++++-- ceres/src/pack/import_repo.rs | 6 +- .../migration/src/m20250314_025943_init.rs | 79 +++++++++++-------- jupiter/src/storage/git_db_storage.rs | 2 +- jupiter/src/storage/init.rs | 25 +++--- mercury/src/internal/object/tree.rs | 4 +- 9 files changed, 135 insertions(+), 89 deletions(-) diff --git a/ceres/src/api_service/import_api_service.rs b/ceres/src/api_service/import_api_service.rs index 7c898ac24..b9f51ba96 100644 --- a/ceres/src/api_service/import_api_service.rs +++ b/ceres/src/api_service/import_api_service.rs @@ -12,7 +12,7 @@ use mercury::internal::object::commit::Commit; use mercury::internal::object::tree::Tree; use mercury::internal::object::tree::TreeItem; -use crate::api_service::ApiHandler; +use crate::api_service::{ApiHandler, GitObjectCache}; use crate::model::git::CreateFileInfo; use crate::protocol::repo::Repo; @@ -113,7 +113,7 @@ impl ApiHandler for ImportApiService { ) { let storage = self.context.services.git_db_storage.clone(); let trees = storage - .get_trees_by_hashes(self.repo.repo_id, hashes) + .get_trees_by_hashes(self.repo.repo_id, &hashes) .await .unwrap(); for tree in trees { @@ -148,20 +148,21 @@ impl ApiHandler for ImportApiService { async fn traverse_commit_history( &self, path: &Path, - start_commit: Commit, + start_commit: &Commit, target: &TreeItem, + cache: &mut GitObjectCache, ) -> Commit { let mut target_commit = start_commit.clone(); let mut visited = HashSet::new(); let mut p_stack = VecDeque::new(); visited.insert(start_commit.id); - p_stack.push_back(start_commit); + p_stack.push_back(start_commit.clone()); while let Some(commit) = p_stack.pop_front() { - let root_tree = self.get_tree_by_hash(&commit.tree_id.to_string()).await; + let root_tree = self.get_tree_from_cache(commit.tree_id, cache).await; let reachable = self - .reachable_in_tree(&root_tree, path, target) + .reachable_in_tree(&root_tree, path, target, cache) .await .unwrap(); if reachable { @@ -173,7 +174,7 @@ impl ApiHandler for ImportApiService { } } if target_commit.committer.timestamp > commit.committer.timestamp { - target_commit = commit; + target_commit = commit.clone(); } let parent_commits = self.get_commits_by_hashes(p_ids).await.unwrap(); p_stack.extend(parent_commits); diff --git a/ceres/src/api_service/mod.rs b/ceres/src/api_service/mod.rs index 29b1fd248..3ba3d606b 100644 --- a/ceres/src/api_service/mod.rs +++ b/ceres/src/api_service/mod.rs @@ -10,6 +10,7 @@ use common::errors::MegaError; use jupiter::{context::Context, utils::converter::generate_git_keep_with_timestamp}; use mercury::{ errors::GitError, + hash::SHA1, internal::object::{ commit::Commit, tree::{Tree, TreeItem, TreeItemMode}, @@ -24,6 +25,11 @@ use crate::model::git::{ pub mod import_api_service; pub mod mono_api_service; +#[derive(Debug, Default, Clone)] +pub struct GitObjectCache { + trees: HashMap, +} + #[async_trait] pub trait ApiHandler: Send + Sync { fn get_context(&self) -> Context; @@ -63,6 +69,15 @@ pub trait ApiHandler: Send + Sync { async fn get_tree_by_hash(&self, hash: &str) -> Tree; + async fn get_tree_from_cache(&self, oid: SHA1, cache: &mut GitObjectCache) -> Tree { + if let Some(tree) = cache.trees.get(&oid) { + return tree.clone(); + } + let tree = self.get_tree_by_hash(&oid.to_string()).await; + cache.trees.insert(oid, tree.clone()); + tree + } + async fn get_tree_relate_commit(&self, t_hash: &str) -> Commit; async fn add_trees_to_map( @@ -82,8 +97,9 @@ pub trait ApiHandler: Send + Sync { async fn traverse_commit_history( &self, path: &Path, - commit: Commit, + commit: &Commit, target: &TreeItem, + cache: &mut GitObjectCache, ) -> Commit; async fn get_blob_as_string(&self, file_path: PathBuf) -> Result, GitError> { @@ -133,6 +149,7 @@ pub trait ApiHandler: Send + Sync { } async fn get_tree_commit_info(&self, path: PathBuf) -> Result, GitError> { + let mut cache = GitObjectCache::default(); match self.search_tree_by_path(&path).await? { Some(tree) => { let mut item_to_commit = HashMap::new(); @@ -157,7 +174,6 @@ pub trait ApiHandler: Send + Sync { ) .await; - let mut items = Vec::new(); let commit_ids: HashSet = item_to_commit.values().cloned().collect(); let commits = self .get_commits_by_hashes(commit_ids.into_iter().collect()) @@ -166,34 +182,33 @@ pub trait ApiHandler: Send + Sync { let commit_map: HashMap = commits.into_iter().map(|x| (x.id.to_string(), x)).collect(); - let root_commit: Option = None; + let mut root_commit: Option = None; + let mut item_to_commit_map: HashMap> = HashMap::new(); for item in tree.tree_items { - let mut info: TreeCommitItem = item.clone().into(); if let Some(commit_id) = item_to_commit.get(&item.id.to_string()) { let commit = if let Some(commit) = commit_map.get(commit_id) { - commit + commit.to_owned() } else { - tracing::warn!("failed fecth commit: {}", commit_id); - let root_commit = if let Some(ref root_commit) = root_commit { - root_commit.clone() - } else { - self.get_root_commit().await - }; - &self - .traverse_commit_history(&path, root_commit, &item) + tracing::warn!("failed fecth from commit map: {}", commit_id); + if root_commit.is_none() { + root_commit = Some(self.get_root_commit().await); + } + let root_commit = root_commit.as_ref().unwrap().clone(); + self.traverse_commit_history(&path, &root_commit, &item, &mut cache) .await }; - info.oid = commit.id.to_string(); - info.message = commit.format_message(); - info.date = commit.committer.timestamp.to_string(); + item_to_commit_map.insert(item, Some(commit)); } - items.push(info); } + let mut items: Vec = item_to_commit_map + .into_iter() + .map(TreeCommitItem::from) + .collect(); // sort with type and date items.sort_by(|a, b| { a.content_type .cmp(&b.content_type) - .then(b.date.cmp(&a.date)) + .then(a.name.cmp(&b.name)) }); Ok(items) } @@ -403,6 +418,7 @@ pub trait ApiHandler: Send + Sync { root_tree: &Tree, path: &Path, target: &TreeItem, + cache: &mut GitObjectCache, ) -> Result { let relative_path = self.strip_relative(path).unwrap(); let mut search_tree = root_tree.clone(); @@ -416,7 +432,7 @@ pub trait ApiHandler: Send + Sync { .iter() .find(|x| x.name == target_name); if let Some(search_res) = search_res { - search_tree = self.get_tree_by_hash(&search_res.id.to_string()).await; + search_tree = self.get_tree_from_cache(search_res.id, cache).await; } else { return Ok(false); } diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index 45ec09929..46923c2c9 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -18,7 +18,7 @@ use mercury::internal::object::blob::Blob; use mercury::internal::object::commit::Commit; use mercury::internal::object::tree::{Tree, TreeItem, TreeItemMode}; -use crate::api_service::ApiHandler; +use crate::api_service::{ApiHandler, GitObjectCache}; use crate::model::git::CreateFileInfo; use crate::protocol::mr::MergeRequest; @@ -190,7 +190,13 @@ impl ApiHandler for MonoApiService { Ok(commits.into_iter().map(|x| x.into()).collect()) } - async fn traverse_commit_history(&self, _: &Path, _: Commit, _: &TreeItem) -> Commit { + async fn traverse_commit_history( + &self, + _: &Path, + _: &Commit, + _: &TreeItem, + _: &mut GitObjectCache, + ) -> Commit { unreachable!() } } @@ -324,6 +330,7 @@ impl MonoApiService { .await .expect("Failed to execute libra init"); // libra remote add origin http://localhost:8000/project + // TODO remove hard-code here Command::new("libra") .arg("remote") .arg("add") diff --git a/ceres/src/model/git.rs b/ceres/src/model/git.rs index c13e55242..af90facc9 100644 --- a/ceres/src/model/git.rs +++ b/ceres/src/model/git.rs @@ -1,7 +1,10 @@ use serde::{Deserialize, Serialize}; use utoipa::{IntoParams, ToSchema}; -use mercury::internal::object::tree::{TreeItem, TreeItemMode}; +use mercury::internal::object::{ + commit::Commit, + tree::{TreeItem, TreeItemMode}, +}; #[derive(PartialEq, Eq, Debug, Clone, Default, Serialize, Deserialize)] pub struct CreateFileInfo { @@ -69,18 +72,27 @@ pub struct TreeCommitItem { pub date: String, } -impl From for TreeCommitItem { - fn from(value: TreeItem) -> Self { +impl From<(TreeItem, Option)> for TreeCommitItem { + fn from((item, commit): (TreeItem, Option)) -> Self { TreeCommitItem { - name: value.name, - content_type: if value.mode == TreeItemMode::Tree { + name: item.name.clone(), + content_type: if item.mode == TreeItemMode::Tree { "directory".to_owned() } else { "file".to_owned() }, - oid: String::new(), - message: String::new(), - date: String::new(), + oid: commit + .as_ref() + .map(|x| x.id.to_string()) + .unwrap_or_default(), + message: commit + .as_ref() + .map(|x| x.format_message()) + .unwrap_or_default(), + date: commit + .as_ref() + .map(|x| x.committer.timestamp.to_string()) + .unwrap_or_default(), } } } diff --git a/ceres/src/pack/import_repo.rs b/ceres/src/pack/import_repo.rs index e8d02c765..8bbe58aeb 100644 --- a/ceres/src/pack/import_repo.rs +++ b/ceres/src/pack/import_repo.rs @@ -204,7 +204,7 @@ impl PackHandler for ImportRepo { } } - let want_tree_ids = want_commits.iter().map(|c| c.tree_id.to_string()).collect(); + let want_tree_ids = &want_commits.iter().map(|c| c.tree_id.to_string()).collect(); let want_trees: HashMap = storage .get_trees_by_hashes(self.repo.repo_id, want_tree_ids) .await @@ -222,7 +222,7 @@ impl PackHandler for ImportRepo { let have_trees = storage .get_trees_by_hashes( self.repo.repo_id, - have_commits.iter().map(|x| x.tree.clone()).collect(), + &have_commits.iter().map(|x| x.tree.clone()).collect(), ) .await .unwrap(); @@ -266,7 +266,7 @@ impl PackHandler for ImportRepo { .context .services .git_db_storage - .get_trees_by_hashes(self.repo.repo_id, hashes) + .get_trees_by_hashes(self.repo.repo_id, &hashes) .await .unwrap() .into_iter() diff --git a/jupiter/migration/src/m20250314_025943_init.rs b/jupiter/migration/src/m20250314_025943_init.rs index 0a979c239..dc967aecb 100644 --- a/jupiter/migration/src/m20250314_025943_init.rs +++ b/jupiter/migration/src/m20250314_025943_init.rs @@ -2,7 +2,7 @@ use extension::postgres::Type; use sea_orm_migration::{ prelude::*, schema::*, - sea_orm::{EnumIter, Iterable}, + sea_orm::{DatabaseBackend, EnumIter, Iterable}, }; use crate::pk_bigint; @@ -13,41 +13,50 @@ pub struct Migration; #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .create_type( - Type::create() - .as_enum(MergeStatusEnum) - .values(MergeStatus::iter()) - .to_owned(), - ) - .await?; - - manager - .create_type( - Type::create() - .as_enum(StorageTypeEnum) - .values(StorageType::iter()) - .to_owned(), - ) - .await?; - - manager - .create_type( - Type::create() - .as_enum(RefTypeEnum) - .values(RefType::iter()) - .to_owned(), - ) - .await?; + let backend = manager.get_database_backend(); + + match backend { + DatabaseBackend::Postgres => { + manager + .create_type( + Type::create() + .as_enum(StorageTypeEnum) + .values(StorageType::iter()) + .to_owned(), + ) + .await?; + + manager + .create_type( + Type::create() + .as_enum(RefTypeEnum) + .values(RefType::iter()) + .to_owned(), + ) + .await?; + + manager + .create_type( + Type::create() + .as_enum(ConvTypeEnum) + .values(ConvType::iter()) + .to_owned(), + ) + .await?; + manager + .create_type( + Type::create() + .as_enum(MergeStatusEnum) + .values(MergeStatus::iter()) + .to_owned(), + ) + .await?; + } - manager - .create_type( - Type::create() - .as_enum(ConvTypeEnum) - .values(ConvType::iter()) - .to_owned(), - ) - .await?; + DatabaseBackend::Sqlite | DatabaseBackend::MySql => { + // Do not create enum in sqlite + } + } manager .create_table( diff --git a/jupiter/src/storage/git_db_storage.rs b/jupiter/src/storage/git_db_storage.rs index cf2eae787..3f356c71b 100644 --- a/jupiter/src/storage/git_db_storage.rs +++ b/jupiter/src/storage/git_db_storage.rs @@ -296,7 +296,7 @@ impl GitDbStorage { pub async fn get_trees_by_hashes( &self, repo_id: i64, - hashes: Vec, + hashes: &Vec, ) -> Result, MegaError> { Ok(git_tree::Entity::find() .filter(git_tree::Column::RepoId.eq(repo_id)) diff --git a/jupiter/src/storage/init.rs b/jupiter/src/storage/init.rs index 952ac487d..bb7426be2 100644 --- a/jupiter/src/storage/init.rs +++ b/jupiter/src/storage/init.rs @@ -16,12 +16,9 @@ use crate::utils::id_generator; pub async fn database_connection(db_config: &DbConfig) -> DatabaseConnection { id_generator::set_up_options().unwrap(); - if db_config.db_type == "postgres" { + let conn = if db_config.db_type == "postgres" { match postgres_connection(db_config).await { - Ok(conn) => { - Migrator::up(&conn, None).await.unwrap(); - conn - } + Ok(conn) => conn, Err(e) => { log::error!("Failed to connect to postgres: {}", e); log::info!("Falling back to sqlite"); @@ -32,7 +29,10 @@ pub async fn database_connection(db_config: &DbConfig) -> DatabaseConnection { } } else { sqlite_connection(db_config).await.unwrap() - } + }; + Migrator::up(&conn, None).await.unwrap(); + + conn } async fn postgres_connection(db_config: &DbConfig) -> Result { @@ -56,16 +56,17 @@ async fn sqlite_connection(db_config: &DbConfig) -> Result Result<(), TransactionError> { conn.transaction::<_, _, DbErr>(|txn| { Box::pin(async move { diff --git a/mercury/src/internal/object/tree.rs b/mercury/src/internal/object/tree.rs index 0cd6ce38a..ec8bdcdca 100644 --- a/mercury/src/internal/object/tree.rs +++ b/mercury/src/internal/object/tree.rs @@ -28,7 +28,7 @@ use std::fmt::Display; /// that entry. The mode is a three-digit octal number that encodes both the permissions and the /// type of the object. The first digit specifies the object type, and the remaining two digits /// specify the file mode or permissions. -#[derive(PartialEq, Eq, Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(PartialEq, Eq, Debug, Clone, Copy, Serialize, Deserialize, Hash)] pub enum TreeItemMode { Blob, BlobExecutable, @@ -129,7 +129,7 @@ impl TreeItemMode { /// 100644 hello-world\0 /// 040000 data\0 /// ``` -#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)] +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Hash)] pub struct TreeItem { pub mode: TreeItemMode, pub id: SHA1,