Skip to content

fix(jupiter): support sqlite migration #1030

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
genedna merged 1 commit into from
May 15, 2025
Merged

fix(jupiter): support sqlite migration #1030

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions ceres/src/api_service/import_api_service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use mercury::internal::object::commit::Commit;
use mercury::internal::object::tree::Tree;
use mercury::internal::object::tree::TreeItem;

use crate::api_service::ApiHandler;
use crate::api_service::{ApiHandler, GitObjectCache};
use crate::model::git::CreateFileInfo;
use crate::protocol::repo::Repo;

Expand Down Expand Up @@ -113,7 +113,7 @@ impl ApiHandler for ImportApiService {
) {
let storage = self.context.services.git_db_storage.clone();
let trees = storage
.get_trees_by_hashes(self.repo.repo_id, hashes)
.get_trees_by_hashes(self.repo.repo_id, &hashes)
.await
.unwrap();
for tree in trees {
Expand Down Expand Up @@ -148,20 +148,21 @@ impl ApiHandler for ImportApiService {
async fn traverse_commit_history(
&self,
path: &Path,
start_commit: Commit,
start_commit: &Commit,
target: &TreeItem,
cache: &mut GitObjectCache,
) -> Commit {
let mut target_commit = start_commit.clone();
let mut visited = HashSet::new();
let mut p_stack = VecDeque::new();

visited.insert(start_commit.id);
p_stack.push_back(start_commit);
p_stack.push_back(start_commit.clone());

while let Some(commit) = p_stack.pop_front() {
let root_tree = self.get_tree_by_hash(&commit.tree_id.to_string()).await;
let root_tree = self.get_tree_from_cache(commit.tree_id, cache).await;
let reachable = self
.reachable_in_tree(&root_tree, path, target)
.reachable_in_tree(&root_tree, path, target, cache)
.await
.unwrap();
if reachable {
Expand All @@ -173,7 +174,7 @@ impl ApiHandler for ImportApiService {
}
}
if target_commit.committer.timestamp > commit.committer.timestamp {
target_commit = commit;
target_commit = commit.clone();
}
let parent_commits = self.get_commits_by_hashes(p_ids).await.unwrap();
p_stack.extend(parent_commits);
Expand Down
54 changes: 35 additions & 19 deletions ceres/src/api_service/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use common::errors::MegaError;
use jupiter::{context::Context, utils::converter::generate_git_keep_with_timestamp};
use mercury::{
errors::GitError,
hash::SHA1,
internal::object::{
commit::Commit,
tree::{Tree, TreeItem, TreeItemMode},
Expand All @@ -24,6 +25,11 @@ use crate::model::git::{
pub mod import_api_service;
pub mod mono_api_service;

#[derive(Debug, Default, Clone)]
pub struct GitObjectCache {
trees: HashMap<SHA1, Tree>,
}

#[async_trait]
pub trait ApiHandler: Send + Sync {
fn get_context(&self) -> Context;
Expand Down Expand Up @@ -63,6 +69,15 @@ pub trait ApiHandler: Send + Sync {

async fn get_tree_by_hash(&self, hash: &str) -> Tree;

async fn get_tree_from_cache(&self, oid: SHA1, cache: &mut GitObjectCache) -> Tree {
if let Some(tree) = cache.trees.get(&oid) {
return tree.clone();
}
let tree = self.get_tree_by_hash(&oid.to_string()).await;
cache.trees.insert(oid, tree.clone());
tree
}

async fn get_tree_relate_commit(&self, t_hash: &str) -> Commit;

async fn add_trees_to_map(
Expand All @@ -82,8 +97,9 @@ pub trait ApiHandler: Send + Sync {
async fn traverse_commit_history(
&self,
path: &Path,
commit: Commit,
commit: &Commit,
target: &TreeItem,
cache: &mut GitObjectCache,
) -> Commit;

async fn get_blob_as_string(&self, file_path: PathBuf) -> Result<Option<String>, GitError> {
Expand Down Expand Up @@ -133,6 +149,7 @@ pub trait ApiHandler: Send + Sync {
}

async fn get_tree_commit_info(&self, path: PathBuf) -> Result<Vec<TreeCommitItem>, GitError> {
let mut cache = GitObjectCache::default();
match self.search_tree_by_path(&path).await? {
Some(tree) => {
let mut item_to_commit = HashMap::new();
Expand All @@ -157,7 +174,6 @@ pub trait ApiHandler: Send + Sync {
)
.await;

let mut items = Vec::new();
let commit_ids: HashSet<String> = item_to_commit.values().cloned().collect();
let commits = self
.get_commits_by_hashes(commit_ids.into_iter().collect())
Expand All @@ -166,34 +182,33 @@ pub trait ApiHandler: Send + Sync {
let commit_map: HashMap<String, Commit> =
commits.into_iter().map(|x| (x.id.to_string(), x)).collect();

let root_commit: Option<Commit> = None;
let mut root_commit: Option<Commit> = None;
let mut item_to_commit_map: HashMap<TreeItem, Option<Commit>> = HashMap::new();
for item in tree.tree_items {
let mut info: TreeCommitItem = item.clone().into();
if let Some(commit_id) = item_to_commit.get(&item.id.to_string()) {
let commit = if let Some(commit) = commit_map.get(commit_id) {
commit
commit.to_owned()
} else {
tracing::warn!("failed fecth commit: {}", commit_id);
let root_commit = if let Some(ref root_commit) = root_commit {
root_commit.clone()
} else {
self.get_root_commit().await
};
&self
.traverse_commit_history(&path, root_commit, &item)
tracing::warn!("failed fecth from commit map: {}", commit_id);
if root_commit.is_none() {
root_commit = Some(self.get_root_commit().await);
}
let root_commit = root_commit.as_ref().unwrap().clone();
self.traverse_commit_history(&path, &root_commit, &item, &mut cache)
.await
};
info.oid = commit.id.to_string();
info.message = commit.format_message();
info.date = commit.committer.timestamp.to_string();
item_to_commit_map.insert(item, Some(commit));
}
items.push(info);
}
let mut items: Vec<TreeCommitItem> = item_to_commit_map
.into_iter()
.map(TreeCommitItem::from)
.collect();
// sort with type and date
items.sort_by(|a, b| {
a.content_type
.cmp(&b.content_type)
.then(b.date.cmp(&a.date))
.then(a.name.cmp(&b.name))
});
Ok(items)
}
Expand Down Expand Up @@ -403,6 +418,7 @@ pub trait ApiHandler: Send + Sync {
root_tree: &Tree,
path: &Path,
target: &TreeItem,
cache: &mut GitObjectCache,
) -> Result<bool, GitError> {
let relative_path = self.strip_relative(path).unwrap();
let mut search_tree = root_tree.clone();
Expand All @@ -416,7 +432,7 @@ pub trait ApiHandler: Send + Sync {
.iter()
.find(|x| x.name == target_name);
if let Some(search_res) = search_res {
search_tree = self.get_tree_by_hash(&search_res.id.to_string()).await;
search_tree = self.get_tree_from_cache(search_res.id, cache).await;
} else {
return Ok(false);
}
Expand Down
11 changes: 9 additions & 2 deletions ceres/src/api_service/mono_api_service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use mercury::internal::object::blob::Blob;
use mercury::internal::object::commit::Commit;
use mercury::internal::object::tree::{Tree, TreeItem, TreeItemMode};

use crate::api_service::ApiHandler;
use crate::api_service::{ApiHandler, GitObjectCache};
use crate::model::git::CreateFileInfo;
use crate::protocol::mr::MergeRequest;

Expand Down Expand Up @@ -190,7 +190,13 @@ impl ApiHandler for MonoApiService {
Ok(commits.into_iter().map(|x| x.into()).collect())
}

async fn traverse_commit_history(&self, _: &Path, _: Commit, _: &TreeItem) -> Commit {
async fn traverse_commit_history(
&self,
_: &Path,
_: &Commit,
_: &TreeItem,
_: &mut GitObjectCache,
) -> Commit {
unreachable!()
}
}
Expand Down Expand Up @@ -324,6 +330,7 @@ impl MonoApiService {
.await
.expect("Failed to execute libra init");
// libra remote add origin http://localhost:8000/project
// TODO remove hard-code here
Command::new("libra")
.arg("remote")
.arg("add")
Expand Down
28 changes: 20 additions & 8 deletions ceres/src/model/git.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
use serde::{Deserialize, Serialize};
use utoipa::{IntoParams, ToSchema};

use mercury::internal::object::tree::{TreeItem, TreeItemMode};
use mercury::internal::object::{
commit::Commit,
tree::{TreeItem, TreeItemMode},
};

#[derive(PartialEq, Eq, Debug, Clone, Default, Serialize, Deserialize)]
pub struct CreateFileInfo {
Expand Down Expand Up @@ -69,18 +72,27 @@ pub struct TreeCommitItem {
pub date: String,
}

impl From<TreeItem> for TreeCommitItem {
fn from(value: TreeItem) -> Self {
impl From<(TreeItem, Option<Commit>)> for TreeCommitItem {
fn from((item, commit): (TreeItem, Option<Commit>)) -> Self {
TreeCommitItem {
name: value.name,
content_type: if value.mode == TreeItemMode::Tree {
name: item.name.clone(),
content_type: if item.mode == TreeItemMode::Tree {
"directory".to_owned()
} else {
"file".to_owned()
},
oid: String::new(),
message: String::new(),
date: String::new(),
oid: commit
.as_ref()
.map(|x| x.id.to_string())
.unwrap_or_default(),
message: commit
.as_ref()
.map(|x| x.format_message())
.unwrap_or_default(),
date: commit
.as_ref()
.map(|x| x.committer.timestamp.to_string())
.unwrap_or_default(),
}
}
}
Expand Down
6 changes: 3 additions & 3 deletions ceres/src/pack/import_repo.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ impl PackHandler for ImportRepo {
}
}

let want_tree_ids = want_commits.iter().map(|c| c.tree_id.to_string()).collect();
let want_tree_ids = &want_commits.iter().map(|c| c.tree_id.to_string()).collect();

Copilot AI May 15, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Binding a reference to the result of collect() may lead to lifetime issues; consider explicitly creating a vector variable to hold the collected values before referencing it.

Suggested change
let want_tree_ids = &want_commits.iter().map(|c| c.tree_id.to_string()).collect();
let want_tree_ids_vec: Vec<String> = want_commits.iter().map(|c| c.tree_id.to_string()).collect();
let want_tree_ids = &want_tree_ids_vec;

Copilot uses AI. Check for mistakes.
let want_trees: HashMap<SHA1, Tree> = storage
.get_trees_by_hashes(self.repo.repo_id, want_tree_ids)
.await
Expand All @@ -222,7 +222,7 @@ impl PackHandler for ImportRepo {
let have_trees = storage
.get_trees_by_hashes(
self.repo.repo_id,
have_commits.iter().map(|x| x.tree.clone()).collect(),
&have_commits.iter().map(|x| x.tree.clone()).collect(),
)
.await
.unwrap();
Expand Down Expand Up @@ -266,7 +266,7 @@ impl PackHandler for ImportRepo {
.context
.services
.git_db_storage
.get_trees_by_hashes(self.repo.repo_id, hashes)
.get_trees_by_hashes(self.repo.repo_id, &hashes)
.await
.unwrap()
.into_iter()
Expand Down
79 changes: 44 additions & 35 deletions jupiter/migration/src/m20250314_025943_init.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use extension::postgres::Type;
use sea_orm_migration::{
prelude::*,
schema::*,
sea_orm::{EnumIter, Iterable},
sea_orm::{DatabaseBackend, EnumIter, Iterable},
};

use crate::pk_bigint;
Expand All @@ -13,41 +13,50 @@ pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.create_type(
Type::create()
.as_enum(MergeStatusEnum)
.values(MergeStatus::iter())
.to_owned(),
)
.await?;

manager
.create_type(
Type::create()
.as_enum(StorageTypeEnum)
.values(StorageType::iter())
.to_owned(),
)
.await?;

manager
.create_type(
Type::create()
.as_enum(RefTypeEnum)
.values(RefType::iter())
.to_owned(),
)
.await?;
let backend = manager.get_database_backend();

match backend {
DatabaseBackend::Postgres => {
manager
.create_type(
Type::create()
.as_enum(StorageTypeEnum)
.values(StorageType::iter())
.to_owned(),
)
.await?;

manager
.create_type(
Type::create()
.as_enum(RefTypeEnum)
.values(RefType::iter())
.to_owned(),
)
.await?;

manager
.create_type(
Type::create()
.as_enum(ConvTypeEnum)
.values(ConvType::iter())
.to_owned(),
)
.await?;
manager
.create_type(
Type::create()
.as_enum(MergeStatusEnum)
.values(MergeStatus::iter())
.to_owned(),
)
.await?;
}

manager
.create_type(
Type::create()
.as_enum(ConvTypeEnum)
.values(ConvType::iter())
.to_owned(),
)
.await?;
DatabaseBackend::Sqlite | DatabaseBackend::MySql => {
// Do not create enum in sqlite
}
}

manager
.create_table(
Expand Down
2 changes: 1 addition & 1 deletion jupiter/src/storage/git_db_storage.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ impl GitDbStorage {
pub async fn get_trees_by_hashes(
&self,
repo_id: i64,
hashes: Vec<String>,
hashes: &Vec<String>,
) -> Result<Vec<git_tree::Model>, MegaError> {
Ok(git_tree::Entity::find()
.filter(git_tree::Column::RepoId.eq(repo_id))
Expand Down
Loading