Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 102 additions & 42 deletions ceres/src/api_service/import_api_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use std::collections::VecDeque;
use std::path::Component;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;

use async_trait::async_trait;

Expand All @@ -13,6 +14,8 @@ use mercury::hash::SHA1;
use mercury::internal::object::commit::Commit;
use mercury::internal::object::tree::Tree;
use mercury::internal::object::tree::TreeItem;
use mercury::internal::object::tree::TreeItemMode;
use tokio::sync::Mutex;

use crate::api_service::{ApiHandler, GitObjectCache};
use crate::model::git::CreateFileInfo;
Expand Down Expand Up @@ -101,19 +104,28 @@ impl ApiHandler for ImportApiService {
commit.map(|x| x.into())
}

async fn get_tree_relate_commit(&self, t_hash: &str) -> Commit {
let storage = self.storage.git_db_storage();
let tree_info = storage
.get_tree_by_hash(self.repo.repo_id, t_hash)
.await
.unwrap()
.unwrap();
storage
.get_commit_by_hash(self.repo.repo_id, &tree_info.commit_id)
async fn get_tree_relate_commit(
&self,
t_hash: SHA1,
path: PathBuf,
) -> Result<Commit, GitError> {
let file_name = match path.file_name() {
Some(name) => name.to_string_lossy().to_string(),
None => {
return Err(GitError::CustomError("Invalid Path Input".to_string()));
}
};

let search_item = TreeItem::new(TreeItemMode::Tree, t_hash, file_name);
let cache = GitObjectCache::new();
let root_commit = Arc::new(self.get_root_commit().await);

let parent = match path.parent() {
Some(p) => p,
None => return Err(GitError::CustomError("Invalid Path Input".to_string())),
};
self.traverse_commit_history(parent, root_commit, &search_item, cache)
.await
.unwrap()
.unwrap()
.into()
}

async fn get_commits_by_hashes(&self, c_hashes: Vec<String>) -> Result<Vec<Commit>, GitError> {
Expand All @@ -129,15 +141,15 @@ impl ApiHandler for ImportApiService {
&self,
path: PathBuf,
) -> Result<HashMap<TreeItem, Option<Commit>>, GitError> {
let mut cache = GitObjectCache::default();
let root_commit = self.get_root_commit().await;
let cache = GitObjectCache::new();
let root_commit = Arc::new(self.get_root_commit().await);
match self.search_tree_by_path(&path).await? {
Some(tree) => {
let mut result: HashMap<TreeItem, Option<Commit>> = HashMap::new();
for item in tree.tree_items {
let commit = self
.traverse_commit_history(&path, &root_commit, &item, &mut cache)
.await;
.traverse_commit_history(&path, root_commit.clone(), &item, cache.clone())
.await?;
result.insert(item, Some(commit));
}
Ok(result)
Expand All @@ -148,31 +160,74 @@ impl ApiHandler for ImportApiService {
}

impl ImportApiService {
/// Traverses the commit history starting from a given commit, looking for the earliest commit
/// (based on committer timestamp) where the target `TreeItem` is reachable at the given path.
///
/// The function performs a breadth-first search (BFS) through the commit graph, checking for the
/// target's existence in each commit's tree. It uses a commit and tree cache to avoid redundant
/// repository lookups.
///
/// # Arguments
/// * `path_components` - The path to search, pre-split into components to avoid repeated parsing.
/// * `start_commit` - The commit to start traversal from.
/// * `target` - The tree item we want to find under path (e.g., file or subdirectory).
/// * `cache` - A shared, mutable cache of commits and trees to speed up lookups.
///
/// # Returns
/// The earliest commit (by timestamp) in which the target path contains the given `TreeItem`.
///
/// # Algorithm
/// 1. Initialize a queue with the starting commit.
/// 2. Track visited commit IDs to prevent cycles.
/// 3. For each commit in the queue:
/// - Load its root tree from the cache (or repository if not cached).
/// - Check if the `target` is reachable at the given path.
/// - If reachable:
/// - Add unvisited parent commits to the queue.
/// - If this commit has an earlier timestamp than the current best match, update the result.
/// 4. Return the earliest matching commit.
///
/// # Performance Notes
/// - Uses `Arc<Commit>` internally to avoid cloning commits during traversal.
/// - Commit and tree lookups are cached in `GitObjectCache`.
///
/// # Locking
/// - `cache` is wrapped in `Arc<Mutex<_>>` for safe concurrent access across async calls.
async fn traverse_commit_history(
&self,
path: &Path,
start_commit: &Commit,
target: &TreeItem,
cache: &mut GitObjectCache,
) -> Commit {
start_commit: Arc<Commit>,
search_item: &TreeItem,
cache: Arc<Mutex<GitObjectCache>>,
) -> Result<Commit, GitError> {
let mut target_commit = start_commit.clone();
let mut visited = HashSet::new();
let mut p_stack = VecDeque::new();

visited.insert(start_commit.id);
p_stack.push_back(start_commit.clone());
p_stack.push_back(start_commit);

while let Some(commit) = p_stack.pop_front() {
let root_tree = self.get_tree_from_cache(commit.tree_id, cache).await;
let reachable = self
.reachable_in_tree(&root_tree, path, target, cache)
.await
.unwrap();
let root_tree = {
let mut cache_lock = cache.lock().await;
self.get_tree_from_cache(commit.tree_id, &mut cache_lock)
.await?
};

let reachable = {
let mut cache_lock = cache.lock().await;
self.reachable_in_tree(root_tree, path, search_item, &mut cache_lock)
.await?
};

if reachable {
for p_id in commit.parent_commit_ids.clone() {
for &p_id in &commit.parent_commit_ids {
if !visited.contains(&p_id) {
let p_commit = self.get_commit_from_cache(p_id, cache).await.unwrap();
p_stack.push_back(p_commit);
let p_commit = {
let mut cache_lock = cache.lock().await;
self.get_commit_from_cache(p_id, &mut cache_lock).await?
};
p_stack.push_back(p_commit.clone());
visited.insert(p_id);
}
}
Expand All @@ -181,44 +236,49 @@ impl ImportApiService {
}
}
}
target_commit
Ok((*target_commit).clone())
}

async fn get_tree_from_cache(&self, oid: SHA1, cache: &mut GitObjectCache) -> Tree {
async fn get_tree_from_cache(
&self,
oid: SHA1,
cache: &mut GitObjectCache,
) -> Result<Arc<Tree>, GitError> {
if let Some(tree) = cache.trees.get(&oid) {
return tree.clone();
return Ok(tree.clone());
}
let tree = self.get_tree_by_hash(&oid.to_string()).await;
let tree = Arc::new(self.get_tree_by_hash(&oid.to_string()).await);
cache.trees.insert(oid, tree.clone());
tree
Ok(tree)
}

async fn get_commit_from_cache(
&self,
oid: SHA1,
cache: &mut GitObjectCache,
) -> Result<Commit, GitError> {
) -> Result<Arc<Commit>, GitError> {
if let Some(commit) = cache.commits.get(&oid) {
return Ok(commit.clone());
}
match self.get_commit_by_hash(&oid.to_string()).await {
Some(c) => {
cache.commits.insert(oid, c.clone());
Ok(c)
let commit = Arc::new(c);
cache.commits.insert(oid, commit.clone());
Ok(commit)
}
None => Err(GitError::InvalidCommitObject),
}
}

async fn reachable_in_tree(
&self,
root_tree: &Tree,
root_tree: Arc<Tree>,
path: &Path,
target: &TreeItem,
search_item: &TreeItem,
cache: &mut GitObjectCache,
) -> Result<bool, GitError> {
let relative_path = self.strip_relative(path).unwrap();
let mut search_tree = root_tree.clone();
let mut search_tree = root_tree;
// first find search tree by path
for component in relative_path.components() {
// root tree already found
Expand All @@ -229,14 +289,14 @@ impl ImportApiService {
.iter()
.find(|x| x.name == target_name);
if let Some(search_res) = search_res {
search_tree = self.get_tree_from_cache(search_res.id, cache).await;
search_tree = self.get_tree_from_cache(search_res.id, cache).await?;
} else {
return Ok(false);
}
}
}
// check item exist under search tree
if search_tree.tree_items.iter().any(|x| x == target) {
if search_tree.tree_items.iter().any(|x| x == search_item) {
return Ok(true);
}
Ok(false)
Expand Down
16 changes: 12 additions & 4 deletions ceres/src/api_service/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use std::{
collections::{HashMap, VecDeque},
path::{Component, Path, PathBuf},
sync::Arc,
};

use async_trait::async_trait;
Expand All @@ -17,6 +18,7 @@ use mercury::{
ObjectTrait,
},
};
use tokio::sync::Mutex;

use crate::model::git::{
CreateFileInfo, LatestCommitInfo, TreeBriefItem, TreeCommitItem, TreeHashItem,
Expand All @@ -27,8 +29,14 @@ pub mod mono_api_service;

#[derive(Debug, Default, Clone)]
pub struct GitObjectCache {
trees: HashMap<SHA1, Tree>,
commits: HashMap<SHA1, Commit>,
trees: HashMap<SHA1, Arc<Tree>>,
commits: HashMap<SHA1, Arc<Commit>>,
}

impl GitObjectCache {
pub fn new() -> Arc<Mutex<GitObjectCache>> {
Arc::new(Mutex::new(GitObjectCache::default()))
}
}

#[async_trait]
Expand Down Expand Up @@ -68,7 +76,7 @@ pub trait ApiHandler: Send + Sync {

async fn get_commit_by_hash(&self, hash: &str) -> Option<Commit>;

async fn get_tree_relate_commit(&self, t_hash: &str) -> Commit;
async fn get_tree_relate_commit(&self, t_hash: SHA1, path: PathBuf) -> Result<Commit, GitError>;

async fn get_commits_by_hashes(&self, c_hashes: Vec<String>) -> Result<Vec<Commit>, GitError>;

Expand Down Expand Up @@ -96,7 +104,7 @@ pub trait ApiHandler: Send + Sync {
"can't find target parent tree under latest commit".to_string(),
));
};
let commit = self.get_tree_relate_commit(&tree.id.to_string()).await;
let commit = self.get_tree_relate_commit(tree.id, path).await?;
Ok(commit.into())
}

Expand Down
12 changes: 8 additions & 4 deletions ceres/src/api_service/mono_api_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -191,15 +191,19 @@ impl ApiHandler for MonoApiService {
}
}

async fn get_tree_relate_commit(&self, t_hash: &str) -> Commit {
async fn get_tree_relate_commit(&self, t_hash: SHA1, _: PathBuf) -> Result<Commit, GitError> {
let storage = self.storage.mono_storage();
let tree_info = storage.get_tree_by_hash(t_hash).await.unwrap().unwrap();
storage
let tree_info = storage
.get_tree_by_hash(&t_hash.to_string())
.await
.unwrap()
.unwrap();
Ok(storage
.get_commit_by_hash(&tree_info.commit_id)
.await
.unwrap()
.unwrap()
.into()
.into())
}

async fn get_commits_by_hashes(&self, c_hashes: Vec<String>) -> Result<Vec<Commit>, GitError> {
Expand Down
1 change: 0 additions & 1 deletion jupiter/callisto/src/git_blob.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ pub struct Model {
pub blob_id: String,
pub name: Option<String>,
pub size: i32,
pub commit_id: String,
pub created_at: DateTime,
}

Expand Down
1 change: 0 additions & 1 deletion jupiter/callisto/src/git_tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ pub struct Model {
#[sea_orm(column_type = "VarBinary(StringLen::None)")]
pub sub_trees: Vec<u8>,
pub size: i32,
pub commit_id: String,
pub created_at: DateTime,
}

Expand Down
42 changes: 42 additions & 0 deletions jupiter/src/migration/m20250815_075653_remove_commit_id.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
use sea_orm_migration::prelude::*;

#[derive(DeriveMigrationName)]
pub struct Migration;

#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(GitBlob::Table)
.drop_column("commit_id")
.to_owned(),
)
.await?;

manager
.alter_table(
Table::alter()
.table(GitTree::Table)
.drop_column("commit_id")
.to_owned(),
)
.await?;
Ok(())
}

async fn down(&self, _: &SchemaManager) -> Result<(), DbErr> {
Comment thread
benjamin-747 marked this conversation as resolved.
Ok(())
}
}

#[derive(DeriveIden)]
enum GitTree {
Table,
}

#[derive(DeriveIden)]
enum GitBlob {
Table,
}
2 changes: 2 additions & 0 deletions jupiter/src/migration/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ mod m20250710_073119_create_reactions;
mod m20250725_103004_add_note;
mod m20250804_151214_alter_builds_end_at;
mod m20250812_022434_alter_mega_mr;
mod m20250815_075653_remove_commit_id;
/// Creates a primary key column definition with big integer type.
///
/// # Arguments
Expand Down Expand Up @@ -81,6 +82,7 @@ impl MigratorTrait for Migrator {
Box::new(m20250725_103004_add_note::Migration),
Box::new(m20250804_151214_alter_builds_end_at::Migration),
Box::new(m20250812_022434_alter_mega_mr::Migration),
Box::new(m20250815_075653_remove_commit_id::Migration),
]
}
}
Expand Down
Loading
Loading