diff --git a/Cargo.toml b/Cargo.toml index f26311a76..97c0175dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,16 +24,16 @@ callisto = { path = "jupiter/callisto" } gemini = { path = "gemini" } vault = { path = "vault" } -anyhow = "1.0.83" -serde = "1.0.201" +anyhow = "1.0.86" +serde = "1.0.203" serde_json = "1.0.117" tracing = "0.1.40" tracing-subscriber = "0.3.18" tracing-appender = "0.2" -thiserror = "1.0.60" +thiserror = "1.0.61" rand = "0.8.5" smallvec = "1.13.2" -tokio = "1.37.0" +tokio = "1.38.0" tokio-stream = "0.1.15" tokio-test = "0.4.4" rayon = "1.10.0" @@ -55,11 +55,11 @@ tower-http = "0.5.2" tower = "0.4.13" hex = "0.4.3" sea-orm = "0.12.15" -flate2 = "1.0.28" +flate2 = "1.0.30" bstr = "1.9.1" colored = "2.1.0" idgenerator = "2.0.0" num_cpus = "1.16.0" config = "0.14.0" shadow-rs = "0.29.0" -reqwest = "0.12.4" +reqwest = "0.12.5" diff --git a/archived/git/src/structure/conversion.rs b/archived/git/src/structure/conversion.rs index 1ff388deb..794f00573 100644 --- a/archived/git/src/structure/conversion.rs +++ b/archived/git/src/structure/conversion.rs @@ -121,7 +121,7 @@ impl PackProtocol { let mut traversal_list: Vec = vec![commit.clone()]; let mut want_commits: Vec = vec![commit]; - // tarverse commit's all parents to find the commit that client does not have + // traverse commit's all parents to find the commit that client does not have while let Some(temp) = traversal_list.pop() { for p_commit_id in temp.parent_commit_ids { let p_commit_id = &p_commit_id.to_plain_str(); diff --git a/ceres/src/api_service/import_api_service.rs b/ceres/src/api_service/import_api_service.rs new file mode 100644 index 000000000..d04386fd9 --- /dev/null +++ b/ceres/src/api_service/import_api_service.rs @@ -0,0 +1,236 @@ +use std::collections::HashMap; +use std::collections::HashSet; +use std::collections::VecDeque; +use std::path::Path; +use std::path::PathBuf; + +use axum::async_trait; + +use callisto::raw_blob; +use common::errors::MegaError; +use jupiter::context::Context; +use mercury::errors::GitError; +use mercury::internal::object::commit::Commit; +use mercury::internal::object::tree::Tree; +use mercury::internal::object::tree::TreeItem; +use venus::import_repo::repo::Repo; + +use crate::api_service::ApiHandler; +use crate::model::create_file::CreateFileInfo; + +#[derive(Clone)] +pub struct ImportApiService { + pub context: Context, + pub repo: Repo, +} + +#[async_trait] +impl ApiHandler for ImportApiService { + async fn create_monorepo_file(&self, _: CreateFileInfo) -> Result<(), GitError> { + return Err(GitError::CustomError( + "import dir does not support create file".to_string(), + )); + } + + async fn get_raw_blob_by_hash(&self, hash: &str) -> Result, MegaError> { + self.context + .services + .mega_storage + .get_raw_blob_by_hash(hash) + .await + } + + fn strip_relative(&self, path: &Path) -> Result { + if let Ok(relative_path) = path.strip_prefix(self.repo.repo_path.clone()) { + Ok(relative_path.to_path_buf()) + } else { + Err(GitError::ConversionError( + "The full path does not start with the base path.".to_string(), + )) + } + } + async fn get_root_commit(&self) -> Commit { + let storage = self.context.services.git_db_storage.clone(); + let refs = storage.get_default_ref(&self.repo).await.unwrap().unwrap(); + storage + .get_commit_by_hash(&self.repo, &refs.ref_hash) + .await + .unwrap() + .unwrap() + .into() + } + + async fn get_root_tree(&self) -> Tree { + let storage = self.context.services.git_db_storage.clone(); + let refs = storage.get_default_ref(&self.repo).await.unwrap().unwrap(); + + let root_commit = storage + .get_commit_by_hash(&self.repo, &refs.ref_hash) + .await + .unwrap() + .unwrap(); + storage + .get_tree_by_hash(&self.repo, &root_commit.tree) + .await + .unwrap() + .unwrap() + .into() + } + + async fn get_tree_by_hash(&self, hash: &str) -> Tree { + self.context + .services + .git_db_storage + .get_tree_by_hash(&self.repo, hash) + .await + .unwrap() + .unwrap() + .into() + } + + async fn get_tree_relate_commit(&self, t_hash: &str) -> Commit { + let storage = self.context.services.git_db_storage.clone(); + let tree_info = storage + .get_tree_by_hash(&self.repo, t_hash) + .await + .unwrap() + .unwrap(); + storage + .get_commit_by_hash(&self.repo, &tree_info.commit_id) + .await + .unwrap() + .unwrap() + .into() + } + + async fn add_trees_to_map( + &self, + item_to_commit: &mut HashMap, + hashes: Vec, + ) { + let storage = self.context.services.git_db_storage.clone(); + let trees = storage + .get_trees_by_hashes(&self.repo, hashes) + .await + .unwrap(); + for tree in trees { + item_to_commit.insert(tree.tree_id, tree.commit_id); + } + } + + async fn add_blobs_to_map( + &self, + item_to_commit: &mut HashMap, + hashes: Vec, + ) { + let storage = self.context.services.git_db_storage.clone(); + let blobs = storage + .get_blobs_by_hashes(&self.repo, hashes) + .await + .unwrap(); + for blob in blobs { + item_to_commit.insert(blob.blob_id, blob.commit_id); + } + } + + async fn get_commits_by_hashes(&self, c_hashes: Vec) -> Result, GitError> { + let storage = self.context.services.git_db_storage.clone(); + let commits = storage + .get_commits_by_hashes(&self.repo, &c_hashes) + .await + .unwrap(); + Ok(commits.into_iter().map(|x| x.into()).collect()) + } + + async fn traverse_commit_history( + &self, + path: &Path, + start_commit: Commit, + target: TreeItem, + ) -> Commit { + let mut target_commit = start_commit.clone(); + let mut visited = HashSet::new(); + let mut p_stack = VecDeque::new(); + + visited.insert(start_commit.id); + p_stack.push_back(start_commit); + + while let Some(commit) = p_stack.pop_front() { + let root_tree = self.get_tree_by_hash(&commit.tree_id.to_plain_str()).await; + let reachable = self + .reachable_in_tree(&root_tree, path, target.clone()) + .await + .unwrap(); + if reachable { + let mut p_ids = vec![]; + for p_id in commit.parent_commit_ids.clone() { + if !visited.contains(&p_id) { + p_ids.push(p_id.to_plain_str()); + visited.insert(p_id); + } + } + if target_commit.committer.timestamp > commit.committer.timestamp { + target_commit = commit; + } + let parent_commits = self.get_commits_by_hashes(p_ids).await.unwrap(); + p_stack.extend(parent_commits); + } + } + target_commit + } +} + +impl ImportApiService { + // pub async fn get_objects_data( + // &self, + // object_id: &str, + // repo_path: &str, + // ) -> Result { + // let node = match self.storage.get_node_by_hash(object_id, repo_path).await { + // Ok(Some(node)) => node, + // _ => return Err((StatusCode::NOT_FOUND, "Blob not found".to_string())), + // }; + // let raw_data = match self.storage.get_obj_data_by_id(object_id).await { + // Ok(Some(model)) => model, + // _ => return Err((StatusCode::NOT_FOUND, "Blob not found".to_string())), + // }; + // let file_name = format!("inline; filename=\"{}\"", node.name.unwrap()); + // let res = Response::builder() + // .header("Content-Type", "application/octet-stream") + // .header("Content-Disposition", file_name) + // .body(raw_data.data.into()) + // .unwrap(); + // Ok(res) + // } + + // pub async fn count_object_num( + // &self, + // repo_path: &str, + // ) -> Result, (StatusCode, String)> { + // let query_res = self.storage.count_obj_from_node(repo_path).await.unwrap(); + // let tree = query_res + // .iter() + // .find(|x| x.node_type == "tree") + // .map(|x| x.count) + // .unwrap_or_default() + // .try_into() + // .unwrap(); + // let blob = query_res + // .iter() + // .find(|x| x.node_type == "blob") + // .map(|x| x.count) + // .unwrap_or_default() + // .try_into() + // .unwrap(); + // let commit = self.storage.count_obj_from_commit(repo_path).await.unwrap().try_into().unwrap(); + // let counter = GitTypeCounter { + // commit, + // tree, + // blob, + // tag: 0, + // ofs_delta: 0, + // ref_delta: 0, + // }; + // Ok(Json(counter)) + // } +} diff --git a/ceres/src/api_service/mod.rs b/ceres/src/api_service/mod.rs new file mode 100644 index 000000000..d5a79b886 --- /dev/null +++ b/ceres/src/api_service/mod.rs @@ -0,0 +1,274 @@ +use std::{ + collections::{HashMap, HashSet}, + path::{Component, Path, PathBuf}, +}; + +use axum::async_trait; + +use callisto::raw_blob; +use common::errors::MegaError; +use mercury::{ + errors::GitError, + internal::object::{ + commit::Commit, + tree::{Tree, TreeItem, TreeItemMode}, + }, +}; + +use crate::model::{ + create_file::CreateFileInfo, + objects::{ + BlobObjects, LatestCommitInfo, TreeBriefInfo, TreeBriefItem, TreeCommitInfo, + TreeCommitItem, UserInfo, + }, +}; + +pub mod import_api_service; +pub mod mono_api_service; + +#[async_trait] +pub trait ApiHandler: Send + Sync { + async fn create_monorepo_file(&self, file_info: CreateFileInfo) -> Result<(), GitError>; + + async fn get_raw_blob_by_hash(&self, hash: &str) -> Result, MegaError>; + + fn strip_relative(&self, path: &Path) -> Result; + + async fn get_root_commit(&self) -> Commit; + + async fn get_root_tree(&self) -> Tree; + + async fn get_tree_by_hash(&self, hash: &str) -> Tree; + + async fn get_tree_relate_commit(&self, t_hash: &str) -> Commit; + + async fn add_trees_to_map( + &self, + item_to_commit: &mut HashMap, + hashes: Vec, + ); + + async fn add_blobs_to_map( + &self, + item_to_commit: &mut HashMap, + hashes: Vec, + ); + + async fn get_commits_by_hashes(&self, c_hashes: Vec) -> Result, GitError>; + + async fn traverse_commit_history( + &self, + path: &Path, + commit: Commit, + target: TreeItem, + ) -> Commit; + + async fn get_blob_as_string( + &self, + path: PathBuf, + filename: &str, + ) -> Result { + let (_, tree) = self.search_tree_by_path(&path).await.unwrap(); + let mut plain_text = String::new(); + if let Some(item) = tree.tree_items.into_iter().find(|x| x.name == filename) { + plain_text = match self.get_raw_blob_by_hash(&item.id.to_plain_str()).await { + Ok(Some(model)) => String::from_utf8(model.data.unwrap()).unwrap(), + _ => String::new(), + }; + } + Ok(BlobObjects { plain_text }) + } + + async fn get_latest_commit(&self, path: PathBuf) -> Result { + let (_, tree) = self.search_tree_by_path(&path).await.unwrap(); + let commit = self.get_tree_relate_commit(&tree.id.to_plain_str()).await; + self.convert_commit_to_info(commit) + } + + async fn get_tree_info(&self, path: PathBuf) -> Result { + match self.search_tree_by_path(&path).await { + Ok((_, tree)) => { + let mut items = Vec::new(); + for item in tree.tree_items { + let mut info: TreeBriefItem = item.clone().into(); + path.join(item.name) + .to_str() + .unwrap() + .clone_into(&mut info.path); + items.push(info); + } + Ok(TreeBriefInfo { + total_count: items.len(), + items, + }) + } + Err(_) => Ok(TreeBriefInfo { + total_count: 0, + items: Vec::new(), + }), + } + } + + async fn get_tree_commit_info(&self, path: PathBuf) -> Result { + match self.search_tree_by_path(&path).await { + Ok((_, tree)) => { + let mut item_to_commit = HashMap::new(); + + self.add_trees_to_map( + &mut item_to_commit, + tree.tree_items + .iter() + .filter(|x| x.mode == TreeItemMode::Tree) + .map(|x| x.id.to_plain_str()) + .collect(), + ) + .await; + + self.add_blobs_to_map( + &mut item_to_commit, + tree.tree_items + .iter() + .filter(|x| x.mode == TreeItemMode::Blob) + .map(|x| x.id.to_plain_str()) + .collect(), + ) + .await; + + let mut items = Vec::new(); + let commit_ids: HashSet = item_to_commit.values().cloned().collect(); + let commits = self + .get_commits_by_hashes(commit_ids.into_iter().collect()) + .await + .unwrap(); + let commit_map: HashMap = commits + .into_iter() + .map(|x| (x.id.to_plain_str(), x)) + .collect(); + + for item in tree.tree_items { + let mut info: TreeCommitItem = item.clone().into(); + if let Some(commit_id) = item_to_commit.get(&item.id.to_plain_str()) { + let commit = if let Some(commit) = commit_map.get(commit_id) { + commit + } else { + tracing::error!("failed fecth commit: {}", commit_id); + &self + .traverse_commit_history(&path, self.get_root_commit().await, item) + .await + }; + info.oid = commit.id.to_plain_str(); + info.message = commit.format_message(); + info.date = commit.committer.timestamp.to_string(); + } + items.push(info); + } + Ok(TreeCommitInfo { + total_count: items.len(), + items, + }) + } + Err(_) => Ok(TreeCommitInfo { + total_count: 0, + items: Vec::new(), + }), + } + } + + fn convert_commit_to_info(&self, commit: Commit) -> Result { + let message = commit.format_message(); + let committer = UserInfo { + display_name: commit.committer.name, + ..Default::default() + }; + let author = UserInfo { + display_name: commit.author.name, + ..Default::default() + }; + + let res = LatestCommitInfo { + oid: commit.id.to_plain_str(), + date: commit.committer.timestamp.to_string(), + short_message: message, + author, + committer, + status: "success".to_string(), + }; + Ok(res) + } + + /// Searches for a tree and affected parent by path. + /// + /// This function asynchronously searches for a tree by the provided path. + /// + /// # Arguments + /// + /// * `path` - A reference to the path to search. + /// + /// # Returns + /// + /// Returns a tuple containing a vector of parent trees to be updated and + /// the target tree if found, or an error of type `GitError`. + async fn search_tree_by_path(&self, path: &Path) -> Result<(Vec, Tree), GitError> { + let relative_path = self.strip_relative(path).unwrap(); + let root_tree = self.get_root_tree().await; + let mut search_tree = root_tree.clone(); + let mut update_tree = vec![root_tree]; + let component_num = relative_path.components().count(); + + for (index, component) in relative_path.components().enumerate() { + // root tree already found + if component != Component::RootDir { + let target_name = component.as_os_str().to_str().unwrap(); + let search_res = search_tree + .tree_items + .iter() + .find(|x| x.name == target_name); + + if let Some(search_res) = search_res { + let res = self.get_tree_by_hash(&search_res.id.to_plain_str()).await; + search_tree = res.clone(); + // skip last component + if index != component_num - 1 { + update_tree.push(res); + } + } else { + return Err(GitError::ConversionError( + "can't find target parent tree under latest commit".to_string(), + )); + } + } + } + Ok((update_tree, search_tree)) + } + + async fn reachable_in_tree( + &self, + root_tree: &Tree, + path: &Path, + target: TreeItem, + ) -> Result { + let relative_path = self.strip_relative(path).unwrap(); + let mut search_tree = root_tree.clone(); + // first find search tree by path + for component in relative_path.components() { + // root tree already found + if component != Component::RootDir { + let target_name = component.as_os_str().to_str().unwrap(); + let search_res = search_tree + .tree_items + .iter() + .find(|x| x.name == target_name); + if let Some(search_res) = search_res { + search_tree = self.get_tree_by_hash(&search_res.id.to_plain_str()).await; + } else { + return Ok(false); + } + } + } + // check item exist under search tree + if search_tree.tree_items.into_iter().any(|x| x == target) { + return Ok(true); + } + Ok(false) + } +} diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs new file mode 100644 index 000000000..dd04b5137 --- /dev/null +++ b/ceres/src/api_service/mono_api_service.rs @@ -0,0 +1,332 @@ +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use axum::async_trait; + +use callisto::db_enums::{ConvType, MergeStatus}; +use callisto::{mega_blob, mega_tree, raw_blob}; +use common::errors::MegaError; +use jupiter::context::Context; +use jupiter::storage::batch_save_model; +use mercury::errors::GitError; +use mercury::hash::SHA1; +use mercury::internal::object::blob::Blob; +use mercury::internal::object::commit::Commit; +use mercury::internal::object::tree::{Tree, TreeItem, TreeItemMode}; +use venus::monorepo::converter; +use venus::monorepo::mr::MergeOperation; + +use crate::api_service::ApiHandler; +use crate::model::create_file::CreateFileInfo; +use crate::model::objects::{CommonResult, MrInfoItem}; + +#[derive(Clone)] +pub struct MonoApiService { + pub context: Context, +} + +#[async_trait] +impl ApiHandler for MonoApiService { + async fn create_monorepo_file(&self, file_info: CreateFileInfo) -> Result<(), GitError> { + let storage = self.context.services.mega_storage.clone(); + let path = PathBuf::from(file_info.path); + let mut save_trees = vec![]; + + let (update_trees, search_tree) = self.search_tree_by_path(&path).await.unwrap(); + let mut t_items = search_tree.tree_items; + + let new_item = if file_info.is_directory { + if t_items + .iter() + .any(|x| x.mode == TreeItemMode::Tree && x.name == file_info.name) + { + return Err(GitError::CustomError("Duplicate name".to_string())); + } + let blob = converter::generate_git_keep_with_timestamp(); + let tree_item = TreeItem { + mode: TreeItemMode::Blob, + id: blob.id, + name: String::from(".gitkeep"), + }; + let child_tree = Tree::from_tree_items(vec![tree_item]).unwrap(); + save_trees.push(child_tree.clone()); + TreeItem { + mode: TreeItemMode::Tree, + id: child_tree.id, + name: file_info.name.clone(), + } + } else { + let blob = Blob::from_content(&file_info.content.unwrap()); + let mega_blob: mega_blob::Model = (&blob).into(); + let mega_blob: mega_blob::ActiveModel = mega_blob.into(); + let raw_blob: raw_blob::Model = blob.clone().into(); + let raw_blob: raw_blob::ActiveModel = raw_blob.into(); + batch_save_model(storage.get_connection(), vec![mega_blob]) + .await + .unwrap(); + batch_save_model(storage.get_connection(), vec![raw_blob]) + .await + .unwrap(); + TreeItem { + mode: TreeItemMode::Blob, + id: blob.id, + name: file_info.name.clone(), + } + }; + t_items.push(new_item); + let p_tree = Tree::from_tree_items(t_items).unwrap(); + + let refs = storage.get_ref("/").await.unwrap().unwrap(); + let commit = Commit::from_tree_id( + p_tree.id, + vec![SHA1::from_str(&refs.ref_commit_hash).unwrap()], + &format!("create file {} commit", file_info.name), + ); + + let commit_id = self + .update_parent_tree(path, update_trees, commit) + .await + .unwrap(); + save_trees.push(p_tree); + + for save_t in save_trees { + let mut tree_model: mega_tree::Model = save_t.into(); + tree_model.commit_id.clone_from(&commit_id); + let tree_model: mega_tree::ActiveModel = tree_model.into(); + batch_save_model(storage.get_connection(), vec![tree_model]) + .await + .unwrap(); + } + Ok(()) + } + + async fn get_raw_blob_by_hash(&self, hash: &str) -> Result, MegaError> { + self.context + .services + .mega_storage + .get_raw_blob_by_hash(hash) + .await + } + + fn strip_relative(&self, path: &Path) -> Result { + Ok(path.to_path_buf()) + } + + async fn get_root_commit(&self) -> Commit { + unreachable!() + } + + async fn get_root_tree(&self) -> Tree { + let storage = self.context.services.mega_storage.clone(); + let refs = storage.get_ref("/").await.unwrap().unwrap(); + + storage + .get_tree_by_hash(&refs.ref_tree_hash) + .await + .unwrap() + .unwrap() + .into() + } + + async fn get_tree_by_hash(&self, hash: &str) -> Tree { + self.context + .services + .mega_storage + .get_tree_by_hash(hash) + .await + .unwrap() + .unwrap() + .into() + } + + async fn get_tree_relate_commit(&self, t_hash: &str) -> Commit { + let storage = self.context.services.mega_storage.clone(); + let tree_info = storage.get_tree_by_hash(t_hash).await.unwrap().unwrap(); + storage + .get_commit_by_hash(&tree_info.commit_id) + .await + .unwrap() + .unwrap() + .into() + } + + async fn add_trees_to_map( + &self, + item_to_commit: &mut HashMap, + hashes: Vec, + ) { + let storage = self.context.services.mega_storage.clone(); + let trees = storage.get_trees_by_hashes(hashes).await.unwrap(); + for tree in trees { + item_to_commit.insert(tree.tree_id, tree.commit_id); + } + } + + async fn add_blobs_to_map( + &self, + item_to_commit: &mut HashMap, + hashes: Vec, + ) { + let storage = self.context.services.mega_storage.clone(); + let blobs = storage.get_mega_blobs_by_hashes(hashes).await.unwrap(); + for blob in blobs { + item_to_commit.insert(blob.blob_id, blob.commit_id); + } + } + + async fn get_commits_by_hashes(&self, c_hashes: Vec) -> Result, GitError> { + let storage = self.context.services.mega_storage.clone(); + let commits = storage.get_commits_by_hashes(&c_hashes).await.unwrap(); + Ok(commits.into_iter().map(|x| x.into()).collect()) + } + + async fn traverse_commit_history(&self, _: &Path, _: Commit, _: TreeItem) -> Commit { + unreachable!() + } +} + +impl MonoApiService { + pub async fn init_monorepo(&self) { + self.context.services.mega_storage.init_monorepo().await + } + + pub async fn mr_list(&self, status: &str) -> Result>, MegaError> { + let status = if status == "open" { + vec![MergeStatus::Open] + } else if status == "closed" { + vec![MergeStatus::Closed, MergeStatus::Merged] + } else { + return Err(MegaError::with_message("Invalid status name")); + }; + let storage = self.context.services.mega_storage.clone(); + let mr_list = storage.get_mr_by_status(status).await.unwrap(); + let res = CommonResult { + data: Some(mr_list.into_iter().map(|m| m.into()).collect()), + result: true, + err_message: String::new(), + }; + Ok(res) + } + + pub async fn merge_mr(&self, op: MergeOperation) -> Result<(), MegaError> { + let storage = self.context.services.mega_storage.clone(); + if let Some(mut mr) = storage.get_open_mr_by_id(op.mr_id).await.unwrap() { + let refs = storage.get_ref(&mr.path).await.unwrap().unwrap(); + + if mr.from_hash == refs.ref_commit_hash { + // update mr + mr.merge(op.comment); + storage.update_mr(mr.clone()).await.unwrap(); + + let commit: Commit = storage + .get_commit_by_hash(&mr.to_hash) + .await + .unwrap() + .unwrap() + .into(); + + // add conversation + storage + .add_mr_conversation(mr.id, 0, ConvType::Merged) + .await + .unwrap(); + if mr.path != "/" { + let path = PathBuf::from(mr.path.clone()); + // beacuse only parent tree is needed so we skip current directory + let (tree_vec, _) = self + .search_tree_by_path(path.parent().unwrap()) + .await + .unwrap(); + self.update_parent_tree(path, tree_vec, commit) + .await + .unwrap(); + // remove refs start with path + storage.remove_refs(&mr.path).await.unwrap(); + // TODO: self.clean_dangling_commits().await; + } + } else { + return Err(MegaError::with_message("ref hash conflict")); + } + } else { + return Err(MegaError::with_message("Invalid mr id")); + } + Ok(()) + } + + async fn update_parent_tree( + &self, + mut path: PathBuf, + mut tree_vec: Vec, + commit: Commit, + ) -> Result { + let storage = self.context.services.mega_storage.clone(); + let mut save_trees = Vec::new(); + let mut p_commit_id = String::new(); + + let mut target_hash = commit.tree_id; + + while let Some(mut tree) = tree_vec.pop() { + let cloned_path = path.clone(); + let name = cloned_path.file_name().unwrap().to_str().unwrap(); + path.pop(); + + let index = tree.tree_items.iter().position(|x| x.name == name).unwrap(); + tree.tree_items[index].id = target_hash; + let new_tree = Tree::from_tree_items(tree.tree_items).unwrap(); + target_hash = new_tree.id; + + let model: mega_tree::Model = new_tree.into(); + save_trees.push(model); + + let p_ref = storage.get_ref(path.to_str().unwrap()).await.unwrap(); + if let Some(mut p_ref) = p_ref { + if path == Path::new("/") { + let p_commit = Commit::new( + commit.author.clone(), + commit.committer.clone(), + target_hash, + vec![SHA1::from_str(&p_ref.ref_commit_hash).unwrap()], + &commit.message, + ); + p_commit_id = p_commit.id.to_plain_str(); + // update p_ref + p_ref.ref_commit_hash = p_commit.id.to_plain_str(); + p_ref.ref_tree_hash = target_hash.to_plain_str(); + storage.update_ref(p_ref).await.unwrap(); + storage.save_mega_commits(vec![p_commit]).await.unwrap(); + } else { + storage.remove_ref(p_ref).await.unwrap(); + } + } + } + let save_trees: Vec = save_trees + .into_iter() + .map(|mut x| { + p_commit_id.clone_into(&mut x.commit_id); + x.into() + }) + .collect(); + + batch_save_model(storage.get_connection(), save_trees) + .await + .unwrap(); + Ok(p_commit_id) + } +} + +#[cfg(test)] +mod test { + use std::path::PathBuf; + + #[test] + pub fn test() { + let mut full_path = PathBuf::from("/project/rust/mega"); + for _ in 0..3 { + let cloned_path = full_path.clone(); // Clone full_path + let name = cloned_path.file_name().unwrap().to_str().unwrap(); + full_path.pop(); + println!("name: {}, path: {:?}", name, full_path); + } + } +} diff --git a/ceres/src/lib.rs b/ceres/src/lib.rs index 9454f1c3f..94cd55038 100644 --- a/ceres/src/lib.rs +++ b/ceres/src/lib.rs @@ -1,4 +1,6 @@ +pub mod api_service; pub mod http; pub mod lfs; -pub mod protocol; pub mod pack; +pub mod protocol; +pub mod model; diff --git a/gateway/src/model/create_file.rs b/ceres/src/model/create_file.rs similarity index 100% rename from gateway/src/model/create_file.rs rename to ceres/src/model/create_file.rs diff --git a/gateway/src/model/mod.rs b/ceres/src/model/mod.rs similarity index 100% rename from gateway/src/model/mod.rs rename to ceres/src/model/mod.rs diff --git a/gateway/src/model/objects.rs b/ceres/src/model/objects.rs similarity index 66% rename from gateway/src/model/objects.rs rename to ceres/src/model/objects.rs index b405d1c37..b9c80ca47 100644 --- a/gateway/src/model/objects.rs +++ b/ceres/src/model/objects.rs @@ -1,3 +1,4 @@ +use callisto::mega_mr; use serde::{Deserialize, Serialize}; use mercury::internal::object::tree::{TreeItem, TreeItemMode}; @@ -88,3 +89,47 @@ impl From for TreeBriefItem { pub struct BlobObjects { pub plain_text: String, } + +#[derive(Serialize, Deserialize)] +pub struct MrInfoItem { + pub title: String, + pub status: String, + pub open_date: String, + pub merge_date: String, +} + +impl From for MrInfoItem { + fn from(value: mega_mr::Model) -> Self { + Self { + title: String::new(), + status: value.status.to_string(), + open_date: value.created_at.to_string(), + merge_date: value.merge_date.unwrap().to_string(), + } + } +} + +#[derive(PartialEq, Eq, Debug, Clone, Default, Serialize, Deserialize)] + +pub struct CommonResult { + pub result: bool, + pub data: Option, + pub err_message: String, +} + +impl CommonResult { + pub fn success(data: Option) -> Self { + CommonResult { + result: true, + data, + err_message: "".to_owned(), + } + } + pub fn failed(err_message: &str) -> Self { + CommonResult { + result: false, + data: None, + err_message: err_message.to_string(), + } + } +} diff --git a/gateway/src/model/query.rs b/ceres/src/model/query.rs similarity index 100% rename from gateway/src/model/query.rs rename to ceres/src/model/query.rs diff --git a/ceres/src/pack/handler.rs b/ceres/src/pack/handler.rs index 9825260f0..dd57617f3 100644 --- a/ceres/src/pack/handler.rs +++ b/ceres/src/pack/handler.rs @@ -31,6 +31,37 @@ use venus::import_repo::import_refs::{RefCommand, Refs}; pub trait PackHandler: Send + Sync { async fn head_hash(&self) -> (String, Vec); + async fn handle_receiver(&self, rx: Receiver) -> Result<(), GitError>; + + /// Asynchronously retrieves the full pack data for the specified repository path. + /// This function collects commits and nodes from the storage and packs them into + /// a single binary vector. There is no need to build the entire tree; the function + /// only sends all the data related to this repository. + /// + /// # Returns + /// * `Result, GitError>` - The packed binary data as a vector of bytes. + /// + async fn full_pack(&self) -> Result>, GitError>; + + async fn incremental_pack( + &self, + want: Vec, + have: Vec, + ) -> Result>, GitError>; + + async fn get_trees_by_hashes(&self, hashes: Vec) -> Result, MegaError>; + + async fn get_blobs_by_hashes( + &self, + hashes: Vec, + ) -> Result, MegaError>; + + async fn update_refs(&self, refs: &RefCommand) -> Result<(), GitError>; + + async fn check_commit_exist(&self, hash: &str) -> bool; + + async fn check_default_branch(&self) -> bool; + fn find_head_hash(&self, refs: Vec) -> (String, Vec) { let mut head_hash = ZERO_ID.to_string(); for git_ref in refs.iter() { @@ -59,24 +90,6 @@ pub trait PackHandler: Send + Sync { Ok(receiver) } - async fn handle_receiver(&self, rx: Receiver) -> Result<(), GitError>; - - /// Asynchronously retrieves the full pack data for the specified repository path. - /// This function collects commits and nodes from the storage and packs them into - /// a single binary vector. There is no need to build the entire tree; the function - /// only sends all the data related to this repository. - /// - /// # Returns - /// * `Result, GitError>` - The packed binary data as a vector of bytes. - /// - async fn full_pack(&self) -> Result>, GitError>; - - async fn incremental_pack( - &self, - want: Vec, - have: Vec, - ) -> Result>, GitError>; - async fn traverse_for_count( &self, tree: Tree, @@ -160,17 +173,4 @@ pub trait PackHandler: Send + Sync { sender.send(tree.into()).await.unwrap(); } } - - async fn get_trees_by_hashes(&self, hashes: Vec) -> Result, MegaError>; - - async fn get_blobs_by_hashes( - &self, - hashes: Vec, - ) -> Result, MegaError>; - - async fn update_refs(&self, refs: &RefCommand) -> Result<(), GitError>; - - async fn check_commit_exist(&self, hash: &str) -> bool; - - async fn check_default_branch(&self) -> bool; } diff --git a/ceres/src/pack/import_repo.rs b/ceres/src/pack/import_repo.rs index 756d87c5e..8c8bd90f3 100644 --- a/ceres/src/pack/import_repo.rs +++ b/ceres/src/pack/import_repo.rs @@ -1,5 +1,6 @@ use std::{ collections::{HashMap, HashSet}, + path::PathBuf, str::FromStr, sync::{ atomic::{AtomicUsize, Ordering}, @@ -28,7 +29,11 @@ use venus::import_repo::{ repo::Repo, }; -use crate::pack::handler::PackHandler; +use crate::{ + api_service::{mono_api_service::MonoApiService, ApiHandler}, + model::create_file::CreateFileInfo, + pack::handler::PackHandler, +}; pub struct ImportRepo { pub context: Context, @@ -50,6 +55,7 @@ impl PackHandler for ImportRepo { } async fn handle_receiver(&self, receiver: Receiver) -> Result<(), GitError> { + self.create_tree_not_exist().await; let storage = self.context.services.git_db_storage.clone(); let mut entry_list = vec![]; let mut join_tasks = vec![]; @@ -311,3 +317,21 @@ impl PackHandler for ImportRepo { storage.default_branch_exist(&self.repo).await.unwrap() } } + +impl ImportRepo { + async fn create_tree_not_exist(&self) { + let path = PathBuf::from(self.repo.repo_path.clone()); + let f_name = path.file_name().unwrap().to_str().unwrap(); + let api_service = MonoApiService { + context: self.context.clone(), + }; + let req = CreateFileInfo { + is_directory: true, + name: f_name.to_owned(), + path: path.parent().unwrap().to_str().unwrap().to_owned(), + content: None, + }; + + if (api_service.create_monorepo_file(req).await).is_ok() {} + } +} diff --git a/gateway/src/api_service/router.rs b/gateway/src/api_router.rs similarity index 72% rename from gateway/src/api_service/router.rs rename to gateway/src/api_router.rs index b79943726..37edc9ee9 100644 --- a/gateway/src/api_service/router.rs +++ b/gateway/src/api_router.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::{collections::HashMap, path::PathBuf}; use axum::{ extract::{Query, State}, @@ -9,23 +9,20 @@ use axum::{ }; use jupiter::context::Context; -use venus::{ - import_repo::repo::Repo, - monorepo::mr::{CommonResult, MergeOperation}, -}; +use venus::{import_repo::repo::Repo, monorepo::mr::MergeOperation}; -use crate::{ - api_service::import_service::ImportRepoService, +use ceres::{ + api_service::{ + import_api_service::ImportApiService, mono_api_service::MonoApiService, ApiHandler, + }, model::{ create_file::CreateFileInfo, + objects::{ + BlobObjects, CommonResult, LatestCommitInfo, MrInfoItem, TreeBriefInfo, TreeCommitInfo, + }, query::{BlobContentQuery, CodePreviewQuery}, }, }; -use crate::{ - api_service::mono_service::MonorepoService, - model::objects::{BlobObjects, LatestCommitInfo, TreeCommitInfo}, -}; -use crate::{api_service::ApiHandler, model::objects::TreeBriefInfo}; #[derive(Clone)] pub struct ApiServiceState { @@ -33,9 +30,9 @@ pub struct ApiServiceState { } impl ApiServiceState { - pub fn monorepo(&self) -> MonorepoService { - MonorepoService { - storage: self.context.services.mega_storage.clone(), + pub fn monorepo(&self) -> MonoApiService { + MonoApiService { + context: self.context.clone(), } } @@ -51,14 +48,14 @@ impl ApiServiceState { .unwrap() { let repo: Repo = model.into(); - return Box::new(ImportRepoService { - storage: self.context.services.git_db_storage.clone(), + return Box::new(ImportApiService { + context: self.context.clone(), repo, }); } } - Box::new(MonorepoService { - storage: self.context.services.mega_storage.clone(), + Box::new(MonoApiService { + context: self.context.clone(), }) } } @@ -74,7 +71,8 @@ pub fn routers() -> Router { let preview_code = Router::new() .route("/latest-commit", get(get_latest_commit)) .route("/tree-commit-info", get(get_tree_commit_info)) - .route("/tree", get(get_tree_info)); + .route("/tree", get(get_tree_info)) + .route("/mr-list", get(get_mr_list)); Router::new().merge(router_v1).merge(preview_code) } @@ -123,15 +121,15 @@ async fn init(state: State) { async fn create_file( state: State, Json(json): Json, -) -> Result, (StatusCode, String)> { +) -> Result>, (StatusCode, String)> { let res = state - .monorepo() + .api_handler(json.path.clone().into()) + .await .create_monorepo_file(json.clone()) .await; - let res = if res.is_err() { - CommonResult::failed(&res.err().unwrap().to_string()) - } else { - CommonResult::succrss() + let res = match res { + Ok(_) => CommonResult::success(None), + Err(err) => CommonResult::failed(&err.to_string()), }; Ok(Json(res)) } @@ -139,12 +137,11 @@ async fn create_file( async fn merge( state: State, Json(json): Json, -) -> Result, (StatusCode, String)> { +) -> Result>, (StatusCode, String)> { let res = state.monorepo().merge_mr(json.clone()).await; - let res = if res.is_err() { - CommonResult::failed(&res.err().unwrap().to_string()) - } else { - CommonResult::succrss() + let res = match res { + Ok(_) => CommonResult::success(None), + Err(err) => CommonResult::failed(&err.to_string()), }; Ok(Json(res)) } @@ -187,3 +184,16 @@ async fn get_tree_commit_info( .unwrap(); Ok(Json(res)) } + +async fn get_mr_list( + Query(query): Query>, + state: State, +) -> Result>>, (StatusCode, String)> { + let status = query.get("status").unwrap(); + let res = state.monorepo().mr_list(status).await; + let res = match res { + Ok(res) => res, + Err(err) => CommonResult::failed(&err.to_string()), + }; + Ok(Json(res)) +} diff --git a/gateway/src/api_service/import_service.rs b/gateway/src/api_service/import_service.rs deleted file mode 100644 index 8937d072c..000000000 --- a/gateway/src/api_service/import_service.rs +++ /dev/null @@ -1,309 +0,0 @@ -use std::path::{Component, Path}; -use std::{path::PathBuf, sync::Arc}; - -use axum::async_trait; - -use jupiter::storage::git_db_storage::GitDbStorage; -use mercury::errors::GitError; -use mercury::internal::object::commit::Commit; -use mercury::internal::object::tree::Tree; -use venus::import_repo::repo::Repo; - -use crate::api_service::ApiHandler; -use crate::model::objects::{BlobObjects, LatestCommitInfo, TreeBriefInfo, TreeCommitInfo}; - - -#[derive(Clone)] -pub struct ImportRepoService { - pub storage: Arc, - pub repo: Repo, -} - -#[async_trait] -impl ApiHandler for ImportRepoService { - async fn get_blob_as_string(&self, _path: PathBuf, _filename: &str) -> Result { - unimplemented!() - } - - async fn get_latest_commit(&self, path: PathBuf) -> Result { - let (_, tree) = self.search_tree_by_path(&path).await.unwrap(); - let tree_info = self - .storage - .get_tree_by_hash(&self.repo, &tree.id.to_plain_str()) - .await - .unwrap() - .unwrap(); - let commit: Commit = self - .storage - .get_commit_by_hash(&self.repo, &tree_info.commit_id) - .await - .unwrap() - .unwrap() - .into(); - self.convert_commit_to_info(commit) - } - - async fn get_tree_info(&self, _path: PathBuf) -> Result { - unimplemented!() - } - - async fn get_tree_commit_info(&self, _path: PathBuf) -> Result { - unimplemented!() - } -} - -impl ImportRepoService { - /// Searches for a tree and affected parent by path. - /// - /// This function asynchronously searches for a tree by the provided path. - /// - /// # Arguments - /// - /// * `path` - A reference to the path to search. - /// - /// # Returns - /// - /// Returns a tuple containing a vector of parent trees to be updated and - /// the target tree if found, or an error of type `GitError`. - async fn search_tree_by_path(&self, path: &Path) -> Result<(Vec, Tree), GitError> { - let refs = self.storage.get_default_ref(&self.repo).await.unwrap().unwrap(); - - let root_commit = self.storage.get_commit_by_hash(&self.repo, &refs.ref_hash).await.unwrap().unwrap(); - let root_tree: Tree = self - .storage - .get_tree_by_hash(&self.repo, &root_commit.tree) - .await - .unwrap() - .unwrap() - .into(); - let mut search_tree = root_tree.clone(); - let mut update_tree = vec![root_tree]; - - let component_num = path.components().count(); - - for (index, component) in path.components().enumerate() { - // root tree already found - if component != Component::RootDir { - let target_name = component.as_os_str().to_str().unwrap(); - let search_res = search_tree - .tree_items - .iter() - .find(|x| x.name == target_name); - - if let Some(search_res) = search_res { - let hash = search_res.id.to_plain_str(); - let res: Tree = self - .storage - .get_tree_by_hash(&self.repo, &hash) - .await - .unwrap() - .unwrap() - .into(); - search_tree = res.clone(); - if index != component_num - 1 { - update_tree.push(res); - } - } else { - return Err(GitError::ConversionError( - "can't find target parent tree under latest commit".to_string(), - )); - } - } - } - Ok((update_tree, search_tree)) - } - - // pub async fn get_blob_objects( - // &self, - // object_id: &str, - // ) -> Result, (StatusCode, String)> { - // let blob_data = match self.storage.get_blobs_by_repo_id(object_id).await { - // Ok(Some(node)) => { - // if node.object_type == "blob" { - // node.data - // } else { - // return Err((StatusCode::NOT_FOUND, "Blob not found".to_string())); - // } - // } - // _ => return Err((StatusCode::NOT_FOUND, "Blob not found".to_string())), - // }; - - // let row_data = match String::from_utf8(blob_data) { - // Ok(str) => str, - // _ => { - // return Err(( - // StatusCode::INTERNAL_SERVER_ERROR, - // "Can not convert blob to readable txt".to_string(), - // )) - // } - // }; - - // let data = BlobObjects { row_data }; - // Ok(Json(data)) - // } - - // pub async fn get_directories( - // &self, - // query: DirectoryQuery, - // ) -> Result, (StatusCode, String)> { - // let DirectoryQuery { - // object_id, - // repo_path, - // } = query; - // if let Some(obj_id) = object_id { - // self.get_tree_objects(&obj_id, &repo_path).await - // } else { - // let directory = self - // .storage - // .get_directory_by_full_path(&repo_path) - // .await - // .unwrap(); - // match directory { - // Some(dir) => { - // if dir.is_repo { - // // find commit by path - // let commit_id = match self.storage.search_refs(&repo_path).await { - // Ok(refs) if !refs.is_empty() => refs[0].ref_git_id.clone(), - // _ => { - // return Err(( - // StatusCode::NOT_FOUND, - // "repo_path might not valid".to_string(), - // )) - // } - // }; - // // find tree by commit - // let tree_id = match self - // .storage - // .get_commit_by_hash(&commit_id, &repo_path) - // .await - // { - // Ok(Some(commit)) => commit.tree, - // _ => return Err((StatusCode::NOT_FOUND, "Tree not found".to_string())), - // }; - // self.get_tree_objects(&tree_id, &repo_path).await - // } else { - // let dirs = self.storage.get_directory_by_pid(dir.id).await.unwrap(); - // let items = dirs.into_iter().map(|x| x.into()).collect(); - // let data = Directories { items }; - // Ok(Json(data)) - // } - // } - // None => Err(( - // StatusCode::NOT_FOUND, - // "repo_path might not valid".to_string(), - // )), - // } - // } - // } - - // pub async fn get_tree_objects( - // &self, - // object_id: &str, - // repo_path: &str, - // ) -> Result, (StatusCode, String)> { - // let tree_data = match self.storage.get_obj_data_by_id(object_id).await { - // Ok(Some(node)) => { - // if node.object_type == "tree" { - // node.data - // } else { - // return Err((StatusCode::NOT_FOUND, "Tree not found".to_string())); - // } - // } - // _ => return Err((StatusCode::NOT_FOUND, "Tree not found".to_string())), - // }; - - // let tree = Tree::new_from_data(tree_data); - // let child_ids = tree - // .tree_items - // .iter() - // .map(|tree_item| tree_item.id.to_plain_str()) - // .collect(); - - // let child_nodes = self - // .storage - // .get_nodes_by_hashes(child_ids, repo_path) - // .await - // .unwrap(); - - // let mut items: Vec = child_nodes - // .iter() - // .map(|node| Item::from(node.clone())) - // .collect(); - // let related_commit_ids = child_nodes.into_iter().map(|x| x.last_commit).collect(); - // let related_c = self - // .storage - // .get_commit_by_hashes(related_commit_ids, repo_path) - // .await - // .unwrap(); - // let mut related_c_map: HashMap = HashMap::new(); - // for c in related_c { - // related_c_map.insert(c.git_id.clone(), c.into()); - // } - - // for item in &mut items { - // let related_c_id = item.commit_id.clone().unwrap(); - // let commit = related_c_map.get(&related_c_id).unwrap(); - // item.commit_msg = Some(utils::remove_useless_str( - // commit.message.clone(), - // SIGNATURE_END.to_owned(), - // )); - // item.commit_date = Some(commit.committer.timestamp.to_string()); - // } - - // let data = Directories { items }; - // Ok(Json(data)) - // } - - // pub async fn get_objects_data( - // &self, - // object_id: &str, - // repo_path: &str, - // ) -> Result { - // let node = match self.storage.get_node_by_hash(object_id, repo_path).await { - // Ok(Some(node)) => node, - // _ => return Err((StatusCode::NOT_FOUND, "Blob not found".to_string())), - // }; - // let raw_data = match self.storage.get_obj_data_by_id(object_id).await { - // Ok(Some(model)) => model, - // _ => return Err((StatusCode::NOT_FOUND, "Blob not found".to_string())), - // }; - // let file_name = format!("inline; filename=\"{}\"", node.name.unwrap()); - // let res = Response::builder() - // .header("Content-Type", "application/octet-stream") - // .header("Content-Disposition", file_name) - // .body(raw_data.data.into()) - // .unwrap(); - // Ok(res) - // } - - // pub async fn count_object_num( - // &self, - // repo_path: &str, - // ) -> Result, (StatusCode, String)> { - // let query_res = self.storage.count_obj_from_node(repo_path).await.unwrap(); - // let tree = query_res - // .iter() - // .find(|x| x.node_type == "tree") - // .map(|x| x.count) - // .unwrap_or_default() - // .try_into() - // .unwrap(); - // let blob = query_res - // .iter() - // .find(|x| x.node_type == "blob") - // .map(|x| x.count) - // .unwrap_or_default() - // .try_into() - // .unwrap(); - // let commit = self.storage.count_obj_from_commit(repo_path).await.unwrap().try_into().unwrap(); - // let counter = GitTypeCounter { - // commit, - // tree, - // blob, - // tag: 0, - // ofs_delta: 0, - // ref_delta: 0, - // }; - // Ok(Json(counter)) - // } -} diff --git a/gateway/src/api_service/mod.rs b/gateway/src/api_service/mod.rs deleted file mode 100644 index e2379b637..000000000 --- a/gateway/src/api_service/mod.rs +++ /dev/null @@ -1,58 +0,0 @@ -use std::path::PathBuf; - -use axum::async_trait; - -use mercury::{errors::GitError, internal::object::commit::Commit}; - -use crate::model::objects::{ - BlobObjects, LatestCommitInfo, TreeBriefInfo, TreeCommitInfo, UserInfo, -}; - -pub mod import_service; -pub mod mono_service; -pub mod router; - -const SIGNATURE_END: &str = "-----END PGP SIGNATURE-----"; - -#[async_trait] -pub trait ApiHandler: Send + Sync { - async fn get_blob_as_string(&self, path: PathBuf, filename: &str) -> Result; - - async fn get_latest_commit(&self, path: PathBuf) -> Result; - - async fn get_tree_info(&self, path: PathBuf) -> Result; - - async fn get_tree_commit_info(&self, path: PathBuf) -> Result; - - fn convert_commit_to_info(&self, commit: Commit) -> Result { - let message = self.remove_useless_str(commit.message.clone(), SIGNATURE_END.to_owned()); - let committer = UserInfo { - display_name: commit.committer.name, - ..Default::default() - }; - let author = UserInfo { - display_name: commit.author.name, - ..Default::default() - }; - - let res = LatestCommitInfo { - oid: commit.id.to_plain_str(), - date: commit.committer.timestamp.to_string(), - short_message: message, - author, - committer, - status: "success".to_string(), - }; - Ok(res) - } - - fn remove_useless_str(&self, content: String, remove_str: String) -> String { - if let Some(index) = content.find(&remove_str) { - let filtered_text = &content[index + remove_str.len()..].replace('\n', ""); - let truncated_text = filtered_text.chars().take(50).collect::(); - truncated_text.to_owned() - } else { - content - } - } -} diff --git a/gateway/src/api_service/mono_service.rs b/gateway/src/api_service/mono_service.rs deleted file mode 100644 index 7e0e27336..000000000 --- a/gateway/src/api_service/mono_service.rs +++ /dev/null @@ -1,436 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::path::{Component, Path, PathBuf}; -use std::str::FromStr; -use std::sync::Arc; - -use axum::async_trait; - -use callisto::db_enums::ConvType; -use callisto::{mega_blob, mega_commit, mega_tree, raw_blob}; -use common::errors::MegaError; -use jupiter::storage::batch_save_model; -use jupiter::storage::mega_storage::MegaStorage; -use mercury::errors::GitError; -use mercury::hash::SHA1; -use mercury::internal::object::blob::Blob; -use mercury::internal::object::commit::Commit; -use mercury::internal::object::tree::{Tree, TreeItem, TreeItemMode}; -use venus::monorepo::converter; -use venus::monorepo::mr::MergeOperation; - -use crate::api_service::{ApiHandler, SIGNATURE_END}; -use crate::model::create_file::CreateFileInfo; -use crate::model::objects::{ - BlobObjects, LatestCommitInfo, TreeBriefInfo, TreeBriefItem, TreeCommitInfo, TreeCommitItem, -}; - -#[derive(Clone)] -pub struct MonorepoService { - pub storage: Arc, -} - -#[async_trait] -impl ApiHandler for MonorepoService { - async fn get_blob_as_string( - &self, - path: PathBuf, - filename: &str, - ) -> Result { - let (_, tree) = self.search_tree_by_path(&path).await.unwrap(); - let mut plain_text = String::new(); - if let Some(item) = tree.tree_items.into_iter().find(|x| x.name == filename) { - plain_text = match self - .storage - .get_raw_blob_by_hash(&item.id.to_plain_str()) - .await - { - Ok(Some(model)) => String::from_utf8(model.data.unwrap()).unwrap(), - _ => String::new(), - }; - } - Ok(BlobObjects { plain_text }) - } - - async fn get_latest_commit(&self, path: PathBuf) -> Result { - let (_, tree) = self.search_tree_by_path(&path).await.unwrap(); - let tree_info = self - .storage - .get_tree_by_hash(&tree.id.to_plain_str()) - .await - .unwrap() - .unwrap(); - let commit: Commit = self - .storage - .get_commit_by_hash(&tree_info.commit_id) - .await - .unwrap() - .unwrap() - .into(); - self.convert_commit_to_info(commit) - } - - async fn get_tree_info(&self, path: PathBuf) -> Result { - match self.search_tree_by_path(&path).await { - Ok((_, tree)) => { - let mut items = Vec::new(); - for item in tree.tree_items { - let mut info: TreeBriefItem = item.clone().into(); - path.join(item.name) - .to_str() - .unwrap() - .clone_into(&mut info.path); - items.push(info); - } - Ok(TreeBriefInfo { - total_count: items.len(), - items, - }) - } - Err(_) => Ok(TreeBriefInfo { - total_count: 0, - items: Vec::new(), - }), - } - } - - async fn get_tree_commit_info(&self, path: PathBuf) -> Result { - match self.search_tree_by_path(&path).await { - Ok((_, tree)) => { - // let mut commit_map = HashMap::new(); - let mut item_to_commit = HashMap::new(); - - let trees = self - .storage - .get_trees_by_hashes( - tree.tree_items - .iter() - .filter(|x| x.mode == TreeItemMode::Tree) - .map(|x| x.id.to_plain_str()) - .collect(), - ) - .await - .unwrap(); - - for tree in trees { - item_to_commit.insert(tree.tree_id, tree.commit_id); - } - - let blobs = self - .storage - .get_mega_blobs_by_hashes( - tree.tree_items - .iter() - .filter(|x| x.mode == TreeItemMode::Blob) - .map(|x| x.id.to_plain_str()) - .collect(), - ) - .await - .unwrap(); - - for blob in blobs { - item_to_commit.insert(blob.blob_id, blob.commit_id); - } - - let mut items = Vec::new(); - let commit_ids: HashSet = item_to_commit.values().cloned().collect(); - - let commits = self - .storage - .get_commits_by_hashes(&commit_ids.into_iter().collect()) - .await - .unwrap(); - let commit_map: HashMap = commits - .into_iter() - .map(|x| (x.commit_id.clone(), x)) - .collect(); - - for item in tree.tree_items { - let mut info: TreeCommitItem = item.clone().into(); - if let Some(commit_id) = item_to_commit.get(&item.id.to_plain_str()) { - if let Some(model) = commit_map.get(commit_id) { - let commit: Commit = model.clone().into(); - info.oid = commit.id.to_plain_str(); - info.message = self.remove_useless_str( - commit.message.clone(), - SIGNATURE_END.to_owned(), - ); - info.date = commit.committer.timestamp.to_string(); - } else { - tracing::error!("failed fecth commit: {}", commit_id) - } - } - items.push(info); - } - Ok(TreeCommitInfo { - total_count: items.len(), - items, - }) - } - Err(_) => Ok(TreeCommitInfo { - total_count: 0, - items: Vec::new(), - }), - } - } -} - -impl MonorepoService { - pub async fn init_monorepo(&self) { - self.storage.init_monorepo().await - } - - pub async fn create_monorepo_file(&self, file_info: CreateFileInfo) -> Result<(), GitError> { - let path = PathBuf::from(file_info.path); - let mut save_trees = vec![]; - - let (update_trees, search_tree) = self.search_tree_by_path(&path).await.unwrap(); - let mut t_items = search_tree.tree_items; - - let new_item = if file_info.is_directory { - if t_items - .iter() - .any(|x| x.mode == TreeItemMode::Tree && x.name == file_info.name) - { - return Err(GitError::CustomError("Duplicate name".to_string())); - } - let blob = converter::generate_git_keep_with_timestamp(); - let tree_item = TreeItem { - mode: TreeItemMode::Blob, - id: blob.id, - name: String::from(".gitkeep"), - }; - let child_tree = Tree::from_tree_items(vec![tree_item]).unwrap(); - save_trees.push(child_tree.clone()); - TreeItem { - mode: TreeItemMode::Tree, - id: child_tree.id, - name: file_info.name.clone(), - } - } else { - let blob = Blob::from_content(&file_info.content.unwrap()); - let mega_blob: mega_blob::Model = (&blob).into(); - let mega_blob: mega_blob::ActiveModel = mega_blob.into(); - let raw_blob: raw_blob::Model = blob.clone().into(); - let raw_blob: raw_blob::ActiveModel = raw_blob.into(); - batch_save_model(self.storage.get_connection(), vec![mega_blob]) - .await - .unwrap(); - batch_save_model(self.storage.get_connection(), vec![raw_blob]) - .await - .unwrap(); - TreeItem { - mode: TreeItemMode::Blob, - id: blob.id, - name: file_info.name.clone(), - } - }; - t_items.push(new_item); - let p_tree = Tree::from_tree_items(t_items).unwrap(); - - let refs = self.storage.get_ref("/").await.unwrap().unwrap(); - let commit = Commit::from_tree_id( - p_tree.id, - vec![SHA1::from_str(&refs.ref_commit_hash).unwrap()], - &format!("create file {} commit", file_info.name), - ); - - let commit_id = self - .update_parent_tree(path, update_trees, commit) - .await - .unwrap(); - save_trees.push(p_tree); - - for save_t in save_trees { - let mut tree_model: mega_tree::Model = save_t.into(); - tree_model.commit_id.clone_from(&commit_id); - let tree_model: mega_tree::ActiveModel = tree_model.into(); - batch_save_model(self.storage.get_connection(), vec![tree_model]) - .await - .unwrap(); - } - Ok(()) - } - - pub async fn merge_mr(&self, op: MergeOperation) -> Result<(), MegaError> { - if let Some(mut mr) = self.storage.get_open_mr_by_id(op.mr_id).await.unwrap() { - let refs = self.storage.get_ref(&mr.path).await.unwrap().unwrap(); - - if mr.from_hash == refs.ref_commit_hash { - // update mr - mr.merge(op.comment); - self.storage.update_mr(mr.clone()).await.unwrap(); - - let commit: Commit = self - .storage - .get_commit_by_hash(&mr.to_hash) - .await - .unwrap() - .unwrap() - .into(); - - // add conversation - self.storage - .add_mr_conversation(mr.id, 0, ConvType::Merged) - .await - .unwrap(); - if mr.path != "/" { - let path = PathBuf::from(mr.path.clone()); - // beacuse only parent tree is needed so we skip current directory - let (tree_vec, _) = self - .search_tree_by_path(path.parent().unwrap()) - .await - .unwrap(); - self.update_parent_tree(path, tree_vec, commit) - .await - .unwrap(); - // remove refs start with path - self.storage.remove_refs(&mr.path).await.unwrap(); - // TODO: self.clean_dangling_commits().await; - } - } else { - return Err(MegaError::with_message("ref hash conflict")); - } - } else { - return Err(MegaError::with_message("Invalid mr id")); - } - Ok(()) - } - - /// Searches for a tree and affected parent by path. - /// - /// This function asynchronously searches for a tree by the provided path. - /// - /// # Arguments - /// - /// * `path` - A reference to the path to search. - /// - /// # Returns - /// - /// Returns a tuple containing a vector of parent trees to be updated and - /// the target tree if found, or an error of type `GitError`. - async fn search_tree_by_path(&self, path: &Path) -> Result<(Vec, Tree), GitError> { - let refs = self.storage.get_ref("/").await.unwrap().unwrap(); - - let root_tree: Tree = self - .storage - .get_tree_by_hash(&refs.ref_tree_hash) - .await - .unwrap() - .unwrap() - .into(); - let mut search_tree = root_tree.clone(); - let mut update_tree = vec![root_tree]; - - let component_num = path.components().count(); - - for (index, component) in path.components().enumerate() { - // root tree already found - if component != Component::RootDir { - let target_name = component.as_os_str().to_str().unwrap(); - let search_res = search_tree - .tree_items - .iter() - .find(|x| x.name == target_name); - - if let Some(search_res) = search_res { - let hash = search_res.id.to_plain_str(); - let res: Tree = self - .storage - .get_tree_by_hash(&hash) - .await - .unwrap() - .unwrap() - .into(); - search_tree = res.clone(); - // skip last component - if index != component_num - 1 { - update_tree.push(res); - } - } else { - return Err(GitError::ConversionError( - "can't find target parent tree under latest commit".to_string(), - )); - } - } - } - Ok((update_tree, search_tree)) - } - - async fn update_parent_tree( - &self, - mut path: PathBuf, - mut tree_vec: Vec, - commit: Commit, - ) -> Result { - let mut save_trees = Vec::new(); - let mut p_commit_id = String::new(); - - let mut target_hash = commit.tree_id; - - while let Some(mut tree) = tree_vec.pop() { - let cloned_path = path.clone(); - let name = cloned_path.file_name().unwrap().to_str().unwrap(); - path.pop(); - - let index = tree.tree_items.iter().position(|x| x.name == name).unwrap(); - tree.tree_items[index].id = target_hash; - let new_tree = Tree::from_tree_items(tree.tree_items).unwrap(); - target_hash = new_tree.id; - - let model: mega_tree::Model = new_tree.into(); - save_trees.push(model); - - let p_ref = self.storage.get_ref(path.to_str().unwrap()).await.unwrap(); - if let Some(mut p_ref) = p_ref { - if path == Path::new("/") { - let p_commit = Commit::new( - commit.author.clone(), - commit.committer.clone(), - target_hash, - vec![SHA1::from_str(&p_ref.ref_commit_hash).unwrap()], - &commit.message, - ); - p_commit_id = p_commit.id.to_plain_str(); - // update p_ref - p_ref.ref_commit_hash = p_commit.id.to_plain_str(); - p_ref.ref_tree_hash = target_hash.to_plain_str(); - self.storage.update_ref(p_ref).await.unwrap(); - self.storage - .save_mega_commits(vec![p_commit]) - .await - .unwrap(); - } else { - self.storage.remove_ref(p_ref).await.unwrap(); - } - } - } - let save_trees: Vec = save_trees - .into_iter() - .map(|mut x| { - p_commit_id.clone_into(&mut x.commit_id); - x.into() - }) - .collect(); - - batch_save_model(self.storage.get_connection(), save_trees) - .await - .unwrap(); - Ok(p_commit_id) - } -} - -#[cfg(test)] -mod test { - use std::path::PathBuf; - - #[test] - pub fn test() { - let mut full_path = PathBuf::from("/project/rust/mega"); - for _ in 0..3 { - let cloned_path = full_path.clone(); // Clone full_path - let name = cloned_path.file_name().unwrap().to_str().unwrap(); - full_path.pop(); - println!("name: {}, path: {:?}", name, full_path); - } - } -} diff --git a/gateway/src/https_server.rs b/gateway/src/https_server.rs index d6443fdbd..3c9654fe1 100644 --- a/gateway/src/https_server.rs +++ b/gateway/src/https_server.rs @@ -28,9 +28,9 @@ use common::model::{CommonOptions, GetParams}; use jupiter::context::Context; use jupiter::raw_storage::local_storage::LocalStorage; -use crate::api_service::router::ApiServiceState; +use crate::api_router::{self, ApiServiceState}; +use crate::lfs; use crate::relay_server::run_relay_server; -use crate::{api_service, lfs}; #[derive(Args, Clone, Debug)] pub struct HttpOptions { @@ -137,10 +137,7 @@ pub async fn app(config: Config, host: String, port: u16) -> Router { // add TraceLayer for log record // add CorsLayer to add cors header Router::new() - .nest( - "/api/v1", - api_service::router::routers().with_state(api_state), - ) + .nest("/api/v1", api_router::routers().with_state(api_state)) .route( "/*path", get(get_method_router) diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index f7b616955..625ea25ea 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -1,9 +1,8 @@ -mod api_service; +mod api_router; mod git_protocol; pub mod https_server; pub mod init; mod lfs; -mod model; pub mod relay_server; pub mod ssh_server; diff --git a/gateway/src/relay_server.rs b/gateway/src/relay_server.rs index f95b2a28c..a12c2d4cd 100644 --- a/gateway/src/relay_server.rs +++ b/gateway/src/relay_server.rs @@ -16,8 +16,7 @@ use tower_http::cors::{Any, CorsLayer}; use tower_http::decompression::RequestDecompressionLayer; use tower_http::trace::TraceLayer; -use crate::api_service; -use crate::api_service::router::ApiServiceState; +use crate::api_router::{self, ApiServiceState}; pub async fn run_relay_server(config: Config, host: String, port: u16) { let app = app(config.clone(), host.clone(), port).await; @@ -66,7 +65,7 @@ pub async fn app(config: Config, host: String, port: u16) -> Router { Router::new() .nest( "/api/v1", - api_service::router::routers().with_state(api_state), + api_router::routers().with_state(api_state), ) .route( "/*path", diff --git a/jupiter/callisto/src/db_enums.rs b/jupiter/callisto/src/db_enums.rs index 559c227a4..2f90c9d9e 100644 --- a/jupiter/callisto/src/db_enums.rs +++ b/jupiter/callisto/src/db_enums.rs @@ -1,4 +1,5 @@ use core::fmt; +use std::fmt::Display; use sea_orm::{DeriveActiveEnum, EnumIter}; @@ -35,6 +36,17 @@ pub enum MergeStatus { Closed, } +impl Display for MergeStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match self { + MergeStatus::Open => "open", + MergeStatus::Merged => "merged", + MergeStatus::Closed => "closed", + }; + write!(f, "{}", s) + } +} + #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] #[sea_orm(rs_type = "String", db_type = "String(Some(1))")] pub enum RefType { diff --git a/jupiter/src/storage/git_db_storage.rs b/jupiter/src/storage/git_db_storage.rs index 9431f6991..8f13949f8 100644 --- a/jupiter/src/storage/git_db_storage.rs +++ b/jupiter/src/storage/git_db_storage.rs @@ -3,9 +3,10 @@ use std::sync::{Arc, Mutex}; use async_trait::async_trait; use futures::Stream; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use sea_orm::sea_query::Expr; use sea_orm::{ - ActiveModelTrait, ColumnTrait, DatabaseConnection, DbErr, EntityTrait, IntoActiveModel, - QueryFilter, Set, + ActiveModelTrait, ColumnTrait, DatabaseConnection, DbBackend, DbErr, EntityTrait, + IntoActiveModel, QueryFilter, QueryTrait, Set, }; use sea_orm::{PaginatorTrait, QueryOrder}; @@ -179,10 +180,10 @@ impl GitDbStorage { &self, repo_path: &str, ) -> Result, MegaError> { - let result = git_repo::Entity::find() - .filter(git_repo::Column::RepoPath.eq(repo_path)) - .one(self.get_connection()) - .await?; + let query = git_repo::Entity::find() + .filter(Expr::cust(format!("'{}' LIKE repo_path || '%'", repo_path))); + tracing::debug!("{}", query.build(DbBackend::Postgres).to_string()); + let result = query.one(self.get_connection()).await?; Ok(result) } @@ -293,6 +294,19 @@ impl GitDbStorage { .unwrap()) } + pub async fn get_blobs_by_hashes( + &self, + repo: &Repo, + hashes: Vec, + ) -> Result, MegaError> { + Ok(git_blob::Entity::find() + .filter(git_blob::Column::RepoId.eq(repo.repo_id)) + .filter(git_blob::Column::BlobId.is_in(hashes)) + .all(self.get_connection()) + .await + .unwrap()) + } + pub async fn get_raw_blobs( &self, hashes: Vec, diff --git a/jupiter/src/storage/mega_storage.rs b/jupiter/src/storage/mega_storage.rs index a49b54ad0..093893a73 100644 --- a/jupiter/src/storage/mega_storage.rs +++ b/jupiter/src/storage/mega_storage.rs @@ -121,6 +121,18 @@ impl MegaStorage { Ok(None) } + pub async fn get_mr_by_status( + &self, + status: Vec, + ) -> Result, MegaError> { + let model = mega_mr::Entity::find() + .filter(mega_mr::Column::Status.is_in(status)) + .all(self.get_connection()) + .await + .unwrap(); + Ok(model) + } + pub async fn get_open_mr_by_id(&self, mr_id: i64) -> Result, MegaError> { let model = mega_mr::Entity::find_by_id(mr_id) .filter(mega_mr::Column::Status.eq(MergeStatus::Open)) diff --git a/mercury/src/internal/object/commit.rs b/mercury/src/internal/object/commit.rs index 16acb5b48..23d4f518d 100644 --- a/mercury/src/internal/object/commit.rs +++ b/mercury/src/internal/object/commit.rs @@ -103,6 +103,19 @@ impl Commit { .unwrap(); Commit::new(author, committer, tree_id, parent_commit_ids, message) } + + pub fn format_message(&self) -> String { + let mut has_signature = false; + for line in self.message.lines() { + if has_signature && !line.trim().is_empty() { + return line.to_owned(); + } + if line.contains("-----END PGP SIGNATURE-----") { + has_signature = true; + } + } + self.message.lines().next().unwrap().to_owned() + } } impl ObjectTrait for Commit { @@ -151,7 +164,7 @@ impl ObjectTrait for Commit { let message = unsafe { String::from_utf8_unchecked(commit[commit.find_byte(0x0a).unwrap() + 1..].to_vec()) }; - + Ok(Commit { id: hash, tree_id, diff --git a/moon/package.json b/moon/package.json index 2fa95dd15..3c52f6f33 100644 --- a/moon/package.json +++ b/moon/package.json @@ -16,6 +16,7 @@ "@tailwindcss/forms": "^0.5.7", "antd": "^5.18.3", "axios": "^1.7.2", + "date-fns": "^3.6.0", "github-markdown-css": "^5.6.1", "highlight.js": "^11.9.0", "lexical": "^0.16.0", diff --git a/moon/src/pages/index.js b/moon/src/pages/index.js index e8382cd1d..884d75905 100644 --- a/moon/src/pages/index.js +++ b/moon/src/pages/index.js @@ -11,6 +11,7 @@ import Markdown from 'react-markdown'; import Bottombar from '../components/Bottombar'; import TopNavbar from '../components/TopNavbar'; import '../styles/index.css'; +import { formatDistance, fromUnixTime } from 'date-fns' const HomePage = ({ rootDirectory, directory, readmeContent, fileContent, TreeData }) => { @@ -311,7 +312,9 @@ const HomePage = ({ rootDirectory, directory, readmeContent, fileContent, TreeDa )} {project.message} - {project.date} + + {formatDistance(fromUnixTime(project.date), new Date(), { addSuffix: true })} + ))} diff --git a/venus/src/monorepo/mr.rs b/venus/src/monorepo/mr.rs index 65b36a263..6c7889a15 100644 --- a/venus/src/monorepo/mr.rs +++ b/venus/src/monorepo/mr.rs @@ -76,24 +76,3 @@ pub struct MergeOperation { pub mr_id: i64, } -#[derive(PartialEq, Eq, Debug, Clone, Default, Serialize, Deserialize)] - -pub struct CommonResult { - pub result: bool, - pub err_message: String, -} - -impl CommonResult { - pub fn succrss() -> Self { - CommonResult { - result: true, - err_message: "".to_owned(), - } - } - pub fn failed(err_message: &str) -> Self { - CommonResult { - result: false, - err_message: err_message.to_string(), - } - } -}