diff --git a/docs/images/Scorpio_Commit.png b/docs/images/Scorpio_Commit.png new file mode 100644 index 000000000..21ad0e358 Binary files /dev/null and b/docs/images/Scorpio_Commit.png differ diff --git a/scorpio/Cargo.toml b/scorpio/Cargo.toml index 171c8436e..cb7715e9f 100644 --- a/scorpio/Cargo.toml +++ b/scorpio/Cargo.toml @@ -50,7 +50,7 @@ thiserror = "2.0.12" crossbeam = "0.8.4" fs_extra = "1.2" dashmap = "6.1.0" - +chrono = { workspace = true } [features] diff --git a/scorpio/doc/api.md b/scorpio/doc/api.md index 4ae042a34..27efb3b50 100644 --- a/scorpio/doc/api.md +++ b/scorpio/doc/api.md @@ -70,7 +70,7 @@ This server provides endpoints to manage file system mounting and configuration **Response (JSON)**: ```json { - "status": "Success", + "status": "Success", "message": "Directory unmounted successfully" } ``` @@ -85,7 +85,7 @@ This server provides endpoints to manage file system mounting and configuration { "status": "Success", "config": { - "mega_url": "http://example.com", + "mega_url": "http://example.com", "mount_path": "path/to/mount", "store_path": "path/to/store" } @@ -100,7 +100,7 @@ This server provides endpoints to manage file system mounting and configuration **Request Body (JSON)**: ```json { - "mega_url": "http://example.com", + "mega_url": "http://example.com", "mount_path": "new/mount/path", "store_path": "new/store/path" } @@ -111,54 +111,92 @@ This server provides endpoints to manage file system mounting and configuration { "status": "Success", "config": { - "mega_url": "http://example.com", + "mega_url": "http://example.com", "mount_path": "new/mount/path", "store_path": "new/store/path" } } ``` -### 6. **Git Status** +### 6. **Git Add** +**URL**: `/api/git/add` +**Method**: POST +**Description**: Add added, deleted, and modified files to the temporary storage area. + +**Request Body (JSON)**: +```json +{ + "mono_path": "path/to/add", +} +``` + +**Response (JSON)**: +```json +{ + "status_code": 200, +} +``` + +### 7. **Git Status** **URL**: `/api/git/status` **Method**: GET **Description**: Retrieves the status of the Git repository. **Query Parameters**: -- `filter` (optional): Filter specific output based on the input string. +- `path` : The target path whose status needs to be checked. **Response (JSON)**: ```json { - "status_code": 200, - "output": "Git status output" + "status": "Success", + "mono_path": "target/path", + "upper_path": "upper/folder/of/mono_path", + "lower_path": "lower/folder/of/mono_path", + "message": "Status of mono_path", } ``` -### 7. **Git Commit** -**URL**: `/api/git/commit` -**Method**: POST +### 8. **Git Commit** +**URL**: `/api/git/commit` +**Method**: POST **Description**: Commits changes in the Git repository with a given message. **Request Body (JSON)**: ```json { - "message": "Commit message" + "mono_path": "commit/path", + "message": "Commit message", } ``` **Response (JSON)**: ```json { - "status_code": 200, - "output": "Commit successful" + "status": "Success", + "commit": { + "id": "The Commit hash", + "tree_id": "New hash of root tree", + "parent_commit_ids": "The hash of last version", + "author": "The author of this repository", + committer: "The committer of current Commit", + message: "Commit message", + }, + "msg": "Detailed information", } ``` -### 8. **Git Push** -**URL**: `/api/git/push` -**Method**: POST +### 9. **Git Push** +**URL**: `/api/git/push` +**Method**: POST **Description**: Pushes committed changes to the remote repository. +**Request Body (JSON)**: +```json +{ + "mono_path": "push/path", +} +``` + **Response (JSON)**: ```json { @@ -167,6 +205,25 @@ This server provides endpoints to manage file system mounting and configuration } ``` +### 10. **Git Reset** +**URL**: `/api/git/reset` +**Method**: POST +**Description**: Reset the repository, undoing all modifications. + +**Request Body (JSON)**: +```json +{ + "path": "reset/path", +} +``` + +**Response (JSON)**: +```json +{ + "status_code": 200, +} +``` + ## Data Structures ### MountRequest ```rust @@ -210,10 +267,58 @@ struct ConfigRequest { } ``` +### AddReq +```rust +struct AddReq { + mono_path: String, +} +``` + +### GitStatus +```rust +struct GitStatus { + status: String, + mono_path: String, + upper_path: String, + lower_path: String, + message: String, +} +``` + ### GitStatusParams ```rust struct GitStatusParams { - filter: Option, + path: String, +} +``` + +### CommitPayload +```rust +struct CommitPayload { + mono_path: String, + message: String, } ``` +### CommitResp +```rust +struct CommitResp { + status: String, + commit: Option, + msg: String, +} +``` + +### PushRequest +```rust +struct PushRequest { + mono_path: String, +} +``` + +### ResetReq +```rust +struct ResetReq { + path: String, +} +``` \ No newline at end of file diff --git a/scorpio/src/daemon/git.rs b/scorpio/src/daemon/git.rs index b3c11ace7..8f8b1c891 100644 --- a/scorpio/src/daemon/git.rs +++ b/scorpio/src/daemon/git.rs @@ -24,6 +24,7 @@ pub(super) struct GitStatusParams { path: String, } +/// Handles the git status request. pub(super) async fn git_status_handler( Query(params): Query, State(state): State, @@ -76,6 +77,8 @@ pub(super) struct CommitResp { commit: Option, msg: String, } + +/// Handles the git commit request. #[axum::debug_handler] pub(super) async fn git_commit_handler( State(state): State, @@ -106,6 +109,7 @@ pub(super) struct AddReq { mono_path: String, } +/// Handles the git add request. pub(super) async fn git_add_handler( State(state): State, axum::Json(req): axum::Json, @@ -127,6 +131,7 @@ pub(super) struct ResetReq { path: String, } +/// Handles the git reset request. pub(super) async fn git_reset_handler( State(state): State, axum::Json(req): axum::Json, @@ -159,9 +164,10 @@ pub(super) async fn git_reset_handler( #[derive(serde::Deserialize)] pub(super) struct PushRequest { - monopath: String, + mono_path: String, } +/// Handles the git push request. pub(super) async fn git_push_handler( State(state): State, axum::Json(payload): axum::Json, @@ -170,7 +176,7 @@ pub(super) async fn git_push_handler( .manager .lock() .await - .push_commit(&payload.monopath) + .push_commit(&payload.mono_path) .await { Ok(response) => { diff --git a/scorpio/src/manager/mod.rs b/scorpio/src/manager/mod.rs index ea60d7e83..459836c6a 100644 --- a/scorpio/src/manager/mod.rs +++ b/scorpio/src/manager/mod.rs @@ -52,6 +52,9 @@ impl ScorpioManager { fs::write(file_path, content)?; Ok(()) } + + /// Integrate the temporary storage area files, merge + /// them into a Tree object and output Commit pub async fn mono_commit( &self, mono_path: String, @@ -151,6 +154,7 @@ impl ScorpioManager { Err(Box::from("WorkDir not found")) } + /// Pushes a commit to the remote mono repository. pub async fn push_commit( &self, mono_path: &str, @@ -162,48 +166,13 @@ impl ScorpioManager { let temp_store_area = TempStoreArea::new(&modified_path)?; println!("OK1"); let base_url = config::base_url(); - let url = format!("{}/{}/git-receive-pack", base_url, mono_path); + let url = format!("{}/{}.git/git-receive-pack", base_url, mono_path); println!("START"); - let res = push::push(&work_path, &url, &temp_store_area.index_db).await?; + let res = push::push_core(&work_path, &url, &temp_store_area.index_db).await?; println!("END"); Ok(res) } - /* - pub async fn push_commit( - &self, - mono_path: &str, - ) -> Result> { - let work_dir = self.select_work(mono_path)?; // TODO : deal with error. - let store_path = config::store_path(); - let mut path = store_path.to_string(); - path.push_str(&work_dir.hash); - path.push_str("commit"); - - // check path is exist - if !tokio::fs::try_exists(&path).await.unwrap_or(false) { - eprintln!("Path does not exist: {}", path); - return Err(Box::new(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!("Path does not exist: {}", path), - ))); - } - // read the file as the body to send - let commit_data = tokio::fs::read(&path).await?; - - // Send Commit data to remote mono. - let base_url = config::base_url(); - let url = format!("{}/{}/git-receive-pack", base_url, mono_path); - let client = reqwest::Client::new(); - client - .post(&url) - .header("Content-Type", "application/x-git-receive-pack-request") - .body(Bytes::from(commit_data)) - .send() - .await - .map_err(|e| Box::new(e) as Box) - } - */ pub fn check_before_mount(&self, mono_path: &str) -> Result<(), String> { for work in &self.works { @@ -227,6 +196,7 @@ impl ScorpioManager { } } + /// Adds a mono file to the Scorpio manager's workspace. pub async fn mono_add(&self, mono_path: &str) -> Result<(), Box> { // The OS path cannot be used, and should be mapped from // the FUSE system to the path under Upper. diff --git a/scorpio/src/manager/push.rs b/scorpio/src/manager/push.rs index a1ad7011d..1a28b472d 100644 --- a/scorpio/src/manager/push.rs +++ b/scorpio/src/manager/push.rs @@ -1,7 +1,8 @@ use bytes::BytesMut; use ceres::protocol::smart::add_pkt_line_string; +use chrono::DateTime; use regex::Regex; -use reqwest::{header::CONTENT_TYPE, Client, Response, Url}; +use reqwest::{header::CONTENT_TYPE, ClientBuilder, Response, Url}; use std::io::{Error, ErrorKind}; use std::{path::Path, str::FromStr}; use tokio::sync::mpsc; @@ -21,6 +22,8 @@ use mercury::{ }, }; +/// Use Git style to package Commit, Tree objects, and +/// Blobs objects pub async fn pack(commit: Commit, trees: Vec, blob: Vec) -> Vec { let len = trees.len() + blob.len() + 1; // let (tx, rx) = mpsc::channel::(); @@ -46,10 +49,12 @@ pub async fn pack(commit: Commit, trees: Vec, blob: Vec) -> Vec pack_data } +/// Convert a String to a SHA1 hash fn string_to_sha(hash: &str) -> std::io::Result { SHA1::from_str(hash).map_err(|e| Error::new(ErrorKind::InvalidData, e)) } +/// Extract commit information from a commit file fn extract_commit_from_bytes(commitpath: &Path) -> std::io::Result { let commit_string = std::fs::read_to_string(commitpath)?; // This function uses regular expressions to extract the @@ -66,7 +71,7 @@ fn extract_commit_from_bytes(commitpath: &Path) -> std::io::Result { (?P[a-zA-Z0-9_-]+) [[:blank:]]+ <(?P[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[[:alpha:]]{2,})>\n - .*\n + Date: (?P.*)\n \n # committer @@ -74,7 +79,7 @@ fn extract_commit_from_bytes(commitpath: &Path) -> std::io::Result { (?P[a-zA-Z0-9_-]+) [[:blank:]]+ <(?P[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[[:alpha:]]{2,})>\n - .*\n + Date: (?P.*)\n \n # commit message @@ -95,13 +100,53 @@ fn extract_commit_from_bytes(commitpath: &Path) -> std::io::Result { let parent_hash = extract_data("parent_hash"); let author = extract_data("author"); let author_email = extract_data("author_email"); + let author_time = extract_data("author_time"); let committer = extract_data("committer"); let committer_email = extract_data("committer_email"); + let commit_time = extract_data("commit_time"); let message = extract_data("message"); - let author_sign = Signature::new(SignatureType::Author, author, author_email); - let committer_sign = - Signature::new(SignatureType::Committer, committer, committer_email); + println!("author_time = {}", author_time); + println!("commit_time = {}", commit_time); + + // This part of the code is to prevent the timestamp + // change from causing the Commit Hash to change + let author_time = DateTime::parse_from_str(&author_time, "%Y-%m-%d %H:%M:%S UTC %z") + .map_err(|e| Error::new(ErrorKind::InvalidData, e))? + .to_utc(); + let commit_time = DateTime::parse_from_str(&commit_time, "%Y-%m-%d %H:%M:%S UTC %z") + .map_err(|e| Error::new(ErrorKind::InvalidData, e))? + .to_utc(); + /* + println!("author_time = {:?}", author_time); + println!("commit_time = {:?}", commit_time); + println!("now = {:?}", chrono::Utc::now()); + println!("now = {}", chrono::Utc::now()); + */ + let author_sign = Signature::from_data( + format!( + "{} {} <{}> {} +0800", + SignatureType::Author, + author, + author_email, + author_time.timestamp() + ) + .to_string() + .into_bytes(), + ) + .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; + let committer_sign = Signature::from_data( + format!( + "{} {} <{}> {} +0800", + SignatureType::Author, + committer, + committer_email, + commit_time.to_utc().timestamp() + ) + .to_string() + .into_bytes(), + ) + .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; Ok(Commit::new( author_sign, @@ -118,11 +163,16 @@ fn extract_commit_from_bytes(commitpath: &Path) -> std::io::Result { } } -pub async fn push(work_path: &Path, url: &str, index_db: &sled::Db) -> std::io::Result { +/// Push the commit to the remote repository +pub async fn push_core( + work_path: &Path, + url: &str, + index_db: &sled::Db, +) -> std::io::Result { let new_dbpath = work_path.join("new_tree.db"); let commitpath = work_path.join("commit"); - println!("PART1"); + println!("\x1b[34m[PART1]\x1b[0m"); // check path is exist if !tokio::fs::try_exists(&commitpath).await.unwrap_or(false) { eprintln!("Path does not exist: {}", commitpath.display()); @@ -133,11 +183,14 @@ pub async fn push(work_path: &Path, url: &str, index_db: &sled::Db) -> std::io:: } // read the file as the body to send let commit = extract_commit_from_bytes(&commitpath)?; + /* println!("commit id = {}", commit.id._to_string()); println!("commit tree_id = {}", commit.tree_id._to_string()); + println!("commit = {:?}", commit); + */ - println!("PART2"); - println!("new_dbpath = {}", new_dbpath.display()); + println!("\x1b[34m[PART2]\x1b[0m"); + // println!("new_dbpath = {}", new_dbpath.display()); let new_tree_db = sled::open(new_dbpath)?; println!("Fin"); let hashmap = new_tree_db.db_tree_list()?; @@ -150,7 +203,7 @@ pub async fn push(work_path: &Path, url: &str, index_db: &sled::Db) -> std::io:: let remote_hash = string_to_sha(work_path.file_name().unwrap().to_str().unwrap())?; - println!("PART3"); + println!("\x1b[34m[PART3]\x1b[0m"); let mut data = BytesMut::new(); add_pkt_line_string( &mut data, @@ -160,25 +213,85 @@ pub async fn push(work_path: &Path, url: &str, index_db: &sled::Db) -> std::io:: ), ); data.extend_from_slice(b"0000"); - tracing::debug!("{:?}", data); + + // tracing::debug!("{:?}", data); data.extend(pack(commit, trees, blobs).await); - println!("PART4"); - let request = Client::new(); + println!("\x1b[34m[PART4]\x1b[0m"); + let request = ClientBuilder::new().build().unwrap(); + // println!("data = {:?}", data.clone().freeze()); + // println!("url = {url}"); let url = Url::from_str(url).unwrap(); + let res = request - .post(url.join("git-receive-pack").unwrap()) + .post(url) .header(CONTENT_TYPE, "application/x-git-receive-pack-request") .body(data.freeze()); - let res = res.send().await.unwrap(); + println!("send_pack request: {:?}", res); + + let res = match res.send().await { + Ok(response) => response, + Err(e) => { + eprintln!("\x1b[31mFailed to send request: {:?}\x1b[0m", e); + return Err(std::io::Error::other( + "Failed to send request", + )); + } + }; if res.status() != 200 { eprintln!("status code: {}", res.status()); } else { - println!("[scorpio]:push seccess!") + println!("[scorpio]:\x1b[32mpush seccess!\x1b[0m"); } Ok(res) } + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[tokio::test] + async fn test_extract_commit_from_bytes() { + let tmp_path = PathBuf::from("/tmp/tmp13384"); + let commit_data = r###"tree: 73a486aae34e5c2afe7bc164a54569b98133af4d +parent: 54f28ffd6c8aece72eb9138bfaa44ad0dacfb2ff +author MEGA +Date: 2025-06-06 03:33:05 UTC +0800 + +committer MEGA +Date: 2025-06-06 03:33:05 UTC +0800 + +Added some tmp files"###; + std::fs::create_dir_all(&tmp_path).expect("Failed to create tmp directory"); + let commit_path = tmp_path.join("commit"); + std::fs::write(&commit_path, commit_data).expect("Failed to write commit data"); + + match extract_commit_from_bytes(&commit_path) { + Ok(commit) => { + std::fs::remove_dir_all(&tmp_path).expect("Failed to remove tmp directory"); + assert_eq!( + &commit.id._to_string(), + "1c85c8908b8eb0e4bc62ff834cae3038ca2930f5" + ); + assert_eq!( + &commit.tree_id._to_string(), + "73a486aae34e5c2afe7bc164a54569b98133af4d" + ); + assert_eq!( + &commit.parent_commit_ids[0]._to_string(), + "54f28ffd6c8aece72eb9138bfaa44ad0dacfb2ff" + ); + println!("\x1b[32mParse successful!\x1b[0m"); + } + Err(e) => { + std::fs::remove_dir_all(&tmp_path).expect("Failed to remove tmp directory"); + eprintln!("\x1b[31mParse failed: {}\x1b[0m", e); + } + } + } +} diff --git a/scorpio/src/manager/reset.rs b/scorpio/src/manager/reset.rs index 0c54c91df..77f266f61 100644 --- a/scorpio/src/manager/reset.rs +++ b/scorpio/src/manager/reset.rs @@ -1,6 +1,7 @@ use std::fs; use std::path::Path; +/// Reset the repository, discarding all changes. pub fn reset_core(work_path: &Path) -> Result<(), Box> { let modified_path = work_path.join("modifiedstore"); let upper_path = work_path.join("upper"); diff --git a/scorpio/src/manager/store.rs b/scorpio/src/manager/store.rs index 0bfbe2897..b91ce5051 100644 --- a/scorpio/src/manager/store.rs +++ b/scorpio/src/manager/store.rs @@ -47,7 +47,8 @@ impl TreeStore for sled::Db { // By returning a HashMap, we avoid using a double pointer loop structure in diff.rs. Ok((path, encoded_value)) => { // Convert the IVec to a string and then to a PathBuf - let path = std::str::from_utf8(&path).unwrap_or("Invalid UTF8 path"); + let path = std::str::from_utf8(&path) + .map_err(|_| Error::new(ErrorKind::InvalidData, "Invalid UTF8 path"))?; let decoded_tree: Result = bincode::deserialize(&encoded_value) .map_err(|_| Error::other("Deserialization error")); Ok((PathBuf::from(path), decoded_tree?)) @@ -185,7 +186,11 @@ impl BlobFsStore for PathBuf { hashmap .values() .map(|hash| { - let hash_path = object_path.join(&hash[0..2]); + let hash_flag = hash.get(0..2).ok_or(Error::new( + ErrorKind::InvalidInput, + "Hash must be 40 characters long", + ))?; + let hash_path = object_path.join(hash_flag); let sha_hash = SHA1::from_str(hash).map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; diff --git a/scorpio/src/util/mod.rs b/scorpio/src/util/mod.rs index 4e8c9192d..736090616 100644 --- a/scorpio/src/util/mod.rs +++ b/scorpio/src/util/mod.rs @@ -54,14 +54,9 @@ impl From for PathBuf { } } impl From for String { - fn from(val: GPath) -> Self { - val.path.iter().fold(String::new(), |acc, part| { - match (acc.is_empty(), part.is_empty()) { - (true, _) => part.clone(), - (false, true) => acc, - (false, false) => acc + "/" + part, - } - }) + fn from(mut val: GPath) -> Self { + val.path.retain(|part| !part.is_empty()); + val.path.join("/") } } impl Display for GPath {