From f4b221258de2ad731f29bda419f6ce2cdc83cf5a Mon Sep 17 00:00:00 2001 From: "benjamin.747" Date: Mon, 2 Sep 2024 16:45:04 +0800 Subject: [PATCH 1/2] [lfs] split lfs api route, now need add `git config lfs.url` before using lfs --- ceres/src/lfs/handler.rs | 157 ++++++++++----------- ceres/src/lfs/mod.rs | 21 --- common/src/config.rs | 8 +- common/src/model.rs | 8 +- docker/config.toml | 14 +- gateway/src/https_server.rs | 16 +-- jupiter/src/context.rs | 20 ++- jupiter/src/raw_storage/local_storage.rs | 12 +- jupiter/src/raw_storage/mod.rs | 72 +++++----- jupiter/src/storage/git_fs_storage.rs | 16 +-- mega/config.toml | 8 +- mono/config.toml | 6 +- mono/src/{lfs.rs => api/lfs/lfs_router.rs} | 143 +++++++++---------- mono/src/api/lfs/mod.rs | 1 + mono/src/api/mod.rs | 1 + mono/src/git_protocol/http.rs | 7 +- mono/src/lib.rs | 1 - mono/src/main.rs | 1 - mono/src/server/https_server.rs | 87 ++---------- 19 files changed, 243 insertions(+), 356 deletions(-) rename mono/src/{lfs.rs => api/lfs/lfs_router.rs} (71%) create mode 100644 mono/src/api/lfs/mod.rs diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index 6737b897f..96774298a 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -5,23 +5,23 @@ use std::sync::Arc; use anyhow::Result; use bytes::Bytes; use chrono::{prelude::*, Duration}; -use jupiter::storage::lfs_storage::LfsStorage; use rand::prelude::*; use callisto::{lfs_locks, lfs_objects, lfs_split_relations}; use common::errors::{GitLFSError, MegaError}; +use jupiter::context::Context; +use jupiter::storage::lfs_storage::LfsStorage; use crate::lfs::lfs_structs::{ BatchRequest, LockList, LockRequest, ObjectError, UnlockRequest, VerifiableLockList, VerifiableLockRequest, }; -use crate::lfs::lfs_structs::{Link, Lock, LockListQuery, MetaObject, Representation, RequestVars}; -use crate::lfs::LfsConfig; -use super::lfs_structs::ChunkRepresentation; +use crate::lfs::lfs_structs::ChunkRepresentation; +use crate::lfs::lfs_structs::{Link, Lock, LockListQuery, MetaObject, Representation, RequestVars}; pub async fn lfs_retrieve_lock( - config: &LfsConfig, + storage: Arc, query: LockListQuery, ) -> Result { let mut lock_list = LockList { @@ -29,7 +29,7 @@ pub async fn lfs_retrieve_lock( next_cursor: "".to_string(), }; match lfs_get_filtered_locks( - config.context.services.lfs_storage.clone(), + storage, &query.refspec, &query.path, &query.cursor, @@ -49,7 +49,7 @@ pub async fn lfs_retrieve_lock( } pub async fn lfs_verify_lock( - config: &LfsConfig, + storage: Arc, req: VerifiableLockRequest, ) -> Result { let mut limit = req.limit.unwrap_or(0); @@ -57,7 +57,7 @@ pub async fn lfs_verify_lock( limit = 100; } let res = lfs_get_filtered_locks( - config.context.services.lfs_storage.clone(), + storage, &req.refs.name, "", &req.cursor.clone().unwrap_or("".to_string()).to_string(), @@ -87,9 +87,12 @@ pub async fn lfs_verify_lock( Ok(lock_list) } -pub async fn lfs_create_lock(config: &LfsConfig, req: LockRequest) -> Result { +pub async fn lfs_create_lock( + storage: Arc, + req: LockRequest, +) -> Result { let res = lfs_get_filtered_locks( - config.context.services.lfs_storage.clone(), + storage.clone(), &req.refs.name, &req.path.to_string(), "", @@ -127,13 +130,7 @@ pub async fn lfs_create_lock(config: &LfsConfig, req: LockRequest) -> Result Ok(lock), Err(_) => Err(GitLFSError::GeneralError( "Failed when adding locks!".to_string(), @@ -142,7 +139,7 @@ pub async fn lfs_create_lock(config: &LfsConfig, req: LockRequest) -> Result, id: &str, unlock_request: UnlockRequest, ) -> Result { @@ -150,7 +147,7 @@ pub async fn lfs_delete_lock( return Err(GitLFSError::GeneralError("Invalid lock id!".to_string())); } let res = delete_lock( - config.context.services.lfs_storage.clone(), + storage, &unlock_request.refs.name, None, id, @@ -182,7 +179,7 @@ pub async fn lfs_delete_lock( /// Reference: /// 1. [Git LFS Batch API](https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md) pub async fn lfs_process_batch( - config: &LfsConfig, + context: &Context, mut batch_vars: BatchRequest, ) -> Result, GitLFSError> { let bvo = &mut batch_vars.objects; @@ -190,18 +187,28 @@ pub async fn lfs_process_batch( request.authorization = "".to_string(); } let mut response_objects = Vec::::new(); - let server_url = format!("http://{}:{}", config.host, config.port); - - let storage = config.context.services.lfs_storage.clone(); + let storage = context.services.lfs_storage.clone(); + let config = context.config.lfs.clone(); + let server_url = context.config.lfs.url.clone(); for object in &batch_vars.objects { let meta = lfs_get_meta(storage.clone(), object).await; // Found let found = meta.is_ok(); let mut meta = meta.unwrap_or_default(); - if found && lfs_file_exist(config, &meta).await { + if found && lfs_file_exist(context, &meta).await { // original download method, split mode use `` - response_objects.push(represent(object, &meta, batch_vars.operation == "download", false, false, &server_url).await); + response_objects.push( + represent( + object, + &meta, + batch_vars.operation == "download", + false, + false, + &server_url, + ) + .await, + ); continue; } // Not found @@ -230,24 +237,24 @@ pub async fn lfs_process_batch( /// if server enable split, then return a list of chunk ids. /// else return an error. pub async fn lfs_fetch_chunk_ids( - config: &LfsConfig, + context: &Context, fetch_vars: &RequestVars, ) -> Result, GitLFSError> { + let config = context.config.lfs.clone(); + if !config.enable_split { return Err(GitLFSError::GeneralError( "Server didn't run in `split` mode, didn't support chunk ids".to_string(), )); } + let storage = context.services.lfs_storage.clone(); - let meta = lfs_get_meta(config.context.services.lfs_storage.clone(), fetch_vars) + let meta = lfs_get_meta(storage.clone(), fetch_vars) .await .map_err(|_| GitLFSError::GeneralError("".to_string()))?; assert!(meta.splited, "database didn't match the split mode"); - let relations = config - .context - .services - .lfs_storage + let relations = storage .get_lfs_relations(fetch_vars.oid.clone()) .await .map_err(|_| GitLFSError::GeneralError("".to_string()))?; @@ -258,7 +265,7 @@ pub async fn lfs_fetch_chunk_ids( )); } let mut response_objects = Vec::::new(); - let server_url = format!("http://{}:{}", config.host, config.port); + let server_url = context.config.lfs.url.clone(); for relation in relations { // Reuse RequestArgs to create a link @@ -286,11 +293,15 @@ pub async fn lfs_fetch_chunk_ids( /// Upload object to storage. /// if server enable split, split the object and upload each part to storage, save the relationship to database. pub async fn lfs_upload_object( - config: &LfsConfig, + context: &Context, request_vars: &RequestVars, body_bytes: &[u8], ) -> Result<(), GitLFSError> { - let meta = lfs_get_meta(config.context.services.lfs_storage.clone(), request_vars) + let config = context.config.lfs.clone(); + let lfs_storage = context.services.lfs_storage.clone(); + let raw_storage = context.services.raw_storage.clone(); + + let meta = lfs_get_meta(lfs_storage.clone(), request_vars) .await .unwrap(); if config.enable_split && meta.splited { @@ -301,12 +312,9 @@ pub async fn lfs_upload_object( for chunk in body_bytes.chunks(config.split_size) { // sha256 let sub_id = sha256::digest(chunk); - let res = config - .lfs_storage - .put_object(0, &sub_id, chunk) - .await; + let res = raw_storage.put_object(&sub_id, chunk).await; if res.is_err() { - lfs_delete_meta(config.context.services.lfs_storage.clone(), request_vars) + lfs_delete_meta(lfs_storage.clone(), request_vars) .await .unwrap(); // TODO: whether/how to delete the uploaded blocks. @@ -319,21 +327,18 @@ pub async fn lfs_upload_object( // save the relationship to database let mut offset = 0; for sub_id in sub_ids { - let db = config.context.services.lfs_storage.clone(); + // let db = config.context.services.lfs_storage.clone(); let size = min(config.split_size as i64, body_bytes.len() as i64 - offset); - lfs_put_relation(db, &meta.oid, &sub_id, offset, size) + lfs_put_relation(lfs_storage.clone(), &meta.oid, &sub_id, offset, size) .await .unwrap(); offset += size; } } else { // normal mode - let res = config - .lfs_storage - .put_object(0, &meta.oid, body_bytes) - .await; + let res = raw_storage.put_object(&meta.oid, body_bytes).await; if res.is_err() { - lfs_delete_meta(config.context.services.lfs_storage.clone(), request_vars) + lfs_delete_meta(lfs_storage.clone(), request_vars) .await .unwrap(); return Err(GitLFSError::GeneralError(String::from( @@ -347,17 +352,20 @@ pub async fn lfs_upload_object( /// Download object from storage. /// when server enable split, if OID is a complete object, then splice the object and return it. pub async fn lfs_download_object( - config: &LfsConfig, + context: Context, request_vars: &RequestVars, ) -> Result { + let config = context.config.lfs; + let stg = context.services.lfs_storage.clone(); + let raw_storage = context.services.raw_storage.clone(); if config.enable_split { - let meta = lfs_get_meta(config.context.services.lfs_storage.clone(), request_vars).await; - let relation_db = config.context.services.lfs_storage.clone(); + let meta = lfs_get_meta(stg.clone(), request_vars).await; + // let relation_db = context.services.lfs_storage.clone(); match meta { Ok(meta) => { // client didn't support split, splice the object and return it. - let relations = relation_db.get_lfs_relations(meta.oid).await.unwrap(); + let relations = stg.get_lfs_relations(meta.oid).await.unwrap(); if relations.is_empty() { return Err(GitLFSError::GeneralError( "oid didn't have chunks".to_string(), @@ -365,11 +373,7 @@ pub async fn lfs_download_object( } let mut bytes = vec![0u8; meta.size as usize]; for relation in relations { - let sub_bytes = config - .lfs_storage - .get_object(0, &relation.sub_oid) - .await - .unwrap(); + let sub_bytes = raw_storage.get_object(&relation.sub_oid).await.unwrap(); let offset = relation.offset as usize; let size = relation.size as usize; bytes[offset..offset + size].copy_from_slice(&sub_bytes); @@ -379,32 +383,19 @@ pub async fn lfs_download_object( Err(_) => { // check if the oid is a part of a split object, if so, return the part. let sub_oid = request_vars.oid.clone(); - if !lfs_check_sub_oid_exist(relation_db, &sub_oid) - .await - .unwrap() - { + if !lfs_check_sub_oid_exist(stg, &sub_oid).await.unwrap() { return Err(GitLFSError::GeneralError( "oid didn't belong to any object".to_string(), )); } - let bytes = config - .lfs_storage - .get_object(0, &sub_oid) - .await - .unwrap(); + let bytes = raw_storage.get_object(&sub_oid).await.unwrap(); Ok(bytes) } } } else { - let meta = lfs_get_meta(config.context.services.lfs_storage.clone(), request_vars) - .await - .unwrap(); - let bytes = config - .lfs_storage - .get_object(0, &meta.oid) - .await - .unwrap(); + let meta = lfs_get_meta(stg, request_vars).await.unwrap(); + let bytes = raw_storage.get_object(&meta.oid).await.unwrap(); Ok(bytes) } } @@ -475,27 +466,23 @@ fn create_link(href: &str, header: &HashMap) -> Link { } /// check if meta file exist in storage. -async fn lfs_file_exist(config: &LfsConfig, meta: &MetaObject) -> bool { +async fn lfs_file_exist(context: &Context, meta: &MetaObject) -> bool { + let config = context.config.lfs.clone(); + let storage = context.services.lfs_storage.clone(); + let raw_storage = context.services.raw_storage.clone(); if meta.splited && config.enable_split { - let relations = config - .context - .services - .lfs_storage + let relations = storage .get_lfs_relations(meta.oid.clone()) .await .unwrap(); if relations.is_empty() { return false; } - relations.iter().all(|relation| { - config - .lfs_storage - .exist_object(0, &relation.sub_oid) - }) + relations + .iter() + .all(|relation| raw_storage.exist_object(&relation.sub_oid)) } else { - config - .lfs_storage - .exist_object(0, &meta.oid) + raw_storage.exist_object(&meta.oid) } } diff --git a/ceres/src/lfs/mod.rs b/ceres/src/lfs/mod.rs index 67a4c3fa1..ce9bd7eff 100644 --- a/ceres/src/lfs/mod.rs +++ b/ceres/src/lfs/mod.rs @@ -1,23 +1,2 @@ -use std::sync::Arc; - -use jupiter::{context::Context, raw_storage::RawStorage}; - pub mod handler; pub mod lfs_structs; - -#[derive(Clone)] -pub struct LfsConfig { - pub host: String, - - pub port: u16, - - pub context: Context, - - pub lfs_storage: Arc, - - pub repo_name: String, - - pub enable_split: bool, - - pub split_size: usize, -} diff --git a/common/src/config.rs b/common/src/config.rs index 60edf441d..0a6164290 100644 --- a/common/src/config.rs +++ b/common/src/config.rs @@ -60,7 +60,7 @@ impl Default for Config { }) .collect::>() .join("\n"); - + // default config path: $MEGA_BASE_DIR/etc/config.toml // ensure the directory exists std::fs::create_dir_all(base_dir.join("etc")).unwrap(); @@ -206,7 +206,6 @@ pub struct StorageConfig { pub raw_obj_storage_type: String, pub big_obj_threshold: usize, pub raw_obj_local_path: PathBuf, - pub lfs_obj_local_path: PathBuf, pub obs_access_key: String, pub obs_secret_key: String, pub obs_region: String, @@ -219,7 +218,6 @@ impl Default for StorageConfig { raw_obj_storage_type: String::from("LOCAL"), big_obj_threshold: 1024, raw_obj_local_path: PathBuf::from("/tmp/.mega/objects"), - lfs_obj_local_path: PathBuf::from("/tmp/.mega/lfs"), obs_access_key: String::new(), obs_secret_key: String::new(), obs_region: String::from("cn-east-3"), @@ -262,6 +260,8 @@ impl Default for PackConfig { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct LFSConfig { + pub url: String, + pub lfs_obj_local_path: PathBuf, pub enable_split: bool, pub split_size: usize, } @@ -269,6 +269,8 @@ pub struct LFSConfig { impl Default for LFSConfig { fn default() -> Self { Self { + url: "http://localhost:8000".to_string(), + lfs_obj_local_path: PathBuf::from("/tmp/.mega/lfs"), enable_split: true, split_size: 1024 * 1024 * 1024, } diff --git a/common/src/model.rs b/common/src/model.rs index 3a443472b..21778b572 100644 --- a/common/src/model.rs +++ b/common/src/model.rs @@ -17,15 +17,9 @@ pub struct ZtmOptions { } #[derive(Deserialize, Debug)] -pub struct GetParams { +pub struct InfoRefsParams { pub service: Option, pub refspec: Option, - pub id: Option, - pub path: Option, - pub limit: Option, - pub cursor: Option, - pub identifier: Option, - pub port: Option, } #[derive(PartialEq, Eq, Debug, Clone, Default, Serialize, Deserialize)] diff --git a/docker/config.toml b/docker/config.toml index afa6b541d..5556eb647 100644 --- a/docker/config.toml +++ b/docker/config.toml @@ -49,8 +49,6 @@ big_obj_threshold = 1024 # set the local path of the project storage raw_obj_local_path = "${base_dir}/objects" -lfs_obj_local_path = "${base_dir}/lfs" - obs_access_key = "" obs_secret_key = "" @@ -80,6 +78,12 @@ clean_cache_after_decode = true channel_message_size = 1_000_000 [lfs] +# LFS Server url +url = "https://git.gitmono.com" + +# set the local path of the lfs storage +lfs_obj_local_path = "${base_dir}/lfs" + ## IMPORTANT: The 'enable_split' feature can only be enabled for new databases. Existing databases do not support this feature. # Enable or disable splitting large files into smaller chunks enable_split = true # Default is disabled. Set to true to enable file splitting. @@ -91,3 +95,9 @@ split_size = 20971520 # Default size is 20MB (20971520 bytes) # GitHub OAuth application client id and secret github_client_id = "" github_client_secret = "" + +# Used redirect to ui after login +ui_domain = "https://console.gitmono.com" + +# Set .gitmono.com on Production +cookie_domain = ".gitmono.com" \ No newline at end of file diff --git a/gateway/src/https_server.rs b/gateway/src/https_server.rs index 43569cfde..a3316c676 100644 --- a/gateway/src/https_server.rs +++ b/gateway/src/https_server.rs @@ -17,10 +17,9 @@ use common::config::Config; use common::model::{CommonOptions, ZtmOptions}; use gemini::ztm::agent::{run_ztm_client, LocalZTMAgent}; use jupiter::context::Context; +use mono::api::lfs::lfs_router; use mono::api::MonoApiServiceState; -use mono::server::https_server::{ - get_method_router, post_method_router, put_method_router, AppState, -}; +use mono::server::https_server::{get_method_router, post_method_router, AppState}; use crate::api::{github_router, nostr_router, ztm_router, MegaApiServiceState}; @@ -157,6 +156,10 @@ pub async fn app( // add TraceLayer for log record // add CorsLayer to add cors header Router::new() + .nest( + "/", + lfs_router::routers().with_state(mono_api_state.clone()), + ) .nest( "/api/v1/mono", mono::api::api_router::routers().with_state(mono_api_state.clone()), @@ -166,12 +169,7 @@ pub async fn app( mega_routers().with_state(mega_api_state.clone()), ) // Using Regular Expressions for Path Matching in Protocol - .route( - "/*path", - get(get_method_router) - .post(post_method_router) - .put(put_method_router), - ) + .route("/*path", get(get_method_router).post(post_method_router)) .layer( ServiceBuilder::new().layer(CorsLayer::new().allow_origin(Any).allow_headers(vec![ http::header::AUTHORIZATION, diff --git a/jupiter/src/context.rs b/jupiter/src/context.rs index c6c583f18..1c06f7922 100644 --- a/jupiter/src/context.rs +++ b/jupiter/src/context.rs @@ -1,11 +1,14 @@ -use std::sync::Arc; +use std::{env, path::PathBuf, sync::Arc}; use common::config::Config; -use crate::storage::{ - git_db_storage::GitDbStorage, init::database_connection, lfs_storage::LfsStorage, - mono_storage::MonoStorage, mq_storage::MQStorage, user_storage::UserStorage, - ztm_storage::ZTMStorage, +use crate::{ + raw_storage::{local_storage::LocalStorage, RawStorage}, + storage::{ + git_db_storage::GitDbStorage, init::database_connection, lfs_storage::LfsStorage, + mono_storage::MonoStorage, mq_storage::MQStorage, user_storage::UserStorage, + ztm_storage::ZTMStorage, + }, }; #[derive(Clone)] @@ -37,6 +40,7 @@ pub struct Service { pub ztm_storage: Arc, pub mq_storage: Arc, pub user_storage: UserStorage, + pub raw_storage: Arc, } impl Service { @@ -53,6 +57,9 @@ impl Service { ztm_storage: Arc::new(ZTMStorage::new(connection.clone()).await), mq_storage: Arc::new(MQStorage::new(connection.clone()).await), user_storage: UserStorage::new(connection.clone()).await, + raw_storage: Arc::new(LocalStorage::init( + config.lfs.lfs_obj_local_path.clone(), + )), } } @@ -68,6 +75,9 @@ impl Service { ztm_storage: Arc::new(ZTMStorage::mock()), mq_storage: Arc::new(MQStorage::mock()), user_storage: UserStorage::mock(), + raw_storage: Arc::new(LocalStorage::init( + PathBuf::from(env::current_dir().unwrap().parent().unwrap()).join("tests"), + )), }) } } diff --git a/jupiter/src/raw_storage/local_storage.rs b/jupiter/src/raw_storage/local_storage.rs index fd278c74a..7f0161970 100644 --- a/jupiter/src/raw_storage/local_storage.rs +++ b/jupiter/src/raw_storage/local_storage.rs @@ -67,9 +67,8 @@ impl RawStorage for LocalStorage { Ok(()) } - async fn get_object(&self, repo_id: i64, object_id: &str) -> Result { + async fn get_object(&self, object_id: &str) -> Result { let path = Path::new(&self.base_path) - .join(repo_id.to_string()) .join("objects") .join(self.transform_path(object_id)); let mut file = @@ -81,12 +80,10 @@ impl RawStorage for LocalStorage { async fn put_object( &self, - repo_id: i64, object_id: &str, body_content: &[u8], ) -> Result { let path = Path::new(&self.base_path) - .join(repo_id.to_string()) .join("objects") .join(self.transform_path(object_id)); let dir = path.parent().unwrap(); @@ -97,9 +94,8 @@ impl RawStorage for LocalStorage { Ok(path.to_str().unwrap().to_string()) } - fn exist_object(&self, repo_id: i64, object_id: &str) -> bool { + fn exist_object(&self, object_id: &str) -> bool { let path = Path::new(&self.base_path) - .join(repo_id.to_string()) .join("objects") .join(self.transform_path(object_id)); Path::exists(&path) @@ -124,9 +120,9 @@ mod tests { source.push("tests/objects"); let local_storage = LocalStorage::init(source.clone()); - assert!(local_storage.put_object(0, &oid, &content).await.is_ok()); + assert!(local_storage.put_object(&oid, &content).await.is_ok()); - assert!(local_storage.exist_object(0, &oid)); + assert!(local_storage.exist_object(&oid)); } #[tokio::test] diff --git a/jupiter/src/raw_storage/mod.rs b/jupiter/src/raw_storage/mod.rs index b3f473af0..6de5ba020 100644 --- a/jupiter/src/raw_storage/mod.rs +++ b/jupiter/src/raw_storage/mod.rs @@ -1,18 +1,13 @@ use std::{ - env, - fs::File, - io::Read, path::{self, PathBuf}, sync::Arc, }; use async_trait::async_trait; use bytes::Bytes; -use handlebars::Handlebars; use callisto::db_enums::StorageType; use common::errors::MegaError; -use mercury::internal::pack::entry::Entry; use crate::raw_storage::local_storage::LocalStorage; @@ -48,11 +43,10 @@ pub trait RawStorage: Sync + Send { ref_hash: &str, ) -> Result<(), MegaError>; - async fn get_object(&self, repo_id: i64, object_id: &str) -> Result; + async fn get_object(&self, object_id: &str) -> Result; async fn put_object( &self, - repo_id: i64, object_id: &str, body_content: &[u8], ) -> Result; @@ -71,39 +65,39 @@ pub trait RawStorage: Sync + Send { // } // save a entry and return the b_link file - async fn convert_blink(&self, repo_id: i64, entry: &Entry) -> Result, MegaError> { - let location = self - .put_object(repo_id, &entry.hash.to_plain_str(), &entry.data) - .await - .unwrap(); - let handlebars = Handlebars::new(); - - let path = env::current_dir().unwrap().join("b_link.txt"); - let mut file = File::open(path).unwrap(); - let mut template = String::new(); - file.read_to_string(&mut template).unwrap(); - - let mut context = serde_json::Map::new(); - context.insert( - "objectType".to_string(), - serde_json::json!(entry.obj_type.to_string()), - ); - context.insert( - "sha1".to_string(), - serde_json::json!(entry.hash.to_plain_str()), - ); - context.insert( - "type".to_string(), - serde_json::json!(self.get_storage_type().to_string()), - ); - context.insert("location".to_string(), serde_json::json!(location)); - - let rendered = handlebars.render_template(&template, &context).unwrap(); - - Ok(rendered.into_bytes()) - } + // async fn convert_blink(&self, entry: &Entry) -> Result, MegaError> { + // let location = self + // .put_object( &entry.hash.to_plain_str(), &entry.data) + // .await + // .unwrap(); + // let handlebars = Handlebars::new(); + + // let path = env::current_dir().unwrap().join("b_link.txt"); + // let mut file = File::open(path).unwrap(); + // let mut template = String::new(); + // file.read_to_string(&mut template).unwrap(); + + // let mut context = serde_json::Map::new(); + // context.insert( + // "objectType".to_string(), + // serde_json::json!(entry.obj_type.to_string()), + // ); + // context.insert( + // "sha1".to_string(), + // serde_json::json!(entry.hash.to_plain_str()), + // ); + // context.insert( + // "type".to_string(), + // serde_json::json!(self.get_storage_type().to_string()), + // ); + // context.insert("location".to_string(), serde_json::json!(location)); + + // let rendered = handlebars.render_template(&template, &context).unwrap(); + + // Ok(rendered.into_bytes()) + // } - fn exist_object(&self, repo_id: i64, object_id: &str) -> bool; + fn exist_object(&self, object_id: &str) -> bool; fn transform_path(&self, sha1: &str) -> String { if sha1.len() < 5 { diff --git a/jupiter/src/storage/git_fs_storage.rs b/jupiter/src/storage/git_fs_storage.rs index e8fd95a74..4b6a3dab3 100644 --- a/jupiter/src/storage/git_fs_storage.rs +++ b/jupiter/src/storage/git_fs_storage.rs @@ -63,28 +63,20 @@ impl GitFsStorage { } } - pub async fn save_entry(&self, repo_id: i64, entry_list: Vec) -> Result<(), MegaError> { + pub async fn save_entry(&self, entry_list: Vec) -> Result<(), MegaError> { for entry in entry_list { self.raw_storage - .put_object(repo_id, &entry.hash.to_plain_str(), &entry.data) + .put_object(&entry.hash.to_plain_str(), &entry.data) .await .unwrap(); } Ok(()) } - pub async fn get_entry_by_sha1( - &self, - repo_id: i64, - sha1_vec: Vec<&str>, - ) -> Result, MegaError> { + pub async fn get_entry_by_sha1(&self, sha1_vec: Vec<&str>) -> Result, MegaError> { let mut res: Vec = Vec::new(); for sha1 in sha1_vec { - let data = self - .raw_storage - .get_object(repo_id, sha1) - .await - .unwrap(); + let data = self.raw_storage.get_object(sha1).await.unwrap(); let (type_num, _) = utils::read_type_and_size(&mut Cursor::new(&data)).unwrap(); let obj_type = ObjectType::from_u8(type_num).unwrap(); let hash = SHA1::new(&data.to_vec()); diff --git a/mega/config.toml b/mega/config.toml index 4742d168d..f75f2ca81 100644 --- a/mega/config.toml +++ b/mega/config.toml @@ -49,8 +49,6 @@ big_obj_threshold = 1024 # set the local path of the project storage raw_obj_local_path = "${base_dir}/objects" -lfs_obj_local_path = "${base_dir}/lfs" - obs_access_key = "" obs_secret_key = "" @@ -80,6 +78,12 @@ clean_cache_after_decode = true channel_message_size = 1_000_000 [lfs] +# LFS Server url +url = "http://localhost:8000" + +# set the local path of the lfs storage +lfs_obj_local_path = "${base_dir}/lfs" + ## IMPORTANT: The 'enable_split' feature can only be enabled for new databases. Existing databases do not support this feature. # Enable or disable splitting large files into smaller chunks enable_split = true # Default is disabled. Set to true to enable file splitting. diff --git a/mono/config.toml b/mono/config.toml index ac752e074..2cdb5d7c3 100644 --- a/mono/config.toml +++ b/mono/config.toml @@ -49,9 +49,6 @@ big_obj_threshold = 1024 # set the local path of the project storage raw_obj_local_path = "${base_dir}/objects" -# set the local path of the lfs storage -lfs_obj_local_path = "${base_dir}/lfs" - obs_access_key = "" obs_secret_key = "" @@ -84,6 +81,9 @@ channel_message_size = 1_000_000 # LFS Server url url = "http://localhost:8000" +# set the local path of the lfs storage +lfs_obj_local_path = "${base_dir}/lfs" + ## IMPORTANT: The 'enable_split' feature can only be enabled for new databases. Existing databases do not support this feature. # Enable or disable splitting large files into smaller chunks enable_split = true # Default is disabled. Set to true to enable file splitting. diff --git a/mono/src/lfs.rs b/mono/src/api/lfs/lfs_router.rs similarity index 71% rename from mono/src/lfs.rs rename to mono/src/api/lfs/lfs_router.rs index 765cad358..d11150eb4 100644 --- a/mono/src/lfs.rs +++ b/mono/src/api/lfs/lfs_router.rs @@ -43,42 +43,47 @@ //! when using these handlers in a web application to prevent unauthorized access. use axum::{ body::Body, - extract::{FromRequest, State}, + extract::{Path, Query, State}, http::{Request, StatusCode}, response::Response, - Json, + routing::{get, post, put}, + Json, Router, }; +use futures::TryStreamExt; use ceres::lfs::{ handler, lfs_structs::{ - BatchResponse, FetchchunkResponse, LockList, LockListQuery, LockRequest, LockResponse, - RequestVars, UnlockRequest, UnlockResponse, VerifiableLockRequest, + BatchRequest, BatchResponse, FetchchunkResponse, LockList, LockListQuery, LockRequest, + LockResponse, RequestVars, UnlockRequest, UnlockResponse, VerifiableLockRequest, }, - LfsConfig, }; -use common::{errors::GitLFSError, model::GetParams}; -use futures::TryStreamExt; +use common::errors::GitLFSError; -use crate::server::https_server::AppState; +use crate::api::MonoApiServiceState; const LFS_CONTENT_TYPE: &str = "application/vnd.git-lfs+json"; -pub async fn lfs_retrieve_lock( - config: &LfsConfig, - params: GetParams, -) -> Result, (StatusCode, String)> { - // Load query parameters into struct. - let lock_list_query = LockListQuery { - path: params.path.unwrap_or_default(), - id: params.id.unwrap_or_default(), - cursor: params.cursor.unwrap_or_default(), - limit: params.limit.unwrap_or_default(), - refspec: params.refspec.unwrap_or_default(), - }; +/// The [LFS Server Discovery](https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md) +/// document describes the server LFS discovery protocol. +pub fn routers() -> Router { + Router::new() + .route("/objects/:object_id", get(lfs_download_object)) + .route("/objects/:object_id", put(lfs_upload_object)) + .route("/locks", get(list_locks)) + .route("/locks", post(create_lock)) + .route("/locks/verify", post(list_locks_for_verification)) + .route("/locks/:id/unlock", post(delete_lock)) + .route("/objects/batch", post(lfs_process_batch)) + .route("/objects/chunkids", get(lfs_fetch_chunk_ids)) +} +pub async fn list_locks( + state: State, + Query(query): Query, +) -> Result, (StatusCode, String)> { let result: Result = - handler::lfs_retrieve_lock(config, lock_list_query).await; + handler::lfs_retrieve_lock(state.context.services.lfs_storage.clone(), query).await; match result { Ok(lock_list) => { let body = serde_json::to_string(&lock_list).unwrap_or_default(); @@ -91,18 +96,18 @@ pub async fn lfs_retrieve_lock( } } -pub async fn lfs_verify_lock( - state: State, - config: &LfsConfig, - req: Request, +pub async fn list_locks_for_verification( + state: State, + // req: Request, + Json(json): Json, ) -> Result, (StatusCode, String)> { - tracing::info!("req: {:?}", req); + // tracing::info!("req: {:?}", req); - let request = Json::from_request(req, &state) - .await - .unwrap_or_else(|_| Json(VerifiableLockRequest::default())); + // let request = Json::from_request(req, &state) + // .await + // .unwrap_or_else(|_| Json(VerifiableLockRequest::default())); - let result = handler::lfs_verify_lock(config, request.0).await; + let result = handler::lfs_verify_lock(state.context.services.lfs_storage.clone(), json).await; match result { Ok(lock_list) => { let body = serde_json::to_string(&lock_list).unwrap_or_default(); @@ -120,16 +125,15 @@ pub async fn lfs_verify_lock( } } -pub async fn lfs_create_lock( - state: State, - config: &LfsConfig, - req: Request, +pub async fn create_lock( + state: State, + Json(json): Json, ) -> Result, (StatusCode, String)> { - let request = Json::from_request(req, &state) - .await - .unwrap_or_else(|_| Json(LockRequest::default())); + // let request = Json::from_request(req, &state) + // .await + // .unwrap_or_else(|_| Json(LockRequest::default())); - let result = handler::lfs_create_lock(config, request.0).await; + let result = handler::lfs_create_lock(state.context.services.lfs_storage.clone(), json).await; match result { Ok(lock) => { let lock_response = LockResponse { @@ -152,19 +156,13 @@ pub async fn lfs_create_lock( } } -pub async fn lfs_delete_lock( - state: State, - config: &LfsConfig, - path: &str, - req: Request, +pub async fn delete_lock( + state: State, + Path(id): Path, + Json(json): Json, ) -> Result { - let tokens: Vec<&str> = path.split('/').collect(); - let id = tokens[tokens.len() - 2]; - let request = Json::from_request(req, &state) - .await - .unwrap_or_else(|_| Json(UnlockRequest::default())); - - let result = handler::lfs_delete_lock(config, id, request.0).await; + let result = + handler::lfs_delete_lock(state.context.services.lfs_storage.clone(), &id, json).await; match result { Ok(lock) => { @@ -188,12 +186,10 @@ pub async fn lfs_delete_lock( } pub async fn lfs_process_batch( - state: State, - config: &LfsConfig, - req: Request, + state: State, + Json(json): Json, ) -> Result, (StatusCode, String)> { - let request = Json::from_request(req, &state).await.unwrap(); - let result = handler::lfs_process_batch(config, request.0).await; + let result = handler::lfs_process_batch(&state.context, json).await; match result { Ok(response_objects) => { @@ -220,21 +216,20 @@ pub async fn lfs_process_batch( } pub async fn lfs_fetch_chunk_ids( - state: State, - config: &LfsConfig, - req: Request, + state: State, + Json(json): Json, ) -> Result, (StatusCode, String)> { - let request = Json::from_request(req, &state).await; - if request.is_err() { - return Err((StatusCode::BAD_REQUEST, "Invalid request".to_string())); - } - let request = request.unwrap(); - let result = handler::lfs_fetch_chunk_ids(config, &request).await; + // let request = Json::from_request(req, &state).await; + // if request.is_err() { + // return Err((StatusCode::BAD_REQUEST, "Invalid request".to_string())); + // } + // let request = request.unwrap(); + let result = handler::lfs_fetch_chunk_ids(&state.context, &json).await; match result { Ok(response) => { let size = response.iter().fold(0, |acc, chunk| acc + chunk.size); let fetch_response = FetchchunkResponse { - oid: request.oid.clone(), + oid: json.oid.clone(), size, chunks: response, }; @@ -255,17 +250,16 @@ pub async fn lfs_fetch_chunk_ids( } pub async fn lfs_download_object( - config: &LfsConfig, - path: &str, + state: State, + Path(oid): Path, ) -> Result { - let tokens: Vec<&str> = path.split('/').collect(); // Load request parameters into struct. let request_vars = RequestVars { - oid: tokens[tokens.len() - 1].to_owned(), + oid, authorization: "".to_owned(), ..Default::default() }; - let result = handler::lfs_download_object(config, &request_vars).await; + let result = handler::lfs_download_object(state.context.clone(), &request_vars).await; match result { Ok(bytes) => Ok(Response::builder().body(Body::from(bytes)).unwrap()), Err(err) => Ok({ @@ -278,14 +272,13 @@ pub async fn lfs_download_object( } pub async fn lfs_upload_object( - config: &LfsConfig, - path: &str, + state: State, + Path(oid): Path, req: Request, ) -> Result, (StatusCode, String)> { - let tokens: Vec<&str> = path.split('/').collect(); // Load request parameters into struct. let request_vars = RequestVars { - oid: tokens[tokens.len() - 1].to_string(), + oid, authorization: "".to_string(), ..Default::default() }; @@ -301,7 +294,7 @@ pub async fn lfs_upload_object( .await .unwrap(); - let result = handler::lfs_upload_object(config, &request_vars, &body_bytes).await; + let result = handler::lfs_upload_object(&state.context, &request_vars, &body_bytes).await; match result { Ok(_) => Ok(Response::builder() .header("Content-Type", LFS_CONTENT_TYPE) diff --git a/mono/src/api/lfs/mod.rs b/mono/src/api/lfs/mod.rs new file mode 100644 index 000000000..d91746fbb --- /dev/null +++ b/mono/src/api/lfs/mod.rs @@ -0,0 +1 @@ +pub mod lfs_router; \ No newline at end of file diff --git a/mono/src/api/mod.rs b/mono/src/api/mod.rs index 2b99a4ec5..191433636 100644 --- a/mono/src/api/mod.rs +++ b/mono/src/api/mod.rs @@ -16,6 +16,7 @@ pub mod api_router; pub mod mr_router; pub mod oauth; pub mod user; +pub mod lfs; #[derive(Clone)] pub struct MonoApiServiceState { diff --git a/mono/src/git_protocol/http.rs b/mono/src/git_protocol/http.rs index 30d19b8ee..752a9296d 100644 --- a/mono/src/git_protocol/http.rs +++ b/mono/src/git_protocol/http.rs @@ -4,14 +4,13 @@ use anyhow::Result; use axum::body::Body; use axum::http::{HeaderValue, Request, Response, StatusCode}; use bytes::{Bytes, BytesMut}; -use common::errors::ProtocolError; use futures::{stream, TryStreamExt}; use tokio::io::AsyncReadExt; use tokio_stream::StreamExt; -use common::model::GetParams; - use ceres::protocol::{smart, ServiceType, SmartProtocol}; +use common::errors::ProtocolError; +use common::model::InfoRefsParams; // # Discovering Reference // HTTP clients that support the "smart" protocol (or both the "smart" and "dumb" protocols) MUST @@ -20,7 +19,7 @@ use ceres::protocol::{smart, ServiceType, SmartProtocol}; // where $servicename MUST be the service name the client wishes to contact to complete the operation. // The request MUST NOT contain additional query parameters. pub async fn git_info_refs( - params: GetParams, + params: InfoRefsParams, mut pack_protocol: SmartProtocol, ) -> Result, (StatusCode, String)> { let service_name = params.service.unwrap(); diff --git a/mono/src/lib.rs b/mono/src/lib.rs index f2c107494..b5ee9056c 100644 --- a/mono/src/lib.rs +++ b/mono/src/lib.rs @@ -2,7 +2,6 @@ pub mod api; pub mod cli; mod commands; pub mod git_protocol; -pub mod lfs; pub mod server; #[cfg(test)] diff --git a/mono/src/main.rs b/mono/src/main.rs index 076e2e00e..0d7c8ece3 100644 --- a/mono/src/main.rs +++ b/mono/src/main.rs @@ -8,7 +8,6 @@ pub mod api; mod cli; mod commands; pub mod git_protocol; -pub mod lfs; pub mod server; fn main() { diff --git a/mono/src/server/https_server.rs b/mono/src/server/https_server.rs index 9359aca71..02426f8a8 100644 --- a/mono/src/server/https_server.rs +++ b/mono/src/server/https_server.rs @@ -1,8 +1,6 @@ use std::net::SocketAddr; -use std::ops::Deref; use std::path::PathBuf; use std::str::FromStr; -use std::sync::Arc; use anyhow::Result; use async_session::MemoryStore; @@ -21,17 +19,15 @@ use tower_http::cors::{Any, CorsLayer}; use tower_http::decompression::RequestDecompressionLayer; use tower_http::trace::TraceLayer; -use ceres::lfs::LfsConfig; use ceres::protocol::{ServiceType, SmartProtocol, TransportProtocol}; use common::config::Config; -use common::model::{CommonOptions, GetParams}; +use common::model::{CommonOptions, InfoRefsParams}; use jupiter::context::Context; -use jupiter::raw_storage::local_storage::LocalStorage; use crate::api::api_router::{self}; +use crate::api::lfs::lfs_router; use crate::api::oauth::{self, oauth_client}; use crate::api::MonoApiServiceState; -use crate::lfs; #[derive(Args, Clone, Debug)] pub struct HttpOptions { @@ -65,22 +61,6 @@ pub struct AppState { pub common: CommonOptions, } -impl From for LfsConfig { - fn from(value: AppState) -> Self { - Self { - host: value.host, - port: value.port, - context: value.context.clone(), - lfs_storage: Arc::new(LocalStorage::init( - value.context.config.storage.lfs_obj_local_path, - )), - repo_name: String::from("repo_name"), - enable_split: value.context.config.lfs.enable_split, - split_size: value.context.config.lfs.split_size, - } - } -} - pub fn remove_git_suffix(uri: Uri, git_suffix: &str) -> PathBuf { PathBuf::from(uri.path().replace(".git", "").replace(git_suffix, "")) } @@ -144,20 +124,16 @@ pub async fn app(config: Config, host: String, port: u16, common: CommonOptions) // add TraceLayer for log record // add CorsLayer to add cors header Router::new() + .nest("/", lfs_router::routers().with_state(api_state.clone())) .nest( "/api/v1", api_router::routers().with_state(api_state.clone()), ) - .nest( - "/auth", - oauth::routers().with_state(api_state.clone()), - ) + .nest("/auth", oauth::routers().with_state(api_state.clone())) // Using Regular Expressions for Path Matching in Protocol .route( "/*path", - get(get_method_router) - .post(post_method_router) - .put(put_method_router), + get(get_method_router).post(post_method_router), // .put(put_method_router), ) .layer( ServiceBuilder::new().layer(CorsLayer::new().allow_origin(Any).allow_headers(vec![ @@ -171,20 +147,7 @@ pub async fn app(config: Config, host: String, port: u16, common: CommonOptions) } lazy_static! { - /// The [LFS Server Discovery](https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md) - /// document describes the server LFS discovery protocol. - /// /// The following regular expressions are used to match the LFS server discovery protocol. - /// - static ref OBJECTS_REGEX: Regex = Regex::new(r"/objects/[a-z0-9]+$").unwrap(); - static ref LOCKS_REGEX: Regex = Regex::new(r"/locks$").unwrap(); - - static ref REGEX_LOCKS_VERIFY: Regex = Regex::new(r"/locks/verify$").unwrap(); - static ref REGEX_UNLOCK: Regex = Regex::new(r"/unlock$").unwrap(); - static ref REGEX_OBJECTS_BATCH: Regex = Regex::new(r"/objects/batch$").unwrap(); - - static ref REGEX_OBJECTS_CHUNKIDS: Regex = Regex::new(r"/objects/chunkids$").unwrap(); - /// Git Protocol static ref INFO_REFS_REGEX: Regex = Regex::new(r"/info/refs$").unwrap(); static ref REGEX_GIT_UPLOAD_PACK: Regex = Regex::new(r"/git-upload-pack$").unwrap(); @@ -193,16 +156,10 @@ lazy_static! { pub async fn get_method_router( state: State, - Query(params): Query, + Query(params): Query, uri: Uri, ) -> Result, (StatusCode, String)> { - let lfs_config: LfsConfig = state.deref().to_owned().into(); - // Routing LFS services. - if OBJECTS_REGEX.is_match(uri.path()) { - lfs::lfs_download_object(&lfs_config, uri.path()).await - } else if LOCKS_REGEX.is_match(uri.path()) { - lfs::lfs_retrieve_lock(&lfs_config, params).await - } else if INFO_REFS_REGEX.is_match(uri.path()) { + if INFO_REFS_REGEX.is_match(uri.path()) { let pack_protocol = SmartProtocol::new( remove_git_suffix(uri, "/info/refs"), state.context.clone(), @@ -222,19 +179,7 @@ pub async fn post_method_router( uri: Uri, req: Request, ) -> Result { - let lfs_config: LfsConfig = state.deref().to_owned().into(); - // Routing LFS services. - if REGEX_LOCKS_VERIFY.is_match(uri.path()) { - lfs::lfs_verify_lock(state, &lfs_config, req).await - } else if LOCKS_REGEX.is_match(uri.path()) { - lfs::lfs_create_lock(state, &lfs_config, req).await - } else if REGEX_UNLOCK.is_match(uri.path()) { - lfs::lfs_delete_lock(state, &lfs_config, uri.path(), req).await - } else if REGEX_OBJECTS_BATCH.is_match(uri.path()) { - lfs::lfs_process_batch(state, &lfs_config, req).await - } else if REGEX_OBJECTS_CHUNKIDS.is_match(uri.path()) { - lfs::lfs_fetch_chunk_ids(state, &lfs_config, req).await - } else if REGEX_GIT_UPLOAD_PACK.is_match(uri.path()) { + if REGEX_GIT_UPLOAD_PACK.is_match(uri.path()) { let mut pack_protocol = SmartProtocol::new( remove_git_suffix(uri.clone(), "/git-upload-pack"), state.context.clone(), @@ -258,21 +203,5 @@ pub async fn post_method_router( } } -pub async fn put_method_router( - state: State, - uri: Uri, - req: Request, -) -> Result, (StatusCode, String)> { - let lfs_config: LfsConfig = state.deref().to_owned().into(); - if OBJECTS_REGEX.is_match(uri.path()) { - lfs::lfs_upload_object(&lfs_config, uri.path(), req).await - } else { - Err(( - StatusCode::NOT_FOUND, - String::from("Operation not supported"), - )) - } -} - #[cfg(test)] mod tests {} From 8828a027680909274a8a1b841c641467a4b1ca1d Mon Sep 17 00:00:00 2001 From: "benjamin.747" Date: Mon, 2 Sep 2024 17:00:43 +0800 Subject: [PATCH 2/2] [github action] add git config lfs.url in action config --- .github/workflows/base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 91dea4611..18cf1eebb 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -81,6 +81,7 @@ jobs: git lfs install git config --global user.email "mega@github.com" git config --global user.name "Mega" + git config --global lfs.url http://localhost:8000 - uses: actions-rs/cargo@v1 with: command: test