From 3a24d3e2bc832c6a2c4982f737ae83a654ee3c33 Mon Sep 17 00:00:00 2001 From: Neon Date: Fri, 13 Jun 2025 17:52:53 +0800 Subject: [PATCH 01/18] feat(vault): integrate vault with jupiter storage --- common/src/errors.rs | 8 +- jupiter/callisto/src/mod.rs | 1 + jupiter/callisto/src/vault.rs | 20 +++++ jupiter/migration/src/lib.rs | 2 + .../src/m20250610_000001_add_vault_storage.rs | 40 +++++++++ jupiter/src/context.rs | 9 +- jupiter/src/storage/mod.rs | 1 + jupiter/src/storage/vault_storage.rs | 89 +++++++++++++++++++ vault/Cargo.toml | 2 + vault/src/integration/jupiter_backend.rs | 63 +++++++++++++ vault/src/integration/mod.rs | 1 + vault/src/lib.rs | 2 + 12 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 jupiter/callisto/src/vault.rs create mode 100644 jupiter/migration/src/m20250610_000001_add_vault_storage.rs create mode 100644 jupiter/src/storage/vault_storage.rs create mode 100644 vault/src/integration/jupiter_backend.rs create mode 100644 vault/src/integration/mod.rs diff --git a/common/src/errors.rs b/common/src/errors.rs index 093da1c1e..5bced73b1 100644 --- a/common/src/errors.rs +++ b/common/src/errors.rs @@ -28,16 +28,16 @@ impl MegaError { panic!("{}", self.error.as_ref().unwrap()); } - pub fn unknown_subcommand(cmd: &str) -> MegaError { + pub fn unknown_subcommand(cmd: impl AsRef) -> MegaError { MegaError { - error: anyhow::anyhow!("Unknown subcommand: {}", cmd).into(), + error: anyhow::anyhow!("Unknown subcommand: {}", cmd.as_ref()).into(), code: 1, } } - pub fn with_message(msg: &str) -> MegaError { + pub fn with_message(msg: impl AsRef) -> MegaError { MegaError { - error: anyhow::anyhow!("Error Message: {}", msg).into(), + error: anyhow::anyhow!("Error Message: {}", msg.as_ref()).into(), code: 0, } } diff --git a/jupiter/callisto/src/mod.rs b/jupiter/callisto/src/mod.rs index 1ceda5a4c..552a3de39 100644 --- a/jupiter/callisto/src/mod.rs +++ b/jupiter/callisto/src/mod.rs @@ -34,3 +34,4 @@ pub mod relay_repo_info; pub mod sea_orm_active_enums; pub mod ssh_keys; pub mod user; +pub mod vault; diff --git a/jupiter/callisto/src/vault.rs b/jupiter/callisto/src/vault.rs new file mode 100644 index 000000000..8e548c248 --- /dev/null +++ b/jupiter/callisto/src/vault.rs @@ -0,0 +1,20 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.10 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "vault")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = true)] + pub id: i64, + #[sea_orm(unique, indexed)] + pub key: String, + #[sea_orm(column_type = "VarBinary(StringLen::None)")] + pub value: Vec, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/jupiter/migration/src/lib.rs b/jupiter/migration/src/lib.rs index 7571a0ba9..c3a0de635 100644 --- a/jupiter/migration/src/lib.rs +++ b/jupiter/migration/src/lib.rs @@ -4,6 +4,7 @@ use sea_orm_migration::schema::big_integer; mod m20250314_025943_init; mod m20250427_031332_add_mr_refs_tag; mod m20250605_013340_alter_mega_mr_index; +mod m20250610_000001_add_vault_storage; pub struct Migrator; @@ -14,6 +15,7 @@ impl MigratorTrait for Migrator { Box::new(m20250314_025943_init::Migration), Box::new(m20250427_031332_add_mr_refs_tag::Migration), Box::new(m20250605_013340_alter_mega_mr_index::Migration), + Box::new(m20250610_000001_add_vault_storage::Migration), ] } } diff --git a/jupiter/migration/src/m20250610_000001_add_vault_storage.rs b/jupiter/migration/src/m20250610_000001_add_vault_storage.rs new file mode 100644 index 000000000..f6ce51f9b --- /dev/null +++ b/jupiter/migration/src/m20250610_000001_add_vault_storage.rs @@ -0,0 +1,40 @@ +use sea_orm_migration::{ + prelude::*, + schema::{binary, string}, +}; + +use crate::pk_bigint; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Vault::Table) + .if_not_exists() + .col(pk_bigint(Vault::Id)) + .col(string(Vault::Key)) + .col(binary(Vault::Value)) + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, _: &SchemaManager) -> Result<(), DbErr> { + Ok(()) + } +} + +#[derive(DeriveIden)] +enum Vault { + Table, + Id, + Key, + Value, +} diff --git a/jupiter/src/context.rs b/jupiter/src/context.rs index c6b9a9426..72f745076 100644 --- a/jupiter/src/context.rs +++ b/jupiter/src/context.rs @@ -8,7 +8,7 @@ use crate::{ git_db_storage::GitDbStorage, init::database_connection, issue_storage::IssueStorage, lfs_db_storage::LfsDbStorage, mono_storage::MonoStorage, mq_storage::MQStorage, mr_storage::MrStorage, raw_db_storage::RawDbStorage, relay_storage::RelayStorage, - user_storage::UserStorage, + user_storage::UserStorage, vault_storage::VaultStorage, }, }; @@ -46,6 +46,10 @@ impl Context { self.services.lfs_file_storage() } + pub fn vault_stg(&self) -> VaultStorage { + self.services.vault_storage.clone() + } + pub fn mock() -> Self { Context { services: Service::mock(), @@ -63,6 +67,7 @@ pub struct Service { pub relay_storage: RelayStorage, pub mq_storage: MQStorage, user_storage: UserStorage, + pub vault_storage: VaultStorage, mr_storage: MrStorage, issue_storage: IssueStorage, lfs_file_storage: Arc, @@ -82,6 +87,7 @@ impl Service { user_storage: UserStorage::new(connection.clone()).await, mr_storage: MrStorage::new(connection.clone()).await, issue_storage: IssueStorage::new(connection.clone()).await, + vault_storage: VaultStorage::new(connection.clone()).await, lfs_file_storage: lfs_storage::init(config.lfs.clone(), lfs_db_storage.clone()).await, } } @@ -119,6 +125,7 @@ impl Service { relay_storage: RelayStorage::mock(), mq_storage: MQStorage::mock(), user_storage: UserStorage::mock(), + vault_storage: VaultStorage::mock(), lfs_file_storage: Arc::new(LocalStorage::mock()), mr_storage: MrStorage::mock(), issue_storage: IssueStorage::mock(), diff --git a/jupiter/src/storage/mod.rs b/jupiter/src/storage/mod.rs index d94cb9ce1..d7f63256c 100644 --- a/jupiter/src/storage/mod.rs +++ b/jupiter/src/storage/mod.rs @@ -8,6 +8,7 @@ pub mod mr_storage; pub mod raw_db_storage; pub mod relay_storage; pub mod user_storage; +pub mod vault_storage; use sea_orm::{sea_query::OnConflict, ActiveModelTrait, ConnectionTrait, DbErr, EntityTrait}; diff --git a/jupiter/src/storage/vault_storage.rs b/jupiter/src/storage/vault_storage.rs new file mode 100644 index 000000000..060420aaf --- /dev/null +++ b/jupiter/src/storage/vault_storage.rs @@ -0,0 +1,89 @@ +use std::sync::Arc; + +use callisto::vault::*; +use common::errors::MegaError; +use sea_orm::*; + +#[derive(Clone)] +pub struct VaultStorage { + pub connection: Arc, +} + +impl VaultStorage { + pub fn get_connection(&self) -> &DatabaseConnection { + &self.connection + } + + pub async fn new(connection: Arc) -> Self { + VaultStorage { connection } + } + + pub fn mock() -> Self { + VaultStorage { + connection: Arc::new(DatabaseConnection::default()), + } + } + + pub async fn list_keys(&self, prefix: impl AsRef) -> Result, MegaError> { + Entity::find() + .order_by_asc(Column::Key.like(format!("{}%", prefix.as_ref()).as_str())) + .select_column(Column::Key) + .into_tuple::() + .all(self.get_connection()) + .await + .map_err(|e| { + MegaError::with_message(format!( + "Failed to list vault with prefix: {}, {}", + prefix.as_ref(), + e + )) + }) + } + + pub async fn load(&self, key: impl AsRef) -> Result { + Entity::find() + .filter(Column::Key.eq(key.as_ref())) + .one(self.get_connection()) + .await? + .ok_or_else(|| { + MegaError::with_message(format!("Vault key '{}' not found", key.as_ref()).as_str()) + }) + } + + pub async fn save(&self, key: impl AsRef, value: Vec) -> Result<(), MegaError> { + let model = Model { + id: 0, + key: key.as_ref().to_string(), + value, + } + .into_active_model(); + + match model.save(self.get_connection()).await { + Ok(_) => Ok(()), + Err(e) => Err(MegaError::with_message(format!( + "Failed to save vault entry '{}': {}", + key.as_ref(), + e + ))), + } + } + + pub async fn delete(&self, key: impl AsRef) -> Result<(), MegaError> { + let model = Entity::find() + .filter(Column::Key.eq(key.as_ref())) + .one(self.get_connection()) + .await? + .ok_or_else(|| { + MegaError::with_message(format!("Vault key '{}' not found", key.as_ref()).as_str()) + })?; + + match model.delete(self.get_connection()).await { + Ok(_) => Ok(()), + Err(e) => Err(MegaError::with_message(format!( + "Failed to delete vault entry '{}': {}", + key.as_ref(), + e + ))), + } + } +} diff --git a/vault/Cargo.toml b/vault/Cargo.toml index 784f48ac7..62259b32d 100644 --- a/vault/Cargo.toml +++ b/vault/Cargo.toml @@ -5,6 +5,8 @@ edition = "2021" [dependencies] common = { workspace = true} +jupiter = { workspace = true } + rusty_vault = { workspace = true } serde_json = { workspace = true } go-defer = { workspace = true } diff --git a/vault/src/integration/jupiter_backend.rs b/vault/src/integration/jupiter_backend.rs new file mode 100644 index 000000000..0ed8770da --- /dev/null +++ b/vault/src/integration/jupiter_backend.rs @@ -0,0 +1,63 @@ +use jupiter::context::Context; +use rusty_vault::storage::Backend; +use tokio::runtime::Handle; + +pub struct JupiterBackend { + ctx: Context, + rt: Handle, +} + +impl JupiterBackend { + pub fn new(ctx: Context) -> Self { + let rt = tokio::runtime::Handle::current(); + JupiterBackend { ctx, rt } + } +} + +impl Backend for JupiterBackend { + fn list(&self, prefix: &str) -> Result, rusty_vault::errors::RvError> { + let service = self.ctx.vault_stg(); + self.rt.block_on(async move { + match service.list_keys(prefix).await { + Ok(keys) => Ok(keys), + Err(_) => Err(rusty_vault::errors::RvError::ErrPhysicalBackendKeyInvalid), + } + }) + } + + fn get(&self, key: &str) -> Result, rusty_vault::errors::RvError> { + let service = self.ctx.vault_stg(); + self.rt.block_on(async move { + match service.load(key).await { + Ok(model) => { + let entry = rusty_vault::storage::BackendEntry { + key: model.key, + value: model.value, + }; + Ok(Some(entry)) + } + Err(_) => Err(rusty_vault::errors::RvError::ErrPhysicalBackendKeyInvalid), + } + }) + } + + fn put(&self, entry: &rusty_vault::storage::BackendEntry) -> Result<(), rusty_vault::errors::RvError> { + let service = self.ctx.vault_stg(); + self.rt.block_on(async move { + match service.save(&entry.key, entry.value.clone()).await { + Ok(_) => Ok(()), + Err(_) => Err(rusty_vault::errors::RvError::ErrPhysicalBackendKeyInvalid), + } + }) + } + + fn delete(&self, key: &str) -> Result<(), rusty_vault::errors::RvError> { + let service = self.ctx.vault_stg(); + self.rt.block_on(async move { + match service.delete(key).await { + Ok(_) => Ok(()), + Err(_) => Err(rusty_vault::errors::RvError::ErrPhysicalBackendKeyInvalid), + } + }) + } +} diff --git a/vault/src/integration/mod.rs b/vault/src/integration/mod.rs new file mode 100644 index 000000000..0cce9c672 --- /dev/null +++ b/vault/src/integration/mod.rs @@ -0,0 +1 @@ +pub mod jupiter_backend; diff --git a/vault/src/lib.rs b/vault/src/lib.rs index bd6180446..a0cd702b5 100644 --- a/vault/src/lib.rs +++ b/vault/src/lib.rs @@ -1,3 +1,5 @@ +pub mod integration; + pub mod nostr; pub mod pgp; pub mod pki; From 608c21b3d32fda5d135bbbc301f03260fc96c669 Mon Sep 17 00:00:00 2001 From: Neon Date: Fri, 13 Jun 2025 19:21:39 +0800 Subject: [PATCH 02/18] vault: update dep version --- vault/Cargo.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vault/Cargo.toml b/vault/Cargo.toml index 62259b32d..97ff53e26 100644 --- a/vault/Cargo.toml +++ b/vault/Cargo.toml @@ -7,7 +7,6 @@ edition = "2021" common = { workspace = true} jupiter = { workspace = true } -rusty_vault = { workspace = true } serde_json = { workspace = true } go-defer = { workspace = true } openssl = { workspace = true } @@ -20,3 +19,8 @@ tokio = { workspace = true, features = ["full"] } async-std = { workspace = true } pgp = { workspace = true } smallvec = { workspace = true } + +[dependencies.rusty_vault] +git = "https://github.com/Tongsuo-Project/RustyVault" +tag = "v0.2.2" +features = ["sync_handler"] From de2ba8c3264d78a443f4eff067802df58a050ec4 Mon Sep 17 00:00:00 2001 From: Neon Date: Sun, 15 Jun 2025 09:41:00 +0800 Subject: [PATCH 03/18] vault: make it sync --- vault/src/pki.rs | 13 ++++--------- vault/src/vault.rs | 28 ++++++++++++---------------- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/vault/src/pki.rs b/vault/src/pki.rs index 8018ce3bd..6a14a1d2b 100644 --- a/vault/src/pki.rs +++ b/vault/src/pki.rs @@ -24,7 +24,6 @@ async fn init_ca() -> CoreInfo { // init CA if not let token = &c.token; if read_api(&c.core.read().unwrap(), token, "pki/ca/pem") - .await .is_err() { // err = not found @@ -62,7 +61,7 @@ async fn config_ca(core: Arc>, token: &str) { .unwrap() .clone(); - let resp = write_api(&core, token, "sys/mounts/pki/", Some(mount_data)).await; + let resp = write_api(&core, token, "sys/mounts/pki/", Some(mount_data)); assert!(resp.is_ok()); } @@ -83,7 +82,6 @@ pub async fn config_role(core: Arc>, token: &str, data: Value) { &format!("pki/roles/{}", ROLE), Some(role_data), ) - .await }); assert!(result.is_ok()); } @@ -117,8 +115,7 @@ async fn generate_root(core: Arc>, token: &str, exported: bool) { ) .as_str(), Some(req_data), - ) - .await; + ); assert!(resp.is_ok()); } @@ -143,7 +140,6 @@ pub async fn issue_cert(data: Value) -> (String, String) { &format!("pki/issue/{}", ROLE), Some(issue_data), ) - .await }); assert!(resp.is_ok()); let resp_body = resp.unwrap(); @@ -187,7 +183,6 @@ pub async fn get_root_cert() -> String { let resp_ca_pem = async_std::task::block_on(async { let core = ca().await.core.read().unwrap(); read_api(&core, &ca().await.token, "pki/ca/pem") - .await .unwrap() .unwrap() }); @@ -254,7 +249,7 @@ mod tests_raw { let mut req = Request::new(path); req.operation = Operation::Read; req.client_token = token.to_string(); - let resp = core.handle_request(&mut req).await; + let resp = core.handle_request(&mut req); assert_eq!(resp.is_ok(), is_ok); resp } @@ -271,7 +266,7 @@ mod tests_raw { req.client_token = token.to_string(); req.body = data; - let resp = core.handle_request(&mut req).await; + let resp = core.handle_request(&mut req); println!("path: {}, req.body: {:?}", path, req.body); assert_eq!(resp.is_ok(), is_ok); resp diff --git a/vault/src/vault.rs b/vault/src/vault.rs index 35dc4c237..de9b2325b 100644 --- a/vault/src/vault.rs +++ b/vault/src/vault.rs @@ -109,14 +109,14 @@ fn init() -> CoreInfo { } } -pub async fn read_api(core: &Core, token: &str, path: &str) -> Result, RvError> { +pub fn read_api(core: &Core, token: &str, path: &str) -> Result, RvError> { let mut req = Request::new(path); req.operation = Operation::Read; req.client_token = token.to_string(); - core.handle_request(&mut req).await // !Send + core.handle_request(&mut req) } -pub async fn write_api( +pub fn write_api( core: &Core, token: &str, path: &str, @@ -127,16 +127,16 @@ pub async fn write_api( req.client_token = token.to_string(); req.body = data; - let resp = core.handle_request(&mut req).await; // !Send + let resp = core.handle_request(&mut req); println!("path: {}, req.body: {:?}", path, req.body); resp } -pub async fn delete_api(core: &Core, token: &str, path: &str) -> Result, RvError> { +pub fn delete_api(core: &Core, token: &str, path: &str) -> Result, RvError> { let mut req = Request::new(path); req.operation = Operation::Delete; req.client_token = token.to_string(); - core.handle_request(&mut req).await // !Send + core.handle_request(&mut req) } /// Write a secret to the vault (k-v) @@ -144,33 +144,30 @@ pub async fn write_secret( name: &str, data: Option>, ) -> Result, RvError> { - // async_std: stop spread of `!Send` (RwLockReadGuard cross .await), for `tokio::spawn` - async_std::task::block_on(write_api( + write_api( &CORE.core.read().unwrap(), &CORE.token, &format!("secret/{}", name), data, - )) + ) } /// Read a secret from the vault (k-v) pub async fn read_secret(name: &str) -> Result, RvError> { - // async_std: stop spread of `!Send` (RwLockReadGuard cross .await), for `tokio::spawn` - async_std::task::block_on(read_api( + read_api( &CORE.core.read().unwrap(), &CORE.token, &format!("secret/{}", name), - )) + ) } /// Delete a secret from the vault (k-v) pub async fn delete_secret(name: &str) -> Result, RvError> { - // async_std: stop spread of `!Send` (RwLockReadGuard cross .await), for `tokio::spawn` - async_std::task::block_on(delete_api( + delete_api( &CORE.core.read().unwrap(), &CORE.token, &format!("secret/{}", name), - )) + ) } #[cfg(test)] @@ -200,7 +197,6 @@ mod tests { assert!(read_secret("foo").await.unwrap().is_none()); assert!( read_api(&CORE.core.read().unwrap(), &CORE.token, "secret1/foo") - .await .is_err() ); } From bfbab323e38e5824248be3c25496efc7ed6e26a1 Mon Sep 17 00:00:00 2001 From: Neon Date: Sun, 15 Jun 2025 09:58:36 +0800 Subject: [PATCH 04/18] monobean: use the same dirs from mega::common --- monobean/src/config.rs | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/monobean/src/config.rs b/monobean/src/config.rs index e731cfa78..08f4de201 100644 --- a/monobean/src/config.rs +++ b/monobean/src/config.rs @@ -80,9 +80,9 @@ macro_rules! get_setting { /// The directory is determined in the following priority order: /// 1. Uses the `MONOBEAN_BASE_DIR` environment variable if set /// 2. Falls back to system default paths when environment variable is not set: -/// - On Linux: `~/.local/share/monobean` -/// - On Windows: `C:\Users\{UserName}\AppData\Local\monobean` -/// - On macOS: `~/Library/Application Support/monobean` +/// - On Linux: `~/.local/share/mega/monobean` +/// - On Windows: `C:\Users\{UserName}\AppData\Local\mega\monobean` +/// - On macOS: `~/Library/Application Support/mega/monobean` /// /// # Returns /// A PathBuf containing the base directory path @@ -94,16 +94,9 @@ macro_rules! get_setting { /// pub fn monobean_base() -> PathBuf { // Get the base directory from the environment variable or use the default - let base_dir = std::env::var("MONOBEAN_BASE_DIR").unwrap_or_else(|_| { - let base_dirs = directories::BaseDirs::new().unwrap(); - base_dirs - .data_local_dir() - .join("monobean") - .to_str() - .unwrap() - .to_string() - }); - PathBuf::from(base_dir) + std::env::var("MONOBEAN_BASE_DIR") + .map(|inner| PathBuf::from(inner)) + .unwrap_or_else(|_| common::config::mega_base().join("monobean")) } /// Retrieves the cache directory path for Monobean @@ -111,9 +104,9 @@ pub fn monobean_base() -> PathBuf { /// The directory is determined in the following priority order: /// 1. Uses the `MONOBEAN_CACHE_DIR` environment variable if set /// 2. Falls back to system default paths when environment variable is not set: -/// - On Linux: `~/.cache/monobean` -/// - On Windows: `C:\Users\{username}\AppData\Local\Cache\monobean` -/// - On macOS: `~/Library/Caches/monobean` +/// - On Linux: `~/.cache/mega/monobean` +/// - On Windows: `C:\Users\{username}\AppData\Local\Cache\mega\monobean` +/// - On macOS: `~/Library/Caches/mega/monobean` /// /// # Returns /// A PathBuf containing the cache directory path @@ -125,16 +118,9 @@ pub fn monobean_base() -> PathBuf { /// pub fn monobean_cache() -> PathBuf { // Get the cache directory from the environment variable or use the default - let cache_dir = std::env::var("MONOBEAN_CACHE_DIR").unwrap_or_else(|_| { - let base_dirs = directories::BaseDirs::new().unwrap(); - base_dirs - .cache_dir() - .join("monobean") - .to_str() - .unwrap() - .to_string() - }); - PathBuf::from(cache_dir) + std::env::var("MONOBEAN_CACHE_DIR") + .map(|inner| PathBuf::from(inner)) + .unwrap_or_else(|_| common::config::mega_cache().join("monobean")) } /// TODO: So ugly... From 43c1485360f0562ed34b712546e048b92fb93b58 Mon Sep 17 00:00:00 2001 From: Neon Date: Sun, 15 Jun 2025 16:48:07 +0800 Subject: [PATCH 05/18] jupiter: merged migrator --- Cargo.toml | 3 - gateway/Cargo.toml | 3 +- gateway/src/api/github_router.rs | 12 +- jupiter/{migration => }/.env | 0 jupiter/Cargo.toml | 9 +- jupiter/README.md | 54 +++++++ jupiter/migration/Cargo.toml | 19 --- jupiter/migration/README.md | 52 ------- jupiter/migration/src/lib.rs | 25 --- jupiter/migration/src/main.rs | 6 - jupiter/src/context.rs | 147 +++++++++--------- jupiter/src/lib.rs | 1 + jupiter/src/main.rs | 7 + .../migrator}/m20250314_025943_init.rs | 2 +- .../m20250427_031332_add_mr_refs_tag.rs | 0 .../m20250605_013340_alter_mega_mr_index.rs | 0 .../m20250610_000001_add_vault_storage.rs | 2 +- jupiter/src/migrator/mod.rs | 108 +++++++++++++ jupiter/src/storage/init.rs | 6 +- 19 files changed, 257 insertions(+), 199 deletions(-) rename jupiter/{migration => }/.env (100%) delete mode 100644 jupiter/migration/Cargo.toml delete mode 100644 jupiter/migration/README.md delete mode 100644 jupiter/migration/src/lib.rs delete mode 100644 jupiter/migration/src/main.rs create mode 100644 jupiter/src/main.rs rename jupiter/{migration/src => src/migrator}/m20250314_025943_init.rs (99%) rename jupiter/{migration/src => src/migrator}/m20250427_031332_add_mr_refs_tag.rs (100%) rename jupiter/{migration/src => src/migrator}/m20250605_013340_alter_mega_mr_index.rs (100%) rename jupiter/{migration/src => src/migrator}/m20250610_000001_add_vault_storage.rs (96%) create mode 100644 jupiter/src/migrator/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 2196c5bbc..83a9e1616 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,6 @@ members = [ "common", "mercury", "jupiter", - "jupiter/migration", "ceres", "libra", "vault", @@ -28,7 +27,6 @@ mercury = { path = "mercury" } jupiter = { path = "jupiter" } ceres = { path = "ceres" } callisto = { path = "jupiter/callisto" } -migration = { path = "jupiter/migration" } gemini = { path = "gemini" } vault = { path = "vault" } saturn = { path = "saturn" } @@ -50,7 +48,6 @@ tokio-stream = "0.1.17" tokio-test = "0.4.4" tokio-util = "0.7.15" clap = "4.5.39" -async-std = "1.13.1" async-trait = "0.1.88" async-stream = "0.3.6" bytes = "1.10.1" diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 9c84975d5..a7d38fe02 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -29,9 +29,8 @@ tower-http = { workspace = true, features = [ ] } tokio = { workspace = true, features = ["net"] } reqwest = { workspace = true, features = ["json"] } -lazy_static = { workspace = true } chrono = { workspace = true } quinn = { workspace = true } utoipa = { workspace = true, features = ["axum_extras"] } utoipa-axum = { workspace = true } -utoipa-swagger-ui = { workspace = true, features = ["axum"] } \ No newline at end of file +utoipa-swagger-ui = { workspace = true, features = ["axum"] } diff --git a/gateway/src/api/github_router.rs b/gateway/src/api/github_router.rs index bd2a60329..785af3ce2 100644 --- a/gateway/src/api/github_router.rs +++ b/gateway/src/api/github_router.rs @@ -3,18 +3,10 @@ use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; use axum::routing::post; use axum::Json; -use lazy_static::lazy_static; use reqwest::Client; use serde_json::Value; use utoipa_axum::router::OpenApiRouter; -lazy_static! { - static ref CLIENT: Client = Client::builder() - .user_agent("Mega/0.0.1") // IMPORTANT, or 403 Forbidden - .build() - .unwrap(); -} - pub fn routers() -> OpenApiRouter { OpenApiRouter::new().route("/github/webhook", post(webhook)) } @@ -49,6 +41,10 @@ pub async fn get_pr_commits(pr_url: &str) -> Value { /// Send a GET request to the given URL and return the JSON response. async fn get_request(url: &str) -> Value { + static CLIENT: Client = Client::builder() + .user_agent("Mega/0.0.1") // IMPORTANT, or 403 Forbidden + .build() + .unwrap(); let resp = CLIENT.get(url).send().await.unwrap(); resp.json().await.unwrap() } diff --git a/jupiter/migration/.env b/jupiter/.env similarity index 100% rename from jupiter/migration/.env rename to jupiter/.env diff --git a/jupiter/Cargo.toml b/jupiter/Cargo.toml index b3c3ed3b1..cebd789e3 100644 --- a/jupiter/Cargo.toml +++ b/jupiter/Cargo.toml @@ -9,13 +9,15 @@ edition = "2021" name = "jupiter" path = "src/lib.rs" +[[bin]] +name = "migrator" +path = "src/main.rs" [dependencies] callisto = { workspace = true } common = { workspace = true } mercury = { workspace = true } saturn = { workspace = true } -migration = { workspace = true } sea-orm = { workspace = true, features = [ "sqlx-postgres", @@ -24,6 +26,8 @@ sea-orm = { workspace = true, features = [ "runtime-tokio-rustls", "macros", ] } +sea-orm-migration = { workspace = true } + tracing = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } @@ -39,6 +43,3 @@ ring = { workspace = true } aws-config = { workspace = true, features = ["behavior-version-latest"] } aws-sdk-s3 = { workspace = true, features = ["rt-tokio"] } anyhow = { workspace = true } - -[dev-dependencies] -tokio = { workspace = true, features = ["macros"] } diff --git a/jupiter/README.md b/jupiter/README.md index 01cee69f4..d0e97df0f 100644 --- a/jupiter/README.md +++ b/jupiter/README.md @@ -1 +1,55 @@ ## Jupiter Module - Monorepo and Mega Database Storage Engine + +### Migration Guideline +1. Generate entity files +sea-orm-cli migrate generate + +```bash + +cd mega/jupiter/src/migration + +sea-orm-cli generate entity -u postgres://username:password@hostname/mono -o ../callisto/src --with-serde both + +``` + +2. Running Migrator CLI + +- Generate a new migration file + ```sh + cargo run -- generate MIGRATION_NAME + ``` +- Apply all pending migrations + ```sh + cargo run + ``` + ```sh + cargo run -- up + ``` +- Apply first 10 pending migrations + ```sh + cargo run -- up -n 10 + ``` +- Rollback last applied migrations + ```sh + cargo run -- down + ``` +- Rollback last 10 applied migrations + ```sh + cargo run -- down -n 10 + ``` +- Drop all tables from the database, then reapply all migrations + ```sh + cargo run -- fresh + ``` +- Rollback all applied migrations, then reapply all migrations + ```sh + cargo run -- refresh + ``` +- Rollback all applied migrations + ```sh + cargo run -- reset + ``` +- Check the status of all migrations + ```sh + cargo run -- status + ``` diff --git a/jupiter/migration/Cargo.toml b/jupiter/migration/Cargo.toml deleted file mode 100644 index 319517f31..000000000 --- a/jupiter/migration/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "migration" -version = "0.1.0" -edition = "2024" -publish = false - -[lib] -name = "migration" -path = "src/lib.rs" - -[dependencies] -async-std = { workspace = true, features = ["attributes", "tokio1"] } -sea-orm-migration = { workspace = true, features = [ - # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. - # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. - # e.g. - "runtime-tokio-rustls", # `ASYNC_RUNTIME` feature - "sqlx-postgres", # `DATABASE_DRIVER` feature -]} \ No newline at end of file diff --git a/jupiter/migration/README.md b/jupiter/migration/README.md deleted file mode 100644 index 6a363b772..000000000 --- a/jupiter/migration/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Generate entity files -sea-orm-cli migrate generate - -```bash - -cd mega/jupiter/migration - -sea-orm-cli generate entity -u postgres://username:password@hostname/mono -o ../callisto/src --with-serde both - -``` - -# Running Migrator CLI - -- Generate a new migration file - ```sh - cargo run -- generate MIGRATION_NAME - ``` -- Apply all pending migrations - ```sh - cargo run - ``` - ```sh - cargo run -- up - ``` -- Apply first 10 pending migrations - ```sh - cargo run -- up -n 10 - ``` -- Rollback last applied migrations - ```sh - cargo run -- down - ``` -- Rollback last 10 applied migrations - ```sh - cargo run -- down -n 10 - ``` -- Drop all tables from the database, then reapply all migrations - ```sh - cargo run -- fresh - ``` -- Rollback all applied migrations, then reapply all migrations - ```sh - cargo run -- refresh - ``` -- Rollback all applied migrations - ```sh - cargo run -- reset - ``` -- Check the status of all migrations - ```sh - cargo run -- status - ``` diff --git a/jupiter/migration/src/lib.rs b/jupiter/migration/src/lib.rs deleted file mode 100644 index c3a0de635..000000000 --- a/jupiter/migration/src/lib.rs +++ /dev/null @@ -1,25 +0,0 @@ -pub use sea_orm_migration::prelude::*; -use sea_orm_migration::schema::big_integer; - -mod m20250314_025943_init; -mod m20250427_031332_add_mr_refs_tag; -mod m20250605_013340_alter_mega_mr_index; -mod m20250610_000001_add_vault_storage; - -pub struct Migrator; - -#[async_trait::async_trait] -impl MigratorTrait for Migrator { - fn migrations() -> Vec> { - vec![ - Box::new(m20250314_025943_init::Migration), - Box::new(m20250427_031332_add_mr_refs_tag::Migration), - Box::new(m20250605_013340_alter_mega_mr_index::Migration), - Box::new(m20250610_000001_add_vault_storage::Migration), - ] - } -} - -pub fn pk_bigint(name: T) -> ColumnDef { - big_integer(name).primary_key().take() -} diff --git a/jupiter/migration/src/main.rs b/jupiter/migration/src/main.rs deleted file mode 100644 index c6b6e48db..000000000 --- a/jupiter/migration/src/main.rs +++ /dev/null @@ -1,6 +0,0 @@ -use sea_orm_migration::prelude::*; - -#[async_std::main] -async fn main() { - cli::run_cli(migration::Migrator).await; -} diff --git a/jupiter/src/context.rs b/jupiter/src/context.rs index 72f745076..efda0f255 100644 --- a/jupiter/src/context.rs +++ b/jupiter/src/context.rs @@ -12,71 +12,26 @@ use crate::{ }, }; -#[derive(Clone)] -pub struct Context { - pub services: Arc, - pub config: Arc, -} - -impl Context { - pub async fn new(config: Arc) -> Self { - Context { - services: Service::shared(&config).await, - config, - } - } - - pub fn user_stg(&self) -> UserStorage { - self.services.user_storage() - } - - pub fn issue_stg(&self) -> IssueStorage { - self.services.issue_storage() - } - - pub fn mr_stg(&self) -> MrStorage { - self.services.mr_storage() - } - - pub fn lfs_stg(&self) -> LfsDbStorage { - self.services.lfs_db_storage() - } - - pub fn lfs_file_stg(&self) -> Arc { - self.services.lfs_file_storage() - } - - pub fn vault_stg(&self) -> VaultStorage { - self.services.vault_storage.clone() - } - - pub fn mock() -> Self { - Context { - services: Service::mock(), - config: Arc::new(Config::mock()), - } - } -} - #[derive(Clone)] pub struct Service { pub mono_storage: MonoStorage, pub git_db_storage: GitDbStorage, pub raw_db_storage: RawDbStorage, - lfs_db_storage: LfsDbStorage, + pub lfs_db_storage: LfsDbStorage, pub relay_storage: RelayStorage, pub mq_storage: MQStorage, - user_storage: UserStorage, + pub user_storage: UserStorage, pub vault_storage: VaultStorage, - mr_storage: MrStorage, - issue_storage: IssueStorage, - lfs_file_storage: Arc, + pub mr_storage: MrStorage, + pub issue_storage: IssueStorage, + pub lfs_file_storage: Arc, } impl Service { async fn new(config: &Config) -> Self { let connection = Arc::new(database_connection(&config.database).await); let lfs_db_storage = LfsDbStorage::new(connection.clone()).await; + Self { mono_storage: MonoStorage::new(connection.clone()).await, git_db_storage: GitDbStorage::new(connection.clone()).await, @@ -92,30 +47,6 @@ impl Service { } } - async fn shared(config: &Config) -> Arc { - Arc::new(Self::new(config).await) - } - - fn issue_storage(&self) -> IssueStorage { - self.issue_storage.clone() - } - - fn mr_storage(&self) -> MrStorage { - self.mr_storage.clone() - } - - fn user_storage(&self) -> UserStorage { - self.user_storage.clone() - } - - fn lfs_db_storage(&self) -> LfsDbStorage { - self.lfs_db_storage.clone() - } - - fn lfs_file_storage(&self) -> Arc { - self.lfs_file_storage.clone() - } - fn mock() -> Arc { Arc::new(Self { mono_storage: MonoStorage::mock(), @@ -132,3 +63,69 @@ impl Service { }) } } + +#[derive(Clone)] +pub struct Context { + pub services: Arc, + pub config: Arc, +} + +impl Context { + pub async fn new(config: Arc) -> Self { + Context { + services: Service::new(&config).await.into(), + config, + } + } + + pub fn mono_storage(&self) -> MonoStorage { + self.services.mono_storage.clone() + } + + pub fn git_db_storage(&self) -> GitDbStorage { + self.services.git_db_storage.clone() + } + + pub fn raw_db_storage(&self) -> RawDbStorage { + self.services.raw_db_storage.clone() + } + + pub fn lfs_db_storage(&self) -> LfsDbStorage { + self.services.lfs_db_storage.clone() + } + + pub fn relay_storage(&self) -> RelayStorage { + self.services.relay_storage.clone() + } + + pub fn mq_storage(&self) -> MQStorage { + self.services.mq_storage.clone() + } + + pub fn user_storage(&self) -> UserStorage { + self.services.user_storage.clone() + } + + pub fn vault_storage(&self) -> VaultStorage { + self.services.vault_storage.clone() + } + + pub fn mr_storage(&self) -> MrStorage { + self.services.mr_storage.clone() + } + + pub fn issue_storage(&self) -> IssueStorage { + self.services.issue_storage.clone() + } + + pub fn lfs_file_storage(&self) -> Arc { + self.services.lfs_file_storage.clone() + } + + pub fn mock() -> Self { + Context { + services: Service::mock(), + config: Arc::new(Config::mock()), + } + } +} diff --git a/jupiter/src/lib.rs b/jupiter/src/lib.rs index 982170302..5560803cf 100644 --- a/jupiter/src/lib.rs +++ b/jupiter/src/lib.rs @@ -1,4 +1,5 @@ pub mod context; pub mod lfs_storage; pub mod storage; +pub mod migrator; pub mod utils; diff --git a/jupiter/src/main.rs b/jupiter/src/main.rs new file mode 100644 index 000000000..10ff6d532 --- /dev/null +++ b/jupiter/src/main.rs @@ -0,0 +1,7 @@ +use jupiter::migrator; +use sea_orm_migration::prelude::*; + +#[tokio::main] +async fn main() { + cli::run_cli(migrator::Migrator).await; +} diff --git a/jupiter/migration/src/m20250314_025943_init.rs b/jupiter/src/migrator/m20250314_025943_init.rs similarity index 99% rename from jupiter/migration/src/m20250314_025943_init.rs rename to jupiter/src/migrator/m20250314_025943_init.rs index dc967aecb..7b6480f96 100644 --- a/jupiter/migration/src/m20250314_025943_init.rs +++ b/jupiter/src/migrator/m20250314_025943_init.rs @@ -5,7 +5,7 @@ use sea_orm_migration::{ sea_orm::{DatabaseBackend, EnumIter, Iterable}, }; -use crate::pk_bigint; +use crate::migrator::pk_bigint; #[derive(DeriveMigrationName)] pub struct Migration; diff --git a/jupiter/migration/src/m20250427_031332_add_mr_refs_tag.rs b/jupiter/src/migrator/m20250427_031332_add_mr_refs_tag.rs similarity index 100% rename from jupiter/migration/src/m20250427_031332_add_mr_refs_tag.rs rename to jupiter/src/migrator/m20250427_031332_add_mr_refs_tag.rs diff --git a/jupiter/migration/src/m20250605_013340_alter_mega_mr_index.rs b/jupiter/src/migrator/m20250605_013340_alter_mega_mr_index.rs similarity index 100% rename from jupiter/migration/src/m20250605_013340_alter_mega_mr_index.rs rename to jupiter/src/migrator/m20250605_013340_alter_mega_mr_index.rs diff --git a/jupiter/migration/src/m20250610_000001_add_vault_storage.rs b/jupiter/src/migrator/m20250610_000001_add_vault_storage.rs similarity index 96% rename from jupiter/migration/src/m20250610_000001_add_vault_storage.rs rename to jupiter/src/migrator/m20250610_000001_add_vault_storage.rs index f6ce51f9b..7491d70cf 100644 --- a/jupiter/migration/src/m20250610_000001_add_vault_storage.rs +++ b/jupiter/src/migrator/m20250610_000001_add_vault_storage.rs @@ -3,7 +3,7 @@ use sea_orm_migration::{ schema::{binary, string}, }; -use crate::pk_bigint; +use crate::migrator::pk_bigint; #[derive(DeriveMigrationName)] pub struct Migration; diff --git a/jupiter/src/migrator/mod.rs b/jupiter/src/migrator/mod.rs new file mode 100644 index 000000000..9f90febdf --- /dev/null +++ b/jupiter/src/migrator/mod.rs @@ -0,0 +1,108 @@ +//! Database migration module for the Jupiter application. +//! +//! This module provides database migration functionality using SeaORM's migration framework. +//! It contains all migration files and utilities for managing database schema changes. +//! +//! # Overview +//! +//! The migrator handles database schema evolution through versioned migration files. +//! Each migration is represented as a separate module and implements the `MigrationTrait`. +//! +//! # Migration Files +//! +//! - `m20250314_025943_init` - Initial database schema setup +//! - `m20250427_031332_add_mr_refs_tag` - Adds merge request reference tagging +//! - `m20250605_013340_alter_mega_mr_index` - Modifies merge request indexing +//! - `m20250610_000001_add_vault_storage` - Adds vault storage functionality +//! +//! # Usage +//! +//! ```rust +//! use jupiter::migrator::apply_migrations; +//! +//! // Apply pending migrations +//! apply_migrations(&db, false).await?; +//! +//! // Refresh all migrations (development only) +//! apply_migrations(&db, true).await?; +//! ``` +//! +//! # Safety +//! + +/// Creates a primary key column definition with big integer type. +/// +/// # Arguments +/// +/// * `name` - The name of the column that implements `IntoIden` +/// +/// # Returns +/// +/// A `ColumnDef` configured as a primary key big integer column + +/// The main migrator struct that implements the migration trait. +/// +/// This struct is responsible for managing all database migrations in the correct order. + +/// Applies database migrations to the given database connection. +/// +/// # Arguments +/// +/// * `db` - Reference to the database connection +/// * `refresh` - If true, refreshes all migrations (drops and recreates). If false, applies pending migrations only +/// +/// # Returns +/// +/// * `Ok(())` - If migrations were applied successfully +/// * `Err(MegaError)` - If migration failed, with error details logged +/// +/// # Errors +/// +/// Returns `MegaError` when: +/// - Database connection fails +/// - Migration SQL execution fails +/// - Schema validation errors occur +use sea_orm::DatabaseConnection; +use sea_orm_migration::prelude::*; +use sea_orm_migration::schema::big_integer; +use tracing::log; + +use common::errors::MegaError; + +mod m20250314_025943_init; +mod m20250427_031332_add_mr_refs_tag; +mod m20250605_013340_alter_mega_mr_index; +mod m20250610_000001_add_vault_storage; + +pub(self) fn pk_bigint(name: T) -> ColumnDef { + big_integer(name).primary_key().take() +} + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![ + Box::new(m20250314_025943_init::Migration), + Box::new(m20250427_031332_add_mr_refs_tag::Migration), + Box::new(m20250605_013340_alter_mega_mr_index::Migration), + Box::new(m20250610_000001_add_vault_storage::Migration), + ] + } +} + + +pub async fn apply_migrations( + db: &DatabaseConnection, + refresh: bool, +) -> Result<(), MegaError> { + match refresh { + true => Migrator::refresh(db).await, + false => Migrator::up(db, None).await, + } + .map_err(|e| { + log::error!("Failed to apply migrations: {}", e); + e.into() + }) +} diff --git a/jupiter/src/storage/init.rs b/jupiter/src/storage/init.rs index f6548bb10..b87e54422 100644 --- a/jupiter/src/storage/init.rs +++ b/jupiter/src/storage/init.rs @@ -1,10 +1,10 @@ +use common::config::DbConfig; use common::errors::MegaError; -use migration::{Migrator, MigratorTrait}; use sea_orm::{ConnectOptions, Database, DatabaseConnection}; use std::{path::Path, time::Duration}; use tracing::log; -use common::config::DbConfig; +use crate::migrator::apply_migrations; use crate::utils::id_generator; @@ -27,7 +27,7 @@ pub async fn database_connection(db_config: &DbConfig) -> DatabaseConnection { } else { sqlite_connection(db_config).await.unwrap() }; - Migrator::up(&conn, None).await.unwrap(); + apply_migrations(&conn, false).await.expect("Failed to apply migrations"); conn } From 353f577af6fce6354c217f5081c7dbf4f3618796 Mon Sep 17 00:00:00 2001 From: Neon Date: Sun, 15 Jun 2025 18:06:43 +0800 Subject: [PATCH 06/18] vault: reconstructed core integration --- vault/src/integration/vault_core.rs | 158 ++++++++++++++++++++++ vault/src/vault.rs | 203 ---------------------------- 2 files changed, 158 insertions(+), 203 deletions(-) create mode 100644 vault/src/integration/vault_core.rs delete mode 100644 vault/src/vault.rs diff --git a/vault/src/integration/vault_core.rs b/vault/src/integration/vault_core.rs new file mode 100644 index 000000000..27a10d3d2 --- /dev/null +++ b/vault/src/integration/vault_core.rs @@ -0,0 +1,158 @@ +use std::sync::{Arc, RwLock}; + +use crate::integration::jupiter_backend::JupiterBackend; +use common::errors::MegaError; +use jupiter::context::Context; + +use rusty_vault::{ + core::Core, + logical::{Operation, Request, Response}, + storage::{barrier_aes_gcm, Backend}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use tracing::log; + +const CORE_KEY_FILE: &str = "core_key.json"; // where the core key is stored, like `root_token` + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CoreKey { + secret_shares: Vec>, + root_token: String, +} + +#[derive(Clone)] +pub struct VaultCore { + core: Arc>, + key: CoreKey, +} + +impl VaultCore { + pub fn new(ctx: Context) -> Self { + let dir = common::config::mega_base().join("vault"); + let key_path = dir.join(CORE_KEY_FILE); + + std::fs::create_dir_all(&dir).expect("Failed to create vault directory"); + + let backend: Arc = Arc::new(JupiterBackend::new(ctx)); + let barrier = barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend)); + let seal_config = rusty_vault::core::SealConfig { + secret_shares: 10, + secret_threshold: 5, + }; + + let core = Core { + physical: backend, + barrier: Arc::new(barrier), + ..Default::default() + }; + let core = Arc::new(RwLock::new(core)); + + let key = { + let mut managed_core = core.write().unwrap(); + managed_core + .config(core.clone(), None) + .expect("Failed to configure vault core"); + + let core_key = if !managed_core + .inited() + .expect("Failed to check if vault is initialized") + { + let result = managed_core + .init(&seal_config) + .expect("Failed to initialize vault"); + let core_key = CoreKey { + secret_shares: Vec::from(&result.secret_shares[..]), + root_token: result.root_token, + }; + let file = std::fs::File::create(key_path).unwrap(); + serde_json::to_writer_pretty(file, &core_key).unwrap(); + + core_key + } else { + let key_data = + std::fs::read(&key_path).expect("Failed to read vault core key file"); + serde_json::from_slice::(&key_data) + .expect("Failed to deserialize core key") + }; + + for i in 0..seal_config.secret_threshold { + let key = &core_key.secret_shares[i as usize]; + let unseal = managed_core.unseal(key); + assert!(unseal.is_ok()); + } + + log::debug!( + "Vault core initialized with root token: {}", + core_key.root_token + ); + + core_key + }; + + Self { core, key } + } + + pub fn token(&self) -> &str { + &self.key.root_token + } + + pub(crate) fn read_api(&self, path: impl AsRef) -> Result, MegaError> { + let mut req = Request::new(path.as_ref()); + req.operation = Operation::Read; + req.client_token = self.token().to_string(); + let guard = self.core.read().unwrap(); + guard + .handle_request(&mut req) + .map_err(|_| MegaError::with_message("Failed to read from vault API")) + } + + pub(crate) fn write_api( + &self, + path: impl AsRef, + data: Option>, + ) -> Result, MegaError> { + let mut req = Request::new(path.as_ref()); + req.operation = Operation::Write; + req.client_token = self.token().to_string(); + req.body = data; + let guard = self.core.read().unwrap(); + guard + .handle_request(&mut req) + .map_err(|_| MegaError::with_message("Failed to write to vault API")) + } + + pub(crate) fn delete_api(&self, path: impl AsRef) -> Result, MegaError> { + let mut req = Request::new(path.as_ref()); + req.operation = Operation::Delete; + req.client_token = self.token().to_string(); + let guard = self.core.read().unwrap(); + guard + .handle_request(&mut req) + .map_err(|_| MegaError::with_message("Failed to delete from vault API")) + } + + pub fn write_secret( + &self, + name: &str, + data: Option>, + ) -> Result<(), MegaError> { + self.write_api(&format!("secret/{}", name), data) + .map_err(|_| MegaError::with_message(format!("Failed to write secret: {}", name)))?; + Ok(()) + } + + pub fn read_secret(&self, name: &str) -> Result>, MegaError> { + let resp = self + .read_api(&format!("secret/{}", name)) + .map_err(|_| MegaError::with_message(format!("Failed to read secret: {}", name)))?; + + Ok(resp.map(|r| r.data).flatten()) + } + + pub fn delete_secret(&self, name: &str) -> Result<(), MegaError> { + self.delete_api(&format!("secret/{}", name)) + .map_err(|_| MegaError::with_message(format!("Failed to delete secret: {}", name)))?; + Ok(()) + } +} diff --git a/vault/src/vault.rs b/vault/src/vault.rs deleted file mode 100644 index de9b2325b..000000000 --- a/vault/src/vault.rs +++ /dev/null @@ -1,203 +0,0 @@ -use lazy_static::lazy_static; -use rusty_vault::core::{Core, SealConfig}; -use rusty_vault::errors::RvError; -use rusty_vault::logical::{Operation, Request, Response}; -use rusty_vault::storage; -use rusty_vault::storage::barrier_aes_gcm; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; -use std::collections::HashMap; -use std::fs; -use std::sync::{Arc, RwLock}; - -#[derive(Serialize, Deserialize, Debug)] -struct CoreKey { - secret_shares: Vec>, - root_token: String, -} - -#[derive(Clone)] -pub struct CoreInfo { - pub core: Arc>, - pub token: String, -} - -// coding in `lazy_static!` with copilot seems lagging, so using function `init` instead -lazy_static! { - pub static ref CORE: CoreInfo = init(); -} - -/// Initialize the vault core, used in `lazy_static!` -fn init() -> CoreInfo { - const CORE_KEY_FILE: &str = "core_key.json"; // where the core key is stored, like `root_token` - let dir = common::config::mega_base().join("vault"); - let core_key_path = dir.join(CORE_KEY_FILE); - // let dir = env::temp_dir().join("rusty_vault_pki_module"); // TODO: 改成数据库？ - - let inited = dir.exists(); - if !inited { - assert!(fs::create_dir(&dir).is_ok()); - } - - let mut conf: HashMap = HashMap::new(); - conf.insert( - "path".to_string(), - Value::String(dir.to_string_lossy().into_owned()), - ); - - let backend = storage::new_backend("file", &conf).unwrap(); // file or database - let barrier = barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend)); - - let c = Arc::new(RwLock::new(Core { - physical: backend, - barrier: Arc::new(barrier), - ..Default::default() - })); - - let root_token; - { - let mut core = c.write().unwrap(); - assert!(core.config(Arc::clone(&c), None).is_ok()); - - let seal_config = SealConfig { - secret_shares: 10, - secret_threshold: 5, - }; - - let mut unsealed = false; - if !inited { - let result = core.init(&seal_config); - assert!(result.is_ok()); - let init_result = result.unwrap(); - println!("init_result: {:?}", init_result); - - for i in 0..seal_config.secret_threshold { - let key = &init_result.secret_shares[i as usize]; - let unseal = core.unseal(key); - assert!(unseal.is_ok()); - unsealed = unseal.unwrap(); - } - - root_token = init_result.root_token; - - let core_key = CoreKey { - secret_shares: Vec::from(&init_result.secret_shares[..]), - root_token: root_token.clone(), - }; - let file = fs::File::create(core_key_path).unwrap(); - serde_json::to_writer_pretty(file, &core_key).unwrap(); - } else { - let file = fs::File::open(core_key_path).unwrap(); - let core_key: CoreKey = serde_json::from_reader(file).unwrap(); - root_token = core_key.root_token.clone(); - - for i in 0..seal_config.secret_threshold { - let key = &core_key.secret_shares[i as usize]; - let unseal = core.unseal(key); - assert!(unseal.is_ok()); - unsealed = unseal.unwrap(); - } - } - - assert!(unsealed); - println!("root_token: {:?}", root_token); - } - - CoreInfo { - core: c, - token: root_token, - } -} - -pub fn read_api(core: &Core, token: &str, path: &str) -> Result, RvError> { - let mut req = Request::new(path); - req.operation = Operation::Read; - req.client_token = token.to_string(); - core.handle_request(&mut req) -} - -pub fn write_api( - core: &Core, - token: &str, - path: &str, - data: Option>, -) -> Result, RvError> { - let mut req = Request::new(path); - req.operation = Operation::Write; - req.client_token = token.to_string(); - req.body = data; - - let resp = core.handle_request(&mut req); - println!("path: {}, req.body: {:?}", path, req.body); - resp -} - -pub fn delete_api(core: &Core, token: &str, path: &str) -> Result, RvError> { - let mut req = Request::new(path); - req.operation = Operation::Delete; - req.client_token = token.to_string(); - core.handle_request(&mut req) -} - -/// Write a secret to the vault (k-v) -pub async fn write_secret( - name: &str, - data: Option>, -) -> Result, RvError> { - write_api( - &CORE.core.read().unwrap(), - &CORE.token, - &format!("secret/{}", name), - data, - ) -} - -/// Read a secret from the vault (k-v) -pub async fn read_secret(name: &str) -> Result, RvError> { - read_api( - &CORE.core.read().unwrap(), - &CORE.token, - &format!("secret/{}", name), - ) -} - -/// Delete a secret from the vault (k-v) -pub async fn delete_secret(name: &str) -> Result, RvError> { - delete_api( - &CORE.core.read().unwrap(), - &CORE.token, - &format!("secret/{}", name), - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[allow(clippy::await_holding_lock)] - #[tokio::test] - async fn test_secret() { - // create secret - let kv_data = json!({ - "foo": "bar", - "id": "oqpXWgEhXa1WDqMWBnpUW4jvrxGqJKVuJATy4MSPdKNS", - }) - .as_object() - .unwrap() - .clone(); - write_secret("keyInfo", Some(kv_data.clone())) - .await - .unwrap(); - - let secret = read_secret("keyInfo").await.unwrap().unwrap().data; - assert_eq!(secret, Some(kv_data)); - println!("secret: {:?}", secret.unwrap()); - - assert!(read_secret("foo").await.unwrap().is_none()); - assert!( - read_api(&CORE.core.read().unwrap(), &CORE.token, "secret1/foo") - .is_err() - ); - } -} From c0521f7c8f20daa2ab074d2ed84dac1e2bd4e7b6 Mon Sep 17 00:00:00 2001 From: Neon Date: Sun, 15 Jun 2025 21:33:05 +0800 Subject: [PATCH 07/18] vault: reconstruct pki --- vault/src/pki.rs | 300 ++++++++++++++++++++--------------------------- 1 file changed, 125 insertions(+), 175 deletions(-) diff --git a/vault/src/pki.rs b/vault/src/pki.rs index 6a14a1d2b..a53662fed 100644 --- a/vault/src/pki.rs +++ b/vault/src/pki.rs @@ -1,38 +1,23 @@ use std::cmp::Ordering; -use std::sync::{Arc, RwLock}; use std::time::{SystemTime, UNIX_EPOCH}; use openssl::asn1::Asn1Time; use openssl::x509::X509; -use rusty_vault::core::Core; use serde_json::{json, Value}; -use tokio::sync::OnceCell; -use super::vault::{read_api, write_api, CoreInfo, CORE}; +use crate::integration::vault_core::VaultCore; -const ROLE: &str = "test"; +// FIXME: A more official and robust ROLE name +const ROLE: &str = "test-role"; -// DO NOT use `async_static!`, which will cause some compile errors -static _CA: OnceCell = OnceCell::const_new(); -// Automatically initialize CA when you first use it -pub async fn ca() -> &'static CoreInfo { - _CA.get_or_init(|| async { init_ca().await }).await -} -#[allow(clippy::await_holding_lock)] -async fn init_ca() -> CoreInfo { - let c = CORE.clone(); - // init CA if not - let token = &c.token; - if read_api(&c.core.read().unwrap(), token, "pki/ca/pem") - .is_err() - { +impl VaultCore { + /// Initialize the Vault CA + async fn init_ca(&self) { // err = not found - config_ca(c.core.clone(), token).await; - generate_root(c.core.clone(), token, false).await; - config_role( - c.core.clone(), - token, - json!({ // TODO You may want to customize this + if self.read_api("pki/ca/pem").is_err() { + self.config_ca().await; + self.generate_root(false).await; + self.config_role(json!({ // TODO You may want to customize this "ttl": "60d", "max_ttl": "365d", "key_type": "rsa", @@ -42,180 +27,145 @@ async fn init_ca() -> CoreInfo { "locality": "Beijing", "organization": "OpenAtom-Mega", "no_store": false, - }), - ) - .await; + })) + .await; + } } - c -} - -#[allow(clippy::await_holding_lock)] -async fn config_ca(core: Arc>, token: &str) { - let core = core.read().unwrap(); - - // mount pki backend to path: pki/ - let mount_data = json!({ - "type": "pki", - }) - .as_object() - .unwrap() - .clone(); - - let resp = write_api(&core, token, "sys/mounts/pki/", Some(mount_data)); - assert!(resp.is_ok()); -} -/// - `data`: see [RoleEntry](rusty_vault::modules::pki::path_roles) -#[allow(clippy::await_holding_lock)] -pub async fn config_role(core: Arc>, token: &str, data: Value) { - let role_data = data + async fn config_ca(&self) { + // mount pki backend to path: pki/ + let mount_data = json!({ + "type": "pki", + }) .as_object() - .expect("`data` must be a JSON object") + .unwrap() .clone(); - // config role - let result = async_std::task::block_on(async { - let core = core.read().unwrap(); - write_api( - &core, - token, - &format!("pki/roles/{}", ROLE), - Some(role_data), - ) - }); - assert!(result.is_ok()); -} - -/// generate root cert, so that you can read from `pki/ca/pem` -/// - if `exported` is true, then the response will contain `private key` -#[allow(clippy::await_holding_lock)] -async fn generate_root(core: Arc>, token: &str, exported: bool) { - let core = core.read().unwrap(); - - let key_type = "rsa"; - let key_bits = 4096; - let common_name = "mega-ca"; - let req_data = json!({ - "common_name": common_name, - "ttl": "365d", - "country": "cn", - "key_type": key_type, - "key_bits": key_bits, - }) - .as_object() - .unwrap() - .clone(); - - let resp = write_api( - &core, - token, - format!( - "pki/root/generate/{}", - if exported { "exported" } else { "internal" } - ) - .as_str(), - Some(req_data), - ); - assert!(resp.is_ok()); -} + self.write_api("sys/mounts/pki/", Some(mount_data)) + .expect("Failed to mount pki backend"); + } -/// issue certificate -/// - `data`: see [issue_path](rusty_vault::modules::pki::path_issue) -/// - return: `(cert_pem, private_key)` -#[allow(clippy::await_holding_lock)] -pub async fn issue_cert(data: Value) -> (String, String) { - // let dns_sans = ["test.com", "a.test.com", "b.test.com"]; - let issue_data = data + /// generate root cert, so that you can read from `pki/ca/pem` + /// - if `exported` is true, then the response will contain `private key` + async fn generate_root(&self, exported: bool) { + let key_type = "rsa"; + let key_bits = 4096; + let common_name = "mega-ca"; + let req_data = json!({ + "common_name": common_name, + "ttl": "365d", + "country": "cn", + "key_type": key_type, + "key_bits": key_bits, + }) .as_object() - .expect("`data` must be a JSON object") + .unwrap() .clone(); - // issue cert - let resp = async_std::task::block_on(async { - let core = ca().await.core.read().unwrap(); - let token = &ca().await.token; - write_api( - &core, - token, - &format!("pki/issue/{}", ROLE), - Some(issue_data), + self.write_api( + format!( + "pki/root/generate/{}", + if exported { "exported" } else { "internal" } + ) + .as_str(), + Some(req_data), ) - }); - assert!(resp.is_ok()); - let resp_body = resp.unwrap(); - let cert_data = resp_body.unwrap().data.unwrap(); - - ( - cert_data["certificate"].as_str().unwrap().to_owned(), // TODO may add root cert (chain) in it - cert_data["private_key"].as_str().unwrap().to_owned(), - ) -} + .expect("Failed to generate root cert"); + } -/// Verify certificate: time & signature -pub async fn verify_cert(cert_pem: &[u8]) -> bool { - let ca_cert = X509::from_pem(get_root_cert().await.as_ref()).unwrap(); + /// - `data`: see [RoleEntry](rusty_vault::modules::pki::path_roles) + pub async fn config_role(&self, data: Value) { + let role_data = data + .as_object() + .expect("`data` must be a JSON object") + .clone(); - let cert = X509::from_pem(cert_pem).unwrap(); - // verify time - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs() as i64; - let now = Asn1Time::from_unix(now).unwrap(); - let not_before = cert.not_before(); - let not_after = cert.not_after(); - match now.compare(not_before) { - Ok(Ordering::Less) | Err(_) => return false, - _ => {} - } - match now.compare(not_after) { - Ok(Ordering::Greater) | Err(_) => return false, - _ => {} + // config role + self.write_api(&format!("pki/roles/{}", ROLE), Some(role_data)) + .expect("Failed to configure role"); } - // verify signature - cert.verify(&ca_cert.public_key().unwrap()).unwrap() -} - -#[allow(clippy::await_holding_lock)] -/// Get root certificate of CA -pub async fn get_root_cert() -> String { - let resp_ca_pem = async_std::task::block_on(async { - let core = ca().await.core.read().unwrap(); - read_api(&core, &ca().await.token, "pki/ca/pem") - .unwrap() - .unwrap() - }); + /// issue certificate + /// - `data`: see [issue_path](rusty_vault::modules::pki::path_issue) + /// - return: `(cert_pem, private_key)` + pub async fn issue_cert(&self, data: Value) -> (String, String) { + // let dns_sans = ["test.com", "a.test.com", "b.test.com"]; + let issue_data = data + .as_object() + .expect("`data` must be a JSON object") + .clone(); - let ca_data = resp_ca_pem.data.unwrap(); + // issue cert + let resp = self.write_api(&format!("pki/issue/{}", ROLE), Some(issue_data)); + let resp_body = resp.unwrap(); + let cert_data = resp_body.unwrap().data.unwrap(); - ca_data["certificate"].as_str().unwrap().to_owned() -} + ( + cert_data["certificate"].as_str().unwrap().to_owned(), // TODO may add root cert (chain) in it + cert_data["private_key"].as_str().unwrap().to_owned(), + ) + } -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - use std::io::Write; + /// Verify certificate: time & signature + pub async fn verify_cert(&self, cert_pem: &[u8]) -> bool { + let ca_cert = X509::from_pem(self.get_root_cert().await.as_ref()).unwrap(); - #[tokio::test] - async fn test_pki_issue() { - let (cert_pem, private_key) = issue_cert(json!({ - "ttl": "10d", - "common_name": "oqpXWgEhXa1WDqMWBnpUW4jvrxGqJKVuJATy4MSPdKNS", //nostr id - })) - .await; + let cert = X509::from_pem(cert_pem).unwrap(); + // verify time + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() as i64; + let now = Asn1Time::from_unix(now).unwrap(); + let not_before = cert.not_before(); + let not_after = cert.not_after(); + match now.compare(not_before) { + Ok(Ordering::Less) | Err(_) => return false, + _ => {} + } + match now.compare(not_after) { + Ok(Ordering::Greater) | Err(_) => return false, + _ => {} + } - println!("cert_pem: {}", cert_pem); - println!("private_key: {}", private_key); + // verify signature + cert.verify(&ca_cert.public_key().unwrap()).unwrap() + } - assert!(verify_cert(cert_pem.as_ref()).await); + /// Get root certificate of CA + pub async fn get_root_cert(&self) -> String { + let resp_ca_pem = self.read_api("pki/ca/pem").unwrap().unwrap(); + let ca_data = resp_ca_pem.data.unwrap(); - let mut file = fs::File::create("/tmp/cert.crt").unwrap(); - file.write_all(cert_pem.as_ref()).unwrap(); + ca_data["certificate"].as_str().unwrap().to_owned() } } +// TODO use mock core to test +// #[cfg(test)] +// mod tests { +// use super::*; +// use std::fs; +// use std::io::Write; + +// #[tokio::test] +// async fn test_pki_issue() { +// let (cert_pem, private_key) = issue_cert(json!({ +// "ttl": "10d", +// "common_name": "oqpXWgEhXa1WDqMWBnpUW4jvrxGqJKVuJATy4MSPdKNS", //nostr id +// })) +// .await; + +// println!("cert_pem: {}", cert_pem); +// println!("private_key: {}", private_key); + +// assert!(verify_cert(cert_pem.as_ref()).await); + +// let mut file = fs::File::create("/tmp/cert.crt").unwrap(); +// file.write_all(cert_pem.as_ref()).unwrap(); +// } +// } + #[allow(clippy::await_holding_lock)] #[cfg(test)] mod tests_raw { From c2b6b722665e6613200306ff3c9cc9eddfe8afd2 Mon Sep 17 00:00:00 2001 From: Neon Date: Sun, 15 Jun 2025 21:36:33 +0800 Subject: [PATCH 08/18] vault: reconstruct pgp --- vault/src/pgp.rs | 423 ++++++++++++++++++++++++----------------------- 1 file changed, 212 insertions(+), 211 deletions(-) diff --git a/vault/src/pgp.rs b/vault/src/pgp.rs index f9db57bd8..fd125d4d2 100644 --- a/vault/src/pgp.rs +++ b/vault/src/pgp.rs @@ -9,230 +9,231 @@ use pgp::types::SecretKeyTrait; pub use pgp::KeyType; use pgp::{SecretKeyParams, SecretKeyParamsBuilder, SubkeyParamsBuilder}; -use crate::vault::{delete_secret, read_secret, write_secret}; +use crate::integration::vault_core::VaultCore; const VAULT_KEY: &str = "pgp-signed-secret"; -/// Generates a PGP key pair (public and secret) based on the provided parameters. -/// -/// # Arguments -/// -/// * `params`: The parameters for the secret key, such as key type, size, and usage flags. -/// * `passwd`: An optional passphrase to encrypt the secret key. If `None`, the key is not encrypted. -/// -/// # Returns -/// -/// A tuple containing the armored string representations of the public and secret keys. -/// The first element is the public key, and the second is the secret key. -pub fn gen_pgp_keypair( - params: SecretKeyParams, - passwd: Option, -) -> (SignedPublicKey, SignedSecretKey) { - let mut rng = secp256k1::rand::rngs::OsRng; - let key = params - .generate(rng) - .expect("failed to generate secret key, encrypted"); - - let signed_key = key - .sign(&mut rng, || { - if let Some(passwd) = passwd.clone() { - passwd - } else { - "".into() - } - }) - .expect("failed to sign key"); - - let pub_key = signed_key.public_key(); - let signed_pub = pub_key - .sign(rng, &signed_key, || { - if let Some(passwd) = passwd { - passwd - } else { - "".into() - } - }) - .expect("failed to sign key"); +impl VaultCore { + /// Generates a PGP key pair (public and secret) based on the provided parameters. + /// + /// # Arguments + /// + /// * `params`: The parameters for the secret key, such as key type, size, and usage flags. + /// * `passwd`: An optional passphrase to encrypt the secret key. If `None`, the key is not encrypted. + /// + /// # Returns + /// + /// A tuple containing the armored string representations of the public and secret keys. + /// The first element is the public key, and the second is the secret key. + pub fn gen_pgp_keypair( + &self, + params: SecretKeyParams, + passwd: Option, + ) -> (SignedPublicKey, SignedSecretKey) { + let mut rng = secp256k1::rand::rngs::OsRng; + let key = params + .generate(rng) + .expect("failed to generate secret key, encrypted"); + + let signed_key = key + .sign(&mut rng, || { + if let Some(passwd) = passwd.clone() { + passwd + } else { + "".into() + } + }) + .expect("failed to sign key"); + + let pub_key = signed_key.public_key(); + let signed_pub = pub_key + .sign(rng, &signed_key, || { + if let Some(passwd) = passwd { + passwd + } else { + "".into() + } + }) + .expect("failed to sign key"); + + (signed_pub, signed_key) + } - (signed_pub, signed_key) -} + /// Loads the public key from the vault. + /// + /// # Returns + /// + /// An `Option` containing the `SignedPublicKey` if the key is found in the vault, otherwise `None`. + pub fn load_pub_key(&self) -> Option { + let key = self.read_secret(VAULT_KEY).unwrap(); + if let Some(data) = key { + let key = data["pub_key"].as_str().unwrap(); + let (key, _headers) = SignedPublicKey::from_string(key).expect("failed to parse key"); + key.verify().expect("invalid key"); + Some(key) + } else { + None + } + } -/// Loads the public key from the vault. -/// -/// # Returns -/// -/// An `Option` containing the `SignedPublicKey` if the key is found in the vault, otherwise `None`. -pub async fn load_pub_key() -> Option { - let key = read_secret(VAULT_KEY).await.unwrap(); - if let Some(key) = key { - let data = key.data.unwrap(); - let key = data["pub_key"].as_str().unwrap(); - let (key, _headers) = SignedPublicKey::from_string(key).expect("failed to parse key"); - key.verify().expect("invalid key"); - Some(key) - } else { - None + /// Loads the public key from the vault. + /// + /// # Returns + /// + /// An `Option` containing the `SignedPublicKey` if the key is found in the vault, otherwise `None`. + pub async fn load_sec_key(&self) -> Option { + let key = self.read_secret(VAULT_KEY).unwrap(); + if let Some(data) = key { + let key = data["sec_key"].as_str().unwrap(); + let (key, _headers) = SignedSecretKey::from_string(key).expect("failed to parse key"); + key.verify().expect("invalid key"); + Some(key) + } else { + None + } } -} -/// Loads the public key from the vault. -/// -/// # Returns -/// -/// An `Option` containing the `SignedPublicKey` if the key is found in the vault, otherwise `None`. -pub async fn load_sec_key() -> Option { - let key = read_secret(VAULT_KEY).await.unwrap(); - if let Some(key) = key { - let data = key.data.unwrap(); - let key = data["sec_key"].as_str().unwrap(); - let (key, _headers) = SignedSecretKey::from_string(key).expect("failed to parse key"); - key.verify().expect("invalid key"); - Some(key) - } else { - None + /// Saves the public and secret keys to the vault. + /// + /// # Arguments + /// + /// * `pub_key`: The armored string representation of the public key. + /// * `sec_key`: The armored string representation of the secret key. + /// + /// # Panics + /// + /// When input is invalid. + pub fn save_keys(&self, pub_key: SignedPublicKey, sec_key: SignedSecretKey) { + let pub_key = pub_key.to_armored_string(None.into()).unwrap(); + let sec_key = sec_key.to_armored_string(None.into()).unwrap(); + let data = serde_json::json!({ + "pub_key": pub_key, + "sec_key": sec_key, + }) + .as_object() + .unwrap() + .clone(); + self.write_secret(VAULT_KEY, Some(data)) + .unwrap_or_else(|e| { + panic!("Failed to write PGP keys: {:?}", e); + }); } -} -/// Saves the public and secret keys to the vault. -/// -/// # Arguments -/// -/// * `pub_key`: The armored string representation of the public key. -/// * `sec_key`: The armored string representation of the secret key. -/// -/// # Panics -/// -/// When input is invalid. -pub async fn save_keys(pub_key: SignedPublicKey, sec_key: SignedSecretKey) { - let pub_key = pub_key.to_armored_string(None.into()).unwrap(); - let sec_key = sec_key.to_armored_string(None.into()).unwrap(); - let data = serde_json::json!({ - "pub_key": pub_key, - "sec_key": sec_key, - }) - .as_object() - .unwrap() - .clone(); - write_secret(VAULT_KEY, Some(data)) - .await - .unwrap_or_else(|e| { - panic!("Failed to write PGP keys: {:?}", e); + /// Deletes the key pair from the vault. + pub fn delete_keys(&self) { + self.delete_secret(VAULT_KEY).unwrap_or_else(|e| { + panic!("Failed to delete PGP keys: {:?}", e); }); -} - -/// Deletes the key pair from the vault. -pub async fn delete_keys() { - delete_secret(VAULT_KEY).await.unwrap_or_else(|e| { - panic!("Failed to delete PGP keys: {:?}", e); - }); -} + } -/// Creates a set of parameters for generating a PGP secret key. -/// -/// This function simplifies the creation of `SecretKeyParams` by pre-configuring several options -/// such as key version, key type, capabilities (certify and sign), preferred algorithms, and subkeys. -/// -/// # Arguments -/// -/// * `key_type`: The type of key to generate (e.g., RSA, ECDSA). -/// * `passwd`: An optional passphrase to encrypt the secret key. If `None`, the key is not encrypted. -/// * `uid`: The user ID associated with the key. This is typically an email address or name. -/// -/// # Returns -/// -/// A `SecretKeyParams` object configured with the specified parameters, ready for key generation. -pub fn params(key_type: pgp::KeyType, passwd: Option, uid: &str) -> SecretKeyParams { - let version = pgp::types::KeyVersion::V6; - - let mut key_params = SecretKeyParamsBuilder::default(); - key_params - .version(version) - .key_type(key_type.clone()) - .can_certify(true) - .can_sign(true) - .primary_user_id(uid.into()) - .preferred_symmetric_algorithms(smallvec![ - pgp::crypto::sym::SymmetricKeyAlgorithm::AES256, - pgp::crypto::sym::SymmetricKeyAlgorithm::AES192, - pgp::crypto::sym::SymmetricKeyAlgorithm::AES128, - ]) - .preferred_hash_algorithms(smallvec![ - pgp::crypto::hash::HashAlgorithm::SHA2_256, - pgp::crypto::hash::HashAlgorithm::SHA2_384, - pgp::crypto::hash::HashAlgorithm::SHA2_512, - pgp::crypto::hash::HashAlgorithm::SHA2_224, - pgp::crypto::hash::HashAlgorithm::SHA1, - ]) - .preferred_compression_algorithms(smallvec![ - pgp::types::CompressionAlgorithm::ZLIB, - pgp::types::CompressionAlgorithm::ZIP, - ]) - .passphrase(passwd.clone()) - .subkey( - SubkeyParamsBuilder::default() - .version(version) - .key_type(key_type) - .passphrase(passwd) - .can_encrypt(true) - .build() - .unwrap(), - ) - .build() - .unwrap() + /// Creates a set of parameters for generating a PGP secret key. + /// + /// This function simplifies the creation of `SecretKeyParams` by pre-configuring several options + /// such as key version, key type, capabilities (certify and sign), preferred algorithms, and subkeys. + /// + /// # Arguments + /// + /// * `key_type`: The type of key to generate (e.g., RSA, ECDSA). + /// * `passwd`: An optional passphrase to encrypt the secret key. If `None`, the key is not encrypted. + /// * `uid`: The user ID associated with the key. This is typically an email address or name. + /// + /// # Returns + /// + /// A `SecretKeyParams` object configured with the specified parameters, ready for key generation. + pub fn params(key_type: pgp::KeyType, passwd: Option, uid: &str) -> SecretKeyParams { + let version = pgp::types::KeyVersion::V6; + + let mut key_params = SecretKeyParamsBuilder::default(); + key_params + .version(version) + .key_type(key_type.clone()) + .can_certify(true) + .can_sign(true) + .primary_user_id(uid.into()) + .preferred_symmetric_algorithms(smallvec![ + pgp::crypto::sym::SymmetricKeyAlgorithm::AES256, + pgp::crypto::sym::SymmetricKeyAlgorithm::AES192, + pgp::crypto::sym::SymmetricKeyAlgorithm::AES128, + ]) + .preferred_hash_algorithms(smallvec![ + pgp::crypto::hash::HashAlgorithm::SHA2_256, + pgp::crypto::hash::HashAlgorithm::SHA2_384, + pgp::crypto::hash::HashAlgorithm::SHA2_512, + pgp::crypto::hash::HashAlgorithm::SHA2_224, + pgp::crypto::hash::HashAlgorithm::SHA1, + ]) + .preferred_compression_algorithms(smallvec![ + pgp::types::CompressionAlgorithm::ZLIB, + pgp::types::CompressionAlgorithm::ZIP, + ]) + .passphrase(passwd.clone()) + .subkey( + SubkeyParamsBuilder::default() + .version(version) + .key_type(key_type) + .passphrase(passwd) + .can_encrypt(true) + .build() + .unwrap(), + ) + .build() + .unwrap() + } } +// TODO use mock core to test #[cfg(test)] mod tests { - use pgp::KeyType; - - use super::*; - - #[test] - fn test_gen_pgp_keypair() { - const PASSWD: &str = "hello"; - const KEY_TYPE: KeyType = KeyType::Rsa(2048); - const UID: &str = "test"; - let params = params(KEY_TYPE, Some(PASSWD.into()), UID); - let (pk, sk) = gen_pgp_keypair(params, Some(PASSWD.into())); - - assert!(pk.verify().is_ok()); - assert!(sk.verify().is_ok()); - } - - #[tokio::test] - async fn test_save_load_delete_keys() { - const PASSWD: &str = "hello"; - const KEY_TYPE: KeyType = KeyType::Rsa(2048); - const UID: &str = "test"; - let params = params(KEY_TYPE, Some(PASSWD.into()), UID); - let (pk, sk) = gen_pgp_keypair(params, Some(PASSWD.into())); - - save_keys(pk.clone(), sk.clone()).await; - - let loaded_pub_key = load_pub_key().await; - assert!(loaded_pub_key.is_some()); - let loaded_pub_key = loaded_pub_key.unwrap(); - assert!(loaded_pub_key.verify().is_ok()); - assert_eq!( - loaded_pub_key.to_armored_string(None.into()).unwrap(), - pk.to_armored_string(None.into()).unwrap() - ); - - let loaded_sec_key = load_sec_key().await; - assert!(loaded_sec_key.is_some()); - let loaded_sec_key = loaded_sec_key.unwrap(); - assert!(loaded_sec_key.verify().is_ok()); - assert_eq!( - loaded_sec_key.to_armored_string(None.into()).unwrap(), - sk.to_armored_string(None.into()).unwrap() - ); - - delete_keys().await; - - let loaded_pub_key = load_pub_key().await; - assert!(loaded_pub_key.is_none()); - - let loaded_sec_key = load_sec_key().await; - assert!(loaded_sec_key.is_none()); - } + // use pgp::KeyType; + + // use super::*; + + // #[test] + // fn test_gen_pgp_keypair() { + // const PASSWD: &str = "hello"; + // const KEY_TYPE: KeyType = KeyType::Rsa(2048); + // const UID: &str = "test"; + // let params = params(KEY_TYPE, Some(PASSWD.into()), UID); + // let (pk, sk) = gen_pgp_keypair(params, Some(PASSWD.into())); + + // assert!(pk.verify().is_ok()); + // assert!(sk.verify().is_ok()); + // } + + // #[tokio::test] + // async fn test_save_load_delete_keys() { + // const PASSWD: &str = "hello"; + // const KEY_TYPE: KeyType = KeyType::Rsa(2048); + // const UID: &str = "test"; + // let params = params(KEY_TYPE, Some(PASSWD.into()), UID); + // let (pk, sk) = gen_pgp_keypair(params, Some(PASSWD.into())); + + // save_keys(pk.clone(), sk.clone()).await; + + // let loaded_pub_key = load_pub_key().await; + // assert!(loaded_pub_key.is_some()); + // let loaded_pub_key = loaded_pub_key.unwrap(); + // assert!(loaded_pub_key.verify().is_ok()); + // assert_eq!( + // loaded_pub_key.to_armored_string(None.into()).unwrap(), + // pk.to_armored_string(None.into()).unwrap() + // ); + + // let loaded_sec_key = load_sec_key().await; + // assert!(loaded_sec_key.is_some()); + // let loaded_sec_key = loaded_sec_key.unwrap(); + // assert!(loaded_sec_key.verify().is_ok()); + // assert_eq!( + // loaded_sec_key.to_armored_string(None.into()).unwrap(), + // sk.to_armored_string(None.into()).unwrap() + // ); + + // delete_keys().await; + + // let loaded_pub_key = load_pub_key().await; + // assert!(loaded_pub_key.is_none()); + + // let loaded_sec_key = load_sec_key().await; + // assert!(loaded_sec_key.is_none()); + // } } From a5417db26f95a7bffde0e4bf03ed41b545b25bd8 Mon Sep 17 00:00:00 2001 From: Neon Date: Sun, 15 Jun 2025 22:07:02 +0800 Subject: [PATCH 09/18] vault: reconstruct nostr --- vault/Cargo.toml | 5 +- vault/src/integration/jupiter_backend.rs | 14 +++--- vault/src/integration/mod.rs | 1 + vault/src/lib.rs | 53 -------------------- vault/src/nostr.rs | 63 ++++++++++++++++++++++++ vault/src/pki.rs | 48 +++++++++++++----- 6 files changed, 109 insertions(+), 75 deletions(-) diff --git a/vault/Cargo.toml b/vault/Cargo.toml index 97ff53e26..051feea2c 100644 --- a/vault/Cargo.toml +++ b/vault/Cargo.toml @@ -4,19 +4,18 @@ version = "0.1.0" edition = "2021" [dependencies] -common = { workspace = true} jupiter = { workspace = true } +common = { workspace = true} +tracing = { workspace = true } serde_json = { workspace = true } go-defer = { workspace = true } openssl = { workspace = true } hex = { workspace = true } -lazy_static = { workspace = true } bs58 = { workspace = true } serde = { workspace = true, features = ["derive"] } secp256k1 = { workspace = true, features = ["serde", "rand", "hashes"] } tokio = { workspace = true, features = ["full"] } -async-std = { workspace = true } pgp = { workspace = true } smallvec = { workspace = true } diff --git a/vault/src/integration/jupiter_backend.rs b/vault/src/integration/jupiter_backend.rs index 0ed8770da..6adfe4feb 100644 --- a/vault/src/integration/jupiter_backend.rs +++ b/vault/src/integration/jupiter_backend.rs @@ -1,14 +1,14 @@ -use jupiter::context::Context; +use jupiter::context::StorageContext; use rusty_vault::storage::Backend; use tokio::runtime::Handle; pub struct JupiterBackend { - ctx: Context, + ctx: StorageContext, rt: Handle, } impl JupiterBackend { - pub fn new(ctx: Context) -> Self { + pub fn new(ctx: StorageContext) -> Self { let rt = tokio::runtime::Handle::current(); JupiterBackend { ctx, rt } } @@ -16,7 +16,7 @@ impl JupiterBackend { impl Backend for JupiterBackend { fn list(&self, prefix: &str) -> Result, rusty_vault::errors::RvError> { - let service = self.ctx.vault_stg(); + let service = self.ctx.vault_storage(); self.rt.block_on(async move { match service.list_keys(prefix).await { Ok(keys) => Ok(keys), @@ -26,7 +26,7 @@ impl Backend for JupiterBackend { } fn get(&self, key: &str) -> Result, rusty_vault::errors::RvError> { - let service = self.ctx.vault_stg(); + let service = self.ctx.vault_storage(); self.rt.block_on(async move { match service.load(key).await { Ok(model) => { @@ -42,7 +42,7 @@ impl Backend for JupiterBackend { } fn put(&self, entry: &rusty_vault::storage::BackendEntry) -> Result<(), rusty_vault::errors::RvError> { - let service = self.ctx.vault_stg(); + let service = self.ctx.vault_storage(); self.rt.block_on(async move { match service.save(&entry.key, entry.value.clone()).await { Ok(_) => Ok(()), @@ -52,7 +52,7 @@ impl Backend for JupiterBackend { } fn delete(&self, key: &str) -> Result<(), rusty_vault::errors::RvError> { - let service = self.ctx.vault_stg(); + let service = self.ctx.vault_storage(); self.rt.block_on(async move { match service.delete(key).await { Ok(_) => Ok(()), diff --git a/vault/src/integration/mod.rs b/vault/src/integration/mod.rs index 0cce9c672..6207f1d4b 100644 --- a/vault/src/integration/mod.rs +++ b/vault/src/integration/mod.rs @@ -1 +1,2 @@ pub mod jupiter_backend; +pub mod vault_core; diff --git a/vault/src/lib.rs b/vault/src/lib.rs index a0cd702b5..47095c59b 100644 --- a/vault/src/lib.rs +++ b/vault/src/lib.rs @@ -3,56 +3,3 @@ pub mod integration; pub mod nostr; pub mod pgp; pub mod pki; -pub mod vault; - -/// Initialize the Nostr ID if it's not found. -/// - return: `(Nostr ID, secret_key)` -/// - You can get `Public Key` by just `base58::decode(nostr)` -pub async fn init() -> (String, String) { - use crate::vault::{read_secret, write_secret}; - - let mut id = read_secret("id").await.unwrap(); - if id.is_none() { - println!("Nostr ID not found, generating new one..."); - let (nostr, (secret_key, _)) = nostr::generate_nostr_id(); - let data = serde_json::json!({ - "nostr": nostr, - "secret_key": secret_key.display_secret().to_string(), - }) - .as_object() - .unwrap() - .clone(); - write_secret("id", Some(data)).await.unwrap_or_else(|e| { - panic!("Failed to write Nostr ID: {:?}", e); - }); - id = read_secret("id").await.unwrap(); - } - let id_data = id.unwrap().data.unwrap(); - ( - id_data["nostr"].as_str().unwrap().to_string(), - id_data["secret_key"].as_str().unwrap().to_string(), - ) -} - -pub async fn get_peerid() -> String { - let (id, _sk) = init().await; - id -} - -pub async fn get_keypair() -> secp256k1::Keypair { - let (_, sk) = init().await; - let secp = secp256k1::Secp256k1::new(); - secp256k1::Keypair::from_seckey_str(&secp, &sk).unwrap() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_init() { - let id = init().await; - println!("Nostr ID: {:?}", id.0); - println!("Secret Key: {:?}", id.1); // private key - } -} diff --git a/vault/src/nostr.rs b/vault/src/nostr.rs index cbfa8df9a..a639e869b 100644 --- a/vault/src/nostr.rs +++ b/vault/src/nostr.rs @@ -1,5 +1,19 @@ use secp256k1::{rand, PublicKey, Secp256k1, SecretKey}; +use tracing::log; +use crate::integration::vault_core::VaultCore; + +const NOSTR_IDENTITY_KEY: &str = "nostr_identity_key"; + +/// Generates a new Nostr ID and returns it along with the secret and public keys. +/// +/// The Nostr ID is a base58-encoded string of the public key. +/// +/// # Returns +/// +/// A tuple containing: +/// - The Nostr ID as a `String` +/// - A tuple of `(SecretKey, PublicKey)` pub fn generate_nostr_id() -> (String, (SecretKey, PublicKey)) { let secp = Secp256k1::new(); let secret_key = SecretKey::new(&mut rand::thread_rng()); @@ -9,11 +23,60 @@ pub fn generate_nostr_id() -> (String, (SecretKey, PublicKey)) { (nostr, (secret_key, public_key)) } +impl VaultCore { + + /// Initialize the Nostr ID if it's not found. + /// - return: `(Nostr ID, secret_key)` + /// - You can get `Public Key` by just `base58::decode(nostr)` + pub fn load_nostr_id(&self) -> (String, String) { + self.read_secret(NOSTR_IDENTITY_KEY).expect("Failed to read Nostr ID from vault").map(|data| { + let nostr = data["nostr"].as_str().unwrap().to_string(); + let secret_key = data["secret_key"].as_str().unwrap().to_string(); + (nostr, secret_key) + }).unwrap_or_else(|| { + log::debug!("Nostr ID not found in vault, generating new one..."); + let (nostr, (secret_key, _)) = generate_nostr_id(); + let data = serde_json::json!({ + "nostr": nostr, + "secret_key": secret_key.display_secret().to_string(), + }) + .as_object() + .unwrap() + .clone(); + + self.write_secret(NOSTR_IDENTITY_KEY, Some(data.clone())).expect("Failed to write Nostr ID to vault"); + (nostr, secret_key.display_secret().to_string()) + }) + } + + /// Initialize the Nostr ID and return it along with the secret key. + pub fn nostr_get_peerid(&self) -> String { + let (id, _sk) = self.load_nostr_id(); + id + } + + /// Initialize the Nostr ID and return it along with the secret key. + pub fn get_keypair(&self) -> secp256k1::Keypair { + let (_, sk) = self.load_nostr_id(); + let secp = secp256k1::Secp256k1::new(); + secp256k1::Keypair::from_seckey_str(&secp, &sk).unwrap() + } +} + + #[cfg(test)] mod tests { use super::*; use secp256k1::Message; + // TODO use mock vault core for testing + // #[tokio::test] + // async fn test_init() { + // let id = init().await; + // println!("Nostr ID: {:?}", id.0); + // println!("Secret Key: {:?}", id.1); // private key + // } + #[test] fn test_generate_nostr_id() { let (nostr, keypair) = generate_nostr_id(); diff --git a/vault/src/pki.rs b/vault/src/pki.rs index a53662fed..e77c6db10 100644 --- a/vault/src/pki.rs +++ b/vault/src/pki.rs @@ -12,11 +12,11 @@ const ROLE: &str = "test-role"; impl VaultCore { /// Initialize the Vault CA - async fn init_ca(&self) { + fn init_ca(&self) { // err = not found if self.read_api("pki/ca/pem").is_err() { - self.config_ca().await; - self.generate_root(false).await; + self.config_ca(); + self.generate_root(false); self.config_role(json!({ // TODO You may want to customize this "ttl": "60d", "max_ttl": "365d", @@ -27,12 +27,11 @@ impl VaultCore { "locality": "Beijing", "organization": "OpenAtom-Mega", "no_store": false, - })) - .await; + })); } } - async fn config_ca(&self) { + fn config_ca(&self) { // mount pki backend to path: pki/ let mount_data = json!({ "type": "pki", @@ -47,7 +46,7 @@ impl VaultCore { /// generate root cert, so that you can read from `pki/ca/pem` /// - if `exported` is true, then the response will contain `private key` - async fn generate_root(&self, exported: bool) { + fn generate_root(&self, exported: bool) { let key_type = "rsa"; let key_bits = 4096; let common_name = "mega-ca"; @@ -74,7 +73,27 @@ impl VaultCore { } /// - `data`: see [RoleEntry](rusty_vault::modules::pki::path_roles) - pub async fn config_role(&self, data: Value) { + /// - This function configures a role for issuing certificates. + /// - The `ROLE` constant is used as the role name. + /// + /// # Arguments + /// - `data`: A JSON object containing the role configuration data. + /// + /// # Example + /// ```json + /// { + /// "ttl": "60d", + /// "max_ttl": "365d", + /// "key_type": "rsa", + /// "key_bits": 4096, + /// "country": "CN", + /// "province": "Beijing", + /// "locality": "Beijing", + /// "organization": "Open Atom-Mega", + /// "no_store": false + /// } + /// ``` + pub fn config_role(&self, data: Value) { let role_data = data .as_object() .expect("`data` must be a JSON object") @@ -88,7 +107,7 @@ impl VaultCore { /// issue certificate /// - `data`: see [issue_path](rusty_vault::modules::pki::path_issue) /// - return: `(cert_pem, private_key)` - pub async fn issue_cert(&self, data: Value) -> (String, String) { + pub fn issue_cert(&self, data: Value) -> (String, String) { // let dns_sans = ["test.com", "a.test.com", "b.test.com"]; let issue_data = data .as_object() @@ -107,8 +126,13 @@ impl VaultCore { } /// Verify certificate: time & signature - pub async fn verify_cert(&self, cert_pem: &[u8]) -> bool { - let ca_cert = X509::from_pem(self.get_root_cert().await.as_ref()).unwrap(); + /// # Arguments + /// - `cert_pem`: The PEM-encoded certificate to verify. + /// + /// # Returns + /// - `true` if the certificate is valid, `false` otherwise. + pub fn verify_cert(&self, cert_pem: &[u8]) -> bool { + let ca_cert = X509::from_pem(self.get_root_cert().as_ref()).unwrap(); let cert = X509::from_pem(cert_pem).unwrap(); // verify time @@ -133,7 +157,7 @@ impl VaultCore { } /// Get root certificate of CA - pub async fn get_root_cert(&self) -> String { + pub fn get_root_cert(&self) -> String { let resp_ca_pem = self.read_api("pki/ca/pem").unwrap().unwrap(); let ca_data = resp_ca_pem.data.unwrap(); From c3595a8736721bb7f76cc4061b281d4f394c9517 Mon Sep 17 00:00:00 2001 From: Neon Date: Mon, 16 Jun 2025 10:54:24 +0800 Subject: [PATCH 10/18] chore: partially resolve vault and jupiter context api change --- aries/src/service/api/nostr_router.rs | 8 +++---- aries/src/service/relay_server.rs | 8 +++---- jupiter/src/context.rs | 26 ++++++++++++++-------- mega/src/commands/service/http.rs | 5 +++-- mega/src/commands/service/mod.rs | 4 ++-- mega/src/commands/service/multi.rs | 6 ++--- mega/src/commands/service/ssh.rs | 5 +++-- mono/src/api/api_router.rs | 4 ++-- mono/src/api/lfs/lfs_router.rs | 18 +++++++-------- mono/src/api/mod.rs | 32 ++++++++++++++++----------- mono/src/api/oauth/mod.rs | 4 ++-- mono/src/commands/service/http.rs | 7 +++--- mono/src/commands/service/mod.rs | 11 ++++----- mono/src/commands/service/multi.rs | 8 +++---- mono/src/commands/service/ssh.rs | 5 ++--- mono/src/context.rs | 30 +++++++++++++++++++++++++ mono/src/git_protocol/http.rs | 18 +++++++-------- mono/src/git_protocol/ssh.rs | 12 +++++----- mono/src/lib.rs | 1 + mono/src/server/https_server.rs | 23 ++++++++++--------- mono/src/server/ssh_server.rs | 31 +++++++++++++------------- 21 files changed, 154 insertions(+), 112 deletions(-) create mode 100644 mono/src/context.rs diff --git a/aries/src/service/api/nostr_router.rs b/aries/src/service/api/nostr_router.rs index 822d8ca38..106c8f3ef 100644 --- a/aries/src/service/api/nostr_router.rs +++ b/aries/src/service/api/nostr_router.rs @@ -53,7 +53,7 @@ async fn receive( } }; //save - let storage = state.context.services.relay_storage.clone(); + let storage = state.storage.services.relay_storage.clone(); if storage .get_nostr_event_by_id(&ztm_nostr_event.id) .await @@ -89,7 +89,7 @@ async fn receive( filters: filters_json.clone(), id: Uuid::new_v4().to_string(), }; - let storage = state.context.services.relay_storage.clone(); + let storage = state.storage.services.relay_storage.clone(); let req_list: Vec = storage .get_all_nostr_req_by_subscription_id(&subscription_id.to_string()) .await @@ -113,7 +113,7 @@ pub async fn event_list( Query(_query): Query>, state: State, ) -> Result>, (StatusCode, String)> { - let storage = state.context.services.relay_storage.clone(); + let storage = state.storage.services.relay_storage.clone(); let event_list: Vec = storage .get_all_nostr_event() .await @@ -128,7 +128,7 @@ pub async fn req_list( Query(_query): Query>, state: State, ) -> Result>, (StatusCode, String)> { - let storage = state.context.services.relay_storage.clone(); + let storage = state.storage.services.relay_storage.clone(); let req_list: Vec = storage .get_all_nostr_req() .await diff --git a/aries/src/service/relay_server.rs b/aries/src/service/relay_server.rs index 2a511d26b..cb683ef3a 100644 --- a/aries/src/service/relay_server.rs +++ b/aries/src/service/relay_server.rs @@ -4,7 +4,7 @@ use axum::routing::get; use axum::{Json, Router}; use clap::Parser; use common::config::Config; -use jupiter::context::Context; +use jupiter::context::Storage; use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; @@ -29,7 +29,7 @@ pub struct RelayOptions { #[derive(Clone)] pub struct AppState { - pub context: Context, + pub storage: Storage, pub relay_option: RelayOptions, } @@ -38,7 +38,7 @@ pub async fn run_relay_server(config: Arc, option: RelayOptions) { let server_url = format!("{}:{}", option.host, option.relay_port); tracing::info!("start relay server: {server_url}"); tokio::spawn(async move { - let context = Context::new(config).await; + let context = Storage::new(config).await; gemini::p2p::relay::run(context, option.host, option.relay_port).await }); let addr = SocketAddr::from_str(&server_url).unwrap(); @@ -50,7 +50,7 @@ pub async fn run_relay_server(config: Arc, option: RelayOptions) { pub async fn app(config: Arc, relay_option: RelayOptions) -> Router { let state = AppState { - context: Context::new(config).await, + storage: Storage::new(config).await, relay_option, }; diff --git a/jupiter/src/context.rs b/jupiter/src/context.rs index efda0f255..a2ae4a4c8 100644 --- a/jupiter/src/context.rs +++ b/jupiter/src/context.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::sync::{Arc, LazyLock, Weak}; use common::config::Config; @@ -43,7 +43,7 @@ impl Service { mr_storage: MrStorage::new(connection.clone()).await, issue_storage: IssueStorage::new(connection.clone()).await, vault_storage: VaultStorage::new(connection.clone()).await, - lfs_file_storage: lfs_storage::init(config.lfs.clone(), lfs_db_storage.clone()).await, + lfs_file_storage: lfs_storage::init(config.lfs.clone(), lfs_db_storage).await, } } @@ -65,19 +65,23 @@ impl Service { } #[derive(Clone)] -pub struct Context { +pub struct Storage { pub services: Arc, - pub config: Arc, + pub config: Weak, } -impl Context { +impl Storage { pub async fn new(config: Arc) -> Self { - Context { + Storage { services: Service::new(&config).await.into(), - config, + config: Arc::downgrade(&config), } } + pub fn config(&self) -> Arc { + self.config.upgrade().expect("Config has been dropped") + } + pub fn mono_storage(&self) -> MonoStorage { self.services.mono_storage.clone() } @@ -123,9 +127,13 @@ impl Context { } pub fn mock() -> Self { - Context { + // During test time, we don't need a AppContext, + // Put config in a leaked static variable thus the weak reference will always be valid. + static CONFIG: LazyLock> = LazyLock::new(|| Config::mock().into()); + + Storage { services: Service::mock(), - config: Arc::new(Config::mock()), + config: Arc::downgrade(&*CONFIG), } } } diff --git a/mega/src/commands/service/http.rs b/mega/src/commands/service/http.rs index 400aff70f..270fa5f55 100644 --- a/mega/src/commands/service/http.rs +++ b/mega/src/commands/service/http.rs @@ -2,13 +2,14 @@ use clap::{ArgMatches, Args, Command, FromArgMatches}; use common::errors::MegaResult; use gateway::https_server::{self, HttpOptions}; -use jupiter::context::Context; +use jupiter::context::Storage; +use mono::context::AppContext; pub fn cli() -> Command { HttpOptions::augment_args_for_update(Command::new("http").about("Start Mega HTTP server")) } -pub(crate) async fn exec(context: Context, args: &ArgMatches) -> MegaResult { +pub(crate) async fn exec(context: AppContext, args: &ArgMatches) -> MegaResult { let server_matchers = HttpOptions::from_arg_matches(args) .map_err(|err| err.exit()) .unwrap(); diff --git a/mega/src/commands/service/mod.rs b/mega/src/commands/service/mod.rs index a644fa774..fbcca445b 100644 --- a/mega/src/commands/service/mod.rs +++ b/mega/src/commands/service/mod.rs @@ -8,7 +8,7 @@ use clap::{ArgMatches, Command}; use std::sync::Arc; use common::{config::Config, errors::MegaResult}; -use jupiter::context::Context; +use jupiter::context::Storage; mod http; mod multi; @@ -28,7 +28,7 @@ pub fn cli() -> Command { #[tokio::main] pub(crate) async fn exec(config: Config, args: &ArgMatches) -> MegaResult { let config = Arc::new(config); - let context = Context::new(config.clone()).await; + let context = Storage::new(config.clone()).await; context .services diff --git a/mega/src/commands/service/multi.rs b/mega/src/commands/service/multi.rs index 3d1be7935..d9b3be530 100644 --- a/mega/src/commands/service/multi.rs +++ b/mega/src/commands/service/multi.rs @@ -5,8 +5,8 @@ use common::{ model::{CommonHttpOptions, P2pOptions}, }; use gateway::https_server::{self, HttpOptions}; -use jupiter::context::Context; -use mono::server::ssh_server::{self, SshCustom, SshOptions}; +use jupiter::context::Storage; +use mono::{context::AppContext, server::ssh_server::{self, SshCustom, SshOptions}}; #[derive(Debug, PartialEq, Clone, ValueEnum)] pub enum StartCommand { @@ -34,7 +34,7 @@ pub fn cli() -> Command { ) } -pub(crate) async fn exec(context: Context, args: &ArgMatches) -> MegaResult { +pub(crate) async fn exec(context: AppContext, args: &ArgMatches) -> MegaResult { let server_matchers = StartOptions::from_arg_matches(args) .map_err(|err| err.exit()) .unwrap(); diff --git a/mega/src/commands/service/ssh.rs b/mega/src/commands/service/ssh.rs index 012a74e8f..ad0463f34 100644 --- a/mega/src/commands/service/ssh.rs +++ b/mega/src/commands/service/ssh.rs @@ -1,7 +1,8 @@ use clap::{ArgMatches, Args, Command, FromArgMatches}; use common::errors::MegaResult; -use jupiter::context::Context; +use jupiter::context::Storage; +use mono::context::AppContext; use mono::server::ssh_server::start_server; use mono::server::ssh_server::SshOptions; @@ -9,7 +10,7 @@ pub fn cli() -> Command { SshOptions::augment_args_for_update(Command::new("ssh").about("Start Git SSH server")) } -pub(crate) async fn exec(context: Context, args: &ArgMatches) -> MegaResult { +pub(crate) async fn exec(context: AppContext, args: &ArgMatches) -> MegaResult { let server_matchers = SshOptions::from_arg_matches(args) .map_err(|err| err.exit()) .unwrap(); diff --git a/mono/src/api/api_router.rs b/mono/src/api/api_router.rs index 37c31674a..9be8b6e97 100644 --- a/mono/src/api/api_router.rs +++ b/mono/src/api/api_router.rs @@ -288,10 +288,10 @@ async fn path_can_be_cloned( state: State, ) -> Result>, ApiError> { let path: PathBuf = query.path.clone().into(); - let import_dir = state.context.config.monorepo.import_dir.clone(); + let import_dir = state.storage.config.monorepo.import_dir.clone(); let res = if path.starts_with(&import_dir) { state - .context + .storage .services .git_db_storage .find_git_repo_exact_match(path.to_str().unwrap()) diff --git a/mono/src/api/lfs/lfs_router.rs b/mono/src/api/lfs/lfs_router.rs index a6c1fba18..61191beee 100644 --- a/mono/src/api/lfs/lfs_router.rs +++ b/mono/src/api/lfs/lfs_router.rs @@ -87,7 +87,7 @@ pub async fn list_locks( Query(query): Query, ) -> Result, (StatusCode, String)> { let result: Result = - handler::lfs_retrieve_lock(state.context.lfs_stg(), query).await; + handler::lfs_retrieve_lock(state.storage.lfs_db_storage(), query).await; match result { Ok(lock_list) => { let body = serde_json::to_string(&lock_list).unwrap_or_default(); @@ -104,7 +104,7 @@ pub async fn list_locks_for_verification( state: State, Json(json): Json, ) -> Result, (StatusCode, String)> { - let result = handler::lfs_verify_lock(state.context.lfs_stg(), json).await; + let result = handler::lfs_verify_lock(state.storage.lfs_db_storage(), json).await; match result { Ok(lock_list) => { let body = serde_json::to_string(&lock_list).unwrap_or_default(); @@ -126,7 +126,7 @@ pub async fn create_lock( state: State, Json(json): Json, ) -> Result, (StatusCode, String)> { - let result = handler::lfs_create_lock(state.context.lfs_stg(), json).await; + let result = handler::lfs_create_lock(state.storage.lfs_db_storage(), json).await; match result { Ok(lock) => { let lock_response = LockResponse { @@ -154,7 +154,7 @@ pub async fn delete_lock( Path(id): Path, Json(json): Json, ) -> Result { - let result = handler::lfs_delete_lock(state.context.lfs_stg(), &id, json).await; + let result = handler::lfs_delete_lock(state.storage.lfs_db_storage(), &id, json).await; match result { Ok(lock) => { @@ -181,7 +181,7 @@ pub async fn lfs_process_batch( state: State, Json(json): Json, ) -> Result, (StatusCode, String)> { - let result = handler::lfs_process_batch(&state.context, json, &state.listen_addr).await; + let result = handler::lfs_process_batch(&state.storage, json, &state.listen_addr).await; match result { Ok(res) => { @@ -206,7 +206,7 @@ pub async fn lfs_fetch_chunk_ids( state: State, Path(oid): Path, ) -> Result, (StatusCode, String)> { - let result = handler::lfs_fetch_chunk_ids(&state.context, &oid, &state.listen_addr).await; + let result = handler::lfs_fetch_chunk_ids(&state.storage, &oid, &state.listen_addr).await; match result { Ok(response) => { let size = response.iter().fold(0, |acc, chunk| acc + chunk.size); @@ -235,7 +235,7 @@ pub async fn lfs_download_object( state: State, Path(oid): Path, ) -> Result { - let result = handler::lfs_download_object(state.context.clone(), oid.clone()).await; + let result = handler::lfs_download_object(state.storage.clone(), oid.clone()).await; match result { Ok(byte_stream) => Ok(Response::builder() .header("Content-Type", LFS_CONTENT_TYPE) @@ -268,7 +268,7 @@ pub async fn lfs_download_chunk( )); } let result = handler::lfs_download_chunk( - state.context.clone(), + state.storage.clone(), &origin_object_id, &chunk_id, offset.unwrap(), @@ -310,7 +310,7 @@ pub async fn lfs_upload_object( .await .unwrap(); - let result = handler::lfs_upload_object(&state.context, &req_obj, body_bytes).await; + let result = handler::lfs_upload_object(&state.storage, &req_obj, body_bytes).await; match result { Ok(_) => Ok(Response::builder() .header("Content-Type", LFS_CONTENT_TYPE) diff --git a/mono/src/api/mod.rs b/mono/src/api/mod.rs index 81b2a863c..899fdd229 100644 --- a/mono/src/api/mod.rs +++ b/mono/src/api/mod.rs @@ -17,7 +17,7 @@ use ceres::{ }; use common::errors::ProtocolError; use jupiter::{ - context::Context, + context::Storage, storage::{issue_storage::IssueStorage, mr_storage::MrStorage, user_storage::UserStorage}, }; @@ -50,7 +50,7 @@ pub type GithubClient< #[derive(Clone)] pub struct MonoApiServiceState { - pub context: Context, + pub storage: Storage, pub oauth_client: Option, // TODO: Replace MemoryStore pub store: Option, @@ -71,34 +71,34 @@ impl FromRef for GithubClient { impl FromRef for UserStorage { fn from_ref(state: &MonoApiServiceState) -> Self { - state.context.user_stg() + state.storage.user_storage() } } impl MonoApiServiceState { fn monorepo(&self) -> MonoApiService { MonoApiService { - context: self.context.clone(), + storage: self.storage.clone(), } } fn issue_stg(&self) -> IssueStorage { - self.context.issue_stg() + self.storage.issue_storage() } fn mr_stg(&self) -> MrStorage { - self.context.mr_stg() + self.storage.mr_storage() } fn user_stg(&self) -> UserStorage { - self.context.user_stg() + self.storage.user_storage() } async fn api_handler(&self, path: PathBuf) -> Result, ProtocolError> { - let import_dir = self.context.config.monorepo.import_dir.clone(); + let import_dir = self.storage.config().monorepo.import_dir.clone(); if path.starts_with(&import_dir) && path != import_dir { if let Some(model) = self - .context + .storage .services .git_db_storage .find_git_repo_like_path(path.to_str().unwrap()) @@ -107,16 +107,22 @@ impl MonoApiServiceState { { let repo: Repo = model.into(); return Ok(Box::new(ImportApiService { - context: self.context.clone(), + storage: self.storage.clone(), repo, })); } } - Ok(Box::new(MonoApiService { - context: self.context.clone(), - })) + let ret: Box = Box::new(MonoApiService { + storage: self.storage.clone(), + }); + + // Rust-analyzer cannot infer the type of `ret` correctly and always reports an error. + // Use `.into()` to workaround this issue. + #[allow(clippy::useless_conversion)] + Ok(ret.into()) } } + pub mod util { use std::path::PathBuf; diff --git a/mono/src/api/oauth/mod.rs b/mono/src/api/oauth/mod.rs index 4e7c5ecad..152128ac7 100644 --- a/mono/src/api/oauth/mod.rs +++ b/mono/src/api/oauth/mod.rs @@ -52,7 +52,7 @@ async fn login_authorized( State(oauth_client): State, ) -> Result { let store: MemoryStore = MemoryStore::from_ref(&state); - let config = state.context.config.oauth.as_ref().unwrap(); + let config = state.storage.config.oauth.as_ref().unwrap(); let http_client = reqwest::ClientBuilder::new() .redirect(reqwest::redirect::Policy::none()) @@ -134,7 +134,7 @@ async fn logout( TypedHeader(cookies): TypedHeader, ) -> Result { let store: MemoryStore = MemoryStore::from_ref(&state); - let full_config = state.context.config.clone(); + let full_config = state.storage.config.clone(); let config = full_config.oauth.as_ref().unwrap(); let cookie = cookies .get(COOKIE_NAME) diff --git a/mono/src/commands/service/http.rs b/mono/src/commands/service/http.rs index 234cd96d1..ee7642a74 100644 --- a/mono/src/commands/service/http.rs +++ b/mono/src/commands/service/http.rs @@ -1,15 +1,14 @@ use clap::{ArgMatches, Args, Command, FromArgMatches}; -use crate::server::https_server::{self}; +use crate::{context::AppContext, server::https_server::{self}}; use common::{errors::MegaResult, model::CommonHttpOptions}; -use jupiter::context::Context; pub fn cli() -> Command { CommonHttpOptions::augment_args_for_update(Command::new("http").about("Start Mega HTTP server")) } -pub(crate) async fn exec(ctx: Context, args: &ArgMatches) -> MegaResult { - let server_matchers = CommonHttpOptions::from_arg_matches(args) +pub(crate) async fn exec(ctx: AppContext, args: &ArgMatches) -> MegaResult { + let server_matchers: CommonHttpOptions = CommonHttpOptions::from_arg_matches(args) .map_err(|err| err.exit()) .unwrap(); diff --git a/mono/src/commands/service/mod.rs b/mono/src/commands/service/mod.rs index 6c8029556..e7ab45a57 100644 --- a/mono/src/commands/service/mod.rs +++ b/mono/src/commands/service/mod.rs @@ -8,7 +8,9 @@ use clap::{ArgMatches, Command}; use std::sync::Arc; use common::{config::Config, errors::MegaResult}; -use jupiter::context::Context; +use jupiter::context::Storage; + +use crate::context::AppContext; pub mod http; pub mod multi; @@ -28,12 +30,7 @@ pub fn cli() -> Command { #[tokio::main] pub(crate) async fn exec(config: Config, args: &ArgMatches) -> MegaResult { let config = Arc::new(config); - let context = Context::new(config.clone()).await; - context - .services - .mono_storage - .init_monorepo(&config.monorepo) - .await; + let context = AppContext::new(config.clone()).await; let (cmd, subcommand_args) = match args.subcommand() { Some((cmd, args)) => (cmd, args), diff --git a/mono/src/commands/service/multi.rs b/mono/src/commands/service/multi.rs index 05c0e36aa..e18699038 100644 --- a/mono/src/commands/service/multi.rs +++ b/mono/src/commands/service/multi.rs @@ -1,10 +1,10 @@ use clap::{ArgMatches, Args, Command, FromArgMatches, ValueEnum}; -use jupiter::context::Context; +use jupiter::context::Storage; -use crate::server::{ +use crate::{context::AppContext, server::{ https_server::{self}, ssh_server::{self, SshCustom, SshOptions}, -}; +}}; use common::{errors::MegaResult, model::CommonHttpOptions}; #[derive(Debug, PartialEq, Clone, ValueEnum)] @@ -31,7 +31,7 @@ pub fn cli() -> Command { ) } -pub(crate) async fn exec(ctx: Context, args: &ArgMatches) -> MegaResult { +pub(crate) async fn exec(ctx: AppContext, args: &ArgMatches) -> MegaResult { let server_matchers = StartOptions::from_arg_matches(args) .map_err(|err| err.exit()) .unwrap(); diff --git a/mono/src/commands/service/ssh.rs b/mono/src/commands/service/ssh.rs index 685ecd918..6b73f8074 100644 --- a/mono/src/commands/service/ssh.rs +++ b/mono/src/commands/service/ssh.rs @@ -1,14 +1,13 @@ use clap::{ArgMatches, Args, Command, FromArgMatches}; -use crate::server::ssh_server::{start_server, SshOptions}; +use crate::{context::AppContext, server::ssh_server::{start_server, SshOptions}}; use common::errors::MegaResult; -use jupiter::context::Context; pub fn cli() -> Command { SshOptions::augment_args_for_update(Command::new("ssh").about("Start Git SSH server")) } -pub(crate) async fn exec(ctx: Context, args: &ArgMatches) -> MegaResult { +pub(crate) async fn exec(ctx: AppContext, args: &ArgMatches) -> MegaResult { let server_matchers = SshOptions::from_arg_matches(args) .map_err(|err| err.exit()) .unwrap(); diff --git a/mono/src/context.rs b/mono/src/context.rs new file mode 100644 index 000000000..299da6506 --- /dev/null +++ b/mono/src/context.rs @@ -0,0 +1,30 @@ +use std::{ops::Deref, sync::Arc}; + +/// This is the main application context for the Mono application. +// It holds shared state and configuration for the application. +// Including database connections, configuration settings, encrypted vault functions, etc. +#[derive(Clone)] +pub struct AppContext { + /// The storage sub-context for the from jupiter abstract layer. + pub storage: jupiter::context::Storage, + + /// The vault core for managing encrypted data. + pub vault: vault::integration::vault_core::VaultCore, + + /// The configuration settings for the application. + pub config: Arc, +} + +impl AppContext { + /// Creates a new application context with the given configuration. + pub async fn new(config: Arc) -> Self { + let storage = jupiter::context::Storage::new(config.clone()).await; + let vault = vault::integration::vault_core::VaultCore::new(storage.clone()); + + Self { + storage, + vault, + config, + } + } +} diff --git a/mono/src/git_protocol/http.rs b/mono/src/git_protocol/http.rs index fafdd675d..fd88fb892 100644 --- a/mono/src/git_protocol/http.rs +++ b/mono/src/git_protocol/http.rs @@ -8,7 +8,7 @@ use base64::prelude::*; use bytes::{Bytes, BytesMut}; use futures::{stream, TryStreamExt}; use http::HeaderMap; -use jupiter::context::Context; +use jupiter::context::Storage; use tokio::io::AsyncReadExt; use tokio_stream::StreamExt; @@ -41,7 +41,7 @@ pub async fn git_info_refs( Ok(response) } -async fn http_auth(header: &HeaderMap, context: &Context) -> bool { +async fn http_auth(header: &HeaderMap, storage: &Storage) -> bool { for (k, v) in header { if k == http::header::AUTHORIZATION { let decoded = general_purpose::STANDARD @@ -58,22 +58,22 @@ async fn http_auth(header: &HeaderMap, context: &Context) -> bool { let username = parts.next().unwrap_or(""); let token = parts.next().unwrap_or(""); tracing::debug!("{}, {}", username, token); - let auth_config = context.config.authentication.clone(); + let auth_config = storage.config().authentication.clone(); if auth_config.enable_test_user && username == auth_config.test_user_name && token == auth_config.test_user_token { return true; } - match context - .user_stg() + match storage + .user_storage() .find_user_by_name(username) .await .unwrap() { Some(user) => { - return context - .user_stg() + return storage + .user_storage() .check_token(user.id, token) .await .unwrap(); @@ -185,8 +185,8 @@ pub async fn git_receive_pack( req: Request, mut pack_protocol: SmartProtocol, ) -> Result, ProtocolError> { - if pack_protocol.context.config.authentication.enable_http_auth - && !http_auth(req.headers(), &pack_protocol.context).await + if pack_protocol.storage.config().authentication.enable_http_auth + && !http_auth(req.headers(), &pack_protocol.storage).await { return auth_failed(); } diff --git a/mono/src/git_protocol/ssh.rs b/mono/src/git_protocol/ssh.rs index e037bbde5..5d390e2b7 100644 --- a/mono/src/git_protocol/ssh.rs +++ b/mono/src/git_protocol/ssh.rs @@ -15,7 +15,7 @@ use ceres::lfs::lfs_structs::Link; use ceres::protocol::smart::{self}; use ceres::protocol::ServiceType; use ceres::protocol::{SmartProtocol, TransportProtocol}; -use jupiter::context::Context; +use jupiter::context::Storage; use tokio::sync::Mutex; use crate::git_protocol::http::search_subsequence; @@ -26,7 +26,7 @@ type ClientMap = HashMap<(usize, ChannelId), Channel>; pub struct SshServer { pub clients: Arc>, pub id: usize, - pub context: Context, + pub storage: Storage, pub smart_protocol: Option, pub data_combined: BytesMut, } @@ -84,7 +84,7 @@ impl server::Handler for SshServer { let path = path.replace(".git", "").replace('\'', ""); let mut smart_protocol = SmartProtocol::new( PathBuf::from(&path), - self.context.clone(), + self.storage.clone(), TransportProtocol::Ssh, ); match command[0] { @@ -106,7 +106,7 @@ impl server::Handler for SshServer { // back to the hybrid protocol using `git-lfs-authenticate`. "git-lfs-authenticate" => { let mut header = HashMap::new(); - let config = smart_protocol.context.config.clone(); + let config = smart_protocol.storage.config(); header.insert("Accept".to_string(), "application/vnd.git-lfs".to_string()); let link = Link { href: config.lfs.ssh.http_url.clone(), @@ -133,8 +133,8 @@ impl server::Handler for SshServer { tracing::info!("auth_publickey: {} / {}", user, fingerprint); let res = self - .context - .user_stg() + .storage + .user_storage() .search_ssh_key_finger(&fingerprint) .await .unwrap(); diff --git a/mono/src/lib.rs b/mono/src/lib.rs index 54fe65595..b8b57cc14 100644 --- a/mono/src/lib.rs +++ b/mono/src/lib.rs @@ -1,5 +1,6 @@ pub mod api; pub mod cli; mod commands; +pub mod context; pub mod git_protocol; pub mod server; diff --git a/mono/src/server/https_server.rs b/mono/src/server/https_server.rs index 3b16737c7..b5b65e9da 100644 --- a/mono/src/server/https_server.rs +++ b/mono/src/server/https_server.rs @@ -20,7 +20,7 @@ use tower_http::trace::TraceLayer; use ceres::protocol::{ServiceType, SmartProtocol, TransportProtocol}; use common::errors::ProtocolError; use common::model::{CommonHttpOptions, InfoRefsParams}; -use jupiter::context::Context; +use jupiter::context::Storage; use utoipa::OpenApi; use utoipa_axum::router::OpenApiRouter; use utoipa_swagger_ui::SwaggerUi; @@ -29,10 +29,11 @@ use crate::api::api_router::{self}; use crate::api::lfs::lfs_router; use crate::api::oauth::{self, oauth_client}; use crate::api::MonoApiServiceState; +use crate::context::AppContext; #[derive(Clone)] pub struct AppState { - pub context: Context, + pub storage: Storage, pub host: String, pub port: u16, } @@ -41,10 +42,10 @@ pub fn remove_git_suffix(uri: Uri, git_suffix: &str) -> PathBuf { PathBuf::from(uri.path().replace(".git", "").replace(git_suffix, "")) } -pub async fn start_http(context: Context, options: CommonHttpOptions) { +pub async fn start_http(ctx: AppContext, options: CommonHttpOptions) { let CommonHttpOptions { host, port } = options.clone(); - let app = app(context, host.clone(), port).await; + let app = app(ctx.storage, host.clone(), port).await; let server_url = format!("{}:{}", host, port); @@ -84,16 +85,16 @@ pub async fn start_http(context: Context, options: CommonHttpOptions) { /// - GET end of `Regex::new(r"/info/refs$")` /// - POST end of `Regex::new(r"/git-upload-pack$")` /// - POST end of `Regex::new(r"/git-receive-pack$")` -pub async fn app(context: Context, host: String, port: u16) -> Router { +pub async fn app(storage: Storage, host: String, port: u16) -> Router { let state = AppState { host: host.clone(), port, - context: context.clone(), + storage: storage.clone(), }; - let config = context.config.clone(); + let config = storage.config(); let api_state = MonoApiServiceState { - context: context.clone(), + storage: storage.clone(), oauth_client: Some(oauth_client(config.oauth.clone().unwrap()).unwrap()), store: Some(MemoryStore::new()), listen_addr: format!("http://{}:{}", host, port), @@ -144,7 +145,7 @@ pub async fn get_method_router( if INFO_REFS_REGEX.is_match(uri.path()) { let pack_protocol = SmartProtocol::new( remove_git_suffix(uri, "/info/refs"), - state.context.clone(), + state.storage.clone(), TransportProtocol::Http, ); crate::git_protocol::http::git_info_refs(params, pack_protocol).await @@ -163,7 +164,7 @@ pub async fn post_method_router( if REGEX_GIT_UPLOAD_PACK.is_match(uri.path()) { let mut pack_protocol = SmartProtocol::new( remove_git_suffix(uri.clone(), "/git-upload-pack"), - state.context.clone(), + state.storage.clone(), TransportProtocol::Http, ); pack_protocol.service_type = Some(ServiceType::UploadPack); @@ -171,7 +172,7 @@ pub async fn post_method_router( } else if REGEX_GIT_RECEIVE_PACK.is_match(uri.path()) { let mut pack_protocol = SmartProtocol::new( remove_git_suffix(uri.clone(), "/git-receive-pack"), - state.context.clone(), + state.storage.clone(), TransportProtocol::Http, ); pack_protocol.service_type = Some(ServiceType::ReceivePack); diff --git a/mono/src/server/ssh_server.rs b/mono/src/server/ssh_server.rs index 6a0b19794..04c5e4383 100644 --- a/mono/src/server/ssh_server.rs +++ b/mono/src/server/ssh_server.rs @@ -13,11 +13,9 @@ use russh::{ }; use common::model::CommonHttpOptions; -use jupiter::context::Context; use tokio::sync::Mutex; -use vault::vault::{read_secret, write_secret}; -use crate::git_protocol::ssh::SshServer; +use crate::{ context::AppContext, git_protocol::ssh::SshServer}; #[derive(Args, Clone, Debug)] pub struct SshOptions { @@ -35,9 +33,9 @@ pub struct SshCustom { } /// start a ssh server -pub async fn start_server(context: Context, command: &SshOptions) { +pub async fn start_server(ctx: AppContext, command: &SshOptions) { // we need to persist the key to prevent key expired after server restart. - let p_key = load_key().await; + let p_key = load_key(ctx.clone()); let ru_config = russh::server::Config { auth_rejection_time: std::time::Duration::from_secs(3), keys: vec![p_key], @@ -55,10 +53,11 @@ pub async fn start_server(context: Context, command: &SshOptions) { common: CommonHttpOptions { host, .. }, custom: SshCustom { ssh_port }, } = command; + let mut ssh_server = SshServer { clients: Arc::new(Mutex::new(HashMap::new())), id: 0, - context, + storage: ctx.storage.clone(), smart_protocol: None, data_combined: BytesMut::new(), }; @@ -67,11 +66,10 @@ pub async fn start_server(context: Context, command: &SshOptions) { ssh_server.run_on_address(ru_config, addr).await.unwrap(); } -pub async fn load_key() -> PrivateKey { - let ssh_key = read_secret("ssh_server_key").await.unwrap(); +pub fn load_key(ctx: AppContext) -> PrivateKey { + let ssh_key = ctx.vault.read_secret("ssh_server_key").unwrap(); if let Some(ssh_key) = ssh_key { - let data = ssh_key.data.unwrap(); - let secret_key = data["secret_key"].as_str().unwrap(); + let secret_key = ssh_key["secret_key"].as_str().unwrap(); PrivateKey::from_openssh(secret_key).unwrap() } else { // generate a keypair if not exists @@ -83,11 +81,12 @@ pub async fn load_key() -> PrivateKey { .as_object() .unwrap() .clone(); - write_secret("ssh_server_key", Some(secret)) - .await - .unwrap_or_else(|e| { - panic!("Failed to write ssh_server_key: {:?}", e); - }); - keys + + match ctx.vault.write_secret("ssh_server_key", Some(secret)) { + Ok(_) => keys, + Err(e) => { + panic!("Failed to write SSH server key to vault: {}", e); + } + } } } From 47234f0a8799492fcbac71a8dff38f71fd9924c8 Mon Sep 17 00:00:00 2001 From: Neon Date: Mon, 16 Jun 2025 10:56:50 +0800 Subject: [PATCH 11/18] chore: resolve api changes in ceres --- ceres/src/api_service/import_api_service.rs | 20 +++++----- ceres/src/api_service/mod.rs | 4 +- ceres/src/api_service/mono_api_service.rs | 38 +++++++++---------- ceres/src/lfs/handler.rs | 42 ++++++++++----------- ceres/src/pack/import_repo.rs | 34 ++++++++--------- ceres/src/pack/monorepo.rs | 22 +++++------ ceres/src/protocol/mod.rs | 20 +++++----- ceres/src/protocol/smart.rs | 2 +- 8 files changed, 91 insertions(+), 91 deletions(-) diff --git a/ceres/src/api_service/import_api_service.rs b/ceres/src/api_service/import_api_service.rs index bff4228df..63df0a144 100644 --- a/ceres/src/api_service/import_api_service.rs +++ b/ceres/src/api_service/import_api_service.rs @@ -7,7 +7,7 @@ use std::path::PathBuf; use async_trait::async_trait; -use jupiter::context::Context; +use jupiter::context::Storage; use mercury::errors::GitError; use mercury::hash::SHA1; use mercury::internal::object::commit::Commit; @@ -20,14 +20,14 @@ use crate::protocol::repo::Repo; #[derive(Clone)] pub struct ImportApiService { - pub context: Context, + pub storage: Storage, pub repo: Repo, } #[async_trait] impl ApiHandler for ImportApiService { - fn get_context(&self) -> Context { - self.context.clone() + fn get_context(&self) -> Storage { + self.storage.clone() } async fn create_monorepo_file(&self, _: CreateFileInfo) -> Result<(), GitError> { @@ -47,7 +47,7 @@ impl ApiHandler for ImportApiService { } async fn get_root_commit(&self) -> Commit { - let storage = self.context.services.git_db_storage.clone(); + let storage = self.storage.services.git_db_storage.clone(); let refs = storage .get_default_ref(self.repo.repo_id) .await @@ -62,7 +62,7 @@ impl ApiHandler for ImportApiService { } async fn get_root_tree(&self) -> Tree { - let storage = self.context.services.git_db_storage.clone(); + let storage = self.storage.services.git_db_storage.clone(); let refs = storage .get_default_ref(self.repo.repo_id) .await @@ -83,7 +83,7 @@ impl ApiHandler for ImportApiService { } async fn get_tree_by_hash(&self, hash: &str) -> Tree { - self.context + self.storage .services .git_db_storage .get_tree_by_hash(self.repo.repo_id, hash) @@ -94,7 +94,7 @@ impl ApiHandler for ImportApiService { } async fn get_commit_by_hash(&self, hash: &str) -> Option { - let storage = self.context.services.git_db_storage.clone(); + let storage = self.storage.services.git_db_storage.clone(); let commit = storage .get_commit_by_hash(self.repo.repo_id, hash) .await @@ -103,7 +103,7 @@ impl ApiHandler for ImportApiService { } async fn get_tree_relate_commit(&self, t_hash: &str) -> Commit { - let storage = self.context.services.git_db_storage.clone(); + let storage = self.storage.services.git_db_storage.clone(); let tree_info = storage .get_tree_by_hash(self.repo.repo_id, t_hash) .await @@ -118,7 +118,7 @@ impl ApiHandler for ImportApiService { } async fn get_commits_by_hashes(&self, c_hashes: Vec) -> Result, GitError> { - let storage = self.context.services.git_db_storage.clone(); + let storage = self.storage.services.git_db_storage.clone(); let commits = storage .get_commits_by_hashes(self.repo.repo_id, &c_hashes) .await diff --git a/ceres/src/api_service/mod.rs b/ceres/src/api_service/mod.rs index dd59bc61d..728508bf7 100644 --- a/ceres/src/api_service/mod.rs +++ b/ceres/src/api_service/mod.rs @@ -7,7 +7,7 @@ use async_trait::async_trait; use callisto::raw_blob; use common::errors::MegaError; -use jupiter::{context::Context, utils::converter::generate_git_keep_with_timestamp}; +use jupiter::{context::Storage, utils::converter::generate_git_keep_with_timestamp}; use mercury::{ errors::GitError, hash::SHA1, @@ -33,7 +33,7 @@ pub struct GitObjectCache { #[async_trait] pub trait ApiHandler: Send + Sync { - fn get_context(&self) -> Context; + fn get_context(&self) -> Storage; async fn create_monorepo_file(&self, file_info: CreateFileInfo) -> Result<(), GitError>; diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index 18ad703f0..2ec5890ac 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -9,7 +9,7 @@ use tokio::process::Command; use callisto::sea_orm_active_enums::ConvTypeEnum; use callisto::{mega_blob, mega_tree, raw_blob}; use common::errors::MegaError; -use jupiter::context::Context; +use jupiter::context::Storage; use jupiter::storage::batch_save_model; use jupiter::utils::converter::generate_git_keep_with_timestamp; use mercury::errors::GitError; @@ -24,13 +24,13 @@ use crate::protocol::mr::MergeRequest; #[derive(Clone)] pub struct MonoApiService { - pub context: Context, + pub storage: Storage, } #[async_trait] impl ApiHandler for MonoApiService { - fn get_context(&self) -> Context { - self.context.clone() + fn get_context(&self) -> Storage { + self.storage.clone() } /// Creates a new file or directory in the monorepo based on the provided file information. @@ -43,7 +43,7 @@ impl ApiHandler for MonoApiService { /// /// Returns `Ok(())` on success, or a `GitError` on failure. async fn create_monorepo_file(&self, file_info: CreateFileInfo) -> Result<(), GitError> { - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); let path = PathBuf::from(file_info.path); let mut save_trees = vec![]; @@ -127,7 +127,7 @@ impl ApiHandler for MonoApiService { } async fn get_root_tree(&self) -> Tree { - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); let refs = storage.get_ref("/").await.unwrap().unwrap(); storage @@ -139,7 +139,7 @@ impl ApiHandler for MonoApiService { } async fn get_tree_by_hash(&self, hash: &str) -> Tree { - self.context + self.storage .services .mono_storage .get_tree_by_hash(hash) @@ -150,7 +150,7 @@ impl ApiHandler for MonoApiService { } async fn get_commit_by_hash(&self, hash: &str) -> Option { - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); match storage.get_commit_by_hash(hash).await { Ok(Some(commit)) => Some(commit.into()), _ => None, @@ -158,7 +158,7 @@ impl ApiHandler for MonoApiService { } async fn get_tree_relate_commit(&self, t_hash: &str) -> Commit { - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); let tree_info = storage.get_tree_by_hash(t_hash).await.unwrap().unwrap(); storage .get_commit_by_hash(&tree_info.commit_id) @@ -169,7 +169,7 @@ impl ApiHandler for MonoApiService { } async fn get_commits_by_hashes(&self, c_hashes: Vec) -> Result, GitError> { - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); let commits = storage.get_commits_by_hashes(&c_hashes).await.unwrap(); Ok(commits.into_iter().map(|x| x.into()).collect()) } @@ -182,7 +182,7 @@ impl ApiHandler for MonoApiService { Some(tree) => { let mut item_to_commit = HashMap::new(); - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); let tree_hashes = tree .tree_items .iter() @@ -234,7 +234,7 @@ impl ApiHandler for MonoApiService { impl MonoApiService { pub async fn merge_mr(&self, mr: &mut MergeRequest) -> Result<(), MegaError> { - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); let refs = storage.get_ref(&mr.path).await.unwrap().unwrap(); if mr.from_hash == refs.ref_commit_hash { @@ -262,14 +262,14 @@ impl MonoApiService { // update mr mr.merge(); // add conversation - self.context - .mr_stg() + self.storage + .mr_storage() .add_mr_conversation(&mr.link, 0, ConvTypeEnum::Merged, None) .await .unwrap(); // update mr status last - self.context - .mr_stg() + self.storage + .mr_storage() .update_mr(mr.clone().into()) .await .unwrap(); @@ -285,7 +285,7 @@ impl MonoApiService { mut tree_vec: Vec, commit: Commit, ) -> Result { - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); let mut save_trees = Vec::new(); let mut p_commit_id = String::new(); @@ -340,9 +340,9 @@ impl MonoApiService { } pub async fn content_diff(&self, mr_link: &str, listen_addr: &str) -> Result { - let stg = self.context.mr_stg(); + let stg = self.storage.mr_storage(); if let Some(mr) = stg.get_mr(mr_link).await.unwrap() { - let base_path = self.context.config.base_dir.clone(); + let base_path = self.storage.config().base_dir.clone(); env::set_current_dir(&base_path).unwrap(); let clone_path = base_path.join(mr_link); if !fs::exists(&clone_path).unwrap() { diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index b336129ea..1473e96e2 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -10,7 +10,7 @@ use tokio_stream::wrappers::ReceiverStream; use callisto::lfs_locks; use common::config::PackConfig; use common::errors::{GitLFSError, MegaError}; -use jupiter::context::Context; +use jupiter::context::Storage; use jupiter::storage::lfs_db_storage::LfsDbStorage; use crate::lfs::lfs_structs::{ @@ -175,16 +175,16 @@ pub async fn lfs_delete_lock( /// Reference: /// 1. [Git LFS Batch API](https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md) pub async fn lfs_process_batch( - context: &Context, + storage: &Storage, request: BatchRequest, listen_addr: &str, ) -> Result { let objects = request.objects; let mut response_objects = Vec::new(); - let file_storage = context.lfs_file_stg(); - let db_storage = context.lfs_stg(); - let config = context.config.lfs.clone(); + let file_storage = storage.lfs_file_storage(); + let db_storage = storage.lfs_db_storage(); + let config = storage.config().lfs.clone(); for object in objects { let meta_res = lfs_get_meta(db_storage.clone(), &object.oid).await.unwrap(); let meta = match meta_res { @@ -243,18 +243,18 @@ pub async fn lfs_process_batch( /// if server enable split, then return a list of chunk ids. /// else return an error. pub async fn lfs_fetch_chunk_ids( - context: &Context, + storage: &Storage, oid: &str, listen_addr: &str, ) -> Result, GitLFSError> { - let config = context.config.lfs.clone(); + let config = storage.config().lfs.clone(); if !config.local.enable_split { return Err(GitLFSError::GeneralError( "Server didn't run in `split` mode, didn't support chunk ids".to_string(), )); } - let db_storage = context.lfs_stg(); + let db_storage = storage.lfs_db_storage(); let meta = lfs_get_meta(db_storage.clone(), oid) .await @@ -281,8 +281,8 @@ pub async fn lfs_fetch_chunk_ids( size: relation.size, ..Default::default() }; - let download_url = context - .lfs_file_stg() + let download_url = storage + .lfs_file_storage() .download_url(&req_obj.oid, listen_addr) .await .unwrap(); @@ -299,13 +299,13 @@ pub async fn lfs_fetch_chunk_ids( /// Upload object to storage. /// if server enable split, split the object and upload each part to storage, save the relationship to database. pub async fn lfs_upload_object( - context: &Context, + storage: &Storage, req_obj: &RequestObject, body_bytes: Vec, ) -> Result<(), GitLFSError> { - let config = context.config.lfs.clone(); - let db_storage = context.lfs_stg(); - let file_storage = context.lfs_file_stg(); + let config = storage.config().lfs.clone(); + let db_storage = storage.lfs_db_storage(); + let file_storage = storage.lfs_file_storage(); let meta = if let Some(meta) = lfs_get_meta(db_storage.clone(), &req_obj.oid).await? { tracing::debug!("upload lfs object {} size: {}", meta.oid, meta.size); @@ -350,11 +350,11 @@ pub async fn lfs_upload_object( /// Download object from storage. /// when server enable split, if OID is a complete object, then splice the object and return it. pub async fn lfs_download_object( - context: Context, + storage: Storage, oid: String, ) -> Result>, GitLFSError> { - let db_storage = context.lfs_stg(); - let file_storage = context.lfs_file_stg(); + let db_storage = storage.lfs_db_storage(); + let file_storage = storage.lfs_file_storage(); let meta = lfs_get_meta(db_storage.clone(), &oid).await?; match meta { @@ -418,15 +418,15 @@ pub async fn lfs_download_object( /// It's used when server didn't have splited chunk, but client request a chunk. /// If the server enable split, then the chunk must be a splited chunk, rather than a random part of the object. pub async fn lfs_download_chunk( - context: Context, + storage: Storage, origin_oid: &str, chunk_oid: &String, offset: u64, size: u64, ) -> Result { - let config = &context.config.lfs; - let db_storage = context.lfs_stg(); - let file_storage = context.lfs_file_stg(); + let config = &storage.config().lfs; + let db_storage = storage.lfs_db_storage(); + let file_storage = storage.lfs_file_storage(); // check if the chunk is already exist. if config.local.enable_split { diff --git a/ceres/src/pack/import_repo.rs b/ceres/src/pack/import_repo.rs index 4513d1c81..fb932100b 100644 --- a/ceres/src/pack/import_repo.rs +++ b/ceres/src/pack/import_repo.rs @@ -18,7 +18,7 @@ use tokio_stream::wrappers::ReceiverStream; use callisto::{mega_tree, raw_blob, sea_orm_active_enums::RefTypeEnum}; use common::errors::MegaError; -use jupiter::{context::Context, storage::batch_save_model}; +use jupiter::{context::Storage, storage::batch_save_model}; use mercury::{ errors::GitError, internal::{ @@ -38,7 +38,7 @@ use crate::{ }; pub struct ImportRepo { - pub context: Context, + pub storage: Storage, pub repo: Repo, pub command_list: Vec, } @@ -47,7 +47,7 @@ pub struct ImportRepo { impl PackHandler for ImportRepo { async fn head_hash(&self) -> (String, Vec) { let result = self - .context + .storage .services .git_db_storage .get_ref(self.repo.repo_id) @@ -62,7 +62,7 @@ impl PackHandler for ImportRepo { &self, mut receiver: UnboundedReceiver, ) -> Result, GitError> { - let storage = self.context.services.git_db_storage.clone(); + let storage = self.storage.services.git_db_storage.clone(); let mut entry_list = vec![]; let semaphore = Arc::new(Semaphore::new(8)); let mut join_tasks = vec![]; @@ -101,12 +101,12 @@ impl PackHandler for ImportRepo { } async fn full_pack(&self, _: Vec) -> Result>, GitError> { - let pack_config = &self.context.config.pack; + let pack_config = &self.storage.config().pack; let (entry_tx, entry_rx) = mpsc::channel(pack_config.channel_message_size); let (stream_tx, stream_rx) = mpsc::channel(pack_config.channel_message_size); - let storage = self.context.services.git_db_storage.clone(); - let raw_storage = self.context.services.raw_db_storage.clone(); + let storage = self.storage.services.git_db_storage.clone(); + let raw_storage = self.storage.services.raw_db_storage.clone(); let total = storage.get_obj_count_by_repo_id(self.repo.repo_id).await; let encoder = PackEncoder::new(total, 0, stream_tx); encoder.encode_async(entry_rx).await.unwrap(); @@ -192,8 +192,8 @@ impl PackHandler for ImportRepo { have: Vec, ) -> Result>, GitError> { let mut want_clone = want.clone(); - let pack_config = &self.context.config.pack; - let storage = self.context.services.git_db_storage.clone(); + let pack_config = &self.storage.config().pack; + let storage = self.storage.services.git_db_storage.clone(); let obj_num = AtomicUsize::new(0); let mut exist_objs = HashSet::new(); @@ -285,7 +285,7 @@ impl PackHandler for ImportRepo { async fn get_trees_by_hashes(&self, hashes: Vec) -> Result, MegaError> { Ok(self - .context + .storage .services .git_db_storage .get_trees_by_hashes(self.repo.repo_id, hashes) @@ -300,7 +300,7 @@ impl PackHandler for ImportRepo { &self, hashes: Vec, ) -> Result, MegaError> { - self.context + self.storage .services .raw_db_storage .get_raw_blobs_by_hashes(hashes) @@ -308,7 +308,7 @@ impl PackHandler for ImportRepo { } async fn update_refs(&self, _: Option, refs: &RefCommand) -> Result<(), GitError> { - let storage = self.context.services.git_db_storage.clone(); + let storage = self.storage.services.git_db_storage.clone(); match refs.command_type { CommandType::Create => { storage @@ -331,7 +331,7 @@ impl PackHandler for ImportRepo { } async fn check_commit_exist(&self, hash: &str) -> bool { - self.context + self.storage .services .git_db_storage .get_commit_by_hash(self.repo.repo_id, hash) @@ -341,7 +341,7 @@ impl PackHandler for ImportRepo { } async fn check_default_branch(&self) -> bool { - let storage = self.context.services.git_db_storage.clone(); + let storage = self.storage.services.git_db_storage.clone(); storage .default_branch_exist(self.repo.repo_id) .await @@ -364,14 +364,14 @@ impl ImportRepo { let path = PathBuf::from(self.repo.repo_path.clone()); let mono_api_service = MonoApiService { - context: self.context.clone(), + storage: self.storage.clone(), }; - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); let save_trees = mono_api_service.search_and_create_tree(&path).await?; let mut root_ref = storage.get_ref("/").await.unwrap().unwrap(); let latest_commit: Commit = self - .context + .storage .services .git_db_storage .get_commit_by_hash(self.repo.repo_id, &commit_id) diff --git a/ceres/src/pack/monorepo.rs b/ceres/src/pack/monorepo.rs index e67f37a44..adcbf1628 100644 --- a/ceres/src/pack/monorepo.rs +++ b/ceres/src/pack/monorepo.rs @@ -16,7 +16,7 @@ use common::{ errors::MegaError, utils::{self, MEGA_BRANCH_NAME}, }; -use jupiter::{context::Context, storage::mr_storage::MrStorage}; +use jupiter::{context::Storage, storage::mr_storage::MrStorage}; use mercury::internal::{object::ObjectTrait, pack::encode::PackEncoder}; use mercury::{ errors::GitError, @@ -36,7 +36,7 @@ use crate::{ }; pub struct MonoRepo { - pub context: Context, + pub storage: Storage, pub path: PathBuf, pub from_hash: String, pub to_hash: String, @@ -45,7 +45,7 @@ pub struct MonoRepo { #[async_trait] impl PackHandler for MonoRepo { async fn head_hash(&self) -> (String, Vec) { - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); let result = storage.get_refs(self.path.to_str().unwrap()).await.unwrap(); @@ -129,7 +129,7 @@ impl PackHandler for MonoRepo { &self, mut receiver: UnboundedReceiver, ) -> Result, GitError> { - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); let mut entry_list = Vec::new(); let mut join_tasks = vec![]; let mut current_commit_id = String::new(); @@ -178,8 +178,8 @@ impl PackHandler for MonoRepo { have: Vec, ) -> Result>, GitError> { let mut want_clone = want.clone(); - let pack_config = &self.context.config.pack; - let storage = self.context.services.mono_storage.clone(); + let pack_config = &self.storage.config().pack; + let storage = self.storage.services.mono_storage.clone(); let obj_num = AtomicUsize::new(0); let mut exist_objs = HashSet::new(); @@ -264,7 +264,7 @@ impl PackHandler for MonoRepo { async fn get_trees_by_hashes(&self, hashes: Vec) -> Result, MegaError> { Ok(self - .context + .storage .services .mono_storage .get_trees_by_hashes(hashes) @@ -279,7 +279,7 @@ impl PackHandler for MonoRepo { &self, hashes: Vec, ) -> Result, MegaError> { - self.context + self.storage .services .raw_db_storage .get_raw_blobs_by_hashes(hashes) @@ -287,7 +287,7 @@ impl PackHandler for MonoRepo { } async fn update_refs(&self, commit: Option, refs: &RefCommand) -> Result<(), GitError> { - let storage = self.context.services.mono_storage.clone(); + let storage = self.storage.services.mono_storage.clone(); if let Some(c) = commit { let mr_link = self.handle_mr(&c.format_message()).await?; @@ -313,7 +313,7 @@ impl PackHandler for MonoRepo { } async fn check_commit_exist(&self, hash: &str) -> bool { - self.context + self.storage .services .mono_storage .get_commit_by_hash(hash) @@ -329,7 +329,7 @@ impl PackHandler for MonoRepo { impl MonoRepo { async fn handle_mr(&self, title: &str) -> Result { - let storage = self.context.mr_stg(); + let storage = self.storage.mr_storage(); let path_str = self.path.to_str().unwrap(); match storage.get_open_mr_by_path(path_str).await.unwrap() { diff --git a/ceres/src/protocol/mod.rs b/ceres/src/protocol/mod.rs index dce72e74a..432ead224 100644 --- a/ceres/src/protocol/mod.rs +++ b/ceres/src/protocol/mod.rs @@ -7,7 +7,7 @@ use common::{ utils::ZERO_ID, }; use import_refs::RefCommand; -use jupiter::context::Context; +use jupiter::context::Storage; use repo::Repo; use crate::pack::{import_repo::ImportRepo, monorepo::MonoRepo, PackHandler}; @@ -24,7 +24,7 @@ pub struct SmartProtocol { pub path: PathBuf, pub command_list: Vec, pub service_type: Option, - pub context: Context, + pub storage: Storage, } #[derive(Debug, PartialEq, Clone, Copy, Default)] @@ -125,33 +125,33 @@ pub struct RefUpdateRequest { } impl SmartProtocol { - pub fn new(path: PathBuf, context: Context, transport_protocol: TransportProtocol) -> Self { + pub fn new(path: PathBuf, storage: Storage, transport_protocol: TransportProtocol) -> Self { SmartProtocol { transport_protocol, capabilities: Vec::new(), path, command_list: Vec::new(), service_type: None, - context, + storage, } } pub fn mock() -> Self { - let context = Context::mock(); + let context = Storage::mock(); SmartProtocol { transport_protocol: TransportProtocol::default(), capabilities: Vec::new(), path: PathBuf::new(), command_list: Vec::new(), service_type: None, - context, + storage: context, } } pub async fn pack_handler(&self) -> Result, ProtocolError> { - let import_dir = self.context.config.monorepo.import_dir.clone(); + let import_dir = self.storage.config().monorepo.import_dir.clone(); if self.path.starts_with(import_dir.clone()) { - let storage = self.context.services.git_db_storage.clone(); + let storage = self.storage.services.git_db_storage.clone(); let path_str = self.path.to_str().unwrap(); let model = storage.find_git_repo_exact_match(path_str).await.unwrap(); let repo = if let Some(repo) = model { @@ -169,13 +169,13 @@ impl SmartProtocol { } }; Ok(Arc::new(ImportRepo { - context: self.context.clone(), + storage: self.storage.clone(), repo, command_list: self.command_list.clone(), })) } else { let mut res = MonoRepo { - context: self.context.clone(), + storage: self.storage.clone(), path: self.path.clone(), from_hash: String::new(), to_hash: String::new(), diff --git a/ceres/src/protocol/smart.rs b/ceres/src/protocol/smart.rs index 4d7e36b86..996bf77db 100644 --- a/ceres/src/protocol/smart.rs +++ b/ceres/src/protocol/smart.rs @@ -213,7 +213,7 @@ impl SmartProtocol { let pack_handler = self.pack_handler().await?; //1. unpack progress let receiver = pack_handler - .unpack_stream(&self.context.config.pack, data_stream) + .unpack_stream(&self.storage.config().pack, data_stream) .await?; // do not block main thread here. From 0e3e4e27cde4db470d31dd1b5eddf4a278924449 Mon Sep 17 00:00:00 2001 From: Neon Date: Mon, 23 Jun 2025 18:56:50 +0800 Subject: [PATCH 12/18] gemini: adapt for vault api change --- gateway/src/https_server.rs | 19 +- gemini/Cargo.toml | 3 +- gemini/src/ca/client.rs | 106 +- gemini/src/ca/mod.rs | 34 +- gemini/src/ca/server.rs | 41 +- gemini/src/lfs/mod.rs | 325 +++-- gemini/src/nostr/event.rs | 6 +- gemini/src/nostr/mod.rs | 4 +- gemini/src/p2p/client.rs | 1459 +++++++++++----------- gemini/src/p2p/mod.rs | 12 + gemini/src/p2p/relay.rs | 1168 ++++++++--------- gemini/src/util.rs | 29 +- jupiter/src/context.rs | 139 --- jupiter/src/lib.rs | 3 +- jupiter/src/storage.rs | 241 ++++ jupiter/src/storage/mod.rs | 101 -- mono/Cargo.toml | 4 + mono/src/api/api_router.rs | 2 +- mono/src/api/mod.rs | 2 +- mono/src/api/oauth/mod.rs | 4 +- mono/src/commands/service/http.rs | 2 +- mono/src/commands/service/mod.rs | 5 +- mono/src/commands/service/multi.rs | 1 - mono/src/context.rs | 13 +- mono/src/git_protocol/http.rs | 2 +- mono/src/git_protocol/ssh.rs | 2 +- mono/src/lib.rs | 2 +- mono/src/main.rs | 1 + mono/src/server/https_server.rs | 2 +- mono/src/server/ssh_server.rs | 1 + vault/Cargo.toml | 2 +- vault/src/integration/jupiter_backend.rs | 6 +- vault/src/integration/mod.rs | 2 + vault/src/integration/vault_core.rs | 43 +- vault/src/lib.rs | 57 + vault/src/nostr.rs | 12 +- vault/src/pgp.rs | 2 +- vault/src/pki.rs | 3 +- 38 files changed, 1999 insertions(+), 1861 deletions(-) delete mode 100644 jupiter/src/context.rs create mode 100644 jupiter/src/storage.rs delete mode 100644 jupiter/src/storage/mod.rs diff --git a/gateway/src/https_server.rs b/gateway/src/https_server.rs index ee3bec10e..2dcae9bd4 100644 --- a/gateway/src/https_server.rs +++ b/gateway/src/https_server.rs @@ -2,6 +2,7 @@ use axum::routing::get; use axum::{http, Router}; use clap::Args; +use mono::context::AppContext; use quinn::rustls; use tower::ServiceBuilder; use tower_http::cors::{Any, CorsLayer}; @@ -9,7 +10,7 @@ use tower_http::decompression::RequestDecompressionLayer; use tower_http::trace::TraceLayer; use common::model::{CommonHttpOptions, P2pOptions}; -use jupiter::context::Context; +use jupiter::context::Storage; use mono::api::lfs::lfs_router; use mono::api::MonoApiServiceState; use mono::server::https_server::{get_method_router, post_method_router, AppState}; @@ -27,7 +28,7 @@ pub struct HttpOptions { pub p2p: P2pOptions, } -pub async fn http_server(context: Context, options: HttpOptions) { +pub async fn http_server(context: AppContext, options: HttpOptions) { let HttpOptions { common: CommonHttpOptions { host, port, .. }, p2p, @@ -39,7 +40,7 @@ pub async fn http_server(context: Context, options: HttpOptions) { check_run_with_p2p(context.clone(), options.p2p.clone()); - let app = app(context, host.clone(), port, p2p.clone()).await; + let app = app(context.storage, host.clone(), port, p2p.clone()).await; let server_url = format!("{}:{}", host, port); @@ -49,15 +50,15 @@ pub async fn http_server(context: Context, options: HttpOptions) { .unwrap(); } -pub async fn app(context: Context, host: String, port: u16, p2p: P2pOptions) -> Router { +pub async fn app(storage: Storage, host: String, port: u16, p2p: P2pOptions) -> Router { let state = AppState { host: host.clone(), port, - context: context.clone(), + storage: storage.clone(), }; let mono_api_state = MonoApiServiceState { - context: context.clone(), + storage: storage.clone(), oauth_client: None, store: None, listen_addr: format!("http://{}:{}", host, port), @@ -103,7 +104,7 @@ pub async fn app(context: Context, host: String, port: u16, p2p: P2pOptions) -> router } -pub fn check_run_with_p2p(context: Context, p2p: P2pOptions) { +pub fn check_run_with_p2p(context: AppContext, p2p: P2pOptions) { //Mega server join a ztm mesh match p2p.bootstrap_node { Some(bootstrap_node) => { @@ -113,7 +114,9 @@ pub fn check_run_with_p2p(context: Context, p2p: P2pOptions) { ); tokio::spawn(async move { - if let Err(e) = gemini::p2p::client::run(context, bootstrap_node).await { + if let Err(e) = + gemini::p2p::client::run(context.storage, context.vault, bootstrap_node).await + { tracing::error!("P2P client closed:{}", e) } }); diff --git a/gemini/Cargo.toml b/gemini/Cargo.toml index 52274933c..85ea14ef3 100644 --- a/gemini/Cargo.toml +++ b/gemini/Cargo.toml @@ -28,7 +28,6 @@ chrono = { workspace = true } secp256k1 = { workspace = true, features = ["serde", "rand", "hashes"] } quinn = { workspace = true } anyhow = { workspace = true } -lazy_static = { workspace = true } rcgen = { workspace = true, features = ["x509-parser", "pem", "crypto"] } dashmap = { workspace = true } uuid = { workspace = true } @@ -38,4 +37,4 @@ futures = { workspace = true } #[dev-dependencies] -#tracing-subscriber = { workspace = true } \ No newline at end of file +#tracing-subscriber = { workspace = true } diff --git a/gemini/src/ca/client.rs b/gemini/src/ca/client.rs index f4f828b6d..9f0dd5500 100644 --- a/gemini/src/ca/client.rs +++ b/gemini/src/ca/client.rs @@ -1,66 +1,80 @@ use anyhow::Result; use anyhow::{anyhow, Ok}; use rcgen::{CertificateParams, KeyPair}; -use reqwest::Client; -use vault::get_peerid; +use quinn::rustls::pki_types::pem::PemObject; +use quinn::rustls::pki_types::CertificateDer; +use quinn::rustls::pki_types::PrivateKeyDer; + +use crate::p2p::client::P2PClient; use super::{get_from_vault, save_to_vault}; static USER_KEY: &str = "user_key"; -pub async fn get_user_key() -> String { - match get_from_vault(USER_KEY.to_string()).await { - Some(key) => key, - None => { - let user_key = KeyPair::generate().unwrap(); - save_to_vault(USER_KEY.to_string(), user_key.serialize_pem()).await; - user_key.serialize_pem() +impl P2PClient { + pub fn get_user_key(&self) -> String { + match get_from_vault(&self.vault, USER_KEY.to_string()) { + Some(key) => key, + None => { + let user_key = KeyPair::generate().unwrap(); + save_to_vault(&self.vault, USER_KEY.to_string(), user_key.serialize_pem()); + user_key.serialize_pem() + } } } -} -pub async fn get_user_cert_from_ca(ca: String) -> Result { - let name = get_peerid().await; - //request to ca - let url = format!("{ca}/api/v1/ca/certificates/{name}"); - let url = add_http_to_url(url); - let client = Client::new(); - let response = client.get(url.clone()).send().await?; - if response.status().is_success() { - //cert exists - return Ok(response.text().await?); - } + pub async fn get_user_cert_from_ca(&self, ca: impl AsRef) -> Result<(CertificateDer<'static>, PrivateKeyDer<'static>)> { + let name = self.get_peer_id(); + // Request to ca + let url = format!("{}/api/v1/ca/certificates/{name}", ca.as_ref()); + let url = add_http_to_url(url); + let response = self.http_client.get(url.clone()).send().await?; + if response.status().is_success() { + //cert exists + let cert = response.text().await?; + let cert = CertificateDer::from_pem_slice(cert.as_bytes())?; + let key = self.get_user_key(); + let key = PrivateKeyDer::from_pem_slice(key.as_bytes())?; + return Ok((cert, key)); + } - let params = CertificateParams::new(vec![name])?; + let params = CertificateParams::new(vec![name])?; - let key = get_user_key().await; - let key = KeyPair::from_pem(&key)?; - let user_csr = params.serialize_request(&key)?; - //request a new cert - let response = client - .post(url) - .body(user_csr.pem().unwrap()) - .send() - .await - .unwrap(); - if response.status().is_success() { - return Ok(response.text().await.unwrap()); - } + let key = self.get_user_key(); + let key = KeyPair::from_pem(&key)?; + let user_csr = params.serialize_request(&key)?; + //request a new cert + let response = self.http_client + .post(url) + .body(user_csr.pem().unwrap()) + .send() + .await + .unwrap(); - Err(anyhow!("get user certificate from ca failed")) -} + if !response.status().is_success() { + return Err(anyhow!("get user certificate from ca failed")); + } + + let cert = CertificateDer::from_pem_slice(response.text().await.unwrap().as_bytes())?; + let key = self.get_user_key(); + let key = PrivateKeyDer::from_pem_slice(key.as_bytes())?; + Ok((cert, key)) -pub async fn get_ca_cert_from_ca(ca: String) -> Result { - //request to ca - let url = format!("{ca}/api/v1/ca/certificates/ca"); - let url = add_http_to_url(url); - let client = Client::new(); - let response = client.get(url.clone()).send().await?; - if response.status().is_success() { - return Ok(response.text().await?); } - Err(anyhow!("get user certificate from ca failed")) + pub async fn get_ca_cert_from_ca(&self, ca: impl AsRef) -> Result> { + //request to ca + let url = format!("{}/api/v1/ca/certificates/ca", ca.as_ref()); + let url = add_http_to_url(url); + let response = self.http_client.get(url.clone()).send().await?; + if response.status().is_success() { + let cert = response.text().await?; + let cert = CertificateDer::from_pem_slice(cert.as_bytes())?; + return Ok(cert); + } + + Err(anyhow!("get user certificate from ca failed")) + } } fn add_http_to_url(url: String) -> String { diff --git a/gemini/src/ca/mod.rs b/gemini/src/ca/mod.rs index 17a80b070..b58e4f3e5 100755 --- a/gemini/src/ca/mod.rs +++ b/gemini/src/ca/mod.rs @@ -1,9 +1,10 @@ use serde_json::json; +use vault::integration::{vault_core::VaultCoreInterface, VaultCore}; pub mod client; pub mod server; -async fn save_to_vault(key: String, value: String) { +fn save_to_vault(vault: &VaultCore, key: String, value: String) { let key_f = format!("ca_{key}"); let kv_data = json!({ key_f.clone(): value, @@ -11,33 +12,28 @@ async fn save_to_vault(key: String, value: String) { .as_object() .unwrap() .clone(); - vault::vault::write_secret(key_f.as_str(), Some(kv_data.clone())) - .await + vault + .write_secret(key_f.as_str(), Some(kv_data.clone())) .unwrap(); } -async fn get_from_vault(key: String) -> Option { +fn get_from_vault(vault: &VaultCore, key: String) -> Option { let key_f = format!("ca_{key}"); - let secret = match vault::vault::read_secret(key_f.as_str()).await.unwrap() { - Some(res) => res.data, + match vault.read_secret(key_f.as_str()).unwrap() { + Some(res) => res + .get(key_f.as_str()) + .map(|v| { + v.as_str() + .map(|vv| String::from(vv.trim_matches(char::is_control))) + }) + .flatten(), None => return None, - }; - - match secret { - Some(m) => { - let s = m.get(key_f.as_str()).unwrap().as_str().unwrap().to_string(); - let s = s.trim_matches(char::is_control).to_string(); - Some(s) - } - None => None, } } -async fn _delete_to_vault(key: String) { +fn _delete_to_vault(vault: &VaultCore, key: String) { let key_f = format!("ca_{key}"); - vault::vault::write_secret(key_f.as_str(), None) - .await - .unwrap(); + vault.delete_secret(key_f.as_str()).unwrap(); } #[cfg(test)] diff --git a/gemini/src/ca/server.rs b/gemini/src/ca/server.rs index 46dfb56b9..e4bcdc3b4 100644 --- a/gemini/src/ca/server.rs +++ b/gemini/src/ca/server.rs @@ -7,6 +7,7 @@ use rcgen::{ use anyhow::anyhow; use anyhow::Result; +use vault::integration::VaultCore; use crate::ca::save_to_vault; @@ -17,57 +18,57 @@ static ROOT_KEY: &str = "root_key"; static USER_KEY_PRE: &str = "user_"; -pub async fn get_root_cert_pem() -> String { - match get_from_vault(ROOT_CERT.to_string()).await { +pub fn get_root_cert_pem(vault: &VaultCore) -> String { + match get_from_vault(vault,ROOT_CERT.to_string()) { Some(cert) => cert, - None => init_self_signed_cert().await.0, + None => init_self_signed_cert(vault).0, } } -pub async fn get_root_cert_der() -> CertificateDer<'static> { - let cert = get_root_cert_pem().await; +pub fn get_root_cert_der(vault: &VaultCore) -> CertificateDer<'static> { + let cert = get_root_cert_pem(vault); let cert = CertificateDer::from_pem_slice(cert.as_bytes()).unwrap(); cert } -pub async fn get_root_key_pem() -> String { - match get_from_vault(ROOT_KEY.to_string()).await { +pub fn get_root_key_pem(vault: &VaultCore) -> String { + match get_from_vault(vault, ROOT_KEY.to_string()) { Some(key) => key, - None => init_self_signed_cert().await.1, + None => init_self_signed_cert(vault).1, } } -pub async fn get_root_key_der() -> PrivateKeyDer<'static> { - let key = get_root_key_pem().await; +pub fn get_root_key_der(vault: &VaultCore) -> PrivateKeyDer<'static> { + let key = get_root_key_pem(vault); let key = PrivateKeyDer::from_pem_slice(key.as_bytes()).unwrap(); key } -async fn init_self_signed_cert() -> (String, String) { +fn init_self_signed_cert(vault: &VaultCore) -> (String, String) { let subject_alt_names = vec!["localhost".to_string()]; let CertifiedKey { cert, key_pair } = generate_simple_self_signed(subject_alt_names).unwrap(); - save_to_vault(ROOT_CERT.to_string(), cert.pem()).await; - save_to_vault(ROOT_KEY.to_string(), key_pair.serialize_pem()).await; + save_to_vault(vault, ROOT_CERT.to_string(), cert.pem()); + save_to_vault(vault, ROOT_KEY.to_string(), key_pair.serialize_pem()); (cert.pem(), key_pair.serialize_pem()) } -pub async fn get_certificate(name: String) -> Result { +pub fn get_certificate(vault: &VaultCore, name: String) -> Result { if name == "ca" { - return Ok(get_root_cert_pem().await); + return Ok(get_root_cert_pem(vault)); } - let cert_option = get_from_vault(add_user_key_pre(name)).await; + let cert_option = get_from_vault(vault, add_user_key_pre(name)); match cert_option { Some(cert) => Ok(cert), None => Err(anyhow!("Username not found")), } } -pub async fn issue_certificate(name: String, csr: String) -> Result { +pub fn issue_certificate(vault: &VaultCore, name: String, csr: String) -> Result { tracing::info!("sign_certificate, name:{name},csr:{csr}"); - let ca_key = KeyPair::from_pem(get_root_key_pem().await.as_str()).unwrap(); - let params = CertificateParams::from_ca_cert_pem(get_root_cert_pem().await.as_str()).unwrap(); + let ca_key = KeyPair::from_pem(get_root_key_pem(vault).as_str()).unwrap(); + let params = CertificateParams::from_ca_cert_pem(get_root_cert_pem(vault).as_str()).unwrap(); let ca_cert = params.self_signed(&ca_key).unwrap(); let csrd = match CertificateSigningRequestDer::from_pem_slice(csr.as_bytes()) { @@ -77,7 +78,7 @@ pub async fn issue_certificate(name: String, csr: String) -> Result { let csrq = CertificateSigningRequestParams::from_der(&csrd).unwrap(); let user_cert = csrq.signed_by(&ca_cert, &ca_key).unwrap(); - save_to_vault(add_user_key_pre(name), user_cert.pem()).await; + save_to_vault(vault, add_user_key_pre(name), user_cert.pem()); Ok(user_cert.pem()) } diff --git a/gemini/src/lfs/mod.rs b/gemini/src/lfs/mod.rs index 86f9c366e..f39b7aeb5 100644 --- a/gemini/src/lfs/mod.rs +++ b/gemini/src/lfs/mod.rs @@ -1,180 +1,179 @@ use std::collections::HashSet; -use reqwest::{get, Client}; +use reqwest::get; -use crate::{util::handle_response, LFSInfo, LFSInfoPostBody, LFSInfoRes}; +use crate::{p2p::client::P2PClient, util::handle_response, LFSInfo, LFSInfoPostBody, LFSInfoRes}; -/// share lfs -/// -/// ## paras -/// - `bootstrap_node`: bootstrap_node -/// - `file_hash`: file_hash -/// - `hash_type`: hash_type -/// - `file_size`: file_size -/// - `origin`: origin -/// -/// ## Example -/// Here is an example of the JSON payload: -/// ```json -/// { -/// "bootstrap_node":"https://gitmono.org/relay", -/// "file_hash":"52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6", -/// "hash_type":"sha256", -/// "file_size":199246498, -/// "origin":"p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6" -///} -/// ``` -/// This method will send a Post request to the relay to share lfs -/// -pub async fn share_lfs( - bootstrap_node: String, - file_hash: String, - hash_type: String, - file_size: i64, - origin: String, -) { - let lfs = LFSInfoPostBody { - file_hash, - hash_type, - file_size, - peer_id: vault::get_peerid().await, - origin, - }; - tracing::info!("Share lfs {:?}", lfs); - let json = serde_json::to_string(&lfs).unwrap(); +impl P2PClient { + /// share lfs + /// + /// ## paras + /// - `bootstrap_node`: bootstrap_node + /// - `file_hash`: file_hash + /// - `hash_type`: hash_type + /// - `file_size`: file_size + /// - `origin`: origin + /// + /// ## Example + /// Here is an example of the JSON payload: + /// ```json + /// { + /// "bootstrap_node":"https://gitmono.org/relay", + /// "file_hash":"52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6", + /// "hash_type":"sha256", + /// "file_size":199246498, + /// "origin":"p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6" + ///} + /// ``` + /// This method will send a Post request to the relay to share lfs + /// + pub async fn share_lfs( + &self, + file_hash: String, + hash_type: String, + file_size: i64, + origin: String, + ) { + let lfs = LFSInfoPostBody { + file_hash, + hash_type, + file_size, + peer_id: self.get_peer_id(), + origin, + }; + tracing::info!("Share lfs {:?}", lfs); + let json = serde_json::to_string(&lfs).unwrap(); - let client = Client::new(); - let url = format!("{}/api/v1/lfs_share", bootstrap_node); - let response = client - .post(url) - .header("content-type", "application/json") - .body(json) - .send() - .await - .unwrap(); + let url = format!("{}/api/v1/lfs_share", self.get_bootstrap_node()); + let response = self + .http_client + .post(url) + .header("content-type", "application/json") + .body(json) + .send() + .await + .unwrap(); - if response.status().is_success() { - tracing::info!("Share lfs {} successfully!", lfs.file_hash); - } else { - let context = response.text().await.unwrap(); - tracing::error!("Share lfs {} failed,{}", lfs.file_hash, context); + if response.status().is_success() { + tracing::info!("Share lfs {} successfully!", lfs.file_hash); + } else { + let context = response.text().await.unwrap(); + tracing::error!("Share lfs {} failed,{}", lfs.file_hash, context); + } } -} -/// get lfs chunks info -/// -/// ## paras -/// - `bootstrap_node`: bootstrap_node -/// - `file_hash`: file_hash -/// -/// for example -/// ```json -/// { -/// "bootstrap_node":"https://gitmono.org/relay", -/// "file_hash":"52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6", -///} -/// ``` -/// This method will send a GET request to the relay to get lfs chunks info -/// -pub async fn get_lfs_chunks_info(bootstrap_node: String, file_hash: String) -> Option { - let url = format!( - "{}/api/v1/lfs_chunk?file_hash={}", - bootstrap_node, file_hash - ); - let lfs_info: LFSInfoRes = match get(url.clone()).await { - Ok(response) => { - if !response.status().is_success() { - println!("Get lfs chuncks info failed {}", url); + /// get lfs chunks info + /// + /// ## paras + /// - `bootstrap_node`: bootstrap_node + /// - `file_hash`: file_hash + /// + /// for example + /// ```json + /// { + /// "bootstrap_node":"https://gitmono.org/relay", + /// "file_hash":"52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6", + ///} + /// ``` + /// This method will send a GET request to the relay to get lfs chunks info + /// + pub async fn get_lfs_chunks_info(&self, file_hash: String) -> Option { + let url = format!( + "{}/api/v1/lfs_chunk?file_hash={}", + self.get_bootstrap_node(), + file_hash + ); + let lfs_info: LFSInfoRes = match get(url.clone()).await { + Ok(response) => { + if !response.status().is_success() { + println!("Get lfs chuncks info failed {}", url); + return None; + } + let body = response.text().await.unwrap(); + let lfs_info: LFSInfoRes = serde_json::from_str(&body).unwrap(); + lfs_info + } + Err(_) => { + println!("Get lfs chuncks info failed {}", url); return None; } - let body = response.text().await.unwrap(); - let lfs_info: LFSInfoRes = serde_json::from_str(&body).unwrap(); - lfs_info - } - Err(_) => { - println!("Get lfs chuncks info failed {}", url); - return None; - } - }; - Some(lfs_info) -} + }; + Some(lfs_info) + } -/// create lfs download local ports -/// -/// ## Paras -/// - `bootstrap_node`: bootstrap_node -/// - `ztm_agent_port`: ztm_agent_port -/// - `file_uri`: file_uri -/// -/// for example -/// ```json -/// { -/// "bootstrap_node":"https://gitmono.org/relay", -/// "ztm_agent_port":777, -/// "file_uri":"p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6" -///} -/// ``` -/// ## Return -/// local_port1, local_port2,... -/// -/// Each port is for a remote peer -pub async fn create_lfs_download_tunnel( - bootstrap_node: String, - _ztm_agent_port: u16, - file_uri: String, -) -> Result, String> { - let file_hash = match get_file_hash_from_origin(file_uri) { - Ok(file_hash) => file_hash, - Err(_) => { - return Err("invalid file_uri".to_string()); - } - }; - // get public lfs by bootstrap_node - let url = format!("{bootstrap_node}/api/v1/lfs_list"); - let request_result = reqwest::get(url.clone()).await; - let response_text = match handle_response(request_result).await { - Ok(s) => s, - Err(s) => { - tracing::error!("GET {url} failed,{s}"); - return Err(s); - } - }; - let lfs_list: Vec = match serde_json::from_slice(response_text.as_bytes()) { - Ok(p) => p, - Err(e) => { - tracing::error!("{}", e); - return Err(e.to_string()); - } - }; - let peer_id = vault::get_peerid().await; - let peer_list: HashSet = lfs_list - .iter() - .filter(|x| x.file_hash == file_hash && x.peer_online && x.peer_id != peer_id) - .map(|x| x.peer_id.clone()) - .collect(); - tracing::info!("Search lfs[{}] download peer:{:?}", file_hash, peer_list); + /// create lfs download local ports + /// + /// ## Paras + /// - `bootstrap_node`: bootstrap_node + /// - `ztm_agent_port`: ztm_agent_port + /// - `file_uri`: file_uri + /// + /// for example + /// ```json + /// { + /// "bootstrap_node":"https://gitmono.org/relay", + /// "ztm_agent_port":777, + /// "file_uri":"p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6" + ///} + /// ``` + /// ## Return + /// local_port1, local_port2,... + /// + /// Each port is for a remote peer + pub async fn create_lfs_download_tunnel(&self, file_uri: String) -> Result, String> { + let file_hash = match self.get_file_hash_from_origin(file_uri) { + Ok(file_hash) => file_hash, + Err(_) => { + return Err("invalid file_uri".to_string()); + } + }; + // get public lfs by bootstrap_node + let url = format!("{}/api/v1/lfs_list", self.get_bootstrap_node()); + let request_result = reqwest::get(url.clone()).await; + let response_text = match handle_response(request_result).await { + Ok(s) => s, + Err(s) => { + tracing::error!("GET {url} failed,{s}"); + return Err(s); + } + }; + let lfs_list: Vec = match serde_json::from_slice(response_text.as_bytes()) { + Ok(p) => p, + Err(e) => { + tracing::error!("{}", e); + return Err(e.to_string()); + } + }; + let peer_id = self.get_peer_id(); + let peer_list: HashSet = lfs_list + .iter() + .filter(|x| x.file_hash == file_hash && x.peer_online && x.peer_id != peer_id) + .map(|x| x.peer_id.clone()) + .collect(); + tracing::info!("Search lfs[{}] download peer:{:?}", file_hash, peer_list); - let tunnel_list: Vec = vec![]; - // for peer_id in peer_list { - // match get_or_create_remote_mega_tunnel(ztm_agent_port, peer_id).await { - // Ok(port) => { - // tunnel_list.push(port); - // } - // Err(s) => { - // tracing::error!("{}", s); - // } - // } - // } - Ok(tunnel_list) -} + let tunnel_list: Vec = vec![]; + // for peer_id in peer_list { + // match get_or_create_remote_mega_tunnel(ztm_agent_port, peer_id).await { + // Ok(port) => { + // tunnel_list.push(port); + // } + // Err(s) => { + // tracing::error!("{}", s); + // } + // } + // } + Ok(tunnel_list) + } -pub fn get_file_hash_from_origin(origin: String) -> Result { - // p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6 - let words: Vec<&str> = origin.split('/').collect(); - if words.len() <= 4 { - return Err("invalid origin".to_string()); + pub fn get_file_hash_from_origin(&self, origin: String) -> Result { + // p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6 + let words: Vec<&str> = origin.split('/').collect(); + if words.len() <= 4 { + return Err("invalid origin".to_string()); + } + Ok(words.get(4).unwrap().to_string()) } - Ok(words.get(4).unwrap().to_string()) } #[cfg(test)] diff --git a/gemini/src/nostr/event.rs b/gemini/src/nostr/event.rs index ed5090e16..4c414879e 100644 --- a/gemini/src/nostr/event.rs +++ b/gemini/src/nostr/event.rs @@ -11,7 +11,6 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::fmt; use std::str::FromStr; -use vault::init; use super::GitEvent; @@ -152,12 +151,11 @@ impl TryFrom for NostrEvent { } impl NostrEvent { - pub async fn new(tags: Vec, content: String) -> Self { + pub async fn new(sk: impl AsRef, tags: Vec, content: String) -> Self { let created_at = get_utc_timestamp(); - let (_, sk) = init().await; let secp = Secp256k1::new(); - let keypair = secp256k1::Keypair::from_seckey_str(&secp, &sk).unwrap(); + let keypair = secp256k1::Keypair::from_seckey_str(&secp, sk.as_ref()).unwrap(); Self::new_with_timestamp(keypair, created_at, NostrKind::Mega, tags, content) } diff --git a/gemini/src/nostr/mod.rs b/gemini/src/nostr/mod.rs index 04a756059..86cf61826 100644 --- a/gemini/src/nostr/mod.rs +++ b/gemini/src/nostr/mod.rs @@ -22,9 +22,9 @@ pub struct GitEventReq { } impl GitEventReq { - pub async fn to_git_event(&self, identifier: String, commit: String) -> GitEvent { + pub async fn to_git_event(&self, peer_id: String, identifier: String, commit: String) -> GitEvent { GitEvent { - peer: vault::get_peerid().await, + peer: peer_id, uri: identifier, action: self.action.clone(), r#ref: "".to_string(), diff --git a/gemini/src/p2p/client.rs b/gemini/src/p2p/client.rs index 1ea419cae..a5b6b8595 100644 --- a/gemini/src/p2p/client.rs +++ b/gemini/src/p2p/client.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; @@ -17,14 +18,10 @@ use ceres::protocol::repo::Repo; use common::utils::generate_id; use dashmap::DashMap; use futures_util::{StreamExt, TryStreamExt}; -use jupiter::context::Context; -use lazy_static::lazy_static; +use jupiter::storage::Storage; use mercury::internal::object::types::ObjectType; use mercury::internal::pack::Pack; use quinn::crypto::rustls::QuicClientConfig; -use quinn::rustls::pki_types::pem::PemObject; -use quinn::rustls::pki_types::CertificateDer; -use quinn::rustls::pki_types::PrivateKeyDer; use quinn::Connection; use quinn::{rustls, ClientConfig, Endpoint}; use std::result::Result::Ok; @@ -36,7 +33,7 @@ use tokio_util::io::ReaderStream; use tracing::error; use tracing::info; use uuid::Uuid; -use vault::get_peerid; +use vault::integration::vault_core::VaultCore; use super::{LFSHeader, ALPN_QUIC_HTTP}; use crate::nostr::client_message::{ClientMessage, Filter, SubscriptionId}; @@ -50,753 +47,819 @@ use crate::util::{ get_git_model_by_path, get_path_from_identifier, get_peer_id_from_identifier, get_repo_path, parse_pointer_data, repo_path_to_identifier, }; -use crate::{ca, Node, RepoInfo}; +use crate::{Node, RepoInfo}; -struct MsgSingletonConnection { - conn: Arc, -} -static INSTANCE: OnceLock = OnceLock::new(); +type ReqSenderType = Sender>; -struct BootstrapNode { - bootstrap_node: String, -} -static BOOTSTRAP_NODE_INSTANCE: OnceLock = OnceLock::new(); +#[derive(Clone)] +pub struct P2PClient { + pub storage: Storage, + pub vault: VaultCore, -type ReqSenderType = Sender>; + req_senders: DashMap>>>, + bootstrap_node: OnceLock, + connection: OnceLock>, -lazy_static! { - //oneshot sender map - static ref REQ_SENDER_MAP: DashMap>>> = - DashMap::new(); + pub http_client: reqwest::Client, + pub peer_id: Arc, } -impl MsgSingletonConnection { - fn new(conn: Arc) -> Self { - MsgSingletonConnection { conn } +impl P2PClient { + pub fn new(storage: Storage, vault: VaultCore) -> Self { + let peer_id = vault.load_nostr_peerid(); + P2PClient { + storage, + vault, + req_senders: DashMap::new(), + bootstrap_node: OnceLock::new(), + connection: OnceLock::new(), + http_client: Default::default(), + + peer_id: Arc::from(peer_id), + } } - pub fn init(conn: Arc) { - INSTANCE - .set(Self::new(conn)) - .unwrap_or_else(|_| panic!("Singleton already initialized!")); + pub fn get_bootstrap_node(&self) -> Cow { + let ref_str = self + .bootstrap_node + .get() + .expect("Bootstrap node must be set before using P2P client") + .as_str(); + Cow::Borrowed(ref_str) } - pub fn instance() -> &'static Self { - INSTANCE.get().expect( - "MsgSingletonConnection not initialized. Connecting to bootstrap node may have failed.", - ) + pub fn get_connection(&self) -> Arc { + self.connection + .get() + .expect("Connection must be set before using P2P client") + .clone() } - pub fn get_connection() -> Arc { - MsgSingletonConnection::instance().conn.clone() - } -} -impl BootstrapNode { - fn new(bootstrap_node: String) -> Self { - BootstrapNode { bootstrap_node } + /// Some methods keeps asking for a String typed peer_id, then for it. + /// In ordinary cases, use self.peer_id directly for better performance. + pub fn get_peer_id(&self) -> String { + String::from(self.peer_id.as_ref()) } - pub fn init(bootstrap_node: String) { - BOOTSTRAP_NODE_INSTANCE - .set(Self::new(bootstrap_node)) - .unwrap_or_else(|_| panic!("Singleton already initialized!")); + /// Some methods are designed to be used with an Arc to cross threads. + pub fn wrapped_client(&self) -> Arc { + Arc::new(self.clone()) } - pub fn instance() -> &'static Self { - BOOTSTRAP_NODE_INSTANCE - .get() - .expect("Connecting to bootstrap node may have failed.") - } + pub async fn run(&self, bootstrap_node: String) -> Result<()> { + let peer_id = self.peer_id.clone(); + let (tx, mut rx) = mpsc::channel(8); - pub fn get() -> String { - BootstrapNode::instance().bootstrap_node.clone() - } -} + self.bootstrap_node + .set(bootstrap_node.clone()) + .expect("Bootstrap node must be set only once"); -pub async fn run(context: Context, bootstrap_node: String) -> Result<()> { - let connection = match get_client_connection(bootstrap_node.clone()).await { - Ok(connection) => connection, - Err(e) => { - bail!("P2P: Connect to {} failed, {}", bootstrap_node, e); - } - }; - - let connection = Arc::new(connection); + let connection = match self.new_client_connection().await { + Ok(connection) => Arc::new(connection), + Err(e) => { + bail!( + "P2P: Connect to {} failed, {}", + self.get_bootstrap_node(), + e + ); + } + }; - MsgSingletonConnection::init(connection.clone()); - BootstrapNode::init(bootstrap_node.clone()); + self.connection + .set(connection.clone()) + .expect("Connection must be set only once"); - let (tx, mut rx) = mpsc::channel(8); + // Register msg connection to relay + let (mut send, _) = connection.clone().open_bi().await?; + send.write_all(format!("{}|{}", peer_id, "MSG").as_bytes()) + .await?; + send.finish()?; - let peer_id = get_peerid().await; + let client = self.wrapped_client(); + tokio::spawn(async move { + if let Err(e) = client.run_ping_task(peer_id).await { + error!("P2P: Ping Task Error, {}", e); + } + }); - // Register msg connection to relay - let connection_clone = MsgSingletonConnection::get_connection(); - let (mut send, _) = connection_clone.open_bi().await?; - send.write_all(format!("{}|{}", peer_id.clone(), "MSG").as_bytes()) - .await?; - send.finish()?; + let client = self.wrapped_client(); + tokio::spawn(async move { + if let Err(e) = client.receive_quic_msg_task(tx.clone()).await { + error!("P2P: Receive quic msg Error, {}", e); + } + }); - tokio::spawn(async move { - if let Err(e) = run_ping_task(peer_id.clone()).await { - error!("P2P: Ping Task Error, {}", e); + while let Some(message) = rx.recv().await { + let client = self.wrapped_client(); + tokio::spawn(async move { + if let Err(e) = client.handle_quic_msg_task(message).await { + error!("P2P: Handle quic msg Error, {}", e); + } + }); } - }); + Ok(()) + } - let connection_clone = connection.clone(); - tokio::spawn(async move { - if let Err(e) = receive_quic_msg_task(connection_clone, tx.clone()).await { - error!("P2P: Receive quic msg Error, {}", e); - } - }); + async fn new_client_connection(&self) -> Result { + let bootstrap_node = self + .bootstrap_node + .get() + .ok_or_else(|| anyhow!("Bootstrap node must be set before using P2P client"))?; + let (user_cert, user_key) = self.get_user_cert_from_ca(bootstrap_node).await?; + let ca_cert = self.get_ca_cert_from_ca(bootstrap_node).await?; + + let mut roots = rustls::RootCertStore::empty(); + + roots.add(ca_cert)?; + + let mut client_crypto = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_client_auth_cert([user_cert].to_vec(), user_key)?; + client_crypto.alpn_protocols = ALPN_QUIC_HTTP.iter().map(|&x| x.into()).collect(); + client_crypto.enable_early_data = true; + let client_config = ClientConfig::new(Arc::new(QuicClientConfig::try_from(client_crypto)?)); + let mut endpoint = Endpoint::client(SocketAddr::from_str("[::]:0")?)?; + endpoint.set_default_client_config(client_config); + + let server_addr: SocketAddr = self.bootstrap_node.get().unwrap().parse()?; + let connection = endpoint + .connect(server_addr, "localhost")? + .await + .map_err(|e| anyhow!("failed to connect: {}", e))?; + + let remote_address = connection.remote_address(); + let stable_id = connection.stable_id(); + info!("Established connection: {remote_address:#?},{stable_id:#?}"); + Ok(connection) + } - while let Some(message) = rx.recv().await { - let context = context.clone(); - tokio::spawn(async move { - if let Err(e) = handle_quic_msg_task(context, message).await { - error!("P2P: Handle quic msg Error, {}", e); - } + // pub async fn call(to_peer_id: String, func: String, data: Vec) -> Result> { + // let (tx, rx) = tokio::sync::oneshot::channel(); + // + // let connection = MsgSingletonConnection::get_connection(); + // + // let connection_clone = connection.clone(); + // let local_peer_id = get_peerid().await; + // tokio::spawn(async move { + // let (mut sender, _) = connection_clone.open_bi().await.unwrap(); + // let send = RequestData { + // from: local_peer_id.clone(), + // data: data.clone(), + // func: func.to_string(), + // action: Action::Call, + // to: to_peer_id.to_string(), + // req_id: Uuid::new_v4().into(), + // }; + // let json = serde_json::to_string(&send).unwrap(); + // sender.write_all(json.as_bytes()).await.unwrap(); + // sender.finish().unwrap(); + // }); + // + // let connection_clone = connection.clone(); + // + // tokio::spawn(async move { + // let (_, mut quic_recv) = connection_clone.accept_bi().await.unwrap(); + // let buffer = quic_recv.read_to_end(1024 * 1024).await.unwrap(); + // info!("QUIC Received:\n{}", String::from_utf8_lossy(&buffer)); + // if tx.send(buffer).is_err() { + // info!("Receiver closed"); + // } + // }); + // let message = rx.await?; + // let data: ResponseData = serde_json::from_slice(&message)?; + // Ok(data.data) + // } + + pub async fn send(&self, to_peer_id: String, func: String, data: Vec) -> Result<()> { + let peer_id = self.get_peer_id(); + let client = self.wrapped_client(); + let t = tokio::spawn(async move { + let send = RequestData { + from: peer_id, + data: data.clone(), + func: func.to_string(), + action: Action::Send, + to: to_peer_id.to_string(), + req_id: Uuid::new_v4().into(), + }; + if let Err(e) = client.send_request(send).await { + error!("failed to send request: {e}"); + }; }); + let _ = join!(t); + Ok(()) } - Ok(()) -} -async fn run_ping_task(peer_id: String) -> Result<()> { - loop { - let connection_clone = MsgSingletonConnection::get_connection(); - let (mut quic_send, _) = connection_clone.open_bi().await?; + pub async fn repo_share(&self, path: String) -> Result { + let db = self.storage.services.git_db_storage.clone(); + let client = self.wrapped_client(); - let ping = RequestData { - from: peer_id.clone(), - data: vec![], - func: "".to_string(), - action: Action::Ping, - to: "relay".to_string(), - req_id: Uuid::new_v4().into(), + let repo: git_repo::Model = + match get_git_model_by_path(self.storage.clone(), path.clone()).await { + None => { + bail!("Repo not found: {}", path); + } + Some(repo) => repo, + }; + let commit = match db.get_last_commit_by_repo_id(repo.id).await { + Ok(commit) => commit, + Err(e) => bail!(e), + }; + let commit = match commit { + Some(commit) => commit, + None => { + bail!("Repo commit error"); + } }; - let json = serde_json::to_string(&ping)?; - quic_send.write_all(json.as_ref()).await?; - quic_send.finish()?; - tokio::time::sleep(Duration::from_secs(60)).await; + let mut repo_info: RepoInfo = repo.clone().into(); + let identifier = repo_path_to_identifier(self.peer_id.as_ref(), repo.repo_path).await; + repo_info.identifier = identifier.clone(); + repo_info.commit = commit.commit_id; + repo_info.update_time = commit.created_at.and_utc().timestamp(); + repo_info.origin = self.get_peer_id(); + + let local_peer_id = self.get_peer_id(); + let req_id: String = Uuid::new_v4().into(); + + let (tx, rx) = tokio::sync::oneshot::channel(); + self.req_senders + .insert(req_id.clone(), Arc::new(Mutex::new(Some(tx)))); + tokio::spawn(async move { + let send = RequestData { + from: local_peer_id, + data: repo_info.to_json().into_bytes(), + func: "".to_string(), + action: Action::RepoShare, + to: "".to_string(), + req_id: req_id.clone(), + }; + + if let Err(e) = client.send_request(send).await { + error!("failed to send request: {e}"); + } + }); + let _message = wait_rx_with_timeout(rx).await?; + info!("Repo share success: {}", identifier); + Ok(identifier) } -} -async fn receive_quic_msg_task( - connection: Arc, - tx: mpsc::Sender>, -) -> Result<()> { - loop { - let (_, mut quic_recv) = connection.accept_bi().await?; - let buffer = quic_recv.read_to_end(1024 * 1024).await?; - info!("QUIC Received:\n{}", String::from_utf8_lossy(&buffer)); - if tx.send(buffer).await.is_err() { - info!("Receiver closed"); - } + pub async fn repo_clone(&self, identifier: String) -> Result { + let remote_peer_id = match get_peer_id_from_identifier(identifier.clone()) { + Ok(p) => p, + Err(_e) => { + bail!("Identifier invalid"); + } + }; + let path = match get_path_from_identifier(identifier.clone()) { + Ok(p) => p, + Err(_e) => { + bail!("Identifier invalid"); + } + }; + let path = get_repo_path(path); + self.request_git_clone(path, remote_peer_id).await?; + Ok(identifier.clone()) } -} -async fn handle_quic_msg_task(context: Context, message: Vec) -> Result<()> { - let data: ResponseData = serde_json::from_slice(&message)?; - match data.func.as_str() { - "request_git_clone" => { - let path = String::from_utf8(data.data)?; - response_git_clone(context.clone(), path, data.from).await?; - } - "request_lfs" => { - let oid = String::from_utf8(data.data)?; - response_lfs(context.clone(), oid, data.from).await?; - } - "nostr" => { - receive_nostr(data.data).await?; - } - "" => { - if let Some(sender) = REQ_SENDER_MAP.get(data.req_id.as_str()) { - let mut guard = sender.lock().await; - if let Some(tx) = guard.take() { - tx.send(data.data).expect("Sender error"); - } - } - } - _ => { - error!("Unsupported function"); + async fn request_git_clone(&self, path: String, to_peer_id: String) -> Result<()> { + let db = self.storage.services.git_db_storage.clone(); + let model = match db.find_git_repo_exact_match(path.as_str()).await { + Ok(model) => model, + Err(e) => bail!(e), + }; + if model.is_some() { + bail!("Repo path already exists"); } - } - Ok(()) -} -async fn get_client_connection(bootstrap_node: String) -> Result { - let (user_cert, user_key) = get_user_cert_from_ca(bootstrap_node.clone()).await?; - let ca_cert = get_ca_cert_from_ca(bootstrap_node.clone()).await?; - - let mut roots = rustls::RootCertStore::empty(); - - roots.add(ca_cert)?; - - let mut client_crypto = rustls::ClientConfig::builder() - .with_root_certificates(roots) - .with_client_auth_cert([user_cert].to_vec(), user_key)?; - client_crypto.alpn_protocols = ALPN_QUIC_HTTP.iter().map(|&x| x.into()).collect(); - client_crypto.enable_early_data = true; - let client_config = ClientConfig::new(Arc::new(QuicClientConfig::try_from(client_crypto)?)); - let mut endpoint = Endpoint::client(SocketAddr::from_str("[::]:0")?)?; - endpoint.set_default_client_config(client_config); - - let server_addr: SocketAddr = bootstrap_node.parse()?; - let connection = endpoint - .connect(server_addr, "localhost")? - .await - .map_err(|e| anyhow!("failed to connect: {}", e))?; - - let remote_address = connection.remote_address(); - let stable_id = connection.stable_id(); - info!("Established connection: {remote_address:#?},{stable_id:#?}"); - Ok(connection) -} + // Register file connection to relay + let file_connection = self.new_client_connection().await?; + let (mut file_sender, mut _file_receiver) = file_connection.open_bi().await?; + file_sender + .write_all( + format!( + "git-clone-{}-{}|{}", + self.peer_id, to_peer_id, "REQUEST_GIT_CLONE" + ) + .as_bytes(), + ) + .await?; + file_sender.finish()?; + + //send git clone request msg via msg connection + let (mut msg_sender, _) = self.get_connection().open_bi().await?; -// pub async fn call(to_peer_id: String, func: String, data: Vec) -> Result> { -// let (tx, rx) = tokio::sync::oneshot::channel(); -// -// let connection = MsgSingletonConnection::get_connection(); -// -// let connection_clone = connection.clone(); -// let local_peer_id = get_peerid().await; -// tokio::spawn(async move { -// let (mut sender, _) = connection_clone.open_bi().await.unwrap(); -// let send = RequestData { -// from: local_peer_id.clone(), -// data: data.clone(), -// func: func.to_string(), -// action: Action::Call, -// to: to_peer_id.to_string(), -// req_id: Uuid::new_v4().into(), -// }; -// let json = serde_json::to_string(&send).unwrap(); -// sender.write_all(json.as_bytes()).await.unwrap(); -// sender.finish().unwrap(); -// }); -// -// let connection_clone = connection.clone(); -// -// tokio::spawn(async move { -// let (_, mut quic_recv) = connection_clone.accept_bi().await.unwrap(); -// let buffer = quic_recv.read_to_end(1024 * 1024).await.unwrap(); -// info!("QUIC Received:\n{}", String::from_utf8_lossy(&buffer)); -// if tx.send(buffer).is_err() { -// info!("Receiver closed"); -// } -// }); -// let message = rx.await?; -// let data: ResponseData = serde_json::from_slice(&message)?; -// Ok(data.data) -// } - -pub async fn send(to_peer_id: String, func: String, data: Vec) -> Result<()> { - let local_peer_id = get_peerid().await; - let t = tokio::spawn(async move { let send = RequestData { - from: local_peer_id.clone(), - data: data.clone(), - func: func.to_string(), + from: self.get_peer_id(), + data: path.as_bytes().to_vec(), + func: "request_git_clone".to_string(), action: Action::Send, to: to_peer_id.to_string(), req_id: Uuid::new_v4().into(), }; - if let Err(e) = send_request(send).await { - error!("failed to send request: {e}"); + let json = serde_json::to_string(&send)?; + msg_sender.write_all(json.as_bytes()).await?; + msg_sender.finish()?; + + //receive header + let (_file_sender, mut file_receiver) = file_connection.accept_bi().await?; + let mut header_buf = [0u8; 1024]; + let header = match file_receiver.read(&mut header_buf).await? { + Some(len) => String::from_utf8_lossy(&header_buf[..len]), + None => { + bail!("failed to read header"); + } }; - }); - let _ = join!(t); - Ok(()) -} - -async fn send_request(data: RequestData) -> Result<()> { - let connection = MsgSingletonConnection::get_connection(); - let (mut sender, _) = connection.open_bi().await?; - let json = serde_json::to_string(&data)?; - sender.write_all(json.as_bytes()).await?; - sender.finish()?; - Ok(()) -} - -pub async fn repo_share(context: Context, path: String) -> Result { - let storage = context.services.git_db_storage.clone(); - - let repo: git_repo::Model = match get_git_model_by_path(context.clone(), path.clone()).await { - None => { - bail!("Repo not found: {}", path); + let header: GitCloneHeader = serde_json::from_str(&header)?; + let (target_id, from, git_path) = (header.target, header.from, header.git_path); + if target_id != *self.peer_id { + bail!("Invalid Connection stream,target_id != peer_id") } - Some(repo) => repo, - }; - let commit = match storage.get_last_commit_by_repo_id(repo.id).await { - Ok(commit) => commit, - Err(e) => bail!(e), - }; - let commit = match commit { - Some(commit) => commit, - None => { - bail!("Repo commit error"); + if git_path != path { + bail!("Invalid Connection stream,target_path != request_path") } - }; - let mut repo_info: RepoInfo = repo.clone().into(); - let identifier = repo_path_to_identifier(repo.repo_path).await; - repo_info.identifier = identifier.clone(); - repo_info.commit = commit.commit_id; - repo_info.update_time = commit.created_at.and_utc().timestamp(); - repo_info.origin = get_peerid().await; - - let local_peer_id = get_peerid().await; - let req_id: String = Uuid::new_v4().into(); - - let (tx, rx) = tokio::sync::oneshot::channel(); - REQ_SENDER_MAP.insert(req_id.clone(), Arc::new(Mutex::new(Some(tx)))); - tokio::spawn(async move { - let send = RequestData { - from: local_peer_id.clone(), - data: repo_info.to_json().into_bytes(), - func: "".to_string(), - action: Action::RepoShare, - to: "".to_string(), - req_id: req_id.clone(), - }; - - if let Err(e) = send_request(send).await { - error!("failed to send request: {e}"); + info!( + "Receive git clone response from [{}], path:{}", + from, git_path + ); + + //Receive git encode objects + let (_file_sender, file_receiver) = file_connection.accept_bi().await?; + + let stream = ReaderStream::new(file_receiver).map_err(axum::Error::new); + let repo = Repo::new(get_repo_path(path).parse()?, false); + + //decode the git objects + let (sender, mut receiver) = mpsc::unbounded_channel(); + + let p = Pack::new( + None, + Some(1024 * 1024 * 1024 * 4), + Some(self.storage.config().pack.pack_decode_cache_path.clone()), + self.storage.config().pack.clean_cache_after_decode, + ); + p.decode_stream(stream, sender).await; + let mut entry_list = vec![]; + while let Some(entry) = receiver.recv().await { + entry_list.push(entry); } - }); - let _message = wait_rx_with_timeout(rx).await?; - info!("Repo share success: {}", identifier); - Ok(identifier) -} -pub async fn repo_clone(context: Context, identifier: String) -> Result { - let remote_peer_id = match get_peer_id_from_identifier(identifier.clone()) { - Ok(p) => p, - Err(_e) => { - bail!("Identifier invalid"); + // deal lfs blob + let mut task = vec![]; + for blob in entry_list.iter().filter(|e| e.obj_type == ObjectType::Blob) { + let oid = parse_pointer_data(&blob.data); + if let Some(oid) = oid { + let client = self.wrapped_client(); + let to_peer_id = to_peer_id.clone(); + let t = tokio::spawn(async move { + //try to download lfs + match client + .request_lfs(oid.0.to_string(), to_peer_id.clone()) + .await + { + Ok(_) => {} + Err(e) => { + error!("failed request lfs: {e}"); + } + }; + }); + task.push(t); + } } - }; - let path = match get_path_from_identifier(identifier.clone()) { - Ok(p) => p, - Err(_e) => { - bail!("Identifier invalid"); + futures::future::join_all(task).await; + //Save to db + if let Err(e) = db.save_git_repo(repo.clone().into()).await { + bail!("failed to save git repo: {}", e); + }; + if let Err(e) = db.save_entry(repo.repo_id, entry_list).await { + bail!("failed to save entry for repo: {}", e); + }; + for x in header.branches { + let r = import_refs::Model { + id: generate_id(), + repo_id: repo.repo_id, + ref_name: x.ref_name, + ref_git_id: x.ref_git_id, + ref_type: x.ref_type.clone(), + default_branch: x.default_branch, + created_at: chrono::Utc::now().naive_utc(), + updated_at: chrono::Utc::now().naive_utc(), + }; + if let Err(e) = db.save_ref(repo.repo_id, r).await { + bail!("failed to save reference: {}", e); + } } - }; - let path = get_repo_path(path); - request_git_clone(context, path, remote_peer_id).await?; - Ok(identifier.clone()) -} -async fn request_git_clone(context: Context, path: String, to_peer_id: String) -> Result<()> { - let storage = context.services.git_db_storage.clone(); - let model = match storage.find_git_repo_exact_match(path.as_str()).await { - Ok(model) => model, - Err(e) => bail!(e), - }; - if model.is_some() { - bail!("Repo path already exists"); + info!( + "Git clone from[{}] with path[{}] successfully", + to_peer_id, git_path + ); + + Ok(()) } - // Register file connection to relay - let bootstrap_node = BootstrapNode::get(); - let file_connection = get_client_connection(bootstrap_node.clone()).await?; - let (mut file_sender, mut _file_receiver) = file_connection.open_bi().await?; - let peer_id = get_peerid().await; - file_sender - .write_all( - format!( - "git-clone-{}-{}|{}", - peer_id.clone(), - to_peer_id, - "REQUEST_GIT_CLONE" + async fn response_git_clone(&self, path: String, to_peer_id: String) -> Result<()> { + let db = self.storage.services.git_db_storage.clone(); + + let repo: Repo = match get_git_model_by_path(self.storage.clone(), path.clone()).await { + None => { + bail!("Repo not found: {}", path); + } + Some(repo) => repo.into(), + }; + // Register file connection to relay + let file_connection = self.new_client_connection().await?; + let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; + + file_sender + .write_all( + format!( + "git-clone-{}-{}|{}", + self.peer_id, to_peer_id, "RESPONSE_GIT_CLONE" + ) + .as_bytes(), ) - .as_bytes(), - ) - .await?; - file_sender.finish()?; - - //send git clone request msg via msg connection - let (mut msg_sender, _) = MsgSingletonConnection::get_connection().open_bi().await?; - - let send = RequestData { - from: get_peerid().await, - data: path.as_bytes().to_vec(), - func: "request_git_clone".to_string(), - action: Action::Send, - to: to_peer_id.to_string(), - req_id: Uuid::new_v4().into(), - }; - let json = serde_json::to_string(&send)?; - msg_sender.write_all(json.as_bytes()).await?; - msg_sender.finish()?; - - //receive header - let (_file_sender, mut file_receiver) = file_connection.accept_bi().await?; - let mut header_buf = [0u8; 1024]; - let header = match file_receiver.read(&mut header_buf).await? { - Some(len) => String::from_utf8_lossy(&header_buf[..len]), - None => { - bail!("failed to read header"); + .await?; + file_sender.finish()?; + + //send git clone header + let refs = match db.get_ref(repo.repo_id).await { + Ok(refs) => refs, + Err(e) => bail!("failed to fetch refs: {e}"), + }; + let header = GitCloneHeader { + from: self.get_peer_id(), + target: to_peer_id.clone(), + git_path: path.clone(), + branches: refs, + }; + let header = serde_json::to_string(&header)?; + let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; + file_sender.write_all(header.as_bytes()).await?; + file_sender.finish()?; + + //send encoded git objects + let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; + let mut receiver = self.get_encode_git_objects_by_repo(repo).await?; + while let Some(data) = receiver.recv().await { + file_sender.write_all(&data).await?; } - }; - let header: GitCloneHeader = serde_json::from_str(&header)?; - let (target_id, from, git_path) = (header.target, header.from, header.git_path); - if target_id != peer_id { - bail!("Invalid Connection stream,target_id != peer_id") - } - if git_path != path { - bail!("Invalid Connection stream,target_path != request_path") + file_sender.finish()?; + + //wait finish msg + match file_connection.accept_bi().await { + Ok((_file_sender, mut file_receiver)) => { + file_receiver.read_to_end(1024).await?; + } + Err(_) => { + info!("Git clone connection closed."); + } + }; + info!( + "Send git clone data to[{}] with path[{}] successfully", + to_peer_id, path + ); + Ok(()) } - info!( - "Receive git clone response from [{}], path:{}", - from, git_path - ); - - //Receive git encode objects - let (_file_sender, file_receiver) = file_connection.accept_bi().await?; - - let stream = ReaderStream::new(file_receiver).map_err(axum::Error::new); - let repo = Repo::new(get_repo_path(path).parse()?, false); - - //decode the git objects - let (sender, mut receiver) = mpsc::unbounded_channel(); - let p = Pack::new( - None, - Some(1024 * 1024 * 1024 * 4), - Some(context.config.pack.pack_decode_cache_path.clone()), - context.config.pack.clean_cache_after_decode, - ); - p.decode_stream(stream, sender).await; - let mut entry_list = vec![]; - while let Some(entry) = receiver.recv().await { - entry_list.push(entry); + + pub async fn repo_subscribe(&self, identifier: String) -> Result<()> { + let filters = vec![Filter::new().repo_uri(identifier)]; + let subscription_id = self.get_peer_id(); + let client_req = ClientMessage::new_req(SubscriptionId::new(subscription_id), filters); + + let relay_message = self.send_nostr_msg(client_req).await?; + info!("Subscribe repo result: {}", relay_message.as_json()); + Ok(()) } - // deal lfs blob - let mut task = vec![]; - for blob in entry_list.iter().filter(|e| e.obj_type == ObjectType::Blob) { - let oid = parse_pointer_data(&blob.data); - if let Some(oid) = oid { - let context = context.clone(); - let to_peer_id = to_peer_id.clone(); - let t = tokio::spawn(async move { - //try to download lfs - match request_lfs(context.clone(), oid.0.to_string(), to_peer_id.clone()).await { - Ok(_) => {} - Err(e) => { - error!("failed request lfs: {e}"); - } - }; - }); - task.push(t); - } + pub async fn send_git_event(&self, git_event: GitEvent) -> Result<()> { + let keypair = self.vault.load_nostr_secp_pair(); + let event = NostrEvent::new_git_event(keypair, git_event); + let client_message = ClientMessage::new_event(event); + let relay_message = self.send_nostr_msg(client_message).await?; + info!("Sent git event result: {}", relay_message.as_json()); + Ok(()) } - futures::future::join_all(task).await; - //Save to db - if let Err(e) = storage.save_git_repo(repo.clone().into()).await { - bail!("failed to save git repo: {}", e); - }; - if let Err(e) = storage.save_entry(repo.repo_id, entry_list).await { - bail!("failed to save entry for repo: {}", e); - }; - for x in header.branches { - let r = import_refs::Model { - id: generate_id(), - repo_id: repo.repo_id, - ref_name: x.ref_name, - ref_git_id: x.ref_git_id, - ref_type: x.ref_type.clone(), - default_branch: x.default_branch, - created_at: chrono::Utc::now().naive_utc(), - updated_at: chrono::Utc::now().naive_utc(), + + pub async fn get_peers(&self) -> Result> { + let (tx, rx) = tokio::sync::oneshot::channel(); + let req_id: String = Uuid::new_v4().into(); + + self.req_senders + .insert(req_id.clone(), Arc::new(Mutex::new(Some(tx)))); + let send = RequestData { + from: self.get_peer_id(), + data: vec![], + func: "".to_string(), + action: Action::Peers, + to: "".to_string(), + req_id: req_id.clone(), }; - if let Err(e) = storage.save_ref(repo.repo_id, r).await { - bail!("failed to save reference: {}", e); + if let Err(e) = self.wrapped_client().send_request(send).await { + error!("failed to get peers: {e}"); } + + let res = wait_rx_with_timeout(rx).await?; + let peers: Vec = serde_json::from_slice(res.as_slice())?; + Ok(peers) } - info!( - "Git clone from[{}] with path[{}] successfully", - to_peer_id, git_path - ); + pub async fn get_repos(&self) -> Result> { + let (tx, rx) = tokio::sync::oneshot::channel(); + let req_id: String = Uuid::new_v4().into(); - Ok(()) -} + self.req_senders + .insert(req_id.clone(), Arc::new(Mutex::new(Some(tx)))); + let send = RequestData { + from: self.get_peer_id(), + data: vec![], + func: "".to_string(), + action: Action::Repos, + to: "".to_string(), + req_id: req_id.clone(), + }; -async fn response_git_clone(context: Context, path: String, to_peer_id: String) -> Result<()> { - let bootstrap_node = BootstrapNode::get(); - let storage = context.services.git_db_storage.clone(); + if let Err(e) = self.wrapped_client().send_request(send).await { + error!("failed to get repos: {e}"); + } + let res = wait_rx_with_timeout(rx).await?; + let repo_list: Vec = serde_json::from_slice(res.as_slice())?; + Ok(repo_list) + } - let repo: Repo = match get_git_model_by_path(context.clone(), path.clone()).await { - None => { - bail!("Repo not found: {}", path); + async fn get_encode_git_objects_by_repo(&self, repo: Repo) -> Result>> { + let import_repo = ImportRepo { + storage: self.storage.clone(), + repo, + command_list: vec![], + }; + match import_repo.full_pack(vec![]).await { + Ok(s) => Ok(s.into_inner()), + Err(e) => bail!("full pack repo failed: {}", e), } - Some(repo) => repo.into(), - }; - // Register file connection to relay - let file_connection = get_client_connection(bootstrap_node).await?; - let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; - let peer_id = get_peerid().await; - file_sender - .write_all( - format!( - "git-clone-{}-{}|{}", - peer_id.clone(), - to_peer_id, - "RESPONSE_GIT_CLONE" - ) - .as_bytes(), - ) - .await?; - file_sender.finish()?; - - //send git clone header - let refs = match storage.get_ref(repo.repo_id).await { - Ok(refs) => refs, - Err(e) => bail!("failed to fetch refs: {e}"), - }; - let header = GitCloneHeader { - from: peer_id.clone(), - target: to_peer_id.clone(), - git_path: path.clone(), - branches: refs, - }; - let header = serde_json::to_string(&header)?; - let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; - file_sender.write_all(header.as_bytes()).await?; - file_sender.finish()?; - - //send encoded git objects - let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; - let mut receiver = get_encode_git_objects_by_repo(context, repo).await?; - while let Some(data) = receiver.recv().await { - file_sender.write_all(&data).await?; } - file_sender.finish()?; - //wait finish msg - match file_connection.accept_bi().await { - Ok((_file_sender, mut file_receiver)) => { - file_receiver.read_to_end(1024).await?; + async fn request_lfs(self: Arc, oid: String, to_peer_id: String) -> Result<()> { + // Register file connection to relay + let file_connection = self.new_client_connection().await?; + let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; + file_sender + .write_all(format!("lfs-{}-{}|{}", self.peer_id, to_peer_id, "REQUEST_LFS").as_bytes()) + .await?; + file_sender.finish()?; + + //send request lfs request msg via msg connection + let (mut msg_sender, _) = self.get_connection().open_bi().await?; + + let send = RequestData { + from: self.get_peer_id(), + data: oid.as_bytes().to_vec(), + func: "request_lfs".to_string(), + action: Action::Send, + to: to_peer_id.to_string(), + req_id: Uuid::new_v4().into(), + }; + let json = serde_json::to_string(&send)?; + msg_sender.write_all(json.as_bytes()).await?; + msg_sender.finish()?; + + //receive header + let (_file_sender, mut file_receiver) = file_connection.accept_bi().await?; + let mut header_buf = [0u8; 1024]; + let len = match file_receiver.read(&mut header_buf).await? { + Some(n) => n, + None => bail!("failed to read header"), + }; + let header = String::from_utf8_lossy(&header_buf[..len]); + let header: LFSHeader = serde_json::from_str(&header)?; + info!("LFS handle receive, {:?}", header); + if header.target != *self.peer_id { + bail!("Invalid Connection stream,target_id != peer_id") } - Err(_) => { - info!("Git clone connection closed."); + if oid != header.oid { + bail!("Invalid Connection stream,oid != header.oid") } - }; - info!( - "Send git clone data to[{}] with path[{}] successfully", - to_peer_id, path - ); - Ok(()) -} - -async fn request_lfs(context: Context, oid: String, to_peer_id: String) -> Result<()> { - let bootstrap_node = BootstrapNode::get(); - // Register file connection to relay - let file_connection = get_client_connection(bootstrap_node).await?; - let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; - let peer_id = get_peerid().await; - file_sender - .write_all(format!("lfs-{}-{}|{}", peer_id.clone(), to_peer_id, "REQUEST_LFS").as_bytes()) - .await?; - file_sender.finish()?; - - //send request lfs request msg via msg connection - let (mut msg_sender, _) = MsgSingletonConnection::get_connection().open_bi().await?; - - let send = RequestData { - from: get_peerid().await, - data: oid.as_bytes().to_vec(), - func: "request_lfs".to_string(), - action: Action::Send, - to: to_peer_id.to_string(), - req_id: Uuid::new_v4().into(), - }; - let json = serde_json::to_string(&send)?; - msg_sender.write_all(json.as_bytes()).await?; - msg_sender.finish()?; - - //receive header - let (_file_sender, mut file_receiver) = file_connection.accept_bi().await?; - let mut header_buf = [0u8; 1024]; - let len = match file_receiver.read(&mut header_buf).await? { - Some(n) => n, - None => bail!("failed to read header"), - }; - let header = String::from_utf8_lossy(&header_buf[..len]); - let header: LFSHeader = serde_json::from_str(&header)?; - info!("LFS handle receive, {:?}", header); - if header.target != peer_id { - bail!("Invalid Connection stream,target_id != peer_id") - } - if oid != header.oid { - bail!("Invalid Connection stream,oid != header.oid") - } - info!( - "Start download lfs from [{}], oid:{}, size:{}", - header.from, header.oid, header.size - ); - - //Receive lfs data - let (_file_sender, mut file_receiver) = file_connection.accept_bi().await?; - let mut data: Vec = vec![]; - let mut buffer = vec![0; 1024 * 8]; - while let Ok(bytes_read) = file_receiver.read(&mut buffer).await { - match bytes_read { - Some(bytes_read) => { - data.append(&mut buffer[..bytes_read].to_vec()); + info!( + "Start download lfs from [{}], oid:{}, size:{}", + header.from, header.oid, header.size + ); + + //Receive lfs data + let (_file_sender, mut file_receiver) = file_connection.accept_bi().await?; + let mut data: Vec = vec![]; + let mut buffer = vec![0; 1024 * 8]; + while let Ok(bytes_read) = file_receiver.read(&mut buffer).await { + match bytes_read { + Some(bytes_read) => { + data.append(&mut buffer[..bytes_read].to_vec()); + } + None => { + break; + } } - None => { - break; + } + // let data = file_receiver.read_to_end(header.size as usize).await?; + info!( + "Download lfs from [{}], oid:{}, size:{} successfully", + header.from, header.oid, header.size + ); + let splited = self.storage.config().lfs.local.enable_split; + let meta_to = lfs_objects::Model { + oid: header.oid, + size: header.size, + exist: true, + splited, + }; + + let res = self.storage.lfs_db_storage().new_lfs_object(meta_to).await; + match res { + Ok(_) => {} + Err(e) => { + error!("Insert lfs object failed:{}", e); } } - } - // let data = file_receiver.read_to_end(header.size as usize).await?; - info!( - "Download lfs from [{}], oid:{}, size:{} successfully", - header.from, header.oid, header.size - ); - let config = context.config.lfs.clone(); - let meta_to = lfs_objects::Model { - oid: header.oid, - size: header.size, - exist: true, - splited: config.local.enable_split, - }; - - let res = context.lfs_stg().new_lfs_object(meta_to).await; - match res { - Ok(_) => {} - Err(e) => { - error!("Insert lfs object failed:{}", e); + + // Load request parameters into struct. + let req_obj = RequestObject { + oid, + ..Default::default() + }; + + let result = handler::lfs_upload_object(&self.storage.clone(), &req_obj, data).await; + + match result { + Ok(_) => { + info!("Upload lfs successfully",); + } + Err(e) => { + error!("Upload lfs failed:{}", e); + } } + Ok(()) } - // Load request parameters into struct. - let req_obj = RequestObject { - oid, - ..Default::default() - }; + async fn response_lfs(self: Arc, oid: String, to_peer_id: String) -> Result<()> { + info!("oid:{}", oid.clone()); + let result = match self + .storage + .lfs_db_storage() + .get_lfs_object(oid.as_str()) + .await + { + Ok(m) => m, + Err(e) => bail!(e), + }; - let result = handler::lfs_upload_object(&context.clone(), &req_obj, data).await; + let lfs_object = match result { + None => { + bail!("LFS not found: {}", oid); + } + Some(o) => o, + }; - match result { - Ok(_) => { - info!("Upload lfs successfully",); - } - Err(e) => { - error!("Upload lfs failed:{}", e); + // Register lfs connection to relay + let file_connection = self.new_client_connection().await?; + let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; + file_sender + .write_all(format!("lfs-{}-{}|{}", self.peer_id, to_peer_id, "RESPONSE_LFS").as_bytes()) + .await?; + file_sender.finish()?; + + //send lfs header + let header = LFSHeader { + from: self.get_peer_id(), + target: to_peer_id.clone(), + oid: oid.clone(), + size: lfs_object.size, + }; + let header = serde_json::to_string(&header)?; + let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; + file_sender.write_all(header.as_bytes()).await?; + file_sender.finish()?; + + //send data + let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; + let mut result = lfs_download_object(self.storage.clone(), oid.clone()).await?; + while let Some(d) = result.next().await { + match d { + Ok(bytes_chunk) => { + info!("bytes_chunk:{}", bytes_chunk.len()); + file_sender.write_all(&bytes_chunk).await?; + } + Err(e) => { + bail!("LFS send error: {}", e); + } + } } + file_sender.finish()?; + + //wait finish msg + match file_connection.accept_bi().await { + Ok((_file_sender, mut file_receiver)) => { + file_receiver.read_to_end(1024).await?; + } + Err(_) => { + info!("LFS connection closed."); + } + }; + info!( + "Send lfs data to[{}], oid: {} successfully", + to_peer_id, oid + ); + Ok(()) } - Ok(()) -} -async fn response_lfs(context: Context, oid: String, to_peer_id: String) -> Result<()> { - let bootstrap_node = BootstrapNode::get(); - info!("oid:{}", oid.clone()); - let result = match context.lfs_stg().get_lfs_object(oid.as_str()).await { - Ok(m) => m, - Err(e) => bail!(e), - }; - - let lfs_object = match result { - None => { - bail!("LFS not found: {}", oid); + async fn receive_quic_msg_task(self: Arc, tx: mpsc::Sender>) -> Result<()> { + loop { + let (_, mut quic_recv) = self.get_connection().accept_bi().await?; + let buffer = quic_recv.read_to_end(1024 * 1024).await?; + info!("QUIC Received:\n{}", String::from_utf8_lossy(&buffer)); + if tx.send(buffer).await.is_err() { + info!("Receiver closed"); + } } - Some(o) => o, - }; - // Register lfs connection to relay - let file_connection = get_client_connection(bootstrap_node).await?; - let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; - let peer_id = get_peerid().await; - file_sender - .write_all(format!("lfs-{}-{}|{}", peer_id.clone(), to_peer_id, "RESPONSE_LFS").as_bytes()) - .await?; - file_sender.finish()?; - - //send lfs header - let header = LFSHeader { - from: peer_id.clone(), - target: to_peer_id.clone(), - oid: oid.clone(), - size: lfs_object.size, - }; - let header = serde_json::to_string(&header)?; - let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; - file_sender.write_all(header.as_bytes()).await?; - file_sender.finish()?; - - //send data - let (mut file_sender, _file_receiver) = file_connection.open_bi().await?; - let mut result = lfs_download_object(context.clone(), oid.clone()).await?; - while let Some(d) = result.next().await { - match d { - Ok(bytes_chunk) => { - info!("bytes_chunk:{}", bytes_chunk.len()); - file_sender.write_all(&bytes_chunk).await?; + } + + async fn handle_quic_msg_task(self: Arc, message: Vec) -> Result<()> { + let data: ResponseData = serde_json::from_slice(&message)?; + match data.func.as_str() { + "request_git_clone" => { + let path = String::from_utf8(data.data)?; + self.response_git_clone(path, data.from).await?; } - Err(e) => { - bail!("LFS send error: {}", e); + "request_lfs" => { + let oid = String::from_utf8(data.data)?; + self.response_lfs(oid, data.from).await?; + } + "nostr" => { + receive_nostr(data.data).await?; + } + "" => { + if let Some(sender) = self.req_senders.get(data.req_id.as_str()) { + let mut guard = sender.lock().await; + if let Some(tx) = guard.take() { + tx.send(data.data).expect("Sender error"); + } + } + } + _ => { + error!("Unsupported function"); } } + Ok(()) } - file_sender.finish()?; - //wait finish msg - match file_connection.accept_bi().await { - Ok((_file_sender, mut file_receiver)) => { - file_receiver.read_to_end(1024).await?; + async fn run_ping_task(self: Arc, peer_id: Arc) -> Result<()> { + loop { + let (mut quic_send, _) = self.get_connection().open_bi().await?; + + let ping = RequestData { + from: String::from(peer_id.as_ref()), + data: vec![], + func: "".to_string(), + action: Action::Ping, + to: "relay".to_string(), + req_id: Uuid::new_v4().into(), + }; + let json = serde_json::to_string(&ping)?; + quic_send.write_all(json.as_ref()).await?; + quic_send.finish()?; + tokio::time::sleep(Duration::from_secs(60)).await; } - Err(_) => { - info!("LFS connection closed."); - } - }; - info!( - "Send lfs data to[{}], oid: {} successfully", - to_peer_id, oid - ); - Ok(()) -} - -pub async fn repo_subscribe(identifier: String) -> Result<()> { - let filters = vec![Filter::new().repo_uri(identifier)]; - let subscription_id = get_peerid().await; - let client_req = ClientMessage::new_req(SubscriptionId::new(subscription_id), filters); - - let relay_message = send_nostr_msg(client_req).await?; - info!("Subscribe repo result: {}", relay_message.as_json()); - Ok(()) -} + } -pub async fn send_git_event(git_event: GitEvent) -> Result<()> { - let keypair = vault::get_keypair().await; - let event = NostrEvent::new_git_event(keypair, git_event); - let client_message = ClientMessage::new_event(event); - let relay_message = send_nostr_msg(client_message).await?; - info!("Sent git event result: {}", relay_message.as_json()); - Ok(()) -} + async fn send_request(self: Arc, data: RequestData) -> Result<()> { + let connection = self.get_connection(); + let (mut sender, _) = connection.open_bi().await?; + let json = serde_json::to_string(&data)?; + sender.write_all(json.as_bytes()).await?; + sender.finish()?; + Ok(()) + } -async fn send_nostr_msg(client_message: ClientMessage) -> Result { - let (tx, rx) = tokio::sync::oneshot::channel(); - let req_id: String = Uuid::new_v4().into(); - let local_peer_id = get_peerid().await; - REQ_SENDER_MAP.insert(req_id.clone(), Arc::new(Mutex::new(Some(tx)))); - tokio::spawn(async move { - let send = RequestData { - from: local_peer_id.clone(), - data: client_message.as_json().as_bytes().to_vec(), - func: "".to_string(), - action: Action::Nostr, - to: "".to_string(), - req_id: req_id.clone(), - }; - if let Err(e) = send_request(send).await { - error!("failed to send nostr msg: {e}"); - } - }); - let data = rx.await?; - let data = RelayMessage::from_json(data)?; - Ok(data) + async fn send_nostr_msg(&self, client_message: ClientMessage) -> Result { + let peer_id = self.get_peer_id(); + let client = self.wrapped_client(); + let req_id: String = Uuid::new_v4().into(); + let (tx, rx) = tokio::sync::oneshot::channel(); + self.req_senders + .insert(req_id.clone(), Arc::new(Mutex::new(Some(tx)))); + tokio::spawn(async move { + let send = RequestData { + from: peer_id, + data: client_message.as_json().as_bytes().to_vec(), + func: "".to_string(), + action: Action::Nostr, + to: "".to_string(), + req_id: req_id.clone(), + }; + if let Err(e) = client.send_request(send).await { + error!("failed to send nostr msg: {e}"); + } + }); + let data = rx.await?; + let data = RelayMessage::from_json(data)?; + Ok(data) + } } async fn receive_nostr(data: Vec) -> Result<()> { @@ -805,50 +868,6 @@ async fn receive_nostr(data: Vec) -> Result<()> { Ok(()) } -pub async fn get_peers() -> Result> { - let (tx, rx) = tokio::sync::oneshot::channel(); - let req_id: String = Uuid::new_v4().into(); - let local_peer_id = get_peerid().await; - REQ_SENDER_MAP.insert(req_id.clone(), Arc::new(Mutex::new(Some(tx)))); - let send = RequestData { - from: local_peer_id.clone(), - data: vec![], - func: "".to_string(), - action: Action::Peers, - to: "".to_string(), - req_id: req_id.clone(), - }; - if let Err(e) = send_request(send).await { - error!("failed to get peers: {e}"); - } - - let res = wait_rx_with_timeout(rx).await?; - let peers: Vec = serde_json::from_slice(res.as_slice())?; - Ok(peers) -} - -pub async fn get_repos() -> Result> { - let (tx, rx) = tokio::sync::oneshot::channel(); - let req_id: String = Uuid::new_v4().into(); - let local_peer_id = get_peerid().await; - REQ_SENDER_MAP.insert(req_id.clone(), Arc::new(Mutex::new(Some(tx)))); - let send = RequestData { - from: local_peer_id.clone(), - data: vec![], - func: "".to_string(), - action: Action::Repos, - to: "".to_string(), - req_id: req_id.clone(), - }; - - if let Err(e) = send_request(send).await { - error!("failed to get repos: {e}"); - } - let res = wait_rx_with_timeout(rx).await?; - let repo_list: Vec = serde_json::from_slice(res.as_slice())?; - Ok(repo_list) -} - async fn wait_rx_with_timeout(rx: tokio::sync::oneshot::Receiver>) -> Result> { match time::timeout(Duration::from_secs(5), rx).await { Ok(r) => Ok(r.clone()?), @@ -857,31 +876,3 @@ async fn wait_rx_with_timeout(rx: tokio::sync::oneshot::Receiver>) -> Re } } } - -async fn get_encode_git_objects_by_repo(context: Context, repo: Repo) -> Result>> { - let import_repo = ImportRepo { - context: context.clone(), - repo, - command_list: vec![], - }; - match import_repo.full_pack(vec![]).await { - Ok(s) => Ok(s.into_inner()), - Err(e) => bail!("full pack repo failed: {}", e), - } -} - -async fn get_user_cert_from_ca( - ca: String, -) -> Result<(CertificateDer<'static>, PrivateKeyDer<'static>)> { - let cert = ca::client::get_user_cert_from_ca(ca).await?; - let cert = CertificateDer::from_pem_slice(cert.as_bytes())?; - let key = ca::client::get_user_key().await; - let key = PrivateKeyDer::from_pem_slice(key.as_bytes())?; - Ok((cert, key)) -} - -async fn get_ca_cert_from_ca(ca: String) -> Result> { - let cert = ca::client::get_ca_cert_from_ca(ca).await?; - let cert = CertificateDer::from_pem_slice(cert.as_bytes())?; - Ok(cert) -} diff --git a/gemini/src/p2p/mod.rs b/gemini/src/p2p/mod.rs index 5fdcd7ef0..b0fe14a89 100644 --- a/gemini/src/p2p/mod.rs +++ b/gemini/src/p2p/mod.rs @@ -2,6 +2,9 @@ use std::fmt; use callisto::import_refs; use serde::{Deserialize, Serialize}; +use vault::{integration::VaultCore, Vault}; + +use crate::p2p::client::P2PClient; pub mod client; pub mod relay; @@ -87,6 +90,15 @@ pub struct LFSHeader { pub size: i64, } +impl Vault for P2PClient { + type Core = VaultCore; + const VAULT_PREFIX: &'static str = "p2p_client"; + + fn core(&self) -> &Self::Core { + &self.vault + } +} + #[cfg(test)] mod tests { // use crate::nostr::GitEventReq; diff --git a/gemini/src/p2p/relay.rs b/gemini/src/p2p/relay.rs index 1a4023db9..dbb9802aa 100644 --- a/gemini/src/p2p/relay.rs +++ b/gemini/src/p2p/relay.rs @@ -13,9 +13,7 @@ use anyhow::anyhow; use anyhow::Result; use callisto::{relay_node, relay_nostr_event, relay_nostr_req, relay_repo_info}; use dashmap::DashMap; -use jupiter::context::Context; -use jupiter::storage::relay_storage::RelayStorage; -use lazy_static::lazy_static; +use jupiter::storage::Storage; use quinn::rustls::pki_types::{CertificateDer, PrivateKeyDer}; use quinn::rustls::server::WebPkiClientVerifier; use quinn::{ @@ -30,653 +28,699 @@ use std::{net::SocketAddr, str::FromStr, sync::Arc}; use tokio::sync::{mpsc, OnceCell}; use tracing::{error, info}; use uuid::Uuid; - -lazy_static! { - static ref MSG_CONNECTION_MAP: DashMap> = DashMap::new(); - static ref GIT_OBJECTS_CONNECTION_MAP: DashMap> = DashMap::new(); - static ref LFS_CONNECTION_MAP: DashMap> = DashMap::new(); - static ref REQ_ID_MAP: DashMap> = DashMap::new(); - static ref NOSTR_EVENT_QUEUE: OnceCell> = OnceCell::new(); +use vault::integration::VaultCore; + +#[derive(Clone)] +pub struct P2PRelay { + pub storage: Storage, + pub vault: VaultCore, + + msg_connection_map: DashMap>, + git_objects_connection_map: DashMap>, + lfs_connection_map: DashMap>, + req_id_map: DashMap>, + nostr_event_queue: OnceCell>, } -pub async fn run(content: Context, host: String, port: u16) -> Result<()> { - let server_config = get_server_config().await?; - let addr = format!("{}:{}", host, port); - let endpoint = - quinn::Endpoint::server(server_config, SocketAddr::from_str(addr.as_str()).unwrap())?; - info!("Quic server listening on udp {}", endpoint.local_addr()?); - - //Nostr event sender channel - let (tx, mut rx) = mpsc::channel(32); - NOSTR_EVENT_QUEUE.set(tx)?; - tokio::spawn(async move { - while let Some((peer_id, nostr_event)) = rx.recv().await { - send_nostr_event(peer_id, nostr_event).await.unwrap(); +impl P2PRelay { + pub fn new(storage: Storage, vault: VaultCore) -> Self { + Self { + storage, + vault, + msg_connection_map: DashMap::new(), + git_objects_connection_map: DashMap::new(), + lfs_connection_map: DashMap::new(), + req_id_map: DashMap::new(), + nostr_event_queue: OnceCell::default(), } - }); + } - let storage_clone = content.services.relay_storage.clone(); - tokio::spawn(async move { - loop { - tokio::time::sleep(Duration::from_secs(60)).await; - check_node_status(Arc::from(storage_clone.clone())).await; - } - }); - - while let Some(conn) = endpoint.accept().await { - let storage_clone = content.services.relay_storage.clone(); - { - info!("accepting connection"); - - let fut = handle_connection(conn, Arc::new(storage_clone)); - tokio::spawn(async move { - if let Err(e) = fut.await { - error!("connection failed: {reason}", reason = e.to_string()); - // remove_close_connection(); - } - }); - } + pub fn wrapped_relay(&self) -> Arc { + Arc::new(self.clone()) } - Ok(()) -} + pub async fn run(&self, host: String, port: u16) -> Result<()> { + let server_config = self.get_server_config().await?; + let addr = format!("{}:{}", host, port); + let endpoint = + quinn::Endpoint::server(server_config, SocketAddr::from_str(addr.as_str()).unwrap())?; + info!("Quic server listening on udp {}", endpoint.local_addr()?); + + //Nostr event sender channel + let wrapped = self.wrapped_relay(); + let (tx, mut rx) = mpsc::channel(32); + self.nostr_event_queue.set(tx)?; + + tokio::spawn(async move { + while let Some((peer_id, nostr_event)) = rx.recv().await { + wrapped.clone() + .send_nostr_event(peer_id, nostr_event) + .await + .unwrap(); + } + }); -pub async fn get_server_config() -> Result { - rustls::crypto::ring::default_provider() - .install_default() - .expect("Failed to install rustls crypto provider"); + let wrapped = self.wrapped_relay(); + tokio::spawn(async move { + loop { + tokio::time::sleep(Duration::from_secs(60)).await; + wrapped.clone().check_node_status().await; + } + }); - let (certs, key) = get_root_certificate_from_vault().await?; + while let Some(conn) = endpoint.accept().await { + { + info!("accepting connection"); - let mut roots = rustls::RootCertStore::empty(); - for c in certs.clone() { - roots.add(c)?; + let wrapped = self.wrapped_relay(); + tokio::spawn(async move { + if let Err(e) = wrapped.handle_connection(conn).await { + error!("connection failed: {reason}", reason = e.to_string()); + // remove_close_connection(); + } + }); + } + } + + Ok(()) } - let client_verifier = WebPkiClientVerifier::builder(roots.into()) - .build() - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + pub async fn get_server_config(&self) -> Result { + rustls::crypto::ring::default_provider() + .install_default() + .expect("Failed to install rustls crypto provider"); - let mut server_crypto = rustls::ServerConfig::builder() - .with_client_cert_verifier(client_verifier) - .with_single_cert(certs, key)?; - server_crypto.alpn_protocols = ALPN_QUIC_HTTP.iter().map(|&x| x.into()).collect(); - server_crypto.max_early_data_size = u32::MAX; + let (certs, key) = self.get_root_certificate_from_vault()?; - let mut server_config = - quinn::ServerConfig::with_crypto(Arc::new(QuicServerConfig::try_from(server_crypto)?)); + let mut roots = rustls::RootCertStore::empty(); + for c in &certs { + roots.add(c.clone())?; + } - let mut transport_config = TransportConfig::default(); - transport_config.max_idle_timeout(Some(IdleTimeout::from(VarInt::from_u32(300_000)))); - transport_config.keep_alive_interval(Some(Duration::from_secs(15))); - server_config.transport_config(transport_config.into()); + let client_verifier = WebPkiClientVerifier::builder(roots.into()) + .build() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - Ok(server_config) -} + let mut server_crypto = rustls::ServerConfig::builder() + .with_client_cert_verifier(client_verifier) + .with_single_cert(certs, key)?; + server_crypto.alpn_protocols = ALPN_QUIC_HTTP.iter().map(|&x| x.into()).collect(); + server_crypto.max_early_data_size = u32::MAX; -async fn handle_connection(conn: quinn::Incoming, relay_storage: Arc) -> Result<()> { - let connection = conn.await?; - - let remote_address = connection.remote_address(); - let local_ip = connection.local_ip().unwrap(); - let stable_id = connection.stable_id(); - info!("Established connection: {remote_address:#?},{local_ip:#?},{stable_id:#?}"); - let connection = Arc::new(connection); - - let (mut _send, mut recv) = connection.accept_bi().await.unwrap(); - let mut buf = [0u8; 1024]; - let len = recv.read(&mut buf).await.unwrap().unwrap(); - let registration = String::from_utf8_lossy(&buf[..len]); - - //register: key |ConnectionType (MSG/REQUEST_GIT_CLONE/REQUEST_LFS) - let parts: Vec<&str> = registration.split('|').collect(); - let (key, connection_type) = (parts[0], parts[1]); - info!("Key:{}, Connection_type:{}", key, connection_type); - match connection_type { - "MSG" => { - MSG_CONNECTION_MAP.insert(key.to_string(), connection.clone()); - msg_handle_receive(connection.clone(), relay_storage).await?; - } - "REQUEST_GIT_CLONE" => { - GIT_OBJECTS_CONNECTION_MAP.insert(key.to_string(), connection.clone()); - } - "RESPONSE_GIT_CLONE" => { - git_clone_handle_receive(connection.clone()).await?; - } - "REQUEST_LFS" => { - LFS_CONNECTION_MAP.insert(key.to_string(), connection.clone()); - } - "RESPONSE_LFS" => { - lfs_handle_receive(connection.clone()).await?; - } - _ => {} - } + let mut server_config = + quinn::ServerConfig::with_crypto(Arc::new(QuicServerConfig::try_from(server_crypto)?)); - Ok(()) -} + let mut transport_config = TransportConfig::default(); + transport_config.max_idle_timeout(Some(IdleTimeout::from(VarInt::from_u32(300_000)))); + transport_config.keep_alive_interval(Some(Duration::from_secs(15))); + server_config.transport_config(transport_config.into()); -fn _remove_close_connection() { - MSG_CONNECTION_MAP.retain(|_, v| v.close_reason().is_some()); - GIT_OBJECTS_CONNECTION_MAP.retain(|_, v| v.close_reason().is_some()); - LFS_CONNECTION_MAP.retain(|_, v| v.close_reason().is_some()); - REQ_ID_MAP.retain(|_, v| v.close_reason().is_some()); -} + Ok(server_config) + } -async fn check_node_status(relay_storage: Arc) { - let nodes = relay_storage.get_all_node().await.unwrap(); - for n in nodes { - let now = get_utc_timestamp(); - if now - n.last_online_time > 60_000 { - let mut node = n.clone(); - node.online = false; - if let Err(e) = relay_storage.update_node(node).await { - error!("Failed to update node: {:?}", e); + async fn handle_connection(&self, conn: quinn::Incoming) -> Result<()> { + let connection = conn.await?; + + let remote_address = connection.remote_address(); + let local_ip = connection.local_ip().unwrap(); + let stable_id = connection.stable_id(); + info!("Established connection: {remote_address:#?},{local_ip:#?},{stable_id:#?}"); + let connection = Arc::new(connection); + + let (mut _send, mut recv) = connection.accept_bi().await.unwrap(); + let mut buf = [0u8; 1024]; + let len = recv.read(&mut buf).await.unwrap().unwrap(); + let registration = String::from_utf8_lossy(&buf[..len]); + + //register: key |ConnectionType (MSG/REQUEST_GIT_CLONE/REQUEST_LFS) + let parts: Vec<&str> = registration.split('|').collect(); + let (key, connection_type) = (parts[0], parts[1]); + info!("Key:{}, Connection_type:{}", key, connection_type); + match connection_type { + "MSG" => { + self.msg_connection_map + .insert(key.to_string(), connection.clone()); + self.msg_handle_receive(connection.clone()).await?; + } + "REQUEST_GIT_CLONE" => { + self.git_objects_connection_map + .insert(key.to_string(), connection.clone()); + } + "RESPONSE_GIT_CLONE" => { + self.git_clone_handle_receive(connection.clone()).await?; + } + "REQUEST_LFS" => { + self.lfs_connection_map.insert(key.to_string(), connection.clone()); + } + "RESPONSE_LFS" => { + self.lfs_handle_receive(connection.clone()).await?; } + _ => {} } + + Ok(()) } -} -async fn msg_handle_receive( - connection: Arc, - relay_storage: Arc, -) -> Result<()> { - loop { - let connection_clone = connection.clone(); - let stream = connection_clone.accept_bi().await; - let (_sender, mut recv) = match stream { - Err(quinn::ConnectionError::ApplicationClosed { .. }) => { - info!("connection closed"); - return Ok(()); - } - Err(e) => { - info!("connection error:{}", e); - return Err(e.into()); + fn _remove_close_connection(&self) { + self.msg_connection_map.retain(|_, v| v.close_reason().is_some()); + self.git_objects_connection_map.retain(|_, v| v.close_reason().is_some()); + self.lfs_connection_map.retain(|_, v| v.close_reason().is_some()); + self.req_id_map.retain(|_, v| v.close_reason().is_some()); + } + + async fn check_node_status(self: Arc) { + let relay_storage = self.storage.relay_storage().clone(); + let nodes = relay_storage.get_all_node().await.unwrap(); + for n in nodes { + let now = get_utc_timestamp(); + if now - n.last_online_time > 60_000 { + let mut node = n.clone(); + node.online = false; + if let Err(e) = relay_storage.update_node(node).await { + error!("Failed to update node: {:?}", e); + } } - Ok(s) => s, - }; - let buffer_vec = recv.read_to_end(1024 * 10).await?; - if buffer_vec.is_empty() { - error!("QUIC Received is empty"); - return Ok(()); } - let result = String::from_utf8_lossy(&buffer_vec); + } - let data: RequestData = match serde_json::from_str(&result) { - Ok(data) => data, - Err(e) => { - error!("QUIC Received Error:{:?}", e); - return Err(anyhow!("QUIC Received Error:{:?}", e)); - } - }; - info!( - "QUIC Received Message from[{}], Action[{}]", - data.from, data.action - ); - match data.action { - Action::Ping => { - let storage = relay_storage.clone(); - - let node = relay_node::Model { - peer_id: data.from.clone(), - r#type: "mega_server".to_string(), - online: true, - last_online_time: get_utc_timestamp(), - }; - match storage.insert_or_update_node(node).await { - Ok(_) => send_back(data, "ok".as_bytes().to_vec(), connection_clone).await?, - Err(_) => { - send_back_err(data, "Ping with error".to_string(), connection_clone).await? - } + async fn msg_handle_receive(&self, connection: Arc) -> Result<()> { + loop { + let relay_storage = self.storage.relay_storage().clone(); + let connection_clone = connection.clone(); + let stream = connection_clone.accept_bi().await; + let (_sender, mut recv) = match stream { + Err(quinn::ConnectionError::ApplicationClosed { .. }) => { + info!("connection closed"); + return Ok(()); + } + Err(e) => { + info!("connection error:{}", e); + return Err(e.into()); } + Ok(s) => s, + }; + let buffer_vec = recv.read_to_end(1024 * 10).await?; + if buffer_vec.is_empty() { + error!("QUIC Received is empty"); + return Ok(()); } - Action::Send => { - let connection = match MSG_CONNECTION_MAP.get(data.to.as_str()) { - None => { - error!("Failed to find connection to {}", data.to); - return Err(anyhow!("Failed to find connection to {}", data.to)); - } - Some(conn) => conn, - }; + let result = String::from_utf8_lossy(&buffer_vec); - let reponse = ResponseData { - from: data.from.to_string(), - data: data.data, - func: data.func.clone(), - err: "".to_string(), - to: data.to.to_string(), - req_id: data.req_id, - }; - let json = serde_json::to_string(&reponse)?; - let (mut send, _) = connection.open_bi().await?; - send.write_all(json.as_bytes()).await?; - send.finish()?; - } - Action::Call => { - { - let connection_to = match MSG_CONNECTION_MAP.get(data.to.as_str()) { - None => { - error!("Failed to find connection to {}", data.to); - return Err(anyhow!("Failed to find connection to {}", data.to)); - } - Some(conn) => conn, - }; - let response = ResponseData { - from: data.from.to_string(), - data: data.data, - func: data.func.clone(), - err: "".to_string(), - to: data.to.to_string(), - req_id: data.req_id.clone(), + let data: RequestData = match serde_json::from_str(&result) { + Ok(data) => data, + Err(e) => { + error!("QUIC Received Error:{:?}", e); + return Err(anyhow!("QUIC Received Error:{:?}", e)); + } + }; + info!( + "QUIC Received Message from[{}], Action[{}]", + data.from, data.action + ); + match data.action { + Action::Ping => { + let storage = relay_storage.clone(); + + let node = relay_node::Model { + peer_id: data.from.clone(), + r#type: "mega_server".to_string(), + online: true, + last_online_time: get_utc_timestamp(), }; - let json = serde_json::to_string(&response)?; - let (mut send, _) = connection_to.open_bi().await?; - send.write_all(json.as_bytes()).await?; - send.finish()?; + match storage.insert_or_update_node(node).await { + Ok(_) => { + self.send_back(data, "ok".as_bytes().to_vec(), connection_clone).await? + } + Err(_) => { + self.send_back_err(data, "Ping with error".to_string(), connection_clone) + .await? + } + } } - let from_connection = connection_clone; - REQ_ID_MAP.insert(data.req_id.to_string(), from_connection.clone()); - } - Action::Callback => { - { - let connection = match REQ_ID_MAP.get(data.req_id.as_str()) { + Action::Send => { + let connection = match self.msg_connection_map.get(data.to.as_str()) { None => { - error!("Failed to find connection req {}", data.req_id); - return Err(anyhow!("Failed to find connection req {}", data.req_id)); + error!("Failed to find connection to {}", data.to); + return Err(anyhow!("Failed to find connection to {}", data.to)); } Some(conn) => conn, }; - let response = ResponseData { + + let reponse = ResponseData { from: data.from.to_string(), data: data.data, func: data.func.clone(), err: "".to_string(), to: data.to.to_string(), - req_id: data.req_id.clone(), + req_id: data.req_id, }; - let json = serde_json::to_string(&response)?; + let json = serde_json::to_string(&reponse)?; let (mut send, _) = connection.open_bi().await?; send.write_all(json.as_bytes()).await?; send.finish()?; } - REQ_ID_MAP.remove(data.req_id.as_str()); - } - - Action::RepoShare => { - let repo_info: RepoInfo = serde_json::from_slice(data.data.as_slice())?; - let repo_info_model: relay_repo_info::Model = repo_info.clone().into(); - let storage = relay_storage.clone(); - match storage.insert_or_update_repo_info(repo_info_model).await { - Ok(_) => { - send_back(data, repo_info.identifier.into_bytes(), connection.clone()) - .await? + Action::Call => { + { + let connection_to = match self.msg_connection_map.get(data.to.as_str()) { + None => { + error!("Failed to find connection to {}", data.to); + return Err(anyhow!("Failed to find connection to {}", data.to)); + } + Some(conn) => conn, + }; + let response = ResponseData { + from: data.from.to_string(), + data: data.data, + func: data.func.clone(), + err: "".to_string(), + to: data.to.to_string(), + req_id: data.req_id.clone(), + }; + let json = serde_json::to_string(&response)?; + let (mut send, _) = connection_to.open_bi().await?; + send.write_all(json.as_bytes()).await?; + send.finish()?; } - Err(_) => { - send_back_err(data, "Repo share failed".to_string(), connection.clone()) - .await? + let from_connection = connection_clone; + self.req_id_map.insert(data.req_id.to_string(), from_connection.clone()); + } + Action::Callback => { + { + let connection = match self.req_id_map.get(data.req_id.as_str()) { + None => { + error!("Failed to find connection req {}", data.req_id); + return Err(anyhow!( + "Failed to find connection req {}", + data.req_id + )); + } + Some(conn) => conn, + }; + let response = ResponseData { + from: data.from.to_string(), + data: data.data, + func: data.func.clone(), + err: "".to_string(), + to: data.to.to_string(), + req_id: data.req_id.clone(), + }; + let json = serde_json::to_string(&response)?; + let (mut send, _) = connection.open_bi().await?; + send.write_all(json.as_bytes()).await?; + send.finish()?; } + self.req_id_map.remove(data.req_id.as_str()); } - } - Action::Nostr => { - info!("Nostr data:{}", String::from_utf8(data.data.clone())?); - let client_msg: ClientMessage = - match serde_json::from_slice(data.data.clone().as_slice()) { - Ok(client_msg) => client_msg, - Err(e) => { - let relay_msg = - RelayMessage::new_ok(EventId::empty(), false, e.to_string()); - send_back( - data, - relay_msg.as_json().as_bytes().to_vec(), - connection.clone(), - ) - .await?; - continue; + Action::RepoShare => { + let repo_info: RepoInfo = serde_json::from_slice(data.data.as_slice())?; + let repo_info_model: relay_repo_info::Model = repo_info.clone().into(); + let storage = relay_storage.clone(); + match storage.insert_or_update_repo_info(repo_info_model).await { + Ok(_) => { + self.send_back(data, repo_info.identifier.into_bytes(), connection.clone()) + .await? + } + Err(_) => { + self.send_back_err(data, "Repo share failed".to_string(), connection.clone()) + .await? } - }; - let relay_msg = - nostr_handle(relay_storage.clone(), client_msg, data.from.clone()).await; - send_back( - data, - relay_msg.as_json().as_bytes().to_vec(), - connection.clone(), - ) - .await?; - } - Action::Peers => { - match relay_storage.get_all_node().await { - Ok(peers) => { - let peers: Vec = peers.iter().map(|p| p.clone().into()).collect(); - let res = serde_json::to_string(&peers)?; - send_back(data, res.into_bytes(), connection.clone()).await? - } - Err(_) => { - send_back_err(data, "Get peers failed".to_string(), connection.clone()) - .await? } - }; - } + } + + Action::Nostr => { + info!("Nostr data:{}", String::from_utf8(data.data.clone())?); + let client_msg: ClientMessage = + match serde_json::from_slice(data.data.clone().as_slice()) { + Ok(client_msg) => client_msg, + Err(e) => { + let relay_msg = + RelayMessage::new_ok(EventId::empty(), false, e.to_string()); + self.send_back( + data, + relay_msg.as_json().as_bytes().to_vec(), + connection.clone(), + ) + .await?; + continue; + } + }; + let relay_msg = + self.nostr_handle( client_msg, data.from.clone()).await; + self.send_back( + data, + relay_msg.as_json().as_bytes().to_vec(), + connection.clone(), + ) + .await?; + } + Action::Peers => { + match relay_storage.get_all_node().await { + Ok(peers) => { + let peers: Vec = peers.iter().map(|p| p.clone().into()).collect(); + let res = serde_json::to_string(&peers)?; + self.send_back(data, res.into_bytes(), connection.clone()).await? + } + Err(_) => { + self.send_back_err(data, "Get peers failed".to_string(), connection.clone()) + .await? + } + }; + } - Action::Repos => { - match relay_storage.get_all_repo_info().await { - Ok(repo_list) => { - let mut repo_list: Vec = - repo_list.iter().map(|p| p.clone().into()).collect(); - for r in repo_list.iter_mut() { - if let Ok(peer_id) = get_peer_id_from_identifier(r.identifier.clone()) { - let node = relay_storage - .get_node_by_id(peer_id.as_str()) - .await - .unwrap(); - if let Some(node) = node { - r.peer_online = node.online; + Action::Repos => { + match relay_storage.get_all_repo_info().await { + Ok(repo_list) => { + let mut repo_list: Vec = + repo_list.iter().map(|p| p.clone().into()).collect(); + for r in repo_list.iter_mut() { + if let Ok(peer_id) = + get_peer_id_from_identifier(r.identifier.clone()) + { + let node = relay_storage + .get_node_by_id(peer_id.as_str()) + .await + .unwrap(); + if let Some(node) = node { + r.peer_online = node.online; + } } } + let res = serde_json::to_string(&repo_list.clone())?; + self.send_back(data, res.into_bytes(), connection.clone()).await? } - let res = serde_json::to_string(&repo_list.clone())?; - send_back(data, res.into_bytes(), connection.clone()).await? - } - Err(_) => { - send_back_err(data, "Get repos failed".to_string(), connection.clone()) - .await? - } - }; + Err(_) => { + self.send_back_err(data, "Get repos failed".to_string(), connection.clone()) + .await? + } + }; + } } - } - { - let peers: Vec = MSG_CONNECTION_MAP - .iter() - .map(|entry| entry.key().clone()) - .collect(); - info!("Online peers num: {}", peers.len()); - for x in peers { - info!("Online peer: {}", x.to_string()); + { + let peers: Vec = self.msg_connection_map + .iter() + .map(|entry| entry.key().clone()) + .collect(); + info!("Online peers num: {}", peers.len()); + for x in peers { + info!("Online peer: {}", x.to_string()); + } } } } -} -async fn nostr_handle( - relay_storage: Arc, - client_message: ClientMessage, - from: String, -) -> RelayMessage { - match client_message { - ClientMessage::Event(nostr_event) => { - match nostr_event.verify() { - Ok(_) => {} - Err(e) => { - return RelayMessage::new_ok(EventId::empty(), false, e.to_string()); + async fn nostr_handle( + &self, + client_message: ClientMessage, + from: String, + ) -> RelayMessage { + let relay_storage = self.storage.relay_storage().clone(); + match client_message { + ClientMessage::Event(nostr_event) => { + match nostr_event.verify() { + Ok(_) => {} + Err(e) => { + return RelayMessage::new_ok(EventId::empty(), false, e.to_string()); + } } - } - let relay_nostr_event: relay_nostr_event::Model = match nostr_event.clone().try_into() { - Ok(n) => n, - Err(e) => { - return RelayMessage::new_ok(EventId::empty(), false, e.to_string()); + let relay_nostr_event: relay_nostr_event::Model = + match nostr_event.clone().try_into() { + Ok(n) => n, + Err(e) => { + return RelayMessage::new_ok(EventId::empty(), false, e.to_string()); + } + }; + //save + if relay_storage + .get_nostr_event_by_id(&relay_nostr_event.id) + .await + .unwrap() + .is_some() + { + return RelayMessage::new_ok( + EventId::empty(), + false, + "Duplicate submission".to_string(), + ); } - }; - //save - if relay_storage - .get_nostr_event_by_id(&relay_nostr_event.id) - .await - .unwrap() - .is_some() - { - return RelayMessage::new_ok( - EventId::empty(), - false, - "Duplicate submission".to_string(), - ); + relay_storage + .insert_nostr_event(relay_nostr_event) + .await + .unwrap(); + + //Event is forwarded to subscribed nodes + let _ = self.transfer_git_event_to_subscribers(nostr_event.clone(), from) + .await; + RelayMessage::new_ok(nostr_event.id, true, "ok".to_string()) } - relay_storage - .insert_nostr_event(relay_nostr_event) - .await - .unwrap(); - - //Event is forwarded to subscribed nodes - let _ = - transfer_git_event_to_subscribers(relay_storage, nostr_event.clone(), from).await; - RelayMessage::new_ok(nostr_event.id, true, "ok".to_string()) - } - ClientMessage::Req { - subscription_id, - filters, - } => { - //subscribe message - //save - let filters_json = serde_json::to_string(&filters).unwrap(); - let ztm_nostr_req = relay_nostr_req::Model { - subscription_id: subscription_id.to_string(), - filters: filters_json.clone(), - id: Uuid::new_v4().to_string(), - }; - let req_list: Vec = relay_storage - .get_all_nostr_req_by_subscription_id(&subscription_id.to_string()) - .await - .unwrap(); - match req_list.iter().find(|&x| x.filters == filters_json) { - Some(_) => {} - None => { - relay_storage.insert_nostr_req(ztm_nostr_req).await.unwrap(); + ClientMessage::Req { + subscription_id, + filters, + } => { + //subscribe message + //save + let filters_json = serde_json::to_string(&filters).unwrap(); + let ztm_nostr_req = relay_nostr_req::Model { + subscription_id: subscription_id.to_string(), + filters: filters_json.clone(), + id: Uuid::new_v4().to_string(), + }; + let req_list: Vec = relay_storage + .get_all_nostr_req_by_subscription_id(&subscription_id.to_string()) + .await + .unwrap(); + match req_list.iter().find(|&x| x.filters == filters_json) { + Some(_) => {} + None => { + relay_storage.insert_nostr_req(ztm_nostr_req).await.unwrap(); + } } + RelayMessage::new_ok(EventId::empty(), true, "ok".to_string()) } - RelayMessage::new_ok(EventId::empty(), true, "ok".to_string()) } } -} -async fn send_back( - request_data: RequestData, - data: Vec, - connection: Arc, -) -> Result<()> { - let response = ResponseData { - from: request_data.to.to_string(), - data, - func: request_data.func.clone(), - err: "".to_string(), - to: request_data.from.to_string(), - req_id: request_data.req_id.clone(), - }; - - let json = serde_json::to_string(&response)?; - let (mut send, _) = connection.open_bi().await?; - send.write_all(json.as_bytes()).await?; - send.finish()?; - Ok(()) -} - -async fn send_back_err( - request_data: RequestData, - err: String, - connection: Arc, -) -> Result<()> { - let response = ResponseData { - from: request_data.to.to_string(), - data: vec![], - func: request_data.func.clone(), - err, - to: request_data.from.to_string(), - req_id: request_data.req_id.clone(), - }; - - let json = serde_json::to_string(&response)?; - let (mut send, _) = connection.open_bi().await?; - send.write_all(json.as_bytes()).await?; - send.finish()?; - Ok(()) -} + async fn send_back( + &self, + request_data: RequestData, + data: Vec, + connection: Arc, + ) -> Result<()> { + let response = ResponseData { + from: request_data.to.to_string(), + data, + func: request_data.func.clone(), + err: "".to_string(), + to: request_data.from.to_string(), + req_id: request_data.req_id.clone(), + }; -async fn transfer_git_event_to_subscribers( - relay_storage: Arc, - nostr_event: NostrEvent, - from: String, -) -> Result<()> { - // only support p2p_uri subscription - let mut uri = String::new(); - for tag in nostr_event.clone().tags { - if let Tag::Generic(TagKind::URI, t) = tag { - if !t.is_empty() { - uri = t.first().unwrap().to_string(); - } - } + let json = serde_json::to_string(&response)?; + let (mut send, _) = connection.open_bi().await?; + send.write_all(json.as_bytes()).await?; + send.finish()?; + Ok(()) } - if uri.is_empty() { - return Ok(()); + + async fn send_back_err( + &self, + request_data: RequestData, + err: String, + connection: Arc, + ) -> Result<()> { + let response = ResponseData { + from: request_data.to.to_string(), + data: vec![], + func: request_data.func.clone(), + err, + to: request_data.from.to_string(), + req_id: request_data.req_id.clone(), + }; + + let json = serde_json::to_string(&response)?; + let (mut send, _) = connection.open_bi().await?; + send.write_all(json.as_bytes()).await?; + send.finish()?; + Ok(()) } - let req_list: Vec = relay_storage - .get_all_nostr_req() - .await - .unwrap() - .iter() - .map(|x| x.clone().into()) - .collect(); - let mut subscription_id_set: HashSet = HashSet::new(); - for req in req_list { - for filter in req.clone().filters { - if let Some(uri_vec) = filter.generic_tags.get(&TagKind::URI.to_string()) { - if uri_vec.is_empty() { - continue; + + async fn transfer_git_event_to_subscribers( + &self, + nostr_event: NostrEvent, + from: String, + ) -> Result<()> { + // only support p2p_uri subscription + let mut uri = String::new(); + let relay_storage = self.storage.relay_storage().clone(); + for tag in nostr_event.clone().tags { + if let Tag::Generic(TagKind::URI, t) = tag { + if !t.is_empty() { + uri = t.first().unwrap().to_string(); } - let req_uri = uri_vec.first().unwrap(); - if *req_uri == uri { - subscription_id_set.insert(req.subscription_id.clone()); + } + } + if uri.is_empty() { + return Ok(()); + } + let req_list: Vec = relay_storage + .get_all_nostr_req() + .await + .unwrap() + .iter() + .map(|x| x.clone().into()) + .collect(); + let mut subscription_id_set: HashSet = HashSet::new(); + for req in req_list { + for filter in req.clone().filters { + if let Some(uri_vec) = filter.generic_tags.get(&TagKind::URI.to_string()) { + if uri_vec.is_empty() { + continue; + } + let req_uri = uri_vec.first().unwrap(); + if *req_uri == uri { + subscription_id_set.insert(req.subscription_id.clone()); + } } } } - } - info!("subscription_id_set:{:?}", subscription_id_set); - for x in subscription_id_set { - if x == from { - continue; + info!("subscription_id_set:{:?}", subscription_id_set); + for x in subscription_id_set { + if x == from { + continue; + } + //send to queue + let tx = self.nostr_event_queue.get().unwrap().clone(); + tx.send((x, nostr_event.clone())).await?; } - //send to queue - let tx = NOSTR_EVENT_QUEUE.get().unwrap().clone(); - tx.send((x, nostr_event.clone())).await?; + Ok(()) } - Ok(()) -} -async fn send_nostr_event(peer_id: String, nostr_event: NostrEvent) -> Result<()> { - if let Some(conn) = MSG_CONNECTION_MAP.get(peer_id.clone().as_str()) { - if conn.close_reason().is_some() { - return Ok(()); + async fn send_nostr_event( + self: Arc, + peer_id: String, + nostr_event: NostrEvent, + ) -> Result<()> { + if let Some(conn) = self.msg_connection_map.get(peer_id.clone().as_str()) { + if conn.close_reason().is_some() { + return Ok(()); + } + let data = + RelayMessage::new_event(SubscriptionId::new(peer_id.clone()), nostr_event.clone()) + .as_json(); + let response = ResponseData { + from: "relay".to_string(), + data: data.as_bytes().to_vec(), + func: "nostr".to_string(), + err: "".to_string(), + to: peer_id.clone(), + req_id: "".to_string(), + }; + + let json = serde_json::to_string(&response)?; + let (mut send, _) = conn.open_bi().await?; + send.write_all(json.as_bytes()).await?; + send.finish()?; + info!( + "Send nostr evnet[{}] to {} success", + nostr_event.id.inner(), + peer_id + ); } - let data = - RelayMessage::new_event(SubscriptionId::new(peer_id.clone()), nostr_event.clone()) - .as_json(); - let response = ResponseData { - from: "relay".to_string(), - data: data.as_bytes().to_vec(), - func: "nostr".to_string(), - err: "".to_string(), - to: peer_id.clone(), - req_id: "".to_string(), - }; + Ok(()) + } - let json = serde_json::to_string(&response)?; - let (mut send, _) = conn.open_bi().await?; - send.write_all(json.as_bytes()).await?; - send.finish()?; + async fn git_clone_handle_receive(&self, connection: Arc) -> Result<()> { + let connection_clone: Arc = connection.clone(); + let (_file_sender, mut file_receiver) = connection_clone.accept_bi().await?; + + //read header + let mut header_buf = [0u8; 4096]; + let len = file_receiver.read(&mut header_buf).await?.unwrap(); + let header = String::from_utf8_lossy(&header_buf[..len]); + let header: GitCloneHeader = serde_json::from_str(&header)?; + let (target_id, from, git_path) = (header.target, header.from, header.git_path); info!( - "Send nostr evnet[{}] to {} success", - nostr_event.id.inner(), - peer_id + "File handle receive, target_id:{}, from:{}, file_path:{}", + target_id, from, git_path ); + let key = format!("git-clone-{}-{}", target_id, from); + + if let Some(target_conn) = self.git_objects_connection_map.get(&key) { + info!("Find target connection to {}", target_id); + //header data + info!("Send git clone header to {}", target_id); + let (mut target_sender, _) = target_conn.open_bi().await?; + target_sender.write_all(&header_buf[..len]).await?; + target_sender.finish()?; + + //git objects data + info!("Send git clone objects to {}", target_id); + let (mut target_sender, _) = target_conn.open_bi().await?; + let (_file_sender, mut file_receiver) = connection_clone.accept_bi().await?; + tokio::io::copy(&mut file_receiver, &mut target_sender).await?; + target_sender.finish()?; + + //send finish to from peer + let (mut sender, _) = connection_clone.open_bi().await?; + sender.write_all("finish".as_bytes()).await?; + sender.finish()?; + info!("Finish git clone to provider:{}", from); + } else { + connection_clone.close(VarInt::from_u32(1), "Cannot find target peer".as_bytes()); + } + Ok(()) } - Ok(()) -} -async fn git_clone_handle_receive(connection: Arc) -> Result<()> { - let connection_clone: Arc = connection.clone(); - let (_file_sender, mut file_receiver) = connection_clone.accept_bi().await?; - - //read header - let mut header_buf = [0u8; 4096]; - let len = file_receiver.read(&mut header_buf).await?.unwrap(); - let header = String::from_utf8_lossy(&header_buf[..len]); - let header: GitCloneHeader = serde_json::from_str(&header)?; - let (target_id, from, git_path) = (header.target, header.from, header.git_path); - info!( - "File handle receive, target_id:{}, from:{}, file_path:{}", - target_id, from, git_path - ); - let key = format!("git-clone-{}-{}", target_id, from); - - if let Some(target_conn) = GIT_OBJECTS_CONNECTION_MAP.get(&key) { - info!("Find target connection to {}", target_id); - //header data - info!("Send git clone header to {}", target_id); - let (mut target_sender, _) = target_conn.open_bi().await?; - target_sender.write_all(&header_buf[..len]).await?; - target_sender.finish()?; - - //git objects data - info!("Send git clone objects to {}", target_id); - let (mut target_sender, _) = target_conn.open_bi().await?; + async fn lfs_handle_receive(&self, connection: Arc) -> Result<()> { + let connection_clone: Arc = connection.clone(); let (_file_sender, mut file_receiver) = connection_clone.accept_bi().await?; - tokio::io::copy(&mut file_receiver, &mut target_sender).await?; - target_sender.finish()?; - - //send finish to from peer - let (mut sender, _) = connection_clone.open_bi().await?; - sender.write_all("finish".as_bytes()).await?; - sender.finish()?; - info!("Finish git clone to provider:{}", from); - } else { - connection_clone.close(VarInt::from_u32(1), "Cannot find target peer".as_bytes()); - } - Ok(()) -} -async fn lfs_handle_receive(connection: Arc) -> Result<()> { - let connection_clone: Arc = connection.clone(); - let (_file_sender, mut file_receiver) = connection_clone.accept_bi().await?; - - //read header - let mut header_buf = [0u8; 4096]; - let len = file_receiver.read(&mut header_buf).await?.unwrap(); - let header = String::from_utf8_lossy(&header_buf[..len]); - let header: LFSHeader = serde_json::from_str(&header)?; - let (target_id, from, oid, size) = (header.target, header.from, header.oid, header.size); - info!( - "LFS handle receive, target_id:{}, from:{}, oid:{}: size:{}", - target_id, from, oid, size - ); - let key = format!("lfs-{}-{}", target_id, from); - - if let Some(target_conn) = LFS_CONNECTION_MAP.get(&key) { - info!("Find target connection to {}", target_id); - //header data - info!("Send lfs header to {}", target_id); - let (mut target_sender, _) = target_conn.open_bi().await?; - target_sender.write_all(&header_buf[..len]).await?; - target_sender.finish()?; - - //lfs data - info!("Send lfs data to {}", target_id); - let (mut target_sender, _) = target_conn.open_bi().await?; - let (_file_sender, mut file_receiver) = connection_clone.accept_bi().await?; - tokio::io::copy(&mut file_receiver, &mut target_sender).await?; - target_sender.finish()?; - - //send finish to from peer - let (mut sender, _) = connection_clone.open_bi().await?; - sender.write_all("finish".as_bytes()).await?; - sender.finish()?; - info!("Finish lfs to provider:{}", from); - } else { - connection_clone.close(VarInt::from_u32(1), "Cannot find target peer".as_bytes()); + //read header + let mut header_buf = [0u8; 4096]; + let len = file_receiver.read(&mut header_buf).await?.unwrap(); + let header = String::from_utf8_lossy(&header_buf[..len]); + let header: LFSHeader = serde_json::from_str(&header)?; + let (target_id, from, oid, size) = (header.target, header.from, header.oid, header.size); + info!( + "LFS handle receive, target_id:{}, from:{}, oid:{}: size:{}", + target_id, from, oid, size + ); + let key = format!("lfs-{}-{}", target_id, from); + + if let Some(target_conn) = self.lfs_connection_map.get(&key) { + info!("Find target connection to {}", target_id); + //header data + info!("Send lfs header to {}", target_id); + let (mut target_sender, _) = target_conn.open_bi().await?; + target_sender.write_all(&header_buf[..len]).await?; + target_sender.finish()?; + + //lfs data + info!("Send lfs data to {}", target_id); + let (mut target_sender, _) = target_conn.open_bi().await?; + let (_file_sender, mut file_receiver) = connection_clone.accept_bi().await?; + tokio::io::copy(&mut file_receiver, &mut target_sender).await?; + target_sender.finish()?; + + //send finish to from peer + let (mut sender, _) = connection_clone.open_bi().await?; + sender.write_all("finish".as_bytes()).await?; + sender.finish()?; + info!("Finish lfs to provider:{}", from); + } else { + connection_clone.close(VarInt::from_u32(1), "Cannot find target peer".as_bytes()); + } + Ok(()) } - Ok(()) -} -//Relay -pub async fn get_root_certificate_from_vault( -) -> Result<(Vec>, PrivateKeyDer<'static>)> { - let cert = ca::server::get_root_cert_der().await; - let key = ca::server::get_root_key_der().await; + //Relay + pub fn get_root_certificate_from_vault( + &self, + ) -> Result<(Vec>, PrivateKeyDer<'static>)> { + let cert = ca::server::get_root_cert_der(&self.vault); + let key = ca::server::get_root_key_der(&self.vault); - Ok((vec![cert], key)) + Ok((vec![cert], key)) + } } diff --git a/gemini/src/util.rs b/gemini/src/util.rs index 49d0f9da9..8b9477abd 100644 --- a/gemini/src/util.rs +++ b/gemini/src/util.rs @@ -1,13 +1,14 @@ use callisto::git_repo; -use jupiter::context::Context; +use jupiter::storage::Storage; use std::{ net::TcpListener, time::{SystemTime, UNIX_EPOCH}, }; -pub fn get_short_peer_id(peer_id: String) -> String { +pub fn get_short_peer_id(peer_id: impl AsRef) -> String { + let peer_id = peer_id.as_ref(); if peer_id.len() <= 7 { - return peer_id; + return peer_id.to_string(); } peer_id[0..7].to_string() } @@ -45,23 +46,21 @@ pub async fn handle_response( } } -pub async fn repo_alias_to_identifier(alias: String) -> String { - let peer_id = vault::get_peerid().await; - format!("p2p://{}/{alias}", peer_id.clone()) +pub async fn repo_alias_to_identifier(peer_id: impl AsRef, alias: impl AsRef) -> String { + format!("p2p://{}/{}", peer_id.as_ref(), alias.as_ref()) } -pub async fn repo_path_to_identifier(repo_path: String) -> String { - let peer_id = vault::get_peerid().await; - if repo_path.starts_with("/") { - return format!("p2p://{}{repo_path}.git", peer_id.clone()); +pub async fn repo_path_to_identifier(peer_id: impl AsRef, repo_path: impl AsRef) -> String { + if repo_path.as_ref().starts_with("/") { + return format!("p2p://{}{}.git", peer_id.as_ref(), repo_path.as_ref()); } - format!("p2p://{}/{repo_path}.git", peer_id.clone()) + format!("p2p://{}/{}.git", peer_id.as_ref(), repo_path.as_ref()) } -pub async fn get_ztm_app_tunnel_bound_name(remote_peer_id: String) -> String { +pub async fn get_ztm_app_tunnel_bound_name(local_peer_id: impl AsRef, remote_peer_id: impl AsRef) -> String { format!( "{}_{}", - get_short_peer_id(vault::get_peerid().await), + get_short_peer_id(local_peer_id), get_short_peer_id(remote_peer_id) ) } @@ -73,8 +72,8 @@ pub fn get_repo_path(mut path: String) -> String { path.to_string() } -pub async fn get_git_model_by_path(context: Context, path: String) -> Option { - let git_model = context +pub async fn get_git_model_by_path(storage: Storage, path: String) -> Option { + let git_model = storage .services .git_db_storage .find_git_repo_exact_match(get_repo_path(path).as_str()) diff --git a/jupiter/src/context.rs b/jupiter/src/context.rs deleted file mode 100644 index a2ae4a4c8..000000000 --- a/jupiter/src/context.rs +++ /dev/null @@ -1,139 +0,0 @@ -use std::sync::{Arc, LazyLock, Weak}; - -use common::config::Config; - -use crate::{ - lfs_storage::{self, local_storage::LocalStorage, LfsFileStorage}, - storage::{ - git_db_storage::GitDbStorage, init::database_connection, issue_storage::IssueStorage, - lfs_db_storage::LfsDbStorage, mono_storage::MonoStorage, mq_storage::MQStorage, - mr_storage::MrStorage, raw_db_storage::RawDbStorage, relay_storage::RelayStorage, - user_storage::UserStorage, vault_storage::VaultStorage, - }, -}; - -#[derive(Clone)] -pub struct Service { - pub mono_storage: MonoStorage, - pub git_db_storage: GitDbStorage, - pub raw_db_storage: RawDbStorage, - pub lfs_db_storage: LfsDbStorage, - pub relay_storage: RelayStorage, - pub mq_storage: MQStorage, - pub user_storage: UserStorage, - pub vault_storage: VaultStorage, - pub mr_storage: MrStorage, - pub issue_storage: IssueStorage, - pub lfs_file_storage: Arc, -} - -impl Service { - async fn new(config: &Config) -> Self { - let connection = Arc::new(database_connection(&config.database).await); - let lfs_db_storage = LfsDbStorage::new(connection.clone()).await; - - Self { - mono_storage: MonoStorage::new(connection.clone()).await, - git_db_storage: GitDbStorage::new(connection.clone()).await, - raw_db_storage: RawDbStorage::new(connection.clone()).await, - lfs_db_storage: lfs_db_storage.clone(), - relay_storage: RelayStorage::new(connection.clone()).await, - mq_storage: MQStorage::new(connection.clone()).await, - user_storage: UserStorage::new(connection.clone()).await, - mr_storage: MrStorage::new(connection.clone()).await, - issue_storage: IssueStorage::new(connection.clone()).await, - vault_storage: VaultStorage::new(connection.clone()).await, - lfs_file_storage: lfs_storage::init(config.lfs.clone(), lfs_db_storage).await, - } - } - - fn mock() -> Arc { - Arc::new(Self { - mono_storage: MonoStorage::mock(), - git_db_storage: GitDbStorage::mock(), - raw_db_storage: RawDbStorage::mock(), - lfs_db_storage: LfsDbStorage::mock(), - relay_storage: RelayStorage::mock(), - mq_storage: MQStorage::mock(), - user_storage: UserStorage::mock(), - vault_storage: VaultStorage::mock(), - lfs_file_storage: Arc::new(LocalStorage::mock()), - mr_storage: MrStorage::mock(), - issue_storage: IssueStorage::mock(), - }) - } -} - -#[derive(Clone)] -pub struct Storage { - pub services: Arc, - pub config: Weak, -} - -impl Storage { - pub async fn new(config: Arc) -> Self { - Storage { - services: Service::new(&config).await.into(), - config: Arc::downgrade(&config), - } - } - - pub fn config(&self) -> Arc { - self.config.upgrade().expect("Config has been dropped") - } - - pub fn mono_storage(&self) -> MonoStorage { - self.services.mono_storage.clone() - } - - pub fn git_db_storage(&self) -> GitDbStorage { - self.services.git_db_storage.clone() - } - - pub fn raw_db_storage(&self) -> RawDbStorage { - self.services.raw_db_storage.clone() - } - - pub fn lfs_db_storage(&self) -> LfsDbStorage { - self.services.lfs_db_storage.clone() - } - - pub fn relay_storage(&self) -> RelayStorage { - self.services.relay_storage.clone() - } - - pub fn mq_storage(&self) -> MQStorage { - self.services.mq_storage.clone() - } - - pub fn user_storage(&self) -> UserStorage { - self.services.user_storage.clone() - } - - pub fn vault_storage(&self) -> VaultStorage { - self.services.vault_storage.clone() - } - - pub fn mr_storage(&self) -> MrStorage { - self.services.mr_storage.clone() - } - - pub fn issue_storage(&self) -> IssueStorage { - self.services.issue_storage.clone() - } - - pub fn lfs_file_storage(&self) -> Arc { - self.services.lfs_file_storage.clone() - } - - pub fn mock() -> Self { - // During test time, we don't need a AppContext, - // Put config in a leaked static variable thus the weak reference will always be valid. - static CONFIG: LazyLock> = LazyLock::new(|| Config::mock().into()); - - Storage { - services: Service::mock(), - config: Arc::downgrade(&*CONFIG), - } - } -} diff --git a/jupiter/src/lib.rs b/jupiter/src/lib.rs index 5560803cf..9a41004bb 100644 --- a/jupiter/src/lib.rs +++ b/jupiter/src/lib.rs @@ -1,5 +1,4 @@ -pub mod context; -pub mod lfs_storage; pub mod storage; +pub mod lfs_storage; pub mod migrator; pub mod utils; diff --git a/jupiter/src/storage.rs b/jupiter/src/storage.rs new file mode 100644 index 000000000..38f29807a --- /dev/null +++ b/jupiter/src/storage.rs @@ -0,0 +1,241 @@ +pub mod git_db_storage; +pub mod init; +pub mod issue_storage; +pub mod lfs_db_storage; +pub mod mono_storage; +pub mod mq_storage; +pub mod mr_storage; +pub mod raw_db_storage; +pub mod relay_storage; +pub mod user_storage; +pub mod vault_storage; + +use sea_orm::{sea_query::OnConflict, ActiveModelTrait, ConnectionTrait, DbErr, EntityTrait}; + +use common::errors::MegaError; + +use std::sync::{Arc, LazyLock, Weak}; + +use common::config::Config; + +use crate::lfs_storage::{self, local_storage::LocalStorage, LfsFileStorage}; +use crate::storage::init::database_connection; +use crate::storage::{ + git_db_storage::GitDbStorage, issue_storage::IssueStorage, lfs_db_storage::LfsDbStorage, + mono_storage::MonoStorage, mq_storage::MQStorage, mr_storage::MrStorage, + raw_db_storage::RawDbStorage, relay_storage::RelayStorage, user_storage::UserStorage, + vault_storage::VaultStorage, +}; + +#[derive(Clone)] +pub struct Service { + pub mono_storage: MonoStorage, + pub git_db_storage: GitDbStorage, + pub raw_db_storage: RawDbStorage, + pub lfs_db_storage: LfsDbStorage, + pub relay_storage: RelayStorage, + pub mq_storage: MQStorage, + pub user_storage: UserStorage, + pub vault_storage: VaultStorage, + pub mr_storage: MrStorage, + pub issue_storage: IssueStorage, + pub lfs_file_storage: Arc, +} + +impl Service { + async fn new(config: &Config) -> Self { + let connection = Arc::new(database_connection(&config.database).await); + let lfs_db_storage = LfsDbStorage::new(connection.clone()).await; + + Self { + mono_storage: MonoStorage::new(connection.clone()).await, + git_db_storage: GitDbStorage::new(connection.clone()).await, + raw_db_storage: RawDbStorage::new(connection.clone()).await, + lfs_db_storage: lfs_db_storage.clone(), + relay_storage: RelayStorage::new(connection.clone()).await, + mq_storage: MQStorage::new(connection.clone()).await, + user_storage: UserStorage::new(connection.clone()).await, + mr_storage: MrStorage::new(connection.clone()).await, + issue_storage: IssueStorage::new(connection.clone()).await, + vault_storage: VaultStorage::new(connection.clone()).await, + lfs_file_storage: lfs_storage::init(config.lfs.clone(), lfs_db_storage).await, + } + } + + fn mock() -> Arc { + Arc::new(Self { + mono_storage: MonoStorage::mock(), + git_db_storage: GitDbStorage::mock(), + raw_db_storage: RawDbStorage::mock(), + lfs_db_storage: LfsDbStorage::mock(), + relay_storage: RelayStorage::mock(), + mq_storage: MQStorage::mock(), + user_storage: UserStorage::mock(), + vault_storage: VaultStorage::mock(), + lfs_file_storage: Arc::new(LocalStorage::mock()), + mr_storage: MrStorage::mock(), + issue_storage: IssueStorage::mock(), + }) + } +} + +#[derive(Clone)] +pub struct Storage { + pub services: Arc, + pub config: Weak, +} + +impl Storage { + pub async fn new(config: Arc) -> Self { + Storage { + services: Service::new(&config).await.into(), + config: Arc::downgrade(&config), + } + } + + pub fn config(&self) -> Arc { + self.config.upgrade().expect("Config has been dropped") + } + + pub fn mono_storage(&self) -> MonoStorage { + self.services.mono_storage.clone() + } + + pub fn git_db_storage(&self) -> GitDbStorage { + self.services.git_db_storage.clone() + } + + pub fn raw_db_storage(&self) -> RawDbStorage { + self.services.raw_db_storage.clone() + } + + pub fn lfs_db_storage(&self) -> LfsDbStorage { + self.services.lfs_db_storage.clone() + } + + pub fn relay_storage(&self) -> RelayStorage { + self.services.relay_storage.clone() + } + + pub fn mq_storage(&self) -> MQStorage { + self.services.mq_storage.clone() + } + + pub fn user_storage(&self) -> UserStorage { + self.services.user_storage.clone() + } + + pub fn vault_storage(&self) -> VaultStorage { + self.services.vault_storage.clone() + } + + pub fn mr_storage(&self) -> MrStorage { + self.services.mr_storage.clone() + } + + pub fn issue_storage(&self) -> IssueStorage { + self.services.issue_storage.clone() + } + + pub fn lfs_file_storage(&self) -> Arc { + self.services.lfs_file_storage.clone() + } + + pub fn mock() -> Self { + // During test time, we don't need a AppContext, + // Put config in a leaked static variable thus the weak reference will always be valid. + static CONFIG: LazyLock> = LazyLock::new(|| Config::mock().into()); + + Storage { + services: Service::mock(), + config: Arc::downgrade(&*CONFIG), + } + } +} + +/// Performs batch saving of models in the database. +/// +/// The method takes a vector of models to be saved and performs batch inserts using the given entity type `E`. +/// The models should implement the `ActiveModelTrait` trait, which provides the necessary functionality for saving and inserting the models. +/// +/// The method splits the models into smaller chunks, each containing models configured by chunk_size, and inserts them into the database using the `E::insert_many` function. +/// The results of each insertion are collected into a vector of futures. +/// +/// Note: Currently, SQLx does not support packets larger than 16MB. +/// # Arguments +/// +/// * `save_models` - A vector of models to be saved. +/// +/// # Generic Constraints +/// +/// * `E` - The entity type that implements the `EntityTrait` trait. +/// * `A` - The model type that implements the `ActiveModelTrait` trait and is convertible from the corresponding model type of `E`. +/// +/// # Errors +/// +/// Returns a `MegaError` if an error occurs during the batch save operation. +pub async fn batch_save_model( + connection: &impl ConnectionTrait, + save_models: Vec, +) -> Result<(), MegaError> +where + E: EntityTrait, + A: ActiveModelTrait + From<::Model> + Send, +{ + let onconflict = OnConflict::new().do_nothing().to_owned().into(); + batch_save_model_with_conflict( + connection, + save_models, + onconflict, + ) + .await +} + +/// Performs batch saving of models in the database with conflict resolution. +/// +/// This function allows saving models in batches while specifying conflict resolution behavior using the `OnConflict` parameter. +/// It is intended for advanced use cases where fine-grained control over conflict handling is required. +/// +/// # Arguments +/// +/// * `connection` - A reference to the database connection. +/// * `save_models` - A vector of models to be saved. +/// * `onconflict` - Specifies the conflict resolution strategy to be used during insertion. +/// +/// # Generic Constraints +/// +/// * `E` - The entity type that implements the `EntityTrait` trait. +/// * `A` - The model type that implements the `ActiveModelTrait` trait and is convertible from the corresponding model type of `E`. +/// +/// # Errors +/// +/// Returns a `MegaError` if an error occurs during the batch save operation. +/// Note: The function ignores `DbErr::RecordNotInserted` errors, which may lead to silent failures. +/// Use this function with caution and ensure that the `OnConflict` parameter is configured correctly to avoid unintended consequences. +pub async fn batch_save_model_with_conflict( + connection: &impl ConnectionTrait, + save_models: Vec, + onconflict: OnConflict, +) -> Result<(), MegaError> +where + E: EntityTrait, + A: ActiveModelTrait + From<::Model> + Send, +{ + // notice that sqlx not support packets larger than 16MB now + let futures = save_models.chunks(1000).map(|chunk| { + let insert = E::insert_many(chunk.iter().cloned()).on_conflict(onconflict.clone()); + let conn = connection; + async move { + match insert.exec(conn).await { + Ok(_) => Ok(()), + Err(DbErr::RecordNotInserted) => { + // ignore not inserted err + Ok(()) + } + Err(e) => Err(e), + } + } + }); + futures::future::try_join_all(futures).await?; + Ok(()) +} diff --git a/jupiter/src/storage/mod.rs b/jupiter/src/storage/mod.rs deleted file mode 100644 index d7f63256c..000000000 --- a/jupiter/src/storage/mod.rs +++ /dev/null @@ -1,101 +0,0 @@ -pub mod git_db_storage; -pub mod init; -pub mod issue_storage; -pub mod lfs_db_storage; -pub mod mono_storage; -pub mod mq_storage; -pub mod mr_storage; -pub mod raw_db_storage; -pub mod relay_storage; -pub mod user_storage; -pub mod vault_storage; - -use sea_orm::{sea_query::OnConflict, ActiveModelTrait, ConnectionTrait, DbErr, EntityTrait}; - -use common::errors::MegaError; - -/// Performs batch saving of models in the database. -/// -/// The method takes a vector of models to be saved and performs batch inserts using the given entity type `E`. -/// The models should implement the `ActiveModelTrait` trait, which provides the necessary functionality for saving and inserting the models. -/// -/// The method splits the models into smaller chunks, each containing models configured by chunk_size, and inserts them into the database using the `E::insert_many` function. -/// The results of each insertion are collected into a vector of futures. -/// -/// Note: Currently, SQLx does not support packets larger than 16MB. -/// # Arguments -/// -/// * `save_models` - A vector of models to be saved. -/// -/// # Generic Constraints -/// -/// * `E` - The entity type that implements the `EntityTrait` trait. -/// * `A` - The model type that implements the `ActiveModelTrait` trait and is convertible from the corresponding model type of `E`. -/// -/// # Errors -/// -/// Returns a `MegaError` if an error occurs during the batch save operation. -pub async fn batch_save_model( - connection: &impl ConnectionTrait, - save_models: Vec, -) -> Result<(), MegaError> -where - E: EntityTrait, - A: ActiveModelTrait + From<::Model> + Send, -{ - batch_save_model_with_conflict( - connection, - save_models, - OnConflict::new().do_nothing().to_owned(), - ) - .await -} - -/// Performs batch saving of models in the database with conflict resolution. -/// -/// This function allows saving models in batches while specifying conflict resolution behavior using the `OnConflict` parameter. -/// It is intended for advanced use cases where fine-grained control over conflict handling is required. -/// -/// # Arguments -/// -/// * `connection` - A reference to the database connection. -/// * `save_models` - A vector of models to be saved. -/// * `onconflict` - Specifies the conflict resolution strategy to be used during insertion. -/// -/// # Generic Constraints -/// -/// * `E` - The entity type that implements the `EntityTrait` trait. -/// * `A` - The model type that implements the `ActiveModelTrait` trait and is convertible from the corresponding model type of `E`. -/// -/// # Errors -/// -/// Returns a `MegaError` if an error occurs during the batch save operation. -/// Note: The function ignores `DbErr::RecordNotInserted` errors, which may lead to silent failures. -/// Use this function with caution and ensure that the `OnConflict` parameter is configured correctly to avoid unintended consequences. -pub async fn batch_save_model_with_conflict( - connection: &impl ConnectionTrait, - save_models: Vec, - onconflict: OnConflict, -) -> Result<(), MegaError> -where - E: EntityTrait, - A: ActiveModelTrait + From<::Model> + Send, -{ - // notice that sqlx not support packets larger than 16MB now - let futures = save_models.chunks(1000).map(|chunk| { - let insert = E::insert_many(chunk.iter().cloned()).on_conflict(onconflict.clone()); - let conn = connection; - async move { - match insert.exec(conn).await { - Ok(_) => Ok(()), - Err(DbErr::RecordNotInserted) => { - // ignore not inserted err - Ok(()) - } - Err(e) => Err(e), - } - } - }); - futures::future::try_join_all(futures).await?; - Ok(()) -} diff --git a/mono/Cargo.toml b/mono/Cargo.toml index ef02a76d4..81824b20a 100644 --- a/mono/Cargo.toml +++ b/mono/Cargo.toml @@ -7,6 +7,10 @@ keywords = ["git", "monorepo", "decentralized"] license = "MIT OR Apache-2.0" readme = "README.md" +[lib] +name = "mono" +path = "src/lib.rs" + [[bin]] name = "mono" path = "src/main.rs" diff --git a/mono/src/api/api_router.rs b/mono/src/api/api_router.rs index 9be8b6e97..22f25b7d0 100644 --- a/mono/src/api/api_router.rs +++ b/mono/src/api/api_router.rs @@ -288,7 +288,7 @@ async fn path_can_be_cloned( state: State, ) -> Result>, ApiError> { let path: PathBuf = query.path.clone().into(); - let import_dir = state.storage.config.monorepo.import_dir.clone(); + let import_dir = state.storage.config().monorepo.import_dir.clone(); let res = if path.starts_with(&import_dir) { state .storage diff --git a/mono/src/api/mod.rs b/mono/src/api/mod.rs index 899fdd229..06444ca99 100644 --- a/mono/src/api/mod.rs +++ b/mono/src/api/mod.rs @@ -17,7 +17,7 @@ use ceres::{ }; use common::errors::ProtocolError; use jupiter::{ - context::Storage, + storage::Storage, storage::{issue_storage::IssueStorage, mr_storage::MrStorage, user_storage::UserStorage}, }; diff --git a/mono/src/api/oauth/mod.rs b/mono/src/api/oauth/mod.rs index 152128ac7..980ccfb66 100644 --- a/mono/src/api/oauth/mod.rs +++ b/mono/src/api/oauth/mod.rs @@ -52,7 +52,7 @@ async fn login_authorized( State(oauth_client): State, ) -> Result { let store: MemoryStore = MemoryStore::from_ref(&state); - let config = state.storage.config.oauth.as_ref().unwrap(); + let config = state.storage.config().oauth.as_ref().unwrap().clone(); let http_client = reqwest::ClientBuilder::new() .redirect(reqwest::redirect::Policy::none()) @@ -134,7 +134,7 @@ async fn logout( TypedHeader(cookies): TypedHeader, ) -> Result { let store: MemoryStore = MemoryStore::from_ref(&state); - let full_config = state.storage.config.clone(); + let full_config = state.storage.config(); let config = full_config.oauth.as_ref().unwrap(); let cookie = cookies .get(COOKIE_NAME) diff --git a/mono/src/commands/service/http.rs b/mono/src/commands/service/http.rs index ee7642a74..0ac61a8e2 100644 --- a/mono/src/commands/service/http.rs +++ b/mono/src/commands/service/http.rs @@ -1,6 +1,6 @@ use clap::{ArgMatches, Args, Command, FromArgMatches}; -use crate::{context::AppContext, server::https_server::{self}}; +use crate::{ context::AppContext, server::https_server::{self}}; use common::{errors::MegaResult, model::CommonHttpOptions}; pub fn cli() -> Command { diff --git a/mono/src/commands/service/mod.rs b/mono/src/commands/service/mod.rs index e7ab45a57..bbc9b1969 100644 --- a/mono/src/commands/service/mod.rs +++ b/mono/src/commands/service/mod.rs @@ -5,10 +5,8 @@ //! use clap::{ArgMatches, Command}; -use std::sync::Arc; use common::{config::Config, errors::MegaResult}; -use jupiter::context::Storage; use crate::context::AppContext; @@ -29,8 +27,7 @@ pub fn cli() -> Command { // It determines which subcommand was used and calls the appropriate function. #[tokio::main] pub(crate) async fn exec(config: Config, args: &ArgMatches) -> MegaResult { - let config = Arc::new(config); - let context = AppContext::new(config.clone()).await; + let context = AppContext::new(config).await; let (cmd, subcommand_args) = match args.subcommand() { Some((cmd, args)) => (cmd, args), diff --git a/mono/src/commands/service/multi.rs b/mono/src/commands/service/multi.rs index e18699038..9051e5d80 100644 --- a/mono/src/commands/service/multi.rs +++ b/mono/src/commands/service/multi.rs @@ -1,5 +1,4 @@ use clap::{ArgMatches, Args, Command, FromArgMatches, ValueEnum}; -use jupiter::context::Storage; use crate::{context::AppContext, server::{ https_server::{self}, diff --git a/mono/src/context.rs b/mono/src/context.rs index 299da6506..8e1e770a6 100644 --- a/mono/src/context.rs +++ b/mono/src/context.rs @@ -1,12 +1,12 @@ -use std::{ops::Deref, sync::Arc}; +use std::sync::Arc; /// This is the main application context for the Mono application. -// It holds shared state and configuration for the application. -// Including database connections, configuration settings, encrypted vault functions, etc. +/// It holds shared state and configuration for the application. +/// Including database connections, configuration settings, encrypted vault functions, etc. #[derive(Clone)] pub struct AppContext { /// The storage sub-context for the from jupiter abstract layer. - pub storage: jupiter::context::Storage, + pub storage: jupiter::storage::Storage, /// The vault core for managing encrypted data. pub vault: vault::integration::vault_core::VaultCore, @@ -17,8 +17,9 @@ pub struct AppContext { impl AppContext { /// Creates a new application context with the given configuration. - pub async fn new(config: Arc) -> Self { - let storage = jupiter::context::Storage::new(config.clone()).await; + pub async fn new(config: common::config::Config) -> Self { + let config = Arc::new(config); + let storage = jupiter::storage::Storage::new(config.clone()).await; let vault = vault::integration::vault_core::VaultCore::new(storage.clone()); Self { diff --git a/mono/src/git_protocol/http.rs b/mono/src/git_protocol/http.rs index fd88fb892..a600fbe55 100644 --- a/mono/src/git_protocol/http.rs +++ b/mono/src/git_protocol/http.rs @@ -8,7 +8,7 @@ use base64::prelude::*; use bytes::{Bytes, BytesMut}; use futures::{stream, TryStreamExt}; use http::HeaderMap; -use jupiter::context::Storage; +use jupiter::storage::Storage; use tokio::io::AsyncReadExt; use tokio_stream::StreamExt; diff --git a/mono/src/git_protocol/ssh.rs b/mono/src/git_protocol/ssh.rs index 5d390e2b7..f30f5e0c2 100644 --- a/mono/src/git_protocol/ssh.rs +++ b/mono/src/git_protocol/ssh.rs @@ -15,7 +15,7 @@ use ceres::lfs::lfs_structs::Link; use ceres::protocol::smart::{self}; use ceres::protocol::ServiceType; use ceres::protocol::{SmartProtocol, TransportProtocol}; -use jupiter::context::Storage; +use jupiter::storage::Storage; use tokio::sync::Mutex; use crate::git_protocol::http::search_subsequence; diff --git a/mono/src/lib.rs b/mono/src/lib.rs index b8b57cc14..559932e9c 100644 --- a/mono/src/lib.rs +++ b/mono/src/lib.rs @@ -1,6 +1,6 @@ +pub mod context; pub mod api; pub mod cli; mod commands; -pub mod context; pub mod git_protocol; pub mod server; diff --git a/mono/src/main.rs b/mono/src/main.rs index d10afc4c3..951dee95d 100644 --- a/mono/src/main.rs +++ b/mono/src/main.rs @@ -5,6 +5,7 @@ mod cli; mod commands; +mod context; pub mod api; pub mod git_protocol; diff --git a/mono/src/server/https_server.rs b/mono/src/server/https_server.rs index b5b65e9da..43e72bbef 100644 --- a/mono/src/server/https_server.rs +++ b/mono/src/server/https_server.rs @@ -20,7 +20,7 @@ use tower_http::trace::TraceLayer; use ceres::protocol::{ServiceType, SmartProtocol, TransportProtocol}; use common::errors::ProtocolError; use common::model::{CommonHttpOptions, InfoRefsParams}; -use jupiter::context::Storage; +use jupiter::storage::Storage; use utoipa::OpenApi; use utoipa_axum::router::OpenApiRouter; use utoipa_swagger_ui::SwaggerUi; diff --git a/mono/src/server/ssh_server.rs b/mono/src/server/ssh_server.rs index 04c5e4383..01095f246 100644 --- a/mono/src/server/ssh_server.rs +++ b/mono/src/server/ssh_server.rs @@ -14,6 +14,7 @@ use russh::{ use common::model::CommonHttpOptions; use tokio::sync::Mutex; +use vault::integration::vault_core::VaultCoreInterface; use crate::{ context::AppContext, git_protocol::ssh::SshServer}; diff --git a/vault/Cargo.toml b/vault/Cargo.toml index 051feea2c..c7db024b9 100644 --- a/vault/Cargo.toml +++ b/vault/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "vault" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] jupiter = { workspace = true } diff --git a/vault/src/integration/jupiter_backend.rs b/vault/src/integration/jupiter_backend.rs index 6adfe4feb..940008ee3 100644 --- a/vault/src/integration/jupiter_backend.rs +++ b/vault/src/integration/jupiter_backend.rs @@ -1,14 +1,14 @@ -use jupiter::context::StorageContext; +use jupiter::storage::Storage; use rusty_vault::storage::Backend; use tokio::runtime::Handle; pub struct JupiterBackend { - ctx: StorageContext, + ctx: Storage, rt: Handle, } impl JupiterBackend { - pub fn new(ctx: StorageContext) -> Self { + pub fn new(ctx: Storage) -> Self { let rt = tokio::runtime::Handle::current(); JupiterBackend { ctx, rt } } diff --git a/vault/src/integration/mod.rs b/vault/src/integration/mod.rs index 6207f1d4b..718b898b1 100644 --- a/vault/src/integration/mod.rs +++ b/vault/src/integration/mod.rs @@ -1,2 +1,4 @@ +pub use vault_core::VaultCore; + pub mod jupiter_backend; pub mod vault_core; diff --git a/vault/src/integration/vault_core.rs b/vault/src/integration/vault_core.rs index 27a10d3d2..5ab0370f4 100644 --- a/vault/src/integration/vault_core.rs +++ b/vault/src/integration/vault_core.rs @@ -2,12 +2,12 @@ use std::sync::{Arc, RwLock}; use crate::integration::jupiter_backend::JupiterBackend; use common::errors::MegaError; -use jupiter::context::Context; +use jupiter::storage::Storage; use rusty_vault::{ core::Core, logical::{Operation, Request, Response}, - storage::{barrier_aes_gcm, Backend}, + storage::{Backend, barrier_aes_gcm}, }; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; @@ -24,11 +24,28 @@ struct CoreKey { #[derive(Clone)] pub struct VaultCore { core: Arc>, - key: CoreKey, + key: Arc, +} + +/// This is a tool trait that provides methods to interact with the vault core. +/// Commonly you don't need to implement this trait, but use `VaultCore` directly. +/// It provides methods to read, write, and delete secrets in the vault. +pub trait VaultCoreInterface { + fn token(&self) -> &str; + fn read_api(&self, path: impl AsRef) -> Result, MegaError>; + fn write_api( + &self, + path: impl AsRef, + data: Option>, + ) -> Result, MegaError>; + fn delete_api(&self, path: impl AsRef) -> Result, MegaError>; + fn write_secret(&self, name: &str, data: Option>) -> Result<(), MegaError>; + fn read_secret(&self, name: &str) -> Result>, MegaError>; + fn delete_secret(&self, name: &str) -> Result<(), MegaError>; } impl VaultCore { - pub fn new(ctx: Context) -> Self { + pub fn new(ctx: Storage) -> Self { let dir = common::config::mega_base().join("vault"); let key_path = dir.join(CORE_KEY_FILE); @@ -87,17 +104,19 @@ impl VaultCore { core_key.root_token ); - core_key + core_key.into() }; Self { core, key } } +} - pub fn token(&self) -> &str { +impl VaultCoreInterface for VaultCore { + fn token(&self) -> &str { &self.key.root_token } - pub(crate) fn read_api(&self, path: impl AsRef) -> Result, MegaError> { + fn read_api(&self, path: impl AsRef) -> Result, MegaError> { let mut req = Request::new(path.as_ref()); req.operation = Operation::Read; req.client_token = self.token().to_string(); @@ -107,7 +126,7 @@ impl VaultCore { .map_err(|_| MegaError::with_message("Failed to read from vault API")) } - pub(crate) fn write_api( + fn write_api( &self, path: impl AsRef, data: Option>, @@ -122,7 +141,7 @@ impl VaultCore { .map_err(|_| MegaError::with_message("Failed to write to vault API")) } - pub(crate) fn delete_api(&self, path: impl AsRef) -> Result, MegaError> { + fn delete_api(&self, path: impl AsRef) -> Result, MegaError> { let mut req = Request::new(path.as_ref()); req.operation = Operation::Delete; req.client_token = self.token().to_string(); @@ -132,7 +151,7 @@ impl VaultCore { .map_err(|_| MegaError::with_message("Failed to delete from vault API")) } - pub fn write_secret( + fn write_secret( &self, name: &str, data: Option>, @@ -142,7 +161,7 @@ impl VaultCore { Ok(()) } - pub fn read_secret(&self, name: &str) -> Result>, MegaError> { + fn read_secret(&self, name: &str) -> Result>, MegaError> { let resp = self .read_api(&format!("secret/{}", name)) .map_err(|_| MegaError::with_message(format!("Failed to read secret: {}", name)))?; @@ -150,7 +169,7 @@ impl VaultCore { Ok(resp.map(|r| r.data).flatten()) } - pub fn delete_secret(&self, name: &str) -> Result<(), MegaError> { + fn delete_secret(&self, name: &str) -> Result<(), MegaError> { self.delete_api(&format!("secret/{}", name)) .map_err(|_| MegaError::with_message(format!("Failed to delete secret: {}", name)))?; Ok(()) diff --git a/vault/src/lib.rs b/vault/src/lib.rs index 47095c59b..dec206c89 100644 --- a/vault/src/lib.rs +++ b/vault/src/lib.rs @@ -1,5 +1,62 @@ +use crate::integration::vault_core::VaultCoreInterface; + pub mod integration; pub mod nostr; pub mod pgp; pub mod pki; + +/// A trait that defines the interface for a vault. +/// It provides methods to save, get, and delete secrets. +/// You can conviniently implement this trait for your structs that need to interact with a vault. +/// It is designed to be used with a vault core implementation, such as `VaultCore`. +/// +/// # Example: +/// ```rust +/// use vault::Vault; +/// use vault::integration::vault_core::VaultCore; +/// struct MyVault { +/// core: VaultCore, +/// } +/// +/// impl Vault for MyVault { +/// type Core = VaultCore; +/// const VAULT_PREFIX: &'static str = "my_vault_key_prefix"; +/// fn core(&self) -> &Self::Core { +/// &self.core +/// } +/// } +/// ``` +pub trait Vault { + type Core: VaultCoreInterface; + const VAULT_PREFIX: &'static str; + + fn core(&self) -> &Self::Core; + + /// Save a secret to the vault. + fn save_to_vault(&self, key: impl AsRef, value: impl AsRef) { + let key_f = format!("{}_{}", Self::VAULT_PREFIX, key.as_ref()); + let kv_data = serde_json::json!({ + "data": value.as_ref(), + }) + .as_object() + .unwrap() + .clone(); + _ = self.core().write_secret(key_f.as_str(), Some(kv_data)); + } + + /// Get a secret from the vault. + fn get_from_vault(&self, key: String) -> Option { + let key_f = format!("{}_{}", Self::VAULT_PREFIX, key); + match self.core().read_secret(key_f.as_str()) { + Ok(Some(data)) => data.get("data").and_then(|v| v.as_str().map(String::from)), + Ok(None) | Err(_) => None, + } + } + + /// Delete a secret from the vault. + fn delete_from_vault(&self, key: String) { + let key_f = format!("{}_{}", Self::VAULT_PREFIX, key); + _ = self.core().delete_secret(key_f.as_str()); + } +} diff --git a/vault/src/nostr.rs b/vault/src/nostr.rs index a639e869b..c89d7f3d6 100644 --- a/vault/src/nostr.rs +++ b/vault/src/nostr.rs @@ -1,7 +1,7 @@ use secp256k1::{rand, PublicKey, Secp256k1, SecretKey}; use tracing::log; -use crate::integration::vault_core::VaultCore; +use crate::integration::vault_core::{VaultCore, VaultCoreInterface}; const NOSTR_IDENTITY_KEY: &str = "nostr_identity_key"; @@ -28,7 +28,7 @@ impl VaultCore { /// Initialize the Nostr ID if it's not found. /// - return: `(Nostr ID, secret_key)` /// - You can get `Public Key` by just `base58::decode(nostr)` - pub fn load_nostr_id(&self) -> (String, String) { + pub fn load_nostr_pair(&self) -> (String, String) { self.read_secret(NOSTR_IDENTITY_KEY).expect("Failed to read Nostr ID from vault").map(|data| { let nostr = data["nostr"].as_str().unwrap().to_string(); let secret_key = data["secret_key"].as_str().unwrap().to_string(); @@ -50,14 +50,14 @@ impl VaultCore { } /// Initialize the Nostr ID and return it along with the secret key. - pub fn nostr_get_peerid(&self) -> String { - let (id, _sk) = self.load_nostr_id(); + pub fn load_nostr_peerid(&self) -> String { + let (id, _sk) = self.load_nostr_pair(); id } /// Initialize the Nostr ID and return it along with the secret key. - pub fn get_keypair(&self) -> secp256k1::Keypair { - let (_, sk) = self.load_nostr_id(); + pub fn load_nostr_secp_pair(&self) -> secp256k1::Keypair { + let (_, sk) = self.load_nostr_pair(); let secp = secp256k1::Secp256k1::new(); secp256k1::Keypair::from_seckey_str(&secp, &sk).unwrap() } diff --git a/vault/src/pgp.rs b/vault/src/pgp.rs index fd125d4d2..7c598257d 100644 --- a/vault/src/pgp.rs +++ b/vault/src/pgp.rs @@ -9,7 +9,7 @@ use pgp::types::SecretKeyTrait; pub use pgp::KeyType; use pgp::{SecretKeyParams, SecretKeyParamsBuilder, SubkeyParamsBuilder}; -use crate::integration::vault_core::VaultCore; +use crate::integration::vault_core::{VaultCore, VaultCoreInterface}; const VAULT_KEY: &str = "pgp-signed-secret"; diff --git a/vault/src/pki.rs b/vault/src/pki.rs index e77c6db10..f40743f9e 100644 --- a/vault/src/pki.rs +++ b/vault/src/pki.rs @@ -5,11 +5,12 @@ use openssl::asn1::Asn1Time; use openssl::x509::X509; use serde_json::{json, Value}; -use crate::integration::vault_core::VaultCore; +use crate::integration::vault_core::{VaultCore, VaultCoreInterface}; // FIXME: A more official and robust ROLE name const ROLE: &str = "test-role"; +#[allow(unused)] impl VaultCore { /// Initialize the Vault CA fn init_ca(&self) { From ad2c6304796e0c14d4eb304208a8a6c15940195c Mon Sep 17 00:00:00 2001 From: Neon Date: Mon, 23 Jun 2025 19:57:02 +0800 Subject: [PATCH 13/18] context: taken as isolated module --- Cargo.toml | 2 ++ aries/Cargo.toml | 1 + aries/src/service/api/ca_router.rs | 8 ++++---- aries/src/service/relay_server.rs | 16 ++++++++++++---- ceres/src/api_service/import_api_service.rs | 2 +- ceres/src/api_service/mod.rs | 2 +- ceres/src/api_service/mono_api_service.rs | 2 +- ceres/src/lfs/handler.rs | 2 +- ceres/src/pack/import_repo.rs | 2 +- ceres/src/pack/monorepo.rs | 2 +- ceres/src/protocol/mod.rs | 2 +- context/Cargo.toml | 19 +++++++++++++++++++ mono/src/context.rs => context/src/lib.rs | 18 ++++++++++++++++++ gateway/Cargo.toml | 1 + gateway/src/api/github_router.rs | 12 +++++++----- gateway/src/https_server.rs | 7 ++++--- mega/Cargo.toml | 1 + mega/src/commands/service/http.rs | 3 +-- mega/src/commands/service/mod.rs | 11 ++--------- mega/src/commands/service/multi.rs | 4 ++-- mega/src/commands/service/ssh.rs | 3 +-- mono/Cargo.toml | 1 + mono/src/commands/service/http.rs | 4 ++-- mono/src/commands/service/mod.rs | 2 +- mono/src/commands/service/multi.rs | 3 ++- mono/src/commands/service/ssh.rs | 4 ++-- mono/src/lib.rs | 1 - mono/src/main.rs | 1 - mono/src/server/https_server.rs | 2 +- mono/src/server/ssh_server.rs | 3 ++- 30 files changed, 93 insertions(+), 48 deletions(-) create mode 100644 context/Cargo.toml rename mono/src/context.rs => context/src/lib.rs (67%) diff --git a/Cargo.toml b/Cargo.toml index 83a9e1616..bddf07bad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "third-party", "observatory", "scorpio", + "context", ] default-members = ["mega", "mono", "libra", "aries", "orion", "orion-server"] resolver = "1" @@ -32,6 +33,7 @@ vault = { path = "vault" } saturn = { path = "saturn" } mono = { path = "mono" } orion = { path = "orion" } +context = { path = "context" } anyhow = "1.0.98" serde = "1.0.219" diff --git a/aries/Cargo.toml b/aries/Cargo.toml index 005bd2370..76d2fd26b 100644 --- a/aries/Cargo.toml +++ b/aries/Cargo.toml @@ -13,6 +13,7 @@ common = { workspace = true } callisto = { workspace = true } gemini = { workspace = true } jupiter = { workspace = true } +vault = { workspace = true } tokio = { workspace = true } clap = { workspace = true, features = ["derive"] } diff --git a/aries/src/service/api/ca_router.rs b/aries/src/service/api/ca_router.rs index 6b2e60d81..d38689acb 100644 --- a/aries/src/service/api/ca_router.rs +++ b/aries/src/service/api/ca_router.rs @@ -15,7 +15,7 @@ pub fn routers() -> Router { } async fn get_method_router( - _state: State, + state: State, Query(_params): Query, uri: Uri, ) -> Result, (StatusCode, String)> { @@ -29,7 +29,7 @@ async fn get_method_router( return Err((StatusCode::BAD_REQUEST, "Bad request".to_string())); } }; - return match gemini::ca::server::get_certificate(name).await { + return match gemini::ca::server::get_certificate(&state.vault, name) { Ok(cert) => Ok(Response::builder().body(Body::from(cert)).unwrap()), Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), }; @@ -41,7 +41,7 @@ async fn get_method_router( } async fn post_method_router( - _state: State, + state: State, uri: Uri, req: Request, ) -> Result, (StatusCode, String)> { @@ -55,7 +55,7 @@ async fn post_method_router( }; let bytes = to_bytes(req.into_body(), usize::MAX).await.unwrap(); let csr = String::from_utf8(bytes.to_vec()).unwrap(); - return match gemini::ca::server::issue_certificate(name, csr).await { + return match gemini::ca::server::issue_certificate(&state.vault, name, csr) { Ok(cert) => Ok(Response::builder().body(Body::from(cert)).unwrap()), Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), }; diff --git a/aries/src/service/relay_server.rs b/aries/src/service/relay_server.rs index cb683ef3a..ef6c1f9cd 100644 --- a/aries/src/service/relay_server.rs +++ b/aries/src/service/relay_server.rs @@ -4,7 +4,8 @@ use axum::routing::get; use axum::{Json, Router}; use clap::Parser; use common::config::Config; -use jupiter::context::Storage; +use jupiter::storage::Storage; +use vault::integration::VaultCore; use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; @@ -30,6 +31,7 @@ pub struct RelayOptions { #[derive(Clone)] pub struct AppState { pub storage: Storage, + pub vault: VaultCore, pub relay_option: RelayOptions, } @@ -38,8 +40,10 @@ pub async fn run_relay_server(config: Arc, option: RelayOptions) { let server_url = format!("{}:{}", option.host, option.relay_port); tracing::info!("start relay server: {server_url}"); tokio::spawn(async move { - let context = Storage::new(config).await; - gemini::p2p::relay::run(context, option.host, option.relay_port).await + let storage = Storage::new(config).await; + let vault = VaultCore::new(storage.clone()); + let relay = gemini::p2p::relay::P2PRelay::new(storage, vault); + relay.run(option.host, option.relay_port).await }); let addr = SocketAddr::from_str(&server_url).unwrap(); let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); @@ -49,8 +53,12 @@ pub async fn run_relay_server(config: Arc, option: RelayOptions) { } pub async fn app(config: Arc, relay_option: RelayOptions) -> Router { + let storage = Storage::new(config).await; + let vault = VaultCore::new(storage.clone()); + let state = AppState { - storage: Storage::new(config).await, + storage, + vault, relay_option, }; diff --git a/ceres/src/api_service/import_api_service.rs b/ceres/src/api_service/import_api_service.rs index 63df0a144..9f3b0bc3d 100644 --- a/ceres/src/api_service/import_api_service.rs +++ b/ceres/src/api_service/import_api_service.rs @@ -7,7 +7,7 @@ use std::path::PathBuf; use async_trait::async_trait; -use jupiter::context::Storage; +use jupiter::storage::Storage; use mercury::errors::GitError; use mercury::hash::SHA1; use mercury::internal::object::commit::Commit; diff --git a/ceres/src/api_service/mod.rs b/ceres/src/api_service/mod.rs index 728508bf7..54c234211 100644 --- a/ceres/src/api_service/mod.rs +++ b/ceres/src/api_service/mod.rs @@ -7,7 +7,7 @@ use async_trait::async_trait; use callisto::raw_blob; use common::errors::MegaError; -use jupiter::{context::Storage, utils::converter::generate_git_keep_with_timestamp}; +use jupiter::{storage::Storage, utils::converter::generate_git_keep_with_timestamp}; use mercury::{ errors::GitError, hash::SHA1, diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index 2ec5890ac..0e3028eaf 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -9,7 +9,7 @@ use tokio::process::Command; use callisto::sea_orm_active_enums::ConvTypeEnum; use callisto::{mega_blob, mega_tree, raw_blob}; use common::errors::MegaError; -use jupiter::context::Storage; +use jupiter::storage::Storage; use jupiter::storage::batch_save_model; use jupiter::utils::converter::generate_git_keep_with_timestamp; use mercury::errors::GitError; diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index 1473e96e2..0ae4e4d20 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -10,7 +10,7 @@ use tokio_stream::wrappers::ReceiverStream; use callisto::lfs_locks; use common::config::PackConfig; use common::errors::{GitLFSError, MegaError}; -use jupiter::context::Storage; +use jupiter::storage::Storage; use jupiter::storage::lfs_db_storage::LfsDbStorage; use crate::lfs::lfs_structs::{ diff --git a/ceres/src/pack/import_repo.rs b/ceres/src/pack/import_repo.rs index fb932100b..4416d73da 100644 --- a/ceres/src/pack/import_repo.rs +++ b/ceres/src/pack/import_repo.rs @@ -18,7 +18,7 @@ use tokio_stream::wrappers::ReceiverStream; use callisto::{mega_tree, raw_blob, sea_orm_active_enums::RefTypeEnum}; use common::errors::MegaError; -use jupiter::{context::Storage, storage::batch_save_model}; +use jupiter::{storage::Storage, storage::batch_save_model}; use mercury::{ errors::GitError, internal::{ diff --git a/ceres/src/pack/monorepo.rs b/ceres/src/pack/monorepo.rs index adcbf1628..4273d5e05 100644 --- a/ceres/src/pack/monorepo.rs +++ b/ceres/src/pack/monorepo.rs @@ -16,7 +16,7 @@ use common::{ errors::MegaError, utils::{self, MEGA_BRANCH_NAME}, }; -use jupiter::{context::Storage, storage::mr_storage::MrStorage}; +use jupiter::{storage::Storage, storage::mr_storage::MrStorage}; use mercury::internal::{object::ObjectTrait, pack::encode::PackEncoder}; use mercury::{ errors::GitError, diff --git a/ceres/src/protocol/mod.rs b/ceres/src/protocol/mod.rs index 432ead224..de6bca5e7 100644 --- a/ceres/src/protocol/mod.rs +++ b/ceres/src/protocol/mod.rs @@ -7,7 +7,7 @@ use common::{ utils::ZERO_ID, }; use import_refs::RefCommand; -use jupiter::context::Storage; +use jupiter::storage::Storage; use repo::Repo; use crate::pack::{import_repo::ImportRepo, monorepo::MonoRepo, PackHandler}; diff --git a/context/Cargo.toml b/context/Cargo.toml new file mode 100644 index 000000000..7c7419019 --- /dev/null +++ b/context/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "context" +version = "0.1.0" +edition = "2024" + +[lib] +name = "context" +path = "src/lib.rs" + +[features] +default = [] +p2p = ["gemini"] + +[dependencies] +common = { workspace = true } +jupiter = { workspace = true } +vault = { workspace = true } + +gemini = { workspace = true, optional = true } diff --git a/mono/src/context.rs b/context/src/lib.rs similarity index 67% rename from mono/src/context.rs rename to context/src/lib.rs index 8e1e770a6..b168e19a0 100644 --- a/mono/src/context.rs +++ b/context/src/lib.rs @@ -11,6 +11,10 @@ pub struct AppContext { /// The vault core for managing encrypted data. pub vault: vault::integration::vault_core::VaultCore, + /// The client for P2P communication. + #[cfg(feature = "p2p")] + pub client: gemini::p2p::client::P2PClient, + /// The configuration settings for the application. pub config: Arc, } @@ -21,11 +25,25 @@ impl AppContext { let config = Arc::new(config); let storage = jupiter::storage::Storage::new(config.clone()).await; let vault = vault::integration::vault_core::VaultCore::new(storage.clone()); + #[cfg(feature = "p2p")] + let client = gemini::p2p::client::P2PClient::new(storage.clone(), vault.clone()); + + storage + .services + .mono_storage + .init_monorepo(&config.monorepo) + .await; Self { storage, vault, config, + #[cfg(feature = "p2p")] + client, } } + + pub fn wrapped_context(&self) -> Arc { + Arc::new(self.clone()) + } } diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index a7d38fe02..38bcff04d 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -15,6 +15,7 @@ common = { workspace = true } jupiter = { workspace = true } callisto = { workspace = true } gemini = { workspace = true } +context = { workspace = true , features = ["p2p"]} axum = { workspace = true } tower = { workspace = true } diff --git a/gateway/src/api/github_router.rs b/gateway/src/api/github_router.rs index 785af3ce2..c93173916 100644 --- a/gateway/src/api/github_router.rs +++ b/gateway/src/api/github_router.rs @@ -5,6 +5,7 @@ use axum::routing::post; use axum::Json; use reqwest::Client; use serde_json::Value; +use std::sync::LazyLock; use utoipa_axum::router::OpenApiRouter; pub fn routers() -> OpenApiRouter { @@ -38,13 +39,14 @@ pub async fn get_pr_files(pr_url: &str) -> Value { pub async fn get_pr_commits(pr_url: &str) -> Value { get_request(&format!("{}/commits", pr_url)).await } - /// Send a GET request to the given URL and return the JSON response. async fn get_request(url: &str) -> Value { - static CLIENT: Client = Client::builder() - .user_agent("Mega/0.0.1") // IMPORTANT, or 403 Forbidden - .build() - .unwrap(); + static CLIENT: LazyLock = LazyLock::new(|| { + Client::builder() + .user_agent("Mega/0.0.1") // IMPORTANT, or 403 Forbidden + .build() + .unwrap() + }); let resp = CLIENT.get(url).send().await.unwrap(); resp.json().await.unwrap() } diff --git a/gateway/src/https_server.rs b/gateway/src/https_server.rs index 2dcae9bd4..5bb196a6d 100644 --- a/gateway/src/https_server.rs +++ b/gateway/src/https_server.rs @@ -2,7 +2,7 @@ use axum::routing::get; use axum::{http, Router}; use clap::Args; -use mono::context::AppContext; +use context::AppContext; use quinn::rustls; use tower::ServiceBuilder; use tower_http::cors::{Any, CorsLayer}; @@ -10,7 +10,7 @@ use tower_http::decompression::RequestDecompressionLayer; use tower_http::trace::TraceLayer; use common::model::{CommonHttpOptions, P2pOptions}; -use jupiter::context::Storage; +use jupiter::storage::Storage; use mono::api::lfs::lfs_router; use mono::api::MonoApiServiceState; use mono::server::https_server::{get_method_router, post_method_router, AppState}; @@ -113,9 +113,10 @@ pub fn check_run_with_p2p(context: AppContext, p2p: P2pOptions) { bootstrap_node.clone() ); + let client = context.client.wrapped_client(); tokio::spawn(async move { if let Err(e) = - gemini::p2p::client::run(context.storage, context.vault, bootstrap_node).await + client.run(bootstrap_node).await { tracing::error!("P2P client closed:{}", e) } diff --git a/mega/Cargo.toml b/mega/Cargo.toml index 48b6f449b..b4360d2b7 100644 --- a/mega/Cargo.toml +++ b/mega/Cargo.toml @@ -16,6 +16,7 @@ mono = { workspace = true } jupiter = { workspace = true } gateway = { workspace = true } common = { workspace = true } +context = { workspace = true } tokio = { workspace = true, features = ["macros"] } clap = { workspace = true, features = ["derive"] } diff --git a/mega/src/commands/service/http.rs b/mega/src/commands/service/http.rs index 270fa5f55..f80f2acb8 100644 --- a/mega/src/commands/service/http.rs +++ b/mega/src/commands/service/http.rs @@ -2,8 +2,7 @@ use clap::{ArgMatches, Args, Command, FromArgMatches}; use common::errors::MegaResult; use gateway::https_server::{self, HttpOptions}; -use jupiter::context::Storage; -use mono::context::AppContext; +use context::AppContext; pub fn cli() -> Command { HttpOptions::augment_args_for_update(Command::new("http").about("Start Mega HTTP server")) diff --git a/mega/src/commands/service/mod.rs b/mega/src/commands/service/mod.rs index fbcca445b..c8aa3332c 100644 --- a/mega/src/commands/service/mod.rs +++ b/mega/src/commands/service/mod.rs @@ -8,7 +8,7 @@ use clap::{ArgMatches, Command}; use std::sync::Arc; use common::{config::Config, errors::MegaResult}; -use jupiter::context::Storage; +use context::AppContext; mod http; mod multi; @@ -27,14 +27,7 @@ pub fn cli() -> Command { // It determines which subcommand was used and calls the appropriate function. #[tokio::main] pub(crate) async fn exec(config: Config, args: &ArgMatches) -> MegaResult { - let config = Arc::new(config); - let context = Storage::new(config.clone()).await; - - context - .services - .mono_storage - .init_monorepo(&config.monorepo) - .await; + let context = AppContext::new(config).await; let (cmd, subcommand_args) = match args.subcommand() { Some((cmd, args)) => (cmd, args), diff --git a/mega/src/commands/service/multi.rs b/mega/src/commands/service/multi.rs index d9b3be530..8097673be 100644 --- a/mega/src/commands/service/multi.rs +++ b/mega/src/commands/service/multi.rs @@ -5,8 +5,8 @@ use common::{ model::{CommonHttpOptions, P2pOptions}, }; use gateway::https_server::{self, HttpOptions}; -use jupiter::context::Storage; -use mono::{context::AppContext, server::ssh_server::{self, SshCustom, SshOptions}}; +use mono::{server::ssh_server::{self, SshCustom, SshOptions}}; +use context::AppContext; #[derive(Debug, PartialEq, Clone, ValueEnum)] pub enum StartCommand { diff --git a/mega/src/commands/service/ssh.rs b/mega/src/commands/service/ssh.rs index ad0463f34..080728491 100644 --- a/mega/src/commands/service/ssh.rs +++ b/mega/src/commands/service/ssh.rs @@ -1,8 +1,7 @@ use clap::{ArgMatches, Args, Command, FromArgMatches}; use common::errors::MegaResult; -use jupiter::context::Storage; -use mono::context::AppContext; +use context::AppContext; use mono::server::ssh_server::start_server; use mono::server::ssh_server::SshOptions; diff --git a/mono/Cargo.toml b/mono/Cargo.toml index 81824b20a..c9193e238 100644 --- a/mono/Cargo.toml +++ b/mono/Cargo.toml @@ -22,6 +22,7 @@ jupiter = { workspace = true } ceres = { workspace = true } vault = { workspace = true } saturn = { workspace = true } +context = { workspace = true } anyhow = { workspace = true } axum = { workspace = true } diff --git a/mono/src/commands/service/http.rs b/mono/src/commands/service/http.rs index 0ac61a8e2..0265b8559 100644 --- a/mono/src/commands/service/http.rs +++ b/mono/src/commands/service/http.rs @@ -1,6 +1,6 @@ use clap::{ArgMatches, Args, Command, FromArgMatches}; - -use crate::{ context::AppContext, server::https_server::{self}}; +use context::AppContext; +use crate::{ server::https_server::{self}}; use common::{errors::MegaResult, model::CommonHttpOptions}; pub fn cli() -> Command { diff --git a/mono/src/commands/service/mod.rs b/mono/src/commands/service/mod.rs index bbc9b1969..76097138a 100644 --- a/mono/src/commands/service/mod.rs +++ b/mono/src/commands/service/mod.rs @@ -8,7 +8,7 @@ use clap::{ArgMatches, Command}; use common::{config::Config, errors::MegaResult}; -use crate::context::AppContext; +use context::AppContext; pub mod http; pub mod multi; diff --git a/mono/src/commands/service/multi.rs b/mono/src/commands/service/multi.rs index 9051e5d80..985365a15 100644 --- a/mono/src/commands/service/multi.rs +++ b/mono/src/commands/service/multi.rs @@ -1,9 +1,10 @@ use clap::{ArgMatches, Args, Command, FromArgMatches, ValueEnum}; -use crate::{context::AppContext, server::{ +use crate::{ server::{ https_server::{self}, ssh_server::{self, SshCustom, SshOptions}, }}; +use context::AppContext; use common::{errors::MegaResult, model::CommonHttpOptions}; #[derive(Debug, PartialEq, Clone, ValueEnum)] diff --git a/mono/src/commands/service/ssh.rs b/mono/src/commands/service/ssh.rs index 6b73f8074..2db666aef 100644 --- a/mono/src/commands/service/ssh.rs +++ b/mono/src/commands/service/ssh.rs @@ -1,6 +1,6 @@ use clap::{ArgMatches, Args, Command, FromArgMatches}; - -use crate::{context::AppContext, server::ssh_server::{start_server, SshOptions}}; +use context::AppContext; +use crate::{server::ssh_server::{start_server, SshOptions}}; use common::errors::MegaResult; pub fn cli() -> Command { diff --git a/mono/src/lib.rs b/mono/src/lib.rs index 559932e9c..54fe65595 100644 --- a/mono/src/lib.rs +++ b/mono/src/lib.rs @@ -1,4 +1,3 @@ -pub mod context; pub mod api; pub mod cli; mod commands; diff --git a/mono/src/main.rs b/mono/src/main.rs index 951dee95d..d10afc4c3 100644 --- a/mono/src/main.rs +++ b/mono/src/main.rs @@ -5,7 +5,6 @@ mod cli; mod commands; -mod context; pub mod api; pub mod git_protocol; diff --git a/mono/src/server/https_server.rs b/mono/src/server/https_server.rs index 43e72bbef..abfdf14ac 100644 --- a/mono/src/server/https_server.rs +++ b/mono/src/server/https_server.rs @@ -29,7 +29,7 @@ use crate::api::api_router::{self}; use crate::api::lfs::lfs_router; use crate::api::oauth::{self, oauth_client}; use crate::api::MonoApiServiceState; -use crate::context::AppContext; +use context::AppContext; #[derive(Clone)] pub struct AppState { diff --git a/mono/src/server/ssh_server.rs b/mono/src/server/ssh_server.rs index 01095f246..009d802cc 100644 --- a/mono/src/server/ssh_server.rs +++ b/mono/src/server/ssh_server.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use bytes::BytesMut; use clap::Args; +use context::AppContext; use ed25519_dalek::pkcs8::spki::der::pem::LineEnding; use russh::{ keys::{ssh_key::rand_core::OsRng, Algorithm, PrivateKey}, @@ -16,7 +17,7 @@ use common::model::CommonHttpOptions; use tokio::sync::Mutex; use vault::integration::vault_core::VaultCoreInterface; -use crate::{ context::AppContext, git_protocol::ssh::SshServer}; +use crate::{ git_protocol::ssh::SshServer}; #[derive(Args, Clone, Debug)] pub struct SshOptions { From 645d655e8d060db8245cc76fac13ac9e98464417 Mon Sep 17 00:00:00 2001 From: Neon Date: Mon, 23 Jun 2025 21:05:23 +0800 Subject: [PATCH 14/18] cleanup --- Cargo.toml | 2 - aries/src/service/relay_server.rs | 2 +- ceres/src/api_service/mono_api_service.rs | 2 +- ceres/src/lfs/handler.rs | 2 +- ceres/src/pack/import_repo.rs | 2 +- ceres/src/pack/monorepo.rs | 2 +- gateway/src/https_server.rs | 4 +- gemini/src/ca/client.rs | 16 +- gemini/src/ca/mod.rs | 13 +- gemini/src/ca/server.rs | 2 +- gemini/src/lfs/mod.rs | 326 +++++++++++----------- gemini/src/nostr/mod.rs | 7 +- gemini/src/p2p/relay.rs | 82 ++++-- gemini/src/util.rs | 10 +- jupiter/src/lib.rs | 2 +- jupiter/src/migrator/mod.rs | 68 ++--- jupiter/src/storage.rs | 9 +- jupiter/src/storage/init.rs | 4 +- libra/src/internal/protocol/lfs_client.rs | 2 + mega/src/commands/service/http.rs | 2 +- mega/src/commands/service/mod.rs | 1 - mega/src/commands/service/multi.rs | 4 +- mega/tests/service_test.rs | 11 +- mono/src/api/api_router.rs | 3 +- mono/src/commands/service/http.rs | 4 +- mono/src/commands/service/multi.rs | 6 +- mono/src/commands/service/ssh.rs | 4 +- mono/src/git_protocol/http.rs | 6 +- mono/src/server/ssh_server.rs | 2 +- vault/src/integration/jupiter_backend.rs | 10 +- vault/src/integration/vault_core.rs | 14 +- vault/src/nostr.rs | 40 +-- vault/src/pgp.rs | 2 +- vault/src/pki.rs | 8 +- 34 files changed, 358 insertions(+), 316 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bddf07bad..47f8d63c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,9 +13,7 @@ members = [ "saturn", "orion", "orion-server", - "third-party", "observatory", - "scorpio", "context", ] default-members = ["mega", "mono", "libra", "aries", "orion", "orion-server"] diff --git a/aries/src/service/relay_server.rs b/aries/src/service/relay_server.rs index ef6c1f9cd..dea28fff0 100644 --- a/aries/src/service/relay_server.rs +++ b/aries/src/service/relay_server.rs @@ -5,7 +5,6 @@ use axum::{Json, Router}; use clap::Parser; use common::config::Config; use jupiter::storage::Storage; -use vault::integration::VaultCore; use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; @@ -13,6 +12,7 @@ use tower::ServiceBuilder; use tower_http::cors::{Any, CorsLayer}; use tower_http::decompression::RequestDecompressionLayer; use tower_http::trace::TraceLayer; +use vault::integration::VaultCore; use super::api; diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index 0e3028eaf..5573cb074 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -9,8 +9,8 @@ use tokio::process::Command; use callisto::sea_orm_active_enums::ConvTypeEnum; use callisto::{mega_blob, mega_tree, raw_blob}; use common::errors::MegaError; -use jupiter::storage::Storage; use jupiter::storage::batch_save_model; +use jupiter::storage::Storage; use jupiter::utils::converter::generate_git_keep_with_timestamp; use mercury::errors::GitError; use mercury::hash::SHA1; diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index 0ae4e4d20..211d56cea 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -10,8 +10,8 @@ use tokio_stream::wrappers::ReceiverStream; use callisto::lfs_locks; use common::config::PackConfig; use common::errors::{GitLFSError, MegaError}; -use jupiter::storage::Storage; use jupiter::storage::lfs_db_storage::LfsDbStorage; +use jupiter::storage::Storage; use crate::lfs::lfs_structs::{ BatchRequest, BatchResponse, ChunkDownloadObject, Link, Lock, LockList, LockListQuery, diff --git a/ceres/src/pack/import_repo.rs b/ceres/src/pack/import_repo.rs index 4416d73da..9e1037ed3 100644 --- a/ceres/src/pack/import_repo.rs +++ b/ceres/src/pack/import_repo.rs @@ -18,7 +18,7 @@ use tokio_stream::wrappers::ReceiverStream; use callisto::{mega_tree, raw_blob, sea_orm_active_enums::RefTypeEnum}; use common::errors::MegaError; -use jupiter::{storage::Storage, storage::batch_save_model}; +use jupiter::{storage::batch_save_model, storage::Storage}; use mercury::{ errors::GitError, internal::{ diff --git a/ceres/src/pack/monorepo.rs b/ceres/src/pack/monorepo.rs index 4273d5e05..1305540d9 100644 --- a/ceres/src/pack/monorepo.rs +++ b/ceres/src/pack/monorepo.rs @@ -16,7 +16,7 @@ use common::{ errors::MegaError, utils::{self, MEGA_BRANCH_NAME}, }; -use jupiter::{storage::Storage, storage::mr_storage::MrStorage}; +use jupiter::{storage::mr_storage::MrStorage, storage::Storage}; use mercury::internal::{object::ObjectTrait, pack::encode::PackEncoder}; use mercury::{ errors::GitError, diff --git a/gateway/src/https_server.rs b/gateway/src/https_server.rs index 5bb196a6d..3af238f4c 100644 --- a/gateway/src/https_server.rs +++ b/gateway/src/https_server.rs @@ -115,9 +115,7 @@ pub fn check_run_with_p2p(context: AppContext, p2p: P2pOptions) { let client = context.client.wrapped_client(); tokio::spawn(async move { - if let Err(e) = - client.run(bootstrap_node).await - { + if let Err(e) = client.run(bootstrap_node).await { tracing::error!("P2P client closed:{}", e) } }); diff --git a/gemini/src/ca/client.rs b/gemini/src/ca/client.rs index 9f0dd5500..d63a0a174 100644 --- a/gemini/src/ca/client.rs +++ b/gemini/src/ca/client.rs @@ -1,9 +1,9 @@ use anyhow::Result; use anyhow::{anyhow, Ok}; -use rcgen::{CertificateParams, KeyPair}; use quinn::rustls::pki_types::pem::PemObject; use quinn::rustls::pki_types::CertificateDer; use quinn::rustls::pki_types::PrivateKeyDer; +use rcgen::{CertificateParams, KeyPair}; use crate::p2p::client::P2PClient; @@ -23,7 +23,10 @@ impl P2PClient { } } - pub async fn get_user_cert_from_ca(&self, ca: impl AsRef) -> Result<(CertificateDer<'static>, PrivateKeyDer<'static>)> { + pub async fn get_user_cert_from_ca( + &self, + ca: impl AsRef, + ) -> Result<(CertificateDer<'static>, PrivateKeyDer<'static>)> { let name = self.get_peer_id(); // Request to ca let url = format!("{}/api/v1/ca/certificates/{name}", ca.as_ref()); @@ -44,7 +47,8 @@ impl P2PClient { let key = KeyPair::from_pem(&key)?; let user_csr = params.serialize_request(&key)?; //request a new cert - let response = self.http_client + let response = self + .http_client .post(url) .body(user_csr.pem().unwrap()) .send() @@ -59,10 +63,12 @@ impl P2PClient { let key = self.get_user_key(); let key = PrivateKeyDer::from_pem_slice(key.as_bytes())?; Ok((cert, key)) - } - pub async fn get_ca_cert_from_ca(&self, ca: impl AsRef) -> Result> { + pub async fn get_ca_cert_from_ca( + &self, + ca: impl AsRef, + ) -> Result> { //request to ca let url = format!("{}/api/v1/ca/certificates/ca", ca.as_ref()); let url = add_http_to_url(url); diff --git a/gemini/src/ca/mod.rs b/gemini/src/ca/mod.rs index b58e4f3e5..97c9a7725 100755 --- a/gemini/src/ca/mod.rs +++ b/gemini/src/ca/mod.rs @@ -20,14 +20,11 @@ fn save_to_vault(vault: &VaultCore, key: String, value: String) { fn get_from_vault(vault: &VaultCore, key: String) -> Option { let key_f = format!("ca_{key}"); match vault.read_secret(key_f.as_str()).unwrap() { - Some(res) => res - .get(key_f.as_str()) - .map(|v| { - v.as_str() - .map(|vv| String::from(vv.trim_matches(char::is_control))) - }) - .flatten(), - None => return None, + Some(res) => res.get(key_f.as_str()).and_then(|v| { + v.as_str() + .map(|vv| String::from(vv.trim_matches(char::is_control))) + }), + None => None, } } diff --git a/gemini/src/ca/server.rs b/gemini/src/ca/server.rs index e4bcdc3b4..f7a76cf24 100644 --- a/gemini/src/ca/server.rs +++ b/gemini/src/ca/server.rs @@ -19,7 +19,7 @@ static ROOT_KEY: &str = "root_key"; static USER_KEY_PRE: &str = "user_"; pub fn get_root_cert_pem(vault: &VaultCore) -> String { - match get_from_vault(vault,ROOT_CERT.to_string()) { + match get_from_vault(vault, ROOT_CERT.to_string()) { Some(cert) => cert, None => init_self_signed_cert(vault).0, } diff --git a/gemini/src/lfs/mod.rs b/gemini/src/lfs/mod.rs index f39b7aeb5..b9bafedda 100644 --- a/gemini/src/lfs/mod.rs +++ b/gemini/src/lfs/mod.rs @@ -1,179 +1,181 @@ use std::collections::HashSet; -use reqwest::get; +use reqwest::{get, Client}; -use crate::{p2p::client::P2PClient, util::handle_response, LFSInfo, LFSInfoPostBody, LFSInfoRes}; +use crate::{util::handle_response, LFSInfo, LFSInfoPostBody, LFSInfoRes}; -impl P2PClient { - /// share lfs - /// - /// ## paras - /// - `bootstrap_node`: bootstrap_node - /// - `file_hash`: file_hash - /// - `hash_type`: hash_type - /// - `file_size`: file_size - /// - `origin`: origin - /// - /// ## Example - /// Here is an example of the JSON payload: - /// ```json - /// { - /// "bootstrap_node":"https://gitmono.org/relay", - /// "file_hash":"52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6", - /// "hash_type":"sha256", - /// "file_size":199246498, - /// "origin":"p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6" - ///} - /// ``` - /// This method will send a Post request to the relay to share lfs - /// - pub async fn share_lfs( - &self, - file_hash: String, - hash_type: String, - file_size: i64, - origin: String, - ) { - let lfs = LFSInfoPostBody { - file_hash, - hash_type, - file_size, - peer_id: self.get_peer_id(), - origin, - }; - tracing::info!("Share lfs {:?}", lfs); - let json = serde_json::to_string(&lfs).unwrap(); +/// share lfs +/// +/// ## paras +/// - `bootstrap_node`: bootstrap_node +/// - `file_hash`: file_hash +/// - `hash_type`: hash_type +/// - `file_size`: file_size +/// - `origin`: origin +/// +/// ## Example +/// Here is an example of the JSON payload: +/// ```json +/// { +/// "bootstrap_node":"https://gitmono.org/relay", +/// "file_hash":"52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6", +/// "hash_type":"sha256", +/// "file_size":199246498, +/// "origin":"p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6" +///} +/// ``` +/// This method will send a Post request to the relay to share lfs +/// +pub async fn share_lfs( + peer_id: String, + bootstrap_node: String, + file_hash: String, + hash_type: String, + file_size: i64, + origin: String, +) { + let lfs = LFSInfoPostBody { + file_hash, + hash_type, + file_size, + peer_id, + origin, + }; + tracing::info!("Share lfs {:?}", lfs); + let json = serde_json::to_string(&lfs).unwrap(); - let url = format!("{}/api/v1/lfs_share", self.get_bootstrap_node()); - let response = self - .http_client - .post(url) - .header("content-type", "application/json") - .body(json) - .send() - .await - .unwrap(); + let client = Client::new(); + let url = format!("{}/api/v1/lfs_share", bootstrap_node); + let response = client + .post(url) + .header("content-type", "application/json") + .body(json) + .send() + .await + .unwrap(); - if response.status().is_success() { - tracing::info!("Share lfs {} successfully!", lfs.file_hash); - } else { - let context = response.text().await.unwrap(); - tracing::error!("Share lfs {} failed,{}", lfs.file_hash, context); - } + if response.status().is_success() { + tracing::info!("Share lfs {} successfully!", lfs.file_hash); + } else { + let context = response.text().await.unwrap(); + tracing::error!("Share lfs {} failed,{}", lfs.file_hash, context); } +} - /// get lfs chunks info - /// - /// ## paras - /// - `bootstrap_node`: bootstrap_node - /// - `file_hash`: file_hash - /// - /// for example - /// ```json - /// { - /// "bootstrap_node":"https://gitmono.org/relay", - /// "file_hash":"52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6", - ///} - /// ``` - /// This method will send a GET request to the relay to get lfs chunks info - /// - pub async fn get_lfs_chunks_info(&self, file_hash: String) -> Option { - let url = format!( - "{}/api/v1/lfs_chunk?file_hash={}", - self.get_bootstrap_node(), - file_hash - ); - let lfs_info: LFSInfoRes = match get(url.clone()).await { - Ok(response) => { - if !response.status().is_success() { - println!("Get lfs chuncks info failed {}", url); - return None; - } - let body = response.text().await.unwrap(); - let lfs_info: LFSInfoRes = serde_json::from_str(&body).unwrap(); - lfs_info - } - Err(_) => { - println!("Get lfs chuncks info failed {}", url); +/// get lfs chunks info +/// +/// ## paras +/// - `bootstrap_node`: bootstrap_node +/// - `file_hash`: file_hash +/// +/// for example +/// ```json +/// { +/// "bootstrap_node":"https://gitmono.org/relay", +/// "file_hash":"52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6", +///} +/// ``` +/// This method will send a GET request to the relay to get lfs chunks info +/// +pub async fn get_lfs_chunks_info(bootstrap_node: String, file_hash: String) -> Option { + let url = format!( + "{}/api/v1/lfs_chunk?file_hash={}", + bootstrap_node, file_hash + ); + let lfs_info: LFSInfoRes = match get(url.clone()).await { + Ok(response) => { + if !response.status().is_success() { + println!("Get lfs chuncks info failed {}", url); return None; } - }; - Some(lfs_info) - } + let body = response.text().await.unwrap(); + let lfs_info: LFSInfoRes = serde_json::from_str(&body).unwrap(); + lfs_info + } + Err(_) => { + println!("Get lfs chuncks info failed {}", url); + return None; + } + }; + Some(lfs_info) +} - /// create lfs download local ports - /// - /// ## Paras - /// - `bootstrap_node`: bootstrap_node - /// - `ztm_agent_port`: ztm_agent_port - /// - `file_uri`: file_uri - /// - /// for example - /// ```json - /// { - /// "bootstrap_node":"https://gitmono.org/relay", - /// "ztm_agent_port":777, - /// "file_uri":"p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6" - ///} - /// ``` - /// ## Return - /// local_port1, local_port2,... - /// - /// Each port is for a remote peer - pub async fn create_lfs_download_tunnel(&self, file_uri: String) -> Result, String> { - let file_hash = match self.get_file_hash_from_origin(file_uri) { - Ok(file_hash) => file_hash, - Err(_) => { - return Err("invalid file_uri".to_string()); - } - }; - // get public lfs by bootstrap_node - let url = format!("{}/api/v1/lfs_list", self.get_bootstrap_node()); - let request_result = reqwest::get(url.clone()).await; - let response_text = match handle_response(request_result).await { - Ok(s) => s, - Err(s) => { - tracing::error!("GET {url} failed,{s}"); - return Err(s); - } - }; - let lfs_list: Vec = match serde_json::from_slice(response_text.as_bytes()) { - Ok(p) => p, - Err(e) => { - tracing::error!("{}", e); - return Err(e.to_string()); - } - }; - let peer_id = self.get_peer_id(); - let peer_list: HashSet = lfs_list - .iter() - .filter(|x| x.file_hash == file_hash && x.peer_online && x.peer_id != peer_id) - .map(|x| x.peer_id.clone()) - .collect(); - tracing::info!("Search lfs[{}] download peer:{:?}", file_hash, peer_list); +/// create lfs download local ports +/// +/// ## Paras +/// - `bootstrap_node`: bootstrap_node +/// - `ztm_agent_port`: ztm_agent_port +/// - `file_uri`: file_uri +/// +/// for example +/// ```json +/// { +/// "bootstrap_node":"https://gitmono.org/relay", +/// "ztm_agent_port":777, +/// "file_uri":"p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6" +///} +/// ``` +/// ## Return +/// local_port1, local_port2,... +/// +/// Each port is for a remote peer +pub async fn create_lfs_download_tunnel( + peer_id: String, + bootstrap_node: String, + _ztm_agent_port: u16, + file_uri: String, +) -> Result, String> { + let file_hash = match get_file_hash_from_origin(file_uri) { + Ok(file_hash) => file_hash, + Err(_) => { + return Err("invalid file_uri".to_string()); + } + }; + // get public lfs by bootstrap_node + let url = format!("{bootstrap_node}/api/v1/lfs_list"); + let request_result = reqwest::get(url.clone()).await; + let response_text = match handle_response(request_result).await { + Ok(s) => s, + Err(s) => { + tracing::error!("GET {url} failed,{s}"); + return Err(s); + } + }; + let lfs_list: Vec = match serde_json::from_slice(response_text.as_bytes()) { + Ok(p) => p, + Err(e) => { + tracing::error!("{}", e); + return Err(e.to_string()); + } + }; + let peer_list: HashSet = lfs_list + .iter() + .filter(|x| x.file_hash == file_hash && x.peer_online && x.peer_id != peer_id) + .map(|x| x.peer_id.clone()) + .collect(); + tracing::info!("Search lfs[{}] download peer:{:?}", file_hash, peer_list); - let tunnel_list: Vec = vec![]; - // for peer_id in peer_list { - // match get_or_create_remote_mega_tunnel(ztm_agent_port, peer_id).await { - // Ok(port) => { - // tunnel_list.push(port); - // } - // Err(s) => { - // tracing::error!("{}", s); - // } - // } - // } - Ok(tunnel_list) - } + let tunnel_list: Vec = vec![]; + // for peer_id in peer_list { + // match get_or_create_remote_mega_tunnel(ztm_agent_port, peer_id).await { + // Ok(port) => { + // tunnel_list.push(port); + // } + // Err(s) => { + // tracing::error!("{}", s); + // } + // } + // } + Ok(tunnel_list) +} - pub fn get_file_hash_from_origin(&self, origin: String) -> Result { - // p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6 - let words: Vec<&str> = origin.split('/').collect(); - if words.len() <= 4 { - return Err("invalid origin".to_string()); - } - Ok(words.get(4).unwrap().to_string()) +pub fn get_file_hash_from_origin(origin: String) -> Result { + // p2p://t14id7uQxwneJ2PnPtaA3GSUwxTx6HTaq1UkayQVWSPT/sha256/52c90a86cb034b7a1c4beb79304fa76bd0a6cbb7b168c3a935076c714bd1c6b6 + let words: Vec<&str> = origin.split('/').collect(); + if words.len() <= 4 { + return Err("invalid origin".to_string()); } + Ok(words.get(4).unwrap().to_string()) } #[cfg(test)] diff --git a/gemini/src/nostr/mod.rs b/gemini/src/nostr/mod.rs index 86cf61826..c995f8b16 100644 --- a/gemini/src/nostr/mod.rs +++ b/gemini/src/nostr/mod.rs @@ -22,7 +22,12 @@ pub struct GitEventReq { } impl GitEventReq { - pub async fn to_git_event(&self, peer_id: String, identifier: String, commit: String) -> GitEvent { + pub async fn to_git_event( + &self, + peer_id: String, + identifier: String, + commit: String, + ) -> GitEvent { GitEvent { peer: peer_id, uri: identifier, diff --git a/gemini/src/p2p/relay.rs b/gemini/src/p2p/relay.rs index dbb9802aa..eb1e4df49 100644 --- a/gemini/src/p2p/relay.rs +++ b/gemini/src/p2p/relay.rs @@ -73,7 +73,8 @@ impl P2PRelay { tokio::spawn(async move { while let Some((peer_id, nostr_event)) = rx.recv().await { - wrapped.clone() + wrapped + .clone() .send_nostr_event(peer_id, nostr_event) .await .unwrap(); @@ -170,7 +171,8 @@ impl P2PRelay { self.git_clone_handle_receive(connection.clone()).await?; } "REQUEST_LFS" => { - self.lfs_connection_map.insert(key.to_string(), connection.clone()); + self.lfs_connection_map + .insert(key.to_string(), connection.clone()); } "RESPONSE_LFS" => { self.lfs_handle_receive(connection.clone()).await?; @@ -182,9 +184,12 @@ impl P2PRelay { } fn _remove_close_connection(&self) { - self.msg_connection_map.retain(|_, v| v.close_reason().is_some()); - self.git_objects_connection_map.retain(|_, v| v.close_reason().is_some()); - self.lfs_connection_map.retain(|_, v| v.close_reason().is_some()); + self.msg_connection_map + .retain(|_, v| v.close_reason().is_some()); + self.git_objects_connection_map + .retain(|_, v| v.close_reason().is_some()); + self.lfs_connection_map + .retain(|_, v| v.close_reason().is_some()); self.req_id_map.retain(|_, v| v.close_reason().is_some()); } @@ -249,11 +254,16 @@ impl P2PRelay { }; match storage.insert_or_update_node(node).await { Ok(_) => { - self.send_back(data, "ok".as_bytes().to_vec(), connection_clone).await? + self.send_back(data, "ok".as_bytes().to_vec(), connection_clone) + .await? } Err(_) => { - self.send_back_err(data, "Ping with error".to_string(), connection_clone) - .await? + self.send_back_err( + data, + "Ping with error".to_string(), + connection_clone, + ) + .await? } } } @@ -302,7 +312,8 @@ impl P2PRelay { send.finish()?; } let from_connection = connection_clone; - self.req_id_map.insert(data.req_id.to_string(), from_connection.clone()); + self.req_id_map + .insert(data.req_id.to_string(), from_connection.clone()); } Action::Callback => { { @@ -338,12 +349,20 @@ impl P2PRelay { let storage = relay_storage.clone(); match storage.insert_or_update_repo_info(repo_info_model).await { Ok(_) => { - self.send_back(data, repo_info.identifier.into_bytes(), connection.clone()) - .await? + self.send_back( + data, + repo_info.identifier.into_bytes(), + connection.clone(), + ) + .await? } Err(_) => { - self.send_back_err(data, "Repo share failed".to_string(), connection.clone()) - .await? + self.send_back_err( + data, + "Repo share failed".to_string(), + connection.clone(), + ) + .await? } } } @@ -365,8 +384,7 @@ impl P2PRelay { continue; } }; - let relay_msg = - self.nostr_handle( client_msg, data.from.clone()).await; + let relay_msg = self.nostr_handle(client_msg, data.from.clone()).await; self.send_back( data, relay_msg.as_json().as_bytes().to_vec(), @@ -379,11 +397,16 @@ impl P2PRelay { Ok(peers) => { let peers: Vec = peers.iter().map(|p| p.clone().into()).collect(); let res = serde_json::to_string(&peers)?; - self.send_back(data, res.into_bytes(), connection.clone()).await? + self.send_back(data, res.into_bytes(), connection.clone()) + .await? } Err(_) => { - self.send_back_err(data, "Get peers failed".to_string(), connection.clone()) - .await? + self.send_back_err( + data, + "Get peers failed".to_string(), + connection.clone(), + ) + .await? } }; } @@ -407,18 +430,24 @@ impl P2PRelay { } } let res = serde_json::to_string(&repo_list.clone())?; - self.send_back(data, res.into_bytes(), connection.clone()).await? + self.send_back(data, res.into_bytes(), connection.clone()) + .await? } Err(_) => { - self.send_back_err(data, "Get repos failed".to_string(), connection.clone()) - .await? + self.send_back_err( + data, + "Get repos failed".to_string(), + connection.clone(), + ) + .await? } }; } } { - let peers: Vec = self.msg_connection_map + let peers: Vec = self + .msg_connection_map .iter() .map(|entry| entry.key().clone()) .collect(); @@ -430,11 +459,7 @@ impl P2PRelay { } } - async fn nostr_handle( - &self, - client_message: ClientMessage, - from: String, - ) -> RelayMessage { + async fn nostr_handle(&self, client_message: ClientMessage, from: String) -> RelayMessage { let relay_storage = self.storage.relay_storage().clone(); match client_message { ClientMessage::Event(nostr_event) => { @@ -470,7 +495,8 @@ impl P2PRelay { .unwrap(); //Event is forwarded to subscribed nodes - let _ = self.transfer_git_event_to_subscribers(nostr_event.clone(), from) + let _ = self + .transfer_git_event_to_subscribers(nostr_event.clone(), from) .await; RelayMessage::new_ok(nostr_event.id, true, "ok".to_string()) } diff --git a/gemini/src/util.rs b/gemini/src/util.rs index 8b9477abd..3292ebecd 100644 --- a/gemini/src/util.rs +++ b/gemini/src/util.rs @@ -50,14 +50,20 @@ pub async fn repo_alias_to_identifier(peer_id: impl AsRef, alias: impl AsRe format!("p2p://{}/{}", peer_id.as_ref(), alias.as_ref()) } -pub async fn repo_path_to_identifier(peer_id: impl AsRef, repo_path: impl AsRef) -> String { +pub async fn repo_path_to_identifier( + peer_id: impl AsRef, + repo_path: impl AsRef, +) -> String { if repo_path.as_ref().starts_with("/") { return format!("p2p://{}{}.git", peer_id.as_ref(), repo_path.as_ref()); } format!("p2p://{}/{}.git", peer_id.as_ref(), repo_path.as_ref()) } -pub async fn get_ztm_app_tunnel_bound_name(local_peer_id: impl AsRef, remote_peer_id: impl AsRef) -> String { +pub async fn get_ztm_app_tunnel_bound_name( + local_peer_id: impl AsRef, + remote_peer_id: impl AsRef, +) -> String { format!( "{}_{}", get_short_peer_id(local_peer_id), diff --git a/jupiter/src/lib.rs b/jupiter/src/lib.rs index 9a41004bb..c97820e8a 100644 --- a/jupiter/src/lib.rs +++ b/jupiter/src/lib.rs @@ -1,4 +1,4 @@ -pub mod storage; pub mod lfs_storage; pub mod migrator; +pub mod storage; pub mod utils; diff --git a/jupiter/src/migrator/mod.rs b/jupiter/src/migrator/mod.rs index 9f90febdf..f9fcff09e 100644 --- a/jupiter/src/migrator/mod.rs +++ b/jupiter/src/migrator/mod.rs @@ -26,9 +26,17 @@ //! // Refresh all migrations (development only) //! apply_migrations(&db, true).await?; //! ``` -//! -//! # Safety -//! +use sea_orm::DatabaseConnection; +use sea_orm_migration::prelude::*; +use sea_orm_migration::schema::big_integer; +use tracing::log; + +use common::errors::MegaError; + +mod m20250314_025943_init; +mod m20250427_031332_add_mr_refs_tag; +mod m20250605_013340_alter_mega_mr_index; +mod m20250610_000001_add_vault_storage; /// Creates a primary key column definition with big integer type. /// @@ -39,10 +47,28 @@ /// # Returns /// /// A `ColumnDef` configured as a primary key big integer column +fn pk_bigint(name: T) -> ColumnDef { + big_integer(name).primary_key().take() +} + /// The main migrator struct that implements the migration trait. /// /// This struct is responsible for managing all database migrations in the correct order. +pub struct Migrator; + + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![ + Box::new(m20250314_025943_init::Migration), + Box::new(m20250427_031332_add_mr_refs_tag::Migration), + Box::new(m20250605_013340_alter_mega_mr_index::Migration), + Box::new(m20250610_000001_add_vault_storage::Migration), + ] + } +} /// Applies database migrations to the given database connection. /// @@ -62,41 +88,7 @@ /// - Database connection fails /// - Migration SQL execution fails /// - Schema validation errors occur -use sea_orm::DatabaseConnection; -use sea_orm_migration::prelude::*; -use sea_orm_migration::schema::big_integer; -use tracing::log; - -use common::errors::MegaError; - -mod m20250314_025943_init; -mod m20250427_031332_add_mr_refs_tag; -mod m20250605_013340_alter_mega_mr_index; -mod m20250610_000001_add_vault_storage; - -pub(self) fn pk_bigint(name: T) -> ColumnDef { - big_integer(name).primary_key().take() -} - -pub struct Migrator; - -#[async_trait::async_trait] -impl MigratorTrait for Migrator { - fn migrations() -> Vec> { - vec![ - Box::new(m20250314_025943_init::Migration), - Box::new(m20250427_031332_add_mr_refs_tag::Migration), - Box::new(m20250605_013340_alter_mega_mr_index::Migration), - Box::new(m20250610_000001_add_vault_storage::Migration), - ] - } -} - - -pub async fn apply_migrations( - db: &DatabaseConnection, - refresh: bool, -) -> Result<(), MegaError> { +pub async fn apply_migrations(db: &DatabaseConnection, refresh: bool) -> Result<(), MegaError> { match refresh { true => Migrator::refresh(db).await, false => Migrator::up(db, None).await, diff --git a/jupiter/src/storage.rs b/jupiter/src/storage.rs index 38f29807a..80e1a1568 100644 --- a/jupiter/src/storage.rs +++ b/jupiter/src/storage.rs @@ -182,13 +182,8 @@ where E: EntityTrait, A: ActiveModelTrait + From<::Model> + Send, { - let onconflict = OnConflict::new().do_nothing().to_owned().into(); - batch_save_model_with_conflict( - connection, - save_models, - onconflict, - ) - .await + let onconflict = OnConflict::new().do_nothing().to_owned(); + batch_save_model_with_conflict(connection, save_models, onconflict).await } /// Performs batch saving of models in the database with conflict resolution. diff --git a/jupiter/src/storage/init.rs b/jupiter/src/storage/init.rs index b87e54422..eed5faae7 100644 --- a/jupiter/src/storage/init.rs +++ b/jupiter/src/storage/init.rs @@ -27,7 +27,9 @@ pub async fn database_connection(db_config: &DbConfig) -> DatabaseConnection { } else { sqlite_connection(db_config).await.unwrap() }; - apply_migrations(&conn, false).await.expect("Failed to apply migrations"); + apply_migrations(&conn, false) + .await + .expect("Failed to apply migrations"); conn } diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index b80f92b73..af403d2aa 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -541,6 +541,7 @@ impl LFSClient { &self, file_uri: &str, // p2p protocol path: impl AsRef, + peer_id: String, mut reporter: Option<( &mut (dyn FnMut(f64) -> anyhow::Result<()> + Send), // progress callback f64, // step @@ -554,6 +555,7 @@ impl LFSClient { let hash = gemini::lfs::get_file_hash_from_origin(file_uri.to_owned()).unwrap(); tracing::info!("Downloading LFS file: {}", hash); let peer_ports = gemini::lfs::create_lfs_download_tunnel( + peer_id.clone(), bootstrap_node.clone(), *ztm_agent_port, file_uri.to_owned(), diff --git a/mega/src/commands/service/http.rs b/mega/src/commands/service/http.rs index f80f2acb8..eb80698f5 100644 --- a/mega/src/commands/service/http.rs +++ b/mega/src/commands/service/http.rs @@ -1,8 +1,8 @@ use clap::{ArgMatches, Args, Command, FromArgMatches}; use common::errors::MegaResult; -use gateway::https_server::{self, HttpOptions}; use context::AppContext; +use gateway::https_server::{self, HttpOptions}; pub fn cli() -> Command { HttpOptions::augment_args_for_update(Command::new("http").about("Start Mega HTTP server")) diff --git a/mega/src/commands/service/mod.rs b/mega/src/commands/service/mod.rs index c8aa3332c..5ba1fe670 100644 --- a/mega/src/commands/service/mod.rs +++ b/mega/src/commands/service/mod.rs @@ -5,7 +5,6 @@ //! use clap::{ArgMatches, Command}; -use std::sync::Arc; use common::{config::Config, errors::MegaResult}; use context::AppContext; diff --git a/mega/src/commands/service/multi.rs b/mega/src/commands/service/multi.rs index 8097673be..cb3d57a6b 100644 --- a/mega/src/commands/service/multi.rs +++ b/mega/src/commands/service/multi.rs @@ -4,9 +4,9 @@ use common::{ errors::MegaResult, model::{CommonHttpOptions, P2pOptions}, }; -use gateway::https_server::{self, HttpOptions}; -use mono::{server::ssh_server::{self, SshCustom, SshOptions}}; use context::AppContext; +use gateway::https_server::{self, HttpOptions}; +use mono::server::ssh_server::{self, SshCustom, SshOptions}; #[derive(Debug, PartialEq, Clone, ValueEnum)] pub enum StartCommand { diff --git a/mega/tests/service_test.rs b/mega/tests/service_test.rs index dafca36d8..565d33029 100644 --- a/mega/tests/service_test.rs +++ b/mega/tests/service_test.rs @@ -4,10 +4,15 @@ use serial_test::serial; #[serial] async fn check_mono_service_status() -> Result<(), reqwest::Error> { let client = reqwest::Client::new(); - let response = client.get("http://127.0.0.1:8000/api/v1/status") + let response = client + .get("http://127.0.0.1:8000/api/v1/status") .send() .await?; - - assert_eq!(response.status(), reqwest::StatusCode::OK, "Service status API did not return 200 OK"); + + assert_eq!( + response.status(), + reqwest::StatusCode::OK, + "Service status API did not return 200 OK" + ); Ok(()) } diff --git a/mono/src/api/api_router.rs b/mono/src/api/api_router.rs index 22f25b7d0..4eea1efb4 100644 --- a/mono/src/api/api_router.rs +++ b/mono/src/api/api_router.rs @@ -12,7 +12,8 @@ use http::StatusCode; use ceres::{ api_service::ApiHandler, model::git::{ - BlobContentQuery, CodePreviewQuery, CreateFileInfo, LatestCommitInfo, TreeBriefItem, TreeCommitItem, TreeHashItem, TreeQuery + BlobContentQuery, CodePreviewQuery, CreateFileInfo, LatestCommitInfo, TreeBriefItem, + TreeCommitItem, TreeHashItem, TreeQuery, }, }; use common::model::CommonResult; diff --git a/mono/src/commands/service/http.rs b/mono/src/commands/service/http.rs index 0265b8559..659f50f84 100644 --- a/mono/src/commands/service/http.rs +++ b/mono/src/commands/service/http.rs @@ -1,7 +1,7 @@ +use crate::server::https_server::{self}; use clap::{ArgMatches, Args, Command, FromArgMatches}; -use context::AppContext; -use crate::{ server::https_server::{self}}; use common::{errors::MegaResult, model::CommonHttpOptions}; +use context::AppContext; pub fn cli() -> Command { CommonHttpOptions::augment_args_for_update(Command::new("http").about("Start Mega HTTP server")) diff --git a/mono/src/commands/service/multi.rs b/mono/src/commands/service/multi.rs index 985365a15..fe027ce6b 100644 --- a/mono/src/commands/service/multi.rs +++ b/mono/src/commands/service/multi.rs @@ -1,11 +1,11 @@ use clap::{ArgMatches, Args, Command, FromArgMatches, ValueEnum}; -use crate::{ server::{ +use crate::server::{ https_server::{self}, ssh_server::{self, SshCustom, SshOptions}, -}}; -use context::AppContext; +}; use common::{errors::MegaResult, model::CommonHttpOptions}; +use context::AppContext; #[derive(Debug, PartialEq, Clone, ValueEnum)] pub enum StartCommand { diff --git a/mono/src/commands/service/ssh.rs b/mono/src/commands/service/ssh.rs index 2db666aef..46b8c88d1 100644 --- a/mono/src/commands/service/ssh.rs +++ b/mono/src/commands/service/ssh.rs @@ -1,7 +1,7 @@ +use crate::server::ssh_server::{start_server, SshOptions}; use clap::{ArgMatches, Args, Command, FromArgMatches}; -use context::AppContext; -use crate::{server::ssh_server::{start_server, SshOptions}}; use common::errors::MegaResult; +use context::AppContext; pub fn cli() -> Command { SshOptions::augment_args_for_update(Command::new("ssh").about("Start Git SSH server")) diff --git a/mono/src/git_protocol/http.rs b/mono/src/git_protocol/http.rs index a600fbe55..519b61d9a 100644 --- a/mono/src/git_protocol/http.rs +++ b/mono/src/git_protocol/http.rs @@ -185,7 +185,11 @@ pub async fn git_receive_pack( req: Request, mut pack_protocol: SmartProtocol, ) -> Result, ProtocolError> { - if pack_protocol.storage.config().authentication.enable_http_auth + if pack_protocol + .storage + .config() + .authentication + .enable_http_auth && !http_auth(req.headers(), &pack_protocol.storage).await { return auth_failed(); diff --git a/mono/src/server/ssh_server.rs b/mono/src/server/ssh_server.rs index 009d802cc..139d8ab27 100644 --- a/mono/src/server/ssh_server.rs +++ b/mono/src/server/ssh_server.rs @@ -17,7 +17,7 @@ use common::model::CommonHttpOptions; use tokio::sync::Mutex; use vault::integration::vault_core::VaultCoreInterface; -use crate::{ git_protocol::ssh::SshServer}; +use crate::git_protocol::ssh::SshServer; #[derive(Args, Clone, Debug)] pub struct SshOptions { diff --git a/vault/src/integration/jupiter_backend.rs b/vault/src/integration/jupiter_backend.rs index 940008ee3..99ddbd1a6 100644 --- a/vault/src/integration/jupiter_backend.rs +++ b/vault/src/integration/jupiter_backend.rs @@ -25,7 +25,10 @@ impl Backend for JupiterBackend { }) } - fn get(&self, key: &str) -> Result, rusty_vault::errors::RvError> { + fn get( + &self, + key: &str, + ) -> Result, rusty_vault::errors::RvError> { let service = self.ctx.vault_storage(); self.rt.block_on(async move { match service.load(key).await { @@ -41,7 +44,10 @@ impl Backend for JupiterBackend { }) } - fn put(&self, entry: &rusty_vault::storage::BackendEntry) -> Result<(), rusty_vault::errors::RvError> { + fn put( + &self, + entry: &rusty_vault::storage::BackendEntry, + ) -> Result<(), rusty_vault::errors::RvError> { let service = self.ctx.vault_storage(); self.rt.block_on(async move { match service.save(&entry.key, entry.value.clone()).await { diff --git a/vault/src/integration/vault_core.rs b/vault/src/integration/vault_core.rs index 5ab0370f4..99193941c 100644 --- a/vault/src/integration/vault_core.rs +++ b/vault/src/integration/vault_core.rs @@ -151,26 +151,22 @@ impl VaultCoreInterface for VaultCore { .map_err(|_| MegaError::with_message("Failed to delete from vault API")) } - fn write_secret( - &self, - name: &str, - data: Option>, - ) -> Result<(), MegaError> { - self.write_api(&format!("secret/{}", name), data) + fn write_secret(&self, name: &str, data: Option>) -> Result<(), MegaError> { + self.write_api(format!("secret/{}", name), data) .map_err(|_| MegaError::with_message(format!("Failed to write secret: {}", name)))?; Ok(()) } fn read_secret(&self, name: &str) -> Result>, MegaError> { let resp = self - .read_api(&format!("secret/{}", name)) + .read_api(format!("secret/{}", name)) .map_err(|_| MegaError::with_message(format!("Failed to read secret: {}", name)))?; - Ok(resp.map(|r| r.data).flatten()) + Ok(resp.and_then(|r| r.data)) } fn delete_secret(&self, name: &str) -> Result<(), MegaError> { - self.delete_api(&format!("secret/{}", name)) + self.delete_api(format!("secret/{}", name)) .map_err(|_| MegaError::with_message(format!("Failed to delete secret: {}", name)))?; Ok(()) } diff --git a/vault/src/nostr.rs b/vault/src/nostr.rs index c89d7f3d6..9213170af 100644 --- a/vault/src/nostr.rs +++ b/vault/src/nostr.rs @@ -1,4 +1,4 @@ -use secp256k1::{rand, PublicKey, Secp256k1, SecretKey}; +use secp256k1::{PublicKey, Secp256k1, SecretKey, rand}; use tracing::log; use crate::integration::vault_core::{VaultCore, VaultCoreInterface}; @@ -24,29 +24,32 @@ pub fn generate_nostr_id() -> (String, (SecretKey, PublicKey)) { } impl VaultCore { - /// Initialize the Nostr ID if it's not found. /// - return: `(Nostr ID, secret_key)` /// - You can get `Public Key` by just `base58::decode(nostr)` pub fn load_nostr_pair(&self) -> (String, String) { - self.read_secret(NOSTR_IDENTITY_KEY).expect("Failed to read Nostr ID from vault").map(|data| { - let nostr = data["nostr"].as_str().unwrap().to_string(); - let secret_key = data["secret_key"].as_str().unwrap().to_string(); - (nostr, secret_key) - }).unwrap_or_else(|| { - log::debug!("Nostr ID not found in vault, generating new one..."); - let (nostr, (secret_key, _)) = generate_nostr_id(); - let data = serde_json::json!({ - "nostr": nostr, - "secret_key": secret_key.display_secret().to_string(), + self.read_secret(NOSTR_IDENTITY_KEY) + .expect("Failed to read Nostr ID from vault") + .map(|data| { + let nostr = data["nostr"].as_str().unwrap().to_string(); + let secret_key = data["secret_key"].as_str().unwrap().to_string(); + (nostr, secret_key) }) - .as_object() - .unwrap() - .clone(); + .unwrap_or_else(|| { + log::debug!("Nostr ID not found in vault, generating new one..."); + let (nostr, (secret_key, _)) = generate_nostr_id(); + let data = serde_json::json!({ + "nostr": nostr, + "secret_key": secret_key.display_secret().to_string(), + }) + .as_object() + .unwrap() + .clone(); - self.write_secret(NOSTR_IDENTITY_KEY, Some(data.clone())).expect("Failed to write Nostr ID to vault"); - (nostr, secret_key.display_secret().to_string()) - }) + self.write_secret(NOSTR_IDENTITY_KEY, Some(data.clone())) + .expect("Failed to write Nostr ID to vault"); + (nostr, secret_key.display_secret().to_string()) + }) } /// Initialize the Nostr ID and return it along with the secret key. @@ -63,7 +66,6 @@ impl VaultCore { } } - #[cfg(test)] mod tests { use super::*; diff --git a/vault/src/pgp.rs b/vault/src/pgp.rs index 7c598257d..4631caeb7 100644 --- a/vault/src/pgp.rs +++ b/vault/src/pgp.rs @@ -4,9 +4,9 @@ /// using asynchronous operations. use smallvec::smallvec; +pub use pgp::KeyType; pub use pgp::composed::{Deserializable, SignedPublicKey, SignedSecretKey}; use pgp::types::SecretKeyTrait; -pub use pgp::KeyType; use pgp::{SecretKeyParams, SecretKeyParamsBuilder, SubkeyParamsBuilder}; use crate::integration::vault_core::{VaultCore, VaultCoreInterface}; diff --git a/vault/src/pki.rs b/vault/src/pki.rs index f40743f9e..7ca94082d 100644 --- a/vault/src/pki.rs +++ b/vault/src/pki.rs @@ -3,7 +3,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use openssl::asn1::Asn1Time; use openssl::x509::X509; -use serde_json::{json, Value}; +use serde_json::{Value, json}; use crate::integration::vault_core::{VaultCore, VaultCoreInterface}; @@ -101,7 +101,7 @@ impl VaultCore { .clone(); // config role - self.write_api(&format!("pki/roles/{}", ROLE), Some(role_data)) + self.write_api(format!("pki/roles/{}", ROLE), Some(role_data)) .expect("Failed to configure role"); } @@ -116,7 +116,7 @@ impl VaultCore { .clone(); // issue cert - let resp = self.write_api(&format!("pki/issue/{}", ROLE), Some(issue_data)); + let resp = self.write_api(format!("pki/issue/{}", ROLE), Some(issue_data)); let resp_body = resp.unwrap(); let cert_data = resp_body.unwrap().data.unwrap(); @@ -213,7 +213,7 @@ mod tests_raw { storage, storage::barrier_aes_gcm, }; - use serde_json::{json, Map, Value}; + use serde_json::{Map, Value, json}; async fn test_read_api( core: &Core, From a790147edffb3b596aac425557659c111c1dd07e Mon Sep 17 00:00:00 2001 From: Neon Date: Tue, 24 Jun 2025 21:49:26 +0800 Subject: [PATCH 15/18] tests(vault, jupiter): pass the tests --- ceres/src/api_service/mono_api_service.rs | 2 +- jupiter/Cargo.toml | 7 +- jupiter/src/lib.rs | 3 + jupiter/src/main.rs | 7 -- .../m20250610_000001_add_vault_storage.rs | 30 +++-- jupiter/src/migrator/mod.rs | 29 ++++- jupiter/src/storage/vault_storage.rs | 34 +++--- jupiter/src/tests.rs | 64 ++++++++++ mono/Cargo.toml | 1 + mono/src/server/https_server.rs | 4 +- vault/Cargo.toml | 5 + vault/src/integration/jupiter_backend.rs | 98 +++++++++++----- vault/src/integration/vault_core.rs | 110 +++++++++++++++++- 13 files changed, 320 insertions(+), 74 deletions(-) delete mode 100644 jupiter/src/main.rs create mode 100644 jupiter/src/tests.rs diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index 3a07deab1..8e210ee79 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -267,7 +267,7 @@ impl MonoApiService { // update mr status last self.storage .mr_storage() - .merge_mr(mr.clone().into()) + .merge_mr(mr.clone()) .await .unwrap(); } else { diff --git a/jupiter/Cargo.toml b/jupiter/Cargo.toml index cebd789e3..3e97c5f1d 100644 --- a/jupiter/Cargo.toml +++ b/jupiter/Cargo.toml @@ -9,10 +9,6 @@ edition = "2021" name = "jupiter" path = "src/lib.rs" -[[bin]] -name = "migrator" -path = "src/main.rs" - [dependencies] callisto = { workspace = true } common = { workspace = true } @@ -36,10 +32,11 @@ futures = { workspace = true } serde_json = { workspace = true } idgenerator = { workspace = true } serde = { workspace = true } -tokio = { workspace = true, features = ["macros"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } uuid = { workspace = true } hex = { workspace = true } ring = { workspace = true } aws-config = { workspace = true, features = ["behavior-version-latest"] } aws-sdk-s3 = { workspace = true, features = ["rt-tokio"] } anyhow = { workspace = true } +tempfile = { workspace = true } diff --git a/jupiter/src/lib.rs b/jupiter/src/lib.rs index c97820e8a..c4786d412 100644 --- a/jupiter/src/lib.rs +++ b/jupiter/src/lib.rs @@ -2,3 +2,6 @@ pub mod lfs_storage; pub mod migrator; pub mod storage; pub mod utils; + +// FIXME: use a global tests module instead +pub mod tests; diff --git a/jupiter/src/main.rs b/jupiter/src/main.rs deleted file mode 100644 index 10ff6d532..000000000 --- a/jupiter/src/main.rs +++ /dev/null @@ -1,7 +0,0 @@ -use jupiter::migrator; -use sea_orm_migration::prelude::*; - -#[tokio::main] -async fn main() { - cli::run_cli(migrator::Migrator).await; -} diff --git a/jupiter/src/migrator/m20250610_000001_add_vault_storage.rs b/jupiter/src/migrator/m20250610_000001_add_vault_storage.rs index 7491d70cf..36af4daa9 100644 --- a/jupiter/src/migrator/m20250610_000001_add_vault_storage.rs +++ b/jupiter/src/migrator/m20250610_000001_add_vault_storage.rs @@ -1,9 +1,4 @@ -use sea_orm_migration::{ - prelude::*, - schema::{binary, string}, -}; - -use crate::migrator::pk_bigint; +use sea_orm_migration::{prelude::*, schema::big_integer}; #[derive(DeriveMigrationName)] pub struct Migration; @@ -16,9 +11,20 @@ impl MigrationTrait for Migration { Table::create() .table(Vault::Table) .if_not_exists() - .col(pk_bigint(Vault::Id)) - .col(string(Vault::Key)) - .col(binary(Vault::Value)) + .col(big_integer(Vault::Id).primary_key().auto_increment()) + .col(ColumnDef::new(Vault::Key).string().not_null().unique_key()) + .col(ColumnDef::new(Vault::Value).binary().not_null()) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .name("vault_key_unique") + .table(Vault::Table) + .col(Vault::Key) + .unique() .to_owned(), ) .await?; @@ -26,8 +32,10 @@ impl MigrationTrait for Migration { Ok(()) } - async fn down(&self, _: &SchemaManager) -> Result<(), DbErr> { - Ok(()) + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(Vault::Table).to_owned()) + .await } } diff --git a/jupiter/src/migrator/mod.rs b/jupiter/src/migrator/mod.rs index 30f9a51b3..2409f7556 100644 --- a/jupiter/src/migrator/mod.rs +++ b/jupiter/src/migrator/mod.rs @@ -14,10 +14,12 @@ //! - `m20250427_031332_add_mr_refs_tag` - Adds merge request reference tagging //! - `m20250605_013340_alter_mega_mr_index` - Modifies merge request indexing //! - `m20250610_000001_add_vault_storage` - Adds vault storage functionality +//! - `m20250613_033821_alter_user_id` - Alters user ID column definitions +//! - `m20250618_065050_add_label` - Adds label functionality to issues //! //! # Usage //! -//! ```rust +//! ```rust,ignore //! use jupiter::migrator::apply_migrations; //! //! // Apply pending migrations @@ -100,3 +102,28 @@ pub async fn apply_migrations(db: &DatabaseConnection, refresh: bool) -> Result< e.into() }) } + +#[cfg(test)] +mod tests { + use crate::tests::test_db_connection; + + use super::*; + + #[tokio::test] + async fn test_apply_migrations() { + let (db, _) = test_db_connection().await; + // Apply migrations to the mock database + let result = apply_migrations(&db, false).await; + assert!( + result.is_ok(), + "Failed to apply migrations: {:?}", + result.err() + ); + + // Verify that the migrations were applied correctly + let applied_migrations = Migrator::get_applied_migrations(&db).await.unwrap(); + assert!(!applied_migrations.is_empty(), "No migrations were applied"); + + // Additional assertions can be added here to verify the state of the database + } +} diff --git a/jupiter/src/storage/vault_storage.rs b/jupiter/src/storage/vault_storage.rs index 060420aaf..ecdae8f8c 100644 --- a/jupiter/src/storage/vault_storage.rs +++ b/jupiter/src/storage/vault_storage.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use callisto::vault::*; use common::errors::MegaError; use sea_orm::*; +use sea_orm_migration::prelude::OnConflict; #[derive(Clone)] pub struct VaultStorage { @@ -26,8 +27,8 @@ impl VaultStorage { pub async fn list_keys(&self, prefix: impl AsRef) -> Result, MegaError> { Entity::find() - .order_by_asc(Column::Key.like(format!("{}%", prefix.as_ref()).as_str())) .select_column(Column::Key) + .filter(Column::Key.like(format!("{}%", prefix.as_ref()).as_str())) .into_tuple::() .all(self.get_connection()) .await @@ -40,25 +41,30 @@ impl VaultStorage { }) } - pub async fn load(&self, key: impl AsRef) -> Result { - Entity::find() + pub async fn load(&self, key: impl AsRef) -> Result, MegaError> { + let found = Entity::find() .filter(Column::Key.eq(key.as_ref())) .one(self.get_connection()) - .await? - .ok_or_else(|| { - MegaError::with_message(format!("Vault key '{}' not found", key.as_ref()).as_str()) - }) + .await?; + Ok(found) } pub async fn save(&self, key: impl AsRef, value: Vec) -> Result<(), MegaError> { - let model = Model { - id: 0, - key: key.as_ref().to_string(), - value, - } - .into_active_model(); + let model = ActiveModel { + id: NotSet, + key: Set(key.as_ref().to_string()), + value: Set(value), + }; - match model.save(self.get_connection()).await { + match Entity::insert(model) + .on_conflict( + OnConflict::column(Column::Key) + .update_column(Column::Value) + .to_owned(), + ) + .exec(self.get_connection()) + .await + { Ok(_) => Ok(()), Err(e) => Err(MegaError::with_message(format!( "Failed to save vault entry '{}': {}", diff --git a/jupiter/src/tests.rs b/jupiter/src/tests.rs new file mode 100644 index 000000000..bb7736feb --- /dev/null +++ b/jupiter/src/tests.rs @@ -0,0 +1,64 @@ +use sea_orm::{ConnectOptions, Database, DatabaseConnection}; +use tempfile::TempDir; +use tracing::log; + +use std::sync::{Arc, LazyLock}; + +use common::config::Config; + +use crate::lfs_storage::local_storage::LocalStorage; +use crate::migrator::apply_migrations; +use crate::storage::{ + git_db_storage::GitDbStorage, issue_storage::IssueStorage, lfs_db_storage::LfsDbStorage, + mono_storage::MonoStorage, mq_storage::MQStorage, mr_storage::MrStorage, + raw_db_storage::RawDbStorage, relay_storage::RelayStorage, user_storage::UserStorage, + vault_storage::VaultStorage, +}; +use crate::storage::{Service, Storage}; + +pub async fn test_db_connection() -> (DatabaseConnection, TempDir) { + let temp_dir = TempDir::new().expect("Failed to create temporary directory"); + let db_url = format!("sqlite://{}/test.db", temp_dir.path().to_string_lossy()); + std::fs::File::create(temp_dir.path().join("test.db")) + .expect("Failed to create test database file"); + + let mut opt = ConnectOptions::new(db_url); + opt.max_connections(5) + .min_connections(1) + .sqlx_logging(true) + .sqlx_logging_level(log::LevelFilter::Debug); + + let db = Database::connect(opt) + .await + .expect("Failed to connect to mock database"); + + (db, temp_dir) +} + +pub async fn test_storage() -> Storage { + static CONFIG: LazyLock> = LazyLock::new(|| Config::mock().into()); + let (connection, _) = test_db_connection().await; + let connection = Arc::new(connection); + let lfs_db_storage = LfsDbStorage::new(connection.clone()).await; + let config = CONFIG.clone(); + + let svc = Service { + mono_storage: MonoStorage::new(connection.clone()).await, + git_db_storage: GitDbStorage::new(connection.clone()).await, + raw_db_storage: RawDbStorage::new(connection.clone()).await, + lfs_db_storage: lfs_db_storage.clone(), + relay_storage: RelayStorage::new(connection.clone()).await, + mq_storage: MQStorage::new(connection.clone()).await, + user_storage: UserStorage::new(connection.clone()).await, + mr_storage: MrStorage::new(connection.clone()).await, + issue_storage: IssueStorage::new(connection.clone()).await, + vault_storage: VaultStorage::new(connection.clone()).await, + lfs_file_storage: Arc::new(LocalStorage::mock()), // fix it until you really use it. + }; + + apply_migrations(&connection, true).await.unwrap(); + Storage { + services: Arc::new(svc), + config: Arc::downgrade(&config), + } +} diff --git a/mono/Cargo.toml b/mono/Cargo.toml index c9193e238..a3e6604eb 100644 --- a/mono/Cargo.toml +++ b/mono/Cargo.toml @@ -67,3 +67,4 @@ jemallocator = { workspace = true } mimalloc = { workspace = true } [dev-dependencies] +tempfile = { workspace = true} diff --git a/mono/src/server/https_server.rs b/mono/src/server/https_server.rs index 20053c092..a0852ff49 100644 --- a/mono/src/server/https_server.rs +++ b/mono/src/server/https_server.rs @@ -222,7 +222,9 @@ mod test { #[test] fn generate_swagger_json() { - let mut file = fs::File::create("gitmono.json").unwrap(); + let temp_dir = tempfile::tempdir().expect("Failed to create temporary directory"); + let temp_path = temp_dir.path().join("gitmono.json"); + let mut file = fs::File::create(temp_path).unwrap(); let json = ApiDoc::openapi().to_pretty_json().unwrap(); file.write_all(json.as_bytes()).unwrap(); println!("{}", json); diff --git a/vault/Cargo.toml b/vault/Cargo.toml index c7db024b9..8d01cec1e 100644 --- a/vault/Cargo.toml +++ b/vault/Cargo.toml @@ -19,7 +19,12 @@ tokio = { workspace = true, features = ["full"] } pgp = { workspace = true } smallvec = { workspace = true } + [dependencies.rusty_vault] git = "https://github.com/Tongsuo-Project/RustyVault" tag = "v0.2.2" features = ["sync_handler"] + +[dev-dependencies] +tempfile = { workspace = true } +rand = { workspace = true } diff --git a/vault/src/integration/jupiter_backend.rs b/vault/src/integration/jupiter_backend.rs index 99ddbd1a6..06b39f85a 100644 --- a/vault/src/integration/jupiter_backend.rs +++ b/vault/src/integration/jupiter_backend.rs @@ -4,25 +4,33 @@ use tokio::runtime::Handle; pub struct JupiterBackend { ctx: Storage, - rt: Handle, } impl JupiterBackend { pub fn new(ctx: Storage) -> Self { - let rt = tokio::runtime::Handle::current(); - JupiterBackend { ctx, rt } + JupiterBackend { ctx } } } impl Backend for JupiterBackend { fn list(&self, prefix: &str) -> Result, rusty_vault::errors::RvError> { let service = self.ctx.vault_storage(); - self.rt.block_on(async move { - match service.list_keys(prefix).await { - Ok(keys) => Ok(keys), - Err(_) => Err(rusty_vault::errors::RvError::ErrPhysicalBackendKeyInvalid), - } + let handle = Handle::current(); + let prefix = prefix.to_string(); + std::thread::spawn(move || { + // Using Handle::block_on to run async code in the new thread. + handle.block_on(async { + match service.list_keys(prefix).await { + Ok(keys) => Ok(keys), + Err(e) => { + println!("list {:?}", e); + Err(rusty_vault::errors::RvError::ErrAuthModuleDisabled) + } + } + }) }) + .join() + .unwrap() } fn get( @@ -30,18 +38,32 @@ impl Backend for JupiterBackend { key: &str, ) -> Result, rusty_vault::errors::RvError> { let service = self.ctx.vault_storage(); - self.rt.block_on(async move { - match service.load(key).await { - Ok(model) => { - let entry = rusty_vault::storage::BackendEntry { - key: model.key, - value: model.value, - }; - Ok(Some(entry)) + let handle = Handle::current(); + let key = key.to_string(); + std::thread::spawn(move || { + // Using Handle::block_on to run async code in the new thread. + handle.block_on(async move { + match service.load(key).await { + Ok(Some(model)) => { + let entry = rusty_vault::storage::BackendEntry { + key: model.key, + value: model.value, + }; + Ok(Some(entry)) + } + Ok(None) => Ok(None), + Err(e) => { + println!("get {:?}", e); + Err(rusty_vault::errors::RvError::ErrAuthModuleDisabled) + } } - Err(_) => Err(rusty_vault::errors::RvError::ErrPhysicalBackendKeyInvalid), - } + }) + }) + .join() + .inspect_err(|e| { + eprintln!("Error in JupiterBackend::get: {:?}", e); }) + .unwrap() } fn put( @@ -49,21 +71,41 @@ impl Backend for JupiterBackend { entry: &rusty_vault::storage::BackendEntry, ) -> Result<(), rusty_vault::errors::RvError> { let service = self.ctx.vault_storage(); - self.rt.block_on(async move { - match service.save(&entry.key, entry.value.clone()).await { - Ok(_) => Ok(()), - Err(_) => Err(rusty_vault::errors::RvError::ErrPhysicalBackendKeyInvalid), - } + let handle = Handle::current(); + let entry_clone = entry.clone(); + std::thread::spawn(move || { + // Using Handle::block_on to run async code in the new thread. + handle.block_on(async move { + match service.save(entry_clone.key, entry_clone.value).await { + Ok(_) => Ok(()), + Err(e) => { + println!("put {:?}", e); + Err(rusty_vault::errors::RvError::ErrAuthModuleDisabled) + } + } + }) }) + .join() + .unwrap() } fn delete(&self, key: &str) -> Result<(), rusty_vault::errors::RvError> { let service = self.ctx.vault_storage(); - self.rt.block_on(async move { - match service.delete(key).await { - Ok(_) => Ok(()), - Err(_) => Err(rusty_vault::errors::RvError::ErrPhysicalBackendKeyInvalid), - } + let handle = Handle::current(); + let key = key.to_string(); + std::thread::spawn(move || { + // Using Handle::block_on to run async code in the new thread. + handle.block_on(async move { + match service.delete(key).await { + Ok(_) => Ok(()), + Err(e) => { + println!("delete {:?}", e); + Err(rusty_vault::errors::RvError::ErrAuthModuleDisabled) + } + } + }) }) + .join() + .unwrap() } } diff --git a/vault/src/integration/vault_core.rs b/vault/src/integration/vault_core.rs index 99193941c..14c4b4305 100644 --- a/vault/src/integration/vault_core.rs +++ b/vault/src/integration/vault_core.rs @@ -1,4 +1,7 @@ -use std::sync::{Arc, RwLock}; +use std::{ + path::PathBuf, + sync::{Arc, RwLock}, +}; use crate::integration::jupiter_backend::JupiterBackend; use common::errors::MegaError; @@ -50,7 +53,17 @@ impl VaultCore { let key_path = dir.join(CORE_KEY_FILE); std::fs::create_dir_all(&dir).expect("Failed to create vault directory"); + Self::config(ctx, key_path) + } + pub async fn mock(key_path: PathBuf) -> Self { + std::fs::create_dir_all(key_path.parent().unwrap()) + .expect("Failed to create mock vault directory"); + let storage = jupiter::tests::test_storage().await; + Self::config(storage, key_path) + } + + fn config(ctx: Storage, key_path: PathBuf) -> Self { let backend: Arc = Arc::new(JupiterBackend::new(ctx)); let barrier = barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend)); let seal_config = rusty_vault::core::SealConfig { @@ -71,10 +84,7 @@ impl VaultCore { .config(core.clone(), None) .expect("Failed to configure vault core"); - let core_key = if !managed_core - .inited() - .expect("Failed to check if vault is initialized") - { + let core_key = if !key_path.exists() { let result = managed_core .init(&seal_config) .expect("Failed to initialize vault"); @@ -87,6 +97,7 @@ impl VaultCore { core_key } else { + println!("Using existing vault core key file: {}", key_path.display()); let key_data = std::fs::read(&key_path).expect("Failed to read vault core key file"); serde_json::from_slice::(&key_data) @@ -96,7 +107,7 @@ impl VaultCore { for i in 0..seal_config.secret_threshold { let key = &core_key.secret_shares[i as usize]; let unseal = managed_core.unseal(key); - assert!(unseal.is_ok()); + assert!(unseal.is_ok(), "Unseal error: {:?}", unseal.err()); } log::debug!( @@ -171,3 +182,90 @@ impl VaultCoreInterface for VaultCore { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn test_vault_core_initialization() { + let temp_dir = tempfile::tempdir().expect("Failed to create temporary directory"); + let key_path = temp_dir.path().join(CORE_KEY_FILE); + let vault_core = VaultCore::mock(key_path).await; + + assert!( + !vault_core.token().is_empty(), + "Vault core token should not be empty" + ); + assert!( + vault_core.core.read().unwrap().inited().unwrap(), + "Vault core should be initialized" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn test_vault_api() { + let temp_dir = tempfile::tempdir().expect("Failed to create temporary directory"); + let key_path = temp_dir.path().join(CORE_KEY_FILE); + let vault_core = VaultCore::mock(key_path).await; + + let random_pairs = (0..1024) + .map(|_| { + ( + rand::random::().to_string(), + rand::random::().to_string(), + ) + }) + .collect::>(); + let data: HashMap> = random_pairs + .into_iter() + .map(|(k, v)| { + ( + k, + serde_json::json!({ + "data": v, + }) + .as_object() + .unwrap() + .clone(), + ) + }) + .collect(); + + // Write secrets to the vault and store them in a map + for (name, value) in &data { + vault_core + .write_secret(name.as_str(), Some(value.clone())) + .expect("Failed to write secret"); + } + + // Read secrets from the vault and verify their values + for (name, value) in &data { + let read_value = vault_core + .read_secret(name.as_str()) + .expect("Failed to read secret") + .expect("Secret should exist"); + assert_eq!( + read_value, *value, + "Read value does not match written value for {}", + name + ); + } + + // Delete secrets from the vault and verify they are removed + for name in data.keys() { + vault_core + .delete_secret(name.as_str()) + .expect("Failed to delete secret"); + + let read_value = vault_core.read_secret(name.as_str()); + assert!(read_value.is_ok()); + assert!( + read_value.unwrap().is_none(), + "Secret {} should be deleted but still exists", + name + ); + } + } +} From bbb98784220a20bc92569347d84e743c95b2fd9b Mon Sep 17 00:00:00 2001 From: Neon Date: Tue, 24 Jun 2025 22:14:41 +0800 Subject: [PATCH 16/18] config: add scorpio to workspace --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 751e8ed6f..82b2e37a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "orion", "orion-server", "observatory", + "scorpio", "context", ] default-members = ["mega", "mono", "libra", "aries", "orion", "orion-server"] From 7ef3710862588e0578d0a7a1882c712166410a5e Mon Sep 17 00:00:00 2001 From: Neon Date: Wed, 25 Jun 2025 14:53:10 +0800 Subject: [PATCH 17/18] fix(jupiter): IO error in test --- jupiter/src/migrator/mod.rs | 3 ++- jupiter/src/tests.rs | 18 +++++++++--------- vault/src/integration/vault_core.rs | 17 +++++++---------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/jupiter/src/migrator/mod.rs b/jupiter/src/migrator/mod.rs index 2409f7556..22e47b5bf 100644 --- a/jupiter/src/migrator/mod.rs +++ b/jupiter/src/migrator/mod.rs @@ -111,7 +111,8 @@ mod tests { #[tokio::test] async fn test_apply_migrations() { - let (db, _) = test_db_connection().await; + let temp_dir = tempfile::TempDir::new().expect("Failed to create temporary directory"); + let db = test_db_connection(temp_dir.path()).await; // Apply migrations to the mock database let result = apply_migrations(&db, false).await; assert!( diff --git a/jupiter/src/tests.rs b/jupiter/src/tests.rs index bb7736feb..a45a176e5 100644 --- a/jupiter/src/tests.rs +++ b/jupiter/src/tests.rs @@ -1,7 +1,7 @@ use sea_orm::{ConnectOptions, Database, DatabaseConnection}; -use tempfile::TempDir; use tracing::log; +use std::path::Path; use std::sync::{Arc, LazyLock}; use common::config::Config; @@ -16,10 +16,9 @@ use crate::storage::{ }; use crate::storage::{Service, Storage}; -pub async fn test_db_connection() -> (DatabaseConnection, TempDir) { - let temp_dir = TempDir::new().expect("Failed to create temporary directory"); - let db_url = format!("sqlite://{}/test.db", temp_dir.path().to_string_lossy()); - std::fs::File::create(temp_dir.path().join("test.db")) +pub async fn test_db_connection(temp_dir: impl AsRef) -> DatabaseConnection { + let db_url = format!("sqlite://{}/test.db", temp_dir.as_ref().to_string_lossy()); + std::fs::File::create(temp_dir.as_ref().join("test.db")) .expect("Failed to create test database file"); let mut opt = ConnectOptions::new(db_url); @@ -32,12 +31,12 @@ pub async fn test_db_connection() -> (DatabaseConnection, TempDir) { .await .expect("Failed to connect to mock database"); - (db, temp_dir) + db } -pub async fn test_storage() -> Storage { +pub async fn test_storage(temp_dir: impl AsRef) -> Storage { static CONFIG: LazyLock> = LazyLock::new(|| Config::mock().into()); - let (connection, _) = test_db_connection().await; + let connection = test_db_connection(temp_dir).await; let connection = Arc::new(connection); let lfs_db_storage = LfsDbStorage::new(connection.clone()).await; let config = CONFIG.clone(); @@ -53,10 +52,11 @@ pub async fn test_storage() -> Storage { mr_storage: MrStorage::new(connection.clone()).await, issue_storage: IssueStorage::new(connection.clone()).await, vault_storage: VaultStorage::new(connection.clone()).await, - lfs_file_storage: Arc::new(LocalStorage::mock()), // fix it until you really use it. + lfs_file_storage: Arc::new(LocalStorage::mock()), // fix it when you really use it. }; apply_migrations(&connection, true).await.unwrap(); + Storage { services: Arc::new(svc), config: Arc::downgrade(&config), diff --git a/vault/src/integration/vault_core.rs b/vault/src/integration/vault_core.rs index 14c4b4305..fa0eabec4 100644 --- a/vault/src/integration/vault_core.rs +++ b/vault/src/integration/vault_core.rs @@ -56,13 +56,6 @@ impl VaultCore { Self::config(ctx, key_path) } - pub async fn mock(key_path: PathBuf) -> Self { - std::fs::create_dir_all(key_path.parent().unwrap()) - .expect("Failed to create mock vault directory"); - let storage = jupiter::tests::test_storage().await; - Self::config(storage, key_path) - } - fn config(ctx: Storage, key_path: PathBuf) -> Self { let backend: Arc = Arc::new(JupiterBackend::new(ctx)); let barrier = barrier_aes_gcm::AESGCMBarrier::new(Arc::clone(&backend)); @@ -185,6 +178,8 @@ impl VaultCoreInterface for VaultCore { #[cfg(test)] mod tests { + use jupiter::tests::test_storage; + use super::*; use std::collections::HashMap; @@ -192,7 +187,8 @@ mod tests { async fn test_vault_core_initialization() { let temp_dir = tempfile::tempdir().expect("Failed to create temporary directory"); let key_path = temp_dir.path().join(CORE_KEY_FILE); - let vault_core = VaultCore::mock(key_path).await; + let storage = test_storage(temp_dir.path()).await; + let vault_core = VaultCore::config(storage, key_path); assert!( !vault_core.token().is_empty(), @@ -208,9 +204,10 @@ mod tests { async fn test_vault_api() { let temp_dir = tempfile::tempdir().expect("Failed to create temporary directory"); let key_path = temp_dir.path().join(CORE_KEY_FILE); - let vault_core = VaultCore::mock(key_path).await; + let storage = test_storage(temp_dir.path()).await; + let vault_core = VaultCore::config(storage, key_path); - let random_pairs = (0..1024) + let random_pairs = (0..128) .map(|_| { ( rand::random::().to_string(), From b0a0bf773f90e94f817cf0d2568f39492b74b8f3 Mon Sep 17 00:00:00 2001 From: Neon <71858127+yyk808@users.noreply.github.com> Date: Wed, 25 Jun 2025 17:16:39 +0800 Subject: [PATCH 18/18] Update vault/src/integration/vault_core.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- vault/src/integration/vault_core.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vault/src/integration/vault_core.rs b/vault/src/integration/vault_core.rs index fa0eabec4..c4dd33345 100644 --- a/vault/src/integration/vault_core.rs +++ b/vault/src/integration/vault_core.rs @@ -127,7 +127,7 @@ impl VaultCoreInterface for VaultCore { let guard = self.core.read().unwrap(); guard .handle_request(&mut req) - .map_err(|_| MegaError::with_message("Failed to read from vault API")) + .map_err(|e| MegaError::with_message(format!("Failed to read from vault API: {}", e))) } fn write_api(