Skip to content

add test authentication user #699

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
genedna merged 2 commits into from
Nov 21, 2024
Merged

add test authentication user #699

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
28ebe55
add test authentication user
benjamin-747 Nov 21, 2024
aad0e26
update package.json
benjamin-747 Nov 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,19 @@ mega = { path = "mega" }
mono = { path = "mono" }
libra = { path = "libra" }

anyhow = "1.0.90"
serde = "1.0.214"
anyhow = "1.0.93"
serde = "1.0.215"
serde_json = "1.0.132"
tracing = "0.1.40"
tracing-subscriber = "0.3.18"
tracing-appender = "0.2"
thiserror = "2.0.0"
thiserror = "2.0.3"
rand = "0.8.5"
smallvec = "1.13.2"
tokio = "1.41.0"
tokio = "1.41.1"
tokio-stream = "0.1.16"
tokio-test = "0.4.4"
clap = "4.5.20"
clap = "4.5.21"
async-trait = "0.1.83"
async-stream = "0.3.6"
bytes = "1.8.0"
Expand All @@ -66,8 +66,8 @@ tower-http = "0.6.1"
tower = "0.5.1"
hex = "0.4.3"
sea-orm = "1.1.1"
flate2 = "1.0.34"
bstr = "1.10.0"
flate2 = "1.0.35"
bstr = "1.11.0"
colored = "2.1.0"
idgenerator = "2.0.0"
num_cpus = "1.16.0"
Expand All @@ -80,10 +80,10 @@ regex = "1.11.1"
ed25519-dalek = "2.1.1"
ctrlc = "3.4.4"
git2 = "0.19.0"
tempfile = "3.13.0"
tempfile = "3.14.0"
home = "0.5.9"
ring = "0.17.8"
cedar-policy = "4.2.1"
cedar-policy = "4.2.2"
secp256k1 = "0.30.0"
oauth2 = "4.4.2"
base64 = "0.22.1"
Expand Down
9 changes: 7 additions & 2 deletions ceres/src/pack/import_repo.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ impl PackHandler for ImportRepo {
self.find_head_hash(refs)
}

async fn handle_receiver(&self, receiver: Receiver<Entry>) -> Result<(), GitError> {
async fn handle_receiver(&self, receiver: Receiver<Entry>) -> Result<String, GitError> {
let storage = self.context.services.git_db_storage.clone();
let mut entry_list = vec![];
let mut join_tasks = vec![];
Expand All @@ -80,7 +80,7 @@ impl PackHandler for ImportRepo {
.await
.unwrap();
self.attach_to_monorepo_parent().await.unwrap();
Ok(())
Ok(String::new())
}

async fn full_pack(&self) -> Result<ReceiverStream<Vec<u8>>, GitError> {
Expand Down Expand Up @@ -294,6 +294,11 @@ impl PackHandler for ImportRepo {
.await
}

async fn handle_mr(&self, _: &str) -> Result<(), GitError> {
// do nothing in import_repo
Ok(())
}

async fn update_refs(&self, refs: &RefCommand) -> Result<(), GitError> {
let storage = self.context.services.git_db_storage.clone();
match refs.command_type {
Expand Down
4 changes: 3 additions & 1 deletion ceres/src/pack/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ pub mod monorepo;
pub trait PackHandler: Send + Sync {
async fn head_hash(&self) -> (String, Vec<Refs>);

async fn handle_receiver(&self, rx: Receiver<Entry>) -> Result<(), GitError>;
async fn handle_receiver(&self, rx: Receiver<Entry>) -> Result<String, GitError>;

/// Asynchronously retrieves the full pack data for the specified repository path.
/// This function collects commits and nodes from the storage and packs them into
Expand All @@ -63,6 +63,8 @@ pub trait PackHandler: Send + Sync {
hashes: Vec<String>,
) -> Result<Vec<raw_blob::Model>, MegaError>;

async fn handle_mr(&self, title: &str) -> Result<(), GitError>;

async fn update_refs(&self, refs: &RefCommand) -> Result<(), GitError>;

async fn check_commit_exist(&self, hash: &str) -> bool;
Expand Down
132 changes: 68 additions & 64 deletions ceres/src/pack/monorepo.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@ use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;

use callisto::{db_enums::ConvType, raw_blob};
use common::{errors::MegaError, utils::{self, MEGA_BRANCH_NAME}};
use common::{
errors::MegaError,
utils::{self, MEGA_BRANCH_NAME},
};
use jupiter::{context::Context, storage::mr_storage::MrStorage};
use mercury::internal::{object::ObjectTrait, pack::encode::PackEncoder};
use mercury::{
Expand Down Expand Up @@ -119,33 +122,43 @@ impl PackHandler for MonoRepo {
self.find_head_hash(refs)
}

async fn handle_receiver(&self, receiver: Receiver<Entry>) -> Result<(), GitError> {
let storage = self.context.mr_stg();
let path_str = self.path.to_str().unwrap();

let unpack_res = self.save_entry(receiver).await;
match unpack_res {
Ok(title) => match storage.get_open_mr_by_path(path_str).await.unwrap() {
Some(mr) => {
let mut mr = mr.into();
self.handle_existing_mr(&mut mr, &storage).await
async fn handle_receiver(&self, receiver: Receiver<Entry>) -> Result<String, GitError> {
let storage = self.context.services.mono_storage.clone();
let mut entry_list = Vec::new();
let mut join_tasks = vec![];
let mut current_commit_id = String::new();
let mut mr_title = String::new();
for entry in receiver {
if current_commit_id.is_empty() {
if entry.obj_type == ObjectType::Commit {
current_commit_id = entry.hash.to_plain_str();
let commit = Commit::from_bytes(&entry.data, entry.hash).unwrap();
mr_title = commit.format_message();
}
} else {
if entry.obj_type == ObjectType::Commit {
return Err(GitError::CustomError(
"only single commit support in each push".to_string(),
));
}
None => {
let link: String = utils::generate_link();
let mr = MergeRequest {
path: path_str.to_owned(),
from_hash: self.from_hash.clone(),
to_hash: self.to_hash.clone(),
link,
title,
..Default::default()
};
storage.save_mr(mr.clone().into()).await.unwrap();
Ok(())
if entry_list.len() >= 1000 {
let stg_clone = storage.clone();
let commit_id = current_commit_id.clone();
let handle = tokio::spawn(async move {
stg_clone.save_entry(&commit_id, entry_list).await.unwrap();
});
join_tasks.push(handle);
entry_list = vec![];
}
},
Err(err) => Err(err),
}
entry_list.push(entry);
}
join_all(join_tasks).await;
storage
.save_entry(&current_commit_id, entry_list)
.await
.unwrap();
Ok(mr_title)
}

// monorepo full pack should follow the shallow clone command 'git clone --depth=1'
Expand Down Expand Up @@ -305,6 +318,36 @@ impl PackHandler for MonoRepo {
.await
}

async fn handle_mr(&self, title: &str) -> Result<(), GitError> {
let storage = self.context.mr_stg();
let path_str = self.path.to_str().unwrap();

match storage.get_open_mr_by_path(path_str).await.unwrap() {
Some(mr) => {
let mut mr = mr.into();
self.handle_existing_mr(&mut mr, &storage).await
}
None => {
if self.from_hash == "0".repeat(40) {
return Err(GitError::CustomError(String::from(
"Can not init directory under monorepo directory!",
)));
}
let link: String = utils::generate_link();
let mr = MergeRequest {
path: path_str.to_owned(),
from_hash: self.from_hash.clone(),
to_hash: self.to_hash.clone(),
link,
title: title.to_string(),
..Default::default()
};
storage.save_mr(mr.clone().into()).await.unwrap();
Ok(())
}
}
}

async fn update_refs(&self, _: &RefCommand) -> Result<(), GitError> {
//do nothing in monorepo because we use mr to handle refs update
Ok(())
Expand Down Expand Up @@ -366,43 +409,4 @@ impl MonoRepo {
&to[..6]
)
}

async fn save_entry(&self, receiver: Receiver<Entry>) -> Result<String, GitError> {
let storage = self.context.services.mono_storage.clone();
let mut entry_list = Vec::new();
let mut join_tasks = vec![];
let mut current_commit_id = String::new();
let mut mr_title = String::new();
for entry in receiver {
if current_commit_id.is_empty() {
if entry.obj_type == ObjectType::Commit {
current_commit_id = entry.hash.to_plain_str();
let commit = Commit::from_bytes(&entry.data, entry.hash).unwrap();
mr_title = commit.format_message();
}
} else {
if entry.obj_type == ObjectType::Commit {
return Err(GitError::CustomError(
"only single commit support in each push".to_string(),
));
}
if entry_list.len() >= 1000 {
let stg_clone = storage.clone();
let commit_id = current_commit_id.clone();
let handle = tokio::spawn(async move {
stg_clone.save_entry(&commit_id, entry_list).await.unwrap();
});
join_tasks.push(handle);
entry_list = vec![];
}
}
entry_list.push(entry);
}
join_all(join_tasks).await;
storage
.save_entry(&current_commit_id, entry_list)
.await
.unwrap();
Ok(mr_title)
}
}
5 changes: 4 additions & 1 deletion ceres/src/protocol/smart.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,10 @@ impl SmartProtocol {
// b.The reference being pushed could be a non-fast-forward reference and the update hooks or configuration could be set to not allow that, etc.
// c.Also, some references can be updated while others can be rejected.
match unpack_result {
Ok(_) => {
Ok(ref title) => {
if let Err(e) = pack_handler.handle_mr(title).await {
command.failed(e.to_string());
}
if !default_exist {
command.default_branch = true;
default_exist = true;
Expand Down
22 changes: 20 additions & 2 deletions common/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ pub struct Config {
pub storage: StorageConfig,
pub monorepo: MonoConfig,
pub pack: PackConfig,
pub authentication: AuthConfig,
pub lfs: LFSConfig,
// Not used in mega app
#[serde(default)]
Expand Down Expand Up @@ -209,7 +210,6 @@ impl Default for StorageConfig {
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct MonoConfig {
pub import_dir: PathBuf,
pub enable_http_auth: bool,
pub admin: String,
pub root_dirs: Vec<String>,
}
Expand All @@ -218,7 +218,6 @@ impl Default for MonoConfig {
fn default() -> Self {
Self {
import_dir: PathBuf::from("/third-part"),
enable_http_auth: false,
admin: String::from("admin"),
root_dirs: vec![
"third-part".to_string(),
Expand All @@ -230,6 +229,25 @@ impl Default for MonoConfig {
}
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct AuthConfig {
pub enable_http_auth: bool,
pub enable_test_user: bool,
pub test_user_name: String,
pub test_user_token: String,
}

impl Default for AuthConfig {
fn default() -> Self {
Self {
enable_http_auth: false,
enable_test_user: false,
test_user_name: String::from("mega"),
test_user_token: String::from("mega")
}
}
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct PackConfig {
pub pack_decode_mem_size: usize,
Expand Down
18 changes: 15 additions & 3 deletions docker/config.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,27 @@ obs_region = "cn-east-3"
# Override the endpoint URL used for remote storage services
obs_endpoint = "https://obs.cn-east-3.myhuaweicloud.com"

[authentication]
# Support http authentication, login in with github and generate token before push
enable_http_auth = false

# Enable a test user for debugging and development purposes.
# If set to true, the service allows using a predefined test user for authentication.
enable_test_user = true

# Specify the name of the test user.
# This is only relevant if `enable_test_user` is set to true.
test_user_name = "mega"

# Specify the token for the test user.
# This is used for authentication when `enable_test_user` is set to true.
test_user_token = "mega"

[monorepo]
## Only import directory support multi-branch commit and tag, monorepo only support main branch
## Mega treats files under this directory as import repo and other directories as monorepo
import_dir = "/third-part"

# Support http authentication, login in with github and generate token before push
enable_http_auth = false

# Set System Admin in directory init, replace the admin's github username here
admin = "admin"

Expand Down
18 changes: 15 additions & 3 deletions mega/config.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,27 @@ obs_region = "cn-east-3"
# Override the endpoint URL used for remote storage services
obs_endpoint = "https://obs.cn-east-3.myhuaweicloud.com"

[authentication]
# Support http authentication, login in with github and generate token before push
enable_http_auth = false

# Enable a test user for debugging and development purposes.
# If set to true, the service allows using a predefined test user for authentication.
enable_test_user = true

# Specify the name of the test user.
# This is only relevant if `enable_test_user` is set to true.
test_user_name = "mega"

# Specify the token for the test user.
# This is used for authentication when `enable_test_user` is set to true.
test_user_token = "mega"

[monorepo]
## Only import directory support multi-branch commit and tag, monorepo only support main branch
## Mega treats files under this directory as import repo and other directories as monorepo
import_dir = "/third-part"

# Support http authentication, login in with github and generate token before push
enable_http_auth = false

# Set System Admin in directory init, replace the admin's github username here
admin = "admin"

Expand Down
Loading