Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions common/src/config.rs
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
//! Configuration management for the Mono and Mega application
//! This module provides functionality to load, parse, and manage configuration settings

use std::rc::Rc;
use std::path::PathBuf;
use std::cell::RefCell;
use std::collections::HashMap;
use std::path::PathBuf;
use std::rc::Rc;

pub use config as c;
use c::{ConfigError, FileFormat};
pub use config as c;

use config::{Source, ValueKind};
use config::builder::DefaultState;
use config::{Source, ValueKind};
use serde::{Deserialize, Deserializer, Serialize};

use callisto::sea_orm_active_enums::StorageTypeEnum;
Expand Down Expand Up @@ -45,7 +45,7 @@ pub fn mega_base() -> PathBuf {
.unwrap()
.to_string()
});

PathBuf::from(base_dir)
}

Expand Down Expand Up @@ -77,7 +77,7 @@ pub fn mega_cache() -> PathBuf {
.unwrap()
.to_string()
});

PathBuf::from(cache_dir)
}

Expand All @@ -104,7 +104,7 @@ impl Config {
.prefix_separator("_")
.separator("__"),
);

let config = variable_placeholder_substitute(builder);

Config::from_config(config)
Expand Down
4 changes: 1 addition & 3 deletions jupiter/migration/src/m20250613_033821_alter_user_id.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,7 @@ impl MigrationTrait for Migration {
.await?;
}

DatabaseBackend::Sqlite => {

}
DatabaseBackend::Sqlite => {}
}

Ok(())
Expand Down
11 changes: 8 additions & 3 deletions mega/tests/service_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,15 @@ use serial_test::serial;
#[serial]
async fn check_mono_service_status() -> Result<(), reqwest::Error> {
let client = reqwest::Client::new();
let response = client.get("http://127.0.0.1:8000/api/v1/status")
let response = client
.get("http://127.0.0.1:8000/api/v1/status")
.send()
.await?;

assert_eq!(response.status(), reqwest::StatusCode::OK, "Service status API did not return 200 OK");

assert_eq!(
response.status(),
reqwest::StatusCode::OK,
"Service status API did not return 200 OK"
);
Ok(())
}
8 changes: 1 addition & 7 deletions mercury/delta/src/decode/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,7 @@ pub fn delta_decode(
Ok([instruction]) => instruction,
Err(err) if err.kind() == ErrorKind::UnexpectedEof => break,
Err(err) => {
panic!(
"{}",
GitDeltaError::DeltaDecoderError(format!(
"Wrong instruction in delta :{}",
err
))
);
panic!("{}", format!("Wrong instruction in delta :{err}"));
}
};

Expand Down
5 changes: 1 addition & 4 deletions mercury/zstdelta/src/zstdelta.rs
Original file line number Diff line number Diff line change
Expand Up @@ -148,10 +148,7 @@ pub fn apply(base: &[u8], delta: &[u8]) -> io::Result<Vec<u8>> {
let msg = format!("cannot decompress ({})", explain_error(outsize));
Err(io::Error::other(msg))
} else if outsize != size {
let msg = format!(
"decompress size mismatch (expected {}, got {})",
size, outsize
);
let msg = format!("decompress size mismatch (expected {size}, got {outsize})");
Err(io::Error::other(msg))
} else {
Ok(buf)
Expand Down
176 changes: 176 additions & 0 deletions scorpio/doc/perf_test.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
```sh
#生成测试文件和目录,生成time_和时间相关的目录,/tmp/time_
script/run.sh

#使用rust代码进行访问文件和目录
rustc script/run.rs
#测试系统文件系统访问速度
script/run /tmp/time_
```

普通文件系统测试结果:

```sh
(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /tmp/test_202506162053/1/2/3
已切换到目录: /tmp/test_202506162053/1/2/3
当前工作目录: /tmp/test_202506162053/1/2/3
path "./1M_1_3.bin" true
path "./4" false
path "./4/1M_1_4.bin" true
path "./4/5" false
path "./4/5/1M_1_5.bin" true
path "./4/5/6" false
path "./4/5/6/1M_1_6.bin" true
共找到 4 个文件
./1M_1_3.bin
./4/1M_1_4.bin
./4/5/1M_1_5.bin
./4/5/6/1M_1_6.bin

===== 性能统计 =====
文件数量: 4
cd 加载目录: 2.815µs
总读取字节数: 4.00 MB
Stat 操作时间: 9.318µs
文件读取时间: 2.259ms
Stat 操作速率: 429276.67 文件/秒
读取吞吐量: 1770.36 MB/秒

(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /tmp/test_202506162053/2/2/3
已切换到目录: /tmp/test_202506162053/2/2/3
当前工作目录: /tmp/test_202506162053/2/2/3
path "./4" false
path "./4/5" false
path "./4/5/1M_2_5.bin" true
path "./4/5/6" false
path "./4/5/6/1M_2_6.bin" true
path "./4/1M_2_4.bin" true
path "./1M_2_3.bin" true
共找到 4 个文件
./4/5/1M_2_5.bin
./4/5/6/1M_2_6.bin
./4/1M_2_4.bin
./1M_2_3.bin

===== 性能统计 =====
文件数量: 4
cd 加载目录: 4.789µs
总读取字节数: 4.00 MB
Stat 操作时间: 7.734µs
文件读取时间: 1.836ms
Stat 操作速率: 517196.79 文件/秒
读取吞吐量: 2178.56 MB/秒
```

fuse文件系统性能测试:

```sh
#需要清空scorpio.toml里面的store文件夹的内容
#建议不要更改scorpio.toml里面的load_dir_depth参数,否则要修改创建文件夹深度和访问深度
#由于控制了加载深度,所以下面的测试不会访问到./4/5/6/1M_2_6.bin
#启动mono服务,推送/tmp/test_time_文件夹到mono
#scorpio下 cargo run
#测试首次加载的性能
script/run /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3
script/run /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3

(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3
已切换到目录: /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3
当前工作目录: /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3
path "./4" false
path "./4/5" false
path "./4/5/6" false
path "./4/5/1M_1_5.bin" true
path "./4/1M_1_4.bin" true
path "./1M_1_3.bin" true
共找到 3 个文件
./4/5/1M_1_5.bin
./4/1M_1_4.bin
./1M_1_3.bin

===== 性能统计 =====
文件数量: 3
cd 加载目录: 299.299ms
总读取字节数: 3.00 MB
Stat 操作时间: 16.010µs
文件读取时间: 17.573ms
Stat 操作速率: 187382.89 文件/秒
读取吞吐量: 170.72 MB/秒
(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3
已切换到目录: /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3
当前工作目录: /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3
path "./4" false
path "./4/5" false
path "./4/5/6" false
path "./4/5/1M_2_5.bin" true
path "./4/1M_2_4.bin" true
path "./1M_2_3.bin" true
共找到 3 个文件
./4/5/1M_2_5.bin
./4/1M_2_4.bin
./1M_2_3.bin

===== 性能统计 =====
文件数量: 3
cd 加载目录: 250.112ms
总读取字节数: 3.00 MB
Stat 操作时间: 14.346µs
文件读取时间: 22.745ms
Stat 操作速率: 209117.52 文件/秒
读取吞吐量: 131.90 MB/秒

总结:初次加载目录和文件内容,网络开销大

#测试加载缓存以后读取的性能
script/run /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3
script/run /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3

(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3
已切换到目录: /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3
当前工作目录: /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3
path "./4" false
path "./4/5" false
path "./4/5/6" false
path "./4/5/1M_1_5.bin" true
path "./4/1M_1_4.bin" true
path "./1M_1_3.bin" true
共找到 3 个文件
./4/5/1M_1_5.bin
./4/1M_1_4.bin
./1M_1_3.bin

===== 性能统计 =====
文件数量: 3
cd 加载目录: 36.546ms
总读取字节数: 3.00 MB
Stat 操作时间: 10.790µs
文件读取时间: 11.981ms
Stat 操作速率: 278035.22 文件/秒
读取吞吐量: 250.40 MB/秒

(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3
已切换到目录: /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3
当前工作目录: /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3
path "./4" false
path "./4/5" false
path "./4/5/6" false
path "./4/5/1M_2_5.bin" true
path "./4/1M_2_4.bin" true
path "./1M_2_3.bin" true
共找到 3 个文件
./4/5/1M_2_5.bin
./4/1M_2_4.bin
./1M_2_3.bin

===== 性能统计 =====
文件数量: 3
cd 加载目录: 39.063ms
总读取字节数: 3.00 MB
Stat 操作时间: 15.379µs
文件读取时间: 20.243ms
Stat 操作速率: 195071.20 文件/秒
读取吞吐量: 148.20 MB/秒

总结:cd切换文件夹路径,如果文件夹内容没有变化,也至少有一次网络开销,但实际上,目录变化不会那么频繁,所以不用每次都要进行同步
```

113 changes: 113 additions & 0 deletions scorpio/script/run.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
use std::env;
use std::fs::{self, File, Metadata};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::time::Instant;

fn collect_files_recursively(dir: &Path) -> Vec<PathBuf> {
if dir.file_name() == Some(std::ffi::OsStr::new(".git")) {
return Vec::new(); // 忽略 .git 目录
}
let mut files = Vec::new();
if let Ok(entries) = fs::read_dir(dir) {
for entry in entries.flatten() {
let path = entry.path();
println!("path {:?} {:?}", path, path.is_file());

if path.is_file() {
files.push(path);
} else if path.is_dir() {
files.extend(collect_files_recursively(&path));
}
}
}
files
}

fn main() {
// 解析命令行参数
let args: Vec<String> = env::args().collect();
if args.len() != 2 {
eprintln!("用法: {} <目录路径>", args[0]);
return;
}

let target_dir = Path::new(&args[1]);
if !target_dir.exists() {
eprintln!("错误:指定路径不存在");
return;
}
if !target_dir.is_dir() {
eprintln!("错误:指定路径不是目录");
return;
}

// 切换到目标目录
let start_cd = Instant::now();

if let Err(e) = env::set_current_dir(target_dir) {
eprintln!("错误:无法切换到目录 {}: {}", target_dir.display(), e);
return;
}
let duration_cd = start_cd.elapsed();

println!("已切换到目录: {}", target_dir.display());

// 获取当前工作目录(应该是我们刚切换到的目录)
let current_dir = match env::current_dir() {
Ok(dir) => dir,
Err(e) => {
eprintln!("错误:无法获取当前目录: {}", e);
return;
}
};

println!("当前工作目录: {}", current_dir.display());

// 从当前目录开始递归收集文件(使用相对路径 ".")
let files = collect_files_recursively(Path::new("."));

println!("共找到 {} 个文件", files.len());
for file in &files {
println!("{}", file.display());
}

// 测试 stat 时间
let start_stat = Instant::now();
let _stats: Vec<Option<Metadata>> = files
.iter()
.map(|f| fs::metadata(f).ok())
.collect();
let duration_stat = start_stat.elapsed();

// 测试文件读取时间
let start_read = Instant::now();
let mut total_bytes = 0u64;
for f in &files {
if let Ok(mut file) = File::open(f) {
let mut buffer = Vec::new();
if let Ok(bytes_read) = file.read_to_end(&mut buffer) {
total_bytes += bytes_read as u64;
}
}
}
let duration_read = start_read.elapsed();

// 输出统计信息
println!("\n===== 性能统计 =====");
println!("文件数量: {}", files.len());
println!("cd 加载目录: {:.3?}", duration_cd);

println!("总读取字节数: {:.2} MB", total_bytes as f64 / (1024.0 * 1024.0));
println!("Stat 操作时间: {:.3?}", duration_stat);
println!("文件读取时间: {:.3?}", duration_read);

if duration_stat.as_secs_f64() > 0.0 {
println!("Stat 操作速率: {:.2} 文件/秒", files.len() as f64 / duration_stat.as_secs_f64());
}

if duration_read.as_secs_f64() > 0.0 {
let throughput = (total_bytes as f64 / (1024.0 * 1024.0)) / duration_read.as_secs_f64();
println!("读取吞吐量: {:.2} MB/秒", throughput);
}
}
Loading
Loading