From a03317d00584e8ef86641a4922bc08c3a69d2adf Mon Sep 17 00:00:00 2001 From: yyjeqhc <1772413353@qq.com> Date: Mon, 16 Jun 2025 20:41:02 +0800 Subject: [PATCH] fix parse response from mono and add doc --- common/src/config.rs | 14 +- .../src/m20250613_033821_alter_user_id.rs | 4 +- mega/tests/service_test.rs | 11 +- mercury/delta/src/decode/mod.rs | 8 +- mercury/zstdelta/src/zstdelta.rs | 5 +- scorpio/doc/perf_test.md | 176 ++++++++++++++++++ scorpio/script/run.rs | 113 +++++++++++ scorpio/script/run.sh | 39 ++++ scorpio/src/dicfuse/store.rs | 28 ++- scorpio/src/scolfs/lfs.rs | 1 + 10 files changed, 360 insertions(+), 39 deletions(-) create mode 100644 scorpio/doc/perf_test.md create mode 100644 scorpio/script/run.rs create mode 100755 scorpio/script/run.sh diff --git a/common/src/config.rs b/common/src/config.rs index 480715838..e6f151a71 100644 --- a/common/src/config.rs +++ b/common/src/config.rs @@ -1,16 +1,16 @@ //! Configuration management for the Mono and Mega application //! This module provides functionality to load, parse, and manage configuration settings -use std::rc::Rc; -use std::path::PathBuf; use std::cell::RefCell; use std::collections::HashMap; +use std::path::PathBuf; +use std::rc::Rc; -pub use config as c; use c::{ConfigError, FileFormat}; +pub use config as c; -use config::{Source, ValueKind}; use config::builder::DefaultState; +use config::{Source, ValueKind}; use serde::{Deserialize, Deserializer, Serialize}; use callisto::sea_orm_active_enums::StorageTypeEnum; @@ -45,7 +45,7 @@ pub fn mega_base() -> PathBuf { .unwrap() .to_string() }); - + PathBuf::from(base_dir) } @@ -77,7 +77,7 @@ pub fn mega_cache() -> PathBuf { .unwrap() .to_string() }); - + PathBuf::from(cache_dir) } @@ -104,7 +104,7 @@ impl Config { .prefix_separator("_") .separator("__"), ); - + let config = variable_placeholder_substitute(builder); Config::from_config(config) diff --git a/jupiter/migration/src/m20250613_033821_alter_user_id.rs b/jupiter/migration/src/m20250613_033821_alter_user_id.rs index 43e1780f0..090a1d00a 100644 --- a/jupiter/migration/src/m20250613_033821_alter_user_id.rs +++ b/jupiter/migration/src/m20250613_033821_alter_user_id.rs @@ -53,9 +53,7 @@ impl MigrationTrait for Migration { .await?; } - DatabaseBackend::Sqlite => { - - } + DatabaseBackend::Sqlite => {} } Ok(()) diff --git a/mega/tests/service_test.rs b/mega/tests/service_test.rs index dafca36d8..565d33029 100644 --- a/mega/tests/service_test.rs +++ b/mega/tests/service_test.rs @@ -4,10 +4,15 @@ use serial_test::serial; #[serial] async fn check_mono_service_status() -> Result<(), reqwest::Error> { let client = reqwest::Client::new(); - let response = client.get("http://127.0.0.1:8000/api/v1/status") + let response = client + .get("http://127.0.0.1:8000/api/v1/status") .send() .await?; - - assert_eq!(response.status(), reqwest::StatusCode::OK, "Service status API did not return 200 OK"); + + assert_eq!( + response.status(), + reqwest::StatusCode::OK, + "Service status API did not return 200 OK" + ); Ok(()) } diff --git a/mercury/delta/src/decode/mod.rs b/mercury/delta/src/decode/mod.rs index 4f555cd8f..c795690dd 100644 --- a/mercury/delta/src/decode/mod.rs +++ b/mercury/delta/src/decode/mod.rs @@ -26,13 +26,7 @@ pub fn delta_decode( Ok([instruction]) => instruction, Err(err) if err.kind() == ErrorKind::UnexpectedEof => break, Err(err) => { - panic!( - "{}", - GitDeltaError::DeltaDecoderError(format!( - "Wrong instruction in delta :{}", - err - )) - ); + panic!("{}", format!("Wrong instruction in delta :{err}")); } }; diff --git a/mercury/zstdelta/src/zstdelta.rs b/mercury/zstdelta/src/zstdelta.rs index 2a2f928f7..51b99b300 100644 --- a/mercury/zstdelta/src/zstdelta.rs +++ b/mercury/zstdelta/src/zstdelta.rs @@ -148,10 +148,7 @@ pub fn apply(base: &[u8], delta: &[u8]) -> io::Result> { let msg = format!("cannot decompress ({})", explain_error(outsize)); Err(io::Error::other(msg)) } else if outsize != size { - let msg = format!( - "decompress size mismatch (expected {}, got {})", - size, outsize - ); + let msg = format!("decompress size mismatch (expected {size}, got {outsize})"); Err(io::Error::other(msg)) } else { Ok(buf) diff --git a/scorpio/doc/perf_test.md b/scorpio/doc/perf_test.md new file mode 100644 index 000000000..cdf7b52e4 --- /dev/null +++ b/scorpio/doc/perf_test.md @@ -0,0 +1,176 @@ +```sh +#生成测试文件和目录,生成time_和时间相关的目录，/tmp/time_ +script/run.sh + +#使用rust代码进行访问文件和目录 +rustc script/run.rs +#测试系统文件系统访问速度 +script/run /tmp/time_ +``` + +普通文件系统测试结果： + +```sh +(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /tmp/test_202506162053/1/2/3 +已切换到目录: /tmp/test_202506162053/1/2/3 +当前工作目录: /tmp/test_202506162053/1/2/3 +path "./1M_1_3.bin" true +path "./4" false +path "./4/1M_1_4.bin" true +path "./4/5" false +path "./4/5/1M_1_5.bin" true +path "./4/5/6" false +path "./4/5/6/1M_1_6.bin" true +共找到 4 个文件 +./1M_1_3.bin +./4/1M_1_4.bin +./4/5/1M_1_5.bin +./4/5/6/1M_1_6.bin + +===== 性能统计 ===== +文件数量: 4 +cd 加载目录: 2.815µs +总读取字节数: 4.00 MB +Stat 操作时间: 9.318µs +文件读取时间: 2.259ms +Stat 操作速率: 429276.67 文件/秒 +读取吞吐量: 1770.36 MB/秒 + +(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /tmp/test_202506162053/2/2/3 +已切换到目录: /tmp/test_202506162053/2/2/3 +当前工作目录: /tmp/test_202506162053/2/2/3 +path "./4" false +path "./4/5" false +path "./4/5/1M_2_5.bin" true +path "./4/5/6" false +path "./4/5/6/1M_2_6.bin" true +path "./4/1M_2_4.bin" true +path "./1M_2_3.bin" true +共找到 4 个文件 +./4/5/1M_2_5.bin +./4/5/6/1M_2_6.bin +./4/1M_2_4.bin +./1M_2_3.bin + +===== 性能统计 ===== +文件数量: 4 +cd 加载目录: 4.789µs +总读取字节数: 4.00 MB +Stat 操作时间: 7.734µs +文件读取时间: 1.836ms +Stat 操作速率: 517196.79 文件/秒 +读取吞吐量: 2178.56 MB/秒 +``` + +fuse文件系统性能测试： + +```sh +#需要清空scorpio.toml里面的store文件夹的内容 +#建议不要更改scorpio.toml里面的load_dir_depth参数，否则要修改创建文件夹深度和访问深度 +#由于控制了加载深度，所以下面的测试不会访问到./4/5/6/1M_2_6.bin +#启动mono服务，推送/tmp/test_time_文件夹到mono +#scorpio下 cargo run +#测试首次加载的性能 +script/run /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3 +script/run /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3 + +(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3 +已切换到目录: /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3 +当前工作目录: /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3 +path "./4" false +path "./4/5" false +path "./4/5/6" false +path "./4/5/1M_1_5.bin" true +path "./4/1M_1_4.bin" true +path "./1M_1_3.bin" true +共找到 3 个文件 +./4/5/1M_1_5.bin +./4/1M_1_4.bin +./1M_1_3.bin + +===== 性能统计 ===== +文件数量: 3 +cd 加载目录: 299.299ms +总读取字节数: 3.00 MB +Stat 操作时间: 16.010µs +文件读取时间: 17.573ms +Stat 操作速率: 187382.89 文件/秒 +读取吞吐量: 170.72 MB/秒 +(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3 +已切换到目录: /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3 +当前工作目录: /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3 +path "./4" false +path "./4/5" false +path "./4/5/6" false +path "./4/5/1M_2_5.bin" true +path "./4/1M_2_4.bin" true +path "./1M_2_3.bin" true +共找到 3 个文件 +./4/5/1M_2_5.bin +./4/1M_2_4.bin +./1M_2_3.bin + +===== 性能统计 ===== +文件数量: 3 +cd 加载目录: 250.112ms +总读取字节数: 3.00 MB +Stat 操作时间: 14.346µs +文件读取时间: 22.745ms +Stat 操作速率: 209117.52 文件/秒 +读取吞吐量: 131.90 MB/秒 + +总结：初次加载目录和文件内容，网络开销大 + +#测试加载缓存以后读取的性能 +script/run /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3 +script/run /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3 + +(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3 +已切换到目录: /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3 +当前工作目录: /home/luxian/megadir/mount/third-party/test_202506162053/1/2/3 +path "./4" false +path "./4/5" false +path "./4/5/6" false +path "./4/5/1M_1_5.bin" true +path "./4/1M_1_4.bin" true +path "./1M_1_3.bin" true +共找到 3 个文件 +./4/5/1M_1_5.bin +./4/1M_1_4.bin +./1M_1_3.bin + +===== 性能统计 ===== +文件数量: 3 +cd 加载目录: 36.546ms +总读取字节数: 3.00 MB +Stat 操作时间: 10.790µs +文件读取时间: 11.981ms +Stat 操作速率: 278035.22 文件/秒 +读取吞吐量: 250.40 MB/秒 + +(base) root@yyjeqhc:~/git/mega/scorpio/script# ./run /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3 +已切换到目录: /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3 +当前工作目录: /home/luxian/megadir/mount/third-party/test_202506162053/2/2/3 +path "./4" false +path "./4/5" false +path "./4/5/6" false +path "./4/5/1M_2_5.bin" true +path "./4/1M_2_4.bin" true +path "./1M_2_3.bin" true +共找到 3 个文件 +./4/5/1M_2_5.bin +./4/1M_2_4.bin +./1M_2_3.bin + +===== 性能统计 ===== +文件数量: 3 +cd 加载目录: 39.063ms +总读取字节数: 3.00 MB +Stat 操作时间: 15.379µs +文件读取时间: 20.243ms +Stat 操作速率: 195071.20 文件/秒 +读取吞吐量: 148.20 MB/秒 + +总结：cd切换文件夹路径，如果文件夹内容没有变化，也至少有一次网络开销，但实际上，目录变化不会那么频繁，所以不用每次都要进行同步 +``` + diff --git a/scorpio/script/run.rs b/scorpio/script/run.rs new file mode 100644 index 000000000..0f2d283de --- /dev/null +++ b/scorpio/script/run.rs @@ -0,0 +1,113 @@ +use std::env; +use std::fs::{self, File, Metadata}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +fn collect_files_recursively(dir: &Path) -> Vec { + if dir.file_name() == Some(std::ffi::OsStr::new(".git")) { + return Vec::new(); // 忽略 .git 目录 + } + let mut files = Vec::new(); + if let Ok(entries) = fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + println!("path {:?} {:?}", path, path.is_file()); + + if path.is_file() { + files.push(path); + } else if path.is_dir() { + files.extend(collect_files_recursively(&path)); + } + } + } + files +} + +fn main() { + // 解析命令行参数 + let args: Vec = env::args().collect(); + if args.len() != 2 { + eprintln!("用法: {} <目录路径>", args[0]); + return; + } + + let target_dir = Path::new(&args[1]); + if !target_dir.exists() { + eprintln!("错误：指定路径不存在"); + return; + } + if !target_dir.is_dir() { + eprintln!("错误：指定路径不是目录"); + return; + } + + // 切换到目标目录 + let start_cd = Instant::now(); + + if let Err(e) = env::set_current_dir(target_dir) { + eprintln!("错误：无法切换到目录 {}: {}", target_dir.display(), e); + return; + } + let duration_cd = start_cd.elapsed(); + + println!("已切换到目录: {}", target_dir.display()); + + // 获取当前工作目录（应该是我们刚切换到的目录） + let current_dir = match env::current_dir() { + Ok(dir) => dir, + Err(e) => { + eprintln!("错误：无法获取当前目录: {}", e); + return; + } + }; + + println!("当前工作目录: {}", current_dir.display()); + + // 从当前目录开始递归收集文件（使用相对路径 "."） + let files = collect_files_recursively(Path::new(".")); + + println!("共找到 {} 个文件", files.len()); + for file in &files { + println!("{}", file.display()); + } + + // 测试 stat 时间 + let start_stat = Instant::now(); + let _stats: Vec> = files + .iter() + .map(|f| fs::metadata(f).ok()) + .collect(); + let duration_stat = start_stat.elapsed(); + + // 测试文件读取时间 + let start_read = Instant::now(); + let mut total_bytes = 0u64; + for f in &files { + if let Ok(mut file) = File::open(f) { + let mut buffer = Vec::new(); + if let Ok(bytes_read) = file.read_to_end(&mut buffer) { + total_bytes += bytes_read as u64; + } + } + } + let duration_read = start_read.elapsed(); + + // 输出统计信息 + println!("\n===== 性能统计 ====="); + println!("文件数量: {}", files.len()); + println!("cd 加载目录: {:.3?}", duration_cd); + + println!("总读取字节数: {:.2} MB", total_bytes as f64 / (1024.0 * 1024.0)); + println!("Stat 操作时间: {:.3?}", duration_stat); + println!("文件读取时间: {:.3?}", duration_read); + + if duration_stat.as_secs_f64() > 0.0 { + println!("Stat 操作速率: {:.2} 文件/秒", files.len() as f64 / duration_stat.as_secs_f64()); + } + + if duration_read.as_secs_f64() > 0.0 { + let throughput = (total_bytes as f64 / (1024.0 * 1024.0)) / duration_read.as_secs_f64(); + println!("读取吞吐量: {:.2} MB/秒", throughput); + } +} \ No newline at end of file diff --git a/scorpio/script/run.sh b/scorpio/script/run.sh new file mode 100755 index 000000000..69256304c --- /dev/null +++ b/scorpio/script/run.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e +# 参数 +TEST_DIR_BASE="/tmp" # 测试目录基础路径 + +# 1. 创建一个时间相关的临时目录 +timestamp=$(date +%Y%m%d%H%M) +git_dir="test_$timestamp" +base_dir="$TEST_DIR_BASE/$git_dir" +echo "创建目录：$base_dir" + +# 2. 创建深层目录结构 +generate_deep_structure() { + local base_path=$1 + local prefix=$2 + + # 创建深层目录路径：prefix/2/3/4/5/6 + local current_path="$base_path/$prefix" + for level in {2..6}; do + current_path="$current_path/$level" + mkdir -p "$current_path" + + # 在每个层级创建一个1MB文件 + echo "创建文件：$current_path/1M_${prefix}_${level}.bin" + head -c 1M "$current_path/1M_${prefix}_${level}.bin" + done +} + +# 创建两个深层目录结构 +generate_deep_structure "$base_dir" "1" +generate_deep_structure "$base_dir" "2" + +echo "文件结构创建完成：" +echo " 深层目录结构1：1/2/3/4/5/6 (每层1个1MB文件)" +echo " 深层目录结构2：2/2/3/4/5/6 (每层1个1MB文件)" +echo " 总层数：6层" +echo " 总文件数：10个 (每个结构5个文件)" +echo " 总大小：10MB" \ No newline at end of file diff --git a/scorpio/src/dicfuse/store.rs b/scorpio/src/dicfuse/store.rs index abccc2342..811a4ed83 100644 --- a/scorpio/src/dicfuse/store.rs +++ b/scorpio/src/dicfuse/store.rs @@ -46,19 +46,17 @@ pub struct ItemExt { } #[derive(Serialize, Deserialize, Debug, Default)] -struct CommitInfoResponse { +struct TreeInfoResponse { req_result: bool, - data: Vec, + data: Vec, err_message: String, } #[derive(Serialize, Deserialize, Debug, Default)] -struct CommitInfo { +struct TreeInfo { oid: String, name: String, content_type: String, - message: String, - date: String, } #[allow(unused)] @@ -243,7 +241,7 @@ async fn fetch_dir(path: &str) -> Result { } }; - let commit_info: CommitInfoResponse = match response.json().await { + let tree_info: TreeInfoResponse = match response.json().await { Ok(info) => info, Err(e) => { return Err(DictionaryError { @@ -252,13 +250,13 @@ async fn fetch_dir(path: &str) -> Result { } }; - if !commit_info.req_result { + if !tree_info.req_result { return Err(DictionaryError { - message: commit_info.err_message, + message: tree_info.err_message, }); } - let mut data = Vec::with_capacity(commit_info.data.len()); + let mut data = Vec::with_capacity(tree_info.data.len()); let base_path = if path.is_empty() || path == "/" { "".to_string() @@ -268,7 +266,7 @@ async fn fetch_dir(path: &str) -> Result { format!("{}/", path) }; - for info in commit_info.data { + for info in tree_info.data { let full_path = if base_path.is_empty() { format!("/{}", info.name) } else { @@ -313,7 +311,7 @@ async fn fetch_get_dir_hash(path: &str) -> Result info, Err(e) => { return Err(DictionaryError { @@ -322,13 +320,13 @@ async fn fetch_get_dir_hash(path: &str) -> Result Result std::io::Result<( } Ok(()) } +#[allow(dead_code)] pub fn get_oid_by_path(_path: &str) -> String { todo!() // create a old lfs pointer storage. }