diff --git a/config/config.toml b/config/config.toml index 10030a1fd..2b3b34d09 100644 --- a/config/config.toml +++ b/config/config.toml @@ -61,7 +61,7 @@ import_dir = "/third-party" admin = "admin" # Set serveral root dirs in directory init -root_dirs = ["third-party", "project", "doc", "release"] +root_dirs = ["third-party", "project", "doc", "release", "model"] [pack] # The maximum memory used by decode diff --git a/mercury/BUCK b/mercury/BUCK deleted file mode 100644 index c5ce272d5..000000000 --- a/mercury/BUCK +++ /dev/null @@ -1,38 +0,0 @@ -rust_library ( - name = "mercury", - srcs = glob(["src/**/*.rs"]), - crate_root = "src/lib.rs", - deps = [ - "//mercury/delta:delta", - "//common:common", - "//jupiter/callisto:callisto", - "//third-party:flate2", - "//third-party:serde", - "//third-party:bstr", - "//third-party:hex", - "//third-party:thiserror", - "//third-party:tracing", - "//third-party:sha1", - "//third-party:colored", - "//third-party:chrono", - "//third-party:tracing-subscriber", - "//third-party:uuid", - "//third-party:threadpool", - "//third-party:num_cpus", - "//third-party:dashmap", - "//third-party:tokio", - "//third-party:lru-mem", - "//third-party:bincode", - "//third-party:byteorder", - "//third-party:futures-util", - "//third-party:bytes", - "//third-party:axum", - "//third-party:memchr", - "//third-party:encoding_rs", - "//third-party:rayon", - "//third-party:tokio-util", - ], - visibility = [ - 'PUBLIC', - ], -) diff --git a/mercury/README.md b/mercury/README.md index 11174d0a6..d361e246b 100644 --- a/mercury/README.md +++ b/mercury/README.md @@ -1,6 +1,77 @@ ## Mercury Module - Git Internal Module -### Performance +Mercury is a high-performance Rust library for encoding and decoding Git internal objects and Pack files. It provides comprehensive support for Git's internal object storage format with advanced features like delta compression, memory management, and concurrent processing. + +## Overview + +Mercury is designed to handle Git internal objects and Pack files efficiently, supporting both reading and writing operations with optimized memory usage and multi-threaded processing capabilities. The library implements the complete Git Pack format specification with additional optimizations for large-scale Git operations. + +## Key Features + +### 1. Multi-threaded Processing + +- Configurable thread pool for parallel object processing +- Concurrent delta resolution with dependency management +- Asynchronous I/O operations for improved performance + +### 2. Advanced Memory Management + +- LRU-based memory cache with configurable limits +- Automatic disk spillover for large objects +- Memory usage tracking and optimization +- Heap size calculation for accurate memory accounting + +### 3. Delta Compression Support + +- Offset Delta : References objects by pack file offset +- Hash Delta : References objects by SHA-1 hash +- Zstd Delta : Enhanced compression using Zstandard algorithm +- Intelligent delta chain resolution + +### 4. Streaming Support + +- Stream-based pack file processing +- Memory-efficient handling of large pack files +- Support for network streams and file streams + +## Core Algorithms + +### Pack Decoding Algorithm + +1. Read and validate pack header (PACK signature, version, object count) +2. For each object in the pack: + a. Parse object header (type, size) + b. Handle based on object type: + - Base objects: Decompress and store directly + - Delta objects: Add to waitlist until base is available + c. Resolve delta chains when base objects become available +3. Verify pack checksum + +### Delta Resolution Strategy + +- Waitlist Management : Delta objects wait for their base objects +- Dependency Tracking : Maintains offset and hash-based dependency maps +- Chain Resolution : Recursively applies delta operations +- Memory Optimization : Calculates expanded object sizes to prevent OOM + +### Cache Management + +- Two-tier Caching : Memory cache with disk spillover +- LRU Eviction : Least recently used objects are evicted first +- Size-based Limits : Configurable memory limits with accurate tracking +- Async Persistence : Background threads handle disk operations + +### Object Processing Pipeline + +``` +Input Stream → Header Parsing → Object Decoding → Delta Resolution → Cache Storage → Output + ↓ ↓ ↓ ↓ + Validation Decompression Waitlist Mgmt Memory Mgmt +``` + +## Performance Optimizations + +### Memory Allocator Recommendations > [!TIP] > Here are some performance tips that you can use to significantly improve performance when using `Mercury` crates as a dependency. @@ -33,4 +104,18 @@ A simple approach: static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; ``` - \ No newline at end of file +### Concurrent Processing + +- Configurable thread pools for CPU-intensive operations +- Lock-free data structures where possible (DashMap for waitlists) +- Parallel delta application using Rayon + +### 3. I/O Optimization + +- Buffered reading with configurable buffer sizes +- Asynchronous file operations for cache persistence +- Stream-based processing to minimize memory footprint + +### Benchmark + +**TODO** \ No newline at end of file diff --git a/mercury/delta/BUCK b/mercury/delta/BUCK deleted file mode 100644 index b71f287de..000000000 --- a/mercury/delta/BUCK +++ /dev/null @@ -1,14 +0,0 @@ -rust_library ( - name = "delta", - srcs = glob(["src/**/*.rs"]), - crate_root = "src/lib.rs", - features = ["diff_mydrs"], - deps = [ - "//third-party:diffs", - "//third-party:thiserror", - "//third-party:flate2", - ], - visibility = [ - 'PUBLIC', - ], -) diff --git a/mercury/delta/README.md b/mercury/delta/README.md index 0e5d9f7d2..687725ff3 100644 --- a/mercury/delta/README.md +++ b/mercury/delta/README.md @@ -1,4 +1,5 @@ # Git Delta + In Git, delta refers to the differences or changes between files or data objects. It is a measure of the amount of change between two versions. By using delta, Git can more efficiently store and transfer changes to files or data objects. ## Example @@ -17,6 +18,7 @@ If you want to decode a delta data, you need a base data(base_info) and a delta ```rust use delta; + let delta_result:Result, GitDeltaError> = delta::delta_decode(stream, base_info); ``` diff --git a/mercury/delta/src/lib.rs b/mercury/delta/src/lib.rs index 29ca1e605..978a74bd2 100644 --- a/mercury/delta/src/lib.rs +++ b/mercury/delta/src/lib.rs @@ -6,10 +6,12 @@ mod errors; mod utils; pub use decode::delta_decode as decode; + pub fn encode_rate(old_data: &[u8], new_data: &[u8]) -> f64 { let differ = DeltaDiff::new(old_data, new_data); differ.get_ssam_rate() } + pub fn encode(old_data: &[u8], new_data: &[u8]) -> Vec { let differ = DeltaDiff::new(old_data, new_data); differ.encode() diff --git a/mercury/zstdelta/README.md b/mercury/zstdelta/README.md index f555ee3cf..b75231ae5 100644 --- a/mercury/zstdelta/README.md +++ b/mercury/zstdelta/README.md @@ -1 +1,3 @@ -It is a copy of the `zstdelta` crate from the `facebook/sapling` repository, which is licensed under MIT License. \ No newline at end of file +## Zstdelta + +It is a copy of the `zstdelta` crate from the [facebook/sapling](https://github.com/facebook/sapling) repository, which is licensed under MIT License. \ No newline at end of file