diff --git a/README.md b/README.md index e69de29b..d361e246 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,121 @@ +## Mercury Module - Git Internal Module + +Mercury is a high-performance Rust library for encoding and decoding Git internal objects and Pack files. It provides comprehensive support for Git's internal object storage format with advanced features like delta compression, memory management, and concurrent processing. + +## Overview + +Mercury is designed to handle Git internal objects and Pack files efficiently, supporting both reading and writing operations with optimized memory usage and multi-threaded processing capabilities. The library implements the complete Git Pack format specification with additional optimizations for large-scale Git operations. + +## Key Features + +### 1. Multi-threaded Processing + +- Configurable thread pool for parallel object processing +- Concurrent delta resolution with dependency management +- Asynchronous I/O operations for improved performance + +### 2. Advanced Memory Management + +- LRU-based memory cache with configurable limits +- Automatic disk spillover for large objects +- Memory usage tracking and optimization +- Heap size calculation for accurate memory accounting + +### 3. Delta Compression Support + +- Offset Delta : References objects by pack file offset +- Hash Delta : References objects by SHA-1 hash +- Zstd Delta : Enhanced compression using Zstandard algorithm +- Intelligent delta chain resolution + +### 4. Streaming Support + +- Stream-based pack file processing +- Memory-efficient handling of large pack files +- Support for network streams and file streams + +## Core Algorithms + +### Pack Decoding Algorithm + +1. Read and validate pack header (PACK signature, version, object count) +2. For each object in the pack: + a. Parse object header (type, size) + b. Handle based on object type: + - Base objects: Decompress and store directly + - Delta objects: Add to waitlist until base is available + c. Resolve delta chains when base objects become available +3. Verify pack checksum + +### Delta Resolution Strategy + +- Waitlist Management : Delta objects wait for their base objects +- Dependency Tracking : Maintains offset and hash-based dependency maps +- Chain Resolution : Recursively applies delta operations +- Memory Optimization : Calculates expanded object sizes to prevent OOM + +### Cache Management + +- Two-tier Caching : Memory cache with disk spillover +- LRU Eviction : Least recently used objects are evicted first +- Size-based Limits : Configurable memory limits with accurate tracking +- Async Persistence : Background threads handle disk operations + +### Object Processing Pipeline + +``` +Input Stream → Header Parsing → Object Decoding → Delta Resolution → Cache Storage → Output + ↓ ↓ ↓ ↓ + Validation Decompression Waitlist Mgmt Memory Mgmt +``` + +## Performance Optimizations + +### Memory Allocator Recommendations + +> [!TIP] +> Here are some performance tips that you can use to significantly improve performance when using `Mercury` crates as a dependency. + +In certain versions of Rust, using `HashMap` on Windows can lead to performance issues. This is due to the allocation strategy of the internal heap memory allocator. To mitigate these performance issues on Windows, you can use [mimalloc](https://github.com/microsoft/mimalloc). (See [this issue](https://github.com/rust-lang/rust/issues/121747) for more details.) + +On other platforms, you can also experiment with [jemalloc](https://github.com/jemalloc/jemalloc) or [mimalloc](https://github.com/microsoft/mimalloc) to potentially improve performance. + +A simple approach: + +1. Change Cargo.toml to use mimalloc on Windows and jemalloc on other platforms. + + ```toml + [target.'cfg(not(windows))'.dependencies] + jemallocator = "0.5.4" + + [target.'cfg(windows)'.dependencies] + mimalloc = "0.1.43" + ``` + +2. Add `#[global_allocator]` to the main.rs file of the program to specify the allocator. + + ```rust + #[cfg(not(target_os = "windows"))] + #[global_allocator] + static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc; + + #[cfg(target_os = "windows")] + #[global_allocator] + static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + ``` + +### Concurrent Processing + +- Configurable thread pools for CPU-intensive operations +- Lock-free data structures where possible (DashMap for waitlists) +- Parallel delta application using Rayon + +### 3. I/O Optimization + +- Buffered reading with configurable buffer sizes +- Asynchronous file operations for cache persistence +- Stream-based processing to minimize memory footprint + +### Benchmark + +**TODO** \ No newline at end of file diff --git a/mercury/Cargo.toml b/mercury/Cargo.toml new file mode 100644 index 00000000..460a7426 --- /dev/null +++ b/mercury/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "mercury" +version = "0.1.0" +edition = "2024" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +delta = { path = "delta" } +sapling-zstdelta = { path = "zstdelta" } +common = { workspace = true } +callisto = { workspace = true } +flate2 = { workspace = true, features = ["zlib"] } # enable linking against the libz(C lib); better performance +serde = { workspace = true, features = ["derive"] } +bstr = { workspace = true } +hex = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +sha1 = { workspace = true } +colored = { workspace = true } +chrono = { workspace = true } +tracing-subscriber = { workspace = true } +uuid = { workspace = true, features = ["v4"] } +threadpool = { workspace = true } +num_cpus = { workspace = true } +dashmap = { workspace = true } +tokio = {workspace = true } +lru-mem = { workspace = true } +bincode = { workspace = true , features = ["serde"] } +byteorder = { workspace = true } +futures-util = { workspace = true } +bytes = { workspace = true } +axum = { workspace = true } +memchr = { workspace = true } +encoding_rs = { workspace = true } +rayon = { workspace = true } +reqwest = { workspace = true } +ring = { workspace = true } +serde_json = { workspace = true } +ahash = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["full"] } +tokio-util = { workspace = true, features = ["io"] } diff --git a/mercury/delta/Cargo.toml b/mercury/delta/Cargo.toml new file mode 100644 index 00000000..48ab1da3 --- /dev/null +++ b/mercury/delta/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "delta" +version = "0.1.0" +edition = "2024" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +diffs = { workspace = true } +thiserror = { workspace = true } +flate2 = { workspace = true, features = ["zlib"] } +rayon = { workspace = true } +[features] +default = ["diff_mydrs"] +diff_mydrs = [] diff --git a/mercury/delta/README.md b/mercury/delta/README.md new file mode 100644 index 00000000..687725ff --- /dev/null +++ b/mercury/delta/README.md @@ -0,0 +1,33 @@ +# Git Delta + +In Git, delta refers to the differences or changes between files or data objects. It is a measure of the amount of change between two versions. By using delta, Git can more efficiently store and transfer changes to files or data objects. + +## Example + +This module exposes three functions to the outside world: + +```rust +pub fn delta_decode(mut stream : &mut impl Read,base_info: &Vec) -> Result, GitDeltaError> + +pub fn delta_encode_rate(old_data: & [u8], new_data: & [u8]) -> f64 + +pub fn delta_encode(old_data: & [u8], new_data: & [u8]) -> Vec +``` + +If you want to decode a delta data, you need a base data(base_info) and a delta instruction data(stream). + +```rust +use delta; + +let delta_result:Result, GitDeltaError> = delta::delta_decode(stream, base_info); +``` + +On the contrary, if you want to compress another object in delta form based on it, use the encode function + +```rust +use delta; + +let delta_data:Vec = delta::delta_encode(old_data, new_data) ; +``` + +In addition, the `delta::delta_encode_rate` function can represent the compression rate of delta \ No newline at end of file diff --git a/mercury/delta/src/decode/mod.rs b/mercury/delta/src/decode/mod.rs new file mode 100644 index 00000000..c795690d --- /dev/null +++ b/mercury/delta/src/decode/mod.rs @@ -0,0 +1,73 @@ +use crate::{errors::GitDeltaError, utils}; +use std::io::{ErrorKind, Read}; + +const COPY_INSTRUCTION_FLAG: u8 = 1 << 7; +const COPY_OFFSET_BYTES: u8 = 4; +const COPY_SIZE_BYTES: u8 = 3; +const COPY_ZERO_SIZE: usize = 0x10000; + +pub fn delta_decode( + mut stream: &mut impl Read, + base_info: &[u8], +) -> Result, GitDeltaError> { + // Read the bash object size & Result Size + let base_size = utils::read_size_encoding(&mut stream).unwrap(); + if base_info.len() != base_size { + return Err(GitDeltaError::DeltaDecoderError( + "base object len is not equal".to_owned(), + )); + } + + let result_size = utils::read_size_encoding(&mut stream).unwrap(); + let mut buffer = Vec::with_capacity(result_size); + loop { + // Check if the stream has ended, meaning the new object is done + let instruction = match utils::read_bytes(stream) { + Ok([instruction]) => instruction, + Err(err) if err.kind() == ErrorKind::UnexpectedEof => break, + Err(err) => { + panic!("{}", format!("Wrong instruction in delta :{err}")); + } + }; + + if instruction & COPY_INSTRUCTION_FLAG == 0 { + // Data instruction; the instruction byte specifies the number of data bytes + if instruction == 0 { + // Appending 0 bytes doesn't make sense, so git disallows it + panic!( + "{}", + GitDeltaError::DeltaDecoderError(String::from("Invalid data instruction")) + ); + } + + // Append the provided bytes + let mut data = vec![0; instruction as usize]; + stream.read_exact(&mut data).unwrap(); + buffer.extend_from_slice(&data); + // result.extend_from_slice(&data); + } else { + // Copy instruction + let mut nonzero_bytes = instruction; + let offset = + utils::read_partial_int(&mut stream, COPY_OFFSET_BYTES, &mut nonzero_bytes) + .unwrap(); + let mut size = + utils::read_partial_int(&mut stream, COPY_SIZE_BYTES, &mut nonzero_bytes).unwrap(); + if size == 0 { + // Copying 0 bytes doesn't make sense, so git assumes a different size + size = COPY_ZERO_SIZE; + } + // Copy bytes from the base object + let base_data = base_info.get(offset..(offset + size)).ok_or_else(|| { + GitDeltaError::DeltaDecoderError("Invalid copy instruction".to_string()) + }); + + match base_data { + Ok(data) => buffer.extend_from_slice(data), + Err(e) => return Err(e), + } + } + } + assert!(buffer.len() == result_size); + Ok(buffer) +} diff --git a/mercury/delta/src/encode/mod.rs b/mercury/delta/src/encode/mod.rs new file mode 100644 index 00000000..a42e1c00 --- /dev/null +++ b/mercury/delta/src/encode/mod.rs @@ -0,0 +1,266 @@ +use diffs::Diff; +#[cfg(feature = "diff_mydrs")] +use diffs::myers; + +const DATA_INS_LEN: usize = 0x7f; +const VAR_INT_ENCODING_BITS: u8 = 7; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Optype { + Data, + Copy, +} + +#[derive(Debug, Clone, Copy)] +struct DeltaOp { + ins: Optype, + begin: usize, + len: usize, +} + +#[derive(Debug)] +pub struct DeltaDiff<'a> { + ops: Vec, + old_data: &'a [u8], + new_data: &'a [u8], + ssam: usize, + ssam_r: f64, +} + +impl<'a> DeltaDiff<'a> { + /// Diff the two u8 array slice , Type should be same. + /// Return the DeltaDiff struct. + pub fn new(old_data: &'a [u8], new_data: &'a [u8]) -> Self { + let mut delta_diff = DeltaDiff { + ops: vec![], + old_data, + new_data, + ssam: 0, + ssam_r: 0.00, + }; + + #[cfg(feature = "diff_mydrs")] + myers::diff( + &mut delta_diff, + old_data, + 0, + old_data.len(), + new_data, + 0, + new_data.len(), + ) + .unwrap(); + + #[cfg(not(feature = "diff_mydrs"))] + diffs::patience::diff( + &mut delta_diff, + old_data, + 0, + old_data.len(), + new_data, + 0, + new_data.len(), + ) + .unwrap(); + + delta_diff + } + + pub fn encode(&self) -> Vec { + let mut result: Vec = Vec::with_capacity(self.ops.len() * 30); + result.append(&mut write_size_encoding(self.old_data.len())); + result.append(&mut write_size_encoding(self.new_data.len())); + + for op in &self.ops { + result.append(&mut self.decode_op(op)); + } + result + } + + /// + /// Decode the DeltaOp to `Vec` + fn decode_op(&self, op: &DeltaOp) -> Vec { + let mut op_data = vec![]; + + match op.ins { + Optype::Data => { + let instruct = (op.len & 0x7f) as u8; + op_data.push(instruct); + op_data.append(&mut self.new_data[op.begin..op.begin + op.len].to_vec()); + } + + Optype::Copy => { + let mut instruct: u8 = 0x80; + let mut offset = op.begin; + let mut size = op.len; + let mut copy_data = vec![]; + + for i in 0..4 { + let _bit = (offset & 0xff) as u8; + if _bit != 0 { + instruct |= (1 << i) as u8; + copy_data.push(_bit) + } + offset >>= 8; + } + + for i in 4..7 { + let _bit = (size & 0xff) as u8; + if _bit != 0 { + instruct |= (1 << i) as u8; + copy_data.push(_bit) + } + size >>= 8; + } + + op_data.push(instruct); + op_data.append(&mut copy_data); + } + } + + op_data + } + + pub fn get_ssam_rate(&self) -> f64 { + self.ssam_r + } +} + +impl Diff for DeltaDiff<'_> { + type Error = (); + + fn equal(&mut self, _old: usize, _new: usize, _len: usize) -> Result<(), Self::Error> { + self.ssam += _len; + if let Some(tail) = self.ops.last_mut() { + if tail.begin + tail.len == _old && tail.ins == Optype::Copy { + tail.len += _len; + } else { + self.ops.push(DeltaOp { + ins: Optype::Copy, + begin: _old, + len: _len, + }); + } + } else { + self.ops.push(DeltaOp { + ins: Optype::Copy, + begin: _old, + len: _len, + }); + } + + Ok(()) + } + + fn insert(&mut self, _old: usize, _new: usize, _len: usize) -> Result<(), ()> { + let mut len = _len; + let mut new = _new; + + if _len > DATA_INS_LEN { + while len > DATA_INS_LEN { + self.ops.push(DeltaOp { + ins: Optype::Data, + begin: new, + len: DATA_INS_LEN, + }); + + len -= DATA_INS_LEN; + new += DATA_INS_LEN; + } + + self.ops.push(DeltaOp { + ins: Optype::Data, + begin: new, + len, + }); + } else if let Some(tail) = self.ops.last_mut() { + if tail.begin + tail.len == _new + && tail.ins == Optype::Data + && tail.len + _len < DATA_INS_LEN + { + tail.len += _len; + } else { + self.ops.push(DeltaOp { + ins: Optype::Data, + begin: new, + len, + }); + } + } else { + self.ops.push(DeltaOp { + ins: Optype::Data, + begin: new, + len, + }); + } + + Ok(()) + } + + fn finish(&mut self) -> Result<(), Self::Error> { + // compute the ssam rate when finish the diff process. + self.ssam_r = self.ssam as f64 / self.new_data.len() as f64; + Ok(()) + } +} + +fn write_size_encoding(number: usize) -> Vec { + let mut num = vec![]; + let mut number = number; + + loop { + if number >> VAR_INT_ENCODING_BITS > 0 { + num.push((number & 0x7f) as u8 | 0x80); + } else { + num.push((number & 0x7f) as u8); + break; + } + + number >>= VAR_INT_ENCODING_BITS; + } + num +} + +#[cfg(test)] +mod tests { + use flate2::bufread::ZlibDecoder; + use std::env; + use std::fs::File; + use std::io::{self, BufReader, Cursor, Read}; + use std::path::{Path, PathBuf}; + + use crate::decode::delta_decode; + + use super::DeltaDiff; + fn read_zlib_data(path: &Path) -> Result, io::Error> { + // 打开文件进行读取 + let file = File::open(path)?; + let buf_reader = BufReader::new(file); + let mut deflate = ZlibDecoder::new(buf_reader); + let mut result = Vec::new(); + deflate.read_to_end(&mut result)?; + Ok(result) + } + + #[test] + fn test_delta_fn() { + let mut source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + source.push("../tests/diff/16ecdcc8f663777896bd39ca025a041b7f005e"); + let old_data = read_zlib_data(&source).unwrap(); + let mut source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + source.push("../tests/diff/bee0d45f981adf7c2926a0dc04deb7f006bcc3"); + let new_data = read_zlib_data(&source).unwrap(); + + let d = DeltaDiff::new(&old_data, &new_data); + let delta_result = d.encode(); + println!( + "delta size: from {} to {}", + old_data.len(), + delta_result.len() + ); + + let mut reader = Cursor::new(&delta_result); + let rebuild_data = delta_decode(&mut reader, &old_data).expect("delta format error"); + assert_eq!(new_data, rebuild_data); + } +} diff --git a/mercury/delta/src/errors.rs b/mercury/delta/src/errors.rs new file mode 100644 index 00000000..a4003dcd --- /dev/null +++ b/mercury/delta/src/errors.rs @@ -0,0 +1,10 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum GitDeltaError { + #[error("The `{0}` is not a valid git object type.")] + DeltaEncoderError(String), + + #[error("The `{0}` is not a valid git object type.")] + DeltaDecoderError(String), +} diff --git a/mercury/delta/src/lib.rs b/mercury/delta/src/lib.rs new file mode 100644 index 00000000..49b7348b --- /dev/null +++ b/mercury/delta/src/lib.rs @@ -0,0 +1,175 @@ +use encode::DeltaDiff; +use rayon::prelude::*; +use std::hash::{DefaultHasher, Hash, Hasher}; + +mod decode; +mod encode; +mod errors; +mod utils; + +pub use decode::delta_decode as decode; + +const SAMPLE_STEP: usize = 64; +const MIN_DELTA_RATE: f64 = 0.5; + +/// approximate calculation delta rate to enhance efficiency +#[allow(clippy::manual_div_ceil)] +pub fn heuristic_encode_rate(old_data: &[u8], new_data: &[u8]) -> f64 { + let old_len = old_data.len(); + let new_len = new_data.len(); + + if old_len == 0 && new_len == 0 { + return 1.0; + } + if old_len == 0 || new_len == 0 { + return 0.0; + } + + let step = SAMPLE_STEP; + let mut match_count = 0; + let mut sample_count = 0; + + let min_len = old_len.min(new_len); + + let total_samples = (min_len + step - 1) / step; + let mut i = 0; + while i < min_len { + let old_chunk = &old_data[i..(i + step).min(old_len)]; + let new_chunk = &new_data[i..(i + step).min(new_len)]; + + if hash_chunk(old_chunk) == hash_chunk(new_chunk) { + match_count += 1; + } + sample_count += 1; + + // Early stopping condition: + // If all remaining samples matched, the rate would still not reach MIN_DELTA_RATE, so return 0 early + let remaining_samples = total_samples - sample_count; + let max_possible_rate = (match_count + remaining_samples) as f64 / total_samples as f64; + if max_possible_rate < MIN_DELTA_RATE { + return 0.0; + } + + i += step; + } + + match_count as f64 / sample_count as f64 +} + +fn hash_chunk(chunk: &[u8]) -> u64 { + let mut hasher = DefaultHasher::new(); + chunk.hash(&mut hasher); + hasher.finish() +} + +pub fn heuristic_encode_rate_parallel(old_data: &[u8], new_data: &[u8]) -> f64 { + let old_len = old_data.len(); + let new_len = new_data.len(); + + if old_len == 0 && new_len == 0 { + return 1.0; + } + if old_len == 0 || new_len == 0 { + return 0.0; + } + + let min_len = old_len.min(new_len); + + let step = if min_len > 10_000_000 { + 1024 + } else if min_len > 1_000_000 { + 512 + } else if min_len > 100_000 { + 128 + } else { + 16 + }; + + let chunks: Vec<_> = old_data[..min_len] + .chunks(step) + .zip(new_data[..min_len].chunks(step)) + .collect(); + + let match_count: usize = chunks.par_iter().filter(|(a, b)| a == b).count(); + + let rate = match_count as f64 / chunks.len() as f64; + if rate < MIN_DELTA_RATE { 0.0 } else { rate } +} + +pub fn encode_rate(old_data: &[u8], new_data: &[u8]) -> f64 { + let differ = DeltaDiff::new(old_data, new_data); + differ.get_ssam_rate() +} + +pub fn encode(old_data: &[u8], new_data: &[u8]) -> Vec { + let differ = DeltaDiff::new(old_data, new_data); + differ.encode() +} + +#[cfg(test)] +mod tests { + use crate::{encode_rate, heuristic_encode_rate, heuristic_encode_rate_parallel}; + + #[test] + fn test_heuristic_encode_rate() { + let data1 = b"hello world, this is a test for delta rate"; + let data2 = b"hello world, this is a test for delta rate"; + let rate = heuristic_encode_rate(data1, data2); + println!("rate = {}", rate); + assert!((rate - 1.0).abs() < 1e-6, "Expected 1.0 for identical data"); + + let data3 = b"hello world, this is a test for delta rate"; + let data4 = b"hello worll, this is a test for delta rate"; + let rate = encode_rate(data3, data4); + println!("rate = {}", rate); + assert!( + rate > 0.5 && rate < 1.0, + "Expected partial match for similar data" + ); + + let data5 = b"abcdefghijklmno"; + let data6 = b"1234567890!@#"; + let rate = heuristic_encode_rate(data5, data6); + println!("rate = {}", rate); + assert!(rate < 0.2, "Expected low match rate for different data"); + + let data7 = b""; + let data8 = b""; + let rate = heuristic_encode_rate(data7, data8); + println!("rate = {}", rate); + assert_eq!(rate, 1.0, "Empty slices should be fully matching"); + + let rate = heuristic_encode_rate(data7, data1); + assert_eq!(rate, 0.0, "Empty vs non-empty should give 0 rate"); + } + + #[test] + fn test_heuristic_encode_rate_large_files() { + let data1 = vec![0u8; 100_000]; + let data2 = vec![1u8; 100_000]; + let rate = heuristic_encode_rate(&data1, &data2); + println!("Large non-matching data rate = {}", rate); + assert_eq!( + rate, 0.0, + "Large completely different data should early stop with 0 rate" + ); + + let data3 = vec![0u8; 100]; + let mut data4 = vec![0u8; 100]; + + for i in 0..2 { + data4[i] = 1; + } + let rate1 = heuristic_encode_rate_parallel(&data3, &data4); + let rate2 = encode_rate(&data3, &data4); + println!( + "Large partially matching data rate = {}, accurate rate = {}", + rate1, rate2 + ); + + assert!( + (rate2 - rate1).abs() < 0.2, + "Large partially matching data should preserve partial rate" + ); + } +} diff --git a/mercury/delta/src/utils.rs b/mercury/delta/src/utils.rs new file mode 100644 index 00000000..40ffd01b --- /dev/null +++ b/mercury/delta/src/utils.rs @@ -0,0 +1,78 @@ +use std::io::Read; + +const VAR_INT_ENCODING_BITS: u8 = 7; +const VAR_INT_CONTINUE_FLAG: u8 = 1 << VAR_INT_ENCODING_BITS; + +/// Read the next N bytes from the reader +/// +#[inline] +pub fn read_bytes(stream: &mut R) -> std::io::Result<[u8; N]> { + let mut bytes = [0; N]; + stream.read_exact(&mut bytes)?; + + Ok(bytes) +} + +// Read the type and size of the object +pub fn read_size_encoding(stream: &mut R) -> std::io::Result { + let mut value = 0; + let mut length = 0; + + loop { + let (byte_value, more_bytes) = read_var_int_byte(stream).unwrap(); + value |= (byte_value as usize) << length; + if !more_bytes { + return Ok(value); + } + + length += VAR_INT_ENCODING_BITS; + } +} + +/// Reads a partial integer from a stream. +/// +/// # Arguments +/// +/// * `stream` - A mutable reference to a readable stream. +/// * `bytes` - The number of bytes to read from the stream. +/// * `present_bytes` - A mutable reference to a byte indicating which bits are present in the integer value. +/// +/// # Returns +/// +/// This function returns a result of type `io::Result`. If the operation is successful, the integer value +/// read from the stream is returned as `Ok(value)`. Otherwise, an `Err` variant is returned, wrapping an `io::Error` +/// that describes the specific error that occurred. +pub fn read_partial_int( + stream: &mut R, + bytes: u8, + present_bytes: &mut u8, +) -> std::io::Result { + let mut value: usize = 0; + + // Iterate over the byte indices + for byte_index in 0..bytes { + // Check if the current bit is present + if *present_bytes & 1 != 0 { + // Read a byte from the stream + let [byte] = read_bytes(stream)?; + + // Add the byte value to the integer value + value |= (byte as usize) << (byte_index * 8); + } + + // Shift the present bytes to the right + *present_bytes >>= 1; + } + + Ok(value) +} + +/// Returns whether the first bit of u8 is 1 and returns the 7-bit truth value +/// +pub fn read_var_int_byte(stream: &mut R) -> std::io::Result<(u8, bool)> { + let [byte] = read_bytes(stream)?; + let value = byte & !VAR_INT_CONTINUE_FLAG; + let more_bytes = byte & VAR_INT_CONTINUE_FLAG != 0; + + Ok((value, more_bytes)) +} diff --git a/mercury/src/errors.rs b/mercury/src/errors.rs new file mode 100644 index 00000000..eb8cd438 --- /dev/null +++ b/mercury/src/errors.rs @@ -0,0 +1,110 @@ +use std::string::FromUtf8Error; + +use common::errors::MegaError; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum GitError { + #[error("The `{0}` is not a valid git object type.")] + InvalidObjectType(String), + + #[error("The `{0}` is not a valid git blob object.")] + InvalidBlobObject(String), + + #[error("Not a valid git tree object.")] + InvalidTreeObject, + + #[error("The `{0}` is not a valid git tree item.")] + InvalidTreeItem(String), + + #[error("`{0}`.")] + EmptyTreeItems(String), + + #[error("The `{0}` is not a valid git commit signature.")] + InvalidSignatureType(String), + + #[error("Not a valid git commit object.")] + InvalidCommitObject, + + #[error("Invalid Commit: {0}")] + InvalidCommit(String), + + #[error("Not a valid git tag object: {0}")] + InvalidTagObject(String), + + #[error("The `{0}` is not a valid idx file.")] + InvalidIdxFile(String), + + #[error("The `{0}` is not a valid pack file.")] + InvalidPackFile(String), + + #[error("The `{0}` is not a valid pack header.")] + InvalidPackHeader(String), + + #[error("The `{0}` is not a valid index file.")] + InvalidIndexFile(String), + + #[error("The `{0}` is not a valid index header.")] + InvalidIndexHeader(String), + + #[error("Argument parse failed: {0}")] + InvalidArgument(String), + + #[error("IO Error: {0}")] + IOError(#[from] std::io::Error), + + #[error("The {0} is not a valid Hash value ")] + InvalidHashValue(String), + + #[error("Delta Object Error Info:{0}")] + DeltaObjectError(String), + + #[error("The object to be packed is incomplete ,{0}")] + UnCompletedPackObject(String), + + #[error("Error decode in the Object ,info:{0}")] + InvalidObjectInfo(String), + + #[error("Can't found Hash value :{0} from current file")] + NotFountHashValue(String), + + #[error("Can't encode the object which id [{0}] to bytes")] + EncodeObjectError(String), + + #[error("UTF-8 conversion error: {0}")] + ConversionError(String), + + #[error("Can't find parent tree by path: {0}")] + InvalidPathError(String), + + #[error("Can't encode entries to pack: {0}")] + PackEncodeError(String), + + #[error("Can't find specific object: {0}")] + ObjectNotFound(String), + + #[error("Repository not found")] + RepoNotFound, + + #[error("UnAuthorized: {0}")] + UnAuthorized(String), + + #[error("Network Error: {0}")] + NetworkError(String), + + #[error("{0}")] + CustomError(String), +} + +impl From for GitError { + fn from(err: FromUtf8Error) -> Self { + // convert the FromUtf8Error to GitError and return it + GitError::ConversionError(err.to_string()) + } +} + +impl From for GitError { + fn from(err: MegaError) -> Self { + GitError::CustomError(err.to_string()) + } +} diff --git a/mercury/src/hash.rs b/mercury/src/hash.rs new file mode 100644 index 00000000..8543077e --- /dev/null +++ b/mercury/src/hash.rs @@ -0,0 +1,267 @@ +//! In Git, the SHA-1 hash algorithm is widely used to generate unique identifiers for Git objects. +//! Each Git object corresponds to a unique SHA-1 hash value, which is used to identify the object's +//! location in the Git internal and mega database. +//! + +use std::{fmt::Display, io}; + +use bincode::{Decode, Encode}; +use colored::Colorize; +use serde::{Deserialize, Serialize}; +use sha1::Digest; + +use crate::internal::object::types::ObjectType; + +/// The [`SHA1`] struct, encapsulating a `[u8; 20]` array, is specifically designed to represent Git hash IDs. +/// In Git's context, these IDs are 40-character hexadecimal strings generated via the SHA-1 algorithm. +/// Each Git object receives a unique hash ID based on its content, serving as an identifier for its location +/// within the Git internal database. Utilizing a dedicated struct for these hash IDs enhances code readability and +/// maintainability by providing a clear, structured format for their manipulation and storage. +/// +/// ### Change Log +/// +/// In previous versions of the 'mega' project, `Hash` was used to denote hash values. However, in newer versions, +/// `SHA1` is employed for this purpose. Future updates plan to extend support to SHA256 and SHA512, or potentially +/// other hash algorithms. By abstracting the hash model to `Hash`, and using specific imports like `use crate::hash::SHA1` +/// or `use crate::hash::SHA256`, the codebase maintains a high level of clarity and maintainability. This design choice +/// allows for easier adaptation to different hash algorithms while keeping the underlying implementation consistent and +/// understandable. - Nov 26, 2023 (by @genedna) +/// +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + Default, + Deserialize, + Serialize, + Encode, + Decode, +)] +pub struct SHA1(pub [u8; 20]); + +/// Display trait for SHA1. +impl Display for SHA1 { + /// Allows [`SHA1::to_string()`] to be used. + /// Note: If you want a terminal-friendly colorized output, use [`SHA1::to_color_str()`]. + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", hex::encode(self.0)) + } +} + +impl AsRef<[u8]> for SHA1 { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} +/// Implementation of the [`std::str::FromStr`] trait for the [`SHA1`] type. +/// +/// To effectively use the `from_str` method for converting a string to a `SHA1` object, consider the following: +/// 1. The input string `s` should be a pre-calculated hexadecimal string, exactly 40 characters in length. This string +/// represents a SHA1 hash and should conform to the standard SHA1 hash format. +/// 2. It is necessary to explicitly import the `FromStr` trait to utilize the `from_str` method. Include the import +/// statement `use std::str::FromStr;` in your code before invoking the `from_str` function. This import ensures +/// that the `from_str` method is available for converting strings to `SHA1` objects. +impl std::str::FromStr for SHA1 { + type Err = String; + + fn from_str(s: &str) -> Result { + let mut h = SHA1::default(); + if s.len() != 40 { + return Err("The length of the string is not 40".to_string()); + } + let bytes = hex::decode(s).map_err(|e| e.to_string())?; + h.0.copy_from_slice(bytes.as_slice()); + Ok(h) + } +} + +/// Implementation of the `SHA1` struct. +/// +/// The naming conventions for the methods in this implementation are designed to be intuitive and self-explanatory: +/// +/// 1. `new` Prefix: +/// Methods starting with `new` are used for computing an SHA-1 hash from given data, signifying the creation of +/// a new `SHA1` instance. For example, `pub fn new(data: &Vec) -> SHA1` takes a byte vector and calculates its SHA-1 hash. +/// +/// 2. `from` Prefix: +/// Methods beginning with `from` are intended for creating a `SHA1` instance from an existing, pre-calculated value. +/// This implies direct derivation of the `SHA1` object from the provided input. For instance, `pub fn from_bytes(bytes: &[u8]) -> SHA1` +/// constructs a `SHA1` from a 20-byte array representing an SHA-1 hash. +/// +/// 3. `to` Prefix: +/// Methods with the `to` prefix are used for outputting the `SHA1` value in various formats. This prefix indicates a transformation or +/// conversion of the `SHA1` instance into another representation. For example, `pub fn to_string(self) -> String` converts the SHA1 +/// value to a plain hexadecimal string, and `pub fn to_data(self) -> Vec` converts it into a byte vector. The `to` prefix +/// thus serves as a clear indicator that the method is exporting or transforming the SHA1 value into a different format. +/// +/// These method naming conventions (`new`, `from`, `to`) provide clarity and predictability in the API, making it easier for users +/// to understand the intended use and functionality of each method within the `SHA1` struct. +impl SHA1 { + // The size of the SHA-1 hash value in bytes + pub const SIZE: usize = 20; + + /// Calculate the SHA-1 hash of the byte slice, then create a Hash value + pub fn new(data: &[u8]) -> SHA1 { + let h = sha1::Sha1::digest(data); + SHA1::from_bytes(h.as_slice()) + } + /// Create a Hash from the object type and data + /// This function is used to create a SHA1 hash from the object type and data. + /// It constructs a byte vector that includes the object type, the size of the data, + /// and the data itself, and then computes the SHA1 hash of this byte vector. + /// + /// Hash compute <- {Object Type}+{ }+{Object Size（before compress）}+{\x00}+{Object Content(before compress)} + pub fn from_type_and_data(object_type: ObjectType, data: &[u8]) -> SHA1 { + let mut d: Vec = Vec::new(); + d.extend(object_type.to_data().unwrap()); + d.push(b' '); + d.extend(data.len().to_string().as_bytes()); + d.push(b'\x00'); + d.extend(data); + SHA1::new(&d) + } + + /// Create Hash from a byte array, which is a 20-byte array already calculated + pub fn from_bytes(bytes: &[u8]) -> SHA1 { + let mut h = SHA1::default(); + h.0.copy_from_slice(bytes); + h + } + + /// Read the Hash value from the stream + /// This function will read exactly 20 bytes from the stream + pub fn from_stream(data: &mut impl io::Read) -> io::Result { + let mut h = SHA1::default(); + data.read_exact(&mut h.0)?; + Ok(h) + } + + /// Export sha1 value to String with the color + pub fn to_color_str(self) -> String { + self.to_string().red().bold().to_string() + } + + /// Export sha1 value to a byte array + pub fn to_data(self) -> Vec { + self.0.to_vec() + } + + /// [`core::fmt::Display`] is somewhat expensive, + /// use this hack to get a string more efficiently + pub fn _to_string(&self) -> String { + hex::encode(self.0) + } +} + +#[cfg(test)] +mod tests { + + use std::io::BufReader; + use std::io::Read; + use std::io::Seek; + use std::io::SeekFrom; + use std::str::FromStr; + use std::{env, path::PathBuf}; + + use crate::hash::SHA1; + + #[test] + fn test_sha1_new() { + // Example input + let data = "Hello, world!".as_bytes(); + + // Generate SHA1 hash from the input data + let sha1 = SHA1::new(data); + + // Known SHA1 hash for "Hello, world!" + let expected_sha1_hash = "943a702d06f34599aee1f8da8ef9f7296031d699"; + + assert_eq!(sha1.to_string(), expected_sha1_hash); + } + + #[test] + fn test_signature_without_delta() { + let mut source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + source.push("tests/data/packs/pack-1d0e6c14760c956c173ede71cb28f33d921e232f.pack"); + + let f = std::fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + + buffered.seek(SeekFrom::End(-20)).unwrap(); + let mut buffer = vec![0; 20]; + buffered.read_exact(&mut buffer).unwrap(); + let signature = SHA1::from_bytes(buffer.as_ref()); + assert_eq!( + signature.to_string(), + "1d0e6c14760c956c173ede71cb28f33d921e232f" + ); + } + + #[test] + fn test_sha1_from_bytes() { + let sha1 = SHA1::from_bytes(&[ + 0x8a, 0xb6, 0x86, 0xea, 0xfe, 0xb1, 0xf4, 0x47, 0x02, 0x73, 0x8c, 0x8b, 0x0f, 0x24, + 0xf2, 0x56, 0x7c, 0x36, 0xda, 0x6d, + ]); + + assert_eq!(sha1.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); + } + + #[test] + fn test_from_stream() { + let source = [ + 0x8a, 0xb6, 0x86, 0xea, 0xfe, 0xb1, 0xf4, 0x47, 0x02, 0x73, 0x8c, 0x8b, 0x0f, 0x24, + 0xf2, 0x56, 0x7c, 0x36, 0xda, 0x6d, + ]; + let mut reader = std::io::Cursor::new(source); + let sha1 = SHA1::from_stream(&mut reader).unwrap(); + assert_eq!(sha1.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); + } + + #[test] + fn test_sha1_from_str() { + let hash_str = "8ab686eafeb1f44702738c8b0f24f2567c36da6d"; + + match SHA1::from_str(hash_str) { + Ok(hash) => { + assert_eq!(hash.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); + } + Err(e) => println!("Error: {e}"), + } + } + + #[test] + fn test_sha1_to_string() { + let hash_str = "8ab686eafeb1f44702738c8b0f24f2567c36da6d"; + + match SHA1::from_str(hash_str) { + Ok(hash) => { + assert_eq!(hash.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); + } + Err(e) => println!("Error: {e}"), + } + } + + #[test] + fn test_sha1_to_data() { + let hash_str = "8ab686eafeb1f44702738c8b0f24f2567c36da6d"; + + match SHA1::from_str(hash_str) { + Ok(hash) => { + assert_eq!( + hash.to_data(), + vec![ + 0x8a, 0xb6, 0x86, 0xea, 0xfe, 0xb1, 0xf4, 0x47, 0x02, 0x73, 0x8c, 0x8b, + 0x0f, 0x24, 0xf2, 0x56, 0x7c, 0x36, 0xda, 0x6d + ] + ); + } + Err(e) => println!("Error: {e}"), + } + } +} diff --git a/mercury/src/internal/index.rs b/mercury/src/internal/index.rs new file mode 100644 index 00000000..3504805a --- /dev/null +++ b/mercury/src/internal/index.rs @@ -0,0 +1,582 @@ +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; +use sha1::{Digest, Sha1}; +use std::collections::BTreeMap; +use std::fmt::{Display, Formatter}; +use std::fs::{self, File}; +use std::io; +use std::io::{BufReader, Read, Write}; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::errors::GitError; +use crate::hash::SHA1; +use crate::internal::pack::wrapper::Wrapper; +use crate::utils; + +#[derive(PartialEq, Eq, Debug, Clone)] +pub struct Time { + seconds: u32, + nanos: u32, +} +impl Time { + pub fn from_stream(stream: &mut impl Read) -> Result { + let seconds = stream.read_u32::()?; + let nanos = stream.read_u32::()?; + Ok(Time { seconds, nanos }) + } + + #[allow(dead_code)] + fn to_system_time(&self) -> SystemTime { + UNIX_EPOCH + std::time::Duration::new(self.seconds.into(), self.nanos) + } + + pub fn from_system_time(system_time: SystemTime) -> Self { + match system_time.duration_since(UNIX_EPOCH) { + Ok(duration) => { + let seconds = duration + .as_secs() + .try_into() + .expect("Time is too far in the future"); + let nanos = duration.subsec_nanos(); + Time { seconds, nanos } + } + Err(_) => panic!("Time is before the UNIX epoch"), + } + } +} +impl Display for Time { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}:{}", self.seconds, self.nanos) + } +} + +/// 16 bits +#[derive(Debug)] +pub struct Flags { + pub assume_valid: bool, + pub extended: bool, // must be 0 in v2 + pub stage: u8, // 2-bit during merge + pub name_length: u16, // 12-bit +} + +impl From for Flags { + fn from(flags: u16) -> Self { + Flags { + assume_valid: flags & 0x8000 != 0, + extended: flags & 0x4000 != 0, + stage: ((flags & 0x3000) >> 12) as u8, + name_length: flags & 0xFFF, + } + } +} + +impl TryInto for &Flags { + type Error = &'static str; + fn try_into(self) -> Result { + let mut flags = 0u16; + if self.assume_valid { + flags |= 0x8000; // 16 + } + if self.extended { + flags |= 0x4000; // 15 + } + flags |= (self.stage as u16) << 12; // 13-14 + if self.name_length > 0xFFF { + return Err("Name length is too long"); + } + flags |= self.name_length; // 0-11 + Ok(flags) + } +} + +impl Flags { + pub fn new(name_len: u16) -> Self { + Flags { + assume_valid: true, + extended: false, + stage: 0, + name_length: name_len, + } + } +} + +pub struct IndexEntry { + pub ctime: Time, + pub mtime: Time, + pub dev: u32, // 0 for windows + pub ino: u32, // 0 for windows + pub mode: u32, // 0o100644 // 4-bit object type + 3-bit unused + 9-bit unix permission + pub uid: u32, // 0 for windows + pub gid: u32, // 0 for windows + pub size: u32, + pub hash: SHA1, + pub flags: Flags, + pub name: String, +} +impl Display for IndexEntry { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "IndexEntry {{ ctime: {}, mtime: {}, dev: {}, ino: {}, mode: {:o}, uid: {}, gid: {}, size: {}, hash: {}, flags: {:?}, name: {} }}", + self.ctime, + self.mtime, + self.dev, + self.ino, + self.mode, + self.uid, + self.gid, + self.size, + self.hash, + self.flags, + self.name + ) + } +} + +impl IndexEntry { + /** Metadata must be got by [fs::symlink_metadata] to avoid following symlink */ + pub fn new(meta: &fs::Metadata, hash: SHA1, name: String) -> Self { + let mut entry = IndexEntry { + ctime: Time::from_system_time(meta.created().unwrap()), + mtime: Time::from_system_time(meta.modified().unwrap()), + dev: 0, + ino: 0, + uid: 0, + gid: 0, + size: meta.len() as u32, + hash, + flags: Flags::new(name.len() as u16), + name, + mode: 0o100644, + }; + #[cfg(unix)] + { + entry.dev = meta.dev() as u32; + entry.ino = meta.ino() as u32; + entry.uid = meta.uid(); + entry.gid = meta.gid(); + + entry.mode = match meta.mode() & 0o170000/* file mode */ { + 0o100000 => { + match meta.mode() & 0o111 { + 0 => 0o100644, // no execute permission + _ => 0o100755, // with execute permission + } + } + 0o120000 => 0o120000, // symlink + _ => entry.mode, // keep the original mode + } + } + #[cfg(windows)] + { + if meta.is_symlink() { + entry.mode = 0o120000; + } + } + entry + } + + /// - `file`: **to workdir path** + /// - `workdir`: absolute or relative path + pub fn new_from_file(file: &Path, hash: SHA1, workdir: &Path) -> io::Result { + let name = file.to_str().unwrap().to_string(); + let file_abs = workdir.join(file); + let meta = fs::symlink_metadata(file_abs)?; // without following symlink + let index = IndexEntry::new(&meta, hash, name); + Ok(index) + } + + pub fn new_from_blob(name: String, hash: SHA1, size: u32) -> Self { + IndexEntry { + ctime: Time { + seconds: 0, + nanos: 0, + }, + mtime: Time { + seconds: 0, + nanos: 0, + }, + dev: 0, + ino: 0, + mode: 0o100644, + uid: 0, + gid: 0, + size, + hash, + flags: Flags::new(name.len() as u16), + name, + } + } +} + +/// see [index-format](https://git-scm.com/docs/index-format) +///
to Working Dir relative path +pub struct Index { + entries: BTreeMap<(String, u8), IndexEntry>, +} + +impl Index { + pub fn new() -> Self { + Index { + entries: BTreeMap::new(), + } + } + + fn check_header(file: &mut impl Read) -> Result { + let mut magic = [0; 4]; + file.read_exact(&mut magic)?; + if magic != *b"DIRC" { + return Err(GitError::InvalidIndexHeader( + String::from_utf8_lossy(&magic).to_string(), + )); + } + + let version = file.read_u32::()?; + // only support v2 now + if version != 2 { + return Err(GitError::InvalidIndexHeader(version.to_string())); + } + + let entries = file.read_u32::()?; + Ok(entries) + } + + pub fn size(&self) -> usize { + self.entries.len() + } + + pub fn from_file(path: impl AsRef) -> Result { + let file = File::open(path.as_ref())?; // read-only + let total_size = file.metadata()?.len(); + let file = &mut Wrapper::new(BufReader::new(file)); // TODO move Wrapper & utils to a common module + + let num = Index::check_header(file)?; + let mut index = Index::new(); + + for _ in 0..num { + let mut entry = IndexEntry { + ctime: Time::from_stream(file)?, + mtime: Time::from_stream(file)?, + dev: file.read_u32::()?, //utils::read_u32_be(file)?, + ino: file.read_u32::()?, + mode: file.read_u32::()?, + uid: file.read_u32::()?, + gid: file.read_u32::()?, + size: file.read_u32::()?, + hash: utils::read_sha1(file)?, + flags: Flags::from(file.read_u16::()?), + name: String::new(), + }; + let name_len = entry.flags.name_length as usize; + let mut name = vec![0; name_len]; + file.read_exact(&mut name)?; + // The exact encoding is undefined, but the '.' and '/' characters are encoded in 7-bit ASCII + entry.name = String::from_utf8(name)?; // TODO check the encoding + index + .entries + .insert((entry.name.clone(), entry.flags.stage), entry); + + // 1-8 nul bytes as necessary to pad the entry to a multiple of eight bytes + // while keeping the name NUL-terminated. // so at least 1 byte nul + let padding = 8 - ((22 + name_len) % 8); // 22 = sha1 + flags, others are 40 % 8 == 0 + utils::read_bytes(file, padding)?; + } + + // Extensions + while file.bytes_read() + SHA1::SIZE < total_size as usize { + // The remaining 20 bytes must be checksum + let sign = utils::read_bytes(file, 4)?; + println!("{:?}", String::from_utf8(sign.clone())?); + // If the first byte is 'A'...'Z' the extension is optional and can be ignored. + if sign[0] >= b'A' && sign[0] <= b'Z' { + // Optional extension + let size = file.read_u32::()?; + utils::read_bytes(file, size as usize)?; // Ignore the extension + } else { + // 'link' or 'sdir' extension + return Err(GitError::InvalidIndexFile( + "Unsupported extension".to_string(), + )); + } + } + + // check sum + let file_hash = file.final_hash(); + let check_sum = utils::read_sha1(file)?; + if file_hash != check_sum { + return Err(GitError::InvalidIndexFile("Check sum failed".to_string())); + } + assert_eq!(index.size(), num as usize); + Ok(index) + } + + pub fn to_file(&self, path: impl AsRef) -> Result<(), GitError> { + let mut file = File::create(path)?; + let mut hash = Sha1::new(); + + let mut header = Vec::new(); + header.write_all(b"DIRC")?; + header.write_u32::(2u32)?; // version 2 + header.write_u32::(self.entries.len() as u32)?; + file.write_all(&header)?; + hash.update(&header); + + for (_, entry) in self.entries.iter() { + let mut entry_bytes = Vec::new(); + entry_bytes.write_u32::(entry.ctime.seconds)?; + entry_bytes.write_u32::(entry.ctime.nanos)?; + entry_bytes.write_u32::(entry.mtime.seconds)?; + entry_bytes.write_u32::(entry.mtime.nanos)?; + entry_bytes.write_u32::(entry.dev)?; + entry_bytes.write_u32::(entry.ino)?; + entry_bytes.write_u32::(entry.mode)?; + entry_bytes.write_u32::(entry.uid)?; + entry_bytes.write_u32::(entry.gid)?; + entry_bytes.write_u32::(entry.size)?; + entry_bytes.write_all(&entry.hash.0)?; + entry_bytes.write_u16::((&entry.flags).try_into().unwrap())?; + entry_bytes.write_all(entry.name.as_bytes())?; + let padding = 8 - ((22 + entry.name.len()) % 8); + entry_bytes.write_all(&vec![0; padding])?; + + file.write_all(&entry_bytes)?; + hash.update(&entry_bytes); + } + + // Extensions + + // check sum + let file_hash: [u8; 20] = hash.finalize().into(); + file.write_all(&file_hash)?; + Ok(()) + } + + pub fn refresh(&mut self, file: impl AsRef, workdir: &Path) -> Result { + let path = file.as_ref(); + let name = path + .to_str() + .ok_or(GitError::InvalidPathError(format!("{path:?}")))?; + + if let Some(entry) = self.entries.get_mut(&(name.to_string(), 0)) { + let abs_path = workdir.join(path); + let meta = fs::symlink_metadata(&abs_path)?; + // Try creation time; on error, warn and use modification time (or now) + let new_ctime = Time::from_system_time(Self::time_or_now( + "creation time", + &abs_path, + meta.created(), + )); + let new_mtime = Time::from_system_time(Self::time_or_now( + "modification time", + &abs_path, + meta.modified(), + )); + let new_size = meta.len() as u32; + + // re-calculate SHA1 + let mut file = File::open(&abs_path)?; + let mut hasher = Sha1::new(); + io::copy(&mut file, &mut hasher)?; + let new_hash = SHA1::from_bytes(&hasher.finalize()); + + // refresh index + if entry.ctime != new_ctime + || entry.mtime != new_mtime + || entry.size != new_size + || entry.hash != new_hash + { + entry.ctime = new_ctime; + entry.mtime = new_mtime; + entry.size = new_size; + entry.hash = new_hash; + return Ok(true); + } + } + Ok(false) + } + + /// Try to get a timestamp, logging on error, and finally falling back to now. + fn time_or_now(what: &str, path: &Path, res: io::Result) -> SystemTime { + match res { + Ok(ts) => ts, + Err(e) => { + eprintln!( + "warning: failed to get {what} for {path:?}: {e}; using SystemTime::now()", + what = what, + path = path.display() + ); + SystemTime::now() + } + } + } +} + +impl Default for Index { + fn default() -> Self { + Self::new() + } +} + +impl Index { + /// Load index. If it does not exist, return an empty index. + pub fn load(index_file: impl AsRef) -> Result { + let path = index_file.as_ref(); + if !path.exists() { + return Ok(Index::new()); + } + Index::from_file(path) + } + + pub fn update(&mut self, entry: IndexEntry) { + self.add(entry) + } + + pub fn add(&mut self, entry: IndexEntry) { + self.entries + .insert((entry.name.clone(), entry.flags.stage), entry); + } + + pub fn remove(&mut self, name: &str, stage: u8) -> Option { + self.entries.remove(&(name.to_string(), stage)) + } + + pub fn get(&self, name: &str, stage: u8) -> Option<&IndexEntry> { + self.entries.get(&(name.to_string(), stage)) + } + + pub fn tracked(&self, name: &str, stage: u8) -> bool { + self.entries.contains_key(&(name.to_string(), stage)) + } + + pub fn get_hash(&self, file: &str, stage: u8) -> Option { + self.get(file, stage).map(|entry| entry.hash) + } + + pub fn verify_hash(&self, file: &str, stage: u8, hash: &SHA1) -> bool { + let inner_hash = self.get_hash(file, stage); + if let Some(inner_hash) = inner_hash { + &inner_hash == hash + } else { + false + } + } + /// is file modified after last `add` (need hash to confirm content change) + /// - `workdir` is used to rebuild absolute file path + pub fn is_modified(&self, file: &str, stage: u8, workdir: &Path) -> bool { + if let Some(entry) = self.get(file, stage) { + let path_abs = workdir.join(file); + let meta = path_abs.symlink_metadata().unwrap(); + // TODO more fields + let same = entry.ctime + == Time::from_system_time(meta.created().unwrap_or(SystemTime::now())) + && entry.mtime + == Time::from_system_time(meta.modified().unwrap_or(SystemTime::now())) + && entry.size == meta.len() as u32; + + !same + } else { + panic!("File not found in index"); + } + } + + /// Get all entries with the same stage + pub fn tracked_entries(&self, stage: u8) -> Vec<&IndexEntry> { + // ? should use stage or not + self.entries + .iter() + .filter(|(_, entry)| entry.flags.stage == stage) + .map(|(_, entry)| entry) + .collect() + } + + /// Get all tracked files(stage = 0) + pub fn tracked_files(&self) -> Vec { + self.tracked_entries(0) + .iter() + .map(|entry| PathBuf::from(&entry.name)) + .collect() + } + + /// Judge if the file(s) of `dir` is in the index + /// - false if `dir` is a file + pub fn contains_dir_file(&self, dir: &str) -> bool { + let dir = Path::new(dir); + self.entries.iter().any(|((name, _), _)| { + let path = Path::new(name); + path.starts_with(dir) && path != dir // TODO change to is_sub_path! + }) + } + + /// remove all files in `dir` from index + /// - do nothing if `dir` is a file + pub fn remove_dir_files(&mut self, dir: &str) -> Vec { + let dir = Path::new(dir); + let mut removed = Vec::new(); + self.entries.retain(|(name, _), _| { + let path = Path::new(name); + if path.starts_with(dir) && path != dir { + removed.push(name.clone()); + false + } else { + true + } + }); + removed + } + + /// saved to index file + pub fn save(&self, index_file: impl AsRef) -> Result<(), GitError> { + self.to_file(index_file) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_time() { + let time = Time { + seconds: 0, + nanos: 0, + }; + let system_time = time.to_system_time(); + let new_time = Time::from_system_time(system_time); + assert_eq!(time, new_time); + } + + #[test] + fn test_check_header() { + let file = File::open("../tests/data/index/index-2").unwrap(); + let entries = Index::check_header(&mut BufReader::new(file)).unwrap(); + assert_eq!(entries, 2); + } + + #[test] + fn test_index() { + let index = Index::from_file("../tests/data/index/index-760").unwrap(); + assert_eq!(index.size(), 760); + for (_, entry) in index.entries.iter() { + println!("{entry}"); + } + } + + #[test] + fn test_index_to_file() { + let index = Index::from_file("../tests/data/index/index-760").unwrap(); + index.to_file("/tmp/index-760").unwrap(); + let new_index = Index::from_file("/tmp/index-760").unwrap(); + assert_eq!(index.size(), new_index.size()); + } + + #[test] + fn test_index_entry_create() { + let file = Path::new("Cargo.toml"); // use as a normal file + let hash = SHA1::from_bytes(&[0; 20]); + let workdir = Path::new("../"); + let entry = IndexEntry::new_from_file(file, hash, workdir).unwrap(); + println!("{entry}"); + } +} diff --git a/mercury/src/internal/mod.rs b/mercury/src/internal/mod.rs new file mode 100644 index 00000000..8c6da347 --- /dev/null +++ b/mercury/src/internal/mod.rs @@ -0,0 +1,4 @@ +pub mod index; +pub mod object; +pub mod pack; +pub mod zlib; diff --git a/mercury/src/internal/object/blob.rs b/mercury/src/internal/object/blob.rs new file mode 100644 index 00000000..a2a2cc77 --- /dev/null +++ b/mercury/src/internal/object/blob.rs @@ -0,0 +1,116 @@ +//! In Git, a blob (binary large object) is a type of Git object that stores the contents of a file. +//! A blob object contains the binary data of a file, but does not contain any metadata such as +//! the file name or permissions. The structure of a Git blob object is as follows: +//! +//! ```bash +//! blob \0 +//! ``` +//! +//! - `blob` is the object type, indicating that this is a blob object. +//! - `` is the length of the content in bytes, encoded as a string of decimal digits. +//! - `\0` is a null byte, which separates the header from the content. +//! - `` is the binary data of the file, represented as a sequence of bytes. +//! +//! We can create a Git blob object for this file by running the following command: +//! +//! ```bash +//! $ echo "Hello, world!" | git hash-object -w --stdin +//! ``` +//! +//! This will output an SHA-1 hash, which is the ID of the newly created blob object. +//! The contents of the blob object would look something like this: +//! +//! ```bash +//! blob 13\0Hello, world! +//! ``` +//! Git uses blobs to store the contents of files in a repository. Each version of a file is +//! represented by a separate blob object, which can be linked together using Git's commit and tree +//! objects to form a version history of the repository. +//! +use std::fmt::Display; + +use crate::errors::GitError; +use crate::hash::SHA1; +use crate::internal::object::ObjectTrait; +use crate::internal::object::types::ObjectType; + +/// **The Blob Object** +#[derive(Eq, Debug, Clone)] +pub struct Blob { + pub id: SHA1, + pub data: Vec, +} + +impl PartialEq for Blob { + /// The Blob object is equal to another Blob object if their IDs are equal. + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Display for Blob { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "Type: Blob").unwrap(); + writeln!(f, "Size: {}", self.data.len()) + } +} + +impl ObjectTrait for Blob { + /// Creates a new object from a byte slice. + fn from_bytes(data: &[u8], hash: SHA1) -> Result + where + Self: Sized, + { + Ok(Blob { + id: hash, + data: data.to_vec(), + }) + } + + /// Returns the Blob type + fn get_type(&self) -> ObjectType { + ObjectType::Blob + } + + fn get_size(&self) -> usize { + self.data.len() + } + + fn to_data(&self) -> Result, GitError> { + Ok(self.data.clone()) + } +} + +impl Blob { + /// Create a new Blob object from the given content string. + /// - This is a convenience method for creating a Blob from a string. + /// - It converts the string to bytes and then calls `from_content_bytes`. + pub fn from_content(content: &str) -> Self { + let content = content.as_bytes().to_vec(); + Blob::from_content_bytes(content) + } + + /// Create a new Blob object from the given content bytes. + /// - some file content can't be represented as a string (UTF-8), so we need to use bytes. + pub fn from_content_bytes(content: Vec) -> Self { + Blob { + // Calculate the SHA1 hash from the type and content + id: SHA1::from_type_and_data(ObjectType::Blob, &content), + data: content, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_blob_from_content() { + let content = "Hello, world!"; + let blob = Blob::from_content(content); + assert_eq!( + blob.id.to_string(), + "5dd01c177f5d7d1be5346a5bc18a569a7410c2ef" + ); + } +} diff --git a/mercury/src/internal/object/commit.rs b/mercury/src/internal/object/commit.rs new file mode 100644 index 00000000..a0c4b9bb --- /dev/null +++ b/mercury/src/internal/object/commit.rs @@ -0,0 +1,363 @@ +//! The Commit object is a data structure used to represent a specific version of a project's +//! files at a particular point in time. In Git, the commit object is a fundamental data structure +//! that is used to track changes to a repository's files over time. Whenever a developer makes +//! changes to the files in a repository, they create a new commit object that records those changes. +//! +//! Each commit object in Git contains the following information: +//! +//! - A unique SHA-1 hash that identifies the commit. +//! - The author and committer of the commit (which may be different people). +//! - The date and time the commit was made. +//! - A commit message that describes the changes made in the commit. +//! - A reference to the parent commit or commits (in the case of a merge commit) that the new commit is based on. +//! - The contents of the files in the repository at the time the commit was made. +use std::fmt::Display; +use std::str::FromStr; + +use crate::errors::GitError; +use crate::hash::SHA1; +use crate::internal::object::ObjectTrait; +use crate::internal::object::ObjectType; +use crate::internal::object::signature::Signature; +use bincode::{Decode, Encode}; +use bstr::ByteSlice; +use callisto::git_commit; +use callisto::mega_commit; +use serde::Deserialize; +use serde::Serialize; + +/// The `Commit` struct is used to represent a commit object. +/// +/// - The tree object SHA points to the top level tree for this commit, which reflects the complete +/// state of the repository at the time of the commit. The tree object in turn points to blobs and +/// subtrees which represent the files in the repository. +/// - The parent commit SHAs allow Git to construct a linked list of commits and build the full +/// commit history. By chaining together commits in this fashion, Git is able to represent the entire +/// history of a repository with a single commit object at its root. +/// - The author and committer fields contain the name, email address, timestamp and timezone. +/// - The message field contains the commit message, which maybe include signed or DCO. +#[derive(Eq, Debug, Clone, Serialize, Deserialize, Decode, Encode)] +pub struct Commit { + pub id: SHA1, + pub tree_id: SHA1, + pub parent_commit_ids: Vec, + pub author: Signature, + pub committer: Signature, + pub message: String, +} +impl PartialEq for Commit { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Display for Commit { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "tree: {}", self.tree_id)?; + for parent in self.parent_commit_ids.iter() { + writeln!(f, "parent: {parent}")?; + } + writeln!(f, "author {}", self.author)?; + writeln!(f, "committer {}", self.committer)?; + writeln!(f, "{}", self.message) + } +} + +impl Commit { + pub fn new( + author: Signature, + committer: Signature, + tree_id: SHA1, + parent_commit_ids: Vec, + message: &str, + ) -> Commit { + let mut commit = Commit { + id: SHA1::default(), + tree_id, + parent_commit_ids, + author, + committer, + message: message.to_string(), + }; + // Calculate the hash of the commit object + // The hash is calculated from the type and data of the commit object + let hash = SHA1::from_type_and_data(ObjectType::Commit, &commit.to_data().unwrap()); + commit.id = hash; + commit + } + + /// Creates a new commit object from a tree ID and a list of parent commit IDs. + /// This function generates the author and committer signatures using the current time + /// and a fixed email address. + /// It also sets the commit message to the provided string. + /// # Arguments + /// - `tree_id`: The SHA1 hash of the tree object that this commit points to. + /// - `parent_commit_ids`: A vector of SHA1 hashes of the parent commits. + /// - `message`: A string containing the commit message. + /// # Returns + /// A new `Commit` object with the specified tree ID, parent commit IDs, and commit message. + /// The author and committer signatures are generated using the current time and a fixed email address. + pub fn from_tree_id(tree_id: SHA1, parent_commit_ids: Vec, message: &str) -> Commit { + let author = Signature::from_data( + format!( + "author mega {} +0800", + chrono::Utc::now().timestamp() + ) + .to_string() + .into_bytes(), + ) + .unwrap(); + let committer = Signature::from_data( + format!( + "committer mega {} +0800", + chrono::Utc::now().timestamp() + ) + .to_string() + .into_bytes(), + ) + .unwrap(); + Commit::new(author, committer, tree_id, parent_commit_ids, message) + } + + /// Formats the commit message by extracting the first meaningful line. + /// + /// If the message contains a PGP signature, it returns the first non-empty line + /// after the signature block. Otherwise, it returns the first non-empty line + /// in the message. If no such line exists, it returns the original message. + pub fn format_message(&self) -> String { + let mut lines = self.message.lines(); + + // If a PGP signature is present, skip lines until after the signature ends + if let Some(pos) = self + .message + .lines() + .position(|line| line.contains("-----END PGP SIGNATURE-----")) + { + return self + .message + .lines() + .skip(pos + 1) + .find(|line| !line.trim().is_empty()) + .map(|line| line.to_owned()) + .unwrap_or_else(|| self.message.clone()); + } + + // Return the first non-empty line from the start + lines + .find(|line| !line.trim().is_empty()) + .map(|line| line.to_owned()) + .unwrap_or_else(|| self.message.clone()) + } +} + +impl ObjectTrait for Commit { + fn from_bytes(data: &[u8], hash: SHA1) -> Result + where + Self: Sized, + { + let mut commit = data; + // Find the tree id and remove it from the data + let tree_end = commit.find_byte(0x0a).unwrap(); + let tree_id: SHA1 = SHA1::from_str( + String::from_utf8(commit[5..tree_end].to_owned()) // 5 is the length of "tree " + .unwrap() + .as_str(), + ) + .unwrap(); + let binding = commit[tree_end + 1..].to_vec(); // Move past the tree id + commit = &binding; + + // Find the parent commit ids and remove them from the data + let author_begin = commit.find("author").unwrap(); + // Find all parent commit ids + // The parent commit ids are all the lines that start with "parent " + // We can use find_iter to find all occurrences of "parent " + // and then extract the SHA1 hashes from them. + let parent_commit_ids: Vec = commit[..author_begin] + .find_iter("parent") + .map(|parent| { + let parent_end = commit[parent..].find_byte(0x0a).unwrap(); + SHA1::from_str( + // 7 is the length of "parent " + String::from_utf8(commit[parent + 7..parent + parent_end].to_owned()) + .unwrap() + .as_str(), + ) + .unwrap() + }) + .collect(); + let binding = commit[author_begin..].to_vec(); + commit = &binding; + + // Find the author and committer and remove them from the data + // 0x0a is the newline character + let author = + Signature::from_data(commit[..commit.find_byte(0x0a).unwrap()].to_vec()).unwrap(); + + let binding = commit[commit.find_byte(0x0a).unwrap() + 1..].to_vec(); + commit = &binding; + let committer = + Signature::from_data(commit[..commit.find_byte(0x0a).unwrap()].to_vec()).unwrap(); + + // The rest is the message + let message = unsafe { + String::from_utf8_unchecked(commit[commit.find_byte(0x0a).unwrap() + 1..].to_vec()) + }; + Ok(Commit { + id: hash, + tree_id, + parent_commit_ids, + author, + committer, + message, + }) + } + + fn get_type(&self) -> ObjectType { + ObjectType::Commit + } + + fn get_size(&self) -> usize { + 0 + } + + /// [Git-Internals-Git-Objects](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects) + fn to_data(&self) -> Result, GitError> { + let mut data = Vec::new(); + + data.extend(b"tree "); + data.extend(self.tree_id.to_string().as_bytes()); + data.extend(&[0x0a]); + + for parent_tree_id in &self.parent_commit_ids { + data.extend(b"parent "); + data.extend(parent_tree_id.to_string().as_bytes()); + data.extend(&[0x0a]); + } + + data.extend(self.author.to_data()?); + data.extend(&[0x0a]); + data.extend(self.committer.to_data()?); + data.extend(&[0x0a]); + // Important! or Git Server can't parse & reply: unpack-objects abnormal exit + // We can move [0x0a] to message instead here. + // data.extend(&[0x0a]); + data.extend(self.message.as_bytes()); + + Ok(data) + } +} +fn commit_from_model( + commit_id: &str, + tree: &str, + parents_id: &serde_json::Value, + author: Option, + committer: Option, + message: Option, +) -> Commit { + Commit { + id: SHA1::from_str(commit_id).unwrap(), + tree_id: SHA1::from_str(tree).unwrap(), + parent_commit_ids: parents_id + .as_array() + .unwrap() + .iter() + .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap()) + .collect(), + author: Signature::from_data(author.unwrap().into()).unwrap(), + committer: Signature::from_data(committer.unwrap().into()).unwrap(), + message: message.unwrap(), + } +} + +impl From for Commit { + fn from(value: mega_commit::Model) -> Self { + commit_from_model( + &value.commit_id, + &value.tree, + &value.parents_id, + value.author, + value.committer, + value.content, + ) + } +} + +impl From for Commit { + fn from(value: git_commit::Model) -> Self { + commit_from_model( + &value.commit_id, + &value.tree, + &value.parents_id, + value.author, + value.committer, + value.content, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + + fn basic_commit() -> Commit { + let raw_commit = br#"tree 341e54913a3a43069f2927cc0f703e5a9f730df1 +author benjamin.747 1757467768 +0800 +committer benjamin.747 1757491219 +0800 +gpgsig -----BEGIN PGP SIGNATURE----- + + iQJNBAABCAA3FiEEs4MaYUV7JcjxsVMPyqxGczTZ6K4FAmjBMC4ZHGJlbmphbWlu + Ljc0N0BvdXRsb29rLmNvbQAKCRDKrEZzNNnorj73EADNpsyLAHsB3NgoeH+uy9Vq + G2+LRtlvqv3QMK7vbQUadXHlQYWk25SIk+WJ1kG1AnUy5fqOrLSDTA1ny+qwpH8O + +2sKCF/S1wlzqGWjCcRH5/ir9srsGIn9HbNqBjmU22NJ6Dt2jnqoUvtWfPwyqwWg + VpjYlj390cFdXTpH5hMvtlmUQB+zCSKtWQW2Ur64h/UsGtllARlACi+KHQQmA2/p + FLWNddvfJQpPM597DkGohQTD68g0PqOBhUkOHduHq7VHy68DVW+07bPNXK8JhJ8S + 4dyV1sZwcVcov0GcKl0wUbEqzy4gf+zV7DQhkfrSRQMBdo5vCWahYj1AbgaTiu8a + hscshYDuWWqpxBU/+nCxOPskV29uUG1sRyXp3DqmKJZpnO9CVdw3QaVrqnMEeh2S + t/wYRI9aI1A+Mi/DETom5ifTVygMkK+3m1h7pAMOlblFEdZx2sDXPRG2IEUcatr4 + Jb2+7PUJQXxUQnwHC7xHHxRh6a2h8TfEJfSoEyrgzxZ0CRxJ6XMJaJu0UwZ2xMsx + Lgmeu6miB/imwxz5R5RL2yVHbgllSlO5l12AIeBaPoarKXYPSALigQnKCXu5OM3x + Jq5qsSGtxdr6S1VgLyYHR4o69bQjzBp9K47J3IXqvrpo/ZiO/6Mspk2ZRWhGj82q + e3qERPp5b7+hA+M7jKPyJg== + =UeLf + -----END PGP SIGNATURE----- + +test parse commit from bytes +"#; + + let hash = SHA1::from_str("57d7685c60213a9da465cf900f31933be3a7ee39").unwrap(); + Commit::from_bytes(raw_commit, hash).unwrap() + } + + #[test] + fn test_from_bytes_with_gpgsig() { + let commit = basic_commit(); + + assert_eq!( + commit.id, + SHA1::from_str("57d7685c60213a9da465cf900f31933be3a7ee39").unwrap() + ); + + assert_eq!( + commit.tree_id, + SHA1::from_str("341e54913a3a43069f2927cc0f703e5a9f730df1").unwrap() + ); + + assert_eq!(commit.author.name, "benjamin.747"); + assert_eq!(commit.author.email, "benjamin.747@outlook.com"); + + assert_eq!(commit.committer.name, "benjamin.747"); + + // check message content（must contains gpgsig and content） + assert!(commit.message.contains("-----BEGIN PGP SIGNATURE-----")); + assert!(commit.message.contains("-----END PGP SIGNATURE-----")); + assert!(commit.message.contains("test parse commit from bytes")); + } + + #[test] + fn test_format_message_with_pgp_signature() { + let commit = basic_commit(); + assert_eq!(commit.format_message(), "test parse commit from bytes"); + } +} diff --git a/mercury/src/internal/object/mod.rs b/mercury/src/internal/object/mod.rs new file mode 100644 index 00000000..4c2661de --- /dev/null +++ b/mercury/src/internal/object/mod.rs @@ -0,0 +1,47 @@ +pub mod blob; +pub mod commit; +pub mod signature; +pub mod tag; +pub mod tree; +pub mod types; +pub mod utils; + +use std::{ + fmt::Display, + io::{BufRead, Read}, + str::FromStr, +}; + +use sha1::Digest; + +use crate::internal::object::types::ObjectType; +use crate::internal::zlib::stream::inflate::ReadBoxed; +use crate::{errors::GitError, hash::SHA1}; + +pub trait ObjectTrait: Send + Sync + Display { + /// Creates a new object from a byte slice. + fn from_bytes(data: &[u8], hash: SHA1) -> Result + where + Self: Sized; + + /// Generate a new Object from a `ReadBoxed`. + /// the input size,is only for new a vec with directive space allocation + /// the input data stream and output object should be plain base object . + fn from_buf_read(read: &mut ReadBoxed, size: usize) -> Self + where + Self: Sized, + { + let mut content: Vec = Vec::with_capacity(size); + read.read_to_end(&mut content).unwrap(); + let h = read.hash.clone(); + let hash_str = h.finalize(); + Self::from_bytes(&content, SHA1::from_str(&format!("{hash_str:x}")).unwrap()).unwrap() + } + + /// Returns the type of the object. + fn get_type(&self) -> ObjectType; + + fn get_size(&self) -> usize; + + fn to_data(&self) -> Result, GitError>; +} diff --git a/mercury/src/internal/object/signature.rs b/mercury/src/internal/object/signature.rs new file mode 100644 index 00000000..4542f2c2 --- /dev/null +++ b/mercury/src/internal/object/signature.rs @@ -0,0 +1,309 @@ +//! In a Git commit, the author signature contains the name, email address, timestamp, and timezone +//! of the person who authored the commit. This information is stored in a specific format, which +//! consists of the following fields: +//! +//! - Name: The name of the author, encoded as a UTF-8 string. +//! - Email: The email address of the author, encoded as a UTF-8 string. +//! - Timestamp: The timestamp of when the commit was authored, encoded as a decimal number of seconds +//! since the Unix epoch (January 1, 1970, 00:00:00 UTC). +//! - Timezone: The timezone offset of the author's local time from Coordinated Universal Time (UTC), +//! encoded as a string in the format "+HHMM" or "-HHMM". +//! +use std::{fmt::Display, str::FromStr}; + +use bincode::{Decode, Encode}; +use bstr::ByteSlice; +use chrono::Offset; +use serde::{Deserialize, Serialize}; + +use crate::errors::GitError; + +/// In addition to the author signature, Git also includes a "committer" signature, which indicates +/// who committed the changes to the repository. The committer signature is similar in structure to +/// the author signature, but includes the name, email address, and timestamp of the committer instead. +/// This can be useful in situations where multiple people are working on a project and changes are +/// being reviewed and merged by someone other than the original author. +/// +/// In the following example, it's has the only one who authored and committed. +/// ```bash +/// author Eli Ma 1678102132 +0800 +/// committer Quanyi Ma 1678102132 +0800 +/// ``` +/// +/// So, we design a `SignatureType` enum to indicate the signature type. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Decode, Encode)] +pub enum SignatureType { + Author, + Committer, + Tagger, +} + +impl Display for SignatureType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SignatureType::Author => write!(f, "author"), + SignatureType::Committer => write!(f, "committer"), + SignatureType::Tagger => write!(f, "tagger"), + } + } +} +impl FromStr for SignatureType { + type Err = GitError; + /// The `from_str` method is used to convert a string to a `SignatureType` enum. + fn from_str(s: &str) -> Result { + match s { + "author" => Ok(SignatureType::Author), + "committer" => Ok(SignatureType::Committer), + "tagger" => Ok(SignatureType::Tagger), + _ => Err(GitError::InvalidSignatureType(s.to_string())), + } + } +} +impl SignatureType { + /// The `from_data` method is used to convert a `Vec` to a `SignatureType` enum. + pub fn from_data(data: Vec) -> Result { + let s = String::from_utf8(data.to_vec())?; + SignatureType::from_str(s.as_str()) + } + + /// The `to_bytes` method is used to convert a `SignatureType` enum to a `Vec`. + pub fn to_bytes(&self) -> Vec { + match self { + SignatureType::Author => "author".to_string().into_bytes(), + SignatureType::Committer => "committer".to_string().into_bytes(), + SignatureType::Tagger => "tagger".to_string().into_bytes(), + } + } +} + +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Decode, Encode)] +pub struct Signature { + pub signature_type: SignatureType, + pub name: String, + pub email: String, + pub timestamp: usize, + pub timezone: String, +} + +impl Display for Signature { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "{} <{}>", self.name, self.email).unwrap(); + // format the timestamp to a human-readable date format + let date = + chrono::DateTime::::from_timestamp(self.timestamp as i64, 0).unwrap(); + writeln!(f, "Date: {} {}", date, self.timezone) + } +} + +impl Signature { + pub fn from_data(data: Vec) -> Result { + // Make a mutable copy of the input data vector. + let mut sign = data; + + // Find the index of the first space byte in the data vector. + let name_start = sign.find_byte(0x20).unwrap(); + + // Parse the author name from the bytes up to the first space byte. + // If the parsing fails, unwrap will panic. + let signature_type = SignatureType::from_data(sign[..name_start].to_vec())?; + + let (name, email) = { + let email_start = sign.find_byte(0x3C).unwrap(); + let email_end = sign.find_byte(0x3E).unwrap(); + + unsafe { + ( + sign[name_start + 1..email_start - 1] + .to_str_unchecked() + .to_string(), + sign[email_start + 1..email_end] + .to_str_unchecked() + .to_string(), + ) + } + }; + + // Update the data vector to remove the author and email bytes. + sign = sign[sign.find_byte(0x3E).unwrap() + 2..].to_vec(); + + // Find the index of the second space byte in the updated data vector. + let timestamp_split = sign.find_byte(0x20).unwrap(); + + // Parse the timestamp integer from the bytes up to the second space byte. + // If the parsing fails, unwrap will panic. + let timestamp = unsafe { + sign[0..timestamp_split] + .to_str_unchecked() + .parse::() + .unwrap() + }; + + // Parse the timezone string from the bytes after the second space byte. + // If the parsing fails, unwrap will panic. + let timezone = unsafe { sign[timestamp_split + 1..].to_str_unchecked().to_string() }; + + // Return a Result object indicating success + Ok(Signature { + signature_type, + name, + email, + timestamp, + timezone, + }) + } + + pub fn to_data(&self) -> Result, GitError> { + // Create a new empty vector to store the encoded data. + let mut sign = Vec::new(); + + // Append the author name bytes to the data vector, followed by a space byte. + sign.extend_from_slice(&self.signature_type.to_bytes()); + sign.extend_from_slice(&[0x20]); + + // Append the name bytes to the data vector, followed by a space byte. + sign.extend_from_slice(self.name.as_bytes()); + sign.extend_from_slice(&[0x20]); + + // Append the email address bytes to the data vector, enclosed in angle brackets. + sign.extend_from_slice(format!("<{}>", self.email).as_bytes()); + sign.extend_from_slice(&[0x20]); + + // Append the timestamp integer bytes to the data vector, followed by a space byte. + sign.extend_from_slice(self.timestamp.to_string().as_bytes()); + sign.extend_from_slice(&[0x20]); + + // Append the timezone string bytes to the data vector. + sign.extend_from_slice(self.timezone.as_bytes()); + + // Return the data vector as a Result object indicating success. + Ok(sign) + } + + /// Represents a signature with author, email, timestamp, and timezone information. + pub fn new(sign_type: SignatureType, author: String, email: String) -> Signature { + // Get the current local time (with timezone) + let local_time = chrono::Local::now(); + + // Get the offset from UTC in minutes (local time - UTC time) + let offset = local_time.offset().fix().local_minus_utc(); + + // Calculate the hours part of the offset (divide by 3600 to convert from seconds to hours) + let hours = offset / 60 / 60; + + // Calculate the minutes part of the offset (remaining minutes after dividing by 60) + let minutes = offset / 60 % 60; + + // Format the offset as a string (e.g., "+0800", "-0300", etc.) + let offset_str = format!("{hours:+03}{minutes:02}"); + + // Return the Signature struct with the provided information + Signature { + signature_type: sign_type, // The type of signature (e.g., commit, merge) + name: author, // The author's name + email, // The author's email + timestamp: chrono::Utc::now().timestamp() as usize, // The timestamp of the signature (seconds since Unix epoch) + timezone: offset_str, // The timezone offset (e.g., "+0800") + } + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::DateTime; + + use crate::internal::object::signature::{Signature, SignatureType}; + + #[test] + fn test_signature_type_from_str() { + assert_eq!( + SignatureType::from_str("author").unwrap(), + SignatureType::Author + ); + + assert_eq!( + SignatureType::from_str("committer").unwrap(), + SignatureType::Committer + ); + } + + #[test] + fn test_signature_type_from_data() { + assert_eq!( + SignatureType::from_data("author".to_string().into_bytes()).unwrap(), + SignatureType::Author + ); + + assert_eq!( + SignatureType::from_data("committer".to_string().into_bytes()).unwrap(), + SignatureType::Committer + ); + } + + #[test] + fn test_signature_type_to_bytes() { + assert_eq!( + SignatureType::Author.to_bytes(), + "author".to_string().into_bytes() + ); + + assert_eq!( + SignatureType::Committer.to_bytes(), + "committer".to_string().into_bytes() + ); + } + + #[test] + fn test_signature_new_from_data() { + let sign = Signature::from_data( + "author Quanyi Ma 1678101573 +0800" + .to_string() + .into_bytes(), + ) + .unwrap(); + + assert_eq!(sign.signature_type, SignatureType::Author); + assert_eq!(sign.name, "Quanyi Ma"); + assert_eq!(sign.email, "eli@patch.sh"); + assert_eq!(sign.timestamp, 1678101573); + assert_eq!(sign.timezone, "+0800"); + } + + #[test] + fn test_signature_to_data() { + let sign = Signature::from_data( + "committer Quanyi Ma 1678101573 +0800" + .to_string() + .into_bytes(), + ) + .unwrap(); + + let dest = sign.to_data().unwrap(); + + assert_eq!( + dest, + "committer Quanyi Ma 1678101573 +0800" + .to_string() + .into_bytes() + ); + } + + /// When the test case run in the GitHub Action, the timezone is +0000, so we ignore it. + #[test] + #[ignore] + fn test_signature_with_time() { + let sign = Signature::new( + SignatureType::Author, + "MEGA".to_owned(), + "admin@mega.com".to_owned(), + ); + assert_eq!(sign.signature_type, SignatureType::Author); + assert_eq!(sign.name, "MEGA"); + assert_eq!(sign.email, "admin@mega.com"); + // assert_eq!(sign.timezone, "+0800");//it depends on the local timezone + + let naive_datetime = DateTime::from_timestamp(sign.timestamp as i64, 0).unwrap(); + println!("Formatted DateTime: {}", naive_datetime.naive_local()); + } +} diff --git a/mercury/src/internal/object/tag.rs b/mercury/src/internal/object/tag.rs new file mode 100644 index 00000000..bd984281 --- /dev/null +++ b/mercury/src/internal/object/tag.rs @@ -0,0 +1,220 @@ +//! In Git objects there are two types of tags: Lightweight tags and annotated tags. +//! +//! A lightweight tag is simply a pointer to a specific commit in Git's version history, +//! without any additional metadata or information associated with it. It is created by +//! running the `git tag` command with a name for the tag and the commit hash that it points to. +//! +//! An annotated tag, on the other hand, is a Git object in its own right, and includes +//! metadata such as the tagger's name and email address, the date and time the tag was created, +//! and a message describing the tag. It is created by running the `git tag -a` command with +//! a name for the tag, the commit hash that it points to, and the additional metadata that +//! should be associated with the tag. +//! +//! When you create a tag in Git, whether it's a lightweight or annotated tag, Git creates a +//! new object in its object database to represent the tag. This object includes the name of the +//! tag, the hash of the commit it points to, and any additional metadata associated with the +//! tag (in the case of an annotated tag). +//! +//! There is no difference in binary format between lightweight tags and annotated tags in Git, +//! as both are represented using the same lightweight object format in Git's object database. +//! +//! The lightweight tag is a reference to a specific commit in Git's version history, not be stored +//! as a separate object in Git's object database. This means that if you create a lightweight tag +//! and then move the tag to a different commit, the tag will still point to the original commit. +//! +//! The lightweight just a text file with the commit hash in it, and the file name is the tag name. +//! If one of -a, -s, or -u \ is passed, the command creates a tag object, and requires a tag +//! message. Unless -m \ or -F \ is given, an editor is started for the user to type in the +//! tag message. +//! +//! ```bash +//! 4b00093bee9b3ef5afc5f8e3645dc39cfa2f49aa +//! ``` +//! +//! The annotated tag is a Git object in its own right, and includes metadata such as the tagger's +//! name and email address, the date and time the tag was created, and a message describing the tag. +//! +//! So, we can use the `git cat-file -p ` command to get the tag object, and the command not +//! for the lightweight tag. +use std::fmt::Display; +use std::str::FromStr; + +use bstr::ByteSlice; + +use crate::errors::GitError; +use crate::hash::SHA1; +use crate::internal::object::ObjectTrait; +use crate::internal::object::ObjectType; +use crate::internal::object::signature::Signature; + +/// The tag object is used to Annotated tag +#[derive(Eq, Debug, Clone)] +pub struct Tag { + pub id: SHA1, + pub object_hash: SHA1, + pub object_type: ObjectType, + pub tag_name: String, + pub tagger: Signature, + pub message: String, +} + +impl PartialEq for Tag { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Display for Tag { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "object {}\ntype {}\ntag {}\ntagger {}\n\n{}", + self.object_hash, self.object_type, self.tag_name, self.tagger, self.message + ) + } +} + +impl Tag { + // pub fn new_from_meta(meta: Meta) -> Result { + // Ok(Tag::new_from_data(meta.data)) + // } + + // pub fn new_from_file(path: &str) -> Result { + // let meta = Meta::new_from_file(path)?; + // Tag::new_from_meta(meta) + // } + + pub fn new( + object_hash: SHA1, + object_type: ObjectType, + tag_name: String, + tagger: Signature, + message: String, + ) -> Self { + // Serialize the tag data to calculate its hash + let data = format!( + "object {}\ntype {}\ntag {}\ntagger {}\n\n{}", + object_hash, object_type, tag_name, tagger, message + ); + let id = SHA1::from_type_and_data(ObjectType::Tag, data.as_bytes()); + + Self { + id, + object_hash, + object_type, + tag_name, + tagger, + message, + } + } +} + +impl ObjectTrait for Tag { + /// The tag object is used to Annotated tag, it's binary format is: + /// + /// ```bash + /// object 0x0a # The SHA-1 hash of the object that the annotated tag is attached to (usually a commit) + /// type 0x0a #The type of Git object that the annotated tag is attached to (usually 'commit') + /// tag 0x0a # The name of the annotated tag(in UTF-8 encoding) + /// tagger 0x0a # The name, email address, and date of the person who created the annotated tag + /// + /// ``` + fn from_bytes(row_data: &[u8], hash: SHA1) -> Result + where + Self: Sized, + { + let mut headers = row_data; + let mut message_start = 0; + + if let Some(pos) = headers.find(b"\n\n") { + message_start = pos + 2; + headers = &headers[..pos]; + } + + let mut object_hash: Option = None; + let mut object_type: Option = None; + let mut tag_name: Option = None; + let mut tagger: Option = None; + + for line in headers.lines() { + if let Some(s) = line.strip_prefix(b"object ") { + let hash_str = s.to_str().map_err(|_| { + GitError::InvalidTagObject("Invalid UTF-8 in object hash".to_string()) + })?; + object_hash = Some(SHA1::from_str(hash_str).map_err(|_| { + GitError::InvalidTagObject("Invalid object hash format".to_string()) + })?); + } else if let Some(s) = line.strip_prefix(b"type ") { + let type_str = s.to_str().map_err(|_| { + GitError::InvalidTagObject("Invalid UTF-8 in object type".to_string()) + })?; + object_type = Some(ObjectType::from_string(type_str)?); + } else if let Some(s) = line.strip_prefix(b"tag ") { + let tag_str = s.to_str().map_err(|_| { + GitError::InvalidTagObject("Invalid UTF-8 in tag name".to_string()) + })?; + tag_name = Some(tag_str.to_string()); + } else if line.starts_with(b"tagger ") { + tagger = Some(Signature::from_data(line.to_vec())?); + } + } + + let message = if message_start > 0 { + String::from_utf8_lossy(&row_data[message_start..]).to_string() + } else { + String::new() + }; + + Ok(Tag { + id: hash, + object_hash: object_hash + .ok_or_else(|| GitError::InvalidTagObject("Missing object hash".to_string()))?, + object_type: object_type + .ok_or_else(|| GitError::InvalidTagObject("Missing object type".to_string()))?, + tag_name: tag_name + .ok_or_else(|| GitError::InvalidTagObject("Missing tag name".to_string()))?, + tagger: tagger + .ok_or_else(|| GitError::InvalidTagObject("Missing tagger".to_string()))?, + message, + }) + } + + fn get_type(&self) -> ObjectType { + ObjectType::Tag + } + + fn get_size(&self) -> usize { + self.to_data().map(|data| data.len()).unwrap_or(0) + } + + /// + /// ```bash + /// object 0x0a # The SHA-1 hash of the object that the annotated tag is attached to (usually a commit) + /// type 0x0a #The type of Git object that the annotated tag is attached to (usually 'commit') + /// tag 0x0a # The name of the annotated tag(in UTF-8 encoding) + /// tagger 0x0a # The name, email address, and date of the person who created the annotated tag + /// + /// ``` + fn to_data(&self) -> Result, GitError> { + let mut data = Vec::new(); + + data.extend_from_slice(b"object "); + data.extend_from_slice(self.object_hash.to_string().as_bytes()); + data.extend_from_slice(b"\n"); + + data.extend_from_slice(b"type "); + data.extend_from_slice(self.object_type.to_string().as_bytes()); + data.extend_from_slice(b"\n"); + + data.extend_from_slice(b"tag "); + data.extend_from_slice(self.tag_name.as_bytes()); + data.extend_from_slice(b"\n"); + + data.extend_from_slice(&self.tagger.to_data()?); + data.extend_from_slice(b"\n\n"); + + data.extend_from_slice(self.message.as_bytes()); + + Ok(data) + } +} diff --git a/mercury/src/internal/object/tree.rs b/mercury/src/internal/object/tree.rs new file mode 100644 index 00000000..63afe25d --- /dev/null +++ b/mercury/src/internal/object/tree.rs @@ -0,0 +1,410 @@ +//! In Git, a tree object is used to represent the state of a directory at a specific point in time. +//! It stores information about the files and directories within that directory, including their names, +//! permissions, and the IDs of the objects that represent their contents. +//! +//! A tree object can contain other tree objects as well as blob objects, which represent the contents +//! of individual files. The object IDs of these child objects are stored within the tree object itself. +//! +//! When you make a commit in Git, you create a new tree object that represents the state of the +//! repository at that point in time. The parent of the new commit is typically the tree object +//! representing the previous state of the repository. +//! +//! Git uses the tree object to efficiently store and manage the contents of a repository. By +//! representing the contents of a directory as a tree object, Git can quickly determine which files +//! have been added, modified, or deleted between two points in time. This allows Git to perform +//! operations like merging and rebasing more quickly and accurately. +//! +use crate::errors::GitError; +use crate::hash::SHA1; +use crate::internal::object::ObjectTrait; +use crate::internal::object::ObjectType; +use bincode::{Decode, Encode}; +use colored::Colorize; +use encoding_rs::GBK; +use serde::Deserialize; +use serde::Serialize; +use std::fmt::Display; + +/// In Git, the mode field in a tree object's entry specifies the type of the object represented by +/// that entry. The mode is a three-digit octal number that encodes both the permissions and the +/// type of the object. The first digit specifies the object type, and the remaining two digits +/// specify the file mode or permissions. +#[derive(PartialEq, Eq, Debug, Clone, Copy, Serialize, Deserialize, Hash, Encode, Decode)] +pub enum TreeItemMode { + Blob, + BlobExecutable, + Tree, + Commit, + Link, +} + +impl Display for TreeItemMode { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let _print = match *self { + TreeItemMode::Blob => "blob", + TreeItemMode::BlobExecutable => "blob executable", + TreeItemMode::Tree => "tree", + TreeItemMode::Commit => "commit", + TreeItemMode::Link => "link", + }; + + write!(f, "{}", String::from(_print).blue()) + } +} + +impl TreeItemMode { + /// Convert a 32-bit mode to a TreeItemType + /// + /// |0100000000000000| (040000)| Directory| + /// |1000000110100100| (100644)| Regular non-executable file| + /// |1000000110110100| (100664)| Regular non-executable group-writeable file| + /// |1000000111101101| (100755)| Regular executable file| + /// |1010000000000000| (120000)| Symbolic link| + /// |1110000000000000| (160000)| Gitlink| + /// --- + /// # GitLink + /// Gitlink, also known as a submodule, is a feature in Git that allows you to include a Git + /// repository as a subdirectory within another Git repository. This is useful when you want to + /// incorporate code from another project into your own project, without having to manually copy + /// the code into your repository. + /// + /// When you add a submodule to your Git repository, Git stores a reference to the other + /// repository at a specific commit. This means that your repository will always point to a + /// specific version of the other repository, even if changes are made to the submodule's code + /// in the future. + /// + /// To work with a submodule in Git, you use commands like git submodule add, git submodule + /// update, and git submodule init. These commands allow you to add a submodule to your repository, + /// update it to the latest version, and initialize it for use. + /// + /// Submodules can be a powerful tool for managing dependencies between different projects and + /// components. However, they can also add complexity to your workflow, so it's important to + /// understand how they work and when to use them. + pub fn tree_item_type_from_bytes(mode: &[u8]) -> Result { + Ok(match mode { + b"40000" => TreeItemMode::Tree, + b"100644" => TreeItemMode::Blob, + b"100755" => TreeItemMode::BlobExecutable, + b"120000" => TreeItemMode::Link, + b"160000" => TreeItemMode::Commit, + b"100664" => TreeItemMode::Blob, + b"100640" => TreeItemMode::Blob, + _ => { + return Err(GitError::InvalidTreeItem( + String::from_utf8(mode.to_vec()).unwrap(), + )); + } + }) + } + + /// 32-bit mode, split into (high to low bits): + /// - 4-bit object type: valid values in binary are 1000 (regular file), 1010 (symbolic link) and 1110 (gitlink) + /// - 3-bit unused + /// - 9-bit unix permission: Only 0755 and 0644 are valid for regular files. Symbolic links and gitlink have value 0 in this field. + pub fn to_bytes(self) -> &'static [u8] { + match self { + TreeItemMode::Blob => b"100644", + TreeItemMode::BlobExecutable => b"100755", + TreeItemMode::Link => b"120000", + TreeItemMode::Tree => b"40000", + TreeItemMode::Commit => b"160000", + } + } +} + +/// A tree object contains a list of entries, one for each file or directory in the tree. Each entry +/// in the file represents an entry in the tree, and each entry has the following format: +/// +/// ```bash +/// \0 +/// ``` +/// - `` is the mode of the object, represented as a six-digit octal number. The first digit +/// represents the object type (tree, blob, etc.), and the remaining digits represent the file mode or permissions. +/// - `` is the name of the object. +/// - `\0` is a null byte separator. +/// - `` is the ID of the object that represents the contents of the file or +/// directory, represented as a binary SHA-1 hash. +/// +/// # Example +/// ```bash +/// 100644 hello-world\0 +/// 040000 data\0 +/// ``` +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Hash, Encode, Decode)] +pub struct TreeItem { + pub mode: TreeItemMode, + pub id: SHA1, + pub name: String, +} + +impl Display for TreeItem { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{} {} {}", + self.mode, + self.name, + self.id.to_string().blue() + ) + } +} + +impl TreeItem { + // Create a new TreeItem from a mode, id and name + pub fn new(mode: TreeItemMode, id: SHA1, name: String) -> Self { + TreeItem { mode, id, name } + } + + /// Create a new TreeItem from a byte vector, split into a mode, id and name, the TreeItem format is: + /// + /// ```bash + /// \0 + /// ``` + /// + pub fn from_bytes(bytes: &[u8]) -> Result { + let mut parts = bytes.splitn(2, |b| *b == b' '); + let mode = parts.next().unwrap(); + let rest = parts.next().unwrap(); + let mut parts = rest.splitn(2, |b| *b == b'\0'); + let raw_name = parts.next().unwrap(); + let id = parts.next().unwrap(); + + let name = if String::from_utf8(raw_name.to_vec()).is_ok() { + String::from_utf8(raw_name.to_vec()).unwrap() + } else { + let (decoded, _, had_errors) = GBK.decode(raw_name); + if had_errors { + return Err(GitError::InvalidTreeItem(format!( + "Unsupported raw format: {raw_name:?}" + ))); + } else { + decoded.to_string() + } + }; + Ok(TreeItem { + mode: TreeItemMode::tree_item_type_from_bytes(mode)?, + id: SHA1::from_bytes(id), + name, + }) + } + + /// Convert a TreeItem to a byte vector + /// ```rust + /// use std::str::FromStr; + /// use mercury::internal::object::tree::{TreeItem, TreeItemMode}; + /// use mercury::hash::SHA1; + /// + /// let tree_item = TreeItem::new( + /// TreeItemMode::Blob, + /// SHA1::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), + /// "hello-world".to_string(), + /// ); + /// + // let bytes = tree_item.to_bytes(); + /// ``` + pub fn to_data(&self) -> Vec { + let mut bytes = Vec::new(); + + bytes.extend_from_slice(self.mode.to_bytes()); + bytes.push(b' '); + bytes.extend_from_slice(self.name.as_bytes()); + bytes.push(b'\0'); + bytes.extend_from_slice(&self.id.to_data()); + + bytes + } + + pub fn is_tree(&self) -> bool { + self.mode == TreeItemMode::Tree + } + + pub fn is_blob(&self) -> bool { + self.mode == TreeItemMode::Blob + } +} + +/// A tree object is a Git object that represents a directory. It contains a list of entries, one +/// for each file or directory in the tree. +#[derive(Eq, Debug, Clone, Serialize, Deserialize, Encode, Decode)] +pub struct Tree { + pub id: SHA1, + pub tree_items: Vec, +} + +impl PartialEq for Tree { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Display for Tree { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "Tree: {}", self.id.to_string().blue())?; + for item in &self.tree_items { + writeln!(f, "{item}")?; + } + Ok(()) + } +} + +impl Tree { + pub fn from_tree_items(tree_items: Vec) -> Result { + if tree_items.is_empty() { + return Err(GitError::EmptyTreeItems( + "When export tree object to meta, the items is empty" + .parse() + .unwrap(), + )); + } + let mut data = Vec::new(); + for item in &tree_items { + data.extend_from_slice(item.to_data().as_slice()); + } + + Ok(Tree { + id: SHA1::from_type_and_data(ObjectType::Tree, &data), + tree_items, + }) + } + + /// After the subdirectory is changed, the hash value of the tree is recalculated. + pub fn rehash(&mut self) { + let mut data = Vec::new(); + for item in &self.tree_items { + data.extend_from_slice(item.to_data().as_slice()); + } + self.id = SHA1::from_type_and_data(ObjectType::Tree, &data); + } +} + +impl TryFrom<&[u8]> for Tree { + type Error = GitError; + fn try_from(data: &[u8]) -> Result { + let h = SHA1::from_type_and_data(ObjectType::Tree, data); + Tree::from_bytes(data, h) + } +} +impl ObjectTrait for Tree { + fn from_bytes(data: &[u8], hash: SHA1) -> Result + where + Self: Sized, + { + let mut tree_items = Vec::new(); + let mut i = 0; + while i < data.len() { + // Find the position of the null byte (0x00) + if let Some(index) = memchr::memchr(0x00, &data[i..]) { + // Calculate the next position + let next = i + index + 21; + + // Extract the bytes and create a TreeItem + let item_data = &data[i..next]; + let tree_item = TreeItem::from_bytes(item_data)?; + + tree_items.push(tree_item); + + i = next; + } else { + // If no null byte is found, return an error + return Err(GitError::InvalidTreeObject); + } + } + + Ok(Tree { + id: hash, + tree_items, + }) + } + + fn get_type(&self) -> ObjectType { + ObjectType::Tree + } + + fn get_size(&self) -> usize { + todo!() + } + + fn to_data(&self) -> Result, GitError> { + let mut data: Vec = Vec::new(); + + for item in &self.tree_items { + data.extend_from_slice(item.to_data().as_slice()); + //data.push(b'\0'); + } + + Ok(data) + } +} + +#[cfg(test)] +mod tests { + + use std::str::FromStr; + + use crate::hash::SHA1; + use crate::internal::object::tree::{Tree, TreeItem, TreeItemMode}; + + #[test] + fn test_tree_item_new() { + let tree_item = TreeItem::new( + TreeItemMode::Blob, + SHA1::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), + "hello-world".to_string(), + ); + + assert_eq!(tree_item.mode, TreeItemMode::Blob); + assert_eq!( + tree_item.id.to_string(), + "8ab686eafeb1f44702738c8b0f24f2567c36da6d" + ); + } + + #[test] + fn test_tree_item_to_bytes() { + let tree_item = TreeItem::new( + TreeItemMode::Blob, + SHA1::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), + "hello-world".to_string(), + ); + + let bytes = tree_item.to_data(); + assert_eq!( + bytes, + vec![ + 49, 48, 48, 54, 52, 52, 32, 104, 101, 108, 108, 111, 45, 119, 111, 114, 108, 100, + 0, 138, 182, 134, 234, 254, 177, 244, 71, 2, 115, 140, 139, 15, 36, 242, 86, 124, + 54, 218, 109 + ] + ); + } + + #[test] + fn test_tree_item_from_bytes() { + let item = TreeItem::new( + TreeItemMode::Blob, + SHA1::from_str("8ab686eafeb1f44702738c8b0f24f2567c36da6d").unwrap(), + "hello-world".to_string(), + ); + + let bytes = item.to_data(); + let tree_item = TreeItem::from_bytes(bytes.as_slice()).unwrap(); + + assert_eq!(tree_item.mode, TreeItemMode::Blob); + assert_eq!(tree_item.id.to_string(), item.id.to_string()); + } + + #[test] + fn test_from_tree_items() { + let item = TreeItem::new( + TreeItemMode::Blob, + SHA1::from_str("17288789afffb273c8c394bc65e87d899b92897b").unwrap(), + "hello-world".to_string(), + ); + let tree = Tree::from_tree_items(vec![item]).unwrap(); + println!("{}", tree.id); + assert_eq!( + "cf99336fa61439a2f074c7e6de1c1a05579550e2", + tree.id.to_string() + ); + } +} diff --git a/mercury/src/internal/object/types.rs b/mercury/src/internal/object/types.rs new file mode 100644 index 00000000..7d272df8 --- /dev/null +++ b/mercury/src/internal/object/types.rs @@ -0,0 +1,162 @@ +use std::fmt::Display; + +use serde::{Deserialize, Serialize}; + +use crate::errors::GitError; + +/// In Git, each object type is assigned a unique integer value, which is used to identify the +/// type of the object in Git repositories. +/// +/// * `Blob` (1): A Git object that stores the content of a file. +/// * `Tree` (2): A Git object that represents a directory or a folder in a Git repository. +/// * `Commit` (3): A Git object that represents a commit in a Git repository, which contains +/// information such as the author, committer, commit message, and parent commits. +/// * `Tag` (4): A Git object that represents a tag in a Git repository, which is used to mark a +/// specific point in the Git history. +/// * `OffsetDelta` (6): A Git object that represents a delta between two objects, where the delta +/// is stored as an offset to the base object. +/// * `HashDelta` (7): A Git object that represents a delta between two objects, where the delta +/// is stored as a hash of the base object. +/// +/// By assigning unique integer values to each Git object type, Git can easily and efficiently +/// identify the type of an object and perform the appropriate operations on it. when parsing a Git +/// repository, Git can use the integer value of an object's type to determine how to parse +/// the object's content. +#[derive(PartialEq, Eq, Hash, Debug, Clone, Copy, Serialize, Deserialize)] +pub enum ObjectType { + Commit = 1, + Tree, + Blob, + Tag, + OffsetZstdelta, // Private extension for Zstandard-compressed delta objects + OffsetDelta, + HashDelta, +} + +const COMMIT_OBJECT_TYPE: &[u8] = b"commit"; +const TREE_OBJECT_TYPE: &[u8] = b"tree"; +const BLOB_OBJECT_TYPE: &[u8] = b"blob"; +const TAG_OBJECT_TYPE: &[u8] = b"tag"; + +/// Display trait for Git objects type +impl Display for ObjectType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + ObjectType::Blob => write!(f, "blob"), + ObjectType::Tree => write!(f, "tree"), + ObjectType::Commit => write!(f, "commit"), + ObjectType::Tag => write!(f, "tag"), + ObjectType::OffsetZstdelta => write!(f, "OffsetZstdelta"), + ObjectType::OffsetDelta => write!(f, "OffsetDelta"), + ObjectType::HashDelta => write!(f, "HashDelta"), + } + } +} + +/// Display trait for Git objects type +impl ObjectType { + pub fn to_bytes(&self) -> &[u8] { + match self { + ObjectType::Commit => COMMIT_OBJECT_TYPE, + ObjectType::Tree => TREE_OBJECT_TYPE, + ObjectType::Blob => BLOB_OBJECT_TYPE, + ObjectType::Tag => TAG_OBJECT_TYPE, + _ => panic!("can put compute the delta hash value"), + } + } + + /// Parses a string representation of a Git object type and returns an ObjectType value + pub fn from_string(s: &str) -> Result { + match s { + "blob" => Ok(ObjectType::Blob), + "tree" => Ok(ObjectType::Tree), + "commit" => Ok(ObjectType::Commit), + "tag" => Ok(ObjectType::Tag), + _ => Err(GitError::InvalidObjectType(s.to_string())), + } + } + + /// Convert an object type to a byte array. + pub fn to_data(self) -> Result, GitError> { + match self { + ObjectType::Blob => Ok(vec![0x62, 0x6c, 0x6f, 0x62]), + ObjectType::Tree => Ok(vec![0x74, 0x72, 0x65, 0x65]), + ObjectType::Commit => Ok(vec![0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74]), + ObjectType::Tag => Ok(vec![0x74, 0x61, 0x67]), + _ => Err(GitError::InvalidObjectType(self.to_string())), + } + } + + /// Convert an object type to a number. + pub fn to_u8(&self) -> u8 { + match self { + ObjectType::Commit => 1, + ObjectType::Tree => 2, + ObjectType::Blob => 3, + ObjectType::Tag => 4, + ObjectType::OffsetZstdelta => 5, // Type 5 is reserved in standard Git packs; we use it for Zstd delta objects. + ObjectType::OffsetDelta => 6, + ObjectType::HashDelta => 7, + } + } + + /// Convert a number to an object type. + pub fn from_u8(number: u8) -> Result { + match number { + 1 => Ok(ObjectType::Commit), + 2 => Ok(ObjectType::Tree), + 3 => Ok(ObjectType::Blob), + 4 => Ok(ObjectType::Tag), + 5 => Ok(ObjectType::OffsetZstdelta), + 6 => Ok(ObjectType::OffsetDelta), + 7 => Ok(ObjectType::HashDelta), + _ => Err(GitError::InvalidObjectType(format!( + "Invalid object type number: {number}" + ))), + } + } + + pub fn is_base(&self) -> bool { + match self { + ObjectType::Commit => true, + ObjectType::Tree => true, + ObjectType::Blob => true, + ObjectType::Tag => true, + ObjectType::HashDelta => false, + ObjectType::OffsetZstdelta => false, + ObjectType::OffsetDelta => false, + } + } +} + +#[cfg(test)] +mod tests { + use crate::internal::object::types::ObjectType; + + #[test] + fn test_object_type_to_data() { + let blob = ObjectType::Blob; + let blob_bytes = blob.to_data().unwrap(); + assert_eq!(blob_bytes, vec![0x62, 0x6c, 0x6f, 0x62]); + } + + #[test] + fn test_object_type_from_string() { + let tree = ObjectType::from_string("tree").unwrap(); + assert_eq!(tree, ObjectType::Tree); + } + + #[test] + fn test_object_type_to_u8() { + let commit = ObjectType::Commit; + let commit_number = commit.to_u8(); + assert_eq!(commit_number, 1); + } + + #[test] + fn test_object_type_from_u8() { + let tag_number = 4; + let tag = ObjectType::from_u8(tag_number).unwrap(); + assert_eq!(tag, ObjectType::Tag); + } +} diff --git a/mercury/src/internal/object/utils.rs b/mercury/src/internal/object/utils.rs new file mode 100644 index 00000000..b9902bc2 --- /dev/null +++ b/mercury/src/internal/object/utils.rs @@ -0,0 +1,111 @@ +use std::io::{self, Read, Write}; + +use flate2::{Compression, write::ZlibEncoder}; + +const TYPE_BITS: u8 = 3; +const VAR_INT_ENCODING_BITS: u8 = 7; +const TYPE_BYTE_SIZE_BITS: u8 = VAR_INT_ENCODING_BITS - TYPE_BITS; +const VAR_INT_CONTINUE_FLAG: u8 = 1 << VAR_INT_ENCODING_BITS; + +/// Parses a byte slice into a `usize` representing the size of a Git object. +/// +/// This function is intended to be used for converting the bytes, which represent the size portion +/// in a Git object, back into a `usize`. This size is typically compared with the actual length of +/// the object's data part to ensure data integrity. +/// +/// # Parameters +/// * `bytes`: A byte slice (`&[u8]`) representing the size in a serialized Git object. +/// +/// # Returns +/// Returns a `Result` which is: +/// * `Ok(usize)`: On successful parsing, returns the size as a `usize`. +/// * `Err(Box)`: On failure, returns an error in a Box. This error could be +/// due to invalid UTF-8 encoding in the byte slice or a failure to parse the byte slice as a `usize`. +/// +/// # Errors +/// This function handles two main types of errors: +/// 1. `Utf8Error`: If the byte slice is not a valid UTF-8 string, which is necessary for the size representation. +/// 2. `ParseIntError`: If the byte slice does not represent a valid `usize` value. +pub fn parse_size_from_bytes(bytes: &[u8]) -> Result> { + let size_str = std::str::from_utf8(bytes)?; + Ok(size_str.parse::()?) +} + +/// Preserve the last bits of value binary +/// +fn keep_bits(value: usize, bits: u8) -> usize { + value & ((1 << bits) - 1) +} +/// Read the first few fields of the object and parse +/// +pub fn read_type_and_size(stream: &mut R) -> io::Result<(u8, usize)> { + // Object type and uncompressed pack data size + // are stored in a "size-encoding" variable-length integer. + // Bits 4 through 6 store the type and the remaining bits store the size. + let value = read_size_encoding(stream)?; + let object_type = keep_bits(value >> TYPE_BYTE_SIZE_BITS, TYPE_BITS) as u8; + let size = keep_bits(value, TYPE_BYTE_SIZE_BITS) + | (value >> VAR_INT_ENCODING_BITS << TYPE_BYTE_SIZE_BITS); + + Ok((object_type, size)) +} + +/// Read the type and size of the object +/// +pub fn read_size_encoding(stream: &mut R) -> io::Result { + let mut value = 0; + let mut length = 0; + + loop { + let (byte_value, more_bytes) = read_var_int_byte(stream).unwrap(); + value |= (byte_value as usize) << length; + if !more_bytes { + return Ok(value); + } + + length += VAR_INT_ENCODING_BITS; + } +} + +/// Returns whether the first bit of u8 is 1 and returns the 7-bit truth value +/// +pub fn read_var_int_byte(stream: &mut R) -> io::Result<(u8, bool)> { + let [byte] = read_bytes(stream)?; + let value = byte & !VAR_INT_CONTINUE_FLAG; + let more_bytes = byte & VAR_INT_CONTINUE_FLAG != 0; + + Ok((value, more_bytes)) +} + +/// Read the next N bytes from the reader +/// +#[inline] +pub fn read_bytes(stream: &mut R) -> io::Result<[u8; N]> { + let mut bytes = [0; N]; + stream.read_exact(&mut bytes)?; + + Ok(bytes) +} + +pub fn compress_zlib(data: &[u8]) -> io::Result> { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(data)?; + let compressed_data = encoder.finish()?; + Ok(compressed_data) +} + +#[cfg(test)] +mod tests { + use crate::internal::object::utils::parse_size_from_bytes; + + #[test] + fn test_parse_size_from_bytes() -> Result<(), Box> { + let size: usize = 12345; + let size_bytes = size.to_string().as_bytes().to_vec(); + + let parsed_size = parse_size_from_bytes(&size_bytes)?; + + assert_eq!(size, parsed_size); + Ok(()) + } +} diff --git a/mercury/src/internal/pack/cache.rs b/mercury/src/internal/pack/cache.rs new file mode 100644 index 00000000..3eef9aaa --- /dev/null +++ b/mercury/src/internal/pack/cache.rs @@ -0,0 +1,291 @@ +use std::path::Path; +use std::path::PathBuf; +use std::sync::Once; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::sleep; +use std::{fs, io}; + +use dashmap::{DashMap, DashSet}; +use lru_mem::LruCache; +use threadpool::ThreadPool; + +use crate::hash::SHA1; +use crate::internal::pack::cache_object::{ + ArcWrapper, CacheObject, FileLoadStore, MemSizeRecorder, +}; +use crate::time_it; + +pub trait _Cache { + fn new(mem_size: Option, tmp_path: PathBuf, thread_num: usize) -> Self + where + Self: Sized; + fn get_hash(&self, offset: usize) -> Option; + fn insert(&self, offset: usize, hash: SHA1, obj: CacheObject) -> Arc; + fn get_by_offset(&self, offset: usize) -> Option>; + fn get_by_hash(&self, h: SHA1) -> Option>; + fn total_inserted(&self) -> usize; + fn memory_used(&self) -> usize; + fn clear(&self); +} + +impl lru_mem::HeapSize for SHA1 { + fn heap_size(&self) -> usize { + 0 + } +} + +pub struct Caches { + map_offset: DashMap, // offset to hash + hash_set: DashSet, // item in the cache + // dropping large lru cache will take a long time on Windows without multi-thread IO + // because "multi-thread IO" clone Arc, so it won't be dropped in the main thread, + // and `CacheObjects` will be killed by OS after Process ends abnormally + // Solution: use `mimalloc` + lru_cache: Mutex>>, + mem_size: Option, + tmp_path: PathBuf, + path_prefixes: [Once; 256], + pool: Arc, + complete_signal: Arc, +} + +impl Caches { + /// only get object from memory, not from tmp file + fn try_get(&self, hash: SHA1) -> Option> { + let mut map = self.lru_cache.lock().unwrap(); + map.get(&hash).map(|x| x.data.clone()) + } + + /// !IMPORTANT: because of the process of pack, the file must be written / be writing before, so it won't be dead lock + /// fall back to temp to get item. **invoker should ensure the hash is in the cache, or it will block forever** + fn get_fallback(&self, hash: SHA1) -> io::Result> { + let path = self.generate_temp_path(&self.tmp_path, hash); + // read from tmp file + let obj = { + loop { + match Self::read_from_temp(&path) { + Ok(x) => break x, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + sleep(std::time::Duration::from_millis(10)); + continue; + } + Err(e) => return Err(e), // other error + } + } + }; + + let mut map = self.lru_cache.lock().unwrap(); + let obj = Arc::new(obj); + let mut x = ArcWrapper::new( + obj.clone(), + self.complete_signal.clone(), + Some(self.pool.clone()), + ); + x.set_store_path(path); + let _ = map.insert(hash, x); // handle the error + Ok(obj) + } + + /// generate the temp file path, hex string of the hash + fn generate_temp_path(&self, tmp_path: &Path, hash: SHA1) -> PathBuf { + // This is enough for the original path, 2 chars directory, 40 chars hash, and extra slashes + let mut path = PathBuf::with_capacity(self.tmp_path.capacity() + SHA1::SIZE * 2 + 5); + path.push(tmp_path); + let hash_str = hash._to_string(); + path.push(&hash_str[..2]); // use first 2 chars as the directory + self.path_prefixes[hash.as_ref()[0] as usize].call_once(|| { + // Check if the directory exists, if not, create it + if !path.exists() { + fs::create_dir_all(&path).unwrap(); + } + }); + path.push(hash_str); + path + } + + fn read_from_temp(path: &Path) -> io::Result { + let obj = CacheObject::f_load(path)?; + // Deserializing will also create an object but without Construction outside and `::new()` + // So if you want to do sth. while Constructing, impl Deserialize trait yourself + obj.record_mem_size(); + Ok(obj) + } + + pub fn queued_tasks(&self) -> usize { + self.pool.queued_count() + } + + /// memory used by the index (exclude lru_cache which is contained in CacheObject::get_mem_size()) + pub fn memory_used_index(&self) -> usize { + self.map_offset.capacity() * (std::mem::size_of::() + std::mem::size_of::()) + + self.hash_set.capacity() * (std::mem::size_of::()) + } + + /// remove the tmp dir + pub fn remove_tmp_dir(&self) { + time_it!("Remove tmp dir", { + if self.tmp_path.exists() { + fs::remove_dir_all(&self.tmp_path).unwrap(); //very slow + } + }); + } +} + +impl _Cache for Caches { + /// @param size: the size of the memory lru cache. **None means no limit** + /// @param tmp_path: the path to store the cache object in the tmp file + fn new(mem_size: Option, tmp_path: PathBuf, thread_num: usize) -> Self + where + Self: Sized, + { + // `None` means no limit, so no need to create the tmp dir + if mem_size.is_some() { + fs::create_dir_all(&tmp_path).unwrap(); + } + + Caches { + map_offset: DashMap::new(), + hash_set: DashSet::new(), + lru_cache: Mutex::new(LruCache::new(mem_size.unwrap_or(usize::MAX))), + mem_size, + tmp_path, + path_prefixes: [const { Once::new() }; 256], + pool: Arc::new(ThreadPool::new(thread_num)), + complete_signal: Arc::new(AtomicBool::new(false)), + } + } + + fn get_hash(&self, offset: usize) -> Option { + self.map_offset.get(&offset).map(|x| *x) + } + + fn insert(&self, offset: usize, hash: SHA1, obj: CacheObject) -> Arc { + let obj_arc = Arc::new(obj); + { + // ? whether insert to cache directly or only write to tmp file + let mut map = self.lru_cache.lock().unwrap(); + let mut a_obj = ArcWrapper::new( + obj_arc.clone(), + self.complete_signal.clone(), + Some(self.pool.clone()), + ); + if self.mem_size.is_some() { + a_obj.set_store_path(self.generate_temp_path(&self.tmp_path, hash)); + } + let _ = map.insert(hash, a_obj); + } + //order maters as for reading in 'get_by_offset()' + self.hash_set.insert(hash); + self.map_offset.insert(offset, hash); + + obj_arc + } + + fn get_by_offset(&self, offset: usize) -> Option> { + match self.map_offset.get(&offset) { + Some(x) => self.get_by_hash(*x), + None => None, + } + } + + fn get_by_hash(&self, hash: SHA1) -> Option> { + // check if the hash is in the cache( lru or tmp file) + if self.hash_set.contains(&hash) { + match self.try_get(hash) { + Some(x) => Some(x), + None => { + if self.mem_size.is_none() { + panic!("should not be here when mem_size is not set") + } + self.get_fallback(hash).ok() + } + } + } else { + None + } + } + + fn total_inserted(&self) -> usize { + self.hash_set.len() + } + fn memory_used(&self) -> usize { + self.lru_cache.lock().unwrap().current_size() + self.memory_used_index() + } + fn clear(&self) { + time_it!("Caches clear", { + self.complete_signal.store(true, Ordering::Release); + self.pool.join(); + self.lru_cache.lock().unwrap().clear(); + self.hash_set.clear(); + self.hash_set.shrink_to_fit(); + self.map_offset.clear(); + self.map_offset.shrink_to_fit(); + }); + + assert_eq!(self.pool.queued_count(), 0); + assert_eq!(self.pool.active_count(), 0); + assert_eq!(self.lru_cache.lock().unwrap().len(), 0); + } +} + +#[cfg(test)] +mod test { + use std::env; + + use super::*; + use crate::{ + hash::SHA1, + internal::{object::types::ObjectType, pack::cache_object::CacheObjectInfo}, + }; + + #[test] + fn test_cache_single_thread() { + let source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + let tmp_path = source.clone().join("tests/.cache_tmp"); + + if tmp_path.exists() { + fs::remove_dir_all(&tmp_path).unwrap(); + } + + let cache = Caches::new(Some(2048), tmp_path, 1); + let a_hash = SHA1::new(String::from("a").as_bytes()); + let b_hash = SHA1::new(String::from("b").as_bytes()); + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, a_hash), + data_decompressed: vec![0; 800], + mem_recorder: None, + offset: 0, + }; + let b = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, b_hash), + data_decompressed: vec![0; 800], + mem_recorder: None, + offset: 0, + }; + // insert a + cache.insert(a.offset, a_hash, a.clone()); + assert!(cache.hash_set.contains(&a_hash)); + assert!(cache.try_get(a_hash).is_some()); + + // insert b, a should still be in cache + cache.insert(b.offset, b_hash, b.clone()); + assert!(cache.hash_set.contains(&b_hash)); + assert!(cache.try_get(b_hash).is_some()); + assert!(cache.try_get(a_hash).is_some()); + + let c_hash = SHA1::new(String::from("c").as_bytes()); + // insert c which will evict both a and b + let c = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, c_hash), + data_decompressed: vec![0; 1700], + mem_recorder: None, + offset: 0, + }; + cache.insert(c.offset, c_hash, c.clone()); + assert!(cache.try_get(a_hash).is_none()); + assert!(cache.try_get(b_hash).is_none()); + assert!(cache.try_get(c_hash).is_some()); + assert!(cache.get_by_hash(c_hash).is_some()); + } +} diff --git a/mercury/src/internal/pack/cache_object.rs b/mercury/src/internal/pack/cache_object.rs new file mode 100644 index 00000000..c5ec81e4 --- /dev/null +++ b/mercury/src/internal/pack/cache_object.rs @@ -0,0 +1,567 @@ +use std::fs::OpenOptions; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::{fs, io}; +use std::{ops::Deref, sync::Arc}; + +use lru_mem::{HeapSize, MemSize}; +use serde::{Deserialize, Serialize}; +use threadpool::ThreadPool; + +use crate::internal::pack::entry::Entry; +use crate::internal::pack::utils; +use crate::{hash::SHA1, internal::object::types::ObjectType}; + +// /// record heap-size of all CacheObjects, used for memory limit. +// static CACHE_OBJS_MEM_SIZE: AtomicUsize = AtomicUsize::new(0); + +/// file load&store trait +pub trait FileLoadStore: Serialize + for<'a> Deserialize<'a> { + fn f_load(path: &Path) -> Result; + fn f_save(&self, path: &Path) -> Result<(), io::Error>; +} + +// trait alias, so that impl FileLoadStore == impl Serialize + Deserialize +impl Deserialize<'a>> FileLoadStore for T { + fn f_load(path: &Path) -> Result { + let data = fs::read(path)?; + let obj: T = bincode::serde::decode_from_slice(&data, bincode::config::standard()) + .map_err(io::Error::other)? + .0; + Ok(obj) + } + fn f_save(&self, path: &Path) -> Result<(), io::Error> { + if path.exists() { + return Ok(()); + } + let data = bincode::serde::encode_to_vec(self, bincode::config::standard()).unwrap(); + let path = path.with_extension("temp"); + { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path.clone())?; + file.write_all(&data)?; + } + let final_path = path.with_extension(""); + fs::rename(&path, final_path.clone())?; + Ok(()) + } +} + +/// Represents the metadata of a cache object, indicating whether it is a delta or not. +#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] +pub(crate) enum CacheObjectInfo { + /// The object is one of the four basic types: + /// [`ObjectType::Blob`], [`ObjectType::Tree`], [`ObjectType::Commit`], or [`ObjectType::Tag`]. + /// The metadata contains the [`ObjectType`] and the [`SHA1`] hash of the object. + BaseObject(ObjectType, SHA1), + /// The object is an offset delta with a specified offset delta [`usize`], + /// and the size of the expanded object (previously `delta_final_size`). + OffsetDelta(usize, usize), + /// Similar to [`OffsetDelta`], but delta algorithm is `zstd`. + OffsetZstdelta(usize, usize), + /// The object is a hash delta with a specified [`SHA1`] hash, + /// and the size of the expanded object (previously `delta_final_size`). + HashDelta(SHA1, usize), +} + +impl CacheObjectInfo { + /// Get the [`ObjectType`] of the object. + pub(crate) fn object_type(&self) -> ObjectType { + match self { + CacheObjectInfo::BaseObject(obj_type, _) => *obj_type, + CacheObjectInfo::OffsetDelta(_, _) => ObjectType::OffsetDelta, + CacheObjectInfo::OffsetZstdelta(_, _) => ObjectType::OffsetZstdelta, + CacheObjectInfo::HashDelta(_, _) => ObjectType::HashDelta, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct CacheObject { + pub(crate) info: CacheObjectInfo, + pub offset: usize, + pub data_decompressed: Vec, + pub mem_recorder: Option>, // record mem-size of all CacheObjects of a Pack +} + +impl Clone for CacheObject { + fn clone(&self) -> Self { + let obj = CacheObject { + info: self.info.clone(), + offset: self.offset, + data_decompressed: self.data_decompressed.clone(), + mem_recorder: self.mem_recorder.clone(), + }; + obj.record_mem_size(); + obj + } +} + +// ! used by lru_mem to calculate the size of the object, limit the memory usage. +// ! the implementation of HeapSize is not accurate, only calculate the size of the data_decompress +// Note that: mem_size == value_size + heap_size, and we only need to impl HeapSize because value_size is known +impl HeapSize for CacheObject { + /// If a [`CacheObject`] is [`ObjectType::HashDelta`] or [`ObjectType::OffsetDelta`], + /// it will expand to another [`CacheObject`] of other types. To prevent potential OOM, + /// we record the size of the expanded object as well as that of the object itself. + /// + /// Base objects, *i.e.*, [`ObjectType::Blob`], [`ObjectType::Tree`], [`ObjectType::Commit`], + /// and [`ObjectType::Tag`], will not be expanded, so the heap-size of the object is the same + /// as the size of the data. + /// + /// See [Comment in PR #755](https://github.com/web3infra-foundation/mega/pull/755#issuecomment-2543100481) for more details. + fn heap_size(&self) -> usize { + match &self.info { + CacheObjectInfo::BaseObject(_, _) => self.data_decompressed.heap_size(), + CacheObjectInfo::OffsetDelta(_, delta_final_size) + | CacheObjectInfo::OffsetZstdelta(_, delta_final_size) + | CacheObjectInfo::HashDelta(_, delta_final_size) => { + // To those who are concerned about why these two values are added, + // let's consider the lifetime of two `CacheObject`s, say `delta_obj` + // and `final_obj` in the function `Pack::rebuild_delta`. + // + // `delta_obj` is dropped only after `Pack::rebuild_delta` returns, + // but the space for `final_obj` is allocated in that function. + // + // Therefore, during the execution of `Pack::rebuild_delta`, both `delta_obj` + // and `final_obj` coexist. The maximum memory usage is the sum of the memory + // usage of `delta_obj` and `final_obj`. + self.data_decompressed.heap_size() + delta_final_size + } + } + } +} + +impl Drop for CacheObject { + // Check: the heap-size subtracted when Drop is equal to the heap-size recorded + // (cannot change the heap-size during life cycle) + fn drop(&mut self) { + // (&*self).heap_size() != self.heap_size() + if let Some(mem_recorder) = &self.mem_recorder { + mem_recorder.fetch_sub((*self).mem_size(), Ordering::Release); + } + } +} + +/// Heap-size recorder for a class(struct) +///
You should use a static Var to record mem-size +/// and record mem-size after construction & minus it in `drop()` +///
So, variable-size fields in object should NOT be modified to keep heap-size stable. +///
Or, you can record the initial mem-size in this object +///
Or, update it (not impl) +pub trait MemSizeRecorder: MemSize { + fn record_mem_size(&self); + fn set_mem_recorder(&mut self, mem_size: Arc); + // fn get_mem_size() -> usize; +} + +impl MemSizeRecorder for CacheObject { + /// record the mem-size of this `CacheObj` in a `static` `var` + ///
since that, DO NOT modify `CacheObj` after recording + fn record_mem_size(&self) { + if let Some(mem_recorder) = &self.mem_recorder { + mem_recorder.fetch_add(self.mem_size(), Ordering::Release); + } + } + + fn set_mem_recorder(&mut self, mem_recorder: Arc) { + self.mem_recorder = Some(mem_recorder); + } + + // fn get_mem_size() -> usize { + // CACHE_OBJS_MEM_SIZE.load(Ordering::Acquire) + // } +} + +impl CacheObject { + /// Create a new CacheObject which is neither [`ObjectType::OffsetDelta`] nor [`ObjectType::HashDelta`]. + pub fn new_for_undeltified(obj_type: ObjectType, data: Vec, offset: usize) -> Self { + let hash = utils::calculate_object_hash(obj_type, &data); + CacheObject { + info: CacheObjectInfo::BaseObject(obj_type, hash), + offset, + data_decompressed: data, + mem_recorder: None, + } + } + + /// Get the [`ObjectType`] of the object. + pub fn object_type(&self) -> ObjectType { + self.info.object_type() + } + + /// Get the [`SHA1`] hash of the object. + /// + /// If the object is a delta object, return [`None`]. + pub fn base_object_hash(&self) -> Option { + match &self.info { + CacheObjectInfo::BaseObject(_, hash) => Some(*hash), + _ => None, + } + } + + /// Get the offset delta of the object. + /// + /// If the object is not an offset delta, return [`None`]. + pub fn offset_delta(&self) -> Option { + match &self.info { + CacheObjectInfo::OffsetDelta(offset, _) => Some(*offset), + _ => None, + } + } + + /// Get the hash delta of the object. + /// + /// If the object is not a hash delta, return [`None`]. + pub fn hash_delta(&self) -> Option { + match &self.info { + CacheObjectInfo::HashDelta(hash, _) => Some(*hash), + _ => None, + } + } + + /// transform the CacheObject to Entry + pub fn to_entry(&self) -> Entry { + match self.info { + CacheObjectInfo::BaseObject(obj_type, hash) => Entry { + obj_type, + data: self.data_decompressed.clone(), + hash, + chain_len: 0, + }, + _ => { + unreachable!("delta object should not persist!") + } + } + } +} + +/// trait alias for simple use +pub trait ArcWrapperBounds: + HeapSize + Serialize + for<'a> Deserialize<'a> + Send + Sync + 'static +{ +} +// You must impl `Alias Trait` for all the `T` satisfying Constraints +// Or, `T` will not satisfy `Alias Trait` even if it satisfies the Original traits +impl Deserialize<'a> + Send + Sync + 'static> ArcWrapperBounds + for T +{ +} + +/// Implementing encapsulation of Arc to enable third-party Trait HeapSize implementation for the Arc type +/// Because of use Arc in LruCache, the LruCache is not clear whether a pointer will drop the referenced +/// content when it is ejected from the cache, the actual memory usage is not accurate +pub struct ArcWrapper { + pub data: Arc, + complete_signal: Arc, + pool: Option>, + pub store_path: Option, // path to store when drop +} +impl ArcWrapper { + /// Create a new ArcWrapper + pub fn new(data: Arc, share_flag: Arc, pool: Option>) -> Self { + ArcWrapper { + data, + complete_signal: share_flag, + pool, + store_path: None, + } + } + pub fn set_store_path(&mut self, path: PathBuf) { + self.store_path = Some(path); + } +} + +impl HeapSize for ArcWrapper { + fn heap_size(&self) -> usize { + self.data.heap_size() + } +} + +impl Clone for ArcWrapper { + /// clone won't clone the store_path + fn clone(&self) -> Self { + ArcWrapper { + data: self.data.clone(), + complete_signal: self.complete_signal.clone(), + pool: self.pool.clone(), + store_path: None, + } + } +} + +impl Deref for ArcWrapper { + type Target = Arc; + fn deref(&self) -> &Self::Target { + &self.data + } +} +impl Drop for ArcWrapper { + // `drop` will be called in `lru_cache.insert()` when cache full & eject the LRU + // `lru_cache.insert()` is protected by Mutex + fn drop(&mut self) { + if !self.complete_signal.load(Ordering::Acquire) + && let Some(path) = &self.store_path + { + match &self.pool { + Some(pool) => { + let data_copy = self.data.clone(); + let path_copy = path.clone(); + let complete_signal = self.complete_signal.clone(); + // block entire process, wait for IO, Control Memory + // queue size will influence the Memory usage + while pool.queued_count() > 2000 { + std::thread::yield_now(); + } + pool.execute(move || { + if !complete_signal.load(Ordering::Acquire) { + let res = data_copy.f_save(&path_copy); + if let Err(e) = res { + println!("[f_save] {path_copy:?} error: {e:?}"); + } + } + }); + } + None => { + let res = self.data.f_save(path); + if let Err(e) = res { + println!("[f_save] {path:?} error: {e:?}"); + } + } + } + } + } +} +#[cfg(test)] +mod test { + use std::{fs, sync::Mutex}; + + use lru_mem::LruCache; + + use super::*; + #[test] + #[ignore = "only in single thread"] + // 只在单线程测试 + fn test_heap_size_record() { + let mut obj = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, SHA1::default()), + offset: 0, + data_decompressed: vec![0; 1024], + mem_recorder: None, + }; + let mem = Arc::new(AtomicUsize::default()); + assert_eq!(mem.load(Ordering::Relaxed), 0); + obj.set_mem_recorder(mem.clone()); + obj.record_mem_size(); + assert_eq!(mem.load(Ordering::Relaxed), 1120); + drop(obj); + assert_eq!(mem.load(Ordering::Relaxed), 0); + } + + #[test] + fn test_cache_object_with_same_size() { + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, SHA1::default()), + offset: 0, + data_decompressed: vec![0; 1024], + mem_recorder: None, + }; + assert!(a.heap_size() == 1024); + + // let b = ArcWrapper(Arc::new(a.clone())); + let b = ArcWrapper::new(Arc::new(a.clone()), Arc::new(AtomicBool::new(false)), None); + assert!(b.heap_size() == 1024); + } + #[test] + #[ignore] + fn test_cache_object_with_lru() { + let mut cache = LruCache::new(2048); + + let hash_a = SHA1::default(); + let hash_b = SHA1::new(b"b"); // whatever different hash + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_a), + offset: 0, + data_decompressed: vec![0; 1024], + mem_recorder: None, + }; + println!("a.heap_size() = {}", a.heap_size()); + + let b = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_b), + offset: 0, + data_decompressed: vec![0; (1024.0 * 1.5) as usize], + mem_recorder: None, + }; + { + let r = cache.insert( + hash_a.to_string(), + ArcWrapper::new(Arc::new(a.clone()), Arc::new(AtomicBool::new(true)), None), + ); + assert!(r.is_ok()) + } + { + let r = cache.try_insert( + hash_b.to_string(), + ArcWrapper::new(Arc::new(b.clone()), Arc::new(AtomicBool::new(true)), None), + ); + assert!(r.is_err()); + if let Err(lru_mem::TryInsertError::WouldEjectLru { .. }) = r { + // 匹配到指定错误，不需要额外操作 + } else { + panic!("Expected WouldEjectLru error"); + } + // 使用不同的键插入b，这样a会被驱逐 + let r = cache.insert( + hash_b.to_string(), + ArcWrapper::new(Arc::new(b.clone()), Arc::new(AtomicBool::new(true)), None), + ); + assert!(r.is_ok()); + } + { + // a should be ejected + let r = cache.get(&hash_a.to_string()); + assert!(r.is_some()); + } + } + + #[derive(Serialize, Deserialize)] + struct Test { + a: usize, + } + impl Drop for Test { + fn drop(&mut self) { + println!("drop Test"); + } + } + impl HeapSize for Test { + fn heap_size(&self) -> usize { + self.a + } + } + #[test] + fn test_lru_drop() { + println!("insert a"); + let cache = LruCache::new(2048); + let cache = Arc::new(Mutex::new(cache)); + { + let mut c = cache.as_ref().lock().unwrap(); + let _ = c.insert( + "a", + ArcWrapper::new( + Arc::new(Test { a: 1024 }), + Arc::new(AtomicBool::new(true)), + None, + ), + ); + } + println!("insert b, a should be ejected"); + { + let mut c = cache.as_ref().lock().unwrap(); + let _ = c.insert( + "b", + ArcWrapper::new( + Arc::new(Test { a: 1200 }), + Arc::new(AtomicBool::new(true)), + None, + ), + ); + } + let b = { + let mut c = cache.as_ref().lock().unwrap(); + c.get("b").cloned() + }; + println!("insert c, b should not be ejected"); + { + let mut c = cache.as_ref().lock().unwrap(); + let _ = c.insert( + "c", + ArcWrapper::new( + Arc::new(Test { a: 1200 }), + Arc::new(AtomicBool::new(true)), + None, + ), + ); + } + println!("user b: {}", b.as_ref().unwrap().a); + println!("test over, enject all"); + } + + #[test] + fn test_cache_object_serialize() { + let a = CacheObject { + info: CacheObjectInfo::BaseObject(ObjectType::Blob, SHA1::default()), + offset: 0, + data_decompressed: vec![0; 1024], + mem_recorder: None, + }; + let s = bincode::serde::encode_to_vec(&a, bincode::config::standard()).unwrap(); + let b: CacheObject = bincode::serde::decode_from_slice(&s, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(a.info, b.info); + assert_eq!(a.data_decompressed, b.data_decompressed); + assert_eq!(a.offset, b.offset); + } + + #[test] + fn test_arc_wrapper_drop_store() { + let mut path = PathBuf::from(".cache_temp/test_arc_wrapper_drop_store"); + fs::create_dir_all(&path).unwrap(); + path.push("test_obj"); + let mut a = ArcWrapper::new(Arc::new(1024), Arc::new(AtomicBool::new(false)), None); + a.set_store_path(path.clone()); + drop(a); + + assert!(path.exists()); + path.pop(); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + /// test warpper can't correctly store the data when lru eject it + fn test_arc_wrapper_with_lru() { + let mut cache = LruCache::new(1500); + let path = PathBuf::from(".cache_temp/test_arc_wrapper_with_lru"); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + let shared_flag = Arc::new(AtomicBool::new(false)); + + // insert a, a not ejected + let a_path = path.join("a"); + { + let mut a = ArcWrapper::new(Arc::new(Test { a: 1024 }), shared_flag.clone(), None); + a.set_store_path(a_path.clone()); + let b = ArcWrapper::new(Arc::new(1024), shared_flag.clone(), None); + assert!(b.store_path.is_none()); + + println!("insert a with heap size: {:?}", a.heap_size()); + let rt = cache.insert("a", a); + if let Err(e) = rt { + panic!("{}", format!("insert a failed: {:?}", e.to_string())); + } + println!("after insert a, cache used = {}", cache.current_size()); + } + assert!(!a_path.exists()); + + let b_path = path.join("b"); + // insert b, a should be ejected + { + let mut b = ArcWrapper::new(Arc::new(Test { a: 996 }), shared_flag.clone(), None); + b.set_store_path(b_path.clone()); + let rt = cache.insert("b", b); + if let Err(e) = rt { + panic!("{}", format!("insert a failed: {:?}", e.to_string())); + } + println!("after insert b, cache used = {}", cache.current_size()); + } + assert!(a_path.exists()); + assert!(!b_path.exists()); + shared_flag.store(true, Ordering::Release); + fs::remove_dir_all(path).unwrap(); + // should pass even b's path not exists + } +} diff --git a/mercury/src/internal/pack/channel_reader.rs b/mercury/src/internal/pack/channel_reader.rs new file mode 100644 index 00000000..9402eed7 --- /dev/null +++ b/mercury/src/internal/pack/channel_reader.rs @@ -0,0 +1,51 @@ +use std::io::{self}; +use std::io::{BufRead, Read}; +use std::sync::mpsc::Receiver; + +/// Custom BufRead implementation that reads from the channel +pub(crate) struct StreamBufReader { + receiver: Receiver>, + buffer: io::Cursor>, +} + +impl StreamBufReader { + pub(crate) fn new(receiver: Receiver>) -> Self { + StreamBufReader { + receiver, + buffer: io::Cursor::new(Vec::new()), + } + } +} + +impl Read for StreamBufReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.buffer.position() as usize == self.buffer.get_ref().len() { + // buffer has been read completely + match self.receiver.recv() { + Ok(data) => { + self.buffer = io::Cursor::new(data); + } + Err(_) => return Ok(0), // Channel is closed + } + } + self.buffer.read(buf) + } +} + +impl BufRead for StreamBufReader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + if self.buffer.position() as usize == self.buffer.get_ref().len() { + match self.receiver.recv() { + Ok(data) => { + self.buffer = io::Cursor::new(data); + } + Err(_) => return Ok(&[]), // Channel is closed + } + } + self.buffer.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.buffer.consume(amt); + } +} diff --git a/mercury/src/internal/pack/decode.rs b/mercury/src/internal/pack/decode.rs new file mode 100644 index 00000000..e0f100aa --- /dev/null +++ b/mercury/src/internal/pack/decode.rs @@ -0,0 +1,905 @@ +use std::io::{self, BufRead, Cursor, ErrorKind, Read}; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread::{self, JoinHandle}; +use std::time::Instant; + +use axum::Error; +use bytes::Bytes; +use flate2::bufread::ZlibDecoder; +use futures_util::{Stream, StreamExt}; +use threadpool::ThreadPool; +use tokio::sync::mpsc::UnboundedSender; +use uuid::Uuid; + +use crate::errors::GitError; +use crate::hash::SHA1; +use crate::internal::object::types::ObjectType; + +use super::cache_object::CacheObjectInfo; +use crate::internal::pack::cache::_Cache; +use crate::internal::pack::cache::Caches; +use crate::internal::pack::cache_object::{CacheObject, MemSizeRecorder}; +use crate::internal::pack::channel_reader::StreamBufReader; +use crate::internal::pack::entry::Entry; +use crate::internal::pack::waitlist::Waitlist; +use crate::internal::pack::wrapper::Wrapper; +use crate::internal::pack::{DEFAULT_TMP_DIR, Pack, utils}; +use crate::utils::CountingReader; + +/// For the convenience of passing parameters +struct SharedParams { + pub pool: Arc, + pub waitlist: Arc, + pub caches: Arc, + pub cache_objs_mem_size: Arc, + pub callback: Arc, +} + +impl Drop for Pack { + fn drop(&mut self) { + if self.clean_tmp { + self.caches.remove_tmp_dir(); + } + } +} + +impl Pack { + /// # Parameters + /// - `thread_num`: The number of threads to use for decoding and cache, `None` mean use the number of logical CPUs. + /// It can't be zero, or panic
+ /// - `mem_limit`: The maximum size of the memory cache in bytes, or None for unlimited. + /// The 80% of it will be used for [Caches]
+ /// ​**Not very accurate, because of memory alignment and other reasons, overuse about 15%**
+ /// - `temp_path`: The path to a directory for temporary files, default is "./.cache_temp"
+ /// For example, thread_num = 4 will use up to 8 threads (4 for decoding and 4 for cache)
+ /// - `clean_tmp`: whether to remove temp directory when Pack is dropped + pub fn new( + thread_num: Option, + mem_limit: Option, + temp_path: Option, + clean_tmp: bool, + ) -> Self { + let mut temp_path = temp_path.unwrap_or(PathBuf::from(DEFAULT_TMP_DIR)); + // add 8 random characters as subdirectory, check if the directory exists + loop { + let sub_dir = Uuid::new_v4().to_string()[..8].to_string(); + temp_path.push(sub_dir); + if !temp_path.exists() { + break; + } + temp_path.pop(); + } + let thread_num = thread_num.unwrap_or_else(num_cpus::get); + let cache_mem_size = mem_limit.map(|mem_limit| mem_limit * 4 / 5); + Pack { + number: 0, + signature: SHA1::default(), + objects: Vec::new(), + pool: Arc::new(ThreadPool::new(thread_num)), + waitlist: Arc::new(Waitlist::new()), + caches: Arc::new(Caches::new(cache_mem_size, temp_path, thread_num)), + mem_limit, + cache_objs_mem: Arc::new(AtomicUsize::default()), + clean_tmp, + } + } + + /// Checks and reads the header of a Git pack file. + /// + /// This function reads the first 12 bytes of a pack file, which include the b"PACK" magic identifier, + /// the version number, and the number of objects in the pack. It verifies that the magic identifier + /// is correct and that the version number is 2 (which is the version currently supported by Git). + /// It also collects these header bytes for later use, such as for hashing the entire pack file. + /// + /// # Parameters + /// * `pack` - A mutable reference to an object implementing the `Read` trait, + /// representing the source of the pack file data (e.g., file, memory stream). + /// + /// # Returns + /// A `Result` which is: + /// * `Ok((u32, Vec))`: On successful reading and validation of the header, returns a tuple where: + /// - The first element is the number of objects in the pack file (`u32`). + /// - The second element is a vector containing the bytes of the pack file header (`Vec`). + /// * `Err(GitError)`: On failure, returns a [`GitError`] with a description of the issue. + /// + /// # Errors + /// This function can return an error in the following situations: + /// * If the pack file does not start with the "PACK" magic identifier. + /// * If the pack file's version number is not 2. + /// * If there are any issues reading from the provided `pack` source. + pub fn check_header(pack: &mut impl BufRead) -> Result<(u32, Vec), GitError> { + // A vector to store the header data for hashing later + let mut header_data = Vec::new(); + + // Read the first 4 bytes which should be "PACK" + let mut magic = [0; 4]; + // Read the magic "PACK" identifier + let result = pack.read_exact(&mut magic); + match result { + Ok(_) => { + // Store these bytes for later + header_data.extend_from_slice(&magic); + + // Check if the magic bytes match "PACK" + if magic != *b"PACK" { + // If not, return an error indicating invalid pack header + return Err(GitError::InvalidPackHeader(format!( + "{},{},{},{}", + magic[0], magic[1], magic[2], magic[3] + ))); + } + } + Err(e) => { + // If there is an error in reading, return a GitError + return Err(GitError::InvalidPackFile(format!( + "Error reading magic identifier: {e}" + ))); + } + } + + // Read the next 4 bytes for the version number + let mut version_bytes = [0; 4]; + let result = pack.read_exact(&mut version_bytes); // Read the version number + match result { + Ok(_) => { + // Store these bytes + header_data.extend_from_slice(&version_bytes); + + // Convert the version bytes to an u32 integer + let version = u32::from_be_bytes(version_bytes); + if version != 2 { + // Git currently supports version 2, so error if not version 2 + return Err(GitError::InvalidPackFile(format!( + "Version Number is {version}, not 2" + ))); + } + } + Err(e) => { + // If there is an error in reading, return a GitError + return Err(GitError::InvalidPackFile(format!( + "Error reading version number: {e}" + ))); + } + } + + // Read the next 4 bytes for the number of objects in the pack + let mut object_num_bytes = [0; 4]; + // Read the number of objects + let result = pack.read_exact(&mut object_num_bytes); + match result { + Ok(_) => { + // Store these bytes + header_data.extend_from_slice(&object_num_bytes); + // Convert the object number bytes to an u32 integer + let object_num = u32::from_be_bytes(object_num_bytes); + // Return the number of objects and the header data for further processing + Ok((object_num, header_data)) + } + Err(e) => { + // If there is an error in reading, return a GitError + Err(GitError::InvalidPackFile(format!( + "Error reading object number: {e}" + ))) + } + } + } + + /// Decompresses data from a given Read and BufRead source using Zlib decompression. + /// + /// # Parameters + /// * `pack`: A source that implements both Read and BufRead traits (e.g., file, network stream). + /// * `expected_size`: The expected decompressed size of the data. + /// + /// # Returns + /// Returns a `Result` containing either: + /// * A tuple with a `Vec` of the decompressed data and the total number of input bytes processed, + /// * Or a `GitError` in case of a mismatch in expected size or any other reading error. + /// + pub fn decompress_data( + pack: &mut (impl BufRead + Send), + expected_size: usize, + ) -> Result<(Vec, usize), GitError> { + // Create a buffer with the expected size for the decompressed data + let mut buf = Vec::with_capacity(expected_size); + + let mut counting_reader = CountingReader::new(pack); + // Create a new Zlib decoder with the original data + //let mut deflate = ZlibDecoder::new(pack); + let mut deflate = ZlibDecoder::new(&mut counting_reader); + // Attempt to read data to the end of the buffer + match deflate.read_to_end(&mut buf) { + Ok(_) => { + // Check if the length of the buffer matches the expected size + if buf.len() != expected_size { + Err(GitError::InvalidPackFile(format!( + "The object size {} does not match the expected size {}", + buf.len(), + expected_size + ))) + } else { + // If everything is as expected, return the buffer, the original data, and the total number of input bytes processed + let actual_input_bytes = counting_reader.bytes_read as usize; + Ok((buf, actual_input_bytes)) + } + } + Err(e) => { + // If there is an error in reading, return a GitError + Err(GitError::InvalidPackFile(format!( + "Decompression error: {e}" + ))) + } + } + } + + /// Decodes a pack object from a given Read and BufRead source and returns the object as a [`CacheObject`]. + /// + /// # Parameters + /// * `pack`: A source that implements both Read and BufRead traits. + /// * `offset`: A mutable reference to the current offset within the pack. + /// + /// # Returns + /// Returns a `Result` containing either: + /// * A tuple of the next offset in the pack and the original compressed data as `Vec`, + /// * Or a `GitError` in case of any reading or decompression error. + /// + pub fn decode_pack_object( + pack: &mut (impl BufRead + Send), + offset: &mut usize, + ) -> Result { + let init_offset = *offset; + + // Attempt to read the type and size, handle potential errors + let (type_bits, size) = match utils::read_type_and_varint_size(pack, offset) { + Ok(result) => result, + Err(e) => { + // Handle the error e.g., by logging it or converting it to GitError + // and then return from the function + return Err(GitError::InvalidPackFile(format!("Read error: {e}"))); + } + }; + + // Check if the object type is valid + let t = ObjectType::from_u8(type_bits)?; + + match t { + ObjectType::Commit | ObjectType::Tree | ObjectType::Blob | ObjectType::Tag => { + let (data, raw_size) = Pack::decompress_data(pack, size)?; + *offset += raw_size; + Ok(CacheObject::new_for_undeltified(t, data, init_offset)) + } + ObjectType::OffsetDelta | ObjectType::OffsetZstdelta => { + let (delta_offset, bytes) = utils::read_offset_encoding(pack).unwrap(); + *offset += bytes; + + let (data, raw_size) = Pack::decompress_data(pack, size)?; + *offset += raw_size; + + // Count the base object offset: the current offset - delta offset + let base_offset = init_offset + .checked_sub(delta_offset as usize) + .ok_or_else(|| { + GitError::InvalidObjectInfo("Invalid OffsetDelta offset".to_string()) + }) + .unwrap(); + + let mut reader = Cursor::new(&data); + let (_, final_size) = utils::read_delta_object_size(&mut reader)?; + + let obj_info = match t { + ObjectType::OffsetDelta => { + CacheObjectInfo::OffsetDelta(base_offset, final_size) + } + ObjectType::OffsetZstdelta => { + CacheObjectInfo::OffsetZstdelta(base_offset, final_size) + } + _ => unreachable!(), + }; + Ok(CacheObject { + info: obj_info, + offset: init_offset, + data_decompressed: data, + mem_recorder: None, + }) + } + ObjectType::HashDelta => { + // Read 20 bytes to get the reference object SHA1 hash + let ref_sha1 = SHA1::from_stream(pack).unwrap(); + // Offset is incremented by 20 bytes + *offset += SHA1::SIZE; + + let (data, raw_size) = Pack::decompress_data(pack, size)?; + *offset += raw_size; + + let mut reader = Cursor::new(&data); + let (_, final_size) = utils::read_delta_object_size(&mut reader)?; + + Ok(CacheObject { + info: CacheObjectInfo::HashDelta(ref_sha1, final_size), + offset: init_offset, + data_decompressed: data, + mem_recorder: None, + }) + } + } + } + + /// Decodes a pack file from a given Read and BufRead source, for each object in the pack, + /// it decodes the object and processes it using the provided callback function. + pub fn decode( + &mut self, + pack: &mut (impl BufRead + Send), + callback: F, + ) -> Result<(), GitError> + where + F: Fn(Entry, usize) + Sync + Send + 'static, + { + let time = Instant::now(); + let mut last_update_time = time.elapsed().as_millis(); + let log_info = |_i: usize, pack: &Pack| { + tracing::info!( + "time {:.2} s \t decode: {:?} \t dec-num: {} \t cah-num: {} \t Objs: {} MB \t CacheUsed: {} MB", + time.elapsed().as_millis() as f64 / 1000.0, + _i, + pack.pool.queued_count(), + pack.caches.queued_tasks(), + pack.cache_objs_mem_used() / 1024 / 1024, + pack.caches.memory_used() / 1024 / 1024 + ); + }; + let callback = Arc::new(callback); + + let caches = self.caches.clone(); + let mut reader = Wrapper::new(io::BufReader::new(pack)); + + let result = Pack::check_header(&mut reader); + match result { + Ok((object_num, _)) => { + self.number = object_num as usize; + } + Err(e) => { + return Err(e); + } + } + tracing::info!("The pack file has {} objects", self.number); + let mut offset: usize = 12; + let mut i = 0; + while i < self.number { + // log per 1000 objects and 1 second + if i % 1000 == 0 { + let time_now = time.elapsed().as_millis(); + if time_now - last_update_time > 1000 { + log_info(i, self); + last_update_time = time_now; + } + } + // 3 parts: Waitlist + TheadPool + Caches + // hardcode the limit of the tasks of threads_pool queue, to limit memory + while self.pool.queued_count() > 2000 + || self + .mem_limit + .map(|limit| self.memory_used() > limit) + .unwrap_or(false) + { + thread::yield_now(); + } + let r: Result = + Pack::decode_pack_object(&mut reader, &mut offset); + match r { + Ok(mut obj) => { + obj.set_mem_recorder(self.cache_objs_mem.clone()); + obj.record_mem_size(); + + // Wrapper of Arc Params, for convenience to pass + let params = Arc::new(SharedParams { + pool: self.pool.clone(), + waitlist: self.waitlist.clone(), + caches: self.caches.clone(), + cache_objs_mem_size: self.cache_objs_mem.clone(), + callback: callback.clone(), + }); + + let caches = caches.clone(); + let waitlist = self.waitlist.clone(); + self.pool.execute(move || { + match obj.info { + CacheObjectInfo::BaseObject(_, _) => { + Self::cache_obj_and_process_waitlist(params, obj); + } + CacheObjectInfo::OffsetDelta(base_offset, _) + | CacheObjectInfo::OffsetZstdelta(base_offset, _) => { + if let Some(base_obj) = caches.get_by_offset(base_offset) { + Self::process_delta(params, obj, base_obj); + } else { + // You can delete this 'if' block ↑, because there are Second check in 'else' + // It will be more readable, but the performance will be slightly reduced + waitlist.insert_offset(base_offset, obj); + // Second check: prevent that the base_obj thread has finished before the waitlist insert + if let Some(base_obj) = caches.get_by_offset(base_offset) { + Self::process_waitlist(params, base_obj); + } + } + } + CacheObjectInfo::HashDelta(base_ref, _) => { + if let Some(base_obj) = caches.get_by_hash(base_ref) { + Self::process_delta(params, obj, base_obj); + } else { + waitlist.insert_ref(base_ref, obj); + if let Some(base_obj) = caches.get_by_hash(base_ref) { + Self::process_waitlist(params, base_obj); + } + } + } + } + }); + } + Err(e) => { + return Err(e); + } + } + i += 1; + } + log_info(i, self); + let render_hash = reader.final_hash(); + self.signature = SHA1::from_stream(&mut reader).unwrap(); + + if render_hash != self.signature { + return Err(GitError::InvalidPackFile(format!( + "The pack file hash {} does not match the trailer hash {}", + render_hash, self.signature + ))); + } + + let end = utils::is_eof(&mut reader); + if !end { + return Err(GitError::InvalidPackFile( + "The pack file is not at the end".to_string(), + )); + } + + self.pool.join(); // wait for all threads to finish + // !Attention: Caches threadpool may not stop, but it's not a problem (garbage file data) + // So that files != self.number + assert_eq!(self.waitlist.map_offset.len(), 0); + assert_eq!(self.waitlist.map_ref.len(), 0); + assert_eq!(self.number, caches.total_inserted()); + tracing::info!( + "The pack file has been decoded successfully, takes: [ {:?} ]", + time.elapsed() + ); + self.caches.clear(); // clear cached objects & stop threads + assert_eq!(self.cache_objs_mem_used(), 0); // all the objs should be dropped until here + + // impl in Drop Trait + // if self.clean_tmp { + // self.caches.remove_tmp_dir(); + // } + + Ok(()) + } + + /// Decode a Pack in a new thread and send the CacheObjects while decoding. + ///
Attention: It will consume the `pack` and return in a JoinHandle. + pub fn decode_async( + mut self, + mut pack: impl BufRead + Send + 'static, + sender: UnboundedSender, + ) -> JoinHandle { + thread::spawn(move || { + self.decode(&mut pack, move |entry, _| { + if let Err(e) = sender.send(entry) { + eprintln!("Channel full, failed to send entry: {e:?}"); + } + }) + .unwrap(); + self + }) + } + + /// Decodes a `Pack` from a `Stream` of `Bytes`, and sends the `Entry` while decoding. + pub async fn decode_stream( + mut self, + mut stream: impl Stream> + Unpin + Send + 'static, + sender: UnboundedSender, + ) -> Self { + let (tx, rx) = std::sync::mpsc::channel(); + let mut reader = StreamBufReader::new(rx); + tokio::spawn(async move { + while let Some(chunk) = stream.next().await { + let data = chunk.unwrap().to_vec(); + if let Err(e) = tx.send(data) { + eprintln!("Sending Error: {e:?}"); + break; + } + } + }); + // CPU-bound task, so use spawn_blocking + // DO NOT use thread::spawn, because it will block tokio runtime (if single-threaded runtime, like in tests) + tokio::task::spawn_blocking(move || { + self.decode(&mut reader, move |entry: Entry, _| { + // as we used unbound channel here, it will never full so can be send with synchronous + if let Err(e) = sender.send(entry) { + eprintln!("unbound channel Sending Error: {e:?}"); + } + }) + .unwrap(); + self + }) + .await + .unwrap() + } + + /// CacheObjects + Index size of Caches + fn memory_used(&self) -> usize { + self.cache_objs_mem_used() + self.caches.memory_used_index() + } + + /// The total memory used by the CacheObjects of this Pack + fn cache_objs_mem_used(&self) -> usize { + self.cache_objs_mem.load(Ordering::Acquire) + } + + /// Rebuild the Delta Object in a new thread & process the objects waiting for it recursively. + ///
This function must be *static*, because [&self] can't be moved into a new thread. + fn process_delta( + shared_params: Arc, + delta_obj: CacheObject, + base_obj: Arc, + ) { + shared_params.pool.clone().execute(move || { + let mut new_obj = match delta_obj.info { + CacheObjectInfo::OffsetDelta(_, _) | CacheObjectInfo::HashDelta(_, _) => { + Pack::rebuild_delta(delta_obj, base_obj) + } + CacheObjectInfo::OffsetZstdelta(_, _) => { + Pack::rebuild_zstdelta(delta_obj, base_obj) + } + _ => unreachable!(), + }; + + new_obj.set_mem_recorder(shared_params.cache_objs_mem_size.clone()); + new_obj.record_mem_size(); + Self::cache_obj_and_process_waitlist(shared_params, new_obj); //Indirect Recursion + }); + } + + /// Cache the new object & process the objects waiting for it (in multi-threading). + fn cache_obj_and_process_waitlist(shared_params: Arc, new_obj: CacheObject) { + (shared_params.callback)(new_obj.to_entry(), new_obj.offset); + let new_obj = shared_params.caches.insert( + new_obj.offset, + new_obj.base_object_hash().unwrap(), + new_obj, + ); + Self::process_waitlist(shared_params, new_obj); + } + + fn process_waitlist(shared_params: Arc, base_obj: Arc) { + let wait_objs = shared_params + .waitlist + .take(base_obj.offset, base_obj.base_object_hash().unwrap()); + for obj in wait_objs { + // Process the objects waiting for the new object(base_obj = new_obj) + Self::process_delta(shared_params.clone(), obj, base_obj.clone()); + } + } + + /// Reconstruct the Delta Object based on the "base object" + /// and return the new object. + pub fn rebuild_delta(delta_obj: CacheObject, base_obj: Arc) -> CacheObject { + const COPY_INSTRUCTION_FLAG: u8 = 1 << 7; + const COPY_OFFSET_BYTES: u8 = 4; + const COPY_SIZE_BYTES: u8 = 3; + const COPY_ZERO_SIZE: usize = 0x10000; + + let mut stream = Cursor::new(&delta_obj.data_decompressed); + + // Read the base object size + // (Size Encoding) + let (base_size, result_size) = utils::read_delta_object_size(&mut stream).unwrap(); + + // Get the base object data + let base_info = &base_obj.data_decompressed; + assert_eq!(base_info.len(), base_size, "Base object size mismatch"); + + let mut result = Vec::with_capacity(result_size); + + loop { + // Check if the stream has ended, meaning the new object is done + let instruction = match utils::read_bytes(&mut stream) { + Ok([instruction]) => instruction, + Err(err) if err.kind() == ErrorKind::UnexpectedEof => break, + Err(err) => { + panic!( + "{}", + GitError::DeltaObjectError(format!("Wrong instruction in delta :{err}")) + ); + } + }; + + if instruction & COPY_INSTRUCTION_FLAG == 0 { + // Data instruction; the instruction byte specifies the number of data bytes + if instruction == 0 { + // Appending 0 bytes doesn't make sense, so git disallows it + panic!( + "{}", + GitError::DeltaObjectError(String::from("Invalid data instruction")) + ); + } + + // Append the provided bytes + let mut data = vec![0; instruction as usize]; + stream.read_exact(&mut data).unwrap(); + result.extend_from_slice(&data); + } else { + // Copy instruction + // +----------+---------+---------+---------+---------+-------+-------+-------+ + // | 1xxxxxxx | offset1 | offset2 | offset3 | offset4 | size1 | size2 | size3 | + // +----------+---------+---------+---------+---------+-------+-------+-------+ + let mut nonzero_bytes = instruction; + let offset = + utils::read_partial_int(&mut stream, COPY_OFFSET_BYTES, &mut nonzero_bytes) + .unwrap(); + let mut size = + utils::read_partial_int(&mut stream, COPY_SIZE_BYTES, &mut nonzero_bytes) + .unwrap(); + if size == 0 { + // Copying 0 bytes doesn't make sense, so git assumes a different size + size = COPY_ZERO_SIZE; + } + // Copy bytes from the base object + let base_data = base_info.get(offset..(offset + size)).ok_or_else(|| { + GitError::DeltaObjectError("Invalid copy instruction".to_string()) + }); + + match base_data { + Ok(data) => result.extend_from_slice(data), + Err(e) => panic!("{}", e), + } + } + } + assert_eq!(result_size, result.len(), "Result size mismatch"); + + let hash = utils::calculate_object_hash(base_obj.object_type(), &result); + // create new obj from `delta_obj` & `result` instead of modifying `delta_obj` for heap-size recording + CacheObject { + info: CacheObjectInfo::BaseObject(base_obj.object_type(), hash), + offset: delta_obj.offset, + data_decompressed: result, + mem_recorder: None, + } // Canonical form (Complete Object) + // Memory recording will happen after this function returns. See `process_delta` + } + pub fn rebuild_zstdelta(delta_obj: CacheObject, base_obj: Arc) -> CacheObject { + let result = zstdelta::apply(&base_obj.data_decompressed, &delta_obj.data_decompressed) + .expect("Failed to apply zstdelta"); + let hash = utils::calculate_object_hash(base_obj.object_type(), &result); + CacheObject { + info: CacheObjectInfo::BaseObject(base_obj.object_type(), hash), + offset: delta_obj.offset, + data_decompressed: result, + mem_recorder: None, + } // Canonical form (Complete Object) + // Memory recording will happen after this function returns. See `process_delta` + } +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::io::BufReader; + use std::io::Cursor; + use std::io::prelude::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::{env, path::PathBuf}; + + use flate2::Compression; + use flate2::write::ZlibEncoder; + use tokio_util::io::ReaderStream; + + use crate::internal::pack::Pack; + use crate::internal::pack::tests::init_logger; + use futures_util::TryStreamExt; + + #[tokio::test] + async fn test_pack_check_header() { + let res = crate::test_utils::setup_lfs_file().await; + println!("{res:?}"); + let source = res + .get("git-2d187177923cd618a75da6c6db45bb89d92bd504.pack") + .unwrap(); + + let f = fs::File::open(source).unwrap(); + let mut buf_reader = BufReader::new(f); + let (object_num, _) = Pack::check_header(&mut buf_reader).unwrap(); + + assert_eq!(object_num, 358109); + } + + #[test] + fn test_decompress_data() { + let data = b"Hello, world!"; // Sample data to compress and then decompress + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(data).unwrap(); + let compressed_data = encoder.finish().unwrap(); + let compressed_size = compressed_data.len(); + + // Create a cursor for the compressed data to simulate a BufRead source + let mut cursor: Cursor> = Cursor::new(compressed_data); + let expected_size = data.len(); + + // Decompress the data and assert correctness + let result = Pack::decompress_data(&mut cursor, expected_size); + match result { + Ok((decompressed_data, bytes_read)) => { + assert_eq!(bytes_read, compressed_size); + assert_eq!(decompressed_data, data); + } + Err(e) => panic!("Decompression failed: {e:?}"), + } + } + + #[test] + fn test_pack_decode_without_delta() { + let mut source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + source.push("tests/data/packs/pack-1d0e6c14760c956c173ede71cb28f33d921e232f.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); + p.decode(&mut buffered, |_, _| {}).unwrap(); + } + + #[test] + // #[traced_test] + fn test_pack_decode_with_ref_delta() { + init_logger(); + + let mut source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + source.push("tests/data/packs/ref-delta-65d47638aa7cb7c39f1bd1d5011a415439b887a8.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); + p.decode(&mut buffered, |_, _| {}).unwrap(); + } + + #[test] + fn test_pack_decode_no_mem_limit() { + let mut source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + source.push("tests/data/packs/pack-1d0e6c14760c956c173ede71cb28f33d921e232f.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, None, Some(tmp), true); + p.decode(&mut buffered, |_, _| {}).unwrap(); + } + + #[tokio::test] + #[ignore] // Take too long time + async fn test_pack_decode_with_large_file_with_delta_without_ref() { + init_logger(); + let file_map = crate::test_utils::setup_lfs_file().await; + let source = file_map + .get("git-2d187177923cd618a75da6c6db45bb89d92bd504.pack") + .unwrap(); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new( + Some(20), + Some(1024 * 1024 * 1024 * 2), + Some(tmp.clone()), + true, + ); + let rt = p.decode(&mut buffered, |_obj, _offset| { + // println!("{:?} {}", obj.hash.to_string(), offset); + }); + if let Err(e) = rt { + fs::remove_dir_all(tmp).unwrap(); + panic!("Error: {e:?}"); + } + } // it will be stuck on dropping `Pack` on Windows if `mem_size` is None, so we need `mimalloc` + + #[tokio::test] + async fn test_decode_large_file_stream() { + init_logger(); + let file_map = crate::test_utils::setup_lfs_file().await; + let source = file_map + .get("git-2d187177923cd618a75da6c6db45bb89d92bd504.pack") + .unwrap(); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + let f = tokio::fs::File::open(source).await.unwrap(); + let stream = ReaderStream::new(f).map_err(axum::Error::new); + let p = Pack::new( + Some(20), + Some(1024 * 1024 * 1024 * 4), + Some(tmp.clone()), + true, + ); + + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let handle = tokio::spawn(async move { p.decode_stream(stream, tx).await }); + let count = Arc::new(AtomicUsize::new(0)); + let count_c = count.clone(); + // in tests, RUNTIME is single-threaded, so `sync code` will block the tokio runtime + let consume = tokio::spawn(async move { + let mut cnt = 0; + while let Some(_entry) = rx.recv().await { + cnt += 1; + } + tracing::info!("Received: {}", cnt); + count_c.store(cnt, Ordering::Release); + }); + let p = handle.await.unwrap(); + consume.await.unwrap(); + assert_eq!(count.load(Ordering::Acquire), p.number); + assert_eq!(p.number, 358109); + } + + #[tokio::test] + #[ignore] // Take too long time, duplicate with `test_decode_large_file_stream` + async fn test_decode_large_file_async() { + let file_map = crate::test_utils::setup_lfs_file().await; + let source = file_map + .get("git-2d187177923cd618a75da6c6db45bb89d92bd504.pack") + .unwrap(); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + let f = fs::File::open(source).unwrap(); + let buffered = BufReader::new(f); + let p = Pack::new( + Some(20), + Some(1024 * 1024 * 1024 * 2), + Some(tmp.clone()), + true, + ); + + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let handle = p.decode_async(buffered, tx); // new thread + let mut cnt = 0; + while let Ok(_entry) = rx.try_recv() { + cnt += 1; //use entry here + } + let p = handle.join().unwrap(); + assert_eq!(cnt, p.number); + } + + #[test] + fn test_pack_decode_with_delta_without_ref() { + let mut source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + source.push("tests/data/packs/pack-d50df695086eea6253a237cb5ac44af1629e7ced.pack"); + + let tmp = PathBuf::from("/tmp/.cache_temp"); + + let f = fs::File::open(source).unwrap(); + let mut buffered = BufReader::new(f); + let mut p = Pack::new(None, Some(1024 * 1024 * 20), Some(tmp), true); + p.decode(&mut buffered, |_, _| {}).unwrap(); + } + + #[test] + #[ignore] // Take too long time + fn test_pack_decode_multi_task_with_large_file_with_delta_without_ref() { + let task1 = std::thread::spawn(|| { + test_pack_decode_with_large_file_with_delta_without_ref(); + }); + let task2 = std::thread::spawn(|| { + test_pack_decode_with_large_file_with_delta_without_ref(); + }); + + task1.join().unwrap(); + task2.join().unwrap(); + } +} diff --git a/mercury/src/internal/pack/encode.rs b/mercury/src/internal/pack/encode.rs new file mode 100644 index 00000000..a5ab8443 --- /dev/null +++ b/mercury/src/internal/pack/encode.rs @@ -0,0 +1,853 @@ +use std::cmp::Ordering; +use std::collections::VecDeque; +use std::io::Write; + +use crate::internal::object::types::ObjectType; +use crate::time_it; +use crate::{errors::GitError, hash::SHA1, internal::pack::entry::Entry}; +use ahash::AHasher; +use flate2::write::ZlibEncoder; +use rayon::prelude::*; +use sha1::{Digest, Sha1}; +use std::hash::{Hash, Hasher}; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; + +const MAX_CHAIN_LEN: usize = 50; +const MIN_DELTA_RATE: f64 = 0.5; // minimum delta rate +//const MAX_ZSTDELTA_CHAIN_LEN: usize = 50; + +/// A encoder for generating pack files with delta objects. +pub struct PackEncoder { + object_number: usize, + process_index: usize, + window_size: usize, + // window: VecDeque<(Entry, usize)>, // entry and offset + sender: Option>>, + inner_offset: usize, // offset of current entry + inner_hash: Sha1, // Not SHA1 because need update trait + final_hash: Option, + start_encoding: bool, +} + +/// Encode header of pack file (12 byte)
+/// Content: 'PACK', Version(2), number of objects +fn encode_header(object_number: usize) -> Vec { + let mut result: Vec = vec![ + b'P', b'A', b'C', b'K', // The logotype of the Pack File + 0, 0, 0, 2, // generates version 2 only. + ]; + assert_ne!(object_number, 0); // guarantee self.number_of_objects!=0 + assert!(object_number < (1 << 32)); + //TODO: GitError:numbers of objects should < 4G , + result.append((object_number as u32).to_be_bytes().to_vec().as_mut()); // to 4 bytes (network byte order aka. big-endian) + result +} + +/// Encode offset of delta object +fn encode_offset(mut value: usize) -> Vec { + assert_ne!(value, 0, "offset can't be zero"); + let mut bytes = Vec::new(); + + bytes.push((value & 0x7F) as u8); + value >>= 7; + while value != 0 { + value -= 1; + let byte = (value & 0x7F) as u8 | 0x80; // set first bit one + value >>= 7; + bytes.push(byte); + } + bytes.reverse(); + bytes +} + +/// Encode one object, and update the hash +/// @offset: offset of this object if it's a delta object. For other object, it's None +fn encode_one_object(entry: &Entry, offset: Option) -> Result, GitError> { + // try encode as delta + let obj_data = &entry.data; + let obj_data_len = obj_data.len(); + let obj_type_number = entry.obj_type.to_u8(); + + let mut encoded_data = Vec::new(); + + // **header** encoding + let mut header_data = vec![(0x80 | (obj_type_number << 4)) + (obj_data_len & 0x0f) as u8]; + let mut size = obj_data_len >> 4; // 4 bit has been used in first byte + if size > 0 { + while size > 0 { + if size >> 7 > 0 { + header_data.push((0x80 | size) as u8); + size >>= 7; + } else { + header_data.push(size as u8); + break; + } + } + } else { + header_data.push(0); + } + encoded_data.extend(header_data); + + // **offset** encoding + if entry.obj_type == ObjectType::OffsetDelta || entry.obj_type == ObjectType::OffsetZstdelta { + let offset_data = encode_offset(offset.unwrap()); + encoded_data.extend(offset_data); + } else if entry.obj_type == ObjectType::HashDelta { + unreachable!("unsupported type") + } + + // **data** encoding, need zlib compress + let mut inflate = ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + inflate + .write_all(obj_data) + .expect("zlib compress should never failed"); + inflate.flush().expect("zlib flush should never failed"); + let compressed_data = inflate.finish().expect("zlib compress should never failed"); + // self.write_all_and_update(&compressed_data).await; + encoded_data.extend(compressed_data); + Ok(encoded_data) +} + +fn magic_sort(a: &Entry, b: &Entry) -> Ordering { + // let ord = b.obj_type.to_u8().cmp(&a.obj_type.to_u8()); + // if ord != Ordering::Equal { + // return ord; + // } + + // the hash should be file hash not content hash + // todo the feature need larger refactor + // let ord = b.hash.cmp(&a.hash); + // if ord != Ordering::Equal { return ord; } + + let ord = b.data.len().cmp(&a.data.len()); + if ord != Ordering::Equal { + return ord; + } + + // fallback pointer order (newest first) + (a as *const Entry).cmp(&(b as *const Entry)) +} + +fn calc_hash(data: &[u8]) -> u64 { + let mut hasher = AHasher::default(); + data.hash(&mut hasher); + hasher.finish() +} + +fn cheap_similar(a: &[u8], b: &[u8]) -> bool { + let k = a.len().min(b.len()).min(128); + if k == 0 { + return false; + } + calc_hash(&a[..k]) == calc_hash(&b[..k]) +} + +impl PackEncoder { + pub fn new(object_number: usize, window_size: usize, sender: mpsc::Sender>) -> Self { + PackEncoder { + object_number, + window_size, + process_index: 0, + // window: VecDeque::with_capacity(window_size), + sender: Some(sender), + inner_offset: 12, // 12 bytes header + inner_hash: Sha1::new(), + final_hash: None, + start_encoding: false, + } + } + + pub fn drop_sender(&mut self) { + self.sender.take(); // Take the sender out, dropping it + } + + pub async fn send_data(&mut self, data: Vec) { + if let Some(sender) = &self.sender { + sender.send(data).await.unwrap(); + } + } + + /// Get the hash of the pack file. if the pack file is not finished, return None + pub fn get_hash(&self) -> Option { + self.final_hash + } + + /// Encodes entries into a pack file with delta objects and outputs them through the specified writer. + /// # Arguments + /// - `rx` - A receiver channel (`mpsc::Receiver`) from which entries to be encoded are received. + /// # Returns + /// Returns `Ok(())` if encoding is successful, or a `GitError` in case of failure. + /// - Returns a `GitError` if there is a failure during the encoding process. + /// - Returns `PackEncodeError` if an encoding operation is already in progress. + pub async fn encode(&mut self, entry_rx: mpsc::Receiver) -> Result<(), GitError> { + self.inner_encode(entry_rx, false).await + } + + pub async fn encode_with_zstdelta( + &mut self, + entry_rx: mpsc::Receiver, + ) -> Result<(), GitError> { + self.inner_encode(entry_rx, true).await + } + + /// Delta selection heuristics are based on: + /// https://github.com/git/git/blob/master/Documentation/technical/pack-heuristics.adoc + async fn inner_encode( + &mut self, + mut entry_rx: mpsc::Receiver, + enable_zstdelta: bool, + ) -> Result<(), GitError> { + let head = encode_header(self.object_number); + self.send_data(head.clone()).await; + self.inner_hash.update(&head); + + // ensure only one decode can only invoke once + if self.start_encoding { + return Err(GitError::PackEncodeError( + "encoding operation is already in progress".to_string(), + )); + } + + let mut commits: Vec = Vec::new(); + let mut trees: Vec = Vec::new(); + let mut blobs: Vec = Vec::new(); + let mut tags: Vec = Vec::new(); + while let Some(entry) = entry_rx.recv().await { + match entry.obj_type { + ObjectType::Commit => { + commits.push(entry); + } + ObjectType::Tree => { + trees.push(entry); + } + ObjectType::Blob => { + blobs.push(entry); + } + ObjectType::Tag => { + tags.push(entry); + } + _ => {} + } + } + + commits.sort_by(magic_sort); + trees.sort_by(magic_sort); + blobs.sort_by(magic_sort); + tags.sort_by(magic_sort); + tracing::info!( + "numbers : commits: {:?} trees: {:?} blobs:{:?} tag :{:?}", + commits.len(), + trees.len(), + blobs.len(), + tags.len() + ); + + // parallel encoding vec with different object_type + let (commit_results, tree_results, blob_results, tag_results) = tokio::try_join!( + tokio::task::spawn_blocking(move || { + Self::try_as_offset_delta(commits, 10, enable_zstdelta) + }), + tokio::task::spawn_blocking(move || { + Self::try_as_offset_delta(trees, 10, enable_zstdelta) + }), + tokio::task::spawn_blocking(move || { + Self::try_as_offset_delta(blobs, 10, enable_zstdelta) + }), + tokio::task::spawn_blocking(move || { + Self::try_as_offset_delta(tags, 10, enable_zstdelta) + }), + ) + .map_err(|e| GitError::PackEncodeError(format!("Task join error: {e}")))?; + + let all_encoded_data = [ + commit_results + .map_err(|e| GitError::PackEncodeError(format!("Commit encoding error: {e}")))?, + tree_results + .map_err(|e| GitError::PackEncodeError(format!("Tree encoding error: {e}")))?, + blob_results + .map_err(|e| GitError::PackEncodeError(format!("Blob encoding error: {e}")))?, + tag_results + .map_err(|e| GitError::PackEncodeError(format!("Tag encoding error: {e}")))?, + ] + .concat(); + + // 按顺序发送合并后的结果 + for data in all_encoded_data { + self.write_all_and_update(&data).await; + } + + // Hash signature + let hash_result = self.inner_hash.clone().finalize(); + self.final_hash = Some(SHA1::from_bytes(&hash_result)); + self.send_data(hash_result.to_vec()).await; + + self.drop_sender(); + Ok(()) + } + + /// Try to encode as delta using objects in window + /// delta & zstdelta have been gathered here + /// Refs: https://sapling-scm.com/docs/dev/internals/zstdelta/ + /// the sliding window was moved here + /// # Returns + /// - Return (Vec) if success make delta + /// - Return (None) if didn't delta, + fn try_as_offset_delta( + mut bucket: Vec, + window_size: usize, + enable_zstdelta: bool, + ) -> Result>, GitError> { + let mut current_offset = 0usize; + let mut window: VecDeque<(Entry, usize)> = VecDeque::with_capacity(window_size); + let mut res: Vec> = Vec::new(); + + for entry in bucket.iter_mut() { + //let entry_for_window = entry.clone(); + // 每次循环重置最佳基对象选择 + let mut best_base: Option<&(Entry, usize)> = None; + let mut best_rate: f64 = 0.0; + let tie_epsilon: f64 = 0.15; + + let candidates: Vec<_> = window + .par_iter() + .with_min_len(3) + .filter_map(|try_base| { + if try_base.0.obj_type != entry.obj_type { + return None; + } + + if try_base.0.chain_len >= MAX_CHAIN_LEN { + return None; + } + + if try_base.0.hash == entry.hash { + return None; + } + + let sym_ratio = (try_base.0.data.len().min(entry.data.len()) as f64) + / (try_base.0.data.len().max(entry.data.len()) as f64); + if sym_ratio < 0.5 { + return None; + } + + if !cheap_similar(&try_base.0.data, &entry.data) { + return None; + } + + let rate = if (try_base.0.data.len() + entry.data.len()) / 2 > 64 { + delta::heuristic_encode_rate_parallel(&try_base.0.data, &entry.data) + } else { + delta::encode_rate(&try_base.0.data, &entry.data) + // let try_delta_obj = zstdelta::diff(&try_base.0.data, &entry.data).unwrap(); + // 1.0 - try_delta_obj.len() as f64 / entry.data.len() as f64 + }; + + if rate > MIN_DELTA_RATE { + Some((rate, try_base)) + } else { + None + } + }) + .collect(); + + for (rate, try_base) in candidates { + match best_base { + None => { + best_rate = rate; + //best_base_offset = current_offset - try_base.1; + best_base = Some(try_base); + } + Some(best_base_ref) => { + let is_better = if rate > best_rate + tie_epsilon { + true + } else if (rate - best_rate).abs() <= tie_epsilon { + try_base.0.chain_len > best_base_ref.0.chain_len + } else { + false + }; + + if is_better { + best_rate = rate; + best_base = Some(try_base); + } + } + } + } + + let mut entry_for_window = entry.clone(); + + let offset = best_base.map(|best_base| { + let delta = if enable_zstdelta { + entry.obj_type = ObjectType::OffsetZstdelta; + zstdelta::diff(&best_base.0.data, &entry.data) + .map_err(|e| { + GitError::DeltaObjectError(format!("zstdelta diff failed: {e}")) + }) + .unwrap() + } else { + entry.obj_type = ObjectType::OffsetDelta; + delta::encode(&best_base.0.data, &entry.data) + }; + //entry.obj_type = ObjectType::OffsetDelta; + entry.data = delta; + entry.chain_len = best_base.0.chain_len + 1; + current_offset - best_base.1 + }); + + entry_for_window.chain_len = entry.chain_len; + let obj_data = encode_one_object(entry, offset)?; + window.push_back((entry_for_window, current_offset)); + if window.len() > window_size { + window.pop_front(); + } + current_offset += obj_data.len(); + res.push(obj_data); + } + Ok(res) + } + + /// Parallel encode with rayon, only works when window_size == 0 (no delta) + pub async fn parallel_encode( + &mut self, + mut entry_rx: mpsc::Receiver, + ) -> Result<(), GitError> { + if self.window_size != 0 { + return Err(GitError::PackEncodeError( + "parallel encode only works when window_size == 0".to_string(), + )); + } + + let head = encode_header(self.object_number); + self.send_data(head.clone()).await; + self.inner_hash.update(&head); + + // ensure only one decode can only invoke once + if self.start_encoding { + return Err(GitError::PackEncodeError( + "encoding operation is already in progress".to_string(), + )); + } + + let batch_size = usize::max(1000, entry_rx.max_capacity() / 10); // A temporary value, not optimized + tracing::info!("encode with batch size: {}", batch_size); + loop { + let mut batch_entries = Vec::with_capacity(batch_size); + time_it!("parallel encode: receive batch", { + for _ in 0..batch_size { + match entry_rx.recv().await { + Some(entry) => { + batch_entries.push(entry); + self.process_index += 1; + } + None => break, + } + } + }); + + if batch_entries.is_empty() { + break; + } + + // use `collect` will return result in order, refs: https://github.com/rayon-rs/rayon/issues/551#issuecomment-371657900 + let batch_result: Vec> = time_it!("parallel encode: encode batch", { + batch_entries + .par_iter() + .map(|entry| encode_one_object(entry, None).unwrap()) + .collect() + }); + + time_it!("parallel encode: write batch", { + for obj_data in batch_result { + self.write_all_and_update(&obj_data).await; + } + }); + } + + if self.process_index != self.object_number { + panic!( + "not all objects are encoded, process:{}, total:{}", + self.process_index, self.object_number + ); + } + + // hash signature + let hash_result = self.inner_hash.clone().finalize(); + self.final_hash = Some(SHA1::from_bytes(&hash_result)); + self.send_data(hash_result.to_vec()).await; + self.drop_sender(); + Ok(()) + } + + /// Write data to writer and update hash & offset + async fn write_all_and_update(&mut self, data: &[u8]) { + self.inner_hash.update(data); + self.inner_offset += data.len(); + self.send_data(data.to_vec()).await; + } + + /// async version of encode, result data will be returned by JoinHandle. + /// It will consume PackEncoder, so you can't use it after calling this function. + /// when window_size = 0, it executes parallel_encode which retains stream transmission + /// when window_size = 0,it executes encode which uses magic sort and delta. + /// It seems that all other modules rely on this api + pub async fn encode_async( + mut self, + rx: mpsc::Receiver, + ) -> Result, GitError> { + Ok(tokio::spawn(async move { + if self.window_size == 0 { + self.parallel_encode(rx).await.unwrap() + } else { + self.encode(rx).await.unwrap() + } + })) + } + + pub async fn encode_async_with_zstdelta( + mut self, + rx: mpsc::Receiver, + ) -> Result, GitError> { + Ok(tokio::spawn(async move { + // Do not use parallel encode with zstdelta because it make no sense. + self.encode_with_zstdelta(rx).await.unwrap() + })) + } +} + +#[cfg(test)] +mod tests { + use std::env; + use std::sync::Arc; + use std::time::Instant; + use std::{io::Cursor, path::PathBuf}; + use tokio::sync::Mutex; + + use crate::internal::object::blob::Blob; + use crate::internal::pack::utils::read_offset_encoding; + use crate::internal::pack::{Pack, tests::init_logger}; + use crate::time_it; + + use super::*; + + fn check_format(data: &Vec) { + let mut p = Pack::new( + None, + Some(1024 * 1024 * 1024 * 6), // 6GB + Some(PathBuf::from("/tmp/.cache_temp")), + true, + ); + let mut reader = Cursor::new(data); + tracing::debug!("start check format"); + p.decode(&mut reader, |_, _| {}) + .expect("pack file format error"); + } + + #[tokio::test] + async fn test_pack_encoder() { + async fn encode_once(window_size: usize) -> Vec { + let (tx, mut rx) = mpsc::channel(100); + let (entry_tx, entry_rx) = mpsc::channel::(1); + + // make some different objects, or decode will fail + let str_vec = vec!["hello, word", "hello, world.", "!", "123141251251"]; + let encoder = PackEncoder::new(str_vec.len(), window_size, tx); + encoder.encode_async(entry_rx).await.unwrap(); + + for str in str_vec { + let blob = Blob::from_content(str); + let entry: Entry = blob.into(); + entry_tx.send(entry).await.unwrap(); + } + drop(entry_tx); + // assert!(encoder.get_hash().is_some()); + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + result + } + + // without delta + let pack_without_delta = encode_once(0).await; + let pack_without_delta_size = pack_without_delta.len(); + check_format(&pack_without_delta); + + // with delta + let pack_with_delta = encode_once(4).await; + assert!(pack_with_delta.len() <= pack_without_delta_size); + check_format(&pack_with_delta); + } + + async fn get_entries_for_test() -> Arc>> { + let mut source = PathBuf::from(env::current_dir().unwrap().parent().unwrap()); + source.push("tests/data/packs/pack-f8bbb573cef7d851957caceb491c073ee8e8de41.pack"); + // let file_map = crate::test_utils::setup_lfs_file().await; + // let source = file_map + // .get("git-2d187177923cd618a75da6c6db45bb89d92bd504.pack") + // .unwrap(); + // decode pack file to get entries + let mut p = Pack::new(None, None, Some(PathBuf::from("/tmp/.cache_temp")), true); + + let f = std::fs::File::open(source).unwrap(); + tracing::info!("pack file size: {}", f.metadata().unwrap().len()); + let mut reader = std::io::BufReader::new(f); + let entries = Arc::new(Mutex::new(Vec::new())); + let entries_clone = entries.clone(); + p.decode(&mut reader, move |entry, _| { + let mut entries = entries_clone.blocking_lock(); + entries.push(entry); + }) + .unwrap(); + assert_eq!(p.number, entries.lock().await.len()); + tracing::info!("total entries: {}", p.number); + drop(p); + + entries + } + + #[tokio::test] + async fn test_pack_encoder_parallel_large_file() { + init_logger(); + + let start = Instant::now(); + let entries = get_entries_for_test().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + // encode entries with parallel + let (tx, mut rx) = mpsc::channel(1_000_000); + let (entry_tx, entry_rx) = mpsc::channel::(1_000_000); + + let mut encoder = PackEncoder::new(entries_number, 0, tx); + tokio::spawn(async move { + time_it!("test parallel encode", { + encoder.parallel_encode(entry_rx).await.unwrap(); + }); + }); + + // spawn a task to send entries + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx.send(entry.clone()).await.unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", result.len()); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + // check format + check_format(&result); + } + + #[tokio::test] + async fn test_pack_encoder_large_file() { + init_logger(); + let entries = get_entries_for_test().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let start = Instant::now(); + // encode entries + let (tx, mut rx) = mpsc::channel(100_000); + let (entry_tx, entry_rx) = mpsc::channel::(100_000); + + let mut encoder = PackEncoder::new(entries_number, 0, tx); + tokio::spawn(async move { + time_it!("test encode no parallel", { + encoder.encode(entry_rx).await.unwrap(); + }); + }); + + // spawn a task to send entries + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx.send(entry.clone()).await.unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + // // only receive data + // while (rx.recv().await).is_some() { + // // do nothing + // } + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", pack_size); + tracing::info!("original total size: {}", total_original_size); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + tracing::info!( + "space saved: {} bytes", + total_original_size.saturating_sub(pack_size) + ); + } + + #[tokio::test] + async fn test_pack_encoder_with_zstdelta() { + init_logger(); + let entries = get_entries_for_test().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let start = Instant::now(); + let (tx, mut rx) = mpsc::channel(100_000); + let (entry_tx, entry_rx) = mpsc::channel::(100_000); + + let encoder = PackEncoder::new(entries_number, 10, tx); + encoder.encode_async_with_zstdelta(entry_rx).await.unwrap(); + + // spawn a task to send entries + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx.send(entry.clone()).await.unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", pack_size); + tracing::info!("original total size: {}", total_original_size); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + tracing::info!( + "space saved: {} bytes", + total_original_size.saturating_sub(pack_size) + ); + + // check format + check_format(&result); + } + + #[test] + fn test_encode_offset() { + // let value = 11013; + let value = 16389; + + let data = encode_offset(value); + println!("{data:?}"); + let mut reader = Cursor::new(data); + let (result, _) = read_offset_encoding(&mut reader).unwrap(); + println!("result: {result}"); + assert_eq!(result, value as u64); + } + + #[tokio::test] + async fn test_pack_encoder_large_file_with_delta() { + init_logger(); + let entries = get_entries_for_test().await; + let entries_number = entries.lock().await.len(); + + let total_original_size: usize = entries + .lock() + .await + .iter() + .map(|entry| entry.data.len()) + .sum(); + + let (tx, mut rx) = mpsc::channel(100_000); + let (entry_tx, entry_rx) = mpsc::channel::(100_000); + + let encoder = PackEncoder::new(entries_number, 10, tx); + + let start = Instant::now(); // 开始时间 + encoder.encode_async(entry_rx).await.unwrap(); + + // spawn a task to send entries + tokio::spawn(async move { + let entries = entries.lock().await; + for entry in entries.iter() { + entry_tx.send(entry.clone()).await.unwrap(); + } + drop(entry_tx); + tracing::info!("all entries sent"); + }); + + let mut result = Vec::new(); + while let Some(chunk) = rx.recv().await { + result.extend(chunk); + } + + let pack_size = result.len(); + let compression_rate = if total_original_size > 0 { + 1.0 - (pack_size as f64 / total_original_size as f64) + } else { + 0.0 + }; + + let duration = start.elapsed(); + tracing::info!("test executed in: {:.2?}", duration); + tracing::info!("new pack file size: {}", pack_size); + tracing::info!("original total size: {}", total_original_size); + tracing::info!("compression rate: {:.2}%", compression_rate * 100.0); + tracing::info!( + "space saved: {} bytes", + total_original_size.saturating_sub(pack_size) + ); + + // check format + check_format(&result); + } +} diff --git a/mercury/src/internal/pack/entry.rs b/mercury/src/internal/pack/entry.rs new file mode 100644 index 00000000..f18269e1 --- /dev/null +++ b/mercury/src/internal/pack/entry.rs @@ -0,0 +1,80 @@ +use std::hash::{Hash, Hasher}; + +use serde::{Deserialize, Serialize}; + +use crate::hash::SHA1; +use crate::internal::object::ObjectTrait; +use crate::internal::object::blob::Blob; +use crate::internal::object::commit::Commit; +use crate::internal::object::tag::Tag; +use crate::internal::object::tree::Tree; +use crate::internal::object::types::ObjectType; + +/// +/// Git object data from pack file +/// +#[derive(Eq, Clone, Debug, Serialize, Deserialize)] +pub struct Entry { + pub obj_type: ObjectType, + pub data: Vec, + pub hash: SHA1, + pub chain_len: usize, +} + +impl PartialEq for Entry { + fn eq(&self, other: &Self) -> bool { + // hash is enough to compare, right? + self.obj_type == other.obj_type && self.hash == other.hash + } +} + +impl Hash for Entry { + fn hash(&self, state: &mut H) { + self.obj_type.hash(state); + self.hash.hash(state); + } +} + +impl From for Entry { + fn from(value: Blob) -> Self { + Self { + obj_type: ObjectType::Blob, + data: value.data, + hash: value.id, + chain_len: 0, + } + } +} + +impl From for Entry { + fn from(value: Commit) -> Self { + Self { + obj_type: ObjectType::Commit, + data: value.to_data().unwrap(), + hash: value.id, + chain_len: 0, + } + } +} + +impl From for Entry { + fn from(value: Tree) -> Self { + Self { + obj_type: ObjectType::Tree, + data: value.to_data().unwrap(), + hash: value.id, + chain_len: 0, + } + } +} + +impl From for Entry { + fn from(value: Tag) -> Self { + Self { + obj_type: ObjectType::Tag, + data: value.to_data().unwrap(), + hash: value.id, + chain_len: 0, + } + } +} diff --git a/mercury/src/internal/pack/mod.rs b/mercury/src/internal/pack/mod.rs new file mode 100644 index 00000000..604369cf --- /dev/null +++ b/mercury/src/internal/pack/mod.rs @@ -0,0 +1,58 @@ +//! +//! ## Reference +//! 1. Git Pack-Format [Introduce](https://git-scm.com/docs/pack-format) +//! +pub mod cache; +pub mod cache_object; +pub mod channel_reader; +pub mod decode; +pub mod encode; +pub mod entry; +pub mod utils; +pub mod waitlist; +pub mod wrapper; + +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use threadpool::ThreadPool; + +use crate::hash::SHA1; +use crate::internal::object::ObjectTrait; +use crate::internal::pack::cache::Caches; +use crate::internal::pack::waitlist::Waitlist; + +const DEFAULT_TMP_DIR: &str = "./.cache_temp"; +pub struct Pack { + pub number: usize, + pub signature: SHA1, + pub objects: Vec>, + pub pool: Arc, + pub waitlist: Arc, + pub caches: Arc, + pub mem_limit: Option, + pub cache_objs_mem: Arc, // the memory size of CacheObjects in this Pack + pub clean_tmp: bool, +} + +#[cfg(test)] +mod tests { + use tracing_subscriber::util::SubscriberInitExt; + + pub(crate) fn init_logger() { + let _ = tracing_subscriber::fmt::Subscriber::builder() + .with_target(false) + .without_time() + .with_level(true) + .with_max_level(tracing::Level::DEBUG) + .finish() + .try_init(); // avoid multi-init + + // CAUTION: This two is same + // 1. + // tracing_subscriber::fmt().init(); + // + // 2. + // env::set_var("RUST_LOG", "debug"); // must be set if use `fmt::init()`, or no output + // tracing_subscriber::fmt::init(); + } +} diff --git a/mercury/src/internal/pack/utils.rs b/mercury/src/internal/pack/utils.rs new file mode 100644 index 00000000..d98db98a --- /dev/null +++ b/mercury/src/internal/pack/utils.rs @@ -0,0 +1,515 @@ +use sha1::{Digest, Sha1}; +use std::fs; +use std::io::{self, Read}; +use std::path::Path; + +use crate::hash::SHA1; +use crate::internal::object::types::ObjectType; + +/// Checks if the reader has reached EOF (end of file). +/// +/// It attempts to read a single byte from the reader into a buffer. +/// If `Ok(0)` is returned, it means no byte was read, indicating +/// that the end of the stream has been reached and there is no more +/// data left to read. +/// +/// Any other return value means that data was successfully read, so +/// the reader has not reached the end yet. +/// +/// # Arguments +/// +/// * `reader` - The reader to check for EOF state +/// It must implement the `std::io::Read` trait +/// +/// # Returns +/// +/// true if the reader reached EOF, false otherwise +pub fn is_eof(reader: &mut dyn Read) -> bool { + let mut buf = [0; 1]; + matches!(reader.read(&mut buf), Ok(0)) +} + +/// Reads a byte from the given stream and checks if there are more bytes to continue reading. +/// +/// The return value includes two parts: an unsigned integer formed by the first 7 bits of the byte, +/// and a boolean value indicating whether more bytes need to be read. +/// +/// # Parameters +/// * `stream`: The stream from which the byte is read. +/// +/// # Returns +/// Returns an `io::Result` containing a tuple. The first element is the value of the first 7 bits, +/// and the second element is a boolean indicating whether more bytes need to be read. +/// +pub fn read_byte_and_check_continuation(stream: &mut R) -> io::Result<(u8, bool)> { + // Create a buffer for a single byte + let mut bytes = [0; 1]; + + // Read exactly one byte from the stream into the buffer + stream.read_exact(&mut bytes)?; + + // Extract the byte from the buffer + let byte = bytes[0]; + + // Extract the first 7 bits of the byte + let value = byte & 0b0111_1111; + + // Check if the most significant bit (8th bit) is set, indicating more bytes to follow + let msb = byte >= 128; + + // Return the extracted value and the continuation flag + Ok((value, msb)) +} + +/// Reads bytes from the stream and parses the first byte for type and size. +/// Subsequent bytes are read as size bytes and are processed as variable-length +/// integer in little-endian order. The function returns the type and the computed size. +/// +/// # Parameters +/// * `stream`: The stream from which the bytes are read. +/// * `offset`: The offset of the stream. +/// +/// # Returns +/// Returns an `io::Result` containing a tuple of the type and the computed size. +/// +pub fn read_type_and_varint_size( + stream: &mut R, + offset: &mut usize, +) -> io::Result<(u8, usize)> { + let (first_byte, continuation) = read_byte_and_check_continuation(stream)?; + + // Increment the offset by one byte + *offset += 1; + + // Extract the type (bits 2, 3, 4 of the first byte) + let type_bits = (first_byte & 0b0111_0000) >> 4; + + // Initialize size with the last 4 bits of the first byte + let mut size: u64 = (first_byte & 0b0000_1111) as u64; + let mut shift = 4; // Next byte will shift by 4 bits + + let mut more_bytes = continuation; + while more_bytes { + let (next_byte, continuation) = read_byte_and_check_continuation(stream)?; + // Increment the offset by one byte + *offset += 1; + + size |= (next_byte as u64) << shift; + shift += 7; // Each subsequent byte contributes 7 more bits + more_bytes = continuation; + } + + Ok((type_bits, size as usize)) +} + +/// Reads a variable-length integer (VarInt) encoded in little-endian format from a source implementing the Read trait. +/// +/// The VarInt encoding uses the most significant bit (MSB) of each byte as a continuation bit. +/// The continuation bit being 1 indicates that there are following bytes. +/// The actual integer value is encoded in the remaining 7 bits of each byte. +/// +/// # Parameters +/// * `reader`: A source implementing the Read trait (e.g., file, network stream). +/// +/// # Returns +/// Returns a `Result` containing either: +/// * A tuple of the decoded `u64` value and the number of bytes read (`offset`). +/// * An `io::Error` in case of any reading error or if the VarInt is too long. +/// +pub fn read_varint_le(reader: &mut R) -> io::Result<(u64, usize)> { + // The decoded value + let mut value: u64 = 0; + // Bit shift for the next byte + let mut shift = 0; + // Number of bytes read + let mut offset = 0; + + loop { + // A buffer to read a single byte + let mut buf = [0; 1]; + // Read one byte from the reader + reader.read_exact(&mut buf)?; + + // The byte just read + let byte = buf[0]; + if shift > 63 { + // VarInt too long for u64 + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "VarInt too long", + )); + } + + // Take the lower 7 bits of the byte + let byte_value = (byte & 0x7F) as u64; + // Add the byte value to the result, considering the shift + value |= byte_value << shift; + + // Increment the byte count + offset += 1; + // Check if the MSB is 0 (last byte) + if byte & 0x80 == 0 { + break; + } + + // Increment the shift for the next byte + shift += 7; + } + + Ok((value, offset)) +} + +/// The offset for an OffsetDelta object(big-endian order) +/// # Arguments +/// +/// * `stream`: Input Data Stream to read +/// # Returns +/// * (`delta_offset`(unsigned), `consume`) +pub fn read_offset_encoding(stream: &mut R) -> io::Result<(u64, usize)> { + // Like the object length, the offset for an OffsetDelta object + // is stored in a variable number of bytes, + // with the most significant bit of each byte indicating whether more bytes follow. + // However, the object length encoding allows redundant values, + // e.g. the 7-bit value [n] is the same as the 14- or 21-bit values [n, 0] or [n, 0, 0]. + // Instead, the offset encoding adds 1 to the value of each byte except the least significant one. + // And just for kicks, the bytes are ordered from *most* to *least* significant. + let mut value = 0; + let mut offset = 0; + loop { + let (byte_value, more_bytes) = read_byte_and_check_continuation(stream)?; + offset += 1; + value = (value << 7) | byte_value as u64; + if !more_bytes { + return Ok((value, offset)); + } + + value += 1; //important!: for n >= 2 adding 2^7 + 2^14 + ... + 2^(7*(n-1)) to the result + } +} + +/// Read the next N bytes from the reader +/// +#[inline] +pub fn read_bytes(stream: &mut R) -> io::Result<[u8; N]> { + let mut bytes = [0; N]; + stream.read_exact(&mut bytes)?; + + Ok(bytes) +} + +/// Reads a partial integer from a stream. (little-endian order) +/// +/// # Arguments +/// +/// * `stream` - A mutable reference to a readable stream. +/// * `bytes` - The number of bytes to read from the stream. +/// * `present_bytes` - A mutable reference to a byte indicating which bits are present in the integer value. +/// +/// # Returns +/// +/// This function returns a result of type `io::Result`. If the operation is successful, the integer value +/// read from the stream is returned as `Ok(value)`. Otherwise, an `Err` variant is returned, wrapping an `io::Error` +/// that describes the specific error that occurred. +pub fn read_partial_int( + stream: &mut R, + bytes: u8, + present_bytes: &mut u8, +) -> io::Result { + let mut value: usize = 0; + + // Iterate over the byte indices + for byte_index in 0..bytes { + // Check if the current bit is present + if *present_bytes & 1 != 0 { + // Read a byte from the stream + let [byte] = read_bytes(stream)?; + + // Add the byte value to the integer value + value |= (byte as usize) << (byte_index * 8); + } + + // Shift the present bytes to the right + *present_bytes >>= 1; + } + + Ok(value) +} + +/// Reads the base size and result size of a delta object from the given stream. +/// +/// **Note**: The stream MUST be positioned at the start of the delta object. +/// +/// The base size and result size are encoded as variable-length integers in little-endian order. +/// +/// The base size is the size of the base object, and the result size is the size of the result object. +/// +/// # Parameters +/// * `stream`: The stream from which the sizes are read. +/// +/// # Returns +/// Returns a tuple containing the base size and result size. +/// +pub fn read_delta_object_size(stream: &mut R) -> io::Result<(usize, usize)> { + let base_size = read_varint_le(stream)?.0 as usize; + let result_size = read_varint_le(stream)?.0 as usize; + Ok((base_size, result_size)) +} + +/// Calculate the SHA1 hash of the given object. +///
"` \0`" +///
data: The decompressed content of the object +pub fn calculate_object_hash(obj_type: ObjectType, data: &Vec) -> SHA1 { + let mut hash = Sha1::new(); + // Header: " \0" + hash.update(obj_type.to_bytes()); + hash.update(b" "); + hash.update(data.len().to_string()); + hash.update(b"\0"); + + // Decompressed data(raw content) + hash.update(data); + + let re: [u8; 20] = hash.finalize().into(); + SHA1(re) +} +/// Create an empty directory or clear the existing directory. +pub fn create_empty_dir>(path: P) -> io::Result<()> { + let dir = path.as_ref(); + // 删除整个文件夹 + if dir.exists() { + fs::remove_dir_all(dir)?; + } + // 重新创建文件夹 + fs::create_dir_all(dir)?; + Ok(()) +} + +/// Count the number of files in a directory and its subdirectories. +pub fn count_dir_files(path: &Path) -> io::Result { + let mut count = 0; + for entry in fs::read_dir(path)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + count += count_dir_files(&path)?; + } else { + count += 1; + } + } + Ok(count) +} + +/// Count the time taken to execute a block of code. +#[macro_export] +macro_rules! time_it { + ($msg:expr, $block:block) => {{ + let start = std::time::Instant::now(); + let result = $block; + let elapsed = start.elapsed(); + // println!("{}: {:?}", $msg, elapsed); + tracing::info!("{}: {:?}", $msg, elapsed); + result + }}; +} + +#[cfg(test)] +mod tests { + use crate::internal::object::types::ObjectType; + use std::io; + use std::io::Cursor; + use std::io::Read; + + use crate::internal::pack::utils::*; + + #[test] + fn test_calc_obj_hash() { + let hash = calculate_object_hash(ObjectType::Blob, &b"a".to_vec()); + assert_eq!(hash.to_string(), "2e65efe2a145dda7ee51d1741299f848e5bf752e"); + } + + #[test] + fn eof() { + let mut reader = Cursor::new(&b""[..]); + assert!(is_eof(&mut reader)); + } + + #[test] + fn not_eof() { + let mut reader = Cursor::new(&b"abc"[..]); + assert!(!is_eof(&mut reader)); + } + + #[test] + fn eof_midway() { + let mut reader = Cursor::new(&b"abc"[..]); + reader.read_exact(&mut [0; 2]).unwrap(); + assert!(!is_eof(&mut reader)); + } + + #[test] + fn reader_error() { + struct BrokenReader; + impl Read for BrokenReader { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::other("error")) + } + } + + let mut reader = BrokenReader; + assert!(!is_eof(&mut reader)); + } + + // Test case for a byte without a continuation bit (most significant bit is 0) + #[test] + fn test_read_byte_and_check_continuation_no_continuation() { + let data = [0b0101_0101]; // 85 in binary, highest bit is 0 + let mut cursor = Cursor::new(data); + let (value, more_bytes) = read_byte_and_check_continuation(&mut cursor).unwrap(); + + assert_eq!(value, 85); // Expected value is 85 + assert!(!more_bytes); // No more bytes are expected + } + + // Test case for a byte with a continuation bit (most significant bit is 1) + #[test] + fn test_read_byte_and_check_continuation_with_continuation() { + let data = [0b1010_1010]; // 170 in binary, highest bit is 1 + let mut cursor = Cursor::new(data); + let (value, more_bytes) = read_byte_and_check_continuation(&mut cursor).unwrap(); + + assert_eq!(value, 42); // Expected value is 42 (170 - 128) + assert!(more_bytes); // More bytes are expected + } + + // Test cases for edge values, like the minimum and maximum byte values + #[test] + fn test_read_byte_and_check_continuation_edge_cases() { + // Test the minimum value (0) + let data = [0b0000_0000]; + let mut cursor = Cursor::new(data); + let (value, more_bytes) = read_byte_and_check_continuation(&mut cursor).unwrap(); + + assert_eq!(value, 0); // Expected value is 0 + assert!(!more_bytes); // No more bytes are expected + + // Test the maximum value (255) + let data = [0b1111_1111]; + let mut cursor = Cursor::new(data); + let (value, more_bytes) = read_byte_and_check_continuation(&mut cursor).unwrap(); + + assert_eq!(value, 127); // Expected value is 127 (255 - 128) + assert!(more_bytes); // More bytes are expected + } + + // Test with a single byte where msb is 0 (no continuation) + #[test] + fn test_single_byte_no_continuation() { + let data = [0b0101_0101]; // Type: 5 (101), Size: 5 (0101) + let mut offset: usize = 0; + let mut cursor = Cursor::new(data); + let (type_bits, size) = read_type_and_varint_size(&mut cursor, &mut offset).unwrap(); + + assert_eq!(offset, 1); // Offset is 1 + assert_eq!(type_bits, 5); // Expected type is 2 + assert_eq!(size, 5); // Expected size is 5 + } + + // Test with multiple bytes, where continuation occurs + #[test] + fn test_multiple_bytes_with_continuation() { + // Type: 5 (101), Sizes: 5 (0101), 3 (0000011) in little-endian order + let data = [0b1101_0101, 0b0000_0011]; // Second byte's msb is 0 + let mut offset: usize = 0; + let mut cursor = Cursor::new(data); + let (type_bits, size) = read_type_and_varint_size(&mut cursor, &mut offset).unwrap(); + + assert_eq!(offset, 2); // Offset is 2 + assert_eq!(type_bits, 5); // Expected type is 5 + // Expected size 000000110101 + // 110101 = 1 * 2^5 + 1 * 2^4 + 0 * 2^3 + 1 * 2^2 + 0 * 2^1 + 1 * 2^0= 53 + assert_eq!(size, 53); + } + + // Test with edge case where size is spread across multiple bytes + #[test] + fn test_edge_case_size_spread_across_bytes() { + // Type: 1 (001), Sizes: 15 (1111) in little-endian order + let data = [0b0001_1111, 0b0000_0010]; // Second byte's msb is 1 (continuation) + let mut offset: usize = 0; + let mut cursor = Cursor::new(data); + let (type_bits, size) = read_type_and_varint_size(&mut cursor, &mut offset).unwrap(); + + assert_eq!(offset, 1); // Offset is 1 + assert_eq!(type_bits, 1); // Expected type is 1 + // Expected size is 15 + assert_eq!(size, 15); + } + + #[test] + fn test_read_varint_le_single_byte() { + // Single byte: 0x05 (binary: 0000 0101) + // Represents the value 5 with no continuation bit set. + let data = vec![0x05]; + let mut cursor = Cursor::new(data); + let (value, offset) = read_varint_le(&mut cursor).unwrap(); + + assert_eq!(value, 5); + assert_eq!(offset, 1); + } + + #[test] + fn test_read_varint_le_multiple_bytes() { + // Two bytes: 0x85, 0x01 (binary: 1000 0101, 0000 0001) + // Represents the value 133. First byte has the continuation bit set. + let data = vec![0x85, 0x01]; + let mut cursor = Cursor::new(data); + let (value, offset) = read_varint_le(&mut cursor).unwrap(); + + assert_eq!(value, 133); + assert_eq!(offset, 2); + } + + #[test] + fn test_read_varint_le_large_number() { + // Five bytes: 0xFF, 0xFF, 0xFF, 0xFF, 0xF (binary: 1111 1111, 1111 1111, 1111 1111, 1111 1111, 0000 1111) + // Represents the value 134,217,727. All continuation bits are set except in the last byte. + let data = vec![0xFF, 0xFF, 0xFF, 0xFF, 0xF]; + let mut cursor = Cursor::new(data); + let (value, offset) = read_varint_le(&mut cursor).unwrap(); + + assert_eq!(value, 0xFFFFFFFF); + assert_eq!(offset, 5); + } + + #[test] + fn test_read_varint_le_zero() { + // Single byte: 0x00 (binary: 0000 0000) + // Represents the value 0 with no continuation bit set. + let data = vec![0x00]; + let mut cursor = Cursor::new(data); + let (value, offset) = read_varint_le(&mut cursor).unwrap(); + + assert_eq!(value, 0); + assert_eq!(offset, 1); + } + + #[test] + fn test_read_varint_le_too_long() { + let data = vec![ + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01, + ]; + let mut cursor = Cursor::new(data); + let result = read_varint_le(&mut cursor); + + assert!(result.is_err()); + } + + #[test] + fn test_read_offset_encoding() { + let data: Vec = vec![0b_1101_0101, 0b_0000_0101]; + let mut cursor = Cursor::new(data); + let result = read_offset_encoding(&mut cursor); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), (11013, 2)); + } +} diff --git a/mercury/src/internal/pack/waitlist.rs b/mercury/src/internal/pack/waitlist.rs new file mode 100644 index 00000000..13107b79 --- /dev/null +++ b/mercury/src/internal/pack/waitlist.rs @@ -0,0 +1,39 @@ +use crate::hash::SHA1; +use crate::internal::pack::cache_object::CacheObject; +use dashmap::DashMap; + +/// Waitlist for Delta objects while the Base object is not ready. +/// Easier and faster than Channels. +#[derive(Default, Debug)] +pub struct Waitlist { + //TODO Memory Control! + pub map_offset: DashMap>, + pub map_ref: DashMap>, +} + +impl Waitlist { + pub fn new() -> Self { + Self::default() + } + + pub fn insert_offset(&self, offset: usize, obj: CacheObject) { + self.map_offset.entry(offset).or_default().push(obj); + } + + pub fn insert_ref(&self, hash: SHA1, obj: CacheObject) { + self.map_ref.entry(hash).or_default().push(obj); + } + + /// Take objects out (get & remove) + ///
Return Vec::new() if None + pub fn take(&self, offset: usize, hash: SHA1) -> Vec { + let mut res = Vec::new(); + if let Some((_, vec)) = self.map_offset.remove(&offset) { + res.extend(vec); + } + if let Some((_, vec)) = self.map_ref.remove(&hash) { + res.extend(vec); + } + res + } +} diff --git a/mercury/src/internal/pack/wrapper.rs b/mercury/src/internal/pack/wrapper.rs new file mode 100644 index 00000000..057d77a0 --- /dev/null +++ b/mercury/src/internal/pack/wrapper.rs @@ -0,0 +1,132 @@ +use std::io::{self, BufRead, Read}; + +use sha1::{Digest, Sha1}; + +use crate::hash::SHA1; + +/// [`Wrapper`] is a wrapper around a reader that also computes the SHA1 hash of the data read. +/// +/// It is designed to work with any reader that implements `BufRead`. +/// +/// Fields: +/// * `inner`: The inner reader. +/// * `hash`: The SHA1 hash state. +/// * `count_hash`: A flag to indicate whether to compute the hash while reading. +pub struct Wrapper { + inner: R, + hash: Sha1, + bytes_read: usize, +} + +impl Wrapper +where + R: BufRead, +{ + /// Constructs a new [`Wrapper`] with the given reader and a flag to enable or disable hashing. + /// + /// # Parameters + /// * `inner`: The reader to wrap. + /// * `count_hash`: If `true`, the hash is computed while reading; otherwise, it is not. + pub fn new(inner: R) -> Self { + Self { + inner, + hash: Sha1::new(), // Initialize a new SHA1 hasher + bytes_read: 0, + } + } + + pub fn bytes_read(&self) -> usize { + self.bytes_read + } + + /// Returns the final SHA1 hash of the data read so far. + /// + /// This is a clone of the internal hash state finalized into a SHA1 hash. + pub fn final_hash(&self) -> SHA1 { + let re: [u8; 20] = self.hash.clone().finalize().into(); // Clone, finalize, and convert the hash into bytes + SHA1(re) + } +} + +impl BufRead for Wrapper +where + R: BufRead, +{ + /// Provides access to the internal buffer of the wrapped reader without consuming it. + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.inner.fill_buf() // Delegate to the inner reader + } + + /// Consumes data from the buffer and updates the hash if `count_hash` is true. + /// + /// # Parameters + /// * `amt`: The amount of data to consume from the buffer. + fn consume(&mut self, amt: usize) { + let buffer = self.inner.fill_buf().expect("Failed to fill buffer"); + self.hash.update(&buffer[..amt]); // Update hash with the data being consumed + self.inner.consume(amt); // Consume the data from the inner reader + self.bytes_read += amt; + } +} + +impl Read for Wrapper +where + R: BufRead, +{ + /// Reads data into the provided buffer and updates the hash if `count_hash` is true. + ///
[Read::read_exact] calls it internally. + /// + /// # Parameters + /// * `buf`: The buffer to read data into. + /// + /// # Returns + /// Returns the number of bytes read. + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let o = self.inner.read(buf)?; // Read data into the buffer + self.hash.update(&buf[..o]); // Update hash with the data being read + self.bytes_read += o; + Ok(o) // Return the number of bytes read + } +} + +#[cfg(test)] +mod tests { + use std::io::{self, BufReader, Cursor, Read}; + + use sha1::{Digest, Sha1}; + + use crate::internal::pack::wrapper::Wrapper; + + #[test] + fn test_wrapper_read() -> io::Result<()> { + let data = b"Hello, world!"; // Sample data + let cursor = Cursor::new(data.as_ref()); + let buf_reader = BufReader::new(cursor); + let mut wrapper = Wrapper::new(buf_reader); + + let mut buffer = vec![0; data.len()]; + wrapper.read_exact(&mut buffer)?; + + assert_eq!(buffer, data); + Ok(()) + } + + #[test] + fn test_wrapper_hash() -> io::Result<()> { + let data = b"Hello, world!"; + let cursor = Cursor::new(data.as_ref()); + let buf_reader = BufReader::new(cursor); + let mut wrapper = Wrapper::new(buf_reader); + + let mut buffer = vec![0; data.len()]; + wrapper.read_exact(&mut buffer)?; + + let hash_result = wrapper.final_hash(); + let mut hasher = Sha1::new(); + hasher.update(data); + let expected_hash: [u8; 20] = hasher.finalize().into(); + + assert_eq!(hash_result.0, expected_hash); + Ok(()) + } +} diff --git a/mercury/src/internal/zlib/mod.rs b/mercury/src/internal/zlib/mod.rs new file mode 100644 index 00000000..baf29e06 --- /dev/null +++ b/mercury/src/internal/zlib/mod.rs @@ -0,0 +1 @@ +pub mod stream; diff --git a/mercury/src/internal/zlib/stream/inflate.rs b/mercury/src/internal/zlib/stream/inflate.rs new file mode 100644 index 00000000..3d2878f1 --- /dev/null +++ b/mercury/src/internal/zlib/stream/inflate.rs @@ -0,0 +1,105 @@ +use std::{io, io::BufRead}; + +use flate2::{Decompress, FlushDecompress, Status}; +use sha1::{Digest, Sha1, digest::core_api::CoreWrapper}; + +use crate::internal::object::types::ObjectType; + +/// ReadBoxed is to unzip information from a DEFLATE stream, +/// which hash [`BufRead`] trait. +/// For a continuous stream of DEFLATE information, the structure +/// does not read too many bytes to affect subsequent information +/// reads +pub struct ReadBoxed { + /// The reader from which bytes should be decompressed. + pub inner: R, + /// The decompressor doing all the work. + pub decompressor: Box, + /// the [`count_hash`] decide whether to calculate the hash value in the [`read`] method + count_hash: bool, + pub hash: CoreWrapper, +} +impl ReadBoxed +where + R: BufRead, +{ + /// Nen a ReadBoxed for zlib read, the Output ReadBoxed is for the Common Object, + /// but not for the Delta Object,if that ,see new_for_delta method below. + pub fn new(inner: R, obj_type: ObjectType, size: usize) -> Self { + let mut hash = sha1::Sha1::new(); + hash.update(obj_type.to_bytes()); + hash.update(b" "); + hash.update(size.to_string()); + hash.update(b"\0"); + ReadBoxed { + inner, + hash, + count_hash: true, + decompressor: Box::new(Decompress::new(true)), + } + } + + pub fn new_for_delta(inner: R) -> Self { + ReadBoxed { + inner, + hash: Sha1::new(), + count_hash: false, + decompressor: Box::new(Decompress::new(true)), + } + } +} +impl io::Read for ReadBoxed +where + R: BufRead, +{ + fn read(&mut self, into: &mut [u8]) -> io::Result { + let o = read(&mut self.inner, &mut self.decompressor, into)?; + //update the hash value + if self.count_hash { + self.hash.update(&into[..o]); + } + Ok(o) + } +} + +/// Read bytes from `rd` and decompress them using `state` into a pre-allocated fitting buffer `dst`, returning the amount of bytes written. +fn read(rd: &mut impl BufRead, state: &mut Decompress, mut dst: &mut [u8]) -> io::Result { + let mut total_written = 0; + loop { + let (written, consumed, ret, eof); + { + let input = rd.fill_buf()?; + eof = input.is_empty(); + let before_out = state.total_out(); + let before_in = state.total_in(); + let flush = if eof { + FlushDecompress::Finish + } else { + FlushDecompress::None + }; + ret = state.decompress(input, dst, flush); + written = (state.total_out() - before_out) as usize; + total_written += written; + dst = &mut dst[written..]; + consumed = (state.total_in() - before_in) as usize; + } + rd.consume(consumed); + + match ret { + // The stream has officially ended, nothing more to do here. + Ok(Status::StreamEnd) => return Ok(total_written), + // Either input our output are depleted even though the stream is not depleted yet. + Ok(Status::Ok | Status::BufError) if eof || dst.is_empty() => return Ok(total_written), + // Some progress was made in both the input and the output, it must continue to reach the end. + Ok(Status::Ok | Status::BufError) if consumed != 0 || written != 0 => continue, + // A strange state, where zlib makes no progress but isn't done either. Call it out. + Ok(Status::Ok | Status::BufError) => unreachable!("Definitely a bug somewhere"), + Err(..) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "corrupt deflate stream", + )); + } + } + } +} diff --git a/mercury/src/internal/zlib/stream/mod.rs b/mercury/src/internal/zlib/stream/mod.rs new file mode 100644 index 00000000..ff0155e1 --- /dev/null +++ b/mercury/src/internal/zlib/stream/mod.rs @@ -0,0 +1 @@ +pub mod inflate; diff --git a/mercury/src/lib.rs b/mercury/src/lib.rs new file mode 100644 index 00000000..ebeb8743 --- /dev/null +++ b/mercury/src/lib.rs @@ -0,0 +1,93 @@ +//! Mercury is a library for encoding and decoding Git Pack format files or streams. + +pub mod errors; +pub mod hash; +pub mod internal; +pub mod utils; + +// #[cfg(test)] +pub mod test_utils { + use reqwest::Client; + use ring::digest::{Context, SHA256}; + use std::collections::HashMap; + use std::env; + use std::fs::File; + use std::io::copy; + use std::path::PathBuf; + use tokio::io::{AsyncReadExt, AsyncSeekExt}; + use tokio::sync::OnceCell; + + static FILES_READY: OnceCell> = OnceCell::const_new(); + + pub async fn setup_lfs_file() -> HashMap { + FILES_READY + .get_or_init(|| async { + let files_to_download = vec![( + "git-2d187177923cd618a75da6c6db45bb89d92bd504.pack", + "0d1f01ac02481427e83ba6c110c7a3a75edd4264c5af8014d12d6800699c8409", + )]; + + let mut map = HashMap::new(); + let mut base_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + base_dir.pop(); + base_dir.push("tests/data/packs"); + for (name, sha256) in files_to_download { + let file_path = base_dir.join(name); + download_lfs_file_if_not_exist(name, &file_path, sha256).await; + + map.insert(name.to_string(), file_path); + } + + map + }) + .await + .clone() + } + + async fn calculate_checksum(file: &mut tokio::fs::File, checksum: &mut Context) { + file.seek(tokio::io::SeekFrom::Start(0)).await.unwrap(); + let mut buf = [0u8; 8192]; + loop { + let n = file.read(&mut buf).await.unwrap(); + if n == 0 { + break; + } + checksum.update(&buf[..n]); + } + } + + async fn download_file( + url: &str, + output_path: &PathBuf, + ) -> Result<(), Box> { + let response = Client::new().get(url).send().await?; + let mut file = File::create(output_path)?; + let content = response.bytes().await?; + let mut content = content.as_ref(); + copy(&mut content, &mut file)?; + Ok(()) + } + + async fn check_file_hash(output_path: &PathBuf, sha256: &str) -> bool { + if output_path.exists() { + let mut ring_context = Context::new(&SHA256); + let mut file = tokio::fs::File::open(output_path).await.unwrap(); + calculate_checksum(&mut file, &mut ring_context).await; + let checksum = hex::encode(ring_context.finish().as_ref()); + checksum == sha256 + } else { + false + } + } + + async fn download_lfs_file_if_not_exist(file_name: &str, output_path: &PathBuf, sha256: &str) { + let url = format!( + "https://file.gitmega.com/packs/{file_name}", + // "https://gitmono.s3.ap-southeast-2.amazonaws.com/packs/{file_name}", + // "https://gitmono.oss-cn-beijing.aliyuncs.com/{}", + ); + if !check_file_hash(output_path, sha256).await { + download_file(&url, output_path).await.unwrap(); + } + } +} diff --git a/mercury/src/utils.rs b/mercury/src/utils.rs new file mode 100644 index 00000000..2ed62f41 --- /dev/null +++ b/mercury/src/utils.rs @@ -0,0 +1,49 @@ +use crate::hash::SHA1; +use std::io; +use std::io::{BufRead, Read}; + +pub fn read_bytes(file: &mut impl Read, len: usize) -> io::Result> { + let mut buf = vec![0; len]; + file.read_exact(&mut buf)?; + Ok(buf) +} + +pub fn read_sha1(file: &mut impl Read) -> io::Result { + SHA1::from_stream(file) +} + +/// A lightweight wrapper that counts bytes read from the underlying reader. +/// replace deflate.intotal() in decompress_data +pub struct CountingReader { + pub inner: R, + pub bytes_read: u64, +} + +impl CountingReader { + /// Creates a new `CountingReader` wrapping the given reader. + pub fn new(inner: R) -> Self { + Self { + inner, + bytes_read: 0, + } + } +} + +impl Read for CountingReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let n = self.inner.read(buf)?; + self.bytes_read += n as u64; + Ok(n) + } +} + +impl BufRead for CountingReader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.inner.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.bytes_read += amt as u64; + self.inner.consume(amt); + } +} diff --git a/mercury/zstdelta/Cargo.toml b/mercury/zstdelta/Cargo.toml new file mode 100644 index 00000000..7d268b0d --- /dev/null +++ b/mercury/zstdelta/Cargo.toml @@ -0,0 +1,27 @@ +# @generated by autocargo from //eden/scm/lib/zstdelta:zstdelta + +[package] +name = "sapling-zstdelta" +version = "0.1.0" +authors = ["Meta Source Control Team "] +edition = "2024" +homepage = "https://sapling-scm.com/" +repository = "https://github.com/facebook/sapling" +license = "MIT" + +[lib] +name = "zstdelta" + +[[bin]] +name = "zstdelta" +path = "src/main.rs" +doc = false + +[dependencies] +libc = { workspace = true } +zstd-sys = { workspace = true, features = ["experimental"] } + +[dev-dependencies] +quickcheck = { workspace = true } +rand = { workspace = true, features = ["small_rng"] } +rand_chacha = { workspace = true } diff --git a/mercury/zstdelta/LICENSE b/mercury/zstdelta/LICENSE new file mode 100644 index 00000000..b93be905 --- /dev/null +++ b/mercury/zstdelta/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Meta Platforms, Inc. and affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/mercury/zstdelta/README.md b/mercury/zstdelta/README.md new file mode 100644 index 00000000..b75231ae --- /dev/null +++ b/mercury/zstdelta/README.md @@ -0,0 +1,3 @@ +## Zstdelta + +It is a copy of the `zstdelta` crate from the [facebook/sapling](https://github.com/facebook/sapling) repository, which is licensed under MIT License. \ No newline at end of file diff --git a/mercury/zstdelta/src/lib.rs b/mercury/zstdelta/src/lib.rs new file mode 100644 index 00000000..f9c4ec81 --- /dev/null +++ b/mercury/zstdelta/src/lib.rs @@ -0,0 +1,11 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +mod zstdelta; + +pub use crate::zstdelta::apply; +pub use crate::zstdelta::diff; diff --git a/mercury/zstdelta/src/main.rs b/mercury/zstdelta/src/main.rs new file mode 100644 index 00000000..a0860d00 --- /dev/null +++ b/mercury/zstdelta/src/main.rs @@ -0,0 +1,46 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +mod zstdelta; + +use std::env::args; +use std::fs::File; +use std::io; +use std::io::Read; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; +use std::process::exit; + +use crate::zstdelta::apply; +use crate::zstdelta::diff; + +fn read(path: &Path) -> Vec { + let mut buf = Vec::new(); + File::open(path) + .expect("open") + .read_to_end(&mut buf) + .expect("read"); + buf +} + +fn main() { + let args: Vec<_> = args().skip(1).collect(); + if args.len() < 3 { + println!("Usage: zstdelta -c base data > delta\n zstdelta -d base delta > data\n"); + exit(1); + } + let base = read(&PathBuf::from(&args[1])); + let data = read(&PathBuf::from(&args[2])); + let out = if args[0] == "-c" { + diff(&base, &data).expect("diff") + } else { + apply(&base, &data).expect("apply") + }; + + io::stdout().write_all(&out).expect("write"); +} diff --git a/mercury/zstdelta/src/zstdelta.rs b/mercury/zstdelta/src/zstdelta.rs new file mode 100644 index 00000000..b16f1436 --- /dev/null +++ b/mercury/zstdelta/src/zstdelta.rs @@ -0,0 +1,204 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use std::cmp; +use std::ffi::CStr; +use std::io; + +use libc::c_void; +use zstd_sys::ZSTD_CHAINLOG_MIN; +use zstd_sys::ZSTD_CONTENTSIZE_ERROR; +use zstd_sys::ZSTD_CONTENTSIZE_UNKNOWN; +use zstd_sys::ZSTD_DCtx_setMaxWindowSize; +use zstd_sys::ZSTD_HASHLOG_MIN; +use zstd_sys::ZSTD_SEARCHLOG_MIN; +use zstd_sys::ZSTD_WINDOWLOG_MIN; +use zstd_sys::ZSTD_compress_advanced; +use zstd_sys::ZSTD_compressBound; +use zstd_sys::ZSTD_compressionParameters; +use zstd_sys::ZSTD_createCCtx; +use zstd_sys::ZSTD_createDCtx; +use zstd_sys::ZSTD_decompress_usingDict; +use zstd_sys::ZSTD_findDecompressedSize; +use zstd_sys::ZSTD_frameParameters; +use zstd_sys::ZSTD_freeCCtx; +use zstd_sys::ZSTD_freeDCtx; +use zstd_sys::ZSTD_getErrorName; +use zstd_sys::ZSTD_isError; +use zstd_sys::ZSTD_parameters; +use zstd_sys::ZSTD_strategy; + +// They are complex "#define"s that are not exposed by bindgen automatically +const ZSTD_WINDOWLOG_MAX: u32 = 30; +const ZSTD_HASHLOG_MAX: u32 = 30; + +/// Return `y` so `1 << y` is greater than `x`. +/// Note: `1 << y` might be greater than `u64::MAX`. +fn log_base2(x: u64) -> u32 { + 64 - x.leading_zeros() +} + +/// Adjust a value so it is in the given range. +fn clamp(v: u32, min: u32, max: u32) -> u32 { + cmp::max(min, cmp::min(v, max)) +} + +/// Convert zstd error code to a static string. +fn explain_error(code: usize) -> &'static str { + unsafe { + // ZSTD_getErrorName returns a static string. + let name = ZSTD_getErrorName(code); + let cstr = CStr::from_ptr(name); + cstr.to_str().expect("zstd error is utf-8") + } +} + +/// Create a "zstd delta". Compress `data` using dictionary `base`. +pub fn diff(base: &[u8], data: &[u8]) -> io::Result> { + // Customized wlog, hlog to let zstd do better at delta-ing. Use "fast" strategy, which is + // good enough assuming the primary space saving is caused by "delta-ing". + let log = log_base2((data.len() + base.len() + 1) as u64); + let wlog = clamp(log, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + let hlog = clamp(log, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + let cparams = ZSTD_compressionParameters { + windowLog: wlog, + chainLog: ZSTD_CHAINLOG_MIN, // useless using "fast" strategy + hashLog: hlog, + searchLog: ZSTD_SEARCHLOG_MIN, // useless using "fast" strategy + minMatch: 7, // level 1 default (see ZSTD_defaultCParameters) + targetLength: 0, // enable huffman compression of literals (for "fast" strategy) + strategy: ZSTD_strategy::ZSTD_fast, + }; + let fparams = ZSTD_frameParameters { + contentSizeFlag: 1, // needed by `apply` + checksumFlag: 0, // checksum is done at another level + noDictIDFlag: 1, // dictionary is fixed, not reused + }; + let params = ZSTD_parameters { + cParams: cparams, + fParams: fparams, + }; + + unsafe { + let cctx = ZSTD_createCCtx(); + if cctx.is_null() { + return Err(io::Error::other("cannot create CCtx")); + } + + let max_outsize = ZSTD_compressBound(data.len()); + let mut buf: Vec = vec![0; max_outsize]; + + let outsize = ZSTD_compress_advanced( + cctx, + buf.as_mut_ptr() as *mut c_void, + buf.len(), + data.as_ptr() as *const c_void, + data.len(), + base.as_ptr() as *const c_void, + base.len(), + params, + ); + + ZSTD_freeCCtx(cctx); + + if ZSTD_isError(outsize) != 0 { + let msg = format!("cannot compress ({})", explain_error(outsize)); + Err(io::Error::other(msg)) + } else { + buf.set_len(outsize); + Ok(buf) + } + } +} + +/// Apply a zstd `delta` generated by `diff` to `base`. Return reconstructed `data`. +pub fn apply(base: &[u8], delta: &[u8]) -> io::Result> { + unsafe { + let dctx = ZSTD_createDCtx(); + if dctx.is_null() { + return Err(io::Error::other("cannot create DCtx")); + } + ZSTD_DCtx_setMaxWindowSize(dctx, 1 << ZSTD_WINDOWLOG_MAX); + + let size = ZSTD_findDecompressedSize(delta.as_ptr() as *const c_void, delta.len()) as usize; + if size == ZSTD_CONTENTSIZE_ERROR as usize || size == ZSTD_CONTENTSIZE_UNKNOWN as usize { + ZSTD_freeDCtx(dctx); + let msg = "cannot get decompress size"; + return Err(io::Error::other(msg)); + } + + let mut buf = vec![0u8; size]; + + let outsize = ZSTD_decompress_usingDict( + dctx, + buf.as_mut_ptr() as *mut c_void, + size, + delta.as_ptr() as *const c_void, + delta.len(), + base.as_ptr() as *const c_void, + base.len(), + ); + ZSTD_freeDCtx(dctx); + + if ZSTD_isError(outsize) != 0 { + let msg = format!("cannot decompress ({})", explain_error(outsize)); + Err(io::Error::other(msg)) + } else if outsize != size { + let msg = format!("decompress size mismatch (expected {size}, got {outsize})"); + Err(io::Error::other(msg)) + } else { + Ok(buf) + } + } +} + +#[cfg(test)] +mod tests { + use quickcheck::quickcheck; + use rand::RngCore; + use rand::SeedableRng; + use rand_chacha::ChaChaRng; + + use super::*; + + fn check_round_trip(base: &[u8], data: &[u8]) -> bool { + let delta = diff(base, data).expect("delta"); + let reconstructed = apply(base, &delta).expect("apply"); + reconstructed[..] == data[..] + } + + #[test] + fn test_round_trip_manual() { + assert!(check_round_trip(b"", b"")); + assert!(check_round_trip(b"123", b"")); + assert!(check_round_trip(b"", b"123")); + assert!(check_round_trip(b"1234567890", b"3")); + assert!(check_round_trip(b"3", b"1234567890")); + } + + #[test] + fn test_delta_efficiency() { + // 1 MB incompressible random data + let mut base = vec![0u8; 1000000]; + ChaChaRng::from_seed([0; 32]).fill_bytes(base.as_mut()); + // Change a few bytes + let mut data = base.clone(); + data[0] ^= 1; + data[10000] ^= 3; + data[900000] ^= 7; + let delta = diff(&base, &data).expect("diff"); + // Should generate a small delta. + // Note: this will fail if wlog/hlog are not tweaked. + assert!(delta.len() < 200); + } + + quickcheck! { + fn test_round_trip_quickcheck(a: Vec, b: Vec) -> bool { + check_round_trip(&a, &b) + } + } +}