diff --git a/docs/development.md b/docs/development.md index e97a252f1..317708ad8 100644 --- a/docs/development.md +++ b/docs/development.md @@ -66,83 +66,7 @@ $ psql mega -c "GRANT ALL ON ALL FUNCTIONS IN SCHEMA public to mega;" ``` -4. Update config file for local test. For local testing, Mega uses the `config.toml` file to configure the required parameters. - - ```ini - # Fillin the following environment variables with values you set - ## Logging Configuration - [log] - # The path which log file is saved - log_path = "/tmp/.mega/logs" - - # log level - level = "debug" - - # print std log in console, disable it on production for performance - print_std = true - - - [database] - # "sqlite" | "postgres" - # "sqlite" will use `db_path` and ignore `db_url` - db_type = "sqlite" - - # used for sqlite - db_path = "/tmp/.mega/mega.db" - - # database connection url - db_url = "postgres://mega:mega@localhost:5432/mega" - - # db max connection, setting it to twice the number of CPU cores would be appropriate. - max_connection = 32 - - # db min connection, setting it to the number of CPU cores would be appropriate. - min_connection = 16 - - # Whether to disabling SQLx Log - sqlx_logging = false - - - [ssh] - ssh_key_path = "/tmp/.mega/ssh" - - [storage] - # raw object stroage type, can be `local` or `remote` - raw_obj_storage_type = "LOCAL" - - ## If the object file size exceeds the threshold value, it will be handled by file storage instead of the database, Unit is KB - big_obj_threshold = 1024 - - # set the local path of the project storage - raw_obj_local_path = "/tmp/.mega/objects" - - lfs_obj_local_path = "/tmp/.mega/lfs" - - obs_access_key = "" - obs_secret_key = "" - - # cloud storage region - obs_region = "cn-east-3" - - # Override the endpoint URL used for remote storage services - obs_endpoint = "https://obs.cn-east-3.myhuaweicloud.com" - - - [monorepo] - ## Only import directory support multi-branch commit and tag, repo under regular directory only support main branch only - ## Mega treats files in that directory as import repo and other directories as monorepo - import_dir = "/third-part" - - - # The maximum memory used by decode, Unit is GB - pack_decode_mem_size = 4 - - # The location where the object stored when the memory used by decode exceeds the limit - pack_decode_cache_path = "/tmp/.mega/cache" - - clean_cache_after_decode = true - - ``` +4. Update config file for local test. For local testing, Mega uses the `config.toml` file to configure the required parameters. See [Configuration](#configuration). 5. Init the Mega @@ -236,92 +160,7 @@ $ sudo -u postgres psql mega -c "GRANT ALL ON ALL FUNCTIONS IN SCHEMA public to mega;" ``` -4. Config `confg.toml`. - - ```ini - # Fillin the following environment variables with values you set - - ## Logging Configuration - [log] - # The path which log file is saved - log_path = "/tmp/.mega/logs" - - # log level - level = "debug" - - # print std log in console, disable it on production for performance - print_std = true - - - [database] - # "sqlite" | "postgres" - # "sqlite" will use `db_path` and ignore `db_url` - db_type = "sqlite" - - # used for sqlite - db_path = "/tmp/.mega/mega.db" - - # database connection url - db_url = "postgres://mega:mega@localhost:5432/mega" - - # db max connection, setting it to twice the number of CPU cores would be appropriate. - max_connection = 32 - - # db min connection, setting it to the number of CPU cores would be appropriate. - min_connection = 16 - - # Whether to disabling SQLx Log - sqlx_logging = false - - - [ssh] - ssh_key_path = "/tmp/.mega/ssh" - - [storage] - # raw object stroage type, can be `local` or `remote` - raw_obj_storage_type = "LOCAL" - - ## If the object file size exceeds the threshold value, it will be handled by file storage instead of the database, Unit is KB - big_obj_threshold = 1024 - - # set the local path of the project storage - raw_obj_local_path = "/tmp/.mega/objects" - - lfs_obj_local_path = "/tmp/.mega/lfs" - - obs_access_key = "" - obs_secret_key = "" - - # cloud storage region - obs_region = "cn-east-3" - - # Override the endpoint URL used for remote storage services - obs_endpoint = "https://obs.cn-east-3.myhuaweicloud.com" - - - [monorepo] - ## Only import directory support multi-branch commit and tag, repo under regular directory only support main branch only - ## Mega treats files in that directory as import repo and other directories as monorepo - import_dir = "/third-part" - - - # The maximum memory used by decode, Unit is GB - pack_decode_mem_size = 4 - - # The location where the object stored when the memory used by decode exceeds the limit - pack_decode_cache_path = "/tmp/.mega/cache" - - clean_cache_after_decode = true - - [lfs] - ## IMPORTANT: The 'enable_split' feature can only be enabled for new databases. Existing databases do not support this feature. - # Enable or disable splitting large files into smaller chunks - - enable_split = false # Default is disabled. Set to true to enable file splitting. - # Size of each file chunk when splitting is enabled, in bytes. Ignored if splitting is disabled. - split_size = 20971520 # Default size is 20MB (20971520 bytes) - - ``` +4. Config `confg.toml`. See [Configuration](#configuration). 5. Init Mega. @@ -376,87 +215,123 @@ sudo -u postgres psql mega -c "GRANT ALL ON ALL FUNCTIONS IN SCHEMA public to me Config `confg.toml` file for the Mega project. -```ini - # Fillin the following environment variables with values you set +--- +## Configuration +Config `confg.toml` file for the Mega project. + +### Path +- Default: automatically load `config.toml` in current directory. +- Specify manually: use `--config "/path/to/config.toml"` + +### Enhance +- You can use environment variables starting with `MEGA_` to override the configuration in `config.toml`. + - like `MEGA_BASE_DIR` to override `base_dir`. // with `env::set_var()` + - but it seems only not available for nested keys, like `log.log_path`. +- Support `${}` syntax to reference other keys in the same file. + - like `log_path = "${base_dir}/logs"`, `${base_dir}` will be replaced by the value of `base_dir` + - or `key = "${xxx.yyy}/zzz"` (prefix `xxx.` can't be omitted) + - only support `String` type + - substitute from up to down + - see codes in [config.rs](/common/src/config.rs) + +```toml +# The directory where the data files is located, such as logs, database, etc. +# can be overrided by environment variable `MEGA_BASE_DIR` +base_dir = "/tmp/.mega" + +# Filling the following environment variables with values you set +## Logging Configuration +[log] +# The path which log file is saved +log_path = "${base_dir}/logs" - ## Logging Configuration - [log] - # The path which log file is saved - log_path = "/tmp/.mega/logs" +# log level +level = "debug" - # log level - level = "debug" +# print std log in console, disable it on production for performance +print_std = true - # print std log in console, disable it on production for performance - print_std = true +[database] +# "sqlite" | "postgres" +# "sqlite" will use `db_path` and ignore `db_url` +db_type = "sqlite" - [database] - # "sqlite" | "postgres" - # "sqlite" will use `db_path` and ignore `db_url` - db_type = "sqlite" +# used for sqlite +db_path = "${base_dir}/mega.db" - # used for sqlite - db_path = "/tmp/.mega/mega.db" +# database connection url +db_url = "postgres://mega:mega@localhost:5432/mega" - # database connection url - db_url = "postgres://mega:mega@localhost:5432/mega" +# db max connection, setting it to twice the number of CPU cores would be appropriate. +max_connection = 32 - # db max connection, setting it to twice the number of CPU cores would be appropriate. - max_connection = 2 +# db min connection, setting it to the number of CPU cores would be appropriate. +min_connection = 16 - # db min connection, setting it to the number of CPU cores would be appropriate. - min_connection = 4 +# Whether to disabling SQLx Log +sqlx_logging = false - # Whether to disabling SQLx Log - sqlx_logging = false - [ssh] - ssh_key_path = "/tmp/.mega/ssh" +[ssh] +ssh_key_path = "${base_dir}/ssh" - [storage] - # raw object stroage type, can be `local` or `remote` - raw_obj_storage_type = "LOCAL" +[storage] +# raw object stroage type, can be `local` or `remote` +raw_obj_storage_type = "LOCAL" - ## If the object file size exceeds the threshold value, it will be handled by file storage instead of the database, Unit is KB - big_obj_threshold = 1024 +## If the object file size exceeds the threshold value, it will be handled by file storage instead of the database, Unit is KB +big_obj_threshold = 1024 - # set the local path of the project storage - raw_obj_local_path = "/tmp/.mega/objects" +# set the local path of the project storage +raw_obj_local_path = "${base_dir}/objects" - lfs_obj_local_path = "/tmp/.mega/lfs" +lfs_obj_local_path = "${base_dir}/lfs" - obs_access_key = "" - obs_secret_key = "" +obs_access_key = "" +obs_secret_key = "" - # cloud storage region - obs_region = "cn-east-3" +# cloud storage region +obs_region = "cn-east-3" - # Override the endpoint URL used for remote storage services - obs_endpoint = "https://obs.cn-east-3.myhuaweicloud.com" +# Override the endpoint URL used for remote storage services +obs_endpoint = "https://obs.cn-east-3.myhuaweicloud.com" - [monorepo] - ## Only import directory support multi-branch commit and tag, repo under regular directory only support main branch only - ## Mega treats files in that directory as import repo and other directories as monorepo - import_dir = "/third-part" +[monorepo] +## Only import directory support multi-branch commit and tag, monorepo only support main branch +## Mega treats files under this directory as import repo and other directories as monorepo +import_dir = "/third-part" - # The maximum memory used by decode, Unit is GB - pack_decode_mem_size = 4 - # The location where the object stored when the memory used by decode exceeds the limit - pack_decode_cache_path = "/tmp/.mega/cache" +[pack] +# The maximum memory used by decode, Unit is GB +pack_decode_mem_size = 4 - clean_cache_after_decode = true +# The location where the object stored when the memory used by decode exceeds the limit +pack_decode_cache_path = "${base_dir}/cache" - [lfs] - ## IMPORTANT: The 'enable_split' feature can only be enabled for new databases. Existing databases do not support this feature. - # Enable or disable splitting large files into smaller chunks +clean_cache_after_decode = true - enable_split = false # Default is disabled. Set to true to enable file splitting. - # Size of each file chunk when splitting is enabled, in bytes. Ignored if splitting is disabled. - split_size = 20971520 # Default size is 20MB (20971520 bytes) +# The maximum meesage size in channel buffer while decode +channel_message_size = 1_000_000 + + +[ztm] +ca = "http://127.0.0.1:9999" +hub = "http://127.0.0.1:8888" +agent = "http://127.0.0.1:7777" + +[lfs] +## IMPORTANT: The 'enable_split' feature can only be enabled for new databases. Existing databases do not support this feature. +# Enable or disable splitting large files into smaller chunks +enable_split = false # Default is disabled. Set to true to enable file splitting. + +# Size of each file chunk when splitting is enabled, in bytes. Ignored if splitting is disabled. +split_size = 20971520 # Default size is 20MB (20971520 bytes) ``` + +--- ## Database maintenance Currently, the tables of database are created by `.sql` file. @@ -465,7 +340,64 @@ If you want to add a new table or modify the existing table, you need to update ### Attention - Each database corresponds to one `.sql` file, you must modify all of them if you want to update the tables in order to keep the consistency of the database. - DO NOT use `Array` Type in PostgreSQL but use `JSON` instead, for compatibility with SQLite & MySQL. (`JSON` <==> `serde_json::Value`) +--- +## Tests +> Keep in mind that it's impossible to find all bugs. +> +> Tests are the last line of defense. + +### Unit Tests + +Unit tests are small, focused tests that verify the behavior of a single function or module in isolation. + +#### Example: + +```rust +// ...Other Codes + +#[cfg(test)] // indicates this block will only be compiled when running tests +mod tests { + use super::*; + + #[test] // indicates that this function is a test, which will be run by `cargo test` + fn test_add() { + let result = add(1, 1); + assert_eq!(result, 2); // assert is important to tests + } +} +``` + +### Integration Tests +Integration tests verify that different parts of your **library** work correctly together. +They are **external** to your crate and use your code in the same way any other code would. + +#### Steps +You can refer to the implementation of the mega **module**. ([mega/tests](/mega/tests)) +1. Create a `tests` directory at the **same level** as your `src` directory (e.g. `libra/tests`). +2. Add `*.rs` files in this directory. // Each file will be compiled as a separate **crate**. + +#### Attention +- The `tests` in **root** directory (workspace) is NOT integration tests, but some `data` for other tests. +- If you need a common module, use `tests/common/mod.rs` rather than `tests/common.rs`, to declare it's not a test file. +- There is no need to add `#[cfg(test)]` to the `tests` directory. `tests` will be compiled only when running tests. + +#### Run integration tests +The following command will be executed in `GitHub Actions`. + +This command DOES NOT run **Unit Tests** (which could be very messy). +```bash +cargo test --workspace --test '*' -- --nocapture +``` +- `--workspace` : Run tests for **all packages** in the workspace. +- `--test` : Test the specified **integration test**. +- `--` : Pass the following arguments to the test binary. +- `--nocapture` : DO NOT capture the output (e.g. `println!`) of the test. + +If you want to run tests in a specific package, you can use `--package`. + +For more information, please refer to the [rust wiki](https://rustwiki.org/zh-CN/cargo/commands/cargo-test.html). +--- ## Comment Guideline This guide outlines the recommended order for importing dependencies in Rust projects. @@ -476,6 +408,7 @@ This guide outlines the recommended order for importing dependencies in Rust pro ### Function Comments (///) +--- ## Rust Dependency Import Order Guideline This guide outlines the recommended order for importing dependencies in Rust projects. diff --git a/mega/tests/common/mod.rs b/mega/tests/common/mod.rs new file mode 100644 index 000000000..11bce2820 --- /dev/null +++ b/mega/tests/common/mod.rs @@ -0,0 +1 @@ +// use `common/mod.rs` rather than `common.rs`, to declare it's not a test file \ No newline at end of file diff --git a/mega/tests/lfs_test.rs b/mega/tests/lfs_test.rs index 0f601f34e..b36648743 100644 --- a/mega/tests/lfs_test.rs +++ b/mega/tests/lfs_test.rs @@ -1,8 +1,12 @@ +mod common; + /// integration tests for the mega module use std::process::Command; use std::{env, fs, io, thread}; use std::io::Write; +use std::net::TcpStream; use std::path::Path; +use std::time::Duration; use rand::Rng; use tempfile::TempDir; @@ -27,12 +31,8 @@ fn run_git_cmd(args: &[&str]) { } fn is_port_in_use(port: u16) -> bool { - let output = Command::new("lsof") - .arg(format!("-i:{}", port)) - .output() - .expect("Failed to execute command"); - - !output.stdout.is_empty() + TcpStream::connect_timeout(&format!("127.0.0.1:{}", port).parse().unwrap(), Duration::from_millis(1000)) + .is_ok() } fn run_mega_server() { @@ -40,10 +40,10 @@ fn run_mega_server() { let args = vec!["service", "multi", "http"]; mega::cli::parse(Some(args)).expect("Failed to start mega service"); }); - // loop check until 8000 port to be ready + // loop check until port to be ready let mut i = 0; - while !is_port_in_use(PORT) && (i < 10) { - thread::sleep(std::time::Duration::from_secs(1)); + while !is_port_in_use(PORT) && (i < 15) { + thread::sleep(Duration::from_secs(1)); i += 1; } assert!(is_port_in_use(PORT), "mega server not started"); diff --git a/mercury/src/internal/object/commit.rs b/mercury/src/internal/object/commit.rs index 6f7ce45f4..d077a4621 100644 --- a/mercury/src/internal/object/commit.rs +++ b/mercury/src/internal/object/commit.rs @@ -25,11 +25,11 @@ use crate::internal::object::ObjectType; /// The `Commit` struct is used to represent a commit object. /// /// - The tree object SHA points to the top level tree for this commit, which reflects the complete -/// state of the repository at the time of the commit. The tree object in turn points to blobs and -/// subtrees which represent the files in the repository. +/// state of the repository at the time of the commit. The tree object in turn points to blobs and +/// subtrees which represent the files in the repository. /// - The parent commit SHAs allow Git to construct a linked list of commits and build the full -/// commit history. By chaining together commits in this fashion, Git is able to represent the entire -/// history of a repository with a single commit object at its root. +/// commit history. By chaining together commits in this fashion, Git is able to represent the entire +/// history of a repository with a single commit object at its root. /// - The author and committer fields contain the name, email address, timestamp and timezone. /// - The message field contains the commit message, which maybe include signed or DCO. #[allow(unused)] diff --git a/mercury/src/internal/object/signature.rs b/mercury/src/internal/object/signature.rs index 892d966c8..9c223a129 100644 --- a/mercury/src/internal/object/signature.rs +++ b/mercury/src/internal/object/signature.rs @@ -5,9 +5,9 @@ //! - Name: The name of the author, encoded as a UTF-8 string. //! - Email: The email address of the author, encoded as a UTF-8 string. //! - Timestamp: The timestamp of when the commit was authored, encoded as a decimal number of seconds -//! since the Unix epoch (January 1, 1970, 00:00:00 UTC). +//! since the Unix epoch (January 1, 1970, 00:00:00 UTC). //! - Timezone: The timezone offset of the author's local time from Coordinated Universal Time (UTC), -//! encoded as a string in the format "+HHMM" or "-HHMM". +//! encoded as a string in the format "+HHMM" or "-HHMM". //! use std::{fmt::Display, str::FromStr}; diff --git a/mercury/src/internal/object/tree.rs b/mercury/src/internal/object/tree.rs index 8da12dae0..3da39b187 100644 --- a/mercury/src/internal/object/tree.rs +++ b/mercury/src/internal/object/tree.rs @@ -120,11 +120,11 @@ impl TreeItemMode { /// \0 /// ``` /// - `` is the mode of the object, represented as a six-digit octal number. The first digit -/// represents the object type (tree, blob, etc.), and the remaining digits represent the file mode or permissions. +/// represents the object type (tree, blob, etc.), and the remaining digits represent the file mode or permissions. /// - `` is the name of the object. /// - `\0` is a null byte separator. /// - `` is the ID of the object that represents the contents of the file or -/// directory, represented as a binary SHA-1 hash. +/// directory, represented as a binary SHA-1 hash. /// /// # Example /// ```bash diff --git a/mercury/src/internal/object/types.rs b/mercury/src/internal/object/types.rs index 23b3de47a..0d2b7327f 100644 --- a/mercury/src/internal/object/types.rs +++ b/mercury/src/internal/object/types.rs @@ -10,13 +10,13 @@ use crate::errors::GitError; /// * `Blob` (1): A Git object that stores the content of a file. /// * `Tree` (2): A Git object that represents a directory or a folder in a Git repository. /// * `Commit` (3): A Git object that represents a commit in a Git repository, which contains -/// information such as the author, committer, commit message, and parent commits. +/// information such as the author, committer, commit message, and parent commits. /// * `Tag` (4): A Git object that represents a tag in a Git repository, which is used to mark a -/// specific point in the Git history. +/// specific point in the Git history. /// * `OffsetDelta` (6): A Git object that represents a delta between two objects, where the delta -/// is stored as an offset to the base object. +/// is stored as an offset to the base object. /// * `HashDelta` (7): A Git object that represents a delta between two objects, where the delta -/// is stored as a hash of the base object. +/// is stored as a hash of the base object. /// /// By assigning unique integer values to each Git object type, Git can easily and efficiently /// identify the type of an object and perform the appropriate operations on it. when parsing a Git diff --git a/mercury/src/internal/pack/decode.rs b/mercury/src/internal/pack/decode.rs index edfe2a3e9..ab306024a 100644 --- a/mercury/src/internal/pack/decode.rs +++ b/mercury/src/internal/pack/decode.rs @@ -46,12 +46,12 @@ impl Drop for Pack { impl Pack { /// # Parameters /// - `thread_num`: The number of threads to use for decoding and cache, `None` mean use the number of logical CPUs. - /// It can't be zero, or panic
+ /// It can't be zero, or panic
/// - `mem_limit`: The maximum size of the memory cache in bytes, or None for unlimited. - /// The 80% of it will be used for [Caches]
+ /// The 80% of it will be used for [Caches]
/// **Not very accurate, because of memory alignment and other reasons, overuse about 15%**
/// - `temp_path`: The path to a directory for temporary files, default is "./.cache_temp"
- /// For example, thread_num = 4 will use up to 8 threads (4 for decoding and 4 for cache)
+ /// For example, thread_num = 4 will use up to 8 threads (4 for decoding and 4 for cache)
/// - `clean_tmp`: whether to remove temp dir /// pub fn new(thread_num: Option, mem_limit: Option, temp_path: Option, clean_tmp: bool) -> Self { diff --git a/scorpio/Cargo.toml b/scorpio/Cargo.toml index 0b1981436..271f9c28f 100644 --- a/scorpio/Cargo.toml +++ b/scorpio/Cargo.toml @@ -21,6 +21,8 @@ sea-orm = { version= "0.12.15",features = [ ] } once_cell = "1.19.0" tokio = { version = "1.38.1", features = ["full"] } - + +[features] +async-io = [] [workspace]