diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3c6253dd7..1e98c56b2 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:24.04 WORKDIR /home/ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5352bae2d..d12d12579 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -15,5 +15,10 @@ "**/CODE_OF_CONDUCT.md": true, "**/LICENSE": true } + }, + "hostRequirements": { + "cpus": 8, + "memory": "32gb", + "storage": "64gb" } } \ No newline at end of file diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 31ffb9c89..a4187e43c 100644 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -9,22 +9,25 @@ apt-get install -y \ zsh \ vim \ build-essential \ - openssl + openssl \ + libssl-dev \ + fuse3 \ + libfuse3-dev \ + pkg-config \ + postgresql ## Install rustup and common components curl https://sh.rustup.rs -sSf | sh -s -- -y -rustup install nightly +rustup install default rustup component add rustfmt -rustup component add rustfmt --toolchain nightly -rustup component add clippy -rustup component add clippy --toolchain nightly +rustup component add clippy cargo install cargo-expand cargo install cargo-edit -## setup and install oh-my-zsh +## Setup and install oh-my-zsh sh -c "$(curl -fsSL https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh)" cp -R /root/.oh-my-zsh /home/$USERNAME cp /root/.zshrc /home/$USERNAME sed -i -e "s/\/root\/.oh-my-zsh/\/home\/$USERNAME\/.oh-my-zsh/g" /home/$USERNAME/.zshrc -chown -R $USER_UID:$USER_GID /home/$USERNAME/.oh-my-zsh /home/$USERNAME/.zshrc +chown -R $USER_UID:$USER_GID /home/$USERNAME/.oh-my-zsh /home/$USERNAME/.zshrc \ No newline at end of file diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index cadda3c1c..9c502c5b2 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -2,7 +2,7 @@ # # History: # 1. 2023-02-14: Created at 2023-02-14T16:00:00Z by Quanyi Ma -# +# 2. 2024-05-07: Update the `fuse` job to install `fuse3` and `libfuse3-dev` at 2024-05-07T16:00:00Z by Xiaoyang Han # # @@ -11,6 +11,7 @@ on: [ push, pull_request ] name: Base GitHub Action for Check, Test and Lints jobs: + # check: name: Check runs-on: ubuntu-latest @@ -25,6 +26,7 @@ jobs: with: command: check + # clippy: name: Clippy runs-on: ubuntu-latest @@ -40,7 +42,8 @@ jobs: with: command: clippy args: --workspace --all-targets --all-features -- -D warnings - + + # doc: name: Doc runs-on: ubuntu-latest @@ -54,6 +57,8 @@ jobs: - uses: actions-rs/cargo@v1 with: command: doc + + # fuse: name: Fuse Lints runs-on: ubuntu-latest diff --git a/README.md b/README.md index 92d114030..fe8788020 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,10 @@ When it comes to managing large codebases in a centralized manner, trunk-based d Mega will support conventional commits, which are a set of rules for creating clear and concise commit messages. More information on the [Conventional Commits](https://www.conventionalcommits.org/). +### Code Owners + +Mega will support code owners, which are a set of rules for defining who owns a particular piece of code. More information on the [Code Owners](https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners). + ### Decentralized Open Source Collaboration For now, the entire open source community base on Git and GitHub. It's centralized model, and it's not suitable for growing speed of open source world. Mega is working on build a decentralized open source collaboration model with [ZTM](https://github.com/flomesh-io/ztm)(Zero Trust Model) and decentralized social network like [Nostr](https://nostr.com), [Matrix](https://matrix.org) and [Mastodon](https://joinmastodon.org). diff --git a/docs/development.md b/docs/development.md index 6ff9b836c..067a29628 100644 --- a/docs/development.md +++ b/docs/development.md @@ -343,6 +343,95 @@ $ git clone http://localhost:8000/third-part/linux.git ``` +### GitHub Codespace + +If you are using GitHub codespaces, you can follow the steps below to set up the Mega project. When you create a new Codespace, the Mega project will be cloned automatically. You can then follow the steps below to set up the project. + +When the codespace is ready, the PostgreSQL will be installed and started automatically. You can then follow the steps below to set up the database with below steps. + +```bash +## Start PostgreSQL +/etc/init.d/postgresql start + +sudo -u postgres psql mega -c "CREATE DATABASE mega;" +sudo -u postgres psql mega < /workspaces/mega/sql/pg_20240205__init.sql +sudo -u postgres psql mega -c "CREATE USER mega WITH ENCRYPTED PASSWORD 'mega';" +sudo -u postgres psql mega -c "GRANT ALL PRIVILEGES ON DATABASE mega TO mega;" +sudo -u postgres psql mega -c "GRANT ALL ON ALL TABLES IN SCHEMA public to mega;" +sudo -u postgres psql mega -c "GRANT ALL ON ALL TABLES IN SCHEMA public to mega;" +sudo -u postgres psql mega -c "GRANT ALL ON ALL SEQUENCES IN SCHEMA public to mega;" +sudo -u postgres psql mega -c "GRANT ALL ON ALL FUNCTIONS IN SCHEMA public to mega;" +``` + +Config `confg.toml` file for the Mega project. + +```ini + # Fillin the following environment variables with values you set + + ## Logging Configuration + [log] + # The path which log file is saved + log_path = "/tmp/.mega/logs" + + # log level + level = "debug" + + # print std log in console, disable it on production for performance + print_std = true + + + [database] + # database connection url + db_url = "postgres://postgres:postgres@localhost:5432/mega" + + # db max connection, setting it to twice the number of CPU cores would be appropriate. + max_connection = 2 + + # db min connection, setting it to the number of CPU cores would be appropriate. + min_connection = 4 + + # Whether to disabling SQLx Log + sqlx_logging = false + + [ssh] + ssh_key_path = "/tmp/.mega/ssh" + + [storage] + # raw object stroage type, can be `local` or `remote` + raw_obj_storage_type = "LOCAL" + + ## If the object file size exceeds the threshold value, it will be handled by file storage instead of the database, Unit is KB + big_obj_threshold = 1024 + + # set the local path of the project storage + raw_obj_local_path = "/tmp/.mega/objects" + + lfs_obj_local_path = "/tmp/.mega/lfs" + + obs_access_key = "" + obs_secret_key = "" + + # cloud storage region + obs_region = "cn-east-3" + + # Override the endpoint URL used for remote storage services + obs_endpoint = "https://obs.cn-east-3.myhuaweicloud.com" + + + [monorepo] + ## Only import directory support multi-branch commit and tag, repo under regular directory only support main branch only + ## Mega treats files in that directory as import repo and other directories as monorepo + import_dir = "/third-part" + + # The maximum memory used by decode, Unit is GB + pack_decode_mem_size = 4 + + # The location where the object stored when the memory used by decode exceeds the limit + pack_decode_cache_path = "/tmp/.mega/cache" + + clean_cache_after_decode = true +``` + ## Comment Guideline This guide outlines the recommended order for importing dependencies in Rust projects. diff --git a/mercury/src/internal/index.rs b/mercury/src/internal/index.rs index 546d8a6d8..e2fa7cdf3 100644 --- a/mercury/src/internal/index.rs +++ b/mercury/src/internal/index.rs @@ -464,14 +464,6 @@ mod tests { } } - // #[test] - // fn test_libra_index() { - // let index = Index::from_file(PathBuf::from(test::TEST_DIR).join(".libra/index")).unwrap(); - // for (_, entry) in index.entries.iter() { - // println!("{}", entry); - // } - // } - #[test] fn test_index_to_file() { let index = Index::from_file("../tests/data/index/index-760").unwrap(); diff --git a/scorpio/README.md b/scorpio/README.md index c2fe5827f..1328c8ff5 100644 --- a/scorpio/README.md +++ b/scorpio/README.md @@ -8,7 +8,8 @@ The FUSE project consists of two components: the fuse kernel module (maintained When VFS receives a file access request from the user process and this file belongs to a certain fuse file system, it will forward the request to a kernel module named "fuse". Then, "fuse" converts the request into the protocol format agreed upon with the daemon and transmits it to the daemon process. -Currently, there have been many successful fuse based projects +Currently, there have been many successful fuse based projects, + - [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) makes you operate files and directories in S3 bucket like a local file system ![Github stars](https://img.shields.io/github/stars/s3fs-fuse/s3fs-fuse.svg) @@ -35,19 +36,14 @@ Because the code organization requirements are different from the existing popul 7. Incremental repackage: Using multiple package indexes, repackage packaged Git data into fewer package files without interrupting parallel commands. -### Some Related -#### [Vfs for Git](https://github.com/microsoft/VFSForGit) +### Some Related + +#### [VFS for Git](https://github.com/microsoft/VFSForGit) from Microsoft VFS For Git is a preliminary attempt by Microsoft on the Monorepo client, which implemented the FUSE system based on Sqlite and Mutli pack index, achieving on-demand partial pull functionality. The client will perceive the user's "open directory" operation before pulling the code content under the corresponding directory. #### [Sapling](https://sapling-scm.com/) from Meta The structure of Sapling is achieved through a multi-layered architecture, with each checkout corresponding to a mount point, followed by an Overlay layer. At the same time, it provides third-party interfaces for other programs to use, so that some heavy IO and computational parts do not need to be consumed by the performance of the virtual layer. - -#### [Scalar](https://github.com/microsoft/git/blob/HEAD/contrib/scalar/docs/index.md) -Scalar is a tool that helps Git scale to some of the largest Git repositories. Initially, it was a single standalone git plugin based on Vfs for git, inheriting GVFS. No longer using FUSE. It implements aware partial directory management. Users need to manage and register the required workspace directory on their own. - -Ease of use can be improved through the fuse mechanism. - ### How to Use? @@ -65,4 +61,6 @@ Ease of use can be improved through the fuse mechanism. [5] [sapling](https://github.com/facebook/sapling.git) : A cross-platform, highly scalable, Git-compatible source control system. -[6] [fuser](https://github.com/cberner/fuser.git) : A Rust library crate for easy implementation of FUSE filesystems in userspace. \ No newline at end of file +[6] [fuser](https://github.com/cberner/fuser.git) : A Rust library crate for easy implementation of FUSE filesystems in userspace. + +[7] [Scalar](https://github.com/microsoft/git/blob/HEAD/contrib/scalar/docs/index.md) : Scalar is a tool that helps Git scale to some of the largest Git repositories. Initially, it was a single standalone git plugin based on Vfs for git, inheriting GVFS. No longer using FUSE. It implements aware partial directory management. Users need to manage and register the required workspace directory on their own. Ease of use can be improved through the fuse mechanism. diff --git a/scorpio/src/lib.rs b/scorpio/src/lib.rs index 50725e7da..2c5fdbb71 100644 --- a/scorpio/src/lib.rs +++ b/scorpio/src/lib.rs @@ -1,9 +1,14 @@ use std::{ffi::OsStr, path::Path, time::SystemTime}; -use fuser::{Filesystem, KernelConfig, ReplyAttr, ReplyBmap, ReplyCreate, ReplyData, ReplyDirectory, ReplyDirectoryPlus, ReplyEmpty, ReplyEntry, ReplyIoctl, ReplyLock, ReplyLseek, ReplyOpen, ReplyStatfs, ReplyWrite, ReplyXattr, Request, TimeOrNow}; use libc::{c_int, ENOSYS, EPERM}; + +use fuser::{ Filesystem, KernelConfig, ReplyAttr, ReplyBmap, ReplyCreate, ReplyData, ReplyDirectory }; +use fuser::{ ReplyDirectoryPlus, ReplyEmpty, ReplyEntry, ReplyIoctl, ReplyLock, ReplyLseek }; +use fuser::{ ReplyOpen, ReplyStatfs, ReplyWrite, ReplyXattr, Request, TimeOrNow}; use tracing::{debug,warn}; + #[allow(unused)] struct MegaFuse; + impl Filesystem for MegaFuse { /// Initialize filesystem. /// Called before any other filesystem method. diff --git a/scorpio/src/main.rs b/scorpio/src/main.rs new file mode 100644 index 000000000..fbedd9205 --- /dev/null +++ b/scorpio/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} \ No newline at end of file