From aa4eeff1064c852d367f706f653b94108407f647 Mon Sep 17 00:00:00 2001 From: Quanyi Ma Date: Sun, 25 Aug 2024 18:05:59 +0800 Subject: [PATCH 1/3] Refactoring the name of Dockerfile Signed-off-by: Quanyi Ma --- docker/README.md | 2 +- docker/{moon-ui-dockerfile => mono-ui-dockerfile} | 0 neptune/libs/ztm | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename docker/{moon-ui-dockerfile => mono-ui-dockerfile} (100%) diff --git a/docker/README.md b/docker/README.md index a2b68c687..e84e34f6c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -13,7 +13,7 @@ docker buildx build -t mono-engine:0.1-pre-release -f ./docker/mono-engine-docke # docker buildx build -t mono-engine:0.1-pre-debug -f ./docker/mono-engine-dockerfile --build-arg BUILD_TYPE=debug . # build frontend moon image -docker buildx build -t mono-ui:0.1-pre-release -f ./docker/moon-ui-dockerfile . +docker buildx build -t mono-ui:0.1-pre-release -f ./docker/mono-ui-dockerfile . ``` ## Test mono and moon diff --git a/docker/moon-ui-dockerfile b/docker/mono-ui-dockerfile similarity index 100% rename from docker/moon-ui-dockerfile rename to docker/mono-ui-dockerfile diff --git a/neptune/libs/ztm b/neptune/libs/ztm index 2e0ac3ed1..f3ba7aa56 160000 --- a/neptune/libs/ztm +++ b/neptune/libs/ztm @@ -1 +1 @@ -Subproject commit 2e0ac3ed15b8374aeb8227b6a65a83dc6f2a97dc +Subproject commit f3ba7aa56814c5e23b12bb3f7d951b86977fd260 From 24c558eee70603eb7868e46e9c8ed0c363d2deaf Mon Sep 17 00:00:00 2001 From: Quanyi Ma Date: Sun, 25 Aug 2024 20:54:48 +0800 Subject: [PATCH 2/3] Update the README.md of docker Signed-off-by: Quanyi Ma --- docker/README.md | 51 ++++++++++++++---------------------------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/docker/README.md b/docker/README.md index e84e34f6c..c959f3a81 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,20 +6,19 @@ # build postgres image docker buildx build -t mono-pg:0.1-pre-release -f ./docker/mono-pg-dockerfile . -# build backend mono image (default in release mode) +# build backend mono engine image (default in release mode) docker buildx build -t mono-engine:0.1-pre-release -f ./docker/mono-engine-dockerfile . -# build backend mono in debug mode +# build backend mono engine in debug mode # docker buildx build -t mono-engine:0.1-pre-debug -f ./docker/mono-engine-dockerfile --build-arg BUILD_TYPE=debug . -# build frontend moon image +# build frontend mono ui image docker buildx build -t mono-ui:0.1-pre-release -f ./docker/mono-ui-dockerfile . ``` -## Test mono and moon +## Test Mono Engine - -### Test with SQLite +### Test Mono Engine with SQLite ```bash # create network @@ -31,20 +30,15 @@ docker run --rm -it -d --network mono-network --name mono-ui -e NEXT_PUBLIC_API_ visit http://localhost:3000 to see the frontend -## Test with PostgreSQL +### Test Mono Engine with PostgreSQL -[1] start postgres +[1] Initiate volume for mono data and postgres data ```bash -# create network -docker network create mono-network -# run postgres -docker run --rm -it -d --name mono-pg --network mono-network -v /mnt/data/mono/pg-data:/var/lib/postgresql/data -p 5432:5432 mono-pg:0.1-pre-release -docker run --rm -it -d --name mono-engine --network mono-network -v /mnt/data/mono/mono-data:/opt/mega -p 8000:8000 -p 22:9000 mono-engine:0.1-pre-release ``` -[3] edit `config.toml`, change `db_type` to `postgres` and db_url to `postgres://mega:mega@mega-db:5432/mega` +Check `db_type` to `postgres` and db_url to `postgres://mono:mono@mega-db:5432/mono` in the `config.toml` ```toml [database] @@ -54,29 +48,14 @@ db_type = "postgres" db_url = "postgres://mono:mono@mono-pg:5432/mono" ``` -[2] create default config +[2] Start whole mono engine stack ```bash -docker run --rm -it -d --network mono-network --name mono-engine -v ./mega_base:/opt/mega/etc mega-mono:0.1-pre-release -docker stop mega-mono -``` - -[3] edit `mega_base/config.toml`, change `db_type` to `postgres` and db_url to `postgres://mega:mega@mega-db:5432/mega` - -```toml -[database] -db_type = "postgres" - -# used for sqlite -db_path = "${base_dir}/mega.db" - -# database connection url -db_url = "postgres://mega:mega@mega-db:5432/mega" -``` - -[4] Start the mono again, and run the frontend. +# create network +docker network create mono-network -```bash -docker run --rm -it -d --network mega-network --name mega-mono -v ./mega_base:/opt/mega/etc mega-mono:0.1-pre-release -docker run --rm -it -d --network mega-network -e NEXT_PUBLIC_API_URL=http://mega-mono:8000 -p 3000:3000 mega-moon:0.1-pre-release +# run postgres +docker run --rm -it -d --name mono-pg --network mono-network -v /mnt/data/mono/pg-data:/var/lib/postgresql/data -p 5432:5432 mono-pg:0.1-pre-release +docker run --rm -it -d --name mono-engine --network mono-network -v /mnt/data/mono/mono-data:/opt/mega -p 8000:8000 -p 22:9000 mono-engine:0.1-pre-release +docker run --rm -it -d --name mono-ui --network mono-network -e NEXT_PUBLIC_API_URL=http://mono-engine:8000 -p 3000:3000 mono-ui:0.1-pre-release ``` \ No newline at end of file From 02e0c2f4ed31e4416d3a3699f693762496446e4a Mon Sep 17 00:00:00 2001 From: Quanyi Ma Date: Sun, 25 Aug 2024 22:35:33 +0800 Subject: [PATCH 3/3] Add init volume script Signed-off-by: Quanyi Ma --- docker/README.md | 2 +- docker/config.toml | 93 ++++++++++++++++++++++++++++++++++++++++ docker/init-volume.sh | 98 +++++++++++++++++++++++++++++++++++++++++++ neptune/libs/ztm | 2 +- 4 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 docker/config.toml create mode 100755 docker/init-volume.sh diff --git a/docker/README.md b/docker/README.md index c959f3a81..e5cd54e31 100644 --- a/docker/README.md +++ b/docker/README.md @@ -35,7 +35,7 @@ visit http://localhost:3000 to see the frontend [1] Initiate volume for mono data and postgres data ```bash - +./init-volume.sh /mnt/data ./config.toml ``` Check `db_type` to `postgres` and db_url to `postgres://mono:mono@mega-db:5432/mono` in the `config.toml` diff --git a/docker/config.toml b/docker/config.toml new file mode 100644 index 000000000..afa6b541d --- /dev/null +++ b/docker/config.toml @@ -0,0 +1,93 @@ +# the directory where the data files is located, such as logs, database, etc. +# can be overrided by environment variable `MEGA_BASE_DIR` +base_dir = "/tmp/.mono" + +# Filling the following environment variables with values you set +## Logging Configuration +[log] +# The path which log file is saved +log_path = "${base_dir}/logs" + +# log level +level = "debug" + +# print std log in console, disable it on production for performance +print_std = true + + +[database] +# "sqlite" | "postgres" +# "sqlite" will use `db_path` and ignore `db_url` +db_type = "postgres" + +# used for sqlite +db_path = "${base_dir}/mega.db" + +# database connection url +db_url = "postgres://mono:mono@mono-pg:5432/mono" + +# db max connection, setting it to twice the number of CPU cores would be appropriate. +max_connection = 16 + +# db min connection, setting it to the number of CPU cores would be appropriate. +min_connection = 8 + +# Whether to disabling SQLx Log +sqlx_logging = false + + +[ssh] +ssh_key_path = "${base_dir}/etc/ssh" + +[storage] +# raw object stroage type, can be `local` or `remote` +raw_obj_storage_type = "LOCAL" + +## If the object file size exceeds the threshold value, it will be handled by file storage instead of the database, Unit is KB +big_obj_threshold = 1024 + +# set the local path of the project storage +raw_obj_local_path = "${base_dir}/objects" + +lfs_obj_local_path = "${base_dir}/lfs" + +obs_access_key = "" +obs_secret_key = "" + +# cloud storage region +obs_region = "cn-east-3" + +# Override the endpoint URL used for remote storage services +obs_endpoint = "https://obs.cn-east-3.myhuaweicloud.com" + + +[monorepo] +## Only import directory support multi-branch commit and tag, monorepo only support main branch +## Mega treats files under this directory as import repo and other directories as monorepo +import_dir = "/third-part" + + +[pack] +# The maximum memory used by decode, Unit is GB +pack_decode_mem_size = 4 + +# The location where the object stored when the memory used by decode exceeds the limit +pack_decode_cache_path = "${base_dir}/cache" + +clean_cache_after_decode = true + +# The maximum meesage size in channel buffer while decode +channel_message_size = 1_000_000 + +[lfs] +## IMPORTANT: The 'enable_split' feature can only be enabled for new databases. Existing databases do not support this feature. +# Enable or disable splitting large files into smaller chunks +enable_split = true # Default is disabled. Set to true to enable file splitting. + +# Size of each file chunk when splitting is enabled, in bytes. Ignored if splitting is disabled. +split_size = 20971520 # Default size is 20MB (20971520 bytes) + +[oauth] +# GitHub OAuth application client id and secret +github_client_id = "" +github_client_secret = "" diff --git a/docker/init-volume.sh b/docker/init-volume.sh new file mode 100755 index 000000000..6705f3582 --- /dev/null +++ b/docker/init-volume.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Check if both required arguments are provided +if [ $# -ne 2 ]; then + echo "Error: Both directory path and config file path are required" + echo "Usage: $0 " + exit 1 +fi + +# Get the directory path argument +base_dir="$1" + +# Get the config file path +config_file="$2" + +# Check if the config file exists +if [ ! -f "$config_file" ]; then + echo "Error: Config file not found at $config_file" + exit 1 +fi + +# Function to check if a command exists +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +# Check if the mono directory already exists +mono_dir="$base_dir/mono" +if [ -d "$mono_dir" ]; then + read -p "The directory $mono_dir already exists. Do you want to delete it and continue? (y/N): " confirm + if [[ $confirm == [yY] || $confirm == [yY][eE][sS] ]]; then + echo "Deleting existing mono directory. Root password may be required." + if command_exists sudo; then + sudo rm -rf "$mono_dir" + else + echo "Error: sudo is not available. Please run this script with appropriate permissions to delete the directory." + exit 1 + fi + else + echo "Operation cancelled. Exiting..." + exit 0 + fi +fi + +# Create the base directory (if it doesn't exist) +mkdir -p "$base_dir" + +# Create the mono directory +mkdir -p "$mono_dir" + +# Create mono-data and pg-data directories under the mono directory +mono_data_dir="$mono_dir/mono-data" +pg_data_dir="$mono_dir/pg-data" +mkdir -p "$mono_data_dir" +mkdir -p "$pg_data_dir" + +# Create subdirectories in mono-data +mkdir -p "$mono_data_dir/etc" +mkdir -p "$mono_data_dir/cache" +mkdir -p "$mono_data_dir/lfs" +mkdir -p "$mono_data_dir/logs" +mkdir -p "$mono_data_dir/objects" + +# Create ssh and https directories under etc +ssh_dir="$mono_data_dir/etc/ssh" +mkdir -p "$ssh_dir" +mkdir -p "$mono_data_dir/etc/https" + +# Generate SSH key for sshd (non-interactive) +ssh_key_file="$ssh_dir/ssh_host_rsa_key" +ssh-keygen -t rsa -b 4096 -f "$ssh_key_file" -N "" -C "sshd host key" >/dev/null 2>&1 +if [ $? -eq 0 ]; then + echo "SSH host key generated at $ssh_key_file" +else + echo "Warning: Failed to generate SSH host key" +fi + +# Copy config file +cp "$config_file" "$mono_data_dir/etc/config.toml" +echo "Config file copied to $mono_data_dir/etc/config.toml" + +echo "Directory structure has been successfully created:" +echo "$base_dir" +echo "└── mono" +echo " ├── mono-data" +echo " │ ├── etc" +echo " │ │ ├── ssh" +echo " │ │ │ ├── ssh_host_rsa_key" +echo " │ │ │ └── ssh_host_rsa_key.pub" +echo " │ │ ├── https" +echo " │ │ └── config.toml" +echo " │ ├── cache" +echo " │ ├── lfs" +echo " │ ├── logs" +echo " │ └── objects" +echo " └── pg-data" + +echo "Note: Please review and set appropriate permissions if needed." \ No newline at end of file diff --git a/neptune/libs/ztm b/neptune/libs/ztm index f3ba7aa56..2e0ac3ed1 160000 --- a/neptune/libs/ztm +++ b/neptune/libs/ztm @@ -1 +1 @@ -Subproject commit f3ba7aa56814c5e23b12bb3f7d951b86977fd260 +Subproject commit 2e0ac3ed15b8374aeb8227b6a65a83dc6f2a97dc