Refactoring Dockerfiles and add init volume script

docker/README.md

@@ -6,20 +6,19 @@
 # build postgres image
 docker buildx build -t mono-pg:0.1-pre-release -f ./docker/mono-pg-dockerfile .
 
-# build backend mono image (default in release mode)
+# build backend mono engine image (default in release mode)
 docker buildx build -t mono-engine:0.1-pre-release -f ./docker/mono-engine-dockerfile .
 
-# build backend mono in debug mode
+# build backend mono engine in debug mode
 # docker buildx build -t mono-engine:0.1-pre-debug -f ./docker/mono-engine-dockerfile --build-arg BUILD_TYPE=debug .
 
-# build frontend moon image
-docker buildx build -t mono-ui:0.1-pre-release -f ./docker/moon-ui-dockerfile .
+# build frontend mono ui image
+docker buildx build -t mono-ui:0.1-pre-release -f ./docker/mono-ui-dockerfile .
 ```
 
-## Test mono and moon
+## Test Mono Engine
 
-
-### Test with SQLite
+### Test Mono Engine with SQLite
 
 ```bash
 # create network
@@ -31,20 +30,15 @@ docker run --rm -it -d --network mono-network --name mono-ui -e NEXT_PUBLIC_API_
 
 visit http://localhost:3000 to see the frontend
 
-## Test with PostgreSQL
+### Test Mono Engine with PostgreSQL
 
-[1] start postgres
+[1] Initiate volume for mono data and postgres data
 
 ```bash
-# create network
-docker network create mono-network
-
-# run postgres
-docker run --rm -it -d --name mono-pg --network mono-network -v /mnt/data/mono/pg-data:/var/lib/postgresql/data -p 5432:5432 mono-pg:0.1-pre-release
-docker run --rm -it -d --name mono-engine --network mono-network -v /mnt/data/mono/mono-data:/opt/mega -p 8000:8000 -p 22:9000 mono-engine:0.1-pre-release
+./init-volume.sh /mnt/data ./config.toml
 ```
 
-[3] edit `config.toml`, change `db_type` to `postgres` and db_url to `postgres://mega:mega@mega-db:5432/mega`
+Check `db_type` to `postgres` and db_url to `postgres://mono:mono@mega-db:5432/mono` in the `config.toml`
 
 ```toml
 [database]
@@ -54,29 +48,14 @@ db_type = "postgres"
 db_url = "postgres://mono:mono@mono-pg:5432/mono"
 ```
 
-[2] create default config
+[2] Start whole mono engine stack
 
 ```bash
-docker run --rm -it -d --network mono-network --name mono-engine -v ./mega_base:/opt/mega/etc mega-mono:0.1-pre-release
-docker stop mega-mono
-```
-
-[3] edit `mega_base/config.toml`, change `db_type` to `postgres` and db_url to `postgres://mega:mega@mega-db:5432/mega`
-
-```toml
-[database]
-db_type = "postgres"
-
-# used for sqlite
-db_path = "${base_dir}/mega.db"
-
-# database connection url
-db_url = "postgres://mega:mega@mega-db:5432/mega"
-```
-
-[4] Start the mono again, and run the frontend.
+# create network
+docker network create mono-network
 
-```bash
-docker run --rm -it -d --network mega-network --name mega-mono -v ./mega_base:/opt/mega/etc mega-mono:0.1-pre-release
-docker run --rm -it -d --network mega-network -e NEXT_PUBLIC_API_URL=http://mega-mono:8000 -p 3000:3000 mega-moon:0.1-pre-release
+# run postgres
+docker run --rm -it -d --name mono-pg --network mono-network -v /mnt/data/mono/pg-data:/var/lib/postgresql/data -p 5432:5432 mono-pg:0.1-pre-release
+docker run --rm -it -d --name mono-engine --network mono-network -v /mnt/data/mono/mono-data:/opt/mega -p 8000:8000 -p 22:9000 mono-engine:0.1-pre-release
+docker run --rm -it -d --name mono-ui --network mono-network -e NEXT_PUBLIC_API_URL=http://mono-engine:8000 -p 3000:3000 mono-ui:0.1-pre-release
 ```

docker/config.toml

@@ -0,0 +1,93 @@
+# the directory where the data files is located, such as logs, database, etc.
+# can be overrided by environment variable `MEGA_BASE_DIR`
+base_dir = "/tmp/.mono"
+
+# Filling the following environment variables with values you set
+## Logging Configuration
+[log]
+# The path which log file is saved
+log_path = "${base_dir}/logs"
+
+# log level
+level = "debug"
+
+# print std log in console, disable it on production for performance
+print_std = true
+
+
+[database]
+# "sqlite" | "postgres"
+# "sqlite" will use `db_path` and ignore `db_url`
+db_type = "postgres"
+
+# used for sqlite
+db_path = "${base_dir}/mega.db"
+
+# database connection url
+db_url = "postgres://mono:mono@mono-pg:5432/mono"
+
+# db max connection, setting it to twice the number of CPU cores would be appropriate.
+max_connection = 16
+
+# db min connection, setting it to the number of CPU cores would be appropriate.
+min_connection = 8
+
+# Whether to disabling SQLx Log
+sqlx_logging = false
+
+
+[ssh]
+ssh_key_path = "${base_dir}/etc/ssh"
+
+[storage]
+# raw object stroage type, can be `local` or `remote`
+raw_obj_storage_type = "LOCAL"
+
+## If the object file size exceeds the threshold value, it will be handled by file storage instead of the database, Unit is KB
+big_obj_threshold = 1024
+
+# set the local path of the project storage
+raw_obj_local_path = "${base_dir}/objects"
+
+lfs_obj_local_path = "${base_dir}/lfs"
+
+obs_access_key = ""
+obs_secret_key = ""
+
+# cloud storage region
+obs_region = "cn-east-3"
+
+# Override the endpoint URL used for remote storage services
+obs_endpoint = "https://obs.cn-east-3.myhuaweicloud.com"
+
+
+[monorepo]
+## Only import directory support multi-branch commit and tag, monorepo only support main branch
+## Mega treats files under this directory as import repo and other directories as monorepo
+import_dir = "/third-part"
+
+
+[pack]
+# The maximum memory used by decode, Unit is GB
+pack_decode_mem_size = 4
+
+# The location where the object stored when the memory used by decode exceeds the limit
+pack_decode_cache_path = "${base_dir}/cache"
+
+clean_cache_after_decode = true
+
+# The maximum meesage size in channel buffer while decode
+channel_message_size = 1_000_000
+
+[lfs]
+## IMPORTANT: The 'enable_split' feature can only be enabled for new databases. Existing databases do not support this feature.
+# Enable or disable splitting large files into smaller chunks
+enable_split = true  # Default is disabled. Set to true to enable file splitting.
+
+# Size of each file chunk when splitting is enabled, in bytes. Ignored if splitting is disabled.
+split_size = 20971520 # Default size is 20MB (20971520 bytes)
+
+[oauth]
+# GitHub OAuth application client id and secret
+github_client_id = ""
+github_client_secret = ""

docker/init-volume.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# Check if both required arguments are provided
+if [ $# -ne 2 ]; then
+    echo "Error: Both directory path and config file path are required"
+    echo "Usage: $0 <directory_path> <config_file_path>"
+    exit 1
+fi
+
+# Get the directory path argument
+base_dir="$1"
+
+# Get the config file path
+config_file="$2"
+
+# Check if the config file exists
+if [ ! -f "$config_file" ]; then
+    echo "Error: Config file not found at $config_file"
+    exit 1
+fi
+
+# Function to check if a command exists
+command_exists() {
+    command -v "$1" >/dev/null 2>&1
+}
+
+# Check if the mono directory already exists
+mono_dir="$base_dir/mono"
+if [ -d "$mono_dir" ]; then
+    read -p "The directory $mono_dir already exists. Do you want to delete it and continue? (y/N): " confirm
+    if [[ $confirm == [yY] || $confirm == [yY][eE][sS] ]]; then
+        echo "Deleting existing mono directory. Root password may be required."
+        if command_exists sudo; then
+            sudo rm -rf "$mono_dir"
+        else
+            echo "Error: sudo is not available. Please run this script with appropriate permissions to delete the directory."
+            exit 1
+        fi
+    else
+        echo "Operation cancelled. Exiting..."
+        exit 0
+    fi
+fi
+
+# Create the base directory (if it doesn't exist)
+mkdir -p "$base_dir"
+
+# Create the mono directory
+mkdir -p "$mono_dir"
+
+# Create mono-data and pg-data directories under the mono directory
+mono_data_dir="$mono_dir/mono-data"
+pg_data_dir="$mono_dir/pg-data"
+mkdir -p "$mono_data_dir"
+mkdir -p "$pg_data_dir"
+
+# Create subdirectories in mono-data
+mkdir -p "$mono_data_dir/etc"
+mkdir -p "$mono_data_dir/cache"
+mkdir -p "$mono_data_dir/lfs"
+mkdir -p "$mono_data_dir/logs"
+mkdir -p "$mono_data_dir/objects"
+
+# Create ssh and https directories under etc
+ssh_dir="$mono_data_dir/etc/ssh"
+mkdir -p "$ssh_dir"
+mkdir -p "$mono_data_dir/etc/https"
+
+# Generate SSH key for sshd (non-interactive)
+ssh_key_file="$ssh_dir/ssh_host_rsa_key"
+ssh-keygen -t rsa -b 4096 -f "$ssh_key_file" -N "" -C "sshd host key" >/dev/null 2>&1
+if [ $? -eq 0 ]; then
+    echo "SSH host key generated at $ssh_key_file"
+else
+    echo "Warning: Failed to generate SSH host key"
+fi
+
+# Copy config file
+cp "$config_file" "$mono_data_dir/etc/config.toml"
+echo "Config file copied to $mono_data_dir/etc/config.toml"
+
+echo "Directory structure has been successfully created:"
+echo "$base_dir"
+echo "└── mono"
+echo "    ├── mono-data"
+echo "    │   ├── etc"
+echo "    │   │   ├── ssh"
+echo "    │   │   │   ├── ssh_host_rsa_key"
+echo "    │   │   │   └── ssh_host_rsa_key.pub"
+echo "    │   │   ├── https"
+echo "    │   │   └── config.toml"
+echo "    │   ├── cache"
+echo "    │   ├── lfs"
+echo "    │   ├── logs"
+echo "    │   └── objects"
+echo "    └── pg-data"
+
+echo "Note: Please review and set appropriate permissions if needed."