From 395c4d838d400b676788e295cf553f541d1fa637 Mon Sep 17 00:00:00 2001 From: "benjamin.747" Date: Thu, 5 Sep 2024 10:25:30 +0800 Subject: [PATCH] [doc] add sql update README --- ceres/src/lfs/handler.rs | 4 +- docker/mono-pg-dockerfile | 2 +- docs/database.md | 11 - docs/development.md | 6 +- jupiter/src/storage/init.rs | 2 +- sql/README.md | 57 ++++ ...240205__init.sql => pg_20240905__init.sql} | 0 ...0711_init.sql => sqlite_20240905_init.sql} | 0 tests/common/mod.rs | 288 ------------------ tests/compose/http/compose.yaml | 41 --- tests/compose/mega_p2p/compose.yaml | 72 ----- tests/compose/ssh/compose.yaml | 40 --- 12 files changed, 64 insertions(+), 459 deletions(-) create mode 100644 sql/README.md rename sql/postgres/{pg_20240205__init.sql => pg_20240905__init.sql} (100%) rename sql/sqlite/{sqlite_20240711_init.sql => sqlite_20240905_init.sql} (100%) delete mode 100644 tests/common/mod.rs delete mode 100644 tests/compose/http/compose.yaml delete mode 100644 tests/compose/mega_p2p/compose.yaml delete mode 100644 tests/compose/ssh/compose.yaml diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index d6c824ef6..503e78658 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -616,9 +616,9 @@ async fn lfs_add_lock( async fn lfs_get_meta( storage: Arc, - oid: &String, + oid: &str, ) -> Result { - let result = storage.get_lfs_object(oid.clone()).await.unwrap(); + let result = storage.get_lfs_object(oid.to_owned()).await.unwrap(); match result { Some(val) => Ok(MetaObject { diff --git a/docker/mono-pg-dockerfile b/docker/mono-pg-dockerfile index 8211afe67..9c04bb95f 100644 --- a/docker/mono-pg-dockerfile +++ b/docker/mono-pg-dockerfile @@ -17,6 +17,6 @@ EXPOSE 5432 # Add the database initialization script to the container # When the container starts, PostgreSQL will automatically execute all .sql files in the docker-entrypoint-initdb.d/ directory -COPY ./sql/postgres/pg_20240205__init.sql /docker-entrypoint-initdb.d/ +COPY ./sql/postgres/pg_20240905__init.sql /docker-entrypoint-initdb.d/ CMD ["postgres"] diff --git a/docs/database.md b/docs/database.md index 9f5d55b2e..e1a120014 100644 --- a/docs/database.md +++ b/docs/database.md @@ -371,17 +371,6 @@ erDiagram ## 3. Prerequisites -- You need to execute SQL files in a specific order to init the database. - - For example using `PostgreSQL`, execute the files under `sql\postgres`: - - pg_20240205__init.sql - - or if you are using `Mysql`, execute the files under `sql\mysql`: - - mysql_20231106__init.sql - - - Generating entities: Entities can be generated from the database table structure with the following command diff --git a/docs/development.md b/docs/development.md index 317708ad8..9f0518635 100644 --- a/docs/development.md +++ b/docs/development.md @@ -49,7 +49,7 @@ ```bash $ cd mega/sql/postgres - $ psql mega < pg_20240205_init.sql + $ psql mega < pg_YYYYMMDD_init.sql ``` 3. Create user and grant privileges. @@ -142,7 +142,7 @@ ```bash $ cd mega/sql/postgres - $ sudo -u postgres psql mega < pg_20240205__init.sql + $ sudo -u postgres psql mega < pg_YYYYMMDD__init.sql ``` 4.Create user and grant privileges. @@ -204,7 +204,7 @@ When the codespace is ready, the PostgreSQL will be installed and started automa /etc/init.d/postgresql start sudo -u postgres psql mega -c "CREATE DATABASE mega;" -sudo -u postgres psql mega < /workspaces/mega/sql/pg_20240205__init.sql +sudo -u postgres psql mega < /workspaces/mega/sql/pg_YYYYMMDD__init.sql sudo -u postgres psql mega -c "CREATE USER mega WITH ENCRYPTED PASSWORD 'mega';" sudo -u postgres psql mega -c "GRANT ALL PRIVILEGES ON DATABASE mega TO mega;" sudo -u postgres psql mega -c "GRANT ALL ON ALL TABLES IN SCHEMA public to mega;" diff --git a/jupiter/src/storage/init.rs b/jupiter/src/storage/init.rs index e5d905578..7d1a65643 100644 --- a/jupiter/src/storage/init.rs +++ b/jupiter/src/storage/init.rs @@ -51,7 +51,7 @@ async fn setup_sql(conn: &DatabaseConnection) -> Result<(), TransactionError, - pub tree_ids: Vec, - pub blob_ids: Vec, - pub tag_ids: Vec, -} - -// TODO: got some problem on copy content files -// pub fn build_image(config: &P2pTestConfig) { -// let mut child = Command::new("docker") -// .arg("compose") -// .arg("-f") -// .arg(&config.compose_path) -// .arg("build") -// .spawn() -// .expect("Failed to execute command"); -// assert!(child.wait().is_ok()); -// } - -pub fn start_server(config: &P2pTestConfig) { - let path = config.compose_path.clone(); - thread::spawn(move || { - let mut child = Command::new("docker") - .arg("compose") - .arg("-f") - .arg(path) - .arg("up") - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .spawn() - .expect("Failed to execute command"); - // Wait for the child process to finish and get the result - let _ = child.wait().expect("Failed to wait for child process"); - }); -} - -pub async fn lifecycle_check(config: &P2pTestConfig) { - loop { - let resp = reqwest::get(config.lifecycle_url.clone()).await.unwrap(); - if resp.status() == 200 { - println!("lifecycle check passed"); - break; - } else { - println!( - "lifecycle check failed, retrying in {} secs ...", - config.lifecycle_retrying - ); - } - sleep(Duration::from_secs(config.lifecycle_retrying)); - } -} - -pub async fn init_by_pack(config: &P2pTestConfig) { - let pkt_line = format!("00980000000000000000000000000000000000000000 {} refs/heads/master\0 report-status-v2 side-band-64k agent=mega-test\n0000", config.commit_id); - let f = tokio::fs::File::open(&config.pack_path).await.unwrap(); - - match config.protocol { - TransportProtocol::Http | TransportProtocol::P2p => { - let stream = - ReaderStream::new(Cursor::new(Bytes::from(pkt_line))).chain(ReaderStream::new(f)); - - let client = reqwest::Client::new(); - let url = format!("http://localhost:8000{}/git-receive-pack", config.repo_path); - let resp = client - .post(url) - .body(reqwest::Body::wrap_stream(stream)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 200); - println!("resp: {:?}", resp.bytes().await); - } - TransportProtocol::Ssh => { - // Create an asynchronous stream from the pkt_line string - let pkt_line_stream = Cursor::new(pkt_line); - // Combine the pkt_line and file streams - let combined_stream = pkt_line_stream.chain(f); - let mut ssh: Session = Session::connect("git".to_string(), ("localhost", 8100)) - .await - .unwrap(); - let code = ssh - .call( - &format!("git-receive-pack '{}'", config.repo_path), - combined_stream, - ) - .await - .unwrap(); - println!("Exitcode: {:?}", code); - } - _ => todo!(), - } -} - -pub fn clone_by_type(config: &P2pTestConfig, url: &str, into_path: &Path) -> Repository { - match config.protocol { - TransportProtocol::Http => match Repository::clone(url, into_path) { - Ok(repo) => repo, - Err(e) => panic!("failed to clone: {}", e), - }, - TransportProtocol::Ssh => { - // Create callbacks for SSH authentication - let mut callbacks = RemoteCallbacks::new(); - - callbacks.credentials(|_url, username_from_url, _allowed_types| { - Cred::ssh_key( - username_from_url.unwrap(), - None, - Path::new(&format!("{}/.ssh/id_rsa", env::var("HOME").unwrap())), - None, - ) - }); - // Create fetch options - let mut fetch_options = FetchOptions::new(); - fetch_options.remote_callbacks(callbacks); - - // Clone the repository - RepoBuilder::new() - .fetch_options(fetch_options) - .clone(url, into_path) - .expect("Failed to clone repository") - } - _ => todo!(), - } -} - -pub fn push_by_type(config: &P2pTestConfig, repo: &Repository) { - let mut remote = repo.find_remote("origin").unwrap(); - let refspecs = ["refs/heads/master:refs/heads/master"]; - - let mut op = if config.protocol == TransportProtocol::Ssh { - let mut callbacks = RemoteCallbacks::new(); - callbacks.credentials(|_url, username_from_url, _allowed_types| { - Cred::ssh_key( - username_from_url.unwrap(), - None, - Path::new(&format!("{}/.ssh/id_rsa", env::var("HOME").unwrap())), - None, - ) - }); - let mut push_options = PushOptions::new(); - push_options.remote_callbacks(callbacks); - Some(push_options) - } else { - None - }; - remote.push(&refspecs, op.as_mut()).unwrap(); -} - -pub fn stop_server(config: &P2pTestConfig) { - println!("stoping server and cleaning resources..."); - Command::new("docker") - .arg("compose") - .arg("-f") - .arg(&config.compose_path) - .arg("down") - .spawn() - .expect("Failed to execute command"); -} - -struct Client {} - -// More SSH event handlers -// can be defined in this trait -// In this example, we're only using Channel, so these aren't needed. -#[async_trait] -impl client::Handler for Client { - type Error = russh::Error; - - async fn check_server_key( - &mut self, - _server_public_key: &key::PublicKey, - ) -> Result { - Ok(true) - } -} - -/// This struct is a convenience wrapper -/// around a russh client -pub struct Session { - session: client::Handle, -} - -impl Session { - pub async fn connect(user: impl Into, addrs: A) -> Result { - let ssh_config = SshConfig::default(); - let key_pair = load_key(ssh_config.ssh_key_path)?; - let config = client::Config { - inactivity_timeout: Some(Duration::from_secs(5)), - ..<_>::default() - }; - - let config = Arc::new(config); - let sh = Client {}; - - let mut session = client::connect(config, addrs, sh).await?; - let auth_res = session - .authenticate_publickey(user, Arc::new(key_pair)) - .await?; - - if !auth_res { - anyhow::bail!("Authentication failed"); - } - - Ok(Self { session }) - } - - pub async fn call( - &mut self, - command: &str, - data: R, - ) -> Result { - let mut channel = self.session.channel_open_session().await?; - channel.exec(true, command).await?; - // direct send pack to server regardless of the return - channel.data(data).await?; - channel.eof().await?; - - let mut code = 0; - let mut stdout = tokio::io::stdout(); - - loop { - // There's an event available on the session channel - let Some(msg) = channel.wait().await else { - break; - }; - match msg { - // Write data to the terminal - ChannelMsg::Data { ref data } => { - stdout.write_all(data).await?; - stdout.flush().await?; - } - // The command has returned an exit code - ChannelMsg::ExitStatus { exit_status } => { - code = exit_status; - channel.eof().await?; - break; - } - _ => {} - } - } - - stdout - .write_all(format!("exit code:{}\n", code).as_bytes()) - .await?; - Ok(code) - } -} diff --git a/tests/compose/http/compose.yaml b/tests/compose/http/compose.yaml deleted file mode 100644 index 95b960291..000000000 --- a/tests/compose/http/compose.yaml +++ /dev/null @@ -1,41 +0,0 @@ -x-environment: &commonEnvironment - MEGA_DB_MAX_CONNECTIONS: 16 - MEGA_DB_MIN_CONNECTIONS: 2 - MEGA_DB_SQLX_LOGGING: false - MEGA_SSH_KEY: "/tmp/.mega/ssh" - MEGA_OBJ_STORAGR_TYPE: "LOCAL" - MEGA_OBJ_LOCAL_PATH: "/tmp/.mega/objects" - MEGA_BIG_OBJ_THRESHOLD_SIZE: 1024 - GIT_INTERNAL_DECODE_CACHE_SIZE: 1000 - GIT_INTERNAL_DECODE_STORAGE_BATCH_SIZE: 10000 - GIT_INTERNAL_DECODE_STORAGE_TQUEUE_SIZE: 10 - GIT_INTERNAL_DECODE_CACHE_TYEP: "lru" - REDIS_CONFIG: "redis://172.17.0.1:6379" - -services: - http_server: - build: - context: ../../../ - ports: - - "8000:8000" - environment: - <<: *commonEnvironment - MEGA_DB_POSTGRESQL_URL: "postgres://postgres:postgres@172.17.0.1:5433/mega" - command: service https --host 0.0.0.0 - depends_on: - - redis - - postgres - redis: - image: "redis:alpine" - ports: - - "6379:6379" - postgres: - image: postgres:latest - ports: - - "5433:5432" - volumes: - - ../../../sql/postgres/pg_20240115__init.sql:/docker-entrypoint-initdb.d/init.sql - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: mega diff --git a/tests/compose/mega_p2p/compose.yaml b/tests/compose/mega_p2p/compose.yaml deleted file mode 100644 index ee0ce095f..000000000 --- a/tests/compose/mega_p2p/compose.yaml +++ /dev/null @@ -1,72 +0,0 @@ -x-environment: &commonEnvironment - MEGA_DB_MAX_CONNECTIONS: 16 - MEGA_DB_MIN_CONNECTIONS: 2 - MEGA_DB_SQLX_LOGGING: false - MEGA_OBJ_STORAGR_TYPE: "LOCAL" - MEGA_OBJ_LOCAL_PATH: "/tmp/.mega" - MEGA_BIG_OBJ_THRESHOLD_SIZE: 1024 - GIT_INTERNAL_DECODE_CACHE_SIZE: 1000 - GIT_INTERNAL_DECODE_STORAGE_BATCH_SIZE: 10000 - GIT_INTERNAL_DECODE_STORAGE_TQUEUE_SIZE: 10 - GIT_INTERNAL_DECODE_CACHE_TYEP: "lru" - REDIS_CONFIG: "redis://172.17.0.1:6379" - -services: - p2prelay: - build: - context: ../../../ - ports: - - "8200:8200" - environment: *commonEnvironment - command: service p2p --host 0.0.0.0 --p2p-port 8200 --relay-server - depends_on: - - redis - node_a: - image: mega_p2p-p2prelay - ports: - - "8300:8300" #p2p node port - - "8301:8001" #p2p http api port - - "8000:8000" #http server api port - environment: - <<: *commonEnvironment - MEGA_DB_POSTGRESQL_URL: "postgres://postgres:postgres@172.17.0.1:5433/mega" - command: service start http p2p --host 0.0.0.0 --p2p-port 8300 --bootstrap-node /ip4/172.17.0.1/tcp/8200 --secret-key 6b911fd37cdf5c81d4c0adb1ab7fa822ed253ab0ad9aa18d77257c88b29b7181 - depends_on: - - postgres_a - - p2prelay - node_b: - image: mega_p2p-p2prelay - ports: - - "8400:8400" - - "8401:8001" - environment: - <<: *commonEnvironment - MEGA_DB_POSTGRESQL_URL: "postgres://postgres:postgres@172.17.0.1:5434/mega" - command: service p2p --host 0.0.0.0 --p2p-port 8400 --bootstrap-node /ip4/172.17.0.1/tcp/8200 --secret-key 6b911fd37cdf5c81d4c0adb1ab7fa822ed253ab0ad9aa18d77257c88b29b718e - depends_on: - - postgres_b - - p2prelay - redis: - image: "redis:alpine" - ports: - - "6379:6379" - postgres_a: - image: postgres:latest - ports: - - "5433:5432" - volumes: - - ../../../sql/postgres/pg_20240115__init.sql:/docker-entrypoint-initdb.d/init.sql - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: mega - postgres_b: - image: postgres:latest - ports: - - "5434:5432" - volumes: - - ../../../sql/postgres/pg_20240115__init.sql:/docker-entrypoint-initdb.d/init.sql - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: mega diff --git a/tests/compose/ssh/compose.yaml b/tests/compose/ssh/compose.yaml deleted file mode 100644 index 43e19354e..000000000 --- a/tests/compose/ssh/compose.yaml +++ /dev/null @@ -1,40 +0,0 @@ -x-environment: &commonEnvironment - MEGA_DB_MAX_CONNECTIONS: 16 - MEGA_DB_MIN_CONNECTIONS: 2 - MEGA_DB_SQLX_LOGGING: false - MEGA_OBJ_STORAGR_TYPE: "LOCAL" - MEGA_OBJ_LOCAL_PATH: "/tmp/.mega" - MEGA_BIG_OBJ_THRESHOLD_SIZE: 1024 - GIT_INTERNAL_DECODE_CACHE_SIZE: 1000 - GIT_INTERNAL_DECODE_STORAGE_BATCH_SIZE: 10000 - GIT_INTERNAL_DECODE_STORAGE_TQUEUE_SIZE: 10 - GIT_INTERNAL_DECODE_CACHE_TYEP: "lru" - REDIS_CONFIG: "redis://172.17.0.1:6379" - -services: - http_server: - build: - context: ../../../ - ports: - - "8000:8000" - environment: - <<: *commonEnvironment - MEGA_DB_POSTGRESQL_URL: "postgres://postgres:postgres@172.17.0.1:5433/mega" - command: service start ssh https --host 0.0.0.0 - depends_on: - - redis - - postgres - redis: - image: "redis:alpine" - ports: - - "6379:6379" - postgres: - image: postgres:latest - ports: - - "5433:5432" - volumes: - - ../../../sql/postgres/pg_20240115__init.sql:/docker-entrypoint-initdb.d/init.sql - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: mega