diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 9c502c5b2..d240170af 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -6,7 +6,7 @@ # # -on: [ push, pull_request ] +on: [push, pull_request] name: Base GitHub Action for Check, Test and Lints @@ -17,6 +17,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + submodules: recursive - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -32,6 +34,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + submodules: recursive - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -49,6 +53,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + submodules: recursive - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -69,5 +75,6 @@ jobs: profile: minimal toolchain: stable override: true + submodules: recursive - run: sudo apt-get update && sudo apt-get install -y fuse3 libfuse3-dev - - run: cd ./scorpio && cargo clippy --all-targets --all-features -- -D warnings \ No newline at end of file + - run: cd ./scorpio && cargo clippy --all-targets --all-features -- -D warnings diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..95523bd3c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "neptune/libs/ztm"] + path = neptune/libs/ztm + url = git@github.com:flomesh-io/ztm.git + ignore = dirty diff --git a/Cargo.toml b/Cargo.toml index 97c0175dd..1f6b5a920 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "ceres", "libra", "vault", + "neptune", ] exclude = ["craft"] resolver = "1" @@ -23,6 +24,7 @@ ceres = { path = "ceres" } callisto = { path = "jupiter/callisto" } gemini = { path = "gemini" } vault = { path = "vault" } +neptune = { path = "neptune" } anyhow = "1.0.86" serde = "1.0.203" diff --git a/gemini/Cargo.toml b/gemini/Cargo.toml index cdd712775..8d87e9200 100644 --- a/gemini/Cargo.toml +++ b/gemini/Cargo.toml @@ -21,4 +21,4 @@ tokio = { workspace = true, features = ["net"] } chrono = { workspace = true } jupiter = { workspace = true } callisto = { workspace = true } -rust-ztm = { git = "https://github.com/flomesh-io/rust-ztm.git" } \ No newline at end of file +neptune = { workspace = true } \ No newline at end of file diff --git a/gemini/src/ztm/mod.rs b/gemini/src/ztm/mod.rs index 1e649df59..e98a422ee 100644 --- a/gemini/src/ztm/mod.rs +++ b/gemini/src/ztm/mod.rs @@ -271,7 +271,7 @@ pub struct LocalZTM { impl LocalZTM { pub fn start_ztm_agent(self) { tokio::spawn(async move { - rust_ztm::start_agent("ztm_agent.db", self.agent_port); + neptune::start_agent("ztm_agent.db", self.agent_port); }); } } diff --git a/neptune/Cargo.toml b/neptune/Cargo.toml new file mode 100644 index 000000000..3ae508c0f --- /dev/null +++ b/neptune/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "neptune" +version = "0.1.0" +edition = "2021" + +[build-dependencies] +cmake = "0.1.50" +num_cpus = { workspace = true } + +[dependencies] +libc = "0.1.0" +reqwest = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +tokio = { workspace = true, features = ["macros"] } +tokio-test = { workspace = true } + +[features] +agent-ui = [] diff --git a/neptune/README.md b/neptune/README.md new file mode 100644 index 000000000..0d9881a6b --- /dev/null +++ b/neptune/README.md @@ -0,0 +1,11 @@ +# neptune +rust wrap of flomesh-io/ztm + +> [!IMPORTANT] +> +> Building `neptune` requires `cmake`, `clang`, `nodejs`, `npm`, and the `rust toolchain`. +> +> Additionally, building with the `agent-ui` features requires that `vite` be installed. + +# features +- [x] `agent-ui`: build with ztm agent's web ui,for development and testing purpose. diff --git a/neptune/build.rs b/neptune/build.rs new file mode 100644 index 000000000..0507daf35 --- /dev/null +++ b/neptune/build.rs @@ -0,0 +1,112 @@ +use std::{fs, path::Path}; + +use cmake::Config; +fn build_agent_ui() { + let ui = Path::new("libs/ztm/agent/gui"); + if cfg!(feature = "agent-ui") { + if !ui.exists() { + // run npm run build in the libs/ztm/gui + let _ = std::process::Command::new("npm") + .current_dir("libs/ztm/gui") + .arg("run") + .arg("build") + .output() + .expect("failed to run npm run build in ztm/gui"); + } + } else if ui.exists() { + std::fs::remove_dir_all(ui).expect("failed to remove agent/gui"); + } +} + +fn parse_args_to_rustc(dst: &Path) { + // ** `cargo:rustc-*` format is used to pass information to the cargo build system + + // parse to `rustc` to look for dynamic library, used in running + let origin_path = if cfg!(target_os = "macos") { + "@executable_path" + } else if cfg!(target_os = "linux") { + "$ORIGIN" + } else { + "" // windows auto search excutable path + }; + println!( + "cargo:rustc-link-arg=-Wl,-rpath,{}/build,-rpath,{}", + dst.display(), + origin_path + ); + + // add the path to the library to the linker search path, used in build + println!("cargo:rustc-link-search={}/build", dst.display()); + + println!("cargo:rustc-link-lib=pipy"); +} + +/// copy `mega-app/` to `libs/ztm/agent/apps/mega` +fn copy_mega_apps() { + let src = Path::new("mega_app"); + let dst = Path::new("libs/ztm/agent/apps/mega"); + if src.exists() { + if dst.exists() { + fs::remove_dir_all(dst).expect("failed to remove origin agent/apps"); + } + // std::fs::copy(src, dst).expect("failed to copy mega to agent/apps"); + copy_dir_all(src, dst).expect("failed to copy mega to agent/apps"); + } +} + +fn main() { + // check submodule exists + let check_file = Path::new("libs/ztm/pipy/CMakeLists.txt"); + if !check_file.exists() { + panic!("Please run `git submodule update --init --recursive` to get the submodule"); + } + copy_mega_apps(); + + build_agent_ui(); + + /* compile ztm & pipy */ + // run npm install in the libs/ztm + let _ = std::process::Command::new("npm") + .current_dir("libs/ztm/pipy") + .arg("install") + .output() + .expect("failed to run npm install in ztm/pipy"); + + let mut config = Config::new("libs/ztm/pipy"); + + // set to use clang/clang++ to compile + config.define("CMAKE_C_COMPILER", "clang"); + config.define("CMAKE_CXX_COMPILER", "clang++"); + + // compile ztm in pipy + config.define("PIPY_SHARED", "ON"); + config.define("PIPY_GUI", "OFF"); + config.define("PIPY_CODEBASES", "ON"); + config.define( + "PIPY_CUSTOM_CODEBASES", + "ztm/agent:../agent,ztm/hub:../hub,ztm/ca:../ca", + ); + + config.no_build_target(true); + + // build, with half of the cpu + let cups = num_cpus::get() - num_cpus::get() / 2; + std::env::set_var("CMAKE_BUILD_PARALLEL_LEVEL", cups.to_string()); + let dst = config.build(); + + parse_args_to_rustc(&dst); +} + +fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> std::io::Result<()> { + fs::create_dir_all(&dst)?; + for entry in fs::read_dir(src)? { + let entry = entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?; + } else { + fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; + } + } + Ok(()) +} diff --git a/neptune/libs/ztm b/neptune/libs/ztm new file mode 160000 index 000000000..f3ba7aa56 --- /dev/null +++ b/neptune/libs/ztm @@ -0,0 +1 @@ +Subproject commit f3ba7aa56814c5e23b12bb3f7d951b86977fd260 diff --git a/neptune/mega_app/tunnel/api.js b/neptune/mega_app/tunnel/api.js new file mode 100644 index 000000000..46bc584e9 --- /dev/null +++ b/neptune/mega_app/tunnel/api.js @@ -0,0 +1,439 @@ +export default function ({ app, mesh }) { + var currentListens = [] + var currentTargets = {} + + function allEndpoints() { + return mesh.discover() + } + + function allInbound(ep) { + if (ep === app.endpoint.id) { + return getLocalConfig().then( + config => config.inbound + ) + } else { + return requestPeer(ep, new Message( + { + method: 'GET', + path: `/api/inbound`, + } + )).then(res => res ? JSON.decode(res.body) : null) + } + } + + function allOutbound(ep) { + if (ep === app.endpoint.id) { + return getLocalConfig().then( + config => config.outbound + ) + } else { + return requestPeer(ep, new Message( + { + method: 'GET', + path: `/api/outbound`, + } + )).then(res => res ? JSON.decode(res.body) : null) + } + } + + function getInbound(ep, protocol, name) { + if (ep === app.endpoint.id) { + return getLocalConfig().then( + config => config.inbound.find( + i => i.protocol === protocol && i.name === name + ) || null + ) + } else { + return requestPeer(ep, new Message( + { + method: 'GET', + path: `/api/inbound/${protocol}/${name}`, + } + )).then(res => res?.head?.status === 200 ? JSON.decode(res.body) : null) + } + } + + function getOutbound(ep, protocol, name) { + if (ep === app.endpoint.id) { + return getLocalConfig().then( + config => config.outbound.find( + o => o.protocol === protocol && o.name === name + ) || null + ) + } else { + return requestPeer(ep, new Message( + { + method: 'GET', + path: `/api/outbound/${protocol}/${name}`, + } + )).then(res => res?.head?.status === 200 ? JSON.decode(res.body) : null) + } + } + + function setInbound(ep, protocol, name, listens, exits) { + if (ep === app.endpoint.id) { + exits = exits || [] + checkProtocol(protocol) + checkName(name) + checkListens(listens) + checkExits(exits) + return getLocalConfig().then(config => { + var all = config.inbound + var ent = { protocol, name, listens, exits } + var i = all.findIndex(i => i.protocol === protocol && i.name === name) + if (i >= 0) { + all[i] = ent + } else { + all.push(ent) + } + setLocalConfig(config) + applyLocalConfig(config) + }) + } else { + return requestPeer(ep, new Message( + { + method: 'POST', + path: `/api/inbound/${protocol}/${name}`, + }, + JSON.encode({ listens, exits }) + )) + } + } + + function setOutbound(ep, protocol, name, targets, entrances) { + if (ep === app.endpoint.id) { + entrances = entrances || [] + checkProtocol(protocol) + checkName(name) + checkTargets(targets) + checkEntrances(entrances) + return getLocalConfig().then(config => { + var all = config.outbound + var ent = { protocol, name, targets, entrances } + var i = all.findIndex(o => o.protocol === protocol && o.name === name) + if (i >= 0) { + all[i] = ent + } else { + all.push(ent) + } + setLocalConfig(config) + applyLocalConfig(config) + }) + } else { + return requestPeer(ep, new Message( + { + method: 'POST', + path: `/api/outbound/${protocol}/${name}`, + }, + JSON.encode({ targets, entrances }) + )) + } + } + + function deleteInbound(ep, protocol, name) { + if (ep === app.endpoint.id) { + return getLocalConfig().then(config => { + var all = config.inbound + var i = all.findIndex(i => i.protocol === protocol && i.name === name) + if (i >= 0) { + all.splice(i, 1) + setLocalConfig(config) + } + }) + } else { + return requestPeer(ep, new Message( + { + method: 'DELETE', + path: `/api/inbound/${protocol}/${name}`, + } + )) + } + } + + function deleteOutbound(ep, protocol, name) { + if (ep === app.endpoint.id) { + return getLocalConfig().then(config => { + var all = config.outbound + var i = all.findIndex(o => o.protocol === protocol && o.name === name) + if (i >= 0) { + all.splice(i, 1) + setLocalConfig(config) + } + }) + } else { + return requestPeer(ep, new Message( + { + method: 'DELETE', + path: `/api/outbound/${protocol}/${name}`, + } + )) + } + } + + function getLocalConfig() { + return mesh.read('/local/config.json').then( + data => data ? JSON.decode(data) : { inbound: [], outbound: [] } + ) + } + + function setLocalConfig(config) { + mesh.write('/local/config.json', JSON.encode(config)) + } + + function applyLocalConfig(config) { + currentListens.forEach(l => { + var protocol = l.protocol + var ip = l.ip + var port = l.port + if (!config.inbound.some(i => ( + i.protocol === protocol && + i.listens.some(l => l.ip === ip && l.port === port) + ))) { + pipy.listen(`${ip}:${port}`, protocol, null) + app.log(`Stopped ${protocol} listening ${ip}:${port}`) + } + }) + + currentListens = [] + currentTargets = {} + + config.inbound.forEach(i => { + var protocol = i.protocol + var name = i.name + var listens = i.listens + + var $selectedEP + var connectPeer = pipeline($=>$ + .connectHTTPTunnel( + new Message({ + method: 'CONNECT', + path: `/api/outbound/${protocol}/${name}`, + }) + ).to($=>$ + .muxHTTP().to($=>$ + .pipe(() => mesh.connect($selectedEP)) + ) + ) + .onEnd(() => app.log(`Disconnected from ep ${$selectedEP} for ${protocol}/${name}`)) + ) + + var pass = null + var deny = pipeline($=>$.replaceStreamStart(new StreamEnd)) + + switch (protocol) { + case 'tcp': + pass = connectPeer + break + case 'udp': + pass = pipeline($=>$ + .replaceData(data => new Message(data)) + .encodeWebSocket() + .pipe(connectPeer) + .decodeWebSocket() + .replaceMessage(msg => msg.body) + ) + break + } + + var p = pipeline($=>$ + .onStart(() => + ((i.exits && i.exits.length > 0) + ? Promise.resolve(i.exits) + : mesh.discover().then(list => list.map(ep => ep.id)) + ).then(exits => Promise.all( + exits.map( + id => getOutbound(id, protocol, name).then( + o => o ? { ep: id, ...o } : null + ) + ) + )).then(list => { + list = list.filter(o => (o && ( + !o.entrances || + o.entrances.length === 0 || + o.entrances.includes(app.endpoint.id) + ))) + if (list.length > 0) { + $selectedEP = list[Math.floor(Math.random() * list.length)].ep + app.log(`Connect to ep ${$selectedEP} for ${protocol}/${name}`) + } else { + app.log(`No exit found for ${protocol}/${name}`) + } + return new Data + }) + ) + .pipe(() => $selectedEP ? pass : deny) + ) + + listens.forEach(l => { + try { + pipy.listen(`${l.ip}:${l.port}`, protocol, p) + currentListens.push({ protocol, ip: l.ip, port: l.port }) + app.log(`Started ${protocol} listening ${l.ip}:${l.port}`) + } catch (err) { + app.log(`Cannot open port ${l.ip}:${l.port}: ${err}`) + } + }) + }) + + config.outbound.forEach(o => { + var key = `${o.protocol}/${o.name}` + currentTargets[key] = new algo.LoadBalancer(o.targets) + }) + } + + function requestPeer(ep, req) { + var $response + return pipeline($=>$ + .onStart(req) + .muxHTTP().to($=>$ + .pipe(mesh.connect(ep)) + ) + .replaceMessage(res => { + $response = res + return new StreamEnd + }) + .onEnd(() => $response) + ).spawn() + } + + var matchApiOutbound = new http.Match('/api/outbound/{proto}/{name}') + var response200 = new Message({ status: 200 }) + var response404 = new Message({ status: 404 }) + + var $resource + var $target + var $protocol + + var servePeerInbound = pipeline($=>$ + .acceptHTTPTunnel(req => { + var params = matchApiOutbound(req.head.path) + var proto = params?.proto + var name = params?.name + var key = `${proto}/${name}` + var lb = currentTargets[key] + if (lb) { + $resource = lb.allocate() + var target = $resource.target + var host = target.host + var port = target.port + $target = `${host}:${port}` + $protocol = proto + app.log(`Connect to ${$target} for ${key}`) + return response200 + } + app.log(`No target found for ${key}`) + return response404 + }).to($=>$ + .pipe(() => $protocol, { + 'tcp': ($=>$ + .connect(() => $target) + ), + 'udp': ($=>$ + .decodeWebSocket() + .replaceMessage(msg => msg.body) + .connect(() => $target, { protocol: 'udp' }) + .replaceData(data => new Message(data)) + .encodeWebSocket() + ) + }) + .onEnd(() => { + $resource.free() + app.log(`Disconnected from ${$target}`) + }) + ) + ) + + getLocalConfig().then(applyLocalConfig) + + return { + allEndpoints, + allInbound, + allOutbound, + getInbound, + getOutbound, + setInbound, + setOutbound, + deleteInbound, + deleteOutbound, + servePeerInbound, + } +} + +function checkProtocol(protocol) { + switch (protocol) { + case 'tcp': + case 'udp': + return + default: throw `invalid protocol '${protocol}'` + } +} + +function checkName(name) { + if ( + typeof name !== 'string' || + name.indexOf('/') >= 0 + ) throw `invalid name '${name}'` +} + +function checkIP(ip) { + try { + new IP(ip) + } catch { + throw `malformed IP address '${ip}'` + } +} + +function checkHost(host) { + if ( + typeof host !== 'string' || + host.indexOf(':') >= 0 || + host.indexOf('[') >= 0 || + host.indexOf(']') >= 0 + ) throw `invalid host '${host}'` +} + +function checkPort(port) { + if ( + typeof port !== 'number' || + port < 1 || port > 65535 + ) throw `invalid port number: ${port}` +} + +function checkUUID(uuid) { + if ( + typeof uuid !== 'string' || + uuid.length !== 36 || + uuid.charAt(8) != '-' || + uuid.charAt(13) != '-' || + uuid.charAt(18) != '-' || + uuid.charAt(23) != '-' + ) throw `malformed UUID '${uuid}'` +} + +function checkListens(listens) { + if (!(listens instanceof Array)) throw 'invalid listen array' + listens.forEach(l => { + if (typeof l !== 'object' || l === null) throw 'invalid listen' + checkIP(l.ip) + checkPort(l.port) + }) +} + +function checkTargets(targets) { + if (!(targets instanceof Array)) throw 'invalid target array' + targets.forEach(t => { + if (typeof t !== 'object' || t === null) throw 'invalid target' + checkHost(t.host) + checkPort(t.port) + }) +} + +function checkExits(exits) { + if (!(exits instanceof Array)) throw 'invalid exit array' + exits.forEach(e => checkUUID(e)) +} + +function checkEntrances(entrances) { + if (!(entrances instanceof Array)) throw 'invalid entrance array' + entrances.forEach(e => checkUUID(e)) +} diff --git a/neptune/mega_app/tunnel/cli.js b/neptune/mega_app/tunnel/cli.js new file mode 100644 index 000000000..b1068ab52 --- /dev/null +++ b/neptune/mega_app/tunnel/cli.js @@ -0,0 +1,353 @@ +export default function ({ app, api, utils }) { + return pipeline($=>$ + .onStart(argv => main(argv)) + ) + + function main(argv) { + var buffer = new Data + + function output(str) { + buffer.push(str) + } + + function error(err) { + output('ztm: ') + output(err.message || err.toString()) + output('\n') + } + + function flush() { + return [buffer, new StreamEnd] + } + + var endpoints = null + + function allEndpoints() { + if (endpoints) return Promise.resolve(endpoints) + return api.allEndpoints().then(list => (endpoints = list)) + } + + function selectEndpoint(name) { + if (name) { + return allEndpoints().then(endpoints => { + var ep = endpoints.find(ep => ep.id === name) + if (ep) return ep + var list = endpoints.filter(ep => ep.name === name) + if (list.length === 1) return list[0] + if (list.length === 0) throw `Endpoint '${name}' not found` + throw `Ambiguous endpoint name '${name}'` + }) + } else { + return Promise.resolve(app.endpoint) + } + } + + function lookupEndpointNames(list) { + return allEndpoints().then(endpoints => ( + list.map(id => { + var ep = endpoints.find(ep => ep.id === id) + return ep ? ep.name : id + }) + )) + } + + function lookupEndpointIDs(list) { + return allEndpoints().then(endpoints => ( + list.flatMap(name => { + if (endpoints.some(ep => ep.id === name)) return name + var list = endpoints.filter(ep => ep.name === name) + if (list.length === 1) return list[0].id + if (list.length === 0) throw `Endpoint '${name}' not found` + return list.map(ep => ep.id) + }) + )) + } + + try { + return utils.parseArgv(['ztm tunnel', ...argv], { + help: text => Promise.resolve(output(text + '\n')), + notes: objectTypeNotes + objectNameNotes, + commands: [ + + { + title: 'List objects of the specified type', + usage: 'get ', + options: ` + --mesh Specify a mesh by name + --ep Specify an endpoint by name or UUID + `, + notes: objectTypeNotes, + action: (args) => { + var ep = args['--ep'] + switch (validateObjectType(args, 'get')) { + case 'inbound': return getInbound(ep) + case 'outbound': return getOutbound(ep) + } + } + }, + + { + title: 'Show detailed info of the specified object', + usage: 'describe ', + options: ` + --mesh Specify a mesh by name + --ep Specify an endpoint by name or UUID + `, + notes: objectTypeNotes + objectNameNotes, + action: (args) => { + var ep = args['--ep'] + var name = args[''] + switch (validateObjectType(args, 'describe')) { + case 'inbound': return describeInbound(ep, name) + case 'outbound': return describeOutbound(ep, name) + } + } + }, + + { + title: 'Create an object of the specified type', + usage: 'open ', + options: ` + --mesh Specify a mesh by name + --ep Specify an endpoint by name or UUID + + For inbound end: + + --listen <[ip:]port ...> Set local ports to listen on + --exit Select endpoints as the outbound end + + For outbound end: + + --target Set targets to connect to + --entrance Select endpoints as the inbound end + `, + notes: objectTypeNotes + objectNameNotes, + action: (args) => { + var ep = args['--ep'] + var name = args[''] + switch (validateObjectType(args, 'open')) { + case 'inbound': return openInbound(ep, name, args['--listen'], args['--exit']) + case 'outbound': return openOutbound(ep, name, args['--target'], args['--entrance']) + } + } + }, + + { + title: 'Delete the specified object', + usage: 'close ', + options: ` + --mesh Specify a mesh by name + --ep Specify an endpoint by name or UUID + `, + notes: objectTypeNotes + objectNameNotes, + action: (args) => { + var ep = args['--ep'] + var name = args[''] + switch (validateObjectType(args, 'close')) { + case 'inbound': return closeInbound(ep, name) + case 'outbound': return closeOutbound(ep, name) + } + } + }, + ] + + }).then(flush).catch(err => { + error(err) + return flush() + }) + + } catch (err) { + error(err) + return Promise.resolve(flush()) + } + + function getInbound(epName) { + return selectEndpoint(epName).then(ep => + api.allInbound(ep.id) + ).then(list => ( + Promise.all(list.map(i => + lookupEndpointNames(i.exits || []).then(exits => ({ + ...i, + exits, + })) + )).then(list => + printTable(list, { + 'NAME': i => `${i.protocol}/${i.name}`, + 'LISTENS': i => i.listens.map(l => `${l.ip}:${l.port}`).join(', '), + 'EXITS': i => i.exits.join(', '), + }) + ) + )) + } + + function getOutbound(epName) { + return selectEndpoint(epName).then(ep => + api.allOutbound(ep.id) + ).then(list => ( + Promise.all(list.map(o => + lookupEndpointNames(o.entrances || []).then(entrances => ({ + ...o, + entrances, + })) + )).then(list => + printTable(list, { + 'NAME': o => `${o.protocol}/${o.name}`, + 'TARGETS': o => o.targets.map(t => `${t.host}:${t.port}`).join(', '), + 'ENTRANCES': o => o.entrances.join(', '), + }) + ) + )) + } + + function describeInbound(epName, tunnelName) { + var obj = validateObjectName(tunnelName) + return selectEndpoint(epName).then(ep => + api.getInbound(ep.id, obj.protocol, obj.name).then(obj => { + if (!obj) return + return lookupEndpointNames(obj.exits || []).then(exits => { + output(`Inbound ${obj.protocol}/${obj.name}\n`) + output(`Endpoint: ${ep.name} (${ep.id})\n`) + output(`Listens:\n`) + obj.listens.forEach(l => output(` ${l.ip}:${l.port}\n`)) + output(`Exits:\n`) + exits.forEach(e => output(` ${e}\n`)) + if (exits.length === 0) output(` (all endpoints)\n`) + }) + }) + ) + } + + function describeOutbound(epName, tunnelName) { + var obj = validateObjectName(tunnelName) + return selectEndpoint(epName).then(ep => + api.getOutbound(ep.id, obj.protocol, obj.name).then(obj => { + if (!obj) return + return lookupEndpointNames(obj.entrances || []).then(entrances => { + output(`Outbound ${obj.protocol}/${obj.name}\n`) + output(`Endpoint: ${ep.name} (${ep.id})\n`) + output(`Targets:\n`) + obj.targets.forEach(t => output(` ${t.host}:${t.port}\n`)) + output(`Entrances:\n`) + entrances.forEach(e => output(` ${e}\n`)) + if (entrances.length === 0) output(` (all endpoints)\n`) + }) + }) + ) + } + + function openInbound(epName, tunnelName, listens, exits) { + var obj = validateObjectName(tunnelName) + if (!listens || listens.length === 0) throw `Option '--listen' is required` + listens = listens.map(l => validateHostPort(l)).map(({ host, port }) => ({ ip: host, port })) + return lookupEndpointIDs(exits || []).then( + exits => selectEndpoint(epName).then( + ep => api.setInbound(ep.id, obj.protocol, obj.name, listens, exits) + ) + ) + } + + function openOutbound(epName, tunnelName, targets, entrances) { + var obj = validateObjectName(tunnelName) + if (!targets || targets.length === 0) throw `Option '--target' is required` + targets = targets.map(t => validateHostPort(t)) + return lookupEndpointIDs(entrances || []).then( + entrances => selectEndpoint(epName).then( + ep => api.setOutbound(ep.id, obj.protocol, obj.name, targets, entrances) + ) + ) + } + + function closeInbound(epName, tunnelName) { + var obj = validateObjectName(tunnelName) + return selectEndpoint(epName).then(ep => + api.deleteInbound(ep.id, obj.protocol, obj.name) + ) + } + + function closeOutbound(epName, tunnelName) { + var obj = validateObjectName(tunnelName) + return selectEndpoint(epName).then(ep => + api.deleteOutbound(ep.id, obj.protocol, obj.name) + ) + } + + function validateObjectType(args, command) { + var ot = args[''] + switch (ot) { + case 'inbound': + case 'in': + return 'inbound' + case 'outbound': + case 'out': + return 'outbound' + default: throw `Invalid object type '${ot}'. Type 'ztm tunnel ${command} for help.'` + } + } + + function validateObjectName(name) { + if (!name) return + var segs = name.split('/') + if (segs.length === 2) { + var protocol = segs[0] + if (protocol === 'tcp' || protocol === 'udp') { + if (segs[1]) { + return { protocol, name: segs[1] } + } + } + } + throw `Invalid inbound/outbound name '${name}'` + } + + function validateHostPort(addr) { + var i = addr.lastIndexOf(':') + if (i >= 0) { + var host = addr.substring(0,i) + var port = addr.substring(i+1) + } else { + var host = '' + var port = addr + } + port = Number.parseInt(port) + if (Number.isNaN(port)) throw `Invalid port number in '${addr}'` + if (!host) host = '127.0.0.1' + return { host, port } + } + + function printTable(data, columns, indent) { + var head = ' '.repeat(indent || 0) + var cols = Object.entries(columns) + var colHeaders = cols.map(i => i[0]) + var colFormats = cols.map(i => i[1]) + var colSizes = colHeaders.map(name => name.length) + var rows = data.map(row => colFormats.map( + (format, i) => { + var v = format(row).toString() + colSizes[i] = Math.max(colSizes[i], v.length) + return v + } + )) + output(head) + colHeaders.forEach((name, i) => output(name.padEnd(colSizes[i]) + ' ')) + output('\n') + rows.forEach(row => { + output(head) + row.forEach((v, i) => output(v.padEnd(colSizes[i]) + ' ')) + output('\n') + }) + } + } +} + +var objectTypeNotes = ` + Object Types: + + inbound in Inbound end of a tunnel + outbound out Outbound end of a tunnel +` + +var objectNameNotes = ` + Object Names: + + tcp/ Name for a TCP tunnel + udp/ Name for a UDP tunnel +` diff --git a/neptune/mega_app/tunnel/main.js b/neptune/mega_app/tunnel/main.js new file mode 100644 index 000000000..3c2e6b39d --- /dev/null +++ b/neptune/mega_app/tunnel/main.js @@ -0,0 +1,142 @@ +import initAPI from './api.js' +import initCLI from './cli.js' + +export default function ({ app, mesh, utils }) { + var api = initAPI({ app, mesh }) + var cli = initCLI({ app, mesh, utils, api }) + + var $ctx + + var gui = new http.Directory(os.path.join(app.root, 'gui')) + var response = utils.createResponse + var responder = utils.createResponder + + var serveUser = utils.createServer({ + '/cli': { + 'CONNECT': utils.createCLIResponder(cli), + }, + + '/api/endpoints': { + 'GET': responder(() => api.allEndpoints().then( + ret => ret ? response(200, ret) : response(404) + )) + }, + + '/api/endpoints/{ep}/inbound': { + 'GET': responder(({ ep }) => api.allInbound(ep).then( + ret => ret ? response(200, ret) : response(404) + )) + }, + + '/api/endpoints/{ep}/outbound': { + 'GET': responder(({ ep }) => api.allOutbound(ep).then( + ret => ret ? response(200, ret) : response(404) + )) + }, + + '/api/endpoints/{ep}/inbound/{proto}/{name}': { + 'GET': responder(({ ep, proto, name }) => { + return api.getInbound(ep, proto, name).then( + ret => ret ? response(200, ret) : response(404) + ) + }), + + 'POST': responder(({ ep, proto, name }, req) => { + var obj = JSON.decode(req.body) + var listens = obj.listens + var exits = obj.exits || null + return api.setInbound(ep, proto, name, listens, exits).then(response(201)) + }), + + 'DELETE': responder(({ ep, proto, name }) => { + return api.deleteInbound(ep, proto, name).then(response(204)) + }), + + }, + + '/api/endpoints/{ep}/outbound/{proto}/{name}': { + 'GET': responder(({ ep, proto, name }) => { + return api.getOutbound(ep, proto, name).then( + ret => ret ? response(200, ret) : response(404) + ) + }), + + 'POST': responder(({ ep, proto, name }, req) => { + var obj = JSON.decode(req.body) + var targets = obj.targets + var entrances = obj.entrances || null + return api.setOutbound(ep, proto, name, targets, entrances).then(response(201)) + }), + + 'DELETE': responder(({ ep, proto, name }) => { + return api.deleteOutbound(ep, proto, name).then(response(204)) + }), + }, + + '*': { + 'GET': responder((_, req) => { + return Promise.resolve(gui.serve(req) || response(404)) + }) + }, + }) + + var servePeer = utils.createServer({ + '/api/inbound': { + 'GET': responder(() => api.allInbound(app.endpoint.id).then( + ret => ret ? response(200, ret) : response(404) + )) + }, + + '/api/outbound': { + 'GET': responder(() => api.allOutbound(app.endpoint.id).then( + ret => ret ? response(200, ret) : response(404) + )) + }, + + '/api/inbound/{proto}/{name}': { + 'GET': responder(({ proto, name }) => api.getInbound(app.endpoint.id, proto, name).then( + ret => ret ? response(200, ret) : response(404) + )), + + 'POST': responder(({ proto, name }, req) => { + var obj = JSON.decode(req.body) + var listens = obj.listens + var exits = obj.exits || null + return api.setInbound(app.endpoint.id, proto, name, listens, exits).then(response(201)) + }), + + 'DELETE': responder(({ proto, name }) => { + return api.deleteInbound(app.endpoint.id, proto, name).then(response(204)) + }), + }, + + '/api/outbound/{proto}/{name}': { + 'GET': responder(({ proto, name }) => api.getOutbound(app.endpoint.id, proto, name).then( + ret => ret ? response(200, ret) : response(404) + )), + + 'POST': responder(({ proto, name }, req) => { + var obj = JSON.decode(req.body) + var targets = obj.targets + var entrances = obj.entrances || null + return api.setOutbound(app.endpoint.id, proto, name, targets, entrances).then(response(201)) + }), + + 'DELETE': responder(({ proto, name }) => { + return api.deleteOutbound(app.endpoint.id, proto, name).then(response(204)) + }), + + 'CONNECT': api.servePeerInbound, + }, + }) + + return pipeline($=>$ + .onStart(c => void ($ctx = c)) + .pipe(() => { + switch ($ctx.source) { + case 'user': return serveUser + case 'peer': return servePeer + } + }) + ) +} diff --git a/neptune/src/lib.rs b/neptune/src/lib.rs new file mode 100644 index 000000000..18b1a2554 --- /dev/null +++ b/neptune/src/lib.rs @@ -0,0 +1,108 @@ +/// a test demo for pipy +use libc::{c_char, c_int}; +use std::ffi::CString; +use std::thread; + +#[link(name = "pipy", kind = "dylib")] +extern "C" { + pub fn pipy_main(argc: c_int, argv: *const *const c_char) -> c_int; + + pub fn pipy_exit(force: c_int); +} + +/// start ztm agent, like run pipy repo://ztm/agent --database=database --listen=0.0.0.0:listen_port +/// ! only support to start one agent or one hub at one process +pub fn start_agent(database: &str, listen_port: u16) { + let database = database.to_string(); + tracing::info!("start pipy with port: {}", listen_port); + let args = [ + CString::new("ztm-pipy").unwrap(), + CString::new("repo://ztm/agent").unwrap(), + CString::new("--args").unwrap(), + CString::new(format!("--database={}", database)).unwrap(), + CString::new(format!("--listen=0.0.0.0:{}", listen_port)).unwrap(), + ]; + let c_args: Vec<*const c_char> = args.iter().map(|arg| arg.as_ptr()).collect(); + unsafe { + pipy_main(c_args.len() as c_int, c_args.as_ptr()); + } + thread::sleep(std::time::Duration::from_secs(1)); // wait for pipy to start +} + +/// start ztm hub, like run pipy repo://ztm/hub --listen=0.0.0.0:listen_port --name=name --ca=ca +/// ! only support to start one agent or one hub at one process +pub fn start_hub(listen_port: u16, name: Vec, ca: &str) { + let _ = name; // TODO: ignore name + tracing::info!("start pipy with port: {}", listen_port); + let args = [ + CString::new("ztm-pipy").unwrap(), + CString::new("repo://ztm/hub").unwrap(), + CString::new("--args").unwrap(), + CString::new(format!("--listen=0.0.0.0:{}", listen_port)).unwrap(), + CString::new(format!("--ca={}", ca)).unwrap(), + ]; + let c_args: Vec<*const c_char> = args.iter().map(|arg| arg.as_ptr()).collect(); + unsafe { + pipy_main(c_args.len() as c_int, c_args.as_ptr()); + } +} + +/// exit ztm agent or hub +pub fn exit_ztm() { + unsafe { + pipy_exit(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn test_start_agent() { + let port = 7776; + start_agent("test.db", port); + thread::sleep(std::time::Duration::from_secs(1)); + + let resp = reqwest::get(format!("http://127.0.0.1:{}/api/version", port)) + .await + .unwrap(); + tracing::debug!("resp: {:?}", resp); + assert!(resp.status().is_success()); + tracing::info!("ztm agent start success"); + + exit_ztm(); + let resp = reqwest::get(format!("http://127.0.0.1:{}/api/version", port)) + .await + .unwrap(); + tracing::debug!("resp: {:?}", resp); + assert!(resp.status().as_u16() == 502); // 502 + tracing::info!("ztm agent exit success"); + } + + #[tokio::test] + #[should_panic] + /// didn't support multiple agent + async fn test_start_multiple_agent() { + let port1 = 7777; + let port2 = 7778; + start_agent("test1.db", port1); + + let resp = reqwest::get(format!("http://0.0.0.0:{}/api/version", port1)) + .await + .unwrap(); + assert!(resp.status().is_success()); + + start_agent("test2.db", port2); + let resp = reqwest::get(format!("http://0.0.0.0:{}/api/version", port2)) + .await + .unwrap(); + assert!(resp.status().is_success()); + + exit_ztm(); + + let resp = reqwest::get(format!("http://http://0.0.0.0:{}/api/version", port1)) + .await + .unwrap(); + assert!(resp.status().as_u16() == 502); // 502 + } +} diff --git a/neptune/src/main.rs b/neptune/src/main.rs new file mode 100644 index 000000000..6497593d7 --- /dev/null +++ b/neptune/src/main.rs @@ -0,0 +1,15 @@ +use libc::{c_char, c_int}; +/// a test demo for pipy +use std::ffi::CString; + +fn main() { + let args: Vec = std::env::args() + .map(|arg| CString::new(arg).unwrap()) + .collect(); + + let c_args: Vec<*const c_char> = args.iter().map(|arg| arg.as_ptr()).collect(); + + unsafe { + neptune::pipy_main(c_args.len() as c_int, c_args.as_ptr()); + } +} diff --git a/sql/postgres/pg_20240205__init.sql b/sql/postgres/pg_20240205__init.sql index aad29dd22..084227256 100644 --- a/sql/postgres/pg_20240205__init.sql +++ b/sql/postgres/pg_20240205__init.sql @@ -224,7 +224,7 @@ CREATE TABLE IF NOT EXISTS "lfs_split_relations" ( "offset" BIGINT NOT NULL, "size" BIGINT NOT NULL, PRIMARY KEY ("ori_oid", "sub_oid", "offset") -) +); CREATE TABLE IF NOT EXISTS "ztm_node" (