diff --git a/aries/src/main.rs b/aries/src/main.rs index 86bda0973..cdeffde2e 100644 --- a/aries/src/main.rs +++ b/aries/src/main.rs @@ -20,7 +20,6 @@ pub mod service; #[tokio::main] async fn main() { - ctrlc::set_handler(move || { tracing::info!("Received Ctrl-C signal, exiting..."); std::process::exit(0); diff --git a/gemini/src/ztm/agent.rs b/gemini/src/ztm/agent.rs index 6dcb04057..0048b1701 100644 --- a/gemini/src/ztm/agent.rs +++ b/gemini/src/ztm/agent.rs @@ -39,6 +39,7 @@ pub struct Agent { pub struct ZTMEndPoint { pub id: String, pub username: String, + pub name: String, pub online: bool, #[serde(rename = "isLocal")] pub is_local: bool, @@ -216,7 +217,7 @@ impl ZTMAgent for LocalZTMAgent { match self.get_ztm_endpoints().await { Ok(ep_list) => { for ele in ep_list { - if ele.online && ele.username == peer_id { + if ele.online && ele.name == peer_id { return Ok(ele); } } @@ -434,24 +435,25 @@ pub async fn run_ztm_client( agent: LocalZTMAgent, http_port: u16, ) { - let name = peer_id.clone(); - let url = format!("{bootstrap_node}/api/v1/certificate?name={name}"); + //TODO ztm permit control? + // let name = peer_id.clone(); + let url = format!("{bootstrap_node}/api/v1/certificate?name=root"); let request_result = reqwest::get(url).await; let response_text = match handle_response(request_result).await { Ok(s) => s, Err(s) => { - tracing::error!("GET {bootstrap_node}/api/v1/certificate?name={name} failed,{s}"); + tracing::error!("GET {bootstrap_node}/api/v1/certificate?name=root failed,{s}"); return; } }; - let permit: ZTMUserPermit = match serde_json::from_slice(response_text.as_bytes()) { + let mut permit: ZTMUserPermit = match serde_json::from_slice(response_text.as_bytes()) { Ok(p) => p, Err(e) => { tracing::error!("{}", e); return; } }; - + permit.agent.name = peer_id.clone(); // 2. join ztm mesh let mesh = match agent.connect_ztm_hub(permit.clone()).await { Ok(m) => m, diff --git a/neptune/hub/cmdline.js b/neptune/hub/cmdline.js index 9f389f65c..3127bac9e 100644 --- a/neptune/hub/cmdline.js +++ b/neptune/hub/cmdline.js @@ -83,8 +83,8 @@ export default function (argv, { commands, notes, help, fallback }) { var opts = parseOptions(cmd.options, rest) } catch (err) { - var command = pattern.slice(0, best).join(' ') - throw `${err}. Type '${program} help ${command}' for help info.` + var command = ['help', ...pattern.slice(0, best)].join(' ') + throw `${err}. Type '${program} ${command}' for help info.` } return cmd.action({ ...args, ...opts }) @@ -92,7 +92,7 @@ export default function (argv, { commands, notes, help, fallback }) { function matchCommand(pattern, argv) { var i = argv.findIndex( - (arg, i) => arg.startsWith('-') || arg !== pattern[i] + (arg, i) => (arg.length > 1 && arg.startsWith('-')) || arg !== pattern[i] ) return i < 0 ? argv.length : i } @@ -101,7 +101,7 @@ function parseCommand(pattern, argv, rest) { var values = {} var i = argv.findIndex((arg, i) => { var tok = pattern[i] - if (arg.startsWith('-')) return true + if (arg.length > 1 && arg.startsWith('-')) return true if (!tok) throw `Excessive positional argument: ${arg}` if (tok.startsWith('[') || tok.startsWith('<')) { values[tok] = arg @@ -131,7 +131,8 @@ function parseOptions(format, argv) { if (t.endsWith(',')) t = t.substring(0, t.length - 1) aliases.push(t) } else { - if (t.endsWith('...]') || t.endsWith('...>')) type = 'array' + if (t === '...') type = 'remainder array' + else if (t.endsWith('...]') || t.endsWith('...>')) type = 'array' else if (t.startsWith('[')) type = 'optional string' else if (t.startsWith('<')) type = 'string' else type = 'boolean' @@ -147,7 +148,11 @@ function parseOptions(format, argv) { argv.forEach(arg => { if (currentOption) { - if (arg.startsWith('-')) { + if (options[currentOption]?.type === 'remainder array') { + addOption(currentOption, arg) + return + } + if (arg.length > 1 && arg.startsWith('-')) { endOption(currentOption) currentOption = undefined } else { @@ -157,7 +162,7 @@ function parseOptions(format, argv) { } if (arg.startsWith('--')) { currentOption = arg - } else if (arg.startsWith('-')) { + } else if (arg.length > 1 && arg.startsWith('-')) { if (arg.length === 2) { currentOption = arg } else { @@ -182,6 +187,7 @@ function parseOptions(format, argv) { option.value = value break case 'array': + case 'remainder array': option.value ??= [] option.value.push(value) break @@ -220,18 +226,15 @@ function parseOptions(format, argv) { function tokenize(str) { var tokens = str.split(' ').reduce( (a, b) => { - if (typeof a === 'string') a = [a] var last = a.pop() - if (last.startsWith('<')) { - a.push(`${last} ${b}`) - } else if (last.startsWith('[')) { + if (last && (last.startsWith('<') || last.startsWith('['))) { a.push(`${last} ${b}`) } else { a.push(last, b) } if (b.endsWith('>') || b.endsWith(']')) a.push('') return a - } + }, [] ) return tokens instanceof Array ? tokens.filter(t => t) : [tokens] } @@ -239,6 +242,7 @@ function tokenize(str) { function stripIndentation(s, indent) { var lines = s.split('\n') if (lines[0].trim() === '') lines.shift() + if (lines[lines.length - 1].trim() === '') lines.pop() var depth = lines[0].length - lines[0].trimStart().length var padding = ' '.repeat(indent || 0) return lines.map(l => padding + l.substring(depth)).join('\n') diff --git a/neptune/hub/db.js b/neptune/hub/db.js index 61e6c354d..2ea3b740e 100644 --- a/neptune/hub/db.js +++ b/neptune/hub/db.js @@ -16,6 +16,16 @@ function open(pathname) { data TEXT NOT NULL ) `) + + db.exec(` + CREATE TABLE IF NOT EXISTS files ( + path TEXT PRIMARY KEY, + hash TEXT NOT NULL, + size INTEGER NOT NULL, + time REAL NOT NULL, + since REAL NOT NULL + ) + `) } function getCert(name) { @@ -70,6 +80,55 @@ function delKey(name) { .exec() } +function recordToFile(rec) { + return { + pathname: rec.path, + hash: rec.hash, + size: +rec.size, + time: rec.time, + since: rec.since, + } +} + +function allFiles() { + return ( + db.sql('SELECT * FROM files') + .exec() + .map(recordToFile) + ) +} + +function getFile(pathname) { + return ( + db.sql('SELECT * FROM files WHERE path = ?') + .bind(1, pathname) + .exec() + .map(recordToFile)[0] + ) +} + +function setFile(pathname, file) { + var obj = getFile(pathname) + if (obj) { + Object.assign(obj, file) + db.sql('UPDATE files SET hash = ?, size = ?, time = ?, since = ? WHERE path = ?') + .bind(1, obj.hash) + .bind(2, obj.size) + .bind(3, obj.time) + .bind(4, obj.since) + .bind(5, pathname) + .exec() + } else { + db.sql('INSERT INTO files(path, hash, size, time, since) VALUES(?, ?, ?, ?, ?)') + .bind(1, pathname) + .bind(2, file.hash) + .bind(3, file.size) + .bind(4, file.time) + .bind(5, file.since) + .exec() + } +} + export default { open, getCert, @@ -78,4 +137,7 @@ export default { getKey, setKey, delKey, + allFiles, + getFile, + setFile, } diff --git a/neptune/hub/main.js b/neptune/hub/main.js index d6f4e2c15..9ff6be5c8 100755 --- a/neptune/hub/main.js +++ b/neptune/hub/main.js @@ -27,10 +27,6 @@ var routes = Object.entries({ 'GET': () => getFileData, }, - '/api/endpoints/{ep}/services': { - 'GET': () => getServices, - }, - '/api/endpoints/{ep}/apps/{app}': { 'CONNECT': () => connectApp, }, @@ -39,10 +35,6 @@ var routes = Object.entries({ 'CONNECT': () => connectApp, }, - '/api/endpoints/{ep}/services/{proto}/{svc}': { - 'CONNECT': () => connectService, - }, - '/api/filesystem': { 'GET': () => getFilesystem, 'POST': () => findCurrentEndpointSession() ? postFilesystem : noSession, @@ -52,27 +44,6 @@ var routes = Object.entries({ 'GET': () => getFileInfo, }, - '/api/apps': { - 'POST': () => findCurrentEndpointSession() ? postAppStates : noSession, - }, - - '/api/apps/{app}': { - 'GET': () => getAppState, - }, - - '/api/apps/{provider}/{app}': { - 'GET': () => getAppState, - }, - - '/api/services': { - 'GET': () => getServices, - 'POST': () => findCurrentEndpointSession() ? postServices : noSession, - }, - - '/api/services/{proto}/{svc}': { - 'GET': () => getService, - }, - '/api/forward/{ep}/*': { 'GET': () => forwardRequest, 'POST': () => forwardRequest, @@ -91,9 +62,35 @@ var routes = Object.entries({ } ) +// +// endpoints[uuid] = { +// id: 'uuid', +// name: 'ep-xxx', +// username: 'root', +// ip: 'x.x.x.x', +// port: 12345, +// via: '127.0.0.1:8888', +// hubs: ['x.x.x.x:8888'], +// heartbeat: 1723012345678, +// isConnected: true, +// } +// + var endpoints = {} var sessions = {} +// +// files[pathname] = { +// '#': '012345678abcdef', // hash +// '$': 12345, // size +// 'T': 1789012345678, // time +// '+': 1789012345678, // since +// '@': [], // sources +// } +// + +var files = {} + var caCert = null var myCert = null var myKey = null @@ -170,6 +167,7 @@ function endpointName(id) { } function isEndpointOnline(ep) { + if (!ep) return false if (!sessions[ep.id]?.size) return false if (ep.heartbeat + 30*1000 < Date.now()) return false return true @@ -187,9 +185,11 @@ var $hubSelected = null var $pingID function start(listen) { + db.allFiles().forEach(f => { + files[f.pathname] = makeFileInfo(f.hash, f.size, f.time, f.since) + }) - - var username = "hub0" + var username = "root" var caAgent = new http.Agent('localhost:9999') caAgent.request('GET', '/api/certificates/ca').then( function (res) { @@ -277,20 +277,31 @@ function start(listen) { console.info('Hub started at', listen) - } - ) + }) - function clearOutdatedEndpoints() { - Object.values(endpoints).filter(ep => isEndpointOutdated(ep)).forEach( - (ep) => { - console.info(`Endpoint ${ep.name} (uuid = ${ep.id}) outdated`) - if (sessions[ep.id]?.size === 0) { - delete endpoints[ep.id] + new Timeout(60).wait().then(() => { + var outdated = [] + Object.values(endpoints).filter(ep => isEndpointOutdated(ep)).forEach( + (ep) => { + console.info(`Endpoint ${ep.name} (uuid = ${ep.id}) outdated`) + if (sessions[ep.id]?.size === 0) { + outdated.push(ep.id) + delete endpoints[ep.id] + } } + ) + if (outdated.length > 0) { + outdated = Object.fromEntries(outdated.map(ep => [ep, true])) + Object.values(files).forEach(file => { + var sources = file['@'] + if (sources.some(ep => ep in outdated)) { + file['@'] = sources.filter(ep => !(ep in outdated)) + } + }) } - ) - new Timeout(60).wait().then(clearOutdatedEndpoints) + clearOutdatedEndpoints() + }) } clearOutdatedEndpoints() @@ -366,15 +377,46 @@ var getEndpoint = pipeline($=>$ var getFilesystem = pipeline($=>$ .replaceData() .replaceMessage( - function () { - var fs = {} - Object.values(endpoints).forEach(ep => { - if (!ep.files) return - ep.files.forEach(f => { - updateFileInfo(fs, f, ep.id) - }) - }) - return response(200, fs) + function (req) { + var url = new URL(req.head.path) + var since = url.searchParams.get('since') + if (since) { + var since = Number.parseFloat(since) + var until = Date.now() + return new Timeout(1.5).wait().then( + () => response(200, Object.fromEntries( + Object.entries(files).filter( + ([_, v]) => { + var t = v['+'] + return since < t && t <= until + } + ).map( + ([k, v]) => [ + k, { + '#': v['#'], + '$': v['$'], + 'T': v['T'], + '+': v['+'], + } + ] + ) + )) + ) + } else { + return response(200, Object.fromEntries( + Object.entries(files).filter( + ([_, v]) => (v['$'] >= 0) + ).map( + ([k, v]) => [ + k, { + '#': v['#'], + '$': v['$'], + 'T': v['T'], + } + ] + ) + )) + } } ) ) @@ -383,23 +425,20 @@ var postFilesystem = pipeline($=>$ .replaceMessage( function (req) { var body = JSON.decode(req.body) - $endpoint.files = Object.entries(body).map( - ([k, v]) => ({ - pathname: k, - time: v['T'], - hash: v['#'], - size: v['$'], - }) + var username = $endpoint.username + var prefixUser = `/users/${username}/` + var prefixShared = `/shared/${username}` + var matchAppUser = new http.Match(`/apps/{provider}/{appname}/users/${username}/*`) + var matchAppShared = new http.Match(`/apps/{provider}/{appname}/shared/${username}/*`) + var canUpdate = (path) => ( + path.startsWith(prefixUser) || + path.startsWith(prefixShared) || + matchAppUser(path) || + matchAppShared(path) + ) + Object.entries(body).map( + ([k, v]) => updateFileInfo(k, v, $endpoint.id, canUpdate(k)) ) - return new Message({ status: 201 }) - } - ) -) - -var postAppStates = pipeline($=>$ - .replaceMessage( - function (req) { - $endpoint.apps = JSON.decode(req.body) return new Message({ status: 201 }) } ) @@ -409,18 +448,12 @@ var getFileInfo = pipeline($=>$ .replaceData() .replaceMessage( function () { - var fs = {} var pathname = '/' + $params['*'] - Object.values(endpoints).forEach(ep => { - if (!ep.files) return - ep.files.forEach(f => { - if (f.pathname === pathname) { - updateFileInfo(fs, f, ep.id) - } - }) - }) - var info = fs[pathname] - return info ? response(200, info) : response(404) + var info = files[pathname] + if (!info || info['$'] < 0) return response(404) + var sources = info['@'] + if (sources) info['@'] = sources.filter(ep => isEndpointOnline(endpoints[ep])) + return response(200, info) } ) ) @@ -442,110 +475,11 @@ var getFileData = pipeline($=>$ ) ) -var getAppState = pipeline($=>$ - .replaceData() - .replaceMessage( - function () { - var provider = $params.provider - var name = $params.app - var runners = [] - Object.values(endpoints).forEach(ep => { - if (!isEndpointOnline(ep)) return - if (!ep.apps) return - var app = ep.apps.find( - a => a.name === name && (!provider || a.provider === provider) - ) - if (app) { - runners.push({ - id: ep.id, - name: ep.name, - username: app.username, - }) - } - }) - return response(200, { name, provider, endpoints: runners }) - } - ) -) - -var getServices = pipeline($=>$ - .replaceData() - .replaceMessage( - function () { - var services = [] - var collect = (ep) => { - ep.services?.forEach?.( - function (svc) { - var name = svc.name - var protocol = svc.protocol - var s = services.find(s => s.name === name && s.protocol === protocol) - if (!s) services.push(s = { name, protocol, endpoints: [] }) - s.endpoints.push({ id: ep.id, name: ep.name, users: svc.users }) - } - ) - } - if ($params.ep) { - var ep = endpoints[$params.ep] - if (ep && isEndpointOnline(ep)) collect(ep) - } else { - Object.values(endpoints).filter(isEndpointOnline).forEach(collect) - } - return response(200, services) - } - ) -) - -var postServices = pipeline($=>$ - .replaceMessage( - function (req) { - var body = JSON.decode(req.body) - var time = body.time - var last = $endpoint.servicesUpdateTime - if (!last || last <= time) { - var services = body.services - var oldList = $endpoint.services || [] - var newList = services instanceof Array ? services : [] - var who = endpointName($endpoint.id) - console.info(`Received service list (length = ${newList.length}) from ${who}`) - newList.forEach(({ name, protocol }) => { - if (!oldList.some(s => s.name === name && s.protocol === protocol)) { - console.info(`Service ${name} published by ${who}`) - } - }) - oldList.forEach(({ name, protocol }) => { - if (!newList.some(s => s.name === name && s.protocol === protocol)) { - console.info(`Service ${name} deleted by ${who}`) - } - }) - $endpoint.services = newList - $endpoint.servicesUpdateTime = time - } - return new Message({ status: 201 }) - } - ) -) - -var getService = pipeline($=>$ - .replaceData() - .replaceMessage( - function () { - var name = $params.svc - var protocol = $params.proto - var providers = Object.values(endpoints).filter( - ep => isEndpointOnline(ep) && ep.services.some(s => s.name === name && s.protocol === protocol) - ) - if (providers.length === 0) return response(404) - return response(200, { - name, - protocol, - endpoints: providers.map(({ id, name }) => ({ id, name })), - }) - } - ) -) - var muxToAgent = pipeline($=>$ - .muxHTTP(() => $hubSelected, { version: 2 }).to($=>$ + .muxHTTP(() => $hubSelected, { + version: 2, + ping: () => new Timeout(10).wait().then(new Data), + }).to($=>$ .swap(() => $hubSelected) ) ) @@ -601,31 +535,6 @@ var connectApp = pipeline($=>$ ) ) -var connectService = pipeline($=>$ - .acceptHTTPTunnel( - function () { - var svc = $params.svc - var proto = $params.proto - var id = $params.ep - var ep = endpoints[id] - if (!ep) return response(404, 'Endpoint not found') - if (!canConnect($ctx.username, ep, proto, svc)) return response(403) - if (!ep.services.some(s => s.name === svc && s.protocol === proto)) return response(404, 'Service not found') - sessions[id]?.forEach?.(h => $hubSelected = h) - if (!$hubSelected) return response(404, 'Agent not found') - console.info(`Forward to ${svc} at ${endpointName(id)}`) - return response(200) - } - ).to($=>$ - .connectHTTPTunnel( - () => new Message({ - method: 'CONNECT', - path: `/api/services/${$params.proto}/${$params.svc}`, - }) - ).to(muxToAgent) - ) -) - var forwardRequest = pipeline($=>$ .pipe( function (req) { @@ -720,55 +629,58 @@ function findCurrentEndpointSession() { ip: $ctx.ip, port: $ctx.port, via: $ctx.via, - services: [], hubs: [...myNames] } + } else { + $endpoint.username = $ctx.username } $endpoint.isConnected = true return true } -function updateFileInfo(fs, f, ep) { - var e = (fs[f.pathname] ??= { - 'T': 0, - '$': 0, - '#': null, - '@': null, - }) - var t1 = e['T'] - var h1 = e['#'] - var t2 = f.time - var h2 = f.hash - if (h2 === h1) { - e['@'].push(ep) - e['T'] = Math.max(t1, t2) - } else if (t2 > t1) { - e['#'] = h2 - e['$'] = f.size - e['@'] = [ep] - e['T'] = t2 +function makeFileInfo(hash, size, time, since) { + return { + '#': hash, + '$': size, + 'T': time, + '+': since, + '@': [], } } -function canSee(username, ep) { - if (username === 'root') return true - if (username === ep.username) return true - return false +function updateFileInfo(pathname, f, ep, update) { + var e = files[pathname] + if (e || update) { + if (!e) e = files[pathname] = makeFileInfo('', 0, 0, 0) + var t1 = e['T'] + var h1 = e['#'] + var t2 = f['T'] + var h2 = f['#'] + if (h2 === h1) { + var sources = e['@'] + if (!sources.includes(ep)) sources.push(ep) + if (update && t2 > t1) { + e['T'] = t2 + e['+'] = Date.now() + } + } else if (t2 > t1 && update) { + e['#'] = h2 + e['$'] = f['$'] + e['T'] = t2 + e['+'] = Date.now() + e['@'] = [ep] + db.setFile(pathname, { + hash: h2, + size: e['$'], + time: t2, + since: e['+'], + }) + } + } } function canOperate(username, ep) { - if (username === 'root') return true - if (username === ep.username) return true - return false -} - -function canConnect(username, ep, proto, svc) { - if (username === 'root') return true - if (username === ep.username) return true - var s = ep.services.find(({ protocol, name }) => (protocol === proto && name === svc)) - if (!s) return false - if (!s.users) return true - return s.users.includes(username) + return (username === ep.username) } function response(status, body) { diff --git a/neptune/libs/ztm b/neptune/libs/ztm index 2e0ac3ed1..99840a0fe 160000 --- a/neptune/libs/ztm +++ b/neptune/libs/ztm @@ -1 +1 @@ -Subproject commit 2e0ac3ed15b8374aeb8227b6a65a83dc6f2a97dc +Subproject commit 99840a0fe0ab04011cda41689038f7e90564e155 diff --git a/neptune/src/lib.rs b/neptune/src/lib.rs index 3eda46ce7..3fb51e93b 100644 --- a/neptune/src/lib.rs +++ b/neptune/src/lib.rs @@ -13,16 +13,16 @@ extern "C" { /// start ztm agent, like run pipy repo://ztm/agent --database=database --listen=0.0.0.0:listen_port /// ! only support to start one agent or one hub at one process pub fn start_agent(database: &str, listen_port: u16) { - let database = database.to_string(); + let _database = database.to_string(); tracing::info!("start pipy with port: {}", listen_port); let args = [ CString::new("ztm-pipy").unwrap(), CString::new("repo://ztm/agent").unwrap(), - CString::new("--reuse-port").unwrap(), + // CString::new("--reuse-port").unwrap(), CString::new("--args").unwrap(), - // CString::new(format!("--database={}", database)).unwrap(), - CString::new(format!("--data={}", database)).unwrap(), - CString::new(format!("--listen=0.0.0.0:{}", listen_port)).unwrap(), + CString::new("--data").unwrap(), + CString::new(database).unwrap(), + // CString::new(format!("--listen 0.0.0.0:{}", listen_port)).unwrap(), ]; let c_args: Vec<*const c_char> = args.iter().map(|arg| arg.as_ptr()).collect(); unsafe {