From 2682fbce5552f12f827aa97398c6bcce6765ef01 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Thu, 10 Jun 2021 13:05:51 -0400 Subject: [PATCH 01/24] Reload TLS context on adding new cert Signed-off-by: Peter Broadhurst --- src/app.ts | 10 ++-------- src/lib/cert.ts | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/app.ts b/src/app.ts index b2c0438..8bbcdf5 100644 --- a/src/app.ts +++ b/src/app.ts @@ -19,7 +19,7 @@ import https from 'https'; import http from 'http'; import WebSocket from 'ws'; import { init as initConfig, config } from './lib/config'; -import { init as initCert, key, cert, ca } from './lib/cert'; +import { init as initCert, key, cert, ca, genTLSContext } from './lib/cert'; import { createLogger, LogLevelString } from 'bunyan'; import * as utils from './lib/utils'; import { router as apiRouter } from './routers/api'; @@ -46,13 +46,7 @@ export const start = async () => { const apiServer = http.createServer(apiApp); const p2pApp = express(); - const p2pServer = https.createServer({ - key, - cert, - ca, - rejectUnauthorized: true, - requestCert: true, - }, p2pApp); + const p2pServer = https.createServer(genTLSContext(), p2pApp); const wss = new WebSocket.Server({ server: apiServer, verifyClient: (info, cb) => { diff --git a/src/lib/cert.ts b/src/lib/cert.ts index 03a23f7..5e6d471 100644 --- a/src/lib/cert.ts +++ b/src/lib/cert.ts @@ -18,6 +18,7 @@ import * as utils from '../lib/utils'; import { promises as fs } from 'fs'; import path from 'path'; import { createLogger, LogLevelString } from 'bunyan'; +import { Server } from 'https'; const log = createLogger({ name: 'lib/certs.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); @@ -42,3 +43,20 @@ export const loadCAs = async () => { } log.debug(`Loaded ${ca.length} peer certificate(s)`); }; + + +export const genTLSContext = () => { + return { + key, + cert, + ca, + rejectUnauthorized: true, + requestCert: true, + } +} + +export const resetCAs = async (server: Server) => { + loadCAs() + // The most recent context wins (per the Node.js spec), so to get a reload we just add a wildcard context + server.addContext("*", genTLSContext()) +}; From 261367f86f6cd7084549054e5c5873bba80f9bb1 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Thu, 10 Jun 2021 13:07:27 -0400 Subject: [PATCH 02/24] Lint Signed-off-by: Peter Broadhurst --- src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index 8bbcdf5..a806fa5 100644 --- a/src/app.ts +++ b/src/app.ts @@ -19,7 +19,7 @@ import https from 'https'; import http from 'http'; import WebSocket from 'ws'; import { init as initConfig, config } from './lib/config'; -import { init as initCert, key, cert, ca, genTLSContext } from './lib/cert'; +import { init as initCert, genTLSContext } from './lib/cert'; import { createLogger, LogLevelString } from 'bunyan'; import * as utils from './lib/utils'; import { router as apiRouter } from './routers/api'; From 9647855e333c36e91ab52131f5d6d4edb275bc28 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Thu, 10 Jun 2021 13:24:13 -0400 Subject: [PATCH 03/24] Call reset on add/remove Signed-off-by: Peter Broadhurst --- src/routers/api.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routers/api.ts b/src/routers/api.ts index ccf897d..85d8894 100644 --- a/src/routers/api.ts +++ b/src/routers/api.ts @@ -22,7 +22,7 @@ import RequestError from '../lib/request-error'; import { config, persistConfig } from '../lib/config'; import { IStatus } from '../lib/interfaces'; import https from 'https'; -import { key, cert, ca, loadCAs, peerID } from '../lib/cert'; +import { key, cert, ca, peerID, resetCAs } from '../lib/cert'; import * as eventsHandler from '../handlers/events'; import { promises as fs } from 'fs'; import path from 'path'; @@ -93,7 +93,7 @@ router.put('/peers/:id', async (req, res, next) => { config.peers.push(peer); } await persistConfig(); - await loadCAs(); + await resetCAs(); res.send({ status: 'added' }); } catch (err) { next(err); @@ -114,7 +114,7 @@ router.delete('/peers/:id', async (req, res, next) => { } config.peers = config.peers.filter(peer => peer.id !== req.params.id); await persistConfig(); - await loadCAs(); + await resetCAs(); res.send({ status: 'removed' }); } catch (err) { next(err); From 2207ebedd9eaa4df02825c5e91041adac2256a21 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Thu, 10 Jun 2021 13:33:05 -0400 Subject: [PATCH 04/24] Work out how to pass the server to the right place Signed-off-by: Peter Broadhurst --- src/app.ts | 17 +++++++++++++---- src/lib/cert.ts | 8 -------- src/routers/api.ts | 12 +++++++++--- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/app.ts b/src/app.ts index a806fa5..0a0f012 100644 --- a/src/app.ts +++ b/src/app.ts @@ -15,14 +15,14 @@ // limitations under the License. import express from 'express'; -import https from 'https'; +import https, { Server } from 'https'; import http from 'http'; import WebSocket from 'ws'; import { init as initConfig, config } from './lib/config'; -import { init as initCert, genTLSContext } from './lib/cert'; +import { init as initCert, genTLSContext, loadCAs } from './lib/cert'; import { createLogger, LogLevelString } from 'bunyan'; import * as utils from './lib/utils'; -import { router as apiRouter } from './routers/api'; +import { router as apiRouter, setResetP2PCAs } from './routers/api'; import { router as p2pRouter, eventEmitter as p2pEventEmitter } from './routers/p2p'; import RequestError, { errorHandler } from './lib/request-error'; import * as eventsHandler from './handlers/events' @@ -36,8 +36,17 @@ const log = createLogger({ name: 'app.ts', level: utils.constants.LOG_LEVEL as L const swaggerDocument = YAML.load(path.join(__dirname, './swagger.yaml')); +let p2pServer : Server + let delegatedWebSocket: WebSocket | undefined = undefined; +export const resetP2PCAs = async () => { + loadCAs() + // The most recent context wins (per the Node.js spec), so to get a reload we just add a wildcard context + p2pServer.addContext("*", genTLSContext()) +}; +setResetP2PCAs(resetP2PCAs) + export const start = async () => { await initConfig(); await initCert(); @@ -46,7 +55,7 @@ export const start = async () => { const apiServer = http.createServer(apiApp); const p2pApp = express(); - const p2pServer = https.createServer(genTLSContext(), p2pApp); + p2pServer = https.createServer(genTLSContext(), p2pApp); const wss = new WebSocket.Server({ server: apiServer, verifyClient: (info, cb) => { diff --git a/src/lib/cert.ts b/src/lib/cert.ts index 5e6d471..52b550e 100644 --- a/src/lib/cert.ts +++ b/src/lib/cert.ts @@ -18,7 +18,6 @@ import * as utils from '../lib/utils'; import { promises as fs } from 'fs'; import path from 'path'; import { createLogger, LogLevelString } from 'bunyan'; -import { Server } from 'https'; const log = createLogger({ name: 'lib/certs.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); @@ -44,7 +43,6 @@ export const loadCAs = async () => { log.debug(`Loaded ${ca.length} peer certificate(s)`); }; - export const genTLSContext = () => { return { key, @@ -54,9 +52,3 @@ export const genTLSContext = () => { requestCert: true, } } - -export const resetCAs = async (server: Server) => { - loadCAs() - // The most recent context wins (per the Node.js spec), so to get a reload we just add a wildcard context - server.addContext("*", genTLSContext()) -}; diff --git a/src/routers/api.ts b/src/routers/api.ts index 85d8894..21a7c6f 100644 --- a/src/routers/api.ts +++ b/src/routers/api.ts @@ -22,7 +22,7 @@ import RequestError from '../lib/request-error'; import { config, persistConfig } from '../lib/config'; import { IStatus } from '../lib/interfaces'; import https from 'https'; -import { key, cert, ca, peerID, resetCAs } from '../lib/cert'; +import { key, cert, ca, peerID } from '../lib/cert'; import * as eventsHandler from '../handlers/events'; import { promises as fs } from 'fs'; import path from 'path'; @@ -30,6 +30,12 @@ import { v4 as uuidV4 } from 'uuid'; export const router = Router(); +let resetP2PCAs: () => void; + +export const setResetP2PCAs = (_resetP2PCAs: () => void) => { + resetP2PCAs = _resetP2PCAs +} + router.get('/id', async (_req, res, next) => { try { res.send({ @@ -93,7 +99,7 @@ router.put('/peers/:id', async (req, res, next) => { config.peers.push(peer); } await persistConfig(); - await resetCAs(); + await resetP2PCAs(); res.send({ status: 'added' }); } catch (err) { next(err); @@ -114,7 +120,7 @@ router.delete('/peers/:id', async (req, res, next) => { } config.peers = config.peers.filter(peer => peer.id !== req.params.id); await persistConfig(); - await resetCAs(); + await resetP2PCAs(); res.send({ status: 'removed' }); } catch (err) { next(err); From e0c50e1fabe2c841a7005216d135eb147c12a21a Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Thu, 10 Jun 2021 13:44:35 -0400 Subject: [PATCH 05/24] Specify hostname Signed-off-by: Peter Broadhurst --- src/app.ts | 10 +++++----- src/routers/api.ts | 11 ++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/app.ts b/src/app.ts index 0a0f012..030c487 100644 --- a/src/app.ts +++ b/src/app.ts @@ -22,7 +22,7 @@ import { init as initConfig, config } from './lib/config'; import { init as initCert, genTLSContext, loadCAs } from './lib/cert'; import { createLogger, LogLevelString } from 'bunyan'; import * as utils from './lib/utils'; -import { router as apiRouter, setResetP2PCAs } from './routers/api'; +import { router as apiRouter, setAddTLSContext } from './routers/api'; import { router as p2pRouter, eventEmitter as p2pEventEmitter } from './routers/p2p'; import RequestError, { errorHandler } from './lib/request-error'; import * as eventsHandler from './handlers/events' @@ -40,12 +40,12 @@ let p2pServer : Server let delegatedWebSocket: WebSocket | undefined = undefined; -export const resetP2PCAs = async () => { - loadCAs() +export const addTLSContext = async (hostname: string) => { + await loadCAs() // The most recent context wins (per the Node.js spec), so to get a reload we just add a wildcard context - p2pServer.addContext("*", genTLSContext()) + p2pServer.addContext(hostname, genTLSContext()) }; -setResetP2PCAs(resetP2PCAs) +setAddTLSContext(addTLSContext) export const start = async () => { await initConfig(); diff --git a/src/routers/api.ts b/src/routers/api.ts index 21a7c6f..6c09b10 100644 --- a/src/routers/api.ts +++ b/src/routers/api.ts @@ -27,13 +27,14 @@ import * as eventsHandler from '../handlers/events'; import { promises as fs } from 'fs'; import path from 'path'; import { v4 as uuidV4 } from 'uuid'; +import { URL } from 'url'; export const router = Router(); -let resetP2PCAs: () => void; +let addTLSContext: (hostname: string) => Promise; -export const setResetP2PCAs = (_resetP2PCAs: () => void) => { - resetP2PCAs = _resetP2PCAs +export const setAddTLSContext = (_addTLSContext: (hostname: string) => Promise) => { + addTLSContext = _addTLSContext; } router.get('/id', async (_req, res, next) => { @@ -99,7 +100,8 @@ router.put('/peers/:id', async (req, res, next) => { config.peers.push(peer); } await persistConfig(); - await resetP2PCAs(); + let url = new URL(req.body.endpoint) + await addTLSContext(url.hostname); res.send({ status: 'added' }); } catch (err) { next(err); @@ -120,7 +122,6 @@ router.delete('/peers/:id', async (req, res, next) => { } config.peers = config.peers.filter(peer => peer.id !== req.params.id); await persistConfig(); - await resetP2PCAs(); res.send({ status: 'removed' }); } catch (err) { next(err); From 58c0d9cffde7b2590dda521189a23adc1fb33875 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Thu, 10 Jun 2021 14:17:03 -0400 Subject: [PATCH 06/24] Correct PeerID calc on sending a message Signed-off-by: Peter Broadhurst --- src/routers/p2p.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index 8d43cea..152d468 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -31,7 +31,7 @@ router.head('/ping', (_req, res) => { router.post('/messages', async (req, res, next) => { try { const cert = req.client.getPeerCertificate(); - const sender = cert.issuer.O + cert.issuer.OU; + const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const message = await utils.extractMessageFromMultipartForm(req); eventEmitter.emit('event', { type: 'message-received', From 063f8e93ba1e7c4cf015caf79d2aae90bd743db6 Mon Sep 17 00:00:00 2001 From: Gabriel Indik Date: Wed, 16 Jun 2021 16:13:18 -0400 Subject: [PATCH 07/24] Add metadata Signed-off-by: Gabriel Indik Signed-off-by: Peter Broadhurst --- src/handlers/blobs.ts | 31 ++++++++++++++++++++++++++++--- src/lib/interfaces.ts | 5 +++++ src/lib/utils.ts | 5 ++++- src/routers/api.ts | 20 +++++++++++++++++++- src/routers/p2p.ts | 5 +++-- 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/handlers/blobs.ts b/src/handlers/blobs.ts index bd775be..abb393e 100644 --- a/src/handlers/blobs.ts +++ b/src/handlers/blobs.ts @@ -17,7 +17,7 @@ import { promises as fs, createReadStream, createWriteStream } from 'fs'; import path from 'path'; import * as utils from '../lib/utils'; -import { BlobTask, IBlobDeliveredEvent, IBlobFailedEvent, IFile } from "../lib/interfaces"; +import { BlobTask, IBlobDeliveredEvent, IBlobFailedEvent, IFile, IMetadata } from "../lib/interfaces"; import stream from 'stream'; import RequestError from '../lib/request-error'; import crypto from 'crypto'; @@ -36,7 +36,7 @@ export const eventEmitter = new EventEmitter(); export const retreiveBlob = async (filePath: string) => { const resolvedFilePath = path.join(utils.constants.DATA_DIRECTORY, utils.constants.BLOBS_SUBDIRECTORY, filePath); if (!(await utils.fileExists(resolvedFilePath))) { - throw new RequestError(`Blob not found`, 404); + throw new RequestError(`Blob content missing from storage`, 404); } return createReadStream(resolvedFilePath); }; @@ -52,7 +52,7 @@ export const storeBlob = async (file: IFile, filePath: string) => { } }); const writeStream = createWriteStream(resolvedFilePath); - return new Promise((resolve, reject) => { + const blobHash = await new Promise((resolve, reject) => { file.readableStream.on('end', () => { resolve(hash.digest('hex')); }).on('error', err => { @@ -60,6 +60,7 @@ export const storeBlob = async (file: IFile, filePath: string) => { }); file.readableStream.pipe(hashCalculator).pipe(writeStream); }); + return await upsertMetadata(filePath, blobHash); }; export const sendBlob = async (blobPath: string, recipient: string, recipientURL: string, requestID: string | undefined) => { @@ -111,3 +112,27 @@ export const deliverBlob = async ({ blobPath, recipient, recipientURL, requestID log.error(`Failed to deliver blob ${err}`); } }; + +export const retreiveMetadata = async (filePath: string) => { + const resolvedFilePath = path.join(utils.constants.DATA_DIRECTORY, utils.constants.BLOBS_SUBDIRECTORY, filePath + utils.constants.METADATA_SUFFIX); + if (!(await utils.fileExists(resolvedFilePath))) { + throw new RequestError(`Blob not found`, 404); + } + try { + const metadataString = await fs.readFile(resolvedFilePath); + return JSON.parse(metadataString.toString()) as IMetadata; + } catch(err) { + throw new RequestError(`Invalid blob`); + } +}; + +export const upsertMetadata = async (filePath: string, hash: string) => { + const resolvedFilePath = path.join(utils.constants.DATA_DIRECTORY, utils.constants.BLOBS_SUBDIRECTORY, filePath + utils.constants.METADATA_SUFFIX); + await fs.mkdir(path.parse(resolvedFilePath).dir, { recursive: true }); + let metadata: IMetadata = { + hash, + lastUpdate: new Date().getTime() + }; + await fs.writeFile(resolvedFilePath, JSON.stringify(metadata)); + return metadata; +}; diff --git a/src/lib/interfaces.ts b/src/lib/interfaces.ts index c0cc798..0180f18 100644 --- a/src/lib/interfaces.ts +++ b/src/lib/interfaces.ts @@ -124,3 +124,8 @@ export interface ICertData { organization?: string organizationUnit?: string } + +export interface IMetadata { + hash: string + lastUpdate: number +} \ No newline at end of file diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 78983e4..48e8ad9 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -28,6 +28,7 @@ export const constants = { DATA_DIRECTORY: process.env.DATA_DIRECTORY || '/data', PEER_CERTS_SUBDIRECTORY: 'peer-certs', BLOBS_SUBDIRECTORY: 'blobs', + METADATA_SUFFIX: '.metadata.json', RECEIVED_BLOBS_SUBDIRECTORY: 'received', CONFIG_FILE_NAME: 'config.json', CERT_FILE: 'cert.pem', @@ -36,7 +37,9 @@ export const constants = { TRANSFER_HASH_ALGORITHM: 'sha256', REST_API_CALL_MAX_ATTEMPTS: 5, REST_API_CALL_RETRY_DELAY_MS: 500, - MAX_EVENT_QUEUE_SIZE: 1000 + MAX_EVENT_QUEUE_SIZE: 1000, + HASH_HEADER_NAME: 'dx-hash', + LAST_UPDATE_HEADER_NAME: 'dx-last-update' }; const log = createLogger({ name: 'utils.ts', level: constants.LOG_LEVEL as LogLevelString }); diff --git a/src/routers/api.ts b/src/routers/api.ts index 6c09b10..ddf02ac 100644 --- a/src/routers/api.ts +++ b/src/routers/api.ts @@ -151,13 +151,31 @@ router.post('/messages', async (req, res, next) => { } }); +router.head('/blobs/*', async (req, res, next) => { + try { + const blobPath = `/${req.params[0]}`; + if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { + throw new RequestError('Invalid path', 400); + } + const metadata = await blobsHandler.retreiveMetadata(blobPath); + res.setHeader(utils.constants.HASH_HEADER_NAME, metadata.hash); + res.setHeader(utils.constants.LAST_UPDATE_HEADER_NAME, metadata.lastUpdate); + res.status(204).send(); + } catch (err) { + next(err); + } +}); + router.get('/blobs/*', async (req, res, next) => { try { const blobPath = `/${req.params[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } - let blobStream = await blobsHandler.retreiveBlob(blobPath); + const metadata = await blobsHandler.retreiveMetadata(blobPath); + res.setHeader(utils.constants.HASH_HEADER_NAME, metadata.hash); + res.setHeader(utils.constants.LAST_UPDATE_HEADER_NAME, metadata.lastUpdate); + const blobStream = await blobsHandler.retreiveBlob(blobPath); blobStream.on('end', () => res.end()); blobStream.pipe(res); } catch (err) { diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index 152d468..7362d07 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -50,13 +50,14 @@ router.put('/blobs/*', async (req, res, next) => { const sender = cert.issuer.O + cert.issuer.OU; const file = await utils.extractFileFromMultipartForm(req); const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, req.params[0]); - const hash = await blobsHandler.storeBlob(file, blobPath); + const metadata = await blobsHandler.storeBlob(file, blobPath); res.sendStatus(204); eventEmitter.emit('event', { type: 'blob-received', sender, path: blobPath, - hash + hash: metadata.hash, + lastUpdate: metadata.lastUpdate } as IBlobReceivedEvent); } catch (err) { next(err); From e469dc152ca5515df4443ffe675ba6b3c24bc360 Mon Sep 17 00:00:00 2001 From: Gabriel Indik Date: Wed, 16 Jun 2021 16:16:51 -0400 Subject: [PATCH 08/24] Update swagger Signed-off-by: Gabriel Indik Signed-off-by: Peter Broadhurst --- src/swagger.yaml | 50 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/src/swagger.yaml b/src/swagger.yaml index c6c0c64..e4e075c 100644 --- a/src/swagger.yaml +++ b/src/swagger.yaml @@ -146,20 +146,49 @@ schema: $ref: '#/components/schemas/Error' /blobs/{blobPath}: + parameters: + - in: path + name: blobPath + required: true + schema: + type: string + description: Blob path + head: + tags: + - Blobs + description: Retrieve blob metadata + responses: + '204': + description: Blob metadata + headers: + dx-hash: + schema: + type: string + description: Blob hash + dx-last-update: + schema: + type: string + description: Blob last update timestamp + '404': + description: Blob not found + '500': + description: Internal error get: tags: - Blobs description: Retreive blob - parameters: - - in: path - name: blobPath - required: true - schema: - type: string - description: Blob path responses: '200': description: Blob content + headers: + dx-hash: + schema: + type: string + description: Blob hash + dx-last-update: + schema: + type: string + description: Blob last update timestamp content: application/json: schema: @@ -181,13 +210,6 @@ tags: - Blobs description: Store blob - parameters: - - in: path - name: blobPath - required: true - schema: - type: string - description: Blob path requestBody: description: Blob required: true From 2e019cd5db4444dd60d9317ac5cad49003ac9408 Mon Sep 17 00:00:00 2001 From: Gabriel Indik Date: Wed, 16 Jun 2021 16:39:42 -0400 Subject: [PATCH 09/24] Restructure API response Signed-off-by: Gabriel Indik Signed-off-by: Peter Broadhurst --- src/routers/api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routers/api.ts b/src/routers/api.ts index ddf02ac..3cec523 100644 --- a/src/routers/api.ts +++ b/src/routers/api.ts @@ -190,8 +190,8 @@ router.put('/blobs/*', async (req, res, next) => { throw new RequestError('Invalid path', 400); } const file = await utils.extractFileFromMultipartForm(req); - const hash = await blobsHandler.storeBlob(file, blobPath); - res.send({ hash }); + const metadata = await blobsHandler.storeBlob(file, blobPath); + res.send(metadata); } catch (err) { next(err); } From d82e26ac35ab0c0fe1ec40a417ac1ad2d225ee4f Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Sat, 19 Jun 2021 22:51:50 -0400 Subject: [PATCH 10/24] Tweak hashing to avoid incomplete uploads Signed-off-by: Peter Broadhurst --- src/handlers/blobs.ts | 20 +++++++++++--------- src/routers/api.ts | 9 ++++++--- src/routers/p2p.ts | 9 ++++++--- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/handlers/blobs.ts b/src/handlers/blobs.ts index abb393e..b89a60d 100644 --- a/src/handlers/blobs.ts +++ b/src/handlers/blobs.ts @@ -45,17 +45,19 @@ export const storeBlob = async (file: IFile, filePath: string) => { const resolvedFilePath = path.join(utils.constants.DATA_DIRECTORY, utils.constants.BLOBS_SUBDIRECTORY, filePath); await fs.mkdir(path.parse(resolvedFilePath).dir, { recursive: true }); let hash = crypto.createHash(utils.constants.TRANSFER_HASH_ALGORITHM); - let hashCalculator = new stream.Transform({ - async transform(chunk, _enc, cb) { - hash.update(chunk); - cb(undefined, chunk); - } - }); const writeStream = createWriteStream(resolvedFilePath); const blobHash = await new Promise((resolve, reject) => { - file.readableStream.on('end', () => { - resolve(hash.digest('hex')); - }).on('error', err => { + let hashCalculator = new stream.Transform({ + transform(chunk, _enc, cb) { + hash.update(chunk); + cb(undefined, chunk); + }, + flush(cb) { + resolve(hash.digest('hex')); + cb(); + } + }); + file.readableStream.on('error', err => { reject(err); }); file.readableStream.pipe(hashCalculator).pipe(writeStream); diff --git a/src/routers/api.ts b/src/routers/api.ts index 3cec523..6d93c7c 100644 --- a/src/routers/api.ts +++ b/src/routers/api.ts @@ -153,7 +153,8 @@ router.post('/messages', async (req, res, next) => { router.head('/blobs/*', async (req, res, next) => { try { - const blobPath = `/${req.params[0]}`; + const p: any = req.params + const blobPath = `/${p[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } @@ -168,7 +169,8 @@ router.head('/blobs/*', async (req, res, next) => { router.get('/blobs/*', async (req, res, next) => { try { - const blobPath = `/${req.params[0]}`; + const p: any = req.params + const blobPath = `/${p[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } @@ -185,7 +187,8 @@ router.get('/blobs/*', async (req, res, next) => { router.put('/blobs/*', async (req, res, next) => { try { - const blobPath = `/${req.params[0]}`; + const p: any = req.params + const blobPath = `/${p[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index 7362d07..2a5724f 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -30,7 +30,8 @@ router.head('/ping', (_req, res) => { router.post('/messages', async (req, res, next) => { try { - const cert = req.client.getPeerCertificate(); + const r: any = req; + const cert = r.client.getPeerCertificate(); const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const message = await utils.extractMessageFromMultipartForm(req); eventEmitter.emit('event', { @@ -46,10 +47,12 @@ router.post('/messages', async (req, res, next) => { router.put('/blobs/*', async (req, res, next) => { try { - const cert = req.client.getPeerCertificate(); + const r: any = req; + const p: any = req.params; + const cert = r.client.getPeerCertificate(); const sender = cert.issuer.O + cert.issuer.OU; const file = await utils.extractFileFromMultipartForm(req); - const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, req.params[0]); + const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, p[0]); const metadata = await blobsHandler.storeBlob(file, blobPath); res.sendStatus(204); eventEmitter.emit('event', { From 9a121ea1cfe7ef6127886e4efd98d5f5a0968c88 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Sun, 20 Jun 2021 02:35:58 -0400 Subject: [PATCH 11/24] Fix sender Signed-off-by: Peter Broadhurst --- src/routers/p2p.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index 2a5724f..7e99791 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -50,7 +50,7 @@ router.put('/blobs/*', async (req, res, next) => { const r: any = req; const p: any = req.params; const cert = r.client.getPeerCertificate(); - const sender = cert.issuer.O + cert.issuer.OU; + const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const file = await utils.extractFileFromMultipartForm(req); const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, p[0]); const metadata = await blobsHandler.storeBlob(file, blobPath); From 07a138697df4d86567379beff6b11245eebabfff Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 13:35:19 -0400 Subject: [PATCH 12/24] Log events Signed-off-by: Peter Broadhurst --- src/app.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app.ts b/src/app.ts index 030c487..2b10582 100644 --- a/src/app.ts +++ b/src/app.ts @@ -72,6 +72,7 @@ export const start = async () => { messagesEventEmitter.addListener('event', event => eventsHandler.queueEvent(event)); eventsHandler.eventEmitter.addListener('event', event => { + log.info(`Event emitted: ${JSON.stringify(event)}`) if (delegatedWebSocket !== undefined) { delegatedWebSocket.send(JSON.stringify(event)); } From b4f23026d7eeeddc2beb3819bdec5c6f19d90b0f Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 13:50:28 -0400 Subject: [PATCH 13/24] Log events Signed-off-by: Peter Broadhurst --- src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index 2b10582..ca11110 100644 --- a/src/app.ts +++ b/src/app.ts @@ -72,7 +72,7 @@ export const start = async () => { messagesEventEmitter.addListener('event', event => eventsHandler.queueEvent(event)); eventsHandler.eventEmitter.addListener('event', event => { - log.info(`Event emitted: ${JSON.stringify(event)}`) + log.info(`Event emitted ${event.type} `) if (delegatedWebSocket !== undefined) { delegatedWebSocket.send(JSON.stringify(event)); } From 323f4fd821b03aa4b3d8eb0c85f18b2c38c7c868 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 13:57:25 -0400 Subject: [PATCH 14/24] Correlate events with commits Signed-off-by: Peter Broadhurst --- src/app.ts | 3 ++- src/handlers/blobs.ts | 3 +++ src/handlers/messages.ts | 3 +++ src/lib/interfaces.ts | 6 ++++++ src/routers/p2p.ts | 3 +++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index ca11110..0ba77e6 100644 --- a/src/app.ts +++ b/src/app.ts @@ -72,7 +72,7 @@ export const start = async () => { messagesEventEmitter.addListener('event', event => eventsHandler.queueEvent(event)); eventsHandler.eventEmitter.addListener('event', event => { - log.info(`Event emitted ${event.type} `) + log.info(`Event emitted ${event.type}/${event.id}`) if (delegatedWebSocket !== undefined) { delegatedWebSocket.send(JSON.stringify(event)); } @@ -86,6 +86,7 @@ export const start = async () => { try { const messageContent = JSON.parse(message.toLocaleString()); if (messageContent.action === 'commit') { + log.info(`Event comitted ${event?`${event.type}/${event.id}`:`[no event in flight]`}`) eventsHandler.handleCommit(); } } catch (err) { diff --git a/src/handlers/blobs.ts b/src/handlers/blobs.ts index b89a60d..3dc0310 100644 --- a/src/handlers/blobs.ts +++ b/src/handlers/blobs.ts @@ -26,6 +26,7 @@ import https from 'https'; import { key, cert, ca } from '../lib/cert'; import { createLogger, LogLevelString } from 'bunyan'; import EventEmitter from 'events'; +import { v4 as uuidV4 } from 'uuid'; const log = createLogger({ name: 'handlers/blobs.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); @@ -97,6 +98,7 @@ export const deliverBlob = async ({ blobPath, recipient, recipientURL, requestID httpsAgent }); eventEmitter.emit('event', { + id: uuidV4(), type: 'blob-delivered', path: blobPath, recipient, @@ -105,6 +107,7 @@ export const deliverBlob = async ({ blobPath, recipient, recipientURL, requestID log.trace(`Blob delivered`); } catch (err) { eventEmitter.emit('event', { + id: uuidV4(), type: 'blob-failed', path: blobPath, recipient, diff --git a/src/handlers/messages.ts b/src/handlers/messages.ts index d15fdc8..601c8a5 100644 --- a/src/handlers/messages.ts +++ b/src/handlers/messages.ts @@ -21,6 +21,7 @@ import { IMessageDeliveredEvent, IMessageFailedEvent, MessageTask } from '../lib import FormData from 'form-data'; import EventEmitter from 'events'; import { createLogger, LogLevelString } from 'bunyan'; +import { v4 as uuidV4 } from 'uuid'; const log = createLogger({ name: 'handlers/messages.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); @@ -55,6 +56,7 @@ export const deliverMessage = async ({ message, recipient, recipientURL, request httpsAgent }); eventEmitter.emit('event', { + id: uuidV4(), type: 'message-delivered', message, recipient, @@ -63,6 +65,7 @@ export const deliverMessage = async ({ message, recipient, recipientURL, request log.trace(`Message delivered`); } catch(err) { eventEmitter.emit('event', { + id: uuidV4(), type: 'message-failed', message, recipient, diff --git a/src/lib/interfaces.ts b/src/lib/interfaces.ts index 0180f18..497b917 100644 --- a/src/lib/interfaces.ts +++ b/src/lib/interfaces.ts @@ -46,18 +46,21 @@ export type OutboundEvent = IBlobFailedEvent export interface IMessageReceivedEvent { + id: string type: 'message-received' sender: string message: string } export interface IMessageDeliveredEvent { + id: string type: 'message-delivered' recipient: string message: string } export interface IMessageFailedEvent { + id: string type: 'message-failed' recipient: string message: string @@ -65,6 +68,7 @@ export interface IMessageFailedEvent { } export interface IBlobReceivedEvent { + id: string type: 'blob-received' sender: string path: string @@ -72,12 +76,14 @@ export interface IBlobReceivedEvent { } export interface IBlobDeliveredEvent { + id: string type: 'blob-delivered' recipient: string path: string } export interface IBlobFailedEvent { + id: string type: 'blob-failed' recipient: string path: string diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index 7e99791..c5407aa 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -20,6 +20,7 @@ import * as blobsHandler from '../handlers/blobs'; import path from 'path'; import { EventEmitter } from 'events'; import { IBlobReceivedEvent, IMessageReceivedEvent } from '../lib/interfaces'; +import { v4 as uuidV4 } from 'uuid'; export const router = Router(); export const eventEmitter = new EventEmitter(); @@ -35,6 +36,7 @@ router.post('/messages', async (req, res, next) => { const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const message = await utils.extractMessageFromMultipartForm(req); eventEmitter.emit('event', { + id: uuidV4(), type: 'message-received', sender, message @@ -56,6 +58,7 @@ router.put('/blobs/*', async (req, res, next) => { const metadata = await blobsHandler.storeBlob(file, blobPath); res.sendStatus(204); eventEmitter.emit('event', { + id: uuidV4(), type: 'blob-received', sender, path: blobPath, From 91e3ab09b7f910ceba96d21e7904b28af3f62c36 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 14:06:45 -0400 Subject: [PATCH 15/24] Simplify logger to avoid JSON output in firefly-cli Signed-off-by: Peter Broadhurst --- package.json | 4 +-- src/app.ts | 26 +++++++------- src/handlers/blobs.ts | 20 +++++------ src/handlers/events.ts | 4 +-- src/handlers/messages.ts | 14 ++++---- src/index.ts | 5 ++- src/lib/cert.ts | 4 +-- src/lib/logger.ts | 73 ++++++++++++++++++++++++++++++++++++++++ src/lib/utils.ts | 10 +++--- 9 files changed, 116 insertions(+), 44 deletions(-) create mode 100644 src/lib/logger.ts diff --git a/package.json b/package.json index dbbffb9..4ef373c 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,6 @@ "dependencies": { "ajv": "^8.5.0", "axios": "^0.21.1", - "bunyan": "^1.8.15", "busboy": "^0.3.1", "express": "^4.17.1", "form-data": "^4.0.0", @@ -42,6 +41,7 @@ "@types/swagger-ui-express": "^4.1.2", "@types/uuid": "^8.3.0", "@types/ws": "^7.4.4", - "@types/yamljs": "^0.2.31" + "@types/yamljs": "^0.2.31", + "rimraf": "^3.0.2" } } diff --git a/src/app.ts b/src/app.ts index 0ba77e6..110fa92 100644 --- a/src/app.ts +++ b/src/app.ts @@ -15,24 +15,24 @@ // limitations under the License. import express from 'express'; -import https, { Server } from 'https'; import http from 'http'; +import https, { Server } from 'https'; +import path from 'path'; +import swaggerUi from 'swagger-ui-express'; import WebSocket from 'ws'; -import { init as initConfig, config } from './lib/config'; -import { init as initCert, genTLSContext, loadCAs } from './lib/cert'; -import { createLogger, LogLevelString } from 'bunyan'; -import * as utils from './lib/utils'; -import { router as apiRouter, setAddTLSContext } from './routers/api'; -import { router as p2pRouter, eventEmitter as p2pEventEmitter } from './routers/p2p'; -import RequestError, { errorHandler } from './lib/request-error'; -import * as eventsHandler from './handlers/events' +import YAML from 'yamljs'; import { eventEmitter as blobsEventEmitter } from './handlers/blobs'; +import * as eventsHandler from './handlers/events'; import { eventEmitter as messagesEventEmitter } from './handlers/messages'; -import swaggerUi from 'swagger-ui-express'; -import YAML from 'yamljs'; -import path from 'path'; +import { genTLSContext, init as initCert, loadCAs } from './lib/cert'; +import { config, init as initConfig } from './lib/config'; +import { Logger } from './lib/logger'; +import RequestError, { errorHandler } from './lib/request-error'; +import * as utils from './lib/utils'; +import { router as apiRouter, setAddTLSContext } from './routers/api'; +import { eventEmitter as p2pEventEmitter, router as p2pRouter } from './routers/p2p'; -const log = createLogger({ name: 'app.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger("app.ts"); const swaggerDocument = YAML.load(path.join(__dirname, './swagger.yaml')); diff --git a/src/handlers/blobs.ts b/src/handlers/blobs.ts index 3dc0310..a1ecaa6 100644 --- a/src/handlers/blobs.ts +++ b/src/handlers/blobs.ts @@ -14,21 +14,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { promises as fs, createReadStream, createWriteStream } from 'fs'; -import path from 'path'; -import * as utils from '../lib/utils'; -import { BlobTask, IBlobDeliveredEvent, IBlobFailedEvent, IFile, IMetadata } from "../lib/interfaces"; -import stream from 'stream'; -import RequestError from '../lib/request-error'; import crypto from 'crypto'; +import EventEmitter from 'events'; import FormData from 'form-data'; +import { createReadStream, createWriteStream, promises as fs } from 'fs'; import https from 'https'; -import { key, cert, ca } from '../lib/cert'; -import { createLogger, LogLevelString } from 'bunyan'; -import EventEmitter from 'events'; +import path from 'path'; +import stream from 'stream'; import { v4 as uuidV4 } from 'uuid'; +import { ca, cert, key } from '../lib/cert'; +import { BlobTask, IBlobDeliveredEvent, IBlobFailedEvent, IFile, IMetadata } from "../lib/interfaces"; +import { Logger } from '../lib/logger'; +import RequestError from '../lib/request-error'; +import * as utils from '../lib/utils'; -const log = createLogger({ name: 'handlers/blobs.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger("handlers/blobs.ts") let blobQueue: BlobTask[] = []; let sending = false; diff --git a/src/handlers/events.ts b/src/handlers/events.ts index deedf57..5e972e0 100644 --- a/src/handlers/events.ts +++ b/src/handlers/events.ts @@ -14,12 +14,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { createLogger, LogLevelString } from "bunyan"; import EventEmitter from "events"; import { OutboundEvent } from "../lib/interfaces"; +import { Logger } from "../lib/logger"; import * as utils from '../lib/utils'; -const log = createLogger({ name: 'handlers/events.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger("handlers/events.ts") let eventQueue: OutboundEvent[] = []; export const eventEmitter = new EventEmitter(); diff --git a/src/handlers/messages.ts b/src/handlers/messages.ts index 601c8a5..0b35645 100644 --- a/src/handlers/messages.ts +++ b/src/handlers/messages.ts @@ -14,16 +14,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -import https from 'https'; -import * as utils from '../lib/utils'; -import { key, cert, ca } from '../lib/cert'; -import { IMessageDeliveredEvent, IMessageFailedEvent, MessageTask } from '../lib/interfaces'; -import FormData from 'form-data'; import EventEmitter from 'events'; -import { createLogger, LogLevelString } from 'bunyan'; +import FormData from 'form-data'; +import https from 'https'; import { v4 as uuidV4 } from 'uuid'; +import { ca, cert, key } from '../lib/cert'; +import { IMessageDeliveredEvent, IMessageFailedEvent, MessageTask } from '../lib/interfaces'; +import { Logger } from '../lib/logger'; +import * as utils from '../lib/utils'; -const log = createLogger({ name: 'handlers/messages.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger('handlers/messages.ts') let messageQueue: MessageTask[] = []; let sending = false; diff --git a/src/index.ts b/src/index.ts index e13729d..836b903 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,11 +14,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { createLogger, LogLevelString } from 'bunyan'; -import * as utils from './lib/utils'; import { start } from './app'; +import { Logger } from './lib/logger'; -const log = createLogger({ name: 'index.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger("index.ts") start().catch(err => { log.error(`Failed to FireFly Data Exchange ${err}`); diff --git a/src/lib/cert.ts b/src/lib/cert.ts index 52b550e..44c3364 100644 --- a/src/lib/cert.ts +++ b/src/lib/cert.ts @@ -17,9 +17,9 @@ import * as utils from '../lib/utils'; import { promises as fs } from 'fs'; import path from 'path'; -import { createLogger, LogLevelString } from 'bunyan'; +import { Logger } from './logger'; -const log = createLogger({ name: 'lib/certs.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger('lib/certs.ts') export let key: string; export let cert: string; diff --git a/src/lib/logger.ts b/src/lib/logger.ts new file mode 100644 index 0000000..cbb11f2 --- /dev/null +++ b/src/lib/logger.ts @@ -0,0 +1,73 @@ +// Copyright © 2021 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +"use strict"; + +const LEVEL_NONE = 0; +const LEVEL_ERROR = 1; +const LEVEL_WARN = 2; +const LEVEL_INFO = 3; +const LEVEL_DEBUG = 4; +const LEVEL_TRACE = 5; + +const LEVEL_TAGS = { + [LEVEL_NONE]: 'NONE', + [LEVEL_ERROR]: 'ERROR', + [LEVEL_WARN]: 'WARN ', + [LEVEL_INFO]: 'INFO ', + [LEVEL_DEBUG]: 'DEBUG', + [LEVEL_TRACE]: 'TRACE', +}; + +let logLevel = LEVEL_ERROR; + +export function setLogLevel(level?: string) { + if (!level) level = process.env.LOG_LEVEL || 'info'; + for (let [l,t] of Object.entries(LEVEL_TAGS)) { + if (t.trim().toLowerCase() === level.trim().toLowerCase()) { + logLevel = Number(l); + } + } +} + +export class Logger { + + constructor(private loggerName?: string) {} + + error(...args: any[]) { logLevel >= LEVEL_ERROR && this.log('ERROR', ...args); } + warn(...args: any[]) { logLevel >= LEVEL_WARN && this.log('WARN ', ...args); } + info(...args: any[]) { logLevel >= LEVEL_INFO && this.log('INFO ', ...args); } + debug(...args: any[]) { logLevel >= LEVEL_DEBUG && this.log('DEBUG', ...args); } + trace(...args: any[]) { logLevel >= LEVEL_TRACE && this.log('TRACE', ...args); } + + private log(level: string, ...args: any[]) { + const logArgs = []; + for (const arg of args) { + // Special handling of axios errors to avoid massive dumps in log + if (arg?.isAxiosError) { + let data = arg.response?.data; + data = data?.on ? '[stream]' : JSON.stringify(data); + logArgs.push(`HTTP [${arg.response?.status}] ${arg.message}: ${data}`) + } else { + logArgs.push(arg); + } + } + console.log(`${new Date().toISOString()} [${level}]:`, ...logArgs, this.loggerName); + } + +} + +setLogLevel(); diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 48e8ad9..a63b7e7 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -14,14 +14,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +import axios, { AxiosRequestConfig } from 'axios'; +import Busboy from 'busboy'; import { Request } from 'express'; import { promises as fs } from 'fs'; +import { X509 } from 'jsrsasign'; import { ICertData, IFile } from './interfaces'; +import { Logger } from './logger'; import RequestError from './request-error'; -import Busboy from 'busboy'; -import axios, { AxiosRequestConfig } from 'axios'; -import { createLogger, LogLevelString } from 'bunyan'; -import { X509 } from 'jsrsasign'; export const constants = { LOG_LEVEL: process.env.LOG_LEVEL || 'info', @@ -41,7 +41,7 @@ export const constants = { HASH_HEADER_NAME: 'dx-hash', LAST_UPDATE_HEADER_NAME: 'dx-last-update' }; -const log = createLogger({ name: 'utils.ts', level: constants.LOG_LEVEL as LogLevelString }); +const log = new Logger('utils.ts') export const regexp = { FILE_KEY: /^(\/[a-z0-9\+\-\_\.]+)+$/, From 2d2faf15d0c161b37cd595072d09d03f41338f78 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 14:22:37 -0400 Subject: [PATCH 16/24] Logging for TLS context addition Signed-off-by: Peter Broadhurst --- src/app.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app.ts b/src/app.ts index 110fa92..0eefdcf 100644 --- a/src/app.ts +++ b/src/app.ts @@ -43,6 +43,7 @@ let delegatedWebSocket: WebSocket | undefined = undefined; export const addTLSContext = async (hostname: string) => { await loadCAs() // The most recent context wins (per the Node.js spec), so to get a reload we just add a wildcard context + log.info(`Adding TLS context for new peer '%s'`, hostname) p2pServer.addContext(hostname, genTLSContext()) }; setAddTLSContext(addTLSContext) From c757d5027181d69dda4838960511419799802f49 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 14:24:23 -0400 Subject: [PATCH 17/24] Logging for TLS context addition Signed-off-by: Peter Broadhurst --- src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index 0eefdcf..43a2717 100644 --- a/src/app.ts +++ b/src/app.ts @@ -43,7 +43,7 @@ let delegatedWebSocket: WebSocket | undefined = undefined; export const addTLSContext = async (hostname: string) => { await loadCAs() // The most recent context wins (per the Node.js spec), so to get a reload we just add a wildcard context - log.info(`Adding TLS context for new peer '%s'`, hostname) + log.info(`Adding TLS context for new peer '${hostname}'`) p2pServer.addContext(hostname, genTLSContext()) }; setAddTLSContext(addTLSContext) From 814f31f74cdf3fe14c5bccceccd21c1c1b3100b7 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Tue, 22 Jun 2021 09:01:12 -0400 Subject: [PATCH 18/24] Cleaner fix for type issue Signed-off-by: Peter Broadhurst --- src/custom.d.ts | 6 +++++- src/routers/api.ts | 17 +++++++---------- src/routers/p2p.ts | 11 +++++------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/custom.d.ts b/src/custom.d.ts index 1ba0bf2..a4f21be 100644 --- a/src/custom.d.ts +++ b/src/custom.d.ts @@ -16,9 +16,13 @@ export {} -declare global{ +declare global { namespace Express { export interface Request { + params: { + [s: string]: string, + 0: string, + }, client: { authorized: boolean getCertificate: () => { diff --git a/src/routers/api.ts b/src/routers/api.ts index 6d93c7c..755d4da 100644 --- a/src/routers/api.ts +++ b/src/routers/api.ts @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Router } from 'express'; +import { Router, Request } from 'express'; import * as blobsHandler from '../handlers/blobs'; import * as messagesHandler from '../handlers/messages'; import * as utils from '../lib/utils'; @@ -151,10 +151,9 @@ router.post('/messages', async (req, res, next) => { } }); -router.head('/blobs/*', async (req, res, next) => { +router.head('/blobs/*', async (req: Request, res, next) => { try { - const p: any = req.params - const blobPath = `/${p[0]}`; + const blobPath = `/${req.params[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } @@ -167,10 +166,9 @@ router.head('/blobs/*', async (req, res, next) => { } }); -router.get('/blobs/*', async (req, res, next) => { +router.get('/blobs/*', async (req: Request, res, next) => { try { - const p: any = req.params - const blobPath = `/${p[0]}`; + const blobPath = `/${req.params[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } @@ -185,10 +183,9 @@ router.get('/blobs/*', async (req, res, next) => { } }); -router.put('/blobs/*', async (req, res, next) => { +router.put('/blobs/*', async (req: Request, res, next) => { try { - const p: any = req.params - const blobPath = `/${p[0]}`; + const blobPath = `/${req.params[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index c5407aa..a928c87 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -14,13 +14,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Router } from 'express'; +import { Router, Request } from 'express'; import * as utils from '../lib/utils'; import * as blobsHandler from '../handlers/blobs'; import path from 'path'; import { EventEmitter } from 'events'; import { IBlobReceivedEvent, IMessageReceivedEvent } from '../lib/interfaces'; import { v4 as uuidV4 } from 'uuid'; +import '../custom' export const router = Router(); export const eventEmitter = new EventEmitter(); @@ -47,14 +48,12 @@ router.post('/messages', async (req, res, next) => { } }); -router.put('/blobs/*', async (req, res, next) => { +router.put('/blobs/*', async (req: Request, res, next) => { try { - const r: any = req; - const p: any = req.params; - const cert = r.client.getPeerCertificate(); + const cert = req.client.getPeerCertificate(); const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const file = await utils.extractFileFromMultipartForm(req); - const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, p[0]); + const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, req.params[0]); const metadata = await blobsHandler.storeBlob(file, blobPath); res.sendStatus(204); eventEmitter.emit('event', { From 0fa833b08cccc9ed008efefe501af019cf1facf0 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Tue, 22 Jun 2021 09:05:30 -0400 Subject: [PATCH 19/24] Missed one tidy up Signed-off-by: Peter Broadhurst --- src/routers/p2p.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index a928c87..161bd56 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -30,10 +30,9 @@ router.head('/ping', (_req, res) => { res.sendStatus(204); }); -router.post('/messages', async (req, res, next) => { +router.post('/messages', async (req: Request, res, next) => { try { - const r: any = req; - const cert = r.client.getPeerCertificate(); + const cert = req.client.getPeerCertificate(); const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const message = await utils.extractMessageFromMultipartForm(req); eventEmitter.emit('event', { From 652ea260ffacf906ee4eded0e7fd1b0afd2b6aed Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Tue, 22 Jun 2021 09:52:42 -0400 Subject: [PATCH 20/24] Do not import custom Signed-off-by: Peter Broadhurst --- src/routers/p2p.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index 161bd56..39ededf 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -21,7 +21,6 @@ import path from 'path'; import { EventEmitter } from 'events'; import { IBlobReceivedEvent, IMessageReceivedEvent } from '../lib/interfaces'; import { v4 as uuidV4 } from 'uuid'; -import '../custom' export const router = Router(); export const eventEmitter = new EventEmitter(); From 73a1fcb9d114a588b841bed61246c77578ebf536 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Tue, 22 Jun 2021 13:45:47 -0400 Subject: [PATCH 21/24] Fix updating TLS context on add peer Signed-off-by: Peter Broadhurst --- src/app.ts | 10 ++++------ src/routers/api.ts | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/app.ts b/src/app.ts index 43a2717..e85970c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -29,7 +29,7 @@ import { config, init as initConfig } from './lib/config'; import { Logger } from './lib/logger'; import RequestError, { errorHandler } from './lib/request-error'; import * as utils from './lib/utils'; -import { router as apiRouter, setAddTLSContext } from './routers/api'; +import { router as apiRouter, setRefreshCACerts } from './routers/api'; import { eventEmitter as p2pEventEmitter, router as p2pRouter } from './routers/p2p'; const log = new Logger("app.ts"); @@ -40,13 +40,11 @@ let p2pServer : Server let delegatedWebSocket: WebSocket | undefined = undefined; -export const addTLSContext = async (hostname: string) => { +export const refreshCACerts = async () => { await loadCAs() - // The most recent context wins (per the Node.js spec), so to get a reload we just add a wildcard context - log.info(`Adding TLS context for new peer '${hostname}'`) - p2pServer.addContext(hostname, genTLSContext()) + p2pServer.setSecureContext(genTLSContext()) }; -setAddTLSContext(addTLSContext) +setRefreshCACerts(refreshCACerts) export const start = async () => { await initConfig(); diff --git a/src/routers/api.ts b/src/routers/api.ts index 755d4da..4bd369e 100644 --- a/src/routers/api.ts +++ b/src/routers/api.ts @@ -27,14 +27,13 @@ import * as eventsHandler from '../handlers/events'; import { promises as fs } from 'fs'; import path from 'path'; import { v4 as uuidV4 } from 'uuid'; -import { URL } from 'url'; export const router = Router(); -let addTLSContext: (hostname: string) => Promise; +let refreshCACerts: () => Promise; -export const setAddTLSContext = (_addTLSContext: (hostname: string) => Promise) => { - addTLSContext = _addTLSContext; +export const setRefreshCACerts = (fn: () => Promise) => { + refreshCACerts = fn; } router.get('/id', async (_req, res, next) => { @@ -100,8 +99,7 @@ router.put('/peers/:id', async (req, res, next) => { config.peers.push(peer); } await persistConfig(); - let url = new URL(req.body.endpoint) - await addTLSContext(url.hostname); + await refreshCACerts(); res.send({ status: 'added' }); } catch (err) { next(err); From d24c5a6ef9585798c1c97e0f6420402f668072d1 Mon Sep 17 00:00:00 2001 From: Gabriel Indik Date: Tue, 22 Jun 2021 15:46:15 -0400 Subject: [PATCH 22/24] Fix regex Signed-off-by: Peter Broadhurst --- data/member-a/cert.pem | 26 +++++++++++--------------- src/lib/utils.ts | 4 ++-- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/data/member-a/cert.pem b/data/member-a/cert.pem index f984b1b..10b0c5f 100644 --- a/data/member-a/cert.pem +++ b/data/member-a/cert.pem @@ -1,17 +1,13 @@ -----BEGIN CERTIFICATE----- -MIICxDCCAawCCQCwQ58VBeUg+TANBgkqhkiG9w0BAQsFADAkMRIwEAYDVQQDDAls -b2NhbGhvc3QxDjAMBgNVBAoMBW9yZy1hMB4XDTIxMDUyMDAyMjUzN1oXDTIyMDUy -MDAyMjUzN1owJDESMBAGA1UEAwwJbG9jYWxob3N0MQ4wDAYDVQQKDAVvcmctYTCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK/bZPBvaA+V6HoXNP+oOQaK -DPuLgFO3R7TTIBTCJttELv1lbfuw95Up+/5b4dPnnIF3wwE2GZsiMWJV7RgfDAUi -x7saJJIIa2E8gLrmoaG99w+PYQW41OBz0p54tw4abN5RNsgineu1N5pIJOMJ6cMs -D8IYfWSHDjZZvu3F2YCItSVYZlfmGBCcFCf4HASF6m4lUTZCDRBurjOQrwW9mXJv -hkIa1HEa4l7Nq09d8Bokvieq1vHJUB78kYTR027z+sm4H2o4pXjzBOaV/z2yB7+t -yqWbMjm8aq2m/gmpZTjDBPgX9XLMYHWBZWCpk1iHo/eqw+UeZ6kRZ0RR78Co+IsC -AwEAATANBgkqhkiG9w0BAQsFAAOCAQEAH6rQx8khtfzH93YCstHduvEzUCHSyifF -sDyRLvSmS0qHz1liLJyvWbT9xDQCpDZrxhQKJLN2eZxxlNaH6XBEfHQgi2I43hxu -sd2KZ4wOWOk/HyM9BcibKNMtfHJdgQ5EYRc6OWDY1c8bQQfRJUBzrSJKldqfQjqC -mPEeHXDH++yTg2Vfm7GZiogxqSWn/+ILzHNeWrvr0HJ86Guyg/NPKBxs0uasvgI7 -KqW+fcZ/9Vg6a4e+zRTL8EwYBX6dTSwgt4X9wuwMvt/K0qodgW6I4paqEpVJOS+d -Z6WevHgHwZTr+mS3mJ6Y6u/DBD8/06uPAf3T5d/dC6Xtl4sEFkpNAA== +MIICBDCCAamgAwIBAgIQfrTdLe0oABHDVmSAOeaZJTAKBggqhkjOPQQDAjBJMQsw +CQYDVQQGEwJVUzEQMA4GA1UEBwwHUmFsZWlnaDEQMA4GA1UECgwHS2FsZWlkbzEW +MBQGA1UEAwwNY2EuenpwemIyZmRiNTAiGA8yMDIxMDYyMjE2MjkzOVoYDzIwMjQw +NjIyMTYyOTM5WjB2MQswCQYDVQQGEwJVUzEQMA4GA1UEBwwHUmFsZWlnaDEQMA4G +A1UECgwHS2FsZWlkbzETMBEGA1UECwwKenprcWRmMTZ0ajEuMCwGA1UEAwwlenps +eTV6MDZkcy56enB6YjJmZGI1LmthbGVpZG8ubmV0d29yazBZMBMGByqGSM49AgEG +CCqGSM49AwEHA0IABMucOwEUA23cYMNiAKzxoqcy+e6agAvR6FxFldrzeuJ+4vu+ +eVfDjlVIswRCyedqYQj84H6MteSNHHXhXa9dErGjQjBAMAwGA1UdEwEB/wQCMAAw +DgYDVR0PAQH/BAQDAgWgMCAGA1UdJQEB/wQWMBQGCCsGAQUFBwMCBggrBgEFBQcD +ATAKBggqhkjOPQQDAgNJADBGAiEA6MsZUeTgPThToswHVXNbf4z7NI17fB9qxQCp +aSp9144CIQC7alKXsyL2xMucZi7hNRsQlvfKm5zrBS/8qJAEirYOSQ== -----END CERTIFICATE----- diff --git a/src/lib/utils.ts b/src/lib/utils.ts index a63b7e7..8c6a941 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -138,12 +138,12 @@ export const getCertData = (cert: string): ICertData => { const x509 = new X509(); x509.readCertPEM(cert); const subject = x509.getSubjectString(); - const o = subject.match(/O=(.+[^/])/); + const o = subject.match('O=([^\/.]+)'); let certData: ICertData = {}; if(o !== null) { certData.organization = o[1]; } - const ou = subject.match(/OU=(.+[^/])/); + const ou = subject.match('OU=([^\/.]+)'); if(ou !== null) { certData.organizationUnit = ou[1]; } From 33e2c571183e1b15e50ed6f24b2ac5b0a2132aa1 Mon Sep 17 00:00:00 2001 From: Gabriel Indik Date: Tue, 22 Jun 2021 15:49:38 -0400 Subject: [PATCH 23/24] Restore sample Signed-off-by: Peter Broadhurst --- data/member-a/cert.pem | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/data/member-a/cert.pem b/data/member-a/cert.pem index 10b0c5f..f984b1b 100644 --- a/data/member-a/cert.pem +++ b/data/member-a/cert.pem @@ -1,13 +1,17 @@ -----BEGIN CERTIFICATE----- -MIICBDCCAamgAwIBAgIQfrTdLe0oABHDVmSAOeaZJTAKBggqhkjOPQQDAjBJMQsw -CQYDVQQGEwJVUzEQMA4GA1UEBwwHUmFsZWlnaDEQMA4GA1UECgwHS2FsZWlkbzEW -MBQGA1UEAwwNY2EuenpwemIyZmRiNTAiGA8yMDIxMDYyMjE2MjkzOVoYDzIwMjQw -NjIyMTYyOTM5WjB2MQswCQYDVQQGEwJVUzEQMA4GA1UEBwwHUmFsZWlnaDEQMA4G -A1UECgwHS2FsZWlkbzETMBEGA1UECwwKenprcWRmMTZ0ajEuMCwGA1UEAwwlenps -eTV6MDZkcy56enB6YjJmZGI1LmthbGVpZG8ubmV0d29yazBZMBMGByqGSM49AgEG -CCqGSM49AwEHA0IABMucOwEUA23cYMNiAKzxoqcy+e6agAvR6FxFldrzeuJ+4vu+ -eVfDjlVIswRCyedqYQj84H6MteSNHHXhXa9dErGjQjBAMAwGA1UdEwEB/wQCMAAw -DgYDVR0PAQH/BAQDAgWgMCAGA1UdJQEB/wQWMBQGCCsGAQUFBwMCBggrBgEFBQcD -ATAKBggqhkjOPQQDAgNJADBGAiEA6MsZUeTgPThToswHVXNbf4z7NI17fB9qxQCp -aSp9144CIQC7alKXsyL2xMucZi7hNRsQlvfKm5zrBS/8qJAEirYOSQ== +MIICxDCCAawCCQCwQ58VBeUg+TANBgkqhkiG9w0BAQsFADAkMRIwEAYDVQQDDAls +b2NhbGhvc3QxDjAMBgNVBAoMBW9yZy1hMB4XDTIxMDUyMDAyMjUzN1oXDTIyMDUy +MDAyMjUzN1owJDESMBAGA1UEAwwJbG9jYWxob3N0MQ4wDAYDVQQKDAVvcmctYTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK/bZPBvaA+V6HoXNP+oOQaK +DPuLgFO3R7TTIBTCJttELv1lbfuw95Up+/5b4dPnnIF3wwE2GZsiMWJV7RgfDAUi +x7saJJIIa2E8gLrmoaG99w+PYQW41OBz0p54tw4abN5RNsgineu1N5pIJOMJ6cMs +D8IYfWSHDjZZvu3F2YCItSVYZlfmGBCcFCf4HASF6m4lUTZCDRBurjOQrwW9mXJv +hkIa1HEa4l7Nq09d8Bokvieq1vHJUB78kYTR027z+sm4H2o4pXjzBOaV/z2yB7+t +yqWbMjm8aq2m/gmpZTjDBPgX9XLMYHWBZWCpk1iHo/eqw+UeZ6kRZ0RR78Co+IsC +AwEAATANBgkqhkiG9w0BAQsFAAOCAQEAH6rQx8khtfzH93YCstHduvEzUCHSyifF +sDyRLvSmS0qHz1liLJyvWbT9xDQCpDZrxhQKJLN2eZxxlNaH6XBEfHQgi2I43hxu +sd2KZ4wOWOk/HyM9BcibKNMtfHJdgQ5EYRc6OWDY1c8bQQfRJUBzrSJKldqfQjqC +mPEeHXDH++yTg2Vfm7GZiogxqSWn/+ILzHNeWrvr0HJ86Guyg/NPKBxs0uasvgI7 +KqW+fcZ/9Vg6a4e+zRTL8EwYBX6dTSwgt4X9wuwMvt/K0qodgW6I4paqEpVJOS+d +Z6WevHgHwZTr+mS3mJ6Y6u/DBD8/06uPAf3T5d/dC6Xtl4sEFkpNAA== -----END CERTIFICATE----- From 19e15d21513b849b7ffe5647908d409e973523ba Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Tue, 22 Jun 2021 16:15:33 -0400 Subject: [PATCH 24/24] Move TS to devDeps Signed-off-by: Peter Broadhurst --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 4ef373c..2addce3 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,6 @@ "jsrsasign": "^10.3.0", "swagger-ui-express": "^4.1.6", "ts-node": "^9.1.1", - "typescript": "^4.2.4", "uuid": "^8.3.2", "ws": "^7.4.6", "yamljs": "^0.3.0" @@ -42,6 +41,7 @@ "@types/uuid": "^8.3.0", "@types/ws": "^7.4.4", "@types/yamljs": "^0.2.31", - "rimraf": "^3.0.2" + "rimraf": "^3.0.2", + "typescript": "^4.2.4" } }