From 021538792af2c1bca555d5093a6c387ddc504571 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Sat, 19 Jun 2021 22:51:50 -0400 Subject: [PATCH 01/11] Tweak hashing to avoid incomplete uploads Signed-off-by: Peter Broadhurst --- src/handlers/blobs.ts | 20 +++++++++++--------- src/routers/api.ts | 9 ++++++--- src/routers/p2p.ts | 9 ++++++--- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/handlers/blobs.ts b/src/handlers/blobs.ts index abb393e..b89a60d 100644 --- a/src/handlers/blobs.ts +++ b/src/handlers/blobs.ts @@ -45,17 +45,19 @@ export const storeBlob = async (file: IFile, filePath: string) => { const resolvedFilePath = path.join(utils.constants.DATA_DIRECTORY, utils.constants.BLOBS_SUBDIRECTORY, filePath); await fs.mkdir(path.parse(resolvedFilePath).dir, { recursive: true }); let hash = crypto.createHash(utils.constants.TRANSFER_HASH_ALGORITHM); - let hashCalculator = new stream.Transform({ - async transform(chunk, _enc, cb) { - hash.update(chunk); - cb(undefined, chunk); - } - }); const writeStream = createWriteStream(resolvedFilePath); const blobHash = await new Promise((resolve, reject) => { - file.readableStream.on('end', () => { - resolve(hash.digest('hex')); - }).on('error', err => { + let hashCalculator = new stream.Transform({ + transform(chunk, _enc, cb) { + hash.update(chunk); + cb(undefined, chunk); + }, + flush(cb) { + resolve(hash.digest('hex')); + cb(); + } + }); + file.readableStream.on('error', err => { reject(err); }); file.readableStream.pipe(hashCalculator).pipe(writeStream); diff --git a/src/routers/api.ts b/src/routers/api.ts index 3cec523..6d93c7c 100644 --- a/src/routers/api.ts +++ b/src/routers/api.ts @@ -153,7 +153,8 @@ router.post('/messages', async (req, res, next) => { router.head('/blobs/*', async (req, res, next) => { try { - const blobPath = `/${req.params[0]}`; + const p: any = req.params + const blobPath = `/${p[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } @@ -168,7 +169,8 @@ router.head('/blobs/*', async (req, res, next) => { router.get('/blobs/*', async (req, res, next) => { try { - const blobPath = `/${req.params[0]}`; + const p: any = req.params + const blobPath = `/${p[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } @@ -185,7 +187,8 @@ router.get('/blobs/*', async (req, res, next) => { router.put('/blobs/*', async (req, res, next) => { try { - const blobPath = `/${req.params[0]}`; + const p: any = req.params + const blobPath = `/${p[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index 7362d07..2a5724f 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -30,7 +30,8 @@ router.head('/ping', (_req, res) => { router.post('/messages', async (req, res, next) => { try { - const cert = req.client.getPeerCertificate(); + const r: any = req; + const cert = r.client.getPeerCertificate(); const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const message = await utils.extractMessageFromMultipartForm(req); eventEmitter.emit('event', { @@ -46,10 +47,12 @@ router.post('/messages', async (req, res, next) => { router.put('/blobs/*', async (req, res, next) => { try { - const cert = req.client.getPeerCertificate(); + const r: any = req; + const p: any = req.params; + const cert = r.client.getPeerCertificate(); const sender = cert.issuer.O + cert.issuer.OU; const file = await utils.extractFileFromMultipartForm(req); - const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, req.params[0]); + const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, p[0]); const metadata = await blobsHandler.storeBlob(file, blobPath); res.sendStatus(204); eventEmitter.emit('event', { From a51ae508b34a312b8ced52da82a7d8ce10147d3e Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Sun, 20 Jun 2021 02:35:58 -0400 Subject: [PATCH 02/11] Fix sender Signed-off-by: Peter Broadhurst --- src/routers/p2p.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index 2a5724f..7e99791 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -50,7 +50,7 @@ router.put('/blobs/*', async (req, res, next) => { const r: any = req; const p: any = req.params; const cert = r.client.getPeerCertificate(); - const sender = cert.issuer.O + cert.issuer.OU; + const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const file = await utils.extractFileFromMultipartForm(req); const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, p[0]); const metadata = await blobsHandler.storeBlob(file, blobPath); From c5e66621f1ea947da357d39de64037e982f21a60 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 13:35:19 -0400 Subject: [PATCH 03/11] Log events Signed-off-by: Peter Broadhurst --- src/app.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app.ts b/src/app.ts index 030c487..2b10582 100644 --- a/src/app.ts +++ b/src/app.ts @@ -72,6 +72,7 @@ export const start = async () => { messagesEventEmitter.addListener('event', event => eventsHandler.queueEvent(event)); eventsHandler.eventEmitter.addListener('event', event => { + log.info(`Event emitted: ${JSON.stringify(event)}`) if (delegatedWebSocket !== undefined) { delegatedWebSocket.send(JSON.stringify(event)); } From ffbcea92a46841d24ee626e77ec0e0ddf8385b73 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 13:50:28 -0400 Subject: [PATCH 04/11] Log events Signed-off-by: Peter Broadhurst --- src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index 2b10582..ca11110 100644 --- a/src/app.ts +++ b/src/app.ts @@ -72,7 +72,7 @@ export const start = async () => { messagesEventEmitter.addListener('event', event => eventsHandler.queueEvent(event)); eventsHandler.eventEmitter.addListener('event', event => { - log.info(`Event emitted: ${JSON.stringify(event)}`) + log.info(`Event emitted ${event.type} `) if (delegatedWebSocket !== undefined) { delegatedWebSocket.send(JSON.stringify(event)); } From fbe5031a2794ff253ceb9956c2d9f1656cb3e463 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 13:57:25 -0400 Subject: [PATCH 05/11] Correlate events with commits Signed-off-by: Peter Broadhurst --- src/app.ts | 3 ++- src/handlers/blobs.ts | 3 +++ src/handlers/messages.ts | 3 +++ src/lib/interfaces.ts | 6 ++++++ src/routers/p2p.ts | 3 +++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index ca11110..0ba77e6 100644 --- a/src/app.ts +++ b/src/app.ts @@ -72,7 +72,7 @@ export const start = async () => { messagesEventEmitter.addListener('event', event => eventsHandler.queueEvent(event)); eventsHandler.eventEmitter.addListener('event', event => { - log.info(`Event emitted ${event.type} `) + log.info(`Event emitted ${event.type}/${event.id}`) if (delegatedWebSocket !== undefined) { delegatedWebSocket.send(JSON.stringify(event)); } @@ -86,6 +86,7 @@ export const start = async () => { try { const messageContent = JSON.parse(message.toLocaleString()); if (messageContent.action === 'commit') { + log.info(`Event comitted ${event?`${event.type}/${event.id}`:`[no event in flight]`}`) eventsHandler.handleCommit(); } } catch (err) { diff --git a/src/handlers/blobs.ts b/src/handlers/blobs.ts index b89a60d..3dc0310 100644 --- a/src/handlers/blobs.ts +++ b/src/handlers/blobs.ts @@ -26,6 +26,7 @@ import https from 'https'; import { key, cert, ca } from '../lib/cert'; import { createLogger, LogLevelString } from 'bunyan'; import EventEmitter from 'events'; +import { v4 as uuidV4 } from 'uuid'; const log = createLogger({ name: 'handlers/blobs.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); @@ -97,6 +98,7 @@ export const deliverBlob = async ({ blobPath, recipient, recipientURL, requestID httpsAgent }); eventEmitter.emit('event', { + id: uuidV4(), type: 'blob-delivered', path: blobPath, recipient, @@ -105,6 +107,7 @@ export const deliverBlob = async ({ blobPath, recipient, recipientURL, requestID log.trace(`Blob delivered`); } catch (err) { eventEmitter.emit('event', { + id: uuidV4(), type: 'blob-failed', path: blobPath, recipient, diff --git a/src/handlers/messages.ts b/src/handlers/messages.ts index d15fdc8..601c8a5 100644 --- a/src/handlers/messages.ts +++ b/src/handlers/messages.ts @@ -21,6 +21,7 @@ import { IMessageDeliveredEvent, IMessageFailedEvent, MessageTask } from '../lib import FormData from 'form-data'; import EventEmitter from 'events'; import { createLogger, LogLevelString } from 'bunyan'; +import { v4 as uuidV4 } from 'uuid'; const log = createLogger({ name: 'handlers/messages.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); @@ -55,6 +56,7 @@ export const deliverMessage = async ({ message, recipient, recipientURL, request httpsAgent }); eventEmitter.emit('event', { + id: uuidV4(), type: 'message-delivered', message, recipient, @@ -63,6 +65,7 @@ export const deliverMessage = async ({ message, recipient, recipientURL, request log.trace(`Message delivered`); } catch(err) { eventEmitter.emit('event', { + id: uuidV4(), type: 'message-failed', message, recipient, diff --git a/src/lib/interfaces.ts b/src/lib/interfaces.ts index 0180f18..497b917 100644 --- a/src/lib/interfaces.ts +++ b/src/lib/interfaces.ts @@ -46,18 +46,21 @@ export type OutboundEvent = IBlobFailedEvent export interface IMessageReceivedEvent { + id: string type: 'message-received' sender: string message: string } export interface IMessageDeliveredEvent { + id: string type: 'message-delivered' recipient: string message: string } export interface IMessageFailedEvent { + id: string type: 'message-failed' recipient: string message: string @@ -65,6 +68,7 @@ export interface IMessageFailedEvent { } export interface IBlobReceivedEvent { + id: string type: 'blob-received' sender: string path: string @@ -72,12 +76,14 @@ export interface IBlobReceivedEvent { } export interface IBlobDeliveredEvent { + id: string type: 'blob-delivered' recipient: string path: string } export interface IBlobFailedEvent { + id: string type: 'blob-failed' recipient: string path: string diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index 7e99791..c5407aa 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -20,6 +20,7 @@ import * as blobsHandler from '../handlers/blobs'; import path from 'path'; import { EventEmitter } from 'events'; import { IBlobReceivedEvent, IMessageReceivedEvent } from '../lib/interfaces'; +import { v4 as uuidV4 } from 'uuid'; export const router = Router(); export const eventEmitter = new EventEmitter(); @@ -35,6 +36,7 @@ router.post('/messages', async (req, res, next) => { const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const message = await utils.extractMessageFromMultipartForm(req); eventEmitter.emit('event', { + id: uuidV4(), type: 'message-received', sender, message @@ -56,6 +58,7 @@ router.put('/blobs/*', async (req, res, next) => { const metadata = await blobsHandler.storeBlob(file, blobPath); res.sendStatus(204); eventEmitter.emit('event', { + id: uuidV4(), type: 'blob-received', sender, path: blobPath, From 0f7dd548e5a00094269095140399d12bf2223f0e Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 14:06:45 -0400 Subject: [PATCH 06/11] Simplify logger to avoid JSON output in firefly-cli Signed-off-by: Peter Broadhurst --- package.json | 4 +-- src/app.ts | 26 +++++++------- src/handlers/blobs.ts | 20 +++++------ src/handlers/events.ts | 4 +-- src/handlers/messages.ts | 14 ++++---- src/index.ts | 5 ++- src/lib/cert.ts | 4 +-- src/lib/logger.ts | 73 ++++++++++++++++++++++++++++++++++++++++ src/lib/utils.ts | 10 +++--- 9 files changed, 116 insertions(+), 44 deletions(-) create mode 100644 src/lib/logger.ts diff --git a/package.json b/package.json index dbbffb9..4ef373c 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,6 @@ "dependencies": { "ajv": "^8.5.0", "axios": "^0.21.1", - "bunyan": "^1.8.15", "busboy": "^0.3.1", "express": "^4.17.1", "form-data": "^4.0.0", @@ -42,6 +41,7 @@ "@types/swagger-ui-express": "^4.1.2", "@types/uuid": "^8.3.0", "@types/ws": "^7.4.4", - "@types/yamljs": "^0.2.31" + "@types/yamljs": "^0.2.31", + "rimraf": "^3.0.2" } } diff --git a/src/app.ts b/src/app.ts index 0ba77e6..110fa92 100644 --- a/src/app.ts +++ b/src/app.ts @@ -15,24 +15,24 @@ // limitations under the License. import express from 'express'; -import https, { Server } from 'https'; import http from 'http'; +import https, { Server } from 'https'; +import path from 'path'; +import swaggerUi from 'swagger-ui-express'; import WebSocket from 'ws'; -import { init as initConfig, config } from './lib/config'; -import { init as initCert, genTLSContext, loadCAs } from './lib/cert'; -import { createLogger, LogLevelString } from 'bunyan'; -import * as utils from './lib/utils'; -import { router as apiRouter, setAddTLSContext } from './routers/api'; -import { router as p2pRouter, eventEmitter as p2pEventEmitter } from './routers/p2p'; -import RequestError, { errorHandler } from './lib/request-error'; -import * as eventsHandler from './handlers/events' +import YAML from 'yamljs'; import { eventEmitter as blobsEventEmitter } from './handlers/blobs'; +import * as eventsHandler from './handlers/events'; import { eventEmitter as messagesEventEmitter } from './handlers/messages'; -import swaggerUi from 'swagger-ui-express'; -import YAML from 'yamljs'; -import path from 'path'; +import { genTLSContext, init as initCert, loadCAs } from './lib/cert'; +import { config, init as initConfig } from './lib/config'; +import { Logger } from './lib/logger'; +import RequestError, { errorHandler } from './lib/request-error'; +import * as utils from './lib/utils'; +import { router as apiRouter, setAddTLSContext } from './routers/api'; +import { eventEmitter as p2pEventEmitter, router as p2pRouter } from './routers/p2p'; -const log = createLogger({ name: 'app.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger("app.ts"); const swaggerDocument = YAML.load(path.join(__dirname, './swagger.yaml')); diff --git a/src/handlers/blobs.ts b/src/handlers/blobs.ts index 3dc0310..a1ecaa6 100644 --- a/src/handlers/blobs.ts +++ b/src/handlers/blobs.ts @@ -14,21 +14,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { promises as fs, createReadStream, createWriteStream } from 'fs'; -import path from 'path'; -import * as utils from '../lib/utils'; -import { BlobTask, IBlobDeliveredEvent, IBlobFailedEvent, IFile, IMetadata } from "../lib/interfaces"; -import stream from 'stream'; -import RequestError from '../lib/request-error'; import crypto from 'crypto'; +import EventEmitter from 'events'; import FormData from 'form-data'; +import { createReadStream, createWriteStream, promises as fs } from 'fs'; import https from 'https'; -import { key, cert, ca } from '../lib/cert'; -import { createLogger, LogLevelString } from 'bunyan'; -import EventEmitter from 'events'; +import path from 'path'; +import stream from 'stream'; import { v4 as uuidV4 } from 'uuid'; +import { ca, cert, key } from '../lib/cert'; +import { BlobTask, IBlobDeliveredEvent, IBlobFailedEvent, IFile, IMetadata } from "../lib/interfaces"; +import { Logger } from '../lib/logger'; +import RequestError from '../lib/request-error'; +import * as utils from '../lib/utils'; -const log = createLogger({ name: 'handlers/blobs.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger("handlers/blobs.ts") let blobQueue: BlobTask[] = []; let sending = false; diff --git a/src/handlers/events.ts b/src/handlers/events.ts index deedf57..5e972e0 100644 --- a/src/handlers/events.ts +++ b/src/handlers/events.ts @@ -14,12 +14,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { createLogger, LogLevelString } from "bunyan"; import EventEmitter from "events"; import { OutboundEvent } from "../lib/interfaces"; +import { Logger } from "../lib/logger"; import * as utils from '../lib/utils'; -const log = createLogger({ name: 'handlers/events.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger("handlers/events.ts") let eventQueue: OutboundEvent[] = []; export const eventEmitter = new EventEmitter(); diff --git a/src/handlers/messages.ts b/src/handlers/messages.ts index 601c8a5..0b35645 100644 --- a/src/handlers/messages.ts +++ b/src/handlers/messages.ts @@ -14,16 +14,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -import https from 'https'; -import * as utils from '../lib/utils'; -import { key, cert, ca } from '../lib/cert'; -import { IMessageDeliveredEvent, IMessageFailedEvent, MessageTask } from '../lib/interfaces'; -import FormData from 'form-data'; import EventEmitter from 'events'; -import { createLogger, LogLevelString } from 'bunyan'; +import FormData from 'form-data'; +import https from 'https'; import { v4 as uuidV4 } from 'uuid'; +import { ca, cert, key } from '../lib/cert'; +import { IMessageDeliveredEvent, IMessageFailedEvent, MessageTask } from '../lib/interfaces'; +import { Logger } from '../lib/logger'; +import * as utils from '../lib/utils'; -const log = createLogger({ name: 'handlers/messages.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger('handlers/messages.ts') let messageQueue: MessageTask[] = []; let sending = false; diff --git a/src/index.ts b/src/index.ts index e13729d..836b903 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,11 +14,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { createLogger, LogLevelString } from 'bunyan'; -import * as utils from './lib/utils'; import { start } from './app'; +import { Logger } from './lib/logger'; -const log = createLogger({ name: 'index.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger("index.ts") start().catch(err => { log.error(`Failed to FireFly Data Exchange ${err}`); diff --git a/src/lib/cert.ts b/src/lib/cert.ts index 52b550e..44c3364 100644 --- a/src/lib/cert.ts +++ b/src/lib/cert.ts @@ -17,9 +17,9 @@ import * as utils from '../lib/utils'; import { promises as fs } from 'fs'; import path from 'path'; -import { createLogger, LogLevelString } from 'bunyan'; +import { Logger } from './logger'; -const log = createLogger({ name: 'lib/certs.ts', level: utils.constants.LOG_LEVEL as LogLevelString }); +const log = new Logger('lib/certs.ts') export let key: string; export let cert: string; diff --git a/src/lib/logger.ts b/src/lib/logger.ts new file mode 100644 index 0000000..cbb11f2 --- /dev/null +++ b/src/lib/logger.ts @@ -0,0 +1,73 @@ +// Copyright © 2021 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +"use strict"; + +const LEVEL_NONE = 0; +const LEVEL_ERROR = 1; +const LEVEL_WARN = 2; +const LEVEL_INFO = 3; +const LEVEL_DEBUG = 4; +const LEVEL_TRACE = 5; + +const LEVEL_TAGS = { + [LEVEL_NONE]: 'NONE', + [LEVEL_ERROR]: 'ERROR', + [LEVEL_WARN]: 'WARN ', + [LEVEL_INFO]: 'INFO ', + [LEVEL_DEBUG]: 'DEBUG', + [LEVEL_TRACE]: 'TRACE', +}; + +let logLevel = LEVEL_ERROR; + +export function setLogLevel(level?: string) { + if (!level) level = process.env.LOG_LEVEL || 'info'; + for (let [l,t] of Object.entries(LEVEL_TAGS)) { + if (t.trim().toLowerCase() === level.trim().toLowerCase()) { + logLevel = Number(l); + } + } +} + +export class Logger { + + constructor(private loggerName?: string) {} + + error(...args: any[]) { logLevel >= LEVEL_ERROR && this.log('ERROR', ...args); } + warn(...args: any[]) { logLevel >= LEVEL_WARN && this.log('WARN ', ...args); } + info(...args: any[]) { logLevel >= LEVEL_INFO && this.log('INFO ', ...args); } + debug(...args: any[]) { logLevel >= LEVEL_DEBUG && this.log('DEBUG', ...args); } + trace(...args: any[]) { logLevel >= LEVEL_TRACE && this.log('TRACE', ...args); } + + private log(level: string, ...args: any[]) { + const logArgs = []; + for (const arg of args) { + // Special handling of axios errors to avoid massive dumps in log + if (arg?.isAxiosError) { + let data = arg.response?.data; + data = data?.on ? '[stream]' : JSON.stringify(data); + logArgs.push(`HTTP [${arg.response?.status}] ${arg.message}: ${data}`) + } else { + logArgs.push(arg); + } + } + console.log(`${new Date().toISOString()} [${level}]:`, ...logArgs, this.loggerName); + } + +} + +setLogLevel(); diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 48e8ad9..a63b7e7 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -14,14 +14,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +import axios, { AxiosRequestConfig } from 'axios'; +import Busboy from 'busboy'; import { Request } from 'express'; import { promises as fs } from 'fs'; +import { X509 } from 'jsrsasign'; import { ICertData, IFile } from './interfaces'; +import { Logger } from './logger'; import RequestError from './request-error'; -import Busboy from 'busboy'; -import axios, { AxiosRequestConfig } from 'axios'; -import { createLogger, LogLevelString } from 'bunyan'; -import { X509 } from 'jsrsasign'; export const constants = { LOG_LEVEL: process.env.LOG_LEVEL || 'info', @@ -41,7 +41,7 @@ export const constants = { HASH_HEADER_NAME: 'dx-hash', LAST_UPDATE_HEADER_NAME: 'dx-last-update' }; -const log = createLogger({ name: 'utils.ts', level: constants.LOG_LEVEL as LogLevelString }); +const log = new Logger('utils.ts') export const regexp = { FILE_KEY: /^(\/[a-z0-9\+\-\_\.]+)+$/, From e288fe46d408f1a765ae244759533e3347ad495c Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 14:22:37 -0400 Subject: [PATCH 07/11] Logging for TLS context addition Signed-off-by: Peter Broadhurst --- src/app.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app.ts b/src/app.ts index 110fa92..0eefdcf 100644 --- a/src/app.ts +++ b/src/app.ts @@ -43,6 +43,7 @@ let delegatedWebSocket: WebSocket | undefined = undefined; export const addTLSContext = async (hostname: string) => { await loadCAs() // The most recent context wins (per the Node.js spec), so to get a reload we just add a wildcard context + log.info(`Adding TLS context for new peer '%s'`, hostname) p2pServer.addContext(hostname, genTLSContext()) }; setAddTLSContext(addTLSContext) From eb971587e16b90ea82412fd07a19d62d877b5eda Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Mon, 21 Jun 2021 14:24:23 -0400 Subject: [PATCH 08/11] Logging for TLS context addition Signed-off-by: Peter Broadhurst --- src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index 0eefdcf..43a2717 100644 --- a/src/app.ts +++ b/src/app.ts @@ -43,7 +43,7 @@ let delegatedWebSocket: WebSocket | undefined = undefined; export const addTLSContext = async (hostname: string) => { await loadCAs() // The most recent context wins (per the Node.js spec), so to get a reload we just add a wildcard context - log.info(`Adding TLS context for new peer '%s'`, hostname) + log.info(`Adding TLS context for new peer '${hostname}'`) p2pServer.addContext(hostname, genTLSContext()) }; setAddTLSContext(addTLSContext) From 26b4eb0e1ca6a5b639193dccf39bfcce1f277669 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Tue, 22 Jun 2021 09:01:12 -0400 Subject: [PATCH 09/11] Cleaner fix for type issue Signed-off-by: Peter Broadhurst --- src/custom.d.ts | 6 +++++- src/routers/api.ts | 17 +++++++---------- src/routers/p2p.ts | 11 +++++------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/custom.d.ts b/src/custom.d.ts index 1ba0bf2..a4f21be 100644 --- a/src/custom.d.ts +++ b/src/custom.d.ts @@ -16,9 +16,13 @@ export {} -declare global{ +declare global { namespace Express { export interface Request { + params: { + [s: string]: string, + 0: string, + }, client: { authorized: boolean getCertificate: () => { diff --git a/src/routers/api.ts b/src/routers/api.ts index 6d93c7c..755d4da 100644 --- a/src/routers/api.ts +++ b/src/routers/api.ts @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Router } from 'express'; +import { Router, Request } from 'express'; import * as blobsHandler from '../handlers/blobs'; import * as messagesHandler from '../handlers/messages'; import * as utils from '../lib/utils'; @@ -151,10 +151,9 @@ router.post('/messages', async (req, res, next) => { } }); -router.head('/blobs/*', async (req, res, next) => { +router.head('/blobs/*', async (req: Request, res, next) => { try { - const p: any = req.params - const blobPath = `/${p[0]}`; + const blobPath = `/${req.params[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } @@ -167,10 +166,9 @@ router.head('/blobs/*', async (req, res, next) => { } }); -router.get('/blobs/*', async (req, res, next) => { +router.get('/blobs/*', async (req: Request, res, next) => { try { - const p: any = req.params - const blobPath = `/${p[0]}`; + const blobPath = `/${req.params[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } @@ -185,10 +183,9 @@ router.get('/blobs/*', async (req, res, next) => { } }); -router.put('/blobs/*', async (req, res, next) => { +router.put('/blobs/*', async (req: Request, res, next) => { try { - const p: any = req.params - const blobPath = `/${p[0]}`; + const blobPath = `/${req.params[0]}`; if (!utils.regexp.FILE_KEY.test(blobPath) || utils.regexp.CONSECUTIVE_DOTS.test(blobPath)) { throw new RequestError('Invalid path', 400); } diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index c5407aa..a928c87 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -14,13 +14,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Router } from 'express'; +import { Router, Request } from 'express'; import * as utils from '../lib/utils'; import * as blobsHandler from '../handlers/blobs'; import path from 'path'; import { EventEmitter } from 'events'; import { IBlobReceivedEvent, IMessageReceivedEvent } from '../lib/interfaces'; import { v4 as uuidV4 } from 'uuid'; +import '../custom' export const router = Router(); export const eventEmitter = new EventEmitter(); @@ -47,14 +48,12 @@ router.post('/messages', async (req, res, next) => { } }); -router.put('/blobs/*', async (req, res, next) => { +router.put('/blobs/*', async (req: Request, res, next) => { try { - const r: any = req; - const p: any = req.params; - const cert = r.client.getPeerCertificate(); + const cert = req.client.getPeerCertificate(); const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const file = await utils.extractFileFromMultipartForm(req); - const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, p[0]); + const blobPath = path.join(utils.constants.RECEIVED_BLOBS_SUBDIRECTORY, sender, req.params[0]); const metadata = await blobsHandler.storeBlob(file, blobPath); res.sendStatus(204); eventEmitter.emit('event', { From bdc5e995aff190957323f87db39ef366447d11dd Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Tue, 22 Jun 2021 09:05:30 -0400 Subject: [PATCH 10/11] Missed one tidy up Signed-off-by: Peter Broadhurst --- src/routers/p2p.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index a928c87..161bd56 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -30,10 +30,9 @@ router.head('/ping', (_req, res) => { res.sendStatus(204); }); -router.post('/messages', async (req, res, next) => { +router.post('/messages', async (req: Request, res, next) => { try { - const r: any = req; - const cert = r.client.getPeerCertificate(); + const cert = req.client.getPeerCertificate(); const sender = utils.getPeerID(cert.issuer.O, cert.issuer.OU); const message = await utils.extractMessageFromMultipartForm(req); eventEmitter.emit('event', { From 6c7767d6f2bf5ca1b812a67da2ee4d4539d35f0d Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Tue, 22 Jun 2021 09:52:42 -0400 Subject: [PATCH 11/11] Do not import custom Signed-off-by: Peter Broadhurst --- src/routers/p2p.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/routers/p2p.ts b/src/routers/p2p.ts index 161bd56..39ededf 100644 --- a/src/routers/p2p.ts +++ b/src/routers/p2p.ts @@ -21,7 +21,6 @@ import path from 'path'; import { EventEmitter } from 'events'; import { IBlobReceivedEvent, IMessageReceivedEvent } from '../lib/interfaces'; import { v4 as uuidV4 } from 'uuid'; -import '../custom' export const router = Router(); export const eventEmitter = new EventEmitter();