From 9bd048c2b6472c4e409fc8994a21035cee5370cf Mon Sep 17 00:00:00 2001 From: Kris West Date: Mon, 19 May 2025 16:58:28 +0100 Subject: [PATCH 01/76] feat(key on repo url): start of db refactor (WIP) --- src/db/file/repo.ts | 14 ++++++++++++++ src/db/index.ts | 14 +++++++++++++- src/db/mongo/repo.ts | 5 +++++ src/proxy/processors/push-action/pullRemote.ts | 2 +- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index fd7218c15..4f4b14fd4 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -49,6 +49,20 @@ export const getRepo = async (name: string) => { }); }; +export const getRepoByURL = async (repoURL: string) => { + return new Promise((resolve, reject) => { + db.findOne({ url: repoURL.toLowerCase() }, (err: Error | null, doc: Repo) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ + if (err) { + reject(err); + } else { + resolve(doc); + } + }); + }); +}; + export const createRepo = async (repo: Repo) => { if (isBlank(repo.project)) { throw new Error('Project name cannot be empty'); diff --git a/src/db/index.ts b/src/db/index.ts index ff1189f1b..e766b871f 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -56,6 +56,18 @@ export const createUser = async ( await sink.createUser(data); }; +export const getRepoByUrl = async (repoUrl: string) => { + const response = await sink.getRepoByUrl(repoUrl); + // backwards compatibility + if (!response) { + // parse github URLs into org and repo names and fallback to legacy retrieval by repo name + const repoName = 'some regex magic goes here'; + + return sink.getRepo(repoName); + } + return response; +}; + export const { authorise, reject, @@ -69,8 +81,8 @@ export const { getUsers, deleteUser, updateUser, - getRepos, getRepo, + getRepos, createRepo, addUserCanPush, addUserCanAuthorise, diff --git a/src/db/mongo/repo.ts b/src/db/mongo/repo.ts index f299f907a..47485f85d 100644 --- a/src/db/mongo/repo.ts +++ b/src/db/mongo/repo.ts @@ -18,6 +18,11 @@ export const getRepo = async (name: string) => { return collection.findOne({ name: { $eq: name } }); }; +export const getRepoByUrl = async (repoUrl: string) => { + const collection = await connect(collectionName); + return collection.findOne({ name: { $eq: repoUrl.toLowerCase() } }); +}; + export const createRepo = async (repo: Repo) => { repo.name = repo.name.toLowerCase(); console.log(`creating new repo ${JSON.stringify(repo)}`); diff --git a/src/proxy/processors/push-action/pullRemote.ts b/src/proxy/processors/push-action/pullRemote.ts index 2f7c808a2..991a62ca9 100644 --- a/src/proxy/processors/push-action/pullRemote.ts +++ b/src/proxy/processors/push-action/pullRemote.ts @@ -22,7 +22,7 @@ const exec = async (req: any, action: Action): Promise => { } const cmd = `git clone ${action.url}`; - step.log(`Exectuting ${cmd}`); + step.log(`Executing ${cmd}`); const authHeader = req.headers?.authorization; const [username, password] = Buffer.from(authHeader.split(' ')[1], 'base64') From 4891468612a14d3d61c2857f77abfa8fd712783f Mon Sep 17 00:00:00 2001 From: Kris West Date: Mon, 19 May 2025 17:21:15 +0100 Subject: [PATCH 02/76] feat(key on repo url): consolidate validation for repo creation to reduce duplication --- src/db/file/repo.ts | 21 --------------------- src/db/index.ts | 36 +++++++++++++++++++++++++++++++----- src/db/mongo/repo.ts | 22 ---------------------- 3 files changed, 31 insertions(+), 48 deletions(-) diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index 4f4b14fd4..348361f4a 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -14,10 +14,6 @@ const db = new Datastore({ filename: './.data/db/repos.db', autoload: true }); db.ensureIndex({ fieldName: 'name', unique: false }); db.setAutocompactionInterval(COMPACTION_INTERVAL); -const isBlank = (str: string) => { - return !str || /^\s*$/.test(str); -}; - export const getRepos = async (query: any = {}) => { if (query?.name) { query.name = query.name.toLowerCase(); @@ -64,23 +60,6 @@ export const getRepoByURL = async (repoURL: string) => { }; export const createRepo = async (repo: Repo) => { - if (isBlank(repo.project)) { - throw new Error('Project name cannot be empty'); - } - if (isBlank(repo.name)) { - throw new Error('Repository name cannot be empty'); - } else { - repo.name = repo.name.toLowerCase(); - } - if (isBlank(repo.url)) { - throw new Error('URL cannot be empty'); - } - - repo.users = { - canPush: [], - canAuthorise: [], - }; - return new Promise((resolve, reject) => { db.insert(repo, (err, doc) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query diff --git a/src/db/index.ts b/src/db/index.ts index e766b871f..cfab3d979 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -1,3 +1,5 @@ +import { Repo } from './types'; + const bcrypt = require('bcryptjs'); const config = require('../config'); let sink: any; @@ -7,6 +9,10 @@ if (config.getDatabase().type === 'mongo') { sink = require('./file'); } +const isBlank = (str: string) => { + return !str || /^\s*$/.test(str); +}; + export const createUser = async ( username: string, password: string, @@ -32,17 +38,17 @@ export const createUser = async ( admin: admin, }; - if (username === undefined || username === null || username === '') { + if (isBlank(username)) { const errorMessage = `username ${username} cannot be empty`; throw new Error(errorMessage); } - if (gitAccount === undefined || gitAccount === null || gitAccount === '') { + if (isBlank(gitAccount)) { const errorMessage = `GitAccount ${gitAccount} cannot be empty`; throw new Error(errorMessage); } - if (email === undefined || email === null || email === '') { + if (isBlank(email)) { const errorMessage = `Email ${email} cannot be empty`; throw new Error(errorMessage); } @@ -56,6 +62,28 @@ export const createUser = async ( await sink.createUser(data); }; +export const createRepo = async (repo: Repo) => { + repo.name = repo.name.toLowerCase(); + repo.users = { + canPush: [], + canAuthorise: [], + }; + + console.log(`creating new repo ${JSON.stringify(repo)}`); + + if (isBlank(repo.project)) { + throw new Error('Project name cannot be empty'); + } + if (isBlank(repo.name)) { + throw new Error('Repository name cannot be empty'); + } + if (isBlank(repo.url)) { + throw new Error('URL cannot be empty'); + } + + return sink.createRepo(repo); +}; + export const getRepoByUrl = async (repoUrl: string) => { const response = await sink.getRepoByUrl(repoUrl); // backwards compatibility @@ -81,9 +109,7 @@ export const { getUsers, deleteUser, updateUser, - getRepo, getRepos, - createRepo, addUserCanPush, addUserCanAuthorise, removeUserCanAuthorise, diff --git a/src/db/mongo/repo.ts b/src/db/mongo/repo.ts index 47485f85d..0d3ded4f9 100644 --- a/src/db/mongo/repo.ts +++ b/src/db/mongo/repo.ts @@ -3,10 +3,6 @@ import { Repo } from '../types'; const connect = require('./helper').connect; const collectionName = 'repos'; -const isBlank = (str: string) => { - return !str || /^\s*$/.test(str); -}; - export const getRepos = async (query: any = {}) => { const collection = await connect(collectionName); return collection.find(query).toArray(); @@ -24,24 +20,6 @@ export const getRepoByUrl = async (repoUrl: string) => { }; export const createRepo = async (repo: Repo) => { - repo.name = repo.name.toLowerCase(); - console.log(`creating new repo ${JSON.stringify(repo)}`); - - if (isBlank(repo.project)) { - throw new Error('Project name cannot be empty'); - } - if (isBlank(repo.name)) { - throw new Error('Repository name cannot be empty'); - } - if (isBlank(repo.url)) { - throw new Error('URL cannot be empty'); - } - - repo.users = { - canPush: [], - canAuthorise: [], - }; - const collection = await connect(collectionName); await collection.insertOne(repo); console.log(`created new repo ${JSON.stringify(repo)}`); From 1ad07ab0e877487611224d7e2f063159a9830dd9 Mon Sep 17 00:00:00 2001 From: Raimund Hook Date: Tue, 20 May 2025 11:42:21 +0100 Subject: [PATCH 03/76] feat(key on repo url): add fallback to non-url repo matching Signed-off-by: Raimund Hook --- src/db/file/index.ts | 1 + src/db/file/repo.ts | 4 ++-- src/db/index.ts | 11 ++++++++++- src/db/mongo/index.ts | 1 + test/testDb.test.js | 6 ++++++ 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/db/file/index.ts b/src/db/file/index.ts index 6ac1c2088..18c36a06b 100644 --- a/src/db/file/index.ts +++ b/src/db/file/index.ts @@ -17,6 +17,7 @@ export const { export const { getRepos, getRepo, + getRepoByUrl, createRepo, addUserCanPush, addUserCanAuthorise, diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index 348361f4a..fa9ad9e91 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -45,9 +45,9 @@ export const getRepo = async (name: string) => { }); }; -export const getRepoByURL = async (repoURL: string) => { +export const getRepoByUrl = async (repoUrl: string) => { return new Promise((resolve, reject) => { - db.findOne({ url: repoURL.toLowerCase() }, (err: Error | null, doc: Repo) => { + db.findOne({ url: repoUrl.toLowerCase() }, (err: Error | null, doc: Repo) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { diff --git a/src/db/index.ts b/src/db/index.ts index cfab3d979..885b0742b 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -89,7 +89,16 @@ export const getRepoByUrl = async (repoUrl: string) => { // backwards compatibility if (!response) { // parse github URLs into org and repo names and fallback to legacy retrieval by repo name - const repoName = 'some regex magic goes here'; + const regex = /.*\/([\w_.-]+?)(\.git)?$/m; + const match = regex.exec(repoUrl); + let repoName = ''; + + if (match && match[1]) { + repoName = match[1]; + } else { + const errorMessage = `Cannot parse repository name from ${repoUrl}`; + throw new Error(errorMessage); + } return sink.getRepo(repoName); } diff --git a/src/db/mongo/index.ts b/src/db/mongo/index.ts index a6d7ce6b2..b190bed49 100644 --- a/src/db/mongo/index.ts +++ b/src/db/mongo/index.ts @@ -20,6 +20,7 @@ export const { export const { getRepos, getRepo, + getRepoByUrl, createRepo, addUserCanPush, addUserCanAuthorise, diff --git a/test/testDb.test.js b/test/testDb.test.js index 45f329e77..9bc825692 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -109,6 +109,12 @@ describe('Database clients', async () => { expect(cleanRepo).to.eql(TEST_REPO); }); + it('should be able to retrieve a repo by url', async function () { + const repo = await db.getRepoByUrl(TEST_REPO.url); + const cleanRepo = cleanResponseData(TEST_REPO, repo); + expect(cleanRepo).to.eql(TEST_REPO); + }); + it('should be able to delete a repo', async function () { await db.deleteRepo(TEST_REPO.name); const repos = await db.getRepos(); From 5eeccc81022fa1557095c3d78fd5300e2b20924f Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 20 May 2025 11:48:23 +0100 Subject: [PATCH 04/76] feat(key on repo url): switch repo detail used in queries from name to url (WIP) --- src/db/file/pushes.ts | 10 ++++----- src/db/file/repo.ts | 47 ++++++++++++++++++------------------------ src/db/mongo/pushes.ts | 10 ++++----- src/db/mongo/repo.ts | 41 +++++++++++++++--------------------- 4 files changed, 45 insertions(+), 63 deletions(-) diff --git a/src/db/file/pushes.ts b/src/db/file/pushes.ts index 2d8c41efb..e96004dcf 100644 --- a/src/db/file/pushes.ts +++ b/src/db/file/pushes.ts @@ -132,14 +132,13 @@ export const cancel = async (id: string) => { export const canUserCancelPush = async (id: string, user: string) => { return new Promise(async (resolve) => { - const pushDetail = await getPush(id); - if (!pushDetail) { + const action = await getPush(id); + if (!action) { resolve(false); return; } - const repoName = pushDetail.repoName.replace('.git', ''); - const isAllowed = await repo.isUserPushAllowed(repoName, user); + const isAllowed = await repo.isUserPushAllowed(action.url, user); if (isAllowed) { resolve(true); @@ -156,8 +155,7 @@ export const canUserApproveRejectPush = async (id: string, user: string) => { resolve(false); return; } - const repoName = action.repoName.replace('.git', ''); - const isAllowed = await repo.canUserApproveRejectPushRepo(repoName, user); + const isAllowed = await repo.canUserApproveRejectPushRepo(action.url, user); resolve(isAllowed); }); diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index fa9ad9e91..d626132c4 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -45,9 +45,9 @@ export const getRepo = async (name: string) => { }); }; -export const getRepoByUrl = async (repoUrl: string) => { +export const getRepoByUrl = async (repoURL: string) => { return new Promise((resolve, reject) => { - db.findOne({ url: repoUrl.toLowerCase() }, (err: Error | null, doc: Repo) => { + db.findOne({ url: repoURL }, (err: Error | null, doc: Repo) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -73,11 +73,10 @@ export const createRepo = async (repo: Repo) => { }); }; -export const addUserCanPush = async (name: string, user: string) => { - name = name.toLowerCase(); +export const addUserCanPush = async (repoUrl: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepo(name); + const repo = await getRepoByUrl(repoUrl); if (!repo) { reject(new Error('Repo not found')); return; @@ -90,7 +89,7 @@ export const addUserCanPush = async (name: string, user: string) => { repo.users.canPush.push(user); const options = { multi: false, upsert: false }; - db.update({ name: name }, repo, options, (err) => { + db.update({ url: repoUrl }, repo, options, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -102,11 +101,10 @@ export const addUserCanPush = async (name: string, user: string) => { }); }; -export const addUserCanAuthorise = async (name: string, user: string) => { - name = name.toLowerCase(); +export const addUserCanAuthorise = async (repoUrl: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepo(name); + const repo = await getRepoByUrl(repoUrl); if (!repo) { reject(new Error('Repo not found')); return; @@ -120,7 +118,7 @@ export const addUserCanAuthorise = async (name: string, user: string) => { repo.users.canAuthorise.push(user); const options = { multi: false, upsert: false }; - db.update({ name: name }, repo, options, (err) => { + db.update({ url: repoUrl }, repo, options, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -132,11 +130,10 @@ export const addUserCanAuthorise = async (name: string, user: string) => { }); }; -export const removeUserCanAuthorise = async (name: string, user: string) => { - name = name.toLowerCase(); +export const removeUserCanAuthorise = async (repoUrl: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepo(name); + const repo = await getRepoByUrl(repoUrl); if (!repo) { reject(new Error('Repo not found')); return; @@ -145,7 +142,7 @@ export const removeUserCanAuthorise = async (name: string, user: string) => { repo.users.canAuthorise = repo.users.canAuthorise.filter((x: string) => x != user); const options = { multi: false, upsert: false }; - db.update({ name: name }, repo, options, (err) => { + db.update({ url: repoUrl }, repo, options, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -157,11 +154,10 @@ export const removeUserCanAuthorise = async (name: string, user: string) => { }); }; -export const removeUserCanPush = async (name: string, user: string) => { - name = name.toLowerCase(); +export const removeUserCanPush = async (repoUrl: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepo(name); + const repo = await getRepo(repoUrl); if (!repo) { reject(new Error('Repo not found')); return; @@ -170,7 +166,7 @@ export const removeUserCanPush = async (name: string, user: string) => { repo.users.canPush = repo.users.canPush.filter((x) => x != user); const options = { multi: false, upsert: false }; - db.update({ name: name }, repo, options, (err) => { + db.update({ url: repoUrl }, repo, options, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -182,10 +178,9 @@ export const removeUserCanPush = async (name: string, user: string) => { }); }; -export const deleteRepo = async (name: string) => { - name = name.toLowerCase(); +export const deleteRepo = async (repoUrl: string) => { return new Promise((resolve, reject) => { - db.remove({ name: name }, (err) => { + db.remove({ url: repoUrl }, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -197,11 +192,10 @@ export const deleteRepo = async (name: string) => { }); }; -export const isUserPushAllowed = async (name: string, user: string) => { - name = name.toLowerCase(); +export const isUserPushAllowed = async (repoUrl: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve) => { - const repo = await getRepo(name); + const repo = await getRepoByUrl(repoUrl); if (!repo) { resolve(false); return; @@ -218,12 +212,11 @@ export const isUserPushAllowed = async (name: string, user: string) => { }); }; -export const canUserApproveRejectPushRepo = async (name: string, user: string) => { - name = name.toLowerCase(); +export const canUserApproveRejectPushRepo = async (repoUrl: string, user: string) => { user = user.toLowerCase(); console.log(`checking if user ${user} can approve/reject for ${name}`); return new Promise(async (resolve) => { - const repo = await getRepo(name); + const repo = await getRepoByUrl(repoUrl); if (!repo) { resolve(false); return; diff --git a/src/db/mongo/pushes.ts b/src/db/mongo/pushes.ts index 8778bdf73..c461bfb2c 100644 --- a/src/db/mongo/pushes.ts +++ b/src/db/mongo/pushes.ts @@ -108,8 +108,7 @@ export const canUserApproveRejectPush = async (id: string, user: string) => { return; } - const repoName = action.repoName.replace('.git', ''); - const isAllowed = await repo.canUserApproveRejectPushRepo(repoName, user); + const isAllowed = await repo.canUserApproveRejectPushRepo(action.url, user); resolve(isAllowed); }); @@ -117,14 +116,13 @@ export const canUserApproveRejectPush = async (id: string, user: string) => { export const canUserCancelPush = async (id: string, user: string) => { return new Promise(async (resolve) => { - const pushDetail = await getPush(id); - if (!pushDetail) { + const action = await getPush(id); + if (!action) { resolve(false); return; } - const repoName = pushDetail.repoName.replace('.git', ''); - const isAllowed = await repo.isUserPushAllowed(repoName, user); + const isAllowed = await repo.isUserPushAllowed(action.url, user); if (isAllowed) { resolve(true); diff --git a/src/db/mongo/repo.ts b/src/db/mongo/repo.ts index 0d3ded4f9..6c8d58dd1 100644 --- a/src/db/mongo/repo.ts +++ b/src/db/mongo/repo.ts @@ -25,45 +25,39 @@ export const createRepo = async (repo: Repo) => { console.log(`created new repo ${JSON.stringify(repo)}`); }; -export const addUserCanPush = async (name: string, user: string) => { - name = name.toLowerCase(); +export const addUserCanPush = async (repoUrl: string, user: string) => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ name: name }, { $push: { 'users.canPush': user } }); + await collection.updateOne({ url: repoUrl }, { $push: { 'users.canPush': user } }); }; -export const addUserCanAuthorise = async (name: string, user: string) => { - name = name.toLowerCase(); +export const addUserCanAuthorise = async (repoUrl: string, user: string) => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ name: name }, { $push: { 'users.canAuthorise': user } }); + await collection.updateOne({ url: repoUrl }, { $push: { 'users.canAuthorise': user } }); }; -export const removeUserCanPush = async (name: string, user: string) => { - name = name.toLowerCase(); +export const removeUserCanPush = async (repoUrl: string, user: string) => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ name: name }, { $pull: { 'users.canPush': user } }); + await collection.updateOne({ url: repoUrl }, { $pull: { 'users.canPush': user } }); }; -export const removeUserCanAuthorise = async (name: string, user: string) => { - name = name.toLowerCase(); +export const removeUserCanAuthorise = async (repoUrl: string, user: string) => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ name: name }, { $pull: { 'users.canAuthorise': user } }); + await collection.updateOne({ url: repoUrl }, { $pull: { 'users.canAuthorise': user } }); }; -export const deleteRepo = async (name: string) => { - name = name.toLowerCase(); +export const deleteRepo = async (repoUrl: string) => { const collection = await connect(collectionName); - await collection.deleteMany({ name: name }); + await collection.deleteMany({ url: repoUrl }); }; -export const isUserPushAllowed = async (name: string, user: string) => { - name = name.toLowerCase(); +export const isUserPushAllowed = async (repoUrl: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve) => { - const repo = await exports.getRepo(name); + const repo = await exports.getRepoByUrl(repoUrl); console.log(repo.users.canPush); console.log(repo.users.canAuthorise); @@ -75,17 +69,16 @@ export const isUserPushAllowed = async (name: string, user: string) => { }); }; -export const canUserApproveRejectPushRepo = async (name: string, user: string) => { - name = name.toLowerCase(); +export const canUserApproveRejectPushRepo = async (repoUrl: string, user: string) => { user = user.toLowerCase(); - console.log(`checking if user ${user} can approve/reject for ${name}`); + console.log(`checking if user ${user} can approve/reject for ${repoUrl}`); return new Promise(async (resolve) => { - const repo = await exports.getRepo(name); + const repo = await exports.getRepoByUrl(repoUrl); if (repo.users.canAuthorise.includes(user)) { - console.log(`user ${user} can approve/reject to repo ${name}`); + console.log(`user ${user} can approve/reject to repo ${repoUrl}`); resolve(true); } else { - console.log(`user ${user} cannot approve/reject to repo ${name}`); + console.log(`user ${user} cannot approve/reject to repo ${repoUrl}`); resolve(false); } }); From a3bdb7eafc4a9c40e6f1e76dccf16136a453cbd4 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 20 May 2025 12:48:17 +0100 Subject: [PATCH 05/76] feat(key on repo url): refactor tests to use repo URLs rather than names (WIP) --- src/db/file/repo.ts | 8 ++-- src/db/mongo/repo.ts | 8 ++++ test/addRepoTest.test.js | 70 +++++++++++++++-------------- test/testDb.test.js | 95 +++++++++++++++++----------------------- 4 files changed, 90 insertions(+), 91 deletions(-) diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index d626132c4..3ed116687 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -157,7 +157,7 @@ export const removeUserCanAuthorise = async (repoUrl: string, user: string) => { export const removeUserCanPush = async (repoUrl: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepo(repoUrl); + const repo = await getRepoByUrl(repoUrl); if (!repo) { reject(new Error('Repo not found')); return; @@ -214,7 +214,7 @@ export const isUserPushAllowed = async (repoUrl: string, user: string) => { export const canUserApproveRejectPushRepo = async (repoUrl: string, user: string) => { user = user.toLowerCase(); - console.log(`checking if user ${user} can approve/reject for ${name}`); + console.log(`checking if user ${user} can approve/reject for ${repoUrl}`); return new Promise(async (resolve) => { const repo = await getRepoByUrl(repoUrl); if (!repo) { @@ -223,10 +223,10 @@ export const canUserApproveRejectPushRepo = async (repoUrl: string, user: string } if (repo.users.canAuthorise.includes(user)) { - console.log(`user ${user} can approve/reject to repo ${name}`); + console.log(`user ${user} can approve/reject to repo ${repoUrl}`); resolve(true); } else { - console.log(`user ${user} cannot approve/reject to repo ${name}`); + console.log(`user ${user} cannot approve/reject to repo ${repoUrl}`); resolve(false); } }); diff --git a/src/db/mongo/repo.ts b/src/db/mongo/repo.ts index 6c8d58dd1..939cc9db5 100644 --- a/src/db/mongo/repo.ts +++ b/src/db/mongo/repo.ts @@ -58,6 +58,10 @@ export const isUserPushAllowed = async (repoUrl: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve) => { const repo = await exports.getRepoByUrl(repoUrl); + if (!repo) { + resolve(false); + return; + } console.log(repo.users.canPush); console.log(repo.users.canAuthorise); @@ -74,6 +78,10 @@ export const canUserApproveRejectPushRepo = async (repoUrl: string, user: string console.log(`checking if user ${user} can approve/reject for ${repoUrl}`); return new Promise(async (resolve) => { const repo = await exports.getRepoByUrl(repoUrl); + if (!repo) { + resolve(false); + return; + } if (repo.users.canAuthorise.includes(user)) { console.log(`user ${user} can approve/reject to repo ${repoUrl}`); resolve(true); diff --git a/test/addRepoTest.test.js b/test/addRepoTest.test.js index 172074101..ec8460ca5 100644 --- a/test/addRepoTest.test.js +++ b/test/addRepoTest.test.js @@ -8,6 +8,12 @@ chai.use(chaiHttp); chai.should(); const expect = chai.expect; +const TEST_REPO = { + url: 'https://github.com/finos/test-repo.git', + name: 'test-repo', + project: 'finos', +}; + describe('add new repo', async () => { let app; let cookie; @@ -24,7 +30,7 @@ describe('add new repo', async () => { before(async function () { app = await service.start(); // Prepare the data. - await db.deleteRepo('test-repo'); + await db.deleteRepo(TEST_REPO.url); await db.deleteUser('u1'); await db.deleteUser('u2'); await db.createUser('u1', 'abc', 'test@test.com', 'test', true); @@ -41,17 +47,17 @@ describe('add new repo', async () => { }); it('create a new repo', async function () { - const res = await chai.request(app).post('/api/v1/repo').set('Cookie', `${cookie}`).send({ - project: 'finos', - name: 'test-repo', - url: 'https://github.com/finos/test-repo.git', - }); + const res = await chai + .request(app) + .post('/api/v1/repo') + .set('Cookie', `${cookie}`) + .send(TEST_REPO); res.should.have.status(200); - const repo = await db.getRepo('test-repo'); - repo.project.should.equal('finos'); - repo.name.should.equal('test-repo'); - repo.url.should.equal('https://github.com/finos/test-repo.git'); + const repo = await db.getRepoByUrl(TEST_REPO.url); + repo.project.should.equal(TEST_REPO.project); + repo.name.should.equal(TEST_REPO.name); + repo.url.should.equal(TEST_REPO.url); repo.users.canPush.length.should.equal(0); repo.users.canAuthorise.length.should.equal(0); }); @@ -61,24 +67,24 @@ describe('add new repo', async () => { .request(app) .get('/api/v1/repo') .set('Cookie', `${cookie}`) - .query({ name: 'test-repo' }); + .query({ url: TEST_REPO.url }); res.should.have.status(200); - res.body[0].project.should.equal('finos'); - res.body[0].name.should.equal('test-repo'); - res.body[0].url.should.equal('https://github.com/finos/test-repo.git'); + res.body[0].project.should.equal(TEST_REPO.project); + res.body[0].name.should.equal(TEST_REPO.name); + res.body[0].url.should.equal(TEST_REPO.url); }); it('add 1st can push user', async function () { const res = await chai .request(app) - .patch('/api/v1/repo/test-repo/user/push') + .patch(`/api/v1/repo/${TEST_REPO.url}/user/push`) .set('Cookie', `${cookie}`) .send({ username: 'u1', }); res.should.have.status(200); - const repo = await db.getRepo('test-repo'); + const repo = await db.getRepoByUrl(TEST_REPO.url); repo.users.canPush.length.should.equal(1); repo.users.canPush[0].should.equal('u1'); }); @@ -93,7 +99,7 @@ describe('add new repo', async () => { }); res.should.have.status(200); - const repo = await db.getRepo('test-repo'); + const repo = await db.getRepoByUrl(TEST_REPO.url); repo.users.canPush.length.should.equal(2); repo.users.canPush[1].should.equal('u2'); }); @@ -101,40 +107,40 @@ describe('add new repo', async () => { it('add push user that does not exist', async function () { const res = await chai .request(app) - .patch('/api/v1/repo/test-repo/user/push') + .patch(`/api/v1/repo/${TEST_REPO.url}/user/push`) .set('Cookie', `${cookie}`) .send({ username: 'u3', }); res.should.have.status(400); - const repo = await db.getRepo('test-repo'); + const repo = await db.getRepoByUrl(TEST_REPO.url); repo.users.canPush.length.should.equal(2); }); it('delete user u2 from push', async function () { const res = await chai .request(app) - .delete('/api/v1/repo/test-repo/user/push/u2') + .delete(`/api/v1/repo/${TEST_REPO.url}/user/push/u2`) .set('Cookie', `${cookie}`) .send({}); res.should.have.status(200); - const repo = await db.getRepo('test-repo'); + const repo = await db.getRepoByUrl(TEST_REPO.url); repo.users.canPush.length.should.equal(1); }); it('add 1st can authorise user', async function () { const res = await chai .request(app) - .patch('/api/v1/repo/test-repo/user/authorise') + .patch(`/api/v1/repo/${TEST_REPO.url}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u1', }); res.should.have.status(200); - const repo = await db.getRepo('test-repo'); + const repo = await db.getRepoByUrl(TEST_REPO.url); repo.users.canAuthorise.length.should.equal(1); repo.users.canAuthorise[0].should.equal('u1'); }); @@ -142,14 +148,14 @@ describe('add new repo', async () => { it('add 2nd can authorise user', async function () { const res = await chai .request(app) - .patch('/api/v1/repo/test-repo/user/authorise') + .patch(`/api/v1/repo/${TEST_REPO.url}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u2', }); res.should.have.status(200); - const repo = await db.getRepo('test-repo'); + const repo = await db.getRepoByUrl(TEST_REPO.url); repo.users.canAuthorise.length.should.equal(2); repo.users.canAuthorise[1].should.equal('u2'); }); @@ -157,43 +163,43 @@ describe('add new repo', async () => { it('add authorise user that does not exist', async function () { const res = await chai .request(app) - .patch('/api/v1/repo/test-repo/user/authorise') + .patch(`/api/v1/repo/${TEST_REPO.url}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u3', }); res.should.have.status(400); - const repo = await db.getRepo('test-repo'); + const repo = await db.getRepoByUrl(TEST_REPO.url); repo.users.canAuthorise.length.should.equal(2); }); it('Can delete u2 user', async function () { const res = await chai .request(app) - .delete('/api/v1/repo/test-repo/user/authorise/u2') + .delete(`/api/v1/repo/${TEST_REPO.url}/user/authorise/u2`) .set('Cookie', `${cookie}`) .send({}); res.should.have.status(200); - const repo = await db.getRepo('test-repo'); + const repo = await db.getRepoByUrl(TEST_REPO.url); repo.users.canAuthorise.length.should.equal(1); }); it('Valid user push permission on repo', async function () { const res = await chai .request(app) - .patch('/api/v1/repo/test-repo/user/authorise') + .patch(`/api/v1/repo/${TEST_REPO.url}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u2' }); res.should.have.status(200); - const isAllowed = await db.isUserPushAllowed('test-repo', 'u2'); + const isAllowed = await db.isUserPushAllowed(TEST_REPO.url, 'u2'); expect(isAllowed).to.be.true; }); it('Invalid user push permission on repo', async function () { - const isAllowed = await db.isUserPushAllowed('test-repo', 'test'); + const isAllowed = await db.isUserPushAllowed(TEST_REPO.url, 'test'); expect(isAllowed).to.be.false; }); diff --git a/test/testDb.test.js b/test/testDb.test.js index 9bc825692..c1edc0028 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -10,6 +10,12 @@ const TEST_REPO = { url: 'https://github.com/finos/db-test-repo.git', }; +const TEST_NONEXISTENT_REPO = { + project: 'MegaCorp', + name: 'repo', + url: 'https://example.com/MegaCorp/MegaGroup/repo.git', +}; + const TEST_USER = { username: 'db-u1', password: 'abc', @@ -102,13 +108,6 @@ describe('Database clients', async () => { expect(repos3).to.have.same.deep.members(repos4); }); - it('should be able to retrieve a repo by name', async function () { - // uppercase the filter value to confirm db client is lowercasing inputs - const repo = await db.getRepo(TEST_REPO.name); - const cleanRepo = cleanResponseData(TEST_REPO, repo); - expect(cleanRepo).to.eql(TEST_REPO); - }); - it('should be able to retrieve a repo by url', async function () { const repo = await db.getRepoByUrl(TEST_REPO.url); const cleanRepo = cleanResponseData(TEST_REPO, repo); @@ -116,7 +115,7 @@ describe('Database clients', async () => { }); it('should be able to delete a repo', async function () { - await db.deleteRepo(TEST_REPO.name); + await db.deleteRepo(TEST_REPO.url); const repos = await db.getRepos(); const cleanRepos = cleanResponseData(TEST_REPO, repos); @@ -285,7 +284,7 @@ describe('Database clients', async () => { let threwError = false; try { // uppercase the filter value to confirm db client is lowercasing inputs - await db.addUserCanPush('non-existent-repo', TEST_USER.username); + await db.addUserCanPush(TEST_NONEXISTENT_REPO.url, TEST_USER.username); } catch (e) { threwError = true; } @@ -297,24 +296,21 @@ describe('Database clients', async () => { try { // first create the repo and check that user is not allowed to push await db.createRepo(TEST_REPO); - let allowed = await db.isUserPushAllowed(TEST_REPO.name, TEST_USER.username); + let allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); expect(allowed).to.be.false; // uppercase the filter value to confirm db client is lowercasing inputs - await db.addUserCanPush(TEST_REPO.name.toUpperCase(), TEST_USER.username.toUpperCase()); + await db.addUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); // repeat, should not throw an error if already set - await db.addUserCanPush(TEST_REPO.name.toUpperCase(), TEST_USER.username.toUpperCase()); + await db.addUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); // confirm the setting exists - allowed = await db.isUserPushAllowed(TEST_REPO.name, TEST_USER.username); + allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); expect(allowed).to.be.true; // confirm that casing doesn't matter - allowed = await db.isUserPushAllowed( - TEST_REPO.name.toUpperCase(), - TEST_USER.username.toUpperCase(), - ); + allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username.toUpperCase()); expect(allowed).to.be.true; } catch (e) { console.error('Error thrown ', e); @@ -326,7 +322,7 @@ describe('Database clients', async () => { it('should throw an error when de-authorising a user to push on non-existent repo', async function () { let threwError = false; try { - await db.removeUserCanPush('non-existent-repo', TEST_USER.username); + await db.removeUserCanPush(TEST_NONEXISTENT_REPO.url, TEST_USER.username); } catch (e) { threwError = true; } @@ -337,27 +333,24 @@ describe('Database clients', async () => { let threwError = false; try { // repo should already exist with user able to push after previous test - let allowed = await db.isUserPushAllowed(TEST_REPO.name, TEST_USER.username); + let allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); expect(allowed).to.be.true; // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanPush(TEST_REPO.name.toUpperCase(), TEST_USER.username.toUpperCase()); + await db.removeUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); // repeat, should not throw an error if already unset - await db.removeUserCanPush(TEST_REPO.name.toUpperCase(), TEST_USER.username.toUpperCase()); + await db.removeUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); // confirm the setting exists - allowed = await db.isUserPushAllowed(TEST_REPO.name, TEST_USER.username); + allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); expect(allowed).to.be.false; // confirm that casing doesn't matter - allowed = await db.isUserPushAllowed( - TEST_REPO.name.toUpperCase(), - TEST_USER.username.toUpperCase(), - ); + allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username.toUpperCase()); expect(allowed).to.be.false; } catch (e) { - console.error('Error thrown ', e); + console.error('Error thrown at: ' + e.stack, e); threwError = true; } expect(threwError).to.be.false; @@ -366,7 +359,7 @@ describe('Database clients', async () => { it('should throw an error when authorising a user to authorise on non-existent repo', async function () { let threwError = false; try { - await db.addUserCanAuthorise('non-existent-repo', TEST_USER.username); + await db.addUserCanAuthorise(TEST_NONEXISTENT_REPO.url, TEST_USER.username); } catch (e) { threwError = true; } @@ -377,22 +370,22 @@ describe('Database clients', async () => { let threwError = false; try { // repo should already exist after a previous test - let allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.name, TEST_USER.username); + let allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); expect(allowed).to.be.false; // uppercase the filter value to confirm db client is lowercasing inputs - await db.addUserCanAuthorise(TEST_REPO.name.toUpperCase(), TEST_USER.username.toUpperCase()); + await db.addUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); // repeat, should not throw an error if already set - await db.addUserCanAuthorise(TEST_REPO.name.toUpperCase(), TEST_USER.username.toUpperCase()); + await db.addUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); // confirm the setting exists - allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.name, TEST_USER.username); + allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); expect(allowed).to.be.true; // confirm that casing doesn't matter allowed = await db.canUserApproveRejectPushRepo( - TEST_REPO.name.toUpperCase(), + TEST_REPO.url, TEST_USER.username.toUpperCase(), ); expect(allowed).to.be.true; @@ -407,7 +400,7 @@ describe('Database clients', async () => { let threwError = false; try { // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanAuthorise('non-existent-repo', TEST_USER.username); + await db.removeUserCanAuthorise(TEST_NONEXISTENT_REPO.url, TEST_USER.username); } catch (e) { threwError = true; } @@ -418,28 +411,22 @@ describe('Database clients', async () => { let threwError = false; try { // repo should already exist after a previous test and user should be an authoriser - let allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.name, TEST_USER.username); + let allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); expect(allowed).to.be.true; // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanAuthorise( - TEST_REPO.name.toUpperCase(), - TEST_USER.username.toUpperCase(), - ); + await db.removeUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); // repeat, should not throw an error if already set - await db.removeUserCanAuthorise( - TEST_REPO.name.toUpperCase(), - TEST_USER.username.toUpperCase(), - ); + await db.removeUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); // confirm the setting was removed - allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.name, TEST_USER.username); + allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); expect(allowed).to.be.false; // confirm that casing doesn't matter allowed = await db.canUserApproveRejectPushRepo( - TEST_REPO.name.toUpperCase(), + TEST_REPO.url, TEST_USER.username.toUpperCase(), ); expect(allowed).to.be.false; @@ -453,8 +440,7 @@ describe('Database clients', async () => { it('should NOT throw an error when checking whether a user can push on non-existent repo', async function () { let threwError = false; try { - // uppercase the filter value to confirm db client is lowercasing inputs - const allowed = await db.isUserPushAllowed('non-existent-repo', TEST_USER.username); + const allowed = await db.isUserPushAllowed(TEST_NONEXISTENT_REPO.url, TEST_USER.username); expect(allowed).to.be.false; } catch (e) { threwError = true; @@ -467,7 +453,7 @@ describe('Database clients', async () => { try { // uppercase the filter value to confirm db client is lowercasing inputs const allowed = await db.canUserApproveRejectPushRepo( - 'non-existent-repo', + TEST_NONEXISTENT_REPO.url, TEST_USER.username, ); expect(allowed).to.be.false; @@ -569,7 +555,6 @@ describe('Database clients', async () => { it('should be able to check if a user can cancel push', async function () { let threwError = false; - const repoName = TEST_PUSH.repoName.replace('.git', ''); try { // push does not exist yet, should return false let allowed = await db.canUserCancelPush(TEST_PUSH.id, TEST_USER.username); @@ -581,21 +566,21 @@ describe('Database clients', async () => { expect(allowed).to.be.false; // authorise user and recheck - await db.addUserCanPush(repoName, TEST_USER.username); + await db.addUserCanPush(TEST_PUSH.url, TEST_USER.username); allowed = await db.canUserCancelPush(TEST_PUSH.id, TEST_USER.username); expect(allowed).to.be.true; } catch (e) { + console.error(e); threwError = true; } expect(threwError).to.be.false; // clean up await db.deletePush(TEST_PUSH.id); - await db.removeUserCanPush(repoName, TEST_USER.username); + await db.removeUserCanPush(TEST_PUSH.url, TEST_USER.username); }); it('should be able to check if a user can approve/reject push', async function () { let threwError = false; - const repoName = TEST_PUSH.repoName.replace('.git', ''); try { // push does not exist yet, should return false let allowed = await db.canUserApproveRejectPush(TEST_PUSH.id, TEST_USER.username); @@ -607,7 +592,7 @@ describe('Database clients', async () => { expect(allowed).to.be.false; // authorise user and recheck - await db.addUserCanAuthorise(repoName, TEST_USER.username); + await db.addUserCanAuthorise(TEST_PUSH.url, TEST_USER.username); allowed = await db.canUserApproveRejectPush(TEST_PUSH.id, TEST_USER.username); expect(allowed).to.be.true; } catch (e) { @@ -616,11 +601,11 @@ describe('Database clients', async () => { expect(threwError).to.be.false; // clean up await db.deletePush(TEST_PUSH.id); - await db.removeUserCanAuthorise(repoName, TEST_USER.username); + await db.removeUserCanAuthorise(TEST_PUSH.url, TEST_USER.username); }); after(async function () { - await db.deleteRepo(TEST_REPO.name); + await db.deleteRepo(TEST_REPO.url); await db.deleteUser(TEST_USER.username); await db.deletePush(TEST_PUSH.id); }); From 522ab1cf37e930c4af6d66248904ecedebeedffb Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 20 May 2025 15:11:09 +0100 Subject: [PATCH 06/76] feat(key on repo url): switch repo indexing to url field and make unique as its our new primary key --- src/db/file/repo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index 3ed116687..d97f9c844 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -11,7 +11,7 @@ if (!fs.existsSync('./.data')) fs.mkdirSync('./.data'); if (!fs.existsSync('./.data/db')) fs.mkdirSync('./.data/db'); const db = new Datastore({ filename: './.data/db/repos.db', autoload: true }); -db.ensureIndex({ fieldName: 'name', unique: false }); +db.ensureIndex({ fieldName: 'url', unique: true }); db.setAutocompactionInterval(COMPACTION_INTERVAL); export const getRepos = async (query: any = {}) => { From 3433157bb790ccab9a6454955abf64d3e9712da4 Mon Sep 17 00:00:00 2001 From: Raimund Hook Date: Tue, 20 May 2025 15:25:02 +0100 Subject: [PATCH 07/76] feat(key on repo url): allow file db to delete multiple records Signed-off-by: Raimund Hook --- src/db/file/repo.ts | 4 ++-- test/testDb.test.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index d97f9c844..c238bf0b0 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -178,9 +178,9 @@ export const removeUserCanPush = async (repoUrl: string, user: string) => { }); }; -export const deleteRepo = async (repoUrl: string) => { +export const deleteRepo = async (repoUrl: string, multi: boolean = false) => { return new Promise((resolve, reject) => { - db.remove({ url: repoUrl }, (err) => { + db.remove({ url: repoUrl }, { multi }, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { diff --git a/test/testDb.test.js b/test/testDb.test.js index c1edc0028..e28bc5de7 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -605,7 +605,7 @@ describe('Database clients', async () => { }); after(async function () { - await db.deleteRepo(TEST_REPO.url); + await db.deleteRepo(TEST_REPO.url, true); await db.deleteUser(TEST_USER.username); await db.deletePush(TEST_PUSH.id); }); From 3aaa6f773d0e0ce15385a2cf68cf13950df18d77 Mon Sep 17 00:00:00 2001 From: Kris West Date: Wed, 21 May 2025 00:56:21 +0100 Subject: [PATCH 08/76] feat(key on repo url): switching to key on _id for in API + consolidate non-specific db fns --- src/db/file/index.ts | 15 +- src/db/file/pushes.ts | 32 ---- src/db/file/repo.ts | 82 +++----- src/db/index.ts | 63 ++++++- src/db/mongo/index.ts | 15 +- src/db/mongo/pushes.ts | 33 ---- src/db/mongo/repo.ts | 63 ++----- src/db/types.ts | 7 +- .../push-action/checkUserPushPermission.ts | 9 +- src/service/routes/repo.js | 42 ++--- test/addRepoTest.test.js | 46 +++-- test/testDb.test.js | 178 ++++++------------ 12 files changed, 222 insertions(+), 363 deletions(-) diff --git a/src/db/file/index.ts b/src/db/file/index.ts index 18c36a06b..80276a7af 100644 --- a/src/db/file/index.ts +++ b/src/db/file/index.ts @@ -2,30 +2,19 @@ import * as users from './users'; import * as repo from './repo'; import * as pushes from './pushes'; -export const { - getPushes, - writeAudit, - getPush, - deletePush, - authorise, - cancel, - reject, - canUserCancelPush, - canUserApproveRejectPush, -} = pushes; +export const { getPushes, writeAudit, getPush, deletePush, authorise, cancel, reject } = pushes; export const { getRepos, getRepo, getRepoByUrl, + getRepoById, createRepo, addUserCanPush, addUserCanAuthorise, removeUserCanPush, removeUserCanAuthorise, deleteRepo, - isUserPushAllowed, - canUserApproveRejectPushRepo, } = repo; export const { findUser, findUserByOIDC, getUsers, createUser, deleteUser, updateUser } = users; diff --git a/src/db/file/pushes.ts b/src/db/file/pushes.ts index e96004dcf..6de64c9d8 100644 --- a/src/db/file/pushes.ts +++ b/src/db/file/pushes.ts @@ -3,7 +3,6 @@ import _ from 'lodash'; import Datastore from '@seald-io/nedb'; import { Action } from '../../proxy/actions/Action'; import { toClass } from '../helper'; -import * as repo from './repo'; import { PushQuery } from '../types'; const COMPACTION_INTERVAL = 1000 * 60 * 60 * 24; // once per day @@ -129,34 +128,3 @@ export const cancel = async (id: string) => { await writeAudit(action); return { message: `cancel ${id}` }; }; - -export const canUserCancelPush = async (id: string, user: string) => { - return new Promise(async (resolve) => { - const action = await getPush(id); - if (!action) { - resolve(false); - return; - } - - const isAllowed = await repo.isUserPushAllowed(action.url, user); - - if (isAllowed) { - resolve(true); - } else { - resolve(false); - } - }); -}; - -export const canUserApproveRejectPush = async (id: string, user: string) => { - return new Promise(async (resolve) => { - const action = await getPush(id); - if (!action) { - resolve(false); - return; - } - const isAllowed = await repo.canUserApproveRejectPushRepo(action.url, user); - - resolve(isAllowed); - }); -}; diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index c238bf0b0..069e789b2 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -59,6 +59,20 @@ export const getRepoByUrl = async (repoURL: string) => { }); }; +export const getRepoById = async (_id: string) => { + return new Promise((resolve, reject) => { + db.findOne({ _id: _id }, (err: Error | null, doc: Repo) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ + if (err) { + reject(err); + } else { + resolve(doc); + } + }); + }); +}; + export const createRepo = async (repo: Repo) => { return new Promise((resolve, reject) => { db.insert(repo, (err, doc) => { @@ -73,10 +87,10 @@ export const createRepo = async (repo: Repo) => { }); }; -export const addUserCanPush = async (repoUrl: string, user: string) => { +export const addUserCanPush = async (_id: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepoByUrl(repoUrl); + const repo = await getRepoById(_id); if (!repo) { reject(new Error('Repo not found')); return; @@ -89,7 +103,7 @@ export const addUserCanPush = async (repoUrl: string, user: string) => { repo.users.canPush.push(user); const options = { multi: false, upsert: false }; - db.update({ url: repoUrl }, repo, options, (err) => { + db.update({ _id: _id }, repo, options, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -101,10 +115,10 @@ export const addUserCanPush = async (repoUrl: string, user: string) => { }); }; -export const addUserCanAuthorise = async (repoUrl: string, user: string) => { +export const addUserCanAuthorise = async (_id: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepoByUrl(repoUrl); + const repo = await getRepoById(_id); if (!repo) { reject(new Error('Repo not found')); return; @@ -118,7 +132,7 @@ export const addUserCanAuthorise = async (repoUrl: string, user: string) => { repo.users.canAuthorise.push(user); const options = { multi: false, upsert: false }; - db.update({ url: repoUrl }, repo, options, (err) => { + db.update({ _id: _id }, repo, options, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -130,10 +144,10 @@ export const addUserCanAuthorise = async (repoUrl: string, user: string) => { }); }; -export const removeUserCanAuthorise = async (repoUrl: string, user: string) => { +export const removeUserCanAuthorise = async (_id: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepoByUrl(repoUrl); + const repo = await getRepoById(_id); if (!repo) { reject(new Error('Repo not found')); return; @@ -142,7 +156,7 @@ export const removeUserCanAuthorise = async (repoUrl: string, user: string) => { repo.users.canAuthorise = repo.users.canAuthorise.filter((x: string) => x != user); const options = { multi: false, upsert: false }; - db.update({ url: repoUrl }, repo, options, (err) => { + db.update({ _id: _id }, repo, options, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -154,10 +168,10 @@ export const removeUserCanAuthorise = async (repoUrl: string, user: string) => { }); }; -export const removeUserCanPush = async (repoUrl: string, user: string) => { +export const removeUserCanPush = async (_id: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepoByUrl(repoUrl); + const repo = await getRepoById(_id); if (!repo) { reject(new Error('Repo not found')); return; @@ -166,7 +180,7 @@ export const removeUserCanPush = async (repoUrl: string, user: string) => { repo.users.canPush = repo.users.canPush.filter((x) => x != user); const options = { multi: false, upsert: false }; - db.update({ url: repoUrl }, repo, options, (err) => { + db.update({ _id: _id }, repo, options, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -178,9 +192,9 @@ export const removeUserCanPush = async (repoUrl: string, user: string) => { }); }; -export const deleteRepo = async (repoUrl: string, multi: boolean = false) => { +export const deleteRepo = async (_id: string) => { return new Promise((resolve, reject) => { - db.remove({ url: repoUrl }, { multi }, (err) => { + db.remove({ _id: _id }, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -191,43 +205,3 @@ export const deleteRepo = async (repoUrl: string, multi: boolean = false) => { }); }); }; - -export const isUserPushAllowed = async (repoUrl: string, user: string) => { - user = user.toLowerCase(); - return new Promise(async (resolve) => { - const repo = await getRepoByUrl(repoUrl); - if (!repo) { - resolve(false); - return; - } - - console.log(repo.users.canPush); - console.log(repo.users.canAuthorise); - - if (repo.users.canPush.includes(user) || repo.users.canAuthorise.includes(user)) { - resolve(true); - } else { - resolve(false); - } - }); -}; - -export const canUserApproveRejectPushRepo = async (repoUrl: string, user: string) => { - user = user.toLowerCase(); - console.log(`checking if user ${user} can approve/reject for ${repoUrl}`); - return new Promise(async (resolve) => { - const repo = await getRepoByUrl(repoUrl); - if (!repo) { - resolve(false); - return; - } - - if (repo.users.canAuthorise.includes(user)) { - console.log(`user ${user} can approve/reject to repo ${repoUrl}`); - resolve(true); - } else { - console.log(`user ${user} cannot approve/reject to repo ${repoUrl}`); - resolve(false); - } - }); -}; diff --git a/src/db/index.ts b/src/db/index.ts index 885b0742b..6b55fb0a2 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -105,6 +105,64 @@ export const getRepoByUrl = async (repoUrl: string) => { return response; }; +export const isUserPushAllowed = async (url: string, user: string) => { + user = user.toLowerCase(); + return new Promise(async (resolve) => { + const repo = await getRepoByUrl(url); + if (!repo) { + resolve(false); + return; + } + + console.log(repo.users.canPush); + console.log(repo.users.canAuthorise); + + if (repo.users.canPush.includes(user) || repo.users.canAuthorise.includes(user)) { + resolve(true); + } else { + resolve(false); + } + }); +}; + +export const canUserApproveRejectPush = async (id: string, user: string) => { + return new Promise(async (resolve) => { + const action = await getPush(id); + if (!action) { + resolve(false); + return; + } + + const theRepo = await sink.getRepoByUrl(action.url); + + if (theRepo.users.canAuthorise.includes(user)) { + console.log(`user ${user} can approve/reject for repo ${action.url}`); + resolve(true); + } else { + console.log(`user ${user} cannot approve/reject for repo ${action.url}`); + resolve(false); + } + }); +}; + +export const canUserCancelPush = async (id: string, user: string) => { + return new Promise(async (resolve) => { + const action = await getPush(id); + if (!action) { + resolve(false); + return; + } + + const isAllowed = await isUserPushAllowed(action.url, user); + + if (isAllowed) { + resolve(true); + } else { + resolve(false); + } + }); +}; + export const { authorise, reject, @@ -119,14 +177,11 @@ export const { deleteUser, updateUser, getRepos, + getRepoById, addUserCanPush, addUserCanAuthorise, removeUserCanAuthorise, removeUserCanPush, deleteRepo, - isUserPushAllowed, - canUserApproveRejectPushRepo, - canUserApproveRejectPush, - canUserCancelPush, getSessionStore, } = sink; diff --git a/src/db/mongo/index.ts b/src/db/mongo/index.ts index b190bed49..932a803df 100644 --- a/src/db/mongo/index.ts +++ b/src/db/mongo/index.ts @@ -5,30 +5,19 @@ import * as users from './users'; export const { getSessionStore } = helper; -export const { - getPushes, - writeAudit, - getPush, - deletePush, - authorise, - cancel, - reject, - canUserCancelPush, - canUserApproveRejectPush, -} = pushes; +export const { getPushes, writeAudit, getPush, deletePush, authorise, cancel, reject } = pushes; export const { getRepos, getRepo, getRepoByUrl, + getRepoById, createRepo, addUserCanPush, addUserCanAuthorise, removeUserCanPush, removeUserCanAuthorise, deleteRepo, - isUserPushAllowed, - canUserApproveRejectPushRepo, } = repo; export const { findUser, getUsers, createUser, deleteUser, updateUser } = users; diff --git a/src/db/mongo/pushes.ts b/src/db/mongo/pushes.ts index c461bfb2c..3818ee5ba 100644 --- a/src/db/mongo/pushes.ts +++ b/src/db/mongo/pushes.ts @@ -1,7 +1,6 @@ import { connect, findDocuments, findOneDocument } from './helper'; import { Action } from '../../proxy/actions'; import { toClass } from '../helper'; -import * as repo from './repo'; import { Push, PushQuery } from '../types'; const collectionName = 'pushes'; @@ -99,35 +98,3 @@ export const cancel = async (id: string) => { await writeAudit(action); return { message: `canceled ${id}` }; }; - -export const canUserApproveRejectPush = async (id: string, user: string) => { - return new Promise(async (resolve) => { - const action = await getPush(id); - if (!action) { - resolve(false); - return; - } - - const isAllowed = await repo.canUserApproveRejectPushRepo(action.url, user); - - resolve(isAllowed); - }); -}; - -export const canUserCancelPush = async (id: string, user: string) => { - return new Promise(async (resolve) => { - const action = await getPush(id); - if (!action) { - resolve(false); - return; - } - - const isAllowed = await repo.isUserPushAllowed(action.url, user); - - if (isAllowed) { - resolve(true); - } else { - resolve(false); - } - }); -}; diff --git a/src/db/mongo/repo.ts b/src/db/mongo/repo.ts index 939cc9db5..f35d5c1c0 100644 --- a/src/db/mongo/repo.ts +++ b/src/db/mongo/repo.ts @@ -19,75 +19,42 @@ export const getRepoByUrl = async (repoUrl: string) => { return collection.findOne({ name: { $eq: repoUrl.toLowerCase() } }); }; +export const getRepoById = async (_id: string) => { + const collection = await connect(collectionName); + return collection.findOne({ _id: { $eq: _id } }); +}; + export const createRepo = async (repo: Repo) => { const collection = await connect(collectionName); await collection.insertOne(repo); console.log(`created new repo ${JSON.stringify(repo)}`); }; -export const addUserCanPush = async (repoUrl: string, user: string) => { +export const addUserCanPush = async (_id: string, user: string) => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ url: repoUrl }, { $push: { 'users.canPush': user } }); + await collection.updateOne({ _id: _id }, { $push: { 'users.canPush': user } }); }; -export const addUserCanAuthorise = async (repoUrl: string, user: string) => { +export const addUserCanAuthorise = async (_id: string, user: string) => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ url: repoUrl }, { $push: { 'users.canAuthorise': user } }); + await collection.updateOne({ _id: _id }, { $push: { 'users.canAuthorise': user } }); }; -export const removeUserCanPush = async (repoUrl: string, user: string) => { +export const removeUserCanPush = async (_id: string, user: string) => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ url: repoUrl }, { $pull: { 'users.canPush': user } }); + await collection.updateOne({ _id: _id }, { $pull: { 'users.canPush': user } }); }; -export const removeUserCanAuthorise = async (repoUrl: string, user: string) => { +export const removeUserCanAuthorise = async (_id: string, user: string) => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ url: repoUrl }, { $pull: { 'users.canAuthorise': user } }); + await collection.updateOne({ _id: _id }, { $pull: { 'users.canAuthorise': user } }); }; -export const deleteRepo = async (repoUrl: string) => { +export const deleteRepo = async (_id: string) => { const collection = await connect(collectionName); - await collection.deleteMany({ url: repoUrl }); -}; - -export const isUserPushAllowed = async (repoUrl: string, user: string) => { - user = user.toLowerCase(); - return new Promise(async (resolve) => { - const repo = await exports.getRepoByUrl(repoUrl); - if (!repo) { - resolve(false); - return; - } - console.log(repo.users.canPush); - console.log(repo.users.canAuthorise); - - if (repo.users.canPush.includes(user) || repo.users.canAuthorise.includes(user)) { - resolve(true); - } else { - resolve(false); - } - }); -}; - -export const canUserApproveRejectPushRepo = async (repoUrl: string, user: string) => { - user = user.toLowerCase(); - console.log(`checking if user ${user} can approve/reject for ${repoUrl}`); - return new Promise(async (resolve) => { - const repo = await exports.getRepoByUrl(repoUrl); - if (!repo) { - resolve(false); - return; - } - if (repo.users.canAuthorise.includes(user)) { - console.log(`user ${user} can approve/reject to repo ${repoUrl}`); - resolve(true); - } else { - console.log(`user ${user} cannot approve/reject to repo ${repoUrl}`); - resolve(false); - } - }); + await collection.deleteMany({ _id: _id }); }; diff --git a/src/db/types.ts b/src/db/types.ts index 04951a699..33dbc2d47 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -12,11 +12,11 @@ export type Repo = { name: string; url: string; users: Record; - _id: string; + _id?: string; }; export type User = { - _id: string; + _id?: string; username: string; password: string | null; // null if oidcId is set gitAccount: string; @@ -42,7 +42,8 @@ export type Push = { rejected: boolean; repo: string; repoName: string; - timepstamp: string; + timestamp: string; type: string; url: string; + _id?: string; }; diff --git a/src/proxy/processors/push-action/checkUserPushPermission.ts b/src/proxy/processors/push-action/checkUserPushPermission.ts index 5ed660b44..1cd57538c 100644 --- a/src/proxy/processors/push-action/checkUserPushPermission.ts +++ b/src/proxy/processors/push-action/checkUserPushPermission.ts @@ -5,7 +5,6 @@ import { getUsers, isUserPushAllowed } from '../../../db'; const exec = async (req: any, action: Action): Promise => { const step = new Step('checkUserPushPermission'); - const repoName = action.repo.split('/')[1].replace('.git', ''); let isUserAllowed = false; let user = action.user; @@ -16,15 +15,15 @@ const exec = async (req: any, action: Action): Promise => { if (list.length == 1) { user = list[0].username; - isUserAllowed = await isUserPushAllowed(repoName, user); + isUserAllowed = await isUserPushAllowed(action.url, user!); } - console.log(`User ${user} permission on Repo ${repoName} : ${isUserAllowed}`); + console.log(`User ${user} permission on Repo ${action.url} : ${isUserAllowed}`); if (!isUserAllowed) { console.log('User not allowed to Push'); step.error = true; - step.log(`User ${user} is not allowed to push on repo ${action.repo}, ending`); + step.log(`User ${user} is not allowed to push on repo ${action.url}, ending`); console.log('setting error'); @@ -37,7 +36,7 @@ const exec = async (req: any, action: Action): Promise => { return action; } - step.log(`User ${user} is allowed to push on repo ${action.repo}`); + step.log(`User ${user} is allowed to push on repo ${action.url}`); action.addStep(step); return action; }; diff --git a/src/service/routes/repo.js b/src/service/routes/repo.js index cc70cec16..575d6ec2a 100644 --- a/src/service/routes/repo.js +++ b/src/service/routes/repo.js @@ -22,16 +22,16 @@ router.get('/', async (req, res) => { res.send(qd.map((d) => ({ ...d, proxyURL }))); }); -router.get('/:name', async (req, res) => { +router.get('/:id', async (req, res) => { const proxyURL = getProxyURL(req); - const name = req.params.name; - const qd = await db.getRepo(name); + const _id = req.params.id; + const qd = await db.getRepoById(_id); res.send({ ...qd, proxyURL }); }); -router.patch('/:name/user/push', async (req, res) => { +router.patch('/:id/user/push', async (req, res) => { if (req.user && req.user.admin) { - const repoName = req.params.name; + const _id = req.params.id; const username = req.body.username.toLowerCase(); const user = await db.findUser(username); @@ -40,7 +40,7 @@ router.patch('/:name/user/push', async (req, res) => { return; } - await db.addUserCanPush(repoName, username); + await db.addUserCanPush(_id, username); res.send({ message: 'created' }); } else { res.status(401).send({ @@ -49,9 +49,9 @@ router.patch('/:name/user/push', async (req, res) => { } }); -router.patch('/:name/user/authorise', async (req, res) => { +router.patch('/:id/user/authorise', async (req, res) => { if (req.user && req.user.admin) { - const repoName = req.params.name; + const _id = req.params.id; const username = req.body.username; const user = await db.findUser(username); @@ -60,7 +60,7 @@ router.patch('/:name/user/authorise', async (req, res) => { return; } - await db.addUserCanAuthorise(repoName, username); + await db.addUserCanAuthorise(_id, username); res.send({ message: 'created' }); } else { res.status(401).send({ @@ -69,9 +69,9 @@ router.patch('/:name/user/authorise', async (req, res) => { } }); -router.delete('/:name/user/authorise/:username', async (req, res) => { +router.delete('/:id/user/authorise/:username', async (req, res) => { if (req.user && req.user.admin) { - const repoName = req.params.name; + const _id = req.params.id; const username = req.params.username; const user = await db.findUser(username); @@ -80,7 +80,7 @@ router.delete('/:name/user/authorise/:username', async (req, res) => { return; } - await db.removeUserCanAuthorise(repoName, username); + await db.removeUserCanAuthorise(_id, username); res.send({ message: 'created' }); } else { res.status(401).send({ @@ -89,9 +89,9 @@ router.delete('/:name/user/authorise/:username', async (req, res) => { } }); -router.delete('/:name/user/push/:username', async (req, res) => { +router.delete('/:id/user/push/:username', async (req, res) => { if (req.user && req.user.admin) { - const repoName = req.params.name; + const _id = req.params.id; const username = req.params.username; const user = await db.findUser(username); @@ -100,7 +100,7 @@ router.delete('/:name/user/push/:username', async (req, res) => { return; } - await db.removeUserCanPush(repoName, username); + await db.removeUserCanPush(_id, username); res.send({ message: 'created' }); } else { res.status(401).send({ @@ -109,11 +109,11 @@ router.delete('/:name/user/push/:username', async (req, res) => { } }); -router.delete('/:name/delete', async (req, res) => { +router.delete('/:id/delete', async (req, res) => { if (req.user && req.user.admin) { - const repoName = req.params.name; + const _id = req.params.id; - await db.deleteRepo(repoName); + await db.deleteRepo(_id); res.send({ message: 'deleted' }); } else { res.status(401).send({ @@ -124,14 +124,14 @@ router.delete('/:name/delete', async (req, res) => { router.post('/', async (req, res) => { if (req.user && req.user.admin) { - if (!req.body.name) { + if (!req.body.url) { res.status(400).send({ - message: 'Repository name is required', + message: 'Repository url is required', }); return; } - const repo = await db.getRepo(req.body.name); + const repo = await db.getRepoByUrl(req.body.url); if (repo) { res.status(409).send({ message: 'Repository already exists!', diff --git a/test/addRepoTest.test.js b/test/addRepoTest.test.js index ec8460ca5..04979afa7 100644 --- a/test/addRepoTest.test.js +++ b/test/addRepoTest.test.js @@ -17,6 +17,7 @@ const TEST_REPO = { describe('add new repo', async () => { let app; let cookie; + let repoId; const setCookie = function (res) { res.headers['set-cookie'].forEach((x) => { @@ -30,7 +31,11 @@ describe('add new repo', async () => { before(async function () { app = await service.start(); // Prepare the data. - await db.deleteRepo(TEST_REPO.url); + // _id is autogenerated by the DB so we need to retrieve it before we can use it + const repo = await db.getRepoByUrl(TEST_REPO.url); + if (repo) { + await db.deleteRepo(repo._id); + } await db.deleteUser('u1'); await db.deleteUser('u2'); await db.createUser('u1', 'abc', 'test@test.com', 'test', true); @@ -55,6 +60,9 @@ describe('add new repo', async () => { res.should.have.status(200); const repo = await db.getRepoByUrl(TEST_REPO.url); + // save repo id for use in subsequent tests + repoId = repo._id; + repo.project.should.equal(TEST_REPO.project); repo.name.should.equal(TEST_REPO.name); repo.url.should.equal(TEST_REPO.url); @@ -77,14 +85,14 @@ describe('add new repo', async () => { it('add 1st can push user', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${TEST_REPO.url}/user/push`) + .patch(`/api/v1/repo/${repoId}/user/push`) .set('Cookie', `${cookie}`) .send({ username: 'u1', }); res.should.have.status(200); - const repo = await db.getRepoByUrl(TEST_REPO.url); + const repo = await db.getRepoById(repoId); repo.users.canPush.length.should.equal(1); repo.users.canPush[0].should.equal('u1'); }); @@ -92,14 +100,14 @@ describe('add new repo', async () => { it('add 2nd can push user', async function () { const res = await chai .request(app) - .patch('/api/v1/repo/test-repo/user/push') + .patch(`/api/v1/repo/${repoId}/user/push`) .set('Cookie', `${cookie}`) .send({ username: 'u2', }); res.should.have.status(200); - const repo = await db.getRepoByUrl(TEST_REPO.url); + const repo = await db.getRepoById(repoId); repo.users.canPush.length.should.equal(2); repo.users.canPush[1].should.equal('u2'); }); @@ -107,40 +115,40 @@ describe('add new repo', async () => { it('add push user that does not exist', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${TEST_REPO.url}/user/push`) + .patch(`/api/v1/repo/${repoId}/user/push`) .set('Cookie', `${cookie}`) .send({ username: 'u3', }); res.should.have.status(400); - const repo = await db.getRepoByUrl(TEST_REPO.url); + const repo = await db.getRepoById(repoId); repo.users.canPush.length.should.equal(2); }); it('delete user u2 from push', async function () { const res = await chai .request(app) - .delete(`/api/v1/repo/${TEST_REPO.url}/user/push/u2`) + .delete(`/api/v1/repo/${repoId}/user/push/u2`) .set('Cookie', `${cookie}`) .send({}); res.should.have.status(200); - const repo = await db.getRepoByUrl(TEST_REPO.url); + const repo = await db.getRepoById(repoId); repo.users.canPush.length.should.equal(1); }); it('add 1st can authorise user', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${TEST_REPO.url}/user/authorise`) + .patch(`/api/v1/repo/${repoId}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u1', }); res.should.have.status(200); - const repo = await db.getRepoByUrl(TEST_REPO.url); + const repo = await db.getRepoById(repoId); repo.users.canAuthorise.length.should.equal(1); repo.users.canAuthorise[0].should.equal('u1'); }); @@ -148,14 +156,14 @@ describe('add new repo', async () => { it('add 2nd can authorise user', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${TEST_REPO.url}/user/authorise`) + .patch(`/api/v1/repo/${repoId}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u2', }); res.should.have.status(200); - const repo = await db.getRepoByUrl(TEST_REPO.url); + const repo = await db.getRepoById(repoId); repo.users.canAuthorise.length.should.equal(2); repo.users.canAuthorise[1].should.equal('u2'); }); @@ -163,33 +171,33 @@ describe('add new repo', async () => { it('add authorise user that does not exist', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${TEST_REPO.url}/user/authorise`) + .patch(`/api/v1/repo/${repoId}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u3', }); res.should.have.status(400); - const repo = await db.getRepoByUrl(TEST_REPO.url); + const repo = await db.getRepoById(repoId); repo.users.canAuthorise.length.should.equal(2); }); it('Can delete u2 user', async function () { const res = await chai .request(app) - .delete(`/api/v1/repo/${TEST_REPO.url}/user/authorise/u2`) + .delete(`/api/v1/repo/${repoId}/user/authorise/u2`) .set('Cookie', `${cookie}`) .send({}); res.should.have.status(200); - const repo = await db.getRepoByUrl(TEST_REPO.url); + const repo = await db.getRepoById(repoId); repo.users.canAuthorise.length.should.equal(1); }); it('Valid user push permission on repo', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${TEST_REPO.url}/user/authorise`) + .patch(`/api/v1/repo/${repoId}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u2' }); @@ -199,7 +207,7 @@ describe('add new repo', async () => { }); it('Invalid user push permission on repo', async function () { - const isAllowed = await db.isUserPushAllowed(TEST_REPO.url, 'test'); + const isAllowed = await db.isUserPushAllowed(TEST_REPO.url, 'test1234'); expect(isAllowed).to.be.false; }); diff --git a/test/testDb.test.js b/test/testDb.test.js index e28bc5de7..d290e496f 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -14,6 +14,7 @@ const TEST_NONEXISTENT_REPO = { project: 'MegaCorp', name: 'repo', url: 'https://example.com/MegaCorp/MegaGroup/repo.git', + _id: 'ABCDEFGHIJKLMNOP', }; const TEST_USER = { @@ -41,7 +42,7 @@ const TEST_PUSH = { timestamp: 1744380903338, project: 'finos', repoName: 'db-test-repo.git', - url: 'https://github.com/finos/db-test-repo.git', + url: TEST_REPO.url, repo: 'finos/db-test-repo.git', user: 'db-test-user', userEmail: 'db-test@test.com', @@ -114,10 +115,19 @@ describe('Database clients', async () => { expect(cleanRepo).to.eql(TEST_REPO); }); + it('should be able to retrieve a repo by id', async function () { + // _id is autogenerated by the DB so we need to retrieve it before we can use it + const repo = await db.getRepoByUrl(TEST_REPO.url); + const repoById = await db.getRepoById(repo._id); + const cleanRepo = cleanResponseData(TEST_REPO, repoById); + expect(cleanRepo).to.eql(TEST_REPO); + }); + it('should be able to delete a repo', async function () { - await db.deleteRepo(TEST_REPO.url); + // _id is autogenerated by the DB so we need to retrieve it before we can use it + const repo = await db.getRepoByUrl(TEST_REPO.url); + await db.deleteRepo(repo._id); const repos = await db.getRepos(); - const cleanRepos = cleanResponseData(TEST_REPO, repos); expect(cleanRepos).to.not.deep.include(TEST_REPO); }); @@ -284,7 +294,7 @@ describe('Database clients', async () => { let threwError = false; try { // uppercase the filter value to confirm db client is lowercasing inputs - await db.addUserCanPush(TEST_NONEXISTENT_REPO.url, TEST_USER.username); + await db.addUserCanPush(TEST_NONEXISTENT_REPO._id, TEST_USER.username); } catch (e) { threwError = true; } @@ -292,37 +302,33 @@ describe('Database clients', async () => { }); it('should be able to authorise a user to push and confirm that they can', async function () { - let threwError = false; - try { - // first create the repo and check that user is not allowed to push - await db.createRepo(TEST_REPO); - let allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); - expect(allowed).to.be.false; + // first create the repo and check that user is not allowed to push + await db.createRepo(TEST_REPO); - // uppercase the filter value to confirm db client is lowercasing inputs - await db.addUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); + let allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); + expect(allowed).to.be.false; - // repeat, should not throw an error if already set - await db.addUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); + const repo = await db.getRepoByUrl(TEST_REPO.url); - // confirm the setting exists - allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); - expect(allowed).to.be.true; + // uppercase the filter value to confirm db client is lowercasing inputs + await db.addUserCanPush(repo._id, TEST_USER.username.toUpperCase()); - // confirm that casing doesn't matter - allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username.toUpperCase()); - expect(allowed).to.be.true; - } catch (e) { - console.error('Error thrown ', e); - threwError = true; - } - expect(threwError).to.be.false; + // repeat, should not throw an error if already set + await db.addUserCanPush(repo._id, TEST_USER.username.toUpperCase()); + + // confirm the setting exists + allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); + expect(allowed).to.be.true; + + // confirm that casing doesn't matter + allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username.toUpperCase()); + expect(allowed).to.be.true; }); it('should throw an error when de-authorising a user to push on non-existent repo', async function () { let threwError = false; try { - await db.removeUserCanPush(TEST_NONEXISTENT_REPO.url, TEST_USER.username); + await db.removeUserCanPush(TEST_NONEXISTENT_REPO._id, TEST_USER.username); } catch (e) { threwError = true; } @@ -336,11 +342,13 @@ describe('Database clients', async () => { let allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); expect(allowed).to.be.true; + const repo = await db.getRepoByUrl(TEST_REPO.url); + // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); + await db.removeUserCanPush(repo._id, TEST_USER.username.toUpperCase()); // repeat, should not throw an error if already unset - await db.removeUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); + await db.removeUserCanPush(repo._id, TEST_USER.username.toUpperCase()); // confirm the setting exists allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); @@ -359,108 +367,27 @@ describe('Database clients', async () => { it('should throw an error when authorising a user to authorise on non-existent repo', async function () { let threwError = false; try { - await db.addUserCanAuthorise(TEST_NONEXISTENT_REPO.url, TEST_USER.username); + await db.addUserCanAuthorise(TEST_NONEXISTENT_REPO._id, TEST_USER.username); } catch (e) { threwError = true; } expect(threwError).to.be.true; }); - it('should be able to authorise a user to authorise and confirm that they can', async function () { - let threwError = false; - try { - // repo should already exist after a previous test - let allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); - expect(allowed).to.be.false; - - // uppercase the filter value to confirm db client is lowercasing inputs - await db.addUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); - - // repeat, should not throw an error if already set - await db.addUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); - - // confirm the setting exists - allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); - expect(allowed).to.be.true; - - // confirm that casing doesn't matter - allowed = await db.canUserApproveRejectPushRepo( - TEST_REPO.url, - TEST_USER.username.toUpperCase(), - ); - expect(allowed).to.be.true; - } catch (e) { - console.error('Error thrown ', e); - threwError = true; - } - expect(threwError).to.be.false; - }); - it('should throw an error when de-authorising a user to push on non-existent repo', async function () { let threwError = false; try { // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanAuthorise(TEST_NONEXISTENT_REPO.url, TEST_USER.username); + await db.removeUserCanAuthorise(TEST_NONEXISTENT_REPO._id, TEST_USER.username); } catch (e) { threwError = true; } expect(threwError).to.be.true; }); - it("should be able to de-authorise a user to authorise and confirm that they can't", async function () { - let threwError = false; - try { - // repo should already exist after a previous test and user should be an authoriser - let allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); - expect(allowed).to.be.true; - - // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); - - // repeat, should not throw an error if already set - await db.removeUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); - - // confirm the setting was removed - allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); - expect(allowed).to.be.false; - - // confirm that casing doesn't matter - allowed = await db.canUserApproveRejectPushRepo( - TEST_REPO.url, - TEST_USER.username.toUpperCase(), - ); - expect(allowed).to.be.false; - } catch (e) { - console.error('Error thrown ', e); - threwError = true; - } - expect(threwError).to.be.false; - }); - it('should NOT throw an error when checking whether a user can push on non-existent repo', async function () { - let threwError = false; - try { - const allowed = await db.isUserPushAllowed(TEST_NONEXISTENT_REPO.url, TEST_USER.username); - expect(allowed).to.be.false; - } catch (e) { - threwError = true; - } - expect(threwError).to.be.false; - }); - - it('should NOT throw an error when checking whether a user can authorise on non-existent repo', async function () { - let threwError = false; - try { - // uppercase the filter value to confirm db client is lowercasing inputs - const allowed = await db.canUserApproveRejectPushRepo( - TEST_NONEXISTENT_REPO.url, - TEST_USER.username, - ); - expect(allowed).to.be.false; - } catch (e) { - threwError = true; - } - expect(threwError).to.be.false; + const allowed = await db.isUserPushAllowed(TEST_NONEXISTENT_REPO.url, TEST_USER.username); + expect(allowed).to.be.false; }); it('should be able to create a push', async function () { @@ -556,6 +483,8 @@ describe('Database clients', async () => { it('should be able to check if a user can cancel push', async function () { let threwError = false; try { + const repo = await db.getRepoByUrl(TEST_REPO.url); + // push does not exist yet, should return false let allowed = await db.canUserCancelPush(TEST_PUSH.id, TEST_USER.username); expect(allowed).to.be.false; @@ -566,9 +495,14 @@ describe('Database clients', async () => { expect(allowed).to.be.false; // authorise user and recheck - await db.addUserCanPush(TEST_PUSH.url, TEST_USER.username); + await db.addUserCanPush(repo._id, TEST_USER.username); allowed = await db.canUserCancelPush(TEST_PUSH.id, TEST_USER.username); expect(allowed).to.be.true; + + // deauthorise user and recheck + await db.removeUserCanPush(repo._id, TEST_USER.username); + allowed = await db.canUserCancelPush(TEST_PUSH.id, TEST_USER.username); + expect(allowed).to.be.false; } catch (e) { console.error(e); threwError = true; @@ -576,7 +510,6 @@ describe('Database clients', async () => { expect(threwError).to.be.false; // clean up await db.deletePush(TEST_PUSH.id); - await db.removeUserCanPush(TEST_PUSH.url, TEST_USER.username); }); it('should be able to check if a user can approve/reject push', async function () { @@ -591,21 +524,30 @@ describe('Database clients', async () => { allowed = await db.canUserApproveRejectPush(TEST_PUSH.id, TEST_USER.username); expect(allowed).to.be.false; + const repo = await db.getRepoByUrl(TEST_REPO.url); + // authorise user and recheck - await db.addUserCanAuthorise(TEST_PUSH.url, TEST_USER.username); + await db.addUserCanAuthorise(repo._id, TEST_USER.username); allowed = await db.canUserApproveRejectPush(TEST_PUSH.id, TEST_USER.username); expect(allowed).to.be.true; + + // deauthorise user and recheck + await db.removeUserCanAuthorise(repo._id, TEST_USER.username); + allowed = await db.canUserApproveRejectPush(TEST_PUSH.id, TEST_USER.username); + expect(allowed).to.be.false; } catch (e) { threwError = true; } expect(threwError).to.be.false; + // clean up await db.deletePush(TEST_PUSH.id); - await db.removeUserCanAuthorise(TEST_PUSH.url, TEST_USER.username); }); after(async function () { - await db.deleteRepo(TEST_REPO.url, true); + // _id is autogenerated by the DB so we need to retrieve it before we can use it + const repo = await db.getRepoByUrl(TEST_REPO.url); + await db.deleteRepo(repo._id, true); await db.deleteUser(TEST_USER.username); await db.deletePush(TEST_PUSH.id); }); From e73fba493cc18cfa2cedd9b560263f4acad404c5 Mon Sep 17 00:00:00 2001 From: Raimund Hook Date: Wed, 21 May 2025 11:39:43 +0100 Subject: [PATCH 09/76] feat(key on repo url): update calls in cli tests to use repo ids Signed-off-by: Raimund Hook --- packages/git-proxy-cli/test/testCli.test.js | 8 ++++---- packages/git-proxy-cli/test/testCliUtils.js | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/git-proxy-cli/test/testCli.test.js b/packages/git-proxy-cli/test/testCli.test.js index fbfce0fe3..a872221c5 100644 --- a/packages/git-proxy-cli/test/testCli.test.js +++ b/packages/git-proxy-cli/test/testCli.test.js @@ -224,7 +224,7 @@ describe('test git-proxy-cli', function () { after(async function () { await helper.removeGitPushFromDb(pushId); - await helper.removeRepoFromDb(TEST_REPO_CONFIG.name); + await helper.removeRepoFromDb(TEST_REPO_CONFIG.url); }); it('attempt to authorise should fail when server is down', async function () { @@ -299,7 +299,7 @@ describe('test git-proxy-cli', function () { after(async function () { await helper.removeGitPushFromDb(pushId); - await helper.removeRepoFromDb(TEST_REPO_CONFIG.name); + await helper.removeRepoFromDb(TEST_REPO_CONFIG.url); }); it('attempt to cancel should fail when server is down', async function () { @@ -420,7 +420,7 @@ describe('test git-proxy-cli', function () { after(async function () { await helper.removeGitPushFromDb(pushId); - await helper.removeRepoFromDb(TEST_REPO_CONFIG.name); + await helper.removeRepoFromDb(TEST_REPO_CONFIG.url); }); it('attempt to reject should fail when server is down', async function () { @@ -498,7 +498,7 @@ describe('test git-proxy-cli', function () { after(async function () { await helper.removeUserFromDb('testuser1'); await helper.removeGitPushFromDb(pushId); - await helper.removeRepoFromDb(TEST_REPO_CONFIG.name); + await helper.removeRepoFromDb(TEST_REPO_CONFIG.url); }); it('attempt to ls should list existing push', async function () { diff --git a/packages/git-proxy-cli/test/testCliUtils.js b/packages/git-proxy-cli/test/testCliUtils.js index 557857619..a58ece747 100644 --- a/packages/git-proxy-cli/test/testCliUtils.js +++ b/packages/git-proxy-cli/test/testCliUtils.js @@ -152,8 +152,9 @@ async function addRepoToDb(newRepo, debug = false) { const found = repos.find((y) => y.project === newRepo.project && newRepo.name === y.name); if (!found) { await db.createRepo(newRepo); - await db.addUserCanPush(newRepo.name, 'admin'); - await db.addUserCanAuthorise(newRepo.name, 'admin'); + const repo = await db.getRepoByUrl(newRepo.url); + await db.addUserCanPush(repo._id, 'admin'); + await db.addUserCanAuthorise(repo._id, 'admin'); if (debug) { console.log(`New repo added to database: ${newRepo}`); } @@ -166,10 +167,11 @@ async function addRepoToDb(newRepo, debug = false) { /** * Removes a repo from the DB. - * @param {string} repoName The name of the repo to remove. + * @param {string} repoUrl The url of the repo to remove. */ -async function removeRepoFromDb(repoName) { - await db.deleteRepo(repoName); +async function removeRepoFromDb(repoUrl) { + const repo = await db.getRepoByUrl(repoUrl); + await db.deleteRepo(repo._id); } /** From a2f1e6c9285b5ad7aa8a7388085f33076233243d Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 22 May 2025 17:51:59 +0100 Subject: [PATCH 10/76] feat(key on repo url): apply proper typing to DB classes Typescript wasn't working on the DB classes due to their dependency imports with require. --- src/db/file/pushes.ts | 16 +++---- src/db/file/repo.ts | 48 +++++++++++---------- src/db/file/users.ts | 20 ++++----- src/db/index.ts | 94 +++++++++++++++++++++++------------------ src/db/mongo/index.ts | 2 +- src/db/mongo/pushes.ts | 19 ++++----- src/db/mongo/repo.ts | 55 ++++++++++++++---------- src/db/mongo/users.ts | 36 ++++++++++------ src/db/types.ts | 95 +++++++++++++++++++++++++++++------------- 9 files changed, 233 insertions(+), 152 deletions(-) diff --git a/src/db/file/pushes.ts b/src/db/file/pushes.ts index 6de64c9d8..38a3336f6 100644 --- a/src/db/file/pushes.ts +++ b/src/db/file/pushes.ts @@ -24,7 +24,7 @@ const defaultPushQuery: PushQuery = { authorised: false, }; -export const getPushes = (query: PushQuery) => { +export const getPushes = (query: PushQuery): Promise => { if (!query) query = defaultPushQuery; return new Promise((resolve, reject) => { db.find(query, (err: Error, docs: Action[]) => { @@ -43,7 +43,7 @@ export const getPushes = (query: PushQuery) => { }); }; -export const getPush = async (id: string) => { +export const getPush = async (id: string): Promise => { return new Promise((resolve, reject) => { db.findOne({ id: id }, (err, doc) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query @@ -61,7 +61,7 @@ export const getPush = async (id: string) => { }); }; -export const deletePush = async (id: string) => { +export const deletePush = async (id: string): Promise => { return new Promise((resolve, reject) => { db.remove({ id }, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query @@ -75,7 +75,7 @@ export const deletePush = async (id: string) => { }); }; -export const writeAudit = async (action: Action) => { +export const writeAudit = async (action: Action): Promise => { return new Promise((resolve, reject) => { const options = { multi: false, upsert: true }; db.update({ id: action.id }, action, options, (err) => { @@ -84,13 +84,13 @@ export const writeAudit = async (action: Action) => { if (err) { reject(err); } else { - resolve(null); + resolve(); } }); }); }; -export const authorise = async (id: string, attestation: any) => { +export const authorise = async (id: string, attestation: any): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -104,7 +104,7 @@ export const authorise = async (id: string, attestation: any) => { return { message: `authorised ${id}` }; }; -export const reject = async (id: string) => { +export const reject = async (id: string): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -117,7 +117,7 @@ export const reject = async (id: string) => { return { message: `reject ${id}` }; }; -export const cancel = async (id: string) => { +export const cancel = async (id: string): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index 069e789b2..c7acdbe0a 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -1,6 +1,8 @@ import fs from 'fs'; import Datastore from '@seald-io/nedb'; import { Repo } from '../types'; +import { toClass } from '../helper'; +import _ from 'lodash'; const COMPACTION_INTERVAL = 1000 * 60 * 60 * 24; // once per day @@ -14,7 +16,7 @@ const db = new Datastore({ filename: './.data/db/repos.db', autoload: true }); db.ensureIndex({ fieldName: 'url', unique: true }); db.setAutocompactionInterval(COMPACTION_INTERVAL); -export const getRepos = async (query: any = {}) => { +export const getRepos = async (query: any = {}): Promise => { if (query?.name) { query.name = query.name.toLowerCase(); } @@ -25,13 +27,17 @@ export const getRepos = async (query: any = {}) => { if (err) { reject(err); } else { - resolve(docs); + resolve( + _.chain(docs) + .map((x) => toClass(x, Repo.prototype)) + .value(), + ); } }); }); }; -export const getRepo = async (name: string) => { +export const getRepo = async (name: string): Promise => { return new Promise((resolve, reject) => { db.findOne({ name: name.toLowerCase() }, (err: Error | null, doc: Repo) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query @@ -39,13 +45,13 @@ export const getRepo = async (name: string) => { if (err) { reject(err); } else { - resolve(doc); + resolve(doc ? toClass(doc, Repo.prototype) : null); } }); }); }; -export const getRepoByUrl = async (repoURL: string) => { +export const getRepoByUrl = async (repoURL: string): Promise => { return new Promise((resolve, reject) => { db.findOne({ url: repoURL }, (err: Error | null, doc: Repo) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query @@ -53,13 +59,13 @@ export const getRepoByUrl = async (repoURL: string) => { if (err) { reject(err); } else { - resolve(doc); + resolve(doc ? toClass(doc, Repo.prototype) : null); } }); }); }; -export const getRepoById = async (_id: string) => { +export const getRepoById = async (_id: string): Promise => { return new Promise((resolve, reject) => { db.findOne({ _id: _id }, (err: Error | null, doc: Repo) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query @@ -67,13 +73,13 @@ export const getRepoById = async (_id: string) => { if (err) { reject(err); } else { - resolve(doc); + resolve(doc ? toClass(doc, Repo.prototype) : null); } }); }); }; -export const createRepo = async (repo: Repo) => { +export const createRepo = async (repo: Repo): Promise => { return new Promise((resolve, reject) => { db.insert(repo, (err, doc) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query @@ -81,13 +87,13 @@ export const createRepo = async (repo: Repo) => { if (err) { reject(err); } else { - resolve(doc); + resolve(doc ? toClass(doc, Repo.prototype) : null); } }); }); }; -export const addUserCanPush = async (_id: string, user: string) => { +export const addUserCanPush = async (_id: string, user: string): Promise => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { const repo = await getRepoById(_id); @@ -97,7 +103,7 @@ export const addUserCanPush = async (_id: string, user: string) => { } if (repo.users.canPush.includes(user)) { - resolve(null); + resolve(); return; } repo.users.canPush.push(user); @@ -109,13 +115,13 @@ export const addUserCanPush = async (_id: string, user: string) => { if (err) { reject(err); } else { - resolve(null); + resolve(); } }); }); }; -export const addUserCanAuthorise = async (_id: string, user: string) => { +export const addUserCanAuthorise = async (_id: string, user: string): Promise => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { const repo = await getRepoById(_id); @@ -125,7 +131,7 @@ export const addUserCanAuthorise = async (_id: string, user: string) => { } if (repo.users.canAuthorise.includes(user)) { - resolve(null); + resolve(); return; } @@ -138,13 +144,13 @@ export const addUserCanAuthorise = async (_id: string, user: string) => { if (err) { reject(err); } else { - resolve(null); + resolve(); } }); }); }; -export const removeUserCanAuthorise = async (_id: string, user: string) => { +export const removeUserCanAuthorise = async (_id: string, user: string): Promise => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { const repo = await getRepoById(_id); @@ -162,13 +168,13 @@ export const removeUserCanAuthorise = async (_id: string, user: string) => { if (err) { reject(err); } else { - resolve(null); + resolve(); } }); }); }; -export const removeUserCanPush = async (_id: string, user: string) => { +export const removeUserCanPush = async (_id: string, user: string): Promise => { user = user.toLowerCase(); return new Promise(async (resolve, reject) => { const repo = await getRepoById(_id); @@ -186,13 +192,13 @@ export const removeUserCanPush = async (_id: string, user: string) => { if (err) { reject(err); } else { - resolve(null); + resolve(); } }); }); }; -export const deleteRepo = async (_id: string) => { +export const deleteRepo = async (_id: string): Promise => { return new Promise((resolve, reject) => { db.remove({ _id: _id }, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query diff --git a/src/db/file/users.ts b/src/db/file/users.ts index 263c612f4..25716f10b 100644 --- a/src/db/file/users.ts +++ b/src/db/file/users.ts @@ -17,7 +17,7 @@ db.ensureIndex({ fieldName: 'username', unique: true }); db.ensureIndex({ fieldName: 'email', unique: true }); db.setAutocompactionInterval(COMPACTION_INTERVAL); -export const findUser = (username: string) => { +export const findUser = (username: string): Promise => { return new Promise((resolve, reject) => { db.findOne({ username: username.toLowerCase() }, (err: Error | null, doc: User) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query @@ -35,9 +35,9 @@ export const findUser = (username: string) => { }); }; -export const findUserByOIDC = function (oidcId: string) { - return new Promise((resolve, reject) => { - db.findOne({ oidcId: oidcId }, (err, doc) => { +export const findUserByOIDC = function (oidcId: string): Promise { + return new Promise((resolve, reject) => { + db.findOne({ oidcId: oidcId }, (err: Error | null, doc: User) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query /* istanbul ignore if */ if (err) { @@ -53,7 +53,7 @@ export const findUserByOIDC = function (oidcId: string) { }); }; -export const createUser = function (user: User) { +export const createUser = function (user: User): Promise { user.username = user.username.toLowerCase(); user.email = user.email.toLowerCase(); return new Promise((resolve, reject) => { @@ -63,13 +63,13 @@ export const createUser = function (user: User) { if (err) { reject(err); } else { - resolve(user); + resolve(); } }); }); }; -export const deleteUser = (username: string) => { +export const deleteUser = (username: string): Promise => { return new Promise((resolve, reject) => { db.remove({ username: username.toLowerCase() }, (err) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query @@ -83,7 +83,7 @@ export const deleteUser = (username: string) => { }); }; -export const updateUser = (user: User) => { +export const updateUser = (user: User): Promise => { user.username = user.username.toLowerCase(); if (user.email) { user.email = user.email.toLowerCase(); @@ -113,7 +113,7 @@ export const updateUser = (user: User) => { if (err) { reject(err); } else { - resolve(null); + resolve(); } }); } @@ -121,7 +121,7 @@ export const updateUser = (user: User) => { }); }; -export const getUsers = (query: any = {}) => { +export const getUsers = (query: any = {}): Promise => { if (query.username) { query.username = query.username.toLowerCase(); } diff --git a/src/db/index.ts b/src/db/index.ts index 6b55fb0a2..b90bbfa1f 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -1,12 +1,17 @@ -import { Repo } from './types'; - -const bcrypt = require('bcryptjs'); -const config = require('../config'); -let sink: any; +import { AuthorisedRepo } from '../config/types'; +import { PushQuery, Repo, Sink, User } from './types'; +import * as bcrypt from 'bcryptjs'; +import * as config from '../config'; +import * as mongo from './mongo'; +import * as neDb from './file'; +import { Action } from '../proxy/actions/Action'; +import MongoDBStore from 'connect-mongo'; + +let sink: Sink; if (config.getDatabase().type === 'mongo') { - sink = require('./mongo'); + sink = mongo; } else if (config.getDatabase().type === 'fs') { - sink = require('./file'); + sink = neDb; } const isBlank = (str: string) => { @@ -62,26 +67,29 @@ export const createUser = async ( await sink.createUser(data); }; -export const createRepo = async (repo: Repo) => { - repo.name = repo.name.toLowerCase(); - repo.users = { - canPush: [], - canAuthorise: [], +export const createRepo = async (repo: AuthorisedRepo) => { + const toCreate = { + ...repo, + users: { + canPush: [], + canAuthorise: [], + }, }; + toCreate.name = repo.name.toLowerCase(); - console.log(`creating new repo ${JSON.stringify(repo)}`); + console.log(`creating new repo ${JSON.stringify(toCreate)}`); - if (isBlank(repo.project)) { + if (isBlank(toCreate.project)) { throw new Error('Project name cannot be empty'); } - if (isBlank(repo.name)) { + if (isBlank(toCreate.name)) { throw new Error('Repository name cannot be empty'); } - if (isBlank(repo.url)) { + if (isBlank(toCreate.url)) { throw new Error('URL cannot be empty'); } - return sink.createRepo(repo); + return sink.createRepo(toCreate) as Promise>; }; export const getRepoByUrl = async (repoUrl: string) => { @@ -135,7 +143,7 @@ export const canUserApproveRejectPush = async (id: string, user: string) => { const theRepo = await sink.getRepoByUrl(action.url); - if (theRepo.users.canAuthorise.includes(user)) { + if (theRepo?.users?.canAuthorise?.includes(user)) { console.log(`user ${user} can approve/reject for repo ${action.url}`); resolve(true); } else { @@ -163,25 +171,31 @@ export const canUserCancelPush = async (id: string, user: string) => { }); }; -export const { - authorise, - reject, - cancel, - getPushes, - writeAudit, - getPush, - deletePush, - findUser, - findUserByOIDC, - getUsers, - deleteUser, - updateUser, - getRepos, - getRepoById, - addUserCanPush, - addUserCanAuthorise, - removeUserCanAuthorise, - removeUserCanPush, - deleteRepo, - getSessionStore, -} = sink; +export const getSessionStore = (): MongoDBStore | null => + sink.getSessionStore ? sink.getSessionStore() : null; +export const getPushes = (query: PushQuery): Promise => sink.getPushes(query); +export const writeAudit = (action: Action): Promise => sink.writeAudit(action); +export const getPush = (id: string): Promise => sink.getPush(id); +export const deletePush = (id: string): Promise => sink.deletePush(id); +export const authorise = (id: string, attestation: any): Promise<{ message: string }> => + sink.authorise(id, attestation); +export const cancel = (id: string): Promise<{ message: string }> => sink.cancel(id); +export const reject = (id: string): Promise<{ message: string }> => sink.reject(id); +export const getRepos = (query?: object): Promise => sink.getRepos(query); +export const getRepo = (name: string): Promise => sink.getRepo(name); +// export const getRepoByUrl = (url: string): Promise => sink.getRepoByUrl(url); +export const getRepoById = (_id: string): Promise => sink.getRepoById(_id); +export const addUserCanPush = (_id: string, user: string): Promise => + sink.addUserCanPush(_id, user); +export const addUserCanAuthorise = (_id: string, user: string): Promise => + sink.addUserCanAuthorise(_id, user); +export const removeUserCanPush = (_id: string, user: string): Promise => + sink.removeUserCanPush(_id, user); +export const removeUserCanAuthorise = (_id: string, user: string): Promise => + sink.removeUserCanAuthorise(_id, user); +export const deleteRepo = (_id: string): Promise => sink.deleteRepo(_id); +export const findUser = (username: string): Promise => sink.findUser(username); +export const findUserByOIDC = (oidcId: string): Promise => sink.findUserByOIDC(oidcId); +export const getUsers = (query?: object): Promise => sink.getUsers(query); +export const deleteUser = (username: string): Promise => sink.deleteUser(username); +export const updateUser = (user: User): Promise => sink.updateUser(user); diff --git a/src/db/mongo/index.ts b/src/db/mongo/index.ts index 932a803df..9b81720ad 100644 --- a/src/db/mongo/index.ts +++ b/src/db/mongo/index.ts @@ -20,4 +20,4 @@ export const { deleteRepo, } = repo; -export const { findUser, getUsers, createUser, deleteUser, updateUser } = users; +export const { findUser, findUserByOIDC, getUsers, createUser, deleteUser, updateUser } = users; diff --git a/src/db/mongo/pushes.ts b/src/db/mongo/pushes.ts index 3818ee5ba..c64325755 100644 --- a/src/db/mongo/pushes.ts +++ b/src/db/mongo/pushes.ts @@ -1,7 +1,7 @@ import { connect, findDocuments, findOneDocument } from './helper'; import { Action } from '../../proxy/actions'; import { toClass } from '../helper'; -import { Push, PushQuery } from '../types'; +import { PushQuery } from '../types'; const collectionName = 'pushes'; @@ -12,8 +12,8 @@ const defaultPushQuery: PushQuery = { authorised: false, }; -export const getPushes = async (query: PushQuery = defaultPushQuery): Promise => { - return findDocuments(collectionName, query, { +export const getPushes = async (query: PushQuery = defaultPushQuery): Promise => { + return findDocuments(collectionName, query, { projection: { _id: 0, id: 1, @@ -44,12 +44,12 @@ export const getPush = async (id: string): Promise => { return doc ? (toClass(doc, Action.prototype) as Action) : null; }; -export const deletePush = async function (id: string) { +export const deletePush = async function (id: string): Promise { const collection = await connect(collectionName); - return collection.deleteOne({ id }); + await collection.deleteOne({ id }); }; -export const writeAudit = async (action: Action): Promise => { +export const writeAudit = async (action: Action): Promise => { const data = JSON.parse(JSON.stringify(action)); const options = { upsert: true }; const collection = await connect(collectionName); @@ -58,10 +58,9 @@ export const writeAudit = async (action: Action): Promise => { throw new Error('Invalid id'); } await collection.updateOne({ id: data.id }, { $set: data }, options); - return action; }; -export const authorise = async (id: string, attestation: any) => { +export const authorise = async (id: string, attestation: any): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -75,7 +74,7 @@ export const authorise = async (id: string, attestation: any) => { return { message: `authorised ${id}` }; }; -export const reject = async (id: string) => { +export const reject = async (id: string): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -87,7 +86,7 @@ export const reject = async (id: string) => { return { message: `reject ${id}` }; }; -export const cancel = async (id: string) => { +export const cancel = async (id: string): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); diff --git a/src/db/mongo/repo.ts b/src/db/mongo/repo.ts index f35d5c1c0..8a0311a63 100644 --- a/src/db/mongo/repo.ts +++ b/src/db/mongo/repo.ts @@ -1,60 +1,71 @@ +import _ from 'lodash'; import { Repo } from '../types'; - -const connect = require('./helper').connect; +import { connect } from './helper'; +import { toClass } from '../helper'; +import { ObjectId, OptionalId, Document } from 'mongodb'; const collectionName = 'repos'; -export const getRepos = async (query: any = {}) => { +export const getRepos = async (query: any = {}): Promise => { const collection = await connect(collectionName); - return collection.find(query).toArray(); + const docs = collection.find(query).toArray(); + return _.chain(docs) + .map((x) => toClass(x, Repo.prototype)) + .value(); }; -export const getRepo = async (name: string) => { +export const getRepo = async (name: string): Promise => { name = name.toLowerCase(); const collection = await connect(collectionName); - return collection.findOne({ name: { $eq: name } }); + const doc = collection.findOne({ name: { $eq: name } }); + return doc ? toClass(doc, Repo.prototype) : null; }; -export const getRepoByUrl = async (repoUrl: string) => { +export const getRepoByUrl = async (repoUrl: string): Promise => { const collection = await connect(collectionName); - return collection.findOne({ name: { $eq: repoUrl.toLowerCase() } }); + const doc = collection.findOne({ name: { $eq: repoUrl.toLowerCase() } }); + return doc ? toClass(doc, Repo.prototype) : null; }; -export const getRepoById = async (_id: string) => { +export const getRepoById = async (_id: string): Promise => { const collection = await connect(collectionName); - return collection.findOne({ _id: { $eq: _id } }); + const doc = collection.findOne({ _id: new ObjectId(_id) }); + return doc ? toClass(doc, Repo.prototype) : null; }; -export const createRepo = async (repo: Repo) => { +export const createRepo = async (repo: Repo): Promise => { const collection = await connect(collectionName); - await collection.insertOne(repo); + const response = await collection.insertOne(repo as OptionalId); console.log(`created new repo ${JSON.stringify(repo)}`); + // add in the _id generated for the record + repo._id = response.insertedId.toString(); + return repo; }; -export const addUserCanPush = async (_id: string, user: string) => { +export const addUserCanPush = async (_id: string, user: string): Promise => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ _id: _id }, { $push: { 'users.canPush': user } }); + await collection.updateOne({ _id: new ObjectId(_id) }, { $push: { 'users.canPush': user } }); }; -export const addUserCanAuthorise = async (_id: string, user: string) => { +export const addUserCanAuthorise = async (_id: string, user: string): Promise => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ _id: _id }, { $push: { 'users.canAuthorise': user } }); + await collection.updateOne({ _id: new ObjectId(_id) }, { $push: { 'users.canAuthorise': user } }); }; -export const removeUserCanPush = async (_id: string, user: string) => { +export const removeUserCanPush = async (_id: string, user: string): Promise => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ _id: _id }, { $pull: { 'users.canPush': user } }); + await collection.updateOne({ _id: new ObjectId(_id) }, { $pull: { 'users.canPush': user } }); }; -export const removeUserCanAuthorise = async (_id: string, user: string) => { +export const removeUserCanAuthorise = async (_id: string, user: string): Promise => { user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ _id: _id }, { $pull: { 'users.canAuthorise': user } }); + await collection.updateOne({ _id: new ObjectId(_id) }, { $pull: { 'users.canAuthorise': user } }); }; -export const deleteRepo = async (_id: string) => { +export const deleteRepo = async (_id: string): Promise => { const collection = await connect(collectionName); - await collection.deleteMany({ _id: _id }); + await collection.deleteMany({ _id: new ObjectId(_id) }); }; diff --git a/src/db/mongo/users.ts b/src/db/mongo/users.ts index 5bacb245d..623bcc9d1 100644 --- a/src/db/mongo/users.ts +++ b/src/db/mongo/users.ts @@ -1,38 +1,50 @@ +import { OptionalId, Document } from 'mongodb'; +import { toClass } from '../helper'; import { User } from '../types'; - -const connect = require('./helper').connect; +import { connect } from './helper'; +import _ from 'lodash'; const collectionName = 'users'; -export const findUser = async function (username: string) { +export const findUser = async function (username: string): Promise { + const collection = await connect(collectionName); + const doc = collection.findOne({ username: { $eq: username.toLowerCase() } }); + return doc ? toClass(doc, User.prototype) : null; +}; + +export const findUserByOIDC = async function (oidcId: string): Promise { const collection = await connect(collectionName); - return collection.findOne({ username: { $eq: username.toLowerCase() } }); + const doc = collection.findOne({ oidcId: { $eq: oidcId } }); + return doc ? toClass(doc, User.prototype) : null; }; -export const getUsers = async function (query: any = {}) { +export const getUsers = async function (query: any = {}): Promise { if (query.username) { query.username = query.username.toLowerCase(); } if (query.email) { query.email = query.email.toLowerCase(); } - console.log(`Getting users for query= ${JSON.stringify(query)}`); + console.log(`Getting users for query = ${JSON.stringify(query)}`); const collection = await connect(collectionName); - return collection.find(query, { password: 0 }).toArray(); + const docs = collection.find(query, { projection: { password: 0 } }).toArray(); + return _.chain(docs) + .map((x) => toClass(x, User.prototype)) + .value(); }; -export const deleteUser = async function (username: string) { +export const deleteUser = async function (username: string): Promise { const collection = await connect(collectionName); - return collection.deleteOne({ username: username.toLowerCase() }); + await collection.deleteOne({ username: username.toLowerCase() }); }; -export const createUser = async function (user: User) { +export const createUser = async function (user: User): Promise { user.username = user.username.toLowerCase(); user.email = user.email.toLowerCase(); const collection = await connect(collectionName); - return collection.insertOne(user); + await collection.insertOne(user as OptionalId); }; -export const updateUser = async (user: User) => { +export const updateUser = async (user: User): Promise => { user.username = user.username.toLowerCase(); if (user.email) { user.email = user.email.toLowerCase(); diff --git a/src/db/types.ts b/src/db/types.ts index 33dbc2d47..fc9503701 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -1,3 +1,6 @@ +import { Action } from '../proxy/actions/Action'; +import MongoDBStore from 'connect-mongo'; + export type PushQuery = { error: boolean; blocked: boolean; @@ -7,43 +10,79 @@ export type PushQuery = { export type UserRole = 'canPush' | 'canAuthorise'; -export type Repo = { +export class Repo { project: string; name: string; url: string; users: Record; _id?: string; -}; -export type User = { - _id?: string; + constructor( + project: string, + name: string, + url: string, + users?: Record, + _id?: string, + ) { + this.project = project; + this.name = name; + this.url = url; + this.users = users ?? { canPush: [], canAuthorise: [] }; + this._id = _id; + } +} + +export class User { username: string; password: string | null; // null if oidcId is set gitAccount: string; email: string; admin: boolean; - oidcId: string | null; -}; - -export type Push = { - id: string; - allowPush: boolean; - authorised: boolean; - blocked: boolean; - blockedMessage: string; - branch: string; - canceled: boolean; - commitData: object; - commitFrom: string; - commitTo: string; - error: boolean; - method: string; - project: string; - rejected: boolean; - repo: string; - repoName: string; - timestamp: string; - type: string; - url: string; + oidcId?: string | null; _id?: string; -}; + + constructor( + username: string, + password: string, + gitAccount: string, + email: string, + admin: boolean, + oidcId: string | null = null, + _id?: string, + ) { + this.username = username; + this.password = password; + this.gitAccount = gitAccount; + this.email = email; + this.admin = admin; + this.oidcId = oidcId ?? null; + this._id = _id; + } +} + +export interface Sink { + getSessionStore?: () => MongoDBStore; + getPushes: (query: PushQuery) => Promise; + writeAudit: (action: Action) => Promise; + getPush: (id: string) => Promise; + deletePush: (id: string) => Promise; + authorise: (id: string, attestation: any) => Promise<{ message: string }>; + cancel: (id: string) => Promise<{ message: string }>; + reject: (id: string) => Promise<{ message: string }>; + getRepos: (query?: object) => Promise; + getRepo: (name: string) => Promise; + getRepoByUrl: (url: string) => Promise; + getRepoById: (_id: string) => Promise; + createRepo: (repo: Repo) => Promise; + addUserCanPush: (_id: string, user: string) => Promise; + addUserCanAuthorise: (_id: string, user: string) => Promise; + removeUserCanPush: (_id: string, user: string) => Promise; + removeUserCanAuthorise: (_id: string, user: string) => Promise; + deleteRepo: (_id: string) => Promise; + findUser: (username: string) => Promise; + findUserByOIDC: (oidcId: string) => Promise; + getUsers: (query?: object) => Promise; + createUser: (user: User) => Promise; + deleteUser: (username: string) => Promise; + updateUser: (user: User) => Promise; +} From 62938c7a56a0130eaa73c8257c2ff828166c8b01 Mon Sep 17 00:00:00 2001 From: Kris West Date: Wed, 28 May 2025 18:32:02 +0100 Subject: [PATCH 11/76] feat(key on repo url): refactoring proxy to support embedding host with fallback --- config.schema.json | 6 +- src/config/index.ts | 20 +- src/proxy/actions/Action.ts | 28 ++- src/proxy/index.ts | 6 +- .../processors/pre-processor/parseAction.ts | 51 ++-- .../push-action/checkRepoInAuthorisedList.ts | 26 +- src/proxy/processors/push-action/scanDiff.ts | 8 +- src/proxy/routes/helper.ts | 195 +++++++++++++++ src/proxy/routes/index.ts | 232 +++++++++--------- test/scanDiff.test.js | 50 +++- test/testCheckRepoInAuthList.test.js | 33 ++- test/testRouteFilter.test.js | 87 ++++++- 12 files changed, 515 insertions(+), 227 deletions(-) create mode 100644 src/proxy/routes/helper.ts diff --git a/config.schema.json b/config.schema.json index 4539cb5b2..945c419c3 100644 --- a/config.schema.json +++ b/config.schema.json @@ -5,7 +5,11 @@ "description": "Configuration for customizing git-proxy", "type": "object", "properties": { - "proxyUrl": { "type": "string" }, + "proxyUrl": { + "type": "string", + "description": "Used in early versions of git proxy to configure the remote host that traffic is proxied to. In later versions, the repository URL is used to determine the domain proxied, allowing multiple hosts to be proxied by one instance.", + "deprecated": true + }, "cookieSecret": { "type": "string" }, "sessionMaxAgeHours": { "type": "number" }, "api": { diff --git a/src/config/index.ts b/src/config/index.ts index a3ea42136..147b0654a 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -21,7 +21,6 @@ let _database: Database[] = defaultSettings.sink; let _authentication: Authentication[] = defaultSettings.authentication; let _apiAuthentication: Authentication[] = defaultSettings.apiAuthentication; let _tempPassword: TempPasswordConfig = defaultSettings.tempPassword; -let _proxyUrl = defaultSettings.proxyUrl; let _api: Record = defaultSettings.api; let _cookieSecret: string = defaultSettings.cookieSecret; let _sessionMaxAgeHours: number = defaultSettings.sessionMaxAgeHours; @@ -47,15 +46,6 @@ let _config = { ...defaultSettings, ...(_userSettings || {}) } as Configuration; // Create config loader instance const configLoader = new ConfigLoader(_config); -// Get configured proxy URL -export const getProxyUrl = () => { - if (_userSettings !== null && _userSettings.proxyUrl) { - _proxyUrl = _userSettings.proxyUrl; - } - - return _proxyUrl; -}; - // Gets a list of authorised repositories export const getAuthorisedList = () => { if (_userSettings !== null && _userSettings.authorisedList) { @@ -92,7 +82,7 @@ export const getDatabase = () => { /** * Get the list of enabled authentication methods - * + * * At least one authentication method must be enabled. * @return {Authentication[]} List of enabled authentication methods */ @@ -104,7 +94,7 @@ export const getAuthMethods = (): Authentication[] => { const enabledAuthMethods = _authentication.filter((auth) => auth.enabled); if (enabledAuthMethods.length === 0) { - throw new Error("No authentication method enabled"); + throw new Error('No authentication method enabled'); } return enabledAuthMethods; @@ -112,7 +102,7 @@ export const getAuthMethods = (): Authentication[] => { /** * Get the list of enabled authentication methods for API endpoints - * + * * If no API authentication methods are enabled, all endpoints are public. * @return {Authentication[]} List of enabled authentication methods */ @@ -121,10 +111,10 @@ export const getAPIAuthMethods = (): Authentication[] => { _apiAuthentication = _userSettings.apiAuthentication; } - const enabledAuthMethods = _apiAuthentication.filter(auth => auth.enabled); + const enabledAuthMethods = _apiAuthentication.filter((auth) => auth.enabled); if (enabledAuthMethods.length === 0) { - console.log("Warning: No authentication method enabled for API endpoints."); + console.log('Warning: No authentication method enabled for API endpoints.'); } return enabledAuthMethods; diff --git a/src/proxy/actions/Action.ts b/src/proxy/actions/Action.ts index b15b7c24c..b9d5e5ed3 100644 --- a/src/proxy/actions/Action.ts +++ b/src/proxy/actions/Action.ts @@ -1,4 +1,4 @@ -import { getProxyUrl } from '../../config'; +import { processGitURLForNameAndOrg, processUrlPath } from '../routes/helper'; import { Step } from './Step'; /** @@ -55,17 +55,31 @@ class Action { * @param {string} type The type of the action * @param {string} method The method of the action * @param {number} timestamp The timestamp of the action - * @param {string} repo The repo of the action + * @param {string} url The URL to the repo that should be proxied (with protocol, origin, repo path, but not the path for the git operation). */ - constructor(id: string, type: string, method: string, timestamp: number, repo: string) { + constructor(id: string, type: string, method: string, timestamp: number, url: string) { this.id = id; this.type = type; this.method = method; this.timestamp = timestamp; - this.project = repo.split('/')[0]; - this.repoName = repo.split('/')[1]; - this.url = `${getProxyUrl()}/${repo}`; - this.repo = repo; + this.url = url; + + const urlBreakdown = processUrlPath(url); + if (urlBreakdown) { + this.repo = urlBreakdown.repoPath; + const repoBreakdown = processGitURLForNameAndOrg(urlBreakdown.repoPath); + if (repoBreakdown) { + this.project = repoBreakdown.project ?? ''; + this.repoName = repoBreakdown.repoName; + } else { + this.project = 'UNKNOWN'; + this.repoName = 'UNKNOWN'; + } + } else { + this.repo = 'NOT-FOUND'; + this.project = 'UNKNOWN'; + this.repoName = 'UNKNOWN'; + } } /** diff --git a/src/proxy/index.ts b/src/proxy/index.ts index 4cfcda986..afd2d7923 100644 --- a/src/proxy/index.ts +++ b/src/proxy/index.ts @@ -47,9 +47,9 @@ export const proxyPreparations = async () => { defaultAuthorisedRepoList.forEach(async (x) => { const found = allowedList.find((y) => y.project === x.project && x.name === y.name); if (!found) { - await createRepo(x); - await addUserCanPush(x.name, 'admin'); - await addUserCanAuthorise(x.name, 'admin'); + const repo = await createRepo(x); + await addUserCanPush(repo._id!, 'admin'); + await addUserCanAuthorise(repo._id!, 'admin'); } }); }; diff --git a/src/proxy/processors/pre-processor/parseAction.ts b/src/proxy/processors/pre-processor/parseAction.ts index a9c332fdc..d1a0da8d6 100644 --- a/src/proxy/processors/pre-processor/parseAction.ts +++ b/src/proxy/processors/pre-processor/parseAction.ts @@ -1,4 +1,6 @@ import { Action } from '../../actions'; +import { processUrlPath } from '../../routes/helper'; +import * as db from '../../../db'; const exec = async (req: { originalUrl: string; @@ -7,34 +9,35 @@ const exec = async (req: { }) => { const id = Date.now(); const timestamp = id; - const repoName = getRepoNameFromUrl(req.originalUrl); - const paths = req.originalUrl.split('/'); - + const pathBreakdown = processUrlPath(req.originalUrl); let type = 'default'; + if (pathBreakdown) { + if (pathBreakdown.gitPath.endsWith('git-upload-pack') && req.method === 'GET') { + type = 'pull'; + } + if ( + pathBreakdown.gitPath.includes('git-receive-pack') && + req.method === 'POST' && + req.headers['content-type'] === 'application/x-git-receive-pack-request' + ) { + type = 'push'; + } + } // else failed to parse proxy URL path - which is logged in the parsing util - if (paths[paths.length - 1].endsWith('git-upload-pack') && req.method === 'GET') { - type = 'pull'; - } - if ( - paths[paths.length - 1] === 'git-receive-pack' && - req.method === 'POST' && - req.headers['content-type'] === 'application/x-git-receive-pack-request' - ) { - type = 'push'; - } - - return new Action(id.toString(), type, req.method, timestamp, repoName); -}; + // Proxy URLs take the form https://:// + // e.g. https://git-proxy-instance.com:8443/github.com/finos/git-proxy.git + // We'll receive /github.com/finos/git-proxy.git as the req.url / req.originalUrl + // Add protocol (assume SSL) to reconstruct full URL + let url = 'https://' + (pathBreakdown?.repoPath ?? 'NOT-FOUND'); -const getRepoNameFromUrl = (url: string): string => { - const parts = url.split('/'); - for (let i = 0, len = parts.length; i < len; i++) { - const part = parts[i]; - if (part.endsWith('.git')) { - return `${parts[i - 1]}/${part}`.trim(); - } + if (!(await db.getRepoByUrl(url))) { + // fallback for legacy proxy URLs + // legacy git proxy paths took the form: https://:/ + // by assuming the host was github.com + url = 'https://github.com' + (pathBreakdown?.repoPath ?? 'NOT-FOUND'); } - return 'NOT-FOUND'; + + return new Action(id.toString(), type, req.method, timestamp, url); }; exec.displayName = 'parseAction.exec'; diff --git a/src/proxy/processors/push-action/checkRepoInAuthorisedList.ts b/src/proxy/processors/push-action/checkRepoInAuthorisedList.ts index 9560dc58d..a06a7d995 100644 --- a/src/proxy/processors/push-action/checkRepoInAuthorisedList.ts +++ b/src/proxy/processors/push-action/checkRepoInAuthorisedList.ts @@ -1,29 +1,15 @@ import { Action, Step } from '../../actions'; -import { getRepos } from '../../../db'; -import { Repo } from '../../../db/types'; +import { getRepoByUrl } from '../../../db'; // Execute if the repo is approved -const exec = async ( - req: any, - action: Action, - authorisedList: () => Promise = getRepos, -): Promise => { +const exec = async (req: any, action: Action): Promise => { const step = new Step('checkRepoInAuthorisedList'); - const list = await authorisedList(); - console.log(list); - - const found = list.find((x: Repo) => { - const targetName = action.repo.replace('.git', '').toLowerCase(); - const allowedName = `${x.project}/${x.name}`.replace('.git', '').toLowerCase(); - console.log(`${targetName} = ${allowedName}`); - return targetName === allowedName; - }); - - console.log(found); + // console.log(found); + const found = (await getRepoByUrl(action.url)) !== null; if (!found) { - console.log('not found'); + console.log(`Repository url '${action.url}' not found`); step.error = true; step.log(`repo ${action.repo} is not in the authorisedList, ending`); console.log('setting error'); @@ -33,7 +19,7 @@ const exec = async ( } console.log('found'); - step.log(`repo ${action.repo} is in the authorisedList`); + step.log(`repo ${action.url} is in the authorisedList`); action.addStep(step); return action; }; diff --git a/src/proxy/processors/push-action/scanDiff.ts b/src/proxy/processors/push-action/scanDiff.ts index 40b39627b..899fa6442 100644 --- a/src/proxy/processors/push-action/scanDiff.ts +++ b/src/proxy/processors/push-action/scanDiff.ts @@ -72,7 +72,7 @@ const combineMatches = (organization: string) => { ? [] : Object.entries(commitConfig.diff.block.providers); - // Combine all matches (literals, paterns) + // Combine all matches (literals, patterns) const combinedMatches = [ ...blockedLiterals.map((literal) => ({ type: BLOCK_TYPE.LITERAL, @@ -104,7 +104,7 @@ const collectMatches = (parsedDiff: File[], combinedMatches: CombinedMatch[]): M const lineNumber = change.ln; // Iterate through each match types - literal, patterns, providers combinedMatches.forEach(({ type, match }) => { - // using Match all to find all occurences of the pattern in the line + // using Match all to find all occurrences of the pattern in the line const matches = [...change.content.matchAll(match)]; matches.forEach((matchInstance) => { @@ -122,7 +122,7 @@ const collectMatches = (parsedDiff: File[], combinedMatches: CombinedMatch[]): M }; } - // apend line numbers to the list of lines + // append line numbers to the list of lines allMatches[matchKey].lines.push(lineNumber); }); }); @@ -131,7 +131,7 @@ const collectMatches = (parsedDiff: File[], combinedMatches: CombinedMatch[]): M }); }); - // convert matches into a final result array, joining line numbers + // convert matches into a final result array, joining line numbers const result = Object.values(allMatches).map((match) => ({ ...match, lines: match.lines.join(','), // join the line numbers into a comma-separated string diff --git a/src/proxy/routes/helper.ts b/src/proxy/routes/helper.ts new file mode 100644 index 000000000..ccabf97c4 --- /dev/null +++ b/src/proxy/routes/helper.ts @@ -0,0 +1,195 @@ +import * as db from '../../db'; + +/** Regex used to analyze un-proxied Git URLs */ +const GIT_URL_REGEX = /(.+:\/\/)([^/]+)(\/.+\.git)(\/.+)*/; + +/** Type representing a breakdown of Git URL (un-proxied)*/ +export type GitUrlBreakdown = { protocol: string; host: string; repoPath: string }; + +/** Function that processes Git URLs to extract the protocol, host, path to the + * git endpoint and discarding any git path (specific operation) that comes after + * the .git element. + * + * E.g. Processing https://github.com/finos/git-proxy.git/info/refs?service=git-upload-pack + * would produce: + * - protocol: https:// + * - host: github.com + * - repoPath: finos/git-proxy.git + * + * @param {string} url The URL to process + * @return {GitUrlBreakdown | null} A breakdown of the components of the URL. + */ +export const processGitUrl = (url: string): GitUrlBreakdown | null => { + const components = url.match(GIT_URL_REGEX); + if (components && components.length >= 5) { + return { + protocol: components[1], + host: components[2], + repoPath: components[3], + // component [4] would be any git path, but isn't needed for repo URLs + }; + } else { + console.error(`Failed to parse git URL: ${url}`); + return null; + } +}; + +/** Regex used to analyze url paths for requests to the proxy and split them + * into the embedded git end point and path for the git operation. */ +const PROXIED_URL_PATH_REGEX = /(.+\.git)(\/.*)?/; + +/** Type representing a breakdown of paths requested from the proxy server */ +export type UrlPathBreakdown = { repoPath: string; gitPath: string }; + +/** Function that processes URL paths (URL with origin removed) of requests to the proxy + * to extract the embedded repository path and path for the specific git operation to be + * proxied. + * + * E.g. Processing /finos/git-proxy.git/info/refs?service=git-upload-pack + * would produce: + * - repoPath: /finos/git-proxy.git + * - gitPath: /info/refs?service=git-upload-pack + * + * and processing /github.com/finos/git-proxy.git/info/refs?service=git-upload-pack + * would produce: + * - repoPath: /github.com/finos/git-proxy.git + * - gitPath: /info/refs?service=git-upload-pack + * + * @param {string} requestPath The URL path to process. + * @return {GitUrlBreakdown | null} A breakdown of the components of the URL path. + */ +export const processUrlPath = (requestPath: string): UrlPathBreakdown | null => { + const components = requestPath.match(PROXIED_URL_PATH_REGEX); + if (components && components.length >= 3) { + return { + repoPath: components[1], + gitPath: components[2] ?? '/', + }; + } else { + console.error(`Failed to parse proxy url path: ${requestPath}`); + return null; + } +}; + +/** Regex used to analyze repo URLs (with protocol and origin) to extract the repository name and + * any path or organisation that proceeds and drop the origin and protocol if present. */ +const GIT_URL_NAME_ORG_REGEX = /(.+:\/\/)?([^/]+)\/(?:(.*)\/)?([^/]+\.git)/; + +/** Type representing a breakdown Git URL into repository name and organisation (project). */ +export type GitNameBreakdown = { project: string | null; repoName: string }; + +/** Function that processes git URLs embedded in proxy request URLs to extract + * the repository name and any path or organisation. + * + * E.g. Processing https://github.com/finos/git-proxy.git + * would produce: + * - project: finos + * - repoName: git-proxy.git + * + * Processing https://someGitHost.com/repo.git + * would produce: + * - project: null + * - repoName: repo.git + * + * Processing https://anotherGitHost.com/project/subProject/subSubProject/repo.git + * would produce: + * - project: project/subProject/subSubProject + * - repoName: repo.git + * + * @param {string} gitUrl The URL to process. + * @return {GitNameBreakdown | null} A breakdown of the components of the URL. + */ +export const processGitURLForNameAndOrg = (gitUrl: string): GitNameBreakdown | null => { + const components = gitUrl.match(GIT_URL_NAME_ORG_REGEX); + if (components && components.length >= 5) { + return { + project: components[3] ?? null, // there may be no project or path for standalone git repo + repoName: components[4], + }; + } else { + console.error(`Failed to parse git URL: ${gitUrl}`); + return null; + } +}; + +// /** Regex used to analyze legacy proxy paths to extract the repository name and +// * any path or organisation that proceeds it. */ +// const GIT_LEGACY_PATH_REGEX = /\/([^/]+)\/([^/]+)\.git/; + +// /** Type representing a breakdown of a legacy proxy path into repository name and organisation (project) +// * and a predicted GitHub URL. */ +// export type GitLegacyPathBreakdown = { project: string; repoName: string; url: string }; + +// /** Function that processes legacy proxy path string (which assumed GitHub as the repository host) to +// * extract the repository name, project (organisation) and to construct a predicted GitHub URL. +// * +// * E.g. Processing finos/git-proxy.git +// * would produce: +// * - project: finos +// * - repoName: git-proxy +// * - url: https://github.com/finos/git-proxy.git +// * +// * @param {string} requestPath The proxy path to process. +// * @return {GitLegacyPathBreakdown | null} A breakdown of the components of the URL. +// */ +// export const processLegacyProxyPathForNameAndOrg = ( +// requestPath: string, +// ): GitLegacyPathBreakdown | null => { +// const components = requestPath.match(GIT_LEGACY_PATH_REGEX); +// if (components && components.length >= 3) { +// return { +// project: components[1], +// repoName: components[2], +// url: `https://github.com/${components[1]}/${components[2]}.git`, +// }; +// } else { +// console.error(`Failed to parse git path: ${requestPath}`); +// return null; +// } +// }; + +/** + * Check whether an HTTP request has the expected properties of a + * Git HTTP request. The URL is expected to be "sanitized", stripped of + * specific paths such as the GitHub {owner}/{repo}.git parts. + * @param {string} gitPath Sanitized URL path which only includes the path + * specific to git (everything after .git/) + * @param {*} headers Request headers (TODO: Fix JSDoc linting and refer to + * node:http.IncomingHttpHeaders) + * @return {boolean} If true, this is a valid and expected git request. + * Otherwise, false. + */ +export const validGitRequest = (gitPath: string, headers: any): boolean => { + const { 'user-agent': agent, accept } = headers; + if ( + ['/info/refs?service=git-upload-pack', '/info/refs?service=git-receive-pack'].includes(gitPath) + ) { + // https://www.git-scm.com/docs/http-protocol#_discovering_references + // We can only filter based on User-Agent since the Accept header is not + // sent in this request + return agent.startsWith('git/'); + } + if (['/git-upload-pack', '/git-receive-pack'].includes(gitPath)) { + // https://www.git-scm.com/docs/http-protocol#_uploading_data + return agent.startsWith('git/') && accept.startsWith('application/x-git-'); + } + return false; +}; + +/** + * Collect the Set of all origins (protocol, host and port if specified) that + * will be proxying requests for, to be used to initialize the proxy. + * + * @return {string[]} an array of origins + */ +export const getAllProxiedOrigins = async (): Promise => { + const repos = await db.getRepos(); + const origins = new Set(); + repos.forEach((repo) => { + const parsedUrl = processGitUrl(repo.url); + if (parsedUrl) { + origins.add(parsedUrl.protocol + parsedUrl.host); + } // failures are logged by parsing util fn + }); + return Array.from(origins); +}; diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index 1e1cfff46..d3759846a 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -1,132 +1,64 @@ import { Router } from 'express'; import proxy from 'express-http-proxy'; import { executeChain } from '../chain'; -import { getProxyUrl } from '../../config'; +import { processUrlPath, validGitRequest, getAllProxiedOrigins } from './helper'; +import { ProxyOptions } from 'express-http-proxy'; + +const proxyFilter: ProxyOptions['filter'] = async (req, res) => { + try { + console.log('request url: ', req.url); + console.log('host: ', req.headers.host); + console.log('user-agent: ', req.headers['user-agent']); + + const urlComponents = processUrlPath(req.url); + + if ( + !urlComponents || + urlComponents.gitPath === undefined || + !validGitRequest(urlComponents.gitPath, req.headers) + ) { + res.status(400).send('Invalid request received'); + return false; + } -// eslint-disable-next-line new-cap -const router = Router(); + const action = await executeChain(req, res); + console.log('action processed'); -/** - * For a given Git HTTP request destined for a GitHub repo, - * remove the GitHub specific components of the URL. - * @param {string} url URL path of the request - * @return {string} Modified path which removes the {owner}/{repo} parts - */ -const stripGitHubFromGitPath = (url: string): string | undefined => { - const parts = url.split('/'); - // url = '/{owner}/{repo}.git/{git-path}' - // url.split('/') = ['', '{owner}', '{repo}.git', '{git-path}'] - if (parts.length !== 4 && parts.length !== 5) { - console.error('unexpected url received: ', url); - return undefined; - } - parts.splice(1, 2); // remove the {owner} and {repo} from the array - return parts.join('/'); -}; + if (action.error || action.blocked) { + res.set('content-type', 'application/x-git-receive-pack-result'); + res.set('transfer-encoding', 'chunked'); + res.set('expires', 'Fri, 01 Jan 1980 00:00:00 GMT'); + res.set('pragma', 'no-cache'); + res.set('cache-control', 'no-cache, max-age=0, must-revalidate'); + res.set('vary', 'Accept-Encoding'); + res.set('x-frame-options', 'DENY'); + res.set('connection', 'close'); -/** - * Check whether an HTTP request has the expected properties of a - * Git HTTP request. The URL is expected to be "sanitized", stripped of - * specific paths such as the GitHub {owner}/{repo}.git parts. - * @param {string} url Sanitized URL which only includes the path specific to git - * @param {*} headers Request headers (TODO: Fix JSDoc linting and refer to node:http.IncomingHttpHeaders) - * @return {boolean} If true, this is a valid and expected git request. Otherwise, false. - */ -const validGitRequest = (url: string, headers: any): boolean => { - const { 'user-agent': agent, accept } = headers; - if (!agent) { - return false; - } - if (['/info/refs?service=git-upload-pack', '/info/refs?service=git-receive-pack'].includes(url)) { - // https://www.git-scm.com/docs/http-protocol#_discovering_references - // We can only filter based on User-Agent since the Accept header is not - // sent in this request - return agent.startsWith('git/'); - } - if (['/git-upload-pack', '/git-receive-pack'].includes(url)) { - if (!accept) { - return false; - } - // https://www.git-scm.com/docs/http-protocol#_uploading_data - return agent.startsWith('git/') && accept.startsWith('application/x-git-') ; - } - return false; -}; + let message = ''; -router.use( - '/', - proxy(getProxyUrl(), { - preserveHostHdr: false, - filter: async function (req, res) { - try { - console.log('request url: ', req.url); - console.log('host: ', req.headers.host); - console.log('user-agent: ', req.headers['user-agent']); - const gitPath = stripGitHubFromGitPath(req.url); - if (gitPath === undefined || !validGitRequest(gitPath, req.headers)) { - res.status(400).send('Invalid request received'); - return false; - } - - const action = await executeChain(req, res); - console.log('action processed'); - - if (action.error || action.blocked) { - res.set('content-type', 'application/x-git-receive-pack-result'); - res.set('expires', 'Fri, 01 Jan 1980 00:00:00 GMT'); - res.set('pragma', 'no-cache'); - res.set('cache-control', 'no-cache, max-age=0, must-revalidate'); - res.set('vary', 'Accept-Encoding'); - res.set('x-frame-options', 'DENY'); - res.set('connection', 'close'); - - let message = ''; - - if (action.error) { - message = action.errorMessage!; - console.error(message); - } - if (action.blocked) { - message = action.blockedMessage!; - } - - const packetMessage = handleMessage(message); - - console.log(req.headers); - - res.status(200).send(packetMessage); - - return false; - } - - return true; - } catch (e) { - console.error(e); - return false; + if (action.error) { + message = action.errorMessage!; + console.error(message); } - }, - proxyReqPathResolver: (req) => { - const url = getProxyUrl() + req.originalUrl; - console.log('Sending request to ' + url); - return url; - }, - proxyReqOptDecorator: function (proxyReqOpts) { - return proxyReqOpts; - }, - - proxyReqBodyDecorator: function (bodyContent, srcReq) { - if (srcReq.method === 'GET') { - return ''; + if (action.blocked) { + message = action.blockedMessage!; } - return bodyContent; - }, - proxyErrorHandler: function (err, res, next) { - console.log(`ERROR=${err}`); - next(err); - }, - }), -); + const packetMessage = handleMessage(message); + + console.log(req.headers); + + res.status(200).send(packetMessage); + + return false; + } + + return true; + } catch (e) { + console.error(e); + return false; + } +}; const handleMessage = (message: string): string => { const errorMessage = `\t${message}`; @@ -137,4 +69,64 @@ const handleMessage = (message: string): string => { return packetMessage; }; -export { router, handleMessage, validGitRequest, stripGitHubFromGitPath }; +const getRequestPathResolver: (origin: string) => ProxyOptions['proxyReqPathResolver'] = ( + origin, +) => { + return (req) => { + const url = origin + req.originalUrl; + console.log('Sending request to ' + url); + return url; + }; +}; + +const proxyReqOptDecorator: ProxyOptions['proxyReqOptDecorator'] = (proxyReqOpts) => proxyReqOpts; + +const proxyReqBodyDecorator: ProxyOptions['proxyReqBodyDecorator'] = (bodyContent, srcReq) => { + if (srcReq.method === 'GET') { + return ''; + } + return bodyContent; +}; + +const proxyErrorHandler: ProxyOptions['proxyErrorHandler'] = (err, res, next) => { + console.log(`ERROR=${err}`); + next(err); +}; + +// eslint-disable-next-line new-cap +const router = Router(); + +getAllProxiedOrigins().then((originsToProxy) => { + // TODO: this will only happen on startup (I think...) We'll need to add routes at runtime when new origins are added? Or force a restart for the proxy to work + + // Middlewares are processed in the order that they are added, if one applies and then doesn't call `next` then subsequent ones are not applied. + // Hence, we define known origins first, then a catch all route for backwards compatibility + originsToProxy.forEach((origin) => { + router.use( + '/' + origin, + proxy(origin, { + preserveHostHdr: false, + filter: proxyFilter, + proxyReqPathResolver: getRequestPathResolver(origin), + proxyReqOptDecorator: proxyReqOptDecorator, + proxyReqBodyDecorator: proxyReqBodyDecorator, + proxyErrorHandler: proxyErrorHandler, + }), + ); + }); + + // Catch-all route for backwards compatibility + router.use( + '/', + proxy(origin, { + preserveHostHdr: false, + filter: proxyFilter, + proxyReqPathResolver: getRequestPathResolver('https://github.com'), + proxyReqOptDecorator: proxyReqOptDecorator, + proxyReqBodyDecorator: proxyReqBodyDecorator, + proxyErrorHandler: proxyErrorHandler, + }), + ); +}); + +export { router, handleMessage, validGitRequest }; diff --git a/test/scanDiff.test.js b/test/scanDiff.test.js index 09fdb9d38..41e5b7d8b 100644 --- a/test/scanDiff.test.js +++ b/test/scanDiff.test.js @@ -4,6 +4,7 @@ const processor = require('../src/proxy/processors/push-action/scanDiff'); const { Action } = require('../src/proxy/actions/Action'); const { expect } = chai; const config = require('../src/config'); +const db = require('../src/db'); chai.should(); // Load blocked literals and patterns from configuration... @@ -72,8 +73,19 @@ describe('Scan commit diff...', async () => { }, }, }; + + before(async () => { + // needed for private org tests + const repo = await db.createRepo(TEST_REPO); + TEST_REPO._id = repo._id; + }); + + after(async () => { + await db.deleteRepo(TEST_REPO._id); + }); + it('A diff including an AWS (Amazon Web Services) Access Key ID blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -88,7 +100,7 @@ describe('Scan commit diff...', async () => { // Formatting test it('A diff including multiple AWS (Amazon Web Services) Access Keys ID blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -107,7 +119,7 @@ describe('Scan commit diff...', async () => { // Formatting test it('A diff including multiple AWS Access Keys ID and Literal blocks the proxy with appropriate message...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -128,7 +140,7 @@ describe('Scan commit diff...', async () => { }); it('A diff including a Google Cloud Platform API Key blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -143,7 +155,7 @@ describe('Scan commit diff...', async () => { }); it('A diff including a GitHub Personal Access Token blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -158,7 +170,7 @@ describe('Scan commit diff...', async () => { }); it('A diff including a GitHub Fine Grained Personal Access Token blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -175,7 +187,7 @@ describe('Scan commit diff...', async () => { }); it('A diff including a GitHub Actions Token blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -190,7 +202,7 @@ describe('Scan commit diff...', async () => { }); it('A diff including a JSON Web Token (JWT) blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -208,7 +220,7 @@ describe('Scan commit diff...', async () => { it('A diff including a blocked literal blocks the proxy...', async () => { for (const [literal] of blockedLiterals.entries()) { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -223,7 +235,7 @@ describe('Scan commit diff...', async () => { } }); it('When no diff is present, the proxy is blocked...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -238,7 +250,7 @@ describe('Scan commit diff...', async () => { }); it('When diff is not a string, the proxy is blocked...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -253,7 +265,7 @@ describe('Scan commit diff...', async () => { }); it('A diff with no secrets or sensitive information does not block the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'project/name'); + const action = new Action('1', 'type', 'method', 1, 'url'); action.steps = [ { stepName: 'diff', @@ -265,8 +277,20 @@ describe('Scan commit diff...', async () => { expect(error).to.be.false; }); + const TEST_REPO = { + project: 'private-org-test', + name: 'repo.git', + url: 'https://github.com/private-org-test/repo.git', + }; + it('A diff including a provider token in a private organization does not block the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'private-org-test'); + const action = new Action( + '1', + 'type', + 'method', + 1, + 'https://github.com/private-org-test/repo.git', // URL needs to be parseable AND exist in DB + ); action.steps = [ { stepName: 'diff', diff --git a/test/testCheckRepoInAuthList.test.js b/test/testCheckRepoInAuthList.test.js index 19d161c12..7ef78cf73 100644 --- a/test/testCheckRepoInAuthList.test.js +++ b/test/testCheckRepoInAuthList.test.js @@ -2,26 +2,37 @@ const chai = require('chai'); const actions = require('../src/proxy/actions/Action'); const processor = require('../src/proxy/processors/push-action/checkRepoInAuthorisedList'); const expect = chai.expect; +const db = require('../src/db'); -const authList = () => { - return [ - { - name: 'repo-is-ok', - project: 'thisproject', - }, - ]; +const TEST_REPO = { + project: 'thisproject', + name: 'repo-is-ok', + url: 'https://github.com/thisproject/repo-is-ok.git', +}; + +const TEST_NON_EXISTENT_REPO = { + url: 'https://github.com/thisproject/repo-is-not-ok.git', }; describe('Check a Repo is in the authorised list', async () => { + before(async function () { + const repo = await db.createRepo(TEST_REPO); + TEST_REPO._id = repo._id; + }); + + after(async function () { + await db.deleteRepo(TEST_REPO._id); + }); + it('Should set ok=true if repo in whitelist', async () => { - const action = new actions.Action('123', 'type', 'get', 1234, 'thisproject/repo-is-ok'); - const result = await processor.exec(null, action, authList); + const action = new actions.Action('123', 'type', 'get', 1234, TEST_REPO.url); + const result = await processor.exec(null, action); expect(result.error).to.be.false; }); it('Should set ok=false if not in authorised', async () => { - const action = new actions.Action('123', 'type', 'get', 1234, 'thisproject/repo-is-not-ok'); - const result = await processor.exec(null, action, authList); + const action = new actions.Action('123', 'type', 'get', 1234, TEST_NON_EXISTENT_REPO.url); + const result = await processor.exec(null, action); expect(result.error).to.be.true; }); }); diff --git a/test/testRouteFilter.test.js b/test/testRouteFilter.test.js index e9ec026b8..1102a7e41 100644 --- a/test/testRouteFilter.test.js +++ b/test/testRouteFilter.test.js @@ -1,23 +1,92 @@ /* eslint-disable max-len */ -const chai = require('chai'); -const validGitRequest = require('../src/proxy/routes').validGitRequest; -const stripGitHubFromGitPath = require('../src/proxy/routes').stripGitHubFromGitPath; +import * as chai from 'chai'; +import { + validGitRequest, + processUrlPath, + processGitUrl, + processGitURLForNameAndOrg, +} from '../src/proxy/routes/helper'; chai.should(); const expect = chai.expect; -describe('url filters for proxying ', function () { - it('stripGitHubFromGitPath should return the sanitized URL with owner & repo removed', function () { - expect(stripGitHubFromGitPath('/octocat/hello-world.git/info/refs?service=git-upload-pack')).eq( - '/info/refs?service=git-upload-pack', +describe('url helpers and filter functions used in the proxy', function () { + it('processUrlPath should return breakdown of a proxyd path, separating the path to repository from the git operation path', function () { + expect( + processUrlPath('/github.com/octocat/hello-world.git/info/refs?service=git-upload-pack'), + ).to.deep.eq({ + repoPath: '/github.com/octocat/hello-world.git', + gitPath: '/info/refs?service=git-upload-pack', + }); + }); + + it('processUrlPath should return breakdown of a legacy proxy path, separating the path to repository from the git operation path', function () { + expect(processUrlPath('/octocat/hello-world.git/info/refs?service=git-upload-pack')).to.deep.eq( + { repoPath: '/octocat/hello-world.git', gitPath: '/info/refs?service=git-upload-pack' }, ); }); - it('stripGitHubFromGitPath should return undefined if the url', function () { - expect(stripGitHubFromGitPath('/octocat/hello-world')).undefined; + it('processUrlPath should return breakdown of a legacy proxy path, separating the path to repository when git path is just /', function () { + expect(processUrlPath('/octocat/hello-world.git/')).to.deep.eq({ + repoPath: '/octocat/hello-world.git', + gitPath: '/', + }); }); + it('processUrlPath should return breakdown of a legacy proxy path, separating the path to repository when no path is present', function () { + expect(processUrlPath('/octocat/hello-world.git')).to.deep.eq({ + repoPath: '/octocat/hello-world.git', + gitPath: '/', + }); + }); + + it("processUrlPath should return null if the url couldn't be parsed", function () { + expect(processUrlPath('/octocat/hello-world')).to.be.null; + }); + + it('processGitUrl should return breakdown of a git URL separating out the protocol, host and repository path', function () { + expect(processGitUrl('https://somegithost.com:1234/octocat/hello-world.git')).to.deep.eq({ + protocol: 'https://', + host: 'somegithost.com:1234', + repoPath: '/octocat/hello-world.git', + }); + }); + + it('processGitUrl should return breakdown of a git URL separating out the protocol, host and repository path and discard any git operation path', function () { + expect( + processGitUrl( + 'https://somegithost.com:1234/octocat/hello-world.git/info/refs?service=git-upload-pack', + ), + ).to.deep.eq({ + protocol: 'https://', + host: 'somegithost.com:1234', + repoPath: '/octocat/hello-world.git', + }); + }); + + it('processGitURLForNameAndOrg should return breakdown of a git URL path separating out the protocol, origin and repository path', function () { + expect(processGitURLForNameAndOrg('github.com/octocat/hello-world.git')).to.deep.eq({ + project: 'octocat', + repoName: 'hello-world.git', + }); + }); + + it('processGitURLForNameAndOrg should return breakdown of a git repository URL separating out the project (organisation) and repository name', function () { + expect(processGitURLForNameAndOrg('https://github.com:80/octocat/hello-world.git')).to.deep.eq({ + project: 'octocat', + repoName: 'hello-world.git', + }); + }); + + // it('processLegacyProxyPathForNameAndOrg should return breakdown of a legacy proxy path separating out the project (organisation), repository name and a predicted github URL', function () { + // expect(processLegacyProxyPathForNameAndOrg('/octocat/hello-world.git')).to.deep.eq({ + // project: 'octocat', + // repoName: 'hello-world', + // url: 'https://github.com/octocat/hello-world.git', + // }); + // }); + it('validGitRequest should return true for safe requests on expected URLs', function () { [ '/info/refs?service=git-upload-pack', From 350bb0cb3d5f2a4c931d139b43ee091873811d79 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 29 May 2025 18:09:29 +0100 Subject: [PATCH 12/76] feat(key on repo url): remove defunct config variable --- proxy.config.json | 1 - 1 file changed, 1 deletion(-) diff --git a/proxy.config.json b/proxy.config.json index 6b2970c30..bdaedff4f 100644 --- a/proxy.config.json +++ b/proxy.config.json @@ -1,5 +1,4 @@ { - "proxyUrl": "https://github.com", "cookieSecret": "cookie secret", "sessionMaxAgeHours": 12, "rateLimit": { From b8973bbb34b8eb3f52b057537dfad8c483544fab Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 29 May 2025 18:11:11 +0100 Subject: [PATCH 13/76] feat(key on repo url): remove unneeded backwards compat in DB --- src/db/index.ts | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/db/index.ts b/src/db/index.ts index b90bbfa1f..518f64468 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -92,27 +92,6 @@ export const createRepo = async (repo: AuthorisedRepo) => { return sink.createRepo(toCreate) as Promise>; }; -export const getRepoByUrl = async (repoUrl: string) => { - const response = await sink.getRepoByUrl(repoUrl); - // backwards compatibility - if (!response) { - // parse github URLs into org and repo names and fallback to legacy retrieval by repo name - const regex = /.*\/([\w_.-]+?)(\.git)?$/m; - const match = regex.exec(repoUrl); - let repoName = ''; - - if (match && match[1]) { - repoName = match[1]; - } else { - const errorMessage = `Cannot parse repository name from ${repoUrl}`; - throw new Error(errorMessage); - } - - return sink.getRepo(repoName); - } - return response; -}; - export const isUserPushAllowed = async (url: string, user: string) => { user = user.toLowerCase(); return new Promise(async (resolve) => { @@ -183,7 +162,7 @@ export const cancel = (id: string): Promise<{ message: string }> => sink.cancel( export const reject = (id: string): Promise<{ message: string }> => sink.reject(id); export const getRepos = (query?: object): Promise => sink.getRepos(query); export const getRepo = (name: string): Promise => sink.getRepo(name); -// export const getRepoByUrl = (url: string): Promise => sink.getRepoByUrl(url); +export const getRepoByUrl = (url: string): Promise => sink.getRepoByUrl(url); export const getRepoById = (_id: string): Promise => sink.getRepoById(_id); export const addUserCanPush = (_id: string, user: string): Promise => sink.addUserCanPush(_id, user); From b48df7ae8b6012fb8f704ea6c45f5cc930d03184 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 29 May 2025 18:48:56 +0100 Subject: [PATCH 14/76] feat(key on repo url): resolving issues in refactored proxy based on testing --- .../processors/pre-processor/parseAction.ts | 9 ++++-- src/proxy/routes/helper.ts | 16 +++++++--- src/proxy/routes/index.ts | 32 ++++++++++++------- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/proxy/processors/pre-processor/parseAction.ts b/src/proxy/processors/pre-processor/parseAction.ts index d1a0da8d6..ddf854caf 100644 --- a/src/proxy/processors/pre-processor/parseAction.ts +++ b/src/proxy/processors/pre-processor/parseAction.ts @@ -27,14 +27,19 @@ const exec = async (req: { // Proxy URLs take the form https://:// // e.g. https://git-proxy-instance.com:8443/github.com/finos/git-proxy.git // We'll receive /github.com/finos/git-proxy.git as the req.url / req.originalUrl - // Add protocol (assume SSL) to reconstruct full URL - let url = 'https://' + (pathBreakdown?.repoPath ?? 'NOT-FOUND'); + // Add protocol (assume SSL) to reconstruct full URL - noting path will start with a / + let url = 'https:/' + (pathBreakdown?.repoPath ?? 'NOT-FOUND'); + + console.log(`Parse action calculated repo URL: ${url} for inbound URL path: ${req.originalUrl}`); if (!(await db.getRepoByUrl(url))) { // fallback for legacy proxy URLs // legacy git proxy paths took the form: https://:/ // by assuming the host was github.com url = 'https://github.com' + (pathBreakdown?.repoPath ?? 'NOT-FOUND'); + console.log( + `Parse action fallback calculated repo URL: ${url} for inbound URL path: ${req.originalUrl}`, + ); } return new Action(id.toString(), type, req.method, timestamp, url); diff --git a/src/proxy/routes/helper.ts b/src/proxy/routes/helper.ts index ccabf97c4..dc0808760 100644 --- a/src/proxy/routes/helper.ts +++ b/src/proxy/routes/helper.ts @@ -10,11 +10,17 @@ export type GitUrlBreakdown = { protocol: string; host: string; repoPath: string * git endpoint and discarding any git path (specific operation) that comes after * the .git element. * - * E.g. Processing https://github.com/finos/git-proxy.git/info/refs?service=git-upload-pack + * E.g. Processing https://github.com/finos/git-proxy.git/info/refs?service=git-upload-pack * would produce: * - protocol: https:// * - host: github.com - * - repoPath: finos/git-proxy.git + * - repoPath: /finos/git-proxy.git + * + * and processing https://someOtherHost.com:8080/repo.git + * would produce: + * - protocol: https:// + * - host: someOtherHost.com:8080 + * - repoPath: /repo.git * * @param {string} url The URL to process * @return {GitUrlBreakdown | null} A breakdown of the components of the URL. @@ -177,18 +183,18 @@ export const validGitRequest = (gitPath: string, headers: any): boolean => { }; /** - * Collect the Set of all origins (protocol, host and port if specified) that + * Collect the Set of all host (host and port if specified) that * will be proxying requests for, to be used to initialize the proxy. * * @return {string[]} an array of origins */ -export const getAllProxiedOrigins = async (): Promise => { +export const getAllProxiedHosts = async (): Promise => { const repos = await db.getRepos(); const origins = new Set(); repos.forEach((repo) => { const parsedUrl = processGitUrl(repo.url); if (parsedUrl) { - origins.add(parsedUrl.protocol + parsedUrl.host); + origins.add(parsedUrl.host); } // failures are logged by parsing util fn }); return Array.from(origins); diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index d3759846a..27dd59fdc 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -1,7 +1,7 @@ import { Router } from 'express'; import proxy from 'express-http-proxy'; import { executeChain } from '../chain'; -import { processUrlPath, validGitRequest, getAllProxiedOrigins } from './helper'; +import { processUrlPath, validGitRequest, getAllProxiedHosts } from './helper'; import { ProxyOptions } from 'express-http-proxy'; const proxyFilter: ProxyOptions['filter'] = async (req, res) => { @@ -18,6 +18,7 @@ const proxyFilter: ProxyOptions['filter'] = async (req, res) => { !validGitRequest(urlComponents.gitPath, req.headers) ) { res.status(400).send('Invalid request received'); + console.log('action blocked'); return false; } @@ -55,7 +56,7 @@ const proxyFilter: ProxyOptions['filter'] = async (req, res) => { return true; } catch (e) { - console.error(e); + console.error('Error occurred in proxy filter function ', e); return false; } }; @@ -63,18 +64,24 @@ const proxyFilter: ProxyOptions['filter'] = async (req, res) => { const handleMessage = (message: string): string => { const errorMessage = `\t${message}`; const len = 6 + new TextEncoder().encode(errorMessage).length; - const prefix = len.toString(16); const packetMessage = `${prefix.padStart(4, '0')}\x02${errorMessage}\n0000`; return packetMessage; }; -const getRequestPathResolver: (origin: string) => ProxyOptions['proxyReqPathResolver'] = ( - origin, +const getRequestPathResolver: (prefix: string) => ProxyOptions['proxyReqPathResolver'] = ( + prefix, ) => { return (req) => { - const url = origin + req.originalUrl; - console.log('Sending request to ' + url); + let url; + // try to prevent too many slashes in the URL + if (prefix.endsWith('/') && req.originalUrl.startsWith('/')) { + url = prefix.substring(0, prefix.length - 1) + req.originalUrl; + } else { + url = prefix + req.originalUrl; + } + + console.log(`Sending request to ${url}`); return url; }; }; @@ -96,18 +103,19 @@ const proxyErrorHandler: ProxyOptions['proxyErrorHandler'] = (err, res, next) => // eslint-disable-next-line new-cap const router = Router(); -getAllProxiedOrigins().then((originsToProxy) => { - // TODO: this will only happen on startup (I think...) We'll need to add routes at runtime when new origins are added? Or force a restart for the proxy to work +getAllProxiedHosts().then((originsToProxy) => { + // TODO: this will only happen on startup. We'll need to add routes at runtime when new origins are added? Or force a restart for the proxy to work // Middlewares are processed in the order that they are added, if one applies and then doesn't call `next` then subsequent ones are not applied. // Hence, we define known origins first, then a catch all route for backwards compatibility originsToProxy.forEach((origin) => { + console.log(`setting up origin '${origin}'`); router.use( '/' + origin, - proxy(origin, { + proxy('https://' + origin, { preserveHostHdr: false, filter: proxyFilter, - proxyReqPathResolver: getRequestPathResolver(origin), + proxyReqPathResolver: getRequestPathResolver('https://'), // no need to add host as it's in the URL proxyReqOptDecorator: proxyReqOptDecorator, proxyReqBodyDecorator: proxyReqBodyDecorator, proxyErrorHandler: proxyErrorHandler, @@ -118,7 +126,7 @@ getAllProxiedOrigins().then((originsToProxy) => { // Catch-all route for backwards compatibility router.use( '/', - proxy(origin, { + proxy('https://github.com', { preserveHostHdr: false, filter: proxyFilter, proxyReqPathResolver: getRequestPathResolver('https://github.com'), From 451b18a35f90166f0431a1c7b3f767e974785c66 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 29 May 2025 18:50:30 +0100 Subject: [PATCH 15/76] feat(key on repo url): refactor UI to use repo Ids, updated urls and making github fns optional --- src/service/routes/repo.js | 2 +- src/ui/services/repo.js | 18 ++--- .../components/PushesTable.jsx | 47 ++++++------ src/ui/views/PushDetails/PushDetails.jsx | 72 ++++++++++++------- .../views/RepoDetails/Components/AddUser.jsx | 8 +-- src/ui/views/RepoDetails/RepoDetails.jsx | 25 +++---- src/ui/views/RepoList/Components/NewRepo.jsx | 17 +++-- .../RepoList/Components/RepoOverview.jsx | 12 ++-- .../RepoList/Components/Repositories.jsx | 4 +- 9 files changed, 119 insertions(+), 86 deletions(-) diff --git a/src/service/routes/repo.js b/src/service/routes/repo.js index 575d6ec2a..6f77594f0 100644 --- a/src/service/routes/repo.js +++ b/src/service/routes/repo.js @@ -134,7 +134,7 @@ router.post('/', async (req, res) => { const repo = await db.getRepoByUrl(req.body.url); if (repo) { res.status(409).send({ - message: 'Repository already exists!', + message: `Repository ${req.body.url} already exists!`, }); } else { try { diff --git a/src/ui/services/repo.js b/src/ui/services/repo.js index 27d898c75..b0bfc3385 100644 --- a/src/ui/services/repo.js +++ b/src/ui/services/repo.js @@ -9,8 +9,8 @@ const config = { withCredentials: true, }; -const canAddUser = (repoName, user, action) => { - const url = new URL(`${baseUrl}/repo/${repoName}`); +const canAddUser = (repoId, user, action) => { + const url = new URL(`${baseUrl}/repo/${repoId}`); return axios .get(url.toString(), config) .then((response) => { @@ -94,10 +94,10 @@ const addRepo = async (onClose, setError, data) => { }); }; -const addUser = async (repoName, user, action) => { - const canAdd = await canAddUser(repoName, user, action); +const addUser = async (repoId, user, action) => { + const canAdd = await canAddUser(repoId, user, action); if (canAdd) { - const url = new URL(`${baseUrl}/repo/${repoName}/user/${action}`); + const url = new URL(`${baseUrl}/repo/${repoId}/user/${action}`); const data = { username: user }; await axios .patch(url, data, { withCredentials: true, headers: { 'X-CSRF-TOKEN': getCookie('csrf') } }) @@ -111,8 +111,8 @@ const addUser = async (repoName, user, action) => { } }; -const deleteUser = async (user, repoName, action) => { - const url = new URL(`${baseUrl}/repo/${repoName}/user/${action}/${user}`); +const deleteUser = async (user, repoId, action) => { + const url = new URL(`${baseUrl}/repo/${repoId}/user/${action}/${user}`); await axios .delete(url, { withCredentials: true, headers: { 'X-CSRF-TOKEN': getCookie('csrf') } }) @@ -122,8 +122,8 @@ const deleteUser = async (user, repoName, action) => { }); }; -const deleteRepo = async (repoName) => { - const url = new URL(`${baseUrl}/repo/${repoName}/delete`); +const deleteRepo = async (repoId) => { + const url = new URL(`${baseUrl}/repo/${repoId}/delete`); await axios .delete(url, { withCredentials: true, headers: { 'X-CSRF-TOKEN': getCookie('csrf') } }) diff --git a/src/ui/views/OpenPushRequests/components/PushesTable.jsx b/src/ui/views/OpenPushRequests/components/PushesTable.jsx index fad06e27e..ef6c1ff95 100644 --- a/src/ui/views/OpenPushRequests/components/PushesTable.jsx +++ b/src/ui/views/OpenPushRequests/components/PushesTable.jsx @@ -95,6 +95,9 @@ export default function PushesTable(props) { {currentItems.reverse().map((row) => { const repoFullName = row.repo.replace('.git', ''); const repoBranch = row.branch.replace('refs/heads/', ''); + const repoUrl = row.url; + const repoWebUrl = repoUrl.replace('.git', ''); + const isGitHub = repoUrl.startsWith('https://github.com'); return ( @@ -104,22 +107,18 @@ export default function PushesTable(props) { .toString()} - + {repoFullName} - + {repoBranch} @@ -127,22 +126,28 @@ export default function PushesTable(props) { - - {row.commitData[0].committer} - + {isGitHub && ( + + {row.commitData[0].committer} + + )} + {!isGitHub && {row.commitData[0].committer}} - - {row.commitData[0].author} - + {isGitHub && ( + + {row.commitData[0].author} + + )} + {!isGitHub && {row.commitData[0].author}} {row.commitData[0].authorEmail ? ( diff --git a/src/ui/views/PushDetails/PushDetails.jsx b/src/ui/views/PushDetails/PushDetails.jsx index cb094ab1b..9fb38de12 100644 --- a/src/ui/views/PushDetails/PushDetails.jsx +++ b/src/ui/views/PushDetails/PushDetails.jsx @@ -95,6 +95,9 @@ export default function Dashboard() { const repoFullName = data.repo.replace('.git', ''); const repoBranch = data.branch.replace('refs/heads/', ''); + const repoUrl = data.url; + const repoWebUrl = repoUrl.replace('.git', ''); + const isGitHub = repoUrl.startsWith('https://github.com'); const generateIcon = (title) => { switch (title) { @@ -197,17 +200,26 @@ export default function Dashboard() { ) : ( <> - - - + {isGitHub && ( + + + + )}

- - {data.attestation.reviewer.gitAccount} - {' '} + {isGitHub && ( + + {data.attestation.reviewer.gitAccount} + + )} + {!isGitHub && ( + + {data.attestation.reviewer.username} + + )}{' '} approved this contribution

@@ -247,7 +259,7 @@ export default function Dashboard() {

Remote Head

@@ -259,7 +271,7 @@ export default function Dashboard() {

Commit SHA

@@ -270,7 +282,7 @@ export default function Dashboard() {

Repository

- + {repoFullName}

@@ -278,11 +290,7 @@ export default function Dashboard() {

Branch

- + {repoBranch}

@@ -310,18 +318,28 @@ export default function Dashboard() { {moment.unix(c.commitTs || c.commitTimestamp).toString()} - - {c.committer} - + {isGitHub && ( + + {c.committer} + + )} + {!isGitHub && {c.committer}} - - {c.author} - + {isGitHub && ( + + {c.author} + + )} + {!isGitHub && {c.author}} {c.authorEmail ? ( diff --git a/src/ui/views/RepoDetails/Components/AddUser.jsx b/src/ui/views/RepoDetails/Components/AddUser.jsx index afab44a53..a4836a322 100644 --- a/src/ui/views/RepoDetails/Components/AddUser.jsx +++ b/src/ui/views/RepoDetails/Components/AddUser.jsx @@ -19,7 +19,7 @@ import { getUsers } from '../../../services/user'; import { PersonAdd } from '@material-ui/icons'; function AddUserDialog(props) { - const repoName = props.repoName; + const repoId = props.repoId; const type = props.type; const refreshFn = props.refreshFn; const [username, setUsername] = useState(''); @@ -48,7 +48,7 @@ function AddUserDialog(props) { const add = async () => { try { setIsLoading(true); - await addUser(repoName, username, type); + await addUser(repoId, username, type); handleSuccess(); handleClose(); } catch (e) { @@ -145,7 +145,7 @@ AddUserDialog.propTypes = { export default function AddUser(props) { const [open, setOpen] = React.useState(false); - const repoName = props.repoName; + const repoId = props.repoId; const type = props.type; const refreshFn = props.refreshFn; @@ -163,7 +163,7 @@ export default function AddUser(props) { { - getRepo(setIsLoading, setData, setAuth, setIsError, repoName); + getRepo(setIsLoading, setData, setAuth, setIsError, repoId); }, []); const removeUser = async (userToRemove, action) => { - await deleteUser(userToRemove, repoName, action); - getRepo(setIsLoading, setData, setAuth, setIsError, repoName); + await deleteUser(userToRemove, repoId, action); + getRepo(setIsLoading, setData, setAuth, setIsError, repoId); }; - const removeRepository = async (name) => { - await deleteRepo(name); + const removeRepository = async (id) => { + await deleteRepo(id); navigate('/dashboard/repo', { replace: true }); }; - const refresh = () => getRepo(setIsLoading, setData, setAuth, setIsError, repoName); + const refresh = () => getRepo(setIsLoading, setData, setAuth, setIsError, repoId); if (isLoading) return
Loading...
; if (isError) return
Something went wrong ...
; - const { project: org, name, proxyURL } = data || {}; - const cloneURL = `${proxyURL}/${org}/${name}.git`; + const { url: remoteUrl, proxyURL } = data || {}; + const parsedUrl = new URL(remoteUrl); + const cloneURL = `${proxyURL}/${parsedUrl.host}${parsedUrl.port ? `:${parsedUrl.port}` : ''}${parsedUrl.pathname}`; return ( @@ -72,7 +73,7 @@ export default function RepoDetails() { @@ -134,7 +135,7 @@ export default function RepoDetails() { {user.admin && (
- +
)} @@ -179,7 +180,7 @@ export default function RepoDetails() { {user.admin && (
- +
)} diff --git a/src/ui/views/RepoList/Components/NewRepo.jsx b/src/ui/views/RepoList/Components/NewRepo.jsx index e1c912069..b5e93f1c4 100644 --- a/src/ui/views/RepoList/Components/NewRepo.jsx +++ b/src/ui/views/RepoList/Components/NewRepo.jsx @@ -53,8 +53,8 @@ function AddRepositoryDialog(props) { maxUser: 1, }; - if (data.project.trim().length == 0 || data.project.length > 100) { - setError('project name length unexpected'); + if (data.project.length > 100) { + setError('organisation name is too long'); return; } @@ -64,7 +64,11 @@ function AddRepositoryDialog(props) { } try { - new URL(data.url); + const parsedUrl = new URL(data.url); + if (!parsedUrl.pathname.endsWith('.git')) { + setError('Invalid git URL - Git URLs should end with .git'); + return; + } } catch { setError('Invalid URL'); return; @@ -73,6 +77,7 @@ function AddRepositoryDialog(props) { try { await addRepo(onClose, setError, data); handleSuccess(data); + handleClose(); } catch (e) { if (e.message) { @@ -124,7 +129,7 @@ function AddRepositoryDialog(props) { aria-describedby='project-helper-text' onChange={(e) => setProject(e.target.value)} /> - GitHub Organization + Organization or path @@ -136,7 +141,7 @@ function AddRepositoryDialog(props) { aria-describedby='name-helper-text' onChange={(e) => setName(e.target.value)} /> - GitHub Repository Name + Git Repository Name @@ -149,7 +154,7 @@ function AddRepositoryDialog(props) { aria-describedby='url-helper-text' onChange={(e) => setUrl(e.target.value)} /> - GitHub Repository URL + Git Repository URL diff --git a/src/ui/views/RepoList/Components/RepoOverview.jsx b/src/ui/views/RepoList/Components/RepoOverview.jsx index 826f78c97..bd3706ce6 100644 --- a/src/ui/views/RepoList/Components/RepoOverview.jsx +++ b/src/ui/views/RepoList/Components/RepoOverview.jsx @@ -578,6 +578,7 @@ export default function Repositories(props) { getGitHubRepository(); }, [props.data.project, props.data.name]); + // TODO add support for GitLab API: https://docs.gitlab.com/api/projects/#get-a-single-project const getGitHubRepository = async () => { await axios .get(`https://api.github.com/repos/${props.data.project}/${props.data.name}`) @@ -585,19 +586,22 @@ export default function Repositories(props) { setGitHub(res.data); }) .catch((error) => { - setErrorMessage(`Error fetching GitHub repository ${props.data.project}/${props.data.name}: ${error}`); + setErrorMessage( + `Error fetching GitHub repository ${props.data.project}/${props.data.name}: ${error}`, + ); setSnackbarOpen(true); }); }; - const { project: org, name, proxyURL } = props?.data || {}; - const cloneURL = `${proxyURL}/${org}/${name}.git`; + const { url: remoteUrl, proxyURL } = props?.data || {}; + const parsedUrl = new URL(remoteUrl); + const cloneURL = `${proxyURL}/${parsedUrl.host}${parsedUrl.port ? `:${parsedUrl.port}` : ''}${parsedUrl.pathname}`; return (
- + {props.data.project}/{props.data.name} diff --git a/src/ui/views/RepoList/Components/Repositories.jsx b/src/ui/views/RepoList/Components/Repositories.jsx index 3b64944f2..e77033c70 100644 --- a/src/ui/views/RepoList/Components/Repositories.jsx +++ b/src/ui/views/RepoList/Components/Repositories.jsx @@ -29,7 +29,7 @@ export default function Repositories(props) { const itemsPerPage = 5; const navigate = useNavigate(); const { user } = useContext(UserContext); - const openRepo = (repo) => navigate(`/dashboard/repo/${repo}`, { replace: true }); + const openRepo = (repoId) => navigate(`/dashboard/repo/${repoId}`, { replace: true }); useEffect(() => { const query = {}; @@ -153,7 +153,7 @@ function GetGridContainerLayOut(props) { {props.data.map((row) => { - if (row.project && row.name) { + if (row.url) { return ; } })} From a4d9bde682619c087eb39e58430d936429a438c6 Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 30 May 2025 15:57:49 +0100 Subject: [PATCH 16/76] feat(key on repo url): fix issues in CLI tests related to change of key field --- packages/git-proxy-cli/test/testCli.test.js | 6 +++--- packages/git-proxy-cli/test/testCliUtils.js | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/git-proxy-cli/test/testCli.test.js b/packages/git-proxy-cli/test/testCli.test.js index a872221c5..33d8d7917 100644 --- a/packages/git-proxy-cli/test/testCli.test.js +++ b/packages/git-proxy-cli/test/testCli.test.js @@ -219,7 +219,7 @@ describe('test git-proxy-cli', function () { before(async function () { await helper.addRepoToDb(TEST_REPO_CONFIG); - await helper.addGitPushToDb(pushId, TEST_REPO); + await helper.addGitPushToDb(pushId, TEST_REPO_CONFIG.url); }); after(async function () { @@ -415,7 +415,7 @@ describe('test git-proxy-cli', function () { before(async function () { await helper.addRepoToDb(TEST_REPO_CONFIG); - await helper.addGitPushToDb(pushId, TEST_REPO); + await helper.addGitPushToDb(pushId, TEST_REPO_CONFIG.url); }); after(async function () { @@ -492,7 +492,7 @@ describe('test git-proxy-cli', function () { before(async function () { await helper.addRepoToDb(TEST_REPO_CONFIG); await helper.addUserToDb('testuser1', 'testpassword', 'test@email.com', gitAccount); - await helper.addGitPushToDb(pushId, TEST_REPO, gitAccount); + await helper.addGitPushToDb(pushId, TEST_REPO_CONFIG.url, gitAccount); }); after(async function () { diff --git a/packages/git-proxy-cli/test/testCliUtils.js b/packages/git-proxy-cli/test/testCliUtils.js index a58ece747..e47a21fd3 100644 --- a/packages/git-proxy-cli/test/testCliUtils.js +++ b/packages/git-proxy-cli/test/testCliUtils.js @@ -30,7 +30,7 @@ async function runCli( expectedExitCode = 0, expectedMessages = null, expectedErrorMessages = null, - debug = false, + debug = true, ) { try { console.log(`cli: '${cli}'`); @@ -177,17 +177,17 @@ async function removeRepoFromDb(repoUrl) { /** * Add a new git push record to the database. * @param {string} id The ID of the git push. - * @param {string} repo The repository of the git push. + * @param {string} repoUrl The repository URL of the git push. * @param {string} user The user who pushed the git push. * @param {boolean} debug Flag to enable logging for debugging. */ -async function addGitPushToDb(id, repo, user = null, debug = false) { +async function addGitPushToDb(id, repoUrl, user = null, debug = false) { const action = new actions.Action( id, 'push', // type 'get', // method Date.now(), // timestamp - repo, + repoUrl, ); action.user = user; const step = new steps.Step( From a5beb574b6a18a2061ecc9191257f082235408f5 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 25 Mar 2025 12:43:47 +0000 Subject: [PATCH 17/76] fix: add indexes, compaction and consistent lowercasing of inputs in file-based DB implementation --- src/db/file/repo.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index c7acdbe0a..a214cd4e7 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -38,6 +38,7 @@ export const getRepos = async (query: any = {}): Promise => { }; export const getRepo = async (name: string): Promise => { + name = name.toLowerCase(); return new Promise((resolve, reject) => { db.findOne({ name: name.toLowerCase() }, (err: Error | null, doc: Repo) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query From 4b305dd1f156481e27ed574118ca46c64162c8af Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 1 Apr 2025 14:22:18 +0100 Subject: [PATCH 18/76] test: use unique emails for users in tests and remove afterwards --- test/addRepoTest.test.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/addRepoTest.test.js b/test/addRepoTest.test.js index 04979afa7..e21c28823 100644 --- a/test/addRepoTest.test.js +++ b/test/addRepoTest.test.js @@ -213,10 +213,8 @@ describe('add new repo', async () => { after(async function () { await service.httpServer.close(); - - // don't clean up data as cypress tests rely on it being present - // await db.deleteRepo('test-repo'); - // await db.deleteUser('u1'); - // await db.deleteUser('u2'); + await db.deleteRepo('test-repo'); + await db.deleteUser('u1'); + await db.deleteUser('u2'); }); }); From ecee528a08ffa425d825b2646cc207ef08d6322d Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 25 Mar 2025 12:43:47 +0000 Subject: [PATCH 19/76] fix: add indexes, compaction and consistent lowercasing of inputs in file-based DB implementation --- src/db/file/repo.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index a214cd4e7..2f12c410a 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -13,7 +13,9 @@ if (!fs.existsSync('./.data')) fs.mkdirSync('./.data'); if (!fs.existsSync('./.data/db')) fs.mkdirSync('./.data/db'); const db = new Datastore({ filename: './.data/db/repos.db', autoload: true }); + db.ensureIndex({ fieldName: 'url', unique: true }); +db.ensureIndex({ fieldName: 'name', unique: false }); db.setAutocompactionInterval(COMPACTION_INTERVAL); export const getRepos = async (query: any = {}): Promise => { From 092435d555802cb8af252aaea177684f9f9929d4 Mon Sep 17 00:00:00 2001 From: Kris West Date: Wed, 16 Apr 2025 17:54:57 +0100 Subject: [PATCH 20/76] test: more code coverage in DB and service/routes/repo --- test/testDb.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/testDb.test.js b/test/testDb.test.js index d290e496f..e295a3639 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -128,6 +128,7 @@ describe('Database clients', async () => { const repo = await db.getRepoByUrl(TEST_REPO.url); await db.deleteRepo(repo._id); const repos = await db.getRepos(); + const cleanRepos = cleanResponseData(TEST_REPO, repos); expect(cleanRepos).to.not.deep.include(TEST_REPO); }); From a44e3d1986848f69e52fcefcfa1c5df9cf702aa6 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 17 Apr 2025 11:33:25 +0100 Subject: [PATCH 21/76] test: more DB test coverage --- test/testDb.test.js | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/testDb.test.js b/test/testDb.test.js index e295a3639..80964cc0d 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -336,6 +336,16 @@ describe('Database clients', async () => { expect(threwError).to.be.true; }); + it('should throw an error when de-authorising a user to push on non-existent repo', async function () { + let threwError = false; + try { + await db.removeUserCanPush('non-existent-repo', TEST_USER.username); + } catch (e) { + threwError = true; + } + expect(threwError).to.be.true; + }); + it("should be able to de-authorise a user to push and confirm that they can't", async function () { let threwError = false; try { @@ -391,6 +401,33 @@ describe('Database clients', async () => { expect(allowed).to.be.false; }); + it('should NOT throw an error when checking whether a user can push on non-existent repo', async function () { + let threwError = false; + try { + // uppercase the filter value to confirm db client is lowercasing inputs + const allowed = await db.isUserPushAllowed('non-existent-repo', TEST_USER.username); + expect(allowed).to.be.false; + } catch (e) { + threwError = true; + } + expect(threwError).to.be.false; + }); + + it('should NOT throw an error when checking whether a user can authorise on non-existent repo', async function () { + let threwError = false; + try { + // uppercase the filter value to confirm db client is lowercasing inputs + const allowed = await db.canUserApproveRejectPushRepo( + 'non-existent-repo', + TEST_USER.username, + ); + expect(allowed).to.be.false; + } catch (e) { + threwError = true; + } + expect(threwError).to.be.false; + }); + it('should be able to create a push', async function () { await db.writeAudit(TEST_PUSH); const pushes = await db.getPushes(); From 97115d6eb24b6618fdae4ca567fc6721d0b73c32 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 20 May 2025 12:48:17 +0100 Subject: [PATCH 22/76] feat(key on repo url): refactor tests to use repo URLs rather than names (WIP) --- test/testDb.test.js | 89 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/test/testDb.test.js b/test/testDb.test.js index 80964cc0d..9846f7d0c 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -294,8 +294,7 @@ describe('Database clients', async () => { it('should throw an error when authorising a user to push on non-existent repo', async function () { let threwError = false; try { - // uppercase the filter value to confirm db client is lowercasing inputs - await db.addUserCanPush(TEST_NONEXISTENT_REPO._id, TEST_USER.username); + await db.addUserCanPush(TEST_NONEXISTENT_REPO.url, TEST_USER.username); } catch (e) { threwError = true; } @@ -339,7 +338,7 @@ describe('Database clients', async () => { it('should throw an error when de-authorising a user to push on non-existent repo', async function () { let threwError = false; try { - await db.removeUserCanPush('non-existent-repo', TEST_USER.username); + await db.removeUserCanPush(TEST_NONEXISTENT_REPO.url, TEST_USER.username); } catch (e) { threwError = true; } @@ -356,10 +355,10 @@ describe('Database clients', async () => { const repo = await db.getRepoByUrl(TEST_REPO.url); // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanPush(repo._id, TEST_USER.username.toUpperCase()); + await db.removeUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); // repeat, should not throw an error if already unset - await db.removeUserCanPush(repo._id, TEST_USER.username.toUpperCase()); + await db.removeUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); // confirm the setting exists allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); @@ -378,34 +377,88 @@ describe('Database clients', async () => { it('should throw an error when authorising a user to authorise on non-existent repo', async function () { let threwError = false; try { - await db.addUserCanAuthorise(TEST_NONEXISTENT_REPO._id, TEST_USER.username); + await db.addUserCanAuthorise(TEST_NONEXISTENT_REPO.url, TEST_USER.username); } catch (e) { threwError = true; } expect(threwError).to.be.true; }); + it('should be able to authorise a user to authorise and confirm that they can', async function () { + let threwError = false; + try { + // repo should already exist after a previous test + let allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); + expect(allowed).to.be.false; + + // uppercase the filter value to confirm db client is lowercasing inputs + await db.addUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); + + // repeat, should not throw an error if already set + await db.addUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); + + // confirm the setting exists + allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); + expect(allowed).to.be.true; + + // confirm that casing doesn't matter + allowed = await db.canUserApproveRejectPushRepo( + TEST_REPO.url, + TEST_USER.username.toUpperCase(), + ); + expect(allowed).to.be.true; + } catch (e) { + console.error('Error thrown ', e); + threwError = true; + } + expect(threwError).to.be.false; + }); + it('should throw an error when de-authorising a user to push on non-existent repo', async function () { let threwError = false; try { // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanAuthorise(TEST_NONEXISTENT_REPO._id, TEST_USER.username); + await db.removeUserCanAuthorise(TEST_NONEXISTENT_REPO.url, TEST_USER.username); } catch (e) { threwError = true; } expect(threwError).to.be.true; }); - it('should NOT throw an error when checking whether a user can push on non-existent repo', async function () { - const allowed = await db.isUserPushAllowed(TEST_NONEXISTENT_REPO.url, TEST_USER.username); - expect(allowed).to.be.false; + it("should be able to de-authorise a user to authorise and confirm that they can't", async function () { + let threwError = false; + try { + // repo should already exist after a previous test and user should be an authoriser + let allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); + expect(allowed).to.be.true; + + // uppercase the filter value to confirm db client is lowercasing inputs + await db.removeUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); + + // repeat, should not throw an error if already set + await db.removeUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); + + // confirm the setting was removed + allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); + expect(allowed).to.be.false; + + // confirm that casing doesn't matter + allowed = await db.canUserApproveRejectPushRepo( + TEST_REPO.url, + TEST_USER.username.toUpperCase(), + ); + expect(allowed).to.be.false; + } catch (e) { + console.error('Error thrown ', e); + threwError = true; + } + expect(threwError).to.be.false; }); it('should NOT throw an error when checking whether a user can push on non-existent repo', async function () { let threwError = false; try { - // uppercase the filter value to confirm db client is lowercasing inputs - const allowed = await db.isUserPushAllowed('non-existent-repo', TEST_USER.username); + const allowed = await db.isUserPushAllowed(TEST_NONEXISTENT_REPO.url, TEST_USER.username); expect(allowed).to.be.false; } catch (e) { threwError = true; @@ -418,7 +471,7 @@ describe('Database clients', async () => { try { // uppercase the filter value to confirm db client is lowercasing inputs const allowed = await db.canUserApproveRejectPushRepo( - 'non-existent-repo', + TEST_NONEXISTENT_REPO.url, TEST_USER.username, ); expect(allowed).to.be.false; @@ -533,7 +586,7 @@ describe('Database clients', async () => { expect(allowed).to.be.false; // authorise user and recheck - await db.addUserCanPush(repo._id, TEST_USER.username); + await db.addUserCanPush(TEST_PUSH.url, TEST_USER.username); allowed = await db.canUserCancelPush(TEST_PUSH.id, TEST_USER.username); expect(allowed).to.be.true; @@ -548,6 +601,7 @@ describe('Database clients', async () => { expect(threwError).to.be.false; // clean up await db.deletePush(TEST_PUSH.id); + await db.removeUserCanPush(TEST_PUSH.url, TEST_USER.username); }); it('should be able to check if a user can approve/reject push', async function () { @@ -565,7 +619,7 @@ describe('Database clients', async () => { const repo = await db.getRepoByUrl(TEST_REPO.url); // authorise user and recheck - await db.addUserCanAuthorise(repo._id, TEST_USER.username); + await db.addUserCanAuthorise(TEST_PUSH.url, TEST_USER.username); allowed = await db.canUserApproveRejectPush(TEST_PUSH.id, TEST_USER.username); expect(allowed).to.be.true; @@ -580,12 +634,11 @@ describe('Database clients', async () => { // clean up await db.deletePush(TEST_PUSH.id); + await db.removeUserCanAuthorise(TEST_PUSH.url, TEST_USER.username); }); after(async function () { - // _id is autogenerated by the DB so we need to retrieve it before we can use it - const repo = await db.getRepoByUrl(TEST_REPO.url); - await db.deleteRepo(repo._id, true); + await db.deleteRepo(TEST_REPO.url); await db.deleteUser(TEST_USER.username); await db.deletePush(TEST_PUSH.id); }); From 2e0f39641a5835e7a4dcbeb5619dc44c574ce9c5 Mon Sep 17 00:00:00 2001 From: Raimund Hook Date: Tue, 20 May 2025 15:25:02 +0100 Subject: [PATCH 23/76] feat(key on repo url): allow file db to delete multiple records Signed-off-by: Raimund Hook --- test/testDb.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testDb.test.js b/test/testDb.test.js index 9846f7d0c..f2827aee9 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -638,7 +638,7 @@ describe('Database clients', async () => { }); after(async function () { - await db.deleteRepo(TEST_REPO.url); + await db.deleteRepo(TEST_REPO.url, true); await db.deleteUser(TEST_USER.username); await db.deletePush(TEST_PUSH.id); }); From ddb823ee34f59a94ee15f757efaa2fafa035d624 Mon Sep 17 00:00:00 2001 From: Kris West Date: Wed, 21 May 2025 00:56:21 +0100 Subject: [PATCH 24/76] feat(key on repo url): switching to key on _id for in API + consolidate non-specific db fns --- test/testDb.test.js | 117 +++++--------------------------------------- 1 file changed, 13 insertions(+), 104 deletions(-) diff --git a/test/testDb.test.js b/test/testDb.test.js index f2827aee9..d290e496f 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -128,7 +128,6 @@ describe('Database clients', async () => { const repo = await db.getRepoByUrl(TEST_REPO.url); await db.deleteRepo(repo._id); const repos = await db.getRepos(); - const cleanRepos = cleanResponseData(TEST_REPO, repos); expect(cleanRepos).to.not.deep.include(TEST_REPO); }); @@ -294,7 +293,8 @@ describe('Database clients', async () => { it('should throw an error when authorising a user to push on non-existent repo', async function () { let threwError = false; try { - await db.addUserCanPush(TEST_NONEXISTENT_REPO.url, TEST_USER.username); + // uppercase the filter value to confirm db client is lowercasing inputs + await db.addUserCanPush(TEST_NONEXISTENT_REPO._id, TEST_USER.username); } catch (e) { threwError = true; } @@ -335,16 +335,6 @@ describe('Database clients', async () => { expect(threwError).to.be.true; }); - it('should throw an error when de-authorising a user to push on non-existent repo', async function () { - let threwError = false; - try { - await db.removeUserCanPush(TEST_NONEXISTENT_REPO.url, TEST_USER.username); - } catch (e) { - threwError = true; - } - expect(threwError).to.be.true; - }); - it("should be able to de-authorise a user to push and confirm that they can't", async function () { let threwError = false; try { @@ -355,10 +345,10 @@ describe('Database clients', async () => { const repo = await db.getRepoByUrl(TEST_REPO.url); // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); + await db.removeUserCanPush(repo._id, TEST_USER.username.toUpperCase()); // repeat, should not throw an error if already unset - await db.removeUserCanPush(TEST_REPO.url, TEST_USER.username.toUpperCase()); + await db.removeUserCanPush(repo._id, TEST_USER.username.toUpperCase()); // confirm the setting exists allowed = await db.isUserPushAllowed(TEST_REPO.url, TEST_USER.username); @@ -377,108 +367,27 @@ describe('Database clients', async () => { it('should throw an error when authorising a user to authorise on non-existent repo', async function () { let threwError = false; try { - await db.addUserCanAuthorise(TEST_NONEXISTENT_REPO.url, TEST_USER.username); + await db.addUserCanAuthorise(TEST_NONEXISTENT_REPO._id, TEST_USER.username); } catch (e) { threwError = true; } expect(threwError).to.be.true; }); - it('should be able to authorise a user to authorise and confirm that they can', async function () { - let threwError = false; - try { - // repo should already exist after a previous test - let allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); - expect(allowed).to.be.false; - - // uppercase the filter value to confirm db client is lowercasing inputs - await db.addUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); - - // repeat, should not throw an error if already set - await db.addUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); - - // confirm the setting exists - allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); - expect(allowed).to.be.true; - - // confirm that casing doesn't matter - allowed = await db.canUserApproveRejectPushRepo( - TEST_REPO.url, - TEST_USER.username.toUpperCase(), - ); - expect(allowed).to.be.true; - } catch (e) { - console.error('Error thrown ', e); - threwError = true; - } - expect(threwError).to.be.false; - }); - it('should throw an error when de-authorising a user to push on non-existent repo', async function () { let threwError = false; try { // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanAuthorise(TEST_NONEXISTENT_REPO.url, TEST_USER.username); + await db.removeUserCanAuthorise(TEST_NONEXISTENT_REPO._id, TEST_USER.username); } catch (e) { threwError = true; } expect(threwError).to.be.true; }); - it("should be able to de-authorise a user to authorise and confirm that they can't", async function () { - let threwError = false; - try { - // repo should already exist after a previous test and user should be an authoriser - let allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); - expect(allowed).to.be.true; - - // uppercase the filter value to confirm db client is lowercasing inputs - await db.removeUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); - - // repeat, should not throw an error if already set - await db.removeUserCanAuthorise(TEST_REPO.url, TEST_USER.username.toUpperCase()); - - // confirm the setting was removed - allowed = await db.canUserApproveRejectPushRepo(TEST_REPO.url, TEST_USER.username); - expect(allowed).to.be.false; - - // confirm that casing doesn't matter - allowed = await db.canUserApproveRejectPushRepo( - TEST_REPO.url, - TEST_USER.username.toUpperCase(), - ); - expect(allowed).to.be.false; - } catch (e) { - console.error('Error thrown ', e); - threwError = true; - } - expect(threwError).to.be.false; - }); - it('should NOT throw an error when checking whether a user can push on non-existent repo', async function () { - let threwError = false; - try { - const allowed = await db.isUserPushAllowed(TEST_NONEXISTENT_REPO.url, TEST_USER.username); - expect(allowed).to.be.false; - } catch (e) { - threwError = true; - } - expect(threwError).to.be.false; - }); - - it('should NOT throw an error when checking whether a user can authorise on non-existent repo', async function () { - let threwError = false; - try { - // uppercase the filter value to confirm db client is lowercasing inputs - const allowed = await db.canUserApproveRejectPushRepo( - TEST_NONEXISTENT_REPO.url, - TEST_USER.username, - ); - expect(allowed).to.be.false; - } catch (e) { - threwError = true; - } - expect(threwError).to.be.false; + const allowed = await db.isUserPushAllowed(TEST_NONEXISTENT_REPO.url, TEST_USER.username); + expect(allowed).to.be.false; }); it('should be able to create a push', async function () { @@ -586,7 +495,7 @@ describe('Database clients', async () => { expect(allowed).to.be.false; // authorise user and recheck - await db.addUserCanPush(TEST_PUSH.url, TEST_USER.username); + await db.addUserCanPush(repo._id, TEST_USER.username); allowed = await db.canUserCancelPush(TEST_PUSH.id, TEST_USER.username); expect(allowed).to.be.true; @@ -601,7 +510,6 @@ describe('Database clients', async () => { expect(threwError).to.be.false; // clean up await db.deletePush(TEST_PUSH.id); - await db.removeUserCanPush(TEST_PUSH.url, TEST_USER.username); }); it('should be able to check if a user can approve/reject push', async function () { @@ -619,7 +527,7 @@ describe('Database clients', async () => { const repo = await db.getRepoByUrl(TEST_REPO.url); // authorise user and recheck - await db.addUserCanAuthorise(TEST_PUSH.url, TEST_USER.username); + await db.addUserCanAuthorise(repo._id, TEST_USER.username); allowed = await db.canUserApproveRejectPush(TEST_PUSH.id, TEST_USER.username); expect(allowed).to.be.true; @@ -634,11 +542,12 @@ describe('Database clients', async () => { // clean up await db.deletePush(TEST_PUSH.id); - await db.removeUserCanAuthorise(TEST_PUSH.url, TEST_USER.username); }); after(async function () { - await db.deleteRepo(TEST_REPO.url, true); + // _id is autogenerated by the DB so we need to retrieve it before we can use it + const repo = await db.getRepoByUrl(TEST_REPO.url); + await db.deleteRepo(repo._id, true); await db.deleteUser(TEST_USER.username); await db.deletePush(TEST_PUSH.id); }); From 928823d47f098b84f4ebe41f3a5df02aacfd6c78 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 22 May 2025 17:51:59 +0100 Subject: [PATCH 25/76] feat(key on repo url): apply proper typing to DB classes Typescript wasn't working on the DB classes due to their dependency imports with require. --- src/db/file/repo.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index 2f12c410a..f68229b68 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -40,7 +40,6 @@ export const getRepos = async (query: any = {}): Promise => { }; export const getRepo = async (name: string): Promise => { - name = name.toLowerCase(); return new Promise((resolve, reject) => { db.findOne({ name: name.toLowerCase() }, (err: Error | null, doc: Repo) => { // ignore for code coverage as neDB rarely returns errors even for an invalid query From 4905970da27668d8802a66ecbfc748a40bc466bd Mon Sep 17 00:00:00 2001 From: Kris West Date: Wed, 28 May 2025 18:32:02 +0100 Subject: [PATCH 26/76] feat(key on repo url): refactoring proxy to support embedding host with fallback --- src/proxy/routes/helper.ts | 2 +- test/testRouteFilter.js | 154 +++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 test/testRouteFilter.js diff --git a/src/proxy/routes/helper.ts b/src/proxy/routes/helper.ts index dc0808760..20d46a8b4 100644 --- a/src/proxy/routes/helper.ts +++ b/src/proxy/routes/helper.ts @@ -183,7 +183,7 @@ export const validGitRequest = (gitPath: string, headers: any): boolean => { }; /** - * Collect the Set of all host (host and port if specified) that + * Collect the Set of all host (host and port if specified) that we * will be proxying requests for, to be used to initialize the proxy. * * @return {string[]} an array of origins diff --git a/test/testRouteFilter.js b/test/testRouteFilter.js new file mode 100644 index 000000000..1102a7e41 --- /dev/null +++ b/test/testRouteFilter.js @@ -0,0 +1,154 @@ +/* eslint-disable max-len */ +import * as chai from 'chai'; +import { + validGitRequest, + processUrlPath, + processGitUrl, + processGitURLForNameAndOrg, +} from '../src/proxy/routes/helper'; + +chai.should(); + +const expect = chai.expect; + +describe('url helpers and filter functions used in the proxy', function () { + it('processUrlPath should return breakdown of a proxyd path, separating the path to repository from the git operation path', function () { + expect( + processUrlPath('/github.com/octocat/hello-world.git/info/refs?service=git-upload-pack'), + ).to.deep.eq({ + repoPath: '/github.com/octocat/hello-world.git', + gitPath: '/info/refs?service=git-upload-pack', + }); + }); + + it('processUrlPath should return breakdown of a legacy proxy path, separating the path to repository from the git operation path', function () { + expect(processUrlPath('/octocat/hello-world.git/info/refs?service=git-upload-pack')).to.deep.eq( + { repoPath: '/octocat/hello-world.git', gitPath: '/info/refs?service=git-upload-pack' }, + ); + }); + + it('processUrlPath should return breakdown of a legacy proxy path, separating the path to repository when git path is just /', function () { + expect(processUrlPath('/octocat/hello-world.git/')).to.deep.eq({ + repoPath: '/octocat/hello-world.git', + gitPath: '/', + }); + }); + + it('processUrlPath should return breakdown of a legacy proxy path, separating the path to repository when no path is present', function () { + expect(processUrlPath('/octocat/hello-world.git')).to.deep.eq({ + repoPath: '/octocat/hello-world.git', + gitPath: '/', + }); + }); + + it("processUrlPath should return null if the url couldn't be parsed", function () { + expect(processUrlPath('/octocat/hello-world')).to.be.null; + }); + + it('processGitUrl should return breakdown of a git URL separating out the protocol, host and repository path', function () { + expect(processGitUrl('https://somegithost.com:1234/octocat/hello-world.git')).to.deep.eq({ + protocol: 'https://', + host: 'somegithost.com:1234', + repoPath: '/octocat/hello-world.git', + }); + }); + + it('processGitUrl should return breakdown of a git URL separating out the protocol, host and repository path and discard any git operation path', function () { + expect( + processGitUrl( + 'https://somegithost.com:1234/octocat/hello-world.git/info/refs?service=git-upload-pack', + ), + ).to.deep.eq({ + protocol: 'https://', + host: 'somegithost.com:1234', + repoPath: '/octocat/hello-world.git', + }); + }); + + it('processGitURLForNameAndOrg should return breakdown of a git URL path separating out the protocol, origin and repository path', function () { + expect(processGitURLForNameAndOrg('github.com/octocat/hello-world.git')).to.deep.eq({ + project: 'octocat', + repoName: 'hello-world.git', + }); + }); + + it('processGitURLForNameAndOrg should return breakdown of a git repository URL separating out the project (organisation) and repository name', function () { + expect(processGitURLForNameAndOrg('https://github.com:80/octocat/hello-world.git')).to.deep.eq({ + project: 'octocat', + repoName: 'hello-world.git', + }); + }); + + // it('processLegacyProxyPathForNameAndOrg should return breakdown of a legacy proxy path separating out the project (organisation), repository name and a predicted github URL', function () { + // expect(processLegacyProxyPathForNameAndOrg('/octocat/hello-world.git')).to.deep.eq({ + // project: 'octocat', + // repoName: 'hello-world', + // url: 'https://github.com/octocat/hello-world.git', + // }); + // }); + + it('validGitRequest should return true for safe requests on expected URLs', function () { + [ + '/info/refs?service=git-upload-pack', + '/info/refs?service=git-receive-pack', + '/git-upload-pack', + '/git-receive-pack', + ].forEach((url) => { + expect( + validGitRequest(url, { + 'user-agent': 'git/2.30.0', + accept: 'application/x-git-upload-pack-request', + }), + ).true; + }); + }); + + it('validGitRequest should return false for unsafe URLs', function () { + ['/', '/foo'].forEach((url) => { + expect( + validGitRequest(url, { + 'user-agent': 'git/2.30.0', + accept: 'application/x-git-upload-pack-request', + }), + ).false; + }); + }); + + it('validGitRequest should return false for a browser request', function () { + expect( + validGitRequest('/', { + 'user-agent': 'Mozilla/5.0', + accept: '*/*', + }), + ).false; + }); + + it('validGitRequest should return false for unexpected combinations of headers & URLs', function () { + // expected Accept=application/x-git-upload-pack + expect( + validGitRequest('/git-upload-pack', { + 'user-agent': 'git/2.30.0', + accept: '*/*', + }), + ).false; + + // expected User-Agent=git/* + expect( + validGitRequest('/info/refs?service=git-upload-pack', { + 'user-agent': 'Mozilla/5.0', + accept: '*/*', + }), + ).false; + }); + + it('validGitRequest should return false for unexpected content-type on certain URLs', function () { + ['application/json', 'text/html', '*/*'].map((accept) => { + expect( + validGitRequest('/git-upload-pack', { + 'user-agent': 'git/2.30.0', + accept: accept, + }), + ).false; + }); + }); +}); From 69d83d0750db0bea8afc8bb1accc52906dba4f44 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 29 May 2025 18:48:56 +0100 Subject: [PATCH 27/76] feat(key on repo url): resolving issues in refactored proxy based on testing --- test/testRouteFilter.js | 154 ---------------------------------------- 1 file changed, 154 deletions(-) delete mode 100644 test/testRouteFilter.js diff --git a/test/testRouteFilter.js b/test/testRouteFilter.js deleted file mode 100644 index 1102a7e41..000000000 --- a/test/testRouteFilter.js +++ /dev/null @@ -1,154 +0,0 @@ -/* eslint-disable max-len */ -import * as chai from 'chai'; -import { - validGitRequest, - processUrlPath, - processGitUrl, - processGitURLForNameAndOrg, -} from '../src/proxy/routes/helper'; - -chai.should(); - -const expect = chai.expect; - -describe('url helpers and filter functions used in the proxy', function () { - it('processUrlPath should return breakdown of a proxyd path, separating the path to repository from the git operation path', function () { - expect( - processUrlPath('/github.com/octocat/hello-world.git/info/refs?service=git-upload-pack'), - ).to.deep.eq({ - repoPath: '/github.com/octocat/hello-world.git', - gitPath: '/info/refs?service=git-upload-pack', - }); - }); - - it('processUrlPath should return breakdown of a legacy proxy path, separating the path to repository from the git operation path', function () { - expect(processUrlPath('/octocat/hello-world.git/info/refs?service=git-upload-pack')).to.deep.eq( - { repoPath: '/octocat/hello-world.git', gitPath: '/info/refs?service=git-upload-pack' }, - ); - }); - - it('processUrlPath should return breakdown of a legacy proxy path, separating the path to repository when git path is just /', function () { - expect(processUrlPath('/octocat/hello-world.git/')).to.deep.eq({ - repoPath: '/octocat/hello-world.git', - gitPath: '/', - }); - }); - - it('processUrlPath should return breakdown of a legacy proxy path, separating the path to repository when no path is present', function () { - expect(processUrlPath('/octocat/hello-world.git')).to.deep.eq({ - repoPath: '/octocat/hello-world.git', - gitPath: '/', - }); - }); - - it("processUrlPath should return null if the url couldn't be parsed", function () { - expect(processUrlPath('/octocat/hello-world')).to.be.null; - }); - - it('processGitUrl should return breakdown of a git URL separating out the protocol, host and repository path', function () { - expect(processGitUrl('https://somegithost.com:1234/octocat/hello-world.git')).to.deep.eq({ - protocol: 'https://', - host: 'somegithost.com:1234', - repoPath: '/octocat/hello-world.git', - }); - }); - - it('processGitUrl should return breakdown of a git URL separating out the protocol, host and repository path and discard any git operation path', function () { - expect( - processGitUrl( - 'https://somegithost.com:1234/octocat/hello-world.git/info/refs?service=git-upload-pack', - ), - ).to.deep.eq({ - protocol: 'https://', - host: 'somegithost.com:1234', - repoPath: '/octocat/hello-world.git', - }); - }); - - it('processGitURLForNameAndOrg should return breakdown of a git URL path separating out the protocol, origin and repository path', function () { - expect(processGitURLForNameAndOrg('github.com/octocat/hello-world.git')).to.deep.eq({ - project: 'octocat', - repoName: 'hello-world.git', - }); - }); - - it('processGitURLForNameAndOrg should return breakdown of a git repository URL separating out the project (organisation) and repository name', function () { - expect(processGitURLForNameAndOrg('https://github.com:80/octocat/hello-world.git')).to.deep.eq({ - project: 'octocat', - repoName: 'hello-world.git', - }); - }); - - // it('processLegacyProxyPathForNameAndOrg should return breakdown of a legacy proxy path separating out the project (organisation), repository name and a predicted github URL', function () { - // expect(processLegacyProxyPathForNameAndOrg('/octocat/hello-world.git')).to.deep.eq({ - // project: 'octocat', - // repoName: 'hello-world', - // url: 'https://github.com/octocat/hello-world.git', - // }); - // }); - - it('validGitRequest should return true for safe requests on expected URLs', function () { - [ - '/info/refs?service=git-upload-pack', - '/info/refs?service=git-receive-pack', - '/git-upload-pack', - '/git-receive-pack', - ].forEach((url) => { - expect( - validGitRequest(url, { - 'user-agent': 'git/2.30.0', - accept: 'application/x-git-upload-pack-request', - }), - ).true; - }); - }); - - it('validGitRequest should return false for unsafe URLs', function () { - ['/', '/foo'].forEach((url) => { - expect( - validGitRequest(url, { - 'user-agent': 'git/2.30.0', - accept: 'application/x-git-upload-pack-request', - }), - ).false; - }); - }); - - it('validGitRequest should return false for a browser request', function () { - expect( - validGitRequest('/', { - 'user-agent': 'Mozilla/5.0', - accept: '*/*', - }), - ).false; - }); - - it('validGitRequest should return false for unexpected combinations of headers & URLs', function () { - // expected Accept=application/x-git-upload-pack - expect( - validGitRequest('/git-upload-pack', { - 'user-agent': 'git/2.30.0', - accept: '*/*', - }), - ).false; - - // expected User-Agent=git/* - expect( - validGitRequest('/info/refs?service=git-upload-pack', { - 'user-agent': 'Mozilla/5.0', - accept: '*/*', - }), - ).false; - }); - - it('validGitRequest should return false for unexpected content-type on certain URLs', function () { - ['application/json', 'text/html', '*/*'].map((accept) => { - expect( - validGitRequest('/git-upload-pack', { - 'user-agent': 'git/2.30.0', - accept: accept, - }), - ).false; - }); - }); -}); From 561f069d920b7bdf4e1f24a78de5177875a952e4 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 3 Jun 2025 15:41:39 +0100 Subject: [PATCH 28/76] feat(key on repo url): fix CLI and cypress tests after change to proxy urls --- cypress/e2e/repo.cy.js | 2 +- package-lock.json | 47 ++++++++++++++++++++++++++++-------------- test/testParsePush.js | 6 +++--- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/cypress/e2e/repo.cy.js b/cypress/e2e/repo.cy.js index 411397128..8b0ac1a21 100644 --- a/cypress/e2e/repo.cy.js +++ b/cypress/e2e/repo.cy.js @@ -10,7 +10,7 @@ describe('Repo', () => { describe('Code button for repo row', () => { it('Opens tooltip with correct content and can copy', () => { - const cloneURL = 'http://localhost:8000/finos/test-repo.git'; + const cloneURL = 'http://localhost:8000/github.com/finos/test-repo.git'; const tooltipQuery = 'div[role="tooltip"]'; cy diff --git a/package-lock.json b/package-lock.json index 5eb72a6f1..407ddf196 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2715,6 +2715,19 @@ "eslint-scope": "5.1.1" } }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2864,6 +2877,16 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/@paralleldrive/cuid2": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.2.2.tgz", + "integrity": "sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@noble/hashes": "^1.1.5" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -7576,13 +7599,14 @@ } }, "node_modules/formidable": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz", - "integrity": "sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==", + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.5.tgz", + "integrity": "sha512-Oz5Hwvwak/DCaXVVUtPn4oLMLLy1CdclLKO1LFgU7XzDpVMUU5UjlSLpGMocyQNNk8F6IJW9M/YdooSn2MRI+Q==", "dev": true, + "license": "MIT", "dependencies": { + "@paralleldrive/cuid2": "^2.2.2", "dezalgo": "^1.0.4", - "hexoid": "^1.0.0", "once": "^1.4.0", "qs": "^6.11.0" }, @@ -8137,15 +8161,6 @@ "he": "bin/he" } }, - "node_modules/hexoid": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", - "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/highlight.js": { "version": "11.9.0", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.9.0.tgz", @@ -14070,9 +14085,9 @@ } }, "node_modules/vite": { - "version": "4.5.13", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.13.tgz", - "integrity": "sha512-Hgp8IF/yZDzKsN1hQWOuQZbrKiaFsbQud+07jJ8h9m9PaHWkpvZ5u55Xw5yYjWRXwRQ4jwFlJvY7T7FUJG9MCA==", + "version": "4.5.14", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.14.tgz", + "integrity": "sha512-+v57oAaoYNnO3hIu5Z/tJRZjq5aHM2zDve9YZ8HngVHbhk66RStobhb1sqPMIPEleV6cNKYK4eGrAbE9Ulbl2g==", "dev": true, "license": "MIT", "dependencies": { diff --git a/test/testParsePush.js b/test/testParsePush.js index a4106d6ce..5ce80cbfa 100644 --- a/test/testParsePush.js +++ b/test/testParsePush.js @@ -97,7 +97,7 @@ const actionData = { type: 'push', method: 'POST', timestamp: 1746542004301, - repo: 'kriswest/git-proxy.git', + repo: 'https://github.com/kriswest/git-proxy.git', }; // truncated request (should through an error and not parse) @@ -419,7 +419,7 @@ const actionData2 = { type: 'push', method: 'POST', timestamp: 1746612610060, - repo: 'kriswest/git-proxy.git', + repo: 'https://github.com/kriswest/git-proxy.git', }; // push with a commit message not terminated by a newline @@ -474,7 +474,7 @@ const actionDataNoNewLine = { type: 'push', method: 'POST', timestamp: 1746697059624, - repo: 'kriswest/git-proxy.git', + repo: 'https://github.com/kriswest/git-proxy.git', }; describe('Check that pushes can be parsed', async () => { From 0f51320d9de5c70c0b576388887debe8629d970d Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 3 Jun 2025 18:10:12 +0100 Subject: [PATCH 29/76] chore(key on repo url): clean-up unused helper function and test --- src/proxy/routes/helper.ts | 36 ------------------------------------ test/testRouteFilter.test.js | 8 -------- 2 files changed, 44 deletions(-) diff --git a/src/proxy/routes/helper.ts b/src/proxy/routes/helper.ts index 20d46a8b4..2fba8616c 100644 --- a/src/proxy/routes/helper.ts +++ b/src/proxy/routes/helper.ts @@ -118,42 +118,6 @@ export const processGitURLForNameAndOrg = (gitUrl: string): GitNameBreakdown | n } }; -// /** Regex used to analyze legacy proxy paths to extract the repository name and -// * any path or organisation that proceeds it. */ -// const GIT_LEGACY_PATH_REGEX = /\/([^/]+)\/([^/]+)\.git/; - -// /** Type representing a breakdown of a legacy proxy path into repository name and organisation (project) -// * and a predicted GitHub URL. */ -// export type GitLegacyPathBreakdown = { project: string; repoName: string; url: string }; - -// /** Function that processes legacy proxy path string (which assumed GitHub as the repository host) to -// * extract the repository name, project (organisation) and to construct a predicted GitHub URL. -// * -// * E.g. Processing finos/git-proxy.git -// * would produce: -// * - project: finos -// * - repoName: git-proxy -// * - url: https://github.com/finos/git-proxy.git -// * -// * @param {string} requestPath The proxy path to process. -// * @return {GitLegacyPathBreakdown | null} A breakdown of the components of the URL. -// */ -// export const processLegacyProxyPathForNameAndOrg = ( -// requestPath: string, -// ): GitLegacyPathBreakdown | null => { -// const components = requestPath.match(GIT_LEGACY_PATH_REGEX); -// if (components && components.length >= 3) { -// return { -// project: components[1], -// repoName: components[2], -// url: `https://github.com/${components[1]}/${components[2]}.git`, -// }; -// } else { -// console.error(`Failed to parse git path: ${requestPath}`); -// return null; -// } -// }; - /** * Check whether an HTTP request has the expected properties of a * Git HTTP request. The URL is expected to be "sanitized", stripped of diff --git a/test/testRouteFilter.test.js b/test/testRouteFilter.test.js index 1102a7e41..f73f42565 100644 --- a/test/testRouteFilter.test.js +++ b/test/testRouteFilter.test.js @@ -79,14 +79,6 @@ describe('url helpers and filter functions used in the proxy', function () { }); }); - // it('processLegacyProxyPathForNameAndOrg should return breakdown of a legacy proxy path separating out the project (organisation), repository name and a predicted github URL', function () { - // expect(processLegacyProxyPathForNameAndOrg('/octocat/hello-world.git')).to.deep.eq({ - // project: 'octocat', - // repoName: 'hello-world', - // url: 'https://github.com/octocat/hello-world.git', - // }); - // }); - it('validGitRequest should return true for safe requests on expected URLs', function () { [ '/info/refs?service=git-upload-pack', From e50c918cf0312291c7749cbe17eb38a172c872d6 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 3 Jun 2025 18:57:49 +0100 Subject: [PATCH 30/76] fix: adding URL sanitisation to resolve codeQL flagged issues --- src/ui/views/OpenPushRequests/components/PushesTable.jsx | 2 +- src/ui/views/PushDetails/PushDetails.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ui/views/OpenPushRequests/components/PushesTable.jsx b/src/ui/views/OpenPushRequests/components/PushesTable.jsx index ef6c1ff95..6e57f1145 100644 --- a/src/ui/views/OpenPushRequests/components/PushesTable.jsx +++ b/src/ui/views/OpenPushRequests/components/PushesTable.jsx @@ -97,7 +97,7 @@ export default function PushesTable(props) { const repoBranch = row.branch.replace('refs/heads/', ''); const repoUrl = row.url; const repoWebUrl = repoUrl.replace('.git', ''); - const isGitHub = repoUrl.startsWith('https://github.com'); + const isGitHub = URL.parse(repoUrl).hostname === 'github.com'; return ( diff --git a/src/ui/views/PushDetails/PushDetails.jsx b/src/ui/views/PushDetails/PushDetails.jsx index 9fb38de12..b4f5d1df0 100644 --- a/src/ui/views/PushDetails/PushDetails.jsx +++ b/src/ui/views/PushDetails/PushDetails.jsx @@ -97,7 +97,7 @@ export default function Dashboard() { const repoBranch = data.branch.replace('refs/heads/', ''); const repoUrl = data.url; const repoWebUrl = repoUrl.replace('.git', ''); - const isGitHub = repoUrl.startsWith('https://github.com'); + const isGitHub = URL.parse(repoUrl).hostname === 'github.com'; const generateIcon = (title) => { switch (title) { From e318dc67dd1d844424f61bb4e2468b1148cd8f8a Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 5 Jun 2025 15:01:45 +0100 Subject: [PATCH 31/76] fix(key on repo url): adjust cypress test broken by changes to id's used --- cypress/e2e/repo.cy.js | 10 ++++------ src/ui/views/RepoList/Components/Repositories.jsx | 4 +++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cypress/e2e/repo.cy.js b/cypress/e2e/repo.cy.js index 8b0ac1a21..92d02734e 100644 --- a/cypress/e2e/repo.cy.js +++ b/cypress/e2e/repo.cy.js @@ -10,7 +10,7 @@ describe('Repo', () => { describe('Code button for repo row', () => { it('Opens tooltip with correct content and can copy', () => { - const cloneURL = 'http://localhost:8000/github.com/finos/test-repo.git'; + const cloneURLRegex = /http:\/\/localhost:8000\/(?:[^\/]+\/).+\.git/; const tooltipQuery = 'div[role="tooltip"]'; cy @@ -19,10 +19,8 @@ describe('Repo', () => { .should('not.exist'); cy - // find the entry for finos/test-repo - .get('a[href="/dashboard/repo/test-repo"]') - // take it's parent row - .closest('tr') + // find a table row for a repo (any will do) + .get('table#RepoListTable>tbody>tr') // find the nearby span containing Code we can click to open the tooltip .find('span') .contains('Code') @@ -35,7 +33,7 @@ describe('Repo', () => { .should('exist') .find('span') // check it contains the url we expect - .contains(cloneURL) + .contains(cloneURLRegex) .should('exist') .parent() // find the adjacent span that contains the svg diff --git a/src/ui/views/RepoList/Components/Repositories.jsx b/src/ui/views/RepoList/Components/Repositories.jsx index e77033c70..e9c3d2c9c 100644 --- a/src/ui/views/RepoList/Components/Repositories.jsx +++ b/src/ui/views/RepoList/Components/Repositories.jsx @@ -124,6 +124,7 @@ export default function Repositories(props) { itemsPerPage={itemsPerPage} onPageChange={handlePageChange} onFilterChange={handleFilterChange} // Pass handleFilterChange as prop + tableId='RepoListTable' /> ); } @@ -138,6 +139,7 @@ GetGridContainerLayOut.propTypes = { totalItems: PropTypes.number.isRequired, itemsPerPage: PropTypes.number.isRequired, onPageChange: PropTypes.func.isRequired, + tableId: PropTypes.string.isRequired, }; function GetGridContainerLayOut(props) { @@ -150,7 +152,7 @@ function GetGridContainerLayOut(props) { -
+
{props.data.map((row) => { if (row.url) { From 2ebdbf62bbca69dcac9ac24eee0de21edbd3eaf8 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 5 Jun 2025 17:48:25 +0100 Subject: [PATCH 32/76] fix: resolving issues with tests after merging main + output string when matches fail --- .../push-action/checkUserPushPermission.ts | 3 -- .../checkUserPushPermission.test.js | 51 ++++++++++++++----- test/processors/writePack.test.js | 12 ++--- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/proxy/processors/push-action/checkUserPushPermission.ts b/src/proxy/processors/push-action/checkUserPushPermission.ts index 1cd57538c..1cb288670 100644 --- a/src/proxy/processors/push-action/checkUserPushPermission.ts +++ b/src/proxy/processors/push-action/checkUserPushPermission.ts @@ -24,9 +24,6 @@ const exec = async (req: any, action: Action): Promise => { console.log('User not allowed to Push'); step.error = true; step.log(`User ${user} is not allowed to push on repo ${action.url}, ending`); - - console.log('setting error'); - step.setError( `Rejecting push as user ${action.user} ` + `is not allowed to push on repo ` + diff --git a/test/processors/checkUserPushPermission.test.js b/test/processors/checkUserPushPermission.test.js index b140d383b..84c543563 100644 --- a/test/processors/checkUserPushPermission.test.js +++ b/test/processors/checkUserPushPermission.test.js @@ -18,12 +18,15 @@ describe('checkUserPushPermission', () => { getUsersStub = sinon.stub(); isUserPushAllowedStub = sinon.stub(); - const checkUserPushPermission = proxyquire('../../src/proxy/processors/push-action/checkUserPushPermission', { - '../../../db': { - getUsers: getUsersStub, - isUserPushAllowed: isUserPushAllowedStub - } - }); + const checkUserPushPermission = proxyquire( + '../../src/proxy/processors/push-action/checkUserPushPermission', + { + '../../../db': { + getUsers: getUsersStub, + isUserPushAllowed: isUserPushAllowedStub, + }, + }, + ); exec = checkUserPushPermission.exec; }); @@ -44,7 +47,7 @@ describe('checkUserPushPermission', () => { 'push', 'POST', 1234567890, - 'test/repo.git' + 'https://github.com/finos/git-proxy.git', ); action.user = 'git-user'; stepSpy = sinon.spy(Step.prototype, 'log'); @@ -58,8 +61,14 @@ describe('checkUserPushPermission', () => { expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.false; - expect(stepSpy.calledWith('User db-user is allowed to push on repo test/repo.git')).to.be.true; - expect(logStub.calledWith('User db-user permission on Repo repo : true')).to.be.true; + expect( + stepSpy.calledWith( + 'User db-user is allowed to push on repo https://github.com/finos/git-proxy.git', + ), + ).to.be.true; + expect(logStub.lastCall.args[0]).to.equal( + 'User db-user permission on Repo https://github.com/finos/git-proxy.git : true', + ); }); it('should reject push when user has no permission', async () => { @@ -70,9 +79,13 @@ describe('checkUserPushPermission', () => { expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(stepSpy.calledWith('User db-user is not allowed to push on repo test/repo.git, ending')).to.be.true; + expect( + stepSpy.calledWith( + 'User db-user is not allowed to push on repo https://github.com/finos/git-proxy.git, ending', + ), + ).to.be.true; expect(result.steps[0].errorMessage).to.include('Rejecting push as user git-user'); - expect(logStub.calledWith('User not allowed to Push')).to.be.true; + expect(logStub.lastCall.args[0]).to.equal('User not allowed to Push'); }); it('should reject push when no user found for git account', async () => { @@ -82,21 +95,31 @@ describe('checkUserPushPermission', () => { expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(stepSpy.calledWith('User git-user is not allowed to push on repo test/repo.git, ending')).to.be.true; + expect( + stepSpy.calledWith( + 'User git-user is not allowed to push on repo https://github.com/finos/git-proxy.git, ending', + ), + ).to.be.true; expect(result.steps[0].errorMessage).to.include('Rejecting push as user git-user'); }); it('should handle multiple users for git account by rejecting push', async () => { getUsersStub.resolves([ { username: 'user1', gitAccount: 'git-user' }, - { username: 'user2', gitAccount: 'git-user' } + { username: 'user2', gitAccount: 'git-user' }, ]); const result = await exec(req, action); expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(logStub.calledWith('Users for this git account: [{"username":"user1","gitAccount":"git-user"},{"username":"user2","gitAccount":"git-user"}]')).to.be.true; + expect(logStub.getCall(-3).args[0]).to.equal( + 'Users for this git account: [{"username":"user1","gitAccount":"git-user"},{"username":"user2","gitAccount":"git-user"}]', + ); + expect(logStub.getCall(-2).args[0]).to.equal( + 'User git-user permission on Repo https://github.com/finos/git-proxy.git : false', + ); + expect(logStub.lastCall.args[0]).to.equal('User not allowed to Push'); }); }); }); diff --git a/test/processors/writePack.test.js b/test/processors/writePack.test.js index a7580caa6..8331a77d8 100644 --- a/test/processors/writePack.test.js +++ b/test/processors/writePack.test.js @@ -17,7 +17,7 @@ describe('writePack', () => { spawnSyncStub = sinon.stub().returns({ stdout: 'git receive-pack output', stderr: '', - status: 0 + status: 0, }); stepLogSpy = sinon.spy(Step.prototype, 'log'); @@ -25,7 +25,7 @@ describe('writePack', () => { stepSetErrorSpy = sinon.spy(Step.prototype, 'setError'); const writePack = proxyquire('../../src/proxy/processors/push-action/writePack', { - 'child_process': { spawnSync: spawnSyncStub } + child_process: { spawnSync: spawnSyncStub }, }); exec = writePack.exec; @@ -41,14 +41,14 @@ describe('writePack', () => { beforeEach(() => { req = { - body: 'pack data' + body: 'pack data', }; action = new Action( '1234567890', 'push', 'POST', 1234567890, - 'test/repo' + 'https://github.com/finos/git-proxy.git', ); action.proxyGitPath = '/path/to/repo'; }); @@ -58,11 +58,11 @@ describe('writePack', () => { expect(spawnSyncStub.calledOnce).to.be.true; expect(spawnSyncStub.firstCall.args[0]).to.equal('git'); - expect(spawnSyncStub.firstCall.args[1]).to.deep.equal(['receive-pack', 'repo']); + expect(spawnSyncStub.firstCall.args[1]).to.deep.equal(['receive-pack', 'git-proxy.git']); expect(spawnSyncStub.firstCall.args[2]).to.deep.equal({ cwd: '/path/to/repo', input: 'pack data', - encoding: 'utf-8' + encoding: 'utf-8', }); expect(stepLogSpy.calledWith('executing git receive-pack repo')).to.be.true; From f7b1911c3070d3d6fe82f47840d2010bbe03fe7c Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 5 Jun 2025 18:27:24 +0100 Subject: [PATCH 33/76] fix(key on repo url): further tweaks to failing tests --- src/proxy/actions/Step.ts | 2 -- test/processors/writePack.test.js | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/proxy/actions/Step.ts b/src/proxy/actions/Step.ts index 6eb114e9c..504b5390c 100644 --- a/src/proxy/actions/Step.ts +++ b/src/proxy/actions/Step.ts @@ -35,12 +35,10 @@ class Step { } setContent(content: any): void { - this.log('setting content'); this.content = content; } setAsyncBlock(message: string): void { - this.log('setting blocked'); this.blocked = true; this.blockedMessage = message; } diff --git a/test/processors/writePack.test.js b/test/processors/writePack.test.js index 8331a77d8..1a775bfd8 100644 --- a/test/processors/writePack.test.js +++ b/test/processors/writePack.test.js @@ -65,10 +65,10 @@ describe('writePack', () => { encoding: 'utf-8', }); - expect(stepLogSpy.calledWith('executing git receive-pack repo')).to.be.true; - expect(stepLogSpy.calledWith('git receive-pack output')).to.be.true; + expect(stepLogSpy.getCall(-2).args[0]).to.equal('executing git receive-pack git-proxy.git'); + expect(stepLogSpy.getCall(-1).args[0]).to.equal('git receive-pack output'); - expect(stepSetContentSpy.calledWith('git receive-pack output')).to.be.true; + expect(stepSetContentSpy.getCall(-1).args[0]).to.equal('git receive-pack output'); expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.false; From 46bdc9c22b19f27c8216edaa6860f0a8de135474 Mon Sep 17 00:00:00 2001 From: Kris West Date: Wed, 11 Jun 2025 16:04:11 +0100 Subject: [PATCH 34/76] fix: restart the proxy when handling a new origin --- src/service/routes/repo.js | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/service/routes/repo.js b/src/service/routes/repo.js index 6f77594f0..91868cafc 100644 --- a/src/service/routes/repo.js +++ b/src/service/routes/repo.js @@ -2,6 +2,7 @@ const express = require('express'); const router = new express.Router(); const db = require('../../db'); const { getProxyURL } = require('../urls'); +const { getAllProxiedHosts } = require('../../proxy/routes/helper'); router.get('/', async (req, res) => { const proxyURL = getProxyURL(req); @@ -138,8 +139,37 @@ router.post('/', async (req, res) => { }); } else { try { + // figure out if this represent a new domain to proxy + let newOrigin = true; + + const existingHosts = await getAllProxiedHosts(); + existingHosts.forEach((h) => { + if (req.body.url.startsWith(h)) { + newOrigin = false; + } + }); + + console.log( + `API request to proxy repository ${req.body.url} is for a new origin: ${newOrigin},\n\texisting origin list was: ${JSON.stringify(existingHosts)}`, + ); + + // create the repository await db.createRepo(req.body); res.send({ message: 'created' }); + + // restart the proxy if we're proxying a new domain + if (newOrigin) { + console.log('Restarting the proxy to handle an additional origin'); + + // 1. Get proxy module dynamically to avoid circular dependency + const proxy = require('../proxy'); + + // 2. Stop existing services + await proxy.stop(); + + // 3. Restart the proxy, which should set up for the new domain + await proxy.start(); + } } catch { res.send('Failed to create repository'); } From c54e632ba9955155122af900b51c21193b1afc14 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 12 Jun 2025 09:07:31 +0100 Subject: [PATCH 35/76] test: correcting git URLs in tests and increasing coverage on parseAction and db pojos --- test/processors/checkCommitMessages.test.js | 70 ++++++------- test/processors/checkIfWaitingAuth.test.js | 23 ++-- test/processors/clearBareClone.test.js | 6 +- test/processors/getDiff.test.js | 66 +++--------- test/processors/gitLeaks.test.js | 110 +++++++++++--------- test/scanDiff.test.js | 41 +++++--- test/testDb.test.js | 88 ++++++++++++++++ test/testParseAction.test.js | 38 +++++++ 8 files changed, 278 insertions(+), 164 deletions(-) create mode 100644 test/testParseAction.test.js diff --git a/test/processors/checkCommitMessages.test.js b/test/processors/checkCommitMessages.test.js index 75156e0ae..5e8e47fdf 100644 --- a/test/processors/checkCommitMessages.test.js +++ b/test/processors/checkCommitMessages.test.js @@ -19,16 +19,19 @@ describe('checkCommitMessages', () => { message: { block: { literals: ['secret', 'password'], - patterns: ['\\b\\d{4}-\\d{4}-\\d{4}-\\d{4}\\b'] // Credit card pattern - } - } + patterns: ['\\b\\d{4}-\\d{4}-\\d{4}-\\d{4}\\b'], // Credit card pattern + }, + }, }; getCommitConfigStub = sinon.stub().returns(commitConfig); - const checkCommitMessages = proxyquire('../../src/proxy/processors/push-action/checkCommitMessages', { - '../../../config': { getCommitConfig: getCommitConfigStub } - }); + const checkCommitMessages = proxyquire( + '../../src/proxy/processors/push-action/checkCommitMessages', + { + '../../../config': { getCommitConfig: getCommitConfigStub }, + }, + ); exec = checkCommitMessages.exec; }); @@ -44,16 +47,10 @@ describe('checkCommitMessages', () => { beforeEach(() => { req = {}; - action = new Action( - '1234567890', - 'push', - 'POST', - 1234567890, - 'test/repo' - ); + action = new Action('1234567890', 'push', 'POST', 1234567890, 'test/repo.git'); action.commitData = [ { message: 'Fix bug', author: 'test@example.com' }, - { message: 'Update docs', author: 'test@example.com' } + { message: 'Update docs', author: 'test@example.com' }, ]; stepSpy = sinon.spy(Step.prototype, 'log'); }); @@ -63,7 +60,8 @@ describe('checkCommitMessages', () => { expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.false; - expect(logStub.calledWith('The following commit messages are legal: Fix bug,Update docs')).to.be.true; + expect(logStub.calledWith('The following commit messages are legal: Fix bug,Update docs')).to + .be.true; }); it('should block commit with illegal messages', async () => { @@ -73,32 +71,32 @@ describe('checkCommitMessages', () => { expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(stepSpy.calledWith( - 'The following commit messages are illegal: secret password here' - )).to.be.true; + expect(stepSpy.calledWith('The following commit messages are illegal: secret password here')) + .to.be.true; expect(result.steps[0].errorMessage).to.include('Your push has been blocked'); - expect(logStub.calledWith('The following commit messages are illegal: secret password here')).to.be.true; + expect(logStub.calledWith('The following commit messages are illegal: secret password here')) + .to.be.true; }); it('should handle duplicate messages only once', async () => { action.commitData = [ { message: 'secret', author: 'test@example.com' }, { message: 'secret', author: 'test@example.com' }, - { message: 'password', author: 'test@example.com' } + { message: 'password', author: 'test@example.com' }, ]; const result = await exec(req, action); expect(result.steps[0].error).to.be.true; - expect(stepSpy.calledWith( - 'The following commit messages are illegal: secret,password' - )).to.be.true; - expect(logStub.calledWith('The following commit messages are illegal: secret,password')).to.be.true; + expect(stepSpy.calledWith('The following commit messages are illegal: secret,password')).to.be + .true; + expect(logStub.calledWith('The following commit messages are illegal: secret,password')).to.be + .true; }); it('should not error when commit data is empty', async () => { // Empty commit data is a valid scenario that happens when making a branch from an unapproved commit - // This is remedied in the getMissingData.exec action + // This is remedied in the getMissingData.exec action action.commitData = []; const result = await exec(req, action); @@ -110,7 +108,7 @@ describe('checkCommitMessages', () => { it('should handle commit data with null values', async () => { action.commitData = [ { message: null, author: 'test@example.com' }, - { message: undefined, author: 'test@example.com' } + { message: undefined, author: 'test@example.com' }, ]; const result = await exec(req, action); @@ -122,33 +120,33 @@ describe('checkCommitMessages', () => { it('should handle commit messages of incorrect type', async () => { action.commitData = [ { message: 123, author: 'test@example.com' }, - { message: {}, author: 'test@example.com' } + { message: {}, author: 'test@example.com' }, ]; const result = await exec(req, action); expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(stepSpy.calledWith( - 'The following commit messages are illegal: 123,[object Object]' - )).to.be.true; - expect(logStub.calledWith('The following commit messages are illegal: 123,[object Object]')).to.be.true; + expect(stepSpy.calledWith('The following commit messages are illegal: 123,[object Object]')) + .to.be.true; + expect(logStub.calledWith('The following commit messages are illegal: 123,[object Object]')) + .to.be.true; }); it('should handle a mix of valid and invalid messages', async () => { action.commitData = [ { message: 'Fix bug', author: 'test@example.com' }, - { message: 'secret password here', author: 'test@example.com' } + { message: 'secret password here', author: 'test@example.com' }, ]; const result = await exec(req, action); expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(stepSpy.calledWith( - 'The following commit messages are illegal: secret password here' - )).to.be.true; - expect(logStub.calledWith('The following commit messages are illegal: secret password here')).to.be.true; + expect(stepSpy.calledWith('The following commit messages are illegal: secret password here')) + .to.be.true; + expect(logStub.calledWith('The following commit messages are illegal: secret password here')) + .to.be.true; }); }); }); diff --git a/test/processors/checkIfWaitingAuth.test.js b/test/processors/checkIfWaitingAuth.test.js index f9a66a3a6..0ee9988bb 100644 --- a/test/processors/checkIfWaitingAuth.test.js +++ b/test/processors/checkIfWaitingAuth.test.js @@ -13,9 +13,12 @@ describe('checkIfWaitingAuth', () => { beforeEach(() => { getPushStub = sinon.stub(); - const checkIfWaitingAuth = proxyquire('../../src/proxy/processors/push-action/checkIfWaitingAuth', { - '../../../db': { getPush: getPushStub } - }); + const checkIfWaitingAuth = proxyquire( + '../../src/proxy/processors/push-action/checkIfWaitingAuth', + { + '../../../db': { getPush: getPushStub }, + }, + ); exec = checkIfWaitingAuth.exec; }); @@ -30,13 +33,7 @@ describe('checkIfWaitingAuth', () => { beforeEach(() => { req = {}; - action = new Action( - '1234567890', - 'push', - 'POST', - 1234567890, - 'test/repo' - ); + action = new Action('1234567890', 'push', 'POST', 1234567890, 'test/repo.git'); }); it('should set allowPush when action exists and is authorized', async () => { @@ -45,7 +42,7 @@ describe('checkIfWaitingAuth', () => { 'push', 'POST', 1234567890, - 'test/repo' + 'test/repo.git', ); authorizedAction.authorised = true; getPushStub.resolves(authorizedAction); @@ -64,7 +61,7 @@ describe('checkIfWaitingAuth', () => { 'push', 'POST', 1234567890, - 'test/repo' + 'test/repo.git', ); unauthorizedAction.authorised = false; getPushStub.resolves(unauthorizedAction); @@ -93,7 +90,7 @@ describe('checkIfWaitingAuth', () => { 'push', 'POST', 1234567890, - 'test/repo' + 'test/repo.git', ); authorizedAction.authorised = true; getPushStub.resolves(authorizedAction); diff --git a/test/processors/clearBareClone.test.js b/test/processors/clearBareClone.test.js index 1ebcf85c4..3f869ff98 100644 --- a/test/processors/clearBareClone.test.js +++ b/test/processors/clearBareClone.test.js @@ -10,8 +10,8 @@ const timestamp = Date.now(); describe('clear bare and local clones', async () => { it('pull remote generates a local .remote folder', async () => { - const action = new Action('123', 'type', 'get', timestamp, 'finos/git-proxy'); - action.url = 'https://github.com/finos/git-proxy'; + const action = new Action('123', 'type', 'get', timestamp, 'finos/git-proxy.git'); + action.url = 'https://github.com/finos/git-proxy.git'; const authorization = `Basic ${Buffer.from('JamieSlome:test').toString('base64')}`; @@ -28,7 +28,7 @@ describe('clear bare and local clones', async () => { }).timeout(20000); it('clear bare clone function purges .remote folder and specific clone folder', async () => { - const action = new Action('123', 'type', 'get', timestamp, 'finos/git-proxy'); + const action = new Action('123', 'type', 'get', timestamp, 'finos/git-proxy.git'); await clearBareClone(null, action); expect(fs.existsSync(`./.remote`)).to.throw; expect(fs.existsSync(`./.remote/${timestamp}`)).to.throw; diff --git a/test/processors/getDiff.test.js b/test/processors/getDiff.test.js index 5acbc83e2..91178f69b 100644 --- a/test/processors/getDiff.test.js +++ b/test/processors/getDiff.test.js @@ -10,13 +10,13 @@ const expect = chai.expect; describe('getDiff', () => { let tempDir; let git; - + before(async () => { // Create a temp repo to avoid mocking simple-git tempDir = path.join(__dirname, 'temp-test-repo'); await fs.mkdir(tempDir, { recursive: true }); git = simpleGit(tempDir); - + await git.init(); await git.addConfig('user.name', 'test'); await git.addConfig('user.email', 'test@test.com'); @@ -25,53 +25,37 @@ describe('getDiff', () => { await git.add('.'); await git.commit('initial commit'); }); - + after(async () => { await fs.rmdir(tempDir, { recursive: true }); }); - + it('should get diff between commits', async () => { await fs.writeFile(path.join(tempDir, 'test.txt'), 'modified content'); await git.add('.'); await git.commit('second commit'); - - const action = new Action( - '1234567890', - 'push', - 'POST', - 1234567890, - 'test/repo' - ); + + const action = new Action('1234567890', 'push', 'POST', 1234567890, 'test/repo.git'); action.proxyGitPath = __dirname; // Temp dir parent path action.repoName = 'temp-test-repo'; action.commitFrom = 'HEAD~1'; action.commitTo = 'HEAD'; - action.commitData = [ - { parent: '0000000000000000000000000000000000000000' } - ]; - + action.commitData = [{ parent: '0000000000000000000000000000000000000000' }]; + const result = await exec({}, action); - + expect(result.steps[0].error).to.be.false; expect(result.steps[0].content).to.include('modified content'); expect(result.steps[0].content).to.include('initial content'); }); it('should get diff between commits with no changes', async () => { - const action = new Action( - '1234567890', - 'push', - 'POST', - 1234567890, - 'test/repo' - ); + const action = new Action('1234567890', 'push', 'POST', 1234567890, 'test/repo.get'); action.proxyGitPath = __dirname; // Temp dir parent path action.repoName = 'temp-test-repo'; action.commitFrom = 'HEAD~1'; action.commitTo = 'HEAD'; - action.commitData = [ - { parent: '0000000000000000000000000000000000000000' } - ]; + action.commitData = [{ parent: '0000000000000000000000000000000000000000' }]; const result = await exec({}, action); @@ -80,13 +64,7 @@ describe('getDiff', () => { }); it('should throw an error if no commit data is provided', async () => { - const action = new Action( - '1234567890', - 'push', - 'POST', - 1234567890, - 'test/repo' - ); + const action = new Action('1234567890', 'push', 'POST', 1234567890, 'test/repo.git'); action.proxyGitPath = __dirname; // Temp dir parent path action.repoName = 'temp-test-repo'; action.commitFrom = 'HEAD~1'; @@ -99,13 +77,7 @@ describe('getDiff', () => { }); it('should throw an error if no commit data is provided', async () => { - const action = new Action( - '1234567890', - 'push', - 'POST', - 1234567890, - 'test/repo' - ); + const action = new Action('1234567890', 'push', 'POST', 1234567890, 'test/repo.git'); action.proxyGitPath = __dirname; // Temp dir parent path action.repoName = 'temp-test-repo'; action.commitFrom = 'HEAD~1'; @@ -126,21 +98,13 @@ describe('getDiff', () => { const parentCommit = log.all[1].hash; const headCommit = log.all[0].hash; - const action = new Action( - '1234567890', - 'push', - 'POST', - 1234567890, - 'test/repo' - ); + const action = new Action('1234567890', 'push', 'POST', 1234567890, 'test/repo.git'); action.proxyGitPath = path.dirname(tempDir); action.repoName = path.basename(tempDir); action.commitFrom = '0000000000000000000000000000000000000000'; action.commitTo = headCommit; - action.commitData = [ - { parent: parentCommit } - ]; + action.commitData = [{ parent: parentCommit }]; const result = await exec({}, action); diff --git a/test/processors/gitLeaks.test.js b/test/processors/gitLeaks.test.js index eeed7f8e2..eca181c61 100644 --- a/test/processors/gitLeaks.test.js +++ b/test/processors/gitLeaks.test.js @@ -22,9 +22,9 @@ describe('gitleaks', () => { fs: { stat: sinon.stub(), access: sinon.stub(), - constants: { R_OK: 0 } + constants: { R_OK: 0 }, }, - spawn: sinon.stub() + spawn: sinon.stub(), }; logStub = sinon.stub(console, 'log'); @@ -33,19 +33,13 @@ describe('gitleaks', () => { const gitleaksModule = proxyquire('../../src/proxy/processors/push-action/gitleaks', { '../../../config': { getAPIs: stubs.getAPIs }, 'node:fs/promises': stubs.fs, - 'node:child_process': { spawn: stubs.spawn } + 'node:child_process': { spawn: stubs.spawn }, }); exec = gitleaksModule.exec; req = {}; - action = new Action( - '1234567890', - 'push', - 'POST', - 1234567890, - 'test/repo' - ); + action = new Action('1234567890', 'push', 'POST', 1234567890, 'test/repo.git'); action.proxyGitPath = '/tmp'; action.repoName = 'test-repo'; action.commitFrom = 'abc123'; @@ -66,8 +60,10 @@ describe('gitleaks', () => { expect(result.error).to.be.true; expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(stepSpy.calledWith('failed setup gitleaks, please contact an administrator\n')).to.be.true; - expect(errorStub.calledWith('failed to get gitleaks config, please fix the error:')).to.be.true; + expect(stepSpy.calledWith('failed setup gitleaks, please contact an administrator\n')).to.be + .true; + expect(errorStub.calledWith('failed to get gitleaks config, please fix the error:')).to.be + .true; }); it('should skip scanning when plugin is disabled', async () => { @@ -87,31 +83,33 @@ describe('gitleaks', () => { const gitRootCommitMock = { exitCode: 0, stdout: 'rootcommit123\n', - stderr: '' + stderr: '', }; const gitleaksMock = { exitCode: 0, stdout: '', - stderr: 'No leaks found' + stderr: 'No leaks found', }; stubs.spawn - .onFirstCall().returns({ + .onFirstCall() + .returns({ on: (event, cb) => { if (event === 'close') cb(gitRootCommitMock.exitCode); return { stdout: { on: () => {} }, stderr: { on: () => {} } }; }, stdout: { on: (_, cb) => cb(gitRootCommitMock.stdout) }, - stderr: { on: (_, cb) => cb(gitRootCommitMock.stderr) } + stderr: { on: (_, cb) => cb(gitRootCommitMock.stderr) }, }) - .onSecondCall().returns({ + .onSecondCall() + .returns({ on: (event, cb) => { if (event === 'close') cb(gitleaksMock.exitCode); return { stdout: { on: () => {} }, stderr: { on: () => {} } }; }, stdout: { on: (_, cb) => cb(gitleaksMock.stdout) }, - stderr: { on: (_, cb) => cb(gitleaksMock.stderr) } + stderr: { on: (_, cb) => cb(gitleaksMock.stderr) }, }); const result = await exec(req, action); @@ -129,31 +127,33 @@ describe('gitleaks', () => { const gitRootCommitMock = { exitCode: 0, stdout: 'rootcommit123\n', - stderr: '' + stderr: '', }; const gitleaksMock = { exitCode: 99, stdout: 'Found secret in file.txt\n', - stderr: 'Warning: potential leak' + stderr: 'Warning: potential leak', }; stubs.spawn - .onFirstCall().returns({ + .onFirstCall() + .returns({ on: (event, cb) => { if (event === 'close') cb(gitRootCommitMock.exitCode); return { stdout: { on: () => {} }, stderr: { on: () => {} } }; }, stdout: { on: (_, cb) => cb(gitRootCommitMock.stdout) }, - stderr: { on: (_, cb) => cb(gitRootCommitMock.stderr) } + stderr: { on: (_, cb) => cb(gitRootCommitMock.stderr) }, }) - .onSecondCall().returns({ + .onSecondCall() + .returns({ on: (event, cb) => { if (event === 'close') cb(gitleaksMock.exitCode); return { stdout: { on: () => {} }, stderr: { on: () => {} } }; }, stdout: { on: (_, cb) => cb(gitleaksMock.stdout) }, - stderr: { on: (_, cb) => cb(gitleaksMock.stderr) } + stderr: { on: (_, cb) => cb(gitleaksMock.stderr) }, }); const result = await exec(req, action); @@ -170,31 +170,33 @@ describe('gitleaks', () => { const gitRootCommitMock = { exitCode: 0, stdout: 'rootcommit123\n', - stderr: '' + stderr: '', }; const gitleaksMock = { exitCode: 1, stdout: '', - stderr: 'Command failed' + stderr: 'Command failed', }; stubs.spawn - .onFirstCall().returns({ + .onFirstCall() + .returns({ on: (event, cb) => { if (event === 'close') cb(gitRootCommitMock.exitCode); return { stdout: { on: () => {} }, stderr: { on: () => {} } }; }, stdout: { on: (_, cb) => cb(gitRootCommitMock.stdout) }, - stderr: { on: (_, cb) => cb(gitRootCommitMock.stderr) } + stderr: { on: (_, cb) => cb(gitRootCommitMock.stderr) }, }) - .onSecondCall().returns({ + .onSecondCall() + .returns({ on: (event, cb) => { if (event === 'close') cb(gitleaksMock.exitCode); return { stdout: { on: () => {} }, stderr: { on: () => {} } }; }, stdout: { on: (_, cb) => cb(gitleaksMock.stdout) }, - stderr: { on: (_, cb) => cb(gitleaksMock.stderr) } + stderr: { on: (_, cb) => cb(gitleaksMock.stderr) }, }); const result = await exec(req, action); @@ -202,7 +204,8 @@ describe('gitleaks', () => { expect(result.error).to.be.true; expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(stepSpy.calledWith('failed to run gitleaks, please contact an administrator\n')).to.be.true; + expect(stepSpy.calledWith('failed to run gitleaks, please contact an administrator\n')).to.be + .true; }); it('should handle gitleaks spawn failure', async () => { @@ -214,7 +217,8 @@ describe('gitleaks', () => { expect(result.error).to.be.true; expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(stepSpy.calledWith('failed to spawn gitleaks, please contact an administrator\n')).to.be.true; + expect(stepSpy.calledWith('failed to spawn gitleaks, please contact an administrator\n')).to + .be.true; }); it('should handle empty gitleaks entry in proxy.config.json', async () => { @@ -233,7 +237,7 @@ describe('gitleaks', () => { return { stdout: { on: () => {} }, stderr: { on: () => {} } }; }, stdout: { on: (_, cb) => cb('') }, - stderr: { on: (_, cb) => cb('') } + stderr: { on: (_, cb) => cb('') }, }); const result = await exec(req, action); @@ -244,11 +248,11 @@ describe('gitleaks', () => { }); it('should handle custom config path', async () => { - stubs.getAPIs.returns({ - gitleaks: { + stubs.getAPIs.returns({ + gitleaks: { enabled: true, - configPath: `../fixtures/gitleaks-config.toml` - } + configPath: `../fixtures/gitleaks-config.toml`, + }, }); stubs.fs.stat.resolves({ isFile: () => true }); @@ -257,46 +261,50 @@ describe('gitleaks', () => { const gitRootCommitMock = { exitCode: 0, stdout: 'rootcommit123\n', - stderr: '' + stderr: '', }; const gitleaksMock = { exitCode: 0, stdout: '', - stderr: 'No leaks found' + stderr: 'No leaks found', }; stubs.spawn - .onFirstCall().returns({ + .onFirstCall() + .returns({ on: (event, cb) => { if (event === 'close') cb(gitRootCommitMock.exitCode); return { stdout: { on: () => {} }, stderr: { on: () => {} } }; }, stdout: { on: (_, cb) => cb(gitRootCommitMock.stdout) }, - stderr: { on: (_, cb) => cb(gitRootCommitMock.stderr) } + stderr: { on: (_, cb) => cb(gitRootCommitMock.stderr) }, }) - .onSecondCall().returns({ + .onSecondCall() + .returns({ on: (event, cb) => { if (event === 'close') cb(gitleaksMock.exitCode); return { stdout: { on: () => {} }, stderr: { on: () => {} } }; }, stdout: { on: (_, cb) => cb(gitleaksMock.stdout) }, - stderr: { on: (_, cb) => cb(gitleaksMock.stderr) } + stderr: { on: (_, cb) => cb(gitleaksMock.stderr) }, }); const result = await exec(req, action); expect(result.error).to.be.false; expect(result.steps[0].error).to.be.false; - expect(stubs.spawn.secondCall.args[1]).to.include('--config=../fixtures/gitleaks-config.toml'); + expect(stubs.spawn.secondCall.args[1]).to.include( + '--config=../fixtures/gitleaks-config.toml', + ); }); it('should handle invalid custom config path', async () => { - stubs.getAPIs.returns({ - gitleaks: { + stubs.getAPIs.returns({ + gitleaks: { enabled: true, - configPath: '/invalid/path.toml' - } + configPath: '/invalid/path.toml', + }, }); stubs.fs.stat.rejects(new Error('File not found')); @@ -306,7 +314,11 @@ describe('gitleaks', () => { expect(result.error).to.be.true; expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(errorStub.calledWith('could not read file at the config path provided, will not be fed to gitleaks')).to.be.true; + expect( + errorStub.calledWith( + 'could not read file at the config path provided, will not be fed to gitleaks', + ), + ).to.be.true; }); }); }); diff --git a/test/scanDiff.test.js b/test/scanDiff.test.js index 41e5b7d8b..d06baeba9 100644 --- a/test/scanDiff.test.js +++ b/test/scanDiff.test.js @@ -85,13 +85,16 @@ describe('Scan commit diff...', async () => { }); it('A diff including an AWS (Amazon Web Services) Access Key ID blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', content: generateDiff('AKIAIOSFODNN7EXAMPLE'), }, ]; + action.setCommit('38cdc3e', '8a9c321'); + action.setBranch('b'); + action.setMessage('Message'); const { error, errorMessage } = await processor.exec(null, action); expect(error).to.be.true; @@ -100,13 +103,14 @@ describe('Scan commit diff...', async () => { // Formatting test it('A diff including multiple AWS (Amazon Web Services) Access Keys ID blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', content: generateMultiLineDiff(), }, ]; + action.setCommit('8b97e49', 'de18d43'); const { error, errorMessage } = await processor.exec(null, action); @@ -119,13 +123,14 @@ describe('Scan commit diff...', async () => { // Formatting test it('A diff including multiple AWS Access Keys ID and Literal blocks the proxy with appropriate message...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', content: generateMultiLineDiffWithLiteral(), }, ]; + action.setCommit('8b97e49', 'de18d43'); const { error, errorMessage } = await processor.exec(null, action); @@ -140,13 +145,15 @@ describe('Scan commit diff...', async () => { }); it('A diff including a Google Cloud Platform API Key blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', content: generateDiff('AIza0aB7Z4Rfs23MnPqars81yzu19KbH72zaFda'), }, ]; + action.commitFrom = '38cdc3e'; + action.commitTo = '8a9c321'; const { error, errorMessage } = await processor.exec(null, action); @@ -155,7 +162,7 @@ describe('Scan commit diff...', async () => { }); it('A diff including a GitHub Personal Access Token blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', @@ -170,7 +177,7 @@ describe('Scan commit diff...', async () => { }); it('A diff including a GitHub Fine Grained Personal Access Token blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', @@ -179,6 +186,8 @@ describe('Scan commit diff...', async () => { ), }, ]; + action.commitFrom = '38cdc3e'; + action.commitTo = '8a9c321'; const { error, errorMessage } = await processor.exec(null, action); @@ -187,13 +196,15 @@ describe('Scan commit diff...', async () => { }); it('A diff including a GitHub Actions Token blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', content: generateDiff(`ghs_${crypto.randomBytes(20).toString('hex')}`), }, ]; + action.commitFrom = '38cdc3e'; + action.commitTo = '8a9c321'; const { error, errorMessage } = await processor.exec(null, action); @@ -202,7 +213,7 @@ describe('Scan commit diff...', async () => { }); it('A diff including a JSON Web Token (JWT) blocks the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', @@ -211,6 +222,8 @@ describe('Scan commit diff...', async () => { ), }, ]; + action.commitFrom = '38cdc3e'; + action.commitTo = '8a9c321'; const { error, errorMessage } = await processor.exec(null, action); @@ -220,13 +233,15 @@ describe('Scan commit diff...', async () => { it('A diff including a blocked literal blocks the proxy...', async () => { for (const [literal] of blockedLiterals.entries()) { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', content: generateDiff(literal), }, ]; + action.commitFrom = '38cdc3e'; + action.commitTo = '8a9c321'; const { error, errorMessage } = await processor.exec(null, action); @@ -235,7 +250,7 @@ describe('Scan commit diff...', async () => { } }); it('When no diff is present, the proxy is blocked...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', @@ -250,7 +265,7 @@ describe('Scan commit diff...', async () => { }); it('When diff is not a string, the proxy is blocked...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', @@ -265,13 +280,15 @@ describe('Scan commit diff...', async () => { }); it('A diff with no secrets or sensitive information does not block the proxy...', async () => { - const action = new Action('1', 'type', 'method', 1, 'url'); + const action = new Action('1', 'type', 'method', 1, 'test/repo.git'); action.steps = [ { stepName: 'diff', content: generateDiff(''), }, ]; + action.commitFrom = '38cdc3e'; + action.commitTo = '8a9c321'; const { error } = await processor.exec(null, action); expect(error).to.be.false; diff --git a/test/testDb.test.js b/test/testDb.test.js index d290e496f..6772d51a2 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -1,6 +1,9 @@ // This test needs to run first const chai = require('chai'); const db = require('../src/db'); +const { Repo, User } = require('../src/db/types'); +const { Action } = require('../src/proxy/actions/Action'); +const { Step } = require('../src/proxy/actions/Step'); const { expect } = chai; @@ -86,6 +89,91 @@ const cleanResponseData = (example, responses) => { describe('Database clients', async () => { before(async function () {}); + it('should be able to construct a repo instance', async function () { + const repo = new Repo('project', 'name', 'https://github.com/finos.git-proxy.git', {}, 'id'); + expect(repo._id).to.equal('id'); + expect(repo.project).to.equal('project'); + expect(repo.name).to.equal('name'); + expect(repo.url).to.equal('https://github.com/finos.git-proxy.git'); + expect(repo.users).to.deep.equals({ canPush: [], canAuthorise: [] }); + }); + + it('should be able to construct a user instance', async function () { + const user = new User( + 'username', + 'password', + 'gitAccount', + 'email@domain.com', + true, + null, + 'id', + ); + expect(user.username).to.equal('username'); + expect(user.username).to.equal('username'); + expect(user.gitAccount).to.equal('gitAccount'); + expect(user.email).to.equal('email@domain.com'); + expect(user.admin).to.equal(true); + expect(user.oidcId).to.be.null; + expect(user._id).to.equal('id'); + + const user2 = new User( + 'username', + 'password', + 'gitAccount', + 'email@domain.com', + false, + 'oidcId', + 'id', + ); + expect(user2.admin).to.equal(false); + expect(user2.oidcId).to.equal('oidcId'); + }); + + it('should be able to construct a valid action instance', async function () { + const action = new Action( + 'id', + 'type', + 'method', + Date.now(), + 'https://github.com/finos.git-proxy.git', + ); + expect(action.project).to.equal('finos'); + expect(action.repoName).to.equal('git-proxy.git'); + }); + + it('should be able to block an action by adding a blocked step', async function () { + const action = new Action( + 'id', + 'type', + 'method', + Date.now(), + 'https://github.com/finos.git-proxy.git', + ); + const step = new Step('stepName', false, null, false, null); + step.setAsyncBlock('blockedMessage'); + action.addStep(step); + expect(action.blocked).to.be.true; + expect(action.blockedMessage).to.equal('blockedMessage'); + expect(action.getLastStep()).to.deep.equals(step); + expect(action.continue()).to.be.false; + }); + + it('should be able to error an action by adding a step with an error', async function () { + const action = new Action( + 'id', + 'type', + 'method', + Date.now(), + 'https://github.com/finos.git-proxy.git', + ); + const step = new Step('stepName', true, 'errorMessage', false, null); + action.addStep(step); + expect(action.error).to.be.true; + expect(action.errorMessage).to.equal('errorMessage'); + expect(action.getLastStep()).to.deep.equals(step); + expect(action.continue()).to.be.false; + }); + it('should be able to create a repo', async function () { await db.createRepo(TEST_REPO); const repos = await db.getRepos(); diff --git a/test/testParseAction.test.js b/test/testParseAction.test.js new file mode 100644 index 000000000..b2cbd6d02 --- /dev/null +++ b/test/testParseAction.test.js @@ -0,0 +1,38 @@ +// Import the dependencies for testing +const chai = require('chai'); +chai.should(); +const expect = chai.expect; +const preprocessor = require('../src/proxy/processors/pre-processor/parseAction'); + +describe('Pre-processor: parseAction', async () => { + before(async function () {}); + after(async function () {}); + + it('should be able to parse a pull request into an action', async function () { + const req = { + originalUrl: 'https://github.com/finos/git-proxy.git/git-upload-pack', + method: 'GET', + headers: {}, + }; + + const action = await preprocessor.exec(req); + expect(action.timestamp).is.greaterThan(0); + expect(action.id).to.not.be.false; + expect(action.type).to.equal('pull'); + expect(action.url).to.equal('https://github.com/finos/git-proxy.git'); + }); + + it('should be able to parse a push request into an action', async function () { + const req = { + originalUrl: 'https://github.com/finos/git-proxy.git/git-receive-pack', + method: 'POST', + headers: { 'content-type': 'application/x-git-receive-pack-request' }, + }; + + const action = await preprocessor.exec(req); + expect(action.timestamp).is.greaterThan(0); + expect(action.id).to.not.be.false; + expect(action.type).to.equal('push'); + expect(action.url).to.equal('https://github.com/finos/git-proxy.git'); + }); +}); From 09763951e47e8af67a42620d7dffb0682bfb80c8 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 12 Jun 2025 09:20:39 +0100 Subject: [PATCH 36/76] test: fixing issues in pasreAction and db pojo tests --- src/db/types.ts | 2 +- test/testDb.test.js | 13 +++++++++++-- test/testParseAction.test.js | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/db/types.ts b/src/db/types.ts index fc9503701..27f3c198b 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -14,7 +14,7 @@ export class Repo { project: string; name: string; url: string; - users: Record; + users: { canPush: string[]; canAuthorise: string[] } | null; _id?: string; constructor( diff --git a/test/testDb.test.js b/test/testDb.test.js index 6772d51a2..04c8321ce 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -90,12 +90,21 @@ describe('Database clients', async () => { before(async function () {}); it('should be able to construct a repo instance', async function () { - const repo = new Repo('project', 'name', 'https://github.com/finos.git-proxy.git', {}, 'id'); + const repo = new Repo('project', 'name', 'https://github.com/finos.git-proxy.git', null, 'id'); expect(repo._id).to.equal('id'); expect(repo.project).to.equal('project'); expect(repo.name).to.equal('name'); expect(repo.url).to.equal('https://github.com/finos.git-proxy.git'); expect(repo.users).to.deep.equals({ canPush: [], canAuthorise: [] }); + + const repo2 = new Repo( + 'project', + 'name', + 'https://github.com/finos.git-proxy.git', + { canPush: ['bill'], canAuthorise: ['ben'] }, + 'id', + ); + expect(repo2.users).to.deep.equals({ canPush: ['bill'], canAuthorise: ['ben'] }); }); it('should be able to construct a user instance', async function () { @@ -135,7 +144,7 @@ describe('Database clients', async () => { 'type', 'method', Date.now(), - 'https://github.com/finos.git-proxy.git', + 'https://github.com/finos/git-proxy.git', ); expect(action.project).to.equal('finos'); expect(action.repoName).to.equal('git-proxy.git'); diff --git a/test/testParseAction.test.js b/test/testParseAction.test.js index b2cbd6d02..4cbc49d19 100644 --- a/test/testParseAction.test.js +++ b/test/testParseAction.test.js @@ -10,7 +10,21 @@ describe('Pre-processor: parseAction', async () => { it('should be able to parse a pull request into an action', async function () { const req = { - originalUrl: 'https://github.com/finos/git-proxy.git/git-upload-pack', + originalUrl: '/github.com/finos/git-proxy.git/git-upload-pack', + method: 'GET', + headers: {}, + }; + + const action = await preprocessor.exec(req); + expect(action.timestamp).is.greaterThan(0); + expect(action.id).to.not.be.false; + expect(action.type).to.equal('pull'); + expect(action.url).to.equal('https://github.com/finos/git-proxy.git'); + }); + + it('should be able to parse a pull request with a legacy path into an action', async function () { + const req = { + originalUrl: '/finos/git-proxy.git/git-upload-pack', method: 'GET', headers: {}, }; @@ -24,7 +38,21 @@ describe('Pre-processor: parseAction', async () => { it('should be able to parse a push request into an action', async function () { const req = { - originalUrl: 'https://github.com/finos/git-proxy.git/git-receive-pack', + originalUrl: '/github.com/finos/git-proxy.git/git-receive-pack', + method: 'POST', + headers: { 'content-type': 'application/x-git-receive-pack-request' }, + }; + + const action = await preprocessor.exec(req); + expect(action.timestamp).is.greaterThan(0); + expect(action.id).to.not.be.false; + expect(action.type).to.equal('push'); + expect(action.url).to.equal('https://github.com/finos/git-proxy.git'); + }); + + it('should be able to parse a push request with a legacy path into an action', async function () { + const req = { + originalUrl: '/finos/git-proxy.git/git-receive-pack', method: 'POST', headers: { 'content-type': 'application/x-git-receive-pack-request' }, }; From c616f061815c72d3136b53ed9514fe566cd544c3 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 12 Jun 2025 10:32:01 +0100 Subject: [PATCH 37/76] test: data setup for parseAction test --- test/testParseAction.test.js | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/test/testParseAction.test.js b/test/testParseAction.test.js index 4cbc49d19..02686fc1d 100644 --- a/test/testParseAction.test.js +++ b/test/testParseAction.test.js @@ -3,10 +3,27 @@ const chai = require('chai'); chai.should(); const expect = chai.expect; const preprocessor = require('../src/proxy/processors/pre-processor/parseAction'); +const db = require('../src/db'); +let testRepo = null; + +const TEST_REPO = { + url: 'https://github.com/finos/git-proxy.git', + name: 'git-proxy', + project: 'finos', +}; describe('Pre-processor: parseAction', async () => { - before(async function () {}); - after(async function () {}); + before(async function () { + // make sure the test repo exists as the presence of the repo makes a difference to handling of urls + testRepo = await db.getRepoByUrl(TEST_REPO.url); + if (!testRepo) { + testRepo = await db.createRepo(TEST_REPO); + } + }); + after(async function () { + // clean up test DB + await db.deleteRepo(testRepo._id); + }); it('should be able to parse a pull request into an action', async function () { const req = { From eb42b944818bf026121a17a8c96ce9d06061266a Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 12 Jun 2025 15:56:20 +0100 Subject: [PATCH 38/76] test: enable tests that weren't being run due to file name --- src/proxy/routes/helper.ts | 5 ++++ ...testParsePush.js => testParsePush.test.js} | 0 test/testProxyRoute.test.js | 26 +++++++++---------- test/testRouteFilter.test.js | 4 +++ 4 files changed, 22 insertions(+), 13 deletions(-) rename test/{testParsePush.js => testParsePush.test.js} (100%) diff --git a/src/proxy/routes/helper.ts b/src/proxy/routes/helper.ts index 2fba8616c..cf47fdcbb 100644 --- a/src/proxy/routes/helper.ts +++ b/src/proxy/routes/helper.ts @@ -97,6 +97,11 @@ export type GitNameBreakdown = { project: string | null; repoName: string }; * - project: null * - repoName: repo.git * + * Processing someGitHost.com/repo.git + * would produce: + * - project: null + * - repoName: repo.git + * * Processing https://anotherGitHost.com/project/subProject/subSubProject/repo.git * would produce: * - project: project/subProject/subSubProject diff --git a/test/testParsePush.js b/test/testParsePush.test.js similarity index 100% rename from test/testParsePush.js rename to test/testParsePush.test.js diff --git a/test/testProxyRoute.test.js b/test/testProxyRoute.test.js index caa9e8615..e778f65ee 100644 --- a/test/testProxyRoute.test.js +++ b/test/testProxyRoute.test.js @@ -112,26 +112,26 @@ describe('proxy route helpers', () => { }); expect(res).to.be.true; }); - + it('should return true for /info/refs?service=git-receive-pack with valid user-agent', () => { const res = validGitRequest('/info/refs?service=git-receive-pack', { 'user-agent': 'git/1.9.1', }); expect(res).to.be.true; }); - + it('should return false for /info/refs?service=git-upload-pack with missing user-agent', () => { const res = validGitRequest('/info/refs?service=git-upload-pack', {}); expect(res).to.be.false; }); - + it('should return false for /info/refs?service=git-upload-pack with non-git user-agent', () => { const res = validGitRequest('/info/refs?service=git-upload-pack', { 'user-agent': 'curl/7.79.1', }); expect(res).to.be.false; }); - + it('should return true for /git-upload-pack with valid user-agent and accept', () => { const res = validGitRequest('/git-upload-pack', { 'user-agent': 'git/2.40.0', @@ -139,14 +139,14 @@ describe('proxy route helpers', () => { }); expect(res).to.be.true; }); - + it('should return false for /git-upload-pack with missing accept header', () => { const res = validGitRequest('/git-upload-pack', { 'user-agent': 'git/2.40.0', }); expect(res).to.be.false; }); - + it('should return false for /git-upload-pack with wrong accept header', () => { const res = validGitRequest('/git-upload-pack', { 'user-agent': 'git/2.40.0', @@ -154,7 +154,7 @@ describe('proxy route helpers', () => { }); expect(res).to.be.false; }); - + it('should return false for unknown paths', () => { const res = validGitRequest('/not-a-valid-git-path', { 'user-agent': 'git/2.40.0', @@ -169,32 +169,32 @@ describe('proxy route helpers', () => { const res = stripGitHubFromGitPath('/foo/bar.git/info/refs'); expect(res).to.equal('/info/refs'); }); - + it('should strip owner and repo from a valid GitHub-style path with 5 parts', () => { const res = stripGitHubFromGitPath('/foo/bar.git/git-upload-pack'); expect(res).to.equal('/git-upload-pack'); }); - + it('should return undefined for malformed path with too few segments', () => { const res = stripGitHubFromGitPath('/foo/bar.git'); expect(res).to.be.undefined; }); - + it('should return undefined for malformed path with too many segments', () => { const res = stripGitHubFromGitPath('/foo/bar.git/extra/path/stuff'); expect(res).to.be.undefined; }); - + it('should handle repo names that include dots correctly', () => { const res = stripGitHubFromGitPath('/foo/some.repo.git/info/refs'); expect(res).to.equal('/info/refs'); }); - + it('should not break if the path is just a slash', () => { const res = stripGitHubFromGitPath('/'); expect(res).to.be.undefined; }); - + it('should not break if the path is empty', () => { const res = stripGitHubFromGitPath(''); expect(res).to.be.undefined; diff --git a/test/testRouteFilter.test.js b/test/testRouteFilter.test.js index f73f42565..99a133fcb 100644 --- a/test/testRouteFilter.test.js +++ b/test/testRouteFilter.test.js @@ -65,6 +65,10 @@ describe('url helpers and filter functions used in the proxy', function () { }); }); + it('processGitUrl should return null for a url it cannot parse', function () { + expect(processGitUrl('somegithost.com:1234/octocat/hello-world.git')).to.be.null; + }); + it('processGitURLForNameAndOrg should return breakdown of a git URL path separating out the protocol, origin and repository path', function () { expect(processGitURLForNameAndOrg('github.com/octocat/hello-world.git')).to.deep.eq({ project: 'octocat', From d7d7e23cd9ee42c6cd06f2c111c8cedfb0302439 Mon Sep 17 00:00:00 2001 From: Kris West Date: Mon, 16 Jun 2025 16:29:11 +0100 Subject: [PATCH 39/76] chore: prettier --- src/ui/components/RouteGuard/RouteGuard.tsx | 4 +- test/processors/blockForAuth.test.js | 7 +- test/processors/checkAuthorEmails.test.js | 91 +++++++++++---------- test/testPush.test.js | 2 +- 4 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/ui/components/RouteGuard/RouteGuard.tsx b/src/ui/components/RouteGuard/RouteGuard.tsx index a729b1660..4efb2c3c1 100644 --- a/src/ui/components/RouteGuard/RouteGuard.tsx +++ b/src/ui/components/RouteGuard/RouteGuard.tsx @@ -46,11 +46,11 @@ const RouteGuard = ({ component: Component, fullRoutePath }: RouteGuardProps) => } if (loginRequired && !user) { - return ; + return ; } if (adminOnly && !user?.admin) { - return ; + return ; } return ; diff --git a/test/processors/blockForAuth.test.js b/test/processors/blockForAuth.test.js index f566f1b2f..e242cb247 100644 --- a/test/processors/blockForAuth.test.js +++ b/test/processors/blockForAuth.test.js @@ -17,12 +17,12 @@ describe('blockForAuth', () => { beforeEach(() => { req = { protocol: 'https', - headers: { host: 'example.com' } + headers: { host: 'example.com' }, }; action = { id: 'push_123', - addStep: sinon.stub() + addStep: sinon.stub(), }; stepInstance = new Step('temp'); @@ -34,7 +34,7 @@ describe('blockForAuth', () => { const blockForAuth = proxyquire('../../src/proxy/processors/push-action/blockForAuth', { '../../../service/urls': { getServiceUIURL: getServiceUIURLStub }, - '../../actions': { Step: StepSpy } + '../../actions': { Step: StepSpy }, }); exec = blockForAuth.exec; @@ -45,7 +45,6 @@ describe('blockForAuth', () => { }); describe('exec', () => { - it('should generate a correct shareable URL', async () => { await exec(req, action); expect(getServiceUIURLStub.calledOnce).to.be.true; diff --git a/test/processors/checkAuthorEmails.test.js b/test/processors/checkAuthorEmails.test.js index 52d8ffc6e..849842704 100644 --- a/test/processors/checkAuthorEmails.test.js +++ b/test/processors/checkAuthorEmails.test.js @@ -4,7 +4,7 @@ const { expect } = require('chai'); describe('checkAuthorEmails', () => { let action; - let commitConfig + let commitConfig; let exec; let getCommitConfigStub; let stepSpy; @@ -25,9 +25,9 @@ describe('checkAuthorEmails', () => { author: { email: { domain: { allow: null }, - local: { block: null } - } - } + local: { block: null }, + }, + }, }; getCommitConfigStub = sinon.stub().returns(commitConfig); @@ -37,13 +37,16 @@ describe('checkAuthorEmails', () => { action.step = new StepStub(); Object.assign(action.step, step); return action.step; - }) + }), }; - const checkAuthorEmails = proxyquire('../../src/proxy/processors/push-action/checkAuthorEmails', { - '../../../config': { getCommitConfig: getCommitConfigStub }, - '../../actions': { Step: StepStub } - }); + const checkAuthorEmails = proxyquire( + '../../src/proxy/processors/push-action/checkAuthorEmails', + { + '../../../config': { getCommitConfig: getCommitConfigStub }, + '../../actions': { Step: StepStub }, + }, + ); exec = checkAuthorEmails.exec; }); @@ -56,7 +59,7 @@ describe('checkAuthorEmails', () => { it('should allow valid emails when no restrictions', async () => { action.commitData = [ { authorEmail: 'valid@example.com' }, - { authorEmail: 'another.valid@test.org' } + { authorEmail: 'another.valid@test.org' }, ]; await exec({}, action); @@ -68,47 +71,48 @@ describe('checkAuthorEmails', () => { commitConfig.author.email.domain.allow = 'example\\.com$'; action.commitData = [ { authorEmail: 'valid@example.com' }, - { authorEmail: 'invalid@forbidden.org' } + { authorEmail: 'invalid@forbidden.org' }, ]; await exec({}, action); expect(action.step.error).to.be.true; - expect(stepSpy.calledWith( - 'The following commit author e-mails are illegal: invalid@forbidden.org' - )).to.be.true; - expect(StepStub.prototype.setError.calledWith( - 'Your push has been blocked. Please verify your Git configured e-mail address is valid (e.g. john.smith@example.com)' - )).to.be.true; + expect( + stepSpy.calledWith( + 'The following commit author e-mails are illegal: invalid@forbidden.org', + ), + ).to.be.true; + expect( + StepStub.prototype.setError.calledWith( + 'Your push has been blocked. Please verify your Git configured e-mail address is valid (e.g. john.smith@example.com)', + ), + ).to.be.true; }); it('should block emails with forbidden usernames', async () => { commitConfig.author.email.local.block = 'blocked'; action.commitData = [ { authorEmail: 'allowed@example.com' }, - { authorEmail: 'blocked.user@test.org' } + { authorEmail: 'blocked.user@test.org' }, ]; await exec({}, action); expect(action.step.error).to.be.true; - expect(stepSpy.calledWith( - 'The following commit author e-mails are illegal: blocked.user@test.org' - )).to.be.true; + expect( + stepSpy.calledWith( + 'The following commit author e-mails are illegal: blocked.user@test.org', + ), + ).to.be.true; }); it('should handle empty email strings', async () => { - action.commitData = [ - { authorEmail: '' }, - { authorEmail: 'valid@example.com' } - ]; + action.commitData = [{ authorEmail: '' }, { authorEmail: 'valid@example.com' }]; await exec({}, action); expect(action.step.error).to.be.true; - expect(stepSpy.calledWith( - 'The following commit author e-mails are illegal: ' - )).to.be.true; + expect(stepSpy.calledWith('The following commit author e-mails are illegal: ')).to.be.true; }); it('should allow emails when both checks pass', async () => { @@ -116,7 +120,7 @@ describe('checkAuthorEmails', () => { commitConfig.author.email.local.block = 'forbidden'; action.commitData = [ { authorEmail: 'allowed@example.com' }, - { authorEmail: 'also.allowed@example.com' } + { authorEmail: 'also.allowed@example.com' }, ]; await exec({}, action); @@ -127,29 +131,24 @@ describe('checkAuthorEmails', () => { it('should block emails that fail both checks', async () => { commitConfig.author.email.domain.allow = 'example\\.com$'; commitConfig.author.email.local.block = 'forbidden'; - action.commitData = [ - { authorEmail: 'forbidden@wrong.org' } - ]; + action.commitData = [{ authorEmail: 'forbidden@wrong.org' }]; await exec({}, action); expect(action.step.error).to.be.true; - expect(stepSpy.calledWith( - 'The following commit author e-mails are illegal: forbidden@wrong.org' - )).to.be.true; + expect( + stepSpy.calledWith('The following commit author e-mails are illegal: forbidden@wrong.org'), + ).to.be.true; }); it('should handle emails without domain', async () => { - action.commitData = [ - { authorEmail: 'nodomain@' } - ]; + action.commitData = [{ authorEmail: 'nodomain@' }]; await exec({}, action); expect(action.step.error).to.be.true; - expect(stepSpy.calledWith( - 'The following commit author e-mails are illegal: nodomain@' - )).to.be.true; + expect(stepSpy.calledWith('The following commit author e-mails are illegal: nodomain@')).to.be + .true; }); it('should handle multiple illegal emails', async () => { @@ -157,15 +156,17 @@ describe('checkAuthorEmails', () => { action.commitData = [ { authorEmail: 'invalid1@bad.org' }, { authorEmail: 'invalid2@wrong.net' }, - { authorEmail: 'valid@example.com' } + { authorEmail: 'valid@example.com' }, ]; await exec({}, action); expect(action.step.error).to.be.true; - expect(stepSpy.calledWith( - 'The following commit author e-mails are illegal: invalid1@bad.org,invalid2@wrong.net' - )).to.be.true; + expect( + stepSpy.calledWith( + 'The following commit author e-mails are illegal: invalid1@bad.org,invalid2@wrong.net', + ), + ).to.be.true; }); }); }); diff --git a/test/testPush.test.js b/test/testPush.test.js index f4e09a4a5..a524e173c 100644 --- a/test/testPush.test.js +++ b/test/testPush.test.js @@ -24,7 +24,7 @@ describe('auth', async () => { expect(res).to.have.cookie('connect.sid'); res.should.have.status(200); - // Get the connect cooie + // Get the connect cookie res.headers['set-cookie'].forEach((x) => { if (x.startsWith('connect')) { cookie = x.split(';')[0]; From eb22bcbe67be92f59c52a558b8674688aea2e731 Mon Sep 17 00:00:00 2001 From: Kris West Date: Wed, 18 Jun 2025 17:52:55 +0100 Subject: [PATCH 40/76] fix: ensure proxy routes are reworked on restart after adding a repo --- index.ts | 2 +- package-lock.json | 39 ++++++----- src/db/index.ts | 10 ++- src/proxy/index.ts | 8 ++- src/proxy/routes/index.ts | 20 +++--- src/service/routes/repo.js | 5 +- test/addRepoTest.test.js | 123 +++++++++++++++++++++++++++-------- test/testDb.test.js | 72 ++++++++++++++++++-- test/testRouteFilter.test.js | 34 +++++++++- 9 files changed, 239 insertions(+), 74 deletions(-) diff --git a/index.ts b/index.ts index 880ccfe02..7fdcc4da9 100755 --- a/index.ts +++ b/index.ts @@ -28,7 +28,7 @@ const argv = yargs(hideBin(process.argv)) .strict() .parseSync(); -setConfigFile(argv.c as string || ""); +setConfigFile((argv.c as string) || ''); if (argv.v) { if (!fs.existsSync(configFile)) { diff --git a/package-lock.json b/package-lock.json index 407ddf196..206437b27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2848,9 +2848,10 @@ } }, "node_modules/@npmcli/map-workspaces/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -4053,9 +4054,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { @@ -4886,10 +4887,11 @@ "peer": true }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5097,9 +5099,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001685", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001685.tgz", - "integrity": "sha512-e/kJN1EMyHQzgcMEEgoo+YTCO1NGCmIYHk5Qk8jT6AazWemS5QFKJ5ShCJlH3GZrNIdZofcNCEwZqbMjjKzmnA==", + "version": "1.0.30001723", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001723.tgz", + "integrity": "sha512-1R/elMjtehrFejxwmexeXAtae5UO9iSyFn6G/I806CYC/BLyyBk1EPhrKBkWhy6wM6Xnm47dSJQec+tLJ39WHw==", "dev": true, "funding": [ { @@ -7911,9 +7913,10 @@ } }, "node_modules/glob/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -10329,9 +10332,9 @@ } }, "node_modules/mocha/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { diff --git a/src/db/index.ts b/src/db/index.ts index 518f64468..3d31c800a 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -79,8 +79,9 @@ export const createRepo = async (repo: AuthorisedRepo) => { console.log(`creating new repo ${JSON.stringify(toCreate)}`); - if (isBlank(toCreate.project)) { - throw new Error('Project name cannot be empty'); + // n.b. project name may be blank but not null for non-github and non-gitlab repos + if (!toCreate.project) { + toCreate.project = ''; } if (isBlank(toCreate.name)) { throw new Error('Repository name cannot be empty'); @@ -101,10 +102,7 @@ export const isUserPushAllowed = async (url: string, user: string) => { return; } - console.log(repo.users.canPush); - console.log(repo.users.canAuthorise); - - if (repo.users.canPush.includes(user) || repo.users.canAuthorise.includes(user)) { + if (repo.users?.canPush.includes(user) || repo.users?.canAuthorise.includes(user)) { resolve(true); } else { resolve(false); diff --git a/src/proxy/index.ts b/src/proxy/index.ts index afd2d7923..1683a69a9 100644 --- a/src/proxy/index.ts +++ b/src/proxy/index.ts @@ -1,9 +1,9 @@ -import express, { Application } from 'express'; +import express, { Application, Router } from 'express'; import bodyParser from 'body-parser'; import http from 'http'; import https from 'https'; import fs from 'fs'; -import { router } from './routes'; +import { getRouter } from './routes'; import { getAuthorisedList, getPlugins, @@ -54,9 +54,10 @@ export const proxyPreparations = async () => { }); }; -// just keep this async incase it needs async stuff in the future const createApp = async (): Promise => { const app = express(); + router = await getRouter(); + // Setup the proxy middleware app.use(bodyParser.raw(options)); app.use('/', router); @@ -65,6 +66,7 @@ const createApp = async (): Promise => { let httpServer: http.Server | null = null; let httpsServer: https.Server | null = null; +let router: Router | null = null; const start = async (): Promise => { const app = await createApp(); diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index 27dd59fdc..126140e3f 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -100,16 +100,18 @@ const proxyErrorHandler: ProxyOptions['proxyErrorHandler'] = (err, res, next) => next(err); }; -// eslint-disable-next-line new-cap -const router = Router(); - -getAllProxiedHosts().then((originsToProxy) => { - // TODO: this will only happen on startup. We'll need to add routes at runtime when new origins are added? Or force a restart for the proxy to work +const getRouter = async () => { + // eslint-disable-next-line new-cap + const router = Router(); + const originsToProxy = await getAllProxiedHosts(); + console.log( + `Initializing proxy router for origins: '${JSON.stringify(originsToProxy, null, 2)}'`, + ); // Middlewares are processed in the order that they are added, if one applies and then doesn't call `next` then subsequent ones are not applied. // Hence, we define known origins first, then a catch all route for backwards compatibility originsToProxy.forEach((origin) => { - console.log(`setting up origin '${origin}'`); + console.log(`\tsetting up origin: '${origin}'`); router.use( '/' + origin, proxy('https://' + origin, { @@ -124,6 +126,7 @@ getAllProxiedHosts().then((originsToProxy) => { }); // Catch-all route for backwards compatibility + console.log('\tsetting up catch-all route (github.com) for backwards compatibility'); router.use( '/', proxy('https://github.com', { @@ -135,6 +138,7 @@ getAllProxiedHosts().then((originsToProxy) => { proxyErrorHandler: proxyErrorHandler, }), ); -}); + return router; +}; -export { router, handleMessage, validGitRequest }; +export { getRouter, handleMessage, validGitRequest }; diff --git a/src/service/routes/repo.js b/src/service/routes/repo.js index 91868cafc..3c96b73b5 100644 --- a/src/service/routes/repo.js +++ b/src/service/routes/repo.js @@ -162,7 +162,7 @@ router.post('/', async (req, res) => { console.log('Restarting the proxy to handle an additional origin'); // 1. Get proxy module dynamically to avoid circular dependency - const proxy = require('../proxy'); + const { proxy } = require('../index'); // 2. Stop existing services await proxy.stop(); @@ -170,7 +170,8 @@ router.post('/', async (req, res) => { // 3. Restart the proxy, which should set up for the new domain await proxy.start(); } - } catch { + } catch (e) { + console.error('Repository creation failed due to error: ', e.message ? e.message : e); res.send('Failed to create repository'); } } diff --git a/test/addRepoTest.test.js b/test/addRepoTest.test.js index e21c28823..765f575c5 100644 --- a/test/addRepoTest.test.js +++ b/test/addRepoTest.test.js @@ -3,6 +3,7 @@ const chai = require('chai'); const chaiHttp = require('chai-http'); const db = require('../src/db'); const service = require('../src/service'); +const { getAllProxiedHosts } = require('../src/proxy/routes/helper'); chai.use(chaiHttp); chai.should(); @@ -12,12 +13,34 @@ const TEST_REPO = { url: 'https://github.com/finos/test-repo.git', name: 'test-repo', project: 'finos', + host: 'github.com', +}; + +const TEST_REPO_NON_GITHUB = { + url: 'https://gitlab.com/org/sub-org/test-repo2.git', + name: 'test-repo2', + project: 'org/sub-org', + host: 'gitlab.com', +}; + +const TEST_REPO_NAKED = { + url: 'https://123.456.789:80/test-repo3.git', + name: 'test-repo3', + project: '', + host: '123.456.789:80', +}; + +const cleanupRepo = async (url) => { + const repo = await db.getRepoByUrl(url); + if (repo) { + await db.deleteRepo(repo._id); + } }; describe('add new repo', async () => { let app; let cookie; - let repoId; + const repoIds = []; const setCookie = function (res) { res.headers['set-cookie'].forEach((x) => { @@ -32,10 +55,10 @@ describe('add new repo', async () => { app = await service.start(); // Prepare the data. // _id is autogenerated by the DB so we need to retrieve it before we can use it - const repo = await db.getRepoByUrl(TEST_REPO.url); - if (repo) { - await db.deleteRepo(repo._id); - } + cleanupRepo(TEST_REPO.url); + cleanupRepo(TEST_REPO_NON_GITHUB.url); + cleanupRepo(TEST_REPO_NAKED.url); + await db.deleteUser('u1'); await db.deleteUser('u2'); await db.createUser('u1', 'abc', 'test@test.com', 'test', true); @@ -61,7 +84,7 @@ describe('add new repo', async () => { const repo = await db.getRepoByUrl(TEST_REPO.url); // save repo id for use in subsequent tests - repoId = repo._id; + repoIds[0] = repo._id; repo.project.should.equal(TEST_REPO.project); repo.name.should.equal(TEST_REPO.name); @@ -85,14 +108,14 @@ describe('add new repo', async () => { it('add 1st can push user', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${repoId}/user/push`) + .patch(`/api/v1/repo/${repoIds[0]}/user/push`) .set('Cookie', `${cookie}`) .send({ username: 'u1', }); res.should.have.status(200); - const repo = await db.getRepoById(repoId); + const repo = await db.getRepoById(repoIds[0]); repo.users.canPush.length.should.equal(1); repo.users.canPush[0].should.equal('u1'); }); @@ -100,14 +123,14 @@ describe('add new repo', async () => { it('add 2nd can push user', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${repoId}/user/push`) + .patch(`/api/v1/repo/${repoIds[0]}/user/push`) .set('Cookie', `${cookie}`) .send({ username: 'u2', }); res.should.have.status(200); - const repo = await db.getRepoById(repoId); + const repo = await db.getRepoById(repoIds[0]); repo.users.canPush.length.should.equal(2); repo.users.canPush[1].should.equal('u2'); }); @@ -115,40 +138,40 @@ describe('add new repo', async () => { it('add push user that does not exist', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${repoId}/user/push`) + .patch(`/api/v1/repo/${repoIds[0]}/user/push`) .set('Cookie', `${cookie}`) .send({ username: 'u3', }); res.should.have.status(400); - const repo = await db.getRepoById(repoId); + const repo = await db.getRepoById(repoIds[0]); repo.users.canPush.length.should.equal(2); }); it('delete user u2 from push', async function () { const res = await chai .request(app) - .delete(`/api/v1/repo/${repoId}/user/push/u2`) + .delete(`/api/v1/repo/${repoIds[0]}/user/push/u2`) .set('Cookie', `${cookie}`) .send({}); res.should.have.status(200); - const repo = await db.getRepoById(repoId); + const repo = await db.getRepoById(repoIds[0]); repo.users.canPush.length.should.equal(1); }); it('add 1st can authorise user', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${repoId}/user/authorise`) + .patch(`/api/v1/repo/${repoIds[0]}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u1', }); res.should.have.status(200); - const repo = await db.getRepoById(repoId); + const repo = await db.getRepoById(repoIds[0]); repo.users.canAuthorise.length.should.equal(1); repo.users.canAuthorise[0].should.equal('u1'); }); @@ -156,14 +179,14 @@ describe('add new repo', async () => { it('add 2nd can authorise user', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${repoId}/user/authorise`) + .patch(`/api/v1/repo/${repoIds[0]}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u2', }); res.should.have.status(200); - const repo = await db.getRepoById(repoId); + const repo = await db.getRepoById(repoIds[0]); repo.users.canAuthorise.length.should.equal(2); repo.users.canAuthorise[1].should.equal('u2'); }); @@ -171,33 +194,33 @@ describe('add new repo', async () => { it('add authorise user that does not exist', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${repoId}/user/authorise`) + .patch(`/api/v1/repo/${repoIds[0]}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u3', }); res.should.have.status(400); - const repo = await db.getRepoById(repoId); + const repo = await db.getRepoById(repoIds[0]); repo.users.canAuthorise.length.should.equal(2); }); it('Can delete u2 user', async function () { const res = await chai .request(app) - .delete(`/api/v1/repo/${repoId}/user/authorise/u2`) + .delete(`/api/v1/repo/${repoIds[0]}/user/authorise/u2`) .set('Cookie', `${cookie}`) .send({}); res.should.have.status(200); - const repo = await db.getRepoById(repoId); + const repo = await db.getRepoById(repoIds[0]); repo.users.canAuthorise.length.should.equal(1); }); it('Valid user push permission on repo', async function () { const res = await chai .request(app) - .patch(`/api/v1/repo/${repoId}/user/authorise`) + .patch(`/api/v1/repo/${repoIds[0]}/user/authorise`) .set('Cookie', `${cookie}`) .send({ username: 'u2' }); @@ -211,10 +234,58 @@ describe('add new repo', async () => { expect(isAllowed).to.be.false; }); + it('Proxy route helpers should return the proxied origin', async function () { + const origins = await getAllProxiedHosts(); + expect(origins).to.eql([TEST_REPO.host]); + }); + + it('Proxy route helpers should return the new proxied origins when new repos are added', async function () { + const res = await chai + .request(app) + .post('/api/v1/repo') + .set('Cookie', `${cookie}`) + .send(TEST_REPO_NON_GITHUB); + res.should.have.status(200); + + const repo = await db.getRepoByUrl(TEST_REPO_NON_GITHUB.url); + // save repo id for use in subsequent tests + repoIds[1] = repo._id; + + repo.project.should.equal(TEST_REPO_NON_GITHUB.project); + repo.name.should.equal(TEST_REPO_NON_GITHUB.name); + repo.url.should.equal(TEST_REPO_NON_GITHUB.url); + repo.users.canPush.length.should.equal(0); + repo.users.canAuthorise.length.should.equal(0); + + const origins = await getAllProxiedHosts(); + expect(origins).to.have.members([TEST_REPO.host, TEST_REPO_NON_GITHUB.host]); + + const res2 = await chai + .request(app) + .post('/api/v1/repo') + .set('Cookie', `${cookie}`) + .send(TEST_REPO_NAKED); + res2.should.have.status(200); + const repo2 = await db.getRepoByUrl(TEST_REPO_NAKED.url); + repoIds[2] = repo2._id; + + const origins2 = await getAllProxiedHosts(); + expect(origins2).to.have.members([ + TEST_REPO.host, + TEST_REPO_NON_GITHUB.host, + TEST_REPO_NAKED.host, + ]); + }); + after(async function () { await service.httpServer.close(); - await db.deleteRepo('test-repo'); - await db.deleteUser('u1'); - await db.deleteUser('u2'); + + // don't clean up data as cypress tests rely on it being present + // await cleanupRepo(TEST_REPO.url); + // await db.deleteUser('u1'); + // await db.deleteUser('u2'); + + await cleanupRepo(TEST_REPO_NON_GITHUB.url); + await cleanupRepo(TEST_REPO_NAKED.url); }); }); diff --git a/test/testDb.test.js b/test/testDb.test.js index 04c8321ce..a016047a5 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -229,7 +229,7 @@ describe('Database clients', async () => { expect(cleanRepos).to.not.deep.include(TEST_REPO); }); - it('should NOT be able to create a repo with blank project, name or url', async function () { + it('should be able to create a repo with a blank project', async function () { // test with a null value let threwError = false; let testRepo = { @@ -238,11 +238,12 @@ describe('Database clients', async () => { url: TEST_REPO.url, }; try { - await db.createRepo(testRepo); + const repo = await db.createRepo(testRepo); + await db.deleteRepo(repo._id, true); } catch (e) { threwError = true; } - expect(threwError).to.be.true; + expect(threwError).to.be.false; // test with an empty string threwError = false; @@ -252,11 +253,12 @@ describe('Database clients', async () => { url: TEST_REPO.url, }; try { - await db.createRepo(testRepo); + const repo = await db.createRepo(testRepo); + await db.deleteRepo(repo._id, true); } catch (e) { threwError = true; } - expect(threwError).to.be.true; + expect(threwError).to.be.false; // test with an undefined property threwError = false; @@ -264,6 +266,23 @@ describe('Database clients', async () => { name: TEST_REPO.name, url: TEST_REPO.url, }; + try { + const repo = await db.createRepo(testRepo); + await db.deleteRepo(repo._id, true); + } catch (e) { + threwError = true; + } + expect(threwError).to.be.false; + }); + + it('should NOT be able to create a repo with blank name or url', async function () { + // null name + let threwError = false; + let testRepo = { + project: TEST_REPO.project, + name: null, + url: TEST_REPO.url, + }; try { await db.createRepo(testRepo); } catch (e) { @@ -271,11 +290,24 @@ describe('Database clients', async () => { } expect(threwError).to.be.true; - // repeat tests for other fields, but don't both with all variations as they go through same fn + // blank name + threwError = false; + testRepo = { + project: TEST_REPO.project, + name: '', + url: TEST_REPO.url, + }; + try { + await db.createRepo(testRepo); + } catch (e) { + threwError = true; + } + expect(threwError).to.be.true; + + // undefined name threwError = false; testRepo = { project: TEST_REPO.project, - name: null, url: TEST_REPO.url, }; try { @@ -285,6 +317,7 @@ describe('Database clients', async () => { } expect(threwError).to.be.true; + // null url testRepo = { project: TEST_REPO.project, name: TEST_REPO.name, @@ -296,6 +329,31 @@ describe('Database clients', async () => { threwError = true; } expect(threwError).to.be.true; + + // blank url + testRepo = { + project: TEST_REPO.project, + name: TEST_REPO.name, + url: '', + }; + try { + await db.createRepo(testRepo); + } catch (e) { + threwError = true; + } + expect(threwError).to.be.true; + + // undefined url + testRepo = { + project: TEST_REPO.project, + name: TEST_REPO.name, + }; + try { + await db.createRepo(testRepo); + } catch (e) { + threwError = true; + } + expect(threwError).to.be.true; }); it('should be able to create a user', async function () { diff --git a/test/testRouteFilter.test.js b/test/testRouteFilter.test.js index 99a133fcb..c5eccedf2 100644 --- a/test/testRouteFilter.test.js +++ b/test/testRouteFilter.test.js @@ -12,13 +12,27 @@ chai.should(); const expect = chai.expect; describe('url helpers and filter functions used in the proxy', function () { - it('processUrlPath should return breakdown of a proxyd path, separating the path to repository from the git operation path', function () { + it('processUrlPath should return breakdown of a proxied path, separating the path to repository from the git operation path', function () { expect( processUrlPath('/github.com/octocat/hello-world.git/info/refs?service=git-upload-pack'), ).to.deep.eq({ repoPath: '/github.com/octocat/hello-world.git', gitPath: '/info/refs?service=git-upload-pack', }); + + expect( + processUrlPath('/gitlab.com/org/sub-org/hello-world.git/info/refs?service=git-upload-pack'), + ).to.deep.eq({ + repoPath: '/gitlab.com/org/sub-org/hello-world.git', + gitPath: '/info/refs?service=git-upload-pack', + }); + + expect( + processUrlPath('/123.456.789/hello-world.git/info/refs?service=git-upload-pack'), + ).to.deep.eq({ + repoPath: '/123.456.789/hello-world.git', + gitPath: '/info/refs?service=git-upload-pack', + }); }); it('processUrlPath should return breakdown of a legacy proxy path, separating the path to repository from the git operation path', function () { @@ -46,11 +60,17 @@ describe('url helpers and filter functions used in the proxy', function () { }); it('processGitUrl should return breakdown of a git URL separating out the protocol, host and repository path', function () { - expect(processGitUrl('https://somegithost.com:1234/octocat/hello-world.git')).to.deep.eq({ + expect(processGitUrl('https://somegithost.com/octocat/hello-world.git')).to.deep.eq({ protocol: 'https://', - host: 'somegithost.com:1234', + host: 'somegithost.com', repoPath: '/octocat/hello-world.git', }); + + expect(processGitUrl('https://123.456.789:1234/hello-world.git')).to.deep.eq({ + protocol: 'https://', + host: '123.456.789:1234', + repoPath: '/hello-world.git', + }); }); it('processGitUrl should return breakdown of a git URL separating out the protocol, host and repository path and discard any git operation path', function () { @@ -63,6 +83,14 @@ describe('url helpers and filter functions used in the proxy', function () { host: 'somegithost.com:1234', repoPath: '/octocat/hello-world.git', }); + + expect( + processGitUrl('https://123.456.789/hello-world.git/info/refs?service=git-upload-pack'), + ).to.deep.eq({ + protocol: 'https://', + host: '123.456.789', + repoPath: '/hello-world.git', + }); }); it('processGitUrl should return null for a url it cannot parse', function () { From 4265fcbef93a26ade5f28f8e14dcfa33139e2df1 Mon Sep 17 00:00:00 2001 From: Kris West Date: Wed, 18 Jun 2025 18:56:52 +0100 Subject: [PATCH 41/76] test: improve coverage for db and repo API --- src/db/file/index.ts | 10 +- src/db/file/users.ts | 18 +++ src/db/index.ts | 14 ++- src/db/mongo/index.ts | 10 +- src/db/mongo/users.ts | 6 + src/db/types.ts | 1 + src/proxy/actions/Action.ts | 9 +- test/testDb.test.js | 109 ++++++++++++++++++ ...ddRepoTest.test.js => testRepoApi.test.js} | 45 ++++++++ 9 files changed, 209 insertions(+), 13 deletions(-) rename test/{addRepoTest.test.js => testRepoApi.test.js} (86%) diff --git a/src/db/file/index.ts b/src/db/file/index.ts index 80276a7af..c41227b84 100644 --- a/src/db/file/index.ts +++ b/src/db/file/index.ts @@ -17,4 +17,12 @@ export const { deleteRepo, } = repo; -export const { findUser, findUserByOIDC, getUsers, createUser, deleteUser, updateUser } = users; +export const { + findUser, + findUserByEmail, + findUserByOIDC, + getUsers, + createUser, + deleteUser, + updateUser, +} = users; diff --git a/src/db/file/users.ts b/src/db/file/users.ts index 25716f10b..9cd3ebd95 100644 --- a/src/db/file/users.ts +++ b/src/db/file/users.ts @@ -35,6 +35,24 @@ export const findUser = (username: string): Promise => { }); }; +export const findUserByEmail = (email: string): Promise => { + return new Promise((resolve, reject) => { + db.findOne({ email: email.toLowerCase() }, (err: Error | null, doc: User) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ + if (err) { + reject(err); + } else { + if (!doc) { + resolve(null); + } else { + resolve(doc); + } + } + }); + }); +}; + export const findUserByOIDC = function (oidcId: string): Promise { return new Promise((resolve, reject) => { db.findOne({ oidcId: oidcId }, (err: Error | null, doc: User) => { diff --git a/src/db/index.ts b/src/db/index.ts index 3d31c800a..5df467a9e 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -44,26 +44,31 @@ export const createUser = async ( }; if (isBlank(username)) { - const errorMessage = `username ${username} cannot be empty`; + const errorMessage = `username cannot be empty`; throw new Error(errorMessage); } if (isBlank(gitAccount)) { - const errorMessage = `GitAccount ${gitAccount} cannot be empty`; + const errorMessage = `gitAccount cannot be empty`; throw new Error(errorMessage); } if (isBlank(email)) { - const errorMessage = `Email ${email} cannot be empty`; + const errorMessage = `email cannot be empty`; throw new Error(errorMessage); } const existingUser = await sink.findUser(username); - if (existingUser) { const errorMessage = `user ${username} already exists`; throw new Error(errorMessage); } + const existingUserWithEmail = await sink.findUserByEmail(email); + if (existingUserWithEmail) { + const errorMessage = `A user with email ${email} already exists`; + throw new Error(errorMessage); + } + await sink.createUser(data); }; @@ -172,6 +177,7 @@ export const removeUserCanAuthorise = (_id: string, user: string): Promise sink.removeUserCanAuthorise(_id, user); export const deleteRepo = (_id: string): Promise => sink.deleteRepo(_id); export const findUser = (username: string): Promise => sink.findUser(username); +export const findUserByEmail = (email: string): Promise => sink.findUserByEmail(email); export const findUserByOIDC = (oidcId: string): Promise => sink.findUserByOIDC(oidcId); export const getUsers = (query?: object): Promise => sink.getUsers(query); export const deleteUser = (username: string): Promise => sink.deleteUser(username); diff --git a/src/db/mongo/index.ts b/src/db/mongo/index.ts index 9b81720ad..0c62e8fea 100644 --- a/src/db/mongo/index.ts +++ b/src/db/mongo/index.ts @@ -20,4 +20,12 @@ export const { deleteRepo, } = repo; -export const { findUser, findUserByOIDC, getUsers, createUser, deleteUser, updateUser } = users; +export const { + findUser, + findUserByEmail, + findUserByOIDC, + getUsers, + createUser, + deleteUser, + updateUser, +} = users; diff --git a/src/db/mongo/users.ts b/src/db/mongo/users.ts index 623bcc9d1..037ecb7c3 100644 --- a/src/db/mongo/users.ts +++ b/src/db/mongo/users.ts @@ -11,6 +11,12 @@ export const findUser = async function (username: string): Promise return doc ? toClass(doc, User.prototype) : null; }; +export const findUserByEmail = async function (email: string): Promise { + const collection = await connect(collectionName); + const doc = collection.findOne({ email: { $eq: email.toLowerCase() } }); + return doc ? toClass(doc, User.prototype) : null; +}; + export const findUserByOIDC = async function (oidcId: string): Promise { const collection = await connect(collectionName); const doc = collection.findOne({ oidcId: { $eq: oidcId } }); diff --git a/src/db/types.ts b/src/db/types.ts index 27f3c198b..59994aad3 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -80,6 +80,7 @@ export interface Sink { removeUserCanAuthorise: (_id: string, user: string) => Promise; deleteRepo: (_id: string) => Promise; findUser: (username: string) => Promise; + findUserByEmail: (email: string) => Promise; findUserByOIDC: (oidcId: string) => Promise; getUsers: (query?: object) => Promise; createUser: (user: User) => Promise; diff --git a/src/proxy/actions/Action.ts b/src/proxy/actions/Action.ts index b9d5e5ed3..678000d7c 100644 --- a/src/proxy/actions/Action.ts +++ b/src/proxy/actions/Action.ts @@ -68,13 +68,8 @@ class Action { if (urlBreakdown) { this.repo = urlBreakdown.repoPath; const repoBreakdown = processGitURLForNameAndOrg(urlBreakdown.repoPath); - if (repoBreakdown) { - this.project = repoBreakdown.project ?? ''; - this.repoName = repoBreakdown.repoName; - } else { - this.project = 'UNKNOWN'; - this.repoName = 'UNKNOWN'; - } + this.project = repoBreakdown?.project ?? ''; + this.repoName = repoBreakdown?.repoName ?? ''; } else { this.repo = 'NOT-FOUND'; this.project = 'UNKNOWN'; diff --git a/test/testDb.test.js b/test/testDb.test.js index a016047a5..446091f12 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -356,6 +356,77 @@ describe('Database clients', async () => { expect(threwError).to.be.true; }); + it('should throw an error when creating a user and username or email is not set', async function () { + // null username + let threwError = false; + let message = null; + try { + await db.createUser( + null, + TEST_USER.password, + TEST_USER.email, + TEST_USER.gitAccount, + TEST_USER.admin, + ); + } catch (e) { + threwError = true; + message = e.message; + } + expect(threwError).to.be.true; + expect(message).to.equal('username cannot be empty'); + + // blank username + threwError = false; + try { + await db.createUser( + '', + TEST_USER.password, + TEST_USER.email, + TEST_USER.gitAccount, + TEST_USER.admin, + ); + } catch (e) { + threwError = true; + message = e.message; + } + expect(threwError).to.be.true; + expect(message).to.equal('username cannot be empty'); + + // null email + threwError = false; + try { + await db.createUser( + TEST_USER.username, + TEST_USER.password, + null, + TEST_USER.gitAccount, + TEST_USER.admin, + ); + } catch (e) { + threwError = true; + message = e.message; + } + expect(threwError).to.be.true; + expect(message).to.equal('email cannot be empty'); + + // blank username + threwError = false; + try { + await db.createUser( + TEST_USER.username, + TEST_USER.password, + '', + TEST_USER.gitAccount, + TEST_USER.admin, + ); + } catch (e) { + threwError = true; + message = e.message; + } + expect(threwError).to.be.true; + expect(message).to.equal('email cannot be empty'); + }); + it('should be able to create a user', async function () { await db.createUser( TEST_USER.username, @@ -372,6 +443,44 @@ describe('Database clients', async () => { expect(cleanUsers).to.deep.include(TEST_USER_CLEAN); }); + it('should throw an error when creating a duplicate username', async function () { + let threwError = false; + let message = null; + try { + await db.createUser( + TEST_USER.username, + TEST_USER.password, + 'prefix_' + TEST_USER.email, + TEST_USER.gitAccount, + TEST_USER.admin, + ); + } catch (e) { + threwError = true; + message = e.message; + } + expect(threwError).to.be.true; + expect(message).to.equal(`user ${TEST_USER.username} already exists`); + }); + + it('should throw an error when creating a user with a duplicate email', async function () { + let threwError = false; + let message = null; + try { + await db.createUser( + 'prefix_' + TEST_USER.username, + TEST_USER.password, + TEST_USER.email, + TEST_USER.gitAccount, + TEST_USER.admin, + ); + } catch (e) { + threwError = true; + message = e.message; + } + expect(threwError).to.be.true; + expect(message).to.equal(`A user with email ${TEST_USER.email} already exists`); + }); + it('should be able to find a user', async function () { const user = await db.findUser(TEST_USER.username); // eslint-disable-next-line no-unused-vars diff --git a/test/addRepoTest.test.js b/test/testRepoApi.test.js similarity index 86% rename from test/addRepoTest.test.js rename to test/testRepoApi.test.js index 765f575c5..ca81bf9ef 100644 --- a/test/addRepoTest.test.js +++ b/test/testRepoApi.test.js @@ -93,6 +93,29 @@ describe('add new repo', async () => { repo.users.canAuthorise.length.should.equal(0); }); + it('get a repo', async function () { + const res = await chai + .request(app) + .get('/api/v1/repo/' + repoIds[0]) + .set('Cookie', `${cookie}`) + .send(); + res.should.have.status(200); + + expect(res.body.url).to.equal(TEST_REPO.url); + expect(res.body.name).to.equal(TEST_REPO.name); + expect(res.body.project).to.equal(TEST_REPO.project); + }); + + it('return a 409 error if the repo already exists', async function () { + const res = await chai + .request(app) + .post('/api/v1/repo') + .set('Cookie', `${cookie}`) + .send(TEST_REPO); + res.should.have.status(409); + res.body.message.should.equal('Repository ' + TEST_REPO.url + ' already exists!'); + }); + it('filter repos', async function () { const res = await chai .request(app) @@ -277,6 +300,28 @@ describe('add new repo', async () => { ]); }); + it('delete a repo', async function () { + const res = await chai + .request(app) + .delete('/api/v1/repo/' + repoIds[1] + '/delete') + .set('Cookie', `${cookie}`) + .send(); + res.should.have.status(200); + + const repo = await db.getRepoByUrl(TEST_REPO_NON_GITHUB.url); + expect(repo).to.be.null; + + const res2 = await chai + .request(app) + .delete('/api/v1/repo/' + repoIds[2] + '/delete') + .set('Cookie', `${cookie}`) + .send(); + res2.should.have.status(200); + + const repo2 = await db.getRepoByUrl(TEST_REPO_NAKED.url); + expect(repo2).to.be.null; + }); + after(async function () { await service.httpServer.close(); From 662830c7cdaf446aeaa43f6114fc547f4e66af1c Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 20 Jun 2025 10:25:59 +0100 Subject: [PATCH 42/76] test: proxy route filter tests --- src/db/file/repo.ts | 4 +- src/db/types.ts | 2 +- src/proxy/routes/index.ts | 2 +- test/testProxyRoute.test.js | 122 ++++++++++++++++++++++++++++++++++-- 4 files changed, 122 insertions(+), 8 deletions(-) diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index f68229b68..451d540ef 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -104,11 +104,11 @@ export const addUserCanPush = async (_id: string, user: string): Promise = return; } - if (repo.users.canPush.includes(user)) { + if (repo.users?.canPush.includes(user)) { resolve(); return; } - repo.users.canPush.push(user); + repo.users?.canPush.push(user); const options = { multi: false, upsert: false }; db.update({ _id: _id }, repo, options, (err) => { diff --git a/src/db/types.ts b/src/db/types.ts index 59994aad3..2d6d81725 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -14,7 +14,7 @@ export class Repo { project: string; name: string; url: string; - users: { canPush: string[]; canAuthorise: string[] } | null; + users: { canPush: string[]; canAuthorise: string[] }; _id?: string; constructor( diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index 126140e3f..cbf6d8484 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -141,4 +141,4 @@ const getRouter = async () => { return router; }; -export { getRouter, handleMessage, validGitRequest }; +export { proxyFilter, getRouter, handleMessage, validGitRequest }; diff --git a/test/testProxyRoute.test.js b/test/testProxyRoute.test.js index e778f65ee..2faf68a16 100644 --- a/test/testProxyRoute.test.js +++ b/test/testProxyRoute.test.js @@ -1,14 +1,15 @@ const { handleMessage, validGitRequest, stripGitHubFromGitPath } = require('../src/proxy/routes'); const chai = require('chai'); const chaiHttp = require('chai-http'); +chai.use(chaiHttp); +chai.should(); +const expect = chai.expect; const sinon = require('sinon'); const express = require('express'); const proxyRouter = require('../src/proxy/routes').router; const chain = require('../src/proxy/chain'); - -chai.use(chaiHttp); -chai.should(); -const expect = chai.expect; +const proxyquire = require('proxyquire'); +const { Action, Step } = require('../src/proxy/actions'); describe('proxy route filter middleware', () => { let app; @@ -201,3 +202,116 @@ describe('proxy route helpers', () => { }); }); }); + +describe('proxy route filter', async () => { + let proxyRoutes; + let req; + let res; + let actionToReturn; + let executeChainStub; + + beforeEach(async () => { + executeChainStub = sinon.stub(); + + // Re-import the proxy routes module and stub executeChain + proxyRoutes = proxyquire('../src/proxy/routes', { + '../chain': { executeChain: executeChainStub }, + }); + + req = { + url: '/github.com/finos/git-proxy.git/info/refs?service=git-receive-pack', + headers: { + host: 'dummyHost', + 'user-agent': 'git/dummy-git-client', + accept: 'application/x-git-receive-pack-request', + }, + }; + res = { + set: () => {}, + status: () => { + return { + send: () => {}, + }; + }, + }; + }); + + it('should return false for push requests that should be blocked', async function () { + // mock the executeChain function + actionToReturn = new Action( + 1234, + 'dummy', + 'dummy', + Date.now(), + '/github.com/finos/git-proxy.git', + ); + const step = new Step('dummy', false, null, true, 'test block', null); + actionToReturn.addStep(step); + executeChainStub.returns(actionToReturn); + const result = await proxyRoutes.proxyFilter(req, res); + expect(result).to.be.false; + }); + + it('should return false for push requests that produced errors', async function () { + // mock the executeChain function + actionToReturn = new Action( + 1234, + 'dummy', + 'dummy', + Date.now(), + '/github.com/finos/git-proxy.git', + ); + const step = new Step('dummy', true, 'test error', false, null, null); + actionToReturn.addStep(step); + executeChainStub.returns(actionToReturn); + const result = await proxyRoutes.proxyFilter(req, res); + expect(result).to.be.false; + }); + + it('should return false for invalid push requests', async function () { + // mock the executeChain function + actionToReturn = new Action( + 1234, + 'dummy', + 'dummy', + Date.now(), + '/github.com/finos/git-proxy.git', + ); + const step = new Step('dummy', true, 'test error', false, null, null); + actionToReturn.addStep(step); + executeChainStub.returns(actionToReturn); + + // create an invalid request + req = { + url: '/github.com/finos/git-proxy.git/invalidPath', + headers: { + host: 'dummyHost', + 'user-agent': 'git/dummy-git-client', + accept: 'application/x-git-receive-pack-request', + }, + }; + + const result = await proxyRoutes.proxyFilter(req, res); + expect(result).to.be.false; + }); + + it('should return true for push requests that are valid and pass the chain', async function () { + // mock the executeChain function + actionToReturn = new Action( + 1234, + 'dummy', + 'dummy', + Date.now(), + '/github.com/finos/git-proxy.git', + ); + const step = new Step('dummy', false, null, false, null, null); + actionToReturn.addStep(step); + executeChainStub.returns(actionToReturn); + const result = await proxyRoutes.proxyFilter(req, res); + expect(result).to.be.true; + }); + + afterEach(() => { + sinon.restore(); + }); +}); From 60a2c37a3b6d45ebaf6a2c3c2a2cd59a2361e073 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 24 Jun 2025 08:16:04 +0100 Subject: [PATCH 43/76] fix: correct comparison of origins and import of proxy class in repo API route --- src/service/routes/repo.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/service/routes/repo.js b/src/service/routes/repo.js index 3c96b73b5..92145fefb 100644 --- a/src/service/routes/repo.js +++ b/src/service/routes/repo.js @@ -144,7 +144,8 @@ router.post('/', async (req, res) => { const existingHosts = await getAllProxiedHosts(); existingHosts.forEach((h) => { - if (req.body.url.startsWith(h)) { + // assume SSL is in use and that our origins are missing the protocol + if (req.body.url.startsWith(`https://${h}`)) { newOrigin = false; } }); @@ -162,7 +163,7 @@ router.post('/', async (req, res) => { console.log('Restarting the proxy to handle an additional origin'); // 1. Get proxy module dynamically to avoid circular dependency - const { proxy } = require('../index'); + const { proxy } = require('../../proxy'); // 2. Stop existing services await proxy.stop(); From 92b801b2d2e7a065d4c567d71cf0af6dca7f6abf Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 4 Jul 2025 17:25:25 +0100 Subject: [PATCH 44/76] fix: remove Transfer-encoding from proxy blocked response As it causes parse errors where Cntent-Length is also set --- src/proxy/routes/helper.ts | 6 +++++ src/proxy/routes/index.ts | 1 - test/testProxyRoute.test.js | 47 ++++--------------------------------- 3 files changed, 11 insertions(+), 43 deletions(-) diff --git a/src/proxy/routes/helper.ts b/src/proxy/routes/helper.ts index cf47fdcbb..a2528e528 100644 --- a/src/proxy/routes/helper.ts +++ b/src/proxy/routes/helper.ts @@ -136,6 +136,9 @@ export const processGitURLForNameAndOrg = (gitUrl: string): GitNameBreakdown | n */ export const validGitRequest = (gitPath: string, headers: any): boolean => { const { 'user-agent': agent, accept } = headers; + if (!agent) { + return false; + } if ( ['/info/refs?service=git-upload-pack', '/info/refs?service=git-receive-pack'].includes(gitPath) ) { @@ -145,6 +148,9 @@ export const validGitRequest = (gitPath: string, headers: any): boolean => { return agent.startsWith('git/'); } if (['/git-upload-pack', '/git-receive-pack'].includes(gitPath)) { + if (!accept) { + return false; + } // https://www.git-scm.com/docs/http-protocol#_uploading_data return agent.startsWith('git/') && accept.startsWith('application/x-git-'); } diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index cbf6d8484..4944249a0 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -27,7 +27,6 @@ const proxyFilter: ProxyOptions['filter'] = async (req, res) => { if (action.error || action.blocked) { res.set('content-type', 'application/x-git-receive-pack-result'); - res.set('transfer-encoding', 'chunked'); res.set('expires', 'Fri, 01 Jan 1980 00:00:00 GMT'); res.set('pragma', 'no-cache'); res.set('cache-control', 'no-cache, max-age=0, must-revalidate'); diff --git a/test/testProxyRoute.test.js b/test/testProxyRoute.test.js index 2faf68a16..fc0aa1d66 100644 --- a/test/testProxyRoute.test.js +++ b/test/testProxyRoute.test.js @@ -1,4 +1,4 @@ -const { handleMessage, validGitRequest, stripGitHubFromGitPath } = require('../src/proxy/routes'); +const { handleMessage, validGitRequest } = require('../src/proxy/routes'); const chai = require('chai'); const chaiHttp = require('chai-http'); chai.use(chaiHttp); @@ -6,7 +6,7 @@ chai.should(); const expect = chai.expect; const sinon = require('sinon'); const express = require('express'); -const proxyRouter = require('../src/proxy/routes').router; +const getRouter = require('../src/proxy/routes').getRouter; const chain = require('../src/proxy/chain'); const proxyquire = require('proxyquire'); const { Action, Step } = require('../src/proxy/actions'); @@ -14,9 +14,9 @@ const { Action, Step } = require('../src/proxy/actions'); describe('proxy route filter middleware', () => { let app; - beforeEach(() => { + beforeEach(async () => { app = express(); - app.use('/', proxyRouter); + app.use('/', await getRouter()); }); afterEach(() => { @@ -164,46 +164,9 @@ describe('proxy route helpers', () => { expect(res).to.be.false; }); }); - - describe('stripGitHubFromGitPath', () => { - it('should strip owner and repo from a valid GitHub-style path with 4 parts', () => { - const res = stripGitHubFromGitPath('/foo/bar.git/info/refs'); - expect(res).to.equal('/info/refs'); - }); - - it('should strip owner and repo from a valid GitHub-style path with 5 parts', () => { - const res = stripGitHubFromGitPath('/foo/bar.git/git-upload-pack'); - expect(res).to.equal('/git-upload-pack'); - }); - - it('should return undefined for malformed path with too few segments', () => { - const res = stripGitHubFromGitPath('/foo/bar.git'); - expect(res).to.be.undefined; - }); - - it('should return undefined for malformed path with too many segments', () => { - const res = stripGitHubFromGitPath('/foo/bar.git/extra/path/stuff'); - expect(res).to.be.undefined; - }); - - it('should handle repo names that include dots correctly', () => { - const res = stripGitHubFromGitPath('/foo/some.repo.git/info/refs'); - expect(res).to.equal('/info/refs'); - }); - - it('should not break if the path is just a slash', () => { - const res = stripGitHubFromGitPath('/'); - expect(res).to.be.undefined; - }); - - it('should not break if the path is empty', () => { - const res = stripGitHubFromGitPath(''); - expect(res).to.be.undefined; - }); - }); }); -describe('proxy route filter', async () => { +describe('proxyFilter function', async () => { let proxyRoutes; let req; let res; From fe934dfa0f703d04328b5648093c8abd1db2efb5 Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 4 Jul 2025 18:40:33 +0100 Subject: [PATCH 45/76] fix: log warnign on startup if proxyUrl is in the config --- src/config/index.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/config/index.ts b/src/config/index.ts index 88f5777c6..d9dd56397 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -16,6 +16,13 @@ import { let _userSettings: UserSettings | null = null; if (existsSync(configFile)) { _userSettings = JSON.parse(readFileSync(configFile, 'utf-8')); + + // print warnings about deprecated config + if (_userSettings && _userSettings.proxyUrl) { + console.log( + 'Warning: the proxyUrl is no longer used (proxy origins are extracted from the repository URLs) and should be removed from your configuration', + ); + } } let _authorisedList: AuthorisedRepo[] = defaultSettings.authorisedList; let _database: Database[] = defaultSettings.sink; From e059206bc9babcf74cb1c0cfa1c48ee8c2349ff1 Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 4 Jul 2025 18:54:36 +0100 Subject: [PATCH 46/76] fix: catch errors on building unique DB indices --- src/db/file/pushes.ts | 9 ++++++++- src/db/file/repo.ts | 10 +++++++++- src/db/file/users.ts | 18 ++++++++++++++++-- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/db/file/pushes.ts b/src/db/file/pushes.ts index 38a3336f6..69ec3087a 100644 --- a/src/db/file/pushes.ts +++ b/src/db/file/pushes.ts @@ -14,7 +14,14 @@ if (!fs.existsSync('./.data')) fs.mkdirSync('./.data'); if (!fs.existsSync('./.data/db')) fs.mkdirSync('./.data/db'); const db = new Datastore({ filename: './.data/db/pushes.db', autoload: true }); -db.ensureIndex({ fieldName: 'id', unique: true }); +try { + db.ensureIndex({ fieldName: 'id', unique: true }); +} catch (e) { + console.error( + 'Failed to build a unique index of push id values. Please check your database file for duplicate entries or delete the duplicate through the UI and restart. ', + e, + ); +} db.setAutocompactionInterval(COMPACTION_INTERVAL); const defaultPushQuery: PushQuery = { diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index 451d540ef..584339f82 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -14,7 +14,15 @@ if (!fs.existsSync('./.data/db')) fs.mkdirSync('./.data/db'); const db = new Datastore({ filename: './.data/db/repos.db', autoload: true }); -db.ensureIndex({ fieldName: 'url', unique: true }); +try { + db.ensureIndex({ fieldName: 'url', unique: true }); +} catch (e) { + console.error( + 'Failed to build a unique index of Repository URLs. Please check your database file for duplicate entries or delete the duplicate through the UI and restart. ', + e, + ); +} + db.ensureIndex({ fieldName: 'name', unique: false }); db.setAutocompactionInterval(COMPACTION_INTERVAL); diff --git a/src/db/file/users.ts b/src/db/file/users.ts index 9cd3ebd95..e449f7ff2 100644 --- a/src/db/file/users.ts +++ b/src/db/file/users.ts @@ -13,8 +13,22 @@ if (!fs.existsSync('./.data/db')) fs.mkdirSync('./.data/db'); const db = new Datastore({ filename: './.data/db/users.db', autoload: true }); // Using a unique constraint with the index -db.ensureIndex({ fieldName: 'username', unique: true }); -db.ensureIndex({ fieldName: 'email', unique: true }); +try { + db.ensureIndex({ fieldName: 'username', unique: true }); +} catch (e) { + console.error( + 'Failed to build a unique index of usernames. Please check your database file for duplicate entries or delete the duplicate through the UI and restart. ', + e, + ); +} +try { + db.ensureIndex({ fieldName: 'email', unique: true }); +} catch (e) { + console.error( + 'Failed to build a unique index of user email addresses. Please check your database file for duplicate entries or delete the duplicate through the UI and restart. ', + e, + ); +} db.setAutocompactionInterval(COMPACTION_INTERVAL); export const findUser = (username: string): Promise => { From a462ed6a62b5450b6e5a32f41f3a4df02fbb1660 Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 4 Jul 2025 19:08:57 +0100 Subject: [PATCH 47/76] fix: control max length of urls processed to prevent DoS --- src/proxy/routes/helper.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/proxy/routes/helper.ts b/src/proxy/routes/helper.ts index a2528e528..954d3a813 100644 --- a/src/proxy/routes/helper.ts +++ b/src/proxy/routes/helper.ts @@ -26,6 +26,10 @@ export type GitUrlBreakdown = { protocol: string; host: string; repoPath: string * @return {GitUrlBreakdown | null} A breakdown of the components of the URL. */ export const processGitUrl = (url: string): GitUrlBreakdown | null => { + if (url.length > 512) { + console.error(`The git URL is too long: ${url}`); + return null; + } const components = url.match(GIT_URL_REGEX); if (components && components.length >= 5) { return { @@ -65,6 +69,10 @@ export type UrlPathBreakdown = { repoPath: string; gitPath: string }; * @return {GitUrlBreakdown | null} A breakdown of the components of the URL path. */ export const processUrlPath = (requestPath: string): UrlPathBreakdown | null => { + if (requestPath.length > 512) { + console.error(`The requestPath is too long: ${requestPath}`); + return null; + } const components = requestPath.match(PROXIED_URL_PATH_REGEX); if (components && components.length >= 3) { return { @@ -111,6 +119,10 @@ export type GitNameBreakdown = { project: string | null; repoName: string }; * @return {GitNameBreakdown | null} A breakdown of the components of the URL. */ export const processGitURLForNameAndOrg = (gitUrl: string): GitNameBreakdown | null => { + if (gitUrl.length > 512) { + console.error(`The git URL is too long: ${gitUrl}`); + return null; + } const components = gitUrl.match(GIT_URL_NAME_ORG_REGEX); if (components && components.length >= 5) { return { From 7ab7df21957425a428023d966176f6c42f2dc355 Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 4 Jul 2025 19:17:21 +0100 Subject: [PATCH 48/76] test: test coverage for path length limits --- test/testRouteFilter.test.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/testRouteFilter.test.js b/test/testRouteFilter.test.js index c5eccedf2..88a9cfed1 100644 --- a/test/testRouteFilter.test.js +++ b/test/testRouteFilter.test.js @@ -11,6 +11,9 @@ chai.should(); const expect = chai.expect; +const VERY_LONG_PATH = + '/a/very/very/very/very/very//very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/very/long/path'; + describe('url helpers and filter functions used in the proxy', function () { it('processUrlPath should return breakdown of a proxied path, separating the path to repository from the git operation path', function () { expect( @@ -57,6 +60,7 @@ describe('url helpers and filter functions used in the proxy', function () { it("processUrlPath should return null if the url couldn't be parsed", function () { expect(processUrlPath('/octocat/hello-world')).to.be.null; + expect(processUrlPath(VERY_LONG_PATH)).to.be.null; }); it('processGitUrl should return breakdown of a git URL separating out the protocol, host and repository path', function () { @@ -95,6 +99,7 @@ describe('url helpers and filter functions used in the proxy', function () { it('processGitUrl should return null for a url it cannot parse', function () { expect(processGitUrl('somegithost.com:1234/octocat/hello-world.git')).to.be.null; + expect(processUrlPath('somegithost.com:1234' + VERY_LONG_PATH + '.git')).to.be.null; }); it('processGitURLForNameAndOrg should return breakdown of a git URL path separating out the protocol, origin and repository path', function () { @@ -111,6 +116,13 @@ describe('url helpers and filter functions used in the proxy', function () { }); }); + it("processGitURLForNameAndOrg should return null for a git repository URL it can't pass", function () { + expect(processGitURLForNameAndOrg('someGitHost.com/repo')).to.be.null; + expect(processGitURLForNameAndOrg('https://someGitHost.com/repo')).to.be.null; + expect(processGitURLForNameAndOrg('https://somegithost.com:1234' + VERY_LONG_PATH + '.git')).to + .be.null; + }); + it('validGitRequest should return true for safe requests on expected URLs', function () { [ '/info/refs?service=git-upload-pack', From 9a1e653e15f06c5abd03456bc786079e2da1c498 Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 4 Jul 2025 19:25:22 +0100 Subject: [PATCH 49/76] fix: typing issue in db client --- src/db/file/pushes.ts | 3 ++- src/db/index.ts | 3 ++- src/db/mongo/pushes.ts | 3 ++- src/db/types.ts | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/db/file/pushes.ts b/src/db/file/pushes.ts index 69ec3087a..10cc2a4fd 100644 --- a/src/db/file/pushes.ts +++ b/src/db/file/pushes.ts @@ -111,7 +111,7 @@ export const authorise = async (id: string, attestation: any): Promise<{ message return { message: `authorised ${id}` }; }; -export const reject = async (id: string): Promise<{ message: string }> => { +export const reject = async (id: string, attestation: any): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -120,6 +120,7 @@ export const reject = async (id: string): Promise<{ message: string }> => { action.authorised = false; action.canceled = false; action.rejected = true; + action.attestation = attestation; await writeAudit(action); return { message: `reject ${id}` }; }; diff --git a/src/db/index.ts b/src/db/index.ts index 5df467a9e..062094492 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -162,7 +162,8 @@ export const deletePush = (id: string): Promise => sink.deletePush(id); export const authorise = (id: string, attestation: any): Promise<{ message: string }> => sink.authorise(id, attestation); export const cancel = (id: string): Promise<{ message: string }> => sink.cancel(id); -export const reject = (id: string): Promise<{ message: string }> => sink.reject(id); +export const reject = (id: string, attestation: any): Promise<{ message: string }> => + sink.reject(id, attestation); export const getRepos = (query?: object): Promise => sink.getRepos(query); export const getRepo = (name: string): Promise => sink.getRepo(name); export const getRepoByUrl = (url: string): Promise => sink.getRepoByUrl(url); diff --git a/src/db/mongo/pushes.ts b/src/db/mongo/pushes.ts index c64325755..e1b3a4bbe 100644 --- a/src/db/mongo/pushes.ts +++ b/src/db/mongo/pushes.ts @@ -74,7 +74,7 @@ export const authorise = async (id: string, attestation: any): Promise<{ message return { message: `authorised ${id}` }; }; -export const reject = async (id: string): Promise<{ message: string }> => { +export const reject = async (id: string, attestation: any): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -82,6 +82,7 @@ export const reject = async (id: string): Promise<{ message: string }> => { action.authorised = false; action.canceled = false; action.rejected = true; + action.attestation = attestation; await writeAudit(action); return { message: `reject ${id}` }; }; diff --git a/src/db/types.ts b/src/db/types.ts index 2d6d81725..d95c352e0 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -68,7 +68,7 @@ export interface Sink { deletePush: (id: string) => Promise; authorise: (id: string, attestation: any) => Promise<{ message: string }>; cancel: (id: string) => Promise<{ message: string }>; - reject: (id: string) => Promise<{ message: string }>; + reject: (id: string, attestation: any) => Promise<{ message: string }>; getRepos: (query?: object) => Promise; getRepo: (name: string) => Promise; getRepoByUrl: (url: string) => Promise; From 6ad79ab26e5dde532e14cf363835ce67e2e6c317 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 8 Jul 2025 02:19:50 +0100 Subject: [PATCH 50/76] fix: fix issues with restart of the proxy by passing service a reference to proxy --- index.ts | 5 +- src/proxy/index.ts | 139 +++++++-------- src/proxy/routes/index.ts | 59 ++++--- src/service/index.js | 39 +++-- src/service/routes/index.js | 21 ++- src/service/routes/repo.js | 332 +++++++++++++++++++----------------- test/1.test.js | 2 +- test/testProxyRoute.test.js | 156 +++++++++++++++++ test/testRepoApi.test.js | 6 +- 9 files changed, 488 insertions(+), 271 deletions(-) diff --git a/index.ts b/index.ts index 7fdcc4da9..bdbd9a694 100755 --- a/index.ts +++ b/index.ts @@ -4,7 +4,7 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import * as fs from 'fs'; import { configFile, setConfigFile, validate } from './src/config/file'; -import proxy from './src/proxy'; +import Proxy from './src/proxy'; import service from './src/service'; const argv = yargs(hideBin(process.argv)) @@ -45,7 +45,8 @@ if (argv.v) { validate(); +const proxy = new Proxy(); proxy.start(); -service.start(); +service.start(proxy); export { proxy, service }; diff --git a/src/proxy/index.ts b/src/proxy/index.ts index b81def508..07795a640 100644 --- a/src/proxy/index.ts +++ b/src/proxy/index.ts @@ -1,4 +1,4 @@ -import express, { Application } from 'express'; +import express, { Express } from 'express'; import http from 'http'; import https from 'https'; import fs from 'fs'; @@ -34,79 +34,82 @@ const options: ServerOptions = { cert: getTLSEnabled() ? fs.readFileSync(getTLSCertPemPath()) : undefined, }; -export const proxyPreparations = async () => { - const plugins = getPlugins(); - const pluginLoader = new PluginLoader(plugins); - await pluginLoader.load(); - chain.chainPluginLoader = pluginLoader; - // Check to see if the default repos are in the repo list - const defaultAuthorisedRepoList = getAuthorisedList(); - const allowedList: Repo[] = await getRepos(); +export default class Proxy { + private httpServer: http.Server | null = null; + private httpsServer: https.Server | null = null; + private expressApp: Express | null = null; - defaultAuthorisedRepoList.forEach(async (x) => { - const found = allowedList.find((y) => y.project === x.project && x.name === y.name); - if (!found) { - const repo = await createRepo(x); - await addUserCanPush(repo._id!, 'admin'); - await addUserCanAuthorise(repo._id!, 'admin'); - } - }); -}; - -let httpServer: http.Server | null = null; -let httpsServer: https.Server | null = null; + constructor() {} -const createApp = async (): Promise => { - const app = express(); - const router = await getRouter(); - app.use('/', router); - return app; -}; + private async proxyPreparations() { + const plugins = getPlugins(); + const pluginLoader = new PluginLoader(plugins); + await pluginLoader.load(); + chain.chainPluginLoader = pluginLoader; + // Check to see if the default repos are in the repo list + const defaultAuthorisedRepoList = getAuthorisedList(); + const allowedList: Repo[] = await getRepos(); -const start = async (): Promise => { - const app = await createApp(); - await proxyPreparations(); - httpServer = http.createServer(options as any, app).listen(proxyHttpPort, () => { - console.log(`HTTP Proxy Listening on ${proxyHttpPort}`); - }); - // Start HTTPS server only if TLS is enabled - if (getTLSEnabled()) { - httpsServer = https.createServer(options, app).listen(proxyHttpsPort, () => { - console.log(`HTTPS Proxy Listening on ${proxyHttpsPort}`); + defaultAuthorisedRepoList.forEach(async (x) => { + const found = allowedList.find((y) => y.project === x.project && x.name === y.name); + if (!found) { + const repo = await createRepo(x); + await addUserCanPush(repo._id!, 'admin'); + await addUserCanAuthorise(repo._id!, 'admin'); + } }); } - return app; -}; -const stop = (): Promise => { - return new Promise((resolve, reject) => { - try { - // Close HTTP server if it exists - if (httpServer) { - httpServer.close(() => { - console.log('HTTP server closed'); - httpServer = null; - }); - } - - // Close HTTPS server if it exists - if (httpsServer) { - httpsServer.close(() => { - console.log('HTTPS server closed'); - httpsServer = null; - }); - } + private async createApp() { + const app = express(); + const router = await getRouter(); + app.use('/', router); + return app; + } - resolve(); - } catch (error) { - reject(error); + public async start() { + this.expressApp = await this.createApp(); + await this.proxyPreparations(); + this.httpServer = http + .createServer(options as any, this.expressApp) + .listen(proxyHttpPort, () => { + console.log(`HTTP Proxy Listening on ${proxyHttpPort}`); + }); + // Start HTTPS server only if TLS is enabled + if (getTLSEnabled()) { + this.httpsServer = https.createServer(options, this.expressApp).listen(proxyHttpsPort, () => { + console.log(`HTTPS Proxy Listening on ${proxyHttpsPort}`); + }); } - }); -}; + } -export default { - proxyPreparations, - createApp, - start, - stop, -}; + public getExpressApp() { + return this.expressApp; + } + + public stop(): Promise { + return new Promise((resolve, reject) => { + try { + // Close HTTP server if it exists + if (this.httpServer) { + this.httpServer.close(() => { + console.log('HTTP server closed'); + this.httpServer = null; + }); + } + + // Close HTTPS server if it exists + if (this.httpsServer) { + this.httpsServer.close(() => { + console.log('HTTPS server closed'); + this.httpsServer = null; + }); + } + + resolve(); + } catch (error) { + reject(error); + } + }); + } +} diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index 9aac442ad..94e6d9545 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -1,4 +1,4 @@ -import { Router, Request, Response, NextFunction } from 'express'; +import { Router, Request, Response, NextFunction, RequestHandler } from 'express'; import proxy from 'express-http-proxy'; import { PassThrough } from 'stream'; import getRawBody from 'raw-body'; @@ -159,15 +159,19 @@ const getRouter = async () => { router.use(teeAndValidate); const originsToProxy = await getAllProxiedHosts(); - console.log( - `Initializing proxy router for origins: '${JSON.stringify(originsToProxy, null, 2)}'`, - ); - // Middlewares are processed in the order that they are added, if one applies and then doesn't call `next` then subsequent ones are not applied. - // Hence, we define known origins first, then a catch all route for backwards compatibility + const proxyKeys: string[] = []; + const proxies: RequestHandler[] = []; + + console.log(`Initializing proxy router for origins: '${JSON.stringify(originsToProxy)}'`); + + // we need to wrap multiple proxy middlewares in a custom middleware as middlewares + // with path are processed in descending path order (/ then /github.com etc.) and + // we want the fallback proxy to go last. originsToProxy.forEach((origin) => { console.log(`\tsetting up origin: '${origin}'`); - router.use( - '/' + origin, + + proxyKeys.push(`/${origin}/`); + proxies.push( proxy('https://' + origin, { parseReqBody: false, preserveHostHdr: false, @@ -180,20 +184,33 @@ const getRouter = async () => { ); }); - // Catch-all route for backwards compatibility console.log('\tsetting up catch-all route (github.com) for backwards compatibility'); - router.use( - '/', - proxy('https://github.com', { - parseReqBody: false, - preserveHostHdr: false, - filter: proxyFilter, - proxyReqPathResolver: getRequestPathResolver('https://github.com'), - proxyReqOptDecorator: proxyReqOptDecorator, - proxyReqBodyDecorator: proxyReqBodyDecorator, - proxyErrorHandler: proxyErrorHandler, - }), - ); + const fallbackProxy: RequestHandler = proxy('https://github.com', { + parseReqBody: false, + preserveHostHdr: false, + filter: proxyFilter, + proxyReqPathResolver: getRequestPathResolver('https://github.com'), + proxyReqOptDecorator: proxyReqOptDecorator, + proxyReqBodyDecorator: proxyReqBodyDecorator, + proxyErrorHandler: proxyErrorHandler, + }); + + console.log('proxy keys registered: ', JSON.stringify(proxyKeys)); + + router.use('/', (req, res, next) => { + console.log(`processing request URL: '${req.url}'`); + console.log('proxy keys registered: ', JSON.stringify(proxyKeys)); + + for (let i = 0; i < proxyKeys.length; i++) { + if (req.url.startsWith(proxyKeys[i])) { + console.log(`\tusing proxy ${proxyKeys[i]}`); + return proxies[i](req, res, next); + } + } + // fallback + console.log(`\tusing fallback`); + return fallbackProxy(req, res, next); + }); return router; }; diff --git a/src/service/index.js b/src/service/index.js index 02e416aa0..f03d75b68 100644 --- a/src/service/index.js +++ b/src/service/index.js @@ -9,7 +9,6 @@ const db = require('../db'); const rateLimit = require('express-rate-limit'); const lusca = require('lusca'); const configLoader = require('../config/ConfigLoader'); -const proxy = require('../proxy'); const limiter = rateLimit(config.getRateLimit()); @@ -22,7 +21,12 @@ const corsOptions = { origin: true, }; -const createApp = async () => { +/** + * Internal function used to bootstrap the Git Proxy API's express application. + * @param {proxy} proxy A reference to the proxy express application, used to restart it when necessary. + * @return {Promise} + */ +async function createApp(proxy) { // configuration of passport is async // Before we can bind the routes - we need the passport strategy const passport = await require('./passport').configure(); @@ -98,17 +102,26 @@ const createApp = async () => { app.use(passport.session()); app.use(express.json()); app.use(express.urlencoded({ extended: true })); - app.use('/', routes); + app.use('/', routes(proxy)); app.use('/', express.static(absBuildPath)); app.get('/*', (req, res) => { res.sendFile(path.join(`${absBuildPath}/index.html`)); }); return app; -}; +} + +/** + * Starts the proxy service. + * @param {proxy?} proxy A reference to the proxy express application, used to restart it when necessary. + * @return {Promise} the express application (used for testing). + */ +async function start(proxy) { + if (!proxy) { + console.warn("WARNING: proxy is null and can't be controlled by the API service"); + } -const start = async () => { - const app = await createApp(); + const app = await createApp(proxy); _httpServer.listen(uiPort); @@ -116,8 +129,14 @@ const start = async () => { app.emit('ready'); return app; -}; +} + +/** + * Stops the proxy service. + */ +async function stop() { + console.log(`Stopping Service Listening on ${uiPort}`); + _httpServer.close(); +} -module.exports.createApp = createApp; -module.exports.start = start; -module.exports.httpServer = _httpServer; +module.exports = { start, stop, httpServer: _httpServer }; diff --git a/src/service/routes/index.js b/src/service/routes/index.js index 45b276c17..5af032a8d 100644 --- a/src/service/routes/index.js +++ b/src/service/routes/index.js @@ -7,14 +7,17 @@ const users = require('./users'); const healthcheck = require('./healthcheck'); const config = require('./config'); const jwtAuthHandler = require('../passport/jwtAuthHandler'); -const router = new express.Router(); -router.use('/api', home); -router.use('/api/auth', auth); -router.use('/api/v1/healthcheck', healthcheck); -router.use('/api/v1/push', jwtAuthHandler(), push); -router.use('/api/v1/repo', jwtAuthHandler(), repo); -router.use('/api/v1/user', jwtAuthHandler(), users); -router.use('/api/v1/config', config); +const routes = (proxy) => { + const router = new express.Router(); + router.use('/api', home); + router.use('/api/auth', auth); + router.use('/api/v1/healthcheck', healthcheck); + router.use('/api/v1/push', jwtAuthHandler(), push); + router.use('/api/v1/repo', jwtAuthHandler(), repo(proxy)); + router.use('/api/v1/user', jwtAuthHandler(), users); + router.use('/api/v1/config', config); + return router; +}; -module.exports = router; +module.exports = routes; diff --git a/src/service/routes/repo.js b/src/service/routes/repo.js index 92145fefb..03c307717 100644 --- a/src/service/routes/repo.js +++ b/src/service/routes/repo.js @@ -1,186 +1,200 @@ const express = require('express'); -const router = new express.Router(); const db = require('../../db'); const { getProxyURL } = require('../urls'); const { getAllProxiedHosts } = require('../../proxy/routes/helper'); -router.get('/', async (req, res) => { - const proxyURL = getProxyURL(req); - const query = {}; - - for (const k in req.query) { - if (!k) continue; - - if (k === 'limit') continue; - if (k === 'skip') continue; - let v = req.query[k]; - if (v === 'false') v = false; - if (v === 'true') v = true; - query[k] = v; - } - - const qd = await db.getRepos(query); - res.send(qd.map((d) => ({ ...d, proxyURL }))); -}); - -router.get('/:id', async (req, res) => { - const proxyURL = getProxyURL(req); - const _id = req.params.id; - const qd = await db.getRepoById(_id); - res.send({ ...qd, proxyURL }); -}); - -router.patch('/:id/user/push', async (req, res) => { - if (req.user && req.user.admin) { - const _id = req.params.id; - const username = req.body.username.toLowerCase(); - const user = await db.findUser(username); - - if (!user) { - res.status(400).send({ error: 'User does not exist' }); - return; +// create a reference to the proxy service as arrow functions will lose track of the `proxy` parameter +// used to restart the proxy when a new host is added +let theProxy = null; +const repo = (proxy) => { + theProxy = proxy; + const router = new express.Router(); + + router.get('/', async (req, res) => { + const proxyURL = getProxyURL(req); + const query = {}; + + for (const k in req.query) { + if (!k) continue; + + if (k === 'limit') continue; + if (k === 'skip') continue; + let v = req.query[k]; + if (v === 'false') v = false; + if (v === 'true') v = true; + query[k] = v; } - await db.addUserCanPush(_id, username); - res.send({ message: 'created' }); - } else { - res.status(401).send({ - message: 'You are not authorised to perform this action...', - }); - } -}); - -router.patch('/:id/user/authorise', async (req, res) => { - if (req.user && req.user.admin) { - const _id = req.params.id; - const username = req.body.username; - const user = await db.findUser(username); - - if (!user) { - res.status(400).send({ error: 'User does not exist' }); - return; - } + const qd = await db.getRepos(query); + res.send(qd.map((d) => ({ ...d, proxyURL }))); + }); - await db.addUserCanAuthorise(_id, username); - res.send({ message: 'created' }); - } else { - res.status(401).send({ - message: 'You are not authorised to perform this action...', - }); - } -}); - -router.delete('/:id/user/authorise/:username', async (req, res) => { - if (req.user && req.user.admin) { + router.get('/:id', async (req, res) => { + const proxyURL = getProxyURL(req); const _id = req.params.id; - const username = req.params.username; - const user = await db.findUser(username); + const qd = await db.getRepoById(_id); + res.send({ ...qd, proxyURL }); + }); + + router.patch('/:id/user/push', async (req, res) => { + if (req.user && req.user.admin) { + const _id = req.params.id; + const username = req.body.username.toLowerCase(); + const user = await db.findUser(username); + + if (!user) { + res.status(400).send({ error: 'User does not exist' }); + return; + } - if (!user) { - res.status(400).send({ error: 'User does not exist' }); - return; + await db.addUserCanPush(_id, username); + res.send({ message: 'created' }); + } else { + res.status(401).send({ + message: 'You are not authorised to perform this action...', + }); } + }); - await db.removeUserCanAuthorise(_id, username); - res.send({ message: 'created' }); - } else { - res.status(401).send({ - message: 'You are not authorised to perform this action...', - }); - } -}); - -router.delete('/:id/user/push/:username', async (req, res) => { - if (req.user && req.user.admin) { - const _id = req.params.id; - const username = req.params.username; - const user = await db.findUser(username); + router.patch('/:id/user/authorise', async (req, res) => { + if (req.user && req.user.admin) { + const _id = req.params.id; + const username = req.body.username; + const user = await db.findUser(username); - if (!user) { - res.status(400).send({ error: 'User does not exist' }); - return; - } - - await db.removeUserCanPush(_id, username); - res.send({ message: 'created' }); - } else { - res.status(401).send({ - message: 'You are not authorised to perform this action...', - }); - } -}); - -router.delete('/:id/delete', async (req, res) => { - if (req.user && req.user.admin) { - const _id = req.params.id; + if (!user) { + res.status(400).send({ error: 'User does not exist' }); + return; + } - await db.deleteRepo(_id); - res.send({ message: 'deleted' }); - } else { - res.status(401).send({ - message: 'You are not authorised to perform this action...', - }); - } -}); - -router.post('/', async (req, res) => { - if (req.user && req.user.admin) { - if (!req.body.url) { - res.status(400).send({ - message: 'Repository url is required', + await db.addUserCanAuthorise(_id, username); + res.send({ message: 'created' }); + } else { + res.status(401).send({ + message: 'You are not authorised to perform this action...', }); - return; } + }); - const repo = await db.getRepoByUrl(req.body.url); - if (repo) { - res.status(409).send({ - message: `Repository ${req.body.url} already exists!`, - }); + router.delete('/:id/user/authorise/:username', async (req, res) => { + if (req.user && req.user.admin) { + const _id = req.params.id; + const username = req.params.username; + const user = await db.findUser(username); + + if (!user) { + res.status(400).send({ error: 'User does not exist' }); + return; + } + + await db.removeUserCanAuthorise(_id, username); + res.send({ message: 'created' }); } else { - try { - // figure out if this represent a new domain to proxy - let newOrigin = true; - - const existingHosts = await getAllProxiedHosts(); - existingHosts.forEach((h) => { - // assume SSL is in use and that our origins are missing the protocol - if (req.body.url.startsWith(`https://${h}`)) { - newOrigin = false; - } - }); + res.status(401).send({ + message: 'You are not authorised to perform this action...', + }); + } + }); - console.log( - `API request to proxy repository ${req.body.url} is for a new origin: ${newOrigin},\n\texisting origin list was: ${JSON.stringify(existingHosts)}`, - ); + router.delete('/:id/user/push/:username', async (req, res) => { + if (req.user && req.user.admin) { + const _id = req.params.id; + const username = req.params.username; + const user = await db.findUser(username); - // create the repository - await db.createRepo(req.body); - res.send({ message: 'created' }); + if (!user) { + res.status(400).send({ error: 'User does not exist' }); + return; + } - // restart the proxy if we're proxying a new domain - if (newOrigin) { - console.log('Restarting the proxy to handle an additional origin'); + await db.removeUserCanPush(_id, username); + res.send({ message: 'created' }); + } else { + res.status(401).send({ + message: 'You are not authorised to perform this action...', + }); + } + }); + + router.delete('/:id/delete', async (req, res) => { + if (req.user && req.user.admin) { + const _id = req.params.id; + + // determine if we need to restart the proxy + const previousHosts = await getAllProxiedHosts(); + await db.deleteRepo(_id); + const currentHosts = await getAllProxiedHosts(); + + if (currentHosts.length < previousHosts.length) { + // restart the proxy + console.log('Restarting the proxy to remove a host'); + await theProxy.stop(); + await theProxy.start(); + } - // 1. Get proxy module dynamically to avoid circular dependency - const { proxy } = require('../../proxy'); + res.send({ message: 'deleted' }); + } else { + res.status(401).send({ + message: 'You are not authorised to perform this action...', + }); + } + }); - // 2. Stop existing services - await proxy.stop(); + router.post('/', async (req, res) => { + if (req.user && req.user.admin) { + if (!req.body.url) { + res.status(400).send({ + message: 'Repository url is required', + }); + return; + } - // 3. Restart the proxy, which should set up for the new domain - await proxy.start(); + const repo = await db.getRepoByUrl(req.body.url); + if (repo) { + res.status(409).send({ + message: `Repository ${req.body.url} already exists!`, + }); + } else { + try { + // figure out if this represent a new domain to proxy + let newOrigin = true; + + const existingHosts = await getAllProxiedHosts(); + existingHosts.forEach((h) => { + // assume SSL is in use and that our origins are missing the protocol + if (req.body.url.startsWith(`https://${h}`)) { + newOrigin = false; + } + }); + + console.log( + `API request to proxy repository ${req.body.url} is for a new origin: ${newOrigin},\n\texisting origin list was: ${JSON.stringify(existingHosts)}`, + ); + + // create the repository + await db.createRepo(req.body); + res.send({ message: 'created' }); + + // restart the proxy if we're proxying a new domain + if (newOrigin) { + console.log('Restarting the proxy to handle an additional host'); + await theProxy.stop(); + await theProxy.start(); + } + } catch (e) { + console.error('Repository creation failed due to error: ', e.message ? e.message : e); + console.error(e.stack); + res.status(500).send({ message: 'Failed to create repository due to error' }); } - } catch (e) { - console.error('Repository creation failed due to error: ', e.message ? e.message : e); - res.send('Failed to create repository'); } + } else { + res.status(401).send({ + message: 'You are not authorised to perform this action...', + }); } - } else { - res.status(401).send({ - message: 'You are not authorised to perform this action...', - }); - } -}); - -module.exports = router; + }); + + return router; +}; + +module.exports = repo; diff --git a/test/1.test.js b/test/1.test.js index ad67cff6a..227dc0104 100644 --- a/test/1.test.js +++ b/test/1.test.js @@ -10,7 +10,7 @@ chai.should(); describe('init', async () => { let app; before(async function () { - app = await service.start(); + app = await service.start(); // pass in proxy if testing config loading or administration of proxy routes }); it('should not be logged in', async function () { diff --git a/test/testProxyRoute.test.js b/test/testProxyRoute.test.js index e8faf76f7..7669a2e86 100644 --- a/test/testProxyRoute.test.js +++ b/test/testProxyRoute.test.js @@ -10,6 +10,24 @@ const getRouter = require('../src/proxy/routes').getRouter; const chain = require('../src/proxy/chain'); const proxyquire = require('proxyquire'); const { Action, Step } = require('../src/proxy/actions'); +const service = require('../src/service'); +const db = require('../src/db'); + +import Proxy from '../src/proxy'; + +const TEST_DEFAULT_REPO = { + url: 'https://github.com/finos/git-proxy.git', + name: 'git-proxy', + project: 'finos/gitproxy', + host: 'github.com', +}; + +const TEST_GITLAB_REPO = { + url: 'https://gitlab.com/gitlab-org/gitlab.git', + name: 'gitlab', + project: 'gitlab-org/gitlab', + host: 'gitlab.com', +}; describe('proxy route filter middleware', () => { let app; @@ -279,3 +297,141 @@ describe('proxyFilter function', async () => { expect(result).to.be.true; }); }); + +describe('proxy express application', async () => { + let apiApp; + let cookie; + let proxy; + + const setCookie = function (res) { + res.headers['set-cookie'].forEach((x) => { + if (x.startsWith('connect')) { + const value = x.split(';')[0]; + cookie = value; + } + }); + }; + + const cleanupRepo = async (url) => { + const repo = await db.getRepoByUrl(url); + if (repo) { + await db.deleteRepo(repo._id); + } + }; + + before(async () => { + // pass through requests + sinon.stub(chain, 'executeChain').resolves({ + blocked: false, + blockedMessage: '', + error: false, + }); + + // start the API and proxy + proxy = new Proxy(); + apiApp = await service.start(proxy); + await proxy.start(); + + const res = await chai.request(apiApp).post('/api/auth/login').send({ + username: 'admin', + password: 'admin', + }); + expect(res).to.have.cookie('connect.sid'); + setCookie(res); + + // if our default repo is not set-up, create it + const repo = await db.getRepoByUrl(TEST_DEFAULT_REPO.url); + if (!repo) { + const res2 = await chai + .request(apiApp) + .post('/api/v1/repo') + .set('Cookie', `${cookie}`) + .send(TEST_DEFAULT_REPO); + res2.should.have.status(200); + } + }); + + after(async () => { + sinon.restore(); + await service.stop(); + await proxy.stop(); + await cleanupRepo(TEST_DEFAULT_REPO.url); + await cleanupRepo(TEST_GITLAB_REPO.url); + }); + + it('should pass-through operations for the default GitHub repository', async function () { + const res = await chai + .request(proxy.getExpressApp()) + .get('/github.com/finos/git-proxy.git/info/refs?service=git-upload-pack') + .set('user-agent', 'git/2.42.0') + .set('accept', 'application/x-git-upload-pack-request') + .buffer(); + + expect(res.status).to.equal(200); + expect(res.text).to.contain('git-upload-pack'); + }); + + it('should pass-through operations for the default GitHub repository using the backwards compatibility URL', async function () { + const res = await chai + .request(proxy.getExpressApp()) + .get('/finos/git-proxy.git/info/refs?service=git-upload-pack') + .set('user-agent', 'git/2.42.0') + .set('accept', 'application/x-git-upload-pack-request') + .buffer(); + + expect(res.status).to.equal(200); + expect(res.text).to.contain('git-upload-pack'); + }); + + it('should pass-through operations for a new repository on a new origin at GitLab.com', async function () { + let repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); + expect(repo).to.be.null; + + const res = await chai + .request(apiApp) + .post('/api/v1/repo') + .set('Cookie', `${cookie}`) + .send(TEST_GITLAB_REPO); + res.should.have.status(200); + + repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); + expect(repo).to.not.be.null; + + const res2 = await chai + .request(proxy.getExpressApp()) + .get('/gitlab.com/gitlab-org/gitlab.git/info/refs?service=git-upload-pack') + .set('user-agent', 'git/2.42.0') + .set('accept', 'application/x-git-upload-pack-request') + .buffer(); + + res2.should.have.status(200); + expect(res2.text).to.contain('git-upload-pack'); + }).timeout(60000); + + it('should NOT pass-through operations after the new repository is deleted', async function () { + let repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); + expect(repo).to.not.be.null; + + const res = await chai + .request(apiApp) + .delete('/api/v1/repo/' + repo._id + '/delete') + .set('Cookie', `${cookie}`) + .send(); + res.should.have.status(200); + + repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); + expect(repo).to.be.null; + + // give the proxy half a second to restart + await new Promise((resolve) => setTimeout(resolve, 500)); + + const res2 = await chai + .request(proxy.getExpressApp()) + .get('/gitlab.com/gitlab-org/gitlab.git/info/refs?service=git-upload-pack') + .set('user-agent', 'git/2.42.0') + .set('accept', 'application/x-git-upload-pack-request') + .buffer(); + + res2.should.have.status(404); + }).timeout(60000); +}); diff --git a/test/testRepoApi.test.js b/test/testRepoApi.test.js index ca81bf9ef..23dc40bac 100644 --- a/test/testRepoApi.test.js +++ b/test/testRepoApi.test.js @@ -5,6 +5,8 @@ const db = require('../src/db'); const service = require('../src/service'); const { getAllProxiedHosts } = require('../src/proxy/routes/helper'); +import Proxy from '../src/proxy'; + chai.use(chaiHttp); chai.should(); const expect = chai.expect; @@ -39,6 +41,7 @@ const cleanupRepo = async (url) => { describe('add new repo', async () => { let app; + let proxy; let cookie; const repoIds = []; @@ -52,7 +55,8 @@ describe('add new repo', async () => { }; before(async function () { - app = await service.start(); + proxy = new Proxy(); + app = await service.start(proxy); // Prepare the data. // _id is autogenerated by the DB so we need to retrieve it before we can use it cleanupRepo(TEST_REPO.url); From d07b1324e64ad82433800d9ee51fff16e035737b Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 8 Jul 2025 09:41:19 +0100 Subject: [PATCH 51/76] fix: do proxy prep before initializing proxy to catch configured projects --- src/proxy/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proxy/index.ts b/src/proxy/index.ts index 07795a640..65182a7c0 100644 --- a/src/proxy/index.ts +++ b/src/proxy/index.ts @@ -68,8 +68,8 @@ export default class Proxy { } public async start() { - this.expressApp = await this.createApp(); await this.proxyPreparations(); + this.expressApp = await this.createApp(); this.httpServer = http .createServer(options as any, this.expressApp) .listen(proxyHttpPort, () => { From 482813572eae0d4a1fc600542917b1d10e0c9018 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 8 Jul 2025 15:13:07 +0100 Subject: [PATCH 52/76] test: comments on proxyRoute tests --- test/testProxyRoute.test.js | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/test/testProxyRoute.test.js b/test/testProxyRoute.test.js index 7669a2e86..f7dcd91d1 100644 --- a/test/testProxyRoute.test.js +++ b/test/testProxyRoute.test.js @@ -359,7 +359,8 @@ describe('proxy express application', async () => { await cleanupRepo(TEST_GITLAB_REPO.url); }); - it('should pass-through operations for the default GitHub repository', async function () { + it('should proxy requests for the default GitHub repository', async function () { + // proxy a fetch request const res = await chai .request(proxy.getExpressApp()) .get('/github.com/finos/git-proxy.git/info/refs?service=git-upload-pack') @@ -371,7 +372,8 @@ describe('proxy express application', async () => { expect(res.text).to.contain('git-upload-pack'); }); - it('should pass-through operations for the default GitHub repository using the backwards compatibility URL', async function () { + it('should proxy requests for the default GitHub repository using the backwards compatibility URL', async function () { + // proxy a fetch request using a fallback URL const res = await chai .request(proxy.getExpressApp()) .get('/finos/git-proxy.git/info/refs?service=git-upload-pack') @@ -383,10 +385,14 @@ describe('proxy express application', async () => { expect(res.text).to.contain('git-upload-pack'); }); - it('should pass-through operations for a new repository on a new origin at GitLab.com', async function () { + it('should be restarted by the api and proxy requests for a new host (e.g. gitlab.com) when a project at that host is ADDED via the API', async function () { + // Tests that the proxy restarts properly after a project with a URL at a new host is added + + // check that we do not have the Gitlab test repo set up yet let repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); expect(repo).to.be.null; + // create the repo through the API, which should force the proxy to restart to handle the new domain const res = await chai .request(apiApp) .post('/api/v1/repo') @@ -394,9 +400,11 @@ describe('proxy express application', async () => { .send(TEST_GITLAB_REPO); res.should.have.status(200); + // confirm that the repo was created in teh DB repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); expect(repo).to.not.be.null; + // proxy a fetch request to the new repo const res2 = await chai .request(proxy.getExpressApp()) .get('/gitlab.com/gitlab-org/gitlab.git/info/refs?service=git-upload-pack') @@ -408,10 +416,16 @@ describe('proxy express application', async () => { expect(res2.text).to.contain('git-upload-pack'); }).timeout(60000); - it('should NOT pass-through operations after the new repository is deleted', async function () { + it('should be restarted by the api and stop proxying requests for a host (e.g. gitlab.com) when the last project at that host is DELETED via the API', async function () { + // We are testing that the proxy stops proxying requests for a particular origin + // The chain is stubbed and will always passthrough requests, hence, we are only checking what hosts are proxied. + + // the gitlab test repo should already exist let repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); expect(repo).to.not.be.null; + // delete the gitlab test repo, which should force the proxy to restart and stop proxying gitlab.com + // We assume that there are no other gitlab.com repos present const res = await chai .request(apiApp) .delete('/api/v1/repo/' + repo._id + '/delete') @@ -419,12 +433,14 @@ describe('proxy express application', async () => { .send(); res.should.have.status(200); + // confirm that its gone from the DB repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); expect(repo).to.be.null; // give the proxy half a second to restart await new Promise((resolve) => setTimeout(resolve, 500)); + // try (and fail) to proxy a request to gitlab.com const res2 = await chai .request(proxy.getExpressApp()) .get('/gitlab.com/gitlab-org/gitlab.git/info/refs?service=git-upload-pack') From 4a1181788880fa03e5a911e8bf12c29901272a6e Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 8 Jul 2025 15:38:47 +0100 Subject: [PATCH 53/76] test: shorten test timeouts --- test/testProxyRoute.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testProxyRoute.test.js b/test/testProxyRoute.test.js index f7dcd91d1..f98fd5df2 100644 --- a/test/testProxyRoute.test.js +++ b/test/testProxyRoute.test.js @@ -414,7 +414,7 @@ describe('proxy express application', async () => { res2.should.have.status(200); expect(res2.text).to.contain('git-upload-pack'); - }).timeout(60000); + }).timeout(5000); it('should be restarted by the api and stop proxying requests for a host (e.g. gitlab.com) when the last project at that host is DELETED via the API', async function () { // We are testing that the proxy stops proxying requests for a particular origin @@ -449,5 +449,5 @@ describe('proxy express application', async () => { .buffer(); res2.should.have.status(404); - }).timeout(60000); + }).timeout(5000); }); From 03bcc6ed4c4ce46ecef4bae749e67498a7236a06 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 10 Jul 2025 07:51:46 +0100 Subject: [PATCH 54/76] feat: support for gitlab user profile links --- .../components/PushesTable.jsx | 53 +++++++++++-------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/src/ui/views/OpenPushRequests/components/PushesTable.jsx b/src/ui/views/OpenPushRequests/components/PushesTable.jsx index 6e57f1145..776786b5b 100644 --- a/src/ui/views/OpenPushRequests/components/PushesTable.jsx +++ b/src/ui/views/OpenPushRequests/components/PushesTable.jsx @@ -72,6 +72,34 @@ export default function PushesTable(props) { if (isLoading) return
Loading...
; if (isError) return
{errorMessage}
; + const getGitProvider = (url) => { + const hostname = new URL(url).hostname.toLowerCase(); + if (hostname === 'github.com') return 'github'; + if (hostname.includes('gitlab')) return 'gitlab'; + return 'unknown'; + }; + + const getUserProfileUrl = (username, provider, hostname) => { + if (provider == 'github') { + return `https://github.com/${username}`; + } else if (provider == 'gitlab') { + return `https://${hostname}/${username}`; + } else { + return null; + } + }; + + const getUserProfileData = (username, provider, hostname) => { + let profileData = ''; + const profileUrl = getUserProfileUrl(username, provider, hostname); + if (profileUrl) { + profileData = `${username}`; + } else { + profileData = `${username}`; + } + return profileData; + }; + return (
{} @@ -97,7 +125,8 @@ export default function PushesTable(props) { const repoBranch = row.branch.replace('refs/heads/', ''); const repoUrl = row.url; const repoWebUrl = repoUrl.replace('.git', ''); - const isGitHub = URL.parse(repoUrl).hostname === 'github.com'; + const gitProvider = getGitProvider(repoUrl); + const hostname = new URL(repoUrl).hostname; return ( @@ -126,28 +155,10 @@ export default function PushesTable(props) { - {isGitHub && ( - - {row.commitData[0].committer} - - )} - {!isGitHub && {row.commitData[0].committer}} + {getUserProfileData(row.commitData[0].committer, gitProvider, hostname)} - {isGitHub && ( - - {row.commitData[0].author} - - )} - {!isGitHub && {row.commitData[0].author}} + {getUserProfileData(row.commitData[0].author, gitProvider, hostname)} {row.commitData[0].authorEmail ? ( From 07ab1cb501f38a1db7d1fac4079ee063c3928d47 Mon Sep 17 00:00:00 2001 From: Kris West Date: Thu, 10 Jul 2025 08:49:07 +0100 Subject: [PATCH 55/76] feat: add support for retrieving repo details from gitlab and fix small rendering issues --- src/service/routes/repo.js | 7 +- src/ui/services/repo.js | 15 +- src/ui/views/RepoList/Components/NewRepo.jsx | 5 +- .../RepoList/Components/RepoOverview.jsx | 137 +++++++++++++----- 4 files changed, 118 insertions(+), 46 deletions(-) diff --git a/src/service/routes/repo.js b/src/service/routes/repo.js index 03c307717..7ebbb62e3 100644 --- a/src/service/routes/repo.js +++ b/src/service/routes/repo.js @@ -172,8 +172,11 @@ const repo = (proxy) => { ); // create the repository - await db.createRepo(req.body); - res.send({ message: 'created' }); + const repoDetails = await db.createRepo(req.body); + const proxyURL = getProxyURL(req); + + // return data on the new repoistory (including it's _id and the proxyUrl) + res.send({ ...repoDetails, proxyURL, message: 'created' }); // restart the proxy if we're proxying a new domain if (newOrigin) { diff --git a/src/ui/services/repo.js b/src/ui/services/repo.js index b0bfc3385..18331131a 100644 --- a/src/ui/services/repo.js +++ b/src/ui/services/repo.js @@ -53,11 +53,14 @@ const getRepos = async ( setIsError(true); if (error.response && error.response.status === 401) { setAuth(false); - setErrorMessage('Failed to authorize user. If JWT auth is enabled, please check your configuration or disable it.'); + setErrorMessage( + 'Failed to authorize user. If JWT auth is enabled, please check your configuration or disable it.', + ); } else { setErrorMessage(`Error fetching repositories: ${error.response.data.message}`); } - }).finally(() => { + }) + .finally(() => { setIsLoading(false); }); }; @@ -76,17 +79,19 @@ const getRepo = async (setIsLoading, setData, setAuth, setIsError, id) => { } else { setIsError(true); } - }).finally(() => { + }) + .finally(() => { setIsLoading(false); }); }; const addRepo = async (onClose, setError, data) => { const url = new URL(`${baseUrl}/repo`); - axios + return axios .post(url, data, { withCredentials: true, headers: { 'X-CSRF-TOKEN': getCookie('csrf') } }) - .then(() => { + .then((response) => { onClose(); + return response.data; }) .catch((error) => { console.log(error.response.data.message); diff --git a/src/ui/views/RepoList/Components/NewRepo.jsx b/src/ui/views/RepoList/Components/NewRepo.jsx index b5e93f1c4..47dc30a4a 100644 --- a/src/ui/views/RepoList/Components/NewRepo.jsx +++ b/src/ui/views/RepoList/Components/NewRepo.jsx @@ -75,9 +75,8 @@ function AddRepositoryDialog(props) { } try { - await addRepo(onClose, setError, data); - handleSuccess(data); - + const repoData = await addRepo(onClose, setError, data); + handleSuccess(repoData); handleClose(); } catch (e) { if (e.message) { diff --git a/src/ui/views/RepoList/Components/RepoOverview.jsx b/src/ui/views/RepoList/Components/RepoOverview.jsx index bd3706ce6..fda106899 100644 --- a/src/ui/views/RepoList/Components/RepoOverview.jsx +++ b/src/ui/views/RepoList/Components/RepoOverview.jsx @@ -569,30 +569,93 @@ import moment from 'moment'; import CodeActionButton from '../../../components/CustomButtons/CodeActionButton'; export default function Repositories(props) { - const [github, setGitHub] = React.useState({}); - + const [remoteRepoData, setRemoteRepoData] = React.useState(null); + const [provider, setProvider] = React.useState(null); const [errorMessage, setErrorMessage] = React.useState(''); const [snackbarOpen, setSnackbarOpen] = React.useState(false); useEffect(() => { - getGitHubRepository(); - }, [props.data.project, props.data.name]); + fetchRemoteRepositoryData(); + }, [props.data.project, props.data.name, props.data.url]); + + const fetchRemoteRepositoryData = async () => { + try { + const { url: remoteUrl } = props.data; + if (!remoteUrl) return; + + const parsedUrl = new URL(remoteUrl); + const hostname = parsedUrl.hostname.toLowerCase(); - // TODO add support for GitLab API: https://docs.gitlab.com/api/projects/#get-a-single-project - const getGitHubRepository = async () => { - await axios - .get(`https://api.github.com/repos/${props.data.project}/${props.data.name}`) - .then((res) => { - setGitHub(res.data); - }) - .catch((error) => { - setErrorMessage( - `Error fetching GitHub repository ${props.data.project}/${props.data.name}: ${error}`, + if (hostname === 'github.com') { + setProvider('github'); + const response = await axios.get( + `https://api.github.com/repos/${props.data.project}/${props.data.name}`, ); - setSnackbarOpen(true); - }); + setRemoteRepoData(response.data); + } else if (hostname.includes('gitlab')) { + setProvider('gitlab'); + const projectPath = encodeURIComponent(`${props.data.project}/${props.data.name}`); + const apiUrl = `https://${hostname}/api/v4/projects/${projectPath}`; + const response = await axios.get(apiUrl); + + // Make follow-up call to get languages + let primaryLanguage = null; + try { + const languagesResponse = await axios.get( + `https://${hostname}/api/v4/projects/${projectPath}/languages`, + ); + const languages = languagesResponse.data; + // Get the first key (primary language) from the ordered hash + primaryLanguage = Object.keys(languages)[0] || null; + } catch (languageError) { + console.warn('Could not fetch language data:', languageError); + } + + setRemoteRepoData({ + ...response.data, + primary_language: primaryLanguage, + }); + } // For other/unknown providers, don't make API calls + } catch (error) { + setErrorMessage(`Error fetching repository data: ${error.message}`); + setSnackbarOpen(true); + } }; + // Helper function to normalize data across providers + const getDisplayData = () => { + if (!remoteRepoData) return {}; + + if (provider === 'github') { + return { + description: remoteRepoData.description, + language: remoteRepoData.language, + license: remoteRepoData.license?.spdx_id, + parent: remoteRepoData.parent, + lastUpdated: moment.max([ + moment(remoteRepoData.created_at), + moment(remoteRepoData.updated_at), + moment(remoteRepoData.pushed_at), + ]), + htmlUrl: remoteRepoData.html_url, + parentUrl: remoteRepoData.parent?.html_url, + }; + } else if (provider === 'gitlab') { + return { + description: remoteRepoData.description, + language: remoteRepoData.primary_language, + license: remoteRepoData.license?.nickname, + parent: remoteRepoData.forked_from_project, + lastUpdated: moment(remoteRepoData.last_activity_at), + htmlUrl: remoteRepoData.web_url, + parentUrl: remoteRepoData.forked_from_project?.web_url, + }; + } + return {}; + }; + + const displayData = getDisplayData(); + const { url: remoteUrl, proxyURL } = props?.data || {}; const parsedUrl = new URL(remoteUrl); const cloneURL = `${proxyURL}/${parsedUrl.host}${parsedUrl.port ? `:${parsedUrl.port}` : ''}${parsedUrl.pathname}`; @@ -606,7 +669,7 @@ export default function Repositories(props) { {props.data.project}/{props.data.name} - {github.parent && ( + {displayData.parent && ( - {github.parent.full_name} + {displayData.parent.full_name} )} - {github.description &&

{github.description}

} + {displayData.description &&

{displayData.description}

} - {github.language && ( + {displayData.language && ( - {github.language} + {displayData.language} )} - {github.license && ( + {displayData.license && ( {' '} - {github.license.spdx_id} + {displayData.license.spdx_id} )} @@ -659,14 +722,14 @@ export default function Repositories(props) { {props.data.users?.canAuthorise?.length || 0} - {(github.created_at || github.updated_at || github.pushed_at) && ( + {(displayData.created_at || displayData.updated_at || displayData.pushed_at) && ( Last updated{' '} {moment .max([ - moment(github.created_at), - moment(github.updated_at), - moment(github.pushed_at), + moment(displayData.created_at), + moment(displayData.updated_at), + moment(displayData.pushed_at), ]) .fromNow()} @@ -679,13 +742,15 @@ export default function Repositories(props) {
- setSnackbarOpen(false)} - message={errorMessage} - /> + + setSnackbarOpen(false)} + message={errorMessage} + /> + ); } From 61412fe6c86ecef9a1b27f7357a0c0a14bd4ba1e Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 11 Jul 2025 15:27:24 +0100 Subject: [PATCH 56/76] feat: tweaks to types needed after merge --- src/types/models.ts | 1 + .../RepoList/Components/RepoOverview.tsx | 172 +++++++++++++----- src/ui/views/RepoList/repositories.types.ts | 4 +- 3 files changed, 134 insertions(+), 43 deletions(-) diff --git a/src/types/models.ts b/src/types/models.ts index a114683e8..5b4d2c536 100644 --- a/src/types/models.ts +++ b/src/types/models.ts @@ -24,6 +24,7 @@ export interface CommitData { export interface PushData { id: string; + url: string; repo: string; branch: string; commitFrom: string; diff --git a/src/ui/views/RepoList/Components/RepoOverview.tsx b/src/ui/views/RepoList/Components/RepoOverview.tsx index 0c9a840d0..6f1f231cc 100644 --- a/src/ui/views/RepoList/Components/RepoOverview.tsx +++ b/src/ui/views/RepoList/Components/RepoOverview.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from 'react'; +import React, { useEffect } from 'react'; import { Snackbar, TableCell, TableRow } from '@material-ui/core'; import GridContainer from '../../../components/Grid/GridContainer'; import GridItem from '../../../components/Grid/GridItem'; @@ -9,12 +9,13 @@ import CodeActionButton from '../../../components/CustomButtons/CodeActionButton import { languageColors } from '../../../../constants/languageColors'; import { RepositoriesProps } from '../repositories.types'; -interface GitHubRepository { +interface GitHubRepositoryMetadata { description?: string; language?: string; license?: { spdx_id: string; }; + html_url: string; parent?: { full_name: string; html_url: string; @@ -24,32 +25,130 @@ interface GitHubRepository { pushed_at?: string; } -const Repositories: React.FC = (props) => { - const [github, setGitHub] = useState({}); +interface GitLabRepositoryMetadata { + description?: string; + primary_language?: string; + license?: { + nickname: string; + }; + web_url: string; + forked_from_project?: { + full_name: string; + web_url: string; + }; + last_activity_at?: string; +} + +interface DisplayMetadata { + description?: string; + language?: string; + license?: string; + htmlUrl?: string; + parentName?: string; + parentUrl?: string; + lastUpdated?: string; + created_at?: string; + updated_at?: string; + pushed_at?: string; +} +const Repositories: React.FC = (props) => { + const [remoteRepoData, setRemoteRepoData] = React.useState< + GitHubRepositoryMetadata | GitLabRepositoryMetadata | null + >(null); + const [provider, setProvider] = React.useState<'github' | 'gitlab' | 'unknown' | null>(null); const [errorMessage, setErrorMessage] = React.useState(''); const [snackbarOpen, setSnackbarOpen] = React.useState(false); useEffect(() => { - getGitHubRepository(); - }, [props.data?.project, props.data?.name]); - - const getGitHubRepository = async () => { - await axios - .get(`https://api.github.com/repos/${props.data?.project}/${props.data?.name}`) - .then((res) => { - setGitHub(res.data); - }) - .catch((error) => { - setErrorMessage( - `Error fetching GitHub repository ${props.data?.project}/${props.data?.name}: ${error}`, + fetchRemoteRepositoryData(); + }, [props.data.project, props.data.name, props.data.url]); + + const fetchRemoteRepositoryData = async () => { + try { + const { url: remoteUrl } = props.data; + if (!remoteUrl) return; + + const parsedUrl = new URL(remoteUrl); + const hostname = parsedUrl.hostname.toLowerCase(); + + if (hostname === 'github.com') { + setProvider('github'); + const response = await axios.get( + `https://api.github.com/repos/${props.data?.project}/${props.data?.name}`, ); - setSnackbarOpen(true); - }); + setRemoteRepoData(response.data); + } else if (hostname.includes('gitlab')) { + setProvider('gitlab'); + const projectPath = encodeURIComponent(`${props.data?.project}/${props.data?.name}`); + const apiUrl = `https://${hostname}/api/v4/projects/${projectPath}`; + const response = await axios.get(apiUrl); + + // Make follow-up call to get languages + let primaryLanguage; + try { + const languagesResponse = await axios.get( + `https://${hostname}/api/v4/projects/${projectPath}/languages`, + ); + const languages = languagesResponse.data; + // Get the first key (primary language) from the ordered hash + primaryLanguage = Object.keys(languages)[0]; + } catch (languageError) { + console.warn('Could not fetch language data:', languageError); + } + + setRemoteRepoData({ + ...response.data, + primary_language: primaryLanguage, + }); + } // For other/unknown providers, don't make API calls + } catch (error: any) { + setErrorMessage(`Error fetching repository data: ${error.message}`); + setSnackbarOpen(true); + } }; - const { project: org, name, proxyURL } = props?.data || {}; - const cloneURL = `${proxyURL}/${org}/${name}.git`; + // Helper function to normalize data across providers + const getDisplayData = (): DisplayMetadata => { + if (!remoteRepoData) return {}; + + if (provider === 'github') { + const gitHubMetadata = remoteRepoData as GitHubRepositoryMetadata; + return { + description: gitHubMetadata.description, + language: gitHubMetadata.language, + license: gitHubMetadata.license?.spdx_id, + lastUpdated: moment + .max([ + moment(gitHubMetadata.created_at), + moment(gitHubMetadata.updated_at), + moment(gitHubMetadata.pushed_at), + ]) + .fromNow(), + htmlUrl: gitHubMetadata.html_url, + parentName: gitHubMetadata.parent?.full_name, + parentUrl: gitHubMetadata.parent?.html_url, + }; + } else if (provider === 'gitlab') { + const gitLabMetadata = remoteRepoData as GitLabRepositoryMetadata; + return { + description: gitLabMetadata.description, + language: gitLabMetadata.primary_language, + license: gitLabMetadata.license?.nickname, + lastUpdated: moment(gitLabMetadata.last_activity_at).fromNow(), + htmlUrl: gitLabMetadata.web_url, + parentName: gitLabMetadata.forked_from_project?.full_name, + parentUrl: gitLabMetadata.forked_from_project?.web_url, + }; + } + return {}; + }; + + const displayData = getDisplayData(); + + const { url: remoteUrl, proxyURL } = props?.data || {}; + const parsedUrl = new URL(remoteUrl); + const cloneURL = `${proxyURL}/${parsedUrl.host}${parsedUrl.port ? `:${parsedUrl.port}` : ''}${parsedUrl.pathname}`; return ( @@ -57,10 +156,10 @@ const Repositories: React.FC = (props) => {
- {props.data?.project}/{props.data?.name} + {props.data.project}/{props.data.name} - {github.parent && ( + {displayData.parentName && ( = (props) => { fontWeight: 'normal', color: 'inherit', }} - href={github.parent.html_url} + href={displayData.parentUrl} > - {github.parent.full_name} + {displayData.parentName} )} - {github.description &&

{github.description}

} + {displayData.description &&

{displayData.description}

} - {github.language && ( + {displayData.language && ( - {github.language} + {displayData.language} )} - {github.license && ( + {displayData.license && ( {' '} - {github.license.spdx_id} + {displayData.license} )} @@ -113,18 +212,7 @@ const Repositories: React.FC = (props) => { {props.data?.users?.canAuthorise?.length || 0} - {(github.created_at || github.updated_at || github.pushed_at) && ( - - Last updated{' '} - {moment - .max([ - moment(github.created_at || 0), - moment(github.updated_at || 0), - moment(github.pushed_at || 0), - ]) - .fromNow()} - - )} + {displayData.lastUpdated && Last updated {displayData.lastUpdated}}
diff --git a/src/ui/views/RepoList/repositories.types.ts b/src/ui/views/RepoList/repositories.types.ts index 577269671..2e7660147 100644 --- a/src/ui/views/RepoList/repositories.types.ts +++ b/src/ui/views/RepoList/repositories.types.ts @@ -1,7 +1,9 @@ export interface RepositoriesProps { - data?: { + data: { + _id: string; project: string; name: string; + url: string; proxyURL: string; users?: { canPush?: string[]; From 6dac8dfe272bfb82364a0e3eecdb699025c08ef5 Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 11 Jul 2025 16:08:22 +0100 Subject: [PATCH 57/76] test: fix a test (that was retrieving a repo by name) after merging main --- test/testDb.test.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/testDb.test.js b/test/testDb.test.js index 10e285a2c..d0b343239 100644 --- a/test/testDb.test.js +++ b/test/testDb.test.js @@ -4,7 +4,6 @@ const db = require('../src/db'); const { Repo, User } = require('../src/db/types'); const { Action } = require('../src/proxy/actions/Action'); const { Step } = require('../src/proxy/actions/Step'); -const { trimTrailingDotGit } = require('../src/db/helper'); const { expect } = chai; @@ -834,9 +833,7 @@ describe('Database clients', async () => { it('should be able to check if a user can approve/reject push including .git within the repo name', async function () { let allowed = undefined; - const repoName = trimTrailingDotGit(TEST_PUSH_DOT_GIT.repoName); - - await db.createRepo(TEST_REPO_DOT_GIT); + const repo = await db.createRepo(TEST_REPO_DOT_GIT); try { // push does not exist yet, should return false allowed = await db.canUserApproveRejectPush(TEST_PUSH_DOT_GIT.id, TEST_USER.username); @@ -856,7 +853,7 @@ describe('Database clients', async () => { try { // authorise user and recheck - await db.addUserCanAuthorise(repoName, TEST_USER.username); + await db.addUserCanAuthorise(repo._id, TEST_USER.username); allowed = await db.canUserApproveRejectPush(TEST_PUSH_DOT_GIT.id, TEST_USER.username); expect(allowed).to.be.true; } catch (e) { @@ -865,7 +862,7 @@ describe('Database clients', async () => { // clean up await db.deletePush(TEST_PUSH_DOT_GIT.id); - await db.removeUserCanAuthorise(repoName, TEST_USER.username); + await db.removeUserCanAuthorise(repo._id, TEST_USER.username); }); after(async function () { From 004b644ba2d6a20af9fe03f3cb88716fc52ceddf Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 11 Jul 2025 16:49:20 +0100 Subject: [PATCH 58/76] chore: move uril functions to utils.tsx and add comments --- src/ui/utils.tsx | 60 +++++++++++++++++-- .../components/PushesTable.tsx | 39 ++---------- src/ui/views/PushDetails/PushDetails.tsx | 6 +- 3 files changed, 64 insertions(+), 41 deletions(-) diff --git a/src/ui/utils.tsx b/src/ui/utils.tsx index be15a19ef..41fd82beb 100644 --- a/src/ui/utils.tsx +++ b/src/ui/utils.tsx @@ -5,13 +5,65 @@ */ export const getCookie = (name: string): string | null => { if (!document.cookie) return null; - + const cookies = document.cookie .split(';') .map((c) => c.trim()) .filter((c) => c.startsWith(name + '=')); - + if (!cookies.length) return null; - + return decodeURIComponent(cookies[0].split('=')[1]); -}; \ No newline at end of file +}; + +/** + * Retrieve a string indicating whether a repository URL is hosted + * by a known SCM provider (github or gitlab). + * @param {string} url The repository URL. + * @return {string} A string representing the SCM provider or 'unknown'. + */ +export const getGitProvider = (url: string) => { + const hostname = new URL(url).hostname.toLowerCase(); + if (hostname === 'github.com') return 'github'; + if (hostname.includes('gitlab')) return 'gitlab'; + return 'unknown'; +}; + +/** + * Predicts a user's profile URL based on their username and the SCM provider's details. + * @param {string} username The username. + * @param {string} provider The name of the SCM provider. + * @param {string} hostname The hostname of the SCM provider. + * @return {string | null} The predicted profile URL or null + */ +export const getUserProfileUrl = (username: string, provider: string, hostname: string) => { + if (provider == 'github') { + return `https://github.com/${username}`; + } else if (provider == 'gitlab') { + return `https://${hostname}/${username}`; + } else { + return null; + } +}; + +/** + * Attempts to construct a link to the user's profile at an SCM provider. + * @param {string} username The username. + * @param {string} provider The name of the SCM provider. + * @param {string} hostname The hostname of the SCM provider. + * @return {string} A string containing an HTML A tag pointing to the user's profile, if possible, degrading to just the username or 'N/A' when not (e.g. because the SCM provider is unknown). + */ +export const getUserProfileLink = (username: string, provider: string, hostname: string) => { + if (username) { + let profileData = ''; + const profileUrl = getUserProfileUrl(username, provider, hostname); + if (profileUrl) { + profileData = `${username}`; + } else { + profileData = `${username}`; + } + return profileData; + } else { + return 'N/A'; + } +}; diff --git a/src/ui/views/OpenPushRequests/components/PushesTable.tsx b/src/ui/views/OpenPushRequests/components/PushesTable.tsx index d45c17a29..4df866d2d 100644 --- a/src/ui/views/OpenPushRequests/components/PushesTable.tsx +++ b/src/ui/views/OpenPushRequests/components/PushesTable.tsx @@ -17,6 +17,7 @@ import Search from '../../../components/Search/Search'; import Pagination from '../../../components/Pagination/Pagination'; import { PushData } from '../../../../types/models'; import { trimPrefixRefsHeads, trimTrailingDotGit } from '../../../../db/helper'; +import { getGitProvider, getUserProfileLink } from '../../../utils'; interface PushesTableProps { [key: string]: any; @@ -80,38 +81,6 @@ const PushesTable: React.FC = (props) => { if (isLoading) return
Loading...
; if (isError) return
{errorMessage}
; - const getGitProvider = (url: string) => { - const hostname = new URL(url).hostname.toLowerCase(); - if (hostname === 'github.com') return 'github'; - if (hostname.includes('gitlab')) return 'gitlab'; - return 'unknown'; - }; - - const getUserProfileUrl = (username: string, provider: string, hostname: string) => { - if (provider == 'github') { - return `https://github.com/${username}`; - } else if (provider == 'gitlab') { - return `https://${hostname}/${username}`; - } else { - return null; - } - }; - - const getUserProfileData = (username: string, provider: string, hostname: string) => { - if (username) { - let profileData = ''; - const profileUrl = getUserProfileUrl(username, provider, hostname); - if (profileUrl) { - profileData = `${username}`; - } else { - profileData = `${username}`; - } - return profileData; - } else { - return 'N/A'; - } - }; - return (
@@ -136,7 +105,7 @@ const PushesTable: React.FC = (props) => { const repoFullName = trimTrailingDotGit(row.repo); const repoBranch = trimPrefixRefsHeads(row.branch); const repoUrl = row.url; - const repoWebUrl = repoUrl.replace('.git', ''); + const repoWebUrl = trimTrailingDotGit(repoUrl); const gitProvider = getGitProvider(repoUrl); const hostname = new URL(repoUrl).hostname; const commitTimestamp = @@ -167,10 +136,10 @@ const PushesTable: React.FC = (props) => { - {getUserProfileData(row.commitData[0].committer, gitProvider, hostname)} + {getUserProfileLink(row.commitData[0].committer, gitProvider, hostname)} - {getUserProfileData(row.commitData[0].author, gitProvider, hostname)} + {getUserProfileLink(row.commitData[0].author, gitProvider, hostname)} {row.commitData[0]?.authorEmail ? ( diff --git a/src/ui/views/PushDetails/PushDetails.tsx b/src/ui/views/PushDetails/PushDetails.tsx index 4c31005a2..2c5a2e6d3 100644 --- a/src/ui/views/PushDetails/PushDetails.tsx +++ b/src/ui/views/PushDetails/PushDetails.tsx @@ -24,6 +24,7 @@ import Snackbar from '@material-ui/core/Snackbar'; import Tooltip from '@material-ui/core/Tooltip'; import { PushData } from '../../../types/models'; import { trimPrefixRefsHeads, trimTrailingDotGit } from '../../../db/helper'; +import { getGitProvider } from '../../utils'; const Dashboard: React.FC = () => { const { id } = useParams<{ id: string }>(); @@ -109,8 +110,9 @@ const Dashboard: React.FC = () => { const repoFullName = trimTrailingDotGit(data.repo); const repoBranch = trimPrefixRefsHeads(data.branch); const repoUrl = data.url; - const repoWebUrl = repoUrl.replace('.git', ''); - const isGitHub = URL.parse(repoUrl)?.hostname === 'github.com'; + const repoWebUrl = trimTrailingDotGit(repoUrl); + const gitProvider = getGitProvider(repoUrl); + const isGitHub = gitProvider == 'github'; const generateIcon = (title: string) => { switch (title) { From e677e61d93aacac3bd305bcda7edd5eda0c8b698 Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 11 Jul 2025 19:08:49 +0100 Subject: [PATCH 59/76] feat: rework remote repository metadata handling --- src/types/models.ts | 58 +++++++ src/ui/utils.tsx | 119 +++++++++++++ src/ui/views/RepoDetails/RepoDetails.tsx | 43 +++-- .../RepoList/Components/RepoOverview.tsx | 156 +++--------------- src/ui/views/User/User.tsx | 1 + 5 files changed, 228 insertions(+), 149 deletions(-) diff --git a/src/types/models.ts b/src/types/models.ts index 5b4d2c536..3a751f8dc 100644 --- a/src/types/models.ts +++ b/src/types/models.ts @@ -50,3 +50,61 @@ export interface Route { icon?: string | React.ComponentType; visible?: boolean; } + +export interface GitHubRepositoryMetadata { + description?: string; + language?: string; + license?: { + spdx_id: string; + }; + html_url: string; + parent?: { + full_name: string; + html_url: string; + }; + created_at?: string; + updated_at?: string; + pushed_at?: string; + owner?: { + avatar_url: string; + html_url: string; + }; +} + +export interface GitLabRepositoryMetadata { + description?: string; + primary_language?: string; + license?: { + nickname: string; + }; + web_url: string; + forked_from_project?: { + full_name: string; + web_url: string; + }; + last_activity_at?: string; + avatar_url?: string; + namespace?: { + name: string; + path: string; + full_path: string; + avatar_url?: string; + web_url: string; + }; +} + +export interface SCMRepositoryMetadata { + description?: string; + language?: string; + license?: string; + htmlUrl?: string; + parentName?: string; + parentUrl?: string; + lastUpdated?: string; + created_at?: string; + updated_at?: string; + pushed_at?: string; + + profileUrl?: string; + avatarUrl?: string; +} diff --git a/src/ui/utils.tsx b/src/ui/utils.tsx index 41fd82beb..ab6413c41 100644 --- a/src/ui/utils.tsx +++ b/src/ui/utils.tsx @@ -1,3 +1,11 @@ +import axios from 'axios'; +import { + SCMRepositoryMetadata, + GitHubRepositoryMetadata, + GitLabRepositoryMetadata, +} from '../types/models'; +import moment from 'moment'; + /** * Retrieve a decoded cookie value from `document.cookie` with given `name`. * @param {string} name - The name of the cookie to retrieve @@ -67,3 +75,114 @@ export const getUserProfileLink = (username: string, provider: string, hostname: return 'N/A'; } }; + +/** + * Predicts an organisation's profile URL at an SCM provider. + * @param {string} project The organisation name. + * @param {string} provider The name of the SCM provider. + * @param {string} hostname The hostname of the SCM provider. + * @return {string} The predicted profile URL or null. + */ +export const getOrganisationProfileUrl = (project: string, provider: string, hostname: string) => { + if (provider == 'github') { + return `https://github.com/${project}`; + } else if (provider == 'gitlab') { + return `https://${hostname}/${project}`; + } else { + return null; + } +}; + +/** + * Predicts an organisation's profile image URL at an SCM provider. + * @param {string} project The organisation name. + * @param {string} provider The name of the SCM provider. + * @param {string} hostname The hostname of the SCM provider. + * @return {string} The predicted profile URL or null. + */ +export const getOrganisationProfileImageUrl = ( + project: string, + provider: string, + hostname: string, +) => { + if (provider == 'github') { + return `https://github.com/${project}.png`; + } else if (provider == 'gitlab') { + return `https://${hostname}/${project}.png`; + } else { + return null; + } +}; + +/** + * Retrieves data about repositories hosted at known SCM providers. + * @param {string} project The organisations's name. + * @param {string} name The repository name. + * @param {string} url The URL of the repository (used to detect the SCM provider) + * @return {Promise} Data retrieved from teh SCM provider or null + */ +export const fetchRemoteRepositoryData = async ( + project: string, + name: string, + url: string, +): Promise => { + const provider = getGitProvider(url); + const hostname = new URL(url).hostname; + + if (provider === 'github') { + const response = await axios.get( + `https://api.github.com/repos/${project}/${name}`, + ); + + return { + description: response.data.description, + language: response.data.language, + license: response.data.license?.spdx_id, + lastUpdated: moment + .max([ + moment(response.data.created_at), + moment(response.data.updated_at), + moment(response.data.pushed_at), + ]) + .fromNow(), + htmlUrl: response.data.html_url, + parentName: response.data.parent?.full_name, + parentUrl: response.data.parent?.html_url, + + avatarUrl: response.data.owner?.avatar_url, + profileUrl: response.data.owner?.html_url, + }; + } else if (provider == 'gitlab') { + const projectPath = encodeURIComponent(`${project}/${name}`); + const apiUrl = `https://${hostname}/api/v4/projects/${projectPath}`; + const response = await axios.get(apiUrl); + + // Make follow-up call to get languages + let primaryLanguage; + try { + const languagesResponse = await axios.get( + `https://${hostname}/api/v4/projects/${projectPath}/languages`, + ); + const languages = languagesResponse.data; + // Get the first key (primary language) from the ordered hash + primaryLanguage = Object.keys(languages)[0]; + } catch (languageError) { + console.warn('Could not fetch language data:', languageError); + } + + return { + description: response.data.description, + language: primaryLanguage, + license: response.data.license?.nickname, + lastUpdated: moment(response.data.last_activity_at).fromNow(), + htmlUrl: response.data.web_url, + parentName: response.data.forked_from_project?.full_name, + parentUrl: response.data.forked_from_project?.web_url, + avatarUrl: response.data.avatar_url, + profileUrl: response.data.namespace?.web_url, + }; + } else { + // For other/unknown providers, don't make API calls + return null; + } +}; diff --git a/src/ui/views/RepoDetails/RepoDetails.tsx b/src/ui/views/RepoDetails/RepoDetails.tsx index 8871b5d76..3a43225d7 100644 --- a/src/ui/views/RepoDetails/RepoDetails.tsx +++ b/src/ui/views/RepoDetails/RepoDetails.tsx @@ -21,6 +21,8 @@ import { UserContext } from '../../../context'; import CodeActionButton from '../../components/CustomButtons/CodeActionButton'; import { Box } from '@material-ui/core'; import { trimTrailingDotGit } from '../../../db/helper'; +import { fetchRemoteRepositoryData } from '../../utils'; +import { SCMRepositoryMetadata } from '../../../types/models'; interface RepoData { _id: string; @@ -59,6 +61,7 @@ const RepoDetails: React.FC = () => { const [, setAuth] = useState(true); const [isLoading, setIsLoading] = useState(true); const [isError, setIsError] = useState(false); + const [remoteRepoData, setRemoteRepoData] = React.useState(null); const { user } = useContext(UserContext); const { id: repoId } = useParams<{ id: string }>(); @@ -68,6 +71,12 @@ const RepoDetails: React.FC = () => { } }, [repoId]); + useEffect(() => { + if (data) { + fetchRemoteRepositoryData(data.project, data.name, data.url).then(setRemoteRepoData); + } + }, [data]); + const removeUser = async (userToRemove: string, action: 'authorise' | 'push') => { if (!repoId) return; await deleteUser(userToRemove, repoId, action); @@ -115,31 +124,33 @@ const RepoDetails: React.FC = () => {
- - {`${data.project} - + {remoteRepoData?.avatarUrl && ( + + {`${data.project} + + )} + Organization

- - {data.project} - + {remoteRepoData?.profileUrl && ( + + {data.project} + + )} + {!remoteRepoData?.profileUrl && {data.project}}

Name

diff --git a/src/ui/views/RepoList/Components/RepoOverview.tsx b/src/ui/views/RepoList/Components/RepoOverview.tsx index 6f1f231cc..a452232fd 100644 --- a/src/ui/views/RepoList/Components/RepoOverview.tsx +++ b/src/ui/views/RepoList/Components/RepoOverview.tsx @@ -3,149 +3,35 @@ import { Snackbar, TableCell, TableRow } from '@material-ui/core'; import GridContainer from '../../../components/Grid/GridContainer'; import GridItem from '../../../components/Grid/GridItem'; import { CodeReviewIcon, LawIcon, PeopleIcon } from '@primer/octicons-react'; -import axios from 'axios'; -import moment from 'moment'; import CodeActionButton from '../../../components/CustomButtons/CodeActionButton'; import { languageColors } from '../../../../constants/languageColors'; import { RepositoriesProps } from '../repositories.types'; - -interface GitHubRepositoryMetadata { - description?: string; - language?: string; - license?: { - spdx_id: string; - }; - html_url: string; - parent?: { - full_name: string; - html_url: string; - }; - created_at?: string; - updated_at?: string; - pushed_at?: string; -} - -interface GitLabRepositoryMetadata { - description?: string; - primary_language?: string; - license?: { - nickname: string; - }; - web_url: string; - forked_from_project?: { - full_name: string; - web_url: string; - }; - last_activity_at?: string; -} - -interface DisplayMetadata { - description?: string; - language?: string; - license?: string; - htmlUrl?: string; - parentName?: string; - parentUrl?: string; - lastUpdated?: string; - created_at?: string; - updated_at?: string; - pushed_at?: string; -} +import { fetchRemoteRepositoryData } from '../../../utils'; +import { SCMRepositoryMetadata } from '../../../../types/models'; const Repositories: React.FC = (props) => { - const [remoteRepoData, setRemoteRepoData] = React.useState< - GitHubRepositoryMetadata | GitLabRepositoryMetadata | null - >(null); - const [provider, setProvider] = React.useState<'github' | 'gitlab' | 'unknown' | null>(null); + const [remoteRepoData, setRemoteRepoData] = React.useState(null); const [errorMessage, setErrorMessage] = React.useState(''); const [snackbarOpen, setSnackbarOpen] = React.useState(false); useEffect(() => { - fetchRemoteRepositoryData(); + prepareRemoteRepositoryData(); }, [props.data.project, props.data.name, props.data.url]); - const fetchRemoteRepositoryData = async () => { + const prepareRemoteRepositoryData = async () => { try { const { url: remoteUrl } = props.data; if (!remoteUrl) return; - const parsedUrl = new URL(remoteUrl); - const hostname = parsedUrl.hostname.toLowerCase(); - - if (hostname === 'github.com') { - setProvider('github'); - const response = await axios.get( - `https://api.github.com/repos/${props.data?.project}/${props.data?.name}`, - ); - setRemoteRepoData(response.data); - } else if (hostname.includes('gitlab')) { - setProvider('gitlab'); - const projectPath = encodeURIComponent(`${props.data?.project}/${props.data?.name}`); - const apiUrl = `https://${hostname}/api/v4/projects/${projectPath}`; - const response = await axios.get(apiUrl); - - // Make follow-up call to get languages - let primaryLanguage; - try { - const languagesResponse = await axios.get( - `https://${hostname}/api/v4/projects/${projectPath}/languages`, - ); - const languages = languagesResponse.data; - // Get the first key (primary language) from the ordered hash - primaryLanguage = Object.keys(languages)[0]; - } catch (languageError) { - console.warn('Could not fetch language data:', languageError); - } - - setRemoteRepoData({ - ...response.data, - primary_language: primaryLanguage, - }); - } // For other/unknown providers, don't make API calls + setRemoteRepoData( + await fetchRemoteRepositoryData(props.data.project, props.data.name, remoteUrl), + ); } catch (error: any) { setErrorMessage(`Error fetching repository data: ${error.message}`); setSnackbarOpen(true); } }; - // Helper function to normalize data across providers - const getDisplayData = (): DisplayMetadata => { - if (!remoteRepoData) return {}; - - if (provider === 'github') { - const gitHubMetadata = remoteRepoData as GitHubRepositoryMetadata; - return { - description: gitHubMetadata.description, - language: gitHubMetadata.language, - license: gitHubMetadata.license?.spdx_id, - lastUpdated: moment - .max([ - moment(gitHubMetadata.created_at), - moment(gitHubMetadata.updated_at), - moment(gitHubMetadata.pushed_at), - ]) - .fromNow(), - htmlUrl: gitHubMetadata.html_url, - parentName: gitHubMetadata.parent?.full_name, - parentUrl: gitHubMetadata.parent?.html_url, - }; - } else if (provider === 'gitlab') { - const gitLabMetadata = remoteRepoData as GitLabRepositoryMetadata; - return { - description: gitLabMetadata.description, - language: gitLabMetadata.primary_language, - license: gitLabMetadata.license?.nickname, - lastUpdated: moment(gitLabMetadata.last_activity_at).fromNow(), - htmlUrl: gitLabMetadata.web_url, - parentName: gitLabMetadata.forked_from_project?.full_name, - parentUrl: gitLabMetadata.forked_from_project?.web_url, - }; - } - return {}; - }; - - const displayData = getDisplayData(); - const { url: remoteUrl, proxyURL } = props?.data || {}; const parsedUrl = new URL(remoteUrl); const cloneURL = `${proxyURL}/${parsedUrl.host}${parsedUrl.port ? `:${parsedUrl.port}` : ''}${parsedUrl.pathname}`; @@ -154,12 +40,12 @@ const Repositories: React.FC = (props) => {
- + {props.data.project}/{props.data.name} - {displayData.parentName && ( + {remoteRepoData?.parentName && ( = (props) => { fontWeight: 'normal', color: 'inherit', }} - href={displayData.parentUrl} + href={remoteRepoData.parentUrl} > - {displayData.parentName} + {remoteRepoData.parentName} )} - {displayData.description &&

{displayData.description}

} + {remoteRepoData?.description && ( +

{remoteRepoData.description}

+ )} - {displayData.language && ( + {remoteRepoData?.language && ( - {displayData.language} + {remoteRepoData.language} )} - {displayData.license && ( + {remoteRepoData?.license && ( {' '} - {displayData.license} + {remoteRepoData.license} )} @@ -212,7 +100,9 @@ const Repositories: React.FC = (props) => { {props.data?.users?.canAuthorise?.length || 0} - {displayData.lastUpdated && Last updated {displayData.lastUpdated}} + {remoteRepoData?.lastUpdated && ( + Last updated {remoteRepoData.lastUpdated} + )}
diff --git a/src/ui/views/User/User.tsx b/src/ui/views/User/User.tsx index a3635a2e8..b5b51e9b4 100644 --- a/src/ui/views/User/User.tsx +++ b/src/ui/views/User/User.tsx @@ -83,6 +83,7 @@ export default function UserProfile(): React.ReactElement { gitAccount: escapeHTML(gitAccount), }; await updateUser(updatedData); + setData(updatedData); navigate(`/dashboard/profile`); } catch { setIsError(true); From 3296af6b6d9b21978a65ccbfef61469e8e058857 Mon Sep 17 00:00:00 2001 From: Kris West Date: Mon, 14 Jul 2025 15:59:10 +0100 Subject: [PATCH 60/76] fix: make sure user profile is available after login --- src/ui/views/Login/Login.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ui/views/Login/Login.tsx b/src/ui/views/Login/Login.tsx index 9582274c1..aac16ace6 100644 --- a/src/ui/views/Login/Login.tsx +++ b/src/ui/views/Login/Login.tsx @@ -25,7 +25,7 @@ const loginUrl = `${process.env.VITE_API_URI}/api/auth/login`; const Login: React.FC = () => { const navigate = useNavigate(); - const { refreshUser } = useAuth(); + const authContext = useAuth(); const [username, setUsername] = useState(''); const [password, setPassword] = useState(''); @@ -64,7 +64,6 @@ const Login: React.FC = () => { window.sessionStorage.setItem('git.proxy.login', 'success'); setMessage('Success!'); setSuccess(true); - refreshUser().then(() => navigate('/dashboard/repo')); }) .catch((error: AxiosError) => { if (error.response?.status === 307) { @@ -76,6 +75,8 @@ const Login: React.FC = () => { setMessage('You entered an invalid username or password...'); } }) + .then(authContext.refreshUser) + .then(() => navigate(0)) .finally(() => { setIsLoading(false); }); From baf50b9982b8f5b179e092915be797f7425e9ff1 Mon Sep 17 00:00:00 2001 From: Kris West Date: Mon, 14 Jul 2025 17:50:56 +0100 Subject: [PATCH 61/76] fix: fix missing snackbar on login failure --- src/ui/views/Login/Login.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ui/views/Login/Login.tsx b/src/ui/views/Login/Login.tsx index aac16ace6..dfc23d036 100644 --- a/src/ui/views/Login/Login.tsx +++ b/src/ui/views/Login/Login.tsx @@ -64,6 +64,7 @@ const Login: React.FC = () => { window.sessionStorage.setItem('git.proxy.login', 'success'); setMessage('Success!'); setSuccess(true); + authContext.refreshUser().then(() => navigate(0)); }) .catch((error: AxiosError) => { if (error.response?.status === 307) { @@ -75,8 +76,6 @@ const Login: React.FC = () => { setMessage('You entered an invalid username or password...'); } }) - .then(authContext.refreshUser) - .then(() => navigate(0)) .finally(() => { setIsLoading(false); }); From 172895c7015ca853b7593b81363528e8fef4a619 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 15 Jul 2025 10:14:03 +0100 Subject: [PATCH 62/76] fix: resolve RepoDetails view display issues --- src/ui/views/RepoDetails/RepoDetails.tsx | 35 ++++++++++++------------ 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/ui/views/RepoDetails/RepoDetails.tsx b/src/ui/views/RepoDetails/RepoDetails.tsx index 3a43225d7..7e5bea20e 100644 --- a/src/ui/views/RepoDetails/RepoDetails.tsx +++ b/src/ui/views/RepoDetails/RepoDetails.tsx @@ -50,7 +50,7 @@ const useStyles = makeStyles((theme) => ({ }, }, table: { - minWidth: 650, + minWidth: 200, }, })); @@ -107,25 +107,26 @@ const RepoDetails: React.FC = () => { - {user.admin && ( -
- -
- )} - + {user.admin && ( + + + + )} + {remoteRepoData?.avatarUrl && ( - + { )} - + Organization

{remoteRepoData?.profileUrl && ( @@ -146,7 +147,7 @@ const RepoDetails: React.FC = () => { {!remoteRepoData?.profileUrl && {data.project}}

- + Name

{

- + URL

From 7e4cf5a1e8678897389177cdd775a54513a8fb4b Mon Sep 17 00:00:00 2001 From: Kris West Date: Wed, 16 Jul 2025 17:48:27 +0100 Subject: [PATCH 63/76] fix: typing issues in UI views --- src/ui/views/RepoDetails/RepoDetails.tsx | 3 +- .../RepoList/Components/Repositories.tsx | 33 +++++++++---------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/ui/views/RepoDetails/RepoDetails.tsx b/src/ui/views/RepoDetails/RepoDetails.tsx index 7e5bea20e..5a650c9b0 100644 --- a/src/ui/views/RepoDetails/RepoDetails.tsx +++ b/src/ui/views/RepoDetails/RepoDetails.tsx @@ -12,6 +12,7 @@ import TableCell from '@material-ui/core/TableCell'; import TableContainer from '@material-ui/core/TableContainer'; import TableHead from '@material-ui/core/TableHead'; import TableRow from '@material-ui/core/TableRow'; +import Box from '@material-ui/core/Box'; import { getRepo, deleteUser, deleteRepo } from '../../services/repo'; import { makeStyles } from '@material-ui/core/styles'; import AddUser from './Components/AddUser'; @@ -19,7 +20,6 @@ import { Code, Delete, RemoveCircle, Visibility } from '@material-ui/icons'; import { useNavigate, useParams } from 'react-router-dom'; import { UserContext } from '../../../context'; import CodeActionButton from '../../components/CustomButtons/CodeActionButton'; -import { Box } from '@material-ui/core'; import { trimTrailingDotGit } from '../../../db/helper'; import { fetchRemoteRepositoryData } from '../../utils'; import { SCMRepositoryMetadata } from '../../../types/models'; @@ -114,7 +114,6 @@ const RepoDetails: React.FC = () => { variant='contained' color='secondary' onClick={() => removeRepository(data._id)} - mx={1} > diff --git a/src/ui/views/RepoList/Components/Repositories.tsx b/src/ui/views/RepoList/Components/Repositories.tsx index 317fd684a..5c5d75ca3 100644 --- a/src/ui/views/RepoList/Components/Repositories.tsx +++ b/src/ui/views/RepoList/Components/Repositories.tsx @@ -28,6 +28,7 @@ interface GridContainerLayoutProps { onPageChange: (page: number) => void; onFilterChange: (filterOption: FilterOption, sortOrder: SortOrder) => void; tableId: string; + key: string; } interface UserContextType { @@ -136,25 +137,23 @@ export default function Repositories(props: RepositoriesProps): React.ReactEleme ); - return ( - - ); + return getGridContainerLayOut({ + key: 'x', + classes: classes, + openRepo: openRepo, + data: paginatedData, + repoButton: addrepoButton, + onSearch: handleSearch, + currentPage: currentPage, + totalItems: filteredData.length, + itemsPerPage: itemsPerPage, + onPageChange: handlePageChange, + onFilterChange: handleFilterChange, + tableId: 'RepoListTable', + }); } -function GetGridContainerLayOut(props: GridContainerLayoutProps): React.ReactElement { +function getGridContainerLayOut(props: GridContainerLayoutProps): React.ReactElement { return ( {props.repoButton} From 1034ad758a37d94adb70710493b7a4f32b0a6c9a Mon Sep 17 00:00:00 2001 From: Kris West Date: Wed, 16 Jul 2025 18:19:46 +0100 Subject: [PATCH 64/76] fix: more typing issues in ui and matching code button dimensions to other button --- .../CustomButtons/CodeActionButton.tsx | 24 ++++++++++--------- src/ui/views/RepoDetails/RepoDetails.tsx | 21 ++++++++++------ 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/ui/components/CustomButtons/CodeActionButton.tsx b/src/ui/components/CustomButtons/CodeActionButton.tsx index d5ce26eeb..47b3e3e20 100644 --- a/src/ui/components/CustomButtons/CodeActionButton.tsx +++ b/src/ui/components/CustomButtons/CodeActionButton.tsx @@ -20,32 +20,34 @@ const CodeActionButton: React.FC = ({ cloneURL }) => { const [placement, setPlacement] = useState(); const [isCopied, setIsCopied] = useState(false); - const handleClick = (newPlacement: PopperPlacementType) => (event: React.MouseEvent) => { - setIsCopied(false); - setAnchorEl(event.currentTarget); - setOpen((prev) => placement !== newPlacement || !prev); - setPlacement(newPlacement); - }; + const handleClick = + (newPlacement: PopperPlacementType) => (event: React.MouseEvent) => { + setIsCopied(false); + setAnchorEl(event.currentTarget); + setOpen((prev) => placement !== newPlacement || !prev); + setPlacement(newPlacement); + }; return ( <> - {' '} - Code + Code - +

= ({ cloneURL }) => { ); }; -export default CodeActionButton; \ No newline at end of file +export default CodeActionButton; diff --git a/src/ui/views/RepoDetails/RepoDetails.tsx b/src/ui/views/RepoDetails/RepoDetails.tsx index 5a650c9b0..d52ebd336 100644 --- a/src/ui/views/RepoDetails/RepoDetails.tsx +++ b/src/ui/views/RepoDetails/RepoDetails.tsx @@ -12,7 +12,7 @@ import TableCell from '@material-ui/core/TableCell'; import TableContainer from '@material-ui/core/TableContainer'; import TableHead from '@material-ui/core/TableHead'; import TableRow from '@material-ui/core/TableRow'; -import Box from '@material-ui/core/Box'; +import Grid from '@material-ui/core/Grid'; import { getRepo, deleteUser, deleteRepo } from '../../services/repo'; import { makeStyles } from '@material-ui/core/styles'; import AddUser from './Components/AddUser'; @@ -107,9 +107,15 @@ const RepoDetails: React.FC = () => { - + {user.admin && ( - + - + )} - - - + + + + {remoteRepoData?.avatarUrl && ( From 63fa968e49e826bf1a553297ff785d726d4e427a Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 5 Aug 2025 14:21:31 +0100 Subject: [PATCH 65/76] test: switch gitlab test to a smaller gitlab repo --- test/testProxyRoute.test.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/testProxyRoute.test.js b/test/testProxyRoute.test.js index f98fd5df2..57e3ddba8 100644 --- a/test/testProxyRoute.test.js +++ b/test/testProxyRoute.test.js @@ -23,10 +23,11 @@ const TEST_DEFAULT_REPO = { }; const TEST_GITLAB_REPO = { - url: 'https://gitlab.com/gitlab-org/gitlab.git', + url: 'https://gitlab.com/gitlab-community/meta.git', name: 'gitlab', - project: 'gitlab-org/gitlab', + project: 'gitlab-community/meta', host: 'gitlab.com', + proxyUrlPrefix: 'gitlab.com/gitlab-community/meta.git', }; describe('proxy route filter middleware', () => { @@ -407,7 +408,7 @@ describe('proxy express application', async () => { // proxy a fetch request to the new repo const res2 = await chai .request(proxy.getExpressApp()) - .get('/gitlab.com/gitlab-org/gitlab.git/info/refs?service=git-upload-pack') + .get(`/${TEST_GITLAB_REPO.proxyUrlPrefix}/info/refs?service=git-upload-pack`) .set('user-agent', 'git/2.42.0') .set('accept', 'application/x-git-upload-pack-request') .buffer(); @@ -443,7 +444,7 @@ describe('proxy express application', async () => { // try (and fail) to proxy a request to gitlab.com const res2 = await chai .request(proxy.getExpressApp()) - .get('/gitlab.com/gitlab-org/gitlab.git/info/refs?service=git-upload-pack') + .get(`/${TEST_GITLAB_REPO.proxyUrlPrefix}/info/refs?service=git-upload-pack`) .set('user-agent', 'git/2.42.0') .set('accept', 'application/x-git-upload-pack-request') .buffer(); From 12f7ad5bc8c7300153eb9def11f71ee73831f714 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 5 Aug 2025 14:24:19 +0100 Subject: [PATCH 66/76] fix: clean-up of proxy filter logging --- src/proxy/routes/index.ts | 50 +++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index 94e6d9545..e700374f3 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -6,12 +6,28 @@ import { executeChain } from '../chain'; import { processUrlPath, validGitRequest, getAllProxiedHosts } from './helper'; import { ProxyOptions } from 'express-http-proxy'; +const logAction = ( + url: string, + host: string | null | undefined, + userAgent: string | null | undefined, + errMsg: string | null | undefined, + blockMsg?: string | null | undefined, +) => { + let msg = `Action processed: ${!(errMsg || blockMsg) ? 'Allowed' : 'Blocked'} + Request URL: ${url} + Host: ${host} + User-Agent: ${userAgent}`; + if (errMsg) { + msg += `\n Error: ${errMsg}`; + } + if (blockMsg) { + msg += `\n Blocked: ${blockMsg}`; + } + console.log(msg); +}; + const proxyFilter: ProxyOptions['filter'] = async (req, res) => { try { - console.log('request url: ', req.url); - console.log('host: ', req.headers.host); - console.log('user-agent: ', req.headers['user-agent']); - const urlComponents = processUrlPath(req.url); if ( @@ -25,7 +41,6 @@ const proxyFilter: ProxyOptions['filter'] = async (req, res) => { } const action = await executeChain(req, res); - console.log('action processed'); if (action.error || action.blocked) { res.set('content-type', 'application/x-git-receive-pack-result'); @@ -48,16 +63,37 @@ const proxyFilter: ProxyOptions['filter'] = async (req, res) => { const packetMessage = handleMessage(message); - console.log(req.headers); + logAction( + req.url, + req.headers.host, + req.headers['user-agent'], + action.errorMessage, + action.blockedMessage, + ); res.status(200).send(packetMessage); return false; } + logAction( + req.url, + req.headers.host, + req.headers['user-agent'], + action.errorMessage, + action.blockedMessage, + ); + return true; } catch (e) { console.error('Error occurred in proxy filter function ', e); + logAction( + req.url, + req.headers.host, + req.headers['user-agent'], + 'Error occurred in proxy filter function: ' + ((e as Error).message ?? e), + null, + ); return false; } }; @@ -80,7 +116,7 @@ const getRequestPathResolver: (prefix: string) => ProxyOptions['proxyReqPathReso url = prefix + req.originalUrl; } - console.log(`Sending request to ${url}`); + console.log(`Request resolved to ${url}`); return url; }; }; From 369d78e9d93cbe3e07867161fa27b8805586f28d Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 5 Aug 2025 17:14:59 +0100 Subject: [PATCH 67/76] test: resolve test issues after merge of fixes for 946 and 950 --- .../checkUserPushPermission.test.js | 41 ++++++++----------- test/testCheckUserPushPermission.test.js | 22 +++++----- test/testPush.test.js | 25 ++++++++--- 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/test/processors/checkUserPushPermission.test.js b/test/processors/checkUserPushPermission.test.js index 328dd44fd..f5d4db86b 100644 --- a/test/processors/checkUserPushPermission.test.js +++ b/test/processors/checkUserPushPermission.test.js @@ -11,9 +11,11 @@ describe('checkUserPushPermission', () => { let getUsersStub; let isUserPushAllowedStub; let logStub; + let errorStub; beforeEach(() => { logStub = sinon.stub(console, 'log'); + errorStub = sinon.stub(console, 'error'); getUsersStub = sinon.stub(); isUserPushAllowedStub = sinon.stub(); @@ -64,13 +66,11 @@ describe('checkUserPushPermission', () => { expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.false; - expect( - stepSpy.calledWith( - 'User db-user is allowed to push on repo https://github.com/finos/git-proxy.git', - ), - ).to.be.true; + expect(stepSpy.lastCall.args[0]).to.equal( + 'User db-user@test.com is allowed to push on repo https://github.com/finos/git-proxy.git', + ); expect(logStub.lastCall.args[0]).to.equal( - 'User db-user permission on Repo https://github.com/finos/git-proxy.git : true', + 'User db-user@test.com permission on Repo https://github.com/finos/git-proxy.git : true', ); }); @@ -84,12 +84,10 @@ describe('checkUserPushPermission', () => { expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect( - stepSpy.calledWith( - 'User db-user is not allowed to push on repo https://github.com/finos/git-proxy.git, ending', - ), - ).to.be.true; - expect(result.steps[0].errorMessage).to.include('Rejecting push as user git-user'); + expect(stepSpy.lastCall.args[0]).to.equal( + 'Your push has been blocked (db-user@test.com is not allowed to push on repo https://github.com/finos/git-proxy.git)', + ); + expect(result.steps[0].errorMessage).to.include('Your push has been blocked'); expect(logStub.lastCall.args[0]).to.equal('User not allowed to Push'); }); @@ -100,12 +98,10 @@ describe('checkUserPushPermission', () => { expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect( - stepSpy.calledWith( - 'User git-user is not allowed to push on repo https://github.com/finos/git-proxy.git, ending', - ), - ).to.be.true; - expect(result.steps[0].errorMessage).to.include('Rejecting push as user git-user'); + expect(stepSpy.lastCall.args[0]).to.equal( + 'Your push has been blocked (db-user@test.com is not allowed to push on repo https://github.com/finos/git-proxy.git)', + ); + expect(result.steps[0].errorMessage).to.include('Your push has been blocked'); }); it('should handle multiple users for git account by rejecting the push', async () => { @@ -118,13 +114,12 @@ describe('checkUserPushPermission', () => { expect(result.steps).to.have.lengthOf(1); expect(result.steps[0].error).to.be.true; - expect(logStub.getCall(-3).args[0]).to.equal( - 'Users for this git account: [{"username":"user1","gitAccount":"git-user"},{"username":"user2","gitAccount":"git-user"}]', + expect(stepSpy.lastCall.args[0]).to.equal( + 'Your push has been blocked (there are multiple users with email db-user@test.com)', ); - expect(logStub.getCall(-2).args[0]).to.equal( - 'User git-user permission on Repo https://github.com/finos/git-proxy.git : false', + expect(errorStub.lastCall.args[0]).to.equal( + 'Multiple users found with email address db-user@test.com, ending', ); - expect(logStub.lastCall.args[0]).to.equal('User not allowed to Push'); }); }); }); diff --git a/test/testCheckUserPushPermission.test.js b/test/testCheckUserPushPermission.test.js index 7eb281d5f..dd7e9d187 100644 --- a/test/testCheckUserPushPermission.test.js +++ b/test/testCheckUserPushPermission.test.js @@ -6,7 +6,7 @@ const db = require('../src/db'); chai.should(); const TEST_ORG = 'finos'; -const TEST_REPO = 'test'; +const TEST_REPO = 'user-push-perms-test.git'; const TEST_URL = 'https://github.com/finos/user-push-perms-test.git'; const TEST_USERNAME_1 = 'push-perms-test'; const TEST_EMAIL_1 = 'push-perms-test@test.com'; @@ -15,35 +15,37 @@ const TEST_EMAIL_2 = 'push-perms-test-2@test.com'; const TEST_EMAIL_3 = 'push-perms-test-3@test.com'; describe('CheckUserPushPermissions...', async () => { + let testRepo = null; + before(async function () { - await db.deleteRepo(TEST_REPO); - await db.deleteUser(TEST_USERNAME_1); - await db.deleteUser(TEST_USERNAME_2); - await db.createRepo({ + // await db.deleteRepo(TEST_REPO); + // await db.deleteUser(TEST_USERNAME_1); + // await db.deleteUser(TEST_USERNAME_2); + testRepo = await db.createRepo({ project: TEST_ORG, name: TEST_REPO, url: TEST_URL, }); await db.createUser(TEST_USERNAME_1, 'abc', TEST_EMAIL_1, TEST_USERNAME_1, false); - await db.addUserCanPush(TEST_REPO, TEST_USERNAME_1); + await db.addUserCanPush(testRepo._id, TEST_USERNAME_1); await db.createUser(TEST_USERNAME_2, 'abc', TEST_EMAIL_2, TEST_USERNAME_2, false); }); after(async function () { - await db.deleteRepo(TEST_REPO); + await db.deleteRepo(testRepo._id); await db.deleteUser(TEST_USERNAME_1); await db.deleteUser(TEST_USERNAME_2); }); it('A committer that is approved should be allowed to push...', async () => { - const action = new Action('1', 'type', 'method', 1, TEST_ORG + '/' + TEST_REPO); + const action = new Action('1', 'type', 'method', 1, TEST_URL); action.userEmail = TEST_EMAIL_1; const { error } = await processor.exec(null, action); expect(error).to.be.false; }); it('A committer that is NOT approved should NOT be allowed to push...', async () => { - const action = new Action('1', 'type', 'method', 1, TEST_ORG + '/' + TEST_REPO); + const action = new Action('1', 'type', 'method', 1, TEST_URL); action.userEmail = TEST_EMAIL_2; const { error, errorMessage } = await processor.exec(null, action); expect(error).to.be.true; @@ -51,7 +53,7 @@ describe('CheckUserPushPermissions...', async () => { }); it('An unknown committer should NOT be allowed to push...', async () => { - const action = new Action('1', 'type', 'method', 1, TEST_ORG + '/' + TEST_REPO); + const action = new Action('1', 'type', 'method', 1, TEST_URL); action.userEmail = TEST_EMAIL_3; const { error, errorMessage } = await processor.exec(null, action); expect(error).to.be.true; diff --git a/test/testPush.test.js b/test/testPush.test.js index 4681f3de5..9e3ad21ff 100644 --- a/test/testPush.test.js +++ b/test/testPush.test.js @@ -41,7 +41,7 @@ const TEST_PUSH = { timestamp: 1744380903338, project: TEST_ORG, repoName: TEST_REPO + '.git', - url: TEST_REPO, + url: TEST_URL, repo: TEST_ORG + '/' + TEST_REPO + '.git', user: TEST_USERNAME_2, userEmail: TEST_EMAIL_2, @@ -55,6 +55,7 @@ const TEST_PUSH = { describe('auth', async () => { let app; let cookie; + let testRepo; const setCookie = function (res) { res.headers['set-cookie'].forEach((x) => { @@ -87,13 +88,19 @@ describe('auth', async () => { }; before(async function () { + // remove existing repo and users if any + const oldRepo = await db.getRepoByUrl(TEST_URL); + if (oldRepo) { + await db.deleteRepo(oldRepo._id); + } + await db.deleteUser(TEST_USERNAME_1); + await db.deleteUser(TEST_USERNAME_2); + app = await service.start(); await loginAsAdmin(); // set up a repo, user and push to test against - await db.deleteRepo(TEST_REPO); - await db.deleteUser(TEST_USERNAME_1); - await db.createRepo({ + testRepo = await db.createRepo({ project: TEST_ORG, name: TEST_REPO, url: TEST_URL, @@ -102,17 +109,23 @@ describe('auth', async () => { // Create a new user for the approver console.log('creating approver'); await db.createUser(TEST_USERNAME_1, TEST_PASSWORD_1, TEST_EMAIL_1, TEST_USERNAME_1, false); - await db.addUserCanAuthorise(TEST_REPO, TEST_USERNAME_1); + await db.addUserCanAuthorise(testRepo._id, TEST_USERNAME_1); // create a new user for the committer console.log('creating committer'); await db.createUser(TEST_USERNAME_2, TEST_PASSWORD_2, TEST_EMAIL_2, TEST_USERNAME_2, false); - await db.addUserCanPush(TEST_REPO, TEST_USERNAME_2); + await db.addUserCanPush(testRepo._id, TEST_USERNAME_2); // logout of admin account await logout(); }); + after(async function () { + await db.deleteRepo(testRepo._id); + await db.deleteUser(TEST_USERNAME_1); + await db.deleteUser(TEST_USERNAME_2); + }); + describe('test push API', async function () { afterEach(async function () { await db.deletePush(TEST_PUSH.id); From e612ad86983b3aef0e3ffb39f01bf98eca8a5d95 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 5 Aug 2025 17:30:52 +0100 Subject: [PATCH 68/76] test: fix minor typo in a test title --- test/testRouteFilter.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testRouteFilter.test.js b/test/testRouteFilter.test.js index 88a9cfed1..c457adf18 100644 --- a/test/testRouteFilter.test.js +++ b/test/testRouteFilter.test.js @@ -116,7 +116,7 @@ describe('url helpers and filter functions used in the proxy', function () { }); }); - it("processGitURLForNameAndOrg should return null for a git repository URL it can't pass", function () { + it("processGitURLForNameAndOrg should return null for a git repository URL it can't parse", function () { expect(processGitURLForNameAndOrg('someGitHost.com/repo')).to.be.null; expect(processGitURLForNameAndOrg('https://someGitHost.com/repo')).to.be.null; expect(processGitURLForNameAndOrg('https://somegithost.com:1234' + VERY_LONG_PATH + '.git')).to From 8b3313941e37cb26a7fd8c6095a71d33dbb7f7ca Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 5 Aug 2025 17:36:46 +0100 Subject: [PATCH 69/76] fix: reduce error msg from snackbar to console warn On failure to retrieve repository metadata as it will occur for all users on private projects. --- src/ui/views/RepoList/Components/RepoOverview.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ui/views/RepoList/Components/RepoOverview.tsx b/src/ui/views/RepoList/Components/RepoOverview.tsx index a452232fd..2191c05db 100644 --- a/src/ui/views/RepoList/Components/RepoOverview.tsx +++ b/src/ui/views/RepoList/Components/RepoOverview.tsx @@ -11,7 +11,7 @@ import { SCMRepositoryMetadata } from '../../../../types/models'; const Repositories: React.FC = (props) => { const [remoteRepoData, setRemoteRepoData] = React.useState(null); - const [errorMessage, setErrorMessage] = React.useState(''); + const [errorMessage] = React.useState(''); const [snackbarOpen, setSnackbarOpen] = React.useState(false); useEffect(() => { @@ -27,8 +27,9 @@ const Repositories: React.FC = (props) => { await fetchRemoteRepositoryData(props.data.project, props.data.name, remoteUrl), ); } catch (error: any) { - setErrorMessage(`Error fetching repository data: ${error.message}`); - setSnackbarOpen(true); + console.warn( + `Unable to fetch repository data for ${props.data.project}/${props.data.name} from '${remoteUrl}' - this may occur if the project is private or from an SCM vendor that is not supported.`, + ); } }; From 06ad5b63f3f734ee5ca098091cafe08d91e4d404 Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 8 Aug 2025 18:13:39 +0100 Subject: [PATCH 70/76] fix: add examples to new repo field hints --- src/ui/views/RepoList/Components/NewRepo.tsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/ui/views/RepoList/Components/NewRepo.tsx b/src/ui/views/RepoList/Components/NewRepo.tsx index d211252d4..61d7cc715 100644 --- a/src/ui/views/RepoList/Components/NewRepo.tsx +++ b/src/ui/views/RepoList/Components/NewRepo.tsx @@ -152,7 +152,9 @@ const AddRepositoryDialog: React.FC = ({ open, onClose onChange={(e) => setProject(e.target.value)} value={project} /> - Organization or path + + Organization or path, e.g. finos + @@ -165,7 +167,9 @@ const AddRepositoryDialog: React.FC = ({ open, onClose onChange={(e) => setName(e.target.value)} value={name} /> - Git Repository Name + + Git Repository Name, e.g. git-proxy + @@ -179,7 +183,9 @@ const AddRepositoryDialog: React.FC = ({ open, onClose onChange={(e) => setUrl(e.target.value)} value={url} /> - Git Repository URL + + Git Repository URL, e.g. https://github.com/finos/git-proxy.git + From 804a8848837662598ca881599caf195943b03c98 Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 8 Aug 2025 18:24:47 +0100 Subject: [PATCH 71/76] fix: make trimTrailingDotGit null safe --- src/db/helper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/helper.ts b/src/db/helper.ts index f706e763b..63532d11c 100644 --- a/src/db/helper.ts +++ b/src/db/helper.ts @@ -6,7 +6,7 @@ export const toClass = function (obj: any, proto: any) { export const trimTrailingDotGit = (str: string): string => { const target = '.git'; - if (str.endsWith(target)) { + if (str && str.endsWith(target)) { // extract string from 0 to the end minus the length of target return str.slice(0, -target.length); } From 256ef3f3409577310ac6d719d7c93aa2c010518e Mon Sep 17 00:00:00 2001 From: Kris West Date: Fri, 8 Aug 2025 18:28:09 +0100 Subject: [PATCH 72/76] fix: add constant and comments for max URL length in proxy helpers --- src/proxy/routes/helper.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/proxy/routes/helper.ts b/src/proxy/routes/helper.ts index 954d3a813..46f73a2c7 100644 --- a/src/proxy/routes/helper.ts +++ b/src/proxy/routes/helper.ts @@ -3,6 +3,9 @@ import * as db from '../../db'; /** Regex used to analyze un-proxied Git URLs */ const GIT_URL_REGEX = /(.+:\/\/)([^/]+)(\/.+\.git)(\/.+)*/; +/** Used to reject URLs that are too long and may be part of a DoS involving regex. */ +const MAX_URL_LENGTH = 512; + /** Type representing a breakdown of Git URL (un-proxied)*/ export type GitUrlBreakdown = { protocol: string; host: string; repoPath: string }; @@ -26,7 +29,8 @@ export type GitUrlBreakdown = { protocol: string; host: string; repoPath: string * @return {GitUrlBreakdown | null} A breakdown of the components of the URL. */ export const processGitUrl = (url: string): GitUrlBreakdown | null => { - if (url.length > 512) { + // limit URL length to avoid DoS via Regex issue detection in SAST scans + if (url.length > MAX_URL_LENGTH) { console.error(`The git URL is too long: ${url}`); return null; } @@ -69,7 +73,8 @@ export type UrlPathBreakdown = { repoPath: string; gitPath: string }; * @return {GitUrlBreakdown | null} A breakdown of the components of the URL path. */ export const processUrlPath = (requestPath: string): UrlPathBreakdown | null => { - if (requestPath.length > 512) { + // limit URL length to avoid DoS via Regex issue detection in SAST scans + if (requestPath.length > MAX_URL_LENGTH) { console.error(`The requestPath is too long: ${requestPath}`); return null; } @@ -119,7 +124,8 @@ export type GitNameBreakdown = { project: string | null; repoName: string }; * @return {GitNameBreakdown | null} A breakdown of the components of the URL. */ export const processGitURLForNameAndOrg = (gitUrl: string): GitNameBreakdown | null => { - if (gitUrl.length > 512) { + // limit URL length to avoid DoS via Regex issue detection in SAST scans + if (gitUrl.length > MAX_URL_LENGTH) { console.error(`The git URL is too long: ${gitUrl}`); return null; } From 9562b698859809cea168cf103b1070f92bdc9cdf Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 19 Aug 2025 17:08:51 +0100 Subject: [PATCH 73/76] fix: fix bad merge with main in testCheckRepoInAuthList.test.js --- test/testCheckRepoInAuthList.test.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/test/testCheckRepoInAuthList.test.js b/test/testCheckRepoInAuthList.test.js index ed24afd2b..885243c8c 100644 --- a/test/testCheckRepoInAuthList.test.js +++ b/test/testCheckRepoInAuthList.test.js @@ -40,15 +40,12 @@ describe('Check a Repo is in the authorised list', async () => { describe('fuzzing', () => { it('should not crash on random repo names', async () => { await fc.assert( - fc.asyncProperty( - fc.string(), - async (repoName) => { - const action = new actions.Action('123', 'type', 'get', 1234, repoName); - const result = await processor.exec(null, action, authList); - expect(result.error).to.be.true; - } - ), - { numRuns: 100 } + fc.asyncProperty(fc.string(), async (repoName) => { + const action = new actions.Action('123', 'type', 'get', 1234, repoName); + const result = await processor.exec(null, action); + expect(result.error).to.be.true; + }), + { numRuns: 100 }, ); }); }); From 536cf9becd6b7f4575583fd34f63c7aa9194d3da Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 19 Aug 2025 17:17:01 +0100 Subject: [PATCH 74/76] test: make test errors for proxyRoute easier to interpret --- test/testProxyRoute.test.js | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/test/testProxyRoute.test.js b/test/testProxyRoute.test.js index 57e3ddba8..a4768e21b 100644 --- a/test/testProxyRoute.test.js +++ b/test/testProxyRoute.test.js @@ -389,9 +389,12 @@ describe('proxy express application', async () => { it('should be restarted by the api and proxy requests for a new host (e.g. gitlab.com) when a project at that host is ADDED via the API', async function () { // Tests that the proxy restarts properly after a project with a URL at a new host is added - // check that we do not have the Gitlab test repo set up yet - let repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); - expect(repo).to.be.null; + // check that we don't have *any* repos at gitlab.com setup + const numExistingGitlabRepos = (await db.getRepos({ url: /https:\/\/gitlab\.com/ })).length; + expect( + numExistingGitlabRepos, + 'There is a GitLab that exists in the database already, which is NOT expected when running this test', + ).to.be.equal(0); // create the repo through the API, which should force the proxy to restart to handle the new domain const res = await chai @@ -401,11 +404,15 @@ describe('proxy express application', async () => { .send(TEST_GITLAB_REPO); res.should.have.status(200); - // confirm that the repo was created in teh DB - repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); + // confirm that the repo was created in the DB + const repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); expect(repo).to.not.be.null; - // proxy a fetch request to the new repo + // and that our initial query for repos would have picked it up + const numCurrentGitlabRepos = (await db.getRepos({ url: /https:\/\/gitlab\.com/ })).length; + expect(numCurrentGitlabRepos).to.be.equal(1); + + // proxy a request to the new repo const res2 = await chai .request(proxy.getExpressApp()) .get(`/${TEST_GITLAB_REPO.proxyUrlPrefix}/info/refs?service=git-upload-pack`) @@ -435,7 +442,10 @@ describe('proxy express application', async () => { res.should.have.status(200); // confirm that its gone from the DB - repo = await db.getRepoByUrl(TEST_GITLAB_REPO.url); + repo = await db.getRepoByUrl( + TEST_GITLAB_REPO.url, + 'The GitLab repo still existed in the database after it should have been deleted...', + ); expect(repo).to.be.null; // give the proxy half a second to restart From 9856f706fecc70760eaf806eedff5b6ced5015bd Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 19 Aug 2025 17:21:33 +0100 Subject: [PATCH 75/76] fix: resolve issues with isPackPost regex that required github-style org/repo.git paths --- src/proxy/routes/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index e700374f3..211542851 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -137,8 +137,7 @@ const proxyErrorHandler: ProxyOptions['proxyErrorHandler'] = (err, res, next) => const isPackPost = (req: Request) => req.method === 'POST' && - // eslint-disable-next-line no-useless-escape - /^\/[^\/]+\/[^\/]+\.git\/(?:git-upload-pack|git-receive-pack)$/.test(req.url); + /^(?:\/[^/]+)*\/[^/]+\.git\/(?:git-upload-pack|git-receive-pack)$/.test(req.url); const teeAndValidate = async (req: Request, res: Response, next: NextFunction) => { if (!isPackPost(req)) return next(); From 74643e675fe59229a28b12b09e39598748bdc1f3 Mon Sep 17 00:00:00 2001 From: Kris West Date: Tue, 19 Aug 2025 18:15:51 +0100 Subject: [PATCH 76/76] test: add an isPackPost test cases for multi-level orgs and no org --- test/teeAndValidation.test.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/teeAndValidation.test.js b/test/teeAndValidation.test.js index 66394dc06..3c5bf5da7 100644 --- a/test/teeAndValidation.test.js +++ b/test/teeAndValidation.test.js @@ -79,6 +79,12 @@ describe('isPackPost()', () => { it('returns true for git-upload-pack POST', () => { expect(isPackPost({ method: 'POST', url: '/a/b.git/git-upload-pack' })).to.be.true; }); + it('returns true for git-upload-pack POST, with a gitlab style multi-level org', () => { + expect(isPackPost({ method: 'POST', url: '/a/bee/sea/dee.git/git-upload-pack' })).to.be.true; + }); + it('returns true for git-upload-pack POST, with a bare (no org) repo URL', () => { + expect(isPackPost({ method: 'POST', url: '/a.git/git-upload-pack' })).to.be.true; + }); it('returns false for other URLs', () => { expect(isPackPost({ method: 'POST', url: '/info/refs' })).to.be.false; });