diff --git a/workspaces/bulk-import/.changeset/long-schools-learn.md b/workspaces/bulk-import/.changeset/long-schools-learn.md new file mode 100644 index 00000000000..9b2331c8027 --- /dev/null +++ b/workspaces/bulk-import/.changeset/long-schools-learn.md @@ -0,0 +1,5 @@ +--- +'@red-hat-developer-hub/backstage-plugin-bulk-import-backend': minor +--- + +**BREAKING** Changes the behavior of the bulk-import backend plugin to return only repositories that are yet to be imported by filtering out the already imported ones. Therefore, the frontend will not display already imported repositories with status displayed as "Imported" anymore. The frontend fetches all repositories at once on the first page load and then all the pagination and search is done client-side. diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts b/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts index 6f4992978d2..f2e110bc521 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts @@ -21,6 +21,9 @@ export const LOCAL_ADDR = `http://${localHostAndPort}`; export const LOCAL_GITLAB_ADDR = `https://gitlab.com/api/v4`; +export const CATALOG_API_LOCATIONS_LOCAL_ADDR = + /^https?:\/\/localhost:\d+\/api\/catalog\/locations$/; + export function loadTestFixture(filePathFromFixturesDir: string) { return require(`${__dirname}/${filePathFromFixturesDir}`); } @@ -499,4 +502,8 @@ export const DEFAULT_TEST_HANDLERS: RestHandler< return res(ctx.status(404)); }, ), + + rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => + res(ctx.status(200), ctx.json([])), + ), ]; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md index 80828016861..88160336be0 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md @@ -52,7 +52,7 @@ Fetch Repositories in the specified GitHub organization, provided it is accessib | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | | **search** | **String**| returns only the items that match the search string | [optional] [default to null] | | **approvalTool** | **String**| the approvalTool to use | [optional] [default to GIT] | -| **x-scm-tokens** | **String**| **Required.** JSON-encoded map of SCM host URL to user OAuth token. Used to fetch repositories on behalf of the signed-in user. The value must be a JSON object whose keys are SCM integration base URLs and whose values are OAuth bearer tokens (e.g. `{"https://github.com":"ghp_xxx"}`). Requests that omit this header, supply an empty object, or exceed 4 KB are rejected with HTTP 401. | [required] [default to null] | +| **x-scm-tokens** | **String**| Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. The value must be a JSON string whose structure matches SCMTokenMap (keys are SCM base URLs, values are OAuth bearer tokens). | [optional] [default to null] | ### Return type diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md index 668ac3aa25c..5aceccf9073 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md @@ -22,7 +22,7 @@ Fetch Organization Repositories accessible by Backstage Github Integrations | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | | **search** | **String**| returns only the items that match the search string | [optional] [default to null] | | **approvalTool** | **String**| the approvalTool to use | [optional] [default to GIT] | -| **x-scm-tokens** | **String**| **Required.** JSON-encoded map of SCM host URL to user OAuth token. Used to fetch repositories on behalf of the signed-in user. The value must be a JSON object whose keys are SCM integration base URLs and whose values are OAuth bearer tokens (e.g. `{"https://github.com":"ghp_xxx"}`). Requests that omit this header, supply an empty object, or exceed 4 KB are rejected with HTTP 401. | [required] [default to null] | +| **x-scm-tokens** | **String**| Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. The value must be a JSON string whose structure matches SCMTokenMap (keys are SCM base URLs, values are OAuth bearer tokens). | [optional] [default to null] | ### Return type diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts index 1815ddba3e6..7978d9c14f2 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts @@ -23,14 +23,17 @@ const octokit = { paginate: async (fn: any) => { const res = await fn(); if (res) { + if (Array.isArray(res?.data?.repositories)) { + return res.data.repositories; + } return res.data; } return []; }, - apps: { - listReposAccessibleToInstallation: jest.fn().mockReturnValue({ data: [] }), - }, rest: { + apps: { + listReposAccessibleToInstallation: jest.fn(), + }, repos: { listForAuthenticatedUser: jest.fn(), listForOrg: jest.fn(), @@ -134,6 +137,16 @@ describe('GithubApiService tests', () => { }, }, }); + octokit.rest.apps.listReposAccessibleToInstallation.mockReturnValue({ + data: { + repositories: [], + total_count: 0, + repository_selection: 'all', + }, + }); + octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ + data: [], + }); octokit.rest.repos.listForOrg.mockReturnValue({ data: [] }); octokit.rest.users.getByUsername.mockReturnValue({ data: { @@ -211,9 +224,12 @@ describe('GithubApiService tests', () => { type: 'User', }, }); - octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ data: [] }); - octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ - data: ghRepos, + octokit.rest.apps.listReposAccessibleToInstallation.mockReturnValue({ + data: { + repositories: ghRepos, + total_count: ghRepos.length, + repository_selection: 'all', + }, }); const result = await githubApiService.getRepositoriesFromIntegrations(); @@ -239,28 +255,36 @@ describe('GithubApiService tests', () => { ); }); - it('returns an a list of unique repositories and no errors', async () => { - octokit.apps.listReposAccessibleToInstallation + it('returns a list of unique repositories and no errors', async () => { + octokit.rest.apps.listReposAccessibleToInstallation .mockReturnValueOnce({ - data: ghRepos, + data: { + repositories: ghRepos, + total_count: ghRepos.length, + repository_selection: 'all', + }, }) .mockReturnValue({ - data: [ - { - name: 'B', - full_name: 'backstage/B', - url: 'https://api.github.com/repos/backstage/B', - html_url: 'https://github.com/backstage/B', - default_branch: 'main', - }, - { - name: 'C', - full_name: 'backstage/C', - url: 'https://api.github.com/repos/backstage/C', - html_url: 'https://github.com/backstage/C', - default_branch: 'default', - }, - ], + data: { + repositories: [ + { + name: 'B', + full_name: 'backstage/B', + url: 'https://api.github.com/repos/backstage/B', + html_url: 'https://github.com/backstage/B', + default_branch: 'main', + }, + { + name: 'C', + full_name: 'backstage/C', + url: 'https://api.github.com/repos/backstage/C', + html_url: 'https://github.com/backstage/C', + default_branch: 'default', + }, + ], + total_count: 2, + repository_selection: 'all', + }, }); const result = await githubApiService.getRepositoriesFromIntegrations(); @@ -311,14 +335,18 @@ describe('GithubApiService tests', () => { throw customError; }, ); - octokit.apps.listReposAccessibleToInstallation + octokit.rest.apps.listReposAccessibleToInstallation .mockImplementationOnce(async () => { const unauthorizedError = new Error('Bad credentials'); unauthorizedError.name = '401 Unauthorized'; throw unauthorizedError; }) .mockReturnValue({ - data: ghRepos, + data: { + repositories: ghRepos, + total_count: ghRepos.length, + repository_selection: 'all', + }, }); const result = await githubApiService.getRepositoriesFromIntegrations(); @@ -351,9 +379,6 @@ describe('GithubApiService tests', () => { octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ data: ghRepos, }); - octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ - data: [], - }); const result = await githubApiService.getRepositoriesFromIntegrations(); @@ -384,13 +409,11 @@ describe('GithubApiService tests', () => { octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ data: ghRepos, }); - octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ + octokit.rest.apps.listReposAccessibleToInstallation.mockReturnValue({ data: [], }); const result = await githubApiService.getRepositoriesFromIntegrations( - undefined, - undefined, undefined, { 'https://github.com': 'user-oauth-token' }, ); @@ -403,8 +426,6 @@ describe('GithubApiService tests', () => { it('returns empty repositories when userTokens is provided but no host matches an integration', async () => { const result = await githubApiService.getRepositoriesFromIntegrations( - undefined, - undefined, undefined, { 'https://some-other-host.com': 'user-oauth-token' }, ); @@ -418,7 +439,7 @@ describe('GithubApiService tests', () => { octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ data: ghRepos, }); - octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ + octokit.rest.apps.listReposAccessibleToInstallation.mockReturnValue({ data: [], }); @@ -433,13 +454,11 @@ describe('GithubApiService tests', () => { octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ data: ghRepos, }); - octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ + octokit.rest.apps.listReposAccessibleToInstallation.mockReturnValue({ data: [], }); const result = await githubApiService.getRepositoriesFromIntegrations( - undefined, - undefined, undefined, {}, ); diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts index 88532325dbb..a4fa2b74f25 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts @@ -335,13 +335,10 @@ export class GithubApiService implements GitApiService { }, octokit, credential, - ghConfig, repositories, dataFetchErrors, { search, - pageNumber, - pageSize, }, ); } else { @@ -356,8 +353,6 @@ export class GithubApiService implements GitApiService { dataFetchErrors, { search, - pageNumber, - pageSize, }, ); } @@ -383,8 +378,6 @@ export class GithubApiService implements GitApiService { */ async getRepositoriesFromIntegrations( search?: string, - pageNumber: number = DefaultPageNumber, - pageSize: number = DefaultPageSize, userTokens?: Record, ): Promise { const repositories = new Map(); @@ -404,7 +397,7 @@ export class GithubApiService implements GitApiService { userCredential, repositories, dataFetchErrors, - { search, pageNumber, pageSize }, + { search }, ), ); const repoList = Array.from(repositories.values()); @@ -429,13 +422,10 @@ export class GithubApiService implements GitApiService { }, octokit, credential, - ghConfig, repositories, dataFetchErrors, { search, - pageNumber, - pageSize, }, ) : await addGithubTokenRepositories( @@ -448,8 +438,6 @@ export class GithubApiService implements GitApiService { dataFetchErrors, { search, - pageNumber, - pageSize, }, ); this.logger.debug( @@ -463,7 +451,7 @@ export class GithubApiService implements GitApiService { }, ); - return this.buildRepositoryResponse(repositories, result, pageSize); + return this.buildRepositoryResponse(repositories, result, DefaultPageSize); } async filterLocationsAccessibleFromIntegrations( diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts index dcee5b075ac..c683c085184 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts @@ -19,6 +19,8 @@ import type { GithubCredentialsProvider, } from '@backstage/integration'; +import { type RestEndpointMethodTypes } from '@octokit/rest'; + export type { SCMFetchError as GithubFetchError, SCMOrganization as GithubOrganization, @@ -64,3 +66,15 @@ export interface ExtendedGithubCredentialsProvider extends GithubCredentialsProv host: string; }) => Promise; } + +export type AuthenticatedUserRepositoryResponse = + RestEndpointMethodTypes['repos']['listForAuthenticatedUser']['response']; + +export type AuthenticatedUserRepositoryList = + AuthenticatedUserRepositoryResponse['data']; + +export type AppInstallationRepositoriesResponse = + RestEndpointMethodTypes['apps']['listReposAccessibleToInstallation']['response']; + +export type AppInstallationRepositories = + AppInstallationRepositoriesResponse['data']; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/repoUtils.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/repoUtils.ts index 69987bff4bf..caf1a237675 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/repoUtils.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/repoUtils.ts @@ -33,18 +33,21 @@ import { } from '../../service/handlers/handlers'; import type { CustomGithubCredentialsProvider } from '../GithubAppManager'; import type { + AppInstallationRepositories, + AuthenticatedUserRepositoryList, ExtendedGithubCredentials, GithubAppCredentials, GithubFetchError, GithubRepository, } from '../types'; -import { getAllAppOrgs } from './orgUtils'; import { computeTotalCountFromGitHubToken, createCredentialError, handleError, } from './utils'; +const GITHUB_REST_API_MAX_PAGE_SIZE = 100; + export type ValidatedRepo = { ghConfig: GithubIntegrationConfig; credentials: ExtendedGithubCredentials[]; @@ -120,59 +123,37 @@ export async function addGithubAppRepositories( }, octokit: Octokit, credential: GithubAppCredentials, - ghConfig: GithubIntegrationConfig, repositories: Map, errors: Map, reqParams?: { search?: string; - pageNumber?: number; - pageSize?: number; }, ): Promise<{ totalCount?: number }> { - const search = reqParams?.search; - const pageNumber = reqParams?.pageNumber ?? DefaultPageNumber; - const pageSize = reqParams?.pageSize ?? DefaultPageSize; + const lowercaseSearch = reqParams?.search?.toLocaleLowerCase(); let totalCount: number | undefined; + try { - if (search) { - const allOrgsMap = await getAllAppOrgs( - deps.githubCredentialsProvider, - ghConfig, - credential.accountLogin, - ); - const orgSearch: string[] = []; - for (const [_orgUrl, ghOrg] of allOrgsMap) { - orgSearch.push(`org:${ghOrg.name}`); - } - const query = `${search} in:name ${orgSearch.join(' ')}`; - const searchResp = await searchRepos( - octokit, - query, - pageNumber, - pageSize, - ); - totalCount = searchResp.totalCount; - searchResp.repositories.forEach(repo => - repositories.set(repo.full_name, repo), - ); - } else { - const resp = await octokit.apps.listReposAccessibleToInstallation({ - page: pageNumber, - per_page: pageSize, - }); - const repos = resp?.data?.repositories ?? resp?.data; - repos?.forEach(repo => { - repositories.set(repo.full_name, { - name: repo.name, - full_name: repo.full_name, - url: repo.url, - html_url: repo.html_url, - default_branch: repo.default_branch, - updated_at: repo.updated_at, - }); - }); - totalCount = resp?.data?.total_count; - } + const { repositories: allRepositories } = + await listAllRepositoriesAccessibleToInstallation(octokit); + + const filteredRepositories = lowercaseSearch + ? allRepositories.filter(repo => + repo.name.toLocaleLowerCase().includes(lowercaseSearch), + ) + : allRepositories; + + filteredRepositories.forEach(repo => + repositories.set(repo.full_name, { + name: repo.name, + full_name: repo.full_name, + url: repo.url, + html_url: repo.html_url, + default_branch: repo.default_branch, + updated_at: repo.updated_at, + }), + ); + + totalCount = filteredRepositories.length; } catch (err: any) { logErrorIfNeeded( deps.logger, @@ -201,81 +182,33 @@ export async function addGithubTokenRepositories( errors: Map, reqParams?: { search?: string; - pageNumber?: number; - pageSize?: number; }, ): Promise<{ totalCount?: number }> { - const search = reqParams?.search; - const pageNumber = reqParams?.pageNumber ?? DefaultPageNumber; - const pageSize = reqParams?.pageSize ?? DefaultPageSize; + const lowercaseSearch = reqParams?.search?.toLocaleLowerCase(); let totalCount: number | undefined; + try { - if (search) { - // Get currently authenticated user - const username = (await octokit.rest.users.getAuthenticated())?.data - ?.login; - let query = `${search} in:name user:${username}`; + const allRepositories = + await listAllRepositoriesForAuthenticatedUser(octokit); - const allOrgsResp = await octokit.paginate( - octokit.rest.orgs.listForAuthenticatedUser, - { - sort: 'full_name', - direction: 'asc', - }, - ); - const orgSearch: string[] = []; - allOrgsResp?.forEach(org => orgSearch.push(`org:${org.login}`)); - if (orgSearch.length > 0) { - query += ` ${orgSearch.join(' ')}`; - } + const filteredRepositories = lowercaseSearch + ? allRepositories.filter(repo => + repo.name.toLocaleLowerCase().includes(lowercaseSearch), + ) + : allRepositories; - const searchResp = await searchRepos( - octokit, - query, - pageNumber, - pageSize, - ); - totalCount = searchResp.totalCount; - searchResp.repositories.forEach(repo => - repositories.set(repo.full_name, repo), - ); - } else { - /** - * The listForAuthenticatedUser endpoint will grab all the repositories the github token has explicit access to. - * These would include repositories they own, repositories where they are a collaborator, - * and repositories that they can access through an organization membership. - */ - const resp = await octokit.rest.repos.listForAuthenticatedUser({ - page: pageNumber, - per_page: pageSize, - sort: 'full_name', - direction: 'asc', - }); - resp?.data?.forEach(repo => { - repositories.set(repo.full_name, { - name: repo.name, - full_name: repo.full_name, - url: repo.url, - html_url: repo.html_url, - default_branch: repo.default_branch, - updated_at: repo.updated_at, - }); + filteredRepositories.forEach(repo => { + repositories.set(repo.full_name, { + name: repo.name, + full_name: repo.full_name, + url: repo.url, + html_url: repo.html_url, + default_branch: repo.default_branch, + updated_at: repo.updated_at, }); + }); - totalCount = await computeTotalCountFromGitHubToken( - deps, - async (lastPageNumber: number) => - octokit.repos - .listForAuthenticatedUser({ - page: lastPageNumber, - per_page: 100, - }) - .then(lastPageResp => lastPageResp.data.length), - 'repos.listForAuthenticatedUser', - resp?.data?.length, - resp?.headers?.link, - ); - } + totalCount = filteredRepositories.length; } catch (err) { handleError( deps, @@ -446,3 +379,39 @@ export async function createOrUpdateFileInBranch( } } } + +async function listAllRepositoriesForAuthenticatedUser( + octokit: Octokit, +): Promise { + /** + * The listForAuthenticatedUser endpoint will grab all the repositories the github token has explicit access to. + * These would include repositories they own, repositories where they are a collaborator, + * and repositories that they can access through an organization membership. + */ + return await octokit.paginate(octokit.rest.repos.listForAuthenticatedUser, { + per_page: GITHUB_REST_API_MAX_PAGE_SIZE, + sort: 'full_name', + direction: 'asc', + }); +} + +async function listAllRepositoriesAccessibleToInstallation( + octokit: Octokit, +): Promise { + /** + * The octokit pagination smartly extracts data from the response. + * Here, repositories array is extracted from the original listReposAccessibleToInstallation. + */ + const repositories = await octokit.paginate( + octokit.rest.apps.listReposAccessibleToInstallation, + { + per_page: GITHUB_REST_API_MAX_PAGE_SIZE, + }, + ); + + return { + repositories, + total_count: repositories.length, + repository_selection: repositories.repository_selection ?? 'all', + }; +} diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts index cec7800700d..569aff79eb8 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts @@ -467,8 +467,6 @@ describe('GitlabApiService tests', () => { }); const result = await gitlabApiService.getRepositoriesFromIntegrations( - undefined, - undefined, undefined, { 'https://gitlab.com': 'user-gitlab-oauth-token' }, ); @@ -480,8 +478,6 @@ describe('GitlabApiService tests', () => { it('returns empty repositories when userTokens is provided but no host matches an integration', async () => { const result = await gitlabApiService.getRepositoriesFromIntegrations( - undefined, - undefined, undefined, { 'https://some-other-host.com': 'user-gitlab-oauth-token' }, ); @@ -509,12 +505,7 @@ describe('GitlabApiService tests', () => { paginationInfo: { total: 2 }, }); - await gitlabApiService.getRepositoriesFromIntegrations( - undefined, - undefined, - undefined, - {}, - ); + await gitlabApiService.getRepositoriesFromIntegrations(undefined, {}); expect(mockGetAllCredentials).toHaveBeenCalled(); }); diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts index 58f26dee320..e3543ca2f79 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts @@ -319,8 +319,6 @@ export class GitlabApiService implements GitApiService { */ async getRepositoriesFromIntegrations( search?: string, - pageNumber: number = DefaultPageNumber, - pageSize: number = DefaultPageSize, userTokens?: Record, ): Promise { const repositories = new Map(); @@ -336,7 +334,7 @@ export class GitlabApiService implements GitApiService { userCredential, repositories, dataFetchErrors, - { search, pageNumber, pageSize }, + { search }, ), ); const repoList = Array.from(repositories.values()); @@ -367,8 +365,6 @@ export class GitlabApiService implements GitApiService { dataFetchErrors, { search, - pageNumber, - pageSize, }, ); this.logger.debug( @@ -383,7 +379,11 @@ export class GitlabApiService implements GitApiService { ); const repoList = Array.from(repositories.values()); - const totalCount = computeTotalCount(repoList, result.data, pageSize); + const totalCount = computeTotalCount( + repoList, + result.data, + DefaultPageSize, + ); return { repositories: repoList, errors: Array.from(result.errors?.values() ?? []), diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/utils/repoUtils.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/utils/repoUtils.ts index 3cf28daf62a..eb8921e893a 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/utils/repoUtils.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/utils/repoUtils.ts @@ -125,73 +125,34 @@ export async function addGitlabTokenRepositories( errors: Map, reqParams?: { search?: string; - pageNumber?: number; - pageSize?: number; }, ): Promise<{ totalCount?: number }> { - const search = reqParams?.search; - const pageNumber = reqParams?.pageNumber ?? DefaultPageNumber; - const pageSize = reqParams?.pageSize ?? DefaultPageSize; + const lowercaseSearch = reqParams?.search?.toLocaleLowerCase(); let totalCount: number | undefined; - try { - if (search) { - // Use the projects api with the search param - // that api gives us all the things the token has access to including the different projects in various groups - const searchResp = await searchRepos( - gitlab, - search, - pageNumber, - pageSize, - ); - totalCount = searchResp.totalCount; - searchResp.repositories.forEach(repo => - repositories.set(repo.full_name, repo), - ); - } else { - /** - * The Projects.all method with the membership: true option will grab all the repositories/projects the gitlab token has explicit access to. - * These would include repositories they own, repositories where they are a collaborator, - * and repositories that they can access through an organization membership. - */ - const { data, paginationInfo } = await gitlab.Projects.all< - true, - 'offset' - >({ - membership: true, - perPage: pageSize, - page: pageNumber, - showExpanded: true, - }); + try { + const allRepositories = await listAllRepositoriesForAuthenticatedUser( + deps, + gitlab, + ); + const filteredRepositories = lowercaseSearch + ? allRepositories.filter(repo => + repo.name.toLocaleLowerCase().includes(lowercaseSearch), + ) + : allRepositories; - data?.forEach((repo: ProjectSchema) => { - repositories.set(repo.path_with_namespace, { - name: repo.name, - full_name: repo.path_with_namespace, - url: repo._links.self, - html_url: repo.web_url, - default_branch: repo.default_branch, - updated_at: repo?.updated_at, - }); + filteredRepositories.forEach(repo => { + repositories.set(repo.path_with_namespace, { + name: repo.name, + full_name: repo.path_with_namespace, + url: repo._links.self, + html_url: repo.web_url, + default_branch: repo.default_branch, + updated_at: repo?.updated_at, }); + }); - /* - paginationInfo: { - total: , This is the total amount of repos, but will be NaN if the value is above 10k, see: https://github.com/jdalrymple/gitbeaker/issues/839#issuecomment-636482319 - next: , - current: , - previous: , - perPage: , - totalPages: - } - */ - - totalCount = await computeTotalCountFromPaginationInfo( - deps, - paginationInfo, - pageSize, // Not thrilled with this for some reason - ); - } + totalCount = filteredRepositories.length; } catch (err) { handleError( deps, @@ -342,3 +303,31 @@ export async function createOrUpdateFileInBranch( } } } + +async function listAllRepositoriesForAuthenticatedUser( + deps: { + logger: LoggerService; + }, + gitlab: InstanceType>, +): Promise { + try { + /** + * The Projects.all method with the membership: true option will grab all the repositories/projects the gitlab token has explicit access to. + * These would include repositories they own, repositories where they are a collaborator, + * and repositories that they can access through an organization membership. + */ + const allProjects = await gitlab.Projects.all({ + membership: true, + showExpanded: true, + orderBy: 'name', + sort: 'asc', + }); + + return allProjects.data; + } catch (error) { + deps.logger.error( + `Failed to list all repositories for authenticated user: ${error}`, + ); + throw error; + } +} diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts index 8c59552330b..065fddbe27e 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts @@ -45,8 +45,6 @@ export interface GitApiService { getRepositoriesFromIntegrations( search?: string, - pageNumber?: number, - pageSize?: number, userTokens?: Record, ): Promise; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts index ca6d8f4656b..ba824686fbe 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts @@ -19,7 +19,10 @@ import { AuthorizeResult } from '@backstage/plugin-permission-common'; import { rest } from 'msw'; import request from 'supertest'; -import { LOCAL_ADDR } from '../../../../__fixtures__/handlers'; +import { + CATALOG_API_LOCATIONS_LOCAL_ADDR, + LOCAL_ADDR, +} from '../../../../__fixtures__/handlers'; import { setupTest, startBackendServer, @@ -126,6 +129,269 @@ describe('repositories', () => { errors: ['Gitlab Token auth did not succeed'], }); }); + + describe('filtering repositories', () => { + it('returns all repos when no repos are imported yet', async () => { + const { mockCatalogClient } = useTestData(); + + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + approvalTool: 'GITLAB', + errors: [], + repositories: [ + { + defaultBranch: 'main', + errors: [], + id: 'saltypig1/dolbear', + lastUpdate: '2025-07-31T14:52:27.849Z', + name: 'dolbear', + organization: 'saltypig1', + url: 'http://localhost:8765/saltypig1/dolbear', + }, + { + defaultBranch: 'main', + errors: [], + id: 'saltypig1/funtimes', + lastUpdate: '2025-08-15T15:03:44.927Z', + name: 'funtimes', + organization: 'saltypig1', + url: 'http://localhost:8765/saltypig1/funtimes', + }, + { + defaultBranch: 'main', + errors: [], + id: 'saltypig1/swapi-node', + lastUpdate: '2025-07-31T14:54:57.289Z', + name: 'swapi-node', + organization: 'saltypig1', + url: 'http://localhost:8765/saltypig1/swapi-node', + }, + ], + totalCount: 3, + }); + }); + + it('returns empty array when there are no repos to be imported', async () => { + const { server, mockCatalogClient } = useTestData(); + + server.use( + rest.get(`${LOCAL_ADDR}/api/v4/projects`, (_, res, ctx) => + res(ctx.status(200), ctx.json([])), + ), + ); + + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + approvalTool: 'GITLAB', + errors: [], + repositories: [], + totalCount: 0, + }); + }); + + it('returns filtered (not yet imported) repos when some repos are already imported', async () => { + const { server, mockCatalogClient } = useTestData(); + + server.use( + rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => + res( + ctx.status(200), + ctx.json([ + { + data: { + id: 'imported-funtimes', + target: + 'http://localhost:8765/saltypig1/funtimes/blob/main/catalog-info.yaml', + type: 'url', + }, + }, + ]), + ), + ), + ); + + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + approvalTool: 'GITLAB', + errors: [], + repositories: [ + { + defaultBranch: 'main', + errors: [], + id: 'saltypig1/dolbear', + lastUpdate: '2025-07-31T14:52:27.849Z', + name: 'dolbear', + organization: 'saltypig1', + url: 'http://localhost:8765/saltypig1/dolbear', + }, + { + defaultBranch: 'main', + errors: [], + id: 'saltypig1/swapi-node', + lastUpdate: '2025-07-31T14:54:57.289Z', + name: 'swapi-node', + organization: 'saltypig1', + url: 'http://localhost:8765/saltypig1/swapi-node', + }, + ], + totalCount: 2, + }); + }); + + it('returns empty array when all repos are already imported', async () => { + const { server, mockCatalogClient } = useTestData(); + + server.use( + rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => + res( + ctx.status(200), + ctx.json([ + { + data: { + id: 'imported-dolbear', + target: + 'http://localhost:8765/saltypig1/dolbear/blob/main/catalog-info.yaml', + type: 'url', + }, + }, + { + data: { + id: 'imported-funtimes', + target: + 'http://localhost:8765/saltypig1/funtimes/blob/main/catalog-info.yaml', + type: 'url', + }, + }, + { + data: { + id: 'imported-swapi-node', + target: + 'http://localhost:8765/saltypig1/swapi-node/blob/main/catalog-info.yaml', + type: 'url', + }, + }, + ]), + ), + ), + ); + + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + approvalTool: 'GITLAB', + errors: [], + repositories: [], + totalCount: 0, + }); + }); + + it('returns all repos even though a non-root catalog location exists', async () => { + const { server, mockCatalogClient } = useTestData(); + + server.use( + rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => + res( + ctx.status(200), + ctx.json([ + { + data: { + id: 'imported-funtimes-nested', + target: + 'http://localhost:8765/saltypig1/funtimes/blob/main/packages/backend/catalog-info.yaml', + type: 'url', + }, + }, + ]), + ), + ), + ); + + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + approvalTool: 'GITLAB', + errors: [], + repositories: [ + { + defaultBranch: 'main', + errors: [], + id: 'saltypig1/dolbear', + lastUpdate: '2025-07-31T14:52:27.849Z', + name: 'dolbear', + organization: 'saltypig1', + url: 'http://localhost:8765/saltypig1/dolbear', + }, + { + defaultBranch: 'main', + errors: [], + id: 'saltypig1/funtimes', + lastUpdate: '2025-08-15T15:03:44.927Z', + name: 'funtimes', + organization: 'saltypig1', + url: 'http://localhost:8765/saltypig1/funtimes', + }, + { + defaultBranch: 'main', + errors: [], + id: 'saltypig1/swapi-node', + lastUpdate: '2025-07-31T14:54:57.289Z', + name: 'swapi-node', + organization: 'saltypig1', + url: 'http://localhost:8765/saltypig1/swapi-node', + }, + ], + totalCount: 3, + }); + }); + }); }); describe('GET /organizations/{org}/repositories', () => { diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts index 9690db163ed..ae5a98252d4 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts @@ -19,7 +19,10 @@ import { AuthorizeResult } from '@backstage/plugin-permission-common'; import { rest } from 'msw'; import request from 'supertest'; -import { LOCAL_ADDR } from '../../../../__fixtures__/handlers'; +import { + CATALOG_API_LOCATIONS_LOCAL_ADDR, + LOCAL_ADDR, +} from '../../../../__fixtures__/handlers'; import { addHandlersForGHTokenAppErrors, setupTest, @@ -140,6 +143,251 @@ describe('repositories', () => { errors: ['Github Token auth did not succeed'], }); }); + + describe('filtering repositories', () => { + it('returns all repos when no repos are imported yet', async () => { + const { mockCatalogClient } = useTestData(); + + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + errors: [], + repositories: [ + { + defaultBranch: 'master', + errors: [], + id: 'octocat/animated-happiness', + lastUpdate: '2011-01-26T19:14:43Z', + name: 'animated-happiness', + organization: 'octocat', + url: 'http://localhost:8765/octocat/animated-happiness', + }, + + { + defaultBranch: 'master', + errors: [], + id: 'my-user/Lorem-Ipsum', + lastUpdate: '2011-01-26T19:14:43Z', + name: 'Lorem-Ipsum', + organization: 'my-user', + url: 'http://localhost:8765/my-user/Lorem-Ipsum', + }, + ], + totalCount: 2, + }); + }); + + it('returns empty array when there are no repos to be imported', async () => { + const { server, mockCatalogClient } = useTestData(); + + server.use( + rest.get(`${LOCAL_ADDR}/user/repos`, (_, res, ctx) => + res(ctx.status(200), ctx.json([])), + ), + ); + server.use( + rest.get(`${LOCAL_ADDR}/installation/repositories`, (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ total_count: 0, repositories: [] }), + ), + ), + ); + + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + errors: [], + repositories: [], + totalCount: 0, + }); + }); + + it('returns filtered (not yet imported) repos when some repos are already imported', async () => { + const { server, mockCatalogClient } = useTestData(); + + server.use( + rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => + res( + ctx.status(200), + ctx.json([ + { + data: { + id: 'imported-hello-world', + target: + 'http://localhost:8765/octocat/Hello-World/blob/master/catalog-info.yaml', + type: 'url', + }, + }, + ]), + ), + ), + ); + + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + errors: [], + repositories: [ + { + defaultBranch: 'master', + errors: [], + id: 'octocat/animated-happiness', + lastUpdate: '2011-01-26T19:14:43Z', + name: 'animated-happiness', + organization: 'octocat', + url: 'http://localhost:8765/octocat/animated-happiness', + }, + { + defaultBranch: 'master', + errors: [], + id: 'my-user/Lorem-Ipsum', + lastUpdate: '2011-01-26T19:14:43Z', + name: 'Lorem-Ipsum', + organization: 'my-user', + url: 'http://localhost:8765/my-user/Lorem-Ipsum', + }, + ], + totalCount: 2, + }); + }); + + it('returns empty array when all repos are already imported', async () => { + const { server, mockCatalogClient } = useTestData(); + + server.use( + rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => + res( + ctx.status(200), + ctx.json([ + { + data: { + id: 'imported-animated-happiness', + target: + 'http://localhost:8765/octocat/animated-happiness/blob/master/catalog-info.yaml', + type: 'url', + }, + }, + { + data: { + id: 'imported-hello-world', + target: + 'http://localhost:8765/octocat/Hello-World/blob/master/catalog-info.yaml', + type: 'url', + }, + }, + { + data: { + id: 'imported-lorem-ipsum', + target: + 'http://localhost:8765/my-user/Lorem-Ipsum/blob/master/catalog-info.yaml', + type: 'url', + }, + }, + ]), + ), + ), + ); + + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + errors: [], + repositories: [], + totalCount: 0, + }); + }); + + it('returns all repos even though a non-root catalog location exists', async () => { + const { server, mockCatalogClient } = useTestData(); + + server.use( + rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => + res( + ctx.status(200), + ctx.json([ + { + data: { + id: 'imported-animated-happiness-sub', + target: + 'http://localhost:8765/octocat/animated-happiness/blob/master/monorepo/nested/path/catalog-info.yaml', + type: 'url', + }, + }, + ]), + ), + ), + ); + + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + errors: [], + repositories: [ + { + defaultBranch: 'master', + errors: [], + id: 'octocat/animated-happiness', + lastUpdate: '2011-01-26T19:14:43Z', + name: 'animated-happiness', + organization: 'octocat', + url: 'http://localhost:8765/octocat/animated-happiness', + }, + + { + defaultBranch: 'master', + errors: [], + id: 'my-user/Lorem-Ipsum', + lastUpdate: '2011-01-26T19:14:43Z', + name: 'Lorem-Ipsum', + organization: 'my-user', + url: 'http://localhost:8765/my-user/Lorem-Ipsum', + }, + ], + totalCount: 2, + }); + }); + }); }); describe('GET /organizations/{org}/repositories', () => { diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts index 44554f485a9..04da6d49f8e 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts @@ -20,6 +20,7 @@ import type { Config } from '@backstage/config'; import gitUrlParse from 'git-url-parse'; import { CatalogHttpClient } from '../../../catalog/catalogHttpClient'; +import { getCatalogUrl } from '../../../catalog/catalogUtils'; import type { Components } from '../../../generated/openapi'; import { GitApiService } from '../../../scm/GitApiService'; import type { SCMRepositoryResponse } from '../../../scm/types'; @@ -56,11 +57,53 @@ export async function findAllRepositories( }',${pageNumber},${pageSize})..`, ); - const repos = await deps.gitApiService - .getRepositoriesFromIntegrations(search, pageNumber, pageSize, userTokens) - .then(response => formatResponse(deps, response, checkStatus)); + const [alreadyImportedRepositories, allRepositoriesResponse] = + await Promise.all([ + deps.catalogHttpClient.listCatalogUrlLocations(), + deps.gitApiService.getRepositoriesFromIntegrations(search, userTokens), + ]); - return repos; + const alreadyImportedRepositoriesLocationTargets = new Set( + alreadyImportedRepositories.uniqueCatalogUrlLocations.keys(), + ); + + const { repositories: allRepositories, errors } = allRepositoriesResponse; + + const notImportedYetRepositories = allRepositories.filter(repo => { + const catalogUrl = getCatalogUrl( + deps.config, + repo.html_url.replace(/\/$/, ''), + repo.default_branch, + ); + + let alreadyImported = + alreadyImportedRepositoriesLocationTargets.has(catalogUrl); + + if (!alreadyImported) { + // Workaround: when a GitHub repository is imported via Backstage, the + // resulting registered catalog location may use a '/tree/' URL for the + // target instead of the '/blob/' URL format returned by getCatalogUrl. + // To correctly detect already-imported repositories regardless of which + // format was persisted, the '/tree/' variant is also checked here. + // This branch can be removed once catalog locations are consistently + // stored using the same '/blob/' format as getCatalogUrl returns. + alreadyImported = alreadyImportedRepositoriesLocationTargets.has( + catalogUrl.replace('/blob/', '/tree/'), + ); + } + + return !alreadyImported; + }); + + sortRepos(notImportedYetRepositories); + + const gitRepositoryResponse: SCMRepositoryResponse = { + repositories: notImportedYetRepositories, + errors, + totalCount: notImportedYetRepositories.length, + }; + + return await formatResponse(deps, gitRepositoryResponse, checkStatus); } export async function findRepositoriesByOrganization( @@ -81,17 +124,18 @@ export async function findRepositoriesByOrganization( `Getting all repositories for org "${orgName}" - (search,page,size)=(${search},${pageNumber},${pageSize})..`, ); - const glReposByOrg = await deps.gitApiService - .getOrgRepositoriesFromIntegrations( + const glReposByOrg = + await deps.gitApiService.getOrgRepositoriesFromIntegrations( orgName, search, pageNumber, pageSize, userTokens, - ) - .then(response => formatResponse(deps, response, checkStatus)); + ); + + sortRepos(glReposByOrg.repositories); - return glReposByOrg; + return formatResponse(deps, glReposByOrg, checkStatus); } function sortRepos(repoList: Components.Schemas.Repository[]) { @@ -173,8 +217,6 @@ async function formatResponse( }); } - sortRepos(repoList); - return { statusCode: 200, responseBody: { diff --git a/workspaces/bulk-import/plugins/bulk-import/src/components/AddRepositories/RepositoriesTable.tsx b/workspaces/bulk-import/plugins/bulk-import/src/components/AddRepositories/RepositoriesTable.tsx index 374eee183a9..37263ea8933 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/components/AddRepositories/RepositoriesTable.tsx +++ b/workspaces/bulk-import/plugins/bulk-import/src/components/AddRepositories/RepositoriesTable.tsx @@ -95,9 +95,6 @@ export const RepositoriesTable = ({ const { loading, data, error } = useRepositories({ showOrganizations, orgName: drawerOrganization, - page: (drawerOrganization ? drawerPage : localPage) + 1, - querySize: rowsPerPage, - searchString, approvalTool: values.approvalTool, }); @@ -135,12 +132,21 @@ export const RepositoriesTable = ({ ? evaluateRowForRepo(tableData, values.repositories) : evaluateRowForOrg(tableData, values.repositories); + if (searchString) { + filteredRows = filteredRows?.filter(row => { + const targetToSearch = showOrganizations ? row.orgName : row.repoName; + return targetToSearch + ?.toLowerCase() + .includes(searchString.toLowerCase()); + }); + } + filteredRows = [...(filteredRows || [])]?.sort( getComparator('asc', 'repoName'), ); return filteredRows; - }, [tableData, values?.repositories, showOrganizations]); + }, [tableData, values?.repositories, showOrganizations, searchString]); const updateSelectedRepositories = useCallback( (newSelected: AddedRepositories) => { @@ -153,14 +159,20 @@ export const RepositoriesTable = ({ ); const effectivePage = drawerOrganization ? drawerPage : page || 0; + + const paginatedData = useMemo(() => { + const startIndex = effectivePage * rowsPerPage; + return filteredData?.slice(startIndex, startIndex + rowsPerPage) || []; + }, [filteredData, effectivePage, rowsPerPage]); + // Avoid a layout jump when reaching the last page with empty rows. const emptyRows = - effectivePage > 0 ? Math.max(0, rowsPerPage - tableData.length) : 0; + effectivePage > 0 ? Math.max(0, rowsPerPage - paginatedData.length) : 0; const handleClickAllForRepositoriesTable = (drawer?: boolean) => { let newSelectedRows: AddedRepositories = { ...selected }; - const rowsEligibleForSelection = filteredData.filter( + const rowsEligibleForSelection = paginatedData.filter( r => !values.excludedRepositories[r.id], ); const isAllSelected = rowsEligibleForSelection.every( @@ -269,8 +281,8 @@ export const RepositoriesTable = ({ [tableData, selected], ); const selectedRepositoriesOnActivePage = useMemo( - () => filterSelectedRepositoriesOnActivePage(filteredData, selected), - [filteredData, selected], + () => filterSelectedRepositoriesOnActivePage(paginatedData, selected), + [paginatedData, selected], ); const getRowCount = () => { if (drawerOrganization) { @@ -330,7 +342,7 @@ export const RepositoriesTable = ({