diff --git a/workspaces/bulk-import/.changeset/small-games-live.md b/workspaces/bulk-import/.changeset/small-games-live.md new file mode 100644 index 00000000000..d828296cf18 --- /dev/null +++ b/workspaces/bulk-import/.changeset/small-games-live.md @@ -0,0 +1,31 @@ +--- +'@red-hat-developer-hub/backstage-plugin-bulk-import-backend': minor +'@red-hat-developer-hub/backstage-plugin-bulk-import': minor +--- + +## On Behalf of User Access + +This release introduces the ability for the Bulk Import plugin to fetch repository and organization listings **on behalf of the signed-in user**, using their OAuth credentials rather than relying solely on server-side integration credentials (GitHub App, PAT, or GitLab token). + +### What Changed + +**Backend (`bulk-import-backend`)** + +- Added a new `GET /api/bulk-import/scm-hosts` endpoint that returns the configured GitHub and GitLab integration host URLs as a `SCMHostList` object, enabling the frontend to discover which hosts to request OAuth tokens for. +- The `GET /repositories` and `GET /organizations/{organizationName}/repositories` endpoints now **require** the `x-scm-tokens` request header — a JSON map of SCM host base URL to user OAuth token. Requests that omit this header, or supply an empty or oversized header, are rejected with HTTP 401. This ensures repository listings are always scoped to the signed-in user's access and never fall back to server-wide integration credentials. +- The `x-scm-tokens` header is stripped from the request immediately upon receipt, before the permission check and before any audit event is created, so OAuth token values are never persisted in audit logs. +- When user tokens are provided for GitHub, the Octokit response cache is intentionally disabled to prevent cross-user ETag cache leakage. Server-side credential paths are not affected. +- Introduced a shared `GitApiService` interface and common SCM types (`SCMOrganization`, `SCMRepository`, `SCMFetchError`, etc.) to unify the GitHub and GitLab service implementations under a consistent contract. + +**Frontend (`bulk-import`)** + +- The plugin now has a **soft dependency** on `@backstage/integration-react`'s `ScmAuthApi`. If the API is registered in the application, the plugin automatically requests OAuth tokens for each configured SCM host and passes them to the backend to enable user-scoped repository listings. +- Added `getSCMHosts()` to the `BulkImportAPI` interface with a corresponding `GET /api/bulk-import/scm-hosts` client call, used to discover host URLs before requesting user tokens. +- User OAuth tokens are transmitted to the backend via the `X-SCM-Tokens` request header as a JSON-encoded map. +- If the SCM OAuth integration is not configured or token collection fails for all hosts, the repository list query is **blocked** on the frontend and the hook surfaces a descriptive error. This prevents the frontend from firing a request that will always be rejected with 401. + +### Required Configuration + +The GitHub and/or GitLab OAuth provider must be configured in the Backstage application for repository listing to work. Deployments that previously relied on server-side credentials alone for the repository list view must add an SCM OAuth provider to continue using this feature. + +If `ScmAuthApi` is not registered or tokens cannot be obtained for any configured SCM host, users will see an error prompting them to configure the SCM OAuth integration. diff --git a/workspaces/bulk-import/e2e-tests/app.test.ts b/workspaces/bulk-import/e2e-tests/app.test.ts index 66ba2059d55..85e8951de8b 100644 --- a/workspaces/bulk-import/e2e-tests/app.test.ts +++ b/workspaces/bulk-import/e2e-tests/app.test.ts @@ -22,11 +22,13 @@ import { mockBulkImportDryRunResponse, mockBulkImportImportsResponse, mockBulkImportRepositoriesResponse, + mockBulkImportSCMHostsResponse, mockImportByRepoData, mockImportByRepoFrontendData, mockImportsData, mockImportsDryRunData, mockRepositoriesData, + mockSCMHostsData, } from './utils/apiUtils'; import { getPreviewSidebarSnapshots, @@ -51,6 +53,24 @@ test.describe('Bulk Import', () => { context = await browser.newContext(); sharedPage = await context.newPage(); + // The backend's GET /repositories and GET /organizations/{org}/repositories + // endpoints require the X-SCM-Tokens header (HTTP 401 otherwise). In a real + // deployment, the frontend obtains these tokens from the configured GitHub / + // GitLab OAuth provider via ScmAuthApi and sends them with every listing + // request. See plugins/bulk-import/README.md → "Required OAuth Configuration". + // + // In these e2e tests we bypass that requirement in two steps: + // 1. Mock GET /api/bulk-import/scm-hosts to return empty host arrays. + // The useRepositories hook hits the `!urls?.length → return undefined` + // early-return path, so tokenFetchError stays undefined and the query + // fires without any X-SCM-Tokens header. + // 2. Mock GET /api/bulk-import/repositories* with a 200 response so + // Playwright intercepts the token-free request before it ever reaches + // the real backend's 401 guard. + // + // This lets us focus on UI behaviour without needing a real OAuth provider + // set up in the test environment. + await mockBulkImportSCMHostsResponse(sharedPage, mockSCMHostsData); await mockBulkImportRepositoriesResponse(sharedPage, mockRepositoriesData); await sharedPage.goto('/'); diff --git a/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts b/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts index 99e37e1569a..3aba4b81c2c 100644 --- a/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts +++ b/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts @@ -19,13 +19,14 @@ import { Page } from '@playwright/test'; * API route patterns for bulk import endpoints */ export const ApiRoutes = { + scmHosts: '**/api/bulk-import/scm-hosts*', repositories: '**/api/bulk-import/repositories*', importsDryRun: '**/api/bulk-import/imports?dryRun=true*', imports: '**/api/bulk-import/imports', byRepoBackend: - '**/api/bulk-import/import/by-repo?repo=https://github.com/test-org/backend-service*', + '**/api/bulk-import/import/by-repo?repo=https%3A%2F%2Fgithub.com%2Ftest-org%2Fbackend-service*', byRepoFrontend: - '**/api/bulk-import/import/by-repo?repo=https://github.com/test-org/frontend-app*', + '**/api/bulk-import/import/by-repo?repo=https%3A%2F%2Fgithub.com%2Ftest-org%2Ffrontend-app*', } as const; type ApiRouteKey = keyof typeof ApiRoutes; @@ -91,6 +92,12 @@ export const mockBulkImportByRepoFrontendResponse = ( status = 200, ) => mockApiResponse(page, ApiRoutes.byRepoFrontend, responseData, status); +export const mockBulkImportSCMHostsResponse = ( + page: Page, + responseData: object, + status = 200, +) => mockApiResponse(page, ApiRoutes.scmHosts, responseData, status); + // Reusable repository definitions const repositories = { backendService: { @@ -135,6 +142,23 @@ const repositories = { }, } as const; +/** + * Mock data for SCM hosts response. + * Returns empty host lists so the `useRepositories` hook hits the early-return + * path (`!urls?.length → return undefined`) and never attempts token collection. + * This means `tokenFetchError` stays `undefined`, the query is enabled, and the + * frontend fires a request without `X-SCM-Tokens`. + * + * In production the backend would reject such a request with HTTP 401, but the + * Playwright route mock for `ApiRoutes.repositories` intercepts the request + * before it reaches the backend, so the e2e tests still receive the mocked + * repository data regardless of the missing header. + */ +export const mockSCMHostsData = { + github: [], + gitlab: [], +}; + /** Mock data for repositories list response */ export const mockRepositoriesData = { errors: [], diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/README.md b/workspaces/bulk-import/plugins/bulk-import-backend/README.md index 40026e1e498..41f63162cba 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/README.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/README.md @@ -301,13 +301,15 @@ The Bulk Import Backend plugin emits audit events for various operations. Events - **`ping`**: tracks `GET` requests to the `/ping` endpoint, which allows to make sure the bulk import backend is up and running. -- **`org-read`**: tracks `GET` requests to the `/organizations` endpoint, which returns the list of organizations accessible from all configured GitHub Integrations. +- **`scm-hosts-read`**: tracks `GET` requests to the `/scm-hosts-read` endpoint, which returns the list of configured GitHub and GitLab integration host URLs. + +- **`org-read`**: tracks `GET` requests to the `/organizations` endpoint, which returns the list of organizations accessible from all configured SCM Integrations (GitHub and GitLab). Filter on `queryType`. - **`all`**: tracks fetching all organizations. (GET `/organizations`) - **`by-query`**: tracks fetching organization filtered by the query parameter 'search'. (GET `/organizations`) -- **`repo-read`**: tracks `GET` requests to the endpoint, which returns the list of repositories accessible from all configured GitHub Integrations. +- **`repo-read`**: tracks `GET` requests to the endpoint, which returns the list of repositories accessible from all configured SCM Integrations (GitHub and GitLab). Filter on `queryType`. - **`all`**: tracks fetching a list of all repositories accessible by Backstage Github Integrations. (GET `/repositories`) @@ -343,8 +345,43 @@ Example: The bulk import backend plugin provides a REST API to bulk import catalog entities into the catalog. The API is available at the `/api/bulk-import` endpoint. -As a prerequisite, you need to add at least one GitHub Integration (using either a GitHub token or a GitHub App or both) in your app-config YAML file (or a local `app-config.local.yaml` file). -See https://backstage.io/docs/integrations/github/locations/#configuration and https://backstage.io/docs/integrations/github/github-apps/#including-in-integrations-config for more details. +As a prerequisite, you need to add at least one SCM integration in your app-config YAML file (or a local `app-config.local.yaml` file): + +- **GitHub**: Configure a GitHub integration using a GitHub token or a GitHub App (or both). See the [GitHub Locations](https://backstage.io/docs/integrations/github/locations/#configuration) and [GitHub Apps](https://backstage.io/docs/integrations/github/github-apps/#including-in-integrations-config) documentation for details. +- **GitLab** _(optional)_: Configure a GitLab integration if you want to import from GitLab repositories. See the [GitLab Locations](https://backstage.io/docs/integrations/gitlab/locations/) documentation for details. + +### On Behalf of User Access + +The plugin supports fetching repository and organization listings **on behalf of the signed-in user**, using their OAuth credentials rather than the server-wide integration credentials (GitHub App, PAT, or GitLab token). + +#### How It Works + +1. The frontend calls `GET /api/bulk-import/scm-hosts` to retrieve the list of configured SCM integration host URLs, grouped by provider (`github` and `gitlab`). +2. For each host, the frontend requests an OAuth token from the Backstage `ScmAuthApi` (provided by `@backstage/integration-react`). +3. The collected tokens are sent to the backend via the **required** `x-scm-tokens` request header — a JSON-encoded string whose value, when parsed, maps each integration base URL to the user's OAuth token (e.g. `{"https://github.com":"ghp_xxx"}`). +4. The backend uses these user tokens to call the GitHub or GitLab APIs on behalf of the user, so the repository listings reflect what the signed-in user can personally access. + +#### Required OAuth Configuration + +The `x-scm-tokens` header is **required** for `GET /repositories` and `GET /organizations/{organizationName}/repositories`. Requests that omit the header, supply an empty token map, or send a header that exceeds the allowed size are rejected with **HTTP 401**. + +A GitHub and/or GitLab OAuth provider must therefore be configured in the Backstage application for these endpoints to work. Refer to the [Backstage GitHub auth docs](https://backstage.io/docs/auth/github/provider) and [GitLab auth docs](https://backstage.io/docs/auth/gitlab/provider) for setup instructions. + +> **Migration note:** Deployments that previously relied solely on server-side credentials (GitHub App, PAT, or GitLab token) for the repository list view must now also configure an SCM OAuth provider. The server-side credentials are still used for all other operations (import creation, status checks, etc.) and are unaffected by this change. + +#### Security Note + +When user tokens are provided for GitHub, the Octokit response cache is intentionally disabled to prevent cross-user ETag cache leakage. Server-side credential paths are not affected. + +The `x-scm-tokens` header is stripped from the request immediately upon receipt — before the permission check and before any audit event is created — so OAuth token values are never persisted in audit logs. + +#### New API Endpoint + +| Method | Path | Description | +| ------ | ---------------------------- | ------------------------------------------------------------------------------------------- | +| `GET` | `/api/bulk-import/scm-hosts` | Returns configured GitHub and GitLab integration host base URLs as an `SCMHostList` object. | + +The existing `GET /repositories` and `GET /organizations/{organizationName}/repositories` endpoints now **require** the `x-scm-tokens` header. See the [API documentation](api-docs/README.md) for the full request/response specification. ## REST API diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts b/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts index f2f23be8a87..6f4992978d2 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts @@ -122,6 +122,10 @@ export const DEFAULT_TEST_HANDLERS: RestHandler< ); }), + rest.get(`${LOCAL_ADDR}/orgs/my-org-1/repos`, (_, res, ctx) => { + return res(ctx.status(200), ctx.json([])); + }), + rest.get(`${LOCAL_ADDR}/orgs/my-ent-org-1/repos`, (_, res, ctx) => { return res( ctx.status(200), diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES index 2e9aae06917..16f725df267 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES @@ -19,6 +19,7 @@ Models/PullRequest.md Models/Repository.md Models/RepositoryList.md Models/Repository_importStatus.md +Models/SCMHostList.md Models/ScaffolderTask.md Models/Source.md Models/SourceImport.md diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/ManagementApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/ManagementApi.md index b2ff8388f1d..7e516bfba9e 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/ManagementApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/ManagementApi.md @@ -4,9 +4,32 @@ All URIs are relative to *http://localhost:7007/api/bulk-import* | Method | HTTP request | Description | |------------- | ------------- | -------------| +| [**findAllSCMHosts**](ManagementApi.md#findAllSCMHosts) | **GET** /scm-hosts | Retrieve the SCM Integration hosts | | [**ping**](ManagementApi.md#ping) | **GET** /ping | Check the health of the Bulk Import backend router | + +# **findAllSCMHosts** +> SCMHostList findAllSCMHosts() + +Retrieve the SCM Integration hosts + +### Parameters +This endpoint does not need any parameter. + +### Return type + +[**SCMHostList**](../Models/SCMHostList.md) + +### Authorization + +[BearerAuth](../README.md#BearerAuth) + +### HTTP request headers + +- **Content-Type**: Not defined +- **Accept**: application/json + # **ping** > ping_200_response ping() diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md index b8b1d09c925..80828016861 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md @@ -38,7 +38,7 @@ Fetch Organizations accessible by Backstage Github Integrations # **findRepositoriesByOrganization** -> RepositoryList findRepositoriesByOrganization(organizationName, checkImportStatus, pagePerIntegration, sizePerIntegration, search, approvalTool) +> RepositoryList findRepositoriesByOrganization(organizationName, checkImportStatus, pagePerIntegration, sizePerIntegration, search, approvalTool, x-scm-tokens) Fetch Repositories in the specified GitHub organization, provided it is accessible by any of the configured GitHub Integrations. @@ -52,6 +52,7 @@ Fetch Repositories in the specified GitHub organization, provided it is accessib | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | | **search** | **String**| returns only the items that match the search string | [optional] [default to null] | | **approvalTool** | **String**| the approvalTool to use | [optional] [default to GIT] | +| **x-scm-tokens** | **String**| **Required.** JSON-encoded map of SCM host URL to user OAuth token. Used to fetch repositories on behalf of the signed-in user. The value must be a JSON object whose keys are SCM integration base URLs and whose values are OAuth bearer tokens (e.g. `{"https://github.com":"ghp_xxx"}`). Requests that omit this header, supply an empty object, or exceed 4 KB are rejected with HTTP 401. | [required] [default to null] | ### Return type diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md index 801f72290ac..668ac3aa25c 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md @@ -9,7 +9,7 @@ All URIs are relative to *http://localhost:7007/api/bulk-import* # **findAllRepositories** -> RepositoryList findAllRepositories(checkImportStatus, pagePerIntegration, sizePerIntegration, search, approvalTool) +> RepositoryList findAllRepositories(checkImportStatus, pagePerIntegration, sizePerIntegration, search, approvalTool, x-scm-tokens) Fetch Organization Repositories accessible by Backstage Github Integrations @@ -22,6 +22,7 @@ Fetch Organization Repositories accessible by Backstage Github Integrations | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | | **search** | **String**| returns only the items that match the search string | [optional] [default to null] | | **approvalTool** | **String**| the approvalTool to use | [optional] [default to GIT] | +| **x-scm-tokens** | **String**| **Required.** JSON-encoded map of SCM host URL to user OAuth token. Used to fetch repositories on behalf of the signed-in user. The value must be a JSON object whose keys are SCM integration base URLs and whose values are OAuth bearer tokens (e.g. `{"https://github.com":"ghp_xxx"}`). Requests that omit this header, supply an empty object, or exceed 4 KB are rejected with HTTP 401. | [required] [default to null] | ### Return type diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/SCMHostList.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/SCMHostList.md new file mode 100644 index 00000000000..9b4766a1307 --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/SCMHostList.md @@ -0,0 +1,10 @@ +# SCMHostList +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **github** | **List** | | [optional] [default to null] | +| **gitlab** | **List** | | [optional] [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md index a2232b16620..a8675b49b2c 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md @@ -19,7 +19,8 @@ All URIs are relative to *http://localhost:7007/api/bulk-import* *ImportApi* | [**findImportStatusByRepo**](Apis/ImportApi.md#findimportstatusbyrepo) | **GET** /import/by-repo | Get Import Status by repository | *ImportApi* | [**findOrchestratorImportStatusByRepo**](Apis/ImportApi.md#findorchestratorimportstatusbyrepo) | **GET** /orchestrator-import/by-repo | Get Import Status by repository | *ImportApi* | [**findTaskImportStatusByRepo**](Apis/ImportApi.md#findtaskimportstatusbyrepo) | **GET** /task-import/by-repo | Get Import Status by repository | -| *ManagementApi* | [**ping**](Apis/ManagementApi.md#ping) | **GET** /ping | Check the health of the Bulk Import backend router | +| *ManagementApi* | [**findAllSCMHosts**](Apis/ManagementApi.md#findallscmhosts) | **GET** /scm-hosts | Retrieve the SCM Integration hosts | +*ManagementApi* | [**ping**](Apis/ManagementApi.md#ping) | **GET** /ping | Check the health of the Bulk Import backend router | | *OrganizationApi* | [**findAllOrganizations**](Apis/OrganizationApi.md#findallorganizations) | **GET** /organizations | Fetch Organizations accessible by Backstage Github Integrations | *OrganizationApi* | [**findRepositoriesByOrganization**](Apis/OrganizationApi.md#findrepositoriesbyorganization) | **GET** /organizations/{organizationName}/repositories | Fetch Repositories in the specified GitHub organization, provided it is accessible by any of the configured GitHub Integrations. | | *RepositoryApi* | [**findAllRepositories**](Apis/RepositoryApi.md#findallrepositories) | **GET** /repositories | Fetch Organization Repositories accessible by Backstage Github Integrations | @@ -44,6 +45,7 @@ All URIs are relative to *http://localhost:7007/api/bulk-import* - [Repository](./Models/Repository.md) - [RepositoryList](./Models/RepositoryList.md) - [Repository_importStatus](./Models/Repository_importStatus.md) + - [SCMHostList](./Models/SCMHostList.md) - [ScaffolderTask](./Models/ScaffolderTask.md) - [Source](./Models/Source.md) - [SourceImport](./Models/SourceImport.md) diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/scripts/openapi.sh b/workspaces/bulk-import/plugins/bulk-import-backend/scripts/openapi.sh index f863c6cb09a..6eae9017111 100755 --- a/workspaces/bulk-import/plugins/bulk-import-backend/scripts/openapi.sh +++ b/workspaces/bulk-import/plugins/bulk-import-backend/scripts/openapi.sh @@ -36,7 +36,7 @@ cat < "${OPENAPI_DOC_JS_FILE}" // prettier-ignore EOF echo 'const OPENAPI = `' >> "${OPENAPI_DOC_JS_FILE}" -cat ./src/schema/openapi.json | sed 's/\\n/\\\\n/g' >> "${OPENAPI_DOC_JS_FILE}" +cat ./src/schema/openapi.json | sed 's/\\n/\\\\n/g' | sed 's/\\"/\\\\"/g' >> "${OPENAPI_DOC_JS_FILE}" echo '`' >> "${OPENAPI_DOC_JS_FILE}" echo "export const openApiDocument = JSON.parse(OPENAPI);" >> "${OPENAPI_DOC_JS_FILE}" rm -f ./src/schema/openapi.json diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts index ead43ae1c54..601d3dd3eb4 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts @@ -13,6 +13,11 @@ import type { declare namespace Components { export interface HeaderParameters { apiVersionHeaderParam?: Parameters.ApiVersionHeaderParam; + xSCMTokensHeaderParam?: /** + * example: + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} + */ + Parameters.XSCMTokensHeaderParam; } namespace Parameters { export type ApiVersionHeaderParam = "v1" | "v2"; @@ -26,6 +31,11 @@ declare namespace Components { export type SizeQueryParam = number; export type SortColumnQueryParam = "repository.name" | "repository.organization" | "repository.url" | "lastUpdate" | "status"; export type SortOrderQueryParam = "asc" | "desc"; + /** + * example: + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} + */ + export type XSCMTokensHeaderParam = string; } export interface QueryParameters { pagePerIntegrationQueryParam?: Parameters.PagePerIntegrationQueryParam; @@ -239,6 +249,26 @@ declare namespace Components { pagePerIntegration?: number; sizePerIntegration?: number; } + /** + * SCM Host List + */ + export interface SCMHostList { + github?: string[]; + gitlab?: string[]; + } + /** + * SCM Token Map + * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. + * + * example: + * { + * "https://github.com": "ghp_xxx", + * "https://ghe.example.com": "ghe_yyy" + * } + */ + export interface SCMTokenMap { + [name: string]: string; + } /** * Scaffolder Task */ @@ -467,12 +497,24 @@ declare namespace Paths { } } namespace FindAllRepositories { + export interface HeaderParameters { + "x-scm-tokens"?: /** + * example: + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} + */ + Parameters.XScmTokens; + } namespace Parameters { export type ApprovalTool = string; export type CheckImportStatus = boolean; export type PagePerIntegration = number; export type Search = string; export type SizePerIntegration = number; + /** + * example: + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} + */ + export type XScmTokens = string; } export interface QueryParameters { checkImportStatus?: Parameters.CheckImportStatus; @@ -486,6 +528,11 @@ declare namespace Paths { export type $500 = /* Repository List */ Components.Schemas.RepositoryList; } } + namespace FindAllSCMHosts { + namespace Responses { + export type $200 = /* SCM Host List */ Components.Schemas.SCMHostList; + } + } namespace FindAllTaskImports { export interface HeaderParameters { "api-version"?: Parameters.ApiVersion; @@ -547,6 +594,13 @@ declare namespace Paths { } } namespace FindRepositoriesByOrganization { + export interface HeaderParameters { + "x-scm-tokens"?: /** + * example: + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} + */ + Parameters.XScmTokens; + } namespace Parameters { export type ApprovalTool = string; export type CheckImportStatus = boolean; @@ -554,6 +608,11 @@ declare namespace Paths { export type PagePerIntegration = number; export type Search = string; export type SizePerIntegration = number; + /** + * example: + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} + */ + export type XScmTokens = string; } export interface PathParameters { organizationName: Parameters.OrganizationName; @@ -606,6 +665,14 @@ export interface OperationMethods { data?: any, config?: AxiosRequestConfig ): OperationResponse + /** + * findAllSCMHosts - Retrieve the SCM Integration hosts + */ + 'findAllSCMHosts'( + parameters?: Parameters | null, + data?: any, + config?: AxiosRequestConfig + ): OperationResponse /** * findAllOrganizations - Fetch Organizations accessible by Backstage Github Integrations */ @@ -618,7 +685,7 @@ export interface OperationMethods { * findRepositoriesByOrganization - Fetch Repositories in the specified GitHub organization, provided it is accessible by any of the configured GitHub Integrations. */ 'findRepositoriesByOrganization'( - parameters?: Parameters | null, + parameters?: Parameters | null, data?: any, config?: AxiosRequestConfig ): OperationResponse @@ -626,7 +693,7 @@ export interface OperationMethods { * findAllRepositories - Fetch Organization Repositories accessible by Backstage Github Integrations */ 'findAllRepositories'( - parameters?: Parameters | null, + parameters?: Parameters | null, data?: any, config?: AxiosRequestConfig ): OperationResponse @@ -739,6 +806,16 @@ export interface PathsDictionary { config?: AxiosRequestConfig ): OperationResponse } + ['/scm-hosts']: { + /** + * findAllSCMHosts - Retrieve the SCM Integration hosts + */ + 'get'( + parameters?: Parameters | null, + data?: any, + config?: AxiosRequestConfig + ): OperationResponse + } ['/organizations']: { /** * findAllOrganizations - Fetch Organizations accessible by Backstage Github Integrations @@ -754,7 +831,7 @@ export interface PathsDictionary { * findRepositoriesByOrganization - Fetch Repositories in the specified GitHub organization, provided it is accessible by any of the configured GitHub Integrations. */ 'get'( - parameters?: Parameters | null, + parameters?: Parameters | null, data?: any, config?: AxiosRequestConfig ): OperationResponse @@ -764,7 +841,7 @@ export interface PathsDictionary { * findAllRepositories - Fetch Organization Repositories accessible by Backstage Github Integrations */ 'get'( - parameters?: Parameters | null, + parameters?: Parameters | null, data?: any, config?: AxiosRequestConfig ): OperationResponse @@ -892,6 +969,8 @@ export type OrganizationList = Components.Schemas.OrganizationList; export type PullRequest = Components.Schemas.PullRequest; export type Repository = Components.Schemas.Repository; export type RepositoryList = Components.Schemas.RepositoryList; +export type SCMHostList = Components.Schemas.SCMHostList; +export type SCMTokenMap = Components.Schemas.SCMTokenMap; export type ScaffolderTask = Components.Schemas.ScaffolderTask; export type Source = Components.Schemas.Source; export type SourceImport = Components.Schemas.SourceImport; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts index 78973e72058..532a4735c7a 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -// GENERATED FILE. DO NOT EDIT. // eslint-disable // prettier-ignore @@ -81,6 +80,37 @@ const OPENAPI = ` } } }, + "/scm-hosts": { + "get": { + "operationId": "findAllSCMHosts", + "summary": "Retrieve the SCM Integration hosts", + "security": [ + { + "BearerAuth": [] + } + ], + "tags": [ + "Management" + ], + "responses": { + "200": { + "description": "List of Integrations available to the Bulk Import plugin", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCMHostList" + }, + "examples": { + "multipleRepos": { + "$ref": "#/components/examples/scmHosts" + } + } + } + } + } + } + } + }, "/organizations": { "get": { "operationId": "findAllOrganizations", @@ -183,6 +213,9 @@ const OPENAPI = ` }, { "$ref": "#/components/parameters/approvalToolParam" + }, + { + "$ref": "#/components/parameters/xSCMTokensHeaderParam" } ], "responses": { @@ -252,6 +285,9 @@ const OPENAPI = ` }, { "$ref": "#/components/parameters/approvalToolParam" + }, + { + "$ref": "#/components/parameters/xSCMTokensHeaderParam" } ], "responses": { @@ -992,6 +1028,16 @@ const OPENAPI = ` "default": "v1" } }, + "xSCMTokensHeaderParam": { + "in": "header", + "name": "x-scm-tokens", + "description": "Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. The value must be a JSON string whose structure matches SCMTokenMap (keys are SCM base URLs, values are OAuth bearer tokens).\\n", + "required": false, + "schema": { + "type": "string", + "example": "{\\"https://github.com\\":\\"ghp_xxx\\",\\"https://ghe.example.com\\":\\"ghe_yyy\\"}" + } + }, "pagePerIntegrationQueryParam": { "in": "query", "name": "pagePerIntegration", @@ -1097,6 +1143,36 @@ const OPENAPI = ` } }, "schemas": { + "SCMTokenMap": { + "title": "SCM Token Map", + "type": "object", + "description": "Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server.\\n", + "additionalProperties": { + "type": "string" + }, + "example": { + "https://github.com": "ghp_xxx", + "https://ghe.example.com": "ghe_yyy" + } + }, + "SCMHostList": { + "title": "SCM Host List", + "type": "object", + "properties": { + "github": { + "type": "array", + "items": { + "type": "string" + } + }, + "gitlab": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "OrganizationList": { "title": "Organization List", "type": "object", @@ -1556,6 +1632,18 @@ const OPENAPI = ` } }, "examples": { + "scmHosts": { + "summary": "Multiple scmHosts", + "value": { + "github": [ + "https://github.com", + "https://ghe.example.com" + ], + "gitlab": [ + "https://gitlab.com" + ] + } + }, "multipleOrgs": { "summary": "Multiple organizations", "value": { diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts index 6f7401bbadf..1815ddba3e6 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts @@ -378,4 +378,104 @@ describe('GithubApiService tests', () => { totalCount: 0, }); }); + + describe('with userTokens', () => { + it('uses the user-token path for getRepositoriesFromIntegrations when a matching host token is provided', async () => { + octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ + data: ghRepos, + }); + octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ + data: [], + }); + + const result = await githubApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + { 'https://github.com': 'user-oauth-token' }, + ); + + expect(result.repositories).toEqual(ghRepos); + expect(result.errors).toEqual([]); + // getAllCredentials is not called in the user-token path + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('returns empty repositories when userTokens is provided but no host matches an integration', async () => { + const result = await githubApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + { 'https://some-other-host.com': 'user-oauth-token' }, + ); + + expect(result.repositories).toEqual([]); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('falls back to server credentials when userTokens is undefined', async () => { + octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ + data: ghRepos, + }); + octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ + data: [], + }); + + const result = await githubApiService.getRepositoriesFromIntegrations(); + + // Server credentials path is used — getAllCredentials IS called + expect(mockGetAllCredentials).toHaveBeenCalled(); + expect(result.repositories).toEqual(ghRepos); + }); + + it('falls back to server credentials when userTokens is an empty object', async () => { + octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ + data: ghRepos, + }); + octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ + data: [], + }); + + const result = await githubApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + {}, + ); + + expect(mockGetAllCredentials).toHaveBeenCalled(); + expect(result.repositories).toEqual(ghRepos); + }); + + it('uses the user-token path for getOrgRepositoriesFromIntegrations when a matching host token is provided', async () => { + octokit.rest.repos.listForOrg.mockReturnValue({ data: ghRepos }); + + const result = await githubApiService.getOrgRepositoriesFromIntegrations( + 'my-org', + undefined, + undefined, + undefined, + { 'https://github.com': 'user-oauth-token' }, + ); + + expect(result.repositories).toEqual(ghRepos); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('returns empty org repositories when userTokens provided but no host matches', async () => { + const result = await githubApiService.getOrgRepositoriesFromIntegrations( + 'my-org', + undefined, + undefined, + undefined, + { 'https://some-other-host.com': 'user-oauth-token' }, + ); + + expect(result.repositories).toEqual([]); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + }); }); diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts index e7132758b81..88532325dbb 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts @@ -21,6 +21,7 @@ import type { import type { Config } from '@backstage/config'; import { DefaultGithubCredentialsProvider, + GithubCredentials, GithubIntegrationConfig, ScmIntegrations, } from '@backstage/integration'; @@ -34,6 +35,7 @@ import { extractLocationOwnerMap, logErrorIfNeeded, } from '../helpers'; +import { GitApiService } from '../scm/GitApiService'; import { DefaultPageNumber, DefaultPageSize, @@ -72,7 +74,7 @@ import { fetchFromMatchedIntegration, } from './utils/utils'; -export class GithubApiService { +export class GithubApiService implements GitApiService { private readonly logger: LoggerService; private readonly integrations: ScmIntegrations; public readonly githubCredentialsProvider: CustomGithubCredentialsProvider; @@ -93,6 +95,35 @@ export class GithubApiService { this.cache = cacheService; } + private get integrationDeps() { + return { + logger: this.logger, + cache: this.cache, + githubCredentialsProvider: this.githubCredentialsProvider, + }; + } + + private get executionDeps() { + return { + ...this.integrationDeps, + config: this.config, + }; + } + + private buildRepositoryResponse( + repositories: Map, + result: { data: number[]; errors: Map }, + pageSize: number, + ): GithubRepositoryResponse { + const repoList = Array.from(repositories.values()); + const totalCount = computeTotalCount(repoList, result.data, pageSize); + return { + repositories: repoList, + errors: Array.from(result.errors.values()), + totalCount, + }; + } + async getCredentials(repoUrl: string): Promise<{ token: string }> { const provider = DefaultGithubCredentialsProvider.fromIntegrations( this.integrations, @@ -113,11 +144,7 @@ export class GithubApiService { errors?: GithubFetchError[]; }> { const { data, errors } = await fetchFromMatchedIntegration( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, repoUrl, async (octokit, gitUrl) => { @@ -147,17 +174,13 @@ export class GithubApiService { } async getOrganizationsFromIntegrations( - search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + search?: string, ): Promise { const orgs = new Map(); const result = await fetchFromAllIntegrations( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, { dataFetcher: async ( @@ -216,19 +239,82 @@ export class GithubApiService { }; } + private async fetchUserTokenGithubIntegrationRepositories( + userTokens: Record, + fetcher: ( + octokit: Octokit, + credential: GithubCredentials, + errors: Map, + ) => Promise<{ totalCount?: number }>, + ): Promise<{ errors: GithubFetchError[]; totalCount: number | undefined }> { + const allErrors: GithubFetchError[] = []; + let totalCount: number | undefined; + const ghConfigs = this.integrations.github.list().map(i => i.config); + for (const ghConfig of ghConfigs) { + const hostUrl = `https://${ghConfig.host}`; + const token = userTokens[hostUrl]; + if (!token) { + continue; + } + // Synthesize a token credential — same shape buildOcto already accepts + const userCredential: GithubCredentials = { token, type: 'token' }; + + // Intentionally no cache: user-token requests must not share the + // server-credential ETag cache to avoid cross-user data leakage. + const userOctokit = buildOcto( + { logger: this.logger, cache: undefined }, + { credential: userCredential }, + ghConfig.apiBaseUrl, + ); + if (!userOctokit) { + continue; + } + const dataFetchErrors = new Map(); + const result = await fetcher( + userOctokit, + userCredential, + dataFetchErrors, + ); + totalCount = (totalCount ?? 0) + (result.totalCount ?? 0); + dataFetchErrors.forEach(err => allErrors.push(err)); + } + return { errors: allErrors, totalCount }; + } + async getOrgRepositoriesFromIntegrations( orgName: string, search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + userTokens?: Record, ): Promise { const repositories = new Map(); + + // When a user token is present, build a user-scoped Octokit per integration + // config and call addGithubTokenOrgRepositories with it. This scopes the + // listing to repositories the user has access to within the given organization. + // When absent, fall through to the existing server-credential path. + if (userTokens && Object.keys(userTokens).length > 0) { + const { errors: allErrors, totalCount } = + await this.fetchUserTokenGithubIntegrationRepositories( + userTokens, + (userOctokit, userCredential, dataFetchErrors) => + addGithubTokenOrgRepositories( + { logger: this.logger }, + userOctokit, + userCredential, + orgName, + repositories, + dataFetchErrors, + { search, pageNumber, pageSize }, + ), + ); + const repoList = Array.from(repositories.values()); + return { repositories: repoList, errors: allErrors, totalCount }; + } + const result = await fetchFromAllIntegrations( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, { dataFetcher: async ( @@ -287,13 +373,7 @@ export class GithubApiService { }, ); - const repoList = Array.from(repositories.values()); - const totalCount = computeTotalCount(repoList, result.data, pageSize); - return { - repositories: repoList, - errors: Array.from(result.errors?.values() ?? []), - totalCount, - }; + return this.buildRepositoryResponse(repositories, result, pageSize); } /** @@ -305,14 +385,34 @@ export class GithubApiService { search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + userTokens?: Record, ): Promise { const repositories = new Map(); + + // When a user token is present, build a user-scoped Octokit per integration + // config and call addGithubTokenRepositories with it. This makes + // listForAuthenticatedUser return only repos the user personally has access to. + // When absent, fall through to the existing server-credential path. + if (userTokens && Object.keys(userTokens).length > 0) { + const { errors: allErrors, totalCount } = + await this.fetchUserTokenGithubIntegrationRepositories( + userTokens, + (userOctokit, userCredential, dataFetchErrors) => + addGithubTokenRepositories( + { logger: this.logger }, + userOctokit, + userCredential, + repositories, + dataFetchErrors, + { search, pageNumber, pageSize }, + ), + ); + const repoList = Array.from(repositories.values()); + return { repositories: repoList, errors: allErrors, totalCount }; + } + const result = await fetchFromAllIntegrations( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, { dataFetcher: async ( @@ -363,13 +463,7 @@ export class GithubApiService { }, ); - const repoList = Array.from(repositories.values()); - const totalCount = computeTotalCount(repoList, result.data, pageSize); - return { - repositories: repoList, - errors: Array.from(result.errors?.values() ?? []), - totalCount, - }; + return this.buildRepositoryResponse(repositories, result, pageSize); } async filterLocationsAccessibleFromIntegrations( @@ -380,44 +474,36 @@ export class GithubApiService { const allAccessibleAppOrgs = new Set(); const allAccessibleTokenOrgs = new Set(); const allAccessibleUsernames = new Set(); - await fetchFromAllIntegrations( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, - this.integrations, - { - dataFetcher: async ( - octokit: Octokit, - credential: ExtendedGithubCredentials, - ghConfig: GithubIntegrationConfig, - ) => { - if (isGithubAppCredential(credential)) { - const appOrgMap = await getAllAppOrgs( - this.githubCredentialsProvider, - ghConfig, - credential.accountLogin, - ); - for (const [_, ghOrg] of appOrgMap) { - allAccessibleAppOrgs.add(ghOrg.name); - } - } else { - // find authenticated GitHub owner... - const username = (await octokit.rest.users.getAuthenticated())?.data - ?.login; - if (username) { - allAccessibleUsernames.add(username); - } - // ... along with orgs accessible from the token auth - (await octokit.paginate(octokit.rest.orgs.listForAuthenticatedUser)) - ?.map(org => org.login) - ?.forEach(orgName => allAccessibleTokenOrgs.add(orgName)); + await fetchFromAllIntegrations(this.integrationDeps, this.integrations, { + dataFetcher: async ( + octokit: Octokit, + credential: ExtendedGithubCredentials, + ghConfig: GithubIntegrationConfig, + ) => { + if (isGithubAppCredential(credential)) { + const appOrgMap = await getAllAppOrgs( + this.githubCredentialsProvider, + ghConfig, + credential.accountLogin, + ); + for (const [_, ghOrg] of appOrgMap) { + allAccessibleAppOrgs.add(ghOrg.name); } - return {}; - }, + } else { + // find authenticated GitHub owner... + const username = (await octokit.rest.users.getAuthenticated())?.data + ?.login; + if (username) { + allAccessibleUsernames.add(username); + } + // ... along with orgs accessible from the token auth + (await octokit.paginate(octokit.rest.orgs.listForAuthenticatedUser)) + ?.map(org => org.login) + ?.forEach(orgName => allAccessibleTokenOrgs.add(orgName)); + } + return {}; }, - ); + }); return locationUrls.filter(loc => { if (!locationGitOwnerMap.has(loc)) { @@ -444,11 +530,7 @@ export class GithubApiService { prBranch?: string; }> { const { data } = await fetchFromMatchedIntegration( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, repoUrl, async (octokit, gitUrl) => { @@ -483,11 +565,7 @@ export class GithubApiService { body?: string, ): Promise { await fetchFromMatchedIntegration( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, repoUrl, async (octokit, gitUrl) => { @@ -611,11 +689,7 @@ export class GithubApiService { }> { const branchName = getBranchName(this.config); const { data } = await fetchFromMatchedIntegration( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, input.repoUrl, async (octokit, gitUrl) => { @@ -647,11 +721,7 @@ export class GithubApiService { }, ): Promise { const { data } = await fetchFromMatchedIntegration( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, input.repoUrl, async (octokit, gitUrl) => { @@ -699,12 +769,7 @@ export class GithubApiService { const errors: any[] = []; const result = await executeFunctionOnFirstSuccessfulIntegration( - { - logger: this.logger, - cache: this.cache, - config: this.config, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.executionDeps, this.integrations, { repoUrl: input.repoUrl, @@ -880,14 +945,9 @@ export class GithubApiService { repoUrl: string; defaultBranch?: string; fileName: string; - }) { + }): Promise { const fileExists = await executeFunctionOnFirstSuccessfulIntegration( - { - logger: this.logger, - cache: this.cache, - config: this.config, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.executionDeps, this.integrations, { repoUrl: input.repoUrl, @@ -925,14 +985,9 @@ export class GithubApiService { gitUrl: gitUrlParse.GitUrl; comment: string; }, - ) { + ): Promise { await executeFunctionOnFirstSuccessfulIntegration( - { - logger: this.logger, - cache: this.cache, - config: this.config, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.executionDeps, this.integrations, { repoUrl: input.repoUrl, @@ -973,14 +1028,9 @@ export class GithubApiService { async deleteImportBranch(input: { repoUrl: string; gitUrl: gitUrlParse.GitUrl; - }) { + }): Promise { await executeFunctionOnFirstSuccessfulIntegration( - { - logger: this.logger, - cache: this.cache, - config: this.config, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.executionDeps, this.integrations, { repoUrl: input.repoUrl, @@ -1006,14 +1056,9 @@ export class GithubApiService { ); } - async isRepoEmpty(input: { repoUrl: string }) { + async isRepoEmpty(input: { repoUrl: string }): Promise { return await executeFunctionOnFirstSuccessfulIntegration( - { - logger: this.logger, - cache: this.cache, - config: this.config, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.executionDeps, this.integrations, { repoUrl: input.repoUrl, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts index 684ff122a70..dcee5b075ac 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts @@ -14,86 +14,18 @@ * limitations under the License. */ -import type { SerializedError } from '@backstage/errors'; import type { GithubCredentials, GithubCredentialsProvider, } from '@backstage/integration'; -// From https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-organizations -export type GithubOrganization = { - name: string; - id: number; - description?: string; - url?: string; - html_url?: string; - repos_url?: string; - events_url?: string; - hooks_url?: string; - issues_url?: string; - members_url?: string; - public_members_url?: string; - avatar_url?: string; - public_repos?: number; - total_private_repos?: number; - /** - * Number of internal repositories, accessible to all members in a GH enterprise - */ - owned_private_repos?: number; -}; - -export type GithubRepository = { - name: string; - /** - * The full name of the repository in the form of owner/repo - */ - full_name: string; - /** - * The API url to the repository - */ - url: string; - /** - * The HTML URL to the repository - */ - html_url: string; - /** - * The default "main" branch of the repository to place the `catalog-info.yaml` file into - */ - default_branch: string; - /** - * The date-time the repository was last updated at - */ - updated_at?: string | null; -}; - -/** - * The type of credentials produced by the credential provider. - * - * @public - */ - -export type GithubFetchError = - | { - type: 'app'; - appId: number; - error: SerializedError; - } - | { - type: 'token'; - error: SerializedError; - }; - -export type GithubOrganizationResponse = { - organizations: GithubOrganization[]; - errors: GithubFetchError[]; - totalCount?: number; -}; - -export type GithubRepositoryResponse = { - repositories: GithubRepository[]; - errors: GithubFetchError[]; - totalCount?: number; -}; +export type { + SCMFetchError as GithubFetchError, + SCMOrganization as GithubOrganization, + SCMOrganizationResponse as GithubOrganizationResponse, + SCMRepository as GithubRepository, + SCMRepositoryResponse as GithubRepositoryResponse, +} from '../scm/types'; export type AppCredentialFetchResult = AppCredential | AppCredentialError; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/ghUtils.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/ghUtils.ts index dec1b99dde9..0105b38c647 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/ghUtils.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/ghUtils.ts @@ -39,7 +39,7 @@ const RESPONSE_CACHE_TTL_MILLIS = 60 * 60 * 1000; export function buildOcto( deps: { logger: LoggerService; - cache: CacheService; + cache: CacheService | undefined; }, input: { credential: ExtendedGithubCredentials; @@ -81,7 +81,7 @@ export function buildOcto( function registerHooks( deps: { logger: LoggerService; - cache: CacheService; + cache: CacheService | undefined; }, octokit: Octokit, ) { @@ -98,7 +98,7 @@ function registerHooks( // Use ETag from in-memory cache if available const cacheKey = extractCacheKey(options); const existingEtag = await deps.cache - .get(cacheKey) + ?.get(cacheKey) ?.then((val?: any) => val?.etag); if (existingEtag) { options.headers['If-None-Match'] = existingEtag; @@ -115,7 +115,7 @@ function registerHooks( ); // If we get a successful response, the resource has changed, so update the in-memory cache const cacheKey = extractCacheKey(options); - await deps.cache.set( + await deps.cache?.set( cacheKey, { etag: response.headers.etag, @@ -136,7 +136,7 @@ function registerHooks( } // "304 Not Modified" means that the resource hasn't changed, // and we should have a version of it in the cache - return await deps.cache.get(extractCacheKey(options)); + return await deps.cache?.get(extractCacheKey(options)); }); } diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts index 3887e706645..cec7800700d 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts @@ -289,7 +289,11 @@ describe('GitlabApiService tests', () => { paginationInfo: { total: 1 }, }); - const result = await gitlabApiService.getOrganizationsFromIntegrations('A'); + const result = await gitlabApiService.getOrganizationsFromIntegrations( + 1, + 20, + 'A', + ); const expected_response = { organizations: [ @@ -435,4 +439,117 @@ describe('GitlabApiService tests', () => { totalCount: 0, }); }); + + describe('with userTokens', () => { + const glRepos = [ + { + id: '1', + name: 'A', + path_with_namespace: 'backstage/A', + _links: { self: 'https://gitlab.com/api/v4/projects/1' }, + web_url: 'https://gitlab.com/backstage/A', + default_branch: 'master', + }, + { + id: '2', + name: 'B', + path_with_namespace: 'backstage/B', + _links: { self: 'https://gitlab.com/api/v4/projects/2' }, + web_url: 'https://gitlab.com/backstage/B', + default_branch: 'main', + }, + ]; + + it('uses the user-token path for getRepositoriesFromIntegrations when a matching host token is provided', async () => { + gitlabkit.Projects.all.mockReturnValue({ + data: glRepos, + paginationInfo: { total: 2 }, + }); + + const result = await gitlabApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + { 'https://gitlab.com': 'user-gitlab-oauth-token' }, + ); + + expect(result.repositories).toHaveLength(2); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('returns empty repositories when userTokens is provided but no host matches an integration', async () => { + const result = await gitlabApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + { 'https://some-other-host.com': 'user-gitlab-oauth-token' }, + ); + + expect(result.repositories).toEqual([]); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('falls back to server credentials when userTokens is undefined', async () => { + gitlabkit.Projects.all.mockReturnValue({ + data: glRepos, + paginationInfo: { total: 2 }, + }); + + await gitlabApiService.getRepositoriesFromIntegrations(); + + // Server credentials path — getAllCredentials IS called + expect(mockGetAllCredentials).toHaveBeenCalled(); + }); + + it('falls back to server credentials when userTokens is an empty object', async () => { + gitlabkit.Projects.all.mockReturnValue({ + data: glRepos, + paginationInfo: { total: 2 }, + }); + + await gitlabApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + {}, + ); + + expect(mockGetAllCredentials).toHaveBeenCalled(); + }); + + it('uses the user-token path for getOrgRepositoriesFromIntegrations when a matching host token is provided', async () => { + gitlabkit.Groups.allProjects.mockReturnValue({ + data: glRepos, + paginationInfo: { total: 2 }, + }); + + const result = await gitlabApiService.getOrgRepositoriesFromIntegrations( + 'my-group', + undefined, + undefined, + undefined, + { 'https://gitlab.com': 'user-gitlab-oauth-token' }, + ); + + expect(result.repositories).toHaveLength(2); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('returns empty org repositories when userTokens provided but no host matches', async () => { + const result = await gitlabApiService.getOrgRepositoriesFromIntegrations( + 'my-group', + undefined, + undefined, + undefined, + { 'https://some-other-host.com': 'user-gitlab-oauth-token' }, + ); + + expect(result.repositories).toEqual([]); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + }); }); diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts index 732b4a95afc..58f26dee320 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts @@ -34,6 +34,7 @@ import { extractLocationOwnerMap, logErrorIfNeeded, } from '../helpers'; +import { GitApiService } from '../scm/GitApiService'; import { DefaultPageNumber, DefaultPageSize, @@ -63,12 +64,12 @@ import { getCredentialsForConfig, } from './utils/utils'; -export class GitlabApiService { +export class GitlabApiService implements GitApiService { private readonly logger: LoggerService; private readonly integrations: ScmIntegrations; private readonly gitlabCredentialsProvider: CustomGitlabCredentialsProvider; private readonly config: Config; - // Cache for storing ETags (used for efficient caching of unchanged data returned by GitHub) + // Cache for storing ETags (used for efficient caching of unchanged data returned by GitLab) private readonly cache: CacheService; constructor( logger: LoggerService, @@ -147,9 +148,9 @@ export class GitlabApiService { } async getOrganizationsFromIntegrations( - search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + search?: string, ): Promise { const groups = new Map(); const result = await fetchFromAllIntegrations( @@ -200,13 +201,67 @@ export class GitlabApiService { }; } + private async fetchUserTokenGitlabIntegrationRepositories( + userTokens: Record, + fetcher: ( + glApi: InstanceType>, + credential: ExtendedGitlabCredentials, + errors: Map, + ) => Promise<{ totalCount?: number }>, + ): Promise<{ errors: GitlabFetchError[]; totalCount: number | undefined }> { + const allErrors: GitlabFetchError[] = []; + let totalCount: number | undefined; + const glConfigs = this.integrations.gitlab.list().map(i => i.config); + for (const glConfig of glConfigs) { + const normalizedBase = glConfig.baseUrl.replace(/\/$/, ''); + const token = + userTokens[normalizedBase] ?? userTokens[`https://${glConfig.host}`]; + + if (!token) { + continue; + } + const userCredential: ExtendedGitlabCredentials = { token }; + const userGitlab = buildGitlab( + { logger: this.logger, cache: this.cache }, + { credential: userCredential }, + glConfig.baseUrl, // NOTE: baseUrl, NOT apiBaseUrl (Gitlab constructor takes the base, not /api/v4) + ); + const dataFetchErrors = new Map(); + const result = await fetcher(userGitlab, userCredential, dataFetchErrors); + totalCount = (totalCount ?? 0) + (result.totalCount ?? 0); + dataFetchErrors.forEach(err => allErrors.push(err)); + } + return { errors: allErrors, totalCount }; + } + async getOrgRepositoriesFromIntegrations( orgName: string, search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + userTokens?: Record, ): Promise { const orgRepositories = new Map(); + + if (userTokens && Object.keys(userTokens).length > 0) { + const { errors: allErrors, totalCount } = + await this.fetchUserTokenGitlabIntegrationRepositories( + userTokens, + (userGitlab, userCredential, dataFetchErrors) => + addGitlabTokenOrgRepositories( + { logger: this.logger }, + userGitlab, + userCredential, + orgName, + orgRepositories, + dataFetchErrors, + { search, pageNumber, pageSize }, + ), + ); + const repoList = Array.from(orgRepositories.values()); + return { repositories: repoList, errors: allErrors, totalCount }; + } + const result = await fetchFromAllIntegrations( { logger: this.logger, @@ -266,8 +321,28 @@ export class GitlabApiService { search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + userTokens?: Record, ): Promise { const repositories = new Map(); + + if (userTokens && Object.keys(userTokens).length > 0) { + const { errors: allErrors, totalCount } = + await this.fetchUserTokenGitlabIntegrationRepositories( + userTokens, + (userGitlab, userCredential, dataFetchErrors) => + addGitlabTokenRepositories( + { logger: this.logger }, + userGitlab, + userCredential, + repositories, + dataFetchErrors, + { search, pageNumber, pageSize }, + ), + ); + const repoList = Array.from(repositories.values()); + return { repositories: repoList, errors: allErrors, totalCount }; + } + const result = await fetchFromAllIntegrations( { logger: this.logger, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/types.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/types.ts index b2aeed9ef7c..46016ed00ad 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/types.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/types.ts @@ -14,75 +14,18 @@ * limitations under the License. */ -import type { SerializedError } from '@backstage/errors'; import type { GitlabCredentials, GitlabCredentialsProvider, } from '@backstage/integration'; -// From https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-organizations -export type GitlabOrganization = { - name: string; - id: number; - description?: string; - url?: string; - html_url?: string; - avatar_url?: string; - public_repos?: number; - total_private_repos?: number; - /** - * Number of internal repositories, accessible to all members in a GH enterprise - */ - owned_private_repos?: number; -}; - -export type GitlabRepository = { - // id?: string; - name: string; - /** - * The full name of the repository in the form of owner/repo, should be the path_with_namespace property in gitlab - */ - full_name: string; - /** - * The API url to the repository - */ - url: string; - /** - * The HTML URL to the repository, web_url in gitlab - */ - html_url: string; - /** - * The default "main" branch of the repository to place the `catalog-info.yaml` file into - */ - default_branch: string; - /** - * The date-time the repository was last updated at - */ - updated_at?: string | null; -}; - -/** - * The type of credentials produced by the credential provider. - * - * @public - */ - -export type GitlabFetchError = { - type: 'token'; - error: SerializedError; -}; - -export type GitlabOrganizationResponse = { - organizations: GitlabOrganization[]; - errors: GitlabFetchError[]; - totalCount?: number; -}; - -export type GitlabRepositoryResponse = { - repositories: GitlabRepository[]; - errors: GitlabFetchError[]; - totalCount?: number; -}; +export type { + SCMFetchError as GitlabFetchError, + SCMOrganization as GitlabOrganization, + SCMOrganizationResponse as GitlabOrganizationResponse, + SCMRepository as GitlabRepository, + SCMRepositoryResponse as GitlabRepositoryResponse, +} from '../scm/types'; export type ExtendedGitlabCredentials = GitlabCredentials; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml b/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml index 49ade45c0ad..100d78474fb 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml @@ -49,6 +49,24 @@ paths: example: status: 'ok' + /scm-hosts: + get: + operationId: findAllSCMHosts + summary: Retrieve the SCM Integration hosts + security: + - BearerAuth: [] + tags: [Management] + responses: + 200: + description: List of Integrations available to the Bulk Import plugin + content: + application/json: + schema: + $ref: '#/components/schemas/SCMHostList' + examples: + multipleRepos: + $ref: '#/components/examples/scmHosts' + /organizations: get: operationId: findAllOrganizations @@ -105,6 +123,7 @@ paths: - $ref: '#/components/parameters/sizePerIntegrationQueryParam' - $ref: '#/components/parameters/searchQueryParam' - $ref: '#/components/parameters/approvalToolParam' + - $ref: '#/components/parameters/xSCMTokensHeaderParam' responses: 200: description: Org Repository list was fetched successfully with no errors @@ -143,6 +162,7 @@ paths: - $ref: '#/components/parameters/sizePerIntegrationQueryParam' - $ref: '#/components/parameters/searchQueryParam' - $ref: '#/components/parameters/approvalToolParam' + - $ref: '#/components/parameters/xSCMTokensHeaderParam' responses: 200: description: Repository list was fetched successfully with no errors @@ -590,6 +610,18 @@ components: type: string enum: ['v1', 'v2'] default: 'v1' + xSCMTokensHeaderParam: + in: header + name: x-scm-tokens + description: > + Optional JSON-encoded map of SCM host URL to user authentication token. + Used to fetch repositories on behalf of the user for each configured SCM host. + The value must be a JSON string whose structure matches SCMTokenMap + (keys are SCM base URLs, values are OAuth bearer tokens). + required: false + schema: + type: string + example: '{"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"}' pagePerIntegrationQueryParam: in: query @@ -684,6 +716,35 @@ components: default: GIT schemas: + SCMTokenMap: + title: SCM Token Map + type: object + description: > + Map of SCM integration base URL to the user's OAuth access token for that + host. Keys must match the base URLs returned by GET /scm-hosts (e.g. + https://github.com or https://gitlab.corp.com). Values must be + non-empty OAuth bearer tokens scoped to the minimum required access + (read-only repository listing). Unknown keys are silently ignored by + the server. + additionalProperties: + type: string + example: + 'https://github.com': 'ghp_xxx' + 'https://ghe.example.com': 'ghe_yyy' + + SCMHostList: + title: SCM Host List + type: object + properties: + github: + type: array + items: + type: string + gitlab: + type: array + items: + type: string + OrganizationList: title: Organization List type: object @@ -1024,6 +1085,15 @@ components: description: Backstage Permissions Framework JWT examples: + scmHosts: + summary: Multiple scmHosts + value: + github: + - 'https://github.com' + - 'https://ghe.example.com' + gitlab: + - 'https://gitlab.com' + multipleOrgs: summary: Multiple organizations value: diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts new file mode 100644 index 00000000000..8c59552330b --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts @@ -0,0 +1,132 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { LoggerService } from '@backstage/backend-plugin-api'; + +import gitUrlParse from 'git-url-parse'; + +import { + SCMFetchError, + SCMOrganizationResponse, + SCMRepository, + SCMRepositoryResponse, +} from './types'; + +export interface GitApiService { + getRepositoryFromIntegrations( + repoUrl: string, + ): Promise<{ repository?: SCMRepository; errors?: SCMFetchError[] }>; + + getOrganizationsFromIntegrations( + pageNumber: number, + pageSize: number, + search?: string, + ): Promise; + + getOrgRepositoriesFromIntegrations( + orgName: string, + search?: string, + pageNumber?: number, + pageSize?: number, + userTokens?: Record, + ): Promise; + + getRepositoriesFromIntegrations( + search?: string, + pageNumber?: number, + pageSize?: number, + userTokens?: Record, + ): Promise; + + filterLocationsAccessibleFromIntegrations( + locationUrls: string[], + ): Promise; + + getPullRequest( + repoUrl: string, + pullRequestNumber: number, + ): Promise<{ + title?: string; + body?: string; + merged?: boolean; + lastUpdated?: string; + prSha?: string; + prBranch?: string; + }>; + + findImportOpenPr( + logger: LoggerService, + input: { + repoUrl: string; + includeCatalogInfoContent?: boolean; + }, + ): Promise<{ + prNum?: number; + prUrl?: string; + prTitle?: string; + prBody?: string; + prCatalogInfoContent?: string; + lastUpdate?: string; + }>; + + getCatalogInfoFile( + logger: LoggerService, + input: { + repoUrl: string; + prNumber: number; + prHeadSha: string; + }, + ): Promise; + + submitPrToRepo( + logger: LoggerService, + input: { + repoUrl: string; + gitUrl: gitUrlParse.GitUrl; + defaultBranch?: string; + prTitle: string; + prBody: string; + catalogInfoContent: string; + }, + ): Promise<{ + prUrl?: string; + prNumber?: number; + hasChanges?: boolean; + lastUpdate?: string; + errors?: string[]; + }>; + + hasFileInRepo(input: { + repoUrl: string; + defaultBranch?: string; + fileName: string; + }): Promise; + + closeImportPR( + logger: LoggerService, + input: { + repoUrl: string; + gitUrl: gitUrlParse.GitUrl; + comment: string; + }, + ): Promise; + + deleteImportBranch(input: { + repoUrl: string; + gitUrl: gitUrlParse.GitUrl; + }): Promise; + + isRepoEmpty(input: { repoUrl: string }): Promise; +} diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/types.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/types.ts new file mode 100644 index 00000000000..05269275a28 --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/types.ts @@ -0,0 +1,91 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { SerializedError } from '@backstage/errors'; + +/** + * Error type returned when fetching SCM resources from an integration. + * + * @public + */ + +export type SCMFetchError = + | { + type: 'app'; + appId: number; + error: SerializedError; + } + | { + type: 'token'; + error: SerializedError; + }; + +export type SCMOrganization = { + name: string; + id: number; + description?: string; + url?: string; + html_url?: string; + repos_url?: string; + events_url?: string; + hooks_url?: string; + issues_url?: string; + members_url?: string; + public_members_url?: string; + avatar_url?: string; + public_repos?: number; + total_private_repos?: number; + /** + * Number of internal repositories, accessible to all members in a GH enterprise + */ + owned_private_repos?: number; +}; + +export type SCMOrganizationResponse = { + organizations: SCMOrganization[]; + errors: SCMFetchError[]; + totalCount?: number; +}; + +export type SCMRepository = { + name: string; + /** + * The full name of the repository in the form of owner/repo + */ + full_name: string; + /** + * The API url to the repository + */ + url: string; + /** + * The HTML URL to the repository + */ + html_url: string; + /** + * The default "main" branch of the repository to place the `catalog-info.yaml` file into + */ + default_branch: string; + /** + * The date-time the repository was last updated at + */ + updated_at?: string | null; +}; + +export type SCMRepositoryResponse = { + repositories: SCMRepository[]; + errors: SCMFetchError[]; + totalCount?: number; +}; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/bulkImports.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/bulkImports.ts index 08da00f7324..47e86159fd9 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/bulkImports.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/bulkImports.ts @@ -53,6 +53,7 @@ import { paginateArray, parseGitURLForApprovalTool, } from '../../../helpers'; +import { GitApiService } from '../../../scm/GitApiService'; import { DefaultPageNumber, DefaultPageSize, @@ -348,7 +349,7 @@ function findImportCandidates( } async function createPR( - gitApiService: GithubApiService | GitlabApiService, + gitApiService: GitApiService, logger: LoggerService, req: Components.Schemas.ImportRequest, gitUrl: gitUrlParse.GitUrl, @@ -942,11 +943,13 @@ export async function findTaskImportStatusByRepo( repository.approvalTool as unknown as Components.Schemas.ApprovalTool; const pullRequest = await parsePullOrMergeRequestInfo( data.state?.checkpoints, - deps.gitlabApiService, - deps.githubApiService, - approvalTool, - deps.logger, - repoUrl, + { + githubApiService: deps.githubApiService, + gitlabApiService: deps.gitlabApiService, + approvalTool, + logger: deps.logger, + repoUrl, + }, ); if (pullRequest && approvalTool === 'GITLAB') { result.gitlab = { pullRequest }; @@ -1078,11 +1081,19 @@ export async function findOrchestratorImportStatusByRepo( async function parsePullOrMergeRequestInfo( checkpoints: Record, - githubApiService: GitlabApiService, - gitlabApiService: GithubApiService, - approvalTool: Components.Schemas.ApprovalTool, - logger: LoggerService, - repoUrl: string, + { + githubApiService, + gitlabApiService, + approvalTool, + logger, + repoUrl, + }: { + githubApiService: GithubApiService; + gitlabApiService: GitlabApiService; + approvalTool: Components.Schemas.ApprovalTool; + logger: LoggerService; + repoUrl: string; + }, ): Promise { // return errors ? if (approvalTool !== 'GITLAB' && approvalTool !== 'GIT') { @@ -1139,7 +1150,7 @@ export async function deleteImportByRepo( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, repoUrl: string, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/importStatus.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/importStatus.ts index 2a20495b566..3f0346ce9aa 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/importStatus.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/importStatus.ts @@ -23,14 +23,13 @@ import { getCatalogUrl, } from '../../../catalog/catalogUtils'; import type { Components } from '../../../generated/openapi'; -import type { GithubApiService } from '../../../github'; -import { GitlabApiService } from '../../../gitlab'; +import { GitApiService } from '../../../scm/GitApiService'; export async function getImportStatusFromLocations( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, repoUrl: string, @@ -59,7 +58,7 @@ async function getImportStatusWithCheckerFn( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, repoUrl: string, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/organization/organizations.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/organization/organizations.ts index 913621c5cc7..cd3da566099 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/organization/organizations.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/organization/organizations.ts @@ -17,11 +17,8 @@ import type { LoggerService } from '@backstage/backend-plugin-api'; import type { Components } from '../../../generated/openapi'; -import type { - GithubApiService, - GithubOrganizationResponse, -} from '../../../github'; -import { GitlabApiService, GitlabOrganizationResponse } from '../../../gitlab'; +import { GitApiService } from '../../../scm/GitApiService'; +import { SCMOrganizationResponse } from '../../../scm/types'; import { DefaultPageNumber, DefaultPageSize, @@ -30,7 +27,7 @@ import { export async function findAllOrganizations( logger: LoggerService, - gitApiService: GithubApiService | GitlabApiService, + gitApiService: GitApiService, search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, @@ -43,9 +40,9 @@ export async function findAllOrganizations( const allOrgsAccessible = await gitApiService.getOrganizationsFromIntegrations( - search, pageNumber, pageSize, + search, ); const errorList: string[] = []; @@ -80,9 +77,7 @@ export async function findAllOrganizations( }; } -function extractOrgMap( - allOrgsAccessible: GithubOrganizationResponse | GitlabOrganizationResponse, -) { +function extractOrgMap(allOrgsAccessible: SCMOrganizationResponse) { const orgMap = new Map(); for (const org of allOrgsAccessible.organizations ?? []) { let totalRepoCount: number | undefined; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts index cc0e3faddb8..ca6d8f4656b 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts @@ -25,11 +25,16 @@ import { startBackendServer, } from '../../../../__fixtures__/testUtils'; +// Token header used across GitLab repo-listing tests. +const GL_USER_TOKENS = JSON.stringify({ + 'https://gitlab.com': 'test-gl-token', +}); + describe('repositories', () => { const useTestData = setupTest(); describe('GET /repositories', () => { - it('returns 200 when repositories are fetched without errors', async () => { + it('returns 401 when X-SCM-Tokens header is absent', async () => { const { mockCatalogClient } = useTestData(); const backendServer = await startBackendServer( mockCatalogClient, @@ -40,6 +45,22 @@ describe('repositories', () => { .get('/api/bulk-import/repositories') .query({ approvalTool: 'GITLAB' }); + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); + }); + + it('returns 200 when repositories are fetched without errors', async () => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); + expect(response.status).toEqual(200); expect(response.body).toEqual({ approvalTool: 'GITLAB', @@ -96,7 +117,8 @@ describe('repositories', () => { const response = await request(backendServer) .get('/api/bulk-import/repositories') - .query({ approvalTool: 'GITLAB' }); + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); expect(response.status).toEqual(500); expect(response.body).toEqual({ @@ -107,6 +129,21 @@ describe('repositories', () => { }); describe('GET /organizations/{org}/repositories', () => { + it('returns 401 when X-SCM-Tokens header is absent', async () => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/organizations/my-ent-org-1/repositories') + .query({ approvalTool: 'GITLAB' }); + + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); + }); + it('returns 200 when repositories are fetched without errors', async () => { const { mockCatalogClient } = useTestData(); const backendServer = await startBackendServer( @@ -116,7 +153,8 @@ describe('repositories', () => { let response = await request(backendServer) .get('/api/bulk-import/organizations/my-ent-org-1/repositories') - .query({ approvalTool: 'GITLAB' }); + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); expect(response.status).toEqual(200); expect(response.body).toEqual({ @@ -138,7 +176,8 @@ describe('repositories', () => { response = await request(backendServer) .get('/api/bulk-import/organizations/my-ent-org-2/repositories') - .query({ approvalTool: 'GITLAB' }); + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); expect(response.status).toEqual(200); expect(response.body).toEqual({ @@ -169,7 +208,8 @@ describe('repositories', () => { response = await request(backendServer) .get('/api/bulk-import/organizations/my-ent-org--no-repos/repositories') - .query({ approvalTool: 'GITLAB' }); + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); expect(response.status).toEqual(200); expect(response.body).toEqual({ @@ -201,7 +241,8 @@ describe('repositories', () => { const orgReposResp = await request(backendServer) .get('/api/bulk-import/organizations/some-org/repositories') - .query({ approvalTool: 'GITLAB' }); + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); expect(orgReposResp.status).toEqual(500); expect(orgReposResp.body).toEqual({ diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts index e65f33254fd..9690db163ed 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts @@ -26,11 +26,19 @@ import { startBackendServer, } from '../../../../__fixtures__/testUtils'; +// Token header used across repo-listing tests. +// With user tokens provided for github.com only, the GitHub App path +// (enterprise.github.com) is skipped — results are scoped to the user's +// own OAuth credential. +const GH_USER_TOKENS = JSON.stringify({ + 'https://github.com': 'test-user-token', +}); + describe('repositories', () => { const useTestData = setupTest(); describe('GET /repositories', () => { - it('returns 200 when repositories are fetched without errors', async () => { + it('returns 401 when X-SCM-Tokens header is absent', async () => { const { mockCatalogClient } = useTestData(); const backendServer = await startBackendServer( mockCatalogClient, @@ -41,7 +49,25 @@ describe('repositories', () => { '/api/bulk-import/repositories', ); + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); + }); + + it('returns 200 when repositories are fetched without errors', async () => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); + expect(response.status).toEqual(200); + // With user-scoped tokens, only repos accessible via the user's OAuth + // credential (GET /user/repos) are returned. The GitHub App path for + // enterprise.github.com is intentionally skipped. expect(response.body).toEqual({ errors: [], repositories: [ @@ -54,15 +80,6 @@ describe('repositories', () => { organization: 'octocat', url: 'http://localhost:8765/octocat/animated-happiness', }, - { - defaultBranch: 'master', - errors: [], - id: 'octocat/Hello-World', - lastUpdate: '2011-01-26T19:14:43Z', - name: 'Hello-World', - organization: 'octocat', - url: 'http://localhost:8765/octocat/Hello-World', - }, { defaultBranch: 'master', errors: [], @@ -73,18 +90,17 @@ describe('repositories', () => { url: 'http://localhost:8765/my-user/Lorem-Ipsum', }, ], - totalCount: 3, + totalCount: 2, }); }); - it('returns 200 with the errors in the body when repositories are fetched, but errors have occurred', async () => { + it('returns 500 when the user token call fails for all integrations', async () => { const { server, mockCatalogClient } = useTestData(); const backendServer = await startBackendServer( mockCatalogClient, AuthorizeResult.ALLOW, ); - // change the response to 'GET /user/repos' - // to simulate an error retrieving list of orgs from GH Token. + // Override GET /user/repos to simulate a failed user-token credential. server.use( rest.get(`${LOCAL_ADDR}/user/repos`, (_, res, ctx) => res( @@ -94,25 +110,13 @@ describe('repositories', () => { ), ); - const response = await request(backendServer).get( - '/api/bulk-import/repositories', - ); + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); - expect(response.status).toEqual(200); + expect(response.status).toEqual(500); expect(response.body).toEqual({ errors: ['Github Token auth did not succeed'], - repositories: [ - { - defaultBranch: 'master', - errors: [], - id: 'octocat/Hello-World', - lastUpdate: '2011-01-26T19:14:43Z', - name: 'Hello-World', - organization: 'octocat', - url: 'http://localhost:8765/octocat/Hello-World', - }, - ], - totalCount: 1, }); }); @@ -122,35 +126,49 @@ describe('repositories', () => { mockCatalogClient, AuthorizeResult.ALLOW, ); - // change the responses to simulate error retrieving list of repos from all GH integrations. + // Override all GH integration endpoints to simulate total failure. addHandlersForGHTokenAppErrors(server); - const reposResp = await request(backendServer).get( - '/api/bulk-import/repositories', - ); + const reposResp = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); expect(reposResp.status).toEqual(500); + // With user tokens only the user-token path runs; the GitHub App path is + // not taken, so only the user-token error is surfaced. expect(reposResp.body).toEqual({ - errors: [ - 'Github App auth returned an error', - 'Github Token auth did not succeed', - ], + errors: ['Github Token auth did not succeed'], }); }); }); describe('GET /organizations/{org}/repositories', () => { - it('returns 200 when repositories are fetched without errors', async () => { + it('returns 401 when X-SCM-Tokens header is absent', async () => { const { mockCatalogClient } = useTestData(); const backendServer = await startBackendServer( mockCatalogClient, AuthorizeResult.ALLOW, ); - let response = await request(backendServer).get( + const response = await request(backendServer).get( '/api/bulk-import/organizations/my-ent-org-1/repositories', ); + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); + }); + + it('returns 200 when repositories are fetched without errors', async () => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + let response = await request(backendServer) + .get('/api/bulk-import/organizations/my-ent-org-1/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); + expect(response.status).toEqual(200); expect(response.body).toEqual({ errors: [], @@ -168,9 +186,9 @@ describe('repositories', () => { totalCount: 1, }); - response = await request(backendServer).get( - '/api/bulk-import/organizations/my-ent-org-2/repositories', - ); + response = await request(backendServer) + .get('/api/bulk-import/organizations/my-ent-org-2/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); expect(response.status).toEqual(200); expect(response.body).toEqual({ @@ -198,9 +216,9 @@ describe('repositories', () => { totalCount: 2, }); - response = await request(backendServer).get( - '/api/bulk-import/organizations/my-ent-org--no-repos/repositories', - ); + response = await request(backendServer) + .get('/api/bulk-import/organizations/my-ent-org--no-repos/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); expect(response.status).toEqual(200); expect(response.body).toEqual({ @@ -216,12 +234,12 @@ describe('repositories', () => { mockCatalogClient, AuthorizeResult.ALLOW, ); - // change the response to simulate an error retrieving list from GH Token. + // Override GH integration endpoints to simulate failure. addHandlersForGHTokenAppErrors(server); - const orgReposResp = await request(backendServer).get( - '/api/bulk-import/organizations/some-org/repositories', - ); + const orgReposResp = await request(backendServer) + .get('/api/bulk-import/organizations/some-org/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); expect(orgReposResp.status).toEqual(500); expect(orgReposResp.body).toEqual({ diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts index b5dcab249b5..44554f485a9 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts @@ -21,11 +21,8 @@ import gitUrlParse from 'git-url-parse'; import { CatalogHttpClient } from '../../../catalog/catalogHttpClient'; import type { Components } from '../../../generated/openapi'; -import type { - GithubApiService, - GithubRepositoryResponse, -} from '../../../github'; -import { GitlabApiService, GitlabRepositoryResponse } from '../../../gitlab'; +import { GitApiService } from '../../../scm/GitApiService'; +import type { SCMRepositoryResponse } from '../../../scm/types'; import { DefaultPageNumber, DefaultPageSize, @@ -37,21 +34,22 @@ export async function findAllRepositories( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, reqParams?: { - approvalTool?: string; search?: string; checkStatus?: boolean; pageNumber?: number; pageSize?: number; + userTokens?: Record; }, ): Promise> { const search = reqParams?.search; const checkStatus = reqParams?.checkStatus ?? false; const pageNumber = reqParams?.pageNumber ?? DefaultPageNumber; const pageSize = reqParams?.pageSize ?? DefaultPageSize; + const userTokens = reqParams?.userTokens; deps.logger.debug( `Getting all repositories - (search,page,size)=('${ search ?? '' @@ -59,7 +57,7 @@ export async function findAllRepositories( ); const repos = await deps.gitApiService - .getRepositoriesFromIntegrations(search, pageNumber, pageSize) + .getRepositoriesFromIntegrations(search, pageNumber, pageSize, userTokens) .then(response => formatResponse(deps, response, checkStatus)); return repos; @@ -69,7 +67,7 @@ export async function findRepositoriesByOrganization( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, orgName: string, @@ -77,13 +75,20 @@ export async function findRepositoriesByOrganization( checkStatus: boolean = false, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + userTokens?: Record, ): Promise> { deps.logger.debug( `Getting all repositories for org "${orgName}" - (search,page,size)=(${search},${pageNumber},${pageSize})..`, ); const glReposByOrg = await deps.gitApiService - .getOrgRepositoriesFromIntegrations(orgName, search, pageNumber, pageSize) + .getOrgRepositoriesFromIntegrations( + orgName, + search, + pageNumber, + pageSize, + userTokens, + ) .then(response => formatResponse(deps, response, checkStatus)); return glReposByOrg; @@ -114,10 +119,10 @@ async function formatResponse( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, - allReposAccessible: GithubRepositoryResponse | GitlabRepositoryResponse, + allReposAccessible: SCMRepositoryResponse, checkStatus: boolean, ) { const errorList = diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.test.ts new file mode 100644 index 00000000000..08c29eb822d --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.test.ts @@ -0,0 +1,140 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { mockServices } from '@backstage/backend-test-utils'; + +import { findAllSCMHosts } from './scm'; + +describe('findAllSCMHosts', () => { + it('returns github and gitlab host URLs from config', async () => { + const config = mockServices.rootConfig({ + data: { + integrations: { + github: [ + { host: 'github.com', token: 'gh-token' }, + { host: 'enterprise.github.com', token: 'gh-ent-token' }, + ], + gitlab: [ + { + host: 'gitlab.com', + baseUrl: 'https://gitlab.com/', + token: 'gl-token', + }, + ], + }, + }, + }); + + const result = await findAllSCMHosts(config); + + expect(result.statusCode).toBe(200); + expect(result.responseBody?.github).toEqual([ + 'https://github.com', + 'https://enterprise.github.com', + ]); + expect(result.responseBody?.gitlab).toEqual(['https://gitlab.com']); + }); + + it('normalizes gitlab baseUrl by stripping trailing slash', async () => { + const config = mockServices.rootConfig({ + data: { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + baseUrl: 'https://gitlab.com/', + token: 'gl-token', + }, + ], + }, + }, + }); + + const result = await findAllSCMHosts(config); + + expect(result.statusCode).toBe(200); + // trailing slash should be stripped from gitlab baseUrl + expect(result.responseBody?.gitlab).toEqual( + expect.arrayContaining(['https://gitlab.com']), + ); + expect(result.responseBody?.gitlab?.every(u => !u.endsWith('/'))).toBe( + true, + ); + }); + + it('returns only github hosts when no gitlab integrations are explicitly configured', async () => { + const config = mockServices.rootConfig({ + data: { + integrations: { + github: [{ host: 'github.com', token: 'gh-token' }], + }, + }, + }); + + const result = await findAllSCMHosts(config); + + expect(result.statusCode).toBe(200); + expect(result.responseBody?.github).toEqual(['https://github.com']); + // Backstage's ScmIntegrations always adds a default gitlab.com entry + expect(result.responseBody?.gitlab).toContain('https://gitlab.com'); + }); + + it('returns only gitlab hosts when no github integrations are explicitly configured', async () => { + const config = mockServices.rootConfig({ + data: { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + baseUrl: 'https://gitlab.com', + token: 'gl-token', + }, + ], + }, + }, + }); + + const result = await findAllSCMHosts(config); + + expect(result.statusCode).toBe(200); + expect(result.responseBody?.gitlab).toContain('https://gitlab.com'); + // Backstage's ScmIntegrations always adds a default github.com entry + expect(result.responseBody?.github).toContain('https://github.com'); + }); + + it('returns multiple github integrations in order', async () => { + const config = mockServices.rootConfig({ + data: { + integrations: { + github: [ + { host: 'github.com', token: 'token-1' }, + { host: 'ghe.example.com', token: 'token-2' }, + { host: 'another-ghe.corp.com', token: 'token-3' }, + ], + }, + }, + }); + + const result = await findAllSCMHosts(config); + + expect(result.statusCode).toBe(200); + expect(result.responseBody?.github).toEqual([ + 'https://github.com', + 'https://ghe.example.com', + 'https://another-ghe.corp.com', + ]); + }); +}); diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.ts new file mode 100644 index 00000000000..896d518c98e --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.ts @@ -0,0 +1,37 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; + +import type { Paths } from '../../../generated/openapi'; +import type { HandlerResponse } from '../handlers'; + +export async function findAllSCMHosts( + config: Config, +): Promise> { + const integrations = ScmIntegrations.fromConfig(config); + const githubHosts = integrations.github + .list() + .map(i => `https://${i.config.host}`); + const gitlabHosts = integrations.gitlab + .list() + .map(i => i.config.baseUrl.replace(/\/$/, '')); + return { + statusCode: 200, + responseBody: { github: githubHosts, gitlab: gitlabHosts }, + }; +} diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts index 2948783d74a..46d564aba10 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts @@ -28,8 +28,139 @@ import { setupTest, startBackendServer } from '../../__fixtures__/testUtils'; describe('router tests', () => { const useTestData = setupTest(); + describe('x-scm-tokens middleware', () => { + it.each([ + [ + 'GET /repositories', + (req: request.SuperTest, header: string) => + req.get('/api/bulk-import/repositories').set('x-scm-tokens', header), + ], + [ + 'GET /organizations/:org/repositories', + (req: request.SuperTest, header: string) => + req + .get('/api/bulk-import/organizations/my-org-1/repositories') + .set('x-scm-tokens', header), + ], + ])( + '%s: returns 400 when x-scm-tokens does not contain a valid JSON-encoded token map', + async ( + _endpoint: string, + reqHandler: ( + req: request.SuperTest, + header: string, + ) => request.Test, + ) => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const cases = [ + 'not-json', + '["array", "not", "object"]', + '{"host": 123}', + '{"host": ""}', + ]; + + for (const invalidHeader of cases) { + const response = await reqHandler( + request(backendServer), + invalidHeader, + ); + expect(response.status).toEqual(400); + } + }, + ); + + it.each([ + [ + 'GET /repositories', + (req: request.SuperTest, header: string) => + req.get('/api/bulk-import/repositories').set('x-scm-tokens', header), + ], + [ + 'GET /organizations/:org/repositories', + (req: request.SuperTest, header: string) => + req + .get('/api/bulk-import/organizations/my-org-1/repositories') + .set('x-scm-tokens', header), + ], + ])( + '%s: ignores x-scm-tokens and returns 401 when header exceeds size limit', + async ( + _endpoint: string, + reqHandler: ( + req: request.SuperTest, + header: string, + ) => request.Test, + ) => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const oversizedHeader = JSON.stringify({ + 'https://github.com': 'a'.repeat(4097), + }); + + const response = await reqHandler( + request(backendServer), + oversizedHeader, + ); + // Oversized header is silently discarded; the missing valid token then + // triggers the 401 guard added for compliance. + expect(response.status).toEqual(401); + }, + ); + + it.each([ + [ + 'GET /repositories', + (req: request.SuperTest, header: string) => + req.get('/api/bulk-import/repositories').set('x-scm-tokens', header), + ], + [ + 'GET /organizations/:org/repositories', + (req: request.SuperTest, header: string) => + req + .get('/api/bulk-import/organizations/my-org-1/repositories') + .set('x-scm-tokens', header), + ], + ])( + '%s: returns 200 and processes request when x-scm-tokens is a valid token map', + async ( + _endpoint: string, + reqHandler: ( + req: request.SuperTest, + header: string, + ) => request.Test, + ) => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const validHeader = JSON.stringify({ + 'https://github.com': 'gho_validUserToken', + }); + + const response = await reqHandler(request(backendServer), validHeader); + expect(response.status).toEqual(200); + }, + ); + }); + describe('permission framework denial', () => { it.each([ + [ + 'GET /scm-hosts', + async (req: request.SuperTest) => + req.get('/api/bulk-import/scm-hosts'), + ], [ 'GET /organizations', async (req: request.SuperTest) => diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts index 06270dbc611..4dcfb4b0249 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts @@ -79,6 +79,7 @@ import { findAllRepositories, findRepositoriesByOrganization, } from './handlers/repository'; +import { findAllSCMHosts } from './handlers/scm/scm'; /** * Router Options @@ -99,6 +100,7 @@ export interface RouterOptions { namespace Operations { export const PING = 'ping'; + export const FIND_ALL_SCM_HOSTS = 'findAllSCMHosts'; export const FIND_ALL_ORGANIZATIONS = 'findAllOrganizations'; export const FIND_ALL_REPOSITORIES = 'findAllRepositories'; @@ -126,6 +128,42 @@ namespace Operations { 'deleteOrchestratorImportByRepo'; } +const SCM_TOKENS_MAX_BYTES = 4096; + +const isValidTokenMap = (v: unknown): v is Record => + typeof v === 'object' && + v !== null && + Object.values(v).every(val => typeof val === 'string' && val.length > 0); + +function parseScmTokensHeader( + rawTokenHeader: string | undefined, + logger: LoggerService, +): Record | undefined { + let token: Record | undefined; + if (!rawTokenHeader) return undefined; + if (rawTokenHeader.length > SCM_TOKENS_MAX_BYTES) { + logger.warn('x-scm-tokens header exceeds maximum allowed size; ignoring'); + return undefined; + } + try { + token = JSON.parse(rawTokenHeader) as Record; + if (typeof token !== 'object' || token === null || Array.isArray(token)) { + throw new Error('x-scm-tokens must be a JSON object'); + } + } catch (e) { + throw new InputError( + `Invalid x-scm-tokens header: ${(e as Error).message}`, + ); + } + + if (!isValidTokenMap(token)) { + throw new InputError( + 'Invalid x-scm-tokens header: all values must be non-empty strings', + ); + } + return token; +} + /** * Router * @public @@ -156,7 +194,8 @@ export async function createRouter( 'orchestrator_repositories', ); const orchestratorWorkflowDao = new OrchestratorWorkflowDao(knex); - // This should probably be sometype of object that holds all the scm API service objects + // GitHub and GitLab both implement the GitApiService interface; the router + // selects the appropriate service per request based on approvalTool. const githubApiService = new GithubApiService(logger, config, cache); const gitlabApiService = new GitlabApiService(logger, config, cache); const catalogHttpClient = new CatalogHttpClient({ @@ -211,6 +250,14 @@ export async function createRouter( }, ); + api.register( + Operations.FIND_ALL_SCM_HOSTS, + async (_c: Context, _req: Request, res: Response) => { + const result = await findAllSCMHosts(config); + return res.status(result.statusCode).json(result.responseBody); + }, + ); + api.register( Operations.FIND_ALL_ORGANIZATIONS, async (c: Context, _req: Request, res: Response) => { @@ -248,6 +295,18 @@ export async function createRouter( q.pagePerIntegration = stringToNumber(q.pagePerIntegration); q.sizePerIntegration = stringToNumber(q.sizePerIntegration); q.checkImportStatus = stringToBoolean(q.checkImportStatus); + + const userTokens = res.locals.scmTokens as + | Record + | undefined; + + if (!userTokens || Object.keys(userTokens).length === 0) { + return res.status(401).json({ + error: + 'User SCM credentials are required to list repositories. Ensure the SCM OAuth integration is configured.', + }); + } + const response = await findAllRepositories( { logger, @@ -261,7 +320,7 @@ export async function createRouter( checkStatus: q.checkImportStatus, pageNumber: q.pagePerIntegration, pageSize: q.sizePerIntegration, - approvalTool: q.approvalTool, + userTokens, }, ); const repos = response.responseBody?.repositories; @@ -286,6 +345,18 @@ export async function createRouter( q.pagePerIntegration = stringToNumber(q.pagePerIntegration); q.sizePerIntegration = stringToNumber(q.sizePerIntegration); q.checkImportStatus = stringToBoolean(q.checkImportStatus); + + const userTokens = res.locals.scmTokens as + | Record + | undefined; + + if (!userTokens || Object.keys(userTokens).length === 0) { + return res.status(401).json({ + error: + 'User SCM credentials are required to list repositories. Ensure the SCM OAuth integration is configured.', + }); + } + const response = await findRepositoriesByOrganization( { logger, @@ -299,6 +370,7 @@ export async function createRouter( q.checkImportStatus, q.pagePerIntegration, q.sizePerIntegration, + userTokens, ); const repos = response.responseBody?.repositories; return res.status(response.statusCode).json({ @@ -678,15 +750,38 @@ export async function createRouter( }); router.use(permissionIntegrationRouter); + // Strip x-scm-tokens from req.headers before the permission check and audit + // middleware run so that OAuth tokens are never captured in audit logs. + // The parsed map is stored in res.locals.scmTokens for handler use. + router.use((req, res, next) => { + const raw = req.headers['x-scm-tokens']; + const rawStr = Array.isArray(raw) ? raw[0] : raw; + delete req.headers['x-scm-tokens']; + if (rawStr) { + try { + res.locals.scmTokens = parseScmTokensHeader(rawStr, logger); + } catch (e) { + next(e); + return; + } + } + next(); + }); + router.use(async (req, _res, next) => { if (req.path !== '/ping') { - await permissionCheck( - auditor, - api.matchOperation(req as OpenAPIRequest)?.operationId, - permissions, - httpAuth, - req, - ).catch(next); + try { + await permissionCheck( + auditor, + api.matchOperation(req as OpenAPIRequest)?.operationId, + permissions, + httpAuth, + req, + ); + } catch (e) { + next(e); + return; + } } next(); }); @@ -726,6 +821,10 @@ async function createAuditorEventByOperationId( case Operations.PING: auditorEvent = await auditCreateEvent(auditor, 'ping', req); break; + case Operations.FIND_ALL_SCM_HOSTS: + auditorEvent = await auditCreateEvent(auditor, 'scm-hosts-read', req); + break; + case Operations.FIND_ALL_ORGANIZATIONS: auditorEvent = await auditCreateEvent(auditor, 'org-read', req, { queryType: req.query.search ? 'by-query' : 'all', diff --git a/workspaces/bulk-import/plugins/bulk-import/README.md b/workspaces/bulk-import/plugins/bulk-import/README.md index f9e48f6cdb5..5d1b0978db6 100644 --- a/workspaces/bulk-import/plugins/bulk-import/README.md +++ b/workspaces/bulk-import/plugins/bulk-import/README.md @@ -19,6 +19,8 @@ The sections below are relevant for static plugins. If the plugin is expected to - Follow the [GitHub Locations](https://backstage.io/docs/integrations/github/locations) to integrate GitHub integrations in your Backstage instance. For now, the plugin only supports loading catalog entities from github.com or GitHub Enterprise. +- Configure a GitHub and/or GitLab OAuth auth provider and register `ScmAuthApi` from `@backstage/integration-react` in your application. **This is required for repository listing.** The `GET /repositories` and `GET /organizations/{org}/repositories` backend endpoints require user OAuth credentials (sent via the `X-SCM-Tokens` header) and will return HTTP 401 if they are absent. See [Configuring Auth Providers](#configuring-auth-providers) for setup instructions. + --- **NOTE** @@ -72,6 +74,47 @@ g, user:default/, role:default/team_a ); ``` +## On Behalf of User Access + +The Bulk Import plugin can fetch repository and organization listings **on behalf of the signed-in user** using their OAuth credentials, so that users see only the repositories and organizations they personally have access to. + +### How It Works + +When `ScmAuthApi` (from `@backstage/integration-react`) is available in the application, the plugin: + +1. Calls `GET /api/bulk-import/scm-hosts` to discover the configured GitHub and GitLab integration host URLs. +2. Requests an OAuth token for each host from `ScmAuthApi` using a read-only scope (`repoWrite: false`). +3. Passes the collected tokens to the backend via the `X-SCM-Tokens` request header when listing repositories or organizations. + +The backend then uses these user tokens to call the SCM APIs on behalf of the user, returning only what that user can access. + +### Required OAuth Configuration + +GitHub and/or GitLab OAuth providers are **required** for repository and organization listing. The backend enforces this: `GET /repositories` and `GET /organizations/{org}/repositories` return **HTTP 401** if the `X-SCM-Tokens` header is absent or empty. + +If `ScmAuthApi` is not registered in the application, or if token collection fails for every configured SCM host, the frontend blocks the listing request and surfaces a descriptive error prompting the user to configure the OAuth integration. + +> **Migration note:** Deployments that previously relied on server-side integration credentials alone for the repository list view (GitHub App, PAT, or GitLab token) must now also configure an SCM OAuth provider. See [Configuring Auth Providers](#configuring-auth-providers) below. + +### Configuring Auth Providers + +To enable user-scoped repository listings, configure the relevant auth providers in your `app-config.yaml`: + +```yaml +auth: + providers: + github: + development: + clientId: ${GITHUB_CLIENT_ID} + clientSecret: ${GITHUB_CLIENT_SECRET} + gitlab: + development: + clientId: ${GITLAB_CLIENT_ID} + clientSecret: ${GITLAB_CLIENT_SECRET} +``` + +Refer to the Backstage documentation for [GitHub auth](https://backstage.io/docs/auth/github/provider) and [GitLab auth](https://backstage.io/docs/auth/gitlab/provider) for full configuration details. + ## New Frontend System If you're using Backstage's new frontend system, add the plugin to your app: diff --git a/workspaces/bulk-import/plugins/bulk-import/dev/mocks.ts b/workspaces/bulk-import/plugins/bulk-import/dev/mocks.ts index f1933bd136c..4c7259b221f 100644 --- a/workspaces/bulk-import/plugins/bulk-import/dev/mocks.ts +++ b/workspaces/bulk-import/plugins/bulk-import/dev/mocks.ts @@ -131,6 +131,12 @@ export class MockBulkImportApi implements BulkImportAPI { i => i.repository.url === repo, ) as ImportJobStatus; } + + async getSCMHosts(): Promise< + { github: string[]; gitlab: string[] } | Response + > { + return { github: [], gitlab: [] }; + } } export const mockBulkImportApi = new MockBulkImportApi(); diff --git a/workspaces/bulk-import/plugins/bulk-import/package.json b/workspaces/bulk-import/plugins/bulk-import/package.json index ca7f7c191a2..043dcd1d717 100644 --- a/workspaces/bulk-import/plugins/bulk-import/package.json +++ b/workspaces/bulk-import/plugins/bulk-import/package.json @@ -53,6 +53,7 @@ "@backstage/core-components": "^0.18.6", "@backstage/core-plugin-api": "^1.12.2", "@backstage/frontend-plugin-api": "^0.13.4", + "@backstage/integration-react": "^1.2.16", "@backstage/plugin-app-react": "^0.1.0", "@backstage/plugin-catalog-import": "^0.13.9", "@backstage/plugin-catalog-react": "^1.21.6", diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts index 3f36bdef7bf..4ebe3b6a9f5 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts @@ -40,6 +40,19 @@ import { const LOCAL_ADDR = 'https://localhost:7007'; const handlers = [ + rest.get(`${LOCAL_ADDR}/api/bulk-import/scm-hosts`, (req, res, ctx) => { + const test = req.headers.get('Content-Type'); + if (test === 'application/json') { + return res( + ctx.status(200), + ctx.json({ + github: ['https://github.com'], + gitlab: ['https://gitlab.com'], + }), + ); + } + return res(ctx.status(404)); + }), rest.get(`${LOCAL_ADDR}/api/bulk-import/repositories`, (req, res, ctx) => { const searchParam = req.url.searchParams.get('search'); const test = req.headers.get('Content-Type'); @@ -59,7 +72,7 @@ const handlers = [ return res(ctx.status(404)); }), rest.get( - `${LOCAL_ADDR}/api/bulk-import/organizations/org/dessert/repositories`, + `${LOCAL_ADDR}/api/bulk-import/organizations/org%2Fdessert/repositories`, (req, res, ctx) => { const test = req.headers.get('Content-Type'); const searchParam = req.url.searchParams.get('search'); @@ -228,6 +241,78 @@ describe('BulkImportBackendClient with open-pull-requests', () => { }); }); + describe('getSCMHosts', () => { + it('should retrieve SCM hosts successfully', async () => { + const hosts = await bulkImportApi.getSCMHosts(); + expect(hosts).toEqual({ + github: ['https://github.com'], + gitlab: ['https://gitlab.com'], + }); + }); + + it('should return the response object when the server returns a non-200 status', async () => { + server.use( + rest.get( + `${LOCAL_ADDR}/api/bulk-import/scm-hosts`, + (_req, res, ctx) => { + return res(ctx.status(403)); + }, + ), + ); + + const response = await bulkImportApi.getSCMHosts(); + expect(response).toBeInstanceOf(Response); + expect((response as Response).status).toBe(403); + }); + }); + + describe('dataFetcher X-SCM-Tokens header', () => { + let fetchSpy: jest.SpyInstance; + + beforeEach(() => { + fetchSpy = jest.spyOn(globalThis, 'fetch'); + }); + + afterEach(() => { + fetchSpy.mockRestore(); + }); + + it('should send the X-SCM-Tokens header when scmAuthTokens are provided', async () => { + const scmAuthTokens: Record = { + 'https://github.com': 'user-token-abc', + }; + await bulkImportApi.dataFetcher(1, 2, '', ApprovalTool.Git, { + scmAuthTokens, + }); + + const calledHeaders = fetchSpy.mock.calls[0][1]?.headers as Record< + string, + string + >; + expect(calledHeaders['X-SCM-Tokens']).toBe(JSON.stringify(scmAuthTokens)); + }); + + it('should not send the X-SCM-Tokens header when scmAuthTokens are not provided', async () => { + await bulkImportApi.dataFetcher(1, 2, '', ApprovalTool.Git); + + const calledHeaders = fetchSpy.mock.calls[0][1]?.headers as Record< + string, + string + >; + expect(calledHeaders['X-SCM-Tokens']).toBeUndefined(); + }); + + it('should not send X-SCM-Tokens when scmAuthTokens is an empty object', async () => { + await bulkImportApi.dataFetcher(1, 2, '', ApprovalTool.Git, {}); + + const calledHeaders = fetchSpy.mock.calls[0][1]?.headers as Record< + string, + string + >; + expect(calledHeaders['X-SCM-Tokens']).toBeUndefined(); + }); + }); + describe('getRepositories', () => { it('getRepositories should retrieve repositories successfully', async () => { const repositories = await bulkImportApi.dataFetcher( diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts index 1ac12c4e52f..d516c9c7da8 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts @@ -33,12 +33,14 @@ import { SortingOrderEnum, } from '../types'; import { getApi } from '../utils/repository-utils'; +import { IBulkImportRESTPathProvider } from './BulkImportBackendClientBase'; import { OrchestratorBulkImportBackendClientPathProvider } from './OrchestratorBulkImportBackendClientPathProvider'; import { PRBulkImportBackendClientPathProvider } from './PRBulkImportBackendClientPathProvider'; import { ScaffolderBulkImportBackendClientPathProvider } from './ScaffolderBulkImportBackendClientPathProvider'; // @public export type BulkImportAPI = { + getSCMHosts(): Promise<{ github: string[]; gitlab: string[] } | Response>; dataFetcher: ( page: number, size: number, @@ -79,27 +81,6 @@ export const bulkImportApiRef = createApiRef({ id: 'plugin.bulk-import.service', }); -export interface IBulkImportRESTPathProvider { - getCreateImportJobsPath(dryRun?: boolean): string | undefined; - getDeleteImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string; - getGetImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string; - getGetImportJobsPath( - page: number, - size: number, - searchString: string, - sortColumn: AddedRepositoryColumnNameEnum, - sortOrder: SortingOrderEnum, - ): string; -} - export class BulkImportBackendClient implements BulkImportAPI { private readonly configApi: ConfigApi; private readonly identityApi: IdentityApi; @@ -136,6 +117,7 @@ export class BulkImportBackendClient implements BulkImportAPI { options?: APITypes, ) { const { token: idToken } = await this.identityApi.getCredentials(); + const backendUrl = this.configApi.getString('backend.baseUrl'); const jsonResponse = await fetch( getApi(backendUrl, page, size, searchString, approvalTool, options), @@ -143,6 +125,9 @@ export class BulkImportBackendClient implements BulkImportAPI { headers: { 'Content-Type': 'application/json', ...(idToken && { Authorization: `Bearer ${idToken}` }), + ...(options?.scmAuthTokens && { + 'X-SCM-Tokens': JSON.stringify(options.scmAuthTokens), + }), }, }, ); @@ -208,6 +193,25 @@ export class BulkImportBackendClient implements BulkImportAPI { return jsonResponse.status === 204 ? null : await jsonResponse.json(); } + async getSCMHosts() { + const { token: idToken } = await this.identityApi.getCredentials(); + const backendUrl = this.configApi.getString('backend.baseUrl'); + const jsonResponse = await fetch( + `${backendUrl}${this.pathProvider.getSCMHostPath()}`, + { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + ...(idToken && { Authorization: `Bearer ${idToken}` }), + }, + }, + ); + if (jsonResponse.status !== 200 && jsonResponse.status !== 204) { + return jsonResponse; + } + return jsonResponse.json(); + } + async deleteImportAction( repo: string, defaultBranch: string, @@ -216,7 +220,7 @@ export class BulkImportBackendClient implements BulkImportAPI { const { token: idToken } = await this.identityApi.getCredentials(); const backendUrl = this.configApi.getString('backend.baseUrl'); const jsonResponse = await fetch( - `${backendUrl}${this.pathProvider.getDeleteImportActionPath(repo, defaultBranch, approvalTool)}`, + `${backendUrl}${this.pathProvider.getImportActionPath(repo, defaultBranch, approvalTool)}`, { method: 'DELETE', headers: { @@ -240,7 +244,7 @@ export class BulkImportBackendClient implements BulkImportAPI { const { token: idToken } = await this.identityApi.getCredentials(); const backendUrl = this.configApi.getString('backend.baseUrl'); const jsonResponse = await fetch( - `${backendUrl}${this.pathProvider.getGetImportActionPath(repo, defaultBranch, approvalTool)}`, + `${backendUrl}${this.pathProvider.getImportActionPath(repo, defaultBranch, approvalTool)}`, { method: 'GET', headers: { diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts new file mode 100644 index 00000000000..a2a40afaeb1 --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts @@ -0,0 +1,66 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; + +export interface IBulkImportRESTPathProvider { + getCreateImportJobsPath(dryRun?: boolean): string | undefined; + getImportActionPath( + repo: string, + defaultBranch: string, + approvalTool?: string, + ): string; + getGetImportJobsPath( + page: number, + size: number, + searchString: string, + sortColumn: AddedRepositoryColumnNameEnum, + sortOrder: SortingOrderEnum, + ): string; + getSCMHostPath(): string; +} + +export abstract class BulkImportRESTPathProviderBase implements IBulkImportRESTPathProvider { + abstract getCreateImportJobsPath(dryRun?: boolean): string | undefined; + abstract getImportActionPath( + repo: string, + defaultBranch: string, + approvalTool?: string, + ): string; + + protected abstract getImportJobsBasePath(): string; + + getGetImportJobsPath( + page: number, + size: number, + searchString: string, + sortColumn: AddedRepositoryColumnNameEnum, + sortOrder: SortingOrderEnum, + ): string { + const params = new URLSearchParams({ + page: String(page), + size: String(size), + search: searchString, + sortColumn, + sortOrder, + }); + return `${this.getImportJobsBasePath()}?${params.toString()}`; + } + + getSCMHostPath(): string { + return `/api/bulk-import/scm-hosts`; + } +} diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts index 528d6c686e4..5f926ac1802 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts @@ -14,39 +14,26 @@ * limitations under the License. */ -import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; -import { IBulkImportRESTPathProvider } from './BulkImportBackendClient'; +import { BulkImportRESTPathProviderBase } from './BulkImportBackendClientBase'; -export class OrchestratorBulkImportBackendClientPathProvider implements IBulkImportRESTPathProvider { +export class OrchestratorBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { getCreateImportJobsPath(dryRun?: boolean): string | undefined { return dryRun === true ? undefined : `/api/bulk-import/orchestrator-workflows`; } - getDeleteImportActionPath( + getImportActionPath( repo: string, _defaultBranch: string, - approvalTool: string, + approvalTool?: string, ): string { - return `/api/bulk-import/orchestrator-import/by-repo?repo=${repo}&approvalTool=${approvalTool}`; + const params = new URLSearchParams({ repo }); + if (approvalTool) params.set('approvalTool', approvalTool); + return `/api/bulk-import/orchestrator-import/by-repo?${params.toString()}`; } - getGetImportActionPath( - repo: string, - _defaultBranch: string, - approvalTool: string, - ): string { - return `/api/bulk-import/orchestrator-import/by-repo?repo=${repo}&approvalTool=${approvalTool}`; - } - - getGetImportJobsPath( - page: number, - size: number, - searchString: string, - sortColumn: AddedRepositoryColumnNameEnum, - sortOrder: SortingOrderEnum, - ): string { - return `/api/bulk-import/orchestrator-workflows?page=${page}&size=${size}&search=${searchString}&sortColumn=${sortColumn}&sortOrder=${sortOrder}`; + protected getImportJobsBasePath(): string { + return `/api/bulk-import/orchestrator-workflows`; } } diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts index 14133cae91f..f3c7ad753ad 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts @@ -14,39 +14,26 @@ * limitations under the License. */ -import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; -import { IBulkImportRESTPathProvider } from './BulkImportBackendClient'; +import { BulkImportRESTPathProviderBase } from './BulkImportBackendClientBase'; -export class PRBulkImportBackendClientPathProvider implements IBulkImportRESTPathProvider { +export class PRBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { getCreateImportJobsPath(dryRun?: boolean): string { return dryRun ? `/api/bulk-import/imports?dryRun=true` : `/api/bulk-import/imports`; } - getDeleteImportActionPath( + getImportActionPath( repo: string, defaultBranch: string, - approvalTool: string, + approvalTool?: string, ): string { - return `/api/bulk-import/import/by-repo?repo=${repo}&defaultBranch=${defaultBranch}&approvalTool=${approvalTool}`; + const params = new URLSearchParams({ repo, defaultBranch }); + if (approvalTool) params.set('approvalTool', approvalTool); + return `/api/bulk-import/import/by-repo?${params.toString()}`; } - getGetImportActionPath( - repo: string, - defaultBranch: string, - approvalTool: string, - ): string { - return `/api/bulk-import/import/by-repo?repo=${repo}&defaultBranch=${defaultBranch}&approvalTool=${approvalTool}`; - } - - getGetImportJobsPath( - page: number, - size: number, - searchString: string, - sortColumn: AddedRepositoryColumnNameEnum, - sortOrder: SortingOrderEnum, - ): string { - return `/api/bulk-import/imports?page=${page}&size=${size}&search=${searchString}&sortColumn=${sortColumn}&sortOrder=${sortOrder}`; + protected getImportJobsBasePath(): string { + return `/api/bulk-import/imports`; } } diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts index ee3c9610609..a7d59a896fd 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts @@ -14,37 +14,24 @@ * limitations under the License. */ -import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; -import { IBulkImportRESTPathProvider } from './BulkImportBackendClient'; +import { BulkImportRESTPathProviderBase } from './BulkImportBackendClientBase'; -export class ScaffolderBulkImportBackendClientPathProvider implements IBulkImportRESTPathProvider { +export class ScaffolderBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { getCreateImportJobsPath(dryRun?: boolean): string | undefined { return dryRun === true ? undefined : `/api/bulk-import/task-imports`; } - getDeleteImportActionPath( + getImportActionPath( repo: string, _defaultBranch: string, - approvalTool: string, + approvalTool?: string, ): string { - return `/api/bulk-import/task-import/by-repo?repo=${repo}&approvalTool=${approvalTool}`; + const params = new URLSearchParams({ repo }); + if (approvalTool) params.set('approvalTool', approvalTool); + return `/api/bulk-import/task-import/by-repo?${params.toString()}`; } - getGetImportActionPath( - repo: string, - _defaultBranch: string, - approvalTool: string, - ): string { - return `/api/bulk-import/task-import/by-repo?repo=${repo}&approvalTool=${approvalTool}`; - } - - getGetImportJobsPath( - page: number, - size: number, - searchString: string, - sortColumn: AddedRepositoryColumnNameEnum, - sortOrder: SortingOrderEnum, - ): string { - return `/api/bulk-import/task-imports?page=${page}&size=${size}&search=${searchString}&sortColumn=${sortColumn}&sortOrder=${sortOrder}`; + protected getImportJobsBasePath(): string { + return `/api/bulk-import/task-imports`; } } diff --git a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts index 6aa0a02ab2c..32d23804e70 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts @@ -21,9 +21,12 @@ import { mockGetOrganizations, mockGetRepositories } from '../mocks/mockData'; import { ApprovalTool } from '../types'; import { useRepositories } from './useRepositories'; +const mockUseApiHolder = jest.fn(); + jest.mock('@backstage/core-plugin-api', () => ({ ...jest.requireActual('@backstage/core-plugin-api'), useApi: jest.fn(), + useApiHolder: () => mockUseApiHolder(), })); jest.mock('@tanstack/react-query', () => ({ @@ -31,6 +34,13 @@ jest.mock('@tanstack/react-query', () => ({ useQuery: jest.fn(), })); +beforeEach(() => { + // Default: no scmAuth registered → tokenLoading is immediately false + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(undefined), + }); +}); + describe('useRepositories', () => { it('should return repositories', async () => { (useQuery as jest.Mock).mockReturnValue({ @@ -104,4 +114,279 @@ describe('useRepositories', () => { ).toBe(7); }); }); + + describe('scmAuth token collection', () => { + it('skips token fetching and renders successfully when scmAuth is not registered', async () => { + // Default mock: apiHolder.get returns undefined for scmAuthApiRef + (useQuery as jest.Mock).mockReturnValue({ + data: mockGetRepositories, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + expect(result.current.data?.repositories).toBeDefined(); + }); + }); + + it('collects tokens from scmAuth when it is registered and getSCMHosts succeeds', async () => { + const mockGetCredentials = jest + .fn() + .mockResolvedValue({ token: 'user-oauth-token-123' }); + const mockScmAuth = { getCredentials: mockGetCredentials }; + + const mockGetSCMHosts = jest.fn().mockResolvedValue({ + github: ['https://github.com'], + gitlab: [], + }); + const mockBulkImportApi = { + getSCMHosts: mockGetSCMHosts, + dataFetcher: jest.fn(), + }; + + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(mockScmAuth), + }); + + const mockUseApi = jest.requireMock('@backstage/core-plugin-api').useApi; + mockUseApi.mockImplementation((ref: { id: string }) => { + if (ref.id === 'plugin.bulk-import.service') return mockBulkImportApi; + return undefined; + }); + + (useQuery as jest.Mock).mockReturnValue({ + data: mockGetRepositories, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + }); + + expect(mockGetSCMHosts).toHaveBeenCalled(); + expect(mockGetCredentials).toHaveBeenCalledWith({ + url: 'https://github.com', + additionalScope: { repoWrite: false }, + }); + }); + + it('surfaces a tokenFetchError when scmAuth is registered but all token fetches fail', async () => { + const mockGetCredentials = jest + .fn() + .mockRejectedValue(new Error('No OAuth provider for this host')); + const mockScmAuth = { getCredentials: mockGetCredentials }; + + const mockGetSCMHosts = jest.fn().mockResolvedValue({ + github: ['https://github.com'], + gitlab: [], + }); + const mockBulkImportApi = { + getSCMHosts: mockGetSCMHosts, + dataFetcher: jest.fn(), + }; + + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(mockScmAuth), + }); + + const mockUseApi = jest.requireMock('@backstage/core-plugin-api').useApi; + mockUseApi.mockImplementation((ref: { id: string }) => { + if (ref.id === 'plugin.bulk-import.service') return mockBulkImportApi; + return undefined; + }); + + (useQuery as jest.Mock).mockReturnValue({ + data: undefined, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + }); + + // When all token fetches fail, the hook surfaces an error so the UI can + // prompt the user to configure the SCM OAuth integration. + expect(result.current.error?.errors).toEqual([ + 'No user SCM credentials could be obtained. Please ensure your SCM OAuth integration is configured.', + ]); + }); + + it('disables the query when tokenFetchError is set (no request fired without tokens)', async () => { + const mockGetCredentials = jest + .fn() + .mockRejectedValue(new Error('No OAuth provider for this host')); + const mockScmAuth = { getCredentials: mockGetCredentials }; + + const mockGetSCMHosts = jest.fn().mockResolvedValue({ + github: ['https://github.com'], + gitlab: [], + }); + const mockBulkImportApi = { + getSCMHosts: mockGetSCMHosts, + dataFetcher: jest.fn(), + }; + + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(mockScmAuth), + }); + + const mockUseApi = jest.requireMock('@backstage/core-plugin-api').useApi; + mockUseApi.mockImplementation((ref: { id: string }) => { + if (ref.id === 'plugin.bulk-import.service') return mockBulkImportApi; + return undefined; + }); + + (useQuery as jest.Mock).mockClear(); + (useQuery as jest.Mock).mockReturnValue({ + data: undefined, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + }); + + // useQuery must have been called with enabled: false so no HTTP request + // is fired when user tokens are unavailable. + const lastOptions = (useQuery as jest.Mock).mock.calls.at(-1)?.[2]; + expect(lastOptions?.enabled).toBe(false); + }); + + it('does not include raw token values in the React Query key', async () => { + const secretToken = 'super-secret-oauth-token'; + const mockGetCredentials = jest + .fn() + .mockResolvedValue({ token: secretToken }); + const mockScmAuth = { getCredentials: mockGetCredentials }; + + const mockGetSCMHosts = jest.fn().mockResolvedValue({ + github: ['https://github.com'], + gitlab: [], + }); + const mockBulkImportApi = { + getSCMHosts: mockGetSCMHosts, + dataFetcher: jest.fn(), + }; + + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(mockScmAuth), + }); + + const mockUseApi = jest.requireMock('@backstage/core-plugin-api').useApi; + mockUseApi.mockImplementation((ref: { id: string }) => { + if (ref.id === 'plugin.bulk-import.service') return mockBulkImportApi; + return undefined; + }); + + (useQuery as jest.Mock).mockClear(); + (useQuery as jest.Mock).mockReturnValue({ + data: mockGetRepositories, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + }); + + // Inspect the query key from the most recent render — it must never + // expose raw token values; only the sorted host URLs should appear. + const lastQueryKey = (useQuery as jest.Mock).mock.calls.at(-1)?.[0]; + const serialised = JSON.stringify(lastQueryKey); + expect(serialised).not.toContain(secretToken); + expect(serialised).toContain('https://github.com'); + }); + + it('skips token fetching when getSCMHosts returns a Response error', async () => { + const mockScmAuth = { getCredentials: jest.fn() }; + const mockGetSCMHosts = jest + .fn() + .mockResolvedValue(new Response(null, { status: 403 })); + const mockBulkImportApi = { + getSCMHosts: mockGetSCMHosts, + dataFetcher: jest.fn(), + }; + + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(mockScmAuth), + }); + + const mockUseApi = jest.requireMock('@backstage/core-plugin-api').useApi; + mockUseApi.mockImplementation((ref: { id: string }) => { + if (ref.id === 'plugin.bulk-import.service') return mockBulkImportApi; + return undefined; + }); + + (useQuery as jest.Mock).mockReturnValue({ + data: mockGetRepositories, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + }); + + // getCredentials should not be called since getSCMHosts failed + expect(mockScmAuth.getCredentials).not.toHaveBeenCalled(); + }); + }); }); diff --git a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts index fd61953bebe..0c97ea07f5e 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts @@ -21,14 +21,18 @@ import { configApiRef, identityApiRef, useApi, + useApiHolder, } from '@backstage/core-plugin-api'; +import { scmAuthApiRef } from '@backstage/integration-react'; import { useQuery } from '@tanstack/react-query'; import { bulkImportApiRef } from '../api/BulkImportBackendClient'; import { AddRepositoryData, + APITypes, ApprovalTool, + DataFetcherQueryParams, OrgAndRepoResponse, RepositoriesError, } from '../types'; @@ -37,15 +41,6 @@ import { prepareDataForRepositories, } from '../utils/repository-utils'; -export interface DataFetcherQueryParams { - showOrganizations?: boolean; - orgName?: string; - page?: number; - querySize?: number; - searchString?: string; - approvalTool: ApprovalTool; -} - export const useRepositories = ( options: DataFetcherQueryParams, pollInterval?: number, @@ -62,6 +57,8 @@ export const useRepositories = ( const identityApi = useApi(identityApiRef); const configApi = useApi(configApiRef); const bulkImportApi = useApi(bulkImportApiRef); + const apiHolder = useApiHolder(); + const scmAuth = apiHolder.get(scmAuthApiRef); const { value: user } = useAsync(async () => { const identityRef = await identityApi.getBackstageIdentity(); @@ -73,45 +70,81 @@ export const useRepositories = ( return url; }); - const fetchRepositories = async (queryOptions: DataFetcherQueryParams) => { - if (queryOptions?.showOrganizations) { - return await bulkImportApi.dataFetcher( - queryOptions?.page ?? 0, - queryOptions?.querySize ?? 0, - queryOptions?.searchString || '', - queryOptions?.approvalTool, - { - fetchOrganizations: true, - }, - ); + const { + value: scmAuthTokens, + loading: tokenLoading, + error: tokenFetchError, + } = useAsync(async () => { + if (!scmAuth) return undefined; + const hosts = await bulkImportApi.getSCMHosts(); + if (!hosts || hosts instanceof Response || !('github' in hosts)) + return undefined; + const urls = + options.approvalTool === ApprovalTool.Gitlab + ? hosts.gitlab + : hosts.github; + + if (!urls?.length) return undefined; + + const tokenRecord: Record = {}; + for (const url of urls) { + try { + const { token } = await scmAuth.getCredentials({ + url, + additionalScope: { repoWrite: false }, + }); + if (token) tokenRecord[url] = token; + } catch { + // No OAuth provider registered for this host — skip it. + } } - if (queryOptions?.orgName) { - return await bulkImportApi.dataFetcher( - queryOptions?.page ?? 0, - queryOptions?.querySize ?? 0, - queryOptions?.searchString || '', - queryOptions?.approvalTool, - { - orgName: queryOptions?.orgName, - }, + if (Object.keys(tokenRecord).length === 0) { + throw new Error( + 'No user SCM credentials could be obtained. Please ensure your SCM OAuth integration is configured.', ); } - return await bulkImportApi.dataFetcher( - queryOptions?.page ?? 0, - queryOptions?.querySize ?? 0, - queryOptions?.searchString || '', - queryOptions?.approvalTool, + return tokenRecord; + }, [scmAuth, bulkImportApi, options.approvalTool]); + + const fetchRepositories = async (queryOptions: DataFetcherQueryParams) => { + const apiOptions: APITypes = { + ...(queryOptions.showOrganizations && { fetchOrganizations: true }), + ...(queryOptions.orgName && { orgName: queryOptions.orgName }), + scmAuthTokens, + }; + return bulkImportApi.dataFetcher( + queryOptions.page ?? 0, + queryOptions.querySize ?? 0, + queryOptions.searchString ?? '', + queryOptions.approvalTool, + apiOptions, ); }; + const scmAuthHosts = useMemo( + () => + Object.keys(scmAuthTokens ?? {}) + .sort((a, b) => a.localeCompare(b)) + .join(','), + [scmAuthTokens], + ); + const { data: value, error, isLoading: isQueryLoading, } = useQuery( - [options?.showOrganizations ? 'organizations' : 'repositories', options], + [ + options?.showOrganizations ? 'organizations' : 'repositories', + options, + scmAuthHosts, + ], () => fetchRepositories(options), - { refetchInterval: pollInterval || 60000, refetchOnWindowFocus: false }, + { + enabled: !tokenLoading && !tokenFetchError, + refetchInterval: pollInterval || 60000, + refetchOnWindowFocus: false, + }, ); const prepareData = useMemo(() => { @@ -126,9 +159,10 @@ export const useRepositories = ( }, [options?.showOrganizations, value, user, baseUrl]); return { - loading: isQueryLoading, + loading: tokenLoading || isQueryLoading, data: prepareData, error: { + ...(tokenFetchError ? { errors: [tokenFetchError.message] } : {}), ...(error ?? {}), ...((value?.errors && value.errors.length > 0) || (value as any as Response)?.statusText diff --git a/workspaces/bulk-import/plugins/bulk-import/src/types/types.ts b/workspaces/bulk-import/plugins/bulk-import/src/types/types.ts index 8c7796f5264..869c059fc25 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/types/types.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/types/types.ts @@ -182,6 +182,7 @@ export type CreateImportJobRepository = { export type APITypes = { orgName?: string; fetchOrganizations?: boolean; + scmAuthTokens?: Record; }; export type ErrorType = { @@ -207,9 +208,10 @@ export interface RepositoriesError extends Error { } export type DataFetcherQueryParams = { - page: number; - querySize: number; + page?: number; + querySize?: number; showOrganizations?: boolean; orgName?: string; searchString?: string; + approvalTool: ApprovalTool; }; diff --git a/workspaces/bulk-import/plugins/bulk-import/src/utils/repository-utils.tsx b/workspaces/bulk-import/plugins/bulk-import/src/utils/repository-utils.tsx index 99b200901f7..80a2080df4b 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/utils/repository-utils.tsx +++ b/workspaces/bulk-import/plugins/bulk-import/src/utils/repository-utils.tsx @@ -548,13 +548,21 @@ export const getApi = ( approvalTool: string, options?: APITypes, ) => { + const params = new URLSearchParams({ + pagePerIntegration: String(page), + sizePerIntegration: String(size), + search: searchString, + approvalTool, + }); + if (options?.fetchOrganizations) { - return `${backendUrl}/api/bulk-import/organizations?pagePerIntegration=${page}&sizePerIntegration=${size}&search=${searchString}&approvalTool=${approvalTool}`; + return `${backendUrl}/api/bulk-import/organizations?${params.toString()}`; } if (options?.orgName) { - return `${backendUrl}/api/bulk-import/organizations/${options.orgName}/repositories?pagePerIntegration=${page}&sizePerIntegration=${size}&search=${searchString}&approvalTool=${approvalTool}`; + const orgName = encodeURIComponent(options?.orgName); + return `${backendUrl}/api/bulk-import/organizations/${orgName}/repositories?${params.toString()}`; } - return `${backendUrl}/api/bulk-import/repositories?pagePerIntegration=${page}&sizePerIntegration=${size}&search=${searchString}&approvalTool=${approvalTool}`; + return `${backendUrl}/api/bulk-import/repositories?${params.toString()}`; }; export const getCustomisedErrorMessage = ( diff --git a/workspaces/bulk-import/yarn.lock b/workspaces/bulk-import/yarn.lock index cbdb168726c..8a2e3805cbb 100644 --- a/workspaces/bulk-import/yarn.lock +++ b/workspaces/bulk-import/yarn.lock @@ -12410,6 +12410,7 @@ __metadata: "@backstage/dev-utils": "npm:^1.1.19" "@backstage/frontend-defaults": "npm:^0.3.5" "@backstage/frontend-plugin-api": "npm:^0.13.4" + "@backstage/integration-react": "npm:^1.2.16" "@backstage/plugin-app-react": "npm:^0.1.0" "@backstage/plugin-catalog-import": "npm:^0.13.9" "@backstage/plugin-catalog-react": "npm:^1.21.6"