From 56b2e5d55d70609905831c9164273c6ea9a391b8 Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Fri, 27 Mar 2026 11:06:03 -0400 Subject: [PATCH 01/11] feat(bulk-import): add support for on-hehalf-of user access Signed-off-by: Patrick Knight --- .../.changeset/small-games-live.md | 31 ++++ .../plugins/bulk-import-backend/README.md | 43 +++++- .../api-docs/.openapi-generator/FILES | 1 + .../api-docs/Apis/ManagementApi.md | 23 +++ .../api-docs/Apis/OrganizationApi.md | 3 +- .../api-docs/Apis/RepositoryApi.md | 3 +- .../api-docs/Models/SCMHostList.md | 10 ++ .../bulk-import-backend/api-docs/README.md | 4 +- .../src/generated/openapi.d.ts | 93 +++++++++++- .../src/generated/openapidocument.ts | 89 +++++++++++- .../src/github/githubApiService.ts | 105 +++++++++++++- .../bulk-import-backend/src/github/types.ts | 82 +---------- .../src/github/utils/ghUtils.ts | 10 +- .../src/gitlab/gitlabApiService.ts | 81 ++++++++++- .../bulk-import-backend/src/gitlab/types.ts | 71 +--------- .../src/schema/openapi.yaml | 67 +++++++++ .../src/scm/GitApiService.ts | 132 ++++++++++++++++++ .../bulk-import-backend/src/scm/types.ts | 91 ++++++++++++ .../service/handlers/import/bulkImports.ts | 35 +++-- .../service/handlers/import/importStatus.ts | 7 +- .../handlers/organization/organizations.ts | 15 +- .../handlers/repository/repositories.ts | 29 ++-- .../src/service/handlers/scm/scm.ts | 37 +++++ .../bulk-import-backend/src/service/router.ts | 98 +++++++++++-- .../bulk-import/plugins/bulk-import/README.md | 43 ++++++ .../plugins/bulk-import/package.json | 1 + .../src/api/BulkImportBackendClient.ts | 50 +++++++ ...atorBulkImportBackendClientPathProvider.ts | 25 +++- .../PRBulkImportBackendClientPathProvider.ts | 25 +++- ...lderBulkImportBackendClientPathProvider.ts | 25 +++- .../bulk-import/src/hooks/useRepositories.ts | 106 ++++++++------ .../plugins/bulk-import/src/types/types.ts | 6 +- .../src/utils/repository-utils.tsx | 14 +- workspaces/bulk-import/yarn.lock | 23 ++- 34 files changed, 1200 insertions(+), 278 deletions(-) create mode 100644 workspaces/bulk-import/.changeset/small-games-live.md create mode 100644 workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/SCMHostList.md create mode 100644 workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts create mode 100644 workspaces/bulk-import/plugins/bulk-import-backend/src/scm/types.ts create mode 100644 workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.ts diff --git a/workspaces/bulk-import/.changeset/small-games-live.md b/workspaces/bulk-import/.changeset/small-games-live.md new file mode 100644 index 00000000000..59e0a17bc23 --- /dev/null +++ b/workspaces/bulk-import/.changeset/small-games-live.md @@ -0,0 +1,31 @@ +--- +'@red-hat-developer-hub/backstage-plugin-bulk-import-backend': minor +'@red-hat-developer-hub/backstage-plugin-bulk-import': minor +--- + +## On Behalf of User Access + +This release introduces the ability for the Bulk Import plugin to fetch repository and organization listings **on behalf of the signed-in user**, using their OAuth credentials rather than relying solely on server-side integration credentials (GitHub App, PAT, or GitLab token). + +### What Changed + +**Backend (`bulk-import-backend`)** + +- Added a new `GET /api/bulk-import/scm-hosts` endpoint that returns the configured GitHub and GitLab integration host URLs as a `SCMHostList` object, enabling the frontend to discover which hosts to request OAuth tokens for. +- The `GET /repositories` and `GET /organizations/{organizationName}/repositories` endpoints now accept an optional `x-scm-tokens` request header — a JSON map of SCM host base URL to user OAuth token. When tokens are present, repository listings reflect what the signed-in user can personally access rather than the full scope of the server-wide integration credentials. +- When user tokens are provided for GitHub, the Octokit response cache is intentionally disabled to prevent cross-user ETag cache leakage. Server-side credential paths are not affected. +- Introduced a shared `GitApiService` interface and common SCM types (`SCMOrganization`, `SCMRepository`, `SCMFetchError`, etc.) to unify the GitHub and GitLab service implementations under a consistent contract. + +**Frontend (`bulk-import`)** + +- The plugin now has a **soft dependency** on `@backstage/integration-react`'s `ScmAuthApi`. If the API is registered in the application, the plugin automatically requests OAuth tokens for each configured SCM host and passes them to the backend to enable user-scoped repository listings. +- Added `getSCMHosts()` to the `BulkImportAPI` interface with a corresponding `GET /api/bulk-import/scm-hosts` client call, used to discover host URLs before requesting user tokens. +- User OAuth tokens are transmitted to the backend via the `X-SCM-Tokens` request header as a JSON-encoded map. + +### Fallback Behavior + +The GitHub and GitLab auth providers are **soft dependencies**. The plugin degrades gracefully when they are not configured or unavailable: + +- If `ScmAuthApi` is not registered in the application, no user tokens are collected and the backend falls back entirely to server-side credentials. +- If a token cannot be obtained for a specific host (e.g., the user has not signed in via that provider, or no OAuth provider is registered for that host), that host is silently skipped and the backend uses its configured integration credentials for that host. +- Deployments that do not configure GitHub or GitLab OAuth providers continue to work exactly as before with no change in behavior. diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/README.md b/workspaces/bulk-import/plugins/bulk-import-backend/README.md index 40026e1e498..344576b0936 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/README.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/README.md @@ -301,13 +301,15 @@ The Bulk Import Backend plugin emits audit events for various operations. Events - **`ping`**: tracks `GET` requests to the `/ping` endpoint, which allows to make sure the bulk import backend is up and running. -- **`org-read`**: tracks `GET` requests to the `/organizations` endpoint, which returns the list of organizations accessible from all configured GitHub Integrations. +- **`scm-hosts-read`**: tracks `GET` requests to the `/scm-hosts` endpoint, which returns the list of configured GitHub and GitLab integration host URLs. + +- **`org-read`**: tracks `GET` requests to the `/organizations` endpoint, which returns the list of organizations accessible from all configured SCM Integrations (GitHub and GitLab). Filter on `queryType`. - **`all`**: tracks fetching all organizations. (GET `/organizations`) - **`by-query`**: tracks fetching organization filtered by the query parameter 'search'. (GET `/organizations`) -- **`repo-read`**: tracks `GET` requests to the endpoint, which returns the list of repositories accessible from all configured GitHub Integrations. +- **`repo-read`**: tracks `GET` requests to the endpoint, which returns the list of repositories accessible from all configured SCM Integrations (GitHub and GitLab). Filter on `queryType`. - **`all`**: tracks fetching a list of all repositories accessible by Backstage Github Integrations. (GET `/repositories`) @@ -343,8 +345,41 @@ Example: The bulk import backend plugin provides a REST API to bulk import catalog entities into the catalog. The API is available at the `/api/bulk-import` endpoint. -As a prerequisite, you need to add at least one GitHub Integration (using either a GitHub token or a GitHub App or both) in your app-config YAML file (or a local `app-config.local.yaml` file). -See https://backstage.io/docs/integrations/github/locations/#configuration and https://backstage.io/docs/integrations/github/github-apps/#including-in-integrations-config for more details. +As a prerequisite, you need to add at least one SCM integration in your app-config YAML file (or a local `app-config.local.yaml` file): + +- **GitHub**: Configure a GitHub integration using a GitHub token or a GitHub App (or both). See the [GitHub Locations](https://backstage.io/docs/integrations/github/locations/#configuration) and [GitHub Apps](https://backstage.io/docs/integrations/github/github-apps/#including-in-integrations-config) documentation for details. +- **GitLab** _(optional)_: Configure a GitLab integration if you want to import from GitLab repositories. See the [GitLab Locations](https://backstage.io/docs/integrations/gitlab/locations/) documentation for details. + +### On Behalf of User Access + +The plugin supports fetching repository and organization listings **on behalf of the signed-in user**, using their OAuth credentials rather than the server-wide integration credentials (GitHub App, PAT, or GitLab token). + +#### How It Works + +1. The frontend calls `GET /api/bulk-import/scm-hosts` to retrieve the list of configured SCM integration host URLs, grouped by provider (`github` and `gitlab`). +2. For each host, the frontend requests an OAuth token from the Backstage `ScmAuthApi` (provided by `@backstage/integration-react`). +3. The collected tokens are sent to the backend via the optional `x-scm-tokens` request header — a JSON object mapping each integration base URL to the user's OAuth token. +4. The backend uses these user tokens to call the GitHub or GitLab APIs on behalf of the user, so the repository listings reflect what the signed-in user can personally access. + +#### Fallback Behavior + +The GitHub and GitLab auth providers are **soft dependencies**. When user tokens are absent or unavailable, the backend falls back gracefully to server-side credentials: + +- If the `x-scm-tokens` header is not present or is empty, the backend uses its configured integration credentials (GitHub App, PAT, or GitLab token) for all repository listing calls. Existing behavior is fully preserved. +- If a token is provided for some hosts but not others, the backend uses the user token where available and the server-side credential for any host that was omitted. +- Deployments that do not configure GitHub or GitLab OAuth providers continue to work exactly as before with no configuration changes required. + +#### Security Note + +When user tokens are provided for GitHub, the Octokit response cache is intentionally disabled to prevent cross-user ETag cache leakage. Server-side credential paths are not affected. + +#### New API Endpoint + +| Method | Path | Description | +| ------ | ---------------------------- | ------------------------------------------------------------------------------------------- | +| `GET` | `/api/bulk-import/scm-hosts` | Returns configured GitHub and GitLab integration host base URLs as an `SCMHostList` object. | + +The existing `GET /repositories` and `GET /organizations/{organizationName}/repositories` endpoints now accept an optional `x-scm-tokens` header. See the [API documentation](api-docs/README.md) for the full request/response specification. ## REST API diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES index 2e9aae06917..16f725df267 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES @@ -19,6 +19,7 @@ Models/PullRequest.md Models/Repository.md Models/RepositoryList.md Models/Repository_importStatus.md +Models/SCMHostList.md Models/ScaffolderTask.md Models/Source.md Models/SourceImport.md diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/ManagementApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/ManagementApi.md index b2ff8388f1d..7e516bfba9e 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/ManagementApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/ManagementApi.md @@ -4,9 +4,32 @@ All URIs are relative to *http://localhost:7007/api/bulk-import* | Method | HTTP request | Description | |------------- | ------------- | -------------| +| [**findAllSCMHosts**](ManagementApi.md#findAllSCMHosts) | **GET** /scm-hosts | Retrieve the SCM Integration hosts | | [**ping**](ManagementApi.md#ping) | **GET** /ping | Check the health of the Bulk Import backend router | + +# **findAllSCMHosts** +> SCMHostList findAllSCMHosts() + +Retrieve the SCM Integration hosts + +### Parameters +This endpoint does not need any parameter. + +### Return type + +[**SCMHostList**](../Models/SCMHostList.md) + +### Authorization + +[BearerAuth](../README.md#BearerAuth) + +### HTTP request headers + +- **Content-Type**: Not defined +- **Accept**: application/json + # **ping** > ping_200_response ping() diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md index b8b1d09c925..9f4f8a2f820 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md @@ -38,7 +38,7 @@ Fetch Organizations accessible by Backstage Github Integrations # **findRepositoriesByOrganization** -> RepositoryList findRepositoriesByOrganization(organizationName, checkImportStatus, pagePerIntegration, sizePerIntegration, search, approvalTool) +> RepositoryList findRepositoriesByOrganization(organizationName, checkImportStatus, pagePerIntegration, sizePerIntegration, search, approvalTool, x-scm-tokens) Fetch Repositories in the specified GitHub organization, provided it is accessible by any of the configured GitHub Integrations. @@ -52,6 +52,7 @@ Fetch Repositories in the specified GitHub organization, provided it is accessib | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | | **search** | **String**| returns only the items that match the search string | [optional] [default to null] | | **approvalTool** | **String**| the approvalTool to use | [optional] [default to GIT] | +| **x-scm-tokens** | [**Map**](../Models/String.md)| Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. | [optional] [default to null] | ### Return type diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md index 801f72290ac..d4418f0b35e 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md @@ -9,7 +9,7 @@ All URIs are relative to *http://localhost:7007/api/bulk-import* # **findAllRepositories** -> RepositoryList findAllRepositories(checkImportStatus, pagePerIntegration, sizePerIntegration, search, approvalTool) +> RepositoryList findAllRepositories(checkImportStatus, pagePerIntegration, sizePerIntegration, search, approvalTool, x-scm-tokens) Fetch Organization Repositories accessible by Backstage Github Integrations @@ -22,6 +22,7 @@ Fetch Organization Repositories accessible by Backstage Github Integrations | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | | **search** | **String**| returns only the items that match the search string | [optional] [default to null] | | **approvalTool** | **String**| the approvalTool to use | [optional] [default to GIT] | +| **x-scm-tokens** | [**Map**](../Models/String.md)| Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. | [optional] [default to null] | ### Return type diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/SCMHostList.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/SCMHostList.md new file mode 100644 index 00000000000..9b4766a1307 --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/SCMHostList.md @@ -0,0 +1,10 @@ +# SCMHostList +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **github** | **List** | | [optional] [default to null] | +| **gitlab** | **List** | | [optional] [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md index a2232b16620..a8675b49b2c 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md @@ -19,7 +19,8 @@ All URIs are relative to *http://localhost:7007/api/bulk-import* *ImportApi* | [**findImportStatusByRepo**](Apis/ImportApi.md#findimportstatusbyrepo) | **GET** /import/by-repo | Get Import Status by repository | *ImportApi* | [**findOrchestratorImportStatusByRepo**](Apis/ImportApi.md#findorchestratorimportstatusbyrepo) | **GET** /orchestrator-import/by-repo | Get Import Status by repository | *ImportApi* | [**findTaskImportStatusByRepo**](Apis/ImportApi.md#findtaskimportstatusbyrepo) | **GET** /task-import/by-repo | Get Import Status by repository | -| *ManagementApi* | [**ping**](Apis/ManagementApi.md#ping) | **GET** /ping | Check the health of the Bulk Import backend router | +| *ManagementApi* | [**findAllSCMHosts**](Apis/ManagementApi.md#findallscmhosts) | **GET** /scm-hosts | Retrieve the SCM Integration hosts | +*ManagementApi* | [**ping**](Apis/ManagementApi.md#ping) | **GET** /ping | Check the health of the Bulk Import backend router | | *OrganizationApi* | [**findAllOrganizations**](Apis/OrganizationApi.md#findallorganizations) | **GET** /organizations | Fetch Organizations accessible by Backstage Github Integrations | *OrganizationApi* | [**findRepositoriesByOrganization**](Apis/OrganizationApi.md#findrepositoriesbyorganization) | **GET** /organizations/{organizationName}/repositories | Fetch Repositories in the specified GitHub organization, provided it is accessible by any of the configured GitHub Integrations. | | *RepositoryApi* | [**findAllRepositories**](Apis/RepositoryApi.md#findallrepositories) | **GET** /repositories | Fetch Organization Repositories accessible by Backstage Github Integrations | @@ -44,6 +45,7 @@ All URIs are relative to *http://localhost:7007/api/bulk-import* - [Repository](./Models/Repository.md) - [RepositoryList](./Models/RepositoryList.md) - [Repository_importStatus](./Models/Repository_importStatus.md) + - [SCMHostList](./Models/SCMHostList.md) - [ScaffolderTask](./Models/ScaffolderTask.md) - [Source](./Models/Source.md) - [SourceImport](./Models/SourceImport.md) diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts index ead43ae1c54..bc933f7e42a 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts @@ -13,6 +13,7 @@ import type { declare namespace Components { export interface HeaderParameters { apiVersionHeaderParam?: Parameters.ApiVersionHeaderParam; + xSCMTokensHeaderParam?: Parameters.XSCMTokensHeaderParam; } namespace Parameters { export type ApiVersionHeaderParam = "v1" | "v2"; @@ -26,6 +27,17 @@ declare namespace Components { export type SizeQueryParam = number; export type SortColumnQueryParam = "repository.name" | "repository.organization" | "repository.url" | "lastUpdate" | "status"; export type SortOrderQueryParam = "asc" | "desc"; + export type XSCMTokensHeaderParam = /** + * SCM Token Map + * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. "https://github.com" or "https://gitlab.corp.com"). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. + * + * example: + * { + * "https://github.com": "ghp_xxx", + * "https://ghe.example.com": "ghe_yyy" + * } + */ + Schemas.SCMTokenMap; } export interface QueryParameters { pagePerIntegrationQueryParam?: Parameters.PagePerIntegrationQueryParam; @@ -239,6 +251,26 @@ declare namespace Components { pagePerIntegration?: number; sizePerIntegration?: number; } + /** + * SCM Host List + */ + export interface SCMHostList { + github?: string[]; + gitlab?: string[]; + } + /** + * SCM Token Map + * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. "https://github.com" or "https://gitlab.corp.com"). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. + * + * example: + * { + * "https://github.com": "ghp_xxx", + * "https://ghe.example.com": "ghe_yyy" + * } + */ + export interface SCMTokenMap { + [name: string]: string; + } /** * Scaffolder Task */ @@ -467,12 +499,26 @@ declare namespace Paths { } } namespace FindAllRepositories { + export interface HeaderParameters { + "x-scm-tokens"?: Parameters.XScmTokens; + } namespace Parameters { export type ApprovalTool = string; export type CheckImportStatus = boolean; export type PagePerIntegration = number; export type Search = string; export type SizePerIntegration = number; + export type XScmTokens = /** + * SCM Token Map + * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. "https://github.com" or "https://gitlab.corp.com"). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. + * + * example: + * { + * "https://github.com": "ghp_xxx", + * "https://ghe.example.com": "ghe_yyy" + * } + */ + Components.Schemas.SCMTokenMap; } export interface QueryParameters { checkImportStatus?: Parameters.CheckImportStatus; @@ -486,6 +532,11 @@ declare namespace Paths { export type $500 = /* Repository List */ Components.Schemas.RepositoryList; } } + namespace FindAllSCMHosts { + namespace Responses { + export type $200 = /* SCM Host List */ Components.Schemas.SCMHostList; + } + } namespace FindAllTaskImports { export interface HeaderParameters { "api-version"?: Parameters.ApiVersion; @@ -547,6 +598,9 @@ declare namespace Paths { } } namespace FindRepositoriesByOrganization { + export interface HeaderParameters { + "x-scm-tokens"?: Parameters.XScmTokens; + } namespace Parameters { export type ApprovalTool = string; export type CheckImportStatus = boolean; @@ -554,6 +608,17 @@ declare namespace Paths { export type PagePerIntegration = number; export type Search = string; export type SizePerIntegration = number; + export type XScmTokens = /** + * SCM Token Map + * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. "https://github.com" or "https://gitlab.corp.com"). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. + * + * example: + * { + * "https://github.com": "ghp_xxx", + * "https://ghe.example.com": "ghe_yyy" + * } + */ + Components.Schemas.SCMTokenMap; } export interface PathParameters { organizationName: Parameters.OrganizationName; @@ -606,6 +671,14 @@ export interface OperationMethods { data?: any, config?: AxiosRequestConfig ): OperationResponse + /** + * findAllSCMHosts - Retrieve the SCM Integration hosts + */ + 'findAllSCMHosts'( + parameters?: Parameters | null, + data?: any, + config?: AxiosRequestConfig + ): OperationResponse /** * findAllOrganizations - Fetch Organizations accessible by Backstage Github Integrations */ @@ -618,7 +691,7 @@ export interface OperationMethods { * findRepositoriesByOrganization - Fetch Repositories in the specified GitHub organization, provided it is accessible by any of the configured GitHub Integrations. */ 'findRepositoriesByOrganization'( - parameters?: Parameters | null, + parameters?: Parameters | null, data?: any, config?: AxiosRequestConfig ): OperationResponse @@ -626,7 +699,7 @@ export interface OperationMethods { * findAllRepositories - Fetch Organization Repositories accessible by Backstage Github Integrations */ 'findAllRepositories'( - parameters?: Parameters | null, + parameters?: Parameters | null, data?: any, config?: AxiosRequestConfig ): OperationResponse @@ -739,6 +812,16 @@ export interface PathsDictionary { config?: AxiosRequestConfig ): OperationResponse } + ['/scm-hosts']: { + /** + * findAllSCMHosts - Retrieve the SCM Integration hosts + */ + 'get'( + parameters?: Parameters | null, + data?: any, + config?: AxiosRequestConfig + ): OperationResponse + } ['/organizations']: { /** * findAllOrganizations - Fetch Organizations accessible by Backstage Github Integrations @@ -754,7 +837,7 @@ export interface PathsDictionary { * findRepositoriesByOrganization - Fetch Repositories in the specified GitHub organization, provided it is accessible by any of the configured GitHub Integrations. */ 'get'( - parameters?: Parameters | null, + parameters?: Parameters | null, data?: any, config?: AxiosRequestConfig ): OperationResponse @@ -764,7 +847,7 @@ export interface PathsDictionary { * findAllRepositories - Fetch Organization Repositories accessible by Backstage Github Integrations */ 'get'( - parameters?: Parameters | null, + parameters?: Parameters | null, data?: any, config?: AxiosRequestConfig ): OperationResponse @@ -892,6 +975,8 @@ export type OrganizationList = Components.Schemas.OrganizationList; export type PullRequest = Components.Schemas.PullRequest; export type Repository = Components.Schemas.Repository; export type RepositoryList = Components.Schemas.RepositoryList; +export type SCMHostList = Components.Schemas.SCMHostList; +export type SCMTokenMap = Components.Schemas.SCMTokenMap; export type ScaffolderTask = Components.Schemas.ScaffolderTask; export type Source = Components.Schemas.Source; export type SourceImport = Components.Schemas.SourceImport; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts index 78973e72058..e875b7e7d5e 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -// GENERATED FILE. DO NOT EDIT. // eslint-disable // prettier-ignore @@ -81,6 +80,37 @@ const OPENAPI = ` } } }, + "/scm-hosts": { + "get": { + "operationId": "findAllSCMHosts", + "summary": "Retrieve the SCM Integration hosts", + "security": [ + { + "BearerAuth": [] + } + ], + "tags": [ + "Management" + ], + "responses": { + "200": { + "description": "List of Integrations available to the Bulk Import plugin", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SCMHostList" + }, + "examples": { + "multipleRepos": { + "$ref": "#/components/examples/scmHosts" + } + } + } + } + } + } + } + }, "/organizations": { "get": { "operationId": "findAllOrganizations", @@ -183,6 +213,9 @@ const OPENAPI = ` }, { "$ref": "#/components/parameters/approvalToolParam" + }, + { + "$ref": "#/components/parameters/xSCMTokensHeaderParam" } ], "responses": { @@ -252,6 +285,9 @@ const OPENAPI = ` }, { "$ref": "#/components/parameters/approvalToolParam" + }, + { + "$ref": "#/components/parameters/xSCMTokensHeaderParam" } ], "responses": { @@ -992,6 +1028,15 @@ const OPENAPI = ` "default": "v1" } }, + "xSCMTokensHeaderParam": { + "in": "header", + "name": "x-scm-tokens", + "description": "Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host.\\n", + "required": false, + "schema": { + "$ref": "#/components/schemas/SCMTokenMap" + } + }, "pagePerIntegrationQueryParam": { "in": "query", "name": "pagePerIntegration", @@ -1097,6 +1142,36 @@ const OPENAPI = ` } }, "schemas": { + "SCMTokenMap": { + "title": "SCM Token Map", + "type": "object", + "description": "Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. \"https://github.com\" or \"https://gitlab.corp.com\"). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server.\\n", + "additionalProperties": { + "type": "string" + }, + "example": { + "https://github.com": "ghp_xxx", + "https://ghe.example.com": "ghe_yyy" + } + }, + "SCMHostList": { + "title": "SCM Host List", + "type": "object", + "properties": { + "github": { + "type": "array", + "items": { + "type": "string" + } + }, + "gitlab": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "OrganizationList": { "title": "Organization List", "type": "object", @@ -1556,6 +1631,18 @@ const OPENAPI = ` } }, "examples": { + "scmHosts": { + "summary": "Multiple scmHosts", + "value": { + "github": [ + "https://github.com", + "https://ghe.example.com" + ], + "gitlab": [ + "https://gitlab.com" + ] + } + }, "multipleOrgs": { "summary": "Multiple organizations", "value": { diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts index e7132758b81..bbbf4ef550b 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts @@ -21,6 +21,7 @@ import type { import type { Config } from '@backstage/config'; import { DefaultGithubCredentialsProvider, + GithubCredentials, GithubIntegrationConfig, ScmIntegrations, } from '@backstage/integration'; @@ -34,6 +35,7 @@ import { extractLocationOwnerMap, logErrorIfNeeded, } from '../helpers'; +import { GitApiService } from '../scm/GitApiService'; import { DefaultPageNumber, DefaultPageSize, @@ -72,7 +74,7 @@ import { fetchFromMatchedIntegration, } from './utils/utils'; -export class GithubApiService { +export class GithubApiService implements GitApiService { private readonly logger: LoggerService; private readonly integrations: ScmIntegrations; public readonly githubCredentialsProvider: CustomGithubCredentialsProvider; @@ -147,9 +149,9 @@ export class GithubApiService { } async getOrganizationsFromIntegrations( - search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + search?: string, ): Promise { const orgs = new Map(); const result = await fetchFromAllIntegrations( @@ -216,13 +218,80 @@ export class GithubApiService { }; } + private async fetchUserTokenGithubIntegrationRepositories( + userTokens: Record, + fetcher: ( + octokit: Octokit, + credential: GithubCredentials, + errors: Map, + ) => Promise<{ totalCount?: number }>, + ): Promise<{ errors: GithubFetchError[]; totalCount: number | undefined }> { + const allErrors: GithubFetchError[] = []; + let totalCount: number | undefined; + const ghConfigs = this.integrations.github.list().map(i => i.config); + for (const ghConfig of ghConfigs) { + const hostUrl = `https://${ghConfig.host}`; + const token = userTokens[hostUrl]; + if (!token) { + continue; + } + // Synthesize a token credential — same shape buildOcto already accepts + const userCredential: GithubCredentials = { token, type: 'token' }; + + // Intentionally no cache: user-token requests must not share the + // server-credential ETag cache to avoid cross-user data leakage. + const userOctokit = buildOcto( + { logger: this.logger, cache: undefined }, + { credential: userCredential }, + ghConfig.apiBaseUrl, + ); + if (!userOctokit) { + continue; + } + const dataFetchErrors = new Map(); + const result = await fetcher( + userOctokit, + userCredential, + dataFetchErrors, + ); + totalCount = (totalCount ?? 0) + (result.totalCount ?? 0); + dataFetchErrors.forEach(err => allErrors.push(err)); + } + return { errors: allErrors, totalCount }; + } + async getOrgRepositoriesFromIntegrations( orgName: string, search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + userTokens?: Record, ): Promise { const repositories = new Map(); + + // When a user token is present, build a user-scoped Octokit per integration + // config and call addGithubTokenOrgRepositories with it. This scopes the + // listing to repositories the user has access to within the given organization. + // When absent, fall through to the existing server-credential path. + if (userTokens && Object.keys(userTokens).length > 0) { + const { errors: allErrors, totalCount } = + await this.fetchUserTokenGithubIntegrationRepositories( + userTokens, + (userOctokit, userCredential, dataFetchErrors) => + addGithubTokenOrgRepositories( + { logger: this.logger }, + userOctokit, + userCredential, + orgName, + repositories, + dataFetchErrors, + { search, pageNumber, pageSize }, + ), + ); + const repoList = Array.from(repositories.values()); + return { repositories: repoList, errors: allErrors, totalCount }; + } + const result = await fetchFromAllIntegrations( { logger: this.logger, @@ -305,8 +374,32 @@ export class GithubApiService { search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + userTokens?: Record, ): Promise { const repositories = new Map(); + + // When a user token is present, build a user-scoped Octokit per integration + // config and call addGithubTokenRepositories with it. This makes + // listForAuthenticatedUser return only repos the user personally has access to. + // When absent, fall through to the existing server-credential path. + if (userTokens && Object.keys(userTokens).length > 0) { + const { errors: allErrors, totalCount } = + await this.fetchUserTokenGithubIntegrationRepositories( + userTokens, + (userOctokit, userCredential, dataFetchErrors) => + addGithubTokenRepositories( + { logger: this.logger }, + userOctokit, + userCredential, + repositories, + dataFetchErrors, + { search, pageNumber, pageSize }, + ), + ); + const repoList = Array.from(repositories.values()); + return { repositories: repoList, errors: allErrors, totalCount }; + } + const result = await fetchFromAllIntegrations( { logger: this.logger, @@ -880,7 +973,7 @@ export class GithubApiService { repoUrl: string; defaultBranch?: string; fileName: string; - }) { + }): Promise { const fileExists = await executeFunctionOnFirstSuccessfulIntegration( { logger: this.logger, @@ -925,7 +1018,7 @@ export class GithubApiService { gitUrl: gitUrlParse.GitUrl; comment: string; }, - ) { + ): Promise { await executeFunctionOnFirstSuccessfulIntegration( { logger: this.logger, @@ -973,7 +1066,7 @@ export class GithubApiService { async deleteImportBranch(input: { repoUrl: string; gitUrl: gitUrlParse.GitUrl; - }) { + }): Promise { await executeFunctionOnFirstSuccessfulIntegration( { logger: this.logger, @@ -1006,7 +1099,7 @@ export class GithubApiService { ); } - async isRepoEmpty(input: { repoUrl: string }) { + async isRepoEmpty(input: { repoUrl: string }): Promise { return await executeFunctionOnFirstSuccessfulIntegration( { logger: this.logger, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts index 684ff122a70..dcee5b075ac 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/types.ts @@ -14,86 +14,18 @@ * limitations under the License. */ -import type { SerializedError } from '@backstage/errors'; import type { GithubCredentials, GithubCredentialsProvider, } from '@backstage/integration'; -// From https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-organizations -export type GithubOrganization = { - name: string; - id: number; - description?: string; - url?: string; - html_url?: string; - repos_url?: string; - events_url?: string; - hooks_url?: string; - issues_url?: string; - members_url?: string; - public_members_url?: string; - avatar_url?: string; - public_repos?: number; - total_private_repos?: number; - /** - * Number of internal repositories, accessible to all members in a GH enterprise - */ - owned_private_repos?: number; -}; - -export type GithubRepository = { - name: string; - /** - * The full name of the repository in the form of owner/repo - */ - full_name: string; - /** - * The API url to the repository - */ - url: string; - /** - * The HTML URL to the repository - */ - html_url: string; - /** - * The default "main" branch of the repository to place the `catalog-info.yaml` file into - */ - default_branch: string; - /** - * The date-time the repository was last updated at - */ - updated_at?: string | null; -}; - -/** - * The type of credentials produced by the credential provider. - * - * @public - */ - -export type GithubFetchError = - | { - type: 'app'; - appId: number; - error: SerializedError; - } - | { - type: 'token'; - error: SerializedError; - }; - -export type GithubOrganizationResponse = { - organizations: GithubOrganization[]; - errors: GithubFetchError[]; - totalCount?: number; -}; - -export type GithubRepositoryResponse = { - repositories: GithubRepository[]; - errors: GithubFetchError[]; - totalCount?: number; -}; +export type { + SCMFetchError as GithubFetchError, + SCMOrganization as GithubOrganization, + SCMOrganizationResponse as GithubOrganizationResponse, + SCMRepository as GithubRepository, + SCMRepositoryResponse as GithubRepositoryResponse, +} from '../scm/types'; export type AppCredentialFetchResult = AppCredential | AppCredentialError; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/ghUtils.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/ghUtils.ts index dec1b99dde9..0105b38c647 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/ghUtils.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/utils/ghUtils.ts @@ -39,7 +39,7 @@ const RESPONSE_CACHE_TTL_MILLIS = 60 * 60 * 1000; export function buildOcto( deps: { logger: LoggerService; - cache: CacheService; + cache: CacheService | undefined; }, input: { credential: ExtendedGithubCredentials; @@ -81,7 +81,7 @@ export function buildOcto( function registerHooks( deps: { logger: LoggerService; - cache: CacheService; + cache: CacheService | undefined; }, octokit: Octokit, ) { @@ -98,7 +98,7 @@ function registerHooks( // Use ETag from in-memory cache if available const cacheKey = extractCacheKey(options); const existingEtag = await deps.cache - .get(cacheKey) + ?.get(cacheKey) ?.then((val?: any) => val?.etag); if (existingEtag) { options.headers['If-None-Match'] = existingEtag; @@ -115,7 +115,7 @@ function registerHooks( ); // If we get a successful response, the resource has changed, so update the in-memory cache const cacheKey = extractCacheKey(options); - await deps.cache.set( + await deps.cache?.set( cacheKey, { etag: response.headers.etag, @@ -136,7 +136,7 @@ function registerHooks( } // "304 Not Modified" means that the resource hasn't changed, // and we should have a version of it in the cache - return await deps.cache.get(extractCacheKey(options)); + return await deps.cache?.get(extractCacheKey(options)); }); } diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts index 732b4a95afc..58f26dee320 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.ts @@ -34,6 +34,7 @@ import { extractLocationOwnerMap, logErrorIfNeeded, } from '../helpers'; +import { GitApiService } from '../scm/GitApiService'; import { DefaultPageNumber, DefaultPageSize, @@ -63,12 +64,12 @@ import { getCredentialsForConfig, } from './utils/utils'; -export class GitlabApiService { +export class GitlabApiService implements GitApiService { private readonly logger: LoggerService; private readonly integrations: ScmIntegrations; private readonly gitlabCredentialsProvider: CustomGitlabCredentialsProvider; private readonly config: Config; - // Cache for storing ETags (used for efficient caching of unchanged data returned by GitHub) + // Cache for storing ETags (used for efficient caching of unchanged data returned by GitLab) private readonly cache: CacheService; constructor( logger: LoggerService, @@ -147,9 +148,9 @@ export class GitlabApiService { } async getOrganizationsFromIntegrations( - search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + search?: string, ): Promise { const groups = new Map(); const result = await fetchFromAllIntegrations( @@ -200,13 +201,67 @@ export class GitlabApiService { }; } + private async fetchUserTokenGitlabIntegrationRepositories( + userTokens: Record, + fetcher: ( + glApi: InstanceType>, + credential: ExtendedGitlabCredentials, + errors: Map, + ) => Promise<{ totalCount?: number }>, + ): Promise<{ errors: GitlabFetchError[]; totalCount: number | undefined }> { + const allErrors: GitlabFetchError[] = []; + let totalCount: number | undefined; + const glConfigs = this.integrations.gitlab.list().map(i => i.config); + for (const glConfig of glConfigs) { + const normalizedBase = glConfig.baseUrl.replace(/\/$/, ''); + const token = + userTokens[normalizedBase] ?? userTokens[`https://${glConfig.host}`]; + + if (!token) { + continue; + } + const userCredential: ExtendedGitlabCredentials = { token }; + const userGitlab = buildGitlab( + { logger: this.logger, cache: this.cache }, + { credential: userCredential }, + glConfig.baseUrl, // NOTE: baseUrl, NOT apiBaseUrl (Gitlab constructor takes the base, not /api/v4) + ); + const dataFetchErrors = new Map(); + const result = await fetcher(userGitlab, userCredential, dataFetchErrors); + totalCount = (totalCount ?? 0) + (result.totalCount ?? 0); + dataFetchErrors.forEach(err => allErrors.push(err)); + } + return { errors: allErrors, totalCount }; + } + async getOrgRepositoriesFromIntegrations( orgName: string, search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + userTokens?: Record, ): Promise { const orgRepositories = new Map(); + + if (userTokens && Object.keys(userTokens).length > 0) { + const { errors: allErrors, totalCount } = + await this.fetchUserTokenGitlabIntegrationRepositories( + userTokens, + (userGitlab, userCredential, dataFetchErrors) => + addGitlabTokenOrgRepositories( + { logger: this.logger }, + userGitlab, + userCredential, + orgName, + orgRepositories, + dataFetchErrors, + { search, pageNumber, pageSize }, + ), + ); + const repoList = Array.from(orgRepositories.values()); + return { repositories: repoList, errors: allErrors, totalCount }; + } + const result = await fetchFromAllIntegrations( { logger: this.logger, @@ -266,8 +321,28 @@ export class GitlabApiService { search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + userTokens?: Record, ): Promise { const repositories = new Map(); + + if (userTokens && Object.keys(userTokens).length > 0) { + const { errors: allErrors, totalCount } = + await this.fetchUserTokenGitlabIntegrationRepositories( + userTokens, + (userGitlab, userCredential, dataFetchErrors) => + addGitlabTokenRepositories( + { logger: this.logger }, + userGitlab, + userCredential, + repositories, + dataFetchErrors, + { search, pageNumber, pageSize }, + ), + ); + const repoList = Array.from(repositories.values()); + return { repositories: repoList, errors: allErrors, totalCount }; + } + const result = await fetchFromAllIntegrations( { logger: this.logger, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/types.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/types.ts index b2aeed9ef7c..46016ed00ad 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/types.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/types.ts @@ -14,75 +14,18 @@ * limitations under the License. */ -import type { SerializedError } from '@backstage/errors'; import type { GitlabCredentials, GitlabCredentialsProvider, } from '@backstage/integration'; -// From https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-organizations -export type GitlabOrganization = { - name: string; - id: number; - description?: string; - url?: string; - html_url?: string; - avatar_url?: string; - public_repos?: number; - total_private_repos?: number; - /** - * Number of internal repositories, accessible to all members in a GH enterprise - */ - owned_private_repos?: number; -}; - -export type GitlabRepository = { - // id?: string; - name: string; - /** - * The full name of the repository in the form of owner/repo, should be the path_with_namespace property in gitlab - */ - full_name: string; - /** - * The API url to the repository - */ - url: string; - /** - * The HTML URL to the repository, web_url in gitlab - */ - html_url: string; - /** - * The default "main" branch of the repository to place the `catalog-info.yaml` file into - */ - default_branch: string; - /** - * The date-time the repository was last updated at - */ - updated_at?: string | null; -}; - -/** - * The type of credentials produced by the credential provider. - * - * @public - */ - -export type GitlabFetchError = { - type: 'token'; - error: SerializedError; -}; - -export type GitlabOrganizationResponse = { - organizations: GitlabOrganization[]; - errors: GitlabFetchError[]; - totalCount?: number; -}; - -export type GitlabRepositoryResponse = { - repositories: GitlabRepository[]; - errors: GitlabFetchError[]; - totalCount?: number; -}; +export type { + SCMFetchError as GitlabFetchError, + SCMOrganization as GitlabOrganization, + SCMOrganizationResponse as GitlabOrganizationResponse, + SCMRepository as GitlabRepository, + SCMRepositoryResponse as GitlabRepositoryResponse, +} from '../scm/types'; export type ExtendedGitlabCredentials = GitlabCredentials; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml b/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml index 49ade45c0ad..82e60f59c73 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml @@ -49,6 +49,24 @@ paths: example: status: 'ok' + /scm-hosts: + get: + operationId: findAllSCMHosts + summary: Retrieve the SCM Integration hosts + security: + - BearerAuth: [] + tags: [Management] + responses: + 200: + description: List of Integrations available to the Bulk Import plugin + content: + application/json: + schema: + $ref: '#/components/schemas/SCMHostList' + examples: + multipleRepos: + $ref: '#/components/examples/scmHosts' + /organizations: get: operationId: findAllOrganizations @@ -105,6 +123,7 @@ paths: - $ref: '#/components/parameters/sizePerIntegrationQueryParam' - $ref: '#/components/parameters/searchQueryParam' - $ref: '#/components/parameters/approvalToolParam' + - $ref: '#/components/parameters/xSCMTokensHeaderParam' responses: 200: description: Org Repository list was fetched successfully with no errors @@ -143,6 +162,7 @@ paths: - $ref: '#/components/parameters/sizePerIntegrationQueryParam' - $ref: '#/components/parameters/searchQueryParam' - $ref: '#/components/parameters/approvalToolParam' + - $ref: '#/components/parameters/xSCMTokensHeaderParam' responses: 200: description: Repository list was fetched successfully with no errors @@ -590,6 +610,15 @@ components: type: string enum: ['v1', 'v2'] default: 'v1' + xSCMTokensHeaderParam: + in: header + name: x-scm-tokens + description: > + Optional JSON-encoded map of SCM host URL to user authentication token. + Used to fetch repositories on behalf of the user for each configured SCM host. + required: false + schema: + $ref: '#/components/schemas/SCMTokenMap' pagePerIntegrationQueryParam: in: query @@ -684,6 +713,35 @@ components: default: GIT schemas: + SCMTokenMap: + title: SCM Token Map + type: object + description: > + Map of SCM integration base URL to the user's OAuth access token for that + host. Keys must match the base URLs returned by GET /scm-hosts (e.g. + "https://github.com" or "https://gitlab.corp.com"). Values must be + non-empty OAuth bearer tokens scoped to the minimum required access + (read-only repository listing). Unknown keys are silently ignored by + the server. + additionalProperties: + type: string + example: + 'https://github.com': 'ghp_xxx' + 'https://ghe.example.com': 'ghe_yyy' + + SCMHostList: + title: SCM Host List + type: object + properties: + github: + type: array + items: + type: string + gitlab: + type: array + items: + type: string + OrganizationList: title: Organization List type: object @@ -1024,6 +1082,15 @@ components: description: Backstage Permissions Framework JWT examples: + scmHosts: + summary: Multiple scmHosts + value: + github: + - 'https://github.com' + - 'https://ghe.example.com' + gitlab: + - 'https://gitlab.com' + multipleOrgs: summary: Multiple organizations value: diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts new file mode 100644 index 00000000000..8c59552330b --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/GitApiService.ts @@ -0,0 +1,132 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { LoggerService } from '@backstage/backend-plugin-api'; + +import gitUrlParse from 'git-url-parse'; + +import { + SCMFetchError, + SCMOrganizationResponse, + SCMRepository, + SCMRepositoryResponse, +} from './types'; + +export interface GitApiService { + getRepositoryFromIntegrations( + repoUrl: string, + ): Promise<{ repository?: SCMRepository; errors?: SCMFetchError[] }>; + + getOrganizationsFromIntegrations( + pageNumber: number, + pageSize: number, + search?: string, + ): Promise; + + getOrgRepositoriesFromIntegrations( + orgName: string, + search?: string, + pageNumber?: number, + pageSize?: number, + userTokens?: Record, + ): Promise; + + getRepositoriesFromIntegrations( + search?: string, + pageNumber?: number, + pageSize?: number, + userTokens?: Record, + ): Promise; + + filterLocationsAccessibleFromIntegrations( + locationUrls: string[], + ): Promise; + + getPullRequest( + repoUrl: string, + pullRequestNumber: number, + ): Promise<{ + title?: string; + body?: string; + merged?: boolean; + lastUpdated?: string; + prSha?: string; + prBranch?: string; + }>; + + findImportOpenPr( + logger: LoggerService, + input: { + repoUrl: string; + includeCatalogInfoContent?: boolean; + }, + ): Promise<{ + prNum?: number; + prUrl?: string; + prTitle?: string; + prBody?: string; + prCatalogInfoContent?: string; + lastUpdate?: string; + }>; + + getCatalogInfoFile( + logger: LoggerService, + input: { + repoUrl: string; + prNumber: number; + prHeadSha: string; + }, + ): Promise; + + submitPrToRepo( + logger: LoggerService, + input: { + repoUrl: string; + gitUrl: gitUrlParse.GitUrl; + defaultBranch?: string; + prTitle: string; + prBody: string; + catalogInfoContent: string; + }, + ): Promise<{ + prUrl?: string; + prNumber?: number; + hasChanges?: boolean; + lastUpdate?: string; + errors?: string[]; + }>; + + hasFileInRepo(input: { + repoUrl: string; + defaultBranch?: string; + fileName: string; + }): Promise; + + closeImportPR( + logger: LoggerService, + input: { + repoUrl: string; + gitUrl: gitUrlParse.GitUrl; + comment: string; + }, + ): Promise; + + deleteImportBranch(input: { + repoUrl: string; + gitUrl: gitUrlParse.GitUrl; + }): Promise; + + isRepoEmpty(input: { repoUrl: string }): Promise; +} diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/types.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/types.ts new file mode 100644 index 00000000000..05269275a28 --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/scm/types.ts @@ -0,0 +1,91 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { SerializedError } from '@backstage/errors'; + +/** + * Error type returned when fetching SCM resources from an integration. + * + * @public + */ + +export type SCMFetchError = + | { + type: 'app'; + appId: number; + error: SerializedError; + } + | { + type: 'token'; + error: SerializedError; + }; + +export type SCMOrganization = { + name: string; + id: number; + description?: string; + url?: string; + html_url?: string; + repos_url?: string; + events_url?: string; + hooks_url?: string; + issues_url?: string; + members_url?: string; + public_members_url?: string; + avatar_url?: string; + public_repos?: number; + total_private_repos?: number; + /** + * Number of internal repositories, accessible to all members in a GH enterprise + */ + owned_private_repos?: number; +}; + +export type SCMOrganizationResponse = { + organizations: SCMOrganization[]; + errors: SCMFetchError[]; + totalCount?: number; +}; + +export type SCMRepository = { + name: string; + /** + * The full name of the repository in the form of owner/repo + */ + full_name: string; + /** + * The API url to the repository + */ + url: string; + /** + * The HTML URL to the repository + */ + html_url: string; + /** + * The default "main" branch of the repository to place the `catalog-info.yaml` file into + */ + default_branch: string; + /** + * The date-time the repository was last updated at + */ + updated_at?: string | null; +}; + +export type SCMRepositoryResponse = { + repositories: SCMRepository[]; + errors: SCMFetchError[]; + totalCount?: number; +}; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/bulkImports.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/bulkImports.ts index 08da00f7324..47e86159fd9 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/bulkImports.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/bulkImports.ts @@ -53,6 +53,7 @@ import { paginateArray, parseGitURLForApprovalTool, } from '../../../helpers'; +import { GitApiService } from '../../../scm/GitApiService'; import { DefaultPageNumber, DefaultPageSize, @@ -348,7 +349,7 @@ function findImportCandidates( } async function createPR( - gitApiService: GithubApiService | GitlabApiService, + gitApiService: GitApiService, logger: LoggerService, req: Components.Schemas.ImportRequest, gitUrl: gitUrlParse.GitUrl, @@ -942,11 +943,13 @@ export async function findTaskImportStatusByRepo( repository.approvalTool as unknown as Components.Schemas.ApprovalTool; const pullRequest = await parsePullOrMergeRequestInfo( data.state?.checkpoints, - deps.gitlabApiService, - deps.githubApiService, - approvalTool, - deps.logger, - repoUrl, + { + githubApiService: deps.githubApiService, + gitlabApiService: deps.gitlabApiService, + approvalTool, + logger: deps.logger, + repoUrl, + }, ); if (pullRequest && approvalTool === 'GITLAB') { result.gitlab = { pullRequest }; @@ -1078,11 +1081,19 @@ export async function findOrchestratorImportStatusByRepo( async function parsePullOrMergeRequestInfo( checkpoints: Record, - githubApiService: GitlabApiService, - gitlabApiService: GithubApiService, - approvalTool: Components.Schemas.ApprovalTool, - logger: LoggerService, - repoUrl: string, + { + githubApiService, + gitlabApiService, + approvalTool, + logger, + repoUrl, + }: { + githubApiService: GithubApiService; + gitlabApiService: GitlabApiService; + approvalTool: Components.Schemas.ApprovalTool; + logger: LoggerService; + repoUrl: string; + }, ): Promise { // return errors ? if (approvalTool !== 'GITLAB' && approvalTool !== 'GIT') { @@ -1139,7 +1150,7 @@ export async function deleteImportByRepo( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, repoUrl: string, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/importStatus.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/importStatus.ts index 2a20495b566..3f0346ce9aa 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/importStatus.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/import/importStatus.ts @@ -23,14 +23,13 @@ import { getCatalogUrl, } from '../../../catalog/catalogUtils'; import type { Components } from '../../../generated/openapi'; -import type { GithubApiService } from '../../../github'; -import { GitlabApiService } from '../../../gitlab'; +import { GitApiService } from '../../../scm/GitApiService'; export async function getImportStatusFromLocations( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, repoUrl: string, @@ -59,7 +58,7 @@ async function getImportStatusWithCheckerFn( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, repoUrl: string, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/organization/organizations.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/organization/organizations.ts index 913621c5cc7..cd3da566099 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/organization/organizations.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/organization/organizations.ts @@ -17,11 +17,8 @@ import type { LoggerService } from '@backstage/backend-plugin-api'; import type { Components } from '../../../generated/openapi'; -import type { - GithubApiService, - GithubOrganizationResponse, -} from '../../../github'; -import { GitlabApiService, GitlabOrganizationResponse } from '../../../gitlab'; +import { GitApiService } from '../../../scm/GitApiService'; +import { SCMOrganizationResponse } from '../../../scm/types'; import { DefaultPageNumber, DefaultPageSize, @@ -30,7 +27,7 @@ import { export async function findAllOrganizations( logger: LoggerService, - gitApiService: GithubApiService | GitlabApiService, + gitApiService: GitApiService, search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, @@ -43,9 +40,9 @@ export async function findAllOrganizations( const allOrgsAccessible = await gitApiService.getOrganizationsFromIntegrations( - search, pageNumber, pageSize, + search, ); const errorList: string[] = []; @@ -80,9 +77,7 @@ export async function findAllOrganizations( }; } -function extractOrgMap( - allOrgsAccessible: GithubOrganizationResponse | GitlabOrganizationResponse, -) { +function extractOrgMap(allOrgsAccessible: SCMOrganizationResponse) { const orgMap = new Map(); for (const org of allOrgsAccessible.organizations ?? []) { let totalRepoCount: number | undefined; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts index b5dcab249b5..44554f485a9 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts @@ -21,11 +21,8 @@ import gitUrlParse from 'git-url-parse'; import { CatalogHttpClient } from '../../../catalog/catalogHttpClient'; import type { Components } from '../../../generated/openapi'; -import type { - GithubApiService, - GithubRepositoryResponse, -} from '../../../github'; -import { GitlabApiService, GitlabRepositoryResponse } from '../../../gitlab'; +import { GitApiService } from '../../../scm/GitApiService'; +import type { SCMRepositoryResponse } from '../../../scm/types'; import { DefaultPageNumber, DefaultPageSize, @@ -37,21 +34,22 @@ export async function findAllRepositories( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, reqParams?: { - approvalTool?: string; search?: string; checkStatus?: boolean; pageNumber?: number; pageSize?: number; + userTokens?: Record; }, ): Promise> { const search = reqParams?.search; const checkStatus = reqParams?.checkStatus ?? false; const pageNumber = reqParams?.pageNumber ?? DefaultPageNumber; const pageSize = reqParams?.pageSize ?? DefaultPageSize; + const userTokens = reqParams?.userTokens; deps.logger.debug( `Getting all repositories - (search,page,size)=('${ search ?? '' @@ -59,7 +57,7 @@ export async function findAllRepositories( ); const repos = await deps.gitApiService - .getRepositoriesFromIntegrations(search, pageNumber, pageSize) + .getRepositoriesFromIntegrations(search, pageNumber, pageSize, userTokens) .then(response => formatResponse(deps, response, checkStatus)); return repos; @@ -69,7 +67,7 @@ export async function findRepositoriesByOrganization( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, orgName: string, @@ -77,13 +75,20 @@ export async function findRepositoriesByOrganization( checkStatus: boolean = false, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, + userTokens?: Record, ): Promise> { deps.logger.debug( `Getting all repositories for org "${orgName}" - (search,page,size)=(${search},${pageNumber},${pageSize})..`, ); const glReposByOrg = await deps.gitApiService - .getOrgRepositoriesFromIntegrations(orgName, search, pageNumber, pageSize) + .getOrgRepositoriesFromIntegrations( + orgName, + search, + pageNumber, + pageSize, + userTokens, + ) .then(response => formatResponse(deps, response, checkStatus)); return glReposByOrg; @@ -114,10 +119,10 @@ async function formatResponse( deps: { logger: LoggerService; config: Config; - gitApiService: GithubApiService | GitlabApiService; + gitApiService: GitApiService; catalogHttpClient: CatalogHttpClient; }, - allReposAccessible: GithubRepositoryResponse | GitlabRepositoryResponse, + allReposAccessible: SCMRepositoryResponse, checkStatus: boolean, ) { const errorList = diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.ts new file mode 100644 index 00000000000..896d518c98e --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.ts @@ -0,0 +1,37 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; + +import type { Paths } from '../../../generated/openapi'; +import type { HandlerResponse } from '../handlers'; + +export async function findAllSCMHosts( + config: Config, +): Promise> { + const integrations = ScmIntegrations.fromConfig(config); + const githubHosts = integrations.github + .list() + .map(i => `https://${i.config.host}`); + const gitlabHosts = integrations.gitlab + .list() + .map(i => i.config.baseUrl.replace(/\/$/, '')); + return { + statusCode: 200, + responseBody: { github: githubHosts, gitlab: gitlabHosts }, + }; +} diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts index 06270dbc611..c1cdca01c85 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts @@ -79,6 +79,7 @@ import { findAllRepositories, findRepositoriesByOrganization, } from './handlers/repository'; +import { findAllSCMHosts } from './handlers/scm/scm'; /** * Router Options @@ -99,6 +100,7 @@ export interface RouterOptions { namespace Operations { export const PING = 'ping'; + export const FIND_ALL_SCM_HOSTS = 'findAllSCMHosts'; export const FIND_ALL_ORGANIZATIONS = 'findAllOrganizations'; export const FIND_ALL_REPOSITORIES = 'findAllRepositories'; @@ -126,6 +128,53 @@ namespace Operations { 'deleteOrchestratorImportByRepo'; } +const SCM_TOKENS_MAX_BYTES = 4096; + +const isValidTokenMap = (v: unknown): v is Record => + typeof v === 'object' && + v !== null && + Object.values(v).every(val => typeof val === 'string' && val.length > 0); + +function parseScmTokensHeader( + rawTokenHeader: string | undefined, + logger: LoggerService, +): Record | undefined { + let token: Record | undefined; + if (!rawTokenHeader) return undefined; + if (rawTokenHeader.length > SCM_TOKENS_MAX_BYTES) { + logger.warn('x-scm-tokens header exceeds maximum allowed size; ignoring'); + return undefined; + } + try { + token = JSON.parse(rawTokenHeader) as Record; + if (typeof token !== 'object' || token === null || Array.isArray(token)) { + throw new Error('x-scm-tokens must be a JSON object'); + } + } catch (e) { + throw new InputError( + `Invalid x-scm-tokens header: ${(e as Error).message}`, + ); + } + + if (!isValidTokenMap(token)) { + throw new InputError( + 'Invalid x-scm-tokens header: all values must be non-empty strings', + ); + } + return token; +} + +function extractUserTokens( + headers: + | Paths.FindAllRepositories.HeaderParameters + | Paths.FindRepositoriesByOrganization.HeaderParameters, + logger: LoggerService, +): Record | undefined { + const raw = headers['x-scm-tokens'] as string | undefined; + delete headers['x-scm-tokens']; // consumed; strip before any logging path + return parseScmTokensHeader(raw, logger); +} + /** * Router * @public @@ -156,7 +205,8 @@ export async function createRouter( 'orchestrator_repositories', ); const orchestratorWorkflowDao = new OrchestratorWorkflowDao(knex); - // This should probably be sometype of object that holds all the scm API service objects + // GitHub and GitLab both implement the GitApiService interface; the router + // selects the appropriate service per request based on approvalTool. const githubApiService = new GithubApiService(logger, config, cache); const gitlabApiService = new GitlabApiService(logger, config, cache); const catalogHttpClient = new CatalogHttpClient({ @@ -211,6 +261,14 @@ export async function createRouter( }, ); + api.register( + Operations.FIND_ALL_SCM_HOSTS, + async (_c: Context, _req: Request, res: Response) => { + const result = await findAllSCMHosts(config); + return res.status(result.statusCode).json(result.responseBody); + }, + ); + api.register( Operations.FIND_ALL_ORGANIZATIONS, async (c: Context, _req: Request, res: Response) => { @@ -248,6 +306,12 @@ export async function createRouter( q.pagePerIntegration = stringToNumber(q.pagePerIntegration); q.sizePerIntegration = stringToNumber(q.sizePerIntegration); q.checkImportStatus = stringToBoolean(q.checkImportStatus); + + const h: Paths.FindAllRepositories.HeaderParameters = { + ...c.request.headers, + }; + const userTokens = extractUserTokens(h, logger); + const response = await findAllRepositories( { logger, @@ -261,7 +325,7 @@ export async function createRouter( checkStatus: q.checkImportStatus, pageNumber: q.pagePerIntegration, pageSize: q.sizePerIntegration, - approvalTool: q.approvalTool, + userTokens, }, ); const repos = response.responseBody?.repositories; @@ -286,6 +350,12 @@ export async function createRouter( q.pagePerIntegration = stringToNumber(q.pagePerIntegration); q.sizePerIntegration = stringToNumber(q.sizePerIntegration); q.checkImportStatus = stringToBoolean(q.checkImportStatus); + + const h: Paths.FindRepositoriesByOrganization.HeaderParameters = { + ...c.request.headers, + }; + const userTokens = extractUserTokens(h, logger); + const response = await findRepositoriesByOrganization( { logger, @@ -299,6 +369,7 @@ export async function createRouter( q.checkImportStatus, q.pagePerIntegration, q.sizePerIntegration, + userTokens, ); const repos = response.responseBody?.repositories; return res.status(response.statusCode).json({ @@ -680,13 +751,18 @@ export async function createRouter( router.use(async (req, _res, next) => { if (req.path !== '/ping') { - await permissionCheck( - auditor, - api.matchOperation(req as OpenAPIRequest)?.operationId, - permissions, - httpAuth, - req, - ).catch(next); + try { + await permissionCheck( + auditor, + api.matchOperation(req as OpenAPIRequest)?.operationId, + permissions, + httpAuth, + req, + ); + } catch (e) { + next(e); + return; + } } next(); }); @@ -726,6 +802,10 @@ async function createAuditorEventByOperationId( case Operations.PING: auditorEvent = await auditCreateEvent(auditor, 'ping', req); break; + case Operations.FIND_ALL_SCM_HOSTS: + auditorEvent = await auditCreateEvent(auditor, 'scm-read', req); + break; + case Operations.FIND_ALL_ORGANIZATIONS: auditorEvent = await auditCreateEvent(auditor, 'org-read', req, { queryType: req.query.search ? 'by-query' : 'all', diff --git a/workspaces/bulk-import/plugins/bulk-import/README.md b/workspaces/bulk-import/plugins/bulk-import/README.md index f9e48f6cdb5..6c138be85ff 100644 --- a/workspaces/bulk-import/plugins/bulk-import/README.md +++ b/workspaces/bulk-import/plugins/bulk-import/README.md @@ -19,6 +19,8 @@ The sections below are relevant for static plugins. If the plugin is expected to - Follow the [GitHub Locations](https://backstage.io/docs/integrations/github/locations) to integrate GitHub integrations in your Backstage instance. For now, the plugin only supports loading catalog entities from github.com or GitHub Enterprise. +- _(Optional)_ To enable [on behalf of user access](#on-behalf-of-user-access), configure a GitHub and/or GitLab OAuth auth provider and ensure `ScmAuthApi` from `@backstage/integration-react` is registered in your application. Without this, the plugin falls back to server-side integration credentials. + --- **NOTE** @@ -72,6 +74,47 @@ g, user:default/, role:default/team_a ); ``` +## On Behalf of User Access + +The Bulk Import plugin can fetch repository and organization listings **on behalf of the signed-in user** using their OAuth credentials, so that users see only the repositories and organizations they personally have access to. + +### How It Works + +When `ScmAuthApi` (from `@backstage/integration-react`) is available in the application, the plugin: + +1. Calls `GET /api/bulk-import/scm-hosts` to discover the configured GitHub and GitLab integration host URLs. +2. Requests an OAuth token for each host from `ScmAuthApi` using a read-only scope (`repoWrite: false`). +3. Passes the collected tokens to the backend via the `X-SCM-Tokens` request header when listing repositories or organizations. + +The backend then uses these user tokens to call the SCM APIs on behalf of the user, returning only what that user can access. + +### Fallback Behavior + +The GitHub and GitLab auth providers are **soft dependencies** — if they are not configured the plugin degrades gracefully: + +- If `ScmAuthApi` is not registered in the application, no tokens are collected and the backend falls back to server-side credentials (GitHub App, PAT, or GitLab token). +- If a token cannot be obtained for a specific host (e.g., the user has not signed in via that provider, or no OAuth provider is registered for that host), that host is silently skipped. The backend uses its configured integration credentials for any host without a user token. +- Deployments that do not configure GitHub or GitLab OAuth providers continue to work exactly as before with no configuration changes required. + +### Configuring Auth Providers + +To enable user-scoped repository listings, configure the relevant auth providers in your `app-config.yaml`: + +```yaml +auth: + providers: + github: + development: + clientId: ${GITHUB_CLIENT_ID} + clientSecret: ${GITHUB_CLIENT_SECRET} + gitlab: + development: + clientId: ${GITLAB_CLIENT_ID} + clientSecret: ${GITLAB_CLIENT_SECRET} +``` + +Refer to the Backstage documentation for [GitHub auth](https://backstage.io/docs/auth/github/provider) and [GitLab auth](https://backstage.io/docs/auth/gitlab/provider) for full configuration details. + ## New Frontend System If you're using Backstage's new frontend system, add the plugin to your app: diff --git a/workspaces/bulk-import/plugins/bulk-import/package.json b/workspaces/bulk-import/plugins/bulk-import/package.json index ca7f7c191a2..043dcd1d717 100644 --- a/workspaces/bulk-import/plugins/bulk-import/package.json +++ b/workspaces/bulk-import/plugins/bulk-import/package.json @@ -53,6 +53,7 @@ "@backstage/core-components": "^0.18.6", "@backstage/core-plugin-api": "^1.12.2", "@backstage/frontend-plugin-api": "^0.13.4", + "@backstage/integration-react": "^1.2.16", "@backstage/plugin-app-react": "^0.1.0", "@backstage/plugin-catalog-import": "^0.13.9", "@backstage/plugin-catalog-react": "^1.21.6", diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts index 1ac12c4e52f..68bea911d04 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts @@ -39,6 +39,7 @@ import { ScaffolderBulkImportBackendClientPathProvider } from './ScaffolderBulkI // @public export type BulkImportAPI = { + getSCMHosts(): Promise<{ github: string[]; gitlab: string[] } | Response>; dataFetcher: ( page: number, size: number, @@ -98,6 +99,32 @@ export interface IBulkImportRESTPathProvider { sortColumn: AddedRepositoryColumnNameEnum, sortOrder: SortingOrderEnum, ): string; + getSCMHostPath(): string; +} + +export abstract class BulkImportRESTPathProviderBase implements IBulkImportRESTPathProvider { + abstract getCreateImportJobsPath(dryRun?: boolean): string | undefined; + abstract getDeleteImportActionPath( + repo: string, + defaultBranch: string, + approvalTool?: string, + ): string; + abstract getGetImportActionPath( + repo: string, + defaultBranch: string, + approvalTool?: string, + ): string; + abstract getGetImportJobsPath( + page: number, + size: number, + searchString: string, + sortColumn: AddedRepositoryColumnNameEnum, + sortOrder: SortingOrderEnum, + ): string; + + getSCMHostPath(): string { + return `/api/bulk-import/scm-hosts`; + } } export class BulkImportBackendClient implements BulkImportAPI { @@ -136,6 +163,7 @@ export class BulkImportBackendClient implements BulkImportAPI { options?: APITypes, ) { const { token: idToken } = await this.identityApi.getCredentials(); + const backendUrl = this.configApi.getString('backend.baseUrl'); const jsonResponse = await fetch( getApi(backendUrl, page, size, searchString, approvalTool, options), @@ -143,6 +171,9 @@ export class BulkImportBackendClient implements BulkImportAPI { headers: { 'Content-Type': 'application/json', ...(idToken && { Authorization: `Bearer ${idToken}` }), + ...(options?.scmAuthTokens && { + 'X-SCM-Tokens': JSON.stringify(options.scmAuthTokens), + }), }, }, ); @@ -208,6 +239,25 @@ export class BulkImportBackendClient implements BulkImportAPI { return jsonResponse.status === 204 ? null : await jsonResponse.json(); } + async getSCMHosts() { + const { token: idToken } = await this.identityApi.getCredentials(); + const backendUrl = this.configApi.getString('backend.baseUrl'); + const jsonResponse = await fetch( + `${backendUrl}${this.pathProvider.getSCMHostPath()}`, + { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + ...(idToken && { Authorization: `Bearer ${idToken}` }), + }, + }, + ); + if (jsonResponse.status !== 200 && jsonResponse.status !== 204) { + return jsonResponse; + } + return jsonResponse.json(); + } + async deleteImportAction( repo: string, defaultBranch: string, diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts index 528d6c686e4..5b10186e9eb 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts @@ -15,9 +15,9 @@ */ import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; -import { IBulkImportRESTPathProvider } from './BulkImportBackendClient'; +import { BulkImportRESTPathProviderBase } from './BulkImportBackendClient'; -export class OrchestratorBulkImportBackendClientPathProvider implements IBulkImportRESTPathProvider { +export class OrchestratorBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { getCreateImportJobsPath(dryRun?: boolean): string | undefined { return dryRun === true ? undefined @@ -27,17 +27,21 @@ export class OrchestratorBulkImportBackendClientPathProvider implements IBulkImp getDeleteImportActionPath( repo: string, _defaultBranch: string, - approvalTool: string, + approvalTool?: string, ): string { - return `/api/bulk-import/orchestrator-import/by-repo?repo=${repo}&approvalTool=${approvalTool}`; + const params = new URLSearchParams({ repo }); + if (approvalTool) params.set('approvalTool', approvalTool); + return `/api/bulk-import/orchestrator-import/by-repo?${params.toString()}`; } getGetImportActionPath( repo: string, _defaultBranch: string, - approvalTool: string, + approvalTool?: string, ): string { - return `/api/bulk-import/orchestrator-import/by-repo?repo=${repo}&approvalTool=${approvalTool}`; + const params = new URLSearchParams({ repo }); + if (approvalTool) params.set('approvalTool', approvalTool); + return `/api/bulk-import/orchestrator-import/by-repo?${params.toString()}`; } getGetImportJobsPath( @@ -47,6 +51,13 @@ export class OrchestratorBulkImportBackendClientPathProvider implements IBulkImp sortColumn: AddedRepositoryColumnNameEnum, sortOrder: SortingOrderEnum, ): string { - return `/api/bulk-import/orchestrator-workflows?page=${page}&size=${size}&search=${searchString}&sortColumn=${sortColumn}&sortOrder=${sortOrder}`; + const params = new URLSearchParams({ + page: String(page), + size: String(size), + search: searchString, + sortColumn, + sortOrder, + }); + return `/api/bulk-import/orchestrator-workflows?${params.toString()}`; } } diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts index 14133cae91f..ec9727a48fe 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts @@ -15,9 +15,9 @@ */ import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; -import { IBulkImportRESTPathProvider } from './BulkImportBackendClient'; +import { BulkImportRESTPathProviderBase } from './BulkImportBackendClient'; -export class PRBulkImportBackendClientPathProvider implements IBulkImportRESTPathProvider { +export class PRBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { getCreateImportJobsPath(dryRun?: boolean): string { return dryRun ? `/api/bulk-import/imports?dryRun=true` @@ -27,17 +27,21 @@ export class PRBulkImportBackendClientPathProvider implements IBulkImportRESTPat getDeleteImportActionPath( repo: string, defaultBranch: string, - approvalTool: string, + approvalTool?: string, ): string { - return `/api/bulk-import/import/by-repo?repo=${repo}&defaultBranch=${defaultBranch}&approvalTool=${approvalTool}`; + const params = new URLSearchParams({ repo, defaultBranch }); + if (approvalTool) params.set('approvalTool', approvalTool); + return `/api/bulk-import/import/by-repo?${params.toString()}`; } getGetImportActionPath( repo: string, defaultBranch: string, - approvalTool: string, + approvalTool?: string, ): string { - return `/api/bulk-import/import/by-repo?repo=${repo}&defaultBranch=${defaultBranch}&approvalTool=${approvalTool}`; + const params = new URLSearchParams({ repo, defaultBranch }); + if (approvalTool) params.set('approvalTool', approvalTool); + return `/api/bulk-import/import/by-repo?${params.toString()}`; } getGetImportJobsPath( @@ -47,6 +51,13 @@ export class PRBulkImportBackendClientPathProvider implements IBulkImportRESTPat sortColumn: AddedRepositoryColumnNameEnum, sortOrder: SortingOrderEnum, ): string { - return `/api/bulk-import/imports?page=${page}&size=${size}&search=${searchString}&sortColumn=${sortColumn}&sortOrder=${sortOrder}`; + const params = new URLSearchParams({ + page: String(page), + size: String(size), + search: searchString, + sortColumn, + sortOrder, + }); + return `/api/bulk-import/imports?${params.toString()}`; } } diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts index ee3c9610609..f64ee722180 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts @@ -15,9 +15,9 @@ */ import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; -import { IBulkImportRESTPathProvider } from './BulkImportBackendClient'; +import { BulkImportRESTPathProviderBase } from './BulkImportBackendClient'; -export class ScaffolderBulkImportBackendClientPathProvider implements IBulkImportRESTPathProvider { +export class ScaffolderBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { getCreateImportJobsPath(dryRun?: boolean): string | undefined { return dryRun === true ? undefined : `/api/bulk-import/task-imports`; } @@ -25,17 +25,21 @@ export class ScaffolderBulkImportBackendClientPathProvider implements IBulkImpor getDeleteImportActionPath( repo: string, _defaultBranch: string, - approvalTool: string, + approvalTool?: string, ): string { - return `/api/bulk-import/task-import/by-repo?repo=${repo}&approvalTool=${approvalTool}`; + const params = new URLSearchParams({ repo }); + if (approvalTool) params.set('approvalTool', approvalTool); + return `/api/bulk-import/task-import/by-repo?${params.toString()}`; } getGetImportActionPath( repo: string, _defaultBranch: string, - approvalTool: string, + approvalTool?: string, ): string { - return `/api/bulk-import/task-import/by-repo?repo=${repo}&approvalTool=${approvalTool}`; + const params = new URLSearchParams({ repo }); + if (approvalTool) params.set('approvalTool', approvalTool); + return `/api/bulk-import/task-import/by-repo?${params.toString()}`; } getGetImportJobsPath( @@ -45,6 +49,13 @@ export class ScaffolderBulkImportBackendClientPathProvider implements IBulkImpor sortColumn: AddedRepositoryColumnNameEnum, sortOrder: SortingOrderEnum, ): string { - return `/api/bulk-import/task-imports?page=${page}&size=${size}&search=${searchString}&sortColumn=${sortColumn}&sortOrder=${sortOrder}`; + const params = new URLSearchParams({ + page: String(page), + size: String(size), + search: searchString, + sortColumn, + sortOrder, + }); + return `/api/bulk-import/task-imports?${params.toString()}`; } } diff --git a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts index fd61953bebe..3762980bd24 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts @@ -21,14 +21,18 @@ import { configApiRef, identityApiRef, useApi, + useApiHolder, } from '@backstage/core-plugin-api'; +import { scmAuthApiRef } from '@backstage/integration-react'; import { useQuery } from '@tanstack/react-query'; import { bulkImportApiRef } from '../api/BulkImportBackendClient'; import { AddRepositoryData, + APITypes, ApprovalTool, + DataFetcherQueryParams, OrgAndRepoResponse, RepositoriesError, } from '../types'; @@ -37,15 +41,6 @@ import { prepareDataForRepositories, } from '../utils/repository-utils'; -export interface DataFetcherQueryParams { - showOrganizations?: boolean; - orgName?: string; - page?: number; - querySize?: number; - searchString?: string; - approvalTool: ApprovalTool; -} - export const useRepositories = ( options: DataFetcherQueryParams, pollInterval?: number, @@ -62,6 +57,8 @@ export const useRepositories = ( const identityApi = useApi(identityApiRef); const configApi = useApi(configApiRef); const bulkImportApi = useApi(bulkImportApiRef); + const apiHolder = useApiHolder(); + const scmAuth = apiHolder.get(scmAuthApiRef); const { value: user } = useAsync(async () => { const identityRef = await identityApi.getBackstageIdentity(); @@ -73,45 +70,76 @@ export const useRepositories = ( return url; }); - const fetchRepositories = async (queryOptions: DataFetcherQueryParams) => { - if (queryOptions?.showOrganizations) { - return await bulkImportApi.dataFetcher( - queryOptions?.page ?? 0, - queryOptions?.querySize ?? 0, - queryOptions?.searchString || '', - queryOptions?.approvalTool, - { - fetchOrganizations: true, - }, - ); - } - if (queryOptions?.orgName) { - return await bulkImportApi.dataFetcher( - queryOptions?.page ?? 0, - queryOptions?.querySize ?? 0, - queryOptions?.searchString || '', - queryOptions?.approvalTool, - { - orgName: queryOptions?.orgName, - }, - ); + const { value: scmAuthTokens, loading: tokenLoading } = useAsync(async () => { + if (!scmAuth) return undefined; + const hosts = await bulkImportApi.getSCMHosts(); + if (!hosts || hosts instanceof Response || !('github' in hosts)) + return undefined; + const urls = + options.approvalTool === ApprovalTool.Gitlab + ? hosts.gitlab + : hosts.github; + + if (!urls?.length) return undefined; + + const tokenRecord: Record = {}; + for (const url of urls) { + try { + const { token } = await scmAuth.getCredentials({ + url, + additionalScope: { repoWrite: false }, + }); + if (token) tokenRecord[url] = token; + } catch { + // No OAuth provider registered for this host — skip it and fall back + // to server-side credentials for that integration on the backend. + } } - return await bulkImportApi.dataFetcher( - queryOptions?.page ?? 0, - queryOptions?.querySize ?? 0, - queryOptions?.searchString || '', - queryOptions?.approvalTool, + return Object.keys(tokenRecord).length > 0 ? tokenRecord : undefined; + }, [scmAuth, bulkImportApi, options.approvalTool]); + + const fetchRepositories = async (queryOptions: DataFetcherQueryParams) => { + const apiOptions: APITypes = { + ...(queryOptions.showOrganizations && { fetchOrganizations: true }), + ...(queryOptions.orgName && { orgName: queryOptions.orgName }), + scmAuthTokens, + }; + return bulkImportApi.dataFetcher( + queryOptions.page ?? 0, + queryOptions.querySize ?? 0, + queryOptions.searchString ?? '', + queryOptions.approvalTool, + apiOptions, ); }; + const tokenGeneration = useMemo( + () => + scmAuthTokens + ? Object.entries(scmAuthTokens) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => `${k}=${v}`) + .join(',') + : '', + [scmAuthTokens], + ); + const { data: value, error, isLoading: isQueryLoading, } = useQuery( - [options?.showOrganizations ? 'organizations' : 'repositories', options], + [ + options?.showOrganizations ? 'organizations' : 'repositories', + options, + tokenGeneration, + ], () => fetchRepositories(options), - { refetchInterval: pollInterval || 60000, refetchOnWindowFocus: false }, + { + enabled: !tokenLoading, + refetchInterval: pollInterval || 60000, + refetchOnWindowFocus: false, + }, ); const prepareData = useMemo(() => { @@ -126,7 +154,7 @@ export const useRepositories = ( }, [options?.showOrganizations, value, user, baseUrl]); return { - loading: isQueryLoading, + loading: tokenLoading || isQueryLoading, data: prepareData, error: { ...(error ?? {}), diff --git a/workspaces/bulk-import/plugins/bulk-import/src/types/types.ts b/workspaces/bulk-import/plugins/bulk-import/src/types/types.ts index 8c7796f5264..869c059fc25 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/types/types.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/types/types.ts @@ -182,6 +182,7 @@ export type CreateImportJobRepository = { export type APITypes = { orgName?: string; fetchOrganizations?: boolean; + scmAuthTokens?: Record; }; export type ErrorType = { @@ -207,9 +208,10 @@ export interface RepositoriesError extends Error { } export type DataFetcherQueryParams = { - page: number; - querySize: number; + page?: number; + querySize?: number; showOrganizations?: boolean; orgName?: string; searchString?: string; + approvalTool: ApprovalTool; }; diff --git a/workspaces/bulk-import/plugins/bulk-import/src/utils/repository-utils.tsx b/workspaces/bulk-import/plugins/bulk-import/src/utils/repository-utils.tsx index 99b200901f7..80a2080df4b 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/utils/repository-utils.tsx +++ b/workspaces/bulk-import/plugins/bulk-import/src/utils/repository-utils.tsx @@ -548,13 +548,21 @@ export const getApi = ( approvalTool: string, options?: APITypes, ) => { + const params = new URLSearchParams({ + pagePerIntegration: String(page), + sizePerIntegration: String(size), + search: searchString, + approvalTool, + }); + if (options?.fetchOrganizations) { - return `${backendUrl}/api/bulk-import/organizations?pagePerIntegration=${page}&sizePerIntegration=${size}&search=${searchString}&approvalTool=${approvalTool}`; + return `${backendUrl}/api/bulk-import/organizations?${params.toString()}`; } if (options?.orgName) { - return `${backendUrl}/api/bulk-import/organizations/${options.orgName}/repositories?pagePerIntegration=${page}&sizePerIntegration=${size}&search=${searchString}&approvalTool=${approvalTool}`; + const orgName = encodeURIComponent(options?.orgName); + return `${backendUrl}/api/bulk-import/organizations/${orgName}/repositories?${params.toString()}`; } - return `${backendUrl}/api/bulk-import/repositories?pagePerIntegration=${page}&sizePerIntegration=${size}&search=${searchString}&approvalTool=${approvalTool}`; + return `${backendUrl}/api/bulk-import/repositories?${params.toString()}`; }; export const getCustomisedErrorMessage = ( diff --git a/workspaces/bulk-import/yarn.lock b/workspaces/bulk-import/yarn.lock index cbdb168726c..70a785c5eed 100644 --- a/workspaces/bulk-import/yarn.lock +++ b/workspaces/bulk-import/yarn.lock @@ -2762,7 +2762,27 @@ __metadata: languageName: node linkType: hard -"@backstage/integration@npm:^1.19.2, @backstage/integration@npm:^1.20.0": +"@backstage/integration-react@npm:^1.2.16": + version: 1.2.16 + resolution: "@backstage/integration-react@npm:1.2.16" + dependencies: + "@backstage/config": "npm:^1.3.6" + "@backstage/core-plugin-api": "npm:^1.12.4" + "@backstage/integration": "npm:^2.0.0" + "@material-ui/core": "npm:^4.12.2" + peerDependencies: + "@types/react": ^17.0.0 || ^18.0.0 + react: ^17.0.0 || ^18.0.0 + react-dom: ^17.0.0 || ^18.0.0 + react-router-dom: ^6.30.2 + peerDependenciesMeta: + "@types/react": + optional: true + checksum: 10c0/70ff6dca97e1ff797e322771061ad642cd8f138f63fd44b743fd2eab1d975823e55a6bd95a04a4fe05b9974301164ceaae5ff79e91e90520e77710405ddf43ba + languageName: node + linkType: hard + +"@backstage/integration@npm:^1.19.2, @backstage/integration@npm:^1.20.0, @backstage/integration@npm:^1.20.1": version: 1.20.1 resolution: "@backstage/integration@npm:1.20.1" dependencies: @@ -12410,6 +12430,7 @@ __metadata: "@backstage/dev-utils": "npm:^1.1.19" "@backstage/frontend-defaults": "npm:^0.3.5" "@backstage/frontend-plugin-api": "npm:^0.13.4" + "@backstage/integration-react": "npm:^1.2.16" "@backstage/plugin-app-react": "npm:^0.1.0" "@backstage/plugin-catalog-import": "npm:^0.13.9" "@backstage/plugin-catalog-react": "npm:^1.21.6" From d7f97d287c6869fe522f5af06ac6d768a13b3e1d Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Mon, 30 Mar 2026 08:16:16 -0400 Subject: [PATCH 02/11] feat(bulk-import): add tests for on behalf of Signed-off-by: Patrick Knight --- .../src/generated/openapi.d.ts | 8 +- .../src/generated/openapidocument.ts | 2 +- .../src/github/githubApiService.test.ts | 105 +++++++++++ .../src/gitlab/gitlabApiService.test.ts | 118 ++++++++++++ .../src/schema/openapi.yaml | 2 +- .../src/service/handlers/scm/scm.test.ts | 140 ++++++++++++++ .../src/service/router.test.ts | 5 + .../src/api/BulkImportBackendClient.test.ts | 87 ++++++++- .../src/api/BulkImportBackendClient.ts | 54 +----- .../src/api/BulkImportBackendClientBase.ts | 64 +++++++ ...atorBulkImportBackendClientPathProvider.ts | 2 +- .../PRBulkImportBackendClientPathProvider.ts | 2 +- ...lderBulkImportBackendClientPathProvider.ts | 2 +- .../src/hooks/useRepositories.test.ts | 175 ++++++++++++++++++ 14 files changed, 709 insertions(+), 57 deletions(-) create mode 100644 workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.test.ts create mode 100644 workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts index bc933f7e42a..037368dee74 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts @@ -29,7 +29,7 @@ declare namespace Components { export type SortOrderQueryParam = "asc" | "desc"; export type XSCMTokensHeaderParam = /** * SCM Token Map - * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. "https://github.com" or "https://gitlab.corp.com"). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. + * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. * * example: * { @@ -260,7 +260,7 @@ declare namespace Components { } /** * SCM Token Map - * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. "https://github.com" or "https://gitlab.corp.com"). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. + * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. * * example: * { @@ -510,7 +510,7 @@ declare namespace Paths { export type SizePerIntegration = number; export type XScmTokens = /** * SCM Token Map - * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. "https://github.com" or "https://gitlab.corp.com"). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. + * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. * * example: * { @@ -610,7 +610,7 @@ declare namespace Paths { export type SizePerIntegration = number; export type XScmTokens = /** * SCM Token Map - * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. "https://github.com" or "https://gitlab.corp.com"). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. + * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. * * example: * { diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts index e875b7e7d5e..00d08e8472c 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts @@ -1145,7 +1145,7 @@ const OPENAPI = ` "SCMTokenMap": { "title": "SCM Token Map", "type": "object", - "description": "Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. \"https://github.com\" or \"https://gitlab.corp.com\"). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server.\\n", + "description": "Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server.\\n", "additionalProperties": { "type": "string" }, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts index 6f7401bbadf..20b0e7bf4f7 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts @@ -378,4 +378,109 @@ describe('GithubApiService tests', () => { totalCount: 0, }); }); + + describe('with userTokens', () => { + it('uses the user-token path for getRepositoriesFromIntegrations when a matching host token is provided', async () => { + octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ + data: ghRepos, + }); + octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ + data: [], + }); + + const result = await githubApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + { 'https://github.com': 'user-oauth-token' }, + ); + + expect(result.repositories).toEqual(ghRepos); + expect(result.errors).toEqual([]); + // getAllCredentials is not called in the user-token path + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('returns empty repositories when userTokens is provided but no host matches an integration', async () => { + const result = await githubApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + { 'https://some-other-host.com': 'user-oauth-token' }, + ); + + expect(result.repositories).toEqual([]); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('falls back to server credentials when userTokens is undefined', async () => { + octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ + data: ghRepos, + }); + octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ + data: [], + }); + + const result = await githubApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + undefined, + ); + + // Server credentials path is used — getAllCredentials IS called + expect(mockGetAllCredentials).toHaveBeenCalled(); + expect(result.repositories).toEqual(ghRepos); + }); + + it('falls back to server credentials when userTokens is an empty object', async () => { + octokit.rest.repos.listForAuthenticatedUser.mockReturnValue({ + data: ghRepos, + }); + octokit.apps.listReposAccessibleToInstallation.mockReturnValue({ + data: [], + }); + + const result = await githubApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + {}, + ); + + expect(mockGetAllCredentials).toHaveBeenCalled(); + expect(result.repositories).toEqual(ghRepos); + }); + + it('uses the user-token path for getOrgRepositoriesFromIntegrations when a matching host token is provided', async () => { + octokit.rest.repos.listForOrg.mockReturnValue({ data: ghRepos }); + + const result = await githubApiService.getOrgRepositoriesFromIntegrations( + 'my-org', + undefined, + undefined, + undefined, + { 'https://github.com': 'user-oauth-token' }, + ); + + expect(result.repositories).toEqual(ghRepos); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('returns empty org repositories when userTokens provided but no host matches', async () => { + const result = await githubApiService.getOrgRepositoriesFromIntegrations( + 'my-org', + undefined, + undefined, + undefined, + { 'https://some-other-host.com': 'user-oauth-token' }, + ); + + expect(result.repositories).toEqual([]); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + }); }); diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts index 3887e706645..7dd89fa64f6 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts @@ -435,4 +435,122 @@ describe('GitlabApiService tests', () => { totalCount: 0, }); }); + + describe('with userTokens', () => { + const glRepos = [ + { + id: '1', + name: 'A', + path_with_namespace: 'backstage/A', + _links: { self: 'https://gitlab.com/api/v4/projects/1' }, + web_url: 'https://gitlab.com/backstage/A', + default_branch: 'master', + }, + { + id: '2', + name: 'B', + path_with_namespace: 'backstage/B', + _links: { self: 'https://gitlab.com/api/v4/projects/2' }, + web_url: 'https://gitlab.com/backstage/B', + default_branch: 'main', + }, + ]; + + it('uses the user-token path for getRepositoriesFromIntegrations when a matching host token is provided', async () => { + gitlabkit.Projects.all.mockReturnValue({ + data: glRepos, + paginationInfo: { total: 2 }, + }); + + const result = await gitlabApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + { 'https://gitlab.com': 'user-gitlab-oauth-token' }, + ); + + expect(result.repositories).toHaveLength(2); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('returns empty repositories when userTokens is provided but no host matches an integration', async () => { + const result = await gitlabApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + { 'https://some-other-host.com': 'user-gitlab-oauth-token' }, + ); + + expect(result.repositories).toEqual([]); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('falls back to server credentials when userTokens is undefined', async () => { + gitlabkit.Projects.all.mockReturnValue({ + data: glRepos, + paginationInfo: { total: 2 }, + }); + + await gitlabApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + undefined, + ); + + // Server credentials path — getAllCredentials IS called + expect(mockGetAllCredentials).toHaveBeenCalled(); + }); + + it('falls back to server credentials when userTokens is an empty object', async () => { + gitlabkit.Projects.all.mockReturnValue({ + data: glRepos, + paginationInfo: { total: 2 }, + }); + + await gitlabApiService.getRepositoriesFromIntegrations( + undefined, + undefined, + undefined, + {}, + ); + + expect(mockGetAllCredentials).toHaveBeenCalled(); + }); + + it('uses the user-token path for getOrgRepositoriesFromIntegrations when a matching host token is provided', async () => { + gitlabkit.Groups.allProjects.mockReturnValue({ + data: glRepos, + paginationInfo: { total: 2 }, + }); + + const result = await gitlabApiService.getOrgRepositoriesFromIntegrations( + 'my-group', + undefined, + undefined, + undefined, + { 'https://gitlab.com': 'user-gitlab-oauth-token' }, + ); + + expect(result.repositories).toHaveLength(2); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + + it('returns empty org repositories when userTokens provided but no host matches', async () => { + const result = await gitlabApiService.getOrgRepositoriesFromIntegrations( + 'my-group', + undefined, + undefined, + undefined, + { 'https://some-other-host.com': 'user-gitlab-oauth-token' }, + ); + + expect(result.repositories).toEqual([]); + expect(result.errors).toEqual([]); + expect(mockGetAllCredentials).not.toHaveBeenCalled(); + }); + }); }); diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml b/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml index 82e60f59c73..b2e18a71c68 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml @@ -719,7 +719,7 @@ components: description: > Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. - "https://github.com" or "https://gitlab.corp.com"). Values must be + https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.test.ts new file mode 100644 index 00000000000..08c29eb822d --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/scm/scm.test.ts @@ -0,0 +1,140 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { mockServices } from '@backstage/backend-test-utils'; + +import { findAllSCMHosts } from './scm'; + +describe('findAllSCMHosts', () => { + it('returns github and gitlab host URLs from config', async () => { + const config = mockServices.rootConfig({ + data: { + integrations: { + github: [ + { host: 'github.com', token: 'gh-token' }, + { host: 'enterprise.github.com', token: 'gh-ent-token' }, + ], + gitlab: [ + { + host: 'gitlab.com', + baseUrl: 'https://gitlab.com/', + token: 'gl-token', + }, + ], + }, + }, + }); + + const result = await findAllSCMHosts(config); + + expect(result.statusCode).toBe(200); + expect(result.responseBody?.github).toEqual([ + 'https://github.com', + 'https://enterprise.github.com', + ]); + expect(result.responseBody?.gitlab).toEqual(['https://gitlab.com']); + }); + + it('normalizes gitlab baseUrl by stripping trailing slash', async () => { + const config = mockServices.rootConfig({ + data: { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + baseUrl: 'https://gitlab.com/', + token: 'gl-token', + }, + ], + }, + }, + }); + + const result = await findAllSCMHosts(config); + + expect(result.statusCode).toBe(200); + // trailing slash should be stripped from gitlab baseUrl + expect(result.responseBody?.gitlab).toEqual( + expect.arrayContaining(['https://gitlab.com']), + ); + expect(result.responseBody?.gitlab?.every(u => !u.endsWith('/'))).toBe( + true, + ); + }); + + it('returns only github hosts when no gitlab integrations are explicitly configured', async () => { + const config = mockServices.rootConfig({ + data: { + integrations: { + github: [{ host: 'github.com', token: 'gh-token' }], + }, + }, + }); + + const result = await findAllSCMHosts(config); + + expect(result.statusCode).toBe(200); + expect(result.responseBody?.github).toEqual(['https://github.com']); + // Backstage's ScmIntegrations always adds a default gitlab.com entry + expect(result.responseBody?.gitlab).toContain('https://gitlab.com'); + }); + + it('returns only gitlab hosts when no github integrations are explicitly configured', async () => { + const config = mockServices.rootConfig({ + data: { + integrations: { + gitlab: [ + { + host: 'gitlab.com', + baseUrl: 'https://gitlab.com', + token: 'gl-token', + }, + ], + }, + }, + }); + + const result = await findAllSCMHosts(config); + + expect(result.statusCode).toBe(200); + expect(result.responseBody?.gitlab).toContain('https://gitlab.com'); + // Backstage's ScmIntegrations always adds a default github.com entry + expect(result.responseBody?.github).toContain('https://github.com'); + }); + + it('returns multiple github integrations in order', async () => { + const config = mockServices.rootConfig({ + data: { + integrations: { + github: [ + { host: 'github.com', token: 'token-1' }, + { host: 'ghe.example.com', token: 'token-2' }, + { host: 'another-ghe.corp.com', token: 'token-3' }, + ], + }, + }, + }); + + const result = await findAllSCMHosts(config); + + expect(result.statusCode).toBe(200); + expect(result.responseBody?.github).toEqual([ + 'https://github.com', + 'https://ghe.example.com', + 'https://another-ghe.corp.com', + ]); + }); +}); diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts index 2948783d74a..b9ba4e9a786 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts @@ -30,6 +30,11 @@ describe('router tests', () => { describe('permission framework denial', () => { it.each([ + [ + 'GET /scm-hosts', + async (req: request.SuperTest) => + req.get('/api/bulk-import/scm-hosts'), + ], [ 'GET /organizations', async (req: request.SuperTest) => diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts index 3f36bdef7bf..8d61c20db05 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts @@ -40,6 +40,19 @@ import { const LOCAL_ADDR = 'https://localhost:7007'; const handlers = [ + rest.get(`${LOCAL_ADDR}/api/bulk-import/scm-hosts`, (req, res, ctx) => { + const test = req.headers.get('Content-Type'); + if (test === 'application/json') { + return res( + ctx.status(200), + ctx.json({ + github: ['https://github.com'], + gitlab: ['https://gitlab.com'], + }), + ); + } + return res(ctx.status(404)); + }), rest.get(`${LOCAL_ADDR}/api/bulk-import/repositories`, (req, res, ctx) => { const searchParam = req.url.searchParams.get('search'); const test = req.headers.get('Content-Type'); @@ -59,7 +72,7 @@ const handlers = [ return res(ctx.status(404)); }), rest.get( - `${LOCAL_ADDR}/api/bulk-import/organizations/org/dessert/repositories`, + `${LOCAL_ADDR}/api/bulk-import/organizations/org%2Fdessert/repositories`, (req, res, ctx) => { const test = req.headers.get('Content-Type'); const searchParam = req.url.searchParams.get('search'); @@ -228,6 +241,78 @@ describe('BulkImportBackendClient with open-pull-requests', () => { }); }); + describe('getSCMHosts', () => { + it('should retrieve SCM hosts successfully', async () => { + const hosts = await bulkImportApi.getSCMHosts(); + expect(hosts).toEqual({ + github: ['https://github.com'], + gitlab: ['https://gitlab.com'], + }); + }); + + it('should return the response object when the server returns a non-200 status', async () => { + server.use( + rest.get( + `${LOCAL_ADDR}/api/bulk-import/scm-hosts`, + (_req, res, ctx) => { + return res(ctx.status(403)); + }, + ), + ); + + const response = await bulkImportApi.getSCMHosts(); + expect(response).toBeInstanceOf(Response); + expect((response as Response).status).toBe(403); + }); + }); + + describe('dataFetcher X-SCM-Tokens header', () => { + let fetchSpy: jest.SpyInstance; + + beforeEach(() => { + fetchSpy = jest.spyOn(global, 'fetch'); + }); + + afterEach(() => { + fetchSpy.mockRestore(); + }); + + it('should send the X-SCM-Tokens header when scmAuthTokens are provided', async () => { + const scmAuthTokens: Record = { + 'https://github.com': 'user-token-abc', + }; + await bulkImportApi.dataFetcher(1, 2, '', ApprovalTool.Git, { + scmAuthTokens, + }); + + const calledHeaders = fetchSpy.mock.calls[0][1]?.headers as Record< + string, + string + >; + expect(calledHeaders['X-SCM-Tokens']).toBe(JSON.stringify(scmAuthTokens)); + }); + + it('should not send the X-SCM-Tokens header when scmAuthTokens are not provided', async () => { + await bulkImportApi.dataFetcher(1, 2, '', ApprovalTool.Git); + + const calledHeaders = fetchSpy.mock.calls[0][1]?.headers as Record< + string, + string + >; + expect(calledHeaders['X-SCM-Tokens']).toBeUndefined(); + }); + + it('should not send X-SCM-Tokens when scmAuthTokens is an empty object', async () => { + await bulkImportApi.dataFetcher(1, 2, '', ApprovalTool.Git, {}); + + const calledHeaders = fetchSpy.mock.calls[0][1]?.headers as Record< + string, + string + >; + expect(calledHeaders['X-SCM-Tokens']).toBeUndefined(); + }); + }); + describe('getRepositories', () => { it('getRepositories should retrieve repositories successfully', async () => { const repositories = await bulkImportApi.dataFetcher( diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts index 68bea911d04..5c8b36d7a1d 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts @@ -33,10 +33,17 @@ import { SortingOrderEnum, } from '../types'; import { getApi } from '../utils/repository-utils'; +import { + BulkImportRESTPathProviderBase, + IBulkImportRESTPathProvider, +} from './BulkImportBackendClientBase'; import { OrchestratorBulkImportBackendClientPathProvider } from './OrchestratorBulkImportBackendClientPathProvider'; import { PRBulkImportBackendClientPathProvider } from './PRBulkImportBackendClientPathProvider'; import { ScaffolderBulkImportBackendClientPathProvider } from './ScaffolderBulkImportBackendClientPathProvider'; +export type { IBulkImportRESTPathProvider }; +export { BulkImportRESTPathProviderBase }; + // @public export type BulkImportAPI = { getSCMHosts(): Promise<{ github: string[]; gitlab: string[] } | Response>; @@ -80,53 +87,6 @@ export const bulkImportApiRef = createApiRef({ id: 'plugin.bulk-import.service', }); -export interface IBulkImportRESTPathProvider { - getCreateImportJobsPath(dryRun?: boolean): string | undefined; - getDeleteImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string; - getGetImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string; - getGetImportJobsPath( - page: number, - size: number, - searchString: string, - sortColumn: AddedRepositoryColumnNameEnum, - sortOrder: SortingOrderEnum, - ): string; - getSCMHostPath(): string; -} - -export abstract class BulkImportRESTPathProviderBase implements IBulkImportRESTPathProvider { - abstract getCreateImportJobsPath(dryRun?: boolean): string | undefined; - abstract getDeleteImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string; - abstract getGetImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string; - abstract getGetImportJobsPath( - page: number, - size: number, - searchString: string, - sortColumn: AddedRepositoryColumnNameEnum, - sortOrder: SortingOrderEnum, - ): string; - - getSCMHostPath(): string { - return `/api/bulk-import/scm-hosts`; - } -} - export class BulkImportBackendClient implements BulkImportAPI { private readonly configApi: ConfigApi; private readonly identityApi: IdentityApi; diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts new file mode 100644 index 00000000000..1462ac0dc7c --- /dev/null +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts @@ -0,0 +1,64 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; + +export interface IBulkImportRESTPathProvider { + getCreateImportJobsPath(dryRun?: boolean): string | undefined; + getDeleteImportActionPath( + repo: string, + defaultBranch: string, + approvalTool?: string, + ): string; + getGetImportActionPath( + repo: string, + defaultBranch: string, + approvalTool?: string, + ): string; + getGetImportJobsPath( + page: number, + size: number, + searchString: string, + sortColumn: AddedRepositoryColumnNameEnum, + sortOrder: SortingOrderEnum, + ): string; + getSCMHostPath(): string; +} + +export abstract class BulkImportRESTPathProviderBase implements IBulkImportRESTPathProvider { + abstract getCreateImportJobsPath(dryRun?: boolean): string | undefined; + abstract getDeleteImportActionPath( + repo: string, + defaultBranch: string, + approvalTool?: string, + ): string; + abstract getGetImportActionPath( + repo: string, + defaultBranch: string, + approvalTool?: string, + ): string; + abstract getGetImportJobsPath( + page: number, + size: number, + searchString: string, + sortColumn: AddedRepositoryColumnNameEnum, + sortOrder: SortingOrderEnum, + ): string; + + getSCMHostPath(): string { + return `/api/bulk-import/scm-hosts`; + } +} diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts index 5b10186e9eb..97a27c881ad 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts @@ -15,7 +15,7 @@ */ import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; -import { BulkImportRESTPathProviderBase } from './BulkImportBackendClient'; +import { BulkImportRESTPathProviderBase } from './BulkImportBackendClientBase'; export class OrchestratorBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { getCreateImportJobsPath(dryRun?: boolean): string | undefined { diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts index ec9727a48fe..a75fce3bbfb 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts @@ -15,7 +15,7 @@ */ import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; -import { BulkImportRESTPathProviderBase } from './BulkImportBackendClient'; +import { BulkImportRESTPathProviderBase } from './BulkImportBackendClientBase'; export class PRBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { getCreateImportJobsPath(dryRun?: boolean): string { diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts index f64ee722180..2243e672701 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts @@ -15,7 +15,7 @@ */ import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; -import { BulkImportRESTPathProviderBase } from './BulkImportBackendClient'; +import { BulkImportRESTPathProviderBase } from './BulkImportBackendClientBase'; export class ScaffolderBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { getCreateImportJobsPath(dryRun?: boolean): string | undefined { diff --git a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts index 6aa0a02ab2c..eda3eee0e03 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts @@ -21,9 +21,12 @@ import { mockGetOrganizations, mockGetRepositories } from '../mocks/mockData'; import { ApprovalTool } from '../types'; import { useRepositories } from './useRepositories'; +const mockUseApiHolder = jest.fn(); + jest.mock('@backstage/core-plugin-api', () => ({ ...jest.requireActual('@backstage/core-plugin-api'), useApi: jest.fn(), + useApiHolder: () => mockUseApiHolder(), })); jest.mock('@tanstack/react-query', () => ({ @@ -31,6 +34,13 @@ jest.mock('@tanstack/react-query', () => ({ useQuery: jest.fn(), })); +beforeEach(() => { + // Default: no scmAuth registered → tokenLoading is immediately false + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(undefined), + }); +}); + describe('useRepositories', () => { it('should return repositories', async () => { (useQuery as jest.Mock).mockReturnValue({ @@ -104,4 +114,169 @@ describe('useRepositories', () => { ).toBe(7); }); }); + + describe('scmAuth token collection', () => { + it('skips token fetching and renders successfully when scmAuth is not registered', async () => { + // Default mock: apiHolder.get returns undefined for scmAuthApiRef + (useQuery as jest.Mock).mockReturnValue({ + data: mockGetRepositories, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + expect(result.current.data?.repositories).toBeDefined(); + }); + }); + + it('collects tokens from scmAuth when it is registered and getSCMHosts succeeds', async () => { + const mockGetCredentials = jest + .fn() + .mockResolvedValue({ token: 'user-oauth-token-123' }); + const mockScmAuth = { getCredentials: mockGetCredentials }; + + const mockGetSCMHosts = jest.fn().mockResolvedValue({ + github: ['https://github.com'], + gitlab: [], + }); + const mockBulkImportApi = { + getSCMHosts: mockGetSCMHosts, + dataFetcher: jest.fn(), + }; + + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(mockScmAuth), + }); + + const mockUseApi = jest.requireMock('@backstage/core-plugin-api').useApi; + mockUseApi.mockImplementation((ref: { id: string }) => { + if (ref.id === 'plugin.bulk-import.service') return mockBulkImportApi; + return undefined; + }); + + (useQuery as jest.Mock).mockReturnValue({ + data: mockGetRepositories, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + }); + + expect(mockGetSCMHosts).toHaveBeenCalled(); + expect(mockGetCredentials).toHaveBeenCalledWith({ + url: 'https://github.com', + additionalScope: { repoWrite: false }, + }); + }); + + it('skips a host gracefully when scmAuth.getCredentials throws for it', async () => { + const mockGetCredentials = jest + .fn() + .mockRejectedValue(new Error('No OAuth provider for this host')); + const mockScmAuth = { getCredentials: mockGetCredentials }; + + const mockGetSCMHosts = jest.fn().mockResolvedValue({ + github: ['https://github.com'], + gitlab: [], + }); + const mockBulkImportApi = { + getSCMHosts: mockGetSCMHosts, + dataFetcher: jest.fn(), + }; + + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(mockScmAuth), + }); + + const mockUseApi = jest.requireMock('@backstage/core-plugin-api').useApi; + mockUseApi.mockImplementation((ref: { id: string }) => { + if (ref.id === 'plugin.bulk-import.service') return mockBulkImportApi; + return undefined; + }); + + (useQuery as jest.Mock).mockReturnValue({ + data: mockGetRepositories, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + // Should complete without throwing even though getCredentials rejected + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + }); + }); + + it('skips token fetching when getSCMHosts returns a Response error', async () => { + const mockScmAuth = { getCredentials: jest.fn() }; + const mockGetSCMHosts = jest + .fn() + .mockResolvedValue(new Response(null, { status: 403 })); + const mockBulkImportApi = { + getSCMHosts: mockGetSCMHosts, + dataFetcher: jest.fn(), + }; + + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(mockScmAuth), + }); + + const mockUseApi = jest.requireMock('@backstage/core-plugin-api').useApi; + mockUseApi.mockImplementation((ref: { id: string }) => { + if (ref.id === 'plugin.bulk-import.service') return mockBulkImportApi; + return undefined; + }); + + (useQuery as jest.Mock).mockReturnValue({ + data: mockGetRepositories, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + }); + + // getCredentials should not be called since getSCMHosts failed + expect(mockScmAuth.getCredentials).not.toHaveBeenCalled(); + }); + }); }); From 5ffd2964af9a3fe250b6e3a7a6e3ddf0ae8460a6 Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Mon, 30 Mar 2026 09:13:22 -0400 Subject: [PATCH 03/11] feat(bulk-import): fix scm-hosts audit event id Signed-off-by: Patrick Knight --- .../plugins/bulk-import-backend/src/service/router.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts index c1cdca01c85..f9de44ef598 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts @@ -803,7 +803,7 @@ async function createAuditorEventByOperationId( auditorEvent = await auditCreateEvent(auditor, 'ping', req); break; case Operations.FIND_ALL_SCM_HOSTS: - auditorEvent = await auditCreateEvent(auditor, 'scm-read', req); + auditorEvent = await auditCreateEvent(auditor, 'scm-hosts-read', req); break; case Operations.FIND_ALL_ORGANIZATIONS: From add1d9151c7a63c1b9e3764119b1f008a6394b83 Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Mon, 30 Mar 2026 09:21:29 -0400 Subject: [PATCH 04/11] feat(bulk-import): ensure that tokens are not logged to audit logger Signed-off-by: Patrick Knight --- .../.changeset/small-games-live.md | 1 + .../plugins/bulk-import-backend/README.md | 2 + .../src/service/router.test.ts | 124 ++++++++++++++++++ .../bulk-import-backend/src/service/router.ts | 43 +++--- 4 files changed, 151 insertions(+), 19 deletions(-) diff --git a/workspaces/bulk-import/.changeset/small-games-live.md b/workspaces/bulk-import/.changeset/small-games-live.md index 59e0a17bc23..93f723c795b 100644 --- a/workspaces/bulk-import/.changeset/small-games-live.md +++ b/workspaces/bulk-import/.changeset/small-games-live.md @@ -13,6 +13,7 @@ This release introduces the ability for the Bulk Import plugin to fetch reposito - Added a new `GET /api/bulk-import/scm-hosts` endpoint that returns the configured GitHub and GitLab integration host URLs as a `SCMHostList` object, enabling the frontend to discover which hosts to request OAuth tokens for. - The `GET /repositories` and `GET /organizations/{organizationName}/repositories` endpoints now accept an optional `x-scm-tokens` request header — a JSON map of SCM host base URL to user OAuth token. When tokens are present, repository listings reflect what the signed-in user can personally access rather than the full scope of the server-wide integration credentials. +- The `x-scm-tokens` header is stripped from the request immediately upon receipt, before the permission check and before any audit event is created, so OAuth token values are never persisted in audit logs. - When user tokens are provided for GitHub, the Octokit response cache is intentionally disabled to prevent cross-user ETag cache leakage. Server-side credential paths are not affected. - Introduced a shared `GitApiService` interface and common SCM types (`SCMOrganization`, `SCMRepository`, `SCMFetchError`, etc.) to unify the GitHub and GitLab service implementations under a consistent contract. diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/README.md b/workspaces/bulk-import/plugins/bulk-import-backend/README.md index 344576b0936..5e1ea11d4a9 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/README.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/README.md @@ -373,6 +373,8 @@ The GitHub and GitLab auth providers are **soft dependencies**. When user tokens When user tokens are provided for GitHub, the Octokit response cache is intentionally disabled to prevent cross-user ETag cache leakage. Server-side credential paths are not affected. +The `x-scm-tokens` header is stripped from the request immediately upon receipt — before the permission check and before any audit event is created — so OAuth token values are never persisted in audit logs. + #### New API Endpoint | Method | Path | Description | diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts index b9ba4e9a786..e31091f2c74 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts @@ -28,6 +28,130 @@ import { setupTest, startBackendServer } from '../../__fixtures__/testUtils'; describe('router tests', () => { const useTestData = setupTest(); + describe('x-scm-tokens middleware', () => { + it.each([ + [ + 'GET /repositories', + (req: request.SuperTest, header: string) => + req.get('/api/bulk-import/repositories').set('x-scm-tokens', header), + ], + [ + 'GET /organizations/:org/repositories', + (req: request.SuperTest, header: string) => + req + .get('/api/bulk-import/organizations/my-org-1/repositories') + .set('x-scm-tokens', header), + ], + ])( + '%s: returns 400 when x-scm-tokens is not a valid JSON object', + async ( + _endpoint: string, + reqHandler: ( + req: request.SuperTest, + header: string, + ) => request.Test, + ) => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const cases = [ + 'not-json', + '["array", "not", "object"]', + '{"host": 123}', + '{"host": ""}', + ]; + + for (const invalidHeader of cases) { + const response = await reqHandler( + request(backendServer), + invalidHeader, + ); + expect(response.status).toEqual(400); + } + }, + ); + + it.each([ + [ + 'GET /repositories', + (req: request.SuperTest, header: string) => + req.get('/api/bulk-import/repositories').set('x-scm-tokens', header), + ], + [ + 'GET /organizations/:org/repositories', + (req: request.SuperTest, header: string) => + req + .get('/api/bulk-import/organizations/my-org-1/repositories') + .set('x-scm-tokens', header), + ], + ])( + '%s: ignores x-scm-tokens and returns 200 when header exceeds size limit', + async ( + _endpoint: string, + reqHandler: ( + req: request.SuperTest, + header: string, + ) => request.Test, + ) => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const oversizedHeader = JSON.stringify({ + 'https://github.com': 'a'.repeat(4097), + }); + + const response = await reqHandler( + request(backendServer), + oversizedHeader, + ); + expect(response.status).toEqual(200); + }, + ); + + it.each([ + [ + 'GET /repositories', + (req: request.SuperTest, header: string) => + req.get('/api/bulk-import/repositories').set('x-scm-tokens', header), + ], + [ + 'GET /organizations/:org/repositories', + (req: request.SuperTest, header: string) => + req + .get('/api/bulk-import/organizations/my-org-1/repositories') + .set('x-scm-tokens', header), + ], + ])( + '%s: returns 200 and processes request when x-scm-tokens is a valid token map', + async ( + _endpoint: string, + reqHandler: ( + req: request.SuperTest, + header: string, + ) => request.Test, + ) => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const validHeader = JSON.stringify({ + 'https://github.com': 'gho_validUserToken', + }); + + const response = await reqHandler(request(backendServer), validHeader); + expect(response.status).toEqual(200); + }, + ); + }); + describe('permission framework denial', () => { it.each([ [ diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts index f9de44ef598..4398ee5d0ce 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts @@ -164,17 +164,6 @@ function parseScmTokensHeader( return token; } -function extractUserTokens( - headers: - | Paths.FindAllRepositories.HeaderParameters - | Paths.FindRepositoriesByOrganization.HeaderParameters, - logger: LoggerService, -): Record | undefined { - const raw = headers['x-scm-tokens'] as string | undefined; - delete headers['x-scm-tokens']; // consumed; strip before any logging path - return parseScmTokensHeader(raw, logger); -} - /** * Router * @public @@ -307,10 +296,9 @@ export async function createRouter( q.sizePerIntegration = stringToNumber(q.sizePerIntegration); q.checkImportStatus = stringToBoolean(q.checkImportStatus); - const h: Paths.FindAllRepositories.HeaderParameters = { - ...c.request.headers, - }; - const userTokens = extractUserTokens(h, logger); + const userTokens = res.locals.scmTokens as + | Record + | undefined; const response = await findAllRepositories( { @@ -351,10 +339,9 @@ export async function createRouter( q.sizePerIntegration = stringToNumber(q.sizePerIntegration); q.checkImportStatus = stringToBoolean(q.checkImportStatus); - const h: Paths.FindRepositoriesByOrganization.HeaderParameters = { - ...c.request.headers, - }; - const userTokens = extractUserTokens(h, logger); + const userTokens = res.locals.scmTokens as + | Record + | undefined; const response = await findRepositoriesByOrganization( { @@ -749,6 +736,24 @@ export async function createRouter( }); router.use(permissionIntegrationRouter); + // Strip x-scm-tokens from req.headers before the permission check and audit + // middleware run so that OAuth tokens are never captured in audit logs. + // The parsed map is stored in res.locals.scmTokens for handler use. + router.use((req, res, next) => { + const raw = req.headers['x-scm-tokens']; + const rawStr = Array.isArray(raw) ? raw[0] : raw; + delete req.headers['x-scm-tokens']; + if (rawStr) { + try { + res.locals.scmTokens = parseScmTokensHeader(rawStr, logger); + } catch (e) { + next(e); + return; + } + } + next(); + }); + router.use(async (req, _res, next) => { if (req.path !== '/ping') { try { From a99478d916263685e6e00838808502f1cb3ce2ff Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Mon, 30 Mar 2026 09:30:11 -0400 Subject: [PATCH 05/11] feat(bulk-import): fix openapi header type mismatch Signed-off-by: Patrick Knight --- .../plugins/bulk-import-backend/README.md | 2 +- .../api-docs/Apis/OrganizationApi.md | 2 +- .../api-docs/Apis/RepositoryApi.md | 2 +- .../src/generated/openapi.d.ts | 54 +++++++++---------- .../src/generated/openapidocument.ts | 5 +- .../src/schema/openapi.yaml | 5 +- .../src/service/router.test.ts | 2 +- 7 files changed, 35 insertions(+), 37 deletions(-) diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/README.md b/workspaces/bulk-import/plugins/bulk-import-backend/README.md index 5e1ea11d4a9..884e4bc95e0 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/README.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/README.md @@ -358,7 +358,7 @@ The plugin supports fetching repository and organization listings **on behalf of 1. The frontend calls `GET /api/bulk-import/scm-hosts` to retrieve the list of configured SCM integration host URLs, grouped by provider (`github` and `gitlab`). 2. For each host, the frontend requests an OAuth token from the Backstage `ScmAuthApi` (provided by `@backstage/integration-react`). -3. The collected tokens are sent to the backend via the optional `x-scm-tokens` request header — a JSON object mapping each integration base URL to the user's OAuth token. +3. The collected tokens are sent to the backend via the optional `x-scm-tokens` request header — a JSON-encoded string whose value, when parsed, maps each integration base URL to the user's OAuth token (e.g. `{"https://github.com":"ghp_xxx"}`). 4. The backend uses these user tokens to call the GitHub or GitLab APIs on behalf of the user, so the repository listings reflect what the signed-in user can personally access. #### Fallback Behavior diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md index 9f4f8a2f820..88160336be0 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md @@ -52,7 +52,7 @@ Fetch Repositories in the specified GitHub organization, provided it is accessib | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | | **search** | **String**| returns only the items that match the search string | [optional] [default to null] | | **approvalTool** | **String**| the approvalTool to use | [optional] [default to GIT] | -| **x-scm-tokens** | [**Map**](../Models/String.md)| Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. | [optional] [default to null] | +| **x-scm-tokens** | **String**| Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. The value must be a JSON string whose structure matches SCMTokenMap (keys are SCM base URLs, values are OAuth bearer tokens). | [optional] [default to null] | ### Return type diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md index d4418f0b35e..5aceccf9073 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md @@ -22,7 +22,7 @@ Fetch Organization Repositories accessible by Backstage Github Integrations | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | | **search** | **String**| returns only the items that match the search string | [optional] [default to null] | | **approvalTool** | **String**| the approvalTool to use | [optional] [default to GIT] | -| **x-scm-tokens** | [**Map**](../Models/String.md)| Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. | [optional] [default to null] | +| **x-scm-tokens** | **String**| Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. The value must be a JSON string whose structure matches SCMTokenMap (keys are SCM base URLs, values are OAuth bearer tokens). | [optional] [default to null] | ### Return type diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts index 037368dee74..601d3dd3eb4 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts @@ -13,7 +13,11 @@ import type { declare namespace Components { export interface HeaderParameters { apiVersionHeaderParam?: Parameters.ApiVersionHeaderParam; - xSCMTokensHeaderParam?: Parameters.XSCMTokensHeaderParam; + xSCMTokensHeaderParam?: /** + * example: + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} + */ + Parameters.XSCMTokensHeaderParam; } namespace Parameters { export type ApiVersionHeaderParam = "v1" | "v2"; @@ -27,17 +31,11 @@ declare namespace Components { export type SizeQueryParam = number; export type SortColumnQueryParam = "repository.name" | "repository.organization" | "repository.url" | "lastUpdate" | "status"; export type SortOrderQueryParam = "asc" | "desc"; - export type XSCMTokensHeaderParam = /** - * SCM Token Map - * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. - * + /** * example: - * { - * "https://github.com": "ghp_xxx", - * "https://ghe.example.com": "ghe_yyy" - * } + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} */ - Schemas.SCMTokenMap; + export type XSCMTokensHeaderParam = string; } export interface QueryParameters { pagePerIntegrationQueryParam?: Parameters.PagePerIntegrationQueryParam; @@ -500,7 +498,11 @@ declare namespace Paths { } namespace FindAllRepositories { export interface HeaderParameters { - "x-scm-tokens"?: Parameters.XScmTokens; + "x-scm-tokens"?: /** + * example: + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} + */ + Parameters.XScmTokens; } namespace Parameters { export type ApprovalTool = string; @@ -508,17 +510,11 @@ declare namespace Paths { export type PagePerIntegration = number; export type Search = string; export type SizePerIntegration = number; - export type XScmTokens = /** - * SCM Token Map - * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. - * + /** * example: - * { - * "https://github.com": "ghp_xxx", - * "https://ghe.example.com": "ghe_yyy" - * } + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} */ - Components.Schemas.SCMTokenMap; + export type XScmTokens = string; } export interface QueryParameters { checkImportStatus?: Parameters.CheckImportStatus; @@ -599,7 +595,11 @@ declare namespace Paths { } namespace FindRepositoriesByOrganization { export interface HeaderParameters { - "x-scm-tokens"?: Parameters.XScmTokens; + "x-scm-tokens"?: /** + * example: + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} + */ + Parameters.XScmTokens; } namespace Parameters { export type ApprovalTool = string; @@ -608,17 +608,11 @@ declare namespace Paths { export type PagePerIntegration = number; export type Search = string; export type SizePerIntegration = number; - export type XScmTokens = /** - * SCM Token Map - * Map of SCM integration base URL to the user's OAuth access token for that host. Keys must match the base URLs returned by GET /scm-hosts (e.g. https://github.com or https://gitlab.corp.com). Values must be non-empty OAuth bearer tokens scoped to the minimum required access (read-only repository listing). Unknown keys are silently ignored by the server. - * + /** * example: - * { - * "https://github.com": "ghp_xxx", - * "https://ghe.example.com": "ghe_yyy" - * } + * {"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"} */ - Components.Schemas.SCMTokenMap; + export type XScmTokens = string; } export interface PathParameters { organizationName: Parameters.OrganizationName; diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts index 00d08e8472c..7e31dbda1e5 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts @@ -1031,10 +1031,11 @@ const OPENAPI = ` "xSCMTokensHeaderParam": { "in": "header", "name": "x-scm-tokens", - "description": "Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host.\\n", + "description": "Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. The value must be a JSON string whose structure matches SCMTokenMap (keys are SCM base URLs, values are OAuth bearer tokens).\\n", "required": false, "schema": { - "$ref": "#/components/schemas/SCMTokenMap" + "type": "string", + "example": "{\"https://github.com\":\"ghp_xxx\",\"https://ghe.example.com\":\"ghe_yyy\"}" } }, "pagePerIntegrationQueryParam": { diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml b/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml index b2e18a71c68..100d78474fb 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/schema/openapi.yaml @@ -616,9 +616,12 @@ components: description: > Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. + The value must be a JSON string whose structure matches SCMTokenMap + (keys are SCM base URLs, values are OAuth bearer tokens). required: false schema: - $ref: '#/components/schemas/SCMTokenMap' + type: string + example: '{"https://github.com":"ghp_xxx","https://ghe.example.com":"ghe_yyy"}' pagePerIntegrationQueryParam: in: query diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts index e31091f2c74..e664496df10 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts @@ -43,7 +43,7 @@ describe('router tests', () => { .set('x-scm-tokens', header), ], ])( - '%s: returns 400 when x-scm-tokens is not a valid JSON object', + '%s: returns 400 when x-scm-tokens does not contain a valid JSON-encoded token map', async ( _endpoint: string, reqHandler: ( From 5e0af39a6e5e3181f921fe16bafa8f64bbf4c5cb Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Mon, 30 Mar 2026 09:36:50 -0400 Subject: [PATCH 06/11] feat(bulk-import): fix raw token in query key Signed-off-by: Patrick Knight --- .../src/hooks/useRepositories.test.ts | 54 +++++++++++++++++++ .../bulk-import/src/hooks/useRepositories.ts | 13 ++--- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts index eda3eee0e03..bb5b771e4bf 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts @@ -236,6 +236,60 @@ describe('useRepositories', () => { }); }); + it('does not include raw token values in the React Query key', async () => { + const secretToken = 'super-secret-oauth-token'; + const mockGetCredentials = jest + .fn() + .mockResolvedValue({ token: secretToken }); + const mockScmAuth = { getCredentials: mockGetCredentials }; + + const mockGetSCMHosts = jest.fn().mockResolvedValue({ + github: ['https://github.com'], + gitlab: [], + }); + const mockBulkImportApi = { + getSCMHosts: mockGetSCMHosts, + dataFetcher: jest.fn(), + }; + + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(mockScmAuth), + }); + + const mockUseApi = jest.requireMock('@backstage/core-plugin-api').useApi; + mockUseApi.mockImplementation((ref: { id: string }) => { + if (ref.id === 'plugin.bulk-import.service') return mockBulkImportApi; + return undefined; + }); + + (useQuery as jest.Mock).mockClear(); + (useQuery as jest.Mock).mockReturnValue({ + data: mockGetRepositories, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + }); + + // Inspect the query key from the most recent render — it must never + // expose raw token values; only the sorted host URLs should appear. + const lastQueryKey = (useQuery as jest.Mock).mock.calls.at(-1)?.[0]; + const serialised = JSON.stringify(lastQueryKey); + expect(serialised).not.toContain(secretToken); + expect(serialised).toContain('https://github.com'); + }); + it('skips token fetching when getSCMHosts returns a Response error', async () => { const mockScmAuth = { getCredentials: jest.fn() }; const mockGetSCMHosts = jest diff --git a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts index 3762980bd24..39d1a35629e 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts @@ -113,14 +113,11 @@ export const useRepositories = ( ); }; - const tokenGeneration = useMemo( + const scmAuthHosts = useMemo( () => - scmAuthTokens - ? Object.entries(scmAuthTokens) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([k, v]) => `${k}=${v}`) - .join(',') - : '', + Object.keys(scmAuthTokens ?? {}) + .sort() + .join(','), [scmAuthTokens], ); @@ -132,7 +129,7 @@ export const useRepositories = ( [ options?.showOrganizations ? 'organizations' : 'repositories', options, - tokenGeneration, + scmAuthHosts, ], () => fetchRepositories(options), { From 407b69929ff299cbb11bf275d7c383c4018318c4 Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Mon, 30 Mar 2026 10:06:41 -0400 Subject: [PATCH 07/11] feat(bulk-import): fix sonarqube issues Signed-off-by: Patrick Knight --- .../plugins/bulk-import-backend/README.md | 2 +- .../bulk-import-backend/__fixtures__/handlers.ts | 4 ++++ .../plugins/bulk-import-backend/scripts/openapi.sh | 2 +- .../src/generated/openapidocument.ts | 2 +- .../src/gitlab/gitlabApiService.test.ts | 13 ++++++------- .../bulk-import/plugins/bulk-import/dev/mocks.ts | 6 ++++++ .../src/api/BulkImportBackendClient.test.ts | 2 +- .../bulk-import/src/api/BulkImportBackendClient.ts | 8 +------- ...chestratorBulkImportBackendClientPathProvider.ts | 6 ++---- .../api/PRBulkImportBackendClientPathProvider.ts | 4 +--- ...ScaffolderBulkImportBackendClientPathProvider.ts | 6 ++---- .../bulk-import/src/hooks/useRepositories.ts | 2 +- 12 files changed, 27 insertions(+), 30 deletions(-) diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/README.md b/workspaces/bulk-import/plugins/bulk-import-backend/README.md index 884e4bc95e0..9d90448e229 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/README.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/README.md @@ -301,7 +301,7 @@ The Bulk Import Backend plugin emits audit events for various operations. Events - **`ping`**: tracks `GET` requests to the `/ping` endpoint, which allows to make sure the bulk import backend is up and running. -- **`scm-hosts-read`**: tracks `GET` requests to the `/scm-hosts` endpoint, which returns the list of configured GitHub and GitLab integration host URLs. +- **`scm-hosts-read`**: tracks `GET` requests to the `/scm-hosts-read` endpoint, which returns the list of configured GitHub and GitLab integration host URLs. - **`org-read`**: tracks `GET` requests to the `/organizations` endpoint, which returns the list of organizations accessible from all configured SCM Integrations (GitHub and GitLab). diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts b/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts index f2f23be8a87..6f4992978d2 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/handlers.ts @@ -122,6 +122,10 @@ export const DEFAULT_TEST_HANDLERS: RestHandler< ); }), + rest.get(`${LOCAL_ADDR}/orgs/my-org-1/repos`, (_, res, ctx) => { + return res(ctx.status(200), ctx.json([])); + }), + rest.get(`${LOCAL_ADDR}/orgs/my-ent-org-1/repos`, (_, res, ctx) => { return res( ctx.status(200), diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/scripts/openapi.sh b/workspaces/bulk-import/plugins/bulk-import-backend/scripts/openapi.sh index f863c6cb09a..6eae9017111 100755 --- a/workspaces/bulk-import/plugins/bulk-import-backend/scripts/openapi.sh +++ b/workspaces/bulk-import/plugins/bulk-import-backend/scripts/openapi.sh @@ -36,7 +36,7 @@ cat < "${OPENAPI_DOC_JS_FILE}" // prettier-ignore EOF echo 'const OPENAPI = `' >> "${OPENAPI_DOC_JS_FILE}" -cat ./src/schema/openapi.json | sed 's/\\n/\\\\n/g' >> "${OPENAPI_DOC_JS_FILE}" +cat ./src/schema/openapi.json | sed 's/\\n/\\\\n/g' | sed 's/\\"/\\\\"/g' >> "${OPENAPI_DOC_JS_FILE}" echo '`' >> "${OPENAPI_DOC_JS_FILE}" echo "export const openApiDocument = JSON.parse(OPENAPI);" >> "${OPENAPI_DOC_JS_FILE}" rm -f ./src/schema/openapi.json diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts index 7e31dbda1e5..532a4735c7a 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts @@ -1035,7 +1035,7 @@ const OPENAPI = ` "required": false, "schema": { "type": "string", - "example": "{\"https://github.com\":\"ghp_xxx\",\"https://ghe.example.com\":\"ghe_yyy\"}" + "example": "{\\"https://github.com\\":\\"ghp_xxx\\",\\"https://ghe.example.com\\":\\"ghe_yyy\\"}" } }, "pagePerIntegrationQueryParam": { diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts index 7dd89fa64f6..cec7800700d 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/gitlab/gitlabApiService.test.ts @@ -289,7 +289,11 @@ describe('GitlabApiService tests', () => { paginationInfo: { total: 1 }, }); - const result = await gitlabApiService.getOrganizationsFromIntegrations('A'); + const result = await gitlabApiService.getOrganizationsFromIntegrations( + 1, + 20, + 'A', + ); const expected_response = { organizations: [ @@ -493,12 +497,7 @@ describe('GitlabApiService tests', () => { paginationInfo: { total: 2 }, }); - await gitlabApiService.getRepositoriesFromIntegrations( - undefined, - undefined, - undefined, - undefined, - ); + await gitlabApiService.getRepositoriesFromIntegrations(); // Server credentials path — getAllCredentials IS called expect(mockGetAllCredentials).toHaveBeenCalled(); diff --git a/workspaces/bulk-import/plugins/bulk-import/dev/mocks.ts b/workspaces/bulk-import/plugins/bulk-import/dev/mocks.ts index f1933bd136c..4c7259b221f 100644 --- a/workspaces/bulk-import/plugins/bulk-import/dev/mocks.ts +++ b/workspaces/bulk-import/plugins/bulk-import/dev/mocks.ts @@ -131,6 +131,12 @@ export class MockBulkImportApi implements BulkImportAPI { i => i.repository.url === repo, ) as ImportJobStatus; } + + async getSCMHosts(): Promise< + { github: string[]; gitlab: string[] } | Response + > { + return { github: [], gitlab: [] }; + } } export const mockBulkImportApi = new MockBulkImportApi(); diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts index 8d61c20db05..4ebe3b6a9f5 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.test.ts @@ -270,7 +270,7 @@ describe('BulkImportBackendClient with open-pull-requests', () => { let fetchSpy: jest.SpyInstance; beforeEach(() => { - fetchSpy = jest.spyOn(global, 'fetch'); + fetchSpy = jest.spyOn(globalThis, 'fetch'); }); afterEach(() => { diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts index 5c8b36d7a1d..cc951461876 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts @@ -33,17 +33,11 @@ import { SortingOrderEnum, } from '../types'; import { getApi } from '../utils/repository-utils'; -import { - BulkImportRESTPathProviderBase, - IBulkImportRESTPathProvider, -} from './BulkImportBackendClientBase'; +import { IBulkImportRESTPathProvider } from './BulkImportBackendClientBase'; import { OrchestratorBulkImportBackendClientPathProvider } from './OrchestratorBulkImportBackendClientPathProvider'; import { PRBulkImportBackendClientPathProvider } from './PRBulkImportBackendClientPathProvider'; import { ScaffolderBulkImportBackendClientPathProvider } from './ScaffolderBulkImportBackendClientPathProvider'; -export type { IBulkImportRESTPathProvider }; -export { BulkImportRESTPathProviderBase }; - // @public export type BulkImportAPI = { getSCMHosts(): Promise<{ github: string[]; gitlab: string[] } | Response>; diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts index 97a27c881ad..1d4ced23585 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts @@ -36,12 +36,10 @@ export class OrchestratorBulkImportBackendClientPathProvider extends BulkImportR getGetImportActionPath( repo: string, - _defaultBranch: string, + defaultBranch: string, approvalTool?: string, ): string { - const params = new URLSearchParams({ repo }); - if (approvalTool) params.set('approvalTool', approvalTool); - return `/api/bulk-import/orchestrator-import/by-repo?${params.toString()}`; + return this.getDeleteImportActionPath(repo, defaultBranch, approvalTool); } getGetImportJobsPath( diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts index a75fce3bbfb..40151b359ac 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts @@ -39,9 +39,7 @@ export class PRBulkImportBackendClientPathProvider extends BulkImportRESTPathPro defaultBranch: string, approvalTool?: string, ): string { - const params = new URLSearchParams({ repo, defaultBranch }); - if (approvalTool) params.set('approvalTool', approvalTool); - return `/api/bulk-import/import/by-repo?${params.toString()}`; + return this.getDeleteImportActionPath(repo, defaultBranch, approvalTool); } getGetImportJobsPath( diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts index 2243e672701..212b41b8c06 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts @@ -34,12 +34,10 @@ export class ScaffolderBulkImportBackendClientPathProvider extends BulkImportRES getGetImportActionPath( repo: string, - _defaultBranch: string, + defaultBranch: string, approvalTool?: string, ): string { - const params = new URLSearchParams({ repo }); - if (approvalTool) params.set('approvalTool', approvalTool); - return `/api/bulk-import/task-import/by-repo?${params.toString()}`; + return this.getDeleteImportActionPath(repo, defaultBranch, approvalTool); } getGetImportJobsPath( diff --git a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts index 39d1a35629e..0ba8bbcb4aa 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts @@ -116,7 +116,7 @@ export const useRepositories = ( const scmAuthHosts = useMemo( () => Object.keys(scmAuthTokens ?? {}) - .sort() + .sort((a, b) => a.localeCompare(b)) .join(','), [scmAuthTokens], ); From e75cf7a53e0859fa5c28aacc5e2fc99f136364c8 Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Mon, 30 Mar 2026 10:19:20 -0400 Subject: [PATCH 08/11] feat(bulk-import): fix sonarqube duplication issues Signed-off-by: Patrick Knight --- .../src/github/githubApiService.ts | 192 +++++++----------- .../src/api/BulkImportBackendClientBase.ts | 23 ++- ...atorBulkImportBackendClientPathProvider.ts | 26 +-- .../PRBulkImportBackendClientPathProvider.ts | 26 +-- ...lderBulkImportBackendClientPathProvider.ts | 26 +-- workspaces/bulk-import/yarn.lock | 23 +-- 6 files changed, 98 insertions(+), 218 deletions(-) diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts index bbbf4ef550b..88532325dbb 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.ts @@ -95,6 +95,35 @@ export class GithubApiService implements GitApiService { this.cache = cacheService; } + private get integrationDeps() { + return { + logger: this.logger, + cache: this.cache, + githubCredentialsProvider: this.githubCredentialsProvider, + }; + } + + private get executionDeps() { + return { + ...this.integrationDeps, + config: this.config, + }; + } + + private buildRepositoryResponse( + repositories: Map, + result: { data: number[]; errors: Map }, + pageSize: number, + ): GithubRepositoryResponse { + const repoList = Array.from(repositories.values()); + const totalCount = computeTotalCount(repoList, result.data, pageSize); + return { + repositories: repoList, + errors: Array.from(result.errors.values()), + totalCount, + }; + } + async getCredentials(repoUrl: string): Promise<{ token: string }> { const provider = DefaultGithubCredentialsProvider.fromIntegrations( this.integrations, @@ -115,11 +144,7 @@ export class GithubApiService implements GitApiService { errors?: GithubFetchError[]; }> { const { data, errors } = await fetchFromMatchedIntegration( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, repoUrl, async (octokit, gitUrl) => { @@ -155,11 +180,7 @@ export class GithubApiService implements GitApiService { ): Promise { const orgs = new Map(); const result = await fetchFromAllIntegrations( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, { dataFetcher: async ( @@ -293,11 +314,7 @@ export class GithubApiService implements GitApiService { } const result = await fetchFromAllIntegrations( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, { dataFetcher: async ( @@ -356,13 +373,7 @@ export class GithubApiService implements GitApiService { }, ); - const repoList = Array.from(repositories.values()); - const totalCount = computeTotalCount(repoList, result.data, pageSize); - return { - repositories: repoList, - errors: Array.from(result.errors?.values() ?? []), - totalCount, - }; + return this.buildRepositoryResponse(repositories, result, pageSize); } /** @@ -401,11 +412,7 @@ export class GithubApiService implements GitApiService { } const result = await fetchFromAllIntegrations( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, { dataFetcher: async ( @@ -456,13 +463,7 @@ export class GithubApiService implements GitApiService { }, ); - const repoList = Array.from(repositories.values()); - const totalCount = computeTotalCount(repoList, result.data, pageSize); - return { - repositories: repoList, - errors: Array.from(result.errors?.values() ?? []), - totalCount, - }; + return this.buildRepositoryResponse(repositories, result, pageSize); } async filterLocationsAccessibleFromIntegrations( @@ -473,44 +474,36 @@ export class GithubApiService implements GitApiService { const allAccessibleAppOrgs = new Set(); const allAccessibleTokenOrgs = new Set(); const allAccessibleUsernames = new Set(); - await fetchFromAllIntegrations( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, - this.integrations, - { - dataFetcher: async ( - octokit: Octokit, - credential: ExtendedGithubCredentials, - ghConfig: GithubIntegrationConfig, - ) => { - if (isGithubAppCredential(credential)) { - const appOrgMap = await getAllAppOrgs( - this.githubCredentialsProvider, - ghConfig, - credential.accountLogin, - ); - for (const [_, ghOrg] of appOrgMap) { - allAccessibleAppOrgs.add(ghOrg.name); - } - } else { - // find authenticated GitHub owner... - const username = (await octokit.rest.users.getAuthenticated())?.data - ?.login; - if (username) { - allAccessibleUsernames.add(username); - } - // ... along with orgs accessible from the token auth - (await octokit.paginate(octokit.rest.orgs.listForAuthenticatedUser)) - ?.map(org => org.login) - ?.forEach(orgName => allAccessibleTokenOrgs.add(orgName)); + await fetchFromAllIntegrations(this.integrationDeps, this.integrations, { + dataFetcher: async ( + octokit: Octokit, + credential: ExtendedGithubCredentials, + ghConfig: GithubIntegrationConfig, + ) => { + if (isGithubAppCredential(credential)) { + const appOrgMap = await getAllAppOrgs( + this.githubCredentialsProvider, + ghConfig, + credential.accountLogin, + ); + for (const [_, ghOrg] of appOrgMap) { + allAccessibleAppOrgs.add(ghOrg.name); } - return {}; - }, + } else { + // find authenticated GitHub owner... + const username = (await octokit.rest.users.getAuthenticated())?.data + ?.login; + if (username) { + allAccessibleUsernames.add(username); + } + // ... along with orgs accessible from the token auth + (await octokit.paginate(octokit.rest.orgs.listForAuthenticatedUser)) + ?.map(org => org.login) + ?.forEach(orgName => allAccessibleTokenOrgs.add(orgName)); + } + return {}; }, - ); + }); return locationUrls.filter(loc => { if (!locationGitOwnerMap.has(loc)) { @@ -537,11 +530,7 @@ export class GithubApiService implements GitApiService { prBranch?: string; }> { const { data } = await fetchFromMatchedIntegration( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, repoUrl, async (octokit, gitUrl) => { @@ -576,11 +565,7 @@ export class GithubApiService implements GitApiService { body?: string, ): Promise { await fetchFromMatchedIntegration( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, repoUrl, async (octokit, gitUrl) => { @@ -704,11 +689,7 @@ export class GithubApiService implements GitApiService { }> { const branchName = getBranchName(this.config); const { data } = await fetchFromMatchedIntegration( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, input.repoUrl, async (octokit, gitUrl) => { @@ -740,11 +721,7 @@ export class GithubApiService implements GitApiService { }, ): Promise { const { data } = await fetchFromMatchedIntegration( - { - logger: this.logger, - cache: this.cache, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.integrationDeps, this.integrations, input.repoUrl, async (octokit, gitUrl) => { @@ -792,12 +769,7 @@ export class GithubApiService implements GitApiService { const errors: any[] = []; const result = await executeFunctionOnFirstSuccessfulIntegration( - { - logger: this.logger, - cache: this.cache, - config: this.config, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.executionDeps, this.integrations, { repoUrl: input.repoUrl, @@ -975,12 +947,7 @@ export class GithubApiService implements GitApiService { fileName: string; }): Promise { const fileExists = await executeFunctionOnFirstSuccessfulIntegration( - { - logger: this.logger, - cache: this.cache, - config: this.config, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.executionDeps, this.integrations, { repoUrl: input.repoUrl, @@ -1020,12 +987,7 @@ export class GithubApiService implements GitApiService { }, ): Promise { await executeFunctionOnFirstSuccessfulIntegration( - { - logger: this.logger, - cache: this.cache, - config: this.config, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.executionDeps, this.integrations, { repoUrl: input.repoUrl, @@ -1068,12 +1030,7 @@ export class GithubApiService implements GitApiService { gitUrl: gitUrlParse.GitUrl; }): Promise { await executeFunctionOnFirstSuccessfulIntegration( - { - logger: this.logger, - cache: this.cache, - config: this.config, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.executionDeps, this.integrations, { repoUrl: input.repoUrl, @@ -1101,12 +1058,7 @@ export class GithubApiService implements GitApiService { async isRepoEmpty(input: { repoUrl: string }): Promise { return await executeFunctionOnFirstSuccessfulIntegration( - { - logger: this.logger, - cache: this.cache, - config: this.config, - githubCredentialsProvider: this.githubCredentialsProvider, - }, + this.executionDeps, this.integrations, { repoUrl: input.repoUrl, diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts index 1462ac0dc7c..208755cf99f 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts @@ -45,18 +45,33 @@ export abstract class BulkImportRESTPathProviderBase implements IBulkImportRESTP defaultBranch: string, approvalTool?: string, ): string; - abstract getGetImportActionPath( + + getGetImportActionPath( repo: string, defaultBranch: string, approvalTool?: string, - ): string; - abstract getGetImportJobsPath( + ): string { + return this.getDeleteImportActionPath(repo, defaultBranch, approvalTool); + } + + protected abstract getImportJobsBasePath(): string; + + getGetImportJobsPath( page: number, size: number, searchString: string, sortColumn: AddedRepositoryColumnNameEnum, sortOrder: SortingOrderEnum, - ): string; + ): string { + const params = new URLSearchParams({ + page: String(page), + size: String(size), + search: searchString, + sortColumn, + sortOrder, + }); + return `${this.getImportJobsBasePath()}?${params.toString()}`; + } getSCMHostPath(): string { return `/api/bulk-import/scm-hosts`; diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts index 1d4ced23585..7834c64ec0d 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; import { BulkImportRESTPathProviderBase } from './BulkImportBackendClientBase'; export class OrchestratorBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { @@ -34,28 +33,7 @@ export class OrchestratorBulkImportBackendClientPathProvider extends BulkImportR return `/api/bulk-import/orchestrator-import/by-repo?${params.toString()}`; } - getGetImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string { - return this.getDeleteImportActionPath(repo, defaultBranch, approvalTool); - } - - getGetImportJobsPath( - page: number, - size: number, - searchString: string, - sortColumn: AddedRepositoryColumnNameEnum, - sortOrder: SortingOrderEnum, - ): string { - const params = new URLSearchParams({ - page: String(page), - size: String(size), - search: searchString, - sortColumn, - sortOrder, - }); - return `/api/bulk-import/orchestrator-workflows?${params.toString()}`; + protected getImportJobsBasePath(): string { + return `/api/bulk-import/orchestrator-workflows`; } } diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts index 40151b359ac..79b480f1b6c 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; import { BulkImportRESTPathProviderBase } from './BulkImportBackendClientBase'; export class PRBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { @@ -34,28 +33,7 @@ export class PRBulkImportBackendClientPathProvider extends BulkImportRESTPathPro return `/api/bulk-import/import/by-repo?${params.toString()}`; } - getGetImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string { - return this.getDeleteImportActionPath(repo, defaultBranch, approvalTool); - } - - getGetImportJobsPath( - page: number, - size: number, - searchString: string, - sortColumn: AddedRepositoryColumnNameEnum, - sortOrder: SortingOrderEnum, - ): string { - const params = new URLSearchParams({ - page: String(page), - size: String(size), - search: searchString, - sortColumn, - sortOrder, - }); - return `/api/bulk-import/imports?${params.toString()}`; + protected getImportJobsBasePath(): string { + return `/api/bulk-import/imports`; } } diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts index 212b41b8c06..48d0fa8f1d0 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; import { BulkImportRESTPathProviderBase } from './BulkImportBackendClientBase'; export class ScaffolderBulkImportBackendClientPathProvider extends BulkImportRESTPathProviderBase { @@ -32,28 +31,7 @@ export class ScaffolderBulkImportBackendClientPathProvider extends BulkImportRES return `/api/bulk-import/task-import/by-repo?${params.toString()}`; } - getGetImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string { - return this.getDeleteImportActionPath(repo, defaultBranch, approvalTool); - } - - getGetImportJobsPath( - page: number, - size: number, - searchString: string, - sortColumn: AddedRepositoryColumnNameEnum, - sortOrder: SortingOrderEnum, - ): string { - const params = new URLSearchParams({ - page: String(page), - size: String(size), - search: searchString, - sortColumn, - sortOrder, - }); - return `/api/bulk-import/task-imports?${params.toString()}`; + protected getImportJobsBasePath(): string { + return `/api/bulk-import/task-imports`; } } diff --git a/workspaces/bulk-import/yarn.lock b/workspaces/bulk-import/yarn.lock index 70a785c5eed..745e9033c0d 100644 --- a/workspaces/bulk-import/yarn.lock +++ b/workspaces/bulk-import/yarn.lock @@ -2741,28 +2741,7 @@ __metadata: languageName: node linkType: hard -"@backstage/integration-react@npm:^1.2.14, @backstage/integration-react@npm:^1.2.16": - version: 1.2.16 - resolution: "@backstage/integration-react@npm:1.2.16" - dependencies: - "@backstage/config": "npm:^1.3.6" - "@backstage/core-plugin-api": "npm:^1.12.4" - "@backstage/integration": "npm:^2.0.0" - "@material-ui/core": "npm:^4.12.2" - "@material-ui/icons": "npm:^4.9.1" - peerDependencies: - "@types/react": ^17.0.0 || ^18.0.0 - react: ^17.0.0 || ^18.0.0 - react-dom: ^17.0.0 || ^18.0.0 - react-router-dom: ^6.30.2 - peerDependenciesMeta: - "@types/react": - optional: true - checksum: 10c0/70ff6dca97e1ff797e322771061ad642cd8f138f63fd44b743fd2eab1d975823e55a6bd95a04a4fe05b9974301164ceaae5ff79e91e90520e77710405ddf43ba - languageName: node - linkType: hard - -"@backstage/integration-react@npm:^1.2.16": +"@backstage/integration-react@npm:^1.2.14, @backstage/integration-react@npm:^1.2.15, @backstage/integration-react@npm:^1.2.16": version: 1.2.16 resolution: "@backstage/integration-react@npm:1.2.16" dependencies: From 6ebe2b2cf31c29cfdfc76a9a70dc1627fd7a320c Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Mon, 30 Mar 2026 16:52:08 -0400 Subject: [PATCH 09/11] feat(bulk-import): fix e2e tests Signed-off-by: Patrick Knight --- workspaces/bulk-import/e2e-tests/app.test.ts | 3 +++ .../bulk-import/e2e-tests/utils/apiUtils.ts | 21 +++++++++++++++++-- .../src/github/githubApiService.test.ts | 7 +------ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/workspaces/bulk-import/e2e-tests/app.test.ts b/workspaces/bulk-import/e2e-tests/app.test.ts index 66ba2059d55..e7fe31e6c9d 100644 --- a/workspaces/bulk-import/e2e-tests/app.test.ts +++ b/workspaces/bulk-import/e2e-tests/app.test.ts @@ -22,11 +22,13 @@ import { mockBulkImportDryRunResponse, mockBulkImportImportsResponse, mockBulkImportRepositoriesResponse, + mockBulkImportSCMHostsResponse, mockImportByRepoData, mockImportByRepoFrontendData, mockImportsData, mockImportsDryRunData, mockRepositoriesData, + mockSCMHostsData, } from './utils/apiUtils'; import { getPreviewSidebarSnapshots, @@ -51,6 +53,7 @@ test.describe('Bulk Import', () => { context = await browser.newContext(); sharedPage = await context.newPage(); + await mockBulkImportSCMHostsResponse(sharedPage, mockSCMHostsData); await mockBulkImportRepositoriesResponse(sharedPage, mockRepositoriesData); await sharedPage.goto('/'); diff --git a/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts b/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts index 99e37e1569a..3af6bd0acd6 100644 --- a/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts +++ b/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts @@ -19,13 +19,14 @@ import { Page } from '@playwright/test'; * API route patterns for bulk import endpoints */ export const ApiRoutes = { + scmHosts: '**/api/bulk-import/scm-hosts*', repositories: '**/api/bulk-import/repositories*', importsDryRun: '**/api/bulk-import/imports?dryRun=true*', imports: '**/api/bulk-import/imports', byRepoBackend: - '**/api/bulk-import/import/by-repo?repo=https://github.com/test-org/backend-service*', + '**/api/bulk-import/import/by-repo?repo=https%3A%2F%2Fgithub.com%2Ftest-org%2Fbackend-service*', byRepoFrontend: - '**/api/bulk-import/import/by-repo?repo=https://github.com/test-org/frontend-app*', + '**/api/bulk-import/import/by-repo?repo=https%3A%2F%2Fgithub.com%2Ftest-org%2Ffrontend-app*', } as const; type ApiRouteKey = keyof typeof ApiRoutes; @@ -91,6 +92,12 @@ export const mockBulkImportByRepoFrontendResponse = ( status = 200, ) => mockApiResponse(page, ApiRoutes.byRepoFrontend, responseData, status); +export const mockBulkImportSCMHostsResponse = ( + page: Page, + responseData: object, + status = 200, +) => mockApiResponse(page, ApiRoutes.scmHosts, responseData, status); + // Reusable repository definitions const repositories = { backendService: { @@ -135,6 +142,16 @@ const repositories = { }, } as const; +/** + * Mock data for SCM hosts response. + * Returns empty host lists so useRepositories skips the token-fetching flow + * and proceeds directly to fetching repositories without SCM credentials. + */ +export const mockSCMHostsData = { + github: [], + gitlab: [], +}; + /** Mock data for repositories list response */ export const mockRepositoriesData = { errors: [], diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts index 20b0e7bf4f7..1815ddba3e6 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/github/githubApiService.test.ts @@ -422,12 +422,7 @@ describe('GithubApiService tests', () => { data: [], }); - const result = await githubApiService.getRepositoriesFromIntegrations( - undefined, - undefined, - undefined, - undefined, - ); + const result = await githubApiService.getRepositoriesFromIntegrations(); // Server credentials path is used — getAllCredentials IS called expect(mockGetAllCredentials).toHaveBeenCalled(); From 1dcc032fe2a757f30831dedc30cb7a7db588a1a2 Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Tue, 31 Mar 2026 14:57:59 -0400 Subject: [PATCH 10/11] feat(bulk-import): make auth providers a new requirement Signed-off-by: Patrick Knight --- .../.changeset/small-games-live.md | 11 +- workspaces/bulk-import/e2e-tests/app.test.ts | 17 +++ .../bulk-import/e2e-tests/utils/apiUtils.ts | 11 +- .../plugins/bulk-import-backend/README.md | 14 +-- .../api-docs/Apis/OrganizationApi.md | 2 +- .../api-docs/Apis/RepositoryApi.md | 2 +- .../repository/repositories-gitlab.test.ts | 53 +++++++- .../handlers/repository/repositories.test.ts | 118 ++++++++++-------- .../src/service/router.test.ts | 6 +- .../bulk-import-backend/src/service/router.ts | 14 +++ .../bulk-import/plugins/bulk-import/README.md | 12 +- .../src/hooks/useRepositories.test.ts | 62 ++++++++- .../bulk-import/src/hooks/useRepositories.ts | 19 ++- workspaces/bulk-import/yarn.lock | 5 +- 14 files changed, 255 insertions(+), 91 deletions(-) diff --git a/workspaces/bulk-import/.changeset/small-games-live.md b/workspaces/bulk-import/.changeset/small-games-live.md index 93f723c795b..d828296cf18 100644 --- a/workspaces/bulk-import/.changeset/small-games-live.md +++ b/workspaces/bulk-import/.changeset/small-games-live.md @@ -12,7 +12,7 @@ This release introduces the ability for the Bulk Import plugin to fetch reposito **Backend (`bulk-import-backend`)** - Added a new `GET /api/bulk-import/scm-hosts` endpoint that returns the configured GitHub and GitLab integration host URLs as a `SCMHostList` object, enabling the frontend to discover which hosts to request OAuth tokens for. -- The `GET /repositories` and `GET /organizations/{organizationName}/repositories` endpoints now accept an optional `x-scm-tokens` request header — a JSON map of SCM host base URL to user OAuth token. When tokens are present, repository listings reflect what the signed-in user can personally access rather than the full scope of the server-wide integration credentials. +- The `GET /repositories` and `GET /organizations/{organizationName}/repositories` endpoints now **require** the `x-scm-tokens` request header — a JSON map of SCM host base URL to user OAuth token. Requests that omit this header, or supply an empty or oversized header, are rejected with HTTP 401. This ensures repository listings are always scoped to the signed-in user's access and never fall back to server-wide integration credentials. - The `x-scm-tokens` header is stripped from the request immediately upon receipt, before the permission check and before any audit event is created, so OAuth token values are never persisted in audit logs. - When user tokens are provided for GitHub, the Octokit response cache is intentionally disabled to prevent cross-user ETag cache leakage. Server-side credential paths are not affected. - Introduced a shared `GitApiService` interface and common SCM types (`SCMOrganization`, `SCMRepository`, `SCMFetchError`, etc.) to unify the GitHub and GitLab service implementations under a consistent contract. @@ -22,11 +22,10 @@ This release introduces the ability for the Bulk Import plugin to fetch reposito - The plugin now has a **soft dependency** on `@backstage/integration-react`'s `ScmAuthApi`. If the API is registered in the application, the plugin automatically requests OAuth tokens for each configured SCM host and passes them to the backend to enable user-scoped repository listings. - Added `getSCMHosts()` to the `BulkImportAPI` interface with a corresponding `GET /api/bulk-import/scm-hosts` client call, used to discover host URLs before requesting user tokens. - User OAuth tokens are transmitted to the backend via the `X-SCM-Tokens` request header as a JSON-encoded map. +- If the SCM OAuth integration is not configured or token collection fails for all hosts, the repository list query is **blocked** on the frontend and the hook surfaces a descriptive error. This prevents the frontend from firing a request that will always be rejected with 401. -### Fallback Behavior +### Required Configuration -The GitHub and GitLab auth providers are **soft dependencies**. The plugin degrades gracefully when they are not configured or unavailable: +The GitHub and/or GitLab OAuth provider must be configured in the Backstage application for repository listing to work. Deployments that previously relied on server-side credentials alone for the repository list view must add an SCM OAuth provider to continue using this feature. -- If `ScmAuthApi` is not registered in the application, no user tokens are collected and the backend falls back entirely to server-side credentials. -- If a token cannot be obtained for a specific host (e.g., the user has not signed in via that provider, or no OAuth provider is registered for that host), that host is silently skipped and the backend uses its configured integration credentials for that host. -- Deployments that do not configure GitHub or GitLab OAuth providers continue to work exactly as before with no change in behavior. +If `ScmAuthApi` is not registered or tokens cannot be obtained for any configured SCM host, users will see an error prompting them to configure the SCM OAuth integration. diff --git a/workspaces/bulk-import/e2e-tests/app.test.ts b/workspaces/bulk-import/e2e-tests/app.test.ts index e7fe31e6c9d..85e8951de8b 100644 --- a/workspaces/bulk-import/e2e-tests/app.test.ts +++ b/workspaces/bulk-import/e2e-tests/app.test.ts @@ -53,6 +53,23 @@ test.describe('Bulk Import', () => { context = await browser.newContext(); sharedPage = await context.newPage(); + // The backend's GET /repositories and GET /organizations/{org}/repositories + // endpoints require the X-SCM-Tokens header (HTTP 401 otherwise). In a real + // deployment, the frontend obtains these tokens from the configured GitHub / + // GitLab OAuth provider via ScmAuthApi and sends them with every listing + // request. See plugins/bulk-import/README.md → "Required OAuth Configuration". + // + // In these e2e tests we bypass that requirement in two steps: + // 1. Mock GET /api/bulk-import/scm-hosts to return empty host arrays. + // The useRepositories hook hits the `!urls?.length → return undefined` + // early-return path, so tokenFetchError stays undefined and the query + // fires without any X-SCM-Tokens header. + // 2. Mock GET /api/bulk-import/repositories* with a 200 response so + // Playwright intercepts the token-free request before it ever reaches + // the real backend's 401 guard. + // + // This lets us focus on UI behaviour without needing a real OAuth provider + // set up in the test environment. await mockBulkImportSCMHostsResponse(sharedPage, mockSCMHostsData); await mockBulkImportRepositoriesResponse(sharedPage, mockRepositoriesData); await sharedPage.goto('/'); diff --git a/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts b/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts index 3af6bd0acd6..3aba4b81c2c 100644 --- a/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts +++ b/workspaces/bulk-import/e2e-tests/utils/apiUtils.ts @@ -144,8 +144,15 @@ const repositories = { /** * Mock data for SCM hosts response. - * Returns empty host lists so useRepositories skips the token-fetching flow - * and proceeds directly to fetching repositories without SCM credentials. + * Returns empty host lists so the `useRepositories` hook hits the early-return + * path (`!urls?.length → return undefined`) and never attempts token collection. + * This means `tokenFetchError` stays `undefined`, the query is enabled, and the + * frontend fires a request without `X-SCM-Tokens`. + * + * In production the backend would reject such a request with HTTP 401, but the + * Playwright route mock for `ApiRoutes.repositories` intercepts the request + * before it reaches the backend, so the e2e tests still receive the mocked + * repository data regardless of the missing header. */ export const mockSCMHostsData = { github: [], diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/README.md b/workspaces/bulk-import/plugins/bulk-import-backend/README.md index 9d90448e229..41f63162cba 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/README.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/README.md @@ -358,16 +358,16 @@ The plugin supports fetching repository and organization listings **on behalf of 1. The frontend calls `GET /api/bulk-import/scm-hosts` to retrieve the list of configured SCM integration host URLs, grouped by provider (`github` and `gitlab`). 2. For each host, the frontend requests an OAuth token from the Backstage `ScmAuthApi` (provided by `@backstage/integration-react`). -3. The collected tokens are sent to the backend via the optional `x-scm-tokens` request header — a JSON-encoded string whose value, when parsed, maps each integration base URL to the user's OAuth token (e.g. `{"https://github.com":"ghp_xxx"}`). +3. The collected tokens are sent to the backend via the **required** `x-scm-tokens` request header — a JSON-encoded string whose value, when parsed, maps each integration base URL to the user's OAuth token (e.g. `{"https://github.com":"ghp_xxx"}`). 4. The backend uses these user tokens to call the GitHub or GitLab APIs on behalf of the user, so the repository listings reflect what the signed-in user can personally access. -#### Fallback Behavior +#### Required OAuth Configuration -The GitHub and GitLab auth providers are **soft dependencies**. When user tokens are absent or unavailable, the backend falls back gracefully to server-side credentials: +The `x-scm-tokens` header is **required** for `GET /repositories` and `GET /organizations/{organizationName}/repositories`. Requests that omit the header, supply an empty token map, or send a header that exceeds the allowed size are rejected with **HTTP 401**. -- If the `x-scm-tokens` header is not present or is empty, the backend uses its configured integration credentials (GitHub App, PAT, or GitLab token) for all repository listing calls. Existing behavior is fully preserved. -- If a token is provided for some hosts but not others, the backend uses the user token where available and the server-side credential for any host that was omitted. -- Deployments that do not configure GitHub or GitLab OAuth providers continue to work exactly as before with no configuration changes required. +A GitHub and/or GitLab OAuth provider must therefore be configured in the Backstage application for these endpoints to work. Refer to the [Backstage GitHub auth docs](https://backstage.io/docs/auth/github/provider) and [GitLab auth docs](https://backstage.io/docs/auth/gitlab/provider) for setup instructions. + +> **Migration note:** Deployments that previously relied solely on server-side credentials (GitHub App, PAT, or GitLab token) for the repository list view must now also configure an SCM OAuth provider. The server-side credentials are still used for all other operations (import creation, status checks, etc.) and are unaffected by this change. #### Security Note @@ -381,7 +381,7 @@ The `x-scm-tokens` header is stripped from the request immediately upon receipt | ------ | ---------------------------- | ------------------------------------------------------------------------------------------- | | `GET` | `/api/bulk-import/scm-hosts` | Returns configured GitHub and GitLab integration host base URLs as an `SCMHostList` object. | -The existing `GET /repositories` and `GET /organizations/{organizationName}/repositories` endpoints now accept an optional `x-scm-tokens` header. See the [API documentation](api-docs/README.md) for the full request/response specification. +The existing `GET /repositories` and `GET /organizations/{organizationName}/repositories` endpoints now **require** the `x-scm-tokens` header. See the [API documentation](api-docs/README.md) for the full request/response specification. ## REST API diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md index 88160336be0..80828016861 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md @@ -52,7 +52,7 @@ Fetch Repositories in the specified GitHub organization, provided it is accessib | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | | **search** | **String**| returns only the items that match the search string | [optional] [default to null] | | **approvalTool** | **String**| the approvalTool to use | [optional] [default to GIT] | -| **x-scm-tokens** | **String**| Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. The value must be a JSON string whose structure matches SCMTokenMap (keys are SCM base URLs, values are OAuth bearer tokens). | [optional] [default to null] | +| **x-scm-tokens** | **String**| **Required.** JSON-encoded map of SCM host URL to user OAuth token. Used to fetch repositories on behalf of the signed-in user. The value must be a JSON object whose keys are SCM integration base URLs and whose values are OAuth bearer tokens (e.g. `{"https://github.com":"ghp_xxx"}`). Requests that omit this header, supply an empty object, or exceed 4 KB are rejected with HTTP 401. | [required] [default to null] | ### Return type diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md index 5aceccf9073..668ac3aa25c 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md +++ b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md @@ -22,7 +22,7 @@ Fetch Organization Repositories accessible by Backstage Github Integrations | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | | **search** | **String**| returns only the items that match the search string | [optional] [default to null] | | **approvalTool** | **String**| the approvalTool to use | [optional] [default to GIT] | -| **x-scm-tokens** | **String**| Optional JSON-encoded map of SCM host URL to user authentication token. Used to fetch repositories on behalf of the user for each configured SCM host. The value must be a JSON string whose structure matches SCMTokenMap (keys are SCM base URLs, values are OAuth bearer tokens). | [optional] [default to null] | +| **x-scm-tokens** | **String**| **Required.** JSON-encoded map of SCM host URL to user OAuth token. Used to fetch repositories on behalf of the signed-in user. The value must be a JSON object whose keys are SCM integration base URLs and whose values are OAuth bearer tokens (e.g. `{"https://github.com":"ghp_xxx"}`). Requests that omit this header, supply an empty object, or exceed 4 KB are rejected with HTTP 401. | [required] [default to null] | ### Return type diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts index cc0e3faddb8..ca6d8f4656b 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts @@ -25,11 +25,16 @@ import { startBackendServer, } from '../../../../__fixtures__/testUtils'; +// Token header used across GitLab repo-listing tests. +const GL_USER_TOKENS = JSON.stringify({ + 'https://gitlab.com': 'test-gl-token', +}); + describe('repositories', () => { const useTestData = setupTest(); describe('GET /repositories', () => { - it('returns 200 when repositories are fetched without errors', async () => { + it('returns 401 when X-SCM-Tokens header is absent', async () => { const { mockCatalogClient } = useTestData(); const backendServer = await startBackendServer( mockCatalogClient, @@ -40,6 +45,22 @@ describe('repositories', () => { .get('/api/bulk-import/repositories') .query({ approvalTool: 'GITLAB' }); + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); + }); + + it('returns 200 when repositories are fetched without errors', async () => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); + expect(response.status).toEqual(200); expect(response.body).toEqual({ approvalTool: 'GITLAB', @@ -96,7 +117,8 @@ describe('repositories', () => { const response = await request(backendServer) .get('/api/bulk-import/repositories') - .query({ approvalTool: 'GITLAB' }); + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); expect(response.status).toEqual(500); expect(response.body).toEqual({ @@ -107,6 +129,21 @@ describe('repositories', () => { }); describe('GET /organizations/{org}/repositories', () => { + it('returns 401 when X-SCM-Tokens header is absent', async () => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/organizations/my-ent-org-1/repositories') + .query({ approvalTool: 'GITLAB' }); + + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); + }); + it('returns 200 when repositories are fetched without errors', async () => { const { mockCatalogClient } = useTestData(); const backendServer = await startBackendServer( @@ -116,7 +153,8 @@ describe('repositories', () => { let response = await request(backendServer) .get('/api/bulk-import/organizations/my-ent-org-1/repositories') - .query({ approvalTool: 'GITLAB' }); + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); expect(response.status).toEqual(200); expect(response.body).toEqual({ @@ -138,7 +176,8 @@ describe('repositories', () => { response = await request(backendServer) .get('/api/bulk-import/organizations/my-ent-org-2/repositories') - .query({ approvalTool: 'GITLAB' }); + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); expect(response.status).toEqual(200); expect(response.body).toEqual({ @@ -169,7 +208,8 @@ describe('repositories', () => { response = await request(backendServer) .get('/api/bulk-import/organizations/my-ent-org--no-repos/repositories') - .query({ approvalTool: 'GITLAB' }); + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); expect(response.status).toEqual(200); expect(response.body).toEqual({ @@ -201,7 +241,8 @@ describe('repositories', () => { const orgReposResp = await request(backendServer) .get('/api/bulk-import/organizations/some-org/repositories') - .query({ approvalTool: 'GITLAB' }); + .query({ approvalTool: 'GITLAB' }) + .set('X-SCM-Tokens', GL_USER_TOKENS); expect(orgReposResp.status).toEqual(500); expect(orgReposResp.body).toEqual({ diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts index e65f33254fd..9690db163ed 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts @@ -26,11 +26,19 @@ import { startBackendServer, } from '../../../../__fixtures__/testUtils'; +// Token header used across repo-listing tests. +// With user tokens provided for github.com only, the GitHub App path +// (enterprise.github.com) is skipped — results are scoped to the user's +// own OAuth credential. +const GH_USER_TOKENS = JSON.stringify({ + 'https://github.com': 'test-user-token', +}); + describe('repositories', () => { const useTestData = setupTest(); describe('GET /repositories', () => { - it('returns 200 when repositories are fetched without errors', async () => { + it('returns 401 when X-SCM-Tokens header is absent', async () => { const { mockCatalogClient } = useTestData(); const backendServer = await startBackendServer( mockCatalogClient, @@ -41,7 +49,25 @@ describe('repositories', () => { '/api/bulk-import/repositories', ); + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); + }); + + it('returns 200 when repositories are fetched without errors', async () => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); + expect(response.status).toEqual(200); + // With user-scoped tokens, only repos accessible via the user's OAuth + // credential (GET /user/repos) are returned. The GitHub App path for + // enterprise.github.com is intentionally skipped. expect(response.body).toEqual({ errors: [], repositories: [ @@ -54,15 +80,6 @@ describe('repositories', () => { organization: 'octocat', url: 'http://localhost:8765/octocat/animated-happiness', }, - { - defaultBranch: 'master', - errors: [], - id: 'octocat/Hello-World', - lastUpdate: '2011-01-26T19:14:43Z', - name: 'Hello-World', - organization: 'octocat', - url: 'http://localhost:8765/octocat/Hello-World', - }, { defaultBranch: 'master', errors: [], @@ -73,18 +90,17 @@ describe('repositories', () => { url: 'http://localhost:8765/my-user/Lorem-Ipsum', }, ], - totalCount: 3, + totalCount: 2, }); }); - it('returns 200 with the errors in the body when repositories are fetched, but errors have occurred', async () => { + it('returns 500 when the user token call fails for all integrations', async () => { const { server, mockCatalogClient } = useTestData(); const backendServer = await startBackendServer( mockCatalogClient, AuthorizeResult.ALLOW, ); - // change the response to 'GET /user/repos' - // to simulate an error retrieving list of orgs from GH Token. + // Override GET /user/repos to simulate a failed user-token credential. server.use( rest.get(`${LOCAL_ADDR}/user/repos`, (_, res, ctx) => res( @@ -94,25 +110,13 @@ describe('repositories', () => { ), ); - const response = await request(backendServer).get( - '/api/bulk-import/repositories', - ); + const response = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); - expect(response.status).toEqual(200); + expect(response.status).toEqual(500); expect(response.body).toEqual({ errors: ['Github Token auth did not succeed'], - repositories: [ - { - defaultBranch: 'master', - errors: [], - id: 'octocat/Hello-World', - lastUpdate: '2011-01-26T19:14:43Z', - name: 'Hello-World', - organization: 'octocat', - url: 'http://localhost:8765/octocat/Hello-World', - }, - ], - totalCount: 1, }); }); @@ -122,35 +126,49 @@ describe('repositories', () => { mockCatalogClient, AuthorizeResult.ALLOW, ); - // change the responses to simulate error retrieving list of repos from all GH integrations. + // Override all GH integration endpoints to simulate total failure. addHandlersForGHTokenAppErrors(server); - const reposResp = await request(backendServer).get( - '/api/bulk-import/repositories', - ); + const reposResp = await request(backendServer) + .get('/api/bulk-import/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); expect(reposResp.status).toEqual(500); + // With user tokens only the user-token path runs; the GitHub App path is + // not taken, so only the user-token error is surfaced. expect(reposResp.body).toEqual({ - errors: [ - 'Github App auth returned an error', - 'Github Token auth did not succeed', - ], + errors: ['Github Token auth did not succeed'], }); }); }); describe('GET /organizations/{org}/repositories', () => { - it('returns 200 when repositories are fetched without errors', async () => { + it('returns 401 when X-SCM-Tokens header is absent', async () => { const { mockCatalogClient } = useTestData(); const backendServer = await startBackendServer( mockCatalogClient, AuthorizeResult.ALLOW, ); - let response = await request(backendServer).get( + const response = await request(backendServer).get( '/api/bulk-import/organizations/my-ent-org-1/repositories', ); + expect(response.status).toEqual(401); + expect(response.body).toHaveProperty('error'); + }); + + it('returns 200 when repositories are fetched without errors', async () => { + const { mockCatalogClient } = useTestData(); + const backendServer = await startBackendServer( + mockCatalogClient, + AuthorizeResult.ALLOW, + ); + + let response = await request(backendServer) + .get('/api/bulk-import/organizations/my-ent-org-1/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); + expect(response.status).toEqual(200); expect(response.body).toEqual({ errors: [], @@ -168,9 +186,9 @@ describe('repositories', () => { totalCount: 1, }); - response = await request(backendServer).get( - '/api/bulk-import/organizations/my-ent-org-2/repositories', - ); + response = await request(backendServer) + .get('/api/bulk-import/organizations/my-ent-org-2/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); expect(response.status).toEqual(200); expect(response.body).toEqual({ @@ -198,9 +216,9 @@ describe('repositories', () => { totalCount: 2, }); - response = await request(backendServer).get( - '/api/bulk-import/organizations/my-ent-org--no-repos/repositories', - ); + response = await request(backendServer) + .get('/api/bulk-import/organizations/my-ent-org--no-repos/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); expect(response.status).toEqual(200); expect(response.body).toEqual({ @@ -216,12 +234,12 @@ describe('repositories', () => { mockCatalogClient, AuthorizeResult.ALLOW, ); - // change the response to simulate an error retrieving list from GH Token. + // Override GH integration endpoints to simulate failure. addHandlersForGHTokenAppErrors(server); - const orgReposResp = await request(backendServer).get( - '/api/bulk-import/organizations/some-org/repositories', - ); + const orgReposResp = await request(backendServer) + .get('/api/bulk-import/organizations/some-org/repositories') + .set('X-SCM-Tokens', GH_USER_TOKENS); expect(orgReposResp.status).toEqual(500); expect(orgReposResp.body).toEqual({ diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts index e664496df10..46d564aba10 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.test.ts @@ -88,7 +88,7 @@ describe('router tests', () => { .set('x-scm-tokens', header), ], ])( - '%s: ignores x-scm-tokens and returns 200 when header exceeds size limit', + '%s: ignores x-scm-tokens and returns 401 when header exceeds size limit', async ( _endpoint: string, reqHandler: ( @@ -110,7 +110,9 @@ describe('router tests', () => { request(backendServer), oversizedHeader, ); - expect(response.status).toEqual(200); + // Oversized header is silently discarded; the missing valid token then + // triggers the 401 guard added for compliance. + expect(response.status).toEqual(401); }, ); diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts index 4398ee5d0ce..4dcfb4b0249 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/router.ts @@ -300,6 +300,13 @@ export async function createRouter( | Record | undefined; + if (!userTokens || Object.keys(userTokens).length === 0) { + return res.status(401).json({ + error: + 'User SCM credentials are required to list repositories. Ensure the SCM OAuth integration is configured.', + }); + } + const response = await findAllRepositories( { logger, @@ -343,6 +350,13 @@ export async function createRouter( | Record | undefined; + if (!userTokens || Object.keys(userTokens).length === 0) { + return res.status(401).json({ + error: + 'User SCM credentials are required to list repositories. Ensure the SCM OAuth integration is configured.', + }); + } + const response = await findRepositoriesByOrganization( { logger, diff --git a/workspaces/bulk-import/plugins/bulk-import/README.md b/workspaces/bulk-import/plugins/bulk-import/README.md index 6c138be85ff..5d1b0978db6 100644 --- a/workspaces/bulk-import/plugins/bulk-import/README.md +++ b/workspaces/bulk-import/plugins/bulk-import/README.md @@ -19,7 +19,7 @@ The sections below are relevant for static plugins. If the plugin is expected to - Follow the [GitHub Locations](https://backstage.io/docs/integrations/github/locations) to integrate GitHub integrations in your Backstage instance. For now, the plugin only supports loading catalog entities from github.com or GitHub Enterprise. -- _(Optional)_ To enable [on behalf of user access](#on-behalf-of-user-access), configure a GitHub and/or GitLab OAuth auth provider and ensure `ScmAuthApi` from `@backstage/integration-react` is registered in your application. Without this, the plugin falls back to server-side integration credentials. +- Configure a GitHub and/or GitLab OAuth auth provider and register `ScmAuthApi` from `@backstage/integration-react` in your application. **This is required for repository listing.** The `GET /repositories` and `GET /organizations/{org}/repositories` backend endpoints require user OAuth credentials (sent via the `X-SCM-Tokens` header) and will return HTTP 401 if they are absent. See [Configuring Auth Providers](#configuring-auth-providers) for setup instructions. --- @@ -88,13 +88,13 @@ When `ScmAuthApi` (from `@backstage/integration-react`) is available in the appl The backend then uses these user tokens to call the SCM APIs on behalf of the user, returning only what that user can access. -### Fallback Behavior +### Required OAuth Configuration -The GitHub and GitLab auth providers are **soft dependencies** — if they are not configured the plugin degrades gracefully: +GitHub and/or GitLab OAuth providers are **required** for repository and organization listing. The backend enforces this: `GET /repositories` and `GET /organizations/{org}/repositories` return **HTTP 401** if the `X-SCM-Tokens` header is absent or empty. -- If `ScmAuthApi` is not registered in the application, no tokens are collected and the backend falls back to server-side credentials (GitHub App, PAT, or GitLab token). -- If a token cannot be obtained for a specific host (e.g., the user has not signed in via that provider, or no OAuth provider is registered for that host), that host is silently skipped. The backend uses its configured integration credentials for any host without a user token. -- Deployments that do not configure GitHub or GitLab OAuth providers continue to work exactly as before with no configuration changes required. +If `ScmAuthApi` is not registered in the application, or if token collection fails for every configured SCM host, the frontend blocks the listing request and surfaces a descriptive error prompting the user to configure the OAuth integration. + +> **Migration note:** Deployments that previously relied on server-side integration credentials alone for the repository list view (GitHub App, PAT, or GitLab token) must now also configure an SCM OAuth provider. See [Configuring Auth Providers](#configuring-auth-providers) below. ### Configuring Auth Providers diff --git a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts index bb5b771e4bf..32d23804e70 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.test.ts @@ -190,7 +190,7 @@ describe('useRepositories', () => { }); }); - it('skips a host gracefully when scmAuth.getCredentials throws for it', async () => { + it('surfaces a tokenFetchError when scmAuth is registered but all token fetches fail', async () => { const mockGetCredentials = jest .fn() .mockRejectedValue(new Error('No OAuth provider for this host')); @@ -216,7 +216,7 @@ describe('useRepositories', () => { }); (useQuery as jest.Mock).mockReturnValue({ - data: mockGetRepositories, + data: undefined, isLoading: false, error: null, refetch: jest.fn(), @@ -230,10 +230,66 @@ describe('useRepositories', () => { }), ); - // Should complete without throwing even though getCredentials rejected await waitFor(() => { expect(result.current.loading).toBeFalsy(); }); + + // When all token fetches fail, the hook surfaces an error so the UI can + // prompt the user to configure the SCM OAuth integration. + expect(result.current.error?.errors).toEqual([ + 'No user SCM credentials could be obtained. Please ensure your SCM OAuth integration is configured.', + ]); + }); + + it('disables the query when tokenFetchError is set (no request fired without tokens)', async () => { + const mockGetCredentials = jest + .fn() + .mockRejectedValue(new Error('No OAuth provider for this host')); + const mockScmAuth = { getCredentials: mockGetCredentials }; + + const mockGetSCMHosts = jest.fn().mockResolvedValue({ + github: ['https://github.com'], + gitlab: [], + }); + const mockBulkImportApi = { + getSCMHosts: mockGetSCMHosts, + dataFetcher: jest.fn(), + }; + + mockUseApiHolder.mockReturnValue({ + get: jest.fn().mockReturnValue(mockScmAuth), + }); + + const mockUseApi = jest.requireMock('@backstage/core-plugin-api').useApi; + mockUseApi.mockImplementation((ref: { id: string }) => { + if (ref.id === 'plugin.bulk-import.service') return mockBulkImportApi; + return undefined; + }); + + (useQuery as jest.Mock).mockClear(); + (useQuery as jest.Mock).mockReturnValue({ + data: undefined, + isLoading: false, + error: null, + refetch: jest.fn(), + }); + + const { result } = renderHook(() => + useRepositories({ + page: 1, + querySize: 10, + approvalTool: ApprovalTool.Git, + }), + ); + + await waitFor(() => { + expect(result.current.loading).toBeFalsy(); + }); + + // useQuery must have been called with enabled: false so no HTTP request + // is fired when user tokens are unavailable. + const lastOptions = (useQuery as jest.Mock).mock.calls.at(-1)?.[2]; + expect(lastOptions?.enabled).toBe(false); }); it('does not include raw token values in the React Query key', async () => { diff --git a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts index 0ba8bbcb4aa..0c97ea07f5e 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/hooks/useRepositories.ts @@ -70,7 +70,11 @@ export const useRepositories = ( return url; }); - const { value: scmAuthTokens, loading: tokenLoading } = useAsync(async () => { + const { + value: scmAuthTokens, + loading: tokenLoading, + error: tokenFetchError, + } = useAsync(async () => { if (!scmAuth) return undefined; const hosts = await bulkImportApi.getSCMHosts(); if (!hosts || hosts instanceof Response || !('github' in hosts)) @@ -91,11 +95,15 @@ export const useRepositories = ( }); if (token) tokenRecord[url] = token; } catch { - // No OAuth provider registered for this host — skip it and fall back - // to server-side credentials for that integration on the backend. + // No OAuth provider registered for this host — skip it. } } - return Object.keys(tokenRecord).length > 0 ? tokenRecord : undefined; + if (Object.keys(tokenRecord).length === 0) { + throw new Error( + 'No user SCM credentials could be obtained. Please ensure your SCM OAuth integration is configured.', + ); + } + return tokenRecord; }, [scmAuth, bulkImportApi, options.approvalTool]); const fetchRepositories = async (queryOptions: DataFetcherQueryParams) => { @@ -133,7 +141,7 @@ export const useRepositories = ( ], () => fetchRepositories(options), { - enabled: !tokenLoading, + enabled: !tokenLoading && !tokenFetchError, refetchInterval: pollInterval || 60000, refetchOnWindowFocus: false, }, @@ -154,6 +162,7 @@ export const useRepositories = ( loading: tokenLoading || isQueryLoading, data: prepareData, error: { + ...(tokenFetchError ? { errors: [tokenFetchError.message] } : {}), ...(error ?? {}), ...((value?.errors && value.errors.length > 0) || (value as any as Response)?.statusText diff --git a/workspaces/bulk-import/yarn.lock b/workspaces/bulk-import/yarn.lock index 745e9033c0d..8a2e3805cbb 100644 --- a/workspaces/bulk-import/yarn.lock +++ b/workspaces/bulk-import/yarn.lock @@ -2741,7 +2741,7 @@ __metadata: languageName: node linkType: hard -"@backstage/integration-react@npm:^1.2.14, @backstage/integration-react@npm:^1.2.15, @backstage/integration-react@npm:^1.2.16": +"@backstage/integration-react@npm:^1.2.14, @backstage/integration-react@npm:^1.2.16": version: 1.2.16 resolution: "@backstage/integration-react@npm:1.2.16" dependencies: @@ -2749,6 +2749,7 @@ __metadata: "@backstage/core-plugin-api": "npm:^1.12.4" "@backstage/integration": "npm:^2.0.0" "@material-ui/core": "npm:^4.12.2" + "@material-ui/icons": "npm:^4.9.1" peerDependencies: "@types/react": ^17.0.0 || ^18.0.0 react: ^17.0.0 || ^18.0.0 @@ -2761,7 +2762,7 @@ __metadata: languageName: node linkType: hard -"@backstage/integration@npm:^1.19.2, @backstage/integration@npm:^1.20.0, @backstage/integration@npm:^1.20.1": +"@backstage/integration@npm:^1.19.2, @backstage/integration@npm:^1.20.0": version: 1.20.1 resolution: "@backstage/integration@npm:1.20.1" dependencies: From 8b0b0484883cebed62b2e31e486336e098884c5f Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Tue, 7 Apr 2026 13:28:19 -0400 Subject: [PATCH 11/11] feat(bulk-import): consolidate getDeleteImportActionPath and getImportActionPath Signed-off-by: Patrick Knight --- .../src/api/BulkImportBackendClient.ts | 4 ++-- .../src/api/BulkImportBackendClientBase.ts | 17 ++--------------- ...tratorBulkImportBackendClientPathProvider.ts | 2 +- .../PRBulkImportBackendClientPathProvider.ts | 2 +- ...folderBulkImportBackendClientPathProvider.ts | 2 +- 5 files changed, 7 insertions(+), 20 deletions(-) diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts index cc951461876..d516c9c7da8 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClient.ts @@ -220,7 +220,7 @@ export class BulkImportBackendClient implements BulkImportAPI { const { token: idToken } = await this.identityApi.getCredentials(); const backendUrl = this.configApi.getString('backend.baseUrl'); const jsonResponse = await fetch( - `${backendUrl}${this.pathProvider.getDeleteImportActionPath(repo, defaultBranch, approvalTool)}`, + `${backendUrl}${this.pathProvider.getImportActionPath(repo, defaultBranch, approvalTool)}`, { method: 'DELETE', headers: { @@ -244,7 +244,7 @@ export class BulkImportBackendClient implements BulkImportAPI { const { token: idToken } = await this.identityApi.getCredentials(); const backendUrl = this.configApi.getString('backend.baseUrl'); const jsonResponse = await fetch( - `${backendUrl}${this.pathProvider.getGetImportActionPath(repo, defaultBranch, approvalTool)}`, + `${backendUrl}${this.pathProvider.getImportActionPath(repo, defaultBranch, approvalTool)}`, { method: 'GET', headers: { diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts index 208755cf99f..a2a40afaeb1 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/BulkImportBackendClientBase.ts @@ -18,12 +18,7 @@ import { AddedRepositoryColumnNameEnum, SortingOrderEnum } from '../types'; export interface IBulkImportRESTPathProvider { getCreateImportJobsPath(dryRun?: boolean): string | undefined; - getDeleteImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string; - getGetImportActionPath( + getImportActionPath( repo: string, defaultBranch: string, approvalTool?: string, @@ -40,20 +35,12 @@ export interface IBulkImportRESTPathProvider { export abstract class BulkImportRESTPathProviderBase implements IBulkImportRESTPathProvider { abstract getCreateImportJobsPath(dryRun?: boolean): string | undefined; - abstract getDeleteImportActionPath( + abstract getImportActionPath( repo: string, defaultBranch: string, approvalTool?: string, ): string; - getGetImportActionPath( - repo: string, - defaultBranch: string, - approvalTool?: string, - ): string { - return this.getDeleteImportActionPath(repo, defaultBranch, approvalTool); - } - protected abstract getImportJobsBasePath(): string; getGetImportJobsPath( diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts index 7834c64ec0d..5f926ac1802 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/OrchestratorBulkImportBackendClientPathProvider.ts @@ -23,7 +23,7 @@ export class OrchestratorBulkImportBackendClientPathProvider extends BulkImportR : `/api/bulk-import/orchestrator-workflows`; } - getDeleteImportActionPath( + getImportActionPath( repo: string, _defaultBranch: string, approvalTool?: string, diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts index 79b480f1b6c..f3c7ad753ad 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/PRBulkImportBackendClientPathProvider.ts @@ -23,7 +23,7 @@ export class PRBulkImportBackendClientPathProvider extends BulkImportRESTPathPro : `/api/bulk-import/imports`; } - getDeleteImportActionPath( + getImportActionPath( repo: string, defaultBranch: string, approvalTool?: string, diff --git a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts index 48d0fa8f1d0..a7d59a896fd 100644 --- a/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts +++ b/workspaces/bulk-import/plugins/bulk-import/src/api/ScaffolderBulkImportBackendClientPathProvider.ts @@ -21,7 +21,7 @@ export class ScaffolderBulkImportBackendClientPathProvider extends BulkImportRES return dryRun === true ? undefined : `/api/bulk-import/task-imports`; } - getDeleteImportActionPath( + getImportActionPath( repo: string, _defaultBranch: string, approvalTool?: string,