From 11803cf9a40c3376fa5b2c2db31a1067b9ae6292 Mon Sep 17 00:00:00 2001 From: almog8k Date: Thu, 14 May 2026 10:01:20 +0300 Subject: [PATCH 1/7] fix: update default values in values.yaml to be empty --- helm/values.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index 6652853..23a1633 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -17,14 +17,14 @@ storage: forcePathStyle: true secretName: "" sslEnabled: false - region: "us-east-1" + region: "" tilesBucket: "" fs: internalPvc: enabled: false name: "" mountPath: "" - tilesSubPath: "tiles" + tilesSubPath: "" mclabels: @@ -46,8 +46,8 @@ fullnameOverride: "" configManagement: offlineMode: false name: 'cleaner' - version: 'latest' - serverUrl: 'http://localhost:8080/api' + version: '' + serverUrl: '' jobDefinitions: {} @@ -106,7 +106,7 @@ env: resourceAttributes: {} tracing: enabled: false - url: http://localhost:55681/v1/trace + url: "" queue: heartbeatIntervalMs: 1000 dequeueIntervalMs: 3000 From 31514d7f7e02c4829ce18a3773d76ed1be7d8016 Mon Sep 17 00:00:00 2001 From: almog8k Date: Thu, 14 May 2026 10:05:10 +0300 Subject: [PATCH 2/7] feat: enhance error handling and reporting in storage providers with detailed failure reasons --- src/cleaner/errors/errors.ts | 18 +++++ src/cleaner/errors/index.ts | 2 +- .../storageProviders/deleteFailureSummary.ts | 33 ++++++++ .../storageProviders/fsStorageProvider.ts | 26 +++---- .../storageProviders/iStorageProvider.ts | 14 +++- src/cleaner/storageProviders/index.ts | 3 +- .../storageProviders/s3StorageProvider.ts | 24 +++--- .../strategies/tilesDeletionStrategy.ts | 57 ++++++++------ .../deleteFailureSummary.spec.ts | 76 +++++++++++++++++++ .../fsStorageProvider.spec.ts | 27 +++++-- .../s3StorageProvider.spec.ts | 39 +++++++--- tests/tilesDeletionStrategy.spec.ts | 33 +++++++- 12 files changed, 279 insertions(+), 73 deletions(-) create mode 100644 src/cleaner/storageProviders/deleteFailureSummary.ts create mode 100644 tests/storageProviders/deleteFailureSummary.spec.ts diff --git a/src/cleaner/errors/errors.ts b/src/cleaner/errors/errors.ts index 5758c62..bbcdd2e 100644 --- a/src/cleaner/errors/errors.ts +++ b/src/cleaner/errors/errors.ts @@ -13,6 +13,24 @@ export function toError(value: unknown): Error { } } +/** + * Produces a short, human-readable label for a thrown value. Prefers Node `errno` + * codes (e.g. 'ENOENT') when present, then falls back to the error message, then + * to a stringified form. Intended for compact failure summaries surfaced to callers + * (task rejection reasons, grouped failure counts) — not for full stack traces. + */ +export function describeError(value: unknown): string { + if (value instanceof Error) { + const code = (value as NodeJS.ErrnoException).code; + return code ?? value.message; + } + try { + return String(value); + } catch { + return 'non-serializable thrown value'; + } +} + /** * Base class for recoverable errors that can be retried. * Task will be retried if attempts < maxAttempts. diff --git a/src/cleaner/errors/index.ts b/src/cleaner/errors/index.ts index ebada70..0315ef3 100644 --- a/src/cleaner/errors/index.ts +++ b/src/cleaner/errors/index.ts @@ -1,2 +1,2 @@ -export { toError, RecoverableError, UnrecoverableError, ConfigurationError, ValidationError, StrategyNotFoundError } from './errors'; +export { toError, describeError, RecoverableError, UnrecoverableError, ConfigurationError, ValidationError, StrategyNotFoundError } from './errors'; export { ErrorHandler } from './errorHandler'; diff --git a/src/cleaner/storageProviders/deleteFailureSummary.ts b/src/cleaner/storageProviders/deleteFailureSummary.ts new file mode 100644 index 0000000..2cd7aad --- /dev/null +++ b/src/cleaner/storageProviders/deleteFailureSummary.ts @@ -0,0 +1,33 @@ +import type { DeleteFailure } from './iStorageProvider'; + +/** + * Aggregated view of a batch of delete failures — designed to be embedded in a + * task rejection reason or log line without further post-processing. + */ +export interface DeleteFailureSummary { + /** Raw count per reason string, e.g. `{ ENOENT: 150, EACCES: 3 }`. */ + counts: Record; + /** Reasons formatted descending by count, e.g. `'ENOENT=150, EACCES=3'`. */ + summary: string; + /** Up to `sampleSize` `path (reason)` strings, preserving input order. */ + sample: string[]; +} + +/** + * Reduces a list of provider delete failures into a compact, log-friendly + * shape. Lives alongside `IStorageProvider` because it operates purely on + * `DeleteFailure[]` — any caller of `provider.delete()` can use it, regardless + * of which storage backend produced the failures. + */ +export function summarizeDeleteFailures(failures: DeleteFailure[], sampleSize: number): DeleteFailureSummary { + const counts: Record = {}; + for (const { reason } of failures) { + counts[reason] = (counts[reason] ?? 0) + 1; + } + const summary = Object.entries(counts) + .sort(([, a], [, b]) => b - a) + .map(([reason, count]) => `${reason}=${count}`) + .join(', '); + const sample = failures.slice(0, sampleSize).map((f) => `${f.path} (${f.reason})`); + return { counts, summary, sample }; +} diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index fdb4e2b..87734d5 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -1,7 +1,8 @@ import { join } from 'node:path'; import { stat, unlink, rmdir } from 'node:fs/promises'; import type { Logger } from '@map-colonies/js-logger'; -import type { IStorageProvider } from './iStorageProvider'; +import { describeError } from '../errors'; +import type { DeleteFailure, IStorageProvider } from './iStorageProvider'; export class FsStorageProvider implements IStorageProvider { public constructor(private readonly logger: Logger) {} @@ -16,7 +17,7 @@ export class FsStorageProvider implements IStorageProvider { } } - public async delete(paths: string[], storageTarget: string): Promise { + public async delete(paths: string[], storageTarget: string): Promise { if (paths.length === 0) { return []; } @@ -25,31 +26,24 @@ export class FsStorageProvider implements IStorageProvider { const results = await Promise.allSettled( paths.map(async (relativePath) => { - try { - await unlink(join(storageTarget, relativePath)); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return relativePath; // treat missing file as a failed deletion, to be included in the failure report - } - throw error; - } + await unlink(join(storageTarget, relativePath)); }) ); - const failedPaths: string[] = []; + const failures: DeleteFailure[] = []; for (const [idx, result] of results.entries()) { if (result.status === 'rejected') { const relativePath = paths[idx]!; - this.logger.warn({ msg: 'Failed to delete file', path: join(storageTarget, relativePath), error: result.reason }); - failedPaths.push(relativePath); - } else if (result.value !== undefined) { - failedPaths.push(result.value); + const error: unknown = result.reason; + const reason = describeError(error); + this.logger.debug({ msg: 'Failed to delete file', path: join(storageTarget, relativePath), reason, error }); + failures.push({ path: relativePath, reason }); } } await this.cleanupEmptyDirs(paths, storageTarget); - return failedPaths; + return failures; } // Attempts to remove any directories that became empty after file deletion. diff --git a/src/cleaner/storageProviders/iStorageProvider.ts b/src/cleaner/storageProviders/iStorageProvider.ts index c9cbead..e392459 100644 --- a/src/cleaner/storageProviders/iStorageProvider.ts +++ b/src/cleaner/storageProviders/iStorageProvider.ts @@ -1,12 +1,22 @@ +/** + * A single failed deletion paired with a short reason string (e.g. 'ENOENT', + * 'AccessDenied', 'NoSuchKey'). + */ +export interface DeleteFailure { + path: string; + reason: string; +} + export interface IStorageProvider { /** * Deletes a batch of relative file paths within the given storage target. * - S3: storageTarget = bucket name; paths are object keys * - FS: storageTarget = base directory; full path = join(storageTarget, path) * - * Returns the paths that failed to delete. Treats "not found" as success (idempotent). + * Returns one entry per failed deletion. "Not found" is reported as a failure + * (with reason 'ENOENT' / 'NoSuchKey'). */ - delete: (paths: string[], storageTarget: string) => Promise; + delete: (paths: string[], storageTarget: string) => Promise; /** * Returns true if relativePath exists within storageTarget and contains data. diff --git a/src/cleaner/storageProviders/index.ts b/src/cleaner/storageProviders/index.ts index 9f82eee..54b39c1 100644 --- a/src/cleaner/storageProviders/index.ts +++ b/src/cleaner/storageProviders/index.ts @@ -1,3 +1,4 @@ -export type { IStorageProvider } from './iStorageProvider'; +export type { IStorageProvider, DeleteFailure } from './iStorageProvider'; export { S3StorageProvider } from './s3StorageProvider'; export { FsStorageProvider } from './fsStorageProvider'; +export { summarizeDeleteFailures, type DeleteFailureSummary } from './deleteFailureSummary'; diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index 6c0da2e..d554a20 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -2,7 +2,8 @@ import { S3Client, DeleteObjectsCommand, ListObjectsV2Command, NoSuchBucket } from '@aws-sdk/client-s3'; import type { Logger } from '@map-colonies/js-logger'; import type { ConfigType } from '@common/config'; -import type { IStorageProvider } from './iStorageProvider'; +import { describeError } from '../errors'; +import type { DeleteFailure, IStorageProvider } from './iStorageProvider'; const S3_MAX_DELETE_BATCH = 1000; @@ -35,26 +36,29 @@ export class S3StorageProvider implements IStorageProvider { }); } - public async delete(paths: string[], storageTarget: string): Promise { + public async delete(paths: string[], storageTarget: string): Promise { if (paths.length === 0) { return []; } this.logger.debug({ msg: 'Deleting objects from S3', bucket: storageTarget, count: paths.length }); - const failedPaths: string[] = []; + const failures: DeleteFailure[] = []; for (const chunk of this.chunk(paths, S3_MAX_DELETE_BATCH)) { try { const failed = await this.deleteChunk(chunk, storageTarget); - failedPaths.push(...failed); + failures.push(...failed); } catch (error) { - this.logger.error({ msg: 'S3 batch request failed', bucket: storageTarget, error }); - failedPaths.push(...chunk); + // Whole chunk failed (network/auth/etc) — mark every path in it with the same reason + // so the caller still gets a per-path failure list and a human-readable cause. + const reason = describeError(error); + this.logger.error({ msg: 'S3 batch request failed', bucket: storageTarget, reason, error }); + failures.push(...chunk.map((path) => ({ path, reason }))); } } - return failedPaths; + return failures; } public async targetExists(bucket: string, relativePath: string): Promise { @@ -69,14 +73,16 @@ export class S3StorageProvider implements IStorageProvider { } } - private async deleteChunk(paths: string[], bucket: string): Promise { + private async deleteChunk(paths: string[], bucket: string): Promise { const command = new DeleteObjectsCommand({ Bucket: bucket, Delete: { Objects: paths.map((Key) => ({ Key })) }, }); const response = await this.s3Client.send(command); - return (response.Errors ?? []).map((e) => e.Key ?? '').filter(Boolean); // filter out any empty keys just in case, though they shouldn't occur + return (response.Errors ?? []) + .filter((e): e is typeof e & { Key: string } => Boolean(e.Key)) // Only include entries with a Key so every failure maps to a specific path. + .map((e) => ({ path: e.Key, reason: e.Code ?? e.Message ?? 'Unknown' })); } private *chunk(paths: string[], size: number): Generator { diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index b20e9b4..1b58c17 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -5,8 +5,8 @@ import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue import { PERCENTAGE_COMPLETE, SERVICES } from '@common/constants'; import type { ConfigType } from '@common/config'; import { validateSchema } from '../utils'; -import { RecoverableError, UnrecoverableError } from '../errors'; -import type { IStorageProvider } from '../storageProviders'; +import { RecoverableError, UnrecoverableError, describeError } from '../errors'; +import { summarizeDeleteFailures, type DeleteFailure, type IStorageProvider } from '../storageProviders'; import type { TaskContext } from './strategyFactory'; import type { ITaskStrategy } from './taskStrategy'; @@ -53,18 +53,19 @@ export class TilesDeletionStrategy implements ITaskStrategy totalTiles, }); - const failedPaths = await this.deleteAllTiles(provider, storageTarget, params, totalTiles); + const failures = await this.deleteAllTiles(provider, storageTarget, params, totalTiles); - if (failedPaths.length > 0) { - const sample = failedPaths.slice(0, this.failureSampleSize); + if (failures.length > 0) { + const { counts, summary, sample } = summarizeDeleteFailures(failures, this.failureSampleSize); this.logger.error({ msg: 'Tiles deletion partially failed', totalTiles, - failedCount: failedPaths.length, - deletedCount: totalTiles - failedPaths.length, + failedCount: failures.length, + deletedCount: totalTiles - failures.length, + reasonCounts: counts, sample, }); - throw new RecoverableError(`Failed to delete ${failedPaths.length} tiles. Sample: ${sample.join(', ')}`); + throw new RecoverableError(`Failed to delete ${failures.length} tiles. Reasons: ${summary}. Sample: ${sample.join(', ')}`); } this.logger.info({ msg: 'Tiles deletion completed successfully', deletedCount: totalTiles }); @@ -84,9 +85,9 @@ export class TilesDeletionStrategy implements ITaskStrategy storageTarget: string, params: TilesDeletionParams, totalTiles: number - ): Promise { + ): Promise { const { jobId, taskId } = this.taskContext; - const failedPaths: string[] = []; + const failures: DeleteFailure[] = []; const pendingBatches: string[][] = []; let batch: string[] = []; let processedTiles = 0; @@ -97,10 +98,10 @@ export class TilesDeletionStrategy implements ITaskStrategy pendingBatches.push(batch); batch = []; if (pendingBatches.length === this.concurrency) { - processedTiles += await this.flushBatches(provider, storageTarget, pendingBatches, failedPaths); + processedTiles += await this.flushBatches(provider, storageTarget, pendingBatches, failures); const percentage = Math.round((processedTiles / totalTiles) * PERCENTAGE_COMPLETE); await this.queueClient.updateProgress(jobId, taskId, percentage); - this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failedPaths.length }); + this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.length }); } } } @@ -109,33 +110,43 @@ export class TilesDeletionStrategy implements ITaskStrategy pendingBatches.push(batch); } if (pendingBatches.length > 0) { - await this.flushBatches(provider, storageTarget, pendingBatches, failedPaths); - this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failedPaths.length }); - if (failedPaths.length === 0) { + await this.flushBatches(provider, storageTarget, pendingBatches, failures); + this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.length }); + if (failures.length === 0) { await this.queueClient.updateProgress(jobId, taskId, PERCENTAGE_COMPLETE); } } - return failedPaths; + return failures; } /** * Deletes all pending batches concurrently via Promise.allSettled. - * Soft failures (paths returned by provider.delete) and hard failures (rejected promises) - * are both collected into failedPaths; hard-failed batch paths are added by index so nothing - * is silently lost. pendingBatches is cleared in-place for reuse. + * Soft failures (entries returned by provider.delete with reason) and hard failures + * (rejected promises — expanded into per-path failures with the thrown error as + * the reason) are both collected so nothing is silently lost and every failed path + * carries a cause the caller can surface in the task rejection reason. + * pendingBatches is cleared in-place for reuse. * * @returns Total tile paths attempted (not necessarily deleted). */ - private async flushBatches(provider: IStorageProvider, storageTarget: string, pendingBatches: string[][], failedPaths: string[]): Promise { + private async flushBatches( + provider: IStorageProvider, + storageTarget: string, + pendingBatches: string[][], + failures: DeleteFailure[] + ): Promise { const flushedCount = pendingBatches.reduce((sum, b) => sum + b.length, 0); const results = await Promise.allSettled(pendingBatches.map(async (batch) => provider.delete(batch, storageTarget))); for (const [index, result] of results.entries()) { if (result.status === 'fulfilled') { - failedPaths.push(...result.value); + failures.push(...result.value); } else { - this.logger.error({ msg: 'Batch delete threw unexpectedly', error: result.reason }); - failedPaths.push(...(pendingBatches[index] ?? [])); + const error: unknown = result.reason; + const reason = describeError(error); + this.logger.error({ msg: 'Batch delete threw unexpectedly', reason, error }); + const batch = pendingBatches[index] ?? []; + failures.push(...batch.map((path) => ({ path, reason }))); } } pendingBatches.length = 0; diff --git a/tests/storageProviders/deleteFailureSummary.spec.ts b/tests/storageProviders/deleteFailureSummary.spec.ts new file mode 100644 index 0000000..db128ef --- /dev/null +++ b/tests/storageProviders/deleteFailureSummary.spec.ts @@ -0,0 +1,76 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { describe, it, expect } from 'vitest'; +import { summarizeDeleteFailures } from '@src/cleaner/storageProviders'; +import type { DeleteFailure } from '@src/cleaner/storageProviders'; + +describe('summarizeDeleteFailures', () => { + it('should return zero counts, empty summary and empty sample for an empty input', () => { + const result = summarizeDeleteFailures([], 5); + expect(result).toEqual({ counts: {}, summary: '', sample: [] }); + }); + + it('should count occurrences per reason', () => { + const failures: DeleteFailure[] = [ + { path: 'a', reason: 'ENOENT' }, + { path: 'b', reason: 'ENOENT' }, + { path: 'c', reason: 'EACCES' }, + ]; + + const { counts } = summarizeDeleteFailures(failures, 3); + + expect(counts).toEqual({ ENOENT: 2, EACCES: 1 }); + }); + + it('should format the summary with reasons sorted by descending count', () => { + const failures: DeleteFailure[] = [ + { path: 'a', reason: 'EACCES' }, + { path: 'b', reason: 'ENOENT' }, + { path: 'c', reason: 'ENOENT' }, + { path: 'd', reason: 'ENOENT' }, + ]; + + const { summary } = summarizeDeleteFailures(failures, 3); + + expect(summary).toBe('ENOENT=3, EACCES=1'); + }); + + it('should annotate each sampled path with its reason', () => { + const failures: DeleteFailure[] = [ + { path: 'tile/1.png', reason: 'ENOENT' }, + { path: 'tile/2.png', reason: 'EACCES' }, + ]; + + const { sample } = summarizeDeleteFailures(failures, 3); + + expect(sample).toEqual(['tile/1.png (ENOENT)', 'tile/2.png (EACCES)']); + }); + + it('should cap the sample to sampleSize and preserve input order', () => { + const failures: DeleteFailure[] = [ + { path: 'a', reason: 'ENOENT' }, + { path: 'b', reason: 'ENOENT' }, + { path: 'c', reason: 'ENOENT' }, + { path: 'd', reason: 'ENOENT' }, + ]; + + const { sample } = summarizeDeleteFailures(failures, 2); + + expect(sample).toEqual(['a (ENOENT)', 'b (ENOENT)']); + }); + + it('should return all failures in the sample when sampleSize exceeds input length', () => { + const failures: DeleteFailure[] = [{ path: 'a', reason: 'ENOENT' }]; + + const { sample } = summarizeDeleteFailures(failures, 10); + + expect(sample).toEqual(['a (ENOENT)']); + }); + + it('should return an empty sample when sampleSize is 0', () => { + const failures: DeleteFailure[] = [{ path: 'a', reason: 'ENOENT' }]; + + const { sample } = summarizeDeleteFailures(failures, 0); + + expect(sample).toEqual([]); + }); +}); diff --git a/tests/storageProviders/fsStorageProvider.spec.ts b/tests/storageProviders/fsStorageProvider.spec.ts index f824601..41756bc 100644 --- a/tests/storageProviders/fsStorageProvider.spec.ts +++ b/tests/storageProviders/fsStorageProvider.spec.ts @@ -79,22 +79,30 @@ describe('FsStorageProvider', () => { expect(result).toEqual([]); }); - it('should treat ENOENT as a failed deletion (included in failure report)', async () => { + it('should treat ENOENT as a failed deletion tagged with ENOENT reason', async () => { const enoent = Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); vi.mocked(unlink).mockRejectedValue(enoent); const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); - expect(result).toEqual(['tile/10/0/0.png']); + expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'ENOENT' }]); }); - it('should return failed path for non-ENOENT errors', async () => { + it('should return failed path with reason for non-ENOENT errors', async () => { const permError = Object.assign(new Error('EACCES'), { code: 'EACCES' }); vi.mocked(unlink).mockRejectedValue(permError); const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); - expect(result).toEqual(['tile/10/0/0.png']); + expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'EACCES' }]); + }); + + it('should fall back to error message when error has no errno code', async () => { + vi.mocked(unlink).mockRejectedValue(new Error('disk on fire')); + + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + + expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'disk on fire' }]); }); it('should handle mixed success, ENOENT and real errors', async () => { @@ -109,18 +117,21 @@ describe('FsStorageProvider', () => { const paths = ['tile/10/0/0.png', 'tile/10/0/1.png', 'tile/10/0/2.png']; const result = await provider.delete(paths, BASE_PATH); - expect(result).toEqual(['tile/10/0/1.png', 'tile/10/0/2.png']); + expect(result).toEqual([ + { path: 'tile/10/0/1.png', reason: 'ENOENT' }, + { path: 'tile/10/0/2.png', reason: 'EACCES' }, + ]); }); - it('should use relative path as key in failed paths (not the full absolute path)', async () => { + it('should use relative path (not the full absolute path) in failure entries', async () => { const permError = Object.assign(new Error('EACCES'), { code: 'EACCES' }); vi.mocked(unlink).mockRejectedValue(permError); const relativePath = 'layer/v1/10/5/3.png'; const result = await provider.delete([relativePath], BASE_PATH); - expect(result).toEqual([relativePath]); - expect(result[0]).not.toContain(BASE_PATH); + expect(result).toEqual([{ path: relativePath, reason: 'EACCES' }]); + expect(result[0]!.path).not.toContain(BASE_PATH); }); describe('cleanupEmptyDirs', () => { diff --git a/tests/storageProviders/s3StorageProvider.spec.ts b/tests/storageProviders/s3StorageProvider.spec.ts index b9001cc..028129d 100644 --- a/tests/storageProviders/s3StorageProvider.spec.ts +++ b/tests/storageProviders/s3StorageProvider.spec.ts @@ -59,7 +59,7 @@ describe('S3StorageProvider', () => { expect(result).toEqual([]); }); - it('should return failed paths from response.Errors', async () => { + it('should return failed paths tagged with the S3 error Code', async () => { mockSend.mockResolvedValue({ Errors: [{ Key: 'a.txt', Code: 'AccessDenied', Message: 'Forbidden' }], }); @@ -67,10 +67,10 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual(['a.txt']); + expect(result).toEqual([{ path: 'a.txt', reason: 'AccessDenied' }]); }); - it('should treat NoSuchKey as a failed deletion (included in failure report)', async () => { + it('should treat NoSuchKey as a failed deletion tagged with NoSuchKey reason', async () => { mockSend.mockResolvedValue({ Errors: [{ Key: 'missing.txt', Code: 'NoSuchKey', Message: 'Not Found' }], }); @@ -78,10 +78,20 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual(['missing.txt']); + expect(result).toEqual([{ path: 'missing.txt', reason: 'NoSuchKey' }]); }); - it('should return all errors including NoSuchKey', async () => { + it('should fall back to Message when error has no Code', async () => { + mockSend.mockResolvedValue({ + Errors: [{ Key: 'a.txt', Message: 'Something bad' }], + }); + + const result = await provider.delete(['a.txt'], BUCKET); + + expect(result).toEqual([{ path: 'a.txt', reason: 'Something bad' }]); + }); + + it('should return all errors including NoSuchKey with their codes', async () => { mockSend.mockResolvedValue({ Errors: [ { Key: 'a.txt', Code: 'NoSuchKey' }, @@ -93,7 +103,12 @@ describe('S3StorageProvider', () => { const result = await provider.delete(['a.txt', 'b.txt', 'c.txt', 'd.txt'], BUCKET); - expect(result).toEqual(['a.txt', 'b.txt', 'c.txt', 'd.txt']); + expect(result).toEqual([ + { path: 'a.txt', reason: 'NoSuchKey' }, + { path: 'b.txt', reason: 'AccessDenied' }, + { path: 'c.txt', reason: 'NoSuchKey' }, + { path: 'd.txt', reason: 'InternalError' }, + ]); }); it('should batch paths into chunks of 1000 (S3 limit)', async () => { @@ -120,16 +135,22 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual(['object-0.txt', 'object-1000.txt']); + expect(result).toEqual([ + { path: 'object-0.txt', reason: 'AccessDenied' }, + { path: 'object-1000.txt', reason: 'AccessDenied' }, + ]); }); - it('should add entire chunk to failed paths when send throws', async () => { + it('should add entire chunk to failures tagged with the thrown error when send rejects', async () => { mockSend.mockRejectedValue(new Error('Network error')); const paths = ['a.txt', 'b.txt']; const result = await provider.delete(paths, BUCKET); - expect(result).toEqual(paths); + expect(result).toEqual([ + { path: 'a.txt', reason: 'Network error' }, + { path: 'b.txt', reason: 'Network error' }, + ]); }); }); diff --git a/tests/tilesDeletionStrategy.spec.ts b/tests/tilesDeletionStrategy.spec.ts index b2d0c26..69b1c5d 100644 --- a/tests/tilesDeletionStrategy.spec.ts +++ b/tests/tilesDeletionStrategy.spec.ts @@ -214,7 +214,7 @@ describe('TilesDeletionStrategy', () => { }); it('should not call updateProgress with 100 when the final flush has failures', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([tilePath(10, 0, 0)]); + vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'NoSuchKey' }]); await expect(strategy.execute(s3Params)).rejects.toThrow(RecoverableError); @@ -224,15 +224,34 @@ describe('TilesDeletionStrategy', () => { describe('failure handling', () => { it('should throw RecoverableError when provider returns failed paths', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([tilePath(10, 0, 0)]); + vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'NoSuchKey' }]); await expect(strategy.execute(s3Params)).rejects.toThrow(RecoverableError); }); it('should include failed count in RecoverableError message', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([tilePath(10, 0, 0), tilePath(10, 0, 1)]); + vi.mocked(MockS3Provider.delete).mockResolvedValue([ + { path: tilePath(10, 0, 0), reason: 'NoSuchKey' }, + { path: tilePath(10, 0, 1), reason: 'NoSuchKey' }, + ]); - await expect(strategy.execute(s3Params)).rejects.toThrow(/2/); + await expect(strategy.execute(s3Params)).rejects.toThrow(/Failed to delete 2/); + }); + + it('should include grouped reason counts in RecoverableError message', async () => { + vi.mocked(MockS3Provider.delete).mockResolvedValue([ + { path: tilePath(10, 0, 0), reason: 'NoSuchKey' }, + { path: tilePath(10, 0, 1), reason: 'NoSuchKey' }, + { path: tilePath(10, 1, 0), reason: 'AccessDenied' }, + ]); + + await expect(strategy.execute(s3Params)).rejects.toThrow(/Reasons: NoSuchKey=2, AccessDenied=1/); + }); + + it('should include path and reason in the failure sample', async () => { + vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'NoSuchKey' }]); + + await expect(strategy.execute(s3Params)).rejects.toThrow(/layer\/v1\/10\/0\/0\.png \(NoSuchKey\)/); }); it('should resolve successfully when provider returns no failed paths', async () => { @@ -253,6 +272,12 @@ describe('TilesDeletionStrategy', () => { await expect(strategy.execute(s3Params)).rejects.toThrow(/Failed to delete 4/); }); + + it('should tag hard-rejected batch failures with the thrown error message as reason', async () => { + vi.mocked(MockS3Provider.delete).mockRejectedValue(new Error('S3 connection lost')); + + await expect(strategy.execute(s3Params)).rejects.toThrow(/S3 connection lost/); + }); }); }); }); From d97ac9720b6ab7345de90236aabf5db5ebe560f8 Mon Sep 17 00:00:00 2001 From: almog8k Date: Mon, 18 May 2026 17:40:30 +0300 Subject: [PATCH 3/7] feat(tests): enhance tiles deletion strategy with integration tests and helpers --- eslint.config.mjs | 2 +- package-lock.json | 1384 ++++++++++++++++- package.json | 5 +- .../storageProviders/s3StorageProvider.ts | 3 +- tests/helpers/fakes/index.ts | 1 + tests/helpers/fakes/tilesDeletionFakes.ts | 45 + tests/integration/helpers/backendFixtures.ts | 79 + tests/integration/helpers/fsTestKit.ts | 57 + tests/integration/helpers/minioContainer.ts | 49 + tests/integration/helpers/s3TestKit.ts | 84 + tests/integration/helpers/testPoller.ts | 118 ++ tests/integration/helpers/tileFixtures.ts | 68 + .../tilesDeletionStrategy.integration.spec.ts | 69 + vitest.integration.config.mts | 29 + 14 files changed, 1932 insertions(+), 61 deletions(-) create mode 100644 tests/helpers/fakes/tilesDeletionFakes.ts create mode 100644 tests/integration/helpers/backendFixtures.ts create mode 100644 tests/integration/helpers/fsTestKit.ts create mode 100644 tests/integration/helpers/minioContainer.ts create mode 100644 tests/integration/helpers/s3TestKit.ts create mode 100644 tests/integration/helpers/testPoller.ts create mode 100644 tests/integration/helpers/tileFixtures.ts create mode 100644 tests/integration/tilesDeletionStrategy.integration.spec.ts create mode 100644 vitest.integration.config.mts diff --git a/eslint.config.mjs b/eslint.config.mjs index c7cea9d..100e66d 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,4 +1,4 @@ import tsBaseConfig from '@map-colonies/eslint-config/ts-base'; import { config } from '@map-colonies/eslint-config/helpers'; -export default config(tsBaseConfig, { ignores: ['vitest.config.mts'] }); +export default config(tsBaseConfig, { ignores: ['vitest.config.mts', 'vitest.integration.config.mts'] }); diff --git a/package-lock.json b/package-lock.json index 4d16ca5..d960a0c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "prettier": "^3.5.3", "pretty-quick": "^4.1.1", "rimraf": "^6.0.1", + "testcontainers": "^11.14.0", "tsc-alias": "^1.8.11", "type-fest": "^5.1.0", "typescript": "^5.8.2", @@ -1030,6 +1031,13 @@ "node": ">=6.9.0" } }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/@bcoe/v8-coverage": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", @@ -2263,6 +2271,16 @@ "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==", "license": "MIT" }, + "node_modules/@kwsites/file-exists": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", + "integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.1" + } + }, "node_modules/@map-colonies/commitlint-config": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@map-colonies/commitlint-config/-/commitlint-config-1.1.1.tgz", @@ -7115,6 +7133,29 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/docker-modem": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/docker-modem/-/docker-modem-3.0.6.tgz", + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/dockerode": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-4.0.1.tgz", + "integrity": "sha512-cmUpB+dPN955PxBEuXE3f6lKO1hHiIGYJA46IVF3BJpNsZGvtBDcRnlrHYHtOH/B6vtDOyl2kZ2ShAu3mgc27Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/docker-modem": "*", + "@types/node": "*", + "@types/ssh2": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -7286,6 +7327,43 @@ "integrity": "sha512-UE7oxhQLLd9gub6JKIAhDq06T0F6FnztwMNRvYgjeQSBeMc1ZG/tA47EwfduvkuQS8apbkM/lpLpWsaCeYsXVg==", "license": "MIT" }, + "node_modules/@types/ssh2": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz", + "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2-streams": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/ssh2-streams/-/ssh2-streams-0.1.13.tgz", + "integrity": "sha512-faHyY3brO9oLEA0QlcO8N2wT7R0+1sHWZvQ+y3rMLwdY1ZyS1z0W3t65j9PqT4HmQ6ALzNe7RZlNuCNE0wBSWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/ssh2/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/tedious": { "version": "4.0.14", "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", @@ -7993,6 +8071,19 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -8140,6 +8231,170 @@ "node": ">= 8" } }, + "node_modules/archiver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.2", + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^3.0.0", + "zip-stream": "^6.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "glob": "^10.0.0", + "graceful-fs": "^4.2.0", + "is-stream": "^2.0.1", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils/node_modules/brace-expansion": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/archiver-utils/node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/archiver-utils/node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/archiver-utils/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/archiver/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/archiver/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -8169,6 +8424,16 @@ "node": ">=8" } }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -8198,6 +8463,20 @@ "dev": true, "license": "MIT" }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "dev": true, + "license": "MIT" + }, + "node_modules/async-lock": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.1.tgz", + "integrity": "sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==", + "dev": true, + "license": "MIT" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -8236,79 +8515,284 @@ "axios": "0.x || 1.x" } }, + "node_modules/b4a": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz", + "integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, - "node_modules/bignumber.js": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", - "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", - "license": "MIT", - "engines": { - "node": "*" + "node_modules/bare-events": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", + "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } } }, - "node_modules/binary-extensions": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", - "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "node_modules/bare-fs": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.1.tgz", + "integrity": "sha512-WDRsyVN52eAx/lBamKD6uyw8H4228h/x0sGGGegOamM2cd7Pag88GfMQalobXI+HaEUxpCkbKQUDOQqt9wawRw==", "dev": true, - "license": "MIT", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, "engines": { - "node": ">=8" + "bare": ">=1.16.0" }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } } }, - "node_modules/bintrees": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", - "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", - "license": "MIT" - }, - "node_modules/body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", - "license": "MIT", - "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - }, + "node_modules/bare-os": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.9.1.tgz", + "integrity": "sha512-6M5XjcnsygQNPMCMPXSK379xrJFiZ/AEMNBmFEmQW8d/789VQATvriyi5r0HYTL9TkQ26rn3kgdTG3aisbrXkQ==", + "dev": true, + "license": "Apache-2.0", "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" + "bare": ">=1.14.0" } }, - "node_modules/body-parser/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "dev": true, + "license": "Apache-2.0", "dependencies": { - "ms": "2.0.0" + "bare-os": "^3.0.1" } }, - "node_modules/body-parser/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, + "node_modules/bare-stream": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.1.tgz", + "integrity": "sha512-Vp0cnjYyrEC4whYTymQ+YZi6pBpfiICZO3cfRG8sy67ZNWe951urv1x4eW1BKNngw3U+3fPYb5JQvHbCtxH7Ow==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.3.tgz", + "integrity": "sha512-Kccpc7ACfXaxfeInfqKcZtW4pT5YBn1mesc4sCsun6sRwtbJ4h+sNOaksUpYEJUKfN65YWC6Bw2OJEFiKxq8nQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/bl/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/body-parser": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, "node_modules/bowser": { "version": "2.14.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", @@ -8337,12 +8821,67 @@ "node": ">=8" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "license": "MIT" }, + "node_modules/buildcheck": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", + "dev": true, + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/byline": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", + "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -8497,6 +9036,13 @@ "node": ">= 6" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true, + "license": "ISC" + }, "node_modules/cjs-module-lexer": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", @@ -8590,6 +9136,50 @@ "dot-prop": "^5.1.0" } }, + "node_modules/compress-commons": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^6.0.0", + "is-stream": "^2.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/compress-commons/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/compress-commons/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/compressible": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", @@ -8869,6 +9459,75 @@ "typescript": ">=5" } }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "dev": true, + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/crc32-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/crc32-stream/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/cross-env": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", @@ -9100,6 +9759,168 @@ "node": ">=8" } }, + "node_modules/docker-compose": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/docker-compose/-/docker-compose-1.4.2.tgz", + "integrity": "sha512-rPHigTKGaEHpkUmfd69QgaOp+Os5vGJwG/Ry8lcr8W/382AmI+z/D7qoa9BybKIkqNppaIbs8RYeHSevdQjWww==", + "dev": true, + "license": "MIT", + "dependencies": { + "yaml": "^2.2.2" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/docker-modem": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.15.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/docker-modem/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/docker-modem/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/dockerode": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.12.tgz", + "integrity": "sha512-/bCZd6KlGcjZO8Buqmi/vXuqEGVEZ0PNjx/biBNqJD3MhK9DmdiAuKxqfNhflgDESDIiBz3qF+0e55+CpnrUcw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@grpc/grpc-js": "^1.11.1", + "@grpc/proto-loader": "^0.7.13", + "docker-modem": "^5.0.7", + "protobufjs": "^7.3.2", + "tar-fs": "^2.1.4", + "uuid": "^10.0.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode/node_modules/@grpc/proto-loader": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dockerode/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/dockerode/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/dockerode/node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/dockerode/node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dockerode/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/dot-prop": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", @@ -9778,6 +10599,36 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/expect-type": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.2.2.tgz", @@ -9883,6 +10734,13 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "license": "MIT" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -10173,6 +11031,13 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true, + "license": "MIT" + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -10321,6 +11186,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-port": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-7.2.0.tgz", + "integrity": "sha512-afP4W205ONCuMoPBqcR6PSXnzX35KTcJygfJfcp+QY+uwm3p20p1YczWXhlICIzGMCxYBQcySEcOgsJcrkyobg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", @@ -10457,6 +11335,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, "node_modules/graphemer": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", @@ -10601,6 +11486,27 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -11108,6 +12014,59 @@ "json-buffer": "3.0.1" } }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -11439,6 +12398,13 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true, + "license": "MIT" + }, "node_modules/module-details-from-path": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", @@ -11485,6 +12451,14 @@ "url": "https://github.com/sponsors/raouldeheer" } }, + "node_modules/nan": { + "version": "2.27.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.27.0.tgz", + "integrity": "sha512-hC+0LidcL3XE4rp1C4H54KujgXKzbfyTngZTwBByQxsOxCEKZT0MPQ4hOKUH2jU1OYstqdDH4onyHPDzcV0XdQ==", + "dev": true, + "license": "MIT", + "optional": true + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -12568,6 +13542,16 @@ "dev": true, "license": "MIT" }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -12603,6 +13587,59 @@ "node": "^16 || ^18 || >=20" } }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/properties-reader": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/properties-reader/-/properties-reader-3.0.1.tgz", + "integrity": "sha512-WPn+h9RGEExOKdu4bsF4HksG/uzd3cFq3MFtq8PsFeExPse5Ha/VOjQNyHhjboBFwGXGev6muJYTSPAOkROq2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@kwsites/file-exists": "^1.1.1", + "mkdirp": "^3.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/steveukx/properties?sponsor=1" + } + }, + "node_modules/properties-reader/node_modules/mkdirp": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", + "dev": true, + "license": "MIT", + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/protobufjs": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", @@ -12771,6 +13808,39 @@ "string_decoder": "~0.10.x" } }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -12891,6 +13961,16 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -13389,6 +14469,13 @@ "node": ">=18.20 || >=20" } }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", + "dev": true, + "license": "ISC" + }, "node_modules/split2": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", @@ -13398,6 +14485,46 @@ "node": ">= 10.x" } }, + "node_modules/ssh-remote-port-forward": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/ssh-remote-port-forward/-/ssh-remote-port-forward-1.0.4.tgz", + "integrity": "sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ssh2": "^0.5.48", + "ssh2": "^1.4.0" + } + }, + "node_modules/ssh-remote-port-forward/node_modules/@types/ssh2": { + "version": "0.5.52", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-0.5.52.tgz", + "integrity": "sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2-streams": "*" + } + }, + "node_modules/ssh2": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.23.0" + } + }, "node_modules/stable-hash": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stable-hash/-/stable-hash-0.0.5.tgz", @@ -13448,6 +14575,18 @@ "npm": ">=6" } }, + "node_modules/streamx": { + "version": "2.25.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.25.0.tgz", + "integrity": "sha512-0nQuG6jf1w+wddNEEXCF4nTg3LtufWINB5eFEN+5TNZW7KWJp6x87+JFL43vaAUPyCfH1wID+mNVyW6OHtFamg==", + "dev": true, + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, "node_modules/string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", @@ -13592,6 +14731,34 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/tar-fs": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.2.tgz", + "integrity": "sha512-QGxxTxxyleAdyM3kpFs14ymbYmNFrfY+pHj7Z8FgtbZ7w2//VAgLMac7sT6nRpIHjppXO2AwwEOg0bPFVRcmXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.0.tgz", + "integrity": "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/tdigest": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", @@ -13601,6 +14768,16 @@ "bintrees": "1.0.2" } }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, "node_modules/test-exclude": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-7.0.1.tgz", @@ -13687,6 +14864,40 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/testcontainers": { + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/testcontainers/-/testcontainers-11.14.0.tgz", + "integrity": "sha512-r9pniwv/iwzyHaI7gwAvAm4Y+IvjJg3vBWdjrUCaDMc2AXIr4jKbq7jJO18Mw2ybs73pZy1Aj7p/4RVBGMRWjg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@types/dockerode": "^4.0.1", + "archiver": "^7.0.1", + "async-lock": "^1.4.1", + "byline": "^5.0.0", + "debug": "^4.4.3", + "docker-compose": "^1.4.2", + "dockerode": "^4.0.10", + "get-port": "^7.2.0", + "proper-lockfile": "^4.1.2", + "properties-reader": "^3.0.1", + "ssh-remote-port-forward": "^1.0.4", + "tar-fs": "^3.1.2", + "tmp": "^0.2.5", + "undici": "^7.24.5" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-extensions": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-2.4.0.tgz", @@ -13866,6 +15077,16 @@ "node": ">=14.0.0" } }, + "node_modules/tmp": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -14016,6 +15237,13 @@ "integrity": "sha512-Ja03QIJlPuHt4IQ2FfGex4F4JAr8m3jpaHbFbQrgwr7s7L6U8ocrHiF3J1+wf9jzhGKxvDeaCAnGDot8OjGFyA==", "license": "(EDL-1.0 OR EPL-1.0)" }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", + "dev": true, + "license": "Unlicense" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -14096,9 +15324,9 @@ } }, "node_modules/undici": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.16.0.tgz", - "integrity": "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==", + "version": "7.25.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", + "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", "license": "MIT", "engines": { "node": ">=20.18.1" @@ -14625,6 +15853,48 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zip-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.0", + "compress-commons": "^6.0.2", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/zip-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/zip-stream/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/package.json b/package.json index d71fedb..02ccbc6 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,11 @@ "description": "This is template for map colonies jobnik worker service", "main": "./src/index.ts", "scripts": { - "test": "vitest run", + "test": "npm run test:unit && npm run test:integration", + "test:unit": "vitest run", "test:watch": "vitest watch", "test:ui": "vitest --ui", + "test:integration": "vitest run --config vitest.integration.config.mts", "format": "prettier --check .", "format:fix": "prettier --write .", "prelint:fix": "npm run format:fix", @@ -68,6 +70,7 @@ "prettier": "^3.5.3", "pretty-quick": "^4.1.1", "rimraf": "^6.0.1", + "testcontainers": "^11.14.0", "tsc-alias": "^1.8.11", "type-fest": "^5.1.0", "typescript": "^5.8.2", diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index d554a20..9a8d9aa 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -5,8 +5,6 @@ import type { ConfigType } from '@common/config'; import { describeError } from '../errors'; import type { DeleteFailure, IStorageProvider } from './iStorageProvider'; -const S3_MAX_DELETE_BATCH = 1000; - interface S3Config { endpoint: string; accessKeyId: string; @@ -16,6 +14,7 @@ interface S3Config { region: string; } +export const S3_MAX_DELETE_BATCH = 1000; export class S3StorageProvider implements IStorageProvider { private readonly s3Client: S3Client; diff --git a/tests/helpers/fakes/index.ts b/tests/helpers/fakes/index.ts index cc5b372..adb0c82 100644 --- a/tests/helpers/fakes/index.ts +++ b/tests/helpers/fakes/index.ts @@ -1,2 +1,3 @@ export * from './taskFakes'; export * from './pairFakes'; +export * from './tilesDeletionFakes'; diff --git a/tests/helpers/fakes/tilesDeletionFakes.ts b/tests/helpers/fakes/tilesDeletionFakes.ts new file mode 100644 index 0000000..b177d2e --- /dev/null +++ b/tests/helpers/fakes/tilesDeletionFakes.ts @@ -0,0 +1,45 @@ +import { faker } from '@faker-js/faker'; +import { SourceType, type TileRange, type TilesDeletionParams } from '@map-colonies/raster-shared'; + +const EXTENSIONS = ['png', 'jpeg'] as const; + +function buildTileRange(overrides: Partial = {}): TileRange { + const minX = faker.number.int({ min: 0, max: 50 }); + const minY = faker.number.int({ min: 0, max: 50 }); + return { + zoom: faker.number.int({ min: 0, max: 18 }), + minX, + maxX: minX + faker.number.int({ min: 0, max: 5 }), + minY, + maxY: minY + faker.number.int({ min: 0, max: 5 }), + ...overrides, + }; +} + +function buildTilesPath(): string { + return `${faker.word.noun().toLowerCase()}/${faker.string.alphanumeric({ length: 6, casing: 'lower' })}`; +} + +function buildBaseParams(overrides: Partial): Omit { + return { + tilesPath: overrides.tilesPath ?? buildTilesPath(), + fileExtension: overrides.fileExtension ?? faker.helpers.arrayElement(EXTENSIONS), + ranges: overrides.ranges ?? [buildTileRange()], + }; +} + +function buildS3TilesDeletionParams(overrides: Partial = {}): TilesDeletionParams { + return { + sourceProvider: SourceType.S3, + ...buildBaseParams(overrides), + }; +} + +function buildFsTilesDeletionParams(overrides: Partial = {}): TilesDeletionParams { + return { + sourceProvider: SourceType.FS, + ...buildBaseParams(overrides), + }; +} + +export { buildTileRange, buildS3TilesDeletionParams, buildFsTilesDeletionParams }; diff --git a/tests/integration/helpers/backendFixtures.ts b/tests/integration/helpers/backendFixtures.ts new file mode 100644 index 0000000..06bdb21 --- /dev/null +++ b/tests/integration/helpers/backendFixtures.ts @@ -0,0 +1,79 @@ +import { faker } from '@faker-js/faker'; +import { container } from 'tsyringe'; +import { SourceType } from '@map-colonies/raster-shared'; +import type { S3Client } from '@aws-sdk/client-s3'; +import { S3StorageProvider, FsStorageProvider, type IStorageProvider } from '@src/cleaner/storageProviders'; +import { createMockLogger } from '../../helpers/mocks'; +import { startMinio, type MinioHandle } from './minioContainer'; +import { createTestS3Client, deleteBucket, ensureBucket, listAllKeys, putManyTiles } from './s3TestKit'; +import { listAllFiles, makeTempFsBase, rmBase, writeManyTiles } from './fsTestKit'; +import { buildS3ConfigForMinio } from './testPoller'; + +type ProviderType = Exclude; + +interface BackendHandles { + minio: MinioHandle; + s3Client: S3Client; +} + +interface TestStorageContext { + providers: Map; + bucket: string; + fsBase: string; +} + +async function startBackends(): Promise { + const minio = await startMinio(); + const s3Client = createTestS3Client(minio); + return { minio, s3Client }; +} + +async function stopBackends({ minio, s3Client }: BackendHandles): Promise { + s3Client.destroy(); + await minio.stop(); +} + +async function setupTestStorageContext({ minio, s3Client }: BackendHandles): Promise { + const bucket = `test-${faker.string.alphanumeric({ length: 16, casing: 'lower' })}`; + await ensureBucket(s3Client, bucket); + const fsBase = await makeTempFsBase(); + + const providers = new Map([ + [SourceType.S3, new S3StorageProvider(buildS3ConfigForMinio(minio), createMockLogger())], + [SourceType.FS, new FsStorageProvider(createMockLogger())], + ]); + + return { providers, bucket, fsBase }; +} + +async function teardownTestStorageContext(handles: BackendHandles, storageContext: TestStorageContext): Promise { + try { + await Promise.all([deleteBucket(handles.s3Client, storageContext.bucket), rmBase(storageContext.fsBase)]); + } finally { + container.reset(); + } +} +interface ProviderBackend { + sourceProvider: ProviderType; + seed: (paths: string[]) => Promise; + list: (prefix: string) => Promise; +} + +function s3Backend(handles: () => BackendHandles, perTest: () => TestStorageContext): ProviderBackend { + return { + sourceProvider: SourceType.S3, + seed: async (paths) => putManyTiles(handles().s3Client, perTest().bucket, paths), + list: async (prefix) => listAllKeys(handles().s3Client, perTest().bucket, prefix), + }; +} + +function fsBackend(perTest: () => TestStorageContext): ProviderBackend { + return { + sourceProvider: SourceType.FS, + seed: async (paths) => writeManyTiles(perTest().fsBase, paths), + list: async (prefix) => (await listAllFiles(perTest().fsBase)).filter((p) => p.startsWith(prefix)), + }; +} + +export { startBackends, stopBackends, setupTestStorageContext, teardownTestStorageContext, s3Backend, fsBackend }; +export type { BackendHandles, TestStorageContext, ProviderBackend }; diff --git a/tests/integration/helpers/fsTestKit.ts b/tests/integration/helpers/fsTestKit.ts new file mode 100644 index 0000000..b4af196 --- /dev/null +++ b/tests/integration/helpers/fsTestKit.ts @@ -0,0 +1,57 @@ +import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'; +import { join, relative, sep, dirname } from 'node:path'; +import { tmpdir } from 'node:os'; +import { TINY_TILE_BODY } from './tileFixtures'; + +async function makeTempFsBase(): Promise { + return mkdtemp(join(tmpdir(), 'test-dir')); +} + +async function writeTile(basePath: string, relativePath: string): Promise { + const absolute = join(basePath, relativePath); + await mkdir(dirname(absolute), { recursive: true }); + await writeFile(absolute, TINY_TILE_BODY); +} + +async function writeManyTiles(basePath: string, relativePaths: string[]): Promise { + const concurrency = 32; + for (let i = 0; i < relativePaths.length; i += concurrency) { + await Promise.all(relativePaths.slice(i, i + concurrency).map(async (path) => writeTile(basePath, path))); + } +} + +function toForwardSlashes(value: string): string { + return sep === '/' ? value : value.split(sep).join('/'); +} + +async function listAllFiles(basePath: string): Promise { + const entries = await readdir(basePath, { recursive: true, withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + if (entry.isFile()) { + const parentPath = (entry as unknown as { parentPath?: string; path?: string }).parentPath ?? (entry as unknown as { path: string }).path; + const abs = join(parentPath, entry.name); + files.push(toForwardSlashes(relative(basePath, abs))); + } + } + return files.sort(); +} + +async function listAllDirs(basePath: string): Promise { + const entries = await readdir(basePath, { recursive: true, withFileTypes: true }); + const dirs: string[] = []; + for (const entry of entries) { + if (entry.isDirectory()) { + const parentPath = (entry as unknown as { parentPath?: string; path?: string }).parentPath ?? (entry as unknown as { path: string }).path; + const abs = join(parentPath, entry.name); + dirs.push(toForwardSlashes(relative(basePath, abs))); + } + } + return dirs.sort(); +} + +async function rmBase(basePath: string): Promise { + await rm(basePath, { recursive: true, force: true }); +} + +export { makeTempFsBase, writeTile, writeManyTiles, listAllFiles, listAllDirs, rmBase }; diff --git a/tests/integration/helpers/minioContainer.ts b/tests/integration/helpers/minioContainer.ts new file mode 100644 index 0000000..8f53e87 --- /dev/null +++ b/tests/integration/helpers/minioContainer.ts @@ -0,0 +1,49 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { GenericContainer, type StartedTestContainer } from 'testcontainers'; + +interface MinioHandle { + endpoint: string; + accessKeyId: string; + secretAccessKey: string; + stop: () => Promise; +} + +const MINIO_PORT = 9000; +const DEFAULT_USER = 'minioadmin'; +const DEFAULT_PASSWORD = 'minioadmin'; + +async function startMinio(): Promise { + const externalEndpoint = process.env.TEST_MINIO_ENDPOINT; + if (externalEndpoint !== undefined && externalEndpoint !== '') { + return { + endpoint: externalEndpoint, + accessKeyId: process.env.TEST_MINIO_ACCESS_KEY ?? DEFAULT_USER, + secretAccessKey: process.env.TEST_MINIO_SECRET_KEY ?? DEFAULT_PASSWORD, + stop: async (): Promise => Promise.resolve(), + }; + } + console.log('No external Minio endpoint configured, starting testcontainer instance'); + const container: StartedTestContainer = await new GenericContainer('minio/minio:latest') + .withCommand(['server', '/data']) + .withEnvironment({ + MINIO_ROOT_USER: DEFAULT_USER, + MINIO_ROOT_PASSWORD: DEFAULT_PASSWORD, + }) + .withExposedPorts(MINIO_PORT) + .start(); + + return { + endpoint: `http://${container.getHost()}:${container.getMappedPort(MINIO_PORT)}`, + accessKeyId: DEFAULT_USER, + secretAccessKey: DEFAULT_PASSWORD, + stop: async (): Promise => { + try { + await container.stop(); + } catch { + // ignore; Ryuk will clean up on test process exit + } + }, + }; +} + +export { startMinio, type MinioHandle }; diff --git a/tests/integration/helpers/s3TestKit.ts b/tests/integration/helpers/s3TestKit.ts new file mode 100644 index 0000000..b6d7352 --- /dev/null +++ b/tests/integration/helpers/s3TestKit.ts @@ -0,0 +1,84 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { + S3Client, + CreateBucketCommand, + DeleteBucketCommand, + DeleteObjectsCommand, + ListObjectsV2Command, + PutObjectCommand, + BucketAlreadyOwnedByYou, + BucketAlreadyExists, +} from '@aws-sdk/client-s3'; +import { S3_MAX_DELETE_BATCH } from '@src/cleaner/storageProviders/s3StorageProvider'; +import type { MinioHandle } from './minioContainer'; +import { TINY_TILE_BODY } from './tileFixtures'; + +function createTestS3Client(handle: MinioHandle): S3Client { + return new S3Client({ + endpoint: handle.endpoint, + credentials: { + accessKeyId: handle.accessKeyId, + secretAccessKey: handle.secretAccessKey, + }, + region: 'us-east-1', + forcePathStyle: true, + tls: false, + }); +} + +async function ensureBucket(client: S3Client, bucket: string): Promise { + try { + await client.send(new CreateBucketCommand({ Bucket: bucket })); + } catch (err) { + if (err instanceof BucketAlreadyOwnedByYou || err instanceof BucketAlreadyExists) { + return; + } + throw err; + } +} + +async function listAllKeys(client: S3Client, bucket: string, prefix?: string): Promise { + const keys: string[] = []; + let continuationToken: string | undefined; + do { + const response = await client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, ContinuationToken: continuationToken })); + for (const obj of response.Contents ?? []) { + if (obj.Key !== undefined) { + keys.push(obj.Key); + } + } + continuationToken = response.IsTruncated === true ? response.NextContinuationToken : undefined; + } while (continuationToken !== undefined); + return keys.sort(); +} + +async function emptyBucket(client: S3Client, bucket: string): Promise { + const keys = await listAllKeys(client, bucket); + for (let i = 0; i < keys.length; i += S3_MAX_DELETE_BATCH) { + const chunk = keys.slice(i, i + S3_MAX_DELETE_BATCH); + await client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: chunk.map((Key) => ({ Key })) }, + }) + ); + } +} + +async function deleteBucket(client: S3Client, bucket: string): Promise { + await emptyBucket(client, bucket); + await client.send(new DeleteBucketCommand({ Bucket: bucket })); +} + +async function putTile(client: S3Client, bucket: string, key: string): Promise { + await client.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: TINY_TILE_BODY })); +} + +async function putManyTiles(client: S3Client, bucket: string, keys: string[]): Promise { + const concurrency = 20; + for (let i = 0; i < keys.length; i += concurrency) { + await Promise.all(keys.slice(i, i + concurrency).map(async (key) => putTile(client, bucket, key))); + } +} + +export { createTestS3Client, ensureBucket, deleteBucket, emptyBucket, putTile, putManyTiles, listAllKeys }; diff --git a/tests/integration/helpers/testPoller.ts b/tests/integration/helpers/testPoller.ts new file mode 100644 index 0000000..8986700 --- /dev/null +++ b/tests/integration/helpers/testPoller.ts @@ -0,0 +1,118 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +import { vi } from 'vitest'; +import { container } from 'tsyringe'; +import { SourceType } from '@map-colonies/raster-shared'; +import type { ITaskResponse, TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; +import { SERVICES } from '@src/common/constants'; +import type { ConfigType } from '@src/common/config'; +import { TaskPoller } from '@src/worker/taskPoller'; +import { ErrorHandler } from '@src/cleaner/errors'; +import { StrategyFactory, TilesDeletionStrategy } from '@src/cleaner/strategies'; +import type { IStorageProvider } from '@src/cleaner/storageProviders'; +import type { JobTrackerClient } from '@src/cleaner/httpClients'; +import type { PollingPairConfig } from '@src/cleaner/types'; +import { createMockLogger, createMockQueueClient, createMockStrategyConfig, createMockJobTrackerClient } from '../../helpers/mocks'; +import type { MinioHandle } from './minioContainer'; + +const TASK_TYPE = 'tiles-deletion'; +const JOB_TYPE = 'Ingestion_Update'; +const POLLING_PAIR: PollingPairConfig = { jobType: JOB_TYPE, taskType: TASK_TYPE, maxAttempts: 3 }; +const POLLER_WATCHDOG_MS = 30_000; + +function buildS3ConfigForMinio(handle: MinioHandle): ConfigType { + return { + get: () => ({ + endpoint: handle.endpoint, + accessKeyId: handle.accessKeyId, + secretAccessKey: handle.secretAccessKey, + sslEnabled: false, + forcePathStyle: true, + region: 'us-east-1', + }), + } as unknown as ConfigType; +} + +/** + * Returns a queue client whose `dequeue` hands out `task` exactly once for the + * matching pair, then null forever. `ack` / `reject` invoke `onTerminal()` so + * the caller can stop the poller and assert. + */ +function buildSingleShotQueue(task: ITaskResponse, onTerminal: () => void): QueueClient { + const queue = createMockQueueClient(); + let handedOut = false; + + vi.mocked(queue.dequeue).mockImplementation(((jobType: string, taskType: string) => { + if (handedOut || jobType !== JOB_TYPE || taskType !== TASK_TYPE) { + return null; + } + handedOut = true; + return task; + }) as unknown as QueueClient['dequeue']); + + vi.mocked(queue.ack).mockImplementation((() => { + onTerminal(); + }) as unknown as QueueClient['ack']); + + vi.mocked(queue.reject).mockImplementation((() => { + onTerminal(); + }) as unknown as QueueClient['reject']); + + return queue; +} + +interface TestPollerParams { + providers: Map; + bucket: string; + fsBase: string; + task: ITaskResponse; +} + +interface TestPoller { + queueClient: QueueClient; + jobTrackerClient: JobTrackerClient; + runSingleTask: () => Promise; +} + +/** + * Wires the real TaskPoller → StrategyFactory → TilesDeletionStrategy pipeline, + * with real S3/FS providers (supplied by the caller) and a single-shot fake + * QueueClient. The poller terminates as soon as ack/reject fires. + */ +function buildPoller({ providers, bucket, fsBase, task }: TestPollerParams): TestPoller { + const config = createMockStrategyConfig({ + 'strategies.tilesDeletion.s3Bucket': bucket, + 'strategies.tilesDeletion.fsBasePath': fsBase, + 'queue.dequeueIntervalMs': 0, + }); + + container.register(SERVICES.LOGGER, { useValue: createMockLogger() }); + container.register(SERVICES.CONFIG, { useValue: config }); + container.register(SERVICES.STORAGE_PROVIDERS, { useValue: providers }); + + const queueClient = buildSingleShotQueue(task, () => { + void poller.stop(); + }); + container.register(SERVICES.QUEUE_CLIENT, { useValue: queueClient }); + container.register(TASK_TYPE, { useClass: TilesDeletionStrategy }); + + const jobTrackerClient = createMockJobTrackerClient(); + container.register(SERVICES.JOB_TRACKER_CLIENT, { useValue: jobTrackerClient }); + + const strategyFactory = container.resolve(StrategyFactory); + const errorHandler = container.resolve(ErrorHandler); + const poller = new TaskPoller(createMockLogger(), config, queueClient, strategyFactory, errorHandler, [POLLING_PAIR], jobTrackerClient); + + const runSingleTask = async (): Promise => { + const startPromise = poller.start(); + // Watchdog so a regression doesn't hang the whole suite. + const watchdog = new Promise((_, reject) => + setTimeout(() => reject(new Error('Poller did not terminate after task lifecycle')), POLLER_WATCHDOG_MS).unref() + ); + await Promise.race([startPromise, watchdog]); + }; + + return { queueClient, jobTrackerClient, runSingleTask }; +} + +export { TASK_TYPE, JOB_TYPE, POLLING_PAIR, buildS3ConfigForMinio, buildSingleShotQueue, buildPoller }; +export type { TestPoller, TestPollerParams }; diff --git a/tests/integration/helpers/tileFixtures.ts b/tests/integration/helpers/tileFixtures.ts new file mode 100644 index 0000000..7065dbc --- /dev/null +++ b/tests/integration/helpers/tileFixtures.ts @@ -0,0 +1,68 @@ +import type { TileRange, TilesDeletionParams } from '@map-colonies/raster-shared'; + +// First 4 bytes of a PNG file header (\x89 P N G). Content is arbitrary; +// eslint-disable-next-line @typescript-eslint/no-magic-numbers +const TINY_TILE_BODY = Buffer.from([0x89, 0x50, 0x4e, 0x47]); + +function tilePathsForRange(range: TileRange, tilesPath: string, fileExtension: string): string[] { + const paths: string[] = []; + for (let x = range.minX; x <= range.maxX; x++) { + for (let y = range.minY; y <= range.maxY; y++) { + paths.push(`${tilesPath}/${range.zoom}/${x}/${y}.${fileExtension}`); + } + } + return paths; +} + +function tilePathsForRanges(params: TilesDeletionParams): string[] { + return params.ranges.flatMap((range) => tilePathsForRange(range, params.tilesPath, params.fileExtension)); +} + +function paddedRange(range: TileRange, padding: number): TileRange { + return { + zoom: range.zoom, + minX: Math.max(0, range.minX - padding), + maxX: range.maxX + padding, + minY: Math.max(0, range.minY - padding), + maxY: range.maxY + padding, + }; +} + +/** + * Builds the set of tiles inside the `paddedRange` but **outside** the original + * `params.ranges`. These tiles are seeded into the backend before the strategy + * runs and MUST survive — they prove "only the supplied ranges are deleted". + */ +function extraTilePathsAroundRanges(params: TilesDeletionParams, padding: number): string[] { + const targetSet = new Set(tilePathsForRanges(params)); + const extras: string[] = []; + for (const range of params.ranges) { + const wider = paddedRange(range, padding); + for (const path of tilePathsForRange(wider, params.tilesPath, params.fileExtension)) { + if (!targetSet.has(path)) { + extras.push(path); + } + } + } + return extras; +} +/** + * Returns one tile path per (range × zoomDelta) pair, placed at a zoom level + * adjacent to each range's zoom. + * + * Like `extraTilePathsAroundRanges`, the returned paths are seeded before the + * test and must survive after the strategy runs. + * */ +function extraTilePathsAtAdjacentZooms(params: TilesDeletionParams, zoomDeltas: number[]): string[] { + const extras: string[] = []; + for (const range of params.ranges) { + for (const delta of zoomDeltas) { + const zoom = range.zoom + delta; + if (zoom < 0) continue; + extras.push(`${params.tilesPath}/${zoom}/${range.minX}/${range.minY}.${params.fileExtension}`); + } + } + return extras; +} + +export { tilePathsForRange, tilePathsForRanges, paddedRange, extraTilePathsAroundRanges, extraTilePathsAtAdjacentZooms, TINY_TILE_BODY }; diff --git a/tests/integration/tilesDeletionStrategy.integration.spec.ts b/tests/integration/tilesDeletionStrategy.integration.spec.ts new file mode 100644 index 0000000..46d41d5 --- /dev/null +++ b/tests/integration/tilesDeletionStrategy.integration.spec.ts @@ -0,0 +1,69 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { faker } from '@faker-js/faker'; +import { type TilesDeletionParams } from '@map-colonies/raster-shared'; +import { buildTask } from '../helpers/fakes/taskFakes'; +import { + fsBackend, + s3Backend, + setupTestStorageContext, + startBackends, + stopBackends, + teardownTestStorageContext, + type BackendHandles, + type TestStorageContext, +} from './helpers/backendFixtures'; +import { buildPoller, TASK_TYPE } from './helpers/testPoller'; +import { extraTilePathsAroundRanges, extraTilePathsAtAdjacentZooms, tilePathsForRanges } from './helpers/tileFixtures'; + +describe('tiles deletion E2E (polling → strategy → real provider → ack)', () => { + let handles: BackendHandles; + let storageContext: TestStorageContext; + + beforeAll(async () => { + handles = await startBackends(); + }); + + afterAll(async () => { + await stopBackends(handles); + }); + + beforeEach(async () => { + storageContext = await setupTestStorageContext(handles); + }); + + afterEach(async () => { + await teardownTestStorageContext(handles, storageContext); + }); + + describe.each([ + s3Backend( + () => handles, + () => storageContext + ), + fsBackend(() => storageContext), + ])('$sourceProvider provider', (backend) => { + it('polls the task, runs the strategy against the chosen provider, and acks after deleting only the requested tiles', async () => { + const tilesPath = `${faker.string.uuid()}/${faker.string.uuid()}`; + const params: TilesDeletionParams = { + sourceProvider: backend.sourceProvider, + tilesPath, + fileExtension: 'png', + ranges: [{ zoom: 10, minX: 5, maxX: 7, minY: 5, maxY: 7 }], + }; + const target = tilePathsForRanges(params); + const extras = [...extraTilePathsAroundRanges(params, 2), ...extraTilePathsAtAdjacentZooms(params, [-1, 1])]; + await backend.seed([...target, ...extras]); + const task = buildTask({ type: TASK_TYPE, parameters: params }); + + const { runSingleTask, queueClient, jobTrackerClient } = buildPoller({ ...storageContext, task }); + await runSingleTask(); + + const remaining = await backend.list(`${tilesPath}/`); + expect(remaining.sort()).toEqual([...extras].sort()); + expect(queueClient.ack).toHaveBeenCalledWith(task.jobId, task.id); + expect(queueClient.reject).not.toHaveBeenCalled(); + expect(jobTrackerClient.notify).toHaveBeenCalledWith(task.id); + }); + }); +}); diff --git a/vitest.integration.config.mts b/vitest.integration.config.mts new file mode 100644 index 0000000..28104d1 --- /dev/null +++ b/vitest.integration.config.mts @@ -0,0 +1,29 @@ +import { defineConfig, type ViteUserConfig } from 'vitest/config'; +import path from 'path'; +import tsconfig from './tsconfig.json'; + +const pathAlias = Object.fromEntries( + Object.entries(tsconfig.compilerOptions.paths).map(([key, [value]]) => [key.replace('/*', ''), path.resolve(__dirname, value.replace('/*', ''))]) +); + +const reporters: Exclude['reporters'] = ['default']; +if (process.env.GITHUB_ACTIONS) { + reporters.push('github-actions'); +} + +export default defineConfig({ + resolve: { + alias: { + ...pathAlias, + '@map-colonies/raster-shared': path.resolve(__dirname, 'node_modules/@map-colonies/raster-shared/dist/index.js'), + }, + }, + test: { + setupFiles: ['./tests/setup/vite.setup.ts'], + include: ['tests/integration/**/*.integration.spec.ts'], + environment: 'node', + reporters, + testTimeout: 120_000, + hookTimeout: 120_000, + }, +}); From e3d3687ca83b1be16c9768a7f21062d032e6f99c Mon Sep 17 00:00:00 2001 From: almog8k Date: Mon, 18 May 2026 17:41:11 +0300 Subject: [PATCH 4/7] fix: add gisDomain label to mclabels in values.yaml --- helm/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/values.yaml b/helm/values.yaml index be16b49..0369243 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -32,6 +32,7 @@ mclabels: component: backend partOf: core owner: raster + gisDomain: raster prometheus: enabled: true From afdb85dbe0972950f74fe8a0a0f9cab1b3a0c87d Mon Sep 17 00:00:00 2001 From: almog8k Date: Mon, 18 May 2026 17:45:36 +0300 Subject: [PATCH 5/7] fix(tests): randomize file extension for tiles deletion tests --- tests/integration/tilesDeletionStrategy.integration.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/tilesDeletionStrategy.integration.spec.ts b/tests/integration/tilesDeletionStrategy.integration.spec.ts index 46d41d5..dc8535d 100644 --- a/tests/integration/tilesDeletionStrategy.integration.spec.ts +++ b/tests/integration/tilesDeletionStrategy.integration.spec.ts @@ -48,7 +48,7 @@ describe('tiles deletion E2E (polling → strategy → real provider → ack)', const params: TilesDeletionParams = { sourceProvider: backend.sourceProvider, tilesPath, - fileExtension: 'png', + fileExtension: faker.helpers.arrayElement(['png', 'jpg']), ranges: [{ zoom: 10, minX: 5, maxX: 7, minY: 5, maxY: 7 }], }; const target = tilePathsForRanges(params); From e8f3988905e243051a9b79a0992688b02bfc8e57 Mon Sep 17 00:00:00 2001 From: almog8k Date: Tue, 19 May 2026 10:19:24 +0300 Subject: [PATCH 6/7] chore: update @map-colonies/raster-shared to version 8.1.0 --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index d960a0c..c05c28d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@map-colonies/js-logger": "^5.0.0", "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", - "@map-colonies/raster-shared": "^8.1.0-alpha.3", + "@map-colonies/raster-shared": "^8.1.0", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.20.0", "@map-colonies/telemetry": "^10.0.1", @@ -2561,9 +2561,9 @@ "license": "ISC" }, "node_modules/@map-colonies/raster-shared": { - "version": "8.1.0-alpha.3", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.1.0-alpha.3.tgz", - "integrity": "sha512-igF8yhnSeXaUdpx6gW3C5gi4AM9J56jVwgqhh7+Zv0mYb/yBhxv0xsv4Mhyv+41UpFCVduYf90DJZRrY29kLpQ==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.1.0.tgz", + "integrity": "sha512-D/otgsJ7X00NnxlMJUSuj6rC9U779ReZDe4pAmtGsIOuQvASRnjjKIGIj/9nFJth6lbOaptppp0RRGImGd0jIw==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^9.1.0", diff --git a/package.json b/package.json index 02ccbc6..8e796e1 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "@map-colonies/js-logger": "^5.0.0", "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", - "@map-colonies/raster-shared": "^8.1.0-alpha.3", + "@map-colonies/raster-shared": "^8.1.0", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.20.0", "@map-colonies/telemetry": "^10.0.1", From a647c005040a4153fcee3343a97202c00c37a66c Mon Sep 17 00:00:00 2001 From: almog8k Date: Tue, 19 May 2026 10:34:07 +0300 Subject: [PATCH 7/7] test: refactor tiles deletion params to use buildBaseParams helper --- tests/helpers/fakes/tilesDeletionFakes.ts | 21 +++++-------------- .../tilesDeletionStrategy.integration.spec.ts | 10 +++------ 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/tests/helpers/fakes/tilesDeletionFakes.ts b/tests/helpers/fakes/tilesDeletionFakes.ts index b177d2e..cbc4dc8 100644 --- a/tests/helpers/fakes/tilesDeletionFakes.ts +++ b/tests/helpers/fakes/tilesDeletionFakes.ts @@ -2,6 +2,7 @@ import { faker } from '@faker-js/faker'; import { SourceType, type TileRange, type TilesDeletionParams } from '@map-colonies/raster-shared'; const EXTENSIONS = ['png', 'jpeg'] as const; +const PROVIDERS = [SourceType.FS, SourceType.S3] as const; function buildTileRange(overrides: Partial = {}): TileRange { const minX = faker.number.int({ min: 0, max: 50 }); @@ -20,26 +21,14 @@ function buildTilesPath(): string { return `${faker.word.noun().toLowerCase()}/${faker.string.alphanumeric({ length: 6, casing: 'lower' })}`; } -function buildBaseParams(overrides: Partial): Omit { +function buildBaseParams(overrides: Partial = {}): TilesDeletionParams { return { tilesPath: overrides.tilesPath ?? buildTilesPath(), fileExtension: overrides.fileExtension ?? faker.helpers.arrayElement(EXTENSIONS), ranges: overrides.ranges ?? [buildTileRange()], + sourceProvider: overrides.sourceProvider ?? faker.helpers.arrayElement(PROVIDERS), + ...overrides, }; } -function buildS3TilesDeletionParams(overrides: Partial = {}): TilesDeletionParams { - return { - sourceProvider: SourceType.S3, - ...buildBaseParams(overrides), - }; -} - -function buildFsTilesDeletionParams(overrides: Partial = {}): TilesDeletionParams { - return { - sourceProvider: SourceType.FS, - ...buildBaseParams(overrides), - }; -} - -export { buildTileRange, buildS3TilesDeletionParams, buildFsTilesDeletionParams }; +export { buildTileRange, buildBaseParams }; diff --git a/tests/integration/tilesDeletionStrategy.integration.spec.ts b/tests/integration/tilesDeletionStrategy.integration.spec.ts index dc8535d..a1261d5 100644 --- a/tests/integration/tilesDeletionStrategy.integration.spec.ts +++ b/tests/integration/tilesDeletionStrategy.integration.spec.ts @@ -3,6 +3,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from import { faker } from '@faker-js/faker'; import { type TilesDeletionParams } from '@map-colonies/raster-shared'; import { buildTask } from '../helpers/fakes/taskFakes'; +import { buildBaseParams } from '../helpers/fakes/tilesDeletionFakes'; import { fsBackend, s3Backend, @@ -45,12 +46,7 @@ describe('tiles deletion E2E (polling → strategy → real provider → ack)', ])('$sourceProvider provider', (backend) => { it('polls the task, runs the strategy against the chosen provider, and acks after deleting only the requested tiles', async () => { const tilesPath = `${faker.string.uuid()}/${faker.string.uuid()}`; - const params: TilesDeletionParams = { - sourceProvider: backend.sourceProvider, - tilesPath, - fileExtension: faker.helpers.arrayElement(['png', 'jpg']), - ranges: [{ zoom: 10, minX: 5, maxX: 7, minY: 5, maxY: 7 }], - }; + const params: TilesDeletionParams = buildBaseParams({ tilesPath, sourceProvider: backend.sourceProvider }); const target = tilePathsForRanges(params); const extras = [...extraTilePathsAroundRanges(params, 2), ...extraTilePathsAtAdjacentZooms(params, [-1, 1])]; await backend.seed([...target, ...extras]); @@ -58,8 +54,8 @@ describe('tiles deletion E2E (polling → strategy → real provider → ack)', const { runSingleTask, queueClient, jobTrackerClient } = buildPoller({ ...storageContext, task }); await runSingleTask(); - const remaining = await backend.list(`${tilesPath}/`); + expect(remaining.sort()).toEqual([...extras].sort()); expect(queueClient.ack).toHaveBeenCalledWith(task.jobId, task.id); expect(queueClient.reject).not.toHaveBeenCalled();