diff --git a/.changeset/gold-doodles-ask.md b/.changeset/gold-doodles-ask.md new file mode 100644 index 0000000000..6c88f4957e --- /dev/null +++ b/.changeset/gold-doodles-ask.md @@ -0,0 +1,6 @@ +--- +"@human-protocol/sdk": major +"@human-protocol/python-sdk": major +--- + +Remove deprecated storage utilities and tests diff --git a/packages/apps/fortune/exchange-oracle/server/package.json b/packages/apps/fortune/exchange-oracle/server/package.json index cb840b8a31..74e23e701c 100644 --- a/packages/apps/fortune/exchange-oracle/server/package.json +++ b/packages/apps/fortune/exchange-oracle/server/package.json @@ -50,7 +50,7 @@ "ethers": "~6.15.0", "joi": "^17.13.3", "jsonwebtoken": "^9.0.2", - "minio": "7.1.3", + "minio": "8.0.6", "passport": "^0.7.0", "passport-jwt": "^4.0.1", "pg": "8.13.1", diff --git a/packages/apps/fortune/exchange-oracle/server/scripts/setup-kv-store.ts b/packages/apps/fortune/exchange-oracle/server/scripts/setup-kv-store.ts index d28014975b..21c9c9a9bf 100644 --- a/packages/apps/fortune/exchange-oracle/server/scripts/setup-kv-store.ts +++ b/packages/apps/fortune/exchange-oracle/server/scripts/setup-kv-store.ts @@ -73,7 +73,7 @@ async function setupPublicKeyFile( throw new Error('Bucket does not exists'); } - await minioClient.putObject(s3Bucket, keyName, publicKey, { + await minioClient.putObject(s3Bucket, keyName, publicKey, undefined, { 'Content-Type': 'text/plain', 'Cache-Control': 'no-store', }); diff --git a/packages/apps/fortune/exchange-oracle/server/src/common/guards/strategy/jwt.http.ts b/packages/apps/fortune/exchange-oracle/server/src/common/guards/strategy/jwt.http.ts index 7f948fddc8..7c99af6b7a 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/common/guards/strategy/jwt.http.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/common/guards/strategy/jwt.http.ts @@ -3,7 +3,7 @@ import { PassportStrategy } from '@nestjs/passport'; import { ExtractJwt, Strategy } from 'passport-jwt'; import { KVStore__factory } from '@human-protocol/core/typechain-types'; -import { ChainId, NETWORKS, StorageClient } from '@human-protocol/sdk'; +import { ChainId, NETWORKS } from '@human-protocol/sdk'; import { ethers } from 'ethers'; import * as jwt from 'jsonwebtoken'; import { JWT_KVSTORE_KEY, KYC_APPROVED } from '../../../common/constant'; @@ -11,6 +11,7 @@ import { Role } from '../../../common/enums/role'; import { JwtUser } from '../../../common/types/jwt'; import { Web3Service } from '../../../modules/web3/web3.service'; import { AuthError, ValidationError } from '../../errors'; +import { downloadFileFromUrl } from '../../utils/storage'; @Injectable() export class JwtHttpStrategy extends PassportStrategy(Strategy, 'jwt-http') { @@ -47,9 +48,7 @@ export class JwtHttpStrategy extends PassportStrategy(Strategy, 'jwt-http') { address, JWT_KVSTORE_KEY, ); - publicKey = (await StorageClient.downloadFileFromUrl( - url, - )) as string; + publicKey = (await downloadFileFromUrl(url)) as string; this.publicKeyCache.set(cacheKey, { value: publicKey, diff --git a/packages/apps/fortune/exchange-oracle/server/src/common/utils/storage.ts b/packages/apps/fortune/exchange-oracle/server/src/common/utils/storage.ts new file mode 100644 index 0000000000..104070d7af --- /dev/null +++ b/packages/apps/fortune/exchange-oracle/server/src/common/utils/storage.ts @@ -0,0 +1,33 @@ +import axios from 'axios'; +import { HttpStatus } from '@nestjs/common'; + +export const isValidUrl = (maybeUrl: string): boolean => { + try { + const { protocol } = new URL(maybeUrl); + return protocol === 'http:' || protocol === 'https:'; + } catch { + return false; + } +}; + +export async function downloadFileFromUrl(url: string): Promise { + if (!isValidUrl(url)) { + throw new Error('Invalid URL string'); + } + + try { + const { data, status } = await axios.get(url, { + headers: { + 'Content-Type': 'application/json', + }, + }); + + if (status !== HttpStatus.OK) { + throw new Error('Storage file not found'); + } + + return data; + } catch { + throw new Error('Storage file not found'); + } +} diff --git a/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.spec.ts b/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.spec.ts index f3192d8661..d39c5d9dd6 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.spec.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.spec.ts @@ -1,13 +1,7 @@ import { createMock } from '@golevelup/ts-jest'; import { HMToken__factory } from '@human-protocol/core/typechain-types'; -import { - Encryption, - EscrowClient, - OperatorUtils, - StorageClient, -} from '@human-protocol/sdk'; +import { Encryption, EscrowClient, OperatorUtils } from '@human-protocol/sdk'; import { HttpService } from '@nestjs/axios'; -import { BadRequestException, NotFoundException } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { Test } from '@nestjs/testing'; import { of } from 'rxjs'; @@ -31,6 +25,7 @@ import { ServerError, ValidationError, } from '../../common/errors'; +import { downloadFileFromUrl } from '../../common/utils/storage'; import { AssignmentEntity } from '../assignment/assignment.entity'; import { AssignmentRepository } from '../assignment/assignment.repository'; import { StorageService } from '../storage/storage.service'; @@ -50,13 +45,14 @@ jest.mock('@human-protocol/sdk', () => ({ OperatorUtils: { getOperator: jest.fn(), }, - StorageClient: { - downloadFileFromUrl: jest.fn(), - }, Encryption: { build: jest.fn(), }, })); +jest.mock('../../common/utils/storage', () => ({ + ...jest.requireActual('../../common/utils/storage'), + downloadFileFromUrl: jest.fn(), +})); jest.mock('minio', () => { class Client { putObject = jest.fn(); @@ -447,6 +443,7 @@ describe('JobService', () => { }); describe('solveJob', () => { + const downloadFileFromUrlMock = jest.mocked(downloadFileFromUrl); const assignment = { id: 1, jobId: 1, @@ -485,9 +482,7 @@ describe('JobService', () => { storageService.downloadJobSolutions = jest.fn().mockResolvedValueOnce([]); - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValueOnce(manifest); + downloadFileFromUrlMock.mockResolvedValueOnce(manifest); const solutionsUrl = 'http://localhost:9000/solution/0x1234567890123456789012345678901234567890-1.json'; @@ -552,9 +547,7 @@ describe('JobService', () => { }, ]); - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValueOnce(manifest); + downloadFileFromUrlMock.mockResolvedValueOnce(manifest); (Encryption.build as any).mockImplementation(() => ({ decrypt: jest.fn().mockResolvedValue(JSON.stringify(manifest)), @@ -587,9 +580,7 @@ describe('JobService', () => { }, ]); - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValueOnce(manifest); + downloadFileFromUrlMock.mockResolvedValueOnce(manifest); (Encryption.build as any).mockImplementation(() => ({ decrypt: jest.fn().mockResolvedValue(JSON.stringify(manifest)), diff --git a/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.ts b/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.ts index 40bbae2104..27646027f3 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.ts @@ -7,10 +7,10 @@ import { Encryption, EncryptionUtils, EscrowClient, - StorageClient, } from '@human-protocol/sdk'; import { Inject, Injectable } from '@nestjs/common'; +import { downloadFileFromUrl } from '../../common/utils/storage'; import { PGPConfigService } from '../../common/config/pgp-config.service'; import { ErrorAssignment, ErrorJob } from '../../common/constant/errors'; import { SortDirection } from '../../common/enums/collection'; @@ -348,8 +348,7 @@ export class JobService { let manifest: ManifestDto | null = null; try { - const manifestEncrypted = - await StorageClient.downloadFileFromUrl(manifestUrl); + const manifestEncrypted = await downloadFileFromUrl(manifestUrl); if ( typeof manifestEncrypted === 'string' && diff --git a/packages/apps/fortune/exchange-oracle/server/src/modules/storage/storage.service.spec.ts b/packages/apps/fortune/exchange-oracle/server/src/modules/storage/storage.service.spec.ts index a8d561b7b9..0f35c5e1bb 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/modules/storage/storage.service.spec.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/modules/storage/storage.service.spec.ts @@ -2,7 +2,6 @@ import { ChainId, Encryption, EncryptionUtils, - StorageClient, EscrowClient, KVStoreUtils, } from '@human-protocol/sdk'; @@ -13,12 +12,10 @@ import { Web3Service } from '../web3/web3.service'; import { ConfigService } from '@nestjs/config'; import { S3ConfigService } from '../../common/config/s3-config.service'; import { PGPConfigService } from '../../common/config/pgp-config.service'; +import { downloadFileFromUrl } from '../../common/utils/storage'; jest.mock('@human-protocol/sdk', () => ({ ...jest.requireActual('@human-protocol/sdk'), - StorageClient: { - downloadFileFromUrl: jest.fn(), - }, Encryption: { build: jest.fn(), }, @@ -33,6 +30,11 @@ jest.mock('@human-protocol/sdk', () => ({ }, })); +jest.mock('../../common/utils/storage', () => ({ + ...jest.requireActual('../../common/utils/storage'), + downloadFileFromUrl: jest.fn(), +})); + jest.mock('minio', () => { class Client { putObject = jest.fn(); @@ -125,6 +127,7 @@ describe('StorageService', () => { s3ConfigService.bucket, `${escrowAddress}-${chainId}.json`, 'encrypted', + undefined, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', @@ -159,6 +162,7 @@ describe('StorageService', () => { s3ConfigService.bucket, `${escrowAddress}-${chainId}.json`, 'encrypted', + undefined, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', @@ -239,6 +243,8 @@ describe('StorageService', () => { }); describe('downloadJobSolutions', () => { + const downloadFileFromUrlMock = jest.mocked(downloadFileFromUrl); + it('should download the encrypted file correctly', async () => { const workerAddress = '0x1234567890123456789012345678901234567891'; const escrowAddress = '0x1234567890123456789012345678901234567890'; @@ -252,9 +258,7 @@ describe('StorageService', () => { }, ]; - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValue('encrypted-content'); + downloadFileFromUrlMock.mockResolvedValue('encrypted-content'); EncryptionUtils.isEncrypted = jest.fn().mockReturnValue(true); @@ -282,9 +286,7 @@ describe('StorageService', () => { }, ]; - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValue(expectedJobFile); + downloadFileFromUrlMock.mockResolvedValue(expectedJobFile); EncryptionUtils.isEncrypted = jest.fn().mockReturnValue(false); @@ -299,9 +301,7 @@ describe('StorageService', () => { const escrowAddress = '0x1234567890123456789012345678901234567890'; const chainId = ChainId.LOCALHOST; - StorageClient.downloadFileFromUrl = jest - .fn() - .mockRejectedValue('Network error'); + downloadFileFromUrlMock.mockRejectedValue('Network error'); const solutionsFile = await storageService.downloadJobSolutions( escrowAddress, diff --git a/packages/apps/fortune/exchange-oracle/server/src/modules/storage/storage.service.ts b/packages/apps/fortune/exchange-oracle/server/src/modules/storage/storage.service.ts index c885e0b6b1..4259033e6e 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/modules/storage/storage.service.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/modules/storage/storage.service.ts @@ -4,11 +4,11 @@ import { EncryptionUtils, EscrowClient, KVStoreUtils, - StorageClient, } from '@human-protocol/sdk'; import { Inject, Injectable } from '@nestjs/common'; import * as Minio from 'minio'; +import { downloadFileFromUrl } from '../../common/utils/storage'; import logger from '../../logger'; import { PGPConfigService } from '../../common/config/pgp-config.service'; import { S3ConfigService } from '../../common/config/s3-config.service'; @@ -49,7 +49,7 @@ export class StorageService { ): Promise { const url = this.getJobUrl(escrowAddress, chainId); try { - const fileContent = await StorageClient.downloadFileFromUrl(url); + const fileContent = await downloadFileFromUrl(url); if (EncryptionUtils.isEncrypted(fileContent)) { const encryption = await Encryption.build( this.pgpConfigService.privateKey!, @@ -120,6 +120,7 @@ export class StorageService { this.s3ConfigService.bucket, `${escrowAddress}-${chainId}.json`, fileToUpload, + undefined, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', diff --git a/packages/apps/fortune/recording-oracle/package.json b/packages/apps/fortune/recording-oracle/package.json index ae59f2e576..fb17499389 100644 --- a/packages/apps/fortune/recording-oracle/package.json +++ b/packages/apps/fortune/recording-oracle/package.json @@ -38,7 +38,7 @@ "dotenv": "^17.2.2", "helmet": "^7.1.0", "joi": "^17.13.3", - "minio": "7.1.3", + "minio": "8.0.6", "reflect-metadata": "^0.2.2", "rxjs": "^7.2.0" }, diff --git a/packages/apps/fortune/recording-oracle/scripts/setup-kv-store.ts b/packages/apps/fortune/recording-oracle/scripts/setup-kv-store.ts index 211d863379..0e4e9051cb 100644 --- a/packages/apps/fortune/recording-oracle/scripts/setup-kv-store.ts +++ b/packages/apps/fortune/recording-oracle/scripts/setup-kv-store.ts @@ -74,7 +74,7 @@ async function setupPublicKeyFile( throw new Error('Bucket does not exists'); } - await minioClient.putObject(s3Bucket, keyName, publicKey, { + await minioClient.putObject(s3Bucket, keyName, publicKey, undefined, { 'Content-Type': 'text/plain', 'Cache-Control': 'no-store', }); diff --git a/packages/apps/fortune/recording-oracle/src/common/utils/storage.ts b/packages/apps/fortune/recording-oracle/src/common/utils/storage.ts new file mode 100644 index 0000000000..104070d7af --- /dev/null +++ b/packages/apps/fortune/recording-oracle/src/common/utils/storage.ts @@ -0,0 +1,33 @@ +import axios from 'axios'; +import { HttpStatus } from '@nestjs/common'; + +export const isValidUrl = (maybeUrl: string): boolean => { + try { + const { protocol } = new URL(maybeUrl); + return protocol === 'http:' || protocol === 'https:'; + } catch { + return false; + } +}; + +export async function downloadFileFromUrl(url: string): Promise { + if (!isValidUrl(url)) { + throw new Error('Invalid URL string'); + } + + try { + const { data, status } = await axios.get(url, { + headers: { + 'Content-Type': 'application/json', + }, + }); + + if (status !== HttpStatus.OK) { + throw new Error('Storage file not found'); + } + + return data; + } catch { + throw new Error('Storage file not found'); + } +} diff --git a/packages/apps/fortune/recording-oracle/src/modules/job/job.service.spec.ts b/packages/apps/fortune/recording-oracle/src/modules/job/job.service.spec.ts index 22455b4a07..e9094e4d21 100644 --- a/packages/apps/fortune/recording-oracle/src/modules/job/job.service.spec.ts +++ b/packages/apps/fortune/recording-oracle/src/modules/job/job.service.spec.ts @@ -5,7 +5,6 @@ import { EscrowClient, EscrowStatus, KVStoreUtils, - StorageClient, } from '@human-protocol/sdk'; import { HttpService } from '@nestjs/axios'; import { ConfigService } from '@nestjs/config'; @@ -36,6 +35,7 @@ import { Web3Service } from '../web3/web3.service'; import { WebhookDto } from '../webhook/webhook.dto'; import { JobService } from './job.service'; import { HMToken__factory } from '@human-protocol/core/typechain-types'; +import { downloadFileFromUrl } from '@/common/utils/storage'; jest.mock('minio', () => { class Client { @@ -51,19 +51,16 @@ jest.mock('minio', () => { return { Client }; }); +jest.mock('@/common/utils/storage', () => ({ + ...jest.requireActual('@/common/utils/storage'), + downloadFileFromUrl: jest.fn(), +})); + jest.mock('@human-protocol/sdk', () => ({ ...jest.requireActual('@human-protocol/sdk'), EscrowClient: { build: jest.fn().mockImplementation(() => ({})), }, - StorageClient: jest.fn().mockImplementation(() => ({ - downloadFileFromUrl: jest.fn().mockResolvedValue( - JSON.stringify({ - submissionsRequired: 3, - requestType: JobRequestType.FORTUNE, - }), - ), - })), KVStoreUtils: { get: jest.fn(), getPublicKey: jest.fn().mockResolvedValue('publicKey'), @@ -75,6 +72,7 @@ jest.mock('@human-protocol/sdk', () => ({ describe('JobService', () => { let jobService: JobService; + const downloadFileFromUrlMock = jest.mocked(downloadFileFromUrl); jest .spyOn(Web3ConfigService.prototype, 'privateKey', 'get') @@ -193,9 +191,9 @@ describe('JobService', () => { getManifest: jest.fn().mockResolvedValue('http://example.com/manifest'), }; (EscrowClient.build as jest.Mock).mockResolvedValue(escrowClient); - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValue(JSON.stringify(invalidManifest)); + downloadFileFromUrlMock.mockResolvedValue( + JSON.stringify(invalidManifest), + ); const jobSolution: WebhookDto = { escrowAddress: MOCK_ADDRESS, @@ -221,9 +219,9 @@ describe('JobService', () => { getManifest: jest.fn().mockResolvedValue('http://example.com/manifest'), }; (EscrowClient.build as jest.Mock).mockResolvedValue(escrowClient); - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValue(JSON.stringify(invalidManifest)); + downloadFileFromUrlMock.mockResolvedValue( + JSON.stringify(invalidManifest), + ); EncryptionUtils.isEncrypted = jest.fn().mockReturnValueOnce(false); const jobSolution: WebhookDto = { @@ -283,8 +281,7 @@ describe('JobService', () => { }, ]; - StorageClient.downloadFileFromUrl = jest - .fn() + downloadFileFromUrlMock .mockResolvedValueOnce(JSON.stringify(manifest)) .mockResolvedValueOnce(JSON.stringify(existingJobSolutions)) .mockResolvedValue(JSON.stringify(exchangeJobSolutions)); @@ -341,8 +338,7 @@ describe('JobService', () => { }, ]; - StorageClient.downloadFileFromUrl = jest - .fn() + downloadFileFromUrlMock .mockResolvedValueOnce(JSON.stringify(manifest)) .mockResolvedValueOnce(JSON.stringify(existingJobSolutions)) .mockResolvedValue(JSON.stringify(exchangeJobSolutions)); @@ -403,8 +399,7 @@ describe('JobService', () => { }, ]; - StorageClient.downloadFileFromUrl = jest - .fn() + downloadFileFromUrlMock .mockResolvedValueOnce(JSON.stringify(manifest)) .mockResolvedValueOnce(JSON.stringify(existingJobSolutions)) .mockResolvedValue(JSON.stringify(exchangeJobSolutions)); @@ -464,8 +459,7 @@ describe('JobService', () => { }, ]; - StorageClient.downloadFileFromUrl = jest - .fn() + downloadFileFromUrlMock .mockResolvedValueOnce(JSON.stringify(manifest)) .mockResolvedValueOnce(JSON.stringify(existingJobSolutions)) .mockResolvedValue(JSON.stringify(exchangeJobSolutions)); @@ -555,8 +549,7 @@ describe('JobService', () => { solution: 'Solution 4', }, ]; - StorageClient.downloadFileFromUrl = jest - .fn() + downloadFileFromUrlMock .mockResolvedValueOnce(JSON.stringify(manifest)) .mockResolvedValueOnce(JSON.stringify(existingJobSolutions)) .mockResolvedValue(JSON.stringify(exchangeJobSolutions)); @@ -633,8 +626,7 @@ describe('JobService', () => { solution: 'Solution 2', }, ]; - StorageClient.downloadFileFromUrl = jest - .fn() + downloadFileFromUrlMock .mockResolvedValueOnce(JSON.stringify(manifest)) .mockResolvedValueOnce(JSON.stringify(existingJobSolutions)) .mockResolvedValue(JSON.stringify(exchangeJobSolutions)); @@ -716,8 +708,7 @@ describe('JobService', () => { solution: 'ass', }, ]; - StorageClient.downloadFileFromUrl = jest - .fn() + downloadFileFromUrlMock .mockResolvedValueOnce(JSON.stringify(manifest)) .mockResolvedValueOnce(JSON.stringify(existingJobSolutions)) .mockResolvedValue(JSON.stringify(exchangeJobSolutions)); diff --git a/packages/apps/fortune/recording-oracle/src/modules/storage/storage.service.spec.ts b/packages/apps/fortune/recording-oracle/src/modules/storage/storage.service.spec.ts index 855cd7911c..ad41fd39f9 100644 --- a/packages/apps/fortune/recording-oracle/src/modules/storage/storage.service.spec.ts +++ b/packages/apps/fortune/recording-oracle/src/modules/storage/storage.service.spec.ts @@ -4,7 +4,6 @@ import { EncryptionUtils, EscrowClient, KVStoreUtils, - StorageClient, } from '@human-protocol/sdk'; import { ConfigService } from '@nestjs/config'; import { Test } from '@nestjs/testing'; @@ -17,12 +16,10 @@ import { PGPConfigService } from '../../common/config/pgp-config.service'; import { S3ConfigService } from '../../common/config/s3-config.service'; import { Web3Service } from '../web3/web3.service'; import { StorageService } from './storage.service'; +import { downloadFileFromUrl } from '@/common/utils/storage'; jest.mock('@human-protocol/sdk', () => ({ ...jest.requireActual('@human-protocol/sdk'), - StorageClient: { - downloadFileFromUrl: jest.fn(), - }, Encryption: { build: jest.fn(), }, @@ -37,6 +34,11 @@ jest.mock('@human-protocol/sdk', () => ({ }, })); +jest.mock('@/common/utils/storage', () => ({ + ...jest.requireActual('@/common/utils/storage'), + downloadFileFromUrl: jest.fn(), +})); + jest.mock('minio', () => { class Client { putObject = jest.fn(); @@ -133,6 +135,7 @@ describe('StorageService', () => { s3ConfigService.bucket, `${hash}.json`, 'encrypted', + undefined, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', @@ -211,6 +214,7 @@ describe('StorageService', () => { }); describe('download', () => { + const downloadFileFromUrlMock = jest.mocked(downloadFileFromUrl); it('should download the non encrypted file correctly', async () => { const exchangeAddress = '0x1234567890123456789012345678901234567892'; const workerAddress = '0x1234567890123456789012345678901234567891'; @@ -226,9 +230,7 @@ describe('StorageService', () => { ], }; - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValue(expectedJobFile); + downloadFileFromUrlMock.mockResolvedValue(expectedJobFile); EncryptionUtils.isEncrypted = jest.fn().mockReturnValue(false); const solutionsFile = await storageService.download(MOCK_FILE_URL); expect(solutionsFile).toStrictEqual(expectedJobFile); @@ -249,9 +251,7 @@ describe('StorageService', () => { ], }; - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValue('encrypted-content'); + downloadFileFromUrlMock.mockResolvedValue('encrypted-content'); Encryption.build = jest.fn().mockResolvedValue({ decrypt: jest.fn().mockResolvedValue(JSON.stringify(expectedJobFile)), @@ -262,9 +262,7 @@ describe('StorageService', () => { }); it('should return empty array when file cannot be downloaded', async () => { - StorageClient.downloadFileFromUrl = jest - .fn() - .mockRejectedValue('Network error'); + downloadFileFromUrlMock.mockRejectedValue('Network error'); const solutionsFile = await storageService.download(MOCK_FILE_URL); expect(solutionsFile).toStrictEqual([]); diff --git a/packages/apps/fortune/recording-oracle/src/modules/storage/storage.service.ts b/packages/apps/fortune/recording-oracle/src/modules/storage/storage.service.ts index d2abeb58c8..330bc072b3 100644 --- a/packages/apps/fortune/recording-oracle/src/modules/storage/storage.service.ts +++ b/packages/apps/fortune/recording-oracle/src/modules/storage/storage.service.ts @@ -4,7 +4,6 @@ import { EncryptionUtils, EscrowClient, KVStoreUtils, - StorageClient, } from '@human-protocol/sdk'; import { Inject, Injectable } from '@nestjs/common'; import crypto from 'crypto'; @@ -15,6 +14,7 @@ import { ServerError, ValidationError } from '../../common/errors'; import { ISolution } from '../../common/interfaces/job'; import { SaveSolutionsDto } from '../job/job.dto'; import { Web3Service } from '../web3/web3.service'; +import { downloadFileFromUrl } from '../../common/utils/storage'; @Injectable() export class StorageService { @@ -42,7 +42,7 @@ export class StorageService { public async download(url: string): Promise { try { - const fileContent = await StorageClient.downloadFileFromUrl(url); + const fileContent = await downloadFileFromUrl(url); if ( typeof fileContent === 'string' && @@ -126,6 +126,7 @@ export class StorageService { this.s3ConfigService.bucket, `${hash}.json`, fileToUpload, + undefined, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', diff --git a/packages/apps/human-app/server/src/common/utils/storage.ts b/packages/apps/human-app/server/src/common/utils/storage.ts new file mode 100644 index 0000000000..104070d7af --- /dev/null +++ b/packages/apps/human-app/server/src/common/utils/storage.ts @@ -0,0 +1,33 @@ +import axios from 'axios'; +import { HttpStatus } from '@nestjs/common'; + +export const isValidUrl = (maybeUrl: string): boolean => { + try { + const { protocol } = new URL(maybeUrl); + return protocol === 'http:' || protocol === 'https:'; + } catch { + return false; + } +}; + +export async function downloadFileFromUrl(url: string): Promise { + if (!isValidUrl(url)) { + throw new Error('Invalid URL string'); + } + + try { + const { data, status } = await axios.get(url, { + headers: { + 'Content-Type': 'application/json', + }, + }); + + if (status !== HttpStatus.OK) { + throw new Error('Storage file not found'); + } + + return data; + } catch { + throw new Error('Storage file not found'); + } +} diff --git a/packages/apps/human-app/server/src/integrations/kv-store/kv-store.gateway.ts b/packages/apps/human-app/server/src/integrations/kv-store/kv-store.gateway.ts index d57967369f..6b513c8047 100644 --- a/packages/apps/human-app/server/src/integrations/kv-store/kv-store.gateway.ts +++ b/packages/apps/human-app/server/src/integrations/kv-store/kv-store.gateway.ts @@ -1,9 +1,4 @@ -import { - ChainId, - KVStoreKeys, - KVStoreUtils, - StorageClient, -} from '@human-protocol/sdk'; +import { ChainId, KVStoreKeys, KVStoreUtils } from '@human-protocol/sdk'; import { CACHE_MANAGER } from '@nestjs/cache-manager'; import { HttpException, Inject, Injectable } from '@nestjs/common'; import { Cache } from 'cache-manager'; @@ -14,6 +9,7 @@ import { ORACLE_URL_CACHE_KEY, REPUTATION_ORACLE_PUBLIC_KEY, } from '../../common/constants/cache'; +import { downloadFileFromUrl } from '../../common/utils/storage'; @Injectable() export class KvStoreGateway { @@ -119,7 +115,7 @@ export class KvStoreGateway { address, JWT_KVSTORE_KEY, ); - publicKey = (await StorageClient.downloadFileFromUrl(url)) as string; + publicKey = (await downloadFileFromUrl(url)) as string; } catch (e) { if (e.toString().includes('Error: Invalid address')) { throw new HttpException( diff --git a/packages/apps/job-launcher/server/package.json b/packages/apps/job-launcher/server/package.json index c0b6502cab..5bf4ee45c4 100644 --- a/packages/apps/job-launcher/server/package.json +++ b/packages/apps/job-launcher/server/package.json @@ -59,7 +59,7 @@ "helmet": "^7.1.0", "joi": "^17.13.3", "json-stable-stringify": "^1.2.1", - "minio": "7.1.3", + "minio": "8.0.6", "nestjs-minio-client": "^2.2.0", "node-cache": "^5.1.2", "passport": "^0.7.0", diff --git a/packages/apps/job-launcher/server/scripts/setup-kv-store.ts b/packages/apps/job-launcher/server/scripts/setup-kv-store.ts index 63a790a62c..6a07297935 100644 --- a/packages/apps/job-launcher/server/scripts/setup-kv-store.ts +++ b/packages/apps/job-launcher/server/scripts/setup-kv-store.ts @@ -80,7 +80,7 @@ async function setupPublicKeyFile( throw new Error('Bucket does not exists'); } - await minioClient.putObject(s3Bucket, keyName, publicKey, { + await minioClient.putObject(s3Bucket, keyName, publicKey, undefined, { 'Content-Type': 'text/plain', 'Cache-Control': 'no-store', }); diff --git a/packages/apps/job-launcher/server/src/modules/job/job.service.ts b/packages/apps/job-launcher/server/src/modules/job/job.service.ts index a28d3cca0c..723d818338 100644 --- a/packages/apps/job-launcher/server/src/modules/job/job.service.ts +++ b/packages/apps/job-launcher/server/src/modules/job/job.service.ts @@ -7,7 +7,6 @@ import { KVStoreKeys, KVStoreUtils, NETWORKS, - StorageParams, } from '@human-protocol/sdk'; import { Inject, Injectable } from '@nestjs/common'; import { ModuleRef } from '@nestjs/core'; @@ -85,7 +84,6 @@ import { Escrow, Escrow__factory } from '@human-protocol/core/typechain-types'; @Injectable() export class JobService { private readonly logger = logger.child({ context: JobService.name }); - public readonly storageParams: StorageParams; public readonly bucket: string; private cronJobRepository: CronJobRepository; diff --git a/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.ts b/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.ts index 53d73cf339..5399c0be61 100644 --- a/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.ts +++ b/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.ts @@ -1,9 +1,4 @@ -import { - ChainId, - Encryption, - KVStoreUtils, - StorageParams, -} from '@human-protocol/sdk'; +import { ChainId, Encryption, KVStoreUtils } from '@human-protocol/sdk'; import { ValidationError as ClassValidationError, Injectable, @@ -72,7 +67,6 @@ import { @Injectable() export class ManifestService { - public readonly storageParams: StorageParams; public readonly bucket: string; constructor( diff --git a/packages/apps/job-launcher/server/src/modules/storage/storage.service.ts b/packages/apps/job-launcher/server/src/modules/storage/storage.service.ts index 523298f568..37f3558830 100644 --- a/packages/apps/job-launcher/server/src/modules/storage/storage.service.ts +++ b/packages/apps/job-launcher/server/src/modules/storage/storage.service.ts @@ -127,6 +127,7 @@ export class StorageService { this.s3ConfigService.bucket, fileKey, fileContents, + undefined, { 'Content-Type': contentType, 'Cache-Control': 'no-store', diff --git a/packages/apps/reputation-oracle/server/package.json b/packages/apps/reputation-oracle/server/package.json index 7d494794ba..9a8dcf6733 100644 --- a/packages/apps/reputation-oracle/server/package.json +++ b/packages/apps/reputation-oracle/server/package.json @@ -59,7 +59,7 @@ "joi": "^17.13.3", "json-stable-stringify": "^1.2.1", "lodash": "^4.17.21", - "minio": "7.1.3", + "minio": "8.0.6", "passport": "^0.7.0", "passport-jwt": "^4.0.1", "pg": "8.13.1", diff --git a/packages/apps/reputation-oracle/server/scripts/setup-kv-store.ts b/packages/apps/reputation-oracle/server/scripts/setup-kv-store.ts index 5165db585c..45ded5aea5 100644 --- a/packages/apps/reputation-oracle/server/scripts/setup-kv-store.ts +++ b/packages/apps/reputation-oracle/server/scripts/setup-kv-store.ts @@ -80,7 +80,7 @@ async function setupPublicKeyFile( throw new Error('Bucket does not exists'); } - await minioClient.putObject(s3Bucket, keyName, publicKey, { + await minioClient.putObject(s3Bucket, keyName, publicKey, undefined, { 'Content-Type': 'text/plain', 'Cache-Control': 'no-store', }); diff --git a/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts b/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts index 105b797ee2..6103f45ee0 100644 --- a/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts +++ b/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts @@ -108,6 +108,7 @@ export class StorageService { this.s3ConfigService.bucket, fileName, content, + undefined, { 'Content-Type': contentType, 'Cache-Control': 'no-store', diff --git a/packages/sdk/python/human-protocol-sdk/Pipfile b/packages/sdk/python/human-protocol-sdk/Pipfile index b78efee08b..ae9b732d16 100644 --- a/packages/sdk/python/human-protocol-sdk/Pipfile +++ b/packages/sdk/python/human-protocol-sdk/Pipfile @@ -17,7 +17,6 @@ sphinx-autodoc-typehints = "*" [packages] cryptography = "*" -minio = "*" validators = "*" web3 = "*" aiohttp = "<4.0.0" # broken freeze in one of dependencies diff --git a/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage/__init__.py b/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage/__init__.py deleted file mode 100644 index d995f00412..0000000000 --- a/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -This modules contains an s3 client and utilities for files sharing. -""" - -from .storage_client import ( - StorageClient, - StorageClientError, - StorageFileNotFoundError, - Credentials, -) -from .storage_utils import StorageUtils diff --git a/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage/storage_client.py b/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage/storage_client.py deleted file mode 100644 index 47b2dec081..0000000000 --- a/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage/storage_client.py +++ /dev/null @@ -1,379 +0,0 @@ -""" -This client enables to interact with S3 cloud storage services like Amazon S3 Bucket, -Google Cloud Storage and others. - -If credentials are not provided, anonymous access will be used (for downloading files). - -Code Example ------------- - -.. code-block:: python - - from human_protocol_sdk.storage import ( - Credentials, - StorageClient, - ) - - credentials = Credentials( - access_key="my-access-key", - secret_key="my-secret-key", - ) - - storage_client = StorageClient( - endpoint_url="s3.us-west-2.amazonaws.com", - region="us-west-2", - credentials=credentials, - ) - -Module ------- -""" - -import hashlib -import io -import json -import logging -import os -from typing import List, Optional -from warnings import warn - -from minio import Minio - -logging.getLogger("minio").setLevel(logging.INFO) - -DEBUG = "true" in os.getenv("DEBUG", "false").lower() -LOG = logging.getLogger("human_protocol_sdk.storage") -LOG.setLevel(logging.DEBUG if DEBUG else logging.INFO) - - -warn(f"The module {__name__} is deprecated.", DeprecationWarning, stacklevel=2) - - -class StorageClientError(Exception): - """ - Raises when some error happens when interacting with storage. - """ - - pass - - -class StorageFileNotFoundError(StorageClientError): - """ - Raises when some error happens when file is not found by its key. - """ - - pass - - -class Credentials: - """ - A class to represent the credentials required to authenticate with an S3-compatible service. - - Example:: - - credentials = Credentials( - access_key='my-access-key', - secret_key='my-secret-key' - ) - - """ - - def __init__(self, access_key: str, secret_key: str): - """ - Initializes a Credentials instance. - - :param access_key: The access key for the S3-compatible service. - :param secret_key: The secret key for the S3-compatible service. - """ - - self.access_key = access_key - self.secret_key = secret_key - - -class StorageClient: - """ - A class for downloading files from an S3-compatible service. - - :attribute: - - client (Minio): The S3-compatible client used for interacting with the service. - - :example: - .. code-block:: python - - # Download a list of files from an S3-compatible service - client = StorageClient( - endpoint_url='https://s3.us-west-2.amazonaws.com', - region='us-west-2', - credentials=Credentials( - access_key='my-access-key', - secret_key='my-secret-key' - ) - ) - files = ['file1.txt', 'file2.txt'] - bucket = 'my-bucket' - result_files = client.download_files(files=files, bucket=bucket) - - """ - - def __init__( - self, - endpoint_url: str, - region: Optional[str] = None, - credentials: Optional[Credentials] = None, - secure: Optional[bool] = True, - ): - """ - Initializes the StorageClient with the given endpoint_url, region, and credentials. - - If credentials are not provided, anonymous access will be used. - - :param endpoint_url: The URL of the S3-compatible service. - :param region: The region of the S3-compatible service. Defaults to None. - :param credentials: The credentials required to authenticate with the S3-compatible service. - Defaults to None for anonymous access. - :param secure: Flag to indicate to use secure (TLS) connection to S3 service or not. - Defaults to True. - """ - try: - self.client = ( - Minio( - region=region, - endpoint=endpoint_url, - secure=secure, - ) # anonymous access - if credentials is None - else Minio( - access_key=credentials.access_key, - secret_key=credentials.secret_key, - region=region, - endpoint=endpoint_url, - secure=secure, - ) # authenticated access - ) - self.endpoint = endpoint_url - self.secure = secure - except Exception as e: - LOG.error(f"Connection with S3 failed because of: {e}") - raise e - - def download_files(self, files: List[str], bucket: str) -> List[bytes]: - """ - Downloads a list of files from the specified S3-compatible bucket. - - :param files: A list of file keys to download. - :param bucket: The name of the S3-compatible bucket to download from. - - :return: A list of file contents (bytes) downloaded from the bucket. - - :raise StorageClientError: If an error occurs while downloading the files. - :raise StorageFileNotFoundError: If one of the specified files is not found in the bucket. - - :example: - .. code-block:: python - - from human_protocol_sdk.storage import ( - Credentials, - StorageClient, - ) - - credentials = Credentials( - access_key="my-access-key", - secret_key="my-secret-key", - ) - - storage_client = StorageClient( - endpoint_url="s3.us-west-2.amazonaws.com", - region="us-west-2", - credentials=credentials, - ) - - result = storage_client.download_files( - files = ["file1.txt", "file2.txt"], - bucket = "my-bucket" - ) - """ - result_files = [] - for file in files: - try: - response = self.client.get_object(bucket_name=bucket, object_name=file) - result_files.append(response.read()) - except Exception as e: - if hasattr(e, "code") and str(e.code) == "NoSuchKey": - raise StorageFileNotFoundError("No object found - returning empty") - LOG.warning( - f"Reading the key {file} with S3 failed" f" because of: {str(e)}" - ) - raise StorageClientError(str(e)) - return result_files - - def upload_files(self, files: List[dict], bucket: str) -> List[dict]: - """ - Uploads a list of files to the specified S3-compatible bucket. - - :param files: A list of files to upload. - :param bucket: The name of the S3-compatible bucket to upload to. - - :return: List of dict with key, url, hash fields - - :raise StorageClientError: If an error occurs while uploading the files. - - :example: - .. code-block:: python - - from human_protocol_sdk.storage import ( - Credentials, - StorageClient, - ) - - credentials = Credentials( - access_key="my-access-key", - secret_key="my-secret-key", - ) - - storage_client = StorageClient( - endpoint_url="s3.us-west-2.amazonaws.com", - region="us-west-2", - credentials=credentials, - ) - - result = storage_client.upload_files( - files = [{"file": "file content", "key": "file1.txt", "hash": "hash1"}], - bucket = "my-bucket" - ) - """ - result_files = [] - for file in files: - if "file" in file and "key" in file and "hash" in file: - data = file["file"] - hash = file["hash"] - key = file["key"] - else: - try: - artifact = json.dumps(file, sort_keys=True) - except Exception as e: - LOG.error("Can't extract the json from the object") - raise e - data = artifact.encode("utf-8") - hash = hashlib.sha1(data).hexdigest() - key = f"s3{hash}.json" - - url = ( - f"{'https' if self.secure else 'http'}://{self.endpoint}/{bucket}/{key}" - ) - file_exist = None - - try: - # check if file with same hash already exists in bucket - file_exist = self.client.stat_object( - bucket_name=bucket, object_name=key - ) - except Exception as e: - if hasattr(e, "code") and str(e.code) == "NoSuchKey": - # file does not exist in bucket, so upload it - pass - else: - LOG.warning( - f"Reading the key {key} in S3 failed" f" because of: {str(e)}" - ) - raise StorageClientError(str(e)) - - if not file_exist: - # file does not exist in bucket, so upload it - try: - self.client.put_object( - bucket_name=bucket, - object_name=key, - data=io.BytesIO(data), - length=len(data), - ) - LOG.debug(f"Uploaded to S3, key: {key}") - except Exception as e: - raise StorageClientError(str(e)) - - result_files.append({"key": key, "url": url, "hash": hash}) - - return result_files - - def bucket_exists(self, bucket: str) -> bool: - """ - Check if a given bucket exists. - - :param bucket: The name of the bucket to check. - - :return: True if the bucket exists, False otherwise. - - :raise StorageClientError: If an error occurs while checking the bucket. - - :example: - .. code-block:: python - - from human_protocol_sdk.storage import ( - Credentials, - StorageClient, - ) - - credentials = Credentials( - access_key="my-access-key", - secret_key="my-secret-key", - ) - - storage_client = StorageClient( - endpoint_url="s3.us-west-2.amazonaws.com", - region="us-west-2", - credentials=credentials, - ) - - is_exists = storage_client.bucket_exists( - bucket = "my-bucket" - ) - """ - try: - return self.client.bucket_exists(bucket_name=bucket) - except Exception as e: - LOG.warning( - f"Checking the bucket {bucket} in S3 failed" f" because of: {str(e)}" - ) - raise StorageClientError(str(e)) - - def list_objects(self, bucket: str) -> List[str]: - """ - Return a list of all objects in a given bucket. - - :param bucket: The name of the bucket to list objects from. - - :return: A list of object keys in the given bucket. - - :raise StorageClientError: If an error occurs while listing the objects. - - :example: - .. code-block:: python - - from human_protocol_sdk.storage import ( - Credentials, - StorageClient, - ) - - credentials = Credentials( - access_key="my-access-key", - secret_key="my-secret-key", - ) - - storage_client = StorageClient( - endpoint_url="s3.us-west-2.amazonaws.com", - region="us-west-2", - credentials=credentials, - ) - - result = storage_client.list_objects( - bucket = "my-bucket" - ) - """ - try: - objects = list(self.client.list_objects(bucket_name=bucket)) - if objects: - return [obj._object_name for obj in objects] - else: - return [] - except Exception as e: - LOG.warning(f"Listing objects in S3 failed because of: {str(e)}") - raise StorageClientError(str(e)) diff --git a/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage/storage_utils.py b/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage/storage_utils.py deleted file mode 100644 index ee15eb2d9b..0000000000 --- a/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage/storage_utils.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Utility class for storage-related operations. -""" - -import logging -import os -from warnings import warn - -import requests - -from human_protocol_sdk.storage.storage_client import StorageClientError -from human_protocol_sdk.utils import validate_url - -logging.getLogger("minio").setLevel(logging.INFO) - -DEBUG = "true" in os.getenv("DEBUG", "false").lower() -LOG = logging.getLogger("human_protocol_sdk.storage") -LOG.setLevel(logging.DEBUG if DEBUG else logging.INFO) - - -warn(f"The module {__name__} is deprecated.", DeprecationWarning, stacklevel=2) - - -class StorageUtils: - """ - Utility class for storage-related operations. - """ - - @staticmethod - def download_file_from_url(url: str) -> bytes: - """ - Downloads a file from the specified URL. - - :param url: The URL of the file to download. - - :return: The content of the downloaded file. - - :raise StorageClientError: If an error occurs while downloading the file. - - :example: - .. code-block:: python - - from human_protocol_sdk.storage import StorageUtils - - result = StorageUtils.download_file_from_url( - "https://www.example.com/file.txt" - ) - """ - if not validate_url(url): - raise StorageClientError(f"Invalid URL: {url}") - - try: - response = requests.get(url) - response.raise_for_status() - - return response.content - except Exception as e: - raise StorageClientError(str(e)) diff --git a/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/storage/__init__.py b/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/storage/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/storage/test_storage_client.py b/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/storage/test_storage_client.py deleted file mode 100644 index e079badeb5..0000000000 --- a/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/storage/test_storage_client.py +++ /dev/null @@ -1,311 +0,0 @@ -import hashlib -import json -import random -import unittest -from unittest.mock import MagicMock, patch -import types -from minio import S3Error - -from human_protocol_sdk.storage import ( - Credentials, - StorageClient, - StorageClientError, - StorageFileNotFoundError, -) - - -class TestCredentials(unittest.TestCase): - def test_credentials(self): - credentials = Credentials( - access_key="my-access-key", secret_key="my-secret-key" - ) - self.assertEqual(credentials.access_key, "my-access-key") - self.assertEqual(credentials.secret_key, "my-secret-key") - - -class TestStorageClient(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.endpoint_url = "s3.us-west-2.amazonaws.com" - cls.bucket = "my-bucket" - cls.files = ["file1.txt", "file2.txt"] - cls.region = "us-west-2" - cls.credentials = Credentials( - access_key="my-access-key", secret_key="my-secret-key" - ) - - def setUp(self): - self.client = StorageClient( - endpoint_url=self.endpoint_url, - region=self.region, - credentials=self.credentials, - ) - - def test_init_authenticated_access(self): - with patch("human_protocol_sdk.storage.storage_client.Minio") as mock_client: - client = StorageClient( - endpoint_url=self.endpoint_url, - region=self.region, - credentials=self.credentials, - ) - mock_client.assert_called_once_with( - access_key=self.credentials.access_key, - secret_key=self.credentials.secret_key, - region=self.region, - endpoint=self.endpoint_url, - secure=True, - ) - self.assertIsNotNone(client.client) - - def test_init_anonymous_access(self): - with patch("human_protocol_sdk.storage.storage_client.Minio") as mock_client: - client = StorageClient( - endpoint_url=self.endpoint_url, - ) - mock_client.assert_called_once() - self.assertEqual(mock_client.call_args_list[0].kwargs["region"], None) - self.assertEqual( - mock_client.call_args_list[0].kwargs["endpoint"], self.endpoint_url - ) - self.assertIsNotNone(client.client) - - def test_init_error(self): - # Connection error - with patch("human_protocol_sdk.storage.storage_client.Minio") as mock_client: - mock_client.side_effect = Exception("Connection error") - with self.assertRaises(Exception): - StorageClient(endpoint_url=self.endpoint_url) - - def test_download_files(self): - expected_result = [b"file1 contents", b"file2 contents"] - self.client.client.get_object = MagicMock( - side_effect=[ - MagicMock(read=MagicMock(return_value=expected_result[0])), - MagicMock(read=MagicMock(return_value=expected_result[1])), - ] - ) - result = self.client.download_files(files=self.files, bucket=self.bucket) - self.assertEqual(result, expected_result) - - def test_download_files_error(self): - self.client.client.get_object = MagicMock( - side_effect=S3Error( - code="NoSuchKey", - message="Key not found", - resource="", - request_id="", - host_id="", - response="", - ) - ) - with self.assertRaises(StorageFileNotFoundError): - self.client.download_files(files=self.files, bucket=self.bucket) - - def test_download_files_anonymous_error(self): - self.client.client.get_object = MagicMock( - side_effect=S3Error( - code="InvalidAccessKeyId", - message="Access denied", - resource="", - request_id="", - host_id="", - response="", - ) - ) - with self.assertRaises(StorageClientError): - self.client.download_files(files=self.files, bucket=self.bucket) - - def test_download_files_exception(self): - self.client.client.get_object = MagicMock( - side_effect=Exception("Connection error") - ) - with self.assertRaises(StorageClientError): - self.client.download_files(files=self.files, bucket=self.bucket) - - def test_upload_files(self): - file3 = "file3 content" - hash = hashlib.sha1(json.dumps("file3 content").encode("utf-8")).hexdigest() - key3 = f"s3{hash}.json" - - self.client.client.stat_object = MagicMock( - side_effect=S3Error( - code="NoSuchKey", - message="Object does not exist", - resource="", - request_id="", - host_id="", - response="", - ) - ) - self.client.client.put_object = MagicMock() - result = self.client.upload_files(files=[file3], bucket=self.bucket) - self.assertEqual(result[0]["key"], key3) - self.assertEqual( - result[0]["url"], f"https://s3.us-west-2.amazonaws.com/my-bucket/{key3}" - ) - self.assertEqual(result[0]["hash"], hash) - - def test_upload_encrypted_files(self): - encrypted_file = "encrypted file content" - hash = hashlib.sha1(json.dumps(encrypted_file).encode("utf-8")).hexdigest() - encrypted_file_key = f"s3{hash}" - file = { - "file": encrypted_file.encode("utf-8"), - "hash": hash, - "key": encrypted_file_key, - } - - self.client.client.stat_object = MagicMock( - side_effect=S3Error( - code="NoSuchKey", - message="Object does not exist", - resource="", - request_id="", - host_id="", - response="", - ) - ) - self.client.client.put_object = MagicMock() - result = self.client.upload_files(files=[file], bucket=self.bucket) - self.assertEqual(result[0]["key"], encrypted_file_key) - self.assertEqual( - result[0]["url"], - f"https://s3.us-west-2.amazonaws.com/my-bucket/{encrypted_file_key}", - ) - self.assertEqual(result[0]["hash"], hash) - - def test_upload_files_exist(self): - file3 = "file3 content" - hash = hashlib.sha1(json.dumps("file3 content").encode("utf-8")).hexdigest() - key3 = f"s3{hash}.json" - - self.client.client.stat_object = MagicMock( - side_effect=[{"_object_name": "1234567890"}] - ) - self.client.client.put_object = MagicMock() - result = self.client.upload_files(files=[file3], bucket=self.bucket) - self.assertEqual(result[0]["key"], key3) - self.assertEqual( - result[0]["url"], f"https://s3.us-west-2.amazonaws.com/my-bucket/{key3}" - ) - self.assertEqual(result[0]["hash"], hash) - - def test_upload_files_error(self): - file3 = "file3 content" - - # HeadObject error - self.client.client.head_object = MagicMock( - side_effect=S3Error( - code="InvalidAccessKeyId", - message="Access denied", - resource="", - request_id="", - host_id="", - response="", - ) - ) - with self.assertRaises(StorageClientError): - self.client.upload_files(files=[file3], bucket=self.bucket) - - # PutObject error - self.client.client.upload_fileobj = MagicMock( - side_effect=S3Error( - code="InvalidAccessKeyId", - message="Access denied", - resource="", - request_id="", - host_id="", - response="", - ) - ) - with self.assertRaises(StorageClientError): - self.client.upload_files(files=[file3], bucket=self.bucket) - - def test_bucket_exists(self): - self.client.client.bucket_exists = MagicMock(side_effect=[True]) - result = self.client.bucket_exists(bucket=self.bucket) - self.assertEqual(result, True) - - def test_bucket_exists_anonymous(self): - client = StorageClient( - endpoint_url=self.endpoint_url, - ) - client.client.bucket_exists = MagicMock(side_effect=[True]) - result = client.bucket_exists(bucket=self.bucket) - self.assertEqual(result, True) - - def test_bucket_not_exists(self): - self.client.client.bucket_exists = MagicMock(side_effect=[False]) - result = self.client.bucket_exists(bucket=self.bucket) - self.assertEqual(result, False) - - def test_bucket_error(self): - self.client.client.bucket_exists = MagicMock( - side_effect=Exception("Connection error") - ) - with self.assertRaises(StorageClientError): - self.client.bucket_exists(bucket=self.bucket) - - def test_list_objects(self): - file1 = types.SimpleNamespace() - file2 = types.SimpleNamespace() - file1._object_name = "file1" - file2._object_name = "file2" - self.client.client.list_objects = MagicMock(side_effect=[[file1, file2]]) - result = self.client.list_objects(bucket=self.bucket) - self.assertEqual(result, ["file1", "file2"]) - - def test_list_objects_anonymous(self): - client = StorageClient( - endpoint_url=self.endpoint_url, - ) - file1 = types.SimpleNamespace() - file2 = types.SimpleNamespace() - file1._object_name = "file1" - file2._object_name = "file2" - client.client.list_objects = MagicMock(side_effect=[[file1, file2]]) - result = client.list_objects(bucket=self.bucket) - self.assertEqual(result, ["file1", "file2"]) - - def test_list_objects_empty(self): - self.client.client.list_objects = MagicMock(side_effect=[[]]) - result = self.client.list_objects(bucket=self.bucket) - self.assertEqual(result, []) - - def test_list_objects_error(self): - self.client.client.head_bucket = MagicMock( - side_effect=Exception("Connection error") - ) - with self.assertRaises(StorageClientError): - self.client.list_objects(bucket=self.bucket) - - def test_list_objects_length(self): - expected_length = random.randint(1, 10) - mock_client = MagicMock() - mock_client.list_objects.return_value = [ - types.SimpleNamespace(_object_name=f"file{i}") - for i in range(expected_length) - ] - with patch( - "human_protocol_sdk.storage.storage_client.Minio", return_value=mock_client - ): - client = StorageClient(endpoint_url="https://example.com", credentials=None) - object_list = client.list_objects(bucket="my-bucket") - self.assertEqual(len(object_list), expected_length) - - def test_list_objects_length_error(self): - expected_length = random.randint(1, 10) - mock_client = MagicMock() - mock_client.list_objects.return_value = [ - types.SimpleNamespace(_object_name=f"file{i}") - for i in range(expected_length) - ] - with patch( - "human_protocol_sdk.storage.storage_client.Minio", return_value=mock_client - ): - client = StorageClient(endpoint_url="https://example.com", credentials=None) - object_list = client.list_objects(bucket="my-bucket") - if len(object_list) != expected_length: - raise AssertionError( - f"Expected {expected_length} objects, but found {len(object_list)}" - ) diff --git a/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/storage/test_storage_utils.py b/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/storage/test_storage_utils.py deleted file mode 100644 index e8dab760db..0000000000 --- a/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/storage/test_storage_utils.py +++ /dev/null @@ -1,37 +0,0 @@ -import unittest -from unittest.mock import patch - -from human_protocol_sdk.storage import StorageUtils, StorageClientError -import requests - - -class TestStorageUtils(unittest.TestCase): - def test_download_file_from_url(self): - with patch("requests.get") as mock_get: - mock_response = mock_get.return_value - mock_response.raise_for_status.return_value = None - mock_response.content = b"Test file content" - url = "https://www.example.com/file.txt" - - result = StorageUtils.download_file_from_url(url) - - self.assertEqual(result, b"Test file content") - - def test_download_file_from_url_invalid_url(self): - url = "invalid_url" - - with self.assertRaises(StorageClientError) as cm: - StorageUtils.download_file_from_url(url) - self.assertEqual(f"Invalid URL: {url}", str(cm.exception)) - - def test_download_file_from_url_error(self): - with patch("requests.get") as mock_get: - url = "https://www.example.com/file.txt" - mock_response = mock_get.return_value - mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( - f"Not Found for url: {url}", response=mock_response - ) - - with self.assertRaises(StorageClientError) as cm: - StorageUtils.download_file_from_url(url) - self.assertEqual(f"Not Found for url: {url}", str(cm.exception)) diff --git a/packages/sdk/typescript/human-protocol-sdk/package.json b/packages/sdk/typescript/human-protocol-sdk/package.json index 96a26132b8..fed510c51e 100644 --- a/packages/sdk/typescript/human-protocol-sdk/package.json +++ b/packages/sdk/typescript/human-protocol-sdk/package.json @@ -44,7 +44,6 @@ "graphql": "^16.8.1", "graphql-request": "^7.3.4", "graphql-tag": "^2.12.6", - "minio": "7.1.3", "openpgp": "^6.2.2", "secp256k1": "^5.0.1", "validator": "^13.12.0", diff --git a/packages/sdk/typescript/human-protocol-sdk/src/error.ts b/packages/sdk/typescript/human-protocol-sdk/src/error.ts index 3a2d61888d..f5cbe8fd4c 100644 --- a/packages/sdk/typescript/human-protocol-sdk/src/error.ts +++ b/packages/sdk/typescript/human-protocol-sdk/src/error.ts @@ -3,42 +3,6 @@ */ export const ErrorStakingMissing = new Error('Staking contract is missing'); -/** - * @constant {Error} - The Storage client not initialized. - */ -export const ErrorStorageClientNotInitialized = new Error( - 'Storage client not initialized' -); - -/** - * @constant {Error} - The Storage client does not exist. - */ -export const ErrorStorageClientNotExists = new Error( - 'Storage client does not exist' -); - -/** - * @constant {Error} - The Storage credentials are missing. - */ -export const ErrorStorageCredentialsMissing = new Error( - 'Storage credentials are missing' -); - -/** - * @constant {Error} - The Storage bucket not found. - */ -export const ErrorStorageBucketNotFound = new Error('Bucket not found'); - -/** - * @constant {Error} - The Storage file not found. - */ -export const ErrorStorageFileNotFound = new Error('File not found'); - -/** - * @constant {Error} - The Storage file not uploaded. - */ -export const ErrorStorageFileNotUploaded = new Error('File not uploaded'); - /** * @constant {Error} - The KVStore key cannot be empty. */ diff --git a/packages/sdk/typescript/human-protocol-sdk/src/index.ts b/packages/sdk/typescript/human-protocol-sdk/src/index.ts index 047bfb6782..15a363bae2 100644 --- a/packages/sdk/typescript/human-protocol-sdk/src/index.ts +++ b/packages/sdk/typescript/human-protocol-sdk/src/index.ts @@ -1,5 +1,4 @@ import { StakingClient, StakingUtils } from './staking'; -import { StorageClient } from './storage'; import { KVStoreClient, KVStoreUtils } from './kvstore'; import { EscrowClient, EscrowUtils } from './escrow'; import { StatisticsClient } from './statistics'; @@ -27,7 +26,6 @@ export { export { StakingClient, - StorageClient, KVStoreClient, KVStoreUtils, EscrowClient, diff --git a/packages/sdk/typescript/human-protocol-sdk/src/storage.ts b/packages/sdk/typescript/human-protocol-sdk/src/storage.ts deleted file mode 100644 index 81656fcaf9..0000000000 --- a/packages/sdk/typescript/human-protocol-sdk/src/storage.ts +++ /dev/null @@ -1,313 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import axios from 'axios'; -import crypto from 'crypto'; -import * as Minio from 'minio'; -import { - ErrorInvalidUrl, - ErrorStorageBucketNotFound, - ErrorStorageClientNotInitialized, - ErrorStorageFileNotFound, - ErrorStorageFileNotUploaded, -} from './error'; -import { UploadFile, StorageCredentials, StorageParams } from './types'; -import { isValidUrl } from './utils'; -import { HttpStatus } from './constants'; - -/** - * - * @deprecated StorageClient is deprecated. Use Minio.Client directly. - * - * ## Introduction - * - * This client enables interacting with S3 cloud storage services like Amazon S3 Bucket, Google Cloud Storage, and others. - * - * The instance creation of `StorageClient` should be made using its constructor: - * - * ```ts - * constructor(params: StorageParams, credentials?: StorageCredentials) - * ``` - * - * > If credentials are not provided, it uses anonymous access to the bucket for downloading files. - * - * ## Installation - * - * ### npm - * ```bash - * npm install @human-protocol/sdk - * ``` - * - * ### yarn - * ```bash - * yarn install @human-protocol/sdk - * ``` - * - * ## Code example - * - * ```ts - * import { StorageClient, StorageCredentials, StorageParams } from '@human-protocol/sdk'; - * - * const credentials: StorageCredentials = { - * accessKey: 'ACCESS_KEY', - * secretKey: 'SECRET_KEY', - * }; - * const params: StorageParams = { - * endPoint: 'http://localhost', - * port: 9000, - * useSSL: false, - * region: 'us-east-1' - * }; - * - * const storageClient = new StorageClient(params, credentials); - * ``` - */ -export class StorageClient { - private client: Minio.Client; - private clientParams: StorageParams; - - /** - * **Storage client constructor** - * - * @param {StorageParams} params - Cloud storage params - * @param {StorageCredentials} credentials - Optional. Cloud storage access data. If credentials are not provided - use anonymous access to the bucket - */ - constructor(params: StorageParams, credentials?: StorageCredentials) { - try { - this.clientParams = params; - - this.client = new Minio.Client({ - ...params, - accessKey: credentials?.accessKey ?? '', - secretKey: credentials?.secretKey ?? '', - }); - } catch { - throw ErrorStorageClientNotInitialized; - } - } - - /** - * This function downloads files from a bucket. - * - * @param {string[]} keys Array of filenames to download. - * @param {string} bucket Bucket name. - * @returns {Promise} Returns an array of JSON files downloaded and parsed into objects. - * - * **Code example** - * - * ```ts - * import { StorageClient, StorageCredentials, StorageParams } from '@human-protocol/sdk'; - * - * const params: StorageParams = { - * endPoint: 'http://localhost', - * port: 9000, - * useSSL: false, - * region: 'us-east-1' - * }; - * - * const storageClient = new StorageClient(params); - * - * const keys = ['file1.json', 'file2.json']; - * const files = await storageClient.downloadFiles(keys, 'bucket-name'); - * ``` - */ - public async downloadFiles(keys: string[], bucket: string): Promise { - const isBucketExists = await this.client.bucketExists(bucket); - if (!isBucketExists) { - throw ErrorStorageBucketNotFound; - } - - return Promise.all( - keys.map(async (key) => { - try { - const response = await this.client.getObject(bucket, key); - const content = response?.read(); - - return { key, content: JSON.parse(content?.toString('utf-8') || '') }; - } catch { - throw ErrorStorageFileNotFound; - } - }) - ); - } - - /** - * This function downloads files from a URL. - * - * @param {string} url URL of the file to download. - * @returns {Promise} Returns the JSON file downloaded and parsed into an object. - * - * **Code example** - * - * ```ts - * import { StorageClient } from '@human-protocol/sdk'; - * - * const file = await StorageClient.downloadFileFromUrl('http://localhost/file.json'); - * ``` - */ - public static async downloadFileFromUrl(url: string): Promise { - if (!isValidUrl(url)) { - throw ErrorInvalidUrl; - } - - try { - const { data, status } = await axios.get(url, { - headers: { - 'Content-Type': 'application/json', - }, - }); - - if (status !== HttpStatus.OK) { - throw ErrorStorageFileNotFound; - } - - return data; - } catch { - throw ErrorStorageFileNotFound; - } - } - - /** - * This function uploads files to a bucket. - * - * @param {any[]} files Array of objects to upload serialized into JSON. - * @param {string} bucket Bucket name. - * @returns {Promise} Returns an array of uploaded file metadata. - * - * **Code example** - * - * ```ts - * import { StorageClient, StorageCredentials, StorageParams } from '@human-protocol/sdk'; - * - * const credentials: StorageCredentials = { - * accessKey: 'ACCESS_KEY', - * secretKey: 'SECRET_KEY', - * }; - * const params: StorageParams = { - * endPoint: 'http://localhost', - * port: 9000, - * useSSL: false, - * region: 'us-east-1' - * }; - * - * const storageClient = new StorageClient(params, credentials); - * const file1 = { name: 'file1', description: 'description of file1' }; - * const file2 = { name: 'file2', description: 'description of file2' }; - * const files = [file1, file2]; - * const uploadedFiles = await storageClient.uploadFiles(files, 'bucket-name'); - * ``` - */ - public async uploadFiles( - files: any[], - bucket: string - ): Promise { - const isBucketExists = await this.client.bucketExists(bucket); - if (!isBucketExists) { - throw ErrorStorageBucketNotFound; - } - - return Promise.all( - files.map(async (file) => { - const content = JSON.stringify(file); - - const hash = crypto.createHash('sha1').update(content).digest('hex'); - const key = `s3${hash}.json`; - - try { - await this.client.putObject(bucket, key, content, { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - }); - - return { - key, - url: `${this.clientParams.useSSL ? 'https' : 'http'}://${ - this.clientParams.endPoint - }${ - this.clientParams.port ? `:${this.clientParams.port}` : '' - }/${bucket}/${key}`, - hash, - }; - } catch { - throw ErrorStorageFileNotUploaded; - } - }) - ); - } - - /** - * This function checks if a bucket exists. - * - * @param {string} bucket Bucket name. - * @returns {Promise} Returns `true` if exists, `false` if it doesn't. - * - * **Code example** - * - * ```ts - * import { StorageClient, StorageCredentials, StorageParams } from '@human-protocol/sdk'; - * - * const credentials: StorageCredentials = { - * accessKey: 'ACCESS_KEY', - * secretKey: 'SECRET_KEY', - * }; - * const params: StorageParams = { - * endPoint: 'http://localhost', - * port: 9000, - * useSSL: false, - * region: 'us-east-1' - * }; - * - * const storageClient = new StorageClient(params, credentials); - * const exists = await storageClient.bucketExists('bucket-name'); - * ``` - */ - public async bucketExists(bucket: string): Promise { - return this.client.bucketExists(bucket); - } - - /** - * This function lists all file names contained in the bucket. - * - * @param {string} bucket Bucket name. - * @returns {Promise} Returns the list of file names contained in the bucket. - * - * **Code example** - * - * ```ts - * import { StorageClient, StorageCredentials, StorageParams } from '@human-protocol/sdk'; - * - * const credentials: StorageCredentials = { - * accessKey: 'ACCESS_KEY', - * secretKey: 'SECRET_KEY', - * }; - * const params: StorageParams = { - * endPoint: 'http://localhost', - * port: 9000, - * useSSL: false, - * region: 'us-east-1' - * }; - * - * const storageClient = new StorageClient(params, credentials); - * const fileNames = await storageClient.listObjects('bucket-name'); - * ``` - */ - public async listObjects(bucket: string): Promise { - const isBucketExists = await this.client.bucketExists(bucket); - if (!isBucketExists) { - throw ErrorStorageBucketNotFound; - } - - try { - return new Promise((resolve, reject) => { - const keys: string[] = []; - const stream = this.client.listObjectsV2(bucket, '', true, ''); - - stream.on('data', (obj: { name: string }) => keys.push(obj.name)); - stream.on('error', reject); - stream.on('end', () => { - resolve(keys); - }); - }); - } catch (e) { - throw new Error(String(e)); - } - } -} diff --git a/packages/sdk/typescript/human-protocol-sdk/src/types.ts b/packages/sdk/typescript/human-protocol-sdk/src/types.ts index 14497c4bc5..6d5ae3a7ba 100644 --- a/packages/sdk/typescript/human-protocol-sdk/src/types.ts +++ b/packages/sdk/typescript/human-protocol-sdk/src/types.ts @@ -36,63 +36,6 @@ export enum EscrowStatus { ToCancel, } -/** - * AWS/GCP cloud storage access data - * @readonly - * @deprecated StorageClient is deprecated. Use Minio.Client directly. - */ -export type StorageCredentials = { - /** - * Access Key - */ - accessKey: string; - /** - * Secret Key - */ - secretKey: string; -}; - -/** - * @deprecated StorageClient is deprecated. Use Minio.Client directly. - */ -export type StorageParams = { - /** - * Request endPoint - */ - endPoint: string; - /** - * Enable secure (HTTPS) access. Default value set to false - */ - useSSL: boolean; - /** - * Region - */ - region?: string; - /** - * TCP/IP port number. Default value set to 80 for HTTP and 443 for HTTPs - */ - port?: number; -}; - -/** - * Upload file data - * @readonly - */ -export type UploadFile = { - /** - * Uploaded object key - */ - key: string; - /** - * Uploaded object URL - */ - url: string; - /** - * Hash of uploaded object key - */ - hash: string; -}; - /** * Network data */ diff --git a/packages/sdk/typescript/human-protocol-sdk/test/storage.test.ts b/packages/sdk/typescript/human-protocol-sdk/test/storage.test.ts deleted file mode 100644 index 6c1988c700..0000000000 --- a/packages/sdk/typescript/human-protocol-sdk/test/storage.test.ts +++ /dev/null @@ -1,400 +0,0 @@ -// Create a Minio.Client mock -vi.mock('minio', () => { - class Client { - getObject = vi.fn().mockImplementation(() => { - const read = () => { - return JSON.stringify({ key: STORAGE_TEST_FILE_VALUE }); - }; - return Promise.resolve({ read }); - }); // getObject mock - putObject = vi.fn(); // putObject mock - bucketExists = vi.fn().mockImplementation((bucketName) => { - // Add conditional logic here based on the test scenario - if (bucketName === STORAGE_FAKE_BUCKET) { - return Promise.resolve(false); // Return false for fake scenario - } else { - return Promise.resolve(true); // Return true for other scenarios - } - }); - } - - // Return Minio.Client mock - return { Client }; -}); - -import { describe, test, expect, vi, beforeAll } from 'vitest'; -import axios from 'axios'; -import crypto from 'crypto'; -import { - DEFAULT_ENDPOINT, - DEFAULT_PORT, - DEFAULT_PUBLIC_BUCKET, - DEFAULT_REGION, - DEFAULT_USE_SSL, - HttpStatus, - StorageCredentials, - StorageParams, -} from '../src'; -import { - ErrorInvalidUrl, - ErrorStorageFileNotFound, - ErrorStorageFileNotUploaded, -} from '../src/error'; -import { StorageClient } from '../src/storage'; -import { - FAKE_URL, - STORAGE_FAKE_BUCKET, - STORAGE_TEST_ACCESS_KEY, - STORAGE_TEST_FILE_VALUE, - STORAGE_TEST_FILE_VALUE_2, - STORAGE_TEST_SECRET_KEY, -} from './utils/constants'; - -describe('Storage tests', () => { - describe('Client initialization', () => { - test('should set correct credentials', async () => { - const storageCredentials: StorageCredentials = { - accessKey: STORAGE_TEST_ACCESS_KEY, - secretKey: STORAGE_TEST_SECRET_KEY, - }; - - expect(storageCredentials.accessKey).toEqual(STORAGE_TEST_ACCESS_KEY); - expect(storageCredentials.secretKey).toEqual(STORAGE_TEST_SECRET_KEY); - }); - - test('should set correct params', async () => { - const storageParams: StorageParams = { - endPoint: DEFAULT_ENDPOINT, - port: DEFAULT_PORT, - useSSL: DEFAULT_USE_SSL, - region: DEFAULT_REGION, - }; - - expect(storageParams.endPoint).toEqual(DEFAULT_ENDPOINT); - expect(storageParams.port).toEqual(DEFAULT_PORT); - expect(storageParams.useSSL).toEqual(false); - expect(storageParams.region).toEqual(DEFAULT_REGION); - }); - - test('should init client with empty credentials', async () => { - const storageParams: StorageParams = { - endPoint: DEFAULT_ENDPOINT, - port: DEFAULT_PORT, - useSSL: DEFAULT_USE_SSL, - }; - - const storageClient = new StorageClient(storageParams); - - expect(storageClient).toBeInstanceOf(StorageClient); - }); - }); - - describe('Client anonymous access', () => { - let storageClient: StorageClient; - - beforeAll(async () => { - const storageParams: StorageParams = { - endPoint: DEFAULT_ENDPOINT, - port: DEFAULT_PORT, - useSSL: DEFAULT_USE_SSL, - }; - - storageClient = new StorageClient(storageParams); - }); - - test('should return the bucket exists', async () => { - const isExists = await storageClient.bucketExists(DEFAULT_PUBLIC_BUCKET); - expect(isExists).toEqual(true); - }); - - test('should return the bucket does not exist', async () => { - const isExists = await storageClient.bucketExists(STORAGE_FAKE_BUCKET); - expect(isExists).toEqual(false); - }); - - test('should upload the file with success', async () => { - const file = { key: STORAGE_TEST_FILE_VALUE }; - - const uploadedResults = await storageClient.uploadFiles( - [file], - DEFAULT_PUBLIC_BUCKET - ); - - const hash = crypto - .createHash('sha1') - .update(JSON.stringify(file)) - .digest('hex'); - const key = `s3${hash}.json`; - - expect(storageClient['client'].putObject).toHaveBeenCalledWith( - DEFAULT_PUBLIC_BUCKET, - key, - JSON.stringify(file), - { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - } - ); - expect(uploadedResults[0].key).toEqual(key); - expect(uploadedResults[0].url).toEqual( - `http://${DEFAULT_ENDPOINT}:${DEFAULT_PORT}/${DEFAULT_PUBLIC_BUCKET}/${key}` - ); - expect(uploadedResults[0].hash).toEqual(hash); - }); - - test('should not upload the file with an error', async () => { - const file = { key: STORAGE_TEST_FILE_VALUE }; - vi.spyOn(storageClient, 'uploadFiles').mockImplementation(() => { - throw ErrorStorageFileNotUploaded; - }); - expect(() => - storageClient.uploadFiles([file], DEFAULT_PUBLIC_BUCKET) - ).toThrow(ErrorStorageFileNotUploaded); - }); - - test('should download the files with success', async () => { - const file = { key: STORAGE_TEST_FILE_VALUE }; - - const hash = crypto - .createHash('sha1') - .update(JSON.stringify(file)) - .digest('hex'); - const key = `s3${hash}.json`; - - const downloadedResults = await storageClient.downloadFiles( - [key], - DEFAULT_PUBLIC_BUCKET - ); - - expect(storageClient['client'].getObject).toHaveBeenCalledWith( - DEFAULT_PUBLIC_BUCKET, - key - ); - expect(downloadedResults[0].key).toEqual(key); - expect(downloadedResults[0].content).toEqual(file); - }); - - test('should not download the files with an error', async () => { - vi.spyOn(storageClient, 'downloadFiles').mockImplementation(() => { - throw ErrorStorageFileNotFound; - }); - expect(() => - storageClient.downloadFiles( - [STORAGE_TEST_FILE_VALUE], - DEFAULT_PUBLIC_BUCKET - ) - ).toThrow(ErrorStorageFileNotFound); - }); - - test('should fail URL validation', async () => { - await expect(StorageClient.downloadFileFromUrl(FAKE_URL)).rejects.toThrow( - ErrorInvalidUrl - ); - }); - - test('should download the file from URL with success', async () => { - const file = { key: STORAGE_TEST_FILE_VALUE }; - - vi.spyOn(axios, 'get').mockImplementation(() => - Promise.resolve({ data: file, status: HttpStatus.OK }) - ); - - const hash = crypto - .createHash('sha1') - .update(JSON.stringify(file)) - .digest('hex'); - const url = `http://${DEFAULT_ENDPOINT}:${DEFAULT_PORT}/${DEFAULT_PUBLIC_BUCKET}/${hash}.json`; - - const result = await StorageClient.downloadFileFromUrl(url); - expect(result).toEqual(file); - }); - - test('should not download the file from URL with an error', async () => { - const file = { key: STORAGE_TEST_FILE_VALUE }; - - const hash = crypto - .createHash('sha1') - .update(JSON.stringify(file)) - .digest('hex'); - const url = `http://${DEFAULT_ENDPOINT}:${DEFAULT_PORT}/${DEFAULT_PUBLIC_BUCKET}/${hash}.json`; - - vi.spyOn(StorageClient, 'downloadFileFromUrl').mockImplementation(() => { - throw ErrorStorageFileNotFound; - }); - expect(() => StorageClient.downloadFileFromUrl(url)).toThrow( - ErrorStorageFileNotFound - ); - }); - - test('should return a list of objects with success', async () => { - const file1 = { key: STORAGE_TEST_FILE_VALUE }; - const hash1 = crypto - .createHash('sha1') - .update(JSON.stringify(file1)) - .digest('hex'); - const key1 = `s3${hash1}.json`; - - const file2 = { key: STORAGE_TEST_FILE_VALUE_2 }; - const hash2 = crypto - .createHash('sha1') - .update(JSON.stringify(file2)) - .digest('hex'); - const key2 = `s3${hash2}.json`; - - vi.spyOn(storageClient, 'listObjects').mockImplementation(() => - Promise.resolve([key1, key2]) - ); - - const results = await storageClient.listObjects(DEFAULT_PUBLIC_BUCKET); - - expect(results[0]).toEqual(key1); - expect(results[1]).toEqual(key2); - }); - - test('should not return a list of objects with an error', async () => { - vi.spyOn(storageClient, 'listObjects').mockImplementation(() => { - throw new Error(); - }); - expect(() => storageClient.listObjects(DEFAULT_PUBLIC_BUCKET)).toThrow( - new Error() - ); - }); - }); - - describe('Client with credentials', () => { - let storageClient: StorageClient; - - beforeAll(async () => { - const storageCredentials: StorageCredentials = { - accessKey: STORAGE_TEST_ACCESS_KEY, - secretKey: STORAGE_TEST_SECRET_KEY, - }; - - const storageParams: StorageParams = { - endPoint: DEFAULT_ENDPOINT, - port: DEFAULT_PORT, - useSSL: DEFAULT_USE_SSL, - }; - - storageClient = new StorageClient(storageParams, storageCredentials); - }); - - test('should return the bucket exists', async () => { - const isExists = await storageClient.bucketExists(DEFAULT_PUBLIC_BUCKET); - expect(isExists).toEqual(true); - }); - - test('should return the bucket does not exist', async () => { - const isExists = await storageClient.bucketExists(STORAGE_FAKE_BUCKET); - expect(isExists).toEqual(false); - }); - - test('should upload the file with success', async () => { - const file = { key: STORAGE_TEST_FILE_VALUE }; - - const uploadedResults = await storageClient.uploadFiles( - [file], - DEFAULT_PUBLIC_BUCKET - ); - - const hash = crypto - .createHash('sha1') - .update(JSON.stringify(file)) - .digest('hex'); - const key = `s3${hash}.json`; - - expect(storageClient['client'].putObject).toHaveBeenCalledWith( - DEFAULT_PUBLIC_BUCKET, - key, - JSON.stringify(file), - { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - } - ); - expect(uploadedResults[0].key).toEqual(key); - expect(uploadedResults[0].url).toEqual( - `http://${DEFAULT_ENDPOINT}:${DEFAULT_PORT}/${DEFAULT_PUBLIC_BUCKET}/${key}` - ); - expect(uploadedResults[0].hash).toEqual(hash); - }); - - test('should not upload the file with an error', async () => { - const file = { key: STORAGE_TEST_FILE_VALUE }; - vi.spyOn(storageClient, 'uploadFiles').mockImplementation(() => { - throw ErrorStorageFileNotUploaded; - }); - expect(() => - storageClient.uploadFiles([file], DEFAULT_PUBLIC_BUCKET) - ).toThrow(ErrorStorageFileNotUploaded); - }); - - test('should download the file with success', async () => { - const file = { key: STORAGE_TEST_FILE_VALUE }; - - const hash = crypto - .createHash('sha1') - .update(JSON.stringify(file)) - .digest('hex'); - const key = `s3${hash}.json`; - - const downloadedResults = await storageClient.downloadFiles( - [key], - DEFAULT_PUBLIC_BUCKET - ); - - expect(storageClient['client'].getObject).toHaveBeenCalledWith( - DEFAULT_PUBLIC_BUCKET, - key - ); - expect(downloadedResults[0].key).toEqual(key); - expect(downloadedResults[0].content).toEqual(file); - }); - - test('should not download the file with an error', async () => { - vi.spyOn(storageClient, 'downloadFiles').mockImplementation(() => { - throw ErrorStorageFileNotFound; - }); - expect(() => - storageClient.downloadFiles( - [STORAGE_TEST_FILE_VALUE], - DEFAULT_PUBLIC_BUCKET - ) - ).toThrow(ErrorStorageFileNotFound); - }); - - test('should return a list of objects with success', async () => { - const file1 = { key: STORAGE_TEST_FILE_VALUE }; - const hash1 = crypto - .createHash('sha1') - .update(JSON.stringify(file1)) - .digest('hex'); - const key1 = `s3${hash1}.json`; - - const file2 = { key: STORAGE_TEST_FILE_VALUE_2 }; - const hash2 = crypto - .createHash('sha1') - .update(JSON.stringify(file2)) - .digest('hex'); - const key2 = `s3${hash2}.json`; - - vi.spyOn(storageClient, 'listObjects').mockImplementation(() => - Promise.resolve([key1, key2]) - ); - - const results = await storageClient.listObjects(DEFAULT_PUBLIC_BUCKET); - - expect(results[0]).toEqual(key1); - expect(results[1]).toEqual(key2); - }); - - test('should not return a list of objects with an error', async () => { - vi.spyOn(storageClient, 'listObjects').mockImplementation(() => { - throw new Error(); - }); - expect(() => storageClient.listObjects(DEFAULT_PUBLIC_BUCKET)).toThrow( - new Error() - ); - }); - }); -}); diff --git a/yarn.lock b/yarn.lock index b951ec5945..1e635094d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -327,7 +327,7 @@ __metadata: jest: "npm:30.2.0" joi: "npm:^17.13.3" jsonwebtoken: "npm:^9.0.2" - minio: "npm:7.1.3" + minio: "npm:8.0.6" passport: "npm:^0.7.0" passport-jwt: "npm:^4.0.1" pg: "npm:8.13.1" @@ -372,7 +372,7 @@ __metadata: helmet: "npm:^7.1.0" jest: "npm:^30.2.0" joi: "npm:^17.13.3" - minio: "npm:7.1.3" + minio: "npm:8.0.6" prettier: "npm:^3.4.2" reflect-metadata: "npm:^0.2.2" rxjs: "npm:^7.2.0" @@ -638,7 +638,7 @@ __metadata: jest: "npm:30.2.0" joi: "npm:^17.13.3" json-stable-stringify: "npm:^1.2.1" - minio: "npm:7.1.3" + minio: "npm:8.0.6" nestjs-minio-client: "npm:^2.2.0" node-cache: "npm:^5.1.2" passport: "npm:^0.7.0" @@ -714,7 +714,7 @@ __metadata: joi: "npm:^17.13.3" json-stable-stringify: "npm:^1.2.1" lodash: "npm:^4.17.21" - minio: "npm:7.1.3" + minio: "npm:8.0.6" nock: "npm:^14.0.3" passport: "npm:^0.7.0" passport-jwt: "npm:^4.0.1" @@ -4989,7 +4989,6 @@ __metadata: graphql: "npm:^16.8.1" graphql-request: "npm:^7.3.4" graphql-tag: "npm:^2.12.6" - minio: "npm:7.1.3" openpgp: "npm:^6.2.2" prettier: "npm:^3.4.2" secp256k1: "npm:^5.0.1" @@ -15106,6 +15105,13 @@ __metadata: languageName: node linkType: hard +"buffer-crc32@npm:^1.0.0": + version: 1.0.0 + resolution: "buffer-crc32@npm:1.0.0" + checksum: 10c0/8b86e161cee4bb48d5fa622cbae4c18f25e4857e5203b89e23de59e627ab26beb82d9d7999f2b8de02580165f61f83f997beaf02980cdf06affd175b651921ab + languageName: node + linkType: hard + "buffer-equal-constant-time@npm:^1.0.1": version: 1.0.1 resolution: "buffer-equal-constant-time@npm:1.0.1" @@ -23781,6 +23787,28 @@ __metadata: languageName: node linkType: hard +"minio@npm:8.0.6": + version: 8.0.6 + resolution: "minio@npm:8.0.6" + dependencies: + async: "npm:^3.2.4" + block-stream2: "npm:^2.1.0" + browser-or-node: "npm:^2.1.1" + buffer-crc32: "npm:^1.0.0" + eventemitter3: "npm:^5.0.1" + fast-xml-parser: "npm:^4.4.1" + ipaddr.js: "npm:^2.0.1" + lodash: "npm:^4.17.21" + mime-types: "npm:^2.1.35" + query-string: "npm:^7.1.3" + stream-json: "npm:^1.8.0" + through2: "npm:^4.0.2" + web-encoding: "npm:^1.1.5" + xml2js: "npm:^0.5.0 || ^0.6.2" + checksum: 10c0/7b10b1d780f300d1ac8881be32e9b5a1d172f9c3e72bfb86f966d085746aa5e5aa9c54c41d728ba00d8cd8c4969ac219214c5e763d1f7918f53f1081fef1a19a + languageName: node + linkType: hard + "minipass-collect@npm:^2.0.1": version: 2.0.1 resolution: "minipass-collect@npm:2.0.1" @@ -28446,7 +28474,7 @@ __metadata: languageName: node linkType: hard -"stream-json@npm:^1.9.1": +"stream-json@npm:^1.8.0, stream-json@npm:^1.9.1": version: 1.9.1 resolution: "stream-json@npm:1.9.1" dependencies: @@ -31743,7 +31771,7 @@ __metadata: languageName: node linkType: hard -"xml2js@npm:^0.6.2": +"xml2js@npm:^0.5.0 || ^0.6.2, xml2js@npm:^0.6.2": version: 0.6.2 resolution: "xml2js@npm:0.6.2" dependencies: