diff --git a/packages/examples/gcv/.env.example b/packages/examples/gcv/.env.example deleted file mode 100644 index 5912b08eaf..0000000000 --- a/packages/examples/gcv/.env.example +++ /dev/null @@ -1,18 +0,0 @@ -# GCV -GOOGLE_PROJECT_ID= -GOOGLE_PRIVATE_KEY_ID= -GOOGLE_PRIVATE_KEY= -GOOGLE_CLIENT_EMAIL= -GOOGLE_CLIENT_ID= -GOOGLE_AUTH_URI= -GOOGLE_TOKEN_URI= -GOOGLE_AUTH_PROVIDER_X509_CERT_URL= -GOOGLE_CLIENT_X509_CERT_URL= - -# S3 -S3_ENDPOINT=localhost -S3_PORT=9000 -S3_ACCESS_KEY=access-key -S3_SECRET_KEY=secret-key -S3_BUCKET=manifests -S3_USE_SSL=false \ No newline at end of file diff --git a/packages/examples/gcv/.gitignore b/packages/examples/gcv/.gitignore deleted file mode 100644 index a17018a7e2..0000000000 --- a/packages/examples/gcv/.gitignore +++ /dev/null @@ -1,44 +0,0 @@ -# compiled output -/dist -/node_modules - -# Logs -logs -*.log -npm-debug.log* -pnpm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* - -# OS -.DS_Store - -# Tests -/coverage -/.nyc_output - -# IDEs and editors -/.idea -.project -.classpath -.c9/ -*.launch -.settings/ -*.sublime-workspace - -# IDE - VSCode -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json - -.env.development -.env.production - -# Postgres Data -db - -# Results Data -results.json \ No newline at end of file diff --git a/packages/examples/gcv/.prettierrc b/packages/examples/gcv/.prettierrc deleted file mode 100644 index a20502b7f0..0000000000 --- a/packages/examples/gcv/.prettierrc +++ /dev/null @@ -1,4 +0,0 @@ -{ - "singleQuote": true, - "trailingComma": "all" -} diff --git a/packages/examples/gcv/README.md b/packages/examples/gcv/README.md deleted file mode 100644 index 9b6f91f4b0..0000000000 --- a/packages/examples/gcv/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# Images Content Moderation - -Images Content Moderation is a module that uses the Google Cloud Vision API to analyze images for moderation purposes. It can assess images for adult, violent, racy, spoof, and medical content, saving the results in a JSON file. The module is designed to help detect potentially inappropriate content in image datasets stored in a remote bucket. - -## Table of Contents - -- [Prerequisites](#prerequisites) -- [Installation](#installation) -- [Usage](#usage) -- [Methods](#methods) -- [Example Output](#example-output) - -## Prerequisites - -- [Node.js](https://nodejs.org/) (version 14 or higher) -- [Google Cloud Vision API](https://cloud.google.com/vision) enabled and credentials for access - -## Installation - -1. Clone the repository: - ```bash - git clone https://github.com/humanprotocol/human-protocol.git - ``` - -2. Navigate to the project directory: - ```bash - cd human-protocol/packages/examples/gcv - ``` - -3. Install dependencies: - ```bash - npm install - ``` - -4. Set up your Google Cloud Vision API credentials: - ```bash - Create a `.env` file in the project directory (or rename `.env.example` to `.env`) and add your credentials. - ``` - -## Usage - -To use the `VisionModeration` class, import and create an instance by providing the necessary credentials. Then, call `processDataset` with your dataset. - -Example usage: - -```typescript -import { VisionModeration } from './VisionModeration'; -import { StorageDataDto } from './dto/storage'; - -const visionModeration = new VisionModeration('your-project-id', 'your-private-key', 'your-client-email'); -const storageData = new StorageDataDto('your-bucket-name', 'your-folder-name'); - -visionModeration.processDataset(storageData) - .then(response => { - console.log('Moderation Results:', response); - }) - .catch(error => { - console.error('Error processing dataset:', error); - }); -``` - -## Methods - -### `analyzeImagesForModeration(imageUrls: string[]): Promise` -Analyzes a list of image URLs for moderation using the Google Cloud Vision API. - -- **Parameters:** `imageUrls` - an array of URLs pointing to images in the dataset -- **Returns:** a list of moderation results, including scores for adult, violence, racy, spoof, and medical content. - -### `saveResultsToJson(results: any[], jsonFilePath: string)` -Saves the moderation results to a JSON file at the specified path. - -- **Parameters:** - - `results`: the moderation results to save - - `jsonFilePath`: the path where the JSON file should be saved - -### `processDataset(storageData: StorageDataDto): Promise<{ containsAbuse: string; abuseResultsFile: string }>` -Processes all images in the dataset, analyzes them for inappropriate content, and saves results to `results.json`. It returns whether any abusive content was detected. - -- **Parameters:** `storageData` - an object containing details about the storage dataset -- **Returns:** an object with: - - `containsAbuse`: `"true"` if any content is marked as abusive, `"false"` otherwise - - `abuseResultsFile`: the path to `results.json` - -## Example Output - -Example content of `results.json` file: - -```json -[ - { - "imageUrl": "https://yourapp.com/bucket/abuse.jpg", - "moderationResult": { - "adult": "VERY_LIKELY", - "violence": "VERY_UNLIKELY", - "racy": "VERY_LIKELY", - "spoof": "VERY_UNLIKELY", - "medical": "LIKELY" - } - }, - { - "imageUrl": "https://yourapp.com/bucket/cat.jpg", - "moderationResult": { - "adult": "VERY_UNLIKELY", - "violence": "UNLIKELY", - "racy": "VERY_UNLIKELY", - "spoof": "POSSIBLE", - "medical": "VERY_UNLIKELY" - } - } -] -``` - -Example response from `processDataset`: - -```json -{ - "containsAbuse": "true", - "abuseResultsFile": "./results.json" -} -``` \ No newline at end of file diff --git a/packages/examples/gcv/eslint.config.mjs b/packages/examples/gcv/eslint.config.mjs deleted file mode 100644 index c127f82b7f..0000000000 --- a/packages/examples/gcv/eslint.config.mjs +++ /dev/null @@ -1,60 +0,0 @@ -import eslint from '@eslint/js'; -import globals from 'globals'; -import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended'; -import jestPlugin from 'eslint-plugin-jest'; -import tseslint from 'typescript-eslint'; - -/** @type {import('eslint').Linter.FlatConfig[]} */ -const config = tseslint.config( - { - ignores: ['dist', 'node_modules'], - }, - eslint.configs.recommended, - tseslint.configs.recommended, - eslintPluginPrettierRecommended, - { - files: ['**/*.ts', '**/*.js'], - languageOptions: { - globals: { - ...globals.node, - ...globals.es2022, - }, - ecmaVersion: 2022, - sourceType: 'module', - parserOptions: { - projectService: true, - tsconfigRootDir: import.meta.dirname, - }, - }, - plugins: { - jest: jestPlugin, - }, - rules: { - 'no-useless-assignment': 'off', - 'preserve-caught-error': 'off', - '@typescript-eslint/interface-name-prefix': 'off', - '@typescript-eslint/explicit-function-return-type': 'off', - '@typescript-eslint/explicit-module-boundary-types': 'off', - '@typescript-eslint/no-explicit-any': 'off', - '@typescript-eslint/no-empty-function': 'off', - '@/quotes': [ - 'error', - 'single', - { avoidEscape: true, allowTemplateLiterals: true }, - ], - }, - }, - { - files: ['**/*.spec.ts', '**/*.spec.tsx', '**/*.test.ts', '**/*.test.tsx'], - languageOptions: { - globals: { - ...globals.jest, - }, - }, - plugins: { - jest: jestPlugin, - }, - }, -); - -export default config; diff --git a/packages/examples/gcv/jest.config.ts b/packages/examples/gcv/jest.config.ts deleted file mode 100644 index 43fd9c5a46..0000000000 --- a/packages/examples/gcv/jest.config.ts +++ /dev/null @@ -1,14 +0,0 @@ -module.exports = { - coverageDirectory: '../coverage', - collectCoverageFrom: ['**/*.(t|j)s'], - moduleFileExtensions: ['js', 'json', 'ts'], - rootDir: 'src', - testEnvironment: 'node', - testRegex: '.*\\.spec\\.ts$', - transform: { - '^.+\\.(t|j)s$': 'ts-jest', - }, - moduleNameMapper: { - '^uuid$': require.resolve('uuid'), - }, -}; diff --git a/packages/examples/gcv/package.json b/packages/examples/gcv/package.json deleted file mode 100644 index 520ea20f57..0000000000 --- a/packages/examples/gcv/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "@apps/gcv", - "private": true, - "version": "1.0.0", - "license": "UNLICENSED", - "scripts": { - "clean": "tsc --build --clean && rm -rf dist", - "build": "tsc --build", - "start": "node dist/src/index.js", - "test": "jest", - "lint": "eslint \"{src,test}/**/*.ts\" --fix" - }, - "dependencies": { - "@google-cloud/vision": "^4.3.2", - "@nestjs/common": "^11.1.12", - "axios": "^1.7.2", - "dotenv": "^17.2.2", - "xml2js": "^0.6.2" - }, - "devDependencies": { - "@eslint/js": "^10.0.1", - "@types/xml2js": "^0.4.14", - "eslint": "^10.0.3", - "eslint-config-prettier": "^10.1.8", - "eslint-plugin-jest": "^29.15.0", - "eslint-plugin-prettier": "^5.5.5", - "globals": "^16.3.0", - "jest": "^29.7.0", - "prettier": "^3.8.1", - "typescript": "^5.8.3", - "typescript-eslint": "^8.57.0" - } -} diff --git a/packages/examples/gcv/src/constants/errors.ts b/packages/examples/gcv/src/constants/errors.ts deleted file mode 100644 index 762e5d153d..0000000000 --- a/packages/examples/gcv/src/constants/errors.ts +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Represents common error messages. - */ -export enum ErrorCommon { - ErrorProcessingDataset = 'Error processing dataset', -} - -/** - * Represents error messages related to bucket. - */ -export enum ErrorBucket { - NotExist = 'Bucket does not exist', - NotPublic = 'Bucket is not public', - UnableSaveFile = 'Unable to save file', - InvalidProvider = 'Invalid storage provider', - EmptyRegion = 'Region cannot be empty for this storage provider', - InvalidRegion = 'Invalid region for the storage provider', - EmptyBucket = 'bucketName cannot be empty', - FailedToFetchBucketContents = 'Failed to fetch bucket contents', -} diff --git a/packages/examples/gcv/src/dto/storage.ts b/packages/examples/gcv/src/dto/storage.ts deleted file mode 100644 index bc2d8d17da..0000000000 --- a/packages/examples/gcv/src/dto/storage.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { AWSRegions, StorageProviders } from '../enums/storage'; - -export class StorageDataDto { - public provider: StorageProviders; - public region: AWSRegions | null; - public bucketName: string; - public path: string; -} diff --git a/packages/examples/gcv/src/enums/storage.ts b/packages/examples/gcv/src/enums/storage.ts deleted file mode 100644 index 19d763dbe6..0000000000 --- a/packages/examples/gcv/src/enums/storage.ts +++ /dev/null @@ -1,92 +0,0 @@ -export enum StorageProviders { - AWS = 'AWS', - GCS = 'GCS', - LOCAL = 'LOCAL', -} - -export enum AWSRegions { - AF_SOUTH_1 = 'af-south-1', - AP_EAST_1 = 'ap-east-1', - AP_NORTHEAST_1 = 'ap-northeast-1', - AP_NORTHEAST_2 = 'ap-northeast-2', - AP_NORTHEAST_3 = 'ap-northeast-3', - AP_SOUTH_1 = 'ap-south-1', - AP_SOUTH_2 = 'ap-south-2', - AP_SOUTHEAST_1 = 'ap-southeast-1', - AP_SOUTHEAST_2 = 'ap-southeast-2', - AP_SOUTHEAST_3 = 'ap-southeast-3', - AP_SOUTHEAST_4 = 'ap-southeast-4', - CA_CENTRAL_1 = 'ca-central-1', - CN_NORTH_1 = 'cn-north-1', - CN_NORTHWEST_1 = 'cn-northwest-1', - EU_CENTRAL_1 = 'eu-central-1', - EU_CENTRAL_2 = 'eu-central-2', - EU_NORTH_1 = 'eu-north-1', - EU_SOUTH_1 = 'eu-south-1', - EU_SOUTH_2 = 'eu-south-2', - EU_WEST_1 = 'eu-west-1', - EU_WEST_2 = 'eu-west-2', - EU_WEST_3 = 'eu-west-3', - IL_CENTRAL_1 = 'il-central-1', - ME_CENTRAL_1 = 'me-central-1', - ME_SOUTH_1 = 'me-south-1', - SA_EAST_1 = 'sa-east-1', - US_EAST_1 = 'us-east-1', - US_EAST_2 = 'us-east-2', - US_GOV_EAST_1 = 'us-gov-east-1', - US_GOV_WEST_1 = 'us-gov-west-1', - US_WEST_1 = 'us-west-1', - US_WEST_2 = 'us-west-2', -} - -export enum GCSRegions { - ASIA_EAST1 = 'asia-east1', // Taiwan - ASIA_EAST2 = 'asia-east2', // Hong Kong - ASIA_NORTHEAST1 = 'asia-northeast1', // Tokyo - ASIA_NORTHEAST2 = 'asia-northeast2', // Osaka - ASIA_NORTHEAST3 = 'asia-northeast3', // Seoul - ASIA_SOUTH1 = 'asia-south1', // Bombay - ASIA_SOUTH2 = 'asia-south2', // Delhi - ASIA_SOUTHEAST1 = 'asia-southeast1', // Singapore - ASIA_SOUTHEAST2 = 'asia-southeast2', // Jakarta - AUSTRALIA_SOUTHEAST1 = 'australia-southeast1', // Sydney - AUSTRALIA_SOUTHEAST2 = 'australia-southeast2', // Melbourne - EUROPE_CENTRAL2 = 'europe-central2', // Warsaw - EUROPE_NORTH1 = 'europe-north1', // Finland (Low CO2 footprint) - EUROPE_SOUTHWEST1 = 'europe-southwest1', // Madrid - EUROPE_WEST1 = 'europe-west1', // Belgium (Low CO2 footprint) - EUROPE_WEST2 = 'europe-west2', // London (Low CO2 footprint) - EUROPE_WEST3 = 'europe-west3', // Frankfurt (Low CO2 footprint) - EUROPE_WEST4 = 'europe-west4', // Netherlands - EUROPE_WEST6 = 'europe-west6', // Zurich (Low CO2 footprint) - EUROPE_WEST8 = 'europe-west8', // Milan - EUROPE_WEST9 = 'europe-west9', // Paris (Low CO2 footprint) - EUROPE_WEST10 = 'europe-west10', // Berlin - EUROPE_WEST12 = 'europe-west12', // Turin - ME_CENTRAL1 = 'me-central1', // Doha - ME_CENTRAL2 = 'me-central2', // Dammam, Saudi Arabia - ME_WEST1 = 'me-west1', // Tel Aviv - NORTHAMERICA_NORTHEAST1 = 'northamerica-northeast1', // Montreal (Low CO2 footprint) - NORTHAMERICA_NORTHEAST2 = 'northamerica-northeast2', // Toronto (Low CO2 footprint) - SOUTHAMERICA_EAST1 = 'southamerica-east1', // São Paulo (Low CO2 footprint) - SOUTHAMERICA_WEST1 = 'southamerica-west1', // Santiago (Low CO2 footprint) - US_CENTRAL1 = 'us-central1', // Iowa (Low CO2 footprint) - US_EAST1 = 'us-east1', // South Carolina - US_EAST4 = 'us-east4', // Northern Virginia - US_EAST5 = 'us-east5', // Columbus - US_SOUTH1 = 'us-south1', // Dallas - US_WEST1 = 'us-west1', // Oregon (Low CO2 footprint) - US_WEST2 = 'us-west2', // Los Angeles - US_WEST3 = 'us-west3', // Salt Lake City - US_WEST4 = 'us-west4', // Las Vegas -} - -export enum ContentType { - TEXT_PLAIN = 'text/plain', - APPLICATION_JSON = 'application/json', -} - -export enum Extension { - JSON = '.json', - ZIP = '.zip', -} diff --git a/packages/examples/gcv/src/errors/controlled.ts b/packages/examples/gcv/src/errors/controlled.ts deleted file mode 100644 index 32cc401c90..0000000000 --- a/packages/examples/gcv/src/errors/controlled.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { HttpStatus } from '@nestjs/common'; - -export class ControlledError extends Error { - status: HttpStatus; - - constructor(message: string, status: HttpStatus, stack?: string) { - super(message); - this.name = this.constructor.name; - this.status = status; - if (stack) this.stack = stack; - else Error.captureStackTrace(this, this.constructor); - } -} diff --git a/packages/examples/gcv/src/index.ts b/packages/examples/gcv/src/index.ts deleted file mode 100644 index a01941c778..0000000000 --- a/packages/examples/gcv/src/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -import * as dotenv from 'dotenv'; -import { StorageProviders } from './enums/storage'; -import { StorageDataDto } from './dto/storage'; -import { VisionModeration } from './utils/vision'; - -dotenv.config(); - -const visionModeration = new VisionModeration( - process.env.GOOGLE_PROJECT_ID!, - process.env.GOOGLE_PRIVATE_KEY!.replace(/\\n/g, '\n')!, - process.env.GOOGLE_CLIENT_EMAIL!, -); - -(async () => { - const storageData: StorageDataDto = { - provider: StorageProviders.LOCAL, - region: null, - bucketName: process.env.S3_BUCKET!, - path: '', - }; - - console.log(await visionModeration.processDataset(storageData)); -})(); diff --git a/packages/examples/gcv/src/utils/storage.spec.ts b/packages/examples/gcv/src/utils/storage.spec.ts deleted file mode 100644 index 319e89ccee..0000000000 --- a/packages/examples/gcv/src/utils/storage.spec.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { HttpStatus } from '@nestjs/common'; -import axios from 'axios'; -import { parseString } from 'xml2js'; -import { generateBucketUrl, listObjectsInBucket } from './storage'; -import { StorageDataDto } from '../dto/storage'; -import { AWSRegions, StorageProviders } from '../enums/storage'; -import { ErrorBucket } from '../constants/errors'; - -jest.mock('axios'); -jest.mock('xml2js'); - -describe('Storage Utils', () => { - describe('generateBucketUrl', () => { - it('should generate AWS bucket URL with path', () => { - const storageData: StorageDataDto = { - provider: StorageProviders.AWS, - region: AWSRegions.US_EAST_1, - bucketName: 'my-bucket', - path: 'my/path/', - }; - const result = generateBucketUrl(storageData); - expect(result.toString()).toBe( - 'https://my-bucket.s3.us-east-1.amazonaws.com/my/path', - ); - }); - - it('should generate GCS bucket URL without path', () => { - const storageData: StorageDataDto = { - provider: StorageProviders.GCS, - region: null, - bucketName: 'my-bucket', - path: '', - }; - const result = generateBucketUrl(storageData); - expect(result.toString()).toBe( - 'https://my-bucket.storage.googleapis.com/', - ); - }); - - it('should generate local bucket URL', () => { - process.env.S3_ENDPOINT = 'localhost'; - process.env.S3_PORT = '9000'; - const storageData: StorageDataDto = { - provider: StorageProviders.LOCAL, - region: null, - bucketName: 'my-local-bucket', - path: 'path/to/file', - }; - const result = generateBucketUrl(storageData); - expect(result.toString()).toBe( - 'http://localhost:9000/my-local-bucket/path/to/file', - ); - }); - - it('should throw an error for invalid provider', () => { - const storageData: StorageDataDto = { - provider: 'INVALID_PROVIDER' as StorageProviders, - region: null, - bucketName: 'my-bucket', - path: '', - }; - expect(() => generateBucketUrl(storageData)).toThrow( - ErrorBucket.InvalidProvider, - ); - }); - - it('should throw an error for empty bucket name', () => { - const storageData: StorageDataDto = { - provider: StorageProviders.AWS, - region: AWSRegions.US_EAST_1, - bucketName: '', - path: '', - }; - expect(() => generateBucketUrl(storageData)).toThrow( - ErrorBucket.EmptyBucket, - ); - }); - - it('should throw an error for empty region in AWS', () => { - const storageData: StorageDataDto = { - provider: StorageProviders.AWS, - region: null, - bucketName: 'my-bucket', - path: '', - }; - expect(() => generateBucketUrl(storageData)).toThrow( - ErrorBucket.EmptyRegion, - ); - }); - - it('should throw an error for invalid region', () => { - const storageData: StorageDataDto = { - provider: StorageProviders.AWS, - region: 'INVALID_REGION' as AWSRegions, - bucketName: 'my-bucket', - path: '', - }; - expect(() => generateBucketUrl(storageData)).toThrow( - ErrorBucket.InvalidRegion, - ); - }); - }); - - describe('listObjectsInBucket', () => { - it('should return object keys when API responds with valid data', async () => { - const mockUrl = new URL('https://my-bucket.s3.us-east-1.amazonaws.com'); - const mockResponse = { - status: HttpStatus.OK, - data: 'file1.txtfile2.txt', - }; - - (axios.get as jest.Mock).mockResolvedValue(mockResponse); - (parseString as jest.Mock).mockImplementation((data, callback) => { - callback(null, { - ListBucketResult: { - Contents: [{ Key: 'file1.txt' }, { Key: 'file2.txt' }], - }, - }); - }); - - const result = await listObjectsInBucket(mockUrl); - expect(result).toEqual(['file1.txt', 'file2.txt']); - }); - - it('should handle errors from the axios request', async () => { - const mockUrl = new URL('https://my-bucket.s3.us-east-1.amazonaws.com'); - (axios.get as jest.Mock).mockRejectedValue(new Error('Network Error')); - - await expect(listObjectsInBucket(mockUrl)).rejects.toThrow( - 'Network Error', - ); - }); - - it('should handle errors when parsing XML', async () => { - const mockUrl = new URL('https://my-bucket.s3.us-east-1.amazonaws.com'); - const mockResponse = { - status: HttpStatus.OK, - data: '', - }; - - (axios.get as jest.Mock).mockResolvedValue(mockResponse); - (parseString as jest.Mock).mockImplementation((data, callback) => { - callback(new Error('Parse Error'), null); - }); - - await expect(listObjectsInBucket(mockUrl)).rejects.toThrow('Parse Error'); - }); - }); -}); diff --git a/packages/examples/gcv/src/utils/storage.ts b/packages/examples/gcv/src/utils/storage.ts deleted file mode 100644 index f0976d74d8..0000000000 --- a/packages/examples/gcv/src/utils/storage.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { HttpStatus } from '@nestjs/common'; -import axios from 'axios'; -import { parseString } from 'xml2js'; -import { AWSRegions, StorageProviders } from '../enums/storage'; -import { StorageDataDto } from '../dto/storage'; -import { ErrorBucket } from '../constants/errors'; - -export function generateBucketUrl(storageData: StorageDataDto): URL { - if ( - storageData.provider != StorageProviders.AWS && - storageData.provider != StorageProviders.GCS && - storageData.provider != StorageProviders.LOCAL - ) { - throw new Error(ErrorBucket.InvalidProvider); - } - if (!storageData.bucketName) { - throw new Error(ErrorBucket.EmptyBucket); - } - switch (storageData.provider) { - case StorageProviders.AWS: - if (!storageData.region) { - throw new Error(ErrorBucket.EmptyRegion); - } - if (!isRegion(storageData.region)) { - throw new Error(ErrorBucket.InvalidRegion); - } - return new URL( - `https://${storageData.bucketName}.s3.${ - storageData.region - }.amazonaws.com${ - storageData.path ? `/${storageData.path.replace(/\/$/, '')}` : '' - }`, - ); - case StorageProviders.GCS: - return new URL( - `https://${storageData.bucketName}.storage.googleapis.com${ - storageData.path ? `/${storageData.path}` : '' - }`, - ); - case StorageProviders.LOCAL: - return new URL( - `http://${process.env.S3_ENDPOINT}:${process.env.S3_PORT}/${storageData.bucketName}${ - storageData.path ? `/${storageData.path}` : '' - }`, - ); - default: - throw new Error(ErrorBucket.InvalidProvider); - } -} - -function isRegion(value: string): value is AWSRegions { - return Object.values(AWSRegions).includes(value as AWSRegions); -} - -export async function listObjectsInBucket(url: URL): Promise { - let objects: string[] = []; - let nextContinuationToken: string | undefined; - const baseUrl = `${url.protocol}//${url.host}/`; - do { - let requestOptions = `${baseUrl}`; - - if (url.hostname !== 'localhost' && url.hostname !== '127.0.0.1') { - requestOptions += `?list-type=2${ - nextContinuationToken - ? `&continuation-token=${encodeURIComponent(nextContinuationToken)}` - : '' - }${url.pathname ? `&prefix=${url.pathname.replace(/^\//, '')}` : ''}`; - } else { - requestOptions += `${url.pathname ? `${url.pathname.replace(/^\//, '')}` : ''}?list-type=2${ - nextContinuationToken - ? `&continuation-token=${encodeURIComponent(nextContinuationToken)}` - : '' - }`; - } - - const response = await axios.get(requestOptions); - - if (response.status === HttpStatus.OK && response.data) { - parseString(response.data, (err: any, result: any) => { - if (err) { - throw new Error(err); - } - nextContinuationToken = result.ListBucketResult.NextContinuationToken - ? result.ListBucketResult.NextContinuationToken[0] - : undefined; - - const objectKeys = result.ListBucketResult.Contents?.map( - (item: any) => item.Key, - ); - - objects = objects.concat(objectKeys?.flat()); - }); - } else { - throw new Error(ErrorBucket.FailedToFetchBucketContents); - } - } while (nextContinuationToken); - return objects; -} diff --git a/packages/examples/gcv/src/utils/vision.spec.ts b/packages/examples/gcv/src/utils/vision.spec.ts deleted file mode 100644 index 772d1979a6..0000000000 --- a/packages/examples/gcv/src/utils/vision.spec.ts +++ /dev/null @@ -1,229 +0,0 @@ -import * as fs from 'fs'; -import { ImageAnnotatorClient } from '@google-cloud/vision'; -import { generateBucketUrl, listObjectsInBucket } from './storage'; -import { StorageDataDto } from '../dto/storage'; -import { VisionModeration } from './vision'; -import { ErrorCommon } from '../constants/errors'; -import { AWSRegions, StorageProviders } from '../enums/storage'; - -jest.mock('fs'); -jest.mock('@google-cloud/vision'); -jest.mock('./storage'); - -describe('VisionModeration', () => { - let visionModeration: VisionModeration; - let mockBatchAnnotateImages: jest.Mock; - - beforeEach(() => { - mockBatchAnnotateImages = jest.fn(); - (ImageAnnotatorClient as unknown as jest.Mock).mockImplementation(() => ({ - batchAnnotateImages: mockBatchAnnotateImages, - })); - - visionModeration = new VisionModeration( - 'test-project-id', - 'test-private-key', - 'test-client-email', - ); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('analyzeImagesForModeration', () => { - it('should return moderation results for a batch of valid image URLs', async () => { - const mockResponses = { - responses: [ - { - safeSearchAnnotation: { - adult: 'LIKELY', - violence: 'VERY_UNLIKELY', - racy: 'UNLIKELY', - spoof: 'POSSIBLE', - medical: 'VERY_UNLIKELY', - }, - }, - { - safeSearchAnnotation: { - adult: 'VERY_LIKELY', - violence: 'LIKELY', - racy: 'VERY_UNLIKELY', - spoof: 'UNLIKELY', - medical: 'VERY_UNLIKELY', - }, - }, - ], - }; - mockBatchAnnotateImages.mockResolvedValue([mockResponses]); - - const result = await visionModeration.analyzeImagesForModeration([ - 'http://test.com/image1.jpg', - 'http://test.com/image2.jpg', - ]); - - expect(result).toEqual([ - { - imageUrl: 'http://test.com/image1.jpg', - moderationResult: { - adult: 'LIKELY', - violence: 'VERY_UNLIKELY', - racy: 'UNLIKELY', - spoof: 'POSSIBLE', - medical: 'VERY_UNLIKELY', - }, - }, - { - imageUrl: 'http://test.com/image2.jpg', - moderationResult: { - adult: 'VERY_LIKELY', - violence: 'LIKELY', - racy: 'VERY_UNLIKELY', - spoof: 'UNLIKELY', - medical: 'VERY_UNLIKELY', - }, - }, - ]); - }); - - it('should return an empty array if no moderation results are found', async () => { - mockBatchAnnotateImages.mockResolvedValue([ - { - responses: [{}], // No safeSearchAnnotation - }, - ]); - - const result = await visionModeration.analyzeImagesForModeration([ - 'http://test.com/image.jpg', - ]); - - expect(result).toEqual([]); - }); - - it('should return an empty array if an error occurs', async () => { - mockBatchAnnotateImages.mockRejectedValue(new Error('API Error')); - - const result = await visionModeration.analyzeImagesForModeration([ - 'http://test.com/image.jpg', - ]); - - expect(result).toEqual([]); - expect(mockBatchAnnotateImages).toHaveBeenCalled(); - }); - }); - - describe('saveResultsToJson', () => { - it('should overwrite results in the JSON file', () => { - const results = [ - { imageUrl: 'http://test.com/image1.jpg', moderationResult: {} }, - ]; - const jsonFilePath = './results.json'; - - visionModeration.saveResultsToJson(results, jsonFilePath); - - expect(fs.writeFileSync).toHaveBeenCalledWith( - jsonFilePath, - JSON.stringify(results, null, 2), - ); - }); - }); - - describe('processDataset', () => { - it('should process all images in the dataset and save results', async () => { - const mockBucketUrl = { - protocol: 'http:', - host: 'test-bucket.s3.amazonaws.com', - pathname: '/images', - }; - const mockObjectKeys = ['image1.jpg', 'image2.jpg']; - const mockModerationResults = [ - { - imageUrl: 'http://test.com/image1.jpg', - moderationResult: { adult: 'LIKELY' }, - }, - { - imageUrl: 'http://test.com/image2.jpg', - moderationResult: { adult: 'VERY_LIKELY' }, - }, - ]; - - (generateBucketUrl as jest.Mock).mockReturnValue(mockBucketUrl); - (listObjectsInBucket as jest.Mock).mockResolvedValue(mockObjectKeys); - - jest - .spyOn(visionModeration, 'analyzeImagesForModeration') - .mockResolvedValue(mockModerationResults); - - jest.spyOn(visionModeration, 'saveResultsToJson'); - - const storageData: StorageDataDto = { - provider: StorageProviders.AWS, - region: AWSRegions.AF_SOUTH_1, - bucketName: 'test-bucket', - path: 'images', - }; - - const result = await visionModeration.processDataset(storageData); - - expect(generateBucketUrl).toHaveBeenCalledWith(storageData); - expect(listObjectsInBucket).toHaveBeenCalledWith(mockBucketUrl); - expect(visionModeration.saveResultsToJson).toHaveBeenCalledWith( - mockModerationResults, - './results.json', - ); - expect(result).toEqual({ - containsAbuse: 'true', - abuseResultsFile: './results.json', - }); - }); - - it('should return false for containsAbuse if no abusive content is detected', async () => { - const mockBucketUrl = { - protocol: 'http:', - host: 'test-bucket.s3.amazonaws.com', - pathname: '/images', - }; - const mockObjectKeys = ['image1.jpg']; - const mockModerationResults = [ - { - imageUrl: 'http://test.com/image1.jpg', - moderationResult: { adult: 'LIKELY' }, - }, - ]; - - (generateBucketUrl as jest.Mock).mockReturnValue(mockBucketUrl); - (listObjectsInBucket as jest.Mock).mockResolvedValue(mockObjectKeys); - jest - .spyOn(visionModeration, 'analyzeImagesForModeration') - .mockResolvedValue(mockModerationResults); - - const storageData: StorageDataDto = { - provider: StorageProviders.AWS, - region: AWSRegions.AF_SOUTH_1, - bucketName: 'test-bucket', - path: 'images', - }; - - const result = await visionModeration.processDataset(storageData); - - expect(result.containsAbuse).toBe('false'); - }); - - it('should throw an error if processing fails', async () => { - (listObjectsInBucket as jest.Mock).mockRejectedValue( - new Error('Error listing objects'), - ); - - const storageData: StorageDataDto = { - provider: StorageProviders.AWS, - region: AWSRegions.AF_SOUTH_1, - bucketName: 'test-bucket', - path: 'images', - }; - - await expect( - visionModeration.processDataset(storageData), - ).rejects.toThrow(ErrorCommon.ErrorProcessingDataset); - }); - }); -}); diff --git a/packages/examples/gcv/src/utils/vision.ts b/packages/examples/gcv/src/utils/vision.ts deleted file mode 100644 index d4c2203ab4..0000000000 --- a/packages/examples/gcv/src/utils/vision.ts +++ /dev/null @@ -1,124 +0,0 @@ -import * as fs from 'fs'; -import { ImageAnnotatorClient, protos } from '@google-cloud/vision'; -import { generateBucketUrl, listObjectsInBucket } from './storage'; -import { StorageDataDto } from '../dto/storage'; -import { ErrorCommon } from '../constants/errors'; - -export class VisionModeration { - private visionClient: ImageAnnotatorClient; - - constructor(projectId: string, privateKey: string, clientEmail: string) { - this.visionClient = new ImageAnnotatorClient({ - projectId, - credentials: { - private_key: privateKey, - client_email: clientEmail, - }, - }); - } - - /** - * Analyze a batch of remote images for moderation using Google Cloud Vision API. - */ - public async analyzeImagesForModeration(imageUrls: string[]): Promise { - const batchRequest: protos.google.cloud.vision.v1.IBatchAnnotateImagesRequest = - { - requests: imageUrls.map((imageUrl) => ({ - image: { source: { imageUri: imageUrl } }, - features: [{ type: 'SAFE_SEARCH_DETECTION' }], - })), - }; - - try { - const [responses]: any = - await this.visionClient.batchAnnotateImages(batchRequest); - console.log(responses); - return responses.responses - .map( - ( - response: protos.google.cloud.vision.v1.IAnnotateImageResponse, - index: number, - ) => { - const safeSearch = response.safeSearchAnnotation; - if (safeSearch) { - return { - imageUrl: imageUrls[index], - moderationResult: { - adult: safeSearch.adult, - violence: safeSearch.violence, - racy: safeSearch.racy, - spoof: safeSearch.spoof, - medical: safeSearch.medical, - }, - }; - } else { - console.error( - `No safeSearchAnnotation found for the image: ${imageUrls[index]}`, - ); - return null; - } - }, - ) - .filter((result: any) => result !== null); - } catch (error) { - console.error('Error analyzing images:', error); - return []; - } - } - - /** - * Save the moderation results to a JSON file. - * This method now overwrites the existing results.json file each time it's called. - */ - public saveResultsToJson(results: any[], jsonFilePath: string) { - // Directly write the results to the JSON file, overwriting any existing data - fs.writeFileSync(jsonFilePath, JSON.stringify(results, null, 2)); - } - - /** - * Process all images in a dataset for moderation. - */ - public async processDataset( - storageData: StorageDataDto, - ): Promise<{ containsAbuse: string; abuseResultsFile: string }> { - const resultsJsonPath = './results.json'; - let containsAbuse = false; - - try { - const bucketUrl = generateBucketUrl(storageData); - const objectKeys = await listObjectsInBucket(bucketUrl); - const imageUrls = objectKeys.map( - (objectKey) => - `${bucketUrl.protocol}//${bucketUrl.host}${bucketUrl.pathname}/${objectKey}`, - ); - - const moderationResults = - await this.analyzeImagesForModeration(imageUrls); - - if (moderationResults.length > 0) { - this.saveResultsToJson(moderationResults, resultsJsonPath); - - containsAbuse = moderationResults.some( - (result) => - result.moderationResult.adult === 'VERY_LIKELY' || - result.moderationResult.racy === 'VERY_LIKELY' || - result.moderationResult.violence === 'VERY_LIKELY' || - result.moderationResult.spoof === 'VERY_LIKELY' || - result.moderationResult.medical === 'VERY_LIKELY', - ); - - console.log('Processing completed. Results saved to results.json.'); - } else { - console.log('No valid moderation results to save.'); - } - } catch (error) { - console.error('Error processing dataset:', error); - throw new Error(ErrorCommon.ErrorProcessingDataset); - } - - return { - containsAbuse: containsAbuse ? 'true' : 'false', - abuseResultsFile: resultsJsonPath, - }; - } -} diff --git a/packages/examples/gcv/tsconfig.json b/packages/examples/gcv/tsconfig.json deleted file mode 100644 index a1e0956c73..0000000000 --- a/packages/examples/gcv/tsconfig.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "compilerOptions": { - "module": "commonjs", - "declaration": true, - "removeComments": true, - "emitDecoratorMetadata": true, - "experimentalDecorators": true, - "allowSyntheticDefaultImports": true, - "allowJs": true, - "target": "es2023", - "sourceMap": true, - "outDir": "./dist", - "baseUrl": "./", - "incremental": true, - "skipLibCheck": true, - "strictNullChecks": true, - "noImplicitAny": true, - "strictBindCallApply": true, - "forceConsistentCasingInFileNames": true, - "noFallthroughCasesInSwitch": true, - "esModuleInterop": true - } -} diff --git a/yarn.lock b/yarn.lock index 895b8c2d02..2553906294 100644 --- a/yarn.lock +++ b/yarn.lock @@ -412,29 +412,6 @@ __metadata: languageName: unknown linkType: soft -"@apps/gcv@workspace:packages/examples/gcv": - version: 0.0.0-use.local - resolution: "@apps/gcv@workspace:packages/examples/gcv" - dependencies: - "@eslint/js": "npm:^10.0.1" - "@google-cloud/vision": "npm:^4.3.2" - "@nestjs/common": "npm:^11.1.12" - "@types/xml2js": "npm:^0.4.14" - axios: "npm:^1.7.2" - dotenv: "npm:^17.2.2" - eslint: "npm:^10.0.3" - eslint-config-prettier: "npm:^10.1.8" - eslint-plugin-jest: "npm:^29.15.0" - eslint-plugin-prettier: "npm:^5.5.5" - globals: "npm:^16.3.0" - jest: "npm:^29.7.0" - prettier: "npm:^3.8.1" - typescript: "npm:^5.8.3" - typescript-eslint: "npm:^8.57.0" - xml2js: "npm:^0.6.2" - languageName: unknown - linkType: soft - "@apps/human-app-frontend@workspace:packages/apps/human-app/frontend": version: 0.0.0-use.local resolution: "@apps/human-app-frontend@workspace:packages/apps/human-app/frontend"