From 235cf702f40caae3a81cf1c65d0143d52334c256 Mon Sep 17 00:00:00 2001 From: Dmitry Nechay Date: Mon, 30 Jun 2025 16:05:05 +0300 Subject: [PATCH 1/3] fix: missing passport-jwt --- packages/apps/human-app/server/package.json | 5 +++++ yarn.lock | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/packages/apps/human-app/server/package.json b/packages/apps/human-app/server/package.json index aeb54182fc..393f154574 100644 --- a/packages/apps/human-app/server/package.json +++ b/packages/apps/human-app/server/package.json @@ -31,10 +31,12 @@ "@nestjs/common": "^10.2.7", "@nestjs/config": "^3.1.1", "@nestjs/core": "^10.3.10", + "@nestjs/passport": "^10.0.0", "@nestjs/platform-express": "^10.3.10", "@nestjs/schedule": "^4.0.1", "@nestjs/swagger": "^7.4.2", "@nestjs/terminus": "^11.0.0", + "@types/passport-jwt": "^4.0.1", "axios": "^1.7.2", "cache-manager": "^5.4.0", "cache-manager-redis-yet": "^5.1.5", @@ -45,6 +47,8 @@ "jsonwebtoken": "^9.0.2", "jwt-decode": "^4.0.0", "lodash": "^4.17.21", + "passport": "^0.7.0", + "passport-jwt": "^4.0.1", "reflect-metadata": "^0.2.2", "rxjs": "^7.2.0" }, @@ -57,6 +61,7 @@ "@types/jsonwebtoken": "^9.0.7", "@types/lodash": "^4.17.14", "@types/node": "22.10.5", + "@types/passport": "^0", "@types/supertest": "^2.0.15", "@typescript-eslint/eslint-plugin": "^5.0.0", "@typescript-eslint/parser": "^5.0.0", diff --git a/yarn.lock b/yarn.lock index c0928c81b4..901b3ef69d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4274,6 +4274,7 @@ __metadata: "@nestjs/common": "npm:^10.2.7" "@nestjs/config": "npm:^3.1.1" "@nestjs/core": "npm:^10.3.10" + "@nestjs/passport": "npm:^10.0.0" "@nestjs/platform-express": "npm:^10.3.10" "@nestjs/schedule": "npm:^4.0.1" "@nestjs/schematics": "npm:^11.0.2" @@ -4285,6 +4286,8 @@ __metadata: "@types/jsonwebtoken": "npm:^9.0.7" "@types/lodash": "npm:^4.17.14" "@types/node": "npm:22.10.5" + "@types/passport": "npm:^0" + "@types/passport-jwt": "npm:^4.0.1" "@types/supertest": "npm:^2.0.15" "@typescript-eslint/eslint-plugin": "npm:^5.0.0" "@typescript-eslint/parser": "npm:^5.0.0" @@ -4303,6 +4306,8 @@ __metadata: jwt-decode: "npm:^4.0.0" lodash: "npm:^4.17.21" nock: "npm:^13.5.1" + passport: "npm:^0.7.0" + passport-jwt: "npm:^4.0.1" prettier: "npm:^3.4.2" reflect-metadata: "npm:^0.2.2" rxjs: "npm:^7.2.0" From f928c5fe308ef4cec4a8f50d382d0dbf1d215026 Mon Sep 17 00:00:00 2001 From: Dmitry Nechay Date: Mon, 30 Jun 2025 17:05:57 +0300 Subject: [PATCH 2/3] feat: sanitize oracle url after fetch --- .../src/common/guards/strategy/jwt.http.ts | 1 - .../integrations/kv-store/kv-store.gateway.ts | 24 ++++---- .../kv-store/spec/kv-store.gateway.spec.ts | 56 ++++++++++--------- 3 files changed, 45 insertions(+), 36 deletions(-) diff --git a/packages/apps/human-app/server/src/common/guards/strategy/jwt.http.ts b/packages/apps/human-app/server/src/common/guards/strategy/jwt.http.ts index 56e70eb47e..1a06cd60ee 100644 --- a/packages/apps/human-app/server/src/common/guards/strategy/jwt.http.ts +++ b/packages/apps/human-app/server/src/common/guards/strategy/jwt.http.ts @@ -30,7 +30,6 @@ export class JwtHttpStrategy extends PassportStrategy(Strategy, 'jwt-http') { ); done(null, pubKey); } catch (error) { - console.error(error); done(error); } }, diff --git a/packages/apps/human-app/server/src/integrations/kv-store/kv-store.gateway.ts b/packages/apps/human-app/server/src/integrations/kv-store/kv-store.gateway.ts index 5341d6f44a..d57967369f 100644 --- a/packages/apps/human-app/server/src/integrations/kv-store/kv-store.gateway.ts +++ b/packages/apps/human-app/server/src/integrations/kv-store/kv-store.gateway.ts @@ -27,13 +27,13 @@ export class KvStoreGateway { if (cachedData) { return cachedData; } - let fetchedData: string; + let oracleUrl: string; try { const runner = new ethers.JsonRpcProvider(this.configService.rpcUrl); const network = await runner.provider?.getNetwork(); const chainId: ChainId = Number(network?.chainId); - fetchedData = await KVStoreUtils.get(chainId, address, KVStoreKeys.url); + oracleUrl = await KVStoreUtils.get(chainId, address, KVStoreKeys.url); } catch (e) { if (e.toString().includes('Error: Invalid address')) { throw new HttpException( @@ -46,19 +46,23 @@ export class KvStoreGateway { ); } } - if (!fetchedData || fetchedData === '') { + + if (!oracleUrl || oracleUrl === '') { throw new HttpException( `Unable to retrieve URL from address: ${address}`, 400, ); - } else { - await this.cacheManager.set( - key, - fetchedData, - this.configService.cacheTtlExchangeOracleUrl, - ); - return fetchedData; } + + oracleUrl = oracleUrl.replace(/\/$/, ''); + + await this.cacheManager.set( + key, + oracleUrl, + this.configService.cacheTtlExchangeOracleUrl, + ); + + return oracleUrl; } async getJobTypesByAddress( diff --git a/packages/apps/human-app/server/src/integrations/kv-store/spec/kv-store.gateway.spec.ts b/packages/apps/human-app/server/src/integrations/kv-store/spec/kv-store.gateway.spec.ts index 5818c72e22..19470e3bda 100644 --- a/packages/apps/human-app/server/src/integrations/kv-store/spec/kv-store.gateway.spec.ts +++ b/packages/apps/human-app/server/src/integrations/kv-store/spec/kv-store.gateway.spec.ts @@ -1,19 +1,12 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { KvStoreGateway } from '../kv-store.gateway'; -import { ChainId, KVStoreKeys, KVStoreUtils } from '@human-protocol/sdk'; -import { EnvironmentConfigService } from '../../../common/config/environment-config.service'; -import { CACHE_MANAGER } from '@nestjs/cache-manager'; -import { Cache } from 'cache-manager'; -import { ORACLE_URL_CACHE_KEY } from '../../../common/constants/cache'; - -const EXPECTED_URL = 'https://example.com'; jest.mock('@human-protocol/sdk', () => { const actualSdk = jest.requireActual('@human-protocol/sdk'); + const mockedSdk = jest.createMockFromModule< + typeof import('@human-protocol/sdk') + >('@human-protocol/sdk'); + return { ...actualSdk, - KVStoreUtils: { - get: jest.fn().mockResolvedValue('https://example.com'), - }, + KVStoreUtils: mockedSdk.KVStoreUtils, }; }); @@ -26,6 +19,7 @@ jest.mock('ethers', () => { }), }, }; + return { ...actualEthers, ethers: { @@ -35,10 +29,23 @@ jest.mock('ethers', () => { }; }); +import { ChainId, KVStoreKeys, KVStoreUtils } from '@human-protocol/sdk'; +import { CACHE_MANAGER } from '@nestjs/cache-manager'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { Cache } from 'cache-manager'; + +import { KvStoreGateway } from '../kv-store.gateway'; +import { EnvironmentConfigService } from '../../../common/config/environment-config.service'; +import { ORACLE_URL_CACHE_KEY } from '../../../common/constants/cache'; + +const EXPECTED_URL = 'https://example.com'; + +const mockedKvStoreUtils = jest.mocked(KVStoreUtils); + describe('KvStoreGateway', () => { let service: KvStoreGateway; let configService: EnvironmentConfigService; - let mockKVStoreUtils: any; let cacheManager: Cache & { get: jest.Mock; set: jest.Mock }; beforeEach(async () => { @@ -47,9 +54,6 @@ describe('KvStoreGateway', () => { cacheTtlExchangeOracleUrl: 2137, } as any; - mockKVStoreUtils = { - get: jest.fn(), - }; const cacheManagerMock = { get: jest.fn(), set: jest.fn(), @@ -61,36 +65,36 @@ describe('KvStoreGateway', () => { provide: EnvironmentConfigService, useValue: configService, }, - { - provide: KVStoreUtils, - useValue: mockKVStoreUtils, - }, { provide: CACHE_MANAGER, useValue: cacheManagerMock, }, ], }).compile(); + cacheManager = module.get(CACHE_MANAGER); configService = module.get(EnvironmentConfigService); service = module.get(KvStoreGateway); }); afterEach(async () => { - jest.restoreAllMocks(); jest.clearAllMocks(); }); + it('should be defined', () => { expect(service).toBeDefined(); }); + describe('getExchangeOracleUrlByAddress', () => { it('should get data from kvStoreUtils, if not cached', async () => { const testAddress = 'testAddress'; const cacheKey = `${ORACLE_URL_CACHE_KEY}:${testAddress}`; - const expectedData = EXPECTED_URL; - mockKVStoreUtils.get.mockResolvedValue(expectedData); + const expectedUrl = EXPECTED_URL; + mockedKvStoreUtils.get.mockResolvedValue(expectedUrl + '/'); cacheManager.get.mockResolvedValue(undefined); + const result = await service.getExchangeOracleUrlByAddress(testAddress); + expect(KVStoreUtils.get).toHaveBeenCalledWith( ChainId.LOCALHOST, testAddress, @@ -99,17 +103,19 @@ describe('KvStoreGateway', () => { expect(cacheManager.set).toHaveBeenCalledWith( cacheKey, - expectedData, + expectedUrl, configService.cacheTtlExchangeOracleUrl, ); expect(cacheManager.get).toHaveBeenCalledWith(cacheKey); - expect(result).toBe(expectedData); + expect(result).toBe(expectedUrl); }); + it('should get data from cache, if available', async () => { const testAddress = 'testAddress'; const cacheKey = `${ORACLE_URL_CACHE_KEY}:${testAddress}`; const expectedData = EXPECTED_URL; cacheManager.get.mockResolvedValue(expectedData); + const result = await service.getExchangeOracleUrlByAddress(testAddress); expect(KVStoreUtils.get).not.toHaveBeenCalled(); From fe3113aa17e33ad505f2749b61b7152b0b37b48f Mon Sep 17 00:00:00 2001 From: Dmitry Nechay Date: Mon, 30 Jun 2025 17:22:21 +0300 Subject: [PATCH 3/3] feat: add ttl to available jobs --- packages/apps/human-app/server/src/app.module.ts | 1 + .../common/config/environment-config.service.ts | 14 ++++++++++++++ .../jobs-discovery/jobs-discovery.service.ts | 7 ++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/apps/human-app/server/src/app.module.ts b/packages/apps/human-app/server/src/app.module.ts index 6099bfa28e..a693e40889 100644 --- a/packages/apps/human-app/server/src/app.module.ts +++ b/packages/apps/human-app/server/src/app.module.ts @@ -96,6 +96,7 @@ const JOI_BOOLEAN_STRING_SCHEMA = Joi.string().valid('true', 'false'); CACHE_TTL_DAILY_HMT_SPENT: Joi.number(), CACHE_TTL_HCAPTCHA_USER_STATS: Joi.number(), CACHE_TTL_ORACLE_DISCOVERY: Joi.number(), + CACHE_TTL_ORACLE_AVAILABLE_JOBS: Joi.number(), JOB_ASSIGNMENTS_DATA_RETENTION_DAYS: Joi.number(), CACHE_TTL_EXCHANGE_ORACLE_URL: Joi.number(), CACHE_TTL_EXCHANGE_ORACLE_REGISTRATION_NEEDED: Joi.number(), diff --git a/packages/apps/human-app/server/src/common/config/environment-config.service.ts b/packages/apps/human-app/server/src/common/config/environment-config.service.ts index af1cd562f6..54980729d8 100644 --- a/packages/apps/human-app/server/src/common/config/environment-config.service.ts +++ b/packages/apps/human-app/server/src/common/config/environment-config.service.ts @@ -6,6 +6,7 @@ const DEFAULT_CACHE_TTL_HCAPTCHA_USER_STATS = 12 * 60 * 60; const DEFAULT_CACHE_TTL_ORACLE_STATS = 12 * 60 * 60; const DEFAULT_CACHE_TTL_USER_STATS = 15 * 60; const DEFAULT_CACHE_TTL_ORACLE_DISCOVERY = 24 * 60 * 60; +const DEFAULT_CACHE_TTL_ORACLE_AVAILABLE_JOBS = 2 * 60; const DEFAULT_JOB_ASSIGNMENTS_DATA_RETENTION_DAYS = 45; const DEFAULT_CACHE_TTL_DAILY_HMT_SPENT = 24 * 60 * 60; const DEFAULT_CORS_ALLOWED_ORIGIN = 'http://localhost:5173'; @@ -164,6 +165,19 @@ export class EnvironmentConfigService { ); } + /** + * The cache time-to-live (TTL) for oracle's available jobs. + * Default: 2 minutes + */ + get cacheTtlOracleAvailableJobs(): number { + return ( + this.configService.get( + 'CACHE_TTL_ORACLE_AVAILABLE_JOBS', + DEFAULT_CACHE_TTL_ORACLE_AVAILABLE_JOBS, + ) * 1000 + ); + } + /** * Number of days without updates assignments data is retained. * Default: 45 days diff --git a/packages/apps/human-app/server/src/modules/jobs-discovery/jobs-discovery.service.ts b/packages/apps/human-app/server/src/modules/jobs-discovery/jobs-discovery.service.ts index 8e590812d6..8a62cad6b2 100644 --- a/packages/apps/human-app/server/src/modules/jobs-discovery/jobs-discovery.service.ts +++ b/packages/apps/human-app/server/src/modules/jobs-discovery/jobs-discovery.service.ts @@ -124,6 +124,11 @@ export class JobsDiscoveryService { jobs: DiscoveredJob[], ): Promise { const cacheKey = JobsDiscoveryService.makeCacheKeyForOracle(oracleAddress); - await this.cacheManager.set(cacheKey, jobs); + + await this.cacheManager.set( + cacheKey, + jobs, + this.configService.cacheTtlOracleAvailableJobs, + ); } }