From 8158718f8421903c5df639bb5731f6361d39685f Mon Sep 17 00:00:00 2001 From: Joachim Date: Tue, 17 Mar 2026 09:20:21 +0100 Subject: [PATCH 01/10] feat: Implement AI scoring system with provider integration and criteria management - Added new API endpoints for AI provider configuration and application analysis. - Introduced scoring criteria generation from job descriptions using AI. - Implemented bulk creation and updating of scoring criteria for jobs. - Enhanced error handling for missing configurations and application data. - Created database migrations for new tables related to AI scoring and analysis runs. - Developed utility functions for interacting with various AI providers (OpenAI, Anthropic, Google). - Established structured output schemas for scoring evaluations and criteria definitions. --- app/components/CandidateDetailSidebar.vue | 21 +- app/components/PipelineCard.vue | 8 +- app/components/ScoreBreakdown.vue | 239 ++++++ app/components/SettingsSidebar.vue | 9 +- app/pages/dashboard/jobs/[id]/index.vue | 49 +- app/pages/dashboard/jobs/new.vue | 534 ++++++++++++- app/pages/dashboard/settings/ai.vue | 349 ++++++++ package-lock.json | 140 ++++ package.json | 4 + .../api/ai-config/generate-criteria.post.ts | 45 ++ server/api/ai-config/index.get.ts | 32 + server/api/ai-config/index.post.ts | 94 +++ server/api/ai-config/providers.get.ts | 11 + server/api/applications/[id]/analyze.post.ts | 208 +++++ server/api/applications/[id]/scores.get.ts | 73 ++ server/api/jobs/[id]/analyze-all.post.ts | 43 + .../api/jobs/[id]/criteria/generate.post.ts | 69 ++ server/api/jobs/[id]/criteria/index.get.ts | 34 + server/api/jobs/[id]/criteria/index.patch.ts | 49 ++ server/api/jobs/[id]/criteria/index.post.ts | 62 ++ .../0015_closed_william_stryker.sql | 79 ++ .../migrations/meta/0015_snapshot.json | 754 ++++++++++++++++-- server/database/migrations/meta/_journal.json | 7 + server/database/schema/app.ts | 140 ++++ server/utils/ai/provider.ts | 143 ++++ server/utils/ai/scoring.ts | 291 +++++++ server/utils/schemas/scoring.ts | 61 ++ shared/permissions.ts | 4 + 28 files changed, 3449 insertions(+), 103 deletions(-) create mode 100644 app/components/ScoreBreakdown.vue create mode 100644 app/pages/dashboard/settings/ai.vue create mode 100644 server/api/ai-config/generate-criteria.post.ts create mode 100644 server/api/ai-config/index.get.ts create mode 100644 server/api/ai-config/index.post.ts create mode 100644 server/api/ai-config/providers.get.ts create mode 100644 server/api/applications/[id]/analyze.post.ts create mode 100644 server/api/applications/[id]/scores.get.ts create mode 100644 server/api/jobs/[id]/analyze-all.post.ts create mode 100644 server/api/jobs/[id]/criteria/generate.post.ts create mode 100644 server/api/jobs/[id]/criteria/index.get.ts create mode 100644 server/api/jobs/[id]/criteria/index.patch.ts create mode 100644 server/api/jobs/[id]/criteria/index.post.ts create mode 100644 server/database/migrations/0015_closed_william_stryker.sql create mode 100644 server/utils/ai/provider.ts create mode 100644 server/utils/ai/scoring.ts create mode 100644 server/utils/schemas/scoring.ts diff --git a/app/components/CandidateDetailSidebar.vue b/app/components/CandidateDetailSidebar.vue index 8c281e95..6bb42df7 100644 --- a/app/components/CandidateDetailSidebar.vue +++ b/app/components/CandidateDetailSidebar.vue @@ -2,7 +2,7 @@ import { X, User, Calendar, Clock, Hash, MessageSquare, FileText, ExternalLink, Mail, Phone, Upload, Download, Eye, Trash2, - ArrowLeft, AlertTriangle, + ArrowLeft, AlertTriangle, Brain, } from 'lucide-vue-next' import { usePreviewReadOnly } from '~/composables/usePreviewReadOnly' @@ -32,7 +32,7 @@ const hasSubNav = computed(() => { // Tabs // ───────────────────────────────────────────── -const activeTab = ref<'overview' | 'documents' | 'responses'>('overview') +const activeTab = ref<'overview' | 'documents' | 'responses' | 'ai_analysis'>('overview') // ───────────────────────────────────────────── // Fetch application detail @@ -414,6 +414,16 @@ function formatInterviewDate(dateStr: string) { > Responses ({{ responsesCount }}) + @@ -857,6 +867,13 @@ function formatInterviewDate(dateStr: string) { + + + +
+ +
+ diff --git a/app/components/PipelineCard.vue b/app/components/PipelineCard.vue index 4282c97a..476108a8 100644 --- a/app/components/PipelineCard.vue +++ b/app/components/PipelineCard.vue @@ -62,7 +62,13 @@ const transitionClasses: Record = { {{ new Date(createdAt).toLocaleDateString() }} - + {{ score }}pts diff --git a/app/components/ScoreBreakdown.vue b/app/components/ScoreBreakdown.vue new file mode 100644 index 00000000..edbe1df9 --- /dev/null +++ b/app/components/ScoreBreakdown.vue @@ -0,0 +1,239 @@ + + + diff --git a/app/components/SettingsSidebar.vue b/app/components/SettingsSidebar.vue index 58f07933..188c7a84 100644 --- a/app/components/SettingsSidebar.vue +++ b/app/components/SettingsSidebar.vue @@ -1,6 +1,6 @@ + + diff --git a/package-lock.json b/package-lock.json index 0e03e141..5c42f027 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,12 +9,16 @@ "version": "1.1.0", "hasInstallScript": true, "dependencies": { + "@ai-sdk/anthropic": "^3.0.58", + "@ai-sdk/google": "^3.0.43", + "@ai-sdk/openai": "^3.0.41", "@aws-sdk/client-s3": "^3.995.0", "@nuxtjs/i18n": "^10.2.3", "@nuxtjs/mdc": "^0.20.1", "@posthog/nuxt": "^1.5.82", "@tailwindcss/typography": "^0.5.19", "@tailwindcss/vite": "^4.1.18", + "ai": "^6.0.116", "better-auth": "^1.4.18", "better-sqlite3": "^12.6.2", "drizzle-orm": "^0.45.1", @@ -41,6 +45,100 @@ "wait-on": "^9.0.4" } }, + "node_modules/@ai-sdk/anthropic": { + "version": "3.0.58", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.58.tgz", + "integrity": "sha512-/53SACgmVukO4bkms4dpxpRlYhW8Ct6QZRe6sj1Pi5H00hYhxIrqfiLbZBGxkdRvjsBQeP/4TVGsXgH5rQeb8Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "3.0.66", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.66.tgz", + "integrity": "sha512-SIQ0YY0iMuv+07HLsZ+bB990zUJ6S4ujORAh+Jv1V2KGNn73qQKnGO0JBk+w+Res8YqOFSycwDoWcFlQrVxS4A==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19", + "@vercel/oidc": "3.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/google": { + "version": "3.0.43", + "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-3.0.43.tgz", + "integrity": "sha512-NGCgP5g8HBxrNdxvF8Dhww+UKfqAkZAmyYBvbu9YLoBkzAmGKDBGhVptN/oXPB5Vm0jggMdoLycZ8JReQM8Zqg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai": { + "version": "3.0.41", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.41.tgz", + "integrity": "sha512-IZ42A+FO+vuEQCVNqlnAPYQnnUpUfdJIwn1BEDOBywiEHa23fw7PahxVtlX9zm3/zMvTW4JKPzWyvAgDu+SQ2A==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.8.tgz", + "integrity": "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "4.0.19", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.19.tgz", + "integrity": "sha512-3eG55CrSWCu2SXlqq2QCsFjo3+E7+Gmg7i/oRVoSZzIodTuDSfLb3MRje67xE9RFea73Zao7Lm4mADIfUETKGg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@aws-crypto/crc32": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", @@ -6803,6 +6901,15 @@ "node": ">=20" } }, + "node_modules/@vercel/oidc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz", + "integrity": "sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, "node_modules/@vitejs/plugin-vue": { "version": "6.0.4", "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-6.0.4.tgz", @@ -7180,6 +7287,24 @@ "node": ">= 14" } }, + "node_modules/ai": { + "version": "6.0.116", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.116.tgz", + "integrity": "sha512-7yM+cTmyRLeNIXwt4Vj+mrrJgVQ9RMIW5WO0ydoLoYkewIvsMcvUmqS4j2RJTUXaF1HphwmSKUMQ/HypNRGOmA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "3.0.66", + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19", + "@opentelemetry/api": "1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/ajv": { "version": "6.14.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", @@ -9554,6 +9679,15 @@ "bare-events": "^2.7.0" } }, + "node_modules/eventsource-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/execa": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", @@ -11231,6 +11365,12 @@ "license": "MIT", "peer": true }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", diff --git a/package.json b/package.json index d11f5c56..a189facb 100644 --- a/package.json +++ b/package.json @@ -22,12 +22,16 @@ "i18n:crowdin:sync": "npm run i18n:crowdin:upload && npm run i18n:crowdin:download" }, "dependencies": { + "@ai-sdk/anthropic": "^3.0.58", + "@ai-sdk/google": "^3.0.43", + "@ai-sdk/openai": "^3.0.41", "@aws-sdk/client-s3": "^3.995.0", "@nuxtjs/i18n": "^10.2.3", "@nuxtjs/mdc": "^0.20.1", "@posthog/nuxt": "^1.5.82", "@tailwindcss/typography": "^0.5.19", "@tailwindcss/vite": "^4.1.18", + "ai": "^6.0.116", "better-auth": "^1.4.18", "better-sqlite3": "^12.6.2", "drizzle-orm": "^0.45.1", diff --git a/server/api/ai-config/generate-criteria.post.ts b/server/api/ai-config/generate-criteria.post.ts new file mode 100644 index 00000000..a3a46208 --- /dev/null +++ b/server/api/ai-config/generate-criteria.post.ts @@ -0,0 +1,45 @@ +import { eq } from 'drizzle-orm' +import { z } from 'zod' +import { aiConfig } from '../../database/schema' +import { generateCriteriaFromDescription } from '../../utils/ai/scoring' + +const bodySchema = z.object({ + title: z.string().min(1).max(200), + description: z.string().min(1).max(50000), +}) + +/** + * POST /api/ai-config/generate-criteria + * Generate scoring criteria from a job title + description using AI. + * Does NOT require a saved job — used during job creation flow. + */ +export default defineEventHandler(async (event) => { + const session = await requirePermission(event, { scoring: ['create'] }) + const orgId = session.session.activeOrganizationId + + const body = await readValidatedBody(event, bodySchema.parse) + + const config = await db.query.aiConfig.findFirst({ + where: eq(aiConfig.organizationId, orgId), + }) + if (!config) { + throw createError({ + statusCode: 422, + statusMessage: 'AI provider not configured. Set up your AI provider in Settings → AI first.', + }) + } + + const criteria = await generateCriteriaFromDescription( + { + provider: config.provider as 'openai' | 'anthropic' | 'google' | 'openai_compatible', + model: config.model, + apiKeyEncrypted: config.apiKeyEncrypted, + baseUrl: config.baseUrl, + maxTokens: config.maxTokens, + }, + body.title, + body.description, + ) + + return { criteria, source: 'ai' } +}) diff --git a/server/api/ai-config/index.get.ts b/server/api/ai-config/index.get.ts new file mode 100644 index 00000000..bf7da122 --- /dev/null +++ b/server/api/ai-config/index.get.ts @@ -0,0 +1,32 @@ +import { eq } from 'drizzle-orm' +import { aiConfig } from '../../database/schema' + +/** + * GET /api/ai-config + * Fetch the organization's AI provider configuration. + * Returns config WITHOUT the API key (security: never expose encrypted keys). + * Includes a boolean `hasApiKey` indicating whether a key is configured. + */ +export default defineEventHandler(async (event) => { + const session = await requirePermission(event, { scoring: ['read'] }) + const orgId = session.session.activeOrganizationId + + const config = await db.query.aiConfig.findFirst({ + where: eq(aiConfig.organizationId, orgId), + columns: { + id: true, + provider: true, + model: true, + baseUrl: true, + maxTokens: true, + apiKeyEncrypted: true, + createdAt: true, + updatedAt: true, + }, + }) + + if (!config) return null + + const { apiKeyEncrypted, ...rest } = config + return { ...rest, hasApiKey: Boolean(apiKeyEncrypted) } +}) diff --git a/server/api/ai-config/index.post.ts b/server/api/ai-config/index.post.ts new file mode 100644 index 00000000..65bc1548 --- /dev/null +++ b/server/api/ai-config/index.post.ts @@ -0,0 +1,94 @@ +import { eq } from 'drizzle-orm' +import { aiConfig } from '../../database/schema' +import { createAiConfigSchema } from '../../utils/schemas/scoring' +import { encrypt } from '../../utils/encryption' + +/** + * POST /api/ai-config + * Create or update the organization's AI provider configuration. + * API key is encrypted at rest using AES-256-GCM before storage. + */ +export default defineEventHandler(async (event) => { + const session = await requirePermission(event, { scoring: ['create'] }) + const orgId = session.session.activeOrganizationId + const body = await readValidatedBody(event, createAiConfigSchema.parse) + + const existing = await db.query.aiConfig.findFirst({ + where: eq(aiConfig.organizationId, orgId), + columns: { id: true, apiKeyEncrypted: true }, + }) + + if (existing) { + const updateData: Record = { + provider: body.provider, + model: body.model, + baseUrl: body.baseUrl ?? null, + maxTokens: body.maxTokens, + updatedAt: new Date(), + } + + // Only re-encrypt if a new key was provided + if (body.apiKey) { + updateData.apiKeyEncrypted = encrypt(body.apiKey, env.BETTER_AUTH_SECRET) + } + + const [updated] = await db.update(aiConfig) + .set(updateData) + .where(eq(aiConfig.id, existing.id)) + .returning({ + id: aiConfig.id, + provider: aiConfig.provider, + model: aiConfig.model, + baseUrl: aiConfig.baseUrl, + maxTokens: aiConfig.maxTokens, + }) + + recordActivity({ + organizationId: orgId, + actorId: session.user.id, + action: 'updated', + resourceType: 'aiConfig', + resourceId: updated!.id, + }) + + return { config: updated } + } + + // New config requires an API key + if (!body.apiKey) { + throw createError({ + statusCode: 422, + statusMessage: 'API key is required for initial AI configuration.', + }) + } + + const apiKeyEncrypted = encrypt(body.apiKey, env.BETTER_AUTH_SECRET) + + const [created] = await db.insert(aiConfig) + .values({ + organizationId: orgId, + provider: body.provider, + model: body.model, + apiKeyEncrypted, + baseUrl: body.baseUrl ?? null, + maxTokens: body.maxTokens, + }) + .returning({ + id: aiConfig.id, + provider: aiConfig.provider, + model: aiConfig.model, + baseUrl: aiConfig.baseUrl, + maxTokens: aiConfig.maxTokens, + }) + + recordActivity({ + organizationId: orgId, + actorId: session.user.id, + action: 'created', + resourceType: 'aiConfig', + resourceId: created!.id, + }) + + setResponseStatus(event, 201) + return { config: created } +}) diff --git a/server/api/ai-config/providers.get.ts b/server/api/ai-config/providers.get.ts new file mode 100644 index 00000000..a6568499 --- /dev/null +++ b/server/api/ai-config/providers.get.ts @@ -0,0 +1,11 @@ +import { PROVIDER_REGISTRY } from '../../utils/ai/provider' + +/** + * GET /api/ai-config/providers + * Returns the list of supported AI providers with their model options and setup URLs. + * Public within org (no secrets exposed). + */ +export default defineEventHandler(async (event) => { + await requirePermission(event, { scoring: ['read'] }) + return PROVIDER_REGISTRY +}) diff --git a/server/api/applications/[id]/analyze.post.ts b/server/api/applications/[id]/analyze.post.ts new file mode 100644 index 00000000..e22c28e1 --- /dev/null +++ b/server/api/applications/[id]/analyze.post.ts @@ -0,0 +1,208 @@ +import { eq, and } from 'drizzle-orm' +import { + application, aiConfig, scoringCriterion, criterionScore, + analysisRun, document, candidate, +} from '../../../database/schema' +import { scoreApplication, computeCompositeScore } from '../../../utils/ai/scoring' +import type { CriterionDefinition } from '../../../utils/ai/scoring' +import { z } from 'zod' + +const paramsSchema = z.object({ id: z.string().min(1) }) + +/** + * POST /api/applications/:id/analyze + * Run AI analysis on a single application. Scores the candidate against job criteria. + * Stores individual criterion scores + composite score + audit trail. + */ +export default defineEventHandler(async (event) => { + const session = await requirePermission(event, { scoring: ['create'] }) + const orgId = session.session.activeOrganizationId + const { id: applicationId } = await getValidatedRouterParams(event, paramsSchema.parse) + + // Fetch application with candidate, job, and documents + const app = await db.query.application.findFirst({ + where: and(eq(application.id, applicationId), eq(application.organizationId, orgId)), + with: { + candidate: { + columns: { id: true, firstName: true, lastName: true }, + }, + job: { + columns: { id: true, title: true, description: true }, + }, + }, + }) + + if (!app) { + throw createError({ statusCode: 404, statusMessage: 'Application not found' }) + } + + // Fetch AI config + const config = await db.query.aiConfig.findFirst({ + where: eq(aiConfig.organizationId, orgId), + }) + if (!config) { + throw createError({ + statusCode: 422, + statusMessage: 'AI provider not configured. Set up your AI provider in Settings first.', + }) + } + + // Fetch scoring criteria for this job + const criteria = await db.select().from(scoringCriterion) + .where(and( + eq(scoringCriterion.jobId, app.job.id), + eq(scoringCriterion.organizationId, orgId), + )) + + if (criteria.length === 0) { + throw createError({ + statusCode: 422, + statusMessage: 'No scoring criteria defined for this job. Add criteria first.', + }) + } + + // Fetch candidate documents (resume text) + const docs = await db.select({ + parsedContent: document.parsedContent, + type: document.type, + }) + .from(document) + .where(and( + eq(document.candidateId, app.candidate.id), + eq(document.organizationId, orgId), + )) + + const resumeDoc = docs.find(d => d.type === 'resume') + const resumeText = resumeDoc?.parsedContent + ? (typeof resumeDoc.parsedContent === 'string' + ? resumeDoc.parsedContent + : JSON.stringify(resumeDoc.parsedContent)) + : null + + if (!resumeText) { + throw createError({ + statusCode: 422, + statusMessage: 'No parsed resume found for this candidate. Upload a resume first.', + }) + } + + if (!app.job.description) { + throw createError({ + statusCode: 422, + statusMessage: 'Job description is required for AI analysis.', + }) + } + + const criteriaDefinitions: CriterionDefinition[] = criteria.map(c => ({ + key: c.key, + name: c.name, + description: c.description, + category: c.category, + maxScore: c.maxScore, + weight: c.weight, + })) + + const providerConfig = { + provider: config.provider as 'openai' | 'anthropic' | 'openai_compatible', + model: config.model, + apiKeyEncrypted: config.apiKeyEncrypted, + baseUrl: config.baseUrl, + maxTokens: config.maxTokens, + } + + let result + try { + result = await scoreApplication(providerConfig, { + jobTitle: app.job.title, + jobDescription: app.job.description, + criteria: criteriaDefinitions, + resumeText, + coverLetterText: app.coverLetterText, + applicationNotes: app.notes, + }) + } catch (err: any) { + // Record failed analysis run + await db.insert(analysisRun).values({ + organizationId: orgId, + applicationId, + status: 'failed', + provider: config.provider, + model: config.model, + criteriaSnapshot: criteriaDefinitions as any, + errorMessage: err?.message ?? 'Unknown error', + scoredById: session.user.id, + }) + + throw createError({ + statusCode: 502, + statusMessage: `AI analysis failed: ${err?.message ?? 'Unknown error'}`, + }) + } + + // Compute composite score + const compositeScore = computeCompositeScore(criteriaDefinitions, result.scoring.evaluations) + + // Delete previous scores for this application (replace strategy) + await db.delete(criterionScore) + .where(and( + eq(criterionScore.applicationId, applicationId), + eq(criterionScore.organizationId, orgId), + )) + + // Insert individual criterion scores + const scoreValues = result.scoring.evaluations.map(evaluation => ({ + organizationId: orgId, + applicationId, + criterionKey: evaluation.criterionKey, + maxScore: evaluation.maxScore, + applicantScore: evaluation.applicantScore, + confidence: evaluation.confidence, + evidence: evaluation.evidence, + strengths: evaluation.strengths, + gaps: evaluation.gaps, + })) + + if (scoreValues.length > 0) { + await db.insert(criterionScore).values(scoreValues) + } + + // Update application composite score + await db.update(application) + .set({ score: compositeScore, updatedAt: new Date() }) + .where(eq(application.id, applicationId)) + + // Record analysis run + const [run] = await db.insert(analysisRun).values({ + organizationId: orgId, + applicationId, + status: 'completed', + provider: config.provider, + model: config.model, + criteriaSnapshot: criteriaDefinitions as any, + compositeScore, + promptTokens: result.usage.promptTokens, + completionTokens: result.usage.completionTokens, + scoredById: session.user.id, + }).returning() + + recordActivity({ + organizationId: orgId, + actorId: session.user.id, + action: 'scored', + resourceType: 'application', + resourceId: applicationId, + metadata: { + compositeScore, + model: config.model, + criterionCount: result.scoring.evaluations.length, + }, + }) + + return { + compositeScore, + evaluations: result.scoring.evaluations, + summary: result.scoring.summary, + analysisRunId: run!.id, + usage: result.usage, + } +}) diff --git a/server/api/applications/[id]/scores.get.ts b/server/api/applications/[id]/scores.get.ts new file mode 100644 index 00000000..ad8ba026 --- /dev/null +++ b/server/api/applications/[id]/scores.get.ts @@ -0,0 +1,73 @@ +import { eq, and, desc } from 'drizzle-orm' +import { application, criterionScore, analysisRun, scoringCriterion } from '../../../database/schema' +import { z } from 'zod' + +const paramsSchema = z.object({ id: z.string().min(1) }) + +/** + * GET /api/applications/:id/scores + * Get the score breakdown for an application, including individual criterion scores + * and the most recent analysis run details. + */ +export default defineEventHandler(async (event) => { + const session = await requirePermission(event, { scoring: ['read'] }) + const orgId = session.session.activeOrganizationId + const { id: applicationId } = await getValidatedRouterParams(event, paramsSchema.parse) + + // Verify application belongs to org + const app = await db.query.application.findFirst({ + where: and(eq(application.id, applicationId), eq(application.organizationId, orgId)), + columns: { id: true, score: true, jobId: true }, + }) + if (!app) { + throw createError({ statusCode: 404, statusMessage: 'Application not found' }) + } + + // Fetch criterion scores with joined criterion metadata + const rawScores = await db.select({ + criterionKey: criterionScore.criterionKey, + maxScore: criterionScore.maxScore, + score: criterionScore.applicantScore, + confidence: criterionScore.confidence, + evidence: criterionScore.evidence, + strengths: criterionScore.strengths, + gaps: criterionScore.gaps, + criterionName: scoringCriterion.name, + weight: scoringCriterion.weight, + category: scoringCriterion.category, + }) + .from(criterionScore) + .leftJoin(scoringCriterion, and( + eq(scoringCriterion.jobId, app.jobId), + eq(scoringCriterion.key, criterionScore.criterionKey), + )) + .where(and( + eq(criterionScore.applicationId, applicationId), + eq(criterionScore.organizationId, orgId), + )) + + // Fetch latest analysis run + const [latestRun] = await db.select({ + id: analysisRun.id, + status: analysisRun.status, + provider: analysisRun.provider, + model: analysisRun.model, + compositeScore: analysisRun.compositeScore, + promptTokens: analysisRun.promptTokens, + completionTokens: analysisRun.completionTokens, + createdAt: analysisRun.createdAt, + }) + .from(analysisRun) + .where(and( + eq(analysisRun.applicationId, applicationId), + eq(analysisRun.organizationId, orgId), + )) + .orderBy(desc(analysisRun.createdAt)) + .limit(1) + + return { + compositeScore: app.score, + scores: rawScores, + latestRun: latestRun ?? null, + } +}) diff --git a/server/api/jobs/[id]/analyze-all.post.ts b/server/api/jobs/[id]/analyze-all.post.ts new file mode 100644 index 00000000..cf14c1e2 --- /dev/null +++ b/server/api/jobs/[id]/analyze-all.post.ts @@ -0,0 +1,43 @@ +import { eq, and, isNull } from 'drizzle-orm' +import { application, job } from '../../../database/schema' +import { z } from 'zod' + +const paramsSchema = z.object({ id: z.string().min(1) }) + +/** + * POST /api/jobs/:id/analyze-all + * Trigger AI analysis for all unscored applications for a job. + * Returns the list of application IDs that were queued for analysis. + * Client should call /api/applications/:id/analyze for each one. + * This keeps the operation simple and avoids long-running server requests. + */ +export default defineEventHandler(async (event) => { + const session = await requirePermission(event, { scoring: ['create'] }) + const orgId = session.session.activeOrganizationId + const { id: jobId } = await getValidatedRouterParams(event, paramsSchema.parse) + + // Verify job belongs to org + const jobRecord = await db.query.job.findFirst({ + where: and(eq(job.id, jobId), eq(job.organizationId, orgId)), + columns: { id: true }, + }) + if (!jobRecord) { + throw createError({ statusCode: 404, statusMessage: 'Job not found' }) + } + + // Find all applications without scores + const unscoredApps = await db.select({ + id: application.id, + }) + .from(application) + .where(and( + eq(application.jobId, jobId), + eq(application.organizationId, orgId), + isNull(application.score), + )) + + return { + applicationIds: unscoredApps.map(a => a.id), + total: unscoredApps.length, + } +}) diff --git a/server/api/jobs/[id]/criteria/generate.post.ts b/server/api/jobs/[id]/criteria/generate.post.ts new file mode 100644 index 00000000..b4551b40 --- /dev/null +++ b/server/api/jobs/[id]/criteria/generate.post.ts @@ -0,0 +1,69 @@ +import { eq, and } from 'drizzle-orm' +import { aiConfig, job } from '../../../../database/schema' +import { generateCriteriaSchema } from '../../../../utils/schemas/scoring' +import { generateCriteriaFromDescription, PREMADE_CRITERIA } from '../../../../utils/ai/scoring' +import { z } from 'zod' + +const paramsSchema = z.object({ id: z.string().min(1) }) + +/** + * POST /api/jobs/:id/criteria/generate + * Generate scoring criteria from a pre-made template or by AI analysis of the job description. + * Does NOT persist — returns generated criteria for the client to review before saving. + */ +export default defineEventHandler(async (event) => { + const session = await requirePermission(event, { scoring: ['create'] }) + const orgId = session.session.activeOrganizationId + const { id: jobId } = await getValidatedRouterParams(event, paramsSchema.parse) + const body = await readValidatedBody(event, generateCriteriaSchema.parse) + + // Verify job belongs to org + const jobRecord = await db.query.job.findFirst({ + where: and(eq(job.id, jobId), eq(job.organizationId, orgId)), + columns: { id: true, title: true, description: true }, + }) + if (!jobRecord) { + throw createError({ statusCode: 404, statusMessage: 'Job not found' }) + } + + // If a pre-made template is specified, return it directly + if (body.template) { + const template = PREMADE_CRITERIA[body.template] + if (!template) { + throw createError({ statusCode: 400, statusMessage: 'Unknown template' }) + } + return { criteria: template, source: 'template' } + } + + // Otherwise generate from job description using AI + if (!jobRecord.description) { + throw createError({ + statusCode: 422, + statusMessage: 'Job description is required to generate AI criteria. Add a description first.', + }) + } + + const config = await db.query.aiConfig.findFirst({ + where: eq(aiConfig.organizationId, orgId), + }) + if (!config) { + throw createError({ + statusCode: 422, + statusMessage: 'AI provider not configured. Set up your AI provider in Settings first.', + }) + } + + const criteria = await generateCriteriaFromDescription( + { + provider: config.provider as 'openai' | 'anthropic' | 'openai_compatible', + model: config.model, + apiKeyEncrypted: config.apiKeyEncrypted, + baseUrl: config.baseUrl, + maxTokens: config.maxTokens, + }, + jobRecord.title, + jobRecord.description, + ) + + return { criteria, source: 'ai' } +}) diff --git a/server/api/jobs/[id]/criteria/index.get.ts b/server/api/jobs/[id]/criteria/index.get.ts new file mode 100644 index 00000000..bea57add --- /dev/null +++ b/server/api/jobs/[id]/criteria/index.get.ts @@ -0,0 +1,34 @@ +import { eq, and, asc } from 'drizzle-orm' +import { scoringCriterion, job } from '../../../../database/schema' +import { z } from 'zod' + +const paramsSchema = z.object({ id: z.string().min(1) }) + +/** + * GET /api/jobs/:id/criteria + * List all scoring criteria for a job, ordered by displayOrder. + */ +export default defineEventHandler(async (event) => { + const session = await requirePermission(event, { scoring: ['read'] }) + const orgId = session.session.activeOrganizationId + const { id: jobId } = await getValidatedRouterParams(event, paramsSchema.parse) + + // Verify job belongs to org + const jobRecord = await db.query.job.findFirst({ + where: and(eq(job.id, jobId), eq(job.organizationId, orgId)), + columns: { id: true }, + }) + if (!jobRecord) { + throw createError({ statusCode: 404, statusMessage: 'Job not found' }) + } + + const criteria = await db.select() + .from(scoringCriterion) + .where(and( + eq(scoringCriterion.jobId, jobId), + eq(scoringCriterion.organizationId, orgId), + )) + .orderBy(asc(scoringCriterion.displayOrder)) + + return { criteria } +}) diff --git a/server/api/jobs/[id]/criteria/index.patch.ts b/server/api/jobs/[id]/criteria/index.patch.ts new file mode 100644 index 00000000..9b197a5d --- /dev/null +++ b/server/api/jobs/[id]/criteria/index.patch.ts @@ -0,0 +1,49 @@ +import { eq, and } from 'drizzle-orm' +import { scoringCriterion, job } from '../../../../database/schema' +import { updateWeightsSchema } from '../../../../utils/schemas/scoring' +import { z } from 'zod' + +const paramsSchema = z.object({ id: z.string().min(1) }) + +/** + * PATCH /api/jobs/:id/criteria + * Update weights for scoring criteria (slider adjustments). + */ +export default defineEventHandler(async (event) => { + const session = await requirePermission(event, { scoring: ['update'] }) + const orgId = session.session.activeOrganizationId + const { id: jobId } = await getValidatedRouterParams(event, paramsSchema.parse) + const body = await readValidatedBody(event, updateWeightsSchema.parse) + + // Verify job belongs to org + const jobRecord = await db.query.job.findFirst({ + where: and(eq(job.id, jobId), eq(job.organizationId, orgId)), + columns: { id: true }, + }) + if (!jobRecord) { + throw createError({ statusCode: 404, statusMessage: 'Job not found' }) + } + + // Update each criterion weight + await Promise.all( + body.weights.map(w => + db.update(scoringCriterion) + .set({ weight: w.weight, updatedAt: new Date() }) + .where(and( + eq(scoringCriterion.jobId, jobId), + eq(scoringCriterion.organizationId, orgId), + eq(scoringCriterion.key, w.key), + )), + ), + ) + + // Return updated criteria + const criteria = await db.select() + .from(scoringCriterion) + .where(and( + eq(scoringCriterion.jobId, jobId), + eq(scoringCriterion.organizationId, orgId), + )) + + return { criteria } +}) diff --git a/server/api/jobs/[id]/criteria/index.post.ts b/server/api/jobs/[id]/criteria/index.post.ts new file mode 100644 index 00000000..4a33d1c2 --- /dev/null +++ b/server/api/jobs/[id]/criteria/index.post.ts @@ -0,0 +1,62 @@ +import { eq, and } from 'drizzle-orm' +import { scoringCriterion, job } from '../../../../database/schema' +import { bulkCriteriaSchema } from '../../../../utils/schemas/scoring' +import { z } from 'zod' + +const paramsSchema = z.object({ id: z.string().min(1) }) + +/** + * POST /api/jobs/:id/criteria + * Bulk-create scoring criteria for a job. Replaces any existing criteria. + */ +export default defineEventHandler(async (event) => { + const session = await requirePermission(event, { scoring: ['create'] }) + const orgId = session.session.activeOrganizationId + const { id: jobId } = await getValidatedRouterParams(event, paramsSchema.parse) + const body = await readValidatedBody(event, bulkCriteriaSchema.parse) + + // Verify job belongs to org + const jobRecord = await db.query.job.findFirst({ + where: and(eq(job.id, jobId), eq(job.organizationId, orgId)), + columns: { id: true }, + }) + if (!jobRecord) { + throw createError({ statusCode: 404, statusMessage: 'Job not found' }) + } + + // Delete existing criteria for this job (replace strategy) + await db.delete(scoringCriterion) + .where(and( + eq(scoringCriterion.jobId, jobId), + eq(scoringCriterion.organizationId, orgId), + )) + + // Insert new criteria + const values = body.criteria.map((c, index) => ({ + organizationId: orgId, + jobId, + key: c.key, + name: c.name, + description: c.description ?? null, + category: c.category, + maxScore: c.maxScore, + weight: c.weight, + displayOrder: c.displayOrder ?? index, + })) + + const created = await db.insert(scoringCriterion) + .values(values) + .returning() + + recordActivity({ + organizationId: orgId, + actorId: session.user.id, + action: 'created', + resourceType: 'scoringCriteria', + resourceId: jobId, + metadata: { count: created.length }, + }) + + setResponseStatus(event, 201) + return { criteria: created } +}) diff --git a/server/database/migrations/0015_closed_william_stryker.sql b/server/database/migrations/0015_closed_william_stryker.sql new file mode 100644 index 00000000..cd61c47f --- /dev/null +++ b/server/database/migrations/0015_closed_william_stryker.sql @@ -0,0 +1,79 @@ +CREATE TYPE "public"."analysis_run_status" AS ENUM('completed', 'failed', 'partial');--> statement-breakpoint +CREATE TYPE "public"."criterion_category" AS ENUM('technical', 'experience', 'soft_skills', 'education', 'culture', 'custom');--> statement-breakpoint +ALTER TYPE "public"."activity_action" ADD VALUE 'scored';--> statement-breakpoint +CREATE TABLE "ai_config" ( + "id" text PRIMARY KEY NOT NULL, + "organization_id" text NOT NULL, + "provider" text DEFAULT 'openai' NOT NULL, + "model" text DEFAULT 'gpt-4o-mini' NOT NULL, + "api_key_encrypted" text NOT NULL, + "base_url" text, + "max_tokens" integer DEFAULT 4096 NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "analysis_run" ( + "id" text PRIMARY KEY NOT NULL, + "organization_id" text NOT NULL, + "application_id" text NOT NULL, + "status" "analysis_run_status" DEFAULT 'completed' NOT NULL, + "provider" text NOT NULL, + "model" text NOT NULL, + "criteria_snapshot" jsonb, + "composite_score" integer, + "prompt_tokens" integer, + "completion_tokens" integer, + "raw_response" jsonb, + "error_message" text, + "scored_by_id" text NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "criterion_score" ( + "id" text PRIMARY KEY NOT NULL, + "organization_id" text NOT NULL, + "application_id" text NOT NULL, + "criterion_key" text NOT NULL, + "max_score" integer NOT NULL, + "applicant_score" integer NOT NULL, + "confidence" integer NOT NULL, + "evidence" text NOT NULL, + "strengths" jsonb, + "gaps" jsonb, + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "scoring_criterion" ( + "id" text PRIMARY KEY NOT NULL, + "organization_id" text NOT NULL, + "job_id" text NOT NULL, + "key" text NOT NULL, + "name" text NOT NULL, + "description" text, + "category" "criterion_category" DEFAULT 'custom' NOT NULL, + "max_score" integer DEFAULT 10 NOT NULL, + "weight" integer DEFAULT 50 NOT NULL, + "display_order" integer DEFAULT 0 NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "ai_config" ADD CONSTRAINT "ai_config_organization_id_organization_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organization"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "analysis_run" ADD CONSTRAINT "analysis_run_organization_id_organization_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organization"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "analysis_run" ADD CONSTRAINT "analysis_run_application_id_application_id_fk" FOREIGN KEY ("application_id") REFERENCES "public"."application"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "analysis_run" ADD CONSTRAINT "analysis_run_scored_by_id_user_id_fk" FOREIGN KEY ("scored_by_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "criterion_score" ADD CONSTRAINT "criterion_score_organization_id_organization_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organization"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "criterion_score" ADD CONSTRAINT "criterion_score_application_id_application_id_fk" FOREIGN KEY ("application_id") REFERENCES "public"."application"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "scoring_criterion" ADD CONSTRAINT "scoring_criterion_organization_id_organization_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organization"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "scoring_criterion" ADD CONSTRAINT "scoring_criterion_job_id_job_id_fk" FOREIGN KEY ("job_id") REFERENCES "public"."job"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "ai_config_organization_id_idx" ON "ai_config" USING btree ("organization_id");--> statement-breakpoint +CREATE INDEX "analysis_run_organization_id_idx" ON "analysis_run" USING btree ("organization_id");--> statement-breakpoint +CREATE INDEX "analysis_run_application_id_idx" ON "analysis_run" USING btree ("application_id");--> statement-breakpoint +CREATE INDEX "analysis_run_created_at_idx" ON "analysis_run" USING btree ("created_at");--> statement-breakpoint +CREATE INDEX "criterion_score_organization_id_idx" ON "criterion_score" USING btree ("organization_id");--> statement-breakpoint +CREATE INDEX "criterion_score_application_id_idx" ON "criterion_score" USING btree ("application_id");--> statement-breakpoint +CREATE UNIQUE INDEX "criterion_score_app_criterion_idx" ON "criterion_score" USING btree ("application_id","criterion_key");--> statement-breakpoint +CREATE INDEX "scoring_criterion_organization_id_idx" ON "scoring_criterion" USING btree ("organization_id");--> statement-breakpoint +CREATE INDEX "scoring_criterion_job_id_idx" ON "scoring_criterion" USING btree ("job_id");--> statement-breakpoint +CREATE UNIQUE INDEX "scoring_criterion_job_key_idx" ON "scoring_criterion" USING btree ("job_id","key"); \ No newline at end of file diff --git a/server/database/migrations/meta/0015_snapshot.json b/server/database/migrations/meta/0015_snapshot.json index a011c774..0db9a667 100644 --- a/server/database/migrations/meta/0015_snapshot.json +++ b/server/database/migrations/meta/0015_snapshot.json @@ -1,6 +1,6 @@ { - "id": "a7084d77-1232-4856-8bd2-37219ba1e9f3", - "prevId": "1d7cc5c7-139f-4bdf-a5d4-e6620d1189f1", + "id": "b6eb4402-158c-42e9-8a61-a3e72fe04ff9", + "prevId": "a7084d77-1232-4856-8bd2-37219ba1e9f3", "version": "7", "dialect": "postgresql", "tables": { @@ -827,6 +827,294 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.ai_config": { + "name": "ai_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'openai'" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'gpt-4o-mini'" + }, + "api_key_encrypted": { + "name": "api_key_encrypted", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "max_tokens": { + "name": "max_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 4096 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "ai_config_organization_id_idx": { + "name": "ai_config_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "ai_config_organization_id_organization_id_fk": { + "name": "ai_config_organization_id_organization_id_fk", + "tableFrom": "ai_config", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.analysis_run": { + "name": "analysis_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "application_id": { + "name": "application_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "analysis_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'completed'" + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "criteria_snapshot": { + "name": "criteria_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "composite_score": { + "name": "composite_score", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "prompt_tokens": { + "name": "prompt_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "completion_tokens": { + "name": "completion_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "raw_response": { + "name": "raw_response", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scored_by_id": { + "name": "scored_by_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "analysis_run_organization_id_idx": { + "name": "analysis_run_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "analysis_run_application_id_idx": { + "name": "analysis_run_application_id_idx", + "columns": [ + { + "expression": "application_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "analysis_run_created_at_idx": { + "name": "analysis_run_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "analysis_run_organization_id_organization_id_fk": { + "name": "analysis_run_organization_id_organization_id_fk", + "tableFrom": "analysis_run", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "analysis_run_application_id_application_id_fk": { + "name": "analysis_run_application_id_application_id_fk", + "tableFrom": "analysis_run", + "tableTo": "application", + "columnsFrom": [ + "application_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "analysis_run_scored_by_id_user_id_fk": { + "name": "analysis_run_scored_by_id_user_id_fk", + "tableFrom": "analysis_run", + "tableTo": "user", + "columnsFrom": [ + "scored_by_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.application": { "name": "application", "schema": "", @@ -1239,28 +1527,158 @@ "name": "candidate_org_email_idx", "columns": [ { - "expression": "organization_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "email", + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "candidate_organization_id_organization_id_fk": { + "name": "candidate_organization_id_organization_id_fk", + "tableFrom": "candidate", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.comment": { + "name": "comment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "comment_target", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "comment_organization_id_idx": { + "name": "comment_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "comment_target_idx": { + "name": "comment_target_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "comment_author_id_idx": { + "name": "comment_author_id_idx", + "columns": [ + { + "expression": "author_id", "isExpression": false, "asc": true, "nulls": "last" } ], - "isUnique": true, + "isUnique": false, "concurrently": false, "method": "btree", "with": {} } }, "foreignKeys": { - "candidate_organization_id_organization_id_fk": { - "name": "candidate_organization_id_organization_id_fk", - "tableFrom": "candidate", + "comment_organization_id_organization_id_fk": { + "name": "comment_organization_id_organization_id_fk", + "tableFrom": "comment", "tableTo": "organization", "columnsFrom": [ "organization_id" @@ -1270,6 +1688,19 @@ ], "onDelete": "cascade", "onUpdate": "no action" + }, + "comment_author_id_user_id_fk": { + "name": "comment_author_id_user_id_fk", + "tableFrom": "comment", + "tableTo": "user", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" } }, "compositePrimaryKeys": {}, @@ -1278,8 +1709,8 @@ "checkConstraints": {}, "isRLSEnabled": false }, - "public.comment": { - "name": "comment", + "public.criterion_score": { + "name": "criterion_score", "schema": "", "columns": { "id": { @@ -1294,40 +1725,56 @@ "primaryKey": false, "notNull": true }, - "author_id": { - "name": "author_id", + "application_id": { + "name": "application_id", "type": "text", "primaryKey": false, "notNull": true }, - "target_type": { - "name": "target_type", - "type": "comment_target", - "typeSchema": "public", + "criterion_key": { + "name": "criterion_key", + "type": "text", "primaryKey": false, "notNull": true }, - "target_id": { - "name": "target_id", - "type": "text", + "max_score": { + "name": "max_score", + "type": "integer", "primaryKey": false, "notNull": true }, - "body": { - "name": "body", + "applicant_score": { + "name": "applicant_score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "confidence": { + "name": "confidence", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "evidence": { + "name": "evidence", "type": "text", "primaryKey": false, "notNull": true }, - "created_at": { - "name": "created_at", - "type": "timestamp", + "strengths": { + "name": "strengths", + "type": "jsonb", "primaryKey": false, - "notNull": true, - "default": "now()" + "notNull": false }, - "updated_at": { - "name": "updated_at", + "gaps": { + "name": "gaps", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", "type": "timestamp", "primaryKey": false, "notNull": true, @@ -1335,8 +1782,8 @@ } }, "indexes": { - "comment_organization_id_idx": { - "name": "comment_organization_id_idx", + "criterion_score_organization_id_idx": { + "name": "criterion_score_organization_id_idx", "columns": [ { "expression": "organization_id", @@ -1350,17 +1797,11 @@ "method": "btree", "with": {} }, - "comment_target_idx": { - "name": "comment_target_idx", + "criterion_score_application_id_idx": { + "name": "criterion_score_application_id_idx", "columns": [ { - "expression": "target_type", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "target_id", + "expression": "application_id", "isExpression": false, "asc": true, "nulls": "last" @@ -1371,26 +1812,32 @@ "method": "btree", "with": {} }, - "comment_author_id_idx": { - "name": "comment_author_id_idx", + "criterion_score_app_criterion_idx": { + "name": "criterion_score_app_criterion_idx", "columns": [ { - "expression": "author_id", + "expression": "application_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "criterion_key", "isExpression": false, "asc": true, "nulls": "last" } ], - "isUnique": false, + "isUnique": true, "concurrently": false, "method": "btree", "with": {} } }, "foreignKeys": { - "comment_organization_id_organization_id_fk": { - "name": "comment_organization_id_organization_id_fk", - "tableFrom": "comment", + "criterion_score_organization_id_organization_id_fk": { + "name": "criterion_score_organization_id_organization_id_fk", + "tableFrom": "criterion_score", "tableTo": "organization", "columnsFrom": [ "organization_id" @@ -1401,12 +1848,12 @@ "onDelete": "cascade", "onUpdate": "no action" }, - "comment_author_id_user_id_fk": { - "name": "comment_author_id_user_id_fk", - "tableFrom": "comment", - "tableTo": "user", + "criterion_score_application_id_application_id_fk": { + "name": "criterion_score_application_id_application_id_fk", + "tableFrom": "criterion_score", + "tableTo": "application", "columnsFrom": [ - "author_id" + "application_id" ], "columnsTo": [ "id" @@ -2680,6 +3127,177 @@ "policies": {}, "checkConstraints": {}, "isRLSEnabled": false + }, + "public.scoring_criterion": { + "name": "scoring_criterion", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "category": { + "name": "category", + "type": "criterion_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'custom'" + }, + "max_score": { + "name": "max_score", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10 + }, + "weight": { + "name": "weight", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 50 + }, + "display_order": { + "name": "display_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "scoring_criterion_organization_id_idx": { + "name": "scoring_criterion_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scoring_criterion_job_id_idx": { + "name": "scoring_criterion_job_id_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scoring_criterion_job_key_idx": { + "name": "scoring_criterion_job_key_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scoring_criterion_organization_id_organization_id_fk": { + "name": "scoring_criterion_organization_id_organization_id_fk", + "tableFrom": "scoring_criterion", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "scoring_criterion_job_id_job_id_fk": { + "name": "scoring_criterion_job_id_job_id_fk", + "tableFrom": "scoring_criterion", + "tableTo": "job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false } }, "enums": { @@ -2694,7 +3312,17 @@ "comment_added", "member_invited", "member_removed", - "member_role_changed" + "member_role_changed", + "scored" + ] + }, + "public.analysis_run_status": { + "name": "analysis_run_status", + "schema": "public", + "values": [ + "completed", + "failed", + "partial" ] }, "public.application_status": { @@ -2735,6 +3363,18 @@ "job" ] }, + "public.criterion_category": { + "name": "criterion_category", + "schema": "public", + "values": [ + "technical", + "experience", + "soft_skills", + "education", + "culture", + "custom" + ] + }, "public.document_type": { "name": "document_type", "schema": "public", diff --git a/server/database/migrations/meta/_journal.json b/server/database/migrations/meta/_journal.json index d6469ccb..7a3caba5 100644 --- a/server/database/migrations/meta/_journal.json +++ b/server/database/migrations/meta/_journal.json @@ -106,6 +106,13 @@ "when": 1773231779768, "tag": "0015_married_wallflower", "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1773733547482, + "tag": "0015_closed_william_stryker", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/database/schema/app.ts b/server/database/schema/app.ts index 82fe4b8e..cd299ee6 100644 --- a/server/database/schema/app.ts +++ b/server/database/schema/app.ts @@ -357,6 +357,19 @@ export const comment = pgTable('comment', { export const activityActionEnum = pgEnum('activity_action', [ 'created', 'updated', 'deleted', 'status_changed', 'comment_added', 'member_invited', 'member_removed', 'member_role_changed', + 'scored', +]) + +// ───────────────────────────────────────────── +// AI Scoring Enums +// ───────────────────────────────────────────── + +export const criterionCategoryEnum = pgEnum('criterion_category', [ + 'technical', 'experience', 'soft_skills', 'education', 'culture', 'custom', +]) + +export const analysisRunStatusEnum = pgEnum('analysis_run_status', [ + 'completed', 'failed', 'partial', ]) /** @@ -379,6 +392,108 @@ export const activityLog = pgTable('activity_log', { index('activity_log_created_at_idx').on(t.createdAt), ])) +// ───────────────────────────────────────────── +// AI Configuration & Scoring Tables +// ───────────────────────────────────────────── + +/** + * Per-organization AI provider configuration. + * API keys are encrypted at rest using AES-256-GCM (same as calendar tokens). + * Each org can configure their own provider, model, and API key. + */ +export const aiConfig = pgTable('ai_config', { + id: text('id').primaryKey().$defaultFn(() => crypto.randomUUID()), + organizationId: text('organization_id').notNull().references(() => organization.id, { onDelete: 'cascade' }), + provider: text('provider').notNull().default('openai'), + model: text('model').notNull().default('gpt-4o-mini'), + /** AES-256-GCM encrypted API key — NEVER returned to client */ + apiKeyEncrypted: text('api_key_encrypted').notNull(), + /** Optional base URL override (e.g. for Ollama or custom endpoints) */ + baseUrl: text('base_url'), + maxTokens: integer('max_tokens').notNull().default(4096), + createdAt: timestamp('created_at').notNull().defaultNow(), + updatedAt: timestamp('updated_at').notNull().defaultNow(), +}, (t) => ([ + uniqueIndex('ai_config_organization_id_idx').on(t.organizationId), +])) + +/** + * Per-job scoring criteria. Each criterion defines one dimension of evaluation. + * Weights are user-adjustable via sliders and used to compute weighted composite scores. + */ +export const scoringCriterion = pgTable('scoring_criterion', { + id: text('id').primaryKey().$defaultFn(() => crypto.randomUUID()), + organizationId: text('organization_id').notNull().references(() => organization.id, { onDelete: 'cascade' }), + jobId: text('job_id').notNull().references(() => job.id, { onDelete: 'cascade' }), + key: text('key').notNull(), + name: text('name').notNull(), + description: text('description'), + category: criterionCategoryEnum('category').notNull().default('custom'), + maxScore: integer('max_score').notNull().default(10), + /** Weight from 0–100, used by sliders. Default 50 = neutral. */ + weight: integer('weight').notNull().default(50), + displayOrder: integer('display_order').notNull().default(0), + createdAt: timestamp('created_at').notNull().defaultNow(), + updatedAt: timestamp('updated_at').notNull().defaultNow(), +}, (t) => ([ + index('scoring_criterion_organization_id_idx').on(t.organizationId), + index('scoring_criterion_job_id_idx').on(t.jobId), + uniqueIndex('scoring_criterion_job_key_idx').on(t.jobId, t.key), +])) + +/** + * Individual criterion scores computed by AI for each application. + * Stores the raw AI output including evidence and confidence. + */ +export const criterionScore = pgTable('criterion_score', { + id: text('id').primaryKey().$defaultFn(() => crypto.randomUUID()), + organizationId: text('organization_id').notNull().references(() => organization.id, { onDelete: 'cascade' }), + applicationId: text('application_id').notNull().references(() => application.id, { onDelete: 'cascade' }), + criterionKey: text('criterion_key').notNull(), + maxScore: integer('max_score').notNull(), + applicantScore: integer('applicant_score').notNull(), + /** Confidence from 0 to 100 (%). */ + confidence: integer('confidence').notNull(), + evidence: text('evidence').notNull(), + strengths: jsonb('strengths').$type(), + gaps: jsonb('gaps').$type(), + createdAt: timestamp('created_at').notNull().defaultNow(), +}, (t) => ([ + index('criterion_score_organization_id_idx').on(t.organizationId), + index('criterion_score_application_id_idx').on(t.applicationId), + uniqueIndex('criterion_score_app_criterion_idx').on(t.applicationId, t.criterionKey), +])) + +/** + * Audit trail for each AI scoring run. Captures the rubric snapshot, + * model used, token usage, and the raw LLM response for debugging. + */ +export const analysisRun = pgTable('analysis_run', { + id: text('id').primaryKey().$defaultFn(() => crypto.randomUUID()), + organizationId: text('organization_id').notNull().references(() => organization.id, { onDelete: 'cascade' }), + applicationId: text('application_id').notNull().references(() => application.id, { onDelete: 'cascade' }), + status: analysisRunStatusEnum('status').notNull().default('completed'), + /** Provider + model used for this run */ + provider: text('provider').notNull(), + model: text('model').notNull(), + /** Snapshot of criteria at score time for audit trail */ + criteriaSnapshot: jsonb('criteria_snapshot').$type[]>(), + /** Composite weighted score (0–100) */ + compositeScore: integer('composite_score'), + /** Token usage for cost tracking */ + promptTokens: integer('prompt_tokens'), + completionTokens: integer('completion_tokens'), + /** Raw LLM response for debugging (sanitized — no PII stored) */ + rawResponse: jsonb('raw_response'), + errorMessage: text('error_message'), + scoredById: text('scored_by_id').notNull().references(() => user.id, { onDelete: 'cascade' }), + createdAt: timestamp('created_at').notNull().defaultNow(), +}, (t) => ([ + index('analysis_run_organization_id_idx').on(t.organizationId), + index('analysis_run_application_id_idx').on(t.applicationId), + index('analysis_run_created_at_idx').on(t.createdAt), +])) + // ───────────────────────────────────────────── // Relations // ───────────────────────────────────────────── @@ -387,6 +502,7 @@ export const jobRelations = relations(job, ({ one, many }) => ({ organization: one(organization, { fields: [job.organizationId], references: [organization.id] }), applications: many(application), questions: many(jobQuestion), + scoringCriteria: many(scoringCriterion), })) export const candidateRelations = relations(candidate, ({ one, many }) => ({ @@ -401,6 +517,8 @@ export const applicationRelations = relations(application, ({ one, many }) => ({ job: one(job, { fields: [application.jobId], references: [job.id] }), responses: many(questionResponse), interviews: many(interview), + criterionScores: many(criterionScore), + analysisRuns: many(analysisRun), })) export const documentRelations = relations(document, ({ one }) => ({ @@ -454,3 +572,25 @@ export const emailTemplateRelations = relations(emailTemplate, ({ one }) => ({ export const calendarIntegrationRelations = relations(calendarIntegration, ({ one }) => ({ user: one(user, { fields: [calendarIntegration.userId], references: [user.id] }), })) + +// ─── AI Scoring Relations ────────────────────────────────────────── + +export const aiConfigRelations = relations(aiConfig, ({ one }) => ({ + organization: one(organization, { fields: [aiConfig.organizationId], references: [organization.id] }), +})) + +export const scoringCriterionRelations = relations(scoringCriterion, ({ one }) => ({ + organization: one(organization, { fields: [scoringCriterion.organizationId], references: [organization.id] }), + job: one(job, { fields: [scoringCriterion.jobId], references: [job.id] }), +})) + +export const criterionScoreRelations = relations(criterionScore, ({ one }) => ({ + organization: one(organization, { fields: [criterionScore.organizationId], references: [organization.id] }), + application: one(application, { fields: [criterionScore.applicationId], references: [application.id] }), +})) + +export const analysisRunRelations = relations(analysisRun, ({ one }) => ({ + organization: one(organization, { fields: [analysisRun.organizationId], references: [organization.id] }), + application: one(application, { fields: [analysisRun.applicationId], references: [application.id] }), + scoredBy: one(user, { fields: [analysisRun.scoredById], references: [user.id] }), +})) diff --git a/server/utils/ai/provider.ts b/server/utils/ai/provider.ts new file mode 100644 index 00000000..5ab3c34d --- /dev/null +++ b/server/utils/ai/provider.ts @@ -0,0 +1,143 @@ +/** + * AI Provider Abstraction Layer + * + * Supports OpenAI, Anthropic, and custom OpenAI-compatible endpoints. + * Credentials are decrypted per-request from the organization's AI config. + * Never logs or stores raw API keys — only encrypted values in the database. + */ +import { createOpenAI } from '@ai-sdk/openai' +import { createAnthropic } from '@ai-sdk/anthropic' +import { createGoogleGenerativeAI } from '@ai-sdk/google' +import { generateObject } from 'ai' +import type { z } from 'zod' +import { decrypt } from '../encryption' + +export type SupportedProvider = 'openai' | 'anthropic' | 'google' | 'openai_compatible' + +export interface ProviderConfig { + provider: SupportedProvider + model: string + apiKeyEncrypted: string + baseUrl?: string | null + maxTokens: number +} + +/** Well-known providers with links for obtaining API keys */ +export const PROVIDER_REGISTRY: Record = { + openai: { + name: 'OpenAI', + modelsUrl: 'https://platform.openai.com/docs/models', + apiKeyUrl: 'https://platform.openai.com/api-keys', + defaultModel: 'gpt-4.1-mini', + models: ['gpt-4.1', 'gpt-4.1-mini', 'gpt-4.1-nano', 'gpt-4o', 'gpt-4o-mini', 'o3', 'o3-mini', 'o4-mini'], + }, + anthropic: { + name: 'Anthropic', + modelsUrl: 'https://docs.anthropic.com/en/docs/about-claude/models', + apiKeyUrl: 'https://console.anthropic.com/settings/keys', + defaultModel: 'claude-sonnet-4-20250514', + models: ['claude-sonnet-4-20250514', 'claude-opus-4-20250514', 'claude-3-5-haiku-20241022'], + }, + google: { + name: 'Google AI (Gemini)', + modelsUrl: 'https://ai.google.dev/gemini-api/docs/models', + apiKeyUrl: 'https://aistudio.google.com/apikey', + defaultModel: 'gemini-2.5-flash', + models: ['gemini-2.5-pro', 'gemini-2.5-flash', 'gemini-2.0-flash', 'gemini-2.0-flash-lite'], + }, + openai_compatible: { + name: 'OpenAI Compatible', + modelsUrl: '', + apiKeyUrl: '', + defaultModel: '', + models: [], + }, +} + +/** + * Create a language model instance from encrypted config. + * Decrypts the API key just-in-time and never persists it in memory beyond the call. + */ +function createLanguageModel(config: ProviderConfig) { + const secret = env.BETTER_AUTH_SECRET + const apiKey = decrypt(config.apiKeyEncrypted, secret) + + if (!apiKey) { + throw createError({ + statusCode: 500, + statusMessage: 'Failed to decrypt AI API key. The key may be corrupted.', + }) + } + + switch (config.provider) { + case 'openai': + case 'openai_compatible': { + const openai = createOpenAI({ + apiKey, + ...(config.baseUrl ? { baseURL: config.baseUrl } : {}), + }) + return openai(config.model) + } + case 'anthropic': { + const anthropic = createAnthropic({ + apiKey, + ...(config.baseUrl ? { baseURL: config.baseUrl } : {}), + }) + return anthropic(config.model) + } + case 'google': { + const google = createGoogleGenerativeAI({ + apiKey, + ...(config.baseUrl ? { baseURL: config.baseUrl } : {}), + }) + return google(config.model) + } + default: + throw createError({ + statusCode: 400, + statusMessage: `Unsupported AI provider: ${config.provider}`, + }) + } +} + +/** + * Generate a structured JSON response from the AI provider. + * Uses Vercel AI SDK's `generateObject` for reliable schema-conformant output. + */ +export async function generateStructuredOutput( + config: ProviderConfig, + options: { + system: string + prompt: string + schema: z.ZodType + schemaName: string + schemaDescription?: string + }, +): Promise<{ object: T; usage: { promptTokens: number; completionTokens: number } }> { + const model = createLanguageModel(config) + + const result = await generateObject({ + model, + system: options.system, + prompt: options.prompt, + schema: options.schema, + schemaName: options.schemaName, + schemaDescription: options.schemaDescription, + maxTokens: config.maxTokens, + temperature: 0.1, + }) + + return { + object: result.object, + usage: { + promptTokens: result.usage.inputTokens ?? 0, + completionTokens: result.usage.outputTokens ?? 0, + }, + } +} diff --git a/server/utils/ai/scoring.ts b/server/utils/ai/scoring.ts new file mode 100644 index 00000000..15f15c18 --- /dev/null +++ b/server/utils/ai/scoring.ts @@ -0,0 +1,291 @@ +/** + * AI Scoring Engine + * + * Evaluates candidates against job-specific scoring criteria using LLMs. + * Produces structured, evidence-based scores with confidence ratings. + */ +import { z } from 'zod' +import { generateStructuredOutput, type ProviderConfig } from './provider' + +// ─── Scoring Output Schema ──────────────────────────────────────── + +/** Schema for a single criterion evaluation from the LLM */ +const criterionEvaluationSchema = z.object({ + criterionKey: z.string(), + maxScore: z.number().int().min(0), + applicantScore: z.number().int().min(0), + confidence: z.number().min(0).max(100).int(), + evidence: z.string(), + strengths: z.array(z.string()), + gaps: z.array(z.string()), +}) + +/** Full scoring response from the LLM */ +const scoringResponseSchema = z.object({ + evaluations: z.array(criterionEvaluationSchema), + summary: z.string(), +}) + +export type CriterionEvaluation = z.infer +export type ScoringResponse = z.infer + +// ─── Criterion Definition ───────────────────────────────────────── + +export interface CriterionDefinition { + key: string + name: string + description: string | null + category: string + maxScore: number + weight: number +} + +// ─── Pre-made Criteria Templates ────────────────────────────────── + +export const PREMADE_CRITERIA: Record = { + standard: [ + { + key: 'technical_skills', + name: 'Technical Skills', + description: 'Evaluate the candidate\'s technical competencies, tools, programming languages, and frameworks mentioned in their resume against the job requirements.', + category: 'technical', + maxScore: 10, + weight: 50, + }, + { + key: 'relevant_experience', + name: 'Relevant Experience', + description: 'Assess years and quality of experience directly relevant to the role. Consider industry, company size, and scope of responsibilities.', + category: 'experience', + maxScore: 10, + weight: 50, + }, + { + key: 'education_fit', + name: 'Education & Certifications', + description: 'Evaluate educational background and professional certifications relevant to the position requirements.', + category: 'education', + maxScore: 10, + weight: 30, + }, + ], + technical: [ + { + key: 'core_tech_stack', + name: 'Core Tech Stack Match', + description: 'How well the candidate\'s technical skills match the primary technologies required for this role.', + category: 'technical', + maxScore: 10, + weight: 70, + }, + { + key: 'system_design', + name: 'System Design & Architecture', + description: 'Evidence of system design experience, scalability thinking, and architectural decision-making.', + category: 'technical', + maxScore: 10, + weight: 50, + }, + { + key: 'engineering_practices', + name: 'Engineering Practices', + description: 'Testing, CI/CD, code review, documentation, and software development lifecycle experience.', + category: 'technical', + maxScore: 10, + weight: 40, + }, + { + key: 'relevant_experience', + name: 'Relevant Experience', + description: 'Years and depth of experience in similar roles, projects, or domains.', + category: 'experience', + maxScore: 10, + weight: 50, + }, + { + key: 'leadership_collab', + name: 'Leadership & Collaboration', + description: 'Evidence of mentoring, tech leadership, cross-team collaboration, and communication skills.', + category: 'soft_skills', + maxScore: 10, + weight: 30, + }, + ], + non_technical: [ + { + key: 'relevant_experience', + name: 'Relevant Experience', + description: 'Depth and breadth of experience directly applicable to the role responsibilities.', + category: 'experience', + maxScore: 10, + weight: 60, + }, + { + key: 'communication', + name: 'Communication Skills', + description: 'Evidence of written and verbal communication ability from resume quality, cover letter, and described accomplishments.', + category: 'soft_skills', + maxScore: 10, + weight: 50, + }, + { + key: 'domain_knowledge', + name: 'Domain Knowledge', + description: 'Relevant industry or domain expertise that demonstrates understanding of the business context.', + category: 'experience', + maxScore: 10, + weight: 40, + }, + { + key: 'education_fit', + name: 'Education & Certifications', + description: 'Educational background and certifications relevant to the position.', + category: 'education', + maxScore: 10, + weight: 30, + }, + { + key: 'culture_fit', + name: 'Culture & Values Alignment', + description: 'Indicators of alignment with company values, work style, and team culture based on career trajectory and interests.', + category: 'culture', + maxScore: 10, + weight: 30, + }, + ], +} + +// ─── Rubric Generation from Job Description ─────────────────────── + +const generatedCriteriaSchema = z.object({ + criteria: z.array(z.object({ + key: z.string(), + name: z.string(), + description: z.string(), + category: z.enum(['technical', 'experience', 'soft_skills', 'education', 'culture', 'custom']), + maxScore: z.literal(10), + suggestedWeight: z.number().int().min(10).max(100), + })), +}) + +/** + * Use AI to generate scoring criteria from a job description. + * Returns 4–6 criteria tailored to the specific role. + */ +export async function generateCriteriaFromDescription( + config: ProviderConfig, + jobTitle: string, + jobDescription: string, +): Promise { + const result = await generateStructuredOutput(config, { + system: `You are an expert HR analyst specializing in creating objective, unbiased candidate evaluation criteria. +Your task is to analyze a job description and create 4–6 measurable scoring criteria. + +Rules: +- Each criterion must be specific and measurable from a resume/CV +- Avoid criteria that could introduce bias (age, gender, ethnicity, disability) +- Focus on skills, experience, and qualifications that are directly relevant to the role +- Use clear, professional language +- Each key must be unique, lowercase, and use underscores (e.g. "react_expertise") +- Set suggestedWeight higher for more critical criteria (10–100 scale)`, + prompt: `Job Title: ${jobTitle}\n\nJob Description:\n${jobDescription}`, + schema: generatedCriteriaSchema, + schemaName: 'GeneratedCriteria', + schemaDescription: 'Scoring criteria generated from job description', + }) + + return result.object.criteria.map((c, i) => ({ + key: c.key, + name: c.name, + description: c.description, + category: c.category, + maxScore: c.maxScore, + weight: c.suggestedWeight, + })) +} + +// ─── Score Application ──────────────────────────────────────────── + +/** + * Score a single application against the job's scoring criteria. + * Returns structured evaluations for each criterion. + */ +export async function scoreApplication( + config: ProviderConfig, + params: { + jobTitle: string + jobDescription: string + criteria: CriterionDefinition[] + resumeText: string + coverLetterText?: string | null + applicationNotes?: string | null + }, +): Promise<{ scoring: ScoringResponse; usage: { promptTokens: number; completionTokens: number } }> { + const criteriaBlock = params.criteria + .map((c, i) => `${i + 1}. **${c.name}** (key: "${c.key}", max: ${c.maxScore})\n ${c.description ?? 'No description provided.'}`) + .join('\n\n') + + const candidateInfo = [ + `RESUME:\n${params.resumeText}`, + params.coverLetterText ? `\nCOVER LETTER:\n${params.coverLetterText}` : '', + params.applicationNotes ? `\nAPPLICATION NOTES:\n${params.applicationNotes}` : '', + ].filter(Boolean).join('\n') + + const result = await generateStructuredOutput(config, { + system: `You are an expert, unbiased candidate evaluator for an applicant tracking system. +Your task is to objectively evaluate a candidate against specific scoring criteria for a job. + +IMPORTANT RULES: +- Score ONLY based on evidence found in the provided materials (resume, cover letter, notes) +- If information for a criterion is missing, give a low score and note it in gaps +- Be fair and consistent — avoid bias based on name, gender, age, or background +- Confidence reflects how much relevant information was available (0–100) +- Evidence must cite specific details from the candidate's materials +- Each strength and gap must be a single, specific statement +- applicantScore must not exceed maxScore for each criterion +- Provide a brief summary of the overall evaluation`, + prompt: `JOB TITLE: ${params.jobTitle} + +JOB DESCRIPTION: +${params.jobDescription} + +SCORING CRITERIA: +${criteriaBlock} + +CANDIDATE MATERIALS: +${candidateInfo} + +Evaluate this candidate against each criterion. Return your evaluation.`, + schema: scoringResponseSchema, + schemaName: 'CandidateScoring', + schemaDescription: 'Structured candidate evaluation with per-criterion scores', + }) + + return { + scoring: result.object, + usage: result.usage, + } +} + +/** + * Compute a weighted composite score (0–100) from individual criterion scores. + */ +export function computeCompositeScore( + criteria: CriterionDefinition[], + evaluations: CriterionEvaluation[], +): number { + let totalWeightedScore = 0 + let totalWeight = 0 + + for (const criterion of criteria) { + const evaluation = evaluations.find(e => e.criterionKey === criterion.key) + if (!evaluation) continue + + const normalizedScore = (evaluation.applicantScore / evaluation.maxScore) * 100 + totalWeightedScore += normalizedScore * criterion.weight + totalWeight += criterion.weight + } + + if (totalWeight === 0) return 0 + return Math.round(totalWeightedScore / totalWeight) +} diff --git a/server/utils/schemas/scoring.ts b/server/utils/schemas/scoring.ts new file mode 100644 index 00000000..b424f6f1 --- /dev/null +++ b/server/utils/schemas/scoring.ts @@ -0,0 +1,61 @@ +import { z } from 'zod' + +// ─── AI Config Schemas ──────────────────────────────────────────── + +export const createAiConfigSchema = z.object({ + provider: z.enum(['openai', 'anthropic', 'google', 'openai_compatible']), + model: z.string().min(1).max(200), + apiKey: z.string().min(1).max(500).optional(), + baseUrl: z.string().url().max(500).nullish(), + maxTokens: z.number().int().min(256).max(32768).optional().default(4096), +}) + +export const updateAiConfigSchema = z.object({ + provider: z.enum(['openai', 'anthropic', 'google', 'openai_compatible']).optional(), + model: z.string().min(1).max(200).optional(), + apiKey: z.string().min(1).max(500).optional(), + baseUrl: z.string().url().max(500).nullish(), + maxTokens: z.number().int().min(256).max(32768).optional(), +}) + +// ─── Scoring Criterion Schemas ──────────────────────────────────── + +const criterionCategoryValues = ['technical', 'experience', 'soft_skills', 'education', 'culture', 'custom'] as const + +export const createCriterionSchema = z.object({ + key: z.string() + .min(1).max(100) + .regex(/^[a-z][a-z0-9_]*$/, 'Key must be lowercase alphanumeric with underscores, starting with a letter'), + name: z.string().min(1).max(200), + description: z.string().max(1000).nullish(), + category: z.enum(criterionCategoryValues).optional().default('custom'), + maxScore: z.number().int().min(1).max(100).optional().default(10), + weight: z.number().int().min(0).max(100).optional().default(50), + displayOrder: z.number().int().min(0).optional().default(0), +}) + +export const updateCriterionSchema = z.object({ + name: z.string().min(1).max(200).optional(), + description: z.string().max(1000).nullish(), + category: z.enum(criterionCategoryValues).optional(), + maxScore: z.number().int().min(1).max(100).optional(), + weight: z.number().int().min(0).max(100).optional(), + displayOrder: z.number().int().min(0).optional(), +}) + +export const bulkCriteriaSchema = z.object({ + criteria: z.array(createCriterionSchema).min(1).max(20), +}) + +export const updateWeightsSchema = z.object({ + weights: z.array(z.object({ + key: z.string().min(1).max(100), + weight: z.number().int().min(0).max(100), + })).min(1).max(20), +}) + +// ─── Generate Criteria Schema ───────────────────────────────────── + +export const generateCriteriaSchema = z.object({ + template: z.enum(['standard', 'technical', 'non_technical']).optional(), +}) diff --git a/shared/permissions.ts b/shared/permissions.ts index 37a788b6..438578a2 100644 --- a/shared/permissions.ts +++ b/shared/permissions.ts @@ -36,6 +36,7 @@ const atsStatements = { interview: ['create', 'read', 'update', 'delete'], emailTemplate: ['create', 'read', 'update', 'delete'], activityLog: ['read'], + scoring: ['create', 'read', 'update', 'delete'], } as const // ─── Merged statement (Better Auth defaults + ATS resources) ─────── @@ -63,6 +64,7 @@ export const owner = ac.newRole({ interview: ['create', 'read', 'update', 'delete'], emailTemplate: ['create', 'read', 'update', 'delete'], activityLog: ['read'], + scoring: ['create', 'read', 'update', 'delete'], }) export const admin = ac.newRole({ @@ -75,6 +77,7 @@ export const admin = ac.newRole({ interview: ['create', 'read', 'update', 'delete'], emailTemplate: ['create', 'read', 'update', 'delete'], activityLog: ['read'], + scoring: ['create', 'read', 'update', 'delete'], }) export const member = ac.newRole({ @@ -87,4 +90,5 @@ export const member = ac.newRole({ interview: ['create', 'read', 'update'], emailTemplate: ['create', 'read', 'update'], activityLog: ['read'], + scoring: ['create', 'read'], }) From 5222980ea02aa92fa44b048157717bc23c3e370a Mon Sep 17 00:00:00 2001 From: Joachim Date: Tue, 17 Mar 2026 10:29:34 +0100 Subject: [PATCH 02/10] feat: implement autoScoreApplication for AI-driven application scoring - Added a new utility function `autoScoreApplication` to handle AI scoring for job applications. - The function retrieves application details, AI configuration, and scoring criteria. - It processes the candidate's resume and job description to generate scores using an AI provider. - Handles errors gracefully and logs analysis runs for both successful and failed scoring attempts. - Updates application scores and records activity for auditing purposes. --- app/components/AppToasts.vue | 139 + app/components/CandidateDetailSidebar.vue | 9 +- app/composables/useJobs.ts | 1 + app/composables/usePermission.ts | 5 +- app/composables/useToast.ts | 94 + app/layouts/dashboard.vue | 1 + app/pages/dashboard/applications/[id].vue | 5 +- app/pages/dashboard/candidates/[id].vue | 9 +- app/pages/dashboard/interviews/[id].vue | 7 +- app/pages/dashboard/interviews/index.vue | 5 +- .../dashboard/jobs/[id]/application-form.vue | 2 +- app/pages/dashboard/jobs/[id]/index.vue | 90 +- app/pages/dashboard/jobs/new.vue | 80 +- app/pages/dashboard/settings/ai.vue | 54 +- server/api/jobs/[id].get.ts | 1 + server/api/jobs/[id].patch.ts | 1 + server/api/jobs/index.post.ts | 2 + server/api/public/jobs/[slug]/apply.post.ts | 13 +- .../database/migrations/0016_first_spyke.sql | 2 + .../migrations/meta/0016_snapshot.json | 3471 +++++++++++++++++ server/database/migrations/meta/_journal.json | 7 + server/database/schema/app.ts | 4 +- server/utils/ai/autoScore.ts | 143 + server/utils/ai/scoring.ts | 2 +- server/utils/schemas/job.ts | 3 + 25 files changed, 4053 insertions(+), 97 deletions(-) create mode 100644 app/components/AppToasts.vue create mode 100644 app/composables/useToast.ts create mode 100644 server/database/migrations/0016_first_spyke.sql create mode 100644 server/database/migrations/meta/0016_snapshot.json create mode 100644 server/utils/ai/autoScore.ts diff --git a/app/components/AppToasts.vue b/app/components/AppToasts.vue new file mode 100644 index 00000000..d078b63e --- /dev/null +++ b/app/components/AppToasts.vue @@ -0,0 +1,139 @@ + + + diff --git a/app/components/CandidateDetailSidebar.vue b/app/components/CandidateDetailSidebar.vue index 6bb42df7..33e8e1be 100644 --- a/app/components/CandidateDetailSidebar.vue +++ b/app/components/CandidateDetailSidebar.vue @@ -17,6 +17,7 @@ const emit = defineEmits<{ }>() const { handlePreviewReadOnlyError } = usePreviewReadOnly() +const toast = useToast() // Detect if the job sub-nav bar is visible (adds 40px / 2.5rem) const route = useRoute() @@ -120,7 +121,7 @@ async function handleTransition(newStatus: string) { emit('updated') } catch (err: any) { if (handlePreviewReadOnlyError(err)) return - alert(err.data?.statusMessage ?? 'Failed to update status') + toast.error('Failed to update status', { message: err.data?.statusMessage, statusCode: err.data?.statusCode }) } finally { isTransitioning.value = false } @@ -151,7 +152,7 @@ async function saveNotes() { isEditingNotes.value = false } catch (err: any) { if (handlePreviewReadOnlyError(err)) return - alert(err.data?.statusMessage ?? 'Failed to save notes') + toast.error('Failed to save notes', { message: err.data?.statusMessage, statusCode: err.data?.statusCode }) } finally { isSavingNotes.value = false } @@ -242,7 +243,7 @@ async function handleDownload(docId: string) { try { await downloadDocument(docId) } catch { - alert('Failed to download document') + toast.error('Failed to download document') } } @@ -255,7 +256,7 @@ async function handleDeleteDoc(docId: string) { showDocDeleteConfirm.value = null } catch (err: any) { if (handlePreviewReadOnlyError(err)) return - alert(err.data?.statusMessage ?? 'Failed to delete document') + toast.error('Failed to delete document', { message: err.data?.statusMessage, statusCode: err.data?.statusCode }) } finally { isDeletingDoc.value = false } diff --git a/app/composables/useJobs.ts b/app/composables/useJobs.ts index 08120123..b94af842 100644 --- a/app/composables/useJobs.ts +++ b/app/composables/useJobs.ts @@ -30,6 +30,7 @@ export function useJobs(options?: { type?: 'full_time' | 'part_time' | 'contract' | 'internship' requireResume?: boolean requireCoverLetter?: boolean + autoScoreOnApply?: boolean }) { try { const created = await $fetch('/api/jobs', { diff --git a/app/composables/usePermission.ts b/app/composables/usePermission.ts index f3bf9fd3..a7eced7a 100644 --- a/app/composables/usePermission.ts +++ b/app/composables/usePermission.ts @@ -38,6 +38,7 @@ type PermissionRequest = { */ export function usePermission(permissions: PermissionRequest) { const role = ref(null) + const isLoading = ref(true) // Fetch the active member's role and re-fetch when org changes const activeOrgState = authClient.useActiveOrganization() @@ -45,11 +46,13 @@ export function usePermission(permissions: PermissionRequest) { async function fetchRole() { // Reset immediately to avoid stale role from previous org (race condition) role.value = null + isLoading.value = true const { data, error } = await authClient.organization.getActiveMemberRole() if (!error) { role.value = data?.role ?? null } + isLoading.value = false } // Only fetch on the client — during SSR there is no window.location, @@ -72,5 +75,5 @@ export function usePermission(permissions: PermissionRequest) { }) }) - return { allowed, role: readonly(role) } + return { allowed, role: readonly(role), isLoading: readonly(isLoading) } } diff --git a/app/composables/useToast.ts b/app/composables/useToast.ts new file mode 100644 index 00000000..25009bc0 --- /dev/null +++ b/app/composables/useToast.ts @@ -0,0 +1,94 @@ +import type { PostHog } from 'posthog-js' + +export type ToastType = 'error' | 'success' | 'warning' | 'info' + +export interface Toast { + id: string + type: ToastType + title: string + message?: string + details?: string + link?: { label: string; href: string } + duration?: number +} + +const GITHUB_ISSUES_URL = 'https://github.com/reqcore-inc/reqcore/issues/new' + +function getPostHog(): PostHog | undefined { + try { + const $ph = (useNuxtApp() as Record).$posthog as (() => PostHog) | undefined + return $ph?.() + } catch { + return undefined + } +} + +let counter = 0 + +export function useToast() { + const toasts = useState('app-toasts', () => []) + + function add(toast: Omit) { + const id = `toast-${++counter}-${Date.now()}` + const entry: Toast = { id, ...toast } + toasts.value.push(entry) + + const duration = toast.duration ?? (toast.type === 'error' ? 8000 : 4000) + if (duration > 0) { + setTimeout(() => remove(id), duration) + } + + return id + } + + function remove(id: string) { + toasts.value = toasts.value.filter(t => t.id !== id) + } + + function clear() { + toasts.value = [] + } + + /** + * Show an error toast with a link to report the issue on GitHub. + * Also tracks the error in PostHog if the user has consented. + */ + function error(title: string, opts?: { message?: string; details?: string; statusCode?: number; path?: string }) { + if (import.meta.client) { + const ph = getPostHog() + if (ph?.has_opted_in_capturing()) { + ph.capture('app_error', { + error_title: title, + error_message: opts?.message, + error_status_code: opts?.statusCode, + path: opts?.path ?? window.location.pathname, + }) + } + } + + return add({ + type: 'error', + title, + message: opts?.message, + details: opts?.details, + link: { + label: 'Report issue', + href: GITHUB_ISSUES_URL, + }, + }) + } + + function success(title: string, message?: string) { + return add({ type: 'success', title, message }) + } + + function warning(title: string, message?: string) { + return add({ type: 'warning', title, message }) + } + + function info(title: string, message?: string) { + return add({ type: 'info', title, message }) + } + + return { toasts: readonly(toasts), add, remove, clear, error, success, warning, info } +} diff --git a/app/layouts/dashboard.vue b/app/layouts/dashboard.vue index 53d50fe0..2033d59e 100644 --- a/app/layouts/dashboard.vue +++ b/app/layouts/dashboard.vue @@ -17,6 +17,7 @@ const isDemo = computed(() => {