diff --git a/lib/apify/__tests__/apifyWebhookHandler.test.ts b/lib/apify/__tests__/apifyWebhookHandler.test.ts index b2791bcb..372575b7 100644 --- a/lib/apify/__tests__/apifyWebhookHandler.test.ts +++ b/lib/apify/__tests__/apifyWebhookHandler.test.ts @@ -10,6 +10,9 @@ vi.mock("../instagram/handleInstagramProfileScraperResults", () => ({ vi.mock("../instagram/handleInstagramCommentsScraper", () => ({ handleInstagramCommentsScraper: vi.fn(), })); +vi.mock("../linkedin/handleLinkedinProfileScraperResults", () => ({ + handleLinkedinProfileScraperResults: vi.fn(), +})); function makeRequest(body: unknown, raw?: string) { return new NextRequest("http://localhost/api/apify", { diff --git a/lib/apify/__tests__/getApifyResultHandler.test.ts b/lib/apify/__tests__/getApifyResultHandler.test.ts new file mode 100644 index 00000000..c6ad3bf1 --- /dev/null +++ b/lib/apify/__tests__/getApifyResultHandler.test.ts @@ -0,0 +1,28 @@ +import { describe, it, expect, vi } from "vitest"; +import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler"; +import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; +import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; +import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; + +vi.mock("@/lib/apify/instagram/handleInstagramProfileScraperResults", () => ({ + handleInstagramProfileScraperResults: vi.fn(), +})); +vi.mock("@/lib/apify/instagram/handleInstagramCommentsScraper", () => ({ + handleInstagramCommentsScraper: vi.fn(), +})); +vi.mock("@/lib/apify/linkedin/handleLinkedinProfileScraperResults", () => ({ + handleLinkedinProfileScraperResults: vi.fn(), +})); + +describe("getApifyResultHandler", () => { + it("maps the Instagram actor ids to their result handlers", () => { + expect(getApifyResultHandler("dSCLg0C3YEZ83HzYX")).toBe(handleInstagramProfileScraperResults); + expect(getApifyResultHandler("SbK00X0JYCPblD2wp")).toBe(handleInstagramCommentsScraper); + }); + it("maps the harvestapi LinkedIn actor id to its results handler", () => { + expect(getApifyResultHandler("LpVuK3Zozwuipa5bp")).toBe(handleLinkedinProfileScraperResults); + }); + it("returns undefined for an unregistered actor id", () => { + expect(getApifyResultHandler("unknown_actor")).toBeUndefined(); + }); +}); diff --git a/lib/apify/__tests__/startLinkedinProfileScraping.test.ts b/lib/apify/__tests__/startLinkedinProfileScraping.test.ts index e9dc4a73..dcd48700 100644 --- a/lib/apify/__tests__/startLinkedinProfileScraping.test.ts +++ b/lib/apify/__tests__/startLinkedinProfileScraping.test.ts @@ -14,13 +14,19 @@ describe("startLinkedinProfileScraping", () => { it("starts the harvestapi actor with `urls` (the actor's real input key), building the /in/ URL from a handle", async () => { const r = await startLinkedinProfileScraping("sweetmaneth"); expect(apifyClient.actor).toHaveBeenCalledWith("harvestapi/linkedin-profile-scraper"); - expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/sweetmaneth"] }); + expect(start).toHaveBeenCalledWith( + { urls: ["https://www.linkedin.com/in/sweetmaneth"] }, + { webhooks: expect.any(Array) }, + ); expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" }); }); it("passes a full profile URL through unchanged", async () => { await startLinkedinProfileScraping("https://www.linkedin.com/in/drew-thurlow"); - expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/drew-thurlow"] }); + expect(start).toHaveBeenCalledWith( + { urls: ["https://www.linkedin.com/in/drew-thurlow"] }, + { webhooks: expect.any(Array) }, + ); }); it("rejects LinkedIn path prefixes as handles (legacy rows stored 'in') instead of scraping the wrong profile", async () => { await expect(startLinkedinProfileScraping("in")).rejects.toThrow(/Invalid LinkedIn handle/); diff --git a/lib/apify/apifyWebhookHandler.ts b/lib/apify/apifyWebhookHandler.ts index 64c863c1..fdda46e5 100644 --- a/lib/apify/apifyWebhookHandler.ts +++ b/lib/apify/apifyWebhookHandler.ts @@ -1,10 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { validateApifyWebhookRequest } from "@/lib/apify/validateApifyWebhookRequest"; -import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; -import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; - -const INSTAGRAM_PROFILE_ACTOR_ID = "dSCLg0C3YEZ83HzYX"; -const INSTAGRAM_COMMENTS_ACTOR_ID = "SbK00X0JYCPblD2wp"; +import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler"; /** * Handler for `POST /api/apify`. Always responds 200 so Apify does not @@ -20,23 +16,18 @@ export async function apifyWebhookHandler(request: NextRequest): Promise Promise; + +/** + * Apify actor id (from the webhook `eventData.actorId`) → the handler that + * persists that actor's results. Registry, so adding a platform is a single + * entry rather than another `switch` arm. + * + * Instagram and LinkedIn are wired; TikTok/X/YouTube/Threads/Facebook scrapes + * run and return data on the poll endpoint but are NOT persisted yet — their + * start modules must also attach `getApifyWebhooks()` (only IG + LinkedIn do). Each needs a `handleProfileScraperResults` (mirror + * the Instagram one) registered here under its resolved actor id. See + * recoupable/chat#1833 ("persist non-Instagram scrape results"). + */ +const HANDLERS_BY_ACTOR_ID: Record = { + dSCLg0C3YEZ83HzYX: handleInstagramProfileScraperResults, // instagram profile + SbK00X0JYCPblD2wp: handleInstagramCommentsScraper, // instagram comments + LpVuK3Zozwuipa5bp: handleLinkedinProfileScraperResults, // linkedin profile (harvestapi) +}; + +/** + * Look up the result handler for an Apify actor id. + * + * @param actorId - `eventData.actorId` from the Apify webhook payload. + * @returns The registered handler, or undefined if the actor isn't wired. + */ +export function getApifyResultHandler(actorId: string): ApifyResultHandler | undefined { + return HANDLERS_BY_ACTOR_ID[actorId]; +} diff --git a/lib/apify/linkedin/__tests__/handleLinkedinProfileScraperResults.test.ts b/lib/apify/linkedin/__tests__/handleLinkedinProfileScraperResults.test.ts new file mode 100644 index 00000000..497901b0 --- /dev/null +++ b/lib/apify/linkedin/__tests__/handleLinkedinProfileScraperResults.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; + +const listItems = vi.fn(); +vi.mock("@/lib/apify/client", () => ({ + default: { dataset: vi.fn(() => ({ listItems })) }, +})); +const upsertSocials = vi.fn(); +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ + upsertSocials: (...a: unknown[]) => upsertSocials(...a), +})); + +const payload = { + eventData: { actorId: "LpVuK3Zozwuipa5bp" }, + resource: { defaultDatasetId: "ds-li-1" }, +} as never; + +// Trimmed from a real harvestapi run (TNwgXpQmhvoCHsKg8, 2026-07-01). +const REAL_PROFILE = { + publicIdentifier: "sweetmaneth", + linkedinUrl: "https://www.linkedin.com/in/sweetmaneth", + firstName: "sweetman", + lastName: "eth", + headline: "The dev for onchain music.", + about: "I am the builder. I code, architect, and scale the system. I build products to last.", + followerCount: 13203, + connectionsCount: 14242, + photo: "https://media.licdn.com/dms/image/v2/C4D03AQEVoqkcw2x3HA/photo.jpg", + location: { linkedinText: "Columbus, Ohio, United States", countryCode: "US" }, +}; + +beforeEach(() => { + vi.clearAllMocks(); + upsertSocials.mockResolvedValue([]); +}); + +describe("handleLinkedinProfileScraperResults", () => { + it("upserts the socials row from a real harvestapi profile item (keyed on profile_url)", async () => { + listItems.mockResolvedValue({ items: [REAL_PROFILE] }); + const result = await handleLinkedinProfileScraperResults(payload); + + expect(upsertSocials).toHaveBeenCalledWith([ + { + profile_url: "linkedin.com/in/sweetmaneth", + username: "sweetmaneth", + avatar: "https://media.licdn.com/dms/image/v2/C4D03AQEVoqkcw2x3HA/photo.jpg", + bio: "I am the builder. I code, architect, and scale the system. I build products to last.", + followerCount: 13203, + region: "Columbus, Ohio, United States", + }, + ]); + expect(result).toMatchObject({ social: expect.anything() }); + }); + + it("skips 404 wrapper items (harvestapi returns {element:{}, error, status} for missing profiles)", async () => { + // Real shape from run AB81HuE3atfKvhIKh (linkedin.com/in/in → 404). + listItems.mockResolvedValue({ + items: [{ element: {}, error: "Profile not found", status: 404, query: {} }], + }); + const result = await handleLinkedinProfileScraperResults(payload); + expect(upsertSocials).not.toHaveBeenCalled(); + expect(result).toEqual({ social: null }); + }); + + it("no-ops on an empty dataset", async () => { + listItems.mockResolvedValue({ items: [] }); + const result = await handleLinkedinProfileScraperResults(payload); + expect(upsertSocials).not.toHaveBeenCalled(); + expect(result).toEqual({ social: null }); + }); +}); diff --git a/lib/apify/linkedin/handleLinkedinProfileScraperResults.ts b/lib/apify/linkedin/handleLinkedinProfileScraperResults.ts new file mode 100644 index 00000000..d44a5f8f --- /dev/null +++ b/lib/apify/linkedin/handleLinkedinProfileScraperResults.ts @@ -0,0 +1,52 @@ +import apifyClient from "@/lib/apify/client"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; +import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; + +/** + * One profile item from the harvestapi/linkedin-profile-scraper dataset. + * Field names taken from a real run (2026-07-01); missing-profile results + * arrive as a wrapper `{ element: {}, error, status }` instead. + */ +type HarvestApiLinkedinProfile = { + publicIdentifier?: string; + linkedinUrl?: string; + headline?: string; + about?: string; + followerCount?: number; + photo?: string; + location?: { linkedinText?: string }; + error?: string; + element?: unknown; +}; + +/** + * Handles LinkedIn profile scraper (harvestapi) webhook results: persists + * the scraped profile back to `socials` (upsert keyed on `profile_url`), + * so follower counts / avatar / bio refresh instead of living only in the + * poll endpoint. Minimal counterpart of the Instagram results handler — + * no posts/comments pipeline for LinkedIn yet. + * + * @param parsed - Validated Apify webhook payload. + */ +export async function handleLinkedinProfileScraperResults(parsed: ApifyWebhookPayload) { + const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); + const first = items[0] as HarvestApiLinkedinProfile | undefined; + + // Missing/failed profiles come back as a {element, error, status} wrapper. + if (!first || first.error || !first.linkedinUrl) { + return { social: null }; + } + + const social = { + profile_url: normalizeProfileUrl(first.linkedinUrl), + username: first.publicIdentifier, + avatar: first.photo ?? null, + bio: first.about ?? first.headline ?? null, + followerCount: first.followerCount ?? null, + region: first.location?.linkedinText ?? null, + }; + await upsertSocials([social]); + + return { social }; +} diff --git a/lib/apify/linkedin/startLinkedinProfileScraping.ts b/lib/apify/linkedin/startLinkedinProfileScraping.ts index 6038eaa8..ff323194 100644 --- a/lib/apify/linkedin/startLinkedinProfileScraping.ts +++ b/lib/apify/linkedin/startLinkedinProfileScraping.ts @@ -1,4 +1,5 @@ import apifyClient from "@/lib/apify/client"; +import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; import { ApifyRunInfo } from "@/lib/apify/types"; @@ -25,7 +26,9 @@ const startLinkedinProfileScraping = async (handle: string): Promise