From c327f3c2c1c623e1b5b8a8c99f0bc5606e670f65 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Wed, 1 Jul 2026 11:23:18 -0500 Subject: [PATCH 1/2] refactor(apify): registry-based webhook dispatch (foundation for non-IG persistence) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apifyWebhookHandler routed only the two Instagram actors via a hardcoded switch; every other platform (TikTok/X/YouTube/Threads/Facebook, and the new LinkedIn) hits "unhandled actorId" and persists NOTHING — those scrapes return data on the poll endpoint but never write back to socials (recoupable/chat#1833). Extract getApifyResultHandler (actorId → handler registry, SRP) and dispatch through it, so adding a platform is a single registry entry instead of another switch arm. Behavior-preserving — all existing webhook tests pass unchanged. The per-platform handlers (mirror handleInstagramProfileScraperResults) are the documented follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/getApifyResultHandler.test.ts | 21 ++++++++++++ lib/apify/apifyWebhookHandler.ts | 33 +++++++------------ lib/apify/getApifyResultHandler.ts | 32 ++++++++++++++++++ 3 files changed, 65 insertions(+), 21 deletions(-) create mode 100644 lib/apify/__tests__/getApifyResultHandler.test.ts create mode 100644 lib/apify/getApifyResultHandler.ts diff --git a/lib/apify/__tests__/getApifyResultHandler.test.ts b/lib/apify/__tests__/getApifyResultHandler.test.ts new file mode 100644 index 000000000..11ba02ef3 --- /dev/null +++ b/lib/apify/__tests__/getApifyResultHandler.test.ts @@ -0,0 +1,21 @@ +import { describe, it, expect, vi } from "vitest"; +import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler"; +import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; +import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; + +vi.mock("@/lib/apify/instagram/handleInstagramProfileScraperResults", () => ({ + handleInstagramProfileScraperResults: vi.fn(), +})); +vi.mock("@/lib/apify/instagram/handleInstagramCommentsScraper", () => ({ + handleInstagramCommentsScraper: vi.fn(), +})); + +describe("getApifyResultHandler", () => { + it("maps the Instagram actor ids to their result handlers", () => { + expect(getApifyResultHandler("dSCLg0C3YEZ83HzYX")).toBe(handleInstagramProfileScraperResults); + expect(getApifyResultHandler("SbK00X0JYCPblD2wp")).toBe(handleInstagramCommentsScraper); + }); + it("returns undefined for an unregistered actor id", () => { + expect(getApifyResultHandler("unknown_actor")).toBeUndefined(); + }); +}); diff --git a/lib/apify/apifyWebhookHandler.ts b/lib/apify/apifyWebhookHandler.ts index 64c863c17..fdda46e5a 100644 --- a/lib/apify/apifyWebhookHandler.ts +++ b/lib/apify/apifyWebhookHandler.ts @@ -1,10 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { validateApifyWebhookRequest } from "@/lib/apify/validateApifyWebhookRequest"; -import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; -import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; - -const INSTAGRAM_PROFILE_ACTOR_ID = "dSCLg0C3YEZ83HzYX"; -const INSTAGRAM_COMMENTS_ACTOR_ID = "SbK00X0JYCPblD2wp"; +import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler"; /** * Handler for `POST /api/apify`. Always responds 200 so Apify does not @@ -20,23 +16,18 @@ export async function apifyWebhookHandler(request: NextRequest): Promise Promise; + +/** + * Apify actor id (from the webhook `eventData.actorId`) → the handler that + * persists that actor's results. Registry, so adding a platform is a single + * entry rather than another `switch` arm. + * + * Only the Instagram actors are wired today — TikTok/X/YouTube/Threads/Facebook + * (and LinkedIn) scrapes run and return data on the poll endpoint but are NOT + * persisted yet. Each needs a `handleProfileScraperResults` (mirror + * the Instagram one) registered here under its resolved actor id. See + * recoupable/chat#1833 ("persist non-Instagram scrape results"). + */ +const HANDLERS_BY_ACTOR_ID: Record = { + dSCLg0C3YEZ83HzYX: handleInstagramProfileScraperResults, // instagram profile + SbK00X0JYCPblD2wp: handleInstagramCommentsScraper, // instagram comments +}; + +/** + * Look up the result handler for an Apify actor id. + * + * @param actorId - `eventData.actorId` from the Apify webhook payload. + * @returns The registered handler, or undefined if the actor isn't wired. + */ +export function getApifyResultHandler(actorId: string): ApifyResultHandler | undefined { + return HANDLERS_BY_ACTOR_ID[actorId]; +} From 223752f62dd0c2351b85d5beeabb75283939f6ad Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Wed, 1 Jul 2026 18:42:11 -0500 Subject: [PATCH 2/2] feat(linkedin): persist LinkedIn scrape results end-to-end (first non-IG platform) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review, the registry alone wasn't worth shipping — wire one new platform through the full persistence path (recoupable/chat#1833): - startLinkedinProfileScraping now attaches getApifyWebhooks() to the run — the root cause of "nothing but IG persists" is that only the Instagram start modules registered the per-run webhook; every other platform's runs never called /api/apify at all. - New handleLinkedinProfileScraperResults: upserts the scraped profile into socials (keyed on profile_url) — username/avatar/bio/followerCount/region. Field mapping taken from a REAL harvestapi run (TNwgXpQmhvoCHsKg8), including the {element,error,status} wrapper shape for missing profiles (skipped). - Registry: LpVuK3Zozwuipa5bp (harvestapi/linkedin-profile-scraper, id resolved from the Apify API) → the new handler. TikTok/X/YouTube/Threads/Facebook remain poll-only — each needs the same webhook attachment + a handler with its real output shape. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/apifyWebhookHandler.test.ts | 3 + .../__tests__/getApifyResultHandler.test.ts | 7 ++ .../startLinkedinProfileScraping.test.ts | 10 ++- lib/apify/getApifyResultHandler.ts | 8 ++- ...andleLinkedinProfileScraperResults.test.ts | 72 +++++++++++++++++++ .../handleLinkedinProfileScraperResults.ts | 52 ++++++++++++++ .../linkedin/startLinkedinProfileScraping.ts | 5 +- 7 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 lib/apify/linkedin/__tests__/handleLinkedinProfileScraperResults.test.ts create mode 100644 lib/apify/linkedin/handleLinkedinProfileScraperResults.ts diff --git a/lib/apify/__tests__/apifyWebhookHandler.test.ts b/lib/apify/__tests__/apifyWebhookHandler.test.ts index b2791bcbb..372575b75 100644 --- a/lib/apify/__tests__/apifyWebhookHandler.test.ts +++ b/lib/apify/__tests__/apifyWebhookHandler.test.ts @@ -10,6 +10,9 @@ vi.mock("../instagram/handleInstagramProfileScraperResults", () => ({ vi.mock("../instagram/handleInstagramCommentsScraper", () => ({ handleInstagramCommentsScraper: vi.fn(), })); +vi.mock("../linkedin/handleLinkedinProfileScraperResults", () => ({ + handleLinkedinProfileScraperResults: vi.fn(), +})); function makeRequest(body: unknown, raw?: string) { return new NextRequest("http://localhost/api/apify", { diff --git a/lib/apify/__tests__/getApifyResultHandler.test.ts b/lib/apify/__tests__/getApifyResultHandler.test.ts index 11ba02ef3..c6ad3bf15 100644 --- a/lib/apify/__tests__/getApifyResultHandler.test.ts +++ b/lib/apify/__tests__/getApifyResultHandler.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi } from "vitest"; import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler"; import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; +import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; vi.mock("@/lib/apify/instagram/handleInstagramProfileScraperResults", () => ({ handleInstagramProfileScraperResults: vi.fn(), @@ -9,12 +10,18 @@ vi.mock("@/lib/apify/instagram/handleInstagramProfileScraperResults", () => ({ vi.mock("@/lib/apify/instagram/handleInstagramCommentsScraper", () => ({ handleInstagramCommentsScraper: vi.fn(), })); +vi.mock("@/lib/apify/linkedin/handleLinkedinProfileScraperResults", () => ({ + handleLinkedinProfileScraperResults: vi.fn(), +})); describe("getApifyResultHandler", () => { it("maps the Instagram actor ids to their result handlers", () => { expect(getApifyResultHandler("dSCLg0C3YEZ83HzYX")).toBe(handleInstagramProfileScraperResults); expect(getApifyResultHandler("SbK00X0JYCPblD2wp")).toBe(handleInstagramCommentsScraper); }); + it("maps the harvestapi LinkedIn actor id to its results handler", () => { + expect(getApifyResultHandler("LpVuK3Zozwuipa5bp")).toBe(handleLinkedinProfileScraperResults); + }); it("returns undefined for an unregistered actor id", () => { expect(getApifyResultHandler("unknown_actor")).toBeUndefined(); }); diff --git a/lib/apify/__tests__/startLinkedinProfileScraping.test.ts b/lib/apify/__tests__/startLinkedinProfileScraping.test.ts index e9dc4a73b..dcd487008 100644 --- a/lib/apify/__tests__/startLinkedinProfileScraping.test.ts +++ b/lib/apify/__tests__/startLinkedinProfileScraping.test.ts @@ -14,13 +14,19 @@ describe("startLinkedinProfileScraping", () => { it("starts the harvestapi actor with `urls` (the actor's real input key), building the /in/ URL from a handle", async () => { const r = await startLinkedinProfileScraping("sweetmaneth"); expect(apifyClient.actor).toHaveBeenCalledWith("harvestapi/linkedin-profile-scraper"); - expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/sweetmaneth"] }); + expect(start).toHaveBeenCalledWith( + { urls: ["https://www.linkedin.com/in/sweetmaneth"] }, + { webhooks: expect.any(Array) }, + ); expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" }); }); it("passes a full profile URL through unchanged", async () => { await startLinkedinProfileScraping("https://www.linkedin.com/in/drew-thurlow"); - expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/drew-thurlow"] }); + expect(start).toHaveBeenCalledWith( + { urls: ["https://www.linkedin.com/in/drew-thurlow"] }, + { webhooks: expect.any(Array) }, + ); }); it("rejects LinkedIn path prefixes as handles (legacy rows stored 'in') instead of scraping the wrong profile", async () => { await expect(startLinkedinProfileScraping("in")).rejects.toThrow(/Invalid LinkedIn handle/); diff --git a/lib/apify/getApifyResultHandler.ts b/lib/apify/getApifyResultHandler.ts index 2ace709fa..37197c57c 100644 --- a/lib/apify/getApifyResultHandler.ts +++ b/lib/apify/getApifyResultHandler.ts @@ -1,5 +1,6 @@ import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; +import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; /** Persists one Apify actor run's results (posts/socials/etc.). */ @@ -10,15 +11,16 @@ export type ApifyResultHandler = (parsed: ApifyWebhookPayload) => PromiseProfileScraperResults` (mirror + * Instagram and LinkedIn are wired; TikTok/X/YouTube/Threads/Facebook scrapes + * run and return data on the poll endpoint but are NOT persisted yet — their + * start modules must also attach `getApifyWebhooks()` (only IG + LinkedIn do). Each needs a `handleProfileScraperResults` (mirror * the Instagram one) registered here under its resolved actor id. See * recoupable/chat#1833 ("persist non-Instagram scrape results"). */ const HANDLERS_BY_ACTOR_ID: Record = { dSCLg0C3YEZ83HzYX: handleInstagramProfileScraperResults, // instagram profile SbK00X0JYCPblD2wp: handleInstagramCommentsScraper, // instagram comments + LpVuK3Zozwuipa5bp: handleLinkedinProfileScraperResults, // linkedin profile (harvestapi) }; /** diff --git a/lib/apify/linkedin/__tests__/handleLinkedinProfileScraperResults.test.ts b/lib/apify/linkedin/__tests__/handleLinkedinProfileScraperResults.test.ts new file mode 100644 index 000000000..497901b04 --- /dev/null +++ b/lib/apify/linkedin/__tests__/handleLinkedinProfileScraperResults.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; + +const listItems = vi.fn(); +vi.mock("@/lib/apify/client", () => ({ + default: { dataset: vi.fn(() => ({ listItems })) }, +})); +const upsertSocials = vi.fn(); +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ + upsertSocials: (...a: unknown[]) => upsertSocials(...a), +})); + +const payload = { + eventData: { actorId: "LpVuK3Zozwuipa5bp" }, + resource: { defaultDatasetId: "ds-li-1" }, +} as never; + +// Trimmed from a real harvestapi run (TNwgXpQmhvoCHsKg8, 2026-07-01). +const REAL_PROFILE = { + publicIdentifier: "sweetmaneth", + linkedinUrl: "https://www.linkedin.com/in/sweetmaneth", + firstName: "sweetman", + lastName: "eth", + headline: "The dev for onchain music.", + about: "I am the builder. I code, architect, and scale the system. I build products to last.", + followerCount: 13203, + connectionsCount: 14242, + photo: "https://media.licdn.com/dms/image/v2/C4D03AQEVoqkcw2x3HA/photo.jpg", + location: { linkedinText: "Columbus, Ohio, United States", countryCode: "US" }, +}; + +beforeEach(() => { + vi.clearAllMocks(); + upsertSocials.mockResolvedValue([]); +}); + +describe("handleLinkedinProfileScraperResults", () => { + it("upserts the socials row from a real harvestapi profile item (keyed on profile_url)", async () => { + listItems.mockResolvedValue({ items: [REAL_PROFILE] }); + const result = await handleLinkedinProfileScraperResults(payload); + + expect(upsertSocials).toHaveBeenCalledWith([ + { + profile_url: "linkedin.com/in/sweetmaneth", + username: "sweetmaneth", + avatar: "https://media.licdn.com/dms/image/v2/C4D03AQEVoqkcw2x3HA/photo.jpg", + bio: "I am the builder. I code, architect, and scale the system. I build products to last.", + followerCount: 13203, + region: "Columbus, Ohio, United States", + }, + ]); + expect(result).toMatchObject({ social: expect.anything() }); + }); + + it("skips 404 wrapper items (harvestapi returns {element:{}, error, status} for missing profiles)", async () => { + // Real shape from run AB81HuE3atfKvhIKh (linkedin.com/in/in → 404). + listItems.mockResolvedValue({ + items: [{ element: {}, error: "Profile not found", status: 404, query: {} }], + }); + const result = await handleLinkedinProfileScraperResults(payload); + expect(upsertSocials).not.toHaveBeenCalled(); + expect(result).toEqual({ social: null }); + }); + + it("no-ops on an empty dataset", async () => { + listItems.mockResolvedValue({ items: [] }); + const result = await handleLinkedinProfileScraperResults(payload); + expect(upsertSocials).not.toHaveBeenCalled(); + expect(result).toEqual({ social: null }); + }); +}); diff --git a/lib/apify/linkedin/handleLinkedinProfileScraperResults.ts b/lib/apify/linkedin/handleLinkedinProfileScraperResults.ts new file mode 100644 index 000000000..d44a5f8fc --- /dev/null +++ b/lib/apify/linkedin/handleLinkedinProfileScraperResults.ts @@ -0,0 +1,52 @@ +import apifyClient from "@/lib/apify/client"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; +import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; + +/** + * One profile item from the harvestapi/linkedin-profile-scraper dataset. + * Field names taken from a real run (2026-07-01); missing-profile results + * arrive as a wrapper `{ element: {}, error, status }` instead. + */ +type HarvestApiLinkedinProfile = { + publicIdentifier?: string; + linkedinUrl?: string; + headline?: string; + about?: string; + followerCount?: number; + photo?: string; + location?: { linkedinText?: string }; + error?: string; + element?: unknown; +}; + +/** + * Handles LinkedIn profile scraper (harvestapi) webhook results: persists + * the scraped profile back to `socials` (upsert keyed on `profile_url`), + * so follower counts / avatar / bio refresh instead of living only in the + * poll endpoint. Minimal counterpart of the Instagram results handler — + * no posts/comments pipeline for LinkedIn yet. + * + * @param parsed - Validated Apify webhook payload. + */ +export async function handleLinkedinProfileScraperResults(parsed: ApifyWebhookPayload) { + const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); + const first = items[0] as HarvestApiLinkedinProfile | undefined; + + // Missing/failed profiles come back as a {element, error, status} wrapper. + if (!first || first.error || !first.linkedinUrl) { + return { social: null }; + } + + const social = { + profile_url: normalizeProfileUrl(first.linkedinUrl), + username: first.publicIdentifier, + avatar: first.photo ?? null, + bio: first.about ?? first.headline ?? null, + followerCount: first.followerCount ?? null, + region: first.location?.linkedinText ?? null, + }; + await upsertSocials([social]); + + return { social }; +} diff --git a/lib/apify/linkedin/startLinkedinProfileScraping.ts b/lib/apify/linkedin/startLinkedinProfileScraping.ts index 6038eaa89..ff3231944 100644 --- a/lib/apify/linkedin/startLinkedinProfileScraping.ts +++ b/lib/apify/linkedin/startLinkedinProfileScraping.ts @@ -1,4 +1,5 @@ import apifyClient from "@/lib/apify/client"; +import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; import { ApifyRunInfo } from "@/lib/apify/types"; @@ -25,7 +26,9 @@ const startLinkedinProfileScraping = async (handle: string): Promise