-
Notifications
You must be signed in to change notification settings - Fork 10
feat(apify): registry webhook dispatch + LinkedIn persistence end-to-end #738
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| import { describe, it, expect, vi } from "vitest"; | ||
| import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler"; | ||
| import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; | ||
| import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; | ||
| import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; | ||
|
|
||
| vi.mock("@/lib/apify/instagram/handleInstagramProfileScraperResults", () => ({ | ||
| handleInstagramProfileScraperResults: vi.fn(), | ||
| })); | ||
| vi.mock("@/lib/apify/instagram/handleInstagramCommentsScraper", () => ({ | ||
| handleInstagramCommentsScraper: vi.fn(), | ||
| })); | ||
| vi.mock("@/lib/apify/linkedin/handleLinkedinProfileScraperResults", () => ({ | ||
| handleLinkedinProfileScraperResults: vi.fn(), | ||
| })); | ||
|
|
||
| describe("getApifyResultHandler", () => { | ||
| it("maps the Instagram actor ids to their result handlers", () => { | ||
| expect(getApifyResultHandler("dSCLg0C3YEZ83HzYX")).toBe(handleInstagramProfileScraperResults); | ||
| expect(getApifyResultHandler("SbK00X0JYCPblD2wp")).toBe(handleInstagramCommentsScraper); | ||
| }); | ||
| it("maps the harvestapi LinkedIn actor id to its results handler", () => { | ||
| expect(getApifyResultHandler("LpVuK3Zozwuipa5bp")).toBe(handleLinkedinProfileScraperResults); | ||
| }); | ||
| it("returns undefined for an unregistered actor id", () => { | ||
| expect(getApifyResultHandler("unknown_actor")).toBeUndefined(); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; | ||
| import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; | ||
| import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; | ||
| import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; | ||
|
|
||
| /** Persists one Apify actor run's results (posts/socials/etc.). */ | ||
| export type ApifyResultHandler = (parsed: ApifyWebhookPayload) => Promise<unknown>; | ||
|
|
||
| /** | ||
| * Apify actor id (from the webhook `eventData.actorId`) → the handler that | ||
| * persists that actor's results. Registry, so adding a platform is a single | ||
| * entry rather than another `switch` arm. | ||
| * | ||
| * Instagram and LinkedIn are wired; TikTok/X/YouTube/Threads/Facebook scrapes | ||
| * run and return data on the poll endpoint but are NOT persisted yet — their | ||
| * start modules must also attach `getApifyWebhooks()` (only IG + LinkedIn do). Each needs a `handle<Platform>ProfileScraperResults` (mirror | ||
| * the Instagram one) registered here under its resolved actor id. See | ||
| * recoupable/chat#1833 ("persist non-Instagram scrape results"). | ||
| */ | ||
| const HANDLERS_BY_ACTOR_ID: Record<string, ApifyResultHandler> = { | ||
| dSCLg0C3YEZ83HzYX: handleInstagramProfileScraperResults, // instagram profile | ||
| SbK00X0JYCPblD2wp: handleInstagramCommentsScraper, // instagram comments | ||
| LpVuK3Zozwuipa5bp: handleLinkedinProfileScraperResults, // linkedin profile (harvestapi) | ||
| }; | ||
|
|
||
| /** | ||
| * Look up the result handler for an Apify actor id. | ||
| * | ||
| * @param actorId - `eventData.actorId` from the Apify webhook payload. | ||
| * @returns The registered handler, or undefined if the actor isn't wired. | ||
| */ | ||
| export function getApifyResultHandler(actorId: string): ApifyResultHandler | undefined { | ||
| return HANDLERS_BY_ACTOR_ID[actorId]; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| import { describe, it, expect, vi, beforeEach } from "vitest"; | ||
|
|
||
| import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; | ||
|
|
||
| const listItems = vi.fn(); | ||
| vi.mock("@/lib/apify/client", () => ({ | ||
| default: { dataset: vi.fn(() => ({ listItems })) }, | ||
| })); | ||
| const upsertSocials = vi.fn(); | ||
| vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ | ||
| upsertSocials: (...a: unknown[]) => upsertSocials(...a), | ||
| })); | ||
|
|
||
| const payload = { | ||
| eventData: { actorId: "LpVuK3Zozwuipa5bp" }, | ||
| resource: { defaultDatasetId: "ds-li-1" }, | ||
| } as never; | ||
|
|
||
| // Trimmed from a real harvestapi run (TNwgXpQmhvoCHsKg8, 2026-07-01). | ||
| const REAL_PROFILE = { | ||
| publicIdentifier: "sweetmaneth", | ||
| linkedinUrl: "https://www.linkedin.com/in/sweetmaneth", | ||
| firstName: "sweetman", | ||
| lastName: "eth", | ||
| headline: "The dev for onchain music.", | ||
| about: "I am the builder. I code, architect, and scale the system. I build products to last.", | ||
| followerCount: 13203, | ||
| connectionsCount: 14242, | ||
| photo: "https://media.licdn.com/dms/image/v2/C4D03AQEVoqkcw2x3HA/photo.jpg", | ||
| location: { linkedinText: "Columbus, Ohio, United States", countryCode: "US" }, | ||
| }; | ||
|
|
||
| beforeEach(() => { | ||
| vi.clearAllMocks(); | ||
| upsertSocials.mockResolvedValue([]); | ||
| }); | ||
|
|
||
| describe("handleLinkedinProfileScraperResults", () => { | ||
| it("upserts the socials row from a real harvestapi profile item (keyed on profile_url)", async () => { | ||
| listItems.mockResolvedValue({ items: [REAL_PROFILE] }); | ||
| const result = await handleLinkedinProfileScraperResults(payload); | ||
|
|
||
| expect(upsertSocials).toHaveBeenCalledWith([ | ||
| { | ||
| profile_url: "linkedin.com/in/sweetmaneth", | ||
| username: "sweetmaneth", | ||
| avatar: "https://media.licdn.com/dms/image/v2/C4D03AQEVoqkcw2x3HA/photo.jpg", | ||
| bio: "I am the builder. I code, architect, and scale the system. I build products to last.", | ||
| followerCount: 13203, | ||
| region: "Columbus, Ohio, United States", | ||
| }, | ||
| ]); | ||
| expect(result).toMatchObject({ social: expect.anything() }); | ||
| }); | ||
|
|
||
| it("skips 404 wrapper items (harvestapi returns {element:{}, error, status} for missing profiles)", async () => { | ||
| // Real shape from run AB81HuE3atfKvhIKh (linkedin.com/in/in → 404). | ||
| listItems.mockResolvedValue({ | ||
| items: [{ element: {}, error: "Profile not found", status: 404, query: {} }], | ||
| }); | ||
| const result = await handleLinkedinProfileScraperResults(payload); | ||
| expect(upsertSocials).not.toHaveBeenCalled(); | ||
| expect(result).toEqual({ social: null }); | ||
| }); | ||
|
|
||
| it("no-ops on an empty dataset", async () => { | ||
| listItems.mockResolvedValue({ items: [] }); | ||
| const result = await handleLinkedinProfileScraperResults(payload); | ||
| expect(upsertSocials).not.toHaveBeenCalled(); | ||
| expect(result).toEqual({ social: null }); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| import apifyClient from "@/lib/apify/client"; | ||
| import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; | ||
| import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; | ||
| import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; | ||
|
|
||
| /** | ||
| * One profile item from the harvestapi/linkedin-profile-scraper dataset. | ||
| * Field names taken from a real run (2026-07-01); missing-profile results | ||
| * arrive as a wrapper `{ element: {}, error, status }` instead. | ||
| */ | ||
| type HarvestApiLinkedinProfile = { | ||
| publicIdentifier?: string; | ||
| linkedinUrl?: string; | ||
| headline?: string; | ||
| about?: string; | ||
| followerCount?: number; | ||
| photo?: string; | ||
| location?: { linkedinText?: string }; | ||
| error?: string; | ||
| element?: unknown; | ||
| }; | ||
|
|
||
| /** | ||
| * Handles LinkedIn profile scraper (harvestapi) webhook results: persists | ||
| * the scraped profile back to `socials` (upsert keyed on `profile_url`), | ||
| * so follower counts / avatar / bio refresh instead of living only in the | ||
| * poll endpoint. Minimal counterpart of the Instagram results handler — | ||
| * no posts/comments pipeline for LinkedIn yet. | ||
| * | ||
| * @param parsed - Validated Apify webhook payload. | ||
| */ | ||
| export async function handleLinkedinProfileScraperResults(parsed: ApifyWebhookPayload) { | ||
| const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); | ||
| const first = items[0] as HarvestApiLinkedinProfile | undefined; | ||
|
|
||
| // Missing/failed profiles come back as a {element, error, status} wrapper. | ||
| if (!first || first.error || !first.linkedinUrl) { | ||
| return { social: null }; | ||
| } | ||
|
|
||
| const social = { | ||
| profile_url: normalizeProfileUrl(first.linkedinUrl), | ||
| username: first.publicIdentifier, | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: LinkedIn webhook persistence can fail for valid-looking items that include Prompt for AI agents |
||
| avatar: first.photo ?? null, | ||
| bio: first.about ?? first.headline ?? null, | ||
| followerCount: first.followerCount ?? null, | ||
| region: first.location?.linkedinText ?? null, | ||
| }; | ||
| await upsertSocials([social]); | ||
|
|
||
| return { social }; | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P3: Guard that the registry result is actually callable, not just truthy. Otherwise special actor ids such as
__proto__produce an internal handler exception instead of the intended unhandled-actor response.Prompt for AI agents