Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions lib/apify/__tests__/apifyWebhookHandler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ vi.mock("../instagram/handleInstagramProfileScraperResults", () => ({
vi.mock("../instagram/handleInstagramCommentsScraper", () => ({
handleInstagramCommentsScraper: vi.fn(),
}));
vi.mock("../linkedin/handleLinkedinProfileScraperResults", () => ({
handleLinkedinProfileScraperResults: vi.fn(),
}));

function makeRequest(body: unknown, raw?: string) {
return new NextRequest("http://localhost/api/apify", {
Expand Down
28 changes: 28 additions & 0 deletions lib/apify/__tests__/getApifyResultHandler.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { describe, it, expect, vi } from "vitest";
import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler";
import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults";
import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper";
import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults";

vi.mock("@/lib/apify/instagram/handleInstagramProfileScraperResults", () => ({
handleInstagramProfileScraperResults: vi.fn(),
}));
vi.mock("@/lib/apify/instagram/handleInstagramCommentsScraper", () => ({
handleInstagramCommentsScraper: vi.fn(),
}));
vi.mock("@/lib/apify/linkedin/handleLinkedinProfileScraperResults", () => ({
handleLinkedinProfileScraperResults: vi.fn(),
}));

describe("getApifyResultHandler", () => {
it("maps the Instagram actor ids to their result handlers", () => {
expect(getApifyResultHandler("dSCLg0C3YEZ83HzYX")).toBe(handleInstagramProfileScraperResults);
expect(getApifyResultHandler("SbK00X0JYCPblD2wp")).toBe(handleInstagramCommentsScraper);
});
it("maps the harvestapi LinkedIn actor id to its results handler", () => {
expect(getApifyResultHandler("LpVuK3Zozwuipa5bp")).toBe(handleLinkedinProfileScraperResults);
});
it("returns undefined for an unregistered actor id", () => {
expect(getApifyResultHandler("unknown_actor")).toBeUndefined();
});
});
10 changes: 8 additions & 2 deletions lib/apify/__tests__/startLinkedinProfileScraping.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,19 @@ describe("startLinkedinProfileScraping", () => {
it("starts the harvestapi actor with `urls` (the actor's real input key), building the /in/ URL from a handle", async () => {
const r = await startLinkedinProfileScraping("sweetmaneth");
expect(apifyClient.actor).toHaveBeenCalledWith("harvestapi/linkedin-profile-scraper");
expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/sweetmaneth"] });
expect(start).toHaveBeenCalledWith(
{ urls: ["https://www.linkedin.com/in/sweetmaneth"] },
{ webhooks: expect.any(Array) },
);
expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" });
});

it("passes a full profile URL through unchanged", async () => {
await startLinkedinProfileScraping("https://www.linkedin.com/in/drew-thurlow");
expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/drew-thurlow"] });
expect(start).toHaveBeenCalledWith(
{ urls: ["https://www.linkedin.com/in/drew-thurlow"] },
{ webhooks: expect.any(Array) },
);
});
it("rejects LinkedIn path prefixes as handles (legacy rows stored 'in') instead of scraping the wrong profile", async () => {
await expect(startLinkedinProfileScraping("in")).rejects.toThrow(/Invalid LinkedIn handle/);
Expand Down
33 changes: 12 additions & 21 deletions lib/apify/apifyWebhookHandler.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
import { NextRequest, NextResponse } from "next/server";
import { validateApifyWebhookRequest } from "@/lib/apify/validateApifyWebhookRequest";
import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults";
import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper";

const INSTAGRAM_PROFILE_ACTOR_ID = "dSCLg0C3YEZ83HzYX";
const INSTAGRAM_COMMENTS_ACTOR_ID = "SbK00X0JYCPblD2wp";
import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler";

/**
* Handler for `POST /api/apify`. Always responds 200 so Apify does not
Expand All @@ -20,23 +16,18 @@ export async function apifyWebhookHandler(request: NextRequest): Promise<NextRes

const { actorId } = validated.eventData;

const handler = getApifyResultHandler(actorId);
if (!handler) {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P3: Guard that the registry result is actually callable, not just truthy. Otherwise special actor ids such as __proto__ produce an internal handler exception instead of the intended unhandled-actor response.

Prompt for AI agents
Check if this issue is valid — if so, understand the root cause and fix it. At lib/apify/apifyWebhookHandler.ts, line 20:

<comment>Guard that the registry result is actually callable, not just truthy. Otherwise special actor ids such as `__proto__` produce an internal handler exception instead of the intended unhandled-actor response.</comment>

<file context>
@@ -20,23 +16,18 @@ export async function apifyWebhookHandler(request: NextRequest): Promise<NextRes
   const { actorId } = validated.eventData;
 
+  const handler = getApifyResultHandler(actorId);
+  if (!handler) {
+    console.warn(`[WARN] apifyWebhookHandler: unhandled actorId ${actorId}`);
+    return NextResponse.json(
</file context>
Suggested change
if (!handler) {
if (typeof handler !== "function") {

console.warn(`[WARN] apifyWebhookHandler: unhandled actorId ${actorId}`);
return NextResponse.json(
{ status: "error", error: `Unhandled actorId: ${actorId}` },
{ status: 200 },
);
}

try {
switch (actorId) {
case INSTAGRAM_PROFILE_ACTOR_ID: {
const result = await handleInstagramProfileScraperResults(validated);
return NextResponse.json(result, { status: 200 });
}
case INSTAGRAM_COMMENTS_ACTOR_ID: {
const result = await handleInstagramCommentsScraper(validated);
return NextResponse.json(result, { status: 200 });
}
default:
console.warn(`[WARN] apifyWebhookHandler: unhandled actorId ${actorId}`);
return NextResponse.json(
{ status: "error", error: `Unhandled actorId: ${actorId}` },
{ status: 200 },
);
}
const result = await handler(validated);
return NextResponse.json(result, { status: 200 });
} catch (error) {
console.error("[ERROR] apifyWebhookHandler:", error);
return NextResponse.json({ status: "error", error: "Internal server error" }, { status: 200 });
Expand Down
34 changes: 34 additions & 0 deletions lib/apify/getApifyResultHandler.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults";
import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper";
import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults";
import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest";

/** Persists one Apify actor run's results (posts/socials/etc.). */
export type ApifyResultHandler = (parsed: ApifyWebhookPayload) => Promise<unknown>;

/**
* Apify actor id (from the webhook `eventData.actorId`) → the handler that
* persists that actor's results. Registry, so adding a platform is a single
* entry rather than another `switch` arm.
*
* Instagram and LinkedIn are wired; TikTok/X/YouTube/Threads/Facebook scrapes
* run and return data on the poll endpoint but are NOT persisted yet — their
* start modules must also attach `getApifyWebhooks()` (only IG + LinkedIn do). Each needs a `handle<Platform>ProfileScraperResults` (mirror
* the Instagram one) registered here under its resolved actor id. See
* recoupable/chat#1833 ("persist non-Instagram scrape results").
*/
const HANDLERS_BY_ACTOR_ID: Record<string, ApifyResultHandler> = {
dSCLg0C3YEZ83HzYX: handleInstagramProfileScraperResults, // instagram profile
SbK00X0JYCPblD2wp: handleInstagramCommentsScraper, // instagram comments
LpVuK3Zozwuipa5bp: handleLinkedinProfileScraperResults, // linkedin profile (harvestapi)
};

/**
* Look up the result handler for an Apify actor id.
*
* @param actorId - `eventData.actorId` from the Apify webhook payload.
* @returns The registered handler, or undefined if the actor isn't wired.
*/
export function getApifyResultHandler(actorId: string): ApifyResultHandler | undefined {
return HANDLERS_BY_ACTOR_ID[actorId];
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import { describe, it, expect, vi, beforeEach } from "vitest";

import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults";

const listItems = vi.fn();
vi.mock("@/lib/apify/client", () => ({
default: { dataset: vi.fn(() => ({ listItems })) },
}));
const upsertSocials = vi.fn();
vi.mock("@/lib/supabase/socials/upsertSocials", () => ({
upsertSocials: (...a: unknown[]) => upsertSocials(...a),
}));

const payload = {
eventData: { actorId: "LpVuK3Zozwuipa5bp" },
resource: { defaultDatasetId: "ds-li-1" },
} as never;

// Trimmed from a real harvestapi run (TNwgXpQmhvoCHsKg8, 2026-07-01).
const REAL_PROFILE = {
publicIdentifier: "sweetmaneth",
linkedinUrl: "https://www.linkedin.com/in/sweetmaneth",
firstName: "sweetman",
lastName: "eth",
headline: "The dev for onchain music.",
about: "I am the builder. I code, architect, and scale the system. I build products to last.",
followerCount: 13203,
connectionsCount: 14242,
photo: "https://media.licdn.com/dms/image/v2/C4D03AQEVoqkcw2x3HA/photo.jpg",
location: { linkedinText: "Columbus, Ohio, United States", countryCode: "US" },
};

beforeEach(() => {
vi.clearAllMocks();
upsertSocials.mockResolvedValue([]);
});

describe("handleLinkedinProfileScraperResults", () => {
it("upserts the socials row from a real harvestapi profile item (keyed on profile_url)", async () => {
listItems.mockResolvedValue({ items: [REAL_PROFILE] });
const result = await handleLinkedinProfileScraperResults(payload);

expect(upsertSocials).toHaveBeenCalledWith([
{
profile_url: "linkedin.com/in/sweetmaneth",
username: "sweetmaneth",
avatar: "https://media.licdn.com/dms/image/v2/C4D03AQEVoqkcw2x3HA/photo.jpg",
bio: "I am the builder. I code, architect, and scale the system. I build products to last.",
followerCount: 13203,
region: "Columbus, Ohio, United States",
},
]);
expect(result).toMatchObject({ social: expect.anything() });
});

it("skips 404 wrapper items (harvestapi returns {element:{}, error, status} for missing profiles)", async () => {
// Real shape from run AB81HuE3atfKvhIKh (linkedin.com/in/in → 404).
listItems.mockResolvedValue({
items: [{ element: {}, error: "Profile not found", status: 404, query: {} }],
});
const result = await handleLinkedinProfileScraperResults(payload);
expect(upsertSocials).not.toHaveBeenCalled();
expect(result).toEqual({ social: null });
});

it("no-ops on an empty dataset", async () => {
listItems.mockResolvedValue({ items: [] });
const result = await handleLinkedinProfileScraperResults(payload);
expect(upsertSocials).not.toHaveBeenCalled();
expect(result).toEqual({ social: null });
});
});
52 changes: 52 additions & 0 deletions lib/apify/linkedin/handleLinkedinProfileScraperResults.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import apifyClient from "@/lib/apify/client";
import { upsertSocials } from "@/lib/supabase/socials/upsertSocials";
import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl";
import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest";

/**
* One profile item from the harvestapi/linkedin-profile-scraper dataset.
* Field names taken from a real run (2026-07-01); missing-profile results
* arrive as a wrapper `{ element: {}, error, status }` instead.
*/
type HarvestApiLinkedinProfile = {
publicIdentifier?: string;
linkedinUrl?: string;
headline?: string;
about?: string;
followerCount?: number;
photo?: string;
location?: { linkedinText?: string };
error?: string;
element?: unknown;
};

/**
* Handles LinkedIn profile scraper (harvestapi) webhook results: persists
* the scraped profile back to `socials` (upsert keyed on `profile_url`),
* so follower counts / avatar / bio refresh instead of living only in the
* poll endpoint. Minimal counterpart of the Instagram results handler —
* no posts/comments pipeline for LinkedIn yet.
*
* @param parsed - Validated Apify webhook payload.
*/
export async function handleLinkedinProfileScraperResults(parsed: ApifyWebhookPayload) {
const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems();
const first = items[0] as HarvestApiLinkedinProfile | undefined;

// Missing/failed profiles come back as a {element, error, status} wrapper.
if (!first || first.error || !first.linkedinUrl) {
return { social: null };
}

const social = {
profile_url: normalizeProfileUrl(first.linkedinUrl),
username: first.publicIdentifier,

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: LinkedIn webhook persistence can fail for valid-looking items that include linkedinUrl but omit publicIdentifier, because the inserted socials row has no required username. Falling back to the handle parsed from linkedinUrl would keep the upsert valid and consistent with scrape startup.

Prompt for AI agents
Check if this issue is valid — if so, understand the root cause and fix it. At lib/apify/linkedin/handleLinkedinProfileScraperResults.ts, line 43:

<comment>LinkedIn webhook persistence can fail for valid-looking items that include `linkedinUrl` but omit `publicIdentifier`, because the inserted `socials` row has no required `username`. Falling back to the handle parsed from `linkedinUrl` would keep the upsert valid and consistent with scrape startup.</comment>

<file context>
@@ -0,0 +1,52 @@
+
+  const social = {
+    profile_url: normalizeProfileUrl(first.linkedinUrl),
+    username: first.publicIdentifier,
+    avatar: first.photo ?? null,
+    bio: first.about ?? first.headline ?? null,
</file context>

avatar: first.photo ?? null,
bio: first.about ?? first.headline ?? null,
followerCount: first.followerCount ?? null,
region: first.location?.linkedinText ?? null,
};
await upsertSocials([social]);

return { social };
}
5 changes: 4 additions & 1 deletion lib/apify/linkedin/startLinkedinProfileScraping.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import apifyClient from "@/lib/apify/client";
import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks";
import { OUTSTANDING_ERROR } from "@/lib/apify/errors";
import { ApifyRunInfo } from "@/lib/apify/types";

Expand All @@ -25,7 +26,9 @@ const startLinkedinProfileScraping = async (handle: string): Promise<ApifyRunInf
const input = { urls: [targetUrl] };

try {
const run = await apifyClient.actor("harvestapi/linkedin-profile-scraper").start(input);
const run = await apifyClient
.actor("harvestapi/linkedin-profile-scraper")
.start(input, { webhooks: getApifyWebhooks() });

if (!run?.id || !run?.defaultDatasetId) {
console.error("Failed to start LinkedIn profile scraping for handle:", handle);
Expand Down
Loading