From 273c21815b0e6e34de97983d9cbd4edf5090bdc1 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Wed, 1 Jul 2026 11:19:03 -0500 Subject: [PATCH 1/4] feat(apify): add LinkedIn profile scraping (harvestapi) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LinkedIn was the only one of {YouTube, TikTok, LinkedIn, X, Instagram} unsupported by POST /api/socials/{social_id}/scrape — a linkedin.com profile returned 400 "Unsupported social media platform" (recoupable/chat#1833). Add: - lib/apify/linkedin/startLinkedinProfileScraping.ts → harvestapi/linkedin-profile-scraper (mirrors the tiktok/youtube modules). - scrapeProfileUrl: dispatch linkedin.com → the new scraper. - getSocialPlatformByLink: linkedin.com → LINKEDIN (was NONE). Note: harvestapi input shape (profiles: [url]) should be confirmed against the actor on the preview; result persistence is handled in the companion webhook PR. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/apify/__tests__/scrapeProfileUrl.test.ts | 37 ++++++++++++++++ .../linkedin/startLinkedinProfileScraping.ts | 44 +++++++++++++++++++ lib/apify/scrapeProfileUrl.ts | 5 +++ .../__tests__/getSocialPlatformByLink.test.ts | 14 ++++++ lib/artists/getSocialPlatformByLink.ts | 1 + 5 files changed, 101 insertions(+) create mode 100644 lib/apify/__tests__/scrapeProfileUrl.test.ts create mode 100644 lib/apify/linkedin/startLinkedinProfileScraping.ts create mode 100644 lib/artists/__tests__/getSocialPlatformByLink.test.ts diff --git a/lib/apify/__tests__/scrapeProfileUrl.test.ts b/lib/apify/__tests__/scrapeProfileUrl.test.ts new file mode 100644 index 000000000..4694cd789 --- /dev/null +++ b/lib/apify/__tests__/scrapeProfileUrl.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +import { scrapeProfileUrl } from "@/lib/apify/scrapeProfileUrl"; + +const li = vi.fn(); +vi.mock("@/lib/apify/linkedin/startLinkedinProfileScraping", () => ({ + default: (...a: unknown[]) => li(...a), +})); +vi.mock("@/lib/apify/tiktok/startTiktokProfileScraping", () => ({ default: vi.fn() })); +vi.mock("@/lib/apify/instagram/startInstagramProfileScraping", () => ({ + startInstagramProfileScraping: vi.fn(), +})); +vi.mock("@/lib/apify/twitter/startTwitterProfileScraping", () => ({ default: vi.fn() })); +vi.mock("@/lib/apify/threads/startThreadsProfileScraping", () => ({ default: vi.fn() })); +vi.mock("@/lib/apify/youtube/startYoutubeProfileScraping", () => ({ default: vi.fn() })); +vi.mock("@/lib/apify/facebook/startFacebookProfileScraping", () => ({ default: vi.fn() })); +vi.mock("@/lib/socials/getUsernameFromProfileUrl", () => ({ + getUsernameFromProfileUrl: () => "drew-thurlow", +})); + +beforeEach(() => { + vi.clearAllMocks(); + li.mockResolvedValue({ runId: "li-run", datasetId: "li-ds" }); +}); + +describe("scrapeProfileUrl", () => { + it("dispatches linkedin.com to the LinkedIn scraper", async () => { + const r = await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "drew-thurlow"); + expect(li).toHaveBeenCalled(); + expect(r).toMatchObject({ supported: true, runId: "li-run", datasetId: "li-ds" }); + }); + it("returns null for an unsupported platform (spotify)", async () => { + const r = await scrapeProfileUrl("https://open.spotify.com/artist/x", "x"); + expect(r).toBeNull(); + expect(li).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/linkedin/startLinkedinProfileScraping.ts b/lib/apify/linkedin/startLinkedinProfileScraping.ts new file mode 100644 index 000000000..28c887411 --- /dev/null +++ b/lib/apify/linkedin/startLinkedinProfileScraping.ts @@ -0,0 +1,44 @@ +import apifyClient from "@/lib/apify/client"; +import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; +import { ApifyRunInfo } from "@/lib/apify/types"; + +/** + * Starts a LinkedIn profile scrape via the harvestapi actor. + * Mirrors the other `startProfileScraping` modules. + * + * @param handle - A LinkedIn vanity slug (e.g. `drew-thurlow`) or full profile URL. + * @returns Apify run info, or null if the run failed to start. + * @see https://apify.com/harvestapi/linkedin-profile-scraper + */ +const startLinkedinProfileScraping = async (handle: string): Promise => { + const cleanHandle = handle.trim().replace(/^@/, ""); + if (!cleanHandle) { + throw new Error("Invalid LinkedIn handle"); + } + + const targetUrl = cleanHandle.startsWith("http") + ? cleanHandle + : `https://www.linkedin.com/in/${cleanHandle}`; + + const input = { profiles: [targetUrl] }; + + try { + const run = await apifyClient.actor("harvestapi/linkedin-profile-scraper").start(input); + + if (!run?.id || !run?.defaultDatasetId) { + console.error("Failed to start LinkedIn profile scraping for handle:", handle); + return null; + } + + if (run.status === "FAILED" || run.status === "ABORTED") { + throw new Error(OUTSTANDING_ERROR); + } + + return { runId: run.id, datasetId: run.defaultDatasetId }; + } catch (error) { + console.error("Error in startLinkedinProfileScraping:", error); + throw error; + } +}; + +export default startLinkedinProfileScraping; diff --git a/lib/apify/scrapeProfileUrl.ts b/lib/apify/scrapeProfileUrl.ts index 3453d0556..6d30a930a 100644 --- a/lib/apify/scrapeProfileUrl.ts +++ b/lib/apify/scrapeProfileUrl.ts @@ -4,6 +4,7 @@ import startTwitterProfileScraping from "@/lib/apify/twitter/startTwitterProfile import startThreadsProfileScraping from "@/lib/apify/threads/startThreadsProfileScraping"; import startYoutubeProfileScraping from "@/lib/apify/youtube/startYoutubeProfileScraping"; import startFacebookProfileScraping from "@/lib/apify/facebook/startFacebookProfileScraping"; +import startLinkedinProfileScraping from "@/lib/apify/linkedin/startLinkedinProfileScraping"; import { getUsernameFromProfileUrl } from "@/lib/socials/getUsernameFromProfileUrl"; type ScrapeRunner = (handle: string) => Promise<{ @@ -51,6 +52,10 @@ const PLATFORM_SCRAPERS: Array<{ match: (url: string) => url.includes("facebook.com"), scraper: startFacebookProfileScraping, }, + { + match: (url: string) => url.includes("linkedin.com"), + scraper: startLinkedinProfileScraping, + }, ]; export const scrapeProfileUrl = async ( diff --git a/lib/artists/__tests__/getSocialPlatformByLink.test.ts b/lib/artists/__tests__/getSocialPlatformByLink.test.ts new file mode 100644 index 000000000..967d74aaf --- /dev/null +++ b/lib/artists/__tests__/getSocialPlatformByLink.test.ts @@ -0,0 +1,14 @@ +import { describe, it, expect } from "vitest"; +import { getSocialPlatformByLink } from "@/lib/artists/getSocialPlatformByLink"; + +describe("getSocialPlatformByLink", () => { + it("maps linkedin.com to LINKEDIN", () => { + expect(getSocialPlatformByLink("https://linkedin.com/in/drew-thurlow")).toBe("LINKEDIN"); + expect(getSocialPlatformByLink("linkedin.com/company/recoup")).toBe("LINKEDIN"); + }); + it("still maps the existing platforms", () => { + expect(getSocialPlatformByLink("x.com/theasf")).toBe("TWITTER"); + expect(getSocialPlatformByLink("tiktok.com/@apache_207")).toBe("TIKTOK"); + expect(getSocialPlatformByLink("")).toBe("NONE"); + }); +}); diff --git a/lib/artists/getSocialPlatformByLink.ts b/lib/artists/getSocialPlatformByLink.ts index 744ad626e..d76b53710 100644 --- a/lib/artists/getSocialPlatformByLink.ts +++ b/lib/artists/getSocialPlatformByLink.ts @@ -10,6 +10,7 @@ export function getSocialPlatformByLink(link: string): string { if (link.includes("instagram.com")) return "INSTAGRAM"; if (link.includes("spotify.com")) return "SPOTIFY"; if (link.includes("tiktok.com")) return "TIKTOK"; + if (link.includes("linkedin.com")) return "LINKEDIN"; if (link.includes("apple.com")) return "APPLE"; if (link.includes("youtube.")) return "YOUTUBE"; if (link.includes("facebook.com")) return "FACEBOOK"; From eec086ca115bc873e07f5dadc19cb1fd00d60ba2 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Wed, 1 Jul 2026 18:21:02 -0500 Subject: [PATCH 2/4] fix(linkedin): use the actor's real input key (urls) + parse /in/ handles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs caught verifying against the live harvestapi actor + a real LinkedIn social (recoupable/chat#1833): - The actor's input schema takes `urls` ("Profile URLs") / publicIdentifiers / queries — not `profiles`. Sending `profiles` would start a run with no targets. - getUsernameFromProfileUrl returned the LinkedIn path prefix ("in") instead of the handle, so a stored username of "in" would make the scraper build linkedin.com/in/in. Teach it linkedin.com/{in,company,school}/; this also fixes the username updateArtistSocials stores for LinkedIn adds. Verified: harvestapi input schema pulled from the actor's latest build (uZgrDy65OCHs0Chum); RED→GREEN tests for both. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../startLinkedinProfileScraping.test.ts | 25 +++++++++++++++++++ .../linkedin/startLinkedinProfileScraping.ts | 2 +- .../getUsernameFromProfileUrl.test.ts | 20 +++++++++++++++ lib/socials/getUsernameFromProfileUrl.ts | 6 +++++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 lib/apify/__tests__/startLinkedinProfileScraping.test.ts create mode 100644 lib/socials/__tests__/getUsernameFromProfileUrl.test.ts diff --git a/lib/apify/__tests__/startLinkedinProfileScraping.test.ts b/lib/apify/__tests__/startLinkedinProfileScraping.test.ts new file mode 100644 index 000000000..ed2a0d7fa --- /dev/null +++ b/lib/apify/__tests__/startLinkedinProfileScraping.test.ts @@ -0,0 +1,25 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import apifyClient from "@/lib/apify/client"; +import startLinkedinProfileScraping from "@/lib/apify/linkedin/startLinkedinProfileScraping"; + +const start = vi.fn(); +vi.mock("@/lib/apify/client", () => ({ default: { actor: vi.fn(() => ({ start })) } })); + +beforeEach(() => { + vi.clearAllMocks(); + start.mockResolvedValue({ id: "run-1", defaultDatasetId: "ds-1", status: "RUNNING" }); +}); + +describe("startLinkedinProfileScraping", () => { + it("starts the harvestapi actor with `urls` (the actor's real input key), building the /in/ URL from a handle", async () => { + const r = await startLinkedinProfileScraping("sweetmaneth"); + expect(apifyClient.actor).toHaveBeenCalledWith("harvestapi/linkedin-profile-scraper"); + expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/sweetmaneth"] }); + expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" }); + }); + + it("passes a full profile URL through unchanged", async () => { + await startLinkedinProfileScraping("https://www.linkedin.com/in/drew-thurlow"); + expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/drew-thurlow"] }); + }); +}); diff --git a/lib/apify/linkedin/startLinkedinProfileScraping.ts b/lib/apify/linkedin/startLinkedinProfileScraping.ts index 28c887411..6e08d7dd3 100644 --- a/lib/apify/linkedin/startLinkedinProfileScraping.ts +++ b/lib/apify/linkedin/startLinkedinProfileScraping.ts @@ -20,7 +20,7 @@ const startLinkedinProfileScraping = async (handle: string): Promise { + it("extracts the first path segment for standard platforms", () => { + expect(getUsernameFromProfileUrl("https://instagram.com/apacheoficial")).toBe("apacheoficial"); + expect(getUsernameFromProfileUrl("x.com/theasf")).toBe("theasf"); + expect(getUsernameFromProfileUrl("")).toBe(""); + }); + + it("extracts the handle from LinkedIn /in/, /company/, /school/ URLs (not the path prefix)", () => { + expect(getUsernameFromProfileUrl("https://www.linkedin.com/in/sweetmaneth/")).toBe( + "sweetmaneth", + ); + expect(getUsernameFromProfileUrl("linkedin.com/in/drew-thurlow-3a354311")).toBe( + "drew-thurlow-3a354311", + ); + expect(getUsernameFromProfileUrl("https://linkedin.com/company/recoupable")).toBe("recoupable"); + }); +}); diff --git a/lib/socials/getUsernameFromProfileUrl.ts b/lib/socials/getUsernameFromProfileUrl.ts index 385b8d280..b1dae71ac 100644 --- a/lib/socials/getUsernameFromProfileUrl.ts +++ b/lib/socials/getUsernameFromProfileUrl.ts @@ -11,6 +11,12 @@ export const getUsernameFromProfileUrl = (profileUrl: string | null | undefined) try { const normalizedUrl = profileUrl.toLowerCase().trim(); + // LinkedIn nests the handle under a path prefix (/in/, /company/, /school/) — + // the generic first-segment rule would return the prefix itself (e.g. "in"). + const linkedin = normalizedUrl.match(/linkedin\.com\/(?:in|company|school)\/([^/?]+)/); + if (linkedin) { + return linkedin[1]; + } const match = normalizedUrl.match(/(?:\.com|\.net)\/([^/?]+)/); return match ? match[1] : ""; } catch (error) { From bf9a2bbc37845f04d14826cfc383e01b285fc026 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Wed, 1 Jul 2026 18:26:28 -0500 Subject: [PATCH 3/4] fix(linkedin): dispatch the profile URL (not the stored username) to the scraper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live preview test: the run queried linkedin.com/in/in → 404 "Profile not found" — the dispatcher passed the DB row's stored username ("in", written by the old path-prefix helper bug), which the helper fix can't repair for existing rows. Add a per-entry input selector to PLATFORM_SCRAPERS; LinkedIn takes the full profile_url (the field the platform was matched on), making stored-username quality irrelevant for LinkedIn scrapes. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/apify/__tests__/scrapeProfileUrl.test.ts | 8 +++++--- lib/apify/scrapeProfileUrl.ts | 10 +++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/apify/__tests__/scrapeProfileUrl.test.ts b/lib/apify/__tests__/scrapeProfileUrl.test.ts index 4694cd789..d34e5ddee 100644 --- a/lib/apify/__tests__/scrapeProfileUrl.test.ts +++ b/lib/apify/__tests__/scrapeProfileUrl.test.ts @@ -24,9 +24,11 @@ beforeEach(() => { }); describe("scrapeProfileUrl", () => { - it("dispatches linkedin.com to the LinkedIn scraper", async () => { - const r = await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "drew-thurlow"); - expect(li).toHaveBeenCalled(); + it("dispatches linkedin.com to the LinkedIn scraper with the PROFILE URL (not the stored username)", async () => { + // Stored usernames for LinkedIn are unreliable (legacy rows hold the "/in/" path + // prefix), so the URL — the field the platform was matched on — is the input. + const r = await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "in"); + expect(li).toHaveBeenCalledWith("https://www.linkedin.com/in/drew-thurlow"); expect(r).toMatchObject({ supported: true, runId: "li-run", datasetId: "li-ds" }); }); it("returns null for an unsupported platform (spotify)", async () => { diff --git a/lib/apify/scrapeProfileUrl.ts b/lib/apify/scrapeProfileUrl.ts index 6d30a930a..2a4520cce 100644 --- a/lib/apify/scrapeProfileUrl.ts +++ b/lib/apify/scrapeProfileUrl.ts @@ -27,6 +27,12 @@ export type ScrapeProfileResult = ProfileScrapeResult & { const PLATFORM_SCRAPERS: Array<{ match: (url: string) => boolean; scraper: ScrapeRunner; + /** + * What to pass the scraper: the platform handle/username (default) or the + * full profile URL. LinkedIn takes the URL — stored usernames for it are + * unreliable (legacy rows hold the "/in/" path prefix, not the handle). + */ + input?: "url" | "username"; }> = [ { match: (url: string) => url.includes("tiktok.com"), @@ -55,6 +61,7 @@ const PLATFORM_SCRAPERS: Array<{ { match: (url: string) => url.includes("linkedin.com"), scraper: startLinkedinProfileScraping, + input: "url", }, ]; @@ -74,9 +81,10 @@ export const scrapeProfileUrl = async ( } const finalUsername = username || getUsernameFromProfileUrl(profileUrl); + const scraperInput = platform.input === "url" ? profileUrl : finalUsername; try { - const result = await platform.scraper(finalUsername); + const result = await platform.scraper(scraperInput); if (!result) { return { From b65c1d6980d125669a1d57afdc86fec27f268d42 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Wed, 1 Jul 2026 18:29:27 -0500 Subject: [PATCH 4/4] =?UTF-8?q?refactor(linkedin):=20keep=20the=20shared?= =?UTF-8?q?=20dispatcher=20untouched=20(KISS)=20=E2=80=94=20guard=20in=20t?= =?UTF-8?q?he=20module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: don't modify the shared PLATFORM_SCRAPERS config for one platform. Revert the input-selector change; instead startLinkedinProfileScraping rejects LinkedIn path prefixes ("in"/"company"/"school") as handles — legacy rows stored the prefix as username, and scraping linkedin.com/in/in would hit a real, wrong profile. Correct usernames come from the fixed getUsernameFromProfileUrl going forward; bad legacy rows now fail loudly instead of scraping the wrong person. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/apify/__tests__/scrapeProfileUrl.test.ts | 8 +++----- .../__tests__/startLinkedinProfileScraping.test.ts | 7 +++++++ lib/apify/linkedin/startLinkedinProfileScraping.ts | 6 ++++-- lib/apify/scrapeProfileUrl.ts | 10 +--------- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/apify/__tests__/scrapeProfileUrl.test.ts b/lib/apify/__tests__/scrapeProfileUrl.test.ts index d34e5ddee..8ea6e96a8 100644 --- a/lib/apify/__tests__/scrapeProfileUrl.test.ts +++ b/lib/apify/__tests__/scrapeProfileUrl.test.ts @@ -24,11 +24,9 @@ beforeEach(() => { }); describe("scrapeProfileUrl", () => { - it("dispatches linkedin.com to the LinkedIn scraper with the PROFILE URL (not the stored username)", async () => { - // Stored usernames for LinkedIn are unreliable (legacy rows hold the "/in/" path - // prefix), so the URL — the field the platform was matched on — is the input. - const r = await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "in"); - expect(li).toHaveBeenCalledWith("https://www.linkedin.com/in/drew-thurlow"); + it("dispatches linkedin.com to the LinkedIn scraper", async () => { + const r = await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "drew-thurlow"); + expect(li).toHaveBeenCalledWith("drew-thurlow"); expect(r).toMatchObject({ supported: true, runId: "li-run", datasetId: "li-ds" }); }); it("returns null for an unsupported platform (spotify)", async () => { diff --git a/lib/apify/__tests__/startLinkedinProfileScraping.test.ts b/lib/apify/__tests__/startLinkedinProfileScraping.test.ts index ed2a0d7fa..e9dc4a73b 100644 --- a/lib/apify/__tests__/startLinkedinProfileScraping.test.ts +++ b/lib/apify/__tests__/startLinkedinProfileScraping.test.ts @@ -22,4 +22,11 @@ describe("startLinkedinProfileScraping", () => { await startLinkedinProfileScraping("https://www.linkedin.com/in/drew-thurlow"); expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/drew-thurlow"] }); }); + it("rejects LinkedIn path prefixes as handles (legacy rows stored 'in') instead of scraping the wrong profile", async () => { + await expect(startLinkedinProfileScraping("in")).rejects.toThrow(/Invalid LinkedIn handle/); + await expect(startLinkedinProfileScraping("company")).rejects.toThrow( + /Invalid LinkedIn handle/, + ); + expect(start).not.toHaveBeenCalled(); + }); }); diff --git a/lib/apify/linkedin/startLinkedinProfileScraping.ts b/lib/apify/linkedin/startLinkedinProfileScraping.ts index 6e08d7dd3..6038eaa89 100644 --- a/lib/apify/linkedin/startLinkedinProfileScraping.ts +++ b/lib/apify/linkedin/startLinkedinProfileScraping.ts @@ -12,8 +12,10 @@ import { ApifyRunInfo } from "@/lib/apify/types"; */ const startLinkedinProfileScraping = async (handle: string): Promise => { const cleanHandle = handle.trim().replace(/^@/, ""); - if (!cleanHandle) { - throw new Error("Invalid LinkedIn handle"); + // Legacy rows stored the LinkedIn path prefix ("in") as the username — never + // scrape those (linkedin.com/in/in is a real, wrong profile URL); fail loudly. + if (!cleanHandle || ["in", "company", "school"].includes(cleanHandle.toLowerCase())) { + throw new Error(`Invalid LinkedIn handle: "${handle}"`); } const targetUrl = cleanHandle.startsWith("http") diff --git a/lib/apify/scrapeProfileUrl.ts b/lib/apify/scrapeProfileUrl.ts index 2a4520cce..6d30a930a 100644 --- a/lib/apify/scrapeProfileUrl.ts +++ b/lib/apify/scrapeProfileUrl.ts @@ -27,12 +27,6 @@ export type ScrapeProfileResult = ProfileScrapeResult & { const PLATFORM_SCRAPERS: Array<{ match: (url: string) => boolean; scraper: ScrapeRunner; - /** - * What to pass the scraper: the platform handle/username (default) or the - * full profile URL. LinkedIn takes the URL — stored usernames for it are - * unreliable (legacy rows hold the "/in/" path prefix, not the handle). - */ - input?: "url" | "username"; }> = [ { match: (url: string) => url.includes("tiktok.com"), @@ -61,7 +55,6 @@ const PLATFORM_SCRAPERS: Array<{ { match: (url: string) => url.includes("linkedin.com"), scraper: startLinkedinProfileScraping, - input: "url", }, ]; @@ -81,10 +74,9 @@ export const scrapeProfileUrl = async ( } const finalUsername = username || getUsernameFromProfileUrl(profileUrl); - const scraperInput = platform.input === "url" ? profileUrl : finalUsername; try { - const result = await platform.scraper(scraperInput); + const result = await platform.scraper(finalUsername); if (!result) { return {