-
Notifications
You must be signed in to change notification settings - Fork 9
feat(apify): add LinkedIn profile scraping (harvestapi) #737
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
273c218
af9f2ac
eec086c
bf9a2bb
b65c1d6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| import { describe, it, expect, vi, beforeEach } from "vitest"; | ||
|
|
||
| import { scrapeProfileUrl } from "@/lib/apify/scrapeProfileUrl"; | ||
|
|
||
| const li = vi.fn(); | ||
| vi.mock("@/lib/apify/linkedin/startLinkedinProfileScraping", () => ({ | ||
| default: (...a: unknown[]) => li(...a), | ||
| })); | ||
| vi.mock("@/lib/apify/tiktok/startTiktokProfileScraping", () => ({ default: vi.fn() })); | ||
| vi.mock("@/lib/apify/instagram/startInstagramProfileScraping", () => ({ | ||
| startInstagramProfileScraping: vi.fn(), | ||
| })); | ||
| vi.mock("@/lib/apify/twitter/startTwitterProfileScraping", () => ({ default: vi.fn() })); | ||
| vi.mock("@/lib/apify/threads/startThreadsProfileScraping", () => ({ default: vi.fn() })); | ||
| vi.mock("@/lib/apify/youtube/startYoutubeProfileScraping", () => ({ default: vi.fn() })); | ||
| vi.mock("@/lib/apify/facebook/startFacebookProfileScraping", () => ({ default: vi.fn() })); | ||
| vi.mock("@/lib/socials/getUsernameFromProfileUrl", () => ({ | ||
| getUsernameFromProfileUrl: () => "drew-thurlow", | ||
| })); | ||
|
|
||
| beforeEach(() => { | ||
| vi.clearAllMocks(); | ||
| li.mockResolvedValue({ runId: "li-run", datasetId: "li-ds" }); | ||
| }); | ||
|
|
||
| describe("scrapeProfileUrl", () => { | ||
| it("dispatches linkedin.com to the LinkedIn scraper", async () => { | ||
| const r = await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "drew-thurlow"); | ||
| expect(li).toHaveBeenCalledWith("drew-thurlow"); | ||
| expect(r).toMatchObject({ supported: true, runId: "li-run", datasetId: "li-ds" }); | ||
| }); | ||
| it("returns null for an unsupported platform (spotify)", async () => { | ||
| const r = await scrapeProfileUrl("https://open.spotify.com/artist/x", "x"); | ||
| expect(r).toBeNull(); | ||
| expect(li).not.toHaveBeenCalled(); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| import { describe, it, expect, vi, beforeEach } from "vitest"; | ||
| import apifyClient from "@/lib/apify/client"; | ||
| import startLinkedinProfileScraping from "@/lib/apify/linkedin/startLinkedinProfileScraping"; | ||
|
|
||
| const start = vi.fn(); | ||
| vi.mock("@/lib/apify/client", () => ({ default: { actor: vi.fn(() => ({ start })) } })); | ||
|
|
||
| beforeEach(() => { | ||
| vi.clearAllMocks(); | ||
| start.mockResolvedValue({ id: "run-1", defaultDatasetId: "ds-1", status: "RUNNING" }); | ||
| }); | ||
|
|
||
| describe("startLinkedinProfileScraping", () => { | ||
| it("starts the harvestapi actor with `urls` (the actor's real input key), building the /in/ URL from a handle", async () => { | ||
| const r = await startLinkedinProfileScraping("sweetmaneth"); | ||
| expect(apifyClient.actor).toHaveBeenCalledWith("harvestapi/linkedin-profile-scraper"); | ||
| expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/sweetmaneth"] }); | ||
| expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" }); | ||
| }); | ||
|
|
||
| it("passes a full profile URL through unchanged", async () => { | ||
| await startLinkedinProfileScraping("https://www.linkedin.com/in/drew-thurlow"); | ||
| expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/drew-thurlow"] }); | ||
| }); | ||
| it("rejects LinkedIn path prefixes as handles (legacy rows stored 'in') instead of scraping the wrong profile", async () => { | ||
| await expect(startLinkedinProfileScraping("in")).rejects.toThrow(/Invalid LinkedIn handle/); | ||
| await expect(startLinkedinProfileScraping("company")).rejects.toThrow( | ||
| /Invalid LinkedIn handle/, | ||
| ); | ||
| expect(start).not.toHaveBeenCalled(); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| import apifyClient from "@/lib/apify/client"; | ||
| import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; | ||
| import { ApifyRunInfo } from "@/lib/apify/types"; | ||
|
|
||
| /** | ||
| * Starts a LinkedIn profile scrape via the harvestapi actor. | ||
| * Mirrors the other `start<Platform>ProfileScraping` modules. | ||
| * | ||
| * @param handle - A LinkedIn vanity slug (e.g. `drew-thurlow`) or full profile URL. | ||
| * @returns Apify run info, or null if the run failed to start. | ||
| * @see https://apify.com/harvestapi/linkedin-profile-scraper | ||
| */ | ||
| const startLinkedinProfileScraping = async (handle: string): Promise<ApifyRunInfo | null> => { | ||
| const cleanHandle = handle.trim().replace(/^@/, ""); | ||
| // Legacy rows stored the LinkedIn path prefix ("in") as the username — never | ||
| // scrape those (linkedin.com/in/in is a real, wrong profile URL); fail loudly. | ||
| if (!cleanHandle || ["in", "company", "school"].includes(cleanHandle.toLowerCase())) { | ||
| throw new Error(`Invalid LinkedIn handle: "${handle}"`); | ||
| } | ||
|
|
||
| const targetUrl = cleanHandle.startsWith("http") | ||
| ? cleanHandle | ||
| : `https://www.linkedin.com/in/${cleanHandle}`; | ||
|
|
||
| const input = { urls: [targetUrl] }; | ||
|
|
||
| try { | ||
| const run = await apifyClient.actor("harvestapi/linkedin-profile-scraper").start(input); | ||
|
|
||
| if (!run?.id || !run?.defaultDatasetId) { | ||
| console.error("Failed to start LinkedIn profile scraping for handle:", handle); | ||
| return null; | ||
| } | ||
|
|
||
| if (run.status === "FAILED" || run.status === "ABORTED") { | ||
| throw new Error(OUTSTANDING_ERROR); | ||
| } | ||
|
Comment on lines
+35
to
+37
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win 🧩 Analysis chain🌐 Web query:
💡 Result: The Apify JS client's actor.start method starts an actor and immediately returns the run object, which reflects the initial state of the run, such as "READY" or "RUNNING" [1][2][3]. It does not wait for the actor to complete, so it will not return a terminal status like "SUCCEEDED" or "FAILED" at the time of resolution [1][2][3]. If you need to wait for the actor to finish, you should use the actor.call method instead, which polls the run status and returns the run object only after the actor has completed [2][4]. Alternatively, you can use the start method with a waitForFinish option (in some versions/configurations) or manage the wait manually using the returned run ID [5][2]. Citations:
🏁 Script executed: #!/bin/bash
set -euo pipefail
# Inspect the target file with line numbers.
sed -n '1,220p' lib/apify/linkedin/startLinkedinProfileScraping.ts | cat -n
# Find nearby Apify start/call usage for context.
rg -n "actor\\.(start|call)|run\\.status|FAILED|ABORTED" lib/apify -SRepository: recoupable/api Length of output: 3192 Move terminal-status handling out of this start path 🤖 Prompt for AI Agents |
||
|
|
||
| return { runId: run.id, datasetId: run.defaultDatasetId }; | ||
| } catch (error) { | ||
| console.error("Error in startLinkedinProfileScraping:", error); | ||
| throw error; | ||
| } | ||
| }; | ||
|
|
||
| export default startLinkedinProfileScraping; | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| import { describe, it, expect } from "vitest"; | ||
| import { getSocialPlatformByLink } from "@/lib/artists/getSocialPlatformByLink"; | ||
|
|
||
| describe("getSocialPlatformByLink", () => { | ||
| it("maps linkedin.com to LINKEDIN", () => { | ||
| expect(getSocialPlatformByLink("https://linkedin.com/in/drew-thurlow")).toBe("LINKEDIN"); | ||
| expect(getSocialPlatformByLink("linkedin.com/company/recoup")).toBe("LINKEDIN"); | ||
| }); | ||
| it("still maps the existing platforms", () => { | ||
| expect(getSocialPlatformByLink("x.com/theasf")).toBe("TWITTER"); | ||
| expect(getSocialPlatformByLink("tiktok.com/@apache_207")).toBe("TIKTOK"); | ||
| expect(getSocialPlatformByLink("")).toBe("NONE"); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -10,6 +10,7 @@ export function getSocialPlatformByLink(link: string): string { | |||||
| if (link.includes("instagram.com")) return "INSTAGRAM"; | ||||||
| if (link.includes("spotify.com")) return "SPOTIFY"; | ||||||
| if (link.includes("tiktok.com")) return "TIKTOK"; | ||||||
| if (link.includes("linkedin.com")) return "LINKEDIN"; | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: The LinkedIn detection matches any string containing Prompt for AI agents
Suggested change
|
||||||
| if (link.includes("apple.com")) return "APPLE"; | ||||||
| if (link.includes("youtube.")) return "YOUTUBE"; | ||||||
| if (link.includes("facebook.com")) return "FACEBOOK"; | ||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| import { describe, it, expect } from "vitest"; | ||
| import { getUsernameFromProfileUrl } from "@/lib/socials/getUsernameFromProfileUrl"; | ||
|
|
||
| describe("getUsernameFromProfileUrl", () => { | ||
| it("extracts the first path segment for standard platforms", () => { | ||
| expect(getUsernameFromProfileUrl("https://instagram.com/apacheoficial")).toBe("apacheoficial"); | ||
| expect(getUsernameFromProfileUrl("x.com/theasf")).toBe("theasf"); | ||
| expect(getUsernameFromProfileUrl("")).toBe(""); | ||
| }); | ||
|
|
||
| it("extracts the handle from LinkedIn /in/, /company/, /school/ URLs (not the path prefix)", () => { | ||
| expect(getUsernameFromProfileUrl("https://www.linkedin.com/in/sweetmaneth/")).toBe( | ||
| "sweetmaneth", | ||
| ); | ||
| expect(getUsernameFromProfileUrl("linkedin.com/in/drew-thurlow-3a354311")).toBe( | ||
| "drew-thurlow-3a354311", | ||
| ); | ||
| expect(getUsernameFromProfileUrl("https://linkedin.com/company/recoupable")).toBe("recoupable"); | ||
| }); | ||
| }); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🩺 Stability & Availability | 🟠 Major | ⚡ Quick win
No timeout on external Apify call.
apifyClient.actor(...).start(input)has no timeout/abort mechanism. If the Apify API hangs, this call could block the request indefinitely, since it appears to run on a request-handling path viascrapeProfileUrl.🕐 Suggested guard using AbortController
🤖 Prompt for AI Agents