Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions lib/apify/__tests__/scrapeProfileUrl.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { describe, it, expect, vi, beforeEach } from "vitest";

import { scrapeProfileUrl } from "@/lib/apify/scrapeProfileUrl";

const li = vi.fn();
vi.mock("@/lib/apify/linkedin/startLinkedinProfileScraping", () => ({
default: (...a: unknown[]) => li(...a),
}));
vi.mock("@/lib/apify/tiktok/startTiktokProfileScraping", () => ({ default: vi.fn() }));
vi.mock("@/lib/apify/instagram/startInstagramProfileScraping", () => ({
startInstagramProfileScraping: vi.fn(),
}));
vi.mock("@/lib/apify/twitter/startTwitterProfileScraping", () => ({ default: vi.fn() }));
vi.mock("@/lib/apify/threads/startThreadsProfileScraping", () => ({ default: vi.fn() }));
vi.mock("@/lib/apify/youtube/startYoutubeProfileScraping", () => ({ default: vi.fn() }));
vi.mock("@/lib/apify/facebook/startFacebookProfileScraping", () => ({ default: vi.fn() }));
vi.mock("@/lib/socials/getUsernameFromProfileUrl", () => ({
getUsernameFromProfileUrl: () => "drew-thurlow",
}));

beforeEach(() => {
vi.clearAllMocks();
li.mockResolvedValue({ runId: "li-run", datasetId: "li-ds" });
});

describe("scrapeProfileUrl", () => {
it("dispatches linkedin.com to the LinkedIn scraper", async () => {
const r = await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "drew-thurlow");
expect(li).toHaveBeenCalledWith("drew-thurlow");
expect(r).toMatchObject({ supported: true, runId: "li-run", datasetId: "li-ds" });
});
it("returns null for an unsupported platform (spotify)", async () => {
const r = await scrapeProfileUrl("https://open.spotify.com/artist/x", "x");
expect(r).toBeNull();
expect(li).not.toHaveBeenCalled();
});
});
32 changes: 32 additions & 0 deletions lib/apify/__tests__/startLinkedinProfileScraping.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import apifyClient from "@/lib/apify/client";
import startLinkedinProfileScraping from "@/lib/apify/linkedin/startLinkedinProfileScraping";

const start = vi.fn();
vi.mock("@/lib/apify/client", () => ({ default: { actor: vi.fn(() => ({ start })) } }));

beforeEach(() => {
vi.clearAllMocks();
start.mockResolvedValue({ id: "run-1", defaultDatasetId: "ds-1", status: "RUNNING" });
});

describe("startLinkedinProfileScraping", () => {
it("starts the harvestapi actor with `urls` (the actor's real input key), building the /in/ URL from a handle", async () => {
const r = await startLinkedinProfileScraping("sweetmaneth");
expect(apifyClient.actor).toHaveBeenCalledWith("harvestapi/linkedin-profile-scraper");
expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/sweetmaneth"] });
expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" });
});

it("passes a full profile URL through unchanged", async () => {
await startLinkedinProfileScraping("https://www.linkedin.com/in/drew-thurlow");
expect(start).toHaveBeenCalledWith({ urls: ["https://www.linkedin.com/in/drew-thurlow"] });
});
it("rejects LinkedIn path prefixes as handles (legacy rows stored 'in') instead of scraping the wrong profile", async () => {
await expect(startLinkedinProfileScraping("in")).rejects.toThrow(/Invalid LinkedIn handle/);
await expect(startLinkedinProfileScraping("company")).rejects.toThrow(
/Invalid LinkedIn handle/,
);
expect(start).not.toHaveBeenCalled();
});
});
46 changes: 46 additions & 0 deletions lib/apify/linkedin/startLinkedinProfileScraping.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import apifyClient from "@/lib/apify/client";
import { OUTSTANDING_ERROR } from "@/lib/apify/errors";
import { ApifyRunInfo } from "@/lib/apify/types";

/**
* Starts a LinkedIn profile scrape via the harvestapi actor.
* Mirrors the other `start<Platform>ProfileScraping` modules.
*
* @param handle - A LinkedIn vanity slug (e.g. `drew-thurlow`) or full profile URL.
* @returns Apify run info, or null if the run failed to start.
* @see https://apify.com/harvestapi/linkedin-profile-scraper
*/
const startLinkedinProfileScraping = async (handle: string): Promise<ApifyRunInfo | null> => {
const cleanHandle = handle.trim().replace(/^@/, "");
// Legacy rows stored the LinkedIn path prefix ("in") as the username — never
// scrape those (linkedin.com/in/in is a real, wrong profile URL); fail loudly.
if (!cleanHandle || ["in", "company", "school"].includes(cleanHandle.toLowerCase())) {
throw new Error(`Invalid LinkedIn handle: "${handle}"`);
}

const targetUrl = cleanHandle.startsWith("http")
? cleanHandle
: `https://www.linkedin.com/in/${cleanHandle}`;

const input = { urls: [targetUrl] };

try {
const run = await apifyClient.actor("harvestapi/linkedin-profile-scraper").start(input);

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win

No timeout on external Apify call.

apifyClient.actor(...).start(input) has no timeout/abort mechanism. If the Apify API hangs, this call could block the request indefinitely, since it appears to run on a request-handling path via scrapeProfileUrl.

🕐 Suggested guard using AbortController
-    const run = await apifyClient.actor("harvestapi/linkedin-profile-scraper").start(input);
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 30_000);
+    let run;
+    try {
+      run = await apifyClient
+        .actor("harvestapi/linkedin-profile-scraper")
+        .start(input, { signal: controller.signal });
+    } finally {
+      clearTimeout(timeout);
+    }
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@lib/apify/linkedin/startLinkedinProfileScraping.ts` at line 26, The Apify
actor start call in startLinkedinProfileScraping is missing an abort/timeout
guard, so it can hang indefinitely on the request path. Update the scrape flow
around apifyClient.actor("harvestapi/linkedin-profile-scraper").start(input) to
use an AbortController or equivalent timeout mechanism, pass the signal into the
request if supported, and ensure the timeout is handled cleanly by surfacing a
controlled error instead of blocking the request.


if (!run?.id || !run?.defaultDatasetId) {
console.error("Failed to start LinkedIn profile scraping for handle:", handle);
return null;
}

if (run.status === "FAILED" || run.status === "ABORTED") {
throw new Error(OUTSTANDING_ERROR);
}
Comment on lines +35 to +37

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win

🧩 Analysis chain

🌐 Web query:

Does Apify JS client actor().start() return a run with terminal status like FAILED or does it only return the initial queued status?

💡 Result:

The Apify JS client's actor.start method starts an actor and immediately returns the run object, which reflects the initial state of the run, such as "READY" or "RUNNING" [1][2][3]. It does not wait for the actor to complete, so it will not return a terminal status like "SUCCEEDED" or "FAILED" at the time of resolution [1][2][3]. If you need to wait for the actor to finish, you should use the actor.call method instead, which polls the run status and returns the run object only after the actor has completed [2][4]. Alternatively, you can use the start method with a waitForFinish option (in some versions/configurations) or manage the wait manually using the returned run ID [5][2].

Citations:


🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Inspect the target file with line numbers.
sed -n '1,220p' lib/apify/linkedin/startLinkedinProfileScraping.ts | cat -n

# Find nearby Apify start/call usage for context.
rg -n "actor\\.(start|call)|run\\.status|FAILED|ABORTED" lib/apify -S

Repository: recoupable/api

Length of output: 3192


Move terminal-status handling out of this start path .start() resolves before the run finishes, so FAILED/ABORTED won’t be observed here. If this step should wait for completion, use .call(); otherwise, handle terminal states in the polling path instead.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@lib/apify/linkedin/startLinkedinProfileScraping.ts` around lines 33 - 35, The
terminal-status check in startLinkedinProfileScraping’s .start() path is
misplaced because .start() returns before the run completes, so FAILED and
ABORTED will not be seen here. Update this flow by either switching the LinkedIn
run invocation to .call() if this function must wait for completion, or remove
the run.status check from startLinkedinProfileScraping and move terminal-state
handling into the polling/completion path that observes the finished run.


return { runId: run.id, datasetId: run.defaultDatasetId };
} catch (error) {
console.error("Error in startLinkedinProfileScraping:", error);
throw error;
}
};

export default startLinkedinProfileScraping;
5 changes: 5 additions & 0 deletions lib/apify/scrapeProfileUrl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import startTwitterProfileScraping from "@/lib/apify/twitter/startTwitterProfile
import startThreadsProfileScraping from "@/lib/apify/threads/startThreadsProfileScraping";
import startYoutubeProfileScraping from "@/lib/apify/youtube/startYoutubeProfileScraping";
import startFacebookProfileScraping from "@/lib/apify/facebook/startFacebookProfileScraping";
import startLinkedinProfileScraping from "@/lib/apify/linkedin/startLinkedinProfileScraping";
import { getUsernameFromProfileUrl } from "@/lib/socials/getUsernameFromProfileUrl";

type ScrapeRunner = (handle: string) => Promise<{
Expand Down Expand Up @@ -51,6 +52,10 @@ const PLATFORM_SCRAPERS: Array<{
match: (url: string) => url.includes("facebook.com"),
scraper: startFacebookProfileScraping,
},
{
match: (url: string) => url.includes("linkedin.com"),
scraper: startLinkedinProfileScraping,
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
},
];

export const scrapeProfileUrl = async (
Expand Down
14 changes: 14 additions & 0 deletions lib/artists/__tests__/getSocialPlatformByLink.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import { describe, it, expect } from "vitest";
import { getSocialPlatformByLink } from "@/lib/artists/getSocialPlatformByLink";

describe("getSocialPlatformByLink", () => {
it("maps linkedin.com to LINKEDIN", () => {
expect(getSocialPlatformByLink("https://linkedin.com/in/drew-thurlow")).toBe("LINKEDIN");
expect(getSocialPlatformByLink("linkedin.com/company/recoup")).toBe("LINKEDIN");
});
it("still maps the existing platforms", () => {
expect(getSocialPlatformByLink("x.com/theasf")).toBe("TWITTER");
expect(getSocialPlatformByLink("tiktok.com/@apache_207")).toBe("TIKTOK");
expect(getSocialPlatformByLink("")).toBe("NONE");
});
});
1 change: 1 addition & 0 deletions lib/artists/getSocialPlatformByLink.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ export function getSocialPlatformByLink(link: string): string {
if (link.includes("instagram.com")) return "INSTAGRAM";
if (link.includes("spotify.com")) return "SPOTIFY";
if (link.includes("tiktok.com")) return "TIKTOK";
if (link.includes("linkedin.com")) return "LINKEDIN";

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: The LinkedIn detection matches any string containing linkedin.com, so unrelated URLs like notlinkedin.com can be misclassified as LinkedIn. Match the hostname boundary instead of using a raw substring check.

Prompt for AI agents
Check if this issue is valid — if so, understand the root cause and fix it. At lib/artists/getSocialPlatformByLink.ts, line 13:

<comment>The LinkedIn detection matches any string containing `linkedin.com`, so unrelated URLs like `notlinkedin.com` can be misclassified as LinkedIn. Match the hostname boundary instead of using a raw substring check.</comment>

<file context>
@@ -10,6 +10,7 @@ export function getSocialPlatformByLink(link: string): string {
   if (link.includes("instagram.com")) return "INSTAGRAM";
   if (link.includes("spotify.com")) return "SPOTIFY";
   if (link.includes("tiktok.com")) return "TIKTOK";
+  if (link.includes("linkedin.com")) return "LINKEDIN";
   if (link.includes("apple.com")) return "APPLE";
   if (link.includes("youtube.")) return "YOUTUBE";
</file context>
Suggested change
if (link.includes("linkedin.com")) return "LINKEDIN";
if (/(^|\/\/|\.)linkedin\.com(\/|$)/.test(link)) return "LINKEDIN";

if (link.includes("apple.com")) return "APPLE";
if (link.includes("youtube.")) return "YOUTUBE";
if (link.includes("facebook.com")) return "FACEBOOK";
Expand Down
20 changes: 20 additions & 0 deletions lib/socials/__tests__/getUsernameFromProfileUrl.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import { describe, it, expect } from "vitest";
import { getUsernameFromProfileUrl } from "@/lib/socials/getUsernameFromProfileUrl";

describe("getUsernameFromProfileUrl", () => {
it("extracts the first path segment for standard platforms", () => {
expect(getUsernameFromProfileUrl("https://instagram.com/apacheoficial")).toBe("apacheoficial");
expect(getUsernameFromProfileUrl("x.com/theasf")).toBe("theasf");
expect(getUsernameFromProfileUrl("")).toBe("");
});

it("extracts the handle from LinkedIn /in/, /company/, /school/ URLs (not the path prefix)", () => {
expect(getUsernameFromProfileUrl("https://www.linkedin.com/in/sweetmaneth/")).toBe(
"sweetmaneth",
);
expect(getUsernameFromProfileUrl("linkedin.com/in/drew-thurlow-3a354311")).toBe(
"drew-thurlow-3a354311",
);
expect(getUsernameFromProfileUrl("https://linkedin.com/company/recoupable")).toBe("recoupable");
});
});
6 changes: 6 additions & 0 deletions lib/socials/getUsernameFromProfileUrl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@ export const getUsernameFromProfileUrl = (profileUrl: string | null | undefined)

try {
const normalizedUrl = profileUrl.toLowerCase().trim();
// LinkedIn nests the handle under a path prefix (/in/, /company/, /school/) —
// the generic first-segment rule would return the prefix itself (e.g. "in").
const linkedin = normalizedUrl.match(/linkedin\.com\/(?:in|company|school)\/([^/?]+)/);
if (linkedin) {
return linkedin[1];
}
const match = normalizedUrl.match(/(?:\.com|\.net)\/([^/?]+)/);
return match ? match[1] : "";
} catch (error) {
Expand Down
Loading