diff --git a/lib/apify/__tests__/scrapeProfileUrl.test.ts b/lib/apify/__tests__/scrapeProfileUrl.test.ts index 8ea6e96a8..e78bc3f1e 100644 --- a/lib/apify/__tests__/scrapeProfileUrl.test.ts +++ b/lib/apify/__tests__/scrapeProfileUrl.test.ts @@ -26,9 +26,13 @@ beforeEach(() => { describe("scrapeProfileUrl", () => { it("dispatches linkedin.com to the LinkedIn scraper", async () => { const r = await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "drew-thurlow"); - expect(li).toHaveBeenCalledWith("drew-thurlow"); + expect(li).toHaveBeenCalledWith("drew-thurlow", undefined); expect(r).toMatchObject({ supported: true, runId: "li-run", datasetId: "li-ds" }); }); + it("forwards posts to the platform scraper", async () => { + await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "drew-thurlow", 20); + expect(li).toHaveBeenCalledWith("drew-thurlow", 20); + }); it("returns null for an unsupported platform (spotify)", async () => { const r = await scrapeProfileUrl("https://open.spotify.com/artist/x", "x"); expect(r).toBeNull(); diff --git a/lib/apify/__tests__/scrapeProfileUrlBatch.test.ts b/lib/apify/__tests__/scrapeProfileUrlBatch.test.ts new file mode 100644 index 000000000..df92bc9fe --- /dev/null +++ b/lib/apify/__tests__/scrapeProfileUrlBatch.test.ts @@ -0,0 +1,35 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +import { scrapeProfileUrlBatch } from "@/lib/apify/scrapeProfileUrlBatch"; +import { scrapeProfileUrl } from "@/lib/apify/scrapeProfileUrl"; + +vi.mock("@/lib/apify/scrapeProfileUrl", () => ({ scrapeProfileUrl: vi.fn() })); + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(scrapeProfileUrl).mockResolvedValue({ + runId: "r1", + datasetId: "d1", + error: null, + supported: true, + }); +}); + +describe("scrapeProfileUrlBatch", () => { + it("forwards posts to every scrapeProfileUrl call", async () => { + await scrapeProfileUrlBatch( + [ + { profileUrl: "https://x.com/a", username: "a" }, + { profileUrl: "https://youtube.com/@b", username: "b" }, + ], + 20, + ); + expect(scrapeProfileUrl).toHaveBeenCalledWith("https://x.com/a", "a", 20); + expect(scrapeProfileUrl).toHaveBeenCalledWith("https://youtube.com/@b", "b", 20); + }); + + it("passes posts as undefined when omitted (legacy behavior)", async () => { + await scrapeProfileUrlBatch([{ profileUrl: "https://x.com/a", username: "a" }]); + expect(scrapeProfileUrl).toHaveBeenCalledWith("https://x.com/a", "a", undefined); + }); +}); diff --git a/lib/apify/scrapeProfileUrl.ts b/lib/apify/scrapeProfileUrl.ts index 6d30a930a..10af0009d 100644 --- a/lib/apify/scrapeProfileUrl.ts +++ b/lib/apify/scrapeProfileUrl.ts @@ -7,7 +7,10 @@ import startFacebookProfileScraping from "@/lib/apify/facebook/startFacebookProf import startLinkedinProfileScraping from "@/lib/apify/linkedin/startLinkedinProfileScraping"; import { getUsernameFromProfileUrl } from "@/lib/socials/getUsernameFromProfileUrl"; -type ScrapeRunner = (handle: string) => Promise<{ +type ScrapeRunner = ( + handle: string, + posts?: number, +) => Promise<{ runId: string; datasetId: string; error?: string; @@ -61,6 +64,7 @@ const PLATFORM_SCRAPERS: Array<{ export const scrapeProfileUrl = async ( profileUrl: string | null | undefined, username: string, + posts?: number, ): Promise => { if (!profileUrl) { return null; @@ -76,7 +80,7 @@ export const scrapeProfileUrl = async ( const finalUsername = username || getUsernameFromProfileUrl(profileUrl); try { - const result = await platform.scraper(finalUsername); + const result = await platform.scraper(finalUsername, posts); if (!result) { return { diff --git a/lib/apify/scrapeProfileUrlBatch.ts b/lib/apify/scrapeProfileUrlBatch.ts index ff4ccb90a..5b73ea7b3 100644 --- a/lib/apify/scrapeProfileUrlBatch.ts +++ b/lib/apify/scrapeProfileUrlBatch.ts @@ -7,9 +7,12 @@ type ScrapeProfileUrlBatchInput = { export const scrapeProfileUrlBatch = async ( inputs: ScrapeProfileUrlBatchInput[], + posts?: number, ): Promise => { const results = await Promise.all( - inputs.map(({ profileUrl, username }) => scrapeProfileUrl(profileUrl ?? null, username ?? "")), + inputs.map(({ profileUrl, username }) => + scrapeProfileUrl(profileUrl ?? null, username ?? "", posts), + ), ); return results diff --git a/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts b/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts new file mode 100644 index 000000000..325e1f484 --- /dev/null +++ b/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts @@ -0,0 +1,36 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import apifyClient from "@/lib/apify/client"; +import startTwitterProfileScraping from "@/lib/apify/twitter/startTwitterProfileScraping"; + +const start = vi.fn(); +vi.mock("@/lib/apify/client", () => ({ default: { actor: vi.fn(() => ({ start })) } })); + +beforeEach(() => { + vi.clearAllMocks(); + start.mockResolvedValue({ id: "run-1", defaultDatasetId: "ds-1", status: "RUNNING" }); +}); + +describe("startTwitterProfileScraping", () => { + it("defaults to the legacy single-item snapshot (maxItems: 1) when posts is omitted", async () => { + const r = await startTwitterProfileScraping("sweetman_eth"); + expect(apifyClient.actor).toHaveBeenCalledWith("apidojo/twitter-scraper-lite"); + expect(start).toHaveBeenCalledWith( + { twitterHandles: ["sweetman_eth"], sort: "Latest", maxItems: 1 }, + { webhooks: expect.any(Array) }, + ); + expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" }); + }); + + it("passes posts through as maxItems", async () => { + await startTwitterProfileScraping("sweetman_eth", 20); + expect(start).toHaveBeenCalledWith( + { twitterHandles: ["sweetman_eth"], sort: "Latest", maxItems: 20 }, + { webhooks: expect.any(Array) }, + ); + }); + + it("throws on an empty handle", async () => { + await expect(startTwitterProfileScraping(" ")).rejects.toThrow(/Invalid Twitter handle/); + expect(start).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/twitter/startTwitterProfileScraping.ts b/lib/apify/twitter/startTwitterProfileScraping.ts index 52f288038..16821355d 100644 --- a/lib/apify/twitter/startTwitterProfileScraping.ts +++ b/lib/apify/twitter/startTwitterProfileScraping.ts @@ -2,7 +2,10 @@ import apifyClient from "@/lib/apify/client"; import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; import { ApifyRunInfo } from "@/lib/apify/types"; -const startTwitterProfileScraping = async (handle: string): Promise => { +const startTwitterProfileScraping = async ( + handle: string, + posts?: number, +): Promise => { const cleanHandle = handle.trim(); if (!cleanHandle) { @@ -12,7 +15,7 @@ const startTwitterProfileScraping = async (handle: string): Promise ({ default: { actor: vi.fn(() => ({ start })) } })); + +beforeEach(() => { + vi.clearAllMocks(); + start.mockResolvedValue({ id: "run-1", defaultDatasetId: "ds-1", status: "RUNNING" }); +}); + +describe("startYoutubeProfileScraping", () => { + it("defaults to the legacy snapshot (maxResults: 1, Shorts excluded) when posts is omitted", async () => { + const r = await startYoutubeProfileScraping("@mycowtf"); + expect(apifyClient.actor).toHaveBeenCalledWith("streamers/youtube-scraper"); + expect(start).toHaveBeenCalledWith( + expect.objectContaining({ + startUrls: [{ url: "https://www.youtube.com/@mycowtf" }], + maxResults: 1, + maxResultsShorts: 0, + }), + { webhooks: expect.any(Array) }, + ); + expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" }); + }); + + it("passes posts through as maxResults AND maxResultsShorts (Shorts included)", async () => { + await startYoutubeProfileScraping("mycowtf", 20); + expect(start).toHaveBeenCalledWith( + expect.objectContaining({ + startUrls: [{ url: "https://www.youtube.com/@mycowtf" }], + maxResults: 20, + maxResultsShorts: 20, + }), + { webhooks: expect.any(Array) }, + ); + }); + + it("throws on an empty handle", async () => { + await expect(startYoutubeProfileScraping(" ")).rejects.toThrow(/Invalid YouTube handle/); + expect(start).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/youtube/startYoutubeProfileScraping.ts b/lib/apify/youtube/startYoutubeProfileScraping.ts index f5a7324b5..a7f222bea 100644 --- a/lib/apify/youtube/startYoutubeProfileScraping.ts +++ b/lib/apify/youtube/startYoutubeProfileScraping.ts @@ -24,7 +24,10 @@ const DEFAULT_INPUT = { subtitlesFormat: "srt", }; -const startYoutubeProfileScraping = async (handle: string): Promise => { +const startYoutubeProfileScraping = async ( + handle: string, + posts?: number, +): Promise => { const cleanHandle = handle.trim().replace(/^@/, ""); if (!cleanHandle) { @@ -36,6 +39,9 @@ const startYoutubeProfileScraping = async (handle: string): Promise ({ scrapeProfileUrlBatch: vi.fn() })); +vi.mock("@/lib/supabase/account_socials/selectAccountSocials", () => ({ + selectAccountSocials: vi.fn(), +})); +vi.mock("@/lib/networking/getCorsHeaders", () => ({ getCorsHeaders: () => ({}) })); + +const ARTIST_ID = "660e8400-e29b-41d4-a716-446655440000"; + +const makeRequest = (body: unknown) => + new NextRequest("http://localhost/api/artist/socials/scrape", { + method: "POST", + body: JSON.stringify(body), + }); + +describe("postArtistSocialsScrapeHandler", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(selectAccountSocials).mockResolvedValue([ + { social: { profile_url: "https://x.com/a", username: "a" } } as never, + ]); + vi.mocked(scrapeProfileUrlBatch).mockResolvedValue([ + { runId: "r1", datasetId: "d1", error: null }, + ]); + }); + + it("returns 400 when artist_account_id is missing", async () => { + const res = await postArtistSocialsScrapeHandler(makeRequest({})); + expect(res.status).toBe(400); + expect(scrapeProfileUrlBatch).not.toHaveBeenCalled(); + }); + + it("scrapes without a posts depth by default (legacy behavior)", async () => { + const res = await postArtistSocialsScrapeHandler(makeRequest({ artist_account_id: ARTIST_ID })); + expect(res.status).toBe(200); + expect(scrapeProfileUrlBatch).toHaveBeenCalledWith( + [{ profileUrl: "https://x.com/a", username: "a" }], + undefined, + ); + }); + + it("forwards posts to scrapeProfileUrlBatch", async () => { + await postArtistSocialsScrapeHandler(makeRequest({ artist_account_id: ARTIST_ID, posts: 20 })); + expect(scrapeProfileUrlBatch).toHaveBeenCalledWith( + [{ profileUrl: "https://x.com/a", username: "a" }], + 20, + ); + }); + + it("returns [] when the artist has no socials", async () => { + vi.mocked(selectAccountSocials).mockResolvedValue([]); + const res = await postArtistSocialsScrapeHandler(makeRequest({ artist_account_id: ARTIST_ID })); + expect(res.status).toBe(200); + expect(await res.json()).toEqual([]); + expect(scrapeProfileUrlBatch).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/artist/__tests__/validateArtistSocialsScrapeBody.test.ts b/lib/artist/__tests__/validateArtistSocialsScrapeBody.test.ts new file mode 100644 index 000000000..fb431553d --- /dev/null +++ b/lib/artist/__tests__/validateArtistSocialsScrapeBody.test.ts @@ -0,0 +1,36 @@ +import { describe, it, expect, vi } from "vitest"; +import { NextResponse } from "next/server"; + +import { validateArtistSocialsScrapeBody } from "../validateArtistSocialsScrapeBody"; + +vi.mock("@/lib/networking/getCorsHeaders", () => ({ getCorsHeaders: () => ({}) })); + +const ARTIST_ID = "660e8400-e29b-41d4-a716-446655440000"; + +describe("validateArtistSocialsScrapeBody", () => { + it("returns 400 when artist_account_id is missing", () => { + const res = validateArtistSocialsScrapeBody({}) as NextResponse; + expect(res.status).toBe(400); + }); + + it("accepts a body without posts (legacy behavior)", () => { + expect(validateArtistSocialsScrapeBody({ artist_account_id: ARTIST_ID })).toEqual({ + artist_account_id: ARTIST_ID, + }); + }); + + it("accepts a valid posts field", () => { + expect(validateArtistSocialsScrapeBody({ artist_account_id: ARTIST_ID, posts: 20 })).toEqual({ + artist_account_id: ARTIST_ID, + posts: 20, + }); + }); + + it.each([[0], [101], [1.5], ["20"]])("returns 400 for invalid posts %s", posts => { + const res = validateArtistSocialsScrapeBody({ + artist_account_id: ARTIST_ID, + posts, + }) as NextResponse; + expect(res.status).toBe(400); + }); +}); diff --git a/lib/artist/postArtistSocialsScrapeHandler.ts b/lib/artist/postArtistSocialsScrapeHandler.ts index c144ce1a4..24242971f 100644 --- a/lib/artist/postArtistSocialsScrapeHandler.ts +++ b/lib/artist/postArtistSocialsScrapeHandler.ts @@ -36,6 +36,7 @@ export async function postArtistSocialsScrapeHandler(request: NextRequest): Prom profileUrl: social.social?.profile_url, username: social.social?.username, })), + validatedBody.posts, ); return NextResponse.json(results, { diff --git a/lib/artist/validateArtistSocialsScrapeBody.ts b/lib/artist/validateArtistSocialsScrapeBody.ts index a6d5cf8e7..099d3d682 100644 --- a/lib/artist/validateArtistSocialsScrapeBody.ts +++ b/lib/artist/validateArtistSocialsScrapeBody.ts @@ -4,6 +4,12 @@ import { z } from "zod"; export const artistSocialsScrapeBodySchema = z.object({ artist_account_id: z.string().min(1, "artist_account_id body parameter is required"), + posts: z + .number() + .int("posts must be an integer") + .min(1, "posts must be between 1 and 100") + .max(100, "posts must be between 1 and 100") + .optional(), }); export type ArtistSocialsScrapeBody = z.infer; diff --git a/lib/socials/__tests__/postSocialScrapeHandler.test.ts b/lib/socials/__tests__/postSocialScrapeHandler.test.ts index 4458ceb05..54ce9b307 100644 --- a/lib/socials/__tests__/postSocialScrapeHandler.test.ts +++ b/lib/socials/__tests__/postSocialScrapeHandler.test.ts @@ -45,6 +45,17 @@ describe("postSocialScrapeHandler", () => { const res = await postSocialScrapeHandler(request, SOCIAL_ID); expect(res.status).toBe(200); expect(await res.json()).toEqual({ runId: "r1", datasetId: "d1" }); + expect(scrapeProfileUrl).toHaveBeenCalledWith(social.profile_url, social.username, undefined); + }); + + it("forwards validated posts to scrapeProfileUrl", async () => { + vi.mocked(validatePostSocialScrapeRequest).mockResolvedValue({ + social_id: SOCIAL_ID, + posts: 20, + }); + vi.mocked(scrapeProfileUrl).mockResolvedValue({ runId: "r1", datasetId: "d1" } as never); + await postSocialScrapeHandler(request, SOCIAL_ID); + expect(scrapeProfileUrl).toHaveBeenCalledWith(social.profile_url, social.username, 20); }); it("returns 400 when platform unsupported (scrapeProfileUrl returns null)", async () => { diff --git a/lib/socials/__tests__/validatePostSocialScrapeRequest.test.ts b/lib/socials/__tests__/validatePostSocialScrapeRequest.test.ts index e42c6c4ef..d74d2c8d8 100644 --- a/lib/socials/__tests__/validatePostSocialScrapeRequest.test.ts +++ b/lib/socials/__tests__/validatePostSocialScrapeRequest.test.ts @@ -73,9 +73,34 @@ describe("validatePostSocialScrapeRequest", () => { it("returns validated payload when caller has access to an owning artist", async () => { expect(await validatePostSocialScrapeRequest(makeRequest(), SOCIAL_ID)).toEqual({ social_id: SOCIAL_ID, + posts: undefined, }); }); + it("parses a valid posts query param", async () => { + const req = new NextRequest(`http://localhost/api/socials/${SOCIAL_ID}/scrape?posts=20`, { + method: "POST", + headers: { "x-api-key": "k" }, + }); + expect(await validatePostSocialScrapeRequest(req, SOCIAL_ID)).toEqual({ + social_id: SOCIAL_ID, + posts: 20, + }); + }); + + it.each([["0"], ["101"], ["abc"], ["1.5"]])( + "returns 400 for invalid posts query param %s", + async posts => { + const req = new NextRequest( + `http://localhost/api/socials/${SOCIAL_ID}/scrape?posts=${posts}`, + { method: "POST", headers: { "x-api-key": "k" } }, + ); + const res = (await validatePostSocialScrapeRequest(req, SOCIAL_ID)) as NextResponse; + expect(res.status).toBe(400); + expect(validateAuthContext).not.toHaveBeenCalled(); + }, + ); + it("propagates DB error from selectAccountSocials (fails closed as 500)", async () => { vi.mocked(selectAccountSocials).mockRejectedValue(new Error("db blew up")); await expect(validatePostSocialScrapeRequest(makeRequest(), SOCIAL_ID)).rejects.toThrow( diff --git a/lib/socials/postSocialScrapeHandler.ts b/lib/socials/postSocialScrapeHandler.ts index 0e2ebb149..7489d6700 100644 --- a/lib/socials/postSocialScrapeHandler.ts +++ b/lib/socials/postSocialScrapeHandler.ts @@ -30,7 +30,11 @@ export async function postSocialScrapeHandler( ); } - const scrapeResult = await scrapeProfileUrl(social.profile_url, social.username); + const scrapeResult = await scrapeProfileUrl( + social.profile_url, + social.username, + validated.posts, + ); if (scrapeResult) { if (scrapeResult.error) { diff --git a/lib/socials/validatePostSocialScrapeRequest.ts b/lib/socials/validatePostSocialScrapeRequest.ts index 9081f9105..8feefe62c 100644 --- a/lib/socials/validatePostSocialScrapeRequest.ts +++ b/lib/socials/validatePostSocialScrapeRequest.ts @@ -9,6 +9,12 @@ import { checkAccountArtistAccess } from "@/lib/artists/checkAccountArtistAccess export const postSocialScrapeParamsSchema = z.object({ social_id: z.string().uuid("social_id must be a valid UUID"), + posts: z.coerce + .number() + .int("posts must be an integer") + .min(1, "posts must be between 1 and 100") + .max(100, "posts must be between 1 and 100") + .optional(), }); export type PostSocialScrapeParams = z.infer; @@ -17,7 +23,10 @@ export async function validatePostSocialScrapeRequest( request: NextRequest, id: string, ): Promise { - const parsed = postSocialScrapeParamsSchema.safeParse({ social_id: id }); + const parsed = postSocialScrapeParamsSchema.safeParse({ + social_id: id, + posts: request.nextUrl.searchParams.get("posts") ?? undefined, + }); if (!parsed.success) { const issue = parsed.error.issues[0]; return validationErrorResponse(issue.message, issue.path); @@ -51,5 +60,5 @@ export async function validatePostSocialScrapeRequest( return errorResponse("Unauthorized social scrape attempt", 403); } - return { social_id }; + return { social_id, posts: parsed.data.posts }; }