From 31cd24636755bcf9a11ddd0c21ce8987a94ed2cd Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Wed, 1 Jul 2026 19:04:04 -0500 Subject: [PATCH] feat(apify): persist scrape results for the remaining 5 platforms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the persistence work started in #738 (registry + LinkedIn): TikTok, X/Twitter, YouTube, Threads, and Facebook scrapes now write back to socials instead of being poll-only (recoupable/chat#1833). Per platform (mirroring lib/apify/linkedin/): - start module attaches getApifyWebhooks() to the run (the root cause of "nothing persists" — only IG/LinkedIn runs ever called /api/apify). - handleProfileScraperResults upserts socials (keyed profile_url), with the field mapping taken from a REAL actor run of each platform: tiktok authorMeta (run G4YRI0e…), twitter author (ALVMZYX…), youtube channel fields (H0ZrIAs…), threads profile (9iiG1sD…), facebook page (ICZxdJB…). - registry entry under the actor id resolved from the Apify API. Round-trip hazards handled explicitly: twitter URLs are lowercased (actor echoes display casing, stored rows are lowercase — upsert would duplicate); youtube keys on inputChannelUrl (the URL we passed) because the actor's own channelUrl is the /channel/UC… form which never matches the stored @handle. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/apifyWebhookHandler.test.ts | 15 ++++++ .../__tests__/getApifyResultHandler.test.ts | 27 ++++++++++ ...andleFacebookProfileScraperResults.test.ts | 46 ++++++++++++++++ .../handleFacebookProfileScraperResults.ts | 30 +++++++++++ .../facebook/startFacebookProfileScraping.ts | 18 ++++--- lib/apify/getApifyResultHandler.ts | 15 ++++-- ...handleThreadsProfileScraperResults.test.ts | 46 ++++++++++++++++ .../handleThreadsProfileScraperResults.ts | 30 +++++++++++ .../threads/startThreadsProfileScraping.ts | 10 ++-- .../handleTiktokProfileScraperResults.test.ts | 51 ++++++++++++++++++ .../handleTiktokProfileScraperResults.ts | 38 +++++++++++++ .../tiktok/startTiktokProfileScraping.ts | 5 +- ...handleTwitterProfileScraperResults.test.ts | 53 +++++++++++++++++++ .../handleTwitterProfileScraperResults.ts | 42 +++++++++++++++ .../twitter/startTwitterProfileScraping.ts | 5 +- ...handleYoutubeProfileScraperResults.test.ts | 49 +++++++++++++++++ .../handleYoutubeProfileScraperResults.ts | 37 +++++++++++++ .../youtube/startYoutubeProfileScraping.ts | 5 +- 18 files changed, 506 insertions(+), 16 deletions(-) create mode 100644 lib/apify/facebook/__tests__/handleFacebookProfileScraperResults.test.ts create mode 100644 lib/apify/facebook/handleFacebookProfileScraperResults.ts create mode 100644 lib/apify/threads/__tests__/handleThreadsProfileScraperResults.test.ts create mode 100644 lib/apify/threads/handleThreadsProfileScraperResults.ts create mode 100644 lib/apify/tiktok/__tests__/handleTiktokProfileScraperResults.test.ts create mode 100644 lib/apify/tiktok/handleTiktokProfileScraperResults.ts create mode 100644 lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts create mode 100644 lib/apify/twitter/handleTwitterProfileScraperResults.ts create mode 100644 lib/apify/youtube/__tests__/handleYoutubeProfileScraperResults.test.ts create mode 100644 lib/apify/youtube/handleYoutubeProfileScraperResults.ts diff --git a/lib/apify/__tests__/apifyWebhookHandler.test.ts b/lib/apify/__tests__/apifyWebhookHandler.test.ts index 372575b75..98f7aa7d2 100644 --- a/lib/apify/__tests__/apifyWebhookHandler.test.ts +++ b/lib/apify/__tests__/apifyWebhookHandler.test.ts @@ -13,6 +13,21 @@ vi.mock("../instagram/handleInstagramCommentsScraper", () => ({ vi.mock("../linkedin/handleLinkedinProfileScraperResults", () => ({ handleLinkedinProfileScraperResults: vi.fn(), })); +vi.mock("../tiktok/handleTiktokProfileScraperResults", () => ({ + handleTiktokProfileScraperResults: vi.fn(), +})); +vi.mock("../twitter/handleTwitterProfileScraperResults", () => ({ + handleTwitterProfileScraperResults: vi.fn(), +})); +vi.mock("../youtube/handleYoutubeProfileScraperResults", () => ({ + handleYoutubeProfileScraperResults: vi.fn(), +})); +vi.mock("../threads/handleThreadsProfileScraperResults", () => ({ + handleThreadsProfileScraperResults: vi.fn(), +})); +vi.mock("../facebook/handleFacebookProfileScraperResults", () => ({ + handleFacebookProfileScraperResults: vi.fn(), +})); function makeRequest(body: unknown, raw?: string) { return new NextRequest("http://localhost/api/apify", { diff --git a/lib/apify/__tests__/getApifyResultHandler.test.ts b/lib/apify/__tests__/getApifyResultHandler.test.ts index c6ad3bf15..8cfaeab70 100644 --- a/lib/apify/__tests__/getApifyResultHandler.test.ts +++ b/lib/apify/__tests__/getApifyResultHandler.test.ts @@ -3,6 +3,11 @@ import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler"; import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; +import { handleTiktokProfileScraperResults } from "@/lib/apify/tiktok/handleTiktokProfileScraperResults"; +import { handleTwitterProfileScraperResults } from "@/lib/apify/twitter/handleTwitterProfileScraperResults"; +import { handleYoutubeProfileScraperResults } from "@/lib/apify/youtube/handleYoutubeProfileScraperResults"; +import { handleThreadsProfileScraperResults } from "@/lib/apify/threads/handleThreadsProfileScraperResults"; +import { handleFacebookProfileScraperResults } from "@/lib/apify/facebook/handleFacebookProfileScraperResults"; vi.mock("@/lib/apify/instagram/handleInstagramProfileScraperResults", () => ({ handleInstagramProfileScraperResults: vi.fn(), @@ -13,6 +18,21 @@ vi.mock("@/lib/apify/instagram/handleInstagramCommentsScraper", () => ({ vi.mock("@/lib/apify/linkedin/handleLinkedinProfileScraperResults", () => ({ handleLinkedinProfileScraperResults: vi.fn(), })); +vi.mock("@/lib/apify/tiktok/handleTiktokProfileScraperResults", () => ({ + handleTiktokProfileScraperResults: vi.fn(), +})); +vi.mock("@/lib/apify/twitter/handleTwitterProfileScraperResults", () => ({ + handleTwitterProfileScraperResults: vi.fn(), +})); +vi.mock("@/lib/apify/youtube/handleYoutubeProfileScraperResults", () => ({ + handleYoutubeProfileScraperResults: vi.fn(), +})); +vi.mock("@/lib/apify/threads/handleThreadsProfileScraperResults", () => ({ + handleThreadsProfileScraperResults: vi.fn(), +})); +vi.mock("@/lib/apify/facebook/handleFacebookProfileScraperResults", () => ({ + handleFacebookProfileScraperResults: vi.fn(), +})); describe("getApifyResultHandler", () => { it("maps the Instagram actor ids to their result handlers", () => { @@ -22,6 +42,13 @@ describe("getApifyResultHandler", () => { it("maps the harvestapi LinkedIn actor id to its results handler", () => { expect(getApifyResultHandler("LpVuK3Zozwuipa5bp")).toBe(handleLinkedinProfileScraperResults); }); + it("maps every platform's resolved actor id to its results handler", () => { + expect(getApifyResultHandler("GdWCkxBtKWOsKjdch")).toBe(handleTiktokProfileScraperResults); + expect(getApifyResultHandler("nfp1fpt5gUlBwPcor")).toBe(handleTwitterProfileScraperResults); + expect(getApifyResultHandler("h7sDV53CddomktSi5")).toBe(handleYoutubeProfileScraperResults); + expect(getApifyResultHandler("kJdK90pa2hhYYrCK5")).toBe(handleThreadsProfileScraperResults); + expect(getApifyResultHandler("4Hv5RhChiaDk6iwad")).toBe(handleFacebookProfileScraperResults); + }); it("returns undefined for an unregistered actor id", () => { expect(getApifyResultHandler("unknown_actor")).toBeUndefined(); }); diff --git a/lib/apify/facebook/__tests__/handleFacebookProfileScraperResults.test.ts b/lib/apify/facebook/__tests__/handleFacebookProfileScraperResults.test.ts new file mode 100644 index 000000000..e18cc68a5 --- /dev/null +++ b/lib/apify/facebook/__tests__/handleFacebookProfileScraperResults.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { handleFacebookProfileScraperResults } from "@/lib/apify/facebook/handleFacebookProfileScraperResults"; +const listItems = vi.fn(); +vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn(() => ({ listItems })) } })); +const upsertSocials = vi.fn(); +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ + upsertSocials: (...a: unknown[]) => upsertSocials(...a), +})); + +const payload = { + eventData: { actorId: "4Hv5RhChiaDk6iwad" }, + resource: { defaultDatasetId: "ds-1" }, +} as never; +// Trimmed from real run ICZxdJBrtP1jkPITS (2026-07-01). +const REAL_ITEM = { + pageUrl: "https://www.facebook.com/facebook", + facebookUrl: "https://www.facebook.com/facebook", + pageName: "facebook", + title: "Facebook", + profilePictureUrl: "https://scontent.cdn/pic.jpg", + followers: 154000000, +}; +beforeEach(() => { + vi.clearAllMocks(); + upsertSocials.mockResolvedValue([]); +}); + +describe("handleFacebookProfileScraperResults", () => { + it("upserts the page from a real item (keyed on profile_url)", async () => { + listItems.mockResolvedValue({ items: [REAL_ITEM] }); + await handleFacebookProfileScraperResults(payload); + expect(upsertSocials).toHaveBeenCalledWith([ + { + profile_url: "facebook.com/facebook", + username: "facebook", + avatar: "https://scontent.cdn/pic.jpg", + followerCount: 154000000, + }, + ]); + }); + it("no-ops on an empty dataset", async () => { + listItems.mockResolvedValue({ items: [] }); + expect(await handleFacebookProfileScraperResults(payload)).toEqual({ social: null }); + expect(upsertSocials).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/facebook/handleFacebookProfileScraperResults.ts b/lib/apify/facebook/handleFacebookProfileScraperResults.ts new file mode 100644 index 000000000..c78b0db2d --- /dev/null +++ b/lib/apify/facebook/handleFacebookProfileScraperResults.ts @@ -0,0 +1,30 @@ +import apifyClient from "@/lib/apify/client"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; +import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; + +/** Page item from apify~facebook-pages-scraper (real shape, run ICZxdJBrtP1jkPITS). */ +type FacebookPageItem = { + pageUrl?: string; + facebookUrl?: string; + pageName?: string; + profilePictureUrl?: string; + followers?: number; +}; + +/** Persists a Facebook page scrape back to `socials` (upsert on `profile_url`). */ +export async function handleFacebookProfileScraperResults(parsed: ApifyWebhookPayload) { + const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); + const first = items[0] as FacebookPageItem | undefined; + const url = first?.pageUrl ?? first?.facebookUrl; + if (!url) return { social: null }; + + const social = { + profile_url: normalizeProfileUrl(url), + username: first?.pageName, + avatar: first?.profilePictureUrl ?? null, + followerCount: first?.followers ?? null, + }; + await upsertSocials([social]); + return { social }; +} diff --git a/lib/apify/facebook/startFacebookProfileScraping.ts b/lib/apify/facebook/startFacebookProfileScraping.ts index 0a645dc46..3e7e498c9 100644 --- a/lib/apify/facebook/startFacebookProfileScraping.ts +++ b/lib/apify/facebook/startFacebookProfileScraping.ts @@ -1,5 +1,6 @@ import { ApifyRunInfo } from "@/lib/apify/types"; import apifyClient from "@/lib/apify/client"; +import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; const startFacebookProfileScraping = async (handle: string): Promise => { @@ -11,13 +12,16 @@ const startFacebookProfileScraping = async (handle: string): Promise PromiseProfileScraperResults` (mirror + * All supported platforms are wired: each start module attaches + * `getApifyWebhooks()` to its run and registers its resolved actor id here. Each needs a `handleProfileScraperResults` (mirror * the Instagram one) registered here under its resolved actor id. See * recoupable/chat#1833 ("persist non-Instagram scrape results"). */ @@ -21,6 +25,11 @@ const HANDLERS_BY_ACTOR_ID: Record = { dSCLg0C3YEZ83HzYX: handleInstagramProfileScraperResults, // instagram profile SbK00X0JYCPblD2wp: handleInstagramCommentsScraper, // instagram comments LpVuK3Zozwuipa5bp: handleLinkedinProfileScraperResults, // linkedin profile (harvestapi) + GdWCkxBtKWOsKjdch: handleTiktokProfileScraperResults, // tiktok (clockworks~tiktok-scraper) + nfp1fpt5gUlBwPcor: handleTwitterProfileScraperResults, // x/twitter (apidojo~twitter-scraper-lite) + h7sDV53CddomktSi5: handleYoutubeProfileScraperResults, // youtube (streamers~youtube-scraper) + kJdK90pa2hhYYrCK5: handleThreadsProfileScraperResults, // threads (apify~threads-profile-api-scraper) + "4Hv5RhChiaDk6iwad": handleFacebookProfileScraperResults, // facebook (apify~facebook-pages-scraper) }; /** diff --git a/lib/apify/threads/__tests__/handleThreadsProfileScraperResults.test.ts b/lib/apify/threads/__tests__/handleThreadsProfileScraperResults.test.ts new file mode 100644 index 000000000..cfd16f31b --- /dev/null +++ b/lib/apify/threads/__tests__/handleThreadsProfileScraperResults.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { handleThreadsProfileScraperResults } from "@/lib/apify/threads/handleThreadsProfileScraperResults"; +const listItems = vi.fn(); +vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn(() => ({ listItems })) } })); +const upsertSocials = vi.fn(); +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ + upsertSocials: (...a: unknown[]) => upsertSocials(...a), +})); + +const payload = { + eventData: { actorId: "kJdK90pa2hhYYrCK5" }, + resource: { defaultDatasetId: "ds-1" }, +} as never; +// Trimmed from real run 9iiG1sDkpaeWPCWHl (2026-07-01). +const REAL_ITEM = { + username: "zuck", + url: "https://www.threads.net/@zuck", + profile_pic_url: "https://instagram.cdn/pic.jpg", + biography: "Mostly superintelligence and MMA takes", + follower_count: 5642746, +}; +beforeEach(() => { + vi.clearAllMocks(); + upsertSocials.mockResolvedValue([]); +}); + +describe("handleThreadsProfileScraperResults", () => { + it("upserts the profile from a real item (keyed on profile_url)", async () => { + listItems.mockResolvedValue({ items: [REAL_ITEM] }); + await handleThreadsProfileScraperResults(payload); + expect(upsertSocials).toHaveBeenCalledWith([ + { + profile_url: "threads.net/@zuck", + username: "zuck", + avatar: "https://instagram.cdn/pic.jpg", + bio: "Mostly superintelligence and MMA takes", + followerCount: 5642746, + }, + ]); + }); + it("no-ops on an empty dataset", async () => { + listItems.mockResolvedValue({ items: [] }); + expect(await handleThreadsProfileScraperResults(payload)).toEqual({ social: null }); + expect(upsertSocials).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/threads/handleThreadsProfileScraperResults.ts b/lib/apify/threads/handleThreadsProfileScraperResults.ts new file mode 100644 index 000000000..8021ea2e2 --- /dev/null +++ b/lib/apify/threads/handleThreadsProfileScraperResults.ts @@ -0,0 +1,30 @@ +import apifyClient from "@/lib/apify/client"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; +import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; + +/** Profile item from apify~threads-profile-api-scraper (real shape, run 9iiG1sDkpaeWPCWHl). */ +type ThreadsProfileItem = { + username?: string; + url?: string; + profile_pic_url?: string; + biography?: string; + follower_count?: number; +}; + +/** Persists a Threads profile scrape back to `socials` (upsert on `profile_url`). */ +export async function handleThreadsProfileScraperResults(parsed: ApifyWebhookPayload) { + const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); + const first = items[0] as ThreadsProfileItem | undefined; + if (!first?.url) return { social: null }; + + const social = { + profile_url: normalizeProfileUrl(first.url), + username: first.username, + avatar: first.profile_pic_url ?? null, + bio: first.biography || null, + followerCount: first.follower_count ?? null, + }; + await upsertSocials([social]); + return { social }; +} diff --git a/lib/apify/threads/startThreadsProfileScraping.ts b/lib/apify/threads/startThreadsProfileScraping.ts index eeb280bca..d150ce948 100644 --- a/lib/apify/threads/startThreadsProfileScraping.ts +++ b/lib/apify/threads/startThreadsProfileScraping.ts @@ -1,5 +1,6 @@ import { ApifyRunInfo } from "@/lib/apify/types"; import apifyClient from "@/lib/apify/client"; +import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; const startThreadsProfileScraping = async (handle: string): Promise => { @@ -9,9 +10,12 @@ const startThreadsProfileScraping = async (handle: string): Promise ({ default: { dataset: vi.fn(() => ({ listItems })) } })); +const upsertSocials = vi.fn(); +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ + upsertSocials: (...a: unknown[]) => upsertSocials(...a), +})); + +const payload = { + eventData: { actorId: "GdWCkxBtKWOsKjdch" }, + resource: { defaultDatasetId: "ds-1" }, +} as never; +// Trimmed from real run G4YRI0eUI0d5IidDN (2026-07-01). +const REAL_ITEM = { + text: "In welcher Stadt tanzen wir?", + authorMeta: { + name: "apache_207", + profileUrl: "https://www.tiktok.com/@apache_207", + avatar: "https://p16-common-sign.tiktokcdn-us.com/avatar.jpeg", + signature: "", + fans: 917500, + following: 0, + }, +}; +beforeEach(() => { + vi.clearAllMocks(); + upsertSocials.mockResolvedValue([]); +}); + +describe("handleTiktokProfileScraperResults", () => { + it("upserts author profile stats from a real post item (keyed on profile_url)", async () => { + listItems.mockResolvedValue({ items: [REAL_ITEM] }); + await handleTiktokProfileScraperResults(payload); + expect(upsertSocials).toHaveBeenCalledWith([ + { + profile_url: "tiktok.com/@apache_207", + username: "apache_207", + avatar: "https://p16-common-sign.tiktokcdn-us.com/avatar.jpeg", + bio: null, + followerCount: 917500, + followingCount: 0, + }, + ]); + }); + it("no-ops when the dataset is empty or has no authorMeta", async () => { + listItems.mockResolvedValue({ items: [] }); + expect(await handleTiktokProfileScraperResults(payload)).toEqual({ social: null }); + expect(upsertSocials).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/tiktok/handleTiktokProfileScraperResults.ts b/lib/apify/tiktok/handleTiktokProfileScraperResults.ts new file mode 100644 index 000000000..3ff3ac248 --- /dev/null +++ b/lib/apify/tiktok/handleTiktokProfileScraperResults.ts @@ -0,0 +1,38 @@ +import apifyClient from "@/lib/apify/client"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; +import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; + +/** Post item from clockworks~tiktok-scraper (real shape, run G4YRI0eUI0d5IidDN). */ +type TiktokPostItem = { + authorMeta?: { + name?: string; + profileUrl?: string; + avatar?: string; + signature?: string; + fans?: number; + following?: number; + }; +}; + +/** + * Persists a TikTok profile scrape back to `socials` (upsert on `profile_url`). + * The actor returns post items; the author's profile stats ride on + * `authorMeta` of any item. + */ +export async function handleTiktokProfileScraperResults(parsed: ApifyWebhookPayload) { + const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); + const author = (items[0] as TiktokPostItem | undefined)?.authorMeta; + if (!author?.profileUrl) return { social: null }; + + const social = { + profile_url: normalizeProfileUrl(author.profileUrl), + username: author.name, + avatar: author.avatar ?? null, + bio: author.signature || null, + followerCount: author.fans ?? null, + followingCount: author.following ?? null, + }; + await upsertSocials([social]); + return { social }; +} diff --git a/lib/apify/tiktok/startTiktokProfileScraping.ts b/lib/apify/tiktok/startTiktokProfileScraping.ts index 3c28520c2..8684246ac 100644 --- a/lib/apify/tiktok/startTiktokProfileScraping.ts +++ b/lib/apify/tiktok/startTiktokProfileScraping.ts @@ -1,4 +1,5 @@ import apifyClient from "@/lib/apify/client"; +import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; import { ApifyRunInfo } from "@/lib/apify/types"; @@ -19,7 +20,9 @@ const startTiktokProfileScraping = async ( }; try { - const run = await apifyClient.actor("clockworks~tiktok-scraper").start(input); + const run = await apifyClient + .actor("clockworks~tiktok-scraper") + .start(input, { webhooks: getApifyWebhooks() }); if (!run?.id || !run?.defaultDatasetId) { console.error("Failed to start TikTok profile scraping for handle:", handle); diff --git a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts new file mode 100644 index 000000000..fe936d72e --- /dev/null +++ b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts @@ -0,0 +1,53 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { handleTwitterProfileScraperResults } from "@/lib/apify/twitter/handleTwitterProfileScraperResults"; +const listItems = vi.fn(); +vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn(() => ({ listItems })) } })); +const upsertSocials = vi.fn(); +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ + upsertSocials: (...a: unknown[]) => upsertSocials(...a), +})); + +const payload = { + eventData: { actorId: "nfp1fpt5gUlBwPcor" }, + resource: { defaultDatasetId: "ds-1" }, +} as never; +// Trimmed from real run ALVMZYXkh3WHgeGfT (2026-07-01). +const REAL_ITEM = { + type: "tweet", + author: { + userName: "TheASF", + url: "https://x.com/TheASF", + profilePicture: "https://pbs.twimg.com/profile_images/pic.jpg", + description: "The global home for open source software", + location: "Worldwide", + followers: 66208, + following: 210, + }, +}; +beforeEach(() => { + vi.clearAllMocks(); + upsertSocials.mockResolvedValue([]); +}); + +describe("handleTwitterProfileScraperResults", () => { + it("upserts author stats with a LOWERCASED profile_url (matches stored rows; X handles are case-insensitive)", async () => { + listItems.mockResolvedValue({ items: [REAL_ITEM] }); + await handleTwitterProfileScraperResults(payload); + expect(upsertSocials).toHaveBeenCalledWith([ + { + profile_url: "x.com/theasf", + username: "TheASF", + avatar: "https://pbs.twimg.com/profile_images/pic.jpg", + bio: "The global home for open source software", + followerCount: 66208, + followingCount: 210, + region: "Worldwide", + }, + ]); + }); + it("no-ops on an empty dataset", async () => { + listItems.mockResolvedValue({ items: [] }); + expect(await handleTwitterProfileScraperResults(payload)).toEqual({ social: null }); + expect(upsertSocials).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/twitter/handleTwitterProfileScraperResults.ts b/lib/apify/twitter/handleTwitterProfileScraperResults.ts new file mode 100644 index 000000000..50a51ee18 --- /dev/null +++ b/lib/apify/twitter/handleTwitterProfileScraperResults.ts @@ -0,0 +1,42 @@ +import apifyClient from "@/lib/apify/client"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; +import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; + +/** Tweet item from apidojo~twitter-scraper-lite (real shape, run ALVMZYXkh3WHgeGfT). */ +type TweetItem = { + author?: { + userName?: string; + url?: string; + profilePicture?: string; + description?: string; + location?: string; + followers?: number; + following?: number; + }; +}; + +/** + * Persists an X/Twitter profile scrape back to `socials`. The actor returns + * tweet items; profile stats ride on `author`. The URL is lowercased — + * X handles are case-insensitive and the actor echoes display casing + * (`x.com/TheASF`) while stored rows are lowercase (`x.com/theasf`); + * without this the upsert would create a duplicate row. + */ +export async function handleTwitterProfileScraperResults(parsed: ApifyWebhookPayload) { + const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); + const author = (items[0] as TweetItem | undefined)?.author; + if (!author?.url) return { social: null }; + + const social = { + profile_url: normalizeProfileUrl(author.url).toLowerCase(), + username: author.userName, + avatar: author.profilePicture ?? null, + bio: author.description || null, + followerCount: author.followers ?? null, + followingCount: author.following ?? null, + region: author.location || null, + }; + await upsertSocials([social]); + return { social }; +} diff --git a/lib/apify/twitter/startTwitterProfileScraping.ts b/lib/apify/twitter/startTwitterProfileScraping.ts index 1df7f31ae..52f288038 100644 --- a/lib/apify/twitter/startTwitterProfileScraping.ts +++ b/lib/apify/twitter/startTwitterProfileScraping.ts @@ -1,4 +1,5 @@ import apifyClient from "@/lib/apify/client"; +import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; import { ApifyRunInfo } from "@/lib/apify/types"; const startTwitterProfileScraping = async (handle: string): Promise => { @@ -14,7 +15,9 @@ const startTwitterProfileScraping = async (handle: string): Promise ({ default: { dataset: vi.fn(() => ({ listItems })) } })); +const upsertSocials = vi.fn(); +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ + upsertSocials: (...a: unknown[]) => upsertSocials(...a), +})); + +const payload = { + eventData: { actorId: "h7sDV53CddomktSi5" }, + resource: { defaultDatasetId: "ds-1" }, +} as never; +// Trimmed from real run H0ZrIAsJJYXLpCAp7 (2026-07-01). +const REAL_ITEM = { + inputChannelUrl: "https://www.youtube.com/@blackveilbrides", + channelUrl: "https://www.youtube.com/channel/UCDdKEz8e7m5g", + channelUsername: "blackveilbrides", + channelAvatarUrl: "https://yt3.googleusercontent.com/avatar", + channelDescription: "The official artist channel for Black Veil Brides", + channelLocation: "United States", + aboutChannelInfo: { numberOfSubscribers: 2720000, channelTotalViews: 977316227 }, +}; +beforeEach(() => { + vi.clearAllMocks(); + upsertSocials.mockResolvedValue([]); +}); + +describe("handleYoutubeProfileScraperResults", () => { + it("keys on inputChannelUrl (round-trips the stored @handle row, NOT the /channel/UC… url)", async () => { + listItems.mockResolvedValue({ items: [REAL_ITEM] }); + await handleYoutubeProfileScraperResults(payload); + expect(upsertSocials).toHaveBeenCalledWith([ + { + profile_url: "youtube.com/@blackveilbrides", + username: "blackveilbrides", + avatar: "https://yt3.googleusercontent.com/avatar", + bio: "The official artist channel for Black Veil Brides", + followerCount: 2720000, + region: "United States", + }, + ]); + }); + it("no-ops on an empty dataset", async () => { + listItems.mockResolvedValue({ items: [] }); + expect(await handleYoutubeProfileScraperResults(payload)).toEqual({ social: null }); + expect(upsertSocials).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/youtube/handleYoutubeProfileScraperResults.ts b/lib/apify/youtube/handleYoutubeProfileScraperResults.ts new file mode 100644 index 000000000..495e5406d --- /dev/null +++ b/lib/apify/youtube/handleYoutubeProfileScraperResults.ts @@ -0,0 +1,37 @@ +import apifyClient from "@/lib/apify/client"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; +import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; + +/** Video item from streamers~youtube-scraper (real shape, run H0ZrIAsJJYXLpCAp7). */ +type YoutubeVideoItem = { + inputChannelUrl?: string; + channelUsername?: string; + channelAvatarUrl?: string; + channelDescription?: string; + channelLocation?: string; + aboutChannelInfo?: { numberOfSubscribers?: number }; +}; + +/** + * Persists a YouTube channel scrape back to `socials`. Keyed on + * `inputChannelUrl` — the exact URL this service passed to the actor — + * because the actor's own `channelUrl` is the `/channel/UC…` form, which + * would never match the stored `@handle` row (and would upsert a duplicate). + */ +export async function handleYoutubeProfileScraperResults(parsed: ApifyWebhookPayload) { + const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); + const first = items[0] as YoutubeVideoItem | undefined; + if (!first?.inputChannelUrl) return { social: null }; + + const social = { + profile_url: normalizeProfileUrl(first.inputChannelUrl), + username: first.channelUsername, + avatar: first.channelAvatarUrl ?? null, + bio: first.channelDescription || null, + followerCount: first.aboutChannelInfo?.numberOfSubscribers ?? null, + region: first.channelLocation || null, + }; + await upsertSocials([social]); + return { social }; +} diff --git a/lib/apify/youtube/startYoutubeProfileScraping.ts b/lib/apify/youtube/startYoutubeProfileScraping.ts index f829de299..f5a7324b5 100644 --- a/lib/apify/youtube/startYoutubeProfileScraping.ts +++ b/lib/apify/youtube/startYoutubeProfileScraping.ts @@ -1,4 +1,5 @@ import apifyClient from "@/lib/apify/client"; +import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; import { ApifyRunInfo } from "@/lib/apify/types"; const DEFAULT_INPUT = { @@ -37,7 +38,9 @@ const startYoutubeProfileScraping = async (handle: string): Promise