From 360268b97baacbe56934fe3268b63c24672da905 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Mon, 6 Jul 2026 16:55:39 -0500 Subject: [PATCH 01/12] =?UTF-8?q?feat(apify):=20notify=20only=20on=20genui?= =?UTF-8?q?nely=20new=20posts=20=E2=80=94=20diff=20against=20stored=20post?= =?UTF-8?q?s=20first?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Instagram scrape alert fired whenever a scrape returned posts, with no comparison against posts already stored — every scrape re-announced the profile's recent feed as new (observed: 6 of 7 alerts in one day announced posts up to 10 days old). New reusable filterNewPostUrls diffs candidate URLs against posts BEFORE upsert; the alert is gated on a non-empty result. Persistence unchanged (recoupable/chat#1855, PR #3 of 6). Co-Authored-By: Claude Fable 5 --- ...ndleInstagramProfileScraperResults.test.ts | 38 ++++++++++++++++++- .../handleInstagramProfileScraperResults.ts | 16 ++++++-- .../__tests__/filterNewPostUrls.test.ts | 25 ++++++++++++ lib/socials/filterNewPostUrls.ts | 21 ++++++++++ 4 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 lib/socials/__tests__/filterNewPostUrls.test.ts create mode 100644 lib/socials/filterNewPostUrls.ts diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts index 9092a197b..bab22c438 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts @@ -12,6 +12,7 @@ import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccou import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccountArtistIds"; import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails"; import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; +import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn() } })); @@ -26,6 +27,7 @@ vi.mock("../handleInstagramProfileFollowUpRuns", () => ({ handleInstagramProfileFollowUpRuns: vi.fn(), })); vi.mock("@/lib/apify/sendApifyWebhookEmail", () => ({ sendApifyWebhookEmail: vi.fn() })); +vi.mock("@/lib/socials/filterNewPostUrls", () => ({ filterNewPostUrls: vi.fn() })); vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ upsertSocials: vi.fn() })); vi.mock("@/lib/supabase/socials/selectSocials", () => ({ selectSocials: vi.fn(), @@ -51,7 +53,11 @@ const payload = { } as never; describe("handleInstagramProfileScraperResults", () => { - beforeEach(() => vi.clearAllMocks()); + beforeEach(() => { + vi.clearAllMocks(); + // default: everything scraped counts as new (per-test overrides below) + vi.mocked(filterNewPostUrls).mockImplementation(async urls => urls); + }); it("short-circuits when the dataset has no latest posts", async () => { mockDataset([{ username: "alice" }]); @@ -92,4 +98,34 @@ describe("handleInstagramProfileScraperResults", () => { expect(result.social).toEqual({ id: "s1" }); expect(result.posts).toEqual(posts); }); + + it("skips the alert email when no scraped post is genuinely new (chat#1855)", async () => { + mockDataset([ + { + latestPosts: [{ url: "u1", timestamp: "t" }], + username: "alice", + url: "instagram.com/alice", + profilePicUrl: "https://a", + fullName: "Alice", + }, + ]); + vi.mocked(filterNewPostUrls).mockResolvedValue([]); // all already stored + vi.mocked(upsertPosts).mockResolvedValue({ data: null, error: null } as never); + vi.mocked(getPosts).mockResolvedValue([{ id: "p1", post_url: "u1" }] as never); + vi.mocked(uploadLinkToArweave).mockResolvedValue(null); + vi.mocked(upsertSocials).mockResolvedValue([] as never); + vi.mocked(selectSocials).mockResolvedValue([{ id: "s1" }] as never); + vi.mocked(selectAccountSocials).mockResolvedValue([{ account_id: "a1" }] as never); + vi.mocked(getAccountArtistIds).mockResolvedValue([{ account_id: "a1" }] as never); + vi.mocked(selectAccountEmails).mockResolvedValue([{ email: "x@y.com" }] as never); + + const result = await handleInstagramProfileScraperResults(payload); + + expect(sendApifyWebhookEmail).not.toHaveBeenCalled(); + // persistence is unaffected — only the notification is gated + expect(upsertPosts).toHaveBeenCalledOnce(); + expect(upsertSocialPosts).toHaveBeenCalledOnce(); + expect(handleInstagramProfileFollowUpRuns).toHaveBeenCalledOnce(); + expect(result.social).toEqual({ id: "s1" }); + }); }); diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index ff8a971c5..4774c17f0 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -14,6 +14,7 @@ import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; import { getFetchableUrl } from "@/lib/arweave/getFetchableUrl"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; +import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; import type { TablesInsert } from "@/types/database.types"; /** @@ -40,6 +41,9 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP post.url ? [{ post_url: post.url, updated_at: post.timestamp }] : [], ); if (postRows.length === 0) return { posts: [], social: null }; + // Diff BEFORE upserting — afterwards every scraped post exists and nothing + // is distinguishable as new (chat#1855). + const newPostUrls = await filterNewPostUrls(postRows.map(p => p.post_url)); await upsertPosts(postRows); const posts = await getPosts({ postUrls: postRows.map(p => p.post_url) }); @@ -96,10 +100,14 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP // mail outage doesn't block comment scraping and vice versa. let sentEmails = null; try { - sentEmails = await sendApifyWebhookEmail( - firstResult, - accountEmails.map(e => e.email).filter(Boolean), - ); + // Only notify when the scrape actually found posts new to the platform — + // otherwise every scrape re-announces the profile's recent feed. + if (newPostUrls.length > 0) { + sentEmails = await sendApifyWebhookEmail( + firstResult, + accountEmails.map(e => e.email).filter(Boolean), + ); + } } catch (error) { console.error("[WARN] webhook email failed:", error); } diff --git a/lib/socials/__tests__/filterNewPostUrls.test.ts b/lib/socials/__tests__/filterNewPostUrls.test.ts new file mode 100644 index 000000000..0f4be5305 --- /dev/null +++ b/lib/socials/__tests__/filterNewPostUrls.test.ts @@ -0,0 +1,25 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; + +vi.mock("@/lib/supabase/posts/getPosts", () => ({ getPosts: vi.fn() })); + +beforeEach(() => vi.clearAllMocks()); + +describe("filterNewPostUrls", () => { + it("returns only URLs not already stored in posts", async () => { + vi.mocked(getPosts).mockResolvedValue([{ post_url: "u1" }, { post_url: "u3" }] as never); + expect(await filterNewPostUrls(["u1", "u2", "u3", "u4"])).toEqual(["u2", "u4"]); + expect(getPosts).toHaveBeenCalledWith({ postUrls: ["u1", "u2", "u3", "u4"] }); + }); + + it("returns every URL when none are stored", async () => { + vi.mocked(getPosts).mockResolvedValue([] as never); + expect(await filterNewPostUrls(["u1"])).toEqual(["u1"]); + }); + + it("returns [] for empty input without querying", async () => { + expect(await filterNewPostUrls([])).toEqual([]); + expect(getPosts).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/socials/filterNewPostUrls.ts b/lib/socials/filterNewPostUrls.ts new file mode 100644 index 000000000..c71d3e670 --- /dev/null +++ b/lib/socials/filterNewPostUrls.ts @@ -0,0 +1,21 @@ +import { getPosts } from "@/lib/supabase/posts/getPosts"; + +/** + * Returns the subset of post URLs that are genuinely new to the platform — + * i.e. not already present in `posts`. Must run BEFORE the scrape results are + * upserted (afterwards everything exists and nothing is "new"). + * + * Used to gate scrape-alert notifications: a scrape re-reads a profile's + * whole recent feed, so without this diff every scrape re-announces old + * posts as new (recoupable/chat#1855). + * + * @param postUrls - Candidate post URLs from a scrape result. + * @returns URLs with no existing `posts` row. + */ +export async function filterNewPostUrls(postUrls: string[]): Promise { + if (!postUrls.length) return []; + + const existing = await getPosts({ postUrls }); + const known = new Set((existing ?? []).map(post => post.post_url)); + return postUrls.filter(url => !known.has(url)); +} From 9a991be6295bd25e8da98f613dda573271ae269b Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Mon, 6 Jul 2026 17:05:48 -0500 Subject: [PATCH 02/12] feat(apify): one consolidated new-posts digest per scrape batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A roster scrape starts one Apify run per platform, each completing independently — extending per-platform alerts would mean 4+ emails per scrape. This registers every run under a batch_id at scrape start (apify_scraper_runs, columns from recoupable/database#41), records each webhook completion with its genuinely-new post URLs, and when the batch's last run completes sends ONE digest (per-platform sections, BCC-only) via the new digest module. Platforms with nothing new are omitted; a batch with nothing new sends nothing. Instagram's solo alert is suppressed for batch runs; legacy/non-batch runs keep today's behavior (recoupable/chat#1855, PR #4 of 6). Co-Authored-By: Claude Fable 5 --- .../__tests__/apifyWebhookHandler.test.ts | 41 +++++++++++ lib/apify/apifyWebhookHandler.ts | 18 +++++ .../__tests__/maybeSendScrapeDigest.test.ts | 73 +++++++++++++++++++ lib/apify/digest/getScrapeDigestRecipients.ts | 28 +++++++ lib/apify/digest/maybeSendScrapeDigest.ts | 27 +++++++ lib/apify/digest/sendScrapeDigestEmail.ts | 38 ++++++++++ ...ndleInstagramProfileScraperResults.test.ts | 36 +++++++++ .../handleInstagramProfileScraperResults.ts | 10 ++- lib/apify/scrapeProfileUrlBatch.ts | 18 +++-- .../handleTiktokProfileScraperResults.test.ts | 3 + .../handleTiktokProfileScraperResults.ts | 5 +- ...handleTwitterProfileScraperResults.test.ts | 3 + .../handleTwitterProfileScraperResults.ts | 5 +- lib/apify/validateApifyWebhookRequest.ts | 4 + .../postArtistSocialsScrapeHandler.test.ts | 9 ++- lib/artist/postArtistSocialsScrapeHandler.ts | 32 +++++++- .../completeApifyScraperRun.ts | 24 ++++++ .../insertApifyScraperRuns.ts | 19 +++++ .../selectApifyScraperRun.ts | 16 ++++ .../selectApifyScraperRunsByBatch.ts | 17 +++++ lib/supabase/apify_scraper_runs/types.ts | 12 +++ 21 files changed, 421 insertions(+), 17 deletions(-) create mode 100644 lib/apify/digest/__tests__/maybeSendScrapeDigest.test.ts create mode 100644 lib/apify/digest/getScrapeDigestRecipients.ts create mode 100644 lib/apify/digest/maybeSendScrapeDigest.ts create mode 100644 lib/apify/digest/sendScrapeDigestEmail.ts create mode 100644 lib/supabase/apify_scraper_runs/completeApifyScraperRun.ts create mode 100644 lib/supabase/apify_scraper_runs/insertApifyScraperRuns.ts create mode 100644 lib/supabase/apify_scraper_runs/selectApifyScraperRun.ts create mode 100644 lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch.ts create mode 100644 lib/supabase/apify_scraper_runs/types.ts diff --git a/lib/apify/__tests__/apifyWebhookHandler.test.ts b/lib/apify/__tests__/apifyWebhookHandler.test.ts index 98f7aa7d2..3cebad3ef 100644 --- a/lib/apify/__tests__/apifyWebhookHandler.test.ts +++ b/lib/apify/__tests__/apifyWebhookHandler.test.ts @@ -1,4 +1,6 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; +import { completeApifyScraperRun } from "@/lib/supabase/apify_scraper_runs/completeApifyScraperRun"; +import { maybeSendScrapeDigest } from "@/lib/apify/digest/maybeSendScrapeDigest"; import { NextRequest } from "next/server"; import { apifyWebhookHandler } from "../apifyWebhookHandler"; import { handleInstagramProfileScraperResults } from "../instagram/handleInstagramProfileScraperResults"; @@ -44,6 +46,13 @@ const baseBody = { resource: { defaultDatasetId: "ds_1" }, }; +vi.mock("@/lib/supabase/apify_scraper_runs/completeApifyScraperRun", () => ({ + completeApifyScraperRun: vi.fn(async () => null), +})); +vi.mock("@/lib/apify/digest/maybeSendScrapeDigest", () => ({ + maybeSendScrapeDigest: vi.fn(async () => null), +})); + describe("apifyWebhookHandler", () => { beforeEach(() => vi.clearAllMocks()); @@ -60,6 +69,38 @@ describe("apifyWebhookHandler", () => { expect(handleInstagramCommentsScraper).not.toHaveBeenCalled(); }); + it("records batch completion + triggers the digest check when the payload carries a run id (chat#1855)", async () => { + vi.mocked(handleInstagramProfileScraperResults).mockResolvedValue({ + posts: [], + newPostUrls: ["https://instagram.com/p/new1"], + } as never); + vi.mocked(completeApifyScraperRun).mockResolvedValue({ + run_id: "run-9", + batch_id: "batch-7", + } as never); + + const res = await apifyWebhookHandler( + makeRequest({ + ...baseBody, + eventData: { actorId: "dSCLg0C3YEZ83HzYX" }, + resource: { ...baseBody.resource, id: "run-9" }, + }), + ); + + expect(res.status).toBe(200); + expect(completeApifyScraperRun).toHaveBeenCalledWith("run-9", ["https://instagram.com/p/new1"]); + expect(maybeSendScrapeDigest).toHaveBeenCalledWith("batch-7"); + }); + + it("skips digest bookkeeping when the payload has no run id", async () => { + vi.mocked(handleInstagramProfileScraperResults).mockResolvedValue({ posts: [] } as never); + await apifyWebhookHandler( + makeRequest({ ...baseBody, eventData: { actorId: "dSCLg0C3YEZ83HzYX" } }), + ); + expect(completeApifyScraperRun).not.toHaveBeenCalled(); + expect(maybeSendScrapeDigest).not.toHaveBeenCalled(); + }); + it("dispatches comments scraper for the IG comments actor", async () => { vi.mocked(handleInstagramCommentsScraper).mockResolvedValue({ comments: [], diff --git a/lib/apify/apifyWebhookHandler.ts b/lib/apify/apifyWebhookHandler.ts index fdda46e5a..540950e65 100644 --- a/lib/apify/apifyWebhookHandler.ts +++ b/lib/apify/apifyWebhookHandler.ts @@ -1,6 +1,8 @@ import { NextRequest, NextResponse } from "next/server"; import { validateApifyWebhookRequest } from "@/lib/apify/validateApifyWebhookRequest"; import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler"; +import { completeApifyScraperRun } from "@/lib/supabase/apify_scraper_runs/completeApifyScraperRun"; +import { maybeSendScrapeDigest } from "@/lib/apify/digest/maybeSendScrapeDigest"; /** * Handler for `POST /api/apify`. Always responds 200 so Apify does not @@ -27,6 +29,22 @@ export async function apifyWebhookHandler(request: NextRequest): Promise ({ + selectApifyScraperRunsByBatch: vi.fn(), +})); +vi.mock("@/lib/apify/digest/getScrapeDigestRecipients", () => ({ + getScrapeDigestRecipients: vi.fn(), +})); +vi.mock("@/lib/apify/digest/sendScrapeDigestEmail", () => ({ + sendScrapeDigestEmail: vi.fn(), +})); + +const run = (over: Record) => ({ + run_id: "r1", + batch_id: "b1", + account_id: "acct-1", + social_id: "s1", + platform: "instagram", + completed_at: "2026-07-07T00:00:00Z", + new_post_urls: [], + ...over, +}); + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getScrapeDigestRecipients).mockResolvedValue(["owner@example.com"]); + vi.mocked(sendScrapeDigestEmail).mockResolvedValue({ id: "email-1" } as never); +}); + +describe("maybeSendScrapeDigest", () => { + it("does nothing while sibling runs are still incomplete", async () => { + vi.mocked(selectApifyScraperRunsByBatch).mockResolvedValue([ + run({}), + run({ run_id: "r2", platform: "tiktok", completed_at: null }), + ] as never); + expect(await maybeSendScrapeDigest("b1")).toBeNull(); + expect(sendScrapeDigestEmail).not.toHaveBeenCalled(); + }); + + it("sends ONE digest with per-platform new posts when the batch completes", async () => { + vi.mocked(selectApifyScraperRunsByBatch).mockResolvedValue([ + run({ new_post_urls: ["https://instagram.com/p/1"] }), + run({ run_id: "r2", platform: "tiktok", new_post_urls: ["https://tiktok.com/v/2"] }), + run({ run_id: "r3", platform: "x", new_post_urls: [] }), + ] as never); + await maybeSendScrapeDigest("b1"); + expect(sendScrapeDigestEmail).toHaveBeenCalledOnce(); + const arg = vi.mocked(sendScrapeDigestEmail).mock.calls[0][0]; + expect(arg.sections).toEqual([ + { platform: "instagram", postUrls: ["https://instagram.com/p/1"] }, + { platform: "tiktok", postUrls: ["https://tiktok.com/v/2"] }, + ]); // x omitted — nothing new + expect(arg.emails).toEqual(["owner@example.com"]); + }); + + it("sends nothing when the batch completes with zero new posts anywhere", async () => { + vi.mocked(selectApifyScraperRunsByBatch).mockResolvedValue([ + run({}), + run({ run_id: "r2", platform: "tiktok" }), + ] as never); + expect(await maybeSendScrapeDigest("b1")).toBeNull(); + expect(sendScrapeDigestEmail).not.toHaveBeenCalled(); + }); + + it("no-ops for a null batch id (legacy runs)", async () => { + expect(await maybeSendScrapeDigest(null)).toBeNull(); + expect(selectApifyScraperRunsByBatch).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/digest/getScrapeDigestRecipients.ts b/lib/apify/digest/getScrapeDigestRecipients.ts new file mode 100644 index 000000000..a3c31cd90 --- /dev/null +++ b/lib/apify/digest/getScrapeDigestRecipients.ts @@ -0,0 +1,28 @@ +import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccountSocials"; +import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccountArtistIds"; +import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails"; + +/** + * Resolves the digest recipients for a set of scraped socials: every owner + * of every artist watching any of them (same chain the per-platform alert + * used). Recipients span tenants — senders must BCC (chat#1855). + */ +export async function getScrapeDigestRecipients(socialIds: string[]): Promise { + const uniqueSocialIds = Array.from(new Set(socialIds.filter(Boolean))); + if (!uniqueSocialIds.length) return []; + + const accountSocials = ( + await Promise.all( + uniqueSocialIds.map(socialId => selectAccountSocials({ socialId, limit: 10000 })), + ) + ).flat(); + + const accountArtistIds = await getAccountArtistIds({ + artistIds: accountSocials.map(a => a.account_id), + }); + const uniqueAccountIds = Array.from( + new Set(accountArtistIds.map(a => a.account_id).filter((id): id is string => Boolean(id))), + ); + const accountEmails = await selectAccountEmails({ accountIds: uniqueAccountIds }); + return Array.from(new Set(accountEmails.map(e => e.email).filter(Boolean))); +} diff --git a/lib/apify/digest/maybeSendScrapeDigest.ts b/lib/apify/digest/maybeSendScrapeDigest.ts new file mode 100644 index 000000000..9247b9dd1 --- /dev/null +++ b/lib/apify/digest/maybeSendScrapeDigest.ts @@ -0,0 +1,27 @@ +import { selectApifyScraperRunsByBatch } from "@/lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch"; +import { getScrapeDigestRecipients } from "@/lib/apify/digest/getScrapeDigestRecipients"; +import { sendScrapeDigestEmail } from "@/lib/apify/digest/sendScrapeDigestEmail"; + +/** + * Batch-completion check for the one-digest-per-scrape design (chat#1855): + * called after each platform run's results are processed. Sends the single + * consolidated digest when (a) every sibling run in the batch has completed + * and (b) at least one platform found genuinely new posts. Platforms with + * nothing new are omitted; a batch with nothing new sends nothing. + */ +export async function maybeSendScrapeDigest(batchId: string | null | undefined) { + if (!batchId) return null; + + const runs = await selectApifyScraperRunsByBatch(batchId); + if (!runs.length || runs.some(r => !r.completed_at)) return null; + + const sections = runs + .filter(r => (r.new_post_urls?.length ?? 0) > 0) + .map(r => ({ platform: r.platform ?? "other", postUrls: r.new_post_urls ?? [] })); + if (!sections.length) return null; + + const emails = await getScrapeDigestRecipients( + runs.map(r => r.social_id).filter((id): id is string => Boolean(id)), + ); + return await sendScrapeDigestEmail({ emails, sections }); +} diff --git a/lib/apify/digest/sendScrapeDigestEmail.ts b/lib/apify/digest/sendScrapeDigestEmail.ts new file mode 100644 index 000000000..c75df6943 --- /dev/null +++ b/lib/apify/digest/sendScrapeDigestEmail.ts @@ -0,0 +1,38 @@ +import { sendEmailWithResend } from "@/lib/emails/sendEmail"; +import { RECOUP_FROM_EMAIL } from "@/lib/const"; + +export type ScrapeDigestSection = { platform: string; postUrls: string[] }; +export type ScrapeDigestInput = { + emails: string[]; + sections: ScrapeDigestSection[]; + artistName?: string | null; +}; + +/** + * Sends the consolidated new-posts digest: one email per scrape batch, + * one section per platform that found genuinely new posts. Deterministic + * body, direct links to each new post. Recipients span tenants — BCC only + * (chat#1855). + */ +export async function sendScrapeDigestEmail({ emails, sections, artistName }: ScrapeDigestInput) { + if (!emails.length || !sections.length) return null; + + const total = sections.reduce((n, s) => n + s.postUrls.length, 0); + const who = artistName || "your artist"; + const sectionHtml = sections + .map( + s => + `

${s.platform}

    ${s.postUrls + .map(u => `
  • ${u}
  • `) + .join("")}
`, + ) + .join(""); + + return await sendEmailWithResend({ + from: RECOUP_FROM_EMAIL, + to: [RECOUP_FROM_EMAIL], + bcc: emails, + subject: `${who}: ${total} new post${total === 1 ? "" : "s"} found across ${sections.length} platform${sections.length === 1 ? "" : "s"}`, + html: `

New posts found for ${who}:

${sectionHtml}`, + }); +} diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts index bab22c438..2a9850140 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts @@ -13,6 +13,7 @@ import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccoun import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails"; import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; +import { selectApifyScraperRun } from "@/lib/supabase/apify_scraper_runs/selectApifyScraperRun"; vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn() } })); @@ -28,6 +29,9 @@ vi.mock("../handleInstagramProfileFollowUpRuns", () => ({ })); vi.mock("@/lib/apify/sendApifyWebhookEmail", () => ({ sendApifyWebhookEmail: vi.fn() })); vi.mock("@/lib/socials/filterNewPostUrls", () => ({ filterNewPostUrls: vi.fn() })); +vi.mock("@/lib/supabase/apify_scraper_runs/selectApifyScraperRun", () => ({ + selectApifyScraperRun: vi.fn(async () => null), +})); vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ upsertSocials: vi.fn() })); vi.mock("@/lib/supabase/socials/selectSocials", () => ({ selectSocials: vi.fn(), @@ -128,4 +132,36 @@ describe("handleInstagramProfileScraperResults", () => { expect(handleInstagramProfileFollowUpRuns).toHaveBeenCalledOnce(); expect(result.social).toEqual({ id: "s1" }); }); + + it("suppresses the solo email for digest-batch runs (webhook layer sends ONE digest)", async () => { + mockDataset([ + { + latestPosts: [{ url: "u-new", timestamp: "t" }], + username: "alice", + url: "instagram.com/alice", + profilePicUrl: "https://a", + fullName: "Alice", + }, + ]); + vi.mocked(selectApifyScraperRun).mockResolvedValue({ + run_id: "run-1", + batch_id: "b1", + } as never); + vi.mocked(upsertPosts).mockResolvedValue({ data: null, error: null } as never); + vi.mocked(getPosts).mockResolvedValue([{ id: "p1", post_url: "u-new" }] as never); + vi.mocked(uploadLinkToArweave).mockResolvedValue(null); + vi.mocked(upsertSocials).mockResolvedValue([] as never); + vi.mocked(selectSocials).mockResolvedValue([{ id: "s1" }] as never); + vi.mocked(selectAccountSocials).mockResolvedValue([{ account_id: "a1" }] as never); + vi.mocked(getAccountArtistIds).mockResolvedValue([{ account_id: "a1" }] as never); + vi.mocked(selectAccountEmails).mockResolvedValue([{ email: "x@y.com" }] as never); + + const result = await handleInstagramProfileScraperResults({ + ...(payload as Record), + resource: { defaultDatasetId: "ds_1", id: "run-1" }, + } as never); + + expect(sendApifyWebhookEmail).not.toHaveBeenCalled(); // digest covers it + expect(result.newPostUrls).toEqual(["u-new"]); + }); }); diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index 4774c17f0..3244e8eb7 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -15,6 +15,7 @@ import { getFetchableUrl } from "@/lib/arweave/getFetchableUrl"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; +import { selectApifyScraperRun } from "@/lib/supabase/apify_scraper_runs/selectApifyScraperRun"; import type { TablesInsert } from "@/types/database.types"; /** @@ -98,11 +99,16 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP // Email + follow-up scrape are independent side effects; isolate so a // mail outage doesn't block comment scraping and vice versa. + // Digest-batch runs get ONE consolidated email from the webhook layer — + // suppress the per-platform solo email for them (chat#1855). Legacy runs + // (no batch registration) keep the immediate alert. + const registeredRun = parsed.resource.id ? await selectApifyScraperRun(parsed.resource.id) : null; + let sentEmails = null; try { // Only notify when the scrape actually found posts new to the platform — // otherwise every scrape re-announces the profile's recent feed. - if (newPostUrls.length > 0) { + if (newPostUrls.length > 0 && !registeredRun?.batch_id) { sentEmails = await sendApifyWebhookEmail( firstResult, accountEmails.map(e => e.email).filter(Boolean), @@ -118,5 +124,5 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP console.error("[WARN] follow-up scrape failed:", error); } - return { posts, social, accountSocials, accountEmails, sentEmails }; + return { posts, social, accountSocials, accountEmails, sentEmails, newPostUrls }; } diff --git a/lib/apify/scrapeProfileUrlBatch.ts b/lib/apify/scrapeProfileUrlBatch.ts index 5b73ea7b3..0bf1c2fd5 100644 --- a/lib/apify/scrapeProfileUrlBatch.ts +++ b/lib/apify/scrapeProfileUrlBatch.ts @@ -1,5 +1,7 @@ import { ProfileScrapeResult, ScrapeProfileResult, scrapeProfileUrl } from "./scrapeProfileUrl"; +export type BatchProfileScrapeResult = ProfileScrapeResult & { profileUrl: string | null }; + type ScrapeProfileUrlBatchInput = { profileUrl: string | null | undefined; username: string | null | undefined; @@ -8,18 +10,22 @@ type ScrapeProfileUrlBatchInput = { export const scrapeProfileUrlBatch = async ( inputs: ScrapeProfileUrlBatchInput[], posts?: number, -): Promise => { +): Promise => { const results = await Promise.all( - inputs.map(({ profileUrl, username }) => - scrapeProfileUrl(profileUrl ?? null, username ?? "", posts), - ), + inputs.map(async ({ profileUrl, username }) => { + const result = await scrapeProfileUrl(profileUrl ?? null, username ?? "", posts); + return result ? { ...result, profileUrl: profileUrl ?? null } : null; + }), ); return results - .filter((result): result is ScrapeProfileResult => result !== null) - .map(({ runId, datasetId, error }) => ({ + .filter( + (result): result is ScrapeProfileResult & { profileUrl: string | null } => result !== null, + ) + .map(({ runId, datasetId, error, profileUrl }) => ({ runId, datasetId, error, + profileUrl, })); }; diff --git a/lib/apify/tiktok/__tests__/handleTiktokProfileScraperResults.test.ts b/lib/apify/tiktok/__tests__/handleTiktokProfileScraperResults.test.ts index 26c230684..9b84f8ca6 100644 --- a/lib/apify/tiktok/__tests__/handleTiktokProfileScraperResults.test.ts +++ b/lib/apify/tiktok/__tests__/handleTiktokProfileScraperResults.test.ts @@ -1,6 +1,9 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleTiktokProfileScraperResults } from "@/lib/apify/tiktok/handleTiktokProfileScraperResults"; const listItems = vi.fn(); +vi.mock("@/lib/socials/filterNewPostUrls", () => ({ + filterNewPostUrls: vi.fn(async (urls: string[]) => urls), +})); vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn(() => ({ listItems })) } })); const upsertSocials = vi.fn(); vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ diff --git a/lib/apify/tiktok/handleTiktokProfileScraperResults.ts b/lib/apify/tiktok/handleTiktokProfileScraperResults.ts index f7361d09b..d736efb6b 100644 --- a/lib/apify/tiktok/handleTiktokProfileScraperResults.ts +++ b/lib/apify/tiktok/handleTiktokProfileScraperResults.ts @@ -2,6 +2,7 @@ import apifyClient from "@/lib/apify/client"; import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; import { persistPostsForSocial } from "@/lib/apify/persistPostsForSocial"; +import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; import { toIsoDate } from "@/lib/apify/toIsoDate"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; import type { TablesInsert } from "@/types/database.types"; @@ -47,7 +48,9 @@ export async function handleTiktokProfileScraperResults(parsed: ApifyWebhookPayl ? [{ post_url: item.webVideoUrl, updated_at: toIsoDate(item.createTimeISO) }] : [], ); + // Diff before persisting so the digest can report genuinely new posts (chat#1855). + const newPostUrls = await filterNewPostUrls(postRows.map(p => p.post_url)); const { posts } = await persistPostsForSocial({ postRows, profileUrl: social.profile_url }); - return { social, posts }; + return { social, posts, newPostUrls }; } diff --git a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts index f1c03b78c..0497e058b 100644 --- a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts +++ b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts @@ -1,6 +1,9 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleTwitterProfileScraperResults } from "@/lib/apify/twitter/handleTwitterProfileScraperResults"; const listItems = vi.fn(); +vi.mock("@/lib/socials/filterNewPostUrls", () => ({ + filterNewPostUrls: vi.fn(async (urls: string[]) => urls), +})); vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn(() => ({ listItems })) } })); const upsertSocials = vi.fn(); vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ diff --git a/lib/apify/twitter/handleTwitterProfileScraperResults.ts b/lib/apify/twitter/handleTwitterProfileScraperResults.ts index c583cc9ca..4634fae0a 100644 --- a/lib/apify/twitter/handleTwitterProfileScraperResults.ts +++ b/lib/apify/twitter/handleTwitterProfileScraperResults.ts @@ -2,6 +2,7 @@ import apifyClient from "@/lib/apify/client"; import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; import { persistPostsForSocial } from "@/lib/apify/persistPostsForSocial"; +import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; import { toIsoDate } from "@/lib/apify/toIsoDate"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; import type { TablesInsert } from "@/types/database.types"; @@ -52,7 +53,9 @@ export async function handleTwitterProfileScraperResults(parsed: ApifyWebhookPay const postRows: TablesInsert<"posts">[] = (items as TweetItem[]).flatMap(item => item.url ? [{ post_url: item.url, updated_at: toIsoDate(item.createdAt) }] : [], ); + // Diff before persisting so the digest can report genuinely new posts (chat#1855). + const newPostUrls = await filterNewPostUrls(postRows.map(p => p.post_url)); const { posts } = await persistPostsForSocial({ postRows, profileUrl: social.profile_url }); - return { social, posts }; + return { social, posts, newPostUrls }; } diff --git a/lib/apify/validateApifyWebhookRequest.ts b/lib/apify/validateApifyWebhookRequest.ts index 2a446b595..4ccb76eae 100644 --- a/lib/apify/validateApifyWebhookRequest.ts +++ b/lib/apify/validateApifyWebhookRequest.ts @@ -13,6 +13,10 @@ export const apifyWebhookPayloadSchema = z.object({ actorId: z.string().min(1, "eventData.actorId is required"), }), resource: z.object({ + // Run id — present on Apify's default payload; used to complete the + // digest-batch registration (chat#1855). Optional so trimmed payloads + // keep validating. + id: z.string().optional(), defaultDatasetId: z.string().min(1, "resource.defaultDatasetId is required"), }), }); diff --git a/lib/artist/__tests__/postArtistSocialsScrapeHandler.test.ts b/lib/artist/__tests__/postArtistSocialsScrapeHandler.test.ts index 9fc090e47..ee95e0235 100644 --- a/lib/artist/__tests__/postArtistSocialsScrapeHandler.test.ts +++ b/lib/artist/__tests__/postArtistSocialsScrapeHandler.test.ts @@ -9,6 +9,9 @@ import { checkAccountArtistAccess } from "@/lib/artists/checkAccountArtistAccess import { ensureSocialScrapeCredits } from "@/lib/socials/ensureSocialScrapeCredits"; import { deductSocialScrapeCredits } from "@/lib/socials/deductSocialScrapeCredits"; +vi.mock("@/lib/supabase/apify_scraper_runs/insertApifyScraperRuns", () => ({ + insertApifyScraperRuns: vi.fn(async () => ({ data: null, error: null })), +})); vi.mock("@/lib/apify/scrapeProfileUrlBatch", () => ({ scrapeProfileUrlBatch: vi.fn() })); vi.mock("@/lib/supabase/account_socials/selectAccountSocials", () => ({ selectAccountSocials: vi.fn(), @@ -42,8 +45,8 @@ describe("postArtistSocialsScrapeHandler", () => { vi.mocked(ensureSocialScrapeCredits).mockResolvedValue(null); vi.mocked(selectAccountSocials).mockResolvedValue(TWO_SOCIALS as never); vi.mocked(scrapeProfileUrlBatch).mockResolvedValue([ - { runId: "r1", datasetId: "d1", error: null }, - { runId: "r2", datasetId: "d2", error: null }, + { runId: "r1", datasetId: "d1", error: null, profileUrl: null }, + { runId: "r2", datasetId: "d2", error: null, profileUrl: null }, ]); }); @@ -99,7 +102,7 @@ describe("postArtistSocialsScrapeHandler", () => { it("forwards posts and deducts (5 + posts) per profile actually scraped", async () => { vi.mocked(scrapeProfileUrlBatch).mockResolvedValue([ - { runId: "r1", datasetId: "d1", error: null }, + { runId: "r1", datasetId: "d1", error: null, profileUrl: null }, ]); await postArtistSocialsScrapeHandler(makeRequest({ artist_account_id: ARTIST_ID, posts: 20 })); expect(ensureSocialScrapeCredits).toHaveBeenCalledWith(ACCOUNT_ID, 50); diff --git a/lib/artist/postArtistSocialsScrapeHandler.ts b/lib/artist/postArtistSocialsScrapeHandler.ts index 530670824..5028d2279 100644 --- a/lib/artist/postArtistSocialsScrapeHandler.ts +++ b/lib/artist/postArtistSocialsScrapeHandler.ts @@ -8,6 +8,8 @@ import { validateAuthContext } from "@/lib/auth/validateAuthContext"; import { checkAccountArtistAccess } from "@/lib/artists/checkAccountArtistAccess"; import { ensureSocialScrapeCredits } from "@/lib/socials/ensureSocialScrapeCredits"; import { deductSocialScrapeCredits } from "@/lib/socials/deductSocialScrapeCredits"; +import { insertApifyScraperRuns } from "@/lib/supabase/apify_scraper_runs/insertApifyScraperRuns"; +import { getSocialPlatformByLink } from "@/lib/artists/getSocialPlatformByLink"; import { getSocialScrapeCreditCost } from "@/lib/socials/getSocialScrapeCreditCost"; /** @@ -70,12 +72,34 @@ export async function postArtistSocialsScrapeHandler(request: NextRequest): Prom if (results.length) { await deductSocialScrapeCredits(authResult.accountId, costPerProfile * results.length); + + // Register the batch so per-platform webhook completions can assemble + // ONE consolidated new-posts digest instead of an email per platform + // (chat#1855). Registration failure must not fail the scrape. + const batchId = crypto.randomUUID(); + const socialByUrl = new Map( + socials.map(s => [s.social?.profile_url ?? "", s.social?.id ?? null]), + ); + await insertApifyScraperRuns( + results + .filter(r => r.runId) + .map(r => ({ + run_id: r.runId as string, + account_id: authResult.accountId, + social_id: r.profileUrl ? (socialByUrl.get(r.profileUrl) ?? null) : null, + platform: r.profileUrl ? getSocialPlatformByLink(r.profileUrl).toLowerCase() : null, + batch_id: batchId, + })), + ); } - return NextResponse.json(results, { - status: 200, - headers: getCorsHeaders(), - }); + return NextResponse.json( + results.map(({ runId, datasetId, error }) => ({ runId, datasetId, error })), + { + status: 200, + headers: getCorsHeaders(), + }, + ); } catch (error) { console.error("[ERROR] postArtistSocialsScrapeHandler error:", error); return NextResponse.json( diff --git a/lib/supabase/apify_scraper_runs/completeApifyScraperRun.ts b/lib/supabase/apify_scraper_runs/completeApifyScraperRun.ts new file mode 100644 index 000000000..fa8f346c3 --- /dev/null +++ b/lib/supabase/apify_scraper_runs/completeApifyScraperRun.ts @@ -0,0 +1,24 @@ +import supabase from "@/lib/supabase/serverClient"; +import type { ApifyScraperRunRow } from "@/lib/supabase/apify_scraper_runs/types"; + +/** + * Marks a scrape run's results as processed and records which post URLs were + * genuinely new. Returns the updated row (carrying batch_id) or null when the + * run was never registered (legacy/non-batch runs). + */ +export async function completeApifyScraperRun( + runId: string, + newPostUrls: string[], +): Promise { + const { data, error } = await supabase + .from("apify_scraper_runs" as never) + .update({ completed_at: new Date().toISOString(), new_post_urls: newPostUrls } as never) + .eq("run_id", runId) + .select() + .maybeSingle(); + if (error) { + console.error("[ERROR] completeApifyScraperRun:", error); + return null; + } + return (data as ApifyScraperRunRow | null) ?? null; +} diff --git a/lib/supabase/apify_scraper_runs/insertApifyScraperRuns.ts b/lib/supabase/apify_scraper_runs/insertApifyScraperRuns.ts new file mode 100644 index 000000000..35a162908 --- /dev/null +++ b/lib/supabase/apify_scraper_runs/insertApifyScraperRuns.ts @@ -0,0 +1,19 @@ +import supabase from "@/lib/supabase/serverClient"; +import type { ApifyScraperRunRow } from "@/lib/supabase/apify_scraper_runs/types"; + +/** + * Records scrape runs at start time so per-platform webhook completions can + * find their digest-batch siblings (recoupable/chat#1855). + * + * @param runs - One row per started Apify run (run_id, account_id, batch_id, …). + */ +export async function insertApifyScraperRuns( + runs: Pick[], +) { + if (!runs.length) return { data: null, error: null }; + const { data, error } = await supabase + .from("apify_scraper_runs" as never) + .upsert(runs as never[], { onConflict: "run_id", ignoreDuplicates: true }); + if (error) console.error("[ERROR] insertApifyScraperRuns:", error); + return { data, error }; +} diff --git a/lib/supabase/apify_scraper_runs/selectApifyScraperRun.ts b/lib/supabase/apify_scraper_runs/selectApifyScraperRun.ts new file mode 100644 index 000000000..a9602a450 --- /dev/null +++ b/lib/supabase/apify_scraper_runs/selectApifyScraperRun.ts @@ -0,0 +1,16 @@ +import supabase from "@/lib/supabase/serverClient"; +import type { ApifyScraperRunRow } from "@/lib/supabase/apify_scraper_runs/types"; + +/** Returns the registered scrape run for an Apify run id, or null. */ +export async function selectApifyScraperRun(runId: string): Promise { + const { data, error } = await supabase + .from("apify_scraper_runs" as never) + .select("*") + .eq("run_id", runId) + .maybeSingle(); + if (error) { + console.error("[ERROR] selectApifyScraperRun:", error); + return null; + } + return (data as ApifyScraperRunRow | null) ?? null; +} diff --git a/lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch.ts b/lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch.ts new file mode 100644 index 000000000..0dd9457e6 --- /dev/null +++ b/lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch.ts @@ -0,0 +1,17 @@ +import supabase from "@/lib/supabase/serverClient"; +import type { ApifyScraperRunRow } from "@/lib/supabase/apify_scraper_runs/types"; + +/** Returns every scrape run registered under a digest batch. */ +export async function selectApifyScraperRunsByBatch( + batchId: string, +): Promise { + const { data, error } = await supabase + .from("apify_scraper_runs" as never) + .select("*") + .eq("batch_id", batchId); + if (error) { + console.error("[ERROR] selectApifyScraperRunsByBatch:", error); + return []; + } + return (data as ApifyScraperRunRow[] | null) ?? []; +} diff --git a/lib/supabase/apify_scraper_runs/types.ts b/lib/supabase/apify_scraper_runs/types.ts new file mode 100644 index 000000000..faa9d00cf --- /dev/null +++ b/lib/supabase/apify_scraper_runs/types.ts @@ -0,0 +1,12 @@ +/** apify_scraper_runs row incl. the digest-batch columns added in + * recoupable/database#41 (not yet in generated database.types). */ +export type ApifyScraperRunRow = { + run_id: string; + account_id: string; + social_id: string | null; + platform: string | null; + batch_id: string | null; + completed_at: string | null; + new_post_urls: string[] | null; + created_at?: string; +}; From 7960d93c474aa5c3b4c81ab3fc8f03e39afe6e8d Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Mon, 6 Jul 2026 17:09:26 -0500 Subject: [PATCH 03/12] feat(apify): deterministic digest template with direct links to each new post MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the per-send LLM email body (nondeterministic branding, vendor jargon reaching customers, no reliable post links) with a shared deterministic renderer: stable subject/branding, one section per platform, a direct link to every genuinely-new post, chat CTA secondary. Used by both the batch digest and the legacy solo Instagram alert, which now also requires new-post URLs and is BCC-only (recoupable/chat#1855, PR #5 of 6; supersedes the interim body from PR #4 and, for this file, the standalone BCC fix in api#758 — same invariant, tests included). Co-Authored-By: Claude Fable 5 --- .../__tests__/sendApifyWebhookEmail.test.ts | 62 +++++++++++++++++++ .../__tests__/renderScrapeDigestHtml.test.ts | 39 ++++++++++++ lib/apify/digest/renderScrapeDigestHtml.ts | 53 ++++++++++++++++ lib/apify/digest/sendScrapeDigestEmail.ts | 1 + ...ndleInstagramProfileScraperResults.test.ts | 2 +- .../handleInstagramProfileScraperResults.ts | 1 + lib/apify/sendApifyWebhookEmail.ts | 55 ++++++---------- 7 files changed, 177 insertions(+), 36 deletions(-) create mode 100644 lib/apify/__tests__/sendApifyWebhookEmail.test.ts create mode 100644 lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts create mode 100644 lib/apify/digest/renderScrapeDigestHtml.ts diff --git a/lib/apify/__tests__/sendApifyWebhookEmail.test.ts b/lib/apify/__tests__/sendApifyWebhookEmail.test.ts new file mode 100644 index 000000000..65d60d36f --- /dev/null +++ b/lib/apify/__tests__/sendApifyWebhookEmail.test.ts @@ -0,0 +1,62 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; +import { RECOUP_FROM_EMAIL } from "@/lib/const"; + +const sendEmailWithResend = vi.fn(); +vi.mock("@/lib/emails/sendEmail", () => ({ + sendEmailWithResend: (...a: unknown[]) => sendEmailWithResend(...a), +})); +vi.mock("@/lib/composio/getFrontendBaseUrl", () => ({ + getFrontendBaseUrl: () => "https://chat.recoupable.dev", +})); + +const PROFILE = { + fullName: "Ashnikko", + username: "ashnikko", + url: "https://instagram.com/ashnikko", + profilePicUrl: "https://example.com/pic.jpg", + biography: "bio", + followersCount: 100, + followsCount: 10, + latestPosts: [], +} as never; +const NEW_URLS = ["https://instagram.com/p/new1"]; + +beforeEach(() => { + vi.clearAllMocks(); + sendEmailWithResend.mockResolvedValue({ id: "email-1" }); +}); + +describe("sendApifyWebhookEmail", () => { + it("BCCs recipients so no account can see another's address (chat#1855)", async () => { + const recipients = ["manager@customer-a.com", "label@customer-b.com"]; + await sendApifyWebhookEmail(PROFILE, recipients, NEW_URLS); + const payload = sendEmailWithResend.mock.calls[0][0] as Record; + expect(payload.bcc).toEqual(recipients); + expect(payload.to).toEqual([RECOUP_FROM_EMAIL]); + }); + + it("never carries more than our own address in to/cc, regardless of recipient count", async () => { + const recipients = Array.from({ length: 25 }, (_, i) => `user${i}@example.com`); + await sendApifyWebhookEmail(PROFILE, recipients, NEW_URLS); + const payload = sendEmailWithResend.mock.calls[0][0] as Record; + expect([payload.to, payload.cc].flat().filter(Boolean)).toEqual([RECOUP_FROM_EMAIL]); + }); + + it("links the new posts in the deterministic body — no vendor jargon", async () => { + await sendApifyWebhookEmail(PROFILE, ["a@b.com"], NEW_URLS); + const payload = sendEmailWithResend.mock.calls[0][0] as Record; + expect(payload.html).toContain('href="https://instagram.com/p/new1"'); + expect((payload.html + payload.subject).toLowerCase()).not.toContain("apify"); + }); + + it("returns null and sends nothing when there are no recipients", async () => { + expect(await sendApifyWebhookEmail(PROFILE, [], NEW_URLS)).toBeNull(); + expect(sendEmailWithResend).not.toHaveBeenCalled(); + }); + + it("returns null and sends nothing when no post is genuinely new", async () => { + expect(await sendApifyWebhookEmail(PROFILE, ["a@b.com"], [])).toBeNull(); + expect(sendEmailWithResend).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts b/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts new file mode 100644 index 000000000..3562f7310 --- /dev/null +++ b/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts @@ -0,0 +1,39 @@ +import { describe, it, expect } from "vitest"; +import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtml"; + +const SECTIONS = [ + { + platform: "instagram", + postUrls: ["https://instagram.com/p/abc", "https://instagram.com/p/def"], + }, + { platform: "tiktok", postUrls: ["https://tiktok.com/@a/video/1"] }, +]; + +describe("renderScrapeDigestHtml", () => { + it("links every new post, grouped under its platform", () => { + const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + for (const s of SECTIONS) for (const u of s.postUrls) expect(html).toContain(`href="${u}"`); + expect(html.toLowerCase()).toContain("instagram"); + expect(html.toLowerCase()).toContain("tiktok"); + }); + + it("is deterministic — identical input renders identical output", () => { + const a = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + const b = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + expect(a).toEqual(b); + }); + + it("never leaks internal vendor jargon to customers", () => { + const { html, subject } = renderScrapeDigestHtml({ + sections: SECTIONS, + artistName: "Ashnikko", + }); + expect((html + subject).toLowerCase()).not.toContain("apify"); + expect((html + subject).toLowerCase()).not.toContain("dataset"); + }); + + it("counts posts and platforms in the subject", () => { + const { subject } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + expect(subject).toBe("Ashnikko: 3 new posts across 2 platforms"); + }); +}); diff --git a/lib/apify/digest/renderScrapeDigestHtml.ts b/lib/apify/digest/renderScrapeDigestHtml.ts new file mode 100644 index 000000000..f0583a3a8 --- /dev/null +++ b/lib/apify/digest/renderScrapeDigestHtml.ts @@ -0,0 +1,53 @@ +import { getFrontendBaseUrl } from "@/lib/composio/getFrontendBaseUrl"; +import type { ScrapeDigestSection } from "@/lib/apify/digest/sendScrapeDigestEmail"; + +const PLATFORM_LABELS: Record = { + instagram: "Instagram", + tiktok: "TikTok", + x: "X", + twitter: "X", + youtube: "YouTube", + facebook: "Facebook", + threads: "Threads", +}; + +/** + * Deterministic house-style renderer for the new-posts digest (chat#1855). + * Replaces the per-send LLM body: identical input renders identical output, + * every genuinely-new post is linked directly, and internal vendor jargon + * never reaches customers. + */ +export function renderScrapeDigestHtml({ + sections, + artistName, +}: { + sections: ScrapeDigestSection[]; + artistName?: string | null; +}): { subject: string; html: string } { + const who = artistName || "Your artist"; + const total = sections.reduce((n, s) => n + s.postUrls.length, 0); + const subject = `${who}: ${total} new post${total === 1 ? "" : "s"} across ${sections.length} platform${sections.length === 1 ? "" : "s"}`; + + const sectionHtml = sections + .map(section => { + const label = PLATFORM_LABELS[section.platform.toLowerCase()] ?? section.platform; + const items = section.postUrls + .map(url => { + const href = url.startsWith("http") ? url : `https://${url}`; + return `
  • ${href}
  • `; + }) + .join(""); + return `

    ${label}

      ${items}
    `; + }) + .join(""); + + const chatUrl = `${getFrontendBaseUrl()}/?q=${encodeURIComponent("tell me about my artist's latest posts")}`; + const html = `
    +

    New posts from ${who}

    +

    We found ${total} new post${total === 1 ? "" : "s"} since we last checked:

    +${sectionHtml} +

    Ask Recoup about these posts →

    +
    `; + + return { subject, html }; +} diff --git a/lib/apify/digest/sendScrapeDigestEmail.ts b/lib/apify/digest/sendScrapeDigestEmail.ts index c75df6943..3ae915bbc 100644 --- a/lib/apify/digest/sendScrapeDigestEmail.ts +++ b/lib/apify/digest/sendScrapeDigestEmail.ts @@ -1,5 +1,6 @@ import { sendEmailWithResend } from "@/lib/emails/sendEmail"; import { RECOUP_FROM_EMAIL } from "@/lib/const"; +import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtml"; export type ScrapeDigestSection = { platform: string; postUrls: string[] }; export type ScrapeDigestInput = { diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts index 2a9850140..d8310dbd5 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts @@ -97,7 +97,7 @@ describe("handleInstagramProfileScraperResults", () => { expect(upsertPosts).toHaveBeenCalledOnce(); expect(upsertSocialPosts).toHaveBeenCalledOnce(); - expect(sendApifyWebhookEmail).toHaveBeenCalledWith(expect.any(Object), ["x@y.com"]); + expect(sendApifyWebhookEmail).toHaveBeenCalledWith(expect.any(Object), ["x@y.com"], ["u1"]); expect(handleInstagramProfileFollowUpRuns).toHaveBeenCalledOnce(); expect(result.social).toEqual({ id: "s1" }); expect(result.posts).toEqual(posts); diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index 3244e8eb7..de38d4c44 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -112,6 +112,7 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP sentEmails = await sendApifyWebhookEmail( firstResult, accountEmails.map(e => e.email).filter(Boolean), + newPostUrls, ); } } catch (error) { diff --git a/lib/apify/sendApifyWebhookEmail.ts b/lib/apify/sendApifyWebhookEmail.ts index 051db1495..5503b3612 100644 --- a/lib/apify/sendApifyWebhookEmail.ts +++ b/lib/apify/sendApifyWebhookEmail.ts @@ -1,54 +1,39 @@ -import generateText from "@/lib/ai/generateText"; import { sendEmailWithResend } from "@/lib/emails/sendEmail"; import { RECOUP_FROM_EMAIL } from "@/lib/const"; -import { getFrontendBaseUrl } from "@/lib/composio/getFrontendBaseUrl"; +import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtml"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; /** - * Sends an Apify-webhook summary email to the given recipients using - * Resend. Generates the email body via an LLM based on the first - * profile result. + * Sends the legacy single-platform Instagram alert (non-batch scrape runs). + * Batch runs get the consolidated digest instead. Body comes from the shared + * deterministic renderer — the per-send LLM body is gone (chat#1855): stable + * branding, direct links to the new posts, no vendor jargon. * * @param profile - First dataset result from the profile scraper. - * @param emails - Recipient email addresses. - * @returns The Resend response, or `null` when there are no recipients. + * @param emails - Recipient email addresses (cross-tenant — BCC only). + * @param newPostUrls - Post URLs genuinely new to the platform. + * @returns The Resend response, or `null` when there is nothing to send. */ export async function sendApifyWebhookEmail( profile: ApifyInstagramProfileResult, emails: string[], + newPostUrls: string[] = [], ) { - if (!emails?.length) return null; + if (!emails?.length || !newPostUrls.length) return null; - const prompt = `You have a new Apify dataset update. Here is the data: - -Key Data -Full Name: ${profile.fullName} -Username: ${profile.username} -Profile URL: ${profile.url} -Profile Picture: ${profile.profilePicUrl} -Biography: ${profile.biography} -Followers: ${profile.followersCount} -Following: ${profile.followsCount} -Latest Posts: ${(Array.isArray(profile.latestPosts) ? profile.latestPosts : []).map(p => JSON.stringify(p)).join(", ")} -`; - - const { text } = await generateText({ - system: `you are a record label services manager for Recoup. - write beautiful html email. - subject: New Apify Dataset Notification. you're notifying music managers about new posts being available for one of their roster musician's Instagram profile. - include a link to view the instagram profile. - call to action is to open a chat link to learn more about the latest posts using Recoup Chat (AI Agents): ${getFrontendBaseUrl()}/?q=tell%20me%20about%20my%20latest%20Ig%20posts - You'll be passed a dataset summary for a musician profile and their latest posts on instagram. - your goal is to get the recipient to click a cta link to open a chat link to learn more about the latest posts using Recoup Chat (AI Agents). - only include the email body html. - no headers or subject`, - prompt, + const { subject, html } = renderScrapeDigestHtml({ + sections: [{ platform: "instagram", postUrls: newPostUrls }], + artistName: profile.fullName ?? profile.username ?? null, }); + // Recipients span multiple customer accounts (the social's watchers are + // resolved cross-tenant), so they must NEVER share a visible To: line — + // BCC only, with ourselves as the required To: (chat#1855). return await sendEmailWithResend({ from: RECOUP_FROM_EMAIL, - to: emails, - subject: `${profile.fullName ?? profile.username ?? "Your artist"} has new posts on Instagram`, - html: text, + to: [RECOUP_FROM_EMAIL], + bcc: emails, + subject, + html, }); } From c0f0118527131070f14aca528acafb1bc62b3df0 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Thu, 9 Jul 2026 15:36:19 -0500 Subject: [PATCH 04/12] test(apify): guard sendScrapeDigestEmail's renderer wiring + BCC invariant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The branch imported renderScrapeDigestHtml in sendScrapeDigestEmail but never called it — the digest still rendered the old inline body (found during the main-sync conflict resolution; wired in that merge commit). This test fails against the unwired version: it asserts the send payload is byte-identical to the renderer's output, plus the BCC-only invariant and the empty-input no-send. Co-Authored-By: Claude Fable 5 --- .../__tests__/sendScrapeDigestEmail.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 lib/apify/digest/__tests__/sendScrapeDigestEmail.test.ts diff --git a/lib/apify/digest/__tests__/sendScrapeDigestEmail.test.ts b/lib/apify/digest/__tests__/sendScrapeDigestEmail.test.ts new file mode 100644 index 000000000..d63ee1c39 --- /dev/null +++ b/lib/apify/digest/__tests__/sendScrapeDigestEmail.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { sendScrapeDigestEmail } from "@/lib/apify/digest/sendScrapeDigestEmail"; +import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtml"; +import { RECOUP_FROM_EMAIL } from "@/lib/const"; + +const sendEmailWithResend = vi.fn(); +vi.mock("@/lib/emails/sendEmail", () => ({ + sendEmailWithResend: (...a: unknown[]) => sendEmailWithResend(...a), +})); +vi.mock("@/lib/composio/getFrontendBaseUrl", () => ({ + getFrontendBaseUrl: () => "https://chat.recoupable.dev", +})); + +const SECTIONS = [ + { platform: "instagram", postUrls: ["https://instagram.com/p/a", "https://instagram.com/p/b"] }, + { platform: "tiktok", postUrls: ["https://tiktok.com/@x/video/1"] }, +]; + +beforeEach(() => { + vi.clearAllMocks(); + sendEmailWithResend.mockResolvedValue({ id: "email-1" }); +}); + +describe("sendScrapeDigestEmail", () => { + it("sends the deterministic renderer's output, not an ad-hoc body (chat#1855)", async () => { + await sendScrapeDigestEmail({ emails: ["a@b.com"], sections: SECTIONS, artistName: "Nat" }); + const payload = sendEmailWithResend.mock.calls[0][0] as Record; + const rendered = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Nat" }); + expect(payload.subject).toBe(rendered.subject); + expect(payload.html).toBe(rendered.html); + }); + + it("BCCs recipients so no account can see another's address", async () => { + const recipients = ["a@b.com", "c@d.com"]; + await sendScrapeDigestEmail({ emails: recipients, sections: SECTIONS }); + const payload = sendEmailWithResend.mock.calls[0][0] as Record; + expect(payload.bcc).toEqual(recipients); + expect(payload.to).toEqual([RECOUP_FROM_EMAIL]); + }); + + it("sends nothing without recipients or sections", async () => { + expect(await sendScrapeDigestEmail({ emails: [], sections: SECTIONS })).toBeNull(); + expect(await sendScrapeDigestEmail({ emails: ["a@b.com"], sections: [] })).toBeNull(); + expect(sendEmailWithResend).not.toHaveBeenCalled(); + }); +}); From b729ccd3e50ab53eaa974fa0578b1fd54bae1be2 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Thu, 9 Jul 2026 15:54:39 -0500 Subject: [PATCH 05/12] feat(apify): house-style digest template with post media, captions, and dates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v1 deterministic template regressed visual quality vs the old LLM emails (bare h3/ul list). This upgrades the renderer to the DESIGN.md house style — achromatic chrome, card-per-post with 72px thumbnail, escaped caption excerpt, date, and a direct link; black CTA button — while staying deterministic and email-safe (tables + inline styles). Media plumbing: extractPostsFromDatasetItems maps platform dataset items (IG latestPosts, TikTok items) to {url, caption, thumbnailUrl, timestamp}, limited to the genuinely-new URLs. The solo alert enriches from the dataset already in memory; the digest enriches via getRunDigestSection, which re-reads each run's dataset from Apify (source of truth) and degrades to URL-only links on any failure — enrichment never blocks a send. Captions are HTML-escaped so scraped content can't inject markup. Co-Authored-By: Claude Fable 5 --- .../__tests__/sendApifyWebhookEmail.test.ts | 16 +++- .../extractPostsFromDatasetItems.test.ts | 77 +++++++++++++++++++ .../__tests__/maybeSendScrapeDigest.test.ts | 12 ++- .../__tests__/renderScrapeDigestHtml.test.ts | 54 +++++++++++-- .../__tests__/sendScrapeDigestEmail.test.ts | 7 +- .../digest/extractPostsFromDatasetItems.ts | 61 +++++++++++++++ lib/apify/digest/getRunDigestSection.ts | 32 ++++++++ lib/apify/digest/maybeSendScrapeDigest.ts | 9 ++- lib/apify/digest/renderScrapeDigestHtml.ts | 71 +++++++++++++---- lib/apify/digest/sendScrapeDigestEmail.ts | 2 +- lib/apify/sendApifyWebhookEmail.ts | 7 +- 11 files changed, 317 insertions(+), 31 deletions(-) create mode 100644 lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts create mode 100644 lib/apify/digest/extractPostsFromDatasetItems.ts create mode 100644 lib/apify/digest/getRunDigestSection.ts diff --git a/lib/apify/__tests__/sendApifyWebhookEmail.test.ts b/lib/apify/__tests__/sendApifyWebhookEmail.test.ts index 65d60d36f..f5ed448b1 100644 --- a/lib/apify/__tests__/sendApifyWebhookEmail.test.ts +++ b/lib/apify/__tests__/sendApifyWebhookEmail.test.ts @@ -18,7 +18,14 @@ const PROFILE = { biography: "bio", followersCount: 100, followsCount: 10, - latestPosts: [], + latestPosts: [ + { + url: "https://instagram.com/p/new1", + caption: "New tour dates", + displayUrl: "https://cdn.ig/new1.jpg", + timestamp: "2026-07-08T12:00:00.000Z", + }, + ], } as never; const NEW_URLS = ["https://instagram.com/p/new1"]; @@ -50,6 +57,13 @@ describe("sendApifyWebhookEmail", () => { expect((payload.html + payload.subject).toLowerCase()).not.toContain("apify"); }); + it("enriches the body with the post's media and caption from the dataset in hand", async () => { + await sendApifyWebhookEmail(PROFILE, ["a@b.com"], NEW_URLS); + const payload = sendEmailWithResend.mock.calls[0][0] as Record; + expect(payload.html).toContain('src="https://cdn.ig/new1.jpg"'); + expect(payload.html).toContain("New tour dates"); + }); + it("returns null and sends nothing when there are no recipients", async () => { expect(await sendApifyWebhookEmail(PROFILE, [], NEW_URLS)).toBeNull(); expect(sendEmailWithResend).not.toHaveBeenCalled(); diff --git a/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts b/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts new file mode 100644 index 000000000..97e68cb5b --- /dev/null +++ b/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect } from "vitest"; +import { extractPostsFromDatasetItems } from "@/lib/apify/digest/extractPostsFromDatasetItems"; + +const IG_ITEMS = [ + { + latestPosts: [ + { + url: "https://instagram.com/p/new1", + caption: "cap one", + displayUrl: "https://cdn.ig/new1.jpg", + timestamp: "2026-07-08T12:00:00.000Z", + }, + { + url: "https://instagram.com/p/old", + caption: "old", + displayUrl: "https://cdn.ig/old.jpg", + timestamp: "2026-06-01T00:00:00.000Z", + }, + ], + }, +]; + +const TIKTOK_ITEMS = [ + { + webVideoUrl: "https://tiktok.com/@a/video/1", + text: "tt caption", + createTimeISO: "2026-07-09T09:00:00.000Z", + videoMeta: { coverUrl: "https://cdn.tt/1.jpg" }, + }, + { webVideoUrl: "https://tiktok.com/@a/video/2", text: "known", createTimeISO: "2026-01-01T00:00:00.000Z" }, +]; + +describe("extractPostsFromDatasetItems", () => { + it("maps Instagram latestPosts limited to the new URLs, with media", () => { + const posts = extractPostsFromDatasetItems("instagram", IG_ITEMS, [ + "https://instagram.com/p/new1", + ]); + expect(posts).toEqual([ + { + url: "https://instagram.com/p/new1", + caption: "cap one", + thumbnailUrl: "https://cdn.ig/new1.jpg", + timestamp: "2026-07-08T12:00:00.000Z", + }, + ]); + }); + + it("maps TikTok items limited to the new URLs, with cover media", () => { + const posts = extractPostsFromDatasetItems("tiktok", TIKTOK_ITEMS, [ + "https://tiktok.com/@a/video/1", + ]); + expect(posts).toEqual([ + { + url: "https://tiktok.com/@a/video/1", + caption: "tt caption", + thumbnailUrl: "https://cdn.tt/1.jpg", + timestamp: "2026-07-09T09:00:00.000Z", + }, + ]); + }); + + it("falls back to URL-only posts for platforms without an extractor", () => { + const posts = extractPostsFromDatasetItems("youtube", [{ some: "shape" }], [ + "https://youtube.com/watch?v=1", + ]); + expect(posts).toEqual([{ url: "https://youtube.com/watch?v=1" }]); + }); + + it("falls back to URL-only for new URLs missing from the dataset items", () => { + const posts = extractPostsFromDatasetItems("instagram", IG_ITEMS, [ + "https://instagram.com/p/new1", + "https://instagram.com/p/not-in-items", + ]); + expect(posts).toHaveLength(2); + expect(posts[1]).toEqual({ url: "https://instagram.com/p/not-in-items" }); + }); +}); diff --git a/lib/apify/digest/__tests__/maybeSendScrapeDigest.test.ts b/lib/apify/digest/__tests__/maybeSendScrapeDigest.test.ts index 8b2dbaba6..11bc146d7 100644 --- a/lib/apify/digest/__tests__/maybeSendScrapeDigest.test.ts +++ b/lib/apify/digest/__tests__/maybeSendScrapeDigest.test.ts @@ -10,6 +10,14 @@ vi.mock("@/lib/supabase/apify_scraper_runs/selectApifyScraperRuns", () => ({ vi.mock("@/lib/apify/digest/getScrapeDigestRecipients", () => ({ getScrapeDigestRecipients: vi.fn(), })); +vi.mock("@/lib/apify/digest/getRunDigestSection", () => ({ + // URL-only translation of a run row — enrichment is unit-tested separately + getRunDigestSection: vi.fn(async (run: { platform: string; new_post_urls: string[] | null }) => + run.new_post_urls?.length + ? { platform: run.platform, posts: run.new_post_urls.map((url: string) => ({ url })) } + : null, + ), +})); vi.mock("@/lib/apify/digest/sendScrapeDigestEmail", () => ({ sendScrapeDigestEmail: vi.fn(), })); @@ -51,8 +59,8 @@ describe("maybeSendScrapeDigest", () => { expect(sendScrapeDigestEmail).toHaveBeenCalledOnce(); const arg = vi.mocked(sendScrapeDigestEmail).mock.calls[0][0]; expect(arg.sections).toEqual([ - { platform: "instagram", postUrls: ["https://instagram.com/p/1"] }, - { platform: "tiktok", postUrls: ["https://tiktok.com/v/2"] }, + { platform: "instagram", posts: [{ url: "https://instagram.com/p/1" }] }, + { platform: "tiktok", posts: [{ url: "https://tiktok.com/v/2" }] }, ]); // x omitted — nothing new expect(arg.emails).toEqual(["owner@example.com"]); }); diff --git a/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts b/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts index 3562f7310..f16e1ee33 100644 --- a/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts +++ b/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts @@ -4,17 +4,53 @@ import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtm const SECTIONS = [ { platform: "instagram", - postUrls: ["https://instagram.com/p/abc", "https://instagram.com/p/def"], + posts: [ + { + url: "https://instagram.com/p/abc", + caption: "Behind the scenes tour & more", + thumbnailUrl: "https://cdn.example.com/thumb-abc.jpg", + timestamp: "2026-07-08T12:00:00.000Z", + }, + { url: "https://instagram.com/p/def", caption: null, thumbnailUrl: null, timestamp: null }, + ], + }, + { + platform: "tiktok", + posts: [ + { + url: "https://tiktok.com/@a/video/1", + caption: "New single out now", + thumbnailUrl: "https://cdn.example.com/cover-1.jpg", + timestamp: "2026-07-09T09:30:00.000Z", + }, + ], }, - { platform: "tiktok", postUrls: ["https://tiktok.com/@a/video/1"] }, ]; describe("renderScrapeDigestHtml", () => { - it("links every new post, grouped under its platform", () => { + it("links every new post, grouped under its platform label", () => { + const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + for (const s of SECTIONS) for (const p of s.posts) expect(html).toContain(`href="${p.url}"`); + expect(html).toContain("Instagram"); + expect(html).toContain("TikTok"); + }); + + it("renders post media and caption excerpts when available", () => { + const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + expect(html).toContain('src="https://cdn.example.com/thumb-abc.jpg"'); + expect(html).toContain('src="https://cdn.example.com/cover-1.jpg"'); + expect(html).toContain("New single out now"); + }); + + it("escapes HTML in captions so scraped content cannot inject markup", () => { + const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + expect(html).not.toContain("tour"); + expect(html).toContain("<b>tour</b> & more"); + }); + + it("still renders a linked card when a post has no media or caption", () => { const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); - for (const s of SECTIONS) for (const u of s.postUrls) expect(html).toContain(`href="${u}"`); - expect(html.toLowerCase()).toContain("instagram"); - expect(html.toLowerCase()).toContain("tiktok"); + expect(html).toContain('href="https://instagram.com/p/def"'); }); it("is deterministic — identical input renders identical output", () => { @@ -23,6 +59,12 @@ describe("renderScrapeDigestHtml", () => { expect(a).toEqual(b); }); + it("uses the house style — achromatic chrome, no ad-hoc colors", () => { + const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + expect(html).toContain("#0a0a0a"); + expect(html).toContain("#e8e8e8"); + }); + it("never leaks internal vendor jargon to customers", () => { const { html, subject } = renderScrapeDigestHtml({ sections: SECTIONS, diff --git a/lib/apify/digest/__tests__/sendScrapeDigestEmail.test.ts b/lib/apify/digest/__tests__/sendScrapeDigestEmail.test.ts index d63ee1c39..2eed9b8ee 100644 --- a/lib/apify/digest/__tests__/sendScrapeDigestEmail.test.ts +++ b/lib/apify/digest/__tests__/sendScrapeDigestEmail.test.ts @@ -12,8 +12,11 @@ vi.mock("@/lib/composio/getFrontendBaseUrl", () => ({ })); const SECTIONS = [ - { platform: "instagram", postUrls: ["https://instagram.com/p/a", "https://instagram.com/p/b"] }, - { platform: "tiktok", postUrls: ["https://tiktok.com/@x/video/1"] }, + { + platform: "instagram", + posts: [{ url: "https://instagram.com/p/a" }, { url: "https://instagram.com/p/b" }], + }, + { platform: "tiktok", posts: [{ url: "https://tiktok.com/@x/video/1" }] }, ]; beforeEach(() => { diff --git a/lib/apify/digest/extractPostsFromDatasetItems.ts b/lib/apify/digest/extractPostsFromDatasetItems.ts new file mode 100644 index 000000000..8b6119879 --- /dev/null +++ b/lib/apify/digest/extractPostsFromDatasetItems.ts @@ -0,0 +1,61 @@ +import type { ScrapeDigestPost } from "@/lib/apify/digest/renderScrapeDigestHtml"; + +type Extractor = (items: unknown[]) => Map; + +const asRecord = (v: unknown): Record => + v && typeof v === "object" ? (v as Record) : {}; + +const str = (v: unknown): string | null => (typeof v === "string" && v ? v : null); + +/** Instagram profile-scraper dataset: posts ride on item[0].latestPosts. */ +const instagram: Extractor = items => { + const map = new Map(); + const latest = asRecord(items[0]).latestPosts; + for (const raw of Array.isArray(latest) ? latest : []) { + const post = asRecord(raw); + const url = str(post.url); + if (!url) continue; + map.set(url, { + url, + caption: str(post.caption), + thumbnailUrl: str(post.displayUrl), + timestamp: str(post.timestamp), + }); + } + return map; +}; + +/** TikTok scraper dataset: one item per video. */ +const tiktok: Extractor = items => { + const map = new Map(); + for (const raw of items) { + const item = asRecord(raw); + const url = str(item.webVideoUrl); + if (!url) continue; + map.set(url, { + url, + caption: str(item.text), + thumbnailUrl: str(asRecord(item.videoMeta).coverUrl), + timestamp: str(item.createTimeISO), + }); + } + return map; +}; + +const EXTRACTORS: Record = { instagram, tiktok }; + +/** + * Builds rich digest posts (caption, media, timestamp) for a platform's + * genuinely-new post URLs from its raw scraper dataset items. Platforms + * without an extractor — and URLs missing from the items — degrade to + * URL-only posts, so enrichment can never lose a link (chat#1855). + */ +export function extractPostsFromDatasetItems( + platform: string, + items: unknown[], + newPostUrls: string[], +): ScrapeDigestPost[] { + const extractor = EXTRACTORS[platform.toLowerCase()]; + const byUrl = extractor ? extractor(items) : new Map(); + return newPostUrls.map(url => byUrl.get(url) ?? { url }); +} diff --git a/lib/apify/digest/getRunDigestSection.ts b/lib/apify/digest/getRunDigestSection.ts new file mode 100644 index 000000000..563653991 --- /dev/null +++ b/lib/apify/digest/getRunDigestSection.ts @@ -0,0 +1,32 @@ +import apifyClient from "@/lib/apify/client"; +import { parseNewPostUrls } from "@/lib/apify/digest/parseNewPostUrls"; +import { extractPostsFromDatasetItems } from "@/lib/apify/digest/extractPostsFromDatasetItems"; +import type { ScrapeDigestSection } from "@/lib/apify/digest/renderScrapeDigestHtml"; +import type { Tables } from "@/types/database.types"; + +/** + * Builds one platform's digest section for a completed scrape run, enriching + * the stored genuinely-new URLs with caption/media/timestamp by re-reading + * the run's dataset from Apify (the source of truth for scrape content — + * only the URL diff is persisted, chat#1855). Enrichment must never block + * the digest: any failure degrades to URL-only posts. Returns null when the + * run found nothing new. + */ +export async function getRunDigestSection( + run: Tables<"apify_scraper_runs">, +): Promise { + const urls = parseNewPostUrls(run.new_post_urls); + if (!urls.length) return null; + const platform = run.platform ?? "other"; + + try { + const runInfo = await apifyClient.run(run.run_id).get(); + const datasetId = runInfo?.defaultDatasetId; + if (!datasetId) return { platform, posts: urls.map(url => ({ url })) }; + const { items } = await apifyClient.dataset(datasetId).listItems(); + return { platform, posts: extractPostsFromDatasetItems(platform, items, urls) }; + } catch (error) { + console.error("[WARN] digest enrichment failed; sending URL-only links:", error); + return { platform, posts: urls.map(url => ({ url })) }; + } +} diff --git a/lib/apify/digest/maybeSendScrapeDigest.ts b/lib/apify/digest/maybeSendScrapeDigest.ts index cd14c13fd..8efeb57c3 100644 --- a/lib/apify/digest/maybeSendScrapeDigest.ts +++ b/lib/apify/digest/maybeSendScrapeDigest.ts @@ -1,7 +1,8 @@ import { selectApifyScraperRuns } from "@/lib/supabase/apify_scraper_runs/selectApifyScraperRuns"; -import { parseNewPostUrls } from "@/lib/apify/digest/parseNewPostUrls"; +import { getRunDigestSection } from "@/lib/apify/digest/getRunDigestSection"; import { getScrapeDigestRecipients } from "@/lib/apify/digest/getScrapeDigestRecipients"; import { sendScrapeDigestEmail } from "@/lib/apify/digest/sendScrapeDigestEmail"; +import type { ScrapeDigestSection } from "@/lib/apify/digest/renderScrapeDigestHtml"; /** * Batch-completion check for the one-digest-per-scrape design (chat#1855): @@ -16,9 +17,9 @@ export async function maybeSendScrapeDigest(batchId: string | null | undefined) const runs = await selectApifyScraperRuns({ batchId }); if (!runs.length || runs.some(r => !r.completed_at)) return null; - const sections = runs - .map(r => ({ platform: r.platform ?? "other", postUrls: parseNewPostUrls(r.new_post_urls) })) - .filter(s => s.postUrls.length > 0); + const sections = (await Promise.all(runs.map(getRunDigestSection))).filter( + (s): s is ScrapeDigestSection => s !== null, + ); if (!sections.length) return null; const emails = await getScrapeDigestRecipients( diff --git a/lib/apify/digest/renderScrapeDigestHtml.ts b/lib/apify/digest/renderScrapeDigestHtml.ts index f0583a3a8..62339b03e 100644 --- a/lib/apify/digest/renderScrapeDigestHtml.ts +++ b/lib/apify/digest/renderScrapeDigestHtml.ts @@ -1,5 +1,13 @@ import { getFrontendBaseUrl } from "@/lib/composio/getFrontendBaseUrl"; -import type { ScrapeDigestSection } from "@/lib/apify/digest/sendScrapeDigestEmail"; + +export type ScrapeDigestPost = { + url: string; + caption?: string | null; + thumbnailUrl?: string | null; + timestamp?: string | null; +}; + +export type ScrapeDigestSection = { platform: string; posts: ScrapeDigestPost[] }; const PLATFORM_LABELS: Record = { instagram: "Instagram", @@ -11,11 +19,36 @@ const PLATFORM_LABELS: Record = { threads: "Threads", }; +const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + +/** Escapes scraped text so captions can never inject markup into the email. */ +function escapeHtml(text: string): string { + return text + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +/** Deterministic "Jul 8" date label (UTC — no locale/timezone variance). */ +function formatDate(iso: string): string { + const d = new Date(iso); + if (isNaN(d.getTime())) return ""; + return `${MONTHS[d.getUTCMonth()]} ${d.getUTCDate()}`; +} + +function truncate(text: string, max: number): string { + return text.length <= max ? text : `${text.slice(0, max - 1).trimEnd()}…`; +} + /** * Deterministic house-style renderer for the new-posts digest (chat#1855). * Replaces the per-send LLM body: identical input renders identical output, - * every genuinely-new post is linked directly, and internal vendor jargon - * never reaches customers. + * every genuinely-new post is a linked card with its media and caption, and + * internal vendor jargon never reaches customers. Styling follows DESIGN.md — + * achromatic chrome (#0a0a0a on #ffffff, #e8e8e8 borders, #6b6b6b muted); + * color comes from the post media, not the chrome. Email-safe: tables + + * inline styles only, system font stack. */ export function renderScrapeDigestHtml({ sections, @@ -25,29 +58,39 @@ export function renderScrapeDigestHtml({ artistName?: string | null; }): { subject: string; html: string } { const who = artistName || "Your artist"; - const total = sections.reduce((n, s) => n + s.postUrls.length, 0); + const total = sections.reduce((n, s) => n + s.posts.length, 0); const subject = `${who}: ${total} new post${total === 1 ? "" : "s"} across ${sections.length} platform${sections.length === 1 ? "" : "s"}`; const sectionHtml = sections .map(section => { const label = PLATFORM_LABELS[section.platform.toLowerCase()] ?? section.platform; - const items = section.postUrls - .map(url => { - const href = url.startsWith("http") ? url : `https://${url}`; - return `
  • ${href}
  • `; + const cards = section.posts + .map(post => { + const href = post.url.startsWith("http") ? post.url : `https://${post.url}`; + const caption = post.caption ? escapeHtml(truncate(post.caption, 110)) : ""; + const date = post.timestamp ? formatDate(post.timestamp) : ""; + const thumbCell = post.thumbnailUrl + ? `` + : ""; + return `
    ${thumbCell}
    ${caption ? `

    ${caption}

    ` : ""}

    ${date ? `${date} · ` : ""}View post →

    `; }) .join(""); - return `

    ${label}

      ${items}
    `; + return `

    ${label}

    ${cards}`; }) .join(""); const chatUrl = `${getFrontendBaseUrl()}/?q=${encodeURIComponent("tell me about my artist's latest posts")}`; - const html = `
    -

    New posts from ${who}

    -

    We found ${total} new post${total === 1 ? "" : "s"} since we last checked:

    + const html = `
    + +
    +

    New posts

    +

    ${escapeHtml(who)}

    +

    ${total} new post${total === 1 ? "" : "s"} since we last checked

    ${sectionHtml} -

    Ask Recoup about these posts →

    -`; +
    Ask Recoup about these posts →
    +

    You're receiving this because you work with this artist on Recoup.

    +
    +
    `; return { subject, html }; } diff --git a/lib/apify/digest/sendScrapeDigestEmail.ts b/lib/apify/digest/sendScrapeDigestEmail.ts index 93b18a059..3d1cfcf25 100644 --- a/lib/apify/digest/sendScrapeDigestEmail.ts +++ b/lib/apify/digest/sendScrapeDigestEmail.ts @@ -1,8 +1,8 @@ import { sendEmailWithResend } from "@/lib/emails/sendEmail"; import { RECOUP_FROM_EMAIL } from "@/lib/const"; import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtml"; +import type { ScrapeDigestSection } from "@/lib/apify/digest/renderScrapeDigestHtml"; -export type ScrapeDigestSection = { platform: string; postUrls: string[] }; export type ScrapeDigestInput = { emails: string[]; sections: ScrapeDigestSection[]; diff --git a/lib/apify/sendApifyWebhookEmail.ts b/lib/apify/sendApifyWebhookEmail.ts index 5503b3612..adb41a401 100644 --- a/lib/apify/sendApifyWebhookEmail.ts +++ b/lib/apify/sendApifyWebhookEmail.ts @@ -1,6 +1,7 @@ import { sendEmailWithResend } from "@/lib/emails/sendEmail"; import { RECOUP_FROM_EMAIL } from "@/lib/const"; import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtml"; +import { extractPostsFromDatasetItems } from "@/lib/apify/digest/extractPostsFromDatasetItems"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; /** @@ -21,8 +22,12 @@ export async function sendApifyWebhookEmail( ) { if (!emails?.length || !newPostUrls.length) return null; + // The profile IS dataset item 0, so the shared extractor enriches the new + // URLs with caption/media/timestamp from latestPosts already in hand. const { subject, html } = renderScrapeDigestHtml({ - sections: [{ platform: "instagram", postUrls: newPostUrls }], + sections: [ + { platform: "instagram", posts: extractPostsFromDatasetItems("instagram", [profile], newPostUrls) }, + ], artistName: profile.fullName ?? profile.username ?? null, }); From 32c6e96d2c5ff26014589679580c7fc49b89bef5 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Thu, 9 Jul 2026 16:03:29 -0500 Subject: [PATCH 06/12] style: prettier pass on the digest template files Co-Authored-By: Claude Fable 5 --- .../__tests__/extractPostsFromDatasetItems.test.ts | 14 ++++++++++---- lib/apify/sendApifyWebhookEmail.ts | 5 ++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts b/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts index 97e68cb5b..c878204bd 100644 --- a/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts +++ b/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts @@ -27,7 +27,11 @@ const TIKTOK_ITEMS = [ createTimeISO: "2026-07-09T09:00:00.000Z", videoMeta: { coverUrl: "https://cdn.tt/1.jpg" }, }, - { webVideoUrl: "https://tiktok.com/@a/video/2", text: "known", createTimeISO: "2026-01-01T00:00:00.000Z" }, + { + webVideoUrl: "https://tiktok.com/@a/video/2", + text: "known", + createTimeISO: "2026-01-01T00:00:00.000Z", + }, ]; describe("extractPostsFromDatasetItems", () => { @@ -60,9 +64,11 @@ describe("extractPostsFromDatasetItems", () => { }); it("falls back to URL-only posts for platforms without an extractor", () => { - const posts = extractPostsFromDatasetItems("youtube", [{ some: "shape" }], [ - "https://youtube.com/watch?v=1", - ]); + const posts = extractPostsFromDatasetItems( + "youtube", + [{ some: "shape" }], + ["https://youtube.com/watch?v=1"], + ); expect(posts).toEqual([{ url: "https://youtube.com/watch?v=1" }]); }); diff --git a/lib/apify/sendApifyWebhookEmail.ts b/lib/apify/sendApifyWebhookEmail.ts index adb41a401..1f07ce8de 100644 --- a/lib/apify/sendApifyWebhookEmail.ts +++ b/lib/apify/sendApifyWebhookEmail.ts @@ -26,7 +26,10 @@ export async function sendApifyWebhookEmail( // URLs with caption/media/timestamp from latestPosts already in hand. const { subject, html } = renderScrapeDigestHtml({ sections: [ - { platform: "instagram", posts: extractPostsFromDatasetItems("instagram", [profile], newPostUrls) }, + { + platform: "instagram", + posts: extractPostsFromDatasetItems("instagram", [profile], newPostUrls), + }, ], artistName: profile.fullName ?? profile.username ?? null, }); From efab25a08febf1b6e353f200e16855467f752eb5 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Thu, 9 Jul 2026 16:36:54 -0500 Subject: [PATCH 07/12] feat(apify): digest addressed by artist name + per-post engagement stats + fixed chat CTA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three feedback items on the template: - The digest header/subject said 'Your artist' — the assembler never passed a name. getRunDigestSection now also extracts the profile display name from the dataset (IG fullName, TikTok authorMeta.nickName) and maybeSendScrapeDigest addresses the email with the first platform's name. - Post cards now carry compact engagement stats (12.3K likes · 678 comments · 1.2M views · shares) mapped from platform counts (IG likesCount/commentsCount/videoViewCount, TikTok diggCount/commentCount/ playCount/shareCount); omitted entirely when the scraper returns none. - The CTA now always points at https://chat.recoupable.dev — previously it derived from the deployment base URL, which on previews is the API deployment itself. Funnel-tracking landing page tracked as follow-up. Co-Authored-By: Claude Fable 5 --- .../extractArtistNameFromDatasetItems.test.ts | 27 ++++++++++++++ .../extractPostsFromDatasetItems.test.ts | 9 +++++ .../__tests__/maybeSendScrapeDigest.test.ts | 15 ++++++-- .../__tests__/renderScrapeDigestHtml.test.ts | 13 +++++++ .../extractArtistNameFromDatasetItems.ts | 26 ++++++++++++++ .../digest/extractPostsFromDatasetItems.ts | 26 +++++++++++++- lib/apify/digest/getRunDigestSection.ts | 26 +++++++++----- lib/apify/digest/maybeSendScrapeDigest.ts | 8 +++-- lib/apify/digest/renderScrapeDigestHtml.ts | 36 +++++++++++++++++-- 9 files changed, 167 insertions(+), 19 deletions(-) create mode 100644 lib/apify/digest/__tests__/extractArtistNameFromDatasetItems.test.ts create mode 100644 lib/apify/digest/extractArtistNameFromDatasetItems.ts diff --git a/lib/apify/digest/__tests__/extractArtistNameFromDatasetItems.test.ts b/lib/apify/digest/__tests__/extractArtistNameFromDatasetItems.test.ts new file mode 100644 index 000000000..9f8fa9314 --- /dev/null +++ b/lib/apify/digest/__tests__/extractArtistNameFromDatasetItems.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect } from "vitest"; +import { extractArtistNameFromDatasetItems } from "@/lib/apify/digest/extractArtistNameFromDatasetItems"; + +describe("extractArtistNameFromDatasetItems", () => { + it("uses the Instagram profile fullName, falling back to username", () => { + expect( + extractArtistNameFromDatasetItems("instagram", [{ fullName: "National Geographic" }]), + ).toBe("National Geographic"); + expect(extractArtistNameFromDatasetItems("instagram", [{ username: "natgeo" }])).toBe("natgeo"); + }); + + it("uses the TikTok author nickName, falling back to handle", () => { + expect( + extractArtistNameFromDatasetItems("tiktok", [ + { authorMeta: { name: "natgeo", nickName: "National Geographic" } }, + ]), + ).toBe("National Geographic"); + expect(extractArtistNameFromDatasetItems("tiktok", [{ authorMeta: { name: "natgeo" } }])).toBe( + "natgeo", + ); + }); + + it("returns null for unknown platforms or empty items", () => { + expect(extractArtistNameFromDatasetItems("youtube", [{ some: "shape" }])).toBeNull(); + expect(extractArtistNameFromDatasetItems("instagram", [])).toBeNull(); + }); +}); diff --git a/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts b/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts index c878204bd..3a3343506 100644 --- a/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts +++ b/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts @@ -9,6 +9,9 @@ const IG_ITEMS = [ caption: "cap one", displayUrl: "https://cdn.ig/new1.jpg", timestamp: "2026-07-08T12:00:00.000Z", + likesCount: 100, + commentsCount: 5, + videoViewCount: 9000, }, { url: "https://instagram.com/p/old", @@ -26,6 +29,10 @@ const TIKTOK_ITEMS = [ text: "tt caption", createTimeISO: "2026-07-09T09:00:00.000Z", videoMeta: { coverUrl: "https://cdn.tt/1.jpg" }, + diggCount: 200, + commentCount: 10, + playCount: 50000, + shareCount: 7, }, { webVideoUrl: "https://tiktok.com/@a/video/2", @@ -45,6 +52,7 @@ describe("extractPostsFromDatasetItems", () => { caption: "cap one", thumbnailUrl: "https://cdn.ig/new1.jpg", timestamp: "2026-07-08T12:00:00.000Z", + stats: { likes: 100, comments: 5, views: 9000 }, }, ]); }); @@ -59,6 +67,7 @@ describe("extractPostsFromDatasetItems", () => { caption: "tt caption", thumbnailUrl: "https://cdn.tt/1.jpg", timestamp: "2026-07-09T09:00:00.000Z", + stats: { likes: 200, comments: 10, views: 50000, shares: 7 }, }, ]); }); diff --git a/lib/apify/digest/__tests__/maybeSendScrapeDigest.test.ts b/lib/apify/digest/__tests__/maybeSendScrapeDigest.test.ts index 11bc146d7..0608e53d7 100644 --- a/lib/apify/digest/__tests__/maybeSendScrapeDigest.test.ts +++ b/lib/apify/digest/__tests__/maybeSendScrapeDigest.test.ts @@ -14,7 +14,11 @@ vi.mock("@/lib/apify/digest/getRunDigestSection", () => ({ // URL-only translation of a run row — enrichment is unit-tested separately getRunDigestSection: vi.fn(async (run: { platform: string; new_post_urls: string[] | null }) => run.new_post_urls?.length - ? { platform: run.platform, posts: run.new_post_urls.map((url: string) => ({ url })) } + ? { + platform: run.platform, + posts: run.new_post_urls.map((url: string) => ({ url })), + artistName: run.platform === "instagram" ? "National Geographic" : null, + } : null, ), })); @@ -59,10 +63,15 @@ describe("maybeSendScrapeDigest", () => { expect(sendScrapeDigestEmail).toHaveBeenCalledOnce(); const arg = vi.mocked(sendScrapeDigestEmail).mock.calls[0][0]; expect(arg.sections).toEqual([ - { platform: "instagram", posts: [{ url: "https://instagram.com/p/1" }] }, - { platform: "tiktok", posts: [{ url: "https://tiktok.com/v/2" }] }, + { + platform: "instagram", + posts: [{ url: "https://instagram.com/p/1" }], + artistName: "National Geographic", + }, + { platform: "tiktok", posts: [{ url: "https://tiktok.com/v/2" }], artistName: null }, ]); // x omitted — nothing new expect(arg.emails).toEqual(["owner@example.com"]); + expect(arg.artistName).toBe("National Geographic"); // digest addressed by artist, not "Your artist" }); it("sends nothing when the batch completes with zero new posts anywhere", async () => { diff --git a/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts b/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts index f16e1ee33..51ab39d60 100644 --- a/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts +++ b/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts @@ -10,6 +10,7 @@ const SECTIONS = [ caption: "Behind the scenes tour & more", thumbnailUrl: "https://cdn.example.com/thumb-abc.jpg", timestamp: "2026-07-08T12:00:00.000Z", + stats: { likes: 12345, comments: 678, views: 1200000 }, }, { url: "https://instagram.com/p/def", caption: null, thumbnailUrl: null, timestamp: null }, ], @@ -48,6 +49,13 @@ describe("renderScrapeDigestHtml", () => { expect(html).toContain("<b>tour</b> & more"); }); + it("renders compact engagement stats when available", () => { + const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + expect(html).toContain("12.3K likes"); + expect(html).toContain("678 comments"); + expect(html).toContain("1.2M views"); + }); + it("still renders a linked card when a post has no media or caption", () => { const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); expect(html).toContain('href="https://instagram.com/p/def"'); @@ -65,6 +73,11 @@ describe("renderScrapeDigestHtml", () => { expect(html).toContain("#e8e8e8"); }); + it("CTA always points at the chat app, never the API deployment", () => { + const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + expect(html).toContain('href="https://chat.recoupable.dev/?q='); + }); + it("never leaks internal vendor jargon to customers", () => { const { html, subject } = renderScrapeDigestHtml({ sections: SECTIONS, diff --git a/lib/apify/digest/extractArtistNameFromDatasetItems.ts b/lib/apify/digest/extractArtistNameFromDatasetItems.ts new file mode 100644 index 000000000..f41bd2589 --- /dev/null +++ b/lib/apify/digest/extractArtistNameFromDatasetItems.ts @@ -0,0 +1,26 @@ +const asRecord = (v: unknown): Record => + v && typeof v === "object" ? (v as Record) : {}; + +const str = (v: unknown): string | null => (typeof v === "string" && v ? v : null); + +/** + * Pulls the scraped profile's display name out of a platform's raw dataset + * items, so the digest can be addressed by artist instead of "Your artist" + * (chat#1855). Returns null for platforms without a known shape. + */ +export function extractArtistNameFromDatasetItems( + platform: string, + items: unknown[], +): string | null { + const first = asRecord(items[0]); + switch (platform.toLowerCase()) { + case "instagram": + return str(first.fullName) ?? str(first.username); + case "tiktok": { + const author = asRecord(first.authorMeta); + return str(author.nickName) ?? str(author.name); + } + default: + return null; + } +} diff --git a/lib/apify/digest/extractPostsFromDatasetItems.ts b/lib/apify/digest/extractPostsFromDatasetItems.ts index 8b6119879..dfcc55237 100644 --- a/lib/apify/digest/extractPostsFromDatasetItems.ts +++ b/lib/apify/digest/extractPostsFromDatasetItems.ts @@ -1,4 +1,7 @@ -import type { ScrapeDigestPost } from "@/lib/apify/digest/renderScrapeDigestHtml"; +import type { + ScrapeDigestPost, + ScrapeDigestPostStats, +} from "@/lib/apify/digest/renderScrapeDigestHtml"; type Extractor = (items: unknown[]) => Map; @@ -7,6 +10,14 @@ const asRecord = (v: unknown): Record => const str = (v: unknown): string | null => (typeof v === "string" && v ? v : null); +const num = (v: unknown): number | null => (typeof v === "number" && isFinite(v) ? v : null); + +/** Drops the stats object entirely when the scraper returned no counts. */ +const stats = (s: ScrapeDigestPostStats): ScrapeDigestPostStats | undefined => + Object.values(s).some(v => v != null) + ? Object.fromEntries(Object.entries(s).filter(([, v]) => v != null)) + : undefined; + /** Instagram profile-scraper dataset: posts ride on item[0].latestPosts. */ const instagram: Extractor = items => { const map = new Map(); @@ -15,11 +26,17 @@ const instagram: Extractor = items => { const post = asRecord(raw); const url = str(post.url); if (!url) continue; + const postStats = stats({ + likes: num(post.likesCount), + comments: num(post.commentsCount), + views: num(post.videoViewCount), + }); map.set(url, { url, caption: str(post.caption), thumbnailUrl: str(post.displayUrl), timestamp: str(post.timestamp), + ...(postStats && { stats: postStats }), }); } return map; @@ -32,11 +49,18 @@ const tiktok: Extractor = items => { const item = asRecord(raw); const url = str(item.webVideoUrl); if (!url) continue; + const videoStats = stats({ + likes: num(item.diggCount), + comments: num(item.commentCount), + views: num(item.playCount), + shares: num(item.shareCount), + }); map.set(url, { url, caption: str(item.text), thumbnailUrl: str(asRecord(item.videoMeta).coverUrl), timestamp: str(item.createTimeISO), + ...(videoStats && { stats: videoStats }), }); } return map; diff --git a/lib/apify/digest/getRunDigestSection.ts b/lib/apify/digest/getRunDigestSection.ts index 563653991..96b103bc8 100644 --- a/lib/apify/digest/getRunDigestSection.ts +++ b/lib/apify/digest/getRunDigestSection.ts @@ -1,20 +1,24 @@ import apifyClient from "@/lib/apify/client"; import { parseNewPostUrls } from "@/lib/apify/digest/parseNewPostUrls"; import { extractPostsFromDatasetItems } from "@/lib/apify/digest/extractPostsFromDatasetItems"; +import { extractArtistNameFromDatasetItems } from "@/lib/apify/digest/extractArtistNameFromDatasetItems"; import type { ScrapeDigestSection } from "@/lib/apify/digest/renderScrapeDigestHtml"; import type { Tables } from "@/types/database.types"; +/** A digest section plus the profile display name the platform reported. */ +export type RunDigestSection = ScrapeDigestSection & { artistName: string | null }; + /** * Builds one platform's digest section for a completed scrape run, enriching - * the stored genuinely-new URLs with caption/media/timestamp by re-reading - * the run's dataset from Apify (the source of truth for scrape content — - * only the URL diff is persisted, chat#1855). Enrichment must never block - * the digest: any failure degrades to URL-only posts. Returns null when the - * run found nothing new. + * the stored genuinely-new URLs with caption/media/stats and the profile's + * display name by re-reading the run's dataset from Apify (the source of + * truth for scrape content — only the URL diff is persisted, chat#1855). + * Enrichment must never block the digest: any failure degrades to URL-only + * posts. Returns null when the run found nothing new. */ export async function getRunDigestSection( run: Tables<"apify_scraper_runs">, -): Promise { +): Promise { const urls = parseNewPostUrls(run.new_post_urls); if (!urls.length) return null; const platform = run.platform ?? "other"; @@ -22,11 +26,15 @@ export async function getRunDigestSection( try { const runInfo = await apifyClient.run(run.run_id).get(); const datasetId = runInfo?.defaultDatasetId; - if (!datasetId) return { platform, posts: urls.map(url => ({ url })) }; + if (!datasetId) return { platform, posts: urls.map(url => ({ url })), artistName: null }; const { items } = await apifyClient.dataset(datasetId).listItems(); - return { platform, posts: extractPostsFromDatasetItems(platform, items, urls) }; + return { + platform, + posts: extractPostsFromDatasetItems(platform, items, urls), + artistName: extractArtistNameFromDatasetItems(platform, items), + }; } catch (error) { console.error("[WARN] digest enrichment failed; sending URL-only links:", error); - return { platform, posts: urls.map(url => ({ url })) }; + return { platform, posts: urls.map(url => ({ url })), artistName: null }; } } diff --git a/lib/apify/digest/maybeSendScrapeDigest.ts b/lib/apify/digest/maybeSendScrapeDigest.ts index 8efeb57c3..c68b51797 100644 --- a/lib/apify/digest/maybeSendScrapeDigest.ts +++ b/lib/apify/digest/maybeSendScrapeDigest.ts @@ -1,8 +1,8 @@ import { selectApifyScraperRuns } from "@/lib/supabase/apify_scraper_runs/selectApifyScraperRuns"; import { getRunDigestSection } from "@/lib/apify/digest/getRunDigestSection"; +import type { RunDigestSection } from "@/lib/apify/digest/getRunDigestSection"; import { getScrapeDigestRecipients } from "@/lib/apify/digest/getScrapeDigestRecipients"; import { sendScrapeDigestEmail } from "@/lib/apify/digest/sendScrapeDigestEmail"; -import type { ScrapeDigestSection } from "@/lib/apify/digest/renderScrapeDigestHtml"; /** * Batch-completion check for the one-digest-per-scrape design (chat#1855): @@ -18,12 +18,14 @@ export async function maybeSendScrapeDigest(batchId: string | null | undefined) if (!runs.length || runs.some(r => !r.completed_at)) return null; const sections = (await Promise.all(runs.map(getRunDigestSection))).filter( - (s): s is ScrapeDigestSection => s !== null, + (s): s is RunDigestSection => s !== null, ); if (!sections.length) return null; const emails = await getScrapeDigestRecipients( runs.map(r => r.social_id).filter((id): id is string => Boolean(id)), ); - return await sendScrapeDigestEmail({ emails, sections }); + // A batch is one artist's scrape — any platform's profile name addresses it. + const artistName = sections.map(s => s.artistName).find(Boolean) ?? null; + return await sendScrapeDigestEmail({ emails, sections, artistName }); } diff --git a/lib/apify/digest/renderScrapeDigestHtml.ts b/lib/apify/digest/renderScrapeDigestHtml.ts index 62339b03e..178a8598d 100644 --- a/lib/apify/digest/renderScrapeDigestHtml.ts +++ b/lib/apify/digest/renderScrapeDigestHtml.ts @@ -1,12 +1,24 @@ -import { getFrontendBaseUrl } from "@/lib/composio/getFrontendBaseUrl"; +export type ScrapeDigestPostStats = { + likes?: number | null; + comments?: number | null; + views?: number | null; + shares?: number | null; +}; export type ScrapeDigestPost = { url: string; caption?: string | null; thumbnailUrl?: string | null; timestamp?: string | null; + stats?: ScrapeDigestPostStats | null; }; +// Fixed chat-app URL — never derived from the current deployment: on preview +// deployments a derived base URL is the API preview itself, which is where +// customers must never land (funnel-tracking landing page tracked separately +// on chat#1855). +const CHAT_APP_URL = "https://chat.recoupable.dev"; + export type ScrapeDigestSection = { platform: string; posts: ScrapeDigestPost[] }; const PLATFORM_LABELS: Record = { @@ -41,6 +53,23 @@ function truncate(text: string, max: number): string { return text.length <= max ? text : `${text.slice(0, max - 1).trimEnd()}…`; } +/** Deterministic compact count: 678, 12.3K, 1.2M (trailing .0 stripped). */ +function formatCount(n: number): string { + if (n < 1000) return String(n); + const [value, unit] = n < 1_000_000 ? [n / 1000, "K"] : [n / 1_000_000, "M"]; + return `${value.toFixed(1).replace(/\.0$/, "")}${unit}`; +} + +/** "12.3K likes · 678 comments · 1.2M views" from whichever stats exist. */ +function formatStats(stats: ScrapeDigestPostStats): string { + const parts: string[] = []; + if (stats.likes != null) parts.push(`${formatCount(stats.likes)} likes`); + if (stats.comments != null) parts.push(`${formatCount(stats.comments)} comments`); + if (stats.views != null) parts.push(`${formatCount(stats.views)} views`); + if (stats.shares != null) parts.push(`${formatCount(stats.shares)} shares`); + return parts.join(" · "); +} + /** * Deterministic house-style renderer for the new-posts digest (chat#1855). * Replaces the per-send LLM body: identical input renders identical output, @@ -69,17 +98,18 @@ export function renderScrapeDigestHtml({ const href = post.url.startsWith("http") ? post.url : `https://${post.url}`; const caption = post.caption ? escapeHtml(truncate(post.caption, 110)) : ""; const date = post.timestamp ? formatDate(post.timestamp) : ""; + const stats = post.stats ? formatStats(post.stats) : ""; const thumbCell = post.thumbnailUrl ? `` : ""; - return `
    ${thumbCell}
    ${caption ? `

    ${caption}

    ` : ""}

    ${date ? `${date} · ` : ""}View post →

    `; + return `
    ${thumbCell}
    ${caption ? `

    ${caption}

    ` : ""}${stats ? `

    ${stats}

    ` : ""}

    ${date ? `${date} · ` : ""}View post →

    `; }) .join(""); return `

    ${label}

    ${cards}`; }) .join(""); - const chatUrl = `${getFrontendBaseUrl()}/?q=${encodeURIComponent("tell me about my artist's latest posts")}`; + const chatUrl = `${CHAT_APP_URL}/?q=${encodeURIComponent("tell me about my artist's latest posts")}`; const html = ` - +
    ` : ""; return `
    From f4f04e59e98989e818af0689c7efa32d287713a3 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Thu, 9 Jul 2026 16:48:42 -0500 Subject: [PATCH 08/12] feat(apify): Recoup logo in the email header + artist-named roster footer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Header is now a two-cell row with the brand icon top-right (hosted PNG — email clients don't render SVG) linking to recoupable.com. Footer names the artist: 'because {artist} is in your roster on Recoup', falling back to 'this artist' when no profile name resolved. Co-Authored-By: Claude Fable 5 --- .../digest/__tests__/renderScrapeDigestHtml.test.ts | 13 +++++++++++++ lib/apify/digest/renderScrapeDigestHtml.ts | 10 +++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts b/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts index 51ab39d60..50bac1578 100644 --- a/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts +++ b/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts @@ -78,6 +78,19 @@ describe("renderScrapeDigestHtml", () => { expect(html).toContain('href="https://chat.recoupable.dev/?q='); }); + it("shows the Recoup logo linking to recoupable.com", () => { + const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + expect(html).toContain('href="https://recoupable.com"'); + expect(html).toContain('src="https://chat.recoupable.dev/icon-with-background.png"'); + }); + + it("names the artist in the roster footer, with a safe fallback", () => { + const named = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); + expect(named.html).toContain("because Ashnikko is in your roster on Recoup"); + const anon = renderScrapeDigestHtml({ sections: SECTIONS }); + expect(anon.html).toContain("because this artist is in your roster on Recoup"); + }); + it("never leaks internal vendor jargon to customers", () => { const { html, subject } = renderScrapeDigestHtml({ sections: SECTIONS, diff --git a/lib/apify/digest/renderScrapeDigestHtml.ts b/lib/apify/digest/renderScrapeDigestHtml.ts index 178a8598d..d1e505a6d 100644 --- a/lib/apify/digest/renderScrapeDigestHtml.ts +++ b/lib/apify/digest/renderScrapeDigestHtml.ts @@ -18,6 +18,9 @@ export type ScrapeDigestPost = { // customers must never land (funnel-tracking landing page tracked separately // on chat#1855). const CHAT_APP_URL = "https://chat.recoupable.dev"; +const WEBSITE_URL = "https://recoupable.com"; +// Hosted on prod chat — email clients need an absolute PNG (no SVG support). +const LOGO_URL = "https://chat.recoupable.dev/icon-with-background.png"; export type ScrapeDigestSection = { platform: string; posts: ScrapeDigestPost[] }; @@ -113,12 +116,17 @@ export function renderScrapeDigestHtml({ const html = `
    + + + +

    New posts

    ${escapeHtml(who)}

    ${total} new post${total === 1 ? "" : "s"} since we last checked

    +
    Recoup
    ${sectionHtml}
    Ask Recoup about these posts →
    -

    You're receiving this because you work with this artist on Recoup.

    +

    You're receiving this because ${artistName ? escapeHtml(artistName) : "this artist"} is in your roster on Recoup.

    `; From f3bcf9011e5afd9a3bada039d7b0d6f08ebf236e Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Thu, 9 Jul 2026 17:16:52 -0500 Subject: [PATCH 09/12] refactor(apify): strict SRP/DRY pass on the digest renderer + delete orphaned pre-rename supabase files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback: - Every helper gets its own lib file: escapeHtml (lib/emails — repo had none), formatUtcDateLabel, truncateText, formatCompactCount, formatPostStats, getPlatformLabel, extractInstagramPosts, extractTiktokPosts, buildPostStats, asRecord/asStringOrNull/ asNumberOrNull coercers. renderScrapeDigestHtml and extractPostsFromDatasetItems now contain only their eponymous functions. - CHAT_APP_URL / WEBSITE_URL / RECOUP_LOGO_URL move to lib/const.ts as shared constants (no existing shared home found for platform labels — getPlatformLabel is the new one). - Deletes 4 orphaned pre-rename files under lib/supabase/apify_scraper_runs (completeApifyScraperRun, insertApifyScraperRuns, selectApifyScraperRunsByBatch, types): this branch predates #760's renames, so the merge of main added the renamed set alongside the branch's inherited old set — both landed with zero conflicts and zero references to the old files. Also confirms no legacy LLM email generation remains in lib/apify (generateText usage gone with the template rewrite). Co-Authored-By: Claude Fable 5 --- lib/apify/digest/asNumberOrNull.ts | 4 + lib/apify/digest/asRecord.ts | 4 + lib/apify/digest/asStringOrNull.ts | 4 + lib/apify/digest/buildPostStats.ts | 8 ++ lib/apify/digest/extractInstagramPosts.ts | 29 ++++++ .../digest/extractPostsFromDatasetItems.ts | 81 +++-------------- lib/apify/digest/extractTiktokPosts.ts | 29 ++++++ lib/apify/digest/formatCompactCount.ts | 6 ++ lib/apify/digest/formatPostStats.ts | 12 +++ lib/apify/digest/formatUtcDateLabel.ts | 8 ++ lib/apify/digest/getPlatformLabel.ts | 14 +++ lib/apify/digest/renderScrapeDigestHtml.ts | 88 ++++--------------- lib/apify/digest/truncateText.ts | 4 + lib/const.ts | 10 +++ lib/emails/escapeHtml.ts | 11 +++ .../completeApifyScraperRun.ts | 24 ----- .../insertApifyScraperRuns.ts | 19 ---- .../selectApifyScraperRunsByBatch.ts | 17 ---- lib/supabase/apify_scraper_runs/types.ts | 12 --- 19 files changed, 172 insertions(+), 212 deletions(-) create mode 100644 lib/apify/digest/asNumberOrNull.ts create mode 100644 lib/apify/digest/asRecord.ts create mode 100644 lib/apify/digest/asStringOrNull.ts create mode 100644 lib/apify/digest/buildPostStats.ts create mode 100644 lib/apify/digest/extractInstagramPosts.ts create mode 100644 lib/apify/digest/extractTiktokPosts.ts create mode 100644 lib/apify/digest/formatCompactCount.ts create mode 100644 lib/apify/digest/formatPostStats.ts create mode 100644 lib/apify/digest/formatUtcDateLabel.ts create mode 100644 lib/apify/digest/getPlatformLabel.ts create mode 100644 lib/apify/digest/truncateText.ts create mode 100644 lib/emails/escapeHtml.ts delete mode 100644 lib/supabase/apify_scraper_runs/completeApifyScraperRun.ts delete mode 100644 lib/supabase/apify_scraper_runs/insertApifyScraperRuns.ts delete mode 100644 lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch.ts delete mode 100644 lib/supabase/apify_scraper_runs/types.ts diff --git a/lib/apify/digest/asNumberOrNull.ts b/lib/apify/digest/asNumberOrNull.ts new file mode 100644 index 000000000..6b82b5b5a --- /dev/null +++ b/lib/apify/digest/asNumberOrNull.ts @@ -0,0 +1,4 @@ +/** Coerces an unknown dataset value to a finite number, or null. */ +export function asNumberOrNull(v: unknown): number | null { + return typeof v === "number" && isFinite(v) ? v : null; +} diff --git a/lib/apify/digest/asRecord.ts b/lib/apify/digest/asRecord.ts new file mode 100644 index 000000000..25d91f8fb --- /dev/null +++ b/lib/apify/digest/asRecord.ts @@ -0,0 +1,4 @@ +/** Coerces an unknown dataset value to a record, or an empty one. */ +export function asRecord(v: unknown): Record { + return v && typeof v === "object" ? (v as Record) : {}; +} diff --git a/lib/apify/digest/asStringOrNull.ts b/lib/apify/digest/asStringOrNull.ts new file mode 100644 index 000000000..34e2b9553 --- /dev/null +++ b/lib/apify/digest/asStringOrNull.ts @@ -0,0 +1,4 @@ +/** Coerces an unknown dataset value to a non-empty string, or null. */ +export function asStringOrNull(v: unknown): string | null { + return typeof v === "string" && v ? v : null; +} diff --git a/lib/apify/digest/buildPostStats.ts b/lib/apify/digest/buildPostStats.ts new file mode 100644 index 000000000..48ca14c88 --- /dev/null +++ b/lib/apify/digest/buildPostStats.ts @@ -0,0 +1,8 @@ +import type { ScrapeDigestPostStats } from "@/lib/apify/digest/renderScrapeDigestHtml"; + +/** Prunes null counts; returns undefined when the scraper reported none. */ +export function buildPostStats(stats: ScrapeDigestPostStats): ScrapeDigestPostStats | undefined { + return Object.values(stats).some(v => v != null) + ? Object.fromEntries(Object.entries(stats).filter(([, v]) => v != null)) + : undefined; +} diff --git a/lib/apify/digest/extractInstagramPosts.ts b/lib/apify/digest/extractInstagramPosts.ts new file mode 100644 index 000000000..7ae7fca01 --- /dev/null +++ b/lib/apify/digest/extractInstagramPosts.ts @@ -0,0 +1,29 @@ +import { asRecord } from "@/lib/apify/digest/asRecord"; +import { asStringOrNull } from "@/lib/apify/digest/asStringOrNull"; +import { asNumberOrNull } from "@/lib/apify/digest/asNumberOrNull"; +import { buildPostStats } from "@/lib/apify/digest/buildPostStats"; +import type { ScrapeDigestPost } from "@/lib/apify/digest/renderScrapeDigestHtml"; + +/** Instagram profile-scraper dataset: posts ride on item[0].latestPosts. */ +export function extractInstagramPosts(items: unknown[]): Map { + const map = new Map(); + const latest = asRecord(items[0]).latestPosts; + for (const raw of Array.isArray(latest) ? latest : []) { + const post = asRecord(raw); + const url = asStringOrNull(post.url); + if (!url) continue; + const stats = buildPostStats({ + likes: asNumberOrNull(post.likesCount), + comments: asNumberOrNull(post.commentsCount), + views: asNumberOrNull(post.videoViewCount), + }); + map.set(url, { + url, + caption: asStringOrNull(post.caption), + thumbnailUrl: asStringOrNull(post.displayUrl), + timestamp: asStringOrNull(post.timestamp), + ...(stats && { stats }), + }); + } + return map; +} diff --git a/lib/apify/digest/extractPostsFromDatasetItems.ts b/lib/apify/digest/extractPostsFromDatasetItems.ts index dfcc55237..133b191a4 100644 --- a/lib/apify/digest/extractPostsFromDatasetItems.ts +++ b/lib/apify/digest/extractPostsFromDatasetItems.ts @@ -1,78 +1,17 @@ -import type { - ScrapeDigestPost, - ScrapeDigestPostStats, -} from "@/lib/apify/digest/renderScrapeDigestHtml"; +import { extractInstagramPosts } from "@/lib/apify/digest/extractInstagramPosts"; +import { extractTiktokPosts } from "@/lib/apify/digest/extractTiktokPosts"; +import type { ScrapeDigestPost } from "@/lib/apify/digest/renderScrapeDigestHtml"; -type Extractor = (items: unknown[]) => Map; - -const asRecord = (v: unknown): Record => - v && typeof v === "object" ? (v as Record) : {}; - -const str = (v: unknown): string | null => (typeof v === "string" && v ? v : null); - -const num = (v: unknown): number | null => (typeof v === "number" && isFinite(v) ? v : null); - -/** Drops the stats object entirely when the scraper returned no counts. */ -const stats = (s: ScrapeDigestPostStats): ScrapeDigestPostStats | undefined => - Object.values(s).some(v => v != null) - ? Object.fromEntries(Object.entries(s).filter(([, v]) => v != null)) - : undefined; - -/** Instagram profile-scraper dataset: posts ride on item[0].latestPosts. */ -const instagram: Extractor = items => { - const map = new Map(); - const latest = asRecord(items[0]).latestPosts; - for (const raw of Array.isArray(latest) ? latest : []) { - const post = asRecord(raw); - const url = str(post.url); - if (!url) continue; - const postStats = stats({ - likes: num(post.likesCount), - comments: num(post.commentsCount), - views: num(post.videoViewCount), - }); - map.set(url, { - url, - caption: str(post.caption), - thumbnailUrl: str(post.displayUrl), - timestamp: str(post.timestamp), - ...(postStats && { stats: postStats }), - }); - } - return map; -}; - -/** TikTok scraper dataset: one item per video. */ -const tiktok: Extractor = items => { - const map = new Map(); - for (const raw of items) { - const item = asRecord(raw); - const url = str(item.webVideoUrl); - if (!url) continue; - const videoStats = stats({ - likes: num(item.diggCount), - comments: num(item.commentCount), - views: num(item.playCount), - shares: num(item.shareCount), - }); - map.set(url, { - url, - caption: str(item.text), - thumbnailUrl: str(asRecord(item.videoMeta).coverUrl), - timestamp: str(item.createTimeISO), - ...(videoStats && { stats: videoStats }), - }); - } - return map; +const EXTRACTORS: Record Map> = { + instagram: extractInstagramPosts, + tiktok: extractTiktokPosts, }; -const EXTRACTORS: Record = { instagram, tiktok }; - /** - * Builds rich digest posts (caption, media, timestamp) for a platform's - * genuinely-new post URLs from its raw scraper dataset items. Platforms - * without an extractor — and URLs missing from the items — degrade to - * URL-only posts, so enrichment can never lose a link (chat#1855). + * Builds rich digest posts (caption, media, stats, timestamp) for a + * platform's genuinely-new post URLs from its raw scraper dataset items. + * Platforms without an extractor — and URLs missing from the items — + * degrade to URL-only posts, so enrichment can never lose a link (chat#1855). */ export function extractPostsFromDatasetItems( platform: string, diff --git a/lib/apify/digest/extractTiktokPosts.ts b/lib/apify/digest/extractTiktokPosts.ts new file mode 100644 index 000000000..a3346b228 --- /dev/null +++ b/lib/apify/digest/extractTiktokPosts.ts @@ -0,0 +1,29 @@ +import { asRecord } from "@/lib/apify/digest/asRecord"; +import { asStringOrNull } from "@/lib/apify/digest/asStringOrNull"; +import { asNumberOrNull } from "@/lib/apify/digest/asNumberOrNull"; +import { buildPostStats } from "@/lib/apify/digest/buildPostStats"; +import type { ScrapeDigestPost } from "@/lib/apify/digest/renderScrapeDigestHtml"; + +/** TikTok scraper dataset: one item per video. */ +export function extractTiktokPosts(items: unknown[]): Map { + const map = new Map(); + for (const raw of items) { + const item = asRecord(raw); + const url = asStringOrNull(item.webVideoUrl); + if (!url) continue; + const stats = buildPostStats({ + likes: asNumberOrNull(item.diggCount), + comments: asNumberOrNull(item.commentCount), + views: asNumberOrNull(item.playCount), + shares: asNumberOrNull(item.shareCount), + }); + map.set(url, { + url, + caption: asStringOrNull(item.text), + thumbnailUrl: asStringOrNull(asRecord(item.videoMeta).coverUrl), + timestamp: asStringOrNull(item.createTimeISO), + ...(stats && { stats }), + }); + } + return map; +} diff --git a/lib/apify/digest/formatCompactCount.ts b/lib/apify/digest/formatCompactCount.ts new file mode 100644 index 000000000..4194fba3a --- /dev/null +++ b/lib/apify/digest/formatCompactCount.ts @@ -0,0 +1,6 @@ +/** Deterministic compact count: 678, 12.3K, 1.2M (trailing .0 stripped). */ +export function formatCompactCount(n: number): string { + if (n < 1000) return String(n); + const [value, unit] = n < 1_000_000 ? [n / 1000, "K"] : [n / 1_000_000, "M"]; + return `${value.toFixed(1).replace(/\.0$/, "")}${unit}`; +} diff --git a/lib/apify/digest/formatPostStats.ts b/lib/apify/digest/formatPostStats.ts new file mode 100644 index 000000000..09e5a1f24 --- /dev/null +++ b/lib/apify/digest/formatPostStats.ts @@ -0,0 +1,12 @@ +import { formatCompactCount } from "@/lib/apify/digest/formatCompactCount"; +import type { ScrapeDigestPostStats } from "@/lib/apify/digest/renderScrapeDigestHtml"; + +/** "12.3K likes · 678 comments · 1.2M views" from whichever stats exist. */ +export function formatPostStats(stats: ScrapeDigestPostStats): string { + const parts: string[] = []; + if (stats.likes != null) parts.push(`${formatCompactCount(stats.likes)} likes`); + if (stats.comments != null) parts.push(`${formatCompactCount(stats.comments)} comments`); + if (stats.views != null) parts.push(`${formatCompactCount(stats.views)} views`); + if (stats.shares != null) parts.push(`${formatCompactCount(stats.shares)} shares`); + return parts.join(" · "); +} diff --git a/lib/apify/digest/formatUtcDateLabel.ts b/lib/apify/digest/formatUtcDateLabel.ts new file mode 100644 index 000000000..14f7100a1 --- /dev/null +++ b/lib/apify/digest/formatUtcDateLabel.ts @@ -0,0 +1,8 @@ +const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + +/** Deterministic "Jul 8" date label (UTC — no locale/timezone variance). */ +export function formatUtcDateLabel(iso: string): string { + const d = new Date(iso); + if (isNaN(d.getTime())) return ""; + return `${MONTHS[d.getUTCMonth()]} ${d.getUTCDate()}`; +} diff --git a/lib/apify/digest/getPlatformLabel.ts b/lib/apify/digest/getPlatformLabel.ts new file mode 100644 index 000000000..d83196a6c --- /dev/null +++ b/lib/apify/digest/getPlatformLabel.ts @@ -0,0 +1,14 @@ +const PLATFORM_LABELS: Record = { + instagram: "Instagram", + tiktok: "TikTok", + x: "X", + twitter: "X", + youtube: "YouTube", + facebook: "Facebook", + threads: "Threads", +}; + +/** Customer-facing label for a scraper platform key. */ +export function getPlatformLabel(platform: string): string { + return PLATFORM_LABELS[platform.toLowerCase()] ?? platform; +} diff --git a/lib/apify/digest/renderScrapeDigestHtml.ts b/lib/apify/digest/renderScrapeDigestHtml.ts index d1e505a6d..0745b1f90 100644 --- a/lib/apify/digest/renderScrapeDigestHtml.ts +++ b/lib/apify/digest/renderScrapeDigestHtml.ts @@ -1,3 +1,10 @@ +import { CHAT_APP_URL, RECOUP_LOGO_URL, WEBSITE_URL } from "@/lib/const"; +import { escapeHtml } from "@/lib/emails/escapeHtml"; +import { formatUtcDateLabel } from "@/lib/apify/digest/formatUtcDateLabel"; +import { formatPostStats } from "@/lib/apify/digest/formatPostStats"; +import { getPlatformLabel } from "@/lib/apify/digest/getPlatformLabel"; +import { truncateText } from "@/lib/apify/digest/truncateText"; + export type ScrapeDigestPostStats = { likes?: number | null; comments?: number | null; @@ -13,74 +20,18 @@ export type ScrapeDigestPost = { stats?: ScrapeDigestPostStats | null; }; -// Fixed chat-app URL — never derived from the current deployment: on preview -// deployments a derived base URL is the API preview itself, which is where -// customers must never land (funnel-tracking landing page tracked separately -// on chat#1855). -const CHAT_APP_URL = "https://chat.recoupable.dev"; -const WEBSITE_URL = "https://recoupable.com"; -// Hosted on prod chat — email clients need an absolute PNG (no SVG support). -const LOGO_URL = "https://chat.recoupable.dev/icon-with-background.png"; - export type ScrapeDigestSection = { platform: string; posts: ScrapeDigestPost[] }; -const PLATFORM_LABELS: Record = { - instagram: "Instagram", - tiktok: "TikTok", - x: "X", - twitter: "X", - youtube: "YouTube", - facebook: "Facebook", - threads: "Threads", -}; - -const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; - -/** Escapes scraped text so captions can never inject markup into the email. */ -function escapeHtml(text: string): string { - return text - .replace(/&/g, "&") - .replace(//g, ">") - .replace(/"/g, """); -} - -/** Deterministic "Jul 8" date label (UTC — no locale/timezone variance). */ -function formatDate(iso: string): string { - const d = new Date(iso); - if (isNaN(d.getTime())) return ""; - return `${MONTHS[d.getUTCMonth()]} ${d.getUTCDate()}`; -} - -function truncate(text: string, max: number): string { - return text.length <= max ? text : `${text.slice(0, max - 1).trimEnd()}…`; -} - -/** Deterministic compact count: 678, 12.3K, 1.2M (trailing .0 stripped). */ -function formatCount(n: number): string { - if (n < 1000) return String(n); - const [value, unit] = n < 1_000_000 ? [n / 1000, "K"] : [n / 1_000_000, "M"]; - return `${value.toFixed(1).replace(/\.0$/, "")}${unit}`; -} - -/** "12.3K likes · 678 comments · 1.2M views" from whichever stats exist. */ -function formatStats(stats: ScrapeDigestPostStats): string { - const parts: string[] = []; - if (stats.likes != null) parts.push(`${formatCount(stats.likes)} likes`); - if (stats.comments != null) parts.push(`${formatCount(stats.comments)} comments`); - if (stats.views != null) parts.push(`${formatCount(stats.views)} views`); - if (stats.shares != null) parts.push(`${formatCount(stats.shares)} shares`); - return parts.join(" · "); -} - /** * Deterministic house-style renderer for the new-posts digest (chat#1855). * Replaces the per-send LLM body: identical input renders identical output, - * every genuinely-new post is a linked card with its media and caption, and - * internal vendor jargon never reaches customers. Styling follows DESIGN.md — - * achromatic chrome (#0a0a0a on #ffffff, #e8e8e8 borders, #6b6b6b muted); - * color comes from the post media, not the chrome. Email-safe: tables + - * inline styles only, system font stack. + * every genuinely-new post is a linked card with its media, caption, and + * engagement stats, and internal vendor jargon never reaches customers. + * Styling follows DESIGN.md — achromatic chrome (#0a0a0a on #ffffff, #e8e8e8 + * borders, #6b6b6b muted); color comes from the post media, not the chrome. + * Email-safe: tables + inline styles only, system font stack. The CTA is the + * fixed CHAT_APP_URL — never a derived deployment URL (previews would point + * customers at the API deployment itself). */ export function renderScrapeDigestHtml({ sections, @@ -95,20 +46,19 @@ export function renderScrapeDigestHtml({ const sectionHtml = sections .map(section => { - const label = PLATFORM_LABELS[section.platform.toLowerCase()] ?? section.platform; const cards = section.posts .map(post => { const href = post.url.startsWith("http") ? post.url : `https://${post.url}`; - const caption = post.caption ? escapeHtml(truncate(post.caption, 110)) : ""; - const date = post.timestamp ? formatDate(post.timestamp) : ""; - const stats = post.stats ? formatStats(post.stats) : ""; + const caption = post.caption ? escapeHtml(truncateText(post.caption, 110)) : ""; + const date = post.timestamp ? formatUtcDateLabel(post.timestamp) : ""; + const stats = post.stats ? formatPostStats(post.stats) : ""; const thumbCell = post.thumbnailUrl ? `
    ${thumbCell}
    ${caption ? `

    ${caption}

    ` : ""}${stats ? `

    ${stats}

    ` : ""}

    ${date ? `${date} · ` : ""}View post →

    `; }) .join(""); - return `

    ${label}

    ${cards}`; + return `

    ${getPlatformLabel(section.platform)}

    ${cards}`; }) .join(""); @@ -122,7 +72,7 @@ export function renderScrapeDigestHtml({

    ${escapeHtml(who)}

    ${total} new post${total === 1 ? "" : "s"} since we last checked

    RecoupRecoup
    ${sectionHtml}
    Ask Recoup about these posts →
    diff --git a/lib/apify/digest/truncateText.ts b/lib/apify/digest/truncateText.ts new file mode 100644 index 000000000..13c369860 --- /dev/null +++ b/lib/apify/digest/truncateText.ts @@ -0,0 +1,4 @@ +/** Truncates to `max` characters with a trailing ellipsis. */ +export function truncateText(text: string, max: number): string { + return text.length <= max ? text : `${text.slice(0, max - 1).trimEnd()}…`; +} diff --git a/lib/const.ts b/lib/const.ts index 5b178dd05..4a05312c9 100644 --- a/lib/const.ts +++ b/lib/const.ts @@ -15,6 +15,16 @@ export const PRIVY_PROJECT_SECRET = process.env.PRIVY_PROJECT_SECRET; /** Base URL for the public API documentation site */ export const DOCS_BASE_URL = "https://docs.recoupable.dev"; +/** Public marketing site. */ +export const WEBSITE_URL = "https://recoupable.com"; + +/** Chat app — customer-facing emails must always link here, never a derived + * deployment URL (previews would point at the API deployment itself). */ +export const CHAT_APP_URL = "https://chat.recoupable.dev"; + +/** Brand icon hosted on prod chat (email clients need an absolute PNG). */ +export const RECOUP_LOGO_URL = "https://chat.recoupable.dev/icon-with-background.png"; + /** Domain for receiving inbound emails (e.g., support@recoupable.dev) */ export const INBOUND_EMAIL_DOMAIN = "@recoupable.dev"; diff --git a/lib/emails/escapeHtml.ts b/lib/emails/escapeHtml.ts new file mode 100644 index 000000000..c0d8bf6d3 --- /dev/null +++ b/lib/emails/escapeHtml.ts @@ -0,0 +1,11 @@ +/** + * Escapes text for safe interpolation into email/notification HTML, so + * user- or scraper-supplied content can never inject markup. + */ +export function escapeHtml(text: string): string { + return text + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} diff --git a/lib/supabase/apify_scraper_runs/completeApifyScraperRun.ts b/lib/supabase/apify_scraper_runs/completeApifyScraperRun.ts deleted file mode 100644 index fa8f346c3..000000000 --- a/lib/supabase/apify_scraper_runs/completeApifyScraperRun.ts +++ /dev/null @@ -1,24 +0,0 @@ -import supabase from "@/lib/supabase/serverClient"; -import type { ApifyScraperRunRow } from "@/lib/supabase/apify_scraper_runs/types"; - -/** - * Marks a scrape run's results as processed and records which post URLs were - * genuinely new. Returns the updated row (carrying batch_id) or null when the - * run was never registered (legacy/non-batch runs). - */ -export async function completeApifyScraperRun( - runId: string, - newPostUrls: string[], -): Promise { - const { data, error } = await supabase - .from("apify_scraper_runs" as never) - .update({ completed_at: new Date().toISOString(), new_post_urls: newPostUrls } as never) - .eq("run_id", runId) - .select() - .maybeSingle(); - if (error) { - console.error("[ERROR] completeApifyScraperRun:", error); - return null; - } - return (data as ApifyScraperRunRow | null) ?? null; -} diff --git a/lib/supabase/apify_scraper_runs/insertApifyScraperRuns.ts b/lib/supabase/apify_scraper_runs/insertApifyScraperRuns.ts deleted file mode 100644 index 35a162908..000000000 --- a/lib/supabase/apify_scraper_runs/insertApifyScraperRuns.ts +++ /dev/null @@ -1,19 +0,0 @@ -import supabase from "@/lib/supabase/serverClient"; -import type { ApifyScraperRunRow } from "@/lib/supabase/apify_scraper_runs/types"; - -/** - * Records scrape runs at start time so per-platform webhook completions can - * find their digest-batch siblings (recoupable/chat#1855). - * - * @param runs - One row per started Apify run (run_id, account_id, batch_id, …). - */ -export async function insertApifyScraperRuns( - runs: Pick[], -) { - if (!runs.length) return { data: null, error: null }; - const { data, error } = await supabase - .from("apify_scraper_runs" as never) - .upsert(runs as never[], { onConflict: "run_id", ignoreDuplicates: true }); - if (error) console.error("[ERROR] insertApifyScraperRuns:", error); - return { data, error }; -} diff --git a/lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch.ts b/lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch.ts deleted file mode 100644 index 0dd9457e6..000000000 --- a/lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch.ts +++ /dev/null @@ -1,17 +0,0 @@ -import supabase from "@/lib/supabase/serverClient"; -import type { ApifyScraperRunRow } from "@/lib/supabase/apify_scraper_runs/types"; - -/** Returns every scrape run registered under a digest batch. */ -export async function selectApifyScraperRunsByBatch( - batchId: string, -): Promise { - const { data, error } = await supabase - .from("apify_scraper_runs" as never) - .select("*") - .eq("batch_id", batchId); - if (error) { - console.error("[ERROR] selectApifyScraperRunsByBatch:", error); - return []; - } - return (data as ApifyScraperRunRow[] | null) ?? []; -} diff --git a/lib/supabase/apify_scraper_runs/types.ts b/lib/supabase/apify_scraper_runs/types.ts deleted file mode 100644 index faa9d00cf..000000000 --- a/lib/supabase/apify_scraper_runs/types.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** apify_scraper_runs row incl. the digest-batch columns added in - * recoupable/database#41 (not yet in generated database.types). */ -export type ApifyScraperRunRow = { - run_id: string; - account_id: string; - social_id: string | null; - platform: string | null; - batch_id: string | null; - completed_at: string | null; - new_post_urls: string[] | null; - created_at?: string; -}; From 8dc8ba3f0774d6c15c203f60202764d6cbb67628 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Thu, 9 Jul 2026 17:31:48 -0500 Subject: [PATCH 10/12] feat(apify): X/Twitter digest extractor + LinkedIn platform label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback: Twitter's handler already feeds the digest (filterNewPostUrls -> newPostUrls) but had no extractor, so its sections degraded to URL-only links. extractTwitterPosts maps apidojo tweet items (fullText, extendedEntities media, createdAt via toIsoDate, like/reply/view/retweet counts), registered under both 'twitter' and 'x'. Artist-name extraction gains the same aliases (author.name ?? userName). LinkedIn added to platform labels. YouTube and LinkedIn deliberately have no extractors: their results handlers persist only the social row (no posts, no newPostUrls), so they never contribute digest sections today — extractors would be dead code until post persistence ships for them (chat#1833 territory). Co-Authored-By: Claude Fable 5 --- .../extractArtistNameFromDatasetItems.test.ts | 13 +++++++ .../extractPostsFromDatasetItems.test.ts | 35 +++++++++++++++++++ .../__tests__/renderScrapeDigestHtml.test.ts | 8 +++++ .../extractArtistNameFromDatasetItems.ts | 5 +++ .../digest/extractPostsFromDatasetItems.ts | 3 ++ lib/apify/digest/extractTwitterPosts.ts | 32 +++++++++++++++++ lib/apify/digest/getPlatformLabel.ts | 1 + 7 files changed, 97 insertions(+) create mode 100644 lib/apify/digest/extractTwitterPosts.ts diff --git a/lib/apify/digest/__tests__/extractArtistNameFromDatasetItems.test.ts b/lib/apify/digest/__tests__/extractArtistNameFromDatasetItems.test.ts index 9f8fa9314..ab3d1881c 100644 --- a/lib/apify/digest/__tests__/extractArtistNameFromDatasetItems.test.ts +++ b/lib/apify/digest/__tests__/extractArtistNameFromDatasetItems.test.ts @@ -25,3 +25,16 @@ describe("extractArtistNameFromDatasetItems", () => { expect(extractArtistNameFromDatasetItems("instagram", [])).toBeNull(); }); }); + +describe("extractArtistNameFromDatasetItems — twitter", () => { + it("uses the tweet author's display name, falling back to handle", () => { + expect( + extractArtistNameFromDatasetItems("twitter", [ + { author: { name: "Sweetman", userName: "sweetman_eth" } }, + ]), + ).toBe("Sweetman"); + expect(extractArtistNameFromDatasetItems("x", [{ author: { userName: "sweetman_eth" } }])).toBe( + "sweetman_eth", + ); + }); +}); diff --git a/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts b/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts index 3a3343506..60e44e9fc 100644 --- a/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts +++ b/lib/apify/digest/__tests__/extractPostsFromDatasetItems.test.ts @@ -90,3 +90,38 @@ describe("extractPostsFromDatasetItems", () => { expect(posts[1]).toEqual({ url: "https://instagram.com/p/not-in-items" }); }); }); + +const TWEET_ITEMS = [ + { + url: "https://x.com/a/status/1", + fullText: "New single out on all platforms", + createdAt: "Wed Jul 08 19:43:50 +0000 2026", + likeCount: 5, + replyCount: 1, + retweetCount: 2, + viewCount: 748, + extendedEntities: { media: [{ media_url_https: "https://pbs.twimg.com/media/x.jpg" }] }, + }, +]; + +describe("extractPostsFromDatasetItems — twitter", () => { + it("maps tweets limited to the new URLs, with media and stats", () => { + const posts = extractPostsFromDatasetItems("twitter", TWEET_ITEMS, [ + "https://x.com/a/status/1", + ]); + expect(posts).toEqual([ + { + url: "https://x.com/a/status/1", + caption: "New single out on all platforms", + thumbnailUrl: "https://pbs.twimg.com/media/x.jpg", + timestamp: "2026-07-08T19:43:50.000Z", + stats: { likes: 5, comments: 1, views: 748, shares: 2 }, + }, + ]); + }); + + it("maps the x platform alias identically", () => { + const posts = extractPostsFromDatasetItems("x", TWEET_ITEMS, ["https://x.com/a/status/1"]); + expect(posts[0].caption).toBe("New single out on all platforms"); + }); +}); diff --git a/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts b/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts index 50bac1578..de1083d38 100644 --- a/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts +++ b/lib/apify/digest/__tests__/renderScrapeDigestHtml.test.ts @@ -36,6 +36,14 @@ describe("renderScrapeDigestHtml", () => { expect(html).toContain("TikTok"); }); + it("labels every wired platform, including LinkedIn", () => { + const { html } = renderScrapeDigestHtml({ + sections: [{ platform: "linkedin", posts: [{ url: "https://linkedin.com/posts/x" }] }], + artistName: "A", + }); + expect(html).toContain("LinkedIn"); + }); + it("renders post media and caption excerpts when available", () => { const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); expect(html).toContain('src="https://cdn.example.com/thumb-abc.jpg"'); diff --git a/lib/apify/digest/extractArtistNameFromDatasetItems.ts b/lib/apify/digest/extractArtistNameFromDatasetItems.ts index f41bd2589..1d3eddf55 100644 --- a/lib/apify/digest/extractArtistNameFromDatasetItems.ts +++ b/lib/apify/digest/extractArtistNameFromDatasetItems.ts @@ -20,6 +20,11 @@ export function extractArtistNameFromDatasetItems( const author = asRecord(first.authorMeta); return str(author.nickName) ?? str(author.name); } + case "twitter": + case "x": { + const author = asRecord(first.author); + return str(author.name) ?? str(author.userName); + } default: return null; } diff --git a/lib/apify/digest/extractPostsFromDatasetItems.ts b/lib/apify/digest/extractPostsFromDatasetItems.ts index 133b191a4..2ed8793d7 100644 --- a/lib/apify/digest/extractPostsFromDatasetItems.ts +++ b/lib/apify/digest/extractPostsFromDatasetItems.ts @@ -1,10 +1,13 @@ import { extractInstagramPosts } from "@/lib/apify/digest/extractInstagramPosts"; import { extractTiktokPosts } from "@/lib/apify/digest/extractTiktokPosts"; +import { extractTwitterPosts } from "@/lib/apify/digest/extractTwitterPosts"; import type { ScrapeDigestPost } from "@/lib/apify/digest/renderScrapeDigestHtml"; const EXTRACTORS: Record Map> = { instagram: extractInstagramPosts, tiktok: extractTiktokPosts, + twitter: extractTwitterPosts, + x: extractTwitterPosts, }; /** diff --git a/lib/apify/digest/extractTwitterPosts.ts b/lib/apify/digest/extractTwitterPosts.ts new file mode 100644 index 000000000..e9320f340 --- /dev/null +++ b/lib/apify/digest/extractTwitterPosts.ts @@ -0,0 +1,32 @@ +import { asRecord } from "@/lib/apify/digest/asRecord"; +import { asStringOrNull } from "@/lib/apify/digest/asStringOrNull"; +import { asNumberOrNull } from "@/lib/apify/digest/asNumberOrNull"; +import { buildPostStats } from "@/lib/apify/digest/buildPostStats"; +import { toIsoDate } from "@/lib/apify/toIsoDate"; +import type { ScrapeDigestPost } from "@/lib/apify/digest/renderScrapeDigestHtml"; + +/** X/Twitter scraper dataset (apidojo): one item per tweet. */ +export function extractTwitterPosts(items: unknown[]): Map { + const map = new Map(); + for (const raw of items) { + const item = asRecord(raw); + const url = asStringOrNull(item.url); + if (!url) continue; + const media = asRecord(item.extendedEntities).media; + const firstMedia = asRecord(Array.isArray(media) ? media[0] : undefined); + const stats = buildPostStats({ + likes: asNumberOrNull(item.likeCount), + comments: asNumberOrNull(item.replyCount), + views: asNumberOrNull(item.viewCount), + shares: asNumberOrNull(item.retweetCount), + }); + map.set(url, { + url, + caption: asStringOrNull(item.fullText) ?? asStringOrNull(item.text), + thumbnailUrl: asStringOrNull(firstMedia.media_url_https), + timestamp: toIsoDate(asStringOrNull(item.createdAt) ?? undefined) ?? null, + ...(stats && { stats }), + }); + } + return map; +} diff --git a/lib/apify/digest/getPlatformLabel.ts b/lib/apify/digest/getPlatformLabel.ts index d83196a6c..48f0a51f3 100644 --- a/lib/apify/digest/getPlatformLabel.ts +++ b/lib/apify/digest/getPlatformLabel.ts @@ -4,6 +4,7 @@ const PLATFORM_LABELS: Record = { x: "X", twitter: "X", youtube: "YouTube", + linkedin: "LinkedIn", facebook: "Facebook", threads: "Threads", }; From 6c4c8a67c4b50b08a329033b5dde1eb7eabaf25c Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Thu, 9 Jul 2026 17:47:14 -0500 Subject: [PATCH 11/12] =?UTF-8?q?fix(apify):=20only=20persist=20and=20repo?= =?UTF-8?q?rt=20the=20artist's=20own=20tweets=20=E2=80=94=20no=20retweets?= =?UTF-8?q?=20or=20replies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feedback from a real-account digest test: an all-retweet X section reported hundreds of likes the artist never earned (retweet items carry the ORIGINAL author's stats). isOriginalTweet keeps originals and quote tweets (the artist's own words and metrics), drops retweets and replies, applied at the persistence layer in handleTwitterProfileScraperResults so both stored posts and digest sections stay accurate. Items without flags pass (defensive default on schema drift). Co-Authored-By: Claude Fable 5 --- ...handleTwitterProfileScraperResults.test.ts | 18 +++++++++++++++++ .../twitter/__tests__/isOriginalTweet.test.ts | 20 +++++++++++++++++++ .../handleTwitterProfileScraperResults.ts | 10 +++++++++- lib/apify/twitter/isOriginalTweet.ts | 14 +++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 lib/apify/twitter/__tests__/isOriginalTweet.test.ts create mode 100644 lib/apify/twitter/isOriginalTweet.ts diff --git a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts index 0497e058b..3d53e946f 100644 --- a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts +++ b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts @@ -76,4 +76,22 @@ describe("handleTwitterProfileScraperResults", () => { expect(upsertSocials).not.toHaveBeenCalled(); expect(persistPostsForSocial).not.toHaveBeenCalled(); }); + + it("excludes retweets and replies from persisted posts and the newness diff", async () => { + listItems.mockResolvedValue({ + items: [ + { ...REAL_ITEM, url: "https://x.com/a/status/1" }, + { ...REAL_ITEM, url: "https://x.com/other/status/2", isRetweet: true }, + { ...REAL_ITEM, url: "https://x.com/a/status/3", isReply: true }, + ], + }); + await handleTwitterProfileScraperResults(payload as never); + expect(persistPostsForSocial).toHaveBeenCalledWith( + expect.objectContaining({ + postRows: [ + { post_url: "https://x.com/a/status/1", updated_at: "2026-07-02T17:21:21.000Z" }, + ], + }), + ); + }); }); diff --git a/lib/apify/twitter/__tests__/isOriginalTweet.test.ts b/lib/apify/twitter/__tests__/isOriginalTweet.test.ts new file mode 100644 index 000000000..6ac47cecc --- /dev/null +++ b/lib/apify/twitter/__tests__/isOriginalTweet.test.ts @@ -0,0 +1,20 @@ +import { describe, it, expect } from "vitest"; +import { isOriginalTweet } from "@/lib/apify/twitter/isOriginalTweet"; + +describe("isOriginalTweet", () => { + it("keeps original tweets", () => { + expect(isOriginalTweet({ isRetweet: false, isQuote: false, isReply: false })).toBe(true); + }); + it("keeps quote tweets — the artist adds their own words and earns the stats", () => { + expect(isOriginalTweet({ isRetweet: false, isQuote: true, isReply: false })).toBe(true); + }); + it("drops retweets — the content and metrics belong to someone else", () => { + expect(isOriginalTweet({ isRetweet: true, isQuote: false, isReply: false })).toBe(false); + }); + it("drops replies — conversational noise, not roster posts", () => { + expect(isOriginalTweet({ isRetweet: false, isQuote: false, isReply: true })).toBe(false); + }); + it("keeps items without flags (defensive default)", () => { + expect(isOriginalTweet({})).toBe(true); + }); +}); diff --git a/lib/apify/twitter/handleTwitterProfileScraperResults.ts b/lib/apify/twitter/handleTwitterProfileScraperResults.ts index 4634fae0a..766aa6e15 100644 --- a/lib/apify/twitter/handleTwitterProfileScraperResults.ts +++ b/lib/apify/twitter/handleTwitterProfileScraperResults.ts @@ -4,12 +4,16 @@ import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; import { persistPostsForSocial } from "@/lib/apify/persistPostsForSocial"; import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; import { toIsoDate } from "@/lib/apify/toIsoDate"; +import { isOriginalTweet } from "@/lib/apify/twitter/isOriginalTweet"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; import type { TablesInsert } from "@/types/database.types"; /** Tweet item from apidojo~twitter-scraper-lite (real shape, run ALVMZYXkh3WHgeGfT; * tweet fields verified on run bx3asRqfbNnkKgogG). */ type TweetItem = { + isRetweet?: boolean; + isQuote?: boolean; + isReply?: boolean; url?: string; createdAt?: string; author?: { @@ -50,8 +54,12 @@ export async function handleTwitterProfileScraperResults(parsed: ApifyWebhookPay // Tweet URLs keep the author's display casing — the path segment is the // case-sensitive status id's context, and unlike profile keys they are // stored as-is (posts upsert keys on exact post_url). + // Only the artist's own posts count — retweets are someone else's content + // and stats, replies are conversation (chat#1855). const postRows: TablesInsert<"posts">[] = (items as TweetItem[]).flatMap(item => - item.url ? [{ post_url: item.url, updated_at: toIsoDate(item.createdAt) }] : [], + item.url && isOriginalTweet(item) + ? [{ post_url: item.url, updated_at: toIsoDate(item.createdAt) }] + : [], ); // Diff before persisting so the digest can report genuinely new posts (chat#1855). const newPostUrls = await filterNewPostUrls(postRows.map(p => p.post_url)); diff --git a/lib/apify/twitter/isOriginalTweet.ts b/lib/apify/twitter/isOriginalTweet.ts new file mode 100644 index 000000000..22d435dbf --- /dev/null +++ b/lib/apify/twitter/isOriginalTweet.ts @@ -0,0 +1,14 @@ +type TweetFlags = { isRetweet?: boolean; isQuote?: boolean; isReply?: boolean }; + +/** + * True for tweets the artist actually authored — originals and quote tweets + * (their own words on top). Retweets are someone else's content and carry the + * ORIGINAL author's engagement stats, so persisting or reporting them + * misattributes both (chat#1855 feedback: an all-retweet digest section + * showed "hundreds of likes" the artist never earned). Replies are excluded + * as conversational noise rather than roster posts. Items without flags pass + * (defensive default — never silently drop a scrape result on schema drift). + */ +export function isOriginalTweet(item: TweetFlags): boolean { + return !item.isRetweet && !item.isReply; +} From cd5761b769231943b309e19314c10d9663a8f89d Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Thu, 9 Jul 2026 18:08:42 -0500 Subject: [PATCH 12/12] =?UTF-8?q?fix(apify):=20fetch=2010=20X=20timeline?= =?UTF-8?q?=20items=20by=20default=20=E2=80=94=20depth=201=20rarely=20surv?= =?UTF-8?q?ives=20the=20retweet=20filter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real-account evidence (2026-07-09): the user's timeline had 7 retweets above their 2 originals and a quote tweet; a depth-1 (or 3) fetch returned only retweets, which isOriginalTweet now drops — so no authored post could ever reach the digest. Fetch deeper, filter after. Co-Authored-By: Claude Fable 5 --- .../twitter/__tests__/startTwitterProfileScraping.test.ts | 4 ++-- lib/apify/twitter/startTwitterProfileScraping.ts | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts b/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts index 325e1f484..43fc1605a 100644 --- a/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts +++ b/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts @@ -11,11 +11,11 @@ beforeEach(() => { }); describe("startTwitterProfileScraping", () => { - it("defaults to the legacy single-item snapshot (maxItems: 1) when posts is omitted", async () => { + it("defaults to a 10-item fetch when posts is omitted — retweets/replies are filtered AFTER the fetch, so depth 1 would rarely surface an authored post", async () => { const r = await startTwitterProfileScraping("sweetman_eth"); expect(apifyClient.actor).toHaveBeenCalledWith("apidojo/twitter-scraper-lite"); expect(start).toHaveBeenCalledWith( - { twitterHandles: ["sweetman_eth"], sort: "Latest", maxItems: 1 }, + { twitterHandles: ["sweetman_eth"], sort: "Latest", maxItems: 10 }, { webhooks: expect.any(Array) }, ); expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" }); diff --git a/lib/apify/twitter/startTwitterProfileScraping.ts b/lib/apify/twitter/startTwitterProfileScraping.ts index 16821355d..5de2e1bf2 100644 --- a/lib/apify/twitter/startTwitterProfileScraping.ts +++ b/lib/apify/twitter/startTwitterProfileScraping.ts @@ -15,7 +15,10 @@ const startTwitterProfileScraping = async ( const input = { twitterHandles: [cleanHandle], sort: "Latest", - maxItems: posts ?? 1, + // Timeline items include retweets/replies, which are dropped AFTER the + // fetch (isOriginalTweet) — depth 1 would rarely surface an authored + // post for active retweeters (chat#1855). + maxItems: posts ?? 10, }; const run = await apifyClient