From 31dd06006e43464ae3e37f63c7a6910e7192a42a Mon Sep 17 00:00:00 2001 From: Arpit Gupta Date: Sat, 25 Apr 2026 01:04:33 +0530 Subject: [PATCH 01/23] feat(api): add supabase helpers for post_comments, social_posts, and related tables Ports the data-layer primitives the Apify Instagram chain needs: post_comments select/insert, social_posts insert, posts insertPosts/getPostsByUrls, socials selectSocialByProfileUrl, uploadLinkToArweave for profile-pic mirroring. insertSocials now upserts on profile_url so replays are idempotent. Co-Authored-By: Claude Opus 4.7 --- lib/arweave/uploadLinkToArweave.ts | 27 +++++++++++ .../post_comments/insertPostComments.ts | 30 +++++++++++++ .../post_comments/selectPostComments.ts | 45 +++++++++++++++++++ lib/supabase/posts/getPostsByUrls.ts | 21 +++++++++ lib/supabase/posts/insertPosts.ts | 21 +++++++++ .../social_posts/insertSocialPosts.ts | 21 +++++++++ lib/supabase/socials/insertSocials.ts | 5 ++- .../socials/selectSocialByProfileUrl.ts | 23 ++++++++++ 8 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 lib/arweave/uploadLinkToArweave.ts create mode 100644 lib/supabase/post_comments/insertPostComments.ts create mode 100644 lib/supabase/post_comments/selectPostComments.ts create mode 100644 lib/supabase/posts/getPostsByUrls.ts create mode 100644 lib/supabase/posts/insertPosts.ts create mode 100644 lib/supabase/social_posts/insertSocialPosts.ts create mode 100644 lib/supabase/socials/selectSocialByProfileUrl.ts diff --git a/lib/arweave/uploadLinkToArweave.ts b/lib/arweave/uploadLinkToArweave.ts new file mode 100644 index 000000000..d550c1b39 --- /dev/null +++ b/lib/arweave/uploadLinkToArweave.ts @@ -0,0 +1,27 @@ +import { uploadToArweave } from "./uploadToArweave"; + +/** + * Fetches the image at `imageUrl` and uploads its bytes to Arweave. + * Returns the Arweave transaction id (without the `ar://` prefix) on + * success, or `null` if fetch / upload fails — callers should fall + * back to the original URL in that case. + * + * @param imageUrl - Remote image URL. + */ +export async function uploadLinkToArweave( + imageUrl: string | null | undefined, +): Promise { + if (!imageUrl) return null; + try { + const res = await fetch(imageUrl); + if (!res.ok) return null; + const buffer = Buffer.from(await res.arrayBuffer()); + const contentType = res.headers.get("content-type") || "image/png"; + + const transaction = await uploadToArweave(buffer, contentType); + return transaction?.id ?? null; + } catch (err) { + console.error("[ERROR] uploadLinkToArweave:", err); + return null; + } +} diff --git a/lib/supabase/post_comments/insertPostComments.ts b/lib/supabase/post_comments/insertPostComments.ts new file mode 100644 index 000000000..403cf42c0 --- /dev/null +++ b/lib/supabase/post_comments/insertPostComments.ts @@ -0,0 +1,30 @@ +import supabase from "../serverClient"; +import type { Tables, TablesInsert } from "@/types/database.types"; + +/** + * Upserts rows into `post_comments`. Conflicts on + * `(post_id, social_id, comment, commented_at)` are ignored so replays + * of the same Apify dataset do not duplicate comments. + * + * @param comments - Rows to insert. + */ +export async function insertPostComments( + comments: TablesInsert<"post_comments">[], +): Promise[]> { + if (comments.length === 0) return []; + + const { data, error } = await supabase + .from("post_comments") + .upsert(comments, { + onConflict: "post_id,social_id,comment,commented_at", + ignoreDuplicates: true, + }) + .select(); + + if (error) { + console.error("[ERROR] insertPostComments:", error); + throw error; + } + + return data ?? []; +} diff --git a/lib/supabase/post_comments/selectPostComments.ts b/lib/supabase/post_comments/selectPostComments.ts new file mode 100644 index 000000000..18777d010 --- /dev/null +++ b/lib/supabase/post_comments/selectPostComments.ts @@ -0,0 +1,45 @@ +import supabase from "../serverClient"; +import type { Tables } from "@/types/database.types"; +import { getPostsByUrls } from "@/lib/supabase/posts/getPostsByUrls"; + +export type PostCommentWithRelations = Tables<"post_comments"> & { + post?: Tables<"posts"> | null; + social?: Tables<"socials"> | null; +}; + +/** + * Selects `post_comments` rows with joined post + social rows. When + * `postUrls` is provided, comments are filtered to those whose + * underlying post URL matches one of the values. + * + * @param params.postUrls - Optional list of post URLs to restrict to. + */ +export async function selectPostComments({ + postUrls, +}: { + postUrls?: string[]; +} = {}): Promise { + let query = supabase.from("post_comments").select(` + *, + post:posts(*), + social:socials(*) + `); + + if (postUrls && postUrls.length > 0) { + const posts = await getPostsByUrls(postUrls); + if (posts.length === 0) return []; + query = query.in( + "post_id", + posts.map(p => p.id), + ); + } + + const { data, error } = await query; + + if (error) { + console.error("[ERROR] selectPostComments:", error); + throw error; + } + + return (data as PostCommentWithRelations[] | null) ?? []; +} diff --git a/lib/supabase/posts/getPostsByUrls.ts b/lib/supabase/posts/getPostsByUrls.ts new file mode 100644 index 000000000..6d51bc226 --- /dev/null +++ b/lib/supabase/posts/getPostsByUrls.ts @@ -0,0 +1,21 @@ +import supabase from "@/lib/supabase/serverClient"; +import type { Tables } from "@/types/database.types"; + +/** + * Fetches `posts` rows matching any of the given URLs. Returns an + * empty array for empty / invalid input. + * + * @param postUrls - Array of `post_url` values to look up. + */ +export async function getPostsByUrls(postUrls: string[]): Promise[]> { + if (!Array.isArray(postUrls) || postUrls.length === 0) return []; + + const { data, error } = await supabase.from("posts").select("*").in("post_url", postUrls); + + if (error) { + console.error("[ERROR] getPostsByUrls:", error); + return []; + } + + return data ?? []; +} diff --git a/lib/supabase/posts/insertPosts.ts b/lib/supabase/posts/insertPosts.ts new file mode 100644 index 000000000..af3b72803 --- /dev/null +++ b/lib/supabase/posts/insertPosts.ts @@ -0,0 +1,21 @@ +import supabase from "@/lib/supabase/serverClient"; +import type { TablesInsert } from "@/types/database.types"; + +/** + * Upserts an array of posts into the `posts` table. Duplicates on + * `post_url` are ignored so this is safe to call repeatedly for the + * same post set during Apify webhook replays. + * + * @param posts - Rows matching the posts-table insert type. + */ +export async function insertPosts(posts: TablesInsert<"posts">[]) { + const { data, error } = await supabase + .from("posts") + .upsert(posts, { onConflict: "post_url", ignoreDuplicates: true }); + + if (error) { + console.error("[ERROR] insertPosts:", error); + } + + return { data, error }; +} diff --git a/lib/supabase/social_posts/insertSocialPosts.ts b/lib/supabase/social_posts/insertSocialPosts.ts new file mode 100644 index 000000000..81e90cdaf --- /dev/null +++ b/lib/supabase/social_posts/insertSocialPosts.ts @@ -0,0 +1,21 @@ +import supabase from "../serverClient"; +import type { TablesInsert } from "@/types/database.types"; + +/** + * Upserts rows into `social_posts` to link posts with the social account + * that produced them. Conflicts on `(post_id, social_id)` are merged so + * repeated webhook deliveries do not create duplicates. + * + * @param socialPosts - Rows to upsert. + */ +export async function insertSocialPosts(socialPosts: TablesInsert<"social_posts">[]) { + const { data, error } = await supabase + .from("social_posts") + .upsert(socialPosts, { onConflict: "post_id,social_id" }); + + if (error) { + console.error("[ERROR] insertSocialPosts:", error); + } + + return { data, error }; +} diff --git a/lib/supabase/socials/insertSocials.ts b/lib/supabase/socials/insertSocials.ts index f149a0a3c..19738798a 100644 --- a/lib/supabase/socials/insertSocials.ts +++ b/lib/supabase/socials/insertSocials.ts @@ -10,7 +10,10 @@ import type { Tables, TablesInsert } from "@/types/database.types"; export async function insertSocials( socials: TablesInsert<"socials">[], ): Promise[]> { - const { data, error } = await supabase.from("socials").insert(socials).select("*"); + const { data, error } = await supabase + .from("socials") + .upsert(socials, { onConflict: "profile_url" }) + .select("*"); if (error) { console.error("[ERROR] insertSocials:", error); diff --git a/lib/supabase/socials/selectSocialByProfileUrl.ts b/lib/supabase/socials/selectSocialByProfileUrl.ts new file mode 100644 index 000000000..d0aff8cd4 --- /dev/null +++ b/lib/supabase/socials/selectSocialByProfileUrl.ts @@ -0,0 +1,23 @@ +import supabase from "../serverClient"; +import type { Tables } from "@/types/database.types"; + +/** + * Fetches a single `socials` row by normalized `profile_url`. + * Returns `null` when no row matches or the URL is empty. + * + * @param profileUrl - Normalized profile URL. + */ +export async function selectSocialByProfileUrl( + profileUrl: string, +): Promise | null> { + if (!profileUrl) return null; + + const { data } = await supabase + .from("socials") + .select("*") + .eq("profile_url", profileUrl) + .neq("profile_url", "") + .single(); + + return data ?? null; +} From 9edcbac5098fa049978d43fabea111784c0b7b15 Mon Sep 17 00:00:00 2001 From: Arpit Gupta Date: Sat, 25 Apr 2026 01:04:42 +0530 Subject: [PATCH 02/23] feat(api): port apify webhook chain + instagram scrape handlers Adds POST /api/apify receiver, dispatcher, validator, payload schema, and ports chat's Instagram profile + comments handlers so follow-up scrapes run as in-process function calls instead of HTTP hops: - startInstagramProfileScraping accepts one or many handles and registers a webhook pointing at this service - startInstagramCommentsScraping (new) mirrors the above for actor SbK00X0JYCPblD2wp - Profile results persist posts + socials, mirror profile pics to Arweave, notify subscribers via Resend, then trigger comments - Comments results persist into post_comments then kick off fan profile scraping for distinct commenter handles Co-Authored-By: Claude Opus 4.7 --- app/api/apify/route.ts | 19 +++ .../__tests__/handleApifyWebhook.test.ts | 50 ++++++++ .../__tests__/postApifyWebhookHandler.test.ts | 45 +++++++ .../validateApifyWebhookRequest.test.ts | 43 +++++++ lib/apify/apifyPayloadSchema.ts | 20 +++ lib/apify/getApifyWebhooks.ts | 16 +++ lib/apify/getDataset.ts | 31 +++++ lib/apify/handleApifyWebhook.ts | 40 ++++++ .../getOrCreatePostsForComments.test.ts | 43 +++++++ .../getOrCreateSocialsForComments.test.ts | 66 ++++++++++ .../handleInstagramCommentsScraper.test.ts | 71 +++++++++++ ...ndleInstagramProfileScraperResults.test.ts | 85 +++++++++++++ .../instagram/getExistingPostComments.ts | 34 +++++ .../instagram/getOrCreatePostsForComments.ts | 36 ++++++ .../getOrCreateSocialsForComments.ts | 52 ++++++++ .../handleInstagramCommentsScraper.ts | 50 ++++++++ .../handleInstagramProfileFollowUpRuns.ts | 39 ++++++ .../handleInstagramProfileScraperResults.ts | 116 ++++++++++++++++++ .../instagram/saveApifyInstagramComments.ts | 51 ++++++++ .../instagram/saveApifyInstagramPosts.ts | 28 +++++ .../startInstagramCommentsScraping.ts | 45 +++++++ .../startInstagramProfileScraping.ts | 28 +++-- lib/apify/postApifyWebhookHandler.ts | 34 +++++ lib/apify/sendApifyWebhookEmail.ts | 50 ++++++++ lib/apify/types.ts | 43 +++++++ lib/apify/validateApifyWebhookRequest.ts | 34 +++++ 26 files changed, 1161 insertions(+), 8 deletions(-) create mode 100644 app/api/apify/route.ts create mode 100644 lib/apify/__tests__/handleApifyWebhook.test.ts create mode 100644 lib/apify/__tests__/postApifyWebhookHandler.test.ts create mode 100644 lib/apify/__tests__/validateApifyWebhookRequest.test.ts create mode 100644 lib/apify/apifyPayloadSchema.ts create mode 100644 lib/apify/getApifyWebhooks.ts create mode 100644 lib/apify/getDataset.ts create mode 100644 lib/apify/handleApifyWebhook.ts create mode 100644 lib/apify/instagram/__tests__/getOrCreatePostsForComments.test.ts create mode 100644 lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts create mode 100644 lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts create mode 100644 lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts create mode 100644 lib/apify/instagram/getExistingPostComments.ts create mode 100644 lib/apify/instagram/getOrCreatePostsForComments.ts create mode 100644 lib/apify/instagram/getOrCreateSocialsForComments.ts create mode 100644 lib/apify/instagram/handleInstagramCommentsScraper.ts create mode 100644 lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts create mode 100644 lib/apify/instagram/handleInstagramProfileScraperResults.ts create mode 100644 lib/apify/instagram/saveApifyInstagramComments.ts create mode 100644 lib/apify/instagram/saveApifyInstagramPosts.ts create mode 100644 lib/apify/instagram/startInstagramCommentsScraping.ts create mode 100644 lib/apify/postApifyWebhookHandler.ts create mode 100644 lib/apify/sendApifyWebhookEmail.ts create mode 100644 lib/apify/validateApifyWebhookRequest.ts diff --git a/app/api/apify/route.ts b/app/api/apify/route.ts new file mode 100644 index 000000000..5c005e53d --- /dev/null +++ b/app/api/apify/route.ts @@ -0,0 +1,19 @@ +import { NextRequest } from "next/server"; +import { postApifyWebhookHandler } from "@/lib/apify/postApifyWebhookHandler"; + +export const maxDuration = 60; +export const dynamic = "force-dynamic"; +export const fetchCache = "force-no-store"; +export const revalidate = 0; + +/** + * `POST /api/apify` — Apify webhook receiver. Unauthenticated on the + * route itself; authenticity is established by the shared actor token + * that only trusted runs can register. Delegates to the dispatcher. + * + * @param request - Incoming webhook request. + * @returns JSON response describing the processed payload (always 200). + */ +export async function POST(request: NextRequest) { + return postApifyWebhookHandler(request); +} diff --git a/lib/apify/__tests__/handleApifyWebhook.test.ts b/lib/apify/__tests__/handleApifyWebhook.test.ts new file mode 100644 index 000000000..21173084f --- /dev/null +++ b/lib/apify/__tests__/handleApifyWebhook.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { handleApifyWebhook } from "../handleApifyWebhook"; +import { handleInstagramProfileScraperResults } from "../instagram/handleInstagramProfileScraperResults"; +import { handleInstagramCommentsScraper } from "../instagram/handleInstagramCommentsScraper"; + +vi.mock("../instagram/handleInstagramProfileScraperResults", () => ({ + handleInstagramProfileScraperResults: vi.fn(), +})); +vi.mock("../instagram/handleInstagramCommentsScraper", () => ({ + handleInstagramCommentsScraper: vi.fn(), +})); + +const base = { + userId: "u", + createdAt: "2026-01-01T00:00:00Z", + eventType: "ACTOR.RUN.SUCCEEDED", + resource: { defaultDatasetId: "ds_1" }, +}; + +describe("handleApifyWebhook", () => { + beforeEach(() => vi.clearAllMocks()); + + it("dispatches Instagram profile actor to profile handler", async () => { + vi.mocked(handleInstagramProfileScraperResults).mockResolvedValue({ posts: [] } as never); + await handleApifyWebhook({ ...base, eventData: { actorId: "dSCLg0C3YEZ83HzYX" } }); + expect(handleInstagramProfileScraperResults).toHaveBeenCalledOnce(); + expect(handleInstagramCommentsScraper).not.toHaveBeenCalled(); + }); + + it("dispatches Instagram comments actor to comments handler", async () => { + vi.mocked(handleInstagramCommentsScraper).mockResolvedValue({ comments: [] } as never); + await handleApifyWebhook({ ...base, eventData: { actorId: "SbK00X0JYCPblD2wp" } }); + expect(handleInstagramCommentsScraper).toHaveBeenCalledOnce(); + expect(handleInstagramProfileScraperResults).not.toHaveBeenCalled(); + }); + + it("returns fallback for unhandled actor ids without throwing", async () => { + const result = await handleApifyWebhook({ ...base, eventData: { actorId: "unknown" } }); + expect(result).toMatchObject({ posts: [], social: null }); + }); + + it("swallows handler errors and returns fallback", async () => { + vi.mocked(handleInstagramCommentsScraper).mockRejectedValueOnce(new Error("boom")); + const result = await handleApifyWebhook({ + ...base, + eventData: { actorId: "SbK00X0JYCPblD2wp" }, + }); + expect(result).toMatchObject({ posts: [], social: null }); + }); +}); diff --git a/lib/apify/__tests__/postApifyWebhookHandler.test.ts b/lib/apify/__tests__/postApifyWebhookHandler.test.ts new file mode 100644 index 000000000..a2e57a343 --- /dev/null +++ b/lib/apify/__tests__/postApifyWebhookHandler.test.ts @@ -0,0 +1,45 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { NextRequest } from "next/server"; +import { postApifyWebhookHandler } from "../postApifyWebhookHandler"; +import { handleApifyWebhook } from "../handleApifyWebhook"; + +vi.mock("../handleApifyWebhook", () => ({ handleApifyWebhook: vi.fn() })); + +function makeRequest(body: unknown, raw?: string) { + return new NextRequest("http://localhost/api/apify", { + method: "POST", + headers: { "content-type": "application/json" }, + body: raw ?? JSON.stringify(body), + }); +} + +describe("postApifyWebhookHandler", () => { + beforeEach(() => vi.clearAllMocks()); + + it("returns 200 with dispatcher result on valid payload", async () => { + vi.mocked(handleApifyWebhook).mockResolvedValue({ posts: [1, 2] } as never); + + const res = await postApifyWebhookHandler( + makeRequest({ + userId: "u", + createdAt: "2026-01-01T00:00:00Z", + eventType: "ACTOR.RUN.SUCCEEDED", + eventData: { actorId: "dSCLg0C3YEZ83HzYX" }, + resource: { defaultDatasetId: "ds_1" }, + }), + ); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ posts: [1, 2] }); + expect(handleApifyWebhook).toHaveBeenCalledOnce(); + }); + + it("returns 200 for invalid payloads so Apify does not retry", async () => { + const res = await postApifyWebhookHandler(makeRequest({ bogus: true })); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.message).toBe("Invalid payload"); + expect(handleApifyWebhook).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/__tests__/validateApifyWebhookRequest.test.ts b/lib/apify/__tests__/validateApifyWebhookRequest.test.ts new file mode 100644 index 000000000..6568fb475 --- /dev/null +++ b/lib/apify/__tests__/validateApifyWebhookRequest.test.ts @@ -0,0 +1,43 @@ +import { describe, it, expect } from "vitest"; +import { NextRequest } from "next/server"; +import { validateApifyWebhookRequest } from "../validateApifyWebhookRequest"; + +function makeRequest(body: unknown, raw?: string) { + return new NextRequest("http://localhost/api/apify", { + method: "POST", + headers: { "content-type": "application/json" }, + body: raw ?? JSON.stringify(body), + }); +} + +describe("validateApifyWebhookRequest", () => { + it("returns ok+data for a valid Apify payload", async () => { + const req = makeRequest({ + userId: "u", + createdAt: "2026-01-01T00:00:00Z", + eventType: "ACTOR.RUN.SUCCEEDED", + eventData: { actorId: "dSCLg0C3YEZ83HzYX" }, + resource: { defaultDatasetId: "ds_1" }, + }); + + const result = await validateApifyWebhookRequest(req); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.data.eventData.actorId).toBe("dSCLg0C3YEZ83HzYX"); + expect(result.data.resource.defaultDatasetId).toBe("ds_1"); + } + }); + + it("rejects payloads missing required fields", async () => { + const req = makeRequest({ eventData: {}, resource: {} }); + const result = await validateApifyWebhookRequest(req); + expect(result.ok).toBe(false); + }); + + it("rejects non-JSON bodies", async () => { + const req = makeRequest(null, "not json"); + const result = await validateApifyWebhookRequest(req); + expect(result).toEqual({ ok: false, error: "Invalid JSON" }); + }); +}); diff --git a/lib/apify/apifyPayloadSchema.ts b/lib/apify/apifyPayloadSchema.ts new file mode 100644 index 000000000..d2890cb0d --- /dev/null +++ b/lib/apify/apifyPayloadSchema.ts @@ -0,0 +1,20 @@ +import { z } from "zod"; + +/** + * Zod schema for Apify webhook payloads. Only validates the fields we + * branch on + read downstream; the rest of the payload is intentionally + * permissive so Apify schema drift does not drop events. + */ +export const apifyPayloadSchema = z.object({ + userId: z.any(), + createdAt: z.any(), + eventType: z.any(), + eventData: z.object({ + actorId: z.string(), + }), + resource: z.object({ + defaultDatasetId: z.string(), + }), +}); + +export type ApifyPayload = z.infer; diff --git a/lib/apify/getApifyWebhooks.ts b/lib/apify/getApifyWebhooks.ts new file mode 100644 index 000000000..b47276a2b --- /dev/null +++ b/lib/apify/getApifyWebhooks.ts @@ -0,0 +1,16 @@ +import type { WebhookEventType, WebhookUpdateData } from "apify-client"; + +/** + * Webhook config registered on Apify actor runs. The Apify client + * serializes this to base64 before including it on the start request. + * Points at this service's `POST /api/apify` webhook receiver. + */ +export function getApifyWebhooks(): WebhookUpdateData[] { + const eventTypes: WebhookEventType[] = ["ACTOR.RUN.SUCCEEDED"]; + return [ + { + eventTypes, + requestUrl: "https://recoup-api.vercel.app/api/apify", + }, + ]; +} diff --git a/lib/apify/getDataset.ts b/lib/apify/getDataset.ts new file mode 100644 index 000000000..816389f12 --- /dev/null +++ b/lib/apify/getDataset.ts @@ -0,0 +1,31 @@ +/** + * Fetches items from an Apify dataset by id. Uses the REST endpoint + * rather than the SDK so we do not have to materialize the whole + * dataset client just to pull items. + * + * @param datasetId - Apify dataset id. + * @returns Parsed dataset body (array) or `[]` on failure. + */ +export async function getDataset(datasetId: string): Promise { + const token = process.env.APIFY_TOKEN; + if (!token) { + console.error("[ERROR] getDataset: missing APIFY_TOKEN"); + return []; + } + + const response = await fetch( + `https://api.apify.com/v2/datasets/${datasetId}/items?token=${token}`, + { + method: "GET", + headers: { "Content-Type": "application/json" }, + }, + ); + + if (!response.ok) { + console.error(`[ERROR] getDataset: ${response.status} ${response.statusText}`); + return []; + } + + const data = await response.json(); + return Array.isArray(data) ? data : []; +} diff --git a/lib/apify/handleApifyWebhook.ts b/lib/apify/handleApifyWebhook.ts new file mode 100644 index 000000000..2f0a2111f --- /dev/null +++ b/lib/apify/handleApifyWebhook.ts @@ -0,0 +1,40 @@ +import type { ApifyPayload } from "@/lib/apify/apifyPayloadSchema"; +import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; +import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; + +const INSTAGRAM_PROFILE_ACTOR_ID = "dSCLg0C3YEZ83HzYX"; +const INSTAGRAM_COMMENTS_ACTOR_ID = "SbK00X0JYCPblD2wp"; + +const fallbackResponse = { + posts: [], + social: null, + accountSocials: [], + accountArtistIds: [], + accountEmails: [], + sentEmails: null, +}; + +/** + * Dispatches an Apify webhook payload to the handler matching the + * `eventData.actorId`. Unknown actor ids are logged and return an + * empty-shaped response. Never throws — handler failures are caught + * and logged so the webhook route can always reply 200. + * + * @param parsed - Validated Apify webhook payload. + */ +export async function handleApifyWebhook(parsed: ApifyPayload) { + try { + switch (parsed.eventData.actorId) { + case INSTAGRAM_PROFILE_ACTOR_ID: + return await handleInstagramProfileScraperResults(parsed); + case INSTAGRAM_COMMENTS_ACTOR_ID: + return await handleInstagramCommentsScraper(parsed); + default: + console.log(`[WARN] handleApifyWebhook: unhandled actorId ${parsed.eventData.actorId}`); + return fallbackResponse; + } + } catch (e) { + console.error("[ERROR] handleApifyWebhook:", e); + return fallbackResponse; + } +} diff --git a/lib/apify/instagram/__tests__/getOrCreatePostsForComments.test.ts b/lib/apify/instagram/__tests__/getOrCreatePostsForComments.test.ts new file mode 100644 index 000000000..ad2079bc9 --- /dev/null +++ b/lib/apify/instagram/__tests__/getOrCreatePostsForComments.test.ts @@ -0,0 +1,43 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { getOrCreatePostsForComments } from "../getOrCreatePostsForComments"; +import { insertPosts } from "@/lib/supabase/posts/insertPosts"; +import { getPostsByUrls } from "@/lib/supabase/posts/getPostsByUrls"; + +vi.mock("@/lib/supabase/posts/insertPosts", () => ({ insertPosts: vi.fn() })); +vi.mock("@/lib/supabase/posts/getPostsByUrls", () => ({ getPostsByUrls: vi.fn() })); + +describe("getOrCreatePostsForComments", () => { + beforeEach(() => vi.clearAllMocks()); + + it("returns empty map for empty input and does not touch DB", async () => { + const result = await getOrCreatePostsForComments([]); + expect(result.size).toBe(0); + expect(insertPosts).not.toHaveBeenCalled(); + expect(getPostsByUrls).not.toHaveBeenCalled(); + }); + + it("inserts only the URLs missing from Supabase", async () => { + const existing = [{ id: "p1", post_url: "u1" }] as never; + const afterInsert = [ + { id: "p1", post_url: "u1" }, + { id: "p2", post_url: "u2" }, + ] as never; + + vi.mocked(getPostsByUrls).mockResolvedValueOnce(existing).mockResolvedValueOnce(afterInsert); + + const result = await getOrCreatePostsForComments(["u1", "u2", "u2"]); + + expect(insertPosts).toHaveBeenCalledWith([expect.objectContaining({ post_url: "u2" })]); + expect(result.get("u1")).toEqual(existing[0]); + expect(result.get("u2")).toEqual(afterInsert[1]); + }); + + it("skips insert when all posts already exist", async () => { + const existing = [{ id: "p1", post_url: "u1" }] as never; + vi.mocked(getPostsByUrls).mockResolvedValue(existing); + + await getOrCreatePostsForComments(["u1"]); + + expect(insertPosts).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts b/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts new file mode 100644 index 000000000..8ad8adfd6 --- /dev/null +++ b/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts @@ -0,0 +1,66 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { getOrCreateSocialsForComments } from "../getOrCreateSocialsForComments"; +import { insertSocials } from "@/lib/supabase/socials/insertSocials"; + +vi.mock("@/lib/supabase/socials/insertSocials", () => ({ insertSocials: vi.fn() })); + +describe("getOrCreateSocialsForComments", () => { + beforeEach(() => vi.clearAllMocks()); + + it("upserts one row per distinct commenter and keys result by username", async () => { + vi.mocked(insertSocials).mockResolvedValue([ + { id: "s1", username: "alice", profile_url: "instagram.com/alice" }, + { id: "s2", username: "bob", profile_url: "instagram.com/bob" }, + ] as never); + + const result = await getOrCreateSocialsForComments([ + { + id: "c1", + text: "t", + timestamp: "2026-01-01", + ownerUsername: "alice", + ownerProfilePicUrl: "https://a", + postUrl: "u1", + }, + { + id: "c2", + text: "t2", + timestamp: "2026-01-02", + ownerUsername: "alice", + ownerProfilePicUrl: "https://a", + postUrl: "u1", + }, + { + id: "c3", + text: "t3", + timestamp: "2026-01-03", + ownerUsername: "bob", + ownerProfilePicUrl: "https://b", + postUrl: "u2", + }, + ]); + + expect(insertSocials).toHaveBeenCalledOnce(); + const [rows] = vi.mocked(insertSocials).mock.calls[0]; + expect(rows).toHaveLength(2); + expect(result.get("alice")?.id).toBe("s1"); + expect(result.get("bob")?.id).toBe("s2"); + }); + + it("returns empty map and swallows DB errors", async () => { + vi.mocked(insertSocials).mockRejectedValue(new Error("boom")); + + const result = await getOrCreateSocialsForComments([ + { + id: "c1", + text: "t", + timestamp: "2026-01-01", + ownerUsername: "alice", + ownerProfilePicUrl: "https://a", + postUrl: "u1", + }, + ]); + + expect(result.size).toBe(0); + }); +}); diff --git a/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts b/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts new file mode 100644 index 000000000..c34bc5ee9 --- /dev/null +++ b/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { handleInstagramCommentsScraper } from "../handleInstagramCommentsScraper"; +import { getDataset } from "@/lib/apify/getDataset"; +import { saveApifyInstagramComments } from "../saveApifyInstagramComments"; +import { startInstagramProfileScraping } from "../startInstagramProfileScraping"; + +vi.mock("@/lib/apify/getDataset", () => ({ getDataset: vi.fn() })); +vi.mock("../saveApifyInstagramComments", () => ({ saveApifyInstagramComments: vi.fn() })); +vi.mock("../startInstagramProfileScraping", () => ({ + startInstagramProfileScraping: vi.fn(), +})); + +const payload = { + userId: "u", + createdAt: "2026-01-01T00:00:00Z", + eventType: "ACTOR.RUN.SUCCEEDED", + eventData: { actorId: "SbK00X0JYCPblD2wp" }, + resource: { defaultDatasetId: "ds_1" }, +} as never; + +describe("handleInstagramCommentsScraper", () => { + beforeEach(() => vi.clearAllMocks()); + + it("saves comments and enqueues fan profile scrape for distinct usernames", async () => { + vi.mocked(getDataset).mockResolvedValue([ + { + id: "c1", + text: "hi", + timestamp: "t", + ownerUsername: "alice", + ownerProfilePicUrl: "", + postUrl: "u1", + }, + { + id: "c2", + text: "hi2", + timestamp: "t", + ownerUsername: "bob", + ownerProfilePicUrl: "", + postUrl: "u1", + }, + { + id: "c3", + text: "hi3", + timestamp: "t", + ownerUsername: "alice", + ownerProfilePicUrl: "", + postUrl: "u2", + }, + ]); + vi.mocked(startInstagramProfileScraping).mockResolvedValue({ runId: "r", datasetId: "d" }); + + const result = await handleInstagramCommentsScraper(payload); + + expect(saveApifyInstagramComments).toHaveBeenCalledOnce(); + expect(startInstagramProfileScraping).toHaveBeenCalledOnce(); + const [handles] = vi.mocked(startInstagramProfileScraping).mock.calls[0]; + expect(new Set(handles as string[])).toEqual(new Set(["alice", "bob"])); + expect(result.totalComments).toBe(3); + expect(new Set(result.processedPostUrls)).toEqual(new Set(["u1", "u2"])); + }); + + it("returns empty shape without touching handlers when datasetId is missing", async () => { + const result = await handleInstagramCommentsScraper({ + ...payload, + resource: { defaultDatasetId: "" }, + }); + expect(result.totalComments).toBe(0); + expect(saveApifyInstagramComments).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts new file mode 100644 index 000000000..8e7ddee11 --- /dev/null +++ b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { handleInstagramProfileScraperResults } from "../handleInstagramProfileScraperResults"; +import { getDataset } from "@/lib/apify/getDataset"; +import { saveApifyInstagramPosts } from "../saveApifyInstagramPosts"; +import { handleInstagramProfileFollowUpRuns } from "../handleInstagramProfileFollowUpRuns"; +import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; +import { insertSocials } from "@/lib/supabase/socials/insertSocials"; +import { selectSocialByProfileUrl } from "@/lib/supabase/socials/selectSocialByProfileUrl"; +import { insertSocialPosts } from "@/lib/supabase/social_posts/insertSocialPosts"; +import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccountSocials"; +import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccountArtistIds"; +import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails"; +import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; + +vi.mock("@/lib/apify/getDataset", () => ({ getDataset: vi.fn() })); +vi.mock("../saveApifyInstagramPosts", () => ({ saveApifyInstagramPosts: vi.fn() })); +vi.mock("../handleInstagramProfileFollowUpRuns", () => ({ + handleInstagramProfileFollowUpRuns: vi.fn(), +})); +vi.mock("@/lib/apify/sendApifyWebhookEmail", () => ({ sendApifyWebhookEmail: vi.fn() })); +vi.mock("@/lib/supabase/socials/insertSocials", () => ({ insertSocials: vi.fn() })); +vi.mock("@/lib/supabase/socials/selectSocialByProfileUrl", () => ({ + selectSocialByProfileUrl: vi.fn(), +})); +vi.mock("@/lib/supabase/social_posts/insertSocialPosts", () => ({ insertSocialPosts: vi.fn() })); +vi.mock("@/lib/supabase/account_socials/selectAccountSocials", () => ({ + selectAccountSocials: vi.fn(), +})); +vi.mock("@/lib/supabase/account_artist_ids/getAccountArtistIds", () => ({ + getAccountArtistIds: vi.fn(), +})); +vi.mock("@/lib/supabase/account_emails/selectAccountEmails", () => ({ + default: vi.fn(), +})); +vi.mock("@/lib/arweave/uploadLinkToArweave", () => ({ uploadLinkToArweave: vi.fn() })); + +const payload = { + userId: "u", + createdAt: "2026-01-01T00:00:00Z", + eventType: "ACTOR.RUN.SUCCEEDED", + eventData: { actorId: "dSCLg0C3YEZ83HzYX" }, + resource: { defaultDatasetId: "ds_1" }, +} as never; + +describe("handleInstagramProfileScraperResults", () => { + beforeEach(() => vi.clearAllMocks()); + + it("returns the empty shape when the dataset has no latest posts", async () => { + vi.mocked(getDataset).mockResolvedValue([{ username: "alice" }]); + + const result = await handleInstagramProfileScraperResults(payload); + + expect(result).toMatchObject({ posts: [], social: null }); + expect(saveApifyInstagramPosts).not.toHaveBeenCalled(); + }); + + it("persists posts, links social_posts, and fires follow-up runs + email", async () => { + const posts = [{ id: "p1", post_url: "u1", updated_at: "t" }] as never; + vi.mocked(getDataset).mockResolvedValue([ + { + latestPosts: [{ url: "u1", timestamp: "t" }], + username: "alice", + url: "instagram.com/alice", + profilePicUrl: "https://a", + fullName: "Alice", + }, + ]); + vi.mocked(saveApifyInstagramPosts).mockResolvedValue({ supabasePosts: posts }); + vi.mocked(uploadLinkToArweave).mockResolvedValue(null); + vi.mocked(insertSocials).mockResolvedValue([] as never); + vi.mocked(selectSocialByProfileUrl).mockResolvedValue({ id: "s1" } as never); + vi.mocked(selectAccountSocials).mockResolvedValue([{ account_id: "a1" }] as never); + vi.mocked(getAccountArtistIds).mockResolvedValue([{ account_id: "a1" }] as never); + vi.mocked(selectAccountEmails).mockResolvedValue([{ email: "x@y.com" }] as never); + vi.mocked(sendApifyWebhookEmail).mockResolvedValue({ id: "email_1" } as never); + + const result = await handleInstagramProfileScraperResults(payload); + + expect(insertSocialPosts).toHaveBeenCalledOnce(); + expect(sendApifyWebhookEmail).toHaveBeenCalledWith(expect.any(Object), ["x@y.com"]); + expect(handleInstagramProfileFollowUpRuns).toHaveBeenCalledOnce(); + expect(result.social).toEqual({ id: "s1" }); + expect(result.posts).toEqual(posts); + }); +}); diff --git a/lib/apify/instagram/getExistingPostComments.ts b/lib/apify/instagram/getExistingPostComments.ts new file mode 100644 index 000000000..286928b85 --- /dev/null +++ b/lib/apify/instagram/getExistingPostComments.ts @@ -0,0 +1,34 @@ +import { selectPostComments } from "@/lib/supabase/post_comments/selectPostComments"; + +/** + * Partitions `postUrls` into those that already have comments in + * Supabase and those that do not. Used by the follow-up-runs handler + * to decide whether to throttle the comments scraper for known posts. + * + * @param postUrls - Instagram post URLs to check. + */ +export async function getExistingPostComments(postUrls: string[]): Promise<{ + urlsWithComments: string[]; + urlsWithoutComments: string[]; +}> { + if (!postUrls || postUrls.length === 0) { + return { urlsWithComments: [], urlsWithoutComments: [] }; + } + + try { + const existing = await selectPostComments({ postUrls }); + + const withComments = existing + .map(c => c.post?.post_url) + .filter((url): url is string => Boolean(url)); + + const urlsWithComments = Array.from(new Set(withComments)); + const urlsWithoutComments = postUrls.filter(url => !urlsWithComments.includes(url)); + + return { urlsWithComments, urlsWithoutComments }; + } catch (error) { + console.error("[ERROR] getExistingPostComments:", error); + // Assume no comments exist on error so follow-up scrape still fires. + return { urlsWithComments: [], urlsWithoutComments: postUrls }; + } +} diff --git a/lib/apify/instagram/getOrCreatePostsForComments.ts b/lib/apify/instagram/getOrCreatePostsForComments.ts new file mode 100644 index 000000000..33a6e2f17 --- /dev/null +++ b/lib/apify/instagram/getOrCreatePostsForComments.ts @@ -0,0 +1,36 @@ +import { insertPosts } from "@/lib/supabase/posts/insertPosts"; +import { getPostsByUrls } from "@/lib/supabase/posts/getPostsByUrls"; +import type { Tables, TablesInsert } from "@/types/database.types"; + +/** + * Ensures a `posts` row exists for every post URL the comment scraper + * produced, creating any missing rows with `updated_at = now()`. Returns + * a map keyed by `post_url` for efficient lookup during comment insert. + * + * @param postUrls - Instagram post URLs referenced by incoming comments. + */ +export async function getOrCreatePostsForComments( + postUrls: string[], +): Promise>> { + const unique = Array.from(new Set(postUrls.filter(Boolean))); + if (unique.length === 0) return new Map(); + + const existing = await getPostsByUrls(unique); + const existingSet = new Set(existing.map(p => p.post_url)); + + const missing = unique.filter(url => !existingSet.has(url)); + if (missing.length > 0) { + const rows: TablesInsert<"posts">[] = missing.map(url => ({ + post_url: url, + updated_at: new Date().toISOString(), + })); + await insertPosts(rows); + } + + const all = await getPostsByUrls(unique); + const map = new Map>(); + all.forEach(post => { + map.set(post.post_url, post); + }); + return map; +} diff --git a/lib/apify/instagram/getOrCreateSocialsForComments.ts b/lib/apify/instagram/getOrCreateSocialsForComments.ts new file mode 100644 index 000000000..80a516880 --- /dev/null +++ b/lib/apify/instagram/getOrCreateSocialsForComments.ts @@ -0,0 +1,52 @@ +import { insertSocials } from "@/lib/supabase/socials/insertSocials"; +import type { Tables, TablesInsert } from "@/types/database.types"; +import type { ApifyInstagramComment } from "@/lib/apify/types"; + +/** + * Ensures a `socials` row exists for each distinct comment author and + * returns a map from `username` to the upserted social row. Upstream + * `insertSocials` upserts on `profile_url`, so repeated calls are + * idempotent even when the same commenter recurs across posts. + * + * @param comments - Apify Instagram comment dataset items. + */ +export async function getOrCreateSocialsForComments( + comments: ApifyInstagramComment[], +): Promise>> { + const uniqueAuthors = Array.from( + new Map( + comments.map(c => [ + c.ownerUsername, + { + username: c.ownerUsername, + profilePicUrl: c.ownerProfilePicUrl, + profileUrl: `instagram.com/${c.ownerUsername}`, + }, + ]), + ).values(), + ); + + const rows: TablesInsert<"socials">[] = uniqueAuthors + .filter(a => a.username && a.profileUrl) + .map(a => ({ + username: a.username, + profile_url: a.profileUrl, + avatar: a.profilePicUrl, + bio: null, + region: null, + followerCount: null, + followingCount: null, + })); + + try { + const upserted = await insertSocials(rows); + const map = new Map>(); + upserted.forEach(social => { + map.set(social.username, social); + }); + return map; + } catch (error) { + console.error("[ERROR] getOrCreateSocialsForComments:", error); + return new Map(); + } +} diff --git a/lib/apify/instagram/handleInstagramCommentsScraper.ts b/lib/apify/instagram/handleInstagramCommentsScraper.ts new file mode 100644 index 000000000..09164f033 --- /dev/null +++ b/lib/apify/instagram/handleInstagramCommentsScraper.ts @@ -0,0 +1,50 @@ +import { getDataset } from "@/lib/apify/getDataset"; +import { saveApifyInstagramComments } from "@/lib/apify/instagram/saveApifyInstagramComments"; +import { startInstagramProfileScraping } from "@/lib/apify/instagram/startInstagramProfileScraping"; +import type { ApifyPayload } from "@/lib/apify/apifyPayloadSchema"; +import type { ApifyInstagramComment } from "@/lib/apify/types"; + +/** + * Handles Instagram comments scraper Apify webhook results: + * - Persists comments into `post_comments`. + * - Kicks off a fan-profile scrape for the distinct commenter + * usernames so their socials get indexed. + * + * @param parsed - Validated Apify webhook payload. + */ +export async function handleInstagramCommentsScraper(parsed: ApifyPayload) { + const datasetId = parsed.resource.defaultDatasetId; + const empty = { + comments: [] as ApifyInstagramComment[], + processedPostUrls: [] as string[], + totalComments: 0, + }; + + if (!datasetId) return empty; + + try { + const dataset = await getDataset(datasetId); + if (!Array.isArray(dataset)) return empty; + + const comments = dataset as ApifyInstagramComment[]; + const processedPostUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); + const totalComments = comments.length; + + await saveApifyInstagramComments(comments); + + const fanHandles = Array.from(new Set(comments.map(c => c.ownerUsername).filter(Boolean))); + + if (fanHandles.length > 0) { + try { + await startInstagramProfileScraping(fanHandles); + } catch (error) { + console.error("[ERROR] fan profile scrape failed:", error); + } + } + + return { comments, processedPostUrls, totalComments }; + } catch (error) { + console.error("[ERROR] handleInstagramCommentsScraper:", error); + return empty; + } +} diff --git a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts new file mode 100644 index 000000000..9351b1099 --- /dev/null +++ b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts @@ -0,0 +1,39 @@ +import { startInstagramCommentsScraping } from "@/lib/apify/instagram/startInstagramCommentsScraping"; +import { getExistingPostComments } from "@/lib/apify/instagram/getExistingPostComments"; +import type { ApifyInstagramPost, ApifyInstagramProfileResult } from "@/lib/apify/types"; + +/** + * Kicks off a comments-scraper run for the newly-seen posts on a + * profile. Posts already present in `post_comments` use a + * `resultsLimit=1` run (cheap refresh); fully-unseen posts use the + * default `resultsLimit` to backfill. + * + * Only runs when the dataset contains a single profile — multi-profile + * runs are fan lookups and should not trigger comment scraping. + * + * @param dataset - Raw Apify dataset for the profile scrape. + * @param firstResult - First row of the dataset. + */ +export async function handleInstagramProfileFollowUpRuns( + dataset: unknown[], + firstResult: ApifyInstagramProfileResult, +): Promise { + if (dataset.length !== 1) return; + if (!firstResult.latestPosts || firstResult.latestPosts.length === 0) return; + + const postUrls = (firstResult.latestPosts as ApifyInstagramPost[]) + .map(post => post.url) + .filter(Boolean); + + if (postUrls.length === 0) return; + + const { urlsWithComments, urlsWithoutComments } = await getExistingPostComments(postUrls); + + if (urlsWithComments.length > 0) { + await startInstagramCommentsScraping(urlsWithComments, 1); + } + + if (urlsWithoutComments.length > 0) { + await startInstagramCommentsScraping(urlsWithoutComments); + } +} diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts new file mode 100644 index 000000000..c8a212229 --- /dev/null +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -0,0 +1,116 @@ +import { getDataset } from "@/lib/apify/getDataset"; +import { saveApifyInstagramPosts } from "@/lib/apify/instagram/saveApifyInstagramPosts"; +import { handleInstagramProfileFollowUpRuns } from "@/lib/apify/instagram/handleInstagramProfileFollowUpRuns"; +import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; +import { insertSocials } from "@/lib/supabase/socials/insertSocials"; +import { selectSocialByProfileUrl } from "@/lib/supabase/socials/selectSocialByProfileUrl"; +import { insertSocialPosts } from "@/lib/supabase/social_posts/insertSocialPosts"; +import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccountSocials"; +import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccountArtistIds"; +import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails"; +import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; +import { getFetchableUrl } from "@/lib/arweave/getFetchableUrl"; +import type { ApifyInstagramPost, ApifyInstagramProfileResult } from "@/lib/apify/types"; +import type { ApifyPayload } from "@/lib/apify/apifyPayloadSchema"; + +/** + * Handles Instagram profile scraper Apify webhook results: + * - Persists the returned posts + social profile row. + * - Mirrors the profile pic to Arweave. + * - Notifies subscribed account emails via Resend. + * - Queues the comments scraper for the profile's latest posts. + * + * Returns a summary object for logging / downstream inspection. All + * failures inside the chain are logged but allowed to propagate to the + * webhook route's outer try/catch, which always returns 200. + * + * @param parsed - Validated Apify webhook payload. + */ +export async function handleInstagramProfileScraperResults(parsed: ApifyPayload) { + const datasetId = parsed.resource.defaultDatasetId; + const empty = { + posts: [], + social: null, + accountSocials: [] as unknown[], + accountArtistIds: [] as unknown[], + accountEmails: [] as unknown[], + sentEmails: null as unknown, + }; + + if (!datasetId) return empty; + + const dataset = await getDataset(datasetId); + const firstResult = dataset[0] as ApifyInstagramProfileResult | undefined; + if (!firstResult?.latestPosts) return empty; + + const { supabasePosts: posts } = await saveApifyInstagramPosts( + firstResult.latestPosts as ApifyInstagramPost[], + ); + + const arweaveTx = await uploadLinkToArweave( + firstResult.profilePicUrlHD || firstResult.profilePicUrl, + ); + if (arweaveTx) { + firstResult.profilePicUrl = getFetchableUrl(`ar://${arweaveTx}`) ?? firstResult.profilePicUrl; + } + + await insertSocials([ + { + username: firstResult.username ?? "", + avatar: firstResult.profilePicUrl ?? null, + profile_url: firstResult.url ?? "", + bio: firstResult.biography ?? null, + followerCount: firstResult.followersCount ?? null, + followingCount: firstResult.followsCount ?? null, + }, + ]); + + const normalizedUrl = normalizeProfileUrl(firstResult.url); + const social = await selectSocialByProfileUrl(normalizedUrl); + + if (!social) { + return { ...empty, posts }; + } + + if (posts.length) { + await insertSocialPosts( + posts.map(post => ({ + post_id: post.id, + updated_at: post.updated_at, + social_id: social.id, + })), + ); + } + + const accountSocials = await selectAccountSocials({ socialId: social.id, limit: 10000 }); + const accountArtistIds = await getAccountArtistIds({ + artistIds: accountSocials.map(a => a.account_id as string), + }); + + const uniqueAccountIds = Array.from( + new Set( + accountArtistIds + .map(a => (a as unknown as { account_id: string | null }).account_id) + .filter((id): id is string => Boolean(id)), + ), + ); + + const accountEmails = await selectAccountEmails({ accountIds: uniqueAccountIds }); + + const sentEmails = await sendApifyWebhookEmail( + firstResult as unknown as Record, + accountEmails.map(e => e.email).filter(Boolean) as string[], + ); + + await handleInstagramProfileFollowUpRuns(dataset, firstResult); + + return { + posts, + social, + accountSocials, + accountArtistIds, + accountEmails, + sentEmails, + }; +} diff --git a/lib/apify/instagram/saveApifyInstagramComments.ts b/lib/apify/instagram/saveApifyInstagramComments.ts new file mode 100644 index 000000000..521291c04 --- /dev/null +++ b/lib/apify/instagram/saveApifyInstagramComments.ts @@ -0,0 +1,51 @@ +import { insertPostComments } from "@/lib/supabase/post_comments/insertPostComments"; +import { getOrCreatePostsForComments } from "@/lib/apify/instagram/getOrCreatePostsForComments"; +import { getOrCreateSocialsForComments } from "@/lib/apify/instagram/getOrCreateSocialsForComments"; +import type { TablesInsert } from "@/types/database.types"; +import type { ApifyInstagramComment } from "@/lib/apify/types"; + +/** + * Persists Apify Instagram comment dataset items into `post_comments`, + * first ensuring that a matching `posts` and `socials` row exists for + * each comment. Missing references are silently skipped. + * + * @param comments - Apify dataset items. + */ +export async function saveApifyInstagramComments(comments: ApifyInstagramComment[]): Promise { + if (comments.length === 0) return; + + try { + const postUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); + const postsMap = await getOrCreatePostsForComments(postUrls); + const socialsMap = await getOrCreateSocialsForComments(comments); + + const rows: TablesInsert<"post_comments">[] = []; + + for (const comment of comments) { + if (!comment.postUrl || !comment.ownerUsername) continue; + + const post = postsMap.get(comment.postUrl); + const social = socialsMap.get(comment.ownerUsername); + + if (!post || !social) { + console.warn( + `[WARN] saveApifyInstagramComments: missing post/social for comment ${comment.id}`, + ); + continue; + } + + rows.push({ + post_id: post.id, + social_id: social.id, + comment: comment.text, + commented_at: comment.timestamp, + }); + } + + if (rows.length > 0) { + await insertPostComments(rows); + } + } catch (error) { + console.error("[ERROR] saveApifyInstagramComments:", error); + } +} diff --git a/lib/apify/instagram/saveApifyInstagramPosts.ts b/lib/apify/instagram/saveApifyInstagramPosts.ts new file mode 100644 index 000000000..8b8799cde --- /dev/null +++ b/lib/apify/instagram/saveApifyInstagramPosts.ts @@ -0,0 +1,28 @@ +import { insertPosts } from "@/lib/supabase/posts/insertPosts"; +import { getPostsByUrls } from "@/lib/supabase/posts/getPostsByUrls"; +import type { ApifyInstagramPost } from "@/lib/apify/types"; +import type { Tables, TablesInsert } from "@/types/database.types"; + +/** + * Upserts an array of Apify Instagram posts into Supabase and returns + * the resulting rows. + * + * @param apifyPosts - Posts from the Apify Instagram profile scraper. + * @returns Object with Supabase `posts` rows for the provided URLs. + */ +export async function saveApifyInstagramPosts( + apifyPosts: ApifyInstagramPost[], +): Promise<{ supabasePosts: Tables<"posts">[] }> { + if (!apifyPosts?.length) return { supabasePosts: [] }; + + const rows: TablesInsert<"posts">[] = apifyPosts.map(post => ({ + post_url: post.url, + updated_at: post.timestamp, + })); + const postUrls = rows.map(p => p.post_url); + + await insertPosts(rows); + + const supabasePosts = await getPostsByUrls(postUrls); + return { supabasePosts }; +} diff --git a/lib/apify/instagram/startInstagramCommentsScraping.ts b/lib/apify/instagram/startInstagramCommentsScraping.ts new file mode 100644 index 000000000..0ac8d2d0e --- /dev/null +++ b/lib/apify/instagram/startInstagramCommentsScraping.ts @@ -0,0 +1,45 @@ +import apifyClient from "@/lib/apify/client"; +import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; +import { ApifyRunInfo } from "@/lib/apify/types"; +import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; + +/** + * Starts an Apify Instagram comments scraping run for the given post URLs. + * Registers a webhook pointing at `/api/apify` so results are processed + * in-process by `handleInstagramCommentsScraper` on success. + * + * @param postUrls - Array of Instagram post URLs to fetch comments for. + * @param resultsLimit - Optional max comments per post (default 100). + * @returns ApifyRunInfo with runId + datasetId, or null on failure. + */ +export async function startInstagramCommentsScraping( + postUrls: string[], + resultsLimit = 100, +): Promise { + const urls = (postUrls ?? []).filter(Boolean); + + if (urls.length === 0) { + throw new Error("At least one Instagram post URL is required"); + } + + const run = await apifyClient.actor("SbK00X0JYCPblD2wp").start( + { + directUrls: urls, + resultsLimit, + }, + { webhooks: getApifyWebhooks() }, + ); + + if (!run?.id || !run?.defaultDatasetId) { + console.error("Failed to start Instagram comments scraping for urls:", urls); + return null; + } + + if (run.status === "FAILED" || run.status === "ABORTED") { + throw new Error(OUTSTANDING_ERROR); + } + + return { runId: run.id, datasetId: run.defaultDatasetId }; +} + +export default startInstagramCommentsScraping; diff --git a/lib/apify/instagram/startInstagramProfileScraping.ts b/lib/apify/instagram/startInstagramProfileScraping.ts index a07cbec95..64e44ec23 100644 --- a/lib/apify/instagram/startInstagramProfileScraping.ts +++ b/lib/apify/instagram/startInstagramProfileScraping.ts @@ -1,20 +1,32 @@ import apifyClient from "@/lib/apify/client"; import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; import { ApifyRunInfo } from "@/lib/apify/types"; +import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; -const startInstagramProfileScraping = async (handle: string): Promise => { - const cleanHandle = handle.trim().replace(/^@/, ""); +/** + * Starts an Apify Instagram profile scraping run for one or more handles. + * Registers a `webhooks` payload pointing at this service's + * `/api/apify` receiver so follow-up processing runs in-process. + * + * @param handles - A single handle or array of handles to scrape. + * @returns ApifyRunInfo with runId + datasetId, or null on failure. + */ +export async function startInstagramProfileScraping( + handles: string | string[], +): Promise { + const list = Array.isArray(handles) ? handles : [handles]; + const cleanHandles = list.map(h => h.trim().replace(/^@/, "")).filter(h => h.length > 0); - if (!cleanHandle) { + if (cleanHandles.length === 0) { throw new Error("Invalid Instagram handle"); } - const run = await apifyClient.actor("apify~instagram-profile-scraper").start({ - usernames: [cleanHandle], - }); + const run = await apifyClient + .actor("apify~instagram-profile-scraper") + .start({ usernames: cleanHandles }, { webhooks: getApifyWebhooks() }); if (!run?.id || !run?.defaultDatasetId) { - console.error("Failed to start Instagram profile scraping for handle:", handle); + console.error("Failed to start Instagram profile scraping for handles:", cleanHandles); return null; } @@ -23,6 +35,6 @@ const startInstagramProfileScraping = async (handle: string): Promise { + const validation = await validateApifyWebhookRequest(request); + + if (!validation.ok || !validation.data) { + console.warn("[WARN] postApifyWebhookHandler: invalid payload:", validation.error); + return NextResponse.json( + { message: "Invalid payload", error: validation.error }, + { status: 200 }, + ); + } + + try { + const result = await handleApifyWebhook(validation.data); + return NextResponse.json(result, { status: 200 }); + } catch (error) { + console.error("[ERROR] postApifyWebhookHandler:", error); + return NextResponse.json( + { message: "Apify webhook received (handler error)" }, + { status: 200 }, + ); + } +} diff --git a/lib/apify/sendApifyWebhookEmail.ts b/lib/apify/sendApifyWebhookEmail.ts new file mode 100644 index 000000000..18eb3749f --- /dev/null +++ b/lib/apify/sendApifyWebhookEmail.ts @@ -0,0 +1,50 @@ +import generateText from "@/lib/ai/generateText"; +import { sendEmailWithResend } from "@/lib/emails/sendEmail"; +import { RECOUP_FROM_EMAIL } from "@/lib/const"; + +/** + * Sends an Apify-webhook summary email to the given recipients using + * Resend. Generates the email body via an LLM based on the first + * profile result. + * + * @param profile - First dataset result from the profile scraper. + * @param emails - Recipient email addresses. + * @returns The Resend response, or `null` when there are no recipients. + */ +export async function sendApifyWebhookEmail(profile: Record, emails: string[]) { + if (!emails?.length) return null; + + const prompt = `You have a new Apify dataset update. Here is the data: + +Key Data +Full Name: ${profile.fullName} +Username: ${profile.username} +Profile URL: ${profile.url} +Profile Picture: ${profile.profilePicUrl} +Biography: ${profile.biography} +External URL: ${profile.externalUrls} +Followers: ${profile.followersCount} +Following: ${profile.followsCount} +Latest Posts: ${((profile.latestPosts as unknown[]) || []).map(p => JSON.stringify(p)).join(", ")} +`; + + const { text } = await generateText({ + system: `you are a record label services manager for Recoup. + write beautiful html email. + subject: New Apify Dataset Notification. you're notifying music managers about new posts being available for one of their roster musician's Instagram profile. + include a link to view the instagram profile. + call to action is to open a chat link to learn more about the latest posts using Recoup Chat (AI Agents): https://chat.recoupable.com/?q=tell%20me%20about%20my%20latest%20Ig%20posts + You'll be passed a dataset summary for a musician profile and their latest posts on instagram. + your goal is to get the recipient to click a cta link to open a chat link to learn more about the latest posts using Recoup Chat (AI Agents). + only include the email body html. + no headers or subject`, + prompt, + }); + + return await sendEmailWithResend({ + from: RECOUP_FROM_EMAIL, + to: emails, + subject: `${profile.fullName} has new posts on Instagram`, + html: text, + }); +} diff --git a/lib/apify/types.ts b/lib/apify/types.ts index c1493634a..1468c4261 100644 --- a/lib/apify/types.ts +++ b/lib/apify/types.ts @@ -4,3 +4,46 @@ export interface ApifyRunInfo { error?: string; data?: unknown; } + +export type ApifyInstagramPost = { + id: string; + type: string; + shortCode: string; + caption: string; + hashtags: string[]; + mentions: string[]; + url: string; + commentsCount: number; + dimensionsHeight: number; + dimensionsWidth: number; + displayUrl: string; + images: string[]; + alt: string; + likesCount: number; + timestamp: string; + childPosts: ApifyInstagramPost[]; + ownerUsername: string; + ownerId: string; + isCommentsDisabled: boolean; +}; + +export interface ApifyInstagramComment { + id: string; + text: string; + timestamp: string; + ownerUsername: string; + ownerProfilePicUrl: string; + postUrl: string; +} + +export interface ApifyInstagramProfileResult { + latestPosts?: ApifyInstagramPost[]; + profilePicUrlHD?: string; + profilePicUrl?: string; + username?: string; + url?: string; + biography?: string; + followersCount?: number; + followsCount?: number; + fullName?: string; +} diff --git a/lib/apify/validateApifyWebhookRequest.ts b/lib/apify/validateApifyWebhookRequest.ts new file mode 100644 index 000000000..f84ff062f --- /dev/null +++ b/lib/apify/validateApifyWebhookRequest.ts @@ -0,0 +1,34 @@ +import { NextRequest } from "next/server"; +import { apifyPayloadSchema, type ApifyPayload } from "@/lib/apify/apifyPayloadSchema"; + +export interface ApifyWebhookValidationResult { + ok: boolean; + data?: ApifyPayload; + error?: string; +} + +/** + * Parses and validates a `POST /api/apify` request body against the + * `apifyPayloadSchema`. Returns `{ ok: true, data }` on success and + * `{ ok: false, error }` when parsing or schema validation fails. The + * route always returns 200 regardless, matching Apify's retry posture. + * + * @param request - Incoming webhook request. + */ +export async function validateApifyWebhookRequest( + request: NextRequest, +): Promise { + let body: unknown; + try { + body = await request.json(); + } catch { + return { ok: false, error: "Invalid JSON" }; + } + + const parsed = apifyPayloadSchema.safeParse(body); + if (!parsed.success) { + return { ok: false, error: parsed.error.issues[0]?.message ?? "Invalid payload" }; + } + + return { ok: true, data: parsed.data }; +} From 476c98e5a9f890f8812ea040cde3a6ad90cf0842 Mon Sep 17 00:00:00 2001 From: Arpit Gupta Date: Sat, 25 Apr 2026 01:17:49 +0530 Subject: [PATCH 03/23] fix(api): apify webhook URL, field-wipe, tests, default exports - getApifyWebhooks: derive receiver URL from VERCEL_ENV/VERCEL_URL so preview deploys report to themselves, not prod. - insertSocials: strip null/undefined fields before upsert so the comments pipeline no longer wipes bio/region/followerCount on existing rows. - Add unit tests for handleInstagramProfileFollowUpRuns covering the multi-profile gate, empty latestPosts guard, and the two-way fan-out between urlsWithComments (resultsLimit=1) and urlsWithoutComments. - Drop redundant default exports on startInstagramProfileScraping / startInstagramCommentsScraping and convert the one remaining caller in scrapeProfileUrl.ts to a named import. --- lib/apify/getApifyWebhooks.ts | 18 ++++- ...handleInstagramProfileFollowUpRuns.test.ts | 68 +++++++++++++++++++ .../startInstagramCommentsScraping.ts | 2 - .../startInstagramProfileScraping.ts | 2 - lib/apify/scrapeProfileUrl.ts | 2 +- lib/supabase/socials/insertSocials.ts | 19 +++++- 6 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts diff --git a/lib/apify/getApifyWebhooks.ts b/lib/apify/getApifyWebhooks.ts index b47276a2b..2b433ede3 100644 --- a/lib/apify/getApifyWebhooks.ts +++ b/lib/apify/getApifyWebhooks.ts @@ -1,4 +1,20 @@ import type { WebhookEventType, WebhookUpdateData } from "apify-client"; +import { NEW_API_BASE_URL } from "@/lib/consts"; + +/** + * Resolves the base URL Apify should POST its webhook to. Preview deploys + * must report to themselves, not prod — otherwise preview runs pollute the + * production DB via the shared webhook receiver. + */ +const getReceiverBaseUrl = (): string => { + if (process.env.VERCEL_ENV === "production") { + return NEW_API_BASE_URL; + } + if (process.env.VERCEL_URL) { + return `https://${process.env.VERCEL_URL}`; + } + return "http://localhost:3000"; +}; /** * Webhook config registered on Apify actor runs. The Apify client @@ -10,7 +26,7 @@ export function getApifyWebhooks(): WebhookUpdateData[] { return [ { eventTypes, - requestUrl: "https://recoup-api.vercel.app/api/apify", + requestUrl: `${getReceiverBaseUrl()}/api/apify`, }, ]; } diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts new file mode 100644 index 000000000..4d49e6034 --- /dev/null +++ b/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts @@ -0,0 +1,68 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { handleInstagramProfileFollowUpRuns } from "../handleInstagramProfileFollowUpRuns"; +import { startInstagramCommentsScraping } from "../startInstagramCommentsScraping"; +import { getExistingPostComments } from "../getExistingPostComments"; +import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; + +vi.mock("../startInstagramCommentsScraping", () => ({ + startInstagramCommentsScraping: vi.fn(), +})); +vi.mock("../getExistingPostComments", () => ({ + getExistingPostComments: vi.fn(), +})); + +const baseProfile: ApifyInstagramProfileResult = { + username: "alice", + url: "https://instagram.com/alice", + profilePicUrl: "", + profilePicUrlHD: "", + biography: "", + followersCount: 0, + followsCount: 0, + latestPosts: [], +} as unknown as ApifyInstagramProfileResult; + +describe("handleInstagramProfileFollowUpRuns", () => { + beforeEach(() => vi.clearAllMocks()); + + it("does not kick off comments scrape when dataset has more than one profile", async () => { + const dataset = [baseProfile, baseProfile, baseProfile]; + + await handleInstagramProfileFollowUpRuns(dataset, { + ...baseProfile, + latestPosts: [{ url: "https://instagram.com/p/1" }], + } as ApifyInstagramProfileResult); + + expect(startInstagramCommentsScraping).not.toHaveBeenCalled(); + expect(getExistingPostComments).not.toHaveBeenCalled(); + }); + + it("does not kick off comments scrape when latestPosts is empty", async () => { + await handleInstagramProfileFollowUpRuns([baseProfile], { + ...baseProfile, + latestPosts: [], + } as ApifyInstagramProfileResult); + + expect(startInstagramCommentsScraping).not.toHaveBeenCalled(); + expect(getExistingPostComments).not.toHaveBeenCalled(); + }); + + it("fans out two scrapes: resultsLimit=1 for seen urls, default for unseen", async () => { + const url1 = "https://instagram.com/p/1"; + const url2 = "https://instagram.com/p/2"; + + vi.mocked(getExistingPostComments).mockResolvedValue({ + urlsWithComments: [url1], + urlsWithoutComments: [url2], + }); + + await handleInstagramProfileFollowUpRuns([baseProfile], { + ...baseProfile, + latestPosts: [{ url: url1 }, { url: url2 }], + } as ApifyInstagramProfileResult); + + expect(startInstagramCommentsScraping).toHaveBeenCalledTimes(2); + expect(startInstagramCommentsScraping).toHaveBeenCalledWith([url1], 1); + expect(startInstagramCommentsScraping).toHaveBeenCalledWith([url2]); + }); +}); diff --git a/lib/apify/instagram/startInstagramCommentsScraping.ts b/lib/apify/instagram/startInstagramCommentsScraping.ts index 0ac8d2d0e..89a904bcb 100644 --- a/lib/apify/instagram/startInstagramCommentsScraping.ts +++ b/lib/apify/instagram/startInstagramCommentsScraping.ts @@ -41,5 +41,3 @@ export async function startInstagramCommentsScraping( return { runId: run.id, datasetId: run.defaultDatasetId }; } - -export default startInstagramCommentsScraping; diff --git a/lib/apify/instagram/startInstagramProfileScraping.ts b/lib/apify/instagram/startInstagramProfileScraping.ts index 64e44ec23..34c0cdb61 100644 --- a/lib/apify/instagram/startInstagramProfileScraping.ts +++ b/lib/apify/instagram/startInstagramProfileScraping.ts @@ -36,5 +36,3 @@ export async function startInstagramProfileScraping( return { runId: run.id, datasetId: run.defaultDatasetId }; } - -export default startInstagramProfileScraping; diff --git a/lib/apify/scrapeProfileUrl.ts b/lib/apify/scrapeProfileUrl.ts index 27ed3d234..3453d0556 100644 --- a/lib/apify/scrapeProfileUrl.ts +++ b/lib/apify/scrapeProfileUrl.ts @@ -1,5 +1,5 @@ import startTikTokProfileScraping from "@/lib/apify/tiktok/startTiktokProfileScraping"; -import startInstagramProfileScraping from "@/lib/apify/instagram/startInstagramProfileScraping"; +import { startInstagramProfileScraping } from "@/lib/apify/instagram/startInstagramProfileScraping"; import startTwitterProfileScraping from "@/lib/apify/twitter/startTwitterProfileScraping"; import startThreadsProfileScraping from "@/lib/apify/threads/startThreadsProfileScraping"; import startYoutubeProfileScraping from "@/lib/apify/youtube/startYoutubeProfileScraping"; diff --git a/lib/supabase/socials/insertSocials.ts b/lib/supabase/socials/insertSocials.ts index 19738798a..8fa522708 100644 --- a/lib/supabase/socials/insertSocials.ts +++ b/lib/supabase/socials/insertSocials.ts @@ -2,7 +2,22 @@ import supabase from "../serverClient"; import type { Tables, TablesInsert } from "@/types/database.types"; /** - * Inserts one or more social records. + * Strips null/undefined fields so upsert-on-conflict doesn't clobber + * existing column values when a caller has nothing new to say about + * those fields (e.g. the comments pipeline passes null for bio/region). + */ +const stripNullish = (row: TablesInsert<"socials">): TablesInsert<"socials"> => { + const out: Record = {}; + for (const [k, v] of Object.entries(row)) { + if (v !== null && v !== undefined) out[k] = v; + } + return out as TablesInsert<"socials">; +}; + +/** + * Inserts one or more social records. Upserts on `profile_url`; + * null/undefined fields are dropped so we don't overwrite existing + * non-null values with nulls. * * @param socials - Array of social data to insert * @returns Array of inserted social records @@ -12,7 +27,7 @@ export async function insertSocials( ): Promise[]> { const { data, error } = await supabase .from("socials") - .upsert(socials, { onConflict: "profile_url" }) + .upsert(socials.map(stripNullish), { onConflict: "profile_url" }) .select("*"); if (error) { From 8b6e93d4eb731f3e899d7da5e7e6e87259d8b23e Mon Sep 17 00:00:00 2001 From: Arpit Gupta Date: Sat, 25 Apr 2026 01:33:45 +0530 Subject: [PATCH 04/23] refactor(api): tighten apify chain types + fix profile_url round-trip - type sendApifyWebhookEmail profile param as ApifyInstagramProfileResult - drop unknown[] / Record casts across the profile handler - normalize profile_url before upsert so the post-insert lookup matches - drop unused z.any() fields from apifyPayloadSchema - remove dead ArtistQueryRow import from getAccountArtistIds Co-Authored-By: Claude Opus 4.7 --- lib/apify/apifyPayloadSchema.ts | 8 +-- .../handleInstagramProfileFollowUpRuns.ts | 6 +- .../handleInstagramProfileScraperResults.ts | 55 ++++++++++++------- lib/apify/sendApifyWebhookEmail.ts | 11 ++-- .../account_artist_ids/getAccountArtistIds.ts | 11 +--- 5 files changed, 48 insertions(+), 43 deletions(-) diff --git a/lib/apify/apifyPayloadSchema.ts b/lib/apify/apifyPayloadSchema.ts index d2890cb0d..3a2508127 100644 --- a/lib/apify/apifyPayloadSchema.ts +++ b/lib/apify/apifyPayloadSchema.ts @@ -2,13 +2,11 @@ import { z } from "zod"; /** * Zod schema for Apify webhook payloads. Only validates the fields we - * branch on + read downstream; the rest of the payload is intentionally - * permissive so Apify schema drift does not drop events. + * branch on + read downstream (actorId for dispatch, datasetId for + * fetch); extra keys from Apify are stripped silently so upstream + * schema drift does not drop events. */ export const apifyPayloadSchema = z.object({ - userId: z.any(), - createdAt: z.any(), - eventType: z.any(), eventData: z.object({ actorId: z.string(), }), diff --git a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts index 9351b1099..f49204fc5 100644 --- a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts +++ b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts @@ -1,6 +1,6 @@ import { startInstagramCommentsScraping } from "@/lib/apify/instagram/startInstagramCommentsScraping"; import { getExistingPostComments } from "@/lib/apify/instagram/getExistingPostComments"; -import type { ApifyInstagramPost, ApifyInstagramProfileResult } from "@/lib/apify/types"; +import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; /** * Kicks off a comments-scraper run for the newly-seen posts on a @@ -21,9 +21,7 @@ export async function handleInstagramProfileFollowUpRuns( if (dataset.length !== 1) return; if (!firstResult.latestPosts || firstResult.latestPosts.length === 0) return; - const postUrls = (firstResult.latestPosts as ApifyInstagramPost[]) - .map(post => post.url) - .filter(Boolean); + const postUrls = firstResult.latestPosts.map(post => post.url).filter(Boolean); if (postUrls.length === 0) return; diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index c8a212229..36488bfab 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -5,14 +5,27 @@ import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; import { insertSocials } from "@/lib/supabase/socials/insertSocials"; import { selectSocialByProfileUrl } from "@/lib/supabase/socials/selectSocialByProfileUrl"; import { insertSocialPosts } from "@/lib/supabase/social_posts/insertSocialPosts"; -import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccountSocials"; +import { + selectAccountSocials, + type AccountSocialWithSocial, +} from "@/lib/supabase/account_socials/selectAccountSocials"; import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccountArtistIds"; import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails"; import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; import { getFetchableUrl } from "@/lib/arweave/getFetchableUrl"; -import type { ApifyInstagramPost, ApifyInstagramProfileResult } from "@/lib/apify/types"; +import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; import type { ApifyPayload } from "@/lib/apify/apifyPayloadSchema"; +import type { Tables } from "@/types/database.types"; + +type ProfileScraperResult = { + posts: Tables<"posts">[]; + social: Tables<"socials"> | null; + accountSocials: AccountSocialWithSocial[]; + accountArtistIds: Awaited>; + accountEmails: Tables<"account_emails">[]; + sentEmails: Awaited>; +}; /** * Handles Instagram profile scraper Apify webhook results: @@ -27,15 +40,17 @@ import type { ApifyPayload } from "@/lib/apify/apifyPayloadSchema"; * * @param parsed - Validated Apify webhook payload. */ -export async function handleInstagramProfileScraperResults(parsed: ApifyPayload) { +export async function handleInstagramProfileScraperResults( + parsed: ApifyPayload, +): Promise { const datasetId = parsed.resource.defaultDatasetId; - const empty = { + const empty: ProfileScraperResult = { posts: [], social: null, - accountSocials: [] as unknown[], - accountArtistIds: [] as unknown[], - accountEmails: [] as unknown[], - sentEmails: null as unknown, + accountSocials: [], + accountArtistIds: [], + accountEmails: [], + sentEmails: null, }; if (!datasetId) return empty; @@ -44,9 +59,7 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyPayload) const firstResult = dataset[0] as ApifyInstagramProfileResult | undefined; if (!firstResult?.latestPosts) return empty; - const { supabasePosts: posts } = await saveApifyInstagramPosts( - firstResult.latestPosts as ApifyInstagramPost[], - ); + const { supabasePosts: posts } = await saveApifyInstagramPosts(firstResult.latestPosts); const arweaveTx = await uploadLinkToArweave( firstResult.profilePicUrlHD || firstResult.profilePicUrl, @@ -55,18 +68,22 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyPayload) firstResult.profilePicUrl = getFetchableUrl(`ar://${arweaveTx}`) ?? firstResult.profilePicUrl; } + // Normalize once so the upsert and the subsequent lookup agree on the + // `profile_url` key — otherwise the lookup misses and the rest of the + // chain (social_posts link, email, follow-up scrape) is short-circuited. + const normalizedUrl = normalizeProfileUrl(firstResult.url); + await insertSocials([ { username: firstResult.username ?? "", avatar: firstResult.profilePicUrl ?? null, - profile_url: firstResult.url ?? "", + profile_url: normalizedUrl, bio: firstResult.biography ?? null, followerCount: firstResult.followersCount ?? null, followingCount: firstResult.followsCount ?? null, }, ]); - const normalizedUrl = normalizeProfileUrl(firstResult.url); const social = await selectSocialByProfileUrl(normalizedUrl); if (!social) { @@ -85,22 +102,18 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyPayload) const accountSocials = await selectAccountSocials({ socialId: social.id, limit: 10000 }); const accountArtistIds = await getAccountArtistIds({ - artistIds: accountSocials.map(a => a.account_id as string), + artistIds: accountSocials.map(a => a.account_id), }); const uniqueAccountIds = Array.from( - new Set( - accountArtistIds - .map(a => (a as unknown as { account_id: string | null }).account_id) - .filter((id): id is string => Boolean(id)), - ), + new Set(accountArtistIds.map(a => a.account_id).filter((id): id is string => Boolean(id))), ); const accountEmails = await selectAccountEmails({ accountIds: uniqueAccountIds }); const sentEmails = await sendApifyWebhookEmail( - firstResult as unknown as Record, - accountEmails.map(e => e.email).filter(Boolean) as string[], + firstResult, + accountEmails.map(e => e.email).filter(Boolean), ); await handleInstagramProfileFollowUpRuns(dataset, firstResult); diff --git a/lib/apify/sendApifyWebhookEmail.ts b/lib/apify/sendApifyWebhookEmail.ts index 18eb3749f..2d9c6db0f 100644 --- a/lib/apify/sendApifyWebhookEmail.ts +++ b/lib/apify/sendApifyWebhookEmail.ts @@ -1,6 +1,7 @@ import generateText from "@/lib/ai/generateText"; import { sendEmailWithResend } from "@/lib/emails/sendEmail"; import { RECOUP_FROM_EMAIL } from "@/lib/const"; +import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; /** * Sends an Apify-webhook summary email to the given recipients using @@ -11,7 +12,10 @@ import { RECOUP_FROM_EMAIL } from "@/lib/const"; * @param emails - Recipient email addresses. * @returns The Resend response, or `null` when there are no recipients. */ -export async function sendApifyWebhookEmail(profile: Record, emails: string[]) { +export async function sendApifyWebhookEmail( + profile: ApifyInstagramProfileResult, + emails: string[], +) { if (!emails?.length) return null; const prompt = `You have a new Apify dataset update. Here is the data: @@ -22,10 +26,9 @@ Username: ${profile.username} Profile URL: ${profile.url} Profile Picture: ${profile.profilePicUrl} Biography: ${profile.biography} -External URL: ${profile.externalUrls} Followers: ${profile.followersCount} Following: ${profile.followsCount} -Latest Posts: ${((profile.latestPosts as unknown[]) || []).map(p => JSON.stringify(p)).join(", ")} +Latest Posts: ${(profile.latestPosts ?? []).map(p => JSON.stringify(p)).join(", ")} `; const { text } = await generateText({ @@ -44,7 +47,7 @@ Latest Posts: ${((profile.latestPosts as unknown[]) || []).map(p => JSON.stringi return await sendEmailWithResend({ from: RECOUP_FROM_EMAIL, to: emails, - subject: `${profile.fullName} has new posts on Instagram`, + subject: `${profile.fullName ?? profile.username ?? "Your artist"} has new posts on Instagram`, html: text, }); } diff --git a/lib/supabase/account_artist_ids/getAccountArtistIds.ts b/lib/supabase/account_artist_ids/getAccountArtistIds.ts index e4e6b8099..def44ded7 100644 --- a/lib/supabase/account_artist_ids/getAccountArtistIds.ts +++ b/lib/supabase/account_artist_ids/getAccountArtistIds.ts @@ -1,8 +1,4 @@ import supabase from "../serverClient"; -import type { ArtistQueryRow } from "@/lib/artists/getFormattedArtist"; - -// Raw row type returned by this query -export type AccountArtistRow = ArtistQueryRow & { artist_id: string; pinned: boolean }; /** * Get all artists for an array of artist IDs or account IDs, with full info. @@ -11,10 +7,7 @@ export type AccountArtistRow = ArtistQueryRow & { artist_id: string; pinned: boo * @param params Object with artistIds or accountIds array * @returns Array of raw artist rows from database */ -export async function getAccountArtistIds(params: { - artistIds?: string[]; - accountIds?: string[]; -}): Promise { +export async function getAccountArtistIds(params: { artistIds?: string[]; accountIds?: string[] }) { const { artistIds, accountIds } = params; if (!artistIds && !accountIds) { throw new Error("Must provide either artistIds or accountIds"); @@ -44,5 +37,5 @@ export async function getAccountArtistIds(params: { return []; } - return (data || []) as unknown as AccountArtistRow[]; + return data || []; } From 82e02c62a5f5433d42e92fe3a699feec132e8906 Mon Sep 17 00:00:00 2001 From: Arpit Gupta Date: Sat, 25 Apr 2026 01:42:56 +0530 Subject: [PATCH 05/23] refactor(api): address cubic feedback on apify chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - getDataset: send APIFY_TOKEN via Authorization header; wrap in try/catch - uploadLinkToArweave: reject private/loopback URLs; cap download at 10MB - propagate errors from insertPosts / insertSocials / insertSocialPosts / selectSocialByProfileUrl / getPostsByUrls instead of swallowing — the webhook handler's outer catch still guarantees a 200 response - saveApifyInstagramComments: skip rows missing timestamp; rethrow errors - getOrCreatePostsForComments: reuse existing rows when nothing inserted - getOrCreateSocialsForComments: drop swallow-and-return-empty fallback - getExistingPostComments: use a dedicated lightweight existence query (selectPostUrlsWithComments) instead of full comment rows with joins; drop now-unused selectPostComments - startInstagramCommentsScraping: drop status check that is unreliable immediately after actor.start() - handleInstagramProfileFollowUpRuns: type-guard post.url from unvalidated webhook payload before using as string - sendApifyWebhookEmail: Array.isArray guard before .map on latestPosts Co-Authored-By: Claude Opus 4.7 --- lib/apify/getDataset.ts | 27 +++++---- .../getOrCreateSocialsForComments.test.ts | 26 ++++----- .../instagram/getExistingPostComments.ts | 12 ++-- .../instagram/getOrCreatePostsForComments.ts | 4 +- .../getOrCreateSocialsForComments.ts | 17 ++---- .../handleInstagramProfileFollowUpRuns.ts | 4 +- .../instagram/saveApifyInstagramComments.ts | 3 +- .../startInstagramCommentsScraping.ts | 5 -- lib/apify/sendApifyWebhookEmail.ts | 2 +- lib/arweave/uploadLinkToArweave.ts | 58 ++++++++++++++++++- .../post_comments/selectPostComments.ts | 45 -------------- .../selectPostUrlsWithComments.ts | 40 +++++++++++++ lib/supabase/posts/getPostsByUrls.ts | 2 +- lib/supabase/posts/insertPosts.ts | 1 + .../social_posts/insertSocialPosts.ts | 1 + lib/supabase/socials/insertSocials.ts | 2 +- .../socials/selectSocialByProfileUrl.ts | 9 ++- 17 files changed, 154 insertions(+), 104 deletions(-) delete mode 100644 lib/supabase/post_comments/selectPostComments.ts create mode 100644 lib/supabase/post_comments/selectPostUrlsWithComments.ts diff --git a/lib/apify/getDataset.ts b/lib/apify/getDataset.ts index 816389f12..b6f1ae838 100644 --- a/lib/apify/getDataset.ts +++ b/lib/apify/getDataset.ts @@ -13,19 +13,24 @@ export async function getDataset(datasetId: string): Promise { return []; } - const response = await fetch( - `https://api.apify.com/v2/datasets/${datasetId}/items?token=${token}`, - { + try { + const response = await fetch(`https://api.apify.com/v2/datasets/${datasetId}/items`, { method: "GET", - headers: { "Content-Type": "application/json" }, - }, - ); + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + }, + }); - if (!response.ok) { - console.error(`[ERROR] getDataset: ${response.status} ${response.statusText}`); + if (!response.ok) { + console.error(`[ERROR] getDataset: ${response.status} ${response.statusText}`); + return []; + } + + const data = await response.json(); + return Array.isArray(data) ? data : []; + } catch (err) { + console.error("[ERROR] getDataset:", err); return []; } - - const data = await response.json(); - return Array.isArray(data) ? data : []; } diff --git a/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts b/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts index 8ad8adfd6..753030c7b 100644 --- a/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts +++ b/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts @@ -47,20 +47,20 @@ describe("getOrCreateSocialsForComments", () => { expect(result.get("bob")?.id).toBe("s2"); }); - it("returns empty map and swallows DB errors", async () => { + it("propagates DB errors so the webhook handler can log + short-circuit", async () => { vi.mocked(insertSocials).mockRejectedValue(new Error("boom")); - const result = await getOrCreateSocialsForComments([ - { - id: "c1", - text: "t", - timestamp: "2026-01-01", - ownerUsername: "alice", - ownerProfilePicUrl: "https://a", - postUrl: "u1", - }, - ]); - - expect(result.size).toBe(0); + await expect( + getOrCreateSocialsForComments([ + { + id: "c1", + text: "t", + timestamp: "2026-01-01", + ownerUsername: "alice", + ownerProfilePicUrl: "https://a", + postUrl: "u1", + }, + ]), + ).rejects.toThrow("boom"); }); }); diff --git a/lib/apify/instagram/getExistingPostComments.ts b/lib/apify/instagram/getExistingPostComments.ts index 286928b85..6d5d877f9 100644 --- a/lib/apify/instagram/getExistingPostComments.ts +++ b/lib/apify/instagram/getExistingPostComments.ts @@ -1,4 +1,4 @@ -import { selectPostComments } from "@/lib/supabase/post_comments/selectPostComments"; +import { selectPostUrlsWithComments } from "@/lib/supabase/post_comments/selectPostUrlsWithComments"; /** * Partitions `postUrls` into those that already have comments in @@ -16,14 +16,10 @@ export async function getExistingPostComments(postUrls: string[]): Promise<{ } try { - const existing = await selectPostComments({ postUrls }); - - const withComments = existing - .map(c => c.post?.post_url) - .filter((url): url is string => Boolean(url)); - + const withComments = await selectPostUrlsWithComments(postUrls); const urlsWithComments = Array.from(new Set(withComments)); - const urlsWithoutComments = postUrls.filter(url => !urlsWithComments.includes(url)); + const withSet = new Set(urlsWithComments); + const urlsWithoutComments = postUrls.filter(url => !withSet.has(url)); return { urlsWithComments, urlsWithoutComments }; } catch (error) { diff --git a/lib/apify/instagram/getOrCreatePostsForComments.ts b/lib/apify/instagram/getOrCreatePostsForComments.ts index 33a6e2f17..d8cdeb6fe 100644 --- a/lib/apify/instagram/getOrCreatePostsForComments.ts +++ b/lib/apify/instagram/getOrCreatePostsForComments.ts @@ -19,15 +19,17 @@ export async function getOrCreatePostsForComments( const existingSet = new Set(existing.map(p => p.post_url)); const missing = unique.filter(url => !existingSet.has(url)); + + let all = existing; if (missing.length > 0) { const rows: TablesInsert<"posts">[] = missing.map(url => ({ post_url: url, updated_at: new Date().toISOString(), })); await insertPosts(rows); + all = await getPostsByUrls(unique); } - const all = await getPostsByUrls(unique); const map = new Map>(); all.forEach(post => { map.set(post.post_url, post); diff --git a/lib/apify/instagram/getOrCreateSocialsForComments.ts b/lib/apify/instagram/getOrCreateSocialsForComments.ts index 80a516880..2ce9c571a 100644 --- a/lib/apify/instagram/getOrCreateSocialsForComments.ts +++ b/lib/apify/instagram/getOrCreateSocialsForComments.ts @@ -38,15 +38,10 @@ export async function getOrCreateSocialsForComments( followingCount: null, })); - try { - const upserted = await insertSocials(rows); - const map = new Map>(); - upserted.forEach(social => { - map.set(social.username, social); - }); - return map; - } catch (error) { - console.error("[ERROR] getOrCreateSocialsForComments:", error); - return new Map(); - } + const upserted = await insertSocials(rows); + const map = new Map>(); + upserted.forEach(social => { + map.set(social.username, social); + }); + return map; } diff --git a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts index f49204fc5..b9abea26d 100644 --- a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts +++ b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts @@ -21,7 +21,9 @@ export async function handleInstagramProfileFollowUpRuns( if (dataset.length !== 1) return; if (!firstResult.latestPosts || firstResult.latestPosts.length === 0) return; - const postUrls = firstResult.latestPosts.map(post => post.url).filter(Boolean); + const postUrls = firstResult.latestPosts + .map(post => (post && typeof post.url === "string" ? post.url : "")) + .filter(Boolean); if (postUrls.length === 0) return; diff --git a/lib/apify/instagram/saveApifyInstagramComments.ts b/lib/apify/instagram/saveApifyInstagramComments.ts index 521291c04..b6b4662e7 100644 --- a/lib/apify/instagram/saveApifyInstagramComments.ts +++ b/lib/apify/instagram/saveApifyInstagramComments.ts @@ -22,7 +22,7 @@ export async function saveApifyInstagramComments(comments: ApifyInstagramComment const rows: TablesInsert<"post_comments">[] = []; for (const comment of comments) { - if (!comment.postUrl || !comment.ownerUsername) continue; + if (!comment.postUrl || !comment.ownerUsername || !comment.timestamp) continue; const post = postsMap.get(comment.postUrl); const social = socialsMap.get(comment.ownerUsername); @@ -47,5 +47,6 @@ export async function saveApifyInstagramComments(comments: ApifyInstagramComment } } catch (error) { console.error("[ERROR] saveApifyInstagramComments:", error); + throw error; } } diff --git a/lib/apify/instagram/startInstagramCommentsScraping.ts b/lib/apify/instagram/startInstagramCommentsScraping.ts index 89a904bcb..8a42444ce 100644 --- a/lib/apify/instagram/startInstagramCommentsScraping.ts +++ b/lib/apify/instagram/startInstagramCommentsScraping.ts @@ -1,5 +1,4 @@ import apifyClient from "@/lib/apify/client"; -import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; import { ApifyRunInfo } from "@/lib/apify/types"; import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; @@ -35,9 +34,5 @@ export async function startInstagramCommentsScraping( return null; } - if (run.status === "FAILED" || run.status === "ABORTED") { - throw new Error(OUTSTANDING_ERROR); - } - return { runId: run.id, datasetId: run.defaultDatasetId }; } diff --git a/lib/apify/sendApifyWebhookEmail.ts b/lib/apify/sendApifyWebhookEmail.ts index 2d9c6db0f..29ec19a94 100644 --- a/lib/apify/sendApifyWebhookEmail.ts +++ b/lib/apify/sendApifyWebhookEmail.ts @@ -28,7 +28,7 @@ Profile Picture: ${profile.profilePicUrl} Biography: ${profile.biography} Followers: ${profile.followersCount} Following: ${profile.followsCount} -Latest Posts: ${(profile.latestPosts ?? []).map(p => JSON.stringify(p)).join(", ")} +Latest Posts: ${(Array.isArray(profile.latestPosts) ? profile.latestPosts : []).map(p => JSON.stringify(p)).join(", ")} `; const { text } = await generateText({ diff --git a/lib/arweave/uploadLinkToArweave.ts b/lib/arweave/uploadLinkToArweave.ts index d550c1b39..af7182ee0 100644 --- a/lib/arweave/uploadLinkToArweave.ts +++ b/lib/arweave/uploadLinkToArweave.ts @@ -1,21 +1,73 @@ import { uploadToArweave } from "./uploadToArweave"; +const MAX_IMAGE_BYTES = 10 * 1024 * 1024; + +function isSafeHttpUrl(raw: string): boolean { + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + return false; + } + if (parsed.protocol !== "https:" && parsed.protocol !== "http:") return false; + const host = parsed.hostname.toLowerCase(); + if (!host) return false; + if (host === "localhost" || host.endsWith(".localhost")) return false; + if ( + /^(10\.|127\.|0\.|169\.254\.|192\.168\.)/.test(host) || + /^172\.(1[6-9]|2\d|3[01])\./.test(host) || + host === "::1" || + host.startsWith("fc") || + host.startsWith("fd") + ) { + return false; + } + return true; +} + /** * Fetches the image at `imageUrl` and uploads its bytes to Arweave. * Returns the Arweave transaction id (without the `ar://` prefix) on * success, or `null` if fetch / upload fails — callers should fall * back to the original URL in that case. * + * Rejects non-http(s) URLs and private/loopback hosts to avoid SSRF, + * and caps the download at MAX_IMAGE_BYTES so a malicious server + * cannot exhaust memory. + * * @param imageUrl - Remote image URL. */ export async function uploadLinkToArweave( imageUrl: string | null | undefined, ): Promise { if (!imageUrl) return null; + if (!isSafeHttpUrl(imageUrl)) { + console.error("[ERROR] uploadLinkToArweave: rejected unsafe url"); + return null; + } try { - const res = await fetch(imageUrl); - if (!res.ok) return null; - const buffer = Buffer.from(await res.arrayBuffer()); + const res = await fetch(imageUrl, { redirect: "error" }); + if (!res.ok || !res.body) return null; + + const contentLength = Number(res.headers.get("content-length") ?? 0); + if (contentLength > MAX_IMAGE_BYTES) return null; + + const reader = res.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (!value) continue; + total += value.byteLength; + if (total > MAX_IMAGE_BYTES) { + await reader.cancel(); + return null; + } + chunks.push(value); + } + + const buffer = Buffer.concat(chunks.map(c => Buffer.from(c))); const contentType = res.headers.get("content-type") || "image/png"; const transaction = await uploadToArweave(buffer, contentType); diff --git a/lib/supabase/post_comments/selectPostComments.ts b/lib/supabase/post_comments/selectPostComments.ts deleted file mode 100644 index 18777d010..000000000 --- a/lib/supabase/post_comments/selectPostComments.ts +++ /dev/null @@ -1,45 +0,0 @@ -import supabase from "../serverClient"; -import type { Tables } from "@/types/database.types"; -import { getPostsByUrls } from "@/lib/supabase/posts/getPostsByUrls"; - -export type PostCommentWithRelations = Tables<"post_comments"> & { - post?: Tables<"posts"> | null; - social?: Tables<"socials"> | null; -}; - -/** - * Selects `post_comments` rows with joined post + social rows. When - * `postUrls` is provided, comments are filtered to those whose - * underlying post URL matches one of the values. - * - * @param params.postUrls - Optional list of post URLs to restrict to. - */ -export async function selectPostComments({ - postUrls, -}: { - postUrls?: string[]; -} = {}): Promise { - let query = supabase.from("post_comments").select(` - *, - post:posts(*), - social:socials(*) - `); - - if (postUrls && postUrls.length > 0) { - const posts = await getPostsByUrls(postUrls); - if (posts.length === 0) return []; - query = query.in( - "post_id", - posts.map(p => p.id), - ); - } - - const { data, error } = await query; - - if (error) { - console.error("[ERROR] selectPostComments:", error); - throw error; - } - - return (data as PostCommentWithRelations[] | null) ?? []; -} diff --git a/lib/supabase/post_comments/selectPostUrlsWithComments.ts b/lib/supabase/post_comments/selectPostUrlsWithComments.ts new file mode 100644 index 000000000..e343b2909 --- /dev/null +++ b/lib/supabase/post_comments/selectPostUrlsWithComments.ts @@ -0,0 +1,40 @@ +import supabase from "../serverClient"; + +/** + * Returns the subset of `postUrls` for which at least one row exists in + * `post_comments`. Used by the Apify follow-up dispatcher to decide + * whether a post needs a full comments backfill or just a cheap refresh. + * + * Implemented as a narrow two-step lookup (post ids, then a distinct + * `post_id` scan on `post_comments`) so we don't pull whole comment + * rows with joins just to answer an existence question. + */ +export async function selectPostUrlsWithComments(postUrls: string[]): Promise { + if (postUrls.length === 0) return []; + + const { data: posts, error: postsErr } = await supabase + .from("posts") + .select("id, post_url") + .in("post_url", postUrls); + + if (postsErr) { + console.error("[ERROR] selectPostUrlsWithComments (posts):", postsErr); + throw postsErr; + } + + if (!posts || posts.length === 0) return []; + + const postIds = posts.map(p => p.id); + const { data: comments, error: commentsErr } = await supabase + .from("post_comments") + .select("post_id") + .in("post_id", postIds); + + if (commentsErr) { + console.error("[ERROR] selectPostUrlsWithComments (post_comments):", commentsErr); + throw commentsErr; + } + + const postIdsWithComments = new Set((comments ?? []).map(c => c.post_id)); + return posts.filter(p => postIdsWithComments.has(p.id)).map(p => p.post_url); +} diff --git a/lib/supabase/posts/getPostsByUrls.ts b/lib/supabase/posts/getPostsByUrls.ts index 6d51bc226..52b2c38db 100644 --- a/lib/supabase/posts/getPostsByUrls.ts +++ b/lib/supabase/posts/getPostsByUrls.ts @@ -14,7 +14,7 @@ export async function getPostsByUrls(postUrls: string[]): Promise[]) { if (error) { console.error("[ERROR] insertPosts:", error); + throw error; } return { data, error }; diff --git a/lib/supabase/social_posts/insertSocialPosts.ts b/lib/supabase/social_posts/insertSocialPosts.ts index 81e90cdaf..2fce08887 100644 --- a/lib/supabase/social_posts/insertSocialPosts.ts +++ b/lib/supabase/social_posts/insertSocialPosts.ts @@ -15,6 +15,7 @@ export async function insertSocialPosts(socialPosts: TablesInsert<"social_posts" if (error) { console.error("[ERROR] insertSocialPosts:", error); + throw error; } return { data, error }; diff --git a/lib/supabase/socials/insertSocials.ts b/lib/supabase/socials/insertSocials.ts index 8fa522708..9d49e2eba 100644 --- a/lib/supabase/socials/insertSocials.ts +++ b/lib/supabase/socials/insertSocials.ts @@ -32,7 +32,7 @@ export async function insertSocials( if (error) { console.error("[ERROR] insertSocials:", error); - return []; + throw error; } return data || []; diff --git a/lib/supabase/socials/selectSocialByProfileUrl.ts b/lib/supabase/socials/selectSocialByProfileUrl.ts index d0aff8cd4..138f552d5 100644 --- a/lib/supabase/socials/selectSocialByProfileUrl.ts +++ b/lib/supabase/socials/selectSocialByProfileUrl.ts @@ -12,12 +12,17 @@ export async function selectSocialByProfileUrl( ): Promise | null> { if (!profileUrl) return null; - const { data } = await supabase + const { data, error } = await supabase .from("socials") .select("*") .eq("profile_url", profileUrl) .neq("profile_url", "") - .single(); + .maybeSingle(); + + if (error) { + console.error("[ERROR] selectSocialByProfileUrl:", error); + throw error; + } return data ?? null; } From ad297ae38f7cac30924ea2900cc1a80947227021 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Apr 2026 18:51:40 +0000 Subject: [PATCH 06/23] refactor(api): address sweetman feedback on apify chain (PR #483) - Extract getReceiverBaseUrl, isSafeHttpUrl, stripNullish into their own lib files (one-exported-function-per-file SRP). - Rename supabase helpers to match the SDK call they wrap: insertPostComments -> upsertPostComments, insertPosts -> upsertPosts, insertSocialPosts -> upsertSocialPosts. - Rename getPostsByUrls -> getPosts and accept optional postUrls/postIds filters so it's reusable beyond the apify chain. - Split selectPostUrlsWithComments into single-table helpers: getPosts (URL -> id) + new selectPostComments (post_id existence), with selectPostUrlsWithComments as the chaining parent. - Drop selectSocialByProfileUrl in favor of the existing selectSocials({ profile_url }) helper. - Drop explicit Promise<> return types on supabase libs since the SDK already types the return shape. - Update all callers (saveApifyInstagramPosts, saveApifyInstagramComments, getOrCreatePostsForComments, handleInstagramProfileScraperResults) and their tests. Verification: pnpm lint:check clean, pnpm test 2312/2312 pass. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- lib/apify/getApifyWebhooks.ts | 17 +-------- lib/apify/getReceiverBaseUrl.ts | 16 ++++++++ .../getOrCreatePostsForComments.test.ts | 20 +++++----- ...ndleInstagramProfileScraperResults.test.ts | 14 +++---- .../instagram/getOrCreatePostsForComments.ts | 10 ++--- .../handleInstagramProfileScraperResults.ts | 9 +++-- .../instagram/saveApifyInstagramComments.ts | 4 +- .../instagram/saveApifyInstagramPosts.ts | 8 ++-- lib/arweave/isSafeHttpUrl.ts | 27 ++++++++++++++ lib/arweave/uploadLinkToArweave.ts | 24 +----------- .../post_comments/selectPostComments.ts | 30 +++++++++++++++ .../selectPostUrlsWithComments.ts | 35 +++++------------- ...tPostComments.ts => upsertPostComments.ts} | 10 ++--- lib/supabase/posts/getPosts.ts | 37 +++++++++++++++++++ lib/supabase/posts/getPostsByUrls.ts | 21 ----------- .../posts/{insertPosts.ts => upsertPosts.ts} | 4 +- ...ertSocialPosts.ts => upsertSocialPosts.ts} | 4 +- lib/supabase/socials/insertSocials.ts | 20 ++-------- .../socials/selectSocialByProfileUrl.ts | 28 -------------- lib/utils/stripNullish.ts | 13 +++++++ 20 files changed, 178 insertions(+), 173 deletions(-) create mode 100644 lib/apify/getReceiverBaseUrl.ts create mode 100644 lib/arweave/isSafeHttpUrl.ts create mode 100644 lib/supabase/post_comments/selectPostComments.ts rename lib/supabase/post_comments/{insertPostComments.ts => upsertPostComments.ts} (65%) create mode 100644 lib/supabase/posts/getPosts.ts delete mode 100644 lib/supabase/posts/getPostsByUrls.ts rename lib/supabase/posts/{insertPosts.ts => upsertPosts.ts} (83%) rename lib/supabase/social_posts/{insertSocialPosts.ts => upsertSocialPosts.ts} (82%) delete mode 100644 lib/supabase/socials/selectSocialByProfileUrl.ts create mode 100644 lib/utils/stripNullish.ts diff --git a/lib/apify/getApifyWebhooks.ts b/lib/apify/getApifyWebhooks.ts index 2b433ede3..ea4fd6204 100644 --- a/lib/apify/getApifyWebhooks.ts +++ b/lib/apify/getApifyWebhooks.ts @@ -1,20 +1,5 @@ import type { WebhookEventType, WebhookUpdateData } from "apify-client"; -import { NEW_API_BASE_URL } from "@/lib/consts"; - -/** - * Resolves the base URL Apify should POST its webhook to. Preview deploys - * must report to themselves, not prod — otherwise preview runs pollute the - * production DB via the shared webhook receiver. - */ -const getReceiverBaseUrl = (): string => { - if (process.env.VERCEL_ENV === "production") { - return NEW_API_BASE_URL; - } - if (process.env.VERCEL_URL) { - return `https://${process.env.VERCEL_URL}`; - } - return "http://localhost:3000"; -}; +import { getReceiverBaseUrl } from "@/lib/apify/getReceiverBaseUrl"; /** * Webhook config registered on Apify actor runs. The Apify client diff --git a/lib/apify/getReceiverBaseUrl.ts b/lib/apify/getReceiverBaseUrl.ts new file mode 100644 index 000000000..5d841fda7 --- /dev/null +++ b/lib/apify/getReceiverBaseUrl.ts @@ -0,0 +1,16 @@ +import { NEW_API_BASE_URL } from "@/lib/consts"; + +/** + * Resolves the base URL Apify should POST its webhook to. Preview deploys + * must report to themselves, not prod — otherwise preview runs pollute the + * production DB via the shared webhook receiver. + */ +export function getReceiverBaseUrl(): string { + if (process.env.VERCEL_ENV === "production") { + return NEW_API_BASE_URL; + } + if (process.env.VERCEL_URL) { + return `https://${process.env.VERCEL_URL}`; + } + return "http://localhost:3000"; +} diff --git a/lib/apify/instagram/__tests__/getOrCreatePostsForComments.test.ts b/lib/apify/instagram/__tests__/getOrCreatePostsForComments.test.ts index ad2079bc9..8c9b94d05 100644 --- a/lib/apify/instagram/__tests__/getOrCreatePostsForComments.test.ts +++ b/lib/apify/instagram/__tests__/getOrCreatePostsForComments.test.ts @@ -1,10 +1,10 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { getOrCreatePostsForComments } from "../getOrCreatePostsForComments"; -import { insertPosts } from "@/lib/supabase/posts/insertPosts"; -import { getPostsByUrls } from "@/lib/supabase/posts/getPostsByUrls"; +import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; -vi.mock("@/lib/supabase/posts/insertPosts", () => ({ insertPosts: vi.fn() })); -vi.mock("@/lib/supabase/posts/getPostsByUrls", () => ({ getPostsByUrls: vi.fn() })); +vi.mock("@/lib/supabase/posts/upsertPosts", () => ({ upsertPosts: vi.fn() })); +vi.mock("@/lib/supabase/posts/getPosts", () => ({ getPosts: vi.fn() })); describe("getOrCreatePostsForComments", () => { beforeEach(() => vi.clearAllMocks()); @@ -12,8 +12,8 @@ describe("getOrCreatePostsForComments", () => { it("returns empty map for empty input and does not touch DB", async () => { const result = await getOrCreatePostsForComments([]); expect(result.size).toBe(0); - expect(insertPosts).not.toHaveBeenCalled(); - expect(getPostsByUrls).not.toHaveBeenCalled(); + expect(upsertPosts).not.toHaveBeenCalled(); + expect(getPosts).not.toHaveBeenCalled(); }); it("inserts only the URLs missing from Supabase", async () => { @@ -23,21 +23,21 @@ describe("getOrCreatePostsForComments", () => { { id: "p2", post_url: "u2" }, ] as never; - vi.mocked(getPostsByUrls).mockResolvedValueOnce(existing).mockResolvedValueOnce(afterInsert); + vi.mocked(getPosts).mockResolvedValueOnce(existing).mockResolvedValueOnce(afterInsert); const result = await getOrCreatePostsForComments(["u1", "u2", "u2"]); - expect(insertPosts).toHaveBeenCalledWith([expect.objectContaining({ post_url: "u2" })]); + expect(upsertPosts).toHaveBeenCalledWith([expect.objectContaining({ post_url: "u2" })]); expect(result.get("u1")).toEqual(existing[0]); expect(result.get("u2")).toEqual(afterInsert[1]); }); it("skips insert when all posts already exist", async () => { const existing = [{ id: "p1", post_url: "u1" }] as never; - vi.mocked(getPostsByUrls).mockResolvedValue(existing); + vi.mocked(getPosts).mockResolvedValue(existing); await getOrCreatePostsForComments(["u1"]); - expect(insertPosts).not.toHaveBeenCalled(); + expect(upsertPosts).not.toHaveBeenCalled(); }); }); diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts index 8e7ddee11..a9e26f448 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts @@ -5,8 +5,8 @@ import { saveApifyInstagramPosts } from "../saveApifyInstagramPosts"; import { handleInstagramProfileFollowUpRuns } from "../handleInstagramProfileFollowUpRuns"; import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; import { insertSocials } from "@/lib/supabase/socials/insertSocials"; -import { selectSocialByProfileUrl } from "@/lib/supabase/socials/selectSocialByProfileUrl"; -import { insertSocialPosts } from "@/lib/supabase/social_posts/insertSocialPosts"; +import { selectSocials } from "@/lib/supabase/socials/selectSocials"; +import { upsertSocialPosts } from "@/lib/supabase/social_posts/upsertSocialPosts"; import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccountSocials"; import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccountArtistIds"; import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails"; @@ -19,10 +19,10 @@ vi.mock("../handleInstagramProfileFollowUpRuns", () => ({ })); vi.mock("@/lib/apify/sendApifyWebhookEmail", () => ({ sendApifyWebhookEmail: vi.fn() })); vi.mock("@/lib/supabase/socials/insertSocials", () => ({ insertSocials: vi.fn() })); -vi.mock("@/lib/supabase/socials/selectSocialByProfileUrl", () => ({ - selectSocialByProfileUrl: vi.fn(), +vi.mock("@/lib/supabase/socials/selectSocials", () => ({ + selectSocials: vi.fn(), })); -vi.mock("@/lib/supabase/social_posts/insertSocialPosts", () => ({ insertSocialPosts: vi.fn() })); +vi.mock("@/lib/supabase/social_posts/upsertSocialPosts", () => ({ upsertSocialPosts: vi.fn() })); vi.mock("@/lib/supabase/account_socials/selectAccountSocials", () => ({ selectAccountSocials: vi.fn(), })); @@ -68,7 +68,7 @@ describe("handleInstagramProfileScraperResults", () => { vi.mocked(saveApifyInstagramPosts).mockResolvedValue({ supabasePosts: posts }); vi.mocked(uploadLinkToArweave).mockResolvedValue(null); vi.mocked(insertSocials).mockResolvedValue([] as never); - vi.mocked(selectSocialByProfileUrl).mockResolvedValue({ id: "s1" } as never); + vi.mocked(selectSocials).mockResolvedValue([{ id: "s1" }] as never); vi.mocked(selectAccountSocials).mockResolvedValue([{ account_id: "a1" }] as never); vi.mocked(getAccountArtistIds).mockResolvedValue([{ account_id: "a1" }] as never); vi.mocked(selectAccountEmails).mockResolvedValue([{ email: "x@y.com" }] as never); @@ -76,7 +76,7 @@ describe("handleInstagramProfileScraperResults", () => { const result = await handleInstagramProfileScraperResults(payload); - expect(insertSocialPosts).toHaveBeenCalledOnce(); + expect(upsertSocialPosts).toHaveBeenCalledOnce(); expect(sendApifyWebhookEmail).toHaveBeenCalledWith(expect.any(Object), ["x@y.com"]); expect(handleInstagramProfileFollowUpRuns).toHaveBeenCalledOnce(); expect(result.social).toEqual({ id: "s1" }); diff --git a/lib/apify/instagram/getOrCreatePostsForComments.ts b/lib/apify/instagram/getOrCreatePostsForComments.ts index d8cdeb6fe..001286e61 100644 --- a/lib/apify/instagram/getOrCreatePostsForComments.ts +++ b/lib/apify/instagram/getOrCreatePostsForComments.ts @@ -1,5 +1,5 @@ -import { insertPosts } from "@/lib/supabase/posts/insertPosts"; -import { getPostsByUrls } from "@/lib/supabase/posts/getPostsByUrls"; +import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; import type { Tables, TablesInsert } from "@/types/database.types"; /** @@ -15,7 +15,7 @@ export async function getOrCreatePostsForComments( const unique = Array.from(new Set(postUrls.filter(Boolean))); if (unique.length === 0) return new Map(); - const existing = await getPostsByUrls(unique); + const existing = await getPosts({ postUrls: unique }); const existingSet = new Set(existing.map(p => p.post_url)); const missing = unique.filter(url => !existingSet.has(url)); @@ -26,8 +26,8 @@ export async function getOrCreatePostsForComments( post_url: url, updated_at: new Date().toISOString(), })); - await insertPosts(rows); - all = await getPostsByUrls(unique); + await upsertPosts(rows); + all = await getPosts({ postUrls: unique }); } const map = new Map>(); diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index 36488bfab..cf2828f3b 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -3,8 +3,8 @@ import { saveApifyInstagramPosts } from "@/lib/apify/instagram/saveApifyInstagra import { handleInstagramProfileFollowUpRuns } from "@/lib/apify/instagram/handleInstagramProfileFollowUpRuns"; import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; import { insertSocials } from "@/lib/supabase/socials/insertSocials"; -import { selectSocialByProfileUrl } from "@/lib/supabase/socials/selectSocialByProfileUrl"; -import { insertSocialPosts } from "@/lib/supabase/social_posts/insertSocialPosts"; +import { selectSocials } from "@/lib/supabase/socials/selectSocials"; +import { upsertSocialPosts } from "@/lib/supabase/social_posts/upsertSocialPosts"; import { selectAccountSocials, type AccountSocialWithSocial, @@ -84,14 +84,15 @@ export async function handleInstagramProfileScraperResults( }, ]); - const social = await selectSocialByProfileUrl(normalizedUrl); + const matches = await selectSocials({ profile_url: normalizedUrl }); + const social = matches?.[0] ?? null; if (!social) { return { ...empty, posts }; } if (posts.length) { - await insertSocialPosts( + await upsertSocialPosts( posts.map(post => ({ post_id: post.id, updated_at: post.updated_at, diff --git a/lib/apify/instagram/saveApifyInstagramComments.ts b/lib/apify/instagram/saveApifyInstagramComments.ts index b6b4662e7..88883ef3d 100644 --- a/lib/apify/instagram/saveApifyInstagramComments.ts +++ b/lib/apify/instagram/saveApifyInstagramComments.ts @@ -1,4 +1,4 @@ -import { insertPostComments } from "@/lib/supabase/post_comments/insertPostComments"; +import { upsertPostComments } from "@/lib/supabase/post_comments/upsertPostComments"; import { getOrCreatePostsForComments } from "@/lib/apify/instagram/getOrCreatePostsForComments"; import { getOrCreateSocialsForComments } from "@/lib/apify/instagram/getOrCreateSocialsForComments"; import type { TablesInsert } from "@/types/database.types"; @@ -43,7 +43,7 @@ export async function saveApifyInstagramComments(comments: ApifyInstagramComment } if (rows.length > 0) { - await insertPostComments(rows); + await upsertPostComments(rows); } } catch (error) { console.error("[ERROR] saveApifyInstagramComments:", error); diff --git a/lib/apify/instagram/saveApifyInstagramPosts.ts b/lib/apify/instagram/saveApifyInstagramPosts.ts index 8b8799cde..184b31595 100644 --- a/lib/apify/instagram/saveApifyInstagramPosts.ts +++ b/lib/apify/instagram/saveApifyInstagramPosts.ts @@ -1,5 +1,5 @@ -import { insertPosts } from "@/lib/supabase/posts/insertPosts"; -import { getPostsByUrls } from "@/lib/supabase/posts/getPostsByUrls"; +import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; import type { ApifyInstagramPost } from "@/lib/apify/types"; import type { Tables, TablesInsert } from "@/types/database.types"; @@ -21,8 +21,8 @@ export async function saveApifyInstagramPosts( })); const postUrls = rows.map(p => p.post_url); - await insertPosts(rows); + await upsertPosts(rows); - const supabasePosts = await getPostsByUrls(postUrls); + const supabasePosts = await getPosts({ postUrls }); return { supabasePosts }; } diff --git a/lib/arweave/isSafeHttpUrl.ts b/lib/arweave/isSafeHttpUrl.ts new file mode 100644 index 000000000..fabbc1871 --- /dev/null +++ b/lib/arweave/isSafeHttpUrl.ts @@ -0,0 +1,27 @@ +/** + * Returns true if `raw` parses as an absolute http(s) URL whose host is + * not loopback, link-local, or RFC1918 private. Used to gate outbound + * fetches against SSRF via untrusted user-supplied URLs. + */ +export function isSafeHttpUrl(raw: string): boolean { + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + return false; + } + if (parsed.protocol !== "https:" && parsed.protocol !== "http:") return false; + const host = parsed.hostname.toLowerCase(); + if (!host) return false; + if (host === "localhost" || host.endsWith(".localhost")) return false; + if ( + /^(10\.|127\.|0\.|169\.254\.|192\.168\.)/.test(host) || + /^172\.(1[6-9]|2\d|3[01])\./.test(host) || + host === "::1" || + host.startsWith("fc") || + host.startsWith("fd") + ) { + return false; + } + return true; +} diff --git a/lib/arweave/uploadLinkToArweave.ts b/lib/arweave/uploadLinkToArweave.ts index af7182ee0..9674f4e02 100644 --- a/lib/arweave/uploadLinkToArweave.ts +++ b/lib/arweave/uploadLinkToArweave.ts @@ -1,30 +1,8 @@ import { uploadToArweave } from "./uploadToArweave"; +import { isSafeHttpUrl } from "./isSafeHttpUrl"; const MAX_IMAGE_BYTES = 10 * 1024 * 1024; -function isSafeHttpUrl(raw: string): boolean { - let parsed: URL; - try { - parsed = new URL(raw); - } catch { - return false; - } - if (parsed.protocol !== "https:" && parsed.protocol !== "http:") return false; - const host = parsed.hostname.toLowerCase(); - if (!host) return false; - if (host === "localhost" || host.endsWith(".localhost")) return false; - if ( - /^(10\.|127\.|0\.|169\.254\.|192\.168\.)/.test(host) || - /^172\.(1[6-9]|2\d|3[01])\./.test(host) || - host === "::1" || - host.startsWith("fc") || - host.startsWith("fd") - ) { - return false; - } - return true; -} - /** * Fetches the image at `imageUrl` and uploads its bytes to Arweave. * Returns the Arweave transaction id (without the `ar://` prefix) on diff --git a/lib/supabase/post_comments/selectPostComments.ts b/lib/supabase/post_comments/selectPostComments.ts new file mode 100644 index 000000000..96a032b43 --- /dev/null +++ b/lib/supabase/post_comments/selectPostComments.ts @@ -0,0 +1,30 @@ +import supabase from "../serverClient"; + +type SelectPostCommentsParams = { + postIds?: string[]; +}; + +/** + * Selects rows from `post_comments`. When `postIds` is provided, + * returns only the `post_id` column for rows in that set — used as + * a lightweight existence check for posts that already have comments. + * + * @param params - Optional filters. + */ +export async function selectPostComments({ postIds }: SelectPostCommentsParams = {}) { + let query = supabase.from("post_comments").select("post_id"); + + if (postIds !== undefined) { + if (postIds.length === 0) return []; + query = query.in("post_id", postIds); + } + + const { data, error } = await query; + + if (error) { + console.error("[ERROR] selectPostComments:", error); + throw error; + } + + return data ?? []; +} diff --git a/lib/supabase/post_comments/selectPostUrlsWithComments.ts b/lib/supabase/post_comments/selectPostUrlsWithComments.ts index e343b2909..f0c5bab7c 100644 --- a/lib/supabase/post_comments/selectPostUrlsWithComments.ts +++ b/lib/supabase/post_comments/selectPostUrlsWithComments.ts @@ -1,40 +1,23 @@ -import supabase from "../serverClient"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; +import { selectPostComments } from "./selectPostComments"; /** * Returns the subset of `postUrls` for which at least one row exists in * `post_comments`. Used by the Apify follow-up dispatcher to decide * whether a post needs a full comments backfill or just a cheap refresh. * - * Implemented as a narrow two-step lookup (post ids, then a distinct - * `post_id` scan on `post_comments`) so we don't pull whole comment - * rows with joins just to answer an existence question. + * Composes `getPosts` (URL → id) and `selectPostComments` (post_id + * existence) so each underlying supabase call stays single-table. */ -export async function selectPostUrlsWithComments(postUrls: string[]): Promise { +export async function selectPostUrlsWithComments(postUrls: string[]) { if (postUrls.length === 0) return []; - const { data: posts, error: postsErr } = await supabase - .from("posts") - .select("id, post_url") - .in("post_url", postUrls); - - if (postsErr) { - console.error("[ERROR] selectPostUrlsWithComments (posts):", postsErr); - throw postsErr; - } - - if (!posts || posts.length === 0) return []; + const posts = await getPosts({ postUrls }); + if (posts.length === 0) return []; const postIds = posts.map(p => p.id); - const { data: comments, error: commentsErr } = await supabase - .from("post_comments") - .select("post_id") - .in("post_id", postIds); - - if (commentsErr) { - console.error("[ERROR] selectPostUrlsWithComments (post_comments):", commentsErr); - throw commentsErr; - } + const comments = await selectPostComments({ postIds }); - const postIdsWithComments = new Set((comments ?? []).map(c => c.post_id)); + const postIdsWithComments = new Set(comments.map(c => c.post_id)); return posts.filter(p => postIdsWithComments.has(p.id)).map(p => p.post_url); } diff --git a/lib/supabase/post_comments/insertPostComments.ts b/lib/supabase/post_comments/upsertPostComments.ts similarity index 65% rename from lib/supabase/post_comments/insertPostComments.ts rename to lib/supabase/post_comments/upsertPostComments.ts index 403cf42c0..17aab380b 100644 --- a/lib/supabase/post_comments/insertPostComments.ts +++ b/lib/supabase/post_comments/upsertPostComments.ts @@ -1,16 +1,14 @@ import supabase from "../serverClient"; -import type { Tables, TablesInsert } from "@/types/database.types"; +import type { TablesInsert } from "@/types/database.types"; /** * Upserts rows into `post_comments`. Conflicts on * `(post_id, social_id, comment, commented_at)` are ignored so replays * of the same Apify dataset do not duplicate comments. * - * @param comments - Rows to insert. + * @param comments - Rows to upsert. */ -export async function insertPostComments( - comments: TablesInsert<"post_comments">[], -): Promise[]> { +export async function upsertPostComments(comments: TablesInsert<"post_comments">[]) { if (comments.length === 0) return []; const { data, error } = await supabase @@ -22,7 +20,7 @@ export async function insertPostComments( .select(); if (error) { - console.error("[ERROR] insertPostComments:", error); + console.error("[ERROR] upsertPostComments:", error); throw error; } diff --git a/lib/supabase/posts/getPosts.ts b/lib/supabase/posts/getPosts.ts new file mode 100644 index 000000000..9c5ba3c7d --- /dev/null +++ b/lib/supabase/posts/getPosts.ts @@ -0,0 +1,37 @@ +import supabase from "@/lib/supabase/serverClient"; + +type GetPostsParams = { + postUrls?: string[]; + postIds?: string[]; +}; + +/** + * Fetches `posts` rows. When `postUrls` is provided, scopes to rows + * whose `post_url` is in that list. When `postIds` is provided, scopes + * to rows whose `id` is in that list. Returns an empty array when an + * explicit but empty filter list is passed. + * + * @param params - Optional filters. + */ +export async function getPosts({ postUrls, postIds }: GetPostsParams = {}) { + let query = supabase.from("posts").select("*"); + + if (postUrls !== undefined) { + if (postUrls.length === 0) return []; + query = query.in("post_url", postUrls); + } + + if (postIds !== undefined) { + if (postIds.length === 0) return []; + query = query.in("id", postIds); + } + + const { data, error } = await query; + + if (error) { + console.error("[ERROR] getPosts:", error); + throw error; + } + + return data ?? []; +} diff --git a/lib/supabase/posts/getPostsByUrls.ts b/lib/supabase/posts/getPostsByUrls.ts deleted file mode 100644 index 52b2c38db..000000000 --- a/lib/supabase/posts/getPostsByUrls.ts +++ /dev/null @@ -1,21 +0,0 @@ -import supabase from "@/lib/supabase/serverClient"; -import type { Tables } from "@/types/database.types"; - -/** - * Fetches `posts` rows matching any of the given URLs. Returns an - * empty array for empty / invalid input. - * - * @param postUrls - Array of `post_url` values to look up. - */ -export async function getPostsByUrls(postUrls: string[]): Promise[]> { - if (!Array.isArray(postUrls) || postUrls.length === 0) return []; - - const { data, error } = await supabase.from("posts").select("*").in("post_url", postUrls); - - if (error) { - console.error("[ERROR] getPostsByUrls:", error); - throw error; - } - - return data ?? []; -} diff --git a/lib/supabase/posts/insertPosts.ts b/lib/supabase/posts/upsertPosts.ts similarity index 83% rename from lib/supabase/posts/insertPosts.ts rename to lib/supabase/posts/upsertPosts.ts index 55bfa4283..c70448d65 100644 --- a/lib/supabase/posts/insertPosts.ts +++ b/lib/supabase/posts/upsertPosts.ts @@ -8,13 +8,13 @@ import type { TablesInsert } from "@/types/database.types"; * * @param posts - Rows matching the posts-table insert type. */ -export async function insertPosts(posts: TablesInsert<"posts">[]) { +export async function upsertPosts(posts: TablesInsert<"posts">[]) { const { data, error } = await supabase .from("posts") .upsert(posts, { onConflict: "post_url", ignoreDuplicates: true }); if (error) { - console.error("[ERROR] insertPosts:", error); + console.error("[ERROR] upsertPosts:", error); throw error; } diff --git a/lib/supabase/social_posts/insertSocialPosts.ts b/lib/supabase/social_posts/upsertSocialPosts.ts similarity index 82% rename from lib/supabase/social_posts/insertSocialPosts.ts rename to lib/supabase/social_posts/upsertSocialPosts.ts index 2fce08887..c1711e308 100644 --- a/lib/supabase/social_posts/insertSocialPosts.ts +++ b/lib/supabase/social_posts/upsertSocialPosts.ts @@ -8,13 +8,13 @@ import type { TablesInsert } from "@/types/database.types"; * * @param socialPosts - Rows to upsert. */ -export async function insertSocialPosts(socialPosts: TablesInsert<"social_posts">[]) { +export async function upsertSocialPosts(socialPosts: TablesInsert<"social_posts">[]) { const { data, error } = await supabase .from("social_posts") .upsert(socialPosts, { onConflict: "post_id,social_id" }); if (error) { - console.error("[ERROR] insertSocialPosts:", error); + console.error("[ERROR] upsertSocialPosts:", error); throw error; } diff --git a/lib/supabase/socials/insertSocials.ts b/lib/supabase/socials/insertSocials.ts index 9d49e2eba..fcfbc56f9 100644 --- a/lib/supabase/socials/insertSocials.ts +++ b/lib/supabase/socials/insertSocials.ts @@ -1,18 +1,6 @@ import supabase from "../serverClient"; -import type { Tables, TablesInsert } from "@/types/database.types"; - -/** - * Strips null/undefined fields so upsert-on-conflict doesn't clobber - * existing column values when a caller has nothing new to say about - * those fields (e.g. the comments pipeline passes null for bio/region). - */ -const stripNullish = (row: TablesInsert<"socials">): TablesInsert<"socials"> => { - const out: Record = {}; - for (const [k, v] of Object.entries(row)) { - if (v !== null && v !== undefined) out[k] = v; - } - return out as TablesInsert<"socials">; -}; +import { stripNullish } from "@/lib/utils/stripNullish"; +import type { TablesInsert } from "@/types/database.types"; /** * Inserts one or more social records. Upserts on `profile_url`; @@ -22,9 +10,7 @@ const stripNullish = (row: TablesInsert<"socials">): TablesInsert<"socials"> => * @param socials - Array of social data to insert * @returns Array of inserted social records */ -export async function insertSocials( - socials: TablesInsert<"socials">[], -): Promise[]> { +export async function insertSocials(socials: TablesInsert<"socials">[]) { const { data, error } = await supabase .from("socials") .upsert(socials.map(stripNullish), { onConflict: "profile_url" }) diff --git a/lib/supabase/socials/selectSocialByProfileUrl.ts b/lib/supabase/socials/selectSocialByProfileUrl.ts deleted file mode 100644 index 138f552d5..000000000 --- a/lib/supabase/socials/selectSocialByProfileUrl.ts +++ /dev/null @@ -1,28 +0,0 @@ -import supabase from "../serverClient"; -import type { Tables } from "@/types/database.types"; - -/** - * Fetches a single `socials` row by normalized `profile_url`. - * Returns `null` when no row matches or the URL is empty. - * - * @param profileUrl - Normalized profile URL. - */ -export async function selectSocialByProfileUrl( - profileUrl: string, -): Promise | null> { - if (!profileUrl) return null; - - const { data, error } = await supabase - .from("socials") - .select("*") - .eq("profile_url", profileUrl) - .neq("profile_url", "") - .maybeSingle(); - - if (error) { - console.error("[ERROR] selectSocialByProfileUrl:", error); - throw error; - } - - return data ?? null; -} diff --git a/lib/utils/stripNullish.ts b/lib/utils/stripNullish.ts new file mode 100644 index 000000000..cf8e7c656 --- /dev/null +++ b/lib/utils/stripNullish.ts @@ -0,0 +1,13 @@ +/** + * Returns a shallow copy of `row` with `null` and `undefined` fields + * removed. Used before upserts so callers don't clobber existing + * non-null column values just because they have nothing new to say + * about those fields. + */ +export function stripNullish>(row: T): T { + const out: Record = {}; + for (const [k, v] of Object.entries(row)) { + if (v !== null && v !== undefined) out[k] = v; + } + return out as T; +} From beb9197067ffc237801e2759f70d06a46f315530 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 19:00:13 +0000 Subject: [PATCH 07/23] refactor(networking): make getBaseUrl prod-aware and skip apify webhooks on localhost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - getBaseUrl now returns NEW_API_BASE_URL on VERCEL_ENV=production so callers always get the canonical prod alias for cross-deploy callbacks (Apify webhooks, etc.). Preview/local behavior unchanged. - Drop the apify-specific getReceiverBaseUrl wrapper; getApifyWebhooks now uses the shared getBaseUrl helper. - getApifyWebhooks returns [] when the resolved base URL is localhost. Apify cannot reach a developer machine, so registering localhost as a callback URL is meaningless — let the run kick off without one in local dev. - Cover both behaviors with tests. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- lib/apify/__tests__/getApifyWebhooks.test.ts | 46 ++++++++++++++++++++ lib/apify/getApifyWebhooks.ts | 11 ++++- lib/apify/getReceiverBaseUrl.ts | 16 ------- lib/networking/__tests__/getBaseUrl.test.ts | 9 ++++ lib/networking/getBaseUrl.ts | 17 ++++++-- 5 files changed, 77 insertions(+), 22 deletions(-) create mode 100644 lib/apify/__tests__/getApifyWebhooks.test.ts delete mode 100644 lib/apify/getReceiverBaseUrl.ts diff --git a/lib/apify/__tests__/getApifyWebhooks.test.ts b/lib/apify/__tests__/getApifyWebhooks.test.ts new file mode 100644 index 000000000..934e1a8db --- /dev/null +++ b/lib/apify/__tests__/getApifyWebhooks.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { getApifyWebhooks } from "../getApifyWebhooks"; + +describe("getApifyWebhooks", () => { + const originalEnv = process.env; + + beforeEach(() => { + vi.resetModules(); + process.env = { ...originalEnv }; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it("returns the prod-aliased webhook URL when VERCEL_ENV is production", () => { + process.env.VERCEL_ENV = "production"; + process.env.VERCEL_URL = "recoup-api-abc.vercel.app"; + + expect(getApifyWebhooks()).toEqual([ + { + eventTypes: ["ACTOR.RUN.SUCCEEDED"], + requestUrl: "https://recoup-api.vercel.app/api/apify", + }, + ]); + }); + + it("returns the deployment-specific URL on preview deploys", () => { + delete process.env.VERCEL_ENV; + process.env.VERCEL_URL = "preview-xyz.vercel.app"; + + expect(getApifyWebhooks()).toEqual([ + { + eventTypes: ["ACTOR.RUN.SUCCEEDED"], + requestUrl: "https://preview-xyz.vercel.app/api/apify", + }, + ]); + }); + + it("returns no webhooks locally so we don't hand Apify an unreachable URL", () => { + delete process.env.VERCEL_ENV; + delete process.env.VERCEL_URL; + + expect(getApifyWebhooks()).toEqual([]); + }); +}); diff --git a/lib/apify/getApifyWebhooks.ts b/lib/apify/getApifyWebhooks.ts index ea4fd6204..a8b09dcf4 100644 --- a/lib/apify/getApifyWebhooks.ts +++ b/lib/apify/getApifyWebhooks.ts @@ -1,17 +1,24 @@ import type { WebhookEventType, WebhookUpdateData } from "apify-client"; -import { getReceiverBaseUrl } from "@/lib/apify/getReceiverBaseUrl"; +import { getBaseUrl } from "@/lib/networking/getBaseUrl"; /** * Webhook config registered on Apify actor runs. The Apify client * serializes this to base64 before including it on the start request. * Points at this service's `POST /api/apify` webhook receiver. + * + * Returns an empty list in local dev (when the resolved base URL is + * localhost) — Apify cannot reach a developer's machine, so the run + * still kicks off but without a useless callback URL attached. */ export function getApifyWebhooks(): WebhookUpdateData[] { + const baseUrl = getBaseUrl(); + if (baseUrl.startsWith("http://localhost")) return []; + const eventTypes: WebhookEventType[] = ["ACTOR.RUN.SUCCEEDED"]; return [ { eventTypes, - requestUrl: `${getReceiverBaseUrl()}/api/apify`, + requestUrl: `${baseUrl}/api/apify`, }, ]; } diff --git a/lib/apify/getReceiverBaseUrl.ts b/lib/apify/getReceiverBaseUrl.ts deleted file mode 100644 index 5d841fda7..000000000 --- a/lib/apify/getReceiverBaseUrl.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { NEW_API_BASE_URL } from "@/lib/consts"; - -/** - * Resolves the base URL Apify should POST its webhook to. Preview deploys - * must report to themselves, not prod — otherwise preview runs pollute the - * production DB via the shared webhook receiver. - */ -export function getReceiverBaseUrl(): string { - if (process.env.VERCEL_ENV === "production") { - return NEW_API_BASE_URL; - } - if (process.env.VERCEL_URL) { - return `https://${process.env.VERCEL_URL}`; - } - return "http://localhost:3000"; -} diff --git a/lib/networking/__tests__/getBaseUrl.test.ts b/lib/networking/__tests__/getBaseUrl.test.ts index fa678ea10..0fa451c2c 100644 --- a/lib/networking/__tests__/getBaseUrl.test.ts +++ b/lib/networking/__tests__/getBaseUrl.test.ts @@ -21,6 +21,15 @@ describe("getBaseUrl", () => { expect(result).toBe("https://my-app.vercel.app"); }); + it("returns the canonical prod URL when VERCEL_ENV is production", () => { + process.env.VERCEL_ENV = "production"; + process.env.VERCEL_URL = "recoup-api-abc123.vercel.app"; + + const result = getBaseUrl(); + + expect(result).toBe("https://recoup-api.vercel.app"); + }); + it("returns localhost when VERCEL_URL is not set", () => { delete process.env.VERCEL_URL; diff --git a/lib/networking/getBaseUrl.ts b/lib/networking/getBaseUrl.ts index 112d94e50..fdcf83e22 100644 --- a/lib/networking/getBaseUrl.ts +++ b/lib/networking/getBaseUrl.ts @@ -1,10 +1,19 @@ +import { NEW_API_BASE_URL } from "@/lib/consts"; + /** - * Gets the base URL for the current API server. - * Uses VERCEL_URL in Vercel deployments, falls back to localhost. - * - * @returns The base URL string + * Resolves the base URL for the current API server. + * - Production: canonical `NEW_API_BASE_URL` so cross-deploy callbacks + * (Apify webhooks, etc.) always land on the stable prod alias. + * - Preview: the deployment-specific `VERCEL_URL` so previews report + * to themselves and don't pollute prod. + * - Local: `http://localhost:3000`. Note that third parties (Apify, + * etc.) cannot reach this — callers that need a public URL should + * guard against the localhost case. */ export function getBaseUrl(): string { + if (process.env.VERCEL_ENV === "production") { + return NEW_API_BASE_URL; + } if (process.env.VERCEL_URL) { return `https://${process.env.VERCEL_URL}`; } From f0a1d9da6859ba82412939a8723f8f1f77803cc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 19:12:55 +0000 Subject: [PATCH 08/23] refactor(apify): align webhook validator with project pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename validateApifyWebhookRequest -> validateApifyBody and inline the zod schema (apifyBodySchema) inside the validator file. Drop the separate apifyPayloadSchema module and the ApifyPayload type alias; downstream handlers now import the inferred ApifyBody type. - The validator returns NextResponse on failure (status: 200, with the project-standard { status, missing_fields, error } body) or the validated body on success — matching the validate*Body.ts contract used elsewhere in the codebase. Status 200 is intentional: Apify retries on any non-2xx response. - postApifyWebhookHandler now parses request.json() itself and hands the parsed body to validateApifyBody, then uses the standard `instanceof NextResponse` short-circuit. - Update validator test (new shape) and handler test (validator error passthrough + Invalid JSON branch). Verification: pnpm lint:check clean, pnpm test 2317/2317 pass. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- .../__tests__/postApifyWebhookHandler.test.ts | 14 ++++- lib/apify/__tests__/validateApifyBody.test.ts | 41 +++++++++++++++ .../validateApifyWebhookRequest.test.ts | 43 --------------- lib/apify/apifyPayloadSchema.ts | 18 ------- lib/apify/handleApifyWebhook.ts | 6 +-- .../handleInstagramCommentsScraper.ts | 6 +-- .../handleInstagramProfileScraperResults.ts | 6 +-- lib/apify/postApifyWebhookHandler.ts | 21 ++++---- lib/apify/validateApifyBody.ts | 52 +++++++++++++++++++ lib/apify/validateApifyWebhookRequest.ts | 34 ------------ 10 files changed, 126 insertions(+), 115 deletions(-) create mode 100644 lib/apify/__tests__/validateApifyBody.test.ts delete mode 100644 lib/apify/__tests__/validateApifyWebhookRequest.test.ts delete mode 100644 lib/apify/apifyPayloadSchema.ts create mode 100644 lib/apify/validateApifyBody.ts delete mode 100644 lib/apify/validateApifyWebhookRequest.ts diff --git a/lib/apify/__tests__/postApifyWebhookHandler.test.ts b/lib/apify/__tests__/postApifyWebhookHandler.test.ts index a2e57a343..c8c915bac 100644 --- a/lib/apify/__tests__/postApifyWebhookHandler.test.ts +++ b/lib/apify/__tests__/postApifyWebhookHandler.test.ts @@ -34,12 +34,22 @@ describe("postApifyWebhookHandler", () => { expect(handleApifyWebhook).toHaveBeenCalledOnce(); }); - it("returns 200 for invalid payloads so Apify does not retry", async () => { + it("returns 200 with the validator's error response for invalid payloads", async () => { const res = await postApifyWebhookHandler(makeRequest({ bogus: true })); expect(res.status).toBe(200); const body = await res.json(); - expect(body.message).toBe("Invalid payload"); + expect(body.status).toBe("error"); + expect(typeof body.error).toBe("string"); + expect(handleApifyWebhook).not.toHaveBeenCalled(); + }); + + it("returns 200 with an Invalid JSON message for malformed bodies", async () => { + const res = await postApifyWebhookHandler(makeRequest(null, "not json")); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.message).toBe("Invalid JSON"); expect(handleApifyWebhook).not.toHaveBeenCalled(); }); }); diff --git a/lib/apify/__tests__/validateApifyBody.test.ts b/lib/apify/__tests__/validateApifyBody.test.ts new file mode 100644 index 000000000..7496f2dfe --- /dev/null +++ b/lib/apify/__tests__/validateApifyBody.test.ts @@ -0,0 +1,41 @@ +import { describe, it, expect } from "vitest"; +import { NextResponse } from "next/server"; +import { validateApifyBody } from "../validateApifyBody"; + +describe("validateApifyBody", () => { + it("returns the parsed body for a valid Apify payload", () => { + const body = { + userId: "u", + createdAt: "2026-01-01T00:00:00Z", + eventType: "ACTOR.RUN.SUCCEEDED", + eventData: { actorId: "dSCLg0C3YEZ83HzYX" }, + resource: { defaultDatasetId: "ds_1" }, + }; + + const result = validateApifyBody(body); + + expect(result).not.toBeInstanceOf(NextResponse); + if (!(result instanceof NextResponse)) { + expect(result.eventData.actorId).toBe("dSCLg0C3YEZ83HzYX"); + expect(result.resource.defaultDatasetId).toBe("ds_1"); + } + }); + + it("returns a 200 NextResponse with the project error shape on missing fields", async () => { + const result = validateApifyBody({ eventData: {}, resource: {} }); + + expect(result).toBeInstanceOf(NextResponse); + if (result instanceof NextResponse) { + expect(result.status).toBe(200); + const body = await result.json(); + expect(body.status).toBe("error"); + expect(Array.isArray(body.missing_fields)).toBe(true); + expect(typeof body.error).toBe("string"); + } + }); + + it("returns a NextResponse for non-object bodies", () => { + expect(validateApifyBody(undefined)).toBeInstanceOf(NextResponse); + expect(validateApifyBody("not an object")).toBeInstanceOf(NextResponse); + }); +}); diff --git a/lib/apify/__tests__/validateApifyWebhookRequest.test.ts b/lib/apify/__tests__/validateApifyWebhookRequest.test.ts deleted file mode 100644 index 6568fb475..000000000 --- a/lib/apify/__tests__/validateApifyWebhookRequest.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { NextRequest } from "next/server"; -import { validateApifyWebhookRequest } from "../validateApifyWebhookRequest"; - -function makeRequest(body: unknown, raw?: string) { - return new NextRequest("http://localhost/api/apify", { - method: "POST", - headers: { "content-type": "application/json" }, - body: raw ?? JSON.stringify(body), - }); -} - -describe("validateApifyWebhookRequest", () => { - it("returns ok+data for a valid Apify payload", async () => { - const req = makeRequest({ - userId: "u", - createdAt: "2026-01-01T00:00:00Z", - eventType: "ACTOR.RUN.SUCCEEDED", - eventData: { actorId: "dSCLg0C3YEZ83HzYX" }, - resource: { defaultDatasetId: "ds_1" }, - }); - - const result = await validateApifyWebhookRequest(req); - - expect(result.ok).toBe(true); - if (result.ok) { - expect(result.data.eventData.actorId).toBe("dSCLg0C3YEZ83HzYX"); - expect(result.data.resource.defaultDatasetId).toBe("ds_1"); - } - }); - - it("rejects payloads missing required fields", async () => { - const req = makeRequest({ eventData: {}, resource: {} }); - const result = await validateApifyWebhookRequest(req); - expect(result.ok).toBe(false); - }); - - it("rejects non-JSON bodies", async () => { - const req = makeRequest(null, "not json"); - const result = await validateApifyWebhookRequest(req); - expect(result).toEqual({ ok: false, error: "Invalid JSON" }); - }); -}); diff --git a/lib/apify/apifyPayloadSchema.ts b/lib/apify/apifyPayloadSchema.ts deleted file mode 100644 index 3a2508127..000000000 --- a/lib/apify/apifyPayloadSchema.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { z } from "zod"; - -/** - * Zod schema for Apify webhook payloads. Only validates the fields we - * branch on + read downstream (actorId for dispatch, datasetId for - * fetch); extra keys from Apify are stripped silently so upstream - * schema drift does not drop events. - */ -export const apifyPayloadSchema = z.object({ - eventData: z.object({ - actorId: z.string(), - }), - resource: z.object({ - defaultDatasetId: z.string(), - }), -}); - -export type ApifyPayload = z.infer; diff --git a/lib/apify/handleApifyWebhook.ts b/lib/apify/handleApifyWebhook.ts index 2f0a2111f..77d50a97d 100644 --- a/lib/apify/handleApifyWebhook.ts +++ b/lib/apify/handleApifyWebhook.ts @@ -1,4 +1,4 @@ -import type { ApifyPayload } from "@/lib/apify/apifyPayloadSchema"; +import type { ApifyBody } from "@/lib/apify/validateApifyBody"; import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; @@ -20,9 +20,9 @@ const fallbackResponse = { * empty-shaped response. Never throws — handler failures are caught * and logged so the webhook route can always reply 200. * - * @param parsed - Validated Apify webhook payload. + * @param parsed - Validated Apify webhook body. */ -export async function handleApifyWebhook(parsed: ApifyPayload) { +export async function handleApifyWebhook(parsed: ApifyBody) { try { switch (parsed.eventData.actorId) { case INSTAGRAM_PROFILE_ACTOR_ID: diff --git a/lib/apify/instagram/handleInstagramCommentsScraper.ts b/lib/apify/instagram/handleInstagramCommentsScraper.ts index 09164f033..99bb0428d 100644 --- a/lib/apify/instagram/handleInstagramCommentsScraper.ts +++ b/lib/apify/instagram/handleInstagramCommentsScraper.ts @@ -1,7 +1,7 @@ import { getDataset } from "@/lib/apify/getDataset"; import { saveApifyInstagramComments } from "@/lib/apify/instagram/saveApifyInstagramComments"; import { startInstagramProfileScraping } from "@/lib/apify/instagram/startInstagramProfileScraping"; -import type { ApifyPayload } from "@/lib/apify/apifyPayloadSchema"; +import type { ApifyBody } from "@/lib/apify/validateApifyBody"; import type { ApifyInstagramComment } from "@/lib/apify/types"; /** @@ -10,9 +10,9 @@ import type { ApifyInstagramComment } from "@/lib/apify/types"; * - Kicks off a fan-profile scrape for the distinct commenter * usernames so their socials get indexed. * - * @param parsed - Validated Apify webhook payload. + * @param parsed - Validated Apify webhook body. */ -export async function handleInstagramCommentsScraper(parsed: ApifyPayload) { +export async function handleInstagramCommentsScraper(parsed: ApifyBody) { const datasetId = parsed.resource.defaultDatasetId; const empty = { comments: [] as ApifyInstagramComment[], diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index cf2828f3b..4c6306af9 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -15,7 +15,7 @@ import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; import { getFetchableUrl } from "@/lib/arweave/getFetchableUrl"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; -import type { ApifyPayload } from "@/lib/apify/apifyPayloadSchema"; +import type { ApifyBody } from "@/lib/apify/validateApifyBody"; import type { Tables } from "@/types/database.types"; type ProfileScraperResult = { @@ -38,10 +38,10 @@ type ProfileScraperResult = { * failures inside the chain are logged but allowed to propagate to the * webhook route's outer try/catch, which always returns 200. * - * @param parsed - Validated Apify webhook payload. + * @param parsed - Validated Apify webhook body. */ export async function handleInstagramProfileScraperResults( - parsed: ApifyPayload, + parsed: ApifyBody, ): Promise { const datasetId = parsed.resource.defaultDatasetId; const empty: ProfileScraperResult = { diff --git a/lib/apify/postApifyWebhookHandler.ts b/lib/apify/postApifyWebhookHandler.ts index ef3c06d84..36aa1b2ac 100644 --- a/lib/apify/postApifyWebhookHandler.ts +++ b/lib/apify/postApifyWebhookHandler.ts @@ -1,5 +1,5 @@ import { NextRequest, NextResponse } from "next/server"; -import { validateApifyWebhookRequest } from "@/lib/apify/validateApifyWebhookRequest"; +import { validateApifyBody } from "@/lib/apify/validateApifyBody"; import { handleApifyWebhook } from "@/lib/apify/handleApifyWebhook"; /** @@ -11,18 +11,21 @@ import { handleApifyWebhook } from "@/lib/apify/handleApifyWebhook"; * @returns JSON response (always status 200). */ export async function postApifyWebhookHandler(request: NextRequest): Promise { - const validation = await validateApifyWebhookRequest(request); + let body: unknown; + try { + body = await request.json(); + } catch { + console.warn("[WARN] postApifyWebhookHandler: invalid JSON"); + return NextResponse.json({ message: "Invalid JSON" }, { status: 200 }); + } - if (!validation.ok || !validation.data) { - console.warn("[WARN] postApifyWebhookHandler: invalid payload:", validation.error); - return NextResponse.json( - { message: "Invalid payload", error: validation.error }, - { status: 200 }, - ); + const validated = validateApifyBody(body); + if (validated instanceof NextResponse) { + return validated; } try { - const result = await handleApifyWebhook(validation.data); + const result = await handleApifyWebhook(validated); return NextResponse.json(result, { status: 200 }); } catch (error) { console.error("[ERROR] postApifyWebhookHandler:", error); diff --git a/lib/apify/validateApifyBody.ts b/lib/apify/validateApifyBody.ts new file mode 100644 index 000000000..9af7b469e --- /dev/null +++ b/lib/apify/validateApifyBody.ts @@ -0,0 +1,52 @@ +import { NextResponse } from "next/server"; +import { getCorsHeaders } from "@/lib/networking/getCorsHeaders"; +import { z } from "zod"; + +/** + * Schema for `POST /api/apify` webhook bodies. Only validates the + * fields we branch on + read downstream (`actorId` for dispatch, + * `defaultDatasetId` for fetch). Extra keys are stripped so upstream + * schema drift does not drop events. + */ +export const apifyBodySchema = z.object({ + eventData: z.object({ + actorId: z.string({ message: "eventData.actorId is required" }), + }), + resource: z.object({ + defaultDatasetId: z.string({ message: "resource.defaultDatasetId is required" }), + }), +}); + +export type ApifyBody = z.infer; + +/** + * Validates request body for POST /api/apify. + * + * The route's contract with Apify is to always reply 2xx so Apify does + * not retry, so the failure NextResponse uses status 200 with the + * project-standard error body shape. + * + * @param body - The request body + * @returns A NextResponse with an error if validation fails, or the + * validated body if validation passes. + */ +export function validateApifyBody(body: unknown): NextResponse | ApifyBody { + const result = apifyBodySchema.safeParse(body); + + if (!result.success) { + const firstError = result.error.issues[0]; + return NextResponse.json( + { + status: "error", + missing_fields: firstError.path, + error: firstError.message, + }, + { + status: 200, + headers: getCorsHeaders(), + }, + ); + } + + return result.data; +} diff --git a/lib/apify/validateApifyWebhookRequest.ts b/lib/apify/validateApifyWebhookRequest.ts deleted file mode 100644 index f84ff062f..000000000 --- a/lib/apify/validateApifyWebhookRequest.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { NextRequest } from "next/server"; -import { apifyPayloadSchema, type ApifyPayload } from "@/lib/apify/apifyPayloadSchema"; - -export interface ApifyWebhookValidationResult { - ok: boolean; - data?: ApifyPayload; - error?: string; -} - -/** - * Parses and validates a `POST /api/apify` request body against the - * `apifyPayloadSchema`. Returns `{ ok: true, data }` on success and - * `{ ok: false, error }` when parsing or schema validation fails. The - * route always returns 200 regardless, matching Apify's retry posture. - * - * @param request - Incoming webhook request. - */ -export async function validateApifyWebhookRequest( - request: NextRequest, -): Promise { - let body: unknown; - try { - body = await request.json(); - } catch { - return { ok: false, error: "Invalid JSON" }; - } - - const parsed = apifyPayloadSchema.safeParse(body); - if (!parsed.success) { - return { ok: false, error: parsed.error.issues[0]?.message ?? "Invalid payload" }; - } - - return { ok: true, data: parsed.data }; -} From 7dec06fa5884ab672bcdb08ed0ba3d846f98df06 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 19:30:44 +0000 Subject: [PATCH 09/23] refactor(apify): use request validator + drop dispatcher fallback shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename validateApifyBody -> validateApifyWebhookRequest to match the rest of the codebase's request-validator pattern (validate*Request.ts taking a NextRequest). Validator now parses request.json() itself and returns NextResponse(200) | ApifyWebhookPayload, with the project -standard { status, missing_fields, error } error body. Schema and inferred type (ApifyWebhookPayload) live in the validator file. - Delete lib/apify/handleApifyWebhook.ts; the actor dispatch is now a small switch inside postApifyWebhookHandler. The empty fallback shape is gone — unhandled actors and handler exceptions return a clean { status: "error", error } body (still HTTP 200 to avoid Apify retry). - Stop swallowing errors in handleInstagramCommentsScraper. Dataset fetch and comment persistence failures now propagate up to the webhook route, which logs and returns the error response. The inner fan-profile scrape stays best-effort. - Update validator and handler tests; cover unhandled actor, handler throw, validator passthrough, and Invalid JSON branches. Verification: pnpm lint:check clean, pnpm test 2316/2316 pass. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- .../__tests__/handleApifyWebhook.test.ts | 50 ------------ .../__tests__/postApifyWebhookHandler.test.ts | 80 +++++++++++++++---- lib/apify/__tests__/validateApifyBody.test.ts | 41 ---------- .../validateApifyWebhookRequest.test.ts | 55 +++++++++++++ lib/apify/handleApifyWebhook.ts | 40 ---------- .../handleInstagramCommentsScraper.ts | 45 +++++------ .../handleInstagramProfileScraperResults.ts | 6 +- lib/apify/postApifyWebhookHandler.ts | 51 +++++++----- lib/apify/validateApifyBody.ts | 52 ------------ lib/apify/validateApifyWebhookRequest.ts | 62 ++++++++++++++ 10 files changed, 237 insertions(+), 245 deletions(-) delete mode 100644 lib/apify/__tests__/handleApifyWebhook.test.ts delete mode 100644 lib/apify/__tests__/validateApifyBody.test.ts create mode 100644 lib/apify/__tests__/validateApifyWebhookRequest.test.ts delete mode 100644 lib/apify/handleApifyWebhook.ts delete mode 100644 lib/apify/validateApifyBody.ts create mode 100644 lib/apify/validateApifyWebhookRequest.ts diff --git a/lib/apify/__tests__/handleApifyWebhook.test.ts b/lib/apify/__tests__/handleApifyWebhook.test.ts deleted file mode 100644 index 21173084f..000000000 --- a/lib/apify/__tests__/handleApifyWebhook.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; -import { handleApifyWebhook } from "../handleApifyWebhook"; -import { handleInstagramProfileScraperResults } from "../instagram/handleInstagramProfileScraperResults"; -import { handleInstagramCommentsScraper } from "../instagram/handleInstagramCommentsScraper"; - -vi.mock("../instagram/handleInstagramProfileScraperResults", () => ({ - handleInstagramProfileScraperResults: vi.fn(), -})); -vi.mock("../instagram/handleInstagramCommentsScraper", () => ({ - handleInstagramCommentsScraper: vi.fn(), -})); - -const base = { - userId: "u", - createdAt: "2026-01-01T00:00:00Z", - eventType: "ACTOR.RUN.SUCCEEDED", - resource: { defaultDatasetId: "ds_1" }, -}; - -describe("handleApifyWebhook", () => { - beforeEach(() => vi.clearAllMocks()); - - it("dispatches Instagram profile actor to profile handler", async () => { - vi.mocked(handleInstagramProfileScraperResults).mockResolvedValue({ posts: [] } as never); - await handleApifyWebhook({ ...base, eventData: { actorId: "dSCLg0C3YEZ83HzYX" } }); - expect(handleInstagramProfileScraperResults).toHaveBeenCalledOnce(); - expect(handleInstagramCommentsScraper).not.toHaveBeenCalled(); - }); - - it("dispatches Instagram comments actor to comments handler", async () => { - vi.mocked(handleInstagramCommentsScraper).mockResolvedValue({ comments: [] } as never); - await handleApifyWebhook({ ...base, eventData: { actorId: "SbK00X0JYCPblD2wp" } }); - expect(handleInstagramCommentsScraper).toHaveBeenCalledOnce(); - expect(handleInstagramProfileScraperResults).not.toHaveBeenCalled(); - }); - - it("returns fallback for unhandled actor ids without throwing", async () => { - const result = await handleApifyWebhook({ ...base, eventData: { actorId: "unknown" } }); - expect(result).toMatchObject({ posts: [], social: null }); - }); - - it("swallows handler errors and returns fallback", async () => { - vi.mocked(handleInstagramCommentsScraper).mockRejectedValueOnce(new Error("boom")); - const result = await handleApifyWebhook({ - ...base, - eventData: { actorId: "SbK00X0JYCPblD2wp" }, - }); - expect(result).toMatchObject({ posts: [], social: null }); - }); -}); diff --git a/lib/apify/__tests__/postApifyWebhookHandler.test.ts b/lib/apify/__tests__/postApifyWebhookHandler.test.ts index c8c915bac..d24353672 100644 --- a/lib/apify/__tests__/postApifyWebhookHandler.test.ts +++ b/lib/apify/__tests__/postApifyWebhookHandler.test.ts @@ -1,9 +1,15 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; import { postApifyWebhookHandler } from "../postApifyWebhookHandler"; -import { handleApifyWebhook } from "../handleApifyWebhook"; +import { handleInstagramProfileScraperResults } from "../instagram/handleInstagramProfileScraperResults"; +import { handleInstagramCommentsScraper } from "../instagram/handleInstagramCommentsScraper"; -vi.mock("../handleApifyWebhook", () => ({ handleApifyWebhook: vi.fn() })); +vi.mock("../instagram/handleInstagramProfileScraperResults", () => ({ + handleInstagramProfileScraperResults: vi.fn(), +})); +vi.mock("../instagram/handleInstagramCommentsScraper", () => ({ + handleInstagramCommentsScraper: vi.fn(), +})); function makeRequest(body: unknown, raw?: string) { return new NextRequest("http://localhost/api/apify", { @@ -13,43 +19,83 @@ function makeRequest(body: unknown, raw?: string) { }); } +const baseBody = { + userId: "u", + createdAt: "2026-01-01T00:00:00Z", + eventType: "ACTOR.RUN.SUCCEEDED", + resource: { defaultDatasetId: "ds_1" }, +}; + describe("postApifyWebhookHandler", () => { beforeEach(() => vi.clearAllMocks()); - it("returns 200 with dispatcher result on valid payload", async () => { - vi.mocked(handleApifyWebhook).mockResolvedValue({ posts: [1, 2] } as never); + it("dispatches profile scraper results for the IG profile actor", async () => { + vi.mocked(handleInstagramProfileScraperResults).mockResolvedValue({ posts: [1, 2] } as never); const res = await postApifyWebhookHandler( - makeRequest({ - userId: "u", - createdAt: "2026-01-01T00:00:00Z", - eventType: "ACTOR.RUN.SUCCEEDED", - eventData: { actorId: "dSCLg0C3YEZ83HzYX" }, - resource: { defaultDatasetId: "ds_1" }, - }), + makeRequest({ ...baseBody, eventData: { actorId: "dSCLg0C3YEZ83HzYX" } }), ); expect(res.status).toBe(200); expect(await res.json()).toEqual({ posts: [1, 2] }); - expect(handleApifyWebhook).toHaveBeenCalledOnce(); + expect(handleInstagramProfileScraperResults).toHaveBeenCalledOnce(); + expect(handleInstagramCommentsScraper).not.toHaveBeenCalled(); + }); + + it("dispatches comments scraper for the IG comments actor", async () => { + vi.mocked(handleInstagramCommentsScraper).mockResolvedValue({ totalComments: 3 } as never); + + const res = await postApifyWebhookHandler( + makeRequest({ ...baseBody, eventData: { actorId: "SbK00X0JYCPblD2wp" } }), + ); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ totalComments: 3 }); + expect(handleInstagramCommentsScraper).toHaveBeenCalledOnce(); + expect(handleInstagramProfileScraperResults).not.toHaveBeenCalled(); + }); + + it("returns a 200 error response for unhandled actor ids", async () => { + const res = await postApifyWebhookHandler( + makeRequest({ ...baseBody, eventData: { actorId: "unknown_actor" } }), + ); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.status).toBe("error"); + expect(body.error).toContain("unknown_actor"); + expect(handleInstagramProfileScraperResults).not.toHaveBeenCalled(); + expect(handleInstagramCommentsScraper).not.toHaveBeenCalled(); + }); + + it("returns a 200 error response when the dispatched handler throws", async () => { + vi.mocked(handleInstagramProfileScraperResults).mockRejectedValue(new Error("boom")); + + const res = await postApifyWebhookHandler( + makeRequest({ ...baseBody, eventData: { actorId: "dSCLg0C3YEZ83HzYX" } }), + ); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body).toEqual({ status: "error", error: "boom" }); }); - it("returns 200 with the validator's error response for invalid payloads", async () => { + it("propagates the validator's error response for invalid payloads", async () => { const res = await postApifyWebhookHandler(makeRequest({ bogus: true })); expect(res.status).toBe(200); const body = await res.json(); expect(body.status).toBe("error"); expect(typeof body.error).toBe("string"); - expect(handleApifyWebhook).not.toHaveBeenCalled(); + expect(handleInstagramProfileScraperResults).not.toHaveBeenCalled(); + expect(handleInstagramCommentsScraper).not.toHaveBeenCalled(); }); - it("returns 200 with an Invalid JSON message for malformed bodies", async () => { + it("propagates the validator's Invalid JSON response for malformed bodies", async () => { const res = await postApifyWebhookHandler(makeRequest(null, "not json")); expect(res.status).toBe(200); const body = await res.json(); - expect(body.message).toBe("Invalid JSON"); - expect(handleApifyWebhook).not.toHaveBeenCalled(); + expect(body).toEqual({ status: "error", error: "Invalid JSON" }); }); }); diff --git a/lib/apify/__tests__/validateApifyBody.test.ts b/lib/apify/__tests__/validateApifyBody.test.ts deleted file mode 100644 index 7496f2dfe..000000000 --- a/lib/apify/__tests__/validateApifyBody.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { NextResponse } from "next/server"; -import { validateApifyBody } from "../validateApifyBody"; - -describe("validateApifyBody", () => { - it("returns the parsed body for a valid Apify payload", () => { - const body = { - userId: "u", - createdAt: "2026-01-01T00:00:00Z", - eventType: "ACTOR.RUN.SUCCEEDED", - eventData: { actorId: "dSCLg0C3YEZ83HzYX" }, - resource: { defaultDatasetId: "ds_1" }, - }; - - const result = validateApifyBody(body); - - expect(result).not.toBeInstanceOf(NextResponse); - if (!(result instanceof NextResponse)) { - expect(result.eventData.actorId).toBe("dSCLg0C3YEZ83HzYX"); - expect(result.resource.defaultDatasetId).toBe("ds_1"); - } - }); - - it("returns a 200 NextResponse with the project error shape on missing fields", async () => { - const result = validateApifyBody({ eventData: {}, resource: {} }); - - expect(result).toBeInstanceOf(NextResponse); - if (result instanceof NextResponse) { - expect(result.status).toBe(200); - const body = await result.json(); - expect(body.status).toBe("error"); - expect(Array.isArray(body.missing_fields)).toBe(true); - expect(typeof body.error).toBe("string"); - } - }); - - it("returns a NextResponse for non-object bodies", () => { - expect(validateApifyBody(undefined)).toBeInstanceOf(NextResponse); - expect(validateApifyBody("not an object")).toBeInstanceOf(NextResponse); - }); -}); diff --git a/lib/apify/__tests__/validateApifyWebhookRequest.test.ts b/lib/apify/__tests__/validateApifyWebhookRequest.test.ts new file mode 100644 index 000000000..9d0b938b3 --- /dev/null +++ b/lib/apify/__tests__/validateApifyWebhookRequest.test.ts @@ -0,0 +1,55 @@ +import { describe, it, expect } from "vitest"; +import { NextRequest, NextResponse } from "next/server"; +import { validateApifyWebhookRequest } from "../validateApifyWebhookRequest"; + +function makeRequest(body: unknown, raw?: string) { + return new NextRequest("http://localhost/api/apify", { + method: "POST", + headers: { "content-type": "application/json" }, + body: raw ?? JSON.stringify(body), + }); +} + +describe("validateApifyWebhookRequest", () => { + it("returns the parsed payload for a valid Apify request", async () => { + const result = await validateApifyWebhookRequest( + makeRequest({ + userId: "u", + createdAt: "2026-01-01T00:00:00Z", + eventType: "ACTOR.RUN.SUCCEEDED", + eventData: { actorId: "dSCLg0C3YEZ83HzYX" }, + resource: { defaultDatasetId: "ds_1" }, + }), + ); + + expect(result).not.toBeInstanceOf(NextResponse); + if (!(result instanceof NextResponse)) { + expect(result.eventData.actorId).toBe("dSCLg0C3YEZ83HzYX"); + expect(result.resource.defaultDatasetId).toBe("ds_1"); + } + }); + + it("returns a 200 NextResponse with the project error shape on missing fields", async () => { + const result = await validateApifyWebhookRequest(makeRequest({ eventData: {}, resource: {} })); + + expect(result).toBeInstanceOf(NextResponse); + if (result instanceof NextResponse) { + expect(result.status).toBe(200); + const body = await result.json(); + expect(body.status).toBe("error"); + expect(Array.isArray(body.missing_fields)).toBe(true); + expect(typeof body.error).toBe("string"); + } + }); + + it("returns a 200 NextResponse with Invalid JSON for malformed bodies", async () => { + const result = await validateApifyWebhookRequest(makeRequest(null, "not json")); + + expect(result).toBeInstanceOf(NextResponse); + if (result instanceof NextResponse) { + expect(result.status).toBe(200); + const body = await result.json(); + expect(body).toEqual({ status: "error", error: "Invalid JSON" }); + } + }); +}); diff --git a/lib/apify/handleApifyWebhook.ts b/lib/apify/handleApifyWebhook.ts deleted file mode 100644 index 77d50a97d..000000000 --- a/lib/apify/handleApifyWebhook.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { ApifyBody } from "@/lib/apify/validateApifyBody"; -import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; -import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; - -const INSTAGRAM_PROFILE_ACTOR_ID = "dSCLg0C3YEZ83HzYX"; -const INSTAGRAM_COMMENTS_ACTOR_ID = "SbK00X0JYCPblD2wp"; - -const fallbackResponse = { - posts: [], - social: null, - accountSocials: [], - accountArtistIds: [], - accountEmails: [], - sentEmails: null, -}; - -/** - * Dispatches an Apify webhook payload to the handler matching the - * `eventData.actorId`. Unknown actor ids are logged and return an - * empty-shaped response. Never throws — handler failures are caught - * and logged so the webhook route can always reply 200. - * - * @param parsed - Validated Apify webhook body. - */ -export async function handleApifyWebhook(parsed: ApifyBody) { - try { - switch (parsed.eventData.actorId) { - case INSTAGRAM_PROFILE_ACTOR_ID: - return await handleInstagramProfileScraperResults(parsed); - case INSTAGRAM_COMMENTS_ACTOR_ID: - return await handleInstagramCommentsScraper(parsed); - default: - console.log(`[WARN] handleApifyWebhook: unhandled actorId ${parsed.eventData.actorId}`); - return fallbackResponse; - } - } catch (e) { - console.error("[ERROR] handleApifyWebhook:", e); - return fallbackResponse; - } -} diff --git a/lib/apify/instagram/handleInstagramCommentsScraper.ts b/lib/apify/instagram/handleInstagramCommentsScraper.ts index 99bb0428d..8a10dd71d 100644 --- a/lib/apify/instagram/handleInstagramCommentsScraper.ts +++ b/lib/apify/instagram/handleInstagramCommentsScraper.ts @@ -1,18 +1,22 @@ import { getDataset } from "@/lib/apify/getDataset"; import { saveApifyInstagramComments } from "@/lib/apify/instagram/saveApifyInstagramComments"; import { startInstagramProfileScraping } from "@/lib/apify/instagram/startInstagramProfileScraping"; -import type { ApifyBody } from "@/lib/apify/validateApifyBody"; +import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; import type { ApifyInstagramComment } from "@/lib/apify/types"; /** * Handles Instagram comments scraper Apify webhook results: * - Persists comments into `post_comments`. * - Kicks off a fan-profile scrape for the distinct commenter - * usernames so their socials get indexed. + * usernames so their socials get indexed (best-effort: failures + * are logged but don't fail the comments ingestion). * - * @param parsed - Validated Apify webhook body. + * Errors from the dataset fetch or comment persistence propagate up + * to the webhook route, which logs and returns an error response. + * + * @param parsed - Validated Apify webhook payload. */ -export async function handleInstagramCommentsScraper(parsed: ApifyBody) { +export async function handleInstagramCommentsScraper(parsed: ApifyWebhookPayload) { const datasetId = parsed.resource.defaultDatasetId; const empty = { comments: [] as ApifyInstagramComment[], @@ -22,29 +26,24 @@ export async function handleInstagramCommentsScraper(parsed: ApifyBody) { if (!datasetId) return empty; - try { - const dataset = await getDataset(datasetId); - if (!Array.isArray(dataset)) return empty; + const dataset = await getDataset(datasetId); + if (!Array.isArray(dataset)) return empty; - const comments = dataset as ApifyInstagramComment[]; - const processedPostUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); - const totalComments = comments.length; + const comments = dataset as ApifyInstagramComment[]; + const processedPostUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); + const totalComments = comments.length; - await saveApifyInstagramComments(comments); + await saveApifyInstagramComments(comments); - const fanHandles = Array.from(new Set(comments.map(c => c.ownerUsername).filter(Boolean))); + const fanHandles = Array.from(new Set(comments.map(c => c.ownerUsername).filter(Boolean))); - if (fanHandles.length > 0) { - try { - await startInstagramProfileScraping(fanHandles); - } catch (error) { - console.error("[ERROR] fan profile scrape failed:", error); - } + if (fanHandles.length > 0) { + try { + await startInstagramProfileScraping(fanHandles); + } catch (error) { + console.error("[ERROR] fan profile scrape failed:", error); } - - return { comments, processedPostUrls, totalComments }; - } catch (error) { - console.error("[ERROR] handleInstagramCommentsScraper:", error); - return empty; } + + return { comments, processedPostUrls, totalComments }; } diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index 4c6306af9..60015e0e3 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -15,7 +15,7 @@ import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; import { getFetchableUrl } from "@/lib/arweave/getFetchableUrl"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; -import type { ApifyBody } from "@/lib/apify/validateApifyBody"; +import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; import type { Tables } from "@/types/database.types"; type ProfileScraperResult = { @@ -38,10 +38,10 @@ type ProfileScraperResult = { * failures inside the chain are logged but allowed to propagate to the * webhook route's outer try/catch, which always returns 200. * - * @param parsed - Validated Apify webhook body. + * @param parsed - Validated Apify webhook payload. */ export async function handleInstagramProfileScraperResults( - parsed: ApifyBody, + parsed: ApifyWebhookPayload, ): Promise { const datasetId = parsed.resource.defaultDatasetId; const empty: ProfileScraperResult = { diff --git a/lib/apify/postApifyWebhookHandler.ts b/lib/apify/postApifyWebhookHandler.ts index 36aa1b2ac..23860c2ec 100644 --- a/lib/apify/postApifyWebhookHandler.ts +++ b/lib/apify/postApifyWebhookHandler.ts @@ -1,36 +1,49 @@ import { NextRequest, NextResponse } from "next/server"; -import { validateApifyBody } from "@/lib/apify/validateApifyBody"; -import { handleApifyWebhook } from "@/lib/apify/handleApifyWebhook"; +import { validateApifyWebhookRequest } from "@/lib/apify/validateApifyWebhookRequest"; +import { handleInstagramProfileScraperResults } from "@/lib/apify/instagram/handleInstagramProfileScraperResults"; +import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInstagramCommentsScraper"; + +const INSTAGRAM_PROFILE_ACTOR_ID = "dSCLg0C3YEZ83HzYX"; +const INSTAGRAM_COMMENTS_ACTOR_ID = "SbK00X0JYCPblD2wp"; /** * Handler for `POST /api/apify`. Always responds 200 so Apify does not - * retry on our side of a failure — malformed payloads and downstream - * errors are logged and surfaced in the response body. + * retry on our side of a failure — malformed payloads, unknown actors, + * and downstream errors are logged and surfaced as a `status: "error"` + * JSON body. * * @param request - Incoming webhook request. - * @returns JSON response (always status 200). */ export async function postApifyWebhookHandler(request: NextRequest): Promise { - let body: unknown; - try { - body = await request.json(); - } catch { - console.warn("[WARN] postApifyWebhookHandler: invalid JSON"); - return NextResponse.json({ message: "Invalid JSON" }, { status: 200 }); - } + const validated = await validateApifyWebhookRequest(request); + if (validated instanceof NextResponse) return validated; - const validated = validateApifyBody(body); - if (validated instanceof NextResponse) { - return validated; - } + const { actorId } = validated.eventData; try { - const result = await handleApifyWebhook(validated); - return NextResponse.json(result, { status: 200 }); + switch (actorId) { + case INSTAGRAM_PROFILE_ACTOR_ID: { + const result = await handleInstagramProfileScraperResults(validated); + return NextResponse.json(result, { status: 200 }); + } + case INSTAGRAM_COMMENTS_ACTOR_ID: { + const result = await handleInstagramCommentsScraper(validated); + return NextResponse.json(result, { status: 200 }); + } + default: + console.warn(`[WARN] postApifyWebhookHandler: unhandled actorId ${actorId}`); + return NextResponse.json( + { status: "error", error: `Unhandled actorId: ${actorId}` }, + { status: 200 }, + ); + } } catch (error) { console.error("[ERROR] postApifyWebhookHandler:", error); return NextResponse.json( - { message: "Apify webhook received (handler error)" }, + { + status: "error", + error: error instanceof Error ? error.message : String(error), + }, { status: 200 }, ); } diff --git a/lib/apify/validateApifyBody.ts b/lib/apify/validateApifyBody.ts deleted file mode 100644 index 9af7b469e..000000000 --- a/lib/apify/validateApifyBody.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { NextResponse } from "next/server"; -import { getCorsHeaders } from "@/lib/networking/getCorsHeaders"; -import { z } from "zod"; - -/** - * Schema for `POST /api/apify` webhook bodies. Only validates the - * fields we branch on + read downstream (`actorId` for dispatch, - * `defaultDatasetId` for fetch). Extra keys are stripped so upstream - * schema drift does not drop events. - */ -export const apifyBodySchema = z.object({ - eventData: z.object({ - actorId: z.string({ message: "eventData.actorId is required" }), - }), - resource: z.object({ - defaultDatasetId: z.string({ message: "resource.defaultDatasetId is required" }), - }), -}); - -export type ApifyBody = z.infer; - -/** - * Validates request body for POST /api/apify. - * - * The route's contract with Apify is to always reply 2xx so Apify does - * not retry, so the failure NextResponse uses status 200 with the - * project-standard error body shape. - * - * @param body - The request body - * @returns A NextResponse with an error if validation fails, or the - * validated body if validation passes. - */ -export function validateApifyBody(body: unknown): NextResponse | ApifyBody { - const result = apifyBodySchema.safeParse(body); - - if (!result.success) { - const firstError = result.error.issues[0]; - return NextResponse.json( - { - status: "error", - missing_fields: firstError.path, - error: firstError.message, - }, - { - status: 200, - headers: getCorsHeaders(), - }, - ); - } - - return result.data; -} diff --git a/lib/apify/validateApifyWebhookRequest.ts b/lib/apify/validateApifyWebhookRequest.ts new file mode 100644 index 000000000..d37496aa6 --- /dev/null +++ b/lib/apify/validateApifyWebhookRequest.ts @@ -0,0 +1,62 @@ +import { NextRequest, NextResponse } from "next/server"; +import { getCorsHeaders } from "@/lib/networking/getCorsHeaders"; +import { z } from "zod"; + +/** + * Schema for `POST /api/apify` webhook payloads. Only validates the + * fields we branch on + read downstream (`actorId` for dispatch, + * `defaultDatasetId` for fetch). Extra keys are stripped so upstream + * schema drift does not drop events. + */ +export const apifyWebhookPayloadSchema = z.object({ + eventData: z.object({ + actorId: z.string({ message: "eventData.actorId is required" }), + }), + resource: z.object({ + defaultDatasetId: z.string({ message: "resource.defaultDatasetId is required" }), + }), +}); + +export type ApifyWebhookPayload = z.infer; + +/** + * Parses and validates a POST /api/apify webhook request. + * + * Returns a NextResponse on failure (malformed JSON or schema + * mismatch) or the validated payload on success. The route's contract + * with Apify is to always reply 2xx so Apify does not retry, so the + * failure NextResponse uses status 200 with the project-standard + * error body shape. + * + * @param request - Incoming webhook request. + * @returns A NextResponse with an error if validation fails, or the + * validated payload if validation passes. + */ +export async function validateApifyWebhookRequest( + request: NextRequest, +): Promise { + let body: unknown; + try { + body = await request.json(); + } catch { + return NextResponse.json( + { status: "error", error: "Invalid JSON" }, + { status: 200, headers: getCorsHeaders() }, + ); + } + + const result = apifyWebhookPayloadSchema.safeParse(body); + if (!result.success) { + const firstError = result.error.issues[0]; + return NextResponse.json( + { + status: "error", + missing_fields: firstError.path, + error: firstError.message, + }, + { status: 200, headers: getCorsHeaders() }, + ); + } + + return result.data; +} From 1bdc34c30586024c57af02df33a0f6ed97350a9a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 19:38:56 +0000 Subject: [PATCH 10/23] refactor(apify): rename postApifyWebhookHandler -> postApifyHandler The endpoint is `POST /api/apify`, so the handler should follow the project's `postHandler` convention used by postSandboxesFilesHandler, postArtistSocialsScrapeHandler, etc. The "Webhook" suffix doesn't appear in the URL path and was redundant. Renames the file, the test file, the exported function, and updates the route import. No behavior change. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- app/api/apify/route.ts | 4 ++-- ...kHandler.test.ts => postApifyHandler.test.ts} | 16 ++++++++-------- ...pifyWebhookHandler.ts => postApifyHandler.ts} | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) rename lib/apify/__tests__/{postApifyWebhookHandler.test.ts => postApifyHandler.test.ts} (88%) rename lib/apify/{postApifyWebhookHandler.ts => postApifyHandler.ts} (87%) diff --git a/app/api/apify/route.ts b/app/api/apify/route.ts index 5c005e53d..f3d1fe762 100644 --- a/app/api/apify/route.ts +++ b/app/api/apify/route.ts @@ -1,5 +1,5 @@ import { NextRequest } from "next/server"; -import { postApifyWebhookHandler } from "@/lib/apify/postApifyWebhookHandler"; +import { postApifyHandler } from "@/lib/apify/postApifyHandler"; export const maxDuration = 60; export const dynamic = "force-dynamic"; @@ -15,5 +15,5 @@ export const revalidate = 0; * @returns JSON response describing the processed payload (always 200). */ export async function POST(request: NextRequest) { - return postApifyWebhookHandler(request); + return postApifyHandler(request); } diff --git a/lib/apify/__tests__/postApifyWebhookHandler.test.ts b/lib/apify/__tests__/postApifyHandler.test.ts similarity index 88% rename from lib/apify/__tests__/postApifyWebhookHandler.test.ts rename to lib/apify/__tests__/postApifyHandler.test.ts index d24353672..59dcba0f9 100644 --- a/lib/apify/__tests__/postApifyWebhookHandler.test.ts +++ b/lib/apify/__tests__/postApifyHandler.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; -import { postApifyWebhookHandler } from "../postApifyWebhookHandler"; +import { postApifyHandler } from "../postApifyHandler"; import { handleInstagramProfileScraperResults } from "../instagram/handleInstagramProfileScraperResults"; import { handleInstagramCommentsScraper } from "../instagram/handleInstagramCommentsScraper"; @@ -26,13 +26,13 @@ const baseBody = { resource: { defaultDatasetId: "ds_1" }, }; -describe("postApifyWebhookHandler", () => { +describe("postApifyHandler", () => { beforeEach(() => vi.clearAllMocks()); it("dispatches profile scraper results for the IG profile actor", async () => { vi.mocked(handleInstagramProfileScraperResults).mockResolvedValue({ posts: [1, 2] } as never); - const res = await postApifyWebhookHandler( + const res = await postApifyHandler( makeRequest({ ...baseBody, eventData: { actorId: "dSCLg0C3YEZ83HzYX" } }), ); @@ -45,7 +45,7 @@ describe("postApifyWebhookHandler", () => { it("dispatches comments scraper for the IG comments actor", async () => { vi.mocked(handleInstagramCommentsScraper).mockResolvedValue({ totalComments: 3 } as never); - const res = await postApifyWebhookHandler( + const res = await postApifyHandler( makeRequest({ ...baseBody, eventData: { actorId: "SbK00X0JYCPblD2wp" } }), ); @@ -56,7 +56,7 @@ describe("postApifyWebhookHandler", () => { }); it("returns a 200 error response for unhandled actor ids", async () => { - const res = await postApifyWebhookHandler( + const res = await postApifyHandler( makeRequest({ ...baseBody, eventData: { actorId: "unknown_actor" } }), ); @@ -71,7 +71,7 @@ describe("postApifyWebhookHandler", () => { it("returns a 200 error response when the dispatched handler throws", async () => { vi.mocked(handleInstagramProfileScraperResults).mockRejectedValue(new Error("boom")); - const res = await postApifyWebhookHandler( + const res = await postApifyHandler( makeRequest({ ...baseBody, eventData: { actorId: "dSCLg0C3YEZ83HzYX" } }), ); @@ -81,7 +81,7 @@ describe("postApifyWebhookHandler", () => { }); it("propagates the validator's error response for invalid payloads", async () => { - const res = await postApifyWebhookHandler(makeRequest({ bogus: true })); + const res = await postApifyHandler(makeRequest({ bogus: true })); expect(res.status).toBe(200); const body = await res.json(); @@ -92,7 +92,7 @@ describe("postApifyWebhookHandler", () => { }); it("propagates the validator's Invalid JSON response for malformed bodies", async () => { - const res = await postApifyWebhookHandler(makeRequest(null, "not json")); + const res = await postApifyHandler(makeRequest(null, "not json")); expect(res.status).toBe(200); const body = await res.json(); diff --git a/lib/apify/postApifyWebhookHandler.ts b/lib/apify/postApifyHandler.ts similarity index 87% rename from lib/apify/postApifyWebhookHandler.ts rename to lib/apify/postApifyHandler.ts index 23860c2ec..51f8ecf57 100644 --- a/lib/apify/postApifyWebhookHandler.ts +++ b/lib/apify/postApifyHandler.ts @@ -14,7 +14,7 @@ const INSTAGRAM_COMMENTS_ACTOR_ID = "SbK00X0JYCPblD2wp"; * * @param request - Incoming webhook request. */ -export async function postApifyWebhookHandler(request: NextRequest): Promise { +export async function postApifyHandler(request: NextRequest): Promise { const validated = await validateApifyWebhookRequest(request); if (validated instanceof NextResponse) return validated; @@ -31,14 +31,14 @@ export async function postApifyWebhookHandler(request: NextRequest): Promise Date: Tue, 28 Apr 2026 19:40:19 +0000 Subject: [PATCH 11/23] refactor(apify): rename postApifyHandler -> apifyWebhookHandler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HTTP-verb prefix isn't required — handlers across the codebase (getCatalogsHandler, getAccountHandler, etc.) describe what they do, not the verb. Drop the `post` prefix and keep the descriptive name `apifyWebhookHandler` since this is the handler for the apify webhook. Renames file, test file, exported function, and route import. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- app/api/apify/route.ts | 4 ++-- ...ndler.test.ts => apifyWebhookHandler.test.ts} | 16 ++++++++-------- ...ostApifyHandler.ts => apifyWebhookHandler.ts} | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) rename lib/apify/__tests__/{postApifyHandler.test.ts => apifyWebhookHandler.test.ts} (89%) rename lib/apify/{postApifyHandler.ts => apifyWebhookHandler.ts} (88%) diff --git a/app/api/apify/route.ts b/app/api/apify/route.ts index f3d1fe762..045e697ed 100644 --- a/app/api/apify/route.ts +++ b/app/api/apify/route.ts @@ -1,5 +1,5 @@ import { NextRequest } from "next/server"; -import { postApifyHandler } from "@/lib/apify/postApifyHandler"; +import { apifyWebhookHandler } from "@/lib/apify/apifyWebhookHandler"; export const maxDuration = 60; export const dynamic = "force-dynamic"; @@ -15,5 +15,5 @@ export const revalidate = 0; * @returns JSON response describing the processed payload (always 200). */ export async function POST(request: NextRequest) { - return postApifyHandler(request); + return apifyWebhookHandler(request); } diff --git a/lib/apify/__tests__/postApifyHandler.test.ts b/lib/apify/__tests__/apifyWebhookHandler.test.ts similarity index 89% rename from lib/apify/__tests__/postApifyHandler.test.ts rename to lib/apify/__tests__/apifyWebhookHandler.test.ts index 59dcba0f9..7623b0391 100644 --- a/lib/apify/__tests__/postApifyHandler.test.ts +++ b/lib/apify/__tests__/apifyWebhookHandler.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; -import { postApifyHandler } from "../postApifyHandler"; +import { apifyWebhookHandler } from "../apifyWebhookHandler"; import { handleInstagramProfileScraperResults } from "../instagram/handleInstagramProfileScraperResults"; import { handleInstagramCommentsScraper } from "../instagram/handleInstagramCommentsScraper"; @@ -26,13 +26,13 @@ const baseBody = { resource: { defaultDatasetId: "ds_1" }, }; -describe("postApifyHandler", () => { +describe("apifyWebhookHandler", () => { beforeEach(() => vi.clearAllMocks()); it("dispatches profile scraper results for the IG profile actor", async () => { vi.mocked(handleInstagramProfileScraperResults).mockResolvedValue({ posts: [1, 2] } as never); - const res = await postApifyHandler( + const res = await apifyWebhookHandler( makeRequest({ ...baseBody, eventData: { actorId: "dSCLg0C3YEZ83HzYX" } }), ); @@ -45,7 +45,7 @@ describe("postApifyHandler", () => { it("dispatches comments scraper for the IG comments actor", async () => { vi.mocked(handleInstagramCommentsScraper).mockResolvedValue({ totalComments: 3 } as never); - const res = await postApifyHandler( + const res = await apifyWebhookHandler( makeRequest({ ...baseBody, eventData: { actorId: "SbK00X0JYCPblD2wp" } }), ); @@ -56,7 +56,7 @@ describe("postApifyHandler", () => { }); it("returns a 200 error response for unhandled actor ids", async () => { - const res = await postApifyHandler( + const res = await apifyWebhookHandler( makeRequest({ ...baseBody, eventData: { actorId: "unknown_actor" } }), ); @@ -71,7 +71,7 @@ describe("postApifyHandler", () => { it("returns a 200 error response when the dispatched handler throws", async () => { vi.mocked(handleInstagramProfileScraperResults).mockRejectedValue(new Error("boom")); - const res = await postApifyHandler( + const res = await apifyWebhookHandler( makeRequest({ ...baseBody, eventData: { actorId: "dSCLg0C3YEZ83HzYX" } }), ); @@ -81,7 +81,7 @@ describe("postApifyHandler", () => { }); it("propagates the validator's error response for invalid payloads", async () => { - const res = await postApifyHandler(makeRequest({ bogus: true })); + const res = await apifyWebhookHandler(makeRequest({ bogus: true })); expect(res.status).toBe(200); const body = await res.json(); @@ -92,7 +92,7 @@ describe("postApifyHandler", () => { }); it("propagates the validator's Invalid JSON response for malformed bodies", async () => { - const res = await postApifyHandler(makeRequest(null, "not json")); + const res = await apifyWebhookHandler(makeRequest(null, "not json")); expect(res.status).toBe(200); const body = await res.json(); diff --git a/lib/apify/postApifyHandler.ts b/lib/apify/apifyWebhookHandler.ts similarity index 88% rename from lib/apify/postApifyHandler.ts rename to lib/apify/apifyWebhookHandler.ts index 51f8ecf57..544c6f0e5 100644 --- a/lib/apify/postApifyHandler.ts +++ b/lib/apify/apifyWebhookHandler.ts @@ -14,7 +14,7 @@ const INSTAGRAM_COMMENTS_ACTOR_ID = "SbK00X0JYCPblD2wp"; * * @param request - Incoming webhook request. */ -export async function postApifyHandler(request: NextRequest): Promise { +export async function apifyWebhookHandler(request: NextRequest): Promise { const validated = await validateApifyWebhookRequest(request); if (validated instanceof NextResponse) return validated; @@ -31,14 +31,14 @@ export async function postApifyHandler(request: NextRequest): Promise Date: Tue, 28 Apr 2026 19:47:22 +0000 Subject: [PATCH 12/23] refactor(apify): use SDK listItems() in getDataset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Apify SDK already exposes `apifyClient.dataset(id).listItems()` — the same method getScraperResultsHandler uses — so there's no reason to hand-roll a fetch + token + status-check chain in getDataset. Drop the manual implementation in favor of the SDK call, which also picks up the configured client's token from lib/apify/client.ts. Errors now propagate to the caller (apifyWebhookHandler's outer try/catch) instead of being swallowed and returned as `[]`, matching the webhook handler's error-response contract. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- lib/apify/getDataset.ts | 39 ++++++++------------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/lib/apify/getDataset.ts b/lib/apify/getDataset.ts index b6f1ae838..b264987ef 100644 --- a/lib/apify/getDataset.ts +++ b/lib/apify/getDataset.ts @@ -1,36 +1,13 @@ +import apifyClient from "@/lib/apify/client"; + /** - * Fetches items from an Apify dataset by id. Uses the REST endpoint - * rather than the SDK so we do not have to materialize the whole - * dataset client just to pull items. + * Fetches items from an Apify dataset by id via the SDK. Returns the + * items array; errors propagate to the caller so the webhook handler + * can surface them in its error response. * * @param datasetId - Apify dataset id. - * @returns Parsed dataset body (array) or `[]` on failure. */ -export async function getDataset(datasetId: string): Promise { - const token = process.env.APIFY_TOKEN; - if (!token) { - console.error("[ERROR] getDataset: missing APIFY_TOKEN"); - return []; - } - - try { - const response = await fetch(`https://api.apify.com/v2/datasets/${datasetId}/items`, { - method: "GET", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - }, - }); - - if (!response.ok) { - console.error(`[ERROR] getDataset: ${response.status} ${response.statusText}`); - return []; - } - - const data = await response.json(); - return Array.isArray(data) ? data : []; - } catch (err) { - console.error("[ERROR] getDataset:", err); - return []; - } +export async function getDataset(datasetId: string) { + const { items } = await apifyClient.dataset(datasetId).listItems(); + return items; } From 8c26de222abab623e33fdb03e24678b7676020bc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 19:49:39 +0000 Subject: [PATCH 13/23] refactor(apify): drop getDataset wrapper, call SDK directly After switching getDataset to use apifyClient.dataset(id).listItems(), the function was just a 2-line passthrough adding no value. Inline the SDK call at the two call sites in handleInstagramCommentsScraper and handleInstagramProfileScraperResults and delete the wrapper. Tests now mock @/lib/apify/client (matching the existing pattern in getScraperResultsHandler.test.ts) instead of the removed wrapper. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- lib/apify/getDataset.ts | 13 ------------- .../handleInstagramCommentsScraper.test.ts | 11 ++++++++--- .../handleInstagramProfileScraperResults.test.ts | 14 ++++++++++---- .../instagram/handleInstagramCommentsScraper.ts | 8 +++----- .../handleInstagramProfileScraperResults.ts | 4 ++-- 5 files changed, 23 insertions(+), 27 deletions(-) delete mode 100644 lib/apify/getDataset.ts diff --git a/lib/apify/getDataset.ts b/lib/apify/getDataset.ts deleted file mode 100644 index b264987ef..000000000 --- a/lib/apify/getDataset.ts +++ /dev/null @@ -1,13 +0,0 @@ -import apifyClient from "@/lib/apify/client"; - -/** - * Fetches items from an Apify dataset by id via the SDK. Returns the - * items array; errors propagate to the caller so the webhook handler - * can surface them in its error response. - * - * @param datasetId - Apify dataset id. - */ -export async function getDataset(datasetId: string) { - const { items } = await apifyClient.dataset(datasetId).listItems(); - return items; -} diff --git a/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts b/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts index c34bc5ee9..a53acd325 100644 --- a/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts @@ -1,15 +1,20 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleInstagramCommentsScraper } from "../handleInstagramCommentsScraper"; -import { getDataset } from "@/lib/apify/getDataset"; +import apifyClient from "@/lib/apify/client"; import { saveApifyInstagramComments } from "../saveApifyInstagramComments"; import { startInstagramProfileScraping } from "../startInstagramProfileScraping"; -vi.mock("@/lib/apify/getDataset", () => ({ getDataset: vi.fn() })); +vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn() } })); vi.mock("../saveApifyInstagramComments", () => ({ saveApifyInstagramComments: vi.fn() })); vi.mock("../startInstagramProfileScraping", () => ({ startInstagramProfileScraping: vi.fn(), })); +const mockDataset = (items: unknown[]) => + vi + .mocked(apifyClient.dataset) + .mockImplementation(() => ({ listItems: () => Promise.resolve({ items }) }) as never); + const payload = { userId: "u", createdAt: "2026-01-01T00:00:00Z", @@ -22,7 +27,7 @@ describe("handleInstagramCommentsScraper", () => { beforeEach(() => vi.clearAllMocks()); it("saves comments and enqueues fan profile scrape for distinct usernames", async () => { - vi.mocked(getDataset).mockResolvedValue([ + mockDataset([ { id: "c1", text: "hi", diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts index a9e26f448..c7d587fd8 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleInstagramProfileScraperResults } from "../handleInstagramProfileScraperResults"; -import { getDataset } from "@/lib/apify/getDataset"; +import apifyClient from "@/lib/apify/client"; import { saveApifyInstagramPosts } from "../saveApifyInstagramPosts"; import { handleInstagramProfileFollowUpRuns } from "../handleInstagramProfileFollowUpRuns"; import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; @@ -12,7 +12,13 @@ import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccoun import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails"; import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; -vi.mock("@/lib/apify/getDataset", () => ({ getDataset: vi.fn() })); +vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn() } })); + +const mockDataset = (items: unknown[]) => + vi + .mocked(apifyClient.dataset) + .mockImplementation(() => ({ listItems: () => Promise.resolve({ items }) }) as never); + vi.mock("../saveApifyInstagramPosts", () => ({ saveApifyInstagramPosts: vi.fn() })); vi.mock("../handleInstagramProfileFollowUpRuns", () => ({ handleInstagramProfileFollowUpRuns: vi.fn(), @@ -46,7 +52,7 @@ describe("handleInstagramProfileScraperResults", () => { beforeEach(() => vi.clearAllMocks()); it("returns the empty shape when the dataset has no latest posts", async () => { - vi.mocked(getDataset).mockResolvedValue([{ username: "alice" }]); + mockDataset([{ username: "alice" }]); const result = await handleInstagramProfileScraperResults(payload); @@ -56,7 +62,7 @@ describe("handleInstagramProfileScraperResults", () => { it("persists posts, links social_posts, and fires follow-up runs + email", async () => { const posts = [{ id: "p1", post_url: "u1", updated_at: "t" }] as never; - vi.mocked(getDataset).mockResolvedValue([ + mockDataset([ { latestPosts: [{ url: "u1", timestamp: "t" }], username: "alice", diff --git a/lib/apify/instagram/handleInstagramCommentsScraper.ts b/lib/apify/instagram/handleInstagramCommentsScraper.ts index 8a10dd71d..af406a435 100644 --- a/lib/apify/instagram/handleInstagramCommentsScraper.ts +++ b/lib/apify/instagram/handleInstagramCommentsScraper.ts @@ -1,4 +1,4 @@ -import { getDataset } from "@/lib/apify/getDataset"; +import apifyClient from "@/lib/apify/client"; import { saveApifyInstagramComments } from "@/lib/apify/instagram/saveApifyInstagramComments"; import { startInstagramProfileScraping } from "@/lib/apify/instagram/startInstagramProfileScraping"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; @@ -26,10 +26,8 @@ export async function handleInstagramCommentsScraper(parsed: ApifyWebhookPayload if (!datasetId) return empty; - const dataset = await getDataset(datasetId); - if (!Array.isArray(dataset)) return empty; - - const comments = dataset as ApifyInstagramComment[]; + const { items } = await apifyClient.dataset(datasetId).listItems(); + const comments = items as ApifyInstagramComment[]; const processedPostUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); const totalComments = comments.length; diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index 60015e0e3..23ac350ce 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -1,4 +1,4 @@ -import { getDataset } from "@/lib/apify/getDataset"; +import apifyClient from "@/lib/apify/client"; import { saveApifyInstagramPosts } from "@/lib/apify/instagram/saveApifyInstagramPosts"; import { handleInstagramProfileFollowUpRuns } from "@/lib/apify/instagram/handleInstagramProfileFollowUpRuns"; import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; @@ -55,7 +55,7 @@ export async function handleInstagramProfileScraperResults( if (!datasetId) return empty; - const dataset = await getDataset(datasetId); + const { items: dataset } = await apifyClient.dataset(datasetId).listItems(); const firstResult = dataset[0] as ApifyInstagramProfileResult | undefined; if (!firstResult?.latestPosts) return empty; From 612b7f957b37425b1a80bb9b0d045f275a3551ba Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 20:08:22 +0000 Subject: [PATCH 14/23] refactor(apify): prune redundancies in webhook chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulls a bunch of small redundancies out of this PR's apify chain: - Tighten the webhook payload schema with `.min(1)` on actorId and defaultDatasetId. The handlers were carrying empty-shape fallbacks (`if (!datasetId) return empty`) only because the schema permitted empty strings — now those checks are dead and removed. - Drop the comments handler's `empty` fallback object and its redundant `totalComments` field (= `comments.length`). - Drop the profile handler's `empty` fallback shape and bespoke `ProfileScraperResult` type. The only remaining short-circuit (no `latestPosts`) returns a minimal `{ posts: [], social: null }`. - Inline `saveApifyInstagramPosts` into its single caller; the upsert/getPosts pair now lives next to the rest of the persistence chain in `handleInstagramProfileScraperResults`. - Drop `saveApifyInstagramComments`'s log-and-rethrow try/catch — the outer webhook handler already logs and returns an error response. - Inline `getExistingPostComments` into `handleInstagramProfileFollowUpRuns`. The wrapper added a swallow-and-return-empty fallback that masked DB errors; with errors now propagating it's a clean two-liner. - Remove the post-`actor.start()` FAILED/ABORTED check from `startInstagramProfileScraping`. Cubic flagged this as ineffective (terminal states aren't reliably available at kickoff) and it threw an inappropriate `OUTSTANDING_ERROR`. - Drop the unused `error` and `data` fields from `ApifyRunInfo`; nothing in the codebase ever sets them. Verification: pnpm lint:check clean, pnpm test 2315/2315 pass. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- .../__tests__/apifyWebhookHandler.test.ts | 7 +- .../handleInstagramCommentsScraper.test.ts | 11 +-- ...handleInstagramProfileFollowUpRuns.test.ts | 15 ++--- ...ndleInstagramProfileScraperResults.test.ts | 14 ++-- .../instagram/getExistingPostComments.ts | 30 --------- .../handleInstagramCommentsScraper.ts | 14 +--- .../handleInstagramProfileFollowUpRuns.ts | 14 ++-- .../handleInstagramProfileScraperResults.ts | 67 ++++++------------- .../instagram/saveApifyInstagramComments.ts | 56 +++++++--------- .../instagram/saveApifyInstagramPosts.ts | 28 -------- .../startInstagramProfileScraping.ts | 5 -- lib/apify/types.ts | 2 - lib/apify/validateApifyWebhookRequest.ts | 4 +- 13 files changed, 79 insertions(+), 188 deletions(-) delete mode 100644 lib/apify/instagram/getExistingPostComments.ts delete mode 100644 lib/apify/instagram/saveApifyInstagramPosts.ts diff --git a/lib/apify/__tests__/apifyWebhookHandler.test.ts b/lib/apify/__tests__/apifyWebhookHandler.test.ts index 7623b0391..18f1ec1c8 100644 --- a/lib/apify/__tests__/apifyWebhookHandler.test.ts +++ b/lib/apify/__tests__/apifyWebhookHandler.test.ts @@ -43,14 +43,17 @@ describe("apifyWebhookHandler", () => { }); it("dispatches comments scraper for the IG comments actor", async () => { - vi.mocked(handleInstagramCommentsScraper).mockResolvedValue({ totalComments: 3 } as never); + vi.mocked(handleInstagramCommentsScraper).mockResolvedValue({ + comments: [], + processedPostUrls: ["u1", "u2"], + } as never); const res = await apifyWebhookHandler( makeRequest({ ...baseBody, eventData: { actorId: "SbK00X0JYCPblD2wp" } }), ); expect(res.status).toBe(200); - expect(await res.json()).toEqual({ totalComments: 3 }); + expect(await res.json()).toEqual({ comments: [], processedPostUrls: ["u1", "u2"] }); expect(handleInstagramCommentsScraper).toHaveBeenCalledOnce(); expect(handleInstagramProfileScraperResults).not.toHaveBeenCalled(); }); diff --git a/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts b/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts index a53acd325..78b4926e8 100644 --- a/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts @@ -61,16 +61,7 @@ describe("handleInstagramCommentsScraper", () => { expect(startInstagramProfileScraping).toHaveBeenCalledOnce(); const [handles] = vi.mocked(startInstagramProfileScraping).mock.calls[0]; expect(new Set(handles as string[])).toEqual(new Set(["alice", "bob"])); - expect(result.totalComments).toBe(3); + expect(result.comments).toHaveLength(3); expect(new Set(result.processedPostUrls)).toEqual(new Set(["u1", "u2"])); }); - - it("returns empty shape without touching handlers when datasetId is missing", async () => { - const result = await handleInstagramCommentsScraper({ - ...payload, - resource: { defaultDatasetId: "" }, - }); - expect(result.totalComments).toBe(0); - expect(saveApifyInstagramComments).not.toHaveBeenCalled(); - }); }); diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts index 4d49e6034..94f0b8575 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts @@ -1,14 +1,14 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleInstagramProfileFollowUpRuns } from "../handleInstagramProfileFollowUpRuns"; import { startInstagramCommentsScraping } from "../startInstagramCommentsScraping"; -import { getExistingPostComments } from "../getExistingPostComments"; +import { selectPostUrlsWithComments } from "@/lib/supabase/post_comments/selectPostUrlsWithComments"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; vi.mock("../startInstagramCommentsScraping", () => ({ startInstagramCommentsScraping: vi.fn(), })); -vi.mock("../getExistingPostComments", () => ({ - getExistingPostComments: vi.fn(), +vi.mock("@/lib/supabase/post_comments/selectPostUrlsWithComments", () => ({ + selectPostUrlsWithComments: vi.fn(), })); const baseProfile: ApifyInstagramProfileResult = { @@ -34,7 +34,7 @@ describe("handleInstagramProfileFollowUpRuns", () => { } as ApifyInstagramProfileResult); expect(startInstagramCommentsScraping).not.toHaveBeenCalled(); - expect(getExistingPostComments).not.toHaveBeenCalled(); + expect(selectPostUrlsWithComments).not.toHaveBeenCalled(); }); it("does not kick off comments scrape when latestPosts is empty", async () => { @@ -44,17 +44,14 @@ describe("handleInstagramProfileFollowUpRuns", () => { } as ApifyInstagramProfileResult); expect(startInstagramCommentsScraping).not.toHaveBeenCalled(); - expect(getExistingPostComments).not.toHaveBeenCalled(); + expect(selectPostUrlsWithComments).not.toHaveBeenCalled(); }); it("fans out two scrapes: resultsLimit=1 for seen urls, default for unseen", async () => { const url1 = "https://instagram.com/p/1"; const url2 = "https://instagram.com/p/2"; - vi.mocked(getExistingPostComments).mockResolvedValue({ - urlsWithComments: [url1], - urlsWithoutComments: [url2], - }); + vi.mocked(selectPostUrlsWithComments).mockResolvedValue([url1]); await handleInstagramProfileFollowUpRuns([baseProfile], { ...baseProfile, diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts index c7d587fd8..92f0345dd 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts @@ -1,7 +1,8 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleInstagramProfileScraperResults } from "../handleInstagramProfileScraperResults"; import apifyClient from "@/lib/apify/client"; -import { saveApifyInstagramPosts } from "../saveApifyInstagramPosts"; +import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; import { handleInstagramProfileFollowUpRuns } from "../handleInstagramProfileFollowUpRuns"; import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; import { insertSocials } from "@/lib/supabase/socials/insertSocials"; @@ -19,7 +20,8 @@ const mockDataset = (items: unknown[]) => .mocked(apifyClient.dataset) .mockImplementation(() => ({ listItems: () => Promise.resolve({ items }) }) as never); -vi.mock("../saveApifyInstagramPosts", () => ({ saveApifyInstagramPosts: vi.fn() })); +vi.mock("@/lib/supabase/posts/upsertPosts", () => ({ upsertPosts: vi.fn() })); +vi.mock("@/lib/supabase/posts/getPosts", () => ({ getPosts: vi.fn() })); vi.mock("../handleInstagramProfileFollowUpRuns", () => ({ handleInstagramProfileFollowUpRuns: vi.fn(), })); @@ -51,13 +53,13 @@ const payload = { describe("handleInstagramProfileScraperResults", () => { beforeEach(() => vi.clearAllMocks()); - it("returns the empty shape when the dataset has no latest posts", async () => { + it("short-circuits when the dataset has no latest posts", async () => { mockDataset([{ username: "alice" }]); const result = await handleInstagramProfileScraperResults(payload); expect(result).toMatchObject({ posts: [], social: null }); - expect(saveApifyInstagramPosts).not.toHaveBeenCalled(); + expect(upsertPosts).not.toHaveBeenCalled(); }); it("persists posts, links social_posts, and fires follow-up runs + email", async () => { @@ -71,7 +73,8 @@ describe("handleInstagramProfileScraperResults", () => { fullName: "Alice", }, ]); - vi.mocked(saveApifyInstagramPosts).mockResolvedValue({ supabasePosts: posts }); + vi.mocked(upsertPosts).mockResolvedValue({ data: null, error: null } as never); + vi.mocked(getPosts).mockResolvedValue(posts); vi.mocked(uploadLinkToArweave).mockResolvedValue(null); vi.mocked(insertSocials).mockResolvedValue([] as never); vi.mocked(selectSocials).mockResolvedValue([{ id: "s1" }] as never); @@ -82,6 +85,7 @@ describe("handleInstagramProfileScraperResults", () => { const result = await handleInstagramProfileScraperResults(payload); + expect(upsertPosts).toHaveBeenCalledOnce(); expect(upsertSocialPosts).toHaveBeenCalledOnce(); expect(sendApifyWebhookEmail).toHaveBeenCalledWith(expect.any(Object), ["x@y.com"]); expect(handleInstagramProfileFollowUpRuns).toHaveBeenCalledOnce(); diff --git a/lib/apify/instagram/getExistingPostComments.ts b/lib/apify/instagram/getExistingPostComments.ts deleted file mode 100644 index 6d5d877f9..000000000 --- a/lib/apify/instagram/getExistingPostComments.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { selectPostUrlsWithComments } from "@/lib/supabase/post_comments/selectPostUrlsWithComments"; - -/** - * Partitions `postUrls` into those that already have comments in - * Supabase and those that do not. Used by the follow-up-runs handler - * to decide whether to throttle the comments scraper for known posts. - * - * @param postUrls - Instagram post URLs to check. - */ -export async function getExistingPostComments(postUrls: string[]): Promise<{ - urlsWithComments: string[]; - urlsWithoutComments: string[]; -}> { - if (!postUrls || postUrls.length === 0) { - return { urlsWithComments: [], urlsWithoutComments: [] }; - } - - try { - const withComments = await selectPostUrlsWithComments(postUrls); - const urlsWithComments = Array.from(new Set(withComments)); - const withSet = new Set(urlsWithComments); - const urlsWithoutComments = postUrls.filter(url => !withSet.has(url)); - - return { urlsWithComments, urlsWithoutComments }; - } catch (error) { - console.error("[ERROR] getExistingPostComments:", error); - // Assume no comments exist on error so follow-up scrape still fires. - return { urlsWithComments: [], urlsWithoutComments: postUrls }; - } -} diff --git a/lib/apify/instagram/handleInstagramCommentsScraper.ts b/lib/apify/instagram/handleInstagramCommentsScraper.ts index af406a435..1c4c1b2a1 100644 --- a/lib/apify/instagram/handleInstagramCommentsScraper.ts +++ b/lib/apify/instagram/handleInstagramCommentsScraper.ts @@ -17,19 +17,9 @@ import type { ApifyInstagramComment } from "@/lib/apify/types"; * @param parsed - Validated Apify webhook payload. */ export async function handleInstagramCommentsScraper(parsed: ApifyWebhookPayload) { - const datasetId = parsed.resource.defaultDatasetId; - const empty = { - comments: [] as ApifyInstagramComment[], - processedPostUrls: [] as string[], - totalComments: 0, - }; - - if (!datasetId) return empty; - - const { items } = await apifyClient.dataset(datasetId).listItems(); + const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); const comments = items as ApifyInstagramComment[]; const processedPostUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); - const totalComments = comments.length; await saveApifyInstagramComments(comments); @@ -43,5 +33,5 @@ export async function handleInstagramCommentsScraper(parsed: ApifyWebhookPayload } } - return { comments, processedPostUrls, totalComments }; + return { comments, processedPostUrls }; } diff --git a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts index b9abea26d..ff0a13b1c 100644 --- a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts +++ b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts @@ -1,5 +1,5 @@ import { startInstagramCommentsScraping } from "@/lib/apify/instagram/startInstagramCommentsScraping"; -import { getExistingPostComments } from "@/lib/apify/instagram/getExistingPostComments"; +import { selectPostUrlsWithComments } from "@/lib/supabase/post_comments/selectPostUrlsWithComments"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; /** @@ -27,13 +27,15 @@ export async function handleInstagramProfileFollowUpRuns( if (postUrls.length === 0) return; - const { urlsWithComments, urlsWithoutComments } = await getExistingPostComments(postUrls); + const withComments = await selectPostUrlsWithComments(postUrls); + const withSet = new Set(withComments); + const withoutComments = postUrls.filter(url => !withSet.has(url)); - if (urlsWithComments.length > 0) { - await startInstagramCommentsScraping(urlsWithComments, 1); + if (withComments.length > 0) { + await startInstagramCommentsScraping(withComments, 1); } - if (urlsWithoutComments.length > 0) { - await startInstagramCommentsScraping(urlsWithoutComments); + if (withoutComments.length > 0) { + await startInstagramCommentsScraping(withoutComments); } } diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index 23ac350ce..420ff5178 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -1,14 +1,12 @@ import apifyClient from "@/lib/apify/client"; -import { saveApifyInstagramPosts } from "@/lib/apify/instagram/saveApifyInstagramPosts"; +import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; import { handleInstagramProfileFollowUpRuns } from "@/lib/apify/instagram/handleInstagramProfileFollowUpRuns"; import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; import { insertSocials } from "@/lib/supabase/socials/insertSocials"; import { selectSocials } from "@/lib/supabase/socials/selectSocials"; import { upsertSocialPosts } from "@/lib/supabase/social_posts/upsertSocialPosts"; -import { - selectAccountSocials, - type AccountSocialWithSocial, -} from "@/lib/supabase/account_socials/selectAccountSocials"; +import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccountSocials"; import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccountArtistIds"; import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails"; import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; @@ -16,16 +14,7 @@ import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; import { getFetchableUrl } from "@/lib/arweave/getFetchableUrl"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; -import type { Tables } from "@/types/database.types"; - -type ProfileScraperResult = { - posts: Tables<"posts">[]; - social: Tables<"socials"> | null; - accountSocials: AccountSocialWithSocial[]; - accountArtistIds: Awaited>; - accountEmails: Tables<"account_emails">[]; - sentEmails: Awaited>; -}; +import type { TablesInsert } from "@/types/database.types"; /** * Handles Instagram profile scraper Apify webhook results: @@ -34,32 +23,25 @@ type ProfileScraperResult = { * - Notifies subscribed account emails via Resend. * - Queues the comments scraper for the profile's latest posts. * - * Returns a summary object for logging / downstream inspection. All - * failures inside the chain are logged but allowed to propagate to the - * webhook route's outer try/catch, which always returns 200. + * Returns a summary object for downstream inspection. Failures + * propagate to the webhook route's outer try/catch, which logs and + * returns an error response (always HTTP 200 to Apify). * * @param parsed - Validated Apify webhook payload. */ -export async function handleInstagramProfileScraperResults( - parsed: ApifyWebhookPayload, -): Promise { - const datasetId = parsed.resource.defaultDatasetId; - const empty: ProfileScraperResult = { - posts: [], - social: null, - accountSocials: [], - accountArtistIds: [], - accountEmails: [], - sentEmails: null, - }; - - if (!datasetId) return empty; - - const { items: dataset } = await apifyClient.dataset(datasetId).listItems(); +export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookPayload) { + const { items: dataset } = await apifyClient + .dataset(parsed.resource.defaultDatasetId) + .listItems(); const firstResult = dataset[0] as ApifyInstagramProfileResult | undefined; - if (!firstResult?.latestPosts) return empty; + if (!firstResult?.latestPosts) return { posts: [], social: null }; - const { supabasePosts: posts } = await saveApifyInstagramPosts(firstResult.latestPosts); + const postRows: TablesInsert<"posts">[] = firstResult.latestPosts.map(post => ({ + post_url: post.url, + updated_at: post.timestamp, + })); + await upsertPosts(postRows); + const posts = await getPosts({ postUrls: postRows.map(p => p.post_url) }); const arweaveTx = await uploadLinkToArweave( firstResult.profilePicUrlHD || firstResult.profilePicUrl, @@ -87,9 +69,7 @@ export async function handleInstagramProfileScraperResults( const matches = await selectSocials({ profile_url: normalizedUrl }); const social = matches?.[0] ?? null; - if (!social) { - return { ...empty, posts }; - } + if (!social) return { posts, social: null }; if (posts.length) { await upsertSocialPosts( @@ -119,12 +99,5 @@ export async function handleInstagramProfileScraperResults( await handleInstagramProfileFollowUpRuns(dataset, firstResult); - return { - posts, - social, - accountSocials, - accountArtistIds, - accountEmails, - sentEmails, - }; + return { posts, social, accountSocials, accountEmails, sentEmails }; } diff --git a/lib/apify/instagram/saveApifyInstagramComments.ts b/lib/apify/instagram/saveApifyInstagramComments.ts index 88883ef3d..fd14a7fd5 100644 --- a/lib/apify/instagram/saveApifyInstagramComments.ts +++ b/lib/apify/instagram/saveApifyInstagramComments.ts @@ -7,46 +7,42 @@ import type { ApifyInstagramComment } from "@/lib/apify/types"; /** * Persists Apify Instagram comment dataset items into `post_comments`, * first ensuring that a matching `posts` and `socials` row exists for - * each comment. Missing references are silently skipped. + * each comment. Comments referencing missing posts/socials are skipped + * with a warning. Errors propagate up to the webhook route. * * @param comments - Apify dataset items. */ -export async function saveApifyInstagramComments(comments: ApifyInstagramComment[]): Promise { +export async function saveApifyInstagramComments(comments: ApifyInstagramComment[]) { if (comments.length === 0) return; - try { - const postUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); - const postsMap = await getOrCreatePostsForComments(postUrls); - const socialsMap = await getOrCreateSocialsForComments(comments); + const postUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); + const postsMap = await getOrCreatePostsForComments(postUrls); + const socialsMap = await getOrCreateSocialsForComments(comments); - const rows: TablesInsert<"post_comments">[] = []; + const rows: TablesInsert<"post_comments">[] = []; - for (const comment of comments) { - if (!comment.postUrl || !comment.ownerUsername || !comment.timestamp) continue; + for (const comment of comments) { + if (!comment.postUrl || !comment.ownerUsername || !comment.timestamp) continue; - const post = postsMap.get(comment.postUrl); - const social = socialsMap.get(comment.ownerUsername); + const post = postsMap.get(comment.postUrl); + const social = socialsMap.get(comment.ownerUsername); - if (!post || !social) { - console.warn( - `[WARN] saveApifyInstagramComments: missing post/social for comment ${comment.id}`, - ); - continue; - } - - rows.push({ - post_id: post.id, - social_id: social.id, - comment: comment.text, - commented_at: comment.timestamp, - }); + if (!post || !social) { + console.warn( + `[WARN] saveApifyInstagramComments: missing post/social for comment ${comment.id}`, + ); + continue; } - if (rows.length > 0) { - await upsertPostComments(rows); - } - } catch (error) { - console.error("[ERROR] saveApifyInstagramComments:", error); - throw error; + rows.push({ + post_id: post.id, + social_id: social.id, + comment: comment.text, + commented_at: comment.timestamp, + }); + } + + if (rows.length > 0) { + await upsertPostComments(rows); } } diff --git a/lib/apify/instagram/saveApifyInstagramPosts.ts b/lib/apify/instagram/saveApifyInstagramPosts.ts deleted file mode 100644 index 184b31595..000000000 --- a/lib/apify/instagram/saveApifyInstagramPosts.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; -import { getPosts } from "@/lib/supabase/posts/getPosts"; -import type { ApifyInstagramPost } from "@/lib/apify/types"; -import type { Tables, TablesInsert } from "@/types/database.types"; - -/** - * Upserts an array of Apify Instagram posts into Supabase and returns - * the resulting rows. - * - * @param apifyPosts - Posts from the Apify Instagram profile scraper. - * @returns Object with Supabase `posts` rows for the provided URLs. - */ -export async function saveApifyInstagramPosts( - apifyPosts: ApifyInstagramPost[], -): Promise<{ supabasePosts: Tables<"posts">[] }> { - if (!apifyPosts?.length) return { supabasePosts: [] }; - - const rows: TablesInsert<"posts">[] = apifyPosts.map(post => ({ - post_url: post.url, - updated_at: post.timestamp, - })); - const postUrls = rows.map(p => p.post_url); - - await upsertPosts(rows); - - const supabasePosts = await getPosts({ postUrls }); - return { supabasePosts }; -} diff --git a/lib/apify/instagram/startInstagramProfileScraping.ts b/lib/apify/instagram/startInstagramProfileScraping.ts index 34c0cdb61..263f55be2 100644 --- a/lib/apify/instagram/startInstagramProfileScraping.ts +++ b/lib/apify/instagram/startInstagramProfileScraping.ts @@ -1,5 +1,4 @@ import apifyClient from "@/lib/apify/client"; -import { OUTSTANDING_ERROR } from "@/lib/apify/errors"; import { ApifyRunInfo } from "@/lib/apify/types"; import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; @@ -30,9 +29,5 @@ export async function startInstagramProfileScraping( return null; } - if (run.status === "FAILED" || run.status === "ABORTED") { - throw new Error(OUTSTANDING_ERROR); - } - return { runId: run.id, datasetId: run.defaultDatasetId }; } diff --git a/lib/apify/types.ts b/lib/apify/types.ts index 1468c4261..3dd546a0b 100644 --- a/lib/apify/types.ts +++ b/lib/apify/types.ts @@ -1,8 +1,6 @@ export interface ApifyRunInfo { runId: string; datasetId: string; - error?: string; - data?: unknown; } export type ApifyInstagramPost = { diff --git a/lib/apify/validateApifyWebhookRequest.ts b/lib/apify/validateApifyWebhookRequest.ts index d37496aa6..2a446b595 100644 --- a/lib/apify/validateApifyWebhookRequest.ts +++ b/lib/apify/validateApifyWebhookRequest.ts @@ -10,10 +10,10 @@ import { z } from "zod"; */ export const apifyWebhookPayloadSchema = z.object({ eventData: z.object({ - actorId: z.string({ message: "eventData.actorId is required" }), + actorId: z.string().min(1, "eventData.actorId is required"), }), resource: z.object({ - defaultDatasetId: z.string({ message: "resource.defaultDatasetId is required" }), + defaultDatasetId: z.string().min(1, "resource.defaultDatasetId is required"), }), }); From 6b7294d75e640214580da34ce5196e491e05d211 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 20:14:57 +0000 Subject: [PATCH 15/23] refactor(apify): simplify instagram helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - getOrCreateSocialsForComments: collapse the Map.values() dedup + filter + map chain into a single for-loop. Drop the bio/region/ followerCount/followingCount null fields — stripNullish removes them anyway. ~15 lines shorter, same behavior. - getOrCreatePostsForComments: drop the throwaway `existingSet` step (use a Set comprehension on `existing` directly), pull `now` out of the map for clarity. - handleInstagramProfileFollowUpRuns: drop the `(post && typeof post.url === "string" ? post.url : "")` ternary since latestPosts is typed `ApifyInstagramPost[]`. Just `.map(p => p.url)`. - saveApifyInstagramComments: drop the outer postUrls dedup — getOrCreatePostsForComments already dedupes its input. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- .../instagram/getOrCreatePostsForComments.ts | 19 +++------ .../getOrCreateSocialsForComments.ts | 41 ++++++------------- .../handleInstagramProfileFollowUpRuns.ts | 5 +-- .../instagram/saveApifyInstagramComments.ts | 3 +- 4 files changed, 19 insertions(+), 49 deletions(-) diff --git a/lib/apify/instagram/getOrCreatePostsForComments.ts b/lib/apify/instagram/getOrCreatePostsForComments.ts index 001286e61..e13dd4e3a 100644 --- a/lib/apify/instagram/getOrCreatePostsForComments.ts +++ b/lib/apify/instagram/getOrCreatePostsForComments.ts @@ -1,6 +1,6 @@ import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; import { getPosts } from "@/lib/supabase/posts/getPosts"; -import type { Tables, TablesInsert } from "@/types/database.types"; +import type { Tables } from "@/types/database.types"; /** * Ensures a `posts` row exists for every post URL the comment scraper @@ -16,23 +16,14 @@ export async function getOrCreatePostsForComments( if (unique.length === 0) return new Map(); const existing = await getPosts({ postUrls: unique }); - const existingSet = new Set(existing.map(p => p.post_url)); - - const missing = unique.filter(url => !existingSet.has(url)); + const missing = unique.filter(url => !existing.some(p => p.post_url === url)); let all = existing; if (missing.length > 0) { - const rows: TablesInsert<"posts">[] = missing.map(url => ({ - post_url: url, - updated_at: new Date().toISOString(), - })); - await upsertPosts(rows); + const now = new Date().toISOString(); + await upsertPosts(missing.map(url => ({ post_url: url, updated_at: now }))); all = await getPosts({ postUrls: unique }); } - const map = new Map>(); - all.forEach(post => { - map.set(post.post_url, post); - }); - return map; + return new Map(all.map(post => [post.post_url, post])); } diff --git a/lib/apify/instagram/getOrCreateSocialsForComments.ts b/lib/apify/instagram/getOrCreateSocialsForComments.ts index 2ce9c571a..962882072 100644 --- a/lib/apify/instagram/getOrCreateSocialsForComments.ts +++ b/lib/apify/instagram/getOrCreateSocialsForComments.ts @@ -13,35 +13,18 @@ import type { ApifyInstagramComment } from "@/lib/apify/types"; export async function getOrCreateSocialsForComments( comments: ApifyInstagramComment[], ): Promise>> { - const uniqueAuthors = Array.from( - new Map( - comments.map(c => [ - c.ownerUsername, - { - username: c.ownerUsername, - profilePicUrl: c.ownerProfilePicUrl, - profileUrl: `instagram.com/${c.ownerUsername}`, - }, - ]), - ).values(), - ); - - const rows: TablesInsert<"socials">[] = uniqueAuthors - .filter(a => a.username && a.profileUrl) - .map(a => ({ - username: a.username, - profile_url: a.profileUrl, - avatar: a.profilePicUrl, - bio: null, - region: null, - followerCount: null, - followingCount: null, - })); + const seen = new Set(); + const rows: TablesInsert<"socials">[] = []; + for (const c of comments) { + if (!c.ownerUsername || seen.has(c.ownerUsername)) continue; + seen.add(c.ownerUsername); + rows.push({ + username: c.ownerUsername, + profile_url: `instagram.com/${c.ownerUsername}`, + avatar: c.ownerProfilePicUrl, + }); + } const upserted = await insertSocials(rows); - const map = new Map>(); - upserted.forEach(social => { - map.set(social.username, social); - }); - return map; + return new Map(upserted.map(s => [s.username, s])); } diff --git a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts index ff0a13b1c..9b79d35ba 100644 --- a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts +++ b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts @@ -21,10 +21,7 @@ export async function handleInstagramProfileFollowUpRuns( if (dataset.length !== 1) return; if (!firstResult.latestPosts || firstResult.latestPosts.length === 0) return; - const postUrls = firstResult.latestPosts - .map(post => (post && typeof post.url === "string" ? post.url : "")) - .filter(Boolean); - + const postUrls = firstResult.latestPosts.map(p => p.url).filter(Boolean); if (postUrls.length === 0) return; const withComments = await selectPostUrlsWithComments(postUrls); diff --git a/lib/apify/instagram/saveApifyInstagramComments.ts b/lib/apify/instagram/saveApifyInstagramComments.ts index fd14a7fd5..edb8a0245 100644 --- a/lib/apify/instagram/saveApifyInstagramComments.ts +++ b/lib/apify/instagram/saveApifyInstagramComments.ts @@ -15,8 +15,7 @@ import type { ApifyInstagramComment } from "@/lib/apify/types"; export async function saveApifyInstagramComments(comments: ApifyInstagramComment[]) { if (comments.length === 0) return; - const postUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); - const postsMap = await getOrCreatePostsForComments(postUrls); + const postsMap = await getOrCreatePostsForComments(comments.map(c => c.postUrl)); const socialsMap = await getOrCreateSocialsForComments(comments); const rows: TablesInsert<"post_comments">[] = []; From 175370b938a12c95dab9d8baf59370ced6404197 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 20:20:20 +0000 Subject: [PATCH 16/23] fix(security): address SSRF + review feedback on apify chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Triage of unresolved review comments on PR #483: - isSafeHttpUrl (cubic P0): URL.hostname brackets IPv6 literals, so the previous `host === "::1"` and `host.startsWith("fc")` checks never fired. Strip the brackets first. Also tighten the deny-list per coderabbit defense-in-depth: CGNAT (100.64/10), multicast + reserved (224/4 + 240/4), broadcast (255.255.255.255), and all IPv4-mapped IPv6 (`::ffff:*`). Add 12 unit tests. - uploadLinkToArweave: bound the fetch with AbortSignal.timeout(10s) so a slow/hung server can't stall the webhook. - stripNullish: move from `lib/utils/` (project bans the generic utils folder) to `lib/objects/`. Also tighten the return type to `Partial` since the function can drop keys. - startInstagramProfileScraping: dedupe normalized handles before calling the actor — `["@foo", "foo"]` previously scheduled two scrapes for the same profile. - apifyWebhookHandler: stop returning raw exception messages in the response body. Log the detail server-side and return a generic "Internal server error" string per team policy. Verification: pnpm lint:check clean, pnpm test 2327/2327 pass. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- .../__tests__/apifyWebhookHandler.test.ts | 2 +- lib/apify/apifyWebhookHandler.ts | 8 +-- .../startInstagramProfileScraping.ts | 4 +- lib/arweave/__tests__/isSafeHttpUrl.test.ts | 71 +++++++++++++++++++ lib/arweave/isSafeHttpUrl.ts | 43 +++++++---- lib/arweave/uploadLinkToArweave.ts | 11 ++- lib/{utils => objects}/stripNullish.ts | 8 +-- lib/supabase/socials/insertSocials.ts | 2 +- 8 files changed, 120 insertions(+), 29 deletions(-) create mode 100644 lib/arweave/__tests__/isSafeHttpUrl.test.ts rename lib/{utils => objects}/stripNullish.ts (71%) diff --git a/lib/apify/__tests__/apifyWebhookHandler.test.ts b/lib/apify/__tests__/apifyWebhookHandler.test.ts index 18f1ec1c8..b2791bcbb 100644 --- a/lib/apify/__tests__/apifyWebhookHandler.test.ts +++ b/lib/apify/__tests__/apifyWebhookHandler.test.ts @@ -80,7 +80,7 @@ describe("apifyWebhookHandler", () => { expect(res.status).toBe(200); const body = await res.json(); - expect(body).toEqual({ status: "error", error: "boom" }); + expect(body).toEqual({ status: "error", error: "Internal server error" }); }); it("propagates the validator's error response for invalid payloads", async () => { diff --git a/lib/apify/apifyWebhookHandler.ts b/lib/apify/apifyWebhookHandler.ts index 544c6f0e5..64c863c17 100644 --- a/lib/apify/apifyWebhookHandler.ts +++ b/lib/apify/apifyWebhookHandler.ts @@ -39,12 +39,6 @@ export async function apifyWebhookHandler(request: NextRequest): Promise { const list = Array.isArray(handles) ? handles : [handles]; - const cleanHandles = list.map(h => h.trim().replace(/^@/, "")).filter(h => h.length > 0); + const cleanHandles = Array.from( + new Set(list.map(h => h.trim().replace(/^@/, "")).filter(h => h.length > 0)), + ); if (cleanHandles.length === 0) { throw new Error("Invalid Instagram handle"); diff --git a/lib/arweave/__tests__/isSafeHttpUrl.test.ts b/lib/arweave/__tests__/isSafeHttpUrl.test.ts new file mode 100644 index 000000000..b905ce38f --- /dev/null +++ b/lib/arweave/__tests__/isSafeHttpUrl.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect } from "vitest"; +import { isSafeHttpUrl } from "../isSafeHttpUrl"; + +describe("isSafeHttpUrl", () => { + it("accepts public https URLs", () => { + expect(isSafeHttpUrl("https://example.com/avatar.png")).toBe(true); + expect(isSafeHttpUrl("http://cdn.instagram.com/img.jpg")).toBe(true); + }); + + it("rejects malformed URLs", () => { + expect(isSafeHttpUrl("not a url")).toBe(false); + expect(isSafeHttpUrl("")).toBe(false); + }); + + it("rejects non-http(s) protocols", () => { + expect(isSafeHttpUrl("file:///etc/passwd")).toBe(false); + expect(isSafeHttpUrl("ftp://example.com")).toBe(false); + expect(isSafeHttpUrl("javascript:alert(1)")).toBe(false); + }); + + it("rejects loopback hostnames", () => { + expect(isSafeHttpUrl("http://localhost")).toBe(false); + expect(isSafeHttpUrl("http://foo.localhost")).toBe(false); + expect(isSafeHttpUrl("http://127.0.0.1")).toBe(false); + expect(isSafeHttpUrl("http://127.5.6.7")).toBe(false); + }); + + it("rejects RFC1918 private IPv4", () => { + expect(isSafeHttpUrl("http://10.0.0.1")).toBe(false); + expect(isSafeHttpUrl("http://192.168.1.1")).toBe(false); + expect(isSafeHttpUrl("http://172.16.0.1")).toBe(false); + expect(isSafeHttpUrl("http://172.31.255.255")).toBe(false); + }); + + it("accepts public IPv4 outside 172.16/12", () => { + expect(isSafeHttpUrl("http://172.15.0.1")).toBe(true); + expect(isSafeHttpUrl("http://172.32.0.1")).toBe(true); + }); + + it("rejects link-local IPv4 (169.254/16)", () => { + expect(isSafeHttpUrl("http://169.254.169.254")).toBe(false); + }); + + it("rejects CGNAT range (100.64/10)", () => { + expect(isSafeHttpUrl("http://100.64.0.1")).toBe(false); + expect(isSafeHttpUrl("http://100.127.255.255")).toBe(false); + }); + + it("rejects multicast/reserved/broadcast IPv4", () => { + expect(isSafeHttpUrl("http://224.0.0.1")).toBe(false); + expect(isSafeHttpUrl("http://239.255.255.250")).toBe(false); + expect(isSafeHttpUrl("http://240.0.0.1")).toBe(false); + expect(isSafeHttpUrl("http://255.255.255.255")).toBe(false); + }); + + it("rejects IPv6 loopback (with brackets per WHATWG URL)", () => { + expect(isSafeHttpUrl("http://[::1]")).toBe(false); + expect(isSafeHttpUrl("http://[::]")).toBe(false); + }); + + it("rejects unique-local + link-local IPv6", () => { + expect(isSafeHttpUrl("http://[fc00::1]")).toBe(false); + expect(isSafeHttpUrl("http://[fd12::1]")).toBe(false); + expect(isSafeHttpUrl("http://[fe80::1]")).toBe(false); + }); + + it("rejects IPv4-mapped IPv6 of private addresses", () => { + expect(isSafeHttpUrl("http://[::ffff:10.0.0.1]")).toBe(false); + expect(isSafeHttpUrl("http://[::ffff:127.0.0.1]")).toBe(false); + }); +}); diff --git a/lib/arweave/isSafeHttpUrl.ts b/lib/arweave/isSafeHttpUrl.ts index fabbc1871..dbdee197e 100644 --- a/lib/arweave/isSafeHttpUrl.ts +++ b/lib/arweave/isSafeHttpUrl.ts @@ -1,7 +1,12 @@ /** * Returns true if `raw` parses as an absolute http(s) URL whose host is - * not loopback, link-local, or RFC1918 private. Used to gate outbound - * fetches against SSRF via untrusted user-supplied URLs. + * not loopback, link-local, RFC1918 private, CGNAT, multicast, broadcast, + * or an IPv4-mapped IPv6 form. Used to gate outbound fetches against + * SSRF via untrusted user-supplied URLs. + * + * NOTE: this is a parse-time check. It does not protect against DNS + * rebinding — if you accept fully untrusted hostnames, also resolve + * the host and re-check the resolved IP at fetch time. */ export function isSafeHttpUrl(raw: string): boolean { let parsed: URL; @@ -11,17 +16,31 @@ export function isSafeHttpUrl(raw: string): boolean { return false; } if (parsed.protocol !== "https:" && parsed.protocol !== "http:") return false; - const host = parsed.hostname.toLowerCase(); + + // URL.hostname brackets IPv6 literals (`[::1]`) — strip them so the + // textual checks below can match the raw IPv6 form. + let host = parsed.hostname.toLowerCase(); + if (host.startsWith("[") && host.endsWith("]")) { + host = host.slice(1, -1); + } if (!host) return false; + if (host === "localhost" || host.endsWith(".localhost")) return false; - if ( - /^(10\.|127\.|0\.|169\.254\.|192\.168\.)/.test(host) || - /^172\.(1[6-9]|2\d|3[01])\./.test(host) || - host === "::1" || - host.startsWith("fc") || - host.startsWith("fd") - ) { - return false; - } + + // IPv4 private / loopback / link-local / CGNAT / multicast / reserved / broadcast. + if (/^(10\.|127\.|0\.|169\.254\.|192\.168\.)/.test(host)) return false; + if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; + if (/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./.test(host)) return false; // 100.64.0.0/10 + if (/^(22[4-9]|23\d|24\d|25[0-5])\./.test(host)) return false; // 224.0.0.0/4 + 240.0.0.0/4 + if (host === "255.255.255.255") return false; + + // IPv6 loopback / unique-local / link-local. Also block all + // IPv4-mapped IPv6 (`::ffff:*`) — it's almost always a bypass attempt + // and a legitimate caller would just use the IPv4 form directly. + if (host === "::1" || host === "::") return false; + if (/^f[cd][0-9a-f]{2}:/.test(host)) return false; // fc00::/7 + if (/^fe[89ab][0-9a-f]:/.test(host)) return false; // fe80::/10 + if (host.startsWith("::ffff:")) return false; + return true; } diff --git a/lib/arweave/uploadLinkToArweave.ts b/lib/arweave/uploadLinkToArweave.ts index 9674f4e02..12f8a0c92 100644 --- a/lib/arweave/uploadLinkToArweave.ts +++ b/lib/arweave/uploadLinkToArweave.ts @@ -2,6 +2,7 @@ import { uploadToArweave } from "./uploadToArweave"; import { isSafeHttpUrl } from "./isSafeHttpUrl"; const MAX_IMAGE_BYTES = 10 * 1024 * 1024; +const FETCH_TIMEOUT_MS = 10_000; /** * Fetches the image at `imageUrl` and uploads its bytes to Arweave. @@ -10,8 +11,9 @@ const MAX_IMAGE_BYTES = 10 * 1024 * 1024; * back to the original URL in that case. * * Rejects non-http(s) URLs and private/loopback hosts to avoid SSRF, - * and caps the download at MAX_IMAGE_BYTES so a malicious server - * cannot exhaust memory. + * caps the download at MAX_IMAGE_BYTES so a malicious server cannot + * exhaust memory, and bounds the fetch with a timeout so a slow server + * cannot stall the webhook. * * @param imageUrl - Remote image URL. */ @@ -24,7 +26,10 @@ export async function uploadLinkToArweave( return null; } try { - const res = await fetch(imageUrl, { redirect: "error" }); + const res = await fetch(imageUrl, { + redirect: "error", + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); if (!res.ok || !res.body) return null; const contentLength = Number(res.headers.get("content-length") ?? 0); diff --git a/lib/utils/stripNullish.ts b/lib/objects/stripNullish.ts similarity index 71% rename from lib/utils/stripNullish.ts rename to lib/objects/stripNullish.ts index cf8e7c656..676fcdbe6 100644 --- a/lib/utils/stripNullish.ts +++ b/lib/objects/stripNullish.ts @@ -4,10 +4,10 @@ * non-null column values just because they have nothing new to say * about those fields. */ -export function stripNullish>(row: T): T { - const out: Record = {}; - for (const [k, v] of Object.entries(row)) { +export function stripNullish>(row: T): Partial { + const out: Partial = {}; + for (const [k, v] of Object.entries(row) as [keyof T, T[keyof T]][]) { if (v !== null && v !== undefined) out[k] = v; } - return out as T; + return out; } diff --git a/lib/supabase/socials/insertSocials.ts b/lib/supabase/socials/insertSocials.ts index fcfbc56f9..fe81373df 100644 --- a/lib/supabase/socials/insertSocials.ts +++ b/lib/supabase/socials/insertSocials.ts @@ -1,5 +1,5 @@ import supabase from "../serverClient"; -import { stripNullish } from "@/lib/utils/stripNullish"; +import { stripNullish } from "@/lib/objects/stripNullish"; import type { TablesInsert } from "@/types/database.types"; /** From 93c43683167558e5ccef7f49ea7c6e5807e4da93 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 20:21:04 +0000 Subject: [PATCH 17/23] fix(apify): isolate email + follow-up scrape side effects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If sendApifyWebhookEmail throws (mail outage, rate limit, bad recipient list), the follow-up comments scrape currently never runs — one transient mail error skips the comments fan-out for the whole profile. Wrap each side effect in its own try/catch so they fail independently. Per coderabbit review on PR #483. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- .../handleInstagramProfileScraperResults.ts | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index 420ff5178..dcd6992ad 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -92,12 +92,23 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP const accountEmails = await selectAccountEmails({ accountIds: uniqueAccountIds }); - const sentEmails = await sendApifyWebhookEmail( - firstResult, - accountEmails.map(e => e.email).filter(Boolean), - ); + // Email + follow-up scrape are independent side effects; isolate so a + // mail outage doesn't block comment scraping and vice versa. + let sentEmails = null; + try { + sentEmails = await sendApifyWebhookEmail( + firstResult, + accountEmails.map(e => e.email).filter(Boolean), + ); + } catch (error) { + console.error("[WARN] webhook email failed:", error); + } - await handleInstagramProfileFollowUpRuns(dataset, firstResult); + try { + await handleInstagramProfileFollowUpRuns(dataset, firstResult); + } catch (error) { + console.error("[WARN] follow-up scrape failed:", error); + } return { posts, social, accountSocials, accountEmails, sentEmails }; } From 8a06631d7af32798a2c61a76a9a5d8620edab7dc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 20:35:24 +0000 Subject: [PATCH 18/23] refactor(apify): inline saveApifyInstagramComments into its single caller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The helper was used only by handleInstagramCommentsScraper. Inlining makes the scraper handler read as a single coherent flow: fetch dataset → persist comments → fan-out profile scrapes. Tests now mock the underlying primitives (upsertPostComments, getOrCreatePostsForComments, getOrCreateSocialsForComments) directly and assert the rows we pass to the upsert. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- .../handleInstagramCommentsScraper.test.ts | 40 ++++++++++++++-- .../handleInstagramCommentsScraper.ts | 32 +++++++++++-- .../instagram/saveApifyInstagramComments.ts | 47 ------------------- 3 files changed, 65 insertions(+), 54 deletions(-) delete mode 100644 lib/apify/instagram/saveApifyInstagramComments.ts diff --git a/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts b/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts index 78b4926e8..c04bd8251 100644 --- a/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramCommentsScraper.test.ts @@ -1,11 +1,17 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleInstagramCommentsScraper } from "../handleInstagramCommentsScraper"; import apifyClient from "@/lib/apify/client"; -import { saveApifyInstagramComments } from "../saveApifyInstagramComments"; +import { upsertPostComments } from "@/lib/supabase/post_comments/upsertPostComments"; +import { getOrCreatePostsForComments } from "../getOrCreatePostsForComments"; +import { getOrCreateSocialsForComments } from "../getOrCreateSocialsForComments"; import { startInstagramProfileScraping } from "../startInstagramProfileScraping"; vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn() } })); -vi.mock("../saveApifyInstagramComments", () => ({ saveApifyInstagramComments: vi.fn() })); +vi.mock("@/lib/supabase/post_comments/upsertPostComments", () => ({ + upsertPostComments: vi.fn(), +})); +vi.mock("../getOrCreatePostsForComments", () => ({ getOrCreatePostsForComments: vi.fn() })); +vi.mock("../getOrCreateSocialsForComments", () => ({ getOrCreateSocialsForComments: vi.fn() })); vi.mock("../startInstagramProfileScraping", () => ({ startInstagramProfileScraping: vi.fn(), })); @@ -53,15 +59,43 @@ describe("handleInstagramCommentsScraper", () => { postUrl: "u2", }, ]); + vi.mocked(getOrCreatePostsForComments).mockResolvedValue( + new Map([ + ["u1", { id: "p1", post_url: "u1" }], + ["u2", { id: "p2", post_url: "u2" }], + ]) as never, + ); + vi.mocked(getOrCreateSocialsForComments).mockResolvedValue( + new Map([ + ["alice", { id: "s1", username: "alice" }], + ["bob", { id: "s2", username: "bob" }], + ]) as never, + ); vi.mocked(startInstagramProfileScraping).mockResolvedValue({ runId: "r", datasetId: "d" }); const result = await handleInstagramCommentsScraper(payload); - expect(saveApifyInstagramComments).toHaveBeenCalledOnce(); + expect(upsertPostComments).toHaveBeenCalledOnce(); + const [rows] = vi.mocked(upsertPostComments).mock.calls[0]; + expect(rows).toHaveLength(3); + expect(rows[0]).toMatchObject({ post_id: "p1", social_id: "s1", comment: "hi" }); + expect(startInstagramProfileScraping).toHaveBeenCalledOnce(); const [handles] = vi.mocked(startInstagramProfileScraping).mock.calls[0]; expect(new Set(handles as string[])).toEqual(new Set(["alice", "bob"])); expect(result.comments).toHaveLength(3); expect(new Set(result.processedPostUrls)).toEqual(new Set(["u1", "u2"])); }); + + it("skips persistence when the dataset is empty", async () => { + mockDataset([]); + + const result = await handleInstagramCommentsScraper(payload); + + expect(getOrCreatePostsForComments).not.toHaveBeenCalled(); + expect(getOrCreateSocialsForComments).not.toHaveBeenCalled(); + expect(upsertPostComments).not.toHaveBeenCalled(); + expect(startInstagramProfileScraping).not.toHaveBeenCalled(); + expect(result.comments).toEqual([]); + }); }); diff --git a/lib/apify/instagram/handleInstagramCommentsScraper.ts b/lib/apify/instagram/handleInstagramCommentsScraper.ts index 1c4c1b2a1..dd866d0a7 100644 --- a/lib/apify/instagram/handleInstagramCommentsScraper.ts +++ b/lib/apify/instagram/handleInstagramCommentsScraper.ts @@ -1,12 +1,16 @@ import apifyClient from "@/lib/apify/client"; -import { saveApifyInstagramComments } from "@/lib/apify/instagram/saveApifyInstagramComments"; +import { upsertPostComments } from "@/lib/supabase/post_comments/upsertPostComments"; +import { getOrCreatePostsForComments } from "@/lib/apify/instagram/getOrCreatePostsForComments"; +import { getOrCreateSocialsForComments } from "@/lib/apify/instagram/getOrCreateSocialsForComments"; import { startInstagramProfileScraping } from "@/lib/apify/instagram/startInstagramProfileScraping"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; import type { ApifyInstagramComment } from "@/lib/apify/types"; +import type { TablesInsert } from "@/types/database.types"; /** * Handles Instagram comments scraper Apify webhook results: - * - Persists comments into `post_comments`. + * - Persists comments into `post_comments` (auto-creating any missing + * `posts` and commenter `socials` rows first). * - Kicks off a fan-profile scrape for the distinct commenter * usernames so their socials get indexed (best-effort: failures * are logged but don't fail the comments ingestion). @@ -21,10 +25,30 @@ export async function handleInstagramCommentsScraper(parsed: ApifyWebhookPayload const comments = items as ApifyInstagramComment[]; const processedPostUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); - await saveApifyInstagramComments(comments); + if (comments.length > 0) { + const postsMap = await getOrCreatePostsForComments(comments.map(c => c.postUrl)); + const socialsMap = await getOrCreateSocialsForComments(comments); - const fanHandles = Array.from(new Set(comments.map(c => c.ownerUsername).filter(Boolean))); + const rows: TablesInsert<"post_comments">[] = []; + for (const c of comments) { + if (!c.postUrl || !c.ownerUsername || !c.timestamp) continue; + const post = postsMap.get(c.postUrl); + const social = socialsMap.get(c.ownerUsername); + if (!post || !social) { + console.warn(`[WARN] missing post/social for comment ${c.id}`); + continue; + } + rows.push({ + post_id: post.id, + social_id: social.id, + comment: c.text, + commented_at: c.timestamp, + }); + } + if (rows.length > 0) await upsertPostComments(rows); + } + const fanHandles = Array.from(new Set(comments.map(c => c.ownerUsername).filter(Boolean))); if (fanHandles.length > 0) { try { await startInstagramProfileScraping(fanHandles); diff --git a/lib/apify/instagram/saveApifyInstagramComments.ts b/lib/apify/instagram/saveApifyInstagramComments.ts deleted file mode 100644 index edb8a0245..000000000 --- a/lib/apify/instagram/saveApifyInstagramComments.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { upsertPostComments } from "@/lib/supabase/post_comments/upsertPostComments"; -import { getOrCreatePostsForComments } from "@/lib/apify/instagram/getOrCreatePostsForComments"; -import { getOrCreateSocialsForComments } from "@/lib/apify/instagram/getOrCreateSocialsForComments"; -import type { TablesInsert } from "@/types/database.types"; -import type { ApifyInstagramComment } from "@/lib/apify/types"; - -/** - * Persists Apify Instagram comment dataset items into `post_comments`, - * first ensuring that a matching `posts` and `socials` row exists for - * each comment. Comments referencing missing posts/socials are skipped - * with a warning. Errors propagate up to the webhook route. - * - * @param comments - Apify dataset items. - */ -export async function saveApifyInstagramComments(comments: ApifyInstagramComment[]) { - if (comments.length === 0) return; - - const postsMap = await getOrCreatePostsForComments(comments.map(c => c.postUrl)); - const socialsMap = await getOrCreateSocialsForComments(comments); - - const rows: TablesInsert<"post_comments">[] = []; - - for (const comment of comments) { - if (!comment.postUrl || !comment.ownerUsername || !comment.timestamp) continue; - - const post = postsMap.get(comment.postUrl); - const social = socialsMap.get(comment.ownerUsername); - - if (!post || !social) { - console.warn( - `[WARN] saveApifyInstagramComments: missing post/social for comment ${comment.id}`, - ); - continue; - } - - rows.push({ - post_id: post.id, - social_id: social.id, - comment: comment.text, - commented_at: comment.timestamp, - }); - } - - if (rows.length > 0) { - await upsertPostComments(rows); - } -} From 20a7646bfe9f60d426ca4170b671d3de9381773b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Apr 2026 20:46:01 +0000 Subject: [PATCH 19/23] fix(apify): resolve TS errors blocking Vercel build Two real type errors that lint/test/format CI doesn't run but the Next.js production build does: - handleInstagramCommentsScraper: `apifyClient.dataset().listItems()` returns items typed `Record[]` (not the previous `unknown[]` from getDataset). The direct cast to `ApifyInstagramComment[]` no longer has sufficient overlap; route through `unknown` per TS strict rules. - insertSocials: stripNullish now returns `Partial` per cubic's type-soundness fix, but the supabase `.upsert()` overload still expects the full `TablesInsert<"socials">[]`. Cast the cleaned rows back at the call site (missing optional fields are fine at runtime, the SDK just lets the DB defaults apply). Verification: pnpm lint:check clean, tsc --noEmit clean for PR scope, 38 apify tests + 2328 full-suite pass. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- lib/apify/instagram/handleInstagramCommentsScraper.ts | 2 +- lib/supabase/socials/insertSocials.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/apify/instagram/handleInstagramCommentsScraper.ts b/lib/apify/instagram/handleInstagramCommentsScraper.ts index dd866d0a7..16ceb4e98 100644 --- a/lib/apify/instagram/handleInstagramCommentsScraper.ts +++ b/lib/apify/instagram/handleInstagramCommentsScraper.ts @@ -22,7 +22,7 @@ import type { TablesInsert } from "@/types/database.types"; */ export async function handleInstagramCommentsScraper(parsed: ApifyWebhookPayload) { const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); - const comments = items as ApifyInstagramComment[]; + const comments = items as unknown as ApifyInstagramComment[]; const processedPostUrls = Array.from(new Set(comments.map(c => c.postUrl).filter(Boolean))); if (comments.length > 0) { diff --git a/lib/supabase/socials/insertSocials.ts b/lib/supabase/socials/insertSocials.ts index fe81373df..46b11a26c 100644 --- a/lib/supabase/socials/insertSocials.ts +++ b/lib/supabase/socials/insertSocials.ts @@ -11,9 +11,13 @@ import type { TablesInsert } from "@/types/database.types"; * @returns Array of inserted social records */ export async function insertSocials(socials: TablesInsert<"socials">[]) { + // stripNullish drops keys, so the inferred type widens to Partial<…>; + // cast back since the supabase upsert overload still expects the full + // insert type even though missing optional fields are fine at runtime. + const cleaned = socials.map(stripNullish) as TablesInsert<"socials">[]; const { data, error } = await supabase .from("socials") - .upsert(socials.map(stripNullish), { onConflict: "profile_url" }) + .upsert(cleaned, { onConflict: "profile_url" }) .select("*"); if (error) { From a23ae1b9ee113880c4123a788803eb0ab512bb40 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 13:56:44 +0000 Subject: [PATCH 20/23] refactor(apify): inline cross-table orchestrator + rename insertSocials -> upsertSocials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two threads from sweetman: - selectPostUrlsWithComments lived under lib/supabase/post_comments/ but didn't make any direct supabase calls — it just chained getPosts + selectPostComments. Inline the orchestration into handleInstagramProfileFollowUpRuns (its single caller) and delete the misplaced file. Picks up a small win along the way: when no posts row exists yet, skip the post_comments lookup and just backfill all URLs. - insertSocials switched from .insert() to .upsert() in this PR but kept its original name. Rename file + symbol + all callers (lib/artist, lib/apify/instagram, supabase tests). Verification: pnpm lint:check clean, tsc --noEmit clean for PR scope, 2329/2329 tests pass. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- .../getOrCreateSocialsForComments.test.ts | 12 +++---- ...handleInstagramProfileFollowUpRuns.test.ts | 32 +++++++++++++++---- ...ndleInstagramProfileScraperResults.test.ts | 6 ++-- .../getOrCreateSocialsForComments.ts | 6 ++-- .../handleInstagramProfileFollowUpRuns.ts | 22 +++++++------ .../handleInstagramProfileScraperResults.ts | 4 +-- .../__tests__/updateArtistSocials.test.ts | 6 ++-- lib/artist/updateArtistSocials.ts | 4 +-- .../selectPostUrlsWithComments.ts | 23 ------------- .../{insertSocials.ts => upsertSocials.ts} | 4 +-- 10 files changed, 60 insertions(+), 59 deletions(-) delete mode 100644 lib/supabase/post_comments/selectPostUrlsWithComments.ts rename lib/supabase/socials/{insertSocials.ts => upsertSocials.ts} (88%) diff --git a/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts b/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts index 753030c7b..844eec564 100644 --- a/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts +++ b/lib/apify/instagram/__tests__/getOrCreateSocialsForComments.test.ts @@ -1,14 +1,14 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { getOrCreateSocialsForComments } from "../getOrCreateSocialsForComments"; -import { insertSocials } from "@/lib/supabase/socials/insertSocials"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; -vi.mock("@/lib/supabase/socials/insertSocials", () => ({ insertSocials: vi.fn() })); +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ upsertSocials: vi.fn() })); describe("getOrCreateSocialsForComments", () => { beforeEach(() => vi.clearAllMocks()); it("upserts one row per distinct commenter and keys result by username", async () => { - vi.mocked(insertSocials).mockResolvedValue([ + vi.mocked(upsertSocials).mockResolvedValue([ { id: "s1", username: "alice", profile_url: "instagram.com/alice" }, { id: "s2", username: "bob", profile_url: "instagram.com/bob" }, ] as never); @@ -40,15 +40,15 @@ describe("getOrCreateSocialsForComments", () => { }, ]); - expect(insertSocials).toHaveBeenCalledOnce(); - const [rows] = vi.mocked(insertSocials).mock.calls[0]; + expect(upsertSocials).toHaveBeenCalledOnce(); + const [rows] = vi.mocked(upsertSocials).mock.calls[0]; expect(rows).toHaveLength(2); expect(result.get("alice")?.id).toBe("s1"); expect(result.get("bob")?.id).toBe("s2"); }); it("propagates DB errors so the webhook handler can log + short-circuit", async () => { - vi.mocked(insertSocials).mockRejectedValue(new Error("boom")); + vi.mocked(upsertSocials).mockRejectedValue(new Error("boom")); await expect( getOrCreateSocialsForComments([ diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts index 94f0b8575..b5f6c495a 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts @@ -1,14 +1,16 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleInstagramProfileFollowUpRuns } from "../handleInstagramProfileFollowUpRuns"; import { startInstagramCommentsScraping } from "../startInstagramCommentsScraping"; -import { selectPostUrlsWithComments } from "@/lib/supabase/post_comments/selectPostUrlsWithComments"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; +import { selectPostComments } from "@/lib/supabase/post_comments/selectPostComments"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; vi.mock("../startInstagramCommentsScraping", () => ({ startInstagramCommentsScraping: vi.fn(), })); -vi.mock("@/lib/supabase/post_comments/selectPostUrlsWithComments", () => ({ - selectPostUrlsWithComments: vi.fn(), +vi.mock("@/lib/supabase/posts/getPosts", () => ({ getPosts: vi.fn() })); +vi.mock("@/lib/supabase/post_comments/selectPostComments", () => ({ + selectPostComments: vi.fn(), })); const baseProfile: ApifyInstagramProfileResult = { @@ -34,7 +36,7 @@ describe("handleInstagramProfileFollowUpRuns", () => { } as ApifyInstagramProfileResult); expect(startInstagramCommentsScraping).not.toHaveBeenCalled(); - expect(selectPostUrlsWithComments).not.toHaveBeenCalled(); + expect(getPosts).not.toHaveBeenCalled(); }); it("does not kick off comments scrape when latestPosts is empty", async () => { @@ -44,14 +46,18 @@ describe("handleInstagramProfileFollowUpRuns", () => { } as ApifyInstagramProfileResult); expect(startInstagramCommentsScraping).not.toHaveBeenCalled(); - expect(selectPostUrlsWithComments).not.toHaveBeenCalled(); + expect(getPosts).not.toHaveBeenCalled(); }); it("fans out two scrapes: resultsLimit=1 for seen urls, default for unseen", async () => { const url1 = "https://instagram.com/p/1"; const url2 = "https://instagram.com/p/2"; - vi.mocked(selectPostUrlsWithComments).mockResolvedValue([url1]); + vi.mocked(getPosts).mockResolvedValue([ + { id: "p1", post_url: url1 }, + { id: "p2", post_url: url2 }, + ] as never); + vi.mocked(selectPostComments).mockResolvedValue([{ post_id: "p1" }] as never); await handleInstagramProfileFollowUpRuns([baseProfile], { ...baseProfile, @@ -62,4 +68,18 @@ describe("handleInstagramProfileFollowUpRuns", () => { expect(startInstagramCommentsScraping).toHaveBeenCalledWith([url1], 1); expect(startInstagramCommentsScraping).toHaveBeenCalledWith([url2]); }); + + it("backfills comments for all urls when no posts row exists yet", async () => { + const url1 = "https://instagram.com/p/1"; + vi.mocked(getPosts).mockResolvedValue([]); + + await handleInstagramProfileFollowUpRuns([baseProfile], { + ...baseProfile, + latestPosts: [{ url: url1 }], + } as ApifyInstagramProfileResult); + + expect(selectPostComments).not.toHaveBeenCalled(); + expect(startInstagramCommentsScraping).toHaveBeenCalledOnce(); + expect(startInstagramCommentsScraping).toHaveBeenCalledWith([url1]); + }); }); diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts index 92f0345dd..9092a197b 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts @@ -5,7 +5,7 @@ import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; import { getPosts } from "@/lib/supabase/posts/getPosts"; import { handleInstagramProfileFollowUpRuns } from "../handleInstagramProfileFollowUpRuns"; import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; -import { insertSocials } from "@/lib/supabase/socials/insertSocials"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; import { selectSocials } from "@/lib/supabase/socials/selectSocials"; import { upsertSocialPosts } from "@/lib/supabase/social_posts/upsertSocialPosts"; import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccountSocials"; @@ -26,7 +26,7 @@ vi.mock("../handleInstagramProfileFollowUpRuns", () => ({ handleInstagramProfileFollowUpRuns: vi.fn(), })); vi.mock("@/lib/apify/sendApifyWebhookEmail", () => ({ sendApifyWebhookEmail: vi.fn() })); -vi.mock("@/lib/supabase/socials/insertSocials", () => ({ insertSocials: vi.fn() })); +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ upsertSocials: vi.fn() })); vi.mock("@/lib/supabase/socials/selectSocials", () => ({ selectSocials: vi.fn(), })); @@ -76,7 +76,7 @@ describe("handleInstagramProfileScraperResults", () => { vi.mocked(upsertPosts).mockResolvedValue({ data: null, error: null } as never); vi.mocked(getPosts).mockResolvedValue(posts); vi.mocked(uploadLinkToArweave).mockResolvedValue(null); - vi.mocked(insertSocials).mockResolvedValue([] as never); + vi.mocked(upsertSocials).mockResolvedValue([] as never); vi.mocked(selectSocials).mockResolvedValue([{ id: "s1" }] as never); vi.mocked(selectAccountSocials).mockResolvedValue([{ account_id: "a1" }] as never); vi.mocked(getAccountArtistIds).mockResolvedValue([{ account_id: "a1" }] as never); diff --git a/lib/apify/instagram/getOrCreateSocialsForComments.ts b/lib/apify/instagram/getOrCreateSocialsForComments.ts index 962882072..71a0bea54 100644 --- a/lib/apify/instagram/getOrCreateSocialsForComments.ts +++ b/lib/apify/instagram/getOrCreateSocialsForComments.ts @@ -1,11 +1,11 @@ -import { insertSocials } from "@/lib/supabase/socials/insertSocials"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; import type { Tables, TablesInsert } from "@/types/database.types"; import type { ApifyInstagramComment } from "@/lib/apify/types"; /** * Ensures a `socials` row exists for each distinct comment author and * returns a map from `username` to the upserted social row. Upstream - * `insertSocials` upserts on `profile_url`, so repeated calls are + * `upsertSocials` upserts on `profile_url`, so repeated calls are * idempotent even when the same commenter recurs across posts. * * @param comments - Apify Instagram comment dataset items. @@ -25,6 +25,6 @@ export async function getOrCreateSocialsForComments( }); } - const upserted = await insertSocials(rows); + const upserted = await upsertSocials(rows); return new Map(upserted.map(s => [s.username, s])); } diff --git a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts index 9b79d35ba..a84917b83 100644 --- a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts +++ b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts @@ -1,5 +1,6 @@ import { startInstagramCommentsScraping } from "@/lib/apify/instagram/startInstagramCommentsScraping"; -import { selectPostUrlsWithComments } from "@/lib/supabase/post_comments/selectPostUrlsWithComments"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; +import { selectPostComments } from "@/lib/supabase/post_comments/selectPostComments"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; /** @@ -24,15 +25,18 @@ export async function handleInstagramProfileFollowUpRuns( const postUrls = firstResult.latestPosts.map(p => p.url).filter(Boolean); if (postUrls.length === 0) return; - const withComments = await selectPostUrlsWithComments(postUrls); + const posts = await getPosts({ postUrls }); + if (posts.length === 0) { + await startInstagramCommentsScraping(postUrls); + return; + } + + const comments = await selectPostComments({ postIds: posts.map(p => p.id) }); + const idsWithComments = new Set(comments.map(c => c.post_id)); + const withComments = posts.filter(p => idsWithComments.has(p.id)).map(p => p.post_url); const withSet = new Set(withComments); const withoutComments = postUrls.filter(url => !withSet.has(url)); - if (withComments.length > 0) { - await startInstagramCommentsScraping(withComments, 1); - } - - if (withoutComments.length > 0) { - await startInstagramCommentsScraping(withoutComments); - } + if (withComments.length > 0) await startInstagramCommentsScraping(withComments, 1); + if (withoutComments.length > 0) await startInstagramCommentsScraping(withoutComments); } diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index dcd6992ad..194a76dde 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -3,7 +3,7 @@ import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; import { getPosts } from "@/lib/supabase/posts/getPosts"; import { handleInstagramProfileFollowUpRuns } from "@/lib/apify/instagram/handleInstagramProfileFollowUpRuns"; import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; -import { insertSocials } from "@/lib/supabase/socials/insertSocials"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; import { selectSocials } from "@/lib/supabase/socials/selectSocials"; import { upsertSocialPosts } from "@/lib/supabase/social_posts/upsertSocialPosts"; import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccountSocials"; @@ -55,7 +55,7 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP // chain (social_posts link, email, follow-up scrape) is short-circuited. const normalizedUrl = normalizeProfileUrl(firstResult.url); - await insertSocials([ + await upsertSocials([ { username: firstResult.username ?? "", avatar: firstResult.profilePicUrl ?? null, diff --git a/lib/artist/__tests__/updateArtistSocials.test.ts b/lib/artist/__tests__/updateArtistSocials.test.ts index c30888215..5c8876086 100644 --- a/lib/artist/__tests__/updateArtistSocials.test.ts +++ b/lib/artist/__tests__/updateArtistSocials.test.ts @@ -24,8 +24,8 @@ vi.mock("@/lib/supabase/socials/selectSocials", () => ({ selectSocials: (...args: unknown[]) => mockSelectSocials(...args), })); -vi.mock("@/lib/supabase/socials/insertSocials", () => ({ - insertSocials: (...args: unknown[]) => mockInsertSocials(...args), +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ + upsertSocials: (...args: unknown[]) => mockInsertSocials(...args), })); describe("updateArtistSocials", () => { @@ -182,7 +182,7 @@ describe("updateArtistSocials", () => { mockSelectAccountSocials.mockResolvedValue([]); mockSelectSocials.mockResolvedValue([]); - // Mock insertSocials to return different IDs for each call + // Mock upsertSocials to return different IDs for each call mockInsertSocials.mockImplementation(socials => { const social = socials[0]; if (social.profile_url.includes("instagram")) { diff --git a/lib/artist/updateArtistSocials.ts b/lib/artist/updateArtistSocials.ts index 38fa077a5..04c91c5f2 100644 --- a/lib/artist/updateArtistSocials.ts +++ b/lib/artist/updateArtistSocials.ts @@ -5,7 +5,7 @@ import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccou import { deleteAccountSocial } from "@/lib/supabase/account_socials/deleteAccountSocial"; import { insertAccountSocial } from "@/lib/supabase/account_socials/insertAccountSocial"; import { selectSocials } from "@/lib/supabase/socials/selectSocials"; -import { insertSocials } from "@/lib/supabase/socials/insertSocials"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; import type { AccountSocialWithSocial } from "@/lib/supabase/account_socials/selectAccountSocials"; /** @@ -55,7 +55,7 @@ export async function updateArtistSocials( } else { // Create new social record const username = getUsernameFromProfileUrl(value); - const newSocials = await insertSocials([ + const newSocials = await upsertSocials([ { username, profile_url: normalizedUrl, diff --git a/lib/supabase/post_comments/selectPostUrlsWithComments.ts b/lib/supabase/post_comments/selectPostUrlsWithComments.ts deleted file mode 100644 index f0c5bab7c..000000000 --- a/lib/supabase/post_comments/selectPostUrlsWithComments.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { getPosts } from "@/lib/supabase/posts/getPosts"; -import { selectPostComments } from "./selectPostComments"; - -/** - * Returns the subset of `postUrls` for which at least one row exists in - * `post_comments`. Used by the Apify follow-up dispatcher to decide - * whether a post needs a full comments backfill or just a cheap refresh. - * - * Composes `getPosts` (URL → id) and `selectPostComments` (post_id - * existence) so each underlying supabase call stays single-table. - */ -export async function selectPostUrlsWithComments(postUrls: string[]) { - if (postUrls.length === 0) return []; - - const posts = await getPosts({ postUrls }); - if (posts.length === 0) return []; - - const postIds = posts.map(p => p.id); - const comments = await selectPostComments({ postIds }); - - const postIdsWithComments = new Set(comments.map(c => c.post_id)); - return posts.filter(p => postIdsWithComments.has(p.id)).map(p => p.post_url); -} diff --git a/lib/supabase/socials/insertSocials.ts b/lib/supabase/socials/upsertSocials.ts similarity index 88% rename from lib/supabase/socials/insertSocials.ts rename to lib/supabase/socials/upsertSocials.ts index 46b11a26c..8bb80d58e 100644 --- a/lib/supabase/socials/insertSocials.ts +++ b/lib/supabase/socials/upsertSocials.ts @@ -10,7 +10,7 @@ import type { TablesInsert } from "@/types/database.types"; * @param socials - Array of social data to insert * @returns Array of inserted social records */ -export async function insertSocials(socials: TablesInsert<"socials">[]) { +export async function upsertSocials(socials: TablesInsert<"socials">[]) { // stripNullish drops keys, so the inferred type widens to Partial<…>; // cast back since the supabase upsert overload still expects the full // insert type even though missing optional fields are fine at runtime. @@ -21,7 +21,7 @@ export async function insertSocials(socials: TablesInsert<"socials">[]) { .select("*"); if (error) { - console.error("[ERROR] insertSocials:", error); + console.error("[ERROR] upsertSocials:", error); throw error; } From 79a0b25c4d204c322ed0df66e061333d0a519ac8 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 14:00:15 +0000 Subject: [PATCH 21/23] refactor(apify): collapse two-query post-comments check into one handleInstagramProfileFollowUpRuns previously did two round-trips: getPosts to resolve URLs to ids, then selectPostComments to ask "which of these post_ids have at least one comment?". Replace both with a single PostgREST embed query in a new lib/supabase/posts/getPostsWithComments.ts helper that selects posts joined with their post_comments(post_id). A post has comments iff its embedded `post_comments` array is non-empty. Drop the now-unused lib/supabase/post_comments/selectPostComments.ts. Also tightens the no-posts-row case: if no posts row exists yet, just kick off the default backfill scrape for all URLs in one call instead of running an extra DB lookup first. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- ...handleInstagramProfileFollowUpRuns.test.ts | 22 +++++------ .../handleInstagramProfileFollowUpRuns.ts | 9 ++--- .../post_comments/selectPostComments.ts | 30 --------------- lib/supabase/posts/getPostsWithComments.ts | 38 +++++++++++++++++++ 4 files changed, 50 insertions(+), 49 deletions(-) delete mode 100644 lib/supabase/post_comments/selectPostComments.ts create mode 100644 lib/supabase/posts/getPostsWithComments.ts diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts index b5f6c495a..3944c8b27 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts @@ -1,16 +1,14 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleInstagramProfileFollowUpRuns } from "../handleInstagramProfileFollowUpRuns"; import { startInstagramCommentsScraping } from "../startInstagramCommentsScraping"; -import { getPosts } from "@/lib/supabase/posts/getPosts"; -import { selectPostComments } from "@/lib/supabase/post_comments/selectPostComments"; +import { getPostsWithComments } from "@/lib/supabase/posts/getPostsWithComments"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; vi.mock("../startInstagramCommentsScraping", () => ({ startInstagramCommentsScraping: vi.fn(), })); -vi.mock("@/lib/supabase/posts/getPosts", () => ({ getPosts: vi.fn() })); -vi.mock("@/lib/supabase/post_comments/selectPostComments", () => ({ - selectPostComments: vi.fn(), +vi.mock("@/lib/supabase/posts/getPostsWithComments", () => ({ + getPostsWithComments: vi.fn(), })); const baseProfile: ApifyInstagramProfileResult = { @@ -36,7 +34,7 @@ describe("handleInstagramProfileFollowUpRuns", () => { } as ApifyInstagramProfileResult); expect(startInstagramCommentsScraping).not.toHaveBeenCalled(); - expect(getPosts).not.toHaveBeenCalled(); + expect(getPostsWithComments).not.toHaveBeenCalled(); }); it("does not kick off comments scrape when latestPosts is empty", async () => { @@ -46,18 +44,17 @@ describe("handleInstagramProfileFollowUpRuns", () => { } as ApifyInstagramProfileResult); expect(startInstagramCommentsScraping).not.toHaveBeenCalled(); - expect(getPosts).not.toHaveBeenCalled(); + expect(getPostsWithComments).not.toHaveBeenCalled(); }); it("fans out two scrapes: resultsLimit=1 for seen urls, default for unseen", async () => { const url1 = "https://instagram.com/p/1"; const url2 = "https://instagram.com/p/2"; - vi.mocked(getPosts).mockResolvedValue([ - { id: "p1", post_url: url1 }, - { id: "p2", post_url: url2 }, + vi.mocked(getPostsWithComments).mockResolvedValue([ + { id: "p1", post_url: url1, post_comments: [{ post_id: "p1" }] }, + { id: "p2", post_url: url2, post_comments: [] }, ] as never); - vi.mocked(selectPostComments).mockResolvedValue([{ post_id: "p1" }] as never); await handleInstagramProfileFollowUpRuns([baseProfile], { ...baseProfile, @@ -71,14 +68,13 @@ describe("handleInstagramProfileFollowUpRuns", () => { it("backfills comments for all urls when no posts row exists yet", async () => { const url1 = "https://instagram.com/p/1"; - vi.mocked(getPosts).mockResolvedValue([]); + vi.mocked(getPostsWithComments).mockResolvedValue([]); await handleInstagramProfileFollowUpRuns([baseProfile], { ...baseProfile, latestPosts: [{ url: url1 }], } as ApifyInstagramProfileResult); - expect(selectPostComments).not.toHaveBeenCalled(); expect(startInstagramCommentsScraping).toHaveBeenCalledOnce(); expect(startInstagramCommentsScraping).toHaveBeenCalledWith([url1]); }); diff --git a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts index a84917b83..46e6ae1de 100644 --- a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts +++ b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts @@ -1,6 +1,5 @@ import { startInstagramCommentsScraping } from "@/lib/apify/instagram/startInstagramCommentsScraping"; -import { getPosts } from "@/lib/supabase/posts/getPosts"; -import { selectPostComments } from "@/lib/supabase/post_comments/selectPostComments"; +import { getPostsWithComments } from "@/lib/supabase/posts/getPostsWithComments"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; /** @@ -25,15 +24,13 @@ export async function handleInstagramProfileFollowUpRuns( const postUrls = firstResult.latestPosts.map(p => p.url).filter(Boolean); if (postUrls.length === 0) return; - const posts = await getPosts({ postUrls }); + const posts = await getPostsWithComments({ postUrls }); if (posts.length === 0) { await startInstagramCommentsScraping(postUrls); return; } - const comments = await selectPostComments({ postIds: posts.map(p => p.id) }); - const idsWithComments = new Set(comments.map(c => c.post_id)); - const withComments = posts.filter(p => idsWithComments.has(p.id)).map(p => p.post_url); + const withComments = posts.filter(p => p.post_comments?.length).map(p => p.post_url); const withSet = new Set(withComments); const withoutComments = postUrls.filter(url => !withSet.has(url)); diff --git a/lib/supabase/post_comments/selectPostComments.ts b/lib/supabase/post_comments/selectPostComments.ts deleted file mode 100644 index 96a032b43..000000000 --- a/lib/supabase/post_comments/selectPostComments.ts +++ /dev/null @@ -1,30 +0,0 @@ -import supabase from "../serverClient"; - -type SelectPostCommentsParams = { - postIds?: string[]; -}; - -/** - * Selects rows from `post_comments`. When `postIds` is provided, - * returns only the `post_id` column for rows in that set — used as - * a lightweight existence check for posts that already have comments. - * - * @param params - Optional filters. - */ -export async function selectPostComments({ postIds }: SelectPostCommentsParams = {}) { - let query = supabase.from("post_comments").select("post_id"); - - if (postIds !== undefined) { - if (postIds.length === 0) return []; - query = query.in("post_id", postIds); - } - - const { data, error } = await query; - - if (error) { - console.error("[ERROR] selectPostComments:", error); - throw error; - } - - return data ?? []; -} diff --git a/lib/supabase/posts/getPostsWithComments.ts b/lib/supabase/posts/getPostsWithComments.ts new file mode 100644 index 000000000..3b2bf3525 --- /dev/null +++ b/lib/supabase/posts/getPostsWithComments.ts @@ -0,0 +1,38 @@ +import supabase from "@/lib/supabase/serverClient"; + +type GetPostsWithCommentsParams = { + postUrls?: string[]; + postIds?: string[]; +}; + +/** + * Fetches `posts` rows along with the `post_id` of any related + * `post_comments`. Used to check, in a single round-trip, which posts + * already have at least one comment recorded. + * + * Each row's `post_comments` is empty when the post has no comments. + * + * @param params - Optional filters. + */ +export async function getPostsWithComments({ postUrls, postIds }: GetPostsWithCommentsParams = {}) { + let query = supabase.from("posts").select("*, post_comments(post_id)"); + + if (postUrls !== undefined) { + if (postUrls.length === 0) return []; + query = query.in("post_url", postUrls); + } + + if (postIds !== undefined) { + if (postIds.length === 0) return []; + query = query.in("id", postIds); + } + + const { data, error } = await query; + + if (error) { + console.error("[ERROR] getPostsWithComments:", error); + throw error; + } + + return data ?? []; +} From 2b2407c5ddb07161610583c90aaca57b5eb9619e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 14:03:06 +0000 Subject: [PATCH 22/23] refactor(supabase): always include post_comments embed in getPosts Drop the standalone getPostsWithComments helper and fold the post_comments(post_id) embed directly into getPosts. Every caller already either ignored the field or wanted it; the conditional helper was unnecessary indirection. Saves a file + matches the existing single-helper pattern; the embed is cheap (one column from a related table) and supabase returns the field as an empty array when there are no related rows. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- ...handleInstagramProfileFollowUpRuns.test.ts | 14 +++---- .../handleInstagramProfileFollowUpRuns.ts | 4 +- lib/supabase/posts/getPosts.ts | 12 ++++-- lib/supabase/posts/getPostsWithComments.ts | 38 ------------------- 4 files changed, 17 insertions(+), 51 deletions(-) delete mode 100644 lib/supabase/posts/getPostsWithComments.ts diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts index 3944c8b27..cc4e308c3 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileFollowUpRuns.test.ts @@ -1,14 +1,14 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleInstagramProfileFollowUpRuns } from "../handleInstagramProfileFollowUpRuns"; import { startInstagramCommentsScraping } from "../startInstagramCommentsScraping"; -import { getPostsWithComments } from "@/lib/supabase/posts/getPostsWithComments"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; vi.mock("../startInstagramCommentsScraping", () => ({ startInstagramCommentsScraping: vi.fn(), })); -vi.mock("@/lib/supabase/posts/getPostsWithComments", () => ({ - getPostsWithComments: vi.fn(), +vi.mock("@/lib/supabase/posts/getPosts", () => ({ + getPosts: vi.fn(), })); const baseProfile: ApifyInstagramProfileResult = { @@ -34,7 +34,7 @@ describe("handleInstagramProfileFollowUpRuns", () => { } as ApifyInstagramProfileResult); expect(startInstagramCommentsScraping).not.toHaveBeenCalled(); - expect(getPostsWithComments).not.toHaveBeenCalled(); + expect(getPosts).not.toHaveBeenCalled(); }); it("does not kick off comments scrape when latestPosts is empty", async () => { @@ -44,14 +44,14 @@ describe("handleInstagramProfileFollowUpRuns", () => { } as ApifyInstagramProfileResult); expect(startInstagramCommentsScraping).not.toHaveBeenCalled(); - expect(getPostsWithComments).not.toHaveBeenCalled(); + expect(getPosts).not.toHaveBeenCalled(); }); it("fans out two scrapes: resultsLimit=1 for seen urls, default for unseen", async () => { const url1 = "https://instagram.com/p/1"; const url2 = "https://instagram.com/p/2"; - vi.mocked(getPostsWithComments).mockResolvedValue([ + vi.mocked(getPosts).mockResolvedValue([ { id: "p1", post_url: url1, post_comments: [{ post_id: "p1" }] }, { id: "p2", post_url: url2, post_comments: [] }, ] as never); @@ -68,7 +68,7 @@ describe("handleInstagramProfileFollowUpRuns", () => { it("backfills comments for all urls when no posts row exists yet", async () => { const url1 = "https://instagram.com/p/1"; - vi.mocked(getPostsWithComments).mockResolvedValue([]); + vi.mocked(getPosts).mockResolvedValue([]); await handleInstagramProfileFollowUpRuns([baseProfile], { ...baseProfile, diff --git a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts index 46e6ae1de..4696d594b 100644 --- a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts +++ b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts @@ -1,5 +1,5 @@ import { startInstagramCommentsScraping } from "@/lib/apify/instagram/startInstagramCommentsScraping"; -import { getPostsWithComments } from "@/lib/supabase/posts/getPostsWithComments"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; /** @@ -24,7 +24,7 @@ export async function handleInstagramProfileFollowUpRuns( const postUrls = firstResult.latestPosts.map(p => p.url).filter(Boolean); if (postUrls.length === 0) return; - const posts = await getPostsWithComments({ postUrls }); + const posts = await getPosts({ postUrls }); if (posts.length === 0) { await startInstagramCommentsScraping(postUrls); return; diff --git a/lib/supabase/posts/getPosts.ts b/lib/supabase/posts/getPosts.ts index 9c5ba3c7d..1e1f6ec1c 100644 --- a/lib/supabase/posts/getPosts.ts +++ b/lib/supabase/posts/getPosts.ts @@ -6,15 +6,19 @@ type GetPostsParams = { }; /** - * Fetches `posts` rows. When `postUrls` is provided, scopes to rows - * whose `post_url` is in that list. When `postIds` is provided, scopes - * to rows whose `id` is in that list. Returns an empty array when an + * Fetches `posts` rows along with the `post_id` of any related + * `post_comments`. The embedded `post_comments` array is empty when + * the post has no comments — useful as a one-round-trip existence + * check. + * + * When `postUrls` or `postIds` is provided, scopes to rows whose + * `post_url` / `id` is in that list. Returns an empty array when an * explicit but empty filter list is passed. * * @param params - Optional filters. */ export async function getPosts({ postUrls, postIds }: GetPostsParams = {}) { - let query = supabase.from("posts").select("*"); + let query = supabase.from("posts").select("*, post_comments(post_id)"); if (postUrls !== undefined) { if (postUrls.length === 0) return []; diff --git a/lib/supabase/posts/getPostsWithComments.ts b/lib/supabase/posts/getPostsWithComments.ts deleted file mode 100644 index 3b2bf3525..000000000 --- a/lib/supabase/posts/getPostsWithComments.ts +++ /dev/null @@ -1,38 +0,0 @@ -import supabase from "@/lib/supabase/serverClient"; - -type GetPostsWithCommentsParams = { - postUrls?: string[]; - postIds?: string[]; -}; - -/** - * Fetches `posts` rows along with the `post_id` of any related - * `post_comments`. Used to check, in a single round-trip, which posts - * already have at least one comment recorded. - * - * Each row's `post_comments` is empty when the post has no comments. - * - * @param params - Optional filters. - */ -export async function getPostsWithComments({ postUrls, postIds }: GetPostsWithCommentsParams = {}) { - let query = supabase.from("posts").select("*, post_comments(post_id)"); - - if (postUrls !== undefined) { - if (postUrls.length === 0) return []; - query = query.in("post_url", postUrls); - } - - if (postIds !== undefined) { - if (postIds.length === 0) return []; - query = query.in("id", postIds); - } - - const { data, error } = await query; - - if (error) { - console.error("[ERROR] getPostsWithComments:", error); - throw error; - } - - return data ?? []; -} From 1f9ed1eefe001cf3e5cd9395e4238fad96211f8f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 14:10:48 +0000 Subject: [PATCH 23/23] fix(apify): address coderabbit review feedback batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inline (real) issues: - handleInstagramProfileScraperResults: filter latestPosts items with falsy `url` before upsertPosts. A single missing url would otherwise abort the whole webhook run. - handleInstagramProfileFollowUpRuns: dedupe postUrls with a Set before the DB lookup so duplicate latestPosts entries don't queue duplicate scrapes. - startInstagramProfileScraping: lowercase handles before deduping. Instagram usernames are case-insensitive — `["Foo", "foo"]` should collapse to one actor input. - uploadLinkToArweave: reject non-image content-types before buffering. Stops content-type-mismatched fetches from being uploaded to Arweave and labelled as image/png. Quick wins: - getOrCreatePostsForComments: use a Set for the missing-URL check instead of nested `.some()` (O(n²) → O(n)). - upsertSocials: use the canonical `@/lib/supabase/serverClient` import path. Skipped (with rationale): - Function-size nitpicks on profile scraper results, follow-up runs, webhook handler, arweave uploader, profile scraper start. Per prior direction, prefer inlining single-use helpers over splitting. - Explicit return type on upsertSocials. Conflicts with the user's policy of not annotating supabase libs (SDK already types them). Verification: pnpm lint:check clean, pnpm test 2338/2338 pass. https://claude.ai/code/session_017bcTcG7dzfpdkpYsgnVqVS --- lib/apify/instagram/getOrCreatePostsForComments.ts | 3 ++- lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts | 4 +++- .../instagram/handleInstagramProfileScraperResults.ts | 8 ++++---- lib/apify/instagram/startInstagramProfileScraping.ts | 2 +- lib/arweave/uploadLinkToArweave.ts | 4 +++- lib/supabase/socials/upsertSocials.ts | 2 +- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/apify/instagram/getOrCreatePostsForComments.ts b/lib/apify/instagram/getOrCreatePostsForComments.ts index e13dd4e3a..9ce0cc7bb 100644 --- a/lib/apify/instagram/getOrCreatePostsForComments.ts +++ b/lib/apify/instagram/getOrCreatePostsForComments.ts @@ -16,7 +16,8 @@ export async function getOrCreatePostsForComments( if (unique.length === 0) return new Map(); const existing = await getPosts({ postUrls: unique }); - const missing = unique.filter(url => !existing.some(p => p.post_url === url)); + const existingUrls = new Set(existing.map(p => p.post_url)); + const missing = unique.filter(url => !existingUrls.has(url)); let all = existing; if (missing.length > 0) { diff --git a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts index 4696d594b..08a65058d 100644 --- a/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts +++ b/lib/apify/instagram/handleInstagramProfileFollowUpRuns.ts @@ -21,7 +21,9 @@ export async function handleInstagramProfileFollowUpRuns( if (dataset.length !== 1) return; if (!firstResult.latestPosts || firstResult.latestPosts.length === 0) return; - const postUrls = firstResult.latestPosts.map(p => p.url).filter(Boolean); + const postUrls = Array.from( + new Set(firstResult.latestPosts.flatMap(p => (p.url ? [p.url] : []))), + ); if (postUrls.length === 0) return; const posts = await getPosts({ postUrls }); diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index 194a76dde..ff8a971c5 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -36,10 +36,10 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP const firstResult = dataset[0] as ApifyInstagramProfileResult | undefined; if (!firstResult?.latestPosts) return { posts: [], social: null }; - const postRows: TablesInsert<"posts">[] = firstResult.latestPosts.map(post => ({ - post_url: post.url, - updated_at: post.timestamp, - })); + const postRows: TablesInsert<"posts">[] = firstResult.latestPosts.flatMap(post => + post.url ? [{ post_url: post.url, updated_at: post.timestamp }] : [], + ); + if (postRows.length === 0) return { posts: [], social: null }; await upsertPosts(postRows); const posts = await getPosts({ postUrls: postRows.map(p => p.post_url) }); diff --git a/lib/apify/instagram/startInstagramProfileScraping.ts b/lib/apify/instagram/startInstagramProfileScraping.ts index b62d96a5a..605a0cc6d 100644 --- a/lib/apify/instagram/startInstagramProfileScraping.ts +++ b/lib/apify/instagram/startInstagramProfileScraping.ts @@ -15,7 +15,7 @@ export async function startInstagramProfileScraping( ): Promise { const list = Array.isArray(handles) ? handles : [handles]; const cleanHandles = Array.from( - new Set(list.map(h => h.trim().replace(/^@/, "")).filter(h => h.length > 0)), + new Set(list.map(h => h.trim().replace(/^@/, "").toLowerCase()).filter(h => h.length > 0)), ); if (cleanHandles.length === 0) { diff --git a/lib/arweave/uploadLinkToArweave.ts b/lib/arweave/uploadLinkToArweave.ts index 12f8a0c92..153e4b8f6 100644 --- a/lib/arweave/uploadLinkToArweave.ts +++ b/lib/arweave/uploadLinkToArweave.ts @@ -32,6 +32,9 @@ export async function uploadLinkToArweave( }); if (!res.ok || !res.body) return null; + const contentType = (res.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase(); + if (!contentType.startsWith("image/")) return null; + const contentLength = Number(res.headers.get("content-length") ?? 0); if (contentLength > MAX_IMAGE_BYTES) return null; @@ -51,7 +54,6 @@ export async function uploadLinkToArweave( } const buffer = Buffer.concat(chunks.map(c => Buffer.from(c))); - const contentType = res.headers.get("content-type") || "image/png"; const transaction = await uploadToArweave(buffer, contentType); return transaction?.id ?? null; diff --git a/lib/supabase/socials/upsertSocials.ts b/lib/supabase/socials/upsertSocials.ts index 8bb80d58e..90fb2119f 100644 --- a/lib/supabase/socials/upsertSocials.ts +++ b/lib/supabase/socials/upsertSocials.ts @@ -1,4 +1,4 @@ -import supabase from "../serverClient"; +import supabase from "@/lib/supabase/serverClient"; import { stripNullish } from "@/lib/objects/stripNullish"; import type { TablesInsert } from "@/types/database.types";