diff --git a/.changeset/itchy-ravens-feel.md b/.changeset/itchy-ravens-feel.md new file mode 100644 index 0000000000..a845151cc8 --- /dev/null +++ b/.changeset/itchy-ravens-feel.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/server/src/addie/services/industry-alerts.ts b/server/src/addie/services/industry-alerts.ts index f3478eb065..b4e99fba3d 100644 --- a/server/src/addie/services/industry-alerts.ts +++ b/server/src/addie/services/industry-alerts.ts @@ -3,7 +3,7 @@ * * Sends Slack notifications for industry articles with pacing: * - Quality 5 articles: Post immediately - * - Quality 4 articles: Post only if channel has been quiet for 3+ hours + * - Quality 4 articles: Post only if channel has been quiet for 1+ hour * - Quality < 4: Ignored (not relevant enough for our community) * * Posts one article at a time to encourage engagement. @@ -22,7 +22,7 @@ import { } from '../../db/notification-channels-db.js'; // Minimum hours of quiet before posting a quality 4 article -const QUIET_PERIOD_HOURS = 3; +const QUIET_PERIOD_HOURS = 1; interface ArticleToAlert { id: string; @@ -261,7 +261,7 @@ async function getNextArticleForChannel( * Pacing rules: * - Only posts ONE article per channel per run * - Quality 5: Post immediately - * - Quality 4: Post only if channel has been quiet for 3+ hours + * - Quality 4: Post only if channel has been quiet for 1+ hour * - Quality < 4: Ignored (not posted anywhere) */ export async function processAlerts(): Promise<{ diff --git a/server/src/db/industry-feeds-db.ts b/server/src/db/industry-feeds-db.ts index 220003f71f..426ef8a2b1 100644 --- a/server/src/db/industry-feeds-db.ts +++ b/server/src/db/industry-feeds-db.ts @@ -256,12 +256,18 @@ function generateSlug(title: string, guid: string): string { } /** - * Check if RSS article already exists as a perspective + * Check if RSS article already exists as a perspective. + * Checks both (feed_id, guid) for same-feed duplicates and + * external_url for cross-feed duplicates (same article from different feeds). */ -export async function rssArticleExists(feedId: number, guid: string): Promise { +export async function rssArticleExists(feedId: number, guid: string, externalUrl: string): Promise { const result = await query<{ exists: boolean }>( - `SELECT EXISTS(SELECT 1 FROM perspectives WHERE feed_id = $1 AND guid = $2)`, - [feedId, guid] + `SELECT EXISTS( + SELECT 1 FROM perspectives + WHERE (feed_id = $1 AND guid = $2) + OR external_url = $3 + )`, + [feedId, guid, externalUrl] ); return result.rows[0].exists; } @@ -271,8 +277,8 @@ export async function rssArticleExists(feedId: number, guid: string): Promise { - // First check if we already have this article (by guid) - const existing = await rssArticleExists(article.feed_id, article.guid); + // Check if we already have this article (by guid or URL to catch cross-feed duplicates) + const existing = await rssArticleExists(article.feed_id, article.guid, article.link); if (existing) { return null; } diff --git a/server/src/db/migrations/177_index_perspectives_external_url.sql b/server/src/db/migrations/177_index_perspectives_external_url.sql new file mode 100644 index 0000000000..0fb4e4dbbe --- /dev/null +++ b/server/src/db/migrations/177_index_perspectives_external_url.sql @@ -0,0 +1,6 @@ +-- Add index on external_url for cross-feed deduplication +-- When the same article appears in multiple RSS feeds (e.g., Adweek main + Adweek AI), +-- we need to detect duplicates by URL, not just by (feed_id, guid) +CREATE INDEX IF NOT EXISTS idx_perspectives_external_url + ON perspectives(external_url) + WHERE external_url IS NOT NULL; diff --git a/server/src/db/migrations/178_add_chatgpt_news_sources.sql b/server/src/db/migrations/178_add_chatgpt_news_sources.sql new file mode 100644 index 0000000000..b119297035 --- /dev/null +++ b/server/src/db/migrations/178_add_chatgpt_news_sources.sql @@ -0,0 +1,32 @@ +-- Migration: 178_add_chatgpt_news_sources.sql +-- Add sources covering ChatGPT/OpenAI ads news based on tipsheet.ai coverage +-- Verified RSS feeds for publications covering ChatGPT advertising launch + +-- Use DO block to handle potential duplicates gracefully since +-- industry_feeds uses a partial unique index (not a constraint) +DO $$ +BEGIN + INSERT INTO industry_feeds (name, feed_url, category) + SELECT * FROM (VALUES + ('The Verge', 'https://www.theverge.com/rss/index.xml', 'tech'), + ('Wired', 'https://www.wired.com/feed/rss', 'tech'), + ('VentureBeat', 'https://venturebeat.com/feed', 'tech'), + ('Ars Technica', 'https://feeds.arstechnica.com/arstechnica/index', 'tech'), + ('CNBC Tech', 'https://www.cnbc.com/id/19854910/device/rss/rss.html', 'business') + ) AS new_feeds(name, feed_url, category) + WHERE NOT EXISTS ( + SELECT 1 FROM industry_feeds WHERE feed_url = new_feeds.feed_url + ); +END $$; + +-- Note: The following do NOT have free/accessible RSS feeds: +-- - Financial Times (subscription only) +-- - The Information (subscription only) +-- - CNN Business (no RSS) +-- - Reuters Tech (requires authentication) +-- - Variety (paywalled/limited RSS) +-- +-- Consider adding these as email subscriptions: +-- INSERT INTO industry_feeds (name, feed_url, category, email_slug, accepts_email, is_active) VALUES +-- ('tipsheet.ai', 'email://tipsheet', 'ai', 'tipsheet', true, true) +-- ON CONFLICT (feed_url) DO NOTHING; diff --git a/tests/utils/html-entities.test.ts b/tests/utils/html-entities.test.ts index e4dba056b8..9311d0b9f3 100644 --- a/tests/utils/html-entities.test.ts +++ b/tests/utils/html-entities.test.ts @@ -12,6 +12,11 @@ describe('decodeHtmlEntities', () => { it('decodes decimal character references', () => { expect(decodeHtmlEntities("'")).toBe("'"); expect(decodeHtmlEntities(""")).toBe('"'); + // High code point: right single quote (U+2019) - common in Adweek titles + expect(decodeHtmlEntities("’")).toBe('\u2019'); + expect(decodeHtmlEntities("Read All of ADWEEK’s Year-Ahead")).toBe( + "Read All of ADWEEK\u2019s Year-Ahead" + ); }); it('decodes named entities', () => {