fix: Cross-feed article deduplication and new ChatGPT news sources

.changeset/itchy-ravens-feel.md

@@ -0,0 +1,2 @@
+---
+---

server/src/addie/services/industry-alerts.ts

@@ -3,7 +3,7 @@
  *
  * Sends Slack notifications for industry articles with pacing:
  * - Quality 5 articles: Post immediately
- * - Quality 4 articles: Post only if channel has been quiet for 3+ hours
+ * - Quality 4 articles: Post only if channel has been quiet for 1+ hour
  * - Quality < 4: Ignored (not relevant enough for our community)
  *
  * Posts one article at a time to encourage engagement.
@@ -22,7 +22,7 @@ import {
 } from '../../db/notification-channels-db.js';
 
 // Minimum hours of quiet before posting a quality 4 article
-const QUIET_PERIOD_HOURS = 3;
+const QUIET_PERIOD_HOURS = 1;
 
 interface ArticleToAlert {
   id: string;
@@ -261,7 +261,7 @@ async function getNextArticleForChannel(
  * Pacing rules:
  * - Only posts ONE article per channel per run
  * - Quality 5: Post immediately
- * - Quality 4: Post only if channel has been quiet for 3+ hours
+ * - Quality 4: Post only if channel has been quiet for 1+ hour
  * - Quality < 4: Ignored (not posted anywhere)
  */
 export async function processAlerts(): Promise<{

server/src/db/industry-feeds-db.ts

@@ -256,12 +256,18 @@ function generateSlug(title: string, guid: string): string {
 }
 
 /**
- * Check if RSS article already exists as a perspective
+ * Check if RSS article already exists as a perspective.
+ * Checks both (feed_id, guid) for same-feed duplicates and
+ * external_url for cross-feed duplicates (same article from different feeds).
  */
-export async function rssArticleExists(feedId: number, guid: string): Promise<boolean> {
+export async function rssArticleExists(feedId: number, guid: string, externalUrl: string): Promise<boolean> {
   const result = await query<{ exists: boolean }>(
-    `SELECT EXISTS(SELECT 1 FROM perspectives WHERE feed_id = $1 AND guid = $2)`,
-    [feedId, guid]
+    `SELECT EXISTS(
+      SELECT 1 FROM perspectives
+      WHERE (feed_id = $1 AND guid = $2)
+         OR external_url = $3
+    )`,
+    [feedId, guid, externalUrl]
   );
   return result.rows[0].exists;
 }
@@ -271,8 +277,8 @@ export async function rssArticleExists(feedId: number, guid: string): Promise<bo
  * Returns the perspective ID if created, null if it already exists
  */
 export async function createRssPerspective(article: RssArticleInput): Promise<string | null> {
-  // First check if we already have this article (by guid)
-  const existing = await rssArticleExists(article.feed_id, article.guid);
+  // Check if we already have this article (by guid or URL to catch cross-feed duplicates)
+  const existing = await rssArticleExists(article.feed_id, article.guid, article.link);
   if (existing) {
     return null;
   }

server/src/db/migrations/177_index_perspectives_external_url.sql

@@ -0,0 +1,6 @@
+-- Add index on external_url for cross-feed deduplication
+-- When the same article appears in multiple RSS feeds (e.g., Adweek main + Adweek AI),
+-- we need to detect duplicates by URL, not just by (feed_id, guid)
+CREATE INDEX IF NOT EXISTS idx_perspectives_external_url
+  ON perspectives(external_url)
+  WHERE external_url IS NOT NULL;

server/src/db/migrations/178_add_chatgpt_news_sources.sql

@@ -0,0 +1,32 @@
+-- Migration: 178_add_chatgpt_news_sources.sql
+-- Add sources covering ChatGPT/OpenAI ads news based on tipsheet.ai coverage
+-- Verified RSS feeds for publications covering ChatGPT advertising launch
+
+-- Use DO block to handle potential duplicates gracefully since
+-- industry_feeds uses a partial unique index (not a constraint)
+DO $$
+BEGIN
+  INSERT INTO industry_feeds (name, feed_url, category)
+  SELECT * FROM (VALUES
+    ('The Verge', 'https://www.theverge.com/rss/index.xml', 'tech'),
+    ('Wired', 'https://www.wired.com/feed/rss', 'tech'),
+    ('VentureBeat', 'https://venturebeat.com/feed', 'tech'),
+    ('Ars Technica', 'https://feeds.arstechnica.com/arstechnica/index', 'tech'),
+    ('CNBC Tech', 'https://www.cnbc.com/id/19854910/device/rss/rss.html', 'business')
+  ) AS new_feeds(name, feed_url, category)
+  WHERE NOT EXISTS (
+    SELECT 1 FROM industry_feeds WHERE feed_url = new_feeds.feed_url
+  );
+END $$;
+
+-- Note: The following do NOT have free/accessible RSS feeds:
+-- - Financial Times (subscription only)
+-- - The Information (subscription only)
+-- - CNN Business (no RSS)
+-- - Reuters Tech (requires authentication)
+-- - Variety (paywalled/limited RSS)
+--
+-- Consider adding these as email subscriptions:
+-- INSERT INTO industry_feeds (name, feed_url, category, email_slug, accepts_email, is_active) VALUES
+--   ('tipsheet.ai', 'email://tipsheet', 'ai', 'tipsheet', true, true)
+-- ON CONFLICT (feed_url) DO NOTHING;

tests/utils/html-entities.test.ts

@@ -12,6 +12,11 @@ describe('decodeHtmlEntities', () => {
   it('decodes decimal character references', () => {
     expect(decodeHtmlEntities("'")).toBe("'");
     expect(decodeHtmlEntities(""")).toBe('"');
+    // High code point: right single quote (U+2019) - common in Adweek titles
+    expect(decodeHtmlEntities("’")).toBe('\u2019');
+    expect(decodeHtmlEntities("Read All of ADWEEK’s Year-Ahead")).toBe(
+      "Read All of ADWEEK\u2019s Year-Ahead"
+    );
   });
 
   it('decodes named entities', () => {