fix: decode HTML entities in news feed API responses

.changeset/fix-html-entities-news.md

@@ -0,0 +1,4 @@
+---
+---
+
+Fix HTML entity encoding in news feed API responses. RSS feed titles with entities like `'` are now properly decoded before being returned in API responses.

server/src/addie/services/feed-fetcher.ts

@@ -7,6 +7,7 @@
 
 import Parser from 'rss-parser';
 import { logger } from '../../logger.js';
+import { decodeHtmlEntities } from '../../utils/html-entities.js';
 import {
   getFeedsToFetch,
   getFeedById,
@@ -17,33 +18,6 @@ import {
   type RssArticleInput,
 } from '../../db/industry-feeds-db.js';
 
-/**
- * Decode HTML entities in RSS feed content.
- * Handles both named entities and numeric character references.
- */
-function decodeHtmlEntities(text: string): string {
-  return text
-    // Numeric character references (decimal) - must come first
-    // Use fromCodePoint to handle code points beyond BMP (emoji, etc.)
-    .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(parseInt(code, 10)))
-    // Numeric character references (hex)
-    .replace(/&#x([0-9a-fA-F]+);/g, (_, code) => String.fromCodePoint(parseInt(code, 16)))
-    // Named entities
-    .replace(/'/g, "'")
-    .replace(/"/g, '"')
-    .replace(/&/g, '&')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/&nbsp;/g, ' ')
-    .replace(/&ndash;/g, '\u2013')
-    .replace(/&mdash;/g, '\u2014')
-    .replace(/&lsquo;/g, '\u2018')
-    .replace(/&rsquo;/g, '\u2019')
-    .replace(/&ldquo;/g, '\u201C')
-    .replace(/&rdquo;/g, '\u201D')
-    .replace(/&hellip;/g, '\u2026');
-}
-
 const parser = new Parser({
   timeout: 30000,
   headers: {

server/src/http.ts

@@ -58,6 +58,7 @@ import { createPublicBillingRouter } from "./routes/billing-public.js";
 import { createOrganizationsRouter } from "./routes/organizations.js";
 import { createEventsRouter } from "./routes/events.js";
 import { createLatestRouter } from "./routes/latest.js";
+import { decodeHtmlEntities } from "./utils/html-entities.js";
 import { createCommitteeRouters } from "./routes/committees.js";
 import { sendWelcomeEmail, sendUserSignupEmail, emailDb } from "./notifications/email.js";
 import { emailPrefsDb } from "./db/email-preferences-db.js";
@@ -3322,18 +3323,6 @@ export class HTTPServer {
         const ogSiteMatch = html.match(/<meta[^>]+property=["']og:site_name["'][^>]+content=["']([^"']+)["']/i)
           || html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:site_name["']/i);
 
-        // Helper to decode HTML entities
-        const decodeHtmlEntities = (text: string): string => {
-          return text
-            .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
-            .replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10)))
-            .replace(/&quot;/g, '"')
-            .replace(/&apos;/g, "'")
-            .replace(/&lt;/g, '<')
-            .replace(/&gt;/g, '>')
-            .replace(/&amp;/g, '&');
-        };
-
         // Determine title (prefer og:title, then <title>)
         let title = ogTitleMatch?.[1] || titleMatch?.[1] || '';
         title = decodeHtmlEntities(title.trim());

server/src/routes/admin/feeds.ts

@@ -13,6 +13,7 @@
 import { Router } from 'express';
 import { createLogger } from '../../logger.js';
 import { requireAuth, requireAdmin } from '../../middleware/auth.js';
+import { decodeHtmlEntities } from '../../utils/html-entities.js';
 import {
   getAllFeedsWithStats,
   getFeedById,
@@ -24,6 +25,7 @@ import {
   getFeedStats,
   enableFeedEmail,
   disableFeedEmail,
+  type RecentArticle,
 } from '../../db/industry-feeds-db.js';
 import { fetchSingleFeed } from '../../addie/services/feed-fetcher.js';
 
@@ -169,7 +171,13 @@ export function createAdminFeedsRouter(): Router {
       }
 
       const recentArticles = await getRecentArticlesForFeed(feedId, 10);
-      res.json({ feed, recentArticles });
+      // Decode HTML entities in article titles and summaries
+      const decodedArticles = recentArticles.map((article: RecentArticle) => ({
+        ...article,
+        title: decodeHtmlEntities(article.title),
+        summary: article.summary ? decodeHtmlEntities(article.summary) : null,
+      }));
+      res.json({ feed, recentArticles: decodedArticles });
     } catch (error) {
       logger.error({ err: error }, 'Get feed error');
       res.status(500).json({

server/src/routes/committees.ts

@@ -15,6 +15,7 @@ import { eventsDb } from "../db/events-db.js";
 import { invalidateMemberContextCache } from "../addie/index.js";
 import { syncWorkingGroupMembersFromSlack, syncAllWorkingGroupMembersFromSlack } from "../slack/sync.js";
 import { notifyWorkingGroupPost } from "../notifications/slack.js";
+import { decodeHtmlEntities } from "../utils/html-entities.js";
 
 const logger = createLogger("committee-routes");
 
@@ -39,20 +40,6 @@ const workos = AUTH_ENABLED
     })
   : null;
 
-/**
- * Helper to decode HTML entities from fetched pages
- */
-function decodeHtmlEntities(text: string): string {
-  return text
-    .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
-    .replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10)))
-    .replace(/"/g, '"')
-    .replace(/'/g, "'")
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/&amp;/g, '&');
-}
-
 /**
  * Fetch and extract metadata from a URL (for link posts)
  */

server/src/routes/latest.ts

@@ -8,6 +8,7 @@
 import { Router, type Request, type Response } from "express";
 import { createLogger } from "../logger.js";
 import { serveHtmlWithConfig } from "../utils/html-config.js";
+import { decodeHtmlEntities } from "../utils/html-entities.js";
 import {
   getWebsiteChannels,
   getChannelByWebsiteSlug,
@@ -39,6 +40,19 @@ interface LatestSection {
   article_count: number;
 }
 
+/**
+ * Decode HTML entities in article text fields.
+ * This handles cases where RSS feed data contains encoded entities.
+ */
+function decodeArticle<T extends LatestArticle>(article: T): T {
+  return {
+    ...article,
+    title: decodeHtmlEntities(article.title),
+    summary: article.summary ? decodeHtmlEntities(article.summary) : null,
+    addie_notes: article.addie_notes ? decodeHtmlEntities(article.addie_notes) : null,
+  };
+}
+
 /**
  * Create latest content routes
  */
@@ -214,7 +228,7 @@ export function createLatestRouter(): {
       const total = parseInt(countResult.rows[0]?.count || "0", 10);
 
       res.json({
-        articles: result.rows,
+        articles: result.rows.map(decodeArticle),
         pagination: {
           total,
           limit,
@@ -272,7 +286,7 @@ export function createLatestRouter(): {
         [limit]
       );
 
-      res.json({ articles: result.rows });
+      res.json({ articles: result.rows.map(decodeArticle) });
     } catch (error) {
       logger.error({ error }, "Error fetching featured articles");
       res.status(500).json({ error: "Failed to fetch featured articles" });

server/src/utils/html-entities.ts

@@ -0,0 +1,34 @@
+/**
+ * Decode HTML entities in text.
+ * Handles both named entities and numeric character references.
+ *
+ * @param text - The text containing HTML entities to decode
+ * @returns The decoded text with entities replaced by their character equivalents
+ */
+export function decodeHtmlEntities(text: string): string {
+  if (!text) return text;
+  return (
+    text
+      // Numeric character references (decimal) - must come first
+      // Use fromCodePoint to handle code points beyond BMP (emoji, etc.)
+      .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(parseInt(code, 10)))
+      // Numeric character references (hex)
+      .replace(/&#x([0-9a-fA-F]+);/g, (_, code) =>
+        String.fromCodePoint(parseInt(code, 16))
+      )
+      // Named entities
+      .replace(/'/g, "'")
+      .replace(/"/g, '"')
+      .replace(/&/g, '&')
+      .replace(/&lt;/g, '<')
+      .replace(/&gt;/g, '>')
+      .replace(/&nbsp;/g, ' ')
+      .replace(/&ndash;/g, '\u2013')
+      .replace(/&mdash;/g, '\u2014')
+      .replace(/&lsquo;/g, '\u2018')
+      .replace(/&rsquo;/g, '\u2019')
+      .replace(/&ldquo;/g, '\u201C')
+      .replace(/&rdquo;/g, '\u201D')
+      .replace(/&hellip;/g, '\u2026')
+  );
+}

tests/utils/html-entities.test.ts

@@ -0,0 +1,61 @@
+import { decodeHtmlEntities } from '../../server/src/utils/html-entities.js';
+
+describe('decodeHtmlEntities', () => {
+  it('decodes hex character references', () => {
+    expect(decodeHtmlEntities("'")).toBe("'");
+    expect(decodeHtmlEntities(""")).toBe('"');
+    expect(decodeHtmlEntities("IAB warns industry chasing 'shiny pennies'")).toBe(
+      "IAB warns industry chasing 'shiny pennies'"
+    );
+  });
+
+  it('decodes decimal character references', () => {
+    expect(decodeHtmlEntities("'")).toBe("'");
+    expect(decodeHtmlEntities(""")).toBe('"');
+  });
+
+  it('decodes named entities', () => {
+    expect(decodeHtmlEntities("'")).toBe("'");
+    expect(decodeHtmlEntities(""")).toBe('"');
+    expect(decodeHtmlEntities("&")).toBe("&");
+    expect(decodeHtmlEntities("&lt;")).toBe("<");
+    expect(decodeHtmlEntities("&gt;")).toBe(">");
+    expect(decodeHtmlEntities("&nbsp;")).toBe(" ");
+  });
+
+  it('decodes typographic entities', () => {
+    expect(decodeHtmlEntities("&ndash;")).toBe('\u2013');
+    expect(decodeHtmlEntities("&mdash;")).toBe('\u2014');
+    expect(decodeHtmlEntities("&lsquo;")).toBe('\u2018');
+    expect(decodeHtmlEntities("&rsquo;")).toBe('\u2019');
+    expect(decodeHtmlEntities("&ldquo;")).toBe('\u201C');
+    expect(decodeHtmlEntities("&rdquo;")).toBe('\u201D');
+    expect(decodeHtmlEntities("&hellip;")).toBe('\u2026');
+  });
+
+  it('handles mixed content', () => {
+    expect(
+      decodeHtmlEntities("Tom &amp; Jerry said &ldquo;Hello&#x27;s World&rdquo;")
+    ).toBe("Tom & Jerry said \u201CHello's World\u201D");
+  });
+
+  it('handles null and empty strings', () => {
+    expect(decodeHtmlEntities('')).toBe('');
+    // @ts-expect-error testing null handling
+    expect(decodeHtmlEntities(null)).toBe(null);
+    // @ts-expect-error testing undefined handling
+    expect(decodeHtmlEntities(undefined)).toBe(undefined);
+  });
+
+  it('preserves strings without entities', () => {
+    expect(decodeHtmlEntities("Hello World")).toBe("Hello World");
+    expect(decodeHtmlEntities("No entities here!")).toBe("No entities here!");
+  });
+
+  it('handles emoji and high code points', () => {
+    // Emoji encoded as decimal
+    expect(decodeHtmlEntities("&#128512;")).toBe("😀");
+    // Emoji encoded as hex
+    expect(decodeHtmlEntities("&#x1F600;")).toBe("😀");
+  });
+});