diff --git a/.changeset/fix-html-entities-news.md b/.changeset/fix-html-entities-news.md new file mode 100644 index 0000000000..745e2fe647 --- /dev/null +++ b/.changeset/fix-html-entities-news.md @@ -0,0 +1,4 @@ +--- +--- + +Fix HTML entity encoding in news feed API responses. RSS feed titles with entities like `'` are now properly decoded before being returned in API responses. diff --git a/server/src/addie/services/feed-fetcher.ts b/server/src/addie/services/feed-fetcher.ts index 37f3bd706c..a995687248 100644 --- a/server/src/addie/services/feed-fetcher.ts +++ b/server/src/addie/services/feed-fetcher.ts @@ -7,6 +7,7 @@ import Parser from 'rss-parser'; import { logger } from '../../logger.js'; +import { decodeHtmlEntities } from '../../utils/html-entities.js'; import { getFeedsToFetch, getFeedById, @@ -17,33 +18,6 @@ import { type RssArticleInput, } from '../../db/industry-feeds-db.js'; -/** - * Decode HTML entities in RSS feed content. - * Handles both named entities and numeric character references. - */ -function decodeHtmlEntities(text: string): string { - return text - // Numeric character references (decimal) - must come first - // Use fromCodePoint to handle code points beyond BMP (emoji, etc.) - .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(parseInt(code, 10))) - // Numeric character references (hex) - .replace(/&#x([0-9a-fA-F]+);/g, (_, code) => String.fromCodePoint(parseInt(code, 16))) - // Named entities - .replace(/'/g, "'") - .replace(/"/g, '"') - .replace(/&/g, '&') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/ /g, ' ') - .replace(/–/g, '\u2013') - .replace(/—/g, '\u2014') - .replace(/‘/g, '\u2018') - .replace(/’/g, '\u2019') - .replace(/“/g, '\u201C') - .replace(/”/g, '\u201D') - .replace(/…/g, '\u2026'); -} - const parser = new Parser({ timeout: 30000, headers: { diff --git a/server/src/http.ts b/server/src/http.ts index 14200511a4..ef9a56392d 100644 --- a/server/src/http.ts +++ b/server/src/http.ts @@ -58,6 +58,7 @@ import { createPublicBillingRouter } from "./routes/billing-public.js"; import { createOrganizationsRouter } from "./routes/organizations.js"; import { createEventsRouter } from "./routes/events.js"; import { createLatestRouter } from "./routes/latest.js"; +import { decodeHtmlEntities } from "./utils/html-entities.js"; import { createCommitteeRouters } from "./routes/committees.js"; import { sendWelcomeEmail, sendUserSignupEmail, emailDb } from "./notifications/email.js"; import { emailPrefsDb } from "./db/email-preferences-db.js"; @@ -3322,18 +3323,6 @@ export class HTTPServer { const ogSiteMatch = html.match(/]+property=["']og:site_name["'][^>]+content=["']([^"']+)["']/i) || html.match(/]+content=["']([^"']+)["'][^>]+property=["']og:site_name["']/i); - // Helper to decode HTML entities - const decodeHtmlEntities = (text: string): string => { - return text - .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))) - .replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10))) - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/&/g, '&'); - }; - // Determine title (prefer og:title, then ) let title = ogTitleMatch?.[1] || titleMatch?.[1] || ''; title = decodeHtmlEntities(title.trim()); diff --git a/server/src/routes/admin/feeds.ts b/server/src/routes/admin/feeds.ts index fbf87b884f..e951103c51 100644 --- a/server/src/routes/admin/feeds.ts +++ b/server/src/routes/admin/feeds.ts @@ -13,6 +13,7 @@ import { Router } from 'express'; import { createLogger } from '../../logger.js'; import { requireAuth, requireAdmin } from '../../middleware/auth.js'; +import { decodeHtmlEntities } from '../../utils/html-entities.js'; import { getAllFeedsWithStats, getFeedById, @@ -24,6 +25,7 @@ import { getFeedStats, enableFeedEmail, disableFeedEmail, + type RecentArticle, } from '../../db/industry-feeds-db.js'; import { fetchSingleFeed } from '../../addie/services/feed-fetcher.js'; @@ -169,7 +171,13 @@ export function createAdminFeedsRouter(): Router { } const recentArticles = await getRecentArticlesForFeed(feedId, 10); - res.json({ feed, recentArticles }); + // Decode HTML entities in article titles and summaries + const decodedArticles = recentArticles.map((article: RecentArticle) => ({ + ...article, + title: decodeHtmlEntities(article.title), + summary: article.summary ? decodeHtmlEntities(article.summary) : null, + })); + res.json({ feed, recentArticles: decodedArticles }); } catch (error) { logger.error({ err: error }, 'Get feed error'); res.status(500).json({ diff --git a/server/src/routes/committees.ts b/server/src/routes/committees.ts index a484b701b0..48459cc15a 100644 --- a/server/src/routes/committees.ts +++ b/server/src/routes/committees.ts @@ -15,6 +15,7 @@ import { eventsDb } from "../db/events-db.js"; import { invalidateMemberContextCache } from "../addie/index.js"; import { syncWorkingGroupMembersFromSlack, syncAllWorkingGroupMembersFromSlack } from "../slack/sync.js"; import { notifyWorkingGroupPost } from "../notifications/slack.js"; +import { decodeHtmlEntities } from "../utils/html-entities.js"; const logger = createLogger("committee-routes"); @@ -39,20 +40,6 @@ const workos = AUTH_ENABLED }) : null; -/** - * Helper to decode HTML entities from fetched pages - */ -function decodeHtmlEntities(text: string): string { - return text - .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))) - .replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10))) - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/&/g, '&'); -} - /** * Fetch and extract metadata from a URL (for link posts) */ diff --git a/server/src/routes/latest.ts b/server/src/routes/latest.ts index 991c8c2807..f4524b4835 100644 --- a/server/src/routes/latest.ts +++ b/server/src/routes/latest.ts @@ -8,6 +8,7 @@ import { Router, type Request, type Response } from "express"; import { createLogger } from "../logger.js"; import { serveHtmlWithConfig } from "../utils/html-config.js"; +import { decodeHtmlEntities } from "../utils/html-entities.js"; import { getWebsiteChannels, getChannelByWebsiteSlug, @@ -39,6 +40,19 @@ interface LatestSection { article_count: number; } +/** + * Decode HTML entities in article text fields. + * This handles cases where RSS feed data contains encoded entities. + */ +function decodeArticle<T extends LatestArticle>(article: T): T { + return { + ...article, + title: decodeHtmlEntities(article.title), + summary: article.summary ? decodeHtmlEntities(article.summary) : null, + addie_notes: article.addie_notes ? decodeHtmlEntities(article.addie_notes) : null, + }; +} + /** * Create latest content routes */ @@ -214,7 +228,7 @@ export function createLatestRouter(): { const total = parseInt(countResult.rows[0]?.count || "0", 10); res.json({ - articles: result.rows, + articles: result.rows.map(decodeArticle), pagination: { total, limit, @@ -272,7 +286,7 @@ export function createLatestRouter(): { [limit] ); - res.json({ articles: result.rows }); + res.json({ articles: result.rows.map(decodeArticle) }); } catch (error) { logger.error({ error }, "Error fetching featured articles"); res.status(500).json({ error: "Failed to fetch featured articles" }); diff --git a/server/src/utils/html-entities.ts b/server/src/utils/html-entities.ts new file mode 100644 index 0000000000..a8ec1fbb4c --- /dev/null +++ b/server/src/utils/html-entities.ts @@ -0,0 +1,34 @@ +/** + * Decode HTML entities in text. + * Handles both named entities and numeric character references. + * + * @param text - The text containing HTML entities to decode + * @returns The decoded text with entities replaced by their character equivalents + */ +export function decodeHtmlEntities(text: string): string { + if (!text) return text; + return ( + text + // Numeric character references (decimal) - must come first + // Use fromCodePoint to handle code points beyond BMP (emoji, etc.) + .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(parseInt(code, 10))) + // Numeric character references (hex) + .replace(/&#x([0-9a-fA-F]+);/g, (_, code) => + String.fromCodePoint(parseInt(code, 16)) + ) + // Named entities + .replace(/'/g, "'") + .replace(/"/g, '"') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/ /g, ' ') + .replace(/–/g, '\u2013') + .replace(/—/g, '\u2014') + .replace(/‘/g, '\u2018') + .replace(/’/g, '\u2019') + .replace(/“/g, '\u201C') + .replace(/”/g, '\u201D') + .replace(/…/g, '\u2026') + ); +} diff --git a/tests/utils/html-entities.test.ts b/tests/utils/html-entities.test.ts new file mode 100644 index 0000000000..e4dba056b8 --- /dev/null +++ b/tests/utils/html-entities.test.ts @@ -0,0 +1,61 @@ +import { decodeHtmlEntities } from '../../server/src/utils/html-entities.js'; + +describe('decodeHtmlEntities', () => { + it('decodes hex character references', () => { + expect(decodeHtmlEntities("'")).toBe("'"); + expect(decodeHtmlEntities(""")).toBe('"'); + expect(decodeHtmlEntities("IAB warns industry chasing 'shiny pennies'")).toBe( + "IAB warns industry chasing 'shiny pennies'" + ); + }); + + it('decodes decimal character references', () => { + expect(decodeHtmlEntities("'")).toBe("'"); + expect(decodeHtmlEntities(""")).toBe('"'); + }); + + it('decodes named entities', () => { + expect(decodeHtmlEntities("'")).toBe("'"); + expect(decodeHtmlEntities(""")).toBe('"'); + expect(decodeHtmlEntities("&")).toBe("&"); + expect(decodeHtmlEntities("<")).toBe("<"); + expect(decodeHtmlEntities(">")).toBe(">"); + expect(decodeHtmlEntities(" ")).toBe(" "); + }); + + it('decodes typographic entities', () => { + expect(decodeHtmlEntities("–")).toBe('\u2013'); + expect(decodeHtmlEntities("—")).toBe('\u2014'); + expect(decodeHtmlEntities("‘")).toBe('\u2018'); + expect(decodeHtmlEntities("’")).toBe('\u2019'); + expect(decodeHtmlEntities("“")).toBe('\u201C'); + expect(decodeHtmlEntities("”")).toBe('\u201D'); + expect(decodeHtmlEntities("…")).toBe('\u2026'); + }); + + it('handles mixed content', () => { + expect( + decodeHtmlEntities("Tom & Jerry said “Hello's World”") + ).toBe("Tom & Jerry said \u201CHello's World\u201D"); + }); + + it('handles null and empty strings', () => { + expect(decodeHtmlEntities('')).toBe(''); + // @ts-expect-error testing null handling + expect(decodeHtmlEntities(null)).toBe(null); + // @ts-expect-error testing undefined handling + expect(decodeHtmlEntities(undefined)).toBe(undefined); + }); + + it('preserves strings without entities', () => { + expect(decodeHtmlEntities("Hello World")).toBe("Hello World"); + expect(decodeHtmlEntities("No entities here!")).toBe("No entities here!"); + }); + + it('handles emoji and high code points', () => { + // Emoji encoded as decimal + expect(decodeHtmlEntities("😀")).toBe("😀"); + // Emoji encoded as hex + expect(decodeHtmlEntities("😀")).toBe("😀"); + }); +});