diff --git a/.changeset/spicy-doors-flow.md b/.changeset/spicy-doors-flow.md new file mode 100644 index 0000000000..a845151cc8 --- /dev/null +++ b/.changeset/spicy-doors-flow.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/server/src/addie/services/feed-fetcher.ts b/server/src/addie/services/feed-fetcher.ts index b66a193efd..ed346be034 100644 --- a/server/src/addie/services/feed-fetcher.ts +++ b/server/src/addie/services/feed-fetcher.ts @@ -41,6 +41,47 @@ const parser = new Parser({ }, }); +/** + * Validate that content is actually RSS/Atom XML, not HTML + * Some sites disable their RSS feeds and redirect to HTML pages + */ +function validateRssContent(content: string, contentType: string): { valid: boolean; error?: string } { + // Check content type header first + const isXmlContentType = + contentType.includes('xml') || contentType.includes('rss') || contentType.includes('atom'); + + // Check for HTML doctype or opening tags (indicates redirect to HTML page) + const trimmed = content.trim().toLowerCase(); + if (trimmed.startsWith(' { logger.debug({ feedId: feed.id, name: feed.name, url: feed.feed_url }, 'Fetching RSS feed'); - const parsed = await parser.parseURL(feed.feed_url); + // Pre-fetch content to validate it's actually RSS/XML before parsing + // This prevents cryptic XML parsing errors when sites return HTML + const response = await fetch(feed.feed_url, { + headers: { + 'User-Agent': 'AddieBot/1.0 (AgenticAdvertising.org industry monitor)', + Accept: 'application/rss+xml, application/xml, text/xml', + }, + signal: AbortSignal.timeout(30000), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const contentType = response.headers.get('content-type') || ''; + const content = await response.text(); + + // Validate content is RSS/XML, not HTML (e.g., from a redirect) + const validation = validateRssContent(content, contentType); + if (!validation.valid) { + throw new Error(validation.error); + } + + // Parse the validated content + const parsed = await parser.parseString(content); if (!parsed.items || parsed.items.length === 0) { logger.warn({ feedId: feed.id }, 'Feed returned no items'); diff --git a/server/src/db/industry-feeds-db.ts b/server/src/db/industry-feeds-db.ts index 3af6f9f34c..0ee2c4d99c 100644 --- a/server/src/db/industry-feeds-db.ts +++ b/server/src/db/industry-feeds-db.ts @@ -67,11 +67,13 @@ export interface RssPerspective { /** * Get all active feeds that need fetching + * Only returns feeds with a valid feed_url (excludes email-only feeds) */ export async function getFeedsToFetch(): Promise { const result = await query( `SELECT * FROM industry_feeds WHERE is_active = true + AND feed_url IS NOT NULL AND (last_fetched_at IS NULL OR last_fetched_at < NOW() - (fetch_interval_minutes || ' minutes')::interval) ORDER BY last_fetched_at ASC NULLS FIRST diff --git a/server/src/db/migrations/146_allow_null_feed_url.sql b/server/src/db/migrations/146_allow_null_feed_url.sql new file mode 100644 index 0000000000..1f1a8683f5 --- /dev/null +++ b/server/src/db/migrations/146_allow_null_feed_url.sql @@ -0,0 +1,19 @@ +-- Allow feed_url to be NULL for email-only feeds +-- Previously email-only feeds used 'email://slug' placeholder URLs, +-- but this is cleaner and matches how the code/UI expects to work + +-- Drop the NOT NULL constraint on feed_url +ALTER TABLE industry_feeds ALTER COLUMN feed_url DROP NOT NULL; + +-- Drop the UNIQUE constraint since NULL values should be allowed +-- (multiple email-only feeds would all have NULL feed_url) +ALTER TABLE industry_feeds DROP CONSTRAINT IF EXISTS industry_feeds_feed_url_key; + +-- Add a partial unique constraint that only applies to non-null URLs +CREATE UNIQUE INDEX IF NOT EXISTS industry_feeds_feed_url_unique +ON industry_feeds (feed_url) WHERE feed_url IS NOT NULL; + +-- Migrate existing email:// placeholder URLs to NULL +UPDATE industry_feeds +SET feed_url = NULL +WHERE feed_url LIKE 'email://%';