Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .changeset/eighty-sloths-feel.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
---
---
38 changes: 19 additions & 19 deletions server/src/addie/services/feed-fetcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ function validateRssContent(content: string, contentType: string): { valid: bool
}

interface FeedItem {
guid?: string | { _: string };
guid?: string | { _?: string; $?: Record<string, string> };
link?: string;
title?: string;
pubDate?: string;
Expand Down Expand Up @@ -98,12 +98,9 @@ function isRssFetchable(feedUrl: string | null): feedUrl is string {
async function fetchFeed(feed: IndustryFeed): Promise<RssArticleInput[]> {
// Skip email-only feeds (those with non-HTTP URLs like email://)
if (!isRssFetchable(feed.feed_url)) {
logger.debug({ feedId: feed.id, name: feed.name, url: feed.feed_url }, 'Skipping non-HTTP feed');
return [];
}

logger.debug({ feedId: feed.id, name: feed.name, url: feed.feed_url }, 'Fetching RSS feed');

// Pre-fetch content to validate it's actually RSS/XML before parsing
// This prevents cryptic XML parsing errors when sites return HTML
const response = await fetch(feed.feed_url, {
Expand Down Expand Up @@ -131,7 +128,6 @@ async function fetchFeed(feed: IndustryFeed): Promise<RssArticleInput[]> {
const parsed = await parser.parseString(content);

if (!parsed.items || parsed.items.length === 0) {
logger.warn({ feedId: feed.id }, 'Feed returned no items');
return [];
}

Expand All @@ -144,11 +140,18 @@ async function fetchFeed(feed: IndustryFeed): Promise<RssArticleInput[]> {
}

// Generate a stable GUID
// Handle RSS guids that may be objects with _ property (from XML attributes)
// Handle RSS guids that may be objects with _ property (text content) from XML parsing
// Some feeds have guid with only attributes ($) but no text content (_)
const rawGuid = item.guid;
const guid = typeof rawGuid === 'object' && rawGuid !== null && '_' in rawGuid
? rawGuid._
: (rawGuid || item.link);
let guid: string;
if (typeof rawGuid === 'string') {
guid = rawGuid;
} else if (typeof rawGuid === 'object' && rawGuid !== null && '_' in rawGuid && rawGuid._) {
guid = rawGuid._;
} else {
// Fall back to link if guid is missing or empty object
guid = item.link;
}

// Parse publication date
let publishedAt: Date | undefined;
Expand Down Expand Up @@ -183,7 +186,6 @@ async function fetchFeed(feed: IndustryFeed): Promise<RssArticleInput[]> {
});
}

logger.debug({ feedId: feed.id, articlesFound: articles.length }, 'Parsed RSS feed');
return articles;
}

Expand All @@ -198,12 +200,9 @@ export async function processFeedsToFetch(): Promise<{
const feeds = await getFeedsToFetch();

if (feeds.length === 0) {
logger.debug('No feeds need fetching');
return { feedsProcessed: 0, newPerspectives: 0, errors: 0 };
}

logger.debug({ feedCount: feeds.length }, 'Processing RSS feeds');

let totalNewPerspectives = 0;
let errorCount = 0;

Expand All @@ -212,10 +211,8 @@ export async function processFeedsToFetch(): Promise<{
const articles = await fetchFeed(feed);

if (articles.length > 0) {
// Create perspectives from RSS articles
const created = await createRssPerspectivesBatch(articles);
totalNewPerspectives += created;
logger.debug({ feedId: feed.id, created }, 'Created perspectives from RSS');
}

await updateFeedStatus(feed.id, true);
Expand All @@ -230,10 +227,13 @@ export async function processFeedsToFetch(): Promise<{
await new Promise(resolve => setTimeout(resolve, 500));
}

logger.info(
{ feedsProcessed: feeds.length, newPerspectives: totalNewPerspectives, errors: errorCount },
'Completed RSS feed processing'
);
// Only log when there's something notable
if (totalNewPerspectives > 0 || errorCount > 0) {
logger.info(
{ feedsProcessed: feeds.length, newPerspectives: totalNewPerspectives, errors: errorCount },
'RSS feed processing complete'
);
}

return {
feedsProcessed: feeds.length,
Expand Down
5 changes: 1 addition & 4 deletions server/src/db/industry-feeds-db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -243,10 +243,7 @@ function generateSlug(title: string, guid: string): string {
.replace(/^-|-$/g, '')
.substring(0, 80);

// Ensure guid is a string (defensive against RSS parser quirks)
if (typeof guid !== 'string') {
logger.warn({ guid, title }, 'generateSlug received non-string guid');
}
// Defensive: ensure guid is a string even if caller passes unexpected type
const guidStr = String(guid || '');

// Add a hash of the guid to ensure uniqueness
Expand Down