From 87ca627fee7b0fb306297f84354d0201ec799c99 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 21 Mar 2026 13:58:02 +0100 Subject: [PATCH 1/9] feat: default browserless onboarding and request strategies --- app/web/api/v1/root_metadata.rb | 21 +++- app/web/config/local_config.rb | 36 +++++- app/web/domain/auto_source.rb | 2 +- app/web/feeds/source_resolver.rb | 2 +- docker-compose.yml | 15 ++- frontend/src/__tests__/App.contract.test.tsx | 24 +++- frontend/src/__tests__/App.test.tsx | 113 +++++++++++++++++- frontend/src/__tests__/mocks/server.ts | 11 +- frontend/src/api/contracts.ts | 5 + frontend/src/components/App.tsx | 49 ++++++-- frontend/src/components/AppPanels.tsx | 73 +++++++++-- spec/html2rss/web/api/v1_spec.rb | 19 +++ .../web/feeds/source_resolver_spec.rb | 3 +- spec/html2rss/web/local_config_spec.rb | 31 +++++ 14 files changed, 355 insertions(+), 49 deletions(-) diff --git a/app/web/api/v1/root_metadata.rb b/app/web/api/v1/root_metadata.rb index 2d936b5b..dfdd503c 100644 --- a/app/web/api/v1/root_metadata.rb +++ b/app/web/api/v1/root_metadata.rb @@ -7,6 +7,24 @@ module V1 ## # Builds the public metadata payload for the API root endpoint. module RootMetadata + FEATURED_FEEDS = [ + { + path: '/microsoft.com/azure-products.rss', + title: 'Azure product updates', + description: 'Follow Microsoft Azure product announcements from your own instance.' + }, + { + path: '/phys.org/weekly.rss', + title: 'Top science news of the week', + description: 'Try a high-signal feed with stable weekly headlines from the built-in config set.' + }, + { + path: '/softwareleadweekly.com/issues.rss', + title: 'Software Lead Weekly issues', + description: 'Follow a long-running newsletter archive from the embedded config catalog.' + } + ].freeze + class << self # @param router [Roda::RodaRequest] # @return [Hash{Symbol=>Object}] @@ -30,7 +48,8 @@ def instance_payload(_router) feed_creation: { enabled: AutoSource.enabled?, access_token_required: AutoSource.enabled? - } + }, + featured_feeds: FEATURED_FEEDS } end end diff --git a/app/web/config/local_config.rb b/app/web/config/local_config.rb index 4b937d6c..4e7d0cea 100644 --- a/app/web/config/local_config.rb +++ b/app/web/config/local_config.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true require 'yaml' +begin + require 'html2rss/configs' +rescue LoadError + nil +end module Html2rss module Web @@ -17,6 +22,7 @@ class NotFound < RuntimeError; end # raised when the local config shape is invalid class InvalidConfig < RuntimeError; end FEED_EXTENSION_PATTERN = /\.(json|rss|xml)\z/ + EMBEDDED_FEED_NAME_PATTERN = %r{\A[^/]+/.+\z} # Path to local feed configuration file. CONFIG_FILE = 'config/feeds.yml' @@ -27,10 +33,8 @@ class << self # @return [Hash] def find(name) normalized_name = normalize_name(name) - config = snapshot.feeds.fetch(normalized_name.to_sym) do - raise NotFound, "Did not find local feed config at '#{normalized_name}'" - end - config_hash = deep_dup(config.raw) + config_hash = local_feed_config(normalized_name) || embedded_feed_config(normalized_name) + raise NotFound, "Did not find local feed config at '#{normalized_name}'" unless config_hash apply_global_defaults(config_hash) end @@ -76,6 +80,26 @@ def reload!(reason: 'manual') private + # @param normalized_name [String] + # @return [Hash{Symbol=>Object}, nil] + def local_feed_config(normalized_name) + config = snapshot.feeds[normalized_name.to_sym] + return nil unless config + + deep_dup(config.raw) + end + + # @param normalized_name [String] + # @return [Hash{Symbol=>Object}, nil] + def embedded_feed_config(normalized_name) + return nil unless defined?(Html2rss::Configs) + return nil unless normalized_name.match?(EMBEDDED_FEED_NAME_PATTERN) + + deep_dup(Html2rss::Configs.find_by_name(normalized_name)) + rescue Html2rss::Configs::ConfigNotFound + nil + end + # Applies global defaults only when feed-level keys are absent. # # @param config [Hash{Symbol=>Object}] @@ -90,9 +114,9 @@ def apply_global_defaults(config) end # @param name [String, Symbol, #to_s] - # @return [String] basename without extension for feed lookup. + # @return [String] path without feed extension for feed lookup. def normalize_name(name) - File.basename(name.to_s).sub(FEED_EXTENSION_PATTERN, '') + name.to_s.delete_prefix('/').sub(FEED_EXTENSION_PATTERN, '') end # Deep-duplicates nested config structures to avoid mutating shared data. diff --git a/app/web/domain/auto_source.rb b/app/web/domain/auto_source.rb index becd389e..cafb0429 100644 --- a/app/web/domain/auto_source.rb +++ b/app/web/domain/auto_source.rb @@ -21,7 +21,7 @@ def enabled? # @param token_data [Hash{Symbol=>Object}] authenticated account data. # @param strategy [String] # @return [Html2rss::Web::Api::V1::FeedMetadata::Metadata, nil] - def create_stable_feed(name, url, token_data, strategy = 'faraday') + def create_stable_feed(name, url, token_data, strategy = Html2rss::RequestService.default_strategy_name.to_s) return nil unless token_data && FeedAccess.url_allowed_for_username?(token_data[:username], url) feed_token = Auth.generate_feed_token(token_data[:username], url, strategy: strategy) diff --git a/app/web/feeds/source_resolver.rb b/app/web/feeds/source_resolver.rb index 75e1abf4..45ba3c29 100644 --- a/app/web/feeds/source_resolver.rb +++ b/app/web/feeds/source_resolver.rb @@ -69,7 +69,7 @@ def static_cache_identity(feed_name, params) def static_generator_input(config, params) generator_input = config.dup generator_input[:params] = merged_static_params(config, params) - generator_input[:strategy] ||= :faraday + generator_input[:strategy] ||= Html2rss::RequestService.default_strategy_name.to_sym generator_input end diff --git a/docker-compose.yml b/docker-compose.yml index c901f2c8..38273eaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,11 +7,9 @@ services: restart: unless-stopped ports: - "127.0.0.1:4000:4000" - volumes: - - type: bind - source: ./config/feeds.yml - target: /app/config/feeds.yml - read_only: true + env_file: + - path: .env + required: false environment: RACK_ENV: production PORT: 4000 @@ -19,6 +17,13 @@ services: HEALTH_CHECK_TOKEN: ${HEALTH_CHECK_TOKEN:?set HEALTH_CHECK_TOKEN} BROWSERLESS_IO_WEBSOCKET_URL: ws://browserless:4002 BROWSERLESS_IO_API_TOKEN: ${BROWSERLESS_IO_API_TOKEN:?set BROWSERLESS_IO_API_TOKEN} + # Trial runs use the image's bundled config/feeds.yml. + # Uncomment the block below when you want to replace it with your own file. + # volumes: + # - type: bind + # source: ./config/feeds.yml + # target: /app/config/feeds.yml + # read_only: true watchtower: image: containrrr/watchtower diff --git a/frontend/src/__tests__/App.contract.test.tsx b/frontend/src/__tests__/App.contract.test.tsx index b4f08a7a..87d906a6 100644 --- a/frontend/src/__tests__/App.contract.test.tsx +++ b/frontend/src/__tests__/App.contract.test.tsx @@ -18,7 +18,7 @@ describe('App contract', () => { http.post('/api/v1/feeds', async ({ request }) => { const body = (await request.json()) as { url: string; strategy: string }; - expect(body).toEqual({ url: 'https://example.com/articles', strategy: 'ssrf_filter' }); + expect(body).toEqual({ url: 'https://example.com/articles', strategy: 'browserless' }); expect(request.headers.get('authorization')).toBe(`Bearer ${token}`); return HttpResponse.json( @@ -35,7 +35,14 @@ describe('App contract', () => { return HttpResponse.json( { - items: [{ title: 'Contract Item' }], + items: [ + { + title: 'Contract Item', + content_text: 'Contract preview excerpt.', + url: 'https://example.com/contract-item', + date_published: '2024-01-01T00:00:00Z', + }, + ], }, { headers: { 'content-type': 'application/feed+json' }, @@ -47,6 +54,9 @@ describe('App contract', () => { render(); await screen.findByLabelText('Page URL'); + await waitFor(() => { + expect(screen.getByRole('combobox')).toHaveValue('browserless'); + }); const urlInput = screen.getByLabelText('Page URL') as HTMLInputElement; fireEvent.input(urlInput, { target: { value: 'https://example.com/articles' } }); @@ -54,16 +64,18 @@ describe('App contract', () => { fireEvent.click(screen.getByRole('button', { name: 'Generate feed URL' })); await waitFor(() => { + expect(screen.getByText('Your feed is ready')).toBeInTheDocument(); expect(screen.getByText('Example Feed')).toBeInTheDocument(); expect(screen.getByLabelText('Feed URL')).toBeInTheDocument(); expect(screen.getByRole('button', { name: 'Copy feed URL' })).toBeInTheDocument(); expect(screen.getByRole('link', { name: 'Open feed' })).toBeInTheDocument(); - expect(screen.getByRole('link', { name: 'JSON Feed' })).toHaveAttribute( + expect(screen.getByRole('link', { name: 'Open JSON Feed' })).toHaveAttribute( 'href', 'http://localhost:3000/api/v1/feeds/generated-token.json' ); expect(screen.getByRole('button', { name: 'Create another feed' })).toBeInTheDocument(); - expect(screen.getByText('Feed preview')).toBeInTheDocument(); + expect(screen.getByText('Preview')).toBeInTheDocument(); + expect(screen.getByText('Latest items from this feed')).toBeInTheDocument(); expect(screen.getByText('Contract Item')).toBeInTheDocument(); }); }); @@ -89,6 +101,7 @@ describe('App contract', () => { enabled: true, access_token_required: true, }, + featured_feeds: [], }, }, }); @@ -135,6 +148,9 @@ describe('App contract', () => { render(); await screen.findByLabelText('Page URL'); + await waitFor(() => { + expect(screen.getByRole('combobox')).toHaveValue('browserless'); + }); fireEvent.input(screen.getByLabelText('Page URL'), { target: { value: 'https://example.com/articles' }, diff --git a/frontend/src/__tests__/App.test.tsx b/frontend/src/__tests__/App.test.tsx index c5b44a7f..36f032aa 100644 --- a/frontend/src/__tests__/App.test.tsx +++ b/frontend/src/__tests__/App.test.tsx @@ -38,6 +38,7 @@ describe('App', () => { beforeEach(() => { vi.clearAllMocks(); + window.history.replaceState({}, '', 'http://localhost:3000/'); mockUseAccessToken.mockReturnValue({ token: null, @@ -60,6 +61,7 @@ describe('App', () => { enabled: true, access_token_required: true, }, + featured_feeds: [], }, }, isLoading: false, @@ -77,8 +79,8 @@ describe('App', () => { mockUseStrategies.mockReturnValue({ strategies: [ - { id: 'ssrf_filter', name: 'ssrf_filter', display_name: 'Standard (recommended)' }, - { id: 'browserless', name: 'browserless', display_name: 'JavaScript pages' }, + { id: 'faraday', name: 'faraday', display_name: 'Default' }, + { id: 'browserless', name: 'browserless', display_name: 'JavaScript pages (recommended)' }, ], isLoading: false, error: null, @@ -102,6 +104,50 @@ describe('App', () => { }); }); + it('prefers browserless as the default strategy when available', () => { + render(); + + return waitFor(() => { + expect(screen.getByRole('combobox')).toHaveValue('browserless'); + }); + }); + + it('falls back to the first available strategy when browserless is unavailable', () => { + mockUseStrategies.mockReturnValue({ + strategies: [{ id: 'faraday', name: 'faraday', display_name: 'Default' }], + isLoading: false, + error: null, + }); + + render(); + + return waitFor(() => { + expect(screen.getByRole('combobox')).toHaveValue('faraday'); + }); + }); + + it('auto-submits a prefilled url using the resolved default strategy', async () => { + mockUseAccessToken.mockReturnValue({ + token: 'saved-token', + hasToken: true, + saveToken: mockSaveToken, + clearToken: mockClearToken, + isLoading: false, + error: null, + }); + window.history.replaceState({}, '', 'http://localhost:3000/?url=https%3A%2F%2Fexample.com%2Farticles'); + + render(); + + await waitFor(() => { + expect(mockConvertFeed).toHaveBeenCalledWith( + 'https://example.com/articles', + 'browserless', + 'saved-token' + ); + }); + }); + it('shows inline token prompt when submitting without a token', async () => { render(); @@ -123,14 +169,55 @@ describe('App', () => { expect(mockConvertFeed).not.toHaveBeenCalled(); }); + it('promotes included feeds when feed creation is disabled', () => { + mockUseApiMetadata.mockReturnValue({ + metadata: { + api: { + name: 'html2rss-web API', + description: 'RESTful API for converting websites to RSS feeds', + openapi_url: 'http://example.test/openapi.yaml', + }, + instance: { + feed_creation: { + enabled: false, + access_token_required: false, + }, + featured_feeds: [ + { + path: '/microsoft.com/azure-products.rss', + title: 'Azure product updates', + description: 'Follow Microsoft Azure product announcements from your own instance.', + }, + ], + }, + }, + isLoading: false, + error: null, + }); + + render(); + + expect(screen.getByText('Try a working included feed')).toBeInTheDocument(); + expect(screen.getByRole('link', { name: 'Azure product updates' })).toHaveAttribute( + 'href', + '/microsoft.com/azure-products.rss' + ); + expect(screen.getByText('Custom feed generation is disabled for this instance.')).toBeInTheDocument(); + }); + it('renders the result panel when a feed is available', async () => { + vi.spyOn(window, 'fetch').mockResolvedValue({ + ok: true, + json: async () => ({ items: [] }), + } as Response); + mockUseFeedConversion.mockReturnValue({ isConverting: false, result: { id: 'feed-123', name: 'Example Feed', url: 'https://example.com/articles', - strategy: 'ssrf_filter', + strategy: 'faraday', feed_token: 'example-token', public_url: '/api/v1/feeds/example-token', json_public_url: '/api/v1/feeds/example-token.json', @@ -196,7 +283,7 @@ describe('App', () => { expect(mockSaveToken).toHaveBeenCalledWith('token-123'); expect(mockConvertFeed).toHaveBeenCalledWith( 'https://example.com/articles', - 'ssrf_filter', + 'browserless', 'token-123' ); }); @@ -281,4 +368,22 @@ describe('App', () => { expect(bookmarklet.getAttribute('href')).toContain('/?url='); expect(bookmarklet.getAttribute('href')).not.toContain('%27+encodeURIComponent'); }); + + it('shows the utility links in a user-focused order', () => { + render(); + + fireEvent.click(screen.getByRole('button', { name: 'More' })); + + const utilityLinks = screen.getAllByRole('link').map((link) => link.textContent); + expect(utilityLinks).toEqual(['Try included feeds', 'Bookmarklet', 'OpenAPI spec', 'Source code']); + + expect(screen.getByRole('link', { name: 'OpenAPI spec' })).toHaveAttribute( + 'href', + 'http://example.test/openapi.yaml' + ); + expect(screen.getByRole('link', { name: 'Try included feeds' })).toHaveAttribute( + 'href', + 'https://html2rss.github.io/web-application/how-to/use-included-configs/' + ); + }); }); diff --git a/frontend/src/__tests__/mocks/server.ts b/frontend/src/__tests__/mocks/server.ts index 60098f03..00359bc6 100644 --- a/frontend/src/__tests__/mocks/server.ts +++ b/frontend/src/__tests__/mocks/server.ts @@ -16,6 +16,7 @@ export const server = setupServer( enabled: true, access_token_required: true, }, + featured_feeds: [], }, }, }); @@ -26,14 +27,14 @@ export const server = setupServer( data: { strategies: [ { - id: 'ssrf_filter', - name: 'ssrf_filter', - display_name: 'Standard (recommended)', + id: 'faraday', + name: 'faraday', + display_name: 'Default', }, { id: 'browserless', name: 'browserless', - display_name: 'JavaScript pages', + display_name: 'JavaScript pages (recommended)', }, ], }, @@ -64,7 +65,7 @@ export function buildFeedResponse(overrides: FeedResponseOverrides = {}) { id: overrides.id ?? 'feed-123', name: overrides.name ?? 'Example Feed', url: overrides.url ?? 'https://example.com/articles', - strategy: overrides.strategy ?? 'ssrf_filter', + strategy: overrides.strategy ?? 'faraday', feed_token: overrides.feed_token ?? 'example-token', public_url: overrides.public_url ?? '/api/v1/feeds/example-token', json_public_url: overrides.json_public_url ?? '/api/v1/feeds/example-token.json', diff --git a/frontend/src/api/contracts.ts b/frontend/src/api/contracts.ts index 57d0a54b..14867b84 100644 --- a/frontend/src/api/contracts.ts +++ b/frontend/src/api/contracts.ts @@ -10,5 +10,10 @@ export interface ApiMetadataRecord { enabled: boolean; access_token_required: boolean; }; + featured_feeds?: Array<{ + path: string; + title: string; + description: string; + }>; }; } diff --git a/frontend/src/components/App.tsx b/frontend/src/components/App.tsx index 02ccdd75..8fa7d018 100644 --- a/frontend/src/components/App.tsx +++ b/frontend/src/components/App.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'preact/hooks'; +import { useEffect, useRef, useState } from 'preact/hooks'; import { ResultDisplay } from './ResultDisplay'; import { CreateFeedPanel, UtilityStrip, type Strategy } from './AppPanels'; import { useAccessToken } from '../hooks/useAccessToken'; @@ -8,6 +8,8 @@ import { useStrategies } from '../hooks/useStrategies'; const EMPTY_FEED_ERRORS = { url: '', form: '' }; const DEFAULT_FEED_CREATION = { enabled: true, access_token_required: true }; +const preferredStrategy = (strategies: { id: string }[]) => + strategies.find((strategy) => strategy.id === 'browserless')?.id ?? strategies[0]?.id; function BrandLockup() { return ( @@ -42,25 +44,29 @@ export function App() { } = useFeedConversion(); const { strategies, isLoading: strategiesLoading, error: strategiesError } = useStrategies(); - const [feedFormData, setFeedFormData] = useState({ url: '', strategy: 'ssrf_filter' }); + const [feedFormData, setFeedFormData] = useState({ url: '', strategy: '' }); const [feedFieldErrors, setFeedFieldErrors] = useState(EMPTY_FEED_ERRORS); const [showTokenPrompt, setShowTokenPrompt] = useState(false); const [tokenDraft, setTokenDraft] = useState(''); const [tokenError, setTokenError] = useState(''); const [focusCreateComposerKey, setFocusCreateComposerKey] = useState(0); + const autoSubmitUrlRef = useRef(null); + const hasAutoSubmittedRef = useRef(false); + const selectedStrategy = feedFormData.strategy || preferredStrategy(strategies) || ''; useEffect(() => { if (typeof window === 'undefined') return; - if (feedFormData.url) return; const urlParam = new URLSearchParams(window.location.search).get('url'); if (!urlParam) return; + autoSubmitUrlRef.current = urlParam; + if (feedFormData.url) return; setFeedFormData((prev) => ({ ...prev, url: urlParam })); }, [feedFormData.url]); useEffect(() => { - const nextStrategy = strategies[0]?.id; + const nextStrategy = preferredStrategy(strategies); if (!nextStrategy) return; const hasCurrentStrategy = strategies.some((strategy) => strategy.id === feedFormData.strategy); @@ -68,6 +74,8 @@ export function App() { }, [strategies, feedFormData.strategy]); const feedCreation = metadata?.instance.feed_creation ?? DEFAULT_FEED_CREATION; + const featuredFeeds = metadata?.instance.featured_feeds ?? []; + const submitDisabled = isConverting || strategiesLoading || !feedCreation.enabled || showTokenPrompt; const setFeedField = (key: 'url' | 'strategy', value: string) => { setFeedFormData((prev) => ({ ...prev, [key]: value })); @@ -80,7 +88,7 @@ export function App() { }; const strategyHint = (strategy: Strategy) => { - if (strategy.id === 'ssrf_filter') return 'Start here for most pages.'; + if (strategy.id === 'faraday') return 'Start here for most pages.'; if (strategy.id === 'browserless') return 'Use this if the page loads content with JavaScript.'; return strategy.name; }; @@ -96,11 +104,18 @@ export function App() { }; const attemptFeedCreation = async (accessToken: string) => { + const strategy = selectedStrategy; + if (!feedFormData.url.trim()) { setFeedFieldErrors({ ...EMPTY_FEED_ERRORS, url: 'Source URL is required.' }); return false; } + if (!strategy) { + setFeedFieldErrors({ ...EMPTY_FEED_ERRORS, form: 'Strategy is required' }); + return false; + } + if (!feedCreation.enabled) { setFeedFieldErrors({ ...EMPTY_FEED_ERRORS, @@ -117,7 +132,7 @@ export function App() { } try { - await convertFeed(feedFormData.url, feedFormData.strategy, accessToken); + await convertFeed(feedFormData.url, strategy, accessToken); setShowTokenPrompt(false); setTokenError(''); return true; @@ -166,12 +181,23 @@ export function App() { setFocusCreateComposerKey((current) => current + 1); }; + useEffect(() => { + const autoSubmitUrl = autoSubmitUrlRef.current; + if (!autoSubmitUrl || hasAutoSubmittedRef.current) return; + if (strategiesLoading || metadataLoading || tokenLoading) return; + if (feedFormData.url !== autoSubmitUrl || !selectedStrategy) return; + + hasAutoSubmittedRef.current = true; + setFeedFieldErrors(EMPTY_FEED_ERRORS); + void attemptFeedCreation(token ?? ''); + }, [feedFormData.url, metadataLoading, selectedStrategy, strategiesLoading, token, tokenLoading]); + if (metadataLoading || tokenLoading) { return (
-
-