diff --git a/Gemfile b/Gemfile index 41fda845..ff4d4b8a 100644 --- a/Gemfile +++ b/Gemfile @@ -4,14 +4,15 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } -gem 'html2rss', '~> 0.18' -# gem 'html2rss', github: 'html2rss/html2rss', branch: :master +# gem 'html2rss', '~> 0.18' +gem 'html2rss', github: 'html2rss/html2rss', branch: 'codex-pr-auto-fallback-pipeline' gem 'html2rss-configs', github: 'html2rss/html2rss-configs' # Use these instead of the two above (uncomment them) when developing locally: # gem 'html2rss', path: '../html2rss' # gem 'html2rss-configs', path: '../html2rss-configs' +gem 'concurrent-ruby' gem 'parallel' gem 'rack-cache' gem 'rack-timeout' @@ -21,7 +22,6 @@ gem 'zeitwerk' gem 'puma', require: false group :development do - gem 'byebug' gem 'irb', require: false gem 'rake', require: false gem 'rubocop', require: false diff --git a/Gemfile.lock b/Gemfile.lock index c4cb04a0..b03355e7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,28 @@ +GIT + remote: https://github.com/html2rss/html2rss + revision: 54f6f53482352932b69621f1df989f83cd285c72 + branch: codex-pr-auto-fallback-pipeline + specs: + html2rss (0.18.0) + addressable (~> 2.7) + brotli + dry-validation + faraday (> 2.0.1, < 3.0) + faraday-follow_redirects + faraday-gzip (~> 3) + kramdown + mime-types (> 3.0) + nokogiri (>= 1.10, < 2.0) + parallel + puppeteer-ruby + regexp_parser + reverse_markdown (~> 3.0) + rss + sanitize + thor + tzinfo + zeitwerk + GIT remote: https://github.com/html2rss/html2rss-configs revision: 71981aff28d88e4553c206d78bf54d5633bcdd19 @@ -68,8 +93,6 @@ GEM bigdecimal (4.1.2) brotli (0.8.0) builder (3.3.0) - byebug (13.0.0) - reline (>= 0.6.0) climate_control (1.2.0) concurrent-ruby (1.3.6) connection_pool (3.0.2) @@ -138,25 +161,6 @@ GEM fiber-storage fiber-storage (1.0.1) hashdiff (1.2.1) - html2rss (0.18.0) - addressable (~> 2.7) - brotli - dry-validation - faraday (> 2.0.1, < 3.0) - faraday-follow_redirects - faraday-gzip (~> 3) - kramdown - mime-types (> 3.0) - nokogiri (>= 1.10, < 2.0) - parallel - puppeteer-ruby - regexp_parser - reverse_markdown (~> 3.0) - rss - sanitize - thor - tzinfo - zeitwerk i18n (1.14.8) concurrent-ruby (~> 1.0) io-console (0.8.2) @@ -371,9 +375,9 @@ PLATFORMS x86_64-linux-musl DEPENDENCIES - byebug climate_control - html2rss (~> 0.18) + concurrent-ruby + html2rss! html2rss-configs! irb parallel @@ -413,7 +417,6 @@ CHECKSUMS bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd brotli (0.8.0) sha256=0c5a42046b3b603fb109656881147fd76064c034b7d19c1b4fcc32a093a4d55d builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f - byebug (13.0.0) sha256=d2263efe751941ca520fa29744b71972d39cbc41839496706f5d9b22e92ae05d climate_control (1.2.0) sha256=36b21896193fa8c8536fa1cd843a07cf8ddbd03aaba43665e26c53ec1bd70aa5 concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a @@ -442,7 +445,7 @@ CHECKSUMS fiber-local (1.1.0) sha256=c885f94f210fb9b05737de65d511136ea602e00c5105953748aa0f8793489f06 fiber-storage (1.0.1) sha256=f48e5b6d8b0be96dac486332b55cee82240057065dc761c1ea692b2e719240e1 hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 - html2rss (0.18.0) sha256=83e9dd0388d65f1992df4afc9d345cbfd84e8c740be3756815b5e840ac71cf54 + html2rss (0.18.0) html2rss-configs (0.2.0) i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5 io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc diff --git a/app.rb b/app.rb index 6115126c..b6ddce24 100644 --- a/app.rb +++ b/app.rb @@ -13,6 +13,21 @@ module Html2rss module Web + DEFAULT_HEADERS = { + 'X-Content-Type-Options' => 'nosniff', + 'X-XSS-Protection' => '1; mode=block', + 'X-Frame-Options' => 'SAMEORIGIN', + 'X-Permitted-Cross-Domain-Policies' => 'none', + 'Referrer-Policy' => 'strict-origin-when-cross-origin', + 'Permissions-Policy' => 'geolocation=(), microphone=(), camera=()', + 'Strict-Transport-Security' => 'max-age=31536000; includeSubDomains; preload', + 'Cross-Origin-Embedder-Policy' => 'require-corp', + 'Cross-Origin-Opener-Policy' => 'same-origin', + 'Cross-Origin-Resource-Policy' => 'same-origin', + 'X-DNS-Prefetch-Control' => 'off', + 'X-Download-Options' => 'noopen' + }.freeze + ## # Roda app serving RSS feeds via html2rss class App < Roda @@ -32,7 +47,8 @@ class App < Roda HTML FRONTEND_DIST_PATH = 'frontend/dist' - FRONTEND_INDEX_PATH = File.join(FRONTEND_DIST_PATH, 'index.html') + FRONTEND_DIST_INDEX_PATH = File.join(FRONTEND_DIST_PATH, 'index.html') + FRONTEND_SOURCE_INDEX_PATH = 'frontend/index.html' def self.development? = EnvironmentValidator.development? def development? = self.class.development? @@ -43,7 +59,7 @@ def development? = self.class.development? plugin :content_security_policy do |csp| csp.default_src :none - csp.style_src :self, "'unsafe-inline'" + csp.style_src :self csp.script_src :self csp.connect_src :self csp.img_src :self @@ -65,21 +81,7 @@ def development? = self.class.development? csp.block_all_mixed_content csp.upgrade_insecure_requests end - - plugin :default_headers, { - 'X-Content-Type-Options' => 'nosniff', - 'X-XSS-Protection' => '1; mode=block', - 'X-Frame-Options' => 'SAMEORIGIN', - 'X-Permitted-Cross-Domain-Policies' => 'none', - 'Referrer-Policy' => 'strict-origin-when-cross-origin', - 'Permissions-Policy' => 'geolocation=(), microphone=(), camera=()', - 'Strict-Transport-Security' => 'max-age=31536000; includeSubDomains; preload', - 'Cross-Origin-Embedder-Policy' => 'require-corp', - 'Cross-Origin-Opener-Policy' => 'same-origin', - 'Cross-Origin-Resource-Policy' => 'same-origin', - 'X-DNS-Prefetch-Control' => 'off', - 'X-Download-Options' => 'noopen' - } + plugin :default_headers, DEFAULT_HEADERS plugin :json_parser plugin :static, @@ -91,7 +93,7 @@ def development? = self.class.development? plugin :not_allowed plugin :exception_page plugin :error_handler do |error| - next exception_page(error) if development? + next exception_page(error) if development? && !error.is_a?(HttpError) ErrorResponder.respond(request: request, response: response, error: error) end @@ -107,7 +109,16 @@ def development? = self.class.development? def render_index_page(router) router.response['Content-Type'] = 'text/html' - File.exist?(FRONTEND_INDEX_PATH) ? File.read(FRONTEND_INDEX_PATH) : FALLBACK_HTML + index_path = index_page_path + return File.read(index_path) if index_path + + FALLBACK_HTML + end + + def index_page_path + return FRONTEND_DIST_INDEX_PATH if File.exist?(FRONTEND_DIST_INDEX_PATH) + + FRONTEND_SOURCE_INDEX_PATH if development? && File.exist?(FRONTEND_SOURCE_INDEX_PATH) end end end diff --git a/app/web/api/v1/contract.rb b/app/web/api/v1/contract.rb deleted file mode 100644 index bc67beb8..00000000 --- a/app/web/api/v1/contract.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Api - module V1 - module Contract - CODES = { - unauthorized: Html2rss::Web::UnauthorizedError::CODE, - forbidden: Html2rss::Web::ForbiddenError::CODE, - internal_server_error: Html2rss::Web::InternalServerError::CODE - }.freeze - - MESSAGES = { - auto_source_disabled: 'Auto source feature is disabled', - health_check_failed: 'Health check failed' - }.freeze - end - end - end - end -end diff --git a/app/web/api/v1/create_feed.rb b/app/web/api/v1/create_feed.rb index ec96a76a..feb9b702 100644 --- a/app/web/api/v1/create_feed.rb +++ b/app/web/api/v1/create_feed.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -require 'time' require 'json' +require 'time' module Html2rss module Web @@ -11,33 +11,36 @@ module V1 # Creates stable feed records from authenticated API requests. module CreateFeed # rubocop:disable Metrics/ModuleLength FEED_ATTRIBUTE_KEYS = - %i[id name url strategy feed_token public_url json_public_url created_at updated_at].freeze - class << self # rubocop:disable Metrics/ClassLength + %i[id name url feed_token public_url json_public_url created_at updated_at].freeze + FEED_METADATA_KEYS = + %i[id name url username feed_token public_url json_public_url].freeze + + class << self # Creates a feed and returns a normalized API success payload. # # @param request [Rack::Request] HTTP request with auth context. # @return [Hash{Symbol=>Object}] API response payload. + # rubocop:disable Metrics/MethodLength def call(request) - params, feed_data = build_feed_from_request(request) + account = require_account(request) + params = build_create_params(request, account) + feed_data = create_feed(params, account) + emit_create_success(params) Response.success(response: request.response, status: 201, - data: { feed: feed_attributes(feed_data) }, + data: { + feed: feed_attributes(feed_data) + }, meta: { created: true }) rescue StandardError => error emit_create_failure(error) raise end + # rubocop:enable Metrics/MethodLength private - # @return [void] - def ensure_auto_source_enabled! - raise Html2rss::Web::ForbiddenError, Contract::MESSAGES[:auto_source_disabled] unless AutoSource.enabled? - end - - # @param request [Rack::Request] - # @return [Hash] def require_account(request) account = Auth.authenticate(request) raise Html2rss::Web::UnauthorizedError, 'Authentication required' unless account @@ -45,16 +48,32 @@ def require_account(request) account end - # @param params [Hash] - # @param account [Hash] - # @return [Html2rss::Web::Api::V1::FeedMetadata::CreateParams] - def build_create_params(params, account) - url = validated_url(params['url'], account) - FeedMetadata::CreateParams.new( - url: url, - name: FeedMetadata.site_title_for(url), - strategy: normalize_strategy(params['strategy']) - ) + def build_create_params(request, account) + url = validated_url(request_params(request)['url'], account) + FeedMetadata::CreateParams.new(url:, name: FeedMetadata.site_title_for(url)) + end + + def request_params(request) + return request.params unless json_request?(request) + + request.GET.merge(parsed_json_body(request)) + end + + def parsed_json_body(request) + raw_body = request.body.read + request.body.rewind + return {} if raw_body.strip.empty? + + parsed = JSON.parse(raw_body) + raise Html2rss::Web::BadRequestError, 'Invalid JSON payload' unless parsed.is_a?(Hash) + + parsed + rescue JSON::ParserError + raise Html2rss::Web::BadRequestError, 'Invalid JSON payload' + end + + def json_request?(request) + request.env['CONTENT_TYPE'].to_s.include?('application/json') end # @param raw_url [String, nil] @@ -101,76 +120,29 @@ def hostname_input?(url) }ix.match?(url) end - # @param raw_strategy [String, nil] - # @return [String] - def normalize_strategy(raw_strategy) - strategy = raw_strategy.to_s.strip - strategy = default_strategy if strategy.empty? - - raise Html2rss::Web::BadRequestError, 'Unsupported strategy' unless supported_strategy?(strategy) - - strategy - end + # @param params [Html2rss::Web::Api::V1::FeedMetadata::CreateParams] + # @param account [Hash] + # @return [Html2rss::Web::Api::V1::FeedMetadata::Metadata] + def create_feed(params, account) + raise Html2rss::Web::AutoSourceDisabledError unless AutoSource.enabled? - # @return [Array] supported strategy identifiers. - def supported_strategies - Html2rss::RequestService.strategy_names.map(&:to_s) - end + feed_data = AutoSource.create_stable_feed(params.name, params.url, account) + raise Html2rss::Web::InternalServerError, 'Failed to create feed' unless feed_data - # @param strategy [String] - # @return [Boolean] - def supported_strategy?(strategy) - supported_strategies.include?(strategy) + feed_data.is_a?(FeedMetadata::Metadata) ? feed_data : feed_metadata(feed_data) end - # @return [String] default strategy identifier. - def default_strategy - Html2rss::RequestService.default_strategy_name.to_s + # @param feed_data [Hash] + # @return [Html2rss::Web::Api::V1::FeedMetadata::Metadata] + def feed_metadata(feed_data) + FeedMetadata::Metadata.new(**feed_data.slice(*FEED_METADATA_KEYS)) end - # @param feed_data [Hash, Html2rss::Web::Api::V1::FeedMetadata::Metadata] + # @param feed_data [Html2rss::Web::Api::V1::FeedMetadata::Metadata] # @return [Hash{Symbol=>Object}] def feed_attributes(feed_data) timestamp = Time.now.iso8601 - typed_feed = feed_metadata(feed_data) - typed_feed_attributes(typed_feed, timestamp).slice(*FEED_ATTRIBUTE_KEYS) - end - - # @param request [Rack::Request] - # @return [Hash] - def request_params(request) - return request.params unless json_request?(request) - - raw_body = request.body.read - request.body.rewind - return request.params if raw_body.strip.empty? - - parsed = JSON.parse(raw_body) - raise Html2rss::Web::BadRequestError, 'Invalid JSON payload' unless parsed.is_a?(Hash) - - request.params.merge(parsed) - rescue JSON::ParserError - raise Html2rss::Web::BadRequestError, 'Invalid JSON payload' - end - - # @param request [Rack::Request] - # @return [Boolean] - def json_request?(request) - content_type = request.env['CONTENT_TYPE'].to_s - content_type.include?('application/json') - end - - # @param request [Rack::Request] - # @return [Array<(Html2rss::Web::Api::V1::FeedMetadata::CreateParams, Object)>] - def build_feed_from_request(request) - account = require_account(request) - ensure_auto_source_enabled! - params = build_create_params(request_params(request), account) - - feed_data = AutoSource.create_stable_feed(params.name, params.url, account, params.strategy) - raise Html2rss::Web::InternalServerError, 'Failed to create feed' unless feed_data - - [params, feed_data] + feed_data.to_h.merge(created_at: timestamp, updated_at: timestamp).slice(*FEED_ATTRIBUTE_KEYS) end # @param params [Html2rss::Web::Api::V1::FeedMetadata::CreateParams] @@ -179,7 +151,7 @@ def emit_create_success(params) Observability.emit( event_name: 'feed.create', outcome: 'success', - details: { strategy: params.strategy, url: params.url }, + details: { url: params.url }, level: :info ) end @@ -194,21 +166,6 @@ def emit_create_failure(error) level: :warn ) end - - # @param feed_data [Hash, Html2rss::Web::Api::V1::FeedMetadata::Metadata] - # @return [Html2rss::Web::Api::V1::FeedMetadata::Metadata] - def feed_metadata(feed_data) - return feed_data if feed_data.is_a?(FeedMetadata::Metadata) - - FeedMetadata::Metadata.new(**feed_data) - end - - # @param typed_feed [Html2rss::Web::Api::V1::FeedMetadata::Metadata] - # @param timestamp [String] - # @return [Hash{Symbol=>Object}] - def typed_feed_attributes(typed_feed, timestamp) - typed_feed.to_h.merge(created_at: timestamp, updated_at: timestamp) - end end end end diff --git a/app/web/api/v1/feed_metadata.rb b/app/web/api/v1/feed_metadata.rb index dde42b33..c4bdd2e1 100644 --- a/app/web/api/v1/feed_metadata.rb +++ b/app/web/api/v1/feed_metadata.rb @@ -34,7 +34,6 @@ def metadata_attributes(attributes) name: attributes[:name], url: attributes[:url], username: attributes[:username], - strategy: attributes[:strategy], feed_token: attributes[:feed_token], public_url: public_url(attributes[:feed_token]), json_public_url: json_public_url(attributes[:feed_token]) @@ -64,16 +63,16 @@ def json_public_url(feed_token) ## # Feed create parameters contract. - CreateParams = Data.define(:url, :name, :strategy) do + CreateParams = Data.define(:url, :name) do # @return [Hash{Symbol=>Object}] def to_h - { url: url, name: name, strategy: strategy } + { url: url, name: name } end end ## # Feed metadata contract used between creation services and API responses. - Metadata = Data.define(:id, :name, :url, :username, :strategy, :feed_token, :public_url, :json_public_url) do + Metadata = Data.define(:id, :name, :url, :username, :feed_token, :public_url, :json_public_url) do # @return [Hash{Symbol=>Object}] def to_h { @@ -81,7 +80,6 @@ def to_h name: name, url: url, username: username, - strategy: strategy, feed_token: feed_token, public_url: public_url, json_public_url: json_public_url diff --git a/app/web/api/v1/health.rb b/app/web/api/v1/health.rb index 41299803..81f889ce 100644 --- a/app/web/api/v1/health.rb +++ b/app/web/api/v1/health.rb @@ -96,7 +96,7 @@ def bearer_token(request) def verify_configuration! LocalConfig.yaml rescue StandardError - raise Html2rss::Web::InternalServerError, Contract::MESSAGES[:health_check_failed] + raise Html2rss::Web::HealthCheckFailedError end end end diff --git a/app/web/boot.rb b/app/web/boot.rb index 93cd1871..cdf71111 100644 --- a/app/web/boot.rb +++ b/app/web/boot.rb @@ -7,29 +7,33 @@ module Web ## # Boot helpers for code loading and runtime setup. module Boot + @mutex = Mutex.new + @setup = false + @loader = nil + class << self # @param reloadable [Boolean] # @return [Zeitwerk::Loader] def setup!(reloadable: false) - return loader if setup? + @mutex.synchronize do + return @loader if @setup - loader.enable_reloading if reloadable - loader.setup - @setup = true # rubocop:disable ThreadSafety/ClassInstanceVariable - loader + @loader ||= build_loader + @loader.enable_reloading if reloadable + @loader.setup + @setup = true + @loader + end end # @return [Zeitwerk::Loader] def loader - @loader ||= build_loader # rubocop:disable ThreadSafety/ClassInstanceVariable + @mutex.synchronize { @loader ||= build_loader } end # @return [Boolean] def setup? - # Loader setup happens once during process boot. - # rubocop:disable ThreadSafety/ClassInstanceVariable - @setup == true - # rubocop:enable ThreadSafety/ClassInstanceVariable + @mutex.synchronize { @setup == true } end # @return [void] diff --git a/app/web/boot/sentry.rb b/app/web/boot/sentry.rb index e69dc82b..a07c5150 100644 --- a/app/web/boot/sentry.rb +++ b/app/web/boot/sentry.rb @@ -28,6 +28,7 @@ def initialize_sentry! ::Sentry.init do |config| apply_settings(config) end + apply_scope_tags! end # @param config [Object] @@ -45,6 +46,20 @@ def release_name "#{RuntimeEnv.build_tag}+#{RuntimeEnv.git_sha}" end + # @return [void] + def apply_scope_tags! + return unless defined?(::Sentry) && ::Sentry.respond_to?(:configure_scope) + + ::Sentry.configure_scope do |scope| + scope.set_tags( + release: release_name, + environment: RuntimeEnv.rack_env + ) + end + rescue StandardError + nil + end + # @return [Boolean] def sentry_initialized? defined?(::Sentry) && ::Sentry.respond_to?(:initialized?) && ::Sentry.initialized? diff --git a/app/web/config/local_config.rb b/app/web/config/local_config.rb index 9d2b60bc..8810ccb5 100644 --- a/app/web/config/local_config.rb +++ b/app/web/config/local_config.rb @@ -16,6 +16,9 @@ module Web # Keeping lookup/defaulting here gives the rest of the app one predictable # config shape instead of repeating file parsing and fallback logic. module LocalConfig + @mutex = Mutex.new + @snapshot = nil + ## # raised when the local config wasn't found class NotFound < RuntimeError; end @@ -63,9 +66,7 @@ def yaml ## # @return [Html2rss::Web::ConfigSnapshot::Snapshot] def snapshot - return @snapshot if @snapshot # rubocop:disable ThreadSafety/ClassInstanceVariable - - @snapshot = ConfigSnapshot.load(yaml) # rubocop:disable ThreadSafety/ClassInstanceVariable + @mutex.synchronize { @snapshot ||= ConfigSnapshot.load(yaml) } rescue KeyError, TypeError, ArgumentError => error raise InvalidConfig, "Invalid local config: #{error.message}" end @@ -74,7 +75,7 @@ def snapshot # @param reason [String] # @return [nil] def reload!(reason: 'manual') - @snapshot = nil # rubocop:disable ThreadSafety/ClassInstanceVariable + @mutex.synchronize { @snapshot = nil } SecurityLogger.log_cache_lifecycle('local_config', 'reload', reason: reason) nil end diff --git a/app/web/config/runtime_env.rb b/app/web/config/runtime_env.rb index 0760ad83..6a2c6d5c 100644 --- a/app/web/config/runtime_env.rb +++ b/app/web/config/runtime_env.rb @@ -8,18 +8,20 @@ module Web module RuntimeEnv SENSITIVE_KEYS = %w[HTML2RSS_SECRET_KEY HEALTH_CHECK_TOKEN SENTRY_DSN].freeze BOOT_METADATA_KEYS = %w[BUILD_TAG GIT_SHA RACK_ENV SENTRY_ENABLE_LOGS].freeze + @mutex = Mutex.new + @values = nil class << self # @return [void] def capture! - @values = tracked_env_values.freeze # rubocop:disable ThreadSafety/ClassInstanceVariable + @mutex.synchronize { @values = tracked_env_values.freeze } scrub_sensitive_env! nil end # @return [void] def reset! - @values = nil # rubocop:disable ThreadSafety/ClassInstanceVariable + @mutex.synchronize { @values = nil } end # @return [String] @@ -70,8 +72,8 @@ def rack_env def fetch(key, default = :__missing__) return ENV.fetch(key) if ENV.key?(key) - values = @values || {} # rubocop:disable ThreadSafety/ClassInstanceVariable - return values.fetch(key) if values.key?(key) + current_values = @mutex.synchronize { @values || {} } + return current_values.fetch(key) if current_values.key?(key) return default unless default == :__missing__ raise KeyError, "key not found: #{key}" diff --git a/app/web/domain/auto_source.rb b/app/web/domain/auto_source.rb index cafb0429..881c7d73 100644 --- a/app/web/domain/auto_source.rb +++ b/app/web/domain/auto_source.rb @@ -19,10 +19,11 @@ def enabled? # @param name [String, nil] # @param url [String] # @param token_data [Hash{Symbol=>Object}] authenticated account data. - # @param strategy [String] + # @param strategy [String, nil] # @return [Html2rss::Web::Api::V1::FeedMetadata::Metadata, nil] - def create_stable_feed(name, url, token_data, strategy = Html2rss::RequestService.default_strategy_name.to_s) - return nil unless token_data && FeedAccess.url_allowed_for_username?(token_data[:username], url) + def create_stable_feed(name, url, token_data, strategy = nil) + account = AccountManager.get_account_by_username(token_data&.dig(:username)) + return nil unless account && UrlValidator.url_allowed?(account, url) feed_token = Auth.generate_feed_token(token_data[:username], url, strategy: strategy) return nil unless feed_token @@ -35,7 +36,7 @@ def create_stable_feed(name, url, token_data, strategy = Html2rss::RequestServic # @param name [String, nil] # @param url [String] # @param token_data [Hash{Symbol=>Object}] - # @param strategy [String] + # @param strategy [String, nil] # @param feed_token [String] # @return [Hash{Symbol=>Object}] def metadata_attributes(name, url, token_data, strategy, feed_token) diff --git a/app/web/errors/auto_source_disabled_error.rb b/app/web/errors/auto_source_disabled_error.rb new file mode 100644 index 00000000..09331fb5 --- /dev/null +++ b/app/web/errors/auto_source_disabled_error.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Html2rss + module Web + ## + # HTTP 403 error raised when feed creation is disabled by instance policy. + class AutoSourceDisabledError < ForbiddenError + DEFAULT_MESSAGE = 'Auto source feature is disabled' + end + end +end diff --git a/app/web/errors/error_responder.rb b/app/web/errors/error_responder.rb index 4deefe59..c977220f 100644 --- a/app/web/errors/error_responder.rb +++ b/app/web/errors/error_responder.rb @@ -4,125 +4,135 @@ module Html2rss module Web ## # Centralized error rendering for API and XML endpoints. - # - # Keeping this mapping in one place ensures consistent status codes and - # content types without duplicating rescue behavior in routes. module ErrorResponder API_ROOT_PATH = '/api/v1' - INTERNAL_ERROR_CODE = Api::V1::Contract::CODES[:internal_server_error] + EXTRACTION_EMPTY_CODE = 'EXTRACTION_EMPTY' + EXTRACTION_EMPTY_MESSAGE = 'We could not extract feed items from this page yet. ' \ + 'Try a more specific listing URL or explicit selectors.' + INTERNAL_ERROR_CODE = InternalServerError::CODE + NETWORK_ERRORS = Set[ + Timeout::Error, Net::OpenTimeout, Net::ReadTimeout, SocketError, + EOFError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ETIMEDOUT + ].freeze + + AUTH_META = { kind: 'auth', retryable: false, next_action: 'enter_token', retry_action: 'none' }.freeze + INPUT_META = { kind: 'input', retryable: false, next_action: 'correct_input', retry_action: 'none' }.freeze + SERVER_META = { kind: 'server', retryable: false, next_action: 'none', retry_action: 'none' }.freeze + RETRY_META = { retryable: true, next_action: 'retry', retry_action: 'primary' }.freeze class << self # @param request [Rack::Request] # @param response [Rack::Response] # @param error [StandardError] - # @return [String] serialized JSON or XML error body. + # @return [String] def respond(request:, response:, error:) - error_code = resolve_error_code(error) + code = resolve_error_code(error) response.status = resolve_status(error) - emit_error_event(error, error_code, response.status) + emit_error_event(error, code, response.status) write_internal_error_log(request, error) - client_message = client_message_for(error) - - return render_feed_error(request, response, client_message) if RequestTarget.feed?(request) - return render_api_error(response, client_message, error_code) if api_request?(request) + return render_feed_error(request, response, error) if request_target(request) == RequestTarget::FEED + if request_target(request) == RequestTarget::API || request.path.to_s.start_with?(API_ROOT_PATH) + return render_api_error(request, response, + error) + end - render_xml_error(response, client_message) + render_xml_error(response, error) end private - # @param request [Rack::Request] - # @return [Boolean] - def api_request?(request) - RequestTarget.api?(request) || api_path?(request) - end + def render_feed_error(request, response, error) + f = FeedResponseFormat.for_request(request) + response['Content-Type'] = FeedResponseFormat.content_type(f) + msg = client_message_for(error) + return JsonFeedBuilder.build_error_feed(message: msg) if f == FeedResponseFormat::JSON_FEED - # @param request [Rack::Request] - # @return [Boolean] - def api_path?(request) - path = request.path.to_s - path == API_ROOT_PATH || path.start_with?("#{API_ROOT_PATH}/") + XmlBuilder.build_error_feed(message: msg) end - # @param response [Rack::Response] - # @param message [String] - # @param code [String] - # @return [String] JSON error payload. - def render_api_error(response, message, code) + def render_api_error(_request, response, error) response['Content-Type'] = 'application/json' - JSON.generate({ success: false, error: { message: message, code: code } }) + JSON.generate({ success: false, error: failure_payload(error) }) end - # @param response [Rack::Response] - # @param message [String] - # @return [String] negotiated feed error payload. - def render_feed_error(request, response, message) - format = FeedResponseFormat.for_request(request) - response['Content-Type'] = FeedResponseFormat.content_type(format) - return JsonFeedBuilder.build_error_feed(message: message) if format == FeedResponseFormat::JSON_FEED + def render_xml_error(response, error) + response['Content-Type'] = 'application/xml' + XmlBuilder.build_error_feed(message: client_message_for(error)) + end - XmlBuilder.build_error_feed(message: message) + def resolve_error_code(error) + if extraction_empty_failure?(error) + EXTRACTION_EMPTY_CODE + else + (error.respond_to?(:code) ? error.code : INTERNAL_ERROR_CODE) + end end - # @param response [Rack::Response] - # @param message [String] - # @return [String] XML error feed. - def render_xml_error(response, message) - response['Content-Type'] = 'application/xml' - XmlBuilder.build_error_feed(message: message) + def resolve_status(error) + if extraction_empty_failure?(error) + 422 + else + (error.respond_to?(:status) ? error.status : 500) + end end - # @param request [Rack::Request] - # @param error [StandardError] - # @return [String] - def error_log_line(request, error) - request_id_header = request.respond_to?(:get_header) ? request.get_header('HTTP_X_REQUEST_ID') : nil - context = request.env['html2rss.request_context'] - request_id = request_id_header || context&.request_id - return error.message unless request_id + def client_message_for(error) + return EXTRACTION_EMPTY_MESSAGE if extraction_empty_failure?(error) - "[request_id=#{request_id}] #{error.message}" + error.is_a?(HttpError) ? error.message : HttpError::DEFAULT_MESSAGE end - # @param error [StandardError] - # @return [String] - def resolve_error_code(error) - error.respond_to?(:code) ? error.code : INTERNAL_ERROR_CODE + def extraction_empty_failure?(error) + defined?(::Html2rss::NoFeedItemsExtracted) && error_chain(error).any?(::Html2rss::NoFeedItemsExtracted) end - # @param error [StandardError] - # @return [Integer] - def resolve_status(error) - error.respond_to?(:status) ? error.status : 500 + def failure_payload(error) + { message: client_message_for(error), code: resolve_error_code(error) }.merge(failure_metadata(error)) end - # @param error [StandardError] - # @return [String] - def client_message_for(error) - error.is_a?(Html2rss::Web::HttpError) ? error.message : Html2rss::Web::HttpError::DEFAULT_MESSAGE + def failure_metadata(error) + return AUTH_META if error.is_a?(UnauthorizedError) + return INPUT_META if input_failure?(error) + return SERVER_META if error.is_a?(HealthCheckFailedError) + + RETRY_META.merge(kind: error_kind(error)) + end + + def input_failure?(error) + extraction_empty_failure?(error) || error.is_a?(BadRequestError) || error.is_a?(ForbiddenError) + end + + def error_kind(error) + error_chain(error).any? { |e| NETWORK_ERRORS.include?(e.class) } ? 'network' : 'server' + end + + def error_chain(error) + chain = [] + while error && chain.none? { |e| e.equal?(error) } + chain << error + error = error.respond_to?(:cause) ? error.cause : nil + end + chain end - # @param request [Rack::Request] - # @param error [StandardError] - # @return [void] def write_internal_error_log(request, error) - return if error.is_a?(Html2rss::Web::HttpError) + return if error.is_a?(HttpError) - request.env['rack.errors']&.puts(error_log_line(request, error)) + id = request.env['html2rss.request_context']&.request_id || + (request.respond_to?(:get_header) && request.get_header('HTTP_X_REQUEST_ID')) + request.env['rack.errors']&.puts(id ? "[request_id=#{id}] #{error.message}" : error.message) + end + + # @param request [#env] + # @return [Symbol, nil] + def request_target(request) + request.env[RequestTarget::ENV_KEY] end - # @param error [StandardError] - # @param error_code [String] - # @param status [Integer] - # @return [void] def emit_error_event(error, error_code, status) - Observability.emit( - event_name: 'request.error', - outcome: 'failure', - details: { error_class: error.class.name, error_code: error_code, status: status }, - level: :error - ) + Observability.emit(event_name: 'request.error', outcome: 'failure', level: :error, + details: { error_class: error.class.name, error_code: error_code, status: status }) end end end diff --git a/app/web/errors/health_check_failed_error.rb b/app/web/errors/health_check_failed_error.rb new file mode 100644 index 00000000..e581d115 --- /dev/null +++ b/app/web/errors/health_check_failed_error.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Html2rss + module Web + ## + # HTTP 500 error raised when shallow health checks cannot read configuration. + class HealthCheckFailedError < InternalServerError + DEFAULT_MESSAGE = 'Health check failed' + end + end +end diff --git a/app/web/feeds/cache.rb b/app/web/feeds/cache.rb index 3f8f8a16..9b286ba6 100644 --- a/app/web/feeds/cache.rb +++ b/app/web/feeds/cache.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'concurrent/map' require 'digest' require 'time' @@ -9,6 +10,13 @@ module Feeds ## # Small synchronous cache for canonical feed results. module Cache + # rubocop:disable ThreadSafety/ClassInstanceVariable + def self.entries + @entries ||= Concurrent::Map.new + end + # rubocop:enable ThreadSafety/ClassInstanceVariable + private_class_method :entries + Entry = Data.define(:result, :expires_at) class << self @@ -18,32 +26,27 @@ class << self # @yieldreturn [Html2rss::Web::Feeds::Contracts::RenderResult] # @return [Html2rss::Web::Feeds::Contracts::RenderResult] def fetch(key, ttl_seconds:, cacheable: true) - lock.synchronize do - entry = read_entry(key) - return entry.result if fresh?(entry) + entry = read_entry(key) + return entry.result if fresh?(entry) + + result = yield - result = yield - return result unless cacheable_result?(cacheable, result) + return result unless cacheable_result?(cacheable, result) - write_entry(key, ttl_seconds, result) - result - end + write_entry(key, ttl_seconds, result) + result end # @param reason [String] # @return [nil] def clear!(reason: 'manual') - lock.synchronize do - @entries = {} - SecurityLogger.log_cache_lifecycle('feeds_cache', 'clear', reason: reason) - end + entries.clear + SecurityLogger.log_cache_lifecycle('feeds_cache', 'clear', reason: reason) nil end private - # @param key [String] - # @return [Entry, nil] def read_entry(key) entries[key] end @@ -54,10 +57,6 @@ def fresh?(entry) entry && Time.now.utc < entry.expires_at end - # @param key [String] - # @param ttl_seconds [Integer] - # @param result [Html2rss::Web::Feeds::Contracts::RenderResult] - # @return [void] def write_entry(key, ttl_seconds, result) entries[key] = Entry.new(result: result, expires_at: Time.now.utc + normalize_ttl(ttl_seconds)) SecurityLogger.log_cache_lifecycle('feeds_cache', 'write', key_hash: key_hash(key)) @@ -72,16 +71,6 @@ def cacheable_result?(cacheable, result) cacheable end - # @return [Hash{String=>Entry}] - def entries - @entries ||= {} # rubocop:disable ThreadSafety/ClassInstanceVariable - end - - # @return [Mutex] - def lock - @lock ||= Mutex.new # rubocop:disable ThreadSafety/ClassInstanceVariable - end - # @param ttl_seconds [Integer] # @return [Integer] def normalize_ttl(ttl_seconds) diff --git a/app/web/feeds/contracts.rb b/app/web/feeds/contracts.rb index 29aa24cd..a3d098f0 100644 --- a/app/web/feeds/contracts.rb +++ b/app/web/feeds/contracts.rb @@ -20,7 +20,7 @@ module Contracts ## # Shared feed-serving result wrapper. - RenderResult = Data.define(:status, :payload, :message, :ttl_seconds, :cache_key, :error_message) + RenderResult = Data.define(:status, :payload, :message, :ttl_seconds, :cache_key, :error_message, :error_kind) end end end diff --git a/app/web/feeds/responder.rb b/app/web/feeds/responder.rb index 329fdece..88974e94 100644 --- a/app/web/feeds/responder.rb +++ b/app/web/feeds/responder.rb @@ -61,7 +61,7 @@ def emit_response_result(target_kind:, identifier:, feed_request:, resolved_sour # @param result [Html2rss::Web::Feeds::Contracts::RenderResult] # @return [String] def write_response(response:, representation:, result:) - response.status = result.status == :error ? 500 : 200 + response.status = status_for(result.status) response['Content-Type'] = FeedResponseFormat.content_type(representation) apply_cache_headers(response, result) ::Html2rss::Web::HttpCache.vary(response, 'Accept') @@ -92,7 +92,8 @@ def render_result(result, representation) # @param result [Html2rss::Web::Feeds::Contracts::RenderResult] # @return [void] def emit_result(target_kind:, identifier:, resolved_source:, result:) - return emit_success(target_kind:, identifier:, resolved_source:) unless result.status == :error + return emit_success(target_kind:, identifier:, resolved_source:) if result.status == :ok + return emit_empty(target_kind:, identifier:, resolved_source:, result:) if result.status == :empty emit_failure( target_kind:, @@ -109,14 +110,40 @@ def emit_result(target_kind:, identifier:, resolved_source:, result:) # @return [void] def emit_success(target_kind:, identifier:, resolved_source:) details = { - strategy: resolved_source.generator_input[:strategy], url: resolved_source.generator_input.dig(:channel, :url) } + strategy = resolved_source.generator_input[:strategy] + details[:strategy] = strategy if strategy details[:feed_name] = identifier if target_kind == :static Observability.emit(event_name: 'feed.render', outcome: 'success', details:, level: :info) end + # @param target_kind [Symbol] + # @param identifier [String] + # @param resolved_source [Html2rss::Web::Feeds::Contracts::ResolvedSource] + # @param result [Html2rss::Web::Feeds::Contracts::RenderResult] + # @return [void] + def emit_empty(target_kind:, identifier:, resolved_source:, result:) + details = { + url: resolved_source.generator_input.dig(:channel, :url), + reason: empty_reason_for(result) + } + strategy = resolved_source.generator_input[:strategy] + details[:strategy] = strategy if strategy + details[:feed_name] = identifier if target_kind == :static + + Observability.emit(event_name: 'feed.render', outcome: 'failure', details:, level: :warn) + end + + # @param result [Html2rss::Web::Feeds::Contracts::RenderResult] + # @return [String] + def empty_reason_for(result) + return 'content_extraction_empty' if result.error_kind == :extraction_empty + + 'feed_empty' + end + # @param target_kind [Symbol] # @param identifier [String] # @param error [StandardError] @@ -127,6 +154,15 @@ def emit_failure(target_kind:, identifier:, error:) Observability.emit(event_name: 'feed.render', outcome: 'failure', details:, level: :warn) end + + # @param status [Symbol] + # @return [Integer] + def status_for(status) + return 200 if status == :ok + return 422 if status == :empty + + 500 + end end end end diff --git a/app/web/feeds/service.rb b/app/web/feeds/service.rb index f7bee826..6e3765e9 100644 --- a/app/web/feeds/service.rb +++ b/app/web/feeds/service.rb @@ -30,6 +30,8 @@ def build_result(resolved_source, cache_key) feed = Html2rss.feed(resolved_source.generator_input) success_result(feed, resolved_source, cache_key) rescue StandardError => error + return empty_result(error, resolved_source, cache_key) if extraction_empty_error?(error) + error_result(error, resolved_source, cache_key) end @@ -44,7 +46,8 @@ def success_result(feed, resolved_source, cache_key) message: nil, ttl_seconds: resolved_source.ttl_seconds, cache_key: cache_key, - error_message: nil + error_message: nil, + error_kind: nil ) end @@ -68,7 +71,7 @@ def payload_for(feed, resolved_source) feed: feed, site_title: site_title_for(feed, resolved_source.generator_input.dig(:channel, :url)), url: resolved_source.generator_input.dig(:channel, :url), - strategy: resolved_source.generator_input[:strategy].to_s + strategy: resolved_source.generator_input[:strategy]&.to_s ) end @@ -93,7 +96,59 @@ def error_result(error, resolved_source, cache_key) message: Html2rss::Web::HttpError::DEFAULT_MESSAGE, ttl_seconds: resolved_source.ttl_seconds, cache_key: cache_key, - error_message: error.message + error_message: error.message, + error_kind: nil + ) + end + + # @param error [Html2rss::NoFeedItemsExtracted] + # @param resolved_source [Html2rss::Web::Feeds::Contracts::ResolvedSource] + # @param cache_key [String] + # @return [Html2rss::Web::Feeds::Contracts::RenderResult] + def empty_result(error, resolved_source, cache_key) + Contracts::RenderResult.new( + status: :empty, + payload: payload_for_empty_result(resolved_source), + message: nil, + ttl_seconds: resolved_source.ttl_seconds, + cache_key: cache_key, + error_message: error.message, + error_kind: :extraction_empty + ) + end + + # @param error [StandardError] + # @return [Boolean] + def extraction_empty_error?(error) + return false unless defined?(::Html2rss::NoFeedItemsExtracted) + + error_chain(error).any?(::Html2rss::NoFeedItemsExtracted) + end + + # @param error [StandardError, nil] + # @return [Array] + def error_chain(error) + errors = [] + seen = {}.compare_by_identity + current = error + + while current && !seen[current] + errors << current + seen[current] = true + current = current.respond_to?(:cause) ? current.cause : nil + end + + errors + end + + # @param resolved_source [Html2rss::Web::Feeds::Contracts::ResolvedSource] + # @return [Html2rss::Web::Feeds::Contracts::RenderPayload] + def payload_for_empty_result(resolved_source) + Contracts::RenderPayload.new( + feed: nil, + site_title: resolved_source.generator_input.dig(:channel, :url).to_s, + url: resolved_source.generator_input.dig(:channel, :url), + strategy: resolved_source.generator_input[:strategy]&.to_s ) end end diff --git a/app/web/feeds/source_resolver.rb b/app/web/feeds/source_resolver.rb index 45ba3c29..dc34266e 100644 --- a/app/web/feeds/source_resolver.rb +++ b/app/web/feeds/source_resolver.rb @@ -36,13 +36,15 @@ def resolve_static(feed_request) generator_input: generator_input, ttl_seconds: CacheTtl.seconds_from_minutes(generator_input.dig(:channel, :ttl)) ) + rescue Html2rss::Web::LocalConfig::NotFound + raise Html2rss::Web::NotFoundError end # @param feed_request [Html2rss::Web::Feeds::Contracts::Request] # @return [Html2rss::Web::Feeds::Contracts::ResolvedSource] def resolve_token(feed_request) ensure_auto_source_enabled! - feed_token = FeedAccess.authorize_feed_token!(feed_request.token) + feed_token = authorize_feed_token!(feed_request.token) strategy = resolved_strategy(feed_token) generator_input = token_generator_input(feed_token.url, strategy) @@ -69,7 +71,6 @@ def static_cache_identity(feed_name, params) def static_generator_input(config, params) generator_input = config.dup generator_input[:params] = merged_static_params(config, params) - generator_input[:strategy] ||= Html2rss::RequestService.default_strategy_name.to_sym generator_input end @@ -92,14 +93,29 @@ def token_cache_identity(token) def ensure_auto_source_enabled! return if AutoSource.enabled? - raise Html2rss::Web::ForbiddenError, Api::V1::Contract::MESSAGES[:auto_source_disabled] + raise Html2rss::Web::AutoSourceDisabledError + end + + # @param token [String] + # @return [Html2rss::Web::FeedToken] + def authorize_feed_token!(token) + feed_token = Auth.validate_and_decode_feed_token(token) + raise Html2rss::Web::UnauthorizedError, 'Invalid token' unless feed_token + + account = AccountManager.get_account_by_username(feed_token.username) + raise Html2rss::Web::UnauthorizedError, 'Account not found' unless account + return feed_token if UrlValidator.url_allowed?(account, feed_token.url) + + raise Html2rss::Web::ForbiddenError, 'Access Denied' end # @param feed_token [Html2rss::Web::FeedToken] # @return [String] def resolved_strategy(feed_token) strategy = feed_token.strategy.to_s.strip - strategy = Html2rss::RequestService.default_strategy_name.to_s if strategy.empty? + return default_strategy_name if strategy.empty? + return strategy if strategy == default_strategy_name + supported = Html2rss::RequestService.strategy_names.map(&:to_s) raise Html2rss::Web::BadRequestError, 'Unsupported strategy' unless supported.include?(strategy) @@ -110,13 +126,19 @@ def resolved_strategy(feed_token) # @param strategy [String] # @return [Hash{Symbol=>Object}] def token_generator_input(url, strategy) - LocalConfig.global - .slice(:stylesheets, :headers) - .merge( - strategy: strategy.to_sym, - channel: { url: url }, - auto_source: {} - ) + global_config = LocalConfig.global + base_input = global_config.slice(:stylesheets, :headers) + base_input.merge(channel: { url: url }, auto_source: {}, strategy: strategy.to_sym) + end + + # @return [String] + def default_strategy_name + if Html2rss::Config.respond_to?(:default_strategy_name) + configured = Html2rss::Config.default_strategy_name.to_s + end + return configured unless configured.to_s.strip.empty? + + 'auto' end end end diff --git a/app/web/rendering/feed_notice_text.rb b/app/web/rendering/feed_notice_text.rb index ec92ac3c..ed34d3ad 100644 --- a/app/web/rendering/feed_notice_text.rb +++ b/app/web/rendering/feed_notice_text.rb @@ -6,19 +6,17 @@ module Web # Shared copy helpers for rendered feed warnings and fallback documents. module FeedNoticeText EMPTY_FEED_DESCRIPTION_TEMPLATE = <<~DESC - Unable to extract content from %s using the %s strategy. - The site may rely on JavaScript, block automated requests, or expose a structure that needs a different parser. + We could not extract entries from %s right now. + The source may block automated requests, require dynamic rendering, or be temporarily unavailable. DESC EMPTY_FEED_ITEM_TEMPLATE = <<~DESC No entries were extracted from %s. - Possible causes: - - JavaScript-heavy site (try the browserless strategy) - - Anti-bot protection - - Complex or changing markup - - Site blocking automated requests - Try another strategy or reach out to the site owner. + What you can do: + - Try again in a few moments + - Open the original page to confirm content is available + - Reach out to the site owner if access is restricted DESC class << self diff --git a/app/web/rendering/json_feed_builder.rb b/app/web/rendering/json_feed_builder.rb index 0c1e3187..727c4b99 100644 --- a/app/web/rendering/json_feed_builder.rb +++ b/app/web/rendering/json_feed_builder.rb @@ -75,7 +75,7 @@ def build_single_item(item) # @return [Hash{Symbol=>String}] def empty_feed_item(url) { - title: 'Content Extraction Failed', + title: 'Preview unavailable for this source', content_text: FeedNoticeText.empty_feed_item(url: url), url: url } diff --git a/app/web/rendering/xml_builder.rb b/app/web/rendering/xml_builder.rb index 399a4d39..057e997b 100644 --- a/app/web/rendering/xml_builder.rb +++ b/app/web/rendering/xml_builder.rb @@ -53,7 +53,7 @@ def build_empty_feed_warning(url:, strategy:, site_title: nil) build_single_item_feed( title: FeedNoticeText.empty_feed_title(site_title), description: FeedNoticeText.empty_feed_description(url: url, strategy: strategy), - item: { title: 'Content Extraction Failed', description: FeedNoticeText.empty_feed_item(url: url), + item: { title: 'Preview unavailable for this source', description: FeedNoticeText.empty_feed_item(url: url), link: url }, link: url ) diff --git a/app/web/request/request_target.rb b/app/web/request/request_target.rb index 9c7df270..efec5c84 100644 --- a/app/web/request/request_target.rb +++ b/app/web/request/request_target.rb @@ -9,33 +9,6 @@ module RequestTarget API = :api FEED = :feed - - class << self - # @param request [#env] - # @param target [Symbol] - # @return [Symbol] assigned target. - def mark!(request, target) - request.env[ENV_KEY] = target - end - - # @param request [#env] - # @return [Symbol, nil] request target selected by the router. - def current(request) - request.env[ENV_KEY] - end - - # @param request [#env] - # @return [Boolean] - def api?(request) - current(request) == API - end - - # @param request [#env] - # @return [Boolean] - def feed?(request) - current(request) == FEED - end - end end end end diff --git a/app/web/routes/api_v1.rb b/app/web/routes/api_v1.rb index e95a51ad..44f7ed48 100644 --- a/app/web/routes/api_v1.rb +++ b/app/web/routes/api_v1.rb @@ -16,7 +16,7 @@ class << self # @return [void] def call(router) router.on 'api', 'v1' do - RequestTarget.mark!(router, RequestTarget::API) + router.env[RequestTarget::ENV_KEY] = RequestTarget::API router.response['Content-Type'] = 'application/json' HealthRoutes.call(router) diff --git a/app/web/routes/api_v1/feed_routes.rb b/app/web/routes/api_v1/feed_routes.rb index fdfd7c93..2c54cce0 100644 --- a/app/web/routes/api_v1/feed_routes.rb +++ b/app/web/routes/api_v1/feed_routes.rb @@ -13,13 +13,15 @@ class << self def call(router) router.on 'feeds' do router.get String do |token| - RequestTarget.mark!(router, RequestTarget::FEED) + router.env[RequestTarget::ENV_KEY] = RequestTarget::FEED Feeds::Responder.call(request: router, target_kind: :token, identifier: token) end router.post do JSON.generate(Api::V1::CreateFeed.call(router)) end + + raise NotFoundError end end end diff --git a/app/web/routes/feed_pages.rb b/app/web/routes/feed_pages.rb index 535329f6..ea72b160 100644 --- a/app/web/routes/feed_pages.rb +++ b/app/web/routes/feed_pages.rb @@ -20,7 +20,7 @@ def call(router, index_renderer:) next if feed_name.empty? next if feed_name.include?('.') && !feed_name.end_with?('.json', '.xml', '.rss') - RequestTarget.mark!(router, RequestTarget::FEED) + router.env[RequestTarget::ENV_KEY] = RequestTarget::FEED Feeds::Responder.call(request: router, target_kind: :static, identifier: feed_name) end end diff --git a/app/web/security/account_manager.rb b/app/web/security/account_manager.rb index 45d5a37b..3cb5de85 100644 --- a/app/web/security/account_manager.rb +++ b/app/web/security/account_manager.rb @@ -8,19 +8,21 @@ module Web # Keeps config reads cheap by materializing one immutable snapshot and # exposing narrow lookup helpers for auth and authorization flows. module AccountManager + @mutex = Mutex.new + @snapshot = nil + class << self # Forces account snapshot refresh on next access. - # Used by tests and can be used by runtime reload hooks. # # @param reason [String] # @return [nil] def reload!(reason: 'manual') - @snapshot = nil # rubocop:disable ThreadSafety/ClassInstanceVariable + @mutex.synchronize { @snapshot = nil } SecurityLogger.log_cache_lifecycle('account_manager', 'reload', reason: reason) nil end - # @param token [String] + # @param token [String, nil] # @return [Hash{Symbol=>Object}, nil] def get_account(token) return nil unless token @@ -33,7 +35,7 @@ def accounts snapshot[:accounts] end - # @param username [String] + # @param username [String, nil] # @return [Hash{Symbol=>Object}, nil] def get_account_by_username(username) return nil unless username @@ -43,31 +45,10 @@ def get_account_by_username(username) private - # Lazily initializes and memoizes an immutable account snapshot. - # - # @return [Hash{Symbol=>Object}] - # @option return [ArrayObject}>] :accounts frozen account list. - # @option return [Hash{String=>Hash{Symbol=>Object}}] :token_index token lookup table. - # @option return [Hash{String=>Hash{Symbol=>Object}}] :username_index username lookup table. def snapshot - return @snapshot if @snapshot # rubocop:disable ThreadSafety/ClassInstanceVariable - - mutex.synchronize do - @snapshot ||= build_snapshot - end + @mutex.synchronize { @snapshot ||= build_snapshot } end - # @return [Mutex] synchronization primitive for snapshot rebuilds. - def mutex - @mutex ||= Mutex.new # rubocop:disable ThreadSafety/ClassInstanceVariable - end - - # Builds the immutable account snapshot from local configuration. - # - # @return [Hash{Symbol=>Object}] - # @option return [ArrayObject}>] :accounts frozen account list. - # @option return [Hash{String=>Hash{Symbol=>Object}}] :token_index token lookup table. - # @option return [Hash{String=>Hash{Symbol=>Object}}] :username_index username lookup table. def build_snapshot raw_accounts = LocalConfig.global.dig(:auth, :accounts) accounts = normalized_accounts(raw_accounts) diff --git a/app/web/security/auth.rb b/app/web/security/auth.rb index 528d2fa0..564367b1 100644 --- a/app/web/security/auth.rb +++ b/app/web/security/auth.rb @@ -24,10 +24,10 @@ def authenticate(request) # @param username [String] # @param url [String] - # @param strategy [String] + # @param strategy [String, nil] # @param expires_in [Integer] seconds (default: 10 years) # @return [String, nil] signed feed token when generation succeeds. - def generate_feed_token(username, url, strategy:, expires_in: FeedToken::DEFAULT_EXPIRY) + def generate_feed_token(username, url, strategy: nil, expires_in: FeedToken::DEFAULT_EXPIRY) token = FeedToken.create_with_validation( username: username, url: url, diff --git a/app/web/security/feed_access.rb b/app/web/security/feed_access.rb deleted file mode 100644 index a77dce62..00000000 --- a/app/web/security/feed_access.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - ## - # Centralizes account, token, and URL access checks for token-backed feed flows. - module FeedAccess - class << self - # @param username [String, nil] - # @return [Hash{Symbol=>Object}, nil] - def account_for_username(username) - AccountManager.get_account_by_username(username) - end - - # @param token [String] - # @return [Html2rss::Web::FeedToken] - def authorize_feed_token!(token) - feed_token = Auth.validate_and_decode_feed_token(token) - raise Html2rss::Web::UnauthorizedError, 'Invalid token' unless feed_token - - account = account_for_username(feed_token.username) - raise Html2rss::Web::UnauthorizedError, 'Account not found' unless account - raise Html2rss::Web::ForbiddenError, 'Access Denied' unless UrlValidator.url_allowed?(account, feed_token.url) - - feed_token - end - - # @param username [String, nil] - # @param url [String] - # @return [Boolean] - def url_allowed_for_username?(username, url) - account = account_for_username(username) - return false unless account - - UrlValidator.url_allowed?(account, url) - end - end - end - end -end diff --git a/app/web/security/feed_token.rb b/app/web/security/feed_token.rb index 926e8f9d..3e6ca1e6 100644 --- a/app/web/security/feed_token.rb +++ b/app/web/security/feed_token.rb @@ -4,46 +4,36 @@ require 'json' require 'openssl' require 'zlib' + module Html2rss module Web # rubocop:disable Metrics/ModuleLength ## - # Immutable feed token value object with encode/decode and validation helpers. - # - # It keeps signing, validation, and payload shaping in one place so auth - # callers can treat tokens as a single boundary contract. + # Immutable feed token value object with encoding and validation helpers. FeedToken = Data.define(:username, :url, :expires_at, :signature, :strategy) do # @param username [String] # @param url [String] # @param secret_key [String] - # @param strategy [String] + # @param strategy [String, nil] # @param expires_in [Integer] - # @return [Html2rss::Web::FeedToken, nil] signed token object when inputs are valid. - def self.create_with_validation(username:, url:, secret_key:, strategy:, expires_in: FeedToken::DEFAULT_EXPIRY) + # @return [Html2rss::Web::FeedToken, nil] + def self.create_with_validation(username:, url:, secret_key:, strategy: nil, + expires_in: Html2rss::Web::FeedToken::DEFAULT_EXPIRY) return unless valid_inputs?(username, url, secret_key, strategy) expires_at = Time.now.to_i + expires_in.to_i - payload = build_payload(username, url, expires_at, strategy) - signature = generate_signature(secret_key, payload) - - new(username: username, url: url, expires_at: expires_at, signature: signature, strategy: strategy) + signature = generate_signature(secret_key, build_payload(username, url, expires_at, strategy)) + new(username:, url:, expires_at:, signature:, strategy:) end # @param encoded_token [String, nil] - # @return [Html2rss::Web::FeedToken, nil] decoded token when payload is valid. - def self.decode(encoded_token) # rubocop:disable Metrics/MethodLength + # @return [Html2rss::Web::FeedToken, nil] + def self.decode(encoded_token) return unless encoded_token token_data = parse_token_data(encoded_token) return unless valid_token_data?(token_data) - payload = token_data[:p] - new( - username: payload[:u], - url: payload[:l], - expires_at: payload[:e], - signature: token_data[:s], - strategy: payload[:t] - ) + decoded_token(token_data) rescue JSON::ParserError, ArgumentError, Zlib::DataError, Zlib::BufError nil end @@ -51,7 +41,7 @@ def self.decode(encoded_token) # rubocop:disable Metrics/MethodLength # @param encoded_token [String, nil] # @param expected_url [String, nil] # @param secret_key [String] - # @return [Html2rss::Web::FeedToken, nil] validated token bound to expected URL. + # @return [Html2rss::Web::FeedToken, nil] def self.validate_and_decode(encoded_token, expected_url, secret_key) token = decode(encoded_token) return unless token @@ -62,94 +52,112 @@ def self.validate_and_decode(encoded_token, expected_url, secret_key) token end - # @return [String] compressed + URL-safe token representation. + # @return [String] def encode compressed = Zlib::Deflate.deflate(build_token_data.to_json) Base64.urlsafe_encode64(compressed) end - # @return [Boolean] true when token expiration is in the past. + # @return [Boolean] def expired? Time.now.to_i > expires_at end # @param candidate_url [String] - # @return [Boolean] true when candidate URL matches token URL exactly. + # @return [Boolean] def valid_for_url?(candidate_url) url == candidate_url end # @param secret_key [String] - # @return [Boolean] true when signature matches payload under given key. + # @return [Boolean] def valid_signature?(secret_key) - return false unless self.class.valid_secret_key?(secret_key) + return false unless secret_key.is_a?(String) && !secret_key.empty? - expected_signature = self.class.generate_signature(secret_key, payload_for_signature) - secure_compare(signature, expected_signature) + expected_signature = OpenSSL::HMAC.hexdigest( + Html2rss::Web::FeedToken::HMAC_ALGORITHM, + secret_key, + JSON.generate(payload_for_signature) + ) + signatures_match?(signature, expected_signature) end private - # @return [Hash{Symbol=>Object}] canonical payload used for signature checks. + # @return [Hash{Symbol=>Object}] def payload_for_signature - payload = { username: username, url: url, expires_at: expires_at } + payload = { username:, url:, expires_at: } payload[:strategy] = strategy if strategy payload end - # @return [Hash{Symbol=>Object}] compact token envelope. + # @return [Hash{Symbol=>Object}] def build_token_data payload = { u: username, l: url, e: expires_at } payload[:t] = strategy if strategy { p: payload, s: signature } end - # Constant-time compare prevents timing leaks on signature mismatch. - # # @param first [String, nil] # @param second [String, nil] # @return [Boolean] - def secure_compare(first, second) # rubocop:disable Naming/PredicateMethod + def signatures_match?(first, second) return false unless first && second && first.bytesize == second.bytesize first.each_byte.zip(second.each_byte).reduce(0) { |acc, (a, b)| acc | (a ^ b) }.zero? end class << self + private + # @param username [String] # @param url [String] # @param expires_at [Integer] - # @param strategy [String] - # @return [Hash{Symbol=>Object}] signature payload. + # @param strategy [String, nil] + # @return [Hash{Symbol=>Object}] def build_payload(username, url, expires_at, strategy) - { username: username, url: url, expires_at: expires_at, strategy: strategy } + payload = { username:, url:, expires_at: } + payload[:strategy] = strategy if strategy + payload end # @param secret_key [String] # @param payload [Hash, String] - # @return [String] HMAC digest. + # @return [String] def generate_signature(secret_key, payload) data = payload.is_a?(String) ? payload : JSON.generate(payload) - OpenSSL::HMAC.hexdigest(FeedToken::HMAC_ALGORITHM, secret_key, data) + OpenSSL::HMAC.hexdigest(Html2rss::Web::FeedToken::HMAC_ALGORITHM, secret_key, data) end # @param encoded_token [String] - # @return [Hash{Symbol=>Object}] parsed token envelope. + # @return [Hash{Symbol=>Object}] def parse_token_data(encoded_token) - decoded = Base64.urlsafe_decode64(encoded_token) - inflated = Zlib::Inflate.inflate(decoded) + inflated = Zlib::Inflate.inflate(Base64.urlsafe_decode64(encoded_token)) JSON.parse(inflated, symbolize_names: true) end + # @param token_data [Hash{Symbol=>Object}] + # @return [Html2rss::Web::FeedToken] + def decoded_token(token_data) + payload = token_data[:p] + new( + username: payload[:u], + url: payload[:l], + expires_at: payload[:e], + signature: token_data[:s], + strategy: payload[:t] + ) + end + # @param token_data [Object] - # @return [Boolean] true when structure contains required payload/signature keys. + # @return [Boolean] def valid_token_data?(token_data) return false unless token_data.is_a?(Hash) payload = token_data[:p] signature = token_data[:s] payload.is_a?(Hash) && signature.is_a?(String) && !signature.empty? && - FeedToken::COMPRESSED_PAYLOAD_KEYS.all? { |key| payload[key] } + Html2rss::Web::FeedToken::COMPRESSED_PAYLOAD_KEYS.all? { |key| payload[key] } end # @param username [Object] @@ -177,14 +185,15 @@ def valid_secret_key?(secret_key) # @param strategy [Object] # @return [Boolean] def valid_strategy?(strategy) + return true if strategy.nil? + strategy.is_a?(String) && !strategy.empty? && strategy.length <= 50 && strategy.match?(/\A[a-z0-9_]+\z/) end end end - FeedToken::DEFAULT_EXPIRY = 315_360_000 # 10 years in seconds + FeedToken::DEFAULT_EXPIRY = 315_360_000 FeedToken::HMAC_ALGORITHM = 'SHA256' - FeedToken::REQUIRED_TOKEN_KEYS = %i[p s].freeze FeedToken::COMPRESSED_PAYLOAD_KEYS = %i[u l e].freeze end end diff --git a/app/web/security/security_logger.rb b/app/web/security/security_logger.rb index bc0fce99..05df5d08 100644 --- a/app/web/security/security_logger.rb +++ b/app/web/security/security_logger.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true require 'digest' +require_relative '../telemetry/log_event' +require_relative '../security/log_sanitizer' +require_relative '../telemetry/app_logger' + module Html2rss module Web ## diff --git a/app/web/telemetry/app_logger.rb b/app/web/telemetry/app_logger.rb index 8724bab6..88d686ad 100644 --- a/app/web/telemetry/app_logger.rb +++ b/app/web/telemetry/app_logger.rb @@ -3,11 +3,8 @@ require 'json' require 'logger' require 'time' - module Html2rss module Web - ## - # Shared structured logger for application and middleware runtime events. module AppLogger class << self # @return [Logger] @@ -103,6 +100,7 @@ def normalize_logfmt_value(raw_value) def emit_to_sentry(payload) return unless sentry_payload?(payload) + SentryLogs.record_breadcrumb(payload) SentryLogs.emit(payload) rescue StandardError nil diff --git a/app/web/telemetry/sentry_logs.rb b/app/web/telemetry/sentry_logs.rb index f6d55353..33839677 100644 --- a/app/web/telemetry/sentry_logs.rb +++ b/app/web/telemetry/sentry_logs.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative '../security/log_sanitizer' + module Html2rss module Web ## @@ -9,8 +11,26 @@ module SentryLogs OMIT = Object.new.freeze ALLOWED_LEVELS = %i[debug info warn error fatal].freeze SENSITIVE_ATTRIBUTE_KEYS = %w[actor email ip remote_ip user_agent username x_forwarded_for].freeze + BREADCRUMB_KEYS = %i[event_name security_event outcome request_id route_group strategy component details].freeze + BREADCRUMB_CATEGORY_KEYS = %i[event_name security_event component].freeze + BREADCRUMB_MESSAGE_KEYS = %i[message event_name security_event component].freeze class << self + # @param payload [Hash{Symbol=>Object}] + # @return [void] + def record_breadcrumb(payload) + return unless breadcrumb_enabled? + + ::Sentry.add_breadcrumb( + category: breadcrumb_category(payload), + message: breadcrumb_message(payload), + level: breadcrumb_level(payload), + data: breadcrumb_data(payload) + ) + rescue StandardError + nil + end + # @param payload [Hash{Symbol=>Object}] # @return [void] def emit(payload) @@ -31,6 +51,13 @@ def enabled? !logger.nil? end + # @return [Boolean] + def breadcrumb_enabled? + RuntimeEnv.sentry_enabled? && + defined?(::Sentry) && + ::Sentry.respond_to?(:add_breadcrumb) + end + # @return [Object, nil] def logger return unless defined?(::Sentry) && ::Sentry.respond_to?(:logger) @@ -54,6 +81,34 @@ def message(payload) payload[:component] || 'html2rss-web log' end + # @param payload [Hash{Symbol=>Object}] + # @return [String] + def breadcrumb_category(payload) + breadcrumb_label(payload, 'html2rss-web', BREADCRUMB_CATEGORY_KEYS) + end + + # @param payload [Hash{Symbol=>Object}] + # @return [String] + def breadcrumb_message(payload) + breadcrumb_label(payload, 'html2rss-web log', BREADCRUMB_MESSAGE_KEYS) + end + + # @param payload [Hash{Symbol=>Object}] + # @return [String] + def breadcrumb_level(payload) + requested_level = payload.fetch(:level, 'info').to_s.downcase + return 'warning' if requested_level == 'warn' + return requested_level if ALLOWED_LEVELS.map(&:to_s).include?(requested_level) + + 'info' + end + + # @param payload [Hash{Symbol=>Object}] + # @return [Hash{Symbol=>Object}] + def breadcrumb_data(payload) + LogSanitizer.sanitize_details(payload).slice(*BREADCRUMB_KEYS) + end + # @param payload [Hash{Symbol=>Object}] # @return [Hash{Symbol=>Object}] def attributes(payload) @@ -98,6 +153,14 @@ def sanitize_array(key, values) def sensitive_key?(key) SENSITIVE_ATTRIBUTE_KEYS.include?(key.to_s) end + + # @param payload [Hash{Symbol=>Object}] + # @param fallback [String] + # @param keys [Array] + # @return [String] + def breadcrumb_label(payload, fallback, keys) + keys.lazy.map { |key| payload[key] }.find(&:itself) || fallback + end end end end diff --git a/docs/README.md b/docs/README.md index 7329c3a2..f7d9c4cf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,7 +18,7 @@ Welcome! This is the canonical source of truth for contributing to `html2rss-web `html2rss-web` converts arbitrary websites into RSS 2.0 feeds. - **Backend**: Ruby + Roda under the `Html2rss::Web` namespace. -- **Frontend**: Preact + Vite, built into `frontend/dist` and served at `/`. +- **Frontend**: Preact + Vite, built into `frontend/dist` and served at `/` in production. - **Feed extraction**: Delegated to the `html2rss` gem. - **Distribution**: Docker Compose / Dev Container first. @@ -42,6 +42,7 @@ Running the app directly on the host is not supported. | `make setup` | Install Ruby and Node dependencies. | | `make dev` | Run Ruby (port 4000) and frontend (port 4001) dev servers. | | `make ready` | Pre-commit gate: `make quick-check` + `bundle exec rspec`. | +| `make ci-ready` | CI parity gate: `make ready` + `make openapi-verify` + frontend e2e smoke. | | `make test` | Run Ruby and frontend test suites. | | `make lint` | Run all linters. | | `make yard-verify-public-docs` | Enforce typed YARD docs for public methods in `app/`. | @@ -59,6 +60,12 @@ Running the app directly on the host is not supported. | `pnpm run test:run` | Unit tests (Vitest). | | `pnpm run test:contract`| Contract tests with MSW. | +Development routing defaults: + +- `http://127.0.0.1:4000` is API-only in development (`/api/v1` metadata and API endpoints). +- `http://127.0.0.1:4001` is the canonical frontend SPA entrypoint in development. +- Vite keeps proxying `/api` and `/rss.xsl` to `:4000` so frontend code can use same-origin-style paths. + --- ## Contract-Driven Development Loop @@ -68,7 +75,7 @@ To change or add API endpoints, follow this sequence: 1. **Ruby Request Spec**: Define the new behavior or endpoint in `spec/html2rss/web/app_integration_spec.rb` or a dedicated request spec. 2. **OpenAPI Generation**: Run `make openapi` inside the Dev Container to regenerate `public/openapi.yaml` from the spec metadata. 3. **Verify Contract**: Run `make openapi-verify` and `make openapi-lint` to ensure the generated file matches the specs and is valid. -4. **Frontend Client**: The frontend generated client in `frontend/src/api/generated` is updated by the build process. +4. **Frontend Client**: Keep generated client artifacts in `frontend/src/api/generated` aligned with `public/openapi.yaml`. Always verify the contract before committing API changes. @@ -84,6 +91,12 @@ Always run this before pushing or committing: make ready ``` +For frontend changes and API contract/OpenAPI changes, run the CI-parity gate: + +```bash +make ci-ready +``` + ### Testing Layers | Layer | Tooling | Focus | @@ -200,6 +213,17 @@ Canonical event fields: `event_name`, `schema_version`, `request_id`, `route_gro Critical-path event families: auth, feed create, feed render, request errors. +## Sentry Runbook + +When `SENTRY_DSN` is present, Sentry is enabled. `BUILD_TAG` and `GIT_SHA` become the release identifier, and +`RACK_ENV` becomes the environment tag. + +Triage starts with the newest `feed.create`, `feed.render`, and `request.error` events. Confirm the release tag, +route group, strategy, and outcome before deciding whether the failure is retryable, terminal, or user-facing. + +Alert on sustained production `request.error` spikes or repeated `feed.render` failures, then tune thresholds from +real incidents. + --- ## Documentation Policy diff --git a/frontend/src/__tests__/App.contract.test.tsx b/frontend/src/__tests__/App.contract.test.tsx index 79150dce..6bb75741 100644 --- a/frontend/src/__tests__/App.contract.test.tsx +++ b/frontend/src/__tests__/App.contract.test.tsx @@ -16,9 +16,9 @@ describe('App contract', () => { server.use( http.post('/api/v1/feeds', async ({ request }) => { - const body = (await request.json()) as { url: string; strategy: string }; + const body = (await request.json()) as { url: string }; - expect(body).toEqual({ url: 'https://example.com/articles', strategy: 'faraday' }); + expect(body).toEqual({ url: 'https://example.com/articles' }); expect(request.headers.get('authorization')).toBe(`Bearer ${token}`); return HttpResponse.json( diff --git a/frontend/src/__tests__/useFeedConversion.contract.test.ts b/frontend/src/__tests__/useFeedConversion.contract.test.ts index ac13df99..13d646e9 100644 --- a/frontend/src/__tests__/useFeedConversion.contract.test.ts +++ b/frontend/src/__tests__/useFeedConversion.contract.test.ts @@ -10,10 +10,10 @@ describe('useFeedConversion contract', () => { server.use( http.post('/api/v1/feeds', async ({ request }) => { - const body = (await request.json()) as { url: string; strategy: string }; + const body = (await request.json()) as { url: string }; receivedAuthorization = request.headers.get('authorization'); - expect(body).toEqual({ url: 'https://example.com/articles', strategy: 'faraday' }); + expect(body).toEqual({ url: 'https://example.com/articles' }); return HttpResponse.json( buildFeedResponse({ diff --git a/frontend/src/__tests__/useFeedConversion.test.ts b/frontend/src/__tests__/useFeedConversion.test.ts index 7f55096d..ce00639b 100644 --- a/frontend/src/__tests__/useFeedConversion.test.ts +++ b/frontend/src/__tests__/useFeedConversion.test.ts @@ -517,7 +517,6 @@ describe('useFeedConversion', () => { ); expect(await firstRequest.clone().json()).toEqual({ url: 'https://example.com/articles', - strategy: 'faraday', }); }); @@ -577,7 +576,6 @@ describe('useFeedConversion', () => { const retryRequest = fetchMock.mock.calls[1]?.[0] as Request; expect(await retryRequest.clone().json()).toEqual({ url: 'https://example.com/articles', - strategy: 'browserless', }); expect(result.current.result?.retry).toEqual({ automatic: true, @@ -702,7 +700,6 @@ describe('useFeedConversion', () => { const retryRequest = fetchMock.mock.calls[1]?.[0] as Request; expect(await retryRequest.clone().json()).toEqual({ url: 'https://example.com/articles', - strategy: 'browserless', }); expect(result.current.result?.retry).toEqual({ automatic: true, diff --git a/frontend/src/api/generated/types.gen.ts b/frontend/src/api/generated/types.gen.ts index 88d58560..f2c3a272 100644 --- a/frontend/src/api/generated/types.gen.ts +++ b/frontend/src/api/generated/types.gen.ts @@ -42,7 +42,6 @@ export type GetApiMetadataResponse = GetApiMetadataResponses[keyof GetApiMetadat export type CreateFeedData = { body?: { - strategy: string; url: string; }; headers: { @@ -60,7 +59,11 @@ export type CreateFeedErrors = { 401: { error: { code: string; + kind: string; message: string; + next_action: string; + retry_action: string; + retryable: boolean; }; success: boolean; }; @@ -70,7 +73,11 @@ export type CreateFeedErrors = { 403: { error: { code: string; + kind: string; message: string; + next_action: string; + retry_action: string; + retryable: boolean; }; success: boolean; }; @@ -91,7 +98,6 @@ export type CreateFeedResponses = { json_public_url: string; name: string; public_url: string; - strategy: string; updated_at: string; url: string; }; @@ -157,7 +163,11 @@ export type GetHealthStatusErrors = { 401: { error: { code: string; + kind: string; message: string; + next_action: string; + retry_action: string; + retryable: boolean; }; success: boolean; }; @@ -167,7 +177,11 @@ export type GetHealthStatusErrors = { 500: { error: { code: string; + kind: string; message: string; + next_action: string; + retry_action: string; + retryable: boolean; }; success: boolean; }; diff --git a/frontend/src/hooks/useFeedConversion.ts b/frontend/src/hooks/useFeedConversion.ts index 482003ee..c89767e2 100644 --- a/frontend/src/hooks/useFeedConversion.ts +++ b/frontend/src/hooks/useFeedConversion.ts @@ -274,7 +274,6 @@ async function requestFeedCreation(url: string, strategy: string, token: string) }, body: { url, - strategy, }, throwOnError: true, }); diff --git a/public/openapi.yaml b/public/openapi.yaml index ab990919..61084fed 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -103,13 +103,10 @@ paths: application/json: schema: properties: - strategy: - type: string url: type: string required: - url - - strategy type: object responses: '201': @@ -133,8 +130,6 @@ paths: type: string public_url: type: string - strategy: - type: string updated_at: type: string url: @@ -143,7 +138,6 @@ paths: - id - name - url - - strategy - feed_token - public_url - json_public_url @@ -177,11 +171,23 @@ paths: properties: code: type: string + kind: + type: string message: type: string + next_action: + type: string + retry_action: + type: string + retryable: + type: boolean required: - message - code + - kind + - retryable + - next_action + - retry_action type: object success: type: boolean @@ -199,11 +205,23 @@ paths: properties: code: type: string + kind: + type: string message: type: string + next_action: + type: string + retry_action: + type: string + retryable: + type: boolean required: - message - code + - kind + - retryable + - next_action + - retry_action type: object success: type: boolean @@ -339,11 +357,23 @@ paths: properties: code: type: string + kind: + type: string message: type: string + next_action: + type: string + retry_action: + type: string + retryable: + type: boolean required: - message - code + - kind + - retryable + - next_action + - retry_action type: object success: type: boolean @@ -361,11 +391,23 @@ paths: properties: code: type: string + kind: + type: string message: type: string + next_action: + type: string + retry_action: + type: string + retryable: + type: boolean required: - message - code + - kind + - retryable + - next_action + - retry_action type: object success: type: boolean diff --git a/public/rss.xsl b/public/rss.xsl index 02d8cb25..792de60e 100644 --- a/public/rss.xsl +++ b/public/rss.xsl @@ -361,7 +361,7 @@ - +

diff --git a/spec/html2rss/web/api/v1/feed_metadata_spec.rb b/spec/html2rss/web/api/v1/feed_metadata_spec.rb index fb9e55fa..3f6c9f74 100644 --- a/spec/html2rss/web/api/v1/feed_metadata_spec.rb +++ b/spec/html2rss/web/api/v1/feed_metadata_spec.rb @@ -23,7 +23,6 @@ name: 'Example Feed', url: 'https://example.com/articles', username: 'alice', - strategy: 'faraday', feed_token: 'generated-token', public_url: '/api/v1/feeds/generated-token', json_public_url: '/api/v1/feeds/generated-token.json' diff --git a/spec/html2rss/web/api/v1_spec.rb b/spec/html2rss/web/api/v1_spec.rb index a26256a1..a67a2f0a 100644 --- a/spec/html2rss/web/api/v1_spec.rb +++ b/spec/html2rss/web/api/v1_spec.rb @@ -17,7 +17,8 @@ def feed_result message: nil, ttl_seconds: 600, cache_key: 'feed_result:test', - error_message: nil + error_message: nil, + error_kind: nil ) end @@ -28,7 +29,46 @@ def service_error_result message: 'Internal Server Error', ttl_seconds: 600, cache_key: 'feed_result:error', - error_message: 'upstream timeout' + error_message: 'upstream timeout', + error_kind: :network + ) + end + + def empty_result + empty_feed_result(cache_key: 'feed_result:empty') + end + + def extraction_empty_result + empty_feed_result( + cache_key: 'feed_result:extraction-empty', + error_message: 'No feed items extracted after auto fallback', + error_kind: :extraction_empty + ) + end + + # @param cache_key [String] + # @param error_message [String, nil] + # @param error_kind [Symbol, nil] + # @return [Html2rss::Web::Feeds::Contracts::RenderResult] + def empty_feed_result(cache_key:, error_message: nil, error_kind: nil) + Html2rss::Web::Feeds::Contracts::RenderResult.new( + status: :empty, + payload: empty_feed_payload, + message: nil, + ttl_seconds: 600, + cache_key:, + error_message:, + error_kind: + ) + end + + # @return [Html2rss::Web::Feeds::Contracts::RenderPayload] + def empty_feed_payload + Html2rss::Web::Feeds::Contracts::RenderPayload.new( + feed: nil, + site_title: feed_url, + url: feed_url, + strategy: nil ) end @@ -49,14 +89,13 @@ def ghost_feed_token .create_with_validation( username: 'ghost', url: feed_url, - strategy: 'faraday', secret_key: ENV.fetch('HTML2RSS_SECRET_KEY') ) .encode end def valid_feed_token - Html2rss::Web::Auth.generate_feed_token('admin', feed_url, strategy: 'faraday') + Html2rss::Web::Auth.generate_feed_token('admin', feed_url) end def post_feed_request(payload) @@ -201,9 +240,14 @@ def expected_featured_feeds let(:perform_request) { -> { get '/api/v1/health' } } - it_behaves_like 'api error contract', - status: 401, - code: Html2rss::Web::Api::V1::Contract::CODES[:unauthorized] + it_behaves_like 'api error contract', { + status: 401, + code: Html2rss::Web::UnauthorizedError::CODE, + kind: 'auth', + retryable: false, + next_action: 'enter_token', + retry_action: 'none' + } it 'returns health status when token is valid', :aggregate_failures do header 'Authorization', "Bearer #{health_token}" @@ -251,8 +295,12 @@ def expected_featured_feeds expect(last_response.status).to eq(500) json = expect_error_response(last_response, - code: Html2rss::Web::Api::V1::Contract::CODES[:internal_server_error]) - expect(json.dig('error', 'message')).to eq(Html2rss::Web::Api::V1::Contract::MESSAGES[:health_check_failed]) + code: Html2rss::Web::InternalServerError::CODE, + kind: 'server', + retryable: false, + next_action: 'none', + retry_action: 'none') + expect(json.dig('error', 'message')).to eq(Html2rss::Web::HealthCheckFailedError::DEFAULT_MESSAGE) end end @@ -376,6 +424,14 @@ def expected_featured_feeds expect(last_response.body).to include('Invalid token') end + it 'does not expose a feed status endpoint', :aggregate_failures, openapi: false do + get "/api/v1/feeds/#{valid_feed_token}/status" + + expect(last_response.status).to eq(404) + expect(last_response.content_type).to include('application/json') + expect(response_json(last_response).dig('error', 'code')).to eq(Html2rss::Web::NotFoundError::CODE) + end + it 'returns JSON Feed-shaped errors when requested by json extension' do get '/api/v1/feeds/invalid-token.json' @@ -394,7 +450,7 @@ def expected_featured_feeds expect(last_response.status).to eq(403) expect(last_response.content_type).to include('application/xml') - expect(last_response.body).to include(Html2rss::Web::Api::V1::Contract::MESSAGES[:auto_source_disabled]) + expect(last_response.body).to include(Html2rss::Web::AutoSourceDisabledError::DEFAULT_MESSAGE) end it 'returns JSON Feed-shaped forbidden errors when requested through Accept', :aggregate_failures do @@ -433,6 +489,32 @@ def expected_featured_feeds expect([status, content_type, title]).to eq([500, 'application/feed+json', 'Error']) expect(cache_control).to include('no-store') end + + it 'returns 422 for empty extraction feeds in xml representation', :aggregate_failures, openapi: false do + token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/empty-xml", strategy: 'faraday') + allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(extraction_empty_result) + allow(Html2rss::Web::Feeds::RssRenderer).to receive(:call).and_return('') + + get "/api/v1/feeds/#{token}.xml" + + expect(last_response.status).to eq(422) + expect(last_response.content_type).to include('application/xml') + expect(last_response.headers['Cache-Control']).to include('max-age=600') + end + + it 'returns 422 for empty extraction feeds in json feed representation', :aggregate_failures, openapi: false do + token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/empty-json", strategy: 'faraday') + allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(extraction_empty_result) + allow(Html2rss::Web::Feeds::JsonRenderer).to receive(:call) + .and_return('{"version":"https://jsonfeed.org/version/1.1","title":"Content Extraction Issue","items":[]}') + + get "/api/v1/feeds/#{token}.json" + + expect(last_response.status).to eq(422) + expect(last_response.content_type).to eq('application/feed+json') + expect(last_response.headers['Cache-Control']).to include('max-age=600') + expect(JSON.parse(last_response.body).fetch('title')).to eq('Content Extraction Issue') + end end describe 'POST /api/v1/feeds', openapi: { @@ -443,8 +525,7 @@ def expected_featured_feeds } do let(:request_params) do { - url: feed_url, - strategy: 'faraday' + url: feed_url } end @@ -459,9 +540,14 @@ def expected_featured_feeds header 'Authorization', nil end - it_behaves_like 'api error contract', - status: 401, - code: Html2rss::Web::Api::V1::Contract::CODES[:unauthorized] + it_behaves_like 'api error contract', { + status: 401, + code: Html2rss::Web::UnauthorizedError::CODE, + kind: 'auth', + retryable: false, + next_action: 'enter_token', + retry_action: 'none' + } it 'creates a feed when request is valid', :aggregate_failures do header 'Authorization', "Bearer #{admin_token}" @@ -471,18 +557,19 @@ def expected_featured_feeds expect(last_response.status).to eq(201) json = expect_success_response(last_response) expect_feed_payload(json) + expect(json.fetch('data')).not_to have_key('conversion') expect(last_response.headers['Content-Type']).to include('application/json') end it 'normalizes hostname-only input to https before feed creation', :aggregate_failures do - post_feed_request(url: 'example.com/articles', strategy: 'faraday') + post_feed_request(url: 'example.com/articles') expect(last_response.status).to eq(201) json = expect_success_response(last_response) expect(json.dig('data', 'feed', 'url')).to eq('https://example.com/articles') end - it 'returns forbidden for authenticated requests when auto source is disabled', :aggregate_failures do + it 'returns forbidden for authenticated requests when auto source is disabled', :aggregate_failures do # rubocop:disable RSpec/ExampleLength header 'Authorization', "Bearer #{admin_token}" header 'Content-Type', 'application/json' @@ -491,8 +578,15 @@ def expected_featured_feeds end expect(last_response.status).to eq(403) - json = expect_error_response(last_response, code: Html2rss::Web::Api::V1::Contract::CODES[:forbidden]) - expect(json.dig('error', 'message')).to eq(Html2rss::Web::Api::V1::Contract::MESSAGES[:auto_source_disabled]) + json = expect_error_response( + last_response, + code: Html2rss::Web::ForbiddenError::CODE, + kind: 'input', + retryable: false, + next_action: 'correct_input', + retry_action: 'none' + ) + expect(json.dig('error', 'message')).to eq(Html2rss::Web::AutoSourceDisabledError::DEFAULT_MESSAGE) end end end diff --git a/spec/html2rss/web/app_integration_spec.rb b/spec/html2rss/web/app_integration_spec.rb index 86f3a6c4..0766185c 100644 --- a/spec/html2rss/web/app_integration_spec.rb +++ b/spec/html2rss/web/app_integration_spec.rb @@ -46,7 +46,8 @@ message: nil, ttl_seconds: 600, cache_key: 'feed_result:test', - error_message: nil + error_message: nil, + error_kind: nil ) end @@ -65,6 +66,39 @@ .and_return('{"version":"https://jsonfeed.org/version/1.1","items":[]}') end + describe 'GET /create, /token, /result/:token' do # rubocop:disable RSpec/MultipleMemoizedHelpers + it 'returns not found for create route', :aggregate_failures do + get '/create' + + expect(last_response.status).to eq(404) + end + + it 'returns not found for token route', :aggregate_failures do + get '/token' + + expect(last_response.status).to eq(404) + end + + it 'returns not found for result route', :aggregate_failures do + get '/result/generated-token' + + expect(last_response.status).to eq(404) + end + + it 'returns not found for SPA app routes in development mode', :aggregate_failures do + ClimateControl.modify('RACK_ENV' => 'development') do + get '/create' + expect(last_response.status).to eq(404) + + get '/token' + expect(last_response.status).to eq(404) + + get '/result/generated-token' + expect(last_response.status).to eq(404) + end + end + end + describe 'GET /api/v1/feeds/:token' do # rubocop:disable RSpec/MultipleMemoizedHelpers it 'returns unauthorized for invalid tokens' do get '/api/v1/feeds/invalid-token', {}, { 'HTTP_ACCEPT' => 'application/xml' } @@ -182,6 +216,18 @@ ) end + it 'returns 422 when extraction yields an empty feed warning', :aggregate_failures do + unique_empty_url = "#{feed_url}/empty-warning" + empty_token = Html2rss::Web::Auth.generate_feed_token(account[:username], unique_empty_url, strategy: 'faraday') + stub_empty_feed_warning_result + + get "/api/v1/feeds/#{empty_token}.json" + + expect(last_response.status).to eq(422) + expect(last_response.headers['Content-Type']).to eq('application/feed+json') + expect(JSON.parse(last_response.body).fetch('title')).to eq('Content Extraction Issue') + end + # rubocop:disable Metrics/AbcSize, Metrics/MethodLength def stub_escaped_feed_token(raw_token:, encoded_token:) escaped_token_payload = instance_double( @@ -197,14 +243,31 @@ def stub_escaped_feed_token(raw_token:, encoded_token:) .to receive(:validate_and_decode).with(raw_token, feed_url, anything) .and_return(escaped_token_payload) end + + # @return [void] + def stub_empty_feed_warning_result + Html2rss::Web::Feeds::Cache.clear!(reason: 'spec') + allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return( + Html2rss::Web::Feeds::Contracts::RenderResult.new( + status: :empty, + payload: nil, + message: nil, + ttl_seconds: 600, + cache_key: 'feed_result:empty', + error_message: nil, + error_kind: nil + ) + ) + allow(Html2rss::Web::Feeds::JsonRenderer).to receive(:call) + .and_return('{"version":"https://jsonfeed.org/version/1.1","title":"Content Extraction Issue","items":[]}') + end # rubocop:enable Metrics/AbcSize, Metrics/MethodLength end describe 'POST /api/v1/feeds' do # rubocop:disable RSpec/MultipleMemoizedHelpers let(:request_payload) do { - url: feed_url, - strategy: 'faraday' + url: feed_url } end @@ -231,7 +294,15 @@ def stub_escaped_feed_token(raw_token:, encoded_token:) expect(last_response.status).to eq(401) expect(last_response.content_type).to include('application/json') - expect(json_body).to include('error' => include('code' => 'UNAUTHORIZED')) + expect(json_body).to include( + 'error' => include( + 'code' => 'UNAUTHORIZED', + 'kind' => 'auth', + 'retryable' => false, + 'next_action' => 'enter_token', + 'retry_action' => 'none' + ) + ) end end @@ -245,7 +316,15 @@ def stub_escaped_feed_token(raw_token:, encoded_token:) expect(last_response.status).to eq(400) expect(last_response.content_type).to include('application/json') - expect(json_body).to include('error' => include('message' => 'Invalid JSON payload')) + expect(json_body).to include( + 'error' => include( + 'message' => 'Invalid JSON payload', + 'kind' => 'input', + 'retryable' => false, + 'next_action' => 'correct_input', + 'retry_action' => 'none' + ) + ) end it 'returns bad request when URL is missing' do @@ -253,7 +332,13 @@ def stub_escaped_feed_token(raw_token:, encoded_token:) expect(last_response.status).to eq(400) expect(json_body).to include( - 'error' => include('message' => 'URL parameter is required') + 'error' => include( + 'message' => 'URL parameter is required', + 'kind' => 'input', + 'retryable' => false, + 'next_action' => 'correct_input', + 'retry_action' => 'none' + ) ) end @@ -264,16 +349,13 @@ def stub_escaped_feed_token(raw_token:, encoded_token:) expect(last_response.status).to eq(403) expect(json_body).to include( - 'error' => include('message' => 'URL not allowed for this account') - ) - end - - it 'returns bad request for unsupported strategy' do - post '/api/v1/feeds', request_payload.merge(strategy: 'unsupported').to_json, auth_headers - - expect(last_response.status).to eq(400) - expect(json_body).to include( - 'error' => include('message' => 'Unsupported strategy') + 'error' => include( + 'message' => 'URL not allowed for this account', + 'kind' => 'input', + 'retryable' => false, + 'next_action' => 'correct_input', + 'retry_action' => 'none' + ) ) end @@ -284,10 +366,27 @@ def stub_escaped_feed_token(raw_token:, encoded_token:) expect(last_response.status).to eq(500) expect(json_body).to include( - 'error' => include('message' => 'Failed to create feed') + 'error' => include( + 'message' => 'Failed to create feed', + 'kind' => 'server', + 'retryable' => true, + 'next_action' => 'retry', + 'retry_action' => 'primary' + ) ) end + it 'returns corrective extraction-empty failure when auto fallback exhausts' do + no_feed_items_extracted = stub_const('Html2rss::NoFeedItemsExtracted', Class.new(Html2rss::Error)) + allow(Html2rss::Web::AutoSource).to receive(:create_stable_feed) + .and_raise(no_feed_items_extracted, 'No feed items extracted after auto fallback') + + post '/api/v1/feeds', request_payload.to_json, auth_headers + + expect(last_response.status).to eq(422) + expect(json_body).to include('error' => include(extraction_empty_error_fields)) + end + it 'returns created feed metadata' do post '/api/v1/feeds', request_payload.to_json, auth_headers @@ -300,7 +399,21 @@ def stub_escaped_feed_token(raw_token:, encoded_token:) 'public_url' => "/api/v1/feeds/#{feed_token}", 'json_public_url' => "/api/v1/feeds/#{feed_token}.json" ) + expect(json_body.dig('data', 'feed')).not_to have_key('strategy') + expect(json_body.fetch('data')).not_to have_key('conversion') end end end + + # @return [Hash{String=>Object}] + def extraction_empty_error_fields + { + 'code' => Html2rss::Web::ErrorResponder::EXTRACTION_EMPTY_CODE, + 'message' => Html2rss::Web::ErrorResponder::EXTRACTION_EMPTY_MESSAGE, + 'kind' => 'input', + 'retryable' => false, + 'next_action' => 'correct_input', + 'retry_action' => 'none' + } + end end diff --git a/spec/html2rss/web/app_spec.rb b/spec/html2rss/web/app_spec.rb index f34ae32e..cd44b96c 100644 --- a/spec/html2rss/web/app_spec.rb +++ b/spec/html2rss/web/app_spec.rb @@ -36,7 +36,8 @@ def static_feed_result(ttl:) message: nil, ttl_seconds: Html2rss::Web::CacheTtl.seconds_from_minutes(ttl), cache_key: 'feed_result:spec', - error_message: nil + error_message: nil, + error_kind: nil ) end @@ -53,7 +54,8 @@ def static_service_error_result message: 'Internal Server Error', ttl_seconds: 600, cache_key: 'feed_result:error', - error_message: 'upstream timeout' + error_message: 'upstream timeout', + error_kind: :network ) end @@ -87,10 +89,36 @@ def app = described_class get '/' expect(last_response).to be_ok + expect(last_response.body).not_to include('html2rss-web API (development)') expect(last_response.headers['Content-Security-Policy']).to include("default-src 'none'") + expect(last_response.headers['Content-Security-Policy']).to include("script-src 'self'") + expect(last_response.headers['Content-Security-Policy']).to include("style-src 'self'") + expect(last_response.headers['Content-Security-Policy']).not_to include("'unsafe-inline'") expect(last_response.headers['Strict-Transport-Security']).to include('max-age=31536000') end + it 'serves the SPA shell in development when built assets are absent', :aggregate_failures do + allow(File).to receive(:exist?).and_call_original + allow(File).to receive(:exist?).with(Html2rss::Web::App::FRONTEND_DIST_INDEX_PATH).and_return(false) + + ClimateControl.modify('RACK_ENV' => 'development') do + get '/' + end + + expect(last_response).to be_ok + expect(last_response.body).to include('

') + expect(last_response.body).to include('') + end + + it 'does not render SPA app routes as backend paths' do + ClimateControl.modify('RACK_ENV' => 'development') do + get '/create' + end + + expect(last_response.status).to eq(404) + expect(last_response.headers['Content-Type']).to eq('application/xml') + end + it 'does not serve the removed legacy frontend entrypoint' do get '/frontend/index.html' @@ -143,7 +171,6 @@ def app = described_class post '/api/v1/feeds/test-token' expect(last_response.status).to eq(405) - expect(last_response.headers['Allow']).to eq('GET') end it 'coerces string ttl values before cache expiry math' do @@ -155,22 +182,22 @@ def app = described_class expect(last_response.headers['Cache-Control']).to include('max-age=10800') end - it 'renders XML error when static feed generation fails' do + it 'renders XML not found when static feed config is missing' do allow(Html2rss::Web::XmlBuilder).to receive(:build_error_feed).and_return('') get '/missing-feed' - expect(last_response.status).to eq(500) + expect(last_response.status).to eq(404) expect(last_response.headers['Content-Type']).to eq('application/xml') expect(last_response.body).to eq('') end - it 'renders JSON Feed-shaped errors when static json feed generation fails' do + it 'renders JSON Feed-shaped not found errors when static json feed config is missing' do get '/missing-feed.json' expect(json_feed_error_tuple).to eq( - [500, 'application/feed+json', { 'version' => 'https://jsonfeed.org/version/1.1', 'title' => 'Error', - 'description' => 'Failed to generate feed: Internal Server Error' }] + [404, 'application/feed+json', { 'version' => 'https://jsonfeed.org/version/1.1', 'title' => 'Error', + 'description' => 'Failed to generate feed: Not Found' }] ) end @@ -190,9 +217,22 @@ def app = described_class expect(last_response.status).to eq(500) expect(last_response.headers['Content-Type']).to include('application/json') json = JSON.parse(last_response.body) - expect(json.dig('error', 'code')).to eq(Html2rss::Web::Api::V1::Contract::CODES[:internal_server_error]) + expect(json.dig('error', 'code')).to eq(Html2rss::Web::InternalServerError::CODE) expect(json.dig('error', 'message')).to eq('Internal Server Error') end + + it 'keeps API auth failures on the JSON error contract in development mode' do + ClimateControl.modify('RACK_ENV' => 'development') do + post '/api/v1/feeds', JSON.generate(url: 'https://example.com/articles'), + { 'CONTENT_TYPE' => 'application/json', 'HTTP_AUTHORIZATION' => 'Bearer invalid-token' } + end + + expect(last_response.status).to eq(401) + expect(last_response.headers['Content-Type']).to include('application/json') + json = JSON.parse(last_response.body) + expect(json.dig('error', 'code')).to eq(Html2rss::Web::UnauthorizedError::CODE) + expect(json.dig('error', 'message')).to eq('Authentication required') + end end describe '.development?' do diff --git a/spec/html2rss/web/boot/sentry_spec.rb b/spec/html2rss/web/boot/sentry_spec.rb new file mode 100644 index 00000000..4db6b041 --- /dev/null +++ b/spec/html2rss/web/boot/sentry_spec.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Boot::Sentry do + let(:sentry_dsn) { 'https://example@sentry.invalid/1' } + let(:captured_config) { Struct.new(:dsn, :environment, :enable_logs, :send_default_pii, :release).new } + let(:captured_scope) do + Class.new do + attr_reader :tags + + def initialize + @tags = {} + end + + def set_tags(**tags) + @tags = tags + end + end.new + end + let(:fake_sentry) do + config = captured_config + scope = captured_scope + + Module.new.tap do |mod| + mod.define_singleton_method(:initialized?) { false } + mod.define_singleton_method(:init) do |&block| + block.call(config) + end + mod.define_singleton_method(:configure_scope) do |&block| + block.call(scope) + end + end + end + + before do + stub_const('Sentry', fake_sentry) + end + + it 'configures release, environment, and scope tags when a dsn is present', :aggregate_failures do + stub_runtime_env_for_sentry('production') + described_class.send(:initialize_sentry!) + + expect_sentry_configuration + end + + it 'does nothing when a dsn is not present' do + allow(Html2rss::Web::RuntimeEnv).to receive(:sentry_enabled?).and_return(false) + + expect(described_class.send(:configure?)).to be(false) + end + + def stub_runtime_env_for_sentry(rack_env) + allow(Html2rss::Web::RuntimeEnv).to receive_messages( + sentry_enabled?: true, + sentry_dsn: sentry_dsn, + rack_env: rack_env, + build_tag: '2026-03-27', + git_sha: 'abc1234', + sentry_logs_enabled?: false + ) + end + + def expect_sentry_configuration + expect(captured_config).to have_attributes( + dsn: sentry_dsn, + environment: 'production', + enable_logs: false, + send_default_pii: false, + release: '2026-03-27+abc1234' + ) + expect_sentry_scope_tags + end + + def expect_sentry_scope_tags + expect(captured_scope.tags).to eq( + release: '2026-03-27+abc1234', + environment: 'production' + ) + end +end diff --git a/spec/html2rss/web/error_responder_spec.rb b/spec/html2rss/web/error_responder_spec.rb index fe5d7e0f..81aeafe8 100644 --- a/spec/html2rss/web/error_responder_spec.rb +++ b/spec/html2rss/web/error_responder_spec.rb @@ -36,6 +36,17 @@ def api_error_response [response.status, response['Content-Type'], JSON.parse(body)] end + def extraction_empty_api_error_response + no_feed_items_extracted = stub_const('Html2rss::NoFeedItemsExtracted', Class.new(Html2rss::Error)) + response, body = respond_with( + error: no_feed_items_extracted.new('No feed items extracted after auto fallback'), + path: '/api/v1/feeds', + target: Html2rss::Web::RequestTarget::API + ) + + [response.status, response['Content-Type'], JSON.parse(body)] + end + def legacy_error_response allow(Html2rss::Web::XmlBuilder).to receive(:build_error_feed).and_return('') response, body = respond_with( @@ -67,13 +78,17 @@ def xml_preferred_feed_error_response [response['Content-Type'], body.include?('Invalid token')] end - def expected_api_error_response + def expected_api_error_response # rubocop:disable Metrics/MethodLength [500, 'application/json', { 'success' => false, 'error' => { - 'code' => Html2rss::Web::Api::V1::Contract::CODES[:internal_server_error], - 'message' => 'Internal Server Error' + 'code' => Html2rss::Web::InternalServerError::CODE, + 'message' => 'Internal Server Error', + 'kind' => 'server', + 'retryable' => true, + 'next_action' => 'retry', + 'retry_action' => 'primary' } }] end @@ -83,6 +98,12 @@ def expected_api_error_response expect(api_error_response).to eq(expected_api_error_response) end + it 'maps extraction-empty api errors to corrective 422 payloads' do + expect(extraction_empty_api_error_response).to eq( + [422, 'application/json', extraction_empty_api_payload] + ) + end + it 'returns xml error payload for non-api routes' do expect(legacy_error_response).to eq([500, 'application/xml', '']) end @@ -99,4 +120,24 @@ def expected_api_error_response expect(xml_preferred_feed_error_response).to eq(['application/xml', true]) end end + + # @return [Hash{String=>Object}] + def extraction_empty_api_payload + { + 'success' => false, + 'error' => extraction_empty_error_fields + } + end + + # @return [Hash{String=>Object}] + def extraction_empty_error_fields + { + 'code' => Html2rss::Web::ErrorResponder::EXTRACTION_EMPTY_CODE, + 'message' => Html2rss::Web::ErrorResponder::EXTRACTION_EMPTY_MESSAGE, + 'kind' => 'input', + 'retryable' => false, + 'next_action' => 'correct_input', + 'retry_action' => 'none' + } + end end diff --git a/spec/html2rss/web/feed_access_spec.rb b/spec/html2rss/web/feed_access_spec.rb deleted file mode 100644 index 1da5951f..00000000 --- a/spec/html2rss/web/feed_access_spec.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -require_relative '../../../app/web/security/feed_access' - -RSpec.describe Html2rss::Web::FeedAccess do - describe '.url_allowed_for_username?' do - it 'returns true when the user account allows the URL' do - account = { username: 'alice', allowed_urls: ['https://example.com/*'] } - allow(Html2rss::Web::AccountManager).to receive(:get_account_by_username).with('alice').and_return(account) - - expect(described_class.url_allowed_for_username?('alice', 'https://example.com/articles')).to be(true) - end - - it 'returns false when the user account is missing' do - allow(Html2rss::Web::AccountManager).to receive(:get_account_by_username).with('missing').and_return(nil) - - expect(described_class.url_allowed_for_username?('missing', 'https://example.com/articles')).to be(false) - end - end -end diff --git a/spec/html2rss/web/feed_notice_text_spec.rb b/spec/html2rss/web/feed_notice_text_spec.rb new file mode 100644 index 00000000..37649748 --- /dev/null +++ b/spec/html2rss/web/feed_notice_text_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative '../../../app' + +RSpec.describe Html2rss::Web::FeedNoticeText do + describe '.empty_feed_item' do + subject(:message) { described_class.empty_feed_item(url: 'https://example.com/articles') } + + it 'includes actionable product guidance' do + expect(message).to include('What you can do:') + expect(message).to include('Try again in a few moments') + end + + it 'does not mention hidden strategy controls' do + expect(message).not_to include('browserless strategy') + expect(message).not_to include('Try another strategy') + end + end +end diff --git a/spec/html2rss/web/feed_token_spec.rb b/spec/html2rss/web/feed_token_spec.rb new file mode 100644 index 00000000..338ea70c --- /dev/null +++ b/spec/html2rss/web/feed_token_spec.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative '../../../app/web/security/feed_token' +require_relative '../../../app/web/security/url_validator' + +RSpec.describe Html2rss::Web::FeedToken do + describe '.create_with_validation' do + it 'creates a valid feed token' do + token = described_class.create_with_validation( + username: 'alice', + url: 'https://example.com/feed', + secret_key: 'test-secret', + strategy: 'some_strategy' + ) + + expect(token).to be_a(described_class) + expect(token.username).to eq('alice') + end + + it 'stores the normalized attributes' do + token = described_class.create_with_validation( + username: 'alice', + url: 'https://example.com/feed', + secret_key: 'test-secret', + strategy: 'some_strategy' + ) + + expect(token.url).to eq('https://example.com/feed') + expect(token.strategy).to eq('some_strategy') + end + + it 'signs the token' do + token = described_class.create_with_validation( + username: 'alice', + url: 'https://example.com/feed', + secret_key: 'test-secret', + strategy: 'some_strategy' + ) + + expect(token.signature).not_to be_nil + end + + it 'returns nil for invalid username' do + token = described_class.create_with_validation( + username: '', + url: 'https://example.com/feed', + secret_key: 'test-secret' + ) + + expect(token).to be_nil + end + + it 'returns nil for invalid url' do + token = described_class.create_with_validation( + username: 'alice', + url: 'not-a-url', + secret_key: 'test-secret' + ) + + expect(token).to be_nil + end + end + + describe '.validate_and_decode' do + let(:secret_key) { 'test-secret' } + let(:url) { 'https://example.com/feed' } + let(:token) do + described_class.create_with_validation( + username: 'alice', + url:, + secret_key:, + strategy: 'some_strategy' + ) + end + + it 'returns the token when valid' do + expect(described_class.validate_and_decode(token.encode, url, secret_key)).to eq(token) + end + + it 'returns nil for wrong url' do + expect(described_class.validate_and_decode(token.encode, 'https://different.com', secret_key)).to be_nil + end + + it 'returns nil for wrong secret' do + expect(described_class.validate_and_decode(token.encode, url, 'wrong-secret')).to be_nil + end + + it 'returns nil for expired tokens' do + expired = described_class.create_with_validation(username: 'alice', url:, secret_key:, expires_in: -10) + + expect(described_class.validate_and_decode(expired.encode, url, secret_key)).to be_nil + end + end + + describe '.decode' do + let(:token) do + described_class.create_with_validation( + username: 'alice', + url: 'https://example.com/feed', + secret_key: 'test-secret', + strategy: 'some_strategy' + ) + end + + it 'decodes valid payloads' do + expect(described_class.decode(token.encode)).to eq(token) + end + + it 'rejects invalid strings' do + expect(described_class.decode('invalid')).to be_nil + end + + it 'rejects nil payloads' do + expect(described_class.decode(nil)).to be_nil + end + end + + describe '#expired?' do + it 'returns true for past timestamps' do + token = described_class.new('alice', 'https://example.com/feed', Time.now.to_i - 1, 'sig', nil) + + expect(token.expired?).to be(true) + end + + it 'returns false for future timestamps' do + token = described_class.new('alice', 'https://example.com/feed', Time.now.to_i + 3600, 'sig', nil) + + expect(token.expired?).to be(false) + end + end + + describe '#valid_signature?' do + it 'checks the signature against the payload' do + token = described_class.create_with_validation( + username: 'alice', + url: 'https://example.com/feed', + secret_key: 'test-secret' + ) + + expect(token.valid_signature?('test-secret')).to be(true) + expect(token.valid_signature?('wrong-secret')).to be(false) + end + end +end diff --git a/spec/html2rss/web/feeds/cache_spec.rb b/spec/html2rss/web/feeds/cache_spec.rb index 0bcb2b36..cc9c0224 100644 --- a/spec/html2rss/web/feeds/cache_spec.rb +++ b/spec/html2rss/web/feeds/cache_spec.rb @@ -18,7 +18,8 @@ message: nil, ttl_seconds: 60, cache_key: 'feed_result:test', - error_message: nil + error_message: nil, + error_kind: nil ) end diff --git a/spec/html2rss/web/feeds/json_renderer_spec.rb b/spec/html2rss/web/feeds/json_renderer_spec.rb index d14fef9c..15aa5b32 100644 --- a/spec/html2rss/web/feeds/json_renderer_spec.rb +++ b/spec/html2rss/web/feeds/json_renderer_spec.rb @@ -21,7 +21,8 @@ message: nil, ttl_seconds: 600, cache_key: 'feed_result:test', - error_message: nil + error_message: nil, + error_kind: nil ) end diff --git a/spec/html2rss/web/feeds/responder_spec.rb b/spec/html2rss/web/feeds/responder_spec.rb index d1898199..40052713 100644 --- a/spec/html2rss/web/feeds/responder_spec.rb +++ b/spec/html2rss/web/feeds/responder_spec.rb @@ -13,7 +13,8 @@ message: nil, ttl_seconds: 600, cache_key: 'feed_result:test', - error_message: nil + error_message: nil, + error_kind: nil ) end let(:static_config) do @@ -81,7 +82,8 @@ message: 'Internal Server Error', ttl_seconds: 600, cache_key: 'feed_result:error', - error_message: 'timeout' + error_message: 'timeout', + error_kind: :network ) end @@ -102,6 +104,58 @@ end end + context 'with an empty extraction result' do + subject(:write_response) do + described_class.call( + request: request_for(path: '/example.json', accept: 'application/feed+json'), + target_kind: :static, + identifier: 'example.json' + ) + end + + let(:result) do + Html2rss::Web::Feeds::Contracts::RenderResult.new( + status: :empty, + payload: Html2rss::Web::Feeds::Contracts::RenderPayload.new( + feed: nil, + site_title: 'https://example.com', + url: 'https://example.com', + strategy: 'faraday' + ), + message: nil, + ttl_seconds: 600, + cache_key: 'feed_result:empty', + error_message: nil, + error_kind: :extraction_empty + ) + end + + before do + allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(result) + allow(Html2rss::Web::Feeds::JsonRenderer) + .to receive(:call) + .with(result) + .and_return('{"title":"Content Extraction Issue"}') + end + + it 'returns 422 while preserving warning feed payload' do + expect(response_tuple(write_response)).to eq( + [422, 'application/feed+json', '{"title":"Content Extraction Issue"}'] + ) + end + + it 'emits empty extraction as a failure outcome' do + write_response + + expect(Html2rss::Web::Observability).to have_received(:emit).with( + event_name: 'feed.render', + outcome: 'failure', + details: include(strategy: :faraday, url: 'https://example.com', reason: 'content_extraction_empty'), + level: :warn + ) + end + end + context 'when response rendering fails after feed generation succeeds' do subject(:write_response) do described_class.call( diff --git a/spec/html2rss/web/feeds/rss_renderer_spec.rb b/spec/html2rss/web/feeds/rss_renderer_spec.rb index ca07c26a..108b7d40 100644 --- a/spec/html2rss/web/feeds/rss_renderer_spec.rb +++ b/spec/html2rss/web/feeds/rss_renderer_spec.rb @@ -21,7 +21,8 @@ message: nil, ttl_seconds: 600, cache_key: 'feed_result:test', - error_message: nil + error_message: nil, + error_kind: nil ) end diff --git a/spec/html2rss/web/feeds/service_spec.rb b/spec/html2rss/web/feeds/service_spec.rb index 4f085ca4..11c54d06 100644 --- a/spec/html2rss/web/feeds/service_spec.rb +++ b/spec/html2rss/web/feeds/service_spec.rb @@ -11,7 +11,6 @@ source_kind: :static, cache_identity: 'example-feed:abc123', generator_input: { - strategy: :faraday, channel: { url: 'https://example.com/articles' }, auto_source: {} }, @@ -109,7 +108,49 @@ def expected_payload feed: feed, site_title: 'Example Feed', url: 'https://example.com/articles', - strategy: 'faraday' + strategy: nil ) end + + context 'when auto fallback exhausts without feed items' do + let(:no_feed_items_extracted_class) do + stub_const('Html2rss::NoFeedItemsExtracted', Class.new(Html2rss::Error) do + def initialize(attempts:) + @attempts = attempts + super('No feed items extracted after auto fallback') + end + + attr_reader :attempts + end) + end + + before do + allow(Html2rss).to receive(:feed).with(resolved_source.generator_input).and_raise( + no_feed_items_extracted_class.new( + attempts: [ + { strategy: :faraday, items_count: 0, error_class: nil }, + { strategy: :botasaurus, items_count: 0, error_class: nil } + ] + ) + ) + end + + it 'maps the result to empty extraction instead of a server failure', :aggregate_failures do + expect(result.status).to eq(:empty) + expect(result.error_kind).to eq(:extraction_empty) + expect(result.error_message).to include('No feed items extracted after auto fallback') + expect(result.payload).to have_attributes( + url: 'https://example.com/articles', + site_title: 'https://example.com/articles', + strategy: nil + ) + end + + it 'caches the empty result' do + described_class.call(resolved_source) + described_class.call(resolved_source) + + expect(Html2rss).to have_received(:feed).once + end + end end diff --git a/spec/html2rss/web/feeds/source_resolver_spec.rb b/spec/html2rss/web/feeds/source_resolver_spec.rb index d116e63a..e73fb877 100644 --- a/spec/html2rss/web/feeds/source_resolver_spec.rb +++ b/spec/html2rss/web/feeds/source_resolver_spec.rb @@ -29,10 +29,10 @@ def resolved_tuple(resolved) before do allow(Html2rss::Web::LocalConfig).to receive(:find).with('legacy').and_return(config) - allow(Html2rss::RequestService).to receive(:default_strategy_name).and_return(:browserless) end - it 'normalizes the static source into shared generator input', :aggregate_failures do + it 'normalizes the static source into shared generator input without forcing a default strategy', + :aggregate_failures do resolved = described_class.call(feed_request) expect(resolved_tuple(resolved)).to match( @@ -40,9 +40,10 @@ def resolved_tuple(resolved) :static, start_with('static:legacy:'), 900, - include(params: { 'existing' => '1', 'page' => '3' }, strategy: :browserless) + include(params: { 'existing' => '1', 'page' => '3' }) ] ) + expect(resolved.generator_input).not_to have_key(:strategy) end it 'does not mutate the source config hash' do @@ -103,6 +104,55 @@ def resolved_tuple(resolved) include(strategy: :faraday, channel: { url: 'https://example.com/private' }, auto_source: {})] ) end + + it 'defaults blank token strategy to auto', :aggregate_failures do + allow(feed_token).to receive(:strategy).and_return(nil) + + resolved = described_class.call(feed_request) + + expect(resolved.generator_input).to include(channel: { url: 'https://example.com/private' }, auto_source: {}) + expect(resolved.generator_input[:strategy]).to eq(:auto) + end + + it 'defaults blank configured strategy to auto at the generator boundary', :aggregate_failures do + allow(feed_token).to receive(:strategy).and_return(nil) + allow(Html2rss::Config).to receive(:default_strategy_name).and_return(nil) + + resolved = described_class.call(feed_request) + + expect(resolved.generator_input[:strategy]).to eq(:auto) + end + + it 'accepts explicit auto strategy tokens', :aggregate_failures do + allow(feed_token).to receive(:strategy).and_return('auto') + + resolved = described_class.call(feed_request) + + expect(resolved.generator_input[:strategy]).to eq(:auto) + end + + it 'rejects unknown explicit token strategies' do + allow(feed_token).to receive(:strategy).and_return('unknown') + + expect { described_class.call(feed_request) } + .to raise_error(Html2rss::Web::BadRequestError, 'Unsupported strategy') + end + + it 'rejects tokens whose account no longer exists' do + allow(Html2rss::Web::AccountManager).to receive(:get_account_by_username) + .with('admin').and_return(nil) + + expect { described_class.call(feed_request) } + .to raise_error(Html2rss::Web::UnauthorizedError, 'Account not found') + end + + it 'rejects tokens whose URL is no longer allowed' do + allow(Html2rss::Web::UrlValidator).to receive(:url_allowed?) + .with({ username: 'admin' }, 'https://example.com/private').and_return(false) + + expect { described_class.call(feed_request) } + .to raise_error(Html2rss::Web::ForbiddenError, 'Access Denied') + end end end end diff --git a/spec/html2rss/web/json_feed_builder_spec.rb b/spec/html2rss/web/json_feed_builder_spec.rb new file mode 100644 index 00000000..5aede1fe --- /dev/null +++ b/spec/html2rss/web/json_feed_builder_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative '../../../app' + +RSpec.describe Html2rss::Web::JsonFeedBuilder do + describe '.build_empty_feed_warning' do + subject(:payload) do + JSON.parse( + described_class.build_empty_feed_warning( + url: 'https://example.com/articles', + strategy: 'faraday', + site_title: 'Example Site' + ) + ) + end + + it 'uses updated channel description copy' do + expect(payload.fetch('description')).to include('We could not extract entries') + expect(payload.fetch('description')).not_to include('different parser') + end + + it 'uses updated item title and content text' do + first_item = payload.fetch('items').first + expect(first_item.fetch('title')).to eq('Preview unavailable for this source') + expect(first_item.fetch('content_text')).to include('What you can do:') + end + + it 'does not mention hidden strategy controls in item text' do + first_item = payload.fetch('items').first + expect(first_item.fetch('content_text')).not_to include('browserless strategy') + end + end +end diff --git a/spec/html2rss/web/sentry_logs_spec.rb b/spec/html2rss/web/sentry_logs_spec.rb index 27d5bb28..2e4a46eb 100644 --- a/spec/html2rss/web/sentry_logs_spec.rb +++ b/spec/html2rss/web/sentry_logs_spec.rb @@ -3,6 +3,7 @@ require 'spec_helper' require_relative '../../../app/web/config/runtime_env' +require_relative '../../../app/web/telemetry/app_logger' require_relative '../../../app/web/telemetry/sentry_logs' RSpec.describe Html2rss::Web::SentryLogs do @@ -23,6 +24,7 @@ let(:fake_sentry) do Module.new.tap do |mod| mod.define_singleton_method(:logger) { sentry_logger } + mod.define_singleton_method(:add_breadcrumb) { |**| nil } end end let(:raw_payload) do @@ -65,6 +67,22 @@ expect(captured_call).to eq({}) end + it 'adds breadcrumbs for request-critical structured logs even when sentry logs are disabled', :aggregate_failures do + stub_const('Sentry', fake_sentry) + allow(Html2rss::Web::RuntimeEnv).to receive_messages(sentry_enabled?: true, sentry_logs_enabled?: false) + allow(Sentry).to receive(:add_breadcrumb) + + Html2rss::Web::AppLogger.send( + :format_entry, + 'INFO', + Time.now.utc, + nil, + breadcrumb_payload.to_json + ) + + expect(Sentry).to have_received(:add_breadcrumb).with(expected_breadcrumb) + end + it 'falls back to info when an unsupported level is requested', :aggregate_failures do stub_const('Sentry', fake_sentry) allow(Html2rss::Web::RuntimeEnv).to receive_messages(sentry_enabled?: true, sentry_logs_enabled?: true) @@ -80,6 +98,44 @@ def build_sentry_logger logger_class.new(captured_call) end + def breadcrumb_payload + { + event_name: 'feed.create', + outcome: 'failure', + request_id: 'req-123', + route_group: 'api_v1', + strategy: 'faraday', + details: { url: 'https://example.com/articles', fallback: 'browserless' } + } + end + + def expected_breadcrumb + include( + category: 'feed.create', + message: 'feed.create', + level: 'info', + data: breadcrumb_data_matcher + ) + end + + def breadcrumb_data_matcher + include( + event_name: 'feed.create', + outcome: 'failure', + request_id: 'req-123', + route_group: 'api_v1', + strategy: 'faraday', + details: breadcrumb_details_matcher + ) + end + + def breadcrumb_details_matcher + include( + url: include(host: 'example.com', scheme: 'https'), + fallback: 'browserless' + ) + end + def expect_forwarded_payload expect(captured_call).to include(:message, :attributes) expect_forwarded_message diff --git a/spec/html2rss/web/xml_builder_spec.rb b/spec/html2rss/web/xml_builder_spec.rb new file mode 100644 index 00000000..a06d9047 --- /dev/null +++ b/spec/html2rss/web/xml_builder_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'nokogiri' +require_relative '../../../app' + +RSpec.describe Html2rss::Web::XmlBuilder do + describe '.build_empty_feed_warning' do + subject(:xml_doc) do + xml = described_class.build_empty_feed_warning( + url: 'https://example.com/articles', + strategy: 'faraday', + site_title: 'Example Site' + ) + Nokogiri::XML(xml) + end + + it 'uses updated channel description copy' do + description = xml_doc.at_xpath('//channel/description').text + expect(description).to include('We could not extract entries') + expect(description).not_to include('different parser') + end + + it 'uses updated item title and content text' do + expect(xml_doc.at_xpath('//item/title').text).to eq('Preview unavailable for this source') + expect(xml_doc.at_xpath('//item/description').text).to include('What you can do:') + end + + it 'does not mention hidden strategy controls in item text' do + expect(xml_doc.at_xpath('//item/description').text).not_to include('browserless strategy') + end + end +end diff --git a/spec/support/api_contract_helpers.rb b/spec/support/api_contract_helpers.rb index 9b706b0d..c4bb14d5 100644 --- a/spec/support/api_contract_helpers.rb +++ b/spec/support/api_contract_helpers.rb @@ -3,6 +3,13 @@ require 'json' module ApiContractHelpers + OPTIONAL_ERROR_FIELDS = { + kind: 'kind', + retryable: 'retryable', + next_action: 'next_action', + retry_action: 'retry_action' + }.freeze + def response_json(response) JSON.parse(response.body) end @@ -14,16 +21,21 @@ def expect_success_response(response) json end - def expect_error_response(response, code:) + def expect_error_response(response, code:, **expected) json = response_json(response) + error = json.fetch('error') expect(json['success']).to be(false) - expect(json.dig('error', 'code')).to eq(code) + expect(error.fetch('code')).to eq(code) + expect_optional_error_fields(error, expected) yield json if block_given? json end def expect_feed_payload(json) feed = json.fetch('data').fetch('feed') + expect(feed.keys).to contain_exactly( + 'id', 'name', 'url', 'feed_token', 'public_url', 'json_public_url', 'created_at', 'updated_at' + ) expect_feed_identifier_payload(feed) expect_feed_source_payload(feed) feed @@ -37,7 +49,17 @@ def expect_feed_identifier_payload(feed) def expect_feed_source_payload(feed) expect(feed.fetch('url')).to be_a(String) - expect(feed.fetch('strategy')).to be_a(String) + expect(feed.fetch('name')).to be_a(String) + end + + private + + def expect_optional_error_fields(error, expected) + OPTIONAL_ERROR_FIELDS.each do |key, field_name| + next unless expected.key?(key) + + expect(error.fetch(field_name)).to eq(expected[key]) + end end end diff --git a/spec/support/openapi.rb b/spec/support/openapi.rb index 4eaef8a0..788befce 100644 --- a/spec/support/openapi.rb +++ b/spec/support/openapi.rb @@ -158,21 +158,23 @@ normalized_paths[normalized][verb]['description'] ||= normalized_paths[normalized][verb]['summary'] - next unless normalized == '/feeds/{token}' - - normalized_paths[normalized][verb]['parameters'] ||= [] - has_token_param = normalized_paths[normalized][verb]['parameters'].any? do |parameter| - parameter['name'] == 'token' && parameter['in'] == 'path' - end - unless has_token_param - normalized_paths[normalized][verb]['parameters'] << { - 'name' => 'token', - 'in' => 'path', - 'required' => true, - 'schema' => { 'type' => 'string' } - } + if normalized.start_with?('/feeds/{token}') + normalized_paths[normalized][verb]['parameters'] ||= [] + has_token_param = normalized_paths[normalized][verb]['parameters'].any? do |parameter| + parameter['name'] == 'token' && parameter['in'] == 'path' + end + unless has_token_param + normalized_paths[normalized][verb]['parameters'] << { + 'name' => 'token', + 'in' => 'path', + 'required' => true, + 'schema' => { 'type' => 'string' } + } + end end + next unless normalized == '/feeds/{token}' + token_feed_error_statuses.each do |status| response = normalized_paths[normalized][verb].dig('responses', status) next unless response diff --git a/spec/support/shared_examples/api_error_contract_examples.rb b/spec/support/shared_examples/api_error_contract_examples.rb index 08bd1b40..1770cf48 100644 --- a/spec/support/shared_examples/api_error_contract_examples.rb +++ b/spec/support/shared_examples/api_error_contract_examples.rb @@ -1,12 +1,20 @@ # frozen_string_literal: true -RSpec.shared_examples 'api error contract' do |status:, code:, message: nil| - it "returns #{status} with #{code} error payload", :aggregate_failures do +RSpec.shared_examples 'api error contract' do |expected| + it "returns #{expected.fetch(:status)} with #{expected.fetch(:code)} error payload", :aggregate_failures do # rubocop:disable RSpec/ExampleLength perform_request.call - expect(last_response.status).to eq(status) + expect(last_response.status).to eq(expected.fetch(:status)) expect(last_response.content_type).to include('application/json') - json = expect_error_response(last_response, code: code) - expect(json.dig('error', 'message')).to eq(message) if message + json = expect_error_response( + last_response, + code: expected.fetch(:code), + kind: expected.fetch(:kind), + retryable: expected.fetch(:retryable), + next_action: expected.fetch(:next_action), + retry_action: expected.fetch(:retry_action, 'none') + ) + expected_message = expected[:message] + expect(json.dig('error', 'message')).to eq(expected_message) if expected_message end end