From 570a524d866af2c2160272778244f32118b9a780 Mon Sep 17 00:00:00 2001 From: Hendrik Date: Fri, 17 Apr 2026 16:56:13 +0200 Subject: [PATCH] =?UTF-8?q?fix(search):=20unblock=20SearXNG=20403=20?= =?UTF-8?q?=E2=80=94=20config=20+=20headers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SearXNG returned 403 for every query, logging 'X-Forwarded-For nor X-Real-IP header is set!'. Two fixes, both needed: 1. searxng/settings.yml was being overwritten by SearXNG's default config in fresh volumes. Explicitly set limiter: false, public_instance: false, and move secret_key to env lookup via ${SEARXNG_SECRET:-…}. Force a well-known JSON format list. 2. Even with the limiter off, SearXNG's bot detection still nags on missing forwarder headers. The Node client now sends X-Forwarded-For: 127.0.0.1, X-Real-IP: 127.0.0.1 and Accept: json deterministically. Done via a new extraHeaders option on the http wrapper. Co-Authored-By: Claude Opus 4.7 (1M context) --- searxng/settings.yml | 18 +++++++++++++++++- src/lib/server/http.ts | 26 ++++++++++++++++---------- src/lib/server/search/searxng.ts | 11 ++++++++++- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/searxng/settings.yml b/searxng/settings.yml index 52051cf..174117a 100644 --- a/searxng/settings.yml +++ b/searxng/settings.yml @@ -1,15 +1,31 @@ use_default_settings: true + server: - secret_key: 'dev-secret-change-in-prod' + # In production override via env (see docker-compose.prod.yml). + secret_key: ${SEARXNG_SECRET:-dev-secret-change-in-prod} + # Disables rate limiter + bot detection. This is a private internal service + # called only by kochwas — no public exposure, no abuse risk. limiter: false + public_instance: false image_proxy: false default_http_headers: X-Content-Type-Options: nosniff X-Download-Options: noopen X-Robots-Tag: noindex, nofollow + search: formats: - html - json + safe_search: 0 + autocomplete: '' + default_lang: 'de' + ui: default_locale: de + +# Quieten engines that fail on cold start and aren't useful here +enabled_plugins: + - 'Hash plugin' + - 'Tracker URL remover' + - 'Open Access DOI rewrite' diff --git a/src/lib/server/http.ts b/src/lib/server/http.ts index 2009b9c..9a149fc 100644 --- a/src/lib/server/http.ts +++ b/src/lib/server/http.ts @@ -2,9 +2,10 @@ export type FetchOptions = { maxBytes?: number; timeoutMs?: number; userAgent?: string; + extraHeaders?: Record; }; -const DEFAULTS: Required = { +const DEFAULTS: Required> = { maxBytes: 10 * 1024 * 1024, timeoutMs: 10_000, userAgent: 'Kochwas/0.1' @@ -57,14 +58,19 @@ async function readBody( async function doFetch(url: string, opts: FetchOptions): Promise { assertSafeUrl(url); - const merged = { ...DEFAULTS, ...opts }; + const timeoutMs = opts.timeoutMs ?? DEFAULTS.timeoutMs; + const userAgent = opts.userAgent ?? DEFAULTS.userAgent; const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), merged.timeoutMs); + const timer = setTimeout(() => controller.abort(), timeoutMs); + const headers: Record = { + 'user-agent': userAgent, + ...(opts.extraHeaders ?? {}) + }; try { const res = await fetch(url, { signal: controller.signal, redirect: 'follow', - headers: { 'user-agent': merged.userAgent } + headers }); if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`); return res; @@ -74,9 +80,9 @@ async function doFetch(url: string, opts: FetchOptions): Promise { } export async function fetchText(url: string, opts: FetchOptions = {}): Promise { - const merged = { ...DEFAULTS, ...opts }; - const res = await doFetch(url, merged); - const { data } = await readBody(res, merged.maxBytes); + const maxBytes = opts.maxBytes ?? DEFAULTS.maxBytes; + const res = await doFetch(url, opts); + const { data } = await readBody(res, maxBytes); return new TextDecoder('utf-8').decode(data); } @@ -84,8 +90,8 @@ export async function fetchBuffer( url: string, opts: FetchOptions = {} ): Promise<{ data: Uint8Array; contentType: string | null }> { - const merged = { ...DEFAULTS, ...opts }; - const res = await doFetch(url, merged); - const { data } = await readBody(res, merged.maxBytes); + const maxBytes = opts.maxBytes ?? DEFAULTS.maxBytes; + const res = await doFetch(url, opts); + const { data } = await readBody(res, maxBytes); return { data, contentType: res.headers.get('content-type') }; } diff --git a/src/lib/server/search/searxng.ts b/src/lib/server/search/searxng.ts index 08cb973..ca74508 100644 --- a/src/lib/server/search/searxng.ts +++ b/src/lib/server/search/searxng.ts @@ -96,7 +96,16 @@ export async function searchWeb( endpoint.searchParams.set('format', 'json'); endpoint.searchParams.set('language', 'de'); - const body = await fetchText(endpoint.toString(), { timeoutMs: 15_000 }); + const body = await fetchText(endpoint.toString(), { + timeoutMs: 15_000, + // SearXNG's bot detection complains without these; we are the only caller + // and we're not a bot, so satisfy the check deterministically. + extraHeaders: { + 'X-Forwarded-For': '127.0.0.1', + 'X-Real-IP': '127.0.0.1', + Accept: 'application/json' + } + }); let parsed: SearxngResponse; try { parsed = JSON.parse(body) as SearxngResponse;