Hendrik
570a524d86
All checks were successful
Build & Publish Docker Image / build-and-push (push) Successful in 53s
SearXNG returned 403 for every query, logging
'X-Forwarded-For nor X-Real-IP header is set!'. Two fixes, both needed:
1. searxng/settings.yml was being overwritten by SearXNG's default
config in fresh volumes. Explicitly set limiter: false,
public_instance: false, and move secret_key to env lookup via
${SEARXNG_SECRET:-…}. Force a well-known JSON format list.
2. Even with the limiter off, SearXNG's bot detection still nags on
missing forwarder headers. The Node client now sends
X-Forwarded-For: 127.0.0.1, X-Real-IP: 127.0.0.1 and Accept: json
deterministically. Done via a new extraHeaders option on the http
wrapper.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
98 lines
2.7 KiB
TypeScript
98 lines
2.7 KiB
TypeScript
export type FetchOptions = {
|
|
maxBytes?: number;
|
|
timeoutMs?: number;
|
|
userAgent?: string;
|
|
extraHeaders?: Record<string, string>;
|
|
};
|
|
|
|
const DEFAULTS: Required<Omit<FetchOptions, 'extraHeaders'>> = {
|
|
maxBytes: 10 * 1024 * 1024,
|
|
timeoutMs: 10_000,
|
|
userAgent: 'Kochwas/0.1'
|
|
};
|
|
|
|
function assertSafeUrl(url: string): void {
|
|
let u: URL;
|
|
try {
|
|
u = new URL(url);
|
|
} catch {
|
|
throw new Error(`Invalid URL: ${url}`);
|
|
}
|
|
if (u.protocol !== 'http:' && u.protocol !== 'https:') {
|
|
throw new Error(`Unsupported URL scheme: ${u.protocol}`);
|
|
}
|
|
}
|
|
|
|
async function readBody(
|
|
response: Response,
|
|
maxBytes: number
|
|
): Promise<{ data: Uint8Array; total: number }> {
|
|
const reader = response.body?.getReader();
|
|
if (!reader) {
|
|
const buf = new Uint8Array(await response.arrayBuffer());
|
|
if (buf.byteLength > maxBytes) throw new Error(`Response exceeds ${maxBytes} bytes`);
|
|
return { data: buf, total: buf.byteLength };
|
|
}
|
|
const chunks: Uint8Array[] = [];
|
|
let total = 0;
|
|
for (;;) {
|
|
const { value, done } = await reader.read();
|
|
if (done) break;
|
|
if (value) {
|
|
total += value.byteLength;
|
|
if (total > maxBytes) {
|
|
await reader.cancel();
|
|
throw new Error(`Response exceeds ${maxBytes} bytes`);
|
|
}
|
|
chunks.push(value);
|
|
}
|
|
}
|
|
const merged = new Uint8Array(total);
|
|
let offset = 0;
|
|
for (const c of chunks) {
|
|
merged.set(c, offset);
|
|
offset += c.byteLength;
|
|
}
|
|
return { data: merged, total };
|
|
}
|
|
|
|
async function doFetch(url: string, opts: FetchOptions): Promise<Response> {
|
|
assertSafeUrl(url);
|
|
const timeoutMs = opts.timeoutMs ?? DEFAULTS.timeoutMs;
|
|
const userAgent = opts.userAgent ?? DEFAULTS.userAgent;
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
const headers: Record<string, string> = {
|
|
'user-agent': userAgent,
|
|
...(opts.extraHeaders ?? {})
|
|
};
|
|
try {
|
|
const res = await fetch(url, {
|
|
signal: controller.signal,
|
|
redirect: 'follow',
|
|
headers
|
|
});
|
|
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
|
|
return res;
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
export async function fetchText(url: string, opts: FetchOptions = {}): Promise<string> {
|
|
const maxBytes = opts.maxBytes ?? DEFAULTS.maxBytes;
|
|
const res = await doFetch(url, opts);
|
|
const { data } = await readBody(res, maxBytes);
|
|
return new TextDecoder('utf-8').decode(data);
|
|
}
|
|
|
|
export async function fetchBuffer(
|
|
url: string,
|
|
opts: FetchOptions = {}
|
|
): Promise<{ data: Uint8Array; contentType: string | null }> {
|
|
const maxBytes = opts.maxBytes ?? DEFAULTS.maxBytes;
|
|
const res = await doFetch(url, opts);
|
|
const { data } = await readBody(res, maxBytes);
|
|
return { data, contentType: res.headers.get('content-type') };
|
|
}
|