Use FlareSolverr for ArtStation import behind Cloudflare

ArtStation blocks plain HTTP; route JSON fetches through FlareSolverr
with a direct-fetch fallback.
This commit is contained in:
2026-07-24 10:44:18 -04:00
parent 1ce004ffd5
commit ac7eb17854
+52 -2
View File
@@ -16,7 +16,11 @@ import { downloadRemoteImage } from "../src/lib/media.js";
const prisma = new PrismaClient(); const prisma = new PrismaClient();
const USER_AGENT = const USER_AGENT =
"Mozilla/5.0 (compatible; jmartgraphix-importer/1.0; +https://jmartgraphix.com)"; "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
/** FlareSolverr (or direct fetch) — ArtStation sits behind Cloudflare. */
const FLARESOLVERR_URL =
process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1";
function arg(name: string, fallback?: string): string | undefined { function arg(name: string, fallback?: string): string | undefined {
const idx = process.argv.indexOf(`--${name}`); const idx = process.argv.indexOf(`--${name}`);
@@ -28,9 +32,55 @@ function hasFlag(name: string): boolean {
return process.argv.includes(`--${name}`); return process.argv.includes(`--${name}`);
} }
function extractJsonFromFlareBody(body: string): unknown {
// FlareSolverr often wraps JSON in <pre>…</pre>
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i);
const raw = (pre ? pre[1] : body).trim();
return JSON.parse(raw);
}
async function fetchViaFlareSolverr(url: string): Promise<unknown> {
const res = await fetch(FLARESOLVERR_URL, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
cmd: "request.get",
url,
maxTimeout: 90_000,
}),
signal: AbortSignal.timeout(100_000),
});
if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`);
const payload = (await res.json()) as {
status: string;
message?: string;
solution?: { status: number; response?: string };
};
if (payload.status !== "ok" || !payload.solution?.response) {
throw new Error(
`FlareSolverr failed for ${url}: ${payload.message || payload.status}`
);
}
if (payload.solution.status >= 400) {
throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`);
}
return extractJsonFromFlareBody(payload.solution.response);
}
async function fetchJson<T>(url: string): Promise<T> { async function fetchJson<T>(url: string): Promise<T> {
// Prefer FlareSolverr (Cloudflare). Fall back to direct fetch.
try {
return (await fetchViaFlareSolverr(url)) as T;
} catch (flareErr) {
console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`);
}
const res = await fetch(url, { const res = await fetch(url, {
headers: { "User-Agent": USER_AGENT, Accept: "application/json" }, headers: {
"User-Agent": USER_AGENT,
Accept: "application/json, text/plain, */*",
"Accept-Language": "en-US,en;q=0.9",
Referer: "https://www.artstation.com/",
},
signal: AbortSignal.timeout(30_000), signal: AbortSignal.timeout(30_000),
}); });
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`); if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);