diff --git a/server/scripts/import-artstation.ts b/server/scripts/import-artstation.ts index 52820df..a075486 100644 --- a/server/scripts/import-artstation.ts +++ b/server/scripts/import-artstation.ts @@ -16,7 +16,11 @@ import { downloadRemoteImage } from "../src/lib/media.js"; const prisma = new PrismaClient(); const USER_AGENT = - "Mozilla/5.0 (compatible; jmartgraphix-importer/1.0; +https://jmartgraphix.com)"; + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; + +/** FlareSolverr (or direct fetch) — ArtStation sits behind Cloudflare. */ +const FLARESOLVERR_URL = + process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1"; function arg(name: string, fallback?: string): string | undefined { const idx = process.argv.indexOf(`--${name}`); @@ -28,9 +32,55 @@ function hasFlag(name: string): boolean { return process.argv.includes(`--${name}`); } +function extractJsonFromFlareBody(body: string): unknown { + // FlareSolverr often wraps JSON in
+ const pre = body.match(/]*>([\s\S]*?)<\/pre>/i); + const raw = (pre ? pre[1] : body).trim(); + return JSON.parse(raw); +} + +async function fetchViaFlareSolverr(url: string): Promise { + const res = await fetch(FLARESOLVERR_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + cmd: "request.get", + url, + maxTimeout: 90_000, + }), + signal: AbortSignal.timeout(100_000), + }); + if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`); + const payload = (await res.json()) as { + status: string; + message?: string; + solution?: { status: number; response?: string }; + }; + if (payload.status !== "ok" || !payload.solution?.response) { + throw new Error( + `FlareSolverr failed for ${url}: ${payload.message || payload.status}` + ); + } + if (payload.solution.status >= 400) { + throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`); + } + return extractJsonFromFlareBody(payload.solution.response); +} + async function fetchJson(url: string): Promise { + // Prefer FlareSolverr (Cloudflare). Fall back to direct fetch. + try { + return (await fetchViaFlareSolverr(url)) as T; + } catch (flareErr) { + console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`); + } const res = await fetch(url, { - headers: { "User-Agent": USER_AGENT, Accept: "application/json" }, + headers: { + "User-Agent": USER_AGENT, + Accept: "application/json, text/plain, */*", + "Accept-Language": "en-US,en;q=0.9", + Referer: "https://www.artstation.com/", + }, signal: AbortSignal.timeout(30_000), }); if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);