Use FlareSolverr for ArtStation import behind Cloudflare
ArtStation blocks plain HTTP; route JSON fetches through FlareSolverr with a direct-fetch fallback.
This commit is contained in:
@@ -16,7 +16,11 @@ import { downloadRemoteImage } from "../src/lib/media.js";
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
const USER_AGENT =
|
||||
"Mozilla/5.0 (compatible; jmartgraphix-importer/1.0; +https://jmartgraphix.com)";
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
||||
|
||||
/** FlareSolverr (or direct fetch) — ArtStation sits behind Cloudflare. */
|
||||
const FLARESOLVERR_URL =
|
||||
process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1";
|
||||
|
||||
function arg(name: string, fallback?: string): string | undefined {
|
||||
const idx = process.argv.indexOf(`--${name}`);
|
||||
@@ -28,9 +32,55 @@ function hasFlag(name: string): boolean {
|
||||
return process.argv.includes(`--${name}`);
|
||||
}
|
||||
|
||||
function extractJsonFromFlareBody(body: string): unknown {
|
||||
// FlareSolverr often wraps JSON in <pre>…</pre>
|
||||
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i);
|
||||
const raw = (pre ? pre[1] : body).trim();
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
|
||||
async function fetchViaFlareSolverr(url: string): Promise<unknown> {
|
||||
const res = await fetch(FLARESOLVERR_URL, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
cmd: "request.get",
|
||||
url,
|
||||
maxTimeout: 90_000,
|
||||
}),
|
||||
signal: AbortSignal.timeout(100_000),
|
||||
});
|
||||
if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`);
|
||||
const payload = (await res.json()) as {
|
||||
status: string;
|
||||
message?: string;
|
||||
solution?: { status: number; response?: string };
|
||||
};
|
||||
if (payload.status !== "ok" || !payload.solution?.response) {
|
||||
throw new Error(
|
||||
`FlareSolverr failed for ${url}: ${payload.message || payload.status}`
|
||||
);
|
||||
}
|
||||
if (payload.solution.status >= 400) {
|
||||
throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`);
|
||||
}
|
||||
return extractJsonFromFlareBody(payload.solution.response);
|
||||
}
|
||||
|
||||
async function fetchJson<T>(url: string): Promise<T> {
|
||||
// Prefer FlareSolverr (Cloudflare). Fall back to direct fetch.
|
||||
try {
|
||||
return (await fetchViaFlareSolverr(url)) as T;
|
||||
} catch (flareErr) {
|
||||
console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`);
|
||||
}
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": USER_AGENT, Accept: "application/json" },
|
||||
headers: {
|
||||
"User-Agent": USER_AGENT,
|
||||
Accept: "application/json, text/plain, */*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
Referer: "https://www.artstation.com/",
|
||||
},
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
|
||||
|
||||
Reference in New Issue
Block a user