Use FlareSolverr for ArtStation import behind Cloudflare
ArtStation blocks plain HTTP; route JSON fetches through FlareSolverr with a direct-fetch fallback.
This commit is contained in:
@@ -16,7 +16,11 @@ import { downloadRemoteImage } from "../src/lib/media.js";
|
|||||||
|
|
||||||
const prisma = new PrismaClient();
|
const prisma = new PrismaClient();
|
||||||
const USER_AGENT =
|
const USER_AGENT =
|
||||||
"Mozilla/5.0 (compatible; jmartgraphix-importer/1.0; +https://jmartgraphix.com)";
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
||||||
|
|
||||||
|
/** FlareSolverr (or direct fetch) — ArtStation sits behind Cloudflare. */
|
||||||
|
const FLARESOLVERR_URL =
|
||||||
|
process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1";
|
||||||
|
|
||||||
function arg(name: string, fallback?: string): string | undefined {
|
function arg(name: string, fallback?: string): string | undefined {
|
||||||
const idx = process.argv.indexOf(`--${name}`);
|
const idx = process.argv.indexOf(`--${name}`);
|
||||||
@@ -28,9 +32,55 @@ function hasFlag(name: string): boolean {
|
|||||||
return process.argv.includes(`--${name}`);
|
return process.argv.includes(`--${name}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function extractJsonFromFlareBody(body: string): unknown {
|
||||||
|
// FlareSolverr often wraps JSON in <pre>…</pre>
|
||||||
|
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i);
|
||||||
|
const raw = (pre ? pre[1] : body).trim();
|
||||||
|
return JSON.parse(raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchViaFlareSolverr(url: string): Promise<unknown> {
|
||||||
|
const res = await fetch(FLARESOLVERR_URL, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
cmd: "request.get",
|
||||||
|
url,
|
||||||
|
maxTimeout: 90_000,
|
||||||
|
}),
|
||||||
|
signal: AbortSignal.timeout(100_000),
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`);
|
||||||
|
const payload = (await res.json()) as {
|
||||||
|
status: string;
|
||||||
|
message?: string;
|
||||||
|
solution?: { status: number; response?: string };
|
||||||
|
};
|
||||||
|
if (payload.status !== "ok" || !payload.solution?.response) {
|
||||||
|
throw new Error(
|
||||||
|
`FlareSolverr failed for ${url}: ${payload.message || payload.status}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (payload.solution.status >= 400) {
|
||||||
|
throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`);
|
||||||
|
}
|
||||||
|
return extractJsonFromFlareBody(payload.solution.response);
|
||||||
|
}
|
||||||
|
|
||||||
async function fetchJson<T>(url: string): Promise<T> {
|
async function fetchJson<T>(url: string): Promise<T> {
|
||||||
|
// Prefer FlareSolverr (Cloudflare). Fall back to direct fetch.
|
||||||
|
try {
|
||||||
|
return (await fetchViaFlareSolverr(url)) as T;
|
||||||
|
} catch (flareErr) {
|
||||||
|
console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`);
|
||||||
|
}
|
||||||
const res = await fetch(url, {
|
const res = await fetch(url, {
|
||||||
headers: { "User-Agent": USER_AGENT, Accept: "application/json" },
|
headers: {
|
||||||
|
"User-Agent": USER_AGENT,
|
||||||
|
Accept: "application/json, text/plain, */*",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
Referer: "https://www.artstation.com/",
|
||||||
|
},
|
||||||
signal: AbortSignal.timeout(30_000),
|
signal: AbortSignal.timeout(30_000),
|
||||||
});
|
});
|
||||||
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
|
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
|
||||||
|
|||||||
Reference in New Issue
Block a user