/** * ArtStation project importer for jmartgraphix portfolio. * * Usage: * npm run import:artstation -- --user jmartgraphix * npm run import:artstation -- --user jmartgraphix --dry-run * npm run import:artstation -- --url https://www.artstation.com/artwork/XXXX * * Fetches public ArtStation JSON endpoints, downloads images, and creates * draft projects so you can edit/publish from the admin panel afterward. */ import "dotenv/config"; import { PrismaClient, Visibility } from "@prisma/client"; import slugify from "slugify"; import { downloadRemoteImage } from "../src/lib/media.js"; const prisma = new PrismaClient(); const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; /** FlareSolverr (or direct fetch) — ArtStation sits behind Cloudflare. */ const FLARESOLVERR_URL = process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1"; function arg(name: string, fallback?: string): string | undefined { const idx = process.argv.indexOf(`--${name}`); if (idx >= 0 && process.argv[idx + 1]) return process.argv[idx + 1]; return fallback; } function hasFlag(name: string): boolean { return process.argv.includes(`--${name}`); } function extractJsonFromFlareBody(body: string): unknown { // FlareSolverr often wraps JSON in
const pre = body.match(/]*>([\s\S]*?)<\/pre>/i); const raw = (pre ? pre[1] : body).trim(); return JSON.parse(raw); } async function fetchViaFlareSolverr(url: string): Promise { const res = await fetch(FLARESOLVERR_URL, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ cmd: "request.get", url, maxTimeout: 90_000, }), signal: AbortSignal.timeout(100_000), }); if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`); const payload = (await res.json()) as { status: string; message?: string; solution?: { status: number; response?: string }; }; if (payload.status !== "ok" || !payload.solution?.response) { throw new Error( `FlareSolverr failed for ${url}: ${payload.message || payload.status}` ); } if (payload.solution.status >= 400) { throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`); } return extractJsonFromFlareBody(payload.solution.response); } async function fetchJson(url: string): Promise { // Prefer FlareSolverr (Cloudflare). Fall back to direct fetch. try { return (await fetchViaFlareSolverr(url)) as T; } catch (flareErr) { console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`); } const res = await fetch(url, { headers: { "User-Agent": USER_AGENT, Accept: "application/json, text/plain, */*", "Accept-Language": "en-US,en;q=0.9", Referer: "https://www.artstation.com/", }, signal: AbortSignal.timeout(30_000), }); if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`); return res.json() as Promise; } interface AsProjectListItem { id: number; title: string; hash_id?: string; permalink?: string; cover?: { url?: string; small_square_url?: string }; mediums?: { name: string }[]; categories?: { name: string }[]; published_at?: string; } interface AsProjectDetail { id: number; title: string; description?: string; hash_id?: string; permalink?: string; published_at?: string; tags?: string[]; mediums?: { name: string }[]; categories?: { name: string }[]; software_items?: { name: string }[]; assets?: { id: number; title?: string; asset_type?: string; image_url?: string; has_image?: boolean; width?: number; height?: number; }[]; cover?: { url?: string }; user?: { full_name?: string; username?: string }; } // Map ArtStation medium/category names to our category slugs const CATEGORY_MAP: Record = { "3d": "3d-modeling", "3d modeling": "3d-modeling", modeling: "3d-modeling", characters: "3d-modeling", environments: "3d-modeling", animation: "3d-animation", "3d animation": "3d-animation", cad: "cad", product: "cad", industrial: "cad", sculpting: "3d-sculpting", "digital sculpting": "3d-sculpting", ai: "ai-generated-content", "ai art": "ai-generated-content", video: "video", cinematography: "video", photography: "photography", "graphic design": "graphic-design", design: "graphic-design", illustration: "illustration", concept: "illustration", }; async function resolveCategoryIds(names: string[]): Promise { const all = await prisma.category.findMany(); const bySlug = Object.fromEntries(all.map((c) => [c.slug, c.id])); const ids = new Set(); for (const n of names) { const key = n.toLowerCase().trim(); const mapped = CATEGORY_MAP[key]; if (mapped && bySlug[mapped]) ids.add(bySlug[mapped]); const direct = all.find((c) => c.name.toLowerCase() === key || c.slug === slugify(key, { lower: true, strict: true })); if (direct) ids.add(direct.id); } return [...ids]; } async function uniqueSlug(title: string): Promise { const base = slugify(title, { lower: true, strict: true }) || "artwork"; let slug = base; let n = 2; while (await prisma.project.findUnique({ where: { slug } })) { slug = `${base}-${n++}`; } return slug; } async function importProject(hashOrUrl: string, dryRun: boolean): Promise { let hash = hashOrUrl; const m = hashOrUrl.match(/artstation\.com\/artwork\/([A-Za-z0-9]+)/); if (m) hash = m[1]; const detail = await fetchJson( `https://www.artstation.com/projects/${hash}.json` ); const sourceUrl = detail.permalink || `https://www.artstation.com/artwork/${detail.hash_id || hash}`; const existing = await prisma.project.findFirst({ where: { sourceUrl }, }); if (existing) { console.log(` skip (exists): ${detail.title}`); return; } const mediums = (detail.mediums || []).map((m) => m.name); const cats = (detail.categories || []).map((c) => c.name); const software = (detail.software_items || []).map((s) => s.name); const tags = detail.tags || []; const categoryIds = await resolveCategoryIds([...mediums, ...cats]); console.log(` import: ${detail.title} (${(detail.assets || []).length} assets)`); if (dryRun) return; const slug = await uniqueSlug(detail.title); const project = await prisma.project.create({ data: { title: detail.title, slug, description: stripHtml(detail.description || ""), shortDescription: stripHtml(detail.description || "").slice(0, 280) || null, date: detail.published_at ? new Date(detail.published_at) : null, software, externalLinks: [{ label: "ArtStation", url: sourceUrl }], featured: false, displayPriority: 100, visibility: Visibility.draft, sourceUrl, sourcePlatform: "artstation", }, }); for (const cid of categoryIds) { await prisma.projectCategory.create({ data: { projectId: project.id, categoryId: cid }, }); } for (const t of tags) { const tslug = slugify(t, { lower: true, strict: true }); if (!tslug) continue; const tag = await prisma.tag.upsert({ where: { slug: tslug }, create: { name: t, slug: tslug }, update: {}, }); await prisma.projectTag.upsert({ where: { projectId_tagId: { projectId: project.id, tagId: tag.id } }, create: { projectId: project.id, tagId: tag.id }, update: {}, }); } let sortOrder = 0; let thumbnailId: string | null = null; const assets = detail.assets || []; for (const asset of assets) { if (asset.asset_type && asset.asset_type !== "image" && !asset.has_image) continue; const imageUrl = asset.image_url || detail.cover?.url; if (!imageUrl) continue; const saved = await downloadRemoteImage(imageUrl); if (!saved) { console.warn(` failed download: ${imageUrl}`); continue; } const media = await prisma.media.create({ data: { projectId: project.id, type: "image", url: saved.url, thumbnailUrl: saved.thumbnailUrl, filename: saved.filename, mimeType: saved.mimeType, width: saved.width || asset.width, height: saved.height || asset.height, sizeBytes: saved.sizeBytes, alt: asset.title || detail.title, sortOrder: sortOrder++, }, }); if (!thumbnailId) thumbnailId = media.id; } if (thumbnailId) { await prisma.project.update({ where: { id: project.id }, data: { thumbnailId }, }); } console.log(` created draft project ${project.slug}`); } function stripHtml(html: string): string { return html .replace(//gi, "\n") .replace(/<\/p>/gi, "\n\n") .replace(/<[^>]+>/g, "") .replace(/ /g, " ") .replace(/&/g, "&") .replace(/</g, "<") .replace(/>/g, ">") .replace(/"/g, '"') .replace(/\n{3,}/g, "\n\n") .trim(); } async function listUserProjects(username: string): Promise { const all: AsProjectListItem[] = []; let page = 1; for (;;) { // Public profile projects endpoint const url = `https://www.artstation.com/users/${username}/projects.json?page=${page}`; try { const data = await fetchJson<{ data?: AsProjectListItem[] } | AsProjectListItem[]>(url); const batch = Array.isArray(data) ? data : data.data || []; if (batch.length === 0) break; all.push(...batch); if (batch.length < 50) break; page++; if (page > 20) break; } catch (err) { console.error(`Failed to list page ${page}:`, err); break; } } return all; } async function main() { const dryRun = hasFlag("dry-run"); const user = arg("user", "jmartgraphix")!; const singleUrl = arg("url"); console.log(`ArtStation import (user=${user}, dryRun=${dryRun})`); if (singleUrl) { await importProject(singleUrl, dryRun); } else { const list = await listUserProjects(user); console.log(`Found ${list.length} projects`); for (const item of list) { const hash = item.hash_id || String(item.id); try { await importProject(hash, dryRun); await new Promise((r) => setTimeout(r, 500)); // be polite } catch (err) { console.error(` error importing ${item.title}:`, err); } } } console.log("Done. Review drafts in the admin panel and publish when ready."); } main() .catch((e) => { console.error(e); process.exit(1); }) .finally(() => prisma.$disconnect());