Add admin UI for ArtStation re-import

Expose import as a background admin job with status polling, dry-run
and optional publish; share logic with the CLI via a service module.
This commit is contained in:
2026-07-24 10:52:11 -04:00
parent ac7eb17854
commit cef5738044
5 changed files with 718 additions and 324 deletions
+35
View File
@@ -229,8 +229,43 @@ export const api = {
}), }),
reindex: () => reindex: () =>
request<{ ok: boolean; indexed: number }>("/api/v1/admin/reindex", { method: "POST" }), request<{ ok: boolean; indexed: number }>("/api/v1/admin/reindex", { method: "POST" }),
importArtStation: (body: {
username?: string;
dryRun?: boolean;
publish?: boolean;
url?: string;
}) =>
request<{
ok: boolean;
message: string;
status: ArtStationImportStatus;
}>("/api/v1/admin/import/artstation", {
method: "POST",
body: JSON.stringify(body),
}),
importArtStationStatus: () =>
request<ArtStationImportStatus>("/api/v1/admin/import/artstation"),
}; };
export interface ArtStationImportStatus {
running: boolean;
startedAt?: string;
finishedAt?: string;
error?: string;
log: string[];
result?: {
username: string;
dryRun: boolean;
found: number;
imported: number;
skipped: number;
errors: number;
created: { title: string; slug: string }[];
};
}
export function mediaSrc(url?: string | null): string { export function mediaSrc(url?: string | null): string {
if (!url) return ""; if (!url) return "";
if (url.startsWith("http") || url.startsWith("//")) return url; if (url.startsWith("http") || url.startsWith("//")) return url;
+160 -5
View File
@@ -1,6 +1,6 @@
import { useEffect, useState } from "react"; import { useEffect, useRef, useState } from "react";
import { Link } from "react-router-dom"; import { Link } from "react-router-dom";
import { api } from "../../lib/api"; import { api, type ArtStationImportStatus } from "../../lib/api";
export function AdminDashboard() { export function AdminDashboard() {
const [stats, setStats] = useState({ const [stats, setStats] = useState({
@@ -11,8 +11,15 @@ export function AdminDashboard() {
views: 0, views: 0,
}); });
const [msg, setMsg] = useState<string | null>(null); const [msg, setMsg] = useState<string | null>(null);
const [err, setErr] = useState<string | null>(null);
useEffect(() => { const [asUser, setAsUser] = useState("jmartgraphix");
const [asDryRun, setAsDryRun] = useState(false);
const [asPublish, setAsPublish] = useState(false);
const [importStatus, setImportStatus] = useState<ArtStationImportStatus | null>(null);
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
function refreshStats() {
Promise.all([ Promise.all([
api.adminProjects({ perPage: 1 }), api.adminProjects({ perPage: 1 }),
api.adminProjects({ visibility: "published", perPage: 1 }), api.adminProjects({ visibility: "published", perPage: 1 }),
@@ -28,22 +35,77 @@ export function AdminDashboard() {
views: views.length, views: views.length,
}); });
}); });
}
useEffect(() => {
refreshStats();
api.importArtStationStatus().then(setImportStatus).catch(() => {});
return () => {
if (pollRef.current) clearInterval(pollRef.current);
};
}, []); }, []);
function startPolling() {
if (pollRef.current) clearInterval(pollRef.current);
pollRef.current = setInterval(async () => {
try {
const s = await api.importArtStationStatus();
setImportStatus(s);
if (!s.running) {
if (pollRef.current) clearInterval(pollRef.current);
pollRef.current = null;
refreshStats();
if (s.result) {
setMsg(
`ArtStation import finished: ${s.result.imported} new, ${s.result.skipped} skipped, ${s.result.errors} errors (${s.result.found} on profile)`
);
} else if (s.error) {
setErr(s.error);
}
}
} catch {
/* keep polling */
}
}, 2000);
}
async function reindex() { async function reindex() {
setErr(null);
try { try {
const r = await api.reindex(); const r = await api.reindex();
setMsg(`Typesense reindexed ${r.indexed} projects`); setMsg(`Typesense reindexed ${r.indexed} projects`);
} catch (e) { } catch (e) {
setMsg((e as Error).message); setErr((e as Error).message);
} }
} }
async function runArtStationImport() {
setErr(null);
setMsg(null);
try {
const res = await api.importArtStation({
username: asUser.trim() || "jmartgraphix",
dryRun: asDryRun,
publish: asPublish,
});
setImportStatus(res.status);
setMsg(res.message);
if (res.status.running) startPolling();
} catch (e) {
setErr((e as Error).message);
}
}
const running = importStatus?.running;
const logLines = importStatus?.log?.slice(-40) || [];
return ( return (
<div className="admin-page"> <div className="admin-page">
<h1>Dashboard</h1> <h1>Dashboard</h1>
<p className="admin-page__sub">Manage portfolio content protected by Authelia SSO.</p> <p className="admin-page__sub">Manage portfolio content protected by Authelia SSO.</p>
{msg && <div className="admin-msg admin-msg--ok">{msg}</div>} {msg && <div className="admin-msg admin-msg--ok">{msg}</div>}
{err && <div className="admin-msg admin-msg--err">{err}</div>}
<div className="admin-cards"> <div className="admin-cards">
<div className="admin-stat"> <div className="admin-stat">
<strong>{stats.projects}</strong> <strong>{stats.projects}</strong>
@@ -66,6 +128,7 @@ export function AdminDashboard() {
<span>Portfolio views</span> <span>Portfolio views</span>
</div> </div>
</div> </div>
<div className="admin-page__toolbar"> <div className="admin-page__toolbar">
<Link to="/admin/projects/new" className="btn btn--primary"> <Link to="/admin/projects/new" className="btn btn--primary">
New project New project
@@ -77,7 +140,99 @@ export function AdminDashboard() {
Reindex search Reindex search
</button> </button>
</div> </div>
<section>
<section className="admin-import" style={{ marginTop: "2.5rem", maxWidth: 720 }}>
<h2 style={{ fontSize: "1.1rem", margin: "0 0 0.35rem" }}>Import from ArtStation</h2>
<p style={{ color: "var(--text-muted)", margin: "0 0 1rem", fontSize: "0.9rem" }}>
Pull new public projects from your ArtStation profile. Existing entries (matched by
ArtStation URL) are skipped. New work lands as drafts unless you choose publish.
</p>
<div className="field">
<label htmlFor="as-user">ArtStation username</label>
<input
id="as-user"
value={asUser}
onChange={(e) => setAsUser(e.target.value)}
disabled={!!running}
placeholder="jmartgraphix"
/>
</div>
<div className="admin-check-grid" style={{ marginBottom: "1rem" }}>
<label>
<input
type="checkbox"
checked={asDryRun}
disabled={!!running}
onChange={(e) => setAsDryRun(e.target.checked)}
/>
Dry run (list only, no writes)
</label>
<label>
<input
type="checkbox"
checked={asPublish}
disabled={!!running || asDryRun}
onChange={(e) => setAsPublish(e.target.checked)}
/>
Publish new imports immediately
</label>
</div>
<div className="admin-page__toolbar">
<button
type="button"
className="btn btn--primary"
disabled={!!running}
onClick={runArtStationImport}
>
{running ? "Import running…" : "Import from ArtStation"}
</button>
{importStatus?.finishedAt && !running && (
<span style={{ color: "var(--text-dim)", fontSize: "0.85rem" }}>
Last run {new Date(importStatus.finishedAt).toLocaleString()}
</span>
)}
</div>
{importStatus?.result && !running && (
<div
className="admin-msg admin-msg--ok"
style={{ marginTop: "1rem", whiteSpace: "pre-wrap" }}
>
Found {importStatus.result.found} · imported {importStatus.result.imported} · skipped{" "}
{importStatus.result.skipped} · errors {importStatus.result.errors}
{importStatus.result.created.length > 0 && (
<>
{"\n"}New:{" "}
{importStatus.result.created.map((c) => c.title).join(", ")}
</>
)}
</div>
)}
{logLines.length > 0 && (
<pre
style={{
marginTop: "1rem",
padding: "0.85rem 1rem",
background: "var(--bg-elevated)",
border: "1px solid var(--border)",
borderRadius: "var(--radius-sm)",
fontSize: "0.78rem",
lineHeight: 1.45,
maxHeight: 280,
overflow: "auto",
color: "var(--text-muted)",
}}
>
{logLines.join("\n")}
</pre>
)}
</section>
<section style={{ marginTop: "2.5rem" }}>
<h2 style={{ fontSize: "1rem", color: "var(--text-muted)" }}>Quick links for audiences</h2> <h2 style={{ fontSize: "1rem", color: "var(--text-muted)" }}>Quick links for audiences</h2>
<ul style={{ color: "var(--text-dim)", lineHeight: 1.9 }}> <ul style={{ color: "var(--text-dim)", lineHeight: 1.9 }}>
<li> <li>
+14 -319
View File
@@ -1,26 +1,17 @@
/** /**
* ArtStation project importer for jmartgraphix portfolio. * ArtStation project importer CLI.
* *
* Usage: * Usage:
* npm run import:artstation -- --user jmartgraphix * npm run import:artstation -- --user jmartgraphix
* npm run import:artstation -- --user jmartgraphix --dry-run * npm run import:artstation -- --user jmartgraphix --dry-run
* npm run import:artstation -- --url https://www.artstation.com/artwork/XXXX * npm run import:artstation -- --url https://www.artstation.com/artwork/XXXX
* npm run import:artstation -- --user jmartgraphix --publish
* *
* Fetches public ArtStation JSON endpoints, downloads images, and creates * Prefer the Admin → Dashboard “Import from ArtStation” button in production.
* draft projects so you can edit/publish from the admin panel afterward.
*/ */
import "dotenv/config"; import "dotenv/config";
import { PrismaClient, Visibility } from "@prisma/client"; import { runArtStationImport } from "../src/services/artstation-import.js";
import slugify from "slugify"; import { prisma } from "../src/lib/prisma.js";
import { downloadRemoteImage } from "../src/lib/media.js";
const prisma = new PrismaClient();
const USER_AGENT =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
/** FlareSolverr (or direct fetch) — ArtStation sits behind Cloudflare. */
const FLARESOLVERR_URL =
process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1";
function arg(name: string, fallback?: string): string | undefined { function arg(name: string, fallback?: string): string | undefined {
const idx = process.argv.indexOf(`--${name}`); const idx = process.argv.indexOf(`--${name}`);
@@ -32,312 +23,16 @@ function hasFlag(name: string): boolean {
return process.argv.includes(`--${name}`); return process.argv.includes(`--${name}`);
} }
function extractJsonFromFlareBody(body: string): unknown {
// FlareSolverr often wraps JSON in <pre>…</pre>
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i);
const raw = (pre ? pre[1] : body).trim();
return JSON.parse(raw);
}
async function fetchViaFlareSolverr(url: string): Promise<unknown> {
const res = await fetch(FLARESOLVERR_URL, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
cmd: "request.get",
url,
maxTimeout: 90_000,
}),
signal: AbortSignal.timeout(100_000),
});
if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`);
const payload = (await res.json()) as {
status: string;
message?: string;
solution?: { status: number; response?: string };
};
if (payload.status !== "ok" || !payload.solution?.response) {
throw new Error(
`FlareSolverr failed for ${url}: ${payload.message || payload.status}`
);
}
if (payload.solution.status >= 400) {
throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`);
}
return extractJsonFromFlareBody(payload.solution.response);
}
async function fetchJson<T>(url: string): Promise<T> {
// Prefer FlareSolverr (Cloudflare). Fall back to direct fetch.
try {
return (await fetchViaFlareSolverr(url)) as T;
} catch (flareErr) {
console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`);
}
const res = await fetch(url, {
headers: {
"User-Agent": USER_AGENT,
Accept: "application/json, text/plain, */*",
"Accept-Language": "en-US,en;q=0.9",
Referer: "https://www.artstation.com/",
},
signal: AbortSignal.timeout(30_000),
});
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
return res.json() as Promise<T>;
}
interface AsProjectListItem {
id: number;
title: string;
hash_id?: string;
permalink?: string;
cover?: { url?: string; small_square_url?: string };
mediums?: { name: string }[];
categories?: { name: string }[];
published_at?: string;
}
interface AsProjectDetail {
id: number;
title: string;
description?: string;
hash_id?: string;
permalink?: string;
published_at?: string;
tags?: string[];
mediums?: { name: string }[];
categories?: { name: string }[];
software_items?: { name: string }[];
assets?: {
id: number;
title?: string;
asset_type?: string;
image_url?: string;
has_image?: boolean;
width?: number;
height?: number;
}[];
cover?: { url?: string };
user?: { full_name?: string; username?: string };
}
// Map ArtStation medium/category names to our category slugs
const CATEGORY_MAP: Record<string, string> = {
"3d": "3d-modeling",
"3d modeling": "3d-modeling",
modeling: "3d-modeling",
characters: "3d-modeling",
environments: "3d-modeling",
animation: "3d-animation",
"3d animation": "3d-animation",
cad: "cad",
product: "cad",
industrial: "cad",
sculpting: "3d-sculpting",
"digital sculpting": "3d-sculpting",
ai: "ai-generated-content",
"ai art": "ai-generated-content",
video: "video",
cinematography: "video",
photography: "photography",
"graphic design": "graphic-design",
design: "graphic-design",
illustration: "illustration",
concept: "illustration",
};
async function resolveCategoryIds(names: string[]): Promise<string[]> {
const all = await prisma.category.findMany();
const bySlug = Object.fromEntries(all.map((c) => [c.slug, c.id]));
const ids = new Set<string>();
for (const n of names) {
const key = n.toLowerCase().trim();
const mapped = CATEGORY_MAP[key];
if (mapped && bySlug[mapped]) ids.add(bySlug[mapped]);
const direct = all.find((c) => c.name.toLowerCase() === key || c.slug === slugify(key, { lower: true, strict: true }));
if (direct) ids.add(direct.id);
}
return [...ids];
}
async function uniqueSlug(title: string): Promise<string> {
const base = slugify(title, { lower: true, strict: true }) || "artwork";
let slug = base;
let n = 2;
while (await prisma.project.findUnique({ where: { slug } })) {
slug = `${base}-${n++}`;
}
return slug;
}
async function importProject(hashOrUrl: string, dryRun: boolean): Promise<void> {
let hash = hashOrUrl;
const m = hashOrUrl.match(/artstation\.com\/artwork\/([A-Za-z0-9]+)/);
if (m) hash = m[1];
const detail = await fetchJson<AsProjectDetail>(
`https://www.artstation.com/projects/${hash}.json`
);
const sourceUrl =
detail.permalink || `https://www.artstation.com/artwork/${detail.hash_id || hash}`;
const existing = await prisma.project.findFirst({
where: { sourceUrl },
});
if (existing) {
console.log(` skip (exists): ${detail.title}`);
return;
}
const mediums = (detail.mediums || []).map((m) => m.name);
const cats = (detail.categories || []).map((c) => c.name);
const software = (detail.software_items || []).map((s) => s.name);
const tags = detail.tags || [];
const categoryIds = await resolveCategoryIds([...mediums, ...cats]);
console.log(` import: ${detail.title} (${(detail.assets || []).length} assets)`);
if (dryRun) return;
const slug = await uniqueSlug(detail.title);
const project = await prisma.project.create({
data: {
title: detail.title,
slug,
description: stripHtml(detail.description || ""),
shortDescription: stripHtml(detail.description || "").slice(0, 280) || null,
date: detail.published_at ? new Date(detail.published_at) : null,
software,
externalLinks: [{ label: "ArtStation", url: sourceUrl }],
featured: false,
displayPriority: 100,
visibility: Visibility.draft,
sourceUrl,
sourcePlatform: "artstation",
},
});
for (const cid of categoryIds) {
await prisma.projectCategory.create({
data: { projectId: project.id, categoryId: cid },
});
}
for (const t of tags) {
const tslug = slugify(t, { lower: true, strict: true });
if (!tslug) continue;
const tag = await prisma.tag.upsert({
where: { slug: tslug },
create: { name: t, slug: tslug },
update: {},
});
await prisma.projectTag.upsert({
where: { projectId_tagId: { projectId: project.id, tagId: tag.id } },
create: { projectId: project.id, tagId: tag.id },
update: {},
});
}
let sortOrder = 0;
let thumbnailId: string | null = null;
const assets = detail.assets || [];
for (const asset of assets) {
if (asset.asset_type && asset.asset_type !== "image" && !asset.has_image) continue;
const imageUrl = asset.image_url || detail.cover?.url;
if (!imageUrl) continue;
const saved = await downloadRemoteImage(imageUrl);
if (!saved) {
console.warn(` failed download: ${imageUrl}`);
continue;
}
const media = await prisma.media.create({
data: {
projectId: project.id,
type: "image",
url: saved.url,
thumbnailUrl: saved.thumbnailUrl,
filename: saved.filename,
mimeType: saved.mimeType,
width: saved.width || asset.width,
height: saved.height || asset.height,
sizeBytes: saved.sizeBytes,
alt: asset.title || detail.title,
sortOrder: sortOrder++,
},
});
if (!thumbnailId) thumbnailId = media.id;
}
if (thumbnailId) {
await prisma.project.update({
where: { id: project.id },
data: { thumbnailId },
});
}
console.log(` created draft project ${project.slug}`);
}
function stripHtml(html: string): string {
return html
.replace(/<br\s*\/?>/gi, "\n")
.replace(/<\/p>/gi, "\n\n")
.replace(/<[^>]+>/g, "")
.replace(/&nbsp;/g, " ")
.replace(/&amp;/g, "&")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/\n{3,}/g, "\n\n")
.trim();
}
async function listUserProjects(username: string): Promise<AsProjectListItem[]> {
const all: AsProjectListItem[] = [];
let page = 1;
for (;;) {
// Public profile projects endpoint
const url = `https://www.artstation.com/users/${username}/projects.json?page=${page}`;
try {
const data = await fetchJson<{ data?: AsProjectListItem[] } | AsProjectListItem[]>(url);
const batch = Array.isArray(data) ? data : data.data || [];
if (batch.length === 0) break;
all.push(...batch);
if (batch.length < 50) break;
page++;
if (page > 20) break;
} catch (err) {
console.error(`Failed to list page ${page}:`, err);
break;
}
}
return all;
}
async function main() { async function main() {
const dryRun = hasFlag("dry-run"); const result = await runArtStationImport({
const user = arg("user", "jmartgraphix")!; username: arg("user", "jmartgraphix"),
const singleUrl = arg("url"); url: arg("url"),
dryRun: hasFlag("dry-run"),
console.log(`ArtStation import (user=${user}, dryRun=${dryRun})`); publish: hasFlag("publish"),
});
if (singleUrl) { console.log(
await importProject(singleUrl, dryRun); `Summary: found=${result.found} imported=${result.imported} skipped=${result.skipped} errors=${result.errors}`
} else { );
const list = await listUserProjects(user);
console.log(`Found ${list.length} projects`);
for (const item of list) {
const hash = item.hash_id || String(item.id);
try {
await importProject(hash, dryRun);
await new Promise((r) => setTimeout(r, 500)); // be polite
} catch (err) {
console.error(` error importing ${item.title}:`, err);
}
}
}
console.log("Done. Review drafts in the admin panel and publish when ready.");
} }
main() main()
+39
View File
@@ -4,6 +4,10 @@ import { prisma } from "../lib/prisma.js";
import { requireAdmin } from "../lib/auth.js"; import { requireAdmin } from "../lib/auth.js";
import { uniqueCategorySlug, uniqueTagSlug, makeSlug } from "../lib/slug.js"; import { uniqueCategorySlug, uniqueTagSlug, makeSlug } from "../lib/slug.js";
import { reindexAllProjects } from "../lib/typesense.js"; import { reindexAllProjects } from "../lib/typesense.js";
import {
getImportJobStatus,
startArtStationImportJob,
} from "../services/artstation-import.js";
export async function adminMetaRoutes(app: FastifyInstance) { export async function adminMetaRoutes(app: FastifyInstance) {
// Categories // Categories
@@ -279,4 +283,39 @@ export async function adminMetaRoutes(app: FastifyInstance) {
const count = await reindexAllProjects(); const count = await reindexAllProjects();
return { ok: true, indexed: count }; return { ok: true, indexed: count };
}); });
// ArtStation import
app.get("/api/v1/admin/import/artstation", async (req, reply) => {
if (!requireAdmin(req, reply)) return;
return getImportJobStatus();
});
app.post("/api/v1/admin/import/artstation", async (req, reply) => {
if (!requireAdmin(req, reply)) return;
const body = z
.object({
username: z.string().min(1).optional().default("jmartgraphix"),
dryRun: z.boolean().optional().default(false),
publish: z.boolean().optional().default(false),
url: z.string().url().optional(),
})
.parse(req.body ?? {});
const { started, message } = startArtStationImportJob({
username: body.username,
dryRun: body.dryRun,
publish: body.publish,
url: body.url,
defaultCad: true,
});
if (!started) {
return reply.code(409).send({ ok: false, message, status: getImportJobStatus() });
}
return reply.code(202).send({
ok: true,
message,
status: getImportJobStatus(),
});
});
} }
+470
View File
@@ -0,0 +1,470 @@
/**
* ArtStation portfolio importer (shared by CLI + admin API).
* Uses FlareSolverr when available to bypass Cloudflare.
*/
import { Visibility } from "@prisma/client";
import slugify from "slugify";
import { prisma } from "../lib/prisma.js";
import { downloadRemoteImage } from "../lib/media.js";
import { indexProject, reindexAllProjects } from "../lib/typesense.js";
const USER_AGENT =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
const FLARESOLVERR_URL =
process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1";
const CATEGORY_MAP: Record<string, string> = {
"3d": "3d-modeling",
"3d modeling": "3d-modeling",
modeling: "3d-modeling",
characters: "3d-modeling",
environments: "3d-modeling",
animation: "3d-animation",
"3d animation": "3d-animation",
cad: "cad",
product: "cad",
industrial: "cad",
sculpting: "3d-sculpting",
"digital sculpting": "3d-sculpting",
ai: "ai-generated-content",
"ai art": "ai-generated-content",
video: "video",
cinematography: "video",
photography: "photography",
"graphic design": "graphic-design",
design: "graphic-design",
illustration: "illustration",
concept: "illustration",
};
export interface ImportOptions {
username?: string;
url?: string;
dryRun?: boolean;
/** Publish new imports instead of leaving as draft */
publish?: boolean;
/** Assign CAD when no category matched */
defaultCad?: boolean;
onLog?: (line: string) => void;
}
export interface ImportResult {
username: string;
dryRun: boolean;
found: number;
imported: number;
skipped: number;
errors: number;
log: string[];
created: { title: string; slug: string }[];
}
export type ImportJobStatus = {
running: boolean;
startedAt?: string;
finishedAt?: string;
result?: ImportResult;
error?: string;
log: string[];
};
let currentJob: ImportJobStatus = { running: false, log: [] };
export function getImportJobStatus(): ImportJobStatus {
return { ...currentJob, log: [...currentJob.log] };
}
function log(lines: string[], onLog: ImportOptions["onLog"], line: string) {
lines.push(line);
currentJob.log.push(line);
onLog?.(line);
console.log(line);
}
function extractJsonFromFlareBody(body: string): unknown {
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i);
const raw = (pre ? pre[1] : body).trim();
return JSON.parse(raw);
}
async function fetchViaFlareSolverr(url: string): Promise<unknown> {
const res = await fetch(FLARESOLVERR_URL, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
cmd: "request.get",
url,
maxTimeout: 90_000,
}),
signal: AbortSignal.timeout(100_000),
});
if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`);
const payload = (await res.json()) as {
status: string;
message?: string;
solution?: { status: number; response?: string };
};
if (payload.status !== "ok" || !payload.solution?.response) {
throw new Error(
`FlareSolverr failed for ${url}: ${payload.message || payload.status}`
);
}
if (payload.solution.status >= 400) {
throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`);
}
return extractJsonFromFlareBody(payload.solution.response);
}
async function fetchJson<T>(url: string): Promise<T> {
try {
return (await fetchViaFlareSolverr(url)) as T;
} catch (flareErr) {
console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`);
}
const res = await fetch(url, {
headers: {
"User-Agent": USER_AGENT,
Accept: "application/json, text/plain, */*",
"Accept-Language": "en-US,en;q=0.9",
Referer: "https://www.artstation.com/",
},
signal: AbortSignal.timeout(30_000),
});
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
return res.json() as Promise<T>;
}
interface AsProjectListItem {
id: number;
title: string;
hash_id?: string;
permalink?: string;
}
interface AsProjectDetail {
id: number;
title: string;
description?: string;
hash_id?: string;
permalink?: string;
published_at?: string;
tags?: string[];
mediums?: { name: string }[];
categories?: { name: string }[];
software_items?: { name: string }[];
assets?: {
id: number;
title?: string;
asset_type?: string;
image_url?: string;
has_image?: boolean;
width?: number;
height?: number;
}[];
cover?: { url?: string };
}
function stripHtml(html: string): string {
return html
.replace(/<br\s*\/?>/gi, "\n")
.replace(/<\/p>/gi, "\n\n")
.replace(/<[^>]+>/g, "")
.replace(/&nbsp;/g, " ")
.replace(/&amp;/g, "&")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/\n{3,}/g, "\n\n")
.trim();
}
async function resolveCategoryIds(names: string[]): Promise<string[]> {
const all = await prisma.category.findMany();
const bySlug = Object.fromEntries(all.map((c) => [c.slug, c.id]));
const ids = new Set<string>();
for (const n of names) {
const key = n.toLowerCase().trim();
const mapped = CATEGORY_MAP[key];
if (mapped && bySlug[mapped]) ids.add(bySlug[mapped]);
const direct = all.find(
(c) =>
c.name.toLowerCase() === key ||
c.slug === slugify(key, { lower: true, strict: true })
);
if (direct) ids.add(direct.id);
}
return [...ids];
}
async function uniqueSlug(title: string): Promise<string> {
const base = slugify(title, { lower: true, strict: true }) || "artwork";
let slug = base;
let n = 2;
while (await prisma.project.findUnique({ where: { slug } })) {
slug = `${base}-${n++}`;
}
return slug;
}
type ImportProjectOutcome = "imported" | "skipped" | "error";
async function importProject(
hashOrUrl: string,
opts: ImportOptions,
lines: string[]
): Promise<{ outcome: ImportProjectOutcome; title?: string; slug?: string }> {
const onLog = opts.onLog;
let hash = hashOrUrl;
const m = hashOrUrl.match(/artstation\.com\/artwork\/([A-Za-z0-9]+)/);
if (m) hash = m[1];
const detail = await fetchJson<AsProjectDetail>(
`https://www.artstation.com/projects/${hash}.json`
);
const sourceUrl =
detail.permalink ||
`https://www.artstation.com/artwork/${detail.hash_id || hash}`;
const existing = await prisma.project.findFirst({ where: { sourceUrl } });
if (existing) {
log(lines, onLog, ` skip (exists): ${detail.title}`);
return { outcome: "skipped", title: detail.title, slug: existing.slug };
}
const mediums = (detail.mediums || []).map((x) => x.name);
const cats = (detail.categories || []).map((c) => c.name);
const software = (detail.software_items || []).map((s) => s.name);
const tags = detail.tags || [];
let categoryIds = await resolveCategoryIds([...mediums, ...cats]);
if (categoryIds.length === 0 && opts.defaultCad !== false) {
const cad = await prisma.category.findUnique({ where: { slug: "cad" } });
if (cad) categoryIds = [cad.id];
}
log(
lines,
onLog,
` import: ${detail.title} (${(detail.assets || []).length} assets)`
);
if (opts.dryRun) {
return { outcome: "imported", title: detail.title };
}
const visibility = opts.publish ? Visibility.published : Visibility.draft;
const slug = await uniqueSlug(detail.title);
const project = await prisma.project.create({
data: {
title: detail.title,
slug,
description: stripHtml(detail.description || ""),
shortDescription: stripHtml(detail.description || "").slice(0, 280) || null,
date: detail.published_at ? new Date(detail.published_at) : null,
software,
externalLinks: [{ label: "ArtStation", url: sourceUrl }],
featured: false,
displayPriority: 100,
visibility,
sourceUrl,
sourcePlatform: "artstation",
publishedAt: visibility === Visibility.published ? new Date() : null,
},
});
for (const cid of categoryIds) {
await prisma.projectCategory.create({
data: { projectId: project.id, categoryId: cid },
});
}
for (const t of tags) {
const tslug = slugify(t, { lower: true, strict: true });
if (!tslug) continue;
const tag = await prisma.tag.upsert({
where: { slug: tslug },
create: { name: t, slug: tslug },
update: {},
});
await prisma.projectTag.upsert({
where: { projectId_tagId: { projectId: project.id, tagId: tag.id } },
create: { projectId: project.id, tagId: tag.id },
update: {},
});
}
let sortOrder = 0;
let thumbnailId: string | null = null;
for (const asset of detail.assets || []) {
if (asset.asset_type && asset.asset_type !== "image" && !asset.has_image) continue;
const imageUrl = asset.image_url || detail.cover?.url;
if (!imageUrl) continue;
const saved = await downloadRemoteImage(imageUrl);
if (!saved) {
log(lines, onLog, ` failed download: ${imageUrl}`);
continue;
}
const media = await prisma.media.create({
data: {
projectId: project.id,
type: "image",
url: saved.url,
thumbnailUrl: saved.thumbnailUrl,
filename: saved.filename,
mimeType: saved.mimeType,
width: saved.width || asset.width,
height: saved.height || asset.height,
sizeBytes: saved.sizeBytes,
alt: asset.title || detail.title,
sortOrder: sortOrder++,
},
});
if (!thumbnailId) thumbnailId = media.id;
}
if (thumbnailId) {
await prisma.project.update({
where: { id: project.id },
data: { thumbnailId },
});
}
await indexProject(project.id);
log(lines, onLog, ` created ${visibility} project ${project.slug}`);
return { outcome: "imported", title: detail.title, slug: project.slug };
}
async function listUserProjects(username: string): Promise<AsProjectListItem[]> {
const all: AsProjectListItem[] = [];
let page = 1;
for (;;) {
const url = `https://www.artstation.com/users/${username}/projects.json?page=${page}`;
const data = await fetchJson<{ data?: AsProjectListItem[] } | AsProjectListItem[]>(
url
);
const batch = Array.isArray(data) ? data : data.data || [];
if (batch.length === 0) break;
all.push(...batch);
if (batch.length < 50) break;
page++;
if (page > 20) break;
}
return all;
}
/** Synchronous full import (CLI / awaited admin call). */
export async function runArtStationImport(
options: ImportOptions = {}
): Promise<ImportResult> {
const username = (options.username || "jmartgraphix").trim() || "jmartgraphix";
const dryRun = !!options.dryRun;
const lines: string[] = [];
const onLog = options.onLog;
const created: { title: string; slug: string }[] = [];
let imported = 0;
let skipped = 0;
let errors = 0;
let found = 0;
log(lines, onLog, `ArtStation import (user=${username}, dryRun=${dryRun})`);
if (options.url) {
found = 1;
try {
const r = await importProject(options.url, options, lines);
if (r.outcome === "imported") {
imported++;
if (r.slug) created.push({ title: r.title || "", slug: r.slug });
} else if (r.outcome === "skipped") skipped++;
else errors++;
} catch (err) {
errors++;
log(lines, onLog, ` error: ${(err as Error).message}`);
}
} else {
const list = await listUserProjects(username);
found = list.length;
log(lines, onLog, `Found ${list.length} projects`);
for (const item of list) {
const hash = item.hash_id || String(item.id);
try {
const r = await importProject(hash, options, lines);
if (r.outcome === "imported") {
imported++;
if (r.slug) created.push({ title: r.title || item.title, slug: r.slug });
} else if (r.outcome === "skipped") skipped++;
else errors++;
await new Promise((r) => setTimeout(r, 400));
} catch (err) {
errors++;
log(lines, onLog, ` error importing ${item.title}: ${(err as Error).message}`);
}
}
}
if (!dryRun && imported > 0) {
try {
const n = await reindexAllProjects();
log(lines, onLog, `Typesense reindexed (${n} projects)`);
} catch (err) {
log(lines, onLog, `Typesense reindex warning: ${(err as Error).message}`);
}
}
log(
lines,
onLog,
`Done. imported=${imported} skipped=${skipped} errors=${errors}`
);
return {
username,
dryRun,
found,
imported,
skipped,
errors,
log: lines,
created,
};
}
/** Start import in background; only one job at a time. */
export function startArtStationImportJob(options: ImportOptions = {}): {
started: boolean;
message: string;
} {
if (currentJob.running) {
return { started: false, message: "An import is already running" };
}
currentJob = {
running: true,
startedAt: new Date().toISOString(),
log: [],
};
runArtStationImport({
...options,
onLog: () => {
/* already pushed to currentJob.log */
},
})
.then((result) => {
currentJob = {
running: false,
startedAt: currentJob.startedAt,
finishedAt: new Date().toISOString(),
result,
log: result.log,
};
})
.catch((err) => {
currentJob = {
running: false,
startedAt: currentJob.startedAt,
finishedAt: new Date().toISOString(),
error: (err as Error).message,
log: currentJob.log,
};
});
return { started: true, message: "Import started" };
}