Add admin UI for ArtStation re-import
Expose import as a background admin job with status polling, dry-run and optional publish; share logic with the CLI via a service module.
This commit is contained in:
@@ -229,8 +229,43 @@ export const api = {
|
|||||||
}),
|
}),
|
||||||
reindex: () =>
|
reindex: () =>
|
||||||
request<{ ok: boolean; indexed: number }>("/api/v1/admin/reindex", { method: "POST" }),
|
request<{ ok: boolean; indexed: number }>("/api/v1/admin/reindex", { method: "POST" }),
|
||||||
|
|
||||||
|
importArtStation: (body: {
|
||||||
|
username?: string;
|
||||||
|
dryRun?: boolean;
|
||||||
|
publish?: boolean;
|
||||||
|
url?: string;
|
||||||
|
}) =>
|
||||||
|
request<{
|
||||||
|
ok: boolean;
|
||||||
|
message: string;
|
||||||
|
status: ArtStationImportStatus;
|
||||||
|
}>("/api/v1/admin/import/artstation", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
}),
|
||||||
|
|
||||||
|
importArtStationStatus: () =>
|
||||||
|
request<ArtStationImportStatus>("/api/v1/admin/import/artstation"),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export interface ArtStationImportStatus {
|
||||||
|
running: boolean;
|
||||||
|
startedAt?: string;
|
||||||
|
finishedAt?: string;
|
||||||
|
error?: string;
|
||||||
|
log: string[];
|
||||||
|
result?: {
|
||||||
|
username: string;
|
||||||
|
dryRun: boolean;
|
||||||
|
found: number;
|
||||||
|
imported: number;
|
||||||
|
skipped: number;
|
||||||
|
errors: number;
|
||||||
|
created: { title: string; slug: string }[];
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function mediaSrc(url?: string | null): string {
|
export function mediaSrc(url?: string | null): string {
|
||||||
if (!url) return "";
|
if (!url) return "";
|
||||||
if (url.startsWith("http") || url.startsWith("//")) return url;
|
if (url.startsWith("http") || url.startsWith("//")) return url;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { useEffect, useState } from "react";
|
import { useEffect, useRef, useState } from "react";
|
||||||
import { Link } from "react-router-dom";
|
import { Link } from "react-router-dom";
|
||||||
import { api } from "../../lib/api";
|
import { api, type ArtStationImportStatus } from "../../lib/api";
|
||||||
|
|
||||||
export function AdminDashboard() {
|
export function AdminDashboard() {
|
||||||
const [stats, setStats] = useState({
|
const [stats, setStats] = useState({
|
||||||
@@ -11,8 +11,15 @@ export function AdminDashboard() {
|
|||||||
views: 0,
|
views: 0,
|
||||||
});
|
});
|
||||||
const [msg, setMsg] = useState<string | null>(null);
|
const [msg, setMsg] = useState<string | null>(null);
|
||||||
|
const [err, setErr] = useState<string | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
const [asUser, setAsUser] = useState("jmartgraphix");
|
||||||
|
const [asDryRun, setAsDryRun] = useState(false);
|
||||||
|
const [asPublish, setAsPublish] = useState(false);
|
||||||
|
const [importStatus, setImportStatus] = useState<ArtStationImportStatus | null>(null);
|
||||||
|
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||||
|
|
||||||
|
function refreshStats() {
|
||||||
Promise.all([
|
Promise.all([
|
||||||
api.adminProjects({ perPage: 1 }),
|
api.adminProjects({ perPage: 1 }),
|
||||||
api.adminProjects({ visibility: "published", perPage: 1 }),
|
api.adminProjects({ visibility: "published", perPage: 1 }),
|
||||||
@@ -28,22 +35,77 @@ export function AdminDashboard() {
|
|||||||
views: views.length,
|
views: views.length,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
refreshStats();
|
||||||
|
api.importArtStationStatus().then(setImportStatus).catch(() => {});
|
||||||
|
return () => {
|
||||||
|
if (pollRef.current) clearInterval(pollRef.current);
|
||||||
|
};
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
function startPolling() {
|
||||||
|
if (pollRef.current) clearInterval(pollRef.current);
|
||||||
|
pollRef.current = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
const s = await api.importArtStationStatus();
|
||||||
|
setImportStatus(s);
|
||||||
|
if (!s.running) {
|
||||||
|
if (pollRef.current) clearInterval(pollRef.current);
|
||||||
|
pollRef.current = null;
|
||||||
|
refreshStats();
|
||||||
|
if (s.result) {
|
||||||
|
setMsg(
|
||||||
|
`ArtStation import finished: ${s.result.imported} new, ${s.result.skipped} skipped, ${s.result.errors} errors (${s.result.found} on profile)`
|
||||||
|
);
|
||||||
|
} else if (s.error) {
|
||||||
|
setErr(s.error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
/* keep polling */
|
||||||
|
}
|
||||||
|
}, 2000);
|
||||||
|
}
|
||||||
|
|
||||||
async function reindex() {
|
async function reindex() {
|
||||||
|
setErr(null);
|
||||||
try {
|
try {
|
||||||
const r = await api.reindex();
|
const r = await api.reindex();
|
||||||
setMsg(`Typesense reindexed ${r.indexed} projects`);
|
setMsg(`Typesense reindexed ${r.indexed} projects`);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
setMsg((e as Error).message);
|
setErr((e as Error).message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function runArtStationImport() {
|
||||||
|
setErr(null);
|
||||||
|
setMsg(null);
|
||||||
|
try {
|
||||||
|
const res = await api.importArtStation({
|
||||||
|
username: asUser.trim() || "jmartgraphix",
|
||||||
|
dryRun: asDryRun,
|
||||||
|
publish: asPublish,
|
||||||
|
});
|
||||||
|
setImportStatus(res.status);
|
||||||
|
setMsg(res.message);
|
||||||
|
if (res.status.running) startPolling();
|
||||||
|
} catch (e) {
|
||||||
|
setErr((e as Error).message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const running = importStatus?.running;
|
||||||
|
const logLines = importStatus?.log?.slice(-40) || [];
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="admin-page">
|
<div className="admin-page">
|
||||||
<h1>Dashboard</h1>
|
<h1>Dashboard</h1>
|
||||||
<p className="admin-page__sub">Manage portfolio content protected by Authelia SSO.</p>
|
<p className="admin-page__sub">Manage portfolio content protected by Authelia SSO.</p>
|
||||||
{msg && <div className="admin-msg admin-msg--ok">{msg}</div>}
|
{msg && <div className="admin-msg admin-msg--ok">{msg}</div>}
|
||||||
|
{err && <div className="admin-msg admin-msg--err">{err}</div>}
|
||||||
|
|
||||||
<div className="admin-cards">
|
<div className="admin-cards">
|
||||||
<div className="admin-stat">
|
<div className="admin-stat">
|
||||||
<strong>{stats.projects}</strong>
|
<strong>{stats.projects}</strong>
|
||||||
@@ -66,6 +128,7 @@ export function AdminDashboard() {
|
|||||||
<span>Portfolio views</span>
|
<span>Portfolio views</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="admin-page__toolbar">
|
<div className="admin-page__toolbar">
|
||||||
<Link to="/admin/projects/new" className="btn btn--primary">
|
<Link to="/admin/projects/new" className="btn btn--primary">
|
||||||
New project
|
New project
|
||||||
@@ -77,7 +140,99 @@ export function AdminDashboard() {
|
|||||||
Reindex search
|
Reindex search
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
<section>
|
|
||||||
|
<section className="admin-import" style={{ marginTop: "2.5rem", maxWidth: 720 }}>
|
||||||
|
<h2 style={{ fontSize: "1.1rem", margin: "0 0 0.35rem" }}>Import from ArtStation</h2>
|
||||||
|
<p style={{ color: "var(--text-muted)", margin: "0 0 1rem", fontSize: "0.9rem" }}>
|
||||||
|
Pull new public projects from your ArtStation profile. Existing entries (matched by
|
||||||
|
ArtStation URL) are skipped. New work lands as drafts unless you choose publish.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<div className="field">
|
||||||
|
<label htmlFor="as-user">ArtStation username</label>
|
||||||
|
<input
|
||||||
|
id="as-user"
|
||||||
|
value={asUser}
|
||||||
|
onChange={(e) => setAsUser(e.target.value)}
|
||||||
|
disabled={!!running}
|
||||||
|
placeholder="jmartgraphix"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="admin-check-grid" style={{ marginBottom: "1rem" }}>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={asDryRun}
|
||||||
|
disabled={!!running}
|
||||||
|
onChange={(e) => setAsDryRun(e.target.checked)}
|
||||||
|
/>
|
||||||
|
Dry run (list only, no writes)
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={asPublish}
|
||||||
|
disabled={!!running || asDryRun}
|
||||||
|
onChange={(e) => setAsPublish(e.target.checked)}
|
||||||
|
/>
|
||||||
|
Publish new imports immediately
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="admin-page__toolbar">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="btn btn--primary"
|
||||||
|
disabled={!!running}
|
||||||
|
onClick={runArtStationImport}
|
||||||
|
>
|
||||||
|
{running ? "Import running…" : "Import from ArtStation"}
|
||||||
|
</button>
|
||||||
|
{importStatus?.finishedAt && !running && (
|
||||||
|
<span style={{ color: "var(--text-dim)", fontSize: "0.85rem" }}>
|
||||||
|
Last run {new Date(importStatus.finishedAt).toLocaleString()}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{importStatus?.result && !running && (
|
||||||
|
<div
|
||||||
|
className="admin-msg admin-msg--ok"
|
||||||
|
style={{ marginTop: "1rem", whiteSpace: "pre-wrap" }}
|
||||||
|
>
|
||||||
|
Found {importStatus.result.found} · imported {importStatus.result.imported} · skipped{" "}
|
||||||
|
{importStatus.result.skipped} · errors {importStatus.result.errors}
|
||||||
|
{importStatus.result.created.length > 0 && (
|
||||||
|
<>
|
||||||
|
{"\n"}New:{" "}
|
||||||
|
{importStatus.result.created.map((c) => c.title).join(", ")}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{logLines.length > 0 && (
|
||||||
|
<pre
|
||||||
|
style={{
|
||||||
|
marginTop: "1rem",
|
||||||
|
padding: "0.85rem 1rem",
|
||||||
|
background: "var(--bg-elevated)",
|
||||||
|
border: "1px solid var(--border)",
|
||||||
|
borderRadius: "var(--radius-sm)",
|
||||||
|
fontSize: "0.78rem",
|
||||||
|
lineHeight: 1.45,
|
||||||
|
maxHeight: 280,
|
||||||
|
overflow: "auto",
|
||||||
|
color: "var(--text-muted)",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{logLines.join("\n")}
|
||||||
|
</pre>
|
||||||
|
)}
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section style={{ marginTop: "2.5rem" }}>
|
||||||
<h2 style={{ fontSize: "1rem", color: "var(--text-muted)" }}>Quick links for audiences</h2>
|
<h2 style={{ fontSize: "1rem", color: "var(--text-muted)" }}>Quick links for audiences</h2>
|
||||||
<ul style={{ color: "var(--text-dim)", lineHeight: 1.9 }}>
|
<ul style={{ color: "var(--text-dim)", lineHeight: 1.9 }}>
|
||||||
<li>
|
<li>
|
||||||
|
|||||||
@@ -1,26 +1,17 @@
|
|||||||
/**
|
/**
|
||||||
* ArtStation project importer for jmartgraphix portfolio.
|
* ArtStation project importer CLI.
|
||||||
*
|
*
|
||||||
* Usage:
|
* Usage:
|
||||||
* npm run import:artstation -- --user jmartgraphix
|
* npm run import:artstation -- --user jmartgraphix
|
||||||
* npm run import:artstation -- --user jmartgraphix --dry-run
|
* npm run import:artstation -- --user jmartgraphix --dry-run
|
||||||
* npm run import:artstation -- --url https://www.artstation.com/artwork/XXXX
|
* npm run import:artstation -- --url https://www.artstation.com/artwork/XXXX
|
||||||
|
* npm run import:artstation -- --user jmartgraphix --publish
|
||||||
*
|
*
|
||||||
* Fetches public ArtStation JSON endpoints, downloads images, and creates
|
* Prefer the Admin → Dashboard “Import from ArtStation” button in production.
|
||||||
* draft projects so you can edit/publish from the admin panel afterward.
|
|
||||||
*/
|
*/
|
||||||
import "dotenv/config";
|
import "dotenv/config";
|
||||||
import { PrismaClient, Visibility } from "@prisma/client";
|
import { runArtStationImport } from "../src/services/artstation-import.js";
|
||||||
import slugify from "slugify";
|
import { prisma } from "../src/lib/prisma.js";
|
||||||
import { downloadRemoteImage } from "../src/lib/media.js";
|
|
||||||
|
|
||||||
const prisma = new PrismaClient();
|
|
||||||
const USER_AGENT =
|
|
||||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
||||||
|
|
||||||
/** FlareSolverr (or direct fetch) — ArtStation sits behind Cloudflare. */
|
|
||||||
const FLARESOLVERR_URL =
|
|
||||||
process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1";
|
|
||||||
|
|
||||||
function arg(name: string, fallback?: string): string | undefined {
|
function arg(name: string, fallback?: string): string | undefined {
|
||||||
const idx = process.argv.indexOf(`--${name}`);
|
const idx = process.argv.indexOf(`--${name}`);
|
||||||
@@ -32,312 +23,16 @@ function hasFlag(name: string): boolean {
|
|||||||
return process.argv.includes(`--${name}`);
|
return process.argv.includes(`--${name}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractJsonFromFlareBody(body: string): unknown {
|
|
||||||
// FlareSolverr often wraps JSON in <pre>…</pre>
|
|
||||||
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i);
|
|
||||||
const raw = (pre ? pre[1] : body).trim();
|
|
||||||
return JSON.parse(raw);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchViaFlareSolverr(url: string): Promise<unknown> {
|
|
||||||
const res = await fetch(FLARESOLVERR_URL, {
|
|
||||||
method: "POST",
|
|
||||||
headers: { "Content-Type": "application/json" },
|
|
||||||
body: JSON.stringify({
|
|
||||||
cmd: "request.get",
|
|
||||||
url,
|
|
||||||
maxTimeout: 90_000,
|
|
||||||
}),
|
|
||||||
signal: AbortSignal.timeout(100_000),
|
|
||||||
});
|
|
||||||
if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`);
|
|
||||||
const payload = (await res.json()) as {
|
|
||||||
status: string;
|
|
||||||
message?: string;
|
|
||||||
solution?: { status: number; response?: string };
|
|
||||||
};
|
|
||||||
if (payload.status !== "ok" || !payload.solution?.response) {
|
|
||||||
throw new Error(
|
|
||||||
`FlareSolverr failed for ${url}: ${payload.message || payload.status}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (payload.solution.status >= 400) {
|
|
||||||
throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`);
|
|
||||||
}
|
|
||||||
return extractJsonFromFlareBody(payload.solution.response);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchJson<T>(url: string): Promise<T> {
|
|
||||||
// Prefer FlareSolverr (Cloudflare). Fall back to direct fetch.
|
|
||||||
try {
|
|
||||||
return (await fetchViaFlareSolverr(url)) as T;
|
|
||||||
} catch (flareErr) {
|
|
||||||
console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`);
|
|
||||||
}
|
|
||||||
const res = await fetch(url, {
|
|
||||||
headers: {
|
|
||||||
"User-Agent": USER_AGENT,
|
|
||||||
Accept: "application/json, text/plain, */*",
|
|
||||||
"Accept-Language": "en-US,en;q=0.9",
|
|
||||||
Referer: "https://www.artstation.com/",
|
|
||||||
},
|
|
||||||
signal: AbortSignal.timeout(30_000),
|
|
||||||
});
|
|
||||||
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
|
|
||||||
return res.json() as Promise<T>;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface AsProjectListItem {
|
|
||||||
id: number;
|
|
||||||
title: string;
|
|
||||||
hash_id?: string;
|
|
||||||
permalink?: string;
|
|
||||||
cover?: { url?: string; small_square_url?: string };
|
|
||||||
mediums?: { name: string }[];
|
|
||||||
categories?: { name: string }[];
|
|
||||||
published_at?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface AsProjectDetail {
|
|
||||||
id: number;
|
|
||||||
title: string;
|
|
||||||
description?: string;
|
|
||||||
hash_id?: string;
|
|
||||||
permalink?: string;
|
|
||||||
published_at?: string;
|
|
||||||
tags?: string[];
|
|
||||||
mediums?: { name: string }[];
|
|
||||||
categories?: { name: string }[];
|
|
||||||
software_items?: { name: string }[];
|
|
||||||
assets?: {
|
|
||||||
id: number;
|
|
||||||
title?: string;
|
|
||||||
asset_type?: string;
|
|
||||||
image_url?: string;
|
|
||||||
has_image?: boolean;
|
|
||||||
width?: number;
|
|
||||||
height?: number;
|
|
||||||
}[];
|
|
||||||
cover?: { url?: string };
|
|
||||||
user?: { full_name?: string; username?: string };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Map ArtStation medium/category names to our category slugs
|
|
||||||
const CATEGORY_MAP: Record<string, string> = {
|
|
||||||
"3d": "3d-modeling",
|
|
||||||
"3d modeling": "3d-modeling",
|
|
||||||
modeling: "3d-modeling",
|
|
||||||
characters: "3d-modeling",
|
|
||||||
environments: "3d-modeling",
|
|
||||||
animation: "3d-animation",
|
|
||||||
"3d animation": "3d-animation",
|
|
||||||
cad: "cad",
|
|
||||||
product: "cad",
|
|
||||||
industrial: "cad",
|
|
||||||
sculpting: "3d-sculpting",
|
|
||||||
"digital sculpting": "3d-sculpting",
|
|
||||||
ai: "ai-generated-content",
|
|
||||||
"ai art": "ai-generated-content",
|
|
||||||
video: "video",
|
|
||||||
cinematography: "video",
|
|
||||||
photography: "photography",
|
|
||||||
"graphic design": "graphic-design",
|
|
||||||
design: "graphic-design",
|
|
||||||
illustration: "illustration",
|
|
||||||
concept: "illustration",
|
|
||||||
};
|
|
||||||
|
|
||||||
async function resolveCategoryIds(names: string[]): Promise<string[]> {
|
|
||||||
const all = await prisma.category.findMany();
|
|
||||||
const bySlug = Object.fromEntries(all.map((c) => [c.slug, c.id]));
|
|
||||||
const ids = new Set<string>();
|
|
||||||
for (const n of names) {
|
|
||||||
const key = n.toLowerCase().trim();
|
|
||||||
const mapped = CATEGORY_MAP[key];
|
|
||||||
if (mapped && bySlug[mapped]) ids.add(bySlug[mapped]);
|
|
||||||
const direct = all.find((c) => c.name.toLowerCase() === key || c.slug === slugify(key, { lower: true, strict: true }));
|
|
||||||
if (direct) ids.add(direct.id);
|
|
||||||
}
|
|
||||||
return [...ids];
|
|
||||||
}
|
|
||||||
|
|
||||||
async function uniqueSlug(title: string): Promise<string> {
|
|
||||||
const base = slugify(title, { lower: true, strict: true }) || "artwork";
|
|
||||||
let slug = base;
|
|
||||||
let n = 2;
|
|
||||||
while (await prisma.project.findUnique({ where: { slug } })) {
|
|
||||||
slug = `${base}-${n++}`;
|
|
||||||
}
|
|
||||||
return slug;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function importProject(hashOrUrl: string, dryRun: boolean): Promise<void> {
|
|
||||||
let hash = hashOrUrl;
|
|
||||||
const m = hashOrUrl.match(/artstation\.com\/artwork\/([A-Za-z0-9]+)/);
|
|
||||||
if (m) hash = m[1];
|
|
||||||
|
|
||||||
const detail = await fetchJson<AsProjectDetail>(
|
|
||||||
`https://www.artstation.com/projects/${hash}.json`
|
|
||||||
);
|
|
||||||
|
|
||||||
const sourceUrl =
|
|
||||||
detail.permalink || `https://www.artstation.com/artwork/${detail.hash_id || hash}`;
|
|
||||||
const existing = await prisma.project.findFirst({
|
|
||||||
where: { sourceUrl },
|
|
||||||
});
|
|
||||||
if (existing) {
|
|
||||||
console.log(` skip (exists): ${detail.title}`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const mediums = (detail.mediums || []).map((m) => m.name);
|
|
||||||
const cats = (detail.categories || []).map((c) => c.name);
|
|
||||||
const software = (detail.software_items || []).map((s) => s.name);
|
|
||||||
const tags = detail.tags || [];
|
|
||||||
const categoryIds = await resolveCategoryIds([...mediums, ...cats]);
|
|
||||||
|
|
||||||
console.log(` import: ${detail.title} (${(detail.assets || []).length} assets)`);
|
|
||||||
if (dryRun) return;
|
|
||||||
|
|
||||||
const slug = await uniqueSlug(detail.title);
|
|
||||||
const project = await prisma.project.create({
|
|
||||||
data: {
|
|
||||||
title: detail.title,
|
|
||||||
slug,
|
|
||||||
description: stripHtml(detail.description || ""),
|
|
||||||
shortDescription: stripHtml(detail.description || "").slice(0, 280) || null,
|
|
||||||
date: detail.published_at ? new Date(detail.published_at) : null,
|
|
||||||
software,
|
|
||||||
externalLinks: [{ label: "ArtStation", url: sourceUrl }],
|
|
||||||
featured: false,
|
|
||||||
displayPriority: 100,
|
|
||||||
visibility: Visibility.draft,
|
|
||||||
sourceUrl,
|
|
||||||
sourcePlatform: "artstation",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
for (const cid of categoryIds) {
|
|
||||||
await prisma.projectCategory.create({
|
|
||||||
data: { projectId: project.id, categoryId: cid },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const t of tags) {
|
|
||||||
const tslug = slugify(t, { lower: true, strict: true });
|
|
||||||
if (!tslug) continue;
|
|
||||||
const tag = await prisma.tag.upsert({
|
|
||||||
where: { slug: tslug },
|
|
||||||
create: { name: t, slug: tslug },
|
|
||||||
update: {},
|
|
||||||
});
|
|
||||||
await prisma.projectTag.upsert({
|
|
||||||
where: { projectId_tagId: { projectId: project.id, tagId: tag.id } },
|
|
||||||
create: { projectId: project.id, tagId: tag.id },
|
|
||||||
update: {},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let sortOrder = 0;
|
|
||||||
let thumbnailId: string | null = null;
|
|
||||||
const assets = detail.assets || [];
|
|
||||||
for (const asset of assets) {
|
|
||||||
if (asset.asset_type && asset.asset_type !== "image" && !asset.has_image) continue;
|
|
||||||
const imageUrl = asset.image_url || detail.cover?.url;
|
|
||||||
if (!imageUrl) continue;
|
|
||||||
const saved = await downloadRemoteImage(imageUrl);
|
|
||||||
if (!saved) {
|
|
||||||
console.warn(` failed download: ${imageUrl}`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const media = await prisma.media.create({
|
|
||||||
data: {
|
|
||||||
projectId: project.id,
|
|
||||||
type: "image",
|
|
||||||
url: saved.url,
|
|
||||||
thumbnailUrl: saved.thumbnailUrl,
|
|
||||||
filename: saved.filename,
|
|
||||||
mimeType: saved.mimeType,
|
|
||||||
width: saved.width || asset.width,
|
|
||||||
height: saved.height || asset.height,
|
|
||||||
sizeBytes: saved.sizeBytes,
|
|
||||||
alt: asset.title || detail.title,
|
|
||||||
sortOrder: sortOrder++,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
if (!thumbnailId) thumbnailId = media.id;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (thumbnailId) {
|
|
||||||
await prisma.project.update({
|
|
||||||
where: { id: project.id },
|
|
||||||
data: { thumbnailId },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(` created draft project ${project.slug}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
function stripHtml(html: string): string {
|
|
||||||
return html
|
|
||||||
.replace(/<br\s*\/?>/gi, "\n")
|
|
||||||
.replace(/<\/p>/gi, "\n\n")
|
|
||||||
.replace(/<[^>]+>/g, "")
|
|
||||||
.replace(/ /g, " ")
|
|
||||||
.replace(/&/g, "&")
|
|
||||||
.replace(/</g, "<")
|
|
||||||
.replace(/>/g, ">")
|
|
||||||
.replace(/"/g, '"')
|
|
||||||
.replace(/\n{3,}/g, "\n\n")
|
|
||||||
.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
async function listUserProjects(username: string): Promise<AsProjectListItem[]> {
|
|
||||||
const all: AsProjectListItem[] = [];
|
|
||||||
let page = 1;
|
|
||||||
for (;;) {
|
|
||||||
// Public profile projects endpoint
|
|
||||||
const url = `https://www.artstation.com/users/${username}/projects.json?page=${page}`;
|
|
||||||
try {
|
|
||||||
const data = await fetchJson<{ data?: AsProjectListItem[] } | AsProjectListItem[]>(url);
|
|
||||||
const batch = Array.isArray(data) ? data : data.data || [];
|
|
||||||
if (batch.length === 0) break;
|
|
||||||
all.push(...batch);
|
|
||||||
if (batch.length < 50) break;
|
|
||||||
page++;
|
|
||||||
if (page > 20) break;
|
|
||||||
} catch (err) {
|
|
||||||
console.error(`Failed to list page ${page}:`, err);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return all;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
const dryRun = hasFlag("dry-run");
|
const result = await runArtStationImport({
|
||||||
const user = arg("user", "jmartgraphix")!;
|
username: arg("user", "jmartgraphix"),
|
||||||
const singleUrl = arg("url");
|
url: arg("url"),
|
||||||
|
dryRun: hasFlag("dry-run"),
|
||||||
console.log(`ArtStation import (user=${user}, dryRun=${dryRun})`);
|
publish: hasFlag("publish"),
|
||||||
|
});
|
||||||
if (singleUrl) {
|
console.log(
|
||||||
await importProject(singleUrl, dryRun);
|
`Summary: found=${result.found} imported=${result.imported} skipped=${result.skipped} errors=${result.errors}`
|
||||||
} else {
|
);
|
||||||
const list = await listUserProjects(user);
|
|
||||||
console.log(`Found ${list.length} projects`);
|
|
||||||
for (const item of list) {
|
|
||||||
const hash = item.hash_id || String(item.id);
|
|
||||||
try {
|
|
||||||
await importProject(hash, dryRun);
|
|
||||||
await new Promise((r) => setTimeout(r, 500)); // be polite
|
|
||||||
} catch (err) {
|
|
||||||
console.error(` error importing ${item.title}:`, err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log("Done. Review drafts in the admin panel and publish when ready.");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ import { prisma } from "../lib/prisma.js";
|
|||||||
import { requireAdmin } from "../lib/auth.js";
|
import { requireAdmin } from "../lib/auth.js";
|
||||||
import { uniqueCategorySlug, uniqueTagSlug, makeSlug } from "../lib/slug.js";
|
import { uniqueCategorySlug, uniqueTagSlug, makeSlug } from "../lib/slug.js";
|
||||||
import { reindexAllProjects } from "../lib/typesense.js";
|
import { reindexAllProjects } from "../lib/typesense.js";
|
||||||
|
import {
|
||||||
|
getImportJobStatus,
|
||||||
|
startArtStationImportJob,
|
||||||
|
} from "../services/artstation-import.js";
|
||||||
|
|
||||||
export async function adminMetaRoutes(app: FastifyInstance) {
|
export async function adminMetaRoutes(app: FastifyInstance) {
|
||||||
// Categories
|
// Categories
|
||||||
@@ -279,4 +283,39 @@ export async function adminMetaRoutes(app: FastifyInstance) {
|
|||||||
const count = await reindexAllProjects();
|
const count = await reindexAllProjects();
|
||||||
return { ok: true, indexed: count };
|
return { ok: true, indexed: count };
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ArtStation import
|
||||||
|
app.get("/api/v1/admin/import/artstation", async (req, reply) => {
|
||||||
|
if (!requireAdmin(req, reply)) return;
|
||||||
|
return getImportJobStatus();
|
||||||
|
});
|
||||||
|
|
||||||
|
app.post("/api/v1/admin/import/artstation", async (req, reply) => {
|
||||||
|
if (!requireAdmin(req, reply)) return;
|
||||||
|
const body = z
|
||||||
|
.object({
|
||||||
|
username: z.string().min(1).optional().default("jmartgraphix"),
|
||||||
|
dryRun: z.boolean().optional().default(false),
|
||||||
|
publish: z.boolean().optional().default(false),
|
||||||
|
url: z.string().url().optional(),
|
||||||
|
})
|
||||||
|
.parse(req.body ?? {});
|
||||||
|
|
||||||
|
const { started, message } = startArtStationImportJob({
|
||||||
|
username: body.username,
|
||||||
|
dryRun: body.dryRun,
|
||||||
|
publish: body.publish,
|
||||||
|
url: body.url,
|
||||||
|
defaultCad: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!started) {
|
||||||
|
return reply.code(409).send({ ok: false, message, status: getImportJobStatus() });
|
||||||
|
}
|
||||||
|
return reply.code(202).send({
|
||||||
|
ok: true,
|
||||||
|
message,
|
||||||
|
status: getImportJobStatus(),
|
||||||
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,470 @@
|
|||||||
|
/**
|
||||||
|
* ArtStation portfolio importer (shared by CLI + admin API).
|
||||||
|
* Uses FlareSolverr when available to bypass Cloudflare.
|
||||||
|
*/
|
||||||
|
import { Visibility } from "@prisma/client";
|
||||||
|
import slugify from "slugify";
|
||||||
|
import { prisma } from "../lib/prisma.js";
|
||||||
|
import { downloadRemoteImage } from "../lib/media.js";
|
||||||
|
import { indexProject, reindexAllProjects } from "../lib/typesense.js";
|
||||||
|
|
||||||
|
const USER_AGENT =
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
||||||
|
|
||||||
|
const FLARESOLVERR_URL =
|
||||||
|
process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1";
|
||||||
|
|
||||||
|
const CATEGORY_MAP: Record<string, string> = {
|
||||||
|
"3d": "3d-modeling",
|
||||||
|
"3d modeling": "3d-modeling",
|
||||||
|
modeling: "3d-modeling",
|
||||||
|
characters: "3d-modeling",
|
||||||
|
environments: "3d-modeling",
|
||||||
|
animation: "3d-animation",
|
||||||
|
"3d animation": "3d-animation",
|
||||||
|
cad: "cad",
|
||||||
|
product: "cad",
|
||||||
|
industrial: "cad",
|
||||||
|
sculpting: "3d-sculpting",
|
||||||
|
"digital sculpting": "3d-sculpting",
|
||||||
|
ai: "ai-generated-content",
|
||||||
|
"ai art": "ai-generated-content",
|
||||||
|
video: "video",
|
||||||
|
cinematography: "video",
|
||||||
|
photography: "photography",
|
||||||
|
"graphic design": "graphic-design",
|
||||||
|
design: "graphic-design",
|
||||||
|
illustration: "illustration",
|
||||||
|
concept: "illustration",
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface ImportOptions {
|
||||||
|
username?: string;
|
||||||
|
url?: string;
|
||||||
|
dryRun?: boolean;
|
||||||
|
/** Publish new imports instead of leaving as draft */
|
||||||
|
publish?: boolean;
|
||||||
|
/** Assign CAD when no category matched */
|
||||||
|
defaultCad?: boolean;
|
||||||
|
onLog?: (line: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ImportResult {
|
||||||
|
username: string;
|
||||||
|
dryRun: boolean;
|
||||||
|
found: number;
|
||||||
|
imported: number;
|
||||||
|
skipped: number;
|
||||||
|
errors: number;
|
||||||
|
log: string[];
|
||||||
|
created: { title: string; slug: string }[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ImportJobStatus = {
|
||||||
|
running: boolean;
|
||||||
|
startedAt?: string;
|
||||||
|
finishedAt?: string;
|
||||||
|
result?: ImportResult;
|
||||||
|
error?: string;
|
||||||
|
log: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
let currentJob: ImportJobStatus = { running: false, log: [] };
|
||||||
|
|
||||||
|
export function getImportJobStatus(): ImportJobStatus {
|
||||||
|
return { ...currentJob, log: [...currentJob.log] };
|
||||||
|
}
|
||||||
|
|
||||||
|
function log(lines: string[], onLog: ImportOptions["onLog"], line: string) {
|
||||||
|
lines.push(line);
|
||||||
|
currentJob.log.push(line);
|
||||||
|
onLog?.(line);
|
||||||
|
console.log(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractJsonFromFlareBody(body: string): unknown {
|
||||||
|
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i);
|
||||||
|
const raw = (pre ? pre[1] : body).trim();
|
||||||
|
return JSON.parse(raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchViaFlareSolverr(url: string): Promise<unknown> {
|
||||||
|
const res = await fetch(FLARESOLVERR_URL, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
cmd: "request.get",
|
||||||
|
url,
|
||||||
|
maxTimeout: 90_000,
|
||||||
|
}),
|
||||||
|
signal: AbortSignal.timeout(100_000),
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`);
|
||||||
|
const payload = (await res.json()) as {
|
||||||
|
status: string;
|
||||||
|
message?: string;
|
||||||
|
solution?: { status: number; response?: string };
|
||||||
|
};
|
||||||
|
if (payload.status !== "ok" || !payload.solution?.response) {
|
||||||
|
throw new Error(
|
||||||
|
`FlareSolverr failed for ${url}: ${payload.message || payload.status}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (payload.solution.status >= 400) {
|
||||||
|
throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`);
|
||||||
|
}
|
||||||
|
return extractJsonFromFlareBody(payload.solution.response);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchJson<T>(url: string): Promise<T> {
|
||||||
|
try {
|
||||||
|
return (await fetchViaFlareSolverr(url)) as T;
|
||||||
|
} catch (flareErr) {
|
||||||
|
console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`);
|
||||||
|
}
|
||||||
|
const res = await fetch(url, {
|
||||||
|
headers: {
|
||||||
|
"User-Agent": USER_AGENT,
|
||||||
|
Accept: "application/json, text/plain, */*",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
Referer: "https://www.artstation.com/",
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(30_000),
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
|
||||||
|
return res.json() as Promise<T>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface AsProjectListItem {
|
||||||
|
id: number;
|
||||||
|
title: string;
|
||||||
|
hash_id?: string;
|
||||||
|
permalink?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface AsProjectDetail {
|
||||||
|
id: number;
|
||||||
|
title: string;
|
||||||
|
description?: string;
|
||||||
|
hash_id?: string;
|
||||||
|
permalink?: string;
|
||||||
|
published_at?: string;
|
||||||
|
tags?: string[];
|
||||||
|
mediums?: { name: string }[];
|
||||||
|
categories?: { name: string }[];
|
||||||
|
software_items?: { name: string }[];
|
||||||
|
assets?: {
|
||||||
|
id: number;
|
||||||
|
title?: string;
|
||||||
|
asset_type?: string;
|
||||||
|
image_url?: string;
|
||||||
|
has_image?: boolean;
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
}[];
|
||||||
|
cover?: { url?: string };
|
||||||
|
}
|
||||||
|
|
||||||
|
function stripHtml(html: string): string {
|
||||||
|
return html
|
||||||
|
.replace(/<br\s*\/?>/gi, "\n")
|
||||||
|
.replace(/<\/p>/gi, "\n\n")
|
||||||
|
.replace(/<[^>]+>/g, "")
|
||||||
|
.replace(/ /g, " ")
|
||||||
|
.replace(/&/g, "&")
|
||||||
|
.replace(/</g, "<")
|
||||||
|
.replace(/>/g, ">")
|
||||||
|
.replace(/"/g, '"')
|
||||||
|
.replace(/\n{3,}/g, "\n\n")
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveCategoryIds(names: string[]): Promise<string[]> {
|
||||||
|
const all = await prisma.category.findMany();
|
||||||
|
const bySlug = Object.fromEntries(all.map((c) => [c.slug, c.id]));
|
||||||
|
const ids = new Set<string>();
|
||||||
|
for (const n of names) {
|
||||||
|
const key = n.toLowerCase().trim();
|
||||||
|
const mapped = CATEGORY_MAP[key];
|
||||||
|
if (mapped && bySlug[mapped]) ids.add(bySlug[mapped]);
|
||||||
|
const direct = all.find(
|
||||||
|
(c) =>
|
||||||
|
c.name.toLowerCase() === key ||
|
||||||
|
c.slug === slugify(key, { lower: true, strict: true })
|
||||||
|
);
|
||||||
|
if (direct) ids.add(direct.id);
|
||||||
|
}
|
||||||
|
return [...ids];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uniqueSlug(title: string): Promise<string> {
|
||||||
|
const base = slugify(title, { lower: true, strict: true }) || "artwork";
|
||||||
|
let slug = base;
|
||||||
|
let n = 2;
|
||||||
|
while (await prisma.project.findUnique({ where: { slug } })) {
|
||||||
|
slug = `${base}-${n++}`;
|
||||||
|
}
|
||||||
|
return slug;
|
||||||
|
}
|
||||||
|
|
||||||
|
type ImportProjectOutcome = "imported" | "skipped" | "error";
|
||||||
|
|
||||||
|
async function importProject(
|
||||||
|
hashOrUrl: string,
|
||||||
|
opts: ImportOptions,
|
||||||
|
lines: string[]
|
||||||
|
): Promise<{ outcome: ImportProjectOutcome; title?: string; slug?: string }> {
|
||||||
|
const onLog = opts.onLog;
|
||||||
|
let hash = hashOrUrl;
|
||||||
|
const m = hashOrUrl.match(/artstation\.com\/artwork\/([A-Za-z0-9]+)/);
|
||||||
|
if (m) hash = m[1];
|
||||||
|
|
||||||
|
const detail = await fetchJson<AsProjectDetail>(
|
||||||
|
`https://www.artstation.com/projects/${hash}.json`
|
||||||
|
);
|
||||||
|
|
||||||
|
const sourceUrl =
|
||||||
|
detail.permalink ||
|
||||||
|
`https://www.artstation.com/artwork/${detail.hash_id || hash}`;
|
||||||
|
const existing = await prisma.project.findFirst({ where: { sourceUrl } });
|
||||||
|
if (existing) {
|
||||||
|
log(lines, onLog, ` skip (exists): ${detail.title}`);
|
||||||
|
return { outcome: "skipped", title: detail.title, slug: existing.slug };
|
||||||
|
}
|
||||||
|
|
||||||
|
const mediums = (detail.mediums || []).map((x) => x.name);
|
||||||
|
const cats = (detail.categories || []).map((c) => c.name);
|
||||||
|
const software = (detail.software_items || []).map((s) => s.name);
|
||||||
|
const tags = detail.tags || [];
|
||||||
|
let categoryIds = await resolveCategoryIds([...mediums, ...cats]);
|
||||||
|
|
||||||
|
if (categoryIds.length === 0 && opts.defaultCad !== false) {
|
||||||
|
const cad = await prisma.category.findUnique({ where: { slug: "cad" } });
|
||||||
|
if (cad) categoryIds = [cad.id];
|
||||||
|
}
|
||||||
|
|
||||||
|
log(
|
||||||
|
lines,
|
||||||
|
onLog,
|
||||||
|
` import: ${detail.title} (${(detail.assets || []).length} assets)`
|
||||||
|
);
|
||||||
|
if (opts.dryRun) {
|
||||||
|
return { outcome: "imported", title: detail.title };
|
||||||
|
}
|
||||||
|
|
||||||
|
const visibility = opts.publish ? Visibility.published : Visibility.draft;
|
||||||
|
const slug = await uniqueSlug(detail.title);
|
||||||
|
const project = await prisma.project.create({
|
||||||
|
data: {
|
||||||
|
title: detail.title,
|
||||||
|
slug,
|
||||||
|
description: stripHtml(detail.description || ""),
|
||||||
|
shortDescription: stripHtml(detail.description || "").slice(0, 280) || null,
|
||||||
|
date: detail.published_at ? new Date(detail.published_at) : null,
|
||||||
|
software,
|
||||||
|
externalLinks: [{ label: "ArtStation", url: sourceUrl }],
|
||||||
|
featured: false,
|
||||||
|
displayPriority: 100,
|
||||||
|
visibility,
|
||||||
|
sourceUrl,
|
||||||
|
sourcePlatform: "artstation",
|
||||||
|
publishedAt: visibility === Visibility.published ? new Date() : null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const cid of categoryIds) {
|
||||||
|
await prisma.projectCategory.create({
|
||||||
|
data: { projectId: project.id, categoryId: cid },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const t of tags) {
|
||||||
|
const tslug = slugify(t, { lower: true, strict: true });
|
||||||
|
if (!tslug) continue;
|
||||||
|
const tag = await prisma.tag.upsert({
|
||||||
|
where: { slug: tslug },
|
||||||
|
create: { name: t, slug: tslug },
|
||||||
|
update: {},
|
||||||
|
});
|
||||||
|
await prisma.projectTag.upsert({
|
||||||
|
where: { projectId_tagId: { projectId: project.id, tagId: tag.id } },
|
||||||
|
create: { projectId: project.id, tagId: tag.id },
|
||||||
|
update: {},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let sortOrder = 0;
|
||||||
|
let thumbnailId: string | null = null;
|
||||||
|
for (const asset of detail.assets || []) {
|
||||||
|
if (asset.asset_type && asset.asset_type !== "image" && !asset.has_image) continue;
|
||||||
|
const imageUrl = asset.image_url || detail.cover?.url;
|
||||||
|
if (!imageUrl) continue;
|
||||||
|
const saved = await downloadRemoteImage(imageUrl);
|
||||||
|
if (!saved) {
|
||||||
|
log(lines, onLog, ` failed download: ${imageUrl}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const media = await prisma.media.create({
|
||||||
|
data: {
|
||||||
|
projectId: project.id,
|
||||||
|
type: "image",
|
||||||
|
url: saved.url,
|
||||||
|
thumbnailUrl: saved.thumbnailUrl,
|
||||||
|
filename: saved.filename,
|
||||||
|
mimeType: saved.mimeType,
|
||||||
|
width: saved.width || asset.width,
|
||||||
|
height: saved.height || asset.height,
|
||||||
|
sizeBytes: saved.sizeBytes,
|
||||||
|
alt: asset.title || detail.title,
|
||||||
|
sortOrder: sortOrder++,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (!thumbnailId) thumbnailId = media.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (thumbnailId) {
|
||||||
|
await prisma.project.update({
|
||||||
|
where: { id: project.id },
|
||||||
|
data: { thumbnailId },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
await indexProject(project.id);
|
||||||
|
log(lines, onLog, ` created ${visibility} project ${project.slug}`);
|
||||||
|
return { outcome: "imported", title: detail.title, slug: project.slug };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function listUserProjects(username: string): Promise<AsProjectListItem[]> {
|
||||||
|
const all: AsProjectListItem[] = [];
|
||||||
|
let page = 1;
|
||||||
|
for (;;) {
|
||||||
|
const url = `https://www.artstation.com/users/${username}/projects.json?page=${page}`;
|
||||||
|
const data = await fetchJson<{ data?: AsProjectListItem[] } | AsProjectListItem[]>(
|
||||||
|
url
|
||||||
|
);
|
||||||
|
const batch = Array.isArray(data) ? data : data.data || [];
|
||||||
|
if (batch.length === 0) break;
|
||||||
|
all.push(...batch);
|
||||||
|
if (batch.length < 50) break;
|
||||||
|
page++;
|
||||||
|
if (page > 20) break;
|
||||||
|
}
|
||||||
|
return all;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Synchronous full import (CLI / awaited admin call). */
|
||||||
|
export async function runArtStationImport(
|
||||||
|
options: ImportOptions = {}
|
||||||
|
): Promise<ImportResult> {
|
||||||
|
const username = (options.username || "jmartgraphix").trim() || "jmartgraphix";
|
||||||
|
const dryRun = !!options.dryRun;
|
||||||
|
const lines: string[] = [];
|
||||||
|
const onLog = options.onLog;
|
||||||
|
const created: { title: string; slug: string }[] = [];
|
||||||
|
let imported = 0;
|
||||||
|
let skipped = 0;
|
||||||
|
let errors = 0;
|
||||||
|
let found = 0;
|
||||||
|
|
||||||
|
log(lines, onLog, `ArtStation import (user=${username}, dryRun=${dryRun})`);
|
||||||
|
|
||||||
|
if (options.url) {
|
||||||
|
found = 1;
|
||||||
|
try {
|
||||||
|
const r = await importProject(options.url, options, lines);
|
||||||
|
if (r.outcome === "imported") {
|
||||||
|
imported++;
|
||||||
|
if (r.slug) created.push({ title: r.title || "", slug: r.slug });
|
||||||
|
} else if (r.outcome === "skipped") skipped++;
|
||||||
|
else errors++;
|
||||||
|
} catch (err) {
|
||||||
|
errors++;
|
||||||
|
log(lines, onLog, ` error: ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const list = await listUserProjects(username);
|
||||||
|
found = list.length;
|
||||||
|
log(lines, onLog, `Found ${list.length} projects`);
|
||||||
|
for (const item of list) {
|
||||||
|
const hash = item.hash_id || String(item.id);
|
||||||
|
try {
|
||||||
|
const r = await importProject(hash, options, lines);
|
||||||
|
if (r.outcome === "imported") {
|
||||||
|
imported++;
|
||||||
|
if (r.slug) created.push({ title: r.title || item.title, slug: r.slug });
|
||||||
|
} else if (r.outcome === "skipped") skipped++;
|
||||||
|
else errors++;
|
||||||
|
await new Promise((r) => setTimeout(r, 400));
|
||||||
|
} catch (err) {
|
||||||
|
errors++;
|
||||||
|
log(lines, onLog, ` error importing ${item.title}: ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!dryRun && imported > 0) {
|
||||||
|
try {
|
||||||
|
const n = await reindexAllProjects();
|
||||||
|
log(lines, onLog, `Typesense reindexed (${n} projects)`);
|
||||||
|
} catch (err) {
|
||||||
|
log(lines, onLog, `Typesense reindex warning: ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log(
|
||||||
|
lines,
|
||||||
|
onLog,
|
||||||
|
`Done. imported=${imported} skipped=${skipped} errors=${errors}`
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
username,
|
||||||
|
dryRun,
|
||||||
|
found,
|
||||||
|
imported,
|
||||||
|
skipped,
|
||||||
|
errors,
|
||||||
|
log: lines,
|
||||||
|
created,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Start import in background; only one job at a time. */
|
||||||
|
export function startArtStationImportJob(options: ImportOptions = {}): {
|
||||||
|
started: boolean;
|
||||||
|
message: string;
|
||||||
|
} {
|
||||||
|
if (currentJob.running) {
|
||||||
|
return { started: false, message: "An import is already running" };
|
||||||
|
}
|
||||||
|
currentJob = {
|
||||||
|
running: true,
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
log: [],
|
||||||
|
};
|
||||||
|
runArtStationImport({
|
||||||
|
...options,
|
||||||
|
onLog: () => {
|
||||||
|
/* already pushed to currentJob.log */
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.then((result) => {
|
||||||
|
currentJob = {
|
||||||
|
running: false,
|
||||||
|
startedAt: currentJob.startedAt,
|
||||||
|
finishedAt: new Date().toISOString(),
|
||||||
|
result,
|
||||||
|
log: result.log,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
currentJob = {
|
||||||
|
running: false,
|
||||||
|
startedAt: currentJob.startedAt,
|
||||||
|
finishedAt: new Date().toISOString(),
|
||||||
|
error: (err as Error).message,
|
||||||
|
log: currentJob.log,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return { started: true, message: "Import started" };
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user