Add admin UI for ArtStation re-import
Expose import as a background admin job with status polling, dry-run and optional publish; share logic with the CLI via a service module.
This commit is contained in:
@@ -1,26 +1,17 @@
|
||||
/**
|
||||
* ArtStation project importer for jmartgraphix portfolio.
|
||||
* ArtStation project importer CLI.
|
||||
*
|
||||
* Usage:
|
||||
* npm run import:artstation -- --user jmartgraphix
|
||||
* npm run import:artstation -- --user jmartgraphix --dry-run
|
||||
* npm run import:artstation -- --url https://www.artstation.com/artwork/XXXX
|
||||
* npm run import:artstation -- --user jmartgraphix --publish
|
||||
*
|
||||
* Fetches public ArtStation JSON endpoints, downloads images, and creates
|
||||
* draft projects so you can edit/publish from the admin panel afterward.
|
||||
* Prefer the Admin → Dashboard “Import from ArtStation” button in production.
|
||||
*/
|
||||
import "dotenv/config";
|
||||
import { PrismaClient, Visibility } from "@prisma/client";
|
||||
import slugify from "slugify";
|
||||
import { downloadRemoteImage } from "../src/lib/media.js";
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
const USER_AGENT =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
||||
|
||||
/** FlareSolverr (or direct fetch) — ArtStation sits behind Cloudflare. */
|
||||
const FLARESOLVERR_URL =
|
||||
process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1";
|
||||
import { runArtStationImport } from "../src/services/artstation-import.js";
|
||||
import { prisma } from "../src/lib/prisma.js";
|
||||
|
||||
function arg(name: string, fallback?: string): string | undefined {
|
||||
const idx = process.argv.indexOf(`--${name}`);
|
||||
@@ -32,312 +23,16 @@ function hasFlag(name: string): boolean {
|
||||
return process.argv.includes(`--${name}`);
|
||||
}
|
||||
|
||||
function extractJsonFromFlareBody(body: string): unknown {
|
||||
// FlareSolverr often wraps JSON in <pre>…</pre>
|
||||
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i);
|
||||
const raw = (pre ? pre[1] : body).trim();
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
|
||||
async function fetchViaFlareSolverr(url: string): Promise<unknown> {
|
||||
const res = await fetch(FLARESOLVERR_URL, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
cmd: "request.get",
|
||||
url,
|
||||
maxTimeout: 90_000,
|
||||
}),
|
||||
signal: AbortSignal.timeout(100_000),
|
||||
});
|
||||
if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`);
|
||||
const payload = (await res.json()) as {
|
||||
status: string;
|
||||
message?: string;
|
||||
solution?: { status: number; response?: string };
|
||||
};
|
||||
if (payload.status !== "ok" || !payload.solution?.response) {
|
||||
throw new Error(
|
||||
`FlareSolverr failed for ${url}: ${payload.message || payload.status}`
|
||||
);
|
||||
}
|
||||
if (payload.solution.status >= 400) {
|
||||
throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`);
|
||||
}
|
||||
return extractJsonFromFlareBody(payload.solution.response);
|
||||
}
|
||||
|
||||
async function fetchJson<T>(url: string): Promise<T> {
|
||||
// Prefer FlareSolverr (Cloudflare). Fall back to direct fetch.
|
||||
try {
|
||||
return (await fetchViaFlareSolverr(url)) as T;
|
||||
} catch (flareErr) {
|
||||
console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`);
|
||||
}
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": USER_AGENT,
|
||||
Accept: "application/json, text/plain, */*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
Referer: "https://www.artstation.com/",
|
||||
},
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
|
||||
return res.json() as Promise<T>;
|
||||
}
|
||||
|
||||
interface AsProjectListItem {
|
||||
id: number;
|
||||
title: string;
|
||||
hash_id?: string;
|
||||
permalink?: string;
|
||||
cover?: { url?: string; small_square_url?: string };
|
||||
mediums?: { name: string }[];
|
||||
categories?: { name: string }[];
|
||||
published_at?: string;
|
||||
}
|
||||
|
||||
interface AsProjectDetail {
|
||||
id: number;
|
||||
title: string;
|
||||
description?: string;
|
||||
hash_id?: string;
|
||||
permalink?: string;
|
||||
published_at?: string;
|
||||
tags?: string[];
|
||||
mediums?: { name: string }[];
|
||||
categories?: { name: string }[];
|
||||
software_items?: { name: string }[];
|
||||
assets?: {
|
||||
id: number;
|
||||
title?: string;
|
||||
asset_type?: string;
|
||||
image_url?: string;
|
||||
has_image?: boolean;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}[];
|
||||
cover?: { url?: string };
|
||||
user?: { full_name?: string; username?: string };
|
||||
}
|
||||
|
||||
// Map ArtStation medium/category names to our category slugs
|
||||
const CATEGORY_MAP: Record<string, string> = {
|
||||
"3d": "3d-modeling",
|
||||
"3d modeling": "3d-modeling",
|
||||
modeling: "3d-modeling",
|
||||
characters: "3d-modeling",
|
||||
environments: "3d-modeling",
|
||||
animation: "3d-animation",
|
||||
"3d animation": "3d-animation",
|
||||
cad: "cad",
|
||||
product: "cad",
|
||||
industrial: "cad",
|
||||
sculpting: "3d-sculpting",
|
||||
"digital sculpting": "3d-sculpting",
|
||||
ai: "ai-generated-content",
|
||||
"ai art": "ai-generated-content",
|
||||
video: "video",
|
||||
cinematography: "video",
|
||||
photography: "photography",
|
||||
"graphic design": "graphic-design",
|
||||
design: "graphic-design",
|
||||
illustration: "illustration",
|
||||
concept: "illustration",
|
||||
};
|
||||
|
||||
async function resolveCategoryIds(names: string[]): Promise<string[]> {
|
||||
const all = await prisma.category.findMany();
|
||||
const bySlug = Object.fromEntries(all.map((c) => [c.slug, c.id]));
|
||||
const ids = new Set<string>();
|
||||
for (const n of names) {
|
||||
const key = n.toLowerCase().trim();
|
||||
const mapped = CATEGORY_MAP[key];
|
||||
if (mapped && bySlug[mapped]) ids.add(bySlug[mapped]);
|
||||
const direct = all.find((c) => c.name.toLowerCase() === key || c.slug === slugify(key, { lower: true, strict: true }));
|
||||
if (direct) ids.add(direct.id);
|
||||
}
|
||||
return [...ids];
|
||||
}
|
||||
|
||||
async function uniqueSlug(title: string): Promise<string> {
|
||||
const base = slugify(title, { lower: true, strict: true }) || "artwork";
|
||||
let slug = base;
|
||||
let n = 2;
|
||||
while (await prisma.project.findUnique({ where: { slug } })) {
|
||||
slug = `${base}-${n++}`;
|
||||
}
|
||||
return slug;
|
||||
}
|
||||
|
||||
async function importProject(hashOrUrl: string, dryRun: boolean): Promise<void> {
|
||||
let hash = hashOrUrl;
|
||||
const m = hashOrUrl.match(/artstation\.com\/artwork\/([A-Za-z0-9]+)/);
|
||||
if (m) hash = m[1];
|
||||
|
||||
const detail = await fetchJson<AsProjectDetail>(
|
||||
`https://www.artstation.com/projects/${hash}.json`
|
||||
);
|
||||
|
||||
const sourceUrl =
|
||||
detail.permalink || `https://www.artstation.com/artwork/${detail.hash_id || hash}`;
|
||||
const existing = await prisma.project.findFirst({
|
||||
where: { sourceUrl },
|
||||
});
|
||||
if (existing) {
|
||||
console.log(` skip (exists): ${detail.title}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const mediums = (detail.mediums || []).map((m) => m.name);
|
||||
const cats = (detail.categories || []).map((c) => c.name);
|
||||
const software = (detail.software_items || []).map((s) => s.name);
|
||||
const tags = detail.tags || [];
|
||||
const categoryIds = await resolveCategoryIds([...mediums, ...cats]);
|
||||
|
||||
console.log(` import: ${detail.title} (${(detail.assets || []).length} assets)`);
|
||||
if (dryRun) return;
|
||||
|
||||
const slug = await uniqueSlug(detail.title);
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
title: detail.title,
|
||||
slug,
|
||||
description: stripHtml(detail.description || ""),
|
||||
shortDescription: stripHtml(detail.description || "").slice(0, 280) || null,
|
||||
date: detail.published_at ? new Date(detail.published_at) : null,
|
||||
software,
|
||||
externalLinks: [{ label: "ArtStation", url: sourceUrl }],
|
||||
featured: false,
|
||||
displayPriority: 100,
|
||||
visibility: Visibility.draft,
|
||||
sourceUrl,
|
||||
sourcePlatform: "artstation",
|
||||
},
|
||||
});
|
||||
|
||||
for (const cid of categoryIds) {
|
||||
await prisma.projectCategory.create({
|
||||
data: { projectId: project.id, categoryId: cid },
|
||||
});
|
||||
}
|
||||
|
||||
for (const t of tags) {
|
||||
const tslug = slugify(t, { lower: true, strict: true });
|
||||
if (!tslug) continue;
|
||||
const tag = await prisma.tag.upsert({
|
||||
where: { slug: tslug },
|
||||
create: { name: t, slug: tslug },
|
||||
update: {},
|
||||
});
|
||||
await prisma.projectTag.upsert({
|
||||
where: { projectId_tagId: { projectId: project.id, tagId: tag.id } },
|
||||
create: { projectId: project.id, tagId: tag.id },
|
||||
update: {},
|
||||
});
|
||||
}
|
||||
|
||||
let sortOrder = 0;
|
||||
let thumbnailId: string | null = null;
|
||||
const assets = detail.assets || [];
|
||||
for (const asset of assets) {
|
||||
if (asset.asset_type && asset.asset_type !== "image" && !asset.has_image) continue;
|
||||
const imageUrl = asset.image_url || detail.cover?.url;
|
||||
if (!imageUrl) continue;
|
||||
const saved = await downloadRemoteImage(imageUrl);
|
||||
if (!saved) {
|
||||
console.warn(` failed download: ${imageUrl}`);
|
||||
continue;
|
||||
}
|
||||
const media = await prisma.media.create({
|
||||
data: {
|
||||
projectId: project.id,
|
||||
type: "image",
|
||||
url: saved.url,
|
||||
thumbnailUrl: saved.thumbnailUrl,
|
||||
filename: saved.filename,
|
||||
mimeType: saved.mimeType,
|
||||
width: saved.width || asset.width,
|
||||
height: saved.height || asset.height,
|
||||
sizeBytes: saved.sizeBytes,
|
||||
alt: asset.title || detail.title,
|
||||
sortOrder: sortOrder++,
|
||||
},
|
||||
});
|
||||
if (!thumbnailId) thumbnailId = media.id;
|
||||
}
|
||||
|
||||
if (thumbnailId) {
|
||||
await prisma.project.update({
|
||||
where: { id: project.id },
|
||||
data: { thumbnailId },
|
||||
});
|
||||
}
|
||||
|
||||
console.log(` created draft project ${project.slug}`);
|
||||
}
|
||||
|
||||
function stripHtml(html: string): string {
|
||||
return html
|
||||
.replace(/<br\s*\/?>/gi, "\n")
|
||||
.replace(/<\/p>/gi, "\n\n")
|
||||
.replace(/<[^>]+>/g, "")
|
||||
.replace(/ /g, " ")
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
async function listUserProjects(username: string): Promise<AsProjectListItem[]> {
|
||||
const all: AsProjectListItem[] = [];
|
||||
let page = 1;
|
||||
for (;;) {
|
||||
// Public profile projects endpoint
|
||||
const url = `https://www.artstation.com/users/${username}/projects.json?page=${page}`;
|
||||
try {
|
||||
const data = await fetchJson<{ data?: AsProjectListItem[] } | AsProjectListItem[]>(url);
|
||||
const batch = Array.isArray(data) ? data : data.data || [];
|
||||
if (batch.length === 0) break;
|
||||
all.push(...batch);
|
||||
if (batch.length < 50) break;
|
||||
page++;
|
||||
if (page > 20) break;
|
||||
} catch (err) {
|
||||
console.error(`Failed to list page ${page}:`, err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const dryRun = hasFlag("dry-run");
|
||||
const user = arg("user", "jmartgraphix")!;
|
||||
const singleUrl = arg("url");
|
||||
|
||||
console.log(`ArtStation import (user=${user}, dryRun=${dryRun})`);
|
||||
|
||||
if (singleUrl) {
|
||||
await importProject(singleUrl, dryRun);
|
||||
} else {
|
||||
const list = await listUserProjects(user);
|
||||
console.log(`Found ${list.length} projects`);
|
||||
for (const item of list) {
|
||||
const hash = item.hash_id || String(item.id);
|
||||
try {
|
||||
await importProject(hash, dryRun);
|
||||
await new Promise((r) => setTimeout(r, 500)); // be polite
|
||||
} catch (err) {
|
||||
console.error(` error importing ${item.title}:`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log("Done. Review drafts in the admin panel and publish when ready.");
|
||||
const result = await runArtStationImport({
|
||||
username: arg("user", "jmartgraphix"),
|
||||
url: arg("url"),
|
||||
dryRun: hasFlag("dry-run"),
|
||||
publish: hasFlag("publish"),
|
||||
});
|
||||
console.log(
|
||||
`Summary: found=${result.found} imported=${result.imported} skipped=${result.skipped} errors=${result.errors}`
|
||||
);
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
@@ -4,6 +4,10 @@ import { prisma } from "../lib/prisma.js";
|
||||
import { requireAdmin } from "../lib/auth.js";
|
||||
import { uniqueCategorySlug, uniqueTagSlug, makeSlug } from "../lib/slug.js";
|
||||
import { reindexAllProjects } from "../lib/typesense.js";
|
||||
import {
|
||||
getImportJobStatus,
|
||||
startArtStationImportJob,
|
||||
} from "../services/artstation-import.js";
|
||||
|
||||
export async function adminMetaRoutes(app: FastifyInstance) {
|
||||
// Categories
|
||||
@@ -279,4 +283,39 @@ export async function adminMetaRoutes(app: FastifyInstance) {
|
||||
const count = await reindexAllProjects();
|
||||
return { ok: true, indexed: count };
|
||||
});
|
||||
|
||||
// ArtStation import
|
||||
app.get("/api/v1/admin/import/artstation", async (req, reply) => {
|
||||
if (!requireAdmin(req, reply)) return;
|
||||
return getImportJobStatus();
|
||||
});
|
||||
|
||||
app.post("/api/v1/admin/import/artstation", async (req, reply) => {
|
||||
if (!requireAdmin(req, reply)) return;
|
||||
const body = z
|
||||
.object({
|
||||
username: z.string().min(1).optional().default("jmartgraphix"),
|
||||
dryRun: z.boolean().optional().default(false),
|
||||
publish: z.boolean().optional().default(false),
|
||||
url: z.string().url().optional(),
|
||||
})
|
||||
.parse(req.body ?? {});
|
||||
|
||||
const { started, message } = startArtStationImportJob({
|
||||
username: body.username,
|
||||
dryRun: body.dryRun,
|
||||
publish: body.publish,
|
||||
url: body.url,
|
||||
defaultCad: true,
|
||||
});
|
||||
|
||||
if (!started) {
|
||||
return reply.code(409).send({ ok: false, message, status: getImportJobStatus() });
|
||||
}
|
||||
return reply.code(202).send({
|
||||
ok: true,
|
||||
message,
|
||||
status: getImportJobStatus(),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
/**
|
||||
* ArtStation portfolio importer (shared by CLI + admin API).
|
||||
* Uses FlareSolverr when available to bypass Cloudflare.
|
||||
*/
|
||||
import { Visibility } from "@prisma/client";
|
||||
import slugify from "slugify";
|
||||
import { prisma } from "../lib/prisma.js";
|
||||
import { downloadRemoteImage } from "../lib/media.js";
|
||||
import { indexProject, reindexAllProjects } from "../lib/typesense.js";
|
||||
|
||||
const USER_AGENT =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
||||
|
||||
const FLARESOLVERR_URL =
|
||||
process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1";
|
||||
|
||||
const CATEGORY_MAP: Record<string, string> = {
|
||||
"3d": "3d-modeling",
|
||||
"3d modeling": "3d-modeling",
|
||||
modeling: "3d-modeling",
|
||||
characters: "3d-modeling",
|
||||
environments: "3d-modeling",
|
||||
animation: "3d-animation",
|
||||
"3d animation": "3d-animation",
|
||||
cad: "cad",
|
||||
product: "cad",
|
||||
industrial: "cad",
|
||||
sculpting: "3d-sculpting",
|
||||
"digital sculpting": "3d-sculpting",
|
||||
ai: "ai-generated-content",
|
||||
"ai art": "ai-generated-content",
|
||||
video: "video",
|
||||
cinematography: "video",
|
||||
photography: "photography",
|
||||
"graphic design": "graphic-design",
|
||||
design: "graphic-design",
|
||||
illustration: "illustration",
|
||||
concept: "illustration",
|
||||
};
|
||||
|
||||
export interface ImportOptions {
|
||||
username?: string;
|
||||
url?: string;
|
||||
dryRun?: boolean;
|
||||
/** Publish new imports instead of leaving as draft */
|
||||
publish?: boolean;
|
||||
/** Assign CAD when no category matched */
|
||||
defaultCad?: boolean;
|
||||
onLog?: (line: string) => void;
|
||||
}
|
||||
|
||||
export interface ImportResult {
|
||||
username: string;
|
||||
dryRun: boolean;
|
||||
found: number;
|
||||
imported: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
log: string[];
|
||||
created: { title: string; slug: string }[];
|
||||
}
|
||||
|
||||
export type ImportJobStatus = {
|
||||
running: boolean;
|
||||
startedAt?: string;
|
||||
finishedAt?: string;
|
||||
result?: ImportResult;
|
||||
error?: string;
|
||||
log: string[];
|
||||
};
|
||||
|
||||
let currentJob: ImportJobStatus = { running: false, log: [] };
|
||||
|
||||
export function getImportJobStatus(): ImportJobStatus {
|
||||
return { ...currentJob, log: [...currentJob.log] };
|
||||
}
|
||||
|
||||
function log(lines: string[], onLog: ImportOptions["onLog"], line: string) {
|
||||
lines.push(line);
|
||||
currentJob.log.push(line);
|
||||
onLog?.(line);
|
||||
console.log(line);
|
||||
}
|
||||
|
||||
function extractJsonFromFlareBody(body: string): unknown {
|
||||
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i);
|
||||
const raw = (pre ? pre[1] : body).trim();
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
|
||||
async function fetchViaFlareSolverr(url: string): Promise<unknown> {
|
||||
const res = await fetch(FLARESOLVERR_URL, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
cmd: "request.get",
|
||||
url,
|
||||
maxTimeout: 90_000,
|
||||
}),
|
||||
signal: AbortSignal.timeout(100_000),
|
||||
});
|
||||
if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`);
|
||||
const payload = (await res.json()) as {
|
||||
status: string;
|
||||
message?: string;
|
||||
solution?: { status: number; response?: string };
|
||||
};
|
||||
if (payload.status !== "ok" || !payload.solution?.response) {
|
||||
throw new Error(
|
||||
`FlareSolverr failed for ${url}: ${payload.message || payload.status}`
|
||||
);
|
||||
}
|
||||
if (payload.solution.status >= 400) {
|
||||
throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`);
|
||||
}
|
||||
return extractJsonFromFlareBody(payload.solution.response);
|
||||
}
|
||||
|
||||
async function fetchJson<T>(url: string): Promise<T> {
|
||||
try {
|
||||
return (await fetchViaFlareSolverr(url)) as T;
|
||||
} catch (flareErr) {
|
||||
console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`);
|
||||
}
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": USER_AGENT,
|
||||
Accept: "application/json, text/plain, */*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
Referer: "https://www.artstation.com/",
|
||||
},
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
|
||||
return res.json() as Promise<T>;
|
||||
}
|
||||
|
||||
interface AsProjectListItem {
|
||||
id: number;
|
||||
title: string;
|
||||
hash_id?: string;
|
||||
permalink?: string;
|
||||
}
|
||||
|
||||
interface AsProjectDetail {
|
||||
id: number;
|
||||
title: string;
|
||||
description?: string;
|
||||
hash_id?: string;
|
||||
permalink?: string;
|
||||
published_at?: string;
|
||||
tags?: string[];
|
||||
mediums?: { name: string }[];
|
||||
categories?: { name: string }[];
|
||||
software_items?: { name: string }[];
|
||||
assets?: {
|
||||
id: number;
|
||||
title?: string;
|
||||
asset_type?: string;
|
||||
image_url?: string;
|
||||
has_image?: boolean;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}[];
|
||||
cover?: { url?: string };
|
||||
}
|
||||
|
||||
function stripHtml(html: string): string {
|
||||
return html
|
||||
.replace(/<br\s*\/?>/gi, "\n")
|
||||
.replace(/<\/p>/gi, "\n\n")
|
||||
.replace(/<[^>]+>/g, "")
|
||||
.replace(/ /g, " ")
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
async function resolveCategoryIds(names: string[]): Promise<string[]> {
|
||||
const all = await prisma.category.findMany();
|
||||
const bySlug = Object.fromEntries(all.map((c) => [c.slug, c.id]));
|
||||
const ids = new Set<string>();
|
||||
for (const n of names) {
|
||||
const key = n.toLowerCase().trim();
|
||||
const mapped = CATEGORY_MAP[key];
|
||||
if (mapped && bySlug[mapped]) ids.add(bySlug[mapped]);
|
||||
const direct = all.find(
|
||||
(c) =>
|
||||
c.name.toLowerCase() === key ||
|
||||
c.slug === slugify(key, { lower: true, strict: true })
|
||||
);
|
||||
if (direct) ids.add(direct.id);
|
||||
}
|
||||
return [...ids];
|
||||
}
|
||||
|
||||
async function uniqueSlug(title: string): Promise<string> {
|
||||
const base = slugify(title, { lower: true, strict: true }) || "artwork";
|
||||
let slug = base;
|
||||
let n = 2;
|
||||
while (await prisma.project.findUnique({ where: { slug } })) {
|
||||
slug = `${base}-${n++}`;
|
||||
}
|
||||
return slug;
|
||||
}
|
||||
|
||||
type ImportProjectOutcome = "imported" | "skipped" | "error";
|
||||
|
||||
async function importProject(
|
||||
hashOrUrl: string,
|
||||
opts: ImportOptions,
|
||||
lines: string[]
|
||||
): Promise<{ outcome: ImportProjectOutcome; title?: string; slug?: string }> {
|
||||
const onLog = opts.onLog;
|
||||
let hash = hashOrUrl;
|
||||
const m = hashOrUrl.match(/artstation\.com\/artwork\/([A-Za-z0-9]+)/);
|
||||
if (m) hash = m[1];
|
||||
|
||||
const detail = await fetchJson<AsProjectDetail>(
|
||||
`https://www.artstation.com/projects/${hash}.json`
|
||||
);
|
||||
|
||||
const sourceUrl =
|
||||
detail.permalink ||
|
||||
`https://www.artstation.com/artwork/${detail.hash_id || hash}`;
|
||||
const existing = await prisma.project.findFirst({ where: { sourceUrl } });
|
||||
if (existing) {
|
||||
log(lines, onLog, ` skip (exists): ${detail.title}`);
|
||||
return { outcome: "skipped", title: detail.title, slug: existing.slug };
|
||||
}
|
||||
|
||||
const mediums = (detail.mediums || []).map((x) => x.name);
|
||||
const cats = (detail.categories || []).map((c) => c.name);
|
||||
const software = (detail.software_items || []).map((s) => s.name);
|
||||
const tags = detail.tags || [];
|
||||
let categoryIds = await resolveCategoryIds([...mediums, ...cats]);
|
||||
|
||||
if (categoryIds.length === 0 && opts.defaultCad !== false) {
|
||||
const cad = await prisma.category.findUnique({ where: { slug: "cad" } });
|
||||
if (cad) categoryIds = [cad.id];
|
||||
}
|
||||
|
||||
log(
|
||||
lines,
|
||||
onLog,
|
||||
` import: ${detail.title} (${(detail.assets || []).length} assets)`
|
||||
);
|
||||
if (opts.dryRun) {
|
||||
return { outcome: "imported", title: detail.title };
|
||||
}
|
||||
|
||||
const visibility = opts.publish ? Visibility.published : Visibility.draft;
|
||||
const slug = await uniqueSlug(detail.title);
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
title: detail.title,
|
||||
slug,
|
||||
description: stripHtml(detail.description || ""),
|
||||
shortDescription: stripHtml(detail.description || "").slice(0, 280) || null,
|
||||
date: detail.published_at ? new Date(detail.published_at) : null,
|
||||
software,
|
||||
externalLinks: [{ label: "ArtStation", url: sourceUrl }],
|
||||
featured: false,
|
||||
displayPriority: 100,
|
||||
visibility,
|
||||
sourceUrl,
|
||||
sourcePlatform: "artstation",
|
||||
publishedAt: visibility === Visibility.published ? new Date() : null,
|
||||
},
|
||||
});
|
||||
|
||||
for (const cid of categoryIds) {
|
||||
await prisma.projectCategory.create({
|
||||
data: { projectId: project.id, categoryId: cid },
|
||||
});
|
||||
}
|
||||
|
||||
for (const t of tags) {
|
||||
const tslug = slugify(t, { lower: true, strict: true });
|
||||
if (!tslug) continue;
|
||||
const tag = await prisma.tag.upsert({
|
||||
where: { slug: tslug },
|
||||
create: { name: t, slug: tslug },
|
||||
update: {},
|
||||
});
|
||||
await prisma.projectTag.upsert({
|
||||
where: { projectId_tagId: { projectId: project.id, tagId: tag.id } },
|
||||
create: { projectId: project.id, tagId: tag.id },
|
||||
update: {},
|
||||
});
|
||||
}
|
||||
|
||||
let sortOrder = 0;
|
||||
let thumbnailId: string | null = null;
|
||||
for (const asset of detail.assets || []) {
|
||||
if (asset.asset_type && asset.asset_type !== "image" && !asset.has_image) continue;
|
||||
const imageUrl = asset.image_url || detail.cover?.url;
|
||||
if (!imageUrl) continue;
|
||||
const saved = await downloadRemoteImage(imageUrl);
|
||||
if (!saved) {
|
||||
log(lines, onLog, ` failed download: ${imageUrl}`);
|
||||
continue;
|
||||
}
|
||||
const media = await prisma.media.create({
|
||||
data: {
|
||||
projectId: project.id,
|
||||
type: "image",
|
||||
url: saved.url,
|
||||
thumbnailUrl: saved.thumbnailUrl,
|
||||
filename: saved.filename,
|
||||
mimeType: saved.mimeType,
|
||||
width: saved.width || asset.width,
|
||||
height: saved.height || asset.height,
|
||||
sizeBytes: saved.sizeBytes,
|
||||
alt: asset.title || detail.title,
|
||||
sortOrder: sortOrder++,
|
||||
},
|
||||
});
|
||||
if (!thumbnailId) thumbnailId = media.id;
|
||||
}
|
||||
|
||||
if (thumbnailId) {
|
||||
await prisma.project.update({
|
||||
where: { id: project.id },
|
||||
data: { thumbnailId },
|
||||
});
|
||||
}
|
||||
|
||||
await indexProject(project.id);
|
||||
log(lines, onLog, ` created ${visibility} project ${project.slug}`);
|
||||
return { outcome: "imported", title: detail.title, slug: project.slug };
|
||||
}
|
||||
|
||||
async function listUserProjects(username: string): Promise<AsProjectListItem[]> {
|
||||
const all: AsProjectListItem[] = [];
|
||||
let page = 1;
|
||||
for (;;) {
|
||||
const url = `https://www.artstation.com/users/${username}/projects.json?page=${page}`;
|
||||
const data = await fetchJson<{ data?: AsProjectListItem[] } | AsProjectListItem[]>(
|
||||
url
|
||||
);
|
||||
const batch = Array.isArray(data) ? data : data.data || [];
|
||||
if (batch.length === 0) break;
|
||||
all.push(...batch);
|
||||
if (batch.length < 50) break;
|
||||
page++;
|
||||
if (page > 20) break;
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
/** Synchronous full import (CLI / awaited admin call). */
|
||||
export async function runArtStationImport(
|
||||
options: ImportOptions = {}
|
||||
): Promise<ImportResult> {
|
||||
const username = (options.username || "jmartgraphix").trim() || "jmartgraphix";
|
||||
const dryRun = !!options.dryRun;
|
||||
const lines: string[] = [];
|
||||
const onLog = options.onLog;
|
||||
const created: { title: string; slug: string }[] = [];
|
||||
let imported = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
let found = 0;
|
||||
|
||||
log(lines, onLog, `ArtStation import (user=${username}, dryRun=${dryRun})`);
|
||||
|
||||
if (options.url) {
|
||||
found = 1;
|
||||
try {
|
||||
const r = await importProject(options.url, options, lines);
|
||||
if (r.outcome === "imported") {
|
||||
imported++;
|
||||
if (r.slug) created.push({ title: r.title || "", slug: r.slug });
|
||||
} else if (r.outcome === "skipped") skipped++;
|
||||
else errors++;
|
||||
} catch (err) {
|
||||
errors++;
|
||||
log(lines, onLog, ` error: ${(err as Error).message}`);
|
||||
}
|
||||
} else {
|
||||
const list = await listUserProjects(username);
|
||||
found = list.length;
|
||||
log(lines, onLog, `Found ${list.length} projects`);
|
||||
for (const item of list) {
|
||||
const hash = item.hash_id || String(item.id);
|
||||
try {
|
||||
const r = await importProject(hash, options, lines);
|
||||
if (r.outcome === "imported") {
|
||||
imported++;
|
||||
if (r.slug) created.push({ title: r.title || item.title, slug: r.slug });
|
||||
} else if (r.outcome === "skipped") skipped++;
|
||||
else errors++;
|
||||
await new Promise((r) => setTimeout(r, 400));
|
||||
} catch (err) {
|
||||
errors++;
|
||||
log(lines, onLog, ` error importing ${item.title}: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!dryRun && imported > 0) {
|
||||
try {
|
||||
const n = await reindexAllProjects();
|
||||
log(lines, onLog, `Typesense reindexed (${n} projects)`);
|
||||
} catch (err) {
|
||||
log(lines, onLog, `Typesense reindex warning: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
log(
|
||||
lines,
|
||||
onLog,
|
||||
`Done. imported=${imported} skipped=${skipped} errors=${errors}`
|
||||
);
|
||||
|
||||
return {
|
||||
username,
|
||||
dryRun,
|
||||
found,
|
||||
imported,
|
||||
skipped,
|
||||
errors,
|
||||
log: lines,
|
||||
created,
|
||||
};
|
||||
}
|
||||
|
||||
/** Start import in background; only one job at a time. */
|
||||
export function startArtStationImportJob(options: ImportOptions = {}): {
|
||||
started: boolean;
|
||||
message: string;
|
||||
} {
|
||||
if (currentJob.running) {
|
||||
return { started: false, message: "An import is already running" };
|
||||
}
|
||||
currentJob = {
|
||||
running: true,
|
||||
startedAt: new Date().toISOString(),
|
||||
log: [],
|
||||
};
|
||||
runArtStationImport({
|
||||
...options,
|
||||
onLog: () => {
|
||||
/* already pushed to currentJob.log */
|
||||
},
|
||||
})
|
||||
.then((result) => {
|
||||
currentJob = {
|
||||
running: false,
|
||||
startedAt: currentJob.startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
result,
|
||||
log: result.log,
|
||||
};
|
||||
})
|
||||
.catch((err) => {
|
||||
currentJob = {
|
||||
running: false,
|
||||
startedAt: currentJob.startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
error: (err as Error).message,
|
||||
log: currentJob.log,
|
||||
};
|
||||
});
|
||||
return { started: true, message: "Import started" };
|
||||
}
|
||||
Reference in New Issue
Block a user