diff --git a/client/src/lib/api.ts b/client/src/lib/api.ts index 1408d07..e80588b 100644 --- a/client/src/lib/api.ts +++ b/client/src/lib/api.ts @@ -229,8 +229,43 @@ export const api = { }), reindex: () => request<{ ok: boolean; indexed: number }>("/api/v1/admin/reindex", { method: "POST" }), + + importArtStation: (body: { + username?: string; + dryRun?: boolean; + publish?: boolean; + url?: string; + }) => + request<{ + ok: boolean; + message: string; + status: ArtStationImportStatus; + }>("/api/v1/admin/import/artstation", { + method: "POST", + body: JSON.stringify(body), + }), + + importArtStationStatus: () => + request("/api/v1/admin/import/artstation"), }; +export interface ArtStationImportStatus { + running: boolean; + startedAt?: string; + finishedAt?: string; + error?: string; + log: string[]; + result?: { + username: string; + dryRun: boolean; + found: number; + imported: number; + skipped: number; + errors: number; + created: { title: string; slug: string }[]; + }; +} + export function mediaSrc(url?: string | null): string { if (!url) return ""; if (url.startsWith("http") || url.startsWith("//")) return url; diff --git a/client/src/pages/admin/AdminDashboard.tsx b/client/src/pages/admin/AdminDashboard.tsx index 5e6b34f..0b01848 100644 --- a/client/src/pages/admin/AdminDashboard.tsx +++ b/client/src/pages/admin/AdminDashboard.tsx @@ -1,6 +1,6 @@ -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { Link } from "react-router-dom"; -import { api } from "../../lib/api"; +import { api, type ArtStationImportStatus } from "../../lib/api"; export function AdminDashboard() { const [stats, setStats] = useState({ @@ -11,8 +11,15 @@ export function AdminDashboard() { views: 0, }); const [msg, setMsg] = useState(null); + const [err, setErr] = useState(null); - useEffect(() => { + const [asUser, setAsUser] = useState("jmartgraphix"); + const [asDryRun, setAsDryRun] = useState(false); + const [asPublish, setAsPublish] = useState(false); + const [importStatus, setImportStatus] = useState(null); + const pollRef = useRef | null>(null); + + function refreshStats() { Promise.all([ api.adminProjects({ perPage: 1 }), api.adminProjects({ visibility: "published", perPage: 1 }), @@ -28,22 +35,77 @@ export function AdminDashboard() { views: views.length, }); }); + } + + useEffect(() => { + refreshStats(); + api.importArtStationStatus().then(setImportStatus).catch(() => {}); + return () => { + if (pollRef.current) clearInterval(pollRef.current); + }; }, []); + function startPolling() { + if (pollRef.current) clearInterval(pollRef.current); + pollRef.current = setInterval(async () => { + try { + const s = await api.importArtStationStatus(); + setImportStatus(s); + if (!s.running) { + if (pollRef.current) clearInterval(pollRef.current); + pollRef.current = null; + refreshStats(); + if (s.result) { + setMsg( + `ArtStation import finished: ${s.result.imported} new, ${s.result.skipped} skipped, ${s.result.errors} errors (${s.result.found} on profile)` + ); + } else if (s.error) { + setErr(s.error); + } + } + } catch { + /* keep polling */ + } + }, 2000); + } + async function reindex() { + setErr(null); try { const r = await api.reindex(); setMsg(`Typesense reindexed ${r.indexed} projects`); } catch (e) { - setMsg((e as Error).message); + setErr((e as Error).message); } } + async function runArtStationImport() { + setErr(null); + setMsg(null); + try { + const res = await api.importArtStation({ + username: asUser.trim() || "jmartgraphix", + dryRun: asDryRun, + publish: asPublish, + }); + setImportStatus(res.status); + setMsg(res.message); + if (res.status.running) startPolling(); + } catch (e) { + setErr((e as Error).message); + } + } + + const running = importStatus?.running; + const logLines = importStatus?.log?.slice(-40) || []; + return (

Dashboard

Manage portfolio content protected by Authelia SSO.

{msg &&
{msg}
} + {err &&
{err}
} +
{stats.projects} @@ -66,6 +128,7 @@ export function AdminDashboard() { Portfolio views
+
New project @@ -77,7 +140,99 @@ export function AdminDashboard() { Reindex search
-
+ +
+

Import from ArtStation

+

+ Pull new public projects from your ArtStation profile. Existing entries (matched by + ArtStation URL) are skipped. New work lands as drafts unless you choose publish. +

+ +
+ + setAsUser(e.target.value)} + disabled={!!running} + placeholder="jmartgraphix" + /> +
+ +
+ + +
+ +
+ + {importStatus?.finishedAt && !running && ( + + Last run {new Date(importStatus.finishedAt).toLocaleString()} + + )} +
+ + {importStatus?.result && !running && ( +
+ Found {importStatus.result.found} · imported {importStatus.result.imported} · skipped{" "} + {importStatus.result.skipped} · errors {importStatus.result.errors} + {importStatus.result.created.length > 0 && ( + <> + {"\n"}New:{" "} + {importStatus.result.created.map((c) => c.title).join(", ")} + + )} +
+ )} + + {logLines.length > 0 && ( + 
+            {logLines.join("\n")}
+
+ )} +
+ +

Quick links for audiences

  • diff --git a/server/scripts/import-artstation.ts b/server/scripts/import-artstation.ts index a075486..b0440b5 100644 --- a/server/scripts/import-artstation.ts +++ b/server/scripts/import-artstation.ts @@ -1,26 +1,17 @@ /** - * ArtStation project importer for jmartgraphix portfolio. + * ArtStation project importer CLI. * * Usage: * npm run import:artstation -- --user jmartgraphix * npm run import:artstation -- --user jmartgraphix --dry-run * npm run import:artstation -- --url https://www.artstation.com/artwork/XXXX + * npm run import:artstation -- --user jmartgraphix --publish * - * Fetches public ArtStation JSON endpoints, downloads images, and creates - * draft projects so you can edit/publish from the admin panel afterward. + * Prefer the Admin → Dashboard “Import from ArtStation” button in production. */ import "dotenv/config"; -import { PrismaClient, Visibility } from "@prisma/client"; -import slugify from "slugify"; -import { downloadRemoteImage } from "../src/lib/media.js"; - -const prisma = new PrismaClient(); -const USER_AGENT = - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; - -/** FlareSolverr (or direct fetch) — ArtStation sits behind Cloudflare. */ -const FLARESOLVERR_URL = - process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1"; +import { runArtStationImport } from "../src/services/artstation-import.js"; +import { prisma } from "../src/lib/prisma.js"; function arg(name: string, fallback?: string): string | undefined { const idx = process.argv.indexOf(`--${name}`); @@ -32,312 +23,16 @@ function hasFlag(name: string): boolean { return process.argv.includes(`--${name}`); } -function extractJsonFromFlareBody(body: string): unknown { - // FlareSolverr often wraps JSON in 
    …
    - const pre = body.match(/]*>([\s\S]*?)<\/pre>/i); - const raw = (pre ? pre[1] : body).trim(); - return JSON.parse(raw); -} - -async function fetchViaFlareSolverr(url: string): Promise { - const res = await fetch(FLARESOLVERR_URL, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - cmd: "request.get", - url, - maxTimeout: 90_000, - }), - signal: AbortSignal.timeout(100_000), - }); - if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`); - const payload = (await res.json()) as { - status: string; - message?: string; - solution?: { status: number; response?: string }; - }; - if (payload.status !== "ok" || !payload.solution?.response) { - throw new Error( - `FlareSolverr failed for ${url}: ${payload.message || payload.status}` - ); - } - if (payload.solution.status >= 400) { - throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`); - } - return extractJsonFromFlareBody(payload.solution.response); -} - -async function fetchJson(url: string): Promise { - // Prefer FlareSolverr (Cloudflare). Fall back to direct fetch. - try { - return (await fetchViaFlareSolverr(url)) as T; - } catch (flareErr) { - console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`); - } - const res = await fetch(url, { - headers: { - "User-Agent": USER_AGENT, - Accept: "application/json, text/plain, */*", - "Accept-Language": "en-US,en;q=0.9", - Referer: "https://www.artstation.com/", - }, - signal: AbortSignal.timeout(30_000), - }); - if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`); - return res.json() as Promise; -} - -interface AsProjectListItem { - id: number; - title: string; - hash_id?: string; - permalink?: string; - cover?: { url?: string; small_square_url?: string }; - mediums?: { name: string }[]; - categories?: { name: string }[]; - published_at?: string; -} - -interface AsProjectDetail { - id: number; - title: string; - description?: string; - hash_id?: string; - permalink?: string; - published_at?: string; - tags?: string[]; - mediums?: { name: string }[]; - categories?: { name: string }[]; - software_items?: { name: string }[]; - assets?: { - id: number; - title?: string; - asset_type?: string; - image_url?: string; - has_image?: boolean; - width?: number; - height?: number; - }[]; - cover?: { url?: string }; - user?: { full_name?: string; username?: string }; -} - -// Map ArtStation medium/category names to our category slugs -const CATEGORY_MAP: Record = { - "3d": "3d-modeling", - "3d modeling": "3d-modeling", - modeling: "3d-modeling", - characters: "3d-modeling", - environments: "3d-modeling", - animation: "3d-animation", - "3d animation": "3d-animation", - cad: "cad", - product: "cad", - industrial: "cad", - sculpting: "3d-sculpting", - "digital sculpting": "3d-sculpting", - ai: "ai-generated-content", - "ai art": "ai-generated-content", - video: "video", - cinematography: "video", - photography: "photography", - "graphic design": "graphic-design", - design: "graphic-design", - illustration: "illustration", - concept: "illustration", -}; - -async function resolveCategoryIds(names: string[]): Promise { - const all = await prisma.category.findMany(); - const bySlug = Object.fromEntries(all.map((c) => [c.slug, c.id])); - const ids = new Set(); - for (const n of names) { - const key = n.toLowerCase().trim(); - const mapped = CATEGORY_MAP[key]; - if (mapped && bySlug[mapped]) ids.add(bySlug[mapped]); - const direct = all.find((c) => c.name.toLowerCase() === key || c.slug === slugify(key, { lower: true, strict: true })); - if (direct) ids.add(direct.id); - } - return [...ids]; -} - -async function uniqueSlug(title: string): Promise { - const base = slugify(title, { lower: true, strict: true }) || "artwork"; - let slug = base; - let n = 2; - while (await prisma.project.findUnique({ where: { slug } })) { - slug = `${base}-${n++}`; - } - return slug; -} - -async function importProject(hashOrUrl: string, dryRun: boolean): Promise { - let hash = hashOrUrl; - const m = hashOrUrl.match(/artstation\.com\/artwork\/([A-Za-z0-9]+)/); - if (m) hash = m[1]; - - const detail = await fetchJson( - `https://www.artstation.com/projects/${hash}.json` - ); - - const sourceUrl = - detail.permalink || `https://www.artstation.com/artwork/${detail.hash_id || hash}`; - const existing = await prisma.project.findFirst({ - where: { sourceUrl }, - }); - if (existing) { - console.log(` skip (exists): ${detail.title}`); - return; - } - - const mediums = (detail.mediums || []).map((m) => m.name); - const cats = (detail.categories || []).map((c) => c.name); - const software = (detail.software_items || []).map((s) => s.name); - const tags = detail.tags || []; - const categoryIds = await resolveCategoryIds([...mediums, ...cats]); - - console.log(` import: ${detail.title} (${(detail.assets || []).length} assets)`); - if (dryRun) return; - - const slug = await uniqueSlug(detail.title); - const project = await prisma.project.create({ - data: { - title: detail.title, - slug, - description: stripHtml(detail.description || ""), - shortDescription: stripHtml(detail.description || "").slice(0, 280) || null, - date: detail.published_at ? new Date(detail.published_at) : null, - software, - externalLinks: [{ label: "ArtStation", url: sourceUrl }], - featured: false, - displayPriority: 100, - visibility: Visibility.draft, - sourceUrl, - sourcePlatform: "artstation", - }, - }); - - for (const cid of categoryIds) { - await prisma.projectCategory.create({ - data: { projectId: project.id, categoryId: cid }, - }); - } - - for (const t of tags) { - const tslug = slugify(t, { lower: true, strict: true }); - if (!tslug) continue; - const tag = await prisma.tag.upsert({ - where: { slug: tslug }, - create: { name: t, slug: tslug }, - update: {}, - }); - await prisma.projectTag.upsert({ - where: { projectId_tagId: { projectId: project.id, tagId: tag.id } }, - create: { projectId: project.id, tagId: tag.id }, - update: {}, - }); - } - - let sortOrder = 0; - let thumbnailId: string | null = null; - const assets = detail.assets || []; - for (const asset of assets) { - if (asset.asset_type && asset.asset_type !== "image" && !asset.has_image) continue; - const imageUrl = asset.image_url || detail.cover?.url; - if (!imageUrl) continue; - const saved = await downloadRemoteImage(imageUrl); - if (!saved) { - console.warn(` failed download: ${imageUrl}`); - continue; - } - const media = await prisma.media.create({ - data: { - projectId: project.id, - type: "image", - url: saved.url, - thumbnailUrl: saved.thumbnailUrl, - filename: saved.filename, - mimeType: saved.mimeType, - width: saved.width || asset.width, - height: saved.height || asset.height, - sizeBytes: saved.sizeBytes, - alt: asset.title || detail.title, - sortOrder: sortOrder++, - }, - }); - if (!thumbnailId) thumbnailId = media.id; - } - - if (thumbnailId) { - await prisma.project.update({ - where: { id: project.id }, - data: { thumbnailId }, - }); - } - - console.log(` created draft project ${project.slug}`); -} - -function stripHtml(html: string): string { - return html - .replace(//gi, "\n") - .replace(/<\/p>/gi, "\n\n") - .replace(/<[^>]+>/g, "") - .replace(/ /g, " ") - .replace(/&/g, "&") - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, '"') - .replace(/\n{3,}/g, "\n\n") - .trim(); -} - -async function listUserProjects(username: string): Promise { - const all: AsProjectListItem[] = []; - let page = 1; - for (;;) { - // Public profile projects endpoint - const url = `https://www.artstation.com/users/${username}/projects.json?page=${page}`; - try { - const data = await fetchJson<{ data?: AsProjectListItem[] } | AsProjectListItem[]>(url); - const batch = Array.isArray(data) ? data : data.data || []; - if (batch.length === 0) break; - all.push(...batch); - if (batch.length < 50) break; - page++; - if (page > 20) break; - } catch (err) { - console.error(`Failed to list page ${page}:`, err); - break; - } - } - return all; -} - async function main() { - const dryRun = hasFlag("dry-run"); - const user = arg("user", "jmartgraphix")!; - const singleUrl = arg("url"); - - console.log(`ArtStation import (user=${user}, dryRun=${dryRun})`); - - if (singleUrl) { - await importProject(singleUrl, dryRun); - } else { - const list = await listUserProjects(user); - console.log(`Found ${list.length} projects`); - for (const item of list) { - const hash = item.hash_id || String(item.id); - try { - await importProject(hash, dryRun); - await new Promise((r) => setTimeout(r, 500)); // be polite - } catch (err) { - console.error(` error importing ${item.title}:`, err); - } - } - } - - console.log("Done. Review drafts in the admin panel and publish when ready."); + const result = await runArtStationImport({ + username: arg("user", "jmartgraphix"), + url: arg("url"), + dryRun: hasFlag("dry-run"), + publish: hasFlag("publish"), + }); + console.log( + `Summary: found=${result.found} imported=${result.imported} skipped=${result.skipped} errors=${result.errors}` + ); } main() diff --git a/server/src/routes/admin-meta.ts b/server/src/routes/admin-meta.ts index c192f93..2eff91d 100644 --- a/server/src/routes/admin-meta.ts +++ b/server/src/routes/admin-meta.ts @@ -4,6 +4,10 @@ import { prisma } from "../lib/prisma.js"; import { requireAdmin } from "../lib/auth.js"; import { uniqueCategorySlug, uniqueTagSlug, makeSlug } from "../lib/slug.js"; import { reindexAllProjects } from "../lib/typesense.js"; +import { + getImportJobStatus, + startArtStationImportJob, +} from "../services/artstation-import.js"; export async function adminMetaRoutes(app: FastifyInstance) { // Categories @@ -279,4 +283,39 @@ export async function adminMetaRoutes(app: FastifyInstance) { const count = await reindexAllProjects(); return { ok: true, indexed: count }; }); + + // ArtStation import + app.get("/api/v1/admin/import/artstation", async (req, reply) => { + if (!requireAdmin(req, reply)) return; + return getImportJobStatus(); + }); + + app.post("/api/v1/admin/import/artstation", async (req, reply) => { + if (!requireAdmin(req, reply)) return; + const body = z + .object({ + username: z.string().min(1).optional().default("jmartgraphix"), + dryRun: z.boolean().optional().default(false), + publish: z.boolean().optional().default(false), + url: z.string().url().optional(), + }) + .parse(req.body ?? {}); + + const { started, message } = startArtStationImportJob({ + username: body.username, + dryRun: body.dryRun, + publish: body.publish, + url: body.url, + defaultCad: true, + }); + + if (!started) { + return reply.code(409).send({ ok: false, message, status: getImportJobStatus() }); + } + return reply.code(202).send({ + ok: true, + message, + status: getImportJobStatus(), + }); + }); } diff --git a/server/src/services/artstation-import.ts b/server/src/services/artstation-import.ts new file mode 100644 index 0000000..802594e --- /dev/null +++ b/server/src/services/artstation-import.ts @@ -0,0 +1,470 @@ +/** + * ArtStation portfolio importer (shared by CLI + admin API). + * Uses FlareSolverr when available to bypass Cloudflare. + */ +import { Visibility } from "@prisma/client"; +import slugify from "slugify"; +import { prisma } from "../lib/prisma.js"; +import { downloadRemoteImage } from "../lib/media.js"; +import { indexProject, reindexAllProjects } from "../lib/typesense.js"; + +const USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; + +const FLARESOLVERR_URL = + process.env.FLARESOLVERR_URL || "http://flaresolverr:8191/v1"; + +const CATEGORY_MAP: Record = { + "3d": "3d-modeling", + "3d modeling": "3d-modeling", + modeling: "3d-modeling", + characters: "3d-modeling", + environments: "3d-modeling", + animation: "3d-animation", + "3d animation": "3d-animation", + cad: "cad", + product: "cad", + industrial: "cad", + sculpting: "3d-sculpting", + "digital sculpting": "3d-sculpting", + ai: "ai-generated-content", + "ai art": "ai-generated-content", + video: "video", + cinematography: "video", + photography: "photography", + "graphic design": "graphic-design", + design: "graphic-design", + illustration: "illustration", + concept: "illustration", +}; + +export interface ImportOptions { + username?: string; + url?: string; + dryRun?: boolean; + /** Publish new imports instead of leaving as draft */ + publish?: boolean; + /** Assign CAD when no category matched */ + defaultCad?: boolean; + onLog?: (line: string) => void; +} + +export interface ImportResult { + username: string; + dryRun: boolean; + found: number; + imported: number; + skipped: number; + errors: number; + log: string[]; + created: { title: string; slug: string }[]; +} + +export type ImportJobStatus = { + running: boolean; + startedAt?: string; + finishedAt?: string; + result?: ImportResult; + error?: string; + log: string[]; +}; + +let currentJob: ImportJobStatus = { running: false, log: [] }; + +export function getImportJobStatus(): ImportJobStatus { + return { ...currentJob, log: [...currentJob.log] }; +} + +function log(lines: string[], onLog: ImportOptions["onLog"], line: string) { + lines.push(line); + currentJob.log.push(line); + onLog?.(line); + console.log(line); +} + +function extractJsonFromFlareBody(body: string): unknown { + const pre = body.match(/]*>([\s\S]*?)<\/pre>/i); + const raw = (pre ? pre[1] : body).trim(); + return JSON.parse(raw); +} + +async function fetchViaFlareSolverr(url: string): Promise { + const res = await fetch(FLARESOLVERR_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + cmd: "request.get", + url, + maxTimeout: 90_000, + }), + signal: AbortSignal.timeout(100_000), + }); + if (!res.ok) throw new Error(`FlareSolverr HTTP ${res.status} for ${url}`); + const payload = (await res.json()) as { + status: string; + message?: string; + solution?: { status: number; response?: string }; + }; + if (payload.status !== "ok" || !payload.solution?.response) { + throw new Error( + `FlareSolverr failed for ${url}: ${payload.message || payload.status}` + ); + } + if (payload.solution.status >= 400) { + throw new Error(`ArtStation HTTP ${payload.solution.status} for ${url}`); + } + return extractJsonFromFlareBody(payload.solution.response); +} + +async function fetchJson(url: string): Promise { + try { + return (await fetchViaFlareSolverr(url)) as T; + } catch (flareErr) { + console.warn(` FlareSolverr miss, trying direct: ${(flareErr as Error).message}`); + } + const res = await fetch(url, { + headers: { + "User-Agent": USER_AGENT, + Accept: "application/json, text/plain, */*", + "Accept-Language": "en-US,en;q=0.9", + Referer: "https://www.artstation.com/", + }, + signal: AbortSignal.timeout(30_000), + }); + if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`); + return res.json() as Promise; +} + +interface AsProjectListItem { + id: number; + title: string; + hash_id?: string; + permalink?: string; +} + +interface AsProjectDetail { + id: number; + title: string; + description?: string; + hash_id?: string; + permalink?: string; + published_at?: string; + tags?: string[]; + mediums?: { name: string }[]; + categories?: { name: string }[]; + software_items?: { name: string }[]; + assets?: { + id: number; + title?: string; + asset_type?: string; + image_url?: string; + has_image?: boolean; + width?: number; + height?: number; + }[]; + cover?: { url?: string }; +} + +function stripHtml(html: string): string { + return html + .replace(//gi, "\n") + .replace(/<\/p>/gi, "\n\n") + .replace(/<[^>]+>/g, "") + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +async function resolveCategoryIds(names: string[]): Promise { + const all = await prisma.category.findMany(); + const bySlug = Object.fromEntries(all.map((c) => [c.slug, c.id])); + const ids = new Set(); + for (const n of names) { + const key = n.toLowerCase().trim(); + const mapped = CATEGORY_MAP[key]; + if (mapped && bySlug[mapped]) ids.add(bySlug[mapped]); + const direct = all.find( + (c) => + c.name.toLowerCase() === key || + c.slug === slugify(key, { lower: true, strict: true }) + ); + if (direct) ids.add(direct.id); + } + return [...ids]; +} + +async function uniqueSlug(title: string): Promise { + const base = slugify(title, { lower: true, strict: true }) || "artwork"; + let slug = base; + let n = 2; + while (await prisma.project.findUnique({ where: { slug } })) { + slug = `${base}-${n++}`; + } + return slug; +} + +type ImportProjectOutcome = "imported" | "skipped" | "error"; + +async function importProject( + hashOrUrl: string, + opts: ImportOptions, + lines: string[] +): Promise<{ outcome: ImportProjectOutcome; title?: string; slug?: string }> { + const onLog = opts.onLog; + let hash = hashOrUrl; + const m = hashOrUrl.match(/artstation\.com\/artwork\/([A-Za-z0-9]+)/); + if (m) hash = m[1]; + + const detail = await fetchJson( + `https://www.artstation.com/projects/${hash}.json` + ); + + const sourceUrl = + detail.permalink || + `https://www.artstation.com/artwork/${detail.hash_id || hash}`; + const existing = await prisma.project.findFirst({ where: { sourceUrl } }); + if (existing) { + log(lines, onLog, ` skip (exists): ${detail.title}`); + return { outcome: "skipped", title: detail.title, slug: existing.slug }; + } + + const mediums = (detail.mediums || []).map((x) => x.name); + const cats = (detail.categories || []).map((c) => c.name); + const software = (detail.software_items || []).map((s) => s.name); + const tags = detail.tags || []; + let categoryIds = await resolveCategoryIds([...mediums, ...cats]); + + if (categoryIds.length === 0 && opts.defaultCad !== false) { + const cad = await prisma.category.findUnique({ where: { slug: "cad" } }); + if (cad) categoryIds = [cad.id]; + } + + log( + lines, + onLog, + ` import: ${detail.title} (${(detail.assets || []).length} assets)` + ); + if (opts.dryRun) { + return { outcome: "imported", title: detail.title }; + } + + const visibility = opts.publish ? Visibility.published : Visibility.draft; + const slug = await uniqueSlug(detail.title); + const project = await prisma.project.create({ + data: { + title: detail.title, + slug, + description: stripHtml(detail.description || ""), + shortDescription: stripHtml(detail.description || "").slice(0, 280) || null, + date: detail.published_at ? new Date(detail.published_at) : null, + software, + externalLinks: [{ label: "ArtStation", url: sourceUrl }], + featured: false, + displayPriority: 100, + visibility, + sourceUrl, + sourcePlatform: "artstation", + publishedAt: visibility === Visibility.published ? new Date() : null, + }, + }); + + for (const cid of categoryIds) { + await prisma.projectCategory.create({ + data: { projectId: project.id, categoryId: cid }, + }); + } + + for (const t of tags) { + const tslug = slugify(t, { lower: true, strict: true }); + if (!tslug) continue; + const tag = await prisma.tag.upsert({ + where: { slug: tslug }, + create: { name: t, slug: tslug }, + update: {}, + }); + await prisma.projectTag.upsert({ + where: { projectId_tagId: { projectId: project.id, tagId: tag.id } }, + create: { projectId: project.id, tagId: tag.id }, + update: {}, + }); + } + + let sortOrder = 0; + let thumbnailId: string | null = null; + for (const asset of detail.assets || []) { + if (asset.asset_type && asset.asset_type !== "image" && !asset.has_image) continue; + const imageUrl = asset.image_url || detail.cover?.url; + if (!imageUrl) continue; + const saved = await downloadRemoteImage(imageUrl); + if (!saved) { + log(lines, onLog, ` failed download: ${imageUrl}`); + continue; + } + const media = await prisma.media.create({ + data: { + projectId: project.id, + type: "image", + url: saved.url, + thumbnailUrl: saved.thumbnailUrl, + filename: saved.filename, + mimeType: saved.mimeType, + width: saved.width || asset.width, + height: saved.height || asset.height, + sizeBytes: saved.sizeBytes, + alt: asset.title || detail.title, + sortOrder: sortOrder++, + }, + }); + if (!thumbnailId) thumbnailId = media.id; + } + + if (thumbnailId) { + await prisma.project.update({ + where: { id: project.id }, + data: { thumbnailId }, + }); + } + + await indexProject(project.id); + log(lines, onLog, ` created ${visibility} project ${project.slug}`); + return { outcome: "imported", title: detail.title, slug: project.slug }; +} + +async function listUserProjects(username: string): Promise { + const all: AsProjectListItem[] = []; + let page = 1; + for (;;) { + const url = `https://www.artstation.com/users/${username}/projects.json?page=${page}`; + const data = await fetchJson<{ data?: AsProjectListItem[] } | AsProjectListItem[]>( + url + ); + const batch = Array.isArray(data) ? data : data.data || []; + if (batch.length === 0) break; + all.push(...batch); + if (batch.length < 50) break; + page++; + if (page > 20) break; + } + return all; +} + +/** Synchronous full import (CLI / awaited admin call). */ +export async function runArtStationImport( + options: ImportOptions = {} +): Promise { + const username = (options.username || "jmartgraphix").trim() || "jmartgraphix"; + const dryRun = !!options.dryRun; + const lines: string[] = []; + const onLog = options.onLog; + const created: { title: string; slug: string }[] = []; + let imported = 0; + let skipped = 0; + let errors = 0; + let found = 0; + + log(lines, onLog, `ArtStation import (user=${username}, dryRun=${dryRun})`); + + if (options.url) { + found = 1; + try { + const r = await importProject(options.url, options, lines); + if (r.outcome === "imported") { + imported++; + if (r.slug) created.push({ title: r.title || "", slug: r.slug }); + } else if (r.outcome === "skipped") skipped++; + else errors++; + } catch (err) { + errors++; + log(lines, onLog, ` error: ${(err as Error).message}`); + } + } else { + const list = await listUserProjects(username); + found = list.length; + log(lines, onLog, `Found ${list.length} projects`); + for (const item of list) { + const hash = item.hash_id || String(item.id); + try { + const r = await importProject(hash, options, lines); + if (r.outcome === "imported") { + imported++; + if (r.slug) created.push({ title: r.title || item.title, slug: r.slug }); + } else if (r.outcome === "skipped") skipped++; + else errors++; + await new Promise((r) => setTimeout(r, 400)); + } catch (err) { + errors++; + log(lines, onLog, ` error importing ${item.title}: ${(err as Error).message}`); + } + } + } + + if (!dryRun && imported > 0) { + try { + const n = await reindexAllProjects(); + log(lines, onLog, `Typesense reindexed (${n} projects)`); + } catch (err) { + log(lines, onLog, `Typesense reindex warning: ${(err as Error).message}`); + } + } + + log( + lines, + onLog, + `Done. imported=${imported} skipped=${skipped} errors=${errors}` + ); + + return { + username, + dryRun, + found, + imported, + skipped, + errors, + log: lines, + created, + }; +} + +/** Start import in background; only one job at a time. */ +export function startArtStationImportJob(options: ImportOptions = {}): { + started: boolean; + message: string; +} { + if (currentJob.running) { + return { started: false, message: "An import is already running" }; + } + currentJob = { + running: true, + startedAt: new Date().toISOString(), + log: [], + }; + runArtStationImport({ + ...options, + onLog: () => { + /* already pushed to currentJob.log */ + }, + }) + .then((result) => { + currentJob = { + running: false, + startedAt: currentJob.startedAt, + finishedAt: new Date().toISOString(), + result, + log: result.log, + }; + }) + .catch((err) => { + currentJob = { + running: false, + startedAt: currentJob.startedAt, + finishedAt: new Date().toISOString(), + error: (err as Error).message, + log: currentJob.log, + }; + }); + return { started: true, message: "Import started" }; +}