From ff37533037eb12e15839b0efd636c786697ecd11 Mon Sep 17 00:00:00 2001 From: jmartin Date: Fri, 24 Jul 2026 11:35:57 -0400 Subject: [PATCH] Add Flickr photostream import as Photography Import public photos from samuraijkm (NSID resolve + Flickr REST), tag Photography, with admin UI re-run and CLI support. --- client/src/lib/api.ts | 13 + client/src/pages/admin/AdminDashboard.tsx | 141 ++++++++ package.json | 3 +- server/package.json | 3 +- server/scripts/import-flickr.ts | 38 ++ server/src/routes/admin-meta.ts | 38 ++ server/src/services/flickr-import.ts | 421 ++++++++++++++++++++++ 7 files changed, 655 insertions(+), 2 deletions(-) create mode 100644 server/scripts/import-flickr.ts create mode 100644 server/src/services/flickr-import.ts diff --git a/client/src/lib/api.ts b/client/src/lib/api.ts index 9e7c849..42fbb63 100644 --- a/client/src/lib/api.ts +++ b/client/src/lib/api.ts @@ -251,6 +251,19 @@ export const api = { importArtStationStatus: () => request("/api/v1/admin/import/artstation"), + + importFlickr: (body: { username?: string; dryRun?: boolean; publish?: boolean }) => + request<{ + ok: boolean; + message: string; + status: ArtStationImportStatus; + }>("/api/v1/admin/import/flickr", { + method: "POST", + body: JSON.stringify(body), + }), + + importFlickrStatus: () => + request("/api/v1/admin/import/flickr"), }; export interface ArtStationImportStatus { diff --git a/client/src/pages/admin/AdminDashboard.tsx b/client/src/pages/admin/AdminDashboard.tsx index 0b01848..cb4c592 100644 --- a/client/src/pages/admin/AdminDashboard.tsx +++ b/client/src/pages/admin/AdminDashboard.tsx @@ -19,6 +19,12 @@ export function AdminDashboard() { const [importStatus, setImportStatus] = useState(null); const pollRef = useRef | null>(null); + const [flUser, setFlUser] = useState("samuraijkm"); + const [flDryRun, setFlDryRun] = useState(false); + const [flPublish, setFlPublish] = useState(false); + const [flStatus, setFlStatus] = useState(null); + const flPollRef = useRef | null>(null); + function refreshStats() { Promise.all([ api.adminProjects({ perPage: 1 }), @@ -40,8 +46,10 @@ export function AdminDashboard() { useEffect(() => { refreshStats(); api.importArtStationStatus().then(setImportStatus).catch(() => {}); + api.importFlickrStatus().then(setFlStatus).catch(() => {}); return () => { if (pollRef.current) clearInterval(pollRef.current); + if (flPollRef.current) clearInterval(flPollRef.current); }; }, []); @@ -69,6 +77,30 @@ export function AdminDashboard() { }, 2000); } + function startFlickrPolling() { + if (flPollRef.current) clearInterval(flPollRef.current); + flPollRef.current = setInterval(async () => { + try { + const s = await api.importFlickrStatus(); + setFlStatus(s); + if (!s.running) { + if (flPollRef.current) clearInterval(flPollRef.current); + flPollRef.current = null; + refreshStats(); + if (s.result) { + setMsg( + `Flickr import finished: ${s.result.imported} new, ${s.result.skipped} skipped, ${s.result.errors} errors (${s.result.found} on photostream)` + ); + } else if (s.error) { + setErr(s.error); + } + } + } catch { + /* keep polling */ + } + }, 2000); + } + async function reindex() { setErr(null); try { @@ -96,8 +128,27 @@ export function AdminDashboard() { } } + async function runFlickrImport() { + setErr(null); + setMsg(null); + try { + const res = await api.importFlickr({ + username: flUser.trim() || "samuraijkm", + dryRun: flDryRun, + publish: flPublish, + }); + setFlStatus(res.status); + setMsg(res.message); + if (res.status.running) startFlickrPolling(); + } catch (e) { + setErr((e as Error).message); + } + } + const running = importStatus?.running; const logLines = importStatus?.log?.slice(-40) || []; + const flRunning = flStatus?.running; + const flLogLines = flStatus?.log?.slice(-40) || []; return (
@@ -232,6 +283,96 @@ export function AdminDashboard() { )} +
+

Import from Flickr

+

+ Pull public photos from your Flickr photostream and tag them as{" "} + Photography. Existing entries (matched by Flickr URL) are skipped. +

+ +
+ + setFlUser(e.target.value)} + disabled={!!flRunning} + placeholder="samuraijkm" + /> +
+ +
+ + +
+ +
+ + {flStatus?.finishedAt && !flRunning && ( + + Last run {new Date(flStatus.finishedAt).toLocaleString()} + + )} +
+ + {flStatus?.result && !flRunning && ( +
+ Found {flStatus.result.found} · imported {flStatus.result.imported} · skipped{" "} + {flStatus.result.skipped} · errors {flStatus.result.errors} + {flStatus.result.created.length > 0 && ( + <> + {"\n"}New: {flStatus.result.created.map((c) => c.title).join(", ")} + + )} +
+ )} + + {flLogLines.length > 0 && ( + 
+            {flLogLines.join("\n")}
+
+ )} +
+

Quick links for audiences

    diff --git a/package.json b/package.json index 868ee92..f37bb2e 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,8 @@ "db:generate": "npm run db:generate -w server", "db:migrate": "npm run db:migrate -w server", "db:seed": "npm run db:seed -w server", - "import:artstation": "npm run import:artstation -w server" + "import:artstation": "npm run import:artstation -w server", + "import:flickr": "npm run import:flickr -w server" }, "devDependencies": { "concurrently": "^9.1.2" diff --git a/server/package.json b/server/package.json index 3eee661..bd59412 100644 --- a/server/package.json +++ b/server/package.json @@ -12,7 +12,8 @@ "db:migrate:dev": "prisma migrate dev", "db:seed": "tsx prisma/seed.ts", "db:push": "prisma db push", - "import:artstation": "tsx scripts/import-artstation.ts" + "import:artstation": "tsx scripts/import-artstation.ts", + "import:flickr": "tsx scripts/import-flickr.ts" }, "prisma": { "seed": "tsx prisma/seed.ts" diff --git a/server/scripts/import-flickr.ts b/server/scripts/import-flickr.ts new file mode 100644 index 0000000..8153f27 --- /dev/null +++ b/server/scripts/import-flickr.ts @@ -0,0 +1,38 @@ +/** + * Flickr photostream importer CLI. + * + * npm run import:flickr -- --user samuraijkm + * npm run import:flickr -- --user samuraijkm --publish + * npm run import:flickr -- --user samuraijkm --dry-run + */ +import "dotenv/config"; +import { runFlickrImport } from "../src/services/flickr-import.js"; +import { prisma } from "../src/lib/prisma.js"; + +function arg(name: string, fallback?: string): string | undefined { + const idx = process.argv.indexOf(`--${name}`); + if (idx >= 0 && process.argv[idx + 1]) return process.argv[idx + 1]; + return fallback; +} + +function hasFlag(name: string): boolean { + return process.argv.includes(`--${name}`); +} + +async function main() { + const result = await runFlickrImport({ + username: arg("user", "samuraijkm"), + dryRun: hasFlag("dry-run"), + publish: hasFlag("publish"), + }); + console.log( + `Summary: found=${result.found} imported=${result.imported} skipped=${result.skipped} errors=${result.errors}` + ); +} + +main() + .catch((e) => { + console.error(e); + process.exit(1); + }) + .finally(() => prisma.$disconnect()); diff --git a/server/src/routes/admin-meta.ts b/server/src/routes/admin-meta.ts index 1011154..097a5fd 100644 --- a/server/src/routes/admin-meta.ts +++ b/server/src/routes/admin-meta.ts @@ -8,6 +8,10 @@ import { getImportJobStatus, startArtStationImportJob, } from "../services/artstation-import.js"; +import { + getFlickrImportJobStatus, + startFlickrImportJob, +} from "../services/flickr-import.js"; export async function adminMetaRoutes(app: FastifyInstance) { // Categories @@ -317,4 +321,38 @@ export async function adminMetaRoutes(app: FastifyInstance) { status: getImportJobStatus(), }); }); + + // Flickr import + app.get("/api/v1/admin/import/flickr", async (req, reply) => { + if (!requireAdmin(req, reply)) return; + return getFlickrImportJobStatus(); + }); + + app.post("/api/v1/admin/import/flickr", async (req, reply) => { + if (!requireAdmin(req, reply)) return; + const body = z + .object({ + username: z.string().min(1).optional().default("samuraijkm"), + dryRun: z.boolean().optional().default(false), + publish: z.boolean().optional().default(false), + }) + .parse(req.body ?? {}); + + const { started, message } = startFlickrImportJob({ + username: body.username, + dryRun: body.dryRun, + publish: body.publish, + }); + + if (!started) { + return reply + .code(409) + .send({ ok: false, message, status: getFlickrImportJobStatus() }); + } + return reply.code(202).send({ + ok: true, + message, + status: getFlickrImportJobStatus(), + }); + }); } diff --git a/server/src/services/flickr-import.ts b/server/src/services/flickr-import.ts new file mode 100644 index 0000000..a57901d --- /dev/null +++ b/server/src/services/flickr-import.ts @@ -0,0 +1,421 @@ +/** + * Flickr photostream importer (CLI + admin API). + * Resolves NSID + temporary site_key from flickr.com, then pages getPublicPhotos. + */ +import { Visibility } from "@prisma/client"; +import slugify from "slugify"; +import { prisma } from "../lib/prisma.js"; +import { downloadRemoteImage } from "../lib/media.js"; +import { indexProject, reindexAllProjects } from "../lib/typesense.js"; + +const USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; + +export interface FlickrImportOptions { + /** Path alias e.g. samuraijkm or full photos URL */ + username?: string; + dryRun?: boolean; + publish?: boolean; + onLog?: (line: string) => void; +} + +export interface FlickrImportResult { + username: string; + nsid: string; + dryRun: boolean; + found: number; + imported: number; + skipped: number; + errors: number; + log: string[]; + created: { title: string; slug: string }[]; +} + +export type FlickrJobStatus = { + running: boolean; + startedAt?: string; + finishedAt?: string; + result?: FlickrImportResult; + error?: string; + log: string[]; +}; + +let currentJob: FlickrJobStatus = { running: false, log: [] }; + +export function getFlickrImportJobStatus(): FlickrJobStatus { + return { ...currentJob, log: [...currentJob.log] }; +} + +function log(lines: string[], onLog: FlickrImportOptions["onLog"], line: string) { + lines.push(line); + currentJob.log.push(line); + onLog?.(line); + console.log(line); +} + +function stripHtml(html: string): string { + return html + .replace(//gi, "\n") + .replace(/<\/p>/gi, "\n\n") + .replace(/<[^>]+>/g, "") + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +function parseUsername(input: string): string { + const m = input.match(/flickr\.com\/photos\/([^/?#]+)/i); + if (m) return m[1]; + return input.replace(/^@/, "").trim(); +} + +async function fetchText(url: string): Promise { + const res = await fetch(url, { + headers: { + "User-Agent": USER_AGENT, + Accept: "text/html,application/json,*/*", + "Accept-Language": "en-US,en;q=0.9", + }, + signal: AbortSignal.timeout(45_000), + }); + if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`); + return res.text(); +} + +async function resolveApiKey(): Promise { + const html = await fetchText("https://www.flickr.com/photos/"); + const m = + html.match(/site_key\s*=\s*"([a-f0-9]+)"/i) || + html.match(/"api_key"\s*:\s*"([a-f0-9]+)"/i) || + html.match(/apiKey["']?\s*[:=]\s*["']([a-f0-9]+)/i); + if (!m) throw new Error("Could not resolve Flickr site API key from flickr.com"); + return m[1]; +} + +async function resolveNsid(username: string, apiKey: string): Promise { + if (/^\d+@N\d+$/i.test(username)) return username; + + // Prefer profile page scrape + try { + const html = await fetchText(`https://www.flickr.com/photos/${encodeURIComponent(username)}/`); + const m = html.match(/"nsid"\s*:\s*"(\d+@N\d+)"/i) || html.match(/"id"\s*:\s*"(\d+@N\d+)"/); + if (m) return m[1]; + } catch { + /* fall through */ + } + + const url = new URL("https://api.flickr.com/services/rest/"); + url.searchParams.set("method", "flickr.people.findByUsername"); + url.searchParams.set("api_key", apiKey); + url.searchParams.set("username", username); + url.searchParams.set("format", "json"); + url.searchParams.set("nojsoncallback", "1"); + const res = await fetch(url, { + headers: { "User-Agent": USER_AGENT }, + signal: AbortSignal.timeout(30_000), + }); + const data = (await res.json()) as { + stat?: string; + message?: string; + user?: { id?: string; nsid?: string }; + }; + if (data.stat !== "ok" || !(data.user?.nsid || data.user?.id)) { + throw new Error(`Could not resolve Flickr user “${username}”: ${data.message || data.stat}`); + } + return (data.user.nsid || data.user.id)!; +} + +interface FlickrPhoto { + id: string; + title: string; + description?: { _content?: string } | string; + datetaken?: string; + tags?: string; + url_o?: string; + url_k?: string; + url_h?: string; + url_l?: string; + url_c?: string; + url_z?: string; + url_m?: string; + owner?: string; +} + +function bestPhotoUrl(p: FlickrPhoto): string | null { + return ( + p.url_o || + p.url_k || + p.url_h || + p.url_l || + p.url_c || + p.url_z || + p.url_m || + null + ); +} + +function photoDescription(p: FlickrPhoto): string { + const d = p.description; + if (!d) return ""; + if (typeof d === "string") return stripHtml(d); + return stripHtml(d._content || ""); +} + +async function listPublicPhotos( + nsid: string, + apiKey: string +): Promise { + const all: FlickrPhoto[] = []; + let page = 1; + let pages = 1; + while (page <= pages && page <= 40) { + const url = new URL("https://api.flickr.com/services/rest/"); + url.searchParams.set("method", "flickr.people.getPublicPhotos"); + url.searchParams.set("api_key", apiKey); + url.searchParams.set("user_id", nsid); + url.searchParams.set( + "extras", + "description,date_taken,url_o,url_k,url_h,url_l,url_c,url_z,url_m,tags,media" + ); + url.searchParams.set("per_page", "50"); + url.searchParams.set("page", String(page)); + url.searchParams.set("format", "json"); + url.searchParams.set("nojsoncallback", "1"); + const res = await fetch(url, { + headers: { "User-Agent": USER_AGENT }, + signal: AbortSignal.timeout(45_000), + }); + const data = (await res.json()) as { + stat?: string; + message?: string; + photos?: { page: number; pages: number; photo: FlickrPhoto[] }; + }; + if (data.stat !== "ok" || !data.photos) { + throw new Error(`Flickr API error: ${data.message || data.stat}`); + } + pages = data.photos.pages || 1; + all.push(...(data.photos.photo || [])); + page++; + } + return all; +} + +async function uniqueSlug(title: string): Promise { + const base = slugify(title, { lower: true, strict: true }) || "photo"; + let slug = base; + let n = 2; + while (await prisma.project.findUnique({ where: { slug } })) { + slug = `${base}-${n++}`; + } + return slug; +} + +export async function runFlickrImport( + options: FlickrImportOptions = {} +): Promise { + const username = parseUsername(options.username || "samuraijkm"); + const dryRun = !!options.dryRun; + const lines: string[] = []; + const onLog = options.onLog; + const created: { title: string; slug: string }[] = []; + let imported = 0; + let skipped = 0; + let errors = 0; + + log(lines, onLog, `Flickr import (user=${username}, dryRun=${dryRun})`); + + const apiKey = await resolveApiKey(); + log(lines, onLog, "Resolved Flickr API key"); + const nsid = await resolveNsid(username, apiKey); + log(lines, onLog, `Resolved NSID ${nsid}`); + + const photos = await listPublicPhotos(nsid, apiKey); + log(lines, onLog, `Found ${photos.length} public photos`); + + const photoCat = await prisma.category.findUnique({ where: { slug: "photography" } }); + if (!photoCat) throw new Error("Photography category missing — run seed first"); + + for (const photo of photos) { + const sourceUrl = `https://www.flickr.com/photos/${username}/${photo.id}/`; + const title = (photo.title || "").trim() || `Photo ${photo.id}`; + try { + const existing = await prisma.project.findFirst({ where: { sourceUrl } }); + if (existing) { + // Ensure photography category is present + if (!dryRun) { + await prisma.projectCategory.upsert({ + where: { + projectId_categoryId: { + projectId: existing.id, + categoryId: photoCat.id, + }, + }, + create: { projectId: existing.id, categoryId: photoCat.id }, + update: {}, + }); + } + log(lines, onLog, ` skip (exists): ${title}`); + skipped++; + continue; + } + + const imageUrl = bestPhotoUrl(photo); + if (!imageUrl) { + log(lines, onLog, ` skip (no image URL): ${title}`); + skipped++; + continue; + } + + log(lines, onLog, ` import: ${title}`); + if (dryRun) { + imported++; + continue; + } + + const desc = photoDescription(photo); + const tags = (photo.tags || "") + .split(/\s+/) + .map((t) => t.trim()) + .filter(Boolean) + .slice(0, 20); + + const visibility = options.publish ? Visibility.published : Visibility.draft; + const slug = await uniqueSlug(title); + const project = await prisma.project.create({ + data: { + title, + slug, + description: desc, + shortDescription: desc.slice(0, 280) || null, + date: photo.datetaken ? new Date(photo.datetaken) : null, + software: ["Photography"], + externalLinks: [{ label: "Flickr", url: sourceUrl }], + featured: false, + displayPriority: 100, + visibility, + sourceUrl, + sourcePlatform: "flickr", + publishedAt: visibility === Visibility.published ? new Date() : null, + }, + }); + + await prisma.projectCategory.create({ + data: { projectId: project.id, categoryId: photoCat.id }, + }); + + for (const t of tags) { + const tslug = slugify(t, { lower: true, strict: true }); + if (!tslug) continue; + const tag = await prisma.tag.upsert({ + where: { slug: tslug }, + create: { name: t, slug: tslug }, + update: {}, + }); + await prisma.projectTag.upsert({ + where: { projectId_tagId: { projectId: project.id, tagId: tag.id } }, + create: { projectId: project.id, tagId: tag.id }, + update: {}, + }); + } + + const saved = await downloadRemoteImage(imageUrl); + if (saved) { + const media = await prisma.media.create({ + data: { + projectId: project.id, + type: "image", + url: saved.url, + thumbnailUrl: saved.thumbnailUrl, + filename: saved.filename, + mimeType: saved.mimeType, + width: saved.width, + height: saved.height, + sizeBytes: saved.sizeBytes, + alt: title, + sortOrder: 0, + }, + }); + await prisma.project.update({ + where: { id: project.id }, + data: { thumbnailId: media.id }, + }); + } else { + log(lines, onLog, ` warning: image download failed for ${imageUrl}`); + } + + await indexProject(project.id); + imported++; + created.push({ title, slug: project.slug }); + log(lines, onLog, ` created ${visibility} project ${project.slug}`); + await new Promise((r) => setTimeout(r, 250)); + } catch (err) { + errors++; + log(lines, onLog, ` error importing ${title}: ${(err as Error).message}`); + } + } + + if (!dryRun && imported > 0) { + try { + const n = await reindexAllProjects(); + log(lines, onLog, `Typesense reindexed (${n} projects)`); + } catch (err) { + log(lines, onLog, `Typesense reindex warning: ${(err as Error).message}`); + } + } + + log( + lines, + onLog, + `Done. imported=${imported} skipped=${skipped} errors=${errors}` + ); + + return { + username, + nsid, + dryRun, + found: photos.length, + imported, + skipped, + errors, + log: lines, + created, + }; +} + +export function startFlickrImportJob(options: FlickrImportOptions = {}): { + started: boolean; + message: string; +} { + if (currentJob.running) { + return { started: false, message: "A Flickr import is already running" }; + } + currentJob = { + running: true, + startedAt: new Date().toISOString(), + log: [], + }; + runFlickrImport(options) + .then((result) => { + currentJob = { + running: false, + startedAt: currentJob.startedAt, + finishedAt: new Date().toISOString(), + result, + log: result.log, + }; + }) + .catch((err) => { + currentJob = { + running: false, + startedAt: currentJob.startedAt, + finishedAt: new Date().toISOString(), + error: (err as Error).message, + log: currentJob.log, + }; + }); + return { started: true, message: "Flickr import started" }; +}