Add Flickr photostream import as Photography
Import public photos from samuraijkm (NSID resolve + Flickr REST), tag Photography, with admin UI re-run and CLI support.
This commit is contained in:
+2
-1
@@ -12,7 +12,8 @@
|
||||
"db:migrate:dev": "prisma migrate dev",
|
||||
"db:seed": "tsx prisma/seed.ts",
|
||||
"db:push": "prisma db push",
|
||||
"import:artstation": "tsx scripts/import-artstation.ts"
|
||||
"import:artstation": "tsx scripts/import-artstation.ts",
|
||||
"import:flickr": "tsx scripts/import-flickr.ts"
|
||||
},
|
||||
"prisma": {
|
||||
"seed": "tsx prisma/seed.ts"
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Flickr photostream importer CLI.
|
||||
*
|
||||
* npm run import:flickr -- --user samuraijkm
|
||||
* npm run import:flickr -- --user samuraijkm --publish
|
||||
* npm run import:flickr -- --user samuraijkm --dry-run
|
||||
*/
|
||||
import "dotenv/config";
|
||||
import { runFlickrImport } from "../src/services/flickr-import.js";
|
||||
import { prisma } from "../src/lib/prisma.js";
|
||||
|
||||
function arg(name: string, fallback?: string): string | undefined {
|
||||
const idx = process.argv.indexOf(`--${name}`);
|
||||
if (idx >= 0 && process.argv[idx + 1]) return process.argv[idx + 1];
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function hasFlag(name: string): boolean {
|
||||
return process.argv.includes(`--${name}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const result = await runFlickrImport({
|
||||
username: arg("user", "samuraijkm"),
|
||||
dryRun: hasFlag("dry-run"),
|
||||
publish: hasFlag("publish"),
|
||||
});
|
||||
console.log(
|
||||
`Summary: found=${result.found} imported=${result.imported} skipped=${result.skipped} errors=${result.errors}`
|
||||
);
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(() => prisma.$disconnect());
|
||||
@@ -8,6 +8,10 @@ import {
|
||||
getImportJobStatus,
|
||||
startArtStationImportJob,
|
||||
} from "../services/artstation-import.js";
|
||||
import {
|
||||
getFlickrImportJobStatus,
|
||||
startFlickrImportJob,
|
||||
} from "../services/flickr-import.js";
|
||||
|
||||
export async function adminMetaRoutes(app: FastifyInstance) {
|
||||
// Categories
|
||||
@@ -317,4 +321,38 @@ export async function adminMetaRoutes(app: FastifyInstance) {
|
||||
status: getImportJobStatus(),
|
||||
});
|
||||
});
|
||||
|
||||
// Flickr import
|
||||
app.get("/api/v1/admin/import/flickr", async (req, reply) => {
|
||||
if (!requireAdmin(req, reply)) return;
|
||||
return getFlickrImportJobStatus();
|
||||
});
|
||||
|
||||
app.post("/api/v1/admin/import/flickr", async (req, reply) => {
|
||||
if (!requireAdmin(req, reply)) return;
|
||||
const body = z
|
||||
.object({
|
||||
username: z.string().min(1).optional().default("samuraijkm"),
|
||||
dryRun: z.boolean().optional().default(false),
|
||||
publish: z.boolean().optional().default(false),
|
||||
})
|
||||
.parse(req.body ?? {});
|
||||
|
||||
const { started, message } = startFlickrImportJob({
|
||||
username: body.username,
|
||||
dryRun: body.dryRun,
|
||||
publish: body.publish,
|
||||
});
|
||||
|
||||
if (!started) {
|
||||
return reply
|
||||
.code(409)
|
||||
.send({ ok: false, message, status: getFlickrImportJobStatus() });
|
||||
}
|
||||
return reply.code(202).send({
|
||||
ok: true,
|
||||
message,
|
||||
status: getFlickrImportJobStatus(),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
/**
|
||||
* Flickr photostream importer (CLI + admin API).
|
||||
* Resolves NSID + temporary site_key from flickr.com, then pages getPublicPhotos.
|
||||
*/
|
||||
import { Visibility } from "@prisma/client";
|
||||
import slugify from "slugify";
|
||||
import { prisma } from "../lib/prisma.js";
|
||||
import { downloadRemoteImage } from "../lib/media.js";
|
||||
import { indexProject, reindexAllProjects } from "../lib/typesense.js";
|
||||
|
||||
const USER_AGENT =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
||||
|
||||
export interface FlickrImportOptions {
|
||||
/** Path alias e.g. samuraijkm or full photos URL */
|
||||
username?: string;
|
||||
dryRun?: boolean;
|
||||
publish?: boolean;
|
||||
onLog?: (line: string) => void;
|
||||
}
|
||||
|
||||
export interface FlickrImportResult {
|
||||
username: string;
|
||||
nsid: string;
|
||||
dryRun: boolean;
|
||||
found: number;
|
||||
imported: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
log: string[];
|
||||
created: { title: string; slug: string }[];
|
||||
}
|
||||
|
||||
export type FlickrJobStatus = {
|
||||
running: boolean;
|
||||
startedAt?: string;
|
||||
finishedAt?: string;
|
||||
result?: FlickrImportResult;
|
||||
error?: string;
|
||||
log: string[];
|
||||
};
|
||||
|
||||
let currentJob: FlickrJobStatus = { running: false, log: [] };
|
||||
|
||||
export function getFlickrImportJobStatus(): FlickrJobStatus {
|
||||
return { ...currentJob, log: [...currentJob.log] };
|
||||
}
|
||||
|
||||
function log(lines: string[], onLog: FlickrImportOptions["onLog"], line: string) {
|
||||
lines.push(line);
|
||||
currentJob.log.push(line);
|
||||
onLog?.(line);
|
||||
console.log(line);
|
||||
}
|
||||
|
||||
function stripHtml(html: string): string {
|
||||
return html
|
||||
.replace(/<br\s*\/?>/gi, "\n")
|
||||
.replace(/<\/p>/gi, "\n\n")
|
||||
.replace(/<[^>]+>/g, "")
|
||||
.replace(/ /g, " ")
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function parseUsername(input: string): string {
|
||||
const m = input.match(/flickr\.com\/photos\/([^/?#]+)/i);
|
||||
if (m) return m[1];
|
||||
return input.replace(/^@/, "").trim();
|
||||
}
|
||||
|
||||
async function fetchText(url: string): Promise<string> {
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": USER_AGENT,
|
||||
Accept: "text/html,application/json,*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
},
|
||||
signal: AbortSignal.timeout(45_000),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
|
||||
return res.text();
|
||||
}
|
||||
|
||||
async function resolveApiKey(): Promise<string> {
|
||||
const html = await fetchText("https://www.flickr.com/photos/");
|
||||
const m =
|
||||
html.match(/site_key\s*=\s*"([a-f0-9]+)"/i) ||
|
||||
html.match(/"api_key"\s*:\s*"([a-f0-9]+)"/i) ||
|
||||
html.match(/apiKey["']?\s*[:=]\s*["']([a-f0-9]+)/i);
|
||||
if (!m) throw new Error("Could not resolve Flickr site API key from flickr.com");
|
||||
return m[1];
|
||||
}
|
||||
|
||||
async function resolveNsid(username: string, apiKey: string): Promise<string> {
|
||||
if (/^\d+@N\d+$/i.test(username)) return username;
|
||||
|
||||
// Prefer profile page scrape
|
||||
try {
|
||||
const html = await fetchText(`https://www.flickr.com/photos/${encodeURIComponent(username)}/`);
|
||||
const m = html.match(/"nsid"\s*:\s*"(\d+@N\d+)"/i) || html.match(/"id"\s*:\s*"(\d+@N\d+)"/);
|
||||
if (m) return m[1];
|
||||
} catch {
|
||||
/* fall through */
|
||||
}
|
||||
|
||||
const url = new URL("https://api.flickr.com/services/rest/");
|
||||
url.searchParams.set("method", "flickr.people.findByUsername");
|
||||
url.searchParams.set("api_key", apiKey);
|
||||
url.searchParams.set("username", username);
|
||||
url.searchParams.set("format", "json");
|
||||
url.searchParams.set("nojsoncallback", "1");
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": USER_AGENT },
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
const data = (await res.json()) as {
|
||||
stat?: string;
|
||||
message?: string;
|
||||
user?: { id?: string; nsid?: string };
|
||||
};
|
||||
if (data.stat !== "ok" || !(data.user?.nsid || data.user?.id)) {
|
||||
throw new Error(`Could not resolve Flickr user “${username}”: ${data.message || data.stat}`);
|
||||
}
|
||||
return (data.user.nsid || data.user.id)!;
|
||||
}
|
||||
|
||||
interface FlickrPhoto {
|
||||
id: string;
|
||||
title: string;
|
||||
description?: { _content?: string } | string;
|
||||
datetaken?: string;
|
||||
tags?: string;
|
||||
url_o?: string;
|
||||
url_k?: string;
|
||||
url_h?: string;
|
||||
url_l?: string;
|
||||
url_c?: string;
|
||||
url_z?: string;
|
||||
url_m?: string;
|
||||
owner?: string;
|
||||
}
|
||||
|
||||
function bestPhotoUrl(p: FlickrPhoto): string | null {
|
||||
return (
|
||||
p.url_o ||
|
||||
p.url_k ||
|
||||
p.url_h ||
|
||||
p.url_l ||
|
||||
p.url_c ||
|
||||
p.url_z ||
|
||||
p.url_m ||
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
function photoDescription(p: FlickrPhoto): string {
|
||||
const d = p.description;
|
||||
if (!d) return "";
|
||||
if (typeof d === "string") return stripHtml(d);
|
||||
return stripHtml(d._content || "");
|
||||
}
|
||||
|
||||
async function listPublicPhotos(
|
||||
nsid: string,
|
||||
apiKey: string
|
||||
): Promise<FlickrPhoto[]> {
|
||||
const all: FlickrPhoto[] = [];
|
||||
let page = 1;
|
||||
let pages = 1;
|
||||
while (page <= pages && page <= 40) {
|
||||
const url = new URL("https://api.flickr.com/services/rest/");
|
||||
url.searchParams.set("method", "flickr.people.getPublicPhotos");
|
||||
url.searchParams.set("api_key", apiKey);
|
||||
url.searchParams.set("user_id", nsid);
|
||||
url.searchParams.set(
|
||||
"extras",
|
||||
"description,date_taken,url_o,url_k,url_h,url_l,url_c,url_z,url_m,tags,media"
|
||||
);
|
||||
url.searchParams.set("per_page", "50");
|
||||
url.searchParams.set("page", String(page));
|
||||
url.searchParams.set("format", "json");
|
||||
url.searchParams.set("nojsoncallback", "1");
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": USER_AGENT },
|
||||
signal: AbortSignal.timeout(45_000),
|
||||
});
|
||||
const data = (await res.json()) as {
|
||||
stat?: string;
|
||||
message?: string;
|
||||
photos?: { page: number; pages: number; photo: FlickrPhoto[] };
|
||||
};
|
||||
if (data.stat !== "ok" || !data.photos) {
|
||||
throw new Error(`Flickr API error: ${data.message || data.stat}`);
|
||||
}
|
||||
pages = data.photos.pages || 1;
|
||||
all.push(...(data.photos.photo || []));
|
||||
page++;
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
async function uniqueSlug(title: string): Promise<string> {
|
||||
const base = slugify(title, { lower: true, strict: true }) || "photo";
|
||||
let slug = base;
|
||||
let n = 2;
|
||||
while (await prisma.project.findUnique({ where: { slug } })) {
|
||||
slug = `${base}-${n++}`;
|
||||
}
|
||||
return slug;
|
||||
}
|
||||
|
||||
export async function runFlickrImport(
|
||||
options: FlickrImportOptions = {}
|
||||
): Promise<FlickrImportResult> {
|
||||
const username = parseUsername(options.username || "samuraijkm");
|
||||
const dryRun = !!options.dryRun;
|
||||
const lines: string[] = [];
|
||||
const onLog = options.onLog;
|
||||
const created: { title: string; slug: string }[] = [];
|
||||
let imported = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
log(lines, onLog, `Flickr import (user=${username}, dryRun=${dryRun})`);
|
||||
|
||||
const apiKey = await resolveApiKey();
|
||||
log(lines, onLog, "Resolved Flickr API key");
|
||||
const nsid = await resolveNsid(username, apiKey);
|
||||
log(lines, onLog, `Resolved NSID ${nsid}`);
|
||||
|
||||
const photos = await listPublicPhotos(nsid, apiKey);
|
||||
log(lines, onLog, `Found ${photos.length} public photos`);
|
||||
|
||||
const photoCat = await prisma.category.findUnique({ where: { slug: "photography" } });
|
||||
if (!photoCat) throw new Error("Photography category missing — run seed first");
|
||||
|
||||
for (const photo of photos) {
|
||||
const sourceUrl = `https://www.flickr.com/photos/${username}/${photo.id}/`;
|
||||
const title = (photo.title || "").trim() || `Photo ${photo.id}`;
|
||||
try {
|
||||
const existing = await prisma.project.findFirst({ where: { sourceUrl } });
|
||||
if (existing) {
|
||||
// Ensure photography category is present
|
||||
if (!dryRun) {
|
||||
await prisma.projectCategory.upsert({
|
||||
where: {
|
||||
projectId_categoryId: {
|
||||
projectId: existing.id,
|
||||
categoryId: photoCat.id,
|
||||
},
|
||||
},
|
||||
create: { projectId: existing.id, categoryId: photoCat.id },
|
||||
update: {},
|
||||
});
|
||||
}
|
||||
log(lines, onLog, ` skip (exists): ${title}`);
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const imageUrl = bestPhotoUrl(photo);
|
||||
if (!imageUrl) {
|
||||
log(lines, onLog, ` skip (no image URL): ${title}`);
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
log(lines, onLog, ` import: ${title}`);
|
||||
if (dryRun) {
|
||||
imported++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const desc = photoDescription(photo);
|
||||
const tags = (photo.tags || "")
|
||||
.split(/\s+/)
|
||||
.map((t) => t.trim())
|
||||
.filter(Boolean)
|
||||
.slice(0, 20);
|
||||
|
||||
const visibility = options.publish ? Visibility.published : Visibility.draft;
|
||||
const slug = await uniqueSlug(title);
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
title,
|
||||
slug,
|
||||
description: desc,
|
||||
shortDescription: desc.slice(0, 280) || null,
|
||||
date: photo.datetaken ? new Date(photo.datetaken) : null,
|
||||
software: ["Photography"],
|
||||
externalLinks: [{ label: "Flickr", url: sourceUrl }],
|
||||
featured: false,
|
||||
displayPriority: 100,
|
||||
visibility,
|
||||
sourceUrl,
|
||||
sourcePlatform: "flickr",
|
||||
publishedAt: visibility === Visibility.published ? new Date() : null,
|
||||
},
|
||||
});
|
||||
|
||||
await prisma.projectCategory.create({
|
||||
data: { projectId: project.id, categoryId: photoCat.id },
|
||||
});
|
||||
|
||||
for (const t of tags) {
|
||||
const tslug = slugify(t, { lower: true, strict: true });
|
||||
if (!tslug) continue;
|
||||
const tag = await prisma.tag.upsert({
|
||||
where: { slug: tslug },
|
||||
create: { name: t, slug: tslug },
|
||||
update: {},
|
||||
});
|
||||
await prisma.projectTag.upsert({
|
||||
where: { projectId_tagId: { projectId: project.id, tagId: tag.id } },
|
||||
create: { projectId: project.id, tagId: tag.id },
|
||||
update: {},
|
||||
});
|
||||
}
|
||||
|
||||
const saved = await downloadRemoteImage(imageUrl);
|
||||
if (saved) {
|
||||
const media = await prisma.media.create({
|
||||
data: {
|
||||
projectId: project.id,
|
||||
type: "image",
|
||||
url: saved.url,
|
||||
thumbnailUrl: saved.thumbnailUrl,
|
||||
filename: saved.filename,
|
||||
mimeType: saved.mimeType,
|
||||
width: saved.width,
|
||||
height: saved.height,
|
||||
sizeBytes: saved.sizeBytes,
|
||||
alt: title,
|
||||
sortOrder: 0,
|
||||
},
|
||||
});
|
||||
await prisma.project.update({
|
||||
where: { id: project.id },
|
||||
data: { thumbnailId: media.id },
|
||||
});
|
||||
} else {
|
||||
log(lines, onLog, ` warning: image download failed for ${imageUrl}`);
|
||||
}
|
||||
|
||||
await indexProject(project.id);
|
||||
imported++;
|
||||
created.push({ title, slug: project.slug });
|
||||
log(lines, onLog, ` created ${visibility} project ${project.slug}`);
|
||||
await new Promise((r) => setTimeout(r, 250));
|
||||
} catch (err) {
|
||||
errors++;
|
||||
log(lines, onLog, ` error importing ${title}: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!dryRun && imported > 0) {
|
||||
try {
|
||||
const n = await reindexAllProjects();
|
||||
log(lines, onLog, `Typesense reindexed (${n} projects)`);
|
||||
} catch (err) {
|
||||
log(lines, onLog, `Typesense reindex warning: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
log(
|
||||
lines,
|
||||
onLog,
|
||||
`Done. imported=${imported} skipped=${skipped} errors=${errors}`
|
||||
);
|
||||
|
||||
return {
|
||||
username,
|
||||
nsid,
|
||||
dryRun,
|
||||
found: photos.length,
|
||||
imported,
|
||||
skipped,
|
||||
errors,
|
||||
log: lines,
|
||||
created,
|
||||
};
|
||||
}
|
||||
|
||||
export function startFlickrImportJob(options: FlickrImportOptions = {}): {
|
||||
started: boolean;
|
||||
message: string;
|
||||
} {
|
||||
if (currentJob.running) {
|
||||
return { started: false, message: "A Flickr import is already running" };
|
||||
}
|
||||
currentJob = {
|
||||
running: true,
|
||||
startedAt: new Date().toISOString(),
|
||||
log: [],
|
||||
};
|
||||
runFlickrImport(options)
|
||||
.then((result) => {
|
||||
currentJob = {
|
||||
running: false,
|
||||
startedAt: currentJob.startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
result,
|
||||
log: result.log,
|
||||
};
|
||||
})
|
||||
.catch((err) => {
|
||||
currentJob = {
|
||||
running: false,
|
||||
startedAt: currentJob.startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
error: (err as Error).message,
|
||||
log: currentJob.log,
|
||||
};
|
||||
});
|
||||
return { started: true, message: "Flickr import started" };
|
||||
}
|
||||
Reference in New Issue
Block a user