#!/usr/bin/env bun /** * Crawl all active manufacturers of a given tier. * * Usage: * bun run scripts/crawl-all.ts --tier=1 * bun run scripts/crawl-all.ts --tier=1 --dry-run * * Env vars required: * GEARBOX_URL — Base URL of the GearBox instance (default: http://localhost:3000) * GEARBOX_API_KEY — GearBox API key with write access * ANTHROPIC_API_KEY — Anthropic API key (passed through to crawl-manufacturer) */ const GEARBOX_URL = process.env.GEARBOX_URL ?? "http://localhost:3000"; const GEARBOX_API_KEY = process.env.GEARBOX_API_KEY ?? ""; const args = Object.fromEntries( process.argv .slice(2) .filter((a) => a.startsWith("--")) .map((a) => { const [k, v] = a.slice(2).split("="); return [k, v ?? "true"]; }), ); const tier = args.tier ? Number(args.tier) : 1; const dryRun = args["dry-run"] === "true"; async function listActiveManufacturers(targetTier: number) { const res = await fetch(`${GEARBOX_URL}/api/manufacturers`); if (!res.ok) throw new Error(`Failed to list manufacturers: HTTP ${res.status}`); const all = (await res.json()) as Array<{ slug: string; tier: number; active: boolean; name: string; }>; return all.filter((m) => m.active && m.tier === targetTier); } async function main() { if (!GEARBOX_API_KEY) { console.error("GEARBOX_API_KEY env var is required"); process.exit(1); } const manufacturers = await listActiveManufacturers(tier); console.log( `Found ${manufacturers.length} active tier-${tier} manufacturers\n`, ); const results: Array<{ slug: string; status: "ok" | "error"; error?: string; }> = []; for (const m of manufacturers) { console.log(`\n${"─".repeat(50)}`); console.log(`Crawling: ${m.name} (${m.slug})`); try { const extraArgs = dryRun ? ["--dry-run"] : []; const proc = Bun.spawn( [ "bun", "run", "scripts/crawl-manufacturer.ts", `--manufacturer=${m.slug}`, ...extraArgs, ], { stdout: "inherit", stderr: "inherit", env: process.env }, ); const exitCode = await proc.exited; if (exitCode !== 0) throw new Error(`Exited with code ${exitCode}`); results.push({ slug: m.slug, status: "ok" }); } catch (err) { console.error(` ERROR: ${(err as Error).message}`); results.push({ slug: m.slug, status: "error", error: (err as Error).message, }); } } console.log(`\n${"=".repeat(50)}`); console.log("Summary:"); for (const r of results) { const icon = r.status === "ok" ? "✓" : "✗"; console.log(` ${icon} ${r.slug}${r.error ? ` — ${r.error}` : ""}`); } const failed = results.filter((r) => r.status === "error"); if (failed.length > 0) { console.error(`\n${failed.length} manufacturer(s) failed`); process.exit(1); } } main().catch((err) => { console.error(err); process.exit(1); });