feat(14-05): create SQLite-to-Postgres data migration script
- One-time migration script with type conversions (unix timestamps to Date, int to bool) - Migrates all 13 tables in FK dependency order - Resets serial sequences after data migration - Adds db:migrate-from-sqlite npm script
This commit is contained in:
@@ -13,7 +13,8 @@
|
||||
"test": "bun test tests/",
|
||||
"test:e2e": "bunx playwright test",
|
||||
"test:e2e:ui": "bunx playwright test --ui",
|
||||
"lint": "bunx @biomejs/biome check ."
|
||||
"lint": "bunx @biomejs/biome check .",
|
||||
"db:migrate-from-sqlite": "bun run scripts/migrate-sqlite-to-postgres.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^2.4.7",
|
||||
|
||||
284
scripts/migrate-sqlite-to-postgres.ts
Normal file
284
scripts/migrate-sqlite-to-postgres.ts
Normal file
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* One-time SQLite to PostgreSQL data migration script.
|
||||
*
|
||||
* Usage:
|
||||
* DATABASE_URL=postgres://... SQLITE_PATH=gearbox.db bun run scripts/migrate-sqlite-to-postgres.ts
|
||||
*
|
||||
* Environment variables:
|
||||
* DATABASE_URL - PostgreSQL connection string (required)
|
||||
* SQLITE_PATH - Path to SQLite database file (default: "gearbox.db")
|
||||
*/
|
||||
|
||||
import { Database } from "bun:sqlite";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import postgres from "postgres";
|
||||
import * as schema from "../src/db/schema.ts";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Type conversion helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function unixToDate(unix: number | null): Date | null {
|
||||
if (unix === null || unix === undefined) return null;
|
||||
return new Date(unix * 1000); // Unix seconds to JS milliseconds
|
||||
}
|
||||
|
||||
function intToBool(val: number | null): boolean {
|
||||
return val === 1;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generic table migration
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type RowTransform = (row: Record<string, unknown>) => Record<string, unknown>;
|
||||
|
||||
async function migrateTable(
|
||||
sqlite: Database,
|
||||
db: ReturnType<typeof drizzle>,
|
||||
tableName: string,
|
||||
pgTable: any,
|
||||
transform: RowTransform,
|
||||
): Promise<number> {
|
||||
const rows = sqlite.query(`SELECT * FROM ${tableName}`).all() as Record<
|
||||
string,
|
||||
unknown
|
||||
>[];
|
||||
console.log(` ${tableName}: ${rows.length} rows`);
|
||||
|
||||
if (rows.length === 0) return 0;
|
||||
|
||||
for (const row of rows) {
|
||||
try {
|
||||
await db.insert(pgTable).values(transform(row));
|
||||
} catch (err) {
|
||||
const id = row.id ?? row.key ?? "unknown";
|
||||
console.error(` ERROR migrating ${tableName} row (id=${id}):`, err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Sequence reset
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function resetSequences(pg: ReturnType<typeof postgres>) {
|
||||
const tablesWithSerial = [
|
||||
"categories",
|
||||
"items",
|
||||
"threads",
|
||||
"thread_candidates",
|
||||
"setups",
|
||||
"setup_items",
|
||||
"users",
|
||||
"api_keys",
|
||||
"oauth_clients",
|
||||
"oauth_codes",
|
||||
"oauth_tokens",
|
||||
];
|
||||
|
||||
console.log("\nResetting serial sequences...");
|
||||
for (const table of tablesWithSerial) {
|
||||
await pg.unsafe(
|
||||
`SELECT setval(pg_get_serial_sequence('${table}', 'id'), COALESCE((SELECT MAX(id) FROM "${table}"), 0))`,
|
||||
);
|
||||
console.log(` ${table}: sequence reset`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Per-table transform functions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const transforms: Record<string, RowTransform> = {
|
||||
categories: (row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
icon: row.icon,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
}),
|
||||
|
||||
users: (row: any) => ({
|
||||
id: row.id,
|
||||
username: row.username,
|
||||
passwordHash: row.password_hash,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
}),
|
||||
|
||||
settings: (row: any) => ({
|
||||
key: row.key,
|
||||
value: row.value,
|
||||
}),
|
||||
|
||||
items: (row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
weightGrams: row.weight_grams,
|
||||
priceCents: row.price_cents,
|
||||
categoryId: row.category_id,
|
||||
notes: row.notes,
|
||||
productUrl: row.product_url,
|
||||
imageFilename: row.image_filename,
|
||||
imageSourceUrl: row.image_source_url,
|
||||
quantity: row.quantity,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
updatedAt: unixToDate(row.updated_at),
|
||||
}),
|
||||
|
||||
threads: (row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
status: row.status,
|
||||
resolvedCandidateId: row.resolved_candidate_id,
|
||||
categoryId: row.category_id,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
updatedAt: unixToDate(row.updated_at),
|
||||
}),
|
||||
|
||||
sessions: (row: any) => ({
|
||||
id: row.id,
|
||||
userId: row.user_id,
|
||||
expiresAt: unixToDate(row.expires_at),
|
||||
}),
|
||||
|
||||
api_keys: (row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
keyHash: row.key_hash,
|
||||
keyPrefix: row.key_prefix,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
}),
|
||||
|
||||
oauth_clients: (row: any) => ({
|
||||
id: row.id,
|
||||
clientId: row.client_id,
|
||||
clientName: row.client_name,
|
||||
redirectUris: row.redirect_uris,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
}),
|
||||
|
||||
thread_candidates: (row: any) => ({
|
||||
id: row.id,
|
||||
threadId: row.thread_id,
|
||||
name: row.name,
|
||||
weightGrams: row.weight_grams,
|
||||
priceCents: row.price_cents,
|
||||
categoryId: row.category_id,
|
||||
notes: row.notes,
|
||||
productUrl: row.product_url,
|
||||
imageFilename: row.image_filename,
|
||||
imageSourceUrl: row.image_source_url,
|
||||
status: row.status,
|
||||
pros: row.pros,
|
||||
cons: row.cons,
|
||||
sortOrder: row.sort_order,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
updatedAt: unixToDate(row.updated_at),
|
||||
}),
|
||||
|
||||
setups: (row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
updatedAt: unixToDate(row.updated_at),
|
||||
}),
|
||||
|
||||
oauth_codes: (row: any) => ({
|
||||
id: row.id,
|
||||
code: row.code,
|
||||
clientId: row.client_id,
|
||||
codeChallenge: row.code_challenge,
|
||||
codeChallengeMethod: row.code_challenge_method,
|
||||
redirectUri: row.redirect_uri,
|
||||
expiresAt: unixToDate(row.expires_at),
|
||||
used: intToBool(row.used),
|
||||
}),
|
||||
|
||||
oauth_tokens: (row: any) => ({
|
||||
id: row.id,
|
||||
accessTokenHash: row.access_token_hash,
|
||||
refreshTokenHash: row.refresh_token_hash,
|
||||
clientId: row.client_id,
|
||||
expiresAt: unixToDate(row.expires_at),
|
||||
refreshExpiresAt: unixToDate(row.refresh_expires_at),
|
||||
createdAt: unixToDate(row.created_at),
|
||||
}),
|
||||
|
||||
setup_items: (row: any) => ({
|
||||
id: row.id,
|
||||
setupId: row.setup_id,
|
||||
itemId: row.item_id,
|
||||
classification: row.classification,
|
||||
}),
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
const sqlitePath = process.env.SQLITE_PATH || "gearbox.db";
|
||||
const databaseUrl = process.env.DATABASE_URL;
|
||||
|
||||
if (!databaseUrl) {
|
||||
console.error("ERROR: DATABASE_URL environment variable is required");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Migrating from SQLite (${sqlitePath}) to PostgreSQL...\n`);
|
||||
|
||||
const sqlite = new Database(sqlitePath, { readonly: true });
|
||||
const pg = postgres(databaseUrl);
|
||||
const db = drizzle(pg, { schema });
|
||||
|
||||
let totalRows = 0;
|
||||
|
||||
// Migration order: parents before children (respecting foreign keys)
|
||||
const migrationPlan: [string, (typeof schema)[keyof typeof schema]][] = [
|
||||
// Wave 1: No FK dependencies on other app tables
|
||||
["categories", schema.categories],
|
||||
["users", schema.users],
|
||||
["settings", schema.settings],
|
||||
// Wave 2: FK to categories / users
|
||||
["items", schema.items],
|
||||
["threads", schema.threads],
|
||||
["sessions", schema.sessions],
|
||||
["api_keys", schema.apiKeys],
|
||||
["oauth_clients", schema.oauthClients],
|
||||
// Wave 3: FK to threads / items / oauth
|
||||
["thread_candidates", schema.threadCandidates],
|
||||
["setups", schema.setups],
|
||||
["oauth_codes", schema.oauthCodes],
|
||||
["oauth_tokens", schema.oauthTokens],
|
||||
// Wave 4: FK to setups + items
|
||||
["setup_items", schema.setupItems],
|
||||
];
|
||||
|
||||
console.log("Migrating tables...");
|
||||
for (const [tableName, pgTable] of migrationPlan) {
|
||||
const transform = transforms[tableName];
|
||||
if (!transform) {
|
||||
console.error(` No transform defined for table: ${tableName}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const count = await migrateTable(sqlite, db, tableName, pgTable, transform);
|
||||
totalRows += count;
|
||||
}
|
||||
|
||||
// Reset serial sequences so new inserts get correct IDs
|
||||
await resetSequences(pg);
|
||||
|
||||
// Cleanup
|
||||
await pg.end();
|
||||
sqlite.close();
|
||||
|
||||
console.log(`\nMigration complete! ${totalRows} total rows migrated.`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("Migration failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user