From b4c38134e1e12d91579f217187ce0b8804858eb5 Mon Sep 17 00:00:00 2001 From: Jean-Luc Makiola Date: Sat, 4 Apr 2026 12:28:19 +0200 Subject: [PATCH 1/2] feat(14-05): create SQLite-to-Postgres data migration script - One-time migration script with type conversions (unix timestamps to Date, int to bool) - Migrates all 13 tables in FK dependency order - Resets serial sequences after data migration - Adds db:migrate-from-sqlite npm script --- package.json | 3 +- scripts/migrate-sqlite-to-postgres.ts | 284 ++++++++++++++++++++++++++ 2 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 scripts/migrate-sqlite-to-postgres.ts diff --git a/package.json b/package.json index 281ba01..41f478d 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "test": "bun test tests/", "test:e2e": "bunx playwright test", "test:e2e:ui": "bunx playwright test --ui", - "lint": "bunx @biomejs/biome check ." + "lint": "bunx @biomejs/biome check .", + "db:migrate-from-sqlite": "bun run scripts/migrate-sqlite-to-postgres.ts" }, "devDependencies": { "@biomejs/biome": "^2.4.7", diff --git a/scripts/migrate-sqlite-to-postgres.ts b/scripts/migrate-sqlite-to-postgres.ts new file mode 100644 index 0000000..6c86bd4 --- /dev/null +++ b/scripts/migrate-sqlite-to-postgres.ts @@ -0,0 +1,284 @@ +/** + * One-time SQLite to PostgreSQL data migration script. + * + * Usage: + * DATABASE_URL=postgres://... SQLITE_PATH=gearbox.db bun run scripts/migrate-sqlite-to-postgres.ts + * + * Environment variables: + * DATABASE_URL - PostgreSQL connection string (required) + * SQLITE_PATH - Path to SQLite database file (default: "gearbox.db") + */ + +import { Database } from "bun:sqlite"; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import * as schema from "../src/db/schema.ts"; + +// --------------------------------------------------------------------------- +// Type conversion helpers +// --------------------------------------------------------------------------- + +function unixToDate(unix: number | null): Date | null { + if (unix === null || unix === undefined) return null; + return new Date(unix * 1000); // Unix seconds to JS milliseconds +} + +function intToBool(val: number | null): boolean { + return val === 1; +} + +// --------------------------------------------------------------------------- +// Generic table migration +// --------------------------------------------------------------------------- + +type RowTransform = (row: Record) => Record; + +async function migrateTable( + sqlite: Database, + db: ReturnType, + tableName: string, + pgTable: any, + transform: RowTransform, +): Promise { + const rows = sqlite.query(`SELECT * FROM ${tableName}`).all() as Record< + string, + unknown + >[]; + console.log(` ${tableName}: ${rows.length} rows`); + + if (rows.length === 0) return 0; + + for (const row of rows) { + try { + await db.insert(pgTable).values(transform(row)); + } catch (err) { + const id = row.id ?? row.key ?? "unknown"; + console.error(` ERROR migrating ${tableName} row (id=${id}):`, err); + throw err; + } + } + + return rows.length; +} + +// --------------------------------------------------------------------------- +// Sequence reset +// --------------------------------------------------------------------------- + +async function resetSequences(pg: ReturnType) { + const tablesWithSerial = [ + "categories", + "items", + "threads", + "thread_candidates", + "setups", + "setup_items", + "users", + "api_keys", + "oauth_clients", + "oauth_codes", + "oauth_tokens", + ]; + + console.log("\nResetting serial sequences..."); + for (const table of tablesWithSerial) { + await pg.unsafe( + `SELECT setval(pg_get_serial_sequence('${table}', 'id'), COALESCE((SELECT MAX(id) FROM "${table}"), 0))`, + ); + console.log(` ${table}: sequence reset`); + } +} + +// --------------------------------------------------------------------------- +// Per-table transform functions +// --------------------------------------------------------------------------- + +const transforms: Record = { + categories: (row: any) => ({ + id: row.id, + name: row.name, + icon: row.icon, + createdAt: unixToDate(row.created_at), + }), + + users: (row: any) => ({ + id: row.id, + username: row.username, + passwordHash: row.password_hash, + createdAt: unixToDate(row.created_at), + }), + + settings: (row: any) => ({ + key: row.key, + value: row.value, + }), + + items: (row: any) => ({ + id: row.id, + name: row.name, + weightGrams: row.weight_grams, + priceCents: row.price_cents, + categoryId: row.category_id, + notes: row.notes, + productUrl: row.product_url, + imageFilename: row.image_filename, + imageSourceUrl: row.image_source_url, + quantity: row.quantity, + createdAt: unixToDate(row.created_at), + updatedAt: unixToDate(row.updated_at), + }), + + threads: (row: any) => ({ + id: row.id, + name: row.name, + status: row.status, + resolvedCandidateId: row.resolved_candidate_id, + categoryId: row.category_id, + createdAt: unixToDate(row.created_at), + updatedAt: unixToDate(row.updated_at), + }), + + sessions: (row: any) => ({ + id: row.id, + userId: row.user_id, + expiresAt: unixToDate(row.expires_at), + }), + + api_keys: (row: any) => ({ + id: row.id, + name: row.name, + keyHash: row.key_hash, + keyPrefix: row.key_prefix, + createdAt: unixToDate(row.created_at), + }), + + oauth_clients: (row: any) => ({ + id: row.id, + clientId: row.client_id, + clientName: row.client_name, + redirectUris: row.redirect_uris, + createdAt: unixToDate(row.created_at), + }), + + thread_candidates: (row: any) => ({ + id: row.id, + threadId: row.thread_id, + name: row.name, + weightGrams: row.weight_grams, + priceCents: row.price_cents, + categoryId: row.category_id, + notes: row.notes, + productUrl: row.product_url, + imageFilename: row.image_filename, + imageSourceUrl: row.image_source_url, + status: row.status, + pros: row.pros, + cons: row.cons, + sortOrder: row.sort_order, + createdAt: unixToDate(row.created_at), + updatedAt: unixToDate(row.updated_at), + }), + + setups: (row: any) => ({ + id: row.id, + name: row.name, + createdAt: unixToDate(row.created_at), + updatedAt: unixToDate(row.updated_at), + }), + + oauth_codes: (row: any) => ({ + id: row.id, + code: row.code, + clientId: row.client_id, + codeChallenge: row.code_challenge, + codeChallengeMethod: row.code_challenge_method, + redirectUri: row.redirect_uri, + expiresAt: unixToDate(row.expires_at), + used: intToBool(row.used), + }), + + oauth_tokens: (row: any) => ({ + id: row.id, + accessTokenHash: row.access_token_hash, + refreshTokenHash: row.refresh_token_hash, + clientId: row.client_id, + expiresAt: unixToDate(row.expires_at), + refreshExpiresAt: unixToDate(row.refresh_expires_at), + createdAt: unixToDate(row.created_at), + }), + + setup_items: (row: any) => ({ + id: row.id, + setupId: row.setup_id, + itemId: row.item_id, + classification: row.classification, + }), +}; + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + const sqlitePath = process.env.SQLITE_PATH || "gearbox.db"; + const databaseUrl = process.env.DATABASE_URL; + + if (!databaseUrl) { + console.error("ERROR: DATABASE_URL environment variable is required"); + process.exit(1); + } + + console.log(`Migrating from SQLite (${sqlitePath}) to PostgreSQL...\n`); + + const sqlite = new Database(sqlitePath, { readonly: true }); + const pg = postgres(databaseUrl); + const db = drizzle(pg, { schema }); + + let totalRows = 0; + + // Migration order: parents before children (respecting foreign keys) + const migrationPlan: [string, (typeof schema)[keyof typeof schema]][] = [ + // Wave 1: No FK dependencies on other app tables + ["categories", schema.categories], + ["users", schema.users], + ["settings", schema.settings], + // Wave 2: FK to categories / users + ["items", schema.items], + ["threads", schema.threads], + ["sessions", schema.sessions], + ["api_keys", schema.apiKeys], + ["oauth_clients", schema.oauthClients], + // Wave 3: FK to threads / items / oauth + ["thread_candidates", schema.threadCandidates], + ["setups", schema.setups], + ["oauth_codes", schema.oauthCodes], + ["oauth_tokens", schema.oauthTokens], + // Wave 4: FK to setups + items + ["setup_items", schema.setupItems], + ]; + + console.log("Migrating tables..."); + for (const [tableName, pgTable] of migrationPlan) { + const transform = transforms[tableName]; + if (!transform) { + console.error(` No transform defined for table: ${tableName}`); + process.exit(1); + } + const count = await migrateTable(sqlite, db, tableName, pgTable, transform); + totalRows += count; + } + + // Reset serial sequences so new inserts get correct IDs + await resetSequences(pg); + + // Cleanup + await pg.end(); + sqlite.close(); + + console.log(`\nMigration complete! ${totalRows} total rows migrated.`); +} + +main().catch((err) => { + console.error("Migration failed:", err); + process.exit(1); +}); From 85104f3687804c320ee3de25406b0fb0961bf0f1 Mon Sep 17 00:00:00 2001 From: Jean-Luc Makiola Date: Sat, 4 Apr 2026 12:30:31 +0200 Subject: [PATCH 2/2] docs(14-05): complete SQLite-to-Postgres migration script plan - SUMMARY.md with execution results - STATE.md updated with plan 05 completion - ROADMAP.md updated with phase 14 progress - DB-04 requirement marked complete --- .planning/REQUIREMENTS.md | 4 +- .planning/ROADMAP.md | 2 +- .planning/STATE.md | 22 +++-- .../14-postgresql-migration/14-05-SUMMARY.md | 93 +++++++++++++++++++ 4 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 .planning/phases/14-postgresql-migration/14-05-SUMMARY.md diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md index 121fbd8..bf50d1e 100644 --- a/.planning/REQUIREMENTS.md +++ b/.planning/REQUIREMENTS.md @@ -12,7 +12,7 @@ Requirements for this milestone. Each maps to roadmap phases. - [ ] **DB-01**: Application runs on PostgreSQL instead of SQLite - [ ] **DB-02**: All service functions use async database operations - [ ] **DB-03**: Test infrastructure uses PGlite instead of bun:sqlite in-memory databases -- [ ] **DB-04**: Existing SQLite data can be migrated to Postgres via a one-time script +- [x] **DB-04**: Existing SQLite data can be migrated to Postgres via a one-time script - [ ] **DB-05**: Docker Compose provides Postgres for local development ### Authentication @@ -119,7 +119,7 @@ Which phases cover which requirements. Updated during roadmap creation. | DB-01 | Phase 14 | Pending | | DB-02 | Phase 14 | Pending | | DB-03 | Phase 14 | Pending | -| DB-04 | Phase 14 | Pending | +| DB-04 | Phase 14 | Complete | | DB-05 | Phase 14 | Pending | | AUTH-01 | Phase 15 | Pending | | AUTH-02 | Phase 15 | Pending | diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 2be42f5..d49b174 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -188,7 +188,7 @@ Plans: | 11. Candidate Ranking | v1.3 | 2/2 | Complete | 2026-03-16 | | 12. Comparison View | v1.3 | 1/1 | Complete | 2026-03-17 | | 13. Setup Impact Preview | v1.3 | 0/2 | Not started | - | -| 14. PostgreSQL Migration | v2.0 | 0/? | Not started | - | +| 14. PostgreSQL Migration | v2.0 | 5/6 | In Progress | — | | 15. External Authentication | v2.0 | 0/? | Not started | - | | 16. Multi-User Data Model | v2.0 | 0/? | Not started | - | | 17. Object Storage | v2.0 | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index d9a43f1..526e838 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -2,10 +2,10 @@ gsd_state_version: 1.0 milestone: v2.0 milestone_name: Platform Foundation -status: planning -stopped_at: null -last_updated: "2026-04-03" -last_activity: 2026-04-03 — v2.0 roadmap created (Phases 14-18) +status: executing +stopped_at: Completed 14-05-PLAN.md +last_updated: "2026-04-04T10:28:29Z" +last_activity: 2026-04-04 — Completed 14-05 SQLite-to-Postgres migration script progress: total_phases: 5 completed_phases: 0 @@ -26,9 +26,9 @@ See: .planning/PROJECT.md (updated 2026-04-03) ## Current Position Phase: 14 of 18 (PostgreSQL Migration) -Plan: 0 of ? in current phase -Status: Ready to plan -Last activity: 2026-04-03 — v2.0 roadmap created (Phases 14-18) +Plan: 5 of 6 in current phase +Status: Executing +Last activity: 2026-04-04 — Completed 14-05 SQLite-to-Postgres migration script Progress: [----------] 0% (v2.0 milestone) @@ -45,7 +45,9 @@ Progress: [----------] 0% (v2.0 milestone) ### Decisions -Key decisions made during v2.0 planning: +Key decisions made during v2.0 execution: +- [14-05] Used postgres.js unsafe() for sequence reset DDL instead of drizzle-orm sql template +- [14-05] Row-by-row inserts for better error diagnostics during migration - Platform pivot: single-user to multi-user with discovery-first approach - External auth provider (self-hosted, open-source) — Logto vs Authentik OPEN decision - SQLite to Postgres migration — required by auth provider and multi-user concurrency @@ -64,6 +66,6 @@ None active. ## Session Continuity -Last session: 2026-04-03 -Stopped at: v2.0 roadmap created with 5 phases (14-18) covering 30 requirements +Last session: 2026-04-04T10:28:29Z +Stopped at: Completed 14-05-PLAN.md Resume file: None diff --git a/.planning/phases/14-postgresql-migration/14-05-SUMMARY.md b/.planning/phases/14-postgresql-migration/14-05-SUMMARY.md new file mode 100644 index 0000000..8432915 --- /dev/null +++ b/.planning/phases/14-postgresql-migration/14-05-SUMMARY.md @@ -0,0 +1,93 @@ +--- +phase: 14-postgresql-migration +plan: 05 +subsystem: database +tags: [sqlite, postgres, migration, data-migration, bun-sqlite] + +# Dependency graph +requires: + - phase: 14-01 + provides: PostgreSQL schema definitions (Drizzle pgTable) +provides: + - One-time SQLite-to-PostgreSQL data migration script + - db:migrate-from-sqlite npm script +affects: [14-06, deployment, upgrade-docs] + +# Tech tracking +tech-stack: + added: [] + patterns: [dependency-ordered table migration, unix-to-Date conversion, serial sequence reset] + +key-files: + created: + - scripts/migrate-sqlite-to-postgres.ts + modified: + - package.json + +key-decisions: + - "Used postgres.js unsafe() for sequence reset instead of drizzle-orm sql template (simpler for raw DDL)" + - "Row-by-row insert for error tracing (per-row catch identifies failing record)" + +patterns-established: + - "Migration scripts live in scripts/ directory" + - "Type conversion helpers (unixToDate, intToBool) for SQLite-to-Postgres data transforms" + +requirements-completed: [DB-04] + +# Metrics +duration: 2min +completed: 2026-04-04 +--- + +# Phase 14 Plan 05: SQLite-to-Postgres Migration Script Summary + +**One-time data migration script converting all 13 tables from SQLite to PostgreSQL with timestamp/boolean type conversions and serial sequence reset** + +## Performance + +- **Duration:** 2 min +- **Started:** 2026-04-04T10:26:29Z +- **Completed:** 2026-04-04T10:28:29Z +- **Tasks:** 1 +- **Files modified:** 2 + +## Accomplishments +- Created standalone migration script that reads SQLite and writes to PostgreSQL +- Handles all type conversions: unix epoch integers to Date objects, integer booleans to native booleans +- Migrates tables in FK dependency order (4 waves: no-FK, FK-to-parents, FK-to-intermediates, junction tables) +- Resets all 11 serial sequences after migration to prevent duplicate key errors +- Added `db:migrate-from-sqlite` npm script for easy invocation + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Create SQLite-to-Postgres migration script** - `b4c3813` (feat) + +## Files Created/Modified +- `scripts/migrate-sqlite-to-postgres.ts` - One-time migration script with type conversions and sequence reset +- `package.json` - Added db:migrate-from-sqlite script + +## Decisions Made +- Used `postgres.js` `unsafe()` for raw `setval` queries instead of drizzle-orm `sql` template -- simpler for dynamic table name interpolation in DDL +- Row-by-row inserts instead of bulk for better error diagnostics (each failed row logs its ID) + +## Deviations from Plan + +None - plan executed exactly as written. + +## Issues Encountered +- Biome lint flagged unused `sql` import from drizzle-orm (used `pg.unsafe()` instead) and unnecessary suppression comments -- removed both + +## User Setup Required + +None - no external service configuration required. + +## Next Phase Readiness +- Migration script ready for use during SQLite-to-Postgres upgrade +- Requires DATABASE_URL env var and existing SQLite file +- Can be tested against a dev Postgres instance with `docker compose up` + +--- +*Phase: 14-postgresql-migration* +*Completed: 2026-04-04*