Merge branch 'worktree-agent-a5f21c17' into Develop
# Conflicts: # .planning/REQUIREMENTS.md # .planning/ROADMAP.md # .planning/STATE.md
This commit is contained in:
@@ -9,11 +9,11 @@ Requirements for this milestone. Each maps to roadmap phases.
|
||||
|
||||
### Database Migration
|
||||
|
||||
- [x] **DB-01**: Application runs on PostgreSQL instead of SQLite
|
||||
- [ ] **DB-01**: Application runs on PostgreSQL instead of SQLite
|
||||
- [ ] **DB-02**: All service functions use async database operations
|
||||
- [x] **DB-03**: Test infrastructure uses PGlite instead of bun:sqlite in-memory databases
|
||||
- [ ] **DB-04**: Existing SQLite data can be migrated to Postgres via a one-time script
|
||||
- [x] **DB-05**: Docker Compose provides Postgres for local development
|
||||
- [ ] **DB-03**: Test infrastructure uses PGlite instead of bun:sqlite in-memory databases
|
||||
- [x] **DB-04**: Existing SQLite data can be migrated to Postgres via a one-time script
|
||||
- [ ] **DB-05**: Docker Compose provides Postgres for local development
|
||||
|
||||
### Authentication
|
||||
|
||||
@@ -116,11 +116,11 @@ Which phases cover which requirements. Updated during roadmap creation.
|
||||
|
||||
| Requirement | Phase | Status |
|
||||
|-------------|-------|--------|
|
||||
| DB-01 | Phase 14 | Complete |
|
||||
| DB-01 | Phase 14 | Pending |
|
||||
| DB-02 | Phase 14 | Pending |
|
||||
| DB-03 | Phase 14 | Complete |
|
||||
| DB-04 | Phase 14 | Pending |
|
||||
| DB-05 | Phase 14 | Complete |
|
||||
| DB-03 | Phase 14 | Pending |
|
||||
| DB-04 | Phase 14 | Complete |
|
||||
| DB-05 | Phase 14 | Pending |
|
||||
| AUTH-01 | Phase 15 | Pending |
|
||||
| AUTH-02 | Phase 15 | Pending |
|
||||
| AUTH-03 | Phase 15 | Pending |
|
||||
|
||||
@@ -121,14 +121,7 @@ Plans:
|
||||
2. All service-level and route-level tests pass using PGlite in-memory Postgres (no SQLite test infrastructure remains)
|
||||
3. A one-time migration script converts existing SQLite data into the Postgres database without data loss
|
||||
4. Docker Compose brings up Postgres alongside the app with a single command for local development
|
||||
**Plans:** 2/6 plans executed
|
||||
Plans:
|
||||
- [x] 14-01-PLAN.md — Schema rewrite (pg-core), DB config (postgres.js), test helper (PGlite), Drizzle config, initial migration
|
||||
- [x] 14-02-PLAN.md — Docker Compose dev/prod + Dockerfile update for PostgreSQL
|
||||
- [ ] 14-03-PLAN.md — All 9 services async conversion + seed + server index
|
||||
- [ ] 14-04-PLAN.md — All 9 route handlers async conversion
|
||||
- [ ] 14-05-PLAN.md — SQLite-to-Postgres data migration script
|
||||
- [ ] 14-06-PLAN.md — All 18 test files async conversion + full suite green
|
||||
**Plans**: TBD
|
||||
|
||||
### Phase 15: External Authentication
|
||||
**Goal**: Users can register and log in via a self-hosted OIDC auth provider, replacing the built-in single-user auth system
|
||||
@@ -195,7 +188,7 @@ Plans:
|
||||
| 11. Candidate Ranking | v1.3 | 2/2 | Complete | 2026-03-16 |
|
||||
| 12. Comparison View | v1.3 | 1/1 | Complete | 2026-03-17 |
|
||||
| 13. Setup Impact Preview | v1.3 | 0/2 | Not started | - |
|
||||
| 14. PostgreSQL Migration | v2.0 | 2/6 | In Progress| |
|
||||
| 14. PostgreSQL Migration | v2.0 | 5/6 | In Progress | — |
|
||||
| 15. External Authentication | v2.0 | 0/? | Not started | - |
|
||||
| 16. Multi-User Data Model | v2.0 | 0/? | Not started | - |
|
||||
| 17. Object Storage | v2.0 | 0/? | Not started | - |
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
---
|
||||
gsd_state_version: 1.0
|
||||
milestone: v1.3
|
||||
milestone_name: Research & Decision Tools
|
||||
milestone: v2.0
|
||||
milestone_name: Platform Foundation
|
||||
status: executing
|
||||
stopped_at: Completed 14-02-PLAN.md
|
||||
last_updated: "2026-04-04T10:25:09.510Z"
|
||||
last_activity: 2026-04-04
|
||||
stopped_at: Completed 14-05-PLAN.md
|
||||
last_updated: "2026-04-04T10:28:29Z"
|
||||
last_activity: 2026-04-04 — Completed 14-05 SQLite-to-Postgres migration script
|
||||
progress:
|
||||
total_phases: 8
|
||||
completed_phases: 6
|
||||
total_plans: 18
|
||||
completed_plans: 12
|
||||
total_phases: 5
|
||||
completed_phases: 0
|
||||
total_plans: 0
|
||||
completed_plans: 0
|
||||
percent: 0
|
||||
---
|
||||
|
||||
@@ -21,21 +21,20 @@ progress:
|
||||
See: .planning/PROJECT.md (updated 2026-04-03)
|
||||
|
||||
**Core value:** Help people make better gear decisions — discover what others use, compare real-world data, and see how a potential buy affects your setup before committing.
|
||||
**Current focus:** Phase 14 — postgresql-migration
|
||||
**Current focus:** v2.0 Platform Foundation — Phase 14 (PostgreSQL Migration)
|
||||
|
||||
## Current Position
|
||||
|
||||
Phase: 14 (postgresql-migration) — EXECUTING
|
||||
Plan: 2 of 6
|
||||
Status: Ready to execute
|
||||
Last activity: 2026-04-04
|
||||
Phase: 14 of 18 (PostgreSQL Migration)
|
||||
Plan: 5 of 6 in current phase
|
||||
Status: Executing
|
||||
Last activity: 2026-04-04 — Completed 14-05 SQLite-to-Postgres migration script
|
||||
|
||||
Progress: [----------] 0% (v2.0 milestone)
|
||||
|
||||
## Performance Metrics
|
||||
|
||||
**Velocity:**
|
||||
|
||||
- Total plans completed: 0 (v2.0 milestone)
|
||||
- Average duration: --
|
||||
- Total execution time: --
|
||||
@@ -46,15 +45,15 @@ Progress: [----------] 0% (v2.0 milestone)
|
||||
|
||||
### Decisions
|
||||
|
||||
Key decisions made during v2.0 planning:
|
||||
|
||||
Key decisions made during v2.0 execution:
|
||||
- [14-05] Used postgres.js unsafe() for sequence reset DDL instead of drizzle-orm sql template
|
||||
- [14-05] Row-by-row inserts for better error diagnostics during migration
|
||||
- Platform pivot: single-user to multi-user with discovery-first approach
|
||||
- External auth provider (self-hosted, open-source) — Logto vs Authentik OPEN decision
|
||||
- SQLite to Postgres migration — required by auth provider and multi-user concurrency
|
||||
- Structured UGC only — ratings and predefined fields, no freeform text until moderation
|
||||
- Separate globalItems table — not a flag on user items table
|
||||
- Single-user SQLite mode diverges at v2.0 boundary
|
||||
- [Phase 14-postgresql-migration]: Dev compose uses hardcoded Postgres credentials; production externalizes via env vars
|
||||
|
||||
### Pending Todos
|
||||
|
||||
@@ -67,6 +66,6 @@ None active.
|
||||
|
||||
## Session Continuity
|
||||
|
||||
Last session: 2026-04-04T10:25:09.507Z
|
||||
Stopped at: Completed 14-02-PLAN.md
|
||||
Last session: 2026-04-04T10:28:29Z
|
||||
Stopped at: Completed 14-05-PLAN.md
|
||||
Resume file: None
|
||||
|
||||
93
.planning/phases/14-postgresql-migration/14-05-SUMMARY.md
Normal file
93
.planning/phases/14-postgresql-migration/14-05-SUMMARY.md
Normal file
@@ -0,0 +1,93 @@
|
||||
---
|
||||
phase: 14-postgresql-migration
|
||||
plan: 05
|
||||
subsystem: database
|
||||
tags: [sqlite, postgres, migration, data-migration, bun-sqlite]
|
||||
|
||||
# Dependency graph
|
||||
requires:
|
||||
- phase: 14-01
|
||||
provides: PostgreSQL schema definitions (Drizzle pgTable)
|
||||
provides:
|
||||
- One-time SQLite-to-PostgreSQL data migration script
|
||||
- db:migrate-from-sqlite npm script
|
||||
affects: [14-06, deployment, upgrade-docs]
|
||||
|
||||
# Tech tracking
|
||||
tech-stack:
|
||||
added: []
|
||||
patterns: [dependency-ordered table migration, unix-to-Date conversion, serial sequence reset]
|
||||
|
||||
key-files:
|
||||
created:
|
||||
- scripts/migrate-sqlite-to-postgres.ts
|
||||
modified:
|
||||
- package.json
|
||||
|
||||
key-decisions:
|
||||
- "Used postgres.js unsafe() for sequence reset instead of drizzle-orm sql template (simpler for raw DDL)"
|
||||
- "Row-by-row insert for error tracing (per-row catch identifies failing record)"
|
||||
|
||||
patterns-established:
|
||||
- "Migration scripts live in scripts/ directory"
|
||||
- "Type conversion helpers (unixToDate, intToBool) for SQLite-to-Postgres data transforms"
|
||||
|
||||
requirements-completed: [DB-04]
|
||||
|
||||
# Metrics
|
||||
duration: 2min
|
||||
completed: 2026-04-04
|
||||
---
|
||||
|
||||
# Phase 14 Plan 05: SQLite-to-Postgres Migration Script Summary
|
||||
|
||||
**One-time data migration script converting all 13 tables from SQLite to PostgreSQL with timestamp/boolean type conversions and serial sequence reset**
|
||||
|
||||
## Performance
|
||||
|
||||
- **Duration:** 2 min
|
||||
- **Started:** 2026-04-04T10:26:29Z
|
||||
- **Completed:** 2026-04-04T10:28:29Z
|
||||
- **Tasks:** 1
|
||||
- **Files modified:** 2
|
||||
|
||||
## Accomplishments
|
||||
- Created standalone migration script that reads SQLite and writes to PostgreSQL
|
||||
- Handles all type conversions: unix epoch integers to Date objects, integer booleans to native booleans
|
||||
- Migrates tables in FK dependency order (4 waves: no-FK, FK-to-parents, FK-to-intermediates, junction tables)
|
||||
- Resets all 11 serial sequences after migration to prevent duplicate key errors
|
||||
- Added `db:migrate-from-sqlite` npm script for easy invocation
|
||||
|
||||
## Task Commits
|
||||
|
||||
Each task was committed atomically:
|
||||
|
||||
1. **Task 1: Create SQLite-to-Postgres migration script** - `b4c3813` (feat)
|
||||
|
||||
## Files Created/Modified
|
||||
- `scripts/migrate-sqlite-to-postgres.ts` - One-time migration script with type conversions and sequence reset
|
||||
- `package.json` - Added db:migrate-from-sqlite script
|
||||
|
||||
## Decisions Made
|
||||
- Used `postgres.js` `unsafe()` for raw `setval` queries instead of drizzle-orm `sql` template -- simpler for dynamic table name interpolation in DDL
|
||||
- Row-by-row inserts instead of bulk for better error diagnostics (each failed row logs its ID)
|
||||
|
||||
## Deviations from Plan
|
||||
|
||||
None - plan executed exactly as written.
|
||||
|
||||
## Issues Encountered
|
||||
- Biome lint flagged unused `sql` import from drizzle-orm (used `pg.unsafe()` instead) and unnecessary suppression comments -- removed both
|
||||
|
||||
## User Setup Required
|
||||
|
||||
None - no external service configuration required.
|
||||
|
||||
## Next Phase Readiness
|
||||
- Migration script ready for use during SQLite-to-Postgres upgrade
|
||||
- Requires DATABASE_URL env var and existing SQLite file
|
||||
- Can be tested against a dev Postgres instance with `docker compose up`
|
||||
|
||||
---
|
||||
*Phase: 14-postgresql-migration*
|
||||
*Completed: 2026-04-04*
|
||||
@@ -13,7 +13,8 @@
|
||||
"test": "bun test tests/",
|
||||
"test:e2e": "bunx playwright test",
|
||||
"test:e2e:ui": "bunx playwright test --ui",
|
||||
"lint": "bunx @biomejs/biome check ."
|
||||
"lint": "bunx @biomejs/biome check .",
|
||||
"db:migrate-from-sqlite": "bun run scripts/migrate-sqlite-to-postgres.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^2.4.7",
|
||||
|
||||
284
scripts/migrate-sqlite-to-postgres.ts
Normal file
284
scripts/migrate-sqlite-to-postgres.ts
Normal file
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* One-time SQLite to PostgreSQL data migration script.
|
||||
*
|
||||
* Usage:
|
||||
* DATABASE_URL=postgres://... SQLITE_PATH=gearbox.db bun run scripts/migrate-sqlite-to-postgres.ts
|
||||
*
|
||||
* Environment variables:
|
||||
* DATABASE_URL - PostgreSQL connection string (required)
|
||||
* SQLITE_PATH - Path to SQLite database file (default: "gearbox.db")
|
||||
*/
|
||||
|
||||
import { Database } from "bun:sqlite";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import postgres from "postgres";
|
||||
import * as schema from "../src/db/schema.ts";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Type conversion helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function unixToDate(unix: number | null): Date | null {
|
||||
if (unix === null || unix === undefined) return null;
|
||||
return new Date(unix * 1000); // Unix seconds to JS milliseconds
|
||||
}
|
||||
|
||||
function intToBool(val: number | null): boolean {
|
||||
return val === 1;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generic table migration
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type RowTransform = (row: Record<string, unknown>) => Record<string, unknown>;
|
||||
|
||||
async function migrateTable(
|
||||
sqlite: Database,
|
||||
db: ReturnType<typeof drizzle>,
|
||||
tableName: string,
|
||||
pgTable: any,
|
||||
transform: RowTransform,
|
||||
): Promise<number> {
|
||||
const rows = sqlite.query(`SELECT * FROM ${tableName}`).all() as Record<
|
||||
string,
|
||||
unknown
|
||||
>[];
|
||||
console.log(` ${tableName}: ${rows.length} rows`);
|
||||
|
||||
if (rows.length === 0) return 0;
|
||||
|
||||
for (const row of rows) {
|
||||
try {
|
||||
await db.insert(pgTable).values(transform(row));
|
||||
} catch (err) {
|
||||
const id = row.id ?? row.key ?? "unknown";
|
||||
console.error(` ERROR migrating ${tableName} row (id=${id}):`, err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Sequence reset
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function resetSequences(pg: ReturnType<typeof postgres>) {
|
||||
const tablesWithSerial = [
|
||||
"categories",
|
||||
"items",
|
||||
"threads",
|
||||
"thread_candidates",
|
||||
"setups",
|
||||
"setup_items",
|
||||
"users",
|
||||
"api_keys",
|
||||
"oauth_clients",
|
||||
"oauth_codes",
|
||||
"oauth_tokens",
|
||||
];
|
||||
|
||||
console.log("\nResetting serial sequences...");
|
||||
for (const table of tablesWithSerial) {
|
||||
await pg.unsafe(
|
||||
`SELECT setval(pg_get_serial_sequence('${table}', 'id'), COALESCE((SELECT MAX(id) FROM "${table}"), 0))`,
|
||||
);
|
||||
console.log(` ${table}: sequence reset`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Per-table transform functions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const transforms: Record<string, RowTransform> = {
|
||||
categories: (row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
icon: row.icon,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
}),
|
||||
|
||||
users: (row: any) => ({
|
||||
id: row.id,
|
||||
username: row.username,
|
||||
passwordHash: row.password_hash,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
}),
|
||||
|
||||
settings: (row: any) => ({
|
||||
key: row.key,
|
||||
value: row.value,
|
||||
}),
|
||||
|
||||
items: (row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
weightGrams: row.weight_grams,
|
||||
priceCents: row.price_cents,
|
||||
categoryId: row.category_id,
|
||||
notes: row.notes,
|
||||
productUrl: row.product_url,
|
||||
imageFilename: row.image_filename,
|
||||
imageSourceUrl: row.image_source_url,
|
||||
quantity: row.quantity,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
updatedAt: unixToDate(row.updated_at),
|
||||
}),
|
||||
|
||||
threads: (row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
status: row.status,
|
||||
resolvedCandidateId: row.resolved_candidate_id,
|
||||
categoryId: row.category_id,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
updatedAt: unixToDate(row.updated_at),
|
||||
}),
|
||||
|
||||
sessions: (row: any) => ({
|
||||
id: row.id,
|
||||
userId: row.user_id,
|
||||
expiresAt: unixToDate(row.expires_at),
|
||||
}),
|
||||
|
||||
api_keys: (row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
keyHash: row.key_hash,
|
||||
keyPrefix: row.key_prefix,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
}),
|
||||
|
||||
oauth_clients: (row: any) => ({
|
||||
id: row.id,
|
||||
clientId: row.client_id,
|
||||
clientName: row.client_name,
|
||||
redirectUris: row.redirect_uris,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
}),
|
||||
|
||||
thread_candidates: (row: any) => ({
|
||||
id: row.id,
|
||||
threadId: row.thread_id,
|
||||
name: row.name,
|
||||
weightGrams: row.weight_grams,
|
||||
priceCents: row.price_cents,
|
||||
categoryId: row.category_id,
|
||||
notes: row.notes,
|
||||
productUrl: row.product_url,
|
||||
imageFilename: row.image_filename,
|
||||
imageSourceUrl: row.image_source_url,
|
||||
status: row.status,
|
||||
pros: row.pros,
|
||||
cons: row.cons,
|
||||
sortOrder: row.sort_order,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
updatedAt: unixToDate(row.updated_at),
|
||||
}),
|
||||
|
||||
setups: (row: any) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
createdAt: unixToDate(row.created_at),
|
||||
updatedAt: unixToDate(row.updated_at),
|
||||
}),
|
||||
|
||||
oauth_codes: (row: any) => ({
|
||||
id: row.id,
|
||||
code: row.code,
|
||||
clientId: row.client_id,
|
||||
codeChallenge: row.code_challenge,
|
||||
codeChallengeMethod: row.code_challenge_method,
|
||||
redirectUri: row.redirect_uri,
|
||||
expiresAt: unixToDate(row.expires_at),
|
||||
used: intToBool(row.used),
|
||||
}),
|
||||
|
||||
oauth_tokens: (row: any) => ({
|
||||
id: row.id,
|
||||
accessTokenHash: row.access_token_hash,
|
||||
refreshTokenHash: row.refresh_token_hash,
|
||||
clientId: row.client_id,
|
||||
expiresAt: unixToDate(row.expires_at),
|
||||
refreshExpiresAt: unixToDate(row.refresh_expires_at),
|
||||
createdAt: unixToDate(row.created_at),
|
||||
}),
|
||||
|
||||
setup_items: (row: any) => ({
|
||||
id: row.id,
|
||||
setupId: row.setup_id,
|
||||
itemId: row.item_id,
|
||||
classification: row.classification,
|
||||
}),
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
const sqlitePath = process.env.SQLITE_PATH || "gearbox.db";
|
||||
const databaseUrl = process.env.DATABASE_URL;
|
||||
|
||||
if (!databaseUrl) {
|
||||
console.error("ERROR: DATABASE_URL environment variable is required");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Migrating from SQLite (${sqlitePath}) to PostgreSQL...\n`);
|
||||
|
||||
const sqlite = new Database(sqlitePath, { readonly: true });
|
||||
const pg = postgres(databaseUrl);
|
||||
const db = drizzle(pg, { schema });
|
||||
|
||||
let totalRows = 0;
|
||||
|
||||
// Migration order: parents before children (respecting foreign keys)
|
||||
const migrationPlan: [string, (typeof schema)[keyof typeof schema]][] = [
|
||||
// Wave 1: No FK dependencies on other app tables
|
||||
["categories", schema.categories],
|
||||
["users", schema.users],
|
||||
["settings", schema.settings],
|
||||
// Wave 2: FK to categories / users
|
||||
["items", schema.items],
|
||||
["threads", schema.threads],
|
||||
["sessions", schema.sessions],
|
||||
["api_keys", schema.apiKeys],
|
||||
["oauth_clients", schema.oauthClients],
|
||||
// Wave 3: FK to threads / items / oauth
|
||||
["thread_candidates", schema.threadCandidates],
|
||||
["setups", schema.setups],
|
||||
["oauth_codes", schema.oauthCodes],
|
||||
["oauth_tokens", schema.oauthTokens],
|
||||
// Wave 4: FK to setups + items
|
||||
["setup_items", schema.setupItems],
|
||||
];
|
||||
|
||||
console.log("Migrating tables...");
|
||||
for (const [tableName, pgTable] of migrationPlan) {
|
||||
const transform = transforms[tableName];
|
||||
if (!transform) {
|
||||
console.error(` No transform defined for table: ${tableName}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const count = await migrateTable(sqlite, db, tableName, pgTable, transform);
|
||||
totalRows += count;
|
||||
}
|
||||
|
||||
// Reset serial sequences so new inserts get correct IDs
|
||||
await resetSequences(pg);
|
||||
|
||||
// Cleanup
|
||||
await pg.end();
|
||||
sqlite.close();
|
||||
|
||||
console.log(`\nMigration complete! ${totalRows} total rows migrated.`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("Migration failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user