From 35bd2e182802d93cee94d5c3c1f1738ea249fa27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E7=85=9C?= <1871263099@qq.com> Date: Mon, 5 Oct 2026 19:44:18 +0800 Subject: [PATCH] fix: recover empty database migrations and persist Docker icons --- .env.production.example | 2 + Dockerfile | 2 + README.md | 4 +- apps/api/.env.example | 2 + apps/api/package.json | 2 +- apps/api/scripts/database.cjs | 178 +++++++++++++++++++--- apps/api/scripts/test-isolated.cjs | 51 ++----- apps/api/src/backup.ts | 7 +- apps/api/src/icon-files.ts | 63 ++++++++ apps/api/src/icons.ts | 23 ++- apps/api/src/main.ts | 4 +- apps/api/test/database-migrations.test.ts | 99 ++++++++++++ apps/api/test/database-plan.test.ts | 39 +++++ apps/api/test/icon-files.test.ts | 57 +++++++ apps/api/test/icons.test.ts | 20 +++ compose.yaml | 8 + docker.md | 45 +++++- docs/admin-accounts.md | 2 +- docs/architecture.md | 2 +- docs/database-migrations.md | 29 ++++ 20 files changed, 577 insertions(+), 62 deletions(-) create mode 100644 apps/api/src/icon-files.ts create mode 100644 apps/api/test/database-migrations.test.ts create mode 100644 apps/api/test/database-plan.test.ts create mode 100644 apps/api/test/icon-files.test.ts create mode 100644 docs/database-migrations.md diff --git a/.env.production.example b/.env.production.example index 8f14492..040d75d 100644 --- a/.env.production.example +++ b/.env.production.example @@ -7,6 +7,8 @@ ADMIN_PASSWORD=REPLACE_WITH_A_STRONG_INITIAL_PASSWORD API_HOST=0.0.0.0 PORT=3100 +# Container path; Compose binds /opt/worthpath/data/icons on the host here. +ICON_STORAGE_DIR=/app/data/icons API_ALLOWED_HOSTS=worthpath.example.com WEB_ORIGIN=https://worthpath.example.com diff --git a/Dockerfile b/Dockerfile index 10508ba..4591fbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,8 @@ RUN pnpm db:generate && pnpm build \ FROM base AS runtime ENV NODE_ENV=production +ENV ICON_STORAGE_DIR=/app/data/icons +RUN mkdir -p /app/data/icons && chown -R node:node /app/data # Preserve pnpm's workspace links and Prisma CLI for explicit release migrations. COPY --from=build --chown=node:node /app/node_modules ./node_modules COPY --from=build --chown=node:node /app/package.json /app/pnpm-workspace.yaml ./ diff --git a/README.md b/README.md index e3ac6c6..774eec1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Docker 前后端合并容器部署见 [Docker 部署说明](docker.md)。 +Docker 上传图标持久化到宿主机 `/opt/worthpath/data/icons`(容器 `/app/data/icons`);更新已部署项目需同步 Compose 挂载配置并重建镜像,具体步骤见部署说明。数据库继续保留图标内容,用于现有 ZIP 备份及文件缺失修复。 + 个人资产负债与净资产管理。NestJS + React/Vite + TypeScript + MySQL,支持手机和电脑。Node.js 22.12+、pnpm 11、MySQL 8+。 ```powershell @@ -22,7 +24,7 @@ cd E:\WorthPath .\scripts\pnpm.ps1 dev ``` -已有本地环境配置时直接启动,避免重新复制模板。创建数据库前可以在 `apps/api` 中执行 `node scripts/db-preflight.cjs`;脚本先检查同名数据库,仅在不存在时创建空数据库,不删除已有数据。迁移只使用 `migrate deploy`。Windows 上重新生成 Prisma 客户端前需停止 API,以释放其 DLL。 +已有本地环境配置时直接启动,避免重新复制模板。创建数据库前可以在 `apps/api` 中执行 `node scripts/db-preflight.cjs`;脚本先检查同名数据库,仅在不存在时创建空数据库,不删除已有数据。迁移使用 `pnpm db:migrate`,由项目入口按依赖顺序执行 Prisma `migrate deploy`;空库的历史 `005_account_icons` 失败恢复见 [数据库迁移](docs/database-migrations.md)。Windows 上重新生成 Prisma 客户端前需停止 API,以释放其 DLL。 ```powershell pnpm typecheck diff --git a/apps/api/.env.example b/apps/api/.env.example index 9782c86..cac6fca 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -89,3 +89,5 @@ NETWORK_RATE_LIMIT_WINDOW_MS=900000 NETWORK_AUTH_RATE_LIMIT_MAX=30 # 每个来源在窗口内访问单个 OAuth 授权、令牌、撤销或注册端点的次数。 MCP_AUTH_RATE_LIMIT_MAX=100 +# Optional filesystem icon persistence; Docker configures /app/data/icons. +ICON_STORAGE_DIR= diff --git a/apps/api/package.json b/apps/api/package.json index e1d30bf..e6e8b8a 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -5,7 +5,7 @@ "dev": "node scripts/dev.cjs", "build": "tsc", "typecheck": "tsc --noEmit", - "test": "tsx --test --test-concurrency=1 test/calculation.test.ts test/update.test.ts test/zip.test.ts test/metals.test.ts ../web/test/i18n.test.ts test/account-deletion.test.ts test/mcp-hosts.test.ts test/mcp-http.test.ts test/network.test.ts test/web.test.ts ../web/test/quick-entry.test.ts ../web/test/net-worth-chart.test.ts ../web/test/login-preferences.test.ts", + "test": "tsx --test --test-concurrency=1 test/calculation.test.ts test/update.test.ts test/zip.test.ts test/metals.test.ts ../web/test/i18n.test.ts test/account-deletion.test.ts test/mcp-hosts.test.ts test/mcp-http.test.ts test/network.test.ts test/web.test.ts test/database-plan.test.ts test/icon-files.test.ts ../web/test/quick-entry.test.ts ../web/test/net-worth-chart.test.ts ../web/test/login-preferences.test.ts", "db:generate": "prisma generate", "db:migrate": "node scripts/database.cjs deploy", "db:status": "node scripts/database.cjs status", diff --git a/apps/api/scripts/database.cjs b/apps/api/scripts/database.cjs index 614b8f7..62e768c 100644 --- a/apps/api/scripts/database.cjs +++ b/apps/api/scripts/database.cjs @@ -1,19 +1,163 @@ +require('dotenv').config({ quiet: true }); const { spawnSync } = require('node:child_process'); -const command = process.argv[2]; -if (!['deploy', 'status'].includes(command)) process.exit(1); -const p = spawnSync( - process.execPath, - [require.resolve('prisma/build/index.js'), 'migrate', command], - { encoding: 'utf8' }, -); -// Prisma datasource lines can expose local connection metadata; omit them. -const output = (p.stdout || '') - .split(/\r?\n/) - .filter((s) => !s.startsWith('Datasource ') && !s.includes('Environment variables')) - .join('\n'); -if (p.status === 0) console.log(output); -else - console.error( - 'Database migration command failed. Check local database access and migration compatibility. No reset was performed.', +const { mkdtempSync, copyFileSync, mkdirSync, rmSync } = require('node:fs'); +const { join, resolve, dirname, basename } = require('node:path'); +const { tmpdir } = require('node:os'); +const { createHash } = require('node:crypto'); +const mysql = require('mysql2/promise'); + +// Preserve historical names and SQL. Deploy their prerequisites first on a new database. +const initialMigrations = [ + '202610010001_initial', + '202610010002_import_origin', + '202610010003_revision_sequence', + '202610010004_privacy_time_settings', +]; +const schema = resolve(__dirname, '../prisma/schema.prisma'); +const migrations = resolve(__dirname, '../prisma/migrations'); +function bootstrapPlan(tables, history) { + const applied = new Set( + history.filter((r) => r.finished_at && !r.rolled_back_at).map((r) => r.migration_name), ); -process.exitCode = p.status || 0; + const failed = history.filter((r) => !r.finished_at && !r.rolled_back_at); + const businessTables = tables.filter((name) => name.toLowerCase() !== '_prisma_migrations'); + const recover = + failed.length === 1 && + failed[0].migration_name === '005_account_icons' && + Number(failed[0].applied_steps_count) === 0 && + /1824/.test(failed[0].logs || '') && + /Failed to open the referenced table/i.test(failed[0].logs || '') && + businessTables.length === 0 && + applied.size === 0; + if (failed.length && !recover) + throw new Error( + 'Unresolved migration requires manual recovery; no database reset was performed.', + ); + if (initialMigrations.every((name) => applied.has(name))) + return { bootstrap: false, recover: false }; + const prefix = initialMigrations.slice(0, applied.size); + if ( + [...applied].some((name) => !prefix.includes(name)) || + (businessTables.length && !applied.size) + ) + throw new Error('Unrecognized partial schema/history; no bootstrap or reset was performed.'); + return { bootstrap: true, recover }; +} +function runPrisma(args) { + const p = spawnSync(process.execPath, [require.resolve('prisma/build/index.js'), ...args], { + encoding: 'utf8', + }); + const output = (p.stdout || '') + .split(/\r?\n/) + .filter((s) => !s.startsWith('Datasource ') && !s.includes('Environment variables')) + .join('\n'); + if (p.status === 0) { + if (output.trim()) console.log(output); + } else { + const codes = [...new Set(((p.stdout || '') + (p.stderr || '')).match(/P\d{4}/g) || [])]; + console.error( + 'Prisma command failed' + + (codes.length ? ': ' + codes.join(', ') : '') + + '. No database reset was performed.', + ); + } + return p.status === 0 ? 0 : p.status || 1; +} +async function deploy() { + const url = new URL(process.env.DATABASE_URL); + const database = decodeURIComponent(url.pathname.slice(1)); + if (url.protocol !== 'mysql:' || !database) throw new Error('Invalid MySQL configuration.'); + const connection = await mysql.createConnection({ + host: url.hostname, + port: Number(url.port || 3306), + user: decodeURIComponent(url.username), + password: decodeURIComponent(url.password), + charset: 'utf8mb4', + connectTimeout: 10000, + }); + const lock = + 'worthpath:migrate:' + + createHash('sha256').update(database.toLowerCase()).digest('hex').slice(0, 32); + let locked = false, + folder; + try { + const [lockRows] = await connection.execute('SELECT GET_LOCK(?, 30) AS acquired', [lock]); + if (Number(lockRows[0].acquired) !== 1) throw new Error('Migration lock is busy; retry later.'); + locked = true; + const [rows] = await connection.execute( + 'SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA=?', + [database], + ); + const tables = rows.map((r) => r.TABLE_NAME); + const quotedDatabase = '`' + database.replace(/`/g, '``') + '`'; + const [history] = tables.some((name) => name.toLowerCase() === '_prisma_migrations') + ? await connection.query( + 'SELECT migration_name, finished_at, rolled_back_at, applied_steps_count, logs FROM ' + + quotedDatabase + + '.`_prisma_migrations`', + ) + : [[]]; + const plan = bootstrapPlan(tables, history); + if (plan.recover) { + console.log('Verified empty database: recovering the failed 005_account_icons attempt.'); + const status = runPrisma([ + 'migrate', + 'resolve', + '--rolled-back', + '005_account_icons', + '--schema', + schema, + ]); + if (status) return status; + } + if (plan.bootstrap) { + folder = mkdtempSync(join(tmpdir(), 'worthpath-migrations-')); + copyFileSync(schema, join(folder, 'schema.prisma')); + mkdirSync(join(folder, 'migrations')); + copyFileSync( + join(migrations, 'migration_lock.toml'), + join(folder, 'migrations/migration_lock.toml'), + ); + for (const name of initialMigrations) { + mkdirSync(join(folder, 'migrations', name)); + copyFileSync( + join(migrations, name, 'migration.sql'), + join(folder, 'migrations', name, 'migration.sql'), + ); + } + console.log('Deploying initial prerequisites before historical 005-007 migrations.'); + const status = runPrisma(['migrate', 'deploy', '--schema', join(folder, 'schema.prisma')]); + if (status) return status; + } + return runPrisma(['migrate', 'deploy', '--schema', schema]); + } finally { + if ( + folder && + dirname(resolve(folder)) === resolve(tmpdir()) && + basename(folder).startsWith('worthpath-migrations-') + ) + rmSync(folder, { recursive: true, force: true }); + if (locked) await connection.execute('SELECT RELEASE_LOCK(?)', [lock]); + await connection.end(); + } +} +module.exports = { bootstrapPlan, initialMigrations }; +if (require.main === module) { + const command = process.argv[2]; + if (command === 'status') process.exitCode = runPrisma(['migrate', 'status', '--schema', schema]); + else if (command === 'deploy') + deploy() + .then((code) => { + process.exitCode = code; + }) + .catch((error) => { + const safe = /^(Unresolved migration|Unrecognized partial|Migration lock)/.test( + error.message, + ) + ? error.message + : 'Database migration command failed. Check configuration and database access. No reset was performed.'; + console.error(safe); + process.exitCode = 1; + }); + else process.exitCode = 1; +} diff --git a/apps/api/scripts/test-isolated.cjs b/apps/api/scripts/test-isolated.cjs index 09bfb40..1c2c459 100644 --- a/apps/api/scripts/test-isolated.cjs +++ b/apps/api/scripts/test-isolated.cjs @@ -4,8 +4,9 @@ const mysql = require('mysql2/promise'); const { spawn, spawnSync } = require('node:child_process'); const { randomBytes } = require('node:crypto'); const net = require('node:net'); -const fs = require('node:fs'); -const path = require('node:path'); +const { mkdtempSync, rmSync } = require('node:fs'); +const { join, resolve, dirname, basename } = require('node:path'); +const { tmpdir } = require('node:os'); const pkg = require('../package.json'); async function main() { const url = new URL(process.env.DATABASE_URL); @@ -17,9 +18,11 @@ async function main() { password: decodeURIComponent(url.password), }); let api; + let iconDirectory; try { await connection.query('CREATE DATABASE `' + name + '`'); url.pathname = '/' + name; + iconDirectory = mkdtempSync(join(tmpdir(), 'wp_test_icons_')); const port = await new Promise((resolve, reject) => { const server = net.createServer(); server.on('error', reject); @@ -31,6 +34,7 @@ async function main() { const env = { ...process.env, DATABASE_URL: url.toString(), + ICON_STORAGE_DIR: iconDirectory, ADMIN_USERNAME: 'admin', ADMIN_PASSWORD: 'admin', PORT: String(port), @@ -44,14 +48,7 @@ async function main() { TEST_API_URL: 'http://127.0.0.1:' + port + '/api', }; const run = (args) => spawnSync(process.execPath, args, { env, encoding: 'utf8' }); - // Historical 005-007 directories sort before the initial migration. Use the - // current schema only in this disposable database, then exercise our SQL. - const migration = run([ - require.resolve('prisma/build/index.js'), - 'db', - 'push', - '--skip-generate', - ]); + const migration = run([require.resolve('./database.cjs'), 'deploy']); if (migration.status !== 0) { const safe = (migration.stdout + migration.stderr) .split(/\r?\n/) @@ -60,31 +57,6 @@ async function main() { console.error(safe); throw Error('Disposable database migration failed'); } - await connection.query('USE `' + name + '`'); - // db push omits the historical column comments asserted by integration tests. - await connection.query( - "ALTER TABLE `User` MODIFY COLUMN `accountGroupOrder` JSON NULL COMMENT '账户分组显示顺序;空值表示沿用默认顺序'", - ); - await connection.query( - 'ALTER TABLE `User` DROP COLUMN `role`, DROP COLUMN `banned`, DROP COLUMN `mustChangePassword`', - ); - await connection.query( - fs.readFileSync( - path.join(__dirname, '../prisma/migrations/20261005130000_admin_accounts/migration.sql'), - 'utf8', - ), - ); - // Verify the icon column removal migration against its former shape too. - await connection.query('ALTER TABLE `Icon` ADD COLUMN `source` VARCHAR(500) NULL'); - await connection.query( - fs.readFileSync( - path.join( - __dirname, - '../prisma/migrations/20261005150000_remove_icon_source/migration.sql', - ), - 'utf8', - ), - ); const build = run([require.resolve('typescript/bin/tsc')]); if (build.status !== 0) { console.log(build.stdout); @@ -110,6 +82,7 @@ async function main() { 'test/admin-integration.test.ts', 'test/mcp.test.ts', 'test/oauth-duration.test.ts', + 'test/database-migrations.test.ts', ]; const result = spawnSync( process.execPath, @@ -125,6 +98,12 @@ async function main() { } await connection.query('DROP DATABASE `' + name + '`'); await connection.end(); + if ( + iconDirectory && + dirname(resolve(iconDirectory)) === resolve(tmpdir()) && + basename(iconDirectory).startsWith('wp_test_icons_') + ) + rmSync(iconDirectory, { recursive: true, force: true }); console.log('Disposable test database and API cleaned up.'); } } @@ -133,7 +112,7 @@ main().catch((e) => { 'Isolated test run failed: ' + (/Disposable|API build/.test(e.message) ? e.message - : 'check test configuration or database access'), + : 'check test configuration or database access (' + (e.code || e.name) + ')'), ); process.exitCode = 1; }); diff --git a/apps/api/src/backup.ts b/apps/api/src/backup.ts index 4003fd6..f9fc91d 100644 --- a/apps/api/src/backup.ts +++ b/apps/api/src/backup.ts @@ -49,6 +49,7 @@ import { import { createHash } from 'node:crypto'; import { toBusinessDate } from './validation'; import { iconName, validateStoredIcon } from './icons'; +import { persistIconFile } from './icon-files'; import { day, businessTime } from './calculation'; const timestamp = z.iso .datetime() @@ -656,7 +657,11 @@ export class BackupBusinessService implements OnModuleDestroy, OnModuleInit { .parse(raw), b = validateBackup(backup); const iconData = new Map(); - for (const i of b.icons) iconData.set(i.id, await validateStoredIcon(i.image, i.hash)); + for (const i of b.icons) { + const data = await validateStoredIcon(i.image, i.hash); + iconData.set(i.id, data); + await persistIconFile(i.hash, data); + } return this.db.$transaction( async (tx) => { const ps = await tx.position.findMany({ diff --git a/apps/api/src/icon-files.ts b/apps/api/src/icon-files.ts new file mode 100644 index 0000000..c0313ba --- /dev/null +++ b/apps/api/src/icon-files.ts @@ -0,0 +1,63 @@ +import { mkdir, readFile, writeFile, rename, unlink } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { createHash, randomBytes } from 'node:crypto'; + +const digest = (data: Buffer) => createHash('sha256').update(data).digest('hex'); +function target(hash: string) { + if (!/^[a-f0-9]{64}$/.test(hash)) throw Error('Invalid icon hash'); + const directory = process.env.ICON_STORAGE_DIR?.trim(); + return directory ? join(resolve(directory), hash + '.png') : undefined; +} + +export async function initializeIconDirectory() { + const configured = process.env.ICON_STORAGE_DIR?.trim(); + if (!configured) return; + const directory = resolve(configured); + await mkdir(directory, { recursive: true, mode: 0o750 }); + const probe = join(directory, '.write-check-' + randomBytes(16).toString('hex')); + await writeFile(probe, '', { flag: 'wx', mode: 0o600 }); + await unlink(probe); +} + +// Immutable content-addressed files can be shared by multiple icon records. +// Keep the database copy for existing installations and self-contained ZIP backups. +export async function persistIconFile(hash: string, data: Buffer) { + const file = target(hash); + if (!file) return; + if (digest(data) !== hash) throw Error('Invalid icon contents'); + try { + if (digest(await readFile(file)) === hash) return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + await mkdir(resolve(process.env.ICON_STORAGE_DIR!.trim()), { recursive: true, mode: 0o750 }); + const temporary = file + '.' + randomBytes(16).toString('hex') + '.tmp'; + try { + await writeFile(temporary, data, { flag: 'wx', mode: 0o600 }); + try { + await rename(temporary, file); + } catch (error) { + // Windows may reject replacing a file another upload has just published. + // Accept only an already complete file with exactly the expected content. + const existing = await readFile(file).catch(() => undefined); + if (!existing || digest(existing) !== hash) throw error; + } + } finally { + await unlink(temporary).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + }); + } +} + +export async function readIconFile(hash: string, databaseData: Buffer) { + const file = target(hash); + if (!file) return databaseData; + try { + const data = await readFile(file); + if (digest(data) === hash) return data; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + await persistIconFile(hash, databaseData); + return databaseData; +} diff --git a/apps/api/src/icons.ts b/apps/api/src/icons.ts index 412e16c..bb307ea 100644 --- a/apps/api/src/icons.ts +++ b/apps/api/src/icons.ts @@ -12,6 +12,7 @@ import { UseInterceptors, BadRequestException, NotFoundException, + OnModuleInit, } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; import { memoryStorage } from 'multer'; @@ -21,6 +22,7 @@ import { createHash } from 'node:crypto'; import { z } from 'zod'; import { Database } from './database'; import { UserRequest } from './auth'; +import { initializeIconDirectory, persistIconFile, readIconFile } from './icon-files'; export const iconName = z.string().trim().min(1).max(100); export const iconHash = (data: Buffer) => createHash('sha256').update(data).digest('hex'); @@ -78,8 +80,24 @@ export class IconsService { } @Injectable() -export class IconsBusinessService { +export class IconsBusinessService implements OnModuleInit { constructor(private db: Database) {} + async onModuleInit() { + if (!process.env.ICON_STORAGE_DIR?.trim()) return; + await initializeIconDirectory(); + let cursor: string | undefined; + for (;;) { + const rows = await this.db.icon.findMany({ + orderBy: { id: 'asc' }, + take: 100, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + select: { id: true, hash: true, data: true }, + }); + for (const row of rows) await persistIconFile(row.hash, Buffer.from(row.data)); + if (rows.length < 100) break; + cursor = rows[rows.length - 1].id; + } + } async list(r: UserRequest, q = '', page = '1') { const query = z.string().trim().max(100).parse(q); const index = z.coerce.number().int().min(1).max(100000).parse(page); @@ -105,7 +123,7 @@ export class IconsBusinessService { res.setHeader('X-Content-Type-Options', 'nosniff'); // Uploads, built-in seeding and imports already validate stored PNG data. // Preserve essential white artwork rather than applying the cutout twice. - res.send(Buffer.from(icon.data)); + res.send(await readIconFile(icon.hash, Buffer.from(icon.data))); } async upload(r: UserRequest, raw: unknown, file?: Express.Multer.File) { @@ -123,6 +141,7 @@ export class IconsBusinessService { if (!file) throw new BadRequestException('请选择图标文件'); const data = await normalizeIcon(file.buffer), hash = iconHash(data); + await persistIconFile(hash, data); const icon = await this.db.icon.upsert({ where: { ownerId_hash_shared: { ownerId: r.userId, hash, shared } }, create: { name: v.name, ownerId: r.userId, shared, hash, data }, diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts index 54ca22c..680d4d2 100644 --- a/apps/api/src/main.ts +++ b/apps/api/src/main.ts @@ -139,6 +139,8 @@ async function bootstrap() { console.log('WorthPath API ready'); } void bootstrap().catch(() => { - console.error('API startup failed. Check local configuration and database availability.'); + console.error( + 'API startup failed. Check configuration, icon directory permissions and database availability.', + ); process.exitCode = 1; }); diff --git a/apps/api/test/database-migrations.test.ts b/apps/api/test/database-migrations.test.ts new file mode 100644 index 0000000..572dc3b --- /dev/null +++ b/apps/api/test/database-migrations.test.ts @@ -0,0 +1,99 @@ +import 'dotenv/config'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import mysql from 'mysql2/promise'; +import { spawnSync } from 'node:child_process'; +import { randomBytes, createHash } from 'node:crypto'; +import { readdirSync, readFileSync } from 'node:fs'; +import { resolve, join } from 'node:path'; + +// Every scenario owns its random database; never run against the configured business database. +test( + 'real migrations initialize, recover failed 005, preserve data and reject untracked schemas', + { skip: process.env.TEST_ISOLATED !== 'true' }, + async () => { + const url = new URL(process.env.DATABASE_URL!); + const db = await mysql.createConnection({ + host: url.hostname, + port: Number(url.port || 3306), + user: decodeURIComponent(url.username), + password: decodeURIComponent(url.password), + }); + const directory = resolve('prisma/migrations'); + const expected = readdirSync(directory, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name) + .sort(); + const names: string[] = []; + const run = (database: string, raw = false) => { + const target = new URL(url); + target.pathname = '/' + database; + return spawnSync( + process.execPath, + raw + ? [require.resolve('prisma/build/index.js'), 'migrate', 'deploy'] + : [resolve('scripts/database.cjs'), 'deploy'], + { env: { ...process.env, DATABASE_URL: target.toString() }, encoding: 'utf8' }, + ); + }; + const create = async () => { + const name = 'wp_test_migrations_' + randomBytes(8).toString('hex'); + await db.query('CREATE DATABASE `' + name + '`'); + names.push(name); + await db.query('USE `' + name + '`'); + return name; + }; + const verify = async () => { + const [rows] = await db.query( + 'SELECT migration_name, checksum FROM `_prisma_migrations` WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL ORDER BY migration_name', + ); + assert.deepEqual( + rows.map((r) => r.migration_name), + expected, + ); + for (const row of rows) + assert.equal( + row.checksum, + createHash('sha256') + .update(readFileSync(join(directory, row.migration_name, 'migration.sql'))) + .digest('hex'), + ); + const [columns] = await db.query("SHOW COLUMNS FROM `Icon` LIKE 'source'"); + assert.equal(columns.length, 0); + }; + try { + const fresh = await create(); + assert.equal(run(fresh).status, 0, 'empty database deploy must succeed'); + await verify(); + await db.query('CREATE TABLE `migration_test_marker` (`value` INT NOT NULL)'); + await db.query('INSERT INTO `migration_test_marker` VALUES (42)'); + const [before] = await db.query('SELECT * FROM `_prisma_migrations` ORDER BY id'); + assert.equal(run(fresh).status, 0, 'repeat deploy must succeed'); + const [after] = await db.query('SELECT * FROM `_prisma_migrations` ORDER BY id'); + assert.deepEqual(after, before, 'existing migration history must stay unchanged'); + const [marker] = await db.query('SELECT * FROM `migration_test_marker`'); + assert.equal(marker[0].value, 42); + + const failed = await create(); + const broken = run(failed, true); + assert.notEqual(broken.status, 0); + assert.match(broken.stdout + broken.stderr, /1824/); + assert.equal(run(failed).status, 0, 'known failed 005 on an empty database must recover'); + await verify(); + const [rolled] = await db.query( + "SELECT * FROM `_prisma_migrations` WHERE migration_name='005_account_icons' AND rolled_back_at IS NOT NULL", + ); + assert.equal(rolled.length, 1); + + const untracked = await create(); + await db.query('CREATE TABLE `existing_data` (`value` INT NOT NULL)'); + await db.query('INSERT INTO `existing_data` VALUES (7)'); + assert.notEqual(run(untracked).status, 0, 'untracked existing schema must be refused'); + const [preserved] = await db.query('SELECT * FROM `existing_data`'); + assert.equal(preserved[0].value, 7); + } finally { + for (const name of names) await db.query('DROP DATABASE `' + name + '`'); + await db.end(); + } + }, +); diff --git a/apps/api/test/database-plan.test.ts b/apps/api/test/database-plan.test.ts new file mode 100644 index 0000000..64d67d2 --- /dev/null +++ b/apps/api/test/database-plan.test.ts @@ -0,0 +1,39 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +const { bootstrapPlan, initialMigrations } = require('../scripts/database.cjs'); +const applied = (name: string) => ({ migration_name: name, finished_at: new Date() }); +const failure = { + migration_name: '005_account_icons', + applied_steps_count: 0, + logs: "Database error code: 1824\nFailed to open the referenced table 'user'", +}; +test('bootstrap accepts empty databases and prerequisite prefixes; preserves applied histories', () => { + assert.deepEqual(bootstrapPlan([], []), { bootstrap: true, recover: false }); + assert.deepEqual(bootstrapPlan(['_prisma_migrations'], [failure]), { + bootstrap: true, + recover: true, + }); + assert.deepEqual(bootstrapPlan(['User'], [applied(initialMigrations[0])]), { + bootstrap: true, + recover: false, + }); + assert.deepEqual( + bootstrapPlan(['User'], [...initialMigrations.map(applied), applied('005_account_icons')]), + { bootstrap: false, recover: false }, + ); +}); +test('bootstrap refuses unknown failures, partially applied SQL and untracked schemas', () => { + for (const row of [ + { ...failure, migration_name: '006_navigation_transfers' }, + { ...failure, logs: 'Other database error' }, + { ...failure, applied_steps_count: 1 }, + ]) + assert.throws(() => bootstrapPlan([], [row]), /manual recovery/); + assert.throws(() => bootstrapPlan(['Icon'], [failure]), /manual recovery/); + assert.throws(() => bootstrapPlan(['User'], []), /Unrecognized/); + assert.throws(() => bootstrapPlan(['User'], [applied(initialMigrations[1])]), /Unrecognized/); + assert.throws( + () => bootstrapPlan(['User'], [...initialMigrations.map(applied), failure]), + /manual recovery/, + ); +}); diff --git a/apps/api/test/icon-files.test.ts b/apps/api/test/icon-files.test.ts new file mode 100644 index 0000000..02c0fcf --- /dev/null +++ b/apps/api/test/icon-files.test.ts @@ -0,0 +1,57 @@ +import 'reflect-metadata'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, writeFile, readdir, rm } from 'node:fs/promises'; +import { join, resolve, dirname, basename } from 'node:path'; +import { tmpdir } from 'node:os'; +import { createHash } from 'node:crypto'; +import { persistIconFile, readIconFile } from '../src/icon-files'; +import { IconsBusinessService } from '../src/icons'; +import { Database } from '../src/database'; + +test('icon files persist concurrently, repair damage, reject unsafe names and backfill historical pages', async () => { + const directory = await mkdtemp(join(tmpdir(), 'wp_test_icon_files_')); + const previous = process.env.ICON_STORAGE_DIR; + process.env.ICON_STORAGE_DIR = directory; + const data = Buffer.from('stored icon bytes'); + const hash = createHash('sha256').update(data).digest('hex'); + const file = join(directory, hash + '.png'); + try { + await Promise.all(Array.from({ length: 5 }, () => persistIconFile(hash, data))); + assert.deepEqual(await readFile(file), data); + assert.deepEqual(await readdir(directory), [hash + '.png']); + await writeFile(file, 'damaged'); + assert.deepEqual(await readIconFile(hash, data), data); + assert.deepEqual(await readFile(file), data); + await assert.rejects(persistIconFile('../escape', data), /Invalid icon hash/); + await assert.rejects(persistIconFile('0'.repeat(64), data), /Invalid icon contents/); + + const historical = Buffer.from('historical icon on the second page'); + const historicalHash = createHash('sha256').update(historical).digest('hex'); + const rows = Array.from({ length: 101 }, (_, n) => ({ + id: String(n).padStart(3, '0'), + hash: n === 100 ? historicalHash : hash, + data: n === 100 ? historical : data, + })); + const database = { + icon: { + findMany: async (args: any) => { + const start = args.cursor ? rows.findIndex((r) => r.id === args.cursor.id) + 1 : 0; + return rows.slice(start, start + args.take); + }, + }, + } as unknown as Database; + await new IconsBusinessService(database).onModuleInit(); + assert.deepEqual(await readFile(join(directory, historicalHash + '.png')), historical); + assert.equal( + (await readdir(directory)).filter((name) => name.startsWith('.write-check')).length, + 0, + ); + } finally { + if (previous === undefined) delete process.env.ICON_STORAGE_DIR; + else process.env.ICON_STORAGE_DIR = previous; + assert.equal(dirname(resolve(directory)), resolve(tmpdir())); + assert.ok(basename(directory).startsWith('wp_test_icon_files_')); + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/apps/api/test/icons.test.ts b/apps/api/test/icons.test.ts index 6ced2b2..2ed2258 100644 --- a/apps/api/test/icons.test.ts +++ b/apps/api/test/icons.test.ts @@ -7,6 +7,8 @@ import { randomBytes, randomUUID } from 'node:crypto'; import { PrismaClient } from '@prisma/client'; import sharp from 'sharp'; import { readBackupZip } from '../src/zip'; +import { readFile, unlink, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; const base = process.env.TEST_API_URL || 'http://127.0.0.1:3100/api'; test('private and shared icons, account reuse and complete ZIP restoration preserve isolation', async () => { const db = new PrismaClient(), @@ -74,6 +76,21 @@ test('private and shared icons, account reuse and complete ZIP restoration prese assert.equal((await call('/icons')).status, 401); const own = await upload(a.cookie, '我的银行'); assert.equal(own.status, 201); + const stored = await db.icon.findUniqueOrThrow({ where: { id: own.data.id } }); + const persistedFile = process.env.ICON_STORAGE_DIR + ? join(process.env.ICON_STORAGE_DIR, stored.hash + '.png') + : undefined; + if (process.env.TEST_ISOLATED === 'true') { + assert.ok(persistedFile); + assert.deepEqual(await readFile(persistedFile), Buffer.from(stored.data)); + await unlink(persistedFile); + assert.equal((await call('/icons/' + own.data.id + '/image', a.cookie)).status, 200); + assert.deepEqual(await readFile(persistedFile), Buffer.from(stored.data)); + await writeFile(persistedFile, 'corrupt file'); + const repaired = await call('/icons/' + own.data.id + '/image', a.cookie); + assert.deepEqual(repaired.data, Buffer.from(stored.data)); + assert.deepEqual(await readFile(persistedFile), Buffer.from(stored.data)); + } assert.equal('source' in own.data, false); const ownList = await call('/icons?q=' + encodeURIComponent('我的银行'), a.cookie); assert.equal('source' in ownList.data.items.find((i: any) => i.id === own.data.id), false); @@ -172,6 +189,7 @@ test('private and shared icons, account reuse and complete ZIP restoration prese 400, ); assert.equal(await db.icon.count(), before); + if (process.env.TEST_ISOLATED === 'true') await unlink(persistedFile!); assert.equal( ( await call('/backup/restore-fixture', b.cookie, 'POST', { @@ -181,6 +199,8 @@ test('private and shared icons, account reuse and complete ZIP restoration prese ).status, 201, ); + if (process.env.TEST_ISOLATED === 'true') + assert.deepEqual(await readFile(persistedFile!), Buffer.from(stored.data)); const imported = await db.position.findMany({ where: { userId: b.id, importedFromId: { not: null } }, include: { icon: true }, diff --git a/compose.yaml b/compose.yaml index ee2cfe5..ff06bf3 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,6 +6,14 @@ services: dockerfile: Dockerfile env_file: - .env.production + environment: + ICON_STORAGE_DIR: /app/data/icons + volumes: + - type: bind + source: /opt/worthpath/data/icons + target: /app/data/icons + bind: + create_host_path: false ports: - '127.0.0.1:3100:3100' restart: unless-stopped diff --git a/docker.md b/docker.md index e08a883..3f15a43 100644 --- a/docker.md +++ b/docker.md @@ -1,5 +1,42 @@ # WorthPath Docker 部署 +## 更新已部署的项目(尚未上传图片) + +图标持久化新增配置不需要数据库结构迁移,也不需要搬运上传图片。保留服务器已有 `.env.production`、MySQL 与反向代理配置;更新镜像和 Compose 的图标挂载即可。历史空库迁移修复也包含在此次更新中,已完成迁移的数据库不用重复初始化或运行图标 seed。 + +本机重新打包镜像并导出(在源码项目目录执行): + +```powershell +docker build -t worthpath:local . +docker save -o E:\worthpath-local.tar worthpath:local +``` + +将新的 `worthpath-local.tar` 和 `compose.yaml` 传到服务器 `/opt/worthpath`,同步 Compose 时保留服务器已有的环境文件路径、端口和数据库网络配置。若希望只修改现有 Compose,在 `services.app` 下合并以下配置(不要重复创建已有的 `environment` 或 `volumes` 键): + +```yaml +environment: + ICON_STORAGE_DIR: /app/data/icons +volumes: + - type: bind + source: /opt/worthpath/data/icons + target: /app/data/icons + bind: + create_host_path: false +``` + +服务器执行: + +```bash +cd /opt/worthpath +sudo install -d -m 750 -o 1000 -g 1000 /opt/worthpath/data/icons +docker load -i worthpath-local.tar +docker compose up -d --no-build --force-recreate app +docker compose ps +docker compose logs --tail=100 app +``` + +上述使用 `worthpath:local`,服务器 Compose 的 `image` 标签需要一致。Compose 已设置容器路径,已有 `.env.production` 无需额外添加变量。启动时现有内置图标也会自动落盘,因此尚未上传图片时目录也可能已有 PNG。随后在网页上传一张图标,可在服务器执行 `find /opt/worthpath/data/icons -maxdepth 1 -type f -name '*.png'` 检查文件。 + ## 部署结构 前后端合并为一个应用容器:构建时编译 React/Vite 和 NestJS,将网页产物复制到 `apps/api/public`;运行时只启动 `node dist/main.js`。同一端口提供网页、`/api`、MCP 和 OAuth,不启动 Vite 开发服务。 @@ -16,6 +53,10 @@ WorthPath 应用容器 本仓库的 `compose.yaml` 只管理应用,不创建数据库。需要已有 MySQL 8+、已创建的 `worthpath` 数据库及可用的数据库账号。Compose 使用命名镜像,可通过 `WORTHPATH_IMAGE_TAG` 区分版本。运行镜像保留 Prisma CLI 和相关依赖,以便单独执行迁移,不包含后端源码、测试或前端开发目录。 +上传图标持久化到宿主机 `/opt/worthpath/data/icons`,Compose 将它绑定到容器 `/app/data/icons`,并设置 `ICON_STORAGE_DIR=/app/data/icons`。使用 SHA-256 作为 PNG 文件名,相同内容去重。上传及 ZIP 恢复时写入文件;应用启动时分批补齐数据库中的已有图标。数据库继续保留图片内容,支持现有自包含 ZIP 备份,并可自动修复缺失或损坏的图标文件。图标访问仍通过带身份与可见性校验的 `/api/icons/:id/image` 接口;不要将该目录作为 Nginx 静态目录公开。 + +首次启动前创建目录并设置权限(见上面的 `install` 命令);已有目录权限不正确时执行 `sudo chown 1000:1000 /opt/worthpath/data/icons`。Compose 不自动创建宿主机目录,以避免生成 root 所有且不可写的目录。换镜像或重建容器会保留这些文件。图标文件按内容共享,删除账号或清空数据会撤销数据库引用和接口访问,不自动删除磁盘上的内容文件;服务器备份应包含该目录与 MySQL。 + ## Windows 本机安装 Docker 与构建镜像 本机开发/构建使用 Docker Desktop 的 WSL 2 后端,服务器运行使用 Docker Engine。你可以选择直接在 Linux 服务器构建,本机安装 Docker 不是服务器部署的前置条件。 @@ -65,7 +106,9 @@ docker build -t worthpath:local . 2026-10-05:已通过前后端 TypeScript 检查、前后端生产构建,以及网页托管/网络/Host 的 6 项回归测试。检查了真实环境文件的 Git 忽略规则。测试不连接业务数据库。 -当前本机未找到 Docker 命令,WSL 状态检查提示需要安装;因此尚未完成 Linux 镜像构建、Compose 配置验证、容器内迁移或真实 HTTPS/MCP 联通验证。安装完成后先运行上面的环境验证和 `docker build`,再按后续步骤部署到服务器。 +初次验证时本机未找到 Docker 命令。随后配置正式部署环境时,已检测到 Docker CLI,并通过配置解析检查。图标持久化修改通过 57 项单元测试和 30 项隔离业务回归,包含真实上传落盘、备份恢复落盘、缺失或损坏文件修复及权限隔离。Compose 使用 `docker compose config --no-env-resolution --quiet` 校验结构,未读取服务器上的环境文件。服务器目录权限、容器内迁移与真实 HTTPS/MCP 联通仍需在部署环境验证。 + +本机 Docker Desktop 的 Linux 引擎已完成修改后镜像构建,包含前后端生产构建。两个断网临时容器验证了 UID 1000 写入绑定的测试目录,以及第一个容器销毁后第二个容器读取同一 PNG。测试目录已清理,未连接业务数据库;此验证不代表服务器 `/opt/worthpath/data/icons` 的权限已配置。 ## 1. 准备服务器 diff --git a/docs/admin-accounts.md b/docs/admin-accounts.md index f8244c2..56f5520 100644 --- a/docs/admin-accounts.md +++ b/docs/admin-accounts.md @@ -23,4 +23,4 @@ ADMIN_PASSWORD=admin 运行隔离回归:在仓库根目录执行 `pnpm --filter @worthpath/api test:isolated`。测试库名使用随机 `wp_test_` 前缀,API 使用随机本机端口;完成或失败后清理自己创建的服务和数据库。管理员测试只在该隔离流程中运行,避免修改真实管理员。 -已有迁移目录 `005_account_icons`、`006_navigation_transfers`、`007_notes_display` 排在初始迁移之前,因此原有 `migrate deploy` 从空库初始化会失败。本次未重命名历史迁移。隔离测试用当前 Prisma 模型建立临时库,再移除本次字段并执行新增 SQL 来验证增量迁移;现有库迁移正常。全新部署仍需要单独修复历史迁移顺序。 +历史 `005_account_icons`、`006_navigation_transfers`、`007_notes_display` 排在初始迁移之前的问题已由项目迁移入口修复,保留所有历史目录名称和 SQL 校验值。隔离测试通过正式迁移入口建立空库,再运行业务回归;不再使用 `db push` 或手动改列来代替迁移。空库失败恢复与部署命令见 [数据库迁移](database-migrations.md)。 diff --git a/docs/architecture.md b/docs/architecture.md index 54e3f0a..21916d5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -32,7 +32,7 @@ Revision 保存按业务日期生效的绝对金额,每次金额更新新增 ## 账户图标模块 -Icon 存储 name、ownerId、shared、SHA-256 和规范静态 PNG 的 MediumBlob,不存储 source;内置图标来源保留在资源清单中。Position.iconId 外键 SetNull,多个账户共享同一图片。所有图标接口使用已验证身份,读取/检索/赋值均限定 shared=true 或 ownerId=当前用户,响应不返回 ownerId。个人上传默认私有;共享发布须中文名称和明确公开确认。拒绝 SVG、动图、损坏图片、超限像素/文件,重编码移除元数据。内置图标固定 ID 追加初始化;在线请求不会发送用户财务数据。 +Icon 存储 name、ownerId、shared、SHA-256 和规范静态 PNG 的 MediumBlob,不存储 source;内置图标来源保留在资源清单中。配置 `ICON_STORAGE_DIR` 后,上传及备份恢复同时将图片按 SHA-256 文件名持久化,启动时分批补齐旧图标,读取时校验文件并从数据库修复缺失或损坏内容。Docker 将 `/app/data/icons` 绑定到宿主机 `/opt/worthpath/data/icons`,数据库内容继续用于兼容与自包含 ZIP 备份。内容文件可以被多条图标记录共用,删除数据库记录不自动删除文件。Position.iconId 外键 SetNull,多个账户共享同一图片。所有图标接口使用已验证身份,读取/检索/赋值均限定 shared=true 或 ownerId=当前用户,响应不返回 ownerId。个人上传默认私有;共享发布须中文名称和明确公开确认。拒绝 SVG、动图、损坏图片、超限像素/文件,重编码移除元数据。内置图标固定 ID 追加初始化;在线请求不会发送用户财务数据。 当前备份要求完整 icons 数组,ZIP v9 将图标内容放入 icons.json;内容校验和解码在导入事务前完成,图标在事务内以当前用户私有范围重建,账户关联重映射。清空删除私有图标,公开共享图标不因发布者清空而消失。用户删除时图标 ownerId SetNull,不影响他人已引用的公共图标。 diff --git a/docs/database-migrations.md b/docs/database-migrations.md new file mode 100644 index 0000000..661c6b1 --- /dev/null +++ b/docs/database-migrations.md @@ -0,0 +1,29 @@ +# 数据库迁移 + +在仓库根目录执行: + +```sh +pnpm db:migrate +``` + +Docker 部署需要重新构建镜像以包含修复后的迁移脚本,再执行迁移: + +```sh +docker compose build app +docker compose run --rm app node scripts/database.cjs deploy +docker compose up -d app +``` + +请使用以上项目入口。历史目录 `005_account_icons`、`006_navigation_transfers`、`007_notes_display` 排在 `202610010001_initial` 前面,直接运行 Prisma `migrate deploy` 会在空库首先创建 Icon,因 User 表尚不存在而出现 MySQL 1824。项目入口先通过 Prisma 部署四个初始迁移,再部署完整迁移集合。历史 SQL 与目录名称保持原样,已有数据库的记录和校验值不变;重复运行只应用尚未执行的迁移。 + +如果已经遇到该失败,更新代码或镜像后重新运行项目入口即可。脚本仅在确认没有业务表、没有成功迁移、仅有 `005_account_icons` 的 1824 失败且执行步数为零时,通过 Prisma `migrate resolve --rolled-back 005_account_icons` 标记该次失败,再按正确顺序迁移。不会执行 reset、db push 或删除业务表。 + +其他失败、部分执行或没有迁移记录的已有表会拒绝自动恢复,需要根据实际数据库状态手动处理。不要直接把失败记录标记为 applied,也不要对有数据的库执行 reset。Prisma 官方说明见 [生产迁移故障恢复](https://www.prisma.io/docs/orm/v6/prisma-migrate/workflows/patching-and-hotfixing)。 + +隔离回归入口: + +```sh +pnpm --filter @worthpath/api test:isolated +``` + +测试只创建和清理随机 `wp_test_` 库,覆盖真实空库初始化、重现 005 失败并恢复、迁移校验值一致、重复执行保留数据和历史、拒绝未跟踪的已有表,以及完整业务回归。