import { ensureDatabase, getD1 } from "./bootstrap"; import { type RelayNodePrincipal } from "./relay-control-plane"; import { DomainError } from "./repository"; import { authorityAfterMigrationFailure } from "./relay-migration-state"; export type RelayMigrationState = | "quiescing" | "copying" | "switching" | "draining" | "completed" | "failed"; type MigrationRecord = { id: string; tenant_id: string; source_node_id: string; target_node_id: string; source_generation: number; target_generation: number; state: RelayMigrationState; backup_ref: string | null; manifest_sha256: string | null; requested_by: string; reason: string; started_at: string; updated_at: string; switched_at: string | null; completed_at: string | null; last_error_code: string | null; }; export type RelayMigrationAssignment = { migration_id: string; tenant_id: string; role: "source" | "target"; source_node_id: string; target_node_id: string; source_generation: number; target_generation: number; state: "quiescing" | "copying" | "switching" | "draining"; backup_ref?: string; manifest_sha256?: string; finalize_after?: string; }; function changed(result: D1Result): boolean { return Number(result.meta.changes ?? 0) === 1; } function validTenantId(value: string): boolean { return /^tenant_[0-9a-f]{32}$/u.test(value); } function validNodeId(value: string): boolean { return /^node_[A-Za-z0-9][A-Za-z0-9._:-]{0,95}$/u.test(value); } function validMigrationId(value: string): boolean { return /^migration_[0-9a-f]{32}$/u.test(value); } function validBackupRef(value: string): boolean { return /^[0-9a-f]{32}\/g[0-9]{20}-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{16}$/u.test(value); } function validHash(value: string): boolean { return /^[0-9a-f]{64}$/u.test(value); } function transitionAudit( db: D1Database, record: MigrationRecord, actorId: string, action: string, expectedState: RelayMigrationState, now: string, ): D1PreparedStatement { return db.prepare( `INSERT INTO audit_events (id, actor_id, action, target_type, target_id, reason, created_at) SELECT ?1, ?2, ?3, 'relay_migration', ?4, ?5, ?6 WHERE EXISTS ( SELECT 1 FROM relay_migrations WHERE id = ?4 AND state = ?7 )`, ).bind( `audit_${crypto.randomUUID().replaceAll("-", "")}`, actorId, action, record.id, `migration ${record.tenant_id} ${expectedState}`, now, expectedState, ); } async function deterministicMigrationId(actorId: string, idempotencyKey: string): Promise { if (!/^[A-Za-z0-9._:-]{8,128}$/u.test(idempotencyKey)) { throw new DomainError("invalid_idempotency_key", "迁移幂等键无效"); } const digest = await crypto.subtle.digest( "SHA-256", new TextEncoder().encode(`nekonest-cloud/relay-migration/v1\0${actorId}\0${idempotencyKey}`), ); return `migration_${Array.from(new Uint8Array(digest).slice(0, 16), (byte) => byte.toString(16).padStart(2, "0")).join("")}`; } export async function beginRelayMigration(input: { tenantId: string; targetNodeId: string; actorId: string; reason: string; idempotencyKey: string; }): Promise { await ensureDatabase(); const tenantId = input.tenantId.trim(); const targetNodeId = input.targetNodeId.trim(); const actorId = input.actorId.trim(); const reason = input.reason.trim(); if (!validTenantId(tenantId) || !validNodeId(targetNodeId) || !actorId || reason.length < 8 || reason.length > 500) { throw new DomainError("invalid_relay_migration", "迁移租户、目标节点或原因无效"); } const migrationId = await deterministicMigrationId(actorId, input.idempotencyKey.trim()); const db = getD1(); const existing = await db .prepare("SELECT * FROM relay_migrations WHERE id = ?") .bind(migrationId) .first(); if (existing) { if ( existing.tenant_id !== tenantId || existing.target_node_id !== targetNodeId || existing.requested_by !== actorId || existing.reason !== reason ) { throw new DomainError("idempotency_conflict", "迁移幂等键已用于不同请求", 409); } return existing; } const now = new Date().toISOString(); const results = await db.batch([ db .prepare( `INSERT INTO relay_migrations (id, tenant_id, source_node_id, target_node_id, source_generation, target_generation, state, requested_by, reason, started_at, updated_at) SELECT ?1, placements.tenant_id, placements.relay_node_id, targets.id, placements.generation, placements.generation + 1, 'quiescing', ?4, ?5, ?6, ?6 FROM tenant_placements AS placements INNER JOIN relay_nodes AS sources ON sources.id = placements.relay_node_id INNER JOIN relay_nodes AS targets ON targets.id = ?3 WHERE placements.tenant_id = ?2 AND placements.state = 'active' AND placements.relay_node_id IS NOT NULL AND sources.status IN ('active', 'draining') AND targets.status = 'active' AND targets.id <> placements.relay_node_id AND NOT EXISTS ( SELECT 1 FROM relay_migrations AS active WHERE active.tenant_id = placements.tenant_id AND active.state IN ('quiescing', 'copying', 'switching', 'draining') ) AND ( targets.capacity_tenants = 0 OR (SELECT COUNT(*) FROM tenant_placements AS occupied WHERE occupied.relay_node_id = targets.id AND occupied.state IN ('active', 'draining')) < targets.capacity_tenants )`, ) .bind(migrationId, tenantId, targetNodeId, actorId, reason, now), db .prepare( `UPDATE tenant_placements SET state = 'quiescing', last_error_code = NULL, updated_at = ?1 WHERE tenant_id = ?2 AND state = 'active' AND EXISTS ( SELECT 1 FROM relay_migrations WHERE id = ?3 AND tenant_id = ?2 AND source_node_id = tenant_placements.relay_node_id AND source_generation = tenant_placements.generation AND state = 'quiescing' )`, ) .bind(now, tenantId, migrationId), db .prepare( `INSERT INTO audit_events (id, actor_id, action, target_type, target_id, reason, created_at) SELECT ?1, ?2, 'relay.migration.started', 'relay_migration', ?3, ?4, ?5 WHERE EXISTS ( SELECT 1 FROM relay_migrations WHERE id = ?3 AND tenant_id = ?6 AND state = 'quiescing' )`, ) .bind(`audit_${crypto.randomUUID().replaceAll("-", "")}`, actorId, migrationId, reason, now, tenantId), ]); if (!changed(results[0]) || !changed(results[1])) { const raced = await db .prepare("SELECT * FROM relay_migrations WHERE id = ?") .bind(migrationId) .first(); if (raced) return raced; throw new DomainError("relay_migration_conflict", "租户当前不可迁移或目标节点容量不足", 409); } const created = await db .prepare("SELECT * FROM relay_migrations WHERE id = ?") .bind(migrationId) .first(); if (!created) throw new DomainError("relay_migration_indeterminate", "迁移创建结果不确定", 503, true, 5); return created; } export async function relayMigrationAssignments( principal: RelayNodePrincipal, ): Promise { await ensureDatabase(); const rows = await getD1() .prepare( `SELECT * FROM relay_migrations WHERE state IN ('quiescing', 'copying', 'switching', 'draining') AND ((state = 'quiescing' AND source_node_id = ?1) OR (state IN ('copying', 'switching', 'draining') AND target_node_id = ?1)) ORDER BY started_at ASC, id ASC LIMIT 8`, ) .bind(principal.nodeId) .all(); return rows.results.map((row) => { const assignment: RelayMigrationAssignment = { migration_id: row.id, tenant_id: row.tenant_id, role: row.state === "quiescing" ? "source" : "target", source_node_id: row.source_node_id, target_node_id: row.target_node_id, source_generation: row.source_generation, target_generation: row.target_generation, state: row.state as RelayMigrationAssignment["state"], }; if (row.backup_ref) assignment.backup_ref = row.backup_ref; if (row.manifest_sha256) assignment.manifest_sha256 = row.manifest_sha256; if (row.state === "draining" && row.switched_at) { assignment.finalize_after = new Date(new Date(row.switched_at).getTime() + 5 * 60_000).toISOString(); } return assignment; }); } export async function advanceRelayMigration(input: { principal: RelayNodePrincipal; migrationId: string; action: "quiesced" | "copied" | "switched" | "finalized" | "failed"; backupRef?: string; manifestSha256?: string; errorCode?: string; }): Promise { await ensureDatabase(); const migrationId = input.migrationId.trim(); if (!validMigrationId(migrationId)) { throw new DomainError("invalid_relay_migration", "迁移 ID 无效"); } const db = getD1(); const record = await db .prepare("SELECT * FROM relay_migrations WHERE id = ?") .bind(migrationId) .first(); if (!record) throw new DomainError("relay_migration_not_found", "迁移不存在", 404); const sourceAction = input.action === "quiesced"; const expectedNode = sourceAction ? record.source_node_id : record.target_node_id; if (input.action === "failed") { if (![record.source_node_id, record.target_node_id].includes(input.principal.nodeId)) { throw new DomainError("relay_migration_forbidden", "节点不属于该迁移", 403); } } else if (input.principal.nodeId !== expectedNode) { throw new DomainError("relay_migration_forbidden", "节点不能推进该迁移阶段", 403); } const transitions = { quiesced: ["quiescing", "copying"], copied: ["copying", "switching"], switched: ["switching", "draining"], finalized: ["draining", "completed"], } as const; if (input.action !== "failed" && record.state === transitions[input.action][1]) return record; if (input.action !== "failed" && record.state !== transitions[input.action][0]) { throw new DomainError("relay_migration_fence_conflict", "迁移阶段已经变化", 409); } const now = new Date().toISOString(); if (input.action === "quiesced") { const backupRef = input.backupRef?.trim() ?? ""; const manifestSha256 = input.manifestSha256?.trim().toLowerCase() ?? ""; if (!validBackupRef(backupRef) || !validHash(manifestSha256)) { throw new DomainError("invalid_relay_backup", "迁移备份引用或摘要无效"); } const results = await db.batch([ db.prepare( `UPDATE relay_migrations SET state = 'copying', backup_ref = ?1, manifest_sha256 = ?2, updated_at = ?3 WHERE id = ?4 AND state = 'quiescing' AND source_node_id = ?5`, ).bind(backupRef, manifestSha256, now, migrationId, input.principal.nodeId), db.prepare( `UPDATE tenant_placements SET state = 'copying', updated_at = ?1 WHERE tenant_id = ?2 AND relay_node_id = ?3 AND generation = ?4 AND state = 'quiescing'`, ).bind(now, record.tenant_id, record.source_node_id, record.source_generation), transitionAudit(db, record, input.principal.nodeId, "relay.migration.backup_ready", "copying", now), ]); if (!changed(results[0]) || !changed(results[1])) throw new DomainError("relay_migration_fence_conflict", "迁移 quiesce 栅栏冲突", 409); } else if (input.action === "copied") { if (input.backupRef !== record.backup_ref || input.manifestSha256?.toLowerCase() !== record.manifest_sha256) { throw new DomainError("relay_backup_mismatch", "目标节点恢复的备份与控制面不一致", 409); } const results = await db.batch([ db.prepare( `UPDATE relay_migrations SET state = 'switching', updated_at = ?1 WHERE id = ?2 AND state = 'copying' AND target_node_id = ?3`, ).bind(now, migrationId, input.principal.nodeId), db.prepare( `UPDATE tenant_placements SET state = 'switching', updated_at = ?1 WHERE tenant_id = ?2 AND relay_node_id = ?3 AND generation = ?4 AND state = 'copying'`, ).bind(now, record.tenant_id, record.source_node_id, record.source_generation), transitionAudit(db, record, input.principal.nodeId, "relay.migration.copy_verified", "switching", now), ]); if (!changed(results[0]) || !changed(results[1])) throw new DomainError("relay_migration_fence_conflict", "迁移 copy 栅栏冲突", 409); } else if (input.action === "switched") { const results = await db.batch([ db.prepare( `UPDATE relay_migrations SET state = 'draining', switched_at = ?1, updated_at = ?1 WHERE id = ?2 AND state = 'switching' AND target_node_id = ?3`, ).bind(now, migrationId, input.principal.nodeId), db.prepare( `UPDATE tenant_placements SET relay_node_id = ?1, generation = ?2, state = 'draining', updated_at = ?3 WHERE tenant_id = ?4 AND relay_node_id = ?5 AND generation = ?6 AND state = 'switching'`, ).bind(record.target_node_id, record.target_generation, now, record.tenant_id, record.source_node_id, record.source_generation), transitionAudit(db, record, input.principal.nodeId, "relay.migration.switched", "draining", now), ]); if (!changed(results[0]) || !changed(results[1])) throw new DomainError("relay_migration_fence_conflict", "迁移 switch 栅栏冲突", 409); } else if (input.action === "finalized") { if (!record.switched_at || Date.now() < new Date(record.switched_at).getTime() + 5 * 60_000) { throw new DomainError("relay_migration_drain_pending", "旧 generation 尚在排空窗口", 409, true, 5); } const results = await db.batch([ db.prepare( `UPDATE relay_migrations SET state = 'completed', completed_at = ?1, updated_at = ?1 WHERE id = ?2 AND state = 'draining' AND target_node_id = ?3`, ).bind(now, migrationId, input.principal.nodeId), db.prepare( `UPDATE tenant_placements SET state = 'active', updated_at = ?1 WHERE tenant_id = ?2 AND relay_node_id = ?3 AND generation = ?4 AND state = 'draining'`, ).bind(now, record.tenant_id, record.target_node_id, record.target_generation), transitionAudit(db, record, input.principal.nodeId, "relay.migration.completed", "completed", now), ]); if (!changed(results[0]) || !changed(results[1])) throw new DomainError("relay_migration_fence_conflict", "迁移 finalize 栅栏冲突", 409); } else { const errorCode = input.errorCode?.trim().toLowerCase() ?? "relay_migration_failed"; if (!/^[a-z][a-z0-9_]{2,63}$/u.test(errorCode) || ["completed", "failed"].includes(record.state)) { throw new DomainError("invalid_relay_migration_failure", "迁移失败码或阶段无效"); } const authority = authorityAfterMigrationFailure(record); const results = await db.batch([ db.prepare( `UPDATE relay_migrations SET state = 'failed', last_error_code = ?1, completed_at = ?2, updated_at = ?2 WHERE id = ?3 AND state = ?4`, ).bind(errorCode, now, migrationId, record.state), db.prepare( `UPDATE tenant_placements SET relay_node_id = ?1, generation = ?2, state = 'active', last_error_code = ?3, updated_at = ?4 WHERE tenant_id = ?5 AND relay_node_id = ?6 AND generation = ?7 AND state = ?8`, ).bind(authority.nodeId, authority.generation, errorCode, now, record.tenant_id, authority.nodeId, authority.generation, record.state), transitionAudit(db, record, input.principal.nodeId, "relay.migration.failed", "failed", now), ]); if (!changed(results[0]) || !changed(results[1])) throw new DomainError("relay_migration_fence_conflict", "迁移回滚栅栏冲突", 409); } const updated = await db .prepare("SELECT * FROM relay_migrations WHERE id = ?") .bind(migrationId) .first(); if (!updated) throw new DomainError("relay_migration_indeterminate", "迁移状态不确定", 503, true, 5); return updated; }