feat(server): 同步引擎与 seed(阶段 4)

- 引擎:tick(5 分钟 + 防重入 + 启动即跑)· 单轮串行遍历 active key · 日志整批单事务入库(ON CONFLICT DO NOTHING + log_synced_at 取请求发起时刻)· 额度采集(上游字段全组,独立 try)· QPU 采样(基线 / 值变化才追加)· 每轮摘要与超期 warning
- 401 状态机:N=3 摘除(disabled_source=auto_401);403/500/网络不计数;解密失败跳过不计数
- 纯逻辑抽层:policy(计数 / 合并判定)· dedupe(去重键与内容指纹)· log-row(日志映射,与 seed 共用)
- seed 脚本(幂等):假 key + 984 条日志 + QPU 基线 + 示例购买
- 实测:401 端到端 1→2→3 摘除且摘除后不再轮询;幂等重跑新增 0 条
- 实证发现:错误行 request_id 不唯一(8 组 ×3)——记入实施计划「待定问题 #5」与 upstream 文档,待作者裁决
- AGENTS.md / 根 package.json 增 seed 命令
This commit is contained in:
2026-09-15 00:46:46 +08:00
parent 4b5d21a418
commit e1c6d2153e
14 changed files with 623 additions and 37 deletions
+19
View File
@@ -0,0 +1,19 @@
/**
* 上游字段全组(`api_keys` 表 8 个上游列)——登记 / 启用 / 每轮额度采集共用一处,
* 避免三处各写一份导致漏列(api_keys 表设计 §3.4/§3.5:整组一起写)。
*/
import type { UpstreamTokenUsage } from '../upstream/types'
import { expiresAtFromUnix } from './time'
export function upstreamColumns(usage: UpstreamTokenUsage) {
return {
tokenName: usage.name,
totalGranted: usage.total_granted,
totalUsed: usage.total_used,
totalAvailable: usage.total_available,
unlimitedQuota: usage.unlimited_quota,
modelLimits: usage.model_limits,
modelLimitsEnabled: usage.model_limits_enabled,
expiresAt: expiresAtFromUnix(usage.expires_at),
}
}
+1 -15
View File
@@ -5,7 +5,7 @@ import type { PatchApiKeyRequest, RegisterApiKeyRequest } from '@naku/shared'
import { apiError, isUniqueViolation, upstreamErrorResponse } from '../../lib/api-error'
import { getKeySecrets } from '../../lib/keys'
import { toApiKeyRow } from '../../lib/serialize'
import { expiresAtFromUnix } from '../../lib/time'
import { upstreamColumns } from '../../lib/upstream-columns'
import { manageAuth } from '../../middleware/manage-auth'
import { getDb } from '../../db/client'
import { apiKeys } from '../../db/schema'
@@ -21,20 +21,6 @@ export const apiKeysRoutes = new Hono()
apiKeysRoutes.use(manageAuth)
/** 上游字段全组(登记 / 启用验证成功时一起写,见 api_keys 表设计 §3.4/§3.5) */
function upstreamColumns(usage: UpstreamTokenUsage) {
return {
tokenName: usage.name,
totalGranted: usage.total_granted,
totalUsed: usage.total_used,
totalAvailable: usage.total_available,
unlimitedQuota: usage.unlimited_quota,
modelLimits: usage.model_limits,
modelLimitsEnabled: usage.model_limits_enabled,
expiresAt: expiresAtFromUnix(usage.expires_at),
}
}
/** 清单(按 id 升序,不分页) */
apiKeysRoutes.get('/', async (c) => {
const rows = await getDb().select().from(apiKeys).orderBy(asc(apiKeys.id))
+53
View File
@@ -0,0 +1,53 @@
/**
* 去重键单测(usage_logs 表设计 §3.1 + 实施计划「实现假设」#5)。
*/
import { describe, expect, test } from 'bun:test'
import { contentFingerprint, dedupeKeyFor } from './dedupe'
import type { UpstreamLogRecord } from '../upstream/types'
const log = (over: Partial<UpstreamLogRecord> = {}): UpstreamLogRecord => ({
id: 1,
user_id: 5,
created_at: 1789126642,
type: 2,
content: '',
username: 'chendong',
token_name: 'yangsiyue',
model_name: 'glm-5.3-flash',
quota: 702,
prompt_tokens: 29157,
completion_tokens: 1036,
use_time: 30,
is_stream: true,
channel: 7,
channel_name: '',
token_id: 29,
group: 'default',
ip: '',
request_id: '202609111136522416558528268d9d6CRxC30kh',
other: '{}',
...over,
})
describe('dedupeKeyFor', () => {
test('request_id 存在 → `${key_id}:${request_id}`', () => {
expect(dedupeKeyFor(3, log())).toBe('3:202609111136522416558528268d9d6CRxC30kh')
})
test('跨 key 隔离:同一 request_id 在不同 key 下不同键', () => {
expect(dedupeKeyFor(3, log())).not.toBe(dedupeKeyFor(4, log()))
})
test('request_id 为空 → 内容指纹(同输入稳定)、前缀区分 `fp:`', () => {
const a = dedupeKeyFor(3, log({ request_id: '' }))
const b = dedupeKeyFor(3, log({ request_id: '' }))
expect(a).toBe(b)
expect(a.startsWith('fp:')).toBe(true)
})
test('指纹随内容变化(真实不同请求不同键)', () => {
expect(contentFingerprint(3, log())).not.toBe(contentFingerprint(3, log({ quota: 703 })))
expect(contentFingerprint(3, log())).not.toBe(contentFingerprint(4, log()))
})
})
+26
View File
@@ -0,0 +1,26 @@
/**
* 去重键生成(usage_logs 表设计 §3.1 + 实施计划「实现假设」#5):
* `dedupe_key = ${key_id}:${request_id}`;`request_id` 为空时退化为内容指纹
* `sha256(key_id|created_at|model_name|quota|prompt_tokens|completion_tokens)`。
*/
import { createHash } from 'node:crypto'
import type { UpstreamLogRecord } from '../upstream/types'
/** 内容指纹(仅 `request_id` 缺失时启用) */
export function contentFingerprint(keyId: number, log: UpstreamLogRecord): string {
const parts = [
keyId,
log.created_at,
log.model_name ?? '',
log.quota,
log.prompt_tokens,
log.completion_tokens,
].join('|')
return createHash('sha256').update(parts, 'utf8').digest('hex')
}
export function dedupeKeyFor(keyId: number, log: UpstreamLogRecord): string {
const requestId = (log.request_id ?? '').trim()
return requestId ? `${keyId}:${requestId}` : `fp:${contentFingerprint(keyId, log)}`
}
+202 -17
View File
@@ -1,24 +1,206 @@
/**
* 同步引擎(见 `docs/design/同步引擎设计.md`)——骨架占位。
* 同步引擎(见 `docs/design/同步引擎设计.md`):调度 + 单轮执行。
*
* 已定骨架(§3.2 / §3.3 / §3.4):
* - 后端进程内定时器(不引 worker、不拆进程)
* - 周期 5 分钟,固定 tick + 防重入
* - 启动立即跑一轮
* - 单轮串行遍历全部 active key
* - 每轮:日志(整批单事务入库 → log_synced_at 取请求发起时刻)· 额度(上游字段全组 + usage_synced_at)· QPU 采样(每轮一次、值变化才追加)
* - 失败:401 计数(N=3 摘除)· 403/500/网络不计数 · 不单轮重试
* 结构(§3.2 / §3.3 / §3.4):
* - 后端进程内定时器(不引 worker、不拆进程);周期 5 分钟,固定 tick + 防重入;启动立即跑一轮
* - 单轮串行遍历全部 `active` key;每 key 依次:日志(整批单事务入库 → `log_synced_at` 取请求发起时刻)
* → 额度(上游字段全组 + `usage_synced_at`,与日志各自独立 try)
* - 每轮一次 QPU 采样(独立于 key 循环;`qpu_history` 为空时插基线)
* - 失败:401 计数(N=3 摘除,计数只认 401)· 403/500/网络不计数 · 不单轮重试
* - 可观测:`[sync]` 前缀一行式日志;每轮摘要;`log_synced_at` 落后 > 24h 每轮 warning
*/
import { asc, desc, eq } from 'drizzle-orm'
const TICK_MS = 5 * 60 * 1000
import { getDb } from '../db/client'
import { apiKeys, qpuHistory, usageLogs } from '../db/schema'
import { getKeySecrets } from '../lib/keys'
import { upstreamColumns } from '../lib/upstream-columns'
import { getUpstreamClient } from '../upstream/client'
import { errorMessage, UpstreamError } from '../upstream/errors'
import type { UpstreamLogRecord } from '../upstream/types'
import { toLogRow } from './log-row'
import {
combineRoundResult,
nextKeyCounter,
STALE_WARN_MS,
TICK_MS,
type CallResult,
} from './policy'
type KeyRow = typeof apiKeys.$inferSelect
interface RoundSummary {
keyCount: number
okCount: number
failCount: number
inserted: number
}
let running = false
/** 一轮:遍历全部 active key(串行)→ 日志 / 额度;每轮一次 QPU 采样 */
async function runOnce(): Promise<void> {
// TODO(#5): 日志同步 —— GET /api/log/token → 整批单事务入库(ON CONFLICT (dedupe_key) DO NOTHING)→ 写 log_synced_at
// TODO(#6): 额度采集 —— GET /api/usage/token → 上游字段全组 + usage_synced_at(与日志各自独立)
// TODO(#7): QPU 采样 —— GET /api/status → 值变化才追加 qpu_history(每轮一次,独立于 key 循环)
/** 上游错误 → 本轮结果:只有 401 是「key 失效」信号(403 = 账号封禁,不计数) */
function classify(err: unknown): CallResult {
if (err instanceof UpstreamError && err.kind === 'key_rejected' && err.status === 401) {
return 'unauthorized'
}
return 'failed'
}
/**
* 日志整批单事务入库(`ON CONFLICT (dedupe_key) DO NOTHING`)+ 写 `log_synced_at`
* ——全成或全不写:失败则「数据截止」不前移,下一轮幂等重来(§3.3)。
*/
async function persistLogs(
keyId: number,
logs: UpstreamLogRecord[],
syncedAt: Date,
): Promise<number> {
const db = getDb()
const rows = logs.map((log) => toLogRow(keyId, log))
return await db.transaction(async (tx) => {
let inserted = 0
if (rows.length > 0) {
const res = await tx
.insert(usageLogs)
.values(rows)
.onConflictDoNothing({ target: usageLogs.dedupeKey })
.returning({ id: usageLogs.id })
inserted = res.length
}
await tx.update(apiKeys).set({ logSyncedAt: syncedAt }).where(eq(apiKeys.id, keyId))
return inserted
})
}
/** 单 key 一轮:日志 → 额度(各自独立 try)→ 状态机(每轮一次) */
async function syncOneKey(key: KeyRow): Promise<{ round: CallResult; inserted: number }> {
const db = getDb()
let plainKey: string
try {
plainKey = getKeySecrets().decryptKey(Buffer.from(key.keyCiphertext))
} catch (err) {
// 解密失败:记日志、跳过、不影响其他 key;不计数、不摘除(§3.4)
console.error(
`[sync] key_id=${key.id} 凭证解密失败,跳过本轮(需人工重新登记):${errorMessage(err)}`,
)
return { round: 'failed', inserted: 0 }
}
const results: CallResult[] = []
let inserted = 0
// —— 日志同步(独立 try)——
const logRequestedAt = new Date()
const logStartedMs = Date.now()
try {
const logs = await getUpstreamClient().fetchLogs(plainKey)
inserted = await persistLogs(key.id, logs, logRequestedAt)
results.push('ok')
console.log(
`[sync] key_id=${key.id} 日志同步:上游 ${logs.length} 条,新增 ${inserted} 条(${Date.now() - logStartedMs}ms)`,
)
} catch (err) {
results.push(classify(err))
console.error(
`[sync] key_id=${key.id} 日志同步失败(${Date.now() - logStartedMs}ms):${errorMessage(err)}`,
)
}
// —— 额度采集(独立 try;与日志互不影响、各自写各自的时间戳)——
const usageRequestedAt = new Date()
const usageStartedMs = Date.now()
try {
const usage = await getUpstreamClient().fetchTokenUsage(plainKey)
await db
.update(apiKeys)
.set({ usageSyncedAt: usageRequestedAt, ...upstreamColumns(usage) })
.where(eq(apiKeys.id, key.id))
results.push('ok')
} catch (err) {
results.push(classify(err))
console.error(
`[sync] key_id=${key.id} 额度采集失败(${Date.now() - usageStartedMs}ms):${errorMessage(err)}`,
)
}
// —— 401 状态机(每轮一次;401 优先于成功——见实施计划「实现假设」#15)——
const round = combineRoundResult(results)
const next = nextKeyCounter(
{
consecutive401s: key.consecutive401s,
status: key.status as 'active' | 'disabled',
disabledSource: key.disabledSource as 'manual' | 'auto_401' | null,
},
round,
)
const patch: Partial<typeof apiKeys.$inferInsert> = {}
if (next.consecutive401s !== key.consecutive401s) patch.consecutive401s = next.consecutive401s
if (next.status !== key.status) patch.status = next.status
if (next.disabledSource !== key.disabledSource) patch.disabledSource = next.disabledSource
if (Object.keys(patch).length > 0) {
await db.update(apiKeys).set(patch).where(eq(apiKeys.id, key.id))
}
if (next.removed) {
console.warn(
`[sync] key_id=${key.id} 连续 ${next.consecutive401s} 次 401 → 自动摘除(disabled_source=auto_401)`,
)
}
return { round, inserted }
}
/** QPU 采样:每轮一次;`qpu_history` 为空时插基线,值变化才追加(§3.3 / §3.6) */
async function sampleQpu(): Promise<void> {
const db = getDb()
try {
const status = await getUpstreamClient().fetchStatus()
const qpu = status.quota_per_unit
const [last] = await db
.select()
.from(qpuHistory)
.orderBy(desc(qpuHistory.effectiveFrom))
.limit(1)
const effectiveFrom = new Date()
if (!last) {
await db.insert(qpuHistory).values({ qpu, effectiveFrom })
console.log(`[sync] QPU 基线已插入:${qpu}`)
} else if (Number(last.qpu) !== qpu) {
await db.insert(qpuHistory).values({ qpu, effectiveFrom })
console.log(`[sync] QPU 变化:${last.qpu} → ${qpu}`)
}
} catch (err) {
console.error(`[sync] QPU 采样失败:${errorMessage(err)}`)
}
}
/** 一轮:遍历全部 active key(串行)→ 日志 / 额度;随后 QPU 采样(无 key 也跑) */
async function runOnce(): Promise<RoundSummary> {
const db = getDb()
const keys = await db
.select()
.from(apiKeys)
.where(eq(apiKeys.status, 'active'))
.orderBy(asc(apiKeys.id))
const summary: RoundSummary = { keyCount: keys.length, okCount: 0, failCount: 0, inserted: 0 }
const nowMs = Date.now()
for (const key of keys) {
// 「3 天窗口」警示:落后 > 24h 每轮 warning(§3.5)
if (key.logSyncedAt && nowMs - key.logSyncedAt.getTime() > STALE_WARN_MS) {
const hours = Math.floor((nowMs - key.logSyncedAt.getTime()) / 3_600_000)
console.warn(
`[sync] key_id=${key.id} 数据截止已落后 ${hours} 小时(>24h;窗口上限约 3 天,请尽快恢复同步)`,
)
}
const { round, inserted } = await syncOneKey(key)
summary.inserted += inserted
if (round === 'ok') summary.okCount++
else summary.failCount++
}
await sampleQpu()
return summary
}
async function tick(): Promise<void> {
@@ -27,11 +209,14 @@ async function tick(): Promise<void> {
return
}
running = true
const startedMs = Date.now()
try {
await runOnce()
console.log('[sync] 轮次完成(骨架占位:暂无实际采集动作)')
const s = await runOnce()
console.log(
`[sync] 轮次完成:${s.keyCount} key,成功 ${s.okCount},失败 ${s.failCount},新增日志 ${s.inserted} 条,耗时 ${((Date.now() - startedMs) / 1000).toFixed(1)} 秒`,
)
} catch (err) {
console.error('[sync] 轮次异常', err)
console.error(`[sync] 轮次异常:${errorMessage(err)}`)
} finally {
running = false
}
+45
View File
@@ -0,0 +1,45 @@
/**
* 上游日志行 → `usage_logs` 列值(同步引擎与 seed 脚本共用,避免两份映射漂移)。
*/
import { fromUnixSeconds } from '../lib/time'
import type { usageLogs } from '../db/schema'
import type { UpstreamLogRecord } from '../upstream/types'
import { dedupeKeyFor } from './dedupe'
/** 上游 `other`(JSON 字符串)→ jsonb;解析失败退化为空对象(保真优先,不丢整条日志) */
export function parseOther(raw: string | undefined | null): Record<string, unknown> {
if (!raw) return {}
try {
const v = JSON.parse(raw) as unknown
return v && typeof v === 'object' && !Array.isArray(v) ? (v as Record<string, unknown>) : {}
} catch {
return {}
}
}
export function toLogRow(keyId: number, log: UpstreamLogRecord): typeof usageLogs.$inferInsert {
return {
keyId,
dedupeKey: dedupeKeyFor(keyId, log),
userId: log.user_id,
createdAt: fromUnixSeconds(log.created_at),
type: log.type,
content: log.content ?? '',
username: log.username ?? '',
tokenName: log.token_name ?? '',
modelName: log.model_name ?? '',
quota: log.quota,
promptTokens: log.prompt_tokens,
completionTokens: log.completion_tokens,
useTime: log.use_time,
isStream: log.is_stream,
channel: log.channel,
channelName: log.channel_name ?? '',
tokenId: log.token_id,
groupName: log.group ?? 'default',
ip: log.ip ?? '',
requestId: log.request_id ?? '',
upstreamRequestId: log.upstream_request_id ?? null,
other: parseOther(log.other),
}
}
+79
View File
@@ -0,0 +1,79 @@
/**
* 401 状态机与轮次结果合并的单测(同步引擎设计 §3.4 + 实施计划「实现假设」#15)。
*/
import { describe, expect, test } from 'bun:test'
import {
combineRoundResult,
MAX_CONSECUTIVE_401S,
nextKeyCounter,
type KeyCounterNext,
} from './policy'
const active = (consecutive401s = 0): KeyCounterNext => ({
consecutive401s,
status: 'active',
disabledSource: null,
removed: false,
})
describe('combineRoundResult(两个调用合并为「本轮结果」)', () => {
test('任一 401 → unauthorized(401 优先于成功)', () => {
expect(combineRoundResult(['ok', 'unauthorized'])).toBe('unauthorized')
expect(combineRoundResult(['unauthorized', 'failed'])).toBe('unauthorized')
})
test('无 401 且任一成功 → ok', () => {
expect(combineRoundResult(['ok', 'failed'])).toBe('ok')
expect(combineRoundResult(['ok', 'ok'])).toBe('ok')
})
test('全失败 → failed(不动计数)', () => {
expect(combineRoundResult(['failed', 'failed'])).toBe('failed')
})
})
describe('nextKeyCounter(401 计数 / 摘除 / 清零)', () => {
test('成功 → 计数清零(不摘除)', () => {
const next = nextKeyCounter(active(2), 'ok')
expect(next.consecutive401s).toBe(0)
expect(next.status).toBe('active')
expect(next.removed).toBe(false)
})
test('其他失败(403/500/网络)→ 计数不动', () => {
const next = nextKeyCounter(active(1), 'failed')
expect(next.consecutive401s).toBe(1)
expect(next.removed).toBe(false)
})
test('连续 401 累加;未达阈值不摘除', () => {
let state = active(0)
for (let i = 1; i < MAX_CONSECUTIVE_401S; i++) {
const next = nextKeyCounter(state, 'unauthorized')
expect(next.consecutive401s).toBe(i)
expect(next.removed).toBe(false)
expect(next.status).toBe('active')
state = next
}
})
test('第 N 次(N=3)连续 401 → 摘除且 disabled_source = auto_401', () => {
let state: KeyCounterNext = active(0)
for (let i = 1; i <= MAX_CONSECUTIVE_401S; i++) {
state = nextKeyCounter(state, 'unauthorized')
}
expect(state.consecutive401s).toBe(MAX_CONSECUTIVE_401S)
expect(state.status).toBe('disabled')
expect(state.disabledSource).toBe('auto_401')
expect(state.removed).toBe(true)
})
test('中途成功会打断连续性(计数回零)', () => {
let state: KeyCounterNext = active(2)
state = nextKeyCounter(state, 'ok')
state = nextKeyCounter(state, 'unauthorized')
expect(state.consecutive401s).toBe(1)
expect(state.removed).toBe(false)
})
})
+60
View File
@@ -0,0 +1,60 @@
/**
* 同步策略常量与纯决策函数(同步引擎设计 §3.2/§3.4/§3.5)。
* 抽成纯函数是为了把「401 计数 / 摘除」与「本轮结果合并」的规则钉在单测里。
*/
/** 一轮的固定周期:5 分钟(§3.2) */
export const TICK_MS = 5 * 60 * 1000
/** 连续 401 摘除阈值 N = 3(§3.4) */
export const MAX_CONSECUTIVE_401S = 3
/** 落后警示阈值:`log_synced_at` 落后超过 24 小时 → 每轮 warning(§3.5,距 3 天窗口留 2 天余量) */
export const STALE_WARN_MS = 24 * 60 * 60 * 1000
/** 单个上游调用在本轮的结果 */
export type CallResult = 'ok' | 'unauthorized' | 'failed'
/**
* 一轮内两个调用(日志 / 额度)的合并判定(实施计划「实现假设」#15):
* 任一 401 → `unauthorized`(key 失效信号优先);否则任一成功 → `ok`;全失败 → `failed`(不动计数)。
*/
export function combineRoundResult(results: CallResult[]): CallResult {
if (results.includes('unauthorized')) return 'unauthorized'
if (results.includes('ok')) return 'ok'
return 'failed'
}
export interface KeyCounterState {
consecutive401s: number
status: 'active' | 'disabled'
disabledSource: 'manual' | 'auto_401' | null
}
export interface KeyCounterNext extends KeyCounterState {
/** 本轮是否触发自动摘除 */
removed: boolean
}
/**
* 401 状态机(§3.4):401 → 计数 +1(达 N 摘除,`disabled_source = auto_401`);
* 成功 → 计数清零;其他失败(403 / 500 / 网络 / 解密失败)→ 计数不动。
*/
export function nextKeyCounter(state: KeyCounterState, round: CallResult): KeyCounterNext {
if (round === 'ok') {
return { ...state, consecutive401s: 0, removed: false }
}
if (round === 'failed') {
return { ...state, removed: false }
}
const consecutive401s = state.consecutive401s + 1
if (consecutive401s >= MAX_CONSECUTIVE_401S) {
return {
consecutive401s,
status: 'disabled',
disabledSource: 'auto_401',
removed: true,
}
}
return { ...state, consecutive401s, removed: false }
}