refactor: auth_system 全面异步化（aiosqlite），拆掉 web/services 层的 to_thread 桥接

拆掉两类垫片的最后一处：auth_system 原为同步 sqlite3 栈，web 层与 services 的 worker 只能靠 asyncio.to_thread 逐个桥接才不阻塞事件循环——每次 DB 访问都白付一次线程池调度，且「同步调异步」的边界散落在 100+ 个调用点。B1 已把 worker 变成协程，本批把访问层本身也迁到原生异步，两条线在此合拢。改动（按子系统）： - auth_system：db.py 照 core/db.py 现有范式重建为原生异步访问层（aiosqlite + WAL） ——读路径 _q 每次独立连接、无锁、可并发；写路径 _x / write_transaction 用 asyncio.Lock 串行化，写者不阻塞读者。users / cards / credits / ledger / logs / settings / mail / geo / backup 九模块随之全部 async/await；security.py 是纯计算（pbkdf2/normalize）不动。__init__ 去掉 _DB_LOCK 与 get_conn 导出。 - web：app / deps / billing / tasks_registry 与 routers/{admin_site,auth,music, register,user} 直接 await，拆掉包裹的 to_thread。deps.secret_key 改 async，连带 _parse_cookie 也 async（会话解析要读密钥）。 - services：_taskcore / tasks / downloads / podcast 的账本调用与查询直接 await。 4 处 start_task 调用点（download.py）确认已 await。设计取舍（为什么这么做，而不是直接包一层）： - 锁按**事件循环**隔离（_LOOP_LOCKS 以 id(loop) 为键）：asyncio.Lock 绑定创建它的 loop，应用主 loop 与测试各自的 loop 并存，跨 loop 复用会抛 "bound to a different event loop"。 - WAL 切换用**独立锁** _wal_lock 而非 _write_lock：execute() 已持写锁，再在锁内 connect→_configure 会自锁死（asyncio.Lock 非重入）。切换 + 标记在专用锁内串行并二次复查，否则并发首连会同时执行 PRAGMA journal_mode=WAL 争排他锁报 database is locked（core/db.py 同款教训，实测复现）。 - force_wal 保留：journal_mode 持久化在库文件头，库文件被删重建时新文件不继承，必须重设。exists() 判定须在 aiosqlite.connect **之前**（connect 会创建空文件）。 - connect() 用 DB_PATH.parent 兜底建目录，优于 DATA_DIR：生产两者相同，但测试 monkeypatch 只换 DB_PATH 时兜底仍能命中正确目录。留在 to_thread 上的仅剩**正当的阻塞卸载**，不是同步门面：PBKDF2 的 10 万次迭代（登录链路每次都要跑）、smtplib 发信、备份文件的 copy2 / os.replace、secret.key 的文件 IO。原有 aria2 / downloader / zip 侧的 to_thread 未动。不变量逐条保持（迁移只改执行模型，不改语义）： - ledger_settle 幂等：账本非 open 直接 return 0，重启后重放不重复退款； - 终态先到先得：_mark_task_terminal 对已 done/error/canceled 的任务直接 return，堵住 canceled→error 的非法迁移边； - settle 必须在任务锁**外**调用（ledger_settle 要抢 SQLite 写锁，busy 最长 15s，持任务锁会让全站进度/提交/取消随库停摆）； - reserve_and_open 的预扣与开账同事务，消除原两步之间的崩溃丢账窗口。路径真源红线：DATA_DIR / DB_PATH / BACKUP_DIR / SECRET_KEY_PATH 仍是 auth_system.db 的**模块全局**（用与 core.db 相同的表达式，但**不是**它的别名）。测试只 patch auth_system.db.*，若改成 core.db 的别名会静默把测试写到生产库。已核 data/site.db 读写前后 sha256 完全一致。顺带清掉 get_conn：迁移后它只剩「转调 connect()」一层空壳、全仓零消费者（测试直接开裸连接、备份用 db.connect()），保留等于留一条「拿裸连接自己写」的口子。测试：19 个文件同步迁移——10 个文件的 web.deps.secret_key 替身改 async； test_db_conn.py 的多线程首连用例按异步语义拆成 2 条（单 loop gather(16) 覆盖批量并发首连、WAL 就绪后 8 线程各自 loop 建连覆盖跨 loop）；test_billing_scope.py 去掉多余的 @pytest.mark.anyio；被 await 且**正常返回**的 monkeypatch 替身一律改 async （仅 raise 的同步替身仍可用）。门禁：ruff 全过 / pyright 0-0-0（standard 模式含 tests 与顶层模块）/ pytest 384 项全绿（0 失败 0 error 0 skipped；383→384 因 test_db_conn 拆用例）。 data/site.db 读写前后 sha256 一致（a19f5db…6487），无 -wal/-shm 边车残留。下一步 B3：删 core/sync_http.py、core/async_bridge.py、core/http_errors.py 及 tests/test_async_bridge.py——三者在生产侧已零消费者（http_errors 仅 sync_http 自用，requests 已非声明依赖）。
2026-09-28 20:13:19 +08:00
1 parent 69a50c2c31
commit 96af56cc27
@@ -1,17 +1,16 @@
 # -*- coding: utf-8 -*-
 """预扣账本（开账 / 改名 / 补扣 / 结算 / 对账）。

-（自 auth_system.py 拆出，行为与原实现一致。）
+（自 auth_system.py 拆出，行为与原实现一致；DB 访问已迁移为原生异步。）
 """

 import time
 from typing import Any

 from .db import (
-    _DB_LOCK,
    _q,
    _x,
-    get_conn,
+    write_transaction,
 )
 from .logs import (
    add_log,
@@ -20,80 +19,62 @@ from .logs import (
 # ============ 计费账本（实扣、交付、退款、终态的持久化唯一记录） ============


-def ledger_open(task_id: str, user_id: int, reserved: int) -> None:
+async def ledger_open(task_id: str, user_id: int, reserved: int) -> None:
    """任务计费开账：记录本任务实际预扣次数（admin/-1 用户为 0）。
    task_id 唯一；重复 task_id 由任务系统在提交侧拒绝，这里幂等保护。"""
-    with _DB_LOCK:
-        conn = get_conn()
-        try:
-            conn.execute(
-                'INSERT OR IGNORE INTO billing_ledger (task_id, user_id, reserved, status, created_at) '
-                'VALUES (?, ?, ?, \'open\', ?)',
-                (str(task_id), int(user_id), max(0, int(reserved)), int(time.time())))
-            conn.commit()
-        finally:
-            conn.close()
+    await _x(
+        'INSERT OR IGNORE INTO billing_ledger (task_id, user_id, reserved, status, created_at) '
+        'VALUES (?, ?, ?, \'open\', ?)',
+        (str(task_id), int(user_id), max(0, int(reserved)), int(time.time())))


-def reserve_and_open(uid: int, count: int, ledger_key: str) -> int | None:
+async def reserve_and_open(uid: int, count: int, ledger_key: str) -> int | None:
    """预扣与开账同一事务（原两步之间存在崩溃丢账窗口）。
    足额：扣减+开账并返回剩余；不限/管理员：开账(reserved=0)返回 -1；不足：返回 None（不开账）。"""
    count = max(1, int(count))
    key = str(ledger_key)
-    with _DB_LOCK:
-        conn = get_conn()
-        try:
-            row = conn.execute('SELECT role, duration_seconds FROM users WHERE id = ?', (uid,)).fetchone()
-            if not row:
-                return None
-            now = int(time.time())
-            if row['role'] == 'admin':
-                conn.execute('UPDATE users SET last_active = ? WHERE id = ?', (now, uid))
-                conn.execute(
-                    'INSERT OR IGNORE INTO billing_ledger (task_id, user_id, reserved, status, created_at) '
-                    'VALUES (?, ?, 0, \'open\', ?)', (key, int(uid), now))
-                conn.commit()
-                return -1
-            cur = int(row['duration_seconds'] or 0)
-            if cur == -1:
-                conn.execute('UPDATE users SET last_active = ? WHERE id = ?', (now, uid))
-                conn.execute(
-                    'INSERT OR IGNORE INTO billing_ledger (task_id, user_id, reserved, status, created_at) '
-                    'VALUES (?, ?, 0, \'open\', ?)', (key, int(uid), now))
-                conn.commit()
-                return -1
-            if cur < count:
-                return None
-            conn.execute('UPDATE users SET duration_seconds = ?, last_active = ? WHERE id = ?',
-                         (cur - count, now, uid))
-            conn.execute(
+    async with write_transaction() as conn:
+        row = await (await conn.execute('SELECT role, duration_seconds FROM users WHERE id = ?', (uid,))).fetchone()
+        if not row:
+            return None
+        now = int(time.time())
+        if row['role'] == 'admin':
+            await conn.execute('UPDATE users SET last_active = ? WHERE id = ?', (now, uid))
+            await conn.execute(
                'INSERT OR IGNORE INTO billing_ledger (task_id, user_id, reserved, status, created_at) '
-                'VALUES (?, ?, ?, \'open\', ?)', (key, int(uid), count, now))
-            conn.commit()
-            return cur - count
-        finally:
-            conn.close()
+                'VALUES (?, ?, 0, \'open\', ?)', (key, int(uid), now))
+            return -1
+        cur = int(row['duration_seconds'] or 0)
+        if cur == -1:
+            await conn.execute('UPDATE users SET last_active = ? WHERE id = ?', (now, uid))
+            await conn.execute(
+                'INSERT OR IGNORE INTO billing_ledger (task_id, user_id, reserved, status, created_at) '
+                'VALUES (?, ?, 0, \'open\', ?)', (key, int(uid), now))
+            return -1
+        if cur < count:
+            return None
+        await conn.execute('UPDATE users SET duration_seconds = ?, last_active = ? WHERE id = ?',
+                           (cur - count, now, uid))
+        await conn.execute(
+            'INSERT OR IGNORE INTO billing_ledger (task_id, user_id, reserved, status, created_at) '
+            'VALUES (?, ?, ?, \'open\', ?)', (key, int(uid), count, now))
+        return cur - count


-def ledger_rename(old_key: str, new_key: str) -> bool:
+async def ledger_rename(old_key: str, new_key: str) -> bool:
    """把请求级临时账本行改名为正式 task_id（任务接手账目）。
    old 不存在/已关闭/new 已存在 → False。"""
-    with _DB_LOCK:
-        conn = get_conn()
-        try:
-            dup = conn.execute('SELECT 1 FROM billing_ledger WHERE task_id = ?', (str(new_key),)).fetchone()
-            if dup:
-                return False
-            cur = conn.execute(
-                'UPDATE billing_ledger SET task_id = ? WHERE task_id = ? AND status = \'open\'',
-                (str(new_key), str(old_key)))
-            conn.commit()
-            return cur.rowcount == 1
-        finally:
-            conn.close()
+    async with write_transaction() as conn:
+        dup = await (await conn.execute('SELECT 1 FROM billing_ledger WHERE task_id = ?', (str(new_key),))).fetchone()
+        if dup:
+            return False
+        cur = await conn.execute(
+            'UPDATE billing_ledger SET task_id = ? WHERE task_id = ? AND status = \'open\'',
+            (str(new_key), str(old_key)))
+        return cur.rowcount == 1


-def ledger_supplement(task_id: str, extra: int) -> bool:
+async def ledger_supplement(task_id: str, extra: int) -> bool:
    """整台播客等"先扣 1 后按实际数量补扣"场景：
    原子补扣用户余额并同步累加账本 reserved。任一步失败返回 False（调用方必须让任务失败，
    不允许无账本继续交付）。余额不足同样返回 False。"""
@@ -101,39 +82,34 @@ def ledger_supplement(task_id: str, extra: int) -> bool:
    extra = int(extra)
    if extra <= 0:
        return True
-    with _DB_LOCK:
-        conn = get_conn()
-        try:
-            row = conn.execute('SELECT user_id, status FROM billing_ledger WHERE task_id = ?',
-                               (task_id,)).fetchone()
+    try:
+        async with write_transaction() as conn:
+            row = await (await conn.execute('SELECT user_id, status FROM billing_ledger WHERE task_id = ?',
+                                            (task_id,))).fetchone()
            if not row or row['status'] != 'open':
                return False
-            urow = conn.execute('SELECT role, duration_seconds FROM users WHERE id = ?',
-                                (row['user_id'],)).fetchone()
+            urow = await (await conn.execute('SELECT role, duration_seconds FROM users WHERE id = ?',
+                                             (row['user_id'],))).fetchone()
            if not urow or urow['role'] == 'admin' or int(urow['duration_seconds'] or 0) == -1:
                return True  # 不限用户：无需补扣，账本也不加
            cur_credits = int(urow['duration_seconds'] or 0)
            if cur_credits < extra:
                return False   # 余额不足：任务必须失败（不允许扣到 0 硬做）
-            conn.execute('UPDATE users SET duration_seconds = duration_seconds - ? WHERE id = ?',
-                         (extra, row['user_id']))
-            conn.execute('UPDATE billing_ledger SET reserved = reserved + ? WHERE task_id = ?',
-                         (extra, task_id))
-            conn.commit()
+            await conn.execute('UPDATE users SET duration_seconds = duration_seconds - ? WHERE id = ?',
+                               (extra, row['user_id']))
+            await conn.execute('UPDATE billing_ledger SET reserved = reserved + ? WHERE task_id = ?',
+                               (extra, task_id))
            return True
-        except Exception:
-            conn.rollback()
-            return False
-        finally:
-            conn.close()
+    except Exception:
+        return False


-def ledger_get(task_id: str) -> dict[str, Any] | None:
-    rows = _q('SELECT * FROM billing_ledger WHERE task_id = ?', (str(task_id),))
+async def ledger_get(task_id: str) -> dict[str, Any] | None:
+    rows = await _q('SELECT * FROM billing_ledger WHERE task_id = ?', (str(task_id),))
    return dict(rows[0]) if rows else None


-def ledger_settle(task_id: str, delivered: int, final_status: str) -> int:
+async def ledger_settle(task_id: str, delivered: int, final_status: str) -> int:
    """任务计费结算（幂等）：按账本实扣与实际交付计算应退次数，原子退款并落终态。
    - delivered：本次任务实际交付给用户的数量（单曲/单集 0 或 1；批量为成功入包数）
    - 重复调用/重启后再次调用：账本已非 open 直接返回 0，不重复退款
@@ -143,58 +119,53 @@ def ledger_settle(task_id: str, delivered: int, final_status: str) -> int:
    返回实际退款次数（落余额的部分；-1/admin 场景为 0）。"""
    task_id = str(task_id)
    delivered = max(0, int(delivered))
-    with _DB_LOCK:
-        conn = get_conn()
-        try:
-            row = conn.execute('SELECT * FROM billing_ledger WHERE task_id = ?', (task_id,)).fetchone()
-            if not row or row['status'] != 'open':
-                return 0
-            refund = max(0, int(row['reserved']) - delivered)
-            # 退款与账本终态同事务落库：要么都成，要么都不成。
-            # 注意 WHERE 必须排除 admin/-1：向 -1（不限次）账户加次数会把 -1 哨兵值破坏成
-            # -1+N，向 admin 加无意义——因此任务运行期间账户升级为 -1 时退款不落余额，
-            # 账本 refunded 记录的是"按账本应付"口径（该用户已有无限次数，作废可接受）；
-            # 原注释"照样退"与实现相反，已更正为此口径。
-            if refund > 0:
-                conn.execute('UPDATE users SET duration_seconds = duration_seconds + ? '
-                             'WHERE id = ? AND role != \'admin\' AND duration_seconds != -1',
-                             (refund, row['user_id']))
-            conn.execute('UPDATE billing_ledger SET delivered = ?, refunded = ?, status = ?, '
-                         'settled_at = ? WHERE task_id = ?',
-                         (delivered, refund, str(final_status)[:20], int(time.time()), task_id))
-            conn.commit()
-            return refund
-        finally:
-            conn.close()
+    async with write_transaction() as conn:
+        row = await (await conn.execute('SELECT * FROM billing_ledger WHERE task_id = ?', (task_id,))).fetchone()
+        if not row or row['status'] != 'open':
+            return 0
+        refund = max(0, int(row['reserved']) - delivered)
+        # 退款与账本终态同事务落库：要么都成，要么都不成。
+        # 注意 WHERE 必须排除 admin/-1：向 -1（不限次）账户加次数会把 -1 哨兵值破坏成
+        # -1+N，向 admin 加无意义——因此任务运行期间账户升级为 -1 时退款不落余额，
+        # 账本 refunded 记录的是"按账本应付"口径（该用户已有无限次数，作废可接受）；
+        # 原注释"照样退"与实现相反，已更正为此口径。
+        if refund > 0:
+            await conn.execute('UPDATE users SET duration_seconds = duration_seconds + ? '
+                               'WHERE id = ? AND role != \'admin\' AND duration_seconds != -1',
+                               (refund, row['user_id']))
+        await conn.execute('UPDATE billing_ledger SET delivered = ?, refunded = ?, status = ?, '
+                           'settled_at = ? WHERE task_id = ?',
+                           (delivered, refund, str(final_status)[:20], int(time.time()), task_id))
+        return refund


-def ledger_recover_orphans(max_age_seconds: int = 2 * 3600) -> int:
+async def ledger_recover_orphans(max_age_seconds: int = 2 * 3600) -> int:
    """启动对账：任务表在内存，进程重启后账本中仍 open 的行代表
    "已扣款但任务已丢失"——退回全部实扣。仅处理超过宽限期的行，
    避免误伤同库其他存活进程的进行中任务。返回退款笔数。"""
    now = int(time.time())
    recovered = 0
-    rows = _q('SELECT task_id FROM billing_ledger WHERE status = \'open\' AND created_at <= ?',
-              (now - int(max_age_seconds),))
+    rows = await _q('SELECT task_id FROM billing_ledger WHERE status = \'open\' AND created_at <= ?',
+                    (now - int(max_age_seconds),))
    for r in rows:
        try:
-            refunded = ledger_settle(r['task_id'], 0, 'lost_restart')
+            refunded = await ledger_settle(r['task_id'], 0, 'lost_restart')
            if refunded:
-                add_log('success', 'system', 'system', '重启对账退款',
-                        f'任务 {r["task_id"]} 因进程重启丢失，退回 {refunded} 次')
+                await add_log('success', 'system', 'system', '重启对账退款',
+                              f'任务 {r["task_id"]} 因进程重启丢失，退回 {refunded} 次')
            else:
-                ledger_settle(r['task_id'], 0, 'lost_restart')
+                await ledger_settle(r['task_id'], 0, 'lost_restart')
            recovered += 1
        except Exception:
            pass
    return recovered


-def ledger_cleanup_settled(max_age_seconds: int = 90 * 86400) -> int:
+async def ledger_cleanup_settled(max_age_seconds: int = 90 * 86400) -> int:
    """清理已结算的历史账本行（每任务一行、只增不删会让表无限膨胀；
    settled 状态行无对账价值，保留 90 天足够追溯）"""
    try:
-        return _x('DELETE FROM billing_ledger WHERE status != \'open\' AND settled_at <= ?',
-                  (int(time.time()) - int(max_age_seconds),))
+        return await _x('DELETE FROM billing_ledger WHERE status != \'open\' AND settled_at <= ?',
+                        (int(time.time()) - int(max_age_seconds),))
    except Exception:
        return 0