Archived
refactor: 本地文件 IO 归位——重活留线程、微秒级直接同步,移除零使用依赖 aiofiles
为什么:文件 IO 这条线上 to_thread 用得过宽。凡是「单文件 exists / unlink / stat」 这类微秒级元数据操作,线程调度开销(几十微秒起)比操作本身还大,套 to_thread 只是 给同步调用加了层异步外壳,不产生任何并发收益;真正需要卸载的是整库复制、ZIP 整段 拷贝、SQLite 探测这类活。本批按这个口径把两端都摆正,并顺手修掉一处真实缺陷。 改动: - web/routers/admin_site.py 备份上传:原 `content = await f.read()` 会把整库一次性 读进内存(端点对外无大小上限,等于给了上传方一个内存放大器),落盘后还要把文件 重新打开读一次头。改为 _save_upload_stream 按 1 MiB 逐块搬运——内存占用恒为一块, 文件头在写第一块时就地截获(省掉那趟回读)。仍属阻塞 IO,故保留在 to_thread 里。 - web/routers/admin_site.py backup_list:撤掉仅为一次 stat 而拆出的嵌套函数 + to_thread,直接在事件循环里做完。 - web/routers/admin.py:save_cookie / logout / _get_cookie_info 四处 to_thread 改 直接调用——它们读写 1KB 级的 cookie.txt,全程不碰网络。 - music_api/qr_client.py:logout() 的 print 改 logger。该方法既被 CLI 也被 admin 端点调用,在 web 语境里 print 会把内容直接喷到服务端 stdout,绕过 logging 配置、 无法被收集,也污染容器日志。 - auth_system/backup.py:三处单文件 unlink(占位文件回滚、删备份、裁剪 pre_restore_) 改直接同步;把只为配合 to_thread 而拆出的 _backup_path_sync 并回 backup_path。 保留在线程里的:整库 copy2 / os.replace、glob + N 次 stat、SQLite 探测、以及持 跨线程锁的 _reserve_backup_path(内联会让事件循环等一把别的线程可能持有的锁)。 - web/routers/delivery.py:两处 os.path.exists 改直接同步(带 ASYNC240 行内豁免, 与 core/db.py、auth_system/db.py 同一写法),顺带删掉随之未使用的 asyncio 导入。 - pyproject.toml / uv.lock:移除 aiofiles。它全仓零引用,且底层同样是 to_thread—— 对 1KB 的 cookie 文件不会带来任何真异步收益。 验证: - uv run ruff check . → All checks passed! - uv run pyright → 0 errors, 0 warnings, 0 informations - uv run pytest → 392 项,0 失败 / 0 error / 0 skipped (含 test_admin_backup_upload 四条:接受真实 SQLite / 缺字段拒 / 后缀拒 / 后缀对但内容非库必须拒且不留半成品) - data/site.db sha256 前后一致(a19f5dbf…6487),无 -wal/-shm 残留
This commit is contained in:
1 parent
046225e38d
commit
e8cc7cb34a
7 files changed
+76
-53
No files matched your search
+15
-10
@@ -2,7 +2,9 @@
|
||||
"""数据库备份、恢复与自动备份线程。
|
||||
|
||||
(自 auth_system.py 拆出,行为与原实现一致;DB 备份改用 aiosqlite 在线备份
|
||||
API,文件系统/复制等阻塞 IO 经 asyncio.to_thread 卸载。)
|
||||
API。文件系统操作的归位口径:整库复制/替换、glob + N 次 stat、SQLite 探测这类
|
||||
真活经 asyncio.to_thread 卸载;单文件 exists/unlink 是微秒级元数据操作,线程
|
||||
调度开销比操作本身还贵,直接在事件循环里做完。)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -76,9 +78,9 @@ async def create_backup() -> str:
|
||||
pass
|
||||
return name
|
||||
except Exception:
|
||||
# 备份失败:清掉占位文件,不留半个坏备份
|
||||
# 备份失败:清掉占位文件,不留半个坏备份(单文件 unlink 是微秒级操作,直接做)
|
||||
try:
|
||||
await asyncio.to_thread(dest_path.unlink, missing_ok=True)
|
||||
dest_path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
@@ -101,19 +103,22 @@ def _list_backups_sync() -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
async def list_backups() -> list[dict[str, Any]]:
|
||||
# glob + 每份备份一次 stat,份数可达数十:够不上「单次微秒级元数据操作」,
|
||||
# 仍留在线程里(BACKUP_DIR 可能是绑定挂载/网络盘,逐个 stat 会累积)
|
||||
return await asyncio.to_thread(_list_backups_sync)
|
||||
|
||||
|
||||
def _backup_path_sync(name: str) -> Path | None:
|
||||
async def backup_path(name: str) -> Path | None:
|
||||
"""备份文件路径解析(防目录穿越)。
|
||||
|
||||
单文件 exists + 后缀判定,微秒级元数据操作——直接同步做,线程调度开销比
|
||||
操作本身还大。(原先拆出 _backup_path_sync 只为配合 to_thread。)
|
||||
"""
|
||||
name = Path(name).name # 防目录穿越
|
||||
p = db.BACKUP_DIR / name
|
||||
return p if p.exists() and p.suffix == '.db' else None
|
||||
|
||||
|
||||
async def backup_path(name: str) -> Path | None:
|
||||
return await asyncio.to_thread(_backup_path_sync, name)
|
||||
|
||||
|
||||
def _backup_is_valid_db_sync(src: Path) -> bool:
|
||||
"""深度校验备份文件:SQLite 头 + 能真正打开 + 含 users 表,三者缺一不可"""
|
||||
try:
|
||||
@@ -210,7 +215,7 @@ async def delete_backup(name: str) -> bool:
|
||||
# 允许删除 pre_restore_ 安全副本(配额裁剪已保证至少保留
|
||||
# 最近 5 份;原实现一刀切拒删导致恢复频繁时无界累积且管理员手动也删不掉)
|
||||
if p:
|
||||
await asyncio.to_thread(p.unlink)
|
||||
p.unlink() # 单文件 unlink 是微秒级元数据操作,直接同步做(省一次线程跳转)
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -235,7 +240,7 @@ async def prune_backups(keep_normal: int = 20, keep_pre_restore: int = 5) -> Non
|
||||
pre = [b for b in await list_backups() if b['name'].startswith('pre_restore_')]
|
||||
for b in pre[keep_pre_restore:]:
|
||||
try:
|
||||
await asyncio.to_thread((db.BACKUP_DIR / b['name']).unlink)
|
||||
(db.BACKUP_DIR / b['name']).unlink() # 同上:单文件 unlink,直接同步
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
+10
-5
@@ -293,19 +293,24 @@ class QRLoginClient:
|
||||
print(f"获取登录信息失败: {e}")
|
||||
|
||||
def logout(self) -> bool:
|
||||
"""登出(清除 Cookie)"""
|
||||
"""登出(清除 Cookie)。
|
||||
|
||||
输出走 logger 而不是 print:本方法既被 CLI(main)调用,也被 admin 端点
|
||||
(/api/v1/admin/cookie/delete)调用——在 web 语境里 print 会把内容直接喷到
|
||||
服务端 stdout,绕过 logging 配置、无法被收集,也会污染容器日志。
|
||||
"""
|
||||
try:
|
||||
if self.cookie_file.exists():
|
||||
backup = self._backup_cookie("logout")
|
||||
if backup:
|
||||
print(f"Cookie 已备份到: {backup}")
|
||||
self.logger.info(f"Cookie 已备份到: {backup}")
|
||||
if self._clear_cookie():
|
||||
print("已成功登出")
|
||||
self.logger.info("已成功登出")
|
||||
return True
|
||||
print("登出失败")
|
||||
self.logger.error("登出失败")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"登出时发生错误: {e}")
|
||||
self.logger.error(f"登出时发生错误: {e}")
|
||||
return False
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ dependencies = [
|
||||
"uvicorn[standard]>=0.30,<1.0",
|
||||
"aiohttp>=3.10,<4.0",
|
||||
"aiosqlite>=0.20,<1.0",
|
||||
"aiofiles>=24.1",
|
||||
"itsdangerous>=2.2,<3.0",
|
||||
"cryptography>=50.0,<51.0",
|
||||
"qrcode>=7.0,<8.3",
|
||||
|
||||
@@ -2,15 +2,6 @@ version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.12"
|
||||
|
||||
[[package]]
|
||||
name = "aiofiles"
|
||||
version = "25.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/41/c3/534eac40372d8ee36ef40df62ec129bee4fdb5ad9706e58a29be53b2c970/aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2", size = 46354, upload-time = "2025-10-09T20:51:04.358Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aiohappyeyeballs"
|
||||
version = "2.7.1"
|
||||
@@ -1520,7 +1511,6 @@ name = "yunmusic-assistant"
|
||||
version = "3.0.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "aiofiles" },
|
||||
{ name = "aiohttp" },
|
||||
{ name = "aiosqlite" },
|
||||
{ name = "cryptography" },
|
||||
@@ -1545,7 +1535,6 @@ dev = [
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "aiofiles", specifier = ">=24.1" },
|
||||
{ name = "aiohttp", specifier = ">=3.10,<4.0" },
|
||||
{ name = "aiosqlite", specifier = ">=0.20,<1.0" },
|
||||
{ name = "cryptography", specifier = ">=50.0,<51.0" },
|
||||
|
||||
+10
-8
@@ -11,8 +11,10 @@
|
||||
鉴权:管理员密码闸(文案"需要管理员权限,请使用管理员账号登录"),
|
||||
与 auth_system.admin_required 的文案不同,故本模块用本地 admin 闸保持逐字对齐。
|
||||
|
||||
说明:QRLoginManager 已异步化(直接 await);QRLoginClient 的 save_cookie/logout/
|
||||
_get_cookie_info 仍为同步文件操作,经 asyncio.to_thread 执行。
|
||||
说明:QRLoginManager 已异步化(直接 await)。QRLoginClient 的 save_cookie/logout/
|
||||
_get_cookie_info 是**微秒级本地文件操作**——读写 1KB 级的 cookie.txt、os.replace、
|
||||
unlink,全程不碰网络;对它们走 to_thread,线程调度开销比操作本身还大,且并不产生
|
||||
任何并发收益。故直接同步调用(to_thread 在这里只是给同步 API 套了层异步外壳)。
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
@@ -111,12 +113,12 @@ async def admin_qr_check(_admin: AdminUser, request: Request):
|
||||
cookie = f'MUSIC_U={musu_u};os=pc;appver=8.9.70;'
|
||||
if cookie_dict.get('__csrf'):
|
||||
cookie += f"__csrf={cookie_dict['__csrf']};"
|
||||
saved = await asyncio.to_thread(client.save_cookie, cookie)
|
||||
saved = client.save_cookie(cookie)
|
||||
if not saved:
|
||||
return error('登录成功但 Cookie 保存失败', 500)
|
||||
_reset_caches()
|
||||
logger.info('二维码登录成功,Cookie 已自动保存')
|
||||
info = await asyncio.to_thread(client._get_cookie_info)
|
||||
info = client._get_cookie_info()
|
||||
return success({'status': 'success', 'cookie_info': info}, '登录成功,Cookie 已保存')
|
||||
if code == 801:
|
||||
return success({'status': 'waiting'}, '等待扫码')
|
||||
@@ -133,7 +135,7 @@ async def admin_cookie_status(_admin: AdminUser, request: Request):
|
||||
await ratelimit.guard(request, bucket='free', limit=30, window=60)
|
||||
try:
|
||||
client = _admin_client()
|
||||
info = await asyncio.to_thread(client._get_cookie_info)
|
||||
info = client._get_cookie_info()
|
||||
return success(info, '获取 Cookie 状态成功')
|
||||
except Exception:
|
||||
return error('服务暂时不可用,请稍后再试', 500)
|
||||
@@ -145,15 +147,15 @@ async def admin_cookie_delete(_admin: AdminUser, request: Request):
|
||||
await ratelimit.guard(request, bucket='free', limit=30, window=60)
|
||||
try:
|
||||
client = _admin_client()
|
||||
ok = await asyncio.to_thread(client.logout)
|
||||
ok = client.logout()
|
||||
if not ok:
|
||||
info = await asyncio.to_thread(client._get_cookie_info)
|
||||
info = client._get_cookie_info()
|
||||
return error(
|
||||
f"Cookie 清除失败(文件仍存在:{info.get('file_exists')},"
|
||||
f"可能被占用或为挂载点,请手动删除)", 500)
|
||||
_reset_caches()
|
||||
logger.info('管理员已清除 Cookie')
|
||||
info = await asyncio.to_thread(client._get_cookie_info)
|
||||
info = client._get_cookie_info()
|
||||
return success(info, 'Cookie 已清除')
|
||||
except Exception:
|
||||
return error('服务暂时不可用,请稍后再试', 500)
|
||||
|
||||
+37
-15
@@ -3,16 +3,20 @@
|
||||
|
||||
鉴权:与 auth_system.admin_required 同口径(403 + error_code='forbidden')。
|
||||
|
||||
说明:这些端点多为对 auth_system 的薄封装。auth_system 已迁移为**原生异步**
|
||||
(aiosqlite / aiosmtplib 语义下的 to_thread 卸载),故直接 await;仅 PBKDF2 校验
|
||||
(CPU 密集)与本地文件 stat 仍经 asyncio.to_thread 卸载。操作者名取当前管理员用户名。
|
||||
说明:这些端点多为对 auth_system 的薄封装。auth_system 已迁移为**原生异步**,
|
||||
故直接 await。PBKDF2 校验属 CPU 密集且 CPython 在迭代里**释放 GIL**,是这里唯一
|
||||
真正值得占用线程的活(唯有 to_thread 下它才是真并行)。本地文件操作的归位口径是
|
||||
「重活留线程、微秒级直接同步」:备份上传是整库体量,按块流式落盘、仍走 to_thread;
|
||||
stat / unlink 这类元数据操作,线程调度开销比操作本身还大,直接在事件循环里做完。
|
||||
操作者名取当前管理员用户名。
|
||||
"""
|
||||
import asyncio
|
||||
import math
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
from typing import Annotated, Any
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any, BinaryIO
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import FileResponse
|
||||
@@ -28,6 +32,29 @@ router = APIRouter()
|
||||
_RESERVED_USERNAMES = {'admin', 'administrator', 'qfbsh', 'root', 'system', 'moderator', '管理员', '客服'}
|
||||
_EMAIL_RE = re.compile(r'^[\w.%+-]+@[\w.-]+\.[A-Za-z]{2,}$')
|
||||
|
||||
# 备份上传的搬运块大小。上传的是**整库备份**,原实现 `content = await f.read()`
|
||||
# 会把整库一次性读进内存(本端点对外没有大小上限),等于给上传方一个内存放大器;
|
||||
# 改成逐块搬运后,内存占用恒为一块。
|
||||
_UPLOAD_CHUNK = 1 << 20
|
||||
|
||||
_SQLITE_MAGIC = b'SQLite format 3'
|
||||
|
||||
|
||||
def _save_upload_stream(src: BinaryIO, dest: Path) -> bytes:
|
||||
"""把上传流按块写进 dest,并返回文件头(前 16 字节)。
|
||||
|
||||
**阻塞文件 IO**:只应由 asyncio.to_thread 调用。文件头在写第一块时就地截获,
|
||||
省掉「落盘后再把文件重新打开读一次」的那趟回读。
|
||||
"""
|
||||
src.seek(0)
|
||||
header = b''
|
||||
with dest.open('wb') as out:
|
||||
while chunk := src.read(_UPLOAD_CHUNK):
|
||||
if len(header) < 16:
|
||||
header += chunk[:16 - len(header)]
|
||||
out.write(chunk)
|
||||
return header
|
||||
|
||||
|
||||
async def _require_admin(user: Annotated[SiteUser | None, Depends(current_user)]) -> SiteUser:
|
||||
if user is None or not user.is_admin:
|
||||
@@ -442,12 +469,9 @@ async def backup_create(admin: AdminUser):
|
||||
async def backup_list(_admin: AdminUser):
|
||||
backups = await auth.list_backups()
|
||||
interval = await auth.get_setting('backup_interval_hours', '0')
|
||||
|
||||
def _size() -> int:
|
||||
return auth.db.DB_PATH.stat().st_size if auth.db.DB_PATH.exists() else 0
|
||||
|
||||
return success({'backups': backups, 'interval_hours': interval,
|
||||
'db_size': await asyncio.to_thread(_size)})
|
||||
# 主库文件的 stat:微秒级元数据操作,线程调度开销比它本身还大,直接同步做
|
||||
db_size = auth.db.DB_PATH.stat().st_size if auth.db.DB_PATH.exists() else 0
|
||||
return success({'backups': backups, 'interval_hours': interval, 'db_size': db_size})
|
||||
|
||||
|
||||
@router.get('/admin/site/backup/download')
|
||||
@@ -482,11 +506,9 @@ async def backup_upload(admin: AdminUser, request: Request):
|
||||
name = f'{base}-{k}.db'
|
||||
k += 1
|
||||
dest = auth.db.BACKUP_DIR / name
|
||||
content = await f.read()
|
||||
await asyncio.to_thread(dest.write_bytes, content)
|
||||
with dest.open('rb') as hf:
|
||||
header = hf.read(16)
|
||||
if not header.startswith(b'SQLite format 3'):
|
||||
# 流式落盘:整库体量逐块搬运,文件头随第一块一起截获(不再回读一遍)
|
||||
header = await asyncio.to_thread(_save_upload_stream, f.file, dest)
|
||||
if not header.startswith(_SQLITE_MAGIC):
|
||||
dest.unlink()
|
||||
return error('不是有效的 SQLite 数据库文件')
|
||||
await _log(admin, '上传数据库', dest.name)
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
X-Download-Filename(URL 编码,供前端取回原始文件名)
|
||||
- 文件清理仍由监控线程按 35 分钟保留期统一回收(交付请求不删文件)
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
from typing import Annotated, Any
|
||||
from urllib.parse import quote
|
||||
@@ -67,7 +66,8 @@ async def download_result(user: LoginUser, request: Request):
|
||||
|
||||
result = task.get('result') or {}
|
||||
target = result.get('zip_path') or result.get('file_path')
|
||||
target_exists = bool(target) and await asyncio.to_thread(os.path.exists, target)
|
||||
# 微秒级单文件存在性判定,直接同步做(线程调度开销比它本身还大)
|
||||
target_exists = bool(target) and os.path.exists(target) # noqa: ASYNC240
|
||||
if status in ('canceled', 'error') and not target_exists:
|
||||
return error(task.get('msg') or ('任务已取消' if status == 'canceled' else '任务失败'),
|
||||
200 if status == 'canceled' else 500)
|
||||
@@ -95,7 +95,8 @@ async def download_batch_zip_result(user: LoginUser, request: Request):
|
||||
status = task.get('status')
|
||||
result = task.get('result') or {}
|
||||
zip_path = result.get('zip_path')
|
||||
has_deliverable = bool(zip_path) and await asyncio.to_thread(os.path.exists, zip_path)
|
||||
# 同上:微秒级单文件存在性判定
|
||||
has_deliverable = bool(zip_path) and os.path.exists(zip_path) # noqa: ASYNC240
|
||||
if status in ('queued', 'running'):
|
||||
return error('任务还在进行中,请稍后再试', 202)
|
||||
if status == 'paused':
|
||||
|
||||
Reference in new issue
Block a user