Archived
chore(async-infra): 异步基础层 + 工具链 + 清理
异步基础层（core/）：
- db.py aiosqlite + WAL：读路径无锁并发、写路径 asyncio.Lock 串行，
busy_timeout 自旋——实现「读写同时进行」（旧实现用 RLock 全读串行）
- http_client.py aiohttp 连接池 + 有界重试（5xx 重试 / 4xx 不重试）+
响应体上限 + 流式接口；按事件循环隔离 session（支持迁移期双 loop）
工具链：
- pyproject.toml ruff(E/F/W/I/B/UP/C4/ASYNC/S/RUF) + pyright(basic) + pytest(asyncio auto)
- requirements-fastapi.txt FastAPI/aiohttp/aiosqlite/aiofiles 等目标栈
- requirements-dev.txt 开发依赖（pytest/ruff/pyright）
清理：删除无用脚本 _csscheck.py 与产物标记 _css_ok.txt/_pkg_ok.txt，同步 .gitignore
验证：pytest 10/10（含「写 200 条同时 4 读协程零异常」并发用例）；
ruff All checks passed；pyright 0 errors。
This commit is contained in:
snowy-yang committed
1 parent
41356e89c4
commit
9f56f68b66
16 files changed
+706
-42
No files matched your search
@@ -0,0 +1,79 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""异步 SQLite 层测试：读写并发、WAL、事务原子性。"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
import core.db as db
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def temp_db(tmp_path, monkeypatch):
|
||||
"""把 DB 指向临时目录，独立于生产 data/site.db。"""
|
||||
monkeypatch.setattr(db, 'DATA_DIR', tmp_path)
|
||||
monkeypatch.setattr(db, 'DB_PATH', tmp_path / 'site.db')
|
||||
return tmp_path / 'site.db'
|
||||
|
||||
|
||||
async def test_write_then_read(temp_db):
|
||||
await db.execute('CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)')
|
||||
rid = await db.execute('INSERT INTO t (v) VALUES (?)', ('hello',))
|
||||
assert rid == 1
|
||||
row = await db.fetch_one('SELECT v FROM t WHERE id=?', (1,))
|
||||
assert row['v'] == 'hello'
|
||||
assert await db.fetch_val('SELECT COUNT(*) FROM t') == 1
|
||||
|
||||
|
||||
async def test_wal_mode_enabled(temp_db):
|
||||
await db.execute('CREATE TABLE t (id INTEGER PRIMARY KEY)')
|
||||
mode = await db.fetch_val('PRAGMA journal_mode')
|
||||
assert str(mode).lower() == 'wal'
|
||||
|
||||
|
||||
async def test_concurrent_read_write(temp_db):
|
||||
"""核心诉求：写的同时读不被阻塞、且读到一致快照（WAL 快照隔离）。"""
|
||||
await db.execute('CREATE TABLE nums (id INTEGER PRIMARY KEY, v INTEGER)')
|
||||
|
||||
stop = asyncio.Event()
|
||||
read_errors = []
|
||||
write_count = 0
|
||||
|
||||
async def writer():
|
||||
nonlocal write_count
|
||||
for i in range(200):
|
||||
await db.execute('INSERT INTO nums (v) VALUES (?)', (i,))
|
||||
write_count += 1
|
||||
stop.set()
|
||||
|
||||
async def reader():
|
||||
# 写入过程中持续读，不应抛 SQLITE_BUSY / 不应被阻塞致死
|
||||
while not stop.is_set():
|
||||
try:
|
||||
n = await db.fetch_val('SELECT COUNT(*) FROM nums')
|
||||
assert isinstance(n, int)
|
||||
except Exception as e:
|
||||
read_errors.append(e)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
await asyncio.gather(writer(), *[reader() for _ in range(4)])
|
||||
|
||||
assert not read_errors, f'读期间出现异常: {read_errors[:3]}'
|
||||
assert write_count == 200
|
||||
assert await db.fetch_val('SELECT COUNT(*) FROM nums') == 200
|
||||
|
||||
|
||||
async def test_write_transaction_commit_and_rollback(temp_db):
|
||||
await db.execute('CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)')
|
||||
|
||||
async with db.write_transaction() as conn:
|
||||
await conn.execute('INSERT INTO t (v) VALUES (?)', ('a',))
|
||||
await conn.execute('INSERT INTO t (v) VALUES (?)', ('b',))
|
||||
assert await db.fetch_val('SELECT COUNT(*) FROM t') == 2
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
async with db.write_transaction() as conn:
|
||||
await conn.execute('INSERT INTO t (v) VALUES (?)', ('c',))
|
||||
raise RuntimeError('boom')
|
||||
# 回滚：c 不应落库
|
||||
assert await db.fetch_val('SELECT COUNT(*) FROM t') == 2
|
||||
@@ -0,0 +1,113 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""异步 HTTP 客户端测试：连接复用、有界重试、4xx 不重试、响应体上限。"""
|
||||
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
|
||||
from core.http_client import AsyncHTTPClient, UpstreamHTTPError, UpstreamTooLarge
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
async def server():
|
||||
"""本地 aiohttp 测试服务器。"""
|
||||
hits = {'flaky': 0}
|
||||
|
||||
async def ok(request):
|
||||
return web.json_response({'ok': True})
|
||||
|
||||
async def flaky(request):
|
||||
# 首次 500，之后 200 —— 验证 5xx 退避重试
|
||||
hits['flaky'] += 1
|
||||
if hits['flaky'] == 1:
|
||||
return web.Response(status=500, text='boom')
|
||||
return web.json_response({'retried': True})
|
||||
|
||||
async def notfound(request):
|
||||
return web.Response(status=404, text='nope')
|
||||
|
||||
async def big(request):
|
||||
return web.Response(body=b'x' * (2 * 1024 * 1024))
|
||||
|
||||
app = web.Application()
|
||||
app.router.add_get('/ok', ok)
|
||||
app.router.add_get('/flaky', flaky)
|
||||
app.router.add_get('/notfound', notfound)
|
||||
app.router.add_get('/big', big)
|
||||
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, '127.0.0.1', 0)
|
||||
await site.start()
|
||||
port = site._server.sockets[0].getsockname()[1]
|
||||
base = f'http://127.0.0.1:{port}'
|
||||
try:
|
||||
yield base, hits
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
|
||||
|
||||
async def test_get_json(server):
|
||||
base, _ = server
|
||||
c = AsyncHTTPClient()
|
||||
try:
|
||||
assert await c.get_json(base + '/ok') == {'ok': True}
|
||||
finally:
|
||||
await c.close()
|
||||
|
||||
|
||||
async def test_retry_on_5xx(server):
|
||||
base, hits = server
|
||||
c = AsyncHTTPClient()
|
||||
try:
|
||||
data = await c.get_json(base + '/flaky')
|
||||
assert data == {'retried': True}
|
||||
assert hits['flaky'] == 2 # 第一次 500 + 重试成功
|
||||
finally:
|
||||
await c.close()
|
||||
|
||||
|
||||
async def test_no_retry_on_4xx(server):
|
||||
base, _ = server
|
||||
c = AsyncHTTPClient()
|
||||
try:
|
||||
with pytest.raises(UpstreamHTTPError) as ei:
|
||||
await c.get_text(base + '/notfound')
|
||||
assert ei.value.status == 404
|
||||
finally:
|
||||
await c.close()
|
||||
|
||||
|
||||
async def test_response_size_limit(server):
|
||||
base, _ = server
|
||||
c = AsyncHTTPClient(max_bytes=1024 * 1024) # 1MB 上限，服务端返回 2MB
|
||||
try:
|
||||
with pytest.raises(UpstreamTooLarge):
|
||||
await c.get_text(base + '/big')
|
||||
finally:
|
||||
await c.close()
|
||||
|
||||
|
||||
async def test_session_reuse(server):
|
||||
"""同一客户端多次请求复用同一（按 loop 隔离的）session —— 连接池生效。"""
|
||||
base, _ = server
|
||||
c = AsyncHTTPClient()
|
||||
try:
|
||||
await c.get_json(base + '/ok')
|
||||
s1 = await c._session_for_loop()
|
||||
await c.get_json(base + '/ok')
|
||||
s2 = await c._session_for_loop()
|
||||
assert s1 is s2 and not s1.closed
|
||||
finally:
|
||||
await c.close()
|
||||
|
||||
|
||||
async def test_stream(server):
|
||||
base, _ = server
|
||||
c = AsyncHTTPClient()
|
||||
try:
|
||||
async with c.stream('GET', base + '/ok') as resp:
|
||||
assert resp.status == 200
|
||||
body = await resp.read()
|
||||
assert b'ok' in body
|
||||
finally:
|
||||
await c.close()
|
||||
Reference in new issue
Block a user