refactor: 存档层从「一牧场一个 JSON」换成 SQLite(单库单表)

`saves/farm.db` 一张 farms 表,farm_id 主键,**一个牧场一行**。
接口签名不变(save_state / load_state / delete_save 只多了 farm_id 参数),
调用方只改了路径语义。

设计取舍:标量列 + 整份 payload。
* day/gold/level/ap/weather/forecast/net_worth 单独成列——外部工具或 AI
  一条 SELECT 就能排榜、画曲线,不必先把每份存档读进内存;
* 完整状态仍整份存 payload JSON:游戏数据隔三差五长新字段,规范化成
  几十张表意味着每加字段写一次迁移,而存档只做整体读、整体写,
  所以结构化列管可查询,完整性交给 JSON——加字段不用改表结构。

崩溃安全升级:写入包在事务里。原来的"临时文件 + os.replace"能挡进程
崩溃,挡不住写到一半掉电;SQLite 的日志能。读取失败照旧降级开新档。

向后兼容:SAVE_VERSION=2 / MIN_SAVE_VERSION=1 不变。老 JSON 存档由
migrate_json_saves() 在服务器启动时自动导入:同名已存在就跳过(宁可不
迁不覆盖),导入成功原文件改名 *.json.migrated 留底,先写库再改名
保证幂等;坏档一律跳过——迁移不许有能力阻止服务器启动。

异常边界:StorageError = sqlite3.Error,刻意不是 OSError 子类——
调用方原来的 except OSError 全部跟着改,否则写盘失败会穿透出去。

性能实测:PRAGMA journal_mode=WAL 是持久属性,建库时设一次就够,
每次连接重设白花 2.2ms;save_state 约 6.9ms/次,大头是 synchronous=FULL
的 fsync——不省,存档的价值就是不丢档。bench 热路径无回归。

顺手修一个真 bug:connect() 建表失败时连接会泄漏,Windows 上句柄没关
就删不掉文件,而调用方正是靠"读失败→开新档"容错的。补 close。

测试:StorageTestCase 重写(往返/版本/损坏降级/多牧场隔离/迁移),
test_server 断言 *.json 的用例改查库。513 项全过,ruff/pyright 全绿。
端到端:预置 v1/v2 老档起真服务器验 22 项——自动迁移、字段还原、
list_farms() 排榜、重启后进度分毫不差,全过。
This commit is contained in:
snowy-yang committed 2026-09-19 19:04:30 +08:00
1 parent 6268283fc6
commit bc3ae96a38
10 files changed
+563 -113

No files matched your search

+2 -1
View File
@@ -28,6 +28,7 @@ from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, Str
from .. import config
from ..engine import GameError
from ..storage import StorageError
from . import actions as actions_mod
from .hub import (
CLOSE_MESSAGE,
@@ -597,7 +598,7 @@ def _register_admin_routes(app: FastAPI, svc: _Services) -> None:
try:
session.save()
saved.append(session.farm_id)
except OSError as exc:
except (OSError, StorageError) as exc:
raise GameError(
"save_failed", f"{session.farm_id} 存档失败:{exc}", status=500
) from exc
+17 -7
View File
@@ -24,7 +24,13 @@ from typing import Any
from .. import config
from ..engine import FarmEngine, GameError, tick_decision
from ..storage import load_state, save_state
from ..storage import (
DB_NAME,
StorageError,
load_state,
migrate_json_saves,
save_state,
)
DEFAULT_FARM = "default"
MAX_FARM_ID = 24
@@ -68,7 +74,8 @@ class FarmSession:
self.farm_id = farm_id
self.owner = validate_owner(owner) or farm_id
self.save_dir = save_dir
self.path = os.path.join(save_dir, f"{farm_id}.json")
# 所有牧场共用同一个库文件,靠 farm_id 这行主键区分
self.db_path = os.path.join(save_dir, DB_NAME)
self._lock = threading.RLock()
self.engine = FarmEngine(tick_seconds=tick_seconds)
@@ -95,7 +102,7 @@ class FarmSession:
self.dirty = True
def load(self) -> str:
state, message = load_state(self.path)
state, message = load_state(self.db_path, self.farm_id)
if state is not None:
self.engine.state = state
self.load_message = message
@@ -109,7 +116,7 @@ class FarmSession:
def save(self) -> str:
with self._lock:
path = save_state(self.engine.state, self.path)
path = save_state(self.engine.state, self.db_path, self.farm_id)
self.saved_at = time.time()
return path
@@ -122,12 +129,12 @@ class FarmSession:
try:
self.save()
return True
except OSError:
except (OSError, StorageError):
return False
def shutdown(self) -> None:
# 停机时尽力落盘;写失败也不能挡住进程退出
with contextlib.suppress(OSError):
with contextlib.suppress(OSError, StorageError):
self.save()
def touch(self, count_action: bool = False) -> None:
@@ -203,6 +210,9 @@ class SessionManager:
self._lock = threading.RLock()
self._sessions: dict[str, FarmSession] = {}
os.makedirs(save_dir, exist_ok=True)
# 老版本是一牧场一个 JSON;第一次起来时把它们并进库里。
# 迁移失败不该挡住启动,所以这里只看结果、不抛。
self.migrated: list[str] = migrate_json_saves(save_dir)
# ------------------------------------------------------------ 取用
def get(self, farm_id: str, create: bool = True, owner: str = "") -> FarmSession:
@@ -270,7 +280,7 @@ class SessionManager:
continue
try:
session.shutdown()
except OSError:
except (OSError, StorageError):
continue # 存不下就别丢,宁可占着内存
with self._lock:
self._sessions.pop(session.farm_id, None)
+278 -39
View File
@@ -1,13 +1,28 @@
"""存档读写。
"""存档读写(SQLite)。
存档是纯 JSON,方便人工检查,也方便外部工具/AI 直接读取分析。
写入采用"临时文件 + 原子替换",避免中途崩溃导致存档损坏。
存档是一个 SQLite 库文件(默认 ``saves/farm.db``),**一个牧场一行**。
从"一牧场一个 JSON"换过来的理由:
* **崩溃安全**:写入包在事务里。原来的"临时文件 + ``os.replace``"能挡住
进程崩溃,挡不住写到一半掉电;SQLite 的日志能。
* **一个文件装下所有牧场**,不用再往目录里撒几十个小 JSON。
* **常用字段单独成列**(天数 / 金币 / 等级 / 天气 / 净值),外部工具或 AI
可以直接 ``SELECT`` 排个榜、画条曲线,不必先把每份存档读进内存。
* **完整状态仍然整份存在 ``payload`` 里**(JSON)。这是刻意的取舍:
游戏数据隔三差五就长出新字段,规范化成几十张表意味着每加一个字段都要写
一次迁移;而存档从来只做"整体读、整体写",不做部分查询。所以结构化列
只负责**可查询**,完整性交给 JSON——加字段不必改表结构。
老版本的 JSON 存档由 :func:`migrate_json_saves` 自动导入,原文件改名成
``*.json.migrated`` 留在原地(不删——那是玩家的真实进度)。
"""
from __future__ import annotations
import contextlib
import json
import os
import sqlite3
import time
from typing import Any
@@ -18,33 +33,135 @@ SAVE_VERSION = 2 # v2 起:谷仓存货 / 建筑 / 天气
# 一律走默认值,所以没必要因为升了一版就把玩家手里的档作废。
MIN_SAVE_VERSION = 1
DB_NAME = "farm.db"
# 单机模式只有一个牧场,固定用这个名字
DEFAULT_FARM_ID = "default"
# SQLite 在锁冲突时等多久(多牧场共用一个库,写操作是串行的)
BUSY_TIMEOUT = 10.0
# 存储层对外只暴露这一个异常基类:调用方不必认识 sqlite3,
# 换实现时也不用满仓库去改 except 子句。
# 注意它**不是** OSError 的子类——原来那几处 ``except OSError`` 要跟着改,
# 否则写盘失败会直接穿透出去(reap 里"存不下就别丢"的保护就失效了)。
StorageError = sqlite3.Error
_SCHEMA = """
CREATE TABLE IF NOT EXISTS farms (
farm_id TEXT PRIMARY KEY,
version INTEGER NOT NULL,
saved_at TEXT NOT NULL,
updated_at NUMERIC NOT NULL,
day INTEGER NOT NULL,
gold NUMERIC NOT NULL,
level INTEGER NOT NULL,
ap INTEGER NOT NULL,
weather TEXT NOT NULL,
forecast TEXT NOT NULL,
net_worth NUMERIC NOT NULL DEFAULT 0,
payload TEXT NOT NULL
);
"""
def default_save_dir() -> str:
"""默认存档目录:项目根目录下的 saves/。"""
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
return os.path.join(root, "saves")
def default_save_path() -> str:
"""默认存档位置:项目根目录下的 saves/save.json。"""
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
return os.path.join(root, "saves", "save.json")
"""默认存档库:项目根目录下的 saves/farm.db。"""
return os.path.join(default_save_dir(), DB_NAME)
def save_state(state: GameState, path: str | None = None) -> str:
"""把状态写入磁盘,返回实际写入路径。"""
def normalize_farm_id(raw: Any) -> str:
"""把牧场名规范化成主键;空值落到 default。
只做最基本的防御——名字能不能用是会话层(``validate_farm_id``)的事,
存储层不该反过来依赖它。
"""
if raw is None or raw == "":
return DEFAULT_FARM_ID
if isinstance(raw, (int, float)):
raw = str(raw)
if not isinstance(raw, str):
raise ValueError("牧场 ID 必须是字符串")
text = raw.strip()
return text or DEFAULT_FARM_ID
def connect(path: str | None = None) -> sqlite3.Connection:
"""打开(必要时创建)存档库。
每次调用都开一条新连接:存档是"每 60 秒写一次"的低频操作,开连接的开销
可以忽略,换来的是不用操心跨线程共享连接的问题。
"""
path = path or default_save_path()
directory = os.path.dirname(os.path.abspath(path))
os.makedirs(directory, exist_ok=True)
key = os.path.abspath(path)
parent = os.path.dirname(key)
os.makedirs(parent, exist_ok=True)
payload = {
"version": SAVE_VERSION,
"saved_at": time.strftime("%Y-%m-%d %H:%M:%S"),
"state": state.to_dict(),
}
# 注意:to_dict() 里的派生字段在读取时会被忽略,这里只取原始字段
payload["state"] = _raw_state(state)
# 空文件也算新库(比如刚被截断过)。WAL 是写在库文件头里的**持久**属性,
# 建库时设一次就够;每次连接都重设会白花约 2ms(实测),而存档在
# 每 60 秒一次的自动存盘路径上,能省就省。
fresh = not os.path.exists(key) or os.path.getsize(key) == 0
tmp_path = path + ".tmp"
with open(tmp_path, "w", encoding="utf-8") as handle:
json.dump(payload, handle, ensure_ascii=False, indent=2)
handle.flush()
os.fsync(handle.fileno())
os.replace(tmp_path, path)
conn = sqlite3.connect(key, timeout=BUSY_TIMEOUT)
try:
conn.row_factory = sqlite3.Row
if fresh:
conn.execute("PRAGMA journal_mode=WAL")
conn.executescript(_SCHEMA)
except Exception:
# 建表失败(比如这根本不是个 SQLite 文件)也要把句柄还回去。
# Windows 上文件句柄没关就删不掉,而调用方正是靠"读失败→开新档"来容错的。
conn.close()
raise
return conn
def save_state(state: GameState, path: str | None = None, farm_id: str = DEFAULT_FARM_ID) -> str:
"""把一个牧场写进库,返回库文件路径。"""
path = path or default_save_path()
key = normalize_farm_id(farm_id)
payload = json.dumps(_raw_state(state), ensure_ascii=False)
# ``with conn`` 是事务:要么整行更新,要么什么都没发生
with contextlib.closing(connect(path)) as conn, conn:
conn.execute(
"""
INSERT INTO farms
(farm_id, version, saved_at, updated_at, day, gold, level, ap,
weather, forecast, net_worth, payload)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(farm_id) DO UPDATE SET
version = excluded.version,
saved_at = excluded.saved_at,
updated_at = excluded.updated_at,
day = excluded.day,
gold = excluded.gold,
level = excluded.level,
ap = excluded.ap,
weather = excluded.weather,
forecast = excluded.forecast,
net_worth = excluded.net_worth,
payload = excluded.payload
""",
(
key,
SAVE_VERSION,
time.strftime("%Y-%m-%d %H:%M:%S"),
time.time(),
state.day,
state.gold,
state.level,
state.ap,
state.weather,
state.forecast,
state.net_worth,
payload,
),
)
return path
@@ -92,34 +209,42 @@ def _raw_state(state: GameState) -> dict[str, Any]:
}
def load_state(path: str | None = None) -> tuple[GameState | None, str]:
"""读取存档。
def load_state(
path: str | None = None, farm_id: str = DEFAULT_FARM_ID
) -> tuple[GameState | None, str]:
"""读取一个牧场的存档。
返回 ``(state, message)``;失败时 ``state`` 为 None 并给出原因,
调用方应当降级为"开新档"而不是崩溃。
"""
path = path or default_save_path()
key = normalize_farm_id(farm_id)
if not os.path.exists(path):
return None, f"未找到存档 {path}"
try:
with open(path, encoding="utf-8") as handle:
payload = json.load(handle)
except (OSError, json.JSONDecodeError) as exc:
return None, f"存档无法解析:{exc}"
with contextlib.closing(connect(path)) as conn:
row = conn.execute(
"SELECT version, payload FROM farms WHERE farm_id = ?", (key,)
).fetchone()
except sqlite3.DatabaseError as exc:
return None, f"存档无法读取:{exc}"
if not isinstance(payload, dict):
return None, "存档格式错误:根节点不是对象"
if row is None:
return None, f"存档里没有牧场「{key}」"
version = payload.get("version", 0)
version = row["version"]
if not isinstance(version, int) or not (MIN_SAVE_VERSION <= version <= SAVE_VERSION):
return None, (
f"存档版本不兼容(文件 v{version},当前支持 v{MIN_SAVE_VERSION}~v{SAVE_VERSION})"
)
raw = payload.get("state")
try:
raw = json.loads(row["payload"])
except (TypeError, json.JSONDecodeError) as exc:
return None, f"存档内容异常:{exc}"
if not isinstance(raw, dict):
return None, "存档缺少 state 字段"
return None, "存档格式错误:状态不是对象"
try:
state = GameState.from_dict(raw)
@@ -129,10 +254,124 @@ def load_state(path: str | None = None) -> tuple[GameState | None, str]:
return state, f"已载入存档(第 {state.day} 天)"
def delete_save(path: str | None = None) -> bool:
def delete_save(path: str | None = None, farm_id: str = DEFAULT_FARM_ID) -> bool:
"""删掉一个牧场,返回是否真的删到了。
只删那一行,**不删库文件**:同一台服务器上还有别的牧场,
而且连接开开关关,删文件容易留下悬空的 WAL。
"""
path = path or default_save_path()
try:
os.remove(path)
return True
except OSError:
key = normalize_farm_id(farm_id)
if not os.path.exists(path):
return False
try:
with contextlib.closing(connect(path)) as conn, conn:
cursor = conn.execute("DELETE FROM farms WHERE farm_id = ?", (key,))
return cursor.rowcount > 0
except sqlite3.DatabaseError:
return False
def list_farms(path: str | None = None) -> list[dict[str, Any]]:
"""列出库里的所有牧场(按净值降序)。
这是换成 SQLite 之后白拿的能力:一条 SQL 就能排榜、做巡检,
不用把每份存档都读进内存再算。
"""
path = path or default_save_path()
if not os.path.exists(path):
return []
try:
with contextlib.closing(connect(path)) as conn:
rows = conn.execute(
"""
SELECT farm_id, day, gold, level, ap, weather, forecast,
net_worth, saved_at
FROM farms
ORDER BY net_worth DESC, day DESC, farm_id
"""
).fetchall()
except sqlite3.DatabaseError:
return []
return [
{
"farm": row["farm_id"],
"day": row["day"],
"gold": row["gold"],
"level": row["level"],
"ap": row["ap"],
"weather": row["weather"],
"forecast": row["forecast"],
"net_worth": row["net_worth"],
"saved_at": row["saved_at"],
}
for row in rows
]
def migrate_json_saves(save_dir: str) -> list[str]:
"""把老版本的 JSON 存档导入数据库,返回成功导入的牧场名。
* 文件名(去掉 ``.json``)就是牧场名,与旧的"一牧场一个文件"约定一致;
* 新库里已经有同名牧场 → 跳过,并且**不动那个文件**(宁可不迁,也不覆盖);
* 导入成功后把文件改名成 ``<名字>.json.migrated`` 留在原地。
这里对每一份读不懂的档都是"跳过"而不是报错:迁移不该有能力阻止服务器
启动,坏档留给用户自己看。
"""
if not save_dir or not os.path.isdir(save_dir):
return []
db_path = os.path.join(save_dir, DB_NAME)
imported: list[str] = []
for entry in sorted(os.listdir(save_dir)):
if not entry.endswith(".json"):
continue
src = os.path.join(save_dir, entry)
if not os.path.isfile(src):
continue
farm_id = entry[: -len(".json")]
try:
with open(src, encoding="utf-8") as handle:
payload = json.load(handle)
except (OSError, json.JSONDecodeError):
continue
if not isinstance(payload, dict):
continue
version = payload.get("version", 0)
if not isinstance(version, int) or not (MIN_SAVE_VERSION <= version <= SAVE_VERSION):
continue
raw = payload.get("state")
if not isinstance(raw, dict):
continue
if farm_exists(db_path, farm_id):
continue
try:
state = GameState.from_dict(raw)
except (TypeError, ValueError, KeyError):
continue
save_state(state, db_path, farm_id)
# 先写库再改名:万一改名失败,下次启动还会重来一遍(幂等)
os.replace(src, src + ".migrated")
imported.append(normalize_farm_id(farm_id))
return imported
def farm_exists(path: str, farm_id: str) -> bool:
"""库里有没有这个牧场。库文件不存在时返回 False,且不创建它。"""
if not os.path.exists(path):
return False
try:
with contextlib.closing(connect(path)) as conn:
row = conn.execute(
"SELECT 1 FROM farms WHERE farm_id = ?", (normalize_farm_id(farm_id),)
).fetchone()
except sqlite3.DatabaseError:
return False
return row is not None