feat: strengthen backup and mobile workflows
ci / gitleaks (push) Successful in 1m19s
ci / docker (push) Successful in 5m48s

This commit is contained in:
2026-09-16 21:12:52 +08:00
parent 6f38190c92
commit 6c234d7d82
64 changed files with 5059 additions and 635 deletions
+3
View File
@@ -0,0 +1,3 @@
from .router import router
__all__ = ["router"]
+195
View File
@@ -0,0 +1,195 @@
from __future__ import annotations
import hashlib
import json
import shutil
import tempfile
import zipfile
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from uuid import UUID
from fastapi import HTTPException
MAX_ARCHIVE_BYTES = 256 * 1024 * 1024
MAX_ENTRIES = 10_000
MAX_COMPRESSION_RATIO = 200
MAX_METADATA_ENTRY_BYTES = 16 * 1024 * 1024
_CHUNK_SIZE = 1024 * 1024
@dataclass(frozen=True)
class StagedBlob:
archive_path: str
staging_path: Path
size: int
sha256: str
@dataclass(frozen=True)
class ParsedArchive:
backup_id: UUID
archive_sha256: str
entities: dict[str, list[dict]]
files: dict[str, StagedBlob]
staging_dir: Path
def backup_error(code: str, message: str, status_code: int = 422) -> HTTPException:
return HTTPException(status_code, {"code": code, "message": message})
def canonical_json(value) -> bytes:
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
def build_archive_to_path(
output_path: Path, backup_id: UUID, entities: dict[str, list[dict]], files: dict[str, Path]
) -> None:
entity_entries = {f"data/{name}.json": canonical_json(rows) for name, rows in entities.items()}
checksums = {name: hashlib.sha256(data).hexdigest() for name, data in entity_entries.items()}
for name, path in files.items():
digest = hashlib.sha256()
with path.open("rb") as source:
while chunk := source.read(_CHUNK_SIZE):
digest.update(chunk)
checksums[name] = digest.hexdigest()
manifest = {
"format": "dodo-backup", "version": 2, "backup_id": str(backup_id),
"entities": {name: len(rows) for name, rows in entities.items()}, "checksums": checksums,
}
with zipfile.ZipFile(output_path, "w", compression=zipfile.ZIP_DEFLATED, allowZip64=True) as archive:
archive.writestr("manifest.json", canonical_json(manifest))
for name, data in entity_entries.items():
archive.writestr(name, data)
for name, path in files.items():
archive.write(path, name)
def build_archive(backup_id: UUID, entities: dict[str, list[dict]], files: dict[str, bytes]) -> bytes:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
file_paths = {}
for index, (name, data) in enumerate(files.items()):
path = root / str(index)
path.write_bytes(data)
file_paths[name] = path
output = root / "backup.zip"
build_archive_to_path(output, backup_id, entities, file_paths)
return output.read_bytes()
def _safe_name(name: str) -> bool:
path = PurePosixPath(name)
return bool(name) and not name.startswith("/") and "\\" not in name and ".." not in path.parts
def _read_metadata(archive: zipfile.ZipFile, info: zipfile.ZipInfo) -> bytes:
if info.file_size > MAX_METADATA_ENTRY_BYTES:
raise backup_error("backup_size_invalid", "备份元数据过大")
with archive.open(info) as source:
data = source.read(MAX_METADATA_ENTRY_BYTES + 1)
if len(data) > MAX_METADATA_ENTRY_BYTES:
raise backup_error("backup_size_invalid", "备份元数据过大")
return data
def _stream_blob(archive: zipfile.ZipFile, info: zipfile.ZipInfo, destination: Path) -> tuple[int, str]:
digest = hashlib.sha256()
size = 0
with archive.open(info) as source, destination.open("xb") as output:
while chunk := source.read(_CHUNK_SIZE):
size += len(chunk)
digest.update(chunk)
output.write(chunk)
return size, digest.hexdigest()
def parse_archive_path(path: Path, *, max_archive_bytes: int = MAX_ARCHIVE_BYTES) -> ParsedArchive:
size = path.stat().st_size
if size <= 0 or size > max_archive_bytes:
raise backup_error("backup_size_invalid", "备份文件大小无效")
digest = hashlib.sha256()
with path.open("rb") as source:
while chunk := source.read(_CHUNK_SIZE):
digest.update(chunk)
staging_dir = Path(tempfile.mkdtemp(prefix="dodo-backup-blobs-"))
try:
with zipfile.ZipFile(path) as archive:
infos = archive.infolist()
names = [item.filename for item in infos]
if len(infos) > MAX_ENTRIES:
raise backup_error("backup_too_many_entries", "备份条目过多")
if len(names) != len(set(names)):
raise backup_error("backup_duplicate_entry", "备份包含重复条目")
if any(not _safe_name(item.filename) or item.is_dir() or item.flag_bits & 1 for item in infos):
raise backup_error("backup_unsafe_path", "备份包含不安全路径")
if "manifest.json" not in names:
raise backup_error("backup_manifest_missing", "备份缺少 manifest")
if sum(item.file_size for item in infos) > max_archive_bytes:
raise backup_error("backup_size_invalid", "备份解压后过大")
if any(item.file_size and item.compress_size == 0 for item in infos):
raise backup_error("backup_compression_invalid", "备份压缩比异常")
if any(item.compress_size and item.file_size / item.compress_size > MAX_COMPRESSION_RATIO for item in infos):
raise backup_error("backup_compression_invalid", "备份压缩比异常")
by_name = {item.filename: item for item in infos}
manifest = json.loads(_read_metadata(archive, by_name["manifest.json"]))
if manifest.get("format") != "dodo-backup" or manifest.get("version") != 2:
raise backup_error("backup_version_unsupported", "不支持的备份版本")
backup_id = UUID(manifest["backup_id"])
checksums = manifest["checksums"]
declared_entities = manifest["entities"]
if not isinstance(checksums, dict) or not isinstance(declared_entities, dict):
raise TypeError
content_names = set(names) - {"manifest.json"}
if set(checksums) != content_names:
raise backup_error("backup_manifest_mismatch", "manifest 与 ZIP 条目不一致")
entities: dict[str, list[dict]] = {}
files: dict[str, StagedBlob] = {}
for index, name in enumerate(sorted(content_names)):
info = by_name[name]
if name.startswith("data/") and name.endswith(".json"):
data = _read_metadata(archive, info)
actual_digest = hashlib.sha256(data).hexdigest()
value = json.loads(data)
if not isinstance(value, list) or any(not isinstance(row, dict) for row in value):
raise ValueError
entities[name[5:-5]] = value
else:
blob_path = staging_dir / str(index)
blob_size, actual_digest = _stream_blob(archive, info, blob_path)
files[name] = StagedBlob(name, blob_path, blob_size, actual_digest)
if not isinstance(checksums[name], str) or actual_digest != checksums[name]:
raise backup_error("backup_checksum_mismatch", "备份校验和不匹配")
if set(declared_entities) != set(entities):
raise backup_error("backup_manifest_mismatch", "manifest 实体清单不一致")
if any(type(count) is not int or count < 0 or count != len(entities[name]) for name, count in declared_entities.items()):
raise backup_error("backup_manifest_mismatch", "manifest 实体数量不一致")
except HTTPException:
shutil.rmtree(staging_dir, ignore_errors=True)
raise
except (zipfile.BadZipFile, OSError) as exc:
shutil.rmtree(staging_dir, ignore_errors=True)
raise backup_error("backup_invalid_zip", "无效的 ZIP 备份") from exc
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
shutil.rmtree(staging_dir, ignore_errors=True)
raise backup_error("backup_manifest_invalid", "无效的备份 manifest 或数据") from exc
return ParsedArchive(backup_id, digest.hexdigest(), entities, files, staging_dir)
def parse_archive(content: bytes) -> ParsedArchive:
if not content or len(content) > MAX_ARCHIVE_BYTES:
raise backup_error("backup_size_invalid", "备份文件大小无效")
with tempfile.NamedTemporaryFile() as staged:
staged.write(content)
staged.flush()
return parse_archive_path(Path(staged.name))
+5
View File
@@ -0,0 +1,5 @@
"""Legacy JSON/CSV backup compatibility remains in backend.mvp.
The v2 ZIP implementation is isolated in this package. This marker module documents
that the old endpoints intentionally remain available during the gradual migration.
"""
+274
View File
@@ -0,0 +1,274 @@
from __future__ import annotations
import hashlib
import secrets
import shutil
from datetime import timedelta
from pathlib import Path
from fastapi import APIRouter, Depends, File, Query, UploadFile
from fastapi.responses import FileResponse
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession
from starlette.background import BackgroundTask
from backend.auth import current_user, hash_token
from backend.config import get_settings
from backend.db import get_db
from backend.models import BackupPreflight, Countdown, User, new_id, utcnow
from .archive import MAX_ARCHIVE_BYTES as ARCHIVE_DEFAULT_LIMIT
from .archive import backup_error, parse_archive_path
from .schemas import RestoreRequest
from .service import BackupRepairPending, export_v2, restore_v2, validate_archive
router = APIRouter(prefix="/api/v1/backup", tags=["backup"])
MAX_ARCHIVE_BYTES = ARCHIVE_DEFAULT_LIMIT
MAX_PENDING_PREFLIGHTS_PER_USER = 3
_READ_CHUNK = 1024 * 1024
def _staging_root() -> Path:
root = Path(get_settings().backup_staging_dir).resolve()
root.mkdir(parents=True, exist_ok=True)
return root
def _safe_staging_path(value: str) -> Path:
root = _staging_root()
path = Path(value).resolve()
if path.parent != root:
raise backup_error("backup_preflight_invalid", "预检暂存无效", 409)
return path
def _path_size(path: str | None) -> int:
if not path:
return 0
root = Path(path)
if not root.exists():
return 0
if root.is_file():
return root.stat().st_size
return sum(item.stat().st_size for item in root.rglob("*") if item.is_file())
async def _prune(db: AsyncSession) -> None:
now = utcnow()
rows = list((await db.scalars(select(BackupPreflight).where(
BackupPreflight.expires_at <= now,
BackupPreflight.status.in_((
"reserved", "pending", "failed", "consuming", "cleanup_pending",
"repair_pending",
)),
).with_for_update())).all())
from .storage import remove_quarantine, restore_quarantine_dir
attachment_root = Path(get_settings().attachment_dir).resolve()
for row in rows:
try:
if row.status == "repair_pending":
if row.cleanup_path:
restore_quarantine_dir(attachment_root, Path(row.cleanup_path))
row.cleanup_path = None
_safe_staging_path(row.staging_path).unlink(missing_ok=True)
row.status = "failed"
continue
if row.cleanup_path:
remove_quarantine(Path(row.cleanup_path))
row.cleanup_path = None
_safe_staging_path(row.staging_path).unlink(missing_ok=True)
except (OSError, ValueError):
continue
if row.status == "cleanup_pending":
row.status = "consumed"
row.consumed_at = row.consumed_at or now
else:
await db.delete(row)
if rows:
await db.commit()
@router.get("/export.zip")
async def export_zip(user: User = Depends(current_user), db: AsyncSession = Depends(get_db)):
path = await export_v2(db, user)
await db.commit()
return FileResponse(
path,
media_type="application/zip",
filename="dodo-backup-v2.zip",
background=BackgroundTask(path.unlink, missing_ok=True),
)
@router.post("/preflight")
async def preflight(
mode: str = Query(pattern="^(merge|replace)$"),
file: UploadFile = File(...),
user: User = Depends(current_user),
db: AsyncSession = Depends(get_db),
):
await _prune(db)
settings = get_settings()
configured_limit = settings.backup_max_archive_mb * 1024 * 1024
limit = min(MAX_ARCHIVE_BYTES, configured_limit)
max_pending = min(MAX_PENDING_PREFLIGHTS_PER_USER, settings.backup_max_pending_per_user)
max_staged = settings.backup_max_staged_mb_per_user * 1024 * 1024
token = secrets.token_urlsafe(32)
staging = _staging_root() / f"{new_id()}.zip"
reservation = BackupPreflight(
token_hash=hash_token(token), user_id=user.id, backup_id=new_id(),
archive_sha256="0" * 64, archive_size=limit, staging_path=str(staging), mode=mode,
status="reserved", expires_at=utcnow() + timedelta(seconds=settings.backup_preflight_ttl_seconds),
)
# A user-row write serializes quota decisions across workers on PostgreSQL;
# SQLite serializes writers at the database level.
await db.execute(update(User).where(User.id == user.id).values(username=User.username))
pending_count, pending_bytes = (await db.execute(select(
func.count(BackupPreflight.id), func.coalesce(func.sum(BackupPreflight.archive_size), 0)
).where(
BackupPreflight.user_id == user.id,
BackupPreflight.status.in_((
"reserved", "pending", "failed", "consuming", "cleanup_pending",
"repair_pending",
)),
))).one()
pending_rows = list((await db.scalars(select(BackupPreflight).where(
BackupPreflight.user_id == user.id,
BackupPreflight.status.in_((
"reserved", "pending", "failed", "consuming", "cleanup_pending",
"repair_pending",
)),
))).all())
pending_bytes += sum(_path_size(item.cleanup_path) for item in pending_rows)
if pending_count >= max_pending or pending_bytes + limit > max_staged:
await db.rollback()
raise backup_error("backup_preflight_quota", "待处理预检配额已达上限", 429)
db.add(reservation)
await db.flush()
reservation_id = reservation.id
await db.commit()
size = 0
digest = hashlib.sha256()
archive = None
try:
with staging.open("xb") as output:
while chunk := await file.read(_READ_CHUNK):
size += len(chunk)
if size > limit:
raise backup_error("backup_size_invalid", "备份文件大小无效")
digest.update(chunk)
output.write(chunk)
archive = parse_archive_path(staging, max_archive_bytes=limit)
validate_archive(archive)
if mode == "merge":
incoming_pin_ids = {
str(item["id"])
for item in archive.entities["countdowns"]
if item.get("pinned") is True and item.get("archived_at") is None
}
existing_pin_ids = set((await db.scalars(select(Countdown.id).where(
Countdown.user_id == user.id,
Countdown.pinned.is_(True),
Countdown.archived_at.is_(None),
))).all())
if (
incoming_pin_ids
and existing_pin_ids
and incoming_pin_ids != {str(item) for item in existing_pin_ids}
):
raise backup_error("backup_constraint_invalid", "合并恢复会产生多个置顶倒数日")
shutil.rmtree(archive.staging_dir, ignore_errors=True)
if archive.archive_sha256 != digest.hexdigest():
raise backup_error("backup_checksum_mismatch", "备份校验和不匹配")
reservation.backup_id = archive.backup_id
reservation.archive_sha256 = archive.archive_sha256
reservation.archive_size = size
reservation.status = "pending"
await db.commit()
except Exception:
await db.rollback()
if archive is not None:
shutil.rmtree(archive.staging_dir, ignore_errors=True)
failed = await db.get(BackupPreflight, reservation_id)
if failed is not None:
await db.delete(failed)
await db.commit()
staging.unlink(missing_ok=True)
raise
return {"valid": True, "preflight_token": token, "backup_id": archive.backup_id,
"archive_sha256": archive.archive_sha256,
"entities": {name: len(rows) for name, rows in archive.entities.items()}}
@router.post("/restore")
async def restore(
payload: RestoreRequest,
user: User = Depends(current_user),
db: AsyncSession = Depends(get_db),
):
await _prune(db)
token_hash = hash_token(payload.preflight_token)
row = await db.scalar(select(BackupPreflight).where(
BackupPreflight.token_hash == token_hash, BackupPreflight.user_id == user.id
).with_for_update())
if row is None or row.mode != payload.mode or row.expires_at <= utcnow():
raise backup_error("backup_preflight_invalid", "预检令牌无效或已过期", 409)
if row.status == "repair_pending" and row.cleanup_path:
from .storage import restore_quarantine_dir
root = Path(get_settings().attachment_dir).resolve()
try:
restore_quarantine_dir(root, Path(row.cleanup_path))
except OSError as exc:
raise backup_error("backup_repair_pending", "原附件复原尚未完成,请稍后重试", 503) from exc
row.status = "failed"
row.cleanup_path = None
await db.commit()
raise backup_error("backup_restore_retry", "附件已复原,请重新预检后重试", 409)
if row.status == "cleanup_pending":
from .storage import remove_quarantine
try:
if row.cleanup_path:
remove_quarantine(Path(row.cleanup_path))
_safe_staging_path(row.staging_path).unlink(missing_ok=True)
except OSError as exc:
raise backup_error("backup_cleanup_pending", "清理尚未完成,请稍后重试", 503) from exc
row.status = "consumed"
row.cleanup_path = None
await db.commit()
return {"restored": 0, "mode": payload.mode, "already_imported": False, "cleanup_retried": True}
claimed = await db.execute(update(BackupPreflight).where(
BackupPreflight.id == row.id, BackupPreflight.status.in_(("pending", "failed"))
).values(status="consuming", consumed_at=utcnow()))
if claimed.rowcount != 1:
await db.rollback()
raise backup_error("backup_preflight_invalid", "预检令牌已使用", 409)
await db.commit()
staging = _safe_staging_path(row.staging_path)
archive = parse_archive_path(staging)
try:
if archive.archive_sha256 != row.archive_sha256 or archive.backup_id != row.backup_id:
row.status = "failed"
await db.commit()
raise backup_error("backup_preflight_invalid", "预检暂存已改变", 409)
result = await restore_v2(db, user, archive, payload.mode, operation=row)
except BackupRepairPending:
raise
except Exception:
await db.refresh(row)
if row.status not in {"cleanup_pending", "repair_pending"}:
row.status = "failed"
await db.commit()
raise
finally:
shutil.rmtree(archive.staging_dir, ignore_errors=True)
row.status = "cleanup_pending"
await db.commit()
try:
staging.unlink(missing_ok=True)
except OSError as exc:
raise backup_error("backup_cleanup_pending", "数据已恢复,但暂存清理未完成;请使用同一令牌重试", 500) from exc
row.status = "consumed"
await db.commit()
return result
+6
View File
@@ -0,0 +1,6 @@
from pydantic import BaseModel, Field
class RestoreRequest(BaseModel):
preflight_token: str = Field(min_length=32, max_length=128)
mode: str = Field(pattern="^(merge|replace)$")
+771
View File
@@ -0,0 +1,771 @@
from __future__ import annotations
import hashlib
import math
import tempfile
from datetime import date, datetime
from itertools import pairwise
from pathlib import Path
from uuid import UUID, uuid5
from fastapi import HTTPException
from pydantic import ValidationError
from sqlalchemy import Boolean, Date, DateTime, Float, Integer, String, Text, Uuid, delete, select
from sqlalchemy.ext.asyncio import AsyncSession
from backend.config import get_settings
from backend.models import (
Attachment,
BackupImport,
BackupImportEntity,
BackupPreflight,
Countdown,
Folder,
Habit,
HabitLog,
HabitPause,
Memo,
RecurrenceException,
RecurrenceTemplate,
Task,
TaskList,
User,
new_id,
)
from backend.mvp import (
CountdownInput,
HabitCreate,
MemoCreate,
PauseInput,
RecurrenceCreate,
is_occurrence,
parse_rrule,
)
from backend.schemas import FolderCreate, ListCreate, TaskCreate
from .archive import ParsedArchive, backup_error, build_archive_to_path, canonical_json
from .storage import contained_file, quarantine_files, remove_quarantine, restore_quarantine
class BackupRepairPending(HTTPException):
def __init__(self) -> None:
super().__init__(500, {
"code": "backup_repair_pending",
"message": "恢复失败且原附件尚未完全复原;请使用同一令牌重试",
})
ENTITY_MODELS = {
"folders": Folder,
"lists": TaskList,
"tasks": Task,
"recurrences": RecurrenceTemplate,
"recurrence_exceptions": RecurrenceException,
"habits": Habit,
"habit_logs": HabitLog,
"habit_pauses": HabitPause,
"countdowns": Countdown,
"memos": Memo,
"attachments": Attachment,
}
RELATIONS = {
"lists": {"folder_id": "folders"},
"tasks": {"list_id": "lists", "parent_id": "tasks"},
"recurrences": {"task_id": "tasks"},
"recurrence_exceptions": {"template_id": "recurrences"},
"habit_logs": {"habit_id": "habits"},
"habit_pauses": {"habit_id": "habits"},
"attachments": {"task_id": "tasks"},
}
ORDER = tuple(ENTITY_MODELS)
def _value(value):
if isinstance(value, (UUID, date, datetime)):
return value.isoformat() if not isinstance(value, UUID) else str(value)
return value
def _row(row, *, attachment_path: str | None = None) -> dict:
result = {
column.name: _value(getattr(row, column.name))
for column in row.__table__.columns
if column.name != "user_id" and column.name != "storage_name"
}
if attachment_path is not None:
result["archive_path"] = attachment_path
return result
async def export_v2(db: AsyncSession, user: User) -> Path:
# Serialize exports with restores for this user. PostgreSQL keeps the row
# lock until this function's transaction is committed by the route.
await db.scalar(select(User).where(User.id == user.id).with_for_update())
entities: dict[str, list[dict]] = {}
files: dict[str, Path] = {}
root = Path(get_settings().attachment_dir).resolve()
total_size = 0
limit = get_settings().backup_max_archive_mb * 1024 * 1024
for name, model in ENTITY_MODELS.items():
if name in {"recurrence_exceptions", "habit_logs", "habit_pauses"}:
if name == "recurrence_exceptions":
ids = select(RecurrenceTemplate.id).where(RecurrenceTemplate.user_id == user.id)
rows = list((await db.scalars(select(model).where(model.template_id.in_(ids)))).all())
else:
ids = select(Habit.id).where(Habit.user_id == user.id)
rows = list((await db.scalars(select(model).where(model.habit_id.in_(ids)))).all())
else:
rows = list((await db.scalars(select(model).where(model.user_id == user.id))).all())
output = []
for item in rows:
if isinstance(item, Attachment):
archive_path = f"attachments/{item.id}/content"
try:
path = contained_file(root, item.storage_name)
except ValueError as exc:
raise backup_error("backup_attachment_path_invalid", "附件存储路径无效") from exc
if not path.is_file():
raise backup_error("backup_attachment_missing", "附件文件缺失")
actual_size = path.stat().st_size
if actual_size != item.size:
raise backup_error("backup_attachment_size_mismatch", "附件大小不匹配")
total_size += actual_size
if total_size > limit:
raise backup_error("backup_size_invalid", "备份文件大小超出导入合同")
files[archive_path] = path
output.append(_row(item, attachment_path=archive_path))
else:
output.append(_row(item))
entities[name] = output
handle, output_name = tempfile.mkstemp(prefix="dodo-backup-", suffix=".zip")
import os
os.close(handle)
output_path = Path(output_name)
try:
build_archive_to_path(output_path, new_id(), entities, files)
if output_path.stat().st_size > limit:
raise backup_error("backup_size_invalid", "备份文件大小超出导入合同")
return output_path
except Exception:
output_path.unlink(missing_ok=True)
raise
def _validate_scalar(entity: str, column, row: dict) -> None:
name = column.name
if name in {"user_id", "storage_name"}:
return
if name not in row:
if not column.nullable and column.default is None and not column.primary_key:
raise backup_error("backup_entity_invalid", f"{entity} 缺少必填字段")
return
value = row[name]
if value is None:
if not column.nullable and not column.primary_key:
raise backup_error("backup_entity_invalid", f"{entity}.{name} 不可为空")
return
try:
effective_type = getattr(column.type, "impl", column.type)
if isinstance(effective_type, Uuid):
UUID(str(value))
elif isinstance(effective_type, DateTime):
if not isinstance(value, str):
raise TypeError
datetime.fromisoformat(value)
elif isinstance(effective_type, Date):
if not isinstance(value, str):
raise TypeError
date.fromisoformat(value)
elif isinstance(effective_type, Boolean):
if type(value) is not bool:
raise TypeError
elif isinstance(effective_type, Integer):
if type(value) is not int:
raise TypeError
elif isinstance(effective_type, Float):
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
raise TypeError
elif isinstance(effective_type, (String, Text)):
if not isinstance(value, str):
raise TypeError
if isinstance(effective_type, String) and effective_type.length is not None and len(value) > effective_type.length:
raise ValueError
except (TypeError, ValueError, OverflowError) as exc:
raise backup_error("backup_entity_invalid", f"{entity}.{name} 类型无效") from exc
def _parse_habit_days(value: object, *, minimum: int, maximum: int) -> list[int] | None:
if value is None:
return None
if not isinstance(value, str) or not value or any(part.strip() != part for part in value.split(",")):
raise ValueError
parts = value.split(",")
if any(not part.isascii() or not part.isdecimal() for part in parts):
raise ValueError
days = [int(part) for part in parts]
if value != ",".join(map(str, days)):
raise ValueError
if len(days) != len(set(days)) or any(day < minimum or day > maximum for day in days):
raise ValueError
return days
def _validate_habit_graph(parsed: ParsedArchive) -> None:
habits: dict[str, HabitCreate] = {}
try:
for row in parsed.entities["habits"]:
weekdays = _parse_habit_days(row.get("weekdays"), minimum=0, maximum=6)
month_days = _parse_habit_days(row.get("month_days"), minimum=1, maximum=31)
payload = {
field: row.get(field)
for field in HabitCreate.model_fields
if field in row
}
payload["weekdays"] = weekdays
payload["month_days"] = month_days
habit = HabitCreate.model_validate(payload)
if habit.kind == "boolean" and (row.get("target") != 1 or row.get("max_value") != 1):
raise ValueError
if habit.weekdays != weekdays or habit.month_days != month_days:
raise ValueError
if habit.interval_days != row.get("interval_days"):
raise ValueError
habits[str(row["id"])] = habit
for row in parsed.entities["habit_logs"]:
habit = habits[str(row["habit_id"])]
value = row["value"]
if not math.isfinite(value) or value < 0:
raise ValueError
if habit.kind == "boolean":
if value not in (0, 1):
raise ValueError
elif habit.max_value is not None and value > habit.max_value:
raise ValueError
pauses_by_habit: dict[str, list[tuple[date, date]]] = {}
for row in parsed.entities["habit_pauses"]:
pause = PauseInput.model_validate({
"start_date": row.get("start_date"),
"end_date": row.get("end_date"),
})
pauses_by_habit.setdefault(str(row["habit_id"]), []).append(
(pause.start_date, pause.end_date)
)
for pauses in pauses_by_habit.values():
pauses.sort()
if any(current[0] <= previous[1] for previous, current in pairwise(pauses)):
raise ValueError
except (KeyError, TypeError, ValueError, ValidationError) as exc:
raise backup_error("backup_habit_invalid", "习惯数据不符合业务规则") from exc
def _required_trimmed(value: object, *, maximum: int | None = None) -> str:
if not isinstance(value, str) or value != value.strip() or not value:
raise ValueError
if maximum is not None and len(value) > maximum:
raise ValueError
return value
def _ordered_datetimes(row: dict, earlier: str, later: str) -> None:
if row.get(earlier) and row.get(later) and datetime.fromisoformat(row[later]) < datetime.fromisoformat(row[earlier]):
raise ValueError
def _validate_entity_contracts(parsed: ParsedArchive) -> None:
try:
for row in parsed.entities["folders"]:
FolderCreate.model_validate({"name": row.get("name")})
_required_trimmed(row.get("name"), maximum=120)
if row.get("position", 0) < 0:
raise ValueError
_ordered_datetimes(row, "created_at", "deleted_at")
for row in parsed.entities["lists"]:
ListCreate.model_validate({"name": row.get("name"), "folder_id": row.get("folder_id")})
_required_trimmed(row.get("name"), maximum=120)
if row.get("position", 0) < 0:
raise ValueError
_ordered_datetimes(row, "created_at", "deleted_at")
for row in parsed.entities["tasks"]:
task = TaskCreate.model_validate({
"title": row.get("title"),
"list_id": row.get("list_id"),
"description": row.get("description", ""),
"priority": row.get("priority", 0),
"due_at": row.get("due_at"),
"due_has_time": row.get("due_has_time", False),
"parent_id": row.get("parent_id"),
})
_required_trimmed(row.get("title"), maximum=500)
if row.get("version", 1) < 1 or row.get("position", 0) < 0:
raise ValueError
if task.due_at is None and task.due_has_time:
raise ValueError
completed_at = row.get("completed_at")
if row.get("completed") is True and completed_at is None:
raise ValueError
if row.get("completed") is False and completed_at is not None:
raise ValueError
_ordered_datetimes(row, "created_at", "updated_at")
_ordered_datetimes(row, "created_at", "deleted_at")
for row in parsed.entities["countdowns"]:
CountdownInput.model_validate({
field: row[field] for field in CountdownInput.model_fields if field in row
})
_required_trimmed(row.get("title"), maximum=200)
_required_trimmed(row.get("icon"), maximum=32)
_ordered_datetimes(row, "created_at", "updated_at")
_ordered_datetimes(row, "created_at", "archived_at")
for row in parsed.entities["memos"]:
MemoCreate.model_validate({"title": row.get("title"), "content": row.get("content", "")})
_required_trimmed(row.get("title"), maximum=200)
if row.get("version", 1) < 1:
raise ValueError
_ordered_datetimes(row, "created_at", "updated_at")
_ordered_datetimes(row, "created_at", "deleted_at")
allowed_mime = {
"text/plain", "text/csv", "application/pdf", "image/jpeg", "image/png",
"image/gif", "application/json", "application/zip",
}
for row in parsed.entities["attachments"]:
filename = row.get("filename")
mime_type = row.get("mime_type")
if (
not isinstance(filename, str)
or not filename.strip()
or Path(filename).name != filename
or mime_type not in allowed_mime
or row.get("size", -1) < 0
):
raise ValueError
except (KeyError, TypeError, ValueError, ValidationError) as exc:
raise backup_error("backup_entity_invalid", "实体数据不符合业务规则") from exc
def _validate_recurrence_graph(parsed: ParsedArchive) -> None:
tasks = {str(row["id"]): row for row in parsed.entities["tasks"]}
try:
for row in parsed.entities["recurrences"]:
task = tasks[str(row["task_id"])]
recurrence = RecurrenceCreate.model_validate({
"task_id": row["task_id"],
"rrule": row.get("rrule"),
"trigger_mode": row.get("trigger_mode", "scheduled"),
"after_completion_days": row.get("after_completion_days"),
})
starts_at = datetime.fromisoformat(row["starts_at"])
ends_at = datetime.fromisoformat(row["ends_at"]) if row.get("ends_at") else None
last_completed_at = (
datetime.fromisoformat(row["last_completed_at"])
if row.get("last_completed_at")
else None
)
if task.get("parent_id") is not None or task.get("due_at") is None:
raise ValueError
due_at = datetime.fromisoformat(task["due_at"])
if recurrence.trigger_mode == "scheduled":
if recurrence.rrule is None:
raise ValueError
parse_rrule(recurrence.rrule)
if starts_at != due_at:
raise ValueError
if ends_at is not None and ends_at < starts_at:
raise ValueError
if last_completed_at is not None and last_completed_at > starts_at:
raise ValueError
recurrences = {str(row["id"]): row for row in parsed.entities["recurrences"]}
for row in parsed.entities["recurrence_exceptions"]:
recurrence = recurrences[str(row["template_id"])]
if recurrence.get("trigger_mode", "scheduled") != "scheduled":
raise ValueError
occurrence_at = datetime.fromisoformat(row["occurrence_at"])
if not is_occurrence(
recurrence["rrule"], datetime.fromisoformat(recurrence["starts_at"]), occurrence_at
):
raise ValueError
if recurrence.get("ends_at") and occurrence_at > datetime.fromisoformat(recurrence["ends_at"]):
raise ValueError
except (HTTPException, KeyError, TypeError, ValueError, ValidationError) as exc:
raise backup_error("backup_recurrence_invalid", "重复规则不符合业务规则") from exc
def validate_archive(parsed: ParsedArchive) -> None:
unknown = set(parsed.entities) - set(ENTITY_MODELS)
missing = set(ENTITY_MODELS) - set(parsed.entities)
if unknown:
raise backup_error("backup_entity_unknown", "备份包含未知实体")
if missing:
raise backup_error("backup_entity_missing", "备份缺少必需实体")
ids: dict[str, set[str]] = {}
for entity in ORDER:
rows = parsed.entities.get(entity, [])
model = ENTITY_MODELS[entity]
allowed = {column.name for column in model.__table__.columns} | ({"archive_path"} if entity == "attachments" else set())
entity_ids = []
for row in rows:
if set(row) - allowed or "user_id" in row or "storage_name" in row:
raise backup_error("backup_entity_invalid", f"{entity} 包含未知或受保护字段")
for column in model.__table__.columns:
_validate_scalar(entity, column, row)
try:
entity_ids.append(str(UUID(str(row["id"]))))
except (KeyError, TypeError, ValueError) as exc:
raise backup_error("backup_entity_invalid", f"{entity} 包含无效 ID") from exc
if len(entity_ids) != len(set(entity_ids)):
raise backup_error("backup_duplicate_id", f"{entity} 包含重复 ID")
ids[entity] = set(entity_ids)
for entity, fields in RELATIONS.items():
for row in parsed.entities.get(entity, []):
for field, target in fields.items():
value = row.get(field)
if value is not None and str(value) not in ids[target]:
raise backup_error("backup_reference_invalid", f"{entity}.{field} 引用不存在")
attachment_paths = set()
for row in parsed.entities.get("attachments", []):
path = row.get("archive_path")
if not isinstance(path, str) or path not in parsed.files or not path.startswith("attachments/"):
raise backup_error("backup_attachment_missing", "附件内容缺失")
if path in attachment_paths:
raise backup_error("backup_attachment_duplicate", "附件内容被重复引用")
attachment_paths.add(path)
blob = parsed.files[path]
if row.get("size") != blob.size:
raise backup_error("backup_attachment_size_mismatch", "附件大小不匹配")
if set(parsed.files) != attachment_paths:
raise backup_error("backup_unreferenced_file", "备份包含未引用文件")
def unique(entity: str, fields: tuple[str, ...], *, ignore_null: bool = False) -> None:
seen = set()
for row in parsed.entities[entity]:
key = tuple(row.get(field) for field in fields)
if ignore_null and any(value is None for value in key):
continue
if key in seen:
raise backup_error("backup_constraint_invalid", f"{entity} 唯一约束冲突")
seen.add(key)
unique("recurrences", ("task_id",))
unique("recurrence_exceptions", ("template_id", "occurrence_at"))
unique("habit_logs", ("habit_id", "day"))
unique("tasks", ("external_id",), ignore_null=True)
active_pinned = sum(
row.get("pinned") is True and row.get("archived_at") is None
for row in parsed.entities["countdowns"]
)
if active_pinned > 1:
raise backup_error("backup_constraint_invalid", "最多只能有一个置顶倒数日")
_validate_entity_contracts(parsed)
_validate_habit_graph(parsed)
_validate_recurrence_graph(parsed)
inboxes = [row for row in parsed.entities["lists"] if row.get("is_inbox") is True]
if len(inboxes) != 1:
raise backup_error("backup_constraint_invalid", "备份必须包含且仅包含一个收集箱")
tasks = {str(row["id"]): row for row in parsed.entities["tasks"]}
for task_id, row in tasks.items():
parent_id = row.get("parent_id")
if parent_id is None:
continue
parent_id = str(parent_id)
if parent_id == task_id:
raise backup_error("backup_constraint_invalid", "任务不能以自身为父任务")
parent = tasks[parent_id]
if parent.get("parent_id") is not None:
raise backup_error("backup_constraint_invalid", "任务父子关系仅支持一层")
if parent.get("list_id") != row.get("list_id"):
raise backup_error("backup_constraint_invalid", "父子任务必须属于同一清单")
def _ordered_rows(entity: str, rows: list[dict]) -> list[dict]:
if entity != "tasks":
return rows
pending = {str(row["id"]): row for row in rows}
ordered: list[dict] = []
while pending:
ready = [row for row in pending.values() if row.get("parent_id") is None or str(row["parent_id"]) not in pending]
if not ready:
raise backup_error("backup_constraint_invalid", "任务父子关系存在环")
for row in ready:
ordered.append(row)
pending.pop(str(row["id"]))
return ordered
def _normalized_values(values: dict) -> dict:
return {key: _value(value) for key, value in values.items() if key not in {"created_at", "updated_at"}}
def _coerce(model, raw: dict, mapping: dict[str, dict[str, UUID]], user_id: UUID) -> dict:
values = {}
relations = RELATIONS.get(next(name for name, item in ENTITY_MODELS.items() if item is model), {})
for column in model.__table__.columns:
name = column.name
if name == "user_id":
values[name] = user_id
elif name == "storage_name":
continue
elif name in raw:
value = raw[name]
if name in relations and value is not None:
value = mapping[relations[name]][str(value)]
elif value is not None:
column_type = column.type
effective_type = getattr(column_type, "impl", column_type)
if isinstance(effective_type, Uuid):
value = UUID(str(value))
elif isinstance(effective_type, DateTime) and isinstance(value, str):
value = datetime.fromisoformat(value)
elif isinstance(effective_type, Date) and isinstance(value, str):
value = date.fromisoformat(value)
values[name] = value
return values
def _content_digest(entity: str, raw: dict) -> str:
# Source-form relations are deliberate: digest identity remains stable even
# when another user needs different target UUIDs.
payload = {key: value for key, value in raw.items() if key not in {"created_at", "updated_at", "storage_name"}}
return hashlib.sha256(canonical_json(payload)).hexdigest()
async def _all_uuid_primary_keys(db: AsyncSession) -> set[UUID]:
occupied: set[UUID] = set()
for table in User.metadata.sorted_tables:
primary_keys = list(table.primary_key.columns)
if len(primary_keys) != 1:
continue
column = primary_keys[0]
effective_type = getattr(column.type, "impl", column.type)
if isinstance(effective_type, Uuid):
occupied.update((await db.scalars(select(column))).all())
return occupied
async def _mapped_id(
db: AsyncSession, model, source: UUID, user_id: UUID, backup_id: UUID, entity: str,
reserved: set[UUID], incoming: set[UUID],
) -> UUID:
ledger = await db.scalar(select(BackupImportEntity).where(
BackupImportEntity.user_id == user_id,
BackupImportEntity.backup_id == backup_id,
BackupImportEntity.entity_type == entity,
BackupImportEntity.source_id == source,
))
if ledger is not None:
reserved.add(ledger.target_id)
return ledger.target_id
existing = await db.get(model, source)
owner_id = getattr(existing, "user_id", None)
if owner_id is None:
if isinstance(existing, RecurrenceException):
owner_id = await db.scalar(select(RecurrenceTemplate.user_id).where(RecurrenceTemplate.id == existing.template_id))
elif isinstance(existing, (HabitLog, HabitPause)):
owner_id = await db.scalar(select(Habit.user_id).where(Habit.id == existing.habit_id))
if existing is not None and owner_id == user_id:
reserved.add(source)
return source
if source not in reserved:
reserved.add(source)
return source
candidate = uuid5(user_id, str(source))
while candidate in reserved or candidate in incoming:
candidate = uuid5(user_id, str(candidate))
reserved.add(candidate)
return candidate
async def restore_v2(
db: AsyncSession, user: User, parsed: ParsedArchive, mode: str,
*, operation: BackupPreflight | None = None,
) -> dict:
validate_archive(parsed)
# The user row is the cross-worker restore/export mutex on PostgreSQL.
await db.scalar(select(User).where(User.id == user.id).with_for_update())
existing_import = await db.scalar(select(BackupImport).where(
BackupImport.user_id == user.id, BackupImport.backup_id == parsed.backup_id
))
already_imported = existing_import is not None
if mode == "merge" and existing_import and existing_import.archive_sha256 != parsed.archive_sha256:
raise HTTPException(409, {"code": "backup_id_conflict", "message": "备份标识与内容不一致"})
root = Path(get_settings().attachment_dir).resolve()
root.mkdir(parents=True, exist_ok=True)
quarantine = root.parent / f".backup-quarantine-{new_id()}"
moved = []
written: list[Path] = []
try:
if mode == "replace":
old_names = list((await db.scalars(
select(Attachment.storage_name).where(Attachment.user_id == user.id)
)).all())
moved = quarantine_files(root, old_names, quarantine)
await db.execute(delete(RecurrenceException).where(
RecurrenceException.template_id.in_(
select(RecurrenceTemplate.id).where(RecurrenceTemplate.user_id == user.id)
)
))
await db.execute(delete(RecurrenceTemplate).where(RecurrenceTemplate.user_id == user.id))
await db.execute(delete(HabitLog).where(
HabitLog.habit_id.in_(select(Habit.id).where(Habit.user_id == user.id))
))
await db.execute(delete(HabitPause).where(
HabitPause.habit_id.in_(select(Habit.id).where(Habit.user_id == user.id))
))
for model in (Attachment, Memo, Countdown, Task, Habit, TaskList, Folder):
await db.execute(delete(model).where(model.user_id == user.id))
await db.execute(delete(BackupImportEntity).where(BackupImportEntity.user_id == user.id))
await db.execute(delete(BackupImport).where(BackupImport.user_id == user.id))
mapping: dict[str, dict[str, UUID]] = {name: {} for name in ORDER}
incoming_ids: set[UUID] = {
UUID(str(raw["id"]))
for rows in parsed.entities.values()
for raw in rows
}
reserved_ids = await _all_uuid_primary_keys(db)
for entity in ORDER:
model = ENTITY_MODELS[entity]
for raw in _ordered_rows(entity, parsed.entities[entity]):
source = UUID(str(raw["id"]))
mapping[entity][str(source)] = await _mapped_id(
db, model, source, user.id, parsed.backup_id, entity,
reserved_ids, incoming_ids - {source},
)
restored = 0
for entity in ORDER:
model = ENTITY_MODELS[entity]
for raw in _ordered_rows(entity, parsed.entities[entity]):
source = UUID(str(raw["id"]))
target = mapping[entity][str(source)]
digest_payload = dict(raw)
if model is Attachment:
digest_payload["archive_sha256"] = parsed.files[raw["archive_path"]].sha256
content_digest = _content_digest(entity, digest_payload)
ledger = await db.scalar(select(BackupImportEntity).where(
BackupImportEntity.user_id == user.id,
BackupImportEntity.backup_id == parsed.backup_id,
BackupImportEntity.entity_type == entity,
BackupImportEntity.source_id == source,
))
existing = await db.get(model, target)
values = _coerce(model, raw, mapping, user.id)
values["id"] = target
if ledger is not None:
if ledger.target_id != target or ledger.content_digest != content_digest:
raise HTTPException(409, {"code": "backup_entity_conflict", "message": f"{entity} 映射账本冲突"})
if existing is None:
raise HTTPException(409, {"code": "backup_entity_missing", "message": f"{entity} 映射目标不存在"})
current = _row(existing, attachment_path=raw.get("archive_path") if model is Attachment else None)
expected = {key: _value(value) for key, value in values.items() if key not in {"user_id", "storage_name"}}
if model is Attachment:
expected["archive_path"] = raw["archive_path"]
stored = contained_file(root, existing.storage_name)
file_digest = hashlib.sha256()
actual_size = 0
try:
with stored.open("rb") as source_file:
while chunk := source_file.read(1024 * 1024):
actual_size += len(chunk)
file_digest.update(chunk)
except OSError as exc:
raise HTTPException(409, {"code": "backup_entity_missing", "message": "attachments 映射目标文件不存在"}) from exc
blob = parsed.files[raw["archive_path"]]
if actual_size != blob.size or file_digest.hexdigest() != blob.sha256:
raise HTTPException(409, {"code": "backup_entity_conflict", "message": "attachments 映射目标内容冲突"})
if _normalized_values(current) != _normalized_values(expected):
raise HTTPException(409, {"code": "backup_entity_conflict", "message": f"{entity} 映射目标内容冲突"})
continue
if existing is not None and mode == "merge":
current = _row(existing, attachment_path=raw.get("archive_path") if model is Attachment else None)
expected = {key: _value(value) for key, value in values.items() if key not in {"user_id", "storage_name"}}
if model is Attachment:
expected["archive_path"] = raw["archive_path"]
stored = contained_file(root, existing.storage_name)
blob = parsed.files[raw["archive_path"]]
file_digest = hashlib.sha256()
actual_size = 0
try:
with stored.open("rb") as source_file:
while chunk := source_file.read(1024 * 1024):
actual_size += len(chunk)
file_digest.update(chunk)
except OSError as exc:
raise HTTPException(409, {
"code": "backup_entity_missing",
"message": "attachments 已存在但文件不存在",
}) from exc
if actual_size != blob.size or file_digest.hexdigest() != blob.sha256:
raise HTTPException(409, {
"code": "backup_entity_conflict",
"message": "attachments 已存在不同内容",
})
if _normalized_values(current) != _normalized_values(expected):
raise HTTPException(409, {"code": "backup_entity_conflict", "message": f"{entity} 已存在不同内容"})
else:
if model is Attachment:
archive_path = raw["archive_path"]
storage_name = str(new_id())
destination = contained_file(root, storage_name)
digest = hashlib.sha256()
copied_size = 0
blob = parsed.files[archive_path]
with blob.staging_path.open("rb") as source_file, destination.open("xb") as output:
while chunk := source_file.read(1024 * 1024):
copied_size += len(chunk)
digest.update(chunk)
output.write(chunk)
if copied_size != blob.size or digest.hexdigest() != blob.sha256:
destination.unlink(missing_ok=True)
raise backup_error("backup_checksum_mismatch", "附件暂存校验失败", 409)
written.append(destination)
values["storage_name"] = storage_name
db.add(model(**values))
await db.flush()
restored += 1
db.add(BackupImportEntity(
user_id=user.id, backup_id=parsed.backup_id, entity_type=entity,
source_id=source, target_id=target, content_digest=content_digest,
))
await db.flush()
if mode == "replace" or existing_import is None:
db.add(BackupImport(
user_id=user.id, backup_id=parsed.backup_id,
archive_sha256=parsed.archive_sha256, mode=mode,
))
await db.commit()
except HTTPException:
await db.rollback()
for path in written:
path.unlink(missing_ok=True)
try:
restore_quarantine(moved)
except (OSError, ValueError):
if operation is not None:
operation.status = "repair_pending"
operation.cleanup_path = str(quarantine)
await db.commit()
raise BackupRepairPending()
raise
except Exception:
await db.rollback()
for path in written:
path.unlink(missing_ok=True)
try:
restore_quarantine(moved)
except (OSError, ValueError):
if operation is not None:
operation.status = "repair_pending"
operation.cleanup_path = str(quarantine)
await db.commit()
raise BackupRepairPending()
raise
try:
remove_quarantine(quarantine)
except OSError as exc:
if operation is not None:
operation.status = "cleanup_pending"
operation.cleanup_path = str(quarantine)
await db.commit()
raise HTTPException(500, {"code": "backup_cleanup_pending", "message": "数据已恢复,但旧附件清理未完成;请使用同一令牌重试"}) from exc
return {"restored": restored, "mode": mode, "already_imported": already_imported, "cleanup_retried": False}
+52
View File
@@ -0,0 +1,52 @@
import shutil
from pathlib import Path
def contained_file(root: Path, storage_name: str) -> Path:
root = root.resolve()
candidate = (root / storage_name).resolve()
if candidate.parent != root:
raise ValueError("attachment path escapes storage root")
return candidate
def quarantine_files(root: Path, storage_names: list[str], quarantine: Path) -> list[tuple[Path, Path]]:
moved = []
quarantine.mkdir(parents=True, exist_ok=True)
try:
for name in storage_names:
source = contained_file(root, name)
if source.exists():
target = quarantine / name
target.parent.mkdir(parents=True, exist_ok=True)
source.replace(target)
moved.append((source, target))
return moved
except Exception:
restore_quarantine(moved)
raise
def restore_quarantine(moved: list[tuple[Path, Path]]) -> None:
for original, quarantined in reversed(moved):
if quarantined.exists():
original.parent.mkdir(parents=True, exist_ok=True)
quarantined.replace(original)
def restore_quarantine_dir(root: Path, quarantine: Path) -> None:
if not quarantine.exists():
return
for quarantined in sorted(quarantine.rglob("*")):
if not quarantined.is_file():
continue
relative = quarantined.relative_to(quarantine)
original = contained_file(root, relative.as_posix())
original.parent.mkdir(parents=True, exist_ok=True)
quarantined.replace(original)
shutil.rmtree(quarantine)
def remove_quarantine(path: Path) -> None:
if path.exists():
shutil.rmtree(path)