548 lines
42 KiB
Python
548 lines
42 KiB
Python
"""Opt-in migration regression; dedicated TEST database and disposable schemas only."""
|
|
from __future__ import annotations
|
|
|
|
from contextlib import closing, redirect_stdout
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from hashlib import sha256
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
from unittest.mock import patch
|
|
from uuid import uuid4
|
|
|
|
import psycopg
|
|
from psycopg import sql
|
|
|
|
from apps.server.db import PostgresStore, Store, StoreConflict
|
|
from scripts import migrate_learning
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def seed_source(home: Path, *, quality: bool = True, invalid_quality: bool = False,
|
|
discovery: bool = False, invalid_discovery: bool = False,
|
|
reading: bool = False, invalid_reading: bool = False,
|
|
analysis: bool = False, invalid_analysis: bool = False,
|
|
business: bool = False, invalid_business: bool = False,
|
|
reviews: bool = False, invalid_review_parent: bool = False) -> Path:
|
|
source = Store(home)
|
|
stamp = "2026-10-24T00:00:00Z"
|
|
with source.connect(immediate=True) as conn:
|
|
conn.execute("INSERT INTO operators VALUES($1,$2,$3,$4)", ("owner", "fixture", "not-a-real-password-hash", stamp))
|
|
conn.execute("INSERT INTO learning_tasks VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)",
|
|
("task", "owner", "migration fixture", "demo", "http://127.0.0.1/fixture", "fixture only", "test", "{}", "paused", None, stamp, stamp))
|
|
conn.execute("INSERT INTO candidates VALUES($1,$2,$3,$4,$5,$6,$7)", ("candidate", "task", "fixture", 1, "fixture-digest", "{}", stamp))
|
|
conn.execute("INSERT INTO validations VALUES($1,$2,$3,$4,$5,$6,$7)", ("validation", "candidate", "task", "browser", "passed", "{}", stamp))
|
|
if quality:
|
|
# A deliberately malformed source fixture exercises transaction rollback.
|
|
# SQLite foreign-key enforcement is connection-local and disabled here.
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
for key, created in [("quality-first", "2026-10-24T02:00:00Z"), ("quality-second", "2026-10-24T01:00:00Z")]:
|
|
conn.execute("INSERT INTO quality_observations VALUES(?,?,?,?,?,?)",
|
|
(key, "invalid-validation" if invalid_quality else "validation", "candidate", "task",
|
|
json.dumps({"status": "no_issue", "fixture": key}), created))
|
|
if not quality:
|
|
# A real legacy SQLite installation had no quality table at all.
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
conn.execute("DROP TABLE quality_observations")
|
|
if discovery:
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
for key, created, validation in [("discovery-first", "2026-10-24T04:00:00Z", "validation"),
|
|
("discovery-second", "2026-10-24T03:00:00Z", None)]:
|
|
payload = {"fixture": key, "service": "装修", "region": None,
|
|
"selection": [{"source": "http://127.0.0.1/fixture/note", "basis": ["合成迁移样本"]}]}
|
|
conn.execute("INSERT INTO discovery_runs VALUES(?,?,?,?,?,?)",
|
|
(key, "owner", "candidate", "invalid-validation" if invalid_discovery else validation,
|
|
json.dumps(payload, ensure_ascii=False), created))
|
|
else:
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
conn.execute("DROP TABLE discovery_runs")
|
|
if reading:
|
|
assert discovery, "reading fixtures require their source discovery record"
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
for key, created, validation in [("reading-first", "2026-10-24T06:00:00Z", "validation"),
|
|
("reading-second", "2026-10-24T05:00:00Z", None)]:
|
|
payload = {"fixture": key, "status": "partial", "note_id": "a" * 24,
|
|
"detail": {"title": "装修合成样本🏠", "body": "仅迁移测试"},
|
|
"comments": None, "errors": ["合成未覆盖分支"]}
|
|
conn.execute("INSERT INTO reading_runs VALUES(?,?,?,?,?,?,?)",
|
|
(key, "owner", "discovery-first", "candidate", "invalid-validation" if invalid_reading else validation,
|
|
json.dumps(payload, ensure_ascii=False), created))
|
|
else:
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
conn.execute("DROP TABLE reading_runs")
|
|
if analysis:
|
|
assert reading, "analysis fixtures require their reading source"
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
for key, created in [("analysis-first", "2026-10-24T08:00:00Z"), ("analysis-second", "2026-10-24T07:00:00Z")]:
|
|
payload = {"status": "completed", "fixture": key, "results": [{"source_ref": "note", "demand": "insufficient", "evidence": []}], "cost": {"status": "unknown"}}
|
|
conn.execute("INSERT INTO analysis_runs VALUES(?,?,?,?,?)", (key, "owner", "invalid-reading" if invalid_analysis else "reading-first", json.dumps(payload, ensure_ascii=False), created))
|
|
else:
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
conn.execute("DROP TABLE analysis_runs")
|
|
if business:
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
for key, kind, parent, created in [("brief-parse", "parse", None, "2026-10-24T10:00:00Z"),
|
|
("brief-confirm", "confirmed", "invalid-parent" if invalid_business else "brief-parse", "2026-10-24T09:00:00Z")]:
|
|
payload = {"status": "confirmed" if kind == "confirmed" else "needs_clarification", "fixture": key,
|
|
"conditions": {"region": None, "region_scope": "unspecified"}, "cost": {"status": "unknown"}}
|
|
conn.execute("INSERT INTO business_briefs(id,owner_id,kind,parent_id,data_json,created_at) VALUES(?,?,?,?,?,?)",
|
|
(key, "owner", kind, parent, json.dumps(payload, ensure_ascii=False), created))
|
|
else:
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
conn.execute("DROP TABLE business_briefs")
|
|
if reviews:
|
|
assert analysis, "review fixtures require their analysis source"
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
for key, ref, previous, created in [
|
|
("review-first", "note", None, "2026-10-24T14:00:00Z"),
|
|
("review-second", "note", "invalid-previous" if invalid_review_parent else "review-first", "2026-10-24T13:00:00Z"),
|
|
("review-comment", "comment_1", None, "2026-10-24T12:00:00Z"),
|
|
("review-third", "note", "review-second", "2026-10-24T11:00:00Z"),
|
|
]:
|
|
payload = {"fixture": key, "reviewer_source": "maintainer", "decision": "corrected",
|
|
"note": "仅合成人工核对🏠", "labels": {"role": "unknown", "own_demand": "insufficient"}}
|
|
conn.execute("INSERT INTO analysis_reviews(id,owner_id,analysis_run_id,source_ref,previous_id,data_json,created_at) VALUES(?,?,?,?,?,?,?)",
|
|
(key, "owner", "analysis-first", ref, previous, json.dumps(payload, ensure_ascii=False), created))
|
|
else:
|
|
with closing(sqlite3.connect(source.path)) as conn, conn:
|
|
conn.execute("DROP TABLE analysis_reviews")
|
|
return source.path
|
|
|
|
|
|
def run_migration(home: Path, database_url: str, schema: str) -> tuple[dict | None, BaseException | None]:
|
|
output = io.StringIO()
|
|
with patch.dict(os.environ, {"ASSISTANT_DATABASE_URL": database_url,
|
|
"ASSISTANT_DATABASE_SCHEMA": schema,
|
|
"ASSISTANT_LEARNING_HOME": str(home)}), redirect_stdout(output):
|
|
try:
|
|
migrate_learning.main()
|
|
except (SystemExit, StoreConflict, RuntimeError, sqlite3.Error) as error:
|
|
return None, error
|
|
return json.loads(output.getvalue()), None
|
|
|
|
|
|
def main() -> None:
|
|
database_url = os.environ.get("ASSISTANT_TEST_DATABASE_URL", "")
|
|
if not database_url:
|
|
raise SystemExit("Configure the dedicated ASSISTANT_TEST_DATABASE_URL before running migration regression.")
|
|
(ROOT / ".tmp").mkdir(exist_ok=True)
|
|
run_root = Path(tempfile.mkdtemp(prefix="migration-regression-", dir=ROOT / ".tmp"))
|
|
schemas: list[str] = []
|
|
checks: list[str] = []
|
|
|
|
def target(home: Path) -> PostgresStore:
|
|
schema = "assistant_learning_migration_" + uuid4().hex
|
|
schemas.append(schema)
|
|
return PostgresStore(home, database_url, schema=schema)
|
|
|
|
try:
|
|
home = run_root / "new-source"
|
|
original = seed_source(home)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["result"] == "copied", "new source migration failed"
|
|
assert copied["counts"]["quality_observations"] == 2, "quality rows not included in counts"
|
|
quality = store.rows("SELECT id,data_json FROM quality_observations ORDER BY ordinal")
|
|
assert [row["id"] for row in quality] == ["quality-first", "quality-second"], "quality insertion order not preserved"
|
|
assert [json.loads(row["data_json"])["fixture"] for row in quality] == ["quality-first", "quality-second"], "quality payload changed"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest and Path(copied["backup"]).is_file(), "original/backup missing or modified"
|
|
with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup:
|
|
assert backup.execute("SELECT count(*) FROM quality_observations").fetchone()[0] == 2, "backup quality rows absent"
|
|
checks.append("new quality records, payload, insertion order, original and consistent backup preserved")
|
|
|
|
home = run_root / "legacy-source"
|
|
original = seed_source(home, quality=False)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["counts"]["quality_observations"] == 0, "legacy missing table must migrate as empty"
|
|
assert store.one("SELECT count(*) AS n FROM candidates")["n"] == 1, "legacy core data lost"
|
|
assert store.one("SELECT count(*) AS n FROM quality_observations")["n"] == 0, "legacy quality table not empty"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "legacy source modified"
|
|
checks.append("legacy SQLite with no quality table migrates core records and reports zero quality rows")
|
|
|
|
# Prove that the additive table itself participates in non-empty guards.
|
|
# Remove only its foreign keys in this owned fixture schema to seed an
|
|
# otherwise empty target. Production schemas and triggers stay untouched.
|
|
home = run_root / "nonempty-source"
|
|
original = seed_source(home, quality=False)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
with store.connect(immediate=True) as conn:
|
|
for constraint in ("validation_id", "candidate_id", "task_id"):
|
|
conn.execute(sql.SQL("ALTER TABLE quality_observations DROP CONSTRAINT {}").format(sql.Identifier("quality_observations_" + constraint + "_fkey")))
|
|
conn.execute("INSERT INTO quality_observations(id,validation_id,candidate_id,task_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6)",
|
|
("preexisting-quality", "fixture-only", "fixture-only", "fixture-only", "{}", "fixture"))
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, SystemExit) and "already contains records" in str(error), "quality-only nonempty target not rejected"
|
|
assert store.one("SELECT count(*) AS n FROM operators")["n"] == 0, "nonempty guard wrote core data"
|
|
assert store.one("SELECT id FROM quality_observations")["id"] == "preexisting-quality", "existing target record replaced"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "guard changed source"
|
|
checks.append("target containing only quality fixtures rejects migration without copying or replacing data")
|
|
|
|
home = run_root / "rollback-source"
|
|
original = seed_source(home, invalid_quality=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, StoreConflict), "invalid quality reference must fail"
|
|
assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES), "partial migration did not roll back"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite")), "failed migration did not preserve original/backup"
|
|
checks.append("quality reference failure rolls back all copied tables and preserves source/backup")
|
|
|
|
home = run_root / "discovery-source"
|
|
original = seed_source(home, discovery=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as source:
|
|
expected_discovery = source.execute("SELECT id,owner_id,candidate_id,validation_id,data_json,created_at FROM discovery_runs ORDER BY rowid").fetchall()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["result"] == "copied", "discovery source migration failed"
|
|
assert migrate_learning.TABLES.index("discovery_runs") < migrate_learning.TABLES.index("reading_runs"), "reading table must follow its discovery dependency"
|
|
assert copied["counts"]["discovery_runs"] == 2 and copied["counts"]["quality_observations"] == 2, "discovery or quality counts lost"
|
|
discovery = store.rows("SELECT id,owner_id,candidate_id,validation_id,data_json,created_at FROM discovery_runs ORDER BY ordinal")
|
|
actual_discovery = [tuple(row[column] for column in ("id", "owner_id", "candidate_id", "validation_id", "data_json", "created_at")) for row in discovery]
|
|
assert actual_discovery == expected_discovery, "discovery insertion order, nullable reference or exact payload changed"
|
|
assert [row["id"] for row in discovery] == ["discovery-first", "discovery-second"], "discovery must preserve rowid order rather than created_at sorting"
|
|
assert json.loads(discovery[0]["data_json"])["service"] == "装修", "discovery Unicode payload changed"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest and Path(copied["backup"]).is_file(), "discovery original/backup missing or modified"
|
|
with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup:
|
|
assert backup.execute("SELECT id,owner_id,candidate_id,validation_id,data_json,created_at FROM discovery_runs ORDER BY rowid").fetchall() == expected_discovery, "consistent backup lost discovery records"
|
|
checks.append("discovery rows, exact Unicode payload, nullable references, insertion order, original and backup preserved")
|
|
|
|
home = run_root / "legacy-with-quality-source"
|
|
original = seed_source(home, quality=True, discovery=False)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["counts"]["discovery_runs"] == 0, "legacy missing discovery table must migrate as empty"
|
|
assert store.one("SELECT count(*) AS n FROM discovery_runs")["n"] == 0, "legacy discovery target not empty"
|
|
assert store.one("SELECT count(*) AS n FROM quality_observations")["n"] == 2 and store.one("SELECT count(*) AS n FROM candidates")["n"] == 1, "legacy quality/core records lost"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "legacy source changed while skipping discovery table"
|
|
with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup:
|
|
assert not backup.execute("PRAGMA table_info(discovery_runs)").fetchall(), "backup must not invent a new source table"
|
|
checks.append("legacy SQLite with quality but no discovery table preserves existing data and reports zero discovery rows")
|
|
|
|
home = run_root / "discovery-nonempty-source"
|
|
original = seed_source(home)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
# This disposable target contains only discovery data. Remove its three
|
|
# foreign keys solely to construct the otherwise empty guard fixture.
|
|
with store.connect(immediate=True) as conn:
|
|
for constraint in ("owner_id", "candidate_id", "validation_id"):
|
|
conn.execute(sql.SQL("ALTER TABLE discovery_runs DROP CONSTRAINT {}").format(sql.Identifier("discovery_runs_" + constraint + "_fkey")))
|
|
conn.execute("INSERT INTO discovery_runs(id,owner_id,candidate_id,validation_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6)",
|
|
("preexisting-discovery", "fixture-only", "fixture-only", None, "{}", "fixture"))
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, SystemExit) and "already contains records" in str(error), "discovery-only nonempty target not rejected"
|
|
assert store.one("SELECT count(*) AS n FROM operators")["n"] == 0, "discovery nonempty guard copied core records"
|
|
assert store.one("SELECT id FROM discovery_runs")["id"] == "preexisting-discovery", "existing discovery record replaced"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "discovery guard changed source"
|
|
checks.append("target containing only discovery data rejects migration without copying or replacing records")
|
|
|
|
home = run_root / "discovery-rollback-source"
|
|
original = seed_source(home, discovery=True, invalid_discovery=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, StoreConflict), "invalid discovery foreign key must fail"
|
|
assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES), "discovery failure left a partial migration"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "discovery rollback changed source"
|
|
with sqlite3.connect(next((home / "backups").glob("*.sqlite")).as_uri() + "?mode=ro", uri=True) as backup:
|
|
assert backup.execute("SELECT count(*) FROM discovery_runs").fetchone()[0] == 2, "failed discovery migration did not retain a consistent backup"
|
|
checks.append("discovery foreign-key failure rolls back every preceding table and preserves original/backup")
|
|
|
|
home = run_root / "reading-source"
|
|
original = seed_source(home, discovery=True, reading=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
columns = ("id", "owner_id", "discovery_run_id", "candidate_id", "validation_id", "data_json", "created_at")
|
|
with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as source:
|
|
expected_readings = source.execute("SELECT " + ",".join(columns) + " FROM reading_runs ORDER BY rowid").fetchall()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["result"] == "copied" and copied["counts"]["reading_runs"] == 2, "reading source migration failed"
|
|
assert migrate_learning.TABLES.index("reading_runs") > migrate_learning.TABLES.index("discovery_runs"), "reading must follow discovery dependencies"
|
|
readings = store.rows("SELECT " + ",".join(columns) + " FROM reading_runs ORDER BY ordinal")
|
|
assert [tuple(row[column] for column in columns) for row in readings] == expected_readings, "reading payload, order or nullable validation changed"
|
|
assert [row["id"] for row in readings] == ["reading-first", "reading-second"], "reading insertion order not preserved"
|
|
assert json.loads(readings[0]["data_json"])["detail"]["title"] == "装修合成样本🏠", "reading Unicode changed"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest and Path(copied["backup"]).is_file(), "reading original or backup changed"
|
|
with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup:
|
|
assert backup.execute("SELECT " + ",".join(columns) + " FROM reading_runs ORDER BY rowid").fetchall() == expected_readings, "backup reading rows changed"
|
|
checks.append("reading rows, exact Unicode payload, nullable references, insertion order, original and backup preserved")
|
|
|
|
home = run_root / "legacy-with-discovery-source"
|
|
original = seed_source(home, discovery=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["counts"]["reading_runs"] == 0, "legacy missing reading table must migrate as empty"
|
|
assert store.one("SELECT count(*) AS n FROM reading_runs")["n"] == 0 and store.one("SELECT count(*) AS n FROM discovery_runs")["n"] == 2, "legacy reading/discovery records changed"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "legacy source modified"
|
|
with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup:
|
|
assert not backup.execute("PRAGMA table_info(reading_runs)").fetchall(), "backup invented a reading table"
|
|
checks.append("legacy SQLite with discovery but no reading table preserves prior records and reports zero reading rows")
|
|
|
|
home = run_root / "reading-nonempty-source"
|
|
original = seed_source(home)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
with store.connect(immediate=True) as conn:
|
|
for constraint in ("owner_id", "discovery_run_id", "candidate_id", "validation_id"):
|
|
conn.execute(sql.SQL("ALTER TABLE reading_runs DROP CONSTRAINT {}").format(sql.Identifier("reading_runs_" + constraint + "_fkey")))
|
|
conn.execute("INSERT INTO reading_runs(id,owner_id,discovery_run_id,candidate_id,validation_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)",
|
|
("preexisting-reading", "fixture-only", "fixture-only", "fixture-only", None, "{}", "fixture"))
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, SystemExit) and "already contains records" in str(error), "reading-only nonempty target not rejected"
|
|
assert store.one("SELECT count(*) AS n FROM operators")["n"] == 0 and store.one("SELECT id FROM reading_runs")["id"] == "preexisting-reading", "reading guard copied/replaced data"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "reading guard changed source"
|
|
checks.append("target containing only reading records rejects migration without copying or replacing data")
|
|
|
|
home = run_root / "reading-rollback-source"
|
|
original = seed_source(home, discovery=True, reading=True, invalid_reading=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, StoreConflict), "invalid reading reference must fail"
|
|
assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES), "reading failure did not roll back preceding tables"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite")), "reading failure changed original/backup"
|
|
checks.append("reading foreign-key failure rolls back every preceding table and preserves source and backup")
|
|
|
|
home = run_root / "analysis-source"
|
|
original = seed_source(home, discovery=True, reading=True, analysis=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
columns = ("id", "owner_id", "reading_run_id", "data_json", "created_at")
|
|
with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as source:
|
|
expected_analysis = source.execute("SELECT " + ",".join(columns) + " FROM analysis_runs ORDER BY rowid").fetchall()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["counts"]["analysis_runs"] == 2, "analysis source migration failed"
|
|
assert migrate_learning.TABLES.index("analysis_runs") > migrate_learning.TABLES.index("reading_runs"), "analysis must follow reading dependencies"
|
|
actual = store.rows("SELECT " + ",".join(columns) + " FROM analysis_runs ORDER BY ordinal")
|
|
assert [tuple(row[column] for column in columns) for row in actual] == expected_analysis, "analysis exact payload or insertion order changed"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "analysis source changed"
|
|
with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup:
|
|
assert backup.execute("SELECT " + ",".join(columns) + " FROM analysis_runs ORDER BY rowid").fetchall() == expected_analysis, "analysis backup changed"
|
|
for operation in ("UPDATE analysis_runs SET data_json='{}'", "DELETE FROM analysis_runs", "TRUNCATE analysis_runs CASCADE"):
|
|
try:
|
|
store.execute(operation)
|
|
except StoreConflict:
|
|
pass
|
|
else:
|
|
raise AssertionError("migrated analysis facts are mutable")
|
|
checks.append("analysis payload, source reference, insertion order, immutable records, original and consistent backup preserved")
|
|
|
|
home = run_root / "legacy-with-reading-source"
|
|
original = seed_source(home, discovery=True, reading=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["counts"]["analysis_runs"] == 0 and copied["counts"]["reading_runs"] == 2, "legacy missing analysis table lost reading history"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "legacy analysis omission modified source"
|
|
checks.append("legacy SQLite without analysis table preserves readings and reports zero analyses")
|
|
|
|
home = run_root / "analysis-nonempty-source"
|
|
original = seed_source(home)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
with store.connect(immediate=True) as conn:
|
|
for constraint in ("owner_id", "reading_run_id"):
|
|
conn.execute(sql.SQL("ALTER TABLE analysis_runs DROP CONSTRAINT {}").format(sql.Identifier("analysis_runs_" + constraint + "_fkey")))
|
|
conn.execute("INSERT INTO analysis_runs(id,owner_id,reading_run_id,data_json,created_at) VALUES($1,$2,$3,$4,$5)", ("preexisting-analysis", "fixture-only", "fixture-only", "{}", "fixture"))
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, SystemExit) and store.one("SELECT count(*) AS n FROM operators")["n"] == 0, "analysis-only nonempty target was not rejected before copy"
|
|
assert store.one("SELECT id FROM analysis_runs")["id"] == "preexisting-analysis" and sha256(original.read_bytes()).hexdigest() == initial_digest, "analysis guard replaced data or changed source"
|
|
checks.append("target with only analysis records rejects migration without replacement")
|
|
|
|
home = run_root / "analysis-rollback-source"
|
|
original = seed_source(home, discovery=True, reading=True, analysis=True, invalid_analysis=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, StoreConflict), "invalid analysis reference did not fail"
|
|
assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES), "analysis FK failure left partial preceding tables"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite")), "analysis rollback lost original or backup"
|
|
checks.append("invalid analysis reading FK rolls back every table and preserves original/backup")
|
|
|
|
home = run_root / "business-source"
|
|
original = seed_source(home, discovery=True, reading=True, analysis=True, business=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
columns = ("id", "owner_id", "kind", "parent_id", "data_json", "created_at")
|
|
with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as source:
|
|
expected_business = source.execute("SELECT " + ",".join(columns) + " FROM business_briefs ORDER BY rowid").fetchall()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["counts"]["business_briefs"] == 2, "business brief migration failed"
|
|
actual = store.rows("SELECT " + ",".join(columns) + " FROM business_briefs ORDER BY ordinal")
|
|
assert [tuple(row[column] for column in columns) for row in actual] == expected_business, "business payload, self-reference or order changed"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "business migration changed original"
|
|
assert store.one("SELECT version FROM schema_versions WHERE version=8")["version"] == 8, "schema 8 not recorded"
|
|
with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup:
|
|
assert backup.execute("SELECT " + ",".join(columns) + " FROM business_briefs ORDER BY rowid").fetchall() == expected_business
|
|
for operation in ("UPDATE business_briefs SET data_json='{}'", "DELETE FROM business_briefs", "TRUNCATE business_briefs CASCADE"):
|
|
try:
|
|
store.execute(operation)
|
|
except StoreConflict:
|
|
pass
|
|
else:
|
|
raise AssertionError("migrated business conditions are mutable")
|
|
checks.append("business drafts and confirmed rows preserve self-reference, exact payload, insertion order, immutable facts, original and backup")
|
|
|
|
home = run_root / "legacy-with-analysis-source"
|
|
original = seed_source(home, discovery=True, reading=True, analysis=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["counts"]["business_briefs"] == 0 and copied["counts"]["analysis_runs"] == 2
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "legacy no-business migration changed source"
|
|
checks.append("legacy source without business briefs preserves analysis history and reports zero business rows")
|
|
|
|
home = run_root / "business-nonempty-source"
|
|
original = seed_source(home)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
with store.connect(immediate=True) as conn:
|
|
conn.execute("ALTER TABLE business_briefs DROP CONSTRAINT business_briefs_owner_id_fkey")
|
|
conn.execute("INSERT INTO business_briefs(id,owner_id,kind,parent_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6)",
|
|
("preexisting-brief", "fixture-only", "parse", None, "{}", "fixture"))
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, SystemExit) and store.one("SELECT count(*) AS n FROM operators")["n"] == 0
|
|
assert store.one("SELECT id FROM business_briefs")["id"] == "preexisting-brief" and sha256(original.read_bytes()).hexdigest() == initial_digest
|
|
checks.append("target containing only business briefs rejects migration without copying or replacement")
|
|
|
|
home = run_root / "business-rollback-source"
|
|
original = seed_source(home, discovery=True, reading=True, analysis=True, business=True, invalid_business=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, StoreConflict), "invalid business parent must fail"
|
|
assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES)
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite"))
|
|
checks.append("invalid business parent self-reference rolls back every preceding table and preserves original/backup")
|
|
|
|
home = run_root / "reviews-source"
|
|
original = seed_source(home, discovery=True, reading=True, analysis=True, business=True, reviews=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
columns = ("id", "owner_id", "analysis_run_id", "source_ref", "previous_id", "data_json", "created_at")
|
|
with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as source:
|
|
expected_reviews = source.execute("SELECT " + ",".join(columns) + " FROM analysis_reviews ORDER BY rowid").fetchall()
|
|
review_store = target(home)
|
|
copied, error = run_migration(home, database_url, review_store.schema)
|
|
assert error is None and copied["counts"]["analysis_reviews"] == 4, "analysis review migration failed"
|
|
assert migrate_learning.TABLES.index("analysis_reviews") > migrate_learning.TABLES.index("analysis_runs"), "reviews precede their analysis dependency"
|
|
actual = review_store.rows("SELECT " + ",".join(columns) + " FROM analysis_reviews ORDER BY ordinal")
|
|
assert [tuple(row[column] for column in columns) for row in actual] == expected_reviews, "review source, revision order or exact payload changed"
|
|
assert review_store.one("SELECT version FROM schema_versions WHERE version=9")["version"] == 9, "schema 9 not recorded"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest, "review migration changed original"
|
|
with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup:
|
|
assert backup.execute("SELECT " + ",".join(columns) + " FROM analysis_reviews ORDER BY rowid").fetchall() == expected_reviews
|
|
for operation in ("UPDATE analysis_reviews SET data_json='{}'", "DELETE FROM analysis_reviews", "TRUNCATE analysis_reviews", "TRUNCATE analysis_runs CASCADE"):
|
|
try:
|
|
review_store.execute(operation)
|
|
except StoreConflict:
|
|
pass
|
|
else:
|
|
raise AssertionError("migrated review facts are mutable")
|
|
assert review_store.rows("SELECT " + ",".join(columns) + " FROM analysis_reviews ORDER BY ordinal") == actual
|
|
checks.append("review sources, Unicode payload, self-revision chains and insertion order, immutable facts, original and consistent backup preserved")
|
|
|
|
home = run_root / "legacy-with-business-source"
|
|
original = seed_source(home, discovery=True, reading=True, analysis=True, business=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert error is None and copied["counts"]["analysis_reviews"] == 0 and copied["counts"]["analysis_runs"] == 2 and copied["counts"]["business_briefs"] == 2
|
|
assert store.one("SELECT count(*) AS n FROM analysis_reviews")["n"] == 0 and sha256(original.read_bytes()).hexdigest() == initial_digest
|
|
with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup:
|
|
assert not backup.execute("PRAGMA table_info(analysis_reviews)").fetchall(), "backup invented reviews in legacy source"
|
|
checks.append("legacy source without analysis reviews preserves business and analysis history and reports zero review rows")
|
|
|
|
home = run_root / "reviews-nonempty-source"
|
|
original = seed_source(home)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
with store.connect(immediate=True) as conn:
|
|
for constraint in ("owner_id", "analysis_run_id"):
|
|
conn.execute(sql.SQL("ALTER TABLE analysis_reviews DROP CONSTRAINT {}").format(sql.Identifier("analysis_reviews_" + constraint + "_fkey")))
|
|
conn.execute("INSERT INTO analysis_reviews(id,owner_id,analysis_run_id,source_ref,previous_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)",
|
|
("preexisting-review", "fixture-only", "fixture-only", "note", None, "{}", "fixture"))
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, SystemExit) and store.one("SELECT count(*) AS n FROM operators")["n"] == 0
|
|
assert store.one("SELECT id FROM analysis_reviews")["id"] == "preexisting-review" and sha256(original.read_bytes()).hexdigest() == initial_digest
|
|
checks.append("target containing only analysis reviews rejects migration without copying or replacement")
|
|
|
|
home = run_root / "reviews-rollback-source"
|
|
original = seed_source(home, discovery=True, reading=True, analysis=True, business=True, reviews=True, invalid_review_parent=True)
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, StoreConflict), "invalid review previous self-reference must fail"
|
|
assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES)
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite"))
|
|
checks.append("invalid review previous self-reference rolls back every preceding table and preserves original/backup")
|
|
|
|
# Exercise the actual unique indexes with simultaneous independent
|
|
# connections, not only the service's in-process concurrency guard.
|
|
def review_constraints(store):
|
|
query = "INSERT INTO analysis_reviews(id,owner_id,analysis_run_id,source_ref,previous_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)"
|
|
conflict = StoreConflict if store.backend == "postgresql" else sqlite3.IntegrityError
|
|
def insert(values):
|
|
try:
|
|
store.execute(query, values)
|
|
return "inserted"
|
|
except conflict:
|
|
return "conflict"
|
|
for ref, previous in (("race-first", None), ("note", "review-third")):
|
|
jobs = [("race-" + uuid4().hex, "owner", "analysis-first", ref, previous, "{}", "fixture") for _ in range(2)]
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
outcomes = list(pool.map(insert, jobs))
|
|
assert sorted(outcomes) == ["conflict", "inserted"], "concurrent review writes created two first versions or two successors"
|
|
rows_before = store.rows("SELECT " + ",".join(columns) + f" FROM analysis_reviews ORDER BY {store.order_column()}")
|
|
for operation in ("UPDATE analysis_reviews SET data_json='{}'", "DELETE FROM analysis_reviews"):
|
|
try:
|
|
store.execute(operation)
|
|
except conflict:
|
|
pass
|
|
else:
|
|
raise AssertionError("review append-only guard failed")
|
|
assert store.rows("SELECT " + ",".join(columns) + f" FROM analysis_reviews ORDER BY {store.order_column()}") == rows_before
|
|
review_constraints(review_store)
|
|
checks.append("PostgreSQL simultaneous writes permit one first revision and one successor only, retaining immutable history")
|
|
home = run_root / "reviews-sqlite-constraints"
|
|
seed_source(home, discovery=True, reading=True, analysis=True, reviews=True)
|
|
review_constraints(Store(home))
|
|
checks.append("SQLite simultaneous writes enforce both partial unique indexes and immutable revision history")
|
|
|
|
home = run_root / "missing-core-source"
|
|
original = seed_source(home)
|
|
with closing(sqlite3.connect(original)) as source, source:
|
|
source.execute("DROP TABLE auth_attempts")
|
|
initial_digest = sha256(original.read_bytes()).hexdigest()
|
|
store = target(home)
|
|
copied, error = run_migration(home, database_url, store.schema)
|
|
assert copied is None and isinstance(error, sqlite3.OperationalError), "missing core table must remain a hard failure"
|
|
assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES), "missing core table did not roll back earlier rows"
|
|
assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite")), "missing-core failure lost source/backup"
|
|
checks.append("missing core SQLite table still fails and rolls back instead of being treated as an additive omission")
|
|
finally:
|
|
with psycopg.connect(database_url, connect_timeout=5) as connection:
|
|
for schema in schemas:
|
|
connection.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema)))
|
|
report = {"passed": len(checks), "checks": checks, "test_database_only": True,
|
|
"random_schemas_dropped": len(schemas), "real_platform_data": False}
|
|
(run_root / "report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(json.dumps({**report, "artifacts": str(run_root)}, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except psycopg.Error:
|
|
raise SystemExit("Isolated migration PostgreSQL checks failed; driver details withheld.") from None
|