"""Opt-in migration regression; dedicated TEST database and disposable schemas only.""" from __future__ import annotations from contextlib import closing, redirect_stdout from concurrent.futures import ThreadPoolExecutor from hashlib import sha256 import io import json import os from pathlib import Path import sqlite3 import tempfile from unittest.mock import patch from uuid import uuid4 import psycopg from psycopg import sql from apps.server.db import PostgresStore, Store, StoreConflict from scripts import migrate_learning ROOT = Path(__file__).resolve().parents[1] def seed_source(home: Path, *, quality: bool = True, invalid_quality: bool = False, discovery: bool = False, invalid_discovery: bool = False, reading: bool = False, invalid_reading: bool = False, analysis: bool = False, invalid_analysis: bool = False, business: bool = False, invalid_business: bool = False, reviews: bool = False, invalid_review_parent: bool = False) -> Path: source = Store(home) stamp = "2026-10-24T00:00:00Z" with source.connect(immediate=True) as conn: conn.execute("INSERT INTO operators VALUES($1,$2,$3,$4)", ("owner", "fixture", "not-a-real-password-hash", stamp)) conn.execute("INSERT INTO learning_tasks VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)", ("task", "owner", "migration fixture", "demo", "http://127.0.0.1/fixture", "fixture only", "test", "{}", "paused", None, stamp, stamp)) conn.execute("INSERT INTO candidates VALUES($1,$2,$3,$4,$5,$6,$7)", ("candidate", "task", "fixture", 1, "fixture-digest", "{}", stamp)) conn.execute("INSERT INTO validations VALUES($1,$2,$3,$4,$5,$6,$7)", ("validation", "candidate", "task", "browser", "passed", "{}", stamp)) if quality: # A deliberately malformed source fixture exercises transaction rollback. # SQLite foreign-key enforcement is connection-local and disabled here. with closing(sqlite3.connect(source.path)) as conn, conn: for key, created in [("quality-first", "2026-10-24T02:00:00Z"), ("quality-second", "2026-10-24T01:00:00Z")]: conn.execute("INSERT INTO quality_observations VALUES(?,?,?,?,?,?)", (key, "invalid-validation" if invalid_quality else "validation", "candidate", "task", json.dumps({"status": "no_issue", "fixture": key}), created)) if not quality: # A real legacy SQLite installation had no quality table at all. with closing(sqlite3.connect(source.path)) as conn, conn: conn.execute("DROP TABLE quality_observations") if discovery: with closing(sqlite3.connect(source.path)) as conn, conn: for key, created, validation in [("discovery-first", "2026-10-24T04:00:00Z", "validation"), ("discovery-second", "2026-10-24T03:00:00Z", None)]: payload = {"fixture": key, "service": "装修", "region": None, "selection": [{"source": "http://127.0.0.1/fixture/note", "basis": ["合成迁移样本"]}]} conn.execute("INSERT INTO discovery_runs VALUES(?,?,?,?,?,?)", (key, "owner", "candidate", "invalid-validation" if invalid_discovery else validation, json.dumps(payload, ensure_ascii=False), created)) else: with closing(sqlite3.connect(source.path)) as conn, conn: conn.execute("DROP TABLE discovery_runs") if reading: assert discovery, "reading fixtures require their source discovery record" with closing(sqlite3.connect(source.path)) as conn, conn: for key, created, validation in [("reading-first", "2026-10-24T06:00:00Z", "validation"), ("reading-second", "2026-10-24T05:00:00Z", None)]: payload = {"fixture": key, "status": "partial", "note_id": "a" * 24, "detail": {"title": "装修合成样本🏠", "body": "仅迁移测试"}, "comments": None, "errors": ["合成未覆盖分支"]} conn.execute("INSERT INTO reading_runs VALUES(?,?,?,?,?,?,?)", (key, "owner", "discovery-first", "candidate", "invalid-validation" if invalid_reading else validation, json.dumps(payload, ensure_ascii=False), created)) else: with closing(sqlite3.connect(source.path)) as conn, conn: conn.execute("DROP TABLE reading_runs") if analysis: assert reading, "analysis fixtures require their reading source" with closing(sqlite3.connect(source.path)) as conn, conn: for key, created in [("analysis-first", "2026-10-24T08:00:00Z"), ("analysis-second", "2026-10-24T07:00:00Z")]: payload = {"status": "completed", "fixture": key, "results": [{"source_ref": "note", "demand": "insufficient", "evidence": []}], "cost": {"status": "unknown"}} conn.execute("INSERT INTO analysis_runs VALUES(?,?,?,?,?)", (key, "owner", "invalid-reading" if invalid_analysis else "reading-first", json.dumps(payload, ensure_ascii=False), created)) else: with closing(sqlite3.connect(source.path)) as conn, conn: conn.execute("DROP TABLE analysis_runs") if business: with closing(sqlite3.connect(source.path)) as conn, conn: for key, kind, parent, created in [("brief-parse", "parse", None, "2026-10-24T10:00:00Z"), ("brief-confirm", "confirmed", "invalid-parent" if invalid_business else "brief-parse", "2026-10-24T09:00:00Z")]: payload = {"status": "confirmed" if kind == "confirmed" else "needs_clarification", "fixture": key, "conditions": {"region": None, "region_scope": "unspecified"}, "cost": {"status": "unknown"}} conn.execute("INSERT INTO business_briefs(id,owner_id,kind,parent_id,data_json,created_at) VALUES(?,?,?,?,?,?)", (key, "owner", kind, parent, json.dumps(payload, ensure_ascii=False), created)) else: with closing(sqlite3.connect(source.path)) as conn, conn: conn.execute("DROP TABLE business_briefs") if reviews: assert analysis, "review fixtures require their analysis source" with closing(sqlite3.connect(source.path)) as conn, conn: for key, ref, previous, created in [ ("review-first", "note", None, "2026-10-24T14:00:00Z"), ("review-second", "note", "invalid-previous" if invalid_review_parent else "review-first", "2026-10-24T13:00:00Z"), ("review-comment", "comment_1", None, "2026-10-24T12:00:00Z"), ("review-third", "note", "review-second", "2026-10-24T11:00:00Z"), ]: payload = {"fixture": key, "reviewer_source": "maintainer", "decision": "corrected", "note": "仅合成人工核对🏠", "labels": {"role": "unknown", "own_demand": "insufficient"}} conn.execute("INSERT INTO analysis_reviews(id,owner_id,analysis_run_id,source_ref,previous_id,data_json,created_at) VALUES(?,?,?,?,?,?,?)", (key, "owner", "analysis-first", ref, previous, json.dumps(payload, ensure_ascii=False), created)) else: with closing(sqlite3.connect(source.path)) as conn, conn: conn.execute("DROP TABLE analysis_reviews") return source.path def run_migration(home: Path, database_url: str, schema: str) -> tuple[dict | None, BaseException | None]: output = io.StringIO() with patch.dict(os.environ, {"ASSISTANT_DATABASE_URL": database_url, "ASSISTANT_DATABASE_SCHEMA": schema, "ASSISTANT_LEARNING_HOME": str(home)}), redirect_stdout(output): try: migrate_learning.main() except (SystemExit, StoreConflict, RuntimeError, sqlite3.Error) as error: return None, error return json.loads(output.getvalue()), None def main() -> None: database_url = os.environ.get("ASSISTANT_TEST_DATABASE_URL", "") if not database_url: raise SystemExit("Configure the dedicated ASSISTANT_TEST_DATABASE_URL before running migration regression.") (ROOT / ".tmp").mkdir(exist_ok=True) run_root = Path(tempfile.mkdtemp(prefix="migration-regression-", dir=ROOT / ".tmp")) schemas: list[str] = [] checks: list[str] = [] def target(home: Path) -> PostgresStore: schema = "assistant_learning_migration_" + uuid4().hex schemas.append(schema) return PostgresStore(home, database_url, schema=schema) try: home = run_root / "new-source" original = seed_source(home) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["result"] == "copied", "new source migration failed" assert copied["counts"]["quality_observations"] == 2, "quality rows not included in counts" quality = store.rows("SELECT id,data_json FROM quality_observations ORDER BY ordinal") assert [row["id"] for row in quality] == ["quality-first", "quality-second"], "quality insertion order not preserved" assert [json.loads(row["data_json"])["fixture"] for row in quality] == ["quality-first", "quality-second"], "quality payload changed" assert sha256(original.read_bytes()).hexdigest() == initial_digest and Path(copied["backup"]).is_file(), "original/backup missing or modified" with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup: assert backup.execute("SELECT count(*) FROM quality_observations").fetchone()[0] == 2, "backup quality rows absent" checks.append("new quality records, payload, insertion order, original and consistent backup preserved") home = run_root / "legacy-source" original = seed_source(home, quality=False) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["counts"]["quality_observations"] == 0, "legacy missing table must migrate as empty" assert store.one("SELECT count(*) AS n FROM candidates")["n"] == 1, "legacy core data lost" assert store.one("SELECT count(*) AS n FROM quality_observations")["n"] == 0, "legacy quality table not empty" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "legacy source modified" checks.append("legacy SQLite with no quality table migrates core records and reports zero quality rows") # Prove that the additive table itself participates in non-empty guards. # Remove only its foreign keys in this owned fixture schema to seed an # otherwise empty target. Production schemas and triggers stay untouched. home = run_root / "nonempty-source" original = seed_source(home, quality=False) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) with store.connect(immediate=True) as conn: for constraint in ("validation_id", "candidate_id", "task_id"): conn.execute(sql.SQL("ALTER TABLE quality_observations DROP CONSTRAINT {}").format(sql.Identifier("quality_observations_" + constraint + "_fkey"))) conn.execute("INSERT INTO quality_observations(id,validation_id,candidate_id,task_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6)", ("preexisting-quality", "fixture-only", "fixture-only", "fixture-only", "{}", "fixture")) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, SystemExit) and "already contains records" in str(error), "quality-only nonempty target not rejected" assert store.one("SELECT count(*) AS n FROM operators")["n"] == 0, "nonempty guard wrote core data" assert store.one("SELECT id FROM quality_observations")["id"] == "preexisting-quality", "existing target record replaced" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "guard changed source" checks.append("target containing only quality fixtures rejects migration without copying or replacing data") home = run_root / "rollback-source" original = seed_source(home, invalid_quality=True) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, StoreConflict), "invalid quality reference must fail" assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES), "partial migration did not roll back" assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite")), "failed migration did not preserve original/backup" checks.append("quality reference failure rolls back all copied tables and preserves source/backup") home = run_root / "discovery-source" original = seed_source(home, discovery=True) initial_digest = sha256(original.read_bytes()).hexdigest() with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as source: expected_discovery = source.execute("SELECT id,owner_id,candidate_id,validation_id,data_json,created_at FROM discovery_runs ORDER BY rowid").fetchall() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["result"] == "copied", "discovery source migration failed" assert migrate_learning.TABLES.index("discovery_runs") < migrate_learning.TABLES.index("reading_runs"), "reading table must follow its discovery dependency" assert copied["counts"]["discovery_runs"] == 2 and copied["counts"]["quality_observations"] == 2, "discovery or quality counts lost" discovery = store.rows("SELECT id,owner_id,candidate_id,validation_id,data_json,created_at FROM discovery_runs ORDER BY ordinal") actual_discovery = [tuple(row[column] for column in ("id", "owner_id", "candidate_id", "validation_id", "data_json", "created_at")) for row in discovery] assert actual_discovery == expected_discovery, "discovery insertion order, nullable reference or exact payload changed" assert [row["id"] for row in discovery] == ["discovery-first", "discovery-second"], "discovery must preserve rowid order rather than created_at sorting" assert json.loads(discovery[0]["data_json"])["service"] == "装修", "discovery Unicode payload changed" assert sha256(original.read_bytes()).hexdigest() == initial_digest and Path(copied["backup"]).is_file(), "discovery original/backup missing or modified" with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup: assert backup.execute("SELECT id,owner_id,candidate_id,validation_id,data_json,created_at FROM discovery_runs ORDER BY rowid").fetchall() == expected_discovery, "consistent backup lost discovery records" checks.append("discovery rows, exact Unicode payload, nullable references, insertion order, original and backup preserved") home = run_root / "legacy-with-quality-source" original = seed_source(home, quality=True, discovery=False) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["counts"]["discovery_runs"] == 0, "legacy missing discovery table must migrate as empty" assert store.one("SELECT count(*) AS n FROM discovery_runs")["n"] == 0, "legacy discovery target not empty" assert store.one("SELECT count(*) AS n FROM quality_observations")["n"] == 2 and store.one("SELECT count(*) AS n FROM candidates")["n"] == 1, "legacy quality/core records lost" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "legacy source changed while skipping discovery table" with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup: assert not backup.execute("PRAGMA table_info(discovery_runs)").fetchall(), "backup must not invent a new source table" checks.append("legacy SQLite with quality but no discovery table preserves existing data and reports zero discovery rows") home = run_root / "discovery-nonempty-source" original = seed_source(home) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) # This disposable target contains only discovery data. Remove its three # foreign keys solely to construct the otherwise empty guard fixture. with store.connect(immediate=True) as conn: for constraint in ("owner_id", "candidate_id", "validation_id"): conn.execute(sql.SQL("ALTER TABLE discovery_runs DROP CONSTRAINT {}").format(sql.Identifier("discovery_runs_" + constraint + "_fkey"))) conn.execute("INSERT INTO discovery_runs(id,owner_id,candidate_id,validation_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6)", ("preexisting-discovery", "fixture-only", "fixture-only", None, "{}", "fixture")) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, SystemExit) and "already contains records" in str(error), "discovery-only nonempty target not rejected" assert store.one("SELECT count(*) AS n FROM operators")["n"] == 0, "discovery nonempty guard copied core records" assert store.one("SELECT id FROM discovery_runs")["id"] == "preexisting-discovery", "existing discovery record replaced" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "discovery guard changed source" checks.append("target containing only discovery data rejects migration without copying or replacing records") home = run_root / "discovery-rollback-source" original = seed_source(home, discovery=True, invalid_discovery=True) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, StoreConflict), "invalid discovery foreign key must fail" assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES), "discovery failure left a partial migration" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "discovery rollback changed source" with sqlite3.connect(next((home / "backups").glob("*.sqlite")).as_uri() + "?mode=ro", uri=True) as backup: assert backup.execute("SELECT count(*) FROM discovery_runs").fetchone()[0] == 2, "failed discovery migration did not retain a consistent backup" checks.append("discovery foreign-key failure rolls back every preceding table and preserves original/backup") home = run_root / "reading-source" original = seed_source(home, discovery=True, reading=True) initial_digest = sha256(original.read_bytes()).hexdigest() columns = ("id", "owner_id", "discovery_run_id", "candidate_id", "validation_id", "data_json", "created_at") with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as source: expected_readings = source.execute("SELECT " + ",".join(columns) + " FROM reading_runs ORDER BY rowid").fetchall() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["result"] == "copied" and copied["counts"]["reading_runs"] == 2, "reading source migration failed" assert migrate_learning.TABLES.index("reading_runs") > migrate_learning.TABLES.index("discovery_runs"), "reading must follow discovery dependencies" readings = store.rows("SELECT " + ",".join(columns) + " FROM reading_runs ORDER BY ordinal") assert [tuple(row[column] for column in columns) for row in readings] == expected_readings, "reading payload, order or nullable validation changed" assert [row["id"] for row in readings] == ["reading-first", "reading-second"], "reading insertion order not preserved" assert json.loads(readings[0]["data_json"])["detail"]["title"] == "装修合成样本🏠", "reading Unicode changed" assert sha256(original.read_bytes()).hexdigest() == initial_digest and Path(copied["backup"]).is_file(), "reading original or backup changed" with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup: assert backup.execute("SELECT " + ",".join(columns) + " FROM reading_runs ORDER BY rowid").fetchall() == expected_readings, "backup reading rows changed" checks.append("reading rows, exact Unicode payload, nullable references, insertion order, original and backup preserved") home = run_root / "legacy-with-discovery-source" original = seed_source(home, discovery=True) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["counts"]["reading_runs"] == 0, "legacy missing reading table must migrate as empty" assert store.one("SELECT count(*) AS n FROM reading_runs")["n"] == 0 and store.one("SELECT count(*) AS n FROM discovery_runs")["n"] == 2, "legacy reading/discovery records changed" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "legacy source modified" with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup: assert not backup.execute("PRAGMA table_info(reading_runs)").fetchall(), "backup invented a reading table" checks.append("legacy SQLite with discovery but no reading table preserves prior records and reports zero reading rows") home = run_root / "reading-nonempty-source" original = seed_source(home) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) with store.connect(immediate=True) as conn: for constraint in ("owner_id", "discovery_run_id", "candidate_id", "validation_id"): conn.execute(sql.SQL("ALTER TABLE reading_runs DROP CONSTRAINT {}").format(sql.Identifier("reading_runs_" + constraint + "_fkey"))) conn.execute("INSERT INTO reading_runs(id,owner_id,discovery_run_id,candidate_id,validation_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)", ("preexisting-reading", "fixture-only", "fixture-only", "fixture-only", None, "{}", "fixture")) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, SystemExit) and "already contains records" in str(error), "reading-only nonempty target not rejected" assert store.one("SELECT count(*) AS n FROM operators")["n"] == 0 and store.one("SELECT id FROM reading_runs")["id"] == "preexisting-reading", "reading guard copied/replaced data" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "reading guard changed source" checks.append("target containing only reading records rejects migration without copying or replacing data") home = run_root / "reading-rollback-source" original = seed_source(home, discovery=True, reading=True, invalid_reading=True) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, StoreConflict), "invalid reading reference must fail" assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES), "reading failure did not roll back preceding tables" assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite")), "reading failure changed original/backup" checks.append("reading foreign-key failure rolls back every preceding table and preserves source and backup") home = run_root / "analysis-source" original = seed_source(home, discovery=True, reading=True, analysis=True) initial_digest = sha256(original.read_bytes()).hexdigest() columns = ("id", "owner_id", "reading_run_id", "data_json", "created_at") with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as source: expected_analysis = source.execute("SELECT " + ",".join(columns) + " FROM analysis_runs ORDER BY rowid").fetchall() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["counts"]["analysis_runs"] == 2, "analysis source migration failed" assert migrate_learning.TABLES.index("analysis_runs") > migrate_learning.TABLES.index("reading_runs"), "analysis must follow reading dependencies" actual = store.rows("SELECT " + ",".join(columns) + " FROM analysis_runs ORDER BY ordinal") assert [tuple(row[column] for column in columns) for row in actual] == expected_analysis, "analysis exact payload or insertion order changed" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "analysis source changed" with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup: assert backup.execute("SELECT " + ",".join(columns) + " FROM analysis_runs ORDER BY rowid").fetchall() == expected_analysis, "analysis backup changed" for operation in ("UPDATE analysis_runs SET data_json='{}'", "DELETE FROM analysis_runs", "TRUNCATE analysis_runs CASCADE"): try: store.execute(operation) except StoreConflict: pass else: raise AssertionError("migrated analysis facts are mutable") checks.append("analysis payload, source reference, insertion order, immutable records, original and consistent backup preserved") home = run_root / "legacy-with-reading-source" original = seed_source(home, discovery=True, reading=True) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["counts"]["analysis_runs"] == 0 and copied["counts"]["reading_runs"] == 2, "legacy missing analysis table lost reading history" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "legacy analysis omission modified source" checks.append("legacy SQLite without analysis table preserves readings and reports zero analyses") home = run_root / "analysis-nonempty-source" original = seed_source(home) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) with store.connect(immediate=True) as conn: for constraint in ("owner_id", "reading_run_id"): conn.execute(sql.SQL("ALTER TABLE analysis_runs DROP CONSTRAINT {}").format(sql.Identifier("analysis_runs_" + constraint + "_fkey"))) conn.execute("INSERT INTO analysis_runs(id,owner_id,reading_run_id,data_json,created_at) VALUES($1,$2,$3,$4,$5)", ("preexisting-analysis", "fixture-only", "fixture-only", "{}", "fixture")) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, SystemExit) and store.one("SELECT count(*) AS n FROM operators")["n"] == 0, "analysis-only nonempty target was not rejected before copy" assert store.one("SELECT id FROM analysis_runs")["id"] == "preexisting-analysis" and sha256(original.read_bytes()).hexdigest() == initial_digest, "analysis guard replaced data or changed source" checks.append("target with only analysis records rejects migration without replacement") home = run_root / "analysis-rollback-source" original = seed_source(home, discovery=True, reading=True, analysis=True, invalid_analysis=True) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, StoreConflict), "invalid analysis reference did not fail" assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES), "analysis FK failure left partial preceding tables" assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite")), "analysis rollback lost original or backup" checks.append("invalid analysis reading FK rolls back every table and preserves original/backup") home = run_root / "business-source" original = seed_source(home, discovery=True, reading=True, analysis=True, business=True) initial_digest = sha256(original.read_bytes()).hexdigest() columns = ("id", "owner_id", "kind", "parent_id", "data_json", "created_at") with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as source: expected_business = source.execute("SELECT " + ",".join(columns) + " FROM business_briefs ORDER BY rowid").fetchall() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["counts"]["business_briefs"] == 2, "business brief migration failed" actual = store.rows("SELECT " + ",".join(columns) + " FROM business_briefs ORDER BY ordinal") assert [tuple(row[column] for column in columns) for row in actual] == expected_business, "business payload, self-reference or order changed" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "business migration changed original" assert store.one("SELECT version FROM schema_versions WHERE version=8")["version"] == 8, "schema 8 not recorded" with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup: assert backup.execute("SELECT " + ",".join(columns) + " FROM business_briefs ORDER BY rowid").fetchall() == expected_business for operation in ("UPDATE business_briefs SET data_json='{}'", "DELETE FROM business_briefs", "TRUNCATE business_briefs CASCADE"): try: store.execute(operation) except StoreConflict: pass else: raise AssertionError("migrated business conditions are mutable") checks.append("business drafts and confirmed rows preserve self-reference, exact payload, insertion order, immutable facts, original and backup") home = run_root / "legacy-with-analysis-source" original = seed_source(home, discovery=True, reading=True, analysis=True) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["counts"]["business_briefs"] == 0 and copied["counts"]["analysis_runs"] == 2 assert sha256(original.read_bytes()).hexdigest() == initial_digest, "legacy no-business migration changed source" checks.append("legacy source without business briefs preserves analysis history and reports zero business rows") home = run_root / "business-nonempty-source" original = seed_source(home) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) with store.connect(immediate=True) as conn: conn.execute("ALTER TABLE business_briefs DROP CONSTRAINT business_briefs_owner_id_fkey") conn.execute("INSERT INTO business_briefs(id,owner_id,kind,parent_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6)", ("preexisting-brief", "fixture-only", "parse", None, "{}", "fixture")) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, SystemExit) and store.one("SELECT count(*) AS n FROM operators")["n"] == 0 assert store.one("SELECT id FROM business_briefs")["id"] == "preexisting-brief" and sha256(original.read_bytes()).hexdigest() == initial_digest checks.append("target containing only business briefs rejects migration without copying or replacement") home = run_root / "business-rollback-source" original = seed_source(home, discovery=True, reading=True, analysis=True, business=True, invalid_business=True) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, StoreConflict), "invalid business parent must fail" assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES) assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite")) checks.append("invalid business parent self-reference rolls back every preceding table and preserves original/backup") home = run_root / "reviews-source" original = seed_source(home, discovery=True, reading=True, analysis=True, business=True, reviews=True) initial_digest = sha256(original.read_bytes()).hexdigest() columns = ("id", "owner_id", "analysis_run_id", "source_ref", "previous_id", "data_json", "created_at") with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as source: expected_reviews = source.execute("SELECT " + ",".join(columns) + " FROM analysis_reviews ORDER BY rowid").fetchall() review_store = target(home) copied, error = run_migration(home, database_url, review_store.schema) assert error is None and copied["counts"]["analysis_reviews"] == 4, "analysis review migration failed" assert migrate_learning.TABLES.index("analysis_reviews") > migrate_learning.TABLES.index("analysis_runs"), "reviews precede their analysis dependency" actual = review_store.rows("SELECT " + ",".join(columns) + " FROM analysis_reviews ORDER BY ordinal") assert [tuple(row[column] for column in columns) for row in actual] == expected_reviews, "review source, revision order or exact payload changed" assert review_store.one("SELECT version FROM schema_versions WHERE version=9")["version"] == 9, "schema 9 not recorded" assert sha256(original.read_bytes()).hexdigest() == initial_digest, "review migration changed original" with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup: assert backup.execute("SELECT " + ",".join(columns) + " FROM analysis_reviews ORDER BY rowid").fetchall() == expected_reviews for operation in ("UPDATE analysis_reviews SET data_json='{}'", "DELETE FROM analysis_reviews", "TRUNCATE analysis_reviews", "TRUNCATE analysis_runs CASCADE"): try: review_store.execute(operation) except StoreConflict: pass else: raise AssertionError("migrated review facts are mutable") assert review_store.rows("SELECT " + ",".join(columns) + " FROM analysis_reviews ORDER BY ordinal") == actual checks.append("review sources, Unicode payload, self-revision chains and insertion order, immutable facts, original and consistent backup preserved") home = run_root / "legacy-with-business-source" original = seed_source(home, discovery=True, reading=True, analysis=True, business=True) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert error is None and copied["counts"]["analysis_reviews"] == 0 and copied["counts"]["analysis_runs"] == 2 and copied["counts"]["business_briefs"] == 2 assert store.one("SELECT count(*) AS n FROM analysis_reviews")["n"] == 0 and sha256(original.read_bytes()).hexdigest() == initial_digest with sqlite3.connect(Path(copied["backup"]).as_uri() + "?mode=ro", uri=True) as backup: assert not backup.execute("PRAGMA table_info(analysis_reviews)").fetchall(), "backup invented reviews in legacy source" checks.append("legacy source without analysis reviews preserves business and analysis history and reports zero review rows") home = run_root / "reviews-nonempty-source" original = seed_source(home) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) with store.connect(immediate=True) as conn: for constraint in ("owner_id", "analysis_run_id"): conn.execute(sql.SQL("ALTER TABLE analysis_reviews DROP CONSTRAINT {}").format(sql.Identifier("analysis_reviews_" + constraint + "_fkey"))) conn.execute("INSERT INTO analysis_reviews(id,owner_id,analysis_run_id,source_ref,previous_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)", ("preexisting-review", "fixture-only", "fixture-only", "note", None, "{}", "fixture")) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, SystemExit) and store.one("SELECT count(*) AS n FROM operators")["n"] == 0 assert store.one("SELECT id FROM analysis_reviews")["id"] == "preexisting-review" and sha256(original.read_bytes()).hexdigest() == initial_digest checks.append("target containing only analysis reviews rejects migration without copying or replacement") home = run_root / "reviews-rollback-source" original = seed_source(home, discovery=True, reading=True, analysis=True, business=True, reviews=True, invalid_review_parent=True) initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, StoreConflict), "invalid review previous self-reference must fail" assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES) assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite")) checks.append("invalid review previous self-reference rolls back every preceding table and preserves original/backup") # Exercise the actual unique indexes with simultaneous independent # connections, not only the service's in-process concurrency guard. def review_constraints(store): query = "INSERT INTO analysis_reviews(id,owner_id,analysis_run_id,source_ref,previous_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)" conflict = StoreConflict if store.backend == "postgresql" else sqlite3.IntegrityError def insert(values): try: store.execute(query, values) return "inserted" except conflict: return "conflict" for ref, previous in (("race-first", None), ("note", "review-third")): jobs = [("race-" + uuid4().hex, "owner", "analysis-first", ref, previous, "{}", "fixture") for _ in range(2)] with ThreadPoolExecutor(max_workers=2) as pool: outcomes = list(pool.map(insert, jobs)) assert sorted(outcomes) == ["conflict", "inserted"], "concurrent review writes created two first versions or two successors" rows_before = store.rows("SELECT " + ",".join(columns) + f" FROM analysis_reviews ORDER BY {store.order_column()}") for operation in ("UPDATE analysis_reviews SET data_json='{}'", "DELETE FROM analysis_reviews"): try: store.execute(operation) except conflict: pass else: raise AssertionError("review append-only guard failed") assert store.rows("SELECT " + ",".join(columns) + f" FROM analysis_reviews ORDER BY {store.order_column()}") == rows_before review_constraints(review_store) checks.append("PostgreSQL simultaneous writes permit one first revision and one successor only, retaining immutable history") home = run_root / "reviews-sqlite-constraints" seed_source(home, discovery=True, reading=True, analysis=True, reviews=True) review_constraints(Store(home)) checks.append("SQLite simultaneous writes enforce both partial unique indexes and immutable revision history") home = run_root / "missing-core-source" original = seed_source(home) with closing(sqlite3.connect(original)) as source, source: source.execute("DROP TABLE auth_attempts") initial_digest = sha256(original.read_bytes()).hexdigest() store = target(home) copied, error = run_migration(home, database_url, store.schema) assert copied is None and isinstance(error, sqlite3.OperationalError), "missing core table must remain a hard failure" assert all(store.one(f"SELECT count(*) AS n FROM {table}")["n"] == 0 for table in migrate_learning.TABLES), "missing core table did not roll back earlier rows" assert sha256(original.read_bytes()).hexdigest() == initial_digest and list((home / "backups").glob("*.sqlite")), "missing-core failure lost source/backup" checks.append("missing core SQLite table still fails and rolls back instead of being treated as an additive omission") finally: with psycopg.connect(database_url, connect_timeout=5) as connection: for schema in schemas: connection.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema))) report = {"passed": len(checks), "checks": checks, "test_database_only": True, "random_schemas_dropped": len(schemas), "real_platform_data": False} (run_root / "report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") print(json.dumps({**report, "artifacts": str(run_root)}, ensure_ascii=False)) if __name__ == "__main__": try: main() except psycopg.Error: raise SystemExit("Isolated migration PostgreSQL checks failed; driver details withheld.") from None