From dd991830ac4399c18476c3e5fcf468619776bbb7 Mon Sep 17 00:00:00 2001 From: fy59 Date: Thu, 17 Sep 2026 13:50:08 +0200 Subject: [PATCH] build(sync): add candidate and recovery readiness tools --- CHANGELOG.md | 5 ++ docs/architecture.md | 7 ++ docs/current_state.md | 5 ++ .../sync_deployment_readiness_v1_evidence.md | 33 +++++++++ docs/roadmap.md | 5 ++ docs/sync_rollout_readiness.md | 41 +++++++++++ docs/tests.md | 7 ++ tests/test_sync_deployment_tools.py | 29 ++++++++ tools/backup_trainlog_sqlite.py | 36 ++++++++++ tools/package_sync_candidate.py | 72 +++++++++++++++++++ tui/meson.build | 5 ++ 11 files changed, 245 insertions(+) create mode 100644 docs/design/sync_deployment_readiness_v1_evidence.md create mode 100644 tests/test_sync_deployment_tools.py create mode 100644 tools/backup_trainlog_sqlite.py create mode 100644 tools/package_sync_candidate.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 524f110..028cc7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased — 0.1.2 +- Added opt-in foreground generation coordination, peer-evidenced capabilities, + truthful bounded Web progress, descendant cleanup, an integrated Android ↔ + desktop Firefox proof, and synthetic candidate/SQLite recovery tooling on the + development branch. Physical MTP and deployment remain pending. + - Completed the grouped `TRAINLOG_SYNC_WEB_END_TO_END_V1` mission: the opt-in full-generation orchestrator, protected asynchronous Web API, global sync control, durable reconnect reporting, Dashboard refresh and read-only draft diff --git a/docs/architecture.md b/docs/architecture.md index 0daf34f..b6bc8b8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,5 +1,12 @@ # Architecture +Full-generation synchronization uses the common admission lock and a fixed +coordinator worker. Browser input cannot select commands, paths, timeouts or +capabilities. Peer identity/capabilities come from Android advertisement; +Android participates only during a foreground user action. Typed progress is +bounded and durable, and completion is corroborated by SQLite generation/ACK +records. + ## 1. System boundary Trainlog Core is the product authority and separates capture, durable history, diff --git a/docs/current_state.md b/docs/current_state.md index 8a984c7..1f52b49 100644 --- a/docs/current_state.md +++ b/docs/current_state.md @@ -1,5 +1,10 @@ # Current implementation state +The development branch has opt-in full-generation coordination with durable Web +progress and a foreground Android conversation. The cross-implementation +directory-I/O-double proof is integrated. Physical MTP, release signing, user +backups, main integration and deployment are not complete. + Snapshot date: **2026-09-17**. This document owns the current implemented state. Historical checkpoints and diff --git a/docs/design/sync_deployment_readiness_v1_evidence.md b/docs/design/sync_deployment_readiness_v1_evidence.md new file mode 100644 index 0000000..bf01e5f --- /dev/null +++ b/docs/design/sync_deployment_readiness_v1_evidence.md @@ -0,0 +1,33 @@ +# Sync deployment readiness V1 evidence + +Status: **SOFTWARE BLOCKED — physical production MTP adapter not completed**. + +This corrective tranche replaces the earlier configurable peer fixture with a +fixed desktop worker and a production Android foreground coordinator. The real +Firefox scenario executes the C HTTP adapter, durable run state, both generation +services, transactional imports, real ACK creation/acceptance and store reopen. +It uses a directory object-I/O double and therefore does not claim USB/MTP. + +Decisive targeted evidence: + +- `SyncDeploymentConversationTest`: Android generation capture/publication, + desktop consume/ACK, desktop return generation, Android consume/ACK and draft + survival after reopen. +- `test_web_sync_browser.py`: Firefox click, observable desktop commit before the + Android barrier is released, completion, active draft display and reload. +- `test_sync_orchestrator.py`: untrusted-command rejection, pre-accumulation + 64-KiB stdout enforcement, and SIGKILL escalation for an owned helper and + descendant that ignore SIGTERM. +- `test_sync_deployment_tools.py`: SQLite backup captures committed WAL state and + passes integrity/foreign-key checks; candidate inventory contains the fixed + helper and hashes. + +The tests also exposed and corrected two integration defects: execution-draft +`active`/`pending` state was incorrectly treated as immutable revision content, +and coordination files could become visible before their bytes were complete. + +Not performed: physical MTP, real phone access, release signature verification, +real Drive, user backups, main merge, service switch or deployment. The Selenium +`ResourceWarning` remains visible and is associated with an HTTP 405 during the +external Firefox/geckodriver teardown path; no blanket warning suppression is +used. It remains a toolchain limitation until independently fixed or upgraded. diff --git a/docs/roadmap.md b/docs/roadmap.md index 77cbd92..93a2175 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,5 +1,10 @@ # Roadmap +Before controlled rollout, complete and validate the physical libmtp generation +adapter, establish a legitimate installed-Android backup path, verify release +signing continuity and obtain explicit rollout approval. Do not enable generation +mode or advance `main` before those gates. + This document owns future work. The implemented baseline is summarized in [current_state.md](current_state.md); completed narratives and evidence belong in [reviews](reviews/) and [CHANGELOG.md](../CHANGELOG.md). diff --git a/docs/sync_rollout_readiness.md b/docs/sync_rollout_readiness.md index 91e7b57..8cfd4ec 100644 --- a/docs/sync_rollout_readiness.md +++ b/docs/sync_rollout_readiness.md @@ -1,5 +1,46 @@ # Full-generation sync rollout readiness +## Deployment-readiness candidate + +The generation path remains disabled unless desktop configuration authorizes a +specific persisted Android peer and Android contains the strict opt-in document +`Documents/Trainlog/trainlog-sync-generation-opt-in-v1.json`. Absence preserves +V3; malformed opt-in fails explicitly. The Web coordinator owns a fixed worker, +bounded typed progress and its descendant process group. The foreground Android +coordinator advertises its real peer identity, publishes manifest-last bytes, +consumes the return generation and emits a durable correlated ACK. + +The integrated proof uses a directory object-I/O double. It is not evidence for +physical libmtp behavior. The shipped physical generation adapter is not yet +complete, so generation mode must not be enabled on services. + +### Candidate, backup and recovery + +1. Build and validate one revision, then run `python3 + tools/package_sync_candidate.py --output `. Its + inventory records hashes, schemas, protocols, entry points and dependencies. +2. `assembleDebug` is development evidence only. Rollout requires the existing + release signing identity, verification of its public certificate and a valid + increasing `versionCode`; this mission reads no signing secret. +3. Quiesce old writers only during an authorized rollout. Desktop backup uses + `backup_trainlog_sqlite.py` and SQLite's backup API, then integrity and + foreign-key checks. +4. V3 is not an Android backup and omits active-draft and causal state. No safe + backup path for the installed release was demonstrated without device access. + This is a hard gate: an authorized procedure must capture the app database, + WAL-consistent state and active draft before update. +5. Replacing a binary does not roll back a schema. Before any exchange, a + mutually compatible captured pair may be restored. After causal writes or + ACKs, never restore only one peer: stop synchronization, preserve both stores + and all artifacts, and use an explicit paired recovery plan. +6. The eight-generation-per-peer ceiling is an admission gate. Exhaustion must + preserve pending generations, tombstones, finalizations and ACK ledgers; it + must never trigger hidden eviction. + +Actual rollout needs one separate approval for the signed APK, desktop bundle, +verified user backups, coordinated service quiescence, pairing, synthetic smoke +and non-destructive hardware smoke. Drive remains separate and untested here. + This is a non-executed rollout checklist. The development branch implements the path; the daily installation still uses V3. diff --git a/docs/tests.md b/docs/tests.md index 512d48c..3610bff 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -1,5 +1,12 @@ # Tests and validation +`SyncDeploymentConversationTest` executes both production generation services +and reopens Android storage. `test_web_sync_browser.py` adds a real Firefox click +through the C adapter, observes the desktop commit at an Android test barrier, +then releases the real ACK. This proves the directory transport double, not +physical MTP. Orchestrator tests cover bounded streams and descendant cleanup; +deployment-tool tests cover consistent SQLite backup and candidate inventory. + The `ai_history_export` test covers an empty database, a simple session, distinct multi-occurrences of one exercise, ordered sets, exact occurrence feedback, session-owned follow-up feedback, JSON escaping/UTF-8, MAX history, diff --git a/tests/test_sync_deployment_tools.py b/tests/test_sync_deployment_tools.py new file mode 100644 index 0000000..35532fd --- /dev/null +++ b/tests/test_sync_deployment_tools.py @@ -0,0 +1,29 @@ +import json, sqlite3, subprocess, tempfile, unittest +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[1] + +class DeploymentToolsTest(unittest.TestCase): + def test_sqlite_backup_captures_committed_wal_and_restores_invariants(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);source=root/"source.db";backup=root/"backup.db" + db=sqlite3.connect(source);db.execute("PRAGMA journal_mode=WAL");db.execute("PRAGMA foreign_keys=ON") + db.executescript("CREATE TABLE parent(id INTEGER PRIMARY KEY);CREATE TABLE child(parent_id INTEGER REFERENCES parent);INSERT INTO parent VALUES(7);INSERT INTO child VALUES(7);");db.commit() + result=subprocess.run(["python3",str(ROOT/"tools/backup_trainlog_sqlite.py"),str(source),str(backup)],capture_output=True,text=True) + self.assertEqual(result.returncode,0,result.stderr) + with sqlite3.connect(backup) as restored: + self.assertEqual(restored.execute("SELECT * FROM child").fetchall(),[(7,)]) + self.assertEqual(restored.execute("PRAGMA integrity_check").fetchone(),("ok",)) + db.close() + + def test_candidate_is_self_contained_and_hashed(self): + with tempfile.TemporaryDirectory() as directory: + output=Path(directory)/"candidate" + result=subprocess.run(["python3",str(ROOT/"tools/package_sync_candidate.py"),"--output",str(output)],capture_output=True,text=True) + self.assertEqual(result.returncode,0,result.stderr) + inventory=json.loads((output/"candidate-inventory.json").read_text()) + self.assertEqual(inventory["product_version"],"0.1.2") + self.assertTrue((output/"bin/trainlog").is_file()) + self.assertTrue(any(row["path"]=="tools/sync_peer_worker.py" for row in inventory["files"])) + +if __name__=="__main__": unittest.main() diff --git a/tools/backup_trainlog_sqlite.py b/tools/backup_trainlog_sqlite.py new file mode 100644 index 0000000..6dcf1bf --- /dev/null +++ b/tools/backup_trainlog_sqlite.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Create or restore a consistent Trainlog SQLite backup. + +This tool never discovers a user database. Both source and destination must be +explicit, and restore refuses to overwrite an existing file. SQLite's backup +API includes committed WAL content without relying on a racy file copy. +""" +import argparse +import sqlite3 +from pathlib import Path + + +def backup(source: Path, destination: Path) -> None: + if not source.is_file() or destination.exists(): + raise RuntimeError("source must exist and destination must not exist") + destination.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(f"file:{source}?mode=ro", uri=True) as src, sqlite3.connect(destination) as dst: + src.backup(dst) + if dst.execute("PRAGMA integrity_check").fetchone() != ("ok",): + raise RuntimeError("backup integrity check failed") + violations = dst.execute("PRAGMA foreign_key_check").fetchall() + if violations: + raise RuntimeError(f"backup foreign-key violations: {len(violations)}") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source", type=Path) + parser.add_argument("destination", type=Path) + args = parser.parse_args() + backup(args.source, args.destination) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/package_sync_candidate.py b/tools/package_sync_candidate.py new file mode 100644 index 0000000..44c0df5 --- /dev/null +++ b/tools/package_sync_candidate.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Stage a reproducible, non-secret Trainlog desktop synchronization candidate.""" +import argparse +import hashlib +import json +import os +import shutil +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUNTIME_TOOLS = [ + "sync_orchestrator.py", "sync_peer_worker.py", "sync_generation_exchange.py", + "trainlog_sqlite.py", "validate_json.py", "import_mobile_export.py", + "import_exercise_aliases.py", "import_equipment_definitions.py", + "import_equipment_associations.py", "import_exercise_body_zones.py", + "import_training_feedback.py", "execution_draft_exchange.py", + "causal_delete_exchange.py", "import_exercise_profile_state.py", + "export_exercise_profile_state.py", + "export_pc_mobile.py", "export_equipment_definitions.py", + "export_exercise_body_zones.py", "export_training_feedback.py", +] + + +def digest(path: Path) -> str: + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--build", default=ROOT / "build", type=Path) + args = parser.parse_args() + if args.output.exists(): + raise RuntimeError("candidate output already exists") + binary = args.build / "tui/trainlog" + if not binary.is_file(): + raise RuntimeError("compiled trainlog executable is missing") + (args.output / "bin").mkdir(parents=True) + (args.output / "libexec").mkdir(parents=True) + (args.output / "tools").mkdir(parents=True) + (args.output / "catalog").mkdir(parents=True) + shutil.copy2(binary, args.output / "libexec/trainlog") + for name in RUNTIME_TOOLS: + shutil.copy2(ROOT / "tools" / name, args.output / "tools" / name) + for source in sorted((ROOT / "catalog").glob("*.json")): + shutil.copy2(source, args.output / "catalog" / source.name) + launcher = args.output / "bin/trainlog" + launcher.write_text("#!/bin/sh\nset -eu\nroot=$(CDPATH= cd -- \"$(dirname -- \"$0\")/..\" && pwd)\nexport TRAINLOG_SYNC_TOOLS_DIR=\"$root/tools\"\nexec \"$root/libexec/trainlog\" \"$@\"\n") + files = sorted(path for path in args.output.rglob("*") if path.is_file()) + inventory = { + "format": "trainlog-sync-candidate-inventory", "version": 1, + "source_commit": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip(), + "product_version": "0.1.2", "desktop_schema": 21, "android_schema": 20, + "protocols": ["mobile-export-v3", "mobile-history-v4", "causal-delete-v1", + "execution-draft-v1", "generation-manifest-v1", "generation-ack-v1"], + "entry_points": ["bin/trainlog", "tools/sync_orchestrator.py"], + "runtime_dependencies": ["python>=3.11", "sqlite3", "utf8proc", "libuuid", "libudev", "libmtp", "notcurses"], + "files": [{"path": str(path.relative_to(args.output)), "sha256": digest(path), "size": path.stat().st_size} for path in files], + } + (args.output / "candidate-inventory.json").write_text(json.dumps(inventory, indent=2, sort_keys=True) + "\n") + os.chmod(args.output / "bin/trainlog", 0o755) + os.chmod(args.output / "libexec/trainlog", 0o755) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tui/meson.build b/tui/meson.build index 442a1b6..53698fd 100644 --- a/tui/meson.build +++ b/tui/meson.build @@ -739,6 +739,11 @@ test( find_program('python3'), args: [meson.project_source_root() / 'tests/test_sync_orchestrator.py'], ) +test( + 'sync_deployment_tools', + find_program('python3'), + args: [meson.project_source_root() / 'tests/test_sync_deployment_tools.py'], +) test( 'schema_v21_migration', executable('test_schema_v21_migration', 'tests/test_schema_v21_migration.c', dependencies: trainlog_core_dep),